Add gmv aggregate for ctx2

Signed-off-by: Andrew Stein <steinlink@gmail.com>

Author: texodus

.github/workflows/build.yaml

@@ -28,7 +28,7 @@ on:
             - docs/
             - examples/
             - rust/perspective-python/README.md
-    pull_request_target:
+    pull_request:
         branches:
             - master
     workflow_dispatch:

rust/perspective-python/perspective/tests/table/test_view.py

@@ -536,6 +536,39 @@ def test_view_aggregate_gmv(self):
             "MV": [10000, 5900, -2000, -3200, 700, 7100, 800, -2400, -3900],
         }
 
+    def test_view_aggregate_gmv_split_by(self):
+        data = {
+            "division": [
+                "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2",
+                "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2", "D1", "D2",
+            ],
+            "symbol": [
+                "AAPL", "GOOG", "MSFT", "AAPL", "GOOG", "MSFT", "AAPL",
+                "GOOG", "MSFT", "AAPL", "GOOG", "MSFT", "AAPL", "GOOG",
+                "MSFT", "AAPL", "GOOG", "MSFT", "AAPL", "GOOG",
+            ],
+            "MV": [
+                1500, 1200, 1300, 1400, 1600, 1100, 1700, 1800, 1900, 2000,
+                -2100, -2200, -2300, -2400, -2500, -2600, -2700, -2800,
+                -2900, -3000,
+            ],
+        }
+
+        tbl = Table(data)
+        view = tbl.view(
+            aggregates={"MV": "gmv"},
+            group_by=["division"],
+            split_by=["symbol"],
+            columns=["MV"],
+        )
+
+        assert view.to_columns() == {
+            "__ROW_PATH__": [[], ["D1"], ["D2"]],
+            "AAPL|MV": [2800, -2000, 800],
+            "GOOG|MV": [5600, -3200, -2400],
+            "MSFT|MV": [4600, 700, -3900],
+        }
+
     def test_view_aggregate_mean_from_schema(self):
         data = [
             {"a": "a", "x": 1, "y": 200},

rust/perspective-server/cpp/perspective/src/cpp/context_two.cpp

@@ -77,6 +77,19 @@ t_ctx2::init() {
         );
 
         m_trees[treeidx]->init();
+
+        // Tell the tree how many of its `m_pivots` are row pivots and
+        // what the next missing row pivot is (if any). AGGTYPE_GMV uses
+        // this to roll up across row dimensions that aren't embedded in
+        // this particular tree — e.g. m_trees[0] has no row pivots, so
+        // a tree-leaf at e.g. [AAPL] needs to be partitioned by
+        // `row_pivots[0]` from the gstate to compute the gmv value.
+        const auto& row_pivots = m_config.get_row_pivots();
+        std::string next_row_pivot;
+        if (treeidx < row_pivots.size()) {
+            next_row_pivot = row_pivots[treeidx].colname();
+        }
+        m_trees[treeidx]->set_gmv_row_pivot_meta(treeidx, next_row_pivot);
     }
 
     m_rtraversal = std::make_shared<t_traversal>(rtree());
@@ -1171,6 +1184,15 @@ t_ctx2::reset(bool reset_expressions) {
         );
         m_trees[treeidx]->init();
         m_trees[treeidx]->set_deltas_enabled(get_feature_state(CTX_FEAT_DELTA));
+
+        // See [t_ctx2::init] — gmv needs to know how many row pivots
+        // this particular tree carries vs. what's still in the gstate.
+        const auto& row_pivots = m_config.get_row_pivots();
+        std::string next_row_pivot;
+        if (treeidx < row_pivots.size()) {
+            next_row_pivot = row_pivots[treeidx].colname();
+        }
+        m_trees[treeidx]->set_gmv_row_pivot_meta(treeidx, next_row_pivot);
     }
 
     m_rtraversal = std::make_shared<t_traversal>(rtree());

rust/perspective-server/cpp/perspective/src/cpp/sparse_tree.cpp

@@ -14,6 +14,7 @@
 #include <algorithm>
 #include <cmath>
 #include <fstream>
+#include <unordered_map>
 #include <perspective/base.h>
 #include <perspective/compat.h>
 #include <perspective/extract_aggregate.h>
@@ -86,11 +87,31 @@ t_stree::t_stree(
     m_aggspecs(aggspecs),
     m_schema(std::move(schema)),
     m_cur_aggidx(1),
-    m_has_delta(false) {
+    m_has_delta(false),
+    m_num_row_pivots_in_tree(pivots.size()) {
     const auto& g_agg_str = cfg.get_grand_agg_str();
     m_grand_agg_str = g_agg_str.empty() ? "Grand Aggregate" : g_agg_str;
 }
 
+void
+t_stree::set_gmv_row_pivot_meta(
+    t_uindex num_row_pivots_in_tree,
+    const std::string& next_row_pivot_name
+) {
+    m_num_row_pivots_in_tree = num_row_pivots_in_tree;
+    m_next_row_pivot_name = next_row_pivot_name;
+}
+
+t_uindex
+t_stree::get_num_row_pivots_in_tree() const {
+    return m_num_row_pivots_in_tree;
+}
+
+const std::string&
+t_stree::get_next_row_pivot_name() const {
+    return m_next_row_pivot_name;
+}
+
 t_stree::~t_stree() {
     for (auto& iter : m_smap) {
         free(const_cast<char*>(iter.first));
@@ -1816,17 +1837,24 @@ t_stree::update_agg_table(
                 dst->set_scalar(dst_ridx, new_value);
             } break;
             case AGGTYPE_GMV: {
-                // Leaf nodes: plain `sum` of the underlying rows.
-                // Non-leaf nodes: sum over immediate children of
-                // `abs(sum(child_subtree))`. The child sums are recomputed
-                // from the gstate rather than read from already-populated
-                // child rows, so the parent does not see the abs-rolled-up
-                // child value — it sees the child's raw signed sum.
+                // The user-facing rule is "leaf = sum, parent = sum over
+                // immediate row children of |raw sum of child subtree|".
+                // Under split_by this is complicated by t_ctx2 sharding
+                // the data across several t_stree instances — see
+                // [t_stree::set_gmv_row_pivot_meta] for the metadata that
+                // distinguishes "the next pivot in this tree is a row
+                // pivot" from "the next missing row pivot lives in the
+                // gstate".
                 old_value.set(dst->get_scalar(dst_ridx));
                 const auto& col_name = spec.get_dependencies()[0].name();
-                auto dst_dtype = dst->get_dtype();
-                auto sum_reducer = [dst_dtype](std::vector<t_tscalar>& values
-                                   ) -> t_tscalar {
+                const auto dst_dtype = dst->get_dtype();
+                const auto k_tree = m_num_row_pivots_in_tree;
+                const auto& next_row_pivot = m_next_row_pivot_name;
+                const bool has_missing_row_pivot = !next_row_pivot.empty();
+                const auto depth = get_depth(nidx);
+
+                auto sum_reducer =
+                    [dst_dtype](std::vector<t_tscalar>& values) -> t_tscalar {
                     if (values.empty()) {
                         return mknone();
                     }
@@ -1842,7 +1870,14 @@ t_stree::update_agg_table(
                     return v;
                 };
 
-                if (is_leaf(nidx)) {
+                // row_path_len = min(depth, k_tree). A node is a "row-leaf"
+                // iff there are no row pivots missing from this tree AND
+                // all row pivots are consumed at this node's depth — i.e.
+                // depth >= k_tree.
+                const bool is_row_leaf =
+                    !has_missing_row_pivot && depth >= k_tree;
+
+                if (is_row_leaf) {
                     auto pkeys = get_pkeys(nidx);
                     new_value.set(
                         reduce_from_gstate<std::function<
@@ -1854,7 +1889,10 @@ t_stree::update_agg_table(
                             sum_reducer
                         )
                     );
-                } else {
+                } else if (depth < k_tree) {
+                    // Row-parent and this tree's next pivot is the next
+                    // row pivot. Tree children at depth+1 are the
+                    // immediate row children — use them directly.
                     t_tscalar rval;
                     rval.set(std::uint64_t(0));
                     rval.m_type = dst_dtype;
@@ -1873,6 +1911,53 @@ t_stree::update_agg_table(
                         }
                     }
                     new_value.set(rval);
+                } else {
+                    // Row-parent but this tree's pivots beyond `depth`
+                    // (if any) are column pivots, so the immediate row
+                    // children aren't tree children. Read the missing
+                    // row pivot column for this node's pkeys, partition
+                    // by it, sum each partition, sum |partition_sum|.
+                    auto pkeys = get_pkeys(nidx);
+                    std::vector<t_tscalar> pivot_vals;
+                    std::vector<t_tscalar> data_vals;
+                    read_column_from_gstate(
+                        gstate,
+                        expression_master_table,
+                        next_row_pivot,
+                        pkeys,
+                        pivot_vals
+                    );
+                    read_column_from_gstate(
+                        gstate,
+                        expression_master_table,
+                        col_name,
+                        pkeys,
+                        data_vals
+                    );
+
+                    std::unordered_map<t_tscalar, t_tscalar> partials;
+                    for (std::size_t i = 0;
+                         i < pivot_vals.size() && i < data_vals.size();
+                         ++i) {
+                        const auto& x = data_vals[i];
+                        if (!x.is_valid() || x.is_nan()) {
+                            continue;
+                        }
+                        auto& acc = partials[pivot_vals[i]];
+                        if (!acc.is_valid()) {
+                            acc.set(std::uint64_t(0));
+                            acc.m_type = dst_dtype;
+                        }
+                        acc = acc.add(x.coerce_numeric_dtype(dst_dtype));
+                    }
+
+                    t_tscalar rval;
+                    rval.set(std::uint64_t(0));
+                    rval.m_type = dst_dtype;
+                    for (const auto& kv : partials) {
+                        rval = rval.add(kv.second.abs());
+                    }
+                    new_value.set(rval);
                 }
                 dst->set_scalar(dst_ridx, new_value);
             } break;

rust/perspective-server/cpp/perspective/src/include/perspective/sparse_tree.h

@@ -468,6 +468,25 @@ class PERSPECTIVE_EXPORT t_stree {
     t_symtable m_symtable;
     bool m_has_delta;
     std::string m_grand_agg_str;
+
+    // Used by AGGTYPE_GMV under split_by. For t_ctx1 (group_by only) the
+    // tree's pivots are exactly the row pivots, so the default
+    // `m_num_row_pivots_in_tree == m_pivots.size()` makes every tree-leaf
+    // a "real" leaf and the next-row-pivot lookup unused. For t_ctx2 each
+    // m_trees[k] is initialized with the row+col split via
+    // `set_gmv_row_pivot_meta(k, next_row_pivot_name)`, so the gmv kernel
+    // can detect a "tree-leaf but row-parent" node and roll up across the
+    // missing row pivot dimension via the gstate.
+    t_uindex m_num_row_pivots_in_tree;
+    std::string m_next_row_pivot_name;
+
+public:
+    void set_gmv_row_pivot_meta(
+        t_uindex num_row_pivots_in_tree,
+        const std::string& next_row_pivot_name
+    );
+    t_uindex get_num_row_pivots_in_tree() const;
+    const std::string& get_next_row_pivot_name() const;
 };
 
 } // end namespace perspective