feat: DH-18317: Add hierarchical path (#1164)

Fixes https://deephaven.atlassian.net/browse/DH-18317
Examples are in docs

---------

Co-authored-by: Mike Bender <mikebender@deephaven.io>

Author: jnumainville

plugins/plotly-express/docs/icicle.md

@@ -31,6 +31,17 @@ gapminder_recent = (
 
 icicle_plot = dx.icicle(gapminder_recent, names="Continent", values="Pop", parents="World")
 ```
+### An icicle plot with `path`
+
+Instead of manually aggregating and passing in `names` and `parents`, use the `path` argument to specify the hierarchy of the data. The first column is the root category, and the last column is the leaf category. The values are automatically summed up.
+
+```python order=treemap_path_plot,gapminder
+import deephaven.plot.express as dx
+
+gapminder = dx.data.gapminder().update_view("World = `World`")
+
+icicle_path_plot = dx.icicle(gapminder, path=["World", "Continent", "Country"], values="Pop")
+```
 
 ![Icicle Plot Basic Example](./_assets/icicle_plot.png)
 

plugins/plotly-express/docs/sunburst.md

@@ -34,6 +34,18 @@ sunburst_plot = dx.sunburst(gapminder_recent, names="Continent", values="Pop", p
 
 ![Sunburst Plot Basic Example](./_assets/sunburst_plot.png)
 
+### A sunburst plot with `path`
+
+Instead of manually aggregating and passing in `names` and `parents`, use the `path` argument to specify the hierarchy of the data. The first column is the root category, and the last column is the leaf category. The values are automatically summed up.
+
+```python order=treemap_path_plot,gapminder
+import deephaven.plot.express as dx
+
+gapminder = dx.data.gapminder().update_view("World = `World`")
+
+sunburst_path_plot = dx.sunburst(gapminder, path=["World", "Continent", "Country"], values="Pop")
+```
+
 # A nested sunburst plot with branch values
 
 By default, the `branchvalues` argument is set to `"remainder"`.

plugins/plotly-express/docs/treemap.md

@@ -34,6 +34,18 @@ treemap_plot = dx.treemap(gapminder_recent, names="Continent", values="Pop", par
 
 ![Treemap Plot Basic Example](./_assets/treemap_plot.png)
 
+### A treemap plot with `path`
+
+Instead of manually aggregating and passing in `names` and `parents`, use the `path` argument to specify the hierarchy of the data. The first column is the root category, and the last column is the leaf category. The values are automatically summed up.
+
+```python order=treemap_path_plot,gapminder
+import deephaven.plot.express as dx
+
+gapminder = dx.data.gapminder().update_view("World = `World`")
+
+treemap_path_plot = dx.treemap(gapminder, path=["World", "Continent", "Country"], values="Pop")
+```
+
 # A nested treemap plot with branch values
 
 By default, the `branchvalues` argument is set to `"remainder"`.

plugins/plotly-express/src/deephaven/plot/express/plots/PartitionManager.py

@@ -15,6 +15,7 @@
 from .. import DeephavenFigure
 from ..preprocess.Preprocessor import Preprocessor
 from ..shared import get_unique_names
+from ..types import AttachedTransforms, HierarchicalTransforms
 from .subplots import atomic_make_grid
 
 PARTITION_ARGS = {
@@ -191,11 +192,9 @@ class PartitionManager:
         has_color: bool: True if this figure has user set color, False otherwise
         facet_row: str: The facet row
         facet_col: str: The facet col
-        (arg, col), (map, ls, new_col)
-        always_attached: dict[tuple[str, str],
-          tuple[dict[str, str], list[str], str]: The dict mapping the arg and column
-          to the style map, dictionary, and new column name, to be used for
-          AttachedProcessor when dealing with an "always_attached" plot
+        attached_transforms: to be used for AttachedProcessor when dealing with an "always_attached" plot
+        hierarchical_transforms: HierarchicalTransforms: to be used for HierarchicalProcessor when dealing with
+            a hierarchical plot with a path
         marginal_x: Type of marginal on the x-axis, if applicable
         marginal_y: Type of marginal on the y-axis, if applicable
         marg_args: dict[str, Any]: The dictionary of args to pass to marginals
@@ -226,7 +225,8 @@ def __init__(
         self.has_color = None
         self.facet_row = None
         self.facet_col = None
-        self.always_attached = {}
+        self.attached_transforms = AttachedTransforms()
+        self.hierarchical_transforms = HierarchicalTransforms()
 
         self.marginal_x = args.pop("marginal_x", None)
         self.marginal_y = args.pop("marginal_y", None)
@@ -310,7 +310,7 @@ def convert_table_to_long_mode(
 
         args["table"] = self.to_long_mode(table, self.cols)
 
-    def is_by(self, arg: str, map_val: str | list[str] | None = None) -> None:
+    def is_by(self, arg: str, map_val: str | list[str] | dict | None = None) -> None:
         """
         Given that the specific arg is a by arg, prepare the arg depending on
         if it is attached or not
@@ -325,10 +325,16 @@ def is_by(self, arg: str, map_val: str | list[str] | None = None) -> None:
 
         if "always_attached" in self.groups:
             new_col = get_unique_names(self.args["table"], [arg])[arg]
-            self.always_attached[(arg, self.args[arg])] = (
-                map_val,
-                self.args[seq_arg],
-                new_col,
+            if not isinstance(map_val, dict) and map_val is not None:
+                raise TypeError(
+                    f"Expected a dictionary for {arg} map, got {type(map_val)}"
+                )
+            self.attached_transforms.add(
+                by_col=self.args[arg],
+                new_col=new_col,
+                style_map=map_val,
+                style_list=self.args[seq_arg],
+                style=arg,
             )
             # a new column will be constructed so this color is always updated
             self.args[f"attached_{arg}"] = new_col
@@ -395,9 +401,13 @@ def handle_plot_by_arg(
             ):
                 if "always_attached" in self.groups:
                     args["colors"] = args.pop("color")
-                # just keep the argument in place so it can be passed to plotly
-                # express directly
-                pass
+                    if self.args.get("path"):
+                        # is_single_numeric_col must be true so it is safe to pull the first element
+                        if not isinstance(val, str):
+                            val = val[0]
+                        # numeric column that is the source of color need to be aggregated if path is passed
+                        self.hierarchical_transforms.add(avg_col=val)
+                    # otherwise the colors are attached directly
             elif val:
                 self.is_by(arg, args[map_name])
             elif plot_by_cols and (
@@ -511,7 +521,8 @@ def process_partitions(self) -> Table | PartitionedTable:
         self.preprocessor = Preprocessor(
             args,
             self.groups,
-            self.always_attached,
+            self.attached_transforms,
+            self.hierarchical_transforms,
             self.stacked_column_names,
             self.list_param,
         )
@@ -645,15 +656,16 @@ def table_partition_generator(
         column = (
             self.stacked_column_names["value"] if self.stacked_column_names else None
         )
-        if self.preprocessor:
-            tables = self.preprocessor.preprocess_partitioned_tables(
-                self.constituents, column
-            )
-            for table, current_partition in zip(
-                tables, self.current_partition_generator()
-            ):
-                # since this is preprocessed it will always be a tuple
-                yield cast(Tuple[Table, Dict[str, str]], (table, current_partition))
+
+        if self.preprocessor is None:
+            return
+
+        tables = self.preprocessor.preprocess_partitioned_tables(
+            self.constituents, column
+        )
+        for table, current_partition in zip(tables, self.current_partition_generator()):
+            # since this is preprocessed it will always be a tuple
+            yield cast(Tuple[Table, Dict[str, str]], (table, current_partition))
 
     def partition_generator(self) -> Generator[dict[str, Any], None, None]:
         """
@@ -681,13 +693,8 @@ def partition_generator(self) -> Generator[dict[str, Any], None, None]:
                 args["current_partition"] = current_partition
                 args["table"] = table
                 yield args
-        elif (
-            "preprocess_hist" in self.groups
-            or "preprocess_freq" in self.groups
-            or "preprocess_time" in self.groups
-            or "preprocess_heatmap" in self.groups
-        ) and self.preprocessor:
-            # still need to preprocess the base table
+        elif self.preprocessor:
+            # still need to preprocess the base table if preprocessors were created
             table, arg_update = cast(
                 Tuple,
                 [*self.preprocessor.preprocess_partitioned_tables([args["table"]])][0],

plugins/plotly-express/src/deephaven/plot/express/plots/__init__.py

@@ -4,7 +4,7 @@
 from .bar import bar, frequency_bar, timeline
 from .distribution import histogram, violin, strip, box
 from .financial import candlestick, ohlc
-from .hierarchial import treemap, icicle, sunburst, funnel, funnel_area
+from .hierarchical import treemap, icicle, sunburst, funnel, funnel_area
 from .pie import pie
 from ._layer import layer
 from .subplots import make_subplots

…phaven/plot/express/plots/hierarchial.py …haven/plot/express/plots/hierarchical.pyplugins/plotly-express/src/deephaven/plot/express/plots/hierarchial.py renamed to plugins/plotly-express/src/deephaven/plot/express/plots/hierarchical.py plugins/plotly-express/src/deephaven/plot/express/plots/hierarchial.py renamed to plugins/plotly-express/src/deephaven/plot/express/plots/hierarchical.py

@@ -16,6 +16,7 @@ def treemap(
     values: str | None = None,
     parents: str | None = None,
     ids: str | None = None,
+    path: str | list[str] | None = None,
     color: str | list[str] | None = None,
     hover_name: str | None = None,
     color_discrete_sequence: list[str] | None = None,
@@ -39,10 +40,14 @@ def treemap(
       parents: The column containing parents of the sections
       ids: The column containing ids of the sections. Unlike values, these
         must be unique. Values are used for ids if ids are not specified.
+      path: A column or list of columns that describe the hierarchy.
+        The first column is the root, the second column contains the children
+        of the root, and so on. The last column is the leaf.
       color: A column or list of columns that contain color values.
         If only one column is passed, and it contains numeric values, the value
         is used as a value on a continuous color scale. Otherwise, the value is
         used for a plot by on color.
+        If path is provided, only a single color column is allowed.
         See color_discrete_map for additional behaviors.
       hover_name: A column that contains names to bold in the hover tooltip.
       labels: A dictionary of labels mapping columns to new labels.
@@ -84,6 +89,7 @@ def sunburst(
     values: str | None = None,
     parents: str | None = None,
     ids: str | None = None,
+    path: str | list[str] | None = None,
     color: str | list[str] | None = None,
     hover_name: str | None = None,
     color_discrete_sequence: list[str] | None = None,
@@ -107,10 +113,14 @@ def sunburst(
       parents: The column containing parents of the sections
       ids: The column containing ids of the sections. Unlike values, these
         must be unique. Values are used for ids if ids are not specified.
+      path: A column or list of columns that describe the hierarchy.
+        The first column is the root, the second column contains the children
+        of the root, and so on. The last column is the leaf.
       color: A column or list of columns that contain color values.
         If only one column is passed, and it contains numeric values, the value
         is used as a value on a continuous color scale. Otherwise, the value is
         used for a plot by on color.
+        If path is provided, only a single color column is allowed.
         See color_discrete_map for additional behaviors.
       hover_name: A column that contains names to bold in the hover tooltip.
       color_discrete_sequence: A list of colors to sequentially apply to
@@ -152,6 +162,7 @@ def icicle(
     values: str | None = None,
     parents: str | None = None,
     ids: str | None = None,
+    path: str | list[str] | None = None,
     color: str | list[str] | None = None,
     hover_name: str | None = None,
     color_discrete_sequence: list[str] | None = None,
@@ -173,13 +184,17 @@ def icicle(
       names: The column containing names of the sections
       values: The column containing values of the sections
       parents: The column containing parents of the sections
+      ids: The column containing ids of the sections. Unlike values, these
+        must be unique. Values are used for ids if ids are not specified.
+      path: A column or list of columns that describe the hierarchy.
+        The first column is the root, the second column contains the children
+        of the root, and so on. The last column is the leaf.
       color: A column or list of columns that contain color values.
         If only one column is passed, and it contains numeric values, the value
         is used as a value on a continuous color scale. Otherwise, the value is
         used for a plot by on color.
+        If path is provided, only a single color column is allowed.
         See color_discrete_map for additional behaviors.
-      ids: The column containing ids of the sections. Unlike values, these
-        must be unique. Values are used for ids if ids are not specified.
       hover_name: A column that contains names to bold in the hover tooltip.
       color_discrete_sequence: A list of colors to sequentially apply to
         the series. The colors loop, so if there are more series than colors,

plugins/plotly-express/src/deephaven/plot/express/preprocess/AttachedPreprocessor.py

@@ -2,8 +2,11 @@
 
 from typing import Any
 
+from deephaven.table import Table
+
 from .StyleManager import StyleManager
 from ..shared import get_unique_names
+from ..types import AttachedTransforms
 
 
 class AttachedPreprocessor:
@@ -12,37 +15,57 @@ class AttachedPreprocessor:
 
     Attributes:
         args: Args used to create the plot
-        always_attached: The dict mapping the arg and column
+        attached_transforms: The dict mapping the arg and column
           to the style map, dictionary, and new column name, to be used for
           AttachedProcessor when dealing with an "always_attached" plot
+        color_mask: The column to use for the color mask, which controls if a categorical
+          color is valid at this level. Can also be a boolean, which applies to all values.
     """
 
     def __init__(
         self,
         args: dict[str, Any],
-        always_attached: dict[tuple[str, str], tuple[dict[str, str], list[str], str]],
+        attached_transforms: AttachedTransforms,
+        color_mask: str | bool,
     ):
         self.args = args
-        self.always_attached = always_attached
-        self.prepare_preprocess()
+        self.attached_transforms = attached_transforms
+        self.color_mask = color_mask
 
-    def prepare_preprocess(self):
+    def attach_styles(self, table: Table) -> Table:
         """
-        Create a table with styles attached
+        Attach the styles to the table
+
+        Args:
+            table: The table to attach the styles to
         """
-        # create new columns
-        table = self.args["table"]
-        for (arg, col), (map, ls, new_col) in self.always_attached.items():
+        for (by_col, new_col, style_list, style_map) in self.attached_transforms:
             manager_col = get_unique_names(table, [f"{new_col}_manager"])[
                 f"{new_col}_manager"
             ]
-            style_manager = StyleManager(map=map, ls=ls)
+            style_manager = StyleManager(map=style_map, ls=style_list)
 
             table = table.update_view(
                 [
                     f"{manager_col}=style_manager",
-                    f"{new_col}={manager_col}.assign_style({col})",
+                    f"{new_col}={manager_col}.assign_style({by_col}, {self.color_mask})",
                 ]
             )
 
-        self.args["table"] = table
+        return table
+
+    def preprocess_partitioned_tables(
+        self, tables: list[Table], column: str | None = None
+    ):
+        """
+        Preprocess the attached styles
+
+        Args:
+            tables: The tables to process
+            column: column: the column used (ignored)
+
+        Returns:
+            A tuple containing (the new table, an update to make to the args)
+        """
+        for table in tables:
+            yield self.attach_styles(table), {}