Skip to content

Commit 1e6bc04

Browse files
lmeyerovclaude
andcommitted
feat(gfql): native polars multi-source (node-cartesian) MATCH lowering (#1273)
Multi-source MATCH like `MATCH (a {..}), (b {..})` lowers to a `rows(binding_ops=[Node, Node, ...])` op whose node_cartesian mode was previously deferred (returned None -> NotImplementedError) on the polars engine. Implement it natively, mirroring the pandas `_gfql_cartesian_node_bindings_row_table` oracle exactly: - per-alias frame = filter_by_dict + project into the `_gfql_node_alias_lookup_frame` schema (bare `alias` id, `alias.id`, `alias.<prop>`, and the leaked named-op flag `alias.alias = True` that shadows a same-named real property), then a left-major cross join so the row order matches pandas' constant-key merge (no ORDER BY needed). - Decline (honest NIE, no pandas bridge) outside pandas' reliable zone: anonymous node ops (pandas raises a spurious schema error on empty results) and >3 named aliases (pandas' bare-id merge residue collides on the 4th frame). Both engines then error, never diverge. Verified by a differential fuzz (random small graphs x multi-source query shapes incl. WHERE / ORDER BY / property-name-vs-alias collisions) against the pandas oracle: 0 disagreements over ~10k lowered cases across seeds. Adds parametrized parity + explicit pin tests. No new ruff/mypy findings. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL
1 parent c355660 commit 1e6bc04

2 files changed

Lines changed: 162 additions & 2 deletions

File tree

graphistry/compute/gfql/lazy/engine/polars/row_pipeline.py

Lines changed: 97 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -870,6 +870,101 @@ def select_extend_polars(g: Plottable, items: Sequence[SelectItem]) -> Optional[
870870
return _rewrap(g, out)
871871

872872

873+
def _cartesian_node_bindings_polars(
874+
g: Plottable,
875+
ops: "Sequence[Any]",
876+
node_id: Optional[str],
877+
) -> Optional[Plottable]:
878+
"""Native polars cross-product for disconnected MATCH aliases (#1273).
879+
880+
Mirrors the pandas ``_gfql_cartesian_node_bindings_row_table`` oracle: each
881+
node alias is independently filtered, projected into the ``_gfql_node_alias_lookup_frame``
882+
schema (``alias``, ``alias.node_id``, ``alias.<col>``, plus the leaked named-op
883+
FLAG column ``alias.alias = True``), then cross-joined in op order (left-major,
884+
matching pandas' ``merge`` order so no ORDER BY is needed for parity). The bare
885+
``node_id`` residue column that pandas carries (``id_x``/``id_y`` merge suffixes)
886+
is intentionally dropped: no lowered query references it, and dropping avoids
887+
polars cross-join column collisions on 3+ aliases.
888+
889+
Returns None to DECLINE (caller raises the honest NIE) outside the supported
890+
subset: node ``query=`` params, ``alias == node_id`` (pandas' flag column
891+
overwrites the id column — no sane shared semantics), and seeded re-entry
892+
(already gated by the caller). NO-CHEATING: never bridges to pandas.
893+
"""
894+
import polars as pl
895+
from graphistry.compute.ast import ASTNode
896+
from graphistry.compute.gfql.lazy import collect as _lazy_collect
897+
from .predicates import filter_by_dict_polars
898+
899+
nodes = g._nodes
900+
if nodes is None or node_id is None:
901+
return None
902+
node_id = str(node_id)
903+
if node_id not in nodes.columns:
904+
return None
905+
906+
aliases = [op._name for op in ops]
907+
# Decline outside pandas' RELIABLE zone (empirically derived, keeps parity):
908+
# - anonymous node op: the pandas cartesian raises a spurious schema error on
909+
# an EMPTY result when a bare `()` is present (it drops the id column) rather
910+
# than returning empty — declining avoids that divergence.
911+
# - >3 named aliases: the pandas builder's per-alias bare-id merge residue
912+
# collides on the 4th frame ("Passing 'suffixes' which cause duplicate
913+
# columns"). Both engines must not diverge, so decline to the honest NIE.
914+
if any(not isinstance(a, str) for a in aliases):
915+
return None
916+
named = [a for a in aliases if isinstance(a, str)]
917+
if len(named) > 3:
918+
return None
919+
920+
nodes_lf = nodes.lazy()
921+
# filter_by_dict_polars is frame-polymorphic (LazyFrame in -> LazyFrame out) but
922+
# annotated ``pl.DataFrame``; keep the accumulator loose, as this module does.
923+
per_alias: List[Any] = []
924+
for op in ops:
925+
if not isinstance(op, ASTNode) or op.query is not None:
926+
return None
927+
alias = op._name
928+
if not isinstance(alias, str):
929+
return None
930+
if alias == node_id:
931+
# pandas' named-op flag column overwrites the id column here — neither
932+
# engine has sane semantics; decline (mirrors the single-entity
933+
# ``rows_binding_ops_polars`` corner).
934+
return None
935+
try:
936+
matched = filter_by_dict_polars(nodes_lf, op.filter_dict)
937+
except NotImplementedError:
938+
raise
939+
except Exception:
940+
return None
941+
cols = matched.collect_schema().names()
942+
# prop_cols excludes node_id and any real column named == alias: the pandas
943+
# node execute() leaks a boolean FLAG into a column named ``alias``
944+
# (shadowing a same-named real property), which the lookup frame surfaces
945+
# as ``alias.alias = True``. Reproduce that exactly.
946+
prop_cols = [c for c in cols if c != node_id and c != alias]
947+
exprs = [
948+
pl.col(node_id).alias(alias),
949+
pl.col(node_id).alias(f"{alias}.{node_id}"),
950+
pl.lit(True).alias(f"{alias}.{alias}"),
951+
]
952+
exprs.extend(pl.col(c).alias(f"{alias}.{c}") for c in prop_cols)
953+
per_alias.append(matched.select(exprs))
954+
955+
if not per_alias:
956+
return None
957+
state = per_alias[0]
958+
for frame in per_alias[1:]:
959+
# Left-major cross join → same row order as the pandas constant-key merge.
960+
state = state.join(frame, how="cross")
961+
try:
962+
out_df = _lazy_collect(state)
963+
except pl.exceptions.SchemaError:
964+
return None
965+
return _rewrap(g, out_df)
966+
967+
873968
def binding_rows_polars(
874969
g: Plottable,
875970
binding_ops: Sequence[Dict[str, JSONVal]],
@@ -921,7 +1016,8 @@ def _names(lf: pl.LazyFrame) -> List[str]:
9211016
# for malformed op sequences / duplicate aliases — same error as pandas.
9221017
RowPipelineMixin._gfql_validate_binding_ops(ops)
9231018
if RowPipelineMixin._gfql_binding_ops_mode(ops) == "node_cartesian":
924-
return None # MATCH (a), (b) cross joins: deferred (rare; own schema study)
1019+
# MATCH (a), (b), ... disconnected node aliases: native cross-product (#1273).
1020+
return _cartesian_node_bindings_polars(g, ops, node_id)
9251021
if RowPipelineMixin._gfql_is_shortest_path_scalar_binding_ops(ops):
9261022
return None # shortestPath scalar contract: BFS/native backends, pandas-only
9271023

graphistry/tests/compute/gfql/test_engine_polars_binding_rows.py

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,13 @@ def _assert_parity(query, *, order_sensitive=False):
8181
"MATCH (a)-[*2..2]->(b) RETURN count(*) AS c", # exactly-k
8282
"MATCH (a)-[:F*1..2]->(b) RETURN count(*) AS c", # typed var-length
8383
"MATCH (a)-[*1..2]->(b)-[]->(c) RETURN count(*) AS c", # var-length + fixed hop
84+
# node-only cartesian: disconnected multi-source MATCH (#1273). <=3 named aliases.
85+
"MATCH (a), (b) RETURN a.id AS ai, b.id AS bi ORDER BY ai, bi",
86+
"MATCH (a {kind: 'a'}), (b {kind: 'b'}) RETURN a.id AS ai, b.id AS bi",
87+
"MATCH (a {kind: 'a'}), (b) RETURN a.id AS ai, a.age AS aa, b.id AS bi",
88+
"MATCH (a {kind: 'a'}), (b {kind: 'b'}) WHERE a.age > 20 RETURN a.id AS ai, b.id AS bi",
89+
"MATCH (a), (b), (c {kind: 'a'}) RETURN a.id AS ai, b.id AS bi, c.id AS ci ORDER BY ai, bi, ci",
90+
"MATCH (a {kind: 'z'}), (b) RETURN a.id AS ai, b.id AS bi", # empty (no kind=z) keeps schema
8491
]
8592

8693
# Outside the MVP subset: must raise NotImplementedError (honest NIE, no bridge,
@@ -89,6 +96,10 @@ def _assert_parity(query, *, order_sensitive=False):
8996
"MATCH (a)-[*]->(b) RETURN count(*) AS c", # unbounded var-length
9097
"MATCH (a)-[*1..2]-(b) RETURN count(*) AS c", # undirected var-length
9198
"MATCH (a)-[e]->(b) WHERE a.age < b.age RETURN a.id", # cross-alias same-path WHERE
99+
# cartesian outside the pandas-reliable subset (#1273): pandas itself errors or
100+
# is fragile here, so polars declines rather than diverge.
101+
"MATCH (a), (b), (c), (d) RETURN a.id, b.id, c.id, d.id", # >3 named aliases
102+
"MATCH (a), (b), () RETURN a.id, b.id", # anonymous companion
92103
]
93104

94105

@@ -121,6 +132,50 @@ def test_polars_binding_rows_raw_table_meaningful_cols():
121132
pd.testing.assert_frame_equal(a, b, check_dtype=False)
122133

123134

135+
def test_polars_cartesian_binding_rows_raw_meaningful_cols():
136+
"""Raw node-only cartesian rows(binding_ops): polars carries the same meaningful
137+
per-alias schema as pandas (bare ``alias`` id, ``alias.id``, ``alias.<prop>``,
138+
plus the leaked named-op flag ``alias.alias = True``), values equal to pandas."""
139+
bo = serialize_binding_ops([n(name="a"), n(name="b")])
140+
rpd = BASE.gfql([rows(binding_ops=bo)], engine="pandas")._nodes
141+
rpl = BASE.gfql([rows(binding_ops=bo)], engine="polars")._nodes
142+
assert "polars" in type(rpl).__module__
143+
expected = {"a", "a.id", "a.age", "a.kind", "a.a", "b", "b.id", "b.age", "b.kind", "b.b"}
144+
assert expected <= set(rpl.columns)
145+
assert set(rpl.columns) <= set(rpd.columns) # no columns pandas lacks
146+
key = ["a", "b"]
147+
a = rpd[list(rpl.columns)].sort_values(key).reset_index(drop=True)
148+
b = rpl.to_pandas().sort_values(key).reset_index(drop=True)
149+
pd.testing.assert_frame_equal(a, b, check_dtype=False)
150+
151+
152+
def test_polars_cartesian_alias_name_collides_with_property():
153+
"""A node property named the same as a MATCH alias is shadowed by the leaked
154+
named-op flag (``alias.alias = True``) on BOTH engines — polars mirrors the
155+
pandas quirk exactly rather than surfacing the real property value."""
156+
nodes = pd.DataFrame({"id": [0, 1, 2], "kind": ["a", "b", "a"], "a": [10, 20, 30], "b": [1, 2, 3]})
157+
g = graphistry.nodes(nodes, "id").edges(pd.DataFrame({"s": [0], "d": [1]}), "s", "d")
158+
q = "MATCH (a {kind: 'a'}), (b {kind: 'b'}) RETURN a.id AS ai, a.a AS aa, b.id AS bi, b.b AS bb"
159+
rpd = g.gfql(q, engine="pandas")._nodes.reset_index(drop=True)
160+
rpl = g.gfql(q, engine="polars")._nodes.to_pandas().reset_index(drop=True)
161+
assert list(rpd["aa"]) == [True, True] and list(rpd["bb"]) == [True, True] # flag, not 10/30
162+
pd.testing.assert_frame_equal(
163+
rpd.sort_values(["ai", "bi"]).reset_index(drop=True),
164+
rpl[rpd.columns.tolist()].sort_values(["ai", "bi"]).reset_index(drop=True),
165+
check_dtype=False,
166+
)
167+
168+
169+
def test_polars_cartesian_multiplicity_three_aliases():
170+
"""Three-alias cross product has |a|*|b|*|c| rows in left-major order on polars,
171+
identical to pandas."""
172+
q = "MATCH (a {kind: 'a'}), (b {kind: 'b'}), (c {kind: 'a'}) RETURN a.id AS ai, b.id AS bi, c.id AS ci"
173+
rpd = BASE.gfql(q, engine="pandas")._nodes.reset_index(drop=True)
174+
rpl = BASE.gfql(q, engine="polars")._nodes.to_pandas().reset_index(drop=True)
175+
assert len(rpl) == 3 * 2 * 3 # kind a = {0,2,4}, kind b = {1,3}
176+
pd.testing.assert_frame_equal(rpd, rpl[rpd.columns.tolist()], check_dtype=False) # order-exact
177+
178+
124179
def test_binding_rows_projection_pushdown_skips_unused_props():
125180
"""#1711: a query referencing no node properties (count(*)) attaches ZERO
126181
property columns to the binding table; one referencing only b's property
@@ -248,7 +303,16 @@ def test_polars_binding_rows_decline_branches_direct():
248303
assert binding_rows_polars(seeded, bo) is None
249304

250305
g = graphistry.nodes(pl.from_pandas(NODES), "id").edges(pl.from_pandas(EDGES), "s", "d")
251-
assert binding_rows_polars(g, serialize_binding_ops([n(name="a"), n(name="b")])) is None
306+
# node-only cartesian (#1273) is now natively supported for <=3 named aliases;
307+
# it declines only outside the pandas-reliable subset:
308+
# - anonymous node op (pandas raises a spurious schema error on empty results)
309+
assert binding_rows_polars(g, serialize_binding_ops([n(name="a"), n()])) is None
310+
# - >3 named aliases (pandas' bare-id merge residue collides on the 4th frame)
311+
assert binding_rows_polars(
312+
g, serialize_binding_ops([n(name="a"), n(name="b"), n(name="c"), n(name="d")])
313+
) is None
314+
# ...but 2-3 named aliases now lower natively (returns a row table, not None)
315+
assert binding_rows_polars(g, serialize_binding_ops([n(name="a"), n(name="b")])) is not None
252316
assert binding_rows_polars(g, serialize_binding_ops([n(name="a", query="id > 0"), e_forward(), n(name="b")])) is None
253317
assert binding_rows_polars(g, serialize_binding_ops([n(name="a"), e_forward(source_node_match={"kind": "a"}), n(name="b")])) is None
254318
assert binding_rows_polars(g, serialize_binding_ops([n(name="a"), e_forward(label_seeds=True), n(name="b")])) is None

0 commit comments

Comments
 (0)