Skip to content

Commit 289c6ab

Browse files
lmeyerovclaude
andcommitted
refactor(gfql): extract seeded fast-path specializations to dedicated modules (#1755)
Pure code move (no behavior change), following the gfql_fast_paths.py #1731 convention (keep the orchestrator readable; one-directional imports, no back-edge). Moves the #1755 seeded typed-hop specializations out of the big chain.py / gfql_unified.py orchestrators: - NEW graphistry/compute/chain_fast_paths.py: _seeded_typed_hop_pandas_cudf, _seeded_typed_return_dst_pandas_cudf, _seeded_typed_return_dst_polars (moved verbatim; imports only leaves — .ast, .typing, Plottable). - gfql_fast_paths.py (existing): gains _execute_seeded_typed_hop_fast_path. - chain.py / gfql_unified.py: import the moved helpers (call sites unchanged). Import edges (all one-directional, no cycles): chain -> chain_fast_paths; gfql_unified -> gfql_fast_paths -> chain_fast_paths; chain_fast_paths -> leaves. 41 seeded tests pass unchanged; mypy clean on all four files. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01Y6dQEcjdazEnzuvuwf73ZL
1 parent 206a8ab commit 289c6ab

5 files changed

Lines changed: 313 additions & 298 deletions

File tree

graphistry/compute/chain.py

Lines changed: 1 addition & 184 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
from .ast import ASTObject, ASTNode, ASTEdge, Direction, from_json as ASTObject_from_json, serialize_binding_ops
1212
from .typing import DataFrameT, SeriesT
1313
from .util import generate_safe_column_name
14+
from .chain_fast_paths import _seeded_typed_hop_pandas_cudf
1415
from graphistry.compute.validate.validate_schema import validate_chain_schema
1516
from graphistry.compute.gfql.same_path_types import (
1617
WhereComparison,
@@ -684,190 +685,6 @@ def _chain_otel_attrs(
684685
return attrs
685686

686687

687-
def _seeded_scalar_filters(fd: Optional[Dict[str, Any]], df: DataFrameT) -> Optional[Dict[str, Any]]:
688-
"""Resolve a filter dict to plain scalar column==value pairs, or None to bail
689-
to the general path. Mirrors filter_by_dict.resolve_filter_column exactly for
690-
the shapes it accepts: the cypher ``label__X: True`` form maps to ``type``
691-
equality ONLY when no list-valued ``labels`` column exists (labels-containment
692-
is not scalar equality) and the frame is not edge-shaped — same precedence as
693-
the live resolver. Anything else (predicates, non-scalar values, absent
694-
columns) bails, so the full path keeps its exact semantics incl. E301."""
695-
from graphistry.compute.filter_by_dict import _looks_like_edge_dataframe
696-
if not fd:
697-
return {}
698-
cols = set(df.columns)
699-
out: Dict[str, Any] = {}
700-
for k, v in fd.items():
701-
if not isinstance(v, (int, float, str, bool)):
702-
return None # predicate / non-scalar -> bail to the general path
703-
if k in cols:
704-
out[k] = v
705-
elif (isinstance(k, str) and k.startswith("label__") and v is True
706-
and "labels" not in cols and "type" in cols
707-
and not _looks_like_edge_dataframe(df)):
708-
out["type"] = k[len("label__"):]
709-
else:
710-
return None # labels-list / unknown column -> bail
711-
return out
712-
713-
714-
def _seeded_typed_hop_pandas_cudf(
715-
g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge,
716-
src: str, dst: str, node: str, direction: Direction,
717-
) -> Optional[Plottable]:
718-
"""#1755 lever-3: engine-generic (pandas + cuDF) fast path for a scalar-filtered
719-
seeded typed 1-hop. Value-identical to the general seeded branch for the covered
720-
shape (all node/edge filters are plain scalars, directed) — same rows, columns,
721-
and dtypes; row order and RangeIndex may differ — collapsing it into a
722-
few DataFrame filters so a seeded lookup lands sub-ms. Uses only the shared
723-
pandas/cuDF DataFrame API (no numpy array drops) so the same body runs on both
724-
engines. Returns None to fall back for anything it does not cover (predicates,
725-
undirected, missing columns) — the caller then runs the general branch."""
726-
if direction == "undirected":
727-
return None
728-
729-
nodes_df, edges_df = g._nodes, g._edges
730-
if nodes_df is None or edges_df is None:
731-
return None
732-
n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df)
733-
n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df)
734-
ef = _seeded_scalar_filters(e1.edge_match, edges_df)
735-
if n0f is None or n2f is None or ef is None:
736-
return None
737-
from_col, to_col = (src, dst) if direction == "forward" else (dst, src)
738-
739-
# from-side seed FIRST: reduce edges to the seed's out-edges before the
740-
# edge_match compare, so the type filter runs on the tiny frontier rather than
741-
# all edges — this is what makes a seeded lookup sub-ms. The id filter goes
742-
# first (int, unique -> ~1 row in one pass) so any remaining object filters
743-
# (label__X->type) run on that tiny survivor frame, not the whole node table.
744-
if n0f:
745-
seed_nodes = nodes_df
746-
for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1):
747-
seed_nodes = seed_nodes[seed_nodes[k] == v]
748-
edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())]
749-
else:
750-
edges = edges_df
751-
if ef: # typed edge (edge_match) — now on the reduced frontier
752-
for k, v in ef.items():
753-
edges = edges[edges[k] == v]
754-
755-
# Gather candidate endpoint nodes (both endpoints of surviving edges), then run
756-
# the dest filter, dangling-edge drop and final-node selection on the small
757-
# candidate/edge frames. Selecting from nodes_df keeps only real nodes, so the
758-
# endpoint-in-nodes check subsumes the old NaN-endpoint guard. Membership sets
759-
# are dropna()'d: pandas .isin matches NaN<->NaN, but the general branch's BFS
760-
# joins never join on null keys, so a null id/endpoint must not link.
761-
cand = nodes_df[
762-
nodes_df[node].isin(edges[src].dropna()) | nodes_df[node].isin(edges[dst].dropna())
763-
].drop_duplicates(subset=[node])
764-
if n2f: # destination-node filter (to-side)
765-
n2_cand = cand
766-
for k, v in n2f.items():
767-
n2_cand = n2_cand[n2_cand[k] == v]
768-
n2_ok = n2_cand[node]
769-
else:
770-
n2_ok = cand[node]
771-
to_vals = edges[to_col]
772-
keep = edges[src].isin(cand[node].dropna()) & edges[dst].isin(cand[node].dropna()) & to_vals.isin(n2_ok.dropna())
773-
edges = edges[keep]
774-
cand = cand[cand[node].isin(edges[src]) | cand[node].isin(edges[dst])]
775-
return g.nodes(cand).edges(edges)
776-
777-
778-
def _seeded_typed_return_dst_pandas_cudf(
779-
g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge,
780-
src: str, dst: str, node: str, direction: Direction,
781-
) -> Optional[Tuple[DataFrameT, DataFrameT]]:
782-
"""#1755 cypher RETURN-alias fast path: like _seeded_typed_hop_pandas_cudf but
783-
returns ONLY the destination (RETURN-alias) node rows + surviving edges — no
784-
seed-node gather, no Plottable round-trip — so the seeded cypher projection
785-
lands sub-ms. Engine-generic (pandas + cuDF): only the shared DataFrame API,
786-
no numpy array drops. Returns ``(dst_node_rows, edges)`` or None to fall back."""
787-
if direction == "undirected":
788-
return None
789-
nodes_df, edges_df = g._nodes, g._edges
790-
if nodes_df is None or edges_df is None:
791-
return None
792-
n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df)
793-
n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df)
794-
ef = _seeded_scalar_filters(e1.edge_match, edges_df)
795-
if n0f is None or n2f is None or ef is None or not n0f:
796-
return None
797-
from_col, to_col = (src, dst) if direction == "forward" else (dst, src)
798-
# id-first seed reduction: filter by the id column first (int/unique -> ~1 row)
799-
# so any remaining object filters (label__X->type) run on the tiny survivor
800-
# frame, never materializing an object column over the whole node table.
801-
# Membership sets are dropna()'d: pandas .isin matches NaN<->NaN, but the full
802-
# pipeline's joins never join on null keys, so a null id/endpoint must not link.
803-
seed_nodes = nodes_df
804-
for k, v in sorted(n0f.items(), key=lambda kv: 0 if kv[0] == node else 1):
805-
seed_nodes = seed_nodes[seed_nodes[k] == v]
806-
edges = edges_df[edges_df[from_col].isin(seed_nodes[node].dropna())]
807-
if ef:
808-
for k, v in ef.items():
809-
edges = edges[edges[k] == v]
810-
# destination nodes = real nodes that are edge to-endpoints, then the dest
811-
# filter, dangling-edge drop and dedup on the small dst/edge frames.
812-
dstn = nodes_df[nodes_df[node].isin(edges[to_col].dropna())]
813-
if n2f:
814-
for k, v in n2f.items():
815-
dstn = dstn[dstn[k] == v]
816-
edges = edges[edges[to_col].isin(dstn[node].dropna())]
817-
dstn = dstn[dstn[node].isin(edges[to_col].dropna())].drop_duplicates(subset=[node])
818-
return dstn, edges
819-
820-
821-
def _seeded_typed_return_dst_polars(
822-
g: Plottable, n0: ASTNode, n2: ASTNode, e1: ASTEdge,
823-
src: str, dst: str, node: str, direction: Direction,
824-
) -> Optional[Tuple[DataFrameT, DataFrameT]]:
825-
"""#1755 polars analog of _seeded_typed_return_dst_pandas_cudf: same seed-first
826-
reduction (seed out-edges -> typed-edge filter -> destination nodes) expressed
827-
with polars filters, so a seeded cypher RETURN on polars/polars-gpu also lands
828-
sub-ms. Returns ``(dst_node_rows, edges)`` (polars frames) or None to fall back
829-
to the full lazy pipeline. Value-identical node set to the full path for the
830-
covered shape (scalar filters, directed, single hop); row order may differ."""
831-
import polars as pl
832-
if direction == "undirected":
833-
return None
834-
nodes_df, edges_df = g._nodes, g._edges
835-
# Eager polars frames only: LazyFrame has no get_column, and mixed-engine
836-
# node/edge frames must take the full path — decline rather than crash.
837-
if not isinstance(nodes_df, pl.DataFrame) or not isinstance(edges_df, pl.DataFrame):
838-
return None
839-
840-
n0f = _seeded_scalar_filters(n0.filter_dict, nodes_df)
841-
n2f = _seeded_scalar_filters(n2.filter_dict, nodes_df)
842-
ef = _seeded_scalar_filters(e1.edge_match, edges_df)
843-
if n0f is None or n2f is None or ef is None or not n0f:
844-
return None
845-
from_col, to_col = (src, dst) if direction == "forward" else (dst, src)
846-
847-
# from-side seed: reduce the node frame to the seed rows, take their ids.
848-
# Membership sets are drop_nulls()'d (null ids/endpoints never link, matching
849-
# the full pipeline's joins) and passed via .implode() (Series-arg is_in is
850-
# deprecated in polars 1.42, see polars#22149).
851-
seed_nodes = nodes_df
852-
for k, v in n0f.items():
853-
seed_nodes = seed_nodes.filter(pl.col(k) == v)
854-
from_ids = seed_nodes.get_column(node).drop_nulls()
855-
if from_ids.len() == 0:
856-
return nodes_df.clear(), edges_df.clear()
857-
edges = edges_df.filter(pl.col(from_col).is_in(from_ids.implode()))
858-
for k, v in ef.items(): # typed edge on the reduced frontier
859-
edges = edges.filter(pl.col(k) == v)
860-
dst_ids = edges.get_column(to_col).drop_nulls().unique()
861-
dstn = nodes_df.filter(pl.col(node).is_in(dst_ids.implode()))
862-
for k, v in n2f.items(): # destination-node filter
863-
dstn = dstn.filter(pl.col(k) == v)
864-
# drop dangling edges + dedup destination nodes (mirror the pandas tail)
865-
keep_ids = dstn.get_column(node).drop_nulls()
866-
edges = edges.filter(pl.col(to_col).is_in(keep_ids.implode()))
867-
dstn = dstn.filter(pl.col(node).is_in(edges.get_column(to_col).implode())).unique(subset=[node], maintain_order=True)
868-
return dstn, edges
869-
870-
871688
def _try_chain_fast_path(
872689
g_in: Plottable,
873690
ops: List[ASTObject],

0 commit comments

Comments
 (0)