Skip to content

Commit b7d915c

Browse files
author
pixie-agent
committed
dx_evidence_graph: AE evidence-graph viz, rebased onto main
Consolidates PR #62 (entlein/dx-evidence-graph-viz, 28 commits) onto current main. Squashed because ~13 of its commits were UI-build CI fixes (use_default_shell_env, yarn-by-abs-path, set -x, license/runner-label) that main already got via #64 — a linear rebase collided on bazel/ui.bzl repeatedly; a 3-way squash-merge reconciles them cleanly. Net deliverable: dx_evidence_graph PxL script + vis.json + manifest, vispb/vis.proto graph fields, and the GraphWidget (graph.tsx/graph-utils.ts) edge-label/pin/popup rendering wired to the forensic ClickHouse DSN — the dx→pixie attack-graph viz.
1 parent 68cc095 commit b7d915c

11 files changed

Lines changed: 501 additions & 26 deletions

File tree

‎bazel/ui.bzl‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ def _pl_webpack_deps_impl(ctx):
4040

4141
cmd = ui_shared_cmds_start + cp_cmds + [
4242
'pushd "$TMPPATH/src/ui" &> /dev/null',
43-
"yarn install --immutable &> build.log",
43+
"/opt/px_dev/tools/node/bin/yarn install --immutable &> build.log",
4444
# Pick a deterministic mtime so that the output is not volatile.
4545
# This helps ensure that bazel can cache the ui builds as expected.
4646
'tar --mtime="2018-01-01 00:00:00 UTC" -czf "$BASE_PATH/{}" .'.format(out.path),
@@ -87,9 +87,7 @@ def _pl_webpack_library_impl(ctx):
8787
'pushd "$TMPPATH/src/ui" &> /dev/null',
8888
'tar -xzf "$BASE_PATH/{}"'.format(ctx.file.deps.path),
8989
'mv -f "$BASE_PATH/{}" src/pages/credits/licenses.json'.format(ctx.file.licenses.path),
90-
"retval=0",
91-
"output=`yarn build_prod 2>&1` || retval=$?",
92-
'[ "$retval" -eq 0 ] || (echo $output; echo "Build Failed with Code: $retval"; exit $retval)',
90+
"/opt/px_dev/tools/node/bin/yarn build_prod",
9391
'cp dist/bundle.tar.gz "$BASE_PATH/{}"'.format(out.path),
9492
] + ui_shared_cmds_finish
9593

@@ -165,8 +163,8 @@ def _pl_deps_licenses_impl(ctx):
165163
'pushd "$TMPPATH/src/ui" &> /dev/null',
166164
'export LIC_TMPPATH="$(mktemp -d)"',
167165
'tar -xzf "$BASE_PATH/{}"'.format(ctx.file.deps.path),
168-
"yarn license_check --excludePrivatePackages --production --json --out $LIC_TMPPATH/checker.json",
169-
'yarn pnpify node ./tools/licenses/yarn_license_extractor.js --input=$LIC_TMPPATH/checker.json --output="$BASE_PATH/{}"'.format(out.path),
166+
"/opt/px_dev/tools/node/bin/yarn license_check --excludePrivatePackages --production --json --out $LIC_TMPPATH/checker.json",
167+
'/opt/px_dev/tools/node/bin/yarn pnpify node ./tools/licenses/yarn_license_extractor.js --input=$LIC_TMPPATH/checker.json --output="$BASE_PATH/{}"'.format(out.path),
170168
] + ui_shared_cmds_finish
171169

172170
ctx.actions.run_shell(

‎private/cockpit/script_bundles_config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ metadata:
66
data:
77
SCRIPT_BUNDLE_URLS: >-
88
[
9-
"https://k8sstormcenter.github.io/pixie/pxl_scripts/bundle.json"
9+
"/bundle-oss.json"
1010
]
1111
SCRIPT_BUNDLE_DEV: "false"
1212
PL_SCRIPT_MODIFICATION_DISABLED: "false"

‎src/api/proto/vispb/vis.proto‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -379,6 +379,12 @@ message Graph {
379379
// The value at which the edge should be considered 'HIGH'.
380380
int64 high_threshold = 2;
381381
}
382+
message NodeThresholds {
383+
// The value at which the node should be considered 'MEDIUM'.
384+
int64 medium_threshold = 1;
385+
// The value at which the node should be considered 'HIGH'.
386+
int64 high_threshold = 2;
387+
}
382388
oneof input {
383389
// The column which contains the dot-formatted graph file to render.
384390
string dot_column = 1;
@@ -399,6 +405,21 @@ message Graph {
399405
int64 edge_length = 8;
400406
// Whether the graph should start in hierarchy mode as a default.
401407
bool enable_default_hierarchy = 9;
408+
// The column whose value is rendered as a persistent label on the edge. Optional.
409+
// When unset, edges have no label (current behaviour).
410+
string edge_label_column = 10;
411+
// The column whose value overrides the default node id label. Optional.
412+
// When unset, the node id is used as the label (current behaviour).
413+
string node_label_column = 11;
414+
// The column to use to determine what the color (fill) of the node should be. Optional.
415+
// Interpreted via NodeThresholds when set; falls back to the SemanticType-derived shape
416+
// color when unset.
417+
string node_color_column = 12;
418+
// The threshold at which node values are classified as a 'LOW'/'MEDIUM'/'HIGH' color.
419+
// Optional, but must have a NodeColorColumn specified.
420+
NodeThresholds node_thresholds = 13;
421+
// The columns to display when hovering over a node. Optional.
422+
repeated string node_hover_info = 14;
402423
}
403424

404425
// Display traffic between pods or services as a graph.

‎src/pxl_scripts/BUILD.bazel‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,9 +39,10 @@ genrule(
3939
],
4040
outs = ["bundle-oss.json"],
4141
cmd = """
42-
export PATH_PREFIX=$$(dirname $(location //src/pxl_scripts:Makefile))/;
43-
EXECUTABLES=../../$(location //src/pixie_cli:px) make -C $$PATH_PREFIX bundle-oss.json;
44-
cp bundle-oss.json $(@D)/bundle-oss.json
42+
MAKEDIR=$$(dirname $(location //src/pxl_scripts:Makefile));
43+
PX_ABS=$$(pwd)/$(location //src/pixie_cli:px);
44+
( cd $$MAKEDIR && EXECUTABLES=$$PX_ABS make PATH_PREFIX= bundle-oss.json );
45+
cp $$MAKEDIR/bundle-oss.json $(@D)/bundle-oss.json
4546
""",
4647
tools = [
4748
"//src/pixie_cli:px",

‎src/pxl_scripts/px/dx_evidence_graph/README.md‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
# dx_evidence_graph
2+
3+
A Pixie UI dashboard that renders one dx-agent investigation as a
4+
**severity-weighted, all-protocol pod-to-pod attack graph**. Replaces
5+
the latency-weighted HTTP service map in `cluster_overview` for
6+
security work.
7+
8+
* Nodes = pods. Falls back to service → IP, mirroring `net_flow_graph`.
9+
* Edges = the attack path emitted by dx (delivery → egress →
10+
execution → collection → exfil → pivot).
11+
* Display spec: `vispb.Graph`. **`edgeWeightColumn = weight`**
12+
(open-ended UInt16 sum of CRS severity → edge thickness),
13+
**`edgeColorColumn = max_severity`** (discrete 2-5 heat → edge
14+
colour).
15+
* Read source: `forensic_db.dx_attack_graph` via `px.DataFrame`'s
16+
`clickhouse_dsn` kwarg (`src/carnot/planner/objects/dataframe.cc:43`).
17+
18+
## Schema — `forensic_db.dx_attack_graph`
19+
20+
Locked with dx-agent in PR #62 / `entlein/dx#68`. The
21+
`attackgraph.Edge` Go struct is the single source of truth for the
22+
JSON wire format, the ClickHouse row, and the test fixture.
23+
24+
| Column | Type | Role |
25+
|---|---|---|
26+
| `investigation_id` | String | one graph per dx verdict / pivot incident (UI filter key) |
27+
| `ts` | UInt64 | unix nanos |
28+
| `requestor_pod` / `responder_pod` | String | the hop (`ns/pod`); `""` if only an IP is known |
29+
| `requestor_service` / `responder_service` | String | |
30+
| `requestor_ip` / `responder_ip` | String | peer IP when pod unresolved |
31+
| `weight` | UInt16 | Σ CRS severity on the hop — `edgeWeightColumn` |
32+
| `max_severity` | UInt8 | top single-criterion severity (2-5) — `edgeColorColumn` |
33+
| `confidence` | Float32 | verdict confidence |
34+
| `edge_kind` | String | `delivery`/`egress`/`execution`/`collection`/`exfil`/`pivot` |
35+
| `condition` / `criteria` | String | ruled-in condition + criterion label(s) |
36+
| `num_findings` | UInt32 | |
37+
38+
Table DDL (mirrors `kubescape_logs` partition/TTL convention):
39+
40+
```sql
41+
CREATE TABLE forensic_db.dx_attack_graph ( ...columns above... )
42+
ENGINE = MergeTree
43+
PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(ts))
44+
ORDER BY (investigation_id, requestor_pod, responder_pod)
45+
TTL toDateTime(fromUnixTimestamp64Nano(ts)) + INTERVAL 30 DAY DELETE;
46+
```
47+
48+
## Per-rig ClickHouse DSN
49+
50+
The bundled `vis.json` ships with `clickhouse_dsn` **empty** — the
51+
default is intentionally non-credentialed so the bundle stays
52+
portable across clusters. Operators fill the DSN in via the Pixie
53+
UI script-args panel at run time.
54+
55+
For the in-cluster soc deployment the DSN is:
56+
57+
```
58+
forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db
59+
```
60+
61+
`forensic_analyst` has read-only SELECT on `forensic_db`; same
62+
credential the existing `soc/analysis/px_clickhouse/kubescape/observe.pxl`
63+
script uses for `kubescape_logs`. Override in the UI for other rigs.
64+
65+
## Deploy
66+
67+
Bundle build path:
68+
69+
1. `//src/pxl_scripts:script_bundle` walks every `*.pxl` + `vis.json`
70+
under `src/pxl_scripts/` and emits `bundle-oss.json`
71+
(`src/pxl_scripts/BUILD.bazel:34`).
72+
2. `//src/cloud/proxy:proxy_server_image` bakes the bundle in as a
73+
container layer at `/bundle`
74+
(`src/cloud/proxy/BUILD.bazel:36`).
75+
3. `skaffold run -f skaffold/skaffold_cloud.yaml` rebuilds the
76+
cloud-proxy image and applies the Deployment.
77+
78+
Vizier / PEM / standalone-pem images are unaffected — this is a
79+
UI-bundle-only change.
80+
81+
## Out of scope for v1
82+
83+
* `conn_stats` overlay (the "render the benign neighbourhood + light
84+
up the attack path" view). Ship the attack-path-only graph first;
85+
add the join in v2 once the visual has been used on a real
86+
incident.
87+
* Time anchoring relative to `ts` rather than free-form `start_time`.
88+
Operators today use `-15m` defaults; a future widget could centre
89+
the window on the investigation's first `ts`.

‎src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Copyright 2018- The Pixie Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
# SPDX-License-Identifier: Apache-2.0
16+
17+
import px
18+
19+
20+
def dx_attack_graph(start_time: str, clickhouse_dsn: str, table: str):
21+
df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time)
22+
return df[['requestor_pod', 'responder_pod',
23+
'requestor_service', 'responder_service',
24+
'requestor_ip', 'responder_ip',
25+
'weight', 'max_severity', 'confidence',
26+
'edge_kind', 'condition', 'criteria', 'num_findings',
27+
'investigation_id']]

‎src/pxl_scripts/px/dx_evidence_graph/manifest.yaml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
---
2+
short: DX Attack Graph
3+
long: >
4+
Severity-weighted, all-protocol pod-to-pod attack graph for one
5+
dx-agent investigation. Renders attackgraph.Edge records emitted by
6+
dx with weight (sum of CRS evidence severity) on the edges and
7+
max_severity colouring the heat. v0 manual-load only — wires up to
8+
the dx_attack_graph ClickHouse / Pixie ingest in v1. See README.md
9+
in this directory.

‎src/pxl_scripts/px/dx_evidence_graph/vis.json‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
{
2+
"variables": [
3+
{
4+
"name": "start_time",
5+
"type": "PX_STRING",
6+
"description": "Start time of the window.",
7+
"defaultValue": "-15m"
8+
},
9+
{
10+
"name": "clickhouse_dsn",
11+
"type": "PX_STRING",
12+
"description": "ClickHouse DSN: user:pass@host:port/db.",
13+
"defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"
14+
},
15+
{
16+
"name": "table",
17+
"type": "PX_STRING",
18+
"description": "dx_attack_graph_malicious (default; rule-ins only — benign is NOT pulled from ClickHouse) or dx_attack_graph (full table, includes benign).",
19+
"defaultValue": "dx_attack_graph_malicious"
20+
}
21+
],
22+
"globalFuncs": [
23+
{
24+
"outputName": "dx_graph",
25+
"func": {
26+
"name": "dx_attack_graph",
27+
"args": [
28+
{"name": "start_time", "variable": "start_time"},
29+
{"name": "clickhouse_dsn", "variable": "clickhouse_dsn"},
30+
{"name": "table", "variable": "table"}
31+
]
32+
}
33+
}
34+
],
35+
"widgets": [
36+
{
37+
"name": "DX Attack Graph",
38+
"position": {"x": 0, "y": 0, "w": 12, "h": 5},
39+
"globalFuncOutputName": "dx_graph",
40+
"displaySpec": {
41+
"@type": "types.px.dev/px.vispb.Graph",
42+
"adjacencyList": {
43+
"fromColumn": "requestor_pod",
44+
"toColumn": "responder_pod"
45+
},
46+
"edgeWeightColumn": "weight",
47+
"edgeColorColumn": "max_severity",
48+
"edgeLabelColumn": "edge_kind",
49+
"edgeThresholds": {
50+
"mediumThreshold": 3,
51+
"highThreshold": 4
52+
},
53+
"edgeHoverInfo": [
54+
"edge_kind",
55+
"condition",
56+
"criteria",
57+
"weight",
58+
"max_severity",
59+
"confidence",
60+
"num_findings",
61+
"investigation_id"
62+
],
63+
"edgeLength": 500
64+
}
65+
},
66+
{
67+
"name": "Edges",
68+
"position": {"x": 0, "y": 5, "w": 12, "h": 4},
69+
"globalFuncOutputName": "dx_graph",
70+
"displaySpec": {
71+
"@type": "types.px.dev/px.vispb.Table"
72+
}
73+
}
74+
]
75+
}

‎src/ui/src/containers/live-widgets/graph/graph-utils.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,12 @@ export function getGraphOptions(theme: Theme, edgeLength: number): Options {
6363
smooth: false,
6464
scaling: {
6565
max: 5,
66+
label: false,
67+
},
68+
font: {
69+
strokeWidth: 0,
70+
color: theme.palette.text.primary,
71+
face: 'Roboto',
6672
},
6773
arrows: {
6874
to: {

0 commit comments

Comments
 (0)