refactor: remove legacy result marker fields

Author: bobbai00

agent-service/src/agent/tools/result-formatting.spec.ts

@@ -29,24 +29,19 @@ import {
   type SampleRow,
 } from "../../types/execution";
 
-// Convert flat test rows (with an optional embedded __row_index__) into the
-// structured SampleRow[] the summary now carries.
 function toSampleRows(rows: Record<string, any>[]): SampleRow[] {
-  return rows.map((row, i) => {
-    const { __row_index__, ...tuple } = row;
-    return { rowIndex: typeof __row_index__ === "number" ? __row_index__ : i, tuple };
-  });
+  return rows.map((tuple, rowIndex) => ({ rowIndex, tuple }));
 }
 
-// Test convenience: accept the (old) flat fields and assemble the structured
-// OperatorExecutionSummary, so the cases below stay terse.
 interface OpInfoOverrides {
   state?: OperatorState;
   error?: string;
   outputTuples?: number;
   tuplesCount?: number;
   warnings?: string[];
   result?: Record<string, any>[];
+  sampleTuples?: SampleRow[];
+  resultMode?: OperatorResultMode;
 }
 
 function makeExecutionFailure(message: string): WorkflowFatalError {
@@ -66,11 +61,13 @@ function makeOpInfo(overrides: OpInfoOverrides = {}): OperatorExecutionSummary {
     errorMessages: overrides.error ? [makeExecutionFailure(overrides.error)] : [],
   };
   // The result summary is present only when the operator produced a result.
-  if (overrides.result !== undefined) {
+  if (overrides.result !== undefined || overrides.sampleTuples !== undefined) {
     summary.resultSummary = {
-      resultMode: OperatorResultMode.TABLE,
+      resultMode: overrides.resultMode ?? OperatorResultMode.TABLE,
       // Non-arrays are passed through to exercise the "(no result data)" guard.
-      sampleTuples: Array.isArray(overrides.result) ? toSampleRows(overrides.result) : (overrides.result as any),
+      sampleTuples:
+        overrides.sampleTuples ??
+        (Array.isArray(overrides.result) ? toSampleRows(overrides.result) : (overrides.result as any)),
       tuplesCount: overrides.tuplesCount ?? overrides.outputTuples ?? 0,
     };
   }
@@ -125,16 +122,15 @@ describe("formatOperatorResult - table shape and metadata", () => {
     expect(out).toContain("Output table shape: (999, 2)");
   });
 
-  test("filters internal __is_visualization__ key from outer column count", () => {
+  test("counts every result tuple key as a column", () => {
     const out = formatOperatorResult(
       "op1",
       makeOpInfo({
         outputTuples: 1,
-        result: [{ __is_visualization__: true, "html-content": "<x/>" }],
+        result: [{ "html-content": "<x/>", label: "chart" }],
       })
     );
-    // 1 visible column ("html-content") since __is_visualization__ is filtered.
-    expect(out).toContain("Output table shape: (1, 1)");
+    expect(out).toContain("Output table shape: (1, 2)");
   });
 
   test("appends warnings after metadata lines", () => {
@@ -155,14 +151,14 @@ describe("formatOperatorResult - table shape and metadata", () => {
 });
 
 describe("formatOperatorResult - visualization rows", () => {
-  test("strips html-content and json-content payloads when row is flagged as visualization", () => {
+  test("strips html-content and json-content payloads when result mode is visualization", () => {
     const out = formatOperatorResult(
       "op1",
       makeOpInfo({
         outputTuples: 1,
+        resultMode: OperatorResultMode.VISUALIZATION,
         result: [
           {
-            __is_visualization__: true,
             "html-content": "<div>hidden</div>",
             "json-content": '{"big":1}',
             label: "chart",
@@ -176,32 +172,32 @@ describe("formatOperatorResult - visualization rows", () => {
     expect(out).toContain("chart");
   });
 
-  test("__is_visualization__ false leaves the visualization-only fields untouched", () => {
+  test("table result mode leaves visualization payload fields untouched", () => {
     const out = formatOperatorResult(
       "op1",
       makeOpInfo({
         outputTuples: 1,
-        result: [{ __is_visualization__: false, "html-content": "<keep/>" }],
+        resultMode: OperatorResultMode.TABLE,
+        result: [{ "html-content": "<keep/>" }],
       })
     );
     expect(out).toContain("<keep/>");
     expect(out).not.toContain("<skipped: visualization content>");
   });
 
-  test("__is_visualization__ column is excluded from rendered table body and shape agrees", () => {
+  test("table rows render all tuple columns and shape agrees", () => {
     const out = formatOperatorResult(
       "op1",
       makeOpInfo({
         outputTuples: 1,
-        result: [{ __is_visualization__: false, value: 1 }],
+        result: [{ value: 1 }],
       })
     );
     const lines = out.split("\n");
     expect(out).toContain("Output table shape: (1, 1)");
     // Header line is the third line (after brief summary and shape line).
     expect(lines[2]).toBe("\tvalue");
     expect(lines[3]).toBe("0\t1");
-    expect(out).not.toContain("__is_visualization__");
   });
 });
 
@@ -240,14 +236,14 @@ describe("jsonToTableFormat - cell coercion via formatOperatorResult", () => {
 });
 
 describe("jsonToTableFormat - row index gaps", () => {
-  test("inserts ... separator when __row_index__ skips ahead", () => {
+  test("inserts ... separator when rowIndex skips ahead", () => {
     const out = formatOperatorResult(
       "op1",
       makeOpInfo({
         outputTuples: 2,
-        result: [
-          { __row_index__: 0, v: "a" },
-          { __row_index__: 5, v: "b" },
+        sampleTuples: [
+          { rowIndex: 0, tuple: { v: "a" } },
+          { rowIndex: 5, tuple: { v: "b" } },
         ],
       })
     );
@@ -259,22 +255,25 @@ describe("jsonToTableFormat - row index gaps", () => {
     expect(lines[lines.length - 1]).toBe("5\tb");
   });
 
-  test("no separator is emitted between consecutive __row_index__ values", () => {
+  test("no separator is emitted between consecutive rowIndex values", () => {
     const out = formatOperatorResult(
       "op1",
       makeOpInfo({
         outputTuples: 2,
-        result: [
-          { __row_index__: 0, v: "a" },
-          { __row_index__: 1, v: "b" },
+        sampleTuples: [
+          { rowIndex: 0, tuple: { v: "a" } },
+          { rowIndex: 1, tuple: { v: "b" } },
         ],
       })
     );
     expect(out).not.toContain("...\t...");
   });
 
-  test("non-zero starting __row_index__ does not emit a leading gap marker", () => {
-    const out = formatOperatorResult("op1", makeOpInfo({ outputTuples: 1, result: [{ __row_index__: 9, v: "z" }] }));
+  test("non-zero starting rowIndex does not emit a leading gap marker", () => {
+    const out = formatOperatorResult(
+      "op1",
+      makeOpInfo({ outputTuples: 1, sampleTuples: [{ rowIndex: 9, tuple: { v: "z" } }] })
+    );
     expect(out).not.toContain("...\t...");
     expect(out.endsWith("9\tz")).toBe(true);
   });

agent-service/src/agent/tools/result-formatting.ts

@@ -17,7 +17,7 @@
  * under the License.
  */
 
-import type { OperatorExecutionSummary, SampleRow } from "../../types/execution";
+import { OperatorResultMode, type OperatorExecutionSummary, type SampleRow } from "../../types/execution";
 import { formatExecuteOperatorResult, getOperatorWarnings, getVisibleResultHeaders } from "./tools-utility";
 
 export function formatOperatorResult(operatorId: string, opInfo: OperatorExecutionSummary): string {
@@ -31,15 +31,11 @@ export function formatOperatorResult(operatorId: string, opInfo: OperatorExecuti
     return "(no result data)";
   }
 
-  const headers = sampleTuples.length > 0 ? getVisibleResultHeaders(sampleTuples[0].tuple) : [];
-  const columns = headers.length;
-
-  const isViz = sampleTuples.length > 0 && sampleTuples[0].tuple["__is_visualization__"] === true;
+  const isViz = opInfo.resultSummary?.resultMode === OperatorResultMode.VISUALIZATION;
   const rows: SampleRow[] = isViz
     ? sampleTuples.map(({ rowIndex, tuple }) => {
         const cleaned: Record<string, any> = {};
         for (const key of Object.keys(tuple)) {
-          if (key === "__is_visualization__") continue;
           if (key === "html-content" || key === "json-content") {
             cleaned[key] = "<skipped: visualization content>";
           } else {
@@ -50,6 +46,9 @@ export function formatOperatorResult(operatorId: string, opInfo: OperatorExecuti
       })
     : sampleTuples;
 
+  const headers = rows.length > 0 ? getVisibleResultHeaders(rows[0].tuple) : [];
+  const columns = headers.length;
+
   const dataString = jsonToTableFormat(rows);
 
   // Output shape only; input-port shapes are derivable by the agent from the DAG

agent-service/src/agent/tools/tools-utility.spec.ts

@@ -29,37 +29,17 @@ import {
 } from "./tools-utility";
 
 describe("getVisibleResultHeaders", () => {
-  test("returns every key when no internal columns are present", () => {
+  test("returns every key", () => {
     expect(getVisibleResultHeaders({ a: 1, b: 2 })).toEqual(["a", "b"]);
   });
 
-  test("strips __row_index__ from the result", () => {
-    expect(getVisibleResultHeaders({ __row_index__: 0, a: 1 })).toEqual(["a"]);
-  });
-
-  test("strips __is_visualization__ from the result", () => {
-    expect(getVisibleResultHeaders({ __is_visualization__: true, a: 1 })).toEqual(["a"]);
-  });
-
-  test("strips every known internal column at once", () => {
-    expect(getVisibleResultHeaders({ __row_index__: 0, __is_visualization__: true, a: 1, b: 2 })).toEqual(["a", "b"]);
-  });
-
   test("preserves visible column order", () => {
-    expect(getVisibleResultHeaders({ z: 1, __row_index__: 0, a: 2, __is_visualization__: true, m: 3 })).toEqual([
-      "z",
-      "a",
-      "m",
-    ]);
+    expect(getVisibleResultHeaders({ z: 1, a: 2, m: 3 })).toEqual(["z", "a", "m"]);
   });
 
   test("returns an empty array for an empty row", () => {
     expect(getVisibleResultHeaders({})).toEqual([]);
   });
-
-  test("returns an empty array when only internal columns are present", () => {
-    expect(getVisibleResultHeaders({ __row_index__: 0, __is_visualization__: true })).toEqual([]);
-  });
 });
 
 describe("createToolResult", () => {

agent-service/src/agent/tools/tools-utility.ts

@@ -19,10 +19,8 @@
 
 import type { OperatorExecutionSummary } from "../../types/execution";
 
-export const INTERNAL_RESULT_KEYS: ReadonlySet<string> = new Set(["__row_index__", "__is_visualization__"]);
-
 export function getVisibleResultHeaders(row: Record<string, any>): string[] {
-  return Object.keys(row).filter(k => !INTERNAL_RESULT_KEYS.has(k));
+  return Object.keys(row);
 }
 
 // Warnings are the console messages the engine tags with a "WARNING: " title

agent-service/src/types/execution.ts

@@ -85,8 +85,7 @@ export interface ConsoleMessage {
   message: string;
 }
 
-// One sampled output row: its original position plus the row's columns. (A viz
-// payload's tuple still carries an `__is_visualization__` marker.)
+// One sampled output row: its original position plus the row's columns.
 export interface SampleRow {
   rowIndex: number;
   tuple: Record<string, unknown>;

amber/src/main/scala/org/apache/texera/web/resource/SyncExecutionResource.scala

@@ -551,7 +551,6 @@ class SyncExecutionResource extends LazyLogging {
           if (totalCount == 1 && isVisualizationTuple(firstTuple)) {
             val jsonResults =
               ExecutionResultService.convertTuplesToJson(List(firstTuple), isVisualization = true)
-            jsonResults.foreach(_.put("__is_visualization__", true))
             val rows = jsonResults.zipWithIndex.map { case (json, idx) => SampleRow(idx, json) }
             return ("visualization", Some(rows), Some(totalCount), Some(1), Some(false))
           }

frontend/src/app/workspace/component/agent/agent-interaction/agent-interaction.component.ts

@@ -40,7 +40,7 @@ import { NzTooltipDirective } from "ng-zorro-antd/tooltip";
 import { NzInputDirective, NzAutosizeDirective } from "ng-zorro-antd/input";
 import { NzButtonComponent } from "ng-zorro-antd/button";
 import { NzWaveDirective } from "ng-zorro-antd/core/wave";
-import { SampleRow } from "../../../service/agent/agent.service";
+import { OperatorResultMode, SampleRow } from "../../../service/agent/agent.service";
 
 /**
  * AgentInteractionComponent provides a compact interface for users to send feedback
@@ -77,6 +77,7 @@ export class AgentInteractionComponent implements OnInit, OnChanges {
   @Input() operatorId!: string;
   @Input() operatorDisplayName?: string;
   @Input() sampleTuples?: SampleRow[];
+  @Input() resultMode?: OperatorResultMode;
 
   public availableAgents: Array<{ id: string; name: string; isConnected: boolean }> = [];
   public selectedAgentId: string | null = null;
@@ -102,10 +103,9 @@ export class AgentInteractionComponent implements OnInit, OnChanges {
   }
 
   ngOnChanges(changes: SimpleChanges): void {
-    if (changes["sampleTuples"]) {
+    if (changes["sampleTuples"] || changes["resultMode"]) {
       // Only update cached visualization HTML when the actual content changes
-      const newRows = changes["sampleTuples"].currentValue as SampleRow[] | undefined;
-      const htmlContent = newRows?.[0]?.tuple["html-content"];
+      const htmlContent = this.sampleTuples?.[0]?.tuple["html-content"];
       const newHtml = typeof htmlContent === "string" ? htmlContent : null;
       if (newHtml !== this.cachedVisualizationRawHtml) {
         this.cachedVisualizationRawHtml = newHtml;
@@ -177,12 +177,8 @@ export class AgentInteractionComponent implements OnInit, OnChanges {
     return !!this.selectedAgentId && !!this.feedbackMessage.trim();
   }
 
-  /**
-   * Check if sample rows represent a visualization (has __is_visualization__ flag).
-   */
   public isVisualization(): boolean {
-    if (!this.sampleTuples || this.sampleTuples.length === 0) return false;
-    return this.sampleTuples[0].tuple["__is_visualization__"] === true;
+    return this.resultMode === OperatorResultMode.VISUALIZATION;
   }
 
   /**
@@ -197,7 +193,7 @@ export class AgentInteractionComponent implements OnInit, OnChanges {
    */
   public getSampleColumns(): string[] {
     if (!this.sampleTuples || this.sampleTuples.length === 0) return [];
-    return Object.keys(this.sampleTuples[0].tuple).filter(k => k !== "__is_visualization__");
+    return Object.keys(this.sampleTuples[0].tuple);
   }
 
   /**

frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.html

@@ -40,7 +40,8 @@
         <texera-agent-interaction
           [operatorId]="chatPopoverOperator.operatorId"
           [operatorDisplayName]="chatPopoverOperator.displayName"
-          [sampleTuples]="getOperatorSampleTuples(chatPopoverOperator.operatorId)">
+          [sampleTuples]="getOperatorSampleTuples(chatPopoverOperator.operatorId)"
+          [resultMode]="getOperatorResultMode(chatPopoverOperator.operatorId)">
         </texera-agent-interaction>
       </div>
     </div>

frontend/src/app/workspace/component/workflow-editor/workflow-editor.component.ts

@@ -43,7 +43,12 @@ import { isDefined } from "../../../common/util/predicate";
 import { GuiConfigService } from "../../../common/service/gui-config.service";
 import { line, curveCatmullRomClosed } from "d3-shape";
 import concaveman from "concaveman";
-import { AgentService, OperatorExecutionSummary, SampleRow } from "../../service/agent/agent.service";
+import {
+  AgentService,
+  OperatorExecutionSummary,
+  OperatorResultMode,
+  SampleRow,
+} from "../../service/agent/agent.service";
 import { NzNoAnimationDirective } from "ng-zorro-antd/core/animation";
 import { ContextMenuComponent } from "./context-menu/context-menu/context-menu.component";
 import { NgIf } from "@angular/common";
@@ -1700,10 +1705,12 @@ export class WorkflowEditorComponent implements OnInit, AfterViewInit, OnDestroy
     return this.operatorSummaries.get(operatorId)?.resultSummary?.sampleTuples;
   }
 
+  getOperatorResultMode(operatorId: string): OperatorResultMode | undefined {
+    return this.operatorSummaries.get(operatorId)?.resultSummary?.resultMode;
+  }
+
   isOperatorVisualization(operatorId: string): boolean {
-    return (
-      this.operatorSummaries.get(operatorId)?.resultSummary?.sampleTuples?.[0]?.tuple["__is_visualization__"] === true
-    );
+    return this.getOperatorResultMode(operatorId) === OperatorResultMode.VISUALIZATION;
   }
 
   /**