Skip to content

Commit 1e56096

Browse files
hbarthelsclaude
andcommitted
Add full_table mode for Iceberg loading (single-relation output)
Adds an alternative `(full_table :name [T1 T2 ...])` syntax to `iceberg_data`, enabling all columns to be loaded into a single relation keyed by the row ID (UInt128). Previously only the per-column `(columns (column ...) ...)` mode was supported. Proto: adds `IcebergTarget` message and optional `target` field (field 7) to `IcebergData`. Grammar: adds `full_table` nonterminal and relaxes `iceberg_data` to accept `gnf_columns? full_table?` (exactly one must be present at runtime). All SDK parsers/printers regenerated; new round-trip test added. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent 88009c6 commit 1e56096

21 files changed

Lines changed: 14515 additions & 14056 deletions

File tree

meta/src/meta/grammar.y

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@
123123
%nonterm iceberg_catalog_config_scope String
124124
%nonterm iceberg_catalog_uri String
125125
%nonterm iceberg_data logic.IcebergData
126+
%nonterm full_table logic.IcebergTarget
126127
%nonterm iceberg_from_snapshot String
127128
%nonterm iceberg_locator logic.IcebergLocator
128129
%nonterm iceberg_locator_namespace Sequence[String]
@@ -1225,16 +1226,24 @@ iceberg_to_snapshot
12251226
construct: $$ = $3
12261227
deconstruct: $3: String = $$
12271228

1229+
full_table
1230+
: "(" "full_table" relation_id "[" type* "]" ")"
1231+
construct: $$ = logic.IcebergTarget(target_id=$3, types=$5)
1232+
deconstruct:
1233+
$3: logic.RelationId = $$.target_id
1234+
$5: Sequence[logic.Type] = $$.types
1235+
12281236
iceberg_data
1229-
: "(" "iceberg_data" iceberg_locator iceberg_catalog_config gnf_columns iceberg_from_snapshot? iceberg_to_snapshot? boolean_value ")"
1230-
construct: $$ = construct_iceberg_data($3, $4, $5, $6, $7, $8)
1237+
: "(" "iceberg_data" iceberg_locator iceberg_catalog_config gnf_columns? full_table? iceberg_from_snapshot? iceberg_to_snapshot? boolean_value ")"
1238+
construct: $$ = construct_iceberg_data($3, $4, $5, $6, $7, $8, $9)
12311239
deconstruct:
12321240
$3: logic.IcebergLocator = $$.locator
12331241
$4: logic.IcebergCatalogConfig = $$.config
1234-
$5: Sequence[logic.GNFColumn] = $$.columns
1235-
$6: Optional[String] = deconstruct_iceberg_data_from_snapshot_optional($$)
1236-
$7: Optional[String] = deconstruct_iceberg_data_to_snapshot_optional($$)
1237-
$8: Boolean = $$.returns_delta
1242+
$5: Optional[Sequence[logic.GNFColumn]] = $$.columns if not builtin.is_empty($$.columns) else None
1243+
$6: Optional[logic.IcebergTarget] = $$.target if builtin.has_proto_field($$, 'target') else None
1244+
$7: Optional[String] = deconstruct_iceberg_data_from_snapshot_optional($$)
1245+
$8: Optional[String] = deconstruct_iceberg_data_to_snapshot_optional($$)
1246+
$9: Boolean = $$.returns_delta
12381247

12391248
undefine
12401249
: "(" "undefine" fragment_id ")"
@@ -1724,15 +1733,17 @@ def deconstruct_iceberg_catalog_config_scope_optional(msg: logic.IcebergCatalogC
17241733
def construct_iceberg_data(
17251734
locator: logic.IcebergLocator,
17261735
config: logic.IcebergCatalogConfig,
1727-
columns: Sequence[logic.GNFColumn],
1736+
columns_opt: Optional[Sequence[logic.GNFColumn]],
1737+
target_opt: Optional[logic.IcebergTarget],
17281738
from_snapshot_opt: Optional[String],
17291739
to_snapshot_opt: Optional[String],
17301740
returns_delta: Boolean,
17311741
) -> logic.IcebergData:
17321742
return logic.IcebergData(
17331743
locator=locator,
17341744
config=config,
1735-
columns=columns,
1745+
columns=builtin.unwrap_option_or(columns_opt, list[logic.GNFColumn]()),
1746+
target=target_opt,
17361747
from_snapshot=builtin.some(builtin.unwrap_option_or(from_snapshot_opt, "")),
17371748
to_snapshot=builtin.some(builtin.unwrap_option_or(to_snapshot_opt, "")),
17381749
returns_delta=returns_delta,

proto/relationalai/lqp/v1/logic.proto

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -316,13 +316,19 @@ message CSVConfig {
316316
int64 partition_size_mb = 12;
317317
}
318318

319+
message IcebergTarget {
320+
RelationId target_id = 1; // RelPath for output relation
321+
repeated Type types = 2; // row ID type + all column types, in order
322+
}
323+
319324
message IcebergData {
320325
IcebergLocator locator = 1;
321326
IcebergCatalogConfig config = 2;
322327
repeated GNFColumn columns = 3;
323328
optional string from_snapshot = 4;
324329
optional string to_snapshot = 5;
325330
bool returns_delta = 6;
331+
optional IcebergTarget target = 7; // if present, single-relation (full_table) mode
326332
}
327333

328334
message IcebergLocator {

0 commit comments

Comments
 (0)