Remove columns from iceberg export

Author: gbrgr

meta/src/meta/grammar.y

@@ -116,8 +116,6 @@
 %nonterm export_csv_config transactions.ExportCSVConfig
 %nonterm export_csv_path String
 %nonterm export_csv_source transactions.ExportCSVSource
-%nonterm export_iceberg_column transactions.ExportColumn
-%nonterm export_iceberg_columns Sequence[transactions.ExportColumn]
 %nonterm export_iceberg_config transactions.ExportIcebergConfig
 %nonterm export_iceberg_table_def logic.RelationId
 %nonterm iceberg_auth_properties Sequence[Tuple[String, String]]
@@ -1222,13 +1220,6 @@ iceberg_catalog_config
         $5: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.properties)
         $6: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.auth_properties)
 
-export_iceberg_column
-    : "(" "column" STRING boolean_value ")"
-      construct: $$ = transactions.ExportColumn(name=$3, nullable=$4)
-      deconstruct:
-        $3: String = $$.name
-        $4: Boolean = $$.nullable
-
 iceberg_to_snapshot
     : "(" "to_snapshot" STRING ")"
       construct: $$ = $3
@@ -1372,26 +1363,20 @@ export_iceberg_table_def
       construct: $$ = $3
       deconstruct: $3: logic.RelationId = $$
 
-export_iceberg_columns
-    : "(" "columns" export_iceberg_column* ")"
-      construct: $$ = $3
-      deconstruct: $3: Sequence[transactions.ExportColumn] = $$
-
 iceberg_table_properties
     : "(" "table_properties" iceberg_property_entry* ")"
       construct: $$ = $3
       deconstruct: $3: Sequence[Tuple[String, String]] = $$
 
 export_iceberg_config
-    : "(" "export_iceberg_config" iceberg_locator iceberg_catalog_config export_iceberg_table_def export_iceberg_columns iceberg_table_properties config_dict? ")"
-      construct: $$ = construct_export_iceberg_config_full($3, $4, $5, $6, $7, $8)
+    : "(" "export_iceberg_config" iceberg_locator iceberg_catalog_config export_iceberg_table_def iceberg_table_properties config_dict? ")"
+      construct: $$ = construct_export_iceberg_config_full($3, $4, $5, $6, $7)
       deconstruct:
         $3: logic.IcebergLocator = $$.locator
         $4: logic.IcebergCatalogConfig = $$.config
         $5: logic.RelationId = $$.table_def
-        $6: Sequence[transactions.ExportColumn] = $$.columns
-        $7: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.table_properties)
-        $8: Optional[Sequence[Tuple[String, logic.Value]]] = deconstruct_export_iceberg_config_optional($$)
+        $6: Sequence[Tuple[String, String]] = builtin.dict_to_pairs($$.table_properties)
+        $7: Optional[Sequence[Tuple[String, logic.Value]]] = deconstruct_export_iceberg_config_optional($$)
 
 
 %%

proto/relationalai/lqp/v1/transactions.proto

@@ -122,16 +122,10 @@ message ExportCSVSource {
 
 // Iceberg Export config
 
-message ExportColumn {
-  string name = 1;
-  bool nullable = 2;
-}
-
 message ExportIcebergConfig {
   IcebergLocator locator = 1;
   IcebergCatalogConfig config = 2;
   RelationId table_def = 3;              // Which definition to export as a table.
-  repeated ExportColumn columns = 4;
   optional string prefix = 5;            // File name prefix for parquet files that are produced.
   optional int64 target_file_size_bytes = 6;
   string compression = 7;

sdks/julia/LogicalQueryProtocol.jl/Project.toml

@@ -1,6 +1,6 @@
 name = "LogicalQueryProtocol"
 uuid = "a92373ee-6cc4-4662-ae66-0c99a03ebae1"
-version = "0.5.1"
+version = "0.5.2"
 authors = ["RelationalAI"]
 
 [deps]

sdks/python/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lqp"
-version = "0.5.1"
+version = "0.5.2"
 description = "Validate and translate Logical Query Protocol (LQP) S-expressions into Protobuf"
 readme = "README.md"
 authors = [

tests/lqp/iceberg_export.lqp

@@ -21,10 +21,6 @@
             (properties)
             (auth_properties))
           (table_def :sales)
-          (columns
-            (column "id" false)
-            (column "region" false)
-            (column "amount" false))
           (table_properties)))
 
       ;; Full export with scope, property maps, and trailing export options
@@ -42,10 +38,6 @@
               (prop "credential" "***")
               (prop "token" "***")))
           (table_def :sales)
-          (columns
-            (column "id" false)
-            (column "region" false)
-            (column "amount" false))
           (table_properties (prop "write.format.default" "parquet"))
           {:prefix "part" :target_file_size_bytes 268435456 :compression "SNAPPY"}))
 
@@ -61,6 +53,4 @@
             (properties)
             (auth_properties))
           (table_def :sales)
-          (columns
-            (column "id" false))
           (table_properties))))))

tests/pretty/iceberg_export.lqp

@@ -23,7 +23,6 @@
             (properties)
             (auth_properties))
           (table_def :sales)
-          (columns (column "id" false) (column "region" false) (column "amount" false))
           (table_properties)))
       (export_iceberg
         (export_iceberg_config
@@ -37,7 +36,6 @@
             (properties (prop "client.region" "us-east-1"))
             (auth_properties (prop "credential" "***") (prop "token" "***")))
           (table_def :sales)
-          (columns (column "id" false) (column "region" false) (column "amount" false))
           (table_properties (prop "write.format.default" "parquet"))
           { :compression "SNAPPY" :prefix "part" :target_file_size_bytes 268435456}))
       (export_iceberg
@@ -51,5 +49,4 @@
             (properties)
             (auth_properties))
           (table_def :sales)
-          (columns (column "id" false))
           (table_properties))))))

tests/pretty_debug/iceberg_export.lqp

@@ -23,7 +23,6 @@
             (properties)
             (auth_properties))
           (table_def 0x585c5a1121bd88784c1378aa8837195c)
-          (columns (column "id" false) (column "region" false) (column "amount" false))
           (table_properties)))
       (export_iceberg
         (export_iceberg_config
@@ -37,7 +36,6 @@
             (properties (prop "client.region" "us-east-1"))
             (auth_properties (prop "credential" "***") (prop "token" "***")))
           (table_def 0x585c5a1121bd88784c1378aa8837195c)
-          (columns (column "id" false) (column "region" false) (column "amount" false))
           (table_properties (prop "write.format.default" "parquet"))
           { :compression "SNAPPY" :prefix "part" :target_file_size_bytes 268435456}))
       (export_iceberg
@@ -51,7 +49,6 @@
             (properties)
             (auth_properties))
           (table_def 0x585c5a1121bd88784c1378aa8837195c)
-          (columns (column "id" false))
           (table_properties))))))
 
 ;; Debug information