update proto to make it possible to pass table_def vs gnf_def. Distinction is needed because GNF contain key that is not part of the exported iceberg

Author: vustef

proto/relationalai/lqp/v1/transactions.proto

@@ -121,18 +121,25 @@ message ExportCSVSource {
 message ExportIcebergConfig {
   IcebergLocator locator = 1;
   IcebergCatalogConfig config = 2;
-  repeated ExportIcebergColumn columns = 3;
+  ExportIcebergColumns columns = 3;
   optional string prefix = 4;                    // Default: "data"
   optional int64 target_file_size_bytes = 5;     // Default: 512 MB
   string compression = 6;                        // UNCOMPRESSED, SNAPPY, GZIP, LZ4, ZSTD
-  map<string, string> create_table_properties = 7;            // properties for table creation
+  map<string, string> table_properties = 7;      // properties for table creation
 }
 
 message ExportIcebergColumn {
   string name = 1;
-  RelationId column_data = 2;
-  Type type = 3;
-  bool nullable = 4;
+  Type type = 2;
+  bool nullable = 3;
+}
+
+message ExportIcebergColumns {
+  oneof iceberg_columns {
+    repeated RelationId source_gnf_defs = 1;
+    RelationId source_table_def = 2;
+  }
+  repeated ExportIcebergColumn target_columns = 3;
 }
 
 //

tests/lqp/iceberg_export.lqp

@@ -9,7 +9,7 @@
                 (and (= id 1) (= region "us-east") (= amount 100.0))
                 (and (= id 2) (= region "eu-west") (= amount 250.5))))))))
     (reads
-      ;; Minimal export
+      ;; Minimal export with gnf defs as column source
       (export_iceberg
         (export_iceberg_config
           (iceberg_locator
@@ -21,12 +21,14 @@
             (properties)
             (auth_properties))
           (columns
-            (iceberg_column "id" :id INT false)
-            (iceberg_column "region" :region STRING false)
-            (iceberg_column "amount" :amount FLOAT false))
-          (create_table_properties)))
+            (source_gnf_defs :id :region :amount)
+            (target_columns
+              (iceberg_column "id" INT false)
+              (iceberg_column "region" STRING false)
+              (iceberg_column "amount" FLOAT false)))
+          (table_properties)))
 
-      ;; Full export with scope, property maps, and trailing export options
+      ;; Full export with scope, property maps, table source, and trailing export options
       (export_iceberg
         (export_iceberg_config
           (iceberg_locator
@@ -41,10 +43,12 @@
               (prop "credential" "my-credential")
               (prop "token" "secret-token-123")))
           (columns
-            (iceberg_column "id" :id INT false)
-            (iceberg_column "region" :region STRING false)
-            (iceberg_column "amount" :amount FLOAT false))
-          (create_table_properties (prop "write.format.default" "parquet"))
+            (source_table_def :sales)
+            (target_columns
+              (iceberg_column "id" INT false)
+              (iceberg_column "region" STRING false)
+              (iceberg_column "amount" FLOAT false)))
+          (table_properties (prop "write.format.default" "parquet"))
           {:prefix "part" :target_file_size_bytes 268435456 :compression "SNAPPY"}))
 
       ;; Empty namespace on the locator
@@ -59,5 +63,7 @@
             (properties)
             (auth_properties))
           (columns
-            (iceberg_column "id" :id INT false))
-          (create_table_properties))))))
+            (source_gnf_defs :id)
+            (target_columns
+              (iceberg_column "id" INT false)))
+          (table_properties))))))