Skip to content

Commit 30efd71

Browse files
davidwzhaocomnik
davidwzhao
and
comnik
authored
Export CSV config revamp to support table exports (#199)
* Update ExportCSVConfig to support generic CSV sources * Update generated proto files * Update grammar * Try using Any types * Add partition size to csv config * Use separate options for legacy vs new csv export config * Update generated proto and parser * Update partition_size to partition_size_mb * Update grammar to distinguish between new and legacy ExportCSVConfig versions * Update pretty printer generator to support field unwrapping under conditionals * Update generated sdks * Format * Add tests for export_csv_config_v2 * Fix type error * Update generated parser * Go back to having separate grammar rule for export_csv_columns * Update generated parsers and printers * Update Go pretty printer * Update pretty snapshots * Add csv_export attr to test * No need for ignoring validator * Update snaps * Update go snaps * Update grammar * Fix grammar * Add debug snaps for export tests * Add return type annotation * Apply suggestion from @comnik Co-authored-by: Nikolas Göbel <nikolas.goebel@relational.ai> * Reorder csv config options for clarity --------- Co-authored-by: Nikolas Göbel <nikolas.goebel@relational.ai>
1 parent 5e27319 commit 30efd71

27 files changed

Lines changed: 12615 additions & 10564 deletions

‎meta/src/meta/grammar.y‎

Lines changed: 39 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,10 @@
9999
%nonterm exists logic.Exists
100100
%nonterm export transactions.Export
101101
%nonterm export_csv_column transactions.ExportCSVColumn
102-
%nonterm export_csv_columns Sequence[transactions.ExportCSVColumn]
102+
%nonterm export_csv_columns_list Sequence[transactions.ExportCSVColumn]
103103
%nonterm export_csv_config transactions.ExportCSVConfig
104104
%nonterm export_csv_path String
105+
%nonterm export_csv_source transactions.ExportCSVSource
105106
%nonterm false logic.Disjunction
106107
%nonterm ffi logic.FFI
107108
%nonterm ffi_args Sequence[logic.Abstraction]
@@ -178,6 +179,7 @@
178179
%validator_ignore_completeness DecimalValue
179180
%validator_ignore_completeness BeTreeLocator
180181
%validator_ignore_completeness BeTreeConfig
182+
%validator_ignore_completeness ExportCSVColumns
181183

182184
%%
183185

@@ -1079,17 +1081,23 @@ export
10791081
deconstruct: $3: transactions.ExportCSVConfig = $$.csv_config
10801082

10811083
export_csv_config
1082-
: "(" "export_csv_config" export_csv_path export_csv_columns config_dict ")"
1083-
construct: $$ = export_csv_config($3, $4, $5)
1084-
deconstruct:
1084+
: "(" "export_csv_config_v2" export_csv_path export_csv_source csv_config ")"
1085+
construct: $$ = construct_export_csv_config_with_source($3, $4, $5)
1086+
deconstruct if builtin.length($$.data_columns) == 0:
1087+
$3: String = $$.path
1088+
$4: transactions.ExportCSVSource = $$.csv_source
1089+
$5: logic.CSVConfig = $$.csv_config
1090+
| "(" "export_csv_config" export_csv_path export_csv_columns_list config_dict ")"
1091+
construct: $$ = construct_export_csv_config($3, $4, $5)
1092+
deconstruct if builtin.length($$.data_columns) != 0:
10851093
$3: String = $$.path
10861094
$4: Sequence[transactions.ExportCSVColumn] = $$.data_columns
10871095
$5: Sequence[Tuple[String, logic.Value]] = deconstruct_export_csv_config($$)
10881096

10891097
export_csv_path
10901098
: "(" "path" STRING ")"
10911099

1092-
export_csv_columns
1100+
export_csv_columns_list
10931101
: "(" "columns" export_csv_column* ")"
10941102

10951103
export_csv_column
@@ -1099,6 +1107,16 @@ export_csv_column
10991107
$3: String = $$.column_name
11001108
$4: logic.RelationId = $$.column_data
11011109

1110+
export_csv_source
1111+
: "(" "gnf_columns" export_csv_column* ")"
1112+
construct: $$ = transactions.ExportCSVSource(gnf_columns=transactions.ExportCSVColumns(columns=$3))
1113+
deconstruct if builtin.has_proto_field($$, 'gnf_columns'):
1114+
$3: Sequence[transactions.ExportCSVColumn] = $$.gnf_columns.columns
1115+
| "(" "table_def" relation_id ")"
1116+
construct: $$ = transactions.ExportCSVSource(table_def=$3)
1117+
deconstruct if builtin.has_proto_field($$, 'table_def'):
1118+
$3: logic.RelationId = $$.table_def
1119+
11021120

11031121
%%
11041122

@@ -1198,6 +1216,7 @@ def construct_csv_config(config_dict: Sequence[Tuple[String, logic.Value]]) -> l
11981216
decimal_separator: str = _extract_value_string(builtin.dict_get(config, "csv_decimal_separator"), ".")
11991217
encoding: str = _extract_value_string(builtin.dict_get(config, "csv_encoding"), "utf-8")
12001218
compression: str = _extract_value_string(builtin.dict_get(config, "csv_compression"), "auto")
1219+
partition_size_mb: int = _extract_value_int64(builtin.dict_get(config, "csv_partition_size_mb"), 0)
12011220
return logic.CSVConfig(
12021221
header_row=header_row,
12031222
skip=skip,
@@ -1210,6 +1229,7 @@ def construct_csv_config(config_dict: Sequence[Tuple[String, logic.Value]]) -> l
12101229
decimal_separator=decimal_separator,
12111230
encoding=encoding,
12121231
compression=compression,
1232+
partition_size_mb=partition_size_mb,
12131233
)
12141234

12151235

@@ -1275,8 +1295,7 @@ def construct_configure(config_dict: Sequence[Tuple[String, logic.Value]]) -> tr
12751295
ivm_config=ivm_config,
12761296
)
12771297

1278-
1279-
def export_csv_config(
1298+
def construct_export_csv_config(
12801299
path: String,
12811300
columns: Sequence[transactions.ExportCSVColumn],
12821301
config_dict: Sequence[Tuple[String, logic.Value]],
@@ -1301,6 +1320,17 @@ def export_csv_config(
13011320
syntax_escapechar=builtin.some(syntax_escapechar),
13021321
)
13031322

1323+
def construct_export_csv_config_with_source(
1324+
path: String,
1325+
csv_source: transactions.ExportCSVSource,
1326+
csv_config: logic.CSVConfig,
1327+
) -> transactions.ExportCSVConfig:
1328+
return transactions.ExportCSVConfig(
1329+
path=path,
1330+
csv_source=csv_source,
1331+
csv_config=csv_config,
1332+
)
1333+
13041334

13051335
def _make_value_int32(v: Int32) -> logic.Value:
13061336
return logic.Value(int_value=builtin.int32_to_int64(v))
@@ -1362,6 +1392,8 @@ def deconstruct_csv_config(msg: logic.CSVConfig) -> List[Tuple[String, logic.Val
13621392
builtin.list_push(result, builtin.tuple("csv_decimal_separator", _make_value_string(msg.decimal_separator)))
13631393
builtin.list_push(result, builtin.tuple("csv_encoding", _make_value_string(msg.encoding)))
13641394
builtin.list_push(result, builtin.tuple("csv_compression", _make_value_string(msg.compression)))
1395+
if msg.partition_size_mb != 0:
1396+
builtin.list_push(result, builtin.tuple("csv_partition_size_mb", _make_value_int64(msg.partition_size_mb)))
13651397
return builtin.list_sort(result)
13661398

13671399

‎proto/relationalai/lqp/v1/logic.proto‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -309,6 +309,9 @@ message CSVConfig {
309309

310310
// Compression
311311
string compression = 11; // "none", "gzip", "zstd", "auto" (default: "auto")
312+
313+
// Partitioning (for export)
314+
int64 partition_size_mb = 12;
312315
}
313316

314317
message GNFColumn {

‎proto/relationalai/lqp/v1/transactions.proto‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,11 @@ message Snapshot {
8282

8383
message ExportCSVConfig {
8484
string path = 1;
85+
86+
ExportCSVSource csv_source = 10;
87+
CSVConfig csv_config = 11;
88+
89+
// Below are all deprecated in favour of the `csv_config` above.
8590
repeated ExportCSVColumn data_columns = 2;
8691

8792
optional int64 partition_size = 3;
@@ -100,6 +105,17 @@ message ExportCSVColumn {
100105
RelationId column_data = 2;
101106
}
102107

108+
message ExportCSVColumns {
109+
repeated ExportCSVColumn columns = 1;
110+
}
111+
112+
message ExportCSVSource {
113+
oneof csv_source {
114+
ExportCSVColumns gnf_columns = 1;
115+
RelationId table_def = 2;
116+
}
117+
}
118+
103119
//
104120
// Read operations
105121
//

0 commit comments

Comments
 (0)