Merge remote-tracking branch 'origin/main' into nn-meta-sha256

Author: nystrom

README.md

@@ -82,13 +82,13 @@ package to PyPI. To release a new version:
    automatically determined by the release tag in GitHub.
 3. Get approval and merge into `main`.
 4. Create a new GitHub release from `main` with a tag matching the version (e.g.
-   `gh release create v0.3.0 --title "v0.3.0" --generate-notes`). The release must be
+   `gh release create v0.4.0 --title "v0.4.0" --generate-notes`). The release must be
    created after merging the version bump, since the workflow checks out the default branch.
 5. Tag the release for the Go submodule. Go requires a separate tag with the subdirectory
    prefix for modules that live outside the repository root:
    ```bash
-   git tag sdks/go/v0.3.0 v0.3.0
-   git push origin sdks/go/v0.3.0
+   git tag sdks/go/v0.4.0 v0.4.0
+   git push origin sdks/go/v0.4.0
    ```
    Without this tag, `go get` will not be able to resolve the module at the released version.
 

meta/src/meta/grammar.y

@@ -99,9 +99,10 @@
 %nonterm exists logic.Exists
 %nonterm export transactions.Export
 %nonterm export_csv_column transactions.ExportCSVColumn
-%nonterm export_csv_columns Sequence[transactions.ExportCSVColumn]
+%nonterm export_csv_columns_list Sequence[transactions.ExportCSVColumn]
 %nonterm export_csv_config transactions.ExportCSVConfig
 %nonterm export_csv_path String
+%nonterm export_csv_source transactions.ExportCSVSource
 %nonterm false logic.Disjunction
 %nonterm ffi logic.FFI
 %nonterm ffi_args Sequence[logic.Abstraction]
@@ -178,6 +179,7 @@
 %validator_ignore_completeness DecimalValue
 %validator_ignore_completeness BeTreeLocator
 %validator_ignore_completeness BeTreeConfig
+%validator_ignore_completeness ExportCSVColumns
 
 %%
 
@@ -1079,17 +1081,23 @@ export
       deconstruct: $3: transactions.ExportCSVConfig = $$.csv_config
 
 export_csv_config
-    : "(" "export_csv_config" export_csv_path export_csv_columns config_dict ")"
-      construct: $$ = export_csv_config($3, $4, $5)
-      deconstruct:
+    : "(" "export_csv_config_v2" export_csv_path export_csv_source csv_config ")"
+      construct: $$ = construct_export_csv_config_with_source($3, $4, $5)
+      deconstruct if builtin.length($$.data_columns) == 0:
+        $3: String = $$.path
+        $4: transactions.ExportCSVSource = $$.csv_source
+        $5: logic.CSVConfig = $$.csv_config
+    | "(" "export_csv_config" export_csv_path export_csv_columns_list config_dict ")"
+      construct: $$ = construct_export_csv_config($3, $4, $5)
+      deconstruct if builtin.length($$.data_columns) != 0:
         $3: String = $$.path
         $4: Sequence[transactions.ExportCSVColumn] = $$.data_columns
         $5: Sequence[Tuple[String, logic.Value]] = deconstruct_export_csv_config($$)
 
 export_csv_path
     : "(" "path" STRING ")"
 
-export_csv_columns
+export_csv_columns_list
     : "(" "columns" export_csv_column* ")"
 
 export_csv_column
@@ -1099,6 +1107,16 @@ export_csv_column
         $3: String = $$.column_name
         $4: logic.RelationId = $$.column_data
 
+export_csv_source
+    : "(" "gnf_columns" export_csv_column* ")"
+      construct: $$ = transactions.ExportCSVSource(gnf_columns=transactions.ExportCSVColumns(columns=$3))
+      deconstruct if builtin.has_proto_field($$, 'gnf_columns'):
+        $3: Sequence[transactions.ExportCSVColumn] = $$.gnf_columns.columns
+    | "(" "table_def" relation_id ")"
+      construct: $$ = transactions.ExportCSVSource(table_def=$3)
+      deconstruct if builtin.has_proto_field($$, 'table_def'):
+        $3: logic.RelationId = $$.table_def
+
 
 %%
 
@@ -1198,6 +1216,7 @@ def construct_csv_config(config_dict: Sequence[Tuple[String, logic.Value]]) -> l
     decimal_separator: str = _extract_value_string(builtin.dict_get(config, "csv_decimal_separator"), ".")
     encoding: str = _extract_value_string(builtin.dict_get(config, "csv_encoding"), "utf-8")
     compression: str = _extract_value_string(builtin.dict_get(config, "csv_compression"), "auto")
+    partition_size_mb: int = _extract_value_int64(builtin.dict_get(config, "csv_partition_size_mb"), 0)
     return logic.CSVConfig(
         header_row=header_row,
         skip=skip,
@@ -1210,6 +1229,7 @@ def construct_csv_config(config_dict: Sequence[Tuple[String, logic.Value]]) -> l
         decimal_separator=decimal_separator,
         encoding=encoding,
         compression=compression,
+        partition_size_mb=partition_size_mb,
     )
 
 
@@ -1275,8 +1295,7 @@ def construct_configure(config_dict: Sequence[Tuple[String, logic.Value]]) -> tr
         ivm_config=ivm_config,
     )
 
-
-def export_csv_config(
+def construct_export_csv_config(
     path: String,
     columns: Sequence[transactions.ExportCSVColumn],
     config_dict: Sequence[Tuple[String, logic.Value]],
@@ -1301,6 +1320,17 @@ def export_csv_config(
         syntax_escapechar=builtin.some(syntax_escapechar),
     )
 
+def construct_export_csv_config_with_source(
+    path: String,
+    csv_source: transactions.ExportCSVSource,
+    csv_config: logic.CSVConfig,
+) -> transactions.ExportCSVConfig:
+    return transactions.ExportCSVConfig(
+        path=path,
+        csv_source=csv_source,
+        csv_config=csv_config,
+    )
+
 
 def _make_value_int32(v: Int32) -> logic.Value:
     return logic.Value(int_value=builtin.int32_to_int64(v))
@@ -1362,6 +1392,8 @@ def deconstruct_csv_config(msg: logic.CSVConfig) -> List[Tuple[String, logic.Val
     builtin.list_push(result, builtin.tuple("csv_decimal_separator", _make_value_string(msg.decimal_separator)))
     builtin.list_push(result, builtin.tuple("csv_encoding", _make_value_string(msg.encoding)))
     builtin.list_push(result, builtin.tuple("csv_compression", _make_value_string(msg.compression)))
+    if msg.partition_size_mb != 0:
+      builtin.list_push(result, builtin.tuple("csv_partition_size_mb", _make_value_int64(msg.partition_size_mb)))
     return builtin.list_sort(result)
 
 

meta/src/meta/templates/parser.go.template

@@ -496,8 +496,8 @@ func toPascalCase(s string) string {{
 // --- Parse functions ---
 {parse_nonterminal_defns}
 
-// Parse parses the input string and returns the result
-func Parse(input string) (result *pb.Transaction, err error) {{
+// ParseTransaction parses the input string and returns a Transaction
+func ParseTransaction(input string) (result *pb.Transaction, err error) {{
 	defer func() {{
 		if r := recover(); r != nil {{
 			if pe, ok := r.(ParseError); ok {{
@@ -521,3 +521,34 @@ func Parse(input string) (result *pb.Transaction, err error) {{
 	}}
 	return result, nil
 }}
+
+// ParseFragment parses the input string and returns a Fragment
+func ParseFragment(input string) (result *pb.Fragment, err error) {{
+	defer func() {{
+		if r := recover(); r != nil {{
+			if pe, ok := r.(ParseError); ok {{
+				err = pe
+				return
+			}}
+			panic(r)
+		}}
+	}}()
+
+	lexer := NewLexer(input)
+	parser := NewParser(lexer.tokens)
+	result = parser.parse_fragment()
+
+	// Check for unconsumed tokens (except EOF)
+	if parser.pos < len(parser.tokens) {{
+		remainingToken := parser.lookahead(0)
+		if remainingToken.Type != "$" {{
+			return nil, ParseError{{msg: fmt.Sprintf("Unexpected token at end of input: %v", remainingToken)}}
+		}}
+	}}
+	return result, nil
+}}
+
+// Parse parses the input string and returns a Transaction
+func Parse(input string) (result *pb.Transaction, err error) {{
+	return ParseTransaction(input)
+}}

meta/src/meta/templates/parser.jl.template

@@ -280,7 +280,7 @@ end
 # --- Parse functions ---
 {parse_nonterminal_defns}
 
-function parse(input::String)
+function parse_transaction(input::String)
     lexer = Lexer(input)
     parser = ParserState(lexer.tokens)
     result = parse_{start_name}(parser)
@@ -294,8 +294,26 @@ function parse(input::String)
     return result
 end
 
-# Export main parse function and error type
-export parse, ParseError
+function parse_fragment(input::String)
+    lexer = Lexer(input)
+    parser = ParserState(lexer.tokens)
+    result = parse_fragment(parser)
+    # Check for unconsumed tokens (except EOF)
+    if parser.pos <= length(parser.tokens)
+        remaining_token = lookahead(parser, 0)
+        if remaining_token.type != "\$"
+            throw(ParseError("Unexpected token at end of input: $remaining_token"))
+        end
+    end
+    return result
+end
+
+function parse(input::String)
+    return parse_transaction(input)
+end
+
+# Export main parse functions and error type
+export parse, parse_transaction, parse_fragment, ParseError
 # Export scanner functions for testing
 export scan_string, scan_int, scan_float, scan_int128, scan_uint128, scan_decimal
 # Export Lexer for testing

meta/src/meta/templates/parser.py.template

@@ -273,8 +273,8 @@ class Parser:
     # --- Parse methods ---
 {parse_nonterminal_defns}
 
-def parse(input_str: str) -> Any:
-    """Parse input string and return parse tree."""
+def parse_transaction(input_str: str) -> Any:
+    """Parse input string and return a Transaction."""
     lexer = Lexer(input_str)
     parser = Parser(lexer.tokens)
     result = parser.parse_{start_name}()
@@ -284,3 +284,21 @@ def parse(input_str: str) -> Any:
         if remaining_token.type != "$":
             raise ParseError(f"Unexpected token at end of input: {{remaining_token}}")
     return result
+
+
+def parse_fragment(input_str: str) -> Any:
+    """Parse input string and return a Fragment."""
+    lexer = Lexer(input_str)
+    parser = Parser(lexer.tokens)
+    result = parser.parse_fragment()
+    # Check for unconsumed tokens (except EOF)
+    if parser.pos < len(parser.tokens):
+        remaining_token = parser.lookahead(0)
+        if remaining_token.type != "$":
+            raise ParseError(f"Unexpected token at end of input: {{remaining_token}}")
+    return result
+
+
+def parse(input_str: str) -> Any:
+    """Parse input string and return a Transaction."""
+    return parse_transaction(input_str)

proto/relationalai/lqp/v1/logic.proto

@@ -309,6 +309,9 @@ message CSVConfig {
 
   // Compression
   string compression = 11; // "none", "gzip", "zstd", "auto" (default: "auto")
+
+  // Partitioning (for export)
+  int64 partition_size_mb = 12;
 }
 
 message GNFColumn {

proto/relationalai/lqp/v1/transactions.proto

@@ -82,6 +82,11 @@ message Snapshot {
 
 message ExportCSVConfig {
   string path = 1;
+
+  ExportCSVSource csv_source = 10;
+  CSVConfig csv_config = 11;
+
+  // Below are all deprecated in favour of the `csv_config` above.
   repeated ExportCSVColumn data_columns = 2;
 
   optional int64 partition_size = 3;
@@ -100,6 +105,17 @@ message ExportCSVColumn {
   RelationId column_data = 2;
 }
 
+message ExportCSVColumns {
+  repeated ExportCSVColumn columns = 1;
+}
+
+message ExportCSVSource {
+  oneof csv_source {
+    ExportCSVColumns gnf_columns = 1;
+    RelationId table_def = 2;
+  }
+}
+
 //
 // Read operations
 //