-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathparser.py
More file actions
4033 lines (3787 loc) · 186 KB
/
Copy pathparser.py
File metadata and controls
4033 lines (3787 loc) · 186 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
Auto-generated LL(k) recursive-descent parser.
Generated from protobuf specifications.
Do not modify this file! If you need to modify the parser, edit the generator code
in `meta/` or edit the protobuf specification in `proto/v1`.
Command: python -m meta.cli ../proto/relationalai/lqp/v1/fragments.proto ../proto/relationalai/lqp/v1/logic.proto ../proto/relationalai/lqp/v1/transactions.proto --grammar src/meta/grammar.y --parser python
"""
import ast
import bisect
import hashlib
import re
from collections.abc import Sequence
from typing import Any
from lqp.proto.v1 import logic_pb2, fragments_pb2, transactions_pb2
class ParseError(Exception):
"""Parse error exception."""
pass
class Location:
"""Source location (1-based line and column, 0-based byte offset)."""
__slots__ = ("line", "column", "offset")
def __init__(self, line: int, column: int, offset: int):
self.line = line
self.column = column
self.offset = offset
def __repr__(self) -> str:
return f"Location({self.line}, {self.column}, {self.offset})"
def __eq__(self, other) -> bool:
if not isinstance(other, Location):
return NotImplemented
return self.line == other.line and self.column == other.column and self.offset == other.offset
def __hash__(self) -> int:
return hash((self.line, self.column, self.offset))
class Span:
"""Source span from start to stop location."""
__slots__ = ("start", "stop", "type_name")
def __init__(self, start: Location, stop: Location, type_name: str = ""):
self.start = start
self.stop = stop
self.type_name = type_name
def __repr__(self) -> str:
return f"Span({self.start}, {self.stop})"
def __eq__(self, other) -> bool:
if not isinstance(other, Span):
return NotImplemented
return self.start == other.start and self.stop == other.stop
def __hash__(self) -> int:
return hash((self.start, self.stop))
class Token:
"""Token representation."""
def __init__(self, type: str, value: str, start_pos: int, end_pos: int):
self.type = type
self.value = value
self.start_pos = start_pos
self.end_pos = end_pos
@property
def pos(self) -> int:
return self.start_pos
def __repr__(self) -> str:
return f"Token({self.type}, {self.value!r}, {self.start_pos})"
_WHITESPACE_RE = re.compile(r"\s+")
_COMMENT_RE = re.compile(r";;.*")
_TOKEN_SPECS = [
("LITERAL", re.compile(r"::"), lambda x: x),
("LITERAL", re.compile(r"<="), lambda x: x),
("LITERAL", re.compile(r">="), lambda x: x),
("LITERAL", re.compile(r"\#"), lambda x: x),
("LITERAL", re.compile(r"\("), lambda x: x),
("LITERAL", re.compile(r"\)"), lambda x: x),
("LITERAL", re.compile(r"\*"), lambda x: x),
("LITERAL", re.compile(r"\+"), lambda x: x),
("LITERAL", re.compile(r"\-"), lambda x: x),
("LITERAL", re.compile(r"/"), lambda x: x),
("LITERAL", re.compile(r":"), lambda x: x),
("LITERAL", re.compile(r"<"), lambda x: x),
("LITERAL", re.compile(r"="), lambda x: x),
("LITERAL", re.compile(r">"), lambda x: x),
("LITERAL", re.compile(r"\["), lambda x: x),
("LITERAL", re.compile(r"\]"), lambda x: x),
("LITERAL", re.compile(r"\{"), lambda x: x),
("LITERAL", re.compile(r"\|"), lambda x: x),
("LITERAL", re.compile(r"\}"), lambda x: x),
("DECIMAL", re.compile(r"[-]?\d+\.\d+d\d+"), lambda x: Lexer.scan_decimal(x)),
(
"FLOAT32",
re.compile(r"([-]?\d+\.\d+f32|inf32|nan32)"),
lambda x: Lexer.scan_float32(x),
),
("FLOAT", re.compile(r"([-]?\d+\.\d+|inf|nan)"), lambda x: Lexer.scan_float(x)),
("INT32", re.compile(r"[-]?\d+i32"), lambda x: Lexer.scan_int32(x)),
("INT", re.compile(r"[-]?\d+"), lambda x: Lexer.scan_int(x)),
("UINT32", re.compile(r"\d+u32"), lambda x: Lexer.scan_uint32(x)),
("INT128", re.compile(r"[-]?\d+i128"), lambda x: Lexer.scan_int128(x)),
("STRING", re.compile(r'"(?:[^"\\]|\\.)*"'), lambda x: Lexer.scan_string(x)),
(
"SYMBOL",
re.compile(r"[a-zA-Z_][a-zA-Z0-9_.#/-]*"),
lambda x: Lexer.scan_symbol(x),
),
("UINT128", re.compile(r"0x[0-9a-fA-F]+"), lambda x: Lexer.scan_uint128(x)),
]
class Lexer:
"""Tokenizer for the input."""
def __init__(self, input_str: str):
self.input = input_str
self.pos = 0
self.tokens: list[Token] = []
self._tokenize()
def _tokenize(self) -> None:
"""Tokenize the input string."""
while self.pos < len(self.input):
match = _WHITESPACE_RE.match(self.input, self.pos)
if match:
self.pos = match.end()
continue
match = _COMMENT_RE.match(self.input, self.pos)
if match:
self.pos = match.end()
continue
# Collect all matching tokens
candidates = []
for token_type, regex, action in _TOKEN_SPECS:
match = regex.match(self.input, self.pos)
if match:
value = match.group(0)
candidates.append((token_type, value, action, match.end()))
if not candidates:
raise ParseError(
f"Unexpected character at position {self.pos}: {self.input[self.pos]!r}"
)
# Pick the longest match
token_type, value, action, end_pos = max(candidates, key=lambda x: x[3])
self.tokens.append(Token(token_type, action(value), self.pos, end_pos))
self.pos = end_pos
self.tokens.append(Token("$", "", self.pos, self.pos))
@staticmethod
def scan_symbol(s: str) -> str:
"""Parse SYMBOL token."""
return s
@staticmethod
def scan_string(s: str) -> str:
"""Parse STRING token."""
return ast.literal_eval(s)
@staticmethod
def scan_int(n: str) -> int:
"""Parse INT token."""
val = int(n)
if val < -(1 << 63) or val >= (1 << 63):
raise ParseError(f"Integer literal out of 64-bit range: {n}")
return val
@staticmethod
def scan_int32(n: str) -> int:
"""Parse INT32 token."""
n = n[:-3] # Remove "i32" suffix
val = int(n)
if val < -(1 << 31) or val >= (1 << 31):
raise ParseError(f"Int32 literal out of range: {n}")
return val
@staticmethod
def scan_uint32(n: str) -> int:
"""Parse UINT32 token."""
n = n[:-3] # Remove "u32" suffix
val = int(n)
if val < 0 or val >= (1 << 32):
raise ParseError(f"UInt32 literal out of range: {n}")
return val
@staticmethod
def scan_float32(f: str) -> float:
"""Parse FLOAT32 token."""
if f == "inf32":
return float("inf")
elif f == "nan32":
return float("nan")
f = f[:-3] # Remove "f32" suffix
return float(f)
@staticmethod
def scan_float(f: str) -> float:
"""Parse FLOAT token."""
if f == "inf":
return float("inf")
elif f == "nan":
return float("nan")
return float(f)
@staticmethod
def scan_uint128(u: str) -> Any:
"""Parse UINT128 token."""
uint128_val = int(u, 16)
if uint128_val < 0 or uint128_val >= (1 << 128):
raise ParseError(f"UInt128 literal out of range: {u}")
low = uint128_val & 0xFFFFFFFFFFFFFFFF
high = (uint128_val >> 64) & 0xFFFFFFFFFFFFFFFF
return logic_pb2.UInt128Value(low=low, high=high)
@staticmethod
def scan_int128(u: str) -> Any:
"""Parse INT128 token."""
u = u[:-4] # Remove the "i128" suffix
int128_val = int(u)
if int128_val < -(1 << 127) or int128_val >= (1 << 127):
raise ParseError(f"Int128 literal out of range: {u}")
low = int128_val & 0xFFFFFFFFFFFFFFFF
high = (int128_val >> 64) & 0xFFFFFFFFFFFFFFFF
return logic_pb2.Int128Value(low=low, high=high)
@staticmethod
def scan_decimal(d: str) -> Any:
"""Parse DECIMAL token."""
# Decimal is a string like "123.456d12" where the last part after `d` is the
# precision, and the scale is the number of digits between the decimal point and `d`
parts = d.split("d")
if len(parts) != 2:
raise ValueError(f"Invalid decimal format: {d}")
scale = len(parts[0].split(".")[1])
precision = int(parts[1])
# Parse the integer value directly without calling scan_int128 which strips "i128" suffix
int_str = parts[0].replace(".", "")
int128_val = int(int_str)
low = int128_val & 0xFFFFFFFFFFFFFFFF
high = (int128_val >> 64) & 0xFFFFFFFFFFFFFFFF
value = logic_pb2.Int128Value(low=low, high=high)
return logic_pb2.DecimalValue(precision=precision, scale=scale, value=value)
def _compute_line_starts(text: str) -> list[int]:
"""Compute byte offsets where each line starts (0-based)."""
starts = [0]
for i, ch in enumerate(text):
if ch == '\n':
starts.append(i + 1)
return starts
class Parser:
"""LL(k) recursive-descent parser with backtracking."""
def __init__(self, tokens: list[Token], input_str: str):
self.tokens = tokens
self.pos = 0
self.id_to_debuginfo = {}
self._current_fragment_id: bytes | None = None
self._relation_id_to_name = {}
self.provenance: dict[int, Span] = {}
self._line_starts = _compute_line_starts(input_str)
def _make_location(self, offset: int) -> Location:
"""Convert byte offset to Location with 1-based line/column."""
line_idx = bisect.bisect_right(self._line_starts, offset) - 1
col = offset - self._line_starts[line_idx]
return Location(line_idx + 1, col + 1, offset)
def span_start(self) -> int:
"""Return the start offset of the current token."""
return self.lookahead(0).start_pos
def record_span(self, start_offset: int, type_name: str = "") -> None:
"""Record a span from start_offset to the previous token's end.
Uses first-wins semantics: the innermost parse function records first,
and outer wrappers that share the same offset do not overwrite.
"""
if start_offset in self.provenance:
return
if self.pos > 0:
end_offset = self.tokens[self.pos - 1].end_pos
else:
end_offset = start_offset
span = Span(self._make_location(start_offset), self._make_location(end_offset), type_name)
self.provenance[start_offset] = span
def lookahead(self, k: int = 0) -> Token:
"""Get lookahead token at offset k."""
idx = self.pos + k
return self.tokens[idx] if idx < len(self.tokens) else Token("$", "", -1, -1)
def consume_literal(self, expected: str) -> None:
"""Consume a literal token."""
if not self.match_lookahead_literal(expected, 0):
token = self.lookahead(0)
raise ParseError(
f"Expected literal {expected!r} but got {token.type}=`{token.value!r}` at position {token.pos}"
)
self.pos += 1
def consume_terminal(self, expected: str) -> Any:
"""Consume a terminal token and return parsed value."""
if not self.match_lookahead_terminal(expected, 0):
token = self.lookahead(0)
raise ParseError(
f"Expected terminal {expected} but got {token.type}=`{token.value!r}` at position {token.pos}"
)
token = self.lookahead(0)
self.pos += 1
return token.value
def match_lookahead_literal(self, literal: str, k: int) -> bool:
"""Check if lookahead token at position k matches literal.
Supports soft keywords: alphanumeric literals are lexed as SYMBOL tokens,
so we check both LITERAL and SYMBOL token types.
"""
token = self.lookahead(k)
if token.type == "LITERAL" and token.value == literal:
return True
if token.type == "SYMBOL" and token.value == literal:
return True
return False
def match_lookahead_terminal(self, terminal: str, k: int) -> bool:
"""Check if lookahead token at position k matches terminal."""
token = self.lookahead(k)
return token.type == terminal
def start_fragment(
self, fragment_id: fragments_pb2.FragmentId
) -> fragments_pb2.FragmentId:
"""Set current fragment ID for debug info tracking."""
self._current_fragment_id = fragment_id.id
return fragment_id
def relation_id_from_string(self, name: str) -> Any:
"""Create RelationId from string and track mapping for debug info."""
hash_bytes = hashlib.sha256(name.encode()).digest()
# Use big-endian and the lower 128 bits of the hash, consistent with pyrel.
id_high = int.from_bytes(hash_bytes[16:24], byteorder='big')
id_low = int.from_bytes(hash_bytes[24:32], byteorder='big')
relation_id = logic_pb2.RelationId(id_low=id_low, id_high=id_high)
# Store the mapping for the current fragment if we're inside one
if self._current_fragment_id is not None:
if self._current_fragment_id not in self.id_to_debuginfo:
self.id_to_debuginfo[self._current_fragment_id] = {}
key = (relation_id.id_low, relation_id.id_high)
self.id_to_debuginfo[self._current_fragment_id][key] = name
return relation_id
def construct_fragment(
self,
fragment_id: fragments_pb2.FragmentId,
declarations: list[logic_pb2.Declaration],
) -> fragments_pb2.Fragment:
"""Construct Fragment from fragment_id, declarations, and debug info from parser state."""
# Get the debug info for this fragment
debug_info_dict = self.id_to_debuginfo.get(fragment_id.id, {})
# Convert to DebugInfo protobuf
ids = []
orig_names = []
for (id_low, id_high), name in debug_info_dict.items():
ids.append(logic_pb2.RelationId(id_low=id_low, id_high=id_high))
orig_names.append(name)
# Create DebugInfo
debug_info = fragments_pb2.DebugInfo(ids=ids, orig_names=orig_names)
# Clear _current_fragment_id before the return
self._current_fragment_id = None
# Create and return Fragment
return fragments_pb2.Fragment(
id=fragment_id, declarations=declarations, debug_info=debug_info
)
def relation_id_to_string(self, msg) -> str:
"""Stub: only used in pretty printer."""
raise NotImplementedError(
"relation_id_to_string is only available in PrettyPrinter"
)
def relation_id_to_uint128(self, msg):
"""Stub: only used in pretty printer."""
raise NotImplementedError(
"relation_id_to_uint128 is only available in PrettyPrinter"
)
# --- Helper functions ---
def _extract_value_int32(self, value: logic_pb2.Value | None, default: int) -> int:
if value is not None:
assert value is not None
_t2088 = value.HasField("int32_value")
else:
_t2088 = False
if _t2088:
assert value is not None
return value.int32_value
else:
_t2089 = None
return int(default)
def _extract_value_int64(self, value: logic_pb2.Value | None, default: int) -> int:
if value is not None:
assert value is not None
_t2090 = value.HasField("int_value")
else:
_t2090 = False
if _t2090:
assert value is not None
return value.int_value
else:
_t2091 = None
return default
def _extract_value_string(self, value: logic_pb2.Value | None, default: str) -> str:
if value is not None:
assert value is not None
_t2092 = value.HasField("string_value")
else:
_t2092 = False
if _t2092:
assert value is not None
return value.string_value
else:
_t2093 = None
return default
def _extract_value_boolean(self, value: logic_pb2.Value | None, default: bool) -> bool:
if value is not None:
assert value is not None
_t2094 = value.HasField("boolean_value")
else:
_t2094 = False
if _t2094:
assert value is not None
return value.boolean_value
else:
_t2095 = None
return default
def _extract_value_string_list(self, value: logic_pb2.Value | None, default: Sequence[str]) -> Sequence[str]:
if value is not None:
assert value is not None
_t2096 = value.HasField("string_value")
else:
_t2096 = False
if _t2096:
assert value is not None
return [value.string_value]
else:
_t2097 = None
return default
def _try_extract_value_int64(self, value: logic_pb2.Value | None) -> int | None:
if value is not None:
assert value is not None
_t2098 = value.HasField("int_value")
else:
_t2098 = False
if _t2098:
assert value is not None
return value.int_value
else:
_t2099 = None
return None
def _try_extract_value_float64(self, value: logic_pb2.Value | None) -> float | None:
if value is not None:
assert value is not None
_t2100 = value.HasField("float_value")
else:
_t2100 = False
if _t2100:
assert value is not None
return value.float_value
else:
_t2101 = None
return None
def _try_extract_value_bytes(self, value: logic_pb2.Value | None) -> bytes | None:
if value is not None:
assert value is not None
_t2102 = value.HasField("string_value")
else:
_t2102 = False
if _t2102:
assert value is not None
return value.string_value.encode()
else:
_t2103 = None
return None
def _try_extract_value_uint128(self, value: logic_pb2.Value | None) -> logic_pb2.UInt128Value | None:
if value is not None:
assert value is not None
_t2104 = value.HasField("uint128_value")
else:
_t2104 = False
if _t2104:
assert value is not None
return value.uint128_value
else:
_t2105 = None
return None
def construct_csv_config(self, config_dict: Sequence[tuple[str, logic_pb2.Value]], storage_integration_opt: Sequence[tuple[str, logic_pb2.Value]] | None) -> logic_pb2.CSVConfig:
config = dict(config_dict)
_t2106 = self._extract_value_int32(config.get("csv_header_row"), 1)
header_row = _t2106
_t2107 = self._extract_value_int64(config.get("csv_skip"), 0)
skip = _t2107
_t2108 = self._extract_value_string(config.get("csv_new_line"), "")
new_line = _t2108
_t2109 = self._extract_value_string(config.get("csv_delimiter"), ",")
delimiter = _t2109
_t2110 = self._extract_value_string(config.get("csv_quotechar"), '"')
quotechar = _t2110
_t2111 = self._extract_value_string(config.get("csv_escapechar"), '"')
escapechar = _t2111
_t2112 = self._extract_value_string(config.get("csv_comment"), "")
comment = _t2112
_t2113 = self._extract_value_string_list(config.get("csv_missing_strings"), [])
missing_strings = _t2113
_t2114 = self._extract_value_string(config.get("csv_decimal_separator"), ".")
decimal_separator = _t2114
_t2115 = self._extract_value_string(config.get("csv_encoding"), "utf-8")
encoding = _t2115
_t2116 = self._extract_value_string(config.get("csv_compression"), "")
compression = _t2116
_t2117 = self._extract_value_int64(config.get("csv_partition_size_mb"), 0)
partition_size_mb = _t2117
_t2118 = self.construct_csv_storage_integration(storage_integration_opt)
storage_integration = _t2118
_t2119 = logic_pb2.CSVConfig(header_row=header_row, skip=skip, new_line=new_line, delimiter=delimiter, quotechar=quotechar, escapechar=escapechar, comment=comment, missing_strings=missing_strings, decimal_separator=decimal_separator, encoding=encoding, compression=compression, partition_size_mb=partition_size_mb, storage_integration=storage_integration)
return _t2119
def construct_csv_storage_integration(self, storage_integration_opt: Sequence[tuple[str, logic_pb2.Value]] | None) -> logic_pb2.StorageIntegration | None:
if storage_integration_opt is None:
return None
else:
_t2120 = None
assert storage_integration_opt is not None
config = dict(storage_integration_opt)
_t2121 = self._extract_value_string(config.get("provider"), "")
_t2122 = self._extract_value_string(config.get("azure_sas_token"), "")
_t2123 = self._extract_value_string(config.get("s3_region"), "")
_t2124 = self._extract_value_string(config.get("s3_access_key_id"), "")
_t2125 = self._extract_value_string(config.get("s3_secret_access_key"), "")
_t2126 = logic_pb2.StorageIntegration(provider=_t2121, azure_sas_token=_t2122, s3_region=_t2123, s3_access_key_id=_t2124, s3_secret_access_key=_t2125)
return _t2126
def construct_betree_info(self, key_types: Sequence[logic_pb2.Type], value_types: Sequence[logic_pb2.Type], config_dict: Sequence[tuple[str, logic_pb2.Value]]) -> logic_pb2.BeTreeInfo:
config = dict(config_dict)
_t2127 = self._try_extract_value_float64(config.get("betree_config_epsilon"))
epsilon = _t2127
_t2128 = self._try_extract_value_int64(config.get("betree_config_max_pivots"))
max_pivots = _t2128
_t2129 = self._try_extract_value_int64(config.get("betree_config_max_deltas"))
max_deltas = _t2129
_t2130 = self._try_extract_value_int64(config.get("betree_config_max_leaf"))
max_leaf = _t2130
_t2131 = logic_pb2.BeTreeConfig(epsilon=epsilon, max_pivots=max_pivots, max_deltas=max_deltas, max_leaf=max_leaf)
storage_config = _t2131
_t2132 = self._try_extract_value_uint128(config.get("betree_locator_root_pageid"))
root_pageid = _t2132
_t2133 = self._try_extract_value_bytes(config.get("betree_locator_inline_data"))
inline_data = _t2133
_t2134 = self._try_extract_value_int64(config.get("betree_locator_element_count"))
element_count = _t2134
_t2135 = self._try_extract_value_int64(config.get("betree_locator_tree_height"))
tree_height = _t2135
_t2136 = logic_pb2.BeTreeLocator(root_pageid=root_pageid, inline_data=inline_data, element_count=element_count, tree_height=tree_height)
relation_locator = _t2136
_t2137 = logic_pb2.BeTreeInfo(key_types=key_types, value_types=value_types, storage_config=storage_config, relation_locator=relation_locator)
return _t2137
def default_configure(self) -> transactions_pb2.Configure:
_t2138 = transactions_pb2.IVMConfig(level=transactions_pb2.MaintenanceLevel.MAINTENANCE_LEVEL_OFF)
ivm_config = _t2138
_t2139 = transactions_pb2.Configure(semantics_version=0, ivm_config=ivm_config)
return _t2139
def construct_configure(self, config_dict: Sequence[tuple[str, logic_pb2.Value]]) -> transactions_pb2.Configure:
config = dict(config_dict)
maintenance_level_val = config.get("ivm.maintenance_level")
maintenance_level = transactions_pb2.MaintenanceLevel.MAINTENANCE_LEVEL_OFF
if (maintenance_level_val is not None and maintenance_level_val.HasField("string_value")):
if maintenance_level_val.string_value == "off":
maintenance_level = transactions_pb2.MaintenanceLevel.MAINTENANCE_LEVEL_OFF
else:
if maintenance_level_val.string_value == "auto":
maintenance_level = transactions_pb2.MaintenanceLevel.MAINTENANCE_LEVEL_AUTO
else:
if maintenance_level_val.string_value == "all":
maintenance_level = transactions_pb2.MaintenanceLevel.MAINTENANCE_LEVEL_ALL
else:
maintenance_level = transactions_pb2.MaintenanceLevel.MAINTENANCE_LEVEL_OFF
_t2140 = transactions_pb2.IVMConfig(level=maintenance_level)
ivm_config = _t2140
_t2141 = self._extract_value_int64(config.get("semantics_version"), 0)
semantics_version = _t2141
_t2142 = transactions_pb2.Configure(semantics_version=semantics_version, ivm_config=ivm_config)
return _t2142
def construct_export_csv_config(self, path: str, columns: Sequence[transactions_pb2.ExportCSVColumn], config_dict: Sequence[tuple[str, logic_pb2.Value]]) -> transactions_pb2.ExportCSVConfig:
config = dict(config_dict)
_t2143 = self._extract_value_int64(config.get("partition_size"), 0)
partition_size = _t2143
_t2144 = self._extract_value_string(config.get("compression"), "")
compression = _t2144
_t2145 = self._extract_value_boolean(config.get("syntax_header_row"), True)
syntax_header_row = _t2145
_t2146 = self._extract_value_string(config.get("syntax_missing_string"), "")
syntax_missing_string = _t2146
_t2147 = self._extract_value_string(config.get("syntax_delim"), ",")
syntax_delim = _t2147
_t2148 = self._extract_value_string(config.get("syntax_quotechar"), '"')
syntax_quotechar = _t2148
_t2149 = self._extract_value_string(config.get("syntax_escapechar"), "\\")
syntax_escapechar = _t2149
_t2150 = transactions_pb2.ExportCSVConfig(path=path, data_columns=columns, partition_size=partition_size, compression=compression, syntax_header_row=syntax_header_row, syntax_missing_string=syntax_missing_string, syntax_delim=syntax_delim, syntax_quotechar=syntax_quotechar, syntax_escapechar=syntax_escapechar)
return _t2150
def construct_export_csv_config_with_source(self, path: str, csv_source: transactions_pb2.ExportCSVSource, csv_config: logic_pb2.CSVConfig) -> transactions_pb2.ExportCSVConfig:
_t2151 = transactions_pb2.ExportCSVConfig(path=path, csv_source=csv_source, csv_config=csv_config)
return _t2151
def construct_iceberg_catalog_config(self, catalog_uri: str, scope_opt: str | None, property_pairs: Sequence[tuple[str, str]], auth_property_pairs: Sequence[tuple[str, str]]) -> logic_pb2.IcebergCatalogConfig:
props = dict(property_pairs)
auth_props = dict(auth_property_pairs)
_t2152 = logic_pb2.IcebergCatalogConfig(catalog_uri=catalog_uri, scope=(scope_opt if scope_opt is not None else ""), properties=props, auth_properties=auth_props)
return _t2152
def construct_iceberg_data(self, locator: logic_pb2.IcebergLocator, config: logic_pb2.IcebergCatalogConfig, columns: Sequence[logic_pb2.GNFColumn], from_snapshot_opt: str | None, to_snapshot_opt: str | None, returns_delta: bool) -> logic_pb2.IcebergData:
_t2153 = logic_pb2.IcebergData(locator=locator, config=config, columns=columns, from_snapshot=(from_snapshot_opt if from_snapshot_opt is not None else ""), to_snapshot=(to_snapshot_opt if to_snapshot_opt is not None else ""), returns_delta=returns_delta)
return _t2153
def construct_export_iceberg_config_full(self, locator: logic_pb2.IcebergLocator, config: logic_pb2.IcebergCatalogConfig, table_def: logic_pb2.RelationId, table_property_pairs: Sequence[tuple[str, str]], config_dict: Sequence[tuple[str, logic_pb2.Value]] | None) -> transactions_pb2.ExportIcebergConfig:
cfg = dict((config_dict if config_dict is not None else []))
_t2154 = self._extract_value_string(cfg.get("prefix"), "")
prefix = _t2154
_t2155 = self._extract_value_int64(cfg.get("target_file_size_bytes"), 0)
target_file_size_bytes = _t2155
_t2156 = self._extract_value_string(cfg.get("compression"), "")
compression = _t2156
table_props = dict(table_property_pairs)
_t2157 = transactions_pb2.ExportIcebergConfig(locator=locator, config=config, table_def=table_def, prefix=prefix, target_file_size_bytes=target_file_size_bytes, compression=compression, table_properties=table_props)
return _t2157
# --- Parse methods ---
def parse_transaction(self) -> transactions_pb2.Transaction:
span_start673 = self.span_start()
self.consume_literal("(")
self.consume_literal("transaction")
if (self.match_lookahead_literal("(", 0) and self.match_lookahead_literal("configure", 1)):
_t1335 = self.parse_configure()
_t1334 = _t1335
else:
_t1334 = None
configure667 = _t1334
if (self.match_lookahead_literal("(", 0) and self.match_lookahead_literal("sync", 1)):
_t1337 = self.parse_sync()
_t1336 = _t1337
else:
_t1336 = None
sync668 = _t1336
xs669 = []
cond670 = self.match_lookahead_literal("(", 0)
while cond670:
_t1338 = self.parse_epoch()
item671 = _t1338
xs669.append(item671)
cond670 = self.match_lookahead_literal("(", 0)
epochs672 = xs669
self.consume_literal(")")
_t1339 = self.default_configure()
_t1340 = transactions_pb2.Transaction(epochs=epochs672, configure=(configure667 if configure667 is not None else _t1339), sync=sync668)
result674 = _t1340
self.record_span(span_start673, "Transaction")
return result674
def parse_configure(self) -> transactions_pb2.Configure:
span_start676 = self.span_start()
self.consume_literal("(")
self.consume_literal("configure")
_t1341 = self.parse_config_dict()
config_dict675 = _t1341
self.consume_literal(")")
_t1342 = self.construct_configure(config_dict675)
result677 = _t1342
self.record_span(span_start676, "Configure")
return result677
def parse_config_dict(self) -> Sequence[tuple[str, logic_pb2.Value]]:
self.consume_literal("{")
xs678 = []
cond679 = self.match_lookahead_literal(":", 0)
while cond679:
_t1343 = self.parse_config_key_value()
item680 = _t1343
xs678.append(item680)
cond679 = self.match_lookahead_literal(":", 0)
config_key_values681 = xs678
self.consume_literal("}")
return config_key_values681
def parse_config_key_value(self) -> tuple[str, logic_pb2.Value]:
self.consume_literal(":")
symbol682 = self.consume_terminal("SYMBOL")
_t1344 = self.parse_raw_value()
raw_value683 = _t1344
return (symbol682, raw_value683,)
def parse_raw_value(self) -> logic_pb2.Value:
span_start697 = self.span_start()
if self.match_lookahead_literal("true", 0):
_t1345 = 12
else:
if self.match_lookahead_literal("missing", 0):
_t1346 = 11
else:
if self.match_lookahead_literal("false", 0):
_t1347 = 12
else:
if self.match_lookahead_literal("(", 0):
if self.match_lookahead_literal("datetime", 1):
_t1349 = 1
else:
if self.match_lookahead_literal("date", 1):
_t1350 = 0
else:
_t1350 = -1
_t1349 = _t1350
_t1348 = _t1349
else:
if self.match_lookahead_terminal("UINT32", 0):
_t1351 = 7
else:
if self.match_lookahead_terminal("UINT128", 0):
_t1352 = 8
else:
if self.match_lookahead_terminal("STRING", 0):
_t1353 = 2
else:
if self.match_lookahead_terminal("INT32", 0):
_t1354 = 3
else:
if self.match_lookahead_terminal("INT128", 0):
_t1355 = 9
else:
if self.match_lookahead_terminal("INT", 0):
_t1356 = 4
else:
if self.match_lookahead_terminal("FLOAT32", 0):
_t1357 = 5
else:
if self.match_lookahead_terminal("FLOAT", 0):
_t1358 = 6
else:
if self.match_lookahead_terminal("DECIMAL", 0):
_t1359 = 10
else:
_t1359 = -1
_t1358 = _t1359
_t1357 = _t1358
_t1356 = _t1357
_t1355 = _t1356
_t1354 = _t1355
_t1353 = _t1354
_t1352 = _t1353
_t1351 = _t1352
_t1348 = _t1351
_t1347 = _t1348
_t1346 = _t1347
_t1345 = _t1346
prediction684 = _t1345
if prediction684 == 12:
_t1361 = self.parse_boolean_value()
boolean_value696 = _t1361
_t1362 = logic_pb2.Value(boolean_value=boolean_value696)
_t1360 = _t1362
else:
if prediction684 == 11:
self.consume_literal("missing")
_t1364 = logic_pb2.MissingValue()
_t1365 = logic_pb2.Value(missing_value=_t1364)
_t1363 = _t1365
else:
if prediction684 == 10:
decimal695 = self.consume_terminal("DECIMAL")
_t1367 = logic_pb2.Value(decimal_value=decimal695)
_t1366 = _t1367
else:
if prediction684 == 9:
int128694 = self.consume_terminal("INT128")
_t1369 = logic_pb2.Value(int128_value=int128694)
_t1368 = _t1369
else:
if prediction684 == 8:
uint128693 = self.consume_terminal("UINT128")
_t1371 = logic_pb2.Value(uint128_value=uint128693)
_t1370 = _t1371
else:
if prediction684 == 7:
uint32692 = self.consume_terminal("UINT32")
_t1373 = logic_pb2.Value(uint32_value=uint32692)
_t1372 = _t1373
else:
if prediction684 == 6:
float691 = self.consume_terminal("FLOAT")
_t1375 = logic_pb2.Value(float_value=float691)
_t1374 = _t1375
else:
if prediction684 == 5:
float32690 = self.consume_terminal("FLOAT32")
_t1377 = logic_pb2.Value(float32_value=float32690)
_t1376 = _t1377
else:
if prediction684 == 4:
int689 = self.consume_terminal("INT")
_t1379 = logic_pb2.Value(int_value=int689)
_t1378 = _t1379
else:
if prediction684 == 3:
int32688 = self.consume_terminal("INT32")
_t1381 = logic_pb2.Value(int32_value=int32688)
_t1380 = _t1381
else:
if prediction684 == 2:
string687 = self.consume_terminal("STRING")
_t1383 = logic_pb2.Value(string_value=string687)
_t1382 = _t1383
else:
if prediction684 == 1:
_t1385 = self.parse_raw_datetime()
raw_datetime686 = _t1385
_t1386 = logic_pb2.Value(datetime_value=raw_datetime686)
_t1384 = _t1386
else:
if prediction684 == 0:
_t1388 = self.parse_raw_date()
raw_date685 = _t1388
_t1389 = logic_pb2.Value(date_value=raw_date685)
_t1387 = _t1389
else:
raise ParseError("Unexpected token in raw_value" + f": {self.lookahead(0).type}=`{self.lookahead(0).value}`")
_t1384 = _t1387
_t1382 = _t1384
_t1380 = _t1382
_t1378 = _t1380
_t1376 = _t1378
_t1374 = _t1376
_t1372 = _t1374
_t1370 = _t1372
_t1368 = _t1370
_t1366 = _t1368
_t1363 = _t1366
_t1360 = _t1363
result698 = _t1360
self.record_span(span_start697, "Value")
return result698
def parse_raw_date(self) -> logic_pb2.DateValue:
span_start702 = self.span_start()
self.consume_literal("(")
self.consume_literal("date")
int699 = self.consume_terminal("INT")
int_3700 = self.consume_terminal("INT")
int_4701 = self.consume_terminal("INT")
self.consume_literal(")")
_t1390 = logic_pb2.DateValue(year=int(int699), month=int(int_3700), day=int(int_4701))
result703 = _t1390
self.record_span(span_start702, "DateValue")
return result703
def parse_raw_datetime(self) -> logic_pb2.DateTimeValue:
span_start711 = self.span_start()
self.consume_literal("(")
self.consume_literal("datetime")
int704 = self.consume_terminal("INT")
int_3705 = self.consume_terminal("INT")
int_4706 = self.consume_terminal("INT")
int_5707 = self.consume_terminal("INT")
int_6708 = self.consume_terminal("INT")
int_7709 = self.consume_terminal("INT")
if self.match_lookahead_terminal("INT", 0):
_t1391 = self.consume_terminal("INT")
else:
_t1391 = None
int_8710 = _t1391
self.consume_literal(")")
_t1392 = logic_pb2.DateTimeValue(year=int(int704), month=int(int_3705), day=int(int_4706), hour=int(int_5707), minute=int(int_6708), second=int(int_7709), microsecond=int((int_8710 if int_8710 is not None else 0)))
result712 = _t1392
self.record_span(span_start711, "DateTimeValue")
return result712
def parse_boolean_value(self) -> bool:
if self.match_lookahead_literal("true", 0):
_t1393 = 0
else:
if self.match_lookahead_literal("false", 0):
_t1394 = 1
else:
_t1394 = -1
_t1393 = _t1394
prediction713 = _t1393
if prediction713 == 1:
self.consume_literal("false")
_t1395 = False
else:
if prediction713 == 0:
self.consume_literal("true")
_t1396 = True
else:
raise ParseError("Unexpected token in boolean_value" + f": {self.lookahead(0).type}=`{self.lookahead(0).value}`")
_t1395 = _t1396
return _t1395
def parse_sync(self) -> transactions_pb2.Sync:
span_start718 = self.span_start()
self.consume_literal("(")
self.consume_literal("sync")
xs714 = []
cond715 = self.match_lookahead_literal(":", 0)
while cond715:
_t1397 = self.parse_fragment_id()
item716 = _t1397
xs714.append(item716)
cond715 = self.match_lookahead_literal(":", 0)
fragment_ids717 = xs714
self.consume_literal(")")
_t1398 = transactions_pb2.Sync(fragments=fragment_ids717)
result719 = _t1398
self.record_span(span_start718, "Sync")
return result719
def parse_fragment_id(self) -> fragments_pb2.FragmentId:
span_start721 = self.span_start()
self.consume_literal(":")
symbol720 = self.consume_terminal("SYMBOL")
result722 = fragments_pb2.FragmentId(id=symbol720.encode())
self.record_span(span_start721, "FragmentId")
return result722
def parse_epoch(self) -> transactions_pb2.Epoch:
span_start725 = self.span_start()
self.consume_literal("(")
self.consume_literal("epoch")
if (self.match_lookahead_literal("(", 0) and self.match_lookahead_literal("writes", 1)):
_t1400 = self.parse_epoch_writes()
_t1399 = _t1400
else:
_t1399 = None
epoch_writes723 = _t1399
if self.match_lookahead_literal("(", 0):
_t1402 = self.parse_epoch_reads()
_t1401 = _t1402
else:
_t1401 = None
epoch_reads724 = _t1401
self.consume_literal(")")
_t1403 = transactions_pb2.Epoch(writes=(epoch_writes723 if epoch_writes723 is not None else []), reads=(epoch_reads724 if epoch_reads724 is not None else []))
result726 = _t1403
self.record_span(span_start725, "Epoch")