Skip to content

Commit fa6eeb5

Browse files
davidwzhaoclaude
andcommitted
Add storage_integration tests; fix printer name collision
Rename the grammar nonterm to storage_integration so it no longer collides with the auto-generated CSVStorageIntegration message printer, and regenerate the SDK parsers/printers. Add a csv_storage_integration fixture (S3/Azure import + S3 export) plus unit tests covering secret masking. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
1 parent 5f14d6a commit fa6eeb5

12 files changed

Lines changed: 284 additions & 24 deletions

File tree

meta/src/meta/grammar.y

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -91,7 +91,7 @@
9191
%nonterm csv_data logic.CSVData
9292
%nonterm csv_locator_inline_data String
9393
%nonterm csv_locator_paths Sequence[String]
94-
%nonterm csv_storage_integration Sequence[Tuple[String, logic.Value]]
94+
%nonterm storage_integration Sequence[Tuple[String, logic.Value]]
9595
%nonterm csvlocator logic.CSVLocator
9696
%nonterm data logic.Data
9797
%nonterm date logic.DateValue
@@ -1129,13 +1129,13 @@ csvlocator
11291129
$4: Optional[String] = builtin.decode_string($$.inline_data) if builtin.decode_string($$.inline_data) != "" else None
11301130

11311131
csv_config
1132-
: "(" "csv_config" config_dict csv_storage_integration? ")"
1132+
: "(" "csv_config" config_dict storage_integration? ")"
11331133
construct: $$ = construct_csv_config($3, $4)
11341134
deconstruct:
11351135
$3: Sequence[Tuple[String, logic.Value]] = deconstruct_csv_config($$)
11361136
$4: Optional[Sequence[Tuple[String, logic.Value]]] = deconstruct_csv_storage_integration_optional($$)
11371137

1138-
csv_storage_integration
1138+
storage_integration
11391139
: "(" "storage_integration" config_dict ")"
11401140
construct: $$ = $3
11411141
deconstruct: $3: Sequence[Tuple[String, logic.Value]] = $$

sdks/go/src/parser.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4305,18 +4305,18 @@ func (p *Parser) parse_csv_config() *pb.CSVConfig {
43054305
config_dict1177 := _t1976
43064306
var _t1977 [][]interface{}
43074307
if p.matchLookaheadLiteral("(", 0) {
4308-
_t1978 := p.parse_csv_storage_integration()
4308+
_t1978 := p.parse_storage_integration()
43094309
_t1977 = _t1978
43104310
}
4311-
csv_storage_integration1178 := _t1977
4311+
storage_integration1178 := _t1977
43124312
p.consumeLiteral(")")
4313-
_t1979 := p.construct_csv_config(config_dict1177, csv_storage_integration1178)
4313+
_t1979 := p.construct_csv_config(config_dict1177, storage_integration1178)
43144314
result1180 := _t1979
43154315
p.recordSpan(int(span_start1179), "CSVConfig")
43164316
return result1180
43174317
}
43184318

4319-
func (p *Parser) parse_csv_storage_integration() [][]interface{} {
4319+
func (p *Parser) parse_storage_integration() [][]interface{} {
43204320
p.consumeLiteral("(")
43214321
p.consumeLiteral("storage_integration")
43224322
_t1980 := p.parse_config_dict()

sdks/go/src/pretty.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3926,16 +3926,16 @@ func (p *PrettyPrinter) pretty_csv_config(msg *pb.CSVConfig) interface{} {
39263926
if field1409 != nil {
39273927
p.newline()
39283928
opt_val1410 := field1409
3929-
p.pretty_csv_storage_integration(opt_val1410)
3929+
p.pretty_storage_integration(opt_val1410)
39303930
}
39313931
p.dedent()
39323932
p.write(")")
39333933
}
39343934
return nil
39353935
}
39363936

3937-
func (p *PrettyPrinter) pretty_csv_storage_integration(msg [][]interface{}) interface{} {
3938-
flat1413 := p.tryFlat(msg, func() { p.pretty_csv_storage_integration(msg) })
3937+
func (p *PrettyPrinter) pretty_storage_integration(msg [][]interface{}) interface{} {
3938+
flat1413 := p.tryFlat(msg, func() { p.pretty_storage_integration(msg) })
39393939
if flat1413 != nil {
39403940
p.write(*flat1413)
39413941
return nil

sdks/julia/LogicalQueryProtocol.jl/src/parser.jl

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3629,20 +3629,20 @@ function parse_csv_config(parser::ParserState)::Proto.CSVConfig
36293629
_t1963 = parse_config_dict(parser)
36303630
config_dict1177 = _t1963
36313631
if match_lookahead_literal(parser, "(", 0)
3632-
_t1965 = parse_csv_storage_integration(parser)
3632+
_t1965 = parse_storage_integration(parser)
36333633
_t1964 = _t1965
36343634
else
36353635
_t1964 = nothing
36363636
end
3637-
csv_storage_integration1178 = _t1964
3637+
storage_integration1178 = _t1964
36383638
consume_literal!(parser, ")")
3639-
_t1966 = construct_csv_config(parser, config_dict1177, csv_storage_integration1178)
3639+
_t1966 = construct_csv_config(parser, config_dict1177, storage_integration1178)
36403640
result1180 = _t1966
36413641
record_span!(parser, span_start1179, "CSVConfig")
36423642
return result1180
36433643
end
36443644

3645-
function parse_csv_storage_integration(parser::ParserState)::Vector{Tuple{String, Proto.Value}}
3645+
function parse_storage_integration(parser::ParserState)::Vector{Tuple{String, Proto.Value}}
36463646
consume_literal!(parser, "(")
36473647
consume_literal!(parser, "storage_integration")
36483648
_t1967 = parse_config_dict(parser)

sdks/julia/LogicalQueryProtocol.jl/src/pretty.jl

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4007,16 +4007,16 @@ function pretty_csv_config(pp::PrettyPrinter, msg::Proto.CSVConfig)
40074007
if !isnothing(field1409)
40084008
newline(pp)
40094009
opt_val1410 = field1409
4010-
pretty_csv_storage_integration(pp, opt_val1410)
4010+
pretty_storage_integration(pp, opt_val1410)
40114011
end
40124012
dedent!(pp)
40134013
write(pp, ")")
40144014
end
40154015
return nothing
40164016
end
40174017

4018-
function pretty_csv_storage_integration(pp::PrettyPrinter, msg::Vector{Tuple{String, Proto.Value}})
4019-
flat1413 = try_flat(pp, msg, pretty_csv_storage_integration)
4018+
function pretty_storage_integration(pp::PrettyPrinter, msg::Vector{Tuple{String, Proto.Value}})
4019+
flat1413 = try_flat(pp, msg, pretty_storage_integration)
40204020
if !isnothing(flat1413)
40214021
write(pp, flat1413)
40224022
return nothing

sdks/python/src/lqp/gen/parser.py

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3318,18 +3318,18 @@ def parse_csv_config(self) -> logic_pb2.CSVConfig:
33183318
_t1963 = self.parse_config_dict()
33193319
config_dict1177 = _t1963
33203320
if self.match_lookahead_literal("(", 0):
3321-
_t1965 = self.parse_csv_storage_integration()
3321+
_t1965 = self.parse_storage_integration()
33223322
_t1964 = _t1965
33233323
else:
33243324
_t1964 = None
3325-
csv_storage_integration1178 = _t1964
3325+
storage_integration1178 = _t1964
33263326
self.consume_literal(")")
3327-
_t1966 = self.construct_csv_config(config_dict1177, csv_storage_integration1178)
3327+
_t1966 = self.construct_csv_config(config_dict1177, storage_integration1178)
33283328
result1180 = _t1966
33293329
self.record_span(span_start1179, "CSVConfig")
33303330
return result1180
33313331

3332-
def parse_csv_storage_integration(self) -> Sequence[tuple[str, logic_pb2.Value]]:
3332+
def parse_storage_integration(self) -> Sequence[tuple[str, logic_pb2.Value]]:
33333333
self.consume_literal("(")
33343334
self.consume_literal("storage_integration")
33353335
_t1967 = self.parse_config_dict()

sdks/python/src/lqp/gen/pretty.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3438,12 +3438,12 @@ def pretty_csv_config(self, msg: logic_pb2.CSVConfig):
34383438
self.newline()
34393439
assert field1409 is not None
34403440
opt_val1410 = field1409
3441-
self.pretty_csv_storage_integration(opt_val1410)
3441+
self.pretty_storage_integration(opt_val1410)
34423442
self.dedent()
34433443
self.write(")")
34443444

3445-
def pretty_csv_storage_integration(self, msg: Sequence[tuple[str, logic_pb2.Value]]):
3446-
flat1413 = self._try_flat(msg, self.pretty_csv_storage_integration)
3445+
def pretty_storage_integration(self, msg: Sequence[tuple[str, logic_pb2.Value]]):
3446+
flat1413 = self._try_flat(msg, self.pretty_storage_integration)
34473447
if flat1413 is not None:
34483448
assert flat1413 is not None
34493449
self.write(flat1413)

sdks/python/tests/test_generated_pretty.py

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -442,3 +442,73 @@ def test_pretty_from_binary_roundtrip(bin_file):
442442
assert txn.SerializeToString() == re_proto.SerializeToString(), (
443443
f"Binary roundtrip failed for {Path(bin_file).name}"
444444
)
445+
446+
447+
# ---------------------------------------------------------------------------
448+
# CSV storage integration secret masking
449+
#
450+
# Secret credentials are masked as "***" on print, so they cannot round-trip
451+
# through a fixture. These tests cover masking directly instead.
452+
# ---------------------------------------------------------------------------
453+
454+
_STORAGE_INTEGRATION_LQP = """(transaction
455+
(epoch
456+
(writes
457+
(define
458+
(fragment :f1
459+
(csv_data
460+
(csv_locator (paths "s3://private-bucket/data.csv"))
461+
(csv_config {}
462+
(storage_integration {
463+
:provider "s3"
464+
:s3_region "us-west-2"
465+
:s3_access_key_id "AKIAEXAMPLE"
466+
:s3_secret_access_key "topsecret-value"
467+
:azure_sas_token "sas-token-value" }))
468+
(columns (column "id" :x [INT]))
469+
(asof "2025-01-01T00:00:00Z")))))
470+
(reads (output :x :x))))"""
471+
472+
473+
def _parsed_storage_integration():
474+
proto, _ = generated_parse(_STORAGE_INTEGRATION_LQP)
475+
config = (
476+
proto.epochs[0].writes[0].define.fragment.declarations[0].data.csv_data.config
477+
)
478+
return proto, config
479+
480+
481+
def test_storage_integration_parse_preserves_secrets():
482+
"""Parsing keeps the real secret values; masking is a print-only concern."""
483+
_, config = _parsed_storage_integration()
484+
assert config.HasField("storage_integration")
485+
si = config.storage_integration
486+
assert si.provider == "s3"
487+
assert si.s3_region == "us-west-2"
488+
assert si.s3_access_key_id == "AKIAEXAMPLE"
489+
assert si.s3_secret_access_key == "topsecret-value"
490+
assert si.azure_sas_token == "sas-token-value"
491+
492+
493+
def test_storage_integration_secrets_masked_on_print():
494+
"""Secret fields print as "***"; non-secret fields print verbatim."""
495+
proto, _ = _parsed_storage_integration()
496+
printed = pretty(proto)
497+
# Secret values must never leak into the human-readable output.
498+
assert "AKIAEXAMPLE" not in printed
499+
assert "topsecret-value" not in printed
500+
assert "sas-token-value" not in printed
501+
assert ':s3_access_key_id "***"' in printed
502+
assert ':s3_secret_access_key "***"' in printed
503+
assert ':azure_sas_token "***"' in printed
504+
# Non-secret fields are shown as-is.
505+
assert ':provider "s3"' in printed
506+
assert ':s3_region "us-west-2"' in printed
507+
508+
509+
def test_storage_integration_absent_emits_no_block():
510+
"""A CSVConfig without a storage integration prints no storage_integration block."""
511+
config = logic_pb2.CSVConfig()
512+
assert not config.HasField("storage_integration")
513+
printer = PrettyPrinter()
514+
assert printer.deconstruct_csv_storage_integration_optional(config) is None
1 KB
Binary file not shown.
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
(transaction
2+
(epoch
3+
(writes
4+
(define
5+
(fragment :f1
6+
;; CSV imported from a private bucket via an S3 storage integration.
7+
;; Only non-secret fields are used here so the example round-trips;
8+
;; secret credentials (e.g. s3_secret_access_key) are masked as "***"
9+
;; on print and therefore cannot round-trip through a fixture.
10+
(csv_data
11+
(csv_locator
12+
(paths "s3://private-bucket/data.csv"))
13+
(csv_config {}
14+
(storage_integration {
15+
:provider "s3"
16+
:s3_region "us-west-2" }))
17+
(columns
18+
(column "id" :s3_id [INT])
19+
(column "value" :s3_value [STRING]))
20+
(asof "2025-04-01T00:00:00Z"))
21+
22+
;; CSV imported from a private bucket via an Azure storage integration.
23+
(csv_data
24+
(csv_locator
25+
(paths "https://account.blob.core.windows.net/container/data.csv"))
26+
(csv_config { :csv_delimiter "|" }
27+
(storage_integration {
28+
:provider "azure" }))
29+
(columns
30+
(column "id" :azure_id [INT])
31+
(column "value" :azure_value [STRING]))
32+
(asof "2025-04-02T00:00:00Z"))
33+
34+
;; A table to export back out to a private bucket.
35+
(def :exported
36+
([id::INT value::STRING]
37+
(or
38+
(and (= id 1) (= value "a"))
39+
(and (= id 2) (= value "b"))))))))
40+
41+
(reads
42+
(output :s3_id :s3_id)
43+
(output :s3_value :s3_value)
44+
(output :azure_id :azure_id)
45+
(output :azure_value :azure_value)
46+
47+
;; CSV exported to a private bucket via an S3 storage integration.
48+
(export
49+
(export_csv_config_v2
50+
(path "s3://private-bucket/export.csv")
51+
(table_def :exported)
52+
(csv_config { :csv_header_row 1i32 }
53+
(storage_integration {
54+
:provider "s3"
55+
:s3_region "eu-central-1" })))))))

0 commit comments

Comments
 (0)