@@ -43,53 +43,55 @@ pipelines:
4343 - {name: "Alice", age: 30, score: 95.5}
4444 - {name: "Bob", age: 25, score: 88.0}
4545
46+ # TODO - Rethink error handling since schema mismatch in Create transform
47+ # isn't allowed anymore.
4648 # Validate a Beam Row with a predefined schema with error handling
47- - pipeline :
48- type : composite
49- transforms :
50- - type : Create
51- config :
52- elements :
53- - {name: "Alice", age: 30, score: 95.5}
54- - {name: "Bob", age: 25, score: 88.0}
55- - {name: "Charlie", age: 27, score: "apple"}
56- - {name: "David", age: "twenty", score: 90.0}
57- - {name: 30, age: 40, score: 100.0}
58- - type : ValidateWithSchema
59- input : Create
60- config :
61- schema :
62- type : object
63- properties :
64- name :
65- type : string
66- age :
67- type : integer
68- score :
69- type : number
70- required : [name, age, score]
71- error_handling :
72- output : invalid_rows
73- - type : MapToFields
74- input : ValidateWithSchema.invalid_rows
75- config :
76- language : python
77- fields :
78- name : " element.name"
79- age : " element.age"
80- score : " element.score"
81- - type : AssertEqual
82- input : MapToFields
83- config :
84- elements :
85- - {name: "Charlie", age: 27, score: "apple"}
86- - {name: "David", age: "twenty", score: 90.0}
87- - {name: 30, age: 40, score: 100.0}
88- - type : AssertEqual
89- input : ValidateWithSchema
90- config :
91- elements :
92- - {name: "Alice", age: 30, score: 95.5}
93- - {name: "Bob", age: 25, score: 88.0}
49+ # - pipeline:
50+ # type: composite
51+ # transforms:
52+ # - type: Create
53+ # config:
54+ # elements:
55+ # - {name: "Alice", age: 30, score: 95.5}
56+ # - {name: "Bob", age: 25, score: 88.0}
57+ # - {name: "Charlie", age: 27, score: "apple"}
58+ # - {name: "David", age: "twenty", score: 90.0}
59+ # - {name: 30, age: 40, score: 100.0}
60+ # - type: ValidateWithSchema
61+ # input: Create
62+ # config:
63+ # schema:
64+ # type: object
65+ # properties:
66+ # name:
67+ # type: string
68+ # age:
69+ # type: integer
70+ # score:
71+ # type: number
72+ # required: [name, age, score]
73+ # error_handling:
74+ # output: invalid_rows
75+ # - type: MapToFields
76+ # input: ValidateWithSchema.invalid_rows
77+ # config:
78+ # language: python
79+ # fields:
80+ # name: "element.name"
81+ # age: "element.age"
82+ # score: "element.score"
83+ # - type: AssertEqual
84+ # input: MapToFields
85+ # config:
86+ # elements:
87+ # - {name: "Charlie", age: 27, score: "apple"}
88+ # - {name: "David", age: "twenty", score: 90.0}
89+ # - {name: 30, age: 40, score: 100.0}
90+ # - type: AssertEqual
91+ # input: ValidateWithSchema
92+ # config:
93+ # elements:
94+ # - {name: "Alice", age: 30, score: 95.5}
95+ # - {name: "Bob", age: 25, score: 88.0}
9496
9597
0 commit comments