Skip to content

Commit 230d6dc

Browse files
committed
address more gemini comments
1 parent 9b29dc3 commit 230d6dc

2 files changed

Lines changed: 46 additions & 2 deletions

File tree

sdks/python/apache_beam/yaml/yaml_mapping.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
#
1717

1818
"""This module defines the basic MapToFields operation."""
19+
import datetime
20+
from decimal import Decimal
1921
import itertools
2022
import json
2123
import re
@@ -219,6 +221,12 @@ def py_value_to_js_dict(py_value):
219221
py_value = py_value._asdict()
220222
if isinstance(py_value, dict):
221223
return {key: py_value_to_js_dict(value) for key, value in py_value.items()}
224+
elif isinstance(py_value, bytes):
225+
return py_value.decode('utf-8', errors='replace')
226+
elif isinstance(py_value, (datetime.datetime, datetime.date, datetime.time)):
227+
return py_value.isoformat()
228+
elif isinstance(py_value, Decimal):
229+
return float(py_value)
222230
elif not isinstance(py_value, str) and isinstance(py_value, abc.Iterable):
223231
return [py_value_to_js_dict(value) for value in list(py_value)]
224232
else:
@@ -237,8 +245,8 @@ def _expand_javascript_mapping_func(
237245

238246
if expression:
239247
unpacking_code = '\n'.join([
240-
f' var {name} = __row__.{name};' for name in original_fields
241-
if name in expression
248+
f" var {name} = __row__['{name}'];" for name in original_fields
249+
if name in expression and name.isidentifier()
242250
])
243251
source_code = f"""
244252
const udf = function(__row__) {{

sdks/python/apache_beam/yaml/yaml_udf_test.py

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,8 @@
1414
# See the License for the specific language governing permissions and
1515
# limitations under the License.
1616
#
17+
import datetime
18+
from decimal import Decimal
1719
import logging
1820
import os
1921
import shutil
@@ -374,6 +376,40 @@ def g(x):
374376
row=beam.Row(rank=2, values=[7, 8, 9])),
375377
]))
376378

379+
@unittest.skipIf(quickjs is None, 'quickjs-ng not installed.')
380+
def test_map_to_fields_js_non_serializable_types(self):
381+
with beam.Pipeline(options=beam.options.pipeline_options.PipelineOptions(
382+
pickle_library='cloudpickle', yaml_experimental_features=['javascript'
383+
])) as p:
384+
data = [
385+
beam.Row(
386+
b=b'hello',
387+
dt=datetime.datetime(2026, 5, 14, 12, 0, 0),
388+
dec=Decimal('10.5'))
389+
]
390+
elements = p | beam.Create(data)
391+
result = elements | YamlTransform(
392+
'''
393+
type: MapToFields
394+
config:
395+
language: javascript
396+
fields:
397+
b_out:
398+
expression: "b + '_world'"
399+
dt_out:
400+
expression: "dt"
401+
dec_out:
402+
expression: "dec * 2"
403+
''')
404+
assert_that(
405+
result,
406+
equal_to([
407+
beam.Row(
408+
b_out='hello_world',
409+
dt_out='2026-05-14T12:00:00',
410+
dec_out=21.0),
411+
]))
412+
377413

378414
if __name__ == '__main__':
379415
logging.getLogger().setLevel(logging.INFO)

0 commit comments

Comments
 (0)