Skip to content
This repository was archived by the owner on Feb 24, 2026. It is now read-only.

Commit 7862f38

Browse files
lqiu96lqiu96
committed
chore: Add edge cases for user input
1 parent 6f3f4c6 commit 7862f38

4 files changed

Lines changed: 291 additions & 78 deletions

File tree

‎google-cloud-bigquerystorage/src/main/java/com/google/cloud/bigquery/storage/v1/JsonToProtoMessage.java‎

Lines changed: 72 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,11 @@
1515
*/
1616
package com.google.cloud.bigquery.storage.v1;
1717

18+
import static java.time.temporal.ChronoField.HOUR_OF_DAY;
19+
import static java.time.temporal.ChronoField.MINUTE_OF_HOUR;
20+
import static java.time.temporal.ChronoField.NANO_OF_SECOND;
21+
import static java.time.temporal.ChronoField.SECOND_OF_MINUTE;
22+
1823
import com.google.api.pathtemplate.ValidationException;
1924
import com.google.cloud.bigquery.storage.v1.Exceptions.RowIndexToErrorException;
2025
import com.google.common.annotations.VisibleForTesting;
@@ -69,7 +74,31 @@ public class JsonToProtoMessage implements ToProtoConverter<Object> {
6974
.put(FieldDescriptor.Type.STRING, "string")
7075
.put(FieldDescriptor.Type.MESSAGE, "object")
7176
.build();
72-
private static final DateTimeFormatter TIMESTAMP_FORMATTER =
77+
78+
private static final DateTimeFormatter TO_TIMESTAMP_FORMATTER =
79+
new DateTimeFormatterBuilder()
80+
.parseLenient()
81+
.append(DateTimeFormatter.ISO_LOCAL_DATE)
82+
.optionalStart()
83+
.appendLiteral('T')
84+
.optionalEnd()
85+
.appendValue(HOUR_OF_DAY, 2)
86+
.appendLiteral(':')
87+
.appendValue(MINUTE_OF_HOUR, 2)
88+
.optionalStart()
89+
.appendLiteral(':')
90+
.appendValue(SECOND_OF_MINUTE, 2)
91+
.optionalEnd()
92+
.optionalStart()
93+
.appendFraction(NANO_OF_SECOND, 6, 9, true)
94+
.optionalEnd()
95+
.optionalStart()
96+
.appendOffset("+HHMM", "+00:00")
97+
.optionalEnd()
98+
.toFormatter()
99+
.withZone(ZoneOffset.UTC);
100+
101+
private static final DateTimeFormatter FROM_TIMESTAMP_FORMATTER =
73102
new DateTimeFormatterBuilder()
74103
.parseLenient()
75104
.append(DateTimeFormatter.ofPattern("yyyy[/][-]MM[/][-]dd"))
@@ -634,25 +663,8 @@ private void fillField(
634663
return;
635664
}
636665
} else if (fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
637-
if (val instanceof String) {
638-
Double parsed = Doubles.tryParse((String) val);
639-
if (parsed != null) {
640-
protoMsg.setField(fieldDescriptor, parsed.longValue());
641-
return;
642-
}
643-
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
644-
protoMsg.setField(
645-
fieldDescriptor,
646-
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
647-
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
648-
return;
649-
} else if (val instanceof Long) {
650-
protoMsg.setField(fieldDescriptor, val);
651-
return;
652-
} else if (val instanceof Integer) {
653-
protoMsg.setField(fieldDescriptor, Long.valueOf((Integer) val));
654-
return;
655-
}
666+
protoMsg.setField(fieldDescriptor, getTimestampAsLong(val));
667+
return;
656668
}
657669
}
658670
if (val instanceof Integer) {
@@ -919,24 +931,7 @@ private void fillRepeatedField(
919931
}
920932
} else if (fieldSchema != null
921933
&& fieldSchema.getType() == TableFieldSchema.Type.TIMESTAMP) {
922-
if (val instanceof String) {
923-
Double parsed = Doubles.tryParse((String) val);
924-
if (parsed != null) {
925-
protoMsg.addRepeatedField(fieldDescriptor, parsed.longValue());
926-
} else {
927-
TemporalAccessor parsedTime = TIMESTAMP_FORMATTER.parse((String) val);
928-
protoMsg.addRepeatedField(
929-
fieldDescriptor,
930-
parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
931-
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND));
932-
}
933-
} else if (val instanceof Long) {
934-
protoMsg.addRepeatedField(fieldDescriptor, val);
935-
} else if (val instanceof Integer) {
936-
protoMsg.addRepeatedField(fieldDescriptor, Long.valueOf((Integer) val));
937-
} else {
938-
throwWrongFieldType(fieldDescriptor, currentScope, index);
939-
}
934+
protoMsg.addRepeatedField(fieldDescriptor, getTimestampAsLong(val));
940935
} else if (val instanceof Integer) {
941936
protoMsg.addRepeatedField(fieldDescriptor, Long.valueOf((Integer) val));
942937
} else if (val instanceof Long) {
@@ -1050,18 +1045,51 @@ static Instant fromEpochMicros(long micros) {
10501045
@VisibleForTesting
10511046
static String getTimestampAsString(Object val) {
10521047
if (val instanceof String) {
1053-
// Validate the ISO8601 values before sending it to the server
10541048
String value = (String) val;
1049+
Double parsed = Doubles.tryParse(value);
1050+
// If true, it was a numeric value inside a String
1051+
if (parsed != null) {
1052+
return getTimestampAsString(parsed.longValue());
1053+
}
1054+
// Validate the ISO8601 values before sending it to the server. No need to format
1055+
// if it's valid.
10551056
validateTimestamp(value);
1056-
return value;
1057-
} else if (val instanceof Short || val instanceof Integer || val instanceof Long) {
1057+
1058+
// If it's high precision (more than 9 digits), then return the ISO8601 string as-is
1059+
// as JDK does not have a DateTimeFormatter that supports more than nanosecond precision.
1060+
Matcher matcher = ISO8601_TIMESTAMP_HIGH_PRECISION_PATTERN.matcher(value);
1061+
if (matcher.find()) {
1062+
return value;
1063+
}
1064+
// Otherwise, output the timestamp to a standard format before sending it to BQ
1065+
Instant instant = FROM_TIMESTAMP_FORMATTER.parse(value, Instant::from);
1066+
return TO_TIMESTAMP_FORMATTER.format(instant);
1067+
} else if (val instanceof Number) {
10581068
// Micros from epoch will most likely will be represented a Long, but any non-float
10591069
// numeric value can be used
1060-
return fromEpochMicros((Long) val).toString();
1070+
Instant instant = fromEpochMicros(((Number) val).longValue());
1071+
return TO_TIMESTAMP_FORMATTER.format(instant);
10611072
} else if (val instanceof Timestamp) {
10621073
// Convert the Protobuf timestamp class to ISO8601 string
10631074
Timestamp timestamp = (Timestamp) val;
1064-
return Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()).toString();
1075+
return TO_TIMESTAMP_FORMATTER.format(
1076+
Instant.ofEpochSecond(timestamp.getSeconds(), timestamp.getNanos()));
1077+
}
1078+
throw new IllegalArgumentException("The timestamp value passed in is not from a valid type");
1079+
}
1080+
1081+
/* Best effort to try and convert the Object to a long (microseconds since epoch) */
1082+
private long getTimestampAsLong(Object val) {
1083+
if (val instanceof String) {
1084+
Double parsed = Doubles.tryParse((String) val);
1085+
if (parsed != null) {
1086+
return parsed.longValue();
1087+
}
1088+
TemporalAccessor parsedTime = FROM_TIMESTAMP_FORMATTER.parse((String) val);
1089+
return parsedTime.getLong(ChronoField.INSTANT_SECONDS) * 1000000
1090+
+ parsedTime.getLong(ChronoField.MICRO_OF_SECOND);
1091+
} else if (val instanceof Number) {
1092+
return ((Number) val).longValue();
10651093
}
10661094
throw new IllegalArgumentException("The timestamp value passed in is not from a valid type");
10671095
}
@@ -1107,7 +1135,7 @@ static void validateTimestamp(String timestamp) {
11071135

11081136
// It is valid as long as DateTimeFormatter doesn't throw an exception
11091137
try {
1110-
TIMESTAMP_FORMATTER.parse((String) timestamp);
1138+
FROM_TIMESTAMP_FORMATTER.parse((String) timestamp);
11111139
} catch (DateTimeParseException e) {
11121140
throw new IllegalArgumentException(e.getMessage(), e);
11131141
}

‎google-cloud-bigquerystorage/src/test/java/com/google/cloud/bigquery/storage/v1/BQTableSchemaToProtoDescriptorTest.java‎

Lines changed: 19 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -85,10 +85,9 @@ private void assertDesciptorsAreEqual(Descriptor expected, Descriptor actual) {
8585
FieldDescriptor.Type originalType = expectedField.getType();
8686
assertEquals(convertedField.getName(), convertedType, originalType);
8787
// Check mode
88-
assertTrue(
89-
(expectedField.isRepeated() == convertedField.isRepeated())
90-
&& (expectedField.isRequired() == convertedField.isRequired())
91-
&& (expectedField.isOptional() == convertedField.isOptional()));
88+
assertEquals(expectedField.isRepeated(), convertedField.isRepeated());
89+
assertEquals(expectedField.isRequired(), convertedField.isRequired());
90+
assertEquals(expectedField.isOptional(), convertedField.isOptional());
9291
// Recursively check nested messages
9392
if (convertedType == FieldDescriptor.Type.MESSAGE) {
9493
assertDesciptorsAreEqual(expectedField.getMessageType(), convertedField.getMessageType());
@@ -195,17 +194,6 @@ public void testRange() throws Exception {
195194
.setType(TableFieldSchema.Type.TIMESTAMP)
196195
.build())
197196
.build())
198-
.addFields(
199-
TableFieldSchema.newBuilder()
200-
.setName("range_timestamp_higher_precision_miXEd_caSE")
201-
.setType(TableFieldSchema.Type.RANGE)
202-
.setMode(TableFieldSchema.Mode.NULLABLE)
203-
.setRangeElementType(
204-
TableFieldSchema.FieldElementType.newBuilder()
205-
.setType(TableFieldSchema.Type.TIMESTAMP)
206-
.build())
207-
.setTimestampPrecision(Int64Value.newBuilder().setValue(12).build())
208-
.build())
209197
.build();
210198
final Descriptor descriptor =
211199
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);
@@ -424,6 +412,20 @@ public void testStructComplex() throws Exception {
424412
.setMode(TableFieldSchema.Mode.REPEATED)
425413
.setName("test_json")
426414
.build();
415+
final TableFieldSchema TEST_TIMESTAMP_HIGHER_PRECISION =
416+
TableFieldSchema.newBuilder()
417+
.setType(TableFieldSchema.Type.TIMESTAMP)
418+
.setMode(TableFieldSchema.Mode.NULLABLE)
419+
.setName("test_timestamp_higher_precision")
420+
.setTimestampPrecision(Int64Value.newBuilder().setValue(12).build())
421+
.build();
422+
final TableFieldSchema TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED =
423+
TableFieldSchema.newBuilder()
424+
.setType(TableFieldSchema.Type.TIMESTAMP)
425+
.setMode(TableFieldSchema.Mode.REPEATED)
426+
.setName("test_timestamp_higher_precision_repeated")
427+
.setTimestampPrecision(Int64Value.newBuilder().setValue(12).build())
428+
.build();
427429
final TableSchema tableSchema =
428430
TableSchema.newBuilder()
429431
.addFields(0, test_int)
@@ -457,6 +459,8 @@ public void testStructComplex() throws Exception {
457459
.addFields(28, TEST_BIGNUMERIC_DOUBLE)
458460
.addFields(29, TEST_INTERVAL)
459461
.addFields(30, TEST_JSON)
462+
.addFields(31, TEST_TIMESTAMP_HIGHER_PRECISION)
463+
.addFields(32, TEST_TIMESTAMP_HIGHER_PRECISION_REPEATED)
460464
.build();
461465
final Descriptor descriptor =
462466
BQTableSchemaToProtoDescriptor.convertBQTableSchemaToProtoDescriptor(tableSchema);

0 commit comments

Comments
 (0)