Skip to content

Commit e2e67b0

Browse files
Madhavanclaude
andcommitted
fix(connector): make testSchema tolerant of non-shaded Avro and JSON whole-number doubles
Two pre-existing failures in the connector PulsarCassandraSourceTests.testSchema (across Avro and JSON variants), surfaced by running the suite on jdk 17 as well as 11. Neither is caused by the Kafka work; they stem from the messaging-abstraction migration and were partially addressed before (see 51f872b, which made assertMapsEqual tolerant of shaded/non-shaded Avro Utf8). 1) Avro: java.lang.ClassCastException casting org.apache.avro.generic.GenericData$Array to the Pulsar-shaded GenericData$Array. Pulsar's GenericRecord.getField() returns collection (array) columns as NON-shaded Avro objects even though nested records come back shaded, so the shaded-typed assertions throw. Fix: normalizeToShadedAvro() round-trips any non-shaded Avro GenericContainer through binary Avro into the shaded type in genericRecordToMap(), deeply converting arrays and everything nested (records, maps, strings, CQL logical-type records) so the existing shaded assertions work unchanged. The writer uses GenericData.get() (no registered conversions) so raw datums are written verbatim. Also make the map-of-tuple assertion look up its key by string and normalize the tuple value. 2) JSON: "double expected:<1.0> but was:<1>". A whole-number double/float is serialized to JSON as `1`, which Jackson reads as an IntNode, so a type-sensitive equals against the expected Double fails. Compare double/float numerically (via asDouble) instead. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
1 parent ebdc30f commit e2e67b0

7 files changed

Lines changed: 57 additions & 2362 deletions

File tree

connector/src/test/java/com/datastax/oss/pulsar/source/PulsarCassandraSourceTests.java

Lines changed: 57 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -774,7 +774,15 @@ void assertGenericMap(String field, Map<Utf8, Object> gm) {
774774
case "mapoftuple":
775775
log.debug("field={} gm={}", field, gm);
776776
Assert.assertEquals("Incorrect size of map", gm.size(), 1);
777-
assertAvroTupleRecord((GenericData.Record) gm.get(new Utf8("a")));
777+
// The map key may be a shaded or non-shaded Avro Utf8 depending on how Pulsar
778+
// returned it; look it up by string value. The tuple record value may likewise be
779+
// non-shaded, so normalize it to shaded before asserting.
780+
Object tupleValue = gm.entrySet().stream()
781+
.filter(e -> e.getKey().toString().equals("a"))
782+
.map(Map.Entry::getValue)
783+
.findFirst()
784+
.orElse(null);
785+
assertAvroTupleRecord((GenericData.Record) normalizeToShadedAvro(tupleValue));
778786
return;
779787
}
780788
Assert.assertTrue("Unexpected field="+field, false);
@@ -943,10 +951,17 @@ void assertJsonNode(String field, JsonNode node) {
943951
case "tinyint":
944952
case "smallint":
945953
case "int":
946-
case "bigint":
954+
case "bigint": {
955+
Assert.assertEquals("Wrong value for regular field " + field, dataSpecMap.get(field).jsonValue(), node.numberValue());
956+
}
957+
return;
947958
case "double":
948959
case "float": {
949-
Assert.assertEquals("Wrong value for regular field " + field, dataSpecMap.get(field).jsonValue(), node.numberValue());
960+
// A whole-number double/float (e.g. 1.0) is serialized to JSON as `1`, which Jackson
961+
// reads back as an IntNode -- so numberValue() yields an Integer, not a Double, and an
962+
// exact type-sensitive equals fails. Compare numerically instead.
963+
Assert.assertEquals("Wrong value for regular field " + field,
964+
((Number) dataSpecMap.get(field).jsonValue()).doubleValue(), node.asDouble(), 0.0);
950965
}
951966
return;
952967
case "set": {
@@ -1347,13 +1362,51 @@ Map<String, Object> genericRecordToMap(GenericRecord genericRecord) {
13471362
return jsonNodeToMap((JsonNode) genericRecord.getNativeObject());
13481363
} else {
13491364
for (Field field : genericRecord.getFields()) {
1350-
map.put(field.getName(), genericRecord.getField(field));
1365+
map.put(field.getName(), normalizeToShadedAvro(genericRecord.getField(field)));
13511366
}
13521367
}
13531368

13541369
return map;
13551370
}
13561371

1372+
/**
1373+
* Normalize a value that Pulsar returned as a NON-shaded Avro object
1374+
* ({@code org.apache.avro.*}) into the Pulsar-shaded equivalent
1375+
* ({@code org.apache.pulsar.shade.org.apache.avro.*}) by round-tripping it through binary Avro.
1376+
* <p>
1377+
* Pulsar's {@code GenericRecord.getField()} returns collection (array) columns as non-shaded
1378+
* Avro arrays even though nested records come back shaded, so a direct cast to the shaded
1379+
* {@code GenericData.Array} (as the assertions below do) throws {@link ClassCastException}.
1380+
* Round-tripping deeply converts the array and everything nested inside it (records, maps,
1381+
* strings, CQL logical-type records) to shaded objects, so the existing shaded-typed assertions
1382+
* work unchanged. Non-{@code GenericContainer} values (primitives, Maps, Strings, and values
1383+
* that are already shaded) are returned unchanged.
1384+
*/
1385+
private static Object normalizeToShadedAvro(Object value) {
1386+
if (!(value instanceof org.apache.avro.generic.GenericContainer)) {
1387+
return value;
1388+
}
1389+
try {
1390+
org.apache.avro.Schema nonShadedSchema = ((org.apache.avro.generic.GenericContainer) value).getSchema();
1391+
java.io.ByteArrayOutputStream out = new java.io.ByteArrayOutputStream();
1392+
org.apache.avro.io.BinaryEncoder encoder = org.apache.avro.io.EncoderFactory.get().binaryEncoder(out, null);
1393+
org.apache.avro.generic.GenericDatumWriter<Object> writer =
1394+
new org.apache.avro.generic.GenericDatumWriter<>(nonShadedSchema);
1395+
writer.write(value, encoder);
1396+
encoder.flush();
1397+
1398+
Schema shadedSchema = new Schema.Parser().parse(nonShadedSchema.toString());
1399+
org.apache.pulsar.shade.org.apache.avro.io.BinaryDecoder decoder =
1400+
org.apache.pulsar.shade.org.apache.avro.io.DecoderFactory.get()
1401+
.binaryDecoder(out.toByteArray(), null);
1402+
org.apache.pulsar.shade.org.apache.avro.generic.GenericDatumReader<Object> reader =
1403+
new org.apache.pulsar.shade.org.apache.avro.generic.GenericDatumReader<>(shadedSchema);
1404+
return reader.read(null, decoder);
1405+
} catch (java.io.IOException e) {
1406+
throw new RuntimeException("Failed to normalize non-shaded Avro value to shaded", e);
1407+
}
1408+
}
1409+
13571410
static Map<String, Object> jsonNodeToMap(JsonNode jsonNode) {
13581411
Map<String, Object> map = new HashMap<>();
13591412
for (Iterator<String> it = jsonNode.fieldNames(); it.hasNext(); ) {

0 commit comments

Comments
 (0)