Skip to content

Commit 4307a51

Browse files
author
Tobias Kaymak
committed
fix: log warning when neither WrappedArray nor ArraySeq class is found
Per /ultrareview feedback: the fallback branch silently swallowed the second ClassNotFoundException. In practice one of the two classes is always present (Scala 2.12 vs 2.13 stdlib), but a silent skip could mask a broken classpath. Emit a LOG.warn instead.
1 parent 1863616 commit 4307a51

1 file changed

Lines changed: 9 additions & 1 deletion

File tree

runners/spark/src/main/java/org/apache/beam/runners/spark/coders/SparkRunnerKryoRegistrator.java

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
import org.apache.beam.sdk.values.TupleTag;
3131
import org.apache.beam.vendor.guava.v32_1_2_jre.com.google.common.collect.HashBasedTable;
3232
import org.apache.spark.serializer.KryoRegistrator;
33+
import org.slf4j.Logger;
34+
import org.slf4j.LoggerFactory;
3335

3436
/**
3537
* Custom {@link KryoRegistrator}s for Beam's Spark runner needs and registering used class in spark
@@ -45,6 +47,8 @@
4547
})
4648
public class SparkRunnerKryoRegistrator implements KryoRegistrator {
4749

50+
private static final Logger LOG = LoggerFactory.getLogger(SparkRunnerKryoRegistrator.class);
51+
4852
@Override
4953
public void registerClasses(Kryo kryo) {
5054
// MicrobatchSource is serialized as data and may not be Kryo-serializable.
@@ -67,7 +71,11 @@ public void registerClasses(Kryo kryo) {
6771
try {
6872
kryo.register(Class.forName("scala.collection.mutable.WrappedArray$ofRef"));
6973
} catch (ClassNotFoundException ignored) {
70-
// Neither class found; skip registration
74+
LOG.warn(
75+
"Neither scala.collection.mutable.ArraySeq$ofRef (Scala 2.13) nor "
76+
+ "scala.collection.mutable.WrappedArray$ofRef (Scala 2.12) was found on the "
77+
+ "classpath. Kryo serialization of Scala wrapped arrays will fall back to Java "
78+
+ "serialization or fail at runtime if spark.kryo.registrationRequired is true.");
7179
}
7280
}
7381

0 commit comments

Comments
 (0)