File tree Expand file tree Collapse file tree
spark/src/test/scala/org/apache/spark/sql Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -332,38 +332,6 @@ abstract class CometTestBase
332332 }
333333 }
334334
335- // inspired from spark-testing-base
336- protected def assertDataFrameEquals (
337- df : => DataFrame ,
338- checkNativeOperators : Boolean = true ): Unit = {
339-
340- var sparkDf : DataFrame = null
341- withSQLConf(CometConf .COMET_ENABLED .key -> " false" ) {
342- sparkDf = datasetOfRows(spark, df.logicalPlan)
343- }
344- val cometDf = datasetOfRows(spark, df.logicalPlan)
345-
346- // schema match check
347- assert(
348- sparkDf.schema == cometDf.schema,
349- s " Schemas do not match. \n Correct Answer: ${sparkDf.schema}\n Spark Answer: ${cometDf.schema}" )
350-
351- // diff check using except instead of collect (collect errors out on spark driver for long -> timestamp conv)
352- val sparkMinusComet = sparkDf.except(cometDf)
353- val cometMinusSpark = cometDf.except(sparkDf)
354-
355- val diffCount1 = sparkMinusComet.count()
356- val diffCount2 = cometMinusSpark.count()
357-
358- if (diffCount1 != 0 || diffCount2 != 0 ) {
359- fail(" DataFrames are not equal.\n " )
360- }
361-
362- if (checkNativeOperators) {
363- checkCometOperators(stripAQEPlan(df.queryExecution.executedPlan))
364- }
365- }
366-
367335 /**
368336 * A helper function for comparing Comet DataFrame with Spark result using absolute tolerance.
369337 */
You can’t perform that action at this time.
0 commit comments