address feedback

Author: andygrove

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastBooleanBenchmark.scala

@@ -77,6 +77,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Boolean to other types casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, 50/50 true/false
           prepareTable(
             dir,
             spark.sql(s"""
@@ -98,6 +99,7 @@ object CometCastBooleanBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Numeric to Boolean casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL per column, values in {-1, 0, 1} (~33% each)
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToNumericBenchmark.scala

@@ -98,7 +98,13 @@ object CometCastNumericToNumericBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Numeric to Numeric casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
-          // Generate varied numeric data including edge cases
+          // Data distribution: 1% NULL per column
+          // - c_byte: full range -64 to 63
+          // - c_short: full range -16384 to 16383
+          // - c_int: centered around 0 (-2.5M to +2.5M)
+          // - c_long: large positive values (0 to ~5 billion)
+          // - c_float/c_double: 4% special values (NaN/Infinity), rest centered around 0
+          // - c_decimal: values from -25000.00 to +25000.00
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToStringBenchmark.scala

@@ -60,7 +60,13 @@ object CometCastNumericToStringBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Numeric to String casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
-          // Generate varied numeric data including edge cases
+          // Data distribution: 1% NULL per column
+          // - c_bool: 50/50 true/false
+          // - c_byte: full range -64 to 63
+          // - c_short: full range -16384 to 16383
+          // - c_int/c_long: large values centered around 0
+          // - c_float/c_double: 3% special values (NaN/Infinity), rest centered around 0
+          // - c_decimal: values from -25000.00 to +25000.00
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastNumericToTemporalBenchmark.scala

@@ -58,8 +58,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Int to Date casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
-          // Generate INT values representing days since epoch (1970-01-01)
-          // Range: ~-18000 to +18000 days (roughly 1920 to 2020)
+          // Data distribution: 1% NULL, days since epoch spanning ~100 years (1920-2020)
           prepareTable(
             dir,
             spark.sql(s"""
@@ -81,8 +80,7 @@ object CometCastNumericToTemporalBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Long to Timestamp casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
-          // Generate LONG values representing microseconds since epoch
-          // Range: 2020-2021 timestamps
+          // Data distribution: 1% NULL, microseconds since epoch spanning ~1 year from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToNumericBenchmark.scala

@@ -68,8 +68,11 @@ object CometCastStringToNumericBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("String to numeric casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
-          // Generate numeric strings with both integer and decimal values
-          // Also include some special values: nulls (~2%), NaN (~2%), Infinity (~2%)
+          // Data distribution:
+          // - 2% NULL, 2% 'NaN', 2% 'Infinity', 2% '-Infinity'
+          // - 12% small integers (0-98)
+          // - 40% medium integers (0-999,998)
+          // - 40% decimals centered around 0 (approx -5000.00 to +5000.00)
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastStringToTemporalBenchmark.scala

@@ -58,6 +58,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("date data generation", values) { v =>
       withTempPath { dateDir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 10% invalid strings, 90% valid date strings spanning ~10 years
           prepareTable(
             dateDir,
             spark.sql(s"""
@@ -80,6 +81,7 @@ object CometCastStringToTemporalBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("timestamp data generation", values) { v =>
       withTempPath { timestampDir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 10% invalid strings, 90% valid timestamp strings (1970 epoch range)
           prepareTable(
             timestampDir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToNumericBenchmark.scala

@@ -62,6 +62,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Date to Numeric casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, dates spanning ~10 years from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""
@@ -83,6 +84,7 @@ object CometCastTemporalToNumericBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Timestamp to Numeric casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, timestamps spanning ~1 year from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToStringBenchmark.scala

@@ -55,6 +55,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Date to String casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, dates spanning ~10 years from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""
@@ -76,6 +77,7 @@ object CometCastTemporalToStringBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Timestamp to String casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, timestamps spanning ~1 year from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""

spark/src/test/scala/org/apache/spark/sql/benchmark/CometCastTemporalToTemporalBenchmark.scala

@@ -57,6 +57,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Date to Timestamp casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, dates spanning ~10 years from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""
@@ -78,6 +79,7 @@ object CometCastTemporalToTemporalBenchmark extends CometBenchmarkBase {
     runBenchmarkWithTable("Timestamp to Date casts", values) { v =>
       withTempPath { dir =>
         withTempTable("parquetV1Table") {
+          // Data distribution: 1% NULL, timestamps spanning ~1 year from 2020-01-01
           prepareTable(
             dir,
             spark.sql(s"""