Skip to content

Commit f322271

Browse files
jerrypengHeartSaVioR
authored andcommitted
[SPARK-56042][SS] Fix swapped external/internal col family count metrics in RocksDBStateStoreProvider
### What changes were proposed in this pull request? Fix swapped external/internal col family count metrics in RocksDBStateStoreProvider ### Why are the changes needed? ### Does this PR introduce _any_ user-facing change? no ### How was this patch tested? tests added ### Was this patch authored or co-authored using generative AI tooling? no Closes #54874 from jerrypeng/SPARK-56042. Authored-by: Jerry Peng <jerry.peng@databricks.com> Signed-off-by: Jungtaek Lim <kabhwan.opensource@gmail.com>
1 parent a4dcd6d commit f322271

2 files changed

Lines changed: 11 additions & 2 deletions

File tree

sql/core/src/main/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreProvider.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -674,8 +674,8 @@ private[sql] class RocksDBStateStoreProvider
674674
CUSTOM_METRIC_FLUSH_WRITTEN_BYTES -> nativeOpsMetrics("totalBytesWrittenByFlush"),
675675
CUSTOM_METRIC_PINNED_BLOCKS_MEM_USAGE -> rocksDBMetrics.pinnedBlocksMemUsage,
676676
CUSTOM_METRIC_NUM_INTERNAL_COL_FAMILIES_KEYS -> rocksDBMetrics.numInternalKeys,
677-
CUSTOM_METRIC_NUM_EXTERNAL_COL_FAMILIES -> internalColFamilyCnt(),
678-
CUSTOM_METRIC_NUM_INTERNAL_COL_FAMILIES -> externalColFamilyCnt(),
677+
CUSTOM_METRIC_NUM_EXTERNAL_COL_FAMILIES -> externalColFamilyCnt(),
678+
CUSTOM_METRIC_NUM_INTERNAL_COL_FAMILIES -> internalColFamilyCnt(),
679679
CUSTOM_METRIC_NUM_SNAPSHOTS_AUTO_REPAIRED -> rocksDBMetrics.numSnapshotsAutoRepaired,
680680
CUSTOM_METRIC_FORCE_SNAPSHOT -> (if (forceSnapshotOnCommit) 1L else 0L)
681681
) ++ rocksDBMetrics.zipFileBytesUncompressed.map(bytes =>

sql/core/src/test/scala/org/apache/spark/sql/execution/streaming/state/RocksDBStateStoreSuite.scala

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2178,6 +2178,15 @@ class RocksDBStateStoreSuite extends StateStoreSuiteBase[RocksDBStateStoreProvid
21782178
val metricPair = store
21792179
.metrics.customMetrics.find(_._1.name == "rocksdbNumInternalColFamiliesKeys")
21802180
assert(metricPair.isDefined && metricPair.get._2 === 4)
2181+
// Verify external/internal column family count metrics
2182+
val externalColFamilyCount = store
2183+
.metrics.customMetrics.find(_._1.name == "rocksdbNumExternalColumnFamilies")
2184+
// 2 external col families: default + testColFamily
2185+
assert(externalColFamilyCount.isDefined && externalColFamilyCount.get._2 === 2)
2186+
val internalColFamilyCount = store
2187+
.metrics.customMetrics.find(_._1.name == "rocksdbNumInternalColumnFamilies")
2188+
// 1 internal col family: $testIndex
2189+
assert(internalColFamilyCount.isDefined && internalColFamilyCount.get._2 === 1)
21812190
val store1 = provider.getStore(1)
21822191
assert(rowPairsToDataSet(store1.iterator(cfName)) ===
21832192
Set(("a", 0) -> 1, ("b", 0) -> 2, ("c", 0) -> 3, ("d", 0) -> 4, ("e", 0) -> 5))

0 commit comments

Comments
 (0)