Skip to content

Commit 20de5e7

Browse files
LuciferYangNgone51
authored andcommitted
[SPARK-54884][CORE][TESTS] Remove private function totalRunningTasksPerResourceProfile from ExecutorAllocationManager
### What changes were proposed in this pull request? This pr removes the private function `totalRunningTasksPerResourceProfile` from `ExecutorAllocationManager` and refactors `ExecutorAllocationManagerSuite` so that it no longer relies on this private function, which was solely used for testing purposes. ### Why are the changes needed? Code cleanup. ### Does this PR introduce _any_ user-facing change? No ### How was this patch tested? Pass Github Actions ### Was this patch authored or co-authored using generative AI tooling? No Closes #53662 from LuciferYang/remove-totalRunningTasksPerResourceProfile. Authored-by: yangjie01 <yangjie01@baidu.com> Signed-off-by: Yi Wu <yi.wu@databricks.com>
1 parent 2f12bc6 commit 20de5e7

2 files changed

Lines changed: 4 additions & 11 deletions

File tree

core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -329,12 +329,6 @@ private[spark] class ExecutorAllocationManager(
329329
}
330330
}
331331

332-
// Please do not delete this function, the tests in `ExecutorAllocationManagerSuite`
333-
// need to access `listener.totalRunningTasksPerResourceProfile` with `synchronized`.
334-
private def totalRunningTasksPerResourceProfile(id: Int): Int = synchronized {
335-
listener.totalRunningTasksPerResourceProfile(id)
336-
}
337-
338332
/**
339333
* This is called at a fixed interval to regulate the number of pending executor requests
340334
* and number of executors running.

core/src/test/scala/org/apache/spark/ExecutorAllocationManagerSuite.scala

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1924,8 +1924,6 @@ private object ExecutorAllocationManagerSuite extends PrivateMethodTester {
19241924
PrivateMethod[mutable.HashMap[Int, Int]](Symbol("numLocalityAwareTasksPerResourceProfileId"))
19251925
private val _rpIdToHostToLocalTaskCount =
19261926
PrivateMethod[Map[Int, Map[String, Int]]](Symbol("rpIdToHostToLocalTaskCount"))
1927-
private val _totalRunningTasksPerResourceProfile =
1928-
PrivateMethod[Int](Symbol("totalRunningTasksPerResourceProfile"))
19291927

19301928
private val defaultProfile = ResourceProfile.getOrCreateDefaultProfile(new SparkConf)
19311929

@@ -2022,9 +2020,10 @@ private object ExecutorAllocationManagerSuite extends PrivateMethodTester {
20222020
localMap(defaultProfile.id)
20232021
}
20242022

2025-
private def totalRunningTasksPerResourceProfile(manager: ExecutorAllocationManager): Int = {
2026-
manager invokePrivate _totalRunningTasksPerResourceProfile(defaultProfile.id)
2027-
}
2023+
private def totalRunningTasksPerResourceProfile(manager: ExecutorAllocationManager): Int =
2024+
manager.synchronized {
2025+
manager.listener.totalRunningTasksPerResourceProfile(defaultProfile.id)
2026+
}
20282027

20292028
private def hostToLocalTaskCount(
20302029
manager: ExecutorAllocationManager): Map[String, Int] = {

0 commit comments

Comments
 (0)