@@ -39,7 +39,7 @@ class VeloxIteratorApiFsConfSuite extends SharedSparkSession {
3939 WholeStageTransformContext (PlanBuilder .empty())
4040
4141 test(" genPartitions embeds fs.azure.* keys from Hadoop conf into GlutenPartition.fsConf" ) {
42- val hadoopConf = spark.sparkContext.hadoopConfiguration
42+ val hadoopConf = spark.sessionState.newHadoopConf()
4343 hadoopConf.set(" fs.azure.account.auth.type.myaccount.dfs.core.windows.net" , " OAuth" )
4444 hadoopConf.set(" fs.azure.account.oauth.provider.type" , " ClientCredentials" )
4545 try {
@@ -61,7 +61,7 @@ class VeloxIteratorApiFsConfSuite extends SharedSparkSession {
6161 }
6262
6363 test(" genPartitions embeds fs.s3a.* keys from Hadoop conf into GlutenPartition.fsConf" ) {
64- val hadoopConf = spark.sparkContext.hadoopConfiguration
64+ val hadoopConf = spark.sessionState.newHadoopConf()
6565 hadoopConf.set(" fs.s3a.access.key" , " AKIAIOSFODNN7EXAMPLE" )
6666 hadoopConf.set(" fs.s3a.secret.key" , " wJalrXUtnFEMI" )
6767 try {
@@ -78,7 +78,7 @@ class VeloxIteratorApiFsConfSuite extends SharedSparkSession {
7878 }
7979
8080 test(" genPartitions embeds fs.gs.* keys from Hadoop conf into GlutenPartition.fsConf" ) {
81- val hadoopConf = spark.sparkContext.hadoopConfiguration
81+ val hadoopConf = spark.sessionState.newHadoopConf()
8282 hadoopConf.set(" fs.gs.auth.service.account.json.keyfile" , " /tmp/sa.json" )
8383 try {
8484 val partitions = api.genPartitions(emptyWsCtx, Seq (Seq .empty), Seq .empty)
@@ -94,7 +94,7 @@ class VeloxIteratorApiFsConfSuite extends SharedSparkSession {
9494 }
9595
9696 test(" genPartitions does not include non-fs.* keys in GlutenPartition.fsConf" ) {
97- val hadoopConf = spark.sparkContext.hadoopConfiguration
97+ val hadoopConf = spark.sessionState.newHadoopConf()
9898 hadoopConf.set(" fs.s3a.access.key" , " KEY" )
9999 hadoopConf.set(" spark.some.conf" , " value" )
100100 hadoopConf.set(" mapreduce.input.fileinputformat.split.maxsize" , " 128000000" )
@@ -115,7 +115,7 @@ class VeloxIteratorApiFsConfSuite extends SharedSparkSession {
115115
116116 test(" genPartitions produces empty fsConf when no fs.* keys are set" ) {
117117 // Use a key guaranteed not to exist in the Hadoop conf under any test profile.
118- val hadoopConf = spark.sparkContext.hadoopConfiguration
118+ val hadoopConf = spark.sessionState.newHadoopConf()
119119 val uniqueKey = " fs.azure.__test_only_unique_key__"
120120 hadoopConf.unset(uniqueKey)
121121 // Count only keys matching our prefixes.
0 commit comments