Skip to content

Commit e5a79ed

Browse files
fenzhuGitHub Enterprise
authored andcommitted
[CARMEL-6076][Followup] Fix UT failure caused by compacting empty directory (#1010)
* [CARMEL-6076][Followup] Fix UT failure caused by compacting empty directory dddd tttt * show path
1 parent 01fb336 commit e5a79ed

File tree

2 files changed

+15
-9
lines changed

2 files changed

+15
-9
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2521,11 +2521,11 @@ object SQLConf {
25212521
val CONCURRENT_READ_LOCK_CLUSTER =
25222522
buildConf("spark.sql.source.output.concurrent.read.lock.cluster")
25232523
.doc("Clusters that enable read lock for empty path listing. We can configure" +
2524-
"it as comma seperated string list, e.g., 'hermes,apollo'.")
2524+
"it as comma separated string list, e.g., 'hermes,apollo'.")
25252525
.version("3.0.0")
25262526
.internal()
25272527
.stringConf
2528-
.createWithDefault("hermes")
2528+
.createWithDefault("all")
25292529

25302530
val TABLE_COMPACT_FLAG_PATH =
25312531
buildConf("spark.sql.source.table.compact.path")

sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/InMemoryFileIndex.scala

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -216,14 +216,20 @@ object InMemoryFileIndex extends Logging {
216216
))
217217

218218
val validTableLocation: Option[URI] = if (tableLocation != null && tableLocation.isDefined) {
219-
val tl = tableLocation.get.getPath.trim.toLowerCase(Locale.ROOT)
219+
val tl = new Path(tableLocation.get.getPath.trim)
220+
val fs = SparkHadoopWriterUtils.getNewFileSystem(tl, hadoopConf)
221+
val tlp = fs.makeQualified(tl).toString.toLowerCase(Locale.ROOT)
220222
val readLockCluster = sparkSession.sessionState.conf.concurrentReadLockCluster
221-
val readLockClusters: Seq[String] =
222-
readLockCluster.split(",").toSeq.map(_.trim.toLowerCase(Locale.ROOT))
223-
logInfo(s"Configured cluster list to enable read lock: $readLockClusters")
224-
val enforced: Boolean = readLockClusters.exists(cluster => tl.contains(cluster))
225-
logInfo(s"Table location is: $tl, read lock enforced = $enforced")
226-
if (enforced) tableLocation else null
223+
logInfo(s"Configured cluster that enables read lock for empty directory: $readLockCluster")
224+
if (readLockCluster.trim.equalsIgnoreCase("all")) {
225+
tableLocation
226+
} else {
227+
val readLockClusters: Seq[String] =
228+
readLockCluster.split(",").toSeq.map(_.trim.toLowerCase(Locale.ROOT))
229+
val enforced: Boolean = readLockClusters.exists(cluster => tlp.contains(cluster))
230+
logInfo(s"Table location is: $tlp, read lock enforced = $enforced")
231+
if (enforced) tableLocation else null
232+
}
227233
} else {
228234
tableLocation
229235
}

0 commit comments

Comments
 (0)