Skip to content

Commit 38c7d6e

Browse files
committed
Address PR comments
1 parent e231268 commit 38c7d6e

File tree

2 files changed

+11
-1
lines changed

2 files changed

+11
-1
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -558,7 +558,7 @@ case class FileSourceScanExec(
558558
}.getOrElse(Array.empty)
559559
FilePartition(bucketId, partitionedFiles)
560560
}
561-
} getOrElse {
561+
}.getOrElse {
562562
Seq.tabulate(bucketSpec.numBuckets) { bucketId =>
563563
FilePartition(bucketId, prunedFilesGroupedToBuckets.getOrElse(bucketId, Array.empty))
564564
}

sql/core/src/test/scala/org/apache/spark/sql/execution/bucketing/CoalesceBucketsInSortMergeJoinSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,4 +181,14 @@ class CoalesceBucketsInSortMergeJoinSuite extends SQLTestUtils with SharedSparkS
181181
isSortMergeJoin = true))
182182
}
183183
}
184+
185+
test("FileSourceScanExec's metadata should be updated with coalesced info") {
186+
val scan = newFileSourceScanExec(RelationSetting(8, None))
187+
val value = scan.metadata("SelectedBucketsCount")
188+
assert(value === "8 out of 8")
189+
190+
val scanWithCoalescing = scan.copy(optionalNumCoalescedBuckets = Some(4))
191+
val valueWithCoalescing = scanWithCoalescing.metadata("SelectedBucketsCount")
192+
assert(valueWithCoalescing == "8 out of 8 (Coalesced to 4)")
193+
}
184194
}

0 commit comments

Comments
 (0)