Skip to content

Commit 07ed459

Browse files
c21c21
committed
Address comments and try to fix unit test failure
1 parent f5a06ba commit 07ed459

File tree

30 files changed

+1298
-1277
lines changed

30 files changed

+1298
-1277
lines changed

‎sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastNestedLoopJoinExec.scala

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -412,9 +412,9 @@ case class BroadcastNestedLoopJoinExec(
412412
override def doConsume(ctx: CodegenContext, input: Seq[ExprCode], row: ExprCode): String = {
413413
joinType match {
414414
case _: InnerLike => codegenInner(ctx, input)
415-
case x =>
415+
case _ =>
416416
throw new IllegalArgumentException(
417-
s"BroadcastNestedLoopJoin code-gen should not take $x as the JoinType")
417+
s"BroadcastNestedLoopJoin code-gen should not take $joinType as the JoinType")
418418
}
419419
}
420420

@@ -427,12 +427,12 @@ case class BroadcastNestedLoopJoinExec(
427427
val broadcastTerm = ctx.addReferenceObj("broadcastTerm", broadcastArray)
428428

429429
// Inline mutable state since not many join operations in a task
430-
ctx.addMutableState("InternalRow[]", "broadcastArray",
430+
ctx.addMutableState("InternalRow[]", "buildRowArray",
431431
v => s"$v = (InternalRow[]) $broadcastTerm.value();", forceInline = true)
432432
}
433433

434434
private def codegenInner(ctx: CodegenContext, input: Seq[ExprCode]): String = {
435-
val arrayTerm = prepareBroadcast(ctx)
435+
val buildRowArrayTerm = prepareBroadcast(ctx)
436436
val (buildRow, checkCondition, buildVars) = getJoinCondition(ctx, input, streamed, broadcast)
437437

438438
val resultVars = buildSide match {
@@ -445,8 +445,8 @@ case class BroadcastNestedLoopJoinExec(
445445
s"""
446446
|int $arrayIndex = 0;
447447
|UnsafeRow $buildRow;
448-
|while ($arrayIndex < $arrayTerm.length) {
449-
| $buildRow = (UnsafeRow) $arrayTerm[$arrayIndex];
448+
|while ($arrayIndex < $buildRowArrayTerm.length) {
449+
| $buildRow = (UnsafeRow) $buildRowArrayTerm[$arrayIndex];
450450
| $checkCondition {
451451
| $numOutput.add(1);
452452
| ${consume(ctx, resultVars)}

‎sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/explain.txt

Lines changed: 46 additions & 46 deletions
Large diffs are not rendered by default.

‎sql/core/src/test/resources/tpcds-plan-stability/approved-plans-v1_4/q28.sf100/simplified.txt

Lines changed: 62 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
BroadcastNestedLoopJoin
1+
WholeStageCodegen (18)
22
BroadcastNestedLoopJoin
33
BroadcastNestedLoopJoin
44
BroadcastNestedLoopJoin
55
BroadcastNestedLoopJoin
6-
WholeStageCodegen (3)
6+
BroadcastNestedLoopJoin
77
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B1_LP,B1_CNT,B1_CNTD,sum,count,count,count]
88
InputAdapter
99
Exchange #1
@@ -19,88 +19,93 @@ BroadcastNestedLoopJoin
1919
ColumnarToRow
2020
InputAdapter
2121
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
22-
BroadcastExchange #3
23-
WholeStageCodegen (6)
24-
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count]
22+
InputAdapter
23+
BroadcastExchange #3
24+
WholeStageCodegen (5)
25+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B2_LP,B2_CNT,B2_CNTD,sum,count,count,count]
26+
InputAdapter
27+
Exchange #4
28+
WholeStageCodegen (4)
29+
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
30+
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
31+
InputAdapter
32+
Exchange [ss_list_price] #5
33+
WholeStageCodegen (3)
34+
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
35+
Project [ss_list_price]
36+
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
37+
ColumnarToRow
38+
InputAdapter
39+
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
40+
InputAdapter
41+
BroadcastExchange #6
42+
WholeStageCodegen (8)
43+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count]
44+
InputAdapter
45+
Exchange #7
46+
WholeStageCodegen (7)
47+
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
48+
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
49+
InputAdapter
50+
Exchange [ss_list_price] #8
51+
WholeStageCodegen (6)
52+
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
53+
Project [ss_list_price]
54+
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
55+
ColumnarToRow
56+
InputAdapter
57+
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
58+
InputAdapter
59+
BroadcastExchange #9
60+
WholeStageCodegen (11)
61+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count]
2562
InputAdapter
26-
Exchange #4
27-
WholeStageCodegen (5)
63+
Exchange #10
64+
WholeStageCodegen (10)
2865
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
2966
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
3067
InputAdapter
31-
Exchange [ss_list_price] #5
32-
WholeStageCodegen (4)
68+
Exchange [ss_list_price] #11
69+
WholeStageCodegen (9)
3370
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
3471
Project [ss_list_price]
3572
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
3673
ColumnarToRow
3774
InputAdapter
3875
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
39-
BroadcastExchange #6
40-
WholeStageCodegen (9)
41-
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B3_LP,B3_CNT,B3_CNTD,sum,count,count,count]
76+
InputAdapter
77+
BroadcastExchange #12
78+
WholeStageCodegen (14)
79+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count]
4280
InputAdapter
43-
Exchange #7
44-
WholeStageCodegen (8)
81+
Exchange #13
82+
WholeStageCodegen (13)
4583
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
4684
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
4785
InputAdapter
48-
Exchange [ss_list_price] #8
49-
WholeStageCodegen (7)
86+
Exchange [ss_list_price] #14
87+
WholeStageCodegen (12)
5088
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
5189
Project [ss_list_price]
5290
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
5391
ColumnarToRow
5492
InputAdapter
5593
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
56-
BroadcastExchange #9
57-
WholeStageCodegen (12)
58-
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B4_LP,B4_CNT,B4_CNTD,sum,count,count,count]
94+
InputAdapter
95+
BroadcastExchange #15
96+
WholeStageCodegen (17)
97+
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count]
5998
InputAdapter
60-
Exchange #10
61-
WholeStageCodegen (11)
99+
Exchange #16
100+
WholeStageCodegen (16)
62101
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
63102
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
64103
InputAdapter
65-
Exchange [ss_list_price] #11
66-
WholeStageCodegen (10)
104+
Exchange [ss_list_price] #17
105+
WholeStageCodegen (15)
67106
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
68107
Project [ss_list_price]
69108
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
70109
ColumnarToRow
71110
InputAdapter
72111
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
73-
BroadcastExchange #12
74-
WholeStageCodegen (15)
75-
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B5_LP,B5_CNT,B5_CNTD,sum,count,count,count]
76-
InputAdapter
77-
Exchange #13
78-
WholeStageCodegen (14)
79-
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
80-
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
81-
InputAdapter
82-
Exchange [ss_list_price] #14
83-
WholeStageCodegen (13)
84-
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
85-
Project [ss_list_price]
86-
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
87-
ColumnarToRow
88-
InputAdapter
89-
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]
90-
BroadcastExchange #15
91-
WholeStageCodegen (18)
92-
HashAggregate [sum,count,count,count] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),B6_LP,B6_CNT,B6_CNTD,sum,count,count,count]
93-
InputAdapter
94-
Exchange #16
95-
WholeStageCodegen (17)
96-
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),count(ss_list_price),sum,count,count,count,sum,count,count,count]
97-
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
98-
InputAdapter
99-
Exchange [ss_list_price] #17
100-
WholeStageCodegen (16)
101-
HashAggregate [ss_list_price] [avg(UnscaledValue(ss_list_price)),count(ss_list_price),sum,count,count,sum,count,count]
102-
Project [ss_list_price]
103-
Filter [ss_quantity,ss_list_price,ss_coupon_amt,ss_wholesale_cost]
104-
ColumnarToRow
105-
InputAdapter
106-
Scan parquet default.store_sales [ss_quantity,ss_wholesale_cost,ss_list_price,ss_coupon_amt,ss_sold_date_sk]

0 commit comments

Comments
 (0)