Skip to content

Commit be6b1d5

Browse files
pzzspzzs
committed
Update Optimizer.scala
1 parent b01e622 commit be6b1d5

File tree

1 file changed

+56
-82
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer

1 file changed

+56
-82
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

Lines changed: 56 additions & 82 deletions
Original file line numberDiff line numberDiff line change
@@ -36,23 +36,20 @@ object DefaultOptimizer extends Optimizer {
3636
// SubQueries are only needed for analysis and can be removed before execution.
3737
Batch("Remove SubQueries", FixedPoint(100),
3838
EliminateSubQueries) ::
39-
Batch("Transform Condition", FixedPoint(100),
40-
TransformCondition) ::
41-
Batch("Operator Reordering", FixedPoint(100),
39+
Batch("Operator Optimizations", FixedPoint(100),
4240
UnionPushdown,
4341
CombineFilters,
4442
PushPredicateThroughProject,
45-
PushPredicateThroughJoin,
4643
PushPredicateThroughGenerate,
4744
ColumnPruning,
4845
ProjectCollapsing,
49-
CombineLimits) ::
50-
Batch("ConstantFolding", FixedPoint(100),
46+
CombineLimits,
5147
NullPropagation,
5248
OptimizeIn,
5349
ConstantFolding,
5450
LikeSimplification,
5551
BooleanSimplification,
52+
PushPredicateThroughJoin,
5653
SimplifyFilters,
5754
SimplifyCasts,
5855
SimplifyCaseConversionExpressions) ::
@@ -62,80 +59,6 @@ object DefaultOptimizer extends Optimizer {
6259
ConvertToLocalRelation) :: Nil
6360
}
6461

65-
/**
66-
* Transform and/or Condition:
67-
* 1. a && a => a
68-
* 2. (a || b) && (a || c) => a || (b && c)
69-
* 3. a || a => a
70-
* 4. (a && b) || (a && c) => a && (b || c)
71-
*/
72-
object TransformCondition extends Rule[LogicalPlan] with PredicateHelper {
73-
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
74-
case q: LogicalPlan => q transformExpressionsUp {
75-
case and @ And(left, right) => (left, right) match {
76-
77-
// a && a => a
78-
case (l, r) if l fastEquals r => l
79-
// (a || b) && (a || c) => a || (b && c)
80-
case _ =>
81-
// 1. Split left and right to get the disjunctive predicates,
82-
// i.e. lhsSet = (a, b), rhsSet = (a, c)
83-
// 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
84-
// 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
85-
// 4. Apply the formula, get the optimized predicate: common || (ldiff && rdiff)
86-
val lhsSet = splitDisjunctivePredicates(left).toSet
87-
val rhsSet = splitDisjunctivePredicates(right).toSet
88-
val common = lhsSet.intersect(rhsSet)
89-
if (common.isEmpty) {
90-
// No common factors, return the original predicate
91-
and
92-
} else {
93-
val ldiff = lhsSet.diff(common)
94-
val rdiff = rhsSet.diff(common)
95-
if (ldiff.isEmpty || rdiff.isEmpty) {
96-
// (a || b || c || ...) && (a || b) => (a || b)
97-
common.reduce(Or)
98-
} else {
99-
// (a || b || c || ...) && (a || b || d || ...) =>
100-
// ((c || ...) && (d || ...)) || a || b
101-
(common + And(ldiff.reduce(Or), rdiff.reduce(Or))).reduce(Or)
102-
}
103-
}
104-
} // end of And(left, right)
105-
106-
case or @ Or(left, right) => (left, right) match {
107-
108-
case (l, r) if l fastEquals r => l
109-
// (a && b) || (a && c) => a && (b || c)
110-
case _ =>
111-
// 1. Split left and right to get the conjunctive predicates,
112-
// i.e. lhsSet = (a, b), rhsSet = (a, c)
113-
// 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
114-
// 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
115-
// 4. Apply the formula, get the optimized predicate: common && (ldiff || rdiff)
116-
val lhsSet = splitConjunctivePredicates(left).toSet
117-
val rhsSet = splitConjunctivePredicates(right).toSet
118-
val common = lhsSet.intersect(rhsSet)
119-
if (common.isEmpty) {
120-
// No common factors, return the original predicate
121-
or
122-
} else {
123-
val ldiff = lhsSet.diff(common)
124-
val rdiff = rhsSet.diff(common)
125-
if (ldiff.isEmpty || rdiff.isEmpty) {
126-
// (a && b) || (a && b && c && ...) => a && b
127-
common.reduce(And)
128-
} else {
129-
// (a && b && c && ...) || (a && b && d && ...) =>
130-
// ((c && ...) || (d && ...)) && a && b
131-
(common + Or(ldiff.reduce(And), rdiff.reduce(And))).reduce(And)
132-
}
133-
}
134-
} // end of Or(left, right)
135-
}
136-
}
137-
}
138-
13962
/**
14063
* Pushes operations to either side of a Union.
14164
*/
@@ -423,7 +346,32 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
423346
// l && false => false
424347
case (_, Literal(false, BooleanType)) => Literal(false)
425348
// a && a => a
426-
case _ => and
349+
case (l, r) if l fastEquals r => l
350+
// (a || b) && (a || c) => a || (b && c)
351+
case _ =>
352+
// 1. Split left and right to get the disjunctive predicates,
353+
// i.e. lhsSet = (a, b), rhsSet = (a, c)
354+
// 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
355+
// 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
356+
// 4. Apply the formula, get the optimized predicate: common || (ldiff && rdiff)
357+
val lhsSet = splitDisjunctivePredicates(left).toSet
358+
val rhsSet = splitDisjunctivePredicates(right).toSet
359+
val common = lhsSet.intersect(rhsSet)
360+
if (common.isEmpty) {
361+
// No common factors, return the original predicate
362+
and
363+
} else {
364+
val ldiff = lhsSet.diff(common)
365+
val rdiff = rhsSet.diff(common)
366+
if (ldiff.isEmpty || rdiff.isEmpty) {
367+
// (a || b || c || ...) && (a || b) => (a || b)
368+
common.reduce(Or)
369+
} else {
370+
// (a || b || c || ...) && (a || b || d || ...) =>
371+
// ((c || ...) && (d || ...)) || a || b
372+
(common + And(ldiff.reduce(Or), rdiff.reduce(Or))).reduce(Or)
373+
}
374+
}
427375
} // end of And(left, right)
428376

429377
case or @ Or(left, right) => (left, right) match {
@@ -435,7 +383,33 @@ object BooleanSimplification extends Rule[LogicalPlan] with PredicateHelper {
435383
case (Literal(false, BooleanType), r) => r
436384
// l || false => l
437385
case (l, Literal(false, BooleanType)) => l
438-
case _ => or
386+
// a || b = a
387+
case (l, r) if l fastEquals r => l
388+
// (a && b) || (a && c) => a && (b || c)
389+
case _ =>
390+
// 1. Split left and right to get the conjunctive predicates,
391+
// i.e. lhsSet = (a, b), rhsSet = (a, c)
392+
// 2. Find the common predict between lhsSet and rhsSet, i.e. common = (a)
393+
// 3. Remove common predict from lhsSet and rhsSet, i.e. ldiff = (b), rdiff = (c)
394+
// 4. Apply the formula, get the optimized predicate: common && (ldiff || rdiff)
395+
val lhsSet = splitConjunctivePredicates(left).toSet
396+
val rhsSet = splitConjunctivePredicates(right).toSet
397+
val common = lhsSet.intersect(rhsSet)
398+
if (common.isEmpty) {
399+
// No common factors, return the original predicate
400+
or
401+
} else {
402+
val ldiff = lhsSet.diff(common)
403+
val rdiff = rhsSet.diff(common)
404+
if (ldiff.isEmpty || rdiff.isEmpty) {
405+
// (a && b) || (a && b && c && ...) => a && b
406+
common.reduce(And)
407+
} else {
408+
// (a && b && c && ...) || (a && b && d && ...) =>
409+
// ((c && ...) || (d && ...)) && a && b
410+
(common + Or(ldiff.reduce(And), rdiff.reduce(And))).reduce(And)
411+
}
412+
}
439413
} // end of Or(left, right)
440414

441415
case not @ Not(exp) => exp match {

0 commit comments

Comments
 (0)