Skip to content
This repository was archived by the owner on May 9, 2024. It is now read-only.

Commit de64328

Browse files
committed
Address rxin's comments.
1 parent fc9b1ad commit de64328

File tree

1 file changed

+15
-8
lines changed
  • sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis

1 file changed

+15
-8
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala

Lines changed: 15 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -645,9 +645,14 @@ class Analyzer(
645645

646646
/**
647647
* From a Seq of [[NamedExpression]]s, extract expressions containing window expressions and
648-
* other regular expressions that do not contain any window expression.
648+
* other regular expressions that do not contain any window expression. For example, for
649+
* `col1, Sum(col2 + col3) OVER (PARTITION BY col4 ORDER BY col5)`, we will extract
650+
* `col1`, `col2 + col3`, `col4`, and `col5` out and replace them appearances in
651+
* the window expression as attribute references. So, the first returned value will be
652+
* `[Sum(_w0) OVER (PARTITION BY _w1 ORDER BY _w2)]` and the second returned value will be
653+
* [col1, col2 + col3 as _w0, col4 as _w1, col5 as _w2].
649654
*/
650-
private def extractRegularExpressions(
655+
private def extract(
651656
expressions: Seq[NamedExpression]): (Seq[NamedExpression], Seq[NamedExpression]) = {
652657
// First, we partition the input expressions to two part. For the first part,
653658
// every expression in it contain at least one WindowExpression.
@@ -662,8 +667,8 @@ class Analyzer(
662667
val extractedExprBuffer = new ArrayBuffer[NamedExpression]()
663668
def extractExpr(expr: Expression): Expression = expr match {
664669
case ne: NamedExpression =>
665-
// If a named expression is not in regularExpressions, add extract it and replace it
666-
// with an AttributeReference.
670+
// If a named expression is not in regularExpressions, add it to
671+
// extractedExprBuffer and replace it with an AttributeReference.
667672
val missingExpr =
668673
AttributeSet(Seq(expr)) -- (regularExpressions ++ extractedExprBuffer)
669674
if (missingExpr.nonEmpty) {
@@ -709,7 +714,7 @@ class Analyzer(
709714
}
710715

711716
(newExpressionsWithWindowFunctions, regularExpressions ++ extractedExprBuffer)
712-
} // end of extractRegularExpressions
717+
} // end of extract
713718

714719
/**
715720
* Adds operators for Window Expressions. Every Window operator handles a single Window Spec.
@@ -756,6 +761,8 @@ class Analyzer(
756761
if (distinctWindowSpec.length == 0 ) {
757762
failAnalysis(s"$expr does not have any WindowExpression.")
758763
} else if (distinctWindowSpec.length > 1) {
764+
// newExpressionsWithWindowFunctions only have expressions with a single
765+
// WindowExpression. If we reach here, we have a bug.
759766
failAnalysis(s"$expr has multiple Window Specifications ($distinctWindowSpec)." +
760767
s"Please file a bug report with this error message, stack trace, and the query.")
761768
} else {
@@ -790,7 +797,7 @@ class Analyzer(
790797
if child.resolved &&
791798
hasWindowFunction(aggregateExprs) &&
792799
a.expressions.forall(_.resolved) =>
793-
val (windowExpressions, aggregateExpressions) = extractRegularExpressions(aggregateExprs)
800+
val (windowExpressions, aggregateExpressions) = extract(aggregateExprs)
794801
// Create an Aggregate operator to evaluate aggregation functions.
795802
val withAggregate = Aggregate(groupingExprs, aggregateExpressions, child)
796803
// Add a Filter operator for conditions in the Having clause.
@@ -807,7 +814,7 @@ class Analyzer(
807814
case a @ Aggregate(groupingExprs, aggregateExprs, child)
808815
if hasWindowFunction(aggregateExprs) &&
809816
a.expressions.forall(_.resolved) =>
810-
val (windowExpressions, aggregateExpressions) = extractRegularExpressions(aggregateExprs)
817+
val (windowExpressions, aggregateExpressions) = extract(aggregateExprs)
811818
// Create an Aggregate operator to evaluate aggregation functions.
812819
val withAggregate = Aggregate(groupingExprs, aggregateExpressions, child)
813820
// Add Window operators.
@@ -821,7 +828,7 @@ class Analyzer(
821828
// have been resolved.
822829
case p @ Project(projectList, child)
823830
if hasWindowFunction(projectList) && !p.expressions.exists(!_.resolved) =>
824-
val (windowExpressions, regularExpressions) = extractRegularExpressions(projectList)
831+
val (windowExpressions, regularExpressions) = extract(projectList)
825832
// We add a project to get all needed expressions for window expressions from the child
826833
// of the original Project operator.
827834
val withProject = Project(regularExpressions, child)

0 commit comments

Comments
 (0)