Make to local iterator shorter

epahomov · epahomov · commit 8ec8f249f61a · 2014-04-05T10:11:04.000+04:00
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -672,13 +672,7 @@ abstract class RDD[T: ClassTag](
     def collectPartition(p: Int): Array[T] = {
       sc.runJob(this, (iter: Iterator[T]) => iter.toArray, Seq(p), allowLocal = false).head
     }
-    var buffer = Stream.empty[T]
-    for (p <- 0 until this.partitions.length) {
-      buffer = buffer append {
-        collectPartition(p).toStream
-      }
-    }
-    buffer.iterator
+    (0 until partitions.length).iterator.flatMap(i => collectPartition(i))
   }
 
   /**

Original file line number	Diff line number	Diff line change
`@@ -672,13 +672,7 @@ abstract class RDD[T: ClassTag](`
`672`	`672`	`def collectPartition(p: Int): Array[T] = {`
`673`	`673`	`sc.runJob(this, (iter: Iterator[T]) => iter.toArray, Seq(p), allowLocal = false).head`
`674`	`674`	`}`
`675`		`- var buffer = Stream.empty[T]`
`676`		`- for (p <- 0 until this.partitions.length) {`
`677`		`- buffer = buffer append {`
`678`		`- collectPartition(p).toStream`
`679`		`- }`
`680`		`- }`
`681`		`- buffer.iterator`
	`675`	`+ (0 until partitions.length).iterator.flatMap(i => collectPartition(i))`
`682`	`676`	`}`
`683`	`677`
`684`	`678`	`/**`