Skip to content

Commit e93e726

Browse files
author
Liquan Pei
committed
use treeAggregate instead of aggregate
1 parent 1a8fb41 commit e93e726

File tree

1 file changed

+4
-3
lines changed

1 file changed

+4
-3
lines changed

mllib/src/main/scala/org/apache/spark/mllib/feature/Word2Vec.scala

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ import org.apache.spark.SparkContext._
3030
import org.apache.spark.mllib.linalg.{Vector, Vectors}
3131
import org.apache.spark.HashPartitioner
3232
import org.apache.spark.storage.StorageLevel
33+
import org.apache.spark.mllib.rdd.RDDFunctions._
3334
/**
3435
* Entry in vocabulary
3536
*/
@@ -111,9 +112,9 @@ class Word2Vec(
111112
}
112113

113114
private def learnVocabPerPartition(words:RDD[String]) {
114-
115+
115116
}
116-
117+
117118
private def createExpTable(): Array[Double] = {
118119
val expTable = new Array[Double](EXP_TABLE_SIZE)
119120
var i = 0
@@ -254,7 +255,7 @@ class Word2Vec(
254255
val (aggSyn0, aggSyn1, _, _) =
255256
// TODO: broadcast temp instead of serializing it directly
256257
// or initialize the model in each executor
257-
newSentences.aggregate((syn0Global.clone(), syn1Global.clone(), 0, 0))(
258+
newSentences.treeAggregate((syn0Global.clone(), syn1Global.clone(), 0, 0))(
258259
seqOp = (c, v) => (c, v) match {
259260
case ((syn0, syn1, lastWordCount, wordCount), sentence) =>
260261
var lwc = lastWordCount

0 commit comments

Comments
 (0)