Moving the 'LargeIteratorSuite' to simply test persistance of iterators. It doesn't try to invoke a OOM error any more

Author: kellrott

core/src/test/scala/org/apache/spark/storage/FlatmapIteratorSuite.scala

@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.spark.storage
+
+import org.scalatest.FunSuite
+import org.apache.spark.{SharedSparkContext, SparkConf, LocalSparkContext, SparkContext}
+
+
+class FlatmapIteratorSuite extends FunSuite with LocalSparkContext {
+  /* Tests the ability of Spark to deal with user provided iterators from flatMap
+   * calls, that may generate more data then available memory. In any
+   * memory based persistance Spark will unroll the iterator into an ArrayBuffer
+   * for caching, however in the case that the use defines DISK_ONLY persistance,
+   * the iterator will be fed directly to the serializer and written to disk.
+   *
+   * This also tests the ObjectOutputStream reset rate. When serializing using the
+   * Java serialization system, the serializer caches objects to prevent writing redundant
+   * data, however that stops GC of those objects. By calling 'reset' you flush that
+   * info from the serializer, and allow old objects to be GC'd
+   */
+  test("Flatmap Iterator to Disk") {
+    val sconf = new SparkConf().setMaster("local-cluster[1,1,512]")
+      .setAppName("iterator_to_disk_test")
+    sc = new SparkContext(sconf)
+    try {
+      val expand_size = 100
+      val data = sc.parallelize( (1 to 5).toSeq ).
+        flatMap( x => Stream.range(0, expand_size) )
+      var persisted = data.persist(StorageLevel.DISK_ONLY)
+      println(persisted.count())
+      assert( persisted.count() == 500)
+      assert( persisted.filter( _==1 ).count() == 5 )
+    } catch {
+      case _ : OutOfMemoryError => assert(false)
+    }
+  }
+
+  test("Flatmap Iterator to Memory") {
+    val sconf = new SparkConf().setMaster("local-cluster[1,1,512]")
+      .setAppName("iterator_to_disk_test")
+    sc = new SparkContext(sconf)
+    try {
+      val expand_size = 100
+      val data = sc.parallelize( (1 to 5).toSeq ).
+        flatMap( x => Stream.range(0, expand_size) )
+      var persisted = data.persist(StorageLevel.MEMORY_ONLY)
+      println(persisted.count())
+      assert( persisted.count() == 500)
+      assert( persisted.filter( _==1 ).count() == 5 )
+    } catch {
+      case _ : OutOfMemoryError => assert(false)
+    }
+  }
+
+  test("Serializer Reset") {
+    val sconf = new SparkConf().setMaster("local-cluster[1,1,512]")
+      .setAppName("serializer_reset_test")
+      .set("spark.serializer.objectStreamReset", "10")
+
+    sc = new SparkContext(sconf)
+    val expand_size = 500
+    val data = sc.parallelize( Seq(1,2) ).
+      flatMap( x => Stream.range(1, expand_size).
+      map( y => "%d: string test %d".format(y,x) ) )
+    var persisted = data.persist(StorageLevel.MEMORY_ONLY_SER)
+    assert( persisted.filter( _.startsWith("1:") ).count() == 2 )
+  }
+
+}