Skip to content

Commit aeb9e9d

Browse files
ekrivokonmaprmgorbov
authored andcommitted
MapR [SPARK-144] Add insertToMapRDB method for rdd for Java API (apache#205)
(cherry picked from commit 3137e3a)
1 parent b3a4ffa commit aeb9e9d

File tree

1 file changed

+131
-0
lines changed

1 file changed

+131
-0
lines changed

external/maprdb/src/main/scala/com/mapr/db/spark/api/java/MapRDBJavaSparkContext.scala

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,4 +181,135 @@ class MapRDBJavaSparkContext(val sparkContext: SparkContext) {
181181
bulkInsert = false)
182182
}
183183

184+
def insertToMapRDB[K, V <: AnyRef](javaPairRDD: JavaPairRDD[K, V],
185+
keyClazz: Class[K],
186+
valueClazz: Class[V],
187+
tableName: String,
188+
createTable: Boolean,
189+
bulkInsert: Boolean): Unit = {
190+
191+
require(javaPairRDD != null, "RDD can not be null")
192+
require(keyClazz != null, "Key class can not be null")
193+
require(valueClazz != null, "Value class can not be null")
194+
195+
import scala.reflect._
196+
implicit val vct: ClassTag[V] = ClassTag(valueClazz)
197+
implicit val v: OJAIValue[V] = OJAIValue.overrideDefault[V]
198+
199+
implicit val ct: ClassTag[K] = ClassTag(keyClazz)
200+
implicit val f: OJAIKey[K] = MapRDBUtils.getOjaiKey[K]()
201+
202+
PairedDocumentRDDFunctions(javaPairRDD.rdd)
203+
.insertToMapRDB(tableName, createTable, bulkInsert)
204+
}
205+
206+
def insertToMapRDB[K, V <: AnyRef](javaPairRDD: JavaPairRDD[K, V],
207+
keyClazz: Class[K],
208+
valueClazz: Class[V],
209+
tableName: String,
210+
createTable: Boolean): Unit = {
211+
212+
this.insertToMapRDB(javaPairRDD,
213+
keyClazz,
214+
valueClazz,
215+
tableName,
216+
createTable,
217+
bulkInsert = false)
218+
}
219+
220+
def insertToMapRDB[K, V <: AnyRef](javaPairRDD: JavaPairRDD[K, V],
221+
keyClazz: Class[K],
222+
valueClazz: Class[V],
223+
tableName: String): Unit = {
224+
225+
this.insertToMapRDB(javaPairRDD,
226+
keyClazz,
227+
valueClazz,
228+
tableName,
229+
createTable = false,
230+
bulkInsert = false)
231+
}
232+
233+
def insertRowRDDToMapRDB(javaRDD: JavaRDD[Row],
234+
tableName: String,
235+
createTable: Boolean,
236+
bulkInsert: Boolean,
237+
idField: String): Unit = {
238+
239+
require(javaRDD != null, "RDD can not be null")
240+
javaRDD.rdd
241+
.asInstanceOf[RDD[Row]]
242+
.insertToMapRDB(tableName, createTable, bulkInsert, idField)
243+
}
244+
245+
def insertRowRDDToMapRDB(javaRDD: JavaRDD[Row],
246+
tableName: String,
247+
createTable: Boolean,
248+
bulkInsert: Boolean): Unit = {
249+
this.insertRowRDDToMapRDB(javaRDD,
250+
tableName,
251+
createTable,
252+
bulkInsert,
253+
DocumentConstants.ID_KEY)
254+
}
255+
256+
def insertRowRDDToMapRDB(javaRDD: JavaRDD[Row],
257+
tableName: String,
258+
createTable: Boolean): Unit = {
259+
this.insertRowRDDToMapRDB(javaRDD,
260+
tableName,
261+
createTable,
262+
bulkInsert = false,
263+
DocumentConstants.ID_KEY)
264+
}
265+
266+
def insertRowRDDToMapRDB(javaRDD: JavaRDD[Row], tableName: String): Unit = {
267+
this.insertRowRDDToMapRDB(javaRDD,
268+
tableName,
269+
createTable = false,
270+
bulkInsert = false,
271+
DocumentConstants.ID_KEY)
272+
}
273+
274+
def insertToMapRDB[D](javaRDD: JavaRDD[D],
275+
tableName: String,
276+
createTable: Boolean,
277+
bulkInsert: Boolean,
278+
idField: String): Unit = {
279+
280+
require(javaRDD != null, "RDD can not be null")
281+
282+
javaRDD.rdd
283+
.asInstanceOf[RDD[OJAIValue[D]]]
284+
.insertToMapRDB(tableName, createTable, bulkInsert, idField)
285+
}
286+
287+
def insertToMapRDB[D](javaRDD: JavaRDD[D],
288+
tableName: String,
289+
createTable: Boolean,
290+
bulkInsert: Boolean): Unit = {
291+
this.insertToMapRDB(javaRDD,
292+
tableName,
293+
createTable,
294+
bulkInsert,
295+
DocumentConstants.ID_KEY)
296+
}
297+
298+
def insertToMapRDB[D](javaRDD: JavaRDD[D],
299+
tableName: String,
300+
createTable: Boolean): Unit = {
301+
this.insertToMapRDB(javaRDD,
302+
tableName,
303+
createTable,
304+
bulkInsert = false,
305+
DocumentConstants.ID_KEY)
306+
}
307+
308+
def insertToMapRDB[D](javaRDD: JavaRDD[D], tableName: String): Unit = {
309+
this.insertToMapRDB(javaRDD,
310+
tableName,
311+
createTable = false,
312+
bulkInsert = false,
313+
DocumentConstants.ID_KEY)
314+
}
184315
}

0 commit comments

Comments
 (0)