@@ -1049,9 +1049,9 @@ def saveAsNewAPIHadoopDataset(self, conf, keyConverter=None, valueConverter=None
1049
1049
@param valueConverter: (None by default)
1050
1050
"""
1051
1051
jconf = self .ctx ._dictToJavaMap (conf )
1052
- pickled = self ._toPickleSerialization ()
1053
- batched = isinstance (pickled ._jrdd_deserializer , BatchedSerializer )
1054
- self .ctx ._jvm .PythonRDD .saveAsHadoopDataset (pickled ._jrdd , batched , jconf ,
1052
+ pickledRDD = self ._toPickleSerialization ()
1053
+ batched = isinstance (pickledRDD ._jrdd_deserializer , BatchedSerializer )
1054
+ self .ctx ._jvm .PythonRDD .saveAsHadoopDataset (pickledRDD ._jrdd , batched , jconf ,
1055
1055
keyConverter , valueConverter , True )
1056
1056
1057
1057
def saveAsNewAPIHadoopFile (self , path , outputFormatClass , keyClass = None , valueClass = None ,
@@ -1076,9 +1076,9 @@ def saveAsNewAPIHadoopFile(self, path, outputFormatClass, keyClass=None, valueCl
1076
1076
@param conf: Hadoop job configuration, passed in as a dict (None by default)
1077
1077
"""
1078
1078
jconf = self .ctx ._dictToJavaMap (conf )
1079
- pickled = self ._toPickleSerialization ()
1080
- batched = isinstance (pickled ._jrdd_deserializer , BatchedSerializer )
1081
- self .ctx ._jvm .PythonRDD .saveAsNewAPIHadoopFile (pickled ._jrdd , batched , path ,
1079
+ pickledRDD = self ._toPickleSerialization ()
1080
+ batched = isinstance (pickledRDD ._jrdd_deserializer , BatchedSerializer )
1081
+ self .ctx ._jvm .PythonRDD .saveAsNewAPIHadoopFile (pickledRDD ._jrdd , batched , path ,
1082
1082
outputFormatClass , keyClass , valueClass , keyConverter , valueConverter , jconf )
1083
1083
1084
1084
def saveAsHadoopDataset (self , conf , keyConverter = None , valueConverter = None ):
@@ -1093,9 +1093,9 @@ def saveAsHadoopDataset(self, conf, keyConverter=None, valueConverter=None):
1093
1093
@param valueConverter: (None by default)
1094
1094
"""
1095
1095
jconf = self .ctx ._dictToJavaMap (conf )
1096
- pickled = self ._toPickleSerialization ()
1097
- batched = isinstance (pickled ._jrdd_deserializer , BatchedSerializer )
1098
- self .ctx ._jvm .PythonRDD .saveAsHadoopDataset (pickled ._jrdd , batched , jconf ,
1096
+ pickledRDD = self ._toPickleSerialization ()
1097
+ batched = isinstance (pickledRDD ._jrdd_deserializer , BatchedSerializer )
1098
+ self .ctx ._jvm .PythonRDD .saveAsHadoopDataset (pickledRDD ._jrdd , batched , jconf ,
1099
1099
keyConverter , valueConverter , False )
1100
1100
1101
1101
def saveAsHadoopFile (self , path , outputFormatClass , keyClass = None , valueClass = None ,
@@ -1121,9 +1121,9 @@ def saveAsHadoopFile(self, path, outputFormatClass, keyClass=None, valueClass=No
1121
1121
@param compressionCodecClass: (None by default)
1122
1122
"""
1123
1123
jconf = self .ctx ._dictToJavaMap (conf )
1124
- pickled = self ._toPickleSerialization ()
1125
- batched = isinstance (pickled ._jrdd_deserializer , BatchedSerializer )
1126
- self .ctx ._jvm .PythonRDD .saveAsHadoopFile (pickled ._jrdd , batched ,
1124
+ pickledRDD = self ._toPickleSerialization ()
1125
+ batched = isinstance (pickledRDD ._jrdd_deserializer , BatchedSerializer )
1126
+ self .ctx ._jvm .PythonRDD .saveAsHadoopFile (pickledRDD ._jrdd , batched ,
1127
1127
path , outputFormatClass , keyClass , valueClass , keyConverter , valueConverter ,
1128
1128
jconf , compressionCodecClass )
1129
1129
@@ -1138,9 +1138,9 @@ def saveAsSequenceFile(self, path, compressionCodecClass=None):
1138
1138
@param path: path to sequence file
1139
1139
@param compressionCodecClass: (None by default)
1140
1140
"""
1141
- pickled = self ._toPickleSerialization ()
1142
- batched = isinstance (pickled ._jrdd_deserializer , BatchedSerializer )
1143
- self .ctx ._jvm .PythonRDD .saveAsSequenceFile (pickled ._jrdd , batched ,
1141
+ pickledRDD = self ._toPickleSerialization ()
1142
+ batched = isinstance (pickledRDD ._jrdd_deserializer , BatchedSerializer )
1143
+ self .ctx ._jvm .PythonRDD .saveAsSequenceFile (pickledRDD ._jrdd , batched ,
1144
1144
path , compressionCodecClass )
1145
1145
1146
1146
def saveAsPickleFile (self , path , batchSize = 10 ):
0 commit comments