Skip to content

Commit 3ffe566

Browse files
author
Andrew Or
committed
Restore "null" as default for RDD name
Instead, RDDInfo's name field will default to the RDD class name if the RDD's name is not defined. This avoids changing RDD#toString behavior.
1 parent 5fdd89d commit 3ffe566

File tree

4 files changed

+5
-5
lines changed

4 files changed

+5
-5
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -898,7 +898,7 @@ class SparkContext(config: SparkConf) extends Logging with ExecutorAllocationCli
898898
inputFormatClass,
899899
keyClass,
900900
valueClass,
901-
minPartitions).setName(s"HadoopRDD[$path]")
901+
minPartitions).setName(path)
902902
}
903903

904904
/**

core/src/main/scala/org/apache/spark/rdd/RDD.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@ abstract class RDD[T: ClassTag](
140140
val id: Int = sc.newRddId()
141141

142142
/** A friendly name for this RDD */
143-
@transient var name: String = Utils.getFormattedClassName(this)
143+
@transient var name: String = null
144144

145145
/** Assign a name to this RDD */
146146
def setName(_name: String): this.type = {

core/src/main/scala/org/apache/spark/storage/RDDInfo.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ class RDDInfo(
5454

5555
private[spark] object RDDInfo {
5656
def fromRdd(rdd: RDD[_]): RDDInfo = {
57-
val rddName = Option(rdd.name).getOrElse(rdd.id.toString)
57+
val rddName = Option(rdd.name).getOrElse(Utils.getFormattedClassName(rdd))
5858
val parentIds = rdd.dependencies.map(_.rdd.id)
5959
new RDDInfo(rdd.id, rddName, rdd.partitions.length, rdd.getStorageLevel, parentIds, rdd.scope)
6060
}

python/pyspark/sql/dataframe.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,8 +244,8 @@ def explain(self, extended=False):
244244
:param extended: boolean, default ``False``. If ``False``, prints only the physical plan.
245245
246246
>>> df.explain()
247-
PhysicalRDD [age#0,name#1], MapPartitionsRDD MapPartitionsRDD[7] at applySchemaToPythonRDD\
248-
at NativeMethodAccessorImpl.java:...
247+
PhysicalRDD [age#0,name#1], MapPartitionsRDD[...] at applySchemaToPythonRDD at\
248+
NativeMethodAccessorImpl.java:...
249249
250250
>>> df.explain(True)
251251
== Parsed Logical Plan ==

0 commit comments

Comments
 (0)