Skip to content

Commit 32a3f3f

Browse files
ingore it in SparkSubmit instead of SparkContext
1 parent 667cf24 commit 32a3f3f

File tree

3 files changed

+8
-16
lines changed

3 files changed

+8
-16
lines changed

core/src/main/scala/org/apache/spark/SparkContext.scala

Lines changed: 4 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,10 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging {
175175
logInfo("Spark configuration:\n" + conf.toDebugString)
176176
}
177177

178+
// Set Spark driver host and port system properties.
179+
conf.setIfMissing("spark.driver.host", Utils.localHostName())
180+
conf.setIfMissing("spark.driver.port", "0")
181+
178182
val jars: Seq[String] =
179183
conf.getOption("spark.jars").map(_.split(",")).map(_.filter(_.size != 0)).toSeq.flatten
180184

@@ -202,14 +206,6 @@ class SparkContext(config: SparkConf) extends SparkStatusAPI with Logging {
202206

203207
if (master == "yarn-client") System.setProperty("SPARK_YARN_MODE", "true")
204208

205-
// Set Spark driver host and port system properties. Ignore host setting in all cluster modes.
206-
if (master.contains("cluster")) {
207-
conf.set("spark.driver.host", Utils.localHostName())
208-
} else {
209-
conf.setIfMissing("spark.driver.host", Utils.localHostName())
210-
}
211-
conf.setIfMissing("spark.driver.port", "0")
212-
213209
// An asynchronous listener bus for Spark events
214210
private[spark] val listenerBus = new LiveListenerBus
215211

core/src/main/scala/org/apache/spark/deploy/Client.scala

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -155,14 +155,6 @@ object Client {
155155
}
156156
conf.set("spark.akka.askTimeout", "10")
157157
conf.set("akka.loglevel", driverArgs.logLevel.toString.replace("WARN", "WARNING"))
158-
159-
// Set the web ui port to be ephemeral so we don't conflict with other spark processes
160-
// running on the same box
161-
conf.set("spark.ui.port", "0")
162-
163-
// Set the master property to match the requested mode.
164-
conf.set("spark.master", "standalone-cluster")
165-
166158
Logger.getRootLogger.setLevel(driverArgs.logLevel)
167159

168160
val (actorSystem, _) = AkkaUtils.createActorSystem(

core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,6 +279,10 @@ object SparkSubmit {
279279
sysProps.getOrElseUpdate(k, v)
280280
}
281281

282+
if (deployMode == CLUSTER) {
283+
sysProps -= ("spark.driver.host")
284+
}
285+
282286
// Resolve paths in certain spark properties
283287
val pathConfigs = Seq(
284288
"spark.jars",

0 commit comments

Comments
 (0)