Restore deprecation logic for spark.executor.userClassPathFirst

Author: Andrew Or

core/src/main/scala/org/apache/spark/executor/Executor.scala

@@ -92,6 +92,11 @@ private[spark] class Executor(
   private val executorActor = env.actorSystem.actorOf(
     Props(new ExecutorActor(executorId)), "ExecutorActor")
 
+  // Whether to load classes in user jars before those in Spark jars
+  private val userClassPathFirst =
+    conf.getBoolean("spark.executor.userClassPathFirst",
+      conf.getBoolean("spark.files.userClassPathFirst", false))
+
   // Create our ClassLoader
   // do this after SparkEnv creation so can access the SecurityManager
   private val urlClassLoader = createClassLoader()
@@ -309,7 +314,7 @@ private[spark] class Executor(
     val urls = userClassPath.toArray ++ currentJars.keySet.map { uri =>
       new File(uri.split("/").last).toURI.toURL
     }
-    if (conf.getBoolean("spark.executor.userClassPathFirst", false)) {
+    if (userClassPathFirst) {
       new ChildFirstURLClassLoader(urls, currentLoader)
     } else {
       new MutableURLClassLoader(urls, currentLoader)
@@ -324,14 +329,13 @@ private[spark] class Executor(
     val classUri = conf.get("spark.repl.class.uri", null)
     if (classUri != null) {
       logInfo("Using REPL class URI: " + classUri)
-      val userClassPathFirst: java.lang.Boolean =
-        conf.getBoolean("spark.executor.userClassPathFirst", false)
       try {
+        val _userClassPathFirst: java.lang.Boolean = userClassPathFirst
         val klass = Class.forName("org.apache.spark.repl.ExecutorClassLoader")
           .asInstanceOf[Class[_ <: ClassLoader]]
         val constructor = klass.getConstructor(classOf[SparkConf], classOf[String],
           classOf[ClassLoader], classOf[Boolean])
-        constructor.newInstance(conf, classUri, parent, userClassPathFirst)
+        constructor.newInstance(conf, classUri, parent, _userClassPathFirst)
       } catch {
         case _: ClassNotFoundException =>
           logError("Could not find org.apache.spark.repl.ExecutorClassLoader on classpath!")

core/src/test/scala/org/apache/spark/SparkConfSuite.scala

@@ -198,15 +198,23 @@ class SparkConfSuite extends FunSuite with LocalSparkContext with ResetSystemPro
   }
 
   test("deprecated config keys") {
+    val conf = new SparkConf().set("spark.executor.userClassPathFirst", "true")
+    assert(conf.contains("spark.executor.userClassPathFirst"))
+    assert(!conf.contains("spark.files.userClassPathFirst"))
+    assert(conf.get("spark.executor.userClassPathFirst") === "true")
+    assert(conf.get("spark.files.userClassPathFirst") === "true")
+  }
+
+  test("deprecated config does not override original config") {
     val conf = new SparkConf()
       .set("spark.files.userClassPathFirst", "true")
-      .set("spark.yarn.user.classpath.first", "true")
-    assert(conf.contains("spark.files.userClassPathFirst"))
-    assert(conf.contains("spark.executor.userClassPathFirst"))
-    assert(conf.contains("spark.yarn.user.classpath.first"))
-    assert(conf.getBoolean("spark.files.userClassPathFirst", false))
-    assert(conf.getBoolean("spark.executor.userClassPathFirst", false))
-    assert(conf.getBoolean("spark.yarn.user.classpath.first", false))
+      .set("spark.executor.userClassPathFirst", "false")
+    val conf2 = new SparkConf()
+      .set("spark.executor.userClassPathFirst", "false")
+      .set("spark.files.userClassPathFirst", "true")
+    // In both cases, the new config's value should be used
+    assert(conf.get("spark.executor.userClassPathFirst") === "false")
+    assert(conf2.get("spark.executor.userClassPathFirst") === "false")
   }
 
 }

yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala

@@ -955,7 +955,8 @@ object Client extends Logging {
     if (isDriver) {
       conf.getBoolean("spark.driver.userClassPathFirst", false)
     } else {
-      conf.getBoolean("spark.executor.userClassPathFirst", false)
+      conf.getBoolean("spark.executor.userClassPathFirst",
+        conf.getBoolean("spark.files.userClassPathFirst", false))
     }
   }