Merge branch 'upmaster' into SPARK-35253-FOLLOWUP

Author: LuciferYang

common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/RemoteBlockPushResolver.java

@@ -47,6 +47,8 @@
 import com.google.common.collect.Maps;
 import com.google.common.primitives.Ints;
 import com.google.common.primitives.Longs;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 import org.roaringbitmap.RoaringBitmap;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -770,9 +772,9 @@ public int hashCode() {
 
     @Override
     public String toString() {
-      return Objects.toStringHelper(this)
-        .add("appId", appId)
-        .add("shuffleId", shuffleId)
+      return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+        .append("appId", appId)
+        .append("shuffleId", shuffleId)
         .toString();
     }
   }

common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/FinalizeShuffleMerge.java

@@ -19,6 +19,8 @@
 
 import com.google.common.base.Objects;
 import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import org.apache.spark.network.protocol.Encoders;
 
@@ -51,9 +53,9 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return Objects.toStringHelper(this)
-      .add("appId", appId)
-      .add("shuffleId", shuffleId)
+    return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+      .append("appId", appId)
+      .append("shuffleId", shuffleId)
       .toString();
   }
 

common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/MergeStatuses.java

@@ -21,6 +21,8 @@
 
 import com.google.common.base.Objects;
 import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 import org.roaringbitmap.RoaringBitmap;
 
 import org.apache.spark.network.protocol.Encoders;
@@ -76,9 +78,9 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return Objects.toStringHelper(this)
-      .add("shuffleId", shuffleId)
-      .add("reduceId size", reduceIds.length)
+    return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+      .append("shuffleId", shuffleId)
+      .append("reduceId size", reduceIds.length)
       .toString();
   }
 

common/network-shuffle/src/main/java/org/apache/spark/network/shuffle/protocol/PushBlockStream.java

@@ -19,6 +19,8 @@
 
 import com.google.common.base.Objects;
 import io.netty.buffer.ByteBuf;
+import org.apache.commons.lang3.builder.ToStringBuilder;
+import org.apache.commons.lang3.builder.ToStringStyle;
 
 import org.apache.spark.network.protocol.Encoders;
 
@@ -59,12 +61,12 @@ public int hashCode() {
 
   @Override
   public String toString() {
-    return Objects.toStringHelper(this)
-      .add("appId", appId)
-      .add("shuffleId", shuffleId)
-      .add("mapIndex", mapIndex)
-      .add("reduceId", reduceId)
-      .add("index", index)
+    return new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE)
+      .append("appId", appId)
+      .append("shuffleId", shuffleId)
+      .append("mapIndex", mapIndex)
+      .append("reduceId", reduceId)
+      .append("index", index)
       .toString();
   }
 

dev/create-release/spark-rm/Dockerfile

@@ -40,7 +40,9 @@ ARG APT_INSTALL="apt-get install --no-install-recommends -y"
 # TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
 #   See also https://github.com/sphinx-doc/sphinx/issues/7551.
 #   We should use the latest Sphinx version once this is fixed.
-ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.1.2 numpy==1.19.4 pydata_sphinx_theme==0.4.1 ipython==7.19.0 nbsphinx==0.8.0 numpydoc==1.1.0"
+# TODO(SPARK-35375): Jinja2 3.0.0+ causes error when building with Sphinx.
+#   See also https://issues.apache.org/jira/browse/SPARK-35375.
+ARG PIP_PKGS="sphinx==3.0.4 mkdocs==1.1.2 numpy==1.19.4 pydata_sphinx_theme==0.4.1 ipython==7.19.0 nbsphinx==0.8.0 numpydoc==1.1.0 jinja2==2.11.3"
 ARG GEM_PKGS="bundler:2.2.9"
 
 # Install extra needed repos and refresh.

docs/README.md

@@ -67,10 +67,13 @@ To generate API docs for any language, you'll need to install these libraries:
 <!--
 TODO(SPARK-32407): Sphinx 3.1+ does not correctly index nested classes.
 See also https://github.com/sphinx-doc/sphinx/issues/7551.
+
+TODO(SPARK-35375): Jinja2 3.0.0+ causes error when building with Sphinx.
+See also https://issues.apache.org/jira/browse/SPARK-35375.
 -->
 
 ```sh
-$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc
+$ sudo pip install 'sphinx<3.1.0' mkdocs numpy pydata_sphinx_theme ipython nbsphinx numpydoc 'jinja2<3.0.0'
 ```
 
 ## Generating the Documentation HTML

docs/configuration.md

@@ -943,8 +943,8 @@ Apart from these, the following properties are also available, and may be useful
   <td>false</td>
   <td>
     Enables the external shuffle service. This service preserves the shuffle files written by
-    executors so the executors can be safely removed. The external shuffle service
-    must be set up in order to enable it. See
+    executors e.g. so that executors can be safely removed, or so that shuffle fetches can continue in 
+    the event of executor failure. The external shuffle service must be set up in order to enable it. See
     <a href="job-scheduling.html#configuration-and-setup">dynamic allocation
     configuration and setup documentation</a> for more information.
   </td>

docs/job-scheduling.md

@@ -142,13 +142,12 @@ an executor should not be idle if there are still pending tasks to be scheduled.
 
 ### Graceful Decommission of Executors
 
-Before dynamic allocation, a Spark executor exits either on failure or when the associated
-application has also exited. In both scenarios, all state associated with the executor is no
-longer needed and can be safely discarded. With dynamic allocation, however, the application
-is still running when an executor is explicitly removed. If the application attempts to access
-state stored in or written by the executor, it will have to perform a recompute the state. Thus,
-Spark needs a mechanism to decommission an executor gracefully by preserving its state before
-removing it.
+Before dynamic allocation, if a Spark executor exits when the associated application has also exited 
+then all state associated with the executor is no longer needed and can be safely discarded. 
+With dynamic allocation, however, the application is still running when an executor is explicitly 
+removed. If the application attempts to access state stored in or written by the executor, it will 
+have to perform a recompute the state. Thus, Spark needs a mechanism to decommission an executor 
+gracefully by preserving its state before removing it.
 
 This requirement is especially important for shuffles. During a shuffle, the Spark executor first
 writes its own map outputs locally to disk, and then acts as the server for those files when other

python/docs/source/migration_guide/pyspark_3.1_to_3.2.rst

@@ -21,3 +21,5 @@ Upgrading from PySpark 3.1 to 3.2
 =================================
 
 * In Spark 3.2, the PySpark methods from sql, ml, spark_on_pandas modules raise the ``TypeError`` instead of ``ValueError`` when are applied to an param of inappropriate type.
+
+* In Spark 3.2, the traceback from Python UDFs, pandas UDFs and pandas function APIs are simplified by default without the traceback from the internal Python workers. In Spark 3.1 or earlier, the traceback from Python workers was printed out. To restore the behavior before Spark 3.2, you can set ``spark.sql.execution.pyspark.udf.simplifiedTraceback.enabled`` to ``false``.

python/mypy.ini

@@ -85,6 +85,10 @@ disallow_untyped_defs = False
 [mypy-pyspark.sql.utils]
 disallow_untyped_defs = False
 
+[mypy-pyspark.pandas.*]
+strict_optional = False
+disallow_untyped_defs = False
+
 [mypy-pyspark.tests.*]
 disallow_untyped_defs = False
 
@@ -125,6 +129,14 @@ ignore_missing_imports = True
 [mypy-psutil.*]
 ignore_missing_imports = True
 
-# TODO(SPARK-34941): Enable mypy for pandas-on-Spark
-[mypy-pyspark.pandas.*]
-ignore_errors = True
+[mypy-matplotlib.*]
+ignore_missing_imports = True
+
+[mypy-plotly.*]
+ignore_missing_imports = True
+
+[mypy-mlflow.*]
+ignore_missing_imports = True
+
+[mypy-sklearn.*]
+ignore_missing_imports = True