Skip to content

Commit b838f35

Browse files
author
Davies Liu
committed
retry after timeout
1 parent 191524e commit b838f35

File tree

2 files changed

+11
-1
lines changed

2 files changed

+11
-1
lines changed

core/src/main/scala/org/apache/spark/api/python/PythonRDD.scala

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -613,7 +613,16 @@ private[spark] object PythonRDD extends Logging {
613613
setDaemon(true)
614614
override def run() {
615615
try {
616-
val sock = serverSocket.accept()
616+
var sock: Socket = null
617+
try {
618+
sock = serverSocket.accept()
619+
} catch {
620+
case e: SocketTimeoutException =>
621+
// there is a small chance that the client had connected, so retry
622+
logWarning("Timed out after 4 seconds, retry once")
623+
serverSocket.setSoTimeout(10)
624+
sock = serverSocket.accept()
625+
}
617626
val out = new DataOutputStream(new BufferedOutputStream(sock.getOutputStream))
618627
try {
619628
writeIteratorToStream(items, out)

python/pyspark/rdd.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -113,6 +113,7 @@ def _parse_memory(s):
113113

114114
def _load_from_socket(port, serializer):
115115
sock = socket.socket()
116+
sock.settimeout(5)
116117
try:
117118
sock.connect(("localhost", port))
118119
rf = sock.makefile("rb", 65536)

0 commit comments

Comments
 (0)