Skip to content

Commit a34a114

Browse files
foxishash211
authored andcommitted
Filter nodes to only try and send files to external IPs (apache#106)
* Filter node addresses * Added comment
1 parent 3d80fff commit a34a114

File tree

1 file changed

+7
-2
lines changed
  • resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes

1 file changed

+7
-2
lines changed

resource-managers/kubernetes/core/src/main/scala/org/apache/spark/deploy/kubernetes/Client.scala

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -745,9 +745,14 @@ private[spark] class Client(
745745
val nodeUrls = kubernetesClient.nodes.list.getItems.asScala
746746
.filterNot(node => node.getSpec.getUnschedulable != null &&
747747
node.getSpec.getUnschedulable)
748-
.flatMap(_.getStatus.getAddresses.asScala.map(address => {
748+
.flatMap(_.getStatus.getAddresses.asScala)
749+
// The list contains hostnames, internal and external IP addresses.
750+
// we want only external IP addresses in our list
751+
// (https://kubernetes.io/docs/admin/node/#addresses)
752+
.filter(_.getType == "ExternalIP")
753+
.map(address => {
749754
s"$urlScheme://${address.getAddress}:$servicePort"
750-
})).toSet
755+
}).toSet
751756
require(nodeUrls.nonEmpty, "No nodes found to contact the driver!")
752757
val (trustManager, sslContext): (X509TrustManager, SSLContext) =
753758
if (driverSubmitSslOptions.enabled) {

0 commit comments

Comments
 (0)