Skip to content

Commit 202f6e7

Browse files
mccheahmccheah
authored
Add the API module for the async shuffle upload plugin. (#641)
1 parent 55a26e9 commit 202f6e7

File tree

8 files changed

+331
-5
lines changed

8 files changed

+331
-5
lines changed

‎async-shuffle-upload/async-shuffle-upload-api/pom.xml

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
19+
<modelVersion>4.0.0</modelVersion>
20+
<parent>
21+
<groupId>org.apache.spark</groupId>
22+
<artifactId>spark-parent_2.11</artifactId>
23+
<version>3.0.0-SNAPSHOT</version>
24+
<relativePath>../../pom.xml</relativePath>
25+
</parent>
26+
27+
<artifactId>spark-async-shuffle-upload-api</artifactId>
28+
<properties>
29+
<sbt.project.name>spark-async-shuffle-upload-api</sbt.project.name>
30+
</properties>
31+
<packaging>jar</packaging>
32+
<name>Spark Project Async Shuffle Upload API</name>
33+
<url>http://spark.apache.org/</url>
34+
<dependencies>
35+
<dependency>
36+
<groupId>org.immutables</groupId>
37+
<artifactId>value</artifactId>
38+
</dependency>
39+
<dependency>
40+
<groupId>com.fasterxml.jackson.datatype</groupId>
41+
<artifactId>jackson-datatype-jdk8</artifactId>
42+
</dependency>
43+
<dependency>
44+
<groupId>com.palantir.safe-logging</groupId>
45+
<artifactId>safe-logging</artifactId>
46+
</dependency>
47+
<dependency>
48+
<groupId>com.palantir.safe-logging</groupId>
49+
<artifactId>preconditions</artifactId>
50+
</dependency>
51+
<dependency>
52+
<groupId>org.apache.spark</groupId>
53+
<artifactId>spark-async-shuffle-upload-immutables</artifactId>
54+
<version>${project.version}</version>
55+
<scope>provided</scope>
56+
</dependency>
57+
<dependency>
58+
<groupId>org.assertj</groupId>
59+
<artifactId>assertj-core</artifactId>
60+
<scope>test</scope>
61+
</dependency>
62+
<dependency>
63+
<groupId>junit</groupId>
64+
<artifactId>junit</artifactId>
65+
<scope>test</scope>
66+
</dependency>
67+
</dependencies>
68+
</project>
69+

‎async-shuffle-upload/async-shuffle-upload-api/src/main/java/com/palantir/spark/shuffle/async/api/SparkShuffleApiConstants.java

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.palantir.spark.shuffle.async.api;
19+
20+
public final class SparkShuffleApiConstants {
21+
22+
// Identifiers used by the spark shuffle plugin
23+
// TODO make these consistent in naming - but keep in mind that changing these would result in runtime errors.
24+
public static final String SHUFFLE_PLUGIN_APP_NAME_CONF = "spark.plugin.shuffle.async.appName";
25+
public static final String SHUFFLE_BASE_URI_CONF = "spark.shuffle.hadoop.async.base-uri";
26+
public static final String SHUFFLE_S3A_CREDS_FILE_CONF = "spark.plugin.shuffle.async.s3a.credsFile";
27+
public static final String SHUFFLE_S3A_ENDPOINT_CONF = "spark.shuffle.hadoop.async.s3a.endpoint";
28+
public static final String METRICS_FACTORY_CLASS_CONF = "spark.plugin.shuffle.async.metricsFactoryClass";
29+
30+
private SparkShuffleApiConstants() {}
31+
32+
}

‎async-shuffle-upload/async-shuffle-upload-api/src/main/java/com/palantir/spark/shuffle/async/api/SparkShuffleAwsCredentials.java

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.palantir.spark.shuffle.async.api;
19+
20+
import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
21+
import com.fasterxml.jackson.core.JsonProcessingException;
22+
import com.fasterxml.jackson.databind.ObjectMapper;
23+
import com.fasterxml.jackson.databind.annotation.JsonDeserialize;
24+
import com.fasterxml.jackson.databind.annotation.JsonSerialize;
25+
import com.palantir.logsafe.UnsafeArg;
26+
import com.palantir.logsafe.exceptions.SafeIllegalArgumentException;
27+
import com.palantir.spark.shuffle.async.immutables.ImmutablesStyle;
28+
import java.io.IOException;
29+
import java.nio.charset.StandardCharsets;
30+
import org.immutables.value.Value;
31+
32+
@ImmutablesStyle
33+
@Value.Immutable
34+
@JsonSerialize(as = ImmutableSparkShuffleAwsCredentials.class)
35+
@JsonDeserialize(as = ImmutableSparkShuffleAwsCredentials.class)
36+
@JsonIgnoreProperties(ignoreUnknown = true)
37+
public abstract class SparkShuffleAwsCredentials {
38+
39+
private static final ObjectMapper MAPPER = new ObjectMapper();
40+
41+
public abstract String accessKeyId();
42+
43+
public abstract String secretAccessKey();
44+
45+
public abstract String sessionToken();
46+
47+
public final byte[] toBytes() {
48+
try {
49+
return MAPPER.writeValueAsString(this).getBytes(StandardCharsets.UTF_8);
50+
} catch (JsonProcessingException e) {
51+
throw new RuntimeException(e);
52+
}
53+
}
54+
55+
public static SparkShuffleAwsCredentials fromBytes(byte[] bytes) {
56+
try {
57+
return MAPPER.readValue(new String(bytes, StandardCharsets.UTF_8), SparkShuffleAwsCredentials.class);
58+
} catch (IOException e) {
59+
throw new SafeIllegalArgumentException(
60+
"Could not deserialize bytes as AWS credentials.",
61+
UnsafeArg.of("cause", e));
62+
}
63+
}
64+
65+
public static Builder builder() {
66+
return new Builder();
67+
}
68+
69+
public static final class Builder extends ImmutableSparkShuffleAwsCredentials.Builder {}
70+
}

‎async-shuffle-upload/async-shuffle-upload-api/src/test/java/com/palantir/spark/shuffle/async/api/SparkShuffleAwsCredentialsSuite.java

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.palantir.spark.shuffle.async.api;
19+
20+
import static org.assertj.core.api.Assertions.assertThat;
21+
import static org.assertj.core.api.Assertions.fail;
22+
23+
import java.nio.charset.StandardCharsets;
24+
import org.junit.Test;
25+
26+
public final class SparkShuffleAwsCredentialsSuite {
27+
28+
@Test
29+
public void testSerialize() {
30+
SparkShuffleAwsCredentials creds = SparkShuffleAwsCredentials.builder()
31+
.accessKeyId("access-key")
32+
.secretAccessKey("secret-key")
33+
.sessionToken("session-token")
34+
.build();
35+
byte[] bytes = creds.toBytes();
36+
assertThat(new String(bytes, StandardCharsets.UTF_8))
37+
.isEqualTo("{\"accessKeyId\":\"access-key\","
38+
+ "\"secretAccessKey\":\"secret-key\","
39+
+ "\"sessionToken\":\"session-token\"}");
40+
}
41+
42+
@Test
43+
public void testDeserialize() {
44+
String serializedString = "{\"accessKeyId\":\"access-key\","
45+
+ "\"secretAccessKey\":\"secret-key\","
46+
+ "\"sessionToken\":\"session-token\"}";
47+
48+
SparkShuffleAwsCredentials creds =
49+
SparkShuffleAwsCredentials.fromBytes(serializedString.getBytes(StandardCharsets.UTF_8));
50+
}
51+
}

‎async-shuffle-upload/immutables/pom.xml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
19+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.spark</groupId>
23+
<artifactId>spark-parent_2.11</artifactId>
24+
<version>3.0.0-SNAPSHOT</version>
25+
<relativePath>../../pom.xml</relativePath>
26+
</parent>
27+
28+
<artifactId>spark-async-shuffle-upload-immutables</artifactId>
29+
<properties>
30+
<sbt.project.name>spark-async-shuffle-upload-immutables</sbt.project.name>
31+
</properties>
32+
<packaging>jar</packaging>
33+
<name>Spark Project Async Shuffle Upload Immutables</name>
34+
<url>http://spark.apache.org/</url>
35+
<dependencies>
36+
<dependency>
37+
<groupId>org.immutables</groupId>
38+
<artifactId>value</artifactId>
39+
</dependency>
40+
</dependencies>
41+
</project>

‎async-shuffle-upload/immutables/src/main/java/com/palantir/spark/shuffle/async/immutables/ImmutablesStyle.java

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.palantir.spark.shuffle.async.immutables;
19+
20+
import java.lang.annotation.ElementType;
21+
import java.lang.annotation.Retention;
22+
import java.lang.annotation.RetentionPolicy;
23+
import java.lang.annotation.Target;
24+
import org.immutables.value.Value;
25+
26+
@Target({ElementType.PACKAGE, ElementType.TYPE})
27+
@Retention(RetentionPolicy.SOURCE)
28+
@Value.Style(
29+
visibility = Value.Style.ImplementationVisibility.PACKAGE,
30+
overshadowImplementation = true,
31+
jdkOnly = true,
32+
get = {"get*", "is*"})
33+
public @interface ImmutablesStyle {}

‎core/pom.xml

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -388,14 +388,13 @@
388388
<scope>test</scope>
389389
</dependency>
390390
<dependency>
391-
<groupId>com.palantir.safe-logging</groupId>
392-
<artifactId>safe-logging</artifactId>
393-
<version>1.5.1</version>
391+
<groupId>org.apache.commons</groupId>
392+
<artifactId>commons-crypto</artifactId>
394393
</dependency>
395394

396395
<dependency>
397-
<groupId>org.apache.commons</groupId>
398-
<artifactId>commons-crypto</artifactId>
396+
<groupId>com.palantir.safe-logging</groupId>
397+
<artifactId>safe-logging</artifactId>
399398
</dependency>
400399

401400
</dependencies>

‎pom.xml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,10 @@
114114
<module>sql/hive</module>
115115
<module>streaming</module>
116116
<module>tools</module>
117+
118+
<!-- Async shuffle upload modules-->
119+
<module>async-shuffle-upload/immutables</module>
120+
<module>async-shuffle-upload/async-shuffle-upload-api</module>
117121
<!-- See additional modules enabled by profiles below -->
118122
</modules>
119123

@@ -216,6 +220,9 @@
216220
-->
217221
<arrow.version>0.12.0</arrow.version>
218222

223+
<!-- Async shuffle upload plugin dependency versions -->
224+
<safe-logging.version>1.13.0</safe-logging.version>
225+
219226
<test.java.home>${java.home}</test.java.home>
220227
<test.exclude.tags></test.exclude.tags>
221228
<test.include.tags></test.include.tags>
@@ -2305,6 +2312,30 @@
23052312
<artifactId>kafka-clients</artifactId>
23062313
<version>${kafka.version}</version>
23072314
</dependency>
2315+
2316+
<dependency>
2317+
<groupId>com.palantir.safe-logging</groupId>
2318+
<artifactId>safe-logging</artifactId>
2319+
<version>${safe-logging.version}</version>
2320+
</dependency>
2321+
2322+
<!-- Dependencies required for the async shuffle upload plugin. -->
2323+
<dependency>
2324+
<groupId>org.immutables</groupId>
2325+
<artifactId>value</artifactId>
2326+
<version>2.8.2</version>
2327+
<scope>provided</scope>
2328+
</dependency>
2329+
<dependency>
2330+
<groupId>com.palantir.safe-logging</groupId>
2331+
<artifactId>preconditions</artifactId>
2332+
<version>${safe-logging.version}</version>
2333+
</dependency>
2334+
<dependency>
2335+
<groupId>org.assertj</groupId>
2336+
<artifactId>assertj-core</artifactId>
2337+
<version>3.15.0</version>
2338+
</dependency>
23082339
</dependencies>
23092340
</dependencyManagement>
23102341

0 commit comments

Comments
 (0)