Make PySpark the only Spark image. (#436)

* Make PySpark the only Spark image.

* Add link to spark examples.

* Update changelog

* Remove pyspark from GH workflows.

* Remove PySpark RH project.

* Update image_tools/conf.py

Co-authored-by: Malte Sander <[email protected]>

---------

Co-authored-by: Malte Sander <[email protected]>

Author: razvan

.github/workflows/dev.yaml

@@ -26,7 +26,6 @@ jobs:
           - krb5
           - nifi
           - opa
-          - pyspark-k8s
           - spark-k8s
           - superset
           - testing-tools
@@ -93,7 +92,6 @@ jobs:
           - kafka
           - nifi
           - opa
-          - pyspark-k8s
           - spark-k8s
           - superset
           - trino

.github/workflows/release.yml

@@ -29,7 +29,6 @@ jobs:
           - krb5
           - nifi
           - opa
-          - pyspark-k8s
           - spark-k8s
           - superset
           - testing-tools
@@ -101,7 +100,6 @@ jobs:
           - kafka
           - nifi
           - opa
-          - pyspark-k8s
           - spark-k8s
           - superset
           - trino

image_tools/conf.py

@@ -248,7 +248,7 @@
         ],
     },
     {
-        "name": "pyspark-k8s",
+        "name": "spark-k8s",
         "versions": [
             {
                 "product": "3.2.1",
@@ -316,37 +316,6 @@
                 "java-base": "17",
                 "python": "3.11",
                 "hadoop_short_version": "3",
-                "hadoop_long_version": "3.3.4",
-                "aws_java_sdk_bundle": "1.12.262",
-                "azure_storage": "7.0.1",
-                "azure_keyvault_core": "1.0.0",
-                "jackson_dataformat_xml": "2.14.2",
-                "stax2_api": "4.2.1",
-                "woodstox_core": "6.5.0",
-                "vector": "0.31.0",
-            },
-        ],
-    },
-    {
-        "name": "spark-k8s",
-        "versions": [
-            {
-                "product": "3.3.0",
-                "java-base": "11",
-                "hadoop_short_version": "3",
-                "hadoop_long_version": "3.3.3",
-                "aws_java_sdk_bundle": "1.11.1026",
-                "azure_storage": "7.0.1",
-                "azure_keyvault_core": "1.0.0",
-                "jackson_dataformat_xml": "2.13.3",
-                "stax2_api": "4.2.1",
-                "woodstox_core": "6.2.1",
-                "vector": "0.31.0",
-            },
-            {
-                "product": "3.4.0",
-                "java-base": "11",
-                "hadoop_short_version": "3",
                 "hadoop_long_version": "3.3.4",  # https://github.com/apache/spark/blob/1db2f5c36b120c213432fc658c9fd24fc73cb45e/pom.xml#L122
                 "aws_java_sdk_bundle": "1.12.262",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.3.4
                 "azure_storage": "7.0.1",  # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4
@@ -452,7 +421,6 @@
     "kafka": {"id": "625ff25b91bdcd4b49c823a4"},
     "nifi": {"id": "625586a32e9e14bc8118e203"},
     "opa": {"id": "6255838bea1feb8bec4aaaa3"},
-    "pyspark-k8s": {"id": "625581e28801ddc4aad30d5c"},
     "spark-k8s": {"id": "62613e81f8ce82a2f247dda5"},
     "superset": {"id": "62557e5fea1feb8bec4aaaa0"},
     "tools": {"id": "62557cd575ab7e30884aaaa0"},