Skip to content

Commit 89ab3db

Browse files
andre-marcos-perezandre-marcos-perez
authored
Merge pull request #95 from cluster-apps-on-docker/feature/bump-version
[Feature] Bump version to latest
2 parents d908cb6 + ee2b278 commit 89ab3db

File tree

13 files changed

+79
-248
lines changed

13 files changed

+79
-248
lines changed

‎.github/actions/build-jupyterlab/action.yml‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,10 +19,6 @@ inputs:
1919
DOCKERHUB_PWD:
2020
description: 'Docker Hub password to push images with'
2121
required: true
22-
IS_LATEST:
23-
description: 'Indicate whether the images is latest'
24-
required: false
25-
default: "false"
2622
runs:
2723
using: "composite"
2824
steps:
@@ -51,5 +47,4 @@ runs:
5147
docker tag jupyterlab:${{ inputs.JUPYTERLAB_VERSION }}-spark-${{ inputs.SPARK_VERSION }} ${{ inputs.DOCKERHUB_USR }}/jupyterlab:latest
5248
echo ${{ inputs.DOCKERHUB_PWD }} | docker login -u ${{ inputs.DOCKERHUB_USR }} --password-stdin
5349
if [ "${GITHUB_REF}" == "refs/heads/master" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/jupyterlab:${{ inputs.JUPYTERLAB_VERSION }}-spark-${{ inputs.SPARK_VERSION }}; fi
54-
if [ "${GITHUB_REF}" == "refs/heads/master" ] && [ "${{ inputs.IS_LATEST }}" == "true" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/jupyterlab:latest; fi
55-
50+
if [ "${GITHUB_REF}" == "refs/heads/master" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/jupyterlab:latest; fi

‎.github/actions/build-spark/action.yml‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,10 +16,6 @@ inputs:
1616
DOCKERHUB_PWD:
1717
description: 'Docker Hub password to push images with'
1818
required: true
19-
IS_LATEST:
20-
description: 'Indicate whether the images is latest'
21-
required: false
22-
default: ""
2319
runs:
2420
using: "composite"
2521
steps:
@@ -55,7 +51,7 @@ runs:
5551
docker tag spark-master:${{ inputs.SPARK_VERSION }} ${{ inputs.DOCKERHUB_USR }}/spark-master:latest
5652
echo ${{ inputs.DOCKERHUB_PWD }} | docker login -u ${{ inputs.DOCKERHUB_USR }} --password-stdin
5753
if [ "${GITHUB_REF}" == "refs/heads/master" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/spark-master:${{ inputs.SPARK_VERSION }}; fi
58-
if [ "${GITHUB_REF}" == "refs/heads/master" ] && [ "${{ inputs.IS_LATEST }}" == "true" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/spark-master:latest; fi
54+
if [ "${GITHUB_REF}" == "refs/heads/master" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/spark-master:latest; fi
5955
- name: Build & Push Spark Worker Image
6056
shell: bash
6157
run: |
@@ -69,4 +65,4 @@ runs:
6965
docker tag spark-worker:${{ inputs.SPARK_VERSION }} ${{ inputs.DOCKERHUB_USR }}/spark-worker:latest
7066
echo ${{ inputs.DOCKERHUB_PWD }} | docker login -u ${{ inputs.DOCKERHUB_USR }} --password-stdin
7167
if [ "${GITHUB_REF}" == "refs/heads/master" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/spark-worker:${{ inputs.SPARK_VERSION }}; fi
72-
if [ "${GITHUB_REF}" == "refs/heads/master" ] && [ "${{ inputs.IS_LATEST }}" == "true" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/spark-worker:latest; fi
68+
if [ "${GITHUB_REF}" == "refs/heads/master" ]; then docker push ${{ inputs.DOCKERHUB_USR }}/spark-worker:latest; fi

‎.github/workflows/master.yml‎

Lines changed: 9 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,5 @@
11
name: build-master
22
on:
3-
schedule:
4-
- cron: '0 0 * * *'
53
push:
64
branches:
75
- 'master'
@@ -15,35 +13,14 @@ jobs:
1513
steps:
1614
- name: Checkout Code
1715
uses: actions/checkout@v2
18-
- name: Build Spark v3.0.0 (latest)
16+
- name: Build Spark v3.5.7 (latest)
1917
uses: ./.github/actions/build-spark
2018
with:
2119
DOCKERHUB_USR: ${DOCKERHUB_USR}
2220
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
23-
SCALA_VERSION: "2.12.10"
24-
SPARK_VERSION: "3.0.0"
25-
HADOOP_VERSION: "3.2"
26-
IS_LATEST: "true"
27-
spark-images:
28-
name: Spark Images
29-
runs-on: ubuntu-latest
30-
env:
31-
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
32-
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
33-
strategy:
34-
matrix:
35-
spark_version: [ "2.4.4", "2.4.0" ]
36-
steps:
37-
- name: Checkout Code
38-
uses: actions/checkout@v2
39-
- name: Build Spark v2.4.4 + v2.4.0
40-
uses: ./.github/actions/build-spark
41-
with:
42-
DOCKERHUB_USR: ${DOCKERHUB_USR}
43-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
44-
SCALA_VERSION: "2.11.12"
45-
SPARK_VERSION: ${{ matrix.spark_version }}
46-
HADOOP_VERSION: "2.7"
21+
SCALA_VERSION: "2.12.20"
22+
SPARK_VERSION: "3.5.7"
23+
HADOOP_VERSION: "3"
4724
jupyterlab-latest-images:
4825
name: JupyterLab Images (latest)
4926
runs-on: ubuntu-latest
@@ -53,45 +30,12 @@ jobs:
5330
steps:
5431
- name: Checkout Code
5532
uses: actions/checkout@v2
56-
- name: Build JupyterLab v3.0.0 and Spark v3.0.0 (latest)
57-
uses: ./.github/actions/build-jupyterlab
58-
with:
59-
DOCKERHUB_USR: ${DOCKERHUB_USR}
60-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
61-
SCALA_VERSION: "2.12.10"
62-
SCALA_KERNEL_VERSION: "0.10.9"
63-
SPARK_VERSION: "3.0.0"
64-
JUPYTERLAB_VERSION: "3.0.0"
65-
IS_LATEST: "true"
66-
- name: Build JupyterLab v2.1.4 and Spark v3.0.0
67-
uses: ./.github/actions/build-jupyterlab
68-
with:
69-
DOCKERHUB_USR: ${DOCKERHUB_USR}
70-
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
71-
SCALA_VERSION: "2.12.10"
72-
SCALA_KERNEL_VERSION: "0.10.9"
73-
SPARK_VERSION: "3.0.0"
74-
JUPYTERLAB_VERSION: "2.1.4"
75-
IS_LATEST: "false"
76-
jupyterlab-images:
77-
name: JupyterLab Images
78-
runs-on: ubuntu-latest
79-
env:
80-
DOCKERHUB_USR: ${{ secrets.DOCKERHUB_USR }}
81-
DOCKERHUB_PWD: ${{ secrets.DOCKERHUB_PWD }}
82-
strategy:
83-
matrix:
84-
spark_version: [ "2.4.4", "2.4.0" ]
85-
jupyterlab_version: [ "3.0.0", "2.1.4" ]
86-
steps:
87-
- name: Checkout Code
88-
uses: actions/checkout@v2
89-
- name: Build JupyterLab v3.0.0 + v2.1.4 and Spark v2.4.4 + v2.4.0
33+
- name: Build JupyterLab v4.4.10 and Spark v3.5.7 (latest)
9034
uses: ./.github/actions/build-jupyterlab
9135
with:
9236
DOCKERHUB_USR: ${DOCKERHUB_USR}
9337
DOCKERHUB_PWD: ${DOCKERHUB_PWD}
94-
SCALA_VERSION: "2.11.12"
95-
SCALA_KERNEL_VERSION: "0.6.0"
96-
SPARK_VERSION: ${{ matrix.spark_version }}
97-
JUPYTERLAB_VERSION: ${{ matrix.jupyterlab_version }}
38+
SCALA_VERSION: "2.12.20"
39+
SCALA_KERNEL_VERSION: "0.14.2"
40+
SPARK_VERSION: "3.5.7"
41+
JUPYTERLAB_VERSION: "4.4.10"

‎.github/workflows/staging.yml‎

Lines changed: 0 additions & 95 deletions
This file was deleted.

‎README.md‎

Lines changed: 11 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ docker-compose up
4040
### Cluster overview
4141

4242
| Application | URL | Description |
43-
| --------------- | ---------------------------------------- | ---------------------------------------------------------- |
43+
|-----------------|------------------------------------------|------------------------------------------------------------|
4444
| JupyterLab | [localhost:8888](http://localhost:8888/) | Cluster interface with built-in Jupyter notebooks |
4545
| Spark Driver | [localhost:4040](http://localhost:4040/) | Spark Driver web ui |
4646
| Spark Master | [localhost:8080](http://localhost:8080/) | Spark Master node |
@@ -104,28 +104,27 @@ docker-compose up
104104
- Infra
105105

106106
| Component | Version |
107-
| -------------- | ------- |
107+
|----------------|---------|
108108
| Docker Engine | 1.13.0+ |
109109
| Docker Compose | 1.10.0+ |
110110

111-
- Languages and Kernels
111+
- Languages
112112

113-
| Spark | Hadoop | Scala | [Scala Kernel](https://almond.sh/) | Python | [Python Kernel](https://ipython.org/) | R | [R Kernel](https://irkernel.github.io/) |
114-
| ----- | ------ | ------- | ---------------------------------- | ------ | ------------------------------------- | ----- | --------------------------------------- |
115-
| 3.x | 3.2 | 2.12.10 | 0.10.9 | 3.7.3 | 7.19.0 | 3.5.2 | 1.1.1 |
116-
| 2.x | 2.7 | 2.11.12 | 0.6.0 | 3.7.3 | 7.19.0 | 3.5.2 | 1.1.1 |
113+
| Spark | Hadoop | Scala | [Scala Kernel](https://almond.sh/) | Python | R |
114+
|-------|--------|---------|------------------------------------|--------|-------|
115+
| 3.5.7 | 3 | 2.12.20 | 0.14.2 | 3.12.3 | 4.3.3 |
117116

118117
- Apps
119118

120-
| Component | Version | Docker Tag |
121-
| -------------- | ----------------------- | ---------------------------------------------------- |
122-
| Apache Spark | 2.4.0 \| 2.4.4 \| 3.0.0 | **\<spark-version>** |
123-
| JupyterLab | 2.1.4 \| 3.0.0 | **\<jupyterlab-version>**-spark-**\<spark-version>** |
119+
| Component | Version | Docker Tag |
120+
|--------------|---------|------------------------------------------------------|
121+
| Apache Spark | 3.5.7 | **\<spark-version>** |
122+
| JupyterLab | 4.4.10 | **\<jupyterlab-version>**-spark-**\<spark-version>** |
124123

125124
## <a name="metrics"></a>Metrics
126125

127126
| Image | Size | Downloads |
128-
| -------------------------------------------------------------- | ---------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------- |
127+
|----------------------------------------------------------------|------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------|
129128
| [JupyterLab](https://hub.docker.com/r/andreper/jupyterlab) | ![docker-size-jupyterlab](https://img.shields.io/docker/image-size/andreper/jupyterlab/latest) | ![docker-pull](https://img.shields.io/docker/pulls/andreper/jupyterlab) |
130129
| [Spark Master](https://hub.docker.com/r/andreper/spark-master) | ![docker-size-master](https://img.shields.io/docker/image-size/andreper/spark-master/latest) | ![docker-pull](https://img.shields.io/docker/pulls/andreper/spark-master) |
131130
| [Spark Worker](https://hub.docker.com/r/andreper/spark-worker) | ![docker-size-worker](https://img.shields.io/docker/image-size/andreper/spark-worker/latest) | ![docker-pull](https://img.shields.io/docker/pulls/andreper/spark-worker) |

‎build/build.sh‎

Lines changed: 10 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,28 +8,16 @@
88

99
BUILD_DATE="$(date -u +'%Y-%m-%d')"
1010

11-
SHOULD_BUILD_BASE="$(grep -m 1 build_base build.yml | grep -o -P '(?<=").*(?=")')"
12-
SHOULD_BUILD_SPARK="$(grep -m 1 build_spark build.yml | grep -o -P '(?<=").*(?=")')"
13-
SHOULD_BUILD_JUPYTERLAB="$(grep -m 1 build_jupyter build.yml | grep -o -P '(?<=").*(?=")')"
14-
15-
SPARK_VERSION="$(grep -m 1 spark build.yml | grep -o -P '(?<=").*(?=")')"
16-
JUPYTERLAB_VERSION="$(grep -m 1 jupyterlab build.yml | grep -o -P '(?<=").*(?=")')"
17-
18-
SPARK_VERSION_MAJOR=${SPARK_VERSION:0:1}
19-
20-
if [[ "${SPARK_VERSION_MAJOR}" == "2" ]]
21-
then
22-
HADOOP_VERSION="2.7"
23-
SCALA_VERSION="2.11.12"
24-
SCALA_KERNEL_VERSION="0.6.0"
25-
elif [[ "${SPARK_VERSION_MAJOR}" == "3" ]]
26-
then
27-
HADOOP_VERSION="3.2"
28-
SCALA_VERSION="2.12.10"
29-
SCALA_KERNEL_VERSION="0.10.9"
30-
else
31-
exit 1
32-
fi
11+
SHOULD_BUILD_BASE="$(grep -m 1 build_base build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
12+
SHOULD_BUILD_SPARK="$(grep -m 1 build_spark build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
13+
SHOULD_BUILD_JUPYTERLAB="$(grep -m 1 build_jupyter build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
14+
15+
SPARK_VERSION="$(grep -m 1 spark build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
16+
JUPYTERLAB_VERSION="$(grep -m 1 jupyterlab build.yml | sed -E 's/.*"([^"]*)".*/\1/')"
17+
18+
HADOOP_VERSION="3"
19+
SCALA_VERSION="2.12.20"
20+
SCALA_KERNEL_VERSION="0.14.2"
3321

3422
# ----------------------------------------------------------------------------------------------------------------------
3523
# -- Functions----------------------------------------------------------------------------------------------------------

‎build/build.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
applications:
2-
spark: "3.0.0"
3-
jupyterlab: "3.0.0"
2+
spark: "3.5.7"
3+
jupyterlab: "4.4.10"
44
build:
5-
build_base: "true"
6-
build_spark: "true"
5+
build_base: "false"
6+
build_spark: "false"
77
build_jupyterlab: "true"

‎build/docker/base/Dockerfile‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
1-
ARG debian_buster_image_tag=8-jre-slim
2-
FROM openjdk:${debian_buster_image_tag}
3-
LABEL manteiner="Andre Perez <[email protected]>"
1+
ARG java_image_tag=21.0.8-jre-headless-ubuntu-24.04
2+
FROM sapmachine:${java_image_tag}
3+
LABEL maintainer="Andre Perez <[email protected]>"
44

55
# -- Layer: Image Metadata
66

@@ -18,13 +18,16 @@ ARG scala_version
1818
ARG shared_workspace=/opt/workspace
1919

2020
RUN mkdir -p ${shared_workspace}/data && \
21-
mkdir -p /usr/share/man/man1 && \
22-
apt-get update -y && \
23-
apt-get install -y curl python3 r-base && \
24-
ln -s /usr/bin/python3 /usr/bin/python && \
25-
curl https://downloads.lightbend.com/scala/${scala_version}/scala-${scala_version}.deb -k -o scala.deb && \
26-
apt install -y ./scala.deb && \
27-
rm -rf scala.deb /var/lib/apt/lists/*
21+
mkdir -p /usr/share/man/man1
22+
23+
RUN apt-get -y update && \
24+
apt-get -y install curl python3 r-base && \
25+
apt-get -y clean
26+
27+
RUN curl -L https://github.com/scala/scala/releases/download/v${scala_version}/scala-${scala_version}.deb -o scala.deb && \
28+
apt-get -y install ./scala.deb && \
29+
rm -rf scala.deb && \
30+
apt-get -y clean
2831

2932
ENV SCALA_HOME="/usr/bin/scala"
3033
ENV PATH=${PATH}:${SCALA_HOME}/bin

0 commit comments

Comments
 (0)