diff --git a/.github/workflows/dev_jupyterhub-keycloak-spark.yaml b/.github/workflows/dev_jupyterhub-keycloak-spark.yaml new file mode 100644 index 00000000..97e3c241 --- /dev/null +++ b/.github/workflows/dev_jupyterhub-keycloak-spark.yaml @@ -0,0 +1,29 @@ +--- +name: Build and publish Spark for jupyterhub-keycloak demo + +on: + workflow_dispatch: + push: + branches: + - main + # TODO (@NickLarsenNZ): Also build on release branches, but with a stackable0.0.0-dev or stackableYY.M.X tag. + # The current problem is that we don't know the patch level, so do we always build for `.0`, or just drop that + # (eg: stackableYY.M), but then do we do the same for main (eg: 0.0-dev)? + # - release-* + paths: + - demos/jupyterhub-keycloak/Dockerfile + - .github/workflows/dev_jupyterhub-keycloak-spark.yaml + +jobs: + build_image: + name: Reusable Workflow + uses: ./.github/workflows/reusable_build_image.yaml + secrets: + harbor-robot-secret: ${{ secrets.HARBOR_ROBOT_DEMOS_GITHUB_ACTION_BUILD_SECRET }} + slack-token: ${{ secrets.SLACK_CONTAINER_IMAGE_TOKEN }} + with: + image-name: spark + # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that + # the demo is reproducable for the release and it will be automatically replaced for the release branch. + image-version: 3.5.2-python311 + containerfile-path: demos/jupyterhub-keycloak/Dockerfile diff --git a/stacks/jupyterhub-keycloak/Dockerfile b/demos/jupyterhub-keycloak/Dockerfile similarity index 70% rename from stacks/jupyterhub-keycloak/Dockerfile rename to demos/jupyterhub-keycloak/Dockerfile index d847f41e..b3dece7f 100644 --- a/stacks/jupyterhub-keycloak/Dockerfile +++ b/demos/jupyterhub-keycloak/Dockerfile @@ -1,9 +1,9 @@ -# docker build -t oci.stackable.tech/sandbox/spark:3.5.5-python311 -f Dockerfile . -# kind load docker-image oci.stackable.tech/sandbox/spark:3.5.5-python311 -n stackable-data-platform +# docker build -t oci.stackable.tech/demos/spark:3.5.2-python311 -f Dockerfile . +# kind load docker-image oci.stackable.tech/demos/spark:3.5.2-python311 -n stackable-data-platform # or: -# docker push oci.stackable.tech/sandbox/spark:3.5.5-python311 +# docker push oci.stackable.tech/demos/spark:3.5.2-python311 -FROM spark:3.5.5-scala2.12-java17-ubuntu +FROM spark:3.5.2-scala2.12-java17-ubuntu USER root diff --git a/docs/modules/demos/images/jupyterhub-keycloak/load-nb.png b/docs/modules/demos/images/jupyterhub-keycloak/load-nb.png index 4fa36126..92708ac7 100644 Binary files a/docs/modules/demos/images/jupyterhub-keycloak/load-nb.png and b/docs/modules/demos/images/jupyterhub-keycloak/load-nb.png differ diff --git a/docs/modules/demos/pages/jupyterhub-keycloak.adoc b/docs/modules/demos/pages/jupyterhub-keycloak.adoc index 93d2ba1e..90200c1b 100644 --- a/docs/modules/demos/pages/jupyterhub-keycloak.adoc +++ b/docs/modules/demos/pages/jupyterhub-keycloak.adoc @@ -103,7 +103,7 @@ This setup is ideal for interactive data processing. === Spark Configuration -* **Executor Image**: Uses a custom image `oci.stackable.tech/sandbox/spark:3.5.5-python311` (built on the standard Spark image) for the executors, matching the Python version of the notebook. +* **Executor Image**: Uses a custom image `oci.stackable.tech/demos/spark:3.5.2-python311` (built on the standard Spark image) for the executors, matching the Python version of the notebook. * **Resource Allocation**: Configures Spark executor instances, memory, and cores through settings defined in the notebook. * **Hadoop and AWS Libraries**: Includes necessary Hadoop and AWS libraries for S3 operations, matching the notebook image version. diff --git a/stacks/jupyterhub-keycloak/jupyterhub.yaml b/stacks/jupyterhub-keycloak/jupyterhub.yaml index ed11b3dc..375763e1 100644 --- a/stacks/jupyterhub-keycloak/jupyterhub.yaml +++ b/stacks/jupyterhub-keycloak/jupyterhub.yaml @@ -209,7 +209,7 @@ options: image: display_name: Image choices: -{% for image in ["quay.io/jupyter/pyspark-notebook:python-3.11.9", "quay.io/jupyter/pyspark-notebook:spark-3.5.5"] %} +{% for image in ["quay.io/jupyter/pyspark-notebook:python-3.11.9", "quay.io/jupyter/pyspark-notebook:spark-3.5.2"] %} "{{image}}": display_name: "{{image}}" kubespawner_override: diff --git a/stacks/jupyterhub-keycloak/process-s3.ipynb b/stacks/jupyterhub-keycloak/process-s3.ipynb index f13dd25f..c28d3b7c 100644 --- a/stacks/jupyterhub-keycloak/process-s3.ipynb +++ b/stacks/jupyterhub-keycloak/process-s3.ipynb @@ -49,8 +49,8 @@ "acts as the driver. It is important that the versions of spark and python match across the driver (running in the juypyterhub image)\n", "and the executor(s) (running in a separate image, specified below with the `spark.kubernetes.container.image` setting).\n", "\n", - "The jupyterhub image `quay.io/jupyter/pyspark-notebook:spark-3.5.5` uses a base ubuntu image (like the spark images).\n", - "The versions of java match exactly. Python versions can differ at patch level, and the image used below `oci.stackable.tech/sandbox/spark:3.5.5-python311` is built from a `spark:3.5.2-scala2.12-java17-ubuntu` base image with python 3.11 (the same major/minor version as the notebook) installed.\n", + "The jupyterhub image `quay.io/jupyter/pyspark-notebook:spark-3.5.2` uses a base ubuntu image (like the spark images).\n", + "The versions of java match exactly. Python versions can differ at patch level, and the image used below `oci.stackable.tech/demos/spark:3.5.2-python311` is built from a `spark:3.5.2-scala2.12-java17-ubuntu` base image with python 3.11 (the same major/minor version as the notebook) installed.\n", "\n", "## S3\n", "As we will be reading data from an S3 bucket, we need to add the necessary `hadoop` and `aws` libraries in the same hadoop version as the\n", @@ -69,7 +69,7 @@ "NAMESPACE = os.environ.get(\"NAMESPACE\", \"default\")\n", "POD_NAME = os.environ.get(\"HOSTNAME\", f\"jupyter-{os.environ.get('USER', 'default')}-{NAMESPACE}\")\n", "\n", - "EXECUTOR_IMAGE = \"oci.stackable.tech/sandbox/spark:3.5.5-python311\" \n", + "EXECUTOR_IMAGE = \"oci.stackable.tech/demos/spark:3.5.2-python311\" \n", "\n", "spark = (\n", " SparkSession.builder\n",