diff --git a/.github/workflows/build-image.sh b/.github/workflows/build-image.sh index 72670f255c..c6ee19bdb6 100755 --- a/.github/workflows/build-image.sh +++ b/.github/workflows/build-image.sh @@ -13,8 +13,8 @@ if [ "$TAG" == "" ]; then exit 1 fi if [ "$DEFAULT_JDK" = "" ]; then - echo "DEFAULT_JDK not found using 11" - DEFAULT_JDK=11 + echo "DEFAULT_JDK not found using 17" + DEFAULT_JDK=17 else echo "DEFAULT_JDK=$DEFAULT_JDK" fi @@ -55,7 +55,8 @@ if [ ! -f "$JAR" ]; then exit $RC fi fi -for v in 8 11 17; do +# TODO add Java 21 when packeto supports it +for v in 17; do echo "Creating: $REPO:$TAG-jdk$v" pack build --builder gcr.io/paketo-buildpacks/builder:base \ --path "$JAR" \ @@ -93,9 +94,5 @@ for v in 8 11 17; do fi fi done -#if [ "$PUSH" == "true" ]; then -# echo "Pruning Docker" -# docker system prune -f -# docker system prune --volumes -f -#fi + diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 23cf8926a4..b1f7a59832 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -81,7 +81,7 @@ jobs: shell: bash env: TAG: ${{ needs.prepare.outputs.version }} - DEFAULT_JDK: '11' + DEFAULT_JDK: '17' ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} DELETE_TAGS: ${{ inputs.delete-tags }} diff --git a/.github/workflows/workflow.adoc b/.github/workflows/workflow.adoc new file mode 100644 index 0000000000..bbd98775dc --- /dev/null +++ b/.github/workflows/workflow.adoc @@ -0,0 +1,71 @@ += Workflow Reference + +This README serves as a guide to the GitHub Action workflows included in this repository. +It outlines the purpose and functionality of each workflow, detailing their role in the CI and release processes. +Additionally, this document provides an overview of the scripts and actions utilized in these workflows, offering insights into how they work together in SCDF's CI/CD pipeline. + +This document is a work in progress, and as various workflows are updated, the documentation will be revised to reflect both existing and new behaviors. + + +== Building Docker Images and pushing the containers to DockerHub +.This diagram shows the flow of execution of how workflows create Docker imges. +``` +┌─────────────────────────┐ +│ │ +│ │ +│build-snapshot-worker.yml┼────┐ +│ │ │ +│ │ │ +└─────────────────────────┘ │ +┌─────────────────────────┐ │ +│ │ │ +│ │ │ +│ ci-images.yml ┼────┤ ┌─────────────────────────┐ ┌─────────────────────────┐ +│ │ │ │ │ │ │ +│ │ │ │ │ │ │ +└─────────────────────────┘ ├────►│ build-images.yml ┼────────►│ build-image.sh │ +┌─────────────────────────┐ │ │ │ │ │ +│ │ │ │ │ │ │ +│ │ │ └───────────┬─────────────┘ └─────────────────────────┘ +│ ci.yml ┼────┤ │ +│ │ │ │ +│ │ │ ┌───────────┴─────────────┐ +└─────────────────────────┘ │ │ │ +┌─────────────────────────┐ │ │ │ +│ │ │ │ images.json │ +│ │ │ │ │ +│ release-worker.yml ┼────┘ │ │ +│ │ └─────────────────────────┘ +│ │ +└─────────────────────────┘ +``` + +Part of the release and CI process involves creating and pushing images to a registry (such as DockerHub) for the Dataflow server, Skipper server, CTR app, and other components. +This process is managed by the `build-images` (build-images.yml) workflow. While the `build-images` workflow is typically not run independently, it is invoked by other workflows that handle CI builds and releases. +The `build-images` workflow determines which images to create based on the `images.json` file. +This file contains metadata on the primary SCDF components that need to have an associated image. +Each entry specifies the location (directory) where the jar can be found, jar name, and image name for each artifact that will be used to construct the image. +For each entry in the `images.json` file, the workflow calls the `build-image.sh` script, which retrieves the jar, builds the image, and then pushes it to the registry. + +SCDF also provides images for external applications that support some of the optional features that are offered by dataflow. +These include Grafana and Prometheus local. +These images are created and pushed using the docker/build-push-action@v2 action. + +=== Scripts used to build images +As mentioned above, the `build-image.sh` script is responsible for building the specified image based on the parameters provided and then pushing the image to Dockerhub. +This script uses Paketo to build an image for each of the supported Java versions using the corresponding jar file. +The resulting image name will look something like `spring-cloud-dataflow-server:3.0.0-SNAPSHOT-jdk17`. +Additionally, the script creates a default image using the default Java version as specified by the `DEFAULT_JDK` environment variable. + +The format for running the `build-image.sh` is as follows: +[source, bash] +``` +bash +./build-image.sh +``` + +There is an optional `DEFAULT_JDK` environment variable that allows you to set the JDK version for the default image created. +If not the script will set it to its current setting (which as of the writing of this document is `17`). + +NOTE: When new releases of Java are available and are compliant with the SCDF release, they need to be added to the `build-image.sh` script. + diff --git a/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java b/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java index 64441e7100..02c584aee3 100644 --- a/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java +++ b/spring-cloud-dataflow-rest-resource/src/main/java/org/springframework/cloud/dataflow/rest/support/jackson/JobParameterJacksonDeserializer.java @@ -54,6 +54,7 @@ public JobParameter deserialize(JsonParser jsonParser, DeserializationContext de try { jobParameter = new JobParameter(value, Class.forName(type), identifying); } catch (ClassNotFoundException e) { + logger.warn("JobParameter type %s is not supported by DataFlow. Verify type is valid or in classpath.".formatted(type) ); throw new IllegalArgumentException("JobParameter type %s is not supported by DataFlow".formatted(type), e); } } diff --git a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java index 23ea329fde..19e8b263c4 100644 --- a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java +++ b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/DataFlowIT.java @@ -1314,8 +1314,6 @@ public void composedTask() { assertThat(taskBuilder.allTasks().size()).isEqualTo(0); } - //TODO: Boot3x followup - @Disabled("TODO: Boot3x followup Wait for composed Task runner to be ported to 3.x") @Test public void multipleComposedTaskWithArguments() { logger.info("task-multiple-composed-task-with-arguments-test"); diff --git a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java index c7f8b48816..d2527afb3b 100644 --- a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java +++ b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/db/AbstractDataflowTests.java @@ -64,7 +64,8 @@ protected static class EmptyConfig { ClusterContainer.from(TagNames.DATAFLOW_2_8, DATAFLOW_IMAGE_PREFIX + "2.8.4"), ClusterContainer.from(TagNames.DATAFLOW_2_9, DATAFLOW_IMAGE_PREFIX + "2.9.6"), ClusterContainer.from(TagNames.DATAFLOW_2_10, DATAFLOW_IMAGE_PREFIX + "2.10.3"), - ClusterContainer.from(TagNames.DATAFLOW_2_11, DATAFLOW_IMAGE_PREFIX + "2.11.3") + ClusterContainer.from(TagNames.DATAFLOW_2_11, DATAFLOW_IMAGE_PREFIX + "2.11.4"), + ClusterContainer.from(TagNames.DATAFLOW_3_0, DATAFLOW_IMAGE_PREFIX + "3.0.0") ); public final static List SKIPPER_CONTAINERS = Arrays.asList( @@ -72,7 +73,8 @@ protected static class EmptyConfig { ClusterContainer.from(TagNames.SKIPPER_2_7, SKIPPER_IMAGE_PREFIX + "2.7.4"), ClusterContainer.from(TagNames.SKIPPER_2_8, SKIPPER_IMAGE_PREFIX + "2.8.6"), ClusterContainer.from(TagNames.SKIPPER_2_9, SKIPPER_IMAGE_PREFIX + "2.9.3"), - ClusterContainer.from(TagNames.SKIPPER_2_11, SKIPPER_IMAGE_PREFIX + "2.11.3") + ClusterContainer.from(TagNames.SKIPPER_2_11, SKIPPER_IMAGE_PREFIX + "2.11.4"), + ClusterContainer.from(TagNames.SKIPPER_3_0, SKIPPER_IMAGE_PREFIX + "3.0.0") ); public final static List DATABASE_CONTAINERS = Arrays.asList( diff --git a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java index 074b4e5a8a..da55beb2c3 100644 --- a/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java +++ b/spring-cloud-dataflow-server/src/test/java/org/springframework/cloud/dataflow/integration/test/tags/TagNames.java @@ -73,8 +73,11 @@ public abstract class TagNames { public static final String SKIPPER_2_8 = "skipper_2_8"; public static final String SKIPPER_2_9 = "skipper_2_9"; + public static final String SKIPPER_2_11 = "skipper_2_11"; + public static final String SKIPPER_3_0 = "skipper_3_0"; + public static final String SKIPPER_main = "skipper_main"; public static final String DATAFLOW = "dataflow"; @@ -86,7 +89,10 @@ public abstract class TagNames { public static final String DATAFLOW_2_9 = "dataflow_2_9"; public static final String DATAFLOW_2_10 = "dataflow_2_10"; + public static final String DATAFLOW_2_11 = "dataflow_2_11"; + public static final String DATAFLOW_3_0 = "dataflow_3_0"; + public static final String DATAFLOW_main = "dataflow_main"; } diff --git a/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml b/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml index 4280774905..116a1dce3e 100644 --- a/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml +++ b/spring-cloud-dataflow-server/src/test/resources/docker-compose-maven-it-task-import.yml @@ -1,5 +1,3 @@ -version: '3' - # CI specific test Task (scenario) registered from maven resource services: diff --git a/src/docker-compose/docker-compose-prometheus.yml b/src/docker-compose/docker-compose-prometheus.yml index d06f5b0a23..a4a2ba178f 100644 --- a/src/docker-compose/docker-compose-prometheus.yml +++ b/src/docker-compose/docker-compose-prometheus.yml @@ -1,5 +1,3 @@ -version: '3' - # Extends the default docker-compose.yml with Prometheus/Grafana monitoring configuration # Usage: docker-compose -f ./docker-compose.yml -f ./docker-compose-prometheus.yml up services: diff --git a/src/docker-compose/docker-compose.yml b/src/docker-compose/docker-compose.yml index 86b6e63ba1..cf7d830325 100644 --- a/src/docker-compose/docker-compose.yml +++ b/src/docker-compose/docker-compose.yml @@ -1,4 +1,3 @@ -version: '3' # Configuration environment variables: # - DATAFLOW_VERSION and SKIPPER_VERSION specify what DataFlow and Skipper image versions to use. # - STREAM_APPS_URI and TASK_APPS_URI are used to specify what Stream and Task applications to pre-register. @@ -21,7 +20,7 @@ version: '3' services: dataflow-server: user: root - image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-2.11.2-SNAPSHOT}${BP_JVM_VERSION:-} + image: springcloud/spring-cloud-dataflow-server:${DATAFLOW_VERSION:-3.0.0-SNAPSHOT}${BP_JVM_VERSION:-} container_name: dataflow-server ports: - "9393:9393" @@ -64,7 +63,7 @@ services: skipper-server: user: root - image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-2.11.2-SNAPSHOT}${BP_JVM_VERSION:-} + image: springcloud/spring-cloud-skipper-server:${SKIPPER_VERSION:-3.0.0-SNAPSHOT}${BP_JVM_VERSION:-} container_name: skipper-server ports: - "7577:7577"