From 3de91faa3337b33ac7d35d2c75c47accf6607aa3 Mon Sep 17 00:00:00 2001 From: Evan Pagryzinski Date: Wed, 20 Nov 2024 11:45:48 -0500 Subject: [PATCH] Adding poderize scrpits Adding scripts to create podman images and upload them to a docker registry as well as updating the Dockerfile's to play nice with podman --- .../broker/Dockerfile | 2 +- .../eventsbroker/Dockerfile | 2 +- .../redis/Dockerfile | 2 +- scripts/poderize.sh | 374 +++++++++++++++ scripts/poderizeGithub.sh | 449 ++++++++++++++++++ wres-broker/Dockerfile | 2 +- wres-eventsbroker/Dockerfile | 2 +- wres-eventsbroker/docker-entrypoint.sh | 2 - wres-redis/Dockerfile | 2 +- 9 files changed, 829 insertions(+), 8 deletions(-) create mode 100644 scripts/poderize.sh create mode 100644 scripts/poderizeGithub.sh diff --git a/scripts/localWindowsDeployScripts/broker/Dockerfile b/scripts/localWindowsDeployScripts/broker/Dockerfile index fd6130f6e..53d417924 100644 --- a/scripts/localWindowsDeployScripts/broker/Dockerfile +++ b/scripts/localWindowsDeployScripts/broker/Dockerfile @@ -8,7 +8,7 @@ RUN echo "wres_docker:x:498:1370800073::/home/wres_docker:" >> /etc/passwd \ && echo "user:!:1:0:99999:7:::" >> /etc/shadow \ && echo "wres:x:1370800073:" >> /etc/group \ && mkdir /home/wres_docker \ - && chown wres_docker: /home/wres_docker + && chown wres_docker /home/wres_docker # Enable authentication with x509 client certificates (requires some conf too): RUN rabbitmq-plugins enable --offline rabbitmq_auth_mechanism_ssl diff --git a/scripts/localWindowsDeployScripts/eventsbroker/Dockerfile b/scripts/localWindowsDeployScripts/eventsbroker/Dockerfile index d51ebf057..10206f06f 100644 --- a/scripts/localWindowsDeployScripts/eventsbroker/Dockerfile +++ b/scripts/localWindowsDeployScripts/eventsbroker/Dockerfile @@ -62,7 +62,7 @@ WORKDIR ${BROKER_WORK} # Create directories with the correct permissions RUN mkdir -p {${BROKER_HOME},${BROKER_CONFIG},${BROKER_INSTANCE}} && \ - chown ${BROKER_USER}: ${BROKER_INSTANCE} + chown ${BROKER_USER} ${BROKER_INSTANCE} COPY --from=builder /workspace/out ${BROKER_HOME} diff --git a/scripts/localWindowsDeployScripts/redis/Dockerfile b/scripts/localWindowsDeployScripts/redis/Dockerfile index b9c917e28..28d8f793c 100644 --- a/scripts/localWindowsDeployScripts/redis/Dockerfile +++ b/scripts/localWindowsDeployScripts/redis/Dockerfile @@ -6,7 +6,7 @@ RUN echo "wres_persister_docker:x:498:1370800073::/home/wres_persister_docker:" && echo "user:!:1:0:99999:7:::" >> /etc/shadow \ && echo "wres:x:1370800073:" >> /etc/group \ && mkdir -p /home/wres_persister_docker \ - && chown wres_persister_docker: /home/wres_persister_docker + && chown wres_persister_docker /home/wres_persister_docker COPY redis.conf /etc/redis/redis.conf COPY docker-healthcheck /usr/local/bin/ diff --git a/scripts/poderize.sh b/scripts/poderize.sh new file mode 100644 index 000000000..65a587b92 --- /dev/null +++ b/scripts/poderize.sh @@ -0,0 +1,374 @@ +#!/bin/bash + +# poderize.sh +# +# Usage: +# +# cd +# scripts/poderize.sh +# +# Arguments: +# +# All other arguments are optional, so that if one is not specified, then it is assumed +# to be "auto". A version of "auto" will result in the default version, obtained +# through the versions.sh script, being used. +# +# All arguments are positional. For example, if you want to specify the tasker +# version, then you need to specify the core and worker shim versions as "auto", +# first. +# +# Description: +# +# The purpose of this script is to avoid manual errors when tagging an image. +# It is not a replacement for understanding images, containers, and tags. +# Below steps can be run manually as well, and should be run when errors are not +# visible. +# +# Script needs to be run from the root of wres directory so that other scripts +# that depend on the context of the wres directory will work (such as those that +# depend on git depending on the context of directory to work). +# +# This script should be idempotent, meaning you can run it several times in +# a row without damage. This is true for the build steps, tag steps, push steps. +# +# Depends on versions.sh script + + +#============================================================= +# Identify default versions! +#============================================================= + +# Attempt to auto-detect the versions needed. +all_versions=$( scripts/versions.sh ) + +overall_version=$( echo "$all_versions" | grep "^Main version" | cut -d' ' -f3 ) +tasker_version=$( echo "$all_versions" | grep "^wres-tasker version" | cut -d' ' -f3 ) +broker_version=$( echo "$all_versions" | grep "^wres-broker version" | cut -d' ' -f3 ) +redis_version=$( echo "$all_versions" | grep "^wres-redis version" | cut -d' ' -f3 ) +wres_worker_shim_version=$( echo "$all_versions" | grep "^wres-worker version" | cut -d' ' -f3 ) +eventsbroker_version=$( echo "$all_versions" | grep "^wres-eventsbroker version" | cut -d' ' -f3 ) +graphics_version=$( echo "$all_versions" | grep "^wres-vis version" | cut -d' ' -f3 ) +writing_version=$( echo "$all_versions" | grep "^wres-writing version" | cut -d' ' -f3 ) + +# These will be the zip ids, as distinct from the previously-found image ids. +wres_core_version=$overall_version +wres_tasker_version=$tasker_version +wres_vis_version=$graphics_version +wres_writing_version=$writing_version + + +# Sometimes auto-detection of versions does not work, because if no code changed +# then gradle will not create a new zip file. So the caller must specify each +# version with positional args, or "auto" to retain auto-detected version. + +if [[ "$1" != "" && "$1" != "auto" ]] +then + wres_core_version=$1 +fi + +if [[ "$2" != "" && "$2" != "auto" ]] +then + wres_worker_shim_version=$2 +fi + +if [[ "$3" != "" && "$3" != "auto" ]] +then + wres_tasker_version=$3 +fi + +if [[ "$4" != "" && "$4" != "auto" ]] +then + broker_version=$4 +fi + +if [[ "$5" != "" && "$5" != "auto" ]] +then + redis_version=$5 +fi + +if [[ "$6" != "" && "$6" != "auto" ]] +then + eventsbroker_version=$6 +fi + +if [[ "$7" != "" && "$7" != "auto" ]] +then + wres_vis_version=$7 +fi + +if [[ "$8" != "" && "$8" != "auto" ]] +then + wres_writing_version=$8 +fi + +echo "" +echo "VERSIONS USED BASED ON DEFAULTS WITH ARGUMENT OVERRIDES:" +echo "" +echo "Core WRES binary zip version is $wres_core_version" +echo "WRES Worker shim binary zip version is $wres_worker_shim_version" +echo "WRES Tasker binary zip version is $wres_tasker_version" +echo "Primary podman image version is $overall_version" +echo "Tasker podman image version is $tasker_version" +echo "Broker podman image version is $broker_version" +echo "Redis podman image version is $redis_version" +echo "WRES events broker podman image version is $eventsbroker_version" +echo "WRES vis binary zip version is $wres_vis_version" +echo "WRES graphics podman image version is $graphics_version" +echo "WRES writing podman image version is $wres_writing_version" + + +#============================================================= +# Identify zip files and Jenkins URLs; wait for zips +#============================================================= +echo "" +echo "Identifying .zip files required..." + +wres_core_file=wres-${wres_core_version}.zip +worker_shim_file=wres-worker-${wres_worker_shim_version}.zip +tasker_file=wres-tasker-${wres_tasker_version}.zip +vis_file=wres-vis-${wres_vis_version}.zip +writing_file=wres-writing-${wres_writing_version}.zip + +# Ensure the distribution zip files are present for successful podman build --format docker +if [[ ! -f ./build/distributions/$wres_core_file || \ + ! -f ./wres-worker/build/distributions/$worker_shim_file || \ + ! -f ./wres-tasker/build/distributions/$tasker_file || \ + ! -f ./wres-writing/build/distributions/$writing_file || \ + ! -f ./wres-vis/build/distributions/$vis_file ]] +then + echo "" + echo "It appears you are not an automated build server (or something went wrong if you are)." + echo "" + echo "You do not have one of the required files, check the bellow exist" + echo "" + echo "./build/distributions/$wres_core_file" + echo "./wres-worker/build/distributions/$worker_shim_file" + echo "./wres-tasker/build/distributions/$tasker_file" + echo "./wres-writing/build/distributions/$writing_file" + echo "./wres-vis/build/distributions/$vis_file" + echo "" + echo "" + exit 3 +fi + + +#============================================================= +# Build the images +#============================================================= +echo "" +echo "Building images..." + +# Build and tag the worker image which is composed of WRES core and worker shim. +# Tag will be based on the later image version which is WRES core at git root. +echo "Building and tagging worker image..." +worker_image_id=$( podman build --format docker --build-arg version=$wres_core_version --build-arg worker_version=$wres_worker_shim_version --quiet --tag wres/wres-worker:$overall_version . ) +echo "Built wres/wres-worker:$overall_version -- $worker_image_id" + +# Build and tag the tasker image which solely contains the tasker. +echo "Building tasker image..." +pushd wres-tasker +tasker_image_id=$( podman build --format docker --build-arg version=$wres_tasker_version --tag wres/wres-tasker:$tasker_version . ) +popd + +echo "Built wres/wres-tasker:$tasker_version -- $tasker_image_id" + +# Build and tag the broker image +echo "Building broker image..." +pushd wres-broker +broker_image_id=$( podman build --format docker --pull --no-cache --build-arg version=$broker_version --tag wres/wres-broker:$broker_version . ) +popd + +echo "Built wres/wres-broker:$broker_version -- $broker_image_id" + +# Build and tag the redis image +echo "Building redis image..." +pushd wres-redis +redis_image_id=$( podman build --format docker --pull --no-cache --build-arg version=$redis_version --tag wres/wres-redis:$redis_version . ) +popd + +echo "Built wres/wres-redis:$redis_version -- $redis_image_id" + +# Build and tag the eventsbroker image +echo "Building events broker image..." +pushd wres-eventsbroker +eventsbroker_image_id=$( podman build --format docker --no-cache --build-arg version=$eventsbroker_version --tag wres/wres-eventsbroker:$eventsbroker_version . ) +popd + +echo "Built wres/wres-eventsbroker:$eventsbroker_version -- $eventsbroker_image_id" + +# Build and tag the graphics image +echo "Building graphics image..." +pushd wres-vis +graphics_image_id=$( podman build --format docker --build-arg version=$wres_vis_version --tag wres/wres-graphics:$graphics_version . ) +popd + +echo "Built wres/wres-graphics:$graphics_version -- $graphics_image_id" + +# Build and tag the writing image +echo "Building writing image..." +pushd wres-writing +writing_image_id=$( podman build --format docker --build-arg version=$wres_writing_version --tag wres/wres-writing:$writing_version . ) +popd + +echo "Built wres/wres-writing:$writing_version -- $writing_image_id" + +echo "Displaying most recent 20 docker images" +docker image ls | head -n 21 + + +#============================================================= +# Docker Registry +#============================================================= + +# Optional: set environment variable DOCKER_REGISTRY to the FQDN of a docker +# registry (without any path, full fqdn, without scheme) + +if [[ ! -z "$DOCKER_REGISTRY" ]] +then + # Check the format of the registry env var. If something is wrong, then don't use it. + if [[ $DOCKER_REGISTRY =~ ^https?:// ]] +# if [[ $DOCKER_REGISTRY == http* ]] + then + echo "" + echo "You provided a DOCKER_REGISTRY, but it starts with http. Don't include the scheme!" + echo "Skipping pushing the images to the registry!" + # It looks good, try to push to the registry. + else + echo "" + echo "Attempting tagging and pushing images to the registry, https://$DOCKER_REGISTRY ..." + echo "Running docker login https://$DOCKER_REGISTRY..." + podman login https://$DOCKER_REGISTRY + login_success=$? + + if [[ ! login_success ]] + then + echo "Failed to login, not going to try to push to registry. Try again." + exit 2 + fi + + primary_image_dev_status=$( echo ${overall_version} | grep "dev" ) + + if [[ "$primary_image_dev_status" != "" ]] + then + echo "Refusing to tag and push primary docker image version ${overall_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-worker:$overall_version as $DOCKER_REGISTRY/wres/wres-worker/$overall_version..." + podman tag wres/wres-worker:$overall_version $DOCKER_REGISTRY/wres/wres-worker:$overall_version + podman push $DOCKER_REGISTRY/wres/wres-worker:$overall_version + fi + + tasker_image_dev_status=$( echo ${tasker_version} | grep "dev" ) + + if [[ "$tasker_image_dev_status" != "" ]] + then + echo "Refusing to tag and push tasker docker image version ${tasker_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-tasker:$tasker_version as $DOCKER_REGISTRY/wres/wres-tasker/$tasker_version..." + podman tag wres/wres-tasker:$tasker_version $DOCKER_REGISTRY/wres/wres-tasker:$tasker_version + podman push $DOCKER_REGISTRY/wres/wres-tasker:$tasker_version + fi + + broker_image_dev_status=$( echo ${broker_version} | grep "dev" ) + + if [[ "$broker_image_dev_status" != "" ]] + then + echo "Refusing to tag and push broker docker image version ${broker_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-broker:$broker_version as $DOCKER_REGISTRY/wres/wres-broker/$broker_version..." + podman tag wres/wres-broker:$broker_version $DOCKER_REGISTRY/wres/wres-broker:$broker_version + podman push $DOCKER_REGISTRY/wres/wres-broker:$broker_version + fi + + redis_image_dev_status=$( echo ${redis_version} | grep "dev" ) + + if [[ "$redis_image_dev_status" != "" ]] + then + echo "Refusing to tag and push redis docker image version ${redis_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-redis:$redis_version as $DOCKER_REGISTRY/wres/wres-redis/$redis_version..." + podman tag wres/wres-redis:$redis_version $DOCKER_REGISTRY/wres/wres-redis:$redis_version + podman push $DOCKER_REGISTRY/wres/wres-redis:$redis_version + fi + + eventsbroker_image_dev_status=$( echo ${eventsbroker_version} | grep "dev" ) + + if [[ "$eventsbroker_image_dev_status" != "" ]] + then + echo "Refusing to tag and push eventsbroker docker image version ${eventsbroker_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-eventsbroker:$eventsbroker_version as $DOCKER_REGISTRY/wres/wres-eventsbroker/$eventsbroker_version..." + podman tag wres/wres-eventsbroker:$eventsbroker_version $DOCKER_REGISTRY/wres/wres-eventsbroker:$eventsbroker_version + podman push $DOCKER_REGISTRY/wres/wres-eventsbroker:$eventsbroker_version + fi + + graphics_image_dev_status=$( echo ${graphics_version} | grep "dev" ) + + if [[ "$graphics_image_dev_status" != "" ]] + then + echo "Refusing to tag and push graphics docker image version ${graphics_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-graphics:$graphics_version as $DOCKER_REGISTRY/wres/wres-graphics/$graphics_version..." + podman tag wres/wres-graphics:$graphics_version $DOCKER_REGISTRY/wres/wres-graphics:$graphics_version + podman push $DOCKER_REGISTRY/wres/wres-graphics:$graphics_version + fi + + writing_image_dev_status=$( echo ${writing_version} | grep "dev" ) + + if [[ "$writing_image_dev_status" != "" ]] + then + echo "Refusing to tag and push writing docker image version ${writing_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-writing:$writing_version as $DOCKER_REGISTRY/wres/wres-writing/$writing_version..." + podman tag wres/wres-writing:$writing_version $DOCKER_REGISTRY/wres/wres-writing:$writing_version + podman push $DOCKER_REGISTRY/wres/wres-writing:$writing_version + fi + + echo "Tagging and pushing wres/nginx as wres/nginx..." + podman tag wres/nginx $DOCKER_REGISTRY/wres/nginx + podman push $DOCKER_REGISTRY/wres/nginx + fi + +else + echo "" + echo "No variable 'DOCKER_REGISTRY' found, not attempting to docker push." + echo "If you want to automatically push, set DOCKER_REGISTRY to the FQDN of" + echo "an accessible docker registry and this script will attempt to tag and" + echo "push to that registry." + echo "" +fi + +#============================================================= +# Create .yml files +#============================================================= + +echo "" +echo "About to update the .yml files with the new versions based on a template." +echo "If you are only updating some of the images/versions, it is recommended" +echo "you skip this step and do that by manually editing the .ymls." +echo "" + +cp compose-entry.template.yml compose-entry.yml +sed -i "s/TASKER_IMAGE/${tasker_version}/" compose-entry.yml +sed -i "s/BROKER_IMAGE/${broker_version}/" compose-entry.yml +sed -i "s/REDIS_IMAGE/${redis_version}/" compose-entry.yml +sed -i "s/WORKER_IMAGE/${overall_version}/" compose-entry.yml # By design... The tag for the worker image is the "overall_version". +sed -i "s/EVENTS_IMAGE/${eventsbroker_version}/" compose-entry.yml +sed -i "s/GRAPHICS_IMAGE/${graphics_version}/" compose-entry.yml +sed -i "s/WRITING_IMAGE/${writing_version}/" compose-entry.yml +sed -i "s/OVERALL_IMAGE/${overall_version}/" compose-entry.yml + +cp compose-workers.template.yml compose-workers.yml +sed -i "s/TASKER_IMAGE/${tasker_version}/" compose-workers.yml +sed -i "s/BROKER_IMAGE/${broker_version}/" compose-workers.yml +sed -i "s/REDIS_IMAGE/${redis_version}/" compose-workers.yml +sed -i "s/WORKER_IMAGE/${overall_version}/" compose-workers.yml # By design... The tag for the worker image is the "overall_version". +sed -i "s/EVENTS_IMAGE/${eventsbroker_version}/" compose-workers.yml +sed -i "s/GRAPHICS_IMAGE/${graphics_version}/" compose-workers.yml +sed -i "s/WRITING_IMAGE/${writing_version}/" compose-workers.yml +sed -i "s/OVERALL_IMAGE/${overall_version}/" compose-workers.yml + +echo "" +echo "The two .yml files have been updated. Please push them to the repository, if appropriate, or use 'git checkout' to undo the changes." + + + diff --git a/scripts/poderizeGithub.sh b/scripts/poderizeGithub.sh new file mode 100644 index 000000000..90ec17439 --- /dev/null +++ b/scripts/poderizeGithub.sh @@ -0,0 +1,449 @@ +#!/bin/bash + +# poderizeGithub.sh +# +# Usage: +# +# cd +# scripts/poderizeGithub.sh +# +# Arguments: +# +# All arguments are optional, so that if one is not specified, then it is assumed +# to be "auto". A version of "auto" will result in the default version, obtained +# through the versions.sh script, being used. +# +# All arguments are positional. For example, if you want to specify the tasker +# version, then you need to specify the core and worker shim versions as "auto", +# first. +# +# Description: +# +# The purpose of this script is to avoid manual errors when tagging an image from a github release. +# It is not a replacement for understanding podman images, containers, and tags. +# Below steps can be run manually as well, and should be run when errors are not +# visible. +# +# Script needs to be run from the root of wres directory so that other scripts +# that depend on the context of the wres directory will work (such as those that +# depend on git depending on the context of directory to work). +# +# This script should be idempotent, meaning you can run it several times in +# a row without damage. This is true for the build steps, tag steps, push steps. +# +# Depends on versionsGithub.sh script + + +#============================================================= +# Identify default versions! +#============================================================= + +# Attempt to auto-detect the versions needed. +all_versions=$( scripts/versionsGithub.sh ) + +overall_version=$( echo "$all_versions" | grep "^Main version" | cut -d' ' -f3 ) +tasker_version=$( echo "$all_versions" | grep "^wres-tasker version" | cut -d' ' -f3 ) +broker_version=$( echo "$all_versions" | grep "^wres-broker version" | cut -d' ' -f3 ) +redis_version=$( echo "$all_versions" | grep "^wres-redis version" | cut -d' ' -f3 ) +wres_worker_shim_version=$( echo "$all_versions" | grep "^wres-worker version" | cut -d' ' -f3 ) +eventsbroker_version=$( echo "$all_versions" | grep "^wres-eventsbroker version" | cut -d' ' -f3 ) +graphics_version=$( echo "$all_versions" | grep "^wres-vis version" | cut -d' ' -f3 ) +writing_version=$( echo "$all_versions" | grep "^wres-writing version" | cut -d' ' -f3 ) + +# These will be the zip ids, as distinct from the previously-found image ids. +wres_core_version=$overall_version +wres_tasker_version=$tasker_version +wres_vis_version=$graphics_version +wres_writing_version=$writing_version + + +# Sometimes auto-detection of versions does not work, because if no code changed +# then gradle will not create a new zip file. So the caller must specify each +# version with positional args, or "auto" to retain auto-detected version. + +if [[ "$1" != "" && "$1" != "auto" ]] +then + wres_core_version=$1 +fi + +if [[ "$2" != "" && "$2" != "auto" ]] +then + wres_worker_shim_version=$2 +fi + +if [[ "$3" != "" && "$3" != "auto" ]] +then + wres_tasker_version=$3 +fi + +if [[ "$4" != "" && "$4" != "auto" ]] +then + broker_version=$4 +fi + +if [[ "$5" != "" && "$5" != "auto" ]] +then + redis_version=$5 +fi + +if [[ "$6" != "" && "$6" != "auto" ]] +then + eventsbroker_version=$6 +fi + +if [[ "$7" != "" && "$7" != "auto" ]] +then + wres_vis_version=$7 +fi + +if [[ "$8" != "" && "$8" != "auto" ]] +then + wres_writing_version=$8 +fi + +echo "" +echo "VERSIONS USED BASED ON DEFAULTS WITH ARGUMENT OVERRIDES:" +echo "" +echo "Core WRES binary zip version is $wres_core_version" +echo "WRES Worker shim binary zip version is $wres_worker_shim_version" +echo "WRES Tasker binary zip version is $wres_tasker_version" +echo "Primary podman image version is $overall_version" +echo "Tasker podman image version is $tasker_version" +echo "Broker podman image version is $broker_version" +echo "Redis podman image version is $redis_version" +echo "WRES events broker podman image version is $eventsbroker_version" +echo "WRES vis binary zip version is $wres_vis_version" +echo "WRES graphics podman image version is $graphics_version" +echo "WRES writing podman image version is $wres_writing_version" + + +#============================================================= +# Identify zip files and Jenkins URLs; wait for zips +#============================================================= +echo "" +echo "Identifying .zip files required..." + +wres_core_file=wres-${wres_core_version}.zip +worker_shim_file=wres-worker-${wres_worker_shim_version}.zip +tasker_file=wres-tasker-${wres_tasker_version}.zip +vis_file=wres-vis-${wres_vis_version}.zip +writing_file=wres-writing-${wres_writing_version}.zip + +github_workspace=https://github.com/NOAA-OWP/wres/releases/download/staging + +core_url=$github_workspace/$wres_core_file +worker_url=$github_workspace/$worker_shim_file +tasker_url=$github_workspace/$tasker_file +vis_url=$github_workspace/$vis_file +writing_url=$github_workspace/$writing_file + + +# Ensure the distribution zip files are present for successful podman build +if [[ ! -f ./build/distributions/$wres_core_file || \ + ! -f ./wres-worker/build/distributions/$worker_shim_file || \ + ! -f ./wres-tasker/build/distributions/$tasker_file || \ + ! -f ./wres-writing/build/distributions/$writing_file || \ + ! -f ./wres-vis/build/distributions/$vis_file ]] +then + echo "" + echo "It appears you are not an automated build server (or something went wrong if you are)." + echo "" + echo "Please download these files and place them in the stated directory:" + echo "" + + if [[ ! -f ./build/distributions/$wres_core_file ]] + then + echo " $core_url - build/distributions" + fi + + if [[ ! -f ./wres-worker/build/distributions/$worker_shim_file ]] + then + echo " $worker_url - wres-worker/build/distributions" + fi + + if [[ ! -f ./wres-tasker/build/distributions/$tasker_file ]] + then + echo " $tasker_url - wres-tasker/build/distributions" + fi + + if [[ ! -f ./wres-vis/build/distributions/$vis_file ]] + then + echo " $vis_url - wres-vis/build/distributions" + fi + + if [[ ! -f ./wres-writing/build/distributions/$writing_file ]] + then + echo " $writing_url - wres-writing/build/distributions" + fi + echo "" + echo "You can use the following curl commands, with user name and token specified in ~/jenkins_token, to obtain the files:" + echo "" + + if [[ ! -f ./build/distributions/$wres_core_file ]] + then + echo " wget -P ./build/distributions/ $core_url" + fi + + if [[ ! -f ./wres-worker/build/distributions/$worker_shim_file ]] + then + echo " wget -P ./wres-worker/build/distributions/ $worker_url" + fi + + if [[ ! -f ./wres-tasker/build/distributions/$tasker_file ]] + then + echo " wget -P ./wres-tasker/build/distributions/ $tasker_url" + fi + + if [[ ! -f ./wres-vis/build/distributions/$vis_file ]] + then + echo " wget -P ./wres-vis/build/distributions/ $vis_url" + fi + + if [[ ! -f ./wres-writing/build/distributions/$writing_file ]] + then + echo " wget -P ./wres-writing/build/distributions/ $writing_url" + fi + echo "" + echo "You can also use the '-u user:token' option instead of '--config ~/jenkins_token', e.g. '-u :'." + echo "" + echo "The above URLs are only valid if your .zip files are the latest artifact. To pull down old artifacts, identify the Jenkins build number associated with the VLab GIT revision and modify the \"ws\" in the url to be \"/artifact\". For example," + echo "" + echo "$JOB_URL/3686/artifact/wres-vis/build/distributions/wres-vis-20210225-713c981.zip" + echo "" + echo "After they have completely finished downloading and have been completely copied into the local directories, re-run this script." + exit 3 +fi + + +#============================================================= +# Build the images +#============================================================= +echo "" +echo "Building images..." + +# Build and tag the worker image which is composed of WRES core and worker shim. +# Tag will be based on the later image version which is WRES core at git root. +echo "Building and tagging worker image..." +worker_image_id=$( podman build --format docker --build-arg version=$wres_core_version --build-arg worker_version=$wres_worker_shim_version --quiet --tag wres/wres-worker:$overall_version . ) +echo "Built wres/wres-worker:$overall_version -- $worker_image_id" + +# Build and tag the tasker image which solely contains the tasker. +echo "Building tasker image..." +pushd wres-tasker +tasker_image_id=$( podman build --format docker --build-arg version=$wres_tasker_version --quiet --tag wres/wres-tasker:$tasker_version . ) +popd + +echo "Built wres/wres-tasker:$tasker_version -- $tasker_image_id" + +# Build and tag the broker image +echo "Building broker image..." +pushd wres-broker +broker_image_id=$( podman build --format docker --pull --build-arg version=$broker_version --quiet --tag wres/wres-broker:$broker_version . ) +popd + +echo "Built wres/wres-broker:$broker_version -- $broker_image_id" + +# Build and tag the redis image +echo "Building redis image..." +pushd wres-redis +redis_image_id=$( podman build --format docker --pull --build-arg version=$redis_version --quiet --tag wres/wres-redis:$redis_version . ) +popd + +echo "Built wres/wres-redis:$redis_version -- $redis_image_id" + +# Build and tag the eventsbroker image +echo "Building events broker image..." +pushd wres-eventsbroker +eventsbroker_image_id=$( podman build --format docker --build-arg version=$eventsbroker_version --quiet --tag wres/wres-eventsbroker:$eventsbroker_version . ) +popd + +echo "Built wres/wres-eventsbroker:$eventsbroker_version -- $eventsbroker_image_id" + +# Build and tag the graphics image +echo "Building graphics image..." +pushd wres-vis +graphics_image_id=$( podman build --format docker --build-arg version=$wres_vis_version --quiet --tag wres/wres-graphics:$graphics_version . ) +popd + +echo "Built wres/wres-graphics:$graphics_version -- $graphics_image_id" + +# Build and tag the writing image +echo "Building writing image..." +pushd wres-writing +writing_image_id=$( podman build --format docker --build-arg version=$wres_writing_version --quiet --tag wres/wres-writing:$writing_version . ) +popd + +echo "Built wres/wres-writing:$writing_version -- $writing_image_id" + +# Build and tag the nginx image +echo "Building nginx image..." +pushd nginx +nginx_image_id=$( podman build --format docker --build-arg --quiet --tag wres/nginx . ) +popd + +echo "Built wres/nginx" + +echo "Displaying most recent 20 docker images" +podman image ls | head -n 21 + + +#============================================================= +# Docker Registry +#============================================================= + +# Optional: set environment variable DOCKER_REGISTRY to the FQDN of a docker +# registry (without any path, full fqdn, without scheme) + +if [[ ! -z "$DOCKER_REGISTRY" ]] +then + # Check the format of the registry env var. If something is wrong, then don't use it. + if [[ $DOCKER_REGISTRY =~ ^https?:// ]] +# if [[ $DOCKER_REGISTRY == http* ]] + then + echo "" + echo "You provided a DOCKER_REGISTRY, but it starts with http. Don't include the scheme!" + echo "Skipping pushing the images to the registry!" + # It looks good, try to push to the registry. + else + echo "" + echo "Attempting tagging and pushing images to the registry, https://$DOCKER_REGISTRY ..." + echo "Running docker login https://$DOCKER_REGISTRY..." + podman login https://$DOCKER_REGISTRY + login_success=$? + + if [[ ! login_success ]] + then + echo "Failed to login, not going to try to push to registry. Try again." + exit 2 + fi + + primary_image_dev_status=$( echo ${overall_version} | grep "dev" ) + + if [[ "$primary_image_dev_status" != "" ]] + then + echo "Refusing to tag and push primary podman image version ${overall_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-worker:$overall_version as $DOCKER_REGISTRY/wres/wres-worker/$overall_version..." + podman tag wres/wres-worker:$overall_version $DOCKER_REGISTRY/wres/wres-worker:$overall_version + podman push $DOCKER_REGISTRY/wres/wres-worker:$overall_version + fi + + tasker_image_dev_status=$( echo ${tasker_version} | grep "dev" ) + + if [[ "$tasker_image_dev_status" != "" ]] + then + echo "Refusing to tag and push tasker podman image version ${tasker_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-tasker:$tasker_version as $DOCKER_REGISTRY/wres/wres-tasker/$tasker_version..." + podman tag wres/wres-tasker:$tasker_version $DOCKER_REGISTRY/wres/wres-tasker:$tasker_version + podman push $DOCKER_REGISTRY/wres/wres-tasker:$tasker_version + fi + + broker_image_dev_status=$( echo ${broker_version} | grep "dev" ) + + if [[ "$broker_image_dev_status" != "" ]] + then + echo "Refusing to tag and push broker podman image version ${broker_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-broker:$broker_version as $DOCKER_REGISTRY/wres/wres-broker/$broker_version..." + podman tag wres/wres-broker:$broker_version $DOCKER_REGISTRY/wres/wres-broker:$broker_version + podman push $DOCKER_REGISTRY/wres/wres-broker:$broker_version + fi + + redis_image_dev_status=$( echo ${redis_version} | grep "dev" ) + + if [[ "$redis_image_dev_status" != "" ]] + then + echo "Refusing to tag and push redis podman image version ${redis_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-redis:$redis_version as $DOCKER_REGISTRY/wres/wres-redis/$redis_version..." + podman tag wres/wres-redis:$redis_version $DOCKER_REGISTRY/wres/wres-redis:$redis_version + podman push $DOCKER_REGISTRY/wres/wres-redis:$redis_version + fi + + eventsbroker_image_dev_status=$( echo ${eventsbroker_version} | grep "dev" ) + + if [[ "$eventsbroker_image_dev_status" != "" ]] + then + echo "Refusing to tag and push eventsbroker podman image version ${eventsbroker_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-eventsbroker:$eventsbroker_version as $DOCKER_REGISTRY/wres/wres-eventsbroker/$eventsbroker_version..." + podman tag wres/wres-eventsbroker:$eventsbroker_version $DOCKER_REGISTRY/wres/wres-eventsbroker:$eventsbroker_version + podman push $DOCKER_REGISTRY/wres/wres-eventsbroker:$eventsbroker_version + fi + + graphics_image_dev_status=$( echo ${graphics_version} | grep "dev" ) + + if [[ "$graphics_image_dev_status" != "" ]] + then + echo "Refusing to tag and push graphics podman image version ${graphics_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-graphics:$graphics_version as $DOCKER_REGISTRY/wres/wres-graphics/$graphics_version..." + podman tag wres/wres-graphics:$graphics_version $DOCKER_REGISTRY/wres/wres-graphics:$graphics_version + podman push $DOCKER_REGISTRY/wres/wres-graphics:$graphics_version + fi + + writing_image_dev_status=$( echo ${writing_version} | grep "dev" ) + + if [[ "$writing_image_dev_status" != "" ]] + then + echo "Refusing to tag and push writing podman image version ${writing_version} because its Dockerfile has not been committed to the repository yet." + else + echo "Tagging and pushing wres/wres-writing:$writing_version as $DOCKER_REGISTRY/wres/wres-writing/$writing_version..." + podman tag wres/wres-writing:$writing_version $DOCKER_REGISTRY/wres/wres-writing:$writing_version + podman push $DOCKER_REGISTRY/wres/wres-writing:$writing_version + fi + + echo "Tagging and pushing wres/nginx as wres/nginx..." + podman tag wres/nginx $DOCKER_REGISTRY/wres/nginx + podman push $DOCKER_REGISTRY/wres/nginx + fi + +else + echo "" + echo "No variable 'DOCKER_REGISTRY' found, not attempting to docker push." + echo "If you want to automatically push, set DOCKER_REGISTRY to the FQDN of" + echo "an accessible docker registry and this script will attempt to tag and" + echo "push to that registry." + echo "" +fi + +#============================================================= +# Create .yml files +#============================================================= + +echo "" +echo "About to update the .yml files with the new versions based on a template." +echo "If you are only updating some of the images/versions, it is recommended" +echo "you skip this step and do that by manually editing the .ymls." +echo "" +echo "Do you wish to continue with this last step?" +select yn in "Yes" "No"; do + case $yn in + Yes ) echo "Proceeding..."; break;; + No ) exit;; + esac +done + +cp compose-entry.template.yml compose-entry.yml +sed -i "s/TASKER_IMAGE/${tasker_version}/" compose-entry.yml +sed -i "s/BROKER_IMAGE/${broker_version}/" compose-entry.yml +sed -i "s/REDIS_IMAGE/${redis_version}/" compose-entry.yml +sed -i "s/WORKER_IMAGE/${overall_version}/" compose-entry.yml # By design... The tag for the worker image is the "overall_version". +sed -i "s/EVENTS_IMAGE/${eventsbroker_version}/" compose-entry.yml +sed -i "s/GRAPHICS_IMAGE/${graphics_version}/" compose-entry.yml +sed -i "s/WRITING_IMAGE/${writing_version}/" compose-entry.yml +sed -i "s/OVERALL_IMAGE/${overall_version}/" compose-entry.yml + +cp compose-workers.template.yml compose-workers.yml +sed -i "s/TASKER_IMAGE/${tasker_version}/" compose-workers.yml +sed -i "s/BROKER_IMAGE/${broker_version}/" compose-workers.yml +sed -i "s/REDIS_IMAGE/${redis_version}/" compose-workers.yml +sed -i "s/WORKER_IMAGE/${overall_version}/" compose-workers.yml # By design... The tag for the worker image is the "overall_version". +sed -i "s/EVENTS_IMAGE/${eventsbroker_version}/" compose-workers.yml +sed -i "s/GRAPHICS_IMAGE/${graphics_version}/" compose-workers.yml +sed -i "s/WRITING_IMAGE/${writing_version}/" compose-workers.yml +sed -i "s/OVERALL_IMAGE/${overall_version}/" compose-workers.yml + +echo "" +echo "The two .yml files have been updated. Please push them to the repository, if appropriate, or use 'git checkout' to undo the changes." diff --git a/wres-broker/Dockerfile b/wres-broker/Dockerfile index 9931c6866..0233bab86 100644 --- a/wres-broker/Dockerfile +++ b/wres-broker/Dockerfile @@ -8,7 +8,7 @@ RUN echo "wres_docker:x:498:1370800073::/home/wres_docker:" >> /etc/passwd \ && echo "user:!:1:0:99999:7:::" >> /etc/shadow \ && echo "wres:x:1370800073:" >> /etc/group \ && mkdir /home/wres_docker \ - && chown wres_docker: /home/wres_docker + && chown wres_docker /home/wres_docker # Enable authentication with x509 client certificates (requires some conf too): RUN rabbitmq-plugins enable --offline rabbitmq_auth_mechanism_ssl diff --git a/wres-eventsbroker/Dockerfile b/wres-eventsbroker/Dockerfile index 2b870ffa7..fba3332fb 100644 --- a/wres-eventsbroker/Dockerfile +++ b/wres-eventsbroker/Dockerfile @@ -62,7 +62,7 @@ WORKDIR ${BROKER_WORK} # Create directories with the correct permissions RUN mkdir -p {${BROKER_HOME},${BROKER_CONFIG},${BROKER_INSTANCE}} && \ - chown ${BROKER_USER}: ${BROKER_INSTANCE} + chown ${BROKER_USER} ${BROKER_INSTANCE} COPY --from=builder /workspace/out ${BROKER_HOME} diff --git a/wres-eventsbroker/docker-entrypoint.sh b/wres-eventsbroker/docker-entrypoint.sh index 6ffe5cffc..9a0f5f59a 100644 --- a/wres-eventsbroker/docker-entrypoint.sh +++ b/wres-eventsbroker/docker-entrypoint.sh @@ -8,8 +8,6 @@ BROKER_CONFIG_PATH=$BROKER_INSTANCE/etc/ # Properties passed to the broker and then accessible as system properties in the xml configuration files ARTEMIS_CLUSTER_PROPS="-Dactivemq.remoting.amqp.port=${BROKER_AMQP_PORT} -Dactivemq.remoting.http.port=${BROKER_HTTP_PORT} -Dbroker.keystore.path=${BROKER_KEYSTORE_PATH} -Dbroker.keystore.password=${BROKER_KEYSTORE_PASSWORD} -Dbroker.truststore.path=${BROKER_TRUSTSTORE_PATH} -Dbroker.truststore.password=${BROKER_TRUSTSTORE_PASSWORD} -Dhawtio.disableProxy=true -Dhawtio.realm=activemq-cert -Dhawtio.role=wres-eventsbroker-admin -Dhawtio.offline=true -Dhawtio.sessionTimeout=86400 -Dhawtio.rolePrincipalClasses=org.apache.activemq.artemis.spi.core.security.jaas.RolePrincipal" -echo "HANK HERR ####>> $ARTEMIS_CLUSTER_PROPS" - # Set some JVM arguments if not already set if [[ -z $JAVA_ARGS ]]; then JAVA_ARGS="-XX:+PrintClassHistogram -XX:+UseG1GC -XX:+UseStringDeduplication -Xms2048m -Xmx2048m" diff --git a/wres-redis/Dockerfile b/wres-redis/Dockerfile index 0de732a38..8896a63dc 100644 --- a/wres-redis/Dockerfile +++ b/wres-redis/Dockerfile @@ -6,7 +6,7 @@ RUN echo "wres_persister_docker:x:498:1370800073::/home/wres_persister_docker:" && echo "user:!:1:0:99999:7:::" >> /etc/shadow \ && echo "wres:x:1370800073:" >> /etc/group \ && mkdir -p /home/wres_persister_docker \ - && chown wres_persister_docker: /home/wres_persister_docker + && chown wres_persister_docker /home/wres_persister_docker COPY redis.conf /etc/redis/redis.conf COPY docker-healthcheck /usr/local/bin/