WМАgent switch deployment model copy runtime fix 11990 (#1490)

* Parsing the WMA_TAG as early as possible && Forcing runtime scripts checks && Creating mount points based on WMA_VER_RELEASE instead of WMA_TAG

* Calling check_wmatag

* Fix wrong initFlag cleared && _exec_sql call && typos

docker/pypi/wmagent/Dockerfile

@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile-upstream:master
 FROM registry.cern.ch/cmsweb/wmagent-base:pypi-20230705
 MAINTAINER Valentin Kuznetsov [email protected]
 
@@ -7,14 +8,23 @@ ARG WMA_TAG=$TAG
 ENV WMA_TAG=$WMA_TAG
 ENV WMA_ROOT_DIR=/data
 
+# Parsing the WMA_TAG in parts step by step
+ENV WMA_VER_MINOR=${WMA_TAG#*.*.}
+ENV WMA_VER_MAJOR=${WMA_TAG%.$WMA_VER_MINOR}
+ENV WMA_VER_MINOR=${WMA_VER_MINOR%rc*}
+ENV WMA_VER_MINOR=${WMA_VER_MINOR%.*}
+ENV WMA_VER_RELEASE=${WMA_VER_MAJOR}.${WMA_VER_MINOR}
+ENV WMA_VER_PATCH=${WMA_TAG#$WMA_VER_RELEASE}
+ENV WMA_VER_PATCH=${WMA_VER_PATCH#.}
+
 # Basic WMAgent directory structure passed to all scripts through env variables:
 # NOTE: Those should be static and depend only on $WMA_BASE_DIR
 ENV WMA_BASE_DIR=$WMA_ROOT_DIR/srv/wmagent
 ENV WMA_ADMIN_DIR=$WMA_ROOT_DIR/admin/wmagent
 ENV WMA_CERTS_DIR=$WMA_ROOT_DIR/certs
 
 # ENV WMA_HOSTADMIN_DIR=$WMA_ADMIN_DIR/hostadmin
-ENV WMA_CURRENT_DIR=$WMA_BASE_DIR/$WMA_TAG
+ENV WMA_CURRENT_DIR=$WMA_BASE_DIR/$WMA_VER_RELEASE
 ENV WMA_AUTH_DIR=$WMA_CURRENT_DIR/auth/
 ENV WMA_INSTALL_DIR=$WMA_CURRENT_DIR/install
 ENV WMA_STATE_DIR=$WMA_CURRENT_DIR/state

docker/pypi/wmagent/bin/manage-common.sh

@@ -118,6 +118,15 @@ _exec_oracle() {
     fi
 }
 
+_exec_sql() {
+    case $AGENT_FLAVOR in
+        'mysql')
+           _exec_mysql "$@" $wmaDBName ;;
+        'oracle')
+           _exec_oracle "$@" ;;
+    esac
+}
+
 _init_valid(){
     # Auxiliary function to shorten repetitive compares of .init* files to the current WMA_BUILD_ID
     # :param $1: The full path to the .init* file to be checked.

docker/pypi/wmagent/init.sh

@@ -50,6 +50,7 @@ echo "======================================================="
 echo "Starting WMAgent with the following initialisation data:"
 echo "-------------------------------------------------------"
 echo " - WMAgent Version            : $WMA_TAG"
+echo " - WMAgent Release Cycle      : $WMA_VER_RELEASE"
 echo " - WMAgent User               : $WMA_USER"
 echo " - WMAgent Root path          : $WMA_ROOT_DIR"
 echo " - WMAgent Host               : $HOSTNAME"
@@ -118,6 +119,15 @@ _copy_runtime() {
         echo "$FUNCNAME: Copying content from: $WMA_DEPLOY_DIR/etc/ to: $WMA_CONFIG_DIR/"
         cp -ra $WMA_DEPLOY_DIR/etc/ $WMA_CONFIG_DIR/
         echo $WMA_BUILD_ID > $wmaInitRuntime
+
+        # NOTE: If the sql database have been previously initialized, then the
+        #       current call of _copy_runtime has been due to a change of the
+        #       $WMA_TAG - either a patch version or a release candidate change
+        #       and we need to update the record at the database as well
+        _init_valid $wmaInitSqlDB && {
+            echo "$FUNCNAME: Preserving the current WMA_TAG at the wma_init database"
+            _exec_sql "update wma_init set init_value='$WMA_TAG' where init_param='wma_tag';"
+        }
     else
         echo "$FUNCNAME: ERROR: \$WMA_DEPLOY_DIR: $WMA_DEPLOY_DIR is not a root path for \$pythonLib: $pythonLib"
         echo "$FUNCNAME: ERROR: We cannot find the correct WMCore/WMRuntime source to copy at the current host!"
@@ -331,6 +341,29 @@ EOF
     echo "-----------------------------------------------------------------------"
 }
 
+check_wmatag() {
+    # A function to check if there have been any change in the WMA_TAG since last
+    # WMAgent initialization at the host. It does the check by comparing the whole
+    # WMA_TAG from the environment and the one previously recorded at relational
+    # database in the table `wma_init`.
+    #
+    # NOTE: Before we even start checking all the rest of the initialization flags, we should
+    #       check if this agent has ever been used, and if there is any patch version
+    #       or release candidate change between the current one and the one previously
+    #       initialized at the host. If such discrepancy is confirmed, we should enforce Runtime
+    #       code copy by deleting only the $wmaInitSqlDB and let the rest of the init process take over.
+    _init_valid $wmaInitSqlDB && {
+        echo "$FUNCNAME: This agent has been previously initialized. Checking for WMAgent version change since last run."
+        local wmaTagCurr=$WMA_TAG
+        local wmaTagSqlDB=$(_exec_sql "select init_value from wma_init where init_param = 'wma_tag';")
+        [[ $wmaTagCurr == $wmaTagSqlDB ]] || {
+            echo "$FUNCNAME: Found version change since last run: $wmaTagCurr vs. $wmaTagSqlDB"
+            echo "$FUNCNAME: Enforcing Runtime code check and copy if needed."
+            rm $wmaInitRuntime
+        }
+    }
+}
+
 check_wmagent_init() {
     # A function to check all previously populated */.init<step> files
     # from all previous steps and compare them with the /data/.wmaBuildId
@@ -602,6 +635,7 @@ start_agent() {
 main(){
     basic_checks
     check_wmasecrets
+    check_wmatag
     check_wmagent_init || {
         (deploy_to_host)         || { err=$?; echo "ERROR: deploy_to_host"; exit $err ;}
         (check_databases)        || { err=$?; echo "ERROR: check_databases"; exit $err ;}

docker/pypi/wmagent/install.sh

@@ -48,19 +48,12 @@ done
 WMA_TAG_REG="^[0-9]+\.[0-9]+\.[0-9]{1,2}((\.|rc)[0-9]{1,2})?$"
 [[ $WMA_TAG =~ $WMA_TAG_REG ]] || { echo "WMA_TAG: $WMA_TAG does not match requered expression: $WMA_TAG_REG"; echo "EXIT with Error 1"  ; exit 1 ;}
 
-# Parsing the WMA_TAG
-declare -A WMA_VER
-WMA_VER[release]=$(echo $WMA_TAG| sed -nr 's/.*(^[0-9]+\.[0-9]+\.[0-9]+).*$/\1/p')
-WMA_VER[patch]=$(echo $WMA_TAG| sed -nr 's/.*(^[0-9]+\.[0-9]+\.[0-9]+)//p')
-WMA_VER[major]=$(echo $WMA_TAG| sed -nr 's/.*(^[0-9]+\.[0-9]+).*/\1/p')
-WMA_VER[minor]=$(echo $WMA_TAG| sed -nr 's/^[0-9]+\.[0-9]+\.([0-9]+).*$/\1/p')
-
 echo
 echo "======================================================================="
 echo "Starting new WMAgent deployment with the following initialisation data:"
 echo "-----------------------------------------------------------------------"
 echo " - WMAgent Version            : $WMA_TAG"
-echo " - WMAgent Release Cycle      : ${WMA_VER[release]}"
+echo " - WMAgent Release Cycle      : $WMA_VER_RELEASE"
 echo " - WMAgent User               : $WMA_USER"
 echo " - WMAgent Root path          : $WMA_ROOT_DIR"
 echo " - Python  Version            : $(python --version)"
@@ -114,9 +107,10 @@ stepMsg="Generating and preserving current build id"
 echo "-----------------------------------------------------------------------"
 echo "Start $stepMsg"
 
-echo ${WMA_VER[release]} | sha256sum | awk '{print $1}' > $WMA_ROOT_DIR/.wmaBuildId
+echo $WMA_VER_RELEASE | sha256sum | awk '{print $1}' > $WMA_ROOT_DIR/.wmaBuildId
 echo "WMA_BUILD_ID:`cat $WMA_ROOT_DIR/.wmaBuildId`"
 echo "WMA_BUILD_ID preserved at: $WMA_ROOT_DIR/.wmaBuildId "
+
 echo "Done $stepMsg!" && echo
 echo "-----------------------------------------------------------------------"
 

docker/pypi/wmagent/wmagent-docker-run.sh

@@ -43,6 +43,15 @@ while getopts ":t:hp" opt; do
     esac
 done
 
+# Parsing the WMA_TAG in parts step by step
+WMA_VER_MINOR=${WMA_TAG#*.*.}
+WMA_VER_MAJOR=${WMA_TAG%.$WMA_VER_MINOR}
+WMA_VER_MINOR=${WMA_VER_MINOR%rc*}
+WMA_VER_MINOR=${WMA_VER_MINOR%.*}
+WMA_VER_RELEASE=${WMA_VER_MAJOR}.${WMA_VER_MINOR}
+WMA_VER_PATCH=${WMA_TAG#$WMA_VER_RELEASE}
+WMA_VER_PATCH=${WMA_VER_PATCH#.}
+
 wmaUser=$(id -un)
 wmaGroup=$(id -gn)
 
@@ -73,9 +82,9 @@ fi
 
 # create regular mount points at runtime
 [[ -d $HOST_MOUNT_DIR/admin/wmagent ]] || (mkdir -p $HOST_MOUNT_DIR/admin/wmagent) || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install) || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config)  || exit $?
-[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs ;} || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/install ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/install) || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/config  ]] || (mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/config)  || exit $?
+[[ -d $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/logs ]] || { mkdir -p $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/logs ;} || exit $?
 
 # NOTE: Before mounting /etc/tnsnames.ora we should check it exists, otherwise the run will fail on the FNAL agents
 tnsMount=""
@@ -92,9 +101,9 @@ $tnsMount \
 --mount type=bind,source=/etc/condor,target=/etc/condor,readonly \
 --mount type=bind,source=/tmp,target=/tmp \
 --mount type=bind,source=/data/certs,target=/data/certs \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/install,target=/data/srv/wmagent/current/install \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/config,target=/data/srv/wmagent/current/config \
---mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG/logs,target=/data/srv/wmagent/current/logs \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/install,target=/data/srv/wmagent/current/install \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/config,target=/data/srv/wmagent/current/config \
+--mount type=bind,source=$HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE/logs,target=/data/srv/wmagent/current/logs \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/wmagent,target=/data/admin/wmagent \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/passwd,target=/etc/passwd,readonly \
 --mount type=bind,source=$HOST_MOUNT_DIR/admin/etc/group,target=/etc/group,readonly \
@@ -103,17 +112,17 @@ $tnsMount \
 "
 
 $PULL && {
-    echo "Pulling Docker image: registry.cern.ch/cmsweb/wmagent:$WMA_TAG"
+    echo "Pulling Docker image: registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE"
     docker login registry.cern.ch
-    docker pull registry.cern.ch/cmsweb/wmagent:$WMA_TAG
-    docker tag registry.cern.ch/cmsweb/wmagent:$WMA_TAG local/wmagent:$WMA_TAG
-    docker tag registry.cern.ch/cmsweb/wmagent:$WMA_TAG local/wmagent:latest
+    docker pull registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE
+    docker tag registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE local/wmagent:$WMA_VER_RELEASE
+    docker tag registry.cern.ch/cmsweb/wmagent:$WMA_VER_RELEASE local/wmagent:latest
 }
 
-echo "Checking if there is no other wmagent container running and creating a link to the $WMA_TAG in the host mount area."
+echo "Checking if there is no other wmagent container running and creating a link to the $WMA_VER_RELEASE in the host mount area."
 [[ $(docker container inspect -f '{{.State.Status}}' wmagent 2>/dev/null) == 'running' ]] || (
     [[ -h $HOST_MOUNT_DIR/srv/wmagent/current ]] && rm -f $HOST_MOUNT_DIR/srv/wmagent/current
-    ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_TAG $HOST_MOUNT_DIR/srv/wmagent/current )
+    ln -s $HOST_MOUNT_DIR/srv/wmagent/$WMA_VER_RELEASE $HOST_MOUNT_DIR/srv/wmagent/current )
 
 echo "Starting wmagent:$WMA_TAG docker container with user: $wmaUser:$wmaGroup"
 docker run $dockerOpts local/wmagent:$WMA_TAG