Merge pull request #562 from NethServer/feat-core-update

Update core procedure enhancements

core/imageroot/usr/local/agent/actions/update-module/50run_scriptdir

@@ -48,7 +48,7 @@ for script in "${AGENT_INSTALL_DIR}"/update-module.d/* ; do
     echo "Running ${script}..."
 
     if ! command ${script} ; then
-        echo "${SD_WARNING}${script} has exit code $?"
+        echo "${SD_WARNING}${script} has failed"
     fi
 
 done

core/imageroot/var/lib/nethserver/cluster/actions/update-core/50update_core

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import json
+import agent
+import agent.tasks
+import sys, os
+import cluster.modules
+
+request = json.load(sys.stdin)
+core_url = request.get('core_url', '')
+nodes_list = request['nodes']
+force_update = request.get("force", False)
+
+rdb = agent.redis_connect(privileged=True)
+# if the image url is not given, search for the latest core image inside the repo
+if not core_url:
+    core_url = cluster.modules.get_latest_module('core', rdb)
+
+# Start the core update on each node
+update_core_tasks = []
+for node_id in nodes_list:
+    update_core_tasks.append({
+        "agent_id": f'node/{node_id}',
+        "action": "update-core",
+        "data": {
+            "core_url": core_url,
+            "force": force_update,
+        }
+    })
+
+update_core_errors = agent.tasks.runp_brief(update_core_tasks,
+    endpoint = "redis://cluster-leader",
+    progress_callback=agent.get_progress_callback(31, 95),
+)
+
+agent.assert_exp(update_core_errors == 0, 'update-core failed in some nodes')

core/imageroot/var/lib/nethserver/cluster/actions/update-core/50modules → core/imageroot/var/lib/nethserver/cluster/actions/update-core/70update_modules

@@ -1,23 +1,8 @@
 #!/usr/bin/env python3
 
 #
-# Copyright (C) 2021 Nethesis S.r.l.
-# http://www.nethesis.it - [email protected]
-#
-# This script is part of NethServer.
-#
-# NethServer is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License,
-# or any later version.
-#
-# NethServer is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with NethServer.  If not, see COPYING.
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
 #
 
 import json
@@ -26,6 +11,12 @@ import agent.tasks
 import sys, os
 import cluster.modules
 
+request = json.load(sys.stdin)
+force_update = request.get("force", False)
+agent_install_dir = os.environ["AGENT_INSTALL_DIR"]
+
+agent.run_helper("run-scriptdir", f"{agent_install_dir}/update-core-pre-modules.d/")
+
 rdb = agent.redis_connect(privileged=True)
 
 # Update all core modules
@@ -51,16 +42,16 @@ for mid in instances.keys():
         "action": "update-module",
         "data": {
             "module_url": instances[mid]['url'],
-            "instances":  instances[mid]['instances']
+            "instances":  instances[mid]['instances'],
+            "force": force_update,
         }
     })
 
-
 update_module_errors = agent.tasks.runp_brief(update_module_tasks,
     endpoint = "redis://cluster-leader",
     progress_callback=agent.get_progress_callback(75, 95),
 )
 
-agent.assert_exp(update_module_errors == 0)
+agent.run_helper("run-scriptdir", f"{agent_install_dir}/update-core-post-modules.d/")
 
-json.dump({}, fp=sys.stdout)
+agent.assert_exp(update_module_errors == 0, 'update-core failed in some core modules')

core/imageroot/var/lib/nethserver/cluster/actions/update-core/validate-input.json

@@ -33,6 +33,11 @@
                 "type": "integer"
             },
             "minItems": 1
+        },
+        "force": {
+            "description": "Force the core update even if the given `core_url` is already present in the local Podman storage",
+            "type": "boolean",
+            "default": false
         }
     }
 }

core/imageroot/var/lib/nethserver/cluster/bin/run-scriptdir

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+#
+# Simple helper that runs any executable command under the given
+# script_dir, in alphabetical order. If a script has non-zero exit code
+# print a warning and continue with the next one. Exit with non-zero exit
+# code if one or more scripts has failed.
+#
+
+script_dir=${1:?}
+
+if [[ ! -d "${script_dir}" ]]; then
+    exit 0
+fi
+
+# systemd-journal log severity codes:
+SD_DEBUG='<7>'
+SD_WARNING='<4>'
+
+shopt -s nullglob
+errors=0
+for script in "${script_dir}"/* ; do
+
+    if [[ ! -x "${script}" ]]; then
+        echo "${SD_DEBUG}Skipping non-executable file ${script}..."
+        continue
+    fi
+
+    echo "Running ${script}..."
+
+    if ! command ${script} ; then
+        echo "${SD_WARNING}${script} has failed"
+        ((errors++))
+    fi
+
+done
+
+if ((errors > 0)); then
+    exit 1
+fi

core/imageroot/var/lib/nethserver/cluster/update-core-pre-modules.d/50update_grants

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+#
+# Copyright (C) 2024 Nethesis S.r.l.
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+
+import os
+import sys
+import agent
+import cluster.grants
+
+rdb = agent.redis_connect(privileged=True)
+
+#
+# Update existing grants for features #6814, #6825
+#
+cluster.grants.grant(rdb, action_clause="bind-user-domains",  to_clause="accountconsumer", on_clause='cluster')
+cluster.grants.grant(rdb, action_clause="list-modules",  to_clause="accountprovider", on_clause='cluster')
+
+#
+# END of grant updates
+#
+cluster.grants.refresh_permissions(rdb)

core/imageroot/var/lib/nethserver/node/actions/update-core/50update

@@ -14,6 +14,7 @@ import semver
 
 request = json.load(sys.stdin)
 core_url = request['core_url']
+force_update = request.get("force", False)
 
 agent.set_weight(os.path.basename(__file__), 2)
 
@@ -26,11 +27,16 @@ except ValueError:
     aver = semver.parse_version_info('0.0.0')
     bver = semver.parse_version_info('0.1.0')
 
-if aver >= bver:
+if force_update == False and aver >= bver:
     print('The new core image does not update the current one. Nothing to do.', file=sys.stderr)
     sys.exit(0)
 
-agent.run_helper('podman-pull-missing', core_url,
+if force_update:
+    podman_pull_command = ['podman', 'pull']
+else:
+    podman_pull_command = ['podman-pull-missing']
+
+agent.run_helper(*podman_pull_command, core_url,
     progress_callback=agent.get_progress_callback(0,40)
 ).check_returncode()
 
@@ -57,7 +63,7 @@ print('Core image Digest     : ', inspect_image_digest, file=sys.stderr)
 print('Core image RepoDigest : ', inspect_image_repodigest, file=sys.stderr)
 print('Core image extras     : ', extra_images, file=sys.stderr)
 
-agent.run_helper('podman-pull-missing', *extra_images,
+agent.run_helper(*podman_pull_command, *extra_images,
     progress_callback=agent.get_progress_callback(41,70)
 ).check_returncode()
 

core/imageroot/var/lib/nethserver/node/actions/update-core/60run_scriptdir

@@ -27,7 +27,7 @@ for script in "${AGENT_INSTALL_DIR}"/update-core.d/* ; do
     echo "Running ${script}..."
 
     if ! command ${script} ; then
-        echo "${SD_WARNING}${script} has exit code $?"
+        echo "${SD_WARNING}${script} has failed"
     fi
 
 done

core/imageroot/var/lib/nethserver/node/actions/update-core/validate-input.json

@@ -21,6 +21,11 @@
                 "ghcr.io/nethserver/core:1.2.0"
             ],
             "minLength": 1
+        },
+        "force": {
+            "description": "Force the core update even if the given `core_url` is already present in the local Podman storage",
+            "type": "boolean",
+            "default": false
         }
     }
 }

core/imageroot/var/lib/nethserver/node/update-core.d/50update_grants

@@ -31,6 +31,6 @@ cluster.grants.grant(rdb, action_clause="list-modules",  to_clause="accountprovi
 #
 # END of grant updates -- Do not add more grants: this script will be
 # removed in future releases An equivalent script will be implemented
-# under cluster/pre-update-modules.d/
+# under cluster/update-core-pre-modules.d/
 #
 cluster.grants.refresh_permissions(rdb)