diff --git a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py index f0088578b..018e6c70a 100644 --- a/core/imageroot/usr/local/agent/pypkg/cluster/modules.py +++ b/core/imageroot/usr/local/agent/pypkg/cluster/modules.py @@ -32,6 +32,14 @@ import subprocess import datetime +_repo_testing_cache = {} +def _repo_has_testing_flag(rdb, repo_name): + """Retrieve from Redis the testing flag of repo_name one time, then read it from a cache.""" + global _repo_testing_cache + if repo_name not in _repo_testing_cache: + _repo_testing_cache[repo_name] = rdb.hget(f'cluster/repository/{repo_name}', 'testing') == "1" + return _repo_testing_cache[repo_name] + def _urljoin(base_path, *args): '''replace urllib.parse.joinurl because it doesn't handle multiple parameters ''' @@ -51,46 +59,43 @@ def _get_downloaded_logos(): logos[os.path.basename(app)] = logo return logos -def _parse_repository_metadata(repository_name, repository_url, repository_updated, repodata, skip_core_modules = False, skip_testing_versions = False): - modules = [] - +def _calc_certification_level(package, has_subscription=False): + if package['repository_authority'] in ["distfeed.nethserver.org", "subscription.nethserver.com"]: + certification_level = 3 + elif package['repository_authority'] == "forge.nethserver.org": + certification_level = 2 + else: + certification_level = 1 + # If we trust the repo metadata, elevate up to level 5 + if certification_level == 3: + if package["source"].startswith("ghcr.io/nethserver/") or package["source"].startswith("ghcr.io/nethesis/"): + certification_level = 5 if has_subscription else 4 + return certification_level + +def _parse_repository_metadata(repository_name, repository_url, repository_updated, metadata): try: - repodata = json.loads(repodata) - except: - return modules - - def ignore_testing(version): - if skip_testing_versions and version["testing"] is True: - return False - else: - return True - + repodata = json.loads(metadata) + except Exception as ex: + print(agent.SD_WARNING + f"Unable to parse metadata of repository {repository_name} at {repository_url}:", ex, file=sys.stderr) + return [] + modules = [] for package in repodata: - # Skip core modules if flag is enabled - if skip_core_modules and package['versions']: - version = package['versions'][0] - if 'org.nethserver.flags' in version['labels'] and 'core_module' in version['labels']['org.nethserver.flags']: - continue - package["repository"] = repository_name package["repository_updated"] = repository_updated - + try: + package['repository_authority'] = urllib.parse.urlparse(repository_url).hostname + except Exception as ex: + package['repository_authority'] = "" + print(agent.SD_WARNING + f"Unable to parse repository {repository_name} URL: {repository_url}", ex, file=sys.stderr) # Set absolute path for logo if package["logo"]: package["logo"] = _urljoin(repository_url, package["id"], package["logo"]) - # Set absolute path for screenshots screenshots = [] for s in package["screenshots"]: screenshots.append(_urljoin(repository_url, package["id"], s)) package["screenshots"] = screenshots - - # Filter - package["versions"] = list(filter(ignore_testing, package["versions"])) - - if len(package["versions"]) > 0: - modules.append(package) - + modules.append(package) return modules def _get_http_session(): @@ -107,35 +112,32 @@ def _get_http_session(): osession.mount('https://', requests.adapters.HTTPAdapter(max_retries=oretries)) return osession -def _list_repository_modules(rdb, repository_name, repository_url, skip_core_modules = False, skip_testing_versions=False): - key = f'cluster/repository_cache/{repository_name}' - cache = rdb.hgetall(key) - if cache: - return _parse_repository_metadata(repository_name, repository_url, cache["updated"], cache["data"], skip_core_modules, skip_testing_versions) - - url = _urljoin(repository_url, "repodata.json") - try: - hsubscription = rdb.hgetall("cluster/subscription") - with _get_http_session() as osession: - if hsubscription and url.startswith("https://subscription.nethserver.com/"): - # Send system_id for HTTP Basic authentication - osession.auth = (hsubscription["system_id"], hashlib.sha256(hsubscription["auth_token"].encode()).hexdigest()) - resp = osession.get(url) - repodata = resp.text - updated = resp.headers.get('Last-Modified', "") - except Exception as ex: - print(f"Fetching {url}:", ex, file=sys.stderr) - # If repository is not accessible or invalid, just return an empty array - return [] - - modules = _parse_repository_metadata(repository_name, repository_url, updated, repodata, skip_core_modules, skip_testing_versions) +def _list_repository_modules(rdb, repository_name, repository_url): + cache_key = f'cluster/repository_cache/{repository_name}' + hcache = rdb.hgetall(cache_key) + if not hcache: + url = _urljoin(repository_url, "repodata.json") + hsubscription = rdb.hgetall("cluster/subscription") or None + try: + with _get_http_session() as osession: + if hsubscription and url.startswith("https://subscription.nethserver.com/"): + # Send system_id for HTTP Basic authentication + osession.auth = (hsubscription["system_id"], hashlib.sha256(hsubscription["auth_token"].encode()).hexdigest()) + resp = osession.get(url) + repodata_raw = resp.text + updated = resp.headers.get('Last-Modified', "") + except Exception as ex: + print(f"Fetching {url}:", ex, file=sys.stderr) + # If repository is not accessible or invalid, just return an empty array + return [] + hcache = {"data": repodata_raw, "updated": updated} + modules = _parse_repository_metadata(repository_name, repository_url, hcache['updated'], hcache['data']) # Save inside the cache if data is valid if modules: # Save also repodata file date - rdb.hset(key, mapping={"data": repodata, "updated": updated}) + rdb.hset(cache_key, mapping=hcache) # Set cache expiration to 3600 seconds - rdb.expire(key, 3600) - + rdb.expire(cache_key, 3600) return modules class LatestModuleLookupError(Exception): @@ -229,6 +231,9 @@ def _fetch_metadata_json(module_id, image_name): ometadata.setdefault("screenshots", []) ometadata.setdefault("repository", "__local__") ometadata.setdefault("repository_updated", repository_updated_timestamp) + ometadata.setdefault("repository_authority", "__local__") + ometadata["certification_level"] = 0 + ometadata["rootfull"] = False try: ometadata['logo'] = glob(f'{path_prefix}apps/{module_id}/img/*logo*png')[0].removeprefix(path_prefix) except Exception as ex: @@ -236,8 +241,36 @@ def _fetch_metadata_json(module_id, image_name): print(agent.SD_INFO + "_fetch_metadata_json/glob:", ex, file=sys.stderr) return ometadata -def list_available(rdb, skip_core_modules = False): +def list_available(rdb, skip_core_modules=False): """Iterate over enabled repositories and return available modules respecting the repository priority.""" + hsubscription = rdb.hgetall("cluster/subscription") or None + modules = [] + for omod in _get_available_modules(rdb).values(): + if not _repo_has_testing_flag(rdb, omod["repository"]): + # Ignore testing releases for new installations: + omod["versions"] = list(filter(lambda v: v["testing"] is False, omod["versions"])) + if not omod["versions"]: + continue # Ignore modules with no versions + omod["certification_level"] = _calc_certification_level(omod, bool(hsubscription)) + try: + if skip_core_modules and 'core_module' in omod["versions"][0]['labels']['org.nethserver.flags']: + continue # core modules are ignored + except: + pass + try: + package_is_rootfull = omod["versions"][0]["labels"]["org.nethserver.rootfull"] == "1" + except: + package_is_rootfull = False + # Ignore untrusted rootfull application, if a subscription is active + if hsubscription and package_is_rootfull and omod["certification_level"] < 3: + print(agent.SD_WARNING + f"Ignoring image of rootfull application {omod['source']}: certification_level {omod['certification_level']} is too low", file=sys.stderr) + continue # skip package + else: + omod['rootfull'] = package_is_rootfull + modules.append(omod) + return modules + +def _get_available_modules(rdb): modules = {} repositories = [] # List all modules from enabled repositories @@ -250,26 +283,24 @@ def list_available(rdb, skip_core_modules = False): # Skip non-enabled repositories if repo.get("status", "0") != "1": continue - skip_testing_versions = repo.get("testing", "0") != "1" - for rmod in _list_repository_modules(rdb, nrepo, repo["url"], skip_core_modules, skip_testing_versions): + for rmod in _list_repository_modules(rdb, nrepo, repo["url"]): if rmod["source"] in modules: continue # skip duplicated images from lower priority modules modules[rmod["source"]] = rmod rmod['versions'].sort(key=lambda v: _parse_version_object(v["tag"]), reverse=True) # Integrate the available set with instances that do not belong to any # repository. They can be found in the "installed" dict: - for module_source, module_instances in list_installed(rdb, skip_core_modules).items(): + for module_source, module_instances in list_installed(rdb).items(): if module_source in modules: continue _, image_name = module_source.rsplit("/", 1) vmetadata = _fetch_metadata_json(module_instances[0]['id'], image_name) vmetadata["versions"] = list(_synthesize_module_version(oinst) for oinst in module_instances) - vmetadata["installed"] = module_instances + vmetadata["versions"].sort(key=lambda v: _parse_version_object(v["tag"]), reverse=True) vmetadata["source"] = module_source - vmetadata["updates"] = [] vmetadata["id"] = image_name modules[module_source] = vmetadata - return list(modules.values()) + return modules def list_installed(rdb, skip_core_modules = False): installed = {} @@ -314,44 +345,48 @@ def list_installed_core(rdb): return installed - -def list_updates(rdb, skip_core_modules = False): +def list_updates(rdb, skip_core_modules=False, with_testing_update=False): updates = [] - installed = list_installed(rdb, skip_core_modules) - available = list_available(rdb, skip_core_modules) - - for module in available: - if module["source"] not in installed.keys(): - continue - newest_version = None - for version in module["versions"]: - try: - # skip bogus version tag - v = semver.Version.parse(version["tag"]) - except: - continue - # Skip testing versions if testing is disabled - testing = rdb.hget(f'cluster/repository/{module["repository"]}', 'testing') - if testing != "1" and not v.prerelease is None: - continue - newest_version = version["tag"] - break + installed_modules = list_installed(rdb, skip_core_modules) + available_modules = _get_available_modules(rdb) - # Handle multiple instances of the same module - for instance in installed[module["source"]]: + flat_instance_list = list(mi for module_instances in installed_modules.values() for mi in module_instances) + for instance in flat_instance_list: + if not instance['source'] in available_modules: + continue # skip instance if is not available from any repository + try: + current_version = semver.parse_version_info(instance['version']) + except: + continue # skip development version: instance must be updated manually + # Assuming the versions array is sorted in decreasing oreder, look + # up update candidates for both stable and testing updates: + update_candidate = None + testing_update_candidate = None + available_module = available_modules[instance['source']] + repository_name = available_module['repository'] + for atag in list(aver['tag'] for aver in available_module['versions']): try: - cur = semver.Version.parse(instance["version"]) + available_version = semver.parse_version_info(atag) except: - # skip installed instanced with dev version - continue - - # Version are already sorted - # First match is the newest release - if v > cur: - # Create a copy to not change original object - update = instance.copy() - update["update"] = version["tag"] - updates.append(update) + continue # skip non-semver available tag + if available_version <= current_version: + continue # ignore tags that do not update the current one + if update_candidate is None and ( + _repo_has_testing_flag(rdb, repository_name) + or not available_version.prerelease + ): + update_candidate = available_version + if testing_update_candidate is None and with_testing_update and available_version.prerelease: + testing_update_candidate = available_version + + # If a stable or testing candidate has been found, add this + # instance to the updates list. + if update_candidate: + instance['update'] = str(update_candidate) + if testing_update_candidate: + instance['testing_update'] = str(testing_update_candidate) + if 'update' in instance or 'testing_update' in instance: + updates.append(instance) return updates diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update b/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update index de0b8248a..0dd57b815 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update +++ b/core/imageroot/var/lib/nethserver/cluster/actions/add-module/50update @@ -100,7 +100,7 @@ with subprocess.Popen(['podman', 'image', 'inspect', image_url], stdout=subproce inspect_image_repodigest = inspect[0]['RepoDigests'][0] if 'org.nethserver.rootfull' in inspect_labels: - is_rootfull = int(inspect_labels['org.nethserver.rootfull']) == 1 + is_rootfull = inspect_labels['org.nethserver.rootfull'] == "1" else: is_rootfull = False diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription b/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription index 5ea2743e6..e1c535d52 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription +++ b/core/imageroot/var/lib/nethserver/cluster/actions/get-subscription/10get_subscription @@ -12,6 +12,8 @@ import sys import json import requests, urllib3.util +terms_url = "https://docs.nethserver.org/projects/ns8/en/latest/subscription.html#terms-and-conditions" + def _get_http_session(): osession = requests.Session() osession.timeout = 15 # Timout for HTTP connections @@ -95,4 +97,4 @@ elif hsubscription["provider"] == "nsent": elif hsubscription["provider"] == "nscom": hsubscription.update(fetch_subscription_info_nscom(rdb, hsubscription)) -json.dump({"subscription": hsubscription}, fp=sys.stdout) +json.dump({"subscription": hsubscription, "terms_url": terms_url}, fp=sys.stdout) diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read index c0f8f5f02..4772b4e6d 100755 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/50read @@ -35,7 +35,7 @@ def get_module(source, modules): rdb = agent.redis_connect(privileged=True) installed = cluster.modules.list_installed(rdb, skip_core_modules = True) available = cluster.modules.list_available(rdb, skip_core_modules = True) -updates = cluster.modules.list_updates(rdb, skip_core_modules = True) +updates = cluster.modules.list_updates(rdb, skip_core_modules=True, with_testing_update=True) # Prepare variables for later use for a in available: diff --git a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json index a49fe0906..0d8bdd7a9 100644 --- a/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json +++ b/core/imageroot/var/lib/nethserver/cluster/actions/list-modules/validate-output.json @@ -60,6 +60,8 @@ ], "repository": "t3", "repository_updated": "Mon, 28 Jun 2021 14:42:44 GMT", + "certification_level": 2, + "rootfull": false, "updates": [ { "id": "dokuwiki2", @@ -99,6 +101,19 @@ "type": "string", "description": "Unique name of a package" }, + "upstream_name": { + "type": "string", + "description": "The alternative software name and version number, if they differ from the package name and version", + "example": "Nextcloud Hub 7" + }, + "certification_level": { + "type": "integer", + "description": "The higher, the better: 0=unknown certification, 5=max" + }, + "rootfull": { + "type": "boolean", + "description": "True if the application gains full OS privileges when installed" + }, "description": { "type": "object", "description": "A map of language codes (eg. en, it) with the translated description" @@ -135,6 +150,10 @@ "docs": { "type": "object", "parameters": { + "terms_url": { + "type": "uri", + "description": "Optional link to the application Terms & Conditions document" + }, "documentation_url": { "type": "uri", "description": "Link to the package documentation" @@ -217,9 +236,13 @@ "type": "string", "description": "A valid semantic version extracted from image tag" }, + "testing_update": { + "type": "string", + "description": "A semver prerelease tag, greater than 'version' field" + }, "update": { "type": "string", - "description": "A valid semantic version extracted from image tag wich should be greater than 'version' field" + "description": "A semver tag greater than 'version' field" } }, "required": [ @@ -227,8 +250,7 @@ "node", "digest", "source", - "version", - "update" + "version" ] } }, @@ -275,6 +297,8 @@ "authors", "docs", "source", + "certification_level", + "rootfull", "versions", "installed", "updates" diff --git a/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json b/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json index 332401aef..668c376e2 100644 --- a/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json +++ b/core/imageroot/var/lib/nethserver/cluster/repodata-schema.json @@ -74,6 +74,11 @@ "type": "string", "description": "Common name of a package" }, + "upstream_name": { + "type": "string", + "description": "The alternative software name and version number, if they differ from the package name and version", + "example": "Nextcloud Hub 7" + }, "description": { "type": "object", "description": "A map of language code and description of the packaged translated in the indexed language" @@ -110,6 +115,10 @@ "docs": { "type": "object", "parameters": { + "terms_url": { + "type": "uri", + "description": "Optional link to the application Terms & Conditions document" + }, "documentation_url": { "type": "uri", "description": "Link to the package documentation" diff --git a/docs/modules/images.md b/docs/modules/images.md index d3ccc12e3..01a0a7fb3 100644 --- a/docs/modules/images.md +++ b/docs/modules/images.md @@ -55,6 +55,7 @@ Module images can use a list of well-known labels to configure the system: - `no_data_backup`: if present, the modules will need no data backup - `rootless`: if present, the module is rootless (calculated from `org.nethserver.rootfull` label) - `rootfull`: if present, the module is rootfull (calculated from `org.nethserver.rootfull` label) +- `org.nethserver.max-per-node`: maximum number of module instances installed on the same node Labels are set by `build-images.sh`, when the images are built. diff --git a/docs/modules/metadata.md b/docs/modules/metadata.md index 18cf45985..2bb03fb7d 100644 --- a/docs/modules/metadata.md +++ b/docs/modules/metadata.md @@ -22,11 +22,12 @@ without `id`, `logo` and `versions` fields. Example of `metadata.json`: ```json { - "name": "kickstart", + "name": "Kickstart", + "upstream_name": "Kickstart 12", "description": { "en": "My kickstart module" }, - "categories": [], + "categories": ["somecategory"], "authors": [ { "name": "Name Surname", @@ -34,6 +35,7 @@ Example of `metadata.json`: } ], "docs": { + "terms_url": "https://docs.kickstart.com/terms/", "documentation_url": "https://docs.kickstart.com/", "bug_url": "https://github.com/NethServer/dev", "code_url": "https://github.com/author/ns8-kickstart"