From c6be7e81d095a18f08feb39e9d9cc2d78880dc9a Mon Sep 17 00:00:00 2001 From: Tyler Levy Conde Date: Wed, 30 Oct 2024 11:01:21 -0600 Subject: [PATCH 1/5] Use relenv.fetch to download tarball --- salt/utils/relenv.py | 93 ++++++++++---------------------------------- 1 file changed, 20 insertions(+), 73 deletions(-) diff --git a/salt/utils/relenv.py b/salt/utils/relenv.py index ff07b25c97a..d9516eb92d0 100644 --- a/salt/utils/relenv.py +++ b/salt/utils/relenv.py @@ -1,91 +1,38 @@ -import logging import os -import re -import requests +__virtualname__ = "relenv" -import salt.utils.files -import salt.utils.hashutils -import salt.utils.http -import salt.utils.thin +try: + import relenv.fetch # noqa -log = logging.getLogger(__name__) + HAS_RELENV = True +except ImportError: + HAS_RELENV = False + + +def __virtual__(): + if HAS_RELENV: + return __virtualname__ + return (False, "Pip install `relenv` to use this feature") def gen_relenv( - cachedir, kernel, os_arch, - overwrite=False, ): """ Deploy salt-relenv. - :param cachedir: The cache directory where the downloaded tarball will be stored. :param kernel: The detected OS (e.g., 'linux', 'darwin', 'windows'). :param os_arch: The detected architecture (e.g., 'amd64', 'x86_64', 'arm64'). - :param overwrite: Whether to overwrite the existing cached tarball. :return: The path to the recompressed .tgz file. """ - # Set up directories - relenv_dir = os.path.join(cachedir, "relenv", kernel, os_arch) - if not os.path.isdir(relenv_dir): - os.makedirs(relenv_dir) - - relenv_url = get_tarball(kernel, os_arch) - tarball_path = os.path.join(relenv_dir, "salt-relenv.tar.xz") - - # Download the tarball if it doesn't exist or overwrite is True - if overwrite or not os.path.exists(tarball_path): - if not download(cachedir, relenv_url, tarball_path): - return False - - return tarball_path - - -def get_tarball(kernel, arch): - """ - Get the latest Salt onedir tarball URL for the specified kernel and architecture. - :param kernel: The detected OS (e.g., 'linux', 'darwin', 'windows'). - :param arch: The detected architecture (e.g., 'amd64', 'x86_64', 'arm64'). - :return: The URL of the latest tarball. - """ - base_url = "https://repo.saltproject.io/salt/py3/onedir/latest/" - try: - # Request the page listing - response = requests.get(base_url, timeout=60) - response.raise_for_status() - except requests.RequestException as e: - log.error(f"Failed to retrieve tarball listing: {e}") - raise ValueError("Unable to fetch tarball list from repository") - - # Search for tarball filenames that match the kernel and arch - pattern = re.compile(rf'href="(salt-.*-onedir-{kernel}-{arch}\.tar\.xz)"') - matches = pattern.findall(response.text) - if not matches: - log.error(f"No tarballs found for {kernel} and {arch}") - raise ValueError(f"No tarball found for {kernel} {arch}") + triplet = relenv.fetch.get_triplet(machine=os_arch, plat=kernel) + version = os.environ.get("RELENV_FETCH_VERSION", relenv.fetch.__version__) + python = relenv.fetch.platform_versions()[0] - # Return the latest tarball URL - matches.sort() - latest_tarball = matches[-1] - return base_url + latest_tarball + relenv.fetch.fetch(version, triplet, python) - -def download(cachedir, url, destination): - """ - Download the salt artifact from the given destination to the cache. - """ - if not os.path.exists(destination): - log.info(f"Downloading from {url} to {destination}") - with salt.utils.files.fopen(destination, "wb+") as dest_file: - result = salt.utils.http.query( - url=url, - method="GET", - stream=True, - streaming_callback=dest_file.write, - raise_error=True, - ) - if result.get("status") != 200: - log.error(f"Failed to download file from {url}") - return False - return True + return os.path.join( + relenv.fetch.work_dir("build", relenv.fetch.DATA_DIR), + f"{python}-{triplet}.tar.xz", + ) From a379dd86de11d22da8ad478f98fe37695039c563 Mon Sep 17 00:00:00 2001 From: Tyler Levy Conde Date: Thu, 31 Oct 2024 10:16:57 -0600 Subject: [PATCH 2/5] update to new artifactory location --- salt/utils/relenv.py | 116 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 97 insertions(+), 19 deletions(-) diff --git a/salt/utils/relenv.py b/salt/utils/relenv.py index d9516eb92d0..e4919f797e1 100644 --- a/salt/utils/relenv.py +++ b/salt/utils/relenv.py @@ -1,38 +1,116 @@ +import logging import os +import re -__virtualname__ = "relenv" +import requests -try: - import relenv.fetch # noqa +import salt.utils.files +import salt.utils.hashutils +import salt.utils.http +import salt.utils.thin - HAS_RELENV = True -except ImportError: - HAS_RELENV = False +log = logging.getLogger(__name__) - -def __virtual__(): - if HAS_RELENV: - return __virtualname__ - return (False, "Pip install `relenv` to use this feature") +BASE_URL = "https://packages.broadcom.com/artifactory/saltproject-generic/onedir/" def gen_relenv( + cachedir, kernel, os_arch, + overwrite=False, + base_url=BASE_URL, ): """ Deploy salt-relenv. + :param cachedir: The cache directory where the downloaded tarball will be stored. :param kernel: The detected OS (e.g., 'linux', 'darwin', 'windows'). :param os_arch: The detected architecture (e.g., 'amd64', 'x86_64', 'arm64'). + :param overwrite: Whether to overwrite the existing cached tarball. :return: The path to the recompressed .tgz file. """ - triplet = relenv.fetch.get_triplet(machine=os_arch, plat=kernel) - version = os.environ.get("RELENV_FETCH_VERSION", relenv.fetch.__version__) - python = relenv.fetch.platform_versions()[0] + version = get_latest_version(base_url) + + # Set up directories + relenv_dir = os.path.join(cachedir, "relenv", version, kernel, os_arch) + if not os.path.isdir(relenv_dir): + os.makedirs(relenv_dir) + + relenv_url = get_tarball(f"{base_url}/{version}", kernel, os_arch) + tarball_path = os.path.join(relenv_dir, "salt-relenv.tar.xz") + + # Download the tarball if it doesn't exist or overwrite is True + if overwrite or not os.path.exists(tarball_path): + if not download(cachedir, relenv_url, tarball_path): + return False + + return tarball_path - relenv.fetch.fetch(version, triplet, python) - return os.path.join( - relenv.fetch.work_dir("build", relenv.fetch.DATA_DIR), - f"{python}-{triplet}.tar.xz", - ) +def get_tarball(base_url, kernel, arch): + """ + Get the latest Salt onedir tarball URL for the specified kernel and architecture. + :param base_url: The artifactory location + :param kernel: The detected OS (e.g., 'linux', 'darwin', 'windows'). + :param arch: The detected architecture (e.g., 'amd64', 'x86_64', 'arm64'). + :return: The URL of the latest tarball. + """ + + try: + # Request the page listing + response = requests.get(base_url, timeout=60) + response.raise_for_status() + except requests.RequestException as e: + log.error(f"Failed to retrieve tarball listing: {e}") + raise ValueError("Unable to fetch tarball list from repository") + + # Search for tarball filenames that match the kernel and arch + pattern = re.compile(rf'href="(salt-.*-onedir-{kernel}-{arch}\.tar\.xz)"') + matches = pattern.findall(response.text) + if not matches: + log.error(f"No tarballs found for {kernel} and {arch}") + raise ValueError(f"No tarball found for {kernel} {arch}") + + # Return the latest tarball URL + matches.sort() + latest_tarball = matches[-1] + return base_url + latest_tarball + +def get_latest_version(base_url): + try: + response = requests.get(base_url, timeout=60) + response.raise_for_status() + except requests.RequestException as e: + log.error(f"Failed to retrieve directory listing: {e}") + raise ValueError("Unable to fetch directory list from repository") + + # Extract version numbers from hrefs + pattern = re.compile(r'href="(\d+\.\d+)/"') + versions = pattern.findall(response.text) + + if not versions: + log.error("No versions found in directory listing") + raise ValueError("No versions found in directory listing") + + # Sort versions numerically and return the latest one + versions.sort(key=lambda s: list(map(int, s.split('.'))), reverse=True) + return versions[0] + +def download(cachedir, url, destination): + """ + Download the salt artifact from the given destination to the cache. + """ + if not os.path.exists(destination): + log.info(f"Downloading from {url} to {destination}") + with salt.utils.files.fopen(destination, "wb+") as dest_file: + result = salt.utils.http.query( + url=url, + method="GET", + stream=True, + streaming_callback=dest_file.write, + raise_error=True, + ) + if result.get("status") != 200: + log.error(f"Failed to download file from {url}") + return False + return True From d2c23de8549b5fc33677c1421f35a7f0bc142d2b Mon Sep 17 00:00:00 2001 From: Tyler Levy Conde Date: Thu, 31 Oct 2024 10:20:16 -0600 Subject: [PATCH 3/5] fix precommit --- salt/utils/relenv.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/salt/utils/relenv.py b/salt/utils/relenv.py index e4919f797e1..99c8b618018 100644 --- a/salt/utils/relenv.py +++ b/salt/utils/relenv.py @@ -76,6 +76,7 @@ def get_tarball(base_url, kernel, arch): latest_tarball = matches[-1] return base_url + latest_tarball + def get_latest_version(base_url): try: response = requests.get(base_url, timeout=60) @@ -83,7 +84,7 @@ def get_latest_version(base_url): except requests.RequestException as e: log.error(f"Failed to retrieve directory listing: {e}") raise ValueError("Unable to fetch directory list from repository") - + # Extract version numbers from hrefs pattern = re.compile(r'href="(\d+\.\d+)/"') versions = pattern.findall(response.text) @@ -93,9 +94,10 @@ def get_latest_version(base_url): raise ValueError("No versions found in directory listing") # Sort versions numerically and return the latest one - versions.sort(key=lambda s: list(map(int, s.split('.'))), reverse=True) + versions.sort(key=lambda s: list(map(int, s.split("."))), reverse=True) return versions[0] + def download(cachedir, url, destination): """ Download the salt artifact from the given destination to the cache. From 3ce97c354cf711eb9931d6b38b12b9dd486f906f Mon Sep 17 00:00:00 2001 From: Tyler Levy Conde Date: Fri, 1 Nov 2024 11:37:03 -0600 Subject: [PATCH 4/5] Add docstrings --- salt/utils/relenv.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/salt/utils/relenv.py b/salt/utils/relenv.py index 99c8b618018..483c321236e 100644 --- a/salt/utils/relenv.py +++ b/salt/utils/relenv.py @@ -41,7 +41,7 @@ def gen_relenv( # Download the tarball if it doesn't exist or overwrite is True if overwrite or not os.path.exists(tarball_path): - if not download(cachedir, relenv_url, tarball_path): + if not download(relenv_url, tarball_path): return False return tarball_path @@ -78,6 +78,11 @@ def get_tarball(base_url, kernel, arch): def get_latest_version(base_url): + """ + Retrieve the latest version from the base artifactory directory + :param base_url: The artifactory location + :return: The version number of the latest artifact + """ try: response = requests.get(base_url, timeout=60) response.raise_for_status() @@ -98,9 +103,12 @@ def get_latest_version(base_url): return versions[0] -def download(cachedir, url, destination): +def download(url, destination): """ Download the salt artifact from the given destination to the cache. + :param url: The artifact location + :param destination: Path to the downloaded file + :return: True if the download was successful, else False """ if not os.path.exists(destination): log.info(f"Downloading from {url} to {destination}") From 1354b9fc07c3e8c60d26532355c9a0a6f6441150 Mon Sep 17 00:00:00 2001 From: Tyler Levy Conde Date: Tue, 5 Nov 2024 11:14:19 -0700 Subject: [PATCH 5/5] Fix relenv url --- salt/client/ssh/__init__.py | 32 ++++++++++++++++---------------- salt/utils/relenv.py | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/salt/client/ssh/__init__.py b/salt/client/ssh/__init__.py index 140b8bc58cd..8809d348b56 100644 --- a/salt/client/ssh/__init__.py +++ b/salt/client/ssh/__init__.py @@ -1,4 +1,5 @@ """ + Create ssh executor system """ @@ -192,11 +193,7 @@ ) -SSH_SH_SHIM_RELENV = "\n".join( - [ - s.strip() - for s in """ -/bin/sh << 'EOF' +SSH_SH_SHIM_RELENV = """/bin/sh << 'EOF' set -e set -u DEBUG="{DEBUG}" @@ -211,9 +208,11 @@ SUDO_USER="{SUDO_USER}" if [ "$SUDO" ] && [ "$SUDO_USER" ]; then SUDO="$SUDO -u $SUDO_USER"; fi -RELENV_TAR="{THIN_DIR}/salt-relenv.tar.xz" -mkdir -p "{THIN_DIR}" -SALT_CALL_BIN="{THIN_DIR}/salt-call" +THIN_DIR={THIN_DIR} +mkdir -pv $THIN_DIR >&2 +ls "$THIN_DIR" >&2 +RELENV_TAR=$THIN_DIR/salt-relenv.tar.xz +SALT_CALL_BIN="$THIN_DIR/salt-call" # Extract relenv tarball if not already extracted if [ ! -x "$SALT_CALL_BIN" ]; then @@ -223,9 +222,13 @@ exit 11 fi + file $RELENV_TAR >&2 + # Create directory if not exists and extract the tarball - tar --strip-components=1 -xf "$RELENV_TAR" -C "{THIN_DIR}" -fi + tar --strip-components=1 -xf "$RELENV_TAR" -C "$THIN_DIR" +fi; + +file $SALT_CALL_BIN >&2 # Check if Python binary is executable if [ ! -x "$SALT_CALL_BIN" ]; then @@ -236,13 +239,10 @@ echo "{RSTR}" echo "{RSTR}" >&2 -exec $SUDO "$SALT_CALL_BIN" --retcode-passthrough --local --metadata --out=json -lquiet -c "{THIN_DIR}" {ARGS} +exec $SUDO "$SALT_CALL_BIN" --retcode-passthrough --local --metadata --out=json -lquiet -c "$THIN_DIR" {ARGS} EOF -""".split( - "\n" - ) - ] -) +""" + if not salt.utils.platform.is_windows() and not salt.utils.platform.is_junos(): shim_file = os.path.join(os.path.dirname(__file__), "ssh_py_shim.py") diff --git a/salt/utils/relenv.py b/salt/utils/relenv.py index 483c321236e..0180b68538a 100644 --- a/salt/utils/relenv.py +++ b/salt/utils/relenv.py @@ -36,7 +36,7 @@ def gen_relenv( if not os.path.isdir(relenv_dir): os.makedirs(relenv_dir) - relenv_url = get_tarball(f"{base_url}/{version}", kernel, os_arch) + relenv_url = get_tarball(f"{base_url}/{version}/", kernel, os_arch) tarball_path = os.path.join(relenv_dir, "salt-relenv.tar.xz") # Download the tarball if it doesn't exist or overwrite is True