From 4431384bd13e1a330cdee29970d8cac9a83630cc Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Thu, 6 Jan 2022 19:04:41 +0100 Subject: [PATCH 01/18] GL dind and volumes --- Dockerfile | 7 +++++++ src/drivers/gitlab.js | 9 +++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5fa7ffe30..f242d2899 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,6 +53,13 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && apt-get clean \ && rm --recursive --force /var/lib/apt/lists/* +# INSTALL DOCKER +RUN curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh && \ + usermod -aG docker ubuntu \ + sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock \ + && apt-get clean \ + && rm --recursive --force /var/lib/apt/lists/* + # INSTALL PYTHON ARG PYTHON_VERSION=3 RUN add-apt-repository universe --yes \ diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index 160d62c03..8fce22309 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -9,7 +9,7 @@ const ProxyAgent = require('proxy-agent'); const { fetchUploadData, download, exec } = require('../utils'); -const { IN_DOCKER, CI_PIPELINE_ID } = process.env; +const { IN_DOCKER, CI_PIPELINE_ID, CML_GL_DOCKER_VOLUMES } = process.env; const API_VER = 'v4'; class Gitlab { constructor(opts = {}) { @@ -195,7 +195,12 @@ class Gitlab { --wait-timeout ${idleTimeout} \ --executor "${IN_DOCKER ? 'shell' : 'docker'}" \ --docker-image "iterativeai/cml:${gpu ? 'latest-gpu' : 'latest'}" \ - --docker-runtime "${gpu ? 'nvidia' : ''}" \ + ${ + CML_GL_DOCKER_VOLUMES + ? `--docker-volumes=${CML_GL_DOCKER_VOLUMES}` + : '' + } \ + ${gpu ? '--docker-runtime nvidia' : ''} \ ${single ? '--max-builds 1' : ''}`; return spawn(command, { shell: true }); From 2263de96504d3b3b4451c8605600ca87d36b0562 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Thu, 6 Jan 2022 19:20:41 +0100 Subject: [PATCH 02/18] support multiple volumes --- src/drivers/gitlab.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index 8fce22309..114b25995 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -186,6 +186,11 @@ class Gitlab { const { protocol, host } = new URL(this.repo); const { token } = await this.registerRunner({ tags: labels, name }); + + let dockerVolumesTpl = ''; + CML_GL_DOCKER_VOLUMES.split(',').forEach((vol) => { + dockerVolumesTpl += `--docker-volumes ${vol} `; + }); const command = `${bin} --log-format="json" run-single \ --builds-dir "${workdir}" \ --cache-dir "${workdir}" \ @@ -195,12 +200,8 @@ class Gitlab { --wait-timeout ${idleTimeout} \ --executor "${IN_DOCKER ? 'shell' : 'docker'}" \ --docker-image "iterativeai/cml:${gpu ? 'latest-gpu' : 'latest'}" \ - ${ - CML_GL_DOCKER_VOLUMES - ? `--docker-volumes=${CML_GL_DOCKER_VOLUMES}` - : '' - } \ ${gpu ? '--docker-runtime nvidia' : ''} \ + ${dockerVolumesTpl} \ ${single ? '--max-builds 1' : ''}`; return spawn(command, { shell: true }); From 3f5c872a5fe499ec76f16f864cbcf1540e049642 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Thu, 6 Jan 2022 19:23:18 +0100 Subject: [PATCH 03/18] no docker user --- Dockerfile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f242d2899..c76de6908 100644 --- a/Dockerfile +++ b/Dockerfile @@ -55,8 +55,6 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - # INSTALL DOCKER RUN curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh && \ - usermod -aG docker ubuntu \ - sudo setfacl --modify user:ubuntu:rw /var/run/docker.sock \ && apt-get clean \ && rm --recursive --force /var/lib/apt/lists/* From bbfea5d83f200209703be2de9091404f11ae6485 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Thu, 6 Jan 2022 19:36:13 +0100 Subject: [PATCH 04/18] no sudo --- Dockerfile | 2 +- get-docker.sh | 630 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 631 insertions(+), 1 deletion(-) create mode 100644 get-docker.sh diff --git a/Dockerfile b/Dockerfile index c76de6908..9257b7828 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && rm --recursive --force /var/lib/apt/lists/* # INSTALL DOCKER -RUN curl -fsSL https://get.docker.com -o get-docker.sh && sudo sh get-docker.sh && \ +RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \ && apt-get clean \ && rm --recursive --force /var/lib/apt/lists/* diff --git a/get-docker.sh b/get-docker.sh new file mode 100644 index 000000000..209a6da32 --- /dev/null +++ b/get-docker.sh @@ -0,0 +1,630 @@ +#!/bin/sh +set -e +# Docker CE for Linux installation script +# +# See https://docs.docker.com/engine/install/ for the installation steps. +# +# This script is meant for quick & easy install via: +# $ curl -fsSL https://get.docker.com -o get-docker.sh +# $ sh get-docker.sh +# +# For test builds (ie. release candidates): +# $ curl -fsSL https://test.docker.com -o test-docker.sh +# $ sh test-docker.sh +# +# NOTE: Make sure to verify the contents of the script +# you downloaded matches the contents of install.sh +# located at https://github.com/docker/docker-install +# before executing. +# +# Git commit from https://github.com/docker/docker-install when +# the script was uploaded (Should only be modified by upload job): +SCRIPT_COMMIT_SHA="93d2499759296ac1f9c510605fef85052a2c32be" + +# strip "v" prefix if present +VERSION="${VERSION#v}" + +# The channel to install from: +# * nightly +# * test +# * stable +# * edge (deprecated) +DEFAULT_CHANNEL_VALUE="stable" +if [ -z "$CHANNEL" ]; then + CHANNEL=$DEFAULT_CHANNEL_VALUE +fi + +DEFAULT_DOWNLOAD_URL="https://download.docker.com" +if [ -z "$DOWNLOAD_URL" ]; then + DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL +fi + +DEFAULT_REPO_FILE="docker-ce.repo" +if [ -z "$REPO_FILE" ]; then + REPO_FILE="$DEFAULT_REPO_FILE" +fi + +mirror='' +DRY_RUN=${DRY_RUN:-} +while [ $# -gt 0 ]; do + case "$1" in + --mirror) + mirror="$2" + shift + ;; + --dry-run) + DRY_RUN=1 + ;; + --*) + echo "Illegal option $1" + ;; + esac + shift $(( $# > 0 ? 1 : 0 )) +done + +case "$mirror" in + Aliyun) + DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce" + ;; + AzureChinaCloud) + DOWNLOAD_URL="https://mirror.azure.cn/docker-ce" + ;; +esac + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +# version_gte checks if the version specified in $VERSION is at least +# the given CalVer (YY.MM) version. returns 0 (success) if $VERSION is either +# unset (=latest) or newer or equal than the specified version. Returns 1 (fail) +# otherwise. +# +# examples: +# +# VERSION=20.10 +# version_gte 20.10 // 0 (success) +# version_gte 19.03 // 0 (success) +# version_gte 21.10 // 1 (fail) +version_gte() { + if [ -z "$VERSION" ]; then + return 0 + fi + eval calver_compare "$VERSION" "$1" +} + +# calver_compare compares two CalVer (YY.MM) version strings. returns 0 (success) +# if version A is newer or equal than version B, or 1 (fail) otherwise. Patch +# releases and pre-release (-alpha/-beta) are not taken into account +# +# examples: +# +# calver_compare 20.10 19.03 // 0 (success) +# calver_compare 20.10 20.10 // 0 (success) +# calver_compare 19.03 20.10 // 1 (fail) +calver_compare() ( + set +x + + yy_a="$(echo "$1" | cut -d'.' -f1)" + yy_b="$(echo "$2" | cut -d'.' -f1)" + if [ "$yy_a" -lt "$yy_b" ]; then + return 1 + fi + if [ "$yy_a" -gt "$yy_b" ]; then + return 0 + fi + mm_a="$(echo "$1" | cut -d'.' -f2)" + mm_b="$(echo "$2" | cut -d'.' -f2)" + if [ "${mm_a#0}" -lt "${mm_b#0}" ]; then + return 1 + fi + + return 0 +) + +is_dry_run() { + if [ -z "$DRY_RUN" ]; then + return 1 + else + return 0 + fi +} + +is_wsl() { + case "$(uname -r)" in + *microsoft* ) true ;; # WSL 2 + *Microsoft* ) true ;; # WSL 1 + * ) false;; + esac +} + +is_darwin() { + case "$(uname -s)" in + *darwin* ) true ;; + *Darwin* ) true ;; + * ) false;; + esac +} + +deprecation_notice() { + distro=$1 + distro_version=$2 + echo + printf "\033[91;1mDEPRECATION WARNING\033[0m\n" + printf " This Linux distribution (\033[1m%s %s\033[0m) reached end-of-life and is no longer supported by this script.\n" "$distro" "$distro_version" + echo " No updates or security fixes will be released for this distribution, and users are recommended" + echo " to upgrade to a currently maintained version of $distro." + echo + printf "Press \033[1mCtrl+C\033[0m now to abort this script, or wait for the installation to continue." + echo + sleep 10 +} + +get_distribution() { + lsb_dist="" + # Every system that we officially support has /etc/os-release + if [ -r /etc/os-release ]; then + lsb_dist="$(. /etc/os-release && echo "$ID")" + fi + # Returning an empty string here should be alright since the + # case statements don't act unless you provide an actual value + echo "$lsb_dist" +} + +echo_docker_as_nonroot() { + if is_dry_run; then + return + fi + if command_exists docker && [ -e /var/run/docker.sock ]; then + ( + set -x + $sh_c 'docker version' + ) || true + fi + + # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output + echo + echo "================================================================================" + echo + if version_gte "20.10"; then + echo "To run Docker as a non-privileged user, consider setting up the" + echo "Docker daemon in rootless mode for your user:" + echo + echo " dockerd-rootless-setuptool.sh install" + echo + echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode." + echo + fi + echo + echo "To run the Docker daemon as a fully privileged service, but granting non-root" + echo "users access, refer to https://docs.docker.com/go/daemon-access/" + echo + echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent" + echo " to root access on the host. Refer to the 'Docker daemon attack surface'" + echo " documentation for details: https://docs.docker.com/go/attack-surface/" + echo + echo "================================================================================" + echo +} + +# Check if this is a forked Linux distro +check_forked() { + + # Check for lsb_release command existence, it usually exists in forked distros + if command_exists lsb_release; then + # Check if the `-u` option is supported + set +e + lsb_release -a -u > /dev/null 2>&1 + lsb_release_exit_code=$? + set -e + + # Check if the command has exited successfully, it means we're in a forked distro + if [ "$lsb_release_exit_code" = "0" ]; then + # Print info about current distro + cat <<-EOF + You're using '$lsb_dist' version '$dist_version'. + EOF + + # Get the upstream release info + lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]') + dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]') + + # Print info about upstream distro + cat <<-EOF + Upstream release is '$lsb_dist' version '$dist_version'. + EOF + else + if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then + if [ "$lsb_dist" = "osmc" ]; then + # OSMC runs Raspbian + lsb_dist=raspbian + else + # We're Debian and don't even know it! + lsb_dist=debian + fi + dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$dist_version" in + 11) + dist_version="bullseye" + ;; + 10) + dist_version="buster" + ;; + 9) + dist_version="stretch" + ;; + 8) + dist_version="jessie" + ;; + esac + fi + fi + fi +} + +do_install() { + echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA" + + if command_exists docker; then + cat >&2 <<-'EOF' + Warning: the "docker" command appears to already exist on this system. + + If you already have Docker installed, this script can cause trouble, which is + why we're displaying this warning and provide the opportunity to cancel the + installation. + + If you installed the current Docker package using this script and are using it + again to update Docker, you can safely ignore this message. + + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 20 ) + fi + + user="$(id -un 2>/dev/null || true)" + + sh_c='sh -c' + if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo -E sh -c' + elif command_exists su; then + sh_c='su -c' + else + cat >&2 <<-'EOF' + Error: this installer needs the ability to run commands as root. + We are unable to find either "sudo" or "su" available to make this happen. + EOF + exit 1 + fi + fi + + if is_dry_run; then + sh_c="echo" + fi + + # perform some very rudimentary platform detection + lsb_dist=$( get_distribution ) + lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" + + if is_wsl; then + echo + echo "WSL DETECTED: We recommend using Docker Desktop for Windows." + echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" + echo + cat >&2 <<-'EOF' + + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 20 ) + fi + + case "$lsb_dist" in + + ubuntu) + if command_exists lsb_release; then + dist_version="$(lsb_release --codename | cut -f2)" + fi + if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then + dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" + fi + ;; + + debian|raspbian) + dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$dist_version" in + 11) + dist_version="bullseye" + ;; + 10) + dist_version="buster" + ;; + 9) + dist_version="stretch" + ;; + 8) + dist_version="jessie" + ;; + esac + ;; + + centos|rhel|sles) + if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then + dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + + *) + if command_exists lsb_release; then + dist_version="$(lsb_release --release | cut -f2)" + fi + if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then + dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + + esac + + # Check if this is a forked Linux distro + check_forked + + # Print deprecation warnings for distro versions that recently reached EOL, + # but may still be commonly used (especially LTS versions). + case "$lsb_dist.$dist_version" in + debian.stretch|debian.jessie) + deprecation_notice "$lsb_dist" "$dist_version" + ;; + raspbian.stretch|raspbian.jessie) + deprecation_notice "$lsb_dist" "$dist_version" + ;; + ubuntu.xenial|ubuntu.trusty) + deprecation_notice "$lsb_dist" "$dist_version" + ;; + fedora.*) + if [ "$dist_version" -lt 33 ]; then + deprecation_notice "$lsb_dist" "$dist_version" + fi + ;; + esac + + # Run setup for each distro accordingly + case "$lsb_dist" in + ubuntu|debian|raspbian) + pre_reqs="apt-transport-https ca-certificates curl" + if ! command -v gpg > /dev/null; then + pre_reqs="$pre_reqs gnupg" + fi + apt_repo="deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL" + ( + if ! is_dry_run; then + set -x + fi + $sh_c 'apt-get update -qq >/dev/null' + $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null" + $sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | gpg --dearmor --yes -o /usr/share/keyrings/docker-archive-keyring.gpg" + $sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list" + $sh_c 'apt-get update -qq >/dev/null' + ) + pkg_version="" + if [ -n "$VERSION" ]; then + if is_dry_run; then + echo "# WARNING: VERSION pinning is not supported in DRY_RUN" + else + # Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel + pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist" + search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" + pkg_version="$($sh_c "$search_command")" + echo "INFO: Searching repository for VERSION '$VERSION'" + echo "INFO: $search_command" + if [ -z "$pkg_version" ]; then + echo + echo "ERROR: '$VERSION' not found amongst apt-cache madison results" + echo + exit 1 + fi + if version_gte "18.09"; then + search_command="apt-cache madison 'docker-ce-cli' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" + echo "INFO: $search_command" + cli_pkg_version="=$($sh_c "$search_command")" + fi + pkg_version="=$pkg_version" + fi + fi + ( + pkgs="" + if version_gte "18.09"; then + # older versions don't support a cli package + pkgs="$pkgs docker-ce-cli${cli_pkg_version%=}" + fi + if version_gte "20.10" && [ "$(uname -m)" = "x86_64" ]; then + # also install the latest version of the "docker scan" cli-plugin (only supported on x86 currently) + pkgs="$pkgs docker-scan-plugin" + fi + pkgs="$pkgs docker-ce${pkg_version%=}" + if ! is_dry_run; then + set -x + fi + $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $pkgs >/dev/null" + if version_gte "20.10"; then + # Install docker-ce-rootless-extras without "--no-install-recommends", so as to install slirp4netns when available + $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce-rootless-extras${pkg_version%=} >/dev/null" + fi + ) + echo_docker_as_nonroot + exit 0 + ;; + centos|fedora|rhel) + if [ "$(uname -m)" != "s390x" ] && [ "$lsb_dist" = "rhel" ]; then + echo "Packages for RHEL are currently only available for s390x." + exit 1 + fi + yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" + if ! curl -Ifs "$yum_repo" > /dev/null; then + echo "Error: Unable to curl repository file $yum_repo, is it valid?" + exit 1 + fi + if [ "$lsb_dist" = "fedora" ]; then + pkg_manager="dnf" + config_manager="dnf config-manager" + enable_channel_flag="--set-enabled" + disable_channel_flag="--set-disabled" + pre_reqs="dnf-plugins-core" + pkg_suffix="fc$dist_version" + else + pkg_manager="yum" + config_manager="yum-config-manager" + enable_channel_flag="--enable" + disable_channel_flag="--disable" + pre_reqs="yum-utils" + pkg_suffix="el" + fi + ( + if ! is_dry_run; then + set -x + fi + $sh_c "$pkg_manager install -y -q $pre_reqs" + $sh_c "$config_manager --add-repo $yum_repo" + + if [ "$CHANNEL" != "stable" ]; then + $sh_c "$config_manager $disable_channel_flag docker-ce-*" + $sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL" + fi + $sh_c "$pkg_manager makecache" + ) + pkg_version="" + if [ -n "$VERSION" ]; then + if is_dry_run; then + echo "# WARNING: VERSION pinning is not supported in DRY_RUN" + else + pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix" + search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" + pkg_version="$($sh_c "$search_command")" + echo "INFO: Searching repository for VERSION '$VERSION'" + echo "INFO: $search_command" + if [ -z "$pkg_version" ]; then + echo + echo "ERROR: '$VERSION' not found amongst $pkg_manager list results" + echo + exit 1 + fi + if version_gte "18.09"; then + # older versions don't support a cli package + search_command="$pkg_manager list --showduplicates 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" + cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)" + fi + # Cut out the epoch and prefix with a '-' + pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)" + fi + fi + ( + if ! is_dry_run; then + set -x + fi + # install the correct cli version first + if [ -n "$cli_pkg_version" ]; then + $sh_c "$pkg_manager install -y -q docker-ce-cli-$cli_pkg_version" + fi + $sh_c "$pkg_manager install -y -q docker-ce$pkg_version" + if version_gte "20.10"; then + $sh_c "$pkg_manager install -y -q docker-ce-rootless-extras$pkg_version" + fi + ) + echo_docker_as_nonroot + exit 0 + ;; + sles) + if [ "$(uname -m)" != "s390x" ]; then + echo "Packages for SLES are currently only available for s390x" + exit 1 + fi + + sles_version="${dist_version##*.}" + sles_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" + opensuse_repo="https://download.opensuse.org/repositories/security:SELinux/SLE_15_SP$sles_version/security:SELinux.repo" + if ! curl -Ifs "$sles_repo" > /dev/null; then + echo "Error: Unable to curl repository file $sles_repo, is it valid?" + exit 1 + fi + pre_reqs="ca-certificates curl libseccomp2 awk" + ( + if ! is_dry_run; then + set -x + fi + $sh_c "zypper install -y $pre_reqs" + $sh_c "zypper addrepo $sles_repo" + if ! is_dry_run; then + cat >&2 <<-'EOF' + WARNING!! + openSUSE repository (https://download.opensuse.org/repositories/security:SELinux) will be enabled now. + Do you wish to continue? + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 30 ) + fi + $sh_c "zypper addrepo $opensuse_repo" + $sh_c "zypper --gpg-auto-import-keys refresh" + $sh_c "zypper lr -d" + ) + pkg_version="" + if [ -n "$VERSION" ]; then + if is_dry_run; then + echo "# WARNING: VERSION pinning is not supported in DRY_RUN" + else + pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g")" + search_command="zypper search -s --match-exact 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" + pkg_version="$($sh_c "$search_command")" + echo "INFO: Searching repository for VERSION '$VERSION'" + echo "INFO: $search_command" + if [ -z "$pkg_version" ]; then + echo + echo "ERROR: '$VERSION' not found amongst zypper list results" + echo + exit 1 + fi + search_command="zypper search -s --match-exact 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" + # It's okay for cli_pkg_version to be blank, since older versions don't support a cli package + cli_pkg_version="$($sh_c "$search_command")" + pkg_version="-$pkg_version" + + search_command="zypper search -s --match-exact 'docker-ce-rootless-extras' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" + rootless_pkg_version="$($sh_c "$search_command")" + rootless_pkg_version="-$rootless_pkg_version" + fi + fi + ( + if ! is_dry_run; then + set -x + fi + # install the correct cli version first + if [ -n "$cli_pkg_version" ]; then + $sh_c "zypper install -y docker-ce-cli-$cli_pkg_version" + fi + $sh_c "zypper install -y docker-ce$pkg_version" + if version_gte "20.10"; then + $sh_c "zypper install -y docker-ce-rootless-extras$rootless_pkg_version" + fi + ) + echo_docker_as_nonroot + exit 0 + ;; + *) + if [ -z "$lsb_dist" ]; then + if is_darwin; then + echo + echo "ERROR: Unsupported operating system 'macOS'" + echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" + echo + exit 1 + fi + fi + echo + echo "ERROR: Unsupported distribution '$lsb_dist'" + echo + exit 1 + ;; + esac + exit 1 +} + +# wrapped up in a function so that we have some protection against only getting +# half the file during "curl | sh" +do_install From 806bb8717a8225259c838c633008c1a1dd7dceee Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Thu, 6 Jan 2022 19:53:54 +0100 Subject: [PATCH 05/18] remove extra amps --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 9257b7828..2cc07f82a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && rm --recursive --force /var/lib/apt/lists/* # INSTALL DOCKER -RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh && \ +RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh \ && apt-get clean \ && rm --recursive --force /var/lib/apt/lists/* From 711bc72caaa7c03583223386b423348788420330 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Thu, 6 Jan 2022 20:04:59 +0100 Subject: [PATCH 06/18] del get-docker.sh --- .gitignore | 1 + get-docker.sh | 630 -------------------------------------------------- 2 files changed, 1 insertion(+), 630 deletions(-) delete mode 100644 get-docker.sh diff --git a/.gitignore b/.gitignore index e57c16df4..2222db089 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ terraform.* !terraform.test.js crash.log /build +get-docker.sh diff --git a/get-docker.sh b/get-docker.sh deleted file mode 100644 index 209a6da32..000000000 --- a/get-docker.sh +++ /dev/null @@ -1,630 +0,0 @@ -#!/bin/sh -set -e -# Docker CE for Linux installation script -# -# See https://docs.docker.com/engine/install/ for the installation steps. -# -# This script is meant for quick & easy install via: -# $ curl -fsSL https://get.docker.com -o get-docker.sh -# $ sh get-docker.sh -# -# For test builds (ie. release candidates): -# $ curl -fsSL https://test.docker.com -o test-docker.sh -# $ sh test-docker.sh -# -# NOTE: Make sure to verify the contents of the script -# you downloaded matches the contents of install.sh -# located at https://github.com/docker/docker-install -# before executing. -# -# Git commit from https://github.com/docker/docker-install when -# the script was uploaded (Should only be modified by upload job): -SCRIPT_COMMIT_SHA="93d2499759296ac1f9c510605fef85052a2c32be" - -# strip "v" prefix if present -VERSION="${VERSION#v}" - -# The channel to install from: -# * nightly -# * test -# * stable -# * edge (deprecated) -DEFAULT_CHANNEL_VALUE="stable" -if [ -z "$CHANNEL" ]; then - CHANNEL=$DEFAULT_CHANNEL_VALUE -fi - -DEFAULT_DOWNLOAD_URL="https://download.docker.com" -if [ -z "$DOWNLOAD_URL" ]; then - DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL -fi - -DEFAULT_REPO_FILE="docker-ce.repo" -if [ -z "$REPO_FILE" ]; then - REPO_FILE="$DEFAULT_REPO_FILE" -fi - -mirror='' -DRY_RUN=${DRY_RUN:-} -while [ $# -gt 0 ]; do - case "$1" in - --mirror) - mirror="$2" - shift - ;; - --dry-run) - DRY_RUN=1 - ;; - --*) - echo "Illegal option $1" - ;; - esac - shift $(( $# > 0 ? 1 : 0 )) -done - -case "$mirror" in - Aliyun) - DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce" - ;; - AzureChinaCloud) - DOWNLOAD_URL="https://mirror.azure.cn/docker-ce" - ;; -esac - -command_exists() { - command -v "$@" > /dev/null 2>&1 -} - -# version_gte checks if the version specified in $VERSION is at least -# the given CalVer (YY.MM) version. returns 0 (success) if $VERSION is either -# unset (=latest) or newer or equal than the specified version. Returns 1 (fail) -# otherwise. -# -# examples: -# -# VERSION=20.10 -# version_gte 20.10 // 0 (success) -# version_gte 19.03 // 0 (success) -# version_gte 21.10 // 1 (fail) -version_gte() { - if [ -z "$VERSION" ]; then - return 0 - fi - eval calver_compare "$VERSION" "$1" -} - -# calver_compare compares two CalVer (YY.MM) version strings. returns 0 (success) -# if version A is newer or equal than version B, or 1 (fail) otherwise. Patch -# releases and pre-release (-alpha/-beta) are not taken into account -# -# examples: -# -# calver_compare 20.10 19.03 // 0 (success) -# calver_compare 20.10 20.10 // 0 (success) -# calver_compare 19.03 20.10 // 1 (fail) -calver_compare() ( - set +x - - yy_a="$(echo "$1" | cut -d'.' -f1)" - yy_b="$(echo "$2" | cut -d'.' -f1)" - if [ "$yy_a" -lt "$yy_b" ]; then - return 1 - fi - if [ "$yy_a" -gt "$yy_b" ]; then - return 0 - fi - mm_a="$(echo "$1" | cut -d'.' -f2)" - mm_b="$(echo "$2" | cut -d'.' -f2)" - if [ "${mm_a#0}" -lt "${mm_b#0}" ]; then - return 1 - fi - - return 0 -) - -is_dry_run() { - if [ -z "$DRY_RUN" ]; then - return 1 - else - return 0 - fi -} - -is_wsl() { - case "$(uname -r)" in - *microsoft* ) true ;; # WSL 2 - *Microsoft* ) true ;; # WSL 1 - * ) false;; - esac -} - -is_darwin() { - case "$(uname -s)" in - *darwin* ) true ;; - *Darwin* ) true ;; - * ) false;; - esac -} - -deprecation_notice() { - distro=$1 - distro_version=$2 - echo - printf "\033[91;1mDEPRECATION WARNING\033[0m\n" - printf " This Linux distribution (\033[1m%s %s\033[0m) reached end-of-life and is no longer supported by this script.\n" "$distro" "$distro_version" - echo " No updates or security fixes will be released for this distribution, and users are recommended" - echo " to upgrade to a currently maintained version of $distro." - echo - printf "Press \033[1mCtrl+C\033[0m now to abort this script, or wait for the installation to continue." - echo - sleep 10 -} - -get_distribution() { - lsb_dist="" - # Every system that we officially support has /etc/os-release - if [ -r /etc/os-release ]; then - lsb_dist="$(. /etc/os-release && echo "$ID")" - fi - # Returning an empty string here should be alright since the - # case statements don't act unless you provide an actual value - echo "$lsb_dist" -} - -echo_docker_as_nonroot() { - if is_dry_run; then - return - fi - if command_exists docker && [ -e /var/run/docker.sock ]; then - ( - set -x - $sh_c 'docker version' - ) || true - fi - - # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output - echo - echo "================================================================================" - echo - if version_gte "20.10"; then - echo "To run Docker as a non-privileged user, consider setting up the" - echo "Docker daemon in rootless mode for your user:" - echo - echo " dockerd-rootless-setuptool.sh install" - echo - echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode." - echo - fi - echo - echo "To run the Docker daemon as a fully privileged service, but granting non-root" - echo "users access, refer to https://docs.docker.com/go/daemon-access/" - echo - echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent" - echo " to root access on the host. Refer to the 'Docker daemon attack surface'" - echo " documentation for details: https://docs.docker.com/go/attack-surface/" - echo - echo "================================================================================" - echo -} - -# Check if this is a forked Linux distro -check_forked() { - - # Check for lsb_release command existence, it usually exists in forked distros - if command_exists lsb_release; then - # Check if the `-u` option is supported - set +e - lsb_release -a -u > /dev/null 2>&1 - lsb_release_exit_code=$? - set -e - - # Check if the command has exited successfully, it means we're in a forked distro - if [ "$lsb_release_exit_code" = "0" ]; then - # Print info about current distro - cat <<-EOF - You're using '$lsb_dist' version '$dist_version'. - EOF - - # Get the upstream release info - lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]') - dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]') - - # Print info about upstream distro - cat <<-EOF - Upstream release is '$lsb_dist' version '$dist_version'. - EOF - else - if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then - if [ "$lsb_dist" = "osmc" ]; then - # OSMC runs Raspbian - lsb_dist=raspbian - else - # We're Debian and don't even know it! - lsb_dist=debian - fi - dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" - case "$dist_version" in - 11) - dist_version="bullseye" - ;; - 10) - dist_version="buster" - ;; - 9) - dist_version="stretch" - ;; - 8) - dist_version="jessie" - ;; - esac - fi - fi - fi -} - -do_install() { - echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA" - - if command_exists docker; then - cat >&2 <<-'EOF' - Warning: the "docker" command appears to already exist on this system. - - If you already have Docker installed, this script can cause trouble, which is - why we're displaying this warning and provide the opportunity to cancel the - installation. - - If you installed the current Docker package using this script and are using it - again to update Docker, you can safely ignore this message. - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - user="$(id -un 2>/dev/null || true)" - - sh_c='sh -c' - if [ "$user" != 'root' ]; then - if command_exists sudo; then - sh_c='sudo -E sh -c' - elif command_exists su; then - sh_c='su -c' - else - cat >&2 <<-'EOF' - Error: this installer needs the ability to run commands as root. - We are unable to find either "sudo" or "su" available to make this happen. - EOF - exit 1 - fi - fi - - if is_dry_run; then - sh_c="echo" - fi - - # perform some very rudimentary platform detection - lsb_dist=$( get_distribution ) - lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" - - if is_wsl; then - echo - echo "WSL DETECTED: We recommend using Docker Desktop for Windows." - echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" - echo - cat >&2 <<-'EOF' - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - case "$lsb_dist" in - - ubuntu) - if command_exists lsb_release; then - dist_version="$(lsb_release --codename | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then - dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" - fi - ;; - - debian|raspbian) - dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" - case "$dist_version" in - 11) - dist_version="bullseye" - ;; - 10) - dist_version="buster" - ;; - 9) - dist_version="stretch" - ;; - 8) - dist_version="jessie" - ;; - esac - ;; - - centos|rhel|sles) - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - *) - if command_exists lsb_release; then - dist_version="$(lsb_release --release | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - esac - - # Check if this is a forked Linux distro - check_forked - - # Print deprecation warnings for distro versions that recently reached EOL, - # but may still be commonly used (especially LTS versions). - case "$lsb_dist.$dist_version" in - debian.stretch|debian.jessie) - deprecation_notice "$lsb_dist" "$dist_version" - ;; - raspbian.stretch|raspbian.jessie) - deprecation_notice "$lsb_dist" "$dist_version" - ;; - ubuntu.xenial|ubuntu.trusty) - deprecation_notice "$lsb_dist" "$dist_version" - ;; - fedora.*) - if [ "$dist_version" -lt 33 ]; then - deprecation_notice "$lsb_dist" "$dist_version" - fi - ;; - esac - - # Run setup for each distro accordingly - case "$lsb_dist" in - ubuntu|debian|raspbian) - pre_reqs="apt-transport-https ca-certificates curl" - if ! command -v gpg > /dev/null; then - pre_reqs="$pre_reqs gnupg" - fi - apt_repo="deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL" - ( - if ! is_dry_run; then - set -x - fi - $sh_c 'apt-get update -qq >/dev/null' - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null" - $sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | gpg --dearmor --yes -o /usr/share/keyrings/docker-archive-keyring.gpg" - $sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list" - $sh_c 'apt-get update -qq >/dev/null' - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - # Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist" - search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst apt-cache madison results" - echo - exit 1 - fi - if version_gte "18.09"; then - search_command="apt-cache madison 'docker-ce-cli' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" - echo "INFO: $search_command" - cli_pkg_version="=$($sh_c "$search_command")" - fi - pkg_version="=$pkg_version" - fi - fi - ( - pkgs="" - if version_gte "18.09"; then - # older versions don't support a cli package - pkgs="$pkgs docker-ce-cli${cli_pkg_version%=}" - fi - if version_gte "20.10" && [ "$(uname -m)" = "x86_64" ]; then - # also install the latest version of the "docker scan" cli-plugin (only supported on x86 currently) - pkgs="$pkgs docker-scan-plugin" - fi - pkgs="$pkgs docker-ce${pkg_version%=}" - if ! is_dry_run; then - set -x - fi - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $pkgs >/dev/null" - if version_gte "20.10"; then - # Install docker-ce-rootless-extras without "--no-install-recommends", so as to install slirp4netns when available - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce-rootless-extras${pkg_version%=} >/dev/null" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - centos|fedora|rhel) - if [ "$(uname -m)" != "s390x" ] && [ "$lsb_dist" = "rhel" ]; then - echo "Packages for RHEL are currently only available for s390x." - exit 1 - fi - yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" - if ! curl -Ifs "$yum_repo" > /dev/null; then - echo "Error: Unable to curl repository file $yum_repo, is it valid?" - exit 1 - fi - if [ "$lsb_dist" = "fedora" ]; then - pkg_manager="dnf" - config_manager="dnf config-manager" - enable_channel_flag="--set-enabled" - disable_channel_flag="--set-disabled" - pre_reqs="dnf-plugins-core" - pkg_suffix="fc$dist_version" - else - pkg_manager="yum" - config_manager="yum-config-manager" - enable_channel_flag="--enable" - disable_channel_flag="--disable" - pre_reqs="yum-utils" - pkg_suffix="el" - fi - ( - if ! is_dry_run; then - set -x - fi - $sh_c "$pkg_manager install -y -q $pre_reqs" - $sh_c "$config_manager --add-repo $yum_repo" - - if [ "$CHANNEL" != "stable" ]; then - $sh_c "$config_manager $disable_channel_flag docker-ce-*" - $sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL" - fi - $sh_c "$pkg_manager makecache" - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix" - search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst $pkg_manager list results" - echo - exit 1 - fi - if version_gte "18.09"; then - # older versions don't support a cli package - search_command="$pkg_manager list --showduplicates 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" - cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)" - fi - # Cut out the epoch and prefix with a '-' - pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)" - fi - fi - ( - if ! is_dry_run; then - set -x - fi - # install the correct cli version first - if [ -n "$cli_pkg_version" ]; then - $sh_c "$pkg_manager install -y -q docker-ce-cli-$cli_pkg_version" - fi - $sh_c "$pkg_manager install -y -q docker-ce$pkg_version" - if version_gte "20.10"; then - $sh_c "$pkg_manager install -y -q docker-ce-rootless-extras$pkg_version" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - sles) - if [ "$(uname -m)" != "s390x" ]; then - echo "Packages for SLES are currently only available for s390x" - exit 1 - fi - - sles_version="${dist_version##*.}" - sles_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" - opensuse_repo="https://download.opensuse.org/repositories/security:SELinux/SLE_15_SP$sles_version/security:SELinux.repo" - if ! curl -Ifs "$sles_repo" > /dev/null; then - echo "Error: Unable to curl repository file $sles_repo, is it valid?" - exit 1 - fi - pre_reqs="ca-certificates curl libseccomp2 awk" - ( - if ! is_dry_run; then - set -x - fi - $sh_c "zypper install -y $pre_reqs" - $sh_c "zypper addrepo $sles_repo" - if ! is_dry_run; then - cat >&2 <<-'EOF' - WARNING!! - openSUSE repository (https://download.opensuse.org/repositories/security:SELinux) will be enabled now. - Do you wish to continue? - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 30 ) - fi - $sh_c "zypper addrepo $opensuse_repo" - $sh_c "zypper --gpg-auto-import-keys refresh" - $sh_c "zypper lr -d" - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g")" - search_command="zypper search -s --match-exact 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst zypper list results" - echo - exit 1 - fi - search_command="zypper search -s --match-exact 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" - # It's okay for cli_pkg_version to be blank, since older versions don't support a cli package - cli_pkg_version="$($sh_c "$search_command")" - pkg_version="-$pkg_version" - - search_command="zypper search -s --match-exact 'docker-ce-rootless-extras' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" - rootless_pkg_version="$($sh_c "$search_command")" - rootless_pkg_version="-$rootless_pkg_version" - fi - fi - ( - if ! is_dry_run; then - set -x - fi - # install the correct cli version first - if [ -n "$cli_pkg_version" ]; then - $sh_c "zypper install -y docker-ce-cli-$cli_pkg_version" - fi - $sh_c "zypper install -y docker-ce$pkg_version" - if version_gte "20.10"; then - $sh_c "zypper install -y docker-ce-rootless-extras$rootless_pkg_version" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - *) - if [ -z "$lsb_dist" ]; then - if is_darwin; then - echo - echo "ERROR: Unsupported operating system 'macOS'" - echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" - echo - exit 1 - fi - fi - echo - echo "ERROR: Unsupported distribution '$lsb_dist'" - echo - exit 1 - ;; - esac - exit 1 -} - -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" -do_install From 2aba1f9098b95c21ea726abcc798886c2835039b Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Tue, 11 Jan 2022 17:28:09 +0100 Subject: [PATCH 07/18] docker-volumes param --- bin/cml/runner.js | 15 ++++++++++++--- src/drivers/gitlab.js | 17 ++++++++++++----- src/terraform.js | 4 +++- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/bin/cml/runner.js b/bin/cml/runner.js index f6cf03541..825278816 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -115,6 +115,7 @@ const runCloud = async (opts) => { idleTimeout, name, single, + dockerVolumes, cloud, cloudRegion: region, cloudType: type, @@ -163,7 +164,8 @@ const runCloud = async (opts) => { spotPrice, startupScript, awsSecurityGroup, - awsSubnetId + awsSubnetId, + dockerVolumes }); } @@ -215,14 +217,16 @@ const runCloud = async (opts) => { const runLocal = async (opts) => { winston.info(`Launching ${cml.driver} runner`); - const { workdir, name, labels, single, idleTimeout, noRetry } = opts; + const { workdir, name, labels, single, idleTimeout, noRetry, dockerVolumes } = + opts; const proc = await cml.startRunner({ workdir, name, labels, single, - idleTimeout + idleTimeout, + dockerVolumes }); const dataHandler = async (data) => { @@ -412,6 +416,11 @@ exports.handler = async (opts) => { exports.builder = (yargs) => yargs.env('CML_RUNNER').options( kebabcaseKeys({ + dockerVolumes: { + type: 'array', + default: [], + description: 'Docker volumes. This feature is only supported in Gitlab' + }, labels: { type: 'string', default: 'cml', diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index 114b25995..5cba8f732 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -9,7 +9,7 @@ const ProxyAgent = require('proxy-agent'); const { fetchUploadData, download, exec } = require('../utils'); -const { IN_DOCKER, CI_PIPELINE_ID, CML_GL_DOCKER_VOLUMES } = process.env; +const { IN_DOCKER, CI_PIPELINE_ID } = process.env; const API_VER = 'v4'; class Gitlab { constructor(opts = {}) { @@ -162,7 +162,14 @@ class Gitlab { } async startRunner(opts) { - const { workdir, idleTimeout, single, labels, name } = opts; + const { + workdir, + idleTimeout, + single, + labels, + name, + dockerVolumes = [] + } = opts; let gpu = true; try { @@ -178,8 +185,8 @@ class Gitlab { try { const bin = resolve(workdir, 'gitlab-runner'); if (!(await fse.pathExists(bin))) { - const url = - 'https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-linux-amd64'; + const arch = process.platform === 'darwin' ? 'osx-x64' : 'linux-x64'; + const url = `https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-${arch}-amd64`; await download({ url, path: bin }); await fs.chmod(bin, '777'); } @@ -188,7 +195,7 @@ class Gitlab { const { token } = await this.registerRunner({ tags: labels, name }); let dockerVolumesTpl = ''; - CML_GL_DOCKER_VOLUMES.split(',').forEach((vol) => { + dockerVolumes.forEach((vol) => { dockerVolumesTpl += `--docker-volumes ${vol} `; }); const command = `${bin} --log-format="json" run-single \ diff --git a/src/terraform.js b/src/terraform.js index 60be706ad..a3b33d730 100644 --- a/src/terraform.js +++ b/src/terraform.js @@ -86,7 +86,8 @@ const iterativeCmlRunnerTpl = (opts = {}) => { spotPrice, startupScript, awsSecurityGroup, - awsSubnetId + awsSubnetId, + dockerVolumes } = opts; const template = ` @@ -121,6 +122,7 @@ resource "iterative_cml_runner" "runner" { ? `metadata = {\n ${mapCloudMetadata(metadata).join('\n ')}\n }` : '' } + ${dockerVolumes ? `docker_volumes = ${JSON.stringify(dockerVolumes)}` : ''} } `; return template; From 6519de0e62cff070f5e7de9438dd53f6054fa0ed Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Tue, 11 Jan 2022 18:54:04 +0100 Subject: [PATCH 08/18] test tpl --- bin/cml/runner.test.js | 116 +++++++++++++++++++++-------------------- 1 file changed, 59 insertions(+), 57 deletions(-) diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js index 507df22e2..47270fdbe 100644 --- a/bin/cml/runner.test.js +++ b/bin/cml/runner.test.js @@ -5,64 +5,66 @@ describe('CML e2e', () => { const output = await exec(`echo none | node ./bin/cml.js runner --help`); expect(output).toMatchInlineSnapshot(` -"cml.js runner + "cml.js runner -Launch and register a self-hosted runner + Launch and register a self-hosted runner -Options: - --help Show help [boolean] - --version Show version number [boolean] - --log Maximum log level - [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] - --labels One or more user-defined labels for this runner - (delimited with commas) [string] [default: \\"cml\\"] - --idle-timeout Seconds to wait for jobs before shutting down. Set - to -1 to disable timeout [number] [default: 300] - --name Name displayed in the repository once registered - [string] [default: cml-{ID}] - --no-retry Do not restart workflow terminated due to instance - disposal or GitHub Actions timeout [boolean] - --single Exit after running a single job [boolean] - --reuse Don't launch a new runner if an existing one has - the same name or overlapping labels [boolean] - --driver Platform where the repository is hosted. If not - specified, it will be inferred from the - environment [string] [choices: \\"github\\", \\"gitlab\\"] - --repo Repository to be used for registering the runner. - If not specified, it will be inferred from the - environment [string] - --token Personal access token to register a self-hosted - runner on the repository. If not specified, it - will be inferred from the environment [string] - --cloud Cloud to deploy the runner - [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] - --cloud-region Region where the instance is deployed. Choices: - [us-east, us-west, eu-west, eu-north]. Also - accepts native cloud regions - [string] [default: \\"us-west\\"] - --cloud-type Instance type. Choices: [m, l, xl]. Also supports - native types like i.e. t2.micro [string] - --cloud-permission-set Specifies the instance profile in AWS or instance - service account in GCP [string] [default: \\"\\"] - --cloud-metadata Key Value pairs to associate cml-runner instance - on the provider i.e. tags/labels \\"key=value\\" - [array] [default: []] - --cloud-gpu GPU type. - [string] [choices: \\"nogpu\\", \\"k80\\", \\"v100\\", \\"tesla\\"] - --cloud-hdd-size HDD size in GB [number] - --cloud-ssh-private Custom private RSA SSH key. If not provided an - automatically generated throwaway key will be used - [string] - --cloud-spot Request a spot instance [boolean] - --cloud-spot-price Maximum spot instance bidding price in USD. - Defaults to the current spot bidding price - [number] [default: -1] - --cloud-startup-script Run the provided Base64-encoded Linux shell script - during the instance initialization [string] - --cloud-aws-security-group Specifies the security group in AWS - [string] [default: \\"\\"] - --cloud-aws-subnet-id Specifies the subnet to use within AWS - [string] [default: \\"\\"]" -`); + Options: + --help Show help [boolean] + --version Show version number [boolean] + --log Maximum log level + [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] + --docker-volumes Docker volumes. This feature is only supported in + Gitlab [array] [default: []] + --labels One or more user-defined labels for this runner + (delimited with commas) [string] [default: \\"cml\\"] + --idle-timeout Seconds to wait for jobs before shutting down. Set + to -1 to disable timeout [number] [default: 300] + --name Name displayed in the repository once registered + [string] [default: cml-{ID}] + --no-retry Do not restart workflow terminated due to instance + disposal or GitHub Actions timeout [boolean] + --single Exit after running a single job [boolean] + --reuse Don't launch a new runner if an existing one has + the same name or overlapping labels [boolean] + --driver Platform where the repository is hosted. If not + specified, it will be inferred from the + environment [string] [choices: \\"github\\", \\"gitlab\\"] + --repo Repository to be used for registering the runner. + If not specified, it will be inferred from the + environment [string] + --token Personal access token to register a self-hosted + runner on the repository. If not specified, it + will be inferred from the environment [string] + --cloud Cloud to deploy the runner + [string] [choices: \\"aws\\", \\"azure\\", \\"gcp\\", \\"kubernetes\\"] + --cloud-region Region where the instance is deployed. Choices: + [us-east, us-west, eu-west, eu-north]. Also + accepts native cloud regions + [string] [default: \\"us-west\\"] + --cloud-type Instance type. Choices: [m, l, xl]. Also supports + native types like i.e. t2.micro [string] + --cloud-permission-set Specifies the instance profile in AWS or instance + service account in GCP [string] [default: \\"\\"] + --cloud-metadata Key Value pairs to associate cml-runner instance + on the provider i.e. tags/labels \\"key=value\\" + [array] [default: []] + --cloud-gpu GPU type. + [string] [choices: \\"nogpu\\", \\"k80\\", \\"v100\\", \\"tesla\\"] + --cloud-hdd-size HDD size in GB [number] + --cloud-ssh-private Custom private RSA SSH key. If not provided an + automatically generated throwaway key will be used + [string] + --cloud-spot Request a spot instance [boolean] + --cloud-spot-price Maximum spot instance bidding price in USD. + Defaults to the current spot bidding price + [number] [default: -1] + --cloud-startup-script Run the provided Base64-encoded Linux shell script + during the instance initialization [string] + --cloud-aws-security-group Specifies the security group in AWS + [string] [default: \\"\\"] + --cloud-aws-subnet-id Specifies the subnet to use within AWS + [string] [default: \\"\\"]" + `); }); }); From 389b1997fa3a9aedd3f1fdc150b6553a1098fbc5 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Tue, 11 Jan 2022 19:12:27 +0100 Subject: [PATCH 09/18] tf tests --- src/terraform.test.js | 516 ++++++++++++++++++++++++------------------ 1 file changed, 291 insertions(+), 225 deletions(-) diff --git a/src/terraform.test.js b/src/terraform.test.js index cd7cae4a8..0a840b7bb 100644 --- a/src/terraform.test.js +++ b/src/terraform.test.js @@ -4,43 +4,44 @@ describe('Terraform tests', () => { test('default options', async () => { const output = iterativeCmlRunnerTpl({}); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - - - - - - - - - - - - - - - - - - - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + + + + + + + + + + + + + + + + + + + + + + } + " + `); }); test('basic settings', async () => { @@ -63,43 +64,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { awsSecurityGroup: 'mysg' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - aws_security_group = \\"mysg\\" - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + aws_security_group = \\"mysg\\" + + + + } + " + `); }); test('basic settings with runner forever', async () => { @@ -121,43 +123,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { spotPrice: '0.0001' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 0 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 0 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + + + + + } + " + `); }); test('basic settings with metadata', async () => { @@ -180,46 +183,107 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { metadata: { one: 'value', two: null } }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - - - metadata = { - one = \\"value\\" - two = \\"\\" - } -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + + + metadata = { + one = \\"value\\" + two = \\"\\" + } + + } + " + `); + }); + + test('basic settings with docker volumes', async () => { + const output = iterativeCmlRunnerTpl({ + repo: 'https://', + token: 'abc', + driver: 'gitlab', + labels: 'mylabel', + idleTimeout: 300, + name: 'myrunner', + single: true, + cloud: 'aws', + region: 'west', + type: 'mymachinetype', + gpu: 'mygputype', + hddSize: 50, + sshPrivate: 'myprivate', + spot: true, + spotPrice: '0.0001', + dockerVolumes: ['/aa:/aa', '/bb:/bb'] + }); + expect(output).toMatchInlineSnapshot(` + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + + + + docker_volumes = [\\"/aa:/aa\\",\\"/bb:/bb\\"] + } + " + `); }); test('basic settings with permission set', async () => { @@ -243,43 +307,44 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { awsSecurityGroup: 'mysg' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\" - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - - aws_security_group = \\"mysg\\" - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + instance_permission_set = \\"arn:aws:iam::1:instance-profile/x\\" + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + + aws_security_group = \\"mysg\\" + + + + } + " + `); }); test('Startup script', async () => { @@ -302,42 +367,43 @@ resource \\"iterative_cml_runner\\" \\"runner\\" { startupScript: 'c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=' }); expect(output).toMatchInlineSnapshot(` -" - -terraform { - required_providers { - iterative = { - source = \\"iterative/iterative\\" - } - } -} - -provider \\"iterative\\" {} - - -resource \\"iterative_cml_runner\\" \\"runner\\" { - repo = \\"https://\\" - token = \\"abc\\" - driver = \\"gitlab\\" - labels = \\"mylabel\\" - idle_timeout = 300 - name = \\"myrunner\\" - single = \\"true\\" - cloud = \\"aws\\" - region = \\"west\\" - instance_type = \\"mymachinetype\\" - instance_gpu = \\"mygputype\\" - instance_hdd_size = 50 - - ssh_private = \\"myprivate\\" - spot = true - spot_price = 0.0001 - startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\" - - - -} -" -`); + " + + terraform { + required_providers { + iterative = { + source = \\"iterative/iterative\\" + } + } + } + + provider \\"iterative\\" {} + + + resource \\"iterative_cml_runner\\" \\"runner\\" { + repo = \\"https://\\" + token = \\"abc\\" + driver = \\"gitlab\\" + labels = \\"mylabel\\" + idle_timeout = 300 + name = \\"myrunner\\" + single = \\"true\\" + cloud = \\"aws\\" + region = \\"west\\" + instance_type = \\"mymachinetype\\" + instance_gpu = \\"mygputype\\" + instance_hdd_size = 50 + + ssh_private = \\"myprivate\\" + spot = true + spot_price = 0.0001 + startup_script = \\"c3VkbyBlY2hvICdoZWxsbyB3b3JsZCcgPj4gL3Vzci9oZWxsby50eHQ=\\" + + + + + } + " + `); }); }); From acd5ae6c239d14bd737737d853268cd3d6c27e03 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Sun, 16 Jan 2022 11:33:05 +0100 Subject: [PATCH 10/18] update first --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 2cc07f82a..dba38ab53 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,7 +54,7 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && rm --recursive --force /var/lib/apt/lists/* # INSTALL DOCKER -RUN curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh \ +RUN apt update && curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh \ && apt-get clean \ && rm --recursive --force /var/lib/apt/lists/* From 7b9977098ec07300b69b0bc7509cb9ee357bb089 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Sun, 16 Jan 2022 11:34:27 +0100 Subject: [PATCH 11/18] unused --- .gitignore | 1 - get-docker.sh | 630 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 630 insertions(+), 1 deletion(-) create mode 100644 get-docker.sh diff --git a/.gitignore b/.gitignore index 2222db089..e57c16df4 100644 --- a/.gitignore +++ b/.gitignore @@ -10,4 +10,3 @@ terraform.* !terraform.test.js crash.log /build -get-docker.sh diff --git a/get-docker.sh b/get-docker.sh new file mode 100644 index 000000000..209a6da32 --- /dev/null +++ b/get-docker.sh @@ -0,0 +1,630 @@ +#!/bin/sh +set -e +# Docker CE for Linux installation script +# +# See https://docs.docker.com/engine/install/ for the installation steps. +# +# This script is meant for quick & easy install via: +# $ curl -fsSL https://get.docker.com -o get-docker.sh +# $ sh get-docker.sh +# +# For test builds (ie. release candidates): +# $ curl -fsSL https://test.docker.com -o test-docker.sh +# $ sh test-docker.sh +# +# NOTE: Make sure to verify the contents of the script +# you downloaded matches the contents of install.sh +# located at https://github.com/docker/docker-install +# before executing. +# +# Git commit from https://github.com/docker/docker-install when +# the script was uploaded (Should only be modified by upload job): +SCRIPT_COMMIT_SHA="93d2499759296ac1f9c510605fef85052a2c32be" + +# strip "v" prefix if present +VERSION="${VERSION#v}" + +# The channel to install from: +# * nightly +# * test +# * stable +# * edge (deprecated) +DEFAULT_CHANNEL_VALUE="stable" +if [ -z "$CHANNEL" ]; then + CHANNEL=$DEFAULT_CHANNEL_VALUE +fi + +DEFAULT_DOWNLOAD_URL="https://download.docker.com" +if [ -z "$DOWNLOAD_URL" ]; then + DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL +fi + +DEFAULT_REPO_FILE="docker-ce.repo" +if [ -z "$REPO_FILE" ]; then + REPO_FILE="$DEFAULT_REPO_FILE" +fi + +mirror='' +DRY_RUN=${DRY_RUN:-} +while [ $# -gt 0 ]; do + case "$1" in + --mirror) + mirror="$2" + shift + ;; + --dry-run) + DRY_RUN=1 + ;; + --*) + echo "Illegal option $1" + ;; + esac + shift $(( $# > 0 ? 1 : 0 )) +done + +case "$mirror" in + Aliyun) + DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce" + ;; + AzureChinaCloud) + DOWNLOAD_URL="https://mirror.azure.cn/docker-ce" + ;; +esac + +command_exists() { + command -v "$@" > /dev/null 2>&1 +} + +# version_gte checks if the version specified in $VERSION is at least +# the given CalVer (YY.MM) version. returns 0 (success) if $VERSION is either +# unset (=latest) or newer or equal than the specified version. Returns 1 (fail) +# otherwise. +# +# examples: +# +# VERSION=20.10 +# version_gte 20.10 // 0 (success) +# version_gte 19.03 // 0 (success) +# version_gte 21.10 // 1 (fail) +version_gte() { + if [ -z "$VERSION" ]; then + return 0 + fi + eval calver_compare "$VERSION" "$1" +} + +# calver_compare compares two CalVer (YY.MM) version strings. returns 0 (success) +# if version A is newer or equal than version B, or 1 (fail) otherwise. Patch +# releases and pre-release (-alpha/-beta) are not taken into account +# +# examples: +# +# calver_compare 20.10 19.03 // 0 (success) +# calver_compare 20.10 20.10 // 0 (success) +# calver_compare 19.03 20.10 // 1 (fail) +calver_compare() ( + set +x + + yy_a="$(echo "$1" | cut -d'.' -f1)" + yy_b="$(echo "$2" | cut -d'.' -f1)" + if [ "$yy_a" -lt "$yy_b" ]; then + return 1 + fi + if [ "$yy_a" -gt "$yy_b" ]; then + return 0 + fi + mm_a="$(echo "$1" | cut -d'.' -f2)" + mm_b="$(echo "$2" | cut -d'.' -f2)" + if [ "${mm_a#0}" -lt "${mm_b#0}" ]; then + return 1 + fi + + return 0 +) + +is_dry_run() { + if [ -z "$DRY_RUN" ]; then + return 1 + else + return 0 + fi +} + +is_wsl() { + case "$(uname -r)" in + *microsoft* ) true ;; # WSL 2 + *Microsoft* ) true ;; # WSL 1 + * ) false;; + esac +} + +is_darwin() { + case "$(uname -s)" in + *darwin* ) true ;; + *Darwin* ) true ;; + * ) false;; + esac +} + +deprecation_notice() { + distro=$1 + distro_version=$2 + echo + printf "\033[91;1mDEPRECATION WARNING\033[0m\n" + printf " This Linux distribution (\033[1m%s %s\033[0m) reached end-of-life and is no longer supported by this script.\n" "$distro" "$distro_version" + echo " No updates or security fixes will be released for this distribution, and users are recommended" + echo " to upgrade to a currently maintained version of $distro." + echo + printf "Press \033[1mCtrl+C\033[0m now to abort this script, or wait for the installation to continue." + echo + sleep 10 +} + +get_distribution() { + lsb_dist="" + # Every system that we officially support has /etc/os-release + if [ -r /etc/os-release ]; then + lsb_dist="$(. /etc/os-release && echo "$ID")" + fi + # Returning an empty string here should be alright since the + # case statements don't act unless you provide an actual value + echo "$lsb_dist" +} + +echo_docker_as_nonroot() { + if is_dry_run; then + return + fi + if command_exists docker && [ -e /var/run/docker.sock ]; then + ( + set -x + $sh_c 'docker version' + ) || true + fi + + # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output + echo + echo "================================================================================" + echo + if version_gte "20.10"; then + echo "To run Docker as a non-privileged user, consider setting up the" + echo "Docker daemon in rootless mode for your user:" + echo + echo " dockerd-rootless-setuptool.sh install" + echo + echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode." + echo + fi + echo + echo "To run the Docker daemon as a fully privileged service, but granting non-root" + echo "users access, refer to https://docs.docker.com/go/daemon-access/" + echo + echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent" + echo " to root access on the host. Refer to the 'Docker daemon attack surface'" + echo " documentation for details: https://docs.docker.com/go/attack-surface/" + echo + echo "================================================================================" + echo +} + +# Check if this is a forked Linux distro +check_forked() { + + # Check for lsb_release command existence, it usually exists in forked distros + if command_exists lsb_release; then + # Check if the `-u` option is supported + set +e + lsb_release -a -u > /dev/null 2>&1 + lsb_release_exit_code=$? + set -e + + # Check if the command has exited successfully, it means we're in a forked distro + if [ "$lsb_release_exit_code" = "0" ]; then + # Print info about current distro + cat <<-EOF + You're using '$lsb_dist' version '$dist_version'. + EOF + + # Get the upstream release info + lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]') + dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]') + + # Print info about upstream distro + cat <<-EOF + Upstream release is '$lsb_dist' version '$dist_version'. + EOF + else + if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then + if [ "$lsb_dist" = "osmc" ]; then + # OSMC runs Raspbian + lsb_dist=raspbian + else + # We're Debian and don't even know it! + lsb_dist=debian + fi + dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$dist_version" in + 11) + dist_version="bullseye" + ;; + 10) + dist_version="buster" + ;; + 9) + dist_version="stretch" + ;; + 8) + dist_version="jessie" + ;; + esac + fi + fi + fi +} + +do_install() { + echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA" + + if command_exists docker; then + cat >&2 <<-'EOF' + Warning: the "docker" command appears to already exist on this system. + + If you already have Docker installed, this script can cause trouble, which is + why we're displaying this warning and provide the opportunity to cancel the + installation. + + If you installed the current Docker package using this script and are using it + again to update Docker, you can safely ignore this message. + + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 20 ) + fi + + user="$(id -un 2>/dev/null || true)" + + sh_c='sh -c' + if [ "$user" != 'root' ]; then + if command_exists sudo; then + sh_c='sudo -E sh -c' + elif command_exists su; then + sh_c='su -c' + else + cat >&2 <<-'EOF' + Error: this installer needs the ability to run commands as root. + We are unable to find either "sudo" or "su" available to make this happen. + EOF + exit 1 + fi + fi + + if is_dry_run; then + sh_c="echo" + fi + + # perform some very rudimentary platform detection + lsb_dist=$( get_distribution ) + lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" + + if is_wsl; then + echo + echo "WSL DETECTED: We recommend using Docker Desktop for Windows." + echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" + echo + cat >&2 <<-'EOF' + + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 20 ) + fi + + case "$lsb_dist" in + + ubuntu) + if command_exists lsb_release; then + dist_version="$(lsb_release --codename | cut -f2)" + fi + if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then + dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" + fi + ;; + + debian|raspbian) + dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" + case "$dist_version" in + 11) + dist_version="bullseye" + ;; + 10) + dist_version="buster" + ;; + 9) + dist_version="stretch" + ;; + 8) + dist_version="jessie" + ;; + esac + ;; + + centos|rhel|sles) + if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then + dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + + *) + if command_exists lsb_release; then + dist_version="$(lsb_release --release | cut -f2)" + fi + if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then + dist_version="$(. /etc/os-release && echo "$VERSION_ID")" + fi + ;; + + esac + + # Check if this is a forked Linux distro + check_forked + + # Print deprecation warnings for distro versions that recently reached EOL, + # but may still be commonly used (especially LTS versions). + case "$lsb_dist.$dist_version" in + debian.stretch|debian.jessie) + deprecation_notice "$lsb_dist" "$dist_version" + ;; + raspbian.stretch|raspbian.jessie) + deprecation_notice "$lsb_dist" "$dist_version" + ;; + ubuntu.xenial|ubuntu.trusty) + deprecation_notice "$lsb_dist" "$dist_version" + ;; + fedora.*) + if [ "$dist_version" -lt 33 ]; then + deprecation_notice "$lsb_dist" "$dist_version" + fi + ;; + esac + + # Run setup for each distro accordingly + case "$lsb_dist" in + ubuntu|debian|raspbian) + pre_reqs="apt-transport-https ca-certificates curl" + if ! command -v gpg > /dev/null; then + pre_reqs="$pre_reqs gnupg" + fi + apt_repo="deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL" + ( + if ! is_dry_run; then + set -x + fi + $sh_c 'apt-get update -qq >/dev/null' + $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null" + $sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | gpg --dearmor --yes -o /usr/share/keyrings/docker-archive-keyring.gpg" + $sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list" + $sh_c 'apt-get update -qq >/dev/null' + ) + pkg_version="" + if [ -n "$VERSION" ]; then + if is_dry_run; then + echo "# WARNING: VERSION pinning is not supported in DRY_RUN" + else + # Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel + pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist" + search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" + pkg_version="$($sh_c "$search_command")" + echo "INFO: Searching repository for VERSION '$VERSION'" + echo "INFO: $search_command" + if [ -z "$pkg_version" ]; then + echo + echo "ERROR: '$VERSION' not found amongst apt-cache madison results" + echo + exit 1 + fi + if version_gte "18.09"; then + search_command="apt-cache madison 'docker-ce-cli' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" + echo "INFO: $search_command" + cli_pkg_version="=$($sh_c "$search_command")" + fi + pkg_version="=$pkg_version" + fi + fi + ( + pkgs="" + if version_gte "18.09"; then + # older versions don't support a cli package + pkgs="$pkgs docker-ce-cli${cli_pkg_version%=}" + fi + if version_gte "20.10" && [ "$(uname -m)" = "x86_64" ]; then + # also install the latest version of the "docker scan" cli-plugin (only supported on x86 currently) + pkgs="$pkgs docker-scan-plugin" + fi + pkgs="$pkgs docker-ce${pkg_version%=}" + if ! is_dry_run; then + set -x + fi + $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $pkgs >/dev/null" + if version_gte "20.10"; then + # Install docker-ce-rootless-extras without "--no-install-recommends", so as to install slirp4netns when available + $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce-rootless-extras${pkg_version%=} >/dev/null" + fi + ) + echo_docker_as_nonroot + exit 0 + ;; + centos|fedora|rhel) + if [ "$(uname -m)" != "s390x" ] && [ "$lsb_dist" = "rhel" ]; then + echo "Packages for RHEL are currently only available for s390x." + exit 1 + fi + yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" + if ! curl -Ifs "$yum_repo" > /dev/null; then + echo "Error: Unable to curl repository file $yum_repo, is it valid?" + exit 1 + fi + if [ "$lsb_dist" = "fedora" ]; then + pkg_manager="dnf" + config_manager="dnf config-manager" + enable_channel_flag="--set-enabled" + disable_channel_flag="--set-disabled" + pre_reqs="dnf-plugins-core" + pkg_suffix="fc$dist_version" + else + pkg_manager="yum" + config_manager="yum-config-manager" + enable_channel_flag="--enable" + disable_channel_flag="--disable" + pre_reqs="yum-utils" + pkg_suffix="el" + fi + ( + if ! is_dry_run; then + set -x + fi + $sh_c "$pkg_manager install -y -q $pre_reqs" + $sh_c "$config_manager --add-repo $yum_repo" + + if [ "$CHANNEL" != "stable" ]; then + $sh_c "$config_manager $disable_channel_flag docker-ce-*" + $sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL" + fi + $sh_c "$pkg_manager makecache" + ) + pkg_version="" + if [ -n "$VERSION" ]; then + if is_dry_run; then + echo "# WARNING: VERSION pinning is not supported in DRY_RUN" + else + pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix" + search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" + pkg_version="$($sh_c "$search_command")" + echo "INFO: Searching repository for VERSION '$VERSION'" + echo "INFO: $search_command" + if [ -z "$pkg_version" ]; then + echo + echo "ERROR: '$VERSION' not found amongst $pkg_manager list results" + echo + exit 1 + fi + if version_gte "18.09"; then + # older versions don't support a cli package + search_command="$pkg_manager list --showduplicates 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" + cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)" + fi + # Cut out the epoch and prefix with a '-' + pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)" + fi + fi + ( + if ! is_dry_run; then + set -x + fi + # install the correct cli version first + if [ -n "$cli_pkg_version" ]; then + $sh_c "$pkg_manager install -y -q docker-ce-cli-$cli_pkg_version" + fi + $sh_c "$pkg_manager install -y -q docker-ce$pkg_version" + if version_gte "20.10"; then + $sh_c "$pkg_manager install -y -q docker-ce-rootless-extras$pkg_version" + fi + ) + echo_docker_as_nonroot + exit 0 + ;; + sles) + if [ "$(uname -m)" != "s390x" ]; then + echo "Packages for SLES are currently only available for s390x" + exit 1 + fi + + sles_version="${dist_version##*.}" + sles_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" + opensuse_repo="https://download.opensuse.org/repositories/security:SELinux/SLE_15_SP$sles_version/security:SELinux.repo" + if ! curl -Ifs "$sles_repo" > /dev/null; then + echo "Error: Unable to curl repository file $sles_repo, is it valid?" + exit 1 + fi + pre_reqs="ca-certificates curl libseccomp2 awk" + ( + if ! is_dry_run; then + set -x + fi + $sh_c "zypper install -y $pre_reqs" + $sh_c "zypper addrepo $sles_repo" + if ! is_dry_run; then + cat >&2 <<-'EOF' + WARNING!! + openSUSE repository (https://download.opensuse.org/repositories/security:SELinux) will be enabled now. + Do you wish to continue? + You may press Ctrl+C now to abort this script. + EOF + ( set -x; sleep 30 ) + fi + $sh_c "zypper addrepo $opensuse_repo" + $sh_c "zypper --gpg-auto-import-keys refresh" + $sh_c "zypper lr -d" + ) + pkg_version="" + if [ -n "$VERSION" ]; then + if is_dry_run; then + echo "# WARNING: VERSION pinning is not supported in DRY_RUN" + else + pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g")" + search_command="zypper search -s --match-exact 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" + pkg_version="$($sh_c "$search_command")" + echo "INFO: Searching repository for VERSION '$VERSION'" + echo "INFO: $search_command" + if [ -z "$pkg_version" ]; then + echo + echo "ERROR: '$VERSION' not found amongst zypper list results" + echo + exit 1 + fi + search_command="zypper search -s --match-exact 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" + # It's okay for cli_pkg_version to be blank, since older versions don't support a cli package + cli_pkg_version="$($sh_c "$search_command")" + pkg_version="-$pkg_version" + + search_command="zypper search -s --match-exact 'docker-ce-rootless-extras' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" + rootless_pkg_version="$($sh_c "$search_command")" + rootless_pkg_version="-$rootless_pkg_version" + fi + fi + ( + if ! is_dry_run; then + set -x + fi + # install the correct cli version first + if [ -n "$cli_pkg_version" ]; then + $sh_c "zypper install -y docker-ce-cli-$cli_pkg_version" + fi + $sh_c "zypper install -y docker-ce$pkg_version" + if version_gte "20.10"; then + $sh_c "zypper install -y docker-ce-rootless-extras$rootless_pkg_version" + fi + ) + echo_docker_as_nonroot + exit 0 + ;; + *) + if [ -z "$lsb_dist" ]; then + if is_darwin; then + echo + echo "ERROR: Unsupported operating system 'macOS'" + echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" + echo + exit 1 + fi + fi + echo + echo "ERROR: Unsupported distribution '$lsb_dist'" + echo + exit 1 + ;; + esac + exit 1 +} + +# wrapped up in a function so that we have some protection against only getting +# half the file during "curl | sh" +do_install From df1ba1f4f372181a861185b20c25a4da2f7c2eb8 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Sun, 16 Jan 2022 11:35:45 +0100 Subject: [PATCH 12/18] platform --- src/drivers/gitlab.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/drivers/gitlab.js b/src/drivers/gitlab.js index 5cba8f732..41043792c 100644 --- a/src/drivers/gitlab.js +++ b/src/drivers/gitlab.js @@ -185,7 +185,7 @@ class Gitlab { try { const bin = resolve(workdir, 'gitlab-runner'); if (!(await fse.pathExists(bin))) { - const arch = process.platform === 'darwin' ? 'osx-x64' : 'linux-x64'; + const arch = process.platform === 'darwin' ? 'darwin' : 'linux'; const url = `https://gitlab-runner-downloads.s3.amazonaws.com/latest/binaries/gitlab-runner-${arch}-amd64`; await download({ url, path: bin }); await fs.chmod(bin, '777'); From a79926bfae8fe078dc6e937d385cfb8c16f708f1 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Mon, 17 Jan 2022 10:14:08 +0100 Subject: [PATCH 13/18] no-docker --- get-docker.sh | 630 -------------------------------------------------- 1 file changed, 630 deletions(-) delete mode 100644 get-docker.sh diff --git a/get-docker.sh b/get-docker.sh deleted file mode 100644 index 209a6da32..000000000 --- a/get-docker.sh +++ /dev/null @@ -1,630 +0,0 @@ -#!/bin/sh -set -e -# Docker CE for Linux installation script -# -# See https://docs.docker.com/engine/install/ for the installation steps. -# -# This script is meant for quick & easy install via: -# $ curl -fsSL https://get.docker.com -o get-docker.sh -# $ sh get-docker.sh -# -# For test builds (ie. release candidates): -# $ curl -fsSL https://test.docker.com -o test-docker.sh -# $ sh test-docker.sh -# -# NOTE: Make sure to verify the contents of the script -# you downloaded matches the contents of install.sh -# located at https://github.com/docker/docker-install -# before executing. -# -# Git commit from https://github.com/docker/docker-install when -# the script was uploaded (Should only be modified by upload job): -SCRIPT_COMMIT_SHA="93d2499759296ac1f9c510605fef85052a2c32be" - -# strip "v" prefix if present -VERSION="${VERSION#v}" - -# The channel to install from: -# * nightly -# * test -# * stable -# * edge (deprecated) -DEFAULT_CHANNEL_VALUE="stable" -if [ -z "$CHANNEL" ]; then - CHANNEL=$DEFAULT_CHANNEL_VALUE -fi - -DEFAULT_DOWNLOAD_URL="https://download.docker.com" -if [ -z "$DOWNLOAD_URL" ]; then - DOWNLOAD_URL=$DEFAULT_DOWNLOAD_URL -fi - -DEFAULT_REPO_FILE="docker-ce.repo" -if [ -z "$REPO_FILE" ]; then - REPO_FILE="$DEFAULT_REPO_FILE" -fi - -mirror='' -DRY_RUN=${DRY_RUN:-} -while [ $# -gt 0 ]; do - case "$1" in - --mirror) - mirror="$2" - shift - ;; - --dry-run) - DRY_RUN=1 - ;; - --*) - echo "Illegal option $1" - ;; - esac - shift $(( $# > 0 ? 1 : 0 )) -done - -case "$mirror" in - Aliyun) - DOWNLOAD_URL="https://mirrors.aliyun.com/docker-ce" - ;; - AzureChinaCloud) - DOWNLOAD_URL="https://mirror.azure.cn/docker-ce" - ;; -esac - -command_exists() { - command -v "$@" > /dev/null 2>&1 -} - -# version_gte checks if the version specified in $VERSION is at least -# the given CalVer (YY.MM) version. returns 0 (success) if $VERSION is either -# unset (=latest) or newer or equal than the specified version. Returns 1 (fail) -# otherwise. -# -# examples: -# -# VERSION=20.10 -# version_gte 20.10 // 0 (success) -# version_gte 19.03 // 0 (success) -# version_gte 21.10 // 1 (fail) -version_gte() { - if [ -z "$VERSION" ]; then - return 0 - fi - eval calver_compare "$VERSION" "$1" -} - -# calver_compare compares two CalVer (YY.MM) version strings. returns 0 (success) -# if version A is newer or equal than version B, or 1 (fail) otherwise. Patch -# releases and pre-release (-alpha/-beta) are not taken into account -# -# examples: -# -# calver_compare 20.10 19.03 // 0 (success) -# calver_compare 20.10 20.10 // 0 (success) -# calver_compare 19.03 20.10 // 1 (fail) -calver_compare() ( - set +x - - yy_a="$(echo "$1" | cut -d'.' -f1)" - yy_b="$(echo "$2" | cut -d'.' -f1)" - if [ "$yy_a" -lt "$yy_b" ]; then - return 1 - fi - if [ "$yy_a" -gt "$yy_b" ]; then - return 0 - fi - mm_a="$(echo "$1" | cut -d'.' -f2)" - mm_b="$(echo "$2" | cut -d'.' -f2)" - if [ "${mm_a#0}" -lt "${mm_b#0}" ]; then - return 1 - fi - - return 0 -) - -is_dry_run() { - if [ -z "$DRY_RUN" ]; then - return 1 - else - return 0 - fi -} - -is_wsl() { - case "$(uname -r)" in - *microsoft* ) true ;; # WSL 2 - *Microsoft* ) true ;; # WSL 1 - * ) false;; - esac -} - -is_darwin() { - case "$(uname -s)" in - *darwin* ) true ;; - *Darwin* ) true ;; - * ) false;; - esac -} - -deprecation_notice() { - distro=$1 - distro_version=$2 - echo - printf "\033[91;1mDEPRECATION WARNING\033[0m\n" - printf " This Linux distribution (\033[1m%s %s\033[0m) reached end-of-life and is no longer supported by this script.\n" "$distro" "$distro_version" - echo " No updates or security fixes will be released for this distribution, and users are recommended" - echo " to upgrade to a currently maintained version of $distro." - echo - printf "Press \033[1mCtrl+C\033[0m now to abort this script, or wait for the installation to continue." - echo - sleep 10 -} - -get_distribution() { - lsb_dist="" - # Every system that we officially support has /etc/os-release - if [ -r /etc/os-release ]; then - lsb_dist="$(. /etc/os-release && echo "$ID")" - fi - # Returning an empty string here should be alright since the - # case statements don't act unless you provide an actual value - echo "$lsb_dist" -} - -echo_docker_as_nonroot() { - if is_dry_run; then - return - fi - if command_exists docker && [ -e /var/run/docker.sock ]; then - ( - set -x - $sh_c 'docker version' - ) || true - fi - - # intentionally mixed spaces and tabs here -- tabs are stripped by "<<-EOF", spaces are kept in the output - echo - echo "================================================================================" - echo - if version_gte "20.10"; then - echo "To run Docker as a non-privileged user, consider setting up the" - echo "Docker daemon in rootless mode for your user:" - echo - echo " dockerd-rootless-setuptool.sh install" - echo - echo "Visit https://docs.docker.com/go/rootless/ to learn about rootless mode." - echo - fi - echo - echo "To run the Docker daemon as a fully privileged service, but granting non-root" - echo "users access, refer to https://docs.docker.com/go/daemon-access/" - echo - echo "WARNING: Access to the remote API on a privileged Docker daemon is equivalent" - echo " to root access on the host. Refer to the 'Docker daemon attack surface'" - echo " documentation for details: https://docs.docker.com/go/attack-surface/" - echo - echo "================================================================================" - echo -} - -# Check if this is a forked Linux distro -check_forked() { - - # Check for lsb_release command existence, it usually exists in forked distros - if command_exists lsb_release; then - # Check if the `-u` option is supported - set +e - lsb_release -a -u > /dev/null 2>&1 - lsb_release_exit_code=$? - set -e - - # Check if the command has exited successfully, it means we're in a forked distro - if [ "$lsb_release_exit_code" = "0" ]; then - # Print info about current distro - cat <<-EOF - You're using '$lsb_dist' version '$dist_version'. - EOF - - # Get the upstream release info - lsb_dist=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'id' | cut -d ':' -f 2 | tr -d '[:space:]') - dist_version=$(lsb_release -a -u 2>&1 | tr '[:upper:]' '[:lower:]' | grep -E 'codename' | cut -d ':' -f 2 | tr -d '[:space:]') - - # Print info about upstream distro - cat <<-EOF - Upstream release is '$lsb_dist' version '$dist_version'. - EOF - else - if [ -r /etc/debian_version ] && [ "$lsb_dist" != "ubuntu" ] && [ "$lsb_dist" != "raspbian" ]; then - if [ "$lsb_dist" = "osmc" ]; then - # OSMC runs Raspbian - lsb_dist=raspbian - else - # We're Debian and don't even know it! - lsb_dist=debian - fi - dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" - case "$dist_version" in - 11) - dist_version="bullseye" - ;; - 10) - dist_version="buster" - ;; - 9) - dist_version="stretch" - ;; - 8) - dist_version="jessie" - ;; - esac - fi - fi - fi -} - -do_install() { - echo "# Executing docker install script, commit: $SCRIPT_COMMIT_SHA" - - if command_exists docker; then - cat >&2 <<-'EOF' - Warning: the "docker" command appears to already exist on this system. - - If you already have Docker installed, this script can cause trouble, which is - why we're displaying this warning and provide the opportunity to cancel the - installation. - - If you installed the current Docker package using this script and are using it - again to update Docker, you can safely ignore this message. - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - user="$(id -un 2>/dev/null || true)" - - sh_c='sh -c' - if [ "$user" != 'root' ]; then - if command_exists sudo; then - sh_c='sudo -E sh -c' - elif command_exists su; then - sh_c='su -c' - else - cat >&2 <<-'EOF' - Error: this installer needs the ability to run commands as root. - We are unable to find either "sudo" or "su" available to make this happen. - EOF - exit 1 - fi - fi - - if is_dry_run; then - sh_c="echo" - fi - - # perform some very rudimentary platform detection - lsb_dist=$( get_distribution ) - lsb_dist="$(echo "$lsb_dist" | tr '[:upper:]' '[:lower:]')" - - if is_wsl; then - echo - echo "WSL DETECTED: We recommend using Docker Desktop for Windows." - echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" - echo - cat >&2 <<-'EOF' - - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 20 ) - fi - - case "$lsb_dist" in - - ubuntu) - if command_exists lsb_release; then - dist_version="$(lsb_release --codename | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/lsb-release ]; then - dist_version="$(. /etc/lsb-release && echo "$DISTRIB_CODENAME")" - fi - ;; - - debian|raspbian) - dist_version="$(sed 's/\/.*//' /etc/debian_version | sed 's/\..*//')" - case "$dist_version" in - 11) - dist_version="bullseye" - ;; - 10) - dist_version="buster" - ;; - 9) - dist_version="stretch" - ;; - 8) - dist_version="jessie" - ;; - esac - ;; - - centos|rhel|sles) - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - *) - if command_exists lsb_release; then - dist_version="$(lsb_release --release | cut -f2)" - fi - if [ -z "$dist_version" ] && [ -r /etc/os-release ]; then - dist_version="$(. /etc/os-release && echo "$VERSION_ID")" - fi - ;; - - esac - - # Check if this is a forked Linux distro - check_forked - - # Print deprecation warnings for distro versions that recently reached EOL, - # but may still be commonly used (especially LTS versions). - case "$lsb_dist.$dist_version" in - debian.stretch|debian.jessie) - deprecation_notice "$lsb_dist" "$dist_version" - ;; - raspbian.stretch|raspbian.jessie) - deprecation_notice "$lsb_dist" "$dist_version" - ;; - ubuntu.xenial|ubuntu.trusty) - deprecation_notice "$lsb_dist" "$dist_version" - ;; - fedora.*) - if [ "$dist_version" -lt 33 ]; then - deprecation_notice "$lsb_dist" "$dist_version" - fi - ;; - esac - - # Run setup for each distro accordingly - case "$lsb_dist" in - ubuntu|debian|raspbian) - pre_reqs="apt-transport-https ca-certificates curl" - if ! command -v gpg > /dev/null; then - pre_reqs="$pre_reqs gnupg" - fi - apt_repo="deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] $DOWNLOAD_URL/linux/$lsb_dist $dist_version $CHANNEL" - ( - if ! is_dry_run; then - set -x - fi - $sh_c 'apt-get update -qq >/dev/null' - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq $pre_reqs >/dev/null" - $sh_c "curl -fsSL \"$DOWNLOAD_URL/linux/$lsb_dist/gpg\" | gpg --dearmor --yes -o /usr/share/keyrings/docker-archive-keyring.gpg" - $sh_c "echo \"$apt_repo\" > /etc/apt/sources.list.d/docker.list" - $sh_c 'apt-get update -qq >/dev/null' - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - # Will work for incomplete versions IE (17.12), but may not actually grab the "latest" if in the test channel - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/~ce~.*/g" | sed "s/-/.*/g").*-0~$lsb_dist" - search_command="apt-cache madison 'docker-ce' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst apt-cache madison results" - echo - exit 1 - fi - if version_gte "18.09"; then - search_command="apt-cache madison 'docker-ce-cli' | grep '$pkg_pattern' | head -1 | awk '{\$1=\$1};1' | cut -d' ' -f 3" - echo "INFO: $search_command" - cli_pkg_version="=$($sh_c "$search_command")" - fi - pkg_version="=$pkg_version" - fi - fi - ( - pkgs="" - if version_gte "18.09"; then - # older versions don't support a cli package - pkgs="$pkgs docker-ce-cli${cli_pkg_version%=}" - fi - if version_gte "20.10" && [ "$(uname -m)" = "x86_64" ]; then - # also install the latest version of the "docker scan" cli-plugin (only supported on x86 currently) - pkgs="$pkgs docker-scan-plugin" - fi - pkgs="$pkgs docker-ce${pkg_version%=}" - if ! is_dry_run; then - set -x - fi - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq --no-install-recommends $pkgs >/dev/null" - if version_gte "20.10"; then - # Install docker-ce-rootless-extras without "--no-install-recommends", so as to install slirp4netns when available - $sh_c "DEBIAN_FRONTEND=noninteractive apt-get install -y -qq docker-ce-rootless-extras${pkg_version%=} >/dev/null" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - centos|fedora|rhel) - if [ "$(uname -m)" != "s390x" ] && [ "$lsb_dist" = "rhel" ]; then - echo "Packages for RHEL are currently only available for s390x." - exit 1 - fi - yum_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" - if ! curl -Ifs "$yum_repo" > /dev/null; then - echo "Error: Unable to curl repository file $yum_repo, is it valid?" - exit 1 - fi - if [ "$lsb_dist" = "fedora" ]; then - pkg_manager="dnf" - config_manager="dnf config-manager" - enable_channel_flag="--set-enabled" - disable_channel_flag="--set-disabled" - pre_reqs="dnf-plugins-core" - pkg_suffix="fc$dist_version" - else - pkg_manager="yum" - config_manager="yum-config-manager" - enable_channel_flag="--enable" - disable_channel_flag="--disable" - pre_reqs="yum-utils" - pkg_suffix="el" - fi - ( - if ! is_dry_run; then - set -x - fi - $sh_c "$pkg_manager install -y -q $pre_reqs" - $sh_c "$config_manager --add-repo $yum_repo" - - if [ "$CHANNEL" != "stable" ]; then - $sh_c "$config_manager $disable_channel_flag docker-ce-*" - $sh_c "$config_manager $enable_channel_flag docker-ce-$CHANNEL" - fi - $sh_c "$pkg_manager makecache" - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g").*$pkg_suffix" - search_command="$pkg_manager list --showduplicates 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst $pkg_manager list results" - echo - exit 1 - fi - if version_gte "18.09"; then - # older versions don't support a cli package - search_command="$pkg_manager list --showduplicates 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$2}'" - cli_pkg_version="$($sh_c "$search_command" | cut -d':' -f 2)" - fi - # Cut out the epoch and prefix with a '-' - pkg_version="-$(echo "$pkg_version" | cut -d':' -f 2)" - fi - fi - ( - if ! is_dry_run; then - set -x - fi - # install the correct cli version first - if [ -n "$cli_pkg_version" ]; then - $sh_c "$pkg_manager install -y -q docker-ce-cli-$cli_pkg_version" - fi - $sh_c "$pkg_manager install -y -q docker-ce$pkg_version" - if version_gte "20.10"; then - $sh_c "$pkg_manager install -y -q docker-ce-rootless-extras$pkg_version" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - sles) - if [ "$(uname -m)" != "s390x" ]; then - echo "Packages for SLES are currently only available for s390x" - exit 1 - fi - - sles_version="${dist_version##*.}" - sles_repo="$DOWNLOAD_URL/linux/$lsb_dist/$REPO_FILE" - opensuse_repo="https://download.opensuse.org/repositories/security:SELinux/SLE_15_SP$sles_version/security:SELinux.repo" - if ! curl -Ifs "$sles_repo" > /dev/null; then - echo "Error: Unable to curl repository file $sles_repo, is it valid?" - exit 1 - fi - pre_reqs="ca-certificates curl libseccomp2 awk" - ( - if ! is_dry_run; then - set -x - fi - $sh_c "zypper install -y $pre_reqs" - $sh_c "zypper addrepo $sles_repo" - if ! is_dry_run; then - cat >&2 <<-'EOF' - WARNING!! - openSUSE repository (https://download.opensuse.org/repositories/security:SELinux) will be enabled now. - Do you wish to continue? - You may press Ctrl+C now to abort this script. - EOF - ( set -x; sleep 30 ) - fi - $sh_c "zypper addrepo $opensuse_repo" - $sh_c "zypper --gpg-auto-import-keys refresh" - $sh_c "zypper lr -d" - ) - pkg_version="" - if [ -n "$VERSION" ]; then - if is_dry_run; then - echo "# WARNING: VERSION pinning is not supported in DRY_RUN" - else - pkg_pattern="$(echo "$VERSION" | sed "s/-ce-/\\\\.ce.*/g" | sed "s/-/.*/g")" - search_command="zypper search -s --match-exact 'docker-ce' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" - pkg_version="$($sh_c "$search_command")" - echo "INFO: Searching repository for VERSION '$VERSION'" - echo "INFO: $search_command" - if [ -z "$pkg_version" ]; then - echo - echo "ERROR: '$VERSION' not found amongst zypper list results" - echo - exit 1 - fi - search_command="zypper search -s --match-exact 'docker-ce-cli' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" - # It's okay for cli_pkg_version to be blank, since older versions don't support a cli package - cli_pkg_version="$($sh_c "$search_command")" - pkg_version="-$pkg_version" - - search_command="zypper search -s --match-exact 'docker-ce-rootless-extras' | grep '$pkg_pattern' | tail -1 | awk '{print \$6}'" - rootless_pkg_version="$($sh_c "$search_command")" - rootless_pkg_version="-$rootless_pkg_version" - fi - fi - ( - if ! is_dry_run; then - set -x - fi - # install the correct cli version first - if [ -n "$cli_pkg_version" ]; then - $sh_c "zypper install -y docker-ce-cli-$cli_pkg_version" - fi - $sh_c "zypper install -y docker-ce$pkg_version" - if version_gte "20.10"; then - $sh_c "zypper install -y docker-ce-rootless-extras$rootless_pkg_version" - fi - ) - echo_docker_as_nonroot - exit 0 - ;; - *) - if [ -z "$lsb_dist" ]; then - if is_darwin; then - echo - echo "ERROR: Unsupported operating system 'macOS'" - echo "Please get Docker Desktop from https://www.docker.com/products/docker-desktop" - echo - exit 1 - fi - fi - echo - echo "ERROR: Unsupported distribution '$lsb_dist'" - echo - exit 1 - ;; - esac - exit 1 -} - -# wrapped up in a function so that we have some protection against only getting -# half the file during "curl | sh" -do_install From 17afa5ea58d682279e1218955cd3094394ce7189 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Mon, 17 Jan 2022 10:17:14 +0100 Subject: [PATCH 14/18] GitLab vs Gitlab Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- bin/cml/runner.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/cml/runner.js b/bin/cml/runner.js index 825278816..bd5ecb164 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -419,7 +419,7 @@ exports.builder = (yargs) => dockerVolumes: { type: 'array', default: [], - description: 'Docker volumes. This feature is only supported in Gitlab' + description: 'Docker volumes. This feature is only supported in GitLab' }, labels: { type: 'string', From 8e34536e5d4ceb4f466745dc3c8f7523f8f49a07 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Mon, 17 Jan 2022 11:38:40 +0100 Subject: [PATCH 15/18] tests --- bin/cml/runner.test.js | 2 +- package-lock.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/cml/runner.test.js b/bin/cml/runner.test.js index 47270fdbe..b40dc41e7 100644 --- a/bin/cml/runner.test.js +++ b/bin/cml/runner.test.js @@ -15,7 +15,7 @@ describe('CML e2e', () => { --log Maximum log level [string] [choices: \\"error\\", \\"warn\\", \\"info\\", \\"debug\\"] [default: \\"info\\"] --docker-volumes Docker volumes. This feature is only supported in - Gitlab [array] [default: []] + GitLab [array] [default: []] --labels One or more user-defined labels for this runner (delimited with commas) [string] [default: \\"cml\\"] --idle-timeout Seconds to wait for jobs before shutting down. Set diff --git a/package-lock.json b/package-lock.json index e71495425..1004fcf4c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,6 +15,7 @@ "@octokit/core": "^3.5.1", "@octokit/plugin-throttling": "^3.5.2", "@octokit/rest": "18.0.0", + "colors": "1.4.0", "ec2-spot-notification": "^2.0.3", "form-data": "^3.0.1", "fs-extra": "^9.1.0", From 130ff1cfb9eea864ac278de240d8a95e64f03bca Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Mon, 17 Jan 2022 17:34:15 +0100 Subject: [PATCH 16/18] remove docker file --- Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index dba38ab53..ec4576fef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -56,7 +56,9 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - # INSTALL DOCKER RUN apt update && curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh \ && apt-get clean \ - && rm --recursive --force /var/lib/apt/lists/* + && rm --recursive --force /var/lib/apt/lists/* \ + && rm get-docker.sh + # INSTALL PYTHON ARG PYTHON_VERSION=3 From ac9ff91e1239e547afea34070b59092dd7aaf761 Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Wed, 19 Jan 2022 15:59:23 +0100 Subject: [PATCH 17/18] warn for not gl users --- bin/cml/runner.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bin/cml/runner.js b/bin/cml/runner.js index bd5ecb164..545e6fef8 100755 --- a/bin/cml/runner.js +++ b/bin/cml/runner.js @@ -335,12 +335,16 @@ const run = async (opts) => { labels, name, reuse, + dockerVolumes, tfResource, workdir } = opts; cml = new CML({ driver, repo, token }); + if (dockerVolumes.length && cml.driver !== 'gitlab') + winston.warn('Parameters --docker-volumes is only supported in gitlab'); + if (cloud || tfResource) await tf.checkMinVersion(); // prepare tf From 4def4c99ff2a3cfb59aac14262ea565d17f766dc Mon Sep 17 00:00:00 2001 From: DavidGOrtega Date: Fri, 21 Jan 2022 10:20:40 +0100 Subject: [PATCH 18/18] remove docker Co-authored-by: Helio Machado <0x2b3bfa0+git@googlemail.com> --- Dockerfile | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5e4190622..f17f135e1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -53,13 +53,6 @@ RUN curl --location https://apt.releases.hashicorp.com/gpg | sudo apt-key add - && apt-get clean \ && rm --recursive --force /var/lib/apt/lists/* -# INSTALL DOCKER -RUN apt update && curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh \ - && apt-get clean \ - && rm --recursive --force /var/lib/apt/lists/* \ - && rm get-docker.sh - - # INSTALL PYTHON ARG PYTHON_VERSION=3 RUN add-apt-repository universe --yes \