From 2e7d93c9b70acd6cf6bf969025b1a175c7518144 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 17 Mar 2023 15:15:49 -0500 Subject: [PATCH] run the release microk8s job from within a lxc on the jenkins worker --- jobs/release-microk8s/release-microk8s.groovy | 116 +++++++++++++----- 1 file changed, 82 insertions(+), 34 deletions(-) diff --git a/jobs/release-microk8s/release-microk8s.groovy b/jobs/release-microk8s/release-microk8s.groovy index 0b8576edf..bde853859 100644 --- a/jobs/release-microk8s/release-microk8s.groovy +++ b/jobs/release-microk8s/release-microk8s.groovy @@ -14,6 +14,43 @@ def destroy_controller(controller) { """ } +def setup_lxc() { + return """#!/usr/bin/env bash + . \$WORKSPACE/lxc_runner.sh + ci_lxc_init_runner _lxc notrap 1>&2 + echo \${_lxc} + """ +} + +def delete_lxc(container_name) { + return """#!/usr/bin/env bash + . \${WORKSPACE}/cilib.sh + + ci_lxc_delete ${container_name} + """ +} + +def run_lxc(container_name, command) { + return """#!/usr/bin/env bash + . \$WORKSPACE/lxc_runner.sh + + _job_sh=\$(basename \$(mktemp -p \$WORKSPACE --suffix=.sh)) + + # push job script into workspace and run + cat << 'EOF' > \$WORKSPACE/\$_job_sh + + #!/bin/bash + set -eux + set -o allexport + [[ -f \$WORKSPACE/.env ]] && source \$WORKSPACE/.env + set +o allexport + + ${command} + EOF + ci_lxc_job_run ${container_name} -- bash \$LXC_WORKSPACE/\$_job_sh + """ +} + pipeline { agent { label "runner-${params.ARCH}" @@ -42,6 +79,10 @@ pipeline { K8STEAMCI_GPG_PRIVATE= credentials('deb-gpg-private') K8STEAMCI_GPG_KEY = credentials('deb-gpg-key') NOTIFY_EMAIL = credentials('microk8s_notify_email') + LXC_APT_LIST = "awscli,pip,python3-venv,python3-dev,libffi-dev" + LXC_SNAP_LIST = "juju --channel=3.1,juju-wait,juju-crashdump,snapcraft" + LXC_PUSH_LIST = "${SNAPCRAFTCREDS}" + LXC_MOUNT_LIST = ".local,.ssh,.aws,.config" } options { @@ -60,6 +101,11 @@ pipeline { } } } + stage("Setup LXC") { + steps { + lxc_name = sh (script: setup_lxc, returnStdout: true).trim() + } + } stage("Snapcraft login") { steps { sh "snapcraft login --with ${SNAPCRAFTCREDS}" @@ -67,9 +113,9 @@ pipeline { } stage("Setup tox environment") { steps { - sh """ - tox -e py38 -- python -c 'print("Tox Environment Ready")' - """ + sh run_lxc(lxc_name, """ + tox -e py -- python -c 'print("Tox Environment Ready")' + """) } } stage("Run Release Steps") { @@ -93,13 +139,13 @@ pipeline { ] try { - sh """ - . .tox/py38/bin/activate + sh run_lxc(lxc_name, """ + . .tox/py/bin/activate ALWAYS_RELEASE=${params.ALWAYS_RELEASE}\ TRACKS=${params.TRACKS}\ CHANNEL=${channel}\ timeout 6h python jobs/microk8s/release-needed.py - """ + """) } catch (err) { return 0 } @@ -117,47 +163,48 @@ pipeline { } else { error("Aborting build due to unknown arch=${arch}") } - sh destroy_controller(juju_controller) - sh """#!/bin/bash -x - juju bootstrap "${JUJU_CLOUD}" "${juju_controller}" \ - -d "${juju_model}" \ + sh run_lxc(lxc_name, destroy_controller(juju_controller)) + sh run_lxc(lxc_name, """ + juju bootstrap "${JUJU_CLOUD}" "${juju_controller}" \ + --add-model "${juju_model}" \ --model-default test-mode=true \ --model-default resource-tags="owner=k8sci job=${job} stage=${stage}" \ --bootstrap-constraints "mem=8G cores=2" - juju deploy -m "${juju_full_model}" --constraints "${constraints}" ubuntu + juju deploy -m "${juju_full_model}" --constraints "${constraints}" ubuntu + juju-wait -e "${juju_full_model}" -w - juju-wait -e "${juju_full_model}" -w + set +x + AWS_ACCESS_KEY_ID=\$(aws configure get aws_access_key_id) + AWS_SECRET_ACCESS_KEY=\$(aws configure get aws_secret_access_key) - set +x - AWS_ACCESS_KEY_ID=\$(aws configure get aws_access_key_id) - AWS_SECRET_ACCESS_KEY=\$(aws configure get aws_secret_access_key) + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo INSTANCE_TYPE=${eksd_instance_type} | sudo tee -a /etc/environment" + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo STACK_NAME=${juju_controller} | sudo tee -a /etc/environment" + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo AWS_REGION=\$AWS_REGION | sudo tee -a /etc/environment" + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo AWS_ACCESS_KEY_ID=\$AWS_ACCESS_KEY_ID | sudo tee -a /etc/environment" + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo AWS_SECRET_ACCESS_KEY=\$AWS_SECRET_ACCESS_KEY | sudo tee -a /etc/environment" + set -x - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo INSTANCE_TYPE=${eksd_instance_type} | sudo tee -a /etc/environment" - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo STACK_NAME=${juju_controller} | sudo tee -a /etc/environment" - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo AWS_REGION=\$AWS_REGION | sudo tee -a /etc/environment" - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo AWS_ACCESS_KEY_ID=\$AWS_ACCESS_KEY_ID | sudo tee -a /etc/environment" - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- "sudo echo AWS_SECRET_ACCESS_KEY=\$AWS_SECRET_ACCESS_KEY | sudo tee -a /etc/environment" - set -x - - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo snap install lxd' - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo lxd.migrate -yes' || true - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo lxd init --auto' - """ - if (channel == "pre-release"){ - sh """ - juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo snap install snapcraft --classic' + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo snap install lxd' + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo lxd.migrate -yes' || true + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo lxd init --auto' """ + ) + if (channel == "pre-release"){ + sh run_lxc(lxc_name, """ + juju ssh -m "${juju_full_model}" --pty=true ubuntu/0 -- 'sudo snap install snapcraft --classic' + """) } + try { - sh """ - . .tox/py38/bin/activate + sh run_lxc(lxc_name, """ + . .tox/py/bin/activate DRY_RUN=${params.DRY_RUN} ALWAYS_RELEASE=${params.ALWAYS_RELEASE}\ TESTS_BRANCH=${params.TESTS_BRANCH} TRACKS=${params.TRACKS}\ PROXY=${params.PROXY} JUJU_UNIT=ubuntu/0\ JUJU_CONTROLLER=${juju_controller} JUJU_MODEL=${juju_model}\ timeout 6h python jobs/microk8s/${job_name[channel]} - """ + """) } catch (err) { unstable("${job_name[channel]} completed with errors.") emailext( @@ -166,7 +213,7 @@ pipeline { body: "Please go to ${BUILD_URL} and verify the build" ) } finally { - sh destroy_controller(juju_controller) + sh run_lxc(lxc_name, destroy_controller(juju_controller)) } } } @@ -183,9 +230,10 @@ pipeline { def job="release-microk8s" def stage="${channel}-${arch}" def juju_controller="${job}-${stage}" - sh destroy_controller(juju_controller) + sh run_lxc(lxc_name, destroy_controller(juju_controller)) } } + sh delete_lxc(lxc_name) } } }