chore: separated updatecli pipeline

Jenkinsfile_k8s

@@ -123,28 +123,6 @@ pipeline {
             }
           }
         }
-        stage('Updatecli') {
-          steps {
-            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-              script {
-                // TODO: Implement https://github.com/jenkins-infra/pipeline-library/issues/518 to allow using the updatecli() library function
-                withCredentials([
-                  usernamePassword(
-                  credentialsId: 'github-app-updatecli-on-jenkins-infra',
-                  usernameVariable: 'USERNAME_VALUE', // Setting this variable is mandatory, even if of not used when the credentials is a githubApp one
-                  passwordVariable: 'UPDATECLI_GITHUB_TOKEN'
-                  )
-                ]) {
-                  sh 'updatecli version'
-                  sh 'updatecli diff --values ./updatecli/values.yaml --config ./updatecli/updatecli.d'
-                  if (env.BRANCH_IS_PRIMARY) {
-                    sh 'updatecli apply --values ./updatecli/values.yaml --config ./updatecli/updatecli.d'
-                  }
-                }
-              }
-            }
-          }
-        }
       }
     }
     stage('Packer Images') {
@@ -179,18 +157,22 @@ pipeline {
               values 'amazon-ebs'
             }
           }
-          // Do NOT build ARM images in Azure
+          // Only build Ubuntu images for arm64 CPU in AWS (notValues)
           exclude {
             axis {
               name 'cpu_architecture'
               values 'arm64'
             }
+            axis {
+              name 'agent_type'
+              notValues 'ubuntu-22.04'
+            }
             axis {
               name 'compute_type'
-              values 'azure-arm'
+              values 'amazon-ebs'
             }
           }
-          // Only build Ubuntu images for arm64 CPU in AWS
+          // Only build Ubuntu images for arm64 CPU in Azure (notValues)
           exclude {
             axis {
               name 'cpu_architecture'
@@ -202,7 +184,7 @@ pipeline {
             }
             axis {
               name 'compute_type'
-              values 'amazon-ebs'
+              values 'azure-arm'
             }
           }
           // No build on Windows or Docker, not yet implemented

Jenkinsfile_updatecli

@@ -0,0 +1,35 @@
+def cronExpr = env.BRANCH_IS_PRIMARY ? 'H/30 * * * *' : ''
+
+pipeline {
+  agent {
+    docker { image 'jenkinsciinfra/helmfile:2.5.92' }
+  }
+  options {
+    buildDiscarder(logRotator(numToKeepStr: '10'))
+    timeout(time: 30, unit: 'MINUTES')
+    disableConcurrentBuilds()
+  }
+
+  triggers {
+    cron (cronExpr)
+  }
+
+  stages {
+    stage('Check Configuration Update') {
+      // Run updatecli's diff on both push and pull requests (in case a configuration change breaks updatecli)
+      steps {
+        catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+          updatecli(action: 'diff', cronTriggerExpression: cronExpr)
+        }
+      }
+    } // stage
+    stage('Apply Configuration Update') {
+      when {
+        expression { env.BRANCH_IS_PRIMARY }
+      }
+      steps {
+        updatecli(action: 'apply', cronTriggerExpression: cronExpr)
+      }
+    }
+  }
+}

build-jenkins-agent-ubuntu.pkr.hcl

@@ -55,7 +55,8 @@ build {
 
   post-processor "docker-tag" {
     only       = ["docker.ubuntu"]
-    repository = "${var.docker_namespace}/${local.image_name}"
+    # TODO specify architecture in image name with local.image_name
+    repository = format("%s/jenkins-agent-%s-%s",var.docker_namespace, var.agent_os_type, var.agent_os_version)
     tags       = [var.image_version, "latest"]
   }
 }

locals.pkr.hcl

@@ -2,7 +2,8 @@ locals {
   now_unix_timestamp    = formatdate("YYYYMMDDhhmmss", timestamp())
   agent                 = format("%s-%s", var.agent_os_type, var.agent_os_version)
   agent_os_version_safe = replace(var.agent_os_version, ".", "_")
-  image_name            = format("jenkins-agent-%s-%s", var.agent_os_type, var.agent_os_version)
+  image_name            = format("jenkins-agent-%s-%s-%s", var.agent_os_type, var.agent_os_version, var.architecture)
+  unique_image_name     = format("%s-%s", local.image_name, local.now_unix_timestamp)
   aws_spot_instance_types = {
     # 4 vCPU x86 / 16 GB / $0.1504 - https://aws.amazon.com/fr/ec2/instance-types/t3/#Product_Details
     "amd64" = ["t3.xlarge", "t3a.xlarge", "t2.xlarge", "m6a.xlarge"]

podTemplate.yaml

@@ -0,0 +1,40 @@
+apiVersion: "v1"
+kind: "Pod"
+metadata:
+  labels:
+    jenkins: "agent"
+    job: "helmfile"
+spec:
+  nodeSelector:
+    kubernetes.azure.com/agentpool: infracipool
+    kubernetes.io/os: linux
+  tolerations:
+    - key: "os"
+      operator: "Equal"
+      value: "linux"
+      effect: "NoSchedule"
+    - key: "spot"
+      operator: "Equal"
+      value: "true"
+      effect: "NoSchedule"
+    - key: "jenkins"
+      operator: "Equal"
+      value: "infra.ci.jenkins.io"
+      effect: "NoSchedule"
+    - key: "kubernetes.azure.com/scalesetpriority"
+      operator: "Equal"
+      value: "spot"
+      effect: "NoSchedule"
+  containers:
+    - image: "jenkinsciinfra/helmfile:2.5.92"
+      imagePullPolicy: "IfNotPresent"
+      name: "jnlp"
+      resources:
+        limits:
+          memory: "1024Mi"
+          cpu: "2"
+        requests:
+          memory: "256Mi"
+          cpu: "0.5"
+      securityContext:
+        privileged: false

provisioning/tools-versions.yml

@@ -1,5 +1,5 @@
 asdf_version: "0.11.3"
-awscli_version: "2.11.11"
+awscli_version: "2.11.13"
 azurecli_version: "2.47.0"
 chocolatey_version: "1.3.1"
 compose_version: "2.17.2"
@@ -20,15 +20,17 @@ jenkins_remoting_version: "3107.v665000b_51092"
 jq_version: "1.6"
 jxreleaseversion_version: "2.6.10"
 kubectl_version: "1.23.13"
+launchable_version: "1.63.0"
 maven_version: "3.9.1"
 netlifydeploy_version: "0.1.8"
 nodejs_version: "18.13.0"
 openssh_authorized_keys_url: "https://raw.githubusercontent.com/jenkins-infra/aws/main/ec2_agents_authorized_keys"
 packer_version: "1.8.5"
+python3_version: "3.11.3"
 ruby_version: "2.6.10"
 terraform_version: "1.4.4"
 tfsec_version: "1.28.1"
-updatecli_version: "0.48.1"
+updatecli_version: "0.49.1"
 vagrant_version: "2.3.4"
-windows_pwsh_version: "7.3.3"
+windows_pwsh_version: "7.3.4"
 yq_version: "4.25.3"

provisioning/ubuntu-provision.sh

@@ -34,6 +34,7 @@ userhome=/home/jenkins
 groupname=jenkins
 groupid=1001
 asdf_install_dir="${userhome}/.asdf"
+launchable_venv_dir="/usr/local/launchable"
 
 ## This function checks a list of commands are working, and exits with code 1 if not
 function check_commands() {
@@ -567,6 +568,15 @@ function install_playwright_dependencies() {
   rm -rf "${temp_dir}"
 }
 
+## Install Launchable with python3 in its own virtual environment
+function install_launchable() {
+  python3 -m venv "${launchable_venv_dir}"
+  "${launchable_venv_dir}"/bin/pip --require-virtualenv --no-cache-dir install setuptools wheel
+  "${launchable_venv_dir}"/bin/pip --require-virtualenv --no-cache-dir install launchable=="${LAUNCHABLE_VERSION}"
+  # Symlink to a folder in the PATH
+  ln -s "${launchable_venv_dir}/bin/launchable" /usr/local/bin/launchable
+}
+
 ## Ensure that the VM is cleaned up
 function cleanup() {
   export HISTSIZE=0
@@ -650,7 +660,9 @@ function sanity_check() {
   && echo 'playwright install:' \
   && npm install playwright-test \
   && echo 'playwright version:' \
-  && npm @playwright/test --version
+  && npm @playwright/test --version \
+  && echo 'launchable version:' \
+  && launchable --version
   "
   echo "== End of sanity check"
   echo "== Installed packages:"
@@ -694,6 +706,7 @@ function main() {
   install_tfsec
   install_nodejs
   install_playwright_dependencies
+  install_launchable
   cleanup
 }
 

provisioning/windows-provision.ps1

@@ -304,7 +304,7 @@ $downloads = [ordered]@{
             & Remove-Item -Force -Recurse "$baseDir\chocolatey.tmp";
         };
         'cleanupLocal' = 'true';
-        'path' = "$baseDir\ruby26\bin\;C:\HashiCorp\Vagrant\;C:\Program Files\Amazon\AWSCLIV2";
+        'path' = "$baseDir\ruby26\bin\;C:\HashiCorp\Vagrant\;C:\Program Files\Amazon\AWSCLIV2\;c:\python311\;C:\python311\Scripts\;";
         'postInstall' = {
             # Installation of make for Windows
             & "choco.exe" install make --yes --no-progress --limit-output --fail-on-error-output;
@@ -317,6 +317,11 @@ $downloads = [ordered]@{
             & "choco.exe" install awscli --yes --no-progress --limit-output --fail-on-error-output --version "${env:AWSCLI_VERSION}";
             & "choco.exe" install datadog-agent --yes --no-progress --limit-output --fail-on-error-output;
             & "choco.exe" install vcredist2015 --yes --no-progress --limit-output --fail-on-error-output;
+            # Installation of python3 for Launchable
+            & "choco.exe" install python3 --yes --no-progress --limit-output --fail-on-error-output --version "${env:PYTHON3_VERSION}";
+            # Installation of Launchable globally (no other python tool)
+            & "c:\python311\python.exe" -m pip --no-cache-dir --upgrade install setuptools wheel pip;
+            & "c:\python311\python.exe" -m pip --no-cache-dir install launchable=="${env:LAUNCHABLE_VERSION}";
         };
         'sanityCheck'= {
             & choco.exe;
@@ -327,6 +332,8 @@ $downloads = [ordered]@{
             & "$baseDir\ruby26\bin\bundle" -v;
             & updatecli.exe version;
             & yq.exe --version;
+            & "c:\python311\python.exe" --version;
+            & "C:\python311\Scripts\launchable.exe" --version;
         }
     };
 }

sources.pkr.hcl

@@ -1,6 +1,7 @@
 # This source defines all the common settings for any AWS AMI (whatever Operating System)
 source "amazon-ebs" "base" {
-  ami_name            = "${local.image_name}-${var.architecture}-${local.now_unix_timestamp}"
+  # must to be unique to avoid concurrent builds to conflict
+  ami_name            = "${local.unique_image_name}"
   spot_instance_types = local.aws_spot_instance_types[var.architecture]
   spot_price          = "auto"
   # Define custom rootfs for build to avoid later filesystem extension during agent startups
@@ -25,7 +26,6 @@ source "amazon-ebs" "base" {
   # To improve audit and garbage collecting, we provide tags
   tags = {
     imageplatform = var.architecture
-    imagetype     = local.image_name
     timestamp     = local.now_unix_timestamp
     version       = var.image_version
     scm_ref       = var.scm_ref
@@ -35,7 +35,6 @@ source "amazon-ebs" "base" {
 
 # This source defines all the common settings for any Azure image (whatever Operating System)
 source "azure-arm" "base" {
-  managed_image_name                = local.image_name
   managed_image_resource_group_name = local.azure_destination_resource_group
 
   vm_size = local.azure_vm_size[var.architecture]
@@ -56,15 +55,15 @@ source "azure-arm" "base" {
     subscription        = var.azure_subscription_id
     resource_group      = local.azure_destination_resource_group
     gallery_name        = "${var.build_type}_packer_images"
-    image_name          = "${local.image_name}-${var.architecture}"
+    # Not unique name defined in https://github.com/jenkins-infra/azure/blob/bfe56cb4f843b0c8029413090c383f7ac38dde2a/locals.tf#L4-L41
+    image_name          = "${local.image_name}"
     image_version       = var.image_version
     replication_regions = lookup(local.azure_galleries, "${var.build_type}_packer_images", [])
   }
 
   # To improve audit and garbage collecting, we provide tags
   azure_tags = {
     imageplatform = var.architecture
-    imagetype     = local.image_name
     timestamp     = local.now_unix_timestamp
     version       = var.image_version
     scm_ref       = var.scm_ref

updatecli/updatecli.d/launchable.yml

@@ -0,0 +1,48 @@
+---
+name: Bump launchable version
+
+scms:
+  default:
+    kind: github
+    spec:
+      user: "{{ .github.user }}"
+      email: "{{ .github.email }}"
+      owner: "{{ .github.owner }}"
+      repository: "{{ .github.repository }}"
+      token: "{{ requiredEnv .github.token }}"
+      username: "{{ .github.username }}"
+      branch: "{{ .github.branch }}"
+
+sources:
+  lastReleaseVersion:
+    kind: githubrelease
+    name: Get the latest launchable release version
+    spec:
+      owner: "launchableinc"
+      repository: "cli"
+      token: "{{ requiredEnv .github.token }}"
+      username: "{{ .github.username }}"
+      versionfilter:
+        kind: semver
+    transformers:
+      - trimprefix: v
+
+targets:
+  updateVersion:
+    name: "Update the launchable version in the provision-env.yml file"
+    sourceid: lastReleaseVersion
+    kind: yaml
+    spec:
+      file: "provisioning/tools-versions.yml"
+      key: "launchable_version"
+    scmid: default
+
+actions:
+  default:
+    kind: github/pullrequest
+    title: Bump launchable version to {{ source "lastReleaseVersion" }}
+    scmid: default
+    spec:
+      labels:
+        - enhancement
+        - launchable