feat: onboard renovate to support auto-upgrading components (#4913)

Co-authored-by: Devin Wong <[email protected]>
Azure · Sep 16, 2024 · 2d73a30 · 2d73a30
1 parent e26cf97
commit 2d73a30
Show file tree

Hide file tree

Showing 282 changed files with 6,238 additions and 2,549 deletions.
diff --git a/.github/README-RENOVATE.md b/.github/README-RENOVATE.md
diff --git a/.github/README.md b/.github/README.md
@@ -0,0 +1,9 @@
+## AgentBaker GitHub Repository
+
+This repository contains the `.github` folder for the AgentBaker project. The `.github` folder is used to store files related to GitHub-specific features and configurations. It typically includes files such as README.md, issue templates, pull request templates, and workflows.
+
+The purpose of this folder is to provide a centralized location for managing the GitHub-related aspects of the AgentBaker project. It helps streamline the development process by providing standardized templates and configurations for issues, pull requests, and other GitHub features.
+
+Please refer to the individual files within this folder for more detailed information on their purpose and usage.
+
+For the descriptions for Renovate.json, please read [README-RENOVATE.md](README-RENOVATE.md)
diff --git a/.github/images/JSONata_exerciser_example.png b/.github/images/JSONata_exerciser_example.png
diff --git a/.github/images/Renovate_component_datasource.png b/.github/images/Renovate_component_datasource.png
diff --git a/.github/images/Renovate_minor_enabled.png b/.github/images/Renovate_minor_enabled.png
diff --git a/.github/renovate.json b/.github/renovate.json
@@ -1,35 +1,123 @@
 {
+  "extends": [
+    "config:base"
+  ],
   "$schema": "https://docs.renovatebot.com/renovate-schema.json",
   "enabledManagers": ["custom.regex"],
   "prConcurrentLimit": 50,
-  "prHourlyLimit": 20,
+  "prHourlyLimit": 50,
   "branchConcurrentLimit": 50,
+  "separateMinorPatch": true,
   "labels": ["renovate"],
+  "logLevelRemap": [
+    {
+      "matchMessage": "/^Custom manager fetcher/",
+      "newLogLevel": "info"
+    }
+  ],
   "packageRules": [
     {
+      "matchDatasources": ["docker", "custom.deb1804", "custom.deb2004", "custom.deb2204", "custom.deb2404"],
+      "matchUpdateTypes": [
+        "minor"
+      ],
+      "automerge": false,
+      "enabled": false
+    },
+    {
+      "matchDatasources": ["docker", "custom.deb1804", "custom.deb2004", "custom.deb2204", "custom.deb2404"],
       "matchUpdateTypes": [
-        "minor",
         "patch",
         "pin",
         "digest"
       ],
-      "automerge": false
+      "automerge": false,
+      "enabled": true
+    },
+    {
+      "matchPackageNames": ["moby-runc", "moby-containerd"],
+      "assignees": ["devinwong", "anujmaheshwari1", "cameronmeissner", "AlisonB319", "lilypan26", "djsly", "jason1028kr", "UtheMan", "zachary-bailey", "ganeshkumarashok"]
+    },
+    {
+      "matchPackageNames": ["aks/aks-node-ca-watcher"],
+      "assignees": ["UtheMan"]
+    },
+    {
+      "matchPackageNames": ["moby-runc", "moby-containerd"],
+      "extractVersion": "^v?(?<version>.+)$"
     }
   ],
   "customManagers": [
     {
       "customType": "regex",
+      "description": "auto update containerImages in components.json",
       "fileMatch": [
         "parts/linux/cloud-init/artifacts/components.json"
       ],
       "matchStringsStrategy": "any",
       "matchStrings": [
-        ".*?\"downloadURL\":\\s*\"(?<registryUrl>[^/]+)/(?<packageName>[^:]+)[\\s\\S]*?\"multiArchVersions\": \\[\\s*\"(?<currentValue>[^\\\"]+)\".*"
+        "\"renovateTag\":\\s*\"registry=(?<registryUrl>[^,]+), name=(?<packageName>[^\"]+)\",\\s*\"latestVersion\":\\s*\"(?<currentValue>[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?<depType>[^\"]+)\")?"
       ],
       "datasourceTemplate": "docker",
-      "registryUrlTemplate": "https://mcr.microsoft.com"
+      "autoReplaceStringTemplate": "\"renovateTag\": \"registry={{{registryUrl}}}, name={{{packageName}}}\",\n          \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n          \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
+    },
+    {
+      "customType": "regex",
+      "description": "auto update packages for OS ubuntu 18.04 in components.json",
+      "fileMatch": [
+        "parts/linux/cloud-init/artifacts/components.json"
+      ],
+      "matchStringsStrategy": "any",
+      "matchStrings": [
+        "\"renovateTag\":\\s*\"name=(?<packageName>[^,]+), os=ubuntu, release=18\\.04\",\\s*\"versions\":\\s*\\[\\s*\"(?<currentValue>[^\"]+)\""
+      ],
+      "datasourceTemplate": "custom.deb1804",
+      "versioningTemplate": "deb",
+      "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, os=ubuntu, release=18.04\",\n                \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n                \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
+    },
+    {
+      "customType": "regex",
+      "description": "auto update packages for OS ubuntu 20.04 in components.json",
+      "fileMatch": [
+        "parts/linux/cloud-init/artifacts/components.json"
+      ],
+      "matchStringsStrategy": "any",
+      "matchStrings": [
+        "\"renovateTag\":\\s*\"name=(?<packageName>[^,]+), os=ubuntu, release=20\\.04\",\\s*\"latestVersion\":\\s*\"(?<currentValue>[^\"]+)\"[^}]*"
+      ],
+      "datasourceTemplate": "custom.deb2004",
+      "versioningTemplate": "deb",
+      "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, os=ubuntu, release=20.04\",\n                \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n                \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"   
     },
     {
+      "customType": "regex",
+      "description": "auto update packages for OS ubuntu 22.04 in components.json",
+      "fileMatch": [
+        "parts/linux/cloud-init/artifacts/components.json"
+      ],
+      "matchStringsStrategy": "any",
+      "matchStrings": [
+        "\"renovateTag\":\\s*\"name=(?<packageName>[^,]+), os=ubuntu, release=22\\.04\",\\s*\"latestVersion\":\\s*\"(?<currentValue>[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?<depType>[^\"]+)\")?"
+      ],
+      "datasourceTemplate": "custom.deb2204",
+      "versioningTemplate": "deb",
+      "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, os=ubuntu, release=22.04\",\n                \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n                \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
+    },
+    {
+      "customType": "regex",
+      "description": "auto update packages for OS ubuntu 24.04 in components.json",
+      "fileMatch": [
+        "parts/linux/cloud-init/artifacts/components.json"
+      ],
+      "matchStringsStrategy": "any",
+      "matchStrings": [
+        "\"renovateTag\":\\s*\"name=(?<packageName>[^,]+), os=ubuntu, release=24\\.04\",\\s*\"latestVersion\":\\s*\"(?<currentValue>[^\"]+)\"(?:[^}]*\"previousLatestVersion\":\\s*\"(?<depType>[^\"]+)\")?"
+      ],
+      "datasourceTemplate": "custom.deb2404",
+      "versioningTemplate": "deb",
+      "autoReplaceStringTemplate": "\"renovateTag\": \"name={{{packageName}}}, os=ubuntu, release=24.04\",\n                \"latestVersion\": \"{{{newValue}}}\"{{#if depType}},\n                \"previousLatestVersion\": \"{{{currentValue}}}\"{{/if}}"
+    },
+    {      
       "customType": "regex",
       "description": "update version line in any cse_*.sh",
       "fileMatch": [
@@ -40,5 +128,35 @@
         "#\\s*renovate:\\s*(datasource=(?<datasource>.*?) )?depName=(?<depName>.*?)( versioning=(?<versioning>.*?))?\\s*.*?version.*\\\"(?<currentValue>.*)\\\""
       ]
     }
-  ]
-}
+  ],
+  "customDatasources": {
+    "deb1804": {
+      "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/18.04/prod/dists/testing/main/binary-amd64/Packages",
+      "format": "plain",
+      "transformTemplates": [
+        "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $replace(releases[$i + 1].version, /^Version:\\s*/, \"v\") })), function($v) { {\"version\": $v} })}"
+      ]
+    },
+    "deb2004": {
+      "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/20.04/prod/dists/testing/main/binary-amd64/Packages",
+      "format": "plain",
+      "transformTemplates": [
+        "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $replace(releases[$i + 1].version, /^Version:\\s*/, \"v\") })), function($v) { {\"version\": $v} })}"
+      ]
+    },
+    "deb2204": {
+      "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/22.04/prod/dists/testing/main/binary-amd64/Packages",
+      "format": "plain",
+      "transformTemplates": [
+        "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $replace(releases[$i + 1].version, /^Version:\\s*/, \"v\") })), function($v) { {\"version\": $v} })}"
+      ]
+    },
+    "deb2404": {
+      "defaultRegistryUrlTemplate": "https://packages.microsoft.com/ubuntu/24.04/prod/dists/testing/main/binary-amd64/Packages",
+      "format": "plain",
+      "transformTemplates": [
+        "{\"releases\": $map(($index := releases#$i[version=\"Package: {{packageName}}\"].$i; $map($index, function($i) { $replace(releases[$i + 1].version, /^Version:\\s*/, \"v\") })), function($v) { {\"version\": $v} })}"
+      ]
+    }
+  }
+}
diff --git a/e2e/scenario_helpers_test.go b/e2e/scenario_helpers_test.go
@@ -150,16 +150,16 @@ func getExpectedPackageVersions(packageName, distro, release string) []string {
 	packages := gjson.GetBytes(jsonBytes, fmt.Sprintf("Packages.#(name=%s).downloadURIs", packageName))
 
 	for _, packageItem := range packages.Array() {
-		versions := packageItem.Get(fmt.Sprintf("%s.%s.versions", distro, release))
-		if !versions.Exists() {
-			versions = packageItem.Get(fmt.Sprintf("%s.current.versions", distro))
-		}
-		if !versions.Exists() {
-			versions = packageItem.Get("default.current.versions")
-		}
-		if versions.Exists() {
+		// check if versionsV2 exists
+		if packageItem.Get(fmt.Sprintf("%s.%s.versionsV2", distro, release)).Exists() {
+			versions := packageItem.Get(fmt.Sprintf("%s.%s.versionsV2", distro, release))
 			for _, version := range versions.Array() {
-				expectedVersions = append(expectedVersions, version.String())
+				// get versions.latestVersion and append to expectedVersions
+				expectedVersions = append(expectedVersions, version.Get("latestVersion").String())
+				// get versions.previousLatestVersion (if exists) and append to expectedVersions
+				if version.Get("previousLatestVersion").Exists() {
+					expectedVersions = append(expectedVersions, version.Get("previousLatestVersion").String())
+				}
 			}
 		}
 	}

diff --git a/e2e/scenario_test.go b/e2e/scenario_test.go
@@ -950,7 +950,7 @@ func Test_ubuntu2204ContainerdHasCurrentVersion(t *testing.T) {
 			},
 			LiveVMValidators: []*LiveVMValidator{
 				// for containerd we only support one version at a time for each distro/release
-				containerdVersionValidator(getExpectedPackageVersions("containerd", "default", "current")[0]),
+				containerdVersionValidator(getExpectedPackageVersions("containerd", "ubuntu", "r2204")[0]),
 			},
 		},
 	})