CLOUDP-130491: Daily rebuilds of the operator image (#1091)

* Daily docker rebuilds

Author: lsierant

docs/RELEASE_NOTES.md

@@ -1,3 +1,10 @@
+# MongoDB Kubernetes Operator 0.7.6
+
+## Kubernetes Operator
+
+* Changes
+  * `mongodb-kubernetes-operator` image is now rebuilt daily, incorporating updates to system packages and security fixes. The operator binary is built only once during the release process and used without changes in daily rebuilt  
+
 # MongoDB Kubernetes Operator 0.7.5
 
 - Changes

docs/how-to-release.md

@@ -14,6 +14,7 @@
     * Create a PR with the title `Release MongoDB Kubernetes Operator v<operator-version>` (the title must match this pattern).
     * Wait for the tests to pass and merge the PR.
       * Upon approval, all new images for this release will be built and released, and a GitHub release draft will be created.
+        * Dockerfiles for mongodb-kubernetes-operator and mongodb-agent will be uploaded to S3 to be used by daily rebuild process in the enterprise repo.
       * Review and publish the new GitHub release draft, that was prepared
     * Merge helm-charts PR and update submodule to the latest commit on `main` branch.
     * Create a new PR with only bump to the helm-chart submodule.

inventories/operator-inventory.yaml

@@ -5,7 +5,7 @@ images:
   - name: operator-ubi
     vars:
       context: .
-      template_context: scripts/dev/templates
+      template_context: scripts/dev/templates/operator
 
     inputs:
       - operator_image
@@ -14,24 +14,44 @@ images:
     platform: linux/amd64
 
     stages:
-      - name: operator-template-ubi
+#
+# Dev build stages
+#
+      - name: operator-builder-dev
+        task_type: docker_build
+        tags: [ "ubi" ]
+        dockerfile: scripts/dev/templates/operator/Dockerfile.builder
+
+        buildargs:
+          builder_image: $(inputs.params.builder_image)
+
+        labels:
+          quay.expires-after: 48h
+
+        output:
+        - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
+          tag: $(inputs.params.version_id)-context
+
+      - name: operator-template-dev
         task_type: dockerfile_template
         tags: ["ubi"]
-        distro: operator
-
+        template_file_extension: operator
         inputs:
-          - builder
-          - builder_image
           - base_image
 
         output:
-          - dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id)
+          - dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.version_id)
 
-      - name: operator-build-ubi
+      - name: operator-build-dev
         task_type: docker_build
         tags: ["ubi"]
+        dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.version_id)
+
+        inputs:
+          - version_id
 
-        dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id)
+        buildargs:
+          imagebase: $(inputs.params.registry)/$(inputs.params.operator_image_dev):$(inputs.params.version_id)-context
 
         labels:
           quay.expires-after: 48h
@@ -42,14 +62,52 @@ images:
           - registry: $(inputs.params.registry)/$(inputs.params.operator_image_dev)
             tag: latest
 
-      - name: operator-release-ubi
+#
+# Release build stages
+#
+      - name: operator-builder-release
+        task_type: docker_build
+        tags: [ "ubi", "release"]
+
+        inputs:
+          - builder_image
+          - release_version
+
+        dockerfile: scripts/dev/templates/operator/Dockerfile.builder
+
+        labels:
+          quay.expires-after: Never
+
+        buildargs:
+          builder_image: $(inputs.params.builder_image)
+
+        output:
+          - registry: $(inputs.params.registry)/$(inputs.params.operator_image)
+            tag: $(inputs.params.release_version)-context
+
+      - name: operator-template-release
+        task_type: dockerfile_template
+        tags: [ "ubi", "release"]
+        template_file_extension: operator
+        inputs:
+          - base_image
+          - release_version
+
+        output:
+          - dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version)
+          - dockerfile: $(inputs.params.s3_bucket)/mongodb-kubernetes-operator/$(inputs.params.release_version)/ubi/Dockerfile
+
+      - name: operator-build-release
         task_type: docker_build
-        tags: ["release"]
+        tags: [ "ubi", "release"]
 
         inputs:
           - release_version
 
-        dockerfile: scripts/dev/templates/Dockerfile.ubi-$(inputs.params.version_id)
+        dockerfile: scripts/dev/templates/operator/Dockerfile.operator-$(inputs.params.release_version)
+
+        buildargs:
+          imagebase: $(inputs.params.registry)/$(inputs.params.operator_image):$(inputs.params.release_version)-context
 
         labels:
           quay.expires-after: Never

inventory.yaml

@@ -135,7 +135,6 @@ images:
           - registry: $(inputs.params.registry)/$(inputs.params.agent_image_dev)
             tag: $(inputs.params.version_id)-context
 
-
       - name: agent-template-ubi
         task_type: dockerfile_template
         distro: ubi
@@ -144,7 +143,6 @@ images:
         output:
           - dockerfile: scripts/dev/templates/agent/Dockerfile.ubi-$(inputs.params.version_id)
 
-
       - name: agent-ubi-build
         task_type: docker_build
         tags: ["ubi"]
@@ -224,6 +222,9 @@ images:
         labels:
           quay.expires-after: 48h
 
+        buildargs:
+          builder_image: $(inputs.params.builder_image)
+
         output:
           - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image_dev)
             tag: $(inputs.params.version_id)-context
@@ -245,7 +246,6 @@ images:
           - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image_dev)
             tag: latest
 
-
       - name: readiness-init-context-release
         task_type: docker_build
         dockerfile: scripts/dev/templates/readiness/Dockerfile.builder
@@ -254,14 +254,16 @@ images:
         labels:
           quay.expires-after: Never
 
+        buildargs:
+          builder_image: $(inputs.params.builder_image)
+
         inputs:
           - release_version
 
         output:
           - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image)
             tag: $(inputs.params.release_version)-context
 
-
       - name: readiness-init-build-release
         task_type: docker_build
         dockerfile: scripts/dev/templates/readiness/Dockerfile.readiness
@@ -280,7 +282,6 @@ images:
           - registry: $(inputs.params.registry)/$(inputs.params.readiness_probe_image)
             tag: $(inputs.params.release_version)
 
-
   - name: version-post-start-hook-init
     vars:
       context: .
@@ -295,6 +296,9 @@ images:
         dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder
         tags: ["post-start-hook"]
 
+        buildargs:
+          builder_image: $(inputs.params.builder_image)
+
         labels:
           quay.expires-after: 48h
 
@@ -318,7 +322,6 @@ images:
           - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image_dev)
             tag: latest
 
-
       - name: version-post-start-hook-init-context-release
         task_type: docker_build
         dockerfile: scripts/dev/templates/versionhook/Dockerfile.builder
@@ -327,14 +330,16 @@ images:
         labels:
           quay.expires-after: Never
 
+        buildargs:
+          builder_image: $(inputs.params.builder_image)
+
         inputs:
           - release_version
 
         output:
           - registry: $(inputs.params.registry)/$(inputs.params.version_post_start_hook_image)
             tag: $(inputs.params.release_version)-context
 
-
       - name: version-post-start-hook-init-build-release
         task_type: docker_build
         dockerfile: scripts/dev/templates/versionhook/Dockerfile.versionhook

pipeline.py

@@ -18,7 +18,6 @@
     ]
 )
 
-GOLANG_TAG = "1.18.5"
 DEFAULT_IMAGE_TYPE = "ubuntu"
 DEFAULT_NAMESPACE = "default"
 
@@ -82,6 +81,7 @@ def build_readiness_probe_image(config: DevConfig) -> None:
             "release_version": release["readiness-probe"],
             "readiness_probe_image": config.readiness_probe_image,
             "readiness_probe_image_dev": config.readiness_probe_image_dev,
+            "builder_image": release["golang-builder-image"],
         },
     )
 
@@ -98,6 +98,7 @@ def build_version_post_start_hook_image(config: DevConfig) -> None:
             "release_version": release["version-upgrade-hook"],
             "version_post_start_hook_image": config.version_upgrade_hook_image,
             "version_post_start_hook_image_dev": config.version_upgrade_hook_image_dev,
+            "builder_image": release["golang-builder-image"],
         },
     )
 
@@ -111,23 +112,25 @@ def build_operator_ubi_image(config: DevConfig) -> None:
         args={
             "registry": config.repo_url,
             "builder": "true",
-            "builder_image": f"golang:{GOLANG_TAG}",
+            "builder_image": release["golang-builder-image"],
             "base_image": "registry.access.redhat.com/ubi8/ubi-minimal:latest",
             "operator_image": config.operator_image,
             "operator_image_dev": config.operator_image_dev,
             "release_version": release["mongodb-kubernetes-operator"],
+            "s3_bucket": config.s3_bucket,
         },
         inventory="inventories/operator-inventory.yaml",
     )
 
 
 def build_e2e_image(config: DevConfig) -> None:
+    release = _load_release()
     sonar_build_image(
         "e2e",
         config,
         args={
             "registry": config.repo_url,
-            "base_image": f"golang:{GOLANG_TAG}",
+            "base_image": release["golang-builder-image"],
             "e2e_image": config.e2e_image,
         },
         inventory="inventories/e2e-inventory.yaml",

release.json

@@ -1,4 +1,5 @@
 {
+  "golang-builder-image": "golang:1.18",
   "mongodb-kubernetes-operator": "0.7.5",
   "version-upgrade-hook": "1.0.5",
   "readiness-probe": "1.0.11",

requirements.txt

@@ -14,4 +14,5 @@ dnspython==2.0.0
 requests==2.24.0
 pyyaml==5.4.1
 ruamel.yaml==0.17.9
+semver==2.13.0
 rsa>=4.7 # not directly required, pinned by Snyk to avoid a vulnerability

scripts/ci/add_supported_release.py

@@ -8,14 +8,19 @@
 import os
 import subprocess
 import sys
-from typing import Dict, Any
+from typing import Dict, Any, List
 
 import pymongo
 
 LOGLEVEL = os.environ.get("LOGLEVEL", "INFO").upper()
 logging.basicConfig(level=LOGLEVEL)
 
-VALID_IMAGES = frozenset(["mongodb-agent"])
+VALID_IMAGES = frozenset(
+    [
+        "mongodb-agent",
+        "mongodb-kubernetes-operator",
+    ]
+)
 
 
 def get_repo_root() -> str:
@@ -38,7 +43,7 @@ def mongo_client() -> pymongo.MongoClient:
     return pymongo.MongoClient(cnx_str)
 
 
-def add_release_version(image: str, version: str) -> None:
+def add_release_version(image: str, version: str, variants: List[str]) -> None:
     client = mongo_client()
 
     database = os.environ["ATLAS_DATABASE"]
@@ -57,7 +62,7 @@ def add_release_version(image: str, version: str) -> None:
             "version": version,
             "supported": True,
             "eol": year_from_now,
-            "variants": ["ubi", "ubuntu"],
+            "variants": variants,
         }
     )
 
@@ -71,8 +76,17 @@ def main() -> int:
     parser.add_argument(
         "--image-name", help="image to add a new supported version", type=str
     )
+    parser.add_argument(
+        "--variants",
+        help="supported variants, comma-separated, e.g. 'ubi,ubuntu', default=ubi",
+        type=str,
+        default="ubi",
+    )
+
     args = parser.parse_args()
 
+    variants = args.variants.split(",")
+
     if args.image_name not in VALID_IMAGES:
         print(
             "Image {} not supported. Not adding release version.".format(
@@ -82,13 +96,25 @@ def main() -> int:
         return 0
 
     # for now, there is just one version to add as a supported release.
-    version = get_release()[args.image_name]["version"]
-    logging.info("Adding new release: {} {}".format(args.image_name, version))
+    version = get_release_version(args.image_name)
+    logging.info(
+        "Adding new release: {} {}, {}".format(args.image_name, version, variants)
+    )
 
-    add_release_version(args.image_name, version)
+    add_release_version(args.image_name, version, variants)
 
     return 0
 
 
+# get_release_version gets image version from release.json handling both
+# version embedded in the object (for mongodb-agent) and set as a simple string (for other images).
+def get_release_version(image_name: str) -> str:
+    release_obj = get_release()[image_name]
+    if isinstance(release_obj, str):
+        return release_obj
+
+    return release_obj["version"]
+
+
 if __name__ == "__main__":
     sys.exit(main())