Support for Gardener integration tests on GCP

cleanup.py

@@ -39,7 +39,7 @@ def cleanup_image(
     elif release.platform == 'aws':
         cleanup_function = cleanup_aws_images_by_id
     elif release.platform == 'gcp':
-        cleanup_function = None # cleanup_gcp_images
+        cleanup_function = cleanup_gcp_images
     elif release.platform == 'azure':
         cleanup_function = cleanup_azure_community_gallery_images
     elif release.platform == 'openstack':
@@ -121,6 +121,7 @@ def clean_alicloud_images(
 def cleanup_gcp_images(
     release: gm.OnlineReleaseManifest,
     publishing_cfg: gm.PublishingCfg,
+    dry_run: bool = False
 ) -> gm.OnlineReleaseManifest:
     gcp_publishing_cfg: gm.PublishingTargetGCP = publishing_cfg.target(release.platform)
     cfg_factory = ci.util.ctx().cfg_factory()
@@ -134,6 +135,7 @@ def cleanup_gcp_images(
         gcp_project_name=gcp_cfg.project(),
         release=release,
         publishing_cfg=gcp_publishing_cfg,
+        dry_run=dry_run
     )
 
 

glci/gcp.py

@@ -2,6 +2,7 @@
 import tempfile
 import time
 import logging
+import os
 
 import google.cloud.storage.blob
 import google.cloud.storage.client
@@ -13,7 +14,7 @@
 logger = lambda: logging.getLogger(__name__)
 
 
-def upload_image_to_gcp_store(
+def upload_image_to_gcs_bucket(
     storage_client: google.cloud.storage.Client,
     s3_client,
     release: glci.model.OnlineReleaseManifest,
@@ -31,22 +32,33 @@ def upload_image_to_gcp_store(
 
     # XXX: rather do streaming
     with tempfile.TemporaryFile() as tfh:
-        logger().info(f'downloading image from {s3_bucket_name=}')
+        resp = s3_client.get_object(
+            Bucket=s3_bucket_name,
+            Key=raw_image_key,
+        )
+        size = resp['ContentLength']
+        logger().info(f'downloading image from {s3_bucket_name=} to temporary location ({size=})')
+
         s3_client.download_fileobj(
             Bucket=s3_bucket_name,
             Key=raw_image_key,
             Fileobj=tfh,
         )
         logger().info(f'downloaded image from {s3_bucket_name=}')
 
+        # get the size of the temp file on local disk
+        tfh.seek(0, os.SEEK_END)
+        size = tfh.tell()
         tfh.seek(0)
 
-        logger().info(f're-uploading image to gcp {gcp_bucket_name=} {image_blob_name=}')
+        logger().info(f'uploading image from temporary location to gcp {gcp_bucket_name=} {image_blob_name=} ({size=})')
         gcp_bucket = storage_client.get_bucket(gcp_bucket_name)
         image_blob = gcp_bucket.blob(image_blob_name)
         image_blob.upload_from_file(
             tfh,
             content_type='application/x-xz',
+            size=size,
+            timeout=600, # allow for a longer upload timeout on slow connections
         )
         logger().info(f'uploaded image {raw_image_key=} to {image_blob_name=}')
         return image_blob
@@ -56,17 +68,23 @@ def delete_image_from_gcs_bucket(
     storage_client: google.cloud.storage.Client,
     release: glci.model.OnlineReleaseManifest,
     publishing_cfg: glci.model.PublishingTargetGCP,
+    dry_run: bool
 ):
     gcp_bucket_name = publishing_cfg.gcp_bucket_name
     image_blob_name = f'gardenlinux-{release.version}.tar.gz'
 
+    if dry_run:
+        logger().warning(f"DRY RUN: would delete {image_blob_name=} in {gcp_bucket_name=}")
+        return
+
     gcp_bucket = storage_client.get_bucket(gcp_bucket_name)
     image_blob = gcp_bucket.blob(image_blob_name)
     if image_blob.exists():
+        logger().info(f"deleting {image_blob_name=} in {gcp_bucket_name=}")
         image_blob.delete()
 
 
-def upload_image_from_gcp_store(
+def insert_image_to_gce_image_store(
     compute_client,
     image_blob: google.cloud.storage.blob.Blob,
     gcp_project_name: str,
@@ -148,18 +166,23 @@ def delete_image_from_gce_image_store(
     compute_client,
     gcp_project_name: str,
     release: glci.model.OnlineReleaseManifest,
+    dry_run: bool
 ) -> glci.model.OnlineReleaseManifest:
     image_name = _get_image_name_from_release_manifest(release)
 
     images = compute_client.images()
 
+    if dry_run:
+        logger().warning(f"DRY RUN: would delete {image_name=} in {gcp_project_name=}")
+        return
+
+    logger().info(f'deleting stale image {image_name=} from project {gcp_project_name=}')
+
     deletion_rq = images.delete(
         project=gcp_project_name,
         image=image_name,
     )
 
-    logger().info(f'deleting stale image {image_name=} from project {gcp_project_name=}')
-
     resp = deletion_rq.execute()
     op_name = resp['name']
 
@@ -182,7 +205,7 @@ def upload_and_publish_image(
     release: glci.model.OnlineReleaseManifest,
     publishing_cfg: glci.model.PublishingTargetGCP,
 ):
-    image_blob = upload_image_to_gcp_store(
+    image_blob = upload_image_to_gcs_bucket(
         storage_client=storage_client,
         s3_client=s3_client,
         release=release,
@@ -192,7 +215,7 @@ def upload_and_publish_image(
     release_manifest = None
 
     try:
-        release_manifest = upload_image_from_gcp_store(
+        release_manifest = insert_image_to_gce_image_store(
             compute_client=compute_client,
             image_blob=image_blob,
             gcp_project_name=gcp_project_name,
@@ -206,7 +229,7 @@ def upload_and_publish_image(
                 gcp_project_name=gcp_project_name,
                 release=release,
             )
-            release_manifest = upload_image_from_gcp_store(
+            release_manifest = insert_image_to_gce_image_store(
                 compute_client=compute_client,
                 image_blob=image_blob,
                 gcp_project_name=gcp_project_name,
@@ -222,17 +245,20 @@ def cleanup_image(
     gcp_project_name: str,
     release: glci.model.OnlineReleaseManifest,
     publishing_cfg: glci.model.PublishingTargetGCP,
+    dry_run: bool
 ):
     delete_image_from_gce_image_store(
         compute_client=compute_client,
         gcp_project_name=gcp_project_name,
         release=release,
+        dry_run=dry_run
     )
 
     delete_image_from_gcs_bucket(
         storage_client=storage_client,
         release=release,
         publishing_cfg=publishing_cfg,
+        dry_run=dry_run
     )
 
 

publishing-cfg.yaml

@@ -110,3 +110,6 @@
       hyper_v_generations: ['V1']
       publish_to_marketplace: false
       publish_to_community_galleries: true
+    - platform: 'gcp'
+      gcp_cfg_name: 'gardenlinux-integration-test'
+      gcp_bucket_name: 'gardenlinux-test-images'