add cleanup code for GCP (#45)

This allows glci to be used in the Gardener integration test pipeline for GCP as well.
While at it, the names of some functions related to GCP were changed to make them
more descriptive.

cleanup.py

@@ -39,7 +39,7 @@ def cleanup_image(
     elif release.platform == 'aws':
         cleanup_function = cleanup_aws_images_by_id
     elif release.platform == 'gcp':
-        cleanup_function = None # cleanup_gcp_images
+        cleanup_function = cleanup_gcp_images
     elif release.platform == 'azure':
         cleanup_function = cleanup_azure_community_gallery_images
     elif release.platform == 'openstack':
@@ -121,6 +121,7 @@ def clean_alicloud_images(
 def cleanup_gcp_images(
     release: gm.OnlineReleaseManifest,
     publishing_cfg: gm.PublishingCfg,
+    dry_run: bool = False
 ) -> gm.OnlineReleaseManifest:
     gcp_publishing_cfg: gm.PublishingTargetGCP = publishing_cfg.target(release.platform)
     cfg_factory = ci.util.ctx().cfg_factory()
@@ -134,6 +135,7 @@ def cleanup_gcp_images(
         gcp_project_name=gcp_cfg.project(),
         release=release,
         publishing_cfg=gcp_publishing_cfg,
+        dry_run=dry_run
     )
 
 

glci/gcp.py

@@ -2,6 +2,7 @@
 import tempfile
 import time
 import logging
+import os
 
 import google.cloud.storage.blob
 import google.cloud.storage.client
@@ -13,7 +14,7 @@
 logger = lambda: logging.getLogger(__name__)
 
 
-def upload_image_to_gcp_store(
+def upload_image_to_gcs_bucket(
     storage_client: google.cloud.storage.Client,
     s3_client,
     release: glci.model.OnlineReleaseManifest,
@@ -31,22 +32,33 @@ def upload_image_to_gcp_store(
 
     # XXX: rather do streaming
     with tempfile.TemporaryFile() as tfh:
-        logger().info(f'downloading image from {s3_bucket_name=}')
+        resp = s3_client.get_object(
+            Bucket=s3_bucket_name,
+            Key=raw_image_key,
+        )
+        size = resp['ContentLength']
+        logger().info(f'downloading image from {s3_bucket_name=} to temporary location ({size=})')
+
         s3_client.download_fileobj(
             Bucket=s3_bucket_name,
             Key=raw_image_key,
             Fileobj=tfh,
         )
         logger().info(f'downloaded image from {s3_bucket_name=}')
 
+        # get the size of the temp file on local disk
+        tfh.seek(0, os.SEEK_END)
+        size = tfh.tell()
         tfh.seek(0)
 
-        logger().info(f're-uploading image to gcp {gcp_bucket_name=} {image_blob_name=}')
+        logger().info(f'uploading image from temporary location to gcp {gcp_bucket_name=} {image_blob_name=} ({size=})')
         gcp_bucket = storage_client.get_bucket(gcp_bucket_name)
         image_blob = gcp_bucket.blob(image_blob_name)
         image_blob.upload_from_file(
             tfh,
             content_type='application/x-xz',
+            size=size,
+            timeout=600, # allow for a longer upload timeout on slow connections
         )
         logger().info(f'uploaded image {raw_image_key=} to {image_blob_name=}')
         return image_blob
@@ -56,17 +68,23 @@ def delete_image_from_gcs_bucket(
     storage_client: google.cloud.storage.Client,
     release: glci.model.OnlineReleaseManifest,
     publishing_cfg: glci.model.PublishingTargetGCP,
+    dry_run: bool
 ):
     gcp_bucket_name = publishing_cfg.gcp_bucket_name
     image_blob_name = f'gardenlinux-{release.version}.tar.gz'
 
+    if dry_run:
+        logger().warning(f"DRY RUN: would delete {image_blob_name=} in {gcp_bucket_name=}")
+        return
+
     gcp_bucket = storage_client.get_bucket(gcp_bucket_name)
     image_blob = gcp_bucket.blob(image_blob_name)
     if image_blob.exists():
+        logger().info(f"deleting {image_blob_name=} in {gcp_bucket_name=}")
         image_blob.delete()
 
 
-def upload_image_from_gcp_store(
+def insert_image_to_gce_image_store(
     compute_client,
     image_blob: google.cloud.storage.blob.Blob,
     gcp_project_name: str,
@@ -148,18 +166,23 @@ def delete_image_from_gce_image_store(
     compute_client,
     gcp_project_name: str,
     release: glci.model.OnlineReleaseManifest,
+    dry_run: bool
 ) -> glci.model.OnlineReleaseManifest:
     image_name = _get_image_name_from_release_manifest(release)
 
     images = compute_client.images()
 
+    if dry_run:
+        logger().warning(f"DRY RUN: would delete {image_name=} in {gcp_project_name=}")
+        return
+
+    logger().info(f'deleting stale image {image_name=} from project {gcp_project_name=}')
+
     deletion_rq = images.delete(
         project=gcp_project_name,
         image=image_name,
     )
 
-    logger().info(f'deleting stale image {image_name=} from project {gcp_project_name=}')
-
     resp = deletion_rq.execute()
     op_name = resp['name']
 
@@ -182,7 +205,7 @@ def upload_and_publish_image(
     release: glci.model.OnlineReleaseManifest,
     publishing_cfg: glci.model.PublishingTargetGCP,
 ):
-    image_blob = upload_image_to_gcp_store(
+    image_blob = upload_image_to_gcs_bucket(
         storage_client=storage_client,
         s3_client=s3_client,
         release=release,
@@ -192,7 +215,7 @@ def upload_and_publish_image(
     release_manifest = None
 
     try:
-        release_manifest = upload_image_from_gcp_store(
+        release_manifest = insert_image_to_gce_image_store(
             compute_client=compute_client,
             image_blob=image_blob,
             gcp_project_name=gcp_project_name,
@@ -206,7 +229,7 @@ def upload_and_publish_image(
                 gcp_project_name=gcp_project_name,
                 release=release,
             )
-            release_manifest = upload_image_from_gcp_store(
+            release_manifest = insert_image_to_gce_image_store(
                 compute_client=compute_client,
                 image_blob=image_blob,
                 gcp_project_name=gcp_project_name,
@@ -222,17 +245,20 @@ def cleanup_image(
     gcp_project_name: str,
     release: glci.model.OnlineReleaseManifest,
     publishing_cfg: glci.model.PublishingTargetGCP,
+    dry_run: bool
 ):
     delete_image_from_gce_image_store(
         compute_client=compute_client,
         gcp_project_name=gcp_project_name,
         release=release,
+        dry_run=dry_run
     )
 
     delete_image_from_gcs_bucket(
         storage_client=storage_client,
         release=release,
         publishing_cfg=publishing_cfg,
+        dry_run=dry_run
     )
 
 

publishing-cfg.yaml

@@ -110,3 +110,6 @@
       hyper_v_generations: ['V1']
       publish_to_marketplace: false
       publish_to_community_galleries: true
+    - platform: 'gcp'
+      gcp_cfg_name: 'gardenlinux-integration-test'
+      gcp_bucket_name: 'gardenlinux-test-images'