From 035f89df42200ca43f2d0eb9301265240a943bf4 Mon Sep 17 00:00:00 2001 From: a-dubs Date: Thu, 17 Oct 2024 11:46:32 -0400 Subject: [PATCH 1/4] feat(cloud): add two new public methods for uploading files and creating images --- pycloudlib/cloud.py | 91 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/pycloudlib/cloud.py b/pycloudlib/cloud.py index f75cd2f1..abf2f8c8 100644 --- a/pycloudlib/cloud.py +++ b/pycloudlib/cloud.py @@ -1,6 +1,7 @@ # This file is part of pycloudlib. See LICENSE file for license information. """Base class for all other clouds to provide consistent set of functions.""" +import dataclasses import enum import getpass import io @@ -38,6 +39,32 @@ class ImageType(enum.Enum): PRO_FIPS = "Pro FIPS" +@dataclasses.dataclass +class ImageInfo: + """Dataclass that represents an image on any given cloud.""" + + id: str + name: str + + def __str__(self): + """Return a human readable string representation of the image.""" + return f"{self.name} [id: {self.id}]" + + def __repr__(self): + """Return a string representation of the image.""" + return f"ImageInfo(id={self.id}, name={self.name})" + + def __eq__(self, other): + """Check if two ImageInfo objects are equal.""" + if not isinstance(other, ImageInfo): + return False + return self.id == other.id + + def __dict__(self): + """Return a dictionary representation of the image.""" + return {"id": self.id, "name": self.name} + + class BaseCloud(ABC): """Base Cloud Class.""" @@ -371,3 +398,67 @@ def _get_ssh_keys( private_key_path=private_key_path, name=name, ) + + # all actual "Clouds" and not just substrates like LXD and QEMU should support this method + def upload_local_file_to_cloud_storage( + self, + *, + local_file_path: str, + storage_name: str, + remote_file_name: Optional[str] = None, + ) -> str: + """ + Upload a file to a storage destination on the Cloud. + + Args: + local_file_path: The local file path of the image to upload. + storage_name: The name of the storage destination on the Cloud to upload the file to. + remote_file_name: The name of the file in the storage destination. If not provided, + the base name of the local file path will be used. + + Returns: + str: URL of the uploaded file in the storage destination. + """ + raise NotImplementedError + + # most clouds except for like lxd should support this method + def create_image_from_local_file( + self, + *, + local_file_path: str, + image_name: str, + intermediary_storage_name: str, + ): + """ + Upload local image file to storage on the Cloud and then create a custom image from it. + + Args: + local_file_path: The local file path of the image to upload. + image_name: The name to upload the image as and to register. + intermediary_storage_name: The intermediary storage destination on the Cloud to upload + the file to before creating the image. + + Returns: + ImageInfo: Information about the created image. + """ + raise NotImplementedError + + # not all clouds will support this method - depends on how image creation is handled + def _create_image_from_cloud_storage( + self, + *, + image_name: str, + remote_image_file_url: str, + image_description: Optional[str] = None, + ) -> ImageInfo: + """ + Register a custom image in the Cloud from a file in Cloud storage using its url. + + Ideally, this url would be returned from the upload_local_file_to_cloud_storage method. + + Args: + image_name: The name the image will be created with. + remote_image_file_url: The URL of the image file in the Cloud storage. + image_description: (Optional) A description of the image. + """ + raise NotImplementedError From 8cf94e864ac795fdcdd150f2a6bb207c81e241d5 Mon Sep 17 00:00:00 2001 From: a-dubs Date: Thu, 17 Oct 2024 11:46:53 -0400 Subject: [PATCH 2/4] feat(gce): implement both new methods for enabling file uploading and image creation --- examples/gce.py | 97 +++++++++++++++++++++++--- pycloudlib/gce/cloud.py | 151 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 237 insertions(+), 11 deletions(-) diff --git a/examples/gce.py b/examples/gce.py index 3eca925b..401f8a87 100755 --- a/examples/gce.py +++ b/examples/gce.py @@ -2,14 +2,17 @@ # This file is part of pycloudlib. See LICENSE file for license information. """Basic examples of various lifecycle with an GCE instance.""" +import argparse +import datetime import logging import os import pycloudlib -from pycloudlib.cloud import ImageType +from pycloudlib import GCE +from pycloudlib.cloud import ImageInfo, ImageType -def manage_ssh_key(gce): +def manage_ssh_key(gce: GCE): """Manage ssh keys for gce instances.""" pub_key_path = "gce-pubkey" priv_key_path = "gce-privkey" @@ -27,7 +30,7 @@ def manage_ssh_key(gce): gce.use_key(public_key_path=pub_key_path, private_key_path=priv_key_path) -def generic(gce): +def generic(gce: GCE): """Show example of using the GCE library. Connects to GCE and finds the latest daily image. Then runs @@ -39,7 +42,7 @@ def generic(gce): print(inst.execute("lsb_release -a")) -def pro(gce): +def pro(gce: GCE): """Show example of running a GCE PRO machine.""" daily = gce.daily_image("bionic", image_type=ImageType.PRO) with gce.launch(daily) as inst: @@ -47,7 +50,7 @@ def pro(gce): print(inst.execute("sudo ua status --wait")) -def pro_fips(gce): +def pro_fips(gce: GCE): """Show example of running a GCE PRO FIPS machine.""" daily = gce.daily_image("bionic", image_type=ImageType.PRO_FIPS) with gce.launch(daily) as inst: @@ -55,16 +58,92 @@ def pro_fips(gce): print(inst.execute("sudo ua status --wait")) -def demo(): +def custom_image(gce: GCE, image_name): + """Show example of running a GCE custom image.""" + image_id = gce.get_image_id_from_name(image_name) + print(image_id) + with gce.launch(image_id=image_id) as instance: + instance.wait() + print(instance.execute("hostname")) + input("Press Enter to teardown instance") + + +def upload_custom_image(gce: GCE, image_name, local_file_path, bucket_name): + """Show example of uploading a custom image to GCE.""" + new_image: ImageInfo = gce.create_image_from_local_file( + local_file_path=local_file_path, + image_name=image_name, + intermediary_storage_name=bucket_name, + ) + print("created new image:", new_image) + return new_image + + +def demo_image_creation( + local_file_path: str, + bucket_name: str, + image_name_template: str = "gce-example-image-{}", +): + """Show example of creating a custom image on GCE from a local image file.""" + # get short date and time for unique tag + time_tag = datetime.datetime.now().strftime("%Y%m%d-%H%M%S") + tag = f"gce-example-{time_tag}" + + with pycloudlib.GCE(tag=tag) as gce: + manage_ssh_key(gce) + + new_image: ImageInfo = upload_custom_image( + gce, + image_name=image_name_template.format(time_tag), + local_file_path=local_file_path, + bucket_name=bucket_name, + ) + + with gce.launch(new_image.id) as instance: + instance.wait() + print(instance.execute("hostname")) + print(instance.execute("lsb_release -a")) + input("Press Enter to teardown instance") + + +def demo_instances(): """Show examples of launching GCP instances.""" - logging.basicConfig(level=logging.DEBUG) with pycloudlib.GCE(tag="examples") as gce: manage_ssh_key(gce) - generic(gce) pro(gce) pro_fips(gce) +def main(): + """Take in cli args and run GCE demo scripts.""" + parser = argparse.ArgumentParser(description="GCE Demo Script") + parser.add_argument( + "demo_type", + choices=["instance", "image"], + help="Type of demo to run: 'instance' for basic instance launch demo, or " + "'image' for image creation demo", + nargs="?", + default="instance", + ) + parser.add_argument( + "--local_file_path", type=str, help="Local file path for the image creation demo" + ) + parser.add_argument("--bucket_name", type=str, help="Bucket name for the image creation demo") + args = parser.parse_args() + + logging.basicConfig(level=logging.DEBUG) + + if args.demo_type == "instance": + demo_instances() + elif args.demo_type == "image": + if not args.local_file_path or not args.bucket_name: + parser.error("Image creation demo requires --local_file_path and --bucket_name") + demo_image_creation( + local_file_path=args.local_file_path, + bucket_name=args.bucket_name, + ) + + if __name__ == "__main__": - demo() + main() diff --git a/pycloudlib/gce/cloud.py b/pycloudlib/gce/cloud.py index c5f4feff..6c6aab23 100644 --- a/pycloudlib/gce/cloud.py +++ b/pycloudlib/gce/cloud.py @@ -14,9 +14,9 @@ from google.api_core.exceptions import GoogleAPICallError from google.api_core.extended_operation import ExtendedOperation -from google.cloud import compute_v1 +from google.cloud import compute_v1, storage -from pycloudlib.cloud import BaseCloud, ImageType +from pycloudlib.cloud import BaseCloud, ImageInfo, ImageType from pycloudlib.config import ConfigFile from pycloudlib.errors import ( CloudSetupError, @@ -109,6 +109,7 @@ def __init__( self._instances_client = compute_v1.InstancesClient(credentials=credentials) self._zone_operations_client = compute_v1.ZoneOperationsClient(credentials=credentials) self._global_operations_client = compute_v1.GlobalOperationsClient(credentials=credentials) + self._storage_client = storage.Client(credentials=credentials) region = region or self.config.get("region") or "us-west2" zone = zone or self.config.get("zone") or "a" self.project = project @@ -498,3 +499,149 @@ def _wait_for_operation(self, operation, operation_type="global", sleep_seconds= "Expected DONE state, but found {} after waiting {} seconds. " "Check GCE console for more details. \n".format(response.status, sleep_seconds) ) + + def upload_local_file_to_cloud_storage( + self, + *, + local_file_path: str, + storage_name: str, + remote_file_name: Optional[str] = None, + ): + """ + Upload a file to a storage destination on the Cloud. + + Args: + local_file_path: The local file path of the image to upload. + storage_name: The name of the storage destination on the Cloud to upload the file to. + remote_file_name: The name of the file in the storage destination. If not provided, + the base name of the local file path will be used. + + Returns: + str: URL of the uploaded file in the storage destination. + """ + if not remote_file_name: + remote_file_name = os.path.basename(local_file_path) + + bucket = self._storage_client.bucket(storage_name) + blob = bucket.blob(remote_file_name) + + # Check if the file already exists in the destination bucket + if blob.exists(): + self._log.info( + f"File '{remote_file_name}' already exists in bucket '{storage_name}', " + "skipping upload." + ) + else: + self._log.info( + f"Uploading {local_file_path} to {remote_file_name} in bucket {storage_name}..." + ) + blob.upload_from_filename(local_file_path) + self._log.info(f"File {local_file_path} uploaded successfully to {remote_file_name}.") + + return f"http://storage.googleapis.com/{storage_name}/{remote_file_name}" + + def create_image_from_local_file( + self, + *, + local_file_path: str, + image_name: str, + intermediary_storage_name: str, + description: Optional[str] = None, + ) -> ImageInfo: + """ + Upload local image file to storage on the Cloud and then create a custom image from it. + + Args: + local_file_path: The local file path of the image to upload. + image_name: The name to upload the image as and to register. + intermediary_storage_name: The intermediary storage destination on the Cloud to upload + the file to before creating the image. + + Returns: + ImageInfo: Information about the created image. + """ + # Upload the image to GCS + remote_file_name = f"{image_name}.tar.gz" + self._log.info( + "Uploading image '%s' to '%s' as '%s'", + image_name, + intermediary_storage_name, + remote_file_name, + ) + gcs_image_path = self.upload_local_file_to_cloud_storage( + local_file_path=local_file_path, + storage_name=intermediary_storage_name, + remote_file_name=remote_file_name, + ) + # Register the custom image from GCS + return self._create_image_from_cloud_storage( + image_name=image_name, + remote_image_file_url=gcs_image_path, + ) + + def _create_image_from_cloud_storage( + self, + *, + image_name: str, + remote_image_file_url: str, + image_description: Optional[str] = None, + image_family: Optional[str] = None, + ) -> ImageInfo: + """ + Register a custom image in GCE from a file in GCE's Cloud storage using its url. + + Ideally, this url would be returned from the upload_local_file_to_cloud_storage method. + + Args: + image_name: The name the image will be created with. + remote_image_file_url: The URL of the image file in the Cloud storage. + image_description: (Optional) A description of the image. + image_family: (Optional) The family name of the image. + + Returns: + ImageInfo: Information about the created image. + """ + image_body = compute_v1.Image( + name=image_name, + raw_disk=compute_v1.RawDisk(source=remote_image_file_url), + description=image_description, + architecture="x86_64", + ) + if image_family: + image_body.family = image_family + + self._log.info( + "Attempting to register custom image '%s' from GCS path '%s'...", + image_name, + remote_image_file_url, + ) + + try: + operation = self._images_client.insert( + project=self.project, + image_resource=image_body, + ) + self._wait_for_operation(operation) + self._log.info(f"Custom image '{image_name}' registered successfully.") + + except Exception as e: + self._log.error(f"Failed to create custom image from GCS: {e}") + raise e + + return ImageInfo( + id=f"projects/{self.project}/global/images/{image_name}", + name=image_name, + ) + + def _wait_for_operation(self, operation: ExtendedOperation): + """ + Wait for a GCE operation to complete. + + Args: + operation: The operation to wait for. + """ + self._log.debug("Waiting for operation to complete...") + while not operation.done(): + self._log.debug("Operation is still in progress...") + time.sleep(5) + self._log.debug("Operation completed.") From 65746363ac5e69d9ede2bbafa3f58850ff12ac23 Mon Sep 17 00:00:00 2001 From: a-dubs Date: Thu, 9 Jan 2025 16:49:46 -0500 Subject: [PATCH 3/4] feat(oracle): add file upload and image creation functionality --- examples/oracle.py | 67 +++++++++++++--- pycloudlib/oci/cloud.py | 170 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 226 insertions(+), 11 deletions(-) diff --git a/examples/oracle.py b/examples/oracle.py index 74f7cb8a..d4c0e106 100644 --- a/examples/oracle.py +++ b/examples/oracle.py @@ -2,8 +2,8 @@ # This file is part of pycloudlib. See LICENSE file for license information. """Basic examples of various lifecycle with an OCI instance.""" +import argparse import logging -import sys from base64 import b64encode import pycloudlib @@ -46,17 +46,64 @@ def demo( new_instance.wait() new_instance.execute("whoami") +def upload_and_create_image( + local_file_path: str, + image_name: str, + suite: str, + intermediary_storage_name: str, + availability_domain: str = None, + compartment_id: str = None, +): + """Upload a local .img file and create an image from it on OCI.""" + with pycloudlib.OCI( + "oracle-test", + availability_domain=availability_domain, + compartment_id=compartment_id, + ) as client: + image_info = client.create_image_from_local_file( + local_file_path=local_file_path, + image_name=image_name, + intermediary_storage_name=intermediary_storage_name, + suite=suite, + ) + print(f"Created image: {image_info}") + if __name__ == "__main__": + parser = argparse.ArgumentParser(description="OCI example script") + subparsers = parser.add_subparsers(dest="command") + + demo_parser = subparsers.add_parser("demo", help="Run the demo") + demo_parser.add_argument("--availability-domain", type=str, help="Availability domain", required=False) + demo_parser.add_argument("--compartment-id", type=str, help="Compartment ID", required=False) + demo_parser.add_argument("--vcn-name", type=str, help="VCN name", required=False) + + create_image_parser = subparsers.add_parser("create_image", help="Create an image from a local file") + create_image_parser.add_argument("--local-file-path", type=str, required=True, help="Local file path") + create_image_parser.add_argument("--image-name", type=str, required=True, help="Image name") + create_image_parser.add_argument("--intermediary-storage-name", type=str, required=True, help="Intermediary storage name") + create_image_parser.add_argument("--suite", type=str, help="Suite of the image. I.e. 'jammy'", required=True) + create_image_parser.add_argument("--availability-domain", type=str, help="Availability domain") + create_image_parser.add_argument("--compartment-id", type=str, help="Compartment ID") + + args = parser.parse_args() + logging.basicConfig(level=logging.DEBUG) - if len(sys.argv) != 3: - print( - "No arguments passed via command line. " - "Assuming values are set in pycloudlib configuration file." + + if args.command == "demo": + demo( + availability_domain=args.availability_domain, + compartment_id=args.compartment_id, + vcn_name=args.vcn_name, + ) + elif args.command == "create_image": + upload_and_create_image( + local_file_path=args.local_file_path, + image_name=args.image_name, + suite=args.suite, + intermediary_storage_name=args.intermediary_storage_name, + availability_domain=args.availability_domain, + compartment_id=args.compartment_id, ) - demo() else: - passed_availability_domain = sys.argv[1] - passed_compartment_id = sys.argv[2] - passed_vcn_name = sys.argv[3] if len(sys.argv) == 4 else None - demo(passed_availability_domain, passed_compartment_id, passed_vcn_name) + parser.print_help() diff --git a/pycloudlib/oci/cloud.py b/pycloudlib/oci/cloud.py index a3bb60e6..d0b6dce1 100644 --- a/pycloudlib/oci/cloud.py +++ b/pycloudlib/oci/cloud.py @@ -10,7 +10,7 @@ import oci -from pycloudlib.cloud import BaseCloud +from pycloudlib.cloud import BaseCloud, ImageInfo from pycloudlib.config import ConfigFile from pycloudlib.errors import ( CloudSetupError, @@ -367,3 +367,171 @@ def _validate_tag(tag: str): if rules_failed: raise InvalidTagNameError(tag=tag, rules_failed=rules_failed) + + # all actual "Clouds" and not just substrates like LXD and QEMU should support this method + def upload_local_file_to_cloud_storage( + self, + *, + local_file_path: str, + storage_name: str, + remote_file_name: Optional[str] = None, + overwrite_existing: bool = False, + ) -> str: + """ + Upload a file to a storage destination on the Cloud. + + Args: + local_file_path: The local file path of the image to upload. + storage_name: The name of the storage destination on the Cloud to upload the file to. + remote_file_name: The name of the file in the storage destination. If not provided, + the base name of the local file path will be used. + + Returns: + str: URL of the uploaded file in the storage destination. + """ + object_storage_client = oci.object_storage.ObjectStorageClient(self.oci_config) + namespace = object_storage_client.get_namespace().data + bucket_name = storage_name + remote_file_name = remote_file_name or os.path.basename(local_file_path) + + # if remote file name is missing the extension, add it from the local file path + if not remote_file_name.endswith(ext:="."+local_file_path.split('.')[-1]): + remote_file_name += ext + + # check if object already exists in the bucket + try: + object_storage_client.get_object( + namespace, + bucket_name, + remote_file_name + ) + if overwrite_existing: + self._log.warning( + f"Object {remote_file_name} already exists in the bucket {bucket_name}. " + "Overwriting it." + ) + else: + self._log.info( + "Skipping upload as the object already exists in the bucket." + ) + return f"https://objectstorage.{self.region}.oraclecloud.com/n/{namespace}/b/{bucket_name}/o/{remote_file_name}" + except oci.exceptions.ServiceError as e: + if e.status != 404: + raise e + + with open(local_file_path, 'rb') as file: + object_storage_client.put_object( + namespace, + bucket_name, + remote_file_name, + file + ) + + return f"https://objectstorage.{self.region}.oraclecloud.com/n/{namespace}/b/{bucket_name}/o/{remote_file_name}" + + def create_image_from_local_file( + self, + *, + local_file_path: str, + image_name: str, + intermediary_storage_name: str, + suite: str, + ) -> ImageInfo: + """ + Upload local image file to storage on the Cloud and then create a custom image from it. + + Args: + local_file_path: The local file path of the image to upload. + image_name: The name to upload the image as and to register. + intermediary_storage_name: The intermediary storage destination on the Cloud to upload + the file to before creating the image. + suite: The suite of the image to create. I.e. "noble", "jammy", or "focal". + + Returns: + ImageInfo: Information about the created image. + """ + remote_file_url = self.upload_local_file_to_cloud_storage( + local_file_path=local_file_path, + storage_name=intermediary_storage_name, + remote_file_name=image_name, + ) + return self._create_image_from_cloud_storage( + image_name=image_name, + remote_image_file_url=remote_file_url, + suite=suite, + ) + + def parse_remote_object_url(self, remote_object_url: str) -> tuple[str, str, str]: + """ + Parse the remote object URL to extract the namespace, bucket name, and object name. + + Args: + remote_object_url: The URL of the object in the Cloud storage. + + Returns: + tuple[str, str, str]: The namespace, bucket name, and object name. + """ + if not remote_object_url.startswith("https://objectstorage"): + raise ValueError("Invalid URL. Expected a URL from the Oracle Cloud object storage.") + parts = remote_object_url.split("/") + # find the "n/", "b/", and "o/" parts of the URL + namespace_index = parts.index("n") + 1 + bucket_index = parts.index("b") + 1 + object_index = parts.index("o") + 1 + return parts[namespace_index], parts[bucket_index], parts[object_index] + + def _create_image_from_cloud_storage( + self, + *, + image_name: str, + remote_image_file_url: str, + suite: str, + image_description: Optional[str] = None, + ) -> ImageInfo: + """ + Register a custom image in the Cloud from a file in Cloud storage using its url. + + Ideally, this url would be returned from the upload_local_file_to_cloud_storage method. + + Args: + image_name: The name the image will be created with. + remote_image_file_url: The URL of the image file in the Cloud storage. + image_description: (Optional) A description of the image. + """ + suites = { + "plucky": "25.10", + "oracular": "24.10", + "noble": "24.04", + "jammy": "22.04", + "focal": "20.04", + } + if suite not in suites: + raise ValueError(f"Invalid suite. Expected one of {list(suites.keys())}. Found: {suite}") + # parse object name and bucket name from the url + object_namespace, bucket_name, object_name = self.parse_remote_object_url(remote_image_file_url) + self._log.debug(f"Bucket name: {bucket_name}, Object name: {object_name}, Object namespace: {object_namespace}") + image_details = oci.core.models.CreateImageDetails( + compartment_id=self.compartment_id, + display_name=image_name, + image_source_details=oci.core.models.ImageSourceViaObjectStorageTupleDetails( + source_type="objectStorageTuple", + bucket_name=bucket_name, + object_name=object_name, + namespace_name=object_namespace, + operating_system="Canonical Ubuntu", + operating_system_version=suites[suite], + ), + launch_mode="PARAVIRTUALIZED", + freeform_tags={"Description": image_description} if image_description else None + ) + + image_data = self.compute_client.create_image(image_details).data + image_data = wait_till_ready( + func=self.compute_client.get_image, + current_data=image_data, + desired_state="AVAILABLE", + sleep_seconds=30*60, # 30 minutes since image creation can take a while + ) + + return ImageInfo(id=image_data.id, name=image_data.display_name) + From 105f30bb41b8fcc87f8841c2d2b06cc4fca67aa2 Mon Sep 17 00:00:00 2001 From: a-dubs Date: Thu, 13 Feb 2025 10:31:07 -0500 Subject: [PATCH 4/4] fixup! feat(gce): implement both new methods for enabling file uploading and image creation --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index 58bc6ebc..5ffcd83f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ install_requires = boto3 >= 1.14.20 botocore >= 1.17.20 google-cloud-compute + google-cloud-storage googleapis-common-protos >= 1.63.1 ibm-cloud-sdk-core >= 3.14.0 ibm-platform-services