diff --git a/.github/workflows/e2e-suite.yml b/.github/workflows/e2e-suite.yml index 27f32a592..0a7edcdfb 100644 --- a/.github/workflows/e2e-suite.yml +++ b/.github/workflows/e2e-suite.yml @@ -47,16 +47,11 @@ jobs: env: LINODE_CLI_TOKEN: ${{ secrets.LINODE_TOKEN }} - - name: Set release version env - run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV - - name: Add additional information to XML report run: | - echo $RELEASE_VERSION - echo ${{ env.RELEASE_VERSION }} filename=$(ls | grep -E '^[0-9]{12}_cli_test_report\.xml$') python scripts/add_to_xml_test_report.py \ - --branch_name "${{ env.RELEASE_VERSION }}" \ + --branch_name "${GITHUB_REF#refs/*/}" \ --gha_run_id "$GITHUB_RUN_ID" \ --gha_run_number "$GITHUB_RUN_NUMBER" \ --xmlfile "${filename}" diff --git a/.github/workflows/oci-build.yml b/.github/workflows/oci-build.yml index a64f0b776..1d4307467 100644 --- a/.github/workflows/oci-build.yml +++ b/.github/workflows/oci-build.yml @@ -16,6 +16,9 @@ jobs: with: python-version: '3.x' + - name: Install deps + run: make requirements + - name: Set up QEMU uses: docker/setup-qemu-action@2b82ce82d56a2a04d2637cd93a637ae1b359c0a7 # pin@v2.2.0 diff --git a/linodecli/plugins/obj/__init__.py b/linodecli/plugins/obj/__init__.py index a316c07ef..735b7757c 100644 --- a/linodecli/plugins/obj/__init__.py +++ b/linodecli/plugins/obj/__init__.py @@ -78,6 +78,18 @@ INVALID_PAGE_MSG = "No result to show in this page." +def get_available_cluster(cli: CLI): + """Get list of possible clusters for the account""" + return [ + c["id"] + for c in _do_get_request( # pylint: disable=protected-access + cli.config.base_url, + "/object-storage/clusters", + token=cli.config.get_token(), + )["data"] + ] + + def flip_to_page(iterable: Iterable, page: int = 1): """Given a iterable object and return a specific iteration (page)""" iterable = iter(iterable) @@ -451,7 +463,7 @@ def print_help(parser: ArgumentParser): print("See --help for individual commands for more information") -def get_obj_args_parser(): +def get_obj_args_parser(clusters: List[str]): """ Initialize and return the argument parser for the obj plug-in. """ @@ -468,6 +480,7 @@ def get_obj_args_parser(): "--cluster", metavar="CLUSTER", type=str, + choices=clusters, help="The cluster to use for the operation", ) @@ -527,7 +540,8 @@ def call( sys.exit(2) # requirements not met - we can't go on - parser = get_obj_args_parser() + clusters = get_available_cluster(context.client) + parser = get_obj_args_parser(clusters) parsed, args = parser.parse_known_args(args) # don't mind --no-defaults if it's there; the top-level parser already took care of it @@ -710,14 +724,7 @@ def _configure_plugin(client: CLI): """ Configures a default cluster value. """ - clusters = [ - c["id"] - for c in _do_get_request( # pylint: disable=protected-access - client.config.base_url, - "/object-storage/clusters", - token=client.config.get_value("token"), - )["data"] - ] + clusters = get_available_cluster(client) cluster = _default_thing_input( # pylint: disable=protected-access "Configure a default Cluster for operations.", diff --git a/linodecli/plugins/obj/config.py b/linodecli/plugins/obj/config.py index 1d5d56be4..8053b22c7 100644 --- a/linodecli/plugins/obj/config.py +++ b/linodecli/plugins/obj/config.py @@ -1,6 +1,7 @@ """ The config of the object storage plugin. """ +import shutil ENV_ACCESS_KEY_NAME = "LINODE_CLI_OBJ_ACCESS_KEY" ENV_SECRET_KEY_NAME = "LINODE_CLI_OBJ_SECRET_KEY" @@ -15,7 +16,8 @@ # for help commands PLUGIN_BASE = "linode-cli obj" -PROGRESS_BAR_WIDTH = 100 +columns = shutil.get_terminal_size(fallback=(80, 24)).columns +PROGRESS_BAR_WIDTH = columns - 20 if columns > 30 else columns # constant error messages NO_SCOPES_ERROR = """Your OAuth token isn't authorized to create Object Storage keys. diff --git a/linodecli/plugins/obj/helpers.py b/linodecli/plugins/obj/helpers.py index ed00bc964..60d4b2b6c 100644 --- a/linodecli/plugins/obj/helpers.py +++ b/linodecli/plugins/obj/helpers.py @@ -26,8 +26,9 @@ def __call__(self, bytes_amount: int): if not self.size: return self.uploaded += bytes_amount - percentage = self.bar_width * (self.uploaded / self.size) - progress = int(percentage) + percentage = 100 * (self.uploaded / self.size) + uploaded = self.bar_width * (self.uploaded / self.size) + progress = int(uploaded) progress_bar = ("#" * progress) + ("-" * (self.bar_width - progress)) print(f"\r |{progress_bar}| {percentage:.1f}%", end="\r") if self.uploaded == self.size: diff --git a/linodecli/plugins/obj/objects.py b/linodecli/plugins/obj/objects.py index d0c33f048..145aa8345 100644 --- a/linodecli/plugins/obj/objects.py +++ b/linodecli/plugins/obj/objects.py @@ -85,8 +85,14 @@ def upload_object( chunk_size = 1024 * 1024 * parsed.chunk_size + prefix = None + bucket = parsed.bucket + if "/" in parsed.bucket: + bucket = parsed.bucket.split("/")[0] + prefix = parsed.bucket.lstrip(f"{bucket}/") + upload_options = { - "Bucket": parsed.bucket, + "Bucket": bucket, "Config": TransferConfig(multipart_chunksize=chunk_size * MB), } @@ -95,8 +101,10 @@ def upload_object( for file_path in to_upload: print(f"Uploading {file_path.name}:") + upload_options["Key"] = ( + file_path.name if not prefix else f"{prefix}/{file_path.name}" + ) upload_options["Filename"] = str(file_path.resolve()) - upload_options["Key"] = file_path.name upload_options["Callback"] = ProgressPercentage( file_path.stat().st_size, PROGRESS_BAR_WIDTH ) diff --git a/scripts/add_to_xml_test_report.py b/scripts/add_to_xml_test_report.py index 41b4cef17..d30a7964a 100644 --- a/scripts/add_to_xml_test_report.py +++ b/scripts/add_to_xml_test_report.py @@ -1,11 +1,38 @@ import argparse import xml.etree.ElementTree as ET +import requests + +latest_release_url = "https://api.github.com/repos/linode/linode-cli/releases/latest" + + +def get_release_version(): + url = latest_release_url + + try: + response = requests.get(url) + response.raise_for_status() # Check for HTTP errors + + release_info = response.json() + version = release_info["tag_name"] + + # Remove 'v' prefix if it exists + if version.startswith("v"): + version = version[1:] + + return str(version) + + except requests.exceptions.RequestException as e: + print("Error:", e) + except KeyError: + print("Error: Unable to fetch release information from GitHub API.") + # Parse command-line arguments parser = argparse.ArgumentParser(description='Modify XML with workflow information') parser.add_argument('--branch_name', required=True) parser.add_argument('--gha_run_id', required=True) parser.add_argument('--gha_run_number', required=True) +parser.add_argument('--release_tag', required=False) parser.add_argument('--xmlfile', required=True) # Added argument for XML file path args = parser.parse_args() @@ -25,10 +52,14 @@ gha_run_number_element = ET.Element('gha_run_number') gha_run_number_element.text = args.gha_run_number +gha_release_tag_element = ET.Element('release_tag') +gha_release_tag_element.text = get_release_version() + # Add the new elements to the root of the XML root.append(branch_name_element) root.append(gha_run_id_element) root.append(gha_run_number_element) +root.append(gha_release_tag_element) # Save the modified XML modified_xml_file_path = xml_file_path # Overwrite it diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index bb137a005..cc8595c23 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -374,3 +374,31 @@ def nodebalancer_with_default_conf(): res_arr = result.split(",") nodebalancer_id = res_arr[0] delete_target_id(target="nodebalancers", id=nodebalancer_id) + + +def get_regions_with_capabilities(capabilities): + regions = ( + exec_test_command( + [ + "linode-cli", + "regions", + "ls", + "--text", + "--no-headers", + "--format=id,capabilities", + ] + ) + .stdout.decode() + .rstrip() + ) + + regions = regions.split("\n") + + regions_with_all_caps = [] + + for region in regions: + region_name = region.split()[0] + if all(capability in region for capability in capabilities): + regions_with_all_caps.append(region_name) + + return regions_with_all_caps diff --git a/tests/integration/obj/test_obj_plugin.py b/tests/integration/obj/test_obj_plugin.py index d8f37e572..55bf6226f 100644 --- a/tests/integration/obj/test_obj_plugin.py +++ b/tests/integration/obj/test_obj_plugin.py @@ -1,3 +1,4 @@ +import json import logging from dataclasses import dataclass from typing import Callable, Optional @@ -6,16 +7,16 @@ import requests from pytest import MonkeyPatch -from linodecli.configuration.auth import _do_request from linodecli.plugins.obj import ( ENV_ACCESS_KEY_NAME, ENV_SECRET_KEY_NAME, TRUNCATED_MSG, ) from tests.integration.fixture_types import GetTestFilesType, GetTestFileType -from tests.integration.helpers import BASE_URL, count_lines, exec_test_command +from tests.integration.helpers import count_lines, exec_test_command REGION = "us-southeast-1" +CLI_CMD = ["linode-cli", "object-storage"] BASE_CMD = ["linode-cli", "obj", "--cluster", REGION] @@ -50,27 +51,24 @@ def static_site_error(): @pytest.fixture(scope="session") -def keys(token: str): - response = _do_request( - BASE_URL, - requests.post, - "object-storage/keys", - token, - False, - {"label": "cli-integration-test-obj-key"}, - ) - +def keys(): + response = json.loads( + exec_test_command( + CLI_CMD + + [ + "keys-create", + "--label", + "cli-integration-test-obj-key", + "--json", + ], + ).stdout.decode() + )[0] _keys = Keys( access_key=response.get("access_key"), secret_key=response.get("secret_key"), ) yield _keys - _do_request( - BASE_URL, - requests.delete, - f"object-storage/keys/{response['id']}", - token, - ) + exec_test_command(CLI_CMD + ["keys-delete", str(response.get("id"))]) def patch_keys(keys: Keys, monkeypatch: MonkeyPatch): @@ -150,6 +148,51 @@ def test_obj_single_file_single_bucket( assert f1.read() == f2.read() +def test_obj_single_file_single_bucket_with_prefix( + create_bucket: Callable[[Optional[str]], str], + generate_test_files: GetTestFilesType, + keys: Keys, + monkeypatch: MonkeyPatch, +): + patch_keys(keys, monkeypatch) + file_path = generate_test_files()[0] + bucket_name = create_bucket() + exec_test_command( + BASE_CMD + ["put", str(file_path), f"{bucket_name}/prefix"] + ) + process = exec_test_command(BASE_CMD + ["la"]) + output = process.stdout.decode() + + assert f"{bucket_name}/prefix/{file_path.name}" in output + + file_size = file_path.stat().st_size + assert str(file_size) in output + + process = exec_test_command(BASE_CMD + ["ls"]) + output = process.stdout.decode() + assert bucket_name in output + assert file_path.name not in output + + process = exec_test_command(BASE_CMD + ["ls", bucket_name]) + output = process.stdout.decode() + assert bucket_name not in output + assert "prefix" in output + + downloaded_file_path = file_path.parent / f"downloaded_{file_path.name}" + process = exec_test_command( + BASE_CMD + + [ + "get", + bucket_name, + "prefix/" + file_path.name, + str(downloaded_file_path), + ] + ) + output = process.stdout.decode() + with open(downloaded_file_path) as f2, open(file_path) as f1: + assert f1.read() == f2.read() + + def test_multi_files_multi_bucket( create_bucket: Callable[[Optional[str]], str], generate_test_files: GetTestFilesType, diff --git a/tests/integration/vpc/__init__.py b/tests/integration/vpc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/integration/vpc/conftest.py b/tests/integration/vpc/conftest.py new file mode 100644 index 000000000..2736362fc --- /dev/null +++ b/tests/integration/vpc/conftest.py @@ -0,0 +1,99 @@ +import time + +import pytest + +from tests.integration.conftest import get_regions_with_capabilities +from tests.integration.helpers import delete_target_id, exec_test_command + + +@pytest.fixture +def test_vpc_w_subnet(): + region = get_regions_with_capabilities(["VPCs"])[0] + + vpc_label = str(time.time_ns()) + "label" + + subnet_label = str(time.time_ns()) + "label" + + vpc_id = ( + exec_test_command( + [ + "linode-cli", + "vpcs", + "create", + "--label", + vpc_label, + "--region", + region, + "--subnets.ipv4", + "10.0.0.0/24", + "--subnets.label", + subnet_label, + "--no-headers", + "--text", + "--format=id", + ] + ) + .stdout.decode() + .rstrip() + ) + + yield vpc_id + + delete_target_id(target="vpcs", id=vpc_id) + + +@pytest.fixture +def test_vpc_wo_subnet(): + region = get_regions_with_capabilities(["VPCs"])[0] + + label = str(time.time_ns()) + "label" + + vpc_id = ( + exec_test_command( + [ + "linode-cli", + "vpcs", + "create", + "--label", + label, + "--region", + region, + "--no-headers", + "--text", + "--format=id", + ] + ) + .stdout.decode() + .rstrip() + ) + + yield vpc_id + + delete_target_id(target="vpcs", id=vpc_id) + + +@pytest.fixture +def test_subnet(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + subnet_label = str(time.time_ns()) + "label" + res = ( + exec_test_command( + [ + "linode-cli", + "vpcs", + "subnet-create", + "--label", + subnet_label, + "--ipv4", + "10.0.0.0/24", + vpc_id, + "--text", + "--no-headers", + "--delimiter=,", + ] + ) + .stdout.decode() + .rstrip() + ) + + yield res, subnet_label diff --git a/tests/integration/vpc/test_vpc.py b/tests/integration/vpc/test_vpc.py new file mode 100644 index 000000000..ebb063b74 --- /dev/null +++ b/tests/integration/vpc/test_vpc.py @@ -0,0 +1,267 @@ +import re +import time + +from tests.integration.conftest import get_regions_with_capabilities +from tests.integration.helpers import ( + exec_failing_test_command, + exec_test_command, +) + +BASE_CMD = ["linode-cli", "vpcs"] + + +def test_list_vpcs(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + res = ( + exec_test_command(BASE_CMD + ["ls", "--text"]).stdout.decode().rstrip() + ) + headers = ["id", "label", "description", "region"] + + for header in headers: + assert header in res + assert vpc_id in res + + +def test_view_vpc(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + + res = ( + exec_test_command(BASE_CMD + ["view", vpc_id, "--text", "--no-headers"]) + .stdout.decode() + .rstrip() + ) + + assert vpc_id in res + + +def test_update_vpc(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + + new_label = str(time.time_ns()) + "label" + + updated_label = ( + exec_test_command( + BASE_CMD + + [ + "update", + vpc_id, + "--label", + new_label, + "--description", + "new description", + "--text", + "--no-headers", + "--format=label", + ] + ) + .stdout.decode() + .rstrip() + ) + + description = ( + exec_test_command( + BASE_CMD + + ["view", vpc_id, "--text", "--no-headers", "--format=description"] + ) + .stdout.decode() + .rstrip() + ) + + assert new_label == updated_label + assert "new description" in description + + +def test_list_subnets(test_vpc_w_subnet): + vpc_id = test_vpc_w_subnet + + res = ( + exec_test_command( + BASE_CMD + ["subnets-list", vpc_id, "--text", "--delimiter=,"] + ) + .stdout.decode() + .rstrip() + ) + + lines = res.splitlines() + + headers = ["id", "label", "ipv4", "linodes"] + + for header in headers: + assert header in lines[0] + + for line in lines[1:]: + assert re.match( + r"^(\d+),(\w+),(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}/\d+),", line + ), "String format does not match" + + +def test_view_subnet(test_vpc_wo_subnet, test_subnet): + # note calling test_subnet fixture will add subnet to test_vpc_wo_subnet + res, label = test_subnet + + res = res.split(",") + + vpc_subnet_id = res[0] + + vpc_id = test_vpc_wo_subnet + + output = ( + exec_test_command( + BASE_CMD + ["subnet-view", vpc_id, vpc_subnet_id, "--text"] + ) + .stdout.decode() + .rstrip() + ) + + headers = ["id", "label", "ipv4", "linodes"] + + for header in headers: + assert header in output + assert vpc_subnet_id in output + + +def test_update_subnet(test_vpc_w_subnet): + vpc_id = test_vpc_w_subnet + + new_label = str(time.time_ns()) + "label" + + subnet_id = ( + exec_test_command( + BASE_CMD + + ["subnets-list", vpc_id, "--text", "--format=id", "--no-headers"] + ) + .stdout.decode() + .rstrip() + ) + + updated_label = ( + exec_test_command( + BASE_CMD + + [ + "subnet-update", + vpc_id, + subnet_id, + "--label", + new_label, + "--text", + "--format=label", + "--no-headers", + ] + ) + .stdout.decode() + .rstrip() + ) + + assert new_label == updated_label + + +def test_fails_to_create_vpc_invalid_label(): + invalid_label = "invalid_label" + region = get_regions_with_capabilities(["VPCs"])[0] + + res = ( + exec_failing_test_command( + BASE_CMD + ["create", "--label", invalid_label, "--region", region] + ) + .stderr.decode() + .rstrip() + ) + + assert "Request failed: 400" in res + assert "Label must include only ASCII" in res + + +def test_fails_to_create_vpc_duplicate_label(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + label = ( + exec_test_command( + BASE_CMD + + ["view", vpc_id, "--text", "--no-headers", "--format=label"] + ) + .stdout.decode() + .rstrip() + ) + region = get_regions_with_capabilities(["VPCs"])[0] + + res = ( + exec_failing_test_command( + BASE_CMD + ["create", "--label", label, "--region", region] + ) + .stderr.decode() + .rstrip() + ) + + assert "Label must be unique among your VPCs" in res + + +def test_fails_to_update_vpc_invalid_label(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + invalid_label = "invalid_label" + + res = ( + exec_failing_test_command( + BASE_CMD + ["update", vpc_id, "--label", invalid_label] + ) + .stderr.decode() + .rstrip() + ) + + assert "Request failed: 400" in res + assert "Label must include only ASCII" in res + + +def test_fails_to_create_vpc_subnet_w_invalid_label(test_vpc_wo_subnet): + vpc_id = test_vpc_wo_subnet + invalid_label = "invalid_label" + region = get_regions_with_capabilities(["VPCs"])[0] + + res = exec_failing_test_command( + BASE_CMD + + [ + "subnet-create", + "--label", + invalid_label, + "--ipv4", + "10.1.0.0/24", + vpc_id, + ] + ).stderr.decode() + + assert "Request failed: 400" in res + assert "Label must include only ASCII" in res + + +def test_fails_to_update_vpc_subenet_w_invalid_label(test_vpc_w_subnet): + vpc_id = test_vpc_w_subnet + + invalid_label = "invalid_label" + + subnet_id = ( + exec_test_command( + BASE_CMD + + ["subnets-list", vpc_id, "--text", "--format=id", "--no-headers"] + ) + .stdout.decode() + .rstrip() + ) + + res = ( + exec_failing_test_command( + BASE_CMD + + [ + "subnet-update", + vpc_id, + subnet_id, + "--label", + invalid_label, + "--text", + "--format=label", + "--no-headers", + ] + ) + .stderr.decode() + .rstrip() + ) + + assert "Request failed: 400" in res + assert "Label must include only ASCII" in res diff --git a/tests/unit/test_plugin_obj.py b/tests/unit/test_plugin_obj.py index c7e67ca38..22dfaae31 100644 --- a/tests/unit/test_plugin_obj.py +++ b/tests/unit/test_plugin_obj.py @@ -1,10 +1,11 @@ from pytest import CaptureFixture +from linodecli import CLI from linodecli.plugins.obj import get_obj_args_parser, helpers, print_help -def test_print_help(capsys: CaptureFixture): - parser = get_obj_args_parser() +def test_print_help(mock_cli: CLI, capsys: CaptureFixture): + parser = get_obj_args_parser(["us-mia-1"]) print_help(parser) captured_text = capsys.readouterr() assert parser.format_help() in captured_text.out diff --git a/version b/version index f1fd18013..5aa51e513 100755 --- a/version +++ b/version @@ -10,21 +10,12 @@ from packaging.version import parse ENV_LINODE_CLI_VERSION = "LINODE_CLI_VERSION" -def get_version_env(): - return os.getenv(ENV_LINODE_CLI_VERSION) - - -def get_version(ref="HEAD"): +def get_version(): # We want to override the version if an environment variable is specified. # This is useful for certain release and testing pipelines. - version_str = get_version_env() or "0.0.0" - - # Strip the `v` prefix if specified - if version_str.startswith("v"): - version_str = version_str[1:] + version_str = os.getenv(ENV_LINODE_CLI_VERSION) or "0.0.0" return parse(version_str).release -major, minor, patch = get_version() -print("{}.{}.{}".format(major, minor, patch)) +print("{}.{}.{}".format(*get_version()))