From 46d7d4afdb67e3dfafa5a65a11b676e3277e9c44 Mon Sep 17 00:00:00 2001 From: Jen Patague Date: Thu, 19 Dec 2024 10:38:19 -0800 Subject: [PATCH 1/2] PAASTA-18479: Add list-namespaces tool --- paasta_tools/cli/cli.py | 1 + paasta_tools/cli/cmds/autoscale.py | 8 +- paasta_tools/cli/cmds/list_namespaces.py | 77 +++++++++++++++++++ paasta_tools/long_running_service_tools.py | 4 - tests/cli/test_cmds_list_namespaces.py | 88 ++++++++++++++++++++++ 5 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 paasta_tools/cli/cmds/list_namespaces.py create mode 100644 tests/cli/test_cmds_list_namespaces.py diff --git a/paasta_tools/cli/cli.py b/paasta_tools/cli/cli.py index 92f9dd222b..d9ecbe7961 100755 --- a/paasta_tools/cli/cli.py +++ b/paasta_tools/cli/cli.py @@ -110,6 +110,7 @@ def add_subparser(command, subparsers): "itest": "itest", "list-clusters": "list_clusters", "list-deploy-queue": "list_deploy_queue", + "list-namespaces": "list_namespaces", "list": "list", "local-run": "local_run", "logs": "logs", diff --git a/paasta_tools/cli/cmds/autoscale.py b/paasta_tools/cli/cmds/autoscale.py index 4f01d8e336..5d5503e87c 100644 --- a/paasta_tools/cli/cmds/autoscale.py +++ b/paasta_tools/cli/cmds/autoscale.py @@ -39,18 +39,20 @@ def add_subparser(subparsers): ) autoscale_parser.add_argument( - "-s", "--service", help="Service that you want to stop. Like 'example_service'." + "-s", + "--service", + help="Service that you want to autoscale. Like 'example_service'.", ).completer = lazy_choices_completer(list_services) autoscale_parser.add_argument( "-i", "--instance", - help="Instance of the service that you want to stop. Like 'main' or 'canary'.", + help="Instance of the service that you want to autoscale. Like 'main' or 'canary'.", required=True, ).completer = lazy_choices_completer(list_instances) autoscale_parser.add_argument( "-c", "--cluster", - help="The PaaSTA cluster that has the service instance you want to stop. Like 'pnw-prod'.", + help="The PaaSTA cluster that has the service instance you want to autoscale. Like 'pnw-prod'.", required=True, ).completer = lazy_choices_completer(list_clusters) autoscale_parser.add_argument( diff --git a/paasta_tools/cli/cmds/list_namespaces.py b/paasta_tools/cli/cmds/list_namespaces.py new file mode 100644 index 0000000000..fac36846cd --- /dev/null +++ b/paasta_tools/cli/cmds/list_namespaces.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# Copyright 2015-2016 Yelp Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from paasta_tools.cli.utils import get_instance_configs_for_service +from paasta_tools.cli.utils import lazy_choices_completer +from paasta_tools.cli.utils import validate_service_name +from paasta_tools.utils import DEFAULT_SOA_DIR +from paasta_tools.utils import list_clusters +from paasta_tools.utils import list_services + + +def add_subparser(subparsers) -> None: + list_parser = subparsers.add_parser( + "list-namespaces", + help="Lists all k8s namespaces used by instances of a service", + ) + list_parser.add_argument( + "-s", + "--service", + help="Name of the service which you want to list the namespaces for.", + required=True, + ).completer = lazy_choices_completer(list_services) + # Most services likely don't need to filter by cluster/instance, and can add namespaces from all instances + list_parser.add_argument( + "-i", + "--instance", + help="Instance of the service that you want to list namespaces for. Like 'main' or 'canary'.", + required=False, + ) + list_parser.add_argument( + "-c", + "--cluster", + help="Clusters that you want to list namespaces for. Like 'pnw-prod' or 'norcal-stagef'.", + required=False, + ).completer = lazy_choices_completer(list_clusters) + list_parser.add_argument( + "-y", + "-d", + "--soa-dir", + dest="soa_dir", + default=DEFAULT_SOA_DIR, + required=False, + help="define a different soa config directory", + ) + list_parser.set_defaults(command=paasta_list_namespaces) + + +def paasta_list_namespaces(args): + service = args.service + soa_dir = args.soa_dir + validate_service_name(service, soa_dir) + + namespaces = set() + instance_configs = get_instance_configs_for_service( + service=service, soa_dir=soa_dir, instances=args.instance, clusters=args.cluster + ) + for instance in instance_configs: + # We skip non-k8s instance types + if instance.get_instance_type() in ("paasta-native", "adhoc"): + continue + else: + namespaces.add(instance.get_namespace()) + + # Print in list format to be used in iam_roles + print(list(namespaces)) + return 0 diff --git a/paasta_tools/long_running_service_tools.py b/paasta_tools/long_running_service_tools.py index 3106481202..a4fcc0224e 100644 --- a/paasta_tools/long_running_service_tools.py +++ b/paasta_tools/long_running_service_tools.py @@ -149,10 +149,6 @@ def __init__( def get_bounce_method(self) -> str: raise NotImplementedError - def get_namespace(self) -> str: - """Get namespace from config""" - raise NotImplementedError - def get_kubernetes_namespace(self) -> str: """ Only needed on kubernetes LongRunningServiceConfig diff --git a/tests/cli/test_cmds_list_namespaces.py b/tests/cli/test_cmds_list_namespaces.py new file mode 100644 index 0000000000..eb9bb635dc --- /dev/null +++ b/tests/cli/test_cmds_list_namespaces.py @@ -0,0 +1,88 @@ +# tests/cli/cmds/test_list_namespaces.py +from mock import MagicMock +from mock import patch + +from paasta_tools.cli.cmds.list_namespaces import paasta_list_namespaces + + +def test_list_namespaces_no_instances(capfd): + mock_args = MagicMock( + service="fake_service", + instance=None, + cluster=None, + soa_dir="/fake/soa/dir", + ) + with patch( + "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service", + return_value=[], + autospec=True, + ), patch( + "paasta_tools.cli.cmds.list_namespaces.validate_service_name", + autospec=True, + ): + assert paasta_list_namespaces(mock_args) == 0 + stdout, _ = capfd.readouterr() + assert stdout.strip() == "[]" + + +def create_mock_instance_config(instance_type, namespace): + """ + Creates a mock InstanceConfig with specified instance_type and namespace. + + :param instance_type: The type of the instance (e.g., "kubernetes", "paasta-native"). + :param namespace: The namespace associated with the instance. + :return: A mock InstanceConfig object. + """ + mock_instance_config = MagicMock() + mock_instance_config.get_instance_type.return_value = instance_type + mock_instance_config.get_namespace.return_value = namespace + return mock_instance_config + + +def test_list_namespaces_with_instances(capfd): + mock_args = MagicMock( + service="fake_service", + instance=None, + cluster=None, + soa_dir="/fake/soa/dir", + ) + mock_instance_configs = [ + create_mock_instance_config("kubernetes", "k8s_namespace"), + create_mock_instance_config("kubernetes", "k8s_namespace"), + ] + + with patch( + "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service", + return_value=mock_instance_configs, + autospec=True, + ), patch( + "paasta_tools.cli.cmds.list_namespaces.validate_service_name", + autospec=True, + ): + assert paasta_list_namespaces(mock_args) == 0 + stdout, _ = capfd.readouterr() + assert stdout.strip() == "['k8s_namespace']" + + +def test_list_namespaces_skips_non_k8s_instances(capfd): + mock_args = MagicMock( + service="fake_service", + instance=None, + cluster=None, + soa_dir="/fake/soa/dir", + ) + + mock_k8s_instance_config = create_mock_instance_config("eks", "k8s_namespace") + mock_adhoc_instance_config = create_mock_instance_config("adhoc", None) + + with patch( + "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service", + return_value=[mock_k8s_instance_config, mock_adhoc_instance_config], + autospec=True, + ), patch( + "paasta_tools.cli.cmds.list_namespaces.validate_service_name", + autospec=True, + ): + assert paasta_list_namespaces(mock_args) == 0 + stdout, _ = capfd.readouterr() + assert stdout.strip() == "['k8s_namespace']" From b1f316c072bba11a80fcbc38b53ba62a823da7f9 Mon Sep 17 00:00:00 2001 From: Jen Patague Date: Thu, 19 Dec 2024 15:45:30 -0800 Subject: [PATCH 2/2] Handle spark too --- paasta_tools/cli/cmds/list_namespaces.py | 11 ++++- tests/cli/test_cmds_list_namespaces.py | 57 +++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/paasta_tools/cli/cmds/list_namespaces.py b/paasta_tools/cli/cmds/list_namespaces.py index fac36846cd..e5ed945925 100644 --- a/paasta_tools/cli/cmds/list_namespaces.py +++ b/paasta_tools/cli/cmds/list_namespaces.py @@ -15,6 +15,7 @@ from paasta_tools.cli.utils import get_instance_configs_for_service from paasta_tools.cli.utils import lazy_choices_completer from paasta_tools.cli.utils import validate_service_name +from paasta_tools.spark_tools import SPARK_EXECUTOR_NAMESPACE from paasta_tools.utils import DEFAULT_SOA_DIR from paasta_tools.utils import list_clusters from paasta_tools.utils import list_services @@ -69,8 +70,14 @@ def paasta_list_namespaces(args): # We skip non-k8s instance types if instance.get_instance_type() in ("paasta-native", "adhoc"): continue - else: - namespaces.add(instance.get_namespace()) + namespaces.add(instance.get_namespace()) + # Tron instances are TronActionConfigs + if ( + instance.get_instance_type() == "tron" + and instance.get_executor() == "spark" + ): + # We also need paasta-spark for spark executors + namespaces.add(SPARK_EXECUTOR_NAMESPACE) # Print in list format to be used in iam_roles print(list(namespaces)) diff --git a/tests/cli/test_cmds_list_namespaces.py b/tests/cli/test_cmds_list_namespaces.py index eb9bb635dc..eeb03910df 100644 --- a/tests/cli/test_cmds_list_namespaces.py +++ b/tests/cli/test_cmds_list_namespaces.py @@ -3,6 +3,7 @@ from mock import patch from paasta_tools.cli.cmds.list_namespaces import paasta_list_namespaces +from paasta_tools.spark_tools import SPARK_EXECUTOR_NAMESPACE def test_list_namespaces_no_instances(capfd): @@ -39,13 +40,14 @@ def create_mock_instance_config(instance_type, namespace): return mock_instance_config -def test_list_namespaces_with_instances(capfd): +def test_list_namespaces_with_instances_dupe_ns(capfd): mock_args = MagicMock( service="fake_service", instance=None, cluster=None, soa_dir="/fake/soa/dir", ) + mock_instance_configs = [ create_mock_instance_config("kubernetes", "k8s_namespace"), create_mock_instance_config("kubernetes", "k8s_namespace"), @@ -64,6 +66,59 @@ def test_list_namespaces_with_instances(capfd): assert stdout.strip() == "['k8s_namespace']" +def test_list_namespaces_tron(capfd): + mock_args = MagicMock( + service="fake_service", + instance=None, + cluster=None, + soa_dir="/fake/soa/dir", + ) + mock_tron_instance = create_mock_instance_config("tron", "tron") + mock_tron_instance.get_executor.return_value = "paasta" + + with patch( + "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service", + return_value=[mock_tron_instance], + autospec=True, + ), patch( + "paasta_tools.cli.cmds.list_namespaces.validate_service_name", + autospec=True, + ): + assert paasta_list_namespaces(mock_args) == 0 + stdout, _ = capfd.readouterr() + assert "['tron']" + + +def test_list_namespaces_spark(capfd): + mock_args = MagicMock( + service="fake_service", + instance=None, + cluster=None, + soa_dir="/fake/soa/dir", + ) + mock_spark_instance = create_mock_instance_config("tron", "tron") + mock_spark_instance.get_executor.return_value = "spark" + + mock_instance_configs = [ + create_mock_instance_config("kubernetes", "k8s_namespace"), + mock_spark_instance, + ] + + with patch( + "paasta_tools.cli.cmds.list_namespaces.get_instance_configs_for_service", + return_value=mock_instance_configs, + autospec=True, + ), patch( + "paasta_tools.cli.cmds.list_namespaces.validate_service_name", + autospec=True, + ): + assert paasta_list_namespaces(mock_args) == 0 + stdout, _ = capfd.readouterr() + assert f"'{SPARK_EXECUTOR_NAMESPACE}'" in stdout + assert "'tron'" in stdout + assert "'k8s_namespace'" in stdout + + def test_list_namespaces_skips_non_k8s_instances(capfd): mock_args = MagicMock( service="fake_service",