adding create namespace fxnality to reckoner to replace it in helm3 (#…

…165)

* adding create namespace fxnality to reckoner to replace it in helm3

* fixing lack of install of kuberentes module, adding a test for the new option

* adding docs

* testing things and stuff

* phew, finally found that errror

* removing debug line

* making suggested chnanges

CHANGELOG.md

@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## Unreleased
 ### Fixed
 - bug where the helm client version check would fail for helm2 and never proceed to check helm3
+- added create namespace functionality because they removed it from helm3
 
 ## [2.2.0]
 ### Changes

docs/README.md

@@ -154,23 +154,26 @@ Commands:
 You can add `--help` to any `Command` and get output like the one below:
 ```text
 $> reckoner plot --help
-
 Usage: reckoner plot [OPTIONS] COURSE_FILE
 
   Install charts with given arguments as listed in yaml file argument
 
 Options:
-  --dry-run                      Pass --dry-run to helm so no action is taken.
-                                 Implies --debug and skips hooks.
-  --debug                        DEPRECATED - use --dry-run instead, or pass
-                                 to --helm-args
-  -o, --only, --heading <chart>  Only run a specific chart by name
-  --helm-args TEXT               Passes the following arg on to helm, can be
-                                 used more than once. WARNING: Setting this
-                                 will completely override any helm_args in the
-                                 course. Also cannot be used for configuring
-                                 how helm connects to tiller.
-  --continue-on-error            Attempt to install all charts in the course,
-                                 even if any charts or hooks fail to run.
-  --help                         Show this message and exit.
+  --dry-run                       Pass --dry-run to helm so no action is
+                                  taken. Implies --debug and skips hooks.
+  --debug                         DEPRECATED - use --dry-run instead, or pass
+                                  to --helm-args
+  -o, --only, --heading <chart>   Only run a specific chart by name
+  --helm-args TEXT                Passes the following arg on to helm, can be
+                                  used more than once. WARNING: Setting this
+                                  will completely override any helm_args in
+                                  the course. Also cannot be used for
+                                  configuring how helm connects to tiller.
+  --continue-on-error             Attempt to install all charts in the course,
+                                  even if any charts or hooks fail to run.
+  --create-namespace / --no-create-namespace
+                                  Will create the specified nameaspace if it
+                                  does not already exist. Replaces
+                                  functionality lost in Helm3
+  --help                          Show this message and exit.
 ```

end_to_end_testing/run_end_to_end.sh

@@ -15,6 +15,7 @@ set -o errtrace
 E2E_FAILED_TESTS=false
 E2E_FAILED_MESSAGES=()
 E2E_SKIPPED_MESSAGES=()
+HELM_VERSION="${HELM_VERSION:-2}"
 
 function print_status_end_exit() {
     if [ "${#E2E_SKIPPED_MESSAGES[@]}" -gt 0 ]; then echo -e "* * *\nSkipped Tests:"; fi
@@ -151,6 +152,28 @@ function helm_release_key_value_is_type() {
     fi
 }
 
+function e2e_test_namespace_creation_flag_on_chart_install() {
+    if [ "${HELM_VERSION}" -eq "3" ]; then
+        if reckoner plot --no-create-namespace test_create_namespace.yml; then
+            mark_failed "${FUNCNAME[0]}" "With --no-create-namespace set, this should have failed"
+        fi
+
+        if helm_has_release_name_in_namespace "namespace-test" "farglebargle"; then
+            mark_failed "${FUNCNAME[0]}" "Found namespace_test in farglebargle namespace after install and should not have"
+        fi
+
+        if ! reckoner plot test_create_namespace.yml; then
+            mark_failed "${FUNCNAME[0]}" "Without --no-create-namespace set, this should not have failed"
+        fi
+
+        if ! helm_has_release_name_in_namespace "namespace-test" "farglebargle"; then
+            mark_failed "${FUNCNAME[0]}" "Did not find namespace_test in farglebargle namespace after install."
+        fi
+    fi
+
+
+}
+
 function e2e_test_basic_chart_install() {
     if ! reckoner plot test_basic.yml; then
         mark_failed "${FUNCNAME[0]}" "Plot had a bad exit code"

end_to_end_testing/setup_helm2.sh

@@ -1,7 +1,7 @@
 source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/setup_common.sh
 
 echo "Installing Helm"
-curl -sL https://storage.googleapis.com/kubernetes-helm/helm-v2.14.2-linux-amd64.tar.gz | tar xzv linux-amd64/helm
+curl -sL https://storage.googleapis.com/kubernetes-helm/helm-v2.16.1-linux-amd64.tar.gz | tar xzv linux-amd64/helm
 sudo mv linux-amd64/helm /usr/local/bin/helm
 rm -rf linux-amd64
 helm version --client

end_to_end_testing/setup_helm3.sh

@@ -1,17 +1,9 @@
 source "$( cd -P "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"/setup_common.sh
 
 echo "Installing Helm"
-curl -sL https://get.helm.sh/helm-v3.0.1-linux-amd64.tar.gz | tar xzv linux-amd64/helm
+curl -sL https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz | tar xzv linux-amd64/helm
 sudo mv linux-amd64/helm /usr/local/bin/helm
 rm -rf linux-amd64
 helm version
 
-kubectl create namespace infra
-kubectl create namespace test
-kubectl create namespace testing
-kubectl create namespace polaris
-kubectl create namespace another-polaris
-kubectl create namespace a-different-one
-kubectl create namespace redis-test-namespace
-
 helm repo add stable https://kubernetes-charts.storage.googleapis.com

end_to_end_testing/test_create_namespace.yml

@@ -0,0 +1,18 @@
+namespace: farglebargle #namespace to install the chart in, defaults to 'kube-system'
+repositories:
+  test_repo:
+    url: https://kubernetes-charts.storage.googleapis.com
+  incubator:
+    url: https://kubernetes-charts-incubator.storage.googleapis.com
+  fairwinds-stable:
+    url: https://charts.fairwinds.com/stable
+  fairwinds-incubator:
+    url: https://charts.fairwinds.com/incubator
+minimum_versions: #set minimum version requirements here
+  helm: 0.0.0
+  reckoner: 0.0.0
+charts:
+  namespace-test:
+    repository: stable
+    chart: nginx-ingress
+    namespace: farglebargle

reckoner/chart.py

@@ -16,6 +16,8 @@
 
 import logging
 import os
+
+from .kube import create_namespace, list_namespace_names
 from tempfile import NamedTemporaryFile as tempfile
 from .yaml.handler import Handler as yaml_handler
 
@@ -26,10 +28,12 @@
 from .repository import Repository
 from .command_line_caller import call
 
+
 default_repository = {'name': 'stable', 'url': 'https://kubernetes-charts.storage.googleapis.com'}
 
 
 class ChartResult:
+
     def __init__(self, name: str, failed: bool, error_reason: str):
         self.name = name
         self.failed = failed
@@ -238,6 +242,21 @@ def update_dependencies(self):
             except ReckonerCommandException as error:
                 logging.warn("Unable to update chart dependencies: {}".format(error.stderr))
 
+    def create_namespace_if_needed(self):
+        """ Creates the charts specified namespace if it does not already exist
+        Requires `self.config.create_namespace` to true. Caches the existing namespace list
+        in the self.config option to avoid going back to the api for each chart. 
+        """
+
+        if self.config.create_namespace:
+            if self.config.cluster_namespaces is None:
+                self.config.cluster_namespaces = list_namespace_names()
+
+            if self.namespace not in self.config.cluster_namespaces and not self.dryrun:
+                if create_namespace(self.namespace):
+                    logging.info('Namespace {} not found. Creating it now.'.format(self.namespace))
+                    self.config.cluster_namespaces.append(self.namespace)
+
     def install(self, namespace=None, context=None) -> None:
         """
         Description:
@@ -257,6 +276,8 @@ def install(self, namespace=None, context=None) -> None:
 
         # Try to run the install process for mark the result as failed
         try:
+            self.create_namespace_if_needed()
+
             # Fire the pre_install_hook
             self.pre_install_hook()
 

reckoner/cli.py

@@ -48,14 +48,16 @@ def cli(ctx, log_level, *args, **kwargs):
                                   'configuring how helm connects to tiller.', multiple=True)
 @click.option("--continue-on-error", is_flag=True, default=False,
               help="Attempt to install all charts in the course, even if any charts or hooks fail to run.")
-def plot(ctx, course_file=None, dry_run=False, debug=False, only=None, helm_args=None, continue_on_error=False):
+@click.option("--create-namespace/--no-create-namespace", default=True,
+              help="Will create the specified nameaspace if it does not already exist. Replaces functionality lost in Helm3")
+def plot(ctx, course_file=None, dry_run=False, debug=False, only=None, helm_args=None, continue_on_error=False, create_namespace=True):
     """ Install charts with given arguments as listed in yaml file argument """
     try:
         # Check Schema of Course FileA
         with open(course_file.name, 'rb') as course_file_stream:
             validate_course_file(course_file_stream)
         # Load Reckoner
-        r = Reckoner(course_file=course_file, dryrun=dry_run, debug=debug, helm_args=helm_args, continue_on_error=continue_on_error)
+        r = Reckoner(course_file=course_file, dryrun=dry_run, debug=debug, helm_args=helm_args, continue_on_error=continue_on_error, create_namespace=create_namespace)
         # Convert tuple to list
         only = list(only)
         r.install(only)

reckoner/kube.py

@@ -0,0 +1,47 @@
+
+import logging
+import traceback
+
+from kubernetes import client, config
+
+
+def create_namespace(namespace):
+    """ Create a namespace in the configured kubernetes cluster if it does not already exist
+
+    Arguments:
+
+    namespace: The namespace to create
+
+    Returns True on success
+    Raises error in case of failure
+
+    """
+    try:
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        response = v1.create_namespace(
+            client.V1Namespace(
+                metadata=client.V1ObjectMeta(name=namespace)
+            )
+        )
+        return True
+    except Exception as e:
+        logging.error("Unable to create namespace in cluster! {}".format(e))
+        logging.debug(traceback.format_exc())
+        raise e
+
+
+def list_namespace_names():
+    """ Lists namespaces in the configured kubernetes cluster.
+    No arguments
+    Returns list
+    """
+    try:
+        config.load_kube_config()
+        v1 = client.CoreV1Api()
+        namespaces = v1.list_namespace()
+        return [namespace.metadata.name for namespace in namespaces.items]
+    except Exception as e:
+        logging.error("Unable to get namespaces in cluster! {}".format(e))
+        logging.debug(traceback.format_exc())
+        raise e

reckoner/reckoner.py

@@ -63,13 +63,14 @@ class Reckoner(object):
 
     """
 
-    def __init__(self, course_file: BufferedReader = None, dryrun=False, debug=False, helm_args=None, continue_on_error=False):
+    def __init__(self, course_file: BufferedReader = None, dryrun=False, debug=False, helm_args=None, continue_on_error=False, create_namespace=True):
         self.config = Config()
         self.results = ReckonerInstallResults()
         self.config.dryrun = dryrun
         self.config.debug = debug
         self.config.helm_args = helm_args
         self.config.continue_on_error = continue_on_error
+        self.config.create_namespace = create_namespace
         if course_file:
             self.config.course_path = course_file.name
 

reckoner/tests/test_chart.py

@@ -29,6 +29,7 @@
 # would be more easily mockable
 @mock.patch('reckoner.chart.call')
 class TestChartHooks(unittest.TestCase):
+
     def get_chart(self, *args):
         chart = Chart(
             {'name': {
@@ -217,6 +218,8 @@ def test_interpolation_of_env_vars_kube_deploy_spec(self, environMock, chartConf
         chart._check_env_vars()
         self.assertEqual(chart.args[0], 'thing=$(environVar)')
 
+    @mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True))
+    @mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[]))
     @mock.patch('reckoner.chart.Config', autospec=True)
     @mock.patch('reckoner.chart.Repository')
     def test_chart_install(self, repositoryMock, chartConfigMock):
@@ -228,6 +231,8 @@ def test_chart_install(self, repositoryMock, chartConfigMock):
         chartConfig = chartConfigMock()
         chartConfig.course_base_directory = '.'
         chartConfig.dryrun = False
+        chartConfig.create_namespace = True
+        chartConfig.cluster_namespaces = []
 
         debug_args = mock.PropertyMock(debug_args=['fake'])
         type(chart).debug_args = debug_args
@@ -236,6 +241,8 @@ def test_chart_install(self, repositoryMock, chartConfigMock):
         upgrade_call = helm_client_mock.upgrade.call_args
         self.assertEqual(upgrade_call[0][0], ['nameofchart', '', '--namespace', 'fakenamespace'])
 
+    @mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True))
+    @mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[]))
     @mock.patch('reckoner.chart.Config', autospec=True)
     @mock.patch('reckoner.chart.Repository')
     def test_chart_install_with_plugin(self, repositoryMock, chartConfigMock):
@@ -247,6 +254,8 @@ def test_chart_install_with_plugin(self, repositoryMock, chartConfigMock):
         chartConfig = chartConfigMock()
         chartConfig.course_base_directory = '.'
         chartConfig.dryrun = False
+        chartConfig.create_namespace = True
+        chartConfig.cluster_namespaces = []
 
         debug_args = mock.PropertyMock(debug_args=['fake'])
         type(chart).debug_args = debug_args
@@ -258,6 +267,7 @@ def test_chart_install_with_plugin(self, repositoryMock, chartConfigMock):
 
 
 class TestChartResult(unittest.TestCase):
+
     def test_initialize(self):
         c = ChartResult(
             name="fake-result",

reckoner/tests/test_course.py

@@ -26,6 +26,7 @@
 @mock.patch('reckoner.course.get_helm_client', autospec=True)
 @mock.patch('reckoner.course.Config', autospec=True)
 class TestMinVersion(unittest.TestCase):
+
     def test_init_error_fails_min_version_reckoner(self, configMock, helmClientMock, yamlLoadMock, sysMock, repoMock):
         """Tests that minimum version will throw an exit."""
         c = configMock()
@@ -76,6 +77,9 @@ def test_init_error_fails_min_version_helm(self, configMock, helmClientMock, yam
 
 
 class TestIntegrationWithChart(unittest.TestCase):
+
+    @mock.patch('reckoner.chart.create_namespace', mock.MagicMock(return_value=True))
+    @mock.patch('reckoner.chart.list_namespace_names', mock.MagicMock(return_value=[]))
     @mock.patch('reckoner.chart.Config', autospec=True)
     @mock.patch('reckoner.chart.call', autospec=True)
     @mock.patch('reckoner.repository.Repository', autospec=True)
@@ -92,6 +96,9 @@ def test_failed_pre_install_hooks_fail_chart_installation(self, configMock, helm
         chartConfig = chartConfigMock()
         chartConfig.course_base_directory = '.'
         chartConfig.dryrun = False
+        chartConfig.create_namespace = True
+        chartConfig.cluster_namespaces = []
+
         h = helmClientMock(c.helm_args)
         h.client_version = '0.0.1'
 
@@ -119,6 +126,7 @@ def test_failed_pre_install_hooks_fail_chart_installation(self, configMock, helm
 @mock.patch('reckoner.course.yaml_handler', autospec=True)
 @mock.patch('reckoner.course.get_helm_client', autospec=True)
 class TestCourse(unittest.TestCase):
+
     def setUp(self):
         self.course_yaml = {
             'charts': {

setup.py

@@ -43,6 +43,7 @@
           "semver>=2.8.1",
           "ruamel.yaml>=0.16.0",
           "jsonschema>=3.0.2",
+          "kubernetes==10.0.1"
       ],
       entry_points=''' #for click integration
           [console_scripts]