From a31e6ac5adc21e0158a108984b6a12666d39e350 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 26 Oct 2016 12:47:07 +0100
Subject: [PATCH 01/26] Install kubernetes Ubuntu 16.04 packages

---
 admin/acceptance.py           | 94 ++++++++++++++++++++++++++++++++++-
 flocker/provision/_install.py | 61 +++++++++++++++++++++++
 2 files changed, 154 insertions(+), 1 deletion(-)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index 84f2d52787..dc6497dd1b 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -48,6 +48,7 @@
 )
 from flocker.provision._install import (
     ManagedNode,
+    install_kubernetes,
     task_pull_docker_images,
     uninstall_flocker,
     install_flocker,
@@ -335,6 +336,7 @@ def install(ignored):
                 install_flocker(nodes, package_source),
             )
         installing = uninstalling.addCallback(install)
+
         return installing
 
     def ensure_keys(self, reactor):
@@ -355,7 +357,7 @@ def start_cluster(self, reactor):
             )
         else:
             upgrading = succeed(None)
-
+        return upgrading
         def configure(ignored):
             return configured_cluster_for_nodes(
                 reactor,
@@ -388,6 +390,80 @@ def extend_cluster(self, reactor, cluster, count, tag, starting_index):
         raise UsageError("Extending a cluster with managed nodes "
                          "is not implemented yet.")
 
+@implementer(IClusterRunner)
+class KubernetesRunner(object):
+    """
+    """
+    def __init__(self, node_addresses, package_source, distribution,
+                 dataset_backend, dataset_backend_configuration, identity,
+                 cert_path, logging_config):
+        """
+        :param list: A ``list`` of public IP addresses or
+            ``[private_address, public_address]`` lists.
+
+        See ``ManagedRunner`` and ``ManagedNode`` for other parameter
+        documentation.
+        """
+        self._nodes = pvector(
+            make_managed_nodes(node_addresses, distribution)
+        )
+        self.package_source = package_source
+        self.dataset_backend = dataset_backend
+        self.dataset_backend_configuration = dataset_backend_configuration
+        self.identity = identity
+        self.cert_path = cert_path
+        self.logging_config = logging_config
+
+    def ensure_keys(self, reactor):
+        """
+        Assume we have keys, since there's no way of asking the nodes what keys
+        they'll accept.
+        """
+        return succeed(None)
+
+    def start_cluster(self, reactor):
+        """
+        Don't start any nodes.  Give back the addresses of the configured,
+        already-started nodes.
+        """
+        dispatcher = make_dispatcher(reactor)
+        installing = perform(
+            dispatcher,
+            install_kubernetes(self._nodes, self.package_source),
+        )
+
+        def configure(ignored):
+            return configured_cluster_for_nodes(
+                reactor,
+                generate_certificates(
+                    self.identity.name,
+                    self.identity.id,
+                    self._nodes,
+                    self.cert_path,
+                ),
+                self._nodes,
+                self.dataset_backend,
+                self.dataset_backend_configuration,
+                save_backend_configuration(
+                    self.dataset_backend,
+                    self.dataset_backend_configuration
+                ),
+                provider="managed",
+                logging_config=self.logging_config,
+            )
+        configuring = installing.addCallback(configure)
+        return configuring
+
+    def stop_cluster(self, reactor):
+        """
+        Don't stop any nodes.
+        """
+        return succeed(None)
+
+    def extend_cluster(self, reactor, cluster, count, tag, starting_index):
+        raise UsageError("Extending a cluster with managed nodes "
+                         "is not implemented yet.")
+
 
 def generate_certificates(cluster_name, cluster_id, nodes, cert_path):
     """
@@ -1094,6 +1170,22 @@ def _runner_MANAGED(self, package_source, dataset_backend,
             logging_config=self['config'].get('logging'),
         )
 
+    def _runner_KUBERNETES(self, package_source, dataset_backend,
+                           provider_config):
+        if provider_config is None:
+            self._provider_config_missing("kubernetes")
+
+        return KubernetesRunner(
+            node_addresses=provider_config['addresses'],
+            package_source=None,
+            distribution=self['distribution'],
+            dataset_backend=dataset_backend,
+            dataset_backend_configuration=self.dataset_backend_configuration(),
+            identity=self._make_cluster_identity(dataset_backend),
+            cert_path=self['cert-directory'],
+            logging_config=self['config'].get('logging'),
+        )
+
     def _libcloud_runner(self, package_source, dataset_backend,
                          provider, provider_config):
         """
diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 3e7f9304a9..d28535cb37 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1384,6 +1384,55 @@ def task_install_docker(distribution):
         timeout=5.0 * 60.0,
     )
 
+GOOGLE_CLOUD_PACKAGES_KEY = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFUd6rIBCAD6mhKRHDn3UrCeLDp7U5IE7AhhrOCPpqGF7mfTemZYHf/5Jdjx
+cOxoSFlK7zwmFr3lVqJ+tJ9L1wd1K6P7RrtaNwCiZyeNPf/Y86AJ5NJwBe0VD0xH
+TXzPNTqRSByVYtdN94NoltXUYFAAPZYQls0x0nUD1hLMlOlC2HdTPrD1PMCnYq/N
+uL/Vk8sWrcUt4DIS+0RDQ8tKKe5PSV0+PnmaJvdF5CKawhh0qGTklS2MXTyKFoqj
+XgYDfY2EodI9ogT/LGr9Lm/+u4OFPvmN9VN6UG+s0DgJjWvpbmuHL/ZIRwMEn/tp
+uneaLTO7h1dCrXC849PiJ8wSkGzBnuJQUbXnABEBAAG0QEdvb2dsZSBDbG91ZCBQ
+YWNrYWdlcyBBdXRvbWF0aWMgU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNv
+bT6JAT4EEwECACgFAlUd6rICGy8FCQWjmoAGCwkIBwMCBhUIAgkKCwQWAgMBAh4B
+AheAAAoJEDdGwginMXsPcLcIAKi2yNhJMbu4zWQ2tM/rJFovazcY28MF2rDWGOnc
+9giHXOH0/BoMBcd8rw0lgjmOosBdM2JT0HWZIxC/Gdt7NSRA0WOlJe04u82/o3OH
+WDgTdm9MS42noSP0mvNzNALBbQnlZHU0kvt3sV1YsnrxljoIuvxKWLLwren/GVsh
+FLPwONjw3f9Fan6GWxJyn/dkX3OSUGaduzcygw51vksBQiUZLCD2Tlxyr9NvkZYT
+qiaWW78L6regvATsLc9L/dQUiSMQZIK6NglmHE+cuSaoK0H4ruNKeTiQUw/EGFaL
+ecay6Qy/s3Hk7K0QLd+gl0hZ1w1VzIeXLo2BRlqnjOYFX4A=
+=HVTm
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+
+def task_install_kubernetes(distribution):
+    """
+    Return an ``Effect`` for installing Kubernetes
+    """
+    key_path = b"/etc/gc-team@google.com.gpg.pub"
+    source_path = b"/etc/apt/sources.list.d/kubernetes.list"
+
+    return sequence([
+        # Upload the public key rather than downloading from the kubernetes
+        # servers every time.
+        put(GOOGLE_CLOUD_PACKAGES_KEY, key_path),
+        # Upload the apt repo URL
+        put(
+            b"deb http://apt.kubernetes.io/ kubernetes-xenial main\n",
+            source_path
+        ),
+        # Install the key
+        run(command=b"apt-key add {}".format(key_path)),
+        # Install Kubernetes packages
+        run(command=b"apt-get update"),
+        run(command=(
+            b"apt-get install -y "
+            b"kubelet kubeadm kubectl kubernetes-cni"
+        ))
+    ])
+
 
 def task_install_flocker(
     distribution=None,
@@ -1495,6 +1544,7 @@ def provision(distribution, package_source, variants):
     if Variants.DOCKER_HEAD in variants:
         commands.append(task_enable_docker_head_repository(distribution))
     commands.append(task_install_docker(distribution))
+    commands.append(task_install_kubernetes(distribution))
     commands.append(
         task_install_flocker(
             package_source=package_source, distribution=distribution))
@@ -1552,6 +1602,17 @@ def install_flocker(nodes, package_source):
     )
 
 
+def install_kubernetes(nodes, package_source):
+    return _run_on_all_nodes(
+        nodes,
+        task=lambda node: sequence([
+            task_install_kubernetes(
+                distribution=node.distribution,
+            ),
+        ]),
+    )
+
+
 def configure_cluster(
     cluster, dataset_backend_configuration, provider, logging_config=None
 ):

From 6de873b5ee36d93cec90cb66b18e4d6a45a70d70 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 26 Oct 2016 13:18:25 +0100
Subject: [PATCH 02/26] Attempt to configure a master and nodes

---
 admin/acceptance.py           |  4 ++-
 flocker/provision/_install.py | 66 ++++++++++++++++++++++++++++++-----
 2 files changed, 60 insertions(+), 10 deletions(-)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index dc6497dd1b..cec517187e 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -11,6 +11,7 @@
 from base64 import b32encode
 from pipes import quote as shell_quote
 from tempfile import mkdtemp
+from uuid import uuid4
 
 from zope.interface import Interface, implementer
 from characteristic import attributes
@@ -426,10 +427,11 @@ def start_cluster(self, reactor):
         Don't start any nodes.  Give back the addresses of the configured,
         already-started nodes.
         """
+        token = unicode(uuid4())
         dispatcher = make_dispatcher(reactor)
         installing = perform(
             dispatcher,
-            install_kubernetes(self._nodes, self.package_source),
+            install_kubernetes(self._nodes, self.package_source, token),
         )
 
         def configure(ignored):
diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index d28535cb37..2c79838e15 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1434,6 +1434,31 @@ def task_install_kubernetes(distribution):
     ])
 
 
+def task_configure_kubernetes_master(distribution, token):
+    """
+    Return an ``Effect`` for installing Kubernetes
+    """
+    return sequence([
+        run(
+            command=b"kubeadm init --token {}".format(token)
+        )
+    ])
+
+
+def task_configure_kubernetes_node(distribution, token, master_ip):
+    """
+    Return an ``Effect`` for installing Kubernetes
+    """
+    return sequence([
+        run(
+            command=b"kubeadm join --token {} {}".format(
+                token,
+                master_ip,
+            )
+        )
+    ])
+
+
 def task_install_flocker(
     distribution=None,
     package_source=PackageSource(),
@@ -1602,15 +1627,38 @@ def install_flocker(nodes, package_source):
     )
 
 
-def install_kubernetes(nodes, package_source):
-    return _run_on_all_nodes(
-        nodes,
-        task=lambda node: sequence([
-            task_install_kubernetes(
-                distribution=node.distribution,
-            ),
-        ]),
-    )
+def install_kubernetes(nodes, package_source, token):
+    master = nodes[0]
+    workers = nodes[1:]
+    return sequence([
+        _run_on_all_nodes(
+            nodes,
+            task=lambda node: sequence([
+                task_install_kubernetes(
+                    distribution=node.distribution,
+                ),
+            ]),
+        ),
+        _run_on_all_nodes(
+            [master],
+            command=lambda node: sequence([
+                task_configure_kubernetes_master(
+                    distribution=node.distribution,
+                    token=token,
+                )
+            ]),
+        ),
+        _run_on_all_nodes(
+            workers,
+            command=lambda node: sequence([
+                task_configure_kubernetes_node(
+                    distribution=node.distribution,
+                    token=token,
+                    master.public_ip
+                )
+            ]),
+        ),
+    ])
 
 
 def configure_cluster(

From 2498c24303786b4e0375aa21e0344cb21b3bb5af Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 26 Oct 2016 16:06:25 +0100
Subject: [PATCH 03/26] uninstall kubernetes....for easier testing

---
 admin/acceptance.py           | 19 +++++++++++----
 flocker/provision/_install.py | 44 ++++++++++++++++++++++++++++++++---
 2 files changed, 55 insertions(+), 8 deletions(-)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index cec517187e..2e955f9fe5 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -9,9 +9,10 @@
 import json
 from itertools import repeat
 from base64 import b32encode
+from hashlib import sha256
 from pipes import quote as shell_quote
+from random import getrandbits
 from tempfile import mkdtemp
-from uuid import uuid4
 
 from zope.interface import Interface, implementer
 from characteristic import attributes
@@ -50,6 +51,7 @@
 from flocker.provision._install import (
     ManagedNode,
     install_kubernetes,
+    uninstall_kubernetes,
     task_pull_docker_images,
     uninstall_flocker,
     install_flocker,
@@ -427,13 +429,20 @@ def start_cluster(self, reactor):
         Don't start any nodes.  Give back the addresses of the configured,
         already-started nodes.
         """
-        token = unicode(uuid4())
+        # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go
+        random_bytes = sha256(bytes(getrandbits(64))).hexdigest().encode('ascii')
+        token = random_bytes[:6] + b"." + random_bytes[-16:]
         dispatcher = make_dispatcher(reactor)
-        installing = perform(
+        uninstalling = perform(
             dispatcher,
-            install_kubernetes(self._nodes, self.package_source, token),
+            uninstall_kubernetes(self._nodes, self.package_source, token),
+        )
+        installing = uninstalling.addCallback(
+            lambda _ignored: perform(
+                dispatcher,
+                install_kubernetes(self._nodes, self.package_source, token),
+            )
         )
-
         def configure(ignored):
             return configured_cluster_for_nodes(
                 reactor,
diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 2c79838e15..f6ea2caf05 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1630,6 +1630,7 @@ def install_flocker(nodes, package_source):
 def install_kubernetes(nodes, package_source, token):
     master = nodes[0]
     workers = nodes[1:]
+
     return sequence([
         _run_on_all_nodes(
             nodes,
@@ -1641,7 +1642,7 @@ def install_kubernetes(nodes, package_source, token):
         ),
         _run_on_all_nodes(
             [master],
-            command=lambda node: sequence([
+            task=lambda node: sequence([
                 task_configure_kubernetes_master(
                     distribution=node.distribution,
                     token=token,
@@ -1650,17 +1651,54 @@ def install_kubernetes(nodes, package_source, token):
         ),
         _run_on_all_nodes(
             workers,
-            command=lambda node: sequence([
+            task=lambda node: sequence([
                 task_configure_kubernetes_node(
                     distribution=node.distribution,
                     token=token,
-                    master.public_ip
+                    master_ip=master.address
                 )
             ]),
         ),
     ])
 
 
+def uninstall_kubernetes(nodes, package_source, token):
+    master = nodes[0]
+    workers = nodes[1:]
+
+    return sequence([
+        _run_on_all_nodes(
+            nodes,
+            task=lambda node: sequence([
+                run(
+                    command=(
+                        b"apt-get -y remove "
+                        b"kubelet kubeadm kubectl kubernetes-cni"
+                    )
+                ),
+                run(
+                    command=(
+                        b"docker ps --all --quiet | "
+                        b"xargs --no-run-if-empty docker rm --force"
+                    )
+                ),
+                run(
+                    command=(
+                        b"find /var/lib/kubelet "
+                        b"| xargs -n 1 findmnt -n -t tmpfs -o TARGET -T "
+                        b"| uniq | xargs -r umount -v"
+                    )
+                ),
+                run(
+                    command=(
+                        b"rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd"
+                    )
+                ),
+            ]),
+        ),
+    ])
+
+
 def configure_cluster(
     cluster, dataset_backend_configuration, provider, logging_config=None
 ):

From 5bf1611f18f4ac1a26bde56c4f4fe1851571cc17 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 26 Oct 2016 17:03:14 +0100
Subject: [PATCH 04/26] Add weave networking after the nodes  have joined

---
 flocker/provision/_install.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index f6ea2caf05..73ee202931 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1441,7 +1441,8 @@ def task_configure_kubernetes_master(distribution, token):
     return sequence([
         run(
             command=b"kubeadm init --token {}".format(token)
-        )
+        ),
+        run(command=b"kubectl taint nodes --all dedicated-"),
     ])
 
 
@@ -1646,7 +1647,8 @@ def install_kubernetes(nodes, package_source, token):
                 task_configure_kubernetes_master(
                     distribution=node.distribution,
                     token=token,
-                )
+                ),
+
             ]),
         ),
         _run_on_all_nodes(
@@ -1659,6 +1661,12 @@ def install_kubernetes(nodes, package_source, token):
                 )
             ]),
         ),
+        _run_on_all_nodes(
+            [master],
+            task=lambda node: sequence([
+                run(command=b"kubectl apply -f https://git.io/weave-kube")
+            ]),
+        ),
     ])
 
 

From af1e19f438888cbe6df28691df743ee06f17c639 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Thu, 27 Oct 2016 12:18:36 +0100
Subject: [PATCH 05/26] Integrate the kubernetes configuration into the
 existing configure_cluster function

---
 admin/acceptance.py           |  21 ++---
 flocker/provision/_install.py | 141 +++++++++++++++-------------------
 2 files changed, 66 insertions(+), 96 deletions(-)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index 2e955f9fe5..9ad5f05eca 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -9,9 +9,7 @@
 import json
 from itertools import repeat
 from base64 import b32encode
-from hashlib import sha256
 from pipes import quote as shell_quote
-from random import getrandbits
 from tempfile import mkdtemp
 
 from zope.interface import Interface, implementer
@@ -50,8 +48,7 @@
 )
 from flocker.provision._install import (
     ManagedNode,
-    install_kubernetes,
-    uninstall_kubernetes,
+    deconfigure_kubernetes,
     task_pull_docker_images,
     uninstall_flocker,
     install_flocker,
@@ -393,6 +390,7 @@ def extend_cluster(self, reactor, cluster, count, tag, starting_index):
         raise UsageError("Extending a cluster with managed nodes "
                          "is not implemented yet.")
 
+
 @implementer(IClusterRunner)
 class KubernetesRunner(object):
     """
@@ -429,19 +427,10 @@ def start_cluster(self, reactor):
         Don't start any nodes.  Give back the addresses of the configured,
         already-started nodes.
         """
-        # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go
-        random_bytes = sha256(bytes(getrandbits(64))).hexdigest().encode('ascii')
-        token = random_bytes[:6] + b"." + random_bytes[-16:]
         dispatcher = make_dispatcher(reactor)
-        uninstalling = perform(
+        deconfiguring = perform(
             dispatcher,
-            uninstall_kubernetes(self._nodes, self.package_source, token),
-        )
-        installing = uninstalling.addCallback(
-            lambda _ignored: perform(
-                dispatcher,
-                install_kubernetes(self._nodes, self.package_source, token),
-            )
+            deconfigure_kubernetes(self._nodes, self.package_source),
         )
         def configure(ignored):
             return configured_cluster_for_nodes(
@@ -462,7 +451,7 @@ def configure(ignored):
                 provider="managed",
                 logging_config=self.logging_config,
             )
-        configuring = installing.addCallback(configure)
+        configuring = deconfiguring.addCallback(configure)
         return configuring
 
     def stop_cluster(self, reactor):
diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 73ee202931..6f6727d9d2 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -13,6 +13,8 @@
 from effect.retry import retry
 from time import time
 import yaml
+from hashlib import sha256
+from random import getrandbits
 
 from zope.interface import implementer
 
@@ -1407,6 +1409,15 @@ def task_install_docker(distribution):
 """
 
 
+def kubeadm_token_from_cluster(cluster):
+    # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go # noqa
+    hash_bytes = sha256(
+        cluster.certificates.cluster.certificate.getContent()
+    ).hexdigest().encode('ascii')
+    token = hash_bytes[:6] + b"." + hash_bytes[-16:]
+    return token
+
+
 def task_install_kubernetes(distribution):
     """
     Return an ``Effect`` for installing Kubernetes
@@ -1628,62 +1639,12 @@ def install_flocker(nodes, package_source):
     )
 
 
-def install_kubernetes(nodes, package_source, token):
-    master = nodes[0]
-    workers = nodes[1:]
-
-    return sequence([
-        _run_on_all_nodes(
-            nodes,
-            task=lambda node: sequence([
-                task_install_kubernetes(
-                    distribution=node.distribution,
-                ),
-            ]),
-        ),
-        _run_on_all_nodes(
-            [master],
-            task=lambda node: sequence([
-                task_configure_kubernetes_master(
-                    distribution=node.distribution,
-                    token=token,
-                ),
-
-            ]),
-        ),
-        _run_on_all_nodes(
-            workers,
-            task=lambda node: sequence([
-                task_configure_kubernetes_node(
-                    distribution=node.distribution,
-                    token=token,
-                    master_ip=master.address
-                )
-            ]),
-        ),
-        _run_on_all_nodes(
-            [master],
-            task=lambda node: sequence([
-                run(command=b"kubectl apply -f https://git.io/weave-kube")
-            ]),
-        ),
-    ])
-
-
-def uninstall_kubernetes(nodes, package_source, token):
-    master = nodes[0]
-    workers = nodes[1:]
-
+def deconfigure_kubernetes(nodes, package_source):
     return sequence([
         _run_on_all_nodes(
             nodes,
             task=lambda node: sequence([
-                run(
-                    command=(
-                        b"apt-get -y remove "
-                        b"kubelet kubeadm kubectl kubernetes-cni"
-                    )
-                ),
+                run(command=b"systemctl stop kubelet"),
                 run(
                     command=(
                         b"docker ps --all --quiet | "
@@ -1693,7 +1654,9 @@ def uninstall_kubernetes(nodes, package_source, token):
                 run(
                     command=(
                         b"find /var/lib/kubelet "
-                        b"| xargs -n 1 findmnt -n -t tmpfs -o TARGET -T "
+                        b"| xargs --no-run-if-empty "
+                        b"        --max-args 1 "
+                        b"        findmnt -n -t tmpfs -o TARGET -T "
                         b"| uniq | xargs -r umount -v"
                     )
                 ),
@@ -1702,6 +1665,7 @@ def uninstall_kubernetes(nodes, package_source, token):
                         b"rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd"
                     )
                 ),
+                run(command=b"systemctl start kubelet"),
             ]),
         ),
     ])
@@ -1886,6 +1850,10 @@ def configure_control_node(
                     cluster.control_node.distribution
                 )
             ),
+            task_configure_kubernetes_master(
+                distribution=cluster.control_node.distribution,
+                token=kubeadm_token_from_cluster(cluster),
+            )
         ]),
     )
 
@@ -1913,36 +1881,49 @@ def configure_node(
     if provider == "managed":
         setup_action = 'restart'
 
+    if node is cluster.control_node:
+        commands = []
+    else:
+        commands = [
+            task_configure_kubernetes_node(
+                distribution=node.distribution,
+                token=kubeadm_token_from_cluster(cluster),
+                master_ip=cluster.control_node.address,
+            ),
+        ]
+
+    commands.extend([
+        task_install_node_certificates(
+            cluster.certificates.cluster.certificate,
+            certnkey.certificate,
+            certnkey.key),
+        task_install_api_certificates(
+            cluster.certificates.user.certificate,
+            cluster.certificates.user.key),
+        task_enable_docker(node.distribution),
+        if_firewall_available(
+            node.distribution,
+            open_firewall_for_docker_api(node.distribution),
+        ),
+        task_configure_flocker_agent(
+            control_node=cluster.control_node.address,
+            dataset_backend=cluster.dataset_backend,
+            dataset_backend_configuration=(
+                dataset_backend_configuration
+            ),
+            logging_config=logging_config,
+        ),
+        task_enable_docker_plugin(node.distribution),
+        task_enable_flocker_agent(
+            distribution=node.distribution,
+            action=setup_action,
+        ),
+    ])
+
     return run_remotely(
         username='root',
         address=node.address,
-        commands=sequence([
-            task_install_node_certificates(
-                cluster.certificates.cluster.certificate,
-                certnkey.certificate,
-                certnkey.key),
-            task_install_api_certificates(
-                cluster.certificates.user.certificate,
-                cluster.certificates.user.key),
-            task_enable_docker(node.distribution),
-            if_firewall_available(
-                node.distribution,
-                open_firewall_for_docker_api(node.distribution),
-            ),
-            task_configure_flocker_agent(
-                control_node=cluster.control_node.address,
-                dataset_backend=cluster.dataset_backend,
-                dataset_backend_configuration=(
-                    dataset_backend_configuration
-                ),
-                logging_config=logging_config,
-            ),
-            task_enable_docker_plugin(node.distribution),
-            task_enable_flocker_agent(
-                distribution=node.distribution,
-                action=setup_action,
-            ),
-        ]),
+        commands=sequence(commands),
     )
 
 

From eb2f70812f3dd86197e71d86ce9c91ee559ea2e2 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Thu, 27 Oct 2016 12:48:37 +0100
Subject: [PATCH 06/26] Do the kubernetes provisioning from the manages runner.

---
 admin/acceptance.py           | 102 ++++------------------------------
 flocker/provision/_install.py |  20 +++++--
 2 files changed, 25 insertions(+), 97 deletions(-)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index 9ad5f05eca..8c5b63c864 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -306,7 +306,7 @@ def __init__(self, node_addresses, package_source, distribution,
         self.cert_path = cert_path
         self.logging_config = logging_config
 
-    def _upgrade_flocker(self, reactor, nodes, package_source):
+    def _upgrade_flocker(self, dispatcher, nodes, package_source):
         """
         Put the version of Flocker indicated by ``package_source`` onto all of
         the given nodes.
@@ -325,7 +325,7 @@ def _upgrade_flocker(self, reactor, nodes, package_source):
 
         :return: A ``Deferred`` that fires when the software has been upgraded.
         """
-        dispatcher = make_dispatcher(reactor)
+
 
         uninstalling = perform(dispatcher, uninstall_flocker(nodes))
         uninstalling.addErrback(write_failure, logger=None)
@@ -351,87 +351,21 @@ def start_cluster(self, reactor):
         Don't start any nodes.  Give back the addresses of the configured,
         already-started nodes.
         """
+        dispatcher = make_dispatcher(reactor)
         if self.package_source is not None:
             upgrading = self._upgrade_flocker(
-                reactor, self._nodes, self.package_source
+                dispatcher, self._nodes, self.package_source
             )
         else:
             upgrading = succeed(None)
-        return upgrading
-        def configure(ignored):
-            return configured_cluster_for_nodes(
-                reactor,
-                generate_certificates(
-                    self.identity.name,
-                    self.identity.id,
-                    self._nodes,
-                    self.cert_path,
-                ),
-                self._nodes,
-                self.dataset_backend,
-                self.dataset_backend_configuration,
-                save_backend_configuration(
-                    self.dataset_backend,
-                    self.dataset_backend_configuration
-                ),
-                provider="managed",
-                logging_config=self.logging_config,
-            )
-        configuring = upgrading.addCallback(configure)
-        return configuring
-
-    def stop_cluster(self, reactor):
-        """
-        Don't stop any nodes.
-        """
-        return succeed(None)
-
-    def extend_cluster(self, reactor, cluster, count, tag, starting_index):
-        raise UsageError("Extending a cluster with managed nodes "
-                         "is not implemented yet.")
-
-
-@implementer(IClusterRunner)
-class KubernetesRunner(object):
-    """
-    """
-    def __init__(self, node_addresses, package_source, distribution,
-                 dataset_backend, dataset_backend_configuration, identity,
-                 cert_path, logging_config):
-        """
-        :param list: A ``list`` of public IP addresses or
-            ``[private_address, public_address]`` lists.
 
-        See ``ManagedRunner`` and ``ManagedNode`` for other parameter
-        documentation.
-        """
-        self._nodes = pvector(
-            make_managed_nodes(node_addresses, distribution)
+        deconfiguring_kubernetes = upgrading.addCallback(
+            lambda _ignored: perform(
+                dispatcher,
+                deconfigure_kubernetes(self._nodes),
+            )
         )
-        self.package_source = package_source
-        self.dataset_backend = dataset_backend
-        self.dataset_backend_configuration = dataset_backend_configuration
-        self.identity = identity
-        self.cert_path = cert_path
-        self.logging_config = logging_config
 
-    def ensure_keys(self, reactor):
-        """
-        Assume we have keys, since there's no way of asking the nodes what keys
-        they'll accept.
-        """
-        return succeed(None)
-
-    def start_cluster(self, reactor):
-        """
-        Don't start any nodes.  Give back the addresses of the configured,
-        already-started nodes.
-        """
-        dispatcher = make_dispatcher(reactor)
-        deconfiguring = perform(
-            dispatcher,
-            deconfigure_kubernetes(self._nodes, self.package_source),
-        )
         def configure(ignored):
             return configured_cluster_for_nodes(
                 reactor,
@@ -451,7 +385,7 @@ def configure(ignored):
                 provider="managed",
                 logging_config=self.logging_config,
             )
-        configuring = deconfiguring.addCallback(configure)
+        configuring = deconfiguring_kubernetes.addCallback(configure)
         return configuring
 
     def stop_cluster(self, reactor):
@@ -1170,22 +1104,6 @@ def _runner_MANAGED(self, package_source, dataset_backend,
             logging_config=self['config'].get('logging'),
         )
 
-    def _runner_KUBERNETES(self, package_source, dataset_backend,
-                           provider_config):
-        if provider_config is None:
-            self._provider_config_missing("kubernetes")
-
-        return KubernetesRunner(
-            node_addresses=provider_config['addresses'],
-            package_source=None,
-            distribution=self['distribution'],
-            dataset_backend=dataset_backend,
-            dataset_backend_configuration=self.dataset_backend_configuration(),
-            identity=self._make_cluster_identity(dataset_backend),
-            cert_path=self['cert-directory'],
-            logging_config=self['config'].get('logging'),
-        )
-
     def _libcloud_runner(self, package_source, dataset_backend,
                          provider, provider_config):
         """
diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 6f6727d9d2..3751af64cc 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -14,7 +14,6 @@
 from time import time
 import yaml
 from hashlib import sha256
-from random import getrandbits
 
 from zope.interface import implementer
 
@@ -1639,7 +1638,16 @@ def install_flocker(nodes, package_source):
     )
 
 
-def deconfigure_kubernetes(nodes, package_source):
+def deconfigure_kubernetes(nodes):
+    """
+    See: http://kubernetes.io/docs/getting-started-guides/kubeadm/#cleanup
+
+    :param nodes: An iterable of ``Node`` instances on which to de-configure
+        Kubernetes.
+
+    :return: An ``Effect`` which removes all Kubernetes pods / containers and
+        configuration.
+    """
     return sequence([
         _run_on_all_nodes(
             nodes,
@@ -1656,13 +1664,15 @@ def deconfigure_kubernetes(nodes, package_source):
                         b"find /var/lib/kubelet "
                         b"| xargs --no-run-if-empty "
                         b"        --max-args 1 "
-                        b"        findmnt -n -t tmpfs -o TARGET -T "
-                        b"| uniq | xargs -r umount -v"
+                        b"  findmnt --noheadings --types tmpfs "
+                        b"          --output TARGET --target "
+                        b"| uniq | xargs --no-run-if-empty umount"
                     )
                 ),
                 run(
                     command=(
-                        b"rm -rf /etc/kubernetes /var/lib/kubelet /var/lib/etcd"
+                        b"rm -rf "
+                        b"/etc/kubernetes /var/lib/kubelet /var/lib/etcd"
                     )
                 ),
                 run(command=b"systemctl start kubelet"),

From b10116f867c9c41c3efb7f58cd14169ae041bdbb Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Thu, 27 Oct 2016 15:41:47 +0100
Subject: [PATCH 07/26] Import the repo key to a separate keyring and add
 docstrings.

---
 admin/acceptance.py           |  2 --
 flocker/provision/_install.py | 56 +++++++++++++++++++++++++++++------
 2 files changed, 47 insertions(+), 11 deletions(-)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index 8c5b63c864..bb203ab0c1 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -325,8 +325,6 @@ def _upgrade_flocker(self, dispatcher, nodes, package_source):
 
         :return: A ``Deferred`` that fires when the software has been upgraded.
         """
-
-
         uninstalling = perform(dispatcher, uninstall_flocker(nodes))
         uninstalling.addErrback(write_failure, logger=None)
 
diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 3751af64cc..2804e9d836 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1385,6 +1385,8 @@ def task_install_docker(distribution):
         timeout=5.0 * 60.0,
     )
 
+# Hard coded Kubernetes repository key.
+# In PGP ASCII armor.
 GOOGLE_CLOUD_PACKAGES_KEY = """
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
@@ -1409,7 +1411,17 @@ def task_install_docker(distribution):
 
 
 def kubeadm_token_from_cluster(cluster):
-    # See https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go # noqa
+    """
+    Generate a stable ``kubeadmin --token`` parameter for the supplied
+    ``cluster``.
+
+    See: https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/util/tokens.go # noqa
+
+    :param Cluster cluster: The cluster supplied by the acceptance test runner.
+    :returns: A "6<byte>.16<byte>" byte string in the format expected by
+        kubeadm --token.
+    """
+    #
     hash_bytes = sha256(
         cluster.certificates.cluster.certificate.getContent()
     ).hexdigest().encode('ascii')
@@ -1419,22 +1431,32 @@ def kubeadm_token_from_cluster(cluster):
 
 def task_install_kubernetes(distribution):
     """
-    Return an ``Effect`` for installing Kubernetes
+    Install Kubernetes packages.
+
+    :param unicode distribution: The name of the target OS distribution.
+    :returns: an ``Effect`` for installing Kubernetes packages from the
+        Kubernetes repository.
     """
-    key_path = b"/etc/gc-team@google.com.gpg.pub"
-    source_path = b"/etc/apt/sources.list.d/kubernetes.list"
+    if distribution not in ("ubuntu-16.04",):
+        return sequence([])
 
+    key_path = b"/etc/apt/trusted.gpg.d/gc-team@google.com.gpg"
+    source_path = b"/etc/apt/sources.list.d/kubernetes.list"
     return sequence([
         # Upload the public key rather than downloading from the kubernetes
         # servers every time.
-        put(GOOGLE_CLOUD_PACKAGES_KEY, key_path),
+        put(GOOGLE_CLOUD_PACKAGES_KEY, key_path + b".asc"),
         # Upload the apt repo URL
         put(
             b"deb http://apt.kubernetes.io/ kubernetes-xenial main\n",
             source_path
         ),
-        # Install the key
-        run(command=b"apt-key add {}".format(key_path)),
+        # Install the key Kubernetes key
+        run(
+            command=b"apt-key --keyring {} add {}.asc".format(
+                key_path, key_path
+            )
+        ),
         # Install Kubernetes packages
         run(command=b"apt-get update"),
         run(command=(
@@ -1446,20 +1468,36 @@ def task_install_kubernetes(distribution):
 
 def task_configure_kubernetes_master(distribution, token):
     """
-    Return an ``Effect`` for installing Kubernetes
+    Configure a Kubernetes master and allow that node to also run pods.
+
+    :param unicode distribution: The name of the target OS distribution.
+    :param bytes token: A ``kubeadm`` token.
+    :returns: an ``Effect`` for configuring the Kubernetes master node.
     """
+    if distribution not in ("ubuntu-16.04",):
+        return sequence([])
+
     return sequence([
         run(
             command=b"kubeadm init --token {}".format(token)
         ),
+        # Allow pods to be scheduled to the master node too.
         run(command=b"kubectl taint nodes --all dedicated-"),
     ])
 
 
 def task_configure_kubernetes_node(distribution, token, master_ip):
     """
-    Return an ``Effect`` for installing Kubernetes
+    Configure a Kubernetes worker node and join it to the master configured in
+    ``task_configure_kubernetes_master``.
+
+    :param unicode distribution: The name of the target OS distribution.
+    :param bytes token: A ``kubeadm`` token.
+    :returns: an ``Effect`` for running ``kubeadm --join``.
     """
+    if distribution not in ("ubuntu-16.04",):
+        return sequence([])
+
     return sequence([
         run(
             command=b"kubeadm join --token {} {}".format(

From 163132d2a09b1a80d3b3ef8f2216f96faab97e87 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 31 Oct 2016 16:39:18 +0000
Subject: [PATCH 08/26] Install Weave networking daemonset

---
 flocker/provision/_install.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 2804e9d836..2803a8705f 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1465,6 +1465,13 @@ def task_install_kubernetes(distribution):
         ))
     ])
 
+# XXX Maybe copy the entire configuration here to avoid failures due to flaky
+# downloads.
+KUBERNETES_ADDON_WEAVE = (
+    b"https://raw.githubusercontent.com"
+    b"/weaveworks/weave-kube/v1.7.2/weave-daemonset.yaml"
+)
+
 
 def task_configure_kubernetes_master(distribution, token):
     """
@@ -1483,6 +1490,11 @@ def task_configure_kubernetes_master(distribution, token):
         ),
         # Allow pods to be scheduled to the master node too.
         run(command=b"kubectl taint nodes --all dedicated-"),
+        # Install a network addon. The weave daemonset will be started on the
+        # nodes as they join the cluster in ``task_configure_kubernetes_node``.
+        run(
+            command=b"kubectl apply --filename " + KUBERNETES_ADDON_WEAVE
+        ),
     ])
 
 

From b1b803e21ea102a905c383939a4243bfd5dc6582 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Tue, 1 Nov 2016 14:08:40 +0000
Subject: [PATCH 09/26] Non-working Centos installation tasks

---
 flocker/provision/_install.py | 147 +++++++++++++++++++++++++++-------
 1 file changed, 117 insertions(+), 30 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 2803a8705f..b00f3f8619 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -133,6 +133,17 @@ def is_systemd_distribution(distribution):
         distribution == "ubuntu-16.04"
     )
 
+_distribution_to_package_format = {
+    "centos-7": "rpm",
+    "rhel-7.2": "rpm",
+    "ubuntu-16.04": "deb",
+    "ubuntu-14.04": "deb",
+}
+
+
+def package_format_for_distribution(distribution):
+    return _distribution_to_package_format[distribution]
+
 
 def _from_args(sudo):
     """
@@ -1385,9 +1396,10 @@ def task_install_docker(distribution):
         timeout=5.0 * 60.0,
     )
 
-# Hard coded Kubernetes repository key.
-# In PGP ASCII armor.
-GOOGLE_CLOUD_PACKAGES_KEY = """
+# Used for signing yum and apt repo metadata
+# pub   2048R/A7317B0F 2015-04-03 [expires: 2018-04-02]
+# uid   Google Cloud Packages Automatic Signing Key <gc-team@google.com>
+GOOGLE_CLOUD_PACKAGES_KEY_AUTOMATIC = """
 -----BEGIN PGP PUBLIC KEY BLOCK-----
 Version: GnuPG v1
 
@@ -1409,6 +1421,50 @@ def task_install_docker(distribution):
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
+# Used for signing RPM packages.
+# pub   2048R/3E1BA8D5 2015-06-24
+# uid   Google Cloud Packages RPM Signing Key <gc-team@google.com>
+GOOGLE_CLOUD_PACKAGES_KEY_RPM = """
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+Version: GnuPG v1
+
+mQENBFWKtqgBCADmKQWYQF9YoPxLEQZ5XA6DFVg9ZHG4HIuehsSJETMPQ+W9K5c5
+Us5assCZBjG/k5i62SmWb09eHtWsbbEgexURBWJ7IxA8kM3kpTo7bx+LqySDsSC3
+/8JRkiyibVV0dDNv/EzRQsGDxmk5Xl8SbQJ/C2ECSUT2ok225f079m2VJsUGHG+5
+RpyHHgoMaRNedYP8ksYBPSD6sA3Xqpsh/0cF4sm8QtmsxkBmCCIjBa0B0LybDtdX
+XIq5kPJsIrC2zvERIPm1ez/9FyGmZKEFnBGeFC45z5U//pHdB1z03dYKGrKdDpID
+17kNbC5wl24k/IeYyTY9IutMXvuNbVSXaVtRABEBAAG0Okdvb2dsZSBDbG91ZCBQ
+YWNrYWdlcyBSUE0gU2lnbmluZyBLZXkgPGdjLXRlYW1AZ29vZ2xlLmNvbT6JATgE
+EwECACIFAlWKtqgCGy8GCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEPCcOUw+
+G6jV+QwH/0wRH+XovIwLGfkg6kYLEvNPvOIYNQWnrT6zZ+XcV47WkJ+i5SR+QpUI
+udMSWVf4nkv+XVHruxydafRIeocaXY0E8EuIHGBSB2KR3HxG6JbgUiWlCVRNt4Qd
+6udC6Ep7maKEIpO40M8UHRuKrp4iLGIhPm3ELGO6uc8rks8qOBMH4ozU+3PB9a0b
+GnPBEsZdOBI1phyftLyyuEvG8PeUYD+uzSx8jp9xbMg66gQRMP9XGzcCkD+b8w1o
+7v3J3juKKpgvx5Lqwvwv2ywqn/Wr5d5OBCHEw8KtU/tfxycz/oo6XUIshgEbS/+P
+6yKDuYhRp6qxrYXjmAszIT25cftb4d4=
+=/PbX
+-----END PGP PUBLIC KEY BLOCK-----
+"""
+
+KUBERNETES_REPO_APT = """
+deb http://apt.kubernetes.io/ kubernetes-xenial main
+"""
+
+KUBERNETES_REPO_PATH_APT = "/etc/apt/sources.list.d/kubernetes.list"
+
+KUBERNETES_REPO_YUM = """
+[kubernetes]
+name=Kubernetes
+baseurl=http://yum.kubernetes.io/repos/kubernetes-el7-x86_64
+enabled=1
+gpgcheck=1
+repo_gpgcheck=1
+gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
+       https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
+"""
+
+KUBERNETES_REPO_PATH_YUM = "/etc/yum.repos.d/kubernetes.repo"
+
 
 def kubeadm_token_from_cluster(cluster):
     """
@@ -1429,34 +1485,23 @@ def kubeadm_token_from_cluster(cluster):
     return token
 
 
-def task_install_kubernetes(distribution):
-    """
-    Install Kubernetes packages.
-
-    :param unicode distribution: The name of the target OS distribution.
-    :returns: an ``Effect`` for installing Kubernetes packages from the
-        Kubernetes repository.
-    """
-    if distribution not in ("ubuntu-16.04",):
-        return sequence([])
-
-    key_path = b"/etc/apt/trusted.gpg.d/gc-team@google.com.gpg"
-    source_path = b"/etc/apt/sources.list.d/kubernetes.list"
+def task_install_kubernetes_apt():
+    key_path = b"/etc/apt/trusted.gpg.d/google_cloud_packages_automatic"
     return sequence([
         # Upload the public key rather than downloading from the kubernetes
         # servers every time.
-        put(GOOGLE_CLOUD_PACKAGES_KEY, key_path + b".asc"),
-        # Upload the apt repo URL
-        put(
-            b"deb http://apt.kubernetes.io/ kubernetes-xenial main\n",
-            source_path
-        ),
+        put(GOOGLE_CLOUD_PACKAGES_KEY_AUTOMATIC, key_path + b".asc"),
         # Install the key Kubernetes key
         run(
             command=b"apt-key --keyring {} add {}.asc".format(
                 key_path, key_path
             )
         ),
+        # Upload the repo file
+        put(
+            KUBERNETES_REPO_APT,
+            KUBERNETES_REPO_PATH_APT
+        ),
         # Install Kubernetes packages
         run(command=b"apt-get update"),
         run(command=(
@@ -1465,6 +1510,54 @@ def task_install_kubernetes(distribution):
         ))
     ])
 
+
+def task_install_kubernetes_yum():
+    key_paths = [
+        (GOOGLE_CLOUD_PACKAGES_KEY_AUTOMATIC,
+         b"/etc/pki/rpm-gpg/google_cloud_packages_automatic"),
+        (GOOGLE_CLOUD_PACKAGES_KEY_RPM,
+         b"/etc/pki/rpm-gpg/google_cloud_packages_rpm"),
+    ]
+    key_operations = []
+    for key_content, key_path in key_paths:
+        key_operations += [
+            put(key_content, key_path),
+            run(b"rpmkeys --import " + key_path)
+        ]
+    return sequence(
+        # Upload and import YUM and RPM signing keys
+        key_operations + [
+            # Upload the repo file
+            put(
+                KUBERNETES_REPO_YUM,
+                KUBERNETES_REPO_PATH_YUM
+            ),
+            # Install Kubernetes packages
+            run(command=(
+                b"yum install -y "
+                b"kubelet kubeadm kubectl kubernetes-cni"
+            )),
+        ]
+    )
+
+_task_install_kubernetes_variants = {
+    'deb': task_install_kubernetes_apt,
+    'rpm': task_install_kubernetes_yum,
+}
+
+
+def task_install_kubernetes(distribution):
+    """
+    Install Kubernetes packages.
+
+    :param unicode distribution: The name of the target OS distribution.
+    :returns: an ``Effect`` for installing Kubernetes packages from the
+        Kubernetes repository.
+    """
+    package_format = package_format_for_distribution(distribution)
+    return _task_install_kubernetes_variants[package_format]()
+
+
 # XXX Maybe copy the entire configuration here to avoid failures due to flaky
 # downloads.
 KUBERNETES_ADDON_WEAVE = (
@@ -1481,9 +1574,6 @@ def task_configure_kubernetes_master(distribution, token):
     :param bytes token: A ``kubeadm`` token.
     :returns: an ``Effect`` for configuring the Kubernetes master node.
     """
-    if distribution not in ("ubuntu-16.04",):
-        return sequence([])
-
     return sequence([
         run(
             command=b"kubeadm init --token {}".format(token)
@@ -1507,16 +1597,13 @@ def task_configure_kubernetes_node(distribution, token, master_ip):
     :param bytes token: A ``kubeadm`` token.
     :returns: an ``Effect`` for running ``kubeadm --join``.
     """
-    if distribution not in ("ubuntu-16.04",):
-        return sequence([])
-
     return sequence([
         run(
             command=b"kubeadm join --token {} {}".format(
                 token,
                 master_ip,
             )
-        )
+        ),
     ])
 
 
@@ -1913,7 +2000,7 @@ def configure_control_node(
             task_configure_kubernetes_master(
                 distribution=cluster.control_node.distribution,
                 token=kubeadm_token_from_cluster(cluster),
-            )
+            ),
         ]),
     )
 

From 23e6361c46c2be237e70b3d4d7fd9ed1e878ab09 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 7 Nov 2016 17:52:51 +0000
Subject: [PATCH 10/26] Start kublet and docker daemon separately.

---
 flocker/provision/_install.py | 41 ++++++++++++++++++++++++++++++-----
 1 file changed, 36 insertions(+), 5 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index cec1e69f38..0de5e7c0e4 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -829,7 +829,7 @@ def task_install_api_certificates(api_cert, api_key):
         ])
 
 
-def task_enable_docker(distribution):
+def task_configure_docker(distribution):
     """
     Configure docker.
 
@@ -885,6 +885,37 @@ def task_enable_docker(distribution):
         raise DistributionNotSupported(distribution=distribution)
 
 
+def task_start_docker(distribution):
+    """
+    foo
+    """
+    if is_systemd_distribution(distribution):
+        commands = [
+            run_from_args(['systemctl', START, 'docker']),
+        ]
+    elif is_ubuntu(distribution):
+        commands = [
+            run_from_args(['service', 'docker', 'restart']),
+        ]
+    return sequence(commands)
+
+
+def task_start_kubelet(distribution):
+    """
+    foo
+    """
+    if is_systemd_distribution(distribution):
+        commands = [
+            run_from_args(['systemctl', 'enable' 'kubelet']),
+            run_from_args(['systemctl', START, 'kubelet']),
+        ]
+    elif is_ubuntu(distribution):
+        commands = [
+            run_from_args(['service', 'kubelet', 'restart']),
+        ]
+    return sequence(commands)
+
+
 def open_firewalld(service):
     """
     Open firewalld port for a service.
@@ -974,12 +1005,10 @@ def task_enable_docker_plugin(distribution):
         return sequence([
             run_from_args(['systemctl', 'enable', 'flocker-docker-plugin']),
             run_from_args(['systemctl', START, 'flocker-docker-plugin']),
-            run_from_args(['systemctl', START, 'docker']),
         ])
     elif is_ubuntu(distribution):
         return sequence([
             run_from_args(['service', 'flocker-docker-plugin', 'restart']),
-            run_from_args(['service', 'docker', 'restart']),
         ])
     else:
         raise DistributionNotSupported(distribution=distribution)
@@ -1717,7 +1746,9 @@ def provision(distribution, package_source, variants):
     commands.append(
         task_install_docker_plugin(
             package_source=package_source, distribution=distribution))
-    commands.append(task_enable_docker(distribution))
+    commands.append(task_configure_docker(distribution))
+    commands.append(task_start_docker(distribution))
+    commands.append(task_start_kubelet(distribution))
     return sequence(commands)
 
 
@@ -2040,7 +2071,7 @@ def configure_node(
         task_install_api_certificates(
             cluster.certificates.user.certificate,
             cluster.certificates.user.key),
-        task_enable_docker(node.distribution),
+        task_configure_docker(node.distribution),
         if_firewall_available(
             node.distribution,
             open_firewall_for_docker_api(node.distribution),

From 36637e01965c1fa67f7d2c9c353f79d9f4e5ef8a Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Thu, 10 Nov 2016 16:16:17 +0000
Subject: [PATCH 11/26] Juggle the setup steps and disable pre-flight checks to
 get kubadm working on Centos7

---
 flocker/provision/_install.py | 19 +++++++++++++------
 1 file changed, 13 insertions(+), 6 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 0de5e7c0e4..e4ffc58ce4 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -872,7 +872,6 @@ def task_configure_docker(distribution):
                     ExecStart=
                     ExecStart=/usr/bin/dockerd {} {}
                     """.format(unixsock_opt, docker_tls_options))),
-            run_from_args(["systemctl", "enable", "docker.service"]),
         ])
     elif is_ubuntu(distribution):
         return sequence([
@@ -891,6 +890,7 @@ def task_start_docker(distribution):
     """
     if is_systemd_distribution(distribution):
         commands = [
+            run_from_args(["systemctl", "enable", "docker.service"]),
             run_from_args(['systemctl', START, 'docker']),
         ]
     elif is_ubuntu(distribution):
@@ -906,7 +906,7 @@ def task_start_kubelet(distribution):
     """
     if is_systemd_distribution(distribution):
         commands = [
-            run_from_args(['systemctl', 'enable' 'kubelet']),
+            run_from_args(['systemctl', 'enable', 'kubelet']),
             run_from_args(['systemctl', START, 'kubelet']),
         ]
     elif is_ubuntu(distribution):
@@ -1615,15 +1615,19 @@ def task_configure_kubernetes_node(distribution, token, master_ip):
     Configure a Kubernetes worker node and join it to the master configured in
     ``task_configure_kubernetes_master``.
 
+    XXX Skip preflight checks until
+    https://github.com/kubernetes/kubernetes/issues/36301 is resolved.
+
     :param unicode distribution: The name of the target OS distribution.
     :param bytes token: A ``kubeadm`` token.
     :returns: an ``Effect`` for running ``kubeadm --join``.
     """
     return sequence([
         run(
-            command=b"kubeadm join --token {} {}".format(
-                token,
-                master_ip,
+            command=(
+                b"kubeadm join --skip-preflight-checks --token "
+                + token + b" "
+                + master_ip
             )
         ),
     ])
@@ -1746,7 +1750,6 @@ def provision(distribution, package_source, variants):
     commands.append(
         task_install_docker_plugin(
             package_source=package_source, distribution=distribution))
-    commands.append(task_configure_docker(distribution))
     commands.append(task_start_docker(distribution))
     commands.append(task_start_kubelet(distribution))
     return sequence(commands)
@@ -2089,6 +2092,10 @@ def configure_node(
             distribution=node.distribution,
             action=setup_action,
         ),
+        # Restart docker after pushing the Flocker certificates and the Docker
+        # configuration modifications which make it use Flocker certificates
+        # for listening on a TLS / TCP port.
+        task_start_docker(node.distribution)
     ])
 
     return run_remotely(

From cbaa8270e7a3f5752c512b7a76f06fe92f259b51 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 14 Nov 2016 11:49:46 +0000
Subject: [PATCH 12/26] Fix task names for documentation tasks import.

---
 flocker/provision/_tasks.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/flocker/provision/_tasks.py b/flocker/provision/_tasks.py
index da729324a7..a8aab35884 100644
--- a/flocker/provision/_tasks.py
+++ b/flocker/provision/_tasks.py
@@ -6,7 +6,8 @@
 
 from ._install import (
     task_create_flocker_pool_file,
-    task_enable_docker,
+    task_configure_docker,
+    task_start_docker,
     task_install_flocker,
     task_install_ssh_key,
     task_cli_pkg_install,
@@ -22,7 +23,8 @@
 
 __all__ = [
     'task_create_flocker_pool_file',
-    'task_enable_docker',
+    'task_configure_docker',
+    'task_start_docker',
     'task_install_flocker',
     'task_install_ssh_key',
     'task_cli_pkg_install',

From 05bcc3ccf3665b4497d8b6c32e1bce1b73a377c9 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 14 Nov 2016 13:38:34 +0000
Subject: [PATCH 13/26] Docstrings

---
 flocker/provision/_install.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index e4ffc58ce4..28114309cf 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -886,7 +886,7 @@ def task_configure_docker(distribution):
 
 def task_start_docker(distribution):
     """
-    foo
+    Enable and (re)start the Docker daemon.
     """
     if is_systemd_distribution(distribution):
         commands = [
@@ -902,7 +902,7 @@ def task_start_docker(distribution):
 
 def task_start_kubelet(distribution):
     """
-    foo
+    Enable and (re)start the Kubernetes Kubelet.
     """
     if is_systemd_distribution(distribution):
         commands = [

From 2bfa812f4ae60e5b88675ad16d9f926202d2798d Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 14 Nov 2016 14:06:13 +0000
Subject: [PATCH 14/26] Use the correct Ubuntu 14.04 repository

---
 flocker/provision/_install.py | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 28114309cf..77480937fa 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1468,12 +1468,6 @@ def task_install_docker(distribution):
 -----END PGP PUBLIC KEY BLOCK-----
 """
 
-KUBERNETES_REPO_APT = """
-deb http://apt.kubernetes.io/ kubernetes-xenial main
-"""
-
-KUBERNETES_REPO_PATH_APT = "/etc/apt/sources.list.d/kubernetes.list"
-
 KUBERNETES_REPO_YUM = """
 [kubernetes]
 name=Kubernetes
@@ -1519,13 +1513,19 @@ def task_install_kubernetes_apt():
                 key_path, key_path
             )
         ),
-        # Upload the repo file
-        put(
-            KUBERNETES_REPO_APT,
-            KUBERNETES_REPO_PATH_APT
-        ),
-        # Install Kubernetes packages
+        # Install Kubernetes repository
+        run(command=b"apt-get update"),
+        run(command=(
+            b"apt-get install -y "
+            b"apt-transport-https software-properties-common"
+        )),
+        run(command=(
+            b"add-apt-repository -y "
+            b"deb http://apt.kubernetes.io/ "
+            b"kubernetes-$(lsb_release --codename --short) main"
+        )),
         run(command=b"apt-get update"),
+        # Install Kubernetes packages
         run(command=(
             b"apt-get install -y "
             b"kubelet kubeadm kubectl kubernetes-cni"

From ac4233241ee83a96a2f93f72aaa41624c0e5a8ec Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 14 Nov 2016 14:45:59 +0000
Subject: [PATCH 15/26] Use a single argument to add-apt-repository

---
 flocker/provision/_install.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 77480937fa..abac8acd14 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1521,8 +1521,8 @@ def task_install_kubernetes_apt():
         )),
         run(command=(
             b"add-apt-repository -y "
-            b"deb http://apt.kubernetes.io/ "
-            b"kubernetes-$(lsb_release --codename --short) main"
+            b'"deb http://apt.kubernetes.io/ '
+            b'kubernetes-$(lsb_release --codename --short)"'
         )),
         run(command=b"apt-get update"),
         # Install Kubernetes packages

From 6ac35170146e7a08a0f882600a805814bebd2de1 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 14 Nov 2016 15:01:22 +0000
Subject: [PATCH 16/26] Don't attempt to delete kubernetes related containers

---
 flocker/acceptance/testtools.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/flocker/acceptance/testtools.py b/flocker/acceptance/testtools.py
index b353ad224b..e644a79063 100644
--- a/flocker/acceptance/testtools.py
+++ b/flocker/acceptance/testtools.py
@@ -874,7 +874,15 @@ def cleanup_all_containers(_):
                 # they're left over from previous test; they might e.g.
                 # have a volume bind-mounted, preventing its destruction.
                 for container in client.containers():
-                    client.remove_container(container["Id"], force=True)
+                    # Don't attempt to remove containers related to
+                    # orchestration frameworks
+                    protected_container = False
+                    label_keys = container["Labels"].keys()
+                    for key in label_keys:
+                        if key.startswith("io.kubernetes."):
+                            protected_container = True
+                    if not protected_container:
+                        client.remove_container(container["Id"], force=True)
 
         def cleanup_flocker_containers(_):
             cleaning_containers = api_clean_state(

From e98e40c144223b332f220739db2418848190bc93 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Mon, 14 Nov 2016 15:48:49 +0000
Subject: [PATCH 17/26] fix repo line

---
 flocker/provision/_install.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index abac8acd14..b4a6feefe4 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1522,7 +1522,7 @@ def task_install_kubernetes_apt():
         run(command=(
             b"add-apt-repository -y "
             b'"deb http://apt.kubernetes.io/ '
-            b'kubernetes-$(lsb_release --codename --short)"'
+            b'kubernetes-$(lsb_release --codename --short) main"'
         )),
         run(command=b"apt-get update"),
         # Install Kubernetes packages

From 59463a78c0417fa3bbb02e84d7e0ed09e63a5a03 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Tue, 15 Nov 2016 11:05:31 +0000
Subject: [PATCH 18/26] The key must have a .gpg extension or it won't be used
 by the apt tools

---
 flocker/provision/_install.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index b4a6feefe4..f9d1b28a58 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1502,7 +1502,8 @@ def kubeadm_token_from_cluster(cluster):
 
 
 def task_install_kubernetes_apt():
-    key_path = b"/etc/apt/trusted.gpg.d/google_cloud_packages_automatic"
+    # The de-armored key must have a .gpg file extension
+    key_path = b"/etc/apt/trusted.gpg.d/google_cloud_packages_automatic.gpg"
     return sequence([
         # Upload the public key rather than downloading from the kubernetes
         # servers every time.

From b23ef6946d39cb006dfae6f78231de5c66052900 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Tue, 15 Nov 2016 11:07:28 +0000
Subject: [PATCH 19/26] Force ebtables to be installed since it's not a
 declared dependency of kubelet or kubeadm. See
 https://github.com/kubernetes/release/pull/197.

---
 flocker/provision/_install.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index f9d1b28a58..9a807ba512 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1556,9 +1556,11 @@ def task_install_kubernetes_yum():
                 KUBERNETES_REPO_PATH_YUM
             ),
             # Install Kubernetes packages
+            # XXX The ebtables dependency isn't declared by the kubelet
+            # package. See https://github.com/kubernetes/release/pull/197
             run(command=(
                 b"yum install -y "
-                b"kubelet kubeadm kubectl kubernetes-cni"
+                b"ebtables kubelet kubeadm kubectl kubernetes-cni"
             )),
         ]
     )

From 219f5da3eac2d53c95b62951c978d23a6b57fba7 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Tue, 15 Nov 2016 11:13:31 +0000
Subject: [PATCH 20/26] Don't run acceptance tests on Ubuntu 14.04

---
 build.yaml | 113 -----------------------------------------------------
 1 file changed, 113 deletions(-)

diff --git a/build.yaml b/build.yaml
index 5a646eab1b..3251cfeb81 100644
--- a/build.yaml
+++ b/build.yaml
@@ -1063,31 +1063,6 @@ job_type:
       timeout: 45
       directories_to_delete: *run_acceptance_directories_to_delete
 
-    run_acceptance_loopback_on_AWS_Ubuntu_Trusty_for:
-      on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
-      with_modules: *run_full_acceptance_modules
-      with_steps:
-        - { type: 'shell',
-            cli: [ *hashbang, *add_shell_functions,
-                   *cleanup, *setup_venv, *setup_flocker_modules,
-                   *check_version,
-                   'export DISTRIBUTION_NAME=ubuntu-14.04',
-                   *build_sdist, *build_package,
-                   *build_repo_metadata,
-                   *setup_authentication,
-                   'export ACCEPTANCE_TEST_MODULE=${MODULE}',
-                   'export ACCEPTANCE_TEST_PROVIDER=aws',
-                   *run_acceptance_loopback_tests,
-                   *convert_results_to_junit,
-                   *clean_packages,
-                   *exit_with_return_code_from_test ]
-          }
-      clean_repo: true
-      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
-      publish_test_results: true
-      timeout: 45
-      directories_to_delete: *run_acceptance_directories_to_delete
-
     run_acceptance_loopback_on_AWS_Ubuntu_Xenial_for:
       on_nodes_with_labels: 'aws-ubuntu-xenial-T2Medium'
       with_modules: *run_full_acceptance_modules
@@ -1301,36 +1276,6 @@ job_type:
       directories_to_delete: []
       notify_slack: '#nightly-builds'
 
-    run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS:
-      at: '0 6 * * *'
-      # flocker.provision is responsible for creating the test nodes on
-      # so we can actually run run-acceptance-tests from GCE
-      on_nodes_with_labels: 'gce-ubuntu16'
-      with_steps:
-        - { type: 'shell',
-            cli: [ *hashbang, *add_shell_functions,
-                   *cleanup, *setup_venv, *setup_flocker_modules,
-                   *check_version,
-                   'export DISTRIBUTION_NAME=ubuntu-14.04',
-                   *build_sdist, *build_package,
-                   *build_repo_metadata,
-                   *setup_authentication,
-                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
-                   'export ACCEPTANCE_TEST_PROVIDER=aws',
-                   *run_acceptance_tests,
-                   *convert_results_to_junit,
-                   *clean_packages,
-                   *exit_with_return_code_from_test ]
-          }
-      clean_repo: true
-      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
-      publish_test_results: true
-      # Similar to the reasoning for run_acceptance_on_AWS_CentOS_7_with_EBS
-      # but slightly shorter since Ubuntu runs the tests faster.
-      timeout: 90
-      directories_to_delete: []
-      notify_slack: '#nightly-builds'
-
     run_acceptance_on_AWS_Ubuntu_Xenial_with_EBS:
       at: '0 6 * * *'
       # flocker.provision is responsible for creating the test nodes on
@@ -1390,35 +1335,6 @@ job_type:
       directories_to_delete: []
       notify_slack: '#nightly-builds'
 
-    run_acceptance_on_GCE_Ubuntu_Trusty_with_GCE:
-      at: '0 6 * * *'
-      # flocker.provision is responsible for creating the test nodes on
-      # so we can actually run run-acceptance-tests from GCE
-      on_nodes_with_labels: 'gce-ubuntu16'
-      with_steps:
-        - { type: 'shell',
-            cli: [ *hashbang, *add_shell_functions,
-                   *cleanup, *setup_venv, *setup_flocker_modules,
-                   *check_version,
-                   'export DISTRIBUTION_NAME=ubuntu-14.04',
-                   *build_sdist, *build_package,
-                   *build_repo_metadata,
-                   *setup_authentication,
-                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
-                   'export ACCEPTANCE_TEST_PROVIDER=gce',
-                   *run_acceptance_tests,
-                   *convert_results_to_junit,
-                   *clean_packages,
-                   *exit_with_return_code_from_test ]
-          }
-      clean_repo: true
-      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
-      publish_test_results: true
-      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
-      timeout: 90
-      directories_to_delete: []
-      notify_slack: '#nightly-builds'
-
     run_acceptance_on_GCE_Ubuntu_Xenial_with_GCE:
       at: '0 6 * * *'
       # flocker.provision is responsible for creating the test nodes on
@@ -1477,35 +1393,6 @@ job_type:
       directories_to_delete: []
       notify_slack: '#nightly-builds'
 
-    run_acceptance_on_Rackspace_Ubuntu_Trusty_with_Cinder:
-      at: '0 6 * * *'
-      # flocker.provision is responsible for creating the test nodes on
-      # so we can actually run run-acceptance-tests from GCE
-      on_nodes_with_labels: 'gce-ubuntu16'
-      with_steps:
-        - { type: 'shell',
-            cli: [ *hashbang, *add_shell_functions,
-                   *cleanup, *setup_venv, *setup_flocker_modules,
-                   *check_version,
-                   'export DISTRIBUTION_NAME=ubuntu-14.04',
-                   *build_sdist, *build_package,
-                   *build_repo_metadata,
-                   *setup_authentication,
-                   'export ACCEPTANCE_TEST_MODULE=flocker.acceptance',
-                   'export ACCEPTANCE_TEST_PROVIDER=rackspace',
-                   *run_acceptance_tests,
-                   *convert_results_to_junit,
-                   *clean_packages,
-                   *exit_with_return_code_from_test ]
-          }
-      clean_repo: true
-      archive_artifacts: *acceptance_tests_artifacts_ubuntu_special_case
-      publish_test_results: true
-      # Reasoning as for run_acceptance_on_AWS_Ubuntu_Trusty_with_EBS
-      timeout: 90
-      directories_to_delete: []
-      notify_slack: '#nightly-builds'
-
     run_acceptance_on_Rackspace_Ubuntu_Xenial_with_Cinder:
       at: '0 6 * * *'
       # flocker.provision is responsible for creating the test nodes on

From 813980bd3763877917c9b697a08f7576f845e13f Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Tue, 15 Nov 2016 17:34:54 +0000
Subject: [PATCH 21/26] Just enough to connect to Kubernetes API and create a
 namespace for the test

---
 .../endtoend/test_kubernetesplugin.py         | 143 ++++++++++++++++++
 flocker/ca/_validation.py                     |  35 +++++
 2 files changed, 178 insertions(+)
 create mode 100644 flocker/acceptance/endtoend/test_kubernetesplugin.py

diff --git a/flocker/acceptance/endtoend/test_kubernetesplugin.py b/flocker/acceptance/endtoend/test_kubernetesplugin.py
new file mode 100644
index 0000000000..6623292f02
--- /dev/null
+++ b/flocker/acceptance/endtoend/test_kubernetesplugin.py
@@ -0,0 +1,143 @@
+# Copyright ClusterHQ Inc.  See LICENSE file for details.
+
+"""
+Tests for the Flocker Kubernetes plugin.
+"""
+import os
+import json
+from pyrsistent import PClass, field
+from twisted.internet import reactor
+
+from ...testtools import AsyncTestCase, async_runner, random_name
+from ..testtools import (
+    require_cluster, ACCEPTANCE_TEST_TIMEOUT, check_and_decode_json
+)
+
+from ...ca._validation import treq_with_ca
+from twisted.web.http import (
+    CREATED as HTTP_CREATED,
+    OK as HTTP_OK
+)
+from twisted.python.filepath import FilePath
+FLOCKER_ROOT = FilePath(__file__).parent().parent().parent().parent()
+KUBERNETES_DEPLOYMENT = {
+    "apiVersion": "extensions/v1beta1",
+    "metadata": {
+        "name": "nginx-deployment"
+    },
+    "kind": "Deployment",
+    "spec": {
+        "template": {
+            "spec": {
+                "containers": [
+                    {
+                        "image": "nginx:1.7.9",
+                        "name": "nginx",
+                        "ports": [
+                            {
+                                "containerPort": 80
+                            }
+                        ]
+                    }
+                ]
+            },
+            "metadata": {
+                "labels": {
+                    "app": "nginx"
+                }
+            }
+        },
+        "replicas": 3
+    },
+}
+
+
+class KubernetesClient(PClass):
+    client = field()
+    baseurl = field()
+    token = field()
+
+    def namespace_create(self, name):
+        namespace = {
+            "apiVersion": "v1",
+            "kind": "Namespace",
+            "metadata": {
+                "name": name,
+            }
+        }
+        d = self.client.post(
+            self.baseurl + b"/api/v1/namespaces",
+            json.dumps(namespace),
+            headers={
+                b"content-type": b"application/json",
+                b"Authorization": b"Bearer {}".format(self.token),
+            },
+        )
+        d.addCallback(check_and_decode_json, HTTP_CREATED)
+        return d
+
+    def namespace_delete(self, name):
+        d = self.client.delete(
+            self.baseurl + b"/api/v1/namespaces/" + name,
+            headers={
+                b"content-type": b"application/json",
+                b"Authorization": b"Bearer {}".format(self.token),
+            },
+        )
+        d.addCallback(check_and_decode_json, HTTP_OK)
+        return d
+
+
+def kubernetes_client(reactor, api_address, api_port, token):
+    return KubernetesClient(
+        client=treq_with_ca(
+            reactor,
+            ca_path=FLOCKER_ROOT.descendant([".kube", "config", "ca.pem"]),
+            expected_common_name=u"kubernetes",
+        ),
+        baseurl=b"https://%s:%s" % (api_address, api_port),
+        token=token,
+    )
+
+
+def kubernetes_namespace_for_test(test, client):
+    # Namespace must be a DNS label and at most 63 characters
+    namespace_name = random_name(test)
+    namespace_name = namespace_name[-63:]
+    namespace_name = "-".join(filter(None, namespace_name.split("_")))
+    namespace_name = namespace_name.lower()
+
+    d = client.namespace_create(name=namespace_name)
+
+    def delete_namespace():
+        return client.namespace_delete(namespace_name)
+
+    def setup_cleanup(ignored_result):
+        test.addCleanup(delete_namespace)
+
+    d.addCallback(setup_cleanup)
+    d.addCallback(lambda _: namespace_name)
+    return d
+
+
+class KubernetesPluginTests(AsyncTestCase):
+    """
+    Tests for the Kubernetes plugin.
+    """
+    run_tests_with = async_runner(timeout=ACCEPTANCE_TEST_TIMEOUT)
+
+    @require_cluster(1)
+    def test_create_pod(self, cluster):
+        """
+        A pod with a Flocker volume can be created.
+        """
+        client = kubernetes_client(
+            reactor,
+            api_address=cluster.control_node.public_address,
+            api_port=6443,
+            token=os.environ["FLOCKER_ACCEPTANCE_KUBERNETES_TOKEN"]
+        )
+
+        d = kubernetes_namespace_for_test(self, client)
+
+        return d
diff --git a/flocker/ca/_validation.py b/flocker/ca/_validation.py
index 40f322edc8..3089eea57f 100644
--- a/flocker/ca/_validation.py
+++ b/flocker/ca/_validation.py
@@ -138,3 +138,38 @@ def treq_with_authentication(reactor, ca_path, user_cert_path, user_key_path):
     policy = ControlServicePolicy(
         ca_certificate=ca, client_credential=user_credential.credential)
     return HTTPClient(Agent(reactor, contextFactory=policy))
+
+
+@implementer(IPolicyForHTTPS)
+class KubernetesPolicy(PClass):
+    """
+    HTTPS TLS policy for validating the Kubernetes master identity.
+
+    :ivar FlockerCredential client_credential: Client's certificate and
+        private key pair.
+    """
+    ca_certificate = field(mandatory=True)
+    expected_common_name = field(mandatory=True, type=unicode)
+
+    def creatorForNetloc(self, hostname, port):
+        return optionsForClientTLS(
+            self.expected_common_name,
+            trustRoot=self.ca_certificate,
+        )
+
+
+def treq_with_ca(reactor, ca_path, expected_common_name):
+    """
+    Create a ``treq``-API object that trusts a custom certificate authority.
+
+    :param reactor: The reactor to use.
+    :param FilePath ca_path: Absolute path to the public cluster certificate.
+
+    :return: ``treq`` compatible object.
+    """
+    ca = Certificate.loadPEM(ca_path.getContent())
+    policy = KubernetesPolicy(
+        ca_certificate=ca,
+        expected_common_name=expected_common_name,
+    )
+    return HTTPClient(Agent(reactor, contextFactory=policy))

From 56e0afdc89862c8c35bd04a1d4ab446c8c752942 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Tue, 15 Nov 2016 18:13:17 +0000
Subject: [PATCH 22/26] An attempt at a method to create any resource

---
 admin/acceptance.py                           |   1 +
 .../endtoend/test_kubernetesplugin.py         | 152 ++++++++++++++++++
 2 files changed, 153 insertions(+)

diff --git a/admin/acceptance.py b/admin/acceptance.py
index bb203ab0c1..0c99249334 100644
--- a/admin/acceptance.py
+++ b/admin/acceptance.py
@@ -1463,6 +1463,7 @@ def main(reactor, args, base_path, top_level):
     reached_finally = False
 
     def cluster_cleanup():
+        return
         if not reached_finally:
             print "interrupted..."
         print "stopping cluster"
diff --git a/flocker/acceptance/endtoend/test_kubernetesplugin.py b/flocker/acceptance/endtoend/test_kubernetesplugin.py
index 6623292f02..aa99b9b934 100644
--- a/flocker/acceptance/endtoend/test_kubernetesplugin.py
+++ b/flocker/acceptance/endtoend/test_kubernetesplugin.py
@@ -20,6 +20,113 @@
 )
 from twisted.python.filepath import FilePath
 FLOCKER_ROOT = FilePath(__file__).parent().parent().parent().parent()
+
+# Cached output of:
+# curl ...  https://kubernetes:6443/apis/extensions/v1beta1
+KUBERNETES_TYPES = json.loads("""
+[{
+  "kind": "APIResourceList",
+  "groupVersion": "extensions/v1beta1",
+  "resources": [
+    {
+      "name": "daemonsets",
+      "namespaced": true,
+      "kind": "DaemonSet"
+    },
+    {
+      "name": "daemonsets/status",
+      "namespaced": true,
+      "kind": "DaemonSet"
+    },
+    {
+      "name": "deployments",
+      "namespaced": true,
+      "kind": "Deployment"
+    },
+    {
+      "name": "deployments/rollback",
+      "namespaced": true,
+      "kind": "DeploymentRollback"
+    },
+    {
+      "name": "deployments/scale",
+      "namespaced": true,
+      "kind": "Scale"
+    },
+    {
+      "name": "deployments/status",
+      "namespaced": true,
+      "kind": "Deployment"
+    },
+    {
+      "name": "horizontalpodautoscalers",
+      "namespaced": true,
+      "kind": "HorizontalPodAutoscaler"
+    },
+    {
+      "name": "horizontalpodautoscalers/status",
+      "namespaced": true,
+      "kind": "HorizontalPodAutoscaler"
+    },
+    {
+      "name": "ingresses",
+      "namespaced": true,
+      "kind": "Ingress"
+    },
+    {
+      "name": "ingresses/status",
+      "namespaced": true,
+      "kind": "Ingress"
+    },
+    {
+      "name": "jobs",
+      "namespaced": true,
+      "kind": "Job"
+    },
+    {
+      "name": "jobs/status",
+      "namespaced": true,
+      "kind": "Job"
+    },
+    {
+      "name": "networkpolicies",
+      "namespaced": true,
+      "kind": "NetworkPolicy"
+    },
+    {
+      "name": "replicasets",
+      "namespaced": true,
+      "kind": "ReplicaSet"
+    },
+    {
+      "name": "replicasets/scale",
+      "namespaced": true,
+      "kind": "Scale"
+    },
+    {
+      "name": "replicasets/status",
+      "namespaced": true,
+      "kind": "ReplicaSet"
+    },
+    {
+      "name": "replicationcontrollers",
+      "namespaced": true,
+      "kind": "ReplicationControllerDummy"
+    },
+    {
+      "name": "replicationcontrollers/scale",
+      "namespaced": true,
+      "kind": "Scale"
+    },
+    {
+      "name": "thirdpartyresources",
+      "namespaced": false,
+      "kind": "ThirdPartyResource"
+    }
+  ]
+}]
+""")
+
 KUBERNETES_DEPLOYMENT = {
     "apiVersion": "extensions/v1beta1",
     "metadata": {
@@ -87,6 +194,48 @@ def namespace_delete(self, name):
         d.addCallback(check_and_decode_json, HTTP_OK)
         return d
 
+    def create_resource(self, namespace, resource):
+        resource_group_version = resource["apiVersion"]
+        resource_kind = resource["kind"]
+        # Lookup resource list
+        for resource_list in KUBERNETES_TYPES:
+            if resource_list["groupVersion"] == resource_group_version:
+                break
+        else:
+            raise Exception(
+                "resource_group_version not recognized",
+                resource_group_version
+            )
+        # Lookup the "kind"
+        for resource in resource_list["resources"]:
+            if resource["kind"] == resource_kind:
+                break
+        else:
+            raise Exception(
+                "resource_kind not recognized",
+                resource_kind
+            )
+
+        url = "/".join([
+            self.baseurl,
+            "apis",
+            resource_group_version,
+            "namespaces",
+            namespace,
+            resource["name"]
+        ])
+        print "RICHARDW URL:", url
+        d = self.client.post(
+            url,
+            json.dumps(resource),
+            headers={
+                b"content-type": b"application/json",
+                b"Authorization": b"Bearer {}".format(self.token),
+            },
+        )
+        d.addCallback(check_and_decode_json, HTTP_CREATED)
+        return d
+
 
 def kubernetes_client(reactor, api_address, api_port, token):
     return KubernetesClient(
@@ -140,4 +289,7 @@ def test_create_pod(self, cluster):
 
         d = kubernetes_namespace_for_test(self, client)
 
+        def create_deployment(namespace):
+            return client.create_resource(namespace, KUBERNETES_DEPLOYMENT)
+        d.addCallback(create_deployment)
         return d

From 49f7be8150a8f1363e27eb2c36836cf2d025a5cf Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 16 Nov 2016 11:39:35 +0000
Subject: [PATCH 23/26] Add logging to figure out why resources weren't being
 created.

---
 .../endtoend/test_kubernetesplugin.py         | 52 ++++++++++++-------
 1 file changed, 33 insertions(+), 19 deletions(-)

diff --git a/flocker/acceptance/endtoend/test_kubernetesplugin.py b/flocker/acceptance/endtoend/test_kubernetesplugin.py
index aa99b9b934..c7a2d3ef30 100644
--- a/flocker/acceptance/endtoend/test_kubernetesplugin.py
+++ b/flocker/acceptance/endtoend/test_kubernetesplugin.py
@@ -7,7 +7,8 @@
 import json
 from pyrsistent import PClass, field
 from twisted.internet import reactor
-
+from eliot import start_action, Message
+from eliot.twisted import DeferredContext
 from ...testtools import AsyncTestCase, async_runner, random_name
 from ..testtools import (
     require_cluster, ACCEPTANCE_TEST_TIMEOUT, check_and_decode_json
@@ -23,7 +24,7 @@
 
 # Cached output of:
 # curl ...  https://kubernetes:6443/apis/extensions/v1beta1
-KUBERNETES_TYPES = json.loads("""
+KUBERNETES_API_GROUPS = json.loads("""
 [{
   "kind": "APIResourceList",
   "groupVersion": "extensions/v1beta1",
@@ -194,12 +195,13 @@ def namespace_delete(self, name):
         d.addCallback(check_and_decode_json, HTTP_OK)
         return d
 
-    def create_resource(self, namespace, resource):
+    def _endpoint_url_for_resource(self, namespace, resource):
         resource_group_version = resource["apiVersion"]
         resource_kind = resource["kind"]
+
         # Lookup resource list
-        for resource_list in KUBERNETES_TYPES:
-            if resource_list["groupVersion"] == resource_group_version:
+        for group_info in KUBERNETES_API_GROUPS:
+            if group_info["groupVersion"] == resource_group_version:
                 break
         else:
             raise Exception(
@@ -207,8 +209,8 @@ def create_resource(self, namespace, resource):
                 resource_group_version
             )
         # Lookup the "kind"
-        for resource in resource_list["resources"]:
-            if resource["kind"] == resource_kind:
+        for resource_meta in group_info["resources"]:
+            if resource_meta["kind"] == resource_kind:
                 break
         else:
             raise Exception(
@@ -216,25 +218,37 @@ def create_resource(self, namespace, resource):
                 resource_kind
             )
 
-        url = "/".join([
+        return "/".join([
             self.baseurl,
             "apis",
             resource_group_version,
             "namespaces",
             namespace,
-            resource["name"]
+            resource_meta["name"]
         ])
-        print "RICHARDW URL:", url
-        d = self.client.post(
-            url,
-            json.dumps(resource),
-            headers={
-                b"content-type": b"application/json",
-                b"Authorization": b"Bearer {}".format(self.token),
-            },
+
+    def create_resource(self, namespace, resource):
+        url = self._endpoint_url_for_resource(namespace, resource)
+        action = start_action(
+            action_type=u"create_resource",
+            namespace=namespace,
+            resource=resource,
+            url=url,
         )
-        d.addCallback(check_and_decode_json, HTTP_CREATED)
-        return d
+
+        with action.context():
+            d = self.client.post(
+                url,
+                json.dumps(resource),
+                headers={
+                    b"content-type": b"application/json",
+                    b"Authorization": b"Bearer {}".format(self.token),
+                },
+            )
+            d = DeferredContext(d)
+            d.addCallback(check_and_decode_json, HTTP_CREATED)
+            d.addActionFinish()
+            return d.result
 
 
 def kubernetes_client(reactor, api_address, api_port, token):

From 45013598c49621a7a690fece6f45a2a9f8f27c78 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 16 Nov 2016 17:58:21 +0000
Subject: [PATCH 24/26] Attempt to create a pod with a Flocker volume

---
 .../endtoend/test_kubernetesplugin.py         | 230 +++++++++++++++++-
 1 file changed, 224 insertions(+), 6 deletions(-)

diff --git a/flocker/acceptance/endtoend/test_kubernetesplugin.py b/flocker/acceptance/endtoend/test_kubernetesplugin.py
index c7a2d3ef30..0a41ac506e 100644
--- a/flocker/acceptance/endtoend/test_kubernetesplugin.py
+++ b/flocker/acceptance/endtoend/test_kubernetesplugin.py
@@ -5,6 +5,7 @@
 """
 import os
 import json
+import yaml
 from pyrsistent import PClass, field
 from twisted.internet import reactor
 from eliot import start_action, Message
@@ -25,7 +26,193 @@
 # Cached output of:
 # curl ...  https://kubernetes:6443/apis/extensions/v1beta1
 KUBERNETES_API_GROUPS = json.loads("""
-[{
+{"api": {
+  "kind": "APIResourceList",
+  "groupVersion": "v1",
+  "resources": [
+    {
+      "name": "bindings",
+      "namespaced": true,
+      "kind": "Binding"
+    },
+    {
+      "name": "componentstatuses",
+      "namespaced": false,
+      "kind": "ComponentStatus"
+    },
+    {
+      "name": "configmaps",
+      "namespaced": true,
+      "kind": "ConfigMap"
+    },
+    {
+      "name": "endpoints",
+      "namespaced": true,
+      "kind": "Endpoints"
+    },
+    {
+      "name": "events",
+      "namespaced": true,
+      "kind": "Event"
+    },
+    {
+      "name": "limitranges",
+      "namespaced": true,
+      "kind": "LimitRange"
+    },
+    {
+      "name": "namespaces",
+      "namespaced": false,
+      "kind": "Namespace"
+    },
+    {
+      "name": "namespaces/finalize",
+      "namespaced": false,
+      "kind": "Namespace"
+    },
+    {
+      "name": "namespaces/status",
+      "namespaced": false,
+      "kind": "Namespace"
+    },
+    {
+      "name": "nodes",
+      "namespaced": false,
+      "kind": "Node"
+    },
+    {
+      "name": "nodes/proxy",
+      "namespaced": false,
+      "kind": "Node"
+    },
+    {
+      "name": "nodes/status",
+      "namespaced": false,
+      "kind": "Node"
+    },
+    {
+      "name": "persistentvolumeclaims",
+      "namespaced": true,
+      "kind": "PersistentVolumeClaim"
+    },
+    {
+      "name": "persistentvolumeclaims/status",
+      "namespaced": true,
+      "kind": "PersistentVolumeClaim"
+    },
+    {
+      "name": "persistentvolumes",
+      "namespaced": false,
+      "kind": "PersistentVolume"
+    },
+    {
+      "name": "persistentvolumes/status",
+      "namespaced": false,
+      "kind": "PersistentVolume"
+    },
+    {
+      "name": "pods",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "pods/attach",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "pods/binding",
+      "namespaced": true,
+      "kind": "Binding"
+    },
+    {
+      "name": "pods/eviction",
+      "namespaced": true,
+      "kind": "Eviction"
+    },
+    {
+      "name": "pods/exec",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "pods/log",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "pods/portforward",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "pods/proxy",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "pods/status",
+      "namespaced": true,
+      "kind": "Pod"
+    },
+    {
+      "name": "podtemplates",
+      "namespaced": true,
+      "kind": "PodTemplate"
+    },
+    {
+      "name": "replicationcontrollers",
+      "namespaced": true,
+      "kind": "ReplicationController"
+    },
+    {
+      "name": "replicationcontrollers/scale",
+      "namespaced": true,
+      "kind": "Scale"
+    },
+    {
+      "name": "replicationcontrollers/status",
+      "namespaced": true,
+      "kind": "ReplicationController"
+    },
+    {
+      "name": "resourcequotas",
+      "namespaced": true,
+      "kind": "ResourceQuota"
+    },
+    {
+      "name": "resourcequotas/status",
+      "namespaced": true,
+      "kind": "ResourceQuota"
+    },
+    {
+      "name": "secrets",
+      "namespaced": true,
+      "kind": "Secret"
+    },
+    {
+      "name": "serviceaccounts",
+      "namespaced": true,
+      "kind": "ServiceAccount"
+    },
+    {
+      "name": "services",
+      "namespaced": true,
+      "kind": "Service"
+    },
+    {
+      "name": "services/proxy",
+      "namespaced": true,
+      "kind": "Service"
+    },
+    {
+      "name": "services/status",
+      "namespaced": true,
+      "kind": "Service"
+    }
+  ]
+},
+"apis": {
   "kind": "APIResourceList",
   "groupVersion": "extensions/v1beta1",
   "resources": [
@@ -125,7 +312,7 @@
       "kind": "ThirdPartyResource"
     }
   ]
-}]
+}}
 """)
 
 KUBERNETES_DEPLOYMENT = {
@@ -155,10 +342,32 @@
                 }
             }
         },
-        "replicas": 3
+        "replicas": 1
     },
 }
 
+FLOCKER_POD = yaml.safe_load("""\
+apiVersion: v1
+kind: Pod
+metadata:
+  name: flocker-web
+spec:
+  containers:
+    - name: web
+      image: nginx
+      ports:
+        - name: web
+          containerPort: 80
+      volumeMounts:
+          # name must match the volume name below
+          - name: www-root
+            mountPath: "/usr/share/nginx/html"
+  volumes:
+    - name: www-root
+      flocker:
+          datasetName: my-flocker-vol
+""")
+
 
 class KubernetesClient(PClass):
     client = field()
@@ -200,7 +409,7 @@ def _endpoint_url_for_resource(self, namespace, resource):
         resource_kind = resource["kind"]
 
         # Lookup resource list
-        for group_info in KUBERNETES_API_GROUPS:
+        for first_url_segment, group_info in KUBERNETES_API_GROUPS.items():
             if group_info["groupVersion"] == resource_group_version:
                 break
         else:
@@ -220,7 +429,7 @@ def _endpoint_url_for_resource(self, namespace, resource):
 
         return "/".join([
             self.baseurl,
-            "apis",
+            first_url_segment,
             resource_group_version,
             "namespaces",
             namespace,
@@ -264,6 +473,14 @@ def kubernetes_client(reactor, api_address, api_port, token):
 
 
 def kubernetes_namespace_for_test(test, client):
+    """
+    Create a unique Kubernetes namespace in which to create Kubernetes test
+    resources. The namespace will be deleted when the test completes. And
+    Kubernetes *should* then garbage collect all the resources in that
+    namespace.
+    XXX: Although it doesn't always seem to work:
+    https://github.com/kubernetes/kubernetes/issues/36891
+    """
     # Namespace must be a DNS label and at most 63 characters
     namespace_name = random_name(test)
     namespace_name = namespace_name[-63:]
@@ -276,6 +493,7 @@ def delete_namespace():
         return client.namespace_delete(namespace_name)
 
     def setup_cleanup(ignored_result):
+        return
         test.addCleanup(delete_namespace)
 
     d.addCallback(setup_cleanup)
@@ -304,6 +522,6 @@ def test_create_pod(self, cluster):
         d = kubernetes_namespace_for_test(self, client)
 
         def create_deployment(namespace):
-            return client.create_resource(namespace, KUBERNETES_DEPLOYMENT)
+            return client.create_resource(namespace, FLOCKER_POD)
         d.addCallback(create_deployment)
         return d

From ada9477a66da25acb5ff6dfb4eb254c56135be70 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 16 Nov 2016 19:28:16 +0000
Subject: [PATCH 25/26] Configure the Kubernetes Flocker plugin

---
 flocker/provision/_install.py | 69 +++++++++++++++++++++++++++++------
 1 file changed, 57 insertions(+), 12 deletions(-)

diff --git a/flocker/provision/_install.py b/flocker/provision/_install.py
index 9a807ba512..b3f435fc0f 100644
--- a/flocker/provision/_install.py
+++ b/flocker/provision/_install.py
@@ -1013,6 +1013,49 @@ def task_enable_docker_plugin(distribution):
     else:
         raise DistributionNotSupported(distribution=distribution)
 
+# A systemd configuration snippet that compliments the systemd service
+# configuration installed by kubeadm
+KUBELET_FLOCKER_PLUGIN_SYSTEMD_CONFIG = """
+[Service]
+EnvironmentFile=/etc/flocker/env
+"""
+
+# A file containing the location of Flocker control service and certificates to
+# allow the Kubernetes plugin to authenticate with the Flocker REST API.
+ETC_FLOCKER_ENV_TEMPLATE = """
+FLOCKER_CONTROL_SERVICE_HOST={control_service_host}
+FLOCKER_CONTROL_SERVICE_PORT=4523
+FLOCKER_CONTROL_SERVICE_CA_FILE=/etc/flocker/cluster.crt
+FLOCKER_CONTROL_SERVICE_CLIENT_KEY_FILE=/etc/flocker/plugin.key
+FLOCKER_CONTROL_SERVICE_CLIENT_CERT_FILE=/etc/flocker/plugin.crt
+"""
+
+
+def task_enable_kubernetes_plugin(flocker_control_service_host):
+    """
+    Enable the Flocker Kubernetes plugin.
+    By adding a systemd configuration snippet that makes FLOCKER configuration
+    environment variables available to the kubelet.
+
+    :param bytes flocker_control_service_host: The address or hostname of the
+        Flocker control service.
+    """
+    return sequence([
+        put(
+            content=ETC_FLOCKER_ENV_TEMPLATE.format(
+                control_service_host=flocker_control_service_host
+            ),
+            path=b"/etc/flocker/env",
+        ),
+        put(
+            content=KUBELET_FLOCKER_PLUGIN_SYSTEMD_CONFIG,
+            path=(
+                b"/etc/systemd/system/kubelet.service.d/20-flocker-plugin.conf"
+            )
+        ),
+        run_from_args(['systemctl', 'restart', 'kubelet']),
+    ])
+
 
 def task_open_control_firewall(distribution):
     """
@@ -2058,18 +2101,7 @@ def configure_node(
     if provider == "managed":
         setup_action = 'restart'
 
-    if node is cluster.control_node:
-        commands = []
-    else:
-        commands = [
-            task_configure_kubernetes_node(
-                distribution=node.distribution,
-                token=kubeadm_token_from_cluster(cluster),
-                master_ip=cluster.control_node.address,
-            ),
-        ]
-
-    commands.extend([
+    commands = [
         task_install_node_certificates(
             cluster.certificates.cluster.certificate,
             certnkey.certificate,
@@ -2095,6 +2127,19 @@ def configure_node(
             distribution=node.distribution,
             action=setup_action,
         ),
+        task_enable_kubernetes_plugin(cluster.control_node.public_address),
+    ]
+
+    if node is not cluster.control_node:
+        commands = [
+            task_configure_kubernetes_node(
+                distribution=node.distribution,
+                token=kubeadm_token_from_cluster(cluster),
+                master_ip=cluster.control_node.address,
+            ),
+        ]
+
+    commands.extend([
         # Restart docker after pushing the Flocker certificates and the Docker
         # configuration modifications which make it use Flocker certificates
         # for listening on a TLS / TCP port.

From eb2c2570d8f17d7b5abc02db0ca07a5c982aedc6 Mon Sep 17 00:00:00 2001
From: Richard Wall <richard.wall@clusterhq.com>
Date: Wed, 16 Nov 2016 19:39:06 +0000
Subject: [PATCH 26/26] Create a Flocker volume before creating the pod

---
 .../endtoend/test_kubernetesplugin.py         | 63 ++++++++-----------
 1 file changed, 27 insertions(+), 36 deletions(-)

diff --git a/flocker/acceptance/endtoend/test_kubernetesplugin.py b/flocker/acceptance/endtoend/test_kubernetesplugin.py
index 0a41ac506e..5536c2b75e 100644
--- a/flocker/acceptance/endtoend/test_kubernetesplugin.py
+++ b/flocker/acceptance/endtoend/test_kubernetesplugin.py
@@ -8,11 +8,12 @@
 import yaml
 from pyrsistent import PClass, field
 from twisted.internet import reactor
-from eliot import start_action, Message
+from eliot import start_action
 from eliot.twisted import DeferredContext
 from ...testtools import AsyncTestCase, async_runner, random_name
 from ..testtools import (
-    require_cluster, ACCEPTANCE_TEST_TIMEOUT, check_and_decode_json
+    require_cluster, ACCEPTANCE_TEST_TIMEOUT, check_and_decode_json,
+    create_dataset,
 )
 
 from ...ca._validation import treq_with_ca
@@ -315,38 +316,8 @@
 }}
 """)
 
-KUBERNETES_DEPLOYMENT = {
-    "apiVersion": "extensions/v1beta1",
-    "metadata": {
-        "name": "nginx-deployment"
-    },
-    "kind": "Deployment",
-    "spec": {
-        "template": {
-            "spec": {
-                "containers": [
-                    {
-                        "image": "nginx:1.7.9",
-                        "name": "nginx",
-                        "ports": [
-                            {
-                                "containerPort": 80
-                            }
-                        ]
-                    }
-                ]
-            },
-            "metadata": {
-                "labels": {
-                    "app": "nginx"
-                }
-            }
-        },
-        "replicas": 1
-    },
-}
 
-FLOCKER_POD = yaml.safe_load("""\
+FLOCKER_POD_TEMPLATE = """\
 apiVersion: v1
 kind: Pod
 metadata:
@@ -365,8 +336,8 @@
   volumes:
     - name: www-root
       flocker:
-          datasetName: my-flocker-vol
-""")
+          datasetName: {flocker_volume_name}
+"""
 
 
 class KubernetesClient(PClass):
@@ -512,6 +483,7 @@ def test_create_pod(self, cluster):
         """
         A pod with a Flocker volume can be created.
         """
+        flocker_volume_name = random_name(self)
         client = kubernetes_client(
             reactor,
             api_address=cluster.control_node.public_address,
@@ -521,7 +493,26 @@ def test_create_pod(self, cluster):
 
         d = kubernetes_namespace_for_test(self, client)
 
+        def create_flocker_volume(namespace):
+            d = create_dataset(
+                test_case=self,
+                cluster=cluster,
+                metadata=dict(
+                    name=flocker_volume_name
+                )
+            )
+            d.addCallback(lambda _ignored: namespace)
+            return d
+        d.addCallback(create_flocker_volume)
+
         def create_deployment(namespace):
-            return client.create_resource(namespace, FLOCKER_POD)
+            return client.create_resource(
+                namespace,
+                yaml.safe_load(
+                    FLOCKER_POD_TEMPLATE.format(
+                        flocker_volume_name=flocker_volume_name
+                    )
+                )
+            )
         d.addCallback(create_deployment)
         return d