Add cloud-init compatibility for Ubuntu VMs

* Add cloud-init disk with config files
* update root ssh keys via qemu agent in ensure for ubuntu VMs

PL-133325

CHANGES.txt

@@ -18,6 +18,10 @@ Release notes
 
 - rbd: fix cluster load issue due to overuse of rbd.list (PL-133194)
 
+- Add support for cloud-init ubuntu VMs (PL-133325, PL-133372)
+  Add cloud-init `cidata` volume unconditionally for all VMs.
+  Sync SSH root keys for ubuntu VMs via qmp in ensure.
+
 1.6 (2024-10-23)
 ----------------
 

src/fc/qemu/agent.py

@@ -330,6 +330,14 @@ def config_file_staging(self):
     def lock_file(self):
         return self.prefix / "run" / f"qemu.{self.name}.lock"
 
+    @property
+    def users_file(self):
+        return (
+            self.prefix
+            / "etc/qemu/users"
+            / f"{self.cfg['resource_group']}.json"
+        )
+
     def _load_enc(self):
         try:
             with self.config_file.open() as f:
@@ -346,6 +354,10 @@ def _load_enc(self):
                     "Could not load {}".format(self.config_file)
                 )
 
+    def _load_users(self):
+        with self.users_file.open() as f:
+            return json.load(f)
+
     @classmethod
     def handle_consul_event(cls, input: typing.TextIO = sys.stdin):
         # Python in our environments defaults to UTF-8 and we generally
@@ -899,6 +911,7 @@ def _update_from_enc(self):
         self.cfg["root_size"] = self.cfg["disk"] * (1024**3)
         self.cfg["swap_size"] = swap_size(self.cfg["memory"])
         self.cfg["tmp_size"] = tmp_size(self.cfg["disk"])
+        self.cfg["cidata_size"] = 10 * MiB
         self.cfg["ceph_id"] = self.ceph_id
         self.cfg["cpu_model"] = self.cfg.get("cpu_model", "qemu64")
         self.cfg["binary_generation"] = self.binary_generation
@@ -1091,6 +1104,7 @@ def ensure_online_local(self):
             self.ensure_thawed()
             self.mark_qemu_binary_generation()
             self.mark_qemu_guest_properties()
+            self.update_root_ssh_keys_cloudinit()
 
     def cleanup(self):
         """Removes various run and tmp files."""
@@ -1135,6 +1149,23 @@ def mark_qemu_binary_generation(self):
         except Exception as e:
             self.log.error("mark-qemu-binary-generation", reason=str(e))
 
+    def update_root_ssh_keys_cloudinit(self):
+        # TODO: use environment_class_type != 'cloudinit'
+        if self.cfg["environment_class"].lower() != "ubuntu":
+            return
+        self.log.info("update-root-ssh-keys-cloudinit")
+        try:
+            users = self._load_users()
+            to_write_text = util.generate_cloudinit_ssh_keyfile(
+                users, self.cfg["resource_group"]
+            )
+            self.qemu.write_file(
+                "/root/.ssh/authorized_keys_fc",
+                to_write_text.encode("utf-8"),
+            )
+        except Exception as e:
+            self.log.error("update-root-ssh-keys-cloudinit", reason=str(e))
+
     def ensure_online_disk_size(self):
         """Trigger block resize action for the root disk."""
         target_size = self.cfg["root_size"]

src/fc/qemu/default.conf

@@ -28,3 +28,4 @@ lock_host = localhost
 create-vm = create-vm -I {rbd_pool} {name}
 mkfs-xfs = -q -f -K -m crc=1,finobt=1 -d su=4m,sw=1
 mkfs-ext4 = -q -m 1 -E nodiscard
+mkfs-vfat =

src/fc/qemu/hazmat/ceph.py

@@ -4,12 +4,17 @@
 """
 
 import hashlib
+import ipaddress
 import json
 import os
+from pathlib import Path
 from typing import Dict, Optional
 
 import rados
 import rbd
+import yaml
+
+from fc.qemu.util import generate_cloudinit_ssh_keyfile
 
 from ..sysconfig import sysconfig
 from ..timeout import TimeoutError
@@ -333,6 +338,132 @@ def seed(self, enc, generation):
                 f.write(str(generation) + "\n")
 
 
+class CloudInitSpec(VolumeSpecification):
+    suffix = "cidata"
+
+    def pre_start(self):
+        for pool in self.exists_in_pools():
+            if pool != self.desired_pool:
+                self.log.info(
+                    "delete-outdated-cloud-init", pool=pool, image=self.name
+                )
+                self.ceph.remove_volume(self.name, pool)
+
+    def start(self):
+        self.log.info("start-cloud-init")
+        with self.volume.mapped():
+            self.mkfs()
+            self.seed(self.ceph.enc)
+
+    def mkfs(self):
+        self.log.debug("create-fs")
+        device = self.volume.device
+        assert device, f"volume must be mapped first: {device}"
+        self.cmd(f'sgdisk -o "{device}"')
+        self.cmd(f'sgdisk -n 1:: -c "1:{self.suffix}" ' f'-t 1:8300 "{device}"')
+        self.cmd(f"partprobe {device}")
+        self.volume.wait_for_part1dev()
+        options = getattr(self.ceph, "MKFS_VFAT")
+        self.cmd(
+            f'mkfs.vfat {options} -n "{self.suffix}" {self.volume.part1dev}'
+        )
+
+    def seed(self, enc):
+        self.log.info("seed")
+        managed_files = [
+            {
+                "path": "/etc/ssh/sshd_config.d/10-cloud-init-fc.conf",
+                "content": """\
+        AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys_fc
+        """,
+                "permissions": "0644",
+            }
+        ]
+
+        # Improve factoring
+        try:
+            with Path(
+                f"/etc/qemu/users/{self.ceph.cfg['resource_group']}.json"
+            ).open() as f:
+                users = json.load(f)
+                ssh_authorized_keys_content = generate_cloudinit_ssh_keyfile(
+                    users, self.ceph.cfg["resource_group"]
+                )
+                managed_files.append(
+                    {
+                        "path": "/root/.ssh/authorized_keys_fc",
+                        "content": ssh_authorized_keys_content,
+                        "permissions": "0600",
+                    }
+                )
+        except IOError:
+            self.log.error("users-file-not-existing")
+
+        with self.volume.mounted() as target:
+            metadata = target / "meta-data"
+            metadata.touch()
+            with metadata.open("w") as f:
+                yaml.safe_dump({"instance-id": enc["name"]}, f)
+            userdata = target / "user-data"
+            userdata.touch()
+            with userdata.open("w") as f:
+                f.write("#cloud-config\n")
+                yaml.safe_dump(
+                    {
+                        "allow_public_ssh_keys": True,
+                        "ssh_pwauth": False,
+                        "disable_root": False,
+                        "package_update": True,
+                        "packages": ["qemu-guest-agent"],
+                        "hostname": enc["name"],
+                        # don't create ubuntu user, but only root
+                        "users": [{"name": "root"}],
+                        "write_files": managed_files,
+                        "runcmd": [
+                            "systemctl enable --now qemu-guest-agent",
+                            "systemctl restart ssh",
+                        ],
+                    },
+                    f,
+                )
+            networkconfig_path = target / "network-config"
+            networkconfig_path.touch()
+            networkconfig = {"version": 1, "config": []}
+            for ifacename, ifaceconfig in enc["parameters"][
+                "interfaces"
+            ].items():
+                cfg = {
+                    "type": "physical",
+                    "name": "eth" + ifacename,
+                    "mac_address": ifaceconfig["mac"],
+                    "accept-ra": False,
+                    "subnets": [],
+                }
+                for net, netconfig in ifaceconfig["networks"].items():
+                    if not netconfig:
+                        continue
+                    ip_network = ipaddress.ip_network(net)
+                    gateway = ifaceconfig["gateways"][net]
+                    if ip_network.version == 4:
+                        type_ = "static"
+                        nameservers = ["9.9.9.9", "8.8.8.8"]
+                    else:
+                        type_ = "static6"
+                        nameservers = ["2620:fe::fe", "2001:4860:4860::8888"]
+                    for address in netconfig:
+                        cfg["subnets"].append(
+                            {
+                                "type": type_,
+                                "address": f"{address}/{ip_network.prefixlen}",
+                                "gateway": gateway,
+                                "dns_nameservers": nameservers,
+                            }
+                        )
+                networkconfig["config"].append(cfg)
+            with networkconfig_path.open("w") as f:
+                yaml.safe_dump(networkconfig, f)
+
+
 class SwapSpec(VolumeSpecification):
     suffix = "swap"
 
@@ -407,6 +538,7 @@ def __enter__(self):
         RootSpec(self)
         SwapSpec(self)
         TmpSpec(self)
+        CloudInitSpec(self)
         for spec in self.specs.values():
             self.get_volume(spec)
 

src/fc/qemu/qemu.vm.cfg.in

@@ -40,6 +40,15 @@
   aio = "threads"
   cache = "none"
 
+[drive]
+  index = "3"
+  media = "disk"
+  if = "virtio"
+  format = "rbd"
+  file = "rbd:{ceph.volumes[cidata].fullname}:id={{ceph_id}}"
+  aio = "threads"
+  cache = "none"
+
 [device]
   driver = "virtio-rng-pci"
 

src/fc/qemu/sysconfig.py

@@ -85,6 +85,7 @@ def load_system_config(self):
         self.ceph["CEPH_LOCK_HOST"] = self.cp.get("ceph", "lock_host")
         self.ceph["CREATE_VM"] = self.cp.get("ceph", "create-vm")
         self.ceph["MKFS_XFS"] = self.cp.get("ceph", "mkfs-xfs")
+        self.ceph["MKFS_VFAT"] = self.cp.get("ceph", "mkfs-vfat")
 
 
 sysconfig = SysConfig()

src/fc/qemu/util.py

@@ -7,7 +7,7 @@
 import subprocess
 import sys
 import time
-from typing import Dict
+from typing import Dict, List
 
 from structlog import get_logger
 
@@ -168,3 +168,20 @@ def parse_export_format(data: str) -> Dict[str, str]:
         v = v.strip("'\"")
         result[k] = v
     return result
+
+
+def generate_cloudinit_ssh_keyfile(
+    users: List[Dict], resource_group: str
+) -> str:
+
+    authorized_ssh_keys = [
+        u["ssh_pubkey"]
+        for u in users
+        if set(u["permissions"][resource_group]) & set(["sudo-srv", "manager"])
+    ]
+    flattened_ssh_keys = sum(authorized_ssh_keys, [])
+    return (
+        "### managed by Flying Circus - do not edit! ###\n"
+        + "\n".join(flattened_ssh_keys)
+        + "\n"
+    )