From 3ff4c6f0bca5cd523e180061428c196cfa95ecd2 Mon Sep 17 00:00:00 2001 From: Leona Maroni Date: Wed, 22 Jan 2025 11:13:35 +0100 Subject: [PATCH] Add cloud-init compatibility for Ubuntu VMs * Add cloud-init disk with config files * update root ssh keys via qemu agent in ensure for ubuntu VMs PL-133325 --- CHANGES.txt | 4 ++ src/fc/qemu/agent.py | 31 +++++++++ src/fc/qemu/default.conf | 1 + src/fc/qemu/hazmat/ceph.py | 132 +++++++++++++++++++++++++++++++++++++ src/fc/qemu/qemu.vm.cfg.in | 9 +++ src/fc/qemu/sysconfig.py | 1 + src/fc/qemu/util.py | 19 +++++- 7 files changed, 196 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 1c8ddf4..94249e3 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,10 @@ Release notes - rbd: fix cluster load issue due to overuse of rbd.list (PL-133194) +- Add support for cloud-init ubuntu VMs (PL-133325, PL-133372) + Add cloud-init `cidata` volume unconditionally for all VMs. + Sync SSH root keys for ubuntu VMs via qmp in ensure. + 1.6 (2024-10-23) ---------------- diff --git a/src/fc/qemu/agent.py b/src/fc/qemu/agent.py index 715863c..011f42e 100644 --- a/src/fc/qemu/agent.py +++ b/src/fc/qemu/agent.py @@ -330,6 +330,14 @@ def config_file_staging(self): def lock_file(self): return self.prefix / "run" / f"qemu.{self.name}.lock" + @property + def users_file(self): + return ( + self.prefix + / "etc/qemu/users" + / f"{self.cfg['resource_group']}.json" + ) + def _load_enc(self): try: with self.config_file.open() as f: @@ -346,6 +354,10 @@ def _load_enc(self): "Could not load {}".format(self.config_file) ) + def _load_users(self): + with self.users_file.open() as f: + return json.load(f) + @classmethod def handle_consul_event(cls, input: typing.TextIO = sys.stdin): # Python in our environments defaults to UTF-8 and we generally @@ -899,6 +911,7 @@ def _update_from_enc(self): self.cfg["root_size"] = self.cfg["disk"] * (1024**3) self.cfg["swap_size"] = swap_size(self.cfg["memory"]) self.cfg["tmp_size"] = tmp_size(self.cfg["disk"]) + self.cfg["cidata_size"] = 10 * MiB self.cfg["ceph_id"] = self.ceph_id self.cfg["cpu_model"] = self.cfg.get("cpu_model", "qemu64") self.cfg["binary_generation"] = self.binary_generation @@ -1091,6 +1104,7 @@ def ensure_online_local(self): self.ensure_thawed() self.mark_qemu_binary_generation() self.mark_qemu_guest_properties() + self.update_root_ssh_keys_cloudinit() def cleanup(self): """Removes various run and tmp files.""" @@ -1135,6 +1149,23 @@ def mark_qemu_binary_generation(self): except Exception as e: self.log.error("mark-qemu-binary-generation", reason=str(e)) + def update_root_ssh_keys_cloudinit(self): + # TODO: use environment_class_type != 'cloudinit' + if self.cfg["environment_class"].lower() != "ubuntu": + return + self.log.info("update-root-ssh-keys-cloudinit") + try: + users = self._load_users() + to_write_text = util.generate_cloudinit_ssh_keyfile( + users, self.cfg["resource_group"] + ) + self.qemu.write_file( + "/root/.ssh/authorized_keys_fc", + to_write_text.encode("utf-8"), + ) + except Exception as e: + self.log.error("update-root-ssh-keys-cloudinit", reason=str(e)) + def ensure_online_disk_size(self): """Trigger block resize action for the root disk.""" target_size = self.cfg["root_size"] diff --git a/src/fc/qemu/default.conf b/src/fc/qemu/default.conf index 99bdc89..259db53 100644 --- a/src/fc/qemu/default.conf +++ b/src/fc/qemu/default.conf @@ -28,3 +28,4 @@ lock_host = localhost create-vm = create-vm -I {rbd_pool} {name} mkfs-xfs = -q -f -K -m crc=1,finobt=1 -d su=4m,sw=1 mkfs-ext4 = -q -m 1 -E nodiscard +mkfs-vfat = diff --git a/src/fc/qemu/hazmat/ceph.py b/src/fc/qemu/hazmat/ceph.py index 44a46b5..dd4a96e 100644 --- a/src/fc/qemu/hazmat/ceph.py +++ b/src/fc/qemu/hazmat/ceph.py @@ -4,12 +4,17 @@ """ import hashlib +import ipaddress import json import os +from pathlib import Path from typing import Dict, Optional import rados import rbd +import yaml + +from fc.qemu.util import generate_cloudinit_ssh_keyfile from ..sysconfig import sysconfig from ..timeout import TimeoutError @@ -333,6 +338,132 @@ def seed(self, enc, generation): f.write(str(generation) + "\n") +class CloudInitSpec(VolumeSpecification): + suffix = "cidata" + + def pre_start(self): + for pool in self.exists_in_pools(): + if pool != self.desired_pool: + self.log.info( + "delete-outdated-cloud-init", pool=pool, image=self.name + ) + self.ceph.remove_volume(self.name, pool) + + def start(self): + self.log.info("start-cloud-init") + with self.volume.mapped(): + self.mkfs() + self.seed(self.ceph.enc) + + def mkfs(self): + self.log.debug("create-fs") + device = self.volume.device + assert device, f"volume must be mapped first: {device}" + self.cmd(f'sgdisk -o "{device}"') + self.cmd(f'sgdisk -n 1:: -c "1:{self.suffix}" ' f'-t 1:8300 "{device}"') + self.cmd(f"partprobe {device}") + self.volume.wait_for_part1dev() + options = getattr(self.ceph, "MKFS_VFAT") + self.cmd( + f'mkfs.vfat {options} -n "{self.suffix}" {self.volume.part1dev}' + ) + + def seed(self, enc): + self.log.info("seed") + managed_files = [ + { + "path": "/etc/ssh/sshd_config.d/10-cloud-init-fc.conf", + "content": """\ + AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys_fc + """, + "permissions": "0644", + } + ] + + # Improve factoring + try: + with Path( + f"/etc/qemu/users/{self.ceph.cfg['resource_group']}.json" + ).open() as f: + users = json.load(f) + ssh_authorized_keys_content = generate_cloudinit_ssh_keyfile( + users, self.ceph.cfg["resource_group"] + ) + managed_files.append( + { + "path": "/root/.ssh/authorized_keys_fc", + "content": ssh_authorized_keys_content, + "permissions": "0600", + } + ) + except IOError: + self.log.error("users-file-not-existing") + + with self.volume.mounted() as target: + metadata = target / "meta-data" + metadata.touch() + with metadata.open("w") as f: + yaml.safe_dump({"instance-id": enc["name"]}, f) + userdata = target / "user-data" + userdata.touch() + with userdata.open("w") as f: + f.write("#cloud-config\n") + yaml.safe_dump( + { + "allow_public_ssh_keys": True, + "ssh_pwauth": False, + "disable_root": False, + "package_update": True, + "packages": ["qemu-guest-agent"], + "hostname": enc["name"], + # don't create ubuntu user, but only root + "users": [{"name": "root"}], + "write_files": managed_files, + "runcmd": [ + "systemctl enable --now qemu-guest-agent", + "systemctl restart ssh", + ], + }, + f, + ) + networkconfig_path = target / "network-config" + networkconfig_path.touch() + networkconfig = {"version": 1, "config": []} + for ifacename, ifaceconfig in enc["parameters"][ + "interfaces" + ].items(): + cfg = { + "type": "physical", + "name": "eth" + ifacename, + "mac_address": ifaceconfig["mac"], + "accept-ra": False, + "subnets": [], + } + for net, netconfig in ifaceconfig["networks"].items(): + if not netconfig: + continue + ip_network = ipaddress.ip_network(net) + gateway = ifaceconfig["gateways"][net] + if ip_network.version == 4: + type_ = "static" + nameservers = ["9.9.9.9", "8.8.8.8"] + else: + type_ = "static6" + nameservers = ["2620:fe::fe", "2001:4860:4860::8888"] + for address in netconfig: + cfg["subnets"].append( + { + "type": type_, + "address": f"{address}/{ip_network.prefixlen}", + "gateway": gateway, + "dns_nameservers": nameservers, + } + ) + networkconfig["config"].append(cfg) + with networkconfig_path.open("w") as f: + yaml.safe_dump(networkconfig, f) + + class SwapSpec(VolumeSpecification): suffix = "swap" @@ -407,6 +538,7 @@ def __enter__(self): RootSpec(self) SwapSpec(self) TmpSpec(self) + CloudInitSpec(self) for spec in self.specs.values(): self.get_volume(spec) diff --git a/src/fc/qemu/qemu.vm.cfg.in b/src/fc/qemu/qemu.vm.cfg.in index 6bd5e5e..0b74b2e 100644 --- a/src/fc/qemu/qemu.vm.cfg.in +++ b/src/fc/qemu/qemu.vm.cfg.in @@ -40,6 +40,15 @@ aio = "threads" cache = "none" +[drive] + index = "3" + media = "disk" + if = "virtio" + format = "rbd" + file = "rbd:{ceph.volumes[cidata].fullname}:id={{ceph_id}}" + aio = "threads" + cache = "none" + [device] driver = "virtio-rng-pci" diff --git a/src/fc/qemu/sysconfig.py b/src/fc/qemu/sysconfig.py index 6fb3992..2f7d5f5 100644 --- a/src/fc/qemu/sysconfig.py +++ b/src/fc/qemu/sysconfig.py @@ -85,6 +85,7 @@ def load_system_config(self): self.ceph["CEPH_LOCK_HOST"] = self.cp.get("ceph", "lock_host") self.ceph["CREATE_VM"] = self.cp.get("ceph", "create-vm") self.ceph["MKFS_XFS"] = self.cp.get("ceph", "mkfs-xfs") + self.ceph["MKFS_VFAT"] = self.cp.get("ceph", "mkfs-vfat") sysconfig = SysConfig() diff --git a/src/fc/qemu/util.py b/src/fc/qemu/util.py index 6ccaf1a..46fc403 100644 --- a/src/fc/qemu/util.py +++ b/src/fc/qemu/util.py @@ -7,7 +7,7 @@ import subprocess import sys import time -from typing import Dict +from typing import Dict, List from structlog import get_logger @@ -168,3 +168,20 @@ def parse_export_format(data: str) -> Dict[str, str]: v = v.strip("'\"") result[k] = v return result + + +def generate_cloudinit_ssh_keyfile( + users: List[Dict], resource_group: str +) -> str: + + authorized_ssh_keys = [ + u["ssh_pubkey"] + for u in users + if set(u["permissions"][resource_group]) & set(["sudo-srv", "manager"]) + ] + flattened_ssh_keys = sum(authorized_ssh_keys, []) + return ( + "### managed by Flying Circus - do not edit! ###\n" + + "\n".join(flattened_ssh_keys) + + "\n" + )