Add etcd integration tests (#281)

* Initial work for etcd integration tests
* Update openssl commands and fix bootstrap config update
* add etcd marker for integration tests

tests/integration/requirements-test.txt

@@ -1,3 +1,4 @@
 coverage[toml]==7.2.5
 pytest==7.3.1
+PyYAML==6.0.1
 tenacity==8.2.3

tests/integration/templates/etcd/etcd-tls.conf

@@ -0,0 +1,23 @@
+    [req]
+    default_bits  = 4096
+    distinguished_name = req_distinguished_name
+    req_extensions = v3_req
+    prompt = no
+
+    [req_distinguished_name]
+    countryName = US
+    stateOrProvinceName = CA
+    localityName = San Francisco
+    organizationName = etcd
+    commonName = etcd-host
+
+    [v3_req]
+    keyUsage = digitalSignature, keyEncipherment, dataEncipherment
+    extendedKeyUsage = serverAuth, clientAuth
+    subjectAltName = @alt_names
+
+    [alt_names]
+    IP.1 = 127.0.0.1
+    IP.2 = $IP 
+    DNS.1 = localhost
+    DNS.2 = $NAME 

tests/integration/templates/etcd/etcd.service

@@ -0,0 +1,33 @@
+    [Unit]
+    Description=etcd
+    Documentation=https://github.com/etcd-io/etcd
+    Conflicts=etcd.service
+    Conflicts=etcd2.service
+
+    [Service]
+    Type=notify
+    Restart=always
+    RestartSec=5s
+    LimitNOFILE=40000
+    TimeoutStartSec=0
+
+    ExecStart=/tmp/test-etcd/etcd --name $NAME \
+    --data-dir /tmp/etcd/s1 \
+    --listen-client-urls $CLIENT_URL \
+    --advertise-client-urls $CLIENT_URL \
+    --listen-peer-urls $PEER_URL \
+    --initial-advertise-peer-urls $PEER_URL \
+    --initial-cluster "$CLUSTER" \
+    --initial-cluster-token tkn \
+    --initial-cluster-state $CLUSTER_STATE \
+    --client-cert-auth \
+    --trusted-ca-file /tmp/ca-cert.pem \
+    --cert-file /tmp/etcd-cert.pem \
+    --key-file /tmp/etcd-key.pem \
+    --peer-client-cert-auth \
+    --peer-trusted-ca-file /tmp/ca-cert.pem \
+    --peer-cert-file /tmp/etcd-cert.pem \
+    --peer-key-file /tmp/etcd-key.pem
+
+    [Install]
+    WantedBy=multi-user.target

tests/integration/tests/conftest.py

@@ -7,6 +7,7 @@
 
 import pytest
 from test_util import config, harness, util
+from test_util.etcd import EtcdCluster
 
 LOG = logging.getLogger(__name__)
 
@@ -48,7 +49,9 @@ def h() -> harness.Harness:
 def pytest_configure(config):
     config.addinivalue_line(
         "markers",
-        "node_count: Mark a test to specify how many instance nodes need to be created",
+        "node_count: Mark a test to specify how many instance nodes need to be created\n"
+        "disable_k8s_bootstrapping: By default, the first k8s node is bootstrapped. This marker disables that."
+        "etcd_count: Mark a test to specify how many etcd instance nodes need to be created (None by default)",
     )
 
 
@@ -61,13 +64,18 @@ def node_count(request) -> int:
     return int(node_count_arg)
 
 
+@pytest.fixture(scope="function")
+def disable_k8s_bootstrapping(request) -> int:
+    return bool(request.node.get_closest_marker("disable_k8s_bootstrapping"))
+
+
 @pytest.fixture(scope="function")
 def instances(
-    h: harness.Harness, node_count: int, tmp_path: Path
+    h: harness.Harness, node_count: int, tmp_path: Path, disable_k8s_bootstrapping: bool
 ) -> Generator[List[harness.Instance], None, None]:
     """Construct instances for a cluster.
 
-    Bootstrap and setup networking on the first instance.
+    Bootstrap and setup networking on the first instance, if `disable_k8s_bootstrapping` marker is not set.
     """
     if not config.SNAP:
         pytest.fail("Set TEST_SNAP to the path where the snap is")
@@ -86,9 +94,33 @@ def instances(
         instances.append(instance)
         util.setup_k8s_snap(instance, snap_path)
 
-    first_node, *_ = instances
-    first_node.exec(["k8s", "bootstrap"])
+    if not disable_k8s_bootstrapping:
+        first_node, *_ = instances
+        first_node.exec(["k8s", "bootstrap"])
 
     yield instances
 
     _harness_clean(h)
+
+
+@pytest.fixture(scope="function")
+def etcd_count(request) -> int:
+    etcd_count_marker = request.node.get_closest_marker("etcd_count")
+    if not etcd_count_marker:
+        return 0
+    etcd_count_arg, *_ = etcd_count_marker.args
+    return int(etcd_count_arg)
+
+
+@pytest.fixture(scope="function")
+def etcd_cluster(
+    h: harness.Harness, etcd_count: int
+) -> Generator[EtcdCluster, None, None]:
+    """Construct etcd instances for a cluster."""
+    LOG.info(f"Creating {etcd_count} etcd instances")
+
+    cluster = EtcdCluster(h, initial_node_count=etcd_count)
+
+    yield cluster
+
+    _harness_clean(h)

tests/integration/tests/test_etcd.py

@@ -0,0 +1,44 @@
+#
+# Copyright 2024 Canonical, Ltd.
+#
+import logging
+from typing import List
+
+import pytest
+import yaml
+from test_util import harness, util
+from test_util.etcd import EtcdCluster
+
+LOG = logging.getLogger(__name__)
+
+
+@pytest.mark.node_count(1)
+@pytest.mark.etcd_count(1)
+@pytest.mark.disable_k8s_bootstrapping()
+def test_etcd(instances: List[harness.Instance], etcd_cluster: EtcdCluster):
+    k8s_instance = instances[0]
+
+    bootstrap_conf = yaml.safe_dump(
+        {
+            "cluster-config": {"network": {"enabled": True}, "dns": {"enabled": True}},
+            "datastore-type": "external",
+            "datastore-servers": etcd_cluster.client_urls,
+            "datastore-ca-crt": etcd_cluster.ca_cert,
+            "datastore-client-crt": etcd_cluster.cert,
+            "datastore-client-key": etcd_cluster.key,
+        }
+    )
+
+    k8s_instance.exec(
+        ["dd", "of=/root/config.yaml"],
+        input=str.encode(bootstrap_conf),
+    )
+
+    k8s_instance.exec(["k8s", "bootstrap", "--file", "/root/config.yaml"])
+    util.wait_for_dns(k8s_instance)
+    util.wait_for_network(k8s_instance)
+
+    p = k8s_instance.exec(
+        ["systemctl", "is-active", "--quiet", "snap.k8s.k8s-dqlite"], check=False
+    )
+    assert p.returncode != 0, "k8s-dqlite service is still active"

tests/integration/tests/test_loadbalancer.py

@@ -13,28 +13,6 @@
 LOG = logging.getLogger(__name__)
 
 
-def get_default_cidr(instance: harness.Instance, instance_default_ip: str):
-    # ----
-    # 1:  lo    inet 127.0.0.1/8 scope host lo .....
-    # 28: eth0  inet 10.42.254.197/24 metric 100 brd 10.42.254.255 scope global dynamic eth0 ....
-    # ----
-    # Fetching the cidr for the default interface by matching with instance ip from the output
-    p = instance.exec(["ip", "-o", "-f", "inet", "addr", "show"], capture_output=True)
-    out = p.stdout.decode().split(" ")
-    return [i for i in out if instance_default_ip in i][0]
-
-
-def get_default_ip(instance: harness.Instance):
-    # ---
-    # default via 10.42.254.1 dev eth0 proto dhcp src 10.42.254.197 metric 100
-    # ---
-    # Fetching the default IP address from the output, e.g. 10.42.254.197
-    p = instance.exec(
-        ["ip", "-o", "-4", "route", "show", "to", "default"], capture_output=True
-    )
-    return p.stdout.decode().split(" ")[8]
-
-
 def find_suitable_cidr(parent_cidr: str, excluded_ips: List[str]):
     net = ipaddress.IPv4Network(parent_cidr, False)
 
@@ -64,10 +42,10 @@ def test_loadbalancer(instances: List[harness.Instance]):
 
     tester_instance = instances[1]
 
-    instance_default_ip = get_default_ip(instance)
-    tester_instance_default_ip = get_default_ip(tester_instance)
+    instance_default_ip = util.get_default_ip(instance)
+    tester_instance_default_ip = util.get_default_ip(tester_instance)
 
-    instance_default_cidr = get_default_cidr(instance, instance_default_ip)
+    instance_default_cidr = util.get_default_cidr(instance, instance_default_ip)
 
     lb_cidr = find_suitable_cidr(
         parent_cidr=instance_default_cidr,

tests/integration/tests/test_util/config.py

@@ -8,6 +8,15 @@
 
 MANIFESTS_DIR = DIR / ".." / ".." / "templates"
 
+# ETCD_DIR contains all templates required to setup an etcd database.
+ETCD_DIR = MANIFESTS_DIR / "etcd"
+
+# ETCD_URL is the url from which the etcd binaries should be downloaded.
+ETCD_URL = os.getenv("ETCD_URL") or "https://github.com/etcd-io/etcd/releases/download"
+
+# ETCD_VERSION is the version of etcd to use.
+ETCD_VERSION = os.getenv("ETCD_VERSION") or "v3.3.8"
+
 # SNAP is the absolute path to the snap against which we run the integration tests.
 SNAP = os.getenv("TEST_SNAP")