From fa484c8024163181944d1002fe965bda7448caa0 Mon Sep 17 00:00:00 2001
From: dbw7 <danial.bekhit@suse.com>
Date: Wed, 14 Aug 2024 09:34:43 -0400
Subject: [PATCH] Rework manifest install logic (#523)

* rework manifest install logic

* feedback and changes
---
 RELEASE_NOTES.md                              |  2 ++
 .../templates/k3s-multi-node-installer.sh.tpl | 26 ++++++++++++++++++-
 .../k3s-single-node-installer.sh.tpl          | 26 ++++++++++++++++++-
 .../rke2-multi-node-installer.sh.tpl          | 26 ++++++++++++++++++-
 .../rke2-single-node-installer.sh.tpl         | 26 ++++++++++++++++++-
 5 files changed, 102 insertions(+), 4 deletions(-)

diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md
index 5a99dce4..c8793c28 100644
--- a/RELEASE_NOTES.md
+++ b/RELEASE_NOTES.md
@@ -12,6 +12,8 @@
 
 ## Bug Fixes
 
+* [#491](https://github.com/suse-edge/edge-image-builder/issues/491) - Large Helm manifests fail to install
+
 ---
 
 # v1.1.0-rc1
diff --git a/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl b/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl
index 42322189..8459aad5 100644
--- a/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl
+++ b/pkg/combustion/templates/k3s-multi-node-installer.sh.tpl
@@ -38,10 +38,33 @@ CONFIGFILE={{ .configFilePath }}/{{ .initialiserConfigFile }}
 mkdir -p /opt/eib-k8s/manifests
 cp {{ .manifestsPath }}/* /opt/eib-k8s/manifests/
 
+cat <<- 'EOF' > /opt/eib-k8s/create_manifests.sh
+#!/bin/bash
+failed=false
+
+for file in /opt/eib-k8s/manifests/*; do
+    output=$(/opt/bin/kubectl create -f "$file" --kubeconfig=/etc/rancher/k3s/k3s.yaml 2>&1)
+
+    if [ $? != 0 ]; then
+        if [[ "$output" != *"AlreadyExists"* ]]; then
+            failed=true
+        fi
+    fi
+    echo "$output"
+done
+
+if [ $failed = "true" ]; then
+    exit 1
+fi
+EOF
+
+chmod +x /opt/eib-k8s/create_manifests.sh
+
 cat <<- EOF > /etc/systemd/system/kubernetes-resources-install.service
 [Unit]
 Description=Kubernetes Resources Install
 Requires=k3s.service
+PartOf=k3s.service
 After=k3s.service
 ConditionPathExists=/opt/bin/kubectl
 ConditionPathExists=/etc/rancher/k3s/k3s.yaml
@@ -53,7 +76,8 @@ WantedBy=multi-user.target
 Type=oneshot
 Restart=on-failure
 RestartSec=60
-ExecStart=/opt/bin/kubectl apply -f /opt/eib-k8s/manifests --kubeconfig=/etc/rancher/k3s/k3s.yaml
+ExecStartPre=/bin/sh -c 'until systemctl is-active --quiet k3s.service; do sleep 10; done'
+ExecStart=/opt/eib-k8s/create_manifests.sh
 # Disable the service and clean up
 ExecStartPost=/bin/sh -c "systemctl disable kubernetes-resources-install.service"
 ExecStartPost=rm -f /etc/systemd/system/kubernetes-resources-install.service
diff --git a/pkg/combustion/templates/k3s-single-node-installer.sh.tpl b/pkg/combustion/templates/k3s-single-node-installer.sh.tpl
index f3c25e22..8d0e3477 100644
--- a/pkg/combustion/templates/k3s-single-node-installer.sh.tpl
+++ b/pkg/combustion/templates/k3s-single-node-installer.sh.tpl
@@ -12,10 +12,33 @@ umount /var
 mkdir -p /opt/eib-k8s/manifests
 cp {{ .manifestsPath }}/* /opt/eib-k8s/manifests/
 
+cat <<- 'EOF' > /opt/eib-k8s/create_manifests.sh
+#!/bin/bash
+failed=false
+
+for file in /opt/eib-k8s/manifests/*; do
+    output=$(/opt/bin/kubectl create -f "$file" --kubeconfig=/etc/rancher/k3s/k3s.yaml 2>&1)
+
+    if [ $? != 0 ]; then
+        if [[ "$output" != *"AlreadyExists"* ]]; then
+            failed=true
+        fi
+    fi
+    echo "$output"
+done
+
+if [ $failed = "true" ]; then
+    exit 1
+fi
+EOF
+
+chmod +x /opt/eib-k8s/create_manifests.sh
+
 cat <<- EOF > /etc/systemd/system/kubernetes-resources-install.service
 [Unit]
 Description=Kubernetes Resources Install
 Requires=k3s.service
+PartOf=k3s.service
 After=k3s.service
 ConditionPathExists=/opt/bin/kubectl
 ConditionPathExists=/etc/rancher/k3s/k3s.yaml
@@ -27,7 +50,8 @@ WantedBy=multi-user.target
 Type=oneshot
 Restart=on-failure
 RestartSec=60
-ExecStart=/opt/bin/kubectl apply -f /opt/eib-k8s/manifests --kubeconfig=/etc/rancher/k3s/k3s.yaml
+ExecStartPre=/bin/sh -c 'until systemctl is-active --quiet k3s.service; do sleep 10; done'
+ExecStart=/opt/eib-k8s/create_manifests.sh
 # Disable the service and clean up
 ExecStartPost=/bin/sh -c "systemctl disable kubernetes-resources-install.service"
 ExecStartPost=rm -f /etc/systemd/system/kubernetes-resources-install.service
diff --git a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
index 94df592d..da73a9da 100644
--- a/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
+++ b/pkg/combustion/templates/rke2-multi-node-installer.sh.tpl
@@ -38,10 +38,33 @@ CONFIGFILE={{ .configFilePath }}/{{ .initialiserConfigFile }}
 mkdir -p /opt/eib-k8s/manifests
 cp {{ .manifestsPath }}/* /opt/eib-k8s/manifests/
 
+cat <<- 'EOF' > /opt/eib-k8s/create_manifests.sh
+#!/bin/bash
+failed=false
+
+for file in /opt/eib-k8s/manifests/*; do
+    output=$(/opt/eib-k8s/kubectl create -f "$file" --kubeconfig /etc/rancher/rke2/rke2.yaml 2>&1)
+
+    if [ $? != 0 ]; then
+        if [[ "$output" != *"AlreadyExists"* ]]; then
+            failed=true
+        fi
+    fi
+    echo "$output"
+done
+
+if [ $failed = "true" ]; then
+    exit 1
+fi
+EOF
+
+chmod +x /opt/eib-k8s/create_manifests.sh
+
 cat <<- EOF > /etc/systemd/system/kubernetes-resources-install.service
 [Unit]
 Description=Kubernetes Resources Install
 Requires=rke2-server.service
+PartOf=rke2-server.service
 After=rke2-server.service
 ConditionPathExists=/var/lib/rancher/rke2/bin/kubectl
 ConditionPathExists=/etc/rancher/rke2/rke2.yaml
@@ -54,8 +77,9 @@ Type=oneshot
 Restart=on-failure
 RestartSec=60
 # Copy kubectl in order to avoid SELinux permission issues
+ExecStartPre=/bin/sh -c 'until systemctl is-active --quiet rke2-server.service; do sleep 10; done'
 ExecStartPre=cp /var/lib/rancher/rke2/bin/kubectl /opt/eib-k8s/kubectl
-ExecStart=/opt/eib-k8s/kubectl apply -f /opt/eib-k8s/manifests --kubeconfig /etc/rancher/rke2/rke2.yaml
+ExecStart=/opt/eib-k8s/create_manifests.sh
 # Disable the service and clean up
 ExecStartPost=/bin/sh -c "systemctl disable kubernetes-resources-install.service"
 ExecStartPost=rm -f /etc/systemd/system/kubernetes-resources-install.service
diff --git a/pkg/combustion/templates/rke2-single-node-installer.sh.tpl b/pkg/combustion/templates/rke2-single-node-installer.sh.tpl
index 3928463b..2094cc87 100644
--- a/pkg/combustion/templates/rke2-single-node-installer.sh.tpl
+++ b/pkg/combustion/templates/rke2-single-node-installer.sh.tpl
@@ -12,10 +12,33 @@ umount /var
 mkdir -p /opt/eib-k8s/manifests
 cp {{ .manifestsPath }}/* /opt/eib-k8s/manifests/
 
+cat <<- 'EOF' > /opt/eib-k8s/create_manifests.sh
+#!/bin/bash
+failed=false
+
+for file in /opt/eib-k8s/manifests/*; do
+    output=$(/opt/eib-k8s/kubectl create -f "$file" --kubeconfig /etc/rancher/rke2/rke2.yaml 2>&1)
+
+    if [ $? != 0 ]; then
+        if [[ "$output" != *"AlreadyExists"* ]]; then
+            failed=true
+        fi
+    fi
+    echo "$output"
+done
+
+if [ $failed = "true" ]; then
+    exit 1
+fi
+EOF
+
+chmod +x /opt/eib-k8s/create_manifests.sh
+
 cat <<- EOF > /etc/systemd/system/kubernetes-resources-install.service
 [Unit]
 Description=Kubernetes Resources Install
 Requires=rke2-server.service
+PartOf=rke2-server.service
 After=rke2-server.service
 ConditionPathExists=/var/lib/rancher/rke2/bin/kubectl
 ConditionPathExists=/etc/rancher/rke2/rke2.yaml
@@ -28,8 +51,9 @@ Type=oneshot
 Restart=on-failure
 RestartSec=60
 # Copy kubectl in order to avoid SELinux permission issues
+ExecStartPre=/bin/sh -c 'until systemctl is-active --quiet rke2-server.service; do sleep 10; done'
 ExecStartPre=cp /var/lib/rancher/rke2/bin/kubectl /opt/eib-k8s/kubectl
-ExecStart=/opt/eib-k8s/kubectl apply -f /opt/eib-k8s/manifests --kubeconfig /etc/rancher/rke2/rke2.yaml
+ExecStart=/opt/eib-k8s/create_manifests.sh
 # Disable the service and clean up
 ExecStartPost=/bin/sh -c "systemctl disable kubernetes-resources-install.service"
 ExecStartPost=rm -f /etc/systemd/system/kubernetes-resources-install.service