merge updates from upstream

marmila · Feb 26, 2024 · 847391c · 847391c
2 parents 4b3af30 + 831dd46
commit 847391c
Show file tree

Hide file tree

Showing 75 changed files with 573 additions and 358 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -17,7 +17,7 @@ jobs:
         uses: actions/checkout@v4
 
       - name: Set up Python 3.
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.x'
 

diff --git a/Makefile b/Makefile
@@ -84,10 +84,22 @@ shutdown-k3s-master:
 shutdown-gateway:
 	${RUNNER} ansible -b -m shell -a "shutdown -h 1 min" gateway
 
+.PHONY: shutdown-picluster
+shutdown-picluster:
+	${RUNNER} ansible -b -m shell -a "shutdown -h 1 min" picluster
+
 .PHONY: get-argocd-passwd
 get-argocd-passwd:
 	kubectl get secret argocd-initial-admin-secret -o jsonpath='{.data.password}' -n argocd | base64 -d;echo
 
 .PHONY: get-elastic-passwd
 get-elastic-passwd:
-	kubectl get secret efk-es-elastic-user -o jsonpath='{.data.elastic}' -n logging | base64 -d;echo
+	kubectl get secret efk-es-elastic-user -o jsonpath='{.data.elastic}' -n logging | base64 -d;echo
+
+.PHONY: kubernetes-vault-config
+kubernetes-vault-config:
+	${RUNNER} ansible-playbook kubernetes_vault_config.yml
+
+.PHONY: get-pi-status
+get-pi-status:
+	${RUNNER} ansible -b -m shell -a "pi_throttling" raspberrypi
diff --git a/ansible-runner/build/requirements.yml b/ansible-runner/build/requirements.yml
@@ -26,14 +26,12 @@ roles:
     version: v1.0.4
 collections:
   - name: community.general
-    version: 8.0.1
+    version: 8.3.0
   - name: kubernetes.core
-    version: 2.4.0
+    version: 3.0.0
   - name: community.hashi_vault
-    version: 5.0.1
-  - name: community.sops
-    version: 1.6.0
+    version: 6.1.0
   - name: ansible.posix
     version: 1.5.4
   - name: community.crypto
-    version: 2.16.1
+    version: 2.17.1
diff --git a/ansible/external_services.yml b/ansible/external_services.yml
@@ -136,7 +136,7 @@
 ## Install Hashicorp Vault Server
 
 - name: Install Vault Server
-  hosts: gateway
+  hosts: vault
   gather_facts: true
   tags: [vault]
   become: true
@@ -223,7 +223,7 @@
 
 ## Load all credentials into Hashicorp Vault Server
 - name: Load Vault Credentials
-  hosts: gateway
+  hosts: vault
   gather_facts: true
   tags: [vault, credentials]
   become: false

diff --git a/ansible/host_vars/gateway.yml b/ansible/host_vars/gateway.yml
@@ -29,6 +29,7 @@ dnsmasq_additional_dns_hosts:
     desc: "S3 Server"
     hostname: s3
     ip: 89.168.19.79
+    ip: 10.0.0.100
   elasticsearch:
     desc: "Elasticsearch server"
     hostname: elasticsearch
@@ -52,7 +53,7 @@ dnsmasq_additional_dns_hosts:
   vault_server:
     desc: "Vault server"
     hostname: vault
-    ip: 10.0.0.1
+    ip: 10.0.0.11
 dnsmasq_enable_tftp: true
 dnsmasq_tftp_root: /srv/tftp
 dnsmasq_additional_conf: |-
@@ -81,10 +82,8 @@ ntp_allow_hosts: [10.0.0.0/24]
 #########################
 
 # tcp 9100 Prometheus (fluent-bit)
-# tcp 8200, 8201 Vault server
 # udp 69, TFTP server
-# TCP 6443 load balancer K3S API
-in_tcp_port: '{ ssh, https, http, iscsi-target, 9100, 8200, 8201, 6443 }'
+in_tcp_port: '{ ssh, https, http, iscsi-target, 9100 }'
 in_udp_port: '{ snmp, domain, ntp, bootps, 69 }'
 # tcp 9091 minio server
 forward_tcp_port: '{ http, https, ssh, 9091 }'
@@ -144,8 +143,10 @@ nft_forward_host_rules:
     - iifname $wan_interface oifname $lan_interface ip daddr $lan_network tcp dport ssh ct state new accept
   230 http from wan:
     - iifname $wan_interface oifname $lan_interface ip daddr $lan_network tcp dport {http, https} ct state new accept
+  240 haproxy from wan:
+    - iifname $wan_interface oifname $lan_interface ip daddr 10.0.0.11 tcp dport 6443 ct state new accept
   250 port-forwarding from wan:
-    - iifname $wan_interface oifname $lan_interface ip daddr 10.0.0.11 tcp dport 8080 ct state new accept
+    - iifname $wan_interface oifname $lan_interface ip daddr 10.0.0.12 tcp dport 8080 ct state new accept
 # NAT Post-routing
 nft_nat_host_postrouting_rules:
   005 masquerade lan to wan:

diff --git a/ansible/inventory.yml b/ansible/inventory.yml
@@ -60,4 +60,13 @@ all:
           hosts:
             node[3:4]:
             node-esp-1:
+    bootstrap:
+      hosts:
+        node2:
+    vault:
+      hosts:
+        node1:
+    haproxy:
+      hosts:
+        node1:
 
diff --git a/ansible/k3s_bootstrap.yml b/ansible/k3s_bootstrap.yml
@@ -1,7 +1,7 @@
 ---
 
 - name: Bootstrap Cluster
-  hosts: node1
+  hosts: bootstrap
   gather_facts: false
   become: false
 

diff --git a/ansible/k3s_install.yml b/ansible/k3s_install.yml
@@ -1,7 +1,7 @@
 ---
 
 - name: Install load balancer
-  hosts: gateway
+  hosts: haproxy
   gather_facts: true
   tags: [install]
   become: true

diff --git a/ansible/kubernetes_vault_config.yml b/ansible/kubernetes_vault_config.yml
@@ -0,0 +1,60 @@
+---
+
+- name: Configure Vault
+  hosts: vault
+  gather_facts: false
+
+  collections:
+    - kubernetes.core
+
+  environment:
+    # The location of the kubeconfig file on the master.
+    K8S_AUTH_KUBECONFIG: ~/.kube/config
+
+  pre_tasks:
+    # Install Python PIP and jq utility packages
+    - name: Ensure required packages are installed.
+      package:
+        name:
+          - python3-pip
+          - python3-setuptools
+          - jq
+        state: present
+      become: true
+    # Install kubernetes python packages (Ansible kubernetes collection dependency)
+    - name: Ensure kubernetes Python library is installed.
+      pip:
+        name: kubernetes
+        state: present
+      become: true
+    # Install Hashicorp python packages (Ansible hashi module dependency)
+    - name: Ensure hashicorp vault python library is installed.
+      pip:
+        name: hvac
+        state: present
+      become: true
+
+    # Install Helm diff plugin to have a better idempotence check
+    - name: Intall Helm Plugin
+      kubernetes.core.helm_plugin:
+        plugin_path: "https://github.com/databus23/helm-diff"
+        state: present
+
+    - name: Include vault variables
+      include_vars: "vars/vault.yml"
+
+    - name: Include picluster variables
+      include_vars: "vars/picluster.yml"
+
+  tasks:
+    - name: Create namespaces.
+      kubernetes.core.k8s:
+        name: "{{ item }}"
+        api_version: v1
+        kind: Namespace
+        state: present
+      with_items:
+        - "vault"
+
+    - name: Configure Vault integration
+      include_tasks: tasks/configure_vault_integration.yml
diff --git a/ansible/requirements.yml b/ansible/requirements.yml
@@ -26,14 +26,12 @@ roles:
     version: v1.0.4
 collections:
   - name: community.general
-    version: 8.0.1
+    version: 8.3.0
   - name: kubernetes.core
-    version: 2.4.0
+    version: 3.0.0
   - name: community.hashi_vault
-    version: 5.0.1
-  - name: community.sops
-    version: 1.6.0
+    version: 6.1.0
   - name: ansible.posix
     version: 1.5.4
   - name: community.crypto
-    version: 2.16.1
+    version: 2.17.1
diff --git a/ansible/reset_external_services.yml b/ansible/reset_external_services.yml
@@ -28,7 +28,7 @@
         daemon_reload: true
 
 - name: Clean Vault Installation
-  hosts: gateway
+  hosts: vault
   become: true
   gather_facts: false
   tags: [vault]

diff --git a/ansible/tasks/vault_kubernetes_auth_method_config.yml b/ansible/tasks/vault_kubernetes_auth_method_config.yml
@@ -18,7 +18,7 @@
   become: false
   register: vault_login
   changed_when: false
-  delegate_to: gateway
+  delegate_to: node1
 
 - name: Get vault token
   set_fact:

diff --git a/ansible/vars/picluster.yml b/ansible/vars/picluster.yml
@@ -6,10 +6,10 @@
 #######
 
 # k3s version
-k3s_version: v1.28.2+k3s1
+k3s_version: v1.29.1+k3s2
 
 # k3s master node VIP (loadbalancer)
-k3s_api_vip: 10.0.0.1
+k3s_api_vip: 10.0.0.11
 
 # k3s shared token
 k3s_token: "{{ vault.cluster.k3s.token }}"

diff --git a/argocd/bootstrap/argocd/Chart.yaml b/argocd/bootstrap/argocd/Chart.yaml
@@ -3,5 +3,5 @@ name: argocd
 version: 0.0.0
 dependencies:
   - name: argo-cd
-    version: 5.53.3
+    version: 6.3.1
     repository: https://argoproj.github.io/argo-helm
diff --git a/argocd/bootstrap/argocd/values.yaml b/argocd/bootstrap/argocd/values.yaml
@@ -25,26 +25,32 @@ argo-cd:
         return hs
       # Enabling Helm chart rendering with Kustomize
       kustomize.buildOptions: --enable-helm
+      ## Ignore resources
+      # https://argo-cd.readthedocs.io/en/stable/operator-manual/declarative-setup/#resource-exclusioninclusion
+      # Ignore VolumeSnapshot and VolumeSnapshotContent: Created by backup processes.
+      resource.exclusions: |
+        - apiGroups:
+          - snapshot.storage.k8s.io
+          kinds:
+          - VolumeSnapshot
+          - VolumeSnapshotContent
+          clusters:
+          - "*"
 
   server:
-    # Ingress Resource. LoSnghorn dashboard.
+    # Ingress Resource.
     ingress:
       ## Enable creation of ingress resource
       enabled: true
       ## Add ingressClassName to the Ingress
       ingressClassName: nginx
       # ingress host
-      hosts:
-        - argocd.picluster.marmilan.com
-      ## TLS Secret Name
-      tls:
-        - secretName: argocd-tls
-          hosts:
-            - argocd.picluster.marmilan.com
+      hostname: argocd.picluster.marmilan.com
       ## Default ingress path
-      paths:
-        - /
-
+      path: /
+      pathType: Prefix
+      # Enable tls. argocd-server-tls secret is created automatically for hostname
+      tls: true
       ## Ingress annotations
       annotations:
         # Linkerd configuration. Configure Service as Upstream

diff --git a/argocd/bootstrap/crds/cert-manager/kustomization.yaml b/argocd/bootstrap/crds/cert-manager/kustomization.yaml
@@ -4,4 +4,4 @@ resources:
 # Cert-manager helm installation https://cert-manager.io/docs/installation/helm/
 # CDRs can be installed manually.
 # cert-manager helm chart to be installed with value installCDRs=false
-- https://github.com/cert-manager/cert-manager/releases/download/v1.13.3/cert-manager.crds.yaml
+- https://github.com/cert-manager/cert-manager/releases/download/v1.14.2/cert-manager.crds.yaml
diff --git a/argocd/bootstrap/crds/external-secrets/kustomization.yaml b/argocd/bootstrap/crds/external-secrets/kustomization.yaml
@@ -2,6 +2,6 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
 # external-secrets https://github.com/external-secrets/external-secrets/tree/main/deploy/charts/external-secrets
-# helm chart version 0.9.11
+# helm chart version 0.9.12
 # external-secrets helm chart to be installed with value installCDRs=false
-- https://raw.githubusercontent.com/external-secrets/external-secrets/v0.9.11/deploy/crds/bundle.yaml
+- https://raw.githubusercontent.com/external-secrets/external-secrets/v0.9.12/deploy/crds/bundle.yaml
diff --git a/argocd/bootstrap/crds/kube-prometheus-stack/kustomization.yaml b/argocd/bootstrap/crds/kube-prometheus-stack/kustomization.yaml
@@ -6,4 +6,4 @@ resources:
 helmCharts:
 - name: prometheus-operator-crds
   repo: https://prometheus-community.github.io/helm-charts
-  version: "9.0.0"
+  version: "9.0.1"
diff --git a/argocd/bootstrap/crds/traefik/kustomization.yaml b/argocd/bootstrap/crds/traefik/kustomization.yaml
@@ -1,9 +1,9 @@
 apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 resources:
-# Traefik helm chart v23.1.0 CDRs.
+# Traefik helm chart CDRs.
 # Traefik provides CDRs upgrade process (https://github.com/traefik/traefik-helm-chart#upgrading-crds)
 # Issue #581(https://github.com/traefik/traefik-helm-chart/issues/581)
-# Within CDRs direcory in github repository there is a kustomization.yaml.
+# Within CDRs directory in github repository there is a kustomization.yaml.
 # Traefik helm chart to be installed with helm param --skip-cdrs
-- https://github.com/traefik/traefik-helm-chart/traefik/crds/?ref=v23.1.0
+- https://github.com/traefik/traefik-helm-chart/traefik/crds/?ref=v26.0.0
diff --git a/argocd/bootstrap/root/templates/app-set.yaml b/argocd/bootstrap/root/templates/app-set.yaml
@@ -19,6 +19,10 @@ spec:
 {{- if $app.helm }}
     helm:
 {{ toYaml $app.helm | indent 6  }}
+{{- end }}
+{{- if $app.ignoreDifferences }}
+  ignoreDifferences:
+{{ toYaml $app.ignoreDifferences | indent 4  }}
 {{- end }}
   syncPolicy:
     automated:

diff --git a/argocd/bootstrap/root/values.yaml b/argocd/bootstrap/root/values.yaml
@@ -8,6 +8,11 @@ apps:
   - name: crds
     namespace: default
     path: argocd/bootstrap/crds
+    ignoreDifferences:
+      - group: "*"
+        kind: "*"
+        managedFieldsManagers:
+          - external-secrets
     syncWave: 0
     # External Secrets Operator
   - name: external-secrets
@@ -35,13 +40,13 @@ apps:
     path: argocd/system/linkerd
     syncWave: 5
     # Traefik Ingress Controller
-    # - name: traefik
-    #   namespace: traefik
-    #   path: argocd/system/traefik
-    #   syncWave: 6
-    #   helm:
-    #     # skip installation traefik CDRs
-    #     skipCrds: true
+  - name: traefik
+    namespace: traefik
+    path: argocd/system/traefik
+    syncWave: 6
+    helm:
+      # skip installation traefik CDRs
+      skipCrds: true
   - name: nginx
     namespace: nginx
     path: argocd/system/nginx

diff --git a/argocd/system/cert-manager/Chart.yaml b/argocd/system/cert-manager/Chart.yaml
@@ -3,7 +3,7 @@ name: certmanager
 version: 0.0.0
 dependencies:
   - name: cert-manager
-    version: v1.13.3
+    version: v1.14.2
     repository: https://charts.jetstack.io
   - name: trust-manager
     version: v0.8.0