Merge pull request #24 from mhjacks/main

Update common and tests for new common features

common/.ansible-lint

@@ -14,4 +14,7 @@ skip_list:
 exclude_paths:
   - ./ansible/playbooks/vault/vault.yaml
   - ./ansible/playbooks/iib-ci/iib-ci.yaml
+  - ./ansible/playbooks/k8s_secrets/k8s_secrets.yml
+  - ./ansible/playbooks/process_secrets/process_secrets.yml
+  - ./ansible/playbooks/process_secrets/display_secrets_info.yml
   - ./ansible/roles/vault_utils/tests/test.yml

common/.github/linters/.gitleaks.toml

@@ -2,5 +2,7 @@
 # As of v4, gitleaks only matches against filename, not path in the
 # files directive.  Leaving content for backwards compatibility.
 files = [
-  "tests/*.yaml",
+  "ansible/plugins/modules/*.py",
+  "ansible/tests/unit/test_*.py",
+  "ansible/tests/unit/v1/*.yaml",
 ]

common/.github/workflows/chart-branches.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Checkout Code
         uses: actions/checkout@v4
 
-      - uses: dorny/paths-filter@v2
+      - uses: dorny/paths-filter@v3
         id: filter
         with:
           filters: |

common/.gitignore

@@ -5,6 +5,7 @@ __pycache__/
 *.swo
 values-secret.yaml
 .*.expected.yaml
+.vscode
 pattern-vault.init
 pattern-vault.init.bak
 super-linter.log

common/Makefile

@@ -77,9 +77,37 @@ uninstall: ## runs helm uninstall
 	@oc delete csv -n openshift-operators $(CSV)
 
 .PHONY: load-secrets
-load-secrets: ## loads the secrets into the vault
+load-secrets: ## loads the secrets into the backend determined by values-global setting
+	common/scripts/process-secrets.sh $(NAME)
+
+.PHONY: legacy-load-secrets
+legacy-load-secrets: ## loads the secrets into vault (only)
 	common/scripts/vault-utils.sh push_secrets $(NAME)
 
+.PHONY: secrets-backend-vault
+secrets-backend-vault: ## Edits values files to use default Vault+ESO secrets config
+	common/scripts/set-secret-backend.sh vault
+	common/scripts/manage-secret-app.sh vault present
+	common/scripts/manage-secret-app.sh golang-external-secrets present
+	common/scripts/manage-secret-namespace.sh validated-patterns-secrets absent
+	@git diff --exit-code || echo "Secrets backend set to vault, please review changes, commit, and push to activate in the pattern"
+
+.PHONY: secrets-backend-kubernetes
+secrets-backend-kubernetes: ## Edits values file to use Kubernetes+ESO secrets config
+	common/scripts/set-secret-backend.sh kubernetes
+	common/scripts/manage-secret-namespace.sh validated-patterns-secrets present
+	common/scripts/manage-secret-app.sh vault absent
+	common/scripts/manage-secret-app.sh golang-external-secrets present
+	@git diff --exit-code || echo "Secrets backend set to kubernetes, please review changes, commit, and push to activate in the pattern"
+
+.PHONY: secrets-backend-none
+secrets-backend-none: ## Edits values files to remove secrets manager + ESO
+	common/scripts/set-secret-backend.sh none
+	common/scripts/manage-secret-app.sh vault absent
+	common/scripts/manage-secret-app.sh golang-external-secrets absent
+	common/scripts/manage-secret-namespace.sh validated-patterns-secrets absent
+	@git diff --exit-code || echo "Secrets backend set to none, please review changes, commit, and push to activate in the pattern"
+
 .PHONY: load-iib
 load-iib: ## CI target to install Index Image Bundles
 	@set -e; if [ x$(INDEX_IMAGES) != x ]; then \
@@ -99,14 +127,9 @@ load-iib: ## CI target to install Index Image Bundles
 .PHONY: validate-origin
 validate-origin: ## verify the git origin is available
 	@echo "Checking repository:"
-	@echo -n "  $(TARGET_REPO) - branch $(TARGET_BRANCH): "
-	@if [ ! -f /run/.containerenv ]; then\
-		git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
-				echo "OK" ||\
-				(echo "NOT FOUND"; exit 1);\
-	else\
-		echo "Running inside a container: Skipping git ssh checks";\
-	fi
+	@echo -n "  $(TARGET_REPO) - branch '$(TARGET_BRANCH)': "
+	@git ls-remote --exit-code --heads $(TARGET_REPO) $(TARGET_BRANCH) >/dev/null &&\
+		echo "OK" || (echo "NOT FOUND"; exit 1)
 
 .PHONY: validate-cluster
 validate-cluster: ## Do some cluster validations before installing
@@ -130,15 +153,19 @@ validate-schema: ## validates values files against schema in common/clustergroup
 
 .PHONY: validate-prereq
 validate-prereq: ## verify pre-requisites
-	@echo "Checking prerequisites:"
-	@for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done
-	@echo "  Check for '$(EXECUTABLES)': OK"
-	@echo -n "  Check for python-kubernetes: "
-	@if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi
-	@echo "OK"
-	@echo -n "  Check for kubernetes.core collection: "
-	@if ! ansible-galaxy collection list | grep kubernetes.core > /dev/null 2>&1; then echo "Not found"; exit 1; fi
-	@echo "OK"
+	@if [ ! -f /run/.containerenv ]; then\
+	  echo "Checking prerequisites:";\
+	  for t in $(EXECUTABLES); do if ! which $$t > /dev/null 2>&1; then echo "No $$t in PATH"; exit 1; fi; done;\
+	  echo "  Check for '$(EXECUTABLES)': OK";\
+	  echo -n "  Check for python-kubernetes: ";\
+	  if ! ansible -m ansible.builtin.command -a "{{ ansible_python_interpreter }} -c 'import kubernetes'" localhost > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
+	  echo "OK";\
+	  echo -n "  Check for kubernetes.core collection: ";\
+	  if ! ansible-galaxy collection list | grep kubernetes.core > /dev/null 2>&1; then echo "Not found"; exit 1; fi;\
+	  echo "OK";\
+	else\
+	  echo "Skipping prerequisites check as we're running inside a container";\
+	fi
 
 .PHONY: argo-healthcheck
 argo-healthcheck: ## Checks if all argo applications are synced

common/ansible/playbooks/k8s_secrets/k8s_secrets.yml

@@ -0,0 +1,9 @@
+---
+- name: Secrets parsing and direct loading
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  roles:
+   - find_vp_secrets
+   - cluster_pre_check
+   - k8s_secret_utils

common/ansible/playbooks/process_secrets/display_secrets_info.yml

@@ -0,0 +1,29 @@
+---
+- name: Parse and display secrets
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    secrets_backing_store: "vault"
+  tasks:
+    # Set the VALUES_SECRET environment variable to the file to parse
+    - name: Find and decrypt secrets if needed
+      ansible.builtin.include_role:
+        name: find_vp_secrets
+
+    # find_vp_secrets will return a plaintext data structure called values_secrets_data
+    # This will allow us to determine schema version and which backend to use
+    - name: Determine how to load secrets
+      ansible.builtin.set_fact:
+        secrets_yaml: '{{ values_secrets_data | from_yaml }}'
+
+    - name: Parse secrets data
+      no_log: '{{ override_no_log | default(true) }}'
+      parse_secrets_info:
+        values_secrets_plaintext: "{{ values_secrets_data }}"
+        secrets_backing_store: "{{ secrets_backing_store }}"
+      register: secrets_results
+
+    - name: Display secrets data
+      ansible.builtin.debug:
+        var: secrets_results

common/ansible/playbooks/process_secrets/process_secrets.yml

@@ -0,0 +1,50 @@
+---
+- name: Parse and load secrets
+  hosts: localhost
+  connection: local
+  gather_facts: false
+  vars:
+    secrets_role: 'vault_utils'
+    pattern_name: 'common'
+    pattern_dir: '.'
+    secrets_backing_store: 'vault'
+    tasks_from: 'push_parsed_secrets'
+  tasks:
+    - name: "Run secret-loading pre-requisites"
+      ansible.builtin.include_role:
+        name: '{{ item }}'
+      loop:
+        - cluster_pre_check
+        - find_vp_secrets
+
+    # find_vp_secrets will return a plaintext data structure called values_secrets_data
+    # This will allow us to determine schema version and which backend to use
+    - name: Determine how to load secrets
+      ansible.builtin.set_fact:
+        secrets_yaml: '{{ values_secrets_data | from_yaml }}'
+
+    - name: Parse secrets data
+      no_log: '{{ override_no_log | default(true) }}'
+      parse_secrets_info:
+        values_secrets_plaintext: "{{ values_secrets_data }}"
+        secrets_backing_store: "{{ secrets_backing_store }}"
+      register: secrets_results
+
+    # Use the k8s secrets loader when explicitly requested
+    - name: Determine role to use to load secrets
+      ansible.builtin.set_fact:
+        secrets_role: 'k8s_secret_utils'
+        tasks_from: 'inject_k8s_secrets'
+      when:
+        - secrets_backing_store == "kubernetes" or secrets_backing_store == "none"
+        - secrets_yaml['version'] | default('2.0') >= '2.0'
+
+    # secrets_role will have been changed from the default if needed
+    - name: Load secrets using designated role and tasks
+      ansible.builtin.include_role:
+        name: '{{ secrets_role }}'
+        tasks_from: '{{ tasks_from }}'
+      vars:
+        kubernetes_secret_objects: "{{ secrets_results['kubernetes_secret_objects'] }}"
+        vault_policies: "{{ secrets_results['vault_policies'] }}"
+        parsed_secrets: "{{ secrets_results['parsed_secrets'] }}"

common/ansible/playbooks/vault/vault.yaml

@@ -4,4 +4,6 @@
   connection: local
   gather_facts: false
   roles:
+   - find_vp_secrets
+   - cluster_pre_check
    - vault_utils

common/ansible/plugins/module_utils/load_secrets_common.py

@@ -102,3 +102,23 @@ def get_ini_value(inifile, inisection, inikey):
     config = configparser.ConfigParser()
     config.read(inifile)
     return config.get(inisection, inikey, fallback=None)
+
+
+def stringify_dict(input_dict):
+    """
+    Return a dict whose keys and values are all co-erced to strings, for creating labels and annotations in the
+    python Kubernetes module
+
+    Parameters:
+        input_dict(dict): A dictionary of keys and values
+
+    Returns:
+
+        obj: The same dict in the same order but with the keys coerced to str
+    """
+    output_dict = {}
+
+    for key, value in input_dict.items():
+        output_dict[str(key)] = str(value)
+
+    return output_dict