ztp: CNF-14638: Backport CLO6 changes to release-4.16

go.mod

@@ -1,6 +1,6 @@
 module github.com/openshift-kni/cnf-features-deploy
 
-go 1.20
+go 1.21
 
 require (
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f

openshift-ci/Dockerfile.tools

@@ -3,7 +3,7 @@ FROM quay.io/fedora/fedora:37-x86_64
 ENV GOPATH /go
 ENV GOBIN /go/bin
 ENV GOCACHE /go/.cache
-ENV GOVERSION=1.20
+ENV GOVERSION=1.21
 ENV PATH=$PATH:/root/.gimme/versions/go"$GOVERSION".linux.amd64/bin:$GOBIN
 ARG GO_PACKAGE_PATH=github.com/openshift-kni/cnf-features-deploy
 

ztp/gitops-subscriptions/Makefile

@@ -12,6 +12,7 @@ KUSTOMIZE_VERSION=5.2.1
 KUSTOMIZE := $(if $(shell command -v kustomize 2>/dev/null), $(shell command -v kustomize), $(KUSTOMIZE_BIN))
 POLICYGEN_DIR := ../policygenerator
 SOURCE_CRS_DIR := ../source-crs
+PGT2ACMPG_TOOL_DIR := ../tools/pgt2acmpg
 # pgt2acmpg
 ACMPG_FROM_PGT_DIR=./argocd/example/acmpgfrompgt
 
@@ -26,7 +27,7 @@ build:
 	@echo "ZTP: setup ACM policyGenerator kustomize plugin"
 	mkdir -p $(ACM_POLICYGEN_KUSTOMIZE_DIR)
 	cp -r $(SOURCE_CRS_DIR) $(ACM_POLICYGEN_EX_DIR)
-	GOBIN=$(ACM_POLICYGEN_KUSTOMIZE_DIR) go install open-cluster-management.io/policy-generator-plugin/cmd/[email protected]
+	$(PGT2ACMPG_TOOL_DIR)/scripts/build-acmpg-plugin.sh "$(ACM_POLICYGEN_KUSTOMIZE_DIR)"
 
 $(KUSTOMIZE_BIN):
 	@if [[ $(KUSTOMIZE) == $(KUSTOMIZE_BIN) ]] && [ ! -f $(KUSTOMIZE) ]; then \
@@ -59,7 +60,7 @@ build-acmpg-from-pgt: pgt2acmpg-build
 	@echo "ZTP: setup ACM from pgt kustomize plugin"
 	rm -rf $(ACMPG_FROM_PGT_DIR)
 	../tools/pgt2acmpg/pgt2acmpg -i $(POLICYGEN_EX_DIR) -o $(ACMPG_FROM_PGT_DIR) -s ../tools/pgt2acmpg/test/newptpconfig-schema.json -k PtpConfig -g -c ../source-crs
-	GOBIN=$(ACM_POLICYGEN_KUSTOMIZE_DIR) go install open-cluster-management.io/policy-generator-plugin/cmd/[email protected]
+	$(PGT2ACMPG_TOOL_DIR)/scripts/build-acmpg-plugin.sh  "$(ACM_POLICYGEN_KUSTOMIZE_DIR)"
 
 gen-files-acmpg-from-pgt: build-acmpg-from-pgt $(KUSTOMIZE)
 	@mkdir -p $(ACMPG_FROM_PGT_DIR)/out/

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/README.md

@@ -18,7 +18,7 @@ These examples will generate ACM policies same as the DU profile policies genera
 
 
 ## Editing example templates from scratch
-Using ACM Policy Generator templatess is similar to PGT as described in chapter 3 of the following readme describing the overall ZTP process [link](https://github.com/openshift-kni/cnf-features-deploy/blob/master/ztp/gitops-subscriptions/argocd/README.md)
+Using ACM Policy Generator templates is similar to PGT as described in chapter 3 of the following readme describing the overall ZTP process [link](https://github.com/openshift-kni/cnf-features-deploy/blob/master/ztp/gitops-subscriptions/argocd/README.md)
 
 3. Create the ACMPG Template CR for your site in your local clone of the git repository:
    1. Begin by choosing an appropriate example from out/argocd/example/acmpolicygenerator. This directory demonstrates a 3-level policy framework which represents a well-supported low-latency profile tuned for the needs of 5G Telco DU deployments:
@@ -36,7 +36,7 @@ Using ACM Policy Generator templatess is similar to PGT as described in chapter
 
 ## Adding ManagedClustersetbinding 
 A ManagedClusterSet object brings together managed clusters with same access rights. In ZTP, the default clusterset is named `global`.
-With ACM Policy Generator templatess, it is required to specify a clusterset binding. The ManagedClusterSetBinding adds a namespace to to the list of namespaces allowed to managed the managed clusters in the clusterset.
+With ACM Policy Generator templates, it is required to specify a clusterset binding. The ManagedClusterSetBinding adds a namespace to to the list of namespaces allowed to managed the managed clusters in the clusterset.
 The ManagedClusterSetBinding can be added to the ns.yaml file. The managed ManagedClusterSetBinding below adds the `ztp-common`, `ztp-group` and `ztp-site` namespaces to the list of namespaces part of the `global` Clusterset
 
 ```
@@ -70,12 +70,26 @@ spec:
 Creating patches for CR objects containing lists is not currently supported by ACM the Generator Plugin. As a workwaround, the full content of final object must be contained in the patch. 
 The pgt2acmpg tool supports creating such a patch that contains the full content of the CR, see below.
 
-# Converting ACM Policy Generator templatess from exising Policy Gen Templates 
-The pgt2acmpg supports converting Policy Gen Templates to ACM Policy Generator templatess◊. More details can be found at [link](https://github.com/openshift-kni/cnf-features-deploy/ztp/tools/pgt2acmpg/blob/main/README.md)
+# The ACM PolicyGenerator version of the DU reference configuration
+The ACM PolicyGenerator version of this reference configuration is functionally
+identical to the PolicyGenTemplate version. The following sections describe some
+of the key aspects of the PolicyGenerator version.
 
-##### Setup ArgoCD Application
+For more general info using PolicyGenerator follow the [ACM PolicyGenerator
+examples](https://github.com/stolostron/policy-generator-plugin/policy-generator-plugin/tree/main/examples).
 
-The DU profile ACM PolicyGenerator examples can be used with the same [policies-app](https://github.com/openshift-kni/cnf-features-deploy/blob/master/ztp/gitops-subscriptions/argocd/deployment/policies-app.yaml) that is used to deploy the DU profile policygentemplates examples. For more info how to setup the ArgoCD policies application follow the [ReadMe](https://github.com/openshift-kni/cnf-features-deploy/blob/master/ztp/gitops-subscriptions/argocd/README.md) section "Preparation of Hub cluster for ZTP". The Git repo that will be used with the ArgoCD policies application should contain the source-crs directory and must co-exist with the DU profile ACM PolicyGenerator as shown below as example
+
+### ArgoCD Application setup
+
+The ACM PolicyGenerator version of the DU reference can be used with the same
+ArgoCD application
+[policies-app](https://github.com/openshift-kni/cnf-features-deploy/blob/master/ztp/gitops-subscriptions/argocd/deployment/policies-app.yaml)
+that is used to deploy the DU profile policygentemplates examples. For more info
+how to setup the ArgoCD policies application follow the
+[ReadMe](https://github.com/openshift-kni/cnf-features-deploy/blob/master/ztp/gitops-subscriptions/argocd/README.md)
+section "Preparation of Hub cluster for ZTP". The Git repo that will be used
+with the ArgoCD policies application should contain the source-crs directory and
+must co-exist with the DU profile ACM PolicyGenerator as shown below as example
 
 ```
 ├── acmpolicygenerator
@@ -87,6 +101,7 @@ The DU profile ACM PolicyGenerator examples can be used with the same [policies-
 ├── acm-example-sno-site.yaml
 ├── acm-group-du-3node-ranGen.yaml
 ├── acm-group-du-3node-validator-ranGen.yaml
+├── acm-group-du-clo5-cleanup.yaml
 ├── acm-group-du-sno-ranGen.yaml
 ├── acm-group-du-sno-validator-ranGen.yaml
 ├── acm-group-du-standard-ranGen.yaml
@@ -96,4 +111,37 @@ The DU profile ACM PolicyGenerator examples can be used with the same [policies-
 
 ```
 
-For more info using PolicyGenerator follow the [ACM PolicyGenerator examples](https://github.com/stolostron/policy-generator-plugin/policy-generator-plugin/tree/main/examples).
+### Upgrade Cluster Logging Operator to 6.0
+
+The Cluster Logging Operator (CLO) move from version 5.y to 6.0 required
+adaptation to a new API and careful management of the transition during cluster
+upgrades. The Operator itself could upgrade using typical Subscription channel
+changes (as rolled out by TALM), however the API change required a new CR to be
+created *followed by* deletion of the old API CRs. This ordering ensures that
+logs will be streamed from the cluster without interruption or massive
+duplication -- logs will be duplicated while both collectors are running in
+parallel but the new collectors will not restart at the beginning of the log
+files.
+
+The policy which removes the CLO 5.y API artifacts must account for two
+scenarios
+ - upgraded clusters where the 5.y CRDs and artifacts exist and must be removed
+ - newly deployed clusters where the 5.y CRDs never existed and the types are thus unknown
+
+To avoid having two separate policies, one for upgrade and one for newly
+deployed clusters, the `acm-group-du-clo5-cleanup` policy includes
+`ClusterLogging5Cleanup.yaml` which is not a true "source CR". This file is an
+ACM Policy `object-template-raw` which enables us to query for existence of the
+CRD and, if it exists, remove the old API CR and the CRD. This leverages the
+ACM PolicyGenerator support for source files containing object-template-raw
+content which is available from ACM 2.10+.
+
+The `acm-group-du-clo5-cleanup` PolicyGenerator was used to statically generate
+the Policy CR available in the ../policygentemplates directory:
+`group-du-clo5-cleanup-policy.yaml`. This ensures that the Policy applied to the
+hub cluster is consistent whether the environment uses PolicyGenTemplates (and
+includes the statically generated Policy) or PolicyGenerator.
+
+### Other
+
+The pgt2acmpg supports converting Policy Gen Templates to ACM Policy Generator templates. More details can be found at [link](https://github.com/openshift-kni/cnf-features-deploy/ztp/tools/pgt2acmpg/blob/main/README.md)

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/acm-common-ranGen.yaml

@@ -54,6 +54,9 @@ policies:
     - path: source-crs/ClusterLogOperGroup.yaml
     - path: source-crs/ClusterLogSubscription.yaml
     - path: source-crs/ClusterLogOperatorStatus.yaml
+    - path: source-crs/ClusterLogServiceAccount.yaml
+    - path: source-crs/ClusterLogServiceAccountAuditBinding.yaml
+    - path: source-crs/ClusterLogServiceAccountInfrastructureBinding.yaml
     # Ptp operator
     - path: source-crs/PtpSubscriptionNS.yaml
     - path: source-crs/PtpSubscription.yaml

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/acm-group-du-clo5-cleanup.yaml

@@ -0,0 +1,37 @@
+apiVersion: policy.open-cluster-management.io/v1
+kind: PolicyGenerator
+metadata:
+  name: group-du-sno-clo5-cleanup
+placementBindingDefaults:
+  name: group-du-sno-clo5-cleanup-placement-binding
+policyDefaults:
+  namespace: ztp-group
+  # Allow this policy to be unbound from clusters based on a custom label
+  placement:
+    labelSelector:
+      matchExpressions:
+      - key: group-du-sno
+        operator: Exists
+      - key: du-profile
+        operator: In
+        values: ["latest"]
+      - key: clo5-cleanup-done
+        operator: DoesNotExist
+  remediationAction: inform
+  severity: low
+  # standards: []
+  namespaceSelector:
+    exclude:
+      - kube-*
+    include:
+      - '*'
+  evaluationInterval:
+    compliant: 10m
+    noncompliant: 10s
+policies:
+- name: group-du-sno-clo5-cleanup
+  policyAnnotations:
+    ran.openshift.io/ztp-deploy-wave: "11"
+  manifests:
+    - path: source-crs/ClusterLogging5Cleanup.yaml
+

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/acm-group-du-sno-ranGen.yaml

@@ -35,23 +35,28 @@ policies:
       patches:
       - spec:
           outputs:
-            - type: "kafka"
-              name: kafka-open
-              # below url is an example
+          - type: "kafka"
+            name: kafka-open
+            # below url is an example
+            kafka:
               url: tcp://10.46.55.190:9092/test
-          pipelines:
-          - inputRefs:
-            - audit
-            - infrastructure
-            labels:
+          filters:
+          - name: test-labels
+            type: openshiftLabels
+            openshiftLabels:
               label1: test1
               label2: test2
               label3: test3
               label4: test4
-            name: all-to-default
+          pipelines:
+          - name: all-to-default
+            inputRefs:
+            - audit
+            - infrastructure
+            filterRefs:
+            - test-labels
             outputRefs:
             - kafka-open
-    - path: source-crs/ClusterLogging.yaml
     ## The setting below overrides the default "worker" selector predefined in
     ## the source-crs. The change is recommended on SNOs configured with PTP 
     ## event notification for forward compatibility with possible SNO expansion.

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/hub-side-templating/acm-group-du-sno-ranGen-templated.yaml

@@ -41,9 +41,9 @@ policies:
     - path: source-crs/ClusterLogForwarder.yaml
       patches:
       - spec:
+          filters: '{{hub fromConfigMap "" "group-zones-configmap" (printf "%s-cluster-log-fwd-filters" (index .ManagedClusterLabels "group-du-sno-zone")) | toLiteral hub}}'
           outputs: '{{hub fromConfigMap "" "group-zones-configmap" (printf "%s-cluster-log-fwd-outputs" (index .ManagedClusterLabels "group-du-sno-zone")) | toLiteral hub}}'
           pipelines: '{{hub fromConfigMap "" "group-zones-configmap" (printf "%s-cluster-log-fwd-pipelines" (index .ManagedClusterLabels "group-du-sno-zone")) | toLiteral hub}}'
-    - path: source-crs/ClusterLogging.yaml
     ## The setting below overrides the default "worker" selector predefined in
     ## the source-crs. The change is recommended on SNOs configured with PTP 
     ## event notification for forward compatibility with possible SNO expansion.

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/hub-side-templating/configMaps/group-zones-configmap.yaml

@@ -6,8 +6,9 @@ metadata:
   namespace: ztp-group
 data:
   # ClusterLogForwarder.yaml
-  zone-1-cluster-log-fwd-outputs: "[{\"type\":\"kafka\", \"name\":\"kafka-open\", \"url\":\"tcp://10.46.55.190:9092/test\", \"secret\":{\"name\":\"kafka-open-secret\"}}]"
-  zone-1-cluster-log-fwd-pipelines: "[{\"inputRefs\":[\"audit\", \"infrastructure\"], \"labels\": {\"label1\": \"test1\", \"label2\": \"test2\", \"label3\": \"test3\", \"label4\": \"test4\"}, \"name\": \"all-to-default\", \"outputRefs\": [\"kafka-open\"]}]"
+  zone-1-cluster-log-fwd-filters: "[{\"name\":\"test-labels\",\"type\":\"openshiftLabels\",\"openshiftLabels\":{\"label1\":\"test1\",\"label2\":\"test2\",\"label3\":\"test3\",\"label4\":\"test4\"}}]"
+  zone-1-cluster-log-fwd-outputs: "[{\"type\":\"kafka\",\"name\":\"kafka-open\",\"kafka\":{\"url\":\"tcp://10.46.55.190:9092/test\"},\"secret\":{\"name\":\"kafka-open-secret\"}}]"
+  zone-1-cluster-log-fwd-pipelines: "[{\"name\":\"all-to-default\",\"inputRefs\":[\"audit\",\"infrastructure\"],\"filterRefs\":[\"test-labels\"],\"outputRefs\":[\"kafka-open\"]}]"
   # HardwareEvent.yaml
   zone-1-hwevent-logLevel: "debug"
   # StoragePVC.yaml

ztp/gitops-subscriptions/argocd/example/acmpolicygenerator/kustomization.yaml

@@ -19,6 +19,8 @@ generators:
 # These are examples that should be replicated for every individual site. Use just one for each cluster configuration.
 - acm-example-sno-site.yaml
 - acm-example-multinode-site.yaml
+# This policy removes Cluster Logging operator 5.x artifacts after upgrading to 6.x
+- acm-group-du-clo5-cleanup.yaml
 
 resources:
 - ns.yaml

ztp/gitops-subscriptions/argocd/example/configuration-groups.md

@@ -12,7 +12,6 @@ Depending on the exact purpose and configuration for the spokes, some of the sou
 * ClusterLogForwarder.yaml
     * spec.outputs
     * spec.pipelines
-* ClusterLogging.yaml
 * DisableOLMPprof.yaml
 * DisableSnoNetworkDiag.yaml (Cluster Network Operator)
 * HardwareEvent.yaml
@@ -32,7 +31,6 @@ Depending on the exact purpose and configuration for the spokes, some of the sou
 * ClusterLogForwarder.yaml
     * spec.outputs
     * spec.pipelines
-* ClusterLogging.yaml
 * DisableSnoNetworkDiag.yaml (Cluster Network Operator)
     > Note: If users want to configure things apart spec.disableNetworkDiagnostics
 * HardwareEvent.yaml

ztp/gitops-subscriptions/argocd/example/policygentemplates/common-ranGen.yaml

@@ -35,6 +35,12 @@ spec:
       policyName: "subscriptions-policy"
     - fileName: ClusterLogOperatorStatus.yaml
       policyName: "subscriptions-policy"
+    - fileName: ClusterLogServiceAccount.yaml
+      policyName: "subscriptions-policy"
+    - fileName: ClusterLogServiceAccountAuditBinding.yaml
+      policyName: "subscriptions-policy"
+    - fileName: ClusterLogServiceAccountInfrastructureBinding.yaml
+      policyName: "subscriptions-policy"
     - fileName: StorageNS.yaml
       policyName: "subscriptions-policy"
     - fileName: StorageOperGroup.yaml

ztp/gitops-subscriptions/argocd/example/policygentemplates/group-du-clo5-cleanup-policy.yaml

@@ -0,0 +1,94 @@
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: Policy
+metadata:
+    annotations:
+        policy.open-cluster-management.io/categories: CM Configuration Management
+        policy.open-cluster-management.io/controls: CM-2 Baseline Configuration
+        policy.open-cluster-management.io/description: ""
+        policy.open-cluster-management.io/standards: NIST SP 800-53
+        ran.openshift.io/ztp-deploy-wave: "11"
+    name: group-du-sno-clo5-cleanup
+    namespace: ztp-group
+spec:
+    disabled: false
+    policy-templates:
+        - objectDefinition:
+            apiVersion: policy.open-cluster-management.io/v1
+            kind: ConfigurationPolicy
+            metadata:
+                name: group-du-sno-clo5-cleanup
+            spec:
+                evaluationInterval:
+                    compliant: 10m
+                    noncompliant: 10s
+                namespaceSelector:
+                    exclude:
+                        - kube-*
+                    include:
+                        - '*'
+                object-templates-raw: |
+                    {{ if ne (default "" (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "clusterlogforwarders.logging.openshift.io").metadata.name) "" }}
+                    - complianceType: mustnothave
+                      objectDefinition:
+                        apiVersion: logging.openshift.io/v1
+                        kind: ClusterLogForwarder
+                        metadata:
+                          name: instance
+                          namespace: openshift-logging
+                    - complianceType: mustnothave
+                      objectDefinition:
+                        apiVersion: apiextensions.k8s.io/v1
+                        kind: CustomResourceDefinition
+                        metadata:
+                          name: clusterlogforwarders.logging.openshift.io
+                    {{ end }}
+                    {{ if ne (default "" (lookup "apiextensions.k8s.io/v1" "CustomResourceDefinition" "" "clusterloggings.logging.openshift.io").metadata.name) "" }}
+                    - complianceType: mustnothave
+                      objectDefinition:
+                        apiVersion: logging.openshift.io/v1
+                        kind: ClusterLogging
+                        metadata:
+                          name: instance
+                          namespace: openshift-logging
+                    - complianceType: mustnothave
+                      objectDefinition:
+                        apiVersion: apiextensions.k8s.io/v1
+                        kind: CustomResourceDefinition
+                        metadata:
+                          name: clusterloggings.logging.openshift.io
+                    {{ end }}
+                remediationAction: inform
+                severity: low
+    remediationAction: inform
+---
+apiVersion: apps.open-cluster-management.io/v1
+kind: PlacementRule
+metadata:
+    name: placementrule-group-du-sno-clo5-cleanup
+    namespace: ztp-group
+spec:
+    clusterSelector:
+        matchExpressions:
+            - key: group-du-sno
+              operator: Exists
+            - key: du-profile
+              operator: In
+              values:
+                - latest
+            - key: clo5-cleanup-done
+              operator: DoesNotExist
+---
+apiVersion: policy.open-cluster-management.io/v1
+kind: PlacementBinding
+metadata:
+    name: binding-group-du-sno-clo5-cleanup
+    namespace: ztp-group
+placementRef:
+    apiGroup: apps.open-cluster-management.io
+    kind: PlacementRule
+    name: placementrule-group-du-sno-clo5-cleanup
+subjects:
+    - apiGroup: policy.open-cluster-management.io
+      kind: Policy
+      name: group-du-sno-clo5-cleanup