Restart QDR after changing the password (#530) (#534)

* Restart QDR after changing the password

* Fixes bug reported here: #517 (comment)
* Avoids an extra manual step when changing password
* Would affect users who upgrade from earlier STF and subsequently enable basic auth
* Also users who need to change their passwords

* Fixing ansible lint

* Update roles/servicetelemetry/tasks/component_qdr.yml

* Adjust QDR restarts to account for HA

* [smoketest] Wait for qdr-test to be Running

* [smoketest] Wait for QDR password upgrade

* Remove zuul QDR auth override

(cherry picked from commit 16b8197)

Author: csibbitt

ci/vars-zuul-common.yml

@@ -2,6 +2,5 @@
 namespace: "service-telemetry"
 setup_bundle_registry_tls_ca: false
 setup_bundle_registry_auth: false
-__service_telemetry_transports_qdr_auth: none
 base_dir: "{{ sto_dir }}/build"
 logfile_dir: "{{ ansible_user_dir }}/zuul-output/logs/controller"

roles/servicetelemetry/tasks/component_qdr.yml

@@ -163,21 +163,41 @@
         namespace: "{{ ansible_operator_meta.namespace }}"
       register: _qdr_basicauth_object
 
-    # Because https://github.com/interconnectedcloud/qdr-operator/blob/576d2b33dac71437ea2b165caaaf6413220767fe/pkg/controller/interconnect/interconnect_controller.go#L634
-    - name: Perform a one-time upgrade to the default generated password for QDR BasicAuth
-      k8s:
-        definition:
-          kind: Secret
-          apiVersion: v1
-          metadata:
-            name: "{{ ansible_operator_meta.name }}-interconnect-users"
+    - when:
+      - _qdr_basicauth_object.resources[0] is defined and _qdr_basicauth_object.resources[0].metadata.labels.stf_one_time_upgrade is not defined
+      block:
+        # Because https://github.com/interconnectedcloud/qdr-operator/blob/576d2b33dac71437ea2b165caaaf6413220767fe/pkg/controller/interconnect/interconnect_controller.go#L634
+        - name: Perform a one-time upgrade to the default generated password for QDR BasicAuth
+          k8s:
+            definition:
+              kind: Secret
+              apiVersion: v1
+              metadata:
+                name: "{{ ansible_operator_meta.name }}-interconnect-users"
+                namespace: "{{ ansible_operator_meta.namespace }}"
+                labels:
+                  stf_one_time_upgrade: "{{ lookup('pipe', 'date +%s') }}"
+              stringData:
+                guest: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}"
+
+        # label_selectors on the k8s object need kubernetes.core>=2.2.0
+        - name: Get the list of QDR pods
+          k8s_info:
+            api_version: v1
+            kind: Pod
+            namespace: "{{ ansible_operator_meta.namespace }}"
+            label_selectors:
+              - application={{ ansible_operator_meta.name }}-interconnect
+          register: _qdr_pod
+
+        - name: Restart QDR pods to pick up new password
+          k8s:
+            state: absent
+            api_version: v1
+            kind: Pod
             namespace: "{{ ansible_operator_meta.namespace }}"
-            labels:
-              stf_one_time_upgrade: "{{ lookup('pipe', 'date +%s') }}"
-          stringData:
-            guest: "{{ lookup('password', '/dev/null chars=ascii_letters,digits length=32') }}"
-      when:
-        - _qdr_basicauth_object.resources[0] is defined and _qdr_basicauth_object.resources[0].metadata.labels.stf_one_time_upgrade is not defined
+            name: "{{ item.metadata.name }}"
+          loop: "{{ _qdr_pod.resources }}"
 
 - name: Set default Interconnect manifest
   set_fact:

tests/smoketest/smoketest.sh

@@ -59,14 +59,20 @@ oc create configmap stf-smoketest-collectd-entrypoint-script --from-file "${REL}
 oc create configmap stf-smoketest-ceilometer-publisher --from-file "${REL}/ceilometer_publish.py"
 oc create configmap stf-smoketest-ceilometer-entrypoint-script --from-file "${REL}/smoketest_ceilometer_entrypoint.sh"
 
-echo "*** [INFO] Creating Mock OSP Metrics QDR router..."
+echo "*** [INFO] Waiting for QDR password upgrade"
+AMQP_PASS=''
+while [ ${#AMQP_PASS} -lt 32 ]; do AMQP_PASS=$(oc get secret default-interconnect-users -o json | jq -r .data.guest | base64 -d); sleep 3; done
+
+echo "*** [INFO] Creating Mock OSP Metrics QDR..."
 oc delete pod qdr-test
 oc delete service qdr-test
 oc delete configmap qdr-test-config
-AMQP_PASS=$(oc get secret default-interconnect-users -o json | jq -r .data.guest | base64 -d)
 oc create -f <(sed -e "s/<<AMQP_PASS>>/${AMQP_PASS}/;" "${REL}/qdr-test.conf.yaml.template")
 oc create -f "${REL}/qdr-test.yaml"
 
+echo -e "\n* [INFO] Waiting for OSP Metrics QDR pod to be Running\n"
+oc wait --for=jsonpath='{.status.phase}'=Running pod/qdr-test
+
 echo "*** [INFO] Creating smoketest jobs..."
 oc delete job -l app=stf-smoketest
 for NAME in "${CLOUDNAMES[@]}"; do