(Review): Adding OpenEBS NFS provisioner kill experiment (#1248) (#1397)

* (feat): Adding OpenEBS NFS provisioner kill experiment

Signed-off-by: Raj <[email protected]>

build/ansible-runner/Dockerfile

@@ -40,4 +40,4 @@ COPY ./build/plugins/default.py /usr/local/lib/python2.7/dist-packages/ansible/p
 COPY ./build/utils/display.py /usr/local/lib/python2.7/dist-packages/ansible/utils/display.py
 COPY experiments ./experiments
 COPY executor ./executor
-COPY ansiblelint ./ansiblelint
+COPY ansiblelint ./ansiblelint

experiments/openebs/openebs-nfs-provisioner-kill/chaosutil.j2

@@ -0,0 +1,5 @@
+{% if c_lib is defined and c_lib == 'pumba' %}
+c_util: /chaoslib/pumba/pod_failure_by_sigkill.yml
+{% elif c_lib is defined and c_lib == 'containerd' %}
+c_util: /chaoslib/litmus/container_kill/containerd_chaos/crictl-chaos.yml
+{% endif %}

experiments/openebs/openebs-nfs-provisioner-kill/openebs_nfs_provisioner_kill_ansible_logic.yml

@@ -0,0 +1,166 @@
+- hosts: localhost
+  connection: local
+
+  vars:
+    c_experiment: "openebs-nfs-provisioner-kill"
+    nfs_provisioner_ns: "{{ lookup('env', 'NFS_PROVISIONER_NAMESPACE')}}"
+    nfs_provisioner_label: "{{ lookup('env', 'NFS_PROVISIONER_LABEL')}}"
+    nfs_pvc: "{{ lookup('env', 'NFS_PVC') }}"
+    nfs_svc: "{{ lookup('env', 'NFS_SVC') }}"
+    a_label: "{{ lookup('env','APP_LABEL') }}"
+    a_ns: "{{ lookup('env','APP_NAMESPACE') }}"
+    c_lib: "{{ lookup('env', 'LIB') }}"
+    c_container: "{{ lookup('env','TARGET_CONTAINER') }}"
+    lib_image: "{{ lookup('env','LIB_IMAGE') }}"
+    c_engine: "{{ lookup('env','CHAOSENGINE') }}"
+    external_app_check: "{{ lookup('env', 'EXTERNAL_APP_CHECK') }}"
+    chaos_pod_name: "{{ lookup('env','POD_NAME') }}"
+    c_interval: "{{ lookup('env', 'CHAOS_INTERVAL')}}"
+    c_ns: "{{ lookup('env','CHAOS_NAMESPACE') }}"
+    c_duration: "{{ lookup('env','TOTAL_CHAOS_DURATION') }}"
+
+  vars_files:
+    - /mnt/parameters.yml
+
+  tasks:
+
+    - block:
+      - include: openebs_nfs_provisioner_kill_prerequisites.yml
+
+      - include_vars:
+          file: chaosutil.yml
+
+      ## GENERATE EXP RESULT NAME
+      - name: Construct chaos result name (experiment_name)
+        set_fact:
+          c_result: "{{ lookup('env','CHAOSENGINE') }}-{{ c_experiment }}"
+        when: "c_engine is defined and c_engine != ''"
+
+      ## RECORD START-OF-EXPERIMENT IN CHAOS RESULT CR
+      - include_tasks: /utils/runtime/update_chaos_result_resource.yml
+        vars:
+          status: 'SOT'
+          namespace: "{{ c_ns }}"
+
+      ## DISPLAY APP INFORMATION 
+      - name: Display the app information passed via the test job
+        debug: 
+          msg: 
+            - "The application info is as follows:"
+            - "NFS Provisioner Namespace : {{ nfs_provisioner_ns }}"
+            - "NFS Provisioner LABEL     : {{ nfs_provisioner_label }}"
+            - "NFS Provisioner PVC       : {{ nfs_pvc }}"
+            - "NFS Service               : {{ nfs_svc }}"
+
+      - block:
+          - name: Generate a external app run id
+            shell: echo $(mktemp) | cut -d '.' -f 2 
+            register: external_app_rand_string   
+
+          - set_fact:
+              external_app_run_id: "{{ external_app_rand_string.stdout | lower }}"
+
+          - name: Deploying and checking liveness of external application using NFS
+            include_tasks: "/utils/apps/nfs/external_liveness_check.yml"
+            vars:
+              app_ns: "{{ a_ns }}"
+              app_label: "{{ a_label }}"
+              state: "START"
+              run_id: "{{ external_app_run_id }}"
+
+        when: "external_app_check == 'true'"
+
+      ## PRE-CHAOS APPLICATION LIVENESS CHECK
+      - name: Generate a nfs exporter liveness check run id
+        shell: echo $(mktemp) | cut -d '.' -f 2 
+        register: user_app_rand_string   
+
+      - set_fact:
+          user_app_run_id: "{{ user_app_rand_string.stdout | lower }}"
+
+      - name: (Pre) Verify that the AUT (Application Under Test) is running 
+        include_tasks: "/utils/common/status_app_pod.yml"
+        vars:
+          app_ns: "{{ a_ns }}"
+          app_label: "{{ a_label }}"      
+          delay: 2
+          retries: 90
+
+      - name: (Pre) Checking the user's pod volume is using NFS 
+        include_tasks: "/utils/apps/nfs/nfs_mount_check.yml"
+        vars:
+          app_ns: "{{ a_ns }}"
+          app_label: "{{ a_label }}"
+          run_id: "{{ user_app_run_id }}"
+          state: "START"
+
+      ## RECORD EVENT FOR PRE-CHAOS CHECK
+      - include_tasks: /utils/common/generate-kubernetes-chaos-events.yml
+        vars:
+          stage: "PreChaosCheck"
+          exp_pod_name: "{{ chaos_pod_name }}"
+          engine_ns: "{{ c_ns }}"
+          message: "OpenEBS NFS are Running successfully"
+        when: "c_engine is defined and c_engine != ''"
+
+      ## Injecting the NFS kill chaos
+      - name: Injecting the NFS provisioner container with {{ c_lib }} chaos LIB
+        include_tasks: "{{ c_util }}"
+        vars:
+          namespace: "{{ nfs_provisioner_ns }}"
+          app_container: "{{ c_container }}"
+          a_label: "{{ nfs_provisioner_label }}"
+
+      ## POST-CHAOS APPLICATION LIVENESS CHECK
+      - name: (Post) Verify that the AUT (Application Under Test) is running 
+        include_tasks: "/utils/common/status_app_pod.yml"
+        vars:
+          app_ns: "{{ a_ns }}"
+          app_label: "{{ a_label }}"
+          delay: 2
+          retries: 90
+
+      - name: (Post) Re-Checking the user's pod volume is using NFS 
+        include_tasks: "/utils/apps/nfs/nfs_mount_check.yml"
+        vars:
+          app_ns: "{{ a_ns }}"
+          app_label: "{{ a_label }}"
+          run_id: "{{ user_app_run_id }}"
+          state: "END"
+
+      - block:
+        - name: Deploying and checking liveness of external application using NFS
+          include_tasks: "/utils/apps/nfs/external_liveness_check.yml"
+          vars:
+            app_ns: "{{ a_ns }}"
+            app_label: "{{ a_label }}"
+            state: "END"
+            run_id: "{{ external_app_run_id }}"
+        when: "external_app_check == 'true'"
+
+      ## RECORD EVENT FOR POST-CHAOS CHECK
+      - include_tasks: /utils/common/generate-kubernetes-chaos-events.yml
+        vars:
+          stage: "PostChaosCheck"
+          exp_pod_name: "{{ chaos_pod_name }}"
+          engine_ns: "{{ c_ns }}"
+          message: "OpenEBS NFS are Running successfully"
+        when: "c_engine is defined and c_engine != ''"
+
+      - set_fact:
+          flag: "Pass"
+
+      rescue: 
+        - set_fact: 
+            flag: "Fail"
+
+      always: 
+
+        ## Getting failure step from experiment-pod 
+        - include_tasks: /utils/runtime/getting_failure_step.yml
+
+         ## RECORD END-OF-TEST IN CHAOS RESULT CR
+        - include_tasks: /utils/runtime/update_chaos_result_resource.yml
+          vars:
+            status: "EOT"
+            namespace: "{{ c_ns }}"

experiments/openebs/openebs-nfs-provisioner-kill/openebs_nfs_provisioner_kill_k8s_job.yml

@@ -0,0 +1,74 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  generateName: openebs-nfs-provisioner-kill-
+spec:
+  template:
+    metadata:
+      labels:
+        experiment: openebs-nfs-provisioner-kill
+    spec:
+      serviceAccountName: %CHAOS_SERVICE_ACCOUNT%
+      restartPolicy: Never
+      containers:
+      - name: ansibletest
+        image: litmuschaos/ansible-runner:ci
+        imagePullPolicy: Always
+        env:
+          - name: ANSIBLE_STDOUT_CALLBACK
+            value: 'default'
+
+          #provide openebs namespace
+          - name: NFS_PROVISIONER_NAMESPACE
+            value: ''
+
+          # provide application namespace
+          - name: NFS_PROVISIONER_LABEL
+            value: ''
+
+          # provide application pvc
+          - name: NFS_PVC
+            value: ''
+
+          # provide application label
+          - name: APP_LABEL
+            value: ''
+
+          - name: APP_NAMESPACE
+            value: ''
+
+          - name: NFS_SVC
+            value: ''
+
+          # provide lib - It supports pumba and containerd
+          - name: LIB 
+            value: 'pumba'
+
+          # LIB_IMAGE can be - gaiaadm/pumba:0.4.8, gprasath/crictl:ci
+          # For pumba image use : gaiaadm/pumba:0.4.8
+          # For containerd image use : gprasath/crictl:ci
+          - name: LIB_IMAGE  
+            value: 'gaiaadm/pumba:0.4.8'
+
+          - name: TARGET_CONTAINER
+            value: 'nfs-provisioner'
+
+          # EXTERNAL_APP_CHECK can be true/false
+          - name: EXTERNAL_APP_CHECK
+            value: 'false'
+
+          - name: CHAOS_INTERVAL
+            value: '10'
+
+          - name: TOTAL_CHAOS_DURATION
+            value: '20'
+
+        command: ["/bin/bash"]
+        args: ["-c", "ansible-playbook ./experiments/openebs/openebs-nfs-provisioner-kill/openebs_nfs_provisioner_kill_ansible_logic.yml -i /etc/ansible/hosts -vv; exit 0"]
+        volumeMounts:
+          - name: parameters
+            mountPath: /mnt/
+      volumes:
+        - name: parameters
+          configMap:
+            name: openebs-nfs-provisioner-kill

experiments/openebs/openebs-nfs-provisioner-kill/openebs_nfs_provisioner_kill_prerequisites.yml

@@ -0,0 +1,4 @@
+- name: Identify the chaos util to be invoked 
+  template:
+    src: chaosutil.j2
+    dest: chaosutil.yml

utils/apps/nfs/external_applications.j2

@@ -0,0 +1,74 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: write-pod-{{ run_id }}
+  name: write-pod-{{ run_id }}
+  namespace: {{ a_ns }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: write-pod-{{ run_id }}
+  template:
+    metadata:
+      labels:
+        run: write-pod-{{ run_id }}
+    spec:
+      containers:
+      - name: write-pod
+        image: gcr.io/google_containers/busybox:1.24
+        command:
+        - "sleep"
+        - "1000"
+        volumeMounts:
+        - name: nfs-pvc
+          mountPath: "/busybox"
+      volumes:
+      - name: nfs-pvc
+        persistentVolumeClaim:
+          claimName: testclaim-{{ run_id }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  labels:
+    run: read-pod-{{ run_id }}
+  name: read-pod-{{ run_id }}
+  namespace: {{ a_ns }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: read-pod-{{ run_id }}
+  template:
+    metadata:
+      labels:
+        run: read-pod-{{ run_id }}
+    spec:
+      containers:
+      - name: read-pod
+        image: gcr.io/google_containers/busybox:1.24
+        command:
+        - "sleep"
+        - "1000"
+        volumeMounts:
+        - name: nfs-pvc
+          mountPath: "/busybox"
+      volumes:
+      - name: nfs-pvc
+        persistentVolumeClaim:
+          claimName: testclaim-{{ run_id }}
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: testclaim-{{ run_id }}
+  namespace: {{ a_ns }}
+spec:
+  storageClassName: openebs-nfs 
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 10Mi