Merge pull request #568 from IBM/db2u-node-settings

Implement node setting changes as per CP4D documentation

automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-install/tasks/cp4d-prep-wkc-46.yml

@@ -0,0 +1,50 @@
+---
+- name: Prepare yaml file for WKC SCC
+  template:
+    src: "wkc-scc.j2"
+    dest: "{{ status_dir }}/cp4d/wkc-scc.yml"
+
+- name: Apply WKC SCC
+  command: |
+    oc apply -f "{{ status_dir }}/cp4d/wkc-scc.yml"
+  register: _scc_create_result
+  retries: 10
+  delay: 30
+  until: _scc_create_result.rc == 0
+  vars:
+    ansible_callback_diy_runner_retry_msg: >-
+      {%- set result = ansible_callback_diy.result.output -%}
+      {%- set retries_left = result.retries - result.attempts -%}
+      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
+
+- name: Authorize WKC service account to use SCC
+  shell:
+    oc adm policy add-scc-to-user wkc-iis-scc system:serviceaccount:{{ _p_current_cp4d_cluster.project }}:wkc-iis-sa
+
+- name: Insert wkc options into {{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml
+  blockinfile:
+    path: "{{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml"
+    marker: "# {mark} WKC options #"
+    block: |2
+        wkc:
+          install_wkc_core_only: {{ _current_cp4d_cartridge.installation_options.install_wkc_core_only | default(True) }}
+          enableKnowledgeGraph: {{ _current_cp4d_cartridge.installation_options.enableKnowledgeGraph | default(False) }}
+          enableDataQuality: {{ _current_cp4d_cartridge.installation_options.enableDataQuality | default(False) }}
+          enableFactSheet: {{ _current_cp4d_cartridge.installation_options.enableFactSheet | default(False) }}
+          enableMANTA: {{ _current_cp4d_cartridge.installation_options.enableMANTA | default(False) }}
+  when: _p_current_cp4d_cluster.db2u_limited_privileges | default(False)
+
+- name: Insert wkc options into {{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml with elevated privileges
+  blockinfile:
+    path: "{{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml"
+    marker: "# {mark} WKC options #"
+    block: |2
+        wkc:
+          install_wkc_core_only: {{ _current_cp4d_cartridge.installation_options.install_wkc_core_only | default(True) }}
+          enableKnowledgeGraph: {{ _current_cp4d_cartridge.installation_options.enableKnowledgeGraph | default(False) }}
+          enableDataQuality: {{ _current_cp4d_cartridge.installation_options.enableDataQuality | default(False) }}
+          enableFactSheet: {{ _current_cp4d_cartridge.installation_options.enableFactSheet | default(False) }}
+          enableMANTA: {{ _current_cp4d_cartridge.installation_options.enableMANTA | default(False) }}
+          wkc_db2u_set_kernel_params: True
+          iis_db2u_set_kernel_params: True
+  when: not (_p_current_cp4d_cluster.db2u_limited_privileges | default(False))

automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-install/tasks/cp4d-prep-wkc-47.yml

@@ -0,0 +1,11 @@
+---
+- name: Insert wkc options into {{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml
+  blockinfile:
+    path: "{{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml"
+    marker: "# {mark} WKC options #"
+    block: |2
+        wkc:
+          install_wkc_core_only: {{ _current_cp4d_cartridge.installation_options.install_wkc_core_only | default(True) }}
+          enableKnowledgeGraph: {{ _current_cp4d_cartridge.installation_options.enableKnowledgeGraph | default(False) }}
+          enableDataQuality: {{ _current_cp4d_cartridge.installation_options.enableDataQuality | default(False) }}
+          enableFactSheet: {{ _current_cp4d_cartridge.installation_options.enableFactSheet | default(False) }}

automation-roles/50-install-cloud-pak/cp4d/cp4d-cartridge-install/tasks/cp4d-prep-wkc.yml

@@ -1,38 +1,9 @@
 ---
-- name: Prepare yaml file for WKC SCC
-  template:
-    src: "wkc-scc.j2"
-    dest: "{{ status_dir }}/cp4d/wkc-scc.yml"
+- include_tasks: cp4d-prep-wkc-46.yml
+  when: _p_current_cp4d_cluster.cp4d_version < "4.7.0"
 
-- name: Apply WKC SCC
-  command: |
-    oc apply -f "{{ status_dir }}/cp4d/wkc-scc.yml"
-  register: _scc_create_result
-  retries: 10
-  delay: 30
-  until: _scc_create_result.rc == 0
-  vars:
-    ansible_callback_diy_runner_retry_msg: >-
-      {%- set result = ansible_callback_diy.result.output -%}
-      {%- set retries_left = result.retries - result.attempts -%}
-      Retrying: {{ ansible_callback_diy.task.name }} ({{ retries_left }} Retries left) ...
-
-- name: Authorize WKC service account to use SCC
-  shell:
-    oc adm policy add-scc-to-user wkc-iis-scc system:serviceaccount:{{ _p_current_cp4d_cluster.project }}:wkc-iis-sa
-
-- name: Insert wkc options into {{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml
-  blockinfile:
-    path: "{{ status_dir }}/cp4d/{{ _p_current_cp4d_cluster.project }}-install-options.yml"
-    marker: "# {mark} WKC options #"
-    block: |2
-        wkc:
-          install_wkc_core_only: {{ _current_cp4d_cartridge.installation_options.install_wkc_core_only | default(True) }}
-          enableKnowledgeGraph: {{ _current_cp4d_cartridge.installation_options.enableKnowledgeGraph | default(False) }}
-          enableDataQuality: {{ _current_cp4d_cartridge.installation_options.enableDataQuality | default(False) }}
-          enableFactSheet: {{ _current_cp4d_cartridge.installation_options.enableFactSheet | default(False) }}
-          enableMANTA: {{ _current_cp4d_cartridge.installation_options.enableMANTA | default(False) }}
-          wkc_db2u_set_kernel_params: {{ _current_cp4d_cartridge.installation_options.wkc_db2u_set_kernel_params | default(False) }}
+- include_tasks: cp4d-prep-wkc-47.yml
+  when: _p_current_cp4d_cluster.cp4d_version >= "4.7.0"
 
 - block:
   - name: Starting background task to patch UG in case of WKC 4.0.x. Logs are in {{ status_dir }}/log/{{ _p_current_cp4d_cluster.project }}-wkc-40-patch.log

automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/cp4d-db2u-configmap.yml

@@ -0,0 +1,9 @@
+---
+- name: Generate configmap file {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-db2u-product-cm.yml
+  template:
+    src: db2u-product-cm.j2
+    dest: "{{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-db2u-product-cm.yml"
+
+- name: Create configmap db2u-product-cm in project {{ current_cp4d_cluster.project }}
+  shell: |
+    oc apply -f {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-db2u-product-cm.yml

automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/tasks/openshift-prepare-project.yml

@@ -31,8 +31,13 @@
   command: oc new-project {{ current_cp4d_cluster.project }}
   when: _cp4d_cluster_project_exists.rc != 0
 
-# Authorize instance topology
+# Prepare CP4D for versions >= 4.7.0
 - block:
+  # Set configmap to control if Db2U must run with elevated or limited privileges
+  - include_tasks: cp4d-db2u-configmap.yml
+    when:
+    - _db2u_dependency | default(False)
+
   - name: Generate authorize instance script {{ status_dir }}/cp4d/{{ current_cp4d_cluster.project }}-authorize-instance.sh
     template:
       src: authorize-instance.j2

automation-roles/50-install-cloud-pak/cp4d/cp4d-cluster/templates/db2u-product-cm.j2

@@ -0,0 +1,7 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: db2u-product-cm
+  namespace: {{ current_cp4d_cluster.operators_project | default('cpd-operators') }}
+data:
+  DB2U_RUN_WITH_LIMITED_PRIVS: "{{ current_cp4d_cluster.db2u_limited_privileges | default(False) | string | lower }}"

automation-roles/50-install-cloud-pak/cp4d/cp4d-ocp-tuned/files/cp4d-generate-tuned.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+
+# Compute IPC kernel parameters as per IBM Documentation topic
+# https://www.ibm.com/support/knowledgecenter/SSEPGG_11.1.0/com.ibm.db2.luw.qb.server.doc/doc/c0057140.html
+# and generate the Node Tuning Operator CR yaml.
+
+tuned_cr_yaml="/tmp/Db2UnodeTuningCR.yaml"
+mem_limit_Gi=0
+node_label=""
+cr_name="cp4d-ipc"
+cr_profile_name="cp4d-ipc"
+cr_namespace="openshift-cluster-node-tuning-operator"
+create_cr="false"
+delete_cr="false"
+
+usage() {
+    cat <<-USAGE #| fmt
+    Usage: $0 [OPTIONS] [arg]
+
+    OPTIONS:
+    =======
+    * -m|--mem-limit mem_limit  : The memory.limit (Gi) to be applied to Db2U deployment.
+    * [-l|--label node_label]   : The node label to use for dedicated Cp4D deployments.
+    * [-f|--file yaml_output]   : The NodeTuningOperator CR YAML output file. Default /tmp/Db2UnodeTuningCR.yaml.
+    * [-c|--create]             : Create the NodeTuningOperator CR ${cr_name} using the generated CR yaml file.
+    * [-d|--delete]             : Delete the NodeTuningOperator CR ${cr_name}.
+    * [-h|--help]               : Display the help text of the script.
+USAGE
+}
+
+[[ $# -lt 1 ]] && { usage && exit 1; }
+
+while [[ $# -gt 0 ]]; do
+    case "$1" in
+        -f|--file) shift; tuned_cr_yaml=$1
+        ;;
+        -m|--mem-limit) shift; mem_limit_Gi=$1
+        ;;
+        -l|--label) shift; node_label=$1
+        ;;
+        -c|--create) create_cr="true"
+        ;;
+        -d|--delete) delete_cr="true"
+        ;;
+        -h|--help) usage && exit 0
+        ;;
+        *) usage && exit 1
+        ;;
+	esac
+	shift
+done
+
+((ram_in_BYTES=mem_limit_Gi * 1073741824))
+((ram_GB=ram_in_BYTES / (1024 * 1024 * 1024)))
+((IPCMNI_LIMIT=32 * 1024))
+tr ' ' '\n' < /proc/cmdline | grep -q ipcmni_extend && ((IPCMNI_LIMIT=8 * 1024 * 1024))
+
+#
+### =============== functions ================ ###
+#
+# Compute the required kernel IPC parameter values
+compute_kernel_ipc_params() {
+    local PAGESZ=$(getconf PAGESIZE)
+
+    # Global vars
+    ((shmmni=256 * ram_GB))
+    shmmax=${ram_in_BYTES}
+    ((shmall=2 * (ram_in_BYTES / PAGESZ)))
+    ((msgmni=1024 * ram_GB))
+    msgmax=65536
+    msgmnb=${msgmax}
+    SEMMSL=250
+    SEMMNS=256000
+    SEMOPM=32
+    SEMMNI=${shmmni}
+
+    # RH bugzilla https://access.redhat.com/solutions/4968021. Limit SEMMNI, shmmni and msgmni to the max
+    # supported by the Linux kernel -- 32k (default) or 8M if kernel boot parameter 'ipcmni_extend' is set.
+    ((SEMMNI=SEMMNI < IPCMNI_LIMIT ? SEMMNI : IPCMNI_LIMIT))
+    ((shmmni=shmmni < IPCMNI_LIMIT ? shmmni : IPCMNI_LIMIT))
+    ((msgmni=msgmni < IPCMNI_LIMIT ? msgmni : IPCMNI_LIMIT))
+}
+
+# Generate NodeTuning Operator YAML file
+gen_tuned_cr_yaml() {
+    # Generate YAML file for NodeTuning CR and save as ${tuned_cr_yaml}
+    cat <<-EOF > ${tuned_cr_yaml}
+apiVersion: tuned.openshift.io/v1
+kind: Tuned
+metadata:
+  name: ${cr_name}
+  namespace: ${cr_namespace}
+spec:
+  profile:
+  - name: ${cr_profile_name}
+    data: |
+      [main]
+      summary=Tune IPC Kernel parameters on OpenShift nodes running Db2U engine PODs
+      include=openshift-node
+
+      [sysctl]
+      kernel.shmmni = ${shmmni}
+      kernel.shmmax = ${shmmax}
+      kernel.shmall = ${shmall}
+      kernel.sem = ${SEMMSL} ${SEMMNS} ${SEMOPM} ${SEMMNI}
+      kernel.msgmni = ${msgmni}
+      kernel.msgmax = ${msgmax}
+      kernel.msgmnb = ${msgmnb}
+
+  recommend:
+  - match:
+    - label: node-role.kubernetes.io/worker
+EOF
+
+    # Add the optional dedicated label into match array
+    if [[ -n "${node_label}" ]]; then
+        cat <<-EOF >> ${tuned_cr_yaml}
+    - label: icp4data
+      value: ${node_label}
+EOF
+    fi
+
+    # Add the priority and profile keys
+    cat <<-EOF >> ${tuned_cr_yaml}
+    priority: 10
+    profile: ${cr_profile_name}
+EOF
+
+    [[ "${create_cr}" == "true" ]] && return
+    cat <<-MSG
+===============================================================================
+* Successfully generated the Node Tuning Operator Custom Resource Definition as
+  ${tuned_cr_yaml} YAML with Db2U specific IPC sysctl settings.
+
+* Please run 'oc create -f ${tuned_cr_yaml}' on the master node to
+  create the Node Tuning Operator CR to apply those customized sysctl values.
+===============================================================================
+MSG
+}
+
+create_tuned_cr() {
+    echo "Creating the Node Tuning Operator Custom Resource for Db2U IPC kernel parameter tuning ..."
+    oc create -f ${tuned_cr_yaml}
+    sleep 2
+
+    # List the NodeTuning CR and describe
+    oc -n ${cr_namespace} get Tuned/${cr_name}
+    echo ""
+
+    echo "The CR of the Node Tuning Operator deployed"
+    echo "--------------------------------------------"
+    oc -n ${cr_namespace} describe Tuned/${cr_name}
+    echo ""
+}
+
+delete_tuned_cr() {
+    echo "Deleting the Node Tuning Operator Custom Resource used for Db2U IPC kernel parameter tuning ..."
+    oc -n ${cr_namespace} get Tuned/${cr_name} --no-headers -ojsonpath='{.kind}' | grep -iq tuned || \
+        { echo "No matching CR found ..." && exit 0; }
+    oc -n ${cr_namespace} delete Tuned/${cr_name}
+    echo ""
+    sleep 2
+
+    # Get the list of containerized tuned PODs (DaemonSet) deployed on the cluster
+    local tuned_pods=( $(oc -n ${cr_namespace} get po --selector openshift-app=tuned --no-headers -ojsonpath='{.items[*].metadata.name}') )
+    # Remove the tuned profile directory deployed on those PODs
+    for p in "${tuned_pods[@]}"; do
+        echo "Removing the installed tuned profile ${cr_profile_name} on POD: $p"
+        oc -n ${cr_namespace} exec -it $p -- bash -c "rm -fr /etc/tuned/${cr_profile_name}"
+    done
+    echo ""
+}
+
+#
+### ================== Main ==================== ###
+#
+
+[[ "${delete_cr}" == "true" ]] && { delete_tuned_cr && exit 0; }
+
+compute_kernel_ipc_params
+
+gen_tuned_cr_yaml
+
+[[ "${create_cr}" == "true" ]] && create_tuned_cr
+
+exit 0

automation-roles/50-install-cloud-pak/cp4d/cp4d-ocp-tuned/tasks/main.yml

@@ -1,14 +1,37 @@
 ---
-- name: Check if the Tuned {{ cp4d_tuned_name }} exists
-  shell: "oc get Tuned -n openshift-cluster-node-tuning-operator | grep -i '^{{ cp4d_tuned_name }}' | wc -l"
-  register: oc_tuned_exists
-
-- name: Configure Tuned {{ cp4d_tuned_name }}
-  block:
-    - name: Create Tuned yaml
-      template:
-        src: cp4d-tuned.j2
-        dest: "{{ status_dir }}/openshift/cp4d-tuned.yaml"
-    - name: Create Tuned cp4d_ipc
-      shell: "oc apply -f {{ status_dir }}/openshift/cp4d-tuned.yaml"
-  when: oc_tuned_exists.stdout == "0"
+- name: Get first compute node in the cluster
+  shell: |
+    oc get no \
+      -l node-role.kubernetes.io/worker,cluster.ocs.openshift.io/openshift-storage!="" \
+      -o jsonpath='{.items[0].metadata.name}'
+  register: _first_compute_node
+
+- name: Get memory capacity of first compute node {{ _first_compute_node.stdout }}
+  shell:
+    oc get no {{ _first_compute_node.stdout }} -o jsonpath='{.status.capacity.memory}'
+  register: _compute_node_memory_raw
+
+- set_fact:
+    _compute_node_memory: "{{ _compute_node_memory_raw.stdout | regex_search('([0-9]+).*','\\1') | first }}"
+    _memory_unit: "{{ _compute_node_memory_raw.stdout | regex_search('([0-9]+)(.*)','\\2') | first }}"
+
+- set_fact:
+    _compute_node_memory_gb: "{{ (_compute_node_memory | int / (1024*1024)) | int }}"
+  when: _memory_unit=='Ki'
+
+- set_fact:
+    _compute_node_memory_gb: "{{ (_compute_node_memory | int / 1024) | int }}"
+  when: _memory_unit=='Mi'
+
+- debug:
+    var: _compute_node_memory_gb
+
+- name: Generate Tuned resource YAML file {{ status_dir }}/openshift/cp4d-tuned.yaml
+  shell: |
+    {{ role_path }}/files/cp4d-generate-tuned.sh \
+      --mem-limit {{ _compute_node_memory_gb }} \
+      -f {{ status_dir }}/openshift/cp4d-tuned.yaml
+
+- name: Create Tuned cp4d_ipc
+  shell: |
+    oc apply -f {{ status_dir }}/openshift/cp4d-tuned.yaml