Add more tests to cover contaerind/driver container k8s deployments

krepsjan · Mar 24, 2022 · d30b231 · d30b231
1 parent e6baed1
commit d30b231
Show file tree

Hide file tree

Showing 3 changed files with 310 additions and 16 deletions.
diff --git a/virtual/scripts/setup_k8s.sh b/virtual/scripts/setup_k8s.sh
@@ -18,9 +18,12 @@ K8S_CONFIG_DIR="${VIRT_DIR}/config"
 
 ansible_extra_args=""
 if [ ${DEEPOPS_K8S_OPERATOR_EXISTING_SOFTWARE} ]; then
-	ansible_extra_args="${ansible_extra_args} -e deepops_gpu_operator_enabled=true -e gpu_operator_preinstalled_nvidia_software=true"
+	ansible_extra_args="${ansible_extra_args} -e deepops_gpu_operator_enabled=true -e gpu_operator_preinstalled_nvidia_software=${DEEPOPS_K8S_OPERATOR_EXISTING_SOFTWARE}"
 elif [ ${DEEPOPS_K8S_OPERATOR} ]; then
-	ansible_extra_args="${ansible_extra_args} -e deepops_gpu_operator_enabled=true"
+	ansible_extra_args="${ansible_extra_args} -e deepops_gpu_operator_enabled=${DEEPOPS_K8S_OPERATOR}"
+fi
+if [ ${DEEPOPS_K8S_CONTAINER_MANAGER} ]; then
+	ansible_extra_args="${ansible_extra_args} -e container_manager=${DEEPOPS_K8S_CONTAINER_MANAGER}"
 fi
 
 # Deploy the K8s cluster

diff --git a/workloads/jenkins/Jenkinsfile-multi-nightly b/workloads/jenkins/Jenkinsfile-multi-nightly
@@ -2,7 +2,7 @@ pipeline {
   agent any
   environment {
     DEEPOPS_NIGHTLY = 'true'
-    // DEEPOPS_FULL_INSTALL = 'true'
+    DEEPOPS_FULL_INSTALL = 'true'
     // DEEPOPS_VAGRANT_OS = 'ubuntu'
     // DEEPOPS_OS_VERSION = '18.04'
   }
@@ -19,7 +19,6 @@ pipeline {
       }
       steps {
         // TODO: ideally lock should work with declared stages
-        // TODO: determine how to pass a variable into quantity and merge this with the nightly jenkinsfile
         lock(resource: null, label: 'gpu', quantity: 2, variable: 'GPUDATA') {
           echo "Reset repo and unmunge files"
           sh '''
@@ -43,8 +42,10 @@ pipeline {
             bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
           '''
 
-          echo "Cluster Up - MGMT Nodes"
+          echo "Cluster Up - MGMT Nodes - device plugin + docker"
           sh '''
+            export DEEPOPS_K8S_OPERATOR=false
+            export DEEPOPS_K8S_CONTAINER_MANAGER=docker
             bash -x ./workloads/jenkins/scripts/test-cluster-up.sh
           '''
 
@@ -80,7 +81,7 @@ pipeline {
 
           echo "Test Monitoring installation"
           sh '''
-             timeout 2000 bash -x ./workloads/jenkins/scripts/test-monitoring.sh
+             timeout 1200 bash -x ./workloads/jenkins/scripts/test-monitoring.sh
           '''
 
           echo "Test Dashboard installation"
@@ -93,14 +94,172 @@ pipeline {
              timeout 1500 bash -x ./workloads/jenkins/scripts/test-kubeflow-pipeline.sh
           '''
 
-          echo "Start new virtual environment pre-GPU Operator checks"
+          echo "Start new virtual environment"
           sh '''
             bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
           '''
 
-          echo "Cluster Up - MGMT Nodes"
+          echo "Cluster Up - MGMT Nodes gpu operator + docker"
           sh '''
             export DEEPOPS_K8S_OPERATOR=true
+            export DEEPOPS_K8S_CONTAINER_MANAGER=docker
+            bash -x ./workloads/jenkins/scripts/test-cluster-up.sh
+          '''
+
+          echo "Get K8S Cluster Status"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            bash -x ./workloads/jenkins/scripts/get-k8s-debug.sh
+          '''
+
+          echo "Verify we can run a GPU job"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            timeout 500 bash -x ./workloads/jenkins/scripts/run-gpu-job.sh
+          '''
+
+          echo "Verify ingress config"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/verify-ingress-config.sh
+          '''
+
+          echo "Verify local docker registry"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/test-local-registry.sh
+          '''
+
+          echo "Verify rsyslog forwarding is working for the k8s cluster"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/test-rsyslog-k8s.sh
+          '''
+
+          echo "Test Monitoring installation"
+          sh '''
+             timeout 1200 bash -x ./workloads/jenkins/scripts/test-monitoring.sh
+          '''
+
+          echo "Test Dashboard installation"
+          sh '''
+             timeout 180 bash -x ./workloads/jenkins/scripts/test-dashboard.sh
+          '''
+
+          echo "Start new virtual environment"
+          sh '''
+            bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
+          '''
+
+          echo "Cluster Up - MGMT Nodes gpu operator + containerd"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            export DEEPOPS_K8S_CONTAINER_MANAGER=containerd
+            bash -x ./workloads/jenkins/scripts/test-cluster-up.sh
+          '''
+
+          echo "Get K8S Cluster Status"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            bash -x ./workloads/jenkins/scripts/get-k8s-debug.sh
+          '''
+
+          echo "Verify we can run a GPU job"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            timeout 500 bash -x ./workloads/jenkins/scripts/run-gpu-job.sh
+          '''
+
+          echo "Verify ingress config"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/verify-ingress-config.sh
+          '''
+
+          echo "Verify local docker registry"
+          sh '''
+             echo "unsupported configuration" # TODO bash -x ./workloads/jenkins/scripts/test-local-registry.sh
+          '''
+
+          echo "Verify rsyslog forwarding is working for the k8s cluster"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/test-rsyslog-k8s.sh
+          '''
+
+          echo "Test Kubeflow installation"
+          sh '''
+             timeout 4000 bash -x ./workloads/jenkins/scripts/test-kubeflow.sh
+          '''
+
+          echo "Test Monitoring installation"
+          sh '''
+             timeout 1200 bash -x ./workloads/jenkins/scripts/test-monitoring.sh
+          '''
+
+          echo "Test Dashboard installation"
+          sh '''
+             timeout 180 bash -x ./workloads/jenkins/scripts/test-dashboard.sh
+          '''
+
+          echo "Test Kubeflow pipeline"
+          sh '''
+             timeout 1500 bash -x ./workloads/jenkins/scripts/test-kubeflow-pipeline.sh
+          '''
+
+          echo "Start new virtual environment pre-GPU Operator with existing software checks"
+          sh '''
+            bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
+          '''
+
+          echo "Cluster Up - MGMT Nodes gpu operator + docker + drivers"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR_EXISTING_SOFTWARE=true
+            export DEEPOPS_K8S_CONTAINER_MANAGER=docker
+            bash -x ./workloads/jenkins/scripts/test-cluster-up.sh
+          '''
+
+          echo "Get K8S Cluster Status"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            bash -x ./workloads/jenkins/scripts/get-k8s-debug.sh
+          '''
+
+          echo "Verify we can run a GPU job"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR=true
+            timeout 500 bash -x ./workloads/jenkins/scripts/run-gpu-job.sh
+          '''
+
+          echo "Verify ingress config"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/verify-ingress-config.sh
+          '''
+
+          echo "Verify local docker registry"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/test-local-registry.sh
+          '''
+
+          echo "Verify rsyslog forwarding is working for the k8s cluster"
+          sh '''
+             bash -x ./workloads/jenkins/scripts/test-rsyslog-k8s.sh
+          '''
+
+          echo "Test Monitoring installation"
+          sh '''
+             timeout 1200 bash -x ./workloads/jenkins/scripts/test-monitoring.sh
+          '''
+
+          echo "Test Dashboard installation"
+          sh '''
+             timeout 180 bash -x ./workloads/jenkins/scripts/test-dashboard.sh
+          '''
+
+          echo "Start new virtual environment"
+          sh '''
+            bash -x ./workloads/jenkins/scripts/vagrant-startup.sh
+          '''
+
+          echo "Cluster Up - MGMT Nodes gpu operator + containerd + drivers"
+          sh '''
+            export DEEPOPS_K8S_OPERATOR_EXISTING_SOFTWARE=true
+            export DEEPOPS_K8S_CONTAINER_MANAGER=containerd
             bash -x ./workloads/jenkins/scripts/test-cluster-up.sh
           '''
 
@@ -163,7 +322,7 @@ pipeline {
 
           echo "Test DCGM metrics"
           sh '''
-             timeout 600 bash -x ./workloads/jenkins/scripts/test-dcgm-metrics.sh slurm-node
+             timeout 500 bash -x ./workloads/jenkins/scripts/test-dcgm-metrics.sh slurm-node
           '''
 
           echo "Reset repo and unmunge files"