From 50e561d3b3e5ca125bf422560748e106a335d94c Mon Sep 17 00:00:00 2001
From: sonyafenge <45471193+sonyafenge@users.noreply.github.com>
Date: Thu, 13 Aug 2020 16:30:39 -0700
Subject: [PATCH] add apiserver data partition to kube-up nad start-kubemark
 (#577)

---
 cluster/gce/config-default.sh   | 11 +------
 cluster/gce/config-test.sh      | 11 +------
 cluster/gce/util.sh             | 56 +++++++++++++++++++++++++++++++--
 cluster/kube-up.sh              | 19 ++++++++++-
 test/kubemark/start-kubemark.sh | 18 +++++++++++
 5 files changed, 92 insertions(+), 23 deletions(-)

diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh
index bbd44acb535..21e471ce327 100755
--- a/cluster/gce/config-default.sh
+++ b/cluster/gce/config-default.sh
@@ -42,19 +42,10 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 NODE_LABELS="${KUBE_NODE_LABELS:-}"
 WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 APISERVERS_EXTRA_NUM=${APISERVERS_EXTRA_NUM:-0}
+APISERVER_DATAPARTITION_CONFIG=${APISERVER_DATAPARTITION_CONFIG:-"abcdefghijklmnopqrstuvwxyz"}
 WORKLOADCONTROLLER_EXTRA_NUM=${WORKLOADCONTROLLER_EXTRA_NUM:-0}
 #switch to enable/disable kube-controller-manager leader-elect: --leader-elect=true/false
 ENABLE_KCM_LEADER_ELECT=${ENABLE_KCM_LEADER_ELECT:-true}
-APISERVER_SIZE=${APISERVER_SIZE:-${MASTER_SIZE}}
-APISERVER_MIN_CPU_ARCHITECTURE=${APISERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-APISERVER_DISK_TYPE=${APISERVER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-APISERVER_DISK_SIZE=${APISERVER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-APISERVER_ROOT_DISK_SIZE=${APISERVER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
-WORKLOADCONTROLLER_SIZE=${WORKLOADCONTROLLER_SIZE:-${MASTER_SIZE}}
-WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE=${WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-WORKLOADCONTROLLER_DISK_TYPE=${WORKLOADCONTROLLER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-WORKLOADCONTROLLER_DISK_SIZE=${WORKLOADCONTROLLER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-WORKLOADCONTROLLER_ROOT_DISK_SIZE=${WORKLOADCONTROLLER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
 SHARE_PARTITIONSERVER=${SHARE_PARTITIONSERVER:-false}
 PARTITIONSERVER_SIZE=${PARTITIONSERVER_SIZE:-${MASTER_SIZE}}
 PARTITIONSERVER_MIN_CPU_ARCHITECTURE=${PARTITIONSERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh
index ac0ca870967..6f438240707 100755
--- a/cluster/gce/config-test.sh
+++ b/cluster/gce/config-test.sh
@@ -42,19 +42,10 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 NODE_LABELS="${KUBE_NODE_LABELS:-}"
 WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 APISERVERS_EXTRA_NUM=${APISERVERS_EXTRA_NUM:-0}
+APISERVER_DATAPARTITION_CONFIG=${APISERVER_DATAPARTITION_CONFIG:-"abcdefghijklmnopqrstuvwxyz"}
 WORKLOADCONTROLLER_EXTRA_NUM=${WORKLOADCONTROLLER_EXTRA_NUM:-0}
 #switch to enable/disable kube-controller-manager leader-elect: --leader-elect=true/false
 ENABLE_KCM_LEADER_ELECT=${ENABLE_KCM_LEADER_ELECT:-true}
-APISERVER_SIZE=${APISERVER_SIZE:-${MASTER_SIZE}}
-APISERVER_MIN_CPU_ARCHITECTURE=${APISERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-APISERVER_DISK_TYPE=${APISERVER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-APISERVER_DISK_SIZE=${APISERVER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-APISERVER_ROOT_DISK_SIZE=${APISERVER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
-WORKLOADCONTROLLER_SIZE=${WORKLOADCONTROLLER_SIZE:-${MASTER_SIZE}}
-WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE=${WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-WORKLOADCONTROLLER_DISK_TYPE=${WORKLOADCONTROLLER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-WORKLOADCONTROLLER_DISK_SIZE=${WORKLOADCONTROLLER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-WORKLOADCONTROLLER_ROOT_DISK_SIZE=${WORKLOADCONTROLLER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
 SHARE_PARTITIONSERVER=${SHARE_PARTITIONSERVER:-false}
 PARTITIONSERVER_SIZE=${PARTITIONSERVER_SIZE:-${MASTER_SIZE}}
 PARTITIONSERVER_MIN_CPU_ARCHITECTURE=${PARTITIONSERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
index 091c09f170c..0002dd9ffae 100755
--- a/cluster/gce/util.sh
+++ b/cluster/gce/util.sh
@@ -2895,6 +2895,59 @@ function config-partitionserver() {
 
 }
 
+
+##set apiserver datapartition config by apiserver total number
+function set-apiserver-datapartition() {
+  local service_groupid=${1:-0}
+  local total_server=$((APISERVERS_EXTRA_NUM+1))
+  local range_interval=$(( ${#APISERVER_DATAPARTITION_CONFIG}/total_server ))
+
+  if [[ "${service_groupid}" -eq "0" ]]; then
+    APISERVER_ISRANGESTART_VALID=false
+    APISERVER_ISRANGEEND_VALID=true
+    APISERVER_RANGESTART=${APISERVER_DATAPARTITION_CONFIG:0:1}
+    APISERVER_RANGEEND=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid+range_interval-1 )):1}
+  else 
+    if [[ "${service_groupid}" -eq "${APISERVERS_EXTRA_NUM}" ]]; then
+      APISERVER_ISRANGESTART_VALID=true
+      APISERVER_ISRANGEEND_VALID=false
+      APISERVER_RANGESTART=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid-1 )):1}
+      APISERVER_RANGEEND=${APISERVER_DATAPARTITION_CONFIG:$(( ${#APISERVER_DATAPARTITION_CONFIG}-1 )):1}
+    else
+      APISERVER_ISRANGESTART_VALID=true
+      APISERVER_ISRANGEEND_VALID=true
+      APISERVER_RANGESTART=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid-1 )):1}
+      APISERVER_RANGEEND=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid+range_interval-1 )):1}
+    fi
+  fi
+}
+
+function create-apiserver-datapartition-yml {
+  local service_groupid=${1:-0}
+  cat <<EOF >${KUBE_ROOT}/apiserverdatapartition/apidatapartition${service_groupid}.yaml
+apiVersion: v1
+kind: DataPartitionConfig
+serviceGroupId: "${service_groupid}"
+rangeStart: "${APISERVER_RANGESTART}"
+isRangeStartValid: ${APISERVER_ISRANGESTART_VALID}
+rangeEnd: "${APISERVER_RANGEEND}"
+isRangeEndValid: ${APISERVER_ISRANGEEND_VALID}
+metadata:
+  name: "partition-${service_groupid}"
+EOF
+}
+
+function config-apiserver-datapartition {
+  local service_groupid=${1:-0}
+  if [[ -f "${KUBE_ROOT}/apiserverdatapartition/apidatapartition${service_groupid}.yaml" ]]; then
+    sleep 5
+    ${KUBE_ROOT}/cluster/kubectl.sh apply -f "${KUBE_ROOT}/apiserverdatapartition/apidatapartition${service_groupid}.yaml"
+  else 
+    echo "failed to config apiserver datapartition, cannot find required yaml file"
+    exit 1
+  fi
+}
+
 # Adds master replica to etcd cluster.
 #
 # Assumed vars:
@@ -3965,11 +4018,10 @@ function delete-partitionserver() {
   fi
 
   echo "deleting partitionserver firewall: ${server_name}-https"
-  if gcloud compute firewall-rules describe "${server_name}-https" --network "${NETWORK}" --project "${NETWORK_PROJECT}" &>/dev/null; then
+  if gcloud compute firewall-rules describe "${server_name}-https" --project "${NETWORK_PROJECT}" &>/dev/null; then
     gcloud compute firewall-rules delete  \
       --project "${NETWORK_PROJECT}" \
       --quiet \
-      --network "${NETWORK}" \
       "${server_name}-https"
   fi
 
diff --git a/cluster/kube-up.sh b/cluster/kube-up.sh
index 5c4cbc27530..cd06f5b5811 100755
--- a/cluster/kube-up.sh
+++ b/cluster/kube-up.sh
@@ -78,7 +78,24 @@ if [[ "${ENABLE_PROXY:-}" == "true" ]]; then
   echo ""
 fi
 
-echo -e "Done, listing cluster services:\n" >&2
+if [[ "${APISERVERS_EXTRA_NUM:-0}" -gt "0" ]]; then
+  echo "... configing apiserver datapartition" >&2
+  if [[ -d "${KUBE_ROOT}/apiserverdatapartition" ]]; then
+    rm -r ${KUBE_ROOT}/apiserverdatapartition
+  fi
+  mkdir ${KUBE_ROOT}/apiserverdatapartition
+  for (( num=0; num<=${APISERVERS_EXTRA_NUM:-0}; num++ )); do
+    APISERVER_RANGESTART=${APISERVER_RANGESTART:-"${APISERVER_DATAPARTITION_CONFIG:0:1}"}
+    APISERVER_RANGEEND=${APISERVER_RANGEEND:-"${APISERVER_DATAPARTITION_CONFIG:$(( ${#APISERVER_DATAPARTITION_CONFIG}-1 )):1}"}
+    APISERVER_ISRANGESTART_VALID=${APISERVER_ISRANGESTART_VALID:-false}
+    APISERVER_ISRANGEEND_VALID=${APISERVER_ISRANGEEND_VALID:-false}
+    set-apiserver-datapartition $num
+    create-apiserver-datapartition-yml $num
+    config-apiserver-datapartition $num
+  done
+fi
+
+echo -e "\nDone, listing cluster services:\n" >&2
 "${KUBE_ROOT}/cluster/kubectl.sh" cluster-info
 echo
 
diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh
index 18005590240..876231878d1 100755
--- a/test/kubemark/start-kubemark.sh
+++ b/test/kubemark/start-kubemark.sh
@@ -236,3 +236,21 @@ else
   echo "Master IP: ${MASTER_IP}"
 fi
 echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
+
+sleep 5
+echo -e "\nListing kubeamrk cluster details:" >&2
+echo -e "Getting total nodes number:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node | wc -l
+echo
+echo -e "Getting total hollow-nodes number:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node | grep "hollow-node" | wc -l
+echo
+echo -e "Getting endpoints status:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get endpoints -A
+echo
+echo -e "Getting workload controller co status:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get co
+echo
+echo -e "Getting apiserver data partition status:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get datapartition
+echo