add apiserver data partition to kube-up nad start-kubemark (Centaurus…

…Infra#577)
yb01 · Aug 13, 2020 · 50e561d · 50e561d
1 parent 79d954c
commit 50e561d
Show file tree

Hide file tree

Showing 5 changed files with 92 additions and 23 deletions.
diff --git a/cluster/gce/config-default.sh b/cluster/gce/config-default.sh
@@ -42,19 +42,10 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 NODE_LABELS="${KUBE_NODE_LABELS:-}"
 WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 APISERVERS_EXTRA_NUM=${APISERVERS_EXTRA_NUM:-0}
+APISERVER_DATAPARTITION_CONFIG=${APISERVER_DATAPARTITION_CONFIG:-"abcdefghijklmnopqrstuvwxyz"}
 WORKLOADCONTROLLER_EXTRA_NUM=${WORKLOADCONTROLLER_EXTRA_NUM:-0}
 #switch to enable/disable kube-controller-manager leader-elect: --leader-elect=true/false
 ENABLE_KCM_LEADER_ELECT=${ENABLE_KCM_LEADER_ELECT:-true}
-APISERVER_SIZE=${APISERVER_SIZE:-${MASTER_SIZE}}
-APISERVER_MIN_CPU_ARCHITECTURE=${APISERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-APISERVER_DISK_TYPE=${APISERVER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-APISERVER_DISK_SIZE=${APISERVER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-APISERVER_ROOT_DISK_SIZE=${APISERVER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
-WORKLOADCONTROLLER_SIZE=${WORKLOADCONTROLLER_SIZE:-${MASTER_SIZE}}
-WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE=${WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-WORKLOADCONTROLLER_DISK_TYPE=${WORKLOADCONTROLLER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-WORKLOADCONTROLLER_DISK_SIZE=${WORKLOADCONTROLLER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-WORKLOADCONTROLLER_ROOT_DISK_SIZE=${WORKLOADCONTROLLER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
 SHARE_PARTITIONSERVER=${SHARE_PARTITIONSERVER:-false}
 PARTITIONSERVER_SIZE=${PARTITIONSERVER_SIZE:-${MASTER_SIZE}}
 PARTITIONSERVER_MIN_CPU_ARCHITECTURE=${PARTITIONSERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.

diff --git a/cluster/gce/config-test.sh b/cluster/gce/config-test.sh
@@ -42,19 +42,10 @@ NODE_LOCAL_SSDS=${NODE_LOCAL_SSDS:-0}
 NODE_LABELS="${KUBE_NODE_LABELS:-}"
 WINDOWS_NODE_LABELS="${WINDOWS_NODE_LABELS:-}"
 APISERVERS_EXTRA_NUM=${APISERVERS_EXTRA_NUM:-0}
+APISERVER_DATAPARTITION_CONFIG=${APISERVER_DATAPARTITION_CONFIG:-"abcdefghijklmnopqrstuvwxyz"}
 WORKLOADCONTROLLER_EXTRA_NUM=${WORKLOADCONTROLLER_EXTRA_NUM:-0}
 #switch to enable/disable kube-controller-manager leader-elect: --leader-elect=true/false
 ENABLE_KCM_LEADER_ELECT=${ENABLE_KCM_LEADER_ELECT:-true}
-APISERVER_SIZE=${APISERVER_SIZE:-${MASTER_SIZE}}
-APISERVER_MIN_CPU_ARCHITECTURE=${APISERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-APISERVER_DISK_TYPE=${APISERVER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-APISERVER_DISK_SIZE=${APISERVER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-APISERVER_ROOT_DISK_SIZE=${APISERVER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
-WORKLOADCONTROLLER_SIZE=${WORKLOADCONTROLLER_SIZE:-${MASTER_SIZE}}
-WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE=${WORKLOADCONTROLLER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.
-WORKLOADCONTROLLER_DISK_TYPE=${WORKLOADCONTROLLER_DISK_TYPE:-${MASTER_DISK_TYPE}}
-WORKLOADCONTROLLER_DISK_SIZE=${WORKLOADCONTROLLER_DISK_SIZE:-${MASTER_DISK_SIZE}}
-WORKLOADCONTROLLER_ROOT_DISK_SIZE=${WORKLOADCONTROLLER_ROOT_DISK_SIZE:-${MASTER_ROOT_DISK_SIZE}}
 SHARE_PARTITIONSERVER=${SHARE_PARTITIONSERVER:-false}
 PARTITIONSERVER_SIZE=${PARTITIONSERVER_SIZE:-${MASTER_SIZE}}
 PARTITIONSERVER_MIN_CPU_ARCHITECTURE=${PARTITIONSERVER_MIN_CPU_ARCHITECTURE:-${MASTER_MIN_CPU_ARCHITECTURE:-}} # To allow choosing better architectures.

diff --git a/cluster/gce/util.sh b/cluster/gce/util.sh
@@ -2895,6 +2895,59 @@ function config-partitionserver() {
 
 }
 
+
+##set apiserver datapartition config by apiserver total number
+function set-apiserver-datapartition() {
+  local service_groupid=${1:-0}
+  local total_server=$((APISERVERS_EXTRA_NUM+1))
+  local range_interval=$(( ${#APISERVER_DATAPARTITION_CONFIG}/total_server ))
+
+  if [[ "${service_groupid}" -eq "0" ]]; then
+    APISERVER_ISRANGESTART_VALID=false
+    APISERVER_ISRANGEEND_VALID=true
+    APISERVER_RANGESTART=${APISERVER_DATAPARTITION_CONFIG:0:1}
+    APISERVER_RANGEEND=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid+range_interval-1 )):1}
+  else 
+    if [[ "${service_groupid}" -eq "${APISERVERS_EXTRA_NUM}" ]]; then
+      APISERVER_ISRANGESTART_VALID=true
+      APISERVER_ISRANGEEND_VALID=false
+      APISERVER_RANGESTART=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid-1 )):1}
+      APISERVER_RANGEEND=${APISERVER_DATAPARTITION_CONFIG:$(( ${#APISERVER_DATAPARTITION_CONFIG}-1 )):1}
+    else
+      APISERVER_ISRANGESTART_VALID=true
+      APISERVER_ISRANGEEND_VALID=true
+      APISERVER_RANGESTART=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid-1 )):1}
+      APISERVER_RANGEEND=${APISERVER_DATAPARTITION_CONFIG:$(( range_interval*service_groupid+range_interval-1 )):1}
+    fi
+  fi
+}
+
+function create-apiserver-datapartition-yml {
+  local service_groupid=${1:-0}
+  cat <<EOF >${KUBE_ROOT}/apiserverdatapartition/apidatapartition${service_groupid}.yaml
+apiVersion: v1
+kind: DataPartitionConfig
+serviceGroupId: "${service_groupid}"
+rangeStart: "${APISERVER_RANGESTART}"
+isRangeStartValid: ${APISERVER_ISRANGESTART_VALID}
+rangeEnd: "${APISERVER_RANGEEND}"
+isRangeEndValid: ${APISERVER_ISRANGEEND_VALID}
+metadata:
+  name: "partition-${service_groupid}"
+EOF
+}
+
+function config-apiserver-datapartition {
+  local service_groupid=${1:-0}
+  if [[ -f "${KUBE_ROOT}/apiserverdatapartition/apidatapartition${service_groupid}.yaml" ]]; then
+    sleep 5
+    ${KUBE_ROOT}/cluster/kubectl.sh apply -f "${KUBE_ROOT}/apiserverdatapartition/apidatapartition${service_groupid}.yaml"
+  else 
+    echo "failed to config apiserver datapartition, cannot find required yaml file"
+    exit 1
+  fi
+}
+
 # Adds master replica to etcd cluster.
 #
 # Assumed vars:
@@ -3965,11 +4018,10 @@ function delete-partitionserver() {
   fi
 
   echo "deleting partitionserver firewall: ${server_name}-https"
-  if gcloud compute firewall-rules describe "${server_name}-https" --network "${NETWORK}" --project "${NETWORK_PROJECT}" &>/dev/null; then
+  if gcloud compute firewall-rules describe "${server_name}-https" --project "${NETWORK_PROJECT}" &>/dev/null; then
     gcloud compute firewall-rules delete  \
       --project "${NETWORK_PROJECT}" \
       --quiet \
-      --network "${NETWORK}" \
       "${server_name}-https"
   fi
 

diff --git a/cluster/kube-up.sh b/cluster/kube-up.sh
@@ -78,7 +78,24 @@ if [[ "${ENABLE_PROXY:-}" == "true" ]]; then
   echo ""
 fi
 
-echo -e "Done, listing cluster services:\n" >&2
+if [[ "${APISERVERS_EXTRA_NUM:-0}" -gt "0" ]]; then
+  echo "... configing apiserver datapartition" >&2
+  if [[ -d "${KUBE_ROOT}/apiserverdatapartition" ]]; then
+    rm -r ${KUBE_ROOT}/apiserverdatapartition
+  fi
+  mkdir ${KUBE_ROOT}/apiserverdatapartition
+  for (( num=0; num<=${APISERVERS_EXTRA_NUM:-0}; num++ )); do
+    APISERVER_RANGESTART=${APISERVER_RANGESTART:-"${APISERVER_DATAPARTITION_CONFIG:0:1}"}
+    APISERVER_RANGEEND=${APISERVER_RANGEEND:-"${APISERVER_DATAPARTITION_CONFIG:$(( ${#APISERVER_DATAPARTITION_CONFIG}-1 )):1}"}
+    APISERVER_ISRANGESTART_VALID=${APISERVER_ISRANGESTART_VALID:-false}
+    APISERVER_ISRANGEEND_VALID=${APISERVER_ISRANGEEND_VALID:-false}
+    set-apiserver-datapartition $num
+    create-apiserver-datapartition-yml $num
+    config-apiserver-datapartition $num
+  done
+fi
+
+echo -e "\nDone, listing cluster services:\n" >&2
 "${KUBE_ROOT}/cluster/kubectl.sh" cluster-info
 echo
 

diff --git a/test/kubemark/start-kubemark.sh b/test/kubemark/start-kubemark.sh
@@ -236,3 +236,21 @@ else
   echo "Master IP: ${MASTER_IP}"
 fi
 echo "Kubeconfig for kubemark master is written in ${LOCAL_KUBECONFIG}"
+
+sleep 5
+echo -e "\nListing kubeamrk cluster details:" >&2
+echo -e "Getting total nodes number:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node | wc -l
+echo
+echo -e "Getting total hollow-nodes number:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get node | grep "hollow-node" | wc -l
+echo
+echo -e "Getting endpoints status:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get endpoints -A
+echo
+echo -e "Getting workload controller co status:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get co
+echo
+echo -e "Getting apiserver data partition status:" >&2
+"${KUBECTL}" --kubeconfig="${LOCAL_KUBECONFIG}" get datapartition
+echo