-
Notifications
You must be signed in to change notification settings - Fork 18
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Gdb 6524 implement cluster scalability (#40)
* Changed graphdb-utils-configmap update before any jobs * Changed graphdb-node's service to always be headless, fixing a bug with scaling up from 1 node * Cluster scale up, down, patch added * Added waits on proxy to be ready before executing curl commands against it. * Made scale up take precedent over patch and reimplemented patch job * Changed waitAllNodes in graphdb.sh to go from last to first node so it's in the direction of a rolling update * Updated CHANGELOG * Formatted graphdb.sh * Change chart and graphdb version to 10.0.1 * Updated documentation to use 10.0 instead of M3 links
- Loading branch information
Showing
12 changed files
with
362 additions
and
99 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| #!/usr/bin/env bash | ||
| set -eu | ||
|
|
||
| function patchCluster { | ||
| local configLocation=$1 | ||
| local authToken=$2 | ||
| echo "Patching cluster" | ||
| waitService "http://graphdb-cluster-proxy:7200/proxy/ready" "$authToken" | ||
| curl -o patchResponse.json -isSL -m 15 -X PATCH --header "Authorization: Basic ${authToken}" --header 'Content-Type: application/json' --header 'Accept: application/json' -d @"$configLocation" 'http://graphdb-cluster-proxy:7200/rest/cluster/config' | ||
| if grep -q 'HTTP/1.1 200' "patchResponse.json"; then | ||
| echo "Patch successful" | ||
| elif grep -q 'Cluster does not exist.\|HTTP/1.1 412' "patchResponse.json" ; then | ||
| echo "Cluster does not exist" | ||
| else | ||
| echo "Cluster patch failed, received response:" | ||
| cat patchResponse.json | ||
| echo | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| function removeNodes { | ||
| local expectedNodes=$1 | ||
| local authToken=$2 | ||
| local namespace=$3 | ||
| local currentNodes=$(getNodeCountInCurrentCluster "$authToken") | ||
| local nodes="" | ||
| echo "Cluster reported: $currentNodes current nodes" | ||
| echo "Cluster is expected to have: $expectedNodes nodes" | ||
| # if there is no cluster or current nodes are less or equal to expected so no need to remove more, exit | ||
| if [ "$currentNodes" -lt 2 ] || [ "$currentNodes" -le "$expectedNodes" ]; then | ||
| echo "No scaling down of the cluster required" | ||
| exit 0 | ||
| fi | ||
| # if there is a cluster and we wanna scale to 1 node, delete it (we would have exit on the last if in case on no cluster) | ||
| if [ "$expectedNodes" -lt 2 ]; then | ||
| echo "Scaling down to 1 node. Deleting cluster" | ||
| deleteCluster "$authToken" | ||
| exit 0 | ||
| fi | ||
| echo "Scaling the cluster down" | ||
| for ((i = expectedNodes; i < currentNodes; i++)) do | ||
| nodes=${nodes}\"graphdb-node-$i.graphdb-node.${namespace}.svc.cluster.local:7300\" | ||
| if [ $i -lt $(expr $currentNodes - 1) ]; then | ||
| nodes=${nodes}\, | ||
| fi | ||
| done | ||
| nodes=\{\"nodes\":\[${nodes}\]\} | ||
| waitService "http://graphdb-cluster-proxy:7200/proxy/ready" "$authToken" | ||
| curl -o clusterRemove.json -isSL -m 15 -X DELETE --header 'Content-Type: application/json' --header 'Accept: application/json' --header "Authorization: Basic ${authToken}" -d "${nodes}" 'http://graphdb-cluster-proxy:7200/rest/cluster/config/node' | ||
| if grep -q 'HTTP/1.1 200' "clusterRemove.json"; then | ||
| echo "Scaling down successful." | ||
| else | ||
| echo "Issue scaling down:" | ||
| cat clusterRemove.json | ||
| echo | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| function addNodes { | ||
| local expectedNodes=$1 | ||
| local authToken=$2 | ||
| local namespace=$3 | ||
| local timeout=$4 | ||
| local currentNodes=$(getNodeCountInCurrentCluster "$authToken") | ||
| local nodes="" | ||
| echo "Cluster reported: $currentNodes current nodes" | ||
| echo "Cluster is expected to have: $expectedNodes nodes" | ||
| # if there is no cluster or current nodes are more or equal to expected so no need to add more, exit | ||
| if [ "$currentNodes" -lt 2 ] || [ "$currentNodes" -ge "$expectedNodes" ]; then | ||
| echo "No scaling up of the cluster required" | ||
| exit 0 | ||
| fi | ||
| echo "Scaling the cluster up" | ||
| for ((i = currentNodes; i < expectedNodes; i++)) do | ||
| nodes=${nodes}\"graphdb-node-$i.graphdb-node.${namespace}.svc.cluster.local:7300\" | ||
| if [ $i -lt $(expr $expectedNodes - 1) ]; then | ||
| nodes=${nodes}\, | ||
| fi | ||
| done | ||
| nodes=\{\"nodes\":\[${nodes}\]\} | ||
| waitService "http://graphdb-cluster-proxy:7200/proxy/ready" "$authToken" | ||
| curl -o clusterAdd.json -isSL -m ${timeout} -X POST --header 'Content-Type: application/json' --header 'Accept: application/json' --header "Authorization: Basic ${authToken}" -d "${nodes}" 'http://graphdb-cluster-proxy:7200/rest/cluster/config/node' | ||
| if grep -q 'HTTP/1.1 200' "clusterAdd.json"; then | ||
| echo "Scaling successful." | ||
| elif grep -q 'Mismatching fingerprints\|HTTP/1.1 412' "clusterAdd.json"; then | ||
| echo "Issue scaling:" | ||
| cat clusterAdd.json | ||
| echo | ||
| echo "Manual clear of the mismatched repositories will be required to add the node" | ||
| exit 1 | ||
| else | ||
| echo "Issue scaling:" | ||
| cat clusterAdd.json | ||
| echo | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| function deleteCluster { | ||
| local authToken=$1 | ||
| waitService "http://graphdb-node-0.graphdb-node:7200/rest/repositories" "$authToken" | ||
| curl -o response.json -isSL -m 15 -X DELETE --header "Authorization: Basic ${authToken}" --header 'Accept: */*' 'http://graphdb-node-0.graphdb-node:7200/rest/cluster/config?force=false' | ||
| if grep -q 'HTTP/1.1 200' "response.json"; then | ||
| echo "Cluster deletion successful!" | ||
| elif grep -q 'Node is not part of the cluster.\|HTTP/1.1 412' "response.json" ; then | ||
| echo "No cluster present." | ||
| else | ||
| echo "Cluster deletion failed, received response:" | ||
| cat response.json | ||
| echo | ||
| exit 1 | ||
| fi | ||
| } | ||
|
|
||
| function getNodeCountInCurrentCluster { | ||
| local authToken=$1 | ||
| local node_address=http://graphdb-node-0.graphdb-node:7200 | ||
| waitService "${node_address}/rest/repositories" "$authToken" | ||
| curl -o clusterResponse.json -isSL -m 15 -X GET --header 'Content-Type: application/json' --header "Authorization: Basic ${authToken}" --header 'Accept: */*' "${node_address}/rest/cluster/config" | ||
| grep -o 'graphdb-node-' "clusterResponse.json" | grep -c "" | ||
| } | ||
|
|
||
| function waitService { | ||
| local address=$1 | ||
| local authToken=$2 | ||
|
|
||
| local attempt_counter=0 | ||
| local max_attempts=100 | ||
|
|
||
| until $(curl --output /dev/null -fsSL -m 5 -H "Authorization: Basic ${authToken}" --silent --fail ${address}); do | ||
| if [[ ${attempt_counter} -eq ${max_attempts} ]];then | ||
| echo "Max attempts reached" | ||
| exit 1 | ||
| fi | ||
| attempt_counter=$((attempt_counter+1)) | ||
| sleep 5 | ||
| done | ||
| } | ||
|
|
||
| "$@" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| {{- if gt (int .Values.graphdb.clusterConfig.nodesCount) 1 }} | ||
| {{- $authToken := printf "%s:%s" .Values.graphdb.security.provisioningUsername .Values.graphdb.security.provisioningPassword | b64enc }} | ||
| apiVersion: {{ $.Values.versions.job }} | ||
| kind: Job | ||
| metadata: | ||
| name: patch-cluster-job | ||
| labels: | ||
| {{- include "graphdb.labels" . | nindent 4 }} | ||
| annotations: | ||
| "helm.sh/hook": post-upgrade, post-rollback | ||
| "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed | ||
| "helm.sh/hook-weight": "2" | ||
| spec: | ||
| ttlSecondsAfterFinished: 300 | ||
| template: | ||
| spec: | ||
| imagePullSecrets: | ||
| {{- include "combinedImagePullSecrets" $ | nindent 8 }} | ||
| containers: | ||
| - name: patch-cluster | ||
| image: {{ include "renderFullImageName" (dict "globalRegistry" $.Values.global.imageRegistry "image" $.Values.images.graphdb) }} | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| runAsUser: 0 | ||
| volumeMounts: | ||
| - name: graphdb-utils | ||
| mountPath: /tmp/utils | ||
| - name: cluster-config | ||
| mountPath: /tmp/cluster-config | ||
| command: ['sh','-c'] | ||
| args: | ||
| - | | ||
| cp /tmp/cluster-config/cluster-config.json /usr/local/bin/cluster-config.json | ||
| cp /tmp/utils/update-cluster.sh /usr/local/bin/update-cluster.sh; chmod +x /usr/local/bin/update-cluster.sh | ||
| /usr/local/bin/update-cluster.sh patchCluster "/usr/local/bin/cluster-config.json" "{{ $authToken }}" >> /proc/1/fd/1 | ||
| restartPolicy: Never | ||
| volumes: | ||
| - name: cluster-config | ||
| configMap: | ||
| name: graphdb-cluster-config-configmap | ||
| - name: graphdb-utils | ||
| configMap: | ||
| name: graphdb-utils-configmap | ||
| backoffLimit: 4 | ||
| {{- end }} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| {{- $authToken := printf "%s:%s" .Values.graphdb.security.provisioningUsername .Values.graphdb.security.provisioningPassword | b64enc }} | ||
| apiVersion: {{ $.Values.versions.job }} | ||
| kind: Job | ||
| metadata: | ||
| name: scale-down-cluster-job | ||
| labels: | ||
| {{- include "graphdb.labels" . | nindent 4 }} | ||
| annotations: | ||
| "helm.sh/hook": pre-upgrade, pre-rollback | ||
| "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed | ||
| spec: | ||
| ttlSecondsAfterFinished: 300 | ||
| template: | ||
| spec: | ||
| imagePullSecrets: | ||
| {{- include "combinedImagePullSecrets" $ | nindent 8 }} | ||
| containers: | ||
| - name: scale-down-cluster | ||
| image: {{ include "renderFullImageName" (dict "globalRegistry" $.Values.global.imageRegistry "image" $.Values.images.graphdb) }} | ||
| securityContext: | ||
| allowPrivilegeEscalation: false | ||
| runAsUser: 0 | ||
| volumeMounts: | ||
| - name: graphdb-utils | ||
| mountPath: /tmp/utils | ||
| command: ['sh','-c'] | ||
| args: | ||
| - | | ||
| cp /tmp/utils/update-cluster.sh /usr/local/bin/update-cluster.sh; chmod +x /usr/local/bin/update-cluster.sh | ||
| /usr/local/bin/update-cluster.sh removeNodes {{ .Values.graphdb.clusterConfig.nodesCount }} "{{ $authToken }}" {{ $.Release.Namespace }} >> /proc/1/fd/1 | ||
| restartPolicy: Never | ||
| volumes: | ||
| - name: graphdb-utils | ||
| configMap: | ||
| name: graphdb-utils-configmap | ||
| backoffLimit: 4 | ||
|
|
Oops, something went wrong.