From 5061eee768258accaf1e579fbafa263c87e2bbb4 Mon Sep 17 00:00:00 2001 From: Andrew Jefferson <8148776+eastlondoner@users.noreply.github.com> Date: Wed, 7 Jul 2021 14:28:09 +0200 Subject: [PATCH] bump version, add support for sidecar containers, remove the docker-pv helper chart, add bolt keep-alive config (#54) * bump version, add support for sidecar containers, remove the docker-pv helper chart, add bolt keep-alive config * sidecar container helm template test * fixing resource handling for tests. Added more tests * reduce the cpu request for testing --- bin/gcloud-create-gke-cluster | 2 +- build/package | 2 +- build/package-no-sign | 2 +- internal/deploy.go | 8 +++++ internal/helm_template_test.go | 35 +++++++++++++++++++ ...odInitContainerAndCustomInitContainer.yaml | 5 ++- neo4j-docker-desktop-pv/Chart.yaml | 4 +-- neo4j-gcloud-pv/Chart.yaml | 4 +-- neo4j-shared-templates/Chart.yaml | 4 +-- neo4j-standalone/Chart.yaml | 4 +-- neo4j-standalone/templates/neo4j-config.yaml | 6 ++++ .../templates/neo4j-statefulset.yaml | 23 ++++++++++-- neo4j-standalone/values.yaml | 3 ++ 13 files changed, 88 insertions(+), 14 deletions(-) diff --git a/bin/gcloud-create-gke-cluster b/bin/gcloud-create-gke-cluster index 1aa2f62b..b1b1e664 100755 --- a/bin/gcloud-create-gke-cluster +++ b/bin/gcloud-create-gke-cluster @@ -24,7 +24,7 @@ RELEASE_CHANNEL="stable" gcloud container clusters create "${CLOUDSDK_CONTAINER_CLUSTER}" \ --release-channel=${RELEASE_CHANNEL} \ --zone="${CLOUDSDK_COMPUTE_ZONE}" \ - --num-nodes "1" \ + --num-nodes "2" \ --workload-pool="${CLOUDSDK_CORE_PROJECT}.svc.id.goog" \ --preemptible --machine-type "${NODE_MACHINE}" --image-type "COS_CONTAINERD" \ --disk-type "pd-ssd" --disk-size "10" \ diff --git a/build/package b/build/package index f4b46f46..ba7d8222 100755 --- a/build/package +++ b/build/package @@ -24,7 +24,7 @@ PACKAGE_SIGNING_PASSPHRASE="${PACKAGE_SIGNING_PASSPHRASE:?PACKAGE_SIGNING_PASSPH # create the package! echo $PACKAGE_SIGNING_PASSPHRASE | helm package --sign ./neo4j-standalone --key "${PACKAGE_SIGNING_KEY}" --keyring="${PACKAGE_SIGNING_KEYRING}" --passphrase-file - -echo $PACKAGE_SIGNING_PASSPHRASE | helm package --sign ./neo4j-docker-desktop-pv --key "${PACKAGE_SIGNING_KEY}" --keyring="${PACKAGE_SIGNING_KEYRING}" --passphrase-file - +# echo $PACKAGE_SIGNING_PASSPHRASE | helm package --sign ./neo4j-docker-desktop-pv --key "${PACKAGE_SIGNING_KEY}" --keyring="${PACKAGE_SIGNING_KEYRING}" --passphrase-file - echo $PACKAGE_SIGNING_PASSPHRASE | helm package --sign ./neo4j-gcloud-pv --key "${PACKAGE_SIGNING_KEY}" --keyring="${PACKAGE_SIGNING_KEYRING}" --passphrase-file - echo "Copying packaged files to ${BUILD_OUT_DIR}" mkdir -p ${BUILD_OUT_DIR} diff --git a/build/package-no-sign b/build/package-no-sign index fbf20717..3bef2c3e 100755 --- a/build/package-no-sign +++ b/build/package-no-sign @@ -15,7 +15,7 @@ S3_UPLOAD_TO_SUB_FOLDER="${S3_SUB_FOLDER:-neo4j}" # Create packages! # helm package ./neo4j-standalone -helm package ./neo4j-docker-desktop-pv +#helm package ./neo4j-docker-desktop-pv helm package ./neo4j-gcloud-pv # Copy packages diff --git a/internal/deploy.go b/internal/deploy.go index 2c6355f6..e414caa6 100644 --- a/internal/deploy.go +++ b/internal/deploy.go @@ -36,6 +36,10 @@ var ( ) const storageSize = "10Gi" +const cpuRequests = "50m" +const memoryRequests = "900Mi" +const cpuLimits = "1500m" +const memoryLimits = "900Mi" // This changes the working directory to the parent directory if the current working directory doesn't contain a directory called "yaml" func init() { @@ -175,6 +179,10 @@ func baseHelmCommand(helmCommand string, releaseName *ReleaseName, extraHelmArgu "--create-namespace", "--set", "volumes.data.selector.requests.storage="+storageSize, "--set", "neo4j.password="+defaultPassword, + "--set", "neo4j.resources.requests.cpu=" + cpuRequests, + "--set", "neo4j.resources.requests.memory=" + memoryRequests, + "--set", "neo4j.resources.limits.cpu=" + cpuLimits, + "--set", "neo4j.resources.limits.memory=" + memoryLimits, "--set", "ssl.bolt.privateKey.secretName=bolt-key", "--set", "ssl.bolt.publicCertificate.secretName=bolt-cert", "--set", "ssl.bolt.trustedCerts.sources[0].secret.name=bolt-cert", "--set", "ssl.https.privateKey.secretName=https-key", "--set", "ssl.https.publicCertificate.secretName=https-cert", diff --git a/internal/helm_template_test.go b/internal/helm_template_test.go index 0f0e5941..4a5db6e1 100644 --- a/internal/helm_template_test.go +++ b/internal/helm_template_test.go @@ -72,6 +72,38 @@ func TestEnterpriseDoesNotThrowErrorIfLicenseAgreementAccepted(t *testing.T) { } } +// This test is just to check that the produced helm chart doesn't throw any errors +func TestEnterpriseDoesNotThrowIfSet(t *testing.T) { + t.Parallel() + + baseSettings := append(useEnterprise, acceptLicenseAgreement...) + testCases := [][]string{ + baseSettings, + append(baseSettings, "--set", "neo4j.resources.requests.cpu=100m"), + append(baseSettings, "-f", "internal/resources/apocCorePlugin.yaml"), + append(baseSettings, "-f", "internal/resources/csvMetrics.yaml"), + append(baseSettings, "-f", "internal/resources/defaultStorageClass.yaml"), + append(baseSettings, "-f", "internal/resources/jvmAdditionalSettings.yaml"), + append(baseSettings, "-f", "internal/resources/pluginsInitContainer.yaml"), + } + + doTestCase := func(t *testing.T, testCase []string) { + t.Parallel() + manifest, err := helmTemplate(t, testCase...) + if !assert.NoError(t, err) { + return + } + + checkNeo4jManifest(t, manifest) + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + doTestCase(t, testCase) + }) + } +} + // Tests the "default" behaviour that you get if you don't pass in *any* other values and the helm chart defaults are used func TestDefaultEnterpriseHelmTemplate(t *testing.T) { t.Parallel() @@ -105,8 +137,11 @@ func TestDefaultCommunityHelmTemplate(t *testing.T) { neo4jStatefulSet := manifest.first(&appsv1.StatefulSet{}).(*appsv1.StatefulSet) neo4jStatefulSet.GetName() + assert.NotEmpty(t, neo4jStatefulSet.Spec.Template.Spec.Containers) for _, container := range neo4jStatefulSet.Spec.Template.Spec.Containers { assert.NotContains(t, container.Image, "enterprise") + assert.Equal(t, "1", container.Resources.Requests.Cpu().String()) + assert.Equal(t, "2Gi", container.Resources.Requests.Memory().String()) } for _, container := range neo4jStatefulSet.Spec.Template.Spec.InitContainers { assert.NotContains(t, container.Image, "enterprise") diff --git a/internal/resources/chmodInitContainerAndCustomInitContainer.yaml b/internal/resources/chmodInitContainerAndCustomInitContainer.yaml index 04c8a619..cfe806df 100644 --- a/internal/resources/chmodInitContainerAndCustomInitContainer.yaml +++ b/internal/resources/chmodInitContainerAndCustomInitContainer.yaml @@ -9,5 +9,8 @@ volumes: podSpec: initContainers: - name: init-printenv - image: neo4j:4.3.1 command: ['sh', '-c', "printenv"] + + containers: + - name: maintenance-sidecar + command: ['bash', '-c', "while true; do sleep 120; done"] \ No newline at end of file diff --git a/neo4j-docker-desktop-pv/Chart.yaml b/neo4j-docker-desktop-pv/Chart.yaml index 95bcc09c..7c91e437 100644 --- a/neo4j-docker-desktop-pv/Chart.yaml +++ b/neo4j-docker-desktop-pv/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: neo4j-docker-desktop-pv home: https://www.neo4j.com -version: 4.3.1 -appVersion: 4.3.1 +version: 4.3.2 +appVersion: 4.3.2 description: Sets up persistent disks suitable for simple development tasks with Neo4j Helm when using Kubernetes provided by Docker Desktop keywords: - graph diff --git a/neo4j-gcloud-pv/Chart.yaml b/neo4j-gcloud-pv/Chart.yaml index b46b9a6c..94121559 100644 --- a/neo4j-gcloud-pv/Chart.yaml +++ b/neo4j-gcloud-pv/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: neo4j-gcloud-pv home: https://www.neo4j.com -version: 4.3.1 -appVersion: 4.3.1 +version: 4.3.2 +appVersion: 4.3.2 description: Sets up persistent disks suitable for simple development tasks with Neo4j Helm when using Kubernetes provided Google Kubernetes Engine keywords: - graph diff --git a/neo4j-shared-templates/Chart.yaml b/neo4j-shared-templates/Chart.yaml index e2734ee8..94ffa7b9 100644 --- a/neo4j-shared-templates/Chart.yaml +++ b/neo4j-shared-templates/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: neo4j-shared-templates home: https://www.neo4j.com -version: 4.3.1 -appVersion: 4.3.1 +version: 4.3.2 +appVersion: 4.3.2 description: Neo4j is the world's leading graph database keywords: - graph diff --git a/neo4j-standalone/Chart.yaml b/neo4j-standalone/Chart.yaml index 654372f5..ee5066fa 100644 --- a/neo4j-standalone/Chart.yaml +++ b/neo4j-standalone/Chart.yaml @@ -1,8 +1,8 @@ apiVersion: v1 name: neo4j-standalone home: https://www.neo4j.com -version: 4.3.1 -appVersion: 4.3.1 +version: 4.3.2 +appVersion: 4.3.2 description: Neo4j is the world's leading graph database keywords: - graph diff --git a/neo4j-standalone/templates/neo4j-config.yaml b/neo4j-standalone/templates/neo4j-config.yaml index 5d0c24b4..8e525f4a 100644 --- a/neo4j-standalone/templates/neo4j-config.yaml +++ b/neo4j-standalone/templates/neo4j-config.yaml @@ -72,6 +72,12 @@ data: # Helm defaults dbms.mode: "{{ index .Values.config "dbms.mode" | default "SINGLE" }}" + # Bolt keep alive + # this helps to ensure that LoadBalancers do not close bolt connections that are in use but appear idle + dbms.connector.bolt.connection_keep_alive: "30s" + dbms.connector.bolt.connection_keep_alive_for_requests: "ALL" + dbms.connector.bolt.connection_keep_alive_streaming_scheduling_interval: "30s" + # If we set default advertised address it over-rides the bolt address used to populate the browser in a really annoying way # dbms.default_advertised_address: "$(bash -c 'echo ${SERVICE_DOMAIN}')" diff --git a/neo4j-standalone/templates/neo4j-statefulset.yaml b/neo4j-standalone/templates/neo4j-statefulset.yaml index 91884fa3..5962eac9 100644 --- a/neo4j-standalone/templates/neo4j-statefulset.yaml +++ b/neo4j-standalone/templates/neo4j-statefulset.yaml @@ -72,7 +72,8 @@ spec: {{- with $initContainers }} initContainers: {{- range $i, $initContainer := . }} - - {{ $initContainer | toYaml | indent 10 | trimAll " " }} + {{- if not $offlineMaintenanceEnabled | or $initContainer.enabledInOfflineMaintenanceMode }} + - {{ omit $initContainer "enabledInOfflineMaintenanceMode" | toYaml | indent 10 | trimAll " " }} {{- if not ( hasKey $initContainer "image") }} image: "{{ $neo4jImage }}" {{- end }} @@ -80,6 +81,7 @@ spec: volumeMounts: {{ include "neo4j.volumeMounts" ( omit $.Values.volumes "logs" "metrics" ) | nindent 12 }} {{- end }} {{- end }} + {{- end }} {{- end }} containers: - name: "neo4j" @@ -124,10 +126,13 @@ spec: name: backup {{- with .Values.neo4j.resources }} resources: + {{- if hasKey . "requests" | or (hasKey . "limits") }} + {{- omit . "cpu" "memory" | toYaml | nindent 12 }} + {{- else }} requests: {{ . | toYaml | nindent 14 }} limits: {{ . | toYaml | nindent 14 }} + {{- end }} {{- end }} - resources: {} securityContext: {{ omit .Values.securityContext "fsGroupChangePolicy" "fsGroup" | toYaml | nindent 14 }} volumeMounts: - mountPath: "/config/neo4j.conf" @@ -158,6 +163,20 @@ spec: failureThreshold: {{ .Values.startupProbe.failureThreshold }} periodSeconds: {{ .Values.startupProbe.periodSeconds }} {{- end }} + {{- with .Values.podSpec.containers }} + # Extra "sidecar" containers + {{- range $i, $extraContainer := . }} + {{- if not $offlineMaintenanceEnabled | or $extraContainer.enabledInOfflineMaintenanceMode }} + - {{ omit $extraContainer "enabledInOfflineMaintenanceMode" | toYaml | indent 10 | trimAll " " }} + {{- if not ( hasKey $extraContainer "image") }} + image: "{{ $neo4jImage }}" + {{- end }} + {{- if not ( hasKey $extraContainer "volumeMounts") }} + volumeMounts: {{ include "neo4j.volumeMounts" ( omit $.Values.volumes "logs" "metrics" ) | nindent 12 }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} volumes: {{- /* neo4j.conf settings. Using a projected volume allows keys defined later in the list of configMaps to override keys defined earlier in the list of configmaps. */}} - name: neo4j-conf diff --git a/neo4j-standalone/values.yaml b/neo4j-standalone/values.yaml index b7fecfa7..05416ce4 100644 --- a/neo4j-standalone/values.yaml +++ b/neo4j-standalone/values.yaml @@ -265,6 +265,9 @@ podSpec: # initContainers for the Neo4j pod initContainers: [ ] + # additional runtime containers for the Neo4j pod + containers: [ ] + # print the neo4j user password set during install to the `helm install` log logInitialPassword: true