Feature/enable server

.github/workflows/tests.yml

@@ -12,8 +12,10 @@ on:
         default: false
   pull_request:
     paths:
-      - bin/gcloud/*
+      - bin/*
       - internal/*
+      - neo4j*
+      - build/*
     branches:
       - dev
 
@@ -120,6 +122,15 @@ jobs:
           tags: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/reverseproxy:${{ env.CURRENT_DATE }}"
           provenance: false
 
+      - name: Build and push neo4j operations image
+        uses: docker/build-push-action@v5
+        with:
+          context: "{{defaultContext}}:neo4j/neo4j-operations"
+          push: true
+          tags: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-operations:${{ env.CURRENT_DATE }}"
+          provenance: false
+
+
       - name: Build and push backup image
         uses: docker/build-push-action@v5
         if: ${{ !inputs.RELEASE }}
@@ -182,6 +193,7 @@ jobs:
       NEO4J_DOCKER_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j:enterprise-dev"
       NEO4J_REVERSE_PROXY_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/reverseproxy:${{ needs.current-date.outputs.CURRENT_DATE }}"
       NEO4J_DOCKER_BACKUP_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-admin:${{ needs.current-date.outputs.CURRENT_DATE }}"
+      NEO4J_OPERATIONS_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-operations:${{ needs.current-date.outputs.CURRENT_DATE }}"
     container:
       image: ${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/githubactions:latest
       credentials:
@@ -215,13 +227,15 @@ jobs:
           mkdir -p .kube
           cat ${KUBECONFIG} > .kube/config
           CURRENT_DIRECTORY=$(pwd)
-          export KUBECONFIG="${CURRENT_DIRECTORY}/.kube/config"
-          echo "printing kubeconfig path $KUBECONFIG"
+          export KUBECONFIG="${CURRENT_DIRECTORY}/.kube/config"          
           export IPS_PASS=$(gcloud auth print-access-token)
-          if [[ ${{ inputs.RELEASE }} ]]; then
+          #inputs.RELEASE does not hold value when workflow_dispatch is not called
+          ISRELEASE=${{ inputs.RELEASE }}
+          if [[ ${#ISRELEASE} != 0 ]]; then            
             export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}-enterprise"
           fi
           echo "NEO4J_DOCKER_IMG=${NEO4J_DOCKER_IMG}"
+          echo "printing kubeconfig path $KUBECONFIG"
           go test -json -v -timeout ${GO_TEST_TIMEOUT} ./internal/integration_tests/ 2>&1 | tee /tmp/gotest.log | gotestfmt
           go test -json -v -timeout ${GO_TEST_TIMEOUT} ./internal/unit_tests/ 2>&1 | tee /tmp/gotest.log | gotestfmt
 
@@ -238,6 +252,7 @@ jobs:
       NEO4J_REVERSE_PROXY_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/reverseproxy:${{ needs.current-date.outputs.CURRENT_DATE }}"
       NEO4J_DOCKER_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j:community-dev"
       NEO4J_DOCKER_BACKUP_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-admin:${{ needs.current-date.outputs.CURRENT_DATE }}"
+      NEO4J_OPERATIONS_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-operations:${{ needs.current-date.outputs.CURRENT_DATE }}"
     container:
       image: ${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/githubactions:latest
       credentials:
@@ -271,13 +286,15 @@ jobs:
           mkdir -p .kube
           cat ${KUBECONFIG} > .kube/config
           CURRENT_DIRECTORY=$(pwd)
-          export KUBECONFIG="${CURRENT_DIRECTORY}/.kube/config"
-          echo "printing kubeconfig path $KUBECONFIG"
+          export KUBECONFIG="${CURRENT_DIRECTORY}/.kube/config"          
           export IPS_PASS=$(gcloud auth print-access-token)
-          if [[ ${{ inputs.RELEASE }} ]]; then
-            export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}"
+          #inputs.RELEASE does not hold value when workflow_dispatch is not called
+          ISRELEASE=${{ inputs.RELEASE }}
+          if [[ ${#ISRELEASE} != 0 ]]; then            
+            export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}-enterprise"
           fi
           echo "NEO4J_DOCKER_IMG=${NEO4J_DOCKER_IMG}"
+          echo "printing kubeconfig path $KUBECONFIG"
           go test -json -v -timeout ${GO_TEST_TIMEOUT} ./internal/integration_tests/ 2>&1 | tee /tmp/gotest.log | gotestfmt
           go test -json -v -timeout ${GO_TEST_TIMEOUT} ./internal/unit_tests/ 2>&1 | tee /tmp/gotest.log | gotestfmt
 
@@ -294,6 +311,7 @@ jobs:
       NEO4J_DOCKER_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j:enterprise-dev-ubi9"
       NEO4J_REVERSE_PROXY_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/reverseproxy:${{ needs.current-date.outputs.CURRENT_DATE }}"
       NEO4J_DOCKER_BACKUP_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-admin:ubi9-${{ needs.current-date.outputs.CURRENT_DATE }}"
+      NEO4J_OPERATIONS_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-operations:${{ needs.current-date.outputs.CURRENT_DATE }}"
     container:
       image: ${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/githubactions:latest
       credentials:
@@ -329,9 +347,11 @@ jobs:
           CURRENT_DIRECTORY=$(pwd)
           export KUBECONFIG="${CURRENT_DIRECTORY}/.kube/config"
           echo "printing kubeconfig path $KUBECONFIG"
-          export IPS_PASS=$(gcloud auth print-access-token)
-          if [[ ${{ inputs.RELEASE }} ]]; then
-            export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}-enterprise-ubi9"
+          export IPS_PASS=$(gcloud auth print-access-token)          
+          #inputs.RELEASE does not hold value when workflow_dispatch is not called
+          ISRELEASE=${{ inputs.RELEASE }}
+          if [[ ${#ISRELEASE} != 0 ]]; then            
+            export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}-enterprise"
           fi
           echo "NEO4J_DOCKER_IMG=${NEO4J_DOCKER_IMG}"
           go test -json -v -timeout ${GO_TEST_TIMEOUT} ./internal/integration_tests/ 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -350,6 +370,7 @@ jobs:
       NEO4J_DOCKER_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j:community-dev-ubi9"
       NEO4J_REVERSE_PROXY_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/reverseproxy:${{ needs.current-date.outputs.CURRENT_DATE }}"
       NEO4J_DOCKER_BACKUP_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-admin:ubi9-${{ needs.current-date.outputs.CURRENT_DATE }}"
+      NEO4J_OPERATIONS_IMG: "${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/neo4j-operations:${{ needs.current-date.outputs.CURRENT_DATE }}"
     container:
       image: ${{ vars.ARTIFACT_REGISTRY_REPO_NAME }}/githubactions:latest
       credentials:
@@ -386,8 +407,10 @@ jobs:
           export KUBECONFIG="${CURRENT_DIRECTORY}/.kube/config"
           echo "printing kubeconfig path $KUBECONFIG"
           export IPS_PASS=$(gcloud auth print-access-token)
-          if [[ ${{ inputs.RELEASE }} ]]; then
-            export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}-ubi9"
+          #inputs.RELEASE does not hold value when workflow_dispatch is not called
+          ISRELEASE=${{ inputs.RELEASE }}
+          if [[ ${#ISRELEASE} != 0 ]]; then            
+            export NEO4J_DOCKER_IMG="neo4j:${{ inputs.NEO4J_VERSION }}-enterprise"
           fi
           echo "NEO4J_DOCKER_IMG=${NEO4J_DOCKER_IMG}"
           go test -json -v -timeout ${GO_TEST_TIMEOUT} ./internal/integration_tests/ 2>&1 | tee /tmp/gotest.log | gotestfmt
@@ -478,6 +501,14 @@ jobs:
           tags: "neo4j/helm-charts-reverse-proxy:${{ inputs.NEO4J_VERSION }}"
           provenance: false
 
+      - name: Build and push operations image
+        uses: docker/build-push-action@v5
+        with:
+          context: "{{defaultContext}}:neo4j/neo4j-operations"
+          push: true
+          tags: "neo4j/neo4j-operations:${{ inputs.NEO4J_VERSION }}"
+          provenance: false
+
       - name: Build and push backup image
         uses: docker/build-push-action@v5
         with:

bin/gcloud-create-gke-cluster

@@ -15,7 +15,7 @@ CLOUDSDK_COMPUTE_ZONE="${CLOUDSDK_COMPUTE_ZONE:?CLOUDSDK_COMPUTE_ZONE is require
 
 # Parameters
 NODE_MACHINE="e2-standard-4"
-NUM_NODES="${NUM_NODES:-10}"
+NUM_NODES="${NUM_NODES:-11}"
 
 # For more info on release channels see https://cloud.google.com/kubernetes-engine/docs/concepts/release-channels
 RELEASE_CHANNEL="stable"
@@ -27,7 +27,7 @@ gcloud container clusters create "${CLOUDSDK_CONTAINER_CLUSTER}" \
      --num-nodes="${NUM_NODES}" \
      --workload-pool="${CLOUDSDK_CORE_PROJECT}.svc.id.goog" \
      --preemptible --machine-type="${NODE_MACHINE}" --image-type="COS_CONTAINERD" \
-     --disk-type="pd-ssd" --disk-size="10" \
+     --disk-type="pd-ssd" --disk-size="20" \
      --max-pods-per-node=30 --enable-ip-alias \
      --enable-shielded-nodes --metadata=disable-legacy-endpoints=true --no-enable-basic-auth
 

internal/integration_tests/cluster.go

@@ -66,7 +66,7 @@ func removeLabelFromNodes(t *testing.T) error {
 func clusterTests(clusterRelease model.ReleaseName) ([]SubTest, error) {
 
 	subTests := []SubTest{
-		{name: "Install Backup Helm Chart For AWS", test: func(t *testing.T) {
+		{name: "Install Backup Helm Chart For AWS With NodeSelector", test: func(t *testing.T) {
 			t.Parallel()
 			assert.NoError(t, InstallNeo4jBackupAWSHelmChartWithNodeSelector(t, clusterRelease), "Backup to AWS should succeed")
 		}},
@@ -78,6 +78,10 @@ func clusterTests(clusterRelease model.ReleaseName) ([]SubTest, error) {
 			t.Parallel()
 			assert.NoError(t, CheckLogsFormat(t, clusterRelease), "Cluster core logs format should be in JSON")
 		}},
+		{name: "Check Neo4j Operations Pod for enabling server", test: func(t *testing.T) {
+			t.Parallel()
+			assert.NoError(t, CheckNeo4jOperationsPod(t, clusterRelease), "Neo4j Operations Pod should get executed")
+		}},
 		{name: "ImagePullSecret tests", test: func(t *testing.T) {
 			t.Parallel()
 			assert.NoError(t, imagePullSecretTests(t, clusterRelease), "Perform ImagePullSecret Tests")
@@ -344,6 +348,53 @@ func CheckLogsFormat(t *testing.T, releaseName model.ReleaseName) error {
 	return nil
 }
 
+// CheckNeo4jOperationsPod checks whether the neo4j operations pod is executed or not
+func CheckNeo4jOperationsPod(t *testing.T, releaseName model.ReleaseName) error {
+
+	fetchPods := func() (*v1.PodList, error) {
+		pods, err := getPodsWithSpecificLabel(releaseName.Namespace(), "app=neo4j-operations")
+		if err != nil {
+			return &v1.PodList{}, fmt.Errorf("error seen while fetching list of pods \n %v", err)
+		}
+		if len(pods.Items) == 0 {
+			return &v1.PodList{}, fmt.Errorf("no pods found")
+		}
+		if len(pods.Items) > 1 {
+			return &v1.PodList{}, fmt.Errorf("more than one operations pod found")
+		}
+		return pods, nil
+	}
+
+	pods, err := fetchPods()
+	if err != nil {
+		return err
+	}
+	pod := pods.Items[0]
+	for pod.Status.Phase == v1.PodRunning {
+		t.Logf("operations pod in running state..Waiting for it to be completed")
+		time.Sleep(30 * time.Second)
+		pods, err = fetchPods()
+		if err != nil {
+			return err
+		}
+		pod = pods.Items[0]
+	}
+	if pod.Status.Phase != v1.PodSucceeded {
+		return fmt.Errorf("pod phase %v is not succeeded", pod.Status.Phase)
+	}
+
+	out, err := exec.Command("kubectl", "logs", pod.Name, "--namespace", string(releaseName.Namespace())).CombinedOutput()
+	if err != nil {
+		t.Logf("error while fetching operations pod logs")
+		return err
+	}
+	stringOutput := strings.ToLower(string(out))
+	if !strings.Contains(stringOutput, "server is already enabled") {
+		return fmt.Errorf("operations pod does not contain valid logs \n logs := %s", string(out))
+	}
+	return nil
+}
+
 // imagePullSecretTests runs tests related to imagePullSecret feature
 func imagePullSecretTests(t *testing.T, name model.ReleaseName) error {
 	t.Run("Check cluster core has imagePullSecret image", func(t *testing.T) {
@@ -529,7 +580,7 @@ func checkLoadBalancerService(t *testing.T, name model.ReleaseName, expectedEndP
 
 // checkPods checks for the number of pods which should be 5 (3 cluster core + 2 read replica)
 func checkPods(t *testing.T, name model.ReleaseName) error {
-	pods, err := getAllPods(name.Namespace())
+	pods, err := getPodsWithSpecificLabel(name.Namespace(), "helm.neo4j.com/clustering=true")
 	if !assert.NoError(t, err) {
 		return err
 	}

internal/integration_tests/cluster_test.go

@@ -41,9 +41,10 @@ func TestInstallNeo4jClusterInGcloud(t *testing.T) {
 	core1HelmArgs := append(defaultHelmArgs, model.ImagePullSecretArgs...)
 	core1HelmArgs = append(core1HelmArgs, model.NodeSelectorArgs(namespace)...)
 	core2HelmArgs := append(defaultHelmArgs, model.PriorityClassNameArgs(namespace)...)
+	core3HelmArgs := append(defaultHelmArgs, model.EnableServerArgs()...)
 	core1 := clusterCore{model.NewCoreReleaseName(clusterReleaseName, 1), core1HelmArgs}
 	core2 := clusterCore{model.NewCoreReleaseName(clusterReleaseName, 2), core2HelmArgs}
-	core3 := clusterCore{model.NewCoreReleaseName(clusterReleaseName, 3), defaultHelmArgs}
+	core3 := clusterCore{model.NewCoreReleaseName(clusterReleaseName, 3), core3HelmArgs}
 	cores := []clusterCore{core1, core2, core3}
 
 	t.Cleanup(clusterTestCleanup(t, clusterReleaseName, core1, core2, core3, true))

internal/integration_tests/k8s_client.go

@@ -169,6 +169,10 @@ func getSpecificPod(namespace model.Namespace, podName string) (*coreV1.Pod, err
 	return Clientset.CoreV1().Pods(string(namespace)).Get(context.TODO(), podName, v1.GetOptions{})
 }
 
+func getPodsWithSpecificLabel(namespace model.Namespace, label string) (*coreV1.PodList, error) {
+	return Clientset.CoreV1().Pods(string(namespace)).List(context.TODO(), v1.ListOptions{LabelSelector: label})
+}
+
 func getNodesList() (*coreV1.NodeList, error) {
 	return Clientset.CoreV1().Nodes().List(context.TODO(), v1.ListOptions{})
 }

internal/model/default_values.go

@@ -20,6 +20,7 @@ var ImagePullSecretUsername,
 var ImagePullSecretArgs []string
 var (
 	PriorityClassNameArgs func(namespace string) []string
+	EnableServerArgs      func() []string
 	NodeSelectorArgs      func(namespace string) []string
 )
 var NodeSelectorLabel = func(namespace string) string {
@@ -102,6 +103,11 @@ func init() {
 		log.Panic("Please set GCP_SERVICE_ACCOUNT_CRED env variable !!. This environment variable holds the json credentials of GCP service account")
 	}
 
+	_, present = os.LookupEnv("NEO4J_OPERATIONS_IMG")
+	if !present {
+		log.Panic("Please set NEO4J_OPERATIONS_IMG env variable !!.")
+	}
+
 	ImagePullSecretArgs = []string{
 		"--set", fmt.Sprintf("image.customImage=%s", ImagePullSecretCustomImageName),
 		"--set", "image.imagePullSecrets[0]=demo",
@@ -120,6 +126,14 @@ func init() {
 			"--set", fmt.Sprintf("podSpec.priorityClassName=%s", PriorityClassName(namespace)),
 		}
 	}
+	EnableServerArgs = func() []string {
+		return []string{
+			"--set", "neo4j.operations.enableServer=true",
+			"--set", fmt.Sprintf("neo4j.operations.image=%s", os.Getenv("NEO4J_OPERATIONS_IMG")),
+			"--set", "neo4j.operations.protocol=neo4j+ssc",
+		}
+	}
+
 }
 
 const StorageSize = "10Gi"

internal/model/release_values.go

@@ -33,6 +33,13 @@ type HelmValues struct {
 	Analytics                Analytics           `yaml:"analytics"`
 }
 
+type Operations struct {
+	EnableServer bool              `yaml:"enableServer"`
+	Image        string            `yaml:"image"`
+	Protocol     string            `yaml:"protocol"`
+	Labels       map[string]string `yaml:"labels"`
+}
+
 type Analytics struct {
 	Enabled bool `yaml:"enabled"`
 	Type    Type `yaml:"type"`
@@ -110,6 +117,7 @@ type Neo4J struct {
 	OfflineMaintenanceModeEnabled bool        `yaml:"offlineMaintenanceModeEnabled,omitempty"`
 	Resources                     Resources   `yaml:"resources,omitempty"`
 	Labels                        interface{} `yaml:"labels,omitempty"`
+	Operations                    Operations  `yaml:"operations,omitempty"`
 }
 type Requests struct {
 	Storage string `yaml:"storage,omitempty"`