qdrant deployment

.github/workflows/e2e.yaml

@@ -8,7 +8,7 @@ on:
     - cron: '0 5 * * *'  # Runs every day at 5 AM UTC
   workflow_dispatch:
     inputs:
-      spdki_csi_branch:
+      simplyblock_csi_branch:
         description: ''
         required: true
         default: 'master'
@@ -111,12 +111,12 @@ jobs:
         run: |
           sed -i "s/127.0.0.1/${{ steps.bootstrap_k3s.outputs.extra_node_ip }}/g" ${HOME}/.kube/config
 
-      - name: Clone spdk-csi repo
-        run: git clone -b ${{ github.event.inputs.spdki_csi_branch || 'master'}} https://github.com/simplyblock-io/spdk-csi.git
+      - name: Clone simplyblock-csi repo
+        run: git clone -b ${{ github.event.inputs.simplyblock_csi_branch || 'master'}} https://github.com/simplyblock-io/simplyblock-csi.git
 
       - name: Install SPDK-CSI using Helm
         run: |
-          cd spdk-csi/charts/latest/spdk-csi
+          cd simplyblock-csi/charts/latest/spdk-csi
           helm install -n spdk-csi --create-namespace spdk-csi ./ \
              --set csiConfig.simplybk.uuid=${{ steps.bootstrap_cluster.outputs.cluster_id }} \
              --set csiConfig.simplybk.ip=${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }} \
@@ -127,11 +127,31 @@ jobs:
              --set storagenode.spdkImage=simplyblock/spdk:main-latest
 
       - name: Sleep after Helm Installation
-        run: sleep 300
-
+        run: sleep 400
+
+      - name: Check Cluster Status
+        run: |
+          CLUSTER_API_GATEWAY_ENDPOINT=${{ steps.bootstrap_cluster.outputs.cluster_api_gateway_endpoint }}
+          CLUSTER_UUID=${{ steps.bootstrap_cluster.outputs.cluster_id }}
+          CLUSTER_SECRET=${{ steps.bootstrap_cluster.outputs.cluster_secret }}
+
+          response=$(curl -s -X GET "$CLUSTER_API_GATEWAY_ENDPOINT/cluster/$CLUSTER_UUID" \
+            -H "Content-Type: application/json" \
+            -H "Authorization: $CLUSTER_UUID $CLUSTER_SECRET")
+
+          status=$(echo $response | jq -r '.results[0].status')
+
+          if [ "$status" != "active" ]; then
+              echo "Cluster status is not active, current status: $status"
+              exit 1
+          else
+              echo "Cluster status is active"
+              exit 0
+          fi
+
       - name: Run tests
         run: |
-          cd spdk-csi
+          cd simplyblock-csi
           echo "Running tests in namespace ${{ steps.get-namespace.outputs.namespace }}"
           export CSI_NAMESPACE=spdk-csi
           export CGO_ENABLED=1

README.md

@@ -121,6 +121,13 @@ terraform apply -var mgmt_nodes=1 -var storage_nodes=3 -var nr_hugepages=2048 --
 terraform apply -var-file="dev.tfvars" --auto-approve
 ```
 
+### Specifying the Number of Secondary Storage node and Instance Type
+
+```bash
+terraform apply -var mgmt_nodes=1 -var storage_nodes=3 \
+                -var sec_storage_nodes=1 -var sec_storage_nodes_instance_type="m5.large" --auto-approve
+```
+
 ### Save Terraform Output to a File
 
 ```bash

bootstrap-cluster.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 set -euo pipefail
 
-KEY="$HOME/.ssh/simplyblock-ohio.pem"
+KEY="~/.ssh/simplyblock-qdrant.pem"
 
 print_help() {
     echo "Usage: $0 [options]"
@@ -28,9 +28,12 @@ print_help() {
     echo "  --cap-crit <value>                   Set Capacity critical level (optional)"
     echo "  --prov-cap-warn <value>              Set Provision Capacity warning level (optional)"
     echo "  --prov-cap-crit <value>              Set Provision Capacity critical level (optional)"
+    echo "  --ha-type <value>                    Set LVol HA type (optional)"
+    echo "  --enable-node-affinity               Enable node affinity for storage nodes (optional)"
+    echo "  --qpair-count <value>                Set TCP Transport qpair count (optional)"
     echo "  --k8s-snode                          Set Storage node to run on k8s (default: false)"
     echo "  --spdk-debug                         Allow core dumps on storage nodes (optional)"
-    echo "  --enable-ha-jm                       Enable HA JM for ditrib creation"
+    echo "  --disable-ha-jm                      Disable HA JM for distrib creation (optional)"
     echo "  --help                               Print this help message"
     exit 0
 }
@@ -58,10 +61,12 @@ CAP_WARN=""
 CAP_CRIT=""
 PROV_CAP_WARN=""
 PROV_CAP_CRIT=""
-ENABLE_HA_JM=""
+HA_TYPE=""
+ENABLE_NODE_AFFINITY=""
+QPAIR_COUNT=""
+DISABLE_HA_JM="false"
 K8S_SNODE="false"
 
-
 while [[ $# -gt 0 ]]; do
     arg="$1"
     case $arg in
@@ -153,14 +158,26 @@ while [[ $# -gt 0 ]]; do
         PROV_CAP_CRIT="$2"
         shift
         ;;
+    --ha-type)
+        HA_TYPE="$2"
+        shift
+        ;;
+    --enable-node-affinity)
+        ENABLE_NODE_AFFINITY="true"
+        shift
+        ;;
+    --qpair-count)
+        QPAIR_COUNT="$2"
+        shift
+        ;;
     --k8s-snode)
         K8S_SNODE="true"
         ;;
     --spdk-debug)
         SPDK_DEBUG="true"
         ;;
-    --enable-ha-jm)
-        ENABLE_HA_JM="true"
+    --disable-ha-jm)
+        DISABLE_HA_JM="true"
         ;;
     --help)
         print_help
@@ -173,36 +190,37 @@ while [[ $# -gt 0 ]]; do
     shift
 done
 
-SECRET_VALUE=$(terraform output -raw secret_value)
+# SECRET_VALUE=$(terraform output -raw secret_value)
 KEY_NAME=$(terraform output -raw key_name)
 BASTION_IP=$(terraform output -raw bastion_public_ip)
 GRAFANA_ENDPOINT=$(terraform output -raw grafana_invoke_url)
 
 ssh_dir="$HOME/.ssh"
 
-if [ ! -d "$ssh_dir" ]; then
-    mkdir -p "$ssh_dir"
-    echo "Directory $ssh_dir created."
-else
-    echo "Directory $ssh_dir already exists."
-fi
-
-if [[ -n "$SECRET_VALUE" ]]; then
-    KEY="$HOME/.ssh/$KEY_NAME"
-    if [ -f "$HOME/.ssh/$KEY_NAME" ]; then
-        echo "the ssh key: ${KEY} already exits on local"
-    else
-        echo "$SECRET_VALUE" >"$KEY"
-        chmod 400 "$KEY"
-    fi
-else
-    echo "Failed to retrieve secret value. Falling back to default key."
-fi
+# if [ ! -d "$ssh_dir" ]; then
+#     mkdir -p "$ssh_dir"
+#     echo "Directory $ssh_dir created."
+# else
+#     echo "Directory $ssh_dir already exists."
+# fi
+
+# if [[ -n "$SECRET_VALUE" ]]; then
+#     KEY="$HOME/.ssh/$KEY_NAME"
+#     if [ -f "$HOME/.ssh/$KEY_NAME" ]; then
+#         echo "the ssh key: ${KEY} already exits on local"
+#     else
+#         echo "$SECRET_VALUE" >"$KEY"
+#         chmod 400 "$KEY"
+#     fi
+# else
+#     echo "Failed to retrieve secret value. Falling back to default key."
+# fi
 
 mnodes=$(terraform output -raw mgmt_private_ips)
 echo "mgmt_private_ips: ${mnodes}"
 IFS=' ' read -ra mnodes <<<"$mnodes"
 storage_private_ips=$(terraform output -raw storage_private_ips)
+sec_storage_private_ips=$(terraform output -raw sec_storage_private_ips)
 
 echo "bootstrapping cluster..."
 
@@ -229,7 +247,7 @@ echo ""
 echo "Deploying management node..."
 echo ""
 
-command="${SBCLI_CMD} sn deploy-cleaner ; ${SBCLI_CMD} -d cluster create"
+command="sudo docker swarm leave --force ; ${SBCLI_CMD} -d cluster create"
 if [[ -n "$LOG_DEL_INTERVAL" ]]; then
     command+=" --log-del-interval $LOG_DEL_INTERVAL"
 fi
@@ -266,6 +284,15 @@ fi
 if [[ -n "$PROV_CAP_CRIT" ]]; then
     command+=" --prov-cap-crit $PROV_CAP_CRIT"
 fi
+if [[ -n "$HA_TYPE" ]]; then
+    command+=" --ha-type $HA_TYPE"
+fi
+if [[ -n "$ENABLE_NODE_AFFINITY" ]]; then
+    command+=" --enable-node-affinity $ENABLE_NODE_AFFINITY"
+fi
+if [[ -n "$QPAIR_COUNT" ]]; then
+    command+=" --qpair-count $QPAIR_COUNT"
+fi
 echo $command
 
 echo ""
@@ -316,7 +343,7 @@ for ((i = 1; i < ${#mnodes[@]}; i++)); do
         -o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \
         ec2-user@${mnodes[${i}]} "
     MANGEMENT_NODE_IP=${mnodes[0]}
-    ${SBCLI_CMD} mgmt add \${MANGEMENT_NODE_IP} ${CLUSTER_ID} eth0
+    ${SBCLI_CMD} mgmt add \${MANGEMENT_NODE_IP} ${CLUSTER_ID} ${CLUSTER_SECRET} eth0
     "
 done
 
@@ -355,6 +382,9 @@ fi
 if [[ -n "$CPU_MASK" ]]; then
     command+=" --cpu-mask $CPU_MASK"
 fi
+if [ "$DISABLE_HA_JM" == "true" ]; then
+    command+=" --disable-ha-jm"
+fi
 if [ "$SPDK_DEBUG" == "true" ]; then
     command+=" --spdk-debug"
 fi
@@ -367,28 +397,38 @@ if [ "$K8S_SNODE" == "true" ]; then
     :  # Do nothing
 
 else
+    first_three_ips=$(echo "$storage_private_ips" | awk '{print $1, $2, $3}')
     ssh -i "$KEY" -o StrictHostKeyChecking=no \
         -o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \
         ec2-user@${mnodes[0]} "
     MANGEMENT_NODE_IP=${mnodes[0]}
-    for node in ${storage_private_ips}; do
+    for node in ${first_three_ips}; do
         echo ""
         echo "joining node \${node}"
         add_node_command=\"${command} ${CLUSTER_ID} \${node}:5000 eth0\"
         echo "add node command: \${add_node_command}"
         \$add_node_command
         sleep 3
+    done
+
+    for node in ${sec_storage_private_ips}; do
+        echo ""
+        echo "joining secondary node \${node}"
+        add_node_command=\"${command} --is-secondary-node ${CLUSTER_ID} \${node}:5000 eth0\"
+        echo "add node command: \${add_node_command}"
+        \$add_node_command
+        sleep 3
     done"
 
     echo ""
     echo "Running Cluster Activate"
     echo ""
-    
+
     ssh -i "$KEY" -o StrictHostKeyChecking=no \
         -o ProxyCommand="ssh -o StrictHostKeyChecking=no -i \"$KEY\" -W %h:%p ec2-user@${BASTION_IP}" \
         ec2-user@${mnodes[0]} "
     MANGEMENT_NODE_IP=${mnodes[0]}
-    ${SBCLI_CMD} cluster activate ${CLUSTER_ID}
+    ${SBCLI_CMD} -d cluster activate ${CLUSTER_ID}
     "
 fi