Add support for Arbiter pools (#323)

* Add support for Arbiter pools

Signed-off-by: Shree Vatsa N <[email protected]>

mgr/shard.lock

@@ -6,7 +6,7 @@ shards:
 
   crinja:
     git: https://github.com/straight-shoota/crinja.git
-    version: 0.8.0
+    version: 0.8.1
 
   db:
     git: https://github.com/crystal-lang/crystal-db.git
@@ -22,7 +22,7 @@ shards:
 
   kemal:
     git: https://github.com/kemalcr/kemal.git
-    version: 1.3.0
+    version: 1.4.0
 
   moana_types:
     path: ../types
@@ -38,7 +38,7 @@ shards:
 
   volgen:
     git: https://github.com/kadalu/volgen.git
-    version: 0.3.1
+    version: 0.4.1
 
   xattr:
     git: https://github.com/aravindavk/xattr-crystal.git

mgr/src/cmds/pool_create_parser.cr

@@ -178,6 +178,10 @@ module PoolRequestParser
         storage_units["replica"].size,
         storage_units["mirror"].size
       )
+    elsif storage_units["arbiter"].size == 3
+      grp_storage_units = storage_units["arbiter"]
+      dist_group.replica_count = 3
+      dist_group.arbiter_count = 1
     elsif storage_units["disperse"].size > 0 ||
           storage_units["data"].size > 0
       grp_storage_units = storage_units["disperse"] +
@@ -284,30 +288,49 @@ module PoolRequestParser
     req
   end
 
-  # Validate the Pool create request after parsing
   def self.validate(req)
+    validate_pool_name(req)
+    validate_storage_units(req.distribute_groups)
+  end
+
+  def self.validate_pool_name(req)
+    # TODO: Pool name validations
     raise InvalidPoolRequest.new("Pool name not specified (Example: mypool)") if req.name == ""
     raise InvalidPoolRequest.new("Atleast one Storage unit is required") if req.distribute_groups.size == 0
+  end
 
-    # TODO: Pool name validations
+  def self.validate_storage_units(distribute_groups)
+    distribute_groups.each do |dist_grp|
+      validate_storage_units_match_replica_count(dist_grp)
+      validate_storage_units_match_disperse_count(dist_grp)
+      validate_storage_units_node_names(dist_grp.storage_units)
+    end
+  end
 
-    req.distribute_groups.each do |dist_grp|
-      if dist_grp.replica_count > 0 && dist_grp.storage_units.size != dist_grp.replica_count
-        raise InvalidPoolRequest.new(
-          "Number of Storage units not matching #{dist_grp.replica_keyword} count"
-        )
-      end
-      if dist_grp.disperse_count > 0 && dist_grp.storage_units.size != dist_grp.disperse_count
-        raise InvalidPoolRequest.new(
-          "Number of Storage units not matching disperse count"
-        )
-      end
+  def self.validate_storage_units_match_replica_count(dist_grp)
+    replica_arbiter_dist_grp_size = dist_grp.replica_count
+    replica_arbiter_dist_grp_size += 1 if dist_grp.arbiter_count > 0 && dist_grp.replica_count == 2
 
-      dist_grp.storage_units.each do |storage_unit|
-        msg = storage_unit.port > 0 ? "#{storage_unit.port}:" : ""
-        msg += storage_unit.path
-        raise InvalidPoolRequest.new("Node name is not specified for #{msg}") if storage_unit.node.name == ""
-      end
+    if dist_grp.replica_count > 0 && dist_grp.storage_units.size != replica_arbiter_dist_grp_size
+      raise InvalidPoolRequest.new(
+        "Number of Storage units not matching #{dist_grp.replica_keyword} count"
+      )
+    end
+  end
+
+  def self.validate_storage_units_match_disperse_count(dist_grp)
+    if dist_grp.disperse_count > 0 && dist_grp.storage_units.size != dist_grp.disperse_count
+      raise InvalidPoolRequest.new(
+        "Number of Storage units not matching disperse count"
+      )
+    end
+  end
+
+  def self.validate_storage_units_node_names(storage_units)
+    storage_units.each do |storage_unit|
+      msg = storage_unit.port > 0 ? "#{storage_unit.port}:" : ""
+      msg += storage_unit.path
+      raise InvalidPoolRequest.new("Node name is not specified for #{msg}") if storage_unit.node.name == ""
     end
   end
 end

mgr/src/server/plugins/pool_utils.cr

@@ -299,6 +299,7 @@ def services_and_volfiles(req)
 
   # Client Volfile
   tmpl = volfile_get("client")
+
   client_volfile_content = Volgen.generate(tmpl, req.to_json, req.options)
 
   # SHD Volfile
@@ -309,7 +310,7 @@ def services_and_volfiles(req)
   end
 
   req.distribute_groups.each do |dist_grp|
-    dist_grp.storage_units.each do |storage_unit|
+    dist_grp.storage_units.each_with_index do |storage_unit, index|
       # Generate Service Unit
       service = StorageUnitService.new(req.name, storage_unit)
       services[storage_unit.node.name] = [] of MoanaTypes::ServiceUnit unless services[storage_unit.node.name]?
@@ -321,6 +322,13 @@ def services_and_volfiles(req)
       tmpl = volfile_get("storage_unit")
       storage_unit.volume.id = req.id
       storage_unit.volume.name = req.name
+
+      # Handle arbiter pools
+      # Mark every 3rd storage-unit as type arbiter if distribute group has arbiter count > 0
+      if dist_grp.arbiter_count > 0 && ((index + 1) % 3 == 0)
+        storage_unit.type = "arbiter"
+      end
+
       content = Volgen.generate(tmpl, storage_unit.to_json)
       volfiles[storage_unit.node.name] = [] of MoanaTypes::Volfile unless volfiles[storage_unit.node.name]?
       volfiles[storage_unit.node.name] << MoanaTypes::Volfile.new(service.id, content)

tests/Dockerfile

@@ -28,4 +28,4 @@ RUN git clone https://github.com/kadalu/glusterfs && \
 
 RUN curl -fsSL https://crystal-lang.org/install.sh | sudo bash
 
-cmd ["/usr/sbin/init"]
+cmd ["/usr/sbin/init"]

tests/all/pools.t

@@ -45,6 +45,7 @@ nodes.each do |node|
   TEST "mkdir -p /exports/pool19"
   TEST "mkdir -p /exports/pool20a"
   TEST "mkdir -p /exports/pool20b"
+  TEST "mkdir -p /exports/pool21"
 end
 
 USE_NODE nodes[0]
@@ -492,6 +493,35 @@ puts TEST "kadalu node list"
 
 TEST "kadalu pool delete pool20a --mode=script"
 
+# Test for Arbiter Pool type
+USE_NODE nodes[0]
+TEST "kadalu pool create pool21 arbiter server1:/exports/pool21/s1 server2:/exports/pool21/s2 server3:/exports/pool21/s3"
+
+TEST "mkdir -p /mnt/pool21"
+
+puts TEST "kadalu mount pool21 /mnt/pool21"
+puts TEST "df /mnt/pool21"
+
+TEST "mkdir -p /mnt/pool21/d1"
+TEST "echo 'pool21' >> /mnt/pool21/d1/pool21.txt"
+
+content = TEST "cat /exports/pool21/s1/d1/pool21.txt"
+EQUAL content.strip, "pool21", "Test if server1:/exports/pool21/s1/d1/pool21.txt has 'pool21'"
+
+USE_NODE nodes[1]
+content = TEST "cat /exports/pool21/s2/d1/pool21.txt"
+EQUAL content.strip, "pool21", "Test if server2:/exports/pool21/s2/d1/pool21.txt has 'pool21'"
+
+USE_NODE nodes[2]
+# Expect below to be a empty file
+content = TEST "cat /exports/pool21/s3/d1/pool21.txt"
+NOT_EQUAL content.strip, "pool21", "Test if server3:/exports/pool21/s3/d1/pool21.txt has 'pool21'"
+
+USE_NODE nodes[0]
+
+TEST "kadalu pool stop pool21 --mode=script"
+TEST "kadalu pool delete pool21 --mode=script"
+
 nodes.each do |node|
   USE_NODE nodes[0]
   puts TEST "kadalu node remove #{node} --mode=script"

types/src/moana_types.cr

@@ -109,7 +109,9 @@ module MoanaTypes
     end
 
     def type
-      if @replica_count > 0
+      if @replica_count >= 2 && @arbiter_count == 1
+        "Arbiter"
+      elsif @replica_count > 0
         @replica_keyword == "mirror" ? "Mirror" : "Replicate"
       elsif @disperse_count > 0
         "Disperse"