Added S3 user management

Signed-off-by: utkarshbhatthere <[email protected]>

.github/workflows/tests.yml

@@ -52,6 +52,16 @@ jobs:
       with:
         fetch-depth: 0
 
+    - name: Install dependencies
+      run: |
+        # Python script dependencies
+        sudo python -m pip install --upgrade pip
+        sudo pip install flake8 pep8-naming boto3
+
+    - name: Lint check
+      run: |
+        flake8 . --count --show-source --statistics
+
     - name: Install and setup
       run: |
         tests/scripts/actionutils.sh install_microceph
@@ -113,15 +123,9 @@ jobs:
     - name: Exercise RGW
       run: |
         set -eux
-        sudo microceph.ceph status
-        sudo systemctl status snap.microceph.rgw
-        sudo microceph.radosgw-admin user create --uid=test --display-name=test
-        sudo microceph.radosgw-admin key create --uid=test --key-type=s3 --access-key fooAccessKey --secret-key fooSecretKey
-        sudo apt-get -qq install s3cmd
-        echo hello-radosgw > ~/test.txt
-        s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl mb s3://testbucket
-        s3cmd --host localhost --host-bucket="localhost/%(bucket)" --access_key=fooAccessKey --secret_key=fooSecretKey --no-ssl put -P ~/test.txt s3://testbucket
-        curl -s http://localhost/testbucket/test.txt | grep -F hello-radosgw
+        sudo microceph s3-user create testUser --json > keys.json
+        sudo python3 ./scripts/appS3.py http://localhost:80 keys.json --obj-num 2
+        sudo microceph s3-user delete testUser
 
     - name: Test Cluster Config
       run: |

docs/conf.py

@@ -19,8 +19,6 @@
 templates_path = ['_templates']
 exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store', 'sphinxenv']
 
-
-
 # -- Options for HTML output -------------------------------------------------
 # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output
 

docs/how-to/index.rst

@@ -11,4 +11,5 @@ These how-to guides will cover key operations and processes in Microceph.
    service-migration
    metrics
    alerts
+   s3-user
 

docs/how-to/s3-user.rst

@@ -0,0 +1,106 @@
+Use S3 user management on MicroCeph
+===================================
+
+MicroCeph provides an easy to use interface for creating, viewing and deleting s3 users for interfacing with the RGW endpoint.
+This enables smooth and easy access to Object Storage.
+
+.. list-table:: Supported s3-user operations
+   :widths: 30 70
+   :header-rows: 1
+
+   * - Operation
+     - Description
+   * - create
+     - Create provided s3 (radosgw) user with optionally provided access-key and secret
+   * - delete
+     - Delete provided s3 (radosgw) user
+   * - get
+     - Fetch key information of the provided s3 (radosgw) user
+   * - list
+     - List all s3 (radosgw) users
+.. note:: Users can additionally provide --json flag to create and get commands to dump a much detailed 
+
+1. Create an S3 user (optionally provide --access-key --secret and --json)
+
+  .. code-block:: shell
+
+    $ sudo microceph s3-user create newTestUser --access-key=ThisIsAccessKey --secret=ThisIsSecret --json
+    {
+        "user_id": "newTestUser",
+        "display_name": "newTestUser",
+        "email": "",
+        "suspended": 0,
+        "max_buckets": 1000,
+        "subusers": [],
+        "keys": [
+            {
+                "user": "newTestUser",
+                "access_key": "ThisIsAccessKey",
+                "secret_key": "ThisIsSecret"
+            }
+        ],
+        "swift_keys": [],
+        "caps": [],
+        "op_mask": "read, write, delete",
+        "default_placement": "",
+        "default_storage_class": "",
+        "placement_tags": [],
+        "bucket_quota": {
+            "enabled": false,
+            "check_on_raw": false,
+            "max_size": -1,
+            "max_size_kb": 0,
+            "max_objects": -1
+        },
+        "user_quota": {
+            "enabled": false,
+            "check_on_raw": false,
+            "max_size": -1,
+            "max_size_kb": 0,
+            "max_objects": -1
+        },
+        "temp_url_keys": [],
+        "type": "rgw",
+        "mfa_ids": []
+    }
+
+2. List all s3 users :
+
+  .. code-block:: shell
+
+    $ sudo microceph s3-user list
+    +---+-------------+
+    | # |    NAME     |
+    +---+-------------+
+    | 1 | newTestUser |
+    +---+-------------+
+    | 2 | testUser    |
+    +---+-------------+
+
+3. Get details of a an s3 user (optionally use --json flag to get complete details):
+
+  .. code-block:: shell
+
+    $ sudo microceph s3-user get testUser
+    +----------+----------------------+---------+
+    |   NAME   |    ACCESS KEY   |    SECRET    |
+    +----------+----------------------+---------+
+    | testUser | ThisIsAccessKey | ThisIsSecret |
+    +----------+----------------------+---------+
+
+4. Delete an s3 user:
+
+  .. code-block:: shell
+
+   $ sudo microceph s3-user delete newTestUser
+   $ sudo microceph s3-user list
+    +---+----------+
+    | # |   NAME   |
+    +---+----------+
+    | 1 | testUser |
+    +---+----------+
+
+  .. warning:: All the related buckets+objects should be deleted before deletion of the user. 
+
+For more fine-tuned user management use `radosgw-admin CLI <https://docs.ceph.com/en/latest/man/8/radosgw-admin/>`_
+

microceph/api/endpoints.go

@@ -10,11 +10,11 @@ var Endpoints = []rest.Endpoint{
 	disksCmd,
 	resourcesCmd,
 	servicesCmd,
-	rgwServiceCmd,
 	configsCmd,
 	restartServiceCmd,
 	mdsServiceCmd,
 	mgrServiceCmd,
 	monServiceCmd,
 	rgwServiceCmd,
+	s3Cmd,
 }

microceph/api/s3.go

@@ -0,0 +1,79 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+
+	"github.com/canonical/lxd/lxd/response"
+	"github.com/canonical/microceph/microceph/api/types"
+	"github.com/canonical/microceph/microceph/ceph"
+	"github.com/canonical/microcluster/rest"
+	"github.com/canonical/microcluster/state"
+)
+
+// /1.0/resources endpoint.
+var s3Cmd = rest.Endpoint{
+	Path:   "services/rgw/user",
+	Get:    rest.EndpointAction{Handler: cmdS3Get, ProxyTarget: true},
+	Put:    rest.EndpointAction{Handler: cmdS3Put, ProxyTarget: true},
+	Delete: rest.EndpointAction{Handler: cmdS3Delete, ProxyTarget: true},
+}
+
+func cmdS3Get(s *state.State, r *http.Request) response.Response {
+	var err error
+	var req types.S3User
+
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	// If a user name is passed.
+	if len(req.Name) > 0 {
+		getOutput, err := ceph.GetS3User(req)
+		if err != nil {
+			return response.SmartError(err)
+		}
+		return response.SyncResponse(true, getOutput)
+	} else {
+		listOutput, err := ceph.ListS3Users()
+		if err != nil {
+			return response.SmartError(err)
+		}
+		return response.SyncResponse(true, listOutput)
+	}
+}
+
+func cmdS3Put(s *state.State, r *http.Request) response.Response {
+	var err error
+	var req types.S3User
+
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	output, err := ceph.CreateS3User(req)
+	if err != nil {
+		return response.SmartError(err)
+	}
+
+	return response.SyncResponse(true, output)
+}
+
+func cmdS3Delete(s *state.State, r *http.Request) response.Response {
+	var err error
+	var req types.S3User
+
+	err = json.NewDecoder(r.Body).Decode(&req)
+	if err != nil {
+		return response.InternalError(err)
+	}
+
+	err = ceph.DeleteS3User(req.Name)
+	if err != nil {
+		return response.SmartError(err)
+	}
+
+	return response.EmptySyncResponse
+}

microceph/api/types/s3.go

@@ -0,0 +1,9 @@
+// Package types provides shared types and structs.
+package types
+
+// holds the name, access and secretkey required for exposing an S3 user.
+type S3User struct {
+	Name     string `json:"name" yaml:"name"`
+	Key      string `json:"key" yaml:"key"`
+	Secret   string `json:"secret" yaml:"secret"`
+}

microceph/ceph/rgw_s3.go

@@ -0,0 +1,78 @@
+package ceph
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/canonical/microceph/microceph/api/types"
+)
+
+func CreateS3User(user types.S3User) (string, error) {
+	args := []string{
+		"user",
+		"create",
+		fmt.Sprintf("--uid=%s", user.Name),
+		fmt.Sprintf("--display-name=%s", user.Name),
+	}
+
+	if len(user.Key) > 0 {
+		args = append(args, fmt.Sprintf("--access-key=%s", user.Key))
+	}
+
+	if len(user.Secret) > 0 {
+		args = append(args, fmt.Sprintf("--secret=%s", user.Secret))
+	}
+
+	output, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return "", err
+	}
+
+	return output, nil
+}
+
+func GetS3User(user types.S3User) (string, error) {
+	args := []string{
+		"user",
+		"info",
+		fmt.Sprintf("--uid=%s", user.Name),
+	}
+
+	output, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return "", err
+	}
+
+	return output, nil
+}
+
+func ListS3Users() ([]string, error) {
+	args := []string{
+		"user",
+		"list",
+	}
+
+	output, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return []string{}, err
+	}
+
+	ret := []string{}
+	json.Unmarshal([]byte(output), &ret)
+	return ret, nil
+}
+
+func DeleteS3User(name string) error {
+	args := []string{
+		"user",
+		"rm",
+		fmt.Sprintf("--uid=%s", name),
+	}
+
+	_, err := processExec.RunCommand("radosgw-admin", args...)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}