From 94ae9b5717f03fb4943a201be79b4a85e4860e1d Mon Sep 17 00:00:00 2001 From: Utkarsh Bhatt Date: Mon, 7 Oct 2024 18:55:03 +0530 Subject: [PATCH] Added support to perform cluster promotion/demotion Signed-off-by: Utkarsh Bhatt --- .github/workflows/tests.yml | 6 + docs/how-to/perform-site-failover.rst | 71 +++++++++++ .../commands/remote-replication-rbd.rst | 29 +++++ microceph/api/ops_replication.go | 10 ++ microceph/api/types/replication.go | 10 +- microceph/ceph/rbd_mirror.go | 120 +++++++++++++++++- microceph/ceph/rbd_mirror_test.go | 40 ++++++ microceph/ceph/replication.go | 24 +++- microceph/ceph/replication_rbd.go | 113 ++++++++++++++--- .../rbd_mirror_promote_secondary_failure.txt | 10 ++ microceph/client/remote_replication.go | 8 +- .../cmd/microceph/remote_replication_rbd.go | 8 ++ .../remote_replication_rbd_demote.go | 69 ++++++++++ .../remote_replication_rbd_promote.go | 69 ++++++++++ microceph/constants/constants.go | 3 + microceph/go.mod | 4 +- microceph/go.sum | 4 +- tests/scripts/actionutils.sh | 82 ++++++++++++ 18 files changed, 648 insertions(+), 32 deletions(-) create mode 100644 docs/how-to/perform-site-failover.rst create mode 100644 microceph/ceph/test_assets/rbd_mirror_promote_secondary_failure.txt create mode 100644 microceph/cmd/microceph/remote_replication_rbd_demote.go create mode 100644 microceph/cmd/microceph/remote_replication_rbd_promote.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 9fcc3495..704a53dc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -709,6 +709,12 @@ jobs: - name: Verify RBD mirror run : ~/actionutils.sh remote_verify_rbd_mirroring + - name: Failover site A to Site B + run : ~/actionutils.sh remote_failover_to_siteb + + - name: Failback to Site A + run : ~/actionutils.sh remote_failback_to_sitea + - name: Disable RBD mirror run : ~/actionutils.sh remote_disable_rbd_mirroring diff --git a/docs/how-to/perform-site-failover.rst b/docs/how-to/perform-site-failover.rst new file mode 100644 index 00000000..97cefa2f --- /dev/null +++ b/docs/how-to/perform-site-failover.rst @@ -0,0 +1,71 @@ +============================================= +Perform failover for replicated RBD resources +============================================= + +In case of a disaster, all replicated RBD pools can be failed over to a non-primary remote. + +An operator can perform promotion on a non-primary cluster, this will inturn promote all replicated rbd +images in all rbd pools and make them primary. This enables them to be consumed by vms and other workloads. + +Prerequisites +-------------- +1. A primary and a secondary MicroCeph cluster, for example named "primary_cluster" and "secondary_cluster" +2. primary_cluster has imported configurations from secondary_cluster and vice versa. refer to :doc:`import remote <./import-remote-cluster>` +3. RBD remote replication is configured for atleast 1 rbd image. refer to :doc:`configure rbd replication <./configure-rbd-mirroring>` + +Failover to a non-primary remote cluster +----------------------------------------- +List all the resources on 'secondary_cluster' to check primary status. + +.. code-block:: none + + sudo microceph remote replication rbd list + +-----------+------------+------------+---------------------+ + | POOL NAME | IMAGE NAME | IS PRIMARY | LAST LOCAL UPDATE | + +-----------+------------+------------+---------------------+ + | pool_one | image_one | false | 2024-10-14 09:03:17 | + | pool_one | image_two | false | 2024-10-14 09:03:17 | + +-----------+------------+------------+---------------------+ + +An operator can perform cluster wide promotion as follows: + +.. code-block:: none + + sudo microceph remote replication rbd promote --remote primary_cluster --force + +Here, paramter helps microceph filter the resources to promote. + +Verify RBD remote replication primary status +--------------------------------------------- + +List all the resources on 'secondary_cluster' again to check primary status. + +.. code-block:: none + + sudo microceph remote replication rbd status pool_one + +-----------+------------+------------+---------------------+ + | POOL NAME | IMAGE NAME | IS PRIMARY | LAST LOCAL UPDATE | + +-----------+------------+------------+---------------------+ + | pool_one | image_one | true | 2024-10-14 09:06:12 | + | pool_one | image_two | true | 2024-10-14 09:06:12 | + +-----------+------------+------------+---------------------+ + +The status shows that there are 2 replicated images and both of them are now primary. + +Failback to old primary +------------------------ + +Once the disaster struck cluster (primary_cluster) is back online the RBD resources +can be failed back to it, but, by this time the RBD images at the current primary (secondary_cluster) +would have diverged from primary_cluster. Thus, to have a clean sync, the operator must decide +which cluster would be demoted to the non-primary status. This cluster will then receive the +RBD mirror updates from the standing primary. + +Note: Demotion can cause data loss and hence can only be performed with the 'force' flag. + +At primary_cluster (was primary before disaster), perform demotion. +.. code-block:: none + + sudo microceph remote replication rbd demote + + diff --git a/docs/reference/commands/remote-replication-rbd.rst b/docs/reference/commands/remote-replication-rbd.rst index 7d3db611..e3b8fd63 100644 --- a/docs/reference/commands/remote-replication-rbd.rst +++ b/docs/reference/commands/remote-replication-rbd.rst @@ -96,3 +96,32 @@ Usage: --force forcefully disable replication for rbd resource +``promote`` +---------- + +Promote local cluster to primary + +.. code-block:: none + + microceph remote replication rbd promote [flags] + +.. code-block:: none + + --remote remote MicroCeph cluster name + --force forcefully promote site to primary + +``demote`` +------------ + +Demote local cluster to secondary + +Usage: + +.. code-block:: none + + microceph remote replication rbd demote [flags] + +.. code-block:: none + + --remote remote MicroCeph cluster name + diff --git a/microceph/api/ops_replication.go b/microceph/api/ops_replication.go index 8eb66741..57ef4ff0 100644 --- a/microceph/api/ops_replication.go +++ b/microceph/api/ops_replication.go @@ -31,6 +31,7 @@ var opsReplicationCmd = rest.Endpoint{ var opsReplicationWorkloadCmd = rest.Endpoint{ Path: "ops/replication/{wl}", Get: rest.EndpointAction{Handler: getOpsReplicationWorkload, ProxyTarget: false}, + Put: rest.EndpointAction{Handler: putOpsReplicationWorkload, ProxyTarget: false}, } // CRUD Replication @@ -47,6 +48,12 @@ func getOpsReplicationWorkload(s state.State, r *http.Request) response.Response return cmdOpsReplication(s, r, types.ListReplicationRequest) } +// putOpsReplicationWorkload handles list operation +func putOpsReplicationWorkload(s state.State, r *http.Request) response.Response { + // either promote or demote (already encoded in request) + return cmdOpsReplication(s, r, "") +} + // getOpsReplicationResource handles status operation for a certain resource. func getOpsReplicationResource(s state.State, r *http.Request) response.Response { return cmdOpsReplication(s, r, types.StatusReplicationRequest) @@ -104,6 +111,9 @@ func cmdOpsReplication(s state.State, r *http.Request, patchRequest types.Replic return response.SmartError(fmt.Errorf("")) } + // TODO: convert this to debug + logger.Infof("REPOPS: %s received for %s: %s", req.GetWorkloadRequestType(), wl, resource) + return handleReplicationRequest(s, r.Context(), req) } diff --git a/microceph/api/types/replication.go b/microceph/api/types/replication.go index f28aadd5..9b905e34 100644 --- a/microceph/api/types/replication.go +++ b/microceph/api/types/replication.go @@ -11,9 +11,13 @@ type ReplicationRequestType string const ( EnableReplicationRequest ReplicationRequestType = "POST-" + constants.EventEnableReplication ConfigureReplicationRequest ReplicationRequestType = "PUT-" + constants.EventConfigureReplication - DisableReplicationRequest ReplicationRequestType = "DELETE-" + constants.EventDisableReplication - StatusReplicationRequest ReplicationRequestType = "GET-" + constants.EventStatusReplication - ListReplicationRequest ReplicationRequestType = "GET-" + constants.EventListReplication + PromoteReplicationRequest ReplicationRequestType = "PUT-" + constants.EventPromoteReplication + DemoteReplicationRequest ReplicationRequestType = "PUT-" + constants.EventDemoteReplication + // Delete Requests + DisableReplicationRequest ReplicationRequestType = "DELETE-" + constants.EventDisableReplication + // Get Requests + StatusReplicationRequest ReplicationRequestType = "GET-" + constants.EventStatusReplication + ListReplicationRequest ReplicationRequestType = "GET-" + constants.EventListReplication ) type CephWorkloadType string diff --git a/microceph/ceph/rbd_mirror.go b/microceph/ceph/rbd_mirror.go index aa2355ec..545cd98e 100644 --- a/microceph/ceph/rbd_mirror.go +++ b/microceph/ceph/rbd_mirror.go @@ -211,8 +211,8 @@ func DisablePoolMirroring(pool string, peer RbdReplicationPeer, localName string return nil } -// DisableMirroringAllImagesInPool disables mirroring for all images for a pool enabled in pool mirroring mode. -func DisableMirroringAllImagesInPool(poolName string) error { +// DisableAllMirroringImagesInPool disables mirroring for all images for a pool enabled in pool mirroring mode. +func DisableAllMirroringImagesInPool(poolName string) error { poolStatus, err := GetRbdMirrorVerbosePoolStatus(poolName, "", "") if err != nil { err := fmt.Errorf("failed to fetch status for %s pool: %v", poolName, err) @@ -233,6 +233,28 @@ func DisableMirroringAllImagesInPool(poolName string) error { return nil } +// DisableMirroringAllImagesInPool disables mirroring for all images for a pool enabled in pool mirroring mode. +func ResyncAllMirroringImagesInPool(poolName string) error { + poolStatus, err := GetRbdMirrorVerbosePoolStatus(poolName, "", "") + if err != nil { + err := fmt.Errorf("failed to fetch status for %s pool: %v", poolName, err) + logger.Error(err.Error()) + return err + } + + flaggedImages := []string{} + for _, image := range poolStatus.Images { + err := flagImageForResync(poolName, image.Name) + if err != nil { + return fmt.Errorf("failed to resync %s/%s", poolName, image.Name) + } + flaggedImages = append(flaggedImages, image.Name) + } + + logger.Infof("REPRBD: Resynced %v images in %s pool.", flaggedImages, poolName) + return nil +} + // getPeerUUID returns the peer ID for the requested peer name. func getPeerUUID(pool string, peerName string, client string, cluster string) string { poolInfo, err := GetRbdMirrorPoolInfo(pool, cluster, client) @@ -464,6 +486,60 @@ func configureImageFeatures(pool string, image string, op string, feature string return nil } +// enableImageFeatures enables the list of rbd features on the requested resource. +func enableRbdImageFeatures(poolName string, imageName string, features []string) error { + for _, feature := range features { + err := configureImageFeatures(poolName, imageName, "enable", feature) + if err != nil && !strings.Contains(err.Error(), "one or more requested features are already enabled") { + return err + } + } + return nil +} + +// disableRbdImageFeatures disables the list of rbd features on the requested resource. +func disableRbdImageFeatures(poolName string, imageName string, features []string) error { + for _, feature := range features { + err := configureImageFeatures(poolName, imageName, "disable", feature) + if err != nil { + return err + } + } + return nil +} + +// Promote local pool to primary. +func handlePoolPromotion(rh *RbdReplicationHandler, poolName string) error { + err := promotePool(poolName, rh.Request.IsForceOp, "", "") + if err != nil { + if strings.Contains(err.Error(), "image is primary within a remote cluster or demotion is not propagated yet") { + return fmt.Errorf("unable to promote %s, use --force if you understand the risks of this operation: %v", poolName, err) + } + + return err + } + return nil +} + +// Demote local pool to secondary. +func handlePoolDemotion(_ *RbdReplicationHandler, poolName string) error { + return demotePool(poolName, "", "") +} + +// flagImageForResync flags requested mirroring image in the given pool for resync. +func flagImageForResync(poolName string, imageName string) error { + args := []string{ + "mirror", "image", "resync", fmt.Sprintf("%s/%s", poolName, imageName), + } + + _, err := processExec.RunCommand("rbd", args...) + if err != nil { + return err + } + + return nil +} + // peerBootstrapCreate generates peer bootstrap token on remote ceph cluster. func peerBootstrapCreate(pool string, client string, cluster string) (string, error) { args := []string{ @@ -525,6 +601,46 @@ func peerRemove(pool string, peerId string, localName string, remoteName string) return nil } +func promotePool(poolName string, isForce bool, remoteName string, localName string) error { + args := []string{ + "mirror", "pool", "promote", poolName, + } + + if isForce { + args = append(args, "--force") + } + + // add --cluster and --id args + args = appendRemoteClusterArgs(args, remoteName, localName) + + output, err := processExec.RunCommand("rbd", args...) + if err != nil { + return fmt.Errorf("failed to promote pool(%s): %v", poolName, err) + } + + // TODO: Change to debugf + logger.Infof("REPRBD: Promotion Output: %s", output) + return nil +} + +func demotePool(poolName string, remoteName string, localName string) error { + args := []string{ + "mirror", "pool", "demote", poolName, + } + + // add --cluster and --id args + args = appendRemoteClusterArgs(args, remoteName, localName) + + output, err := processExec.RunCommand("rbd", args...) + if err != nil { + return fmt.Errorf("failed to promote pool(%s): %v", poolName, err) + } + + // TODO: Change to debugf + logger.Infof("REPRBD: Demotion Output: %s", output) + return nil +} + // ########################### HELPERS ########################### func IsRemoteConfiguredForRbdMirror(remoteName string) bool { diff --git a/microceph/ceph/rbd_mirror_test.go b/microceph/ceph/rbd_mirror_test.go index b609bf9c..6abf41a8 100644 --- a/microceph/ceph/rbd_mirror_test.go +++ b/microceph/ceph/rbd_mirror_test.go @@ -1,6 +1,7 @@ package ceph import ( + "fmt" "os" "testing" @@ -93,3 +94,42 @@ func (ks *RbdMirrorSuite) TestPoolInfo() { assert.Equal(ks.T(), resp.LocalSiteName, "magical") assert.Equal(ks.T(), resp.Peers[0].RemoteName, "simple") } +func (ks *RbdMirrorSuite) TestPromotePoolOnSecondary() { + r := mocks.NewRunner(ks.T()) + output, _ := os.ReadFile("./test_assets/rbd_mirror_promote_secondary_failure.txt") + + // mocks and expectations + r.On("RunCommand", []interface{}{ + "rbd", "mirror", "pool", "promote", "pool"}...).Return("", fmt.Errorf("%s", string(output))).Once() + r.On("RunCommand", []interface{}{ + "rbd", "mirror", "pool", "promote", "pool", "--force"}...).Return("ok", nil).Once() + processExec = r + + // Method call + rh := RbdReplicationHandler{} + + // Test stardard promotion. + rh.Request.IsForceOp = false + err := handlePoolPromotion(&rh, "pool") + assert.ErrorContains(ks.T(), err, "use --force if you understand the risks of this operation") + + rh.Request.IsForceOp = true + err = handlePoolPromotion(&rh, "pool") + assert.NoError(ks.T(), err) +} + +func (ks *RbdMirrorSuite) TestDemotePoolOnSecondary() { + r := mocks.NewRunner(ks.T()) + + // mocks and expectations + r.On("RunCommand", []interface{}{ + "rbd", "mirror", "pool", "demote", "pool"}...).Return("ok", nil).Once() + processExec = r + + // Method call + rh := RbdReplicationHandler{} + + // Test stardard promotion. + err := handlePoolDemotion(&rh, "pool") + assert.NoError(ks.T(), err) +} diff --git a/microceph/ceph/replication.go b/microceph/ceph/replication.go index 7195ac00..184fe3c2 100644 --- a/microceph/ceph/replication.go +++ b/microceph/ceph/replication.go @@ -32,8 +32,11 @@ type ReplicationHandlerInterface interface { EnableHandler(ctx context.Context, args ...any) error DisableHandler(ctx context.Context, args ...any) error ConfigureHandler(ctx context.Context, args ...any) error - ListHandler(ctx context.Context, args ...any) error StatusHandler(ctx context.Context, args ...any) error + // Cluster wide Operations (don't require any pool/image info.) + ListHandler(ctx context.Context, args ...any) error + PromoteHandler(ctx context.Context, args ...any) error + DemoteHandler(ctx context.Context, args ...any) error } func GetReplicationHandler(name string) ReplicationHandlerInterface { @@ -57,7 +60,9 @@ func GetReplicationStateMachine(initialState ReplicationState) *stateless.StateM Permit(constants.EventEnableReplication, StateEnabledReplication). OnEntryFrom(constants.EventDisableReplication, disableHandler). InternalTransition(constants.EventListReplication, listHandler). - InternalTransition(constants.EventDisableReplication, disableHandler) + InternalTransition(constants.EventDisableReplication, disableHandler). + InternalTransition(constants.EventPromoteReplication, promoteHandler). + InternalTransition(constants.EventDemoteReplication, demoteHandler) // Configure transitions for enabled state. newFsm.Configure(StateEnabledReplication). @@ -66,7 +71,9 @@ func GetReplicationStateMachine(initialState ReplicationState) *stateless.StateM InternalTransition(constants.EventEnableReplication, enableHandler). InternalTransition(constants.EventConfigureReplication, configureHandler). InternalTransition(constants.EventListReplication, listHandler). - InternalTransition(constants.EventStatusReplication, statusHandler) + InternalTransition(constants.EventStatusReplication, statusHandler). + InternalTransition(constants.EventPromoteReplication, promoteHandler). + InternalTransition(constants.EventDemoteReplication, demoteHandler) // Check Event params type. var output *string @@ -80,6 +87,7 @@ func GetReplicationStateMachine(initialState ReplicationState) *stateless.StateM newFsm.SetTriggerParameters(constants.EventConfigureReplication, inputType, outputType, stateType) newFsm.SetTriggerParameters(constants.EventListReplication, inputType, outputType, stateType) newFsm.SetTriggerParameters(constants.EventStatusReplication, inputType, outputType, stateType) + newFsm.SetTriggerParameters(constants.EventPromoteReplication, inputType, outputType, stateType) // Add logger callback for all transitions newFsm.OnTransitioning(logTransitionHandler) @@ -125,3 +133,13 @@ func statusHandler(ctx context.Context, args ...any) error { logger.Infof("REPFSM: Entered Status Handler") return rh.StatusHandler(ctx, args...) } +func promoteHandler(ctx context.Context, args ...any) error { + rh := args[repArgHandler].(ReplicationHandlerInterface) + logger.Infof("REPFSM: Entered Status Handler") + return rh.PromoteHandler(ctx, args...) +} +func demoteHandler(ctx context.Context, args ...any) error { + rh := args[repArgHandler].(ReplicationHandlerInterface) + logger.Infof("REPFSM: Entered Status Handler") + return rh.DemoteHandler(ctx, args...) +} diff --git a/microceph/ceph/replication_rbd.go b/microceph/ceph/replication_rbd.go index 94f0b0de..db064431 100644 --- a/microceph/ceph/replication_rbd.go +++ b/microceph/ceph/replication_rbd.go @@ -238,7 +238,6 @@ func (rh *RbdReplicationHandler) StatusHandler(ctx context.Context, args ...any) } // Also add image info - resp = types.RbdPoolStatus{ Name: rh.Request.SourcePool, Type: string(rh.PoolInfo.Mode), @@ -292,6 +291,57 @@ func (rh *RbdReplicationHandler) StatusHandler(ctx context.Context, args ...any) return nil } +// PromoteHandler promotes sequentially promote all secondary cluster pools to primary. +func (rh *RbdReplicationHandler) PromoteHandler(ctx context.Context, args ...any) error { + // fetch all ceph pools initialised with rbd application. + pools := ListPools("rbd") + + // TODO: make this print debug + logger.Infof("REPRBD: Scan active pools %v", pools) + // fetch verbose pool status for each pool + successPoolList := []string{} + for _, pool := range pools { + err := handlePoolDePro(rh, pool.Name) + if err != nil { + logger.Errorf("Failed to perform (%s) operation on (%s) pool, quitting.", rh.Request.RequestType, pool.Name) + return err + } + + successPoolList = append(successPoolList, pool.Name) + } + + // TODO: Make this print debug. + logger.Infof("REPRBD: Operated %v pools for %s request.", successPoolList, rh.Request.RequestType) + return nil +} + +func (rh *RbdReplicationHandler) DemoteHandler(ctx context.Context, args ...any) error { + if !rh.Request.IsForceOp { + return fmt.Errorf("demotion may cause data loss on this cluster. %s", constants.CliForcePrompt) + } + + // fetch all ceph pools initialised with rbd application. + pools := ListPools("rbd") + + // TODO: make this print debug + logger.Infof("REPRBD: Scan active pools %v", pools) + // fetch verbose pool status for each pool + successPoolList := []string{} + for _, pool := range pools { + err := handlePoolDePro(rh, pool.Name) + if err != nil { + logger.Errorf("Failed to perform (%s) operation on (%s) pool, quitting.", rh.Request.RequestType, pool.Name) + return err + } + + successPoolList = append(successPoolList, pool.Name) + } + + // TODO: Make this print debug. + logger.Infof("REPRBD: Operated %v pools for %s request.", successPoolList, rh.Request.RequestType) + return nil +} + // ################### Helper Functions ################### // Enable handler for pool resource. func handlePoolEnablement(rh *RbdReplicationHandler, localSite string, remoteSite string) error { @@ -370,7 +420,7 @@ func handlePoolDisablement(rh *RbdReplicationHandler, localSite string, remoteSi // If pool in pool mirroring mode, disable all images. if rh.PoolInfo.Mode == types.RbdResourcePool { - err := DisableMirroringAllImagesInPool(rh.Request.SourcePool) + err := DisableAllMirroringImagesInPool(rh.Request.SourcePool) if err != nil { return err } @@ -400,24 +450,53 @@ func handleImageDisablement(rh *RbdReplicationHandler) error { return configureImageMirroring(rh.Request) } -// enableImageFeatures enables the list of rbd features on the requested resource. -func enableRbdImageFeatures(poolName string, imageName string, features []string) error { - for _, feature := range features { - err := configureImageFeatures(poolName, imageName, "enable", feature) - if err != nil && !strings.Contains(err.Error(), "one or more requested features are already enabled") { - return err - } +// handlePoolDePro verifies the pool state/ peers and +func handlePoolDePro(rh *RbdReplicationHandler, poolName string) error { + poolStatus, err := GetRbdMirrorPoolStatus(poolName, "", "") + if err != nil { + logger.Warnf("REPRBD: failed to fetch status for %s pool: %v", poolName, err) + return err } - return nil -} -// disableRbdImageFeatures disables the list of rbd features on the requested resource. -func disableRbdImageFeatures(poolName string, imageName string, features []string) error { - for _, feature := range features { - err := configureImageFeatures(poolName, imageName, "disable", feature) - if err != nil { - return err + poolInfo, err := GetRbdMirrorPoolInfo(poolName, "", "") + if err != nil { + logger.Warnf("REPRBD: failed to fetch status for %s pool: %v", poolName, err) + return err + } + + if poolStatus.State != StateEnabledReplication { + logger.Infof("REPRBD: pool(%s) is not an rbd mirror pool.", poolName) + return nil + } + + logger.Infof("REPRBD: op(%s), pool(%s)", rh.Request.RequestType, poolName) + + for _, peer := range poolInfo.Peers { + logger.Infof("REPRBD: op(%s), pool(%s), peer(%s)", rh.Request.RequestType, poolName, peer.RemoteName) + // Perform operation only if the remote cluster is a known peer. + if peer.RemoteName == rh.Request.RemoteName { + logger.Infof("REPRBD: YES") + if rh.Request.RequestType == types.PromoteReplicationRequest { + logger.Infof("REPRBD: PROMOTE") + // Promote the pool to primary. + err := handlePoolPromotion(rh, poolName) + if err != nil { + // Stop if failed. + logger.Errorf("REPRBD: failed to promote local site to primary: %v", err) + return err + } + } else if rh.Request.RequestType == types.DemoteReplicationRequest { + logger.Infof("REPRBD: DEMOTE") + // Demote the pool to secondary. + err := handlePoolDemotion(rh, poolName) + if err != nil { + // Stop if failed. + logger.Errorf("REPRBD: failed to demote local site to secondary: %v", err) + return err + } + } } } + return nil } diff --git a/microceph/ceph/test_assets/rbd_mirror_promote_secondary_failure.txt b/microceph/ceph/test_assets/rbd_mirror_promote_secondary_failure.txt new file mode 100644 index 00000000..a8d4ea0b --- /dev/null +++ b/microceph/ceph/test_assets/rbd_mirror_promote_secondary_failure.txt @@ -0,0 +1,10 @@ +rbd: failed to 2024-10-09T11:00:10.804+0000 7f65a4bce6c0 -1 librbd::mirror::PromoteRequest: 0x7f6588018e20 handle_get_info: image is primary within a remote cluster or demotion is not propagated yet +promote image image_one: (16) Device or resource busy +2024-10-09T11:00:10.804+0000 7f65a4bce6c0 -1 librbd::io::AioCompletion: 0x7f65980061c0 fail: (16) Device or resource busy +2024-10-09T11:00:10.808+0000 7f65a4bce6c0 -1 librbd::mirror::PromoteRequest: 0x7f658c008c50 handle_get_info: image is primary within a remote cluster or demotion is not propagated yet +2024-10-09T11:00:10.808+0000 7f65a4bce6c0 -1 librbd::io::AioCompletion: 0x7f65980061c0 fail: (16) Device or resource busy +rbd: failed to promote image image_two: (16) Device or resource busy +2024-10-09T11:00:10.812+0000 7f65a53cf6c0 -1 librbd::mirror::PromoteRequest: 0x7f6588018e20 handle_get_info: image is primary within a remote cluster or demotion is not propagated yet +2024-10-09T11:00:10.812+0000 7f65a53cf6c0 -1 librbd::io::AioCompletion: 0x7f658c0069e0 fail: (16) Device or resource busy +rbd: failed to promote image image_three: (16) Device or resource busy +Promoted 0 mirrored images \ No newline at end of file diff --git a/microceph/client/remote_replication.go b/microceph/client/remote_replication.go index cd15f807..95c7c587 100644 --- a/microceph/client/remote_replication.go +++ b/microceph/client/remote_replication.go @@ -7,7 +7,6 @@ import ( "github.com/canonical/lxd/shared/api" "github.com/canonical/microceph/microceph/api/types" - "github.com/canonical/microceph/microceph/constants" microCli "github.com/canonical/microcluster/v2/client" ) @@ -18,8 +17,10 @@ func SendRemoteReplicationRequest(ctx context.Context, c *microCli.Client, data queryCtx, cancel := context.WithTimeout(ctx, time.Second*120) defer cancel() - if data.GetWorkloadRequestType() == constants.EventListReplication { - // list request uses replication/$workload endpoint + // If no API object provided, create API request to the root endpoint. + if len(data.GetAPIObjectId()) == 0 { + // uses replication/$workload endpoint + fmt.Printf("1, %s", string(data.GetWorkloadType())) err = c.Query( queryCtx, data.GetAPIRequestType(), types.ExtendedPathPrefix, api.NewURL().Path("ops", "replication", string(data.GetWorkloadType())), @@ -27,6 +28,7 @@ func SendRemoteReplicationRequest(ctx context.Context, c *microCli.Client, data ) } else { // Other requests use replication/$workload/$resource endpoint + fmt.Printf("2, %s, %s", string(data.GetWorkloadType()), data.GetAPIObjectId()) err = c.Query( queryCtx, data.GetAPIRequestType(), types.ExtendedPathPrefix, api.NewURL().Path("ops", "replication", string(data.GetWorkloadType()), data.GetAPIObjectId()), diff --git a/microceph/cmd/microceph/remote_replication_rbd.go b/microceph/cmd/microceph/remote_replication_rbd.go index 4a595c0a..cadfe3c4 100644 --- a/microceph/cmd/microceph/remote_replication_rbd.go +++ b/microceph/cmd/microceph/remote_replication_rbd.go @@ -34,5 +34,13 @@ func (c *cmdRemoteReplicationRbd) Command() *cobra.Command { remoteReplicationRbdConfigureCmd := cmdRemoteReplicationConfigureRbd{common: c.common} cmd.AddCommand(remoteReplicationRbdConfigureCmd.Command()) + // Replication promote command + remoteReplicationRbdPromoteCmd := cmdRemoteReplicationPromoteRbd{common: c.common} + cmd.AddCommand(remoteReplicationRbdPromoteCmd.Command()) + + // Replication demote command + remoteReplicationRbdDemoteCmd := cmdRemoteReplicationDemoteRbd{common: c.common} + cmd.AddCommand(remoteReplicationRbdDemoteCmd.Command()) + return cmd } diff --git a/microceph/cmd/microceph/remote_replication_rbd_demote.go b/microceph/cmd/microceph/remote_replication_rbd_demote.go new file mode 100644 index 00000000..8be123b5 --- /dev/null +++ b/microceph/cmd/microceph/remote_replication_rbd_demote.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/v2/microcluster" + "github.com/spf13/cobra" +) + +type cmdRemoteReplicationDemoteRbd struct { + common *CmdControl + remoteName string + isForce bool +} + +func (c *cmdRemoteReplicationDemoteRbd) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "demote", + Short: "Demote a primary cluster to non-primary status", + RunE: c.Run, + } + + cmd.Flags().StringVar(&c.remoteName, "remote", "", "remote MicroCeph cluster name") + cmd.Flags().BoolVar(&c.isForce, "yes-i-really-mean-it", false, "demote cluster irrespective of data loss") + cmd.MarkFlagRequired("remote") + return cmd +} + +func (c *cmdRemoteReplicationDemoteRbd) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + m, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagStateDir}) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + payload, err := c.prepareRbdPayload(types.DemoteReplicationRequest) + if err != nil { + return err + } + + _, err = client.SendRemoteReplicationRequest(context.Background(), cli, payload) + if err != nil { + return err + } + + return nil +} + +func (c *cmdRemoteReplicationDemoteRbd) prepareRbdPayload(requestType types.ReplicationRequestType) (types.RbdReplicationRequest, error) { + retReq := types.RbdReplicationRequest{ + RemoteName: c.remoteName, + RequestType: requestType, + ResourceType: types.RbdResourcePool, + SourcePool: "", + IsForceOp: c.isForce, + } + + return retReq, nil +} diff --git a/microceph/cmd/microceph/remote_replication_rbd_promote.go b/microceph/cmd/microceph/remote_replication_rbd_promote.go new file mode 100644 index 00000000..0f6bd194 --- /dev/null +++ b/microceph/cmd/microceph/remote_replication_rbd_promote.go @@ -0,0 +1,69 @@ +package main + +import ( + "context" + + "github.com/canonical/microceph/microceph/api/types" + "github.com/canonical/microceph/microceph/client" + "github.com/canonical/microcluster/v2/microcluster" + "github.com/spf13/cobra" +) + +type cmdRemoteReplicationPromoteRbd struct { + common *CmdControl + remoteName string + isForce bool +} + +func (c *cmdRemoteReplicationPromoteRbd) Command() *cobra.Command { + cmd := &cobra.Command{ + Use: "promote", + Short: "Promote a non-primary cluster to primary status", + RunE: c.Run, + } + + cmd.Flags().StringVar(&c.remoteName, "remote", "", "remote MicroCeph cluster name") + cmd.Flags().BoolVar(&c.isForce, "force", false, "forcefully promote site to primary") + cmd.MarkFlagRequired("remote") + return cmd +} + +func (c *cmdRemoteReplicationPromoteRbd) Run(cmd *cobra.Command, args []string) error { + if len(args) != 0 { + return cmd.Help() + } + + m, err := microcluster.App(microcluster.Args{StateDir: c.common.FlagStateDir}) + if err != nil { + return err + } + + cli, err := m.LocalClient() + if err != nil { + return err + } + + payload, err := c.prepareRbdPayload(types.PromoteReplicationRequest) + if err != nil { + return err + } + + _, err = client.SendRemoteReplicationRequest(context.Background(), cli, payload) + if err != nil { + return err + } + + return nil +} + +func (c *cmdRemoteReplicationPromoteRbd) prepareRbdPayload(requestType types.ReplicationRequestType) (types.RbdReplicationRequest, error) { + retReq := types.RbdReplicationRequest{ + RemoteName: c.remoteName, + RequestType: requestType, + IsForceOp: c.isForce, + ResourceType: types.RbdResourcePool, + SourcePool: "", + } + + return retReq, nil +} diff --git a/microceph/constants/constants.go b/microceph/constants/constants.go index 4fe2b4a8..4a82da31 100644 --- a/microceph/constants/constants.go +++ b/microceph/constants/constants.go @@ -79,3 +79,6 @@ const EventConfigureReplication = "configure_replication" // Rbd features var RbdJournalingEnableFeatureSet = [...]string{"exclusive-lock", "journaling"} + +const EventPromoteReplication = "promote_replication" +const EventDemoteReplication = "demote_replication" diff --git a/microceph/go.mod b/microceph/go.mod index 43c47cfe..70134a4b 100644 --- a/microceph/go.mod +++ b/microceph/go.mod @@ -18,7 +18,7 @@ require ( github.com/stretchr/testify v1.9.0 github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 - gopkg.in/yaml.v2 v2.4.0 + golang.org/x/crypto v0.27.0 ) require ( @@ -62,11 +62,11 @@ require ( go.opentelemetry.io/otel v1.30.0 // indirect go.opentelemetry.io/otel/metric v1.30.0 // indirect go.opentelemetry.io/otel/trace v1.30.0 // indirect - golang.org/x/crypto v0.27.0 // indirect golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect golang.org/x/term v0.24.0 // indirect golang.org/x/text v0.18.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/microceph/go.sum b/microceph/go.sum index 45544a73..5c43320f 100644 --- a/microceph/go.sum +++ b/microceph/go.sum @@ -459,8 +459,8 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.29.0 h1:5ORfpBpCs4HzDYoodCDBbwHzdR5UrLBZ3sOnUJmFoHo= -golang.org/x/net v0.29.0/go.mod h1:gLkgy8jTGERgjzMic6DS9+SP0ajcu6Xu3Orq/SpETg0= +golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys= +golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= diff --git a/tests/scripts/actionutils.sh b/tests/scripts/actionutils.sh index 9131959f..142dd430 100755 --- a/tests/scripts/actionutils.sh +++ b/tests/scripts/actionutils.sh @@ -288,6 +288,88 @@ function remote_verify_rbd_mirroring() { lxc exec node-wrk3 -- sh -c "sudo microceph remote replication rbd list" | grep "pool_two.*image_two" } +function remote_failover_to_siteb() { + set -eux + + # check images are secondary on siteb + img_count=$(lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd list" | grep -c "\"is_primary\":false") + if [[ $img_count -lt 1 ]]; then + echo "Site B has $img_count secondary images" + exit -1 + fi + + # promote site b to primary + lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd promote --force" + + # wait for the site images to show as primary + for index in {1..100}; do + echo "Check run #$index" + list_output=$(lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd list --json") + echo $list_output + images=$(echo $list_output | jq .[].Images) + echo $images + is_primary_count=$(echo $images | grep -c "\"is_primary\": true" || true) + echo $is_primary_count + if [[ $is_primary_count -gt 0 ]] ; then + break + fi + + echo "#################" + sleep 30 + done + + # resolve the split brain situation by demoting the old primary. + lxc exec node-wrk0 -- sh -c "sudo microceph remote replication rbd demote" + + # wait for the site images to show as non-primary + for index in {1..100}; do + echo "Check run #$index" + list_output=$(lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd list --json") + echo $list_output + images=$(echo $list_output | jq .[].Images) + echo $images + is_primary_count=$(echo $images | grep -c "\"is_primary\": false" || true) + echo $is_primary_count + if [[ $is_primary_count -gt 0 ]] ; then + break + fi + + echo "#################" + sleep 30 + done +} + +function remote_failback_to_sitea() { + set -eux + + # check images are secondary on siteb + img_count=$(lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd list" | grep -c "\"is_primary\":false") + if [[ $img_count -lt 1 ]]; then + echo "Site A has $img_count secondary images" + exit -1 + fi + + # demote the current primary + lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd demote" + + # wait for the site images to show as non-primary + for index in {1..100}; do + echo "Check run #$index" + list_output=$(lxc exec node-wrk2 -- sh -c "sudo microceph remote replication rbd list --json") + echo $list_output + images=$(echo $list_output | jq .[].Images) + echo $images + is_primary_count=$(echo $images | grep -c "\"is_primary\": false" || true) + echo $is_primary_count + if [[ $is_primary_count -gt 0 ]] ; then + break + fi + + echo "#################" + sleep 30 + done +} + function remote_disable_rbd_mirroring() { set -eux # check disables fail for image mirroring pools with images currently being mirrored