Skip to content

Commit

Permalink
minor changes implemented in the code and included osd resize.
Browse files Browse the repository at this point in the history 
Signed-off-by: nagendra202 <[email protected]>
  • Loading branch information
@nagendra202
nagendra202 committed Sep 12, 2024
1 parent 923e2bb commit 6b2c183
Show file tree
Hide file tree
Showing 2 changed files with 101 additions and 55 deletions.
42 changes: 31 additions & 11 deletions ocs_ci/helpers/keyrotation_helper.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -311,7 +311,6 @@ def get_osd_dm_crypt(self, device):
"""
cmd = f" get secret rook-ceph-osd-encryption-key-{device} -o jsonpath='{{.data.dmcrypt-key}}'"

dmcrypt_key = self._exec_oc_cmd(cmd=cmd, out_yaml_format=False)
log.info(f"dmcrypt-key of device {device} is {dmcrypt_key}")
return dmcrypt_key
Expand Down Expand Up @@ -409,34 +408,53 @@ def wait_till_keyrotation(self, device_handle):

return True


def enable_key_rotation():
# Enable Keyrotation and verify its enable status at Noobaa and storagecluster end.
"""
Enable Key rotation and verify its status for Noobaa and Storage cluster.
"""
osd_keyrotation = OSDKeyrotation()
noobaa_keyrotation = NoobaaKeyrotation()
osd_keyrotation.enable_keyrotation()
noobaa_keyrotation.enable_keyrotation()

assert (
osd_keyrotation.is_keyrotation_enable()
), "Encryption Key rotation is not enabled for OSDs"
assert (
noobaa_keyrotation.is_keyrotation_enable
), "Keyrotation is not enabled in the storagecluster object."
assert (
noobaa_keyrotation.is_noobaa_keyrotation_enable
), "Keyrotation is not enabled in the noobaa object."
def set_key_rotatio_time(value):
""" value = no.of minutes"""
osd_keyrotation = OSDKeyrotation()
#noobaa_keyrotation = NoobaaKeyrotation()


def set_key_rotation_time(value):
"""
This function will edit the storage cluster and add the value in key rotation schedule
Args:
value: (str) number of minutes
"""
osd_keyrotation = OSDKeyrotation()
schedule = f"*/{value} * * * *"
osd_keyrotation.set_keyrotation_schedule(schedule)


def verify_key_rotation_time(schedule):
# Verify Keyrotation schedule changed at storagecluster object and rook object.
"""
Verify Key rotation schedule changed at storage cluster, rook and Noobaa object.
Args:
schedule: (str)
"""
osd_keyrotation = OSDKeyrotation()
noobaa_keyrotation = NoobaaKeyrotation()
assert (
osd_keyrotation.get_keyrotation_schedule() == schedule
), f"Keyrotation schedule is not set to {schedule} minutes in storagecluster object."
), f"Keyrotation schedule is not set to {schedule} minutes in storage cluster object."
assert (
osd_keyrotation.get_osd_keyrotation_schedule() == schedule
), "KeyRotation is not enabled in the Rook Object."
Expand All @@ -445,16 +463,16 @@ def verify_key_rotation_time(schedule):
), f"Keyrotation schedule is not set to every {schedule} minutes in Noobaa object."


def verify_new_key_after_rotation(tries,delays):
def verify_new_key_after_rotation(tries, delays):
osd_keyrotation = OSDKeyrotation()
noobaa_keyrotation = NoobaaKeyrotation()

# Recored existing OSD keys before rotation is happened.
log.info("Record existing OSD keys before rotation is happened.")
osd_keys_before_rotation = {}
for device in osd_keyrotation.deviceset:
osd_keys_before_rotation[device] = osd_keyrotation.get_osd_dm_crypt(device)

# Recoard Noobaa volume and backend keys before rotation.
log.info("Record Noobaa volume and backend keys before rotation.")
(
old_noobaa_backend_key,
old_noobaa_backend_secret,
Expand All @@ -475,6 +493,7 @@ def verify_new_key_after_rotation(tries,delays):
def compare_old_keys_with_new_keys():
"""
Compare old keys with new keys.
"""
(
new_noobaa_backend_key,
Expand All @@ -497,6 +516,7 @@ def compare_old_keys_with_new_keys():
log.info(
f"Noobaa Volume key rotated {new_noobaa_volume_key} : {new_noobaa_volume_secret}"
)

try:
osd_keyrotation.verify_keyrotation(osd_keys_before_rotation, tries=10, delay=30)
compare_old_keys_with_new_keys()
Expand Down
114 changes: 70 additions & 44 deletions tests/cross_functional/system_test/test_cluster_wide_key_rotation_system.py
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,28 @@
import logging
import pytest
from ocs_ci.helpers.keyrotation_helper import (
verify_key_rotation_time,
enable_key_rotation,
set_key_rotatio_time,
verify_new_key_after_rotation
)
from ocs_ci.framework.testlib import E2ETest

from ocs_ci.framework import config
from ocs_ci.framework.pytest_customization.marks import (
system_test,
magenta_squad,
ignore_leftovers,
)
from ocs_ci.ocs import constants
from ocs_ci.framework import config
from ocs_ci.ocs.ocp import OCP
from ocs_ci.helpers.sanity_helpers import Sanity
from ocs_ci.framework.testlib import E2ETest
from ocs_ci.helpers.e2e_helpers import (
validate_noobaa_rebuild_system,
validate_noobaa_db_backup_recovery_locally_system,
Run_fio_till_cluster_full,
)
from ocs_ci.ocs import warp
from ocs_ci.helpers.keyrotation_helper import (
verify_key_rotation_time,
enable_key_rotation,
set_key_rotation_time,
verify_new_key_after_rotation,
)
from ocs_ci.helpers.osd_resize import basic_resize_osd, get_storage_size
from ocs_ci.helpers.sanity_helpers import Sanity
from ocs_ci.ocs import constants, warp
from ocs_ci.ocs.ocp import OCP

log = logging.getLogger(__name__)

Expand All @@ -36,15 +37,18 @@ class TestKeyRotationWithClusterFull(E2ETest):
2. Set custom taints to all the worker nodes and make sure #1 and #2 still exists after rebooting the pods.
a) Apply custom taint to all nodes
b) Add toleration in storagecluster CR and odf-operator subscription.
c) Verify toleration applied in ODF subscription and Storage Cluster CR are reflecting for other subscriptions, Ceph and nooba components or not.
c) Verify toleration applied in ODF subscription and Storage Cluster CR are reflecting for other subscriptions,
Ceph and nooba components or not.
d) Verify the pods in all nodes are running as per taints applied.
e) Restart all ocs the pods on all nodes.
3. Fill the cluster till the full ratio limits (85%) by running IO from multiple pods and verify key rotation.
4. Once the cluster reaches read-only state, resize the OSD and verify key rotation.
5. Run noobaa_rebuild test and verify key rotation still works as it is.
6. Run noobaa-db backup and recovery locally. Verify key rotation.
"""

@pytest.fixture(autouse=True)
def init_sanity(self):
"""
Expand All @@ -53,6 +57,10 @@ def init_sanity(self):
self.sanity_helpers = Sanity()

def noobaa_rebuild_cleanup(self):
"""
Cleanup function which clears all the noobaa rebuild entries.
"""
# Get the deployment replica count
deploy_obj = OCP(
kind=constants.DEPLOYMENT,
Expand Down Expand Up @@ -80,63 +88,81 @@ def teardown():
request.addfinalizer(teardown)
return warps3

def test_cluster_wide_encryption_key_rotation_system(self,
teardown_project_factory,
pvc_factory,
pod_factory,
threading_lock,
bucket_factory_session,
mcg_obj_session,
bucket_factory,
noobaa_db_backup_and_recovery_locally,
warps3,
):
def test_cluster_wide_encryption_key_rotation_system(
self,
teardown_project_factory,
pvc_factory,
pod_factory,
threading_lock,
bucket_factory_session,
mcg_obj_session,
bucket_factory,
noobaa_db_backup_and_recovery_locally,
warps3,
):
time_interval_to_rotate_key_in_minutes = str(5)
tries = 10
delays = int(time_interval_to_rotate_key_in_minutes)*60/tries
delays = int(time_interval_to_rotate_key_in_minutes) * 60 / tries
log.info("Enabling the key rotation if not done")
enable_key_rotation()
log.info("Setting the key rotation time by editing storage cluster")
set_key_rotatio_time(time_interval_to_rotate_key_in_minutes)
set_key_rotation_time(time_interval_to_rotate_key_in_minutes)
schedule = f"*/{time_interval_to_rotate_key_in_minutes} * * * *"
log.info("Verifying the key rotation time set properly or not")
verify_key_rotation_time(schedule=schedule)
log.info("Verifying the new key generated by comparing it with older key")
verify_new_key_after_rotation(tries, delays)

# Custom taints PR 9808 not yet merged. Will work on it once the merge complete.
# TODO: Custom taints PR 9808 not yet merged. Will include that part once the merge completes.

run_fio_obj = Run_fio_till_cluster_full()
run_fio_obj.run_cluster_full_fio(teardown_project_factory, pvc_factory, pod_factory)
log.info("Verifying the key rotation time is still unchanged after 85% cluster full")
run_fio_obj.run_cluster_full_fio(
teardown_project_factory, pvc_factory, pod_factory
)
log.info(
"Verifying the key rotation time is still unchanged after 85% cluster full"
)
verify_key_rotation_time(schedule=schedule)
log.info("After cluster full 85%, verifying the new key generated by comparing it with older key")
log.info(
"After cluster full 85%, verifying the new key generated by comparing it with older key"
)
verify_new_key_after_rotation(tries, delays)

# Below feature dropped from 4.16 release. Will enable once the feature releases in the future.
#log.info("Performing OSD resize")
# basic_resize_osd(get_storage_size())
#log.info("After OSD resize, checking the key rotation time is unchanged")
# verify_key_rotation_time(schedule=schedule)
#log.info("After OSD resize, verifying the new key generated by comparing it with older key")
# verify_new_key_after_rotation(tries, delays)
# TODO:OSD-resize feature dropped from 4.16 release. It is supported only from 4.17 onwards.
log.info("Performing OSD resize")
basic_resize_osd(get_storage_size())
log.info("After OSD resize, checking the key rotation time is unchanged")
verify_key_rotation_time(schedule=schedule)
log.info(
"After OSD resize, verifying the new key generated by comparing it with older key"
)
verify_new_key_after_rotation(tries, delays)

run_fio_obj.cleanup()
log.info("Triggering noobaa rebuild test")
validate_noobaa_rebuild_system(self, bucket_factory_session, mcg_obj_session)
log.info("After noobaa rebuild, checking the key rotation time is unchanged")
verify_key_rotation_time(schedule=schedule)
log.info("After noobaa rebuild, verifying the new key generated by comparing it with older key")
log.info(
"After noobaa rebuild, verifying the new key generated by comparing it with older key"
)
verify_new_key_after_rotation(tries, delays)
log.info("Starting noobaa rebuild cleanup activity")
self.noobaa_rebuild_cleanup()

log.info("Triggering noobaa db backup and recovery locally")
validate_noobaa_db_backup_recovery_locally_system(self,
bucket_factory,
noobaa_db_backup_and_recovery_locally,
warps3,mcg_obj_session)
log.info("After noobaa db backup and recovery, checking the key rotation time is unchanged")
validate_noobaa_db_backup_recovery_locally_system(
self,
bucket_factory,
noobaa_db_backup_and_recovery_locally,
warps3,
mcg_obj_session,
)
log.info(
"After noobaa db backup and recovery, checking the key rotation time is unchanged"
)
verify_key_rotation_time(schedule=schedule)
log.info("After noobaa db backup and recovery, verifying the new key generated by comparing it with older key")
log.info(
"After noobaa db backup and recovery, verifying the new key generated by comparing it with older key"
)
verify_new_key_after_rotation(tries, delays)

0 comments on commit 6b2c183

Please sign in to comment.