Merge pull request #6 from dell/release_1.5.0

Release version 1.5.0 for PyPowerFlex

ChangeLog.md

@@ -1,5 +1,8 @@
 # PyPowerFlex Change Log
 
+## Version 1.5.0 - released on 27/09/22
+- Added support for 4.0.x release of PowerFlex OS, adding statistics data for storage pool and volume objects.
+
 ## Version 1.4.0 - released on 28/06/22
 - Added configuration operations includes adding/removing of standby MDM, change cluster ownership, change cluster mode, modify performance profile, rename MDM, modify MDM's virtual interface and getting details of MDM cluster entities.
 

PyPowerFlex/__init__.py

@@ -41,7 +41,8 @@ class PowerFlexClient:
         'storage_pool',
         'acceleration_pool',
         'system',
-        'volume'
+        'volume',
+        'utility'
     )
 
     def __init__(self,
@@ -86,6 +87,7 @@ def initialize(self):
                                   objects.AccelerationPool)
         self.__add_storage_entity('system', objects.System)
         self.__add_storage_entity('volume', objects.Volume)
+        self.__add_storage_entity('utility', objects.PowerFlexUtility)
         utils.init_logger(self.configuration.log_level)
         if version.parse(self.system.api_version()) < version.Version('3.0'):
             raise exceptions.PowerFlexClientException(

PyPowerFlex/base_client.py

@@ -30,6 +30,7 @@ class Request:
     def __init__(self, token, configuration):
         self.token = token
         self.configuration = configuration
+        self.__refresh_token = None
 
     @property
     def base_url(self):
@@ -38,6 +39,13 @@ def base_url(self):
             port=self.configuration.gateway_port
         )
 
+    @property
+    def auth_url(self):
+        return 'https://{address}:{port}/rest/auth'.format(
+            address=self.configuration.gateway_address,
+            port=self.configuration.gateway_port
+        )
+
     @property
     def headers(self):
         return {'content-type': 'application/json'}
@@ -54,22 +62,23 @@ def verify_certificate(self):
 
     def send_get_request(self, url, **url_params):
         request_url = self.base_url + url.format(**url_params)
-        self._login()
+        version = self.login()
         r = requests.get(request_url,
                          auth=(
                              self.configuration.username,
                              self.token.get()
                          ),
                          verify=self.verify_certificate,
                          timeout=self.configuration.timeout)
-        self._logout()
+
+        self.logout(version)
         response = r.json()
         return r, response
 
     def send_post_request(self, url, params=None, **url_params):
         if params is None:
             params = dict()
-        self._login()
+        version = self.login()
         request_url = self.base_url + url.format(**url_params)
         r = requests.post(request_url,
                           auth=(
@@ -81,14 +90,14 @@ def send_post_request(self, url, params=None, **url_params):
                           verify=self.verify_certificate,
                           timeout=self.configuration.timeout)
         response = r.json()
-        self._logout()
+        self.logout(version)
         return r, response
 
     def send_mdm_cluster_post_request(self, url, params=None, **url_params):
         if params is None:
             params = dict()
         response = None
-        self._login()
+        version = self.login()
         request_url = self.base_url + url.format(**url_params)
         r = requests.post(request_url,
                           auth=(
@@ -102,9 +111,77 @@ def send_mdm_cluster_post_request(self, url, params=None, **url_params):
 
         if r.content != b'':
             response = r.json()
-        self._logout()
+        self.logout(version)
         return r, response
 
+    # To perform login based on the API version
+    def login(self):
+        version = self.get_api_version()
+        if utils.check_version(version=version):
+            self._appliance_login()
+        else:
+            self._login()
+        return version
+
+    # To perform logout based on the API version
+    def logout(self, version):
+        if utils.check_version(version=version):
+            self._appliance_logout()
+        else:
+            self._logout()
+
+    # Get the Current API version
+    def get_api_version(self):
+        request_url = self.base_url + '/version'
+        self._login()
+        r = requests.get(request_url,
+                         auth=(
+                             self.configuration.username,
+                             self.token.get()),
+                         verify=self.verify_certificate,
+                         timeout=self.configuration.timeout)
+        response = r.json()
+        return response
+
+    # API Login method for 4.0 and above.
+    def _appliance_login(self):
+        request_url = self.auth_url + '/login'
+        payload = {"username": "%s" % self.configuration.username,
+                   "password": "%s" % self.configuration.password
+                   }
+        r = requests.post(request_url, headers=self.headers, json=payload,
+                          verify=self.verify_certificate,
+                          timeout=self.configuration.timeout
+                          )
+        if r.status_code != requests.codes.ok:
+            exc = exceptions.PowerFlexFailQuerying('token')
+            LOG.error(exc.message)
+            raise exc
+        response = r.json()
+        token = response['access_token']
+        self.token.set(token)
+        self.__refresh_token = response['refresh_token']
+
+    # API logout method for 4.0 and above.
+    def _appliance_logout(self):
+        request_url = self.auth_url + '/logout'
+        token = self.token.get()
+        headers = {'Authorization': 'Bearer {0}'.format(token),
+                   'content-type': 'application/json'
+                   }
+        data = {'refresh_token': '{0}'.format(self.__refresh_token)}
+        r = requests.post(request_url, headers=headers, json=data,
+                          verify=self.verify_certificate,
+                          timeout=self.configuration.timeout
+                          )
+
+        if r.status_code != requests.codes.no_content:
+            exc = exceptions.PowerFlexFailQuerying('token')
+            LOG.error(exc.message)
+            raise exc
+        self.token.set("")
+        self.__refresh_token = None
+
     def _login(self):
         request_url = self.base_url + '/login'
 
@@ -148,6 +225,7 @@ class EntityRequest(Request):
     base_relationship_url = base_entity_url + '/relationships/{related}'
     base_object_url = '/instances/{entity}/action/{action}'
     query_mdm_cluster_url = '/instances/{entity}/queryMdmCluster'
+    list_statistics_url = '/types/{entity}/instances/action/{action}'
     entity_name = None
 
     @property

PyPowerFlex/constants.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2022 Dell Inc. or its subsidiaries.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+class StoragePoolConstants:
+    DEFAULT_STATISTICS_PROPERTIES = [
+        "backgroundScanFixedReadErrorCount","pendingMovingOutBckRebuildJobs",
+        "degradedHealthyCapacityInKb","activeMovingOutFwdRebuildJobs",
+        "bckRebuildWriteBwc","netFglUncompressedDataSizeInKb","primaryReadFromDevBwc","BackgroundScannedInMB","volumeIds",
+        "maxUserDataCapacityInKb","persistentChecksumBuilderProgress","rfcacheReadsSkippedAlignedSizeTooLarge",
+        "pendingMovingInRebalanceJobs","rfcacheWritesSkippedHeavyLoad","unusedCapacityInKb","userDataSdcReadLatency",
+        "totalReadBwc","numOfDeviceAtFaultRebuilds","totalWriteBwc","persistentChecksumCapacityInKb","rmPendingAllocatedInKb",
+        "numOfVolumes","rfcacheIosOutstanding","capacityAvailableForVolumeAllocationInKb","numOfMappedToAllVolumes",
+        "netThinUserDataCapacityInKb","backgroundScanFixedCompareErrorCount","volMigrationWriteBwc","thinAndSnapshotRatio",
+        "fglUserDataCapacityInKb","pendingMovingInEnterProtectedMaintenanceModeJobs","activeMovingInNormRebuildJobs",
+        "aggregateCompressionLevel","targetOtherLatency","netUserDataCapacityInKb","pendingMovingOutExitProtectedMaintenanceModeJobs",
+        "overallUsageRatio","volMigrationReadBwc","netCapacityInUseNoOverheadInKb","pendingMovingInBckRebuildJobs",
+        "rfcacheReadsSkippedInternalError","activeBckRebuildCapacityInKb","rebalanceCapacityInKb","pendingMovingInExitProtectedMaintenanceModeJobs",
+        "rfcacheReadsSkippedLowResources","rplJournalCapAllowed","thinCapacityInUseInKb","userDataSdcTrimLatency",
+        "activeMovingInEnterProtectedMaintenanceModeJobs","rfcacheWritesSkippedInternalError","netUserDataCapacityNoTrimInKb",
+        "rfcacheWritesSkippedCacheMiss","degradedFailedCapacityInKb","activeNormRebuildCapacityInKb","fglSparesInKb",
+        "snapCapacityInUseInKb","numOfMigratingVolumes","compressionRatio","rfcacheWriteMiss","primaryReadFromRmcacheBwc",
+        "migratingVtreeIds","numOfVtrees","userDataCapacityNoTrimInKb","rfacheReadHit","compressedDataCompressionRatio",
+        "rplUsedJournalCap","pendingMovingCapacityInKb","numOfSnapshots","pendingFwdRebuildCapacityInKb","tempCapacityInKb",
+        "totalFglMigrationSizeInKb","normRebuildCapacityInKb","logWrittenBlocksInKb","primaryWriteBwc","numOfThickBaseVolumes",
+        "enterProtectedMaintenanceModeReadBwc","activeRebalanceCapacityInKb","numOfReplicationJournalVolumes","rfcacheReadsSkippedLockIos",
+        "unreachableUnusedCapacityInKb","netProvisionedAddressesInKb","trimmedUserDataCapacityInKb","provisionedAddressesInKb",
+        "numOfVolumesInDeletion","pendingMovingOutFwdRebuildJobs","maxCapacityInKb","rmPendingThickInKb","protectedCapacityInKb",
+        "secondaryWriteBwc","normRebuildReadBwc","thinCapacityAllocatedInKb","netFglUserDataCapacityInKb","metadataOverheadInKb",
+        "rebalanceWriteBwc","primaryVacInKb","deviceIds","netSnapshotCapacityInKb","secondaryVacInKb",
+        "numOfDevices","rplTotalJournalCap","failedCapacityInKb","netMetadataOverheadInKb","activeMovingOutBckRebuildJobs",
+        "rfcacheReadsFromCache","activeMovingOutEnterProtectedMaintenanceModeJobs","enterProtectedMaintenanceModeCapacityInKb",
+        "pendingMovingInNormRebuildJobs","failedVacInKb","primaryReadBwc","fglUncompressedDataSizeInKb",
+        "fglCompressedDataSizeInKb","pendingRebalanceCapacityInKb","rfcacheAvgReadTime","semiProtectedCapacityInKb",
+        "pendingMovingOutEnterProtectedMaintenanceModeJobs","mgUserDdataCcapacityInKb","snapshotCapacityInKb",
+        "netMgUserDataCapacityInKb","fwdRebuildReadBwc","rfcacheWritesReceived","netUnusedCapacityInKb",
+        "protectedVacInKb","activeMovingRebalanceJobs","bckRebuildCapacityInKb","activeMovingInFwdRebuildJobs","netTrimmedUserDataCapacityInKb",
+        "pendingMovingRebalanceJobs","numOfMarkedVolumesForReplication","degradedHealthyVacInKb","semiProtectedVacInKb",
+        "userDataReadBwc","pendingBckRebuildCapacityInKb","capacityLimitInKb","vtreeIds","activeMovingCapacityInKb",
+        "targetWriteLatency","pendingExitProtectedMaintenanceModeCapacityInKb","rfcacheIosSkipped","userDataWriteBwc",
+        "inMaintenanceVacInKb","exitProtectedMaintenanceModeReadBwc","netFglSparesInKb","rfcacheReadsSkipped",
+        "activeExitProtectedMaintenanceModeCapacityInKb","activeMovingOutExitProtectedMaintenanceModeJobs","numOfUnmappedVolumes",
+        "tempCapacityVacInKb","volumeAddressSpaceInKb","currentFglMigrationSizeInKb","rfcacheWritesSkippedMaxIoSize",
+        "netMaxUserDataCapacityInKb","numOfMigratingVtrees","atRestCapacityInKb","rfacheWriteHit","bckRebuildReadBwc",
+        "rfcacheSourceDeviceWrites","spareCapacityInKb","enterProtectedMaintenanceModeWriteBwc","rfcacheIoErrors","inaccessibleCapacityInKb",
+        "normRebuildWriteBwc","capacityInUseInKb","rebalanceReadBwc","rfcacheReadsSkippedMaxIoSize","activeMovingInExitProtectedMaintenanceModeJobs",
+        "secondaryReadFromDevBwc","secondaryReadBwc","rfcacheWritesSkippedStuckIo","secondaryReadFromRmcacheBwc",
+        "inMaintenanceCapacityInKb","exposedCapacityInKb","netFglCompressedDataSizeInKb","userDataSdcWriteLatency","inUseVacInKb",
+        "fwdRebuildCapacityInKb","thickCapacityInUseInKb","backgroundScanReadErrorCount","activeMovingInRebalanceJobs",
+        "migratingVolumeIds","rfcacheWritesSkippedLowResources","capacityInUseNoOverheadInKb","exitProtectedMaintenanceModeWriteBwc",
+        "rfcacheSkippedUnlinedWrite","netCapacityInUseInKb","numOfOutgoingMigrations","rfcacheAvgWriteTime",
+        "pendingNormRebuildCapacityInKb","pendingMovingOutNormrebuildJobs","rfcacheSourceDeviceReads","rfcacheReadsPending",
+        "volumeAllocationLimitInKb","rfcacheReadsSkippedHeavyLoad","fwdRebuildWriteBwc","rfcacheReadMiss","targetReadLatency",
+        "userDataCapacityInKb","activeMovingInBckRebuildJobs","movingCapacityInKb","activeEnterProtectedMaintenanceModeCapacityInKb",
+        "backgroundScanCompareErrorCount","pendingMovingInFwdRebuildJobs","rfcacheReadsReceived","spSdsIds",
+        "pendingEnterProtectedMaintenanceModeCapacityInKb","vtreeAddresSpaceInKb","snapCapacityInUseOccupiedInKb",
+        "activeFwdRebuildCapacityInKb","rfcacheReadsSkippedStuckIo","activeMovingOutNormRebuildJobs","rfcacheWritePending",
+        "numOfThinBaseVolumes","degradedFailedVacInKb","userDataTrimBwc","numOfIncomingVtreeMigrations"
+    ]
+
+    DEFAULT_STATISTICS_PROPERTIES_ABOVE_3_5 = ["thinCapacityAllocatedInKm","thinUserDataCapacityInKb"]
+
+class VolumeConstants:
+    DEFAULT_STATISTICS_PROPERTIES = [
+        "rplUsedJournalCap","replicationState","numOfChildVolumes","userDataWriteBwc","rplTotalJournalCap","initiatorSdcId",
+        "userDataSdcReadLatency","userDataSdcTrimLatency","mappedSdcIds","registrationKey","registrationKeys",
+        "descendantVolumeIds","numOfMappedSdcs","reservationType","userDataReadBwc","numOfDescendantVolumes",
+        "replicationJournalVolume","userDataTrimBwc","childVolumeIds","userDataSdcWriteLatency"
+    ]

PyPowerFlex/objects/__init__.py

@@ -23,6 +23,7 @@
 from PyPowerFlex.objects.acceleration_pool import AccelerationPool
 from PyPowerFlex.objects.system import System
 from PyPowerFlex.objects.volume import Volume
+from PyPowerFlex.objects.utility import PowerFlexUtility
 
 
 __all__ = [
@@ -36,4 +37,5 @@
     'AccelerationPool',
     'System',
     'Volume',
+    'PowerFlexUtility'
 ]

PyPowerFlex/objects/storage_pool.py

@@ -180,6 +180,18 @@ def get_volumes(self, storage_pool_id, filter_fields=None, fields=None):
                                 filter_fields,
                                 fields)
 
+    def get_statistics(self, storage_pool_id, fields=None):
+        """Get related PowerFlex Statistics for storage pool.
+
+        :type storage_pool_id: str
+        :type fields: list|tuple
+        :rtype: dict
+        """
+
+        return self.get_related(storage_pool_id,
+                                'Statistics',
+                                fields)
+
     def rename(self, storage_pool_id, name):
         """Rename PowerFlex storage pool.
 

PyPowerFlex/objects/utility.py

@@ -0,0 +1,92 @@
+# Copyright (c) 2022 Dell Inc. or its subsidiaries.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import logging
+
+import requests
+
+from PyPowerFlex import base_client
+from PyPowerFlex import exceptions
+from PyPowerFlex import utils
+from PyPowerFlex.constants import StoragePoolConstants, VolumeConstants
+
+
+LOG = logging.getLogger(__name__)
+
+
+class PowerFlexUtility(base_client.EntityRequest):
+    def __init__(self, token, configuration):
+        super(PowerFlexUtility, self).__init__(token, configuration)
+
+    def get_statistics_for_all_storagepools(self, ids=None, properties=None):
+        """list storagepool statistics for PowerFlex.
+
+        :param ids: list
+        :param properties: list
+        :return: dict
+        """
+
+        action = 'querySelectedStatistics'
+        version = self.get_api_version()
+        default_properties = StoragePoolConstants.DEFAULT_STATISTICS_PROPERTIES
+        if version != '3.5':
+            default_properties = default_properties + StoragePoolConstants.DEFAULT_STATISTICS_PROPERTIES_ABOVE_3_5
+        params = {'properties': default_properties if properties is None else properties}
+        if ids is None:
+            params['allIds'] = ""
+        else:
+            params['ids'] = ids
+
+
+        r, response = self.send_post_request(self.list_statistics_url,
+                                             entity='StoragePool',
+                                             action=action,
+                                             params=params)
+        if r.status_code != requests.codes.ok:
+            msg = ('Failed to list storage pool statistics for PowerFlex. '
+                   'Error: {response}'.format(response=response))
+            LOG.error(msg)
+            raise exceptions.PowerFlexClientException(msg)
+
+        return response
+
+    def get_statistics_for_all_volumes(self, ids=None, properties=None):
+        """list volume statistics for PowerFlex.
+
+        :param ids: list
+        :param properties: list
+        :return: dict
+        """
+
+        action = 'querySelectedStatistics'
+
+        params = {'properties': VolumeConstants.DEFAULT_STATISTICS_PROPERTIES if properties is None else properties}
+        if ids is None:
+            params['allIds'] = ""
+        else:
+            params['ids'] = ids
+
+
+        r, response = self.send_post_request(self.list_statistics_url,
+                                             entity='Volume',
+                                             action=action,
+                                             params=params)
+        if r.status_code != requests.codes.ok:
+            msg = ('Failed to list volume statistics for PowerFlex. '
+                   'Error: {response}'.format(response=response))
+            LOG.error(msg)
+            raise exceptions.PowerFlexClientException(msg)
+
+        return response