Merge branch 'master' into dis-perf-framework

delfin/drivers/fake_storage/__init__.py

@@ -24,7 +24,7 @@
 from oslo_log import log
 from oslo_utils import uuidutils
 
-from delfin import exception
+from delfin import exception, db
 from delfin.common import constants
 from delfin.drivers import driver
 
@@ -66,11 +66,11 @@
 # count of instances for each resource type
 RESOURCE_COUNT_DICT = {
     "storage": 1,
-    "storagePool": 10,
-    "volume": 1000,
-    "port": 10,
-    "controller": 4,
-    "disk": 10,
+    "storagePool": MAX_POOL,
+    "volume": MAX_VOLUME,
+    "port": MAX_PORTS,
+    "controller": MAX_CONTROLLERS,
+    "disk": MAX_DISK,
 }
 
 
@@ -127,7 +127,17 @@ def reset_connection(self, context, **kwargs):
     @wait_random(MIN_WAIT, MAX_WAIT)
     def get_storage(self, context):
         # Do something here
+
         sn = six.text_type(uuidutils.generate_uuid())
+        try:
+            # use existing sn if already registered storage
+            storage = db.storage_get(context, self.storage_id)
+            if storage:
+                sn = storage['serial_number']
+        except exception.StorageNotFound:
+            LOG.debug('Registering new storage')
+        except Exception:
+            LOG.info('Error while retrieving storage from DB')
         total, used, free = self._get_random_capacity()
         raw = random.randint(2000, 3000)
         subscribed = random.randint(3000, 4000)
@@ -156,9 +166,9 @@ def list_storage_pools(self, ctx):
         for idx in range(rd_pools_count):
             total, used, free = self._get_random_capacity()
             p = {
-                "name": "fake_pool_" + str(idx),
+                "name": "storagePool_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_storage_pool_id": "fake_original_id_" + str(idx),
+                "native_storage_pool_id": "storagePool_" + str(idx),
                 "description": "Fake Pool",
                 "status": "normal",
                 "total_capacity": total,
@@ -195,9 +205,9 @@ def list_controllers(self, ctx):
             sts = list(constants.ControllerStatus.ALL)
             sts_len = len(constants.ControllerStatus.ALL) - 1
             c = {
-                "name": "fake_ctrl_" + str(idx),
+                "name": "controller_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_controller_id": "fake_original_id_" + str(idx),
+                "native_controller_id": "controller_" + str(idx),
                 "location": "loc_" + str(random.randint(0, 99)),
                 "status": sts[random.randint(0, sts_len)],
                 "memory_size": total,
@@ -223,9 +233,9 @@ def list_ports(self, ctx):
             logic_type = list(constants.PortLogicalType.ALL)
             logic_type_len = len(constants.PortLogicalType.ALL) - 1
             c = {
-                "name": "fake_port_" + str(idx),
+                "name": "port_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_port_id": "fake_original_id_" + str(idx),
+                "native_port_id": "port_" + str(idx),
                 "location": "location_" + str(random.randint(0, 99)),
                 "connection_status": conn_sts[
                     random.randint(0, conn_sts_len)],
@@ -263,9 +273,9 @@ def list_disks(self, ctx):
             logic_type = list(constants.DiskLogicalType.ALL)
             logic_type_len = len(constants.DiskLogicalType.ALL) - 1
             c = {
-                "name": "fake_disk_" + str(idx),
+                "name": "disk_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_disk_id": "fake_original_id_" + str(idx),
+                "native_disk_id": "disk_" + str(idx),
                 "serial_number": "serial_" + str(random.randint(0, 9999)),
                 "manufacturer": manufacturer[random.randint(0, 4)],
                 "model": "model_" + str(random.randint(0, 9999)),
@@ -298,12 +308,12 @@ def list_quotas(self, ctx):
             hlimit = random.randint(max_cap * 8000, max_cap * 9000)
             user_group = ['usr_', 'grp_']
             q = {
-                "native_quota_id": "fake_original_id_" + str(idx),
+                "native_quota_id": "quota_" + str(idx),
                 "type": qtype[random.randint(0, qtype_len)],
                 "storage_id": self.storage_id,
-                "native_filesystem_id": "fake_original_id_"
+                "native_filesystem_id": "quota_"
                                         + str(random.randint(0, 99)),
-                "native_qtree_id": "fake_original_id_"
+                "native_qtree_id": "qtree_"
                                    + str(random.randint(0, 99)),
                 "capacity_hard_limit": hlimit,
                 "capacity_soft_limit": slimit,
@@ -334,10 +344,10 @@ def list_filesystems(self, ctx):
             security = list(constants.NASSecurityMode.ALL)
             security_len = len(constants.NASSecurityMode.ALL) - 1
             f = {
-                "name": "fake_filesystem_" + str(idx),
+                "name": "filesystem_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_filesystem_id": "fake_original_id_" + str(idx),
-                "native_pool_id": "fake_pool_id_" + str(idx),
+                "native_filesystem_id": "filesystem_" + str(idx),
+                "native_pool_id": "storagePool_" + str(idx),
                 "status": sts[random.randint(0, sts_len)],
                 "type": alloc_type[random.randint(0, alloc_type_len)],
                 "security_mode": security[random.randint(0, security_len)],
@@ -361,10 +371,10 @@ def list_qtrees(self, ctx):
             security_len = len(constants.NASSecurityMode.ALL) - 1
 
             t = {
-                "name": "fake_qtree_" + str(idx),
+                "name": "qtree_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_qtree_id": "fake_original_id_" + str(idx),
-                "native_filesystem_id": "fake_filesystem_id_"
+                "native_qtree_id": "qtree_" + str(idx),
+                "native_filesystem_id": "filesystem_"
                                         + str(random.randint(0, 99)),
                 "security_mode": security[random.randint(0, security_len)],
                 "path": "/path/qtree_" + str(random.randint(0, 99)),
@@ -382,12 +392,12 @@ def list_shares(self, ctx):
             pro = list(constants.ShareProtocol.ALL)
             pro_len = len(constants.ShareProtocol.ALL) - 1
             c = {
-                "name": "fake_share_" + str(idx),
+                "name": "share_" + str(idx),
                 "storage_id": self.storage_id,
-                "native_share_id": "fake_original_id_" + str(idx),
-                "native_filesystem_id": "fake_filesystem_id_"
+                "native_share_id": "share_" + str(idx),
+                "native_filesystem_id": "filesystem_"
                                         + str(random.randint(0, 99)),
-                "native_qtree_id": "fake_qtree_id_"
+                "native_qtree_id": "qtree_"
                                    + str(random.randint(0, 99)),
                 "protocol": pro[random.randint(0, pro_len)],
                 "path": "/path/share_" + str(random.randint(0, 99)),
@@ -467,11 +477,11 @@ def _get_volume_range(self, start, end):
         for i in range(start, end):
             total, used, free = self._get_random_capacity()
             v = {
-                "name": "fake_vol_" + str(i),
+                "name": "volume_" + str(i),
                 "storage_id": self.storage_id,
                 "description": "Fake Volume",
                 "status": "normal",
-                "native_volume_id": "fake_original_id_" + str(i),
+                "native_volume_id": "volume_" + str(i),
                 "wwn": "fake_wwn_" + str(i),
                 "total_capacity": total,
                 "used_capacity": used,
@@ -508,7 +518,7 @@ def get_resource_perf_metrics(self, storage_id, start_time, end_time,
         for i in range(resource_count):
             labels = {'storage_id': storage_id,
                       'resource_type': resource_type,
-                      'resource_id': resource_type + str(i),
+                      'resource_id': resource_type + '_' + str(i),
                       'type': 'RAW'}
             fake_metrics = self._get_random_performance(metric_list,
                                                         start_time, end_time)

delfin/tests/e2e/GetResources.robot

@@ -112,12 +112,12 @@ Get All Storages
 
 Open Application
     ${array_id}=            Register Test Storage
-    Sleep       1s
+    Sleep       5s
 
 Close Application
     @{storages}=            Get All Storages
     FOR     ${storage}      IN                      @{storages}
             ${storage_id}=  Get Value From Json	    ${storage} 	        $..id
             Delete Storage With ID                  ${storage_id[0]}
     END
-    Sleep       1s
+    Sleep       5s

delfin/tests/e2e/RegisterStorage.robot

@@ -66,7 +66,7 @@ Register Storage with valid access_info Test
 
 Register Storage with same access_info Test
     [Tags]    DELFIN
-    Sleep                   1s
+    Sleep                   5s
     ${storage_test}=        Register Test Storage
 
     ${test}=                Load Json From File   ${CURDIR}/test.json

delfin/tests/e2e/RemoveStorage.robot

@@ -12,7 +12,7 @@ ${delfin_url}           http://localhost:8190/v1
 
 Delete Storage with valid storage_id
     [Tags]    DELFIN
-    Sleep                   1s
+    Sleep                   5s
     ${storage_id_test}=     Register Test Storage
     Create Session          delfin      ${delfin_url}
     ${resp_del}=            DELETE On Session    delfin     storages/${storage_id_test}

delfin/tests/e2e/UpdateAccessInfo.robot

@@ -111,9 +111,9 @@ Close Application
             ${storage_id}=  Get Value From Json	    ${storage} 	        $..id
             Delete Storage With ID                  ${storage_id[0]}
     END
-    Sleep       1s
+    Sleep       5s
 
 Open Application
     ${array_id}=            Register Test Storage
-    Sleep       1s
+    Sleep       5s
 

delfin/tests/unit/drivers/test_api.py

@@ -69,11 +69,12 @@ def test_init(self):
         api = API()
         self.assertIsNotNone(api.driver_manager)
 
+    @mock.patch('delfin.db.storage_get')
     @mock.patch('delfin.db.storage_create')
     @mock.patch('delfin.db.access_info_create')
     @mock.patch('delfin.db.storage_get_all')
     def test_discover_storage(self, mock_storage, mock_access_info,
-                              mock_storage_create):
+                              mock_storage_create, mock_get_storage):
         # Case: Positive scenario for fake driver discovery
         storage = copy.deepcopy(STORAGE)
         storage['id'] = '12345'
@@ -85,6 +86,7 @@ def test_discover_storage(self, mock_storage, mock_access_info,
         mock_storage.assert_called()
         mock_access_info.assert_called_with(context, ACCESS_INFO)
         mock_storage_create.assert_called()
+        mock_get_storage.return_value = None
 
         # Case: Register already existing storage
         with self.assertRaises(exception.StorageAlreadyExists) as exc:
@@ -188,18 +190,20 @@ def test_update_access_info(self, mock_storage_get,
         msg = "Storage backend could not be found"
         self.assertIn(msg, str(exc.exception))
 
+    @mock.patch('delfin.db.storage_get')
     @mock.patch('delfin.db.storage_create')
     @mock.patch('delfin.db.access_info_create')
     @mock.patch('delfin.db.storage_get_all')
     def test_remove_storage(self, mock_storage, mock_access_info,
-                            mock_storage_create):
+                            mock_storage_create, mock_get_storage):
         storage = copy.deepcopy(STORAGE)
         storage['id'] = '12345'
         mock_storage.return_value = None
         mock_access_info.return_value = ACCESS_INFO
         mock_storage_create.return_value = storage
         api = API()
         api.discover_storage(context, ACCESS_INFO)
+        mock_get_storage.return_value = None
 
         storage_id = '12345'
 

docker-compose.yml

@@ -26,6 +26,7 @@
 #    $ export DELFIN_RABBITMQ_USER=delfinuser
 #    $ export DELFIN_RABBITMQ_PASS=delfinpass
 #    $ export DELFIN_HOST_IP=192.168.0.2
+#    $ export DELFIN_METRICS_DIR=/var/lib/delfin/metrics
 #
 # 3. Bring up delfin project using following command
 #
@@ -81,10 +82,12 @@ services:
     volumes:
       - ./etc/delfin:/etc/delfin
       - db_data:/var/lib/delfin
+      - metrics_dir:${DELFIN_METRICS_DIR}
     restart: always
     environment:
       - OS_COORDINATION__BACKEND_SERVER=${DELFIN_HOST_IP}:6379
       - OS_DEFAULT__TRANSPORT_URL=rabbit://${DELFIN_RABBITMQ_USER}:${DELFIN_RABBITMQ_PASS}@${DELFIN_HOST_IP}:5672//
+      - OS_PROMETHEUS_EXPORTER__METRICS_DIR=${DELFIN_METRICS_DIR}
     depends_on:
       - redis
       - rabbitmq
@@ -108,11 +111,16 @@ services:
     command: "exporter"
     volumes:
       - ./etc/delfin:/etc/delfin
+      - metrics_dir:${DELFIN_METRICS_DIR}
+    ports:
+      - 8195:8195
     restart: always
     environment:
       - OS_DEFAULT__TRANSPORT_URL=rabbit://${DELFIN_RABBITMQ_USER}:${DELFIN_RABBITMQ_PASS}@${DELFIN_HOST_IP}:5672//
+      - OS_PROMETHEUS_EXPORTER__METRICS_DIR=${DELFIN_METRICS_DIR}
     depends_on:
       - rabbitmq
 
 volumes:
   db_data: {}
+  metrics_dir: {}