Merge pull request #735 from gem/fix-gmf

Fix gmf layer names; display gmpe ranges when selecting event id

svir/dialogs/drive_oq_engine_server_dialog.py

@@ -904,7 +904,8 @@ def show_output_list(self, output_list, calculation_mode):
                             action = 'Download'
                             button = QPushButton()
                             self.connect_button_to_action(
-                                button, action, output, outtype)
+                                button, action, output, outtype,
+                                calculation_mode)
                             self.output_list_tbl.setCellWidget(
                                 row, additional_cols, button)
                             self.calc_list_tbl.setColumnWidth(
@@ -922,7 +923,7 @@ def show_output_list(self, output_list, calculation_mode):
                         continue
                     button = QPushButton()
                     self.connect_button_to_action(
-                        button, action, output, outtype)
+                        button, action, output, outtype, calculation_mode)
                     self.output_list_tbl.setCellWidget(
                         row, additional_cols, button)
                     additional_cols += 1
@@ -931,15 +932,17 @@ def show_output_list(self, output_list, calculation_mode):
                     mod_output['type'] = "%s_aggr" % output['type']
                     button = QPushButton()
                     self.connect_button_to_action(
-                        button, 'Aggregate', mod_output, outtype)
+                        button, 'Aggregate', mod_output, outtype,
+                        calculation_mode)
                     self.output_list_tbl.setCellWidget(
                         row, additional_cols, button)
                     additional_cols += 1
                 # For each output id, add a Download btn per each output type
                 # (like in the webui)
                 action = 'Download'
                 button = QPushButton()
-                self.connect_button_to_action(button, action, output, outtype)
+                self.connect_button_to_action(
+                    button, action, output, outtype, calculation_mode)
                 self.output_list_tbl.setCellWidget(
                     row, additional_cols, button)
                 self.calc_list_tbl.setColumnWidth(
@@ -953,7 +956,8 @@ def show_output_list(self, output_list, calculation_mode):
         self.output_list_tbl.resizeColumnsToContents()
         self.output_list_tbl.resizeRowsToContents()
 
-    def connect_button_to_action(self, button, action, output, outtype):
+    def connect_button_to_action(self, button, action, output, outtype,
+                                 calculation_mode):
         if action in ('Load layer', 'Load from zip', 'Load table',
                       'Show', 'Aggregate'):
             style = 'background-color: blue; color: white;'
@@ -971,10 +975,12 @@ def connect_button_to_action(self, button, action, output, outtype):
             style = 'background-color: #3cb3c5; color: white;'
             button.setText("%s %s" % (action, outtype))
         button.setStyleSheet(style)
-        button.clicked.connect(lambda checked=False, output=output, action=action, outtype=outtype: (  # NOQA
-            self.on_output_action_btn_clicked(output, action, outtype)))
+        button.clicked.connect(lambda checked=False, output=output, action=action, outtype=outtype, calculation_mode=calculation_mode: (  # NOQA
+            self.on_output_action_btn_clicked(
+                output, action, outtype, calculation_mode)))
 
-    def on_output_action_btn_clicked(self, output, action, outtype):
+    def on_output_action_btn_clicked(
+            self, output, action, outtype, calculation_mode):
         output_id = output['id']
         output_type = output['type']
         if action in ['Show', 'Aggregate']:
@@ -1007,7 +1013,8 @@ def on_output_action_btn_clicked(self, output, action, outtype):
             # the asset_risk output type
             if outtype == 'npz' or output_type in (
                     OQ_EXTRACT_TO_LAYER_TYPES | OQ_EXTRACT_TO_VIEW_TYPES):
-                self.open_output(output_id, output_type)
+                self.open_output(
+                    output_id, output_type, calculation_mode=calculation_mode)
             elif outtype == 'csv':
                 dest_folder = tempfile.gettempdir()
                 descr = 'Download %s for calculation %s' % (
@@ -1067,7 +1074,8 @@ def open_full_report(
         self.full_report_dlg.show()
 
     def open_output(
-            self, output_id=None, output_type=None, filepath=None):
+            self, output_id=None, output_type=None, filepath=None,
+            calculation_mode=None):
         assert(output_type is not None)
         if output_type not in OUTPUT_TYPE_LOADERS:
             raise NotImplementedError(output_type)
@@ -1076,7 +1084,8 @@ def open_output(
             self, self.iface, self.viewer_dock,
             self.session, self.hostname, self.current_calc_id,
             output_type, path=filepath,
-            engine_version=self.engine_version)
+            engine_version=self.engine_version,
+            calculation_mode=calculation_mode)
         self.open_output_dlgs[dlg_id] = open_output_dlg
         open_output_dlg.finished[int].connect(
             lambda result: self.del_dlg(dlg_id))

svir/dialogs/load_asset_risk_as_layer_dialog.py

@@ -39,12 +39,12 @@ class LoadAssetRiskAsLayerDialog(LoadOutputAsLayerDialog):
     """
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type='asset_risk', path=None, mode=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type == 'asset_risk'
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            engine_version=engine_version)
+            engine_version=engine_version, calculation_mode=calculation_mode)
 
         self.setWindowTitle(
             'Load Exposure/Risk as layer')

svir/dialogs/load_basic_csv_as_layer_dialog.py

@@ -38,12 +38,12 @@ class LoadCsvAsLayerDialog(LoadOutputAsLayerDialog):
 
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type, path=None, mode=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type in OQ_CSV_TO_LAYER_TYPES, output_type
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            engine_version=engine_version)
+            engine_version=engine_version, calculation_mode=calculation_mode)
         self.create_file_size_indicator()
         self.setWindowTitle('Load %s from CSV, as layer' % output_type)
         self.populate_out_dep_widgets()

svir/dialogs/load_csv_as_layer_dialog.py

@@ -38,12 +38,12 @@ class LoadCsvAsLayerDialog(LoadOutputAsLayerDialog):
 
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type, path=None, mode=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type in OQ_CSV_TO_LAYER_TYPES, output_type
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            engine_version=engine_version)
+            engine_version=engine_version, calculation_mode=calculation_mode)
         self.create_file_size_indicator()
         self.setWindowTitle('Load %s from CSV, as layer' % output_type)
         self.populate_out_dep_widgets()

svir/dialogs/load_dmg_by_asset_as_layer_dialog.py

@@ -42,12 +42,13 @@ class LoadDmgByAssetAsLayerDialog(LoadOutputAsLayerDialog):
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type='dmg_by_asset',
                  path=None, mode=None, zonal_layer_path=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type == 'dmg_by_asset'
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            zonal_layer_path=zonal_layer_path, engine_version=engine_version)
+            zonal_layer_path=zonal_layer_path, engine_version=engine_version,
+            calculation_mode=calculation_mode)
 
         self.setWindowTitle('Load scenario damage by asset as layer')
         self.create_load_selected_only_ckb()

svir/dialogs/load_gmf_data_as_layer_dialog.py

@@ -23,6 +23,7 @@
 # along with OpenQuake.  If not, see <http://www.gnu.org/licenses/>.
 
 import os
+import numpy as np
 from qgis.PyQt.QtWidgets import QInputDialog, QDialog
 from qgis.core import (
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
@@ -39,12 +40,12 @@ class LoadGmfDataAsLayerDialog(LoadOutputAsLayerDialog):
 
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type='gmf_data', path=None, mode=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type == 'gmf_data'
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            engine_version=engine_version)
+            engine_version=engine_version, calculation_mode=calculation_mode)
 
         self.setWindowTitle(
             'Load ground motion fields as layer')
@@ -58,22 +59,38 @@ def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
         self.rlz_or_stat_cbx.setVisible(False)
         self.create_imt_selector()
 
-        log_msg('Extracting number of events. Watch progress in QGIS task bar',
+        self.extract_realizations()
+
+        log_msg('Extracting events. Watch progress in QGIS task bar',
                 level='I', message_bar=self.iface.messageBar())
         self.extract_npz_task = ExtractNpzTask(
-            'Extract number of events', QgsTask.CanCancel, self.session,
-            self.hostname, self.calc_id, 'num_events', self.get_eid,
+            'Extract events', QgsTask.CanCancel, self.session,
+            self.hostname, self.calc_id, 'events', self.get_eid,
             self.on_extract_error)
         QgsApplication.taskManager().addTask(self.extract_npz_task)
 
-    def get_eid(self, num_events_npz):
-        num_events = num_events_npz['num_events']
+    def get_eid(self, events_npz):
+        self.events_npz = events_npz
+        num_events = len(events_npz['array'])
         if 'GEM_QGIS_TEST' in os.environ:
             self.eid, ok = 0, True
+        elif 'scenario' in self.calculation_mode:
+            range_width = self.oqparam['number_of_ground_motion_fields']
+            ranges = {}
+            for gsim_idx, gsim in enumerate(self.gsims):
+                ranges[gsim] = (gsim_idx * range_width,
+                                gsim_idx * range_width + range_width - 1)
+            ranges_str = ''
+            for gsim in ranges:
+                ranges_str += '\n%s: %s' % (gsim, ranges[gsim])
+            self.eid, ok = QInputDialog.getInt(
+                self.drive_engine_dlg,
+                "Select an event ID", "Ranges:%s" % ranges_str,
+                0, 0, num_events - 1)
         else:
             self.eid, ok = QInputDialog.getInt(
                 self.drive_engine_dlg,
-                "Select an event ID", "range (0 - %s)" % (num_events - 1),
+                "Select an event ID", "Range (0 - %s)" % (num_events - 1),
                 0, 0, num_events - 1)
         if not ok:
             self.reject()
@@ -97,7 +114,7 @@ def finalize_init(self, gmf_data_npz):
         self.init_done.emit()
 
     def set_ok_button(self):
-        if not len(self.dataset) and 'GEM_QGIS_TEST' in os.environ:
+        if not len(self.gmf_data) and 'GEM_QGIS_TEST' in os.environ:
             self.ok_button.setEnabled(True)
         else:
             self.ok_button.setEnabled(self.imt_cbx.currentIndex() != -1)
@@ -107,40 +124,44 @@ def show_num_sites(self):
         #       which is always true for scenario calculations.
         #       If different realizations have a different number of sites, we
         #       need to move this block of code inside on_rlz_or_stat_changed()
-        rlz = self.rlz_or_stat_cbx.itemData(
+        rlz_id = self.rlz_or_stat_cbx.itemData(
             self.rlz_or_stat_cbx.currentIndex())
+        rlz_name = 'rlz-%03d' % rlz_id
         try:
-            gmf_data = self.npz_file[rlz]
+            gmf_data = self.npz_file[rlz_name]
         except AttributeError:
             self.num_sites_lbl.setText(self.num_sites_msg % 0)
             return
         else:
             self.num_sites_lbl.setText(self.num_sites_msg % gmf_data.shape)
 
-    def populate_rlz_or_stat_cbx(self):
-        self.rlzs_or_stats = [key for key in sorted(self.npz_file)
-                              if key not in ('imtls', 'array')]
+    def extract_realizations(self):
         with WaitCursorManager(
                 'Extracting...', message_bar=self.iface.messageBar()):
             self.rlzs_npz = extract_npz(
                 self.session, self.hostname, self.calc_id, 'realizations',
                 message_bar=self.iface.messageBar(), params=None)
-        # rlz[1] is the branch_path field
-        self.gsims = [rlz[1].decode('utf8').strip('"')
-                      for rlz in self.rlzs_npz['array']]
+        self.rlzs_or_stats = [
+            rlz_id for rlz_id in self.rlzs_npz['array']['rlz_id']]
+        self.gsims = [branch_path.decode('utf8').strip("\"")
+                      for branch_path in self.rlzs_npz['array']['branch_path']]
+
+    def populate_rlz_or_stat_cbx(self):
         self.rlz_or_stat_cbx.clear()
         self.rlz_or_stat_cbx.setEnabled(True)
         for gsim, rlz in zip(self.gsims, self.rlzs_or_stats):
             # storing gsim as text, rlz as hidden data
             self.rlz_or_stat_cbx.addItem(gsim, userData=rlz)
+        rlz_id = self.events_npz['array'][
+            np.where(self.events_npz['array']['id'] == self.eid)]['rlz_id']
+        self.rlz_or_stat_cbx.setCurrentIndex(
+            self.rlz_or_stat_cbx.itemData(rlz_id))
 
     def on_rlz_or_stat_changed(self):
-        rlz = self.rlz_or_stat_cbx.itemData(
-            self.rlz_or_stat_cbx.currentIndex())
         gmpe = self.rlz_or_stat_cbx.currentText()
         self.rlz_or_stat_lbl.setText("GMPE: %s" % gmpe)
-        self.dataset = self.npz_file[rlz]
-        if not len(self.dataset):
+        self.gmf_data = self.npz_file[self.npz_file.keys()[0]]
+        if not len(self.gmf_data):
             log_msg('No data corresponds to the chosen event and GMPE',
                     level='W', message_bar=self.iface.messageBar())
             if 'GEM_QGIS_TEST' in os.environ:
@@ -159,32 +180,33 @@ def on_imt_changed(self):
         self.set_ok_button()
 
     def accept(self):
-        if not len(self.dataset) and 'GEM_QGIS_TEST' in os.environ:
+        if not len(self.gmf_data) and 'GEM_QGIS_TEST' in os.environ:
             QDialog.accept(self)
         else:
             super().accept()
 
     def load_from_npz(self):
-        for rlz, gsim in zip(self.rlzs_or_stats, self.gsims):
-            # NOTE: selecting only 1 event, we have only 1 gsim
-            with WaitCursorManager('Creating layer for "%s"...'
-                                   % gsim, self.iface.messageBar()):
-                self.build_layer(rlz_or_stat=rlz, gsim=gsim)
-                self.style_maps(self.layer, self.default_field_name,
-                                self.iface, self.output_type)
+        # NOTE: selecting only 1 event, we have only 1 gsim
+        rlz = self.rlz_or_stat_cbx.currentData()
+        gsim = self.rlz_or_stat_cbx.currentText()
+        with WaitCursorManager('Creating layer for "%s"...'
+                               % gsim, self.iface.messageBar()):
+            self.build_layer(rlz_or_stat=rlz, gsim=gsim)
+            self.style_maps(self.layer, self.default_field_name,
+                            self.iface, self.output_type)
         if self.npz_file is not None:
             self.npz_file.close()
 
     def build_layer_name(self, gsim=None, **kwargs):
         self.imt = self.imt_cbx.currentText()
         self.default_field_name = '%s-%s' % (self.imt, self.eid)
-        # NOTE: assuming it's a scenario calculation
-        layer_name = "scenario_gmfs_%s_eid-%s" % (gsim, self.eid)
+        layer_name = "%s_gmfs_%s_eid-%s" % (
+            self.calculation_mode, gsim, self.eid)
         return layer_name
 
     def get_field_types(self, **kwargs):
-        field_types = {name: self.dataset[name].dtype.char
-                       for name in self.dataset.dtype.names}
+        field_types = {name: self.gmf_data[name].dtype.char
+                       for name in self.gmf_data.dtype.names}
         return field_types
 
     def add_field_to_layer(self, field_name, field_type):
@@ -201,7 +223,8 @@ def read_npz_into_layer(self, field_types, rlz_or_stat, **kwargs):
             dataset_field_names = list(self.get_field_types().keys())
             d2l_field_names = dict(
                 list(zip(dataset_field_names[2:], layer_field_names)))
-            for row in self.npz_file[rlz_or_stat]:
+            rlz_name = 'rlz-%03d' % rlz_or_stat
+            for row in self.npz_file[rlz_name]:
                 # add a feature
                 feat = QgsFeature(fields)
                 for field_name in dataset_field_names:

svir/dialogs/load_hcurves_as_layer_dialog.py

@@ -36,12 +36,12 @@ class LoadHazardCurvesAsLayerDialog(LoadOutputAsLayerDialog):
 
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type='hcurves', path=None, mode=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type == 'hcurves'
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            engine_version=engine_version)
+            engine_version=engine_version, calculation_mode=calculation_mode)
 
         self.setWindowTitle(
             'Load hazard curves as layer')

svir/dialogs/load_hmaps_as_layer_dialog.py

@@ -36,12 +36,12 @@ class LoadHazardMapsAsLayerDialog(LoadOutputAsLayerDialog):
     """
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type='hmaps', path=None, mode=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type == 'hmaps'
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            engine_version=engine_version)
+            engine_version=engine_version, calculation_mode=calculation_mode)
 
         self.setWindowTitle(
             'Load hazard maps as layer')

svir/dialogs/load_losses_by_asset_as_layer_dialog.py

@@ -40,12 +40,13 @@ class LoadLossesByAssetAsLayerDialog(LoadOutputAsLayerDialog):
     def __init__(self, drive_engine_dlg, iface, viewer_dock, session, hostname,
                  calc_id, output_type=None,
                  path=None, mode=None, zonal_layer_path=None,
-                 engine_version=None):
+                 engine_version=None, calculation_mode=None):
         assert output_type in ('losses_by_asset', 'avg_losses-stats')
         LoadOutputAsLayerDialog.__init__(
             self, drive_engine_dlg, iface, viewer_dock, session, hostname,
             calc_id, output_type=output_type, path=path, mode=mode,
-            zonal_layer_path=zonal_layer_path, engine_version=engine_version)
+            zonal_layer_path=zonal_layer_path, engine_version=engine_version,
+            calculation_mode=calculation_mode)
 
         if self.output_type == 'losses_by_asset':
             self.setWindowTitle(