Merge pull request #713 from gem/field-types

Set field data types correctly while loading oq-engine outputs

scripts/make_doc.sh

@@ -8,6 +8,6 @@ docker run -d --name qgis -v /tmp/.X11-unix:/tmp/.X11-unix \
  -e DISPLAY=:99 \
  qgis/qgis:final-3_8_3
 
-docker exec -it qgis sh -c "apt update; DEBIAN_FRONTEND=noninteractive apt install -y latexmk texlive-latex-extra python3-matplotlib python3-sphinx python3-sphinx-rtd-theme dvipng"
+docker exec -it qgis sh -c "apt update --allow-releaseinfo-change; DEBIAN_FRONTEND=noninteractive apt install -y latexmk texlive-latex-extra python3-matplotlib python3-sphinx python3-sphinx-rtd-theme dvipng"
 
 docker exec -it qgis sh -c "export PYTHONPATH=$PYTHONPATH:/oq-irmt-qgis; cd /oq-irmt-qgis/svir; make doc"

scripts/run_integration_tests.sh

@@ -24,7 +24,7 @@ docker run -d --name qgis -v /tmp/.X11-unix:/tmp/.X11-unix \
  -e GEM_QGIS_TEST=y \
  qgis/qgis:final-3_8_3
 
-docker exec -it qgis sh -c "apt update; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
+docker exec -it qgis sh -c "apt update --allow-releaseinfo-change; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
 
 docker exec -it qgis sh -c "git clone -q -b $BRANCH --depth=1 https://github.com/gem/oq-engine.git && echo 'Running against oq-engine/$BRANCH'"
 

scripts/run_unit_tests.sh

@@ -9,7 +9,7 @@ docker run -d --name qgis -v /tmp/.X11-unix:/tmp/.X11-unix \
  -e GEM_QGIS_TEST=y \
  qgis/qgis:final-3_8_3
 
-docker exec -it qgis bash -c "apt update; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
+docker exec -it qgis bash -c "apt update --allow-releaseinfo-change; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
 docker exec -it qgis bash -c "python3 -m pip install pytest"
 
 # OGR_SQLITE_JOURNAL=delete prevents QGIS from using WAL, which modifies geopackages even if they are just read

svir/calculations/calculate_utils.py

@@ -35,8 +35,7 @@
 
 from qgis.PyQt.QtCore import QVariant
 
-from svir.utilities.shared import (DOUBLE_FIELD_TYPE_NAME,
-                                   STRING_FIELD_TYPE_NAME,
+from svir.utilities.shared import (
                                    DEBUG,
                                    SUM_BASED_OPERATORS,
                                    MUL_BASED_OPERATORS, DEFAULT_OPERATOR,
@@ -63,18 +62,18 @@ class InvalidFormula(Exception):
     pass
 
 
-def add_numeric_attribute(proposed_attr_name, layer):
-    field = QgsField(proposed_attr_name, QVariant.Double)
-    field.setTypeName(DOUBLE_FIELD_TYPE_NAME)
-    assigned_attr_names = ProcessLayer(layer).add_attributes(
-        [field])
-    assigned_attr_name = assigned_attr_names[proposed_attr_name]
-    return assigned_attr_name
-
-
-def add_textual_attribute(proposed_attr_name, layer):
-    field = QgsField(proposed_attr_name, QVariant.String)
-    field.setTypeName(STRING_FIELD_TYPE_NAME)
+def add_attribute(proposed_attr_name, dtype, layer):
+    if dtype == 'S':
+        qtype = QVariant.String
+        qname = 'String'
+    elif dtype in ('U', 'I'):  # FIXME: what for unsigned int?
+        qtype = QVariant.Int
+        qname = 'integer'
+    else:  # FIXME: treating everything else as double (it might be wrong)
+        qtype = QVariant.Double
+        qname = 'double'
+    field = QgsField(proposed_attr_name, qtype)
+    field.setTypeName(qname)
     assigned_attr_names = ProcessLayer(layer).add_attributes(
         [field])
     assigned_attr_name = assigned_attr_names[proposed_attr_name]
@@ -194,15 +193,13 @@ def get_node_attr_id_and_name(node, layer):
         # deleted it). If it is not there anymore, add a new field
         if layer.fields().indexOf(node_attr_name) == -1:  # not found
             proposed_node_attr_name = node_attr_name
-            node_attr_name = add_numeric_attribute(
-                proposed_node_attr_name, layer)
+            node_attr_name = add_attribute(proposed_node_attr_name, 'F', layer)
             field_was_added = True
         elif DEBUG:
             log_msg('Reusing field %s' % node_attr_name)
     elif 'name' in node:
         proposed_node_attr_name = node['name']
-        node_attr_name = add_numeric_attribute(
-            proposed_node_attr_name, layer)
+        node_attr_name = add_attribute(proposed_node_attr_name, 'F', layer)
         field_was_added = True
     else:  # this corner case should never happen (hopefully)
         raise InvalidNode('This node has no name and it does'

svir/dialogs/load_asset_risk_as_layer_dialog.py

@@ -28,7 +28,6 @@
 from qgis.core import (
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication,)
 from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
-from svir.calculations.calculate_utils import add_numeric_attribute
 from svir.utilities.utils import WaitCursorManager, log_msg
 from svir.ui.multi_select_combo_box import MultiSelectComboBox
 from svir.tasks.extract_npz_task import ExtractNpzTask
@@ -191,21 +190,12 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
                 self.category_cbx.currentText())
         return layer_name
 
-    def get_field_names(self, **kwargs):
-        field_names = [name for name in self.dataset.dtype.names
-                       if name not in self.tag_names and
-                       name not in ['lon', 'lat']]
-        return field_names
-
-    def add_field_to_layer(self, field_name):
-        try:
-            # NOTE: add_numeric_attribute uses the native qgis editing manager
-            added_field_name = add_numeric_attribute(field_name, self.layer)
-        except TypeError as exc:
-            log_msg(str(exc), level='C', message_bar=self.iface.messageBar(),
-                    exception=exc)
-            return
-        return added_field_name
+    def get_field_types(self, **kwargs):
+        field_types = {name: self.dataset[name].dtype.char
+                       for name in self.dataset.dtype.names
+                       if name not in ['lon', 'lat']
+                       and name not in self.tag_names}
+        return field_types
 
     def read_npz_into_layer(self, field_names, **kwargs):
         with edit(self.layer):

svir/dialogs/load_dmg_by_asset_as_layer_dialog.py

@@ -27,7 +27,6 @@
 from qgis.core import (
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
 from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
-from svir.calculations.calculate_utils import add_numeric_attribute
 from svir.utilities.utils import (WaitCursorManager,
                                   log_msg,
                                   get_loss_types,
@@ -157,7 +156,7 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
             layer_name = "dmg_by_asset_%s_%s" % (rlz_or_stat, loss_type)
         return layer_name
 
-    def get_field_names(self, **kwargs):
+    def get_field_types(self, **kwargs):
         loss_type = kwargs['loss_type']
         dmg_state = kwargs['dmg_state']
         if self.aggregate_by_site_ckb.isChecked():
@@ -175,23 +174,19 @@ def get_field_names(self, **kwargs):
             self.default_field_name = "%s_%s_mean" % (
                 self.loss_type_cbx.currentText(),
                 self.dmg_state_cbx.currentText())
-        return field_names
+        # NOTE: assuming that all fields are numeric
+        field_types = {field_name: 'F' for field_name in field_names}
+        return field_types
 
-    def add_field_to_layer(self, field_name):
-        # NOTE: add_numeric_attribute uses the native qgis editing manager
-        added_field_name = add_numeric_attribute(
-            field_name, self.layer)
-        return added_field_name
-
-    def read_npz_into_layer(self, field_names, **kwargs):
+    def read_npz_into_layer(self, field_types, **kwargs):
         if self.aggregate_by_site_ckb.isChecked():
-            self.read_npz_into_layer_aggr_by_site(field_names, **kwargs)
+            self.read_npz_into_layer_aggr_by_site(field_types, **kwargs)
         else:
             # do not aggregate by site, then aggregate by zone afterwards if
             # required
-            self.read_npz_into_layer_no_aggr(field_names, **kwargs)
+            self.read_npz_into_layer_no_aggr(field_types, **kwargs)
 
-    def read_npz_into_layer_no_aggr(self, field_names, **kwargs):
+    def read_npz_into_layer_no_aggr(self, field_types, **kwargs):
         rlz_or_stat = kwargs['rlz_or_stat']
         loss_type = kwargs['loss_type']
         with edit(self.layer):
@@ -200,19 +195,19 @@ def read_npz_into_layer_no_aggr(self, field_names, **kwargs):
             for row in data:
                 # add a feature
                 feat = QgsFeature(self.layer.fields())
-                for field_name_idx, field_name in enumerate(field_names):
+                for field_name, field_type in field_types.items():
                     if field_name in ['lon', 'lat']:
                         continue
                     elif field_name in data.dtype.names:
                         value = row[field_name]
-                        try:
-                            value = float(value)
-                        except ValueError:
+                        if data[field_name].dtype.char == 'S':
                             value = str(value, encoding='utf8').strip('"')
+                        else:
+                            value = float(value)
                     else:
                         value = float(
                             row[loss_type][field_name[len(loss_type)+1:]])
-                    feat.setAttribute(field_names[field_name_idx], value)
+                    feat.setAttribute(field_name, value)
                 feat.setGeometry(QgsGeometry.fromPointXY(
                     QgsPointXY(row['lon'], row['lat'])))
                 feats.append(feat)
@@ -221,7 +216,7 @@ def read_npz_into_layer_no_aggr(self, field_names, **kwargs):
                 msg = 'There was a problem adding features to the layer.'
                 log_msg(msg, level='C', message_bar=self.iface.messageBar())
 
-    def read_npz_into_layer_aggr_by_site(self, field_names, **kwargs):
+    def read_npz_into_layer_aggr_by_site(self, field_types, **kwargs):
         rlz_or_stat = kwargs['rlz_or_stat']
         loss_type = kwargs['loss_type']
         taxonomy = kwargs['taxonomy']
@@ -233,11 +228,14 @@ def read_npz_into_layer_aggr_by_site(self, field_names, **kwargs):
             for row in grouped_by_site:
                 # add a feature
                 feat = QgsFeature(self.layer.fields())
-                for field_name_idx, field_name in enumerate(field_names):
+                field_idx = 0
+                for field_name, field_type in field_types.items():
                     if field_name in ['lon', 'lat']:
+                        field_idx += 1
                         continue
-                    value = float(row[field_name_idx])
-                    feat.setAttribute(field_names[field_name_idx], value)
+                    value = float(row[field_idx])
+                    feat.setAttribute(field_name, value)
+                    field_idx += 1
                 feat.setGeometry(QgsGeometry.fromPointXY(
                     QgsPointXY(row['lon'], row['lat'])))
                 feats.append(feat)

svir/dialogs/load_gmf_data_as_layer_dialog.py

@@ -25,7 +25,7 @@
 from qgis.core import (
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
 from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
-from svir.calculations.calculate_utils import add_numeric_attribute
+from svir.calculations.calculate_utils import add_attribute
 from svir.utilities.utils import WaitCursorManager, log_msg, extract_npz
 from svir.tasks.extract_npz_task import ExtractNpzTask
 
@@ -136,25 +136,23 @@ def build_layer_name(self, gsim=None, **kwargs):
         layer_name = "scenario_gmfs_%s_eid-%s" % (gsim, self.eid)
         return layer_name
 
-    def get_field_names(self, **kwargs):
-        # NOTE: we need a list instead of a tuple, because we want to be able
-        #       to modify the list afterwards, to keep track of the actual
-        #       field names created in the layer, that might be laundered to be
-        #       compliant with shapefiles constraints
-        field_names = list(self.dataset.dtype.names)
-        return field_names
+    def get_field_types(self, **kwargs):
+        field_types = {name: self.dataset[name].dtype.char
+                       for name in self.dataset.dtype.names}
+        return field_types
 
-    def add_field_to_layer(self, field_name):
+    def add_field_to_layer(self, field_name, field_type):
+        # TODO: assuming all attributes are numeric (to be checked!)
         field_name = "%s-%s" % (field_name, self.eid)
-        added_field_name = add_numeric_attribute(field_name, self.layer)
+        added_field_name = add_attribute(field_name, field_type, self.layer)
         return added_field_name
 
-    def read_npz_into_layer(self, field_names, rlz_or_stat, **kwargs):
+    def read_npz_into_layer(self, field_types, rlz_or_stat, **kwargs):
         with edit(self.layer):
             feats = []
             fields = self.layer.fields()
             layer_field_names = [field.name() for field in fields]
-            dataset_field_names = self.get_field_names()
+            dataset_field_names = list(self.get_field_types().keys())
             d2l_field_names = dict(
                 list(zip(dataset_field_names[2:], layer_field_names)))
             for row in self.npz_file[rlz_or_stat]:

svir/dialogs/load_hcurves_as_layer_dialog.py

@@ -25,7 +25,6 @@
 from qgis.core import (
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
 from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
-from svir.calculations.calculate_utils import add_numeric_attribute
 from svir.utilities.utils import log_msg, WaitCursorManager
 from svir.tasks.extract_npz_task import ExtractNpzTask
 
@@ -97,8 +96,8 @@ def build_layer_name(self, **kwargs):
         layer_name = "hcurves_%sy" % investigation_time
         return layer_name
 
-    def get_field_names(self, **kwargs):
-        field_names = []
+    def get_field_types(self, **kwargs):
+        field_types = {}
         for rlz_or_stat in self.rlzs_or_stats:
             if (not self.load_all_rlzs_or_stats_chk.isChecked()
                     and rlz_or_stat != self.rlz_or_stat_cbx.currentText()):
@@ -109,16 +108,13 @@ def get_field_names(self, **kwargs):
                     continue
                 for iml in self.dataset[rlz_or_stat][imt].dtype.names:
                     field_name = "%s_%s_%s" % (rlz_or_stat, imt, iml)
-                    field_names.append(field_name)
-        return field_names
+                    # NOTE: assuming that all fields are numeric
+                    field_types[field_name] = 'F'
+        return field_types
 
     def on_iml_changed(self):
         self.set_ok_button()
 
-    def add_field_to_layer(self, field_name):
-        added_field_name = add_numeric_attribute(field_name, self.layer)
-        return added_field_name
-
     def read_npz_into_layer(self, field_names, **kwargs):
         with edit(self.layer):
             lons = self.npz_file['all']['lon']

svir/dialogs/load_hmaps_as_layer_dialog.py

@@ -26,7 +26,6 @@
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
 from qgis.PyQt.QtCore import Qt
 from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
-from svir.calculations.calculate_utils import add_numeric_attribute
 from svir.utilities.utils import WaitCursorManager, log_msg
 from svir.tasks.extract_npz_task import ExtractNpzTask
 
@@ -154,28 +153,19 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
                 rlz_or_stat, imt, poe, investigation_time)
         return layer_name
 
-    def get_field_names(self, **kwargs):
+    def get_field_types(self, **kwargs):
+        field_types = {}
         if self.load_multicol_ckb.isChecked():
-            field_names = []
             for imt in self.imts:
                 for poe in self.imts[imt]:
                     field_name = "%s-%s" % (imt, poe)
-                    field_names.append(field_name)
+                    field_types[field_name] = 'F'
         else:
             imt = kwargs['imt']
             poe = kwargs['poe']
-            field_names = ['%s-%s' % (imt, poe)]
-        return field_names
-
-    def add_field_to_layer(self, field_name):
-        try:
-            # NOTE: add_numeric_attribute uses the native qgis editing manager
-            added_field_name = add_numeric_attribute(field_name, self.layer)
-        except TypeError as exc:
-            log_msg(str(exc), level='C', message_bar=self.iface.messageBar(),
-                    exception=exc)
-            return
-        return added_field_name
+            field_name = '%s-%s' % (imt, poe)
+            field_types[field_name] = 'F'
+        return field_types
 
     def read_npz_into_layer(self, field_names, **kwargs):
         with edit(self.layer):

svir/dialogs/load_losses_by_asset_as_layer_dialog.py

@@ -27,7 +27,6 @@
 from qgis.core import (
     QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
 from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
-from svir.calculations.calculate_utils import add_numeric_attribute
 from svir.utilities.utils import WaitCursorManager, log_msg, get_loss_types
 from svir.tasks.extract_npz_task import ExtractNpzTask
 
@@ -132,19 +131,13 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
             raise NotImplementedError(self.output_type)
         return layer_name
 
-    def get_field_names(self, **kwargs):
+    def get_field_types(self, **kwargs):
         loss_type = kwargs['loss_type']
-        field_names = ['lon', 'lat', loss_type]
+        field_types = {'lon': 'F', 'lat': 'F', loss_type: 'F'}
         self.default_field_name = loss_type
-        return field_names
+        return field_types
 
-    def add_field_to_layer(self, field_name):
-        # NOTE: add_numeric_attribute uses the native qgis editing manager
-        added_field_name = add_numeric_attribute(
-            field_name, self.layer)
-        return added_field_name
-
-    def read_npz_into_layer(self, field_names, **kwargs):
+    def read_npz_into_layer(self, field_types, **kwargs):
         rlz_or_stat = kwargs['rlz_or_stat']
         loss_type = kwargs['loss_type']
         taxonomy = kwargs['taxonomy']
@@ -155,11 +148,14 @@ def read_npz_into_layer(self, field_names, **kwargs):
             for row in grouped_by_site:
                 # add a feature
                 feat = QgsFeature(self.layer.fields())
-                for field_name_idx, field_name in enumerate(field_names):
+                field_idx = 0
+                for field_name, field_type in field_types.items():
                     if field_name in ['lon', 'lat']:
+                        field_idx += 1
                         continue
-                    value = float(row[field_name_idx])
-                    feat.setAttribute(field_names[field_name_idx], value)
+                    value = float(row[field_idx])
+                    feat.setAttribute(field_name, value)
+                    field_idx += 1
                 feat.setGeometry(QgsGeometry.fromPointXY(
                     QgsPointXY(row['lon'], row['lat'])))
                 feats.append(feat)