Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set field data types correctly while loading oq-engine outputs #713

Merged
merged 10 commits into from
Dec 16, 2019
2 changes: 1 addition & 1 deletion scripts/make_doc.sh
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,6 @@ docker run -d --name qgis -v /tmp/.X11-unix:/tmp/.X11-unix \
-e DISPLAY=:99 \
qgis/qgis:final-3_8_3

docker exec -it qgis sh -c "apt update; DEBIAN_FRONTEND=noninteractive apt install -y latexmk texlive-latex-extra python3-matplotlib python3-sphinx python3-sphinx-rtd-theme dvipng"
docker exec -it qgis sh -c "apt update --allow-releaseinfo-change; DEBIAN_FRONTEND=noninteractive apt install -y latexmk texlive-latex-extra python3-matplotlib python3-sphinx python3-sphinx-rtd-theme dvipng"

docker exec -it qgis sh -c "export PYTHONPATH=$PYTHONPATH:/oq-irmt-qgis; cd /oq-irmt-qgis/svir; make doc"
2 changes: 1 addition & 1 deletion scripts/run_integration_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ docker run -d --name qgis -v /tmp/.X11-unix:/tmp/.X11-unix \
-e GEM_QGIS_TEST=y \
qgis/qgis:final-3_8_3

docker exec -it qgis sh -c "apt update; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
docker exec -it qgis sh -c "apt update --allow-releaseinfo-change; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"

docker exec -it qgis sh -c "git clone -q -b $BRANCH --depth=1 https://github.com/gem/oq-engine.git && echo 'Running against oq-engine/$BRANCH'"

Expand Down
2 changes: 1 addition & 1 deletion scripts/run_unit_tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ docker run -d --name qgis -v /tmp/.X11-unix:/tmp/.X11-unix \
-e GEM_QGIS_TEST=y \
qgis/qgis:final-3_8_3

docker exec -it qgis bash -c "apt update; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
docker exec -it qgis bash -c "apt update --allow-releaseinfo-change; DEBIAN_FRONTEND=noninteractive apt install -y python3-scipy python3-matplotlib python3-pyqt5.qtwebkit"
docker exec -it qgis bash -c "python3 -m pip install pytest"

# OGR_SQLITE_JOURNAL=delete prevents QGIS from using WAL, which modifies geopackages even if they are just read
Expand Down
33 changes: 15 additions & 18 deletions svir/calculations/calculate_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,7 @@

from qgis.PyQt.QtCore import QVariant

from svir.utilities.shared import (DOUBLE_FIELD_TYPE_NAME,
STRING_FIELD_TYPE_NAME,
from svir.utilities.shared import (
DEBUG,
SUM_BASED_OPERATORS,
MUL_BASED_OPERATORS, DEFAULT_OPERATOR,
Expand All @@ -63,18 +62,18 @@ class InvalidFormula(Exception):
pass


def add_numeric_attribute(proposed_attr_name, layer):
field = QgsField(proposed_attr_name, QVariant.Double)
field.setTypeName(DOUBLE_FIELD_TYPE_NAME)
assigned_attr_names = ProcessLayer(layer).add_attributes(
[field])
assigned_attr_name = assigned_attr_names[proposed_attr_name]
return assigned_attr_name


def add_textual_attribute(proposed_attr_name, layer):
field = QgsField(proposed_attr_name, QVariant.String)
field.setTypeName(STRING_FIELD_TYPE_NAME)
def add_attribute(proposed_attr_name, dtype, layer):
if dtype == 'S':
qtype = QVariant.String
qname = 'String'
elif dtype in ('U', 'I'): # FIXME: what for unsigned int?
qtype = QVariant.Int
qname = 'integer'
else: # FIXME: treating everything else as double (it might be wrong)
qtype = QVariant.Double
qname = 'double'
field = QgsField(proposed_attr_name, qtype)
field.setTypeName(qname)
assigned_attr_names = ProcessLayer(layer).add_attributes(
[field])
assigned_attr_name = assigned_attr_names[proposed_attr_name]
Expand Down Expand Up @@ -194,15 +193,13 @@ def get_node_attr_id_and_name(node, layer):
# deleted it). If it is not there anymore, add a new field
if layer.fields().indexOf(node_attr_name) == -1: # not found
proposed_node_attr_name = node_attr_name
node_attr_name = add_numeric_attribute(
proposed_node_attr_name, layer)
node_attr_name = add_attribute(proposed_node_attr_name, 'F', layer)
field_was_added = True
elif DEBUG:
log_msg('Reusing field %s' % node_attr_name)
elif 'name' in node:
proposed_node_attr_name = node['name']
node_attr_name = add_numeric_attribute(
proposed_node_attr_name, layer)
node_attr_name = add_attribute(proposed_node_attr_name, 'F', layer)
field_was_added = True
else: # this corner case should never happen (hopefully)
raise InvalidNode('This node has no name and it does'
Expand Down
22 changes: 6 additions & 16 deletions svir/dialogs/load_asset_risk_as_layer_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
from qgis.core import (
QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication,)
from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
from svir.calculations.calculate_utils import add_numeric_attribute
from svir.utilities.utils import WaitCursorManager, log_msg
from svir.ui.multi_select_combo_box import MultiSelectComboBox
from svir.tasks.extract_npz_task import ExtractNpzTask
Expand Down Expand Up @@ -191,21 +190,12 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
self.category_cbx.currentText())
return layer_name

def get_field_names(self, **kwargs):
field_names = [name for name in self.dataset.dtype.names
if name not in self.tag_names and
name not in ['lon', 'lat']]
return field_names

def add_field_to_layer(self, field_name):
try:
# NOTE: add_numeric_attribute uses the native qgis editing manager
added_field_name = add_numeric_attribute(field_name, self.layer)
except TypeError as exc:
log_msg(str(exc), level='C', message_bar=self.iface.messageBar(),
exception=exc)
return
return added_field_name
def get_field_types(self, **kwargs):
field_types = {name: self.dataset[name].dtype.char
for name in self.dataset.dtype.names
if name not in ['lon', 'lat']
and name not in self.tag_names}
return field_types

def read_npz_into_layer(self, field_names, **kwargs):
with edit(self.layer):
Expand Down
42 changes: 20 additions & 22 deletions svir/dialogs/load_dmg_by_asset_as_layer_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from qgis.core import (
QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
from svir.calculations.calculate_utils import add_numeric_attribute
from svir.utilities.utils import (WaitCursorManager,
log_msg,
get_loss_types,
Expand Down Expand Up @@ -157,7 +156,7 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
layer_name = "dmg_by_asset_%s_%s" % (rlz_or_stat, loss_type)
return layer_name

def get_field_names(self, **kwargs):
def get_field_types(self, **kwargs):
loss_type = kwargs['loss_type']
dmg_state = kwargs['dmg_state']
if self.aggregate_by_site_ckb.isChecked():
Expand All @@ -175,23 +174,19 @@ def get_field_names(self, **kwargs):
self.default_field_name = "%s_%s_mean" % (
self.loss_type_cbx.currentText(),
self.dmg_state_cbx.currentText())
return field_names
# NOTE: assuming that all fields are numeric
field_types = {field_name: 'F' for field_name in field_names}
return field_types

def add_field_to_layer(self, field_name):
# NOTE: add_numeric_attribute uses the native qgis editing manager
added_field_name = add_numeric_attribute(
field_name, self.layer)
return added_field_name

def read_npz_into_layer(self, field_names, **kwargs):
def read_npz_into_layer(self, field_types, **kwargs):
if self.aggregate_by_site_ckb.isChecked():
self.read_npz_into_layer_aggr_by_site(field_names, **kwargs)
self.read_npz_into_layer_aggr_by_site(field_types, **kwargs)
else:
# do not aggregate by site, then aggregate by zone afterwards if
# required
self.read_npz_into_layer_no_aggr(field_names, **kwargs)
self.read_npz_into_layer_no_aggr(field_types, **kwargs)

def read_npz_into_layer_no_aggr(self, field_names, **kwargs):
def read_npz_into_layer_no_aggr(self, field_types, **kwargs):
rlz_or_stat = kwargs['rlz_or_stat']
loss_type = kwargs['loss_type']
with edit(self.layer):
Expand All @@ -200,19 +195,19 @@ def read_npz_into_layer_no_aggr(self, field_names, **kwargs):
for row in data:
# add a feature
feat = QgsFeature(self.layer.fields())
for field_name_idx, field_name in enumerate(field_names):
for field_name, field_type in field_types.items():
if field_name in ['lon', 'lat']:
continue
elif field_name in data.dtype.names:
value = row[field_name]
try:
value = float(value)
except ValueError:
if data[field_name].dtype.char == 'S':
value = str(value, encoding='utf8').strip('"')
else:
value = float(value)
else:
value = float(
row[loss_type][field_name[len(loss_type)+1:]])
feat.setAttribute(field_names[field_name_idx], value)
feat.setAttribute(field_name, value)
feat.setGeometry(QgsGeometry.fromPointXY(
QgsPointXY(row['lon'], row['lat'])))
feats.append(feat)
Expand All @@ -221,7 +216,7 @@ def read_npz_into_layer_no_aggr(self, field_names, **kwargs):
msg = 'There was a problem adding features to the layer.'
log_msg(msg, level='C', message_bar=self.iface.messageBar())

def read_npz_into_layer_aggr_by_site(self, field_names, **kwargs):
def read_npz_into_layer_aggr_by_site(self, field_types, **kwargs):
rlz_or_stat = kwargs['rlz_or_stat']
loss_type = kwargs['loss_type']
taxonomy = kwargs['taxonomy']
Expand All @@ -233,11 +228,14 @@ def read_npz_into_layer_aggr_by_site(self, field_names, **kwargs):
for row in grouped_by_site:
# add a feature
feat = QgsFeature(self.layer.fields())
for field_name_idx, field_name in enumerate(field_names):
field_idx = 0
for field_name, field_type in field_types.items():
if field_name in ['lon', 'lat']:
field_idx += 1
continue
value = float(row[field_name_idx])
feat.setAttribute(field_names[field_name_idx], value)
value = float(row[field_idx])
feat.setAttribute(field_name, value)
field_idx += 1
feat.setGeometry(QgsGeometry.fromPointXY(
QgsPointXY(row['lon'], row['lat'])))
feats.append(feat)
Expand Down
22 changes: 10 additions & 12 deletions svir/dialogs/load_gmf_data_as_layer_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
from qgis.core import (
QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
from svir.calculations.calculate_utils import add_numeric_attribute
from svir.calculations.calculate_utils import add_attribute
from svir.utilities.utils import WaitCursorManager, log_msg, extract_npz
from svir.tasks.extract_npz_task import ExtractNpzTask

Expand Down Expand Up @@ -136,25 +136,23 @@ def build_layer_name(self, gsim=None, **kwargs):
layer_name = "scenario_gmfs_%s_eid-%s" % (gsim, self.eid)
return layer_name

def get_field_names(self, **kwargs):
# NOTE: we need a list instead of a tuple, because we want to be able
# to modify the list afterwards, to keep track of the actual
# field names created in the layer, that might be laundered to be
# compliant with shapefiles constraints
field_names = list(self.dataset.dtype.names)
return field_names
def get_field_types(self, **kwargs):
field_types = {name: self.dataset[name].dtype.char
for name in self.dataset.dtype.names}
return field_types

def add_field_to_layer(self, field_name):
def add_field_to_layer(self, field_name, field_type):
# TODO: assuming all attributes are numeric (to be checked!)
field_name = "%s-%s" % (field_name, self.eid)
added_field_name = add_numeric_attribute(field_name, self.layer)
added_field_name = add_attribute(field_name, field_type, self.layer)
return added_field_name

def read_npz_into_layer(self, field_names, rlz_or_stat, **kwargs):
def read_npz_into_layer(self, field_types, rlz_or_stat, **kwargs):
with edit(self.layer):
feats = []
fields = self.layer.fields()
layer_field_names = [field.name() for field in fields]
dataset_field_names = self.get_field_names()
dataset_field_names = list(self.get_field_types().keys())
d2l_field_names = dict(
list(zip(dataset_field_names[2:], layer_field_names)))
for row in self.npz_file[rlz_or_stat]:
Expand Down
14 changes: 5 additions & 9 deletions svir/dialogs/load_hcurves_as_layer_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
from qgis.core import (
QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
from svir.calculations.calculate_utils import add_numeric_attribute
from svir.utilities.utils import log_msg, WaitCursorManager
from svir.tasks.extract_npz_task import ExtractNpzTask

Expand Down Expand Up @@ -97,8 +96,8 @@ def build_layer_name(self, **kwargs):
layer_name = "hcurves_%sy" % investigation_time
return layer_name

def get_field_names(self, **kwargs):
field_names = []
def get_field_types(self, **kwargs):
field_types = {}
for rlz_or_stat in self.rlzs_or_stats:
if (not self.load_all_rlzs_or_stats_chk.isChecked()
and rlz_or_stat != self.rlz_or_stat_cbx.currentText()):
Expand All @@ -109,16 +108,13 @@ def get_field_names(self, **kwargs):
continue
for iml in self.dataset[rlz_or_stat][imt].dtype.names:
field_name = "%s_%s_%s" % (rlz_or_stat, imt, iml)
field_names.append(field_name)
return field_names
# NOTE: assuming that all fields are numeric
field_types[field_name] = 'F'
return field_types

def on_iml_changed(self):
self.set_ok_button()

def add_field_to_layer(self, field_name):
added_field_name = add_numeric_attribute(field_name, self.layer)
return added_field_name

def read_npz_into_layer(self, field_names, **kwargs):
with edit(self.layer):
lons = self.npz_file['all']['lon']
Expand Down
22 changes: 6 additions & 16 deletions svir/dialogs/load_hmaps_as_layer_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
from qgis.PyQt.QtCore import Qt
from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
from svir.calculations.calculate_utils import add_numeric_attribute
from svir.utilities.utils import WaitCursorManager, log_msg
from svir.tasks.extract_npz_task import ExtractNpzTask

Expand Down Expand Up @@ -154,28 +153,19 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
rlz_or_stat, imt, poe, investigation_time)
return layer_name

def get_field_names(self, **kwargs):
def get_field_types(self, **kwargs):
field_types = {}
if self.load_multicol_ckb.isChecked():
field_names = []
for imt in self.imts:
for poe in self.imts[imt]:
field_name = "%s-%s" % (imt, poe)
field_names.append(field_name)
field_types[field_name] = 'F'
else:
imt = kwargs['imt']
poe = kwargs['poe']
field_names = ['%s-%s' % (imt, poe)]
return field_names

def add_field_to_layer(self, field_name):
try:
# NOTE: add_numeric_attribute uses the native qgis editing manager
added_field_name = add_numeric_attribute(field_name, self.layer)
except TypeError as exc:
log_msg(str(exc), level='C', message_bar=self.iface.messageBar(),
exception=exc)
return
return added_field_name
field_name = '%s-%s' % (imt, poe)
field_types[field_name] = 'F'
return field_types

def read_npz_into_layer(self, field_names, **kwargs):
with edit(self.layer):
Expand Down
24 changes: 10 additions & 14 deletions svir/dialogs/load_losses_by_asset_as_layer_dialog.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
from qgis.core import (
QgsFeature, QgsGeometry, QgsPointXY, edit, QgsTask, QgsApplication)
from svir.dialogs.load_output_as_layer_dialog import LoadOutputAsLayerDialog
from svir.calculations.calculate_utils import add_numeric_attribute
from svir.utilities.utils import WaitCursorManager, log_msg, get_loss_types
from svir.tasks.extract_npz_task import ExtractNpzTask

Expand Down Expand Up @@ -132,19 +131,13 @@ def build_layer_name(self, rlz_or_stat=None, **kwargs):
raise NotImplementedError(self.output_type)
return layer_name

def get_field_names(self, **kwargs):
def get_field_types(self, **kwargs):
loss_type = kwargs['loss_type']
field_names = ['lon', 'lat', loss_type]
field_types = {'lon': 'F', 'lat': 'F', loss_type: 'F'}
self.default_field_name = loss_type
return field_names
return field_types

def add_field_to_layer(self, field_name):
# NOTE: add_numeric_attribute uses the native qgis editing manager
added_field_name = add_numeric_attribute(
field_name, self.layer)
return added_field_name

def read_npz_into_layer(self, field_names, **kwargs):
def read_npz_into_layer(self, field_types, **kwargs):
rlz_or_stat = kwargs['rlz_or_stat']
loss_type = kwargs['loss_type']
taxonomy = kwargs['taxonomy']
Expand All @@ -155,11 +148,14 @@ def read_npz_into_layer(self, field_names, **kwargs):
for row in grouped_by_site:
# add a feature
feat = QgsFeature(self.layer.fields())
for field_name_idx, field_name in enumerate(field_names):
field_idx = 0
for field_name, field_type in field_types.items():
if field_name in ['lon', 'lat']:
field_idx += 1
continue
value = float(row[field_name_idx])
feat.setAttribute(field_names[field_name_idx], value)
value = float(row[field_idx])
feat.setAttribute(field_name, value)
field_idx += 1
feat.setGeometry(QgsGeometry.fromPointXY(
QgsPointXY(row['lon'], row['lat'])))
feats.append(feat)
Expand Down
Loading