diff --git a/.local_docker_tests/Dockerfile b/.local_docker_tests/Dockerfile new file mode 100644 index 0000000..0da29d8 --- /dev/null +++ b/.local_docker_tests/Dockerfile @@ -0,0 +1,33 @@ +ARG QGIS_TEST_VERSION=latest +FROM qgis/qgis:${QGIS_TEST_VERSION} + +RUN apt-get update && \ + apt-get -y install openjdk-8-jre \ + python3-pip \ + && rm -rf /var/lib/apt/lists/* + +# MSSQL: client side +RUN apt-get update +RUN apt-get install -y unixodbc unixodbc-dev odbcinst odbcinst1debian2 libodbc1 libqt5sql5-odbc + +RUN curl https://packages.microsoft.com/keys/microsoft.asc | apt-key add - +RUN curl https://packages.microsoft.com/config/ubuntu/20.04/prod.list | tee /etc/apt/sources.list.d/msprod.list +RUN apt-get update +RUN ACCEPT_EULA=Y apt-get install -y msodbcsql17 mssql-tools + +# Python deps +COPY ./requirements.txt /tmp/ +ENV VIRTUAL_ENV=/opt/venv +RUN python3 -m venv $VIRTUAL_ENV +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +RUN pip3 install -r /tmp/requirements.txt + +# Avoid sqlcmd termination due to locale -- see https://github.com/Microsoft/mssql-docker/issues/163 +RUN echo "nb_NO.UTF-8 UTF-8" > /etc/locale.gen +RUN echo "en_US.UTF-8 UTF-8" >> /etc/locale.gen +RUN locale-gen +ENV PATH="/usr/local/bin:${PATH}" + +ENV LANG=C.UTF-8 + +WORKDIR / diff --git a/.local_docker_tests/commands.txt b/.local_docker_tests/commands.txt new file mode 100644 index 0000000..5a3f795 --- /dev/null +++ b/.local_docker_tests/commands.txt @@ -0,0 +1,13 @@ +docker-compose up -d + +docker exec -it local_docker_tests_qgis_1 bash + +in docker: +/usr/src/.local_docker_tests/run-docker-tests.sh; +java -jar /usr/src/modelbaker/iliwrapper/bin/ili2mssql-5.0.0/ili2mssql-5.0.0.jar --schemaimport --dbhost mssql --dbusr sa --dbpwd '' --dbdatabase gis --dbschema optimal_staedtische_manuel --coalesceCatalogueRef --createEnumTabs --createNumChecks --createUnique --createFk --createFkIdx --coalesceMultiSurface --coalesceMultiLine --coalesceMultiPoint --coalesceArray --beautifyEnumDispName --createGeomIdx --createMetaInfo --expandMultilingual --createTypeConstraint --createEnumTabsWithId --createTidCol --importTid --smart2Inheritance --strokeArcs --createBasketCol --defaultSrsAuth EPSG --defaultSrsCode 2056 --preScript NULL --postScript NULL --models Staedtische_Ortsplanung_V1_1 --iliMetaAttrs NULL /usr/src/tests/testdata/ilimodels/Staedtische_Ortsplanung_V1_1.ili + +lokal: installiere vscode plugin mssql client und connecte: +localhost +1433 +sa +'' diff --git a/.local_docker_tests/docker-compose.yml b/.local_docker_tests/docker-compose.yml new file mode 100644 index 0000000..5c07a9c --- /dev/null +++ b/.local_docker_tests/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3' +services: + mssql: + image: mcr.microsoft.com/mssql/server:2019-CU11-ubuntu-20.04 + environment: + ACCEPT_EULA: Y + SA_PASSWORD: + ports: + - "1433:1433" + + qgis: + build: + context: .. + dockerfile: ./.docker/Dockerfile + args: + QGIS_TEST_VERSION: latest + tty: true + volumes: + - ..:/usr/src + links: + - mssql diff --git a/.local_docker_tests/run-docker-tests.sh b/.local_docker_tests/run-docker-tests.sh new file mode 100755 index 0000000..e2aa842 --- /dev/null +++ b/.local_docker_tests/run-docker-tests.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +#*************************************************************************** +# ------------------- +# begin : 2017-08-24 +# git sha : :%H$ +# copyright : (C) 2017 by OPENGIS.ch +# email : info@opengis.ch +#*************************************************************************** +# +#*************************************************************************** +#* * +#* This program is free software; you can redistribute it and/or modify * +#* it under the terms of the GNU General Public License as published by * +#* the Free Software Foundation; either version 2 of the License, or * +#* (at your option) any later version. * +#* * +#*************************************************************************** + +set -e + +/usr/src/tests/testdata/mssql/setup-mssql.sh diff --git a/.local_docker_tests/statement.sql b/.local_docker_tests/statement.sql new file mode 100644 index 0000000..2e8b657 --- /dev/null +++ b/.local_docker_tests/statement.sql @@ -0,0 +1,234 @@ + + SELECT distinct + tbls.TABLE_SCHEMA AS schemaname + , tbls.TABLE_NAME AS tablename + , Col.Column_Name AS primary_key + , clm.COLUMN_NAME AS geometry_column + , tsrid.setting AS srid + , p.setting AS kind_settings + , alias.setting AS table_alias + , left(c.iliname, charindex('.', c.iliname)-1) AS model + , c.iliname AS ili_name + , STUFF( + (SELECT ';' + CAST(cp.setting AS VARCHAR(MAX)) + FROM optimal_polymorph_manuel.t_ili2db_column_prop cp + WHERE tbls.table_name = cp.tablename + AND clm.COLUMN_NAME = cp.columnname + AND cp.tag IN + ('ch.ehi.ili2db.c1Min', 'ch.ehi.ili2db.c2Min', + 'ch.ehi.ili2db.c1Max', 'ch.ehi.ili2db.c2Max') + order by case cp.tag WHEN 'ch.ehi.ili2db.c1Min' THEN 1 + WHEN 'ch.ehi.ili2db.c2Min' THEN 2 + WHEN 'ch.ehi.ili2db.c1Max' THEN 3 + WHEN 'ch.ehi.ili2db.c2Max' THEN 4 + END + FOR XML PATH(''),TYPE).value('(./text())[1]','VARCHAR(MAX)'),1,1,'' + ) AS extent + , ( SELECT CASE MAX(CHARINDEX('.',cp.setting)) WHEN 0 THEN 0 ELSE MAX( LEN(cp.setting) - CHARINDEX('.',cp.setting) ) END + FROM optimal_polymorph_manuel.t_ili2db_column_prop cp + WHERE tbls.table_name = cp.tablename + AND clm.COLUMN_NAME = cp.columnname + AND cp.tag IN + ('ch.ehi.ili2db.c1Min', 'ch.ehi.ili2db.c2Min', + 'ch.ehi.ili2db.c1Max', 'ch.ehi.ili2db.c2Max') + ) AS coord_decimals + , tgeomtype.setting AS simple_type + , null AS formatted_type + , attrs.sqlname AS attribute_name + , CASE WHEN c.iliname IN ( + SELECT i.baseClass as base, base_names.class, extend_names.class, base_names.model, extend_names.model + FROM optimal_polymorph_manuel.t_ili2db_inheritance i + LEFT JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.',topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 1, charindex('.', thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM optimal_polymorph_manuel.t_ili2db_inheritance + ) AS extended + ) AS extend_names + ON thisClass = extend_names.fullname + LEFT JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.', topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 1, charindex('.',thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM optimal_polymorph_manuel.t_ili2db_inheritance + ) AS topic_level_name + ) AS base_names + ON baseClass = base_names.fullname + -- it's extended + WHERE baseClass IS NOT NULL + -- in a different model + AND base_names.model != extend_names.model + AND ( + -- with the same name + base_names.class = extend_names.class + OR + -- multiple times in the same extended model + (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM optimal_polymorph_manuel.t_ili2db_inheritance JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.', topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 0, charindex('.',thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM optimal_polymorph_manuel.t_ili2db_inheritance + ) AS topic_level_name + ) AS extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 + ) + ) + THEN 0 ELSE 1 END AS relevance + +FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS Tab +INNER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS Col + ON Col.Constraint_Name = Tab.Constraint_Name + AND Col.Table_Name = Tab.Table_Name + AND Col.CONSTRAINT_SCHEMA = Tab.CONSTRAINT_SCHEMA +RIGHT JOIN INFORMATION_SCHEMA.TABLES AS tbls + ON Tab.TABLE_NAME = tbls.TABLE_NAME + AND Tab.CONSTRAINT_SCHEMA = tbls.TABLE_SCHEMA + AND Tab.Constraint_Type = 'PRIMARY KEY' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_TABLE_PROP AS p + ON p.tablename = tbls.TABLE_NAME + AND p.tag = 'ch.ehi.ili2db.tableKind' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_TABLE_PROP AS alias + ON alias.tablename = tbls.TABLE_NAME + AND alias.tag = 'ch.ehi.ili2db.dispName' +LEFT JOIN optimal_polymorph_manuel.t_ili2db_classname AS c + ON tbls.TABLE_NAME = c.sqlname +LEFT JOIN optimal_polymorph_manuel.t_ili2db_attrname AS attrs + ON c.iliname = attrs.iliname +LEFT JOIN INFORMATION_SCHEMA.COLUMNS AS clm + ON clm.TABLE_NAME = tbls.TABLE_NAME + AND clm.TABLE_SCHEMA = tbls.TABLE_SCHEMA + AND clm.DATA_TYPE = 'geometry' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_COLUMN_PROP AS tsrid + ON tbls.TABLE_NAME = tsrid.tablename + AND clm.COLUMN_NAME = tsrid.columnname + AND tsrid.tag='ch.ehi.ili2db.srid' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_COLUMN_PROP AS tgeomtype + ON tbls.TABLE_NAME = tgeomtype.tablename + AND clm.COLUMN_NAME = tgeomtype.columnname + AND tgeomtype.tag= 'ch.ehi.ili2db.geomType' +WHERE tbls.TABLE_TYPE = 'BASE TABLE' AND tbls.TABLE_SCHEMA = 'optimal_polymorph_manuel' + +SELECT distinct + tbls.TABLE_SCHEMA AS schemaname + , tbls.TABLE_NAME AS tablename + , Col.Column_Name AS primary_key + , clm.COLUMN_NAME AS geometry_column + , tsrid.setting AS srid + , p.setting AS kind_settings + , alias.setting AS table_alias + , left(c.iliname, charindex('.', c.iliname)-1) AS model + , c.iliname AS ili_name + , STUFF( + (SELECT ';' + CAST(cp.setting AS VARCHAR(MAX)) + FROM optimal_polymorph_manuel.t_ili2db_column_prop cp + WHERE tbls.table_name = cp.tablename + AND clm.COLUMN_NAME = cp.columnname + AND cp.tag IN + ('ch.ehi.ili2db.c1Min', 'ch.ehi.ili2db.c2Min', + 'ch.ehi.ili2db.c1Max', 'ch.ehi.ili2db.c2Max') + order by case cp.tag WHEN 'ch.ehi.ili2db.c1Min' THEN 1 + WHEN 'ch.ehi.ili2db.c2Min' THEN 2 + WHEN 'ch.ehi.ili2db.c1Max' THEN 3 + WHEN 'ch.ehi.ili2db.c2Max' THEN 4 + END + FOR XML PATH(''),TYPE).value('(./text())[1]','VARCHAR(MAX)'),1,1,'' + ) AS extent + , ( SELECT CASE MAX(CHARINDEX('.',cp.setting)) WHEN 0 THEN 0 ELSE MAX( LEN(cp.setting) - CHARINDEX('.',cp.setting) ) END + FROM optimal_polymorph_manuel.t_ili2db_column_prop cp + WHERE tbls.table_name = cp.tablename + AND clm.COLUMN_NAME = cp.columnname + AND cp.tag IN + ('ch.ehi.ili2db.c1Min', 'ch.ehi.ili2db.c2Min', + 'ch.ehi.ili2db.c1Max', 'ch.ehi.ili2db.c2Max') + ) AS coord_decimals + , tgeomtype.setting AS simple_type + , null AS formatted_type + , attrs.sqlname AS attribute_name + , CASE WHEN c.iliname IN ( + SELECT i.baseClass as base + FROM optimal_polymorph_manuel.t_ili2db_inheritance i + LEFT JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.',topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 1, charindex('.', thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM optimal_polymorph_manuel.t_ili2db_inheritance + ) AS extended + ) AS extend_names + ON thisClass = extend_names.fullname + LEFT JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.', topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 0, charindex('.',thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM optimal_polymorph_manuel.t_ili2db_inheritance + ) AS topic_level_name + ) AS base_names + ON baseClass = base_names.fullname + -- it's extended + WHERE baseClass IS NOT NULL + -- in a different model + AND base_names.model != extend_names.model + AND ( + -- with the same name + base_names.class = extend_names.class + OR + -- multiple times in the same extended model + (SELECT MAX(count) FROM (SELECT COUNT(baseClass) AS count FROM optimal_polymorph_manuel.t_ili2db_inheritance JOIN ( + SELECT fullname, model, topicclass, substring(topicclass, charindex('.', topicclass)+1, len(topicclass)) as class + FROM ( + SELECT + thisClass as fullname, + substring(thisClass, 0, charindex('.',thisClass)-1) as model, + substring(thisClass, charindex('.', thisClass)+1, len(thisClass)) as topicclass + FROM optimal_polymorph_manuel.t_ili2db_inheritance + ) AS topic_level_name + ) AS extend_names ON thisClass = extend_names.fullname WHERE baseClass = i.baseClass GROUP BY baseClass, extend_names.model) AS counts )>1 + ) + ) + THEN 0 ELSE 1 END AS relevance + +FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS Tab +INNER JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS Col + ON Col.Constraint_Name = Tab.Constraint_Name + AND Col.Table_Name = Tab.Table_Name + AND Col.CONSTRAINT_SCHEMA = Tab.CONSTRAINT_SCHEMA +RIGHT JOIN INFORMATION_SCHEMA.TABLES AS tbls + ON Tab.TABLE_NAME = tbls.TABLE_NAME + AND Tab.CONSTRAINT_SCHEMA = tbls.TABLE_SCHEMA + AND Tab.Constraint_Type = 'PRIMARY KEY' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_TABLE_PROP AS p + ON p.tablename = tbls.TABLE_NAME + AND p.tag = 'ch.ehi.ili2db.tableKind' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_TABLE_PROP AS alias + ON alias.tablename = tbls.TABLE_NAME + AND alias.tag = 'ch.ehi.ili2db.dispName' +LEFT JOIN optimal_polymorph_manuel.t_ili2db_classname AS c + ON tbls.TABLE_NAME = c.sqlname +LEFT JOIN optimal_polymorph_manuel.t_ili2db_attrname AS attrs + ON c.iliname = attrs.iliname +LEFT JOIN INFORMATION_SCHEMA.COLUMNS AS clm + ON clm.TABLE_NAME = tbls.TABLE_NAME + AND clm.TABLE_SCHEMA = tbls.TABLE_SCHEMA + AND clm.DATA_TYPE = 'geometry' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_COLUMN_PROP AS tsrid + ON tbls.TABLE_NAME = tsrid.tablename + AND clm.COLUMN_NAME = tsrid.columnname + AND tsrid.tag='ch.ehi.ili2db.srid' +LEFT JOIN optimal_polymorph_manuel.T_ILI2DB_COLUMN_PROP AS tgeomtype + ON tbls.TABLE_NAME = tgeomtype.tablename + AND clm.COLUMN_NAME = tgeomtype.columnname + AND tgeomtype.tag= 'ch.ehi.ili2db.geomType' +WHERE tbls.TABLE_TYPE = 'BASE TABLE' AND tbls.TABLE_SCHEMA = 'optimal_polymorph_manuel' diff --git a/modelbaker/db_factory/gpkg_layer_uri.py b/modelbaker/db_factory/gpkg_layer_uri.py index 3077236..858309a 100644 --- a/modelbaker/db_factory/gpkg_layer_uri.py +++ b/modelbaker/db_factory/gpkg_layer_uri.py @@ -34,4 +34,8 @@ def get_data_source_uri(self, record: dict) -> str: data_source_uri = "{uri}|layername={table}".format( uri=self.uri, table=record["tablename"] ) + if record["geometry_column"]: + data_source_uri = "{} ({})".format( + data_source_uri, record["geometry_column"] + ) return data_source_uri diff --git a/modelbaker/iliwrapper/ili2dbconfig.py b/modelbaker/iliwrapper/ili2dbconfig.py index 2dde42e..67033b8 100644 --- a/modelbaker/iliwrapper/ili2dbconfig.py +++ b/modelbaker/iliwrapper/ili2dbconfig.py @@ -20,6 +20,7 @@ from qgis.core import QgsNetworkAccessManager from qgis.PyQt.QtNetwork import QNetworkProxy +from .globals import DbIliMode from .ili2dbutils import get_all_modeldir_in_path @@ -298,6 +299,10 @@ def to_ili2db_args(self, extra_args=[], with_action=True): elif self.db_ili_version is None or self.db_ili_version > 3: self.append_args(args, ["--createBasketCol=False"]) + print(f"{self.tool} is") + if self.tool == DbIliMode.gpkg: + self.append_args(args, ["--gpkgMultiGeomPerTable"], True) + self.append_args(args, ["--defaultSrsAuth", self.srs_auth]) self.append_args(args, ["--defaultSrsCode", "{}".format(self.srs_code)])