Merge pull request tum-gis#11 from ondics/sddi-odp-2.9.9-datasetfields

SDDI ODP 2.9.9 Branch datasetfields mit diversen Änderungen
it-at-m · May 21, 2024 · 0bac577 · 0bac577
2 parents e81ed82 + 0bf1e9c
commit 0bac577
Show file tree

Hide file tree

Showing 16 changed files with 1,070 additions and 39 deletions.
diff --git a/.github/workflows/edge-sddi-odp.yml b/.github/workflows/edge-sddi-odp.yml
@@ -40,8 +40,8 @@ jobs:
           images: |
             ${{ env.REGISTRY }}/ondics/ckan-${{ matrix.context }}
           tags: |
-            type=edge,branch=sddi-odp-2.9.9,suffix=-2.9.9-dev
-            type=raw,value=odp-katalog-1.0.0-dev
+            type=edge,branch=sddi-odp-2.9.9,suffix=-2.9.9
+            type=raw,value=odp-katalog-1.1.0
           labels: |
             maintainer=Ondics GmbH, Esslingen am Neckar
             org.opencontainers.image.vendor=Ondics GmbH, Esslingen am Neckar
@@ -64,8 +64,8 @@ jobs:
           images: |
             ${{ env.REGISTRY }}/ondics/ckan-${{ matrix.context }}
           tags: |
-            type=edge,branch=sddi-odp-2.9.9,suffix=-2.9.9-debug-dev
-            type=raw,value=odp-katalog-1.0.0-debug-dev
+            type=edge,branch=sddi-odp-2.9.9,suffix=-2.9.9-debug
+            type=raw,value=odp-katalog-1.1.0-debug
           labels: |
             maintainer=Ondics GmbH, Esslingen am Neckar
             org.opencontainers.image.vendor=Ondics GmbH, Esslingen am Neckar
@@ -80,4 +80,4 @@ jobs:
           tags: ${{ steps.meta-debug.outputs.tags }}
           labels: ${{ steps.meta-debug.outputs.labels }}
           build-args: |
-            BASEIMAGE_VERSION=edge-2.9.9-dev
+            BASEIMAGE_VERSION=edge-2.9.9
diff --git a/sddi-odp/CHANGELOG.md b/sddi-odp/CHANGELOG.md
@@ -1,6 +1,16 @@
 CHANGELOG
 =========
 
+21.05.2024 (v1.1.0):
+
+* ENH: API-Link zu Datensatz (in datasetview)
+* ENH: Metadaten-Links zu rdf, json, tt3, n3 (in datasetview)
+* ENH: OpenAPI View Plugin installiert
+
+25.04.2024:
+
+* ENH: Datensatzfelder Musterdatenkatalog, Update-Zyklus und HVD hinzugefügt über ckanext-scheming
+
 18.04.2024:
 
 * FIX: Image Version für debug workflow angepasst

diff --git a/sddi-odp/Dockerfile b/sddi-odp/Dockerfile
@@ -50,13 +50,17 @@ RUN set -ex && \
   ls -lah /wheels
 
 # ckanext-scheming ############################################################
-ARG CKANEXT_SCHEMING_VERSION="f98daec"
+ARG CKANEXT_SCHEMING_VERSION="9a33dcd"
 ENV CKANEXT_SCHEMING_VERSION=${CKANEXT_SCHEMING_VERSION}
-ENV CKANEXT_SCHEMING_GITHUB_URL="https://github.com/MarijaKnezevic/ckanext-scheming"
+ENV CKANEXT_SCHEMING_GITHUB_URL="https://github.com/ckan/ckanext-scheming"
 
-RUN set -ex && \
-  pip wheel --wheel-dir=/wheels \
-    git+${CKANEXT_SCHEMING_GITHUB_URL}.git@${CKANEXT_SCHEMING_VERSION}#egg=ckanext-scheming
+# RUN set -ex && \
+#   pip wheel --wheel-dir=/wheels \
+#     git+${CKANEXT_SCHEMING_GITHUB_URL}.git@${CKANEXT_SCHEMING_VERSION}#egg=ckanext-scheming
+
+# RUN set -ex && \
+#   pip wheel --wheel-dir=/wheels \
+#     git+${CKANEXT_SCHEMING_GITHUB_URL}.git#egg=ckanext-scheming
 
 # ckanext datesearch ##########################################################
 ARG CKANEXT_DATESEARCH_VERSION="1.0.2"
@@ -113,6 +117,18 @@ RUN set -ex && \
   pip wheel --wheel-dir=/wheels \
     git+${CKANEXT_HARVEST_GITHUB_URL}.git@${CKANEXT_HARVEST_VERSION}#egg=ckanext-harvest
 
+# ckanext-geoview #############################################################
+ARG CKANEXT_GEOVIEW_VERSION="v0.1.0"
+ENV CKANEXT_GEOVIEW_VERSION=${CKANEXT_GEOVIEW_VERSION}
+ENV CKANEXT_GEOVIEW_GITHUB_URL="https://github.com/ckan/ckanext-geoview"
+
+RUN set -ex && \
+  mkdir -p /wheels && \
+  pip install -r \
+    https://raw.githubusercontent.com/ckan/ckanext-geoview/${CKANEXT_GEOVIEW_VERSION}/dev-requirements.txt && \
+  pip wheel --wheel-dir=/wheels \
+    git+${CKANEXT_GEOVIEW_GITHUB_URL}.git@${CKANEXT_GEOVIEW_VERSION}#egg=ckanext-geoview
+
 # ckanext-showcase #############################################################
 ARG CKANEXT_SHOWCASE_TAG="v1.6.1"
 ENV CKANEXT_SHOWCASE_TAG=${CKANEXT_SHOWCASE_TAG}
@@ -157,11 +173,31 @@ RUN set -ex && \
   pip wheel --no-deps --wheel-dir=/wheels \
     git+${CKANEXT_DCATDE_GITHUB_URL}.git@${CKANEXT_DCATDE_VERSION}#egg=ckanext-dcatde
 
+# ckanext-xloader ##########################################################
+ARG CKANEXT_XLOADER_VERSION="1.0.1"
+ENV CKANEXT_XLOADER_VERSION=${CKANEXT_XLOADER_VERSION}
+ENV CKANEXT_XLOADER_GITHUB_URL="https://github.com/ckan/ckanext-xloader"
+
+RUN set -ex && \
+  pip install -r \
+    https://raw.githubusercontent.com/ckan/ckanext-xloader/${CKANEXT_XLOADER_VERSION}/requirements.txt && \
+  curl -o /wheels/ckanext-xloader.txt \
+    https://raw.githubusercontent.com/ckan/ckanext-xloader/${CKANEXT_XLOADER_VERSION}/requirements.txt && \
+  pip install -U requests[security] && \
+  pip wheel --wheel-dir=/wheels \
+    git+${CKANEXT_XLOADER_GITHUB_URL}.git@${CKANEXT_XLOADER_VERSION}#egg=ckanext-xloader
+
 # ckanext-ogdmunich #############################################################
-COPY ckan-extensions/ ${APP_DIR}/src
+# COPY ckan-extensions/ ${APP_DIR}/src
+# RUN set -ex && \
+#   cd ${APP_DIR}/src/ckanext-ogdmunich && \
+#   pip wheel --wheel-dir=/wheels .
+
+# ckanext-openapiview #############################################################
+
 RUN set -ex && \
-  cd ${APP_DIR}/src/ckanext-ogdmunich && \
-  pip wheel --wheel-dir=/wheels .
+  pip wheel --wheel-dir=/wheels \
+  git+https://github.com/open-data/ckanext-openapiview.git#egg=ckanext-openapiview
 
 # ckanext-spatial #############################################################
 FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_BUILD_SPATIAL} as extbuild-spatial
@@ -198,9 +234,10 @@ RUN set -ex && \
 FROM ghcr.io/keitaroinc/ckan:${CKAN_VERSION_RUNTIME_STAGE} as runtime
 
 ENV CKAN__PLUGINS "image_view text_view recline_view recline_map_view recline_graph_view recline_grid_view \
-  webpage_view datastore datapusher ogdmunich pages showcase harvest ckan_harvester csw_harvester dcat dcatde \
+  webpage_view openapi_view datastore xloader ogdmunich pages showcase resource_proxy geo_view geojson_view wmts_view shp_view \
+  harvest ckan_harvester csw_harvester dcat dcatde \
   dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data \
-  spatial_metadata spatial_query envvars"
+  spatial_metadata spatial_query scheming_datasets envvars"
 
 # Extra env for compatibility with ckan/base Docker images for downstream k8s
 ENV CKAN_INI=${APP_DIR}/production.ini
@@ -244,8 +281,11 @@ RUN set -ex && \
   pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-spatial
 
 # ckanext-scheming ############################################################
+# RUN set -ex && \
+#   pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming
+
 RUN set -ex && \
-  pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-scheming
+  pip install -e git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming
 
 # ckanext-datesearch ##########################################################
 RUN set -ex && \
@@ -269,6 +309,10 @@ RUN set -ex && \
   pip install -r ${APP_DIR}/ext_wheels/ckanext-harvest.txt && \
   pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-harvest
 
+# ckanext-geoview #############################################################
+RUN set -ex && \
+  pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-geoview
+
 # ckanext-showcase #############################################################
 RUN set -ex && \
   pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-showcase
@@ -287,9 +331,24 @@ RUN set -ex && \
   pip install -r ${APP_DIR}/ext_wheels/ckanext-dcatde.txt && \
   pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-dcatde
 
+# ckanext-xloader #############################################################
+RUN set -ex && \
+  pip install -r ${APP_DIR}/ext_wheels/ckanext-xloader.txt && \
+  pip install -U requests[security] && \
+  pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-xloader
+
 # ckanext-ogdmunich #############################################################
+# RUN set -ex && \
+#   pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-ogdmunich
+
+COPY ckan-extensions/ ${APP_DIR}/src
 RUN set -ex && \
-  pip install --no-index --find-links=${APP_DIR}/ext_wheels ckanext-ogdmunich
+  cd ${APP_DIR}/src/ckanext-ogdmunich && \
+  pip install -e .
+
+# ckanext-openapiview #############################################################
+RUN set -ex && \
+  pip install -e git+https://github.com/open-data/ckanext-openapiview.git#egg=ckanext-openapiview
 
 # Copy additional files
 COPY --chown=ckan:ckan who.ini ${APP_DIR}/who.ini
@@ -306,16 +365,14 @@ RUN set -ex && \
   ckan config-tool "${CKAN_INI}" "ckan.plugins = ${CKAN__PLUGINS}" && \
   ckan config-tool "${CKAN_INI}" "ckan.spatial.srid = 4326" && \
   ckan config-tool "${CKAN_INI}" "ckanext.spatial.search_backend = solr-bbox" && \
-  ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.scheming:ckan_dataset.yaml" && \
-  ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json ckanext.repeating:presets.json ckanext.composite:presets.json" && \
+  ckan config-tool "${CKAN_INI}" "scheming.dataset_schemas = ckanext.ogdmunich:ckan_dataset.yaml" && \
+  ckan config-tool "${CKAN_INI}" "scheming.presets = ckanext.scheming:presets.json" && \
   ckan config-tool "${CKAN_INI}" "scheming.dataset_fallback = false" && \
-  ckan config-tool "${CKAN_INI}" "licenses_group_url = https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/main/ckanext/grouphierarchy/licenses_SDDI.json" && \
+  # ckan config-tool "${CKAN_INI}" "licenses_group_url = https://raw.githubusercontent.com/tum-gis/ckanext-grouphierarchy-sddi/main/ckanext/grouphierarchy/licenses_SDDI.json" && \
+  ckan config-tool "${CKAN_INI}" "licenses_group_url = file:///srv/app/src/ckanext-ogdmunich/ckanext/ogdmunich/resources/licenses-old.json" && \
   ckan config-tool "${CKAN_INI}" "ckanext.password_policy.password_length = 12" && \
   ckan config-tool "${CKAN_INI}" "ckanext.password_policy.failed_logins = 3" && \
   ckan config-tool "${CKAN_INI}" "ckanext.password_policy.user_locked_time = 600" && \
-  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \
-  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.de/{z}/{x}/{y}.png" && \
-  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors." && \
   ckan config-tool "${CKAN_INI}" "ckanext.pages.about_menu = false" && \
   ckan config-tool "${CKAN_INI}" "ckanext.pages.allow_html = true" && \
   ckan config-tool "${CKAN_INI}" "ckanext.pages.editor = ckeditor" && \
@@ -327,6 +384,17 @@ RUN set -ex && \
   ckan config-tool "${CKAN_INI}" "ckan.harvest.log_timeframe = 10" && \
   ckan config-tool "${CKAN_INI}" "ckanext.spatial.harvest.user_name = fwadm" && \
   ckan config-tool "${CKAN_INI}" "ckanext.spatial.harvest.continue_on_validation_errors = True" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.geoview.ol_viewer.formats = wms kml" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.geoview.shp_viewer.srid = 4326" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.geoview.shp_viewer.encoding = UTF-8" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = custom" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.custom.url = https://tile.openstreetmap.org/{z}/{x}/{y}.png" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.attribution = Map tiles by <a href="https://www.openstreetmap.org/copyright">OpenStreetMap</a> contributors" && \
+  # ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.type = wms" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.wms.url = https://geoportal.muenchen.de/geoserver/gsm/wms" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.wms.layer = gsm:g_stadtkarte_gesamt" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.wms.srs = EPSG:25832" && \
+  ckan config-tool "${CKAN_INI}" "ckanext.spatial.common_map.wms.attribution = © <a href="https://www.geodatenservice-muenchen.de">Kommunalreferat – GeodatenService</a>, <a href="https://www.openstreetmap.org/copyright">OpenStreetMap contributors</a>" && \
   echo "${TZ}" > /etc/timezone && \
   mkdir -p ${CKAN_STORAGE_PATH} && \
   chown -R ckan:ckan ${APP_DIR} ${CKAN_STORAGE_PATH} && \

diff --git a/sddi-odp/README.md b/sddi-odp/README.md
@@ -1,9 +1,13 @@
 CKAN SDDI-ODP
 =========
 
-Image für die Erstellung des CKAN Opendata Portals der Stadt München.  
-Als Vorlage wurde das sddi-base Image verwendet.  
-Deployment findet im CAP statt.  
+In diesem Repo wurde das Image [sddi-base](https://github.com/it-at-m/ckan-docker/pkgs/container/ckan-sddi-base) als Vorlage verwendet und an die Anforderungen/Extensions der LHM angepasst (siehe unten).  
+
+CKAN-Version 2.9.9
+
+## Installation 
+
+    $ docker pull ghcr.io/ondics/ckan-sddi-odp:odp-katalog-1.1.0
 
 ## CKAN Extensions
 
@@ -26,9 +30,9 @@ Diese ermöglichen zusammen das Hochladen von strukturierten Ressourcen (wie CSV
 Insgesamt sind folgende Plugins aktiviert:  
 
     ENV CKAN__PLUGINS "image_view text_view recline_view recline_map_view recline_graph_view recline_grid_view \
-  webpage_view datastore datapusher ogdmunich pages showcase harvest ckan_harvester csw_harvester dcat dcatde \
-  dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data \
-  spatial_metadata spatial_query envvars"
+    webpage_view datastore datapusher ogdmunich pages showcase harvest ckan_harvester csw_harvester dcat dcatde \
+    dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data \
+    spatial_metadata spatial_query envvars"
 
 Die folgenden weiteren Extensions gehen über sddi-base hinaus und werden über das Dockerfile installiert und eingerichtet:  
 

diff --git a/sddi-odp/ckan-extensions/ckanext-ogdmunich/MANIFEST.in b/sddi-odp/ckan-extensions/ckanext-ogdmunich/MANIFEST.in
@@ -1,2 +1,2 @@
 include license.json
-recursive-include ckanext/ogdmunich *.html *.json *.js *.less *.css *.yml *.mo *.po *.pot *.ico *.png *.svg *.woff *.woff2 *.eot *.ttf
+recursive-include ckanext/ogdmunich *.html *.json *.js *.less *.css *.yaml *.yml *.mo *.po *.pot *.ico *.png *.svg *.woff *.woff2 *.eot *.ttf
diff --git a/sddi-odp/ckan-extensions/ckanext-ogdmunich/ckanext/ogdmunich/assets/css/ogdmunich_style.css b/sddi-odp/ckan-extensions/ckanext-ogdmunich/ckanext/ogdmunich/assets/css/ogdmunich_style.css
@@ -443,4 +443,10 @@ html {
 .img-fluid {
 	max-width: 100%;
 	height: auto;
-  }
+}
+
+/* package-sidebar */
+ul.ogdmunich-metadata-list {
+	list-style-type: none;
+	padding: 15px;
+}
diff --git a/sddi-odp/ckan-extensions/ckanext-ogdmunich/ckanext/ogdmunich/ckan_dataset.yaml b/sddi-odp/ckan-extensions/ckanext-ogdmunich/ckanext/ogdmunich/ckan_dataset.yaml
@@ -0,0 +1,92 @@
+scheming_version: 2
+dataset_type: dataset
+about: A reimplementation of the default CKAN dataset schema
+about_url: http://github.com/ckan/ckanext-scheming
+
+
+dataset_fields:
+
+- field_name: title
+  label: Title
+  preset: title
+  form_placeholder: eg. A descriptive title
+
+- field_name: name
+  label: URL
+  preset: dataset_slug
+  form_placeholder: eg. my-dataset
+
+- field_name: notes
+  label: Description
+  form_snippet: markdown.html
+  form_placeholder: eg. Some useful notes about the data
+
+- field_name: tag_string
+  label: Tags
+  preset: tag_string_autocomplete
+  form_placeholder: eg. economy, mental health, government
+
+- field_name: license_id
+  label: License
+  form_snippet: license.html
+  help_text: License definitions and additional information can be found at http://opendefinition.org/
+
+- field_name: owner_org
+  label: Organization
+  preset: dataset_organization
+
+- field_name: url
+  label: Source
+  form_placeholder: http://example.com/dataset.json
+  display_property: foaf:homepage
+  display_snippet: link.html
+
+- field_name: maintainer
+  label: Maintainer
+  form_placeholder: Joe Bloggs
+  display_property: dc:contributor
+
+- field_name: maintainer_email
+  label: Maintainer Email
+  form_placeholder: [email protected]
+  display_property: dc:contributor
+  display_snippet: email.html
+  display_email_name_field: maintainer
+
+- field_name: frequency
+  label: Update-Zyklus
+  form_snippet: select.html
+  display_snippet: select.html
+  choices_helper: ogdmunich_frequency_list
+
+- field_name: references
+  label: Musterdatenkatalog
+  form_placeholder: https://musterdatenkatalog.de/def/musterdatensatz/bevoelkerungsstruktur/einwohnerzahl
+  help_text: Musterdatensatz aus <a href="https://bertelsmannstift.github.io/Musterdatenkatalog/def/musterdatensatz.html" target="_blank">Musterdatenkatalog</a>
+  help_allow_html: true
+  validators: url_validator ogdmunich_is_musterdatensatz
+
+- field_name: hvd_category
+  label: Hochwertiger Datensatz
+  form_snippet: select.html
+  choices_helper: ogdmunich_hvd_category_list
+
+
+resource_fields:
+
+- field_name: url
+  label: URL
+  preset: resource_url_upload
+
+- field_name: name
+  label: Name
+  form_placeholder: eg. January 2011 Gold Prices
+
+- field_name: description
+  label: Description
+  form_snippet: markdown.html
+  form_placeholder: Some useful notes about the data
+
+- field_name: format
+  label: Format
+  preset: resource_format_autocomplete