Merge pull request #104 from mjanez/feature/dcat-ap-schemas

Update - CKAN 2.10 compatibility and UI/Schemas/Profiles improvements

.coveragerc

@@ -2,4 +2,4 @@
 omit =
     */site-packages/*
     */python?.?/*
-    ckan/*
+    ckan/*

.dockerignore

@@ -0,0 +1,4 @@
+node_modules
+build
+dist
+*.egg-info/

.github/workflows/test.yml

@@ -1,14 +1,15 @@
-name: schemingdcat-test
-run-name: ${{ github.actor }} probando schemingdcat-test
-on: [workflow_dispatch]
+name: Tests
+env:
+    COLUMNS: 120
+on: [pull_request]
 jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v4
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.9'
       - name: Install requirements
         run: pip install flake8 pycodestyle
       - name: Check syntax
@@ -18,15 +19,23 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: ["2.10", 2.9]
+        include:
+          - ckan-version: "2.11"
+            ckan-image: "ckan/ckan-dev:2.11-py3.10"
+          - ckan-version: "2.10"
+            ckan-image: "ckan/ckan-dev:2.10-py3.10"
+          - ckan-version: "2.9"
+            ckan-image: "ckan/ckan-dev:2.9-py3.9"
       fail-fast: false
+
     name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-latest
     container:
-      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
+      image: ${{ matrix.ckan-image }}
     services:
       solr:
-        image: ckan/ckan-solr:${{ matrix.ckan-version }}
+        # Custom image with spatial support and advanced schema
+        image: ghcr.io/mjanez/ckan-solr:2.10-solr9-spatial
       postgres:
         image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
@@ -35,7 +44,7 @@ jobs:
           POSTGRES_DB: postgres
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       redis:
-          image: redis:3
+          image: redis:7-alpine
     env:
       CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
       CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
@@ -44,25 +53,41 @@ jobs:
       CKAN_REDIS_URL: redis://redis:6379/1
 
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/checkout@v3
-      with:
-        repository: mjanez/ckanext-scheming
-    - name: Install requirements
+    - uses: actions/checkout@v4
+    - name: Install requirements (common)
       run: |
-        pwd
-        echo $GITHUB_WORKSPACE
-        ls -al $GITHUB_WORKSPACE
-        pip install --upgrade pip
+        pip install -r requirements.txt
+        pip install -r dev-requirements.txt
         pip install -e .
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test_subclass.ini
+    - name: Install requirements (2.9)
+      run: |
+        pip install -U pytest-rerunfailures
+      if: ${{ matrix.ckan-version == '2.9' }}
+    - name: Setup other extensions
+      run: |
+        # Install ckanext-dcat
+        git clone https://github.com/mjanez/ckanext-dcat
+        pip install -e ckanext-dcat
+        pip install -r ckanext-dcat/requirements.txt
+
+        # Install ckanext-harvest
+        git clone https://github.com/ckan/ckanext-harvest
+        pip install -e ckanext-harvest
+        pip install -r ckanext-harvest/requirements.txt
+
+        # Install ckanext-scheming
+        git clone https://github.com/ckan/ckanext-scheming
+        pip install -e ckanext-scheming
+
+        # Install ckanext-fluent
+        git clone https://github.com/mjanez/ckanext-fluent
+        pip install -e ckanext-fluent
+        
     - name: Setup extension
       run: |
-        pip install -r test-requirements.txt
         ckan -c test.ini db init
-    - name: Run all tests
-      run: pytest --ckan-ini=test.ini --cov=ckanext.scheming ckanext/scheming/tests
-    - name: Run plugin subclassing tests
-      run: pytest --ckan-ini=test_subclass.ini ckanext/scheming/tests/test_dataset_display.py ckanext/scheming/tests/test_form.py::TestDatasetFormNew ckanext/scheming/tests/test_dataset_logic.py
+        ckan -c test.ini db pending-migrations --apply
+    - name: Run tests
+      run: pytest --log-cli-level=INFO --ckan-ini=test.ini --cov=ckanext.schemingdcat --cov-report=term-missing --cov-append --disable-warnings ckanext/schemingdcat/tests

.gitignore

@@ -1,11 +1,16 @@
 .ropeproject
 node_modules
+**/node_modules/
 bower_components
 .vscode
+.idea
+*.log
 
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
+*.pyc
+.pytest_cache
 
 # C extensions
 *.so
@@ -52,3 +57,9 @@ docs/_build/
 ckanext/schemingdcat/i18n/tools/output/*
 ckanext/schemingdcat/i18n/tools/input/*
 ckanext/schemingdcat/i18n/tools/input/input.yaml
+
+# custom harvesters
+doc/harvesters/*_config.json
+!doc/harvesters/ckan_template_config.json
+!doc/harvesters/xls_template_config.json
+!doc/harvesters/csw_template_config.json

CHANGELOG.md

@@ -1,6 +1,54 @@
 # Changelog
 
-## [Unreleased](https://github.com/ckan/ckanext-dcat/compare/v3.2.1...HEAD)
+## [Unreleased](https://github.com/ckan/ckanext-dcat/compare/v4.0.0...HEAD)
+
+## [v4.0.0](https://github.com/mjanez/ckanext-schemingdcat/compare/v4.0.0...v3.2.2) - 2024-09-18
+* Fix search and index to ensure proper functionality by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/c294495
+* Improve profiles and add DCAT-AP/GeoDCAT-AP 3 by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/7fb647a
+* Bug - CKAN 2.10 - Fix request.args() #41 by @mjanez
+* Develop feature/form-tabs #87 by @mjanez
+* DCAT - Custom DCAT-AP/GeoDCAT-AP (ES/EU) profiles in schemingdcat #93 by @mjanez
+* Schema - Transform contact, publisher, author and maintainer to list of dicts (repeating subfields) #95 by @mjanez
+* Enhanced DCAT-AP profiles to ensure MQA DCAT-AP compliance #99 by @mjanez
+* Feature - Fix tests #22 by @mjanez
+* Improve schemas to align names with ckanext-dcat by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/0d6559b
+* Improve profiles and add DCAT-AP 3 by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/8cb65ff
+* Improve general UI, Home by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/4810b07
+* Update package/resource forms to Tabs by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/ef76bb3
+* Improve i18n and icons by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/4dca2e4
+* Fix and improve templates for CKAN 2.10 by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/c7aabef
+* Fix fa icons for license/social templates by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/f358203
+* Fix before_update by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/7c303d9
+* Improve package_item.html template by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/fdf55e8
+* Improve base admin config template by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/a344aec
+* Update configuration and add stats init to plugin by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/006078e
+* Refactor code to remove debug logging statements by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/3da105a
+* Refactor download metadata template and improve metadata profiles by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/6954453
+* Improve metadata_info and profile badges by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/0ffa59e
+* Refactor new_source_form.html to use Bootstrap 5 data attributes by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/c4d131b
+* Add docs and templates for harvesters by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/e136909
+* Add xls_template_config by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/69e04f3
+* Improve style to improve header language selector dropdown menu by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/66dc66d
+* New DCAT-AP/GeoDCAT-AP Schemas and DCAT Profiles & Form tabs for datasets/resources #97 by @mjanez
+* Fix tabs slug-preview and dataset-map by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/1af58e1
+* Fix Solr indexing by converting dict fields to JSON strings by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/fb8abf6
+* Update open data stats: Skip statistics update during DB initialization by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/e3bb1ee
+* Improve form tabs by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/0cbb625
+* Fix package snippets by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/448113e
+* Fix organization create/edit button bugs by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/4b10208
+* Fix spatial_query snippet bugs by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/3026467
+* Fix xloader logs styles for 2.10 by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/2c02458
+* Fix tags containers by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/commit/50945d0
+* Improve datastore/data api and downloads templates by @mjanez
+
+## [v3.2.2](https://github.com/mjanez/ckanext-schemingdcat/compare/v3.2.2...v3.2.1) - 2024-08-28
+* Feature/ckan harvester improve by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/pull/86
+* Update v3.2.2 by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/pull/85
+* Fix bug when schemingdcat.endpoints_yaml is None by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/pull/89
+* Feature/ckan harvester improve by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/pull/90
+* Fix harvester by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/pull/91
+* Add allow_private_datasets to ckan harvester by @mjanez in https://github.com/mjanez/ckanext-schemingdcat/pull/98
+* Fixed bugs with localised_filesize snippet 43cc8b025e6e7574dad952b1a6b680a4a66dbcff
 
 ## [v3.2.0](https://github.com/mjanez/ckanext-schemingdcat/compare/v3.2.1...v3.2.0) - 2024-05-17
 * Enhance harvester, field mapping validation lib and harvest templates #81