Merge branch 'main' into 4567_set-up-gtag-datalayer

GSA · Apr 9, 2024 · 88dd610 · 88dd610
2 parents d0e02f0 + 6698694
commit 88dd610
Show file tree

Hide file tree

Showing 8 changed files with 62 additions and 91 deletions.
diff --git a/.github/workflows/snyk.yml b/.github/workflows/snyk.yml
@@ -13,10 +13,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v4
-      - name: Set up Python 3.9
+      - name: Set up Python 3.10
         uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: '3.10'
       - name: Display Python version
         run: python -c "import sys; print(sys.version)"
       - name: Install Dependencies

diff --git a/Makefile b/Makefile
@@ -9,42 +9,42 @@ all: build
 # ###############################################
 
 build:
-	docker-compose build --parallel
+	docker compose build --parallel
 
 ci:
-	docker-compose up -d
+	docker compose up -d
 
 clean:
-	docker-compose down -v --remove-orphans
+	docker compose down -v --remove-orphans
 
 cypress:
 	# Turn on local system, and open cypress in interactive mode
 	# If you haven't remapped localhost > ckan, you should change baseURL to "http://localhost:5000" in `e2e/cypress.json`
-	docker-compose up -d && cd e2e && CYPRESS_USER=admin CYPRESS_USER_PASSWORD=password npx [email protected] open
+	docker compose up -d && cd e2e && CYPRESS_USER=admin CYPRESS_USER_PASSWORD=password npx [email protected] open
 
 dev:
 	docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
-	docker-compose build
-	docker-compose up
+	docker compose build
+	docker compose up
 
 debug:
 	docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
-	docker-compose build
-	docker-compose run --service-ports ckan
+	docker compose build
+	docker compose run --service-ports ckan
 
 up:
-	docker-compose up $(ARGS)
+	docker compose up $(ARGS)
 
 update-dependencies:
-	docker-compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
+	docker compose run --rm -T ckan /app/ckan/freeze-requirements.sh $(shell id -u) $(shell id -g)
 
 # ###############################################
 # Test commands
 # ###############################################
 
 test: build
 	# docker build -t ghcr.io/gsa/catalog.data.gov:latest ckan/
-	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
+	docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
 # everytime you added some new variables, you need to swap it with some test values
 # and swap it back after the test. This is because "nginx -t" test cannot read env variables.
@@ -79,31 +79,31 @@ validate-proxy:
 
 quick-bat-test:
 	# if local environment is already build and running
-	docker-compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
+	docker compose -f docker-compose.yml -f docker-compose.test.yml up --abort-on-container-exit test
 
 test-extensions:
 	# test our extensions
 
 	# deal with the CKAN path
-	docker-compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
+	docker compose exec ckan bash -c "ln -sf $(CKAN_HOME)/src/ckan $(CKAN_HOME)/ckan"
 
 	# full test datajson
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-datajson && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datajson/tests --debug=ckanext"
 
 	# full test datagovtheme
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-datagovtheme && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovtheme/tests --debug=ckanext"
 
 	# full test geodatagov
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-geodatagov && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/geodatagov/tests --debug=ckanext"
 
 	# full test geodatagov
-	docker-compose exec ckan bash -c \
+	docker compose exec ckan bash -c \
 		"cd $(CKAN_HOME)/src/ckanext-datagovdatalog && \
 		 nosetests --ckan --with-pylons=$(CKAN_HOME)/src/ckan/test-catalog-next.ini ckanext/datagovdatalog/tests --debug=ckanext"
 
@@ -119,10 +119,10 @@ clear-solr-volume:
 
 unlock-solr-volume:
 	# Corruptible
-	docker-compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock"
+	docker compose run solr /bin/bash -c "rm -rf /var/solr/data/ckan/data/index/write.lock"
 
 search-index-rebuild:
-	docker-compose exec ckan /bin/bash -c "ckan search-index rebuild"
+	docker compose exec ckan /bin/bash -c "ckan search-index rebuild"
 
 copy-src:
 	docker cp catalog-app_ckan_1:$(CKAN_HOME)/src .
@@ -136,7 +136,7 @@ test-import-tool:
 		python -m pytest --vcr-record=none tests/
 
 lint-all:
-	docker-compose exec -T ckan \
+	docker compose exec -T ckan \
 		bash -c "cd $(CKAN_HOME)/src && \
 		 		 pip install pip==20.3.3  && \
 				 pip install flake8 && \
@@ -146,31 +146,31 @@ lint-all:
 # qa:
 # ifeq (${PARAMS}, all)
 # 	# PARAMS=all make qa
-# 	docker-compose exec ckan ckan report generate
+# 	docker compose exec ckan ckan report generate
 # else ifeq (${PARAMS}, openness)
 # 	# PARAMS=openness make qa
 # 	# generate report at /report/openness
-# 	docker-compose exec ckan ckan report generate openness
+# 	docker compose exec ckan ckan report generate openness
 # else ifeq (${PARAMS}, update)
 # 	# PARAMS=update make qa
 # 	# QA is performed when a dataset/resource is archived, or you can run it manually using a ckan command:
-# 	docker-compose exec ckan ckan qa update
+# 	docker compose exec ckan ckan qa update
 # else ifeq (${PARAMS}, archive)
 # 	# PARAMS=archive make qa
 # 	# Archive datasets to perform QA
-# 	docker-compose exec ckan ckan archiver update --queue bulk
+# 	docker compose exec ckan ckan archiver update --queue bulk
 # else ifeq (${PARAMS}, worker)
 # 	# PARAMS=worker make qa
-# 	docker-compose exec ckan ckan jobs worker bulk
+# 	docker compose exec ckan ckan jobs worker bulk
 # endif
 
 update-tracking-info:
 	# https://docs.ckan.org/en/2.8/maintaining/tracking.html
-	docker-compose exec ckan ckan tracking update
+	docker compose exec ckan ckan tracking update
 
 harvest:
 	# Pass any of the following arguments to run them
 	# ARGS=run make harvest
 	# ARGS=gather-consumer make harvest
 	# ARGS=fetch-consumer make harvest
-	docker-compose exec ckan ckan harvester $(ARGS)
+	docker compose exec ckan ckan harvester $(ARGS)
diff --git a/README.md b/README.md
@@ -77,7 +77,7 @@ To test extensions locally you can run
 _TODO: update this for pytest_
 
 ```
-docker-compose exec ckan bash
+docker compose exec ckan bash
 nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/datagovtheme/
 nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/datajson/
 nosetests --ckan --with-pylons=src/ckan/test-catalog-next.ini src/ckanext-datagovtheme/ckanext/geodatagov/
@@ -207,7 +207,7 @@ You can use the ckan template in much the same way as a source install, only
 executing the command inside the CKAN container and setting the mounted `src/`
 folder as output:
 
-    $ docker-compose exec ckan /bin/bash -c \
+    $ docker compose exec ckan /bin/bash -c \
     "ckan generate extension"
 
 The new extension will be created in the `src/` folder. You might need to change
@@ -219,7 +219,7 @@ the owner of its folder to have the appropriate permissions.
 To run a container and be able to add a breakpoint with `pdb` or `ipdb`, run the
 `ckan-dev` container with the `--service-ports` option:
 
-    docker-compose run --service-ports ckan
+    docker compose run --service-ports ckan
 
 This will start a new container, displaying the standard output in your
 terminal. If you add a breakpoint in a source file in the `src` folder (`import

diff --git a/ckan/.snyk b/ckan/.snyk
@@ -3,85 +3,52 @@ version: v1.25.0
 # ignores vulnerabilities until expiry date; change duration by modifying expiry date
 ignore:
   SNYK-PYTHON-BEAKER-575115:
-    - '*':
+    - "*":
         reason: >-
           No remediation available yet; Not affecting us since the storage is
           not accessible to any other client
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2022-12-08T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-6035177:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-10-30T16:50:58.023Z
   SNYK-PYTHON-WERKZEUG-3319936:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-WERKZEUG-3319935:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-02-15T16:20:58.023Z
   SNYK-PYTHON-FLASK-5490129:
-    - '*':
+    - "*":
         reason: >-
           Upgrade path is complex, Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4303
-        expires: 2024-03-31T16:20:58.017Z
+        expires: 2024-05-31T16:20:58.017Z
         created: 2023-05-08T16:20:58.023Z
-  SNYK-PYTHON-WERKZEUG-6041510:
-    - '*':
-        reason: >-
-          Upgrade path is complex, Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4217
-        expires: 2024-03-31T14:44:52.166Z
-        created: 2023-11-02T14:44:52.170Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6050294:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-02-08T00:00:00.000Z
-        created: 2023-11-16T20:31:20.590Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6126975:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-03-14T00:00:00.000Z
-        created: 2023-12-14T00:00:00.000Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6149518:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-04-10T19:28:50.100Z
-        created: 2024-01-11T19:28:50.103Z
   SNYK-PYTHON-PYOPENSSL-6149520:
-    - '*':
+    - "*":
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4532
-        expires: 2024-04-10T19:29:54.032Z
-        created: 2024-01-11T19:29:54.039Z
+        expires: 2024-05-31T19:29:54.032Z
+        created: 2024-01-08T00:00:00.000Z
   SNYK-PYTHON-PYOPENSSL-6157250:
-    - '*':
+    - "*":
         reason: >-
           No remediation available yet; Issue tracked in github:
           https://github.com/GSA/data.gov/issues/4591
-        expires: 2024-04-10T19:29:54.032Z
-  SNYK-PYTHON-CRYPTOGRAPHY-6157248:
-    - '*':
-        reason: >-
-          No remediation available yet; Issue tracked in github:
-          https://github.com/GSA/data.gov/issues/4590
-        expires: 2024-04-10T19:29:54.032Z
+        expires: 2024-05-31T19:29:54.032Z
+        created: 2024-01-14T00:00:00.000Z
 patch: {}
diff --git a/ckan/requirements.in b/ckan/requirements.in
@@ -120,3 +120,7 @@ importlib-resources<6.0
 gevent>=23.9.0
 jinja2>=3.1.3
 cryptography>=42.0.4
+
+# lxml beyond 5.1.0 show error module 'lxml.etree' has no attribute '_ElementStringResult'
+# as in https://github.com/GSA/data.gov/issues/4681
+lxml==5.1.0
diff --git a/ckan/requirements.txt b/ckan/requirements.txt
@@ -4,16 +4,16 @@ Babel==2.10.3
 Beaker==1.11.0
 bleach==5.0.1
 blinker==1.5
-boto3==1.34.69
-botocore==1.34.69
+boto3==1.34.75
+botocore==1.34.75
 certifi==2024.2.2
 cffi==1.16.0
 chardet==5.2.0
 charset-normalizer==3.3.2
 ckan==2.10.4
 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver
 ckanext-datagovcatalog==0.1.0
-ckanext-datagovtheme==0.2.17
+ckanext-datagovtheme==0.2.18
 ckanext-datajson==0.1.23
 ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@e2183dd13bf2dbb0532281d84225f909b8362dd3
 ckanext-envvars==0.0.3
@@ -61,7 +61,7 @@ MarkupSafe==2.1.5
 messytables==0.15.2
 mypy==1.9.0
 mypy-extensions==1.0.0
-newrelic==9.7.1
+newrelic==9.8.0
 nose==1.3.7
 numpy==1.26.4
 oauth2client==4.1.3
@@ -75,9 +75,9 @@ polib==1.1.1
 progressbar==2.5
 progressbar2==3.53.3
 psycopg2==2.9.3
-pyasn1==0.5.1
-pyasn1-modules==0.3.0
-pycparser==2.21
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pycparser==2.22
 PyJWT==2.4.0
 pyOpenSSL==24.1.0
 pyparsing==3.1.2
@@ -120,6 +120,6 @@ Werkzeug==2.0.3
 wheel==0.42.0
 WTForms==3.1.2
 xlrd==2.0.1
-xmlschema==3.1.0
+xmlschema==3.2.0
 zope.event==5.0
 zope.interface==5.4.0
diff --git a/src/README.md b/src/README.md
@@ -2,6 +2,6 @@ This folder is used to install local extensions for development.
 Clone here any extension that you want to work on. This folder in mounted on
 the `ckan-dev` service, so any extension located here will be installed when
 booting up Docker Compose in development mode 
-(`docker-compose -f docker-compose.dev.yml up`).
+(`docker compose -f docker-compose.dev.yml up`).
 If running the CKAN container standalone you will need to mount the folder
 yourself.
diff --git a/tools/harvest_source_import/check_harvest_sources.py b/tools/harvest_source_import/check_harvest_sources.py
@@ -87,7 +87,7 @@
            f'Config: {config}'
     print(info)
 
-    command = (['docker-compose', 'exec', 'ckan', 
+    command = (['docker compose', 'exec', 'ckan', 
                 'ckan', 'harvester', 'run_test',
                 hs.get("id"), '--config=/app/ckan/setup/ckan.ini'])
     out = subprocess.Popen(