From 41b012e485faa96e972638af46ea1e9d63d71d63 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 09:58:01 -0400 Subject: [PATCH 01/73] update: all of the requirements referencing the ckan commit might still have issues... --- ckan/requirements.in | 83 +++++++++++++++++--------------- ckan/requirements.txt | 107 ++++++++++++++++++++++-------------------- 2 files changed, 101 insertions(+), 89 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 576563d5..2665b323 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,5 +1,6 @@ # CKAN requirements and extensions -ckan==2.9.8 +# ckan==2.9.8 +git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7#egg=ckan git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest -e git+https://github.com/ckan/ckanext-spatial.git@master#egg=ckanext-spatial @@ -8,21 +9,26 @@ git+https://github.com/GSA/ckanext-saml2auth.git@create_user_via_saml#egg=ckanex -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver -e git+https://github.com/ckan/ckanext-report.git@master#egg=ckanext-report -ckanext-datagovcatalog>=0.0.3 -ckanext-datagovtheme>=0.1.22 -ckanext-datajson>=0.1.13 -ckanext-envvars>=0.0.2 -ckanext-geodatagov>=0.1.28 -ckanext-googleanalyticsbasic -ckanext-metrics-dashboard +# ckanext-datagovcatalog>=0.0.3 +# ckanext-datagovtheme>=0.1.22 +# ckanext-datajson>=0.1.13 +ckanext-envvars>=0.0.3 +# ckanext-geodatagov>=0.1.28 +# ckanext-googleanalyticsbasic +# ckanext-metrics-dashboard +-e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov +-e git+https://github.com/gsa/ckanext-datagovcatalog.git@ckan-210#egg=ckanext-datagovcatalog +-e git+https://github.com/gsa/ckanext-datagovtheme.git@ckan-210#egg=ckanext-datagovtheme +-e git+https://github.com/gsa/ckanext-datajson.git@ckan-210#egg=ckanext-datajson +-e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@ckan-2.10#egg=ckanext-googleanalyticsbasic +-e git+https://github.com/gsa/ckanext-metrics-dashboard.git@ckan-2.10#egg=ckanext-metrics-dashboard # Pin for saml2auth to work pysaml2==7.0.1 # ckanext-harvest dependencies ckantoolkit>=0.0.7 -pika>=1.1.0 -pyOpenSSL>=22.0.0 +pika>=1.1.0,<1.3.0 # redis==2.10.6 # included in ckan core # requests>=2.11.1 # included in ckan core # six>=1.12.0 # included in dcat @@ -41,7 +47,8 @@ GeoAlchemy2==0.5.0 owslib>=0.28.1 pyparsing>=2.1.10 pyproj==2.6.1 -Shapely>=1.2.13 +Shapely==2.0.1 +geojson==3.0.1 # requests>=1.1.0 # included in ckan core # six # included in dcat @@ -55,52 +62,50 @@ messytables==0.15.2 # https://github.com/ckan/ckanext-report/blob/master/requirements.txt #ckanext-dcat -rdflib==4.2.2 +rdflib==6.1.1 # rdflib-jsonld==0.4.0 # ignoring as cannot build on cloud.gov geomet>=0.2.0 future>=0.18.3 -six>=1.15.0 # ckan doesn't advertise its requirements correctly, so let's add them here. # https://github.com/ckan/ckan/blob/ckan-2.9.5/requirements.txt -alembic==1.0.0 -Babel==2.9.1 # Upgraded from CKAN pin +alembic==1.8.1 +Babel==2.10.3 Beaker==1.11.0 bleach==3.3.0 -click==7.1.2 -dominate==2.4.0 -fanstatic==1.1 +blinker==1.5 +click==8.1.3 +dominate==2.7.0 feedgen==0.9.0 -Flask==2.0 +Flask==2.0.3 Flask-Babel==1.0.0 flask-multistatic==1.0 -Jinja2==3.0 -lxml==4.9.1 +Flask-Login==0.6.1 +Jinja2==3.1.2 +Flask-WTF==1.0.1 pyjwt==2.4.0 -Markdown==3.1.1 # Upgraded from CKAN pin -passlib==1.7.3 +Markdown==3.4.1 +passlib==1.7.4 pastedeploy==2.0.1 # manually kept - remove when #4802 is complete pathtools==0.1.2 # via watchdog -polib==1.0.7 -psycopg2==2.8.6 -python-magic==0.4.15 -pysolr==3.6.0 -python-dateutil>=1.5.0 -pytz==2016.7 -PyUtilib==5.7.1 -pyyaml==5.4 -repoze.who==2.3 -requests==2.25.0 -Routes==1.13 -rq==1.0 +polib==1.1.1 +psycopg2==2.9.3 +PyJWT==2.4.0 +python-magic==0.4.27 +pysolr==3.9.0 +python-dateutil>=2.8.2 +pytz +PyUtilib==6.0.0 +pyyaml==6.0 +requests==2.28.1 +rq==1.11.0 simplejson==3.18.0 # ckan 2.9.5 requires 3.10.0 only # Following can be ignored: https://github.com/ckan/ckan/pull/4450 # sqlalchemy-migrate==0.12.0 -SQLAlchemy==1.3.5 +SQLAlchemy[mypy]==1.4.41 sqlparse==0.4.4 -tzlocal==1.3 -unicodecsv>=0.9 -webassets==0.12.1 +tzlocal==4.2 +webassets==2.0 Werkzeug==2.0 # zope.interface==4.7.2 diff --git a/ckan/requirements.txt b/ckan/requirements.txt index adef1c3c..a0eac4f8 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -1,115 +1,122 @@ -alembic==1.0.0 +alembic==1.8.1 async-timeout==4.0.2 -Babel==2.9.1 +Babel==2.10.3 +backports.zoneinfo==0.2.1 Beaker==1.11.0 bleach==3.3.0 -boto3==1.26.126 -botocore==1.29.126 -certifi==2022.12.7 +blinker==1.5 +boto3==1.26.137 +botocore==1.29.137 +certifi==2023.5.7 cffi==1.15.1 -chardet==3.0.4 -ckan==2.9.8 +chardet==5.1.0 +charset-normalizer==2.1.1 +ckan @ git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -ckanext-datagovcatalog==0.0.5 -ckanext-datagovtheme==0.1.27 -ckanext-datajson==0.1.18 -ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@618928be5a211babafc45103a72b6aab4642e964 +-e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog +-e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson +ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 -ckanext-geodatagov==0.1.34 -ckanext-googleanalyticsbasic==0.2.0 --e git+https://github.com/ckan/ckanext-harvest.git@89a98d7ff5aa3445d8158921669b8d0b04fa41c3#egg=ckanext_harvest -ckanext-metrics-dashboard==0.1.5 +-e git+https://github.com/gsa/ckanext-geodatagov.git@46a3ff790c734bb8cb3fed226f86a123dc6c1975#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic +-e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest +-e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@c2b12a94430034c522b25d282323a064e2d6a03a --e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial +-e git+https://github.com/ckan/ckanext-spatial.git@1205d3aa408e994439fc231ccb5d1cff06d644c6#egg=ckanext_spatial ckantoolkit==0.0.7 -click==7.1.2 +click==8.1.3 cryptography==40.0.2 defusedxml==0.7.1 -dominate==2.4.0 +dominate==2.7.0 elementpath==4.1.2 -fanstatic==1.1 feedgen==0.9.0 -Flask==2.0.0 +Flask==2.0.3 Flask-Babel==1.0.0 +Flask-Login==0.6.1 flask-multistatic==1.0 +Flask-WTF==1.0.1 future==0.18.3 GeoAlchemy2==0.5.0 +geojson==3.0.1 geomet==1.0.0 gevent==22.10.2 greenlet==2.0.2 gunicorn==20.1.0 html5lib==1.1 -idna==2.10 +idna==3.4 +importlib-metadata==6.6.0 importlib-resources==5.12.0 isodate==0.6.1 itsdangerous==2.1.2 -Jinja2==3.0.0 +Jinja2==3.1.2 jmespath==1.0.1 json-table-schema==0.2.1 jsonschema==2.4.0 -lxml==4.9.1 +lxml==4.9.2 Mako==1.2.4 -Markdown==3.1.1 +Markdown==3.4.1 MarkupSafe==2.1.2 messytables==0.15.2 +mypy==1.3.0 +mypy-extensions==1.0.0 newrelic==8.8.0 nose==1.3.7 numpy==1.24.3 -OWSLib==0.29.1 +OWSLib==0.29.2 packaging==23.1 -passlib==1.7.3 +passlib==1.7.4 PasteDeploy==2.0.1 pathtools==0.1.2 -pika==1.3.1 +pika==1.2.1 pip==23.1.2 ply==3.11 -polib==1.0.7 +polib==1.1.1 progressbar==2.5 progressbar2==3.53.3 -psycopg2==2.8.6 +psycopg2==2.9.3 pycparser==2.21 PyJWT==2.4.0 pyOpenSSL==23.1.1 pyparsing==3.0.9 pyproj==2.6.1 pysaml2==7.0.1 -pysolr==3.6.0 +pysolr==3.9.0 python-dateutil==2.8.2 -python-editor==1.0.4 -python-magic==0.4.15 +python-magic==0.4.27 python-utils==3.5.2 -pytz==2016.7 -PyUtilib==5.7.1 -PyYAML==5.4 +pytz==2023.3 +pytz-deprecation-shim==0.1.0.post0 +PyUtilib==6.0.0 +PyYAML==6.0 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219 -rdflib==4.2.2 -redis==4.5.4 -repoze.lru==0.7 -repoze.who==2.3 -requests==2.25.0 +rdflib==6.1.1 +redis==4.5.5 +requests==2.28.1 rfc3987==1.3.8 -Routes==1.13 -rq==1.0 -s3transfer==0.6.0 +rq==1.11.0 +s3transfer==0.6.1 setuptools==67.1.0 shapely==2.0.1 -shutilwhich==1.1.0 simplejson==3.18.0 six==1.16.0 -SQLAlchemy==1.3.5 +SQLAlchemy==1.4.41 +sqlalchemy2-stubs==0.0.2a34 sqlparse==0.4.4 -tzlocal==1.3 -unicodecsv==0.14.1 +tomli==2.0.1 +typing_extensions==4.5.0 +tzdata==2023.3 +tzlocal==4.2 urllib3==1.26.15 -webassets==0.12.1 +webassets==2.0 webencodings==0.5.1 -WebOb==1.8.7 Werkzeug==2.0.0 wheel==0.40.0 +WTForms==3.0.1 xlrd==2.0.1 -xmlschema==2.2.3 +xmlschema==2.3.0 zipp==3.15.0 zope.event==4.6 zope.interface==6.0 From dab302e7b00d02422b870984fc245e97c8e4c8f2 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 11:05:25 -0400 Subject: [PATCH 02/73] new: use custom fork with necessary fixes only --- ckan/requirements.in | 2 +- ckan/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 2665b323..816b69bf 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,6 +1,6 @@ # CKAN requirements and extensions # ckan==2.9.8 -git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7#egg=ckan +git+https://github.com/gsa/ckan.git@ckan-210-temp#egg=ckan git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest -e git+https://github.com/ckan/ckanext-spatial.git@master#egg=ckanext-spatial diff --git a/ckan/requirements.txt b/ckan/requirements.txt index a0eac4f8..21f263a0 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -11,14 +11,14 @@ certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==2.1.1 -ckan @ git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7 +ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog -e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@46a3ff790c734bb8cb3fed226f86a123dc6c1975#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@31b92ada972664b2b532fd63096b3c96ba8e40fd#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From dc7694a86640e8e3ce3c8d17ac3118c26eb21921 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 11:49:18 -0400 Subject: [PATCH 03/73] test: how to fix organization_create? --- e2e/cypress/support/command.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/cypress/support/command.js b/e2e/cypress/support/command.js index 7b2d579d..23a1c32d 100644 --- a/e2e/cypress/support/command.js +++ b/e2e/cypress/support/command.js @@ -101,9 +101,8 @@ Cypress.Commands.add('create_organization', (orgName, orgDesc) => { body: { description: orgDesc, title: orgName, - approval_status: 'approved', - state: 'active', name: orgName, + save: null }, }); }); From c500745700bab24065e5ae958c3f4bcc9dcebf8a Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 11:57:45 -0400 Subject: [PATCH 04/73] refactor: use the 'create_organization_ui' instead? --- e2e/cypress/integration/000_harvest.cy.js | 2 +- e2e/cypress/integration/facets.cy.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/cypress/integration/000_harvest.cy.js b/e2e/cypress/integration/000_harvest.cy.js index 68d9d471..eb178d57 100644 --- a/e2e/cypress/integration/000_harvest.cy.js +++ b/e2e/cypress/integration/000_harvest.cy.js @@ -14,7 +14,7 @@ describe('Harvest', { testIsolation: false }, () => { // Make sure organization does not exist before creating cy.delete_organization(harvestOrg); // Create the organization - cy.create_organization(harvestOrg, 'cypress harvest org description', false); + cy.create_organization_ui(harvestOrg, 'cypress harvest org description'); }); after(() => { diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index 26dfc0e8..24ac3cd9 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -17,7 +17,7 @@ describe('Facets', { testIsolation: false }, () => { }); it('Show datagov facet list on organization page', () => { - cy.create_organization('org-tags', ''); + cy.create_organization_ui('org-tags', ''); cy.visit('/organization/org-tags'); cy.get('.filters h2').its('length').should('be.equal', 10); cy.get('.filters h2').first().contains('Topics'); From 7e91dfd7dc1641498016a328fb64966e2a0bbdaf Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 12:14:08 -0400 Subject: [PATCH 05/73] fix: add missing pre-work (i.e. visit org page) --- e2e/cypress/integration/000_harvest.cy.js | 2 ++ e2e/cypress/integration/facets.cy.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/e2e/cypress/integration/000_harvest.cy.js b/e2e/cypress/integration/000_harvest.cy.js index eb178d57..7c3f7183 100644 --- a/e2e/cypress/integration/000_harvest.cy.js +++ b/e2e/cypress/integration/000_harvest.cy.js @@ -14,6 +14,8 @@ describe('Harvest', { testIsolation: false }, () => { // Make sure organization does not exist before creating cy.delete_organization(harvestOrg); // Create the organization + cy.visit('/organization'); + cy.get('a[class="btn btn-primary"]').click(); cy.create_organization_ui(harvestOrg, 'cypress harvest org description'); }); diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index 24ac3cd9..c1a5ea60 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -17,6 +17,8 @@ describe('Facets', { testIsolation: false }, () => { }); it('Show datagov facet list on organization page', () => { + cy.visit('/organization'); + cy.get('a[class="btn btn-primary"]').click(); cy.create_organization_ui('org-tags', ''); cy.visit('/organization/org-tags'); cy.get('.filters h2').its('length').should('be.equal', 10); From d21288225b9742e41c20fec7231cd20302afb907 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 24 May 2023 08:09:10 -0400 Subject: [PATCH 06/73] update: geodatagov fix /harvest/new template issue -- https://github.com/GSA/ckanext-geodatagov/pull/255/commits/1d6a8071ec625480f091eaacb84ee92b7b703c55 --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 21f263a0..d2c45277 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d3 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@31b92ada972664b2b532fd63096b3c96ba8e40fd#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@936e85a71a09858afa05bc01c9b5e07a060d8d4d#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 3827fd8a689271921ff9e5d95c227b95739cfc5b Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 24 May 2023 09:19:45 -0400 Subject: [PATCH 07/73] update: geodatagov try to test fix for url_for replacement? --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index d2c45277..bc0fae2b 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d3 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@936e85a71a09858afa05bc01c9b5e07a060d8d4d#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 4c3cc787680b042be03484d5cbc043248c5e16c9 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 31 May 2023 17:54:59 -0400 Subject: [PATCH 08/73] new: use ckan 2.10.1 + harvest is too new There was a bad PR merged that caused a recursion error with it, https://github.com/ckan/ckanext-harvest/commit/47100351e70f538d148898b781bc60947419c8cf --- ckan/requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index bc0fae2b..342a6a21 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -11,7 +11,8 @@ certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==2.1.1 -ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 +# ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 +ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog -e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme @@ -20,7 +21,7 @@ ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232 ckanext-envvars==0.0.3 -e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic --e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@bae212784ba309409bb079605e622eb7028740f9#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report From afe1b86dead9b2fe05d7db0e3edd881238a09ccf Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 15 Jun 2023 12:00:30 -0400 Subject: [PATCH 09/73] fix: remove log scope to prevent recursion issue https://gsa-tts.slack.com/archives/C2N85536E/p1686773232427389 --- .env | 1 - 1 file changed, 1 deletion(-) diff --git a/.env b/.env index b8400e1b..c603265e 100644 --- a/.env +++ b/.env @@ -61,7 +61,6 @@ CKAN__HARVEST__MQ__HOSTNAME=redis CKAN__HARVEST__MQ__PORT=6379 CKAN__HARVEST__MQ__REDIS_DB=1 CKAN__HARVEST__LOG_LEVEL=info -CKAN__HARVEST__LOG_SCOPE=0 CKAN__HARVEST__STATUS_MAIL__ALL=True From 3b8cd47db584730a9388f1b1427a6f43fbd1a7f2 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 15 Jun 2023 12:02:51 -0400 Subject: [PATCH 10/73] update: use latest ckanext-harvest --- ckan/requirements.in | 3 +-- ckan/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 816b69bf..0e9263f5 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,6 +1,5 @@ # CKAN requirements and extensions -# ckan==2.9.8 -git+https://github.com/gsa/ckan.git@ckan-210-temp#egg=ckan +ckan==2.10.1 git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest -e git+https://github.com/ckan/ckanext-spatial.git@master#egg=ckanext-spatial diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 342a6a21..4cbdf387 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -21,7 +21,7 @@ ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232 ckanext-envvars==0.0.3 -e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic --e git+https://github.com/ckan/ckanext-harvest.git@bae212784ba309409bb079605e622eb7028740f9#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@e1127fa92d1aac6e7278f0e30303a576f775c4f2#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report From 51b38079a87a701e8f63b63d18f5026f7be0fa80 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 21 Jun 2023 11:59:51 -0400 Subject: [PATCH 11/73] set search_backend to solr-bbox --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index c603265e..20db4387 100644 --- a/.env +++ b/.env @@ -69,7 +69,7 @@ CKANEXT__GEODATAGOV__BUREAU_CSV__URL_DEFAULT=https://resources.data.gov/schemas/ CKAN__SPATIAL__SRID=4326 CKAN__SPATIAL__VALIDATOR__PROFILES=iso19139ngdc -CKANEXT__SPATIAL__SEARCH_BACKEND=solr +CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox CKAN___GOOGLEANALYTICS__IDS=UA-1010101-1 UA-1010101-2 CKAN__TRACKING_ENABLED=true From 0b0a5aaadc44a93da041808cec81980ab4c68a3c Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 21 Jun 2023 12:01:36 -0400 Subject: [PATCH 12/73] set search_backend to solr-bbox --- ckan/setup/ckan.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index 623ffbde..b2a01d56 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -226,7 +226,7 @@ ckan.jobs.timeout = 180 ckan.tracking_enabled = True ## Spatial settings -ckanext.spatial.search_backend = solr +ckanext.spatial.search_backend = solr-bbox ## Harvest settings # ckanext-harvest will use ckan.redis.url if redis configuration From efb413d3142d370271341e03c42292aff556dbfe Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 13:20:26 -0400 Subject: [PATCH 13/73] update: datagovtheme latest UI fixes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 4cbdf387..6338ffa5 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -15,7 +15,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@0ec706c712a8a1b956edc1baecab46350eb6fd14#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 70b521cec3793bc71ed4ef8a0f64f32c4bf44ebd Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 13:22:41 -0400 Subject: [PATCH 14/73] fix: enable csrf enforcement --- ckan/setup/ckan.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index b2a01d56..98bb719a 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -62,6 +62,8 @@ who.log_file = %(cache_dir)s/who_log.ini who.timeout = 900 who.secure = True +ckan.csrf_protection.ignore_extensions=False + ## Database Settings # sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default # Possible tweaks to production sizing From 86b0ffe933aca443e84a9696aef66ccbeb881895 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 15:19:41 -0400 Subject: [PATCH 15/73] update: python 3.8 to 3.9 spatial requires different pyproj for newer version of python 3.9 --- ckan/requirements.in | 2 +- ckan/requirements.txt | 2 +- runtime.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 0e9263f5..8303812f 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -45,7 +45,7 @@ argparse GeoAlchemy2==0.5.0 owslib>=0.28.1 pyparsing>=2.1.10 -pyproj==2.6.1 +pyproj==3.4.1 Shapely==2.0.1 geojson==3.0.1 # requests>=1.1.0 # included in ckan core diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 6338ffa5..1960b05f 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -82,7 +82,7 @@ pycparser==2.21 PyJWT==2.4.0 pyOpenSSL==23.1.1 pyparsing==3.0.9 -pyproj==2.6.1 +pyproj==3.4.1 pysaml2==7.0.1 pysolr==3.9.0 python-dateutil==2.8.2 diff --git a/runtime.txt b/runtime.txt index 5592f898..305091ca 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.x +python-3.9.x From f7c28fea7b5042a69fb1ddc06cd57ac78633c3be Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 15:20:04 -0400 Subject: [PATCH 16/73] fix: db url + init --- .profile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.profile b/.profile index 6b090038..5c80119a 100755 --- a/.profile +++ b/.profile @@ -84,6 +84,7 @@ export CKANEXT__SAML2AUTH__IDP_METADATA__LOCAL_PATH="${HOME}/${CKANEXT__SAML2AUT # Export settings for CKAN via ckanext-envvars export CKAN_REDIS_URL=rediss://:$REDIS_PASSWORD@$REDIS_HOST:$REDIS_PORT export CKAN_SQLALCHEMY_URL=$(vcap_get_service db .credentials.uri) +export CKAN_SQLALCHEMY_URL=${CKAN_SQLALCHEMY_URL/postgres/postgresql} export CKAN___SQLALCHEMY__POOL_SIZE=250 export CKAN___SQLALCHEMY__MAX_OVERFLOW=500 @@ -157,7 +158,7 @@ if [[ $MIGRATE_DB = 'True' ]]; then # Run migrations ckan db upgrade ckan harvester initdb - ckan archiver init - ckan report initdb - ckan qa init + # ckan archiver init + # ckan report initdb + # ckan qa init fi From f69bd76a92781d0b2502fe3049a95d008408aa12 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 16:22:10 -0400 Subject: [PATCH 17/73] update: harvest + datagovcatalog + geodatagov (indirect) --- ckan/requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 1960b05f..20bb39f8 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -11,17 +11,16 @@ certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==2.1.1 -# ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver --e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog +-e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog -e git+https://github.com/gsa/ckanext-datagovtheme.git@0ec706c712a8a1b956edc1baecab46350eb6fd14#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 -e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic --e git+https://github.com/ckan/ckanext-harvest.git@e1127fa92d1aac6e7278f0e30303a576f775c4f2#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report From 8a22d7b03a1ddb8b17a92fc2a89b0882cedf2bee Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 27 Jun 2023 12:40:57 -0400 Subject: [PATCH 18/73] fix: facets org creation cypress was complaining because it couldn't type an empty string :/ --- e2e/cypress/integration/facets.cy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index c1a5ea60..56148478 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -19,7 +19,7 @@ describe('Facets', { testIsolation: false }, () => { it('Show datagov facet list on organization page', () => { cy.visit('/organization'); cy.get('a[class="btn btn-primary"]').click(); - cy.create_organization_ui('org-tags', ''); + cy.create_organization_ui('org-tags', 'tags for org test'); cy.visit('/organization/org-tags'); cy.get('.filters h2').its('length').should('be.equal', 10); cy.get('.filters h2').first().contains('Topics'); From 7637ae86cd5c28bff377e7c412cc39ecedb42c6e Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 28 Jun 2023 09:43:43 -0400 Subject: [PATCH 19/73] disable nginx-authy.conf for now --- proxy/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/nginx.conf b/proxy/nginx.conf index 1605179f..e5e32483 100644 --- a/proxy/nginx.conf +++ b/proxy/nginx.conf @@ -47,7 +47,7 @@ http { # catalog-admin server_name {{env "EXTERNAL_ROUTE_ADMIN"}}; - include nginx-authy.conf; + # include nginx-authy.conf; include nginx-common.conf; } } From 8b840be72957d6b4c3f913c77923c67ba53638c7 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 28 Jun 2023 09:59:11 -0400 Subject: [PATCH 20/73] fix: auth_tkt functionality removed in ckan 2.10 If we knew our tests, we would've seen that this disappeared and caused https://github.com/GSA/data.gov/issues/4371 --- e2e/cypress/integration/login.cy.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/e2e/cypress/integration/login.cy.js b/e2e/cypress/integration/login.cy.js index b7c362a4..807b7b82 100644 --- a/e2e/cypress/integration/login.cy.js +++ b/e2e/cypress/integration/login.cy.js @@ -4,8 +4,6 @@ describe('Login', () => { cy.get('a[href="/user/login"]').click(); cy.login('not-user', 'not-password', true); cy.get('.flash-messages .alert').should('contain', 'Login failed. Bad username or password.'); - // Validate cookie is not set - cy.getCookie('auth_tkt').should('not.exist'); }); it('Valid login attempt', () => { @@ -13,7 +11,5 @@ describe('Login', () => { cy.get('a[href="/user/login"]').click(); cy.login(); cy.get('.nav-tabs>li>a').should('contain', 'My Organizations'); - // Validate cookie is set, in development secure is set to false - cy.getCookie('auth_tkt').should('have.property', 'secure', false); }); }); From 2c7bd7a2be80b01ba26e3c6c614ea38de3ae2a69 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 28 Jun 2023 15:08:32 -0400 Subject: [PATCH 21/73] change the datagovtheme version with the flash message changes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 20bb39f8..74d85d13 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@0ec706c712a8a1b956edc1baecab46350eb6fd14#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@76967b4fcc571b59cea4870b9bcc8352ae75f042#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From b40519aed821fb8c78caa367167dc23a20158b2b Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 28 Jun 2023 21:33:07 -0400 Subject: [PATCH 22/73] create our own auth_tkt cookie --- proxy/nginx-authy.conf | 41 +++++++++++++++++++++++++++++++++++++++-- proxy/nginx.conf | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/proxy/nginx-authy.conf b/proxy/nginx-authy.conf index 27044318..fa934b7f 100644 --- a/proxy/nginx-authy.conf +++ b/proxy/nginx-authy.conf @@ -1,7 +1,41 @@ +################################## +# CKAN used to add an auth_tkt cookie for logged-in user +# It is gone after 2.10. Let us generate our own auth_tkt here. +# If the request comes from login.gov and goes to /user/me, +# we know the user is logged in. +set $check ""; +set $cookiecontent ""; +set $goodauth "auth_tkt=1; PATH=/"; +set $badauth "auth_tkt=0; PATH=/"; + +if ($uri = "/user/me") { + set $check "endpointcheck"; +} +if ($http_referer ~* \.(login|identitysandbox).gov/$ ) { + set $check "${check}+referercheck"; +} +if ($check = "endpointcheck+referercheck") { + set $cookiecontent $goodauth; +} +if ($uri = "/user/logged_out_redirect") { + set $cookiecontent $badauth; +} + +add_header Set-Cookie $cookiecontent; + +if ($uri = "/user/logged_out_redirect") { + return 302 https://{{env "PUBLIC_ROUTE"}}$uri; +} + +########################################## +# If cookie auth_tkt does not have a good value, +# we dont serve the request on the admin site. +# We 302 redirect it to the public site, +# except some special uri listed below: set $authy ""; -# Determine if auth cookie is set -if ($cookie_auth_tkt) { +# Determine if auth cookie is set with good value +if ($cookie_auth_tkt = 1) { set $authy C; } if ($uri = "/user/saml2login") { @@ -10,6 +44,9 @@ if ($uri = "/user/saml2login") { if ($uri = "/acs") { set $authy "${authy}S"; } +if ($uri = "/user/me") { + set $authy "${authy}M"; +} if ($uri = "/api/action/status_show") { set $authy "${authy}H"; } diff --git a/proxy/nginx.conf b/proxy/nginx.conf index e5e32483..1605179f 100644 --- a/proxy/nginx.conf +++ b/proxy/nginx.conf @@ -47,7 +47,7 @@ http { # catalog-admin server_name {{env "EXTERNAL_ROUTE_ADMIN"}}; - # include nginx-authy.conf; + include nginx-authy.conf; include nginx-common.conf; } } From c585d2e67759355dfba194fa8291de1b57ed328c Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Thu, 29 Jun 2023 12:45:37 -0400 Subject: [PATCH 23/73] new version of geodatatheme for the collection fixes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 74d85d13..3f48e435 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@76967b4fcc571b59cea4870b9bcc8352ae75f042#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@aa1a621598c969f3175ce39d0f6806a6cfbbb4d5#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 72649d053f14c67bf9442df20480a1a419d3c215 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 13:26:19 -0400 Subject: [PATCH 24/73] update: datagovtheme resource page fixes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 3f48e435..f33c4448 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@aa1a621598c969f3175ce39d0f6806a6cfbbb4d5#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@7ec4c0be6a7803e5d1358b5e8f605db73f2e5c4c#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 97fd31fb9f2db2ccb305154dad58390c3d420091 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 14:36:48 -0400 Subject: [PATCH 25/73] fix: facet list test the filters class doesn't exist on the org page for some reason.. so use a different class to specify tags --- e2e/cypress/integration/facets.cy.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index 56148478..2d527447 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -21,10 +21,10 @@ describe('Facets', { testIsolation: false }, () => { cy.get('a[class="btn btn-primary"]').click(); cy.create_organization_ui('org-tags', 'tags for org test'); cy.visit('/organization/org-tags'); - cy.get('.filters h2').its('length').should('be.equal', 10); - cy.get('.filters h2').first().contains('Topics'); - cy.get('.filters h2').contains('Harvest Source'); - cy.get('.filters h2').last().contains('Bureaus'); + cy.get('.module-shallow').its('length').should('be.equal', 11); + cy.get('.module-shallow').contains('Topics'); + cy.get('.module-shallow').contains('Harvest Source'); + cy.get('.module-shallow').last().contains('Bureaus'); }); it('Show datagov facet list on group page', () => { From c946632f6a59660bb8537b8ba229ffbef3d1008c Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 14:38:22 -0400 Subject: [PATCH 26/73] fix: ckan version test --- e2e/cypress/integration/ckan_extensions.cy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cypress/integration/ckan_extensions.cy.js b/e2e/cypress/integration/ckan_extensions.cy.js index 7cd0d61e..701c7a74 100644 --- a/e2e/cypress/integration/ckan_extensions.cy.js +++ b/e2e/cypress/integration/ckan_extensions.cy.js @@ -2,7 +2,7 @@ describe('CKAN Extensions', () => { it('Uses CKAN 2.9', () => { cy.request('/api/action/status_show').should((response) => { expect(response.body).to.have.property('success', true); - expect(response.body.result).to.have.property('ckan_version', '2.9.8'); + expect(response.body.result).to.have.property('ckan_version', '2.10.1'); }); }); From c36b9c57e8eab268b8ed021231db144d2026c8cd Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 15:50:46 -0400 Subject: [PATCH 27/73] fix: cypress force selection for harvest source creation --- e2e/cypress/support/command.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/cypress/support/command.js b/e2e/cypress/support/command.js index 23a1c32d..3a258379 100644 --- a/e2e/cypress/support/command.js +++ b/e2e/cypress/support/command.js @@ -214,14 +214,14 @@ Cypress.Commands.add( }); cy.get('#field-notes').type(harvestDesc); - cy.get('[type="radio"]').check(harvestType); + cy.get('[type="radio"]').check(harvestType, { force: true }); // Validate private_datasets defaults to Private cy.get('#field-private_datasets').find(':selected').contains('Private'); - cy.get('#field-private_datasets').select(harvestPrivate); + cy.get('#field-private_datasets').select(harvestPrivate, { force: true }); - cy.get('input[name=save]').click(); + cy.get('input[name=save]').click({ force: true }); } ); From f341e2a4528e30aaa5ac192603e35fed0471963b Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 15:51:13 -0400 Subject: [PATCH 28/73] update: make sure fetch stays alive during harvesting --- ckan/docker-entrypoint.d/10-setup-harvest.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/docker-entrypoint.d/10-setup-harvest.sh b/ckan/docker-entrypoint.d/10-setup-harvest.sh index afa48dfe..6331ad05 100644 --- a/ckan/docker-entrypoint.d/10-setup-harvest.sh +++ b/ckan/docker-entrypoint.d/10-setup-harvest.sh @@ -4,7 +4,12 @@ echo "Init Harvest database tables" ckan harvester initdb echo "turn on gather and fetch workers" -ckan harvester fetch-consumer & +run_fetch () { + until ckan harvester fetch-consumer; do + sleep 1 + done +} +run_fetch & ckan harvester fetch-consumer & ckan harvester gather-consumer & From b231709705d0f97597ad02dc55bcfbb902fb89b5 Mon Sep 17 00:00:00 2001 From: Fuhu Xia <1392461+FuhuXia@users.noreply.github.com> Date: Thu, 6 Jul 2023 12:32:52 -0400 Subject: [PATCH 29/73] correct requirements.txt --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index f33c4448..39e48cf8 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@56736006e2e50cac48bf88f39ee48e7553884a8c#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 8fd60ec77da5f16b76f186647a5623b6dd4051cf Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 10 Jul 2023 12:25:46 -0400 Subject: [PATCH 30/73] test: new spatial parsing algorithm --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 39e48cf8..2ce17f25 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@56736006e2e50cac48bf88f39ee48e7553884a8c#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@de28a3ec18306fa967c26480cca6fc85b34f7a0e#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 75fff7b2da67764c85e13e850234eb5700b15051 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 10 Jul 2023 13:34:37 -0400 Subject: [PATCH 31/73] test: second spatial algorithm --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 2ce17f25..7fc94e1d 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@de28a3ec18306fa967c26480cca6fc85b34f7a0e#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@82757941aa8c23d7e22eeb6385d15729e6bb114c#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From c574d6842cf578bf41c02557f49edd25a2ebaa0c Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 11 Jul 2023 09:29:14 -0400 Subject: [PATCH 32/73] test: third spatial algorithm is the charm ;) --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 7fc94e1d..e0d58ef1 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@82757941aa8c23d7e22eeb6385d15729e6bb114c#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@bbbc9fe21f4960f89c85f4b58be24dbcce3de931#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 66fc670b553996107219bf543bf90ca89dcacc99 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 11 Jul 2023 10:14:46 -0400 Subject: [PATCH 33/73] new: maybe gather is also crashing? --- ckan/docker-entrypoint.d/10-setup-harvest.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/docker-entrypoint.d/10-setup-harvest.sh b/ckan/docker-entrypoint.d/10-setup-harvest.sh index 6331ad05..d9cee2e9 100644 --- a/ckan/docker-entrypoint.d/10-setup-harvest.sh +++ b/ckan/docker-entrypoint.d/10-setup-harvest.sh @@ -9,9 +9,13 @@ run_fetch () { sleep 1 done } +run_gather () { + until ckan harvester gather-consumer; do + sleep 1 + done +} run_fetch & -ckan harvester fetch-consumer & -ckan harvester gather-consumer & +run_gather & echo "check harvest job completion every 10 secs" check_harvester () { From c50e2c31293a588c8539c29552030e9c6c50168e Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 11 Jul 2023 11:47:03 -0400 Subject: [PATCH 34/73] test: fourth and final spatial algorithm for the win --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index e0d58ef1..e0e5c03f 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@bbbc9fe21f4960f89c85f4b58be24dbcce3de931#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@cbbcf599b6273fca56ec60f7b5d8ea2301dae415#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From c2d9dccb0b62b88dd8c6a6e48f01bc169eded0cb Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Tue, 11 Jul 2023 15:33:04 -0400 Subject: [PATCH 35/73] ckanext-spatial v1.1.0 --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index e0e5c03f..4c8fc7d2 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -25,7 +25,7 @@ ckanext-envvars==0.0.3 -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@c2b12a94430034c522b25d282323a064e2d6a03a --e git+https://github.com/ckan/ckanext-spatial.git@1205d3aa408e994439fc231ccb5d1cff06d644c6#egg=ckanext_spatial +-e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial ckantoolkit==0.0.7 click==8.1.3 cryptography==40.0.2 From 8a6481f383e6fbbecc04bb9a21450fc9f0c11b69 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Thu, 13 Jul 2023 18:24:34 -0400 Subject: [PATCH 36/73] udpate datagovtheme --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 4c8fc7d2..f0380829 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@7ec4c0be6a7803e5d1358b5e8f605db73f2e5c4c#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@eb310589aa3007516070fc1cdbfdcf7e3dea116d#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From dc3a50876f1b37ddbd2ea2169c71e054d6155d00 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 09:58:01 -0400 Subject: [PATCH 37/73] update: all of the requirements referencing the ckan commit might still have issues... --- ckan/requirements.in | 82 ++++++++++++++++--------------- ckan/requirements.txt | 109 ++++++++++++++++++++++-------------------- 2 files changed, 102 insertions(+), 89 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index a512d58b..5bb5784c 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,5 +1,6 @@ # CKAN requirements and extensions -git+https://github.com/gsa/ckan.git@ckan-2.9.8-momentjs#egg=ckan +# ckan==2.9.8 +git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7#egg=ckan git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest -e git+https://github.com/ckan/ckanext-spatial.git@master#egg=ckanext-spatial @@ -8,21 +9,26 @@ git+https://github.com/GSA/ckanext-saml2auth.git@create_user_via_saml#egg=ckanex -e git+https://github.com/ckan/ckanext-archiver.git@master#egg=ckanext-archiver -e git+https://github.com/ckan/ckanext-report.git@master#egg=ckanext-report -ckanext-datagovcatalog>=0.0.3 -ckanext-datagovtheme>=0.1.22, ==0.1.* -ckanext-datajson>=0.1.13 -ckanext-envvars>=0.0.2 -ckanext-geodatagov>=0.1.28 -ckanext-googleanalyticsbasic -ckanext-metrics-dashboard +# ckanext-datagovcatalog>=0.0.3 +# ckanext-datagovtheme>=0.1.22 +# ckanext-datajson>=0.1.13 +ckanext-envvars>=0.0.3 +# ckanext-geodatagov>=0.1.28 +# ckanext-googleanalyticsbasic +# ckanext-metrics-dashboard +-e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov +-e git+https://github.com/gsa/ckanext-datagovcatalog.git@ckan-210#egg=ckanext-datagovcatalog +-e git+https://github.com/gsa/ckanext-datagovtheme.git@ckan-210#egg=ckanext-datagovtheme +-e git+https://github.com/gsa/ckanext-datajson.git@ckan-210#egg=ckanext-datajson +-e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@ckan-2.10#egg=ckanext-googleanalyticsbasic +-e git+https://github.com/gsa/ckanext-metrics-dashboard.git@ckan-2.10#egg=ckanext-metrics-dashboard # Pin for saml2auth to work pysaml2==7.0.1 # ckanext-harvest dependencies ckantoolkit>=0.0.7 -pika>=1.1.0 -pyOpenSSL>=22.0.0 +pika>=1.1.0,<1.3.0 # redis==2.10.6 # included in ckan core # requests>=2.11.1 # included in ckan core # six>=1.12.0 # included in dcat @@ -41,7 +47,8 @@ GeoAlchemy2==0.5.0 owslib>=0.28.1 pyparsing>=2.1.10 pyproj==2.6.1 -Shapely>=1.2.13 +Shapely==2.0.1 +geojson==3.0.1 # requests>=1.1.0 # included in ckan core # six # included in dcat @@ -55,51 +62,50 @@ messytables==0.15.2 # https://github.com/ckan/ckanext-report/blob/master/requirements.txt #ckanext-dcat -rdflib==4.2.2 +rdflib==6.1.1 # rdflib-jsonld==0.4.0 # ignoring as cannot build on cloud.gov geomet>=0.2.0 future>=0.18.3 -six>=1.15.0 # ckan doesn't advertise its requirements correctly, so let's add them here. # https://github.com/ckan/ckan/blob/ckan-2.9.5/requirements.txt -alembic==1.0.0 -Babel==2.9.1 # Upgraded from CKAN pin +alembic==1.8.1 +Babel==2.10.3 Beaker==1.11.0 bleach==3.3.0 -click==7.1.2 -dominate==2.4.0 -fanstatic==1.1 +blinker==1.5 +click==8.1.3 +dominate==2.7.0 feedgen==0.9.0 -Flask==2.0 +Flask==2.0.3 Flask-Babel==1.0.0 flask-multistatic==1.0 -Jinja2==3.0 -lxml==4.9.1 +Flask-Login==0.6.1 +Jinja2==3.1.2 +Flask-WTF==1.0.1 pyjwt==2.4.0 -Markdown==3.1.1 # Upgraded from CKAN pin -passlib==1.7.3 +Markdown==3.4.1 +passlib==1.7.4 pastedeploy==2.0.1 # manually kept - remove when #4802 is complete pathtools==0.1.2 # via watchdog -polib==1.0.7 -psycopg2==2.8.6 -python-magic==0.4.15 -pysolr==3.6.0 -python-dateutil>=1.5.0 -pytz==2016.7 -PyUtilib==5.7.1 -pyyaml==5.4 -repoze.who==2.3 -Routes==1.13 -rq==1.0 +polib==1.1.1 +psycopg2==2.9.3 +PyJWT==2.4.0 +python-magic==0.4.27 +pysolr==3.9.0 +python-dateutil>=2.8.2 +pytz +PyUtilib==6.0.0 +pyyaml==6.0 +requests==2.28.1 +rq==1.11.0 simplejson==3.18.0 # ckan 2.9.5 requires 3.10.0 only # Following can be ignored: https://github.com/ckan/ckan/pull/4450 # sqlalchemy-migrate==0.12.0 -SQLAlchemy==1.3.5 +SQLAlchemy[mypy]==1.4.41 sqlparse==0.4.4 -tzlocal==1.3 -unicodecsv>=0.9 -webassets==0.12.1 +tzlocal==4.2 +webassets==2.0 Werkzeug==2.0 # zope.interface==4.7.2 diff --git a/ckan/requirements.txt b/ckan/requirements.txt index ee8442a9..e09d76ff 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -1,115 +1,122 @@ -alembic==1.0.0 +alembic==1.8.1 async-timeout==4.0.2 -Babel==2.9.1 +Babel==2.10.3 +backports.zoneinfo==0.2.1 Beaker==1.11.0 bleach==3.3.0 -boto3==1.26.126 -botocore==1.29.126 -certifi==2022.12.7 +blinker==1.5 +boto3==1.26.137 +botocore==1.29.137 +certifi==2023.5.7 cffi==1.15.1 -chardet==3.0.4 -git+https://github.com/gsa/ckan.git@4af8efa19edf7259530ccde5c0331a5ccd7f7066#egg=ckan +chardet==5.1.0 +charset-normalizer==2.1.1 +ckan @ git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -ckanext-datagovcatalog==0.0.5 -ckanext-datagovtheme==0.1.27 -ckanext-datajson==0.1.18 -ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@618928be5a211babafc45103a72b6aab4642e964 +-e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog +-e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson +ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 -ckanext-geodatagov==0.1.37 -ckanext-googleanalyticsbasic==0.2.0 --e git+https://github.com/ckan/ckanext-harvest.git@89a98d7ff5aa3445d8158921669b8d0b04fa41c3#egg=ckanext_harvest -ckanext-metrics-dashboard==0.1.5 +-e git+https://github.com/gsa/ckanext-geodatagov.git@46a3ff790c734bb8cb3fed226f86a123dc6c1975#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic +-e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest +-e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@c2b12a94430034c522b25d282323a064e2d6a03a --e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial +-e git+https://github.com/ckan/ckanext-spatial.git@1205d3aa408e994439fc231ccb5d1cff06d644c6#egg=ckanext_spatial ckantoolkit==0.0.7 -click==7.1.2 -cryptography==41.0.0 +click==8.1.3 +cryptography==40.0.2 defusedxml==0.7.1 -dominate==2.4.0 +dominate==2.7.0 elementpath==4.1.2 -fanstatic==1.1 feedgen==0.9.0 -Flask==2.0.0 +Flask==2.0.3 Flask-Babel==1.0.0 +Flask-Login==0.6.1 flask-multistatic==1.0 +Flask-WTF==1.0.1 future==0.18.3 GeoAlchemy2==0.5.0 +geojson==3.0.1 geomet==1.0.0 gevent==22.10.2 greenlet==2.0.2 gunicorn==20.1.0 html5lib==1.1 -idna==2.10 +idna==3.4 +importlib-metadata==6.6.0 importlib-resources==5.12.0 isodate==0.6.1 itsdangerous==2.1.2 -Jinja2==3.0.0 +Jinja2==3.1.2 jmespath==1.0.1 json-table-schema==0.2.1 jsonschema==2.4.0 -lxml==4.9.1 +lxml==4.9.2 Mako==1.2.4 -Markdown==3.1.1 +Markdown==3.4.1 MarkupSafe==2.1.2 messytables==0.15.2 +mypy==1.3.0 +mypy-extensions==1.0.0 newrelic==8.8.0 nose==1.3.7 numpy==1.24.3 -OWSLib==0.29.1 +OWSLib==0.29.2 packaging==23.1 -passlib==1.7.3 +passlib==1.7.4 PasteDeploy==2.0.1 pathtools==0.1.2 -pika==1.3.1 +pika==1.2.1 pip==23.1.2 ply==3.11 -polib==1.0.7 +polib==1.1.1 progressbar==2.5 progressbar2==3.53.3 -psycopg2==2.8.6 +psycopg2==2.9.3 pycparser==2.21 PyJWT==2.4.0 pyOpenSSL==23.2.0 pyparsing==3.0.9 pyproj==2.6.1 pysaml2==7.0.1 -pysolr==3.6.0 +pysolr==3.9.0 python-dateutil==2.8.2 -python-editor==1.0.4 -python-magic==0.4.15 +python-magic==0.4.27 python-utils==3.5.2 -pytz==2016.7 -PyUtilib==5.7.1 -PyYAML==5.4 +pytz==2023.3 +pytz-deprecation-shim==0.1.0.post0 +PyUtilib==6.0.0 +PyYAML==6.0 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219 -rdflib==4.2.2 -redis==4.5.4 -repoze.lru==0.7 -repoze.who==2.3 -requests==2.31.0 +rdflib==6.1.1 +redis==4.5.5 +requests==2.28.1 rfc3987==1.3.8 -Routes==1.13 -rq==1.0 -s3transfer==0.6.0 +rq==1.11.0 +s3transfer==0.6.1 setuptools==67.1.0 shapely==2.0.1 -shutilwhich==1.1.0 simplejson==3.18.0 six==1.16.0 -SQLAlchemy==1.3.5 +SQLAlchemy==1.4.41 +sqlalchemy2-stubs==0.0.2a34 sqlparse==0.4.4 -tzlocal==1.3 -unicodecsv==0.14.1 +tomli==2.0.1 +typing_extensions==4.5.0 +tzdata==2023.3 +tzlocal==4.2 urllib3==1.26.15 -webassets==0.12.1 +webassets==2.0 webencodings==0.5.1 -WebOb==1.8.7 Werkzeug==2.0.0 wheel==0.40.0 +WTForms==3.0.1 xlrd==2.0.1 -xmlschema==2.2.3 +xmlschema==2.3.0 zipp==3.15.0 zope.event==4.6 zope.interface==6.0 From cbd98380bac9c56f17caf641a9142fa6816f8907 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 11:05:25 -0400 Subject: [PATCH 38/73] new: use custom fork with necessary fixes only --- ckan/requirements.in | 2 +- ckan/requirements.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 5bb5784c..0e22f7f2 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,6 +1,6 @@ # CKAN requirements and extensions # ckan==2.9.8 -git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7#egg=ckan +git+https://github.com/gsa/ckan.git@ckan-210-temp#egg=ckan git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest -e git+https://github.com/ckan/ckanext-spatial.git@master#egg=ckanext-spatial diff --git a/ckan/requirements.txt b/ckan/requirements.txt index e09d76ff..76e085d7 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -11,14 +11,14 @@ certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==2.1.1 -ckan @ git+https://github.com/ckan/ckan.git@76ebf1ebab92a9bb08649cfe5e758e91edb616b7 +ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog -e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@46a3ff790c734bb8cb3fed226f86a123dc6c1975#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@31b92ada972664b2b532fd63096b3c96ba8e40fd#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 1eb9d067a71ae2ec4c95f4ac8e02e017aa4ea866 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 11:49:18 -0400 Subject: [PATCH 39/73] test: how to fix organization_create? --- e2e/cypress/support/command.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/e2e/cypress/support/command.js b/e2e/cypress/support/command.js index 7b2d579d..23a1c32d 100644 --- a/e2e/cypress/support/command.js +++ b/e2e/cypress/support/command.js @@ -101,9 +101,8 @@ Cypress.Commands.add('create_organization', (orgName, orgDesc) => { body: { description: orgDesc, title: orgName, - approval_status: 'approved', - state: 'active', name: orgName, + save: null }, }); }); From 2cc38a3bfd200db43d6ef5f5c38ca1447400343c Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 11:57:45 -0400 Subject: [PATCH 40/73] refactor: use the 'create_organization_ui' instead? --- e2e/cypress/integration/000_harvest.cy.js | 2 +- e2e/cypress/integration/facets.cy.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/cypress/integration/000_harvest.cy.js b/e2e/cypress/integration/000_harvest.cy.js index 68d9d471..eb178d57 100644 --- a/e2e/cypress/integration/000_harvest.cy.js +++ b/e2e/cypress/integration/000_harvest.cy.js @@ -14,7 +14,7 @@ describe('Harvest', { testIsolation: false }, () => { // Make sure organization does not exist before creating cy.delete_organization(harvestOrg); // Create the organization - cy.create_organization(harvestOrg, 'cypress harvest org description', false); + cy.create_organization_ui(harvestOrg, 'cypress harvest org description'); }); after(() => { diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index 26dfc0e8..24ac3cd9 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -17,7 +17,7 @@ describe('Facets', { testIsolation: false }, () => { }); it('Show datagov facet list on organization page', () => { - cy.create_organization('org-tags', ''); + cy.create_organization_ui('org-tags', ''); cy.visit('/organization/org-tags'); cy.get('.filters h2').its('length').should('be.equal', 10); cy.get('.filters h2').first().contains('Topics'); From 9aa286740372aa34007266905f45d51f3ef02ae3 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 22 May 2023 12:14:08 -0400 Subject: [PATCH 41/73] fix: add missing pre-work (i.e. visit org page) --- e2e/cypress/integration/000_harvest.cy.js | 2 ++ e2e/cypress/integration/facets.cy.js | 2 ++ 2 files changed, 4 insertions(+) diff --git a/e2e/cypress/integration/000_harvest.cy.js b/e2e/cypress/integration/000_harvest.cy.js index eb178d57..7c3f7183 100644 --- a/e2e/cypress/integration/000_harvest.cy.js +++ b/e2e/cypress/integration/000_harvest.cy.js @@ -14,6 +14,8 @@ describe('Harvest', { testIsolation: false }, () => { // Make sure organization does not exist before creating cy.delete_organization(harvestOrg); // Create the organization + cy.visit('/organization'); + cy.get('a[class="btn btn-primary"]').click(); cy.create_organization_ui(harvestOrg, 'cypress harvest org description'); }); diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index 24ac3cd9..c1a5ea60 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -17,6 +17,8 @@ describe('Facets', { testIsolation: false }, () => { }); it('Show datagov facet list on organization page', () => { + cy.visit('/organization'); + cy.get('a[class="btn btn-primary"]').click(); cy.create_organization_ui('org-tags', ''); cy.visit('/organization/org-tags'); cy.get('.filters h2').its('length').should('be.equal', 10); From ccd2bc33c11ebbffe41ae59b995bd9bee7d71bf5 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 24 May 2023 08:09:10 -0400 Subject: [PATCH 42/73] update: geodatagov fix /harvest/new template issue -- https://github.com/GSA/ckanext-geodatagov/pull/255/commits/1d6a8071ec625480f091eaacb84ee92b7b703c55 --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 76e085d7..0cedc98a 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d3 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@31b92ada972664b2b532fd63096b3c96ba8e40fd#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@936e85a71a09858afa05bc01c9b5e07a060d8d4d#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 4b2596c75140a4359f85102b75ac546b9247fe6e Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 24 May 2023 09:19:45 -0400 Subject: [PATCH 43/73] update: geodatagov try to test fix for url_for replacement? --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 0cedc98a..c47edb50 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d3 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@936e85a71a09858afa05bc01c9b5e07a060d8d4d#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 86f4383b9500c9d2927ae10cb556d32051f103f9 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 31 May 2023 17:54:59 -0400 Subject: [PATCH 44/73] new: use ckan 2.10.1 + harvest is too new There was a bad PR merged that caused a recursion error with it, https://github.com/ckan/ckanext-harvest/commit/47100351e70f538d148898b781bc60947419c8cf --- ckan/requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index c47edb50..4f596e82 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -11,7 +11,8 @@ certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==2.1.1 -ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 +# ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 +ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog -e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme @@ -20,7 +21,7 @@ ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232 ckanext-envvars==0.0.3 -e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic --e git+https://github.com/ckan/ckanext-harvest.git@eb73bed1739ac8656d892a609b5ee303e34251ca#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@bae212784ba309409bb079605e622eb7028740f9#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report From 853afd3393af59a854c8c7967ea2b6ad3b0882d3 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 15 Jun 2023 12:00:30 -0400 Subject: [PATCH 45/73] fix: remove log scope to prevent recursion issue https://gsa-tts.slack.com/archives/C2N85536E/p1686773232427389 --- .env | 1 - 1 file changed, 1 deletion(-) diff --git a/.env b/.env index b8400e1b..c603265e 100644 --- a/.env +++ b/.env @@ -61,7 +61,6 @@ CKAN__HARVEST__MQ__HOSTNAME=redis CKAN__HARVEST__MQ__PORT=6379 CKAN__HARVEST__MQ__REDIS_DB=1 CKAN__HARVEST__LOG_LEVEL=info -CKAN__HARVEST__LOG_SCOPE=0 CKAN__HARVEST__STATUS_MAIL__ALL=True From 8bbe6ef4af0c248f6051044dfa7be24d1a4f18d3 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 15 Jun 2023 12:02:51 -0400 Subject: [PATCH 46/73] update: use latest ckanext-harvest --- ckan/requirements.in | 3 +-- ckan/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 0e22f7f2..862355e2 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -1,6 +1,5 @@ # CKAN requirements and extensions -# ckan==2.9.8 -git+https://github.com/gsa/ckan.git@ckan-210-temp#egg=ckan +ckan==2.10.1 git+https://github.com/ckan/ckanext-dcat@master#egg=ckanext-dcat -e git+https://github.com/ckan/ckanext-harvest.git@master#egg=ckanext-harvest -e git+https://github.com/ckan/ckanext-spatial.git@master#egg=ckanext-spatial diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 4f596e82..9926a0c4 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -21,7 +21,7 @@ ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232 ckanext-envvars==0.0.3 -e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic --e git+https://github.com/ckan/ckanext-harvest.git@bae212784ba309409bb079605e622eb7028740f9#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@e1127fa92d1aac6e7278f0e30303a576f775c4f2#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report From ed8cda3c0820e1c01c00451df49e76ea84842a24 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 21 Jun 2023 11:59:51 -0400 Subject: [PATCH 47/73] set search_backend to solr-bbox --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index c603265e..20db4387 100644 --- a/.env +++ b/.env @@ -69,7 +69,7 @@ CKANEXT__GEODATAGOV__BUREAU_CSV__URL_DEFAULT=https://resources.data.gov/schemas/ CKAN__SPATIAL__SRID=4326 CKAN__SPATIAL__VALIDATOR__PROFILES=iso19139ngdc -CKANEXT__SPATIAL__SEARCH_BACKEND=solr +CKANEXT__SPATIAL__SEARCH_BACKEND=solr-bbox CKAN___GOOGLEANALYTICS__IDS=UA-1010101-1 UA-1010101-2 CKAN__TRACKING_ENABLED=true From 02e0e7580f70c92af53fce4324741ba745f8777b Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 21 Jun 2023 12:01:36 -0400 Subject: [PATCH 48/73] set search_backend to solr-bbox --- ckan/setup/ckan.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index fe6ec665..44bde974 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -226,7 +226,7 @@ ckan.jobs.timeout = 180 ckan.tracking_enabled = True ## Spatial settings -ckanext.spatial.search_backend = solr +ckanext.spatial.search_backend = solr-bbox ## Harvest settings # ckanext-harvest will use ckan.redis.url if redis configuration From fe836a1c9102c6e006cedd845213b0bbb572d043 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 13:20:26 -0400 Subject: [PATCH 49/73] update: datagovtheme latest UI fixes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 9926a0c4..28adf1fd 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -15,7 +15,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@8b031c6046f3b480931e948eaf14cdf2f4c8597c#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@0ec706c712a8a1b956edc1baecab46350eb6fd14#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 6ce2c5a9c8a50f6f13a3fe60462c35df7f757228 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 13:22:41 -0400 Subject: [PATCH 50/73] fix: enable csrf enforcement --- ckan/setup/ckan.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ckan/setup/ckan.ini b/ckan/setup/ckan.ini index 44bde974..0091d09c 100644 --- a/ckan/setup/ckan.ini +++ b/ckan/setup/ckan.ini @@ -62,6 +62,8 @@ who.log_file = %(cache_dir)s/who_log.ini who.timeout = 900 who.secure = True +ckan.csrf_protection.ignore_extensions=False + ## Database Settings # sqlalchemy.url = postgresql://ckan_default:pass@localhost/ckan_default # Possible tweaks to production sizing From be7515465f7e653b265950691a9df88ec251a88f Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 15:19:41 -0400 Subject: [PATCH 51/73] update: python 3.8 to 3.9 spatial requires different pyproj for newer version of python 3.9 --- ckan/requirements.in | 2 +- ckan/requirements.txt | 2 +- runtime.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.in b/ckan/requirements.in index 862355e2..0b8c5a7e 100644 --- a/ckan/requirements.in +++ b/ckan/requirements.in @@ -45,7 +45,7 @@ argparse GeoAlchemy2==0.5.0 owslib>=0.28.1 pyparsing>=2.1.10 -pyproj==2.6.1 +pyproj==3.4.1 Shapely==2.0.1 geojson==3.0.1 # requests>=1.1.0 # included in ckan core diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 28adf1fd..f6920bec 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -82,7 +82,7 @@ pycparser==2.21 PyJWT==2.4.0 pyOpenSSL==23.2.0 pyparsing==3.0.9 -pyproj==2.6.1 +pyproj==3.4.1 pysaml2==7.0.1 pysolr==3.9.0 python-dateutil==2.8.2 diff --git a/runtime.txt b/runtime.txt index 5592f898..305091ca 100644 --- a/runtime.txt +++ b/runtime.txt @@ -1 +1 @@ -python-3.8.x +python-3.9.x From 3e03564becf736deab9129b72a121dd4d55f6509 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 15:20:04 -0400 Subject: [PATCH 52/73] fix: db url + init --- .profile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.profile b/.profile index 6f75c477..117824f3 100755 --- a/.profile +++ b/.profile @@ -85,6 +85,7 @@ export CKANEXT__SAML2AUTH__IDP_METADATA__LOCAL_PATH="${HOME}/${CKANEXT__SAML2AUT # Export settings for CKAN via ckanext-envvars export CKAN_REDIS_URL=rediss://:$REDIS_PASSWORD@$REDIS_HOST:$REDIS_PORT export CKAN_SQLALCHEMY_URL=$(vcap_get_service db .credentials.uri) +export CKAN_SQLALCHEMY_URL=${CKAN_SQLALCHEMY_URL/postgres/postgresql} export CKAN___SQLALCHEMY__POOL_SIZE=250 export CKAN___SQLALCHEMY__MAX_OVERFLOW=500 @@ -158,7 +159,7 @@ if [[ $MIGRATE_DB = 'True' ]]; then # Run migrations ckan db upgrade ckan harvester initdb - ckan archiver init - ckan report initdb - ckan qa init + # ckan archiver init + # ckan report initdb + # ckan qa init fi From fbcbafd7c799975c8dec52a6b2957aad103c5a60 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 26 Jun 2023 16:22:10 -0400 Subject: [PATCH 53/73] update: harvest + datagovcatalog + geodatagov (indirect) --- ckan/requirements.txt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index f6920bec..28c3e12b 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -11,17 +11,16 @@ certifi==2023.5.7 cffi==1.15.1 chardet==5.1.0 charset-normalizer==2.1.1 -# ckan @ git+https://github.com/gsa/ckan.git@838b97e7ee14885af5cc4efed98d992b024d37d2 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver --e git+https://github.com/gsa/ckanext-datagovcatalog.git@a3b9c7287814fa5e5b6880712ec314f509c1ecba#egg=ckanext_datagovcatalog +-e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog -e git+https://github.com/gsa/ckanext-datagovtheme.git@0ec706c712a8a1b956edc1baecab46350eb6fd14#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 -e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic --e git+https://github.com/ckan/ckanext-harvest.git@e1127fa92d1aac6e7278f0e30303a576f775c4f2#egg=ckanext_harvest +-e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report From 41edea26586a3840d36fd81cfdf0a1d31f4a6395 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 27 Jun 2023 12:40:57 -0400 Subject: [PATCH 54/73] fix: facets org creation cypress was complaining because it couldn't type an empty string :/ --- e2e/cypress/integration/facets.cy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index c1a5ea60..56148478 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -19,7 +19,7 @@ describe('Facets', { testIsolation: false }, () => { it('Show datagov facet list on organization page', () => { cy.visit('/organization'); cy.get('a[class="btn btn-primary"]').click(); - cy.create_organization_ui('org-tags', ''); + cy.create_organization_ui('org-tags', 'tags for org test'); cy.visit('/organization/org-tags'); cy.get('.filters h2').its('length').should('be.equal', 10); cy.get('.filters h2').first().contains('Topics'); From 4c22dd08d56a3ff9f5486fff8fe820dd5c21c592 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 28 Jun 2023 09:43:43 -0400 Subject: [PATCH 55/73] disable nginx-authy.conf for now --- proxy/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/proxy/nginx.conf b/proxy/nginx.conf index 1605179f..e5e32483 100644 --- a/proxy/nginx.conf +++ b/proxy/nginx.conf @@ -47,7 +47,7 @@ http { # catalog-admin server_name {{env "EXTERNAL_ROUTE_ADMIN"}}; - include nginx-authy.conf; + # include nginx-authy.conf; include nginx-common.conf; } } From 8f42623b5db4d415c32cdb5521b331e100dfbb3e Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Wed, 28 Jun 2023 09:59:11 -0400 Subject: [PATCH 56/73] fix: auth_tkt functionality removed in ckan 2.10 If we knew our tests, we would've seen that this disappeared and caused https://github.com/GSA/data.gov/issues/4371 --- e2e/cypress/integration/login.cy.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/e2e/cypress/integration/login.cy.js b/e2e/cypress/integration/login.cy.js index b7c362a4..807b7b82 100644 --- a/e2e/cypress/integration/login.cy.js +++ b/e2e/cypress/integration/login.cy.js @@ -4,8 +4,6 @@ describe('Login', () => { cy.get('a[href="/user/login"]').click(); cy.login('not-user', 'not-password', true); cy.get('.flash-messages .alert').should('contain', 'Login failed. Bad username or password.'); - // Validate cookie is not set - cy.getCookie('auth_tkt').should('not.exist'); }); it('Valid login attempt', () => { @@ -13,7 +11,5 @@ describe('Login', () => { cy.get('a[href="/user/login"]').click(); cy.login(); cy.get('.nav-tabs>li>a').should('contain', 'My Organizations'); - // Validate cookie is set, in development secure is set to false - cy.getCookie('auth_tkt').should('have.property', 'secure', false); }); }); From a35510275e54fe48a405cb3595ff008be87a8482 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Wed, 28 Jun 2023 15:08:32 -0400 Subject: [PATCH 57/73] change the datagovtheme version with the flash message changes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 28c3e12b..0b8733d1 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@0ec706c712a8a1b956edc1baecab46350eb6fd14#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@76967b4fcc571b59cea4870b9bcc8352ae75f042#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 2a8ccba919022b4a60ac030c4566746c81077fb2 Mon Sep 17 00:00:00 2001 From: Jin-Sun-tts Date: Thu, 29 Jun 2023 12:45:37 -0400 Subject: [PATCH 58/73] new version of geodatatheme for the collection fixes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 0b8733d1..a60953aa 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@76967b4fcc571b59cea4870b9bcc8352ae75f042#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@aa1a621598c969f3175ce39d0f6806a6cfbbb4d5#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 1168be5fc168d2da9e437275b9bb8d41630a7bfe Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 13:26:19 -0400 Subject: [PATCH 59/73] update: datagovtheme resource page fixes --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index a60953aa..63a94852 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@aa1a621598c969f3175ce39d0f6806a6cfbbb4d5#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@7ec4c0be6a7803e5d1358b5e8f605db73f2e5c4c#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From 96c7f82db4b96ead9ab2a65f1f25042ec7a6eef9 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Wed, 28 Jun 2023 21:33:07 -0400 Subject: [PATCH 60/73] create our own auth_tkt cookie --- proxy/nginx-authy.conf | 41 +++++++++++++++++++++++++++++++++++++++-- proxy/nginx.conf | 2 +- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/proxy/nginx-authy.conf b/proxy/nginx-authy.conf index 27044318..fa934b7f 100644 --- a/proxy/nginx-authy.conf +++ b/proxy/nginx-authy.conf @@ -1,7 +1,41 @@ +################################## +# CKAN used to add an auth_tkt cookie for logged-in user +# It is gone after 2.10. Let us generate our own auth_tkt here. +# If the request comes from login.gov and goes to /user/me, +# we know the user is logged in. +set $check ""; +set $cookiecontent ""; +set $goodauth "auth_tkt=1; PATH=/"; +set $badauth "auth_tkt=0; PATH=/"; + +if ($uri = "/user/me") { + set $check "endpointcheck"; +} +if ($http_referer ~* \.(login|identitysandbox).gov/$ ) { + set $check "${check}+referercheck"; +} +if ($check = "endpointcheck+referercheck") { + set $cookiecontent $goodauth; +} +if ($uri = "/user/logged_out_redirect") { + set $cookiecontent $badauth; +} + +add_header Set-Cookie $cookiecontent; + +if ($uri = "/user/logged_out_redirect") { + return 302 https://{{env "PUBLIC_ROUTE"}}$uri; +} + +########################################## +# If cookie auth_tkt does not have a good value, +# we dont serve the request on the admin site. +# We 302 redirect it to the public site, +# except some special uri listed below: set $authy ""; -# Determine if auth cookie is set -if ($cookie_auth_tkt) { +# Determine if auth cookie is set with good value +if ($cookie_auth_tkt = 1) { set $authy C; } if ($uri = "/user/saml2login") { @@ -10,6 +44,9 @@ if ($uri = "/user/saml2login") { if ($uri = "/acs") { set $authy "${authy}S"; } +if ($uri = "/user/me") { + set $authy "${authy}M"; +} if ($uri = "/api/action/status_show") { set $authy "${authy}H"; } diff --git a/proxy/nginx.conf b/proxy/nginx.conf index e5e32483..1605179f 100644 --- a/proxy/nginx.conf +++ b/proxy/nginx.conf @@ -47,7 +47,7 @@ http { # catalog-admin server_name {{env "EXTERNAL_ROUTE_ADMIN"}}; - # include nginx-authy.conf; + include nginx-authy.conf; include nginx-common.conf; } } From f2639718266e26cf8e2b08267c5e6b72d722c49b Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 14:36:48 -0400 Subject: [PATCH 61/73] fix: facet list test the filters class doesn't exist on the org page for some reason.. so use a different class to specify tags --- e2e/cypress/integration/facets.cy.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/cypress/integration/facets.cy.js b/e2e/cypress/integration/facets.cy.js index 56148478..2d527447 100644 --- a/e2e/cypress/integration/facets.cy.js +++ b/e2e/cypress/integration/facets.cy.js @@ -21,10 +21,10 @@ describe('Facets', { testIsolation: false }, () => { cy.get('a[class="btn btn-primary"]').click(); cy.create_organization_ui('org-tags', 'tags for org test'); cy.visit('/organization/org-tags'); - cy.get('.filters h2').its('length').should('be.equal', 10); - cy.get('.filters h2').first().contains('Topics'); - cy.get('.filters h2').contains('Harvest Source'); - cy.get('.filters h2').last().contains('Bureaus'); + cy.get('.module-shallow').its('length').should('be.equal', 11); + cy.get('.module-shallow').contains('Topics'); + cy.get('.module-shallow').contains('Harvest Source'); + cy.get('.module-shallow').last().contains('Bureaus'); }); it('Show datagov facet list on group page', () => { From 656313e4759dde2b469bb562a9c9721f392dfa88 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 14:38:22 -0400 Subject: [PATCH 62/73] fix: ckan version test --- e2e/cypress/integration/ckan_extensions.cy.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/cypress/integration/ckan_extensions.cy.js b/e2e/cypress/integration/ckan_extensions.cy.js index 7cd0d61e..701c7a74 100644 --- a/e2e/cypress/integration/ckan_extensions.cy.js +++ b/e2e/cypress/integration/ckan_extensions.cy.js @@ -2,7 +2,7 @@ describe('CKAN Extensions', () => { it('Uses CKAN 2.9', () => { cy.request('/api/action/status_show').should((response) => { expect(response.body).to.have.property('success', true); - expect(response.body.result).to.have.property('ckan_version', '2.9.8'); + expect(response.body.result).to.have.property('ckan_version', '2.10.1'); }); }); From 95314d5a7708f52bf700b302ab9c627a6dbe7bd8 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 15:50:46 -0400 Subject: [PATCH 63/73] fix: cypress force selection for harvest source creation --- e2e/cypress/support/command.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/e2e/cypress/support/command.js b/e2e/cypress/support/command.js index 23a1c32d..3a258379 100644 --- a/e2e/cypress/support/command.js +++ b/e2e/cypress/support/command.js @@ -214,14 +214,14 @@ Cypress.Commands.add( }); cy.get('#field-notes').type(harvestDesc); - cy.get('[type="radio"]').check(harvestType); + cy.get('[type="radio"]').check(harvestType, { force: true }); // Validate private_datasets defaults to Private cy.get('#field-private_datasets').find(':selected').contains('Private'); - cy.get('#field-private_datasets').select(harvestPrivate); + cy.get('#field-private_datasets').select(harvestPrivate, { force: true }); - cy.get('input[name=save]').click(); + cy.get('input[name=save]').click({ force: true }); } ); From b73ee3d778b90e0b17af2936e8cd92fe21c042ee Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Thu, 29 Jun 2023 15:51:13 -0400 Subject: [PATCH 64/73] update: make sure fetch stays alive during harvesting --- ckan/docker-entrypoint.d/10-setup-harvest.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ckan/docker-entrypoint.d/10-setup-harvest.sh b/ckan/docker-entrypoint.d/10-setup-harvest.sh index afa48dfe..6331ad05 100644 --- a/ckan/docker-entrypoint.d/10-setup-harvest.sh +++ b/ckan/docker-entrypoint.d/10-setup-harvest.sh @@ -4,7 +4,12 @@ echo "Init Harvest database tables" ckan harvester initdb echo "turn on gather and fetch workers" -ckan harvester fetch-consumer & +run_fetch () { + until ckan harvester fetch-consumer; do + sleep 1 + done +} +run_fetch & ckan harvester fetch-consumer & ckan harvester gather-consumer & From 316c6eaef9f6c88648ba1efc7e74dd02fe0f6241 Mon Sep 17 00:00:00 2001 From: Fuhu Xia <1392461+FuhuXia@users.noreply.github.com> Date: Thu, 6 Jul 2023 12:32:52 -0400 Subject: [PATCH 65/73] correct requirements.txt --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 63a94852..bacd7e56 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@ckan-2.10#egg=ckanext-geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@56736006e2e50cac48bf88f39ee48e7553884a8c#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 8ec2305c27e1b13393fa4e8ac7de8df28922c429 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 10 Jul 2023 12:25:46 -0400 Subject: [PATCH 66/73] test: new spatial parsing algorithm --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index bacd7e56..83fbc79a 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@56736006e2e50cac48bf88f39ee48e7553884a8c#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@de28a3ec18306fa967c26480cca6fc85b34f7a0e#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 34c76c11c8329cc8a8ea04371cb3386d6e565d9f Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Mon, 10 Jul 2023 13:34:37 -0400 Subject: [PATCH 67/73] test: second spatial algorithm --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 83fbc79a..058490f1 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@de28a3ec18306fa967c26480cca6fc85b34f7a0e#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@82757941aa8c23d7e22eeb6385d15729e6bb114c#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 6ddb9a7bd3c1b348d6b38bde8cee9b8312d0a633 Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 11 Jul 2023 09:29:14 -0400 Subject: [PATCH 68/73] test: third spatial algorithm is the charm ;) --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 058490f1..37545843 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@82757941aa8c23d7e22eeb6385d15729e6bb114c#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@bbbc9fe21f4960f89c85f4b58be24dbcce3de931#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 50ec751d89516cbc3958f635ca624450eb6acadd Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 11 Jul 2023 10:14:46 -0400 Subject: [PATCH 69/73] new: maybe gather is also crashing? --- ckan/docker-entrypoint.d/10-setup-harvest.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ckan/docker-entrypoint.d/10-setup-harvest.sh b/ckan/docker-entrypoint.d/10-setup-harvest.sh index 6331ad05..d9cee2e9 100644 --- a/ckan/docker-entrypoint.d/10-setup-harvest.sh +++ b/ckan/docker-entrypoint.d/10-setup-harvest.sh @@ -9,9 +9,13 @@ run_fetch () { sleep 1 done } +run_gather () { + until ckan harvester gather-consumer; do + sleep 1 + done +} run_fetch & -ckan harvester fetch-consumer & -ckan harvester gather-consumer & +run_gather & echo "check harvest job completion every 10 secs" check_harvester () { From 7aa5ddf8b593e1f63f1a3d471981dfa10adbc38e Mon Sep 17 00:00:00 2001 From: Nicholas Kumia Date: Tue, 11 Jul 2023 11:47:03 -0400 Subject: [PATCH 70/73] test: fourth and final spatial algorithm for the win --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 37545843..840af468 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -18,7 +18,7 @@ ckan==2.10.1 -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 --e git+https://github.com/gsa/ckanext-geodatagov.git@bbbc9fe21f4960f89c85f4b58be24dbcce3de931#egg=ckanext_geodatagov +-e git+https://github.com/gsa/ckanext-geodatagov.git@cbbcf599b6273fca56ec60f7b5d8ea2301dae415#egg=ckanext_geodatagov -e git+https://github.com/gsa/ckanext-googleanalyticsbasic.git@0de6406608a3156ac368ba3ac14ca21be519a2dd#egg=ckanext_googleanalyticsbasic -e git+https://github.com/ckan/ckanext-harvest.git@9fb44f79809a1c04dfeb0e1ca2540c5ff3cacef4#egg=ckanext_harvest -e git+https://github.com/gsa/ckanext-metrics-dashboard.git@358ad6dbc4ec31ae4e0946cef6609c38920ad5d3#egg=ckanext_metrics_dashboard From 405e56fafad3f61f584d33e40b1eaef158706579 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Tue, 11 Jul 2023 15:33:04 -0400 Subject: [PATCH 71/73] ckanext-spatial v1.1.0 --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index 840af468..acdb7314 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -25,7 +25,7 @@ ckanext-envvars==0.0.3 -e git+https://github.com/ckan/ckanext-qa.git@1731b59d2bf82b06f7866c204b26eb7c6c9ea1f9#egg=ckanext_qa -e git+https://github.com/ckan/ckanext-report.git@3588577f46d17e5f6ef163bb984d0e7016daef71#egg=ckanext_report ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@c2b12a94430034c522b25d282323a064e2d6a03a --e git+https://github.com/ckan/ckanext-spatial.git@1205d3aa408e994439fc231ccb5d1cff06d644c6#egg=ckanext_spatial +-e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial ckantoolkit==0.0.7 click==8.1.3 cryptography==40.0.2 From 1ad8b9dcb8c2db51e23b25b7c08962998cb309c1 Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Thu, 13 Jul 2023 18:24:34 -0400 Subject: [PATCH 72/73] udpate datagovtheme --- ckan/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index acdb7314..a977b6d6 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -14,7 +14,7 @@ charset-normalizer==2.1.1 ckan==2.10.1 -e git+https://github.com/ckan/ckanext-archiver.git@cbfadf9fbf10405958fdef9f77a7faedc05aa20b#egg=ckanext_archiver -e git+https://github.com/gsa/ckanext-datagovcatalog.git@9db6fcf6a5b63743c573f567749a870df1976bd3#egg=ckanext_datagovcatalog --e git+https://github.com/gsa/ckanext-datagovtheme.git@7ec4c0be6a7803e5d1358b5e8f605db73f2e5c4c#egg=ckanext_datagovtheme +-e git+https://github.com/gsa/ckanext-datagovtheme.git@eb310589aa3007516070fc1cdbfdcf7e3dea116d#egg=ckanext_datagovtheme -e git+https://github.com/gsa/ckanext-datajson.git@4d6d09b79d8f932303056747706a31aacd195cf6#egg=ckanext_datajson ckanext-dcat @ git+https://github.com/ckan/ckanext-dcat@14396e6d03aad3118ee5a232e04052a61edd0893 ckanext-envvars==0.0.3 From b8828ac3cb430260f50eed527c0bb9010a92425b Mon Sep 17 00:00:00 2001 From: Fuhu Xia Date: Mon, 17 Jul 2023 09:51:58 -0400 Subject: [PATCH 73/73] update cryptography==41.0.0 requests==2.31.0 --- ckan/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckan/requirements.txt b/ckan/requirements.txt index a977b6d6..355f2a81 100644 --- a/ckan/requirements.txt +++ b/ckan/requirements.txt @@ -28,7 +28,7 @@ ckanext-saml2auth @ git+https://github.com/GSA/ckanext-saml2auth.git@c2b12a94430 -e git+https://github.com/ckan/ckanext-spatial.git@17d5a341cf8f40b35b25df91a18ce72c31195ba3#egg=ckanext_spatial ckantoolkit==0.0.7 click==8.1.3 -cryptography==40.0.2 +cryptography==41.0.0 defusedxml==0.7.1 dominate==2.7.0 elementpath==4.1.2 @@ -94,7 +94,7 @@ PyYAML==6.0 PyZ3950 @ git+https://github.com/danizen/PyZ3950@6d44a4ab85c8bda3a7542c2c9efdfad46c830219 rdflib==6.1.1 redis==4.5.5 -requests==2.28.1 +requests==2.31.0 rfc3987==1.3.8 rq==1.11.0 s3transfer==0.6.1