Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Adapt tests to latest changes in ckanext-harvest #263

Merged
merged 1 commit into from
Feb 27, 2024
Merged

Adapt tests to latest changes in ckanext-harvest #263

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ jobs:
- name: Setup extension
run: |
ckan -c test.ini db init
ckan -c test.ini harvester initdb
ckan -c test.ini db pending-migrations --apply
- name: Run tests
run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=xml --cov-append --disable-warnings ckanext/dcat/tests
- name: Upload coverage report to codecov
Expand Down
9 changes: 9 additions & 0 deletions ckanext/dcat/tests/conftest.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
import pytest

import ckan.plugins as p

@pytest.fixture
def clean_db(reset_db, migrate_db_for):
reset_db()
if p.get_plugin('harvest'):
migrate_db_for('harvest')
12 changes: 1 addition & 11 deletions ckanext/dcat/tests/test_harvester.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,6 @@




# TODO move to ckanext-harvest
@pytest.fixture
def harvest_setup():
harvest_model.setup()
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Perhaps is best to keep the concerns for the clean_db and harvest_setup fixtures separated and keep harvest_setup but refactoring it to use migrate_db_for('harvest') rather than harvest_model.setup()?

Copy link
Member Author

@seitenbau-govdata seitenbau-govdata Feb 7, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could not solve the problem otherwise. Using migrate_db_for('harvest') in harvest_setup or redefining clean_db in the harvester-specific tests (test_harvester.py & test_json_harvester.py) does not seem to solve the issue.
All Tests that include clean_db still throw the following exception:

sqlalchemy.exc.ProgrammingError: (psycopg2.errors.UndefinedTable) relation "harvest_object_extra" does not exist
...
/srv/app/src/ckan/ckan/tests/helpers.py:83: in reset_db
model.repo.rebuild_db()
/srv/app/src/ckan/ckan/model/init.py:229: in rebuild_db
self.delete_all()
/srv/app/src/ckan/ckan/model/init.py:250: in delete_all
connection.execute('delete from "%s"' % table.name)
/usr/lib/python3.8/site-packages/sqlalchemy/engine/base.py:982: in execute
return self.execute_text(object, multiparams, params)
/usr/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1149: in _execute_text
ret = self._execute_context(
/usr/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1247: in _execute_context
self._handle_dbapi_exception(
/usr/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1466: in _handle_dbapi_exception
util.raise_from_cause(sqlalchemy_exception, exc_info)
/usr/lib/python3.8/site-packages/sqlalchemy/util/compat.py:399: in raise_from_cause
reraise(type(exception), exception, tb=exc_tb, cause=cause)
/usr/lib/python3.8/site-packages/sqlalchemy/util/compat.py:153: in reraise
raise value.with_traceback(tb)
/usr/lib/python3.8/site-packages/sqlalchemy/engine/base.py:1243: in _execute_context

Any ideas on how to initialize/create the plugin db tables or to put the right scope for harvest_setup ?

Maybe this was also mentioned in this discussion



@pytest.fixture
def clean_queues():
queue.purge_queues()
Expand Down Expand Up @@ -624,7 +617,7 @@ def _run_full_job(self, harvest_source_id, num_jobs=1, num_objects=1):
self._fetch_queue(num_objects)


@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'clean_queues')
@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'clean_queues')
@pytest.mark.ckan_config('ckan.plugins', 'dcat harvest dcat_rdf_harvester')
class TestDCATHarvestFunctional(FunctionalHarvestTest):

Expand Down Expand Up @@ -1118,7 +1111,6 @@ def test_harvest_create_duplicate_titles(self):
'with_plugins',
'clean_db',
'clean_index',
'harvest_setup',
'clean_queues',
)
@pytest.mark.ckan_config('ckan.plugins', 'dcat harvest dcat_rdf_harvester test_rdf_harvester')
Expand Down Expand Up @@ -1547,7 +1539,6 @@ def test_harvest_import_extensions_point_gets_called(self, reset_calls_counter):
'with_plugins',
'clean_db',
'clean_index',
'harvest_setup',
'clean_queues',
)
@pytest.mark.ckan_config('ckan.plugins', 'dcat harvest dcat_rdf_harvester test_rdf_null_harvester')
Expand Down Expand Up @@ -1608,7 +1599,6 @@ def test_harvest_with_before_create_null(self, reset_calls_counter):
'with_plugins',
'clean_db',
'clean_index',
'harvest_setup',
'clean_queues',
)
@pytest.mark.ckan_config('ckan.plugins', 'dcat harvest dcat_rdf_harvester test_rdf_exception_harvester')
Expand Down
6 changes: 3 additions & 3 deletions ckanext/dcat/tests/test_json_harvester.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,10 @@

from ckanext.dcat.harvesters._json import copy_across_resource_ids, DCATJSONHarvester

from .test_harvester import FunctionalHarvestTest, harvest_setup, clean_queues
from .test_harvester import FunctionalHarvestTest, clean_queues


@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'harvest_setup', 'clean_queues')
@pytest.mark.usefixtures('with_plugins', 'clean_db', 'clean_index', 'clean_queues')
@pytest.mark.ckan_config('ckan.plugins', 'dcat harvest dcat_json_harvester')
class TestDCATJSONHarvestFunctional(FunctionalHarvestTest):

Expand Down Expand Up @@ -284,7 +284,7 @@ def test_not_copied_because_completely_different(self):
)
assert harvested_dataset['resources'][0].get('id') == None

@pytest.mark.usefixtures('clean_db', 'clean_index', 'harvest_setup', 'clean_queues')
@pytest.mark.usefixtures('clean_db', 'clean_index', 'clean_queues')
class TestImportStage(object):

class MockHarvestObject(object):
Expand Down
Loading