Merge branch 'master' of github.com:ckan/ckanext-dcat

CHANGELOG.md

@@ -1,6 +1,10 @@
 # Changelog
 
-## [Unreleased](https://github.com/ckan/ckanext-dcat/compare/v0.0.9...HEAD)
+## [Unreleased](https://github.com/ckan/ckanext-dcat/compare/v1.0.0...HEAD)
+
+* TBD
+
+## [v1.0.0](https://github.com/ckan/ckanext-dcat/compare/v0.0.9...v1.0.0)
 
 * Updating the URLs to dataportals.org (#145)
 * Handle import stage errors (#149)
@@ -9,6 +13,8 @@
 * Ignore auth in internal search call (#156)
 * Support URIRef for dct:language (#158)
 * Support JSON-LD catalogs with @graph (#159)
+* Make read keywords re-usable (#160)
+* Extract read datasets from db to make it re-usable (#161)
 
 ## [v0.0.9](https://github.com/ckan/ckanext-dcat/compare/v0.0.8...v0.0.9) - 2019-01-10
 

README.md

@@ -35,6 +35,7 @@ It also offers other features related to Semantic Data like exposing the necessa
 - [Translation of fields](#translation-of-fields)
 - [Structured Data and Google Dataset Search indexing](#structured-data-and-google-dataset-search-indexing)
 - [Running the Tests](#running-the-tests)
+- [Releases](#releases)
 - [Acknowledgements](#acknowledgements)
 - [Copying and License](#copying-and-license)
 
@@ -912,6 +913,15 @@ Example output of structured data in JSON-LD:
 To run the tests, do:
 
     nosetests --nologcapture --ckan --with-pylons=test.ini ckanext
+
+## Releases
+
+To create a new release, follow these steps:
+
+* Determine new release number based on the rules of [semantic versioning](http://semver.org)
+* Update the CHANGELOG, especially the link for the "Unreleased" section
+* Update the version number in `setup.py`
+* Create a new release on GitHub and add the CHANGELOG of this release as release notes
 
 ## Acknowledgements
 

bin/travis-build.bash

@@ -39,7 +39,12 @@ then
     pip install setuptools==39.0.1
 fi
 
-pip install -r requirements.txt
+if [ -f requirements-py2.txt ]
+then
+    pip install -r requirements-py2.txt
+else
+    pip install -r requirements.txt
+fi
 pip install -r dev-requirements.txt
 cd -
 

ckanext/dcat/harvesters/base.py

@@ -149,11 +149,9 @@ def get_original_url(self, harvest_object_id):
             return obj.source.url
         return None
 
-    def _get_existing_dataset(self, guid):
+    def _read_datasets_from_db(self, guid):
         '''
-        Checks if a dataset with a certain guid extra already exists
-
-        Returns a dict as the ones returned by package_show
+        Returns a database result of datasets matching the given guid.
         '''
 
         datasets = model.Session.query(model.Package.id) \
@@ -162,6 +160,16 @@ def _get_existing_dataset(self, guid):
                                 .filter(model.PackageExtra.value == guid) \
                                 .filter(model.Package.state == 'active') \
                                 .all()
+        return datasets
+
+    def _get_existing_dataset(self, guid):
+        '''
+        Checks if a dataset with a certain guid extra already exists
+
+        Returns a dict as the ones returned by package_show
+        '''
+
+        datasets = self._read_datasets_from_db(guid)
 
         if not datasets:
             return None

ckanext/dcat/profiles.py

@@ -144,6 +144,18 @@ def _distributions(self, dataset):
         for distribution in self.g.objects(dataset, DCAT.distribution):
             yield distribution
 
+    def _keywords(self, dataset_ref):
+        '''
+        Returns all DCAT keywords on a particular dataset
+        '''
+        keywords = self._object_value_list(dataset_ref, DCAT.keyword) or []
+        # Split keywords with commas
+        keywords_with_commas = [k for k in keywords if ',' in k]
+        for keyword in keywords_with_commas:
+            keywords.remove(keyword)
+            keywords.extend([k.strip() for k in keyword.split(',')])
+        return keywords
+
     def _object(self, subject, predicate):
         '''
         Helper for returning the first object for this subject and predicate
@@ -803,16 +815,9 @@ def parse_dataset(self, dataset_dict, dataset_ref):
                 dataset_dict['version'] = value
 
         # Tags
-        keywords = self._object_value_list(dataset_ref, DCAT.keyword) or []
-        # Split keywords with commas
-        keywords_with_commas = [k for k in keywords if ',' in k]
-        for keyword in keywords_with_commas:
-            keywords.remove(keyword)
-            keywords.extend([k.strip() for k in keyword.split(',')])
-
         # replace munge_tag to noop if there's no need to clean tags
         do_clean = toolkit.asbool(config.get(DCAT_CLEAN_TAGS, False))
-        tags_val = [munge_tag(tag) if do_clean else tag for tag in keywords]
+        tags_val = [munge_tag(tag) if do_clean else tag for tag in self._keywords(dataset_ref)]
         tags = [{'name': tag} for tag in tags_val]
         dataset_dict['tags'] = tags
 

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.0.9'
+version = '1.0.0'
 
 setup(
     name='ckanext-dcat',