From 9875e33da295a8207e0d98da60ae1b11e0dfd722 Mon Sep 17 00:00:00 2001 From: Michael Polidori Date: Thu, 5 Dec 2019 07:45:55 -0500 Subject: [PATCH] [feature][m]: add full publisher report functionality (#11) * [feature][m]: add full publisher report functionality * [fix][m]: review fixes * [fix][xs]: add comment for clarity on report CSV file naming --- ckanext/opendatani/controller.py | 73 +++++++++++++++++-- ckanext/opendatani/helpers.py | 73 +++---------------- ckanext/opendatani/plugin.py | 17 +++-- .../opendatani/templates/package/search.html | 13 ++++ .../templates/snippets/organization.html | 6 +- 5 files changed, 106 insertions(+), 76 deletions(-) create mode 100644 ckanext/opendatani/templates/package/search.html diff --git a/ckanext/opendatani/controller.py b/ckanext/opendatani/controller.py index 2969cdc..9084ccf 100644 --- a/ckanext/opendatani/controller.py +++ b/ckanext/opendatani/controller.py @@ -1,18 +1,22 @@ +import logging +import datetime as dt +import requests +import os +import csv +import cStringIO as StringIO + import ckan.lib.base as base from ckan import model import ckan.lib.helpers as h import ckan.lib.mailer as mailer from ckan.common import c, request, _ from ckan.common import config - import ckan.plugins.toolkit as toolkit from ckan.controllers.user import UserController as CoreUserController from ckan.controllers.package import PackageController as CorePackageController - -import logging import ckan.logic as logic -import datetime as dt -import requests +from ckanext.opendatani import helpers + log = logging.getLogger(__name__) @@ -249,3 +253,62 @@ def resource_read(self, id, resource_id): h.flash_error('Sorry, this file is too large to be able to display in the browser, please download the data resource to examine it further.') template = self._resource_template(dataset_type) return render(template, extra_vars=vars) + + +class CustomReportController(CorePackageController): + def prepare_report(self, org): + """ + Creates a CSV publisher report and returns it as a string + :param org: organization + :type org: string + :return: a CSV string + :rtype: string + """ + + try: + data_dict = {'org_name': org} + + if org == '@complete': + data_dict = {} + # Use 'complete' in the file name for full reports + org = org[1:] + else: + # Use 'org-' in the file name for per org reports + org = 'org-' + org + + resource = toolkit.get_action( + 'report_resources_by_organization')({}, data_dict) + + # We need this in case no datasets exist for the given organization + # or the organization doesn't exist + if not resource: + toolkit.abort(404, _('Either the organization does not exist, \ + or it has no datasets.')) + + csvout = StringIO.StringIO() + csvwriter = csv.writer( + csvout, + dialect='excel', + quoting=csv.QUOTE_NONNUMERIC + ) + + fields = resource[0].keys() + csvwriter.writerow(fields) + + for data in resource: + csvwriter.writerow(data.values()) + + csvout.seek(0) + filename = 'publisher-report-{0}-{1}.csv'.format(org, + dt.date.today()) + toolkit.response.headers['Content-Type'] = 'application/csv' + toolkit.response.headers['Content-Disposition'] = \ + 'attachment; filename={0}'.format(filename) + + return csvout.read() + + except Exception as ex: + error = 'Preparing the CSV report failed. Error: {0}'.format(ex) + log.error(error) + h.flash_error(error) + raise diff --git a/ckanext/opendatani/helpers.py b/ckanext/opendatani/helpers.py index 520cedd..ce7ba2a 100644 --- a/ckanext/opendatani/helpers.py +++ b/ckanext/opendatani/helpers.py @@ -1,19 +1,15 @@ import re from six import string_types, text_type +import logging import ckan.model as model from ckan.lib import activity_streams import ckan.logic as logic import ckan.lib.helpers as h - -import logging from ckan.plugins import toolkit -from ckan.common import config -import csv -import json -import os import ckan.authz as authz + log = logging.getLogger(__name__) @@ -124,8 +120,8 @@ def _get_action(action, context_dict, data_dict): def is_admin(user, org): """ - Returns True if user is site admin or admin of the organization, - and the given organization exists. + Returns True if user is site admin or admin of the + organization and the given organization exists :param user: user name :type user: string :param org: organization name @@ -140,70 +136,23 @@ def is_admin(user, org): return any( [(i.get('capacity') == 'admin') - and i.get('name') == org for i in user_orgs])\ + and i.get('name') == org for i in user_orgs]) \ or authz.is_sysadmin(user) def verify_datasets_exist(org): """ - Returns True if the number of datasets (including private) for a given - organization is greater than 0. + Returns True if the number of datasets (including private) + for a given organization is greater than 0. :param org: organization name :type org: string :returns: dataset count :rtype: integer """ - return toolkit.get_action('package_search')({}, { - 'q': 'organization:{0}'.format(org), - 'include_private': True}).get('count') > 0 - - -def prepare_reports(org): - """ - Creates a CSV and JSON publisher report, and stores them under CKAN's - storage path in /storage/publisher-reports/. - :param org: organization - :type org: string - :return: a list containing the file_names of the created archives - :rtype: list - """ - - resource = toolkit.get_action( - 'report_resources_by_organization')({}, {'org_name': org}) - file_names = [] - storage_path = config.get('ckan.storage_path') - file_path = storage_path + '/storage/publisher-reports/' - - if not os.path.exists(file_path): - os.makedirs(file_path) - - for file_type in ['.csv', '.json']: - try: - file_name = 'publisher-report-' + org + file_type - - if file_type == '.csv': - with open(file_path + file_name, 'w') as csvfile: - fields = resource[0].keys() - writer = csv.DictWriter(csvfile, fieldnames=fields, - quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - - for data in resource: - writer.writerow(data) - - file_names.append(file_name) - - if file_type == '.json': - with open(file_path + file_name, 'w') as jsonfile: - jsonfile.writelines(json.dumps(resource)) - - file_names.append(file_name) + data_dict = {'include_private': True} - except Exception as ex: - log.error( - 'An error occured while preparing the {0} archive. Error: {1}' - .format(file_type, ex)) - raise + if org: + data_dict['q'] = 'organization:{0}'.format(org) - return file_names + return toolkit.get_action('package_search')({}, data_dict).get('count') > 0 diff --git a/ckanext/opendatani/plugin.py b/ckanext/opendatani/plugin.py index cc07be2..8935bc6 100644 --- a/ckanext/opendatani/plugin.py +++ b/ckanext/opendatani/plugin.py @@ -1,6 +1,7 @@ import datetime from pylons import config import routes.mapper +import logging import ckan.plugins as plugins import ckan.plugins.toolkit as toolkit @@ -12,9 +13,8 @@ import datetime as dt from ckanext.opendatani.controller import CustomUserController from ckanext.opendatani import helpers - from ckan.common import OrderedDict -import logging + log = logging.getLogger(__name__) @@ -66,8 +66,7 @@ def get_helpers(self): 'package_list': package_list, 'ni_activity_list_to_text': helpers.activity_list_to_text, 'verify_datasets_exist': helpers.verify_datasets_exist, - 'is_admin': helpers.is_admin, - 'prepare_reports': helpers.prepare_reports, + 'is_admin': helpers.is_admin } # IRoutes @@ -100,6 +99,11 @@ def before_map(self, map): m.connect('/dataset/{id}/resource/{resource_id}', action='resource_read') + controller = 'ckanext.opendatani.controller:CustomReportController' + with routes.mapper.SubMapper(map, controller=controller) as m: + m.connect('/publisher-reports/publisher-report-{org}.csv', + action='prepare_report') + return map def after_map(self, map): @@ -182,7 +186,10 @@ def report_resources_by_organization(context, data_dict): report or the organization does not exist.')) data_dict['include_private'] = True - data_dict['q'] = 'organization:{0}'.format(org) + + if org: + data_dict['q'] = 'organization:{0}'.format(org) + results = toolkit.get_action('package_search')({}, data_dict) for item in results['results']: diff --git a/ckanext/opendatani/templates/package/search.html b/ckanext/opendatani/templates/package/search.html new file mode 100644 index 0000000..0b4bf8f --- /dev/null +++ b/ckanext/opendatani/templates/package/search.html @@ -0,0 +1,13 @@ +{% ckan_extends %} + +{% block secondary_content %} + {% block report %} + {% if request.path[-7:] == 'dataset' and h.is_admin(c.user, org) and h.verify_datasets_exist(org) %} +
+
{{ _('Full Publisher Report') }}
+ CSV +
+ {% endif %} + {% endblock %} +{{ super() }} +{% endblock %} diff --git a/ckanext/opendatani/templates/snippets/organization.html b/ckanext/opendatani/templates/snippets/organization.html index 5225748..5a30992 100644 --- a/ckanext/opendatani/templates/snippets/organization.html +++ b/ckanext/opendatani/templates/snippets/organization.html @@ -5,11 +5,9 @@ {% block report %} {% set org = request.path.replace('/organization/', '') %} {% if 'organization' in request.path and h.is_admin(c.user, org) and h.verify_datasets_exist(org) %} - {% set csv, json = h.prepare_reports(org) %}
-

{{ _('Publisher Report') }}

- CSV - JSON +
{{ _('Publisher Report') }}
+ CSV
{% endif %} {% endblock %}