Added extracting and downloading functionality in several file formats

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+omit =
+    */site-packages/*
+    */python?.?/*
+    ckan/*

.gitignore

@@ -0,0 +1,42 @@
+.ropeproject
+node_modules
+bower_components
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+env/
+build/
+develop-eggs/
+dist/
+sdist/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Sphinx documentation
+docs/_build/

.travis.yml

@@ -0,0 +1,11 @@
+language: python
+sudo: required
+python:
+    - "2.7"
+env: PGVERSION=9.1
+install:
+    - bash bin/travis-build.bash
+    - pip install coveralls
+script: sh bin/travis-run.sh
+after_success:
+    - coveralls

MANIFEST.in

@@ -0,0 +1,4 @@
+include README.rst
+include LICENSE
+include requirements.txt
+recursive-include ckanext/dataexplorer *.html *.json *.js *.less *.css *.mo *.config

README.rst

@@ -0,0 +1,166 @@
+.. You should enable this project on travis-ci.org and coveralls.io to make
+   these badges work. The necessary Travis and Coverage config files have been
+   generated for you.
+
+.. image:: https://travis-ci.org/duskobogdanovski/ckanext-dataexplorer.svg?branch=master
+    :target: https://travis-ci.org/duskobogdanovski/ckanext-dataexplorer
+
+.. image:: https://coveralls.io/repos/duskobogdanovski/ckanext-dataexplorer/badge.svg
+  :target: https://coveralls.io/r/duskobogdanovski/ckanext-dataexplorer
+
+.. image:: https://pypip.in/download/ckanext-dataexplorer/badge.svg
+    :target: https://pypi.python.org/pypi//ckanext-dataexplorer/
+    :alt: Downloads
+
+.. image:: https://pypip.in/version/ckanext-dataexplorer/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-dataexplorer/
+    :alt: Latest Version
+
+.. image:: https://pypip.in/py_versions/ckanext-dataexplorer/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-dataexplorer/
+    :alt: Supported Python versions
+
+.. image:: https://pypip.in/status/ckanext-dataexplorer/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-dataexplorer/
+    :alt: Development Status
+
+.. image:: https://pypip.in/license/ckanext-dataexplorer/badge.svg
+    :target: https://pypi.python.org/pypi/ckanext-dataexplorer/
+    :alt: License
+
+=============
+ckanext-dataexplorer
+=============
+
+.. Put a description of your extension here:
+   What does it do? What features does it have?
+   Consider including some screenshots or embedding a video!
+
+
+------------
+Requirements
+------------
+
+For example, you might want to mention here which versions of CKAN this
+extension works with.
+
+
+------------
+Installation
+------------
+
+.. Add any additional install steps to the list below.
+   For example installing any non-Python dependencies or adding any required
+   config settings.
+
+To install ckanext-dataexplorer:
+
+1. Activate your CKAN virtual environment, for example::
+
+     . /usr/lib/ckan/default/bin/activate
+
+2. Install the ckanext-dataexplorer Python package into your virtual environment::
+
+     pip install ckanext-dataexplorer
+
+3. Add ``dataexplorer`` to the ``ckan.plugins`` setting in your CKAN
+   config file (by default the config file is located at
+   ``/etc/ckan/default/production.ini``).
+
+4. Restart CKAN. For example if you've deployed CKAN with Apache on Ubuntu::
+
+     sudo service apache2 reload
+
+
+---------------
+Config Settings
+---------------
+
+Document any optional config settings here. For example::
+
+    # The minimum number of hours to wait before re-checking a resource
+    # (optional, default: 24).
+    ckanext.dataexplorer.some_setting = some_default_value
+
+
+------------------------
+Development Installation
+------------------------
+
+To install ckanext-dataexplorer for development, activate your CKAN virtualenv and
+do::
+
+    git clone https://github.com/duskobogdanovski/ckanext-dataexplorer.git
+    cd ckanext-dataexplorer
+    python setup.py develop
+    pip install -r dev-requirements.txt
+
+
+-----------------
+Running the Tests
+-----------------
+
+To run the tests, do::
+
+    nosetests --nologcapture --with-pylons=test.ini
+
+To run the tests and produce a coverage report, first make sure you have
+coverage installed in your virtualenv (``pip install coverage``) then run::
+
+    nosetests --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.dataexplorer --cover-inclusive --cover-erase --cover-tests
+
+
+---------------------------------
+Registering ckanext-dataexplorer on PyPI
+---------------------------------
+
+ckanext-dataexplorer should be availabe on PyPI as
+https://pypi.python.org/pypi/ckanext-dataexplorer. If that link doesn't work, then
+you can register the project on PyPI for the first time by following these
+steps:
+
+1. Create a source distribution of the project::
+
+     python setup.py sdist
+
+2. Register the project::
+
+     python setup.py register
+
+3. Upload the source distribution to PyPI::
+
+     python setup.py sdist upload
+
+4. Tag the first release of the project on GitHub with the version number from
+   the ``setup.py`` file. For example if the version number in ``setup.py`` is
+   0.0.1 then do::
+
+       git tag 0.0.1
+       git push --tags
+
+
+----------------------------------------
+Releasing a New Version of ckanext-dataexplorer
+----------------------------------------
+
+ckanext-dataexplorer is availabe on PyPI as https://pypi.python.org/pypi/ckanext-dataexplorer.
+To publish a new version to PyPI follow these steps:
+
+1. Update the version number in the ``setup.py`` file.
+   See `PEP 440 <http://legacy.python.org/dev/peps/pep-0440/#public-version-identifiers>`_
+   for how to choose version numbers.
+
+2. Create a source distribution of the new version::
+
+     python setup.py sdist
+
+3. Upload the source distribution to PyPI::
+
+     python setup.py sdist upload
+
+4. Tag the new release of the project on GitHub with the version number from
+   the ``setup.py`` file. For example if the version number in ``setup.py`` is
+   0.0.2 then do::
+
+       git tag 0.0.2
+       git push --tags

bin/travis-build.bash

@@ -0,0 +1,43 @@
+#!/bin/bash
+set -e
+
+echo "This is travis-build.bash..."
+
+echo "Installing the packages that CKAN requires..."
+sudo apt-get update -qq
+sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1
+
+echo "Installing CKAN and its Python dependencies..."
+git clone https://github.com/ckan/ckan
+cd ckan
+export latest_ckan_release_branch=`git branch --all | grep remotes/origin/release-v | sort -r | sed 's/remotes\/origin\///g' | head -n 1`
+echo "CKAN branch: $latest_ckan_release_branch"
+git checkout $latest_ckan_release_branch
+python setup.py develop
+pip install -r requirements.txt --allow-all-external
+pip install -r dev-requirements.txt --allow-all-external
+cd -
+
+echo "Creating the PostgreSQL user and database..."
+sudo -u postgres psql -c "CREATE USER ckan_default WITH PASSWORD 'pass';"
+sudo -u postgres psql -c 'CREATE DATABASE ckan_test WITH OWNER ckan_default;'
+
+echo "SOLR config..."
+# Solr is multicore for tests on ckan master, but it's easier to run tests on
+# Travis single-core. See https://github.com/ckan/ckan/issues/2972
+sed -i -e 's/solr_url.*/solr_url = http:\/\/127.0.0.1:8983\/solr/' ckan/test-core.ini
+
+echo "Initialising the database..."
+cd ckan
+paster db init -c test-core.ini
+cd -
+
+echo "Installing ckanext-dataexplorer and its requirements..."
+python setup.py develop
+pip install -r dev-requirements.txt
+
+echo "Moving test.ini into a subdir..."
+mkdir subdir
+mv test.ini subdir
+
+echo "travis-build.bash is done."

bin/travis-run.sh

@@ -0,0 +1,6 @@
+#!/bin/sh -e
+
+echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
+sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
+sudo service jetty restart
+nosetests --nologcapture --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.dataexplorer --cover-inclusive --cover-erase --cover-tests

ckanext/__init__.py

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+# this is a namespace package
+try:
+    import pkg_resources
+    pkg_resources.declare_namespace(__name__)
+except ImportError:
+    import pkgutil
+    __path__ = pkgutil.extend_path(__path__, __name__)

ckanext/dataexplorer/controllers/dataexplorer.py

@@ -0,0 +1,68 @@
+import logging
+import json
+
+try:
+    # CKAN 2.7 and later
+    from ckan.common import config
+except ImportError:
+    # CKAN 2.6 and earlier
+    from pylons import config
+
+from ckan.lib import base
+from ckan.plugins import toolkit
+from ckan.plugins.toolkit import abort, ObjectNotFound, ValidationError
+from ckan import model, logic
+from ckan.common import c, _, request, response
+from ckanext.dataexplorer.lib import FileWriterService
+
+log = logging.getLogger(__name__)
+
+DUMP_FORMATS = 'csv', 'xlsx', 'json', 'xml'
+
+class DataExplorer(base.BaseController):
+
+    ctrl = 'ckanext.dataexplorer.controllers.dataexplorer:DataExplorer'
+
+    def _get_ctx(self):
+        return {
+            'model': model, 'session': model.Session,
+            'user': c.user,
+            'auth_user_obj': c.userobj,
+            'for_view': True
+        }
+
+    def _get_action(self, action, data_dict):
+        return toolkit.get_action(action)(self._get_ctx(), data_dict)
+
+    def extract(self):
+
+        writer = FileWriterService()
+        columns = []
+
+        if request.method == 'POST':
+            data_dict = dict(request.POST)
+            data = json.loads(data_dict['extract_data'])
+            format = data.pop('format')
+
+            resource_data_info = self._get_action('datastore_info', {'id': data['resource_id']})
+            print resource_data_info
+
+            for key in resource_data_info['schema']:
+                columns.append(key)
+
+            try:
+                resource_data = self._get_action('datastore_search', data)
+            except ObjectNotFound:
+                abort(404, _('DataStore resource not found'))
+
+            try:
+                writer.write_to_file(columns,
+                                     resource_data.get('records'),
+                                     format,
+                                     response)
+            except ValidationError:
+                abort(400, _(
+                    u'Format: must be one of %s') % u', '.join(DUMP_FORMATS))
+
+
+