Clean up six from the codebase

ckanext/harvest/harvesters/ckanharvester.py

@@ -1,11 +1,10 @@
 from __future__ import absolute_import
-import six
 import requests
 from requests.exceptions import HTTPError, RequestException
 
 import datetime
 
-from six.moves.urllib.parse import urlencode
+from urllib.parse import urlencode
 from ckan import model
 from ckan.logic import ValidationError, NotFound, get_action
 from ckan.lib.helpers import json
@@ -119,8 +118,7 @@ def validate_config(self, config):
                     raise ValueError('default_groups must be a *list* of group'
                                      ' names/ids')
                 if config_obj['default_groups'] and \
-                        not isinstance(config_obj['default_groups'][0],
-                                       six.string_types):
+                        not isinstance(config_obj['default_groups'][0], str):
                     raise ValueError('default_groups must be a list of group '
                                      'names/ids (i.e. strings)')
 
@@ -520,7 +518,7 @@ def get_extra(key, package_dict):
                     if existing_extra:
                         package_dict['extras'].remove(existing_extra)
                     # Look for replacement strings
-                    if isinstance(value, six.string_types):
+                    if isinstance(value, str):
                         value = value.format(
                             harvest_source_id=harvest_object.job.source.id,
                             harvest_source_url=harvest_object.job.source.url.strip('/'),

ckanext/harvest/logic/action/update.py

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
 
 import hashlib
+import html
 import json
-import six
 
 import logging
 import datetime
 
 from ckantoolkit import config
 from sqlalchemy import and_, or_
-from six.moves.urllib.parse import urljoin
+from urllib.parse import urljoin
 
 from ckan.lib.search.index import PackageSearchIndex
 from ckan.plugins import toolkit, PluginImplementations
@@ -549,8 +549,10 @@ def harvest_objects_import(context, data_dict):
     last_objects_count = 0
 
     for obj_id in last_objects_ids:
-        if segments and \
-                str(hashlib.md5(six.ensure_binary(obj_id[0])).hexdigest())[0] not in segments:
+        _id = obj_id[0]
+        if isinstance(_id, str):
+            _id = _id.encode()
+        if segments and str(hashlib.md5(_id).hexdigest())[0] not in segments:
             continue
 
         obj = session.query(HarvestObject).get(obj_id)
@@ -798,12 +800,7 @@ def prepare_summary_mail(context, source_id, status):
 def prepare_error_mail(context, source_id, status):
     extra_vars = get_mail_extra_vars(context, source_id, status)
     body = render('emails/error_email.txt', extra_vars)
-    if six.PY34:
-        import html
-        body = html.unescape(body)
-    elif six.PY2:
-        import HTMLParser
-        body = HTMLParser.HTMLParser().unescape(body)
+    body = html.unescape(body)
 
     subject = '{} - Harvesting Job - Error Notification'\
         .format(config.get('ckan.site_title'))

ckanext/harvest/logic/validators.py

@@ -12,11 +12,7 @@
 )
 from ckanext.harvest.model import HarvestSource, UPDATE_FREQUENCIES, HarvestJob
 from ckanext.harvest.interfaces import IHarvester
-
-import six
-from six.moves.urllib.parse import (
-    urlparse, urlunparse
-)
+from urllib.parse import (urlparse, urlunparse)
 
 log = logging.getLogger(__name__)
 
@@ -233,7 +229,7 @@ def harvest_source_convert_from_config(key, data, errors, context):
 
 
 def harvest_source_active_validator(value, context):
-    if isinstance(value, six.string_types):
+    if isinstance(value, str):
         if value.lower() == 'true':
             return True
         else:
@@ -259,6 +255,6 @@ def harvest_object_extras_validator(value, context):
     if not isinstance(value, dict):
         raise Invalid('extras must be a dict')
     for v in value.values():
-        if not isinstance(v, six.string_types):
+        if not isinstance(v, str):
             raise Invalid('extras must be a dict of strings')
     return value

ckanext/harvest/plugin.py

@@ -2,10 +2,8 @@
 
 import os
 import json
-import six
 from logging import getLogger
 
-from six import string_types
 from collections import OrderedDict
 
 from ckan import logic
@@ -113,7 +111,7 @@ def before_dataset_search(self, search_params):
 
         fq = search_params.get("fq", "")
         if "dataset_type:harvest" not in fq:
-            fq = "{0} -dataset_type:harvest".format(fq.encode('utf8') if six.PY2 else fq)
+            fq = "{0} -dataset_type:harvest".format(fq)
             search_params.update({"fq": fq})
 
         return search_params

ckanext/harvest/utils.py

@@ -10,22 +10,12 @@
 
 import ckan.lib.helpers as h
 import ckan.plugins.toolkit as tk
-import six
 from ckan import model
 from ckantoolkit import _
-from six import StringIO
+from io import StringIO
 
 from ckanext.harvest.logic import HarvestJobExists, HarvestSourceInactiveError
 
-try:
-    # Python 2.7
-    xml_parser_exception = etree.ParseError
-except AttributeError:
-    # Python 2.6
-    from xml.parsers import expat
-
-    xml_parser_exception = expat.ExpatError
-
 log = logging.getLogger(__name__)
 
 
@@ -394,8 +384,7 @@ def run_test_harvester(source_id_or_name, force_import):
         if running_jobs:
             print('\nSource "{0}" apparently has a "Running" job:\n{1}'.format(
                 source.get("name") or source["id"], running_jobs))
-
-            resp = six.moves.input("Abort it? (y/n)")
+            resp = input("Abort it? (y/n)")
             if not resp.lower().startswith("y"):
                 sys.exit(1)
             job_dict = tk.get_action("harvest_job_abort")(
@@ -748,7 +737,7 @@ def object_show_view(id, ref_type, response):
             if '<?xml' not in content.split('\n')[0]:
                 content = u'<?xml version="1.0" encoding="UTF-8"?>\n' + content
 
-        except xml_parser_exception:
+        except etree.ParseError:
             try:
                 json.loads(obj['content'])
                 response.content_type = 'application/json; charset=utf-8'
@@ -757,7 +746,11 @@ def object_show_view(id, ref_type, response):
                 pass
 
         response.headers['Content-Length'] = len(content)
-        return (response, six.ensure_str(content))
+
+        if isinstance(content, bytes):
+            content = content.decode("utf-8")
+
+        return (response, content)
 
     except tk.ObjectNotFound as e:
         return tk.abort(404, _(str(e)))

requirements.txt

@@ -1,6 +1,4 @@
 ckantoolkit>=0.0.7
 pika>=1.1.0,<1.3.0
-enum34; python_version < '3.0'  # Required by pika
 redis
 requests>=2.11.1
-six>=1.12.0