Merge pull request #20 from AspenWeb/python3

Add support for python3

.gitignore

@@ -1,6 +1,7 @@
 *.egg-info/
 aspen/tests/log
 env
+.tox/
 *.pyc
 distribute-*
 __pycache__

.travis.yml

@@ -4,12 +4,12 @@ branches:
     - master
 language: python
 python:
-  - 2.7
+  - 3.5
 before_install:
   - pip install --upgrade pip
   - pip --version
 install: python build.py dev
-script: python build.py analyse
+script: python build.py test_cov
 notifications:
   email: false
   irc: 

appveyor.yml

@@ -23,7 +23,7 @@ build: false  # Not a C# project, build stuff at the test step instead.
 test_script:
 
   # Build the compiled extension and run the project tests
-  - "python build.py analyse"
+  - "python build.py test_cov"
 
 branches:
   only:

aspen/configuration/__init__.py

@@ -6,7 +6,6 @@
 import os
 
 from ..exceptions import ConfigurationError
-from ..utils import ascii_dammit
 
 
 def configure(knobs, d, env_prefix, kwargs):
@@ -43,11 +42,11 @@ def parse_conf_var(raw, from_unicode, context, name_in_context):
         value = raw
         extend = False
     try:
-        if isinstance(value, str):
+        if isinstance(value, bytes):
             value = value.decode('US-ASCII')
         return from_unicode(value), extend
     except UnicodeDecodeError as error:
-        value = ascii_dammit(value)
+        value = value.decode('US-ASCII', 'repr')
         error_detail = "Configuration values must be US-ASCII."
     except ValueError as error:
         error_detail = error.args[0]

aspen/configuration/parse.py

@@ -20,17 +20,17 @@ def identity(value):
 
 def media_type(media_type):
     # XXX for now. Read a spec
-    return media_type.encode('US-ASCII')
+    return media_type.encode('ascii').decode('ascii')
 
 def codec(value):
     codecs.lookup(value)
     return value
 
 def yes_no(s):
     s = s.lower()
-    if s in [u'yes', u'true', u'1']:
+    if s in ['yes', 'true', '1']:
         return True
-    if s in [u'no', u'false', u'0']:
+    if s in ['no', 'false', '0']:
         return False
     raise ValueError("must be either yes/true/1 or no/false/0")
 
@@ -47,4 +47,4 @@ def renderer(value):
     if value not in RENDERERS:
         msg = "not one of {%s}" % (','.join(RENDERERS))
         raise ValueError(msg)
-    return value.encode('US-ASCII')
+    return value

aspen/http/request.py

@@ -5,18 +5,21 @@
 from __future__ import print_function
 from __future__ import unicode_literals
 
-
-import cgi
-import urllib
+from six import PY2, text_type as str
+from six.moves.urllib.parse import parse_qs, unquote, unquote_plus
 
 from .mapping import Mapping
 
 
+def _decode(o):
+    return o.decode('utf8') if isinstance(o, bytes) else o
+
+
 def path_decode(bs):
-    return urllib.unquote(bs).decode('UTF-8')
+    return _decode(unquote(bs.encode('ascii') if PY2 else bs))
 
 
-class PathPart(unicode):
+class PathPart(str):
     """A string with a mapping for extra data about it."""
 
     __slots__ = ['params']
@@ -40,17 +43,17 @@ def extract_rfc2396_params(path):
     * path should be raw so we don't split or operate on a decoded character
     * output is decoded
     """
-    pathsegs = path.lstrip(b'/').split(b'/')
+    pathsegs = path.lstrip('/').split('/')
     segments_with_params = []
     for component in pathsegs:
-        parts = component.split(b';')
+        parts = component.split(';')
         params = Mapping()
         segment = path_decode(parts[0])
         for p in parts[1:]:
-            if b'=' in p:
-                k, v = p.split(b'=', 1)
+            if '=' in p:
+                k, v = p.split('=', 1)
             else:
-                k, v = p, b''
+                k, v = p, ''
             params.add(path_decode(k), path_decode(v))
         segments_with_params.append(PathPart(segment, params))
     return segments_with_params
@@ -59,7 +62,7 @@ def extract_rfc2396_params(path):
 def split_path_no_params(path):
     """This splits a path into parts on "/" only (no split on ";" or ",").
     """
-    return [PathPart(path_decode(s)) for s in path.lstrip(b'/').split(b'/')]
+    return [PathPart(path_decode(s)) for s in path.lstrip('/').split('/')]
 
 
 class Path(Mapping):
@@ -79,17 +82,14 @@ class Querystring(Mapping):
     def __init__(self, raw):
         """Takes a string of type application/x-www-form-urlencoded.
         """
-        self.decoded = urllib.unquote_plus(raw).decode('UTF-8')
+        self.decoded = _decode(unquote_plus(raw))
         self.raw = raw
 
         # parse_qs does its own unquote_plus'ing ...
-        as_dict = cgi.parse_qs( raw
-                              , keep_blank_values = True
-                              , strict_parsing = False
-                               )
-
-        # ... but doesn't decode to unicode.
-        for k, vals in as_dict.items():
-            as_dict[k.decode('UTF-8')] = [v.decode('UTF-8') for v in vals]
+        as_dict = parse_qs(raw, keep_blank_values=True, strict_parsing=False)
+
+        # ... but doesn't decode to unicode (in older python versions).
+        for k, vals in list(as_dict.items()):
+            as_dict[_decode(k)] = [_decode(v) for v in vals]
 
         Mapping.__init__(self, as_dict)

aspen/output.py

@@ -3,3 +3,7 @@ class Output(object):
 
     def __init__(self, **kw):
         self.__dict__.update(kw)
+
+    @property
+    def text(self):
+        return self.body.decode(self.charset) if self.charset else None

aspen/request_processor/algorithm.py

@@ -53,7 +53,7 @@ def dispatch_path_to_filesystem(request_processor, path, querystring):
                                 , uripath               = path.decoded
                                 , startdir              = request_processor.www_root
                                  )
-    for k, v in result.wildcards.iteritems():
+    for k, v in result.wildcards.items():
         path[k] = v
     return {'dispatch_result': result}
 
@@ -74,6 +74,6 @@ def render_resource(state, resource):
 
 
 def encode_output(output, request_processor):
-    if isinstance(output.body, unicode):
+    if not isinstance(output.body, bytes):
         output.charset = request_processor.charset_dynamic
         output.body = output.body.encode(output.charset)

aspen/request_processor/dispatcher.py

@@ -10,6 +10,7 @@
 import os
 import posixpath
 from collections import namedtuple
+from functools import reduce
 
 
 class DispatchError(Exception):

aspen/request_processor/typecasting.py

@@ -28,7 +28,7 @@ def apply_typecasters(typecasters, path, state):
        *without* those extensions attached anymore, but with typecast values.
        It also then removes the string-value keys (the ones with the extensions).
     """
-    for part in path.keys():
+    for part in list(path.keys()):
         pieces = part.rsplit('.',1)
         if len(pieces) > 1:
             var, ext = pieces

aspen/simplates/json_.py

@@ -70,7 +70,7 @@ def default(self, obj):
             cls = obj.__class__ # Use this instead of type(obj) because that
                                 # isn't consistent between new- and old-style
                                 # classes, and this is.
-            encode = encoders.get(cls, _json.JSONEncoder.default)
+            encode = encoders.get(cls, super(FriendlyEncoder, self).default)
             return encode(obj)
 
 def lazy_check():

aspen/simplates/pagination.py

@@ -20,7 +20,7 @@ class Page(object):
 
     def __init__(self, content, header='', offset=0):
         self.content = content
-        self.header = header.decode('ascii')
+        self.header = header
         self.offset = offset
 
     @property

aspen/simplates/renderers/__init__.py

@@ -105,9 +105,9 @@ def factories(configuration):
         try:
             capture = {}
             python_syntax = 'from aspen.simplates.renderers.%s import Factory'
-            exec python_syntax % name in capture
+            exec(python_syntax % name, capture)
             make_renderer = capture['Factory'](configuration)
-        except ImportError, err:
+        except ImportError as err:
             make_renderer = err
             err.info = sys.exc_info()
         renderer_factories[name] = make_renderer
@@ -135,7 +135,7 @@ def __init__(self, factory, filepath, raw, media_type, offset):
         self.raw = raw
         self.media_type = media_type
         self.offset = offset
-        self.padded = (b'\n' * offset) + self.raw
+        self.padded = ('\n' * offset) + self.raw
         self.compiled = self.compile(self._filepath, self.padded)
 
     def __call__(self, context):

aspen/simplates/renderers/json_dump.py

@@ -12,7 +12,10 @@ def render_content(self, context):
         output = context['output']
         if not output.media_type:
             output.media_type = context['request_processor'].media_type_json
-        return json_.dumps(eval(self.compiled, globals(), context))
+        r = json_.dumps(eval(self.compiled, globals(), context))
+        if isinstance(r, bytes):
+            r = r.decode('ascii')
+        return r
 
 
 class Factory(Factory):

aspen/simplates/simplate.py

@@ -12,17 +12,6 @@
 renderer_re = re.compile(r'[a-z0-9.-_]+$')
 media_type_re = re.compile(r'[A-Za-z0-9.+*-]+/[A-Za-z0-9.+*-]+$')
 
-MIN_PAGES=2
-MAX_PAGES=None
-
-
-def _ordinal(n):
-    ords = [ 'zero' , 'one' , 'two', 'three', 'four'
-           , 'five', 'six', 'seven', 'eight', 'nine' ]
-    if 0 <= n < len(ords):
-        return ords[n]
-    return str(n)
-
 
 def _decode(raw):
     """As per PEP 263, decode raw data according to the encoding specified in
@@ -31,7 +20,7 @@ def _decode(raw):
     """
     assert type(raw) is bytes  # sanity check
 
-    decl_re = re.compile(r'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
+    decl_re = re.compile(br'^[ \t\f]*#.*coding[:=][ \t]*([-\w.]+)')
 
     def get_declaration(line):
         match = decl_re.match(line)
@@ -51,7 +40,7 @@ def get_declaration(line):
                 # observed behavior.
 
                 encoding = potential
-                munged = b'# encoding set to {0}\n'.format(encoding)
+                munged = b'# encoding set to ' + encoding + b'\n'
 
             else:
 
@@ -61,14 +50,15 @@ def get_declaration(line):
                 # object, we'll get a SyntaxError if we have a well-formed
                 # `coding: # ` line in it.
 
-                munged = b'# encoding NOT set to {0}\n'.format(potential)
+                munged = b'# encoding NOT set to ' + potential + b'\n'
 
             line = line.split(b'#')[0] + munged
 
         fulltext += line
     fulltext += sio.read()
     sio.close()
-    return fulltext.decode(encoding or b'ascii')
+    encoding = encoding.decode('ascii') if encoding else 'ascii'
+    return fulltext.decode(encoding)
 
 
 class SimplateDefaults(object):
@@ -101,8 +91,8 @@ def __init__(self, defaults, fs, raw, default_media_type):
 
         self.defaults = defaults                      # type: SimplateDefaults
         self.fs = fs                                  # type: str
-        self.raw = raw                                # type: str
-        self.decoded = _decode(raw)                   # type: unicode
+        self.raw = raw                                # type: bytes
+        self.decoded = _decode(raw)                   # type: str
         self.default_media_type = default_media_type  # type: str
 
         self.renderers = {}         # mapping of media type to Renderer objects
@@ -158,7 +148,7 @@ def parse_into_pages(self, decoded):
 
         pages = list(split_and_escape(decoded))
         npages = len(pages)
-        blank = [ Page(b'') ]
+        blank = [ Page('') ]
 
         if npages == 1:
             pages = blank + blank + pages
@@ -271,15 +261,13 @@ def _get_renderer_factory(self, media_type, renderer):
                    "renderers (might need third-party libs): %s.")
             raise SyntaxError(msg % (renderer, renderer_re.pattern, possible))
 
-        renderer = renderer.decode('US-ASCII')
-
         make_renderer = factories.get(renderer, None)
         if isinstance(make_renderer, ImportError):
             raise make_renderer
         elif make_renderer is None:
             possible = []
             legend = ''
-            for k, v in sorted(factories.iteritems()):
+            for k, v in sorted(factories.items()):
                 if isinstance(v, ImportError):
                     k = '*' + k
                     legend = " (starred are missing third-party libraries)"

aspen/testing.py

@@ -66,7 +66,11 @@ def simple(self, contents='Greetings, program!', filepath='index.html.spt', urip
         """A helper to create a file and hit it through our machinery.
         """
         if filepath is not None:
-            self.fs.www.mk((filepath, contents))
+            if isinstance(contents, tuple):
+                contents, encoding = contents
+            else:
+                encoding = 'utf8'
+            self.fs.www.mk((filepath, contents, True, encoding))
         if request_processor_configuration is not None:
             self.hydrate_request_processor(**request_processor_configuration)