Merge pull request #88 from liberapay/python3-step2

Get Liberapay running on python3

.flake8

@@ -0,0 +1,4 @@
+[flake8]
+ignore = E127,E226,E301,E302,E303,E309,E402,E701,E711,E712,E731
+max_line_length = 120
+show_source = True

.travis.yml

@@ -1,20 +1,22 @@
 language: python
+python:
+  - 3.5
 addons:
   postgresql: 9.4
 branches:
   only:
     - master
 cache:
   directories:
-    - env/bin
-    - env/lib/python2.7/site-packages
+    - .tox
 install:
- - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf env; fi
- - touch requirements.txt && make env
+ - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf .tox; fi
+ - pip install tox
 before_script:
  - psql -U postgres -c 'CREATE DATABASE liberapay_tests;'
+ - DATABASE_URL=liberapay_tests ./recreate-schema.sh test
  - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rfv tests/py/fixtures; export LIBERAPAY_I18N_TEST=yes; fi
-script: make test
+script: tox
 notifications:
   email: false
   irc:

Makefile

@@ -11,7 +11,6 @@ pip := pip --disable-pip-version-check
 with_local_env := $(env_bin)/honcho run -e defaults.env,local.env
 with_tests_env := $(env_bin)/honcho run -e $(test_env_files)
 py_test := $(with_tests_env) $(env_bin)/py.test
-pep8_ignore = E127,E226,E301,E302,E303,E309,E402,E701,E711,E712,E731
 
 echo:
 	@echo $($(var))
@@ -55,7 +54,7 @@ test-schema: env
 	$(with_tests_env) ./recreate-schema.sh test
 
 pyflakes: env
-	$(env_bin)/flake8 --max-line-length=120 --show-source --ignore=$(pep8_ignore) app.py liberapay tests
+	$(env_bin)/flake8 app.py liberapay tests
 
 test: test-schema pytest
 tests: test

liberapay/__init__.py

@@ -27,20 +27,24 @@ def canonize(request, website):
             new_uri = urlunsplit((scheme, netloc, path, query, fragment))
             request.line = Line(l.method.raw, new_uri, l.version.raw)
         return
-    scheme = request.headers.get('X-Forwarded-Proto', 'http')
-    host = request.headers['Host']
+    scheme = request.headers.get(b'X-Forwarded-Proto', b'http')
+    try:
+        request.hostname = host = request.headers[b'Host'].decode('idna')
+    except UnicodeDecodeError:
+        request.hostname = host = ''
     canonical_host = website.canonical_host
     canonical_scheme = website.canonical_scheme
-    bad_scheme = scheme != canonical_scheme
+    bad_scheme = scheme.decode('ascii', 'replace') != canonical_scheme
     bad_host = False
     if canonical_host:
         if host == canonical_host:
             pass
         elif host.endswith('.'+canonical_host):
             subdomain = host[:-len(canonical_host)-1]
             if subdomain in website.locales:
-                accept_langs = request.headers.get('Accept-Language', '')
-                request.headers['Accept-Language'] = subdomain+','+accept_langs
+                accept_langs = request.headers.get(b'Accept-Language', b'')
+                accept_langs = subdomain.encode('idna') + b',' + accept_langs
+                request.headers[b'Accept-Language'] = accept_langs
             else:
                 bad_host = True
         else:

liberapay/constants.py

@@ -120,7 +120,7 @@ def check_bits(bits):
     MC MT NL NO PL PT RO SE SI SK
 """.split())
 
-SESSION = b'session'
+SESSION = str('session')  # bytes in python2, unicode in python3
 SESSION_REFRESH = timedelta(hours=1)
 SESSION_TIMEOUT = timedelta(hours=6)
 

liberapay/elsewhere/_base.py

@@ -200,7 +200,8 @@ def extract_user_info(self, info):
         if not r.avatar_url:
             gravatar_id = self.x_gravatar_id(r, info, None)
             if r.email and not gravatar_id:
-                gravatar_id = hashlib.md5(r.email.strip().lower()).hexdigest()
+                bs = r.email.strip().lower().encode('utf8')
+                gravatar_id = hashlib.md5(bs).hexdigest()
             if gravatar_id:
                 r.avatar_url = 'https://seccdn.libravatar.org/avatar/'+gravatar_id
         r.is_team = self.x_is_team(r, info, False)

liberapay/main.py

@@ -149,7 +149,7 @@ def _encode_url(url):
     raise Warning('pando.Response.redirect() already exists')
 def _redirect(response, url, code=302):
     response.code = code
-    response.headers['Location'] = response.encode_url(url)
+    response.headers[b'Location'] = response.encode_url(url)
     raise response
 pando.Response.redirect = _redirect
 

liberapay/models/participant.py

@@ -5,11 +5,10 @@
 from email.utils import formataddr
 from hashlib import pbkdf2_hmac, md5
 from os import urandom
-import pickle
 from time import sleep
 import uuid
 
-from six.moves.urllib.parse import urlencode
+from six.moves.urllib.parse import quote, urlencode
 
 import aspen_jinja2_renderer
 from html2text import html2text
@@ -50,7 +49,7 @@
 from liberapay.models.exchange_route import ExchangeRoute
 from liberapay.security.crypto import constant_time_compare
 from liberapay.utils import (
-    b64encode_s, erase_cookie, serialize, set_cookie,
+    b64encode_s, deserialize, erase_cookie, serialize, set_cookie,
     emails, i18n,
 )
 from liberapay.website import website
@@ -215,7 +214,12 @@ def hash_password(cls, password):
         salt = urandom(21)
         rounds = website.app_conf.password_rounds
         hashed = cls._hash_password(password, algo, salt, rounds)
-        hashed = '$'.join((algo, str(rounds), b64encode(salt), b64encode(hashed)))
+        hashed = '$'.join((
+            algo,
+            str(rounds),
+            b64encode(salt).decode('ascii'),
+            b64encode(hashed).decode('ascii')
+        ))
         return hashed
 
     def update_password(self, password, cursor=None):
@@ -353,7 +357,9 @@ def resolve_stub(self):
               FROM elsewhere
              WHERE participant = %s
         """, (self.id,))
-        return rec and '/on/%s/%s/' % (rec.platform, rec.user_name)
+        if rec:
+            return '/on/%s/%s/' % (quote(rec.platform), quote(rec.user_name))
+        return None
 
 
     # Closing
@@ -723,7 +729,7 @@ def dequeue_emails(cls):
                     delete(msg)
                     continue
                 try:
-                    r = p.send_email(msg.spt_name, **pickle.loads(msg.context))
+                    r = p.send_email(msg.spt_name, **deserialize(msg.context))
                     assert r == 1
                 except Exception as e:
                     website.tell_sentry(e, {}, allow_reraise=True)
@@ -734,6 +740,8 @@ def dequeue_emails(cls):
     def set_email_lang(self, accept_lang):
         if not accept_lang:
             return
+        if isinstance(accept_lang, bytes):
+            accept_lang = accept_lang.decode('ascii', 'replace')
         self.db.run("UPDATE participants SET email_lang=%s WHERE id=%s",
                     (accept_lang, self.id))
         self.set_attributes(email_lang=accept_lang)
@@ -843,7 +851,7 @@ def render_notifications(self, state):
         """, (self.id,))
         for id, event, notif_context, is_new in notifs:
             try:
-                notif_context = pickle.loads(notif_context)
+                notif_context = deserialize(notif_context)
                 context = dict(state)
                 self.fill_notification_context(context)
                 context.update(notif_context)
@@ -1038,7 +1046,7 @@ def update_avatar(self, src=None, cursor=None):
         if platform == 'libravatar' or platform is None and email:
             if not email:
                 return
-            avatar_id = md5(email.strip().lower()).hexdigest()
+            avatar_id = md5(email.strip().lower().encode('utf8')).hexdigest()
             avatar_url = 'https://seccdn.libravatar.org/avatar/'+avatar_id
             avatar_url += AVATAR_QUERY
 
@@ -1663,7 +1671,7 @@ def to_dict(self, details=False, inquirer=None):
         #   null - user has no funding goal
         #   3.00 - user wishes to receive at least this amount
         if self.goal != 0:
-            if self.goal > 0:
+            if self.goal and self.goal > 0:
                 goal = str(self.goal)
             else:
                 goal = None

liberapay/renderers/csv_dump.py

@@ -1,21 +1,35 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import csv
-from io import BytesIO
+from io import BytesIO, StringIO
+
+from six import PY2
 
 from aspen.simplates import renderers
 
 
+# The python2 version of the csv module doesn't support unicode
+codec = 'utf8'
+
+
+def maybe_encode(s):
+    return s.encode(codec) if PY2 else s
+
+
 class Renderer(renderers.Renderer):
 
     def render_content(self, context):
         rows = eval(self.compiled, globals(), context)
         if not rows:
             return ''
-        f = BytesIO()
+        if PY2:
+            f = BytesIO()
+            context['output'].charset = codec
+        else:
+            f = StringIO()
         w = csv.writer(f)
         if hasattr(rows[0], '_fields'):
-            w.writerow(rows[0]._fields)
+            w.writerow(map(maybe_encode, rows[0]._fields))
         w.writerows(rows)
         f.seek(0)
         return f.read()

liberapay/renderers/scss.py

@@ -4,7 +4,8 @@
 import os
 import posixpath
 import re
-from urlparse import urlsplit
+
+from six.moves.urllib.parse import urlsplit
 
 import sass
 from aspen import renderers

liberapay/security/authentication.py

@@ -99,13 +99,13 @@ def authenticate_user_if_possible(request, state, user, _):
         return
 
     # HTTP auth
-    if 'Authorization' in request.headers:
-        header = request.headers['authorization']
-        if not header.startswith('Basic '):
+    if b'Authorization' in request.headers:
+        header = request.headers[b'Authorization']
+        if not header.startswith(b'Basic '):
             raise Response(401, 'Unsupported authentication method')
         try:
-            creds = binascii.a2b_base64(header[len('Basic '):]).split(':', 1)
-        except binascii.Error:
+            creds = binascii.a2b_base64(header[len('Basic '):]).decode('utf8').split(':', 1)
+        except (binascii.Error, UnicodeDecodeError):
             raise Response(400, 'Malformed "Authorization" header')
         participant = Participant.authenticate('id', 'password', *creds)
         if not participant:

liberapay/security/crypto.py

@@ -20,8 +20,8 @@
     using_sysrandom = False
 
 
-pool = string.digits + string.letters + string.punctuation
-UNSECURE_RANDOM_STRING = b"".join([random.choice(pool) for i in range(64)])
+pool = string.digits + string.ascii_letters + string.punctuation
+UNSECURE_RANDOM_STRING = "".join([random.choice(pool) for i in range(64)])
 
 
 def get_random_string(length=12,

liberapay/security/csrf.py

@@ -18,9 +18,10 @@
 
 
 TOKEN_LENGTH = 32
+CSRF_TOKEN = str('csrf_token')  # bytes in python2, unicode in python3
 CSRF_TIMEOUT = timedelta(days=7)
 
-_get_new_token = lambda: get_random_string(TOKEN_LENGTH).encode('ascii')
+_get_new_token = lambda: get_random_string(TOKEN_LENGTH)
 _token_re = re.compile(r'^[a-zA-Z0-9]{%d}$' % TOKEN_LENGTH)
 _sanitize_token = lambda t: t if _token_re.match(t) else None
 
@@ -29,7 +30,7 @@ def extract_token_from_cookie(request):
     """Given a Request object, return a csrf_token.
     """
     try:
-        token = request.headers.cookie['csrf_token'].value
+        token = request.headers.cookie[CSRF_TOKEN].value
     except KeyError:
         token = None
     else:
@@ -54,7 +55,7 @@ def reject_forgeries(request, csrf_token):
         if request.line.uri.startswith('/callbacks/'):
             return
         # and requests using HTTP auth
-        if 'Authorization' in request.headers:
+        if b'Authorization' in request.headers:
             return
 
         # Check non-cookie token for match.
@@ -66,7 +67,7 @@ def reject_forgeries(request, csrf_token):
         if second_token == "":
             # Fall back to X-CSRF-TOKEN, to make things easier for AJAX,
             # and possible for PUT/DELETE.
-            second_token = request.headers.get('X-CSRF-TOKEN', '')
+            second_token = request.headers.get(b'X-CSRF-TOKEN', b'').decode('ascii', 'replace')
 
         if not constant_time_compare(second_token, csrf_token):
             raise Response(403, "Bad CSRF cookie")
@@ -78,4 +79,4 @@ def add_token_to_response(response, csrf_token=None):
     if csrf_token:
         # Don't set httponly so that we can POST using XHR.
         # https://github.com/gratipay/gratipay.com/issues/3030
-        response.set_cookie(b'csrf_token', csrf_token, expires=CSRF_TIMEOUT, httponly=False)
+        response.set_cookie(CSRF_TOKEN, csrf_token, expires=CSRF_TIMEOUT, httponly=False)

liberapay/testing/__init__.py

@@ -21,6 +21,7 @@
 from liberapay.models.account_elsewhere import AccountElsewhere
 from liberapay.models.exchange_route import ExchangeRoute
 from liberapay.models.participant import Participant
+from liberapay.security.csrf import CSRF_TOKEN
 from liberapay.testing.vcr import use_cassette
 
 
@@ -40,11 +41,11 @@ def build_wsgi_environ(self, *a, **kw):
         """
 
         # csrf - for both anon and authenticated
-        csrf_token = kw.get('csrf_token', b'ThisIsATokenThatIsThirtyTwoBytes')
+        csrf_token = kw.get('csrf_token', 'ThisIsATokenThatIsThirtyTwoBytes')
         if csrf_token:
             cookies = kw.setdefault('cookies', {})
-            cookies[b'csrf_token'] = csrf_token
-            kw[b'HTTP_X-CSRF-TOKEN'] = csrf_token
+            cookies[CSRF_TOKEN] = csrf_token
+            kw['HTTP_X-CSRF-TOKEN'] = csrf_token
 
         # user authentication
         auth_as = kw.pop('auth_as', None)
@@ -57,7 +58,7 @@ def build_wsgi_environ(self, *a, **kw):
 
     def hit(self, *a, **kw):
         if kw.pop('xhr', False):
-            kw[b'HTTP_X_REQUESTED_WITH'] = b'XMLHttpRequest'
+            kw['HTTP_X_REQUESTED_WITH'] = b'XMLHttpRequest'
 
         # prevent tell_sentry from reraising errors
         if not kw.pop('sentry_reraise', True):