From c72541281b0a5795512d76dd11a44dc47324ce78 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 11:00:30 +0200 Subject: [PATCH 01/29] python3 compat in crypto module --- liberapay/security/crypto.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/liberapay/security/crypto.py b/liberapay/security/crypto.py index bf412fe990..86be731625 100644 --- a/liberapay/security/crypto.py +++ b/liberapay/security/crypto.py @@ -20,8 +20,8 @@ using_sysrandom = False -pool = string.digits + string.letters + string.punctuation -UNSECURE_RANDOM_STRING = b"".join([random.choice(pool) for i in range(64)]) +pool = string.digits + string.ascii_letters + string.punctuation +UNSECURE_RANDOM_STRING = "".join([random.choice(pool) for i in range(64)]) def get_random_string(length=12, From d90ff560c9f5bb4b51dc1122779010dd1a497a09 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 11:04:35 +0200 Subject: [PATCH 02/29] use tox to test both python 2 and 3 on Travis --- .travis.yml | 10 +++++----- tox.ini | 11 +++++++++++ 2 files changed, 16 insertions(+), 5 deletions(-) create mode 100644 tox.ini diff --git a/.travis.yml b/.travis.yml index 57c59ddcb5..9c75ba0eb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,15 +6,15 @@ branches: - master cache: directories: - - env/bin - - env/lib/python2.7/site-packages + - .tox install: - - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf env; fi - - touch requirements.txt && make env + - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rf .tox; fi + - pip install tox before_script: - psql -U postgres -c 'CREATE DATABASE liberapay_tests;' + - make test-schema - if [ "${TRAVIS_BRANCH}" = "master" -a "${TRAVIS_PULL_REQUEST}" = "false" ]; then rm -rfv tests/py/fixtures; export LIBERAPAY_I18N_TEST=yes; fi -script: make test +script: tox notifications: email: false irc: diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000000..9856d641bd --- /dev/null +++ b/tox.ini @@ -0,0 +1,11 @@ +[tox] +envlist = py27,py35 +skipsdist = True + +[testenv] +commands = + honcho run -e defaults.env,tests/test.env,tests/local.env python -m pytest tests/py {posargs} +deps = + --requirement=requirements.txt + --requirement=requirements_dev.txt + --requirement=requirements_tests.txt From 4718e32b7b62cdcade6a9c5078c14fa2dd161783 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 11:05:22 +0200 Subject: [PATCH 03/29] move fake-factory dependency to where it belongs --- requirements_dev.txt | 2 -- requirements_tests.txt | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index 38e49fe509..34b7ee0a6c 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,4 +1,2 @@ -fake-factory==0.3.2 - honcho==0.5.0 diff --git a/requirements_tests.txt b/requirements_tests.txt index 81d90989dd..c65c26a2b6 100644 --- a/requirements_tests.txt +++ b/requirements_tests.txt @@ -20,3 +20,5 @@ PyYAML==3.11 contextlib2==0.4.0 wrapt==1.10.2 vcrpy==1.1.3 + +fake-factory==0.3.2 From 8b51387a792770e756e54bf76c81d555f63dd0eb Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 17:59:41 +0200 Subject: [PATCH 04/29] HTTP headers should always be bytestrings --- liberapay/__init__.py | 14 +++++++----- liberapay/main.py | 2 +- liberapay/models/participant.py | 2 ++ liberapay/security/authentication.py | 10 ++++----- liberapay/security/csrf.py | 4 ++-- liberapay/utils/http_caching.py | 12 +++++----- liberapay/utils/i18n.py | 2 +- tests/py/test_close.py | 2 +- tests/py/test_elsewhere.py | 8 +++---- tests/py/test_hooks.py | 22 +++++++++---------- tests/py/test_pages.py | 4 ++-- tests/py/test_payio.py | 6 ++--- tests/py/test_public_json.py | 2 +- tests/py/test_sign_in.py | 6 ++--- tests/py/test_teams.py | 4 ++-- tests/py/test_utils.py | 4 ++-- www/%username/avatar.spt | 2 +- www/%username/charts.json.spt | 2 +- www/%username/communities.json.spt | 2 +- www/%username/elsewhere/delete.spt | 2 +- www/%username/emails/modify.json.spt | 4 ++-- www/%username/emails/notifications.json.spt | 2 +- www/%username/emails/verify.html.spt | 2 +- www/%username/identity.spt | 2 +- www/%username/public.json.spt | 2 +- www/%username/routes/bank-account.spt | 2 +- www/%username/settings/edit.spt | 2 +- www/%username/tip.spt | 2 +- .../wallet/payin/bankwire/%back_to.spt | 2 +- www/%username/widgets/%type.spt | 4 ++-- www/%username/widgets/button.js.spt | 4 ++-- www/about/charts.json.spt | 4 ++-- www/about/paydays.spt | 2 +- www/admin/payday/%id.spt | 10 ++++----- www/for/%name/%action.spt | 2 +- www/index.html.spt | 2 +- .../%platform/%user_name/%endpoint.json.spt | 6 ++--- www/sign-out.spt | 4 ++-- 38 files changed, 88 insertions(+), 82 deletions(-) diff --git a/liberapay/__init__.py b/liberapay/__init__.py index 413d505bd1..0312a5b5eb 100644 --- a/liberapay/__init__.py +++ b/liberapay/__init__.py @@ -27,11 +27,14 @@ def canonize(request, website): new_uri = urlunsplit((scheme, netloc, path, query, fragment)) request.line = Line(l.method.raw, new_uri, l.version.raw) return - scheme = request.headers.get('X-Forwarded-Proto', 'http') - host = request.headers['Host'] + scheme = request.headers.get(b'X-Forwarded-Proto', b'http') + try: + request.hostname = host = request.headers[b'Host'].decode('idna') + except UnicodeDecodeError: + request.hostname = host = '' canonical_host = website.canonical_host canonical_scheme = website.canonical_scheme - bad_scheme = scheme != canonical_scheme + bad_scheme = scheme.decode('ascii', 'replace') != canonical_scheme bad_host = False if canonical_host: if host == canonical_host: @@ -39,8 +42,9 @@ def canonize(request, website): elif host.endswith('.'+canonical_host): subdomain = host[:-len(canonical_host)-1] if subdomain in website.locales: - accept_langs = request.headers.get('Accept-Language', '') - request.headers['Accept-Language'] = subdomain+','+accept_langs + accept_langs = request.headers.get(b'Accept-Language', b'') + accept_langs = subdomain.encode('idna') + b',' + accept_langs + request.headers[b'Accept-Language'] = accept_langs else: bad_host = True else: diff --git a/liberapay/main.py b/liberapay/main.py index fb6e272353..061dabac14 100644 --- a/liberapay/main.py +++ b/liberapay/main.py @@ -149,7 +149,7 @@ def _encode_url(url): raise Warning('pando.Response.redirect() already exists') def _redirect(response, url, code=302): response.code = code - response.headers['Location'] = response.encode_url(url) + response.headers[b'Location'] = response.encode_url(url) raise response pando.Response.redirect = _redirect diff --git a/liberapay/models/participant.py b/liberapay/models/participant.py index 577daabbd8..767aa9d48c 100644 --- a/liberapay/models/participant.py +++ b/liberapay/models/participant.py @@ -734,6 +734,8 @@ def dequeue_emails(cls): def set_email_lang(self, accept_lang): if not accept_lang: return + if isinstance(accept_lang, bytes): + accept_lang = accept_lang.decode('ascii', 'replace') self.db.run("UPDATE participants SET email_lang=%s WHERE id=%s", (accept_lang, self.id)) self.set_attributes(email_lang=accept_lang) diff --git a/liberapay/security/authentication.py b/liberapay/security/authentication.py index 44a723526c..8e16aca805 100644 --- a/liberapay/security/authentication.py +++ b/liberapay/security/authentication.py @@ -99,13 +99,13 @@ def authenticate_user_if_possible(request, state, user, _): return # HTTP auth - if 'Authorization' in request.headers: - header = request.headers['authorization'] - if not header.startswith('Basic '): + if b'Authorization' in request.headers: + header = request.headers[b'Authorization'] + if not header.startswith(b'Basic '): raise Response(401, 'Unsupported authentication method') try: - creds = binascii.a2b_base64(header[len('Basic '):]).split(':', 1) - except binascii.Error: + creds = binascii.a2b_base64(header[len('Basic '):]).decode('utf8').split(':', 1) + except (binascii.Error, UnicodeDecodeError): raise Response(400, 'Malformed "Authorization" header') participant = Participant.authenticate('id', 'password', *creds) if not participant: diff --git a/liberapay/security/csrf.py b/liberapay/security/csrf.py index 7aa3af6396..45aed4ca07 100644 --- a/liberapay/security/csrf.py +++ b/liberapay/security/csrf.py @@ -54,7 +54,7 @@ def reject_forgeries(request, csrf_token): if request.line.uri.startswith('/callbacks/'): return # and requests using HTTP auth - if 'Authorization' in request.headers: + if b'Authorization' in request.headers: return # Check non-cookie token for match. @@ -66,7 +66,7 @@ def reject_forgeries(request, csrf_token): if second_token == "": # Fall back to X-CSRF-TOKEN, to make things easier for AJAX, # and possible for PUT/DELETE. - second_token = request.headers.get('X-CSRF-TOKEN', '') + second_token = request.headers.get(b'X-CSRF-TOKEN', b'').decode('ascii', 'replace') if not constant_time_compare(second_token, csrf_token): raise Response(403, "Bad CSRF cookie") diff --git a/liberapay/utils/http_caching.py b/liberapay/utils/http_caching.py index 0420c24fd6..8a9cc920f2 100644 --- a/liberapay/utils/http_caching.py +++ b/liberapay/utils/http_caching.py @@ -86,7 +86,7 @@ def try_to_serve_304(dispatch_result, request, etag): # Don't serve one version of a file as if it were another. raise Response(410) - headers_etag = request.headers.get('If-None-Match') + headers_etag = request.headers.get(b'If-None-Match', b'').decode('ascii', 'replace') if not headers_etag: # This client doesn't want a 304. return @@ -107,8 +107,8 @@ def add_caching_to_response(response, request=None, etag=None): """ if not etag: # This is a dynamic resource, disable caching by default - if 'Cache-Control' not in response.headers: - response.headers['Cache-Control'] = 'no-cache' + if b'Cache-Control' not in response.headers: + response.headers[b'Cache-Control'] = b'no-cache' return assert request is not None # sanity check @@ -117,11 +117,11 @@ def add_caching_to_response(response, request=None, etag=None): return # https://developers.google.com/speed/docs/best-practices/caching - response.headers['Etag'] = etag + response.headers[b'Etag'] = etag.encode('ascii') if request.line.uri.querystring.get('etag'): # We can cache "indefinitely" when the querystring contains the etag. - response.headers['Cache-Control'] = 'public, max-age=31536000' + response.headers[b'Cache-Control'] = b'public, max-age=31536000' else: # Otherwise we cache for 1 hour - response.headers['Cache-Control'] = 'public, max-age=3600' + response.headers[b'Cache-Control'] = b'public, max-age=3600' diff --git a/liberapay/utils/i18n.py b/liberapay/utils/i18n.py index 3a7e5d7842..49e34515b3 100644 --- a/liberapay/utils/i18n.py +++ b/liberapay/utils/i18n.py @@ -257,7 +257,7 @@ def get_lang_options(request, locale, previously_used_langs, add_multi=False): def set_up_i18n(website, request, state): - accept_lang = request.headers.get("Accept-Language", "") + accept_lang = request.headers.get(b"Accept-Language", b"").decode('ascii', 'replace') langs = request.accept_langs = list(parse_accept_lang(accept_lang)) loc = match_lang(langs) add_helpers_to_context(state, loc) diff --git a/tests/py/test_close.py b/tests/py/test_close.py index 423ac2abf2..25f4da021c 100644 --- a/tests/py/test_close.py +++ b/tests/py/test_close.py @@ -59,7 +59,7 @@ def test_can_post_to_close_page(self): data = {'disburse_to': 'downstream'} response = self.client.PxST('/alice/settings/close', auth_as=alice, data=data) assert response.code == 302 - assert response.headers['Location'] == '/alice/' + assert response.headers[b'Location'] == b'/alice/' assert Participant.from_username('alice').balance == 0 assert Participant.from_username('bob').balance == 7 diff --git a/tests/py/test_elsewhere.py b/tests/py/test_elsewhere.py index 0f8d55d1e8..3abd522754 100644 --- a/tests/py/test_elsewhere.py +++ b/tests/py/test_elsewhere.py @@ -43,13 +43,13 @@ def test_connect_success(self, gui, gusi, ft): gui.return_value = self.client.website.platforms.github.extract_user_info({'id': 1}) ft.return_value = None - then = '/foobar' + then = b'/foobar' cookie = b64encode_s(json.dumps(['query_data', 'connect', b64encode_s(then), '2'])) response = self.client.GxT('/on/github/associate?state=deadbeef', auth_as=alice, cookies={b'github_deadbeef': cookie}) assert response.code == 302, response.text - assert response.headers['Location'] == then + assert response.headers[b'Location'] == then @mock.patch('requests_oauthlib.OAuth2Session.fetch_token') @mock.patch('liberapay.elsewhere._base.Platform.get_user_self_info') @@ -67,7 +67,7 @@ def test_connect_might_need_confirmation(self, gui, gusi, ft): auth_as=alice, cookies={b'github_deadbeef': cookie}) assert response.code == 302 - assert response.headers['Location'].startswith('/on/confirm.html?id=') + assert response.headers[b'Location'].startswith(b'/on/confirm.html?id=') def test_connect_failure(self): alice = self.make_participant('alice') @@ -200,7 +200,7 @@ def test_take_over(self): response = self.client.PxST('/on/take-over.html', data=data, auth_as=self.bob, cookies=self.connect_cookie) assert response.code == 302 - assert response.headers['Location'] == '/bob/edit' + assert response.headers[b'Location'] == b'/bob/edit' class TestFriendFinder(Harness): diff --git a/tests/py/test_hooks.py b/tests/py/test_hooks.py index a51b8718a4..9a823e53f8 100644 --- a/tests/py/test_hooks.py +++ b/tests/py/test_hooks.py @@ -33,7 +33,7 @@ def test_canonize_canonizes(self): HTTP_X_FORWARDED_PROTO=b'http', ) assert response.code == 302 - assert response.headers['Location'] == b'https://example.com/' + assert response.headers[b'Location'] == b'https://example.com/' def test_no_cookies_over_http(self): """ @@ -103,14 +103,14 @@ def test_i18n_subdomain_is_redirected_to_https(self): ) assert r.code == 302 assert not r.headers.cookie - assert r.headers['Location'] == b'https://en.example.com/' + assert r.headers[b'Location'] == b'https://en.example.com/' class Tests2(Harness): def test_accept_header_is_respected(self): r = self.client.GET('/about/stats', HTTP_ACCEPT=b'application/json') - assert r.headers['Content-Type'] == 'application/json; charset=UTF-8' + assert r.headers[b'Content-Type'] == b'application/json; charset=UTF-8' json.loads(r.body) def test_error_spt_works(self): @@ -119,29 +119,29 @@ def test_error_spt_works(self): def test_cors_is_not_allowed_by_default(self): r = self.client.GET('/') - assert 'Access-Control-Allow-Origin' not in r.headers + assert b'Access-Control-Allow-Origin' not in r.headers def test_cors_is_allowed_for_assets(self): r = self.client.GET('/assets/jquery.min.js') assert r.code == 200 - assert r.headers['Access-Control-Allow-Origin'] == '*' + assert r.headers[b'Access-Control-Allow-Origin'] == b'*' def test_caching_of_assets(self): r = self.client.GET('/assets/jquery.min.js') - assert r.headers['Cache-Control'] == 'public, max-age=3600' - assert 'Vary' not in r.headers + assert r.headers[b'Cache-Control'] == b'public, max-age=3600' + assert b'Vary' not in r.headers assert not r.headers.cookie def test_caching_of_assets_with_etag(self): r = self.client.GET(self.client.website.asset('jquery.min.js')) - assert r.headers['Cache-Control'] == 'public, max-age=31536000' - assert 'Vary' not in r.headers + assert r.headers[b'Cache-Control'] == b'public, max-age=31536000' + assert b'Vary' not in r.headers assert not r.headers.cookie def test_caching_of_simplates(self): r = self.client.GET('/') - assert r.headers['Cache-Control'] == 'no-cache' - assert 'Vary' not in r.headers + assert r.headers[b'Cache-Control'] == b'no-cache' + assert b'Vary' not in r.headers def test_no_csrf_cookie(self): r = self.client.POST('/', csrf_token=False, raise_immediately=False) diff --git a/tests/py/test_pages.py b/tests/py/test_pages.py index c6b423419b..ed2764b163 100644 --- a/tests/py/test_pages.py +++ b/tests/py/test_pages.py @@ -109,7 +109,7 @@ def test_homepage_in_all_supported_langs(self): link_lang = link_lang.format(l, self.website.canonical_scheme, self.website.canonical_host) assert link_lang in r.text - assert r.headers[b'Content-Type'] == 'text/html; charset=UTF-8' + assert r.headers[b'Content-Type'] == b'text/html; charset=UTF-8' def test_escaping_on_homepage(self): alice = self.make_participant('alice') @@ -144,7 +144,7 @@ def test_404(self): def test_anonymous_sign_out_redirects(self): response = self.client.PxST('/sign-out.html') assert response.code == 302 - assert response.headers['Location'] == '/' + assert response.headers[b'Location'] == b'/' def test_sign_out_overwrites_session_cookie(self): alice = self.make_participant('alice') diff --git a/tests/py/test_payio.py b/tests/py/test_payio.py index a7a3506fc5..3f4042c160 100644 --- a/tests/py/test_payio.py +++ b/tests/py/test_payio.py @@ -189,7 +189,7 @@ def test_charge_suspended_user(self): class TestPayinBankWire(MangopayHarness): def test_payin_bank_wire_creation(self): - path = '/janet/wallet/payin/bankwire/' + path = b'/janet/wallet/payin/bankwire/' data = {'amount': str(upcharge_bank_wire(D('10.00'))[0])} r = self.client.PxST(path, data, auth_as=self.janet) @@ -198,8 +198,8 @@ def test_payin_bank_wire_creation(self): self.janet.set_tip_to(self.david, '10.00') r = self.client.PxST(path, data, auth_as=self.janet) assert r.code == 302, r.text - redir = r.headers['Location'] - assert redir.startswith(path+'?exchange_id=') + redir = r.headers[b'Location'] + assert redir.startswith(path+b'?exchange_id=') r = self.client.GET(redir, auth_as=self.janet) assert b'IBAN' in r.body, r.text diff --git a/tests/py/test_public_json.py b/tests/py/test_public_json.py index b9e071bed1..302b361f82 100644 --- a/tests/py/test_public_json.py +++ b/tests/py/test_public_json.py @@ -112,7 +112,7 @@ def test_access_control_allow_origin_header_is_asterisk(self): self.make_participant('alice', balance=100) response = self.client.GET('/alice/public.json') - assert response.headers['Access-Control-Allow-Origin'] == '*' + assert response.headers[b'Access-Control-Allow-Origin'] == b'*' def test_jsonp_works(self): alice = self.make_participant('alice', balance=100) diff --git a/tests/py/test_sign_in.py b/tests/py/test_sign_in.py index 1d51337a73..159c3a3c28 100644 --- a/tests/py/test_sign_in.py +++ b/tests/py/test_sign_in.py @@ -51,7 +51,7 @@ def check_login(self, r, p): def check_with_about_me(self, username, cookies): r = self.client.GET('/about/me/', cookies=cookies, raise_immediately=False) assert r.code == 302 - assert r.headers['Location'] == '/'+username+'/' + assert r.headers[b'Location'] == b'/' + username.encode() + b'/' def test_log_in(self): password = 'password' @@ -126,7 +126,7 @@ def test_email_login(self): data = {'email-login.email': email} r = self.client.POST('/', data, raise_immediately=False) alice = alice.refetch() - assert alice.session_token not in r.headers.raw + assert alice.session_token not in r.headers.raw.decode('ascii') assert alice.session_token not in r.body.decode('utf8') Participant.dequeue_emails() @@ -140,7 +140,7 @@ def test_email_login(self): assert alice2.session_token != alice.session_token # ↑ this means that the link is only valid once assert r.code == 302 - assert r.headers['Location'] == '/alice/?foo=bar' + assert r.headers[b'Location'] == b'/alice/?foo=bar' # ↑ checks that original path and query are preserved # Check that we can change our password diff --git a/tests/py/test_teams.py b/tests/py/test_teams.py index 7896428d96..ad02fc51c3 100644 --- a/tests/py/test_teams.py +++ b/tests/py/test_teams.py @@ -93,7 +93,7 @@ def test_create_close_and_reopen_team(self): alice = self.make_participant('alice') r = self.client.PxST('/about/teams', {'name': 'Team'}, auth_as=alice) assert r.code == 302 - assert r.headers['Location'] == '/Team/edit' + assert r.headers[b'Location'] == b'/Team/edit' t = Participant.from_username('Team') assert t assert t.status == 'active' @@ -106,7 +106,7 @@ def test_create_close_and_reopen_team(self): r = self.client.PxST('/about/teams', {'name': 'Team'}, auth_as=alice) assert r.code == 302 - assert r.headers['Location'] == '/Team/edit' + assert r.headers[b'Location'] == b'/Team/edit' t = t.refetch() assert t.nmembers == 1 assert t.status == 'active' diff --git a/tests/py/test_utils.py b/tests/py/test_utils.py index 91b67bfee1..eecda860bd 100644 --- a/tests/py/test_utils.py +++ b/tests/py/test_utils.py @@ -42,7 +42,7 @@ def test_get_participant_canonicalizes(self): utils.get_participant(state, restrict=False) r = cm.exception assert r.code == 302 - assert r.headers['Location'] == '/alice/?foo=bar' + assert r.headers[b'Location'] == b'/alice/?foo=bar' def test_get_participant_canonicalizes_id_to_username(self): self.make_participant('alice') @@ -52,7 +52,7 @@ def test_get_participant_canonicalizes_id_to_username(self): utils.get_participant(state, restrict=False) r = cm.exception assert r.code == 302 - assert r.headers['Location'] == '/alice/?x=2' + assert r.headers[b'Location'] == b'/alice/?x=2' def test_is_expired(self): expiration = datetime.utcnow() - timedelta(days=40) diff --git a/www/%username/avatar.spt b/www/%username/avatar.spt index 4b4526ff62..fb7bb36b7e 100644 --- a/www/%username/avatar.spt +++ b/www/%username/avatar.spt @@ -24,7 +24,7 @@ if request.method == 'POST': raise Response(400, _("We were unable to get an avatar for you from {0}.", src)) msg = _("Your new avatar URL is: {0}", participant.avatar_url) - if request.headers.get('X-Requested-With') != 'XMLHttpRequest': + if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect(participant.path('edit')+'?success='+b64encode_s(msg)+'#avatar') [---] text/html diff --git a/www/%username/charts.json.spt b/www/%username/charts.json.spt index 4e9c600111..7d6349db87 100644 --- a/www/%username/charts.json.spt +++ b/www/%username/charts.json.spt @@ -75,7 +75,7 @@ if paydays: curpayday['npatrons'] += 1 curpayday['receipts'] += transfer['amount'] -response.headers["Access-Control-Allow-Origin"] = "*" +response.headers[b"Access-Control-Allow-Origin"] = b"*" [---] application/json via jsonp_dump paydays diff --git a/www/%username/communities.json.spt b/www/%username/communities.json.spt index 8c425287e4..3b2bfe89e8 100644 --- a/www/%username/communities.json.spt +++ b/www/%username/communities.json.spt @@ -24,7 +24,7 @@ if request.method == 'POST': is_on = action == 'join' user.update_community_status('memberships', is_on, c_id) - if request.headers.get('X-Requested-With') != 'XMLHttpRequest': + if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect(request.body.get('back_to') or participant.path('edit')) else: diff --git a/www/%username/elsewhere/delete.spt b/www/%username/elsewhere/delete.spt index 2bfd0957b7..a188a032cb 100644 --- a/www/%username/elsewhere/delete.spt +++ b/www/%username/elsewhere/delete.spt @@ -11,7 +11,7 @@ user_id = request.body["user_id"] participant.delete_elsewhere(platform, user_id) -if request.headers.get('X-Requested-With') != 'XMLHttpRequest': +if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect('..') [---] text/html diff --git a/www/%username/emails/modify.json.spt b/www/%username/emails/modify.json.spt index b16d11f5e7..09224eb894 100644 --- a/www/%username/emails/modify.json.spt +++ b/www/%username/emails/modify.json.spt @@ -7,7 +7,7 @@ request.allow("POST") participant = get_participant(state, restrict=True, allow_member=True) if not participant.email_lang: - participant.set_email_lang(request.headers.get("Accept-Language")) + participant.set_email_lang(request.headers.get(b"Accept-Language")) body = request.body out = {} @@ -25,7 +25,7 @@ elif 'remove' in body: else: raise Response(400, 'nothing to do') -if request.headers.get('X-Requested-With') != 'XMLHttpRequest': +if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect('.') [---] application/json via json_dump diff --git a/www/%username/emails/notifications.json.spt b/www/%username/emails/notifications.json.spt index 442853b33c..e10c46ebb0 100644 --- a/www/%username/emails/notifications.json.spt +++ b/www/%username/emails/notifications.json.spt @@ -24,7 +24,7 @@ for field in fields: WHERE id = %s """.format(op), (mask, p_id)) -if request.headers.get('X-Requested-With') != 'XMLHttpRequest': +if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect('.') [---] application/json via json_dump diff --git a/www/%username/emails/verify.html.spt b/www/%username/emails/verify.html.spt index f0cc4ce8c1..a7f48d5dbc 100644 --- a/www/%username/emails/verify.html.spt +++ b/www/%username/emails/verify.html.spt @@ -18,7 +18,7 @@ if ok: nonce = request.qs.get('nonce', '') result = participant.verify_email(email, nonce) if not participant.email_lang: - participant.set_email_lang(request.headers.get("Accept-Language")) + participant.set_email_lang(request.headers.get(b"Accept-Language")) [-----------------------------------------------------------------------------] diff --git a/www/%username/identity.spt b/www/%username/identity.spt index 9afe4ce29d..8783e2ec40 100644 --- a/www/%username/identity.spt +++ b/www/%username/identity.spt @@ -82,7 +82,7 @@ if request.method == 'POST': error = repr_exception(err) website.tell_sentry(err, state, allow_reraise=True) - if error and request.headers.get('X-Requested-With') == 'XMLHttpRequest': + if error and request.headers.get(b'X-Requested-With') == b'XMLHttpRequest': raise Response(400, error) else: diff --git a/www/%username/public.json.spt b/www/%username/public.json.spt index 3a2bf5f830..56693d4b3d 100644 --- a/www/%username/public.json.spt +++ b/www/%username/public.json.spt @@ -3,7 +3,7 @@ from liberapay.utils import get_participant [-----------------------------------------------------------------------------] participant = get_participant(state, restrict=False) -response.headers["Access-Control-Allow-Origin"] = "*" +response.headers[b"Access-Control-Allow-Origin"] = b"*" [---] application/json via jsonp_dump participant.to_dict(details=True, inquirer=user) diff --git a/www/%username/routes/bank-account.spt b/www/%username/routes/bank-account.spt index 73179f5e49..3cce544d0b 100644 --- a/www/%username/routes/bank-account.spt +++ b/www/%username/routes/bank-account.spt @@ -53,7 +53,7 @@ except ResponseException as err: raise Response(400, repr_exception(err)) else: ExchangeRoute.insert(participant, 'mango-ba', ba.Id) - if request.headers.get('X-Requested-With') != 'XMLHttpRequest': + if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect(request.line.uri) [---] text/html diff --git a/www/%username/settings/edit.spt b/www/%username/settings/edit.spt index 81c594dce4..0407c39eb3 100644 --- a/www/%username/settings/edit.spt +++ b/www/%username/settings/edit.spt @@ -44,7 +44,7 @@ elif 'username' in body: p.change_username(body['username']) response.redirect('/'+p.username+'/settings/') -if request.headers.get('X-Requested-With') != 'XMLHttpRequest': +if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect('.') [---] text/html diff --git a/www/%username/tip.spt b/www/%username/tip.spt index 159aedbabc..353328ff55 100644 --- a/www/%username/tip.spt +++ b/www/%username/tip.spt @@ -33,7 +33,7 @@ if request.method == 'POST': else: out["msg"] = _("You are now donating {0} per week to {1}. Thank you!", Money(out['amount'], 'EUR'), tippee_name) - if request.headers.get('X-Requested-With') != 'XMLHttpRequest': + if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': back_to = request.body.get('back_to') or tipper.path('giving/') back_to += '&' if '?' in back_to else '?' back_to += 'success=' + b64encode_s(out["msg"]) diff --git a/www/%username/wallet/payin/bankwire/%back_to.spt b/www/%username/wallet/payin/bankwire/%back_to.spt index 569e23203c..72952c0a6d 100644 --- a/www/%username/wallet/payin/bankwire/%back_to.spt +++ b/www/%username/wallet/payin/bankwire/%back_to.spt @@ -41,7 +41,7 @@ if request.method == 'POST' and request.body.get('action') == 'email': ) if not sent: raise Response(500, _("An unknown error occurred.")) - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + if request.headers.get(b'X-Requested-With') == b'XMLHttpRequest': raise Response(200, json.dumps({'msg': _("The email has been sent.")})) else: response.redirect(request.line.uri) diff --git a/www/%username/widgets/%type.spt b/www/%username/widgets/%type.spt index 8eb6386296..344525fd08 100644 --- a/www/%username/widgets/%type.spt +++ b/www/%username/widgets/%type.spt @@ -17,8 +17,8 @@ if participant.hide_giving and t != 'receiving.js' or \ participant.hide_receiving and t != 'giving.js': raise Response(403) -response.headers['Cache-Control'] = 'public, max-age=600' -response.headers['Vary'] = 'Accept-Language' +response.headers[b'Cache-Control'] = b'public, max-age=600' +response.headers[b'Vary'] = b'Accept-Language' [---] application/javascript via jinja2_html_jswrapped diff --git a/www/about/charts.json.spt b/www/about/charts.json.spt index 9407ee7e2f..b889eb39ce 100644 --- a/www/about/charts.json.spt +++ b/www/about/charts.json.spt @@ -21,9 +21,9 @@ charts = [r._asdict() for r in query_cache.all("""\ for c in charts: c['xTitle'] = c.pop('xtitle') # postgres doesn't respect case here -response.headers["Access-Control-Allow-Origin"] = "*" +response.headers[b"Access-Control-Allow-Origin"] = b"*" -response.headers['Cache-Control'] = 'public, max-age=600' +response.headers[b'Cache-Control'] = b'public, max-age=600' [---] application/json via json_dump charts diff --git a/www/about/paydays.spt b/www/about/paydays.spt index 361beb54d9..4293160be6 100644 --- a/www/about/paydays.spt +++ b/www/about/paydays.spt @@ -1,7 +1,7 @@ [---] query = "SELECT * FROM paydays ORDER BY id DESC" -response.headers["Access-Control-Allow-Origin"] = "*" +response.headers[b"Access-Control-Allow-Origin"] = b"*" [---] application/json via json_dump website.db.all(query) diff --git a/www/admin/payday/%id.spt b/www/admin/payday/%id.spt index cbcd40735f..99f548d2dc 100644 --- a/www/admin/payday/%id.spt +++ b/www/admin/payday/%id.spt @@ -60,10 +60,10 @@ def stream_lines_from_file(f, request, response, maxlen=MAXLEN): The standard for HTTP range requests is https://tools.ietf.org/html/rfc7233 """ file_is_partial = f.name.endswith('.part') - req_range = request.headers.get('Range', '').split('=', 1) - if req_range[0] != 'x-lines': + req_range = request.headers.get(b'Range', b'').split(b'=', 1) + if req_range[0] != b'x-lines': raise Response(416, "unknown range specifier") - req_range = req_range[-1].split('-', 1) + req_range = req_range[-1].split(b'-', 1) first_pos = parse_int(req_range[0], default=0) # NOTE we're ignoring the rest of the requested range with file_seek_lock: @@ -83,8 +83,8 @@ def stream_lines_from_file(f, request, response, maxlen=MAXLEN): size = '*' if file_is_partial else os.fstat(f.fileno()).st_size cr = 'x-lines %s-%s/%s' % (first_pos, last_pos, size) # NOTE the range's unit is bytes, not lines - response.headers['Content-Range'] = cr - response.headers['Content-Type'] = 'text/plain' + response.headers[b'Content-Range'] = cr.encode('ascii') + response.headers[b'Content-Type'] = b'text/plain' response.body = data return response diff --git a/www/for/%name/%action.spt b/www/for/%name/%action.spt index 2b16b6f2c2..f134934b3d 100644 --- a/www/for/%name/%action.spt +++ b/www/for/%name/%action.spt @@ -20,7 +20,7 @@ elif action in ('join', 'leave'): else: raise Response(400) -if request.headers.get('X-Requested-With') != 'XMLHttpRequest': +if request.headers.get(b'X-Requested-With') != b'XMLHttpRequest': response.redirect(request.body.get('back_to') or '/for/'+community.name) [---] text/html diff --git a/www/index.html.spt b/www/index.html.spt index a89ca0a064..a50128d959 100644 --- a/www/index.html.spt +++ b/www/index.html.spt @@ -56,7 +56,7 @@ recent_events = query_cache.all(""" - + % endblock diff --git a/www/on/%platform/%user_name/%endpoint.json.spt b/www/on/%platform/%user_name/%endpoint.json.spt index aff5b7f3d9..d34a53b0e4 100644 --- a/www/on/%platform/%user_name/%endpoint.json.spt +++ b/www/on/%platform/%user_name/%endpoint.json.spt @@ -29,8 +29,8 @@ if account.participant.status != 'stub': next_url = '/%s/%s.json' % (account.participant.username, endpoint) next_url += stringify_qs(request.qs) headers = {} - headers["Access-Control-Allow-Origin"] = "*" - headers["Location"] = Response.encode_url(next_url) + headers[b"Access-Control-Allow-Origin"] = b"*" + headers[b"Location"] = Response.encode_url(next_url) raise Response(302, body='*barrel roll*', headers=headers) @@ -61,7 +61,7 @@ if not user.ANON: out["my_tip"] = str(my_tip) -response.headers["Access-Control-Allow-Origin"] = "*" +response.headers[b"Access-Control-Allow-Origin"] = b"*" [---] application/json via jsonp_dump out diff --git a/www/sign-out.spt b/www/sign-out.spt index dcafbc4f3f..4fa6309df6 100644 --- a/www/sign-out.spt +++ b/www/sign-out.spt @@ -11,13 +11,13 @@ if request.method == 'POST': user.sign_out(response.headers.cookie) state['user'] = ANON - if request.headers.get('X-Requested-With') == 'XMLHttpRequest': + if request.headers.get(b'X-Requested-With') == b'XMLHttpRequest': raise Response(200) if 'back_to' in request.body: back_to = request.body['back_to'] else: - back_to = request.headers.get('referer', '/') + back_to = request.headers.get(b'Referer', b'/') response.redirect(back_to) From 0bbf49944626df0792823bf54b26d6a787a799ff Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:03:54 +0200 Subject: [PATCH 05/29] upgrade raven (from 3.1.4 to 5.25.0!) --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 6538de32c0..3dea6fcd71 100644 --- a/requirements.txt +++ b/requirements.txt @@ -30,7 +30,7 @@ monotonic==1.1 fasteners==0.14.1 mangopaysdk==2.1.0 -raven==3.1.4 +raven==5.25.0 six==1.8.0 libsass==0.11.1 From c9ac08d2e2be182bc08b3cadd8fd119970a5cf87 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:07:18 +0200 Subject: [PATCH 06/29] fix hashing of email addresses for gravatar IDs --- liberapay/elsewhere/_base.py | 3 ++- liberapay/models/participant.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/liberapay/elsewhere/_base.py b/liberapay/elsewhere/_base.py index 69d3c9b5ed..1a0b477da0 100644 --- a/liberapay/elsewhere/_base.py +++ b/liberapay/elsewhere/_base.py @@ -200,7 +200,8 @@ def extract_user_info(self, info): if not r.avatar_url: gravatar_id = self.x_gravatar_id(r, info, None) if r.email and not gravatar_id: - gravatar_id = hashlib.md5(r.email.strip().lower()).hexdigest() + bs = r.email.strip().lower().encode('utf8') + gravatar_id = hashlib.md5(bs).hexdigest() if gravatar_id: r.avatar_url = 'https://seccdn.libravatar.org/avatar/'+gravatar_id r.is_team = self.x_is_team(r, info, False) diff --git a/liberapay/models/participant.py b/liberapay/models/participant.py index 767aa9d48c..8e2e56429d 100644 --- a/liberapay/models/participant.py +++ b/liberapay/models/participant.py @@ -1040,7 +1040,7 @@ def update_avatar(self, src=None, cursor=None): if platform == 'libravatar' or platform is None and email: if not email: return - avatar_id = md5(email.strip().lower()).hexdigest() + avatar_id = md5(email.strip().lower().encode('utf8')).hexdigest() avatar_url = 'https://seccdn.libravatar.org/avatar/'+avatar_id avatar_url += AVATAR_QUERY From c9600a856459f877dd8991105237945f97a98220 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:12:47 +0200 Subject: [PATCH 07/29] b64encode always manipulates bytes but we make our `b64encode_s()` function return `str` for ease of use under python3 --- liberapay/models/participant.py | 7 ++++++- liberapay/utils/__init__.py | 4 +++- tests/py/test_hooks.py | 2 +- tests/py/test_utils.py | 4 ++-- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/liberapay/models/participant.py b/liberapay/models/participant.py index 8e2e56429d..cd6aa7dd6a 100644 --- a/liberapay/models/participant.py +++ b/liberapay/models/participant.py @@ -215,7 +215,12 @@ def hash_password(cls, password): salt = urandom(21) rounds = website.app_conf.password_rounds hashed = cls._hash_password(password, algo, salt, rounds) - hashed = '$'.join((algo, str(rounds), b64encode(salt), b64encode(hashed))) + hashed = '$'.join(( + algo, + str(rounds), + b64encode(salt).decode('ascii'), + b64encode(hashed).decode('ascii') + )) return hashed def update_password(self, password, cursor=None): diff --git a/liberapay/utils/__init__.py b/liberapay/utils/__init__.py index 39ed79c30a..796be59e0c 100644 --- a/liberapay/utils/__init__.py +++ b/liberapay/utils/__init__.py @@ -11,6 +11,7 @@ import pickle import re +from six import PY3 from six.moves.urllib.parse import quote as urlquote from pando import Response, json @@ -150,7 +151,8 @@ def b64encode_s(s): s.decode('utf8') except UnicodeError: prefix = b'.' - return prefix + b64encode(s, b'-_').replace(b'=', b'~') + r = prefix + b64encode(s, b'-_').replace(b'=', b'~') + return r.decode('ascii') if PY3 else r def update_global_stats(website): diff --git a/tests/py/test_hooks.py b/tests/py/test_hooks.py index 9a823e53f8..fce69a8865 100644 --- a/tests/py/test_hooks.py +++ b/tests/py/test_hooks.py @@ -55,7 +55,7 @@ def test_session_cookie_not_set_under_basic_auth(self): password = 'password' alice.update_password(password) - auth_header = b'Basic ' + b64encode(b'%s:%s' % (alice.id, password)) + auth_header = b'Basic ' + b64encode(('%s:%s' % (alice.id, password)).encode('ascii')) response = self.client.GET('/alice/public.json', HTTP_AUTHORIZATION=auth_header, HTTP_X_FORWARDED_PROTO=b'https', diff --git a/tests/py/test_utils.py b/tests/py/test_utils.py index eecda860bd..8ed3ff33da 100644 --- a/tests/py/test_utils.py +++ b/tests/py/test_utils.py @@ -128,10 +128,10 @@ def test_safe_base64_transcode_works_with_binary_data(self): def test_b64encode_s_replaces_slash_with_underscore(self): # TheEnter?prise => VGhlRW50ZXI/cHJpc2U= - assert b64encode_s('TheEnter?prise') == 'VGhlRW50ZXI_cHJpc2U~' + assert b64encode_s('TheEnter?prise') == str('VGhlRW50ZXI_cHJpc2U~') def test_b64encode_s_replaces_equals_with_tilde(self): - assert b64encode_s('TheEnterprise') == 'VGhlRW50ZXJwcmlzZQ~~' + assert b64encode_s('TheEnterprise') == str('VGhlRW50ZXJwcmlzZQ~~') def test_b64decode_s_decodes(self): assert b64decode_s('VGhlRW50ZXI_cHJpc2U~') == 'TheEnter?prise' From 35270fd43f50f9aede5f2b7734811e356de814e2 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:19:34 +0200 Subject: [PATCH 08/29] the stdlib cookies expects `str` --- liberapay/constants.py | 2 +- liberapay/security/csrf.py | 7 ++++--- liberapay/testing/__init__.py | 5 +++-- liberapay/utils/__init__.py | 17 +++++++++-------- tests/py/test_callbacks.py | 3 ++- tests/py/test_elsewhere.py | 10 +++++----- tests/py/test_hooks.py | 10 +++++----- tests/py/test_sign_in.py | 4 ++-- www/on/%platform/associate.spt | 4 ++-- 9 files changed, 33 insertions(+), 29 deletions(-) diff --git a/liberapay/constants.py b/liberapay/constants.py index e2eed9dc1f..aba99f53eb 100644 --- a/liberapay/constants.py +++ b/liberapay/constants.py @@ -120,7 +120,7 @@ def check_bits(bits): MC MT NL NO PL PT RO SE SI SK """.split()) -SESSION = b'session' +SESSION = str('session') # bytes in python2, unicode in python3 SESSION_REFRESH = timedelta(hours=1) SESSION_TIMEOUT = timedelta(hours=6) diff --git a/liberapay/security/csrf.py b/liberapay/security/csrf.py index 45aed4ca07..a7ac2b242e 100644 --- a/liberapay/security/csrf.py +++ b/liberapay/security/csrf.py @@ -18,9 +18,10 @@ TOKEN_LENGTH = 32 +CSRF_TOKEN = str('csrf_token') # bytes in python2, unicode in python3 CSRF_TIMEOUT = timedelta(days=7) -_get_new_token = lambda: get_random_string(TOKEN_LENGTH).encode('ascii') +_get_new_token = lambda: get_random_string(TOKEN_LENGTH) _token_re = re.compile(r'^[a-zA-Z0-9]{%d}$' % TOKEN_LENGTH) _sanitize_token = lambda t: t if _token_re.match(t) else None @@ -29,7 +30,7 @@ def extract_token_from_cookie(request): """Given a Request object, return a csrf_token. """ try: - token = request.headers.cookie['csrf_token'].value + token = request.headers.cookie[CSRF_TOKEN].value except KeyError: token = None else: @@ -78,4 +79,4 @@ def add_token_to_response(response, csrf_token=None): if csrf_token: # Don't set httponly so that we can POST using XHR. # https://github.com/gratipay/gratipay.com/issues/3030 - response.set_cookie(b'csrf_token', csrf_token, expires=CSRF_TIMEOUT, httponly=False) + response.set_cookie(CSRF_TOKEN, csrf_token, expires=CSRF_TIMEOUT, httponly=False) diff --git a/liberapay/testing/__init__.py b/liberapay/testing/__init__.py index 81a59f03ab..c831114dce 100644 --- a/liberapay/testing/__init__.py +++ b/liberapay/testing/__init__.py @@ -21,6 +21,7 @@ from liberapay.models.account_elsewhere import AccountElsewhere from liberapay.models.exchange_route import ExchangeRoute from liberapay.models.participant import Participant +from liberapay.security.csrf import CSRF_TOKEN from liberapay.testing.vcr import use_cassette @@ -40,10 +41,10 @@ def build_wsgi_environ(self, *a, **kw): """ # csrf - for both anon and authenticated - csrf_token = kw.get('csrf_token', b'ThisIsATokenThatIsThirtyTwoBytes') + csrf_token = kw.get('csrf_token', 'ThisIsATokenThatIsThirtyTwoBytes') if csrf_token: cookies = kw.setdefault('cookies', {}) - cookies[b'csrf_token'] = csrf_token + cookies[CSRF_TOKEN] = csrf_token kw[b'HTTP_X-CSRF-TOKEN'] = csrf_token # user authentication diff --git a/liberapay/utils/__init__.py b/liberapay/utils/__init__.py index 796be59e0c..f5df9e2c14 100644 --- a/liberapay/utils/__init__.py +++ b/liberapay/utils/__init__.py @@ -200,23 +200,24 @@ def is_card_expired(exp_year, exp_month): return exp_year < cur_year or exp_year == cur_year and exp_month < cur_month -def set_cookie(cookies, key, value, expires=None, httponly=True, path=b'/'): - cookies[key] = value +def set_cookie(cookies, key, value, expires=None, httponly=True, path='/'): + key = str(key) + cookies[key] = str(value) cookie = cookies[key] if expires: if isinstance(expires, timedelta): expires += utcnow() if isinstance(expires, datetime): - expires = to_rfc822(expires).encode('ascii') - cookie[b'expires'] = expires + expires = to_rfc822(expires) + cookie[str('expires')] = str(expires) if httponly: - cookie[b'httponly'] = True + cookie[str('httponly')] = True if path: - cookie[b'path'] = path + cookie[str('path')] = str(path) if website.canonical_domain: - cookie[b'domain'] = website.canonical_domain + cookie[str('domain')] = str(website.canonical_domain) if website.canonical_scheme == 'https': - cookie[b'secure'] = True + cookie[str('secure')] = True def erase_cookie(cookies, key, **kw): diff --git a/tests/py/test_callbacks.py b/tests/py/test_callbacks.py index 4f77a2c3f9..690c294f03 100644 --- a/tests/py/test_callbacks.py +++ b/tests/py/test_callbacks.py @@ -10,6 +10,7 @@ from liberapay.billing.exchanges import Money, record_exchange from liberapay.models.exchange_route import ExchangeRoute +from liberapay.security.csrf import CSRF_TOKEN from liberapay.testing.emails import EmailHarness from liberapay.testing.mangopay import MangopayHarness @@ -41,7 +42,7 @@ def test_payout_callback(self, Get): payout.Tag = str(e_id) Get.return_value = payout r = self.callback(qs) - assert b'csrf_token' not in r.headers.cookie + assert CSRF_TOKEN not in r.headers.cookie assert r.code == 200, r.text homer = homer.refetch() if status == 'succeeded': diff --git a/tests/py/test_elsewhere.py b/tests/py/test_elsewhere.py index 3abd522754..44d225b813 100644 --- a/tests/py/test_elsewhere.py +++ b/tests/py/test_elsewhere.py @@ -21,7 +21,7 @@ def test_associate_csrf(self): def test_associate_with_empty_cookie_raises_400(self): response = self.client.GxT( '/on/github/associate?state=deadbeef', - cookies={b'github_deadbeef': b''}, + cookies={'github_deadbeef': ''}, ) assert response.code == 400 @@ -47,7 +47,7 @@ def test_connect_success(self, gui, gusi, ft): cookie = b64encode_s(json.dumps(['query_data', 'connect', b64encode_s(then), '2'])) response = self.client.GxT('/on/github/associate?state=deadbeef', auth_as=alice, - cookies={b'github_deadbeef': cookie}) + cookies={'github_deadbeef': cookie}) assert response.code == 302, response.text assert response.headers[b'Location'] == then @@ -65,7 +65,7 @@ def test_connect_might_need_confirmation(self, gui, gusi, ft): cookie = b64encode_s(json.dumps(['query_data', 'connect', '', '2'])) response = self.client.GxT('/on/github/associate?state=deadbeef', auth_as=alice, - cookies={b'github_deadbeef': cookie}) + cookies={'github_deadbeef': cookie}) assert response.code == 302 assert response.headers[b'Location'].startswith(b'/on/confirm.html?id=') @@ -75,7 +75,7 @@ def test_connect_failure(self): url = '/on/facebook/associate?error_message=%s&state=deadbeef' % error cookie = b64encode_s(json.dumps(['query_data', 'connect', '', '2'])) response = self.client.GxT(url, auth_as=alice, - cookies={b'facebook_deadbeef': cookie}) + cookies={'facebook_deadbeef': cookie}) assert response.code == 502, response.text assert error in response.text @@ -170,7 +170,7 @@ def setUp(self): Harness.setUp(self) self.alice_elsewhere = self.make_elsewhere('twitter', -1, 'alice') token, expires = self.alice_elsewhere.make_connect_token() - self.connect_cookie = {b'connect_%s' % self.alice_elsewhere.id: token} + self.connect_cookie = {'connect_%s' % self.alice_elsewhere.id: token} self.bob = self.make_participant('bob') def test_confirm(self): diff --git a/tests/py/test_hooks.py b/tests/py/test_hooks.py index fce69a8865..69ffcc737a 100644 --- a/tests/py/test_hooks.py +++ b/tests/py/test_hooks.py @@ -147,21 +147,21 @@ def test_no_csrf_cookie(self): r = self.client.POST('/', csrf_token=False, raise_immediately=False) assert r.code == 403 assert "Bad CSRF cookie" in r.text - assert b'csrf_token' in r.headers.cookie + assert csrf.CSRF_TOKEN in r.headers.cookie def test_bad_csrf_cookie(self): - r = self.client.POST('/', csrf_token=b'bad_token', raise_immediately=False) + r = self.client.POST('/', csrf_token='bad_token', raise_immediately=False) assert r.code == 403 assert "Bad CSRF cookie" in r.text - assert r.headers.cookie[b'csrf_token'].value != 'bad_token' + assert r.headers.cookie[csrf.CSRF_TOKEN].value != 'bad_token' def test_csrf_cookie_set_for_most_requests(self): r = self.client.GET('/') - assert b'csrf_token' in r.headers.cookie + assert csrf.CSRF_TOKEN in r.headers.cookie def test_no_csrf_cookie_set_for_assets(self): r = self.client.GET('/assets/base.css') - assert b'csrf_token' not in r.headers.cookie + assert csrf.CSRF_TOKEN not in r.headers.cookie def test_sanitize_token_passes_through_good_token(self): token = 'ddddeeeeaaaaddddbbbbeeeeeeeeffff' diff --git a/tests/py/test_sign_in.py b/tests/py/test_sign_in.py index 159c3a3c28..9d788f1d48 100644 --- a/tests/py/test_sign_in.py +++ b/tests/py/test_sign_in.py @@ -38,10 +38,10 @@ def check_login(self, r, p): p = p.refetch() # Basic checks assert r.code == 302 - expected = b'%s:%s' % (p.id, p.session_token) + expected = str('%s:%s') % (p.id, p.session_token) sess_cookie = r.headers.cookie[SESSION] assert sess_cookie.value == expected - expires = sess_cookie[b'expires'] + expires = sess_cookie[str('expires')] assert expires.endswith(' GMT') assert parsedate(expires) > gmtime() # More thorough check diff --git a/www/on/%platform/associate.spt b/www/on/%platform/associate.spt index 65fbd0dae8..3ea96e3872 100644 --- a/www/on/%platform/associate.spt +++ b/www/on/%platform/associate.spt @@ -20,7 +20,7 @@ if platform is None: query_id = platform.get_query_id(request.qs) # Check that we have a cookie that matches the query id (CSRF prevention) -cookie_name = (platform.name+'_'+query_id).encode('ascii') +cookie_name = str(platform.name+'_'+query_id) try: cookie_value = request.headers.cookie[cookie_name].value except KeyError: @@ -83,7 +83,7 @@ if action == 'connect': connect_to.take_over((platform.name, account.user_id)) except NeedConfirmation: token, expires = account.make_connect_token() - response.set_cookie(b'connect_%s' % account.id, token, expires) + response.set_cookie('connect_%s' % account.id, token, expires) response.redirect('/on/confirm.html?id=%s&p_id=%s' % (account.id, connect_to.id)) elif action in {'lock', 'unlock'}: From 23892cf75594da9d4c03b644a1fb5f938b795aaa Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:21:39 +0200 Subject: [PATCH 09/29] make CSV renderer compatible with python3 --- liberapay/renderers/csv_dump.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/liberapay/renderers/csv_dump.py b/liberapay/renderers/csv_dump.py index 7b0f29ce65..6ac3a29682 100644 --- a/liberapay/renderers/csv_dump.py +++ b/liberapay/renderers/csv_dump.py @@ -1,21 +1,35 @@ from __future__ import absolute_import, division, print_function, unicode_literals import csv -from io import BytesIO +from io import BytesIO, StringIO + +from six import PY2 from aspen.simplates import renderers +# The python2 version of the csv module doesn't support unicode +codec = 'utf8' + + +def maybe_encode(s): + return s.encode(codec) if PY2 else s + + class Renderer(renderers.Renderer): def render_content(self, context): rows = eval(self.compiled, globals(), context) if not rows: return '' - f = BytesIO() + if PY2: + f = BytesIO() + context['output'].charset = codec + else: + f = StringIO() w = csv.writer(f) if hasattr(rows[0], '_fields'): - w.writerow(rows[0]._fields) + w.writerow(map(maybe_encode, rows[0]._fields)) w.writerows(rows) f.seek(0) return f.read() From cb8999d44f6aa210c6e176664da9aaf4d0fb9e29 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:22:20 +0200 Subject: [PATCH 10/29] update code for parsing of email simplates --- liberapay/utils/emails.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/liberapay/utils/emails.py b/liberapay/utils/emails.py index 5172255122..599c8cfc5a 100644 --- a/liberapay/utils/emails.py +++ b/liberapay/utils/emails.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from aspen.simplates.pagination import parse_specline, split_and_escape +from aspen.simplates.simplate import _decode from aspen_jinja2_renderer import SimplateLoader from jinja2 import Environment @@ -25,10 +26,10 @@ def compile_email_spt(fpath): r = {} - with open(fpath) as f: - pages = list(split_and_escape(f.read())) + with open(fpath, 'rb') as f: + pages = list(split_and_escape(_decode(f.read()))) for i, page in enumerate(pages, 1): - tmpl = b'\n' * page.offset + page.content + tmpl = '\n' * page.offset + page.content content_type, renderer = parse_specline(page.header) key = 'subject' if i == 1 else content_type env = jinja_env_html if content_type == 'text/html' else jinja_env From 3c3065268df91cb3ffcc8607ed5dafd911027de3 Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:24:24 +0200 Subject: [PATCH 11/29] =?UTF-8?q?response.body=20=E2=86=92=20response.text?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tests/py/test_charts_json.py | 14 +++++++------- tests/py/test_communities.py | 20 ++++++++++---------- tests/py/test_elsewhere.py | 4 ++-- tests/py/test_history.py | 8 ++++---- tests/py/test_hooks.py | 2 +- tests/py/test_notifications.py | 4 ++-- tests/py/test_pages.py | 2 +- tests/py/test_public_json.py | 24 ++++++++++++------------ tests/py/test_search.py | 10 +++++----- tests/py/test_stats.py | 2 +- tests/py/test_tip_json.py | 8 ++++---- tests/py/test_tips_json.py | 10 +++++----- 12 files changed, 54 insertions(+), 54 deletions(-) diff --git a/tests/py/test_charts_json.py b/tests/py/test_charts_json.py index 1bf41731b5..bfd92ca0f5 100644 --- a/tests/py/test_charts_json.py +++ b/tests/py/test_charts_json.py @@ -33,13 +33,13 @@ def run_payday(self): def test_no_payday_returns_empty_list(self): - assert json.loads(self.client.GET('/carl/charts.json').body) == [] + assert json.loads(self.client.GET('/carl/charts.json').text) == [] def test_first_payday_comes_through(self): self.run_payday() # first expected = [{"date": today(), "npatrons": 2, "receipts": 3.00}] - actual = json.loads(self.client.GET('/carl/charts.json').body) + actual = json.loads(self.client.GET('/carl/charts.json').text) assert actual == expected @@ -55,7 +55,7 @@ def test_second_payday_comes_through(self): {"date": today(), "npatrons": 1, "receipts": 5.00}, # most recent first {"date": today(), "npatrons": 2, "receipts": 3.00}, ] - actual = json.loads(self.client.GET('/carl/charts.json').body) + actual = json.loads(self.client.GET('/carl/charts.json').text) assert actual == expected @@ -76,7 +76,7 @@ def test_sandwiched_tipless_payday_comes_through(self): {"date": today(), "npatrons": 0, "receipts": 0.00}, {"date": today(), "npatrons": 2, "receipts": 3.00}, ] - actual = json.loads(self.client.GET('/carl/charts.json').body) + actual = json.loads(self.client.GET('/carl/charts.json').text) assert actual == expected @@ -106,7 +106,7 @@ def test_out_of_band_transfer_gets_included_with_prior_payday(self): "receipts": 3.00, }, ] - actual = json.loads(self.client.GET('/carl/charts.json').body) + actual = json.loads(self.client.GET('/carl/charts.json').text) assert actual == expected @@ -116,7 +116,7 @@ def test_never_received_gives_empty_array(self): self.run_payday() # third expected = [] - actual = json.loads(self.client.GET('/alice/charts.json').body) + actual = json.loads(self.client.GET('/alice/charts.json').text) assert actual == expected @@ -134,7 +134,7 @@ def test_transfer_volume(self): "week_withdrawals": '0.00', "xTitle": utcnow().strftime('%Y-%m-%d'), } - actual = json.loads(self.client.GET('/about/charts.json').body)[0] + actual = json.loads(self.client.GET('/about/charts.json').text)[0] assert actual == expected diff --git a/tests/py/test_communities.py b/tests/py/test_communities.py index f3aa35cb4b..c5aa0109ce 100644 --- a/tests/py/test_communities.py +++ b/tests/py/test_communities.py @@ -40,7 +40,7 @@ def test_joining_and_leaving_community(self): {'do': 'join:'+self.c_id}, auth_as=self.alice, xhr=True) - r = json.loads(response.body) + r = json.loads(response.text) assert r == {} response = self.client.POST('/alice/communities.json', @@ -49,11 +49,11 @@ def test_joining_and_leaving_community(self): response = self.client.GET('/alice/communities.json', auth_as=self.alice) - assert len(json.loads(response.body)) == 0 + assert len(json.loads(response.text)) == 0 def test_get_can_get_communities_for_user(self): response = self.client.GET('/alice/communities.json', auth_as=self.alice) - assert len(json.loads(response.body)) == 0 + assert len(json.loads(response.text)) == 0 class TestCommunityActions(Harness): @@ -77,7 +77,7 @@ def test_subscribe_and_unsubscribe(self): self.client.POST('/for/test/unsubscribe', auth_as=self.bob, xhr=True) response = self.client.GET('/bob/communities.json', auth_as=self.bob) - assert len(json.loads(response.body)) == 0 + assert len(json.loads(response.text)) == 0 def test_join_and_leave(self): with self.assertRaises(AuthRequired): @@ -86,12 +86,12 @@ def test_join_and_leave(self): self.client.POST('/for/test/join', auth_as=self.bob, xhr=True) response = self.client.GET('/bob/communities.json', auth_as=self.bob) - assert len(json.loads(response.body)) == 1 + assert len(json.loads(response.text)) == 1 self.client.POST('/for/test/leave', auth_as=self.bob, xhr=True) response = self.client.GET('/bob/communities.json', auth_as=self.bob) - assert len(json.loads(response.body)) == 0 + assert len(json.loads(response.text)) == 0 class TestCommunityEdit(Harness): @@ -142,7 +142,7 @@ def test_get_non_existing_community(self): def test_get_existing_community(self): response = self.client.GET('/for/test/index.json') - result = json.loads(response.body) + result = json.loads(response.text) # assert len(result['animators']) == 2 # Not implemented yet assert result['name'] == 'test' @@ -152,19 +152,19 @@ def test_post_not_supported(self): def test_limit(self): response = self.client.GET('/for/test/index.json?limit=1') - json.loads(response.body) + json.loads(response.text) # assert len(result['animators']) == 1 # Not implemented yet def test_offset(self): response = self.client.GET('/for/test/index.json?offset=1') - json.loads(response.body) + json.loads(response.text) # assert len(result['animators']) == 1 # Not implemented yet def test_max_limit(self): for i in range(110): self.add_participant(str(i)) response = self.client.GET('/for/test/index.json?limit=200') - json.loads(response.body) + json.loads(response.text) # assert len(result['animators']) == 100 # Not implemented yet def test_invalid_limit(self): diff --git a/tests/py/test_elsewhere.py b/tests/py/test_elsewhere.py index 44d225b813..c7f1889793 100644 --- a/tests/py/test_elsewhere.py +++ b/tests/py/test_elsewhere.py @@ -113,7 +113,7 @@ def test_user_page_shows_pledges(self, get_user_info): bob.set_tip_to(alice, amount) assert alice.receiving == amount r = self.client.GET('/on/github/alice/') - assert str(amount) in r.body, r.body.decode('utf8') + assert str(amount) in r.text, r.text @mock.patch('liberapay.elsewhere._base.Platform.get_user_info') def test_user_page_doesnt_fail_on_at_sign(self, get_user_info): @@ -155,7 +155,7 @@ def test_public_json_not_opted_in(self): assert response.code == 200 - data = json.loads(response.body) + data = json.loads(response.text) assert data['on'] == platform.name def test_public_json_opted_in(self): diff --git a/tests/py/test_history.py b/tests/py/test_history.py index 43b8431d1b..3f280f1e55 100644 --- a/tests/py/test_history.py +++ b/tests/py/test_history.py @@ -136,16 +136,16 @@ def setUp(self): def test_export_json(self): r = self.client.GET('/alice/wallet/export.json', auth_as=self.alice) - assert json.loads(r.body) + assert json.loads(r.text) def test_export_json_aggregate(self): r = self.client.GET('/alice/wallet/export.json?mode=aggregate', auth_as=self.alice) - assert json.loads(r.body) + assert json.loads(r.text) def test_export_json_past_year(self): r = self.client.GET('/alice/wallet/export.json?year=%s' % self.past_year, auth_as=self.alice) - assert len(json.loads(r.body)['exchanges']) == 4 + assert len(json.loads(r.text)['exchanges']) == 4 def test_export_csv(self): r = self.client.GET('/alice/wallet/export.csv?key=exchanges', auth_as=self.alice) - assert r.body.count('\n') == 5 + assert r.text.count('\n') == 5 diff --git a/tests/py/test_hooks.py b/tests/py/test_hooks.py index 69ffcc737a..180a527533 100644 --- a/tests/py/test_hooks.py +++ b/tests/py/test_hooks.py @@ -111,7 +111,7 @@ class Tests2(Harness): def test_accept_header_is_respected(self): r = self.client.GET('/about/stats', HTTP_ACCEPT=b'application/json') assert r.headers[b'Content-Type'] == b'application/json; charset=UTF-8' - json.loads(r.body) + json.loads(r.text) def test_error_spt_works(self): r = self.client.POST('/', csrf_token=False, raise_immediately=False) diff --git a/tests/py/test_notifications.py b/tests/py/test_notifications.py index 26888f96b2..b4b718fe1d 100644 --- a/tests/py/test_notifications.py +++ b/tests/py/test_notifications.py @@ -41,7 +41,7 @@ def test_render_notifications(self): team_url='fake_url', inviter='bob', ) - r = self.client.GET('/alice/notifications.html', auth_as=alice).body + r = self.client.GET('/alice/notifications.html', auth_as=alice).text assert ' len([]) = 0.' in r assert ' 0 diff --git a/tests/py/test_tip_json.py b/tests/py/test_tip_json.py index 7f8f5ddb46..9fbfc44454 100644 --- a/tests/py/test_tip_json.py +++ b/tests/py/test_tip_json.py @@ -30,8 +30,8 @@ def test_get_amount_and_total_back_from_api(self): response2 = self.tip(test_tipper, "test_tippee2", "3.00") # Confirm we get back the right amounts. - first_data = json.loads(response1.body) - second_data = json.loads(response2.body) + first_data = json.loads(response1.text) + second_data = json.loads(response2.text) assert first_data['amount'] == "1.00" assert first_data['total_giving'] == "1.00" assert second_data['amount'] == "3.00" @@ -61,14 +61,14 @@ def test_tip_to_unclaimed(self): alice = self.make_elsewhere('twitter', 1, 'alice') bob = self.make_participant("bob") response = self.tip(bob, alice.participant.username, "10.00") - data = json.loads(response.body) + data = json.loads(response.text) assert response.code == 200 assert data['amount'] == "10.00" assert "alice" in data['msg'] # Stop pledging response = self.tip(bob, alice.participant.username, "0.00") - data = json.loads(response.body) + data = json.loads(response.text) assert response.code == 200 assert data['amount'] == "0.00" assert "alice" in data['msg'] diff --git a/tests/py/test_tips_json.py b/tests/py/test_tips_json.py index 8d175c803e..889e992e3b 100644 --- a/tests/py/test_tips_json.py +++ b/tests/py/test_tips_json.py @@ -25,7 +25,7 @@ def also_prune_variant(self, also_prune, tippees=1): ) assert response.code == 200 - assert len(json.loads(response.body)) == 2 + assert len(json.loads(response.text)) == 2 data = [{'username': 'test_tippee2', 'amount': '1.00'}] response = self.client.POST('/test_tipper/tips.json?also_prune=' + also_prune, @@ -38,7 +38,7 @@ def also_prune_variant(self, also_prune, tippees=1): response = self.client.GET('/test_tipper/tips.json', auth_as=test_tipper) assert response.code == 200 - assert len(json.loads(response.body)) == tippees + assert len(json.loads(response.text)) == tippees def test_get_response(self): test_tipper = self.make_participant("test_tipper") @@ -46,7 +46,7 @@ def test_get_response(self): response = self.client.GET('/test_tipper/tips.json', auth_as=test_tipper) assert response.code == 200 - assert len(json.loads(response.body)) == 0 # empty array + assert len(json.loads(response.text)) == 0 # empty array def test_get_response_with_tips(self): self.make_participant("test_tippee1") @@ -59,10 +59,10 @@ def test_get_response_with_tips(self): ) assert response.code == 200 - assert json.loads(response.body)['amount'] == '1.00' + assert json.loads(response.text)['amount'] == '1.00' response = self.client.GET('/test_tipper/tips.json', auth_as=test_tipper) - data = json.loads(response.body)[0] + data = json.loads(response.text)[0] assert response.code == 200 assert data['username'] == 'test_tippee1' From c6666e85eb3d76126b9b8a87fff951278c407c1c Mon Sep 17 00:00:00 2001 From: Changaco Date: Sat, 27 Aug 2016 18:25:26 +0200 Subject: [PATCH 12/29] can't compare `None` to `int` in python3 --- liberapay/models/participant.py | 2 +- www/%username/edit.spt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/liberapay/models/participant.py b/liberapay/models/participant.py index cd6aa7dd6a..6937b2a93a 100644 --- a/liberapay/models/participant.py +++ b/liberapay/models/participant.py @@ -1670,7 +1670,7 @@ def to_dict(self, details=False, inquirer=None): # null - user has no funding goal # 3.00 - user wishes to receive at least this amount if self.goal != 0: - if self.goal > 0: + if self.goal and self.goal > 0: goal = str(self.goal) else: goal = None diff --git a/www/%username/edit.spt b/www/%username/edit.spt index 664bf40b5b..d25f8c5f29 100644 --- a/www/%username/edit.spt +++ b/www/%username/edit.spt @@ -75,7 +75,7 @@ title = participant.username