diff --git a/LICENSE b/LICENSE index d8fae0e5..ecf3f28e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,5 +1,5 @@ Tor2web -Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project +Copyright (C) 2011-2015 Hermes No Profit Association - GlobaLeaks Project This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by diff --git a/tor2web/t2w.py b/tor2web/t2w.py index 74912b4f..9f249459 100644 --- a/tor2web/t2w.py +++ b/tor2web/t2w.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -48,22 +30,22 @@ from zope.interface import implements from twisted.spread import pb -from twisted.internet import reactor, protocol, defer +from twisted.internet import reactor, protocol, defer, address from twisted.internet.abstract import isIPAddress, isIPv6Address from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint from twisted.protocols.policies import WrappingFactory from twisted.web import http, client, _newclient from twisted.web.error import SchemeNotSupported -from twisted.web.http import datetimeToString, StringTransport, _IdentityTransferDecoder, _ChunkedTransferDecoder, \ - parse_qs +from twisted.web.http import datetimeToString, StringTransport, \ + _IdentityTransferDecoder, _ChunkedTransferDecoder, parse_qs from twisted.web.http_headers import Headers from twisted.web.server import NOT_DONE_YET from twisted.web.template import flattenString, XMLString from twisted.web.iweb import IBodyProducer from twisted.python import log, logfile from twisted.python.compat import networkString, intToBytes +from twisted.python.failure import Failure from twisted.python.filepath import FilePath -from twisted.python.log import err from twisted.internet.task import LoopingCall from tor2web import __version__ from tor2web.utils.config import Config @@ -230,7 +212,7 @@ def spawnT2W(father, childFDs, fds_https, fds_http): childFDs=childFDs) -class Tor2webObj(): +class Tor2webObj(object): def __init__(self): # The destination hidden service identifier self.onion = None @@ -379,14 +361,13 @@ class RedirectAgent(client.RedirectAgent): """ Overridden client.RedirectAgent version where we evaluate and handle automatically only HTTPS redirects """ - def _handleResponse(self, response, method, uri, headers, redirectCount): locationHeaders = response.headers.getRawHeaders('location', []) if locationHeaders: location = self._resolveLocation(uri, locationHeaders[0]) parsed = client._URI.fromBytes(location) if parsed.scheme == 'https': - return client.RedirectAgent._handleResponse(self, response, method, uri, header, redirectCount) + return client.RedirectAgent._handleResponse(self, response, method, uri, headers, redirectCount) return response @@ -439,12 +420,6 @@ def __init__(self, channel, queued, reactor=reactor): self.translation_rexp = {} - def finish(self): - try: - http.Request.finish() - except Exception: - pass - def getRequestHostname(self): """ Function overload to fix ipv6 bug: @@ -496,7 +471,7 @@ def getForwarders(self): port)) else: raise Exception - except: + except Exception: return [] return forwarders @@ -841,7 +816,7 @@ def process(self): ctype = ctype[0] if self.method == b"POST" and ctype: - key, pdict = parse_header(ctype) + key, _ = parse_header(ctype) if key == b'application/x-www-form-urlencoded': args.update(parse_qs(content, 1)) # ################################################################ @@ -923,7 +898,7 @@ def process(self): else: if type(antanistaticmap[staticpath]) == str: - filename, ext = os.path.splitext(staticpath) + _, ext = os.path.splitext(staticpath) self.setHeader(b'content-type', mimetypes.types_map[ext]) content = antanistaticmap[staticpath] defer.returnValue(self.contentFinish(content)) @@ -1236,71 +1211,6 @@ def registerProtocol(self, p): def start_worker(): - global antanistaticmap - global templates - global pool - global ports - - ult = LoopingCall(updateListsTask) - ult.start(600) - - # ############################################################################## - # Static Data loading - # Here we make a file caching to not handle I/O - # at run-time and achieve better performance - # ############################################################################## - antanistaticmap = {} - - # system default static files - sys_static_dir = os.path.join(config.sysdatadir, "static/") - if os.path.exists(sys_static_dir): - for root, dirs, files in os.walk(os.path.join(sys_static_dir)): - for basename in files: - filename = os.path.join(root, basename) - f = FilePath(filename) - antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent() - - # user defined static files - usr_static_dir = os.path.join(config.datadir, "static/") - if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir): - for root, dirs, files in os.walk(os.path.join(usr_static_dir)): - for basename in files: - filename = os.path.join(root, basename) - f = FilePath(filename) - antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent() - # ############################################################################## - - # ############################################################################## - # Templates loading - # Here we make a templates cache in order to not handle I/O - # at run-time and achieve better performance - # ############################################################################## - templates = {} - - # system default templates - sys_tpl_dir = os.path.join(config.sysdatadir, "templates/") - if os.path.exists(sys_tpl_dir): - files = FilePath(sys_tpl_dir).globChildren("*.tpl") - for f in files: - f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) - templates[f.basename()] = PageTemplate(XMLString(f.getContent())) - - # user defined templates - usr_tpl_dir = os.path.join(config.datadir, "templates/") - if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir): - files = FilePath(usr_tpl_dir).globChildren("*.tpl") - for f in files: - f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) - templates[f.basename()] = PageTemplate(XMLString(f.getContent())) - # ############################################################################## - - pool = client.HTTPConnectionPool(reactor, True) - pool.maxPersistentPerHost = config.sockmaxpersistentperhost - pool.cachedConnectionTimeout = config.sockcachedconnectiontimeout - pool.retryAutomatically = config.sockretryautomatically - def nullStartedConnecting(self, connector): pass - pool._factory.startedConnecting = nullStartedConnecting - factory = T2WProxyFactory() # we do not want all workers to die in the same moment @@ -1313,7 +1223,6 @@ def nullStartedConnecting(self, connector): pass config.ssl_dh, config.cipher_list) - fds_https, fds_http = [], [] if 'T2W_FDS_HTTPS' in os.environ: fds_https = [int(x) for x in os.environ['T2W_FDS_HTTPS'].split(",") if x] @@ -1364,8 +1273,6 @@ def set_hosts_map(d): global hosts_map hosts_map = d - global config - rpc("get_white_list").addCallback(set_white_list) rpc("get_black_list").addCallback(set_black_list) rpc("get_blocked_ua_list").addCallback(set_blocked_ua_list) @@ -1463,8 +1370,61 @@ def umask(mask): 'html_t2w': re.compile( r'(href|src|url|action)([\ ]*=[\ ]*[\'\"]?)(?:http:|https:)?//([a-z0-9]{16})\.onion([\ \'\"/])', re.I) } -if 'T2W_FDS_HTTPS' not in os.environ and 'T2W_FDS_HTTP' not in os.environ: +# ############################################################################## +# Static Data loading +# Here we make a file caching to not handle I/O +# at run-time and achieve better performance +# ############################################################################## +antanistaticmap = {} + +# system default static files +sys_static_dir = os.path.join(config.sysdatadir, "static/") +if os.path.exists(sys_static_dir): + for root, dirs, files in os.walk(os.path.join(sys_static_dir)): + for basename in files: + filename = os.path.join(root, basename) + f = FilePath(filename) + antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent() + +# user defined static files +usr_static_dir = os.path.join(config.datadir, "static/") +if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir): + for root, dirs, files in os.walk(os.path.join(usr_static_dir)): + for basename in files: + filename = os.path.join(root, basename) + f = FilePath(filename) + antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent() +# ############################################################################## +templates = {} + +# system default templates +sys_tpl_dir = os.path.join(config.sysdatadir, "templates/") +if os.path.exists(sys_tpl_dir): + files = FilePath(sys_tpl_dir).globChildren("*.tpl") + for f in files: + f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) + templates[f.basename()] = PageTemplate(XMLString(f.getContent())) + +# user defined templates +usr_tpl_dir = os.path.join(config.datadir, "templates/") +if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir): + files = FilePath(usr_tpl_dir).globChildren("*.tpl") + for f in files: + f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) + templates[f.basename()] = PageTemplate(XMLString(f.getContent())) +# ############################################################################## + +ports = [] + +pool = client.HTTPConnectionPool(reactor, True) +pool.maxPersistentPerHost = config.sockmaxpersistentperhost +pool.cachedConnectionTimeout = config.sockcachedconnectiontimeout +pool.retryAutomatically = config.sockretryautomatically +def nullStartedConnecting(self, connector): pass +pool._factory.startedConnecting = nullStartedConnecting + +if 'T2W_FDS_HTTPS' not in os.environ and 'T2W_FDS_HTTP' not in os.environ: set_proctitle("tor2web") def open_listenin_socket(ip, port): @@ -1575,10 +1535,9 @@ def daemon_shutdown(self): t2w_daemon.daemon_shutdown = daemon_shutdown t2w_daemon.rpc_server = T2WRPCServer(config) - t2w_daemon.run(config) + t2w_daemon.run() else: - set_proctitle("tor2web-worker") white_list = [] diff --git a/tor2web/utils/config.py b/tor2web/utils/config.py index eb09c95b..f2f838bd 100644 --- a/tor2web/utils/config.py +++ b/tor2web/utils/config.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -57,7 +39,7 @@ def __init__(self): parser.add_option("-n", "--nodaemon", dest="nodaemon", default=False, action="store_true") parser.add_option("-d", "--rundir", dest="rundir", default='/var/run/tor2web/') parser.add_option("-x", "--command", dest="command", default='start') - (options, args) = parser.parse_args() + options, _ = parser.parse_args() self._file = options.configfile diff --git a/tor2web/utils/daemon.py b/tor2web/utils/daemon.py index 6b27a697..83dea3e3 100644 --- a/tor2web/utils/daemon.py +++ b/tor2web/utils/daemon.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -193,7 +175,6 @@ def change_uid(self): os.setuid(c_uid) def run(self, config): - if self.config.command == 'status': if not self.is_process_running(): exit(1) diff --git a/tor2web/utils/gettor.py b/tor2web/utils/gettor.py index 0cdf3395..0415a0d1 100644 --- a/tor2web/utils/gettor.py +++ b/tor2web/utils/gettor.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2015 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -43,9 +25,7 @@ from twisted.protocols.basic import FileSender from twisted.python.filepath import FilePath from twisted.python.log import err -from twisted.internet import defer from twisted.web.client import getPage, downloadPage - from twisted.web.server import NOT_DONE_YET from tor2web.utils.lists import List @@ -284,7 +264,7 @@ def getTorTask(config): try: with open (latest_tb_file, 'r') as version_file: current_version = version_file.read().replace('\n', '') - except: + except Exception: pass if current_version != latest_version: @@ -326,5 +306,5 @@ def getTorTask(config): with open(latest_tb_file, 'w') as version_file: version_file.write(latest_version) - except: + except Exception: pass diff --git a/tor2web/utils/hostsmap.py b/tor2web/utils/hostsmap.py index 429aff4e..262057fd 100644 --- a/tor2web/utils/hostsmap.py +++ b/tor2web/utils/hostsmap.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2014 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -54,5 +36,5 @@ def read(self, path): host = parts[0] onion = parts[1] self.hosts[host] = onion - except: + except Exception: pass diff --git a/tor2web/utils/lists.py b/tor2web/utils/lists.py index 54a330d4..e0434163 100644 --- a/tor2web/utils/lists.py +++ b/tor2web/utils/lists.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -37,7 +19,6 @@ from collections import OrderedDict from StringIO import StringIO -from OpenSSL import SSL from twisted.internet import reactor, ssl from twisted.internet.task import LoopingCall from twisted.internet.defer import Deferred @@ -213,14 +194,14 @@ def load(self): # simple touch to create non existent files try: open(self.filename, 'a').close() - except: + except Exception: pass try: with open(self.filename, 'r') as fh: for l in fh.readlines(): self.add(re.split("#", l)[0].rstrip("[ , \n,\t]")) - except: + except Exception: pass def dump(self): @@ -231,7 +212,7 @@ def dump(self): with open(self.filename, 'w') as fh: for l in self: fh.write(l + "\n") - except: + except Exception: pass def handleData(self, data): diff --git a/tor2web/utils/mail.py b/tor2web/utils/mail.py index 56fef6b7..b6bfdc6e 100644 --- a/tor2web/utils/mail.py +++ b/tor2web/utils/mail.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` diff --git a/tor2web/utils/misc.py b/tor2web/utils/misc.py index dac3d1a1..5ecb7c8b 100644 --- a/tor2web/utils/misc.py +++ b/tor2web/utils/misc.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` diff --git a/tor2web/utils/socks.py b/tor2web/utils/socks.py index 2523fa52..c554eed9 100644 --- a/tor2web/utils/socks.py +++ b/tor2web/utils/socks.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` diff --git a/tor2web/utils/ssl.py b/tor2web/utils/ssl.py index 6f103f53..b311926c 100644 --- a/tor2web/utils/ssl.py +++ b/tor2web/utils/ssl.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -34,7 +16,8 @@ import os from OpenSSL import SSL -from OpenSSL.crypto import load_certificate, dump_certificate, FILETYPE_PEM +from OpenSSL.crypto import load_certificate, dump_certificate, FILETYPE_PEM, + _raise_current_error from pyasn1.type import univ, constraint, char, namedtype, tag from pyasn1.codec.der.decoder import decode from twisted.internet import ssl diff --git a/tor2web/utils/stats.py b/tor2web/utils/stats.py index 349a9343..04550c81 100644 --- a/tor2web/utils/stats.py +++ b/tor2web/utils/stats.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` diff --git a/tor2web/utils/storage.py b/tor2web/utils/storage.py index 1ebbe361..8c317388 100644 --- a/tor2web/utils/storage.py +++ b/tor2web/utils/storage.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` diff --git a/tor2web/utils/templating.py b/tor2web/utils/templating.py index 5962291e..88816acb 100644 --- a/tor2web/utils/templating.py +++ b/tor2web/utils/templating.py @@ -1,21 +1,3 @@ -""" - Tor2web - Copyright (C) 2012 Hermes No Profit Association - GlobaLeaks Project - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Affero General Public License for more details. - - You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . -""" - """ :mod:`Tor2Web` @@ -41,7 +23,7 @@ def lookupRenderMethod(self, name): if method is None: def renderUsingDict(request, tag): if name.startswith("t2wvar-"): - prefix, var = name.split("-") + _, var = name.split("-") if var in request.var: return tag('%s' % request.var[var]) return tag('undefined-var')