From 24bbce724701f7bb71a72eb6a30c686d6fcd9d95 Mon Sep 17 00:00:00 2001 From: evilaliv3 Date: Fri, 8 May 2015 00:57:13 +0200 Subject: [PATCH] Fix mixed errors thanks to pylint debugger --- tor2web/t2w.py | 134 ++++++++++++++++++---------------------- tor2web/utils/gettor.py | 2 - 2 files changed, 59 insertions(+), 77 deletions(-) diff --git a/tor2web/t2w.py b/tor2web/t2w.py index 74912b4f..177e87e2 100644 --- a/tor2web/t2w.py +++ b/tor2web/t2w.py @@ -48,20 +48,21 @@ from zope.interface import implements from twisted.spread import pb -from twisted.internet import reactor, protocol, defer +from twisted.internet import reactor, protocol, defer, address from twisted.internet.abstract import isIPAddress, isIPv6Address from twisted.internet.endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint from twisted.protocols.policies import WrappingFactory from twisted.web import http, client, _newclient from twisted.web.error import SchemeNotSupported -from twisted.web.http import datetimeToString, StringTransport, _IdentityTransferDecoder, _ChunkedTransferDecoder, \ - parse_qs +from twisted.web.http import datetimeToString, StringTransport, \ + _IdentityTransferDecoder, _ChunkedTransferDecoder, parse_qs from twisted.web.http_headers import Headers from twisted.web.server import NOT_DONE_YET from twisted.web.template import flattenString, XMLString from twisted.web.iweb import IBodyProducer from twisted.python import log, logfile from twisted.python.compat import networkString, intToBytes +from twisted.python.failure import Failure from twisted.python.filepath import FilePath from twisted.python.log import err from twisted.internet.task import LoopingCall @@ -379,14 +380,13 @@ class RedirectAgent(client.RedirectAgent): """ Overridden client.RedirectAgent version where we evaluate and handle automatically only HTTPS redirects """ - def _handleResponse(self, response, method, uri, headers, redirectCount): locationHeaders = response.headers.getRawHeaders('location', []) if locationHeaders: location = self._resolveLocation(uri, locationHeaders[0]) parsed = client._URI.fromBytes(location) if parsed.scheme == 'https': - return client.RedirectAgent._handleResponse(self, response, method, uri, header, redirectCount) + return client.RedirectAgent._handleResponse(self, response, method, uri, headers, redirectCount) return response @@ -1236,71 +1236,6 @@ def registerProtocol(self, p): def start_worker(): - global antanistaticmap - global templates - global pool - global ports - - ult = LoopingCall(updateListsTask) - ult.start(600) - - # ############################################################################## - # Static Data loading - # Here we make a file caching to not handle I/O - # at run-time and achieve better performance - # ############################################################################## - antanistaticmap = {} - - # system default static files - sys_static_dir = os.path.join(config.sysdatadir, "static/") - if os.path.exists(sys_static_dir): - for root, dirs, files in os.walk(os.path.join(sys_static_dir)): - for basename in files: - filename = os.path.join(root, basename) - f = FilePath(filename) - antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent() - - # user defined static files - usr_static_dir = os.path.join(config.datadir, "static/") - if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir): - for root, dirs, files in os.walk(os.path.join(usr_static_dir)): - for basename in files: - filename = os.path.join(root, basename) - f = FilePath(filename) - antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent() - # ############################################################################## - - # ############################################################################## - # Templates loading - # Here we make a templates cache in order to not handle I/O - # at run-time and achieve better performance - # ############################################################################## - templates = {} - - # system default templates - sys_tpl_dir = os.path.join(config.sysdatadir, "templates/") - if os.path.exists(sys_tpl_dir): - files = FilePath(sys_tpl_dir).globChildren("*.tpl") - for f in files: - f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) - templates[f.basename()] = PageTemplate(XMLString(f.getContent())) - - # user defined templates - usr_tpl_dir = os.path.join(config.datadir, "templates/") - if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir): - files = FilePath(usr_tpl_dir).globChildren("*.tpl") - for f in files: - f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) - templates[f.basename()] = PageTemplate(XMLString(f.getContent())) - # ############################################################################## - - pool = client.HTTPConnectionPool(reactor, True) - pool.maxPersistentPerHost = config.sockmaxpersistentperhost - pool.cachedConnectionTimeout = config.sockcachedconnectiontimeout - pool.retryAutomatically = config.sockretryautomatically - def nullStartedConnecting(self, connector): pass - pool._factory.startedConnecting = nullStartedConnecting - factory = T2WProxyFactory() # we do not want all workers to die in the same moment @@ -1313,7 +1248,6 @@ def nullStartedConnecting(self, connector): pass config.ssl_dh, config.cipher_list) - fds_https, fds_http = [], [] if 'T2W_FDS_HTTPS' in os.environ: fds_https = [int(x) for x in os.environ['T2W_FDS_HTTPS'].split(",") if x] @@ -1364,8 +1298,6 @@ def set_hosts_map(d): global hosts_map hosts_map = d - global config - rpc("get_white_list").addCallback(set_white_list) rpc("get_black_list").addCallback(set_black_list) rpc("get_blocked_ua_list").addCallback(set_blocked_ua_list) @@ -1463,8 +1395,61 @@ def umask(mask): 'html_t2w': re.compile( r'(href|src|url|action)([\ ]*=[\ ]*[\'\"]?)(?:http:|https:)?//([a-z0-9]{16})\.onion([\ \'\"/])', re.I) } -if 'T2W_FDS_HTTPS' not in os.environ and 'T2W_FDS_HTTP' not in os.environ: +# ############################################################################## +# Static Data loading +# Here we make a file caching to not handle I/O +# at run-time and achieve better performance +# ############################################################################## +antanistaticmap = {} + +# system default static files +sys_static_dir = os.path.join(config.sysdatadir, "static/") +if os.path.exists(sys_static_dir): + for root, dirs, files in os.walk(os.path.join(sys_static_dir)): + for basename in files: + filename = os.path.join(root, basename) + f = FilePath(filename) + antanistaticmap[filename.replace(sys_static_dir, "")] = f.getContent() + +# user defined static files +usr_static_dir = os.path.join(config.datadir, "static/") +if usr_static_dir != sys_static_dir and os.path.exists(usr_static_dir): + for root, dirs, files in os.walk(os.path.join(usr_static_dir)): + for basename in files: + filename = os.path.join(root, basename) + f = FilePath(filename) + antanistaticmap[filename.replace(usr_static_dir, "")] = f.getContent() +# ############################################################################## + +templates = {} + +# system default templates +sys_tpl_dir = os.path.join(config.sysdatadir, "templates/") +if os.path.exists(sys_tpl_dir): + files = FilePath(sys_tpl_dir).globChildren("*.tpl") + for f in files: + f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) + templates[f.basename()] = PageTemplate(XMLString(f.getContent())) + +# user defined templates +usr_tpl_dir = os.path.join(config.datadir, "templates/") +if usr_tpl_dir != sys_tpl_dir and os.path.exists(usr_tpl_dir): + files = FilePath(usr_tpl_dir).globChildren("*.tpl") + for f in files: + f = FilePath(config.t2w_file_path(os.path.join('templates', f.basename()))) + templates[f.basename()] = PageTemplate(XMLString(f.getContent())) +# ############################################################################## + +ports = [] +pool = client.HTTPConnectionPool(reactor, True) +pool.maxPersistentPerHost = config.sockmaxpersistentperhost +pool.cachedConnectionTimeout = config.sockcachedconnectiontimeout +pool.retryAutomatically = config.sockretryautomatically +def nullStartedConnecting(self, connector): pass +pool._factory.startedConnecting = nullStartedConnecting + +if 'T2W_FDS_HTTPS' not in os.environ and 'T2W_FDS_HTTP' not in os.environ: set_proctitle("tor2web") def open_listenin_socket(ip, port): @@ -1578,7 +1563,6 @@ def daemon_shutdown(self): t2w_daemon.run(config) else: - set_proctitle("tor2web-worker") white_list = [] diff --git a/tor2web/utils/gettor.py b/tor2web/utils/gettor.py index 0cdf3395..65b66474 100644 --- a/tor2web/utils/gettor.py +++ b/tor2web/utils/gettor.py @@ -43,9 +43,7 @@ from twisted.protocols.basic import FileSender from twisted.python.filepath import FilePath from twisted.python.log import err -from twisted.internet import defer from twisted.web.client import getPage, downloadPage - from twisted.web.server import NOT_DONE_YET from tor2web.utils.lists import List