From 4d25a02cd132d826c5a1345c4c7c7ae9d0be08ab Mon Sep 17 00:00:00 2001 From: user Date: Sun, 10 Jun 2018 15:29:32 +0100 Subject: [PATCH 01/40] Only raise security error on ../ --- src/Site/SiteStorage.py | 2 +- src/Ui/UiRequest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Site/SiteStorage.py b/src/Site/SiteStorage.py index 4d08908e1..843d91250 100644 --- a/src/Site/SiteStorage.py +++ b/src/Site/SiteStorage.py @@ -358,7 +358,7 @@ def getPath(self, inner_path): if not inner_path: return self.directory - if ".." in inner_path: + if "../" in inner_path: raise Exception(u"File not allowed: %s" % inner_path) return u"%s/%s" % (self.directory, inner_path) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 604dea5f7..4b60b981b 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -441,7 +441,7 @@ def parsePath(self, path): if path.endswith("/"): path = path + "index.html" - if ".." in path or "./" in path: + if "../" in path or "./" in path: raise SecurityError("Invalid path") match = re.match("/media/(?P
[A-Za-z0-9]+[A-Za-z0-9\._-]+)(?P/.*|$)", path) From d51e9c68f4f3a547cd208a3841accf1df7f9929c Mon Sep 17 00:00:00 2001 From: ValdikSS Date: Tue, 19 Mar 2019 18:21:47 +0300 Subject: [PATCH 02/40] Update ciphersuites --- src/Crypt/CryptConnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index 15009ff45..f530a0177 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -32,7 +32,7 @@ def selectCrypt(self, client_supported): # Return: wrapped socket def wrapSocket(self, sock, crypt, server=False, cert_pin=None): if crypt == "tls-rsa": - ciphers = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:AES128-GCM-SHA256:AES128-SHA256:HIGH:" + ciphers = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-CBC-SHA:ECDHE-ECDSA-AES128-CBC-SHA:ECDHE-RSA-AES128-CBC-SHA:ECDHE-RSA-AES256-CBC-SHA:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:RSA-AES128-SHA:RSA-AES256-SHA:RSA-DES-192-CBC3-SHA:" ciphers += "!aNULL:!eNULL:!EXPORT:!DSS:!DES:!RC4:!3DES:!MD5:!PSK" if server: sock_wrapped = ssl.wrap_socket( From b4ceb6957c47fce7cb1d4ee36e6245cff8771165 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:31:35 +0100 Subject: [PATCH 03/40] Fix return value of bigfile upload post request --- plugins/Bigfile/BigfilePlugin.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/plugins/Bigfile/BigfilePlugin.py b/plugins/Bigfile/BigfilePlugin.py index 484d2b6d8..f711ca34b 100644 --- a/plugins/Bigfile/BigfilePlugin.py +++ b/plugins/Bigfile/BigfilePlugin.py @@ -4,6 +4,7 @@ import shutil import collections import math +import json import msgpack import gevent @@ -96,12 +97,12 @@ def actionBigfileUpload(self): site.content_manager.contents.loadItem(file_info["content_inner_path"]) # reload cache - return { + return json.dumps({ "merkle_root": merkle_root, "piece_num": len(piecemap_info["sha512_pieces"]), "piece_size": piece_size, "inner_path": inner_path - } + }) def readMultipartHeaders(self, wsgi_input): for i in range(100): From 5716b7505f2626a48f544c030f5bd6f16dfaa57d Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:32:09 +0100 Subject: [PATCH 04/40] Add reason for shutdown --- src/Debug/DebugHook.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py index c3956eed1..87888d2f8 100644 --- a/src/Debug/DebugHook.py +++ b/src/Debug/DebugHook.py @@ -1,5 +1,6 @@ import sys import logging +import signal import gevent import gevent.hub @@ -8,15 +9,15 @@ last_error = None -def shutdown(): - print "Shutting down..." +def shutdown(reason="Unknown"): + logging.info("Shutting down (reason: %s)..." % reason) if "file_server" in dir(sys.modules["main"]) and sys.modules["main"].file_server.running: try: if "file_server" in dir(sys.modules["main"]): gevent.spawn(sys.modules["main"].file_server.stop) if "ui_server" in dir(sys.modules["main"]): gevent.spawn(sys.modules["main"].ui_server.stop) - except Exception, err: + except Exception as err: print "Proper shutdown error: %s" % err sys.exit(0) else: From b8b8ce21fa31f2a39f9e828d210f98ac0a132c2a Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:33:12 +0100 Subject: [PATCH 05/40] Pass kwargs to excepthook --- src/Debug/DebugHook.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py index 87888d2f8..bcae34ede 100644 --- a/src/Debug/DebugHook.py +++ b/src/Debug/DebugHook.py @@ -24,7 +24,7 @@ def shutdown(reason="Unknown"): sys.exit(0) # Store last error, ignore notify, allow manual error logging -def handleError(*args): +def handleError(*args, **kwargs): global last_error if not args: # Manual called args = sys.exc_info() @@ -33,22 +33,23 @@ def handleError(*args): silent = False if args[0].__name__ != "Notify": last_error = args + if args[0].__name__ == "KeyboardInterrupt": - shutdown() - return - if not silent and args[0].__name__ != "Notify": + shutdown("Keyboard interrupt") + elif not silent and args[0].__name__ != "Notify": logging.exception("Unhandled exception") if "greenlet.py" not in args[2].tb_frame.f_code.co_filename: # Don't display error twice - sys.__excepthook__(*args) + sys.__excepthook__(*args, **kwargs) # Ignore notify errors -def handleErrorNotify(*args): - if args[0].__name__ == "KeyboardInterrupt": - shutdown() - if args[0].__name__ != "Notify": - logging.exception("Unhandled exception") - sys.__excepthook__(*args) +def handleErrorNotify(*args, **kwargs): + err = args[0] + if err.__name__ == "KeyboardInterrupt": + shutdown("Keyboard interrupt") + elif err.__name__ != "Notify": + logging.error("Unhandled exception: %s" % [args]) + sys.__excepthook__(*args, **kwargs) if config.debug: # Keep last error for /Debug From 74ce0c50ff315ac3ed5e63981f8b41877c479cb4 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:33:27 +0100 Subject: [PATCH 06/40] Proper shutdown at sigterm --- src/Debug/DebugHook.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Debug/DebugHook.py b/src/Debug/DebugHook.py index bcae34ede..1e96125dc 100644 --- a/src/Debug/DebugHook.py +++ b/src/Debug/DebugHook.py @@ -81,6 +81,12 @@ def handleGreenletError(self, context, type, value, tb): gevent.hub.Hub.handle_error = handleGreenletError +try: + signal.signal(signal.SIGTERM, lambda signum, stack_frame: shutdown("SIGTERM")) +except Exception as err: + logging.debug("Error setting up SIGTERM watcher: %s" % err) + + if __name__ == "__main__": import time from gevent import monkey From c19823793838b68cef5f5fad4be2479b414016ae Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:33:41 +0100 Subject: [PATCH 07/40] Remove signal watcher from zeronet.py --- zeronet.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/zeronet.py b/zeronet.py index 8de613473..f75c9e4d5 100755 --- a/zeronet.py +++ b/zeronet.py @@ -11,12 +11,6 @@ def main(): main = None try: - import signal - try: - signal.signal(signal.SIGTERM, lambda signum, stack_frame: sys.exit(0)) - except Exception as err: - print("Error setting up SIGTERM watcher: %s" % err) - app_dir = os.path.dirname(os.path.abspath(__file__)) os.chdir(app_dir) # Change working dir to zeronet.py dir sys.path.insert(0, os.path.join(app_dir, "src/lib")) # External liblary directory From ef892e91da4ffca85f2f8e7fe65d116601d06b13 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:34:27 +0100 Subject: [PATCH 08/40] Add random padding to handshake --- src/Connection/Connection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Connection/Connection.py b/src/Connection/Connection.py index b5f7ae70d..4edd33a29 100644 --- a/src/Connection/Connection.py +++ b/src/Connection/Connection.py @@ -1,5 +1,6 @@ import socket import time +import random import gevent import msgpack @@ -172,7 +173,7 @@ def connect(self): self.sock.connect(sock_address) # Detect protocol - self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo()}) + self.send({"cmd": "handshake", "req_id": 0, "params": self.getHandshakeInfo(), "random": "A" * random.randint(0, 1024)}) event_connected = self.event_connected gevent.spawn(self.messageLoop) connect_res = event_connected.get() # Wait for handshake From 06be430b74fb89b43f1c331fd5210d2b8de81e45 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sat, 23 Mar 2019 03:35:13 +0100 Subject: [PATCH 09/40] Rev3857 --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index fd7b06bc4..80d0e838b 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3853 + self.rev = 3857 self.argv = argv self.action = None self.pending_changes = {} From f66cfc9a5ebeb25909ca5ccf83e52198bfe5a3a7 Mon Sep 17 00:00:00 2001 From: ValdikSS Date: Tue, 19 Mar 2019 18:09:13 +0300 Subject: [PATCH 10/40] Less obvious fake TLS certificate generation This patch adds the following: * Pre-defined CA certificate subjects * Pre-defined popular website domain names * Fake certificate generation for pre-defined popular website domain signed by fake CA with pre-defined subject It should look less suspicious than "example.com" certificates --- src/Crypt/CryptConnection.py | 109 +++++++++++++++++++++++++++--- src/lib/opensslVerify/openssl.cnf | 21 ++---- 2 files changed, 104 insertions(+), 26 deletions(-) diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index 15009ff45..bb089bb5e 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -3,6 +3,7 @@ import os import ssl import hashlib +import random from Config import config from util import SslPatch @@ -11,6 +12,7 @@ class CryptConnectionManager: def __init__(self): + # TODO: UGLY UGLY UGLY # OpenSSL params if sys.platform.startswith("win"): self.openssl_bin = "src\\lib\\opensslVerify\\openssl.exe" @@ -19,6 +21,11 @@ def __init__(self): self.openssl_env = {"OPENSSL_CONF": "src/lib/opensslVerify/openssl.cnf"} self.crypt_supported = [] # Supported cryptos + self.cacert_pem = config.data_dir+"/cacert-rsa.pem" + self.cakey_pem = config.data_dir+"/cakey-rsa.pem" + self.cert_pem = config.data_dir+"/cert-rsa.pem" + self.cert_csr = config.data_dir+"/cert-rsa.csr" + self.key_pem = config.data_dir+"/key-rsa.pem" # Select crypt that supported by both sides # Return: Name of the crypto @@ -36,8 +43,8 @@ def wrapSocket(self, sock, crypt, server=False, cert_pin=None): ciphers += "!aNULL:!eNULL:!EXPORT:!DSS:!DES:!RC4:!3DES:!MD5:!PSK" if server: sock_wrapped = ssl.wrap_socket( - sock, server_side=server, keyfile='%s/key-rsa.pem' % config.data_dir, - certfile='%s/cert-rsa.pem' % config.data_dir, ciphers=ciphers) + sock, server_side=server, keyfile=self.key_pem, + certfile=self.cert_pem, ciphers=ciphers) else: sock_wrapped = ssl.wrap_socket(sock, ciphers=ciphers) if cert_pin: @@ -50,7 +57,7 @@ def wrapSocket(self, sock, crypt, server=False, cert_pin=None): def removeCerts(self): if config.keep_ssl_cert: return False - for file_name in ["cert-rsa.pem", "key-rsa.pem"]: + for file_name in ["cert-rsa.pem", "key-rsa.pem", "cacert-rsa.pem", "cakey-rsa.pem", "cert-rsa.csr"]: file_path = "%s/%s" % (config.data_dir, file_name) if os.path.isfile(file_path): os.unlink(file_path) @@ -66,15 +73,60 @@ def loadCerts(self): # Try to create RSA server cert + sign for connection encryption # Return: True on success def createSslRsaCert(self): - if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir): + casubjects = [ + "/C=US/O=Amazon/OU=Server CA 1B/CN=Amazon", + "/C=US/O=Let's Encrypt/CN=Let's Encrypt Authority X3", + "/C=US/O=DigiCert Inc/OU=www.digicert.com/CN = DigiCert SHA2 High Assurance Server CA", + "/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN = COMODO RSA Domain Validation Secure Server CA" + ] + fakedomains = [ + "yahoo.com", + "amazon.com", + "live.com", + "microsoft.com", + "mail.ru", + "csdn.net", + "bing.com", + "amazon.co.jp", + "office.com", + "imdb.com", + "msn.com", + "samsung.com", + "huawei.com", + "ztedevices.com", + "godaddy.com", + "w3.org", + "gravatar.com", + "creativecommons.org", + "hatena.ne.jp", + "adobe.com", + "opera.com", + "apache.org", + "rambler.ru", + "one.com", + "nationalgeographic.com", + "networksolutions.com", + "php.net", + "python.org", + "phoca.cz", + "debian.org", + "ubuntu.com", + "nazwa.pl", + "symantec.com" + ] + self.openssl_env['CN'] = random.choice(fakedomains) + + if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem): return True # Files already exits import subprocess - cmd = "%s req -x509 -newkey rsa:2048 -sha256 -batch -keyout %s -out %s -nodes -config %s" % helper.shellquote( + # Generate CAcert and CAkey + cmd = "%s req -new -newkey rsa:2048 -days 3650 -nodes -x509 -subj %s -keyout %s -out %s -batch -config %s" % helper.shellquote( self.openssl_bin, - config.data_dir+"/key-rsa.pem", - config.data_dir+"/cert-rsa.pem", - self.openssl_env["OPENSSL_CONF"] + random.choice(casubjects), + self.cakey_pem, + self.cacert_pem, + self.openssl_env["OPENSSL_CONF"], ) proc = subprocess.Popen( cmd.encode(sys.getfilesystemencoding()), @@ -82,9 +134,46 @@ def createSslRsaCert(self): ) back = proc.stdout.read().strip() proc.wait() - logging.debug("Generating RSA cert and key PEM files...%s" % back) + logging.debug("Generating RSA CAcert and CAkey PEM files...%s" % back) - if os.path.isfile("%s/cert-rsa.pem" % config.data_dir) and os.path.isfile("%s/key-rsa.pem" % config.data_dir): + if not (os.path.isfile(self.cacert_pem) and os.path.isfile(self.cakey_pem)): + logging.error("RSA ECC SSL CAcert generation failed, CAcert or CAkey files not exist.") + return False + + # Generate certificate key and signing request + cmd = "%s req -new -newkey rsa:2048 -keyout %s -out %s -subj %s -sha256 -nodes -batch -config %s" % helper.shellquote( + self.openssl_bin, + self.key_pem, + self.cert_csr, + "/CN="+self.openssl_env['CN'], + self.openssl_env["OPENSSL_CONF"], + ) + proc = subprocess.Popen( + cmd.encode(sys.getfilesystemencoding()), + shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=self.openssl_env + ) + back = proc.stdout.read().strip() + proc.wait() + logging.debug("Generating certificate key and signing request...%s" % back) + + # Sign request and generate certificate + cmd = "%s x509 -req -in %s -CA %s -CAkey %s -CAcreateserial -out %s -days 730 -sha256 -extensions x509_ext -extfile %s" % helper.shellquote( + self.openssl_bin, + self.cert_csr, + self.cacert_pem, + self.cakey_pem, + self.cert_pem, + self.openssl_env["OPENSSL_CONF"], + ) + proc = subprocess.Popen( + cmd.encode(sys.getfilesystemencoding()), + shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=self.openssl_env + ) + back = proc.stdout.read().strip() + proc.wait() + logging.debug("Generating RSA cert...%s" % back) + + if os.path.isfile(self.cert_pem) and os.path.isfile(self.key_pem): return True else: logging.error("RSA ECC SSL cert generation failed, cert or key files not exist.") diff --git a/src/lib/opensslVerify/openssl.cnf b/src/lib/opensslVerify/openssl.cnf index a2c48ba17..611780084 100644 --- a/src/lib/opensslVerify/openssl.cnf +++ b/src/lib/opensslVerify/openssl.cnf @@ -1,5 +1,5 @@ [ req ] -prompt = no +prompt = yes default_bits = 2048 default_keyfile = server-key.pem distinguished_name = subject @@ -32,8 +32,8 @@ authorityKeyIdentifier = keyid,issuer basicConstraints = CA:FALSE keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth subjectAltName = @alternate_names -nsComment = "OpenSSL Generated Certificate" # RFC 5280, Section 4.2.1.12 makes EKU optional # CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused @@ -46,8 +46,8 @@ subjectKeyIdentifier = hash basicConstraints = CA:FALSE keyUsage = digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth, serverAuth subjectAltName = @alternate_names -nsComment = "OpenSSL Generated Certificate" # RFC 5280, Section 4.2.1.12 makes EKU optional # CA/Browser Baseline Requirements, Appendix (B)(3)(G) makes me confused @@ -55,16 +55,5 @@ nsComment = "OpenSSL Generated Certificate" [ alternate_names ] -DNS.1 = example.com -DNS.2 = www.example.com -DNS.3 = mail.example.com -DNS.4 = ftp.example.com - -# Add these if you need them. But usually you don't want them or -# need them in production. You may need them for development. -# DNS.5 = localhost -# DNS.6 = localhost.localdomain -# DNS.7 = 127.0.0.1 - -# IPv6 localhost -# DNS.8 = ::1 \ No newline at end of file +DNS.1 = $ENV::CN +DNS.2 = www.$ENV::CN From e333b47c274feb23b2358bac3f50fe197eec6f4c Mon Sep 17 00:00:00 2001 From: tangdou1 <35254744+tangdou1@users.noreply.github.com> Date: Wed, 27 Mar 2019 09:44:36 +0800 Subject: [PATCH 11/40] maybe a bug --- src/Site/Site.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Site/Site.py b/src/Site/Site.py index 1126d0655..29b00cab9 100644 --- a/src/Site/Site.py +++ b/src/Site/Site.py @@ -90,7 +90,7 @@ def loadSettings(self, settings=None): self.settings = settings if "cache" not in settings: settings["cache"] = {} - if "size_files_optional" not in settings: + if "size_optional" not in settings: settings["size_optional"] = 0 if "optional_downloaded" not in settings: settings["optional_downloaded"] = 0 From d504cdf501d3b6cd353ff8617f46657359ddc1d0 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 27 Mar 2019 02:59:41 +0100 Subject: [PATCH 12/40] Formatting CryptConnection.py --- src/Crypt/CryptConnection.py | 94 +++++++----------------------------- 1 file changed, 17 insertions(+), 77 deletions(-) diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index 16cea1bad..0152cbc04 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -12,7 +12,6 @@ class CryptConnectionManager: def __init__(self): - # TODO: UGLY UGLY UGLY # OpenSSL params if sys.platform.startswith("win"): self.openssl_bin = "src\\lib\\opensslVerify\\openssl.exe" @@ -21,11 +20,12 @@ def __init__(self): self.openssl_env = {"OPENSSL_CONF": "src/lib/opensslVerify/openssl.cnf"} self.crypt_supported = [] # Supported cryptos - self.cacert_pem = config.data_dir+"/cacert-rsa.pem" - self.cakey_pem = config.data_dir+"/cakey-rsa.pem" - self.cert_pem = config.data_dir+"/cert-rsa.pem" - self.cert_csr = config.data_dir+"/cert-rsa.csr" - self.key_pem = config.data_dir+"/key-rsa.pem" + + self.cacert_pem = config.data_dir + "/cacert-rsa.pem" + self.cakey_pem = config.data_dir + "/cakey-rsa.pem" + self.cert_pem = config.data_dir + "/cert-rsa.pem" + self.cert_csr = config.data_dir + "/cert-rsa.csr" + self.key_pem = config.data_dir + "/key-rsa.pem" # Select crypt that supported by both sides # Return: Name of the crypto @@ -44,7 +44,8 @@ def wrapSocket(self, sock, crypt, server=False, cert_pin=None): if server: sock_wrapped = ssl.wrap_socket( sock, server_side=server, keyfile=self.key_pem, - certfile=self.cert_pem, ciphers=ciphers) + certfile=self.cert_pem, ciphers=ciphers + ) else: sock_wrapped = ssl.wrap_socket(sock, ciphers=ciphers) if cert_pin: @@ -76,43 +77,16 @@ def createSslRsaCert(self): casubjects = [ "/C=US/O=Amazon/OU=Server CA 1B/CN=Amazon", "/C=US/O=Let's Encrypt/CN=Let's Encrypt Authority X3", - "/C=US/O=DigiCert Inc/OU=www.digicert.com/CN = DigiCert SHA2 High Assurance Server CA", - "/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN = COMODO RSA Domain Validation Secure Server CA" + "/C=US/O=DigiCert Inc/OU=www.digicert.com/CN=DigiCert SHA2 High Assurance Server CA", + "/C=GB/ST=Greater Manchester/L=Salford/O=COMODO CA Limited/CN=COMODO RSA Domain Validation Secure Server CA" ] fakedomains = [ - "yahoo.com", - "amazon.com", - "live.com", - "microsoft.com", - "mail.ru", - "csdn.net", - "bing.com", - "amazon.co.jp", - "office.com", - "imdb.com", - "msn.com", - "samsung.com", - "huawei.com", - "ztedevices.com", - "godaddy.com", - "w3.org", - "gravatar.com", - "creativecommons.org", - "hatena.ne.jp", - "adobe.com", - "opera.com", - "apache.org", - "rambler.ru", - "one.com", - "nationalgeographic.com", - "networksolutions.com", - "php.net", - "python.org", - "phoca.cz", - "debian.org", - "ubuntu.com", - "nazwa.pl", - "symantec.com" + "yahoo.com", "amazon.com", "live.com", "microsoft.com", "mail.ru", "csdn.net", "bing.com", + "amazon.co.jp", "office.com", "imdb.com", "msn.com", "samsung.com", "huawei.com", "ztedevices.com", + "godaddy.com", "w3.org", "gravatar.com", "creativecommons.org", "hatena.ne.jp", + "adobe.com", "opera.com", "apache.org", "rambler.ru", "one.com", "nationalgeographic.com", + "networksolutions.com", "php.net", "python.org", "phoca.cz", "debian.org", "ubuntu.com", + "nazwa.pl", "symantec.com" ] self.openssl_env['CN'] = random.choice(fakedomains) @@ -145,7 +119,7 @@ def createSslRsaCert(self): self.openssl_bin, self.key_pem, self.cert_csr, - "/CN="+self.openssl_env['CN'], + "/CN=" + self.openssl_env['CN'], self.openssl_env["OPENSSL_CONF"], ) proc = subprocess.Popen( @@ -179,39 +153,5 @@ def createSslRsaCert(self): logging.error("RSA ECC SSL cert generation failed, cert or key files not exist.") return False - # Not used yet: Missing on some platform - """def createSslEccCert(self): - return False - import subprocess - - # Create ECC privatekey - proc = subprocess.Popen( - "%s ecparam -name prime256v1 -genkey -out %s/key-ecc.pem" % (self.openssl_bin, config.data_dir), - shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=self.openssl_env - ) - back = proc.stdout.read().strip() - proc.wait() - self.log.debug("Generating ECC privatekey PEM file...%s" % back) - - # Create ECC cert - proc = subprocess.Popen( - "%s req -new -key %s -x509 -nodes -out %s -config %s" % helper.shellquote( - self.openssl_bin, - config.data_dir+"/key-ecc.pem", - config.data_dir+"/cert-ecc.pem", - self.openssl_env["OPENSSL_CONF"] - ), - shell=True, stderr=subprocess.STDOUT, stdout=subprocess.PIPE, env=self.openssl_env - ) - back = proc.stdout.read().strip() - proc.wait() - self.log.debug("Generating ECC cert PEM file...%s" % back) - - if os.path.isfile("%s/cert-ecc.pem" % config.data_dir) and os.path.isfile("%s/key-ecc.pem" % config.data_dir): - return True - else: - self.logging.error("ECC SSL cert generation failed, cert or key files not exits.") - return False - """ manager = CryptConnectionManager() From cdd0f9cda3c810dad2e101511db47b69241ac181 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 27 Mar 2019 02:59:57 +0100 Subject: [PATCH 13/40] Remove srl file --- src/Crypt/CryptConnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index 0152cbc04..ce29c9444 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -58,7 +58,7 @@ def wrapSocket(self, sock, crypt, server=False, cert_pin=None): def removeCerts(self): if config.keep_ssl_cert: return False - for file_name in ["cert-rsa.pem", "key-rsa.pem", "cacert-rsa.pem", "cakey-rsa.pem", "cert-rsa.csr"]: + for file_name in ["cert-rsa.pem", "key-rsa.pem", "cacert-rsa.pem", "cakey-rsa.pem", "cacert-rsa.srl", "cert-rsa.csr"]: file_path = "%s/%s" % (config.data_dir, file_name) if os.path.isfile(file_path): os.unlink(file_path) From 5ab20317d00e1a8fc23c2024466a4fa4ac468b91 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 27 Mar 2019 03:00:44 +0100 Subject: [PATCH 14/40] Fix ssl compatibility with older clients, prefer chacha20-poly1305 if possible --- src/Crypt/CryptConnection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Crypt/CryptConnection.py b/src/Crypt/CryptConnection.py index ce29c9444..0897d2aff 100644 --- a/src/Crypt/CryptConnection.py +++ b/src/Crypt/CryptConnection.py @@ -39,7 +39,7 @@ def selectCrypt(self, client_supported): # Return: wrapped socket def wrapSocket(self, sock, crypt, server=False, cert_pin=None): if crypt == "tls-rsa": - ciphers = "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-CBC-SHA:ECDHE-ECDSA-AES128-CBC-SHA:ECDHE-RSA-AES128-CBC-SHA:ECDHE-RSA-AES256-CBC-SHA:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA:RSA-AES128-SHA:RSA-AES256-SHA:RSA-DES-192-CBC3-SHA:" + ciphers = "ECDHE-RSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-GCM-SHA256:AES128-SHA256:AES256-SHA:" ciphers += "!aNULL:!eNULL:!EXPORT:!DSS:!DES:!RC4:!3DES:!MD5:!PSK" if server: sock_wrapped = ssl.wrap_socket( From 350adeb52d54830422e3413e4bc1119691ed4b99 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 27 Mar 2019 03:01:39 +0100 Subject: [PATCH 15/40] Fix resource loading with origin only referer --- src/Ui/UiRequest.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 71c380d0f..04374e27e 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -246,7 +246,13 @@ def sendHeader(self, status=200, content_type="text/html", noscript=False, allow headers["Connection"] = "Keep-Alive" headers["Keep-Alive"] = "max=25, timeout=30" headers["X-Frame-Options"] = "SAMEORIGIN" - if content_type != "text/html" and self.env.get("HTTP_REFERER") and self.isSameOrigin(self.getReferer(), self.getRequestUrl()): + is_referer_allowed = False + if self.env.get("HTTP_REFERER"): + if self.isSameOrigin(self.getReferer(), self.getRequestUrl()): + is_referer_allowed = True + elif self.getReferer() == "%s://%s/" % (self.env["wsgi.url_scheme"], self.env["HTTP_HOST"]): # Origin-only referer + is_referer_allowed = True + if content_type != "text/html" and is_referer_allowed: headers["Access-Control-Allow-Origin"] = "*" # Allow load font files from css if noscript: From eb88dbbec8d88f58c5c4c060a61f74c9074e591e Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 27 Mar 2019 03:01:56 +0100 Subject: [PATCH 16/40] Rev3860 --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 80d0e838b..b74c4e687 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3857 + self.rev = 3860 self.argv = argv self.action = None self.pending_changes = {} From b3f677f80656cdc101ad29c4824c0336d6d68725 Mon Sep 17 00:00:00 2001 From: tangdou1 <35254744+tangdou1@users.noreply.github.com> Date: Wed, 27 Mar 2019 11:06:41 +0800 Subject: [PATCH 17/40] sitePause & siteResume are also important settings --- plugins/disabled-Multiuser/MultiuserPlugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/disabled-Multiuser/MultiuserPlugin.py b/plugins/disabled-Multiuser/MultiuserPlugin.py index 8e70d3e31..e3e4b54cc 100644 --- a/plugins/disabled-Multiuser/MultiuserPlugin.py +++ b/plugins/disabled-Multiuser/MultiuserPlugin.py @@ -96,7 +96,7 @@ def getCurrentUser(self): class UiWebsocketPlugin(object): def __init__(self, *args, **kwargs): self.multiuser_denied_cmds = ( - "siteDelete", "configSet", "serverShutdown", "serverUpdate", "siteClone", + "sitePause", "siteResume", "siteDelete", "configSet", "serverShutdown", "serverUpdate", "siteClone", "siteSetOwned", "siteSetAutodownloadoptional", "dbReload", "dbRebuild", "mergerSiteDelete", "siteSetLimit", "siteSetAutodownloadBigfileLimit", "optionalLimitSet", "optionalHelp", "optionalHelpRemove", "optionalHelpAll", "optionalFilePin", "optionalFileUnpin", "optionalFileDelete", From 80f3f9d51104e2035e99a17851ecdc35ab14c925 Mon Sep 17 00:00:00 2001 From: krzotr Date: Wed, 3 Apr 2019 17:01:16 +0200 Subject: [PATCH 18/40] OptionalFileList - get list of not downloaded files We can use API command optionalFileList with parameter filter=not_downloaded to get list of all not downloaded files --- plugins/OptionalManager/UiWebsocketPlugin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/plugins/OptionalManager/UiWebsocketPlugin.py b/plugins/OptionalManager/UiWebsocketPlugin.py index 94d3f5018..879fb0add 100644 --- a/plugins/OptionalManager/UiWebsocketPlugin.py +++ b/plugins/OptionalManager/UiWebsocketPlugin.py @@ -132,8 +132,12 @@ def actionOptionalFileList(self, to, address=None, orderby="time_downloaded DESC wheres_raw = [] if "bigfile" in filter: wheres["size >"] = 1024 * 1024 * 10 - if "downloaded" in filter: + + if "not_downloaded" in filter: + wheres["is_downloaded"] = 0 + elif "downloaded" in filter: wheres_raw.append("(is_downloaded = 1 OR is_pinned = 1)") + if "pinned" in filter: wheres["is_pinned"] = 1 From 27c47bb3bd194c0bef5557ff0a677de3c959c25e Mon Sep 17 00:00:00 2001 From: krzotr Date: Thu, 4 Apr 2019 18:39:27 +0200 Subject: [PATCH 19/40] Added support for encrypted connections in Dockerfile - added OpenSSL library --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 7913cf1da..7fcd83cae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ FROM alpine:3.8 ENV HOME /root #Install ZeroNet -RUN apk --no-cache --no-progress add musl-dev gcc python python-dev py2-pip tor \ +RUN apk --no-cache --no-progress add musl-dev gcc python python-dev py2-pip tor openssl \ && pip install --no-cache-dir gevent msgpack \ && apk del musl-dev gcc python-dev py2-pip \ && echo "ControlPort 9051" >> /etc/tor/torrc \ From 7a59a19df116b0899019cb1d8ca506ffa0a3a027 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sun, 7 Apr 2019 12:11:23 +0200 Subject: [PATCH 20/40] Fix double --open_browser (by imachug) --- start.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/start.py b/start.py index 48d4b21c3..fc2bc0765 100644 --- a/start.py +++ b/start.py @@ -9,7 +9,8 @@ def main(): - sys.argv = [sys.argv[0]]+["--open_browser", "default_browser"]+sys.argv[1:] + if "--open_browser" not in sys.argv: + sys.argv = [sys.argv[0]] + ["--open_browser", "default_browser"] + sys.argv[1:] zeronet.main() if __name__ == '__main__': From c0d81021dfbb4324a9933455ca41e467cfa63b06 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Thu, 11 Apr 2019 00:48:16 +0200 Subject: [PATCH 21/40] Rev3861, Escape error detail to avoid XSS (reported by krzotr) --- src/Config.py | 2 +- src/Ui/UiRequest.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Config.py b/src/Config.py index b74c4e687..7ce63c52d 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3860 + self.rev = 3861 self.argv = argv self.action = None self.pending_changes = {} diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 04374e27e..717c5c0b4 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -799,12 +799,12 @@ def error403(self, message="", details=True): # Send file not found error def error404(self, path=""): self.sendHeader(404) - return self.formatError("Not Found", cgi.escape(path.encode("utf8")), details=False) + return self.formatError("Not Found", path.encode("utf8"), details=False) # Internal server error def error500(self, message=":("): self.sendHeader(500) - return self.formatError("Server error", cgi.escape(message)) + return self.formatError("Server error", message) def formatError(self, title, message, details=True): import sys @@ -828,7 +828,7 @@ def formatError(self, title, message, details=True):

Please report it if you think this an error.

Details:

%s
- """ % (title, message, json.dumps(details, indent=4, sort_keys=True)) + """ % (title, cgi.escape(message), cgi.escape(json.dumps(details, indent=4, sort_keys=True))) else: return """

%s

From 85fd08774fbd9e2cd024297372398bafdd50a972 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Thu, 11 Apr 2019 02:28:01 +0200 Subject: [PATCH 22/40] Send noscript header for error message pages --- src/Ui/UiRequest.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 717c5c0b4..43408534d 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -103,7 +103,7 @@ def route(self, path): extra_headers = {"Access-Control-Allow-Origin": "null"} - self.sendHeader(content_type=content_type, extra_headers=extra_headers) + self.sendHeader(content_type=content_type, extra_headers=extra_headers, noscript=True) return "" if path == "/": @@ -787,23 +787,23 @@ def actionTestStream(self): # Send bad request error def error400(self, message=""): - self.sendHeader(400) + self.sendHeader(400, noscript=True) return self.formatError("Bad Request", message) # You are not allowed to access this def error403(self, message="", details=True): - self.sendHeader(403) + self.sendHeader(403, noscript=True) self.log.error("Error 403: %s" % message) return self.formatError("Forbidden", message, details=details) # Send file not found error def error404(self, path=""): - self.sendHeader(404) + self.sendHeader(404, noscript=True) return self.formatError("Not Found", path.encode("utf8"), details=False) # Internal server error def error500(self, message=":("): - self.sendHeader(500) + self.sendHeader(500, noscript=True) return self.formatError("Server error", message) def formatError(self, title, message, details=True): From 7a217a37412e7f3f7b32e3275cc342887a59ad64 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Thu, 11 Apr 2019 02:28:26 +0200 Subject: [PATCH 23/40] Only display error details in debug mode --- src/Ui/UiRequest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 43408534d..8c4c09ddd 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -810,7 +810,7 @@ def formatError(self, title, message, details=True): import sys import gevent - if details: + if details and config.debug: details = {key: val for key, val in self.env.items() if hasattr(val, "endswith") and "COOKIE" not in key} details["version_zeronet"] = "%s r%s" % (config.version, config.rev) details["version_python"] = sys.version @@ -825,7 +825,7 @@ def formatError(self, title, message, details=True):

%s

%s

-

Please report it if you think this an error.

+

Please report it if you think this an error.

Details:

%s
""" % (title, cgi.escape(message), cgi.escape(json.dumps(details, indent=4, sort_keys=True))) From 719df4ac88a06321298711753cad6a79db002f6e Mon Sep 17 00:00:00 2001 From: shortcutme Date: Thu, 11 Apr 2019 02:28:38 +0200 Subject: [PATCH 24/40] Rev3862 --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 7ce63c52d..0e3bdc969 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3861 + self.rev = 3862 self.argv = argv self.action = None self.pending_changes = {} From 7e78fbc16e9ab8c908b94523cc44513831ef0d1e Mon Sep 17 00:00:00 2001 From: shortcutme Date: Fri, 19 Apr 2019 02:37:27 +0200 Subject: [PATCH 25/40] Ignore invalid shared filters --- plugins/ContentFilter/ContentFilterPlugin.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/plugins/ContentFilter/ContentFilterPlugin.py b/plugins/ContentFilter/ContentFilterPlugin.py index 4c30a1406..05f333765 100644 --- a/plugins/ContentFilter/ContentFilterPlugin.py +++ b/plugins/ContentFilter/ContentFilterPlugin.py @@ -145,7 +145,14 @@ def actionFilterIncludeList(self, to, all_sites=False, filters=False): include_site = filter_storage.site_manager.get(include["address"]) if not include_site: continue - content = include_site.storage.loadJson(include["inner_path"]) + try: + content = include_site.storage.loadJson(include["inner_path"]) + include["error"] = None + except Exception as err: + if include_site.settings["own"]: + include_site.log.warning("Error loading filter %s: %s" % (include["inner_path"], err)) + content = {} + include["error"] = str(err) include["mutes"] = content.get("mutes", {}) include["siteblocks"] = content.get("siteblocks", {}) back.append(include) From 129aff2c0cb1ae0fe7244946cb43236b5c2fbb47 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Fri, 19 Apr 2019 02:38:02 +0200 Subject: [PATCH 26/40] Retry peers only once --- src/Peer/Peer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Peer/Peer.py b/src/Peer/Peer.py index b999257e5..536ecf410 100644 --- a/src/Peer/Peer.py +++ b/src/Peer/Peer.py @@ -146,7 +146,7 @@ def request(self, cmd, params={}, stream_to=None): self.log("Send request: %s %s %s %s" % (params.get("site", ""), cmd, params.get("inner_path", ""), params.get("location", ""))) - for retry in range(1, 4): # Retry 3 times + for retry in range(1, 2): # Retry 1 times try: if not self.connection: raise Exception("No connection found") From 3366edc244db29dac74d7e26087407bceb6d07b5 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Fri, 19 Apr 2019 02:38:41 +0200 Subject: [PATCH 27/40] Rev3863 --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 0e3bdc969..5f2361176 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3862 + self.rev = 3863 self.argv = argv self.action = None self.pending_changes = {} From fd7f724e2b7e076f23d2c15e6fae8e20a5b57c0f Mon Sep 17 00:00:00 2001 From: ValdikSS Date: Mon, 22 Apr 2019 01:14:08 +0300 Subject: [PATCH 28/40] Always translate html files to avoid compatibility problems with brackets in url Backport of commit 90fee9788dc7e0e5465dc035c61b1c8417f674f0 from py3 branch --- plugins/TranslateSite/TranslateSitePlugin.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/plugins/TranslateSite/TranslateSitePlugin.py b/plugins/TranslateSite/TranslateSitePlugin.py index 6eefbb779..e049b6923 100644 --- a/plugins/TranslateSite/TranslateSitePlugin.py +++ b/plugins/TranslateSite/TranslateSitePlugin.py @@ -7,11 +7,18 @@ @PluginManager.registerTo("UiRequest") class UiRequestPlugin(object): def actionSiteMedia(self, path, **kwargs): - file_name = path.split("/")[-1] + file_name = path.split("/")[-1].lower() if not file_name: # Path ends with / file_name = "index.html" extension = file_name.split(".")[-1] - if translate.lang != "en" and extension in ["js", "html"]: + if extension == "html": + should_translate = True + elif extension == "js" and translate.lang != "en": + should_translate = True + else: + should_translate = False + + if should_translate: path_parts = self.parsePath(path) kwargs["header_length"] = False file_generator = super(UiRequestPlugin, self).actionSiteMedia(path, **kwargs) From 9b274415e0983210422199103174f3c56e1d1d6d Mon Sep 17 00:00:00 2001 From: shortcutme Date: Mon, 29 Apr 2019 16:36:33 +0200 Subject: [PATCH 29/40] Rev3864, Fix newsfeed sql query with many parameters --- plugins/Newsfeed/NewsfeedPlugin.py | 11 ++++++----- src/Config.py | 2 +- src/Db/DbCursor.py | 10 +++------- src/util/helper.py | 7 +++++++ 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/plugins/Newsfeed/NewsfeedPlugin.py b/plugins/Newsfeed/NewsfeedPlugin.py index 802fa50b3..4e54fae32 100644 --- a/plugins/Newsfeed/NewsfeedPlugin.py +++ b/plugins/Newsfeed/NewsfeedPlugin.py @@ -4,6 +4,7 @@ from Plugin import PluginManager from Db import DbQuery from Debug import Debug +from util import helper @PluginManager.registerTo("UiWebsocket") @@ -66,14 +67,14 @@ def actionFeedQuery(self, to, limit=10, day_limit=3): query = " UNION ".join(query_parts) if ":params" in query: - query = query.replace(":params", ",".join(["?"] * len(params))) - res = site.storage.query(query + " ORDER BY date_added DESC LIMIT %s" % limit, params * query_raw.count(":params")) - else: - res = site.storage.query(query + " ORDER BY date_added DESC LIMIT %s" % limit) + query_params = map(helper.sqlquote, params) + query = query.replace(":params", ",".join(query_params)) + + res = site.storage.query(query + " ORDER BY date_added DESC LIMIT %s" % limit) except Exception as err: # Log error self.log.error("%s feed query %s error: %s" % (address, name, Debug.formatException(err))) - stats.append({"site": site.address, "feed_name": name, "error": str(err), "query": query}) + stats.append({"site": site.address, "feed_name": name, "error": str(err)}) continue for row in res: diff --git a/src/Config.py b/src/Config.py index 5f2361176..c45056a11 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3863 + self.rev = 3864 self.argv = argv self.action = None self.pending_changes = {} diff --git a/src/Db/DbCursor.py b/src/Db/DbCursor.py index 88d898caa..f397ff0cb 100644 --- a/src/Db/DbCursor.py +++ b/src/Db/DbCursor.py @@ -1,5 +1,7 @@ import time import re +from util import helper + # Special sqlite cursor @@ -12,12 +14,6 @@ def __init__(self, conn, db): self.cursor = conn.cursor() self.logging = False - def quoteValue(self, value): - if type(value) is int: - return str(value) - else: - return "'%s'" % value.replace("'", "''") - def execute(self, query, params=None): self.db.last_query_time = time.time() if isinstance(params, dict) and "?" in query: # Make easier select and insert by allowing dict params @@ -35,7 +31,7 @@ def execute(self, query, params=None): operator = "IN" if len(value) > 100: # Embed values in query to avoid "too many SQL variables" error - query_values = ",".join(map(self.quoteValue, value)) + query_values = ",".join(map(helper.sqlquote, value)) else: query_values = ",".join(["?"] * len(value)) values += value diff --git a/src/util/helper.py b/src/util/helper.py index 9b8cb12ae..1c9b26c68 100644 --- a/src/util/helper.py +++ b/src/util/helper.py @@ -72,6 +72,13 @@ def getFreeSpace(): return free_space +def sqlquote(value): + if type(value) is int: + return str(value) + else: + return "'%s'" % value.replace("'", "''") + + def shellquote(*args): if len(args) == 1: return '"%s"' % args[0].replace('"', "") From f08bea7f904fdac1a22fa39d57b359f68a776cfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Fri, 24 May 2019 18:01:22 +0200 Subject: [PATCH 30/40] Create FUNDING.yml --- .github/FUNDING.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .github/FUNDING.yml diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml new file mode 100644 index 000000000..db8c40a50 --- /dev/null +++ b/.github/FUNDING.yml @@ -0,0 +1 @@ +custom: https://zeronet.io/docs/help_zeronet/donate/ From c165d21d9573164bed0dd5939c547cd862dd739b Mon Sep 17 00:00:00 2001 From: shortcutme Date: Fri, 31 May 2019 15:04:03 +0200 Subject: [PATCH 31/40] Rev3865, Fix ZipStream seek support --- plugins/Sidebar/ZipStream.py | 24 ++++++++++++++++++++---- src/Config.py | 2 +- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/plugins/Sidebar/ZipStream.py b/plugins/Sidebar/ZipStream.py index 9d7de2416..ea6283e47 100644 --- a/plugins/Sidebar/ZipStream.py +++ b/plugins/Sidebar/ZipStream.py @@ -2,12 +2,12 @@ import os import zipfile - class ZipStream(file): def __init__(self, dir_path): self.dir_path = dir_path self.pos = 0 - self.zf = zipfile.ZipFile(self, 'w', zipfile.ZIP_DEFLATED, allowZip64 = True) + self.buff_pos = 0 + self.zf = zipfile.ZipFile(self, 'w', zipfile.ZIP_DEFLATED, allowZip64=True) self.buff = StringIO.StringIO() self.file_list = self.getFileList() @@ -27,6 +27,8 @@ def read(self, size=60 * 1024): self.buff.seek(0) back = self.buff.read() self.buff.truncate(0) + self.buff.seek(0) + self.buff_pos += len(back) return back def write(self, data): @@ -36,8 +38,22 @@ def write(self, data): def tell(self): return self.pos - def seek(self, pos, type): - pass + def seek(self, pos, whence=0): + if pos >= self.buff_pos: + self.buff.seek(pos - self.buff_pos, whence) + self.pos = pos def flush(self): pass + + +if __name__ == "__main__": + zs = ZipStream(".") + out = open("out.zip", "wb") + while 1: + data = zs.read() + print("Write %s" % len(data)) + if not data: + break + out.write(data) + out.close() diff --git a/src/Config.py b/src/Config.py index c45056a11..14fb4bb24 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3864 + self.rev = 3865 self.argv = argv self.action = None self.pending_changes = {} From 7418400b527a797d6e834c8e560b1106e177fe51 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Thu, 6 Jun 2019 02:31:08 +0200 Subject: [PATCH 32/40] Rev3866, Fix and test serving files with null bytes --- plugins/Bigfile/BigfilePlugin.py | 7 ++++--- plugins/Bigfile/Test/TestBigfile.py | 29 +++++++++++++++++++++++++++++ src/Config.py | 2 +- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/plugins/Bigfile/BigfilePlugin.py b/plugins/Bigfile/BigfilePlugin.py index f711ca34b..d9b4ff1dc 100644 --- a/plugins/Bigfile/BigfilePlugin.py +++ b/plugins/Bigfile/BigfilePlugin.py @@ -605,9 +605,10 @@ def isReadable(self, site, inner_path, file, pos): if file.read(10) == "\0" * 10: # Looks empty, but makes sures we don't have that piece file_info = site.content_manager.getFileInfo(inner_path) - piece_i = pos / file_info["piece_size"] - if not site.storage.piecefields[file_info["sha512"]][piece_i]: - return False + if "piece_size" in file_info: + piece_i = pos / file_info["piece_size"] + if not site.storage.piecefields[file_info["sha512"]][piece_i]: + return False # Seek back to position we want to read file.seek(pos) return super(FileRequestPlugin, self).isReadable(site, inner_path, file, pos) diff --git a/plugins/Bigfile/Test/TestBigfile.py b/plugins/Bigfile/Test/TestBigfile.py index 2b71ec8e8..de1266824 100644 --- a/plugins/Bigfile/Test/TestBigfile.py +++ b/plugins/Bigfile/Test/TestBigfile.py @@ -491,3 +491,32 @@ def testFileSize(self, file_server, site, site_temp): site_temp.needFile("%s|%s-%s" % (inner_path, 9 * 1024 * 1024, 10 * 1024 * 1024)) assert site_temp.storage.getSize(inner_path) == site.storage.getSize(inner_path) + + @pytest.mark.parametrize("size", [1024 * 3, 1024 * 1024 * 3, 1024 * 1024 * 30]) + def testNullFileRead(self, file_server, site, site_temp, size): + inner_path = "data/optional.iso" + + f = site.storage.open(inner_path, "w") + f.write("\0" * size) + f.close() + assert site.content_manager.sign("content.json", self.privatekey) + + # Init source server + site.connection_server = file_server + file_server.sites[site.address] = site + + # Init client server + site_temp.connection_server = FileServer(file_server.ip, 1545) + site_temp.connection_server.sites[site_temp.address] = site_temp + site_temp.addPeer(file_server.ip, 1544) + + # Download site + site_temp.download(blind_includes=True).join(timeout=5) + + if "piecemap" in site.content_manager.getFileInfo(inner_path): # Bigfile + site_temp.needFile(inner_path + "|all") + else: + site_temp.needFile(inner_path) + + + assert site_temp.storage.getSize(inner_path) == size diff --git a/src/Config.py b/src/Config.py index 14fb4bb24..4a84e6781 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3865 + self.rev = 3866 self.argv = argv self.action = None self.pending_changes = {} From 028d491294ce4fd61e7ef5b28548b80117ffc646 Mon Sep 17 00:00:00 2001 From: Hugh Isaacs II Date: Fri, 5 Jul 2019 14:42:23 -0400 Subject: [PATCH 33/40] Update Sidebar.css Cross browser grabbing cursor. --- plugins/Sidebar/media/Sidebar.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/Sidebar/media/Sidebar.css b/plugins/Sidebar/media/Sidebar.css index 58b2582ec..3af7957b1 100644 --- a/plugins/Sidebar/media/Sidebar.css +++ b/plugins/Sidebar/media/Sidebar.css @@ -3,8 +3,8 @@ } .drag-bg { width: 100%; height: 100%; position: fixed; } -.fixbutton.dragging { cursor: -webkit-grabbing; } -.fixbutton-bg:active { cursor: -webkit-grabbing; } +.fixbutton.dragging { cursor: -webkit-grabbing; cusor: grabbing; } +.fixbutton-bg:active { cursor: -webkit-grabbing; cusor: grabbing; } .body-sidebar, .body-internals { background-color: #666 !important; } From 861e0855973fdb1c0ef73a5d07ef62acf691e1c1 Mon Sep 17 00:00:00 2001 From: kusky <37901668+kusky33@users.noreply.github.com> Date: Sun, 7 Jul 2019 15:11:53 +0200 Subject: [PATCH 34/40] Update it.json --- src/Translate/languages/it.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Translate/languages/it.json b/src/Translate/languages/it.json index 479923282..f3ee5d87a 100644 --- a/src/Translate/languages/it.json +++ b/src/Translate/languages/it.json @@ -39,7 +39,7 @@ " files needs to be downloaded": " i file devono essere scaricati", " downloaded": " scaricati", " download failed": " scaricamento fallito", - "Peers found: ": "Peer trovati: ", + "Peers found: ": "Peers trovati: ", "No peers found": "Nessun peer trovato", "Running out of size limit (": "Superato il limite di spazio (", "Set limit to \" + site_info.next_size_limit + \"MB": "Imposta il limite a \" + site_info.next_size_limit + \"MB", From 67b78ca12d75cb208a3273ad7c9e412e893cc42e Mon Sep 17 00:00:00 2001 From: shortcutme Date: Sun, 18 Aug 2019 03:20:44 +0200 Subject: [PATCH 35/40] Rev3868, Add origin validation to websocket connections --- src/Config.py | 2 +- src/Ui/UiRequest.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Config.py b/src/Config.py index 4a84e6781..8c9fda74f 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3866 + self.rev = 3868 self.argv = argv self.action = None self.pending_changes = {} diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 7fcc3c9e5..25c9ae0e6 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -710,9 +710,19 @@ def actionFile(self, file_path, block_size=64 * 1024, send_header=True, header_l # On websocket connection def actionWebsocket(self): ws = self.env.get("wsgi.websocket") + if ws: - wrapper_key = self.get["wrapper_key"] + # Allow only same-origin websocket requests + origin = self.env.get("HTTP_ORIGIN") + host = self.env.get("HTTP_HOST") + if origin and host: + origin_host = origin.split("://", 1)[-1] + if host != origin_host: + ws.send(json.dumps({"error": "Invalid origin: %s" % origin})) + return self.error403("Invalid origin: %s" % origin) + # Find site by wrapper_key + wrapper_key = self.get["wrapper_key"] site = None for site_check in self.server.sites.values(): if site_check.settings["wrapper_key"] == wrapper_key: From 27a67d9753110f8f511e3fc9a7a800063ebe22b5 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 28 Aug 2019 01:32:02 +0200 Subject: [PATCH 36/40] Allow websocket connection originates from earlier accepted hostnames --- src/Ui/UiRequest.py | 8 ++++++-- src/Ui/UiServer.py | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 25c9ae0e6..667b90a57 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -416,6 +416,9 @@ def renderWrapper(self, site, path, inner_path, title, extra_headers, show_loadi file_url = "/" + address + "/" + inner_path root_url = "/" + address + "/" + if self.isProxyRequest(): + self.server.allowed_ws_origins.add(self.env["HTTP_HOST"]) + # Wrapper variable inits body_style = "" meta_tags = "" @@ -715,9 +718,10 @@ def actionWebsocket(self): # Allow only same-origin websocket requests origin = self.env.get("HTTP_ORIGIN") host = self.env.get("HTTP_HOST") - if origin and host: + # Allow only same-origin websocket requests + if origin: origin_host = origin.split("://", 1)[-1] - if host != origin_host: + if origin_host != host and origin_host not in self.server.allowed_ws_origins: ws.send(json.dumps({"error": "Invalid origin: %s" % origin})) return self.error403("Invalid origin: %s" % origin) diff --git a/src/Ui/UiServer.py b/src/Ui/UiServer.py index 90b6a31ce..03d56162c 100644 --- a/src/Ui/UiServer.py +++ b/src/Ui/UiServer.py @@ -75,6 +75,7 @@ def __init__(self): else: self.allowed_hosts = set([]) self.allow_trans_proxy = config.ui_trans_proxy + self.allowed_ws_origins = set() self.wrapper_nonces = [] self.add_nonces = [] From a121c23973b1d4f849a377f1bdc1cfe76cce88a3 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 28 Aug 2019 01:32:16 +0200 Subject: [PATCH 37/40] Use re.sub to replace template variables --- src/Ui/UiRequest.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Ui/UiRequest.py b/src/Ui/UiRequest.py index 667b90a57..1a2f4b2a8 100644 --- a/src/Ui/UiRequest.py +++ b/src/Ui/UiRequest.py @@ -293,9 +293,12 @@ def sendHeader(self, status=200, content_type="text/html", noscript=False, allow # Renders a template def render(self, template_path, *args, **kwargs): template = open(template_path).read() - for key, val in kwargs.items(): - template = template.replace("{%s}" % key, "%s" % val) - return template.encode("utf8") + def renderReplacer(m): + return "%s" % kwargs.get(m.group(1), "") + + template_rendered = re.sub("{(.*?)}", renderReplacer, template) + + return template_rendered.encode("utf8") # - Actions - From a9b5561c490995f8ac5658f6bfa9b10e6c1f5c4d Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 28 Aug 2019 01:32:28 +0200 Subject: [PATCH 38/40] Rev3870 --- src/Config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Config.py b/src/Config.py index 8c9fda74f..aab299fd7 100644 --- a/src/Config.py +++ b/src/Config.py @@ -13,7 +13,7 @@ class Config(object): def __init__(self, argv): self.version = "0.6.5" - self.rev = 3868 + self.rev = 3870 self.argv = argv self.action = None self.pending_changes = {} From bf771eda5f2175d4ba24376ebf91ecaafde25c48 Mon Sep 17 00:00:00 2001 From: shortcutme Date: Wed, 28 Aug 2019 01:33:32 +0200 Subject: [PATCH 39/40] Restrict setting open_browser values in config file --- src/Ui/UiWebsocket.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Ui/UiWebsocket.py b/src/Ui/UiWebsocket.py index b407e597f..580ff2af2 100644 --- a/src/Ui/UiWebsocket.py +++ b/src/Ui/UiWebsocket.py @@ -1120,6 +1120,11 @@ def actionConfigSet(self, to, key, value): self.response(to, {"error": "Forbidden you cannot set this config key"}) return + if key == "open_browser": + if value not in ["default_browser", "False"]: + self.response(to, {"error": "Forbidden: Invalid value"}) + return + # Remove empty lines from lists if type(value) is list: value = [line for line in value if line] From 9d0cd908fa21cacb5fcbcb0a2b40cfcb5a1d69f1 Mon Sep 17 00:00:00 2001 From: Shyam sam Date: Sat, 18 Jul 2020 18:32:00 +0530 Subject: [PATCH 40/40] arm64 arch docker image request #2568 --- Dockerfile.arm64v8 | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 Dockerfile.arm64v8 diff --git a/Dockerfile.arm64v8 b/Dockerfile.arm64v8 new file mode 100644 index 000000000..d27b76206 --- /dev/null +++ b/Dockerfile.arm64v8 @@ -0,0 +1,34 @@ +FROM alpine:3.12 + +#Base settings +ENV HOME /root + +COPY requirements.txt /root/requirements.txt + +#Install ZeroNet +RUN apk --update --no-cache --no-progress add python3 python3-dev gcc libffi-dev musl-dev make tor openssl \ + && pip3 install -r /root/requirements.txt \ + && apk del python3-dev gcc libffi-dev musl-dev make \ + && echo "ControlPort 9051" >> /etc/tor/torrc \ + && echo "CookieAuthentication 1" >> /etc/tor/torrc + +RUN python3 -V \ + && python3 -m pip list \ + && tor --version \ + && openssl version + +#Add Zeronet source +COPY . /root +VOLUME /root/data + +#Control if Tor proxy is started +ENV ENABLE_TOR false + +WORKDIR /root + +#Set upstart command +CMD (! ${ENABLE_TOR} || tor&) && python3 zeronet.py --ui_ip 0.0.0.0 --fileserver_port 26552 + +#Expose ports +EXPOSE 43110 26552 +