From abca316afb385527a8ba21cda9be06aa9b279cf2 Mon Sep 17 00:00:00 2001 From: Nhan Huynh <> Date: Fri, 14 Sep 2018 17:59:24 -0500 Subject: [PATCH 01/33] Support for SNI and dynamic certificate - Dynamically generate a certificate based on client request using Server Name Idicator - Sign the new certificate with either a static CA certificate, or with a newly generated CA - Add config options to specify a path to static CA certificate - *** NOTE ***: This version only works on windows platform --- fakenet/configs/default.ini | 15 ++- fakenet/listeners/HTTPListener.py | 206 ++++++++++++++++++++++++++---- 2 files changed, 189 insertions(+), 32 deletions(-) diff --git a/fakenet/configs/default.ini b/fakenet/configs/default.ini index 33b1a110..5ffc91ee 100644 --- a/fakenet/configs/default.ini +++ b/fakenet/configs/default.ini @@ -29,7 +29,7 @@ NetworkMode: Auto # DebugLevel: specify fine-grained debug print flags to enable. Enabling all # logging when verbose mode is selected results in overwhelming output, hence # this setting. Valid values (comma-separated) are: -# +# # GENPKT Generic packet information # GENPKTV Packet analysis, displays IP, TCP, UDP fields, very wide output # CB Diverter packet handler callback start/finish logging @@ -90,7 +90,7 @@ FixDNS: Yes # ephemeral change. ModifyLocalDNS: Yes -# Enable 'StopDNSService' to stop Windows DNS client to see the actual +# Enable 'StopDNSService' to stop Windows DNS client to see the actual # processes resolving domains. This is a no-op on Linux, until such time as DNS # caching is observed to interfere with finding the pid associated with a DNS # request. @@ -101,16 +101,16 @@ StopDNSService: Yes # 'DefaultUDPListener' will handle TCP and UDP traffic going to unspecified ports. # # NOTE: Setting default UDP listener will intercept all DNS traffic unless you -# enable a dedicated UDP port 53 DNS listener or add UDP port 53 to the +# enable a dedicated UDP port 53 DNS listener or add UDP port 53 to the # 'BlackListPortsUDP' below so that system's default DNS server is used instead. RedirectAllTraffic: Yes DefaultTCPListener: ProxyTCPListener DefaultUDPListener: ProxyUDPListener -# Specify TCP and UDP ports to ignore when diverting packets. +# Specify TCP and UDP ports to ignore when diverting packets. # For example, you may want to avoid diverting UDP port 53 (DNS) traffic -# when trying to intercept a specific process while allowing the rest to +# when trying to intercept a specific process while allowing the rest to # function normally # # NOTE: This setting is only honored when 'RedirectAllTraffic' is enabled. @@ -133,7 +133,7 @@ BlackListPortsUDP: 67, 68, 137, 138, 443, 1900, 5355 # Listener Configuration # # Listener configuration consists of generic settings used by the diverter which -# are the same for all listeners and listener specific settings. +# are the same for all listeners and listener specific settings. # # NOTE: Listener section names will be used for logging. # @@ -289,6 +289,9 @@ Webroot: defaultFiles/ DumpHTTPPosts: Yes DumpHTTPPostsFilePrefix: http Hidden: False +Static_CA: No +CA_Cert: listeners/ssl_utils/server.pem +CA_Key: listeners/ssl_utils/privkey.pem [SMTPListener] Enabled: True diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index 381cff57..f1cd01e1 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -14,7 +14,10 @@ import mimetypes import time - +import shutil +import traceback +import subprocess +from OpenSSL import crypto from . import * MIME_FILE_RESPONSE = { @@ -29,11 +32,41 @@ 'application/xml': 'FakeNet.html' } + + +def load_cert(certpath): + try: + with open(certpath, "rb") as certfile: + data = certfile.read() + cacert = crypto.load_certificate(crypto.FILETYPE_PEM, data) + except: + traceback.print_exc() + cacert = None + return cacert + + +def load_private_key(keypath): + try: + with open(keypath, "rb") as keyfile: + data = keyfile.read() + privkey = crypto.load_privatekey(crypto.FILETYPE_PEM, data) + except: + traceback.print_exc() + privkey = None + return privkey + + class HTTPListener(object): + SSL_UTILS = os.path.join("listeners", "ssl_utils") + CERT_DIR = os.path.join(SSL_UTILS, "temp_certs") + CN="fakenet.flare" + CA_CERT = os.path.join(SSL_UTILS, "server.pem") + CA_KEY = os.path.join(SSL_UTILS, "privkey.pem") + NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 def taste(self, data, dport): - - request_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', + + request_methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'TRACE', 'OPTIONS', 'CONNECT', 'PATCH'] confidence = 1 if dport in [80, 443] else 0 @@ -53,15 +86,15 @@ def taste(self, data, dport): }) def __init__( - self, - config={}, - name='HTTPListener', - logging_level=logging.DEBUG, + self, + config={}, + name='HTTPListener', + logging_level=logging.DEBUG, ): self.logger = logging.getLogger(name) self.logger.setLevel(logging_level) - + self.config = config self.name = name self.local_ip = '0.0.0.0' @@ -80,10 +113,26 @@ def __init__( self.logger.error('Could not locate webroot directory: %s', path) sys.exit(1) + def prepare_certs(self): + if not os.path.isdir(self.CERT_DIR): + os.makedirs(self.CERT_DIR) + + # Generate and add a root CA, which is used to sign for other certs + if self.config.get("static_ca") == "Yes": + # self.ca_cert, self.ca_key = self.CA_CERT, self.CA_KEY + self.ca_cert = self.config.get("ca_cert", "") + self.ca_key = self.config.get("ca_key", "") + msg = "Using the following root CA: %s" % (self.ca_cert,) + self.logger.info(msg) + self.add_root_ca(self.CA_CERT) + else: + self.logger.info("Generating a new Root CA") + self.ca_cert, self.ca_key = self.create_cert(self.CN) + self.add_root_ca(self.ca_cert) def start(self): self.logger.debug('Starting...') - + self.server = ThreadedHTTPServer((self.local_ip, int(self.config.get('port'))), ThreadedHTTPRequestHandler) self.server.logger = self.logger self.server.config = self.config @@ -91,21 +140,13 @@ def start(self): self.server.extensions_map = self.extensions_map if self.config.get('usessl') == 'Yes': + self.prepare_certs() self.logger.debug('Using SSL socket.') - - keyfile_path = 'listeners/ssl_utils/privkey.pem' - keyfile_path = ListenerBase.abs_config_path(keyfile_path) - if keyfile_path is None: - self.logger.error('Could not locate %s', keyfile_path) - sys.exit(1) - - certfile_path = 'listeners/ssl_utils/server.pem' - certfile_path = ListenerBase.abs_config_path(certfile_path) - if certfile_path is None: - self.logger.error('Could not locate %s', certfile_path) - sys.exit(1) - - self.server.socket = ssl.wrap_socket(self.server.socket, keyfile=keyfile_path, certfile=certfile_path, server_side=True, ciphers='RSA') + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + self.logger.error("setting callback to %s" % str(self.sni_callback)) + ctx.set_servername_callback(self.sni_callback) + ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) + self.server.socket = ctx.wrap_socket(self.server.socket, server_side=True) self.server_thread = threading.Thread(target=self.server.serve_forever) self.server_thread.daemon = True @@ -117,6 +158,119 @@ def stop(self): self.server.shutdown() self.server.server_close() + if self.config.get('usessl' == 'Yes'): + cert = load_cert(self.ca_cert) + if cert is not None: + self.remove_root_ca(cert.get_subject().CN) + try: + shutil.rmtree(self.CERT_DIR) + except: + pass + + def sni_callback(self, sslsock, servername, sslctx): + """ + Callback to handle new SSL ClientHello message. The call back MUST + return None for SSL to continue handling the handshake. Any other + numerical values are used as error codes. + """ + newctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + certfile, keyfile = self.create_cert(servername, self.ca_cert, self.ca_key) + if certfile is None or keyfile is None: + return None + + newctx.check_hostname = False + newctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + sslsock.context = newctx + return None + + def add_root_ca(self, ca_cert_file): + try: + subprocess.check_call( + ["certutil", "-addstore", "Root", ca_cert_file], + shell=True, stdout=None + ) + rc = True + except subprocess.CalledProcessError: + rc = False + self.logger.error("Failed to add root CA") + self.logger.error(traceback.format_exc()) + return rc + + def remove_root_ca(self, cn): + try: + subprocess.check_call( + ["certutil", "-delstore", "Root", cn], + shell=True + ) + rc = True + except subprocess.CalledProcessError: + rc = False + self.logger.error("Failed to add root CA") + self.logger.error(traceback.format_exc()) + return rc + + def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): + """ + Create a cert given the common name, a signing CA, CA private key and + the directory output. + + return: tuple(None, None) on error + tuple(cert_file_path, key_file_path) on success + """ + + f_selfsign = ca_cert is None or ca_key is None + certdir = self.CERT_DIR if cert_dir is None else cert_dir + + certfile = os.path.join(certdir, "%s.crt" % (cn)) + keyfile = os.path.join(certdir, "%s.key" % (cn)) + if os.path.exists(certfile): + return certfile, keyfile + + f_selfsign = True + if ca_cert is not None and ca_key is not None: + f_selfsign = False + cacert = load_cert(ca_cert) + if cacert is None: + return None, None + + cakey = load_private_key(ca_key) + if cakey is None: + return None, None + + # generate crypto keys: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, 2048) + + # Create a cert + cert = crypto.X509() + cert.get_subject().C = "US" + cert.get_subject().CN = cn + cert.set_serial_number(0x31337) + now = time.time() / 1000000 + na = int(now + self.NOT_AFTER_DELTA_SECONDS) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(na) + cert.set_pubkey(key) + if f_selfsign: + cert.set_issuer(cert.get_subject()) + cert.sign(key, "sha1") + else: + cert.set_issuer(cacert.get_subject()) + cert.sign(cakey, "sha1") + + try: + with open(certfile, "wb") as cert_file: + cert_file.write(crypto.dump_certificate( + crypto.FILETYPE_PEM, cert) + ) + with open(keyfile, "wb") as key_file: + key_file.write(crypto.dump_privatekey( + crypto.FILETYPE_PEM, key) + ) + except: + traceback.print_exc() + return None, None + return certfile, keyfile class ThreadedHTTPServer(BaseHTTPServer.HTTPServer): @@ -204,7 +358,7 @@ def do_POST(self): http_f.close() else: - self.server.logger.error('Failed to write HTTP POST headers and data to %s.', http_filename) + self.server.logger.error('Failed to write HTTP POST headers and data to %s.', http_filename) # Get response type based on the requested path response, response_type = self.get_response(self.path) @@ -252,7 +406,7 @@ def get_response(self, path): except Exception, e: self.server.logger.error('Failed to open response file: %s', response_filename) response_type = 'text/html' - else: + else: response = f.read() f.close() @@ -293,7 +447,7 @@ def main(): """ logging.basicConfig(format='%(asctime)s [%(name)15s] %(message)s', datefmt='%m/%d/%y %I:%M:%S %p', level=logging.DEBUG) - + config = {'port': '8443', 'usessl': 'Yes', 'webroot': 'fakenet/defaultFiles' } listener = HTTPListener(config) From 03ed32dff22701be763a29d6a47188cbc33266c7 Mon Sep 17 00:00:00 2001 From: htnhan Date: Fri, 18 Jan 2019 01:41:43 -0600 Subject: [PATCH 02/33] De-couple SSL support from HTTPListener --- fakenet/listeners/HTTPListener.py | 163 +++------------------ fakenet/listeners/ssl_utils/__init__.py | 182 ++++++++++++++++++++++++ 2 files changed, 198 insertions(+), 147 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index f1cd01e1..df97c47d 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -18,6 +18,7 @@ import traceback import subprocess from OpenSSL import crypto +from ssl_utils import SSLWrapper from . import * MIME_FILE_RESPONSE = { @@ -34,32 +35,8 @@ -def load_cert(certpath): - try: - with open(certpath, "rb") as certfile: - data = certfile.read() - cacert = crypto.load_certificate(crypto.FILETYPE_PEM, data) - except: - traceback.print_exc() - cacert = None - return cacert - - -def load_private_key(keypath): - try: - with open(keypath, "rb") as keyfile: - data = keyfile.read() - privkey = crypto.load_privatekey(crypto.FILETYPE_PEM, data) - except: - traceback.print_exc() - privkey = None - return privkey - - class HTTPListener(object): SSL_UTILS = os.path.join("listeners", "ssl_utils") - CERT_DIR = os.path.join(SSL_UTILS, "temp_certs") - CN="fakenet.flare" CA_CERT = os.path.join(SSL_UTILS, "server.pem") CA_KEY = os.path.join(SSL_UTILS, "privkey.pem") NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 @@ -113,23 +90,6 @@ def __init__( self.logger.error('Could not locate webroot directory: %s', path) sys.exit(1) - def prepare_certs(self): - if not os.path.isdir(self.CERT_DIR): - os.makedirs(self.CERT_DIR) - - # Generate and add a root CA, which is used to sign for other certs - if self.config.get("static_ca") == "Yes": - # self.ca_cert, self.ca_key = self.CA_CERT, self.CA_KEY - self.ca_cert = self.config.get("ca_cert", "") - self.ca_key = self.config.get("ca_key", "") - msg = "Using the following root CA: %s" % (self.ca_cert,) - self.logger.info(msg) - self.add_root_ca(self.CA_CERT) - else: - self.logger.info("Generating a new Root CA") - self.ca_cert, self.ca_key = self.create_cert(self.CN) - self.add_root_ca(self.ca_cert) - def start(self): self.logger.debug('Starting...') @@ -140,6 +100,7 @@ def start(self): self.server.extensions_map = self.extensions_map if self.config.get('usessl') == 'Yes': + ''' self.prepare_certs() self.logger.debug('Using SSL socket.') ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) @@ -147,6 +108,18 @@ def start(self): ctx.set_servername_callback(self.sni_callback) ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) self.server.socket = ctx.wrap_socket(self.server.socket, server_side=True) + ''' + config = { + 'static_ca': self.config.get('static_ca'), + 'ca_cert': self.config.get('ca_cert'), + 'ca_key': self.config.get('ca_key'), + } + self.sslwrapper = SSLWrapper(config) + self.logger.error('initializing wrapper socket') + if not self.sslwrapper.initialize(): + raise RuntimeError("Failed to initialize SSLWrapper") + self.server.socket = self.sslwrapper.make_socket(self.server.socket) + self.server_thread = threading.Thread(target=self.server.serve_forever) self.server_thread.daemon = True @@ -159,118 +132,14 @@ def stop(self): self.server.server_close() if self.config.get('usessl' == 'Yes'): - cert = load_cert(self.ca_cert) + cert = self.sslwrap._load_cert(self.ca_cert) if cert is not None: - self.remove_root_ca(cert.get_subject().CN) + self.sslwrap._remove_root_ca(cert.get_subject().CN) try: shutil.rmtree(self.CERT_DIR) except: pass - def sni_callback(self, sslsock, servername, sslctx): - """ - Callback to handle new SSL ClientHello message. The call back MUST - return None for SSL to continue handling the handshake. Any other - numerical values are used as error codes. - """ - newctx = ssl.SSLContext(ssl.PROTOCOL_TLS) - certfile, keyfile = self.create_cert(servername, self.ca_cert, self.ca_key) - if certfile is None or keyfile is None: - return None - - newctx.check_hostname = False - newctx.load_cert_chain(certfile=certfile, keyfile=keyfile) - sslsock.context = newctx - return None - - def add_root_ca(self, ca_cert_file): - try: - subprocess.check_call( - ["certutil", "-addstore", "Root", ca_cert_file], - shell=True, stdout=None - ) - rc = True - except subprocess.CalledProcessError: - rc = False - self.logger.error("Failed to add root CA") - self.logger.error(traceback.format_exc()) - return rc - - def remove_root_ca(self, cn): - try: - subprocess.check_call( - ["certutil", "-delstore", "Root", cn], - shell=True - ) - rc = True - except subprocess.CalledProcessError: - rc = False - self.logger.error("Failed to add root CA") - self.logger.error(traceback.format_exc()) - return rc - - def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): - """ - Create a cert given the common name, a signing CA, CA private key and - the directory output. - - return: tuple(None, None) on error - tuple(cert_file_path, key_file_path) on success - """ - - f_selfsign = ca_cert is None or ca_key is None - certdir = self.CERT_DIR if cert_dir is None else cert_dir - - certfile = os.path.join(certdir, "%s.crt" % (cn)) - keyfile = os.path.join(certdir, "%s.key" % (cn)) - if os.path.exists(certfile): - return certfile, keyfile - - f_selfsign = True - if ca_cert is not None and ca_key is not None: - f_selfsign = False - cacert = load_cert(ca_cert) - if cacert is None: - return None, None - - cakey = load_private_key(ca_key) - if cakey is None: - return None, None - - # generate crypto keys: - key = crypto.PKey() - key.generate_key(crypto.TYPE_RSA, 2048) - - # Create a cert - cert = crypto.X509() - cert.get_subject().C = "US" - cert.get_subject().CN = cn - cert.set_serial_number(0x31337) - now = time.time() / 1000000 - na = int(now + self.NOT_AFTER_DELTA_SECONDS) - cert.gmtime_adj_notBefore(0) - cert.gmtime_adj_notAfter(na) - cert.set_pubkey(key) - if f_selfsign: - cert.set_issuer(cert.get_subject()) - cert.sign(key, "sha1") - else: - cert.set_issuer(cacert.get_subject()) - cert.sign(cakey, "sha1") - - try: - with open(certfile, "wb") as cert_file: - cert_file.write(crypto.dump_certificate( - crypto.FILETYPE_PEM, cert) - ) - with open(keyfile, "wb") as key_file: - key_file.write(crypto.dump_privatekey( - crypto.FILETYPE_PEM, key) - ) - except: - traceback.print_exc() - return None, None - return certfile, keyfile class ThreadedHTTPServer(BaseHTTPServer.HTTPServer): diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index e69de29b..78beafd9 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -0,0 +1,182 @@ + + +import time +import os +import traceback +import subprocess +import logging +import shutil +import ssl +from OpenSSL import crypto + + + +class SSLWrapper(object): + NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 + CERT_DIR = "temp_certs" + CN="fakenet.flare" + + def __init__(self, config): + self.logger = logging.getLogger(self.__class__.__name__) + self.config = config + self.ca_cert = None + self.ca_key = None + + def initialize(self): + certdir = self.config.get('certdir', self.CERT_DIR) + self.logger.error("Cert dir is %s", certdir) + if certdir is None: + return False + if not os.path.isdir(certdir): + os.makedirs(certdir) + + # generate and add root CA, which is used to sign for other certs: + if self.config.get('static_ca') == 'Yes': + self.ca_cert = self.config.get('ca_cert', None) + self.ca_key = self.config.get('ca_key', None) + else: + cn = self.CN + if cn is None: + return False + self.ca_cert, self.ca_key = self.create_cert(cn) + self.logger.error("adding root cert: %s", self.ca_cert) + return self._add_root_ca(self.ca_cert) + + def make_socket(self, s): + self.logger.error('making socket') + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.set_servername_callback(self.sni_callback) + ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) + return ctx.wrap_socket(s, server_side=True) + + def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): + """ + Create a cert given the common name, a signing CA, CA private key and + the directory output. + + return: tuple(None, None) on error + tuple(cert_file_path, key_file_path) on success + """ + + f_selfsign = ca_cert is None or ca_key is None + certdir = self.CERT_DIR if cert_dir is None else cert_dir + certfile = os.path.join(certdir, "%s.crt" % (cn)) + keyfile = os.path.join(certdir, "%s.key" % (cn)) + if os.path.exists(certfile): + return certfile, keyfile + + f_selfsign = True + if ca_cert is not None and ca_key is not None: + f_selfsign = False + cacert = self._load_cert(ca_cert) + if cacert is None: + return None, None + + cakey = self._load_private_key(ca_key) + if cakey is None: + return None, None + + # generate crypto keys: + key = crypto.PKey() + key.generate_key(crypto.TYPE_RSA, 2048) + + # Create a cert + + cert = crypto.X509() + cert.get_subject().C = "US" + cert.get_subject().CN = cn + cert.set_serial_number(0x31337) + now = time.time() / 1000000 + na = int(now + self.NOT_AFTER_DELTA_SECONDS) + cert.gmtime_adj_notBefore(0) + cert.gmtime_adj_notAfter(na) + cert.set_pubkey(key) + if f_selfsign: + cert.set_issuer(cert.get_subject()) + cert.sign(key, "sha1") + else: + cert.set_issuer(cacert.get_subject()) + cert.sign(cakey, "sha1") + + try: + with open(certfile, "wb") as cert_file: + cert_file.write(crypto.dump_certificate( + crypto.FILETYPE_PEM, cert) + ) + with open(keyfile, "wb") as key_file: + key_file.write(crypto.dump_privatekey( + crypto.FILETYPE_PEM, key) + ) + except: + traceback.print_exc() + return None, None + return certfile, keyfile + + def sni_callback(self, sslsock, servername, sslctx): + newctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + certfile, keyfile = self.create_cert(servername, self.ca_cert, self.ca_key) + if certfile is None or keyfile is None: + return None + + newctx.check_hostname = False + newctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + sslsock.context = newctx + return None + + def _load_cert(self, certpath): + try: + with open(certpath, 'rb') as certfile: + data = certfile.read() + self.ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) + except: + traceback.format_exc() + self.ca_cert = None + return self.ca_cert + + def _load_private_key(self, keypath): + try: + with open(keypath, 'rb') as keyfile: + data = keyfile.read() + self.privkey = crypto.load_privatekey(crypto.FILETYPE_PEM, data) + except: + traceback.print_exc() + self.privkey = None + return self.privkey + + def _add_root_ca(self, ca_cert_file): + try: + subprocess.check_call( + ["certutil", "-addstore", "Root", ca_cert_file], + shell=True, stdout=None + ) + rc = True + except subprocess.CalledProcessError: + rc = False + self.logger.error("Failed to add root CA") + self.logger.error(traceback.format_exc()) + return rc + + def _remove_root_ca(self, cn): + try: + subprocess.check_call( + ["certutil", "-delstore", "Root", cn], + shell=True + ) + rc = True + except subprocess.CalledProcessError: + rc = False + self.logger.error("Failed to add root CA") + self.logger.error(traceback.format_exc()) + return rc + + def __del__(self): + cert = self._load_cert(self.ca_cert) + if cert is not None: + self._remove_root_ca(cert.get_subject().CN) + try: + shutil.rmtree(self.config.get('certdir', self.CERT_DIR)) + except: + traceback.print_exc() + return + + From ac6a86550facc2d9025cfde29bd26eee5f589bd4 Mon Sep 17 00:00:00 2001 From: htnhan Date: Fri, 18 Jan 2019 01:42:31 -0600 Subject: [PATCH 03/33] ProxyListener to support SSL and handle SSL traffic --- fakenet/configs/default.ini | 8 +++--- fakenet/listeners/ProxyListener.py | 44 ++++++++++++++++-------------- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/fakenet/configs/default.ini b/fakenet/configs/default.ini index 5ffc91ee..17440fa1 100644 --- a/fakenet/configs/default.ini +++ b/fakenet/configs/default.ini @@ -210,6 +210,9 @@ Listener: ProxyListener Port: 38926 Listeners: HTTPListener, RawListener, FTPListener, DNSListener, POPListener, SMTPListener, TFTPListener, IRCListener, BITSListener Hidden: False +Static_CA: No +CA_Cert: listeners/ssl_utils/server.pem +CA_Key: listeners/ssl_utils/privkey.pem [ProxyUDPListener] Enabled: True @@ -280,7 +283,7 @@ DumpHTTPPostsFilePrefix: http Hidden: False [HTTPListener443] -Enabled: True +Enabled: False Port: 443 Protocol: TCP Listener: HTTPListener @@ -289,9 +292,6 @@ Webroot: defaultFiles/ DumpHTTPPosts: Yes DumpHTTPPostsFilePrefix: http Hidden: False -Static_CA: No -CA_Cert: listeners/ssl_utils/server.pem -CA_Key: listeners/ssl_utils/privkey.pem [SMTPListener] Enabled: True diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index 33b193dd..8b7df934 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -10,7 +10,7 @@ import logging import ssl from OpenSSL import SSL -from ssl_utils import ssl_detector +from ssl_utils import ssl_detector, SSLWrapper from . import * import os @@ -155,7 +155,6 @@ def get_top_listener(config, data, listeners, diverter, orig_src_ip, return top_listener class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler): - def handle(self): @@ -166,21 +165,21 @@ def handle(self): remote_q = Queue.Queue() data = None - ssl_remote_sock = None + # ssl_remote_sock = None - keyfile_path = 'listeners/ssl_utils/privkey.pem' - keyfile_path = ListenerBase.abs_config_path(keyfile_path) - if keyfile_path is None: - self.logger.error('Could not locate %s', keyfile_path) - sys.exit(1) + # keyfile_path = 'listeners/ssl_utils/privkey.pem' + # keyfile_path = ListenerBase.abs_config_path(keyfile_path) + # if keyfile_path is None: + # self.logger.error('Could not locate %s', keyfile_path) + # sys.exit(1) - certfile_path = 'listeners/ssl_utils/server.pem' - certfile_path = ListenerBase.abs_config_path(certfile_path) - if certfile_path is None: - self.logger.error('Could not locate %s', certfile_path) - sys.exit(1) + # certfile_path = 'listeners/ssl_utils/server.pem' + # certfile_path = ListenerBase.abs_config_path(certfile_path) + # if certfile_path is None: + # self.logger.error('Could not locate %s', certfile_path) + # sys.exit(1) - ssl_version = ssl.PROTOCOL_SSLv23 + # ssl_version = ssl.PROTOCOL_SSLv23 try: data = remote_sock.recv(BUF_SZ, socket.MSG_PEEK) @@ -197,14 +196,17 @@ def handle(self): if data: if ssl_detector.looks_like_ssl(data): + config = { + 'static_ca': self.server.config.get('static_ca', False), + 'ca_cert': self.server.config.get('ca_cert'), + 'ca_key': self.server.config.get('ca_key') + } + self.sslwrapper = SSLWrapper(config) + if not self.sslwrapper.initialize(): + raise RuntimeError('Failed to initialize SSLWrapper') + self.server.logger.debug('SSL detected') - ssl_remote_sock = ssl.wrap_socket( - remote_sock, - server_side=True, - do_handshake_on_connect=True, - certfile=certfile_path, - ssl_version=ssl_version, - keyfile=keyfile_path ) + ssl_remote_sock = self.sslwrapper.make_socket(remote_sock) data = ssl_remote_sock.recv(BUF_SZ) orig_src_ip = self.client_address[0] From 772902e3e2154eea316527b96d0e95b5f619ea16 Mon Sep 17 00:00:00 2001 From: htnhan Date: Fri, 18 Jan 2019 02:00:42 -0600 Subject: [PATCH 04/33] certutil should only run on windows --- fakenet/listeners/ssl_utils/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 78beafd9..8fe74283 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -6,6 +6,7 @@ import subprocess import logging import shutil +import sys import ssl from OpenSSL import crypto @@ -144,6 +145,8 @@ def _load_private_key(self, keypath): return self.privkey def _add_root_ca(self, ca_cert_file): + if not sys.platform.startswith('win'): + return False try: subprocess.check_call( ["certutil", "-addstore", "Root", ca_cert_file], @@ -157,6 +160,8 @@ def _add_root_ca(self, ca_cert_file): return rc def _remove_root_ca(self, cn): + if not sys.platform.startswith('win'): + return False try: subprocess.check_call( ["certutil", "-delstore", "Root", cn], From 5ca752b2e079e007703341e44e901ad0f3e9136e Mon Sep 17 00:00:00 2001 From: htnhan Date: Fri, 8 Mar 2019 16:21:43 -0600 Subject: [PATCH 05/33] Fix issue: ssl_remote_sock is undefined on Non-SSL sockets --- fakenet/listeners/ProxyListener.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index 8b7df934..e558341c 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -155,7 +155,7 @@ def get_top_listener(config, data, listeners, diverter, orig_src_ip, return top_listener class ThreadedTCPRequestHandler(SocketServer.BaseRequestHandler): - + def handle(self): remote_sock = self.request @@ -208,6 +208,8 @@ def handle(self): self.server.logger.debug('SSL detected') ssl_remote_sock = self.sslwrapper.make_socket(remote_sock) data = ssl_remote_sock.recv(BUF_SZ) + else: + ssl_remote_sock = None orig_src_ip = self.client_address[0] orig_src_port = self.client_address[1] From f49a91b3f950a636e96eeef606a8972c41809f6b Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:12:40 -0500 Subject: [PATCH 06/33] make_socket() now has a fallback in case SSLContext() fails --- fakenet/listeners/ssl_utils/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 8fe74283..fe4ac005 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -45,10 +45,16 @@ def initialize(self): def make_socket(self, s): self.logger.error('making socket') - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) - ctx.set_servername_callback(self.sni_callback) - ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) - return ctx.wrap_socket(s, server_side=True) + try: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + except: + self.logger.error(traceback.format_exc()) + self.logger.error("Exception when calling ssl.SSLContext") + return ctx.wrap_socket(s, server_side=True) + else: + ctx.set_servername_callback(self.sni_callback) + ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) + return ctx.wrap_socket(s, server_side=True) def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): """ From 63c1c5803ca12c22fec46bfa3547470618003019 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:15:26 -0500 Subject: [PATCH 07/33] rename make_socket() to wrap_socket() --- fakenet/listeners/HTTPListener.py | 2 +- fakenet/listeners/ProxyListener.py | 2 +- fakenet/listeners/ssl_utils/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index df97c47d..5d07df3d 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -118,7 +118,7 @@ def start(self): self.logger.error('initializing wrapper socket') if not self.sslwrapper.initialize(): raise RuntimeError("Failed to initialize SSLWrapper") - self.server.socket = self.sslwrapper.make_socket(self.server.socket) + self.server.socket = self.sslwrapper.wrap_socket(self.server.socket) self.server_thread = threading.Thread(target=self.server.serve_forever) diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index e558341c..d7fc2c65 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -206,7 +206,7 @@ def handle(self): raise RuntimeError('Failed to initialize SSLWrapper') self.server.logger.debug('SSL detected') - ssl_remote_sock = self.sslwrapper.make_socket(remote_sock) + ssl_remote_sock = self.sslwrapper.wrap_socket(remote_sock) data = ssl_remote_sock.recv(BUF_SZ) else: ssl_remote_sock = None diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index fe4ac005..e71fe0b9 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -43,7 +43,7 @@ def initialize(self): self.logger.error("adding root cert: %s", self.ca_cert) return self._add_root_ca(self.ca_cert) - def make_socket(self, s): + def wrap_socket(self, s): self.logger.error('making socket') try: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) From 71da060ada12e27238d2438fcefe2168064193ee Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:24:42 -0500 Subject: [PATCH 08/33] SSLWrapper now initialize everything in __init__() --- fakenet/listeners/HTTPListener.py | 3 --- fakenet/listeners/ProxyListener.py | 3 --- fakenet/listeners/ssl_utils/__init__.py | 15 ++++++--------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index 5d07df3d..8c1bb9ec 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -115,9 +115,6 @@ def start(self): 'ca_key': self.config.get('ca_key'), } self.sslwrapper = SSLWrapper(config) - self.logger.error('initializing wrapper socket') - if not self.sslwrapper.initialize(): - raise RuntimeError("Failed to initialize SSLWrapper") self.server.socket = self.sslwrapper.wrap_socket(self.server.socket) diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index d7fc2c65..dd811cef 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -202,9 +202,6 @@ def handle(self): 'ca_key': self.server.config.get('ca_key') } self.sslwrapper = SSLWrapper(config) - if not self.sslwrapper.initialize(): - raise RuntimeError('Failed to initialize SSLWrapper') - self.server.logger.debug('SSL detected') ssl_remote_sock = self.sslwrapper.wrap_socket(remote_sock) data = ssl_remote_sock.recv(BUF_SZ) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index e71fe0b9..ab23c8b9 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -11,7 +11,6 @@ from OpenSSL import crypto - class SSLWrapper(object): NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 CERT_DIR = "temp_certs" @@ -22,12 +21,12 @@ def __init__(self, config): self.config = config self.ca_cert = None self.ca_key = None - - def initialize(self): + certdir = self.config.get('certdir', self.CERT_DIR) self.logger.error("Cert dir is %s", certdir) if certdir is None: - return False + raise RuntimeError("certdir key is not specified in config") + if not os.path.isdir(certdir): os.makedirs(certdir) @@ -36,12 +35,10 @@ def initialize(self): self.ca_cert = self.config.get('ca_cert', None) self.ca_key = self.config.get('ca_key', None) else: - cn = self.CN - if cn is None: - return False - self.ca_cert, self.ca_key = self.create_cert(cn) + self.ca_cert, self.ca_key = self.create_cert(self.CN) self.logger.error("adding root cert: %s", self.ca_cert) - return self._add_root_ca(self.ca_cert) + if not self._add_root_ca(self.ca_cert): + raise RuntimeError("Failed to add root ca") def wrap_socket(self, s): self.logger.error('making socket') From fcf6b5e2663167146e0aad29952d8da81ce33d08 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:27:38 -0500 Subject: [PATCH 09/33] Use proper logging levels --- fakenet/listeners/HTTPListener.py | 2 +- fakenet/listeners/ssl_utils/__init__.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index 8c1bb9ec..2a8d0082 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -104,7 +104,7 @@ def start(self): self.prepare_certs() self.logger.debug('Using SSL socket.') ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) - self.logger.error("setting callback to %s" % str(self.sni_callback)) + self.logger.info("setting callback to %s" % str(self.sni_callback)) ctx.set_servername_callback(self.sni_callback) ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) self.server.socket = ctx.wrap_socket(self.server.socket, server_side=True) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index ab23c8b9..b6e20a6d 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -23,7 +23,7 @@ def __init__(self, config): self.ca_key = None certdir = self.config.get('certdir', self.CERT_DIR) - self.logger.error("Cert dir is %s", certdir) + self.logger.info("Cert dir is %s", certdir) if certdir is None: raise RuntimeError("certdir key is not specified in config") @@ -36,12 +36,12 @@ def __init__(self, config): self.ca_key = self.config.get('ca_key', None) else: self.ca_cert, self.ca_key = self.create_cert(self.CN) - self.logger.error("adding root cert: %s", self.ca_cert) + self.logger.info("adding root cert: %s", self.ca_cert) if not self._add_root_ca(self.ca_cert): raise RuntimeError("Failed to add root ca") def wrap_socket(self, s): - self.logger.error('making socket') + self.logger.info('making socket') try: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) except: From 1987f4363fb45a54ae4a875a80fa0030a534f68c Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:29:32 -0500 Subject: [PATCH 10/33] Clean up logic --- fakenet/listeners/ssl_utils/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index b6e20a6d..b92b74ff 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -63,9 +63,10 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): """ f_selfsign = ca_cert is None or ca_key is None - certdir = self.CERT_DIR if cert_dir is None else cert_dir - certfile = os.path.join(certdir, "%s.crt" % (cn)) - keyfile = os.path.join(certdir, "%s.key" % (cn)) + if not cert_dir: + cert_dir = self.CERT_DIR + certfile = os.path.join(cert_dir, "%s.crt" % (cn)) + keyfile = os.path.join(cert_dir, "%s.key" % (cn)) if os.path.exists(certfile): return certfile, keyfile From a9e6005c11778dce94cdf0cda8c4d9f1f2d5c096 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:31:38 -0500 Subject: [PATCH 11/33] Fix: failed logic to check make sure both certfile and keyfile exist --- fakenet/listeners/ssl_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index b92b74ff..e5f069dd 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -67,7 +67,7 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): cert_dir = self.CERT_DIR certfile = os.path.join(cert_dir, "%s.crt" % (cn)) keyfile = os.path.join(cert_dir, "%s.key" % (cn)) - if os.path.exists(certfile): + if os.path.exists(certfile) and os.path.exists(keyfile) return certfile, keyfile f_selfsign = True From ed762b457c1fa17a800d758312d37ef2a41c16e9 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:41:26 -0500 Subject: [PATCH 12/33] Fix missing : --- fakenet/listeners/ssl_utils/__init__.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index e5f069dd..3d24bb1f 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -67,13 +67,11 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): cert_dir = self.CERT_DIR certfile = os.path.join(cert_dir, "%s.crt" % (cn)) keyfile = os.path.join(cert_dir, "%s.key" % (cn)) - if os.path.exists(certfile) and os.path.exists(keyfile) + if os.path.exists(certfile) and os.path.exists(keyfile): return certfile, keyfile - f_selfsign = True if ca_cert is not None and ca_key is not None: - f_selfsign = False - cacert = self._load_cert(ca_cert) + `` cacert = self._load_cert(ca_cert) if cacert is None: return None, None From d648c8d0ec08226387bf9261a12c1159c1a335ef Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:44:09 -0500 Subject: [PATCH 13/33] Fix: Invalid syntax -- random back ticks --- fakenet/listeners/ssl_utils/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 3d24bb1f..4c26b54f 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -71,7 +71,7 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): return certfile, keyfile if ca_cert is not None and ca_key is not None: - `` cacert = self._load_cert(ca_cert) + cacert = self._load_cert(ca_cert) if cacert is None: return None, None From b923868b01803648aca28733d1338a7e66ab6cc7 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:57:30 -0500 Subject: [PATCH 14/33] Various renames --- fakenet/listeners/ssl_utils/__init__.py | 48 ++++++++++++------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 4c26b54f..edf58bdf 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -50,7 +50,7 @@ def wrap_socket(self, s): return ctx.wrap_socket(s, server_side=True) else: ctx.set_servername_callback(self.sni_callback) - ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) + ctx.load_cert_chain(cert_file=self.ca_cert, keyfile=self.ca_key) return ctx.wrap_socket(s, server_side=True) def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): @@ -65,18 +65,18 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): f_selfsign = ca_cert is None or ca_key is None if not cert_dir: cert_dir = self.CERT_DIR - certfile = os.path.join(cert_dir, "%s.crt" % (cn)) - keyfile = os.path.join(cert_dir, "%s.key" % (cn)) - if os.path.exists(certfile) and os.path.exists(keyfile): - return certfile, keyfile + cert_file = os.path.join(cert_dir, "%s.crt" % (cn)) + key_file = os.path.join(cert_dir, "%s.key" % (cn)) + if os.path.exists(cert_file) and os.path.exists(key_file): + return cert_file, key_file if ca_cert is not None and ca_key is not None: - cacert = self._load_cert(ca_cert) - if cacert is None: + ca_cert_data = self._load_cert(ca_cert) + if ca_cert_data is None: return None, None - cakey = self._load_private_key(ca_key) - if cakey is None: + ca_key_data = self._load_private_key(ca_key) + if ca_key_data is None: return None, None # generate crypto keys: @@ -98,38 +98,38 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): cert.set_issuer(cert.get_subject()) cert.sign(key, "sha1") else: - cert.set_issuer(cacert.get_subject()) - cert.sign(cakey, "sha1") + cert.set_issuer(ca_cert_data.get_subject()) + cert.sign(ca_key_data, "sha1") try: - with open(certfile, "wb") as cert_file: - cert_file.write(crypto.dump_certificate( + with open(cert_file, "wb") as cert_file_input: + cert_file_input.write(crypto.dump_certificate( crypto.FILETYPE_PEM, cert) ) - with open(keyfile, "wb") as key_file: + with open(key_file, "wb") as key_file: key_file.write(crypto.dump_privatekey( crypto.FILETYPE_PEM, key) ) except: traceback.print_exc() return None, None - return certfile, keyfile + return cert_file, key_file def sni_callback(self, sslsock, servername, sslctx): newctx = ssl.SSLContext(ssl.PROTOCOL_TLS) - certfile, keyfile = self.create_cert(servername, self.ca_cert, self.ca_key) - if certfile is None or keyfile is None: - return None + cert_file, key_file = self.create_cert(servername, self.ca_cert, self.ca_key) + if cert_file is None or key_file is None: + return newctx.check_hostname = False - newctx.load_cert_chain(certfile=certfile, keyfile=keyfile) + newctx.load_cert_chain(certfile=cert_file, keyfile=key_file) sslsock.context = newctx - return None + return def _load_cert(self, certpath): try: - with open(certpath, 'rb') as certfile: - data = certfile.read() + with open(certpath, 'rb') as cert_file_input: + cert_file_input = cert_file.read() self.ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) except: traceback.format_exc() @@ -138,8 +138,8 @@ def _load_cert(self, certpath): def _load_private_key(self, keypath): try: - with open(keypath, 'rb') as keyfile: - data = keyfile.read() + with open(keypath, 'rb') as key_file_input: + data = key_file_input.read() self.privkey = crypto.load_privatekey(crypto.FILETYPE_PEM, data) except: traceback.print_exc() From 430c5be4c134abdc7039c263f214319f716097da Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 09:57:51 -0500 Subject: [PATCH 15/33] We now use traceback.format_exc() output --- fakenet/listeners/ssl_utils/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index edf58bdf..9d87f341 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -132,7 +132,8 @@ def _load_cert(self, certpath): cert_file_input = cert_file.read() self.ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) except: - traceback.format_exc() + self.logger.error(traceback.format_exc()) + self.logger.error("Failed to load certficate") self.ca_cert = None return self.ca_cert From 19b027de803f8f1558e5b38c669c3571b27b41d3 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 23 Mar 2019 10:05:09 -0500 Subject: [PATCH 16/33] More cleanup regarding ssl_utils and __init__.py --- fakenet/listeners/ssl_utils/__init__.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 9d87f341..8bcb4e20 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -146,6 +146,23 @@ def _load_private_key(self, keypath): traceback.print_exc() self.privkey = None return self.privkey + def _run_win_certutil(self, argv): + rc = False + if sys.platform.startswith('win'): + try: + subprocess.check_call(argv, shell=True, stdout=None) + rc = True + except subprocess.CalledProcessError: + self.logger.error('Failed to add root CA') + return rc + + def _add_root_ca(self, ca_cert_file): + argv = ['certutil', '-addstore', 'Root', ca_cert_file] + return self._run_win_certutil(argv) + + def _remove_root_ca(self, cn): + argv = ['certutil', '-delstore', 'Root', cn] + return self._run_win_certutil(argv) def _add_root_ca(self, ca_cert_file): if not sys.platform.startswith('win'): @@ -184,7 +201,7 @@ def __del__(self): try: shutil.rmtree(self.config.get('certdir', self.CERT_DIR)) except: - traceback.print_exc() + self.logger.warn(traceback.format_exc()) return From f477c5d7ab4300fcb75b81a62a095aeb9c759c0a Mon Sep 17 00:00:00 2001 From: htnhan Date: Wed, 27 Mar 2019 00:40:48 -0500 Subject: [PATCH 17/33] Pass network mode config to the listners --- fakenet/fakenet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fakenet/fakenet.py b/fakenet/fakenet.py index f07b466d..3f7a784c 100644 --- a/fakenet/fakenet.py +++ b/fakenet/fakenet.py @@ -187,7 +187,7 @@ def start(self): self.logger.error("%s" % e) else: - + listener_config['networkmode'] = self.diverter_config['networkmode'] listener_provider_instance = listener_provider( listener_config, listener_name, self.logging_level) From 51d5f3282ff5b7a69e8fa103d7b951f46ccc5eab Mon Sep 17 00:00:00 2001 From: htnhan Date: Wed, 27 Mar 2019 00:41:07 -0500 Subject: [PATCH 18/33] Remove old code --- fakenet/listeners/HTTPListener.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index 2a8d0082..23ca5bbc 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -100,15 +100,6 @@ def start(self): self.server.extensions_map = self.extensions_map if self.config.get('usessl') == 'Yes': - ''' - self.prepare_certs() - self.logger.debug('Using SSL socket.') - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) - self.logger.info("setting callback to %s" % str(self.sni_callback)) - ctx.set_servername_callback(self.sni_callback) - ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) - self.server.socket = ctx.wrap_socket(self.server.socket, server_side=True) - ''' config = { 'static_ca': self.config.get('static_ca'), 'ca_cert': self.config.get('ca_cert'), From 88a4b7fa702c1a2904810ed1e40e785302d7ce57 Mon Sep 17 00:00:00 2001 From: htnhan Date: Wed, 27 Mar 2019 00:42:36 -0500 Subject: [PATCH 19/33] Various fixes to add a fallback when failing to generate a cert --- fakenet/listeners/ProxyListener.py | 35 ++++--- fakenet/listeners/ssl_utils/__init__.py | 116 +++++++++++------------- 2 files changed, 72 insertions(+), 79 deletions(-) diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index dd811cef..f11d4c42 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -13,12 +13,15 @@ from ssl_utils import ssl_detector, SSLWrapper from . import * import os +import traceback +import shutil BUF_SZ = 1024 IP = '0.0.0.0' -class ProxyListener(object): +CERT_DIR = "temp_certs" +class ProxyListener(object): def __init__( self, @@ -38,8 +41,12 @@ def __init__( self.logger.info('Starting...') self.logger.debug('Initialized with config:') + if self.config.get('cert_dir', None) is None: + self.config['cert_dir'] = CERT_DIR + for key, value in config.iteritems(): self.logger.debug(' %10s: %s', key, value) + def start(self): @@ -86,6 +93,11 @@ def stop(self): if self.server: self.server.shutdown() self.server.server_close() + self.logger.warn("Trying to cleanup certificate directory") + try: + shutil.rmtree(self.config.get('cert_dir', CERT_DIR)) + except: + pass def acceptListeners(self, listeners): self.server.listeners = listeners @@ -165,22 +177,6 @@ def handle(self): remote_q = Queue.Queue() data = None - # ssl_remote_sock = None - - # keyfile_path = 'listeners/ssl_utils/privkey.pem' - # keyfile_path = ListenerBase.abs_config_path(keyfile_path) - # if keyfile_path is None: - # self.logger.error('Could not locate %s', keyfile_path) - # sys.exit(1) - - # certfile_path = 'listeners/ssl_utils/server.pem' - # certfile_path = ListenerBase.abs_config_path(certfile_path) - # if certfile_path is None: - # self.logger.error('Could not locate %s', certfile_path) - # sys.exit(1) - - # ssl_version = ssl.PROTOCOL_SSLv23 - try: data = remote_sock.recv(BUF_SZ, socket.MSG_PEEK) @@ -194,9 +190,10 @@ def handle(self): self.server.logger.info('recv() error: %s' % e.message) if data: - if ssl_detector.looks_like_ssl(data): config = { + 'cert_dir': self.server.config.get('cert_dir'), + 'networkmode': self.server.config.get('networkmode', None), 'static_ca': self.server.config.get('static_ca', False), 'ca_cert': self.server.config.get('ca_cert'), 'ca_key': self.server.config.get('ca_key') @@ -205,9 +202,11 @@ def handle(self): self.server.logger.debug('SSL detected') ssl_remote_sock = self.sslwrapper.wrap_socket(remote_sock) data = ssl_remote_sock.recv(BUF_SZ) + else: ssl_remote_sock = None + orig_src_ip = self.client_address[0] orig_src_port = self.client_address[1] diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 8bcb4e20..a815888c 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -8,12 +8,13 @@ import shutil import sys import ssl +import random +from listeners import ListenerBase from OpenSSL import crypto class SSLWrapper(object): NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 - CERT_DIR = "temp_certs" CN="fakenet.flare" def __init__(self, config): @@ -21,14 +22,13 @@ def __init__(self, config): self.config = config self.ca_cert = None self.ca_key = None - - certdir = self.config.get('certdir', self.CERT_DIR) - self.logger.info("Cert dir is %s", certdir) - if certdir is None: + + cert_dir = self.config.get('cert_dir', None) + if cert_dir is None: raise RuntimeError("certdir key is not specified in config") - if not os.path.isdir(certdir): - os.makedirs(certdir) + if not os.path.isdir(cert_dir): + os.makedirs(cert_dir) # generate and add root CA, which is used to sign for other certs: if self.config.get('static_ca') == 'Yes': @@ -36,10 +36,11 @@ def __init__(self, config): self.ca_key = self.config.get('ca_key', None) else: self.ca_cert, self.ca_key = self.create_cert(self.CN) - self.logger.info("adding root cert: %s", self.ca_cert) - if not self._add_root_ca(self.ca_cert): - raise RuntimeError("Failed to add root ca") - + if ( not self.config.get('networkmode', None) == 'multihost' and + not self.config.get('static_ca') == 'Yes'): + self.logger.info("adding root cert: %s", self.ca_cert) + self._add_root_ca(self.ca_cert) + def wrap_socket(self, s): self.logger.info('making socket') try: @@ -47,12 +48,27 @@ def wrap_socket(self, s): except: self.logger.error(traceback.format_exc()) self.logger.error("Exception when calling ssl.SSLContext") - return ctx.wrap_socket(s, server_side=True) + return self.wrap_socket_fallback(s) else: ctx.set_servername_callback(self.sni_callback) - ctx.load_cert_chain(cert_file=self.ca_cert, keyfile=self.ca_key) + ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) return ctx.wrap_socket(s, server_side=True) + def wrap_socket_fallback(self, s): + keyfile_path = 'listeners/ssl_utils/privkey.pem' + keyfile_path = ListenerBase.abs_config_path(keyfile_path) + if keyfile_path is None: + raise RuntimeError('Could not locate %s', (key_file,)) + + certfile_path = 'listeners/ssl_utils/server.pem' + certfile_path = ListenerBase.abs_config_path(certfile_path) + if certfile_path is None: + raise RuntimeError('Cound not locate %s' % (certfile_path,)) + + return ssl.wrap_socket(s, keyfile=keyfile_path, certfile=certfile_path, + server_side=True, ciphers='RSA') + + def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): """ Create a cert given the common name, a signing CA, CA private key and @@ -64,7 +80,10 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): f_selfsign = ca_cert is None or ca_key is None if not cert_dir: - cert_dir = self.CERT_DIR + cert_dir = os.path.abspath(self.config.get('cert_dir')) + else: + cert_dir = os.path.abspath(cert_dir) + cert_file = os.path.join(cert_dir, "%s.crt" % (cn)) key_file = os.path.join(cert_dir, "%s.key" % (cn)) if os.path.exists(cert_file) and os.path.exists(key_file): @@ -88,7 +107,7 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): cert = crypto.X509() cert.get_subject().C = "US" cert.get_subject().CN = cn - cert.set_serial_number(0x31337) + cert.set_serial_number(random.randint(1, 0x31337)) now = time.time() / 1000000 na = int(now + self.NOT_AFTER_DELTA_SECONDS) cert.gmtime_adj_notBefore(0) @@ -106,8 +125,8 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): cert_file_input.write(crypto.dump_certificate( crypto.FILETYPE_PEM, cert) ) - with open(key_file, "wb") as key_file: - key_file.write(crypto.dump_privatekey( + with open(key_file, "wb") as key_file_output: + key_file_output.write(crypto.dump_privatekey( crypto.FILETYPE_PEM, key) ) except: @@ -129,13 +148,13 @@ def sni_callback(self, sslsock, servername, sslctx): def _load_cert(self, certpath): try: with open(certpath, 'rb') as cert_file_input: - cert_file_input = cert_file.read() - self.ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) + data = cert_file_input.read() + ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) except: - self.logger.error(traceback.format_exc()) - self.logger.error("Failed to load certficate") - self.ca_cert = None - return self.ca_cert + self.logger.warn(traceback.format_exc()) + self.logger.warn("Failed to load certficate") + ca_cert = None + return ca_cert def _load_private_key(self, keypath): try: @@ -146,14 +165,16 @@ def _load_private_key(self, keypath): traceback.print_exc() self.privkey = None return self.privkey + def _run_win_certutil(self, argv): - rc = False + rc = True if sys.platform.startswith('win'): - try: - subprocess.check_call(argv, shell=True, stdout=None) - rc = True - except subprocess.CalledProcessError: - self.logger.error('Failed to add root CA') + try: + subprocess.check_call(argv, shell=True, stdout=None) + rc = True + except subprocess.CalledProcessError: + self.logger.error('Failed to add root CA') + rc = False return rc def _add_root_ca(self, ca_cert_file): @@ -164,44 +185,17 @@ def _remove_root_ca(self, cn): argv = ['certutil', '-delstore', 'Root', cn] return self._run_win_certutil(argv) - def _add_root_ca(self, ca_cert_file): - if not sys.platform.startswith('win'): - return False - try: - subprocess.check_call( - ["certutil", "-addstore", "Root", ca_cert_file], - shell=True, stdout=None - ) - rc = True - except subprocess.CalledProcessError: - rc = False - self.logger.error("Failed to add root CA") - self.logger.error(traceback.format_exc()) - return rc - - def _remove_root_ca(self, cn): - if not sys.platform.startswith('win'): - return False - try: - subprocess.check_call( - ["certutil", "-delstore", "Root", cn], - shell=True - ) - rc = True - except subprocess.CalledProcessError: - rc = False - self.logger.error("Failed to add root CA") - self.logger.error(traceback.format_exc()) - return rc def __del__(self): cert = self._load_cert(self.ca_cert) - if cert is not None: + if ( cert is not None and + not self.config.get('networkmode', None) == 'multihost' and + not self.config.get('static_ca') == 'Yes'): self._remove_root_ca(cert.get_subject().CN) try: - shutil.rmtree(self.config.get('certdir', self.CERT_DIR)) + shutil.rmtree(self.config.get('cert_dir', self.CERT_DIR)) except: - self.logger.warn(traceback.format_exc()) + pass return From b53c91e5d69eb4d17345a4d373ee069308cee845 Mon Sep 17 00:00:00 2001 From: Michael Bailey Date: Fri, 29 Mar 2019 12:08:31 -0700 Subject: [PATCH 20/33] Exceptions and logging --- fakenet/listeners/ssl_utils/__init__.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index a815888c..e3dc8ebd 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -1,5 +1,3 @@ - - import time import os import traceback @@ -12,6 +10,8 @@ from listeners import ListenerBase from OpenSSL import crypto +g_ssl_fellback = False # Notify only once of SSL static certificate fallback + class SSLWrapper(object): NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 @@ -38,16 +38,19 @@ def __init__(self, config): self.ca_cert, self.ca_key = self.create_cert(self.CN) if ( not self.config.get('networkmode', None) == 'multihost' and not self.config.get('static_ca') == 'Yes'): - self.logger.info("adding root cert: %s", self.ca_cert) + self.logger.debug('adding root cert: %s', self.ca_cert) self._add_root_ca(self.ca_cert) def wrap_socket(self, s): - self.logger.info('making socket') + global g_ssl_fellback try: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) - except: - self.logger.error(traceback.format_exc()) - self.logger.error("Exception when calling ssl.SSLContext") + except AttributeError as e: + if not g_ssl_fellback: + g_ssl_fellback = True + self.logger.error('Exception calling ssl.SSLContext: %s' % + (e.message)) + self.logger.error('Falling back on static certificate') return self.wrap_socket_fallback(s) else: ctx.set_servername_callback(self.sni_callback) From 01be0682e56fbde703e4779f57cff6015131f6e7 Mon Sep 17 00:00:00 2001 From: Michael Bailey Date: Fri, 29 Mar 2019 13:05:59 -0700 Subject: [PATCH 21/33] Exceptions and logging --- fakenet/listeners/ssl_utils/__init__.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index e3dc8ebd..ab637174 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -132,7 +132,7 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): key_file_output.write(crypto.dump_privatekey( crypto.FILETYPE_PEM, key) ) - except: + except IOError: traceback.print_exc() return None, None return cert_file, key_file @@ -149,14 +149,13 @@ def sni_callback(self, sslsock, servername, sslctx): return def _load_cert(self, certpath): + ca_cert = None try: with open(certpath, 'rb') as cert_file_input: data = cert_file_input.read() ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) except: - self.logger.warn(traceback.format_exc()) - self.logger.warn("Failed to load certficate") - ca_cert = None + self.logger.error("Failed to load certficate") return ca_cert def _load_private_key(self, keypath): @@ -190,8 +189,11 @@ def _remove_root_ca(self, cn): def __del__(self): - cert = self._load_cert(self.ca_cert) - if ( cert is not None and + cert = None + if self.ca_cert: + cert = self._load_cert(self.ca_cert) + + if (cert is not None and not self.config.get('networkmode', None) == 'multihost' and not self.config.get('static_ca') == 'Yes'): self._remove_root_ca(cert.get_subject().CN) From 472fe250f04f50313881e16b4c77208427aeb949 Mon Sep 17 00:00:00 2001 From: htnhan Date: Sat, 30 Mar 2019 20:47:57 -0500 Subject: [PATCH 22/33] Fix: Handling case https:// --- fakenet/listeners/ssl_utils/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index ab637174..1a1271d5 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -138,6 +138,8 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): return cert_file, key_file def sni_callback(self, sslsock, servername, sslctx): + if servername is None: + servername = self.CN newctx = ssl.SSLContext(ssl.PROTOCOL_TLS) cert_file, key_file = self.create_cert(servername, self.ca_cert, self.ca_key) if cert_file is None or key_file is None: From 675a65dbead37c57df5cbdad82dae7c88850ddc8 Mon Sep 17 00:00:00 2001 From: htnhan Date: Wed, 3 Apr 2019 02:41:57 -0500 Subject: [PATCH 23/33] SSLWrapper should only be initialized once per listener --- fakenet/listeners/HTTPListener.py | 32 ++++++++++++++---------------- fakenet/listeners/ProxyListener.py | 29 ++++++++++----------------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index a0f6dd4a..d0c649c8 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -94,20 +94,27 @@ def __init__( def start(self): self.logger.debug('Starting...') - self.server = ThreadedHTTPServer((self.local_ip, int(self.config.get('port'))), ThreadedHTTPRequestHandler) + + + config = { + 'cert_dir': self.config.get('cert_dir', 'temp_certs'), + 'networkmode': self.config.get('networkmode', None), + 'static_ca': self.config.get('static_ca', False), + 'ca_cert': self.config.get('ca_cert'), + 'ca_key': self.config.get('ca_key') + } + self.sslwrapper = SSLWrapper(config) + self.server = ThreadedHTTPServer((self.local_ip, + int(self.config.get('port'))), ThreadedHTTPRequestHandler) + self.server.sslwrapper = self.sslwrapper self.server.logger = self.logger self.server.config = self.config self.server.webroot_path = self.webroot_path self.server.extensions_map = self.extensions_map if self.config.get('usessl') == 'Yes': - config = { - 'static_ca': self.config.get('static_ca'), - 'ca_cert': self.config.get('ca_cert'), - 'ca_key': self.config.get('ca_key'), - } - self.sslwrapper = SSLWrapper(config) - self.server.socket = self.sslwrapper.wrap_socket(self.server.socket) + self.server.socket = self.server.sslwrapper.wrap_socket( + self.server.socket) self.server_thread = threading.Thread(target=self.server.serve_forever) @@ -120,15 +127,6 @@ def stop(self): self.server.shutdown() self.server.server_close() - if self.config.get('usessl' == 'Yes'): - cert = self.sslwrap._load_cert(self.ca_cert) - if cert is not None: - self.sslwrap._remove_root_ca(cert.get_subject().CN) - try: - shutil.rmtree(self.CERT_DIR) - except: - pass - class ThreadedHTTPServer(BaseHTTPServer.HTTPServer): diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index 762008fb..f59efa88 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -18,7 +18,6 @@ BUF_SZ = 1024 -CERT_DIR = "temp_certs" class ProxyListener(object): @@ -41,8 +40,6 @@ def __init__( self.logger.debug('Starting...') self.logger.debug('Initialized with config:') - if self.config.get('cert_dir', None) is None: - self.config['cert_dir'] = CERT_DIR for key, value in config.iteritems(): self.logger.debug(' %10s: %s', key, value) @@ -50,6 +47,15 @@ def __init__( def start(self): + config = { + 'cert_dir': self.config.get('cert_dir', 'temp_certs'), + 'networkmode': self.config.get('networkmode', None), + 'static_ca': self.config.get('static_ca', False), + 'ca_cert': self.config.get('ca_cert'), + 'ca_key': self.config.get('ca_key') + } + self.sslwrapper = SSLWrapper(config) + proto = self.config.get('protocol').upper() if proto != None: @@ -59,6 +65,7 @@ def start(self): self.server = ThreadedTCPServer((self.local_ip, int(self.config.get('port'))), ThreadedTCPRequestHandler) + self.server.sslwrapper = self.sslwrapper elif proto == 'UDP': @@ -96,11 +103,6 @@ def stop(self): if self.server: self.server.shutdown() self.server.server_close() - self.logger.warn("Trying to cleanup certificate directory") - try: - shutil.rmtree(self.config.get('cert_dir', CERT_DIR)) - except: - pass def acceptListeners(self, listeners): self.server.listeners = listeners @@ -194,16 +196,7 @@ def handle(self): if data: if ssl_detector.looks_like_ssl(data): - config = { - 'cert_dir': self.server.config.get('cert_dir'), - 'networkmode': self.server.config.get('networkmode', None), - 'static_ca': self.server.config.get('static_ca', False), - 'ca_cert': self.server.config.get('ca_cert'), - 'ca_key': self.server.config.get('ca_key') - } - self.sslwrapper = SSLWrapper(config) - self.server.logger.debug('SSL detected') - ssl_remote_sock = self.sslwrapper.wrap_socket(remote_sock) + ssl_remote_sock = self.server.sslwrapper.wrap_socket(remote_sock) data = ssl_remote_sock.recv(BUF_SZ) else: From 4a13e664df98e12fade1a2d3f988a87ca1025360 Mon Sep 17 00:00:00 2001 From: htnhan Date: Tue, 9 Apr 2019 08:47:00 -0500 Subject: [PATCH 24/33] Listeners now create the SSLWrapper once per startup --- fakenet/listeners/HTTPListener.py | 22 +++++++++++----------- fakenet/listeners/ProxyListener.py | 21 +++++++++------------ 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index d0c649c8..5a02e860 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -79,6 +79,7 @@ def __init__( self.server = None self.name = 'HTTP' self.port = self.config.get('port', 80) + self.sslwrapper = None self.logger.debug('Initialized with config:') for key, value in config.iteritems(): @@ -94,25 +95,24 @@ def __init__( def start(self): self.logger.debug('Starting...') - - - config = { - 'cert_dir': self.config.get('cert_dir', 'temp_certs'), - 'networkmode': self.config.get('networkmode', None), - 'static_ca': self.config.get('static_ca', False), - 'ca_cert': self.config.get('ca_cert'), - 'ca_key': self.config.get('ca_key') - } - self.sslwrapper = SSLWrapper(config) self.server = ThreadedHTTPServer((self.local_ip, int(self.config.get('port'))), ThreadedHTTPRequestHandler) - self.server.sslwrapper = self.sslwrapper self.server.logger = self.logger self.server.config = self.config self.server.webroot_path = self.webroot_path self.server.extensions_map = self.extensions_map if self.config.get('usessl') == 'Yes': + self.logger.debug("HTTP Listener starting with SSL") + config = { + 'cert_dir': self.config.get('cert_dir', 'temp_certs'), + 'networkmode': self.config.get('networkmode', None), + 'static_ca': self.config.get('static_ca', False), + 'ca_cert': self.config.get('ca_cert'), + 'ca_key': self.config.get('ca_key') + } + self.sslwrapper = SSLWrapper(config) + self.server.sslwrapper = sslwrapper self.server.socket = self.server.sslwrapper.wrap_socket( self.server.socket) diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index f59efa88..108ba3f0 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -14,7 +14,6 @@ from . import * import os import traceback -import shutil BUF_SZ = 1024 @@ -47,22 +46,20 @@ def __init__( def start(self): - config = { - 'cert_dir': self.config.get('cert_dir', 'temp_certs'), - 'networkmode': self.config.get('networkmode', None), - 'static_ca': self.config.get('static_ca', False), - 'ca_cert': self.config.get('ca_cert'), - 'ca_key': self.config.get('ca_key') - } - self.sslwrapper = SSLWrapper(config) - proto = self.config.get('protocol').upper() if proto != None: if proto == 'TCP': - self.logger.debug('Starting TCP ...') - + self.logger.debug('Starting TCP ...') + config = { + 'cert_dir': self.config.get('cert_dir', 'temp_certs'), + 'networkmode': self.config.get('networkmode', None), + 'static_ca': self.config.get('static_ca', False), + 'ca_cert': self.config.get('ca_cert'), + 'ca_key': self.config.get('ca_key') + } + self.sslwrapper = SSLWrapper(config) self.server = ThreadedTCPServer((self.local_ip, int(self.config.get('port'))), ThreadedTCPRequestHandler) self.server.sslwrapper = self.sslwrapper From e2395b3f905503535414f930165f77efdb5c25e3 Mon Sep 17 00:00:00 2001 From: htnhan Date: Tue, 9 Apr 2019 08:48:21 -0500 Subject: [PATCH 25/33] Use ignore_errors instead of catch all except --- fakenet/listeners/ssl_utils/__init__.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 1a1271d5..f99d7ab8 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -199,10 +199,7 @@ def __del__(self): not self.config.get('networkmode', None) == 'multihost' and not self.config.get('static_ca') == 'Yes'): self._remove_root_ca(cert.get_subject().CN) - try: - shutil.rmtree(self.config.get('cert_dir', self.CERT_DIR)) - except: - pass + shutil.rmtree(self.config.get('cert_dir'), ignore_errors=True) return From f34814f7be0c731100cc9a6ae5bc83cf2ae4b0b3 Mon Sep 17 00:00:00 2001 From: htnhan Date: Tue, 9 Apr 2019 08:50:34 -0500 Subject: [PATCH 26/33] New template for HTTPS tests --- test/template.ini | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/test/template.ini b/test/template.ini index 37a5a481..a81fccee 100644 --- a/test/template.ini +++ b/test/template.ini @@ -189,10 +189,14 @@ HostBlackList: 6.6.6.6 [ProxyTCPListener] Enabled: True Protocol: TCP +UseSSL: Yes Listener: ProxyListener Port: 38926 Listeners: HTTPListener, RawListener, FTPListener, DNSListener, POPListener, SMTPListener, TFTPListener, IRCListener, BITSListener Hidden: False +Static_CA: No +CA_Cert: listeners/ssl_utils/server.pem +CA_Key: listeners/ssl_utils/privkey.pem [ProxyUDPListener] Enabled: True @@ -317,18 +321,6 @@ DumpHTTPPosts: Yes DumpHTTPPostsFilePrefix: http Hidden: False - -[HTTPListener443] -Enabled: True -Port: 443 -Protocol: TCP -Listener: HTTPListener -UseSSL: Yes -Webroot: defaultFiles/ -DumpHTTPPosts: Yes -DumpHTTPPostsFilePrefix: http -Hidden: False - [SMTPListener] Enabled: True Port: 25 From 054389e7ed5ccf135054fe107cfe8d849913a0ce Mon Sep 17 00:00:00 2001 From: htnhan Date: Tue, 9 Apr 2019 08:52:55 -0500 Subject: [PATCH 27/33] New test cases for HTTPS --- test/test.py | 43 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/test/test.py b/test/test.py index 2ba6ed94..bedc6fea 100644 --- a/test/test.py +++ b/test/test.py @@ -6,6 +6,7 @@ import signal import socket import pyping +import ssl import ftplib import poplib import hashlib @@ -19,10 +20,12 @@ import irc.client import ConfigParser from collections import OrderedDict +from requests.adapters import HTTPAdapter logger = logging.getLogger('FakeNetTests') logging.basicConfig(format='%(message)s', level=logging.INFO) + def is_admin(): result = False try: @@ -549,17 +552,37 @@ def _test_irc(self, hostname, port=6667): irc_tester = IrcTester(hostname, port) return irc_tester.test_irc() - def _test_http(self, hostname, port=None): + def _test_http(self, hostname, port=None, use_ssl=False, verify=False): """Test HTTP Listener""" - retval = False + + # https://stackoverflow.com/a/50215614 + # HACK to force requests to use Windows Certificate store + class SSLContextAdapter(HTTPAdapter): + def init_poolmanager(self, *args, **kwargs): + ctx = ssl.create_default_context() + ctx.load_default_certs() + kwargs['ssl_context'] = ctx + return super(SSLContextAdapter, self).init_poolmanager(*args, **kwargs) + + retval = False + sess = requests.Session() + + if use_ssl: + proto = 'https' + else: + proto = 'http' if port: - url = 'http://%s:%d/asdf.html' % (hostname, port) + url = '%s://%s:%d/asdf.html' % (proto, hostname, port) else: - url = 'http://%s/asdf.html' % (hostname) + url = '%s://%s/asdf.html' % (proto, hostname) + + if use_ssl and verify and sys.platform.startswith('win32'): + adapter = SSLContextAdapter() + sess.mount(url, adapter) try: - r = requests.get(url, timeout=3) + r = sess.get(url, timeout=3, verify=verify) if r.status_code != 200: raise FakeNetTestException('Status code %d' % (r.status_code)) @@ -571,8 +594,11 @@ def _test_http(self, hostname, port=None): retval = True except requests.exceptions.Timeout as e: + pass + except requests.exceptions.SSLError as e: + pass + except ValueError as e: pass - except FakeNetTestException as e: pass @@ -685,7 +711,6 @@ def testGeneral(self, matchspec=[]): udp = socket.SOCK_DGRAM t = OrderedDict() # The tests - t['TCP external IP @ bound'] = (self._test_sk, (tcp, ext_ip, 1337), True) t['TCP external IP @ unbound'] = (self._test_sk, (tcp, ext_ip, 9999), True) t['TCP arbitrary @ bound'] = (self._test_sk, (tcp, arbitrary, 1337), True) @@ -712,6 +737,9 @@ def testGeneral(self, matchspec=[]): t['DNS listener test'] = (self._test_ns, (domain_dne, dns_expected), True) t['HTTP listener test'] = (self._test_http, (arbitrary,), True) + t['HTTPS listener IP test, no verify'] = (self._test_http, (arbitrary,None,True,False), True) + t['HTTPS listener IP test'] = (self._test_http, (arbitrary,None,True,True), False) + t['HTTPS listener hostname test'] = (self._test_http, ('evil.com',None,True,True), True) t['FTP listener test'] = (self._test_ftp, (arbitrary,), True) t['POP3 listener test'] = (self._test_pop, (arbitrary, 110), True) t['SMTP listener test'] = (self._test_smtp, (sender, recipient, smtpmsg, arbitrary), True) @@ -734,7 +762,6 @@ def testGeneral(self, matchspec=[]): t['Listener process whitelist'] = (self._test_http, (arbitrary, self.settings.listener_proc_white), True) t['Listener host blacklist'] = (self._test_http, (arbitrary, self.settings.listener_host_black), True) t['Listener host whitelist'] = (self._test_http, (arbitrary, self.settings.listener_host_black), True) - return self._testGeneric('General', config, t, matchspec) def makeConfig(self, singlehostmode=True, proxied=True, redirectall=True): From 52b5b9f064a7e2e596bb6f1c8cf54477d9611c47 Mon Sep 17 00:00:00 2001 From: htnhan Date: Thu, 25 Apr 2019 14:05:22 -0500 Subject: [PATCH 28/33] ProxyListener does not use UseSSL config keyword --- test/template.ini | 1 - 1 file changed, 1 deletion(-) diff --git a/test/template.ini b/test/template.ini index a81fccee..589e210b 100644 --- a/test/template.ini +++ b/test/template.ini @@ -189,7 +189,6 @@ HostBlackList: 6.6.6.6 [ProxyTCPListener] Enabled: True Protocol: TCP -UseSSL: Yes Listener: ProxyListener Port: 38926 Listeners: HTTPListener, RawListener, FTPListener, DNSListener, POPListener, SMTPListener, TFTPListener, IRCListener, BITSListener From 22a6f6c2037e0bf1f6b7b4f29c32a7435b09643d Mon Sep 17 00:00:00 2001 From: Tina Johnson Date: Fri, 15 Dec 2023 04:58:18 +0100 Subject: [PATCH 29/33] Update SNI branch to Python 3 - Replaced expired CA certificate - Changed certificate sigining algorithm to SHA256 - Added extensions to generated X509 certificates to support interaction with latest browsers Thanks to Nhan Huynh for implementing this feature :) --- fakenet/configs/default.ini | 11 ++- fakenet/configs/fakenet_ca.crt | 19 +++++ fakenet/configs/fakenet_ca.key | 28 +++++++ fakenet/listeners/DNSListener.py | 2 +- fakenet/listeners/HTTPListener.py | 13 ++- fakenet/listeners/ProxyListener.py | 4 +- fakenet/listeners/ssl_utils/__init__.py | 100 +++++++++++++----------- fakenet/listeners/ssl_utils/privkey.pem | 47 ----------- fakenet/listeners/ssl_utils/server.pem | 18 ----- setup.py | 4 +- test/test.py | 38 ++++++--- 11 files changed, 147 insertions(+), 137 deletions(-) create mode 100644 fakenet/configs/fakenet_ca.crt create mode 100644 fakenet/configs/fakenet_ca.key delete mode 100644 fakenet/listeners/ssl_utils/privkey.pem delete mode 100644 fakenet/listeners/ssl_utils/server.pem diff --git a/fakenet/configs/default.ini b/fakenet/configs/default.ini index a88a76e3..7845132e 100644 --- a/fakenet/configs/default.ini +++ b/fakenet/configs/default.ini @@ -199,6 +199,11 @@ BlackListPortsUDP: 67, 68, 137, 138, 443, 1900, 5355 # hostname string, !hostname to insert the actual hostname # of the system, or !random to generate a random hostname # between 1 and 15 characters (inclusive). +# * Static_CA - Set FakeNet to use user provided CA certificate to sign generated certificates. +# * CA_Cert - CA certificate in PEM format to be used when Static_CA config is set. Manually +# add this certificate to Windows trust store before executing FakeNet. +# * CA_Key - CA private key in PEM format to be used when Static_CA config is set. + [ProxyTCPListener] Enabled: True @@ -208,8 +213,8 @@ Port: 38926 Listeners: HTTPListener, RawListener, FTPListener, DNSListener, POPListener, SMTPListener, TFTPListener, IRCListener Hidden: False Static_CA: No -CA_Cert: listeners/ssl_utils/server.pem -CA_Key: listeners/ssl_utils/privkey.pem +CA_Cert: configs/fakenet_ca.crt +CA_Key: configs/fakenet_ca.key [ProxyUDPListener] Enabled: True @@ -286,7 +291,7 @@ Hidden: False # Custom: sample_custom_response.ini [HTTPListener443] -Enabled: False +Enabled: True Port: 443 Protocol: TCP Listener: HTTPListener diff --git a/fakenet/configs/fakenet_ca.crt b/fakenet/configs/fakenet_ca.crt new file mode 100644 index 00000000..4f703acd --- /dev/null +++ b/fakenet/configs/fakenet_ca.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDHzCCAgegAwIBAgIUdwSkHdOM2mMDw094Kha+9Z9/w60wDQYJKoZIhvcNAQEL +BQAwHzELMAkGA1UEBhMCVVMxEDAOBgNVBAMMB2Zha2VuZXQwHhcNMjMxMjE0MDAx +NDUxWhcNMjQwMTEzMDAxNDUxWjAfMQswCQYDVQQGEwJVUzEQMA4GA1UEAwwHZmFr +ZW5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM7pGzS8bX8M3SSQ +mk79puvqGBHwWVpDK82T44N/8mXHJ1R/7jMDq2wpkSjliiAE0mxPpkzMbr9mkeP/ +/31GKAszbJYnurrxxYbLyOdRst2VqoXkWTia61lrsRIGcjwzKe2zyMCcuiyRTLcP +BmYd/ie5AzyHxitlS49cub+QkIODUAKTiZT3mPu6Yw2XvYkg+up69NzC0a/XexUv +PvgbBizquKj/YzMSp5X7ieYGv0xHf8Dhf3mqh9oLk35X/qV3LqdnVPjweCR8X9ze +yhfBbDr1VoBnOe2Nb5hlU9MB//A0hgDYj5TrHa4JrbNkv3lYMd4uv/CBW6o18Ba+ +/zjEvyECAwEAAaNTMFEwHQYDVR0OBBYEFD3wRGMPQdWtBCKRy5c9N5YWnki2MB8G +A1UdIwQYMBaAFD3wRGMPQdWtBCKRy5c9N5YWnki2MA8GA1UdEwEB/wQFMAMBAf8w +DQYJKoZIhvcNAQELBQADggEBAGT5rmafrlv8VIXAgc0iazNd7A6rT7xNLkuF2JGK +7NV3yvsWM6SA+DlG7y70om+eKjd+qxzinxnSt6uFJhcdCqot1LU1u3OZDifTTJmk +31yEYp/+A93qjwe1Ag2rsVcztRl88KtsKrKNohv24iiWfIVDnHo0joerXoaGQwo6 +zXl4GVJBEEAhf2GQRgyXcoWkSrsq8UKtVV9dI5QgIS6vZ65oNQEeoAXH56ihFUBX +hS+4Ko2FfUtxbfbw7tpDaNtqhAzJ+LE4RoDUepyCDXPha0Wb4giGOd5EEubYrFKi +DOdAMiiQT1WLK3/UnMlCOV4lne+g9JwBCXL8C0F0W1fFZY8= +-----END CERTIFICATE----- diff --git a/fakenet/configs/fakenet_ca.key b/fakenet/configs/fakenet_ca.key new file mode 100644 index 00000000..6914c8b0 --- /dev/null +++ b/fakenet/configs/fakenet_ca.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDO6Rs0vG1/DN0k +kJpO/abr6hgR8FlaQyvNk+ODf/JlxydUf+4zA6tsKZEo5YogBNJsT6ZMzG6/ZpHj +//99RigLM2yWJ7q68cWGy8jnUbLdlaqF5Fk4mutZa7ESBnI8Mynts8jAnLoskUy3 +DwZmHf4nuQM8h8YrZUuPXLm/kJCDg1ACk4mU95j7umMNl72JIPrqevTcwtGv13sV +Lz74GwYs6rio/2MzEqeV+4nmBr9MR3/A4X95qofaC5N+V/6ldy6nZ1T48HgkfF/c +3soXwWw69VaAZzntjW+YZVPTAf/wNIYA2I+U6x2uCa2zZL95WDHeLr/wgVuqNfAW +vv84xL8hAgMBAAECggEAAyp6KDPcHCjH5XU4hXGPeGYvkhldQtxqsw2Rr7xpVWrl +dw1q8/dR2kVGVFSlBWe7tIPk0Ew8fD14+xtXG4xhmECAoElTHY+b7VgxkVJPem9h +RD4XOLsP4ba4rlNes+DiUCHKbyHTOox1RXytQZxNbgbVr7tOVNU1nf0rVmzt9Zbw +uPltpCcL+yvnaWmBUgdCLIhIT/HDrp4+uZP+zW7pVm/KMQL0I9OMnm6dCimP88pU +meepDRdxHkHVb347jx0+nWSH2V63wxH6WR8lAb4oAXeQXUgS8Q0lXh70EW/x0Ut3 +neB/mfuaXLVrBcBTbhPTU7PKd/Sc+dkkqhEpNRR3dQKBgQDyLNPyWEJVXwQ6iOom +xxyqm61DZw8IpBYVmfYrOisTUWVC+oNlpsNPzWsgwmWzpcJ8s6ykupWFcWZeJv2h +XPlq+Ai6Ky0v+yESiUtNb+lKuXGvHqTpTxvPgL/20ZMfRIkWs9YBLA+aTh5RC8Vs +i12fRW9JTZTKgyf7PEkj+DjCCwKBgQDauOszqtFs6d36AMVJJ7OlmZnEWaELv9bW +ns7kdeBgr35gzGbDOkyQGBII5SCwpgWbXyxT39T9xuNoWCWXSZdHDco/UlAG47qI +Pq/KmtbrKxvg6yKsPIDV+cexuBrLIvzsaz8qPbfE/W7nxf+FtvDhvnGyqoiYd3DS +XSD5HcwLAwKBgQDdQpG+mF66qx4s8Myl80NAqQ1LSMyWg3xd7hXYdsPGWZaf9Eu6 +wvstXSvkeVf8I5Um4+33bzWO/wWdPhh6pnyG++jVVv9pGBOmYOP48yd9iyLP8bqQ +IyPwmNxKgD3f0nlB0brTxVLYE0llmNCelFJMY18C5SvtPpl31COrBm2s8wKBgG8D +zN28pe+SBIkQOxKWhChZfiKbG5LLHFBy6rAq5GguqwaWuNH+lT3N+dlp8t22ZsIl +3Gn2AjWM7X/Yvbu8LnxyE2Vwcg4NKHBe4PsE/HEAwHW44zBoxTvWO/WIbJEOgTG+ +faEDEnN57wDVDozf/gOWlj8JL6uzdCBSBJps9VPhAoGAWQUkdpxRxEARhL7wYR7g +EidDWKnULiULjlrP7VPMJ5hrZf5PmWyZLWW3SkEUhcf25CnJoUoq1OOB1GjL9lBL +0+tXalLnA/mRxO5ILzwJivHyJKnljuPKyXmpKt8H4KRUXV3Uk2may58Jwn8InuRE +aW956i0O1tgrDOj3tSAT8KI= +-----END PRIVATE KEY----- diff --git a/fakenet/listeners/DNSListener.py b/fakenet/listeners/DNSListener.py index da410b81..2acbe160 100644 --- a/fakenet/listeners/DNSListener.py +++ b/fakenet/listeners/DNSListener.py @@ -19,7 +19,7 @@ class DNSListener(object): def taste(self, data, dport): - confidence = 1 if dport is 53 else 0 + confidence = 1 if dport == 53 else 0 try: d = DNSRecord.parse(data) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index ce7f2f61..421630fd 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -18,11 +18,8 @@ import mimetypes import time -import shutil -import traceback -import subprocess -from OpenSSL import crypto -from ssl_utils import SSLWrapper + +from .ssl_utils import SSLWrapper from . import * MIME_FILE_RESPONSE = { @@ -208,14 +205,14 @@ def start(self): if self.config.get('usessl') == 'Yes': self.logger.debug("HTTP Listener starting with SSL") config = { - 'cert_dir': self.config.get('cert_dir', 'temp_certs'), + 'cert_dir': self.config.get('cert_dir', 'configs/temp_certs'), 'networkmode': self.config.get('networkmode', None), - 'static_ca': self.config.get('static_ca', False), + 'static_ca': self.config.get('static_ca', "No"), 'ca_cert': self.config.get('ca_cert'), 'ca_key': self.config.get('ca_key') } self.sslwrapper = SSLWrapper(config) - self.server.sslwrapper = sslwrapper + self.server.sslwrapper = self.sslwrapper self.server.socket = self.server.sslwrapper.wrap_socket( self.server.socket) diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index 100f1ef4..55747859 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -12,7 +12,7 @@ import logging import ssl from OpenSSL import SSL -from .ssl_utils import ssl_detector +from .ssl_utils import ssl_detector, SSLWrapper from . import * import os import traceback @@ -54,7 +54,7 @@ def start(self): self.logger.debug('Starting TCP ...') config = { - 'cert_dir': self.config.get('cert_dir', 'temp_certs'), + 'cert_dir': self.config.get('cert_dir', 'configs/temp_certs'), 'networkmode': self.config.get('networkmode', None), 'static_ca': self.config.get('static_ca', False), 'ca_cert': self.config.get('ca_cert'), diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index f99d7ab8..337ec60b 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -7,11 +7,11 @@ import sys import ssl import random -from listeners import ListenerBase +from pathlib import Path from OpenSSL import crypto -g_ssl_fellback = False # Notify only once of SSL static certificate fallback - +from fakenet import listeners +from fakenet.listeners import ListenerBase class SSLWrapper(object): NOT_AFTER_DELTA_SECONDS = 300 * 24 * 60 * 60 @@ -22,18 +22,20 @@ def __init__(self, config): self.config = config self.ca_cert = None self.ca_key = None + self.ca_cn = self.CN - cert_dir = self.config.get('cert_dir', None) + cert_dir = self.abs_config_path(self.config.get('cert_dir', None)) if cert_dir is None: - raise RuntimeError("certdir key is not specified in config") + raise RuntimeError("cert_dir key is not specified in config") if not os.path.isdir(cert_dir): os.makedirs(cert_dir) # generate and add root CA, which is used to sign for other certs: if self.config.get('static_ca') == 'Yes': - self.ca_cert = self.config.get('ca_cert', None) - self.ca_key = self.config.get('ca_key', None) + self.ca_cert = self.abs_config_path(self.config.get('ca_cert', None)) + self.ca_key = self.abs_config_path(self.config.get('ca_key', None)) + self.ca_cn = self._load_cert(self.ca_cert).get_subject().CN else: self.ca_cert, self.ca_key = self.create_cert(self.CN) if ( not self.config.get('networkmode', None) == 'multihost' and @@ -42,36 +44,16 @@ def __init__(self, config): self._add_root_ca(self.ca_cert) def wrap_socket(self, s): - global g_ssl_fellback try: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) except AttributeError as e: - if not g_ssl_fellback: - g_ssl_fellback = True - self.logger.error('Exception calling ssl.SSLContext: %s' % - (e.message)) - self.logger.error('Falling back on static certificate') - return self.wrap_socket_fallback(s) + self.logger.error('Exception calling ssl.SSLContext: %s' % + (e.message)) else: - ctx.set_servername_callback(self.sni_callback) + ctx.sni_callback = self.sni_callback ctx.load_cert_chain(certfile=self.ca_cert, keyfile=self.ca_key) return ctx.wrap_socket(s, server_side=True) - def wrap_socket_fallback(self, s): - keyfile_path = 'listeners/ssl_utils/privkey.pem' - keyfile_path = ListenerBase.abs_config_path(keyfile_path) - if keyfile_path is None: - raise RuntimeError('Could not locate %s', (key_file,)) - - certfile_path = 'listeners/ssl_utils/server.pem' - certfile_path = ListenerBase.abs_config_path(certfile_path) - if certfile_path is None: - raise RuntimeError('Cound not locate %s' % (certfile_path,)) - - return ssl.wrap_socket(s, keyfile=keyfile_path, certfile=certfile_path, - server_side=True, ciphers='RSA') - - def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): """ Create a cert given the common name, a signing CA, CA private key and @@ -83,10 +65,10 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): f_selfsign = ca_cert is None or ca_key is None if not cert_dir: - cert_dir = os.path.abspath(self.config.get('cert_dir')) + cert_dir = self.abs_config_path(self.config.get('cert_dir')) else: cert_dir = os.path.abspath(cert_dir) - + cert_file = os.path.join(cert_dir, "%s.crt" % (cn)) key_file = os.path.join(cert_dir, "%s.key" % (cn)) if os.path.exists(cert_file) and os.path.exists(key_file): @@ -108,6 +90,10 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): # Create a cert cert = crypto.X509() + + # Setting certificate version to 3. This is required to use certificate + # extensions which have proven necessary when working with browsers + cert.set_version(2) cert.get_subject().C = "US" cert.get_subject().CN = cn cert.set_serial_number(random.randint(1, 0x31337)) @@ -117,11 +103,21 @@ def create_cert(self, cn, ca_cert=None, ca_key=None, cert_dir=None): cert.gmtime_adj_notAfter(na) cert.set_pubkey(key) if f_selfsign: + extensions = [ + crypto.X509Extension(b'basicConstraints', True, b'CA:TRUE'), + ] cert.set_issuer(cert.get_subject()) - cert.sign(key, "sha1") + cert.add_extensions(extensions) + cert.sign(key, "sha256") else: + alt_name = b'DNS:' + cn.encode() + extensions = [ + crypto.X509Extension(b'basicConstraints', False, b'CA:FALSE'), + crypto.X509Extension(b'subjectAltName', False, alt_name) + ] cert.set_issuer(ca_cert_data.get_subject()) - cert.sign(ca_key_data, "sha1") + cert.add_extensions(extensions) + cert.sign(ca_key_data, "sha256") try: with open(cert_file, "wb") as cert_file_input: @@ -156,8 +152,8 @@ def _load_cert(self, certpath): with open(certpath, 'rb') as cert_file_input: data = cert_file_input.read() ca_cert = crypto.load_certificate(crypto.FILETYPE_PEM, data) - except: - self.logger.error("Failed to load certficate") + except crypto.Error as e: + self.logger.error("Failed to load certficate: %s" % e.message) return ca_cert def _load_private_key(self, keypath): @@ -189,17 +185,29 @@ def _remove_root_ca(self, cn): argv = ['certutil', '-delstore', 'Root', cn] return self._run_win_certutil(argv) - def __del__(self): - cert = None - if self.ca_cert: - cert = self._load_cert(self.ca_cert) - - if (cert is not None and - not self.config.get('networkmode', None) == 'multihost' and - not self.config.get('static_ca') == 'Yes'): - self._remove_root_ca(cert.get_subject().CN) - shutil.rmtree(self.config.get('cert_dir'), ignore_errors=True) + if (not self.config.get('networkmode', None) == 'multihost' and + not self.config.get('static_ca') == 'Yes'): + self._remove_root_ca(self.ca_cn) + shutil.rmtree(self.abs_config_path(self.config.get('cert_dir', None)), ignore_errors=True) return + + def abs_config_path(self, path): + """ + Attempts to return the absolute path of a path from a configuration + setting. + """ + + # Try absolute path first + abspath = os.path.abspath(path) + if os.path.exists(abspath): + return abspath + + if getattr(sys, 'frozen', False) and hasattr(sys, '_MEIPASS'): + abspath = os.path.join(os.getcwd(), path) + else: + abspath = os.path.join(os.fspath(Path(__file__).parents[2]), path) + + return abspath diff --git a/fakenet/listeners/ssl_utils/privkey.pem b/fakenet/listeners/ssl_utils/privkey.pem deleted file mode 100644 index 16369418..00000000 --- a/fakenet/listeners/ssl_utils/privkey.pem +++ /dev/null @@ -1,47 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDzcX3h/MmqeYoj -t3qVhgcV8NwApbBYr6iLmmx384ayykau/0lfcN2NGovofwMSGyqFom42d6GgFLEJ -uaPiQ//RZK6NQl2w1X2wP/Un4SET5Id4sMBShR2zS/5H7GgSlgqwUqKegnel348b -/A2YcdAd4iQ97yBxINxbkciN/d8L2omI4etreD/yghqX5WN2UH+NoSg7FdpvgBIh -aMKsqC9vEqbgXaIFr3jDJVjyEy4wBfvFKTgEAIvHhE2B6B1/ij3eJk0ZKpub3Du2 -gekZTpqajQL4viE1wjvydjJq/ixL2dr3q3+Ko/4xl0bLU8r6+/0CkkFnMxMP2+mp -4xVT9uM3AgMBAAECggEBAJpM7hAGDMCbxp361pzdVbJndtqGKm8b74WEvImO9mpu -YTzcHGJ9BEBCejlD/+tDAsGvAZJOY0g2tTvHyYNJvGS5HxXz4bSKrN7Aux+qxy93 -oxIxXcUwEHIrkaF+yzw0k9PMnLxBT5r4Rxniua9NPC8kvfnvji2GOYVksqylI/G7 -NwBLrYjFOJLXqikVZveyG0uQPXxwtRwaVgDSU7QX8kxXfpwsaZGdnm7yk80hrNte -U4U/A7kv63tE0+qFu9jQVatYwcGAR7kNr9qsFOHvxPbEQkvS6yOUHMdNF8ZpwFbA -vv6O1pt4jV7kAo5TLFoGXgnONBifzZCTD127laH1j1kCgYEA/Te6CsWbZm/6pPFJ -QpVyzDlSam7hCb5rmOneoOeIjo2CLn22DEeu9ZqETj2xLMfA0diV1r23Eia+kHwO -3XdNKLg3/Ue22FpQuV4iCqxDXkwdM/hMeyeLssQmNlGE0+iBaXL8UZCu5zWnw2TD -pkjeS6KGPCMN1MvYdhGIzYxt/zUCgYEA9h5FVLxBA0s40jgILpPvLbIF84cVDF46 -xnFs7c7jQSH+DYK9C7GFYKyDw3UTI5+UtwalxsBYWSLZuTSzdpUocKzT31TmbY2M -nOYb3s7VHrJUXWnCl5m6EW1oqRUVMZuseECg7IABWUaI8qUQAbU2JAD1VeoslpVo -s7C9TzuxCjsCgYEA395l3+Im6uDzkuIz5+cjEEVZhPm0gZ3VmOKjTlSFGasoPhws -WB0EJOXQNTA7tSCBa8V1a95cvXJ9plXX+prgH7EG5ymBETSAC/KaXB9CjFr0sp7C -V1t2Gb1rHzjhG0yDJYxgYWhuCcIZKRmsFBZ2Wh49WWuQbeMu4+vKrBeMpEUCgYEA -4+Wh01XCaZdk1Ru3T9ICHnEDG75QmjRC/nAXKplxS2V7hu0xujs2Qw9br0igYVMq -dNP9+20uIdOogduv9jUlzjfqtJk8CeORedOFqSoipBsDDcCZxKE/b1W9HRv9lQMO -kAdgO75IAW/T7cM0cDBpIquUgWqmwQU2f3U/xreCNvMCfxQQiFTTSyhnZu195hjJ -fgNZ3SODPMkQoh8Pah1E5Hb83umGBPvt4MN2bfm8wWmLEjpU93xiJa2uN8Jzx5Mb -K3hs4JcPtjJ2m2x0aDt1RGFhllEl0l5tOQg+cKy9jnqA/O1wmMOXsFgYS5Fhg1Me -o70kQ6YRH+pBgnP7Pgcu4X0= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIJAPHrk7xP1wNXMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAeFw0xNjAxMjEyMjM3MDdaFw0xNjAyMjAyMjM3MDdaMBQx -EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAPNxfeH8yap5iiO3epWGBxXw3AClsFivqIuabHfzhrLKRq7/SV9w3Y0ai+h/ -AxIbKoWibjZ3oaAUsQm5o+JD/9Fkro1CXbDVfbA/9SfhIRPkh3iwwFKFHbNL/kfs -aBKWCrBSop6Cd6Xfjxv8DZhx0B3iJD3vIHEg3FuRyI393wvaiYjh62t4P/KCGpfl -Y3ZQf42hKDsV2m+AEiFowqyoL28SpuBdogWveMMlWPITLjAF+8UpOAQAi8eETYHo -HX+KPd4mTRkqm5vcO7aB6RlOmpqNAvi+ITXCO/J2Mmr+LEvZ2verf4qj/jGXRstT -yvr7/QKSQWczEw/b6anjFVP24zcCAwEAAaNQME4wHQYDVR0OBBYEFHB5zTo/9cg+ -8kTzB3WDwATror8YMB8GA1UdIwQYMBaAFHB5zTo/9cg+8kTzB3WDwATror8YMAwG -A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACH3jlYfLBR0kGALkJSq9tRI -fB6sjjVOFrZtCt5sOqxh9We3wvMsr2TPAuEVEhiPQ1w1U4x4DTQU6A65N1KOj9Le -anjjlyjNSTqXvNUbxp0wh8x4LJ8DtmsfdXXYp3LsColBmh7JQn/2TL687B9tBXiq -p3P6wkuIOHX45UOQ0kvD4wRf3t+8mL4TmrCH4YHqzbcjI1KXGDfmVQ7i64nYrl3z -v7UCA6Xh0MShv3c0AtX1ccsxObEwjJzWA30zYUQtIWxXpRtLlkiflFU8ak1HzmBU -AHsg1T1KI1012+a1j/LWkKqb91EpUjC/DlaHz8zDOH/S1IEWM6UFjwKYnHLPxoM= ------END CERTIFICATE----- - diff --git a/fakenet/listeners/ssl_utils/server.pem b/fakenet/listeners/ssl_utils/server.pem deleted file mode 100644 index aff78bd3..00000000 --- a/fakenet/listeners/ssl_utils/server.pem +++ /dev/null @@ -1,18 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIC+zCCAeOgAwIBAgIJAPHrk7xP1wNXMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNV -BAMMCWxvY2FsaG9zdDAeFw0xNjAxMjEyMjM3MDdaFw0xNjAyMjAyMjM3MDdaMBQx -EjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoC -ggEBAPNxfeH8yap5iiO3epWGBxXw3AClsFivqIuabHfzhrLKRq7/SV9w3Y0ai+h/ -AxIbKoWibjZ3oaAUsQm5o+JD/9Fkro1CXbDVfbA/9SfhIRPkh3iwwFKFHbNL/kfs -aBKWCrBSop6Cd6Xfjxv8DZhx0B3iJD3vIHEg3FuRyI393wvaiYjh62t4P/KCGpfl -Y3ZQf42hKDsV2m+AEiFowqyoL28SpuBdogWveMMlWPITLjAF+8UpOAQAi8eETYHo -HX+KPd4mTRkqm5vcO7aB6RlOmpqNAvi+ITXCO/J2Mmr+LEvZ2verf4qj/jGXRstT -yvr7/QKSQWczEw/b6anjFVP24zcCAwEAAaNQME4wHQYDVR0OBBYEFHB5zTo/9cg+ -8kTzB3WDwATror8YMB8GA1UdIwQYMBaAFHB5zTo/9cg+8kTzB3WDwATror8YMAwG -A1UdEwQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBACH3jlYfLBR0kGALkJSq9tRI -fB6sjjVOFrZtCt5sOqxh9We3wvMsr2TPAuEVEhiPQ1w1U4x4DTQU6A65N1KOj9Le -anjjlyjNSTqXvNUbxp0wh8x4LJ8DtmsfdXXYp3LsColBmh7JQn/2TL687B9tBXiq -p3P6wkuIOHX45UOQ0kvD4wRf3t+8mL4TmrCH4YHqzbcjI1KXGDfmVQ7i64nYrl3z -v7UCA6Xh0MShv3c0AtX1ccsxObEwjJzWA30zYUQtIWxXpRtLlkiflFU8ak1HzmBU -AHsg1T1KI1012+a1j/LWkKqb91EpUjC/DlaHz8zDOH/S1IEWM6UFjwKYnHLPxoM= ------END CERTIFICATE----- diff --git a/setup.py b/setup.py index 7edfb394..f9394cb0 100644 --- a/setup.py +++ b/setup.py @@ -36,8 +36,8 @@ ], package_dir={'fakenet': 'fakenet'}, package_data={'fakenet': ['*.pem','diverters/*.py', 'listeners/*.py', - 'listeners/ssl_utils/*.py', 'listeners/ssl_utils/*.pem', 'configs/*.ini', 'defaultFiles/*', - 'lib/64/*', 'lib/32/*']}, + 'listeners/ssl_utils/*.py', 'configs/*.crt', 'configs/*.key', + 'configs/*.ini', 'defaultFiles/*', 'lib/64/*', 'lib/32/*']}, entry_points={ "console_scripts": [ "fakenet=fakenet.fakenet:main", diff --git a/test/test.py b/test/test.py index 0f0e4f68..5391102d 100644 --- a/test/test.py +++ b/test/test.py @@ -20,10 +20,12 @@ import requests import netifaces import subprocess +import ssl import irc.client import configparser from collections import OrderedDict from icmplib import ping +from requests.adapters import HTTPAdapter logger = logging.getLogger('FakeNetTests') logging.basicConfig(format='%(message)s', level=logging.INFO) @@ -593,9 +595,20 @@ def _test_irc(self, hostname, port=6667): return irc_tester.test_irc() def _test_http(self, hostname, port=None, scheme=None, uri=None, - teststring=None): + teststring=None, verify=False): """Test HTTP Listener""" + + # https://stackoverflow.com/a/50215614 + # HACK to force requests to use Windows Certificate store + class SSLContextAdapter(HTTPAdapter): + def init_poolmanager(self, *args, **kwargs): + ctx = ssl.create_default_context() + ctx.load_default_certs() + kwargs['ssl_context'] = ctx + return super(SSLContextAdapter, self).init_poolmanager(*args, **kwargs) + retval = False + sess = requests.Session() scheme = scheme if scheme else 'http' uri = uri.lstrip('/') if uri else 'asdf.html' @@ -606,8 +619,12 @@ def _test_http(self, hostname, port=None, scheme=None, uri=None, else: url = '%s://%s/%s' % (scheme, hostname, uri) + if (scheme == "https") and verify and sys.platform.startswith('win32'): + adapter = SSLContextAdapter() + sess.mount(url, adapter) + try: - r = requests.get(url, timeout=3) + r = sess.get(url, timeout=3, verify=verify) if r.status_code != 200: raise FakeNetTestException('Status code %d' % (r.status_code)) @@ -620,6 +637,12 @@ def _test_http(self, hostname, port=None, scheme=None, uri=None, except requests.exceptions.Timeout as e: pass + except requests.exceptions.SSLError as e: + pass + + except ValueError as e: + pass + except FakeNetTestException as e: pass @@ -769,14 +792,9 @@ def testGeneral(self, matchspec=[]): t['DNS listener test'] = (self._test_ns, (domain_dne, dns_expected), True) t['HTTP listener test'] = (self._test_http, (arbitrary,), True) - # Enable HTTPS when we have either added Server Name Indication and Dynamic CA or have modified `_test_http` to - # Ignore certificate issues. Here is the error that arises otherwise. - # Starting new HTTPS connection (1): 8.8.8.8 - # Test HTTP listener test with SSL: Uncaught exception of type : [Errno 1] _ssl.c:510: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed - # Starting new HTTPS connection (1): 8.8.8.8 - # Test HTTP listener test with SSL: Uncaught exception of type : [Errno 1] _ssl.c:510: error:14090086:SSL routines:SSL3_GET_SERVER_CERTIFICATE:certificate verify failed - # [!!!] FAILED: HTTP listener test with SSL - # t['HTTP listener test with SSL'] = (self._test_http, (arbitrary, None, 'https'), True) + t['HTTPS listener IP test, no verify'] = (self._test_http, (arbitrary, None, 'https', None, None, False), True) + t['HTTPS listener IP test'] = (self._test_http, (arbitrary, None, 'https', None, None, True), False) + t['HTTPS listener hostname test'] = (self._test_http, ('evil.com', None, 'https', 'test.html', None, True), True) t['HTTP custom test by URI'] = (self._test_http, (arbitrary, None, None, '/test.txt', 'Wraps this'), True) t['HTTP custom test by hostname'] = (self._test_http, ('some.random.c2.com', None, None, None, 'success'), True) t['HTTP custom test by both URI and hostname'] = (self._test_http, ('both_host.com', None, None, '/and_uri.txt', 'Ahoy'), True) From c5327efecd0db8bd4f882ac6e495ab16a8a3f531 Mon Sep 17 00:00:00 2001 From: Tina Johnson Date: Fri, 15 Dec 2023 05:45:46 +0100 Subject: [PATCH 30/33] Fix to avoid use of TLSv1 or TLSv1_1 --- fakenet/listeners/ssl_utils/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fakenet/listeners/ssl_utils/__init__.py b/fakenet/listeners/ssl_utils/__init__.py index 337ec60b..bd8e2029 100644 --- a/fakenet/listeners/ssl_utils/__init__.py +++ b/fakenet/listeners/ssl_utils/__init__.py @@ -46,6 +46,8 @@ def __init__(self, config): def wrap_socket(self, s): try: ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + ctx.options |= ssl.OP_NO_TLSv1 + ctx.options |= ssl.OP_NO_TLSv1_1 except AttributeError as e: self.logger.error('Exception calling ssl.SSLContext: %s' % (e.message)) @@ -137,6 +139,8 @@ def sni_callback(self, sslsock, servername, sslctx): if servername is None: servername = self.CN newctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + newctx.options |= ssl.OP_NO_TLSv1 + newctx.options |= ssl.OP_NO_TLSv1_1 cert_file, key_file = self.create_cert(servername, self.ca_cert, self.ca_key) if cert_file is None or key_file is None: return From 5567d854eb532f3bfc13b5f05d7d04080471816f Mon Sep 17 00:00:00 2001 From: Tina Johnson Date: Thu, 28 Dec 2023 20:12:24 +0100 Subject: [PATCH 31/33] Documentation update and Python 3.12 related modifications --- LICENSE.txt | 2 +- README.md | 26 +++++++++++++++----------- docs/contributors.md | 9 ++++----- docs/developing.md | 13 +++++-------- docs/srs.md | 9 ++++----- fakenet/defaultFiles/FakeNet.html | 2 +- fakenet/defaultFiles/FakeNet.txt | 2 +- fakenet/diverters/diverterbase.py | 20 +++++++------------- fakenet/fakenet.py | 6 +++--- fakenet/listeners/HTTPListener.py | 17 ++++++++++++++--- fakenet/listeners/ProxyListener.py | 4 ++-- fakenet/listeners/RawListener.py | 17 ++++++++++++++--- test/template.ini | 12 ++++++++++++ test/test.py | 10 +++++----- 14 files changed, 88 insertions(+), 61 deletions(-) diff --git a/LICENSE.txt b/LICENSE.txt index ec2449b5..91adcb11 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -175,7 +175,7 @@ END OF TERMS AND CONDITIONS - Copyright (C) 2018 FireEye, Inc. + Copyright (C) 2016-2024 Mandiant, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 0e8be9e2..5af59db3 100644 --- a/README.md +++ b/README.md @@ -52,20 +52,23 @@ Installation on Linux requires the following dependencies: * libnetfilterqueue development files (e.g. libnetfilter-queue-dev for Ubuntu). -Install these dependencies using the following command: +To install these dependencies, use the following command: sudo apt-get install build-essential python-dev libnetfilter-queue-dev -Install FakeNet-NG as a Python module using pip: +Either install FakeNet-NG as a Python module using pip: pip install https://github.com/mandiant/flare-fakenet-ng/zipball/master -Or by obtaining the latest source code and installing it manually: +Or, by obtaining the latest source code and installing it manually: git clone https://github.com/mandiant/flare-fakenet-ng/ +Next, install Microsoft C++ Build Tools from [here](https://visualstudio.microsoft.com/visual-cpp-build-tools/). + Change directory to the downloaded flare-fakenet-ng and run: + pip install setuptools python setup.py install Execute FakeNet-NG by running 'fakenet' in any directory. @@ -77,30 +80,31 @@ Finally if you would like to avoid installing FakeNet-NG and just want to run it as-is (e.g. for development), then you would need to obtain the source code and install dependencies as follows: -1) Install 64-bit or 32-bit Python 3.7.x for the 64-bit or 32-bit versions +1) Install 64-bit or 32-bit Python 3.12 for the 64-bit or 32-bit versions of Windows respectively. 2) Install Python dependencies: - + ``` pip install pydivert dnslib dpkt pyopenssl pyftpdlib netifaces - + ``` *NOTE*: pydivert will also download and install WinDivert library and driver in the `%PYTHONHOME%\DLLs` directory. FakeNet-NG bundles those files so they are not necessary for normal use. -2b) Optionally, you can install the following module used for testing: - + Optionally, you can install the following module used for testing: + ``` pip install requests + ``` 3) Download the FakeNet-NG source code: git clone https://github.com/mandiant/flare-fakenet-ng -Execute FakeNet-NG by running it with a Python interpreter in a privileged +4) Execute FakeNet-NG by running it with a Python interpreter in a privileged shell: - + ``` python -m fakenet.fakenet - + ``` Usage ===== diff --git a/docs/contributors.md b/docs/contributors.md index ed57688f..1134fa28 100644 --- a/docs/contributors.md +++ b/docs/contributors.md @@ -13,13 +13,13 @@ malware analysis on Windows XP. ## Windows Peter Kacherginsky [implemented -FakeNet-NG](https://www.fireeye.com/blog/threat-research/2016/08/fakenet-ng_next_gen.html) +FakeNet-NG](https://www.mandiant.com/resources/blog/fakenet-ng-next-gen) targeting modern versions of Windows. ## Linux and Core Michael Bailey [implemented FakeNet-NG on -Linux](https://www.fireeye.com/blog/threat-research/2017/07/linux-support-for-fakenet-ng.html), +Linux](https://www.mandiant.com/resources/blog/introducing-linux-support-fakenet-ng-flares-next-generation-dynamic-network-analysis-tool), and later refactored FakeNet-NG to use this as the unified packet processing logic for both Windows and Linux. @@ -30,6 +30,5 @@ Homan developed the original concept of using a protocol "taste" callback to sample traffic and direct clients to the appropriate server ports. Matthew Haigh, Michael Bailey, and Peter Kacherginsky conceptualized the Proxy Listener and Hidden Listener mechanisms for introducing both of these content-based -protocol detection features to FakeNet-NG. Matthew Haigh then [implemented -Content-Based Protocol -Detection](https://www.fireeye.com/blog/threat-research/2017/10/fakenet-content-based-protocol-detection.html). +protocol detection features to FakeNet-NG. Matthew Haigh then implemented +Content-Based Protocol Detection. diff --git a/docs/developing.md b/docs/developing.md index 47a01b8b..6a58db4f 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -181,9 +181,8 @@ utilities (i.e. `pip`). Use an administrative command prompt where applicable for installing Python modules for all users. Pre-requisites: -* Python 2.7 x86 with `pip` -* Visual C++ for Python 2.7 development, available at: - +* Python 3.12 x86 with `pip` +* Microsoft C++ [Build Tools](https://visualstudio.microsoft.com/visual-cpp-build-tools/) Before installing `pyinstaller`, you may wish to take the following steps to prevent the error `ImportError: No module named PyInstaller`: @@ -199,11 +198,10 @@ Install FakeNet-NG to acquire most modules: python setup.py install ``` -Obtain PyDivert 2.0.9, the only version known to work with FakeNet-NG releases -prepared with PyInstaller: +Obtain PyDivert: ``` -pip install pydivert==2.0.9 +pip install pydivert ``` Install `pyinstaller`: @@ -245,7 +243,6 @@ fakenet1.4.3\ | +-- CustomProviderExample.py |   +-- sample_custom_response.ini | +-- sample_raw_response.txt - | +-- sample_raw_tcp_response.txt | +-- defaultFiles\ | +-- FakeNet.gif @@ -260,7 +257,7 @@ fakenet1.4.3\ | +-- listeners\    +-- ssl_utils - +-- __init__.pyc + +-- __init__.py +-- privkey.pem +-- server.pem +-- ssl_detector.py diff --git a/docs/srs.md b/docs/srs.md index 89333cdf..9e9c934c 100644 --- a/docs/srs.md +++ b/docs/srs.md @@ -24,19 +24,18 @@ Analysis](https://nostarch.com/malware). ## History FakeNet-NG was initially released August 3, 2016 by Peter Kacherginsky with support for Windows: [FakeNet-NG: Next Generation Dynamic Network Analysis -Tool](https://www.fireeye.com/blog/threat-research/2016/08/fakenet-ng_next_gen.html). +Tool](https://www.mandiant.com/resources/blog/fakenet-ng-next-gen). On July 5, 2017 FakeNet-NG was updated by Michael Bailey to add support for Linux: [Introducing Linux Support for FakeNet-NG: FLARE's Next Generation Dynamic Network Analysis -Tool](https://www.fireeye.com/blog/threat-research/2017/07/linux-support-for-fakenet-ng.html). +Tool](https://www.mandiant.com/resources/blog/introducing-linux-support-fakenet-ng-flares-next-generation-dynamic-network-analysis-tool). The next significant FakeNet-NG release was by Matthew Haigh on October 23, 2017 to introduce a proxy listener to sample, identify, and route traffic to -the most appropriate listener: [New FakeNet-NG Feature: Content-Based Protocol -Detection](https://www.fireeye.com/blog/threat-research/2017/10/fakenet-content-based-protocol-detection.html). +the most appropriate listener by implementing Content-Based Protocol Detection. -FireEye's [flare-fakenet-ng](https://github.com/fireeye/flare-fakenet-ng) +Mandiant's [flare-fakenet-ng](https://github.com/mandiant/flare-fakenet-ng) repository contains `README.md` which documents usage and configuration; and `docs/internals.md` which describes Diverter internals for Linux. diff --git a/fakenet/defaultFiles/FakeNet.html b/fakenet/defaultFiles/FakeNet.html index 418c0dd4..e7746792 100644 --- a/fakenet/defaultFiles/FakeNet.html +++ b/fakenet/defaultFiles/FakeNet.html @@ -32,6 +32,6 @@

Contact

For bugs, crashes, or other comments please contact The FLARE Team by email -FakeNet@fireeye.com. +FakeNet@mandiant.com. \ No newline at end of file diff --git a/fakenet/defaultFiles/FakeNet.txt b/fakenet/defaultFiles/FakeNet.txt index 10c24c8d..b4b0c326 100644 --- a/fakenet/defaultFiles/FakeNet.txt +++ b/fakenet/defaultFiles/FakeNet.txt @@ -14,4 +14,4 @@ FakeNet-NG is based on the excellent Fakenet tool developed by Andrew Honig and Contact -For bugs, crashes, or other comments please contact the FLARE Team by email FakeNet@fireeye.com \ No newline at end of file +For bugs, crashes, or other comments please contact the FLARE Team by email FakeNet@mandiant.com \ No newline at end of file diff --git a/fakenet/diverters/diverterbase.py b/fakenet/diverters/diverterbase.py index 521040da..83c3c1d8 100644 --- a/fakenet/diverters/diverterbase.py +++ b/fakenet/diverters/diverterbase.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2023 Mandiant, Inc. All rights reserved. +# Copyright (C) 2016-2024 Mandiant, Inc. All rights reserved. import os import abc @@ -1239,12 +1239,6 @@ def formatPkt(self, pkt, pid, comm): Returns: A str containing the log line """ - if pid == None: - pid = 'None' - - if comm == None: - comm = 'None' - logline = '' if pkt.proto == 'UDP': @@ -1252,8 +1246,8 @@ def formatPkt(self, pkt, pid, comm): logline = fmt.format( label=pkt.label, proto=pkt.proto, - pid=pid, - comm=comm, + pid=str(pid), + comm=str(comm), src=pkt.src_ip, sport=pkt.sport, dst=pkt.dst_ip, @@ -1284,8 +1278,8 @@ def formatPkt(self, pkt, pid, comm): logline = fmt.format( label=pkt.label, proto=pkt.proto, - pid=pid, - comm=comm, + pid=str(pid), + comm=str(comm), src=pkt.src_ip, sport=pkt.sport, dst=pkt.dst_ip, @@ -1299,8 +1293,8 @@ def formatPkt(self, pkt, pid, comm): logline = fmt.format( label=pkt.label, proto='UNK', - pid=pid, - comm=comm, + pid=str(pid), + comm=str(comm), src=str(pkt.src_ip), sport=str(pkt.sport), dst=str(pkt.dst_ip), diff --git a/fakenet/fakenet.py b/fakenet/fakenet.py index 7acd3cbd..9dfcf58e 100644 --- a/fakenet/fakenet.py +++ b/fakenet/fakenet.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2023 Mandiant, Inc. All rights reserved. +# Copyright (C) 2016-2024 Mandiant, Inc. All rights reserved. #!/usr/bin/env python # @@ -6,7 +6,7 @@ # analysts and penetration testers. # # Original developer: Peter Kacherginsky -# Current developer: FireEye FLARE Team (FakeNet@fireeye.com) +# Current developer: Mandiant FLARE Team (FakeNet@mandiant.com) import logging import logging.handlers @@ -341,7 +341,7 @@ def main(): Version 3.0 (alpha) _____________________________________________________________ Developed by FLARE Team - Copyright (C) 2016-2022 Mandiant, Inc. All rights reserved. + Copyright (C) 2016-2024 Mandiant, Inc. All rights reserved. _____________________________________________________________ """) diff --git a/fakenet/listeners/HTTPListener.py b/fakenet/listeners/HTTPListener.py index 421630fd..906b4123 100644 --- a/fakenet/listeners/HTTPListener.py +++ b/fakenet/listeners/HTTPListener.py @@ -1,11 +1,12 @@ -# Copyright (C) 2016-2023 Mandiant, Inc. All rights reserved. +# Copyright (C) 2016-2024 Mandiant, Inc. All rights reserved. import logging from configparser import ConfigParser import os import sys -import imp +import importlib.util +import importlib.machinery import threading import socketserver @@ -47,6 +48,16 @@ def qualify_file_path(filename, fallbackdir): return path +def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + class CustomResponse(object): def __init__(self, name, conf, configroot): @@ -84,7 +95,7 @@ def __init__(self, name, conf, configroot): self.handler = None pymod_path = qualify_file_path(conf.get('httpdynamic'), configroot) if pymod_path: - pymod = imp.load_source('cr_' + self.name, pymod_path) + pymod = load_source('cr_' + self.name, pymod_path) funcname = 'HandleHttp' funcname_legacy = 'HandleRequest' if hasattr(pymod, funcname): diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index 55747859..b21f0459 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -138,7 +138,7 @@ def run(self): self.listener_q.put(data) else: self.sock.close() - exit(1) + sys.exit(1) except Exception as e: self.logger.debug('Listener socket exception %s' % e.message) @@ -321,7 +321,7 @@ def main(): TCP_server.shutdown() finally: logger.debug('Closing ProxyListener') - exit(1) + sys.exit(1) logger.debug('Exiting') TCP_server.shutdown() diff --git a/fakenet/listeners/RawListener.py b/fakenet/listeners/RawListener.py index a5ce8c87..d49db6ce 100644 --- a/fakenet/listeners/RawListener.py +++ b/fakenet/listeners/RawListener.py @@ -1,11 +1,12 @@ -# Copyright (C) 2016-2023 Mandiant, Inc. All rights reserved. +# Copyright (C) 2016-2024 Mandiant, Inc. All rights reserved. import logging from configparser import ConfigParser import os import sys -import imp +import importlib.util +import importlib.machinery import base64 import threading @@ -29,6 +30,16 @@ def qualify_file_path(filename, fallbackdir): return path +def load_source(modname, filename): + loader = importlib.machinery.SourceFileLoader(modname, filename) + spec = importlib.util.spec_from_file_location(modname, filename, loader=loader) + module = importlib.util.module_from_spec(spec) + # The module is always executed and not cached in sys.modules. + # Uncomment the following line to cache the module. + # sys.modules[module.__name__] = module + loader.exec_module(module) + return module + class RawCustomResponse(object): def __init__(self, proto, name, conf, configroot): @@ -72,7 +83,7 @@ def __init__(self, proto, name, conf, configroot): pymodpath = qualify_file_path(conf.get(spec_dyn), configroot) if pymodpath: - pymod = imp.load_source('cr_raw_' + self.name, pymodpath) + pymod = load_source('cr_raw_' + self.name, pymodpath) funcname = 'Handle%s' % (proto.capitalize()) if not hasattr(pymod, funcname): raise ValueError('Loaded %s module %s has no function %s' % diff --git a/test/template.ini b/test/template.ini index bf94a156..b83ef6df 100644 --- a/test/template.ini +++ b/test/template.ini @@ -327,6 +327,18 @@ DumpHTTPPostsFilePrefix: http Hidden: False Custom: custom_responses.ini +[HTTPListener443] +Enabled: True +Port: 443 +Protocol: TCP +Listener: HTTPListener +UseSSL: Yes +Webroot: defaultFiles/ +Timeout: 10 +DumpHTTPPosts: Yes +DumpHTTPPostsFilePrefix: http +Hidden: False + [HTTPListener81] Enabled: True Port: 81 diff --git a/test/test.py b/test/test.py index 5391102d..6ec219d4 100644 --- a/test/test.py +++ b/test/test.py @@ -1,4 +1,4 @@ -# Copyright (C) 2016-2022 Mandiant, Inc. All rights reserved. +# Copyright (C) 2016-2024 Mandiant, Inc. All rights reserved. import os import re @@ -665,11 +665,11 @@ def update_hash(buf): f.connect(hostname, port) f.login() f.set_pasv(False) - f.retrbinary('RETR FakeNet.gif', update_hash) + f.retrbinary('RETR FakeNet.txt', update_hash) f.quit() digest = m.digest() - expected = binascii.unhexlify('a6b78c4791dc8110dec6c55f8a756395') + expected = binascii.unhexlify('8DFFD096C5F2C84632F0F8667638A0B0') return (digest == expected) @@ -923,7 +923,7 @@ def __init__(self, startingpath, singlehost=True): self.listener_host_white = 8083 # HTTP listener with host whitelists self.localhost = '127.0.0.1' self.dns_expected = '192.0.2.123' - self.domain_dne = 'does-not-exist-amirite.fireeye.com' + self.domain_dne = 'does-not-exist-amirite.mandiant.com' self.sender = 'from-fakenet@example.org' self.recipient = 'to-fakenet@example.org' self.smtpmsg = 'FakeNet-NG SMTP test email' @@ -943,7 +943,7 @@ def genFakenetCmd(self): (self.fakenet, self.stopflag, self.logpath, self.configpath)) def is_ip(s): - pat = '^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' + pat = r'^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$' return bool(re.match(pat, s)) def main(): From acb0d96dd6fb588ea099f2a05dd3f1b42944c751 Mon Sep 17 00:00:00 2001 From: Tina Johnson Date: Thu, 28 Dec 2023 23:23:33 +0100 Subject: [PATCH 32/33] Fix exception error message syntax --- fakenet/diverters/diverterbase.py | 2 +- fakenet/diverters/linutil.py | 28 ++++++++++++++-------------- fakenet/listeners/ProxyListener.py | 4 ++-- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/fakenet/diverters/diverterbase.py b/fakenet/diverters/diverterbase.py index 83c3c1d8..59efbd1a 100644 --- a/fakenet/diverters/diverterbase.py +++ b/fakenet/diverters/diverterbase.py @@ -937,7 +937,7 @@ def _build_cmd(self, tmpl, pid, comm, src_ip, sport, dst_ip, dport): except KeyError as e: self.logger.error(('Failed to build ExecuteCmd for port %d due ' + 'to erroneous format key: %s') % - (dport, e.message)) + (dport, e)) return cmd diff --git a/fakenet/diverters/linutil.py b/fakenet/diverters/linutil.py index fbdb1289..679656ff 100644 --- a/fakenet/diverters/linutil.py +++ b/fakenet/diverters/linutil.py @@ -145,10 +145,10 @@ def start(self, timeout_sec=0.5): self._bound = True except OSError as e: self.logger.error('Failed to start queue for %s: %s' % - (str(self), e.message)) + (str(self), e)) except RuntimeWarning as e: self.logger.error('Failed to start queue for %s: %s' % - (str(self), e.message)) + (str(self), e)) if not self._bound: return False @@ -166,7 +166,7 @@ def start(self, timeout_sec=0.5): self._thread.start() self._started = True except RuntimeError as e: - self.logger.error('Failed to start queue thread: %s' % (e.message)) + self.logger.error('Failed to start queue thread: %s' % (e)) return self._started @@ -251,7 +251,7 @@ def parse(self, multi=False, max_col=None): retval = cb_retval break except IOError as e: - self.logger.error('Failed accessing %s: %s' % (path, e.message)) + self.logger.error('Failed accessing %s: %s' % (path, e)) # All or nothing retval = [] if multi else None @@ -313,7 +313,7 @@ def linux_capture_iptables(self): ret = p.wait() except OSError as e: self.logger.error('Error executing iptables-save: %s' % - (e.message)) + (e)) return ret @@ -328,7 +328,7 @@ def linux_restore_iptables(self): ret = p.wait() except OSError as e: self.logger.error('Error executing iptables-restore: %s' % - (e.message)) + (e)) return ret @@ -351,7 +351,7 @@ def linux_flush_iptables(self): self.logger.error('Received return code %d from %s' + (ret, cmd)) except OSError as e: - self.logger.error('Error executing %s: %s' % (cmd, e.message)) + self.logger.error('Error executing %s: %s' % (cmd, e)) return rets @@ -388,7 +388,7 @@ def linux_get_current_nfnlq_bindings(self): self.logger.debug(('Failed to open %s to enumerate netfilter ' 'netlink queues, caller may proceed as if ' 'none are in use: %s') % - (procfs_path, e.message)) + (procfs_path, e)) return qnos @@ -445,7 +445,7 @@ def _linux_get_ifaces(self): ifaces.append(fields[0].strip()) except IOError as e: self.logger.error('Failed to open %s to enumerate interfaces: %s' % - (procfs_path, e.message)) + (procfs_path, e)) return ifaces @@ -472,7 +472,7 @@ def linux_modifylocaldns_ephemeral(self): except IOError as e: self.logger.error(('Failed to open %s to save DNS ' + 'configuration: %s') % (resolvconf_path, - e.message)) + e)) if self.old_dns: try: @@ -484,7 +484,7 @@ def linux_modifylocaldns_ephemeral(self): except IOError as e: self.logger.error(('Failed to open %s to modify DNS ' + 'configuration: %s') % (resolvconf_path, - e.message)) + e)) def linux_restore_local_dns(self): resolvconf_path = '/etc/resolv.conf' @@ -496,7 +496,7 @@ def linux_restore_local_dns(self): except IOError as e: self.logger.error(('Failed to open %s to restore DNS ' + 'configuration: %s') % (resolvconf_path, - e.message)) + e)) def linux_find_processes(self, names): """But what if a blacklisted process spawns after we call @@ -618,7 +618,7 @@ def _linux_find_sock_by_endpoint_unsafe(self, ipver, proto_name, ip, port, (line.strip())) except IOError as e: self.logger.error('No such protocol/IP ver (%s) or error: %s' % - (procfs_path, e.message)) + (procfs_path, e)) return inode @@ -701,7 +701,7 @@ def linux_get_comm_by_pid(self, pid): comm = f.read().strip() except IOError as e: self.pdebug(DPROCFS, 'Failed to open %s: %s' % - (procfs_path, e.message)) + (procfs_path, e)) return comm def linux_get_pid_comm_by_endpoint(self, ipver, proto_name, ip, port): diff --git a/fakenet/listeners/ProxyListener.py b/fakenet/listeners/ProxyListener.py index b21f0459..791513ab 100644 --- a/fakenet/listeners/ProxyListener.py +++ b/fakenet/listeners/ProxyListener.py @@ -140,7 +140,7 @@ def run(self): self.sock.close() sys.exit(1) except Exception as e: - self.logger.debug('Listener socket exception %s' % e.message) + self.logger.debug('Listener socket exception %s' % e) class ThreadedTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer): daemon_threads = True @@ -190,7 +190,7 @@ def handle(self): self.server.logger.debug('%s', '-'*80,) except Exception as e: - self.server.logger.warning('recv() error: %s' % e.message) + self.server.logger.warning('recv() error: %s' % e) if data: if ssl_detector.looks_like_ssl(data): From 65ab931254a0c4d40bd7ce5d1df2330502da1441 Mon Sep 17 00:00:00 2001 From: Tina Johnson Date: Fri, 29 Dec 2023 04:38:37 +0100 Subject: [PATCH 33/33] Revert filemd5 change in test.py --- test/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test.py b/test/test.py index 6ec219d4..11b07edb 100644 --- a/test/test.py +++ b/test/test.py @@ -665,11 +665,11 @@ def update_hash(buf): f.connect(hostname, port) f.login() f.set_pasv(False) - f.retrbinary('RETR FakeNet.txt', update_hash) + f.retrbinary('RETR FakeNet.gif', update_hash) f.quit() digest = m.digest() - expected = binascii.unhexlify('8DFFD096C5F2C84632F0F8667638A0B0') + expected = binascii.unhexlify('a6b78c4791dc8110dec6c55f8a756395') return (digest == expected)