From 3aeac7c6317aadebe48025541251744b92a39f7a Mon Sep 17 00:00:00 2001 From: jhao Date: Fri, 26 Mar 2021 16:19:13 +0800 Subject: [PATCH] [update] launcher --- db/MongodbClient.py | 74 --------------------------------------------- db/dbClient.py | 16 ++-------- db/redisClient.py | 21 ++++++++++++- db/ssdbClient.py | 22 ++++++++++++-- helper/launcher.py | 66 ++++++++++++++++++++++++++++++++++++++++ proxyPool.py | 10 ++---- 6 files changed, 112 insertions(+), 97 deletions(-) delete mode 100644 db/MongodbClient.py create mode 100644 helper/launcher.py diff --git a/db/MongodbClient.py b/db/MongodbClient.py deleted file mode 100644 index a30ef6cf1..000000000 --- a/db/MongodbClient.py +++ /dev/null @@ -1,74 +0,0 @@ -# coding: utf-8 -""" -------------------------------------------------- - File Name： MongodbClient.py - Description : 封装mongodb操作 - Author : JHao netAir - date： 2017/3/3 -------------------------------------------------- - Change Activity: - 2017/3/3: - 2017/9/26:完成对mongodb的支持 -------------------------------------------------- -""" -__author__ = 'Maps netAir' - -from pymongo import MongoClient - - -class MongodbClient(object): - def __init__(self, name, host, port, **kwargs): - self.name = name - self.client = MongoClient(host, port, **kwargs) - self.db = self.client.proxy - - def changeTable(self, name): - self.name = name - - def get(self, proxy): - data = self.db[self.name].find_one({'proxy': proxy}) - return data['num'] if data != None else None - - def put(self, proxy, num=1): - if self.db[self.name].find_one({'proxy': proxy}): - return None - else: - self.db[self.name].insert({'proxy': proxy, 'num': num}) - - def pop(self): - data = list(self.db[self.name].aggregate([{'$sample': {'size': 1}}])) - if data: - data = data[0] - value = data['proxy'] - self.delete(value) - return {'proxy': value, 'value': data['num']} - return None - - def delete(self, value): - self.db[self.name].remove({'proxy': value}) - - def getAll(self): - return {p['proxy']: p['num'] for p in self.db[self.name].find()} - - def clean(self): - self.client.drop_database('proxy') - - def delete_all(self): - self.db[self.name].remove() - - def update(self, key, value): - self.db[self.name].update({'proxy': key}, {'$inc': {'num': value}}) - - def exists(self, key): - return True if self.db[self.name].find_one({'proxy': key}) != None else False - - def getNumber(self): - return self.db[self.name].count() - - -if __name__ == "__main__": - db = MongodbClient('first', 'localhost', 27017) - # db.put('127.0.0.1:1') - # db2 = MongodbClient('second', 'localhost', 27017) - # db2.put('127.0.0.1:2') - print(db.pop()) diff --git a/db/dbClient.py b/db/dbClient.py index 411db7d5d..7eb1ceb28 100644 --- a/db/dbClient.py +++ b/db/dbClient.py @@ -53,9 +53,7 @@ def __init__(self, db_conn): init :return: """ - self.db_conn = db_conn self.parseDbConn(db_conn) - self.__printConfig() self.__initDbClient() @classmethod @@ -79,8 +77,6 @@ def __initDbClient(self): __type = "ssdbClient" elif "REDIS" == self.db_type: __type = "redisClient" - elif "MONGODB" == self.db_type: - __type = "mongodbClient" else: pass assert __type, 'type error, Not support DB type: {}'.format(self.db_type) @@ -90,15 +86,6 @@ def __initDbClient(self): password=self.db_pwd, db=self.db_name) - def __printConfig(self): - print("============ DATABASE CONFIGURE =========================") - print("DB_TYPE: %s" % self.db_type) - print("DB_HOST: %s" % self.db_host) - print("DB_PORT: %s" % self.db_port) - print("DB_NAME: %s" % self.db_name) - print("DB_USER: %s" % self.db_user) - print("=========================================================") - def get(self, **kwargs): return self.client.get(**kwargs) @@ -128,3 +115,6 @@ def changeTable(self, name): def getCount(self): return self.client.getCount() + + def test(self): + return self.client.test() diff --git a/db/redisClient.py b/db/redisClient.py index 9e3a46a5a..8948a1ffb 100644 --- a/db/redisClient.py +++ b/db/redisClient.py @@ -13,7 +13,9 @@ """ __author__ = 'JHao' +from redis.exceptions import TimeoutError, ConnectionError, ResponseError from redis.connection import BlockingConnectionPool +from handler.logHandler import LogHandler from random import choice from redis import Redis @@ -38,7 +40,10 @@ def __init__(self, **kwargs): """ self.name = "" kwargs.pop("username") - self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs)) + self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, + timeout=5, + socket_timeout=5, + **kwargs)) def get(self): """ @@ -127,3 +132,17 @@ def changeTable(self, name): :return: """ self.name = name + + def test(self): + log = LogHandler('redis_client') + try: + self.getCount() + except TimeoutError as e: + log.error('redis connection time out: %s' % str(e), exc_info=True) + return e + except ConnectionError as e: + log.error('redis connection error: %s' % str(e), exc_info=True) + return e + except ResponseError as e: + log.error('redis connection error: %s' % str(e), exc_info=True) + return e diff --git a/db/ssdbClient.py b/db/ssdbClient.py index 3e912b61f..97d3d256c 100644 --- a/db/ssdbClient.py +++ b/db/ssdbClient.py @@ -15,8 +15,9 @@ ------------------------------------------------- """ __author__ = 'JHao' - +from redis.exceptions import TimeoutError, ConnectionError, ResponseError from redis.connection import BlockingConnectionPool +from handler.logHandler import LogHandler from random import choice from redis import Redis @@ -39,7 +40,10 @@ def __init__(self, **kwargs): """ self.name = "" kwargs.pop("username") - self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, **kwargs)) + self.__conn = Redis(connection_pool=BlockingConnectionPool(decode_responses=True, + timeout=5, + socket_timeout=5, + **kwargs)) def get(self): """ @@ -128,3 +132,17 @@ def changeTable(self, name): :return: """ self.name = name + + def test(self): + log = LogHandler('ssdb_client') + try: + self.getCount() + except TimeoutError as e: + log.error('ssdb connection time out: %s' % str(e), exc_info=True) + return e + except ConnectionError as e: + log.error('ssdb connection error: %s' % str(e), exc_info=True) + return e + except ResponseError as e: + log.error('ssdb connection error: %s' % str(e), exc_info=True) + return e diff --git a/helper/launcher.py b/helper/launcher.py new file mode 100644 index 000000000..491637bcf --- /dev/null +++ b/helper/launcher.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- +""" +------------------------------------------------- + File Name： launcher + Description : 启动器 + Author : JHao + date： 2021/3/26 +------------------------------------------------- + Change Activity: + 2021/3/26: 启动器 +------------------------------------------------- +""" +__author__ = 'JHao' + +import sys +from db.dbClient import DbClient +from handler.logHandler import LogHandler +from handler.configHandler import ConfigHandler + +log = LogHandler('launcher') + + +def startServer(): + __beforeStart() + from api.proxyApi import runFlask + runFlask() + + +def startScheduler(): + __beforeStart() + from helper.scheduler import runScheduler + runScheduler() + + +def __beforeStart(): + __showVersion() + __showConfigure() + if __checkDBConfig(): + log.info('exit!') + sys.exit() + + +def __showVersion(): + from setting import VERSION + log.info("ProxyPool Version: %s" % VERSION) + + +def __showConfigure(): + conf = ConfigHandler() + log.info("ProxyPool configure HOST: %s" % conf.serverHost) + log.info("ProxyPool configure PORT: %s" % conf.serverPort) + log.info("ProxyPool configure DB_CONN: %s" % conf.dbConn) + log.info("ProxyPool configure PROXY_FETCHER: %s" % conf.fetchers) + + +def __checkDBConfig(): + conf = ConfigHandler() + db = DbClient(conf.dbConn) + log.info("============ DATABASE CONFIGURE ================") + log.info("DB_TYPE: %s" % db.db_type) + log.info("DB_HOST: %s" % db.db_host) + log.info("DB_PORT: %s" % db.db_port) + log.info("DB_NAME: %s" % db.db_name) + log.info("DB_USER: %s" % db.db_user) + log.info("=================================================") + return db.test() diff --git a/proxyPool.py b/proxyPool.py index ec36e2853..59afaadeb 100644 --- a/proxyPool.py +++ b/proxyPool.py @@ -13,7 +13,7 @@ __author__ = 'JHao' import click - +from helper.launcher import startServer, startScheduler from setting import BANNER, VERSION CONTEXT_SETTINGS = dict(help_option_names=['-h', '--help']) @@ -29,18 +29,14 @@ def cli(): def schedule(): """ 启动调度程序 """ click.echo(BANNER) - click.echo("VERSION: %s\n" % VERSION) - from helper.scheduler import runScheduler - runScheduler() + startScheduler() @cli.command(name="server") def server(): """ 启动api服务 """ click.echo(BANNER) - click.echo("VERSION: %s\n" % VERSION) - from api.proxyApi import runFlask - runFlask() + startServer() if __name__ == '__main__':