From 96303aaa0847361f3c37a5ef66258d60b0695633 Mon Sep 17 00:00:00 2001 From: will <157747074@qq.com> Date: Fri, 15 Jun 2018 09:18:33 +0800 Subject: [PATCH 1/3] binance data update bug --- QUANTAXIS/QAFetch/QAbinance.py | 10 ++++++---- QUANTAXIS/QASU/save_binance.py | 10 +++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/QUANTAXIS/QAFetch/QAbinance.py b/QUANTAXIS/QAFetch/QAbinance.py index f1852c4dc..50277935c 100644 --- a/QUANTAXIS/QAFetch/QAbinance.py +++ b/QUANTAXIS/QAFetch/QAbinance.py @@ -47,9 +47,11 @@ def QA_fetch_kline(symbol, start_time, end_time, frequency): raise ConnectTimeout(MESSAGE_NOTICE) klines = json.loads(req.content) if len(klines) == 0: - return datas + break datas.extend(klines) start_time = klines[-1][6] + if len(datas) == 0: + return None frame = pd.DataFrame(datas) frame.columns = columne_names frame['symbol'] = symbol @@ -65,8 +67,8 @@ def QA_fetch_kline(symbol, start_time, end_time, frequency): url = urljoin(Binance_base_url, "/api/v1/klines") start = time.mktime(datetime.datetime(2017, 6, 1).timetuple()) - end = time.mktime(datetime.datetime(2017, 7, 1).timetuple()) + end = time.mktime(datetime.datetime(2017, 8, 1).timetuple()) print(start * 1000) print(end * 1000) - datas = QA_fetch_kline("ETHBTC", start, end) - print(len(datas)) + data = QA_fetch_kline("ETHBTC", start, end, '1h') + print(data[0]) diff --git a/QUANTAXIS/QASU/save_binance.py b/QUANTAXIS/QASU/save_binance.py index ef618080e..1f7d8907d 100644 --- a/QUANTAXIS/QASU/save_binance.py +++ b/QUANTAXIS/QASU/save_binance.py @@ -30,16 +30,16 @@ def QA_SU_save_binance(frequency): if ref.count() > 0: start_stamp = ref.next()['start_time'] / 1000 start_time = datetime.datetime.fromtimestamp(start_stamp) - QA_util_log_info('UPDATE_SYMBOL \n Trying updating {} from {} to {}'.format( - symbol_info['symbol'], start_time, end)) + QA_util_log_info('UPDATE_SYMBOL {} Trying updating {} from {} to {}'.format( + frequency, symbol_info['symbol'], start_time, end)) else: start_time = BINANCE_MIN_DATE - QA_util_log_info('NEW_SYMBOL \n Trying downloading {} from {} to {}'.format( - symbol_info['symbol'], start_time, end)) + QA_util_log_info('NEW_SYMBOL {} Trying downloading {} from {} to {}'.format( + frequency, symbol_info['symbol'], start_time, end)) data = QA_fetch_kline(symbol_info['symbol'], time.mktime(start_time.timetuple()), time.mktime(end.timetuple()), frequency) - if len(data) == 0: + if data is None: QA_util_log_info('SYMBOL {} from {} to {} has no data'.format( symbol_info['symbol'], start_time, end)) continue From 964f90495a03d1f06b915ed23c78f474e1330785 Mon Sep 17 00:00:00 2001 From: will <157747074@qq.com> Date: Fri, 15 Jun 2018 10:52:52 +0800 Subject: [PATCH 2/3] save binance timezone update --- QUANTAXIS/QACmd/__init__.py | 2 +- QUANTAXIS/QAFetch/QAbinance.py | 12 ++++++++---- QUANTAXIS/QASU/save_binance.py | 20 ++++++++++++++------ 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/QUANTAXIS/QACmd/__init__.py b/QUANTAXIS/QACmd/__init__.py index 17cb49d8a..730224747 100755 --- a/QUANTAXIS/QACmd/__init__.py +++ b/QUANTAXIS/QACmd/__init__.py @@ -250,8 +250,8 @@ def do_save(self, arg): # QA_SU_save_stock_info('tdx') elif len(arg) == 1 and arg[0] == "binance": QA_SU_save_symbols() - QA_SU_save_binance_1hour() QA_SU_save_binance_1day() + QA_SU_save_binance_1hour() QA_SU_save_binance_1min() else: for i in arg: diff --git a/QUANTAXIS/QAFetch/QAbinance.py b/QUANTAXIS/QAFetch/QAbinance.py index 50277935c..284d1a326 100644 --- a/QUANTAXIS/QAFetch/QAbinance.py +++ b/QUANTAXIS/QAFetch/QAbinance.py @@ -64,11 +64,15 @@ def QA_fetch_kline(symbol, start_time, end_time, frequency): # a = requests.get(url) # print(a.content) # print(json.loads(a.content)) - + import pytz + from dateutil.tz import * + tz = pytz.timezone("Asia/Shanghai") url = urljoin(Binance_base_url, "/api/v1/klines") - start = time.mktime(datetime.datetime(2017, 6, 1).timetuple()) - end = time.mktime(datetime.datetime(2017, 8, 1).timetuple()) + start = time.mktime(datetime.datetime(2018, 6, 13,tzinfo=tzutc()).timetuple()) + end = time.mktime(datetime.datetime(2018, 6, 14, tzinfo=tzutc()).timetuple()) print(start * 1000) print(end * 1000) - data = QA_fetch_kline("ETHBTC", start, end, '1h') + data = QA_fetch_kline("ETHBTC", start, end, '1d') + print(len(data)) print(data[0]) + print(data[-1]) diff --git a/QUANTAXIS/QASU/save_binance.py b/QUANTAXIS/QASU/save_binance.py index 1f7d8907d..22338470d 100644 --- a/QUANTAXIS/QASU/save_binance.py +++ b/QUANTAXIS/QASU/save_binance.py @@ -1,5 +1,7 @@ import datetime import time +from dateutil.tz import tzutc +from dateutil.relativedelta import relativedelta from QUANTAXIS.QAUtil import (DATABASE, QASETTING, QA_util_get_real_date, QA_util_log_info, QA_util_to_json_from_pandas, trade_date_sse) @@ -8,16 +10,22 @@ import pymongo # binance的历史数据只是从2017年7月开始有，以前的貌似都没有保留 . author:Will -BINANCE_MIN_DATE = datetime.datetime(2017, 7, 1) +BINANCE_MIN_DATE = datetime.datetime(2017, 7, 1, tzinfo=tzutc()) +FREQUANCY_DICT ={ + "1m":relativedelta(minutes=-1), + "1d":relativedelta(days=-1), + "1h":relativedelta(hours=-1) +} def QA_SU_save_binance(frequency): symbol_list = QA_fetch_symbol() col = QASETTING.client.binance[frequency] col.create_index( - [("symbol", pymongo.ASCENDING), ("start_time", pymongo.ASCENDING)]) + [("symbol", pymongo.ASCENDING), ("start_time", pymongo.ASCENDING)],unique=True) - end = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0) + end = datetime.datetime.now(tzutc()) + end += FREQUANCY_DICT.get(frequency) for index, symbol_info in enumerate(symbol_list): QA_util_log_info('The {} of Total {}'.format @@ -28,8 +36,8 @@ def QA_SU_save_binance(frequency): ref = col.find({"symbol": symbol_info['symbol']}).sort("start_time", -1) if ref.count() > 0: - start_stamp = ref.next()['start_time'] / 1000 - start_time = datetime.datetime.fromtimestamp(start_stamp) + start_stamp = ref.next()['close_time'] / 1000 + start_time = datetime.datetime.fromtimestamp(start_stamp, tz=tzutc()) QA_util_log_info('UPDATE_SYMBOL {} Trying updating {} from {} to {}'.format( frequency, symbol_info['symbol'], start_time, end)) else: @@ -38,7 +46,7 @@ def QA_SU_save_binance(frequency): frequency, symbol_info['symbol'], start_time, end)) data = QA_fetch_kline(symbol_info['symbol'], - time.mktime(start_time.timetuple()), time.mktime(end.timetuple()), frequency) + time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency) if data is None: QA_util_log_info('SYMBOL {} from {} to {} has no data'.format( symbol_info['symbol'], start_time, end)) From a281409554d42a73b974cf7bda283a88f03b7a67 Mon Sep 17 00:00:00 2001 From: Vincent yu Date: Sun, 17 Jun 2018 21:07:08 +0800 Subject: [PATCH 3/3] Update CHANGELOG.md --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0166491e6..8de715559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ 3. @几何 优化了setup.py文件 4. 财务数据的存储,获取 5. QA_fetch_financial_report -6. QACLI--> save financialfile +6. QACLI--> save financialfiles 7. QASU.QA_SU_save_financial_files() released in : JUNE 17, 2018