Merge pull request #28 from QUANTAXIS/master

update

CHANGELOG.md

@@ -61,7 +61,7 @@ released in : JUNE 17, 2018
 3. @几何 优化了setup.py文件
 4. 财务数据的存储,获取
 5. QA_fetch_financial_report
-6. QACLI--> save financialfile
+6. QACLI--> save financialfiles
 7. QASU.QA_SU_save_financial_files()
 
 released in : JUNE 17, 2018

QUANTAXIS/QACmd/__init__.py

@@ -251,8 +251,8 @@ def do_save(self, arg):
                 # QA_SU_save_stock_info('tdx')
             elif len(arg) == 1 and arg[0] == "binance":
                 QA_SU_save_symbols()
-                QA_SU_save_binance_1hour()
                 QA_SU_save_binance_1day()
+                QA_SU_save_binance_1hour()
                 QA_SU_save_binance_1min()
             elif len(arg) == 1 and arg[0] == "financialfiles":
                 QA_SU_save_financialfiles()

QUANTAXIS/QAFetch/QAbinance.py

@@ -47,9 +47,11 @@ def QA_fetch_kline(symbol, start_time, end_time, frequency):
             raise ConnectTimeout(MESSAGE_NOTICE)
         klines = json.loads(req.content)
         if len(klines) == 0:
-            return datas
+            break
         datas.extend(klines)
         start_time = klines[-1][6]
+    if len(datas) == 0:
+        return None
     frame = pd.DataFrame(datas)
     frame.columns = columne_names
     frame['symbol'] = symbol
@@ -62,11 +64,15 @@ def QA_fetch_kline(symbol, start_time, end_time, frequency):
     # a = requests.get(url)
     # print(a.content)
     # print(json.loads(a.content))
-
+    import pytz
+    from dateutil.tz import *
+    tz = pytz.timezone("Asia/Shanghai")
     url = urljoin(Binance_base_url, "/api/v1/klines")
-    start = time.mktime(datetime.datetime(2017, 6, 1).timetuple())
-    end = time.mktime(datetime.datetime(2017, 7, 1).timetuple())
+    start = time.mktime(datetime.datetime(2018, 6, 13,tzinfo=tzutc()).timetuple())
+    end = time.mktime(datetime.datetime(2018, 6, 14, tzinfo=tzutc()).timetuple())
     print(start * 1000)
     print(end * 1000)
-    datas = QA_fetch_kline("ETHBTC", start, end)
-    print(len(datas))
+    data = QA_fetch_kline("ETHBTC", start, end, '1d')
+    print(len(data))
+    print(data[0])
+    print(data[-1])

QUANTAXIS/QASU/save_binance.py

@@ -1,5 +1,7 @@
 import datetime
 import time
+from dateutil.tz import tzutc
+from dateutil.relativedelta import relativedelta
 from QUANTAXIS.QAUtil import (DATABASE, QASETTING,
                               QA_util_get_real_date, QA_util_log_info,
                               QA_util_to_json_from_pandas, trade_date_sse)
@@ -8,16 +10,22 @@
 import pymongo
 
 # binance的历史数据只是从2017年7月开始有，以前的貌似都没有保留 . author:Will
-BINANCE_MIN_DATE = datetime.datetime(2017, 7, 1)
+BINANCE_MIN_DATE = datetime.datetime(2017, 7, 1, tzinfo=tzutc())
 
+FREQUANCY_DICT ={
+    "1m":relativedelta(minutes=-1),
+    "1d":relativedelta(days=-1),
+    "1h":relativedelta(hours=-1)
+}
 
 def QA_SU_save_binance(frequency):
     symbol_list = QA_fetch_symbol()
     col = QASETTING.client.binance[frequency]
     col.create_index(
-        [("symbol", pymongo.ASCENDING), ("start_time", pymongo.ASCENDING)])
+        [("symbol", pymongo.ASCENDING), ("start_time", pymongo.ASCENDING)],unique=True)
 
-    end = datetime.datetime.now().replace(hour=0, minute=0, second=0, microsecond=0)
+    end = datetime.datetime.now(tzutc())
+    end += FREQUANCY_DICT.get(frequency)
 
     for index, symbol_info in enumerate(symbol_list):
         QA_util_log_info('The {} of Total {}'.format
@@ -28,18 +36,18 @@ def QA_SU_save_binance(frequency):
         ref = col.find({"symbol": symbol_info['symbol']}).sort("start_time", -1)
 
         if ref.count() > 0:
-            start_stamp = ref.next()['start_time'] / 1000
-            start_time = datetime.datetime.fromtimestamp(start_stamp)
-            QA_util_log_info('UPDATE_SYMBOL \n Trying updating {} from {} to {}'.format(
-                symbol_info['symbol'], start_time, end))
+            start_stamp = ref.next()['close_time'] / 1000
+            start_time = datetime.datetime.fromtimestamp(start_stamp, tz=tzutc())
+            QA_util_log_info('UPDATE_SYMBOL {} Trying updating {} from {} to {}'.format(
+                frequency, symbol_info['symbol'], start_time, end))
         else:
             start_time = BINANCE_MIN_DATE
-            QA_util_log_info('NEW_SYMBOL \n Trying downloading {} from {} to {}'.format(
-                symbol_info['symbol'], start_time, end))
+            QA_util_log_info('NEW_SYMBOL {} Trying downloading {} from {} to {}'.format(
+                frequency, symbol_info['symbol'], start_time, end))
 
         data = QA_fetch_kline(symbol_info['symbol'],
-                              time.mktime(start_time.timetuple()), time.mktime(end.timetuple()), frequency)
-        if len(data) == 0:
+                              time.mktime(start_time.utctimetuple()), time.mktime(end.utctimetuple()), frequency)
+        if data is None:
             QA_util_log_info('SYMBOL {} from {} to {} has no data'.format(
                 symbol_info['symbol'], start_time, end))
             continue