From 2badc51feded5cba1752c56c8687dfb7ae82c47c Mon Sep 17 00:00:00 2001 From: foolcage <5533061@qq.com> Date: Sun, 17 Nov 2024 17:43:58 +0800 Subject: [PATCH] 1)Close WAL mode for most of db 2)add em api get_top_tradable_list 3)add today shoot/top runner 4)Upgrade some requirements to be compatible with Python 3.9 latter --- .pre-commit-config.yaml | 2 +- README-cn.md | 7 ++ README.md | 6 ++ requirements.txt | 6 +- src/zvt/broker/qmt/qmt_quote.py | 2 +- src/zvt/contract/register.py | 5 +- src/zvt/recorders/em/em_api.py | 95 +++++++++++++++++++------- src/zvt/tag/tag_service.py | 1 + src/zvt/tasks/qmt_tick_runner.py | 2 + src/zvt/tasks/stock_pool_runner.py | 3 +- src/zvt/tasks/today_shoot_runner.py | 62 +++++++++++++++++ src/zvt/tasks/today_top_runner.py | 102 ++++++++++++++++++++++++++++ 12 files changed, 260 insertions(+), 33 deletions(-) create mode 100644 src/zvt/tasks/today_shoot_runner.py create mode 100644 src/zvt/tasks/today_top_runner.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 294263cb..078b5580 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -2,7 +2,7 @@ ci: autoupdate_schedule: monthly repos: - repo: https://github.com/psf/black - rev: 21.11b1 + rev: 24.8.0 hooks: - id: black # - repo: https://github.com/PyCQA/flake8 diff --git a/README-cn.md b/README-cn.md index f01d46c7..88a3e67e 100644 --- a/README-cn.md +++ b/README-cn.md @@ -12,6 +12,13 @@ [炒股的三大原理](https://mp.weixin.qq.com/s/FoFR63wFSQIE_AyFubkZ6Q) +**声明** + +本项目目前不保证任何向后兼容性，请谨慎升级。 +随着作者思想的变化，一些以前觉得重要的东西可能也变得不重要，从而可能不会进行维护。 +而一些新的东西的加入对你是否有用，需要自己去评估。 + + **Read this in other languages: [English](README-cn.md).** **详细文档:[https://zvt.readthedocs.io/en/latest/](https://zvt.readthedocs.io/en/latest/)** diff --git a/README.md b/README.md index 2d602092..f965d98e 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,12 @@ [The Three Major Principles of Stock Trading](https://mp.weixin.qq.com/s/FoFR63wFSQIE_AyFubkZ6Q) +**Declaration** + +This project does not currently guarantee any backward compatibility, so please upgrade with caution. +As the author's thoughts evolve, some things that were once considered important may become less so, and thus may not be maintained. +Whether the addition of some new elements will be useful to you needs to be assessed by yourself. + **Read this in other languages: [中文](README-cn.md).** **Read the docs:[https://zvt.readthedocs.io/en/latest/](https://zvt.readthedocs.io/en/latest/)** diff --git a/requirements.txt b/requirements.txt index 5e2e2c38..b706f73d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,16 +1,16 @@ requests==2.31.0 -SQLAlchemy==2.0.28 +SQLAlchemy==2.0.36 pandas==2.0.3 pydantic==2.6.4 arrow==1.2.3 openpyxl==3.1.1 demjson3==3.0.6 plotly==5.13.0 -dash==2.8.1 +dash==2.18.2 jqdatapy==0.1.8 dash-bootstrap-components==1.3.1 dash_daq==0.5.0 -scikit-learn==1.2.1 +scikit-learn==1.5.2 fastapi==0.110.0 fastapi-pagination==0.12.23 apscheduler==3.10.4 diff --git a/src/zvt/broker/qmt/qmt_quote.py b/src/zvt/broker/qmt/qmt_quote.py index 2c26fe14..562aca0d 100644 --- a/src/zvt/broker/qmt/qmt_quote.py +++ b/src/zvt/broker/qmt/qmt_quote.py @@ -282,7 +282,7 @@ def download_capital_data(): def clear_history_quote(): session = get_db_session("qmt", data_schema=StockQuote) session.query(StockQuote).filter(StockQuote.timestamp < current_date()).delete() - start_date = date_time_by_interval(current_date(), -20) + start_date = date_time_by_interval(current_date(), -10) session.query(Stock1mQuote).filter(Stock1mQuote.timestamp < start_date).delete() session.query(StockQuoteLog).filter(StockQuoteLog.timestamp < start_date).delete() session.commit() diff --git a/src/zvt/contract/register.py b/src/zvt/contract/register.py index 029e511e..5b52aae0 100644 --- a/src/zvt/contract/register.py +++ b/src/zvt/contract/register.py @@ -108,7 +108,10 @@ def register_schema( added_columns = [c for c in table.columns if c.name not in existing_columns] index_list = [] with engine.connect() as con: - con.execute(text("PRAGMA journal_mode=WAL;")) + # FIXME: close WAL mode for saving space, most of time no need to write in multiple process + if db_name in ("zvt_info","stock_news","stock_tags"): + con.execute(text("PRAGMA journal_mode=WAL;")) + rs = con.execute(text("PRAGMA INDEX_LIST('{}')".format(table_name))) for row in rs: index_list.append(row[1]) diff --git a/src/zvt/recorders/em/em_api.py b/src/zvt/recorders/em/em_api.py index b8d24790..a0b2e93b 100644 --- a/src/zvt/recorders/em/em_api.py +++ b/src/zvt/recorders/em/em_api.py @@ -8,7 +8,6 @@ import requests import sqlalchemy from requests import Session -from sqlalchemy import func from zvt.api.kdata import generate_kdata_id from zvt.api.utils import value_to_pct, china_stock_code_to_id @@ -589,6 +588,61 @@ def get_future_list(): return df +def get_top_tradable_list(entity_type, fields, limit, entity_flag, exchange=None, return_quote=False): + url = f"https://push2.eastmoney.com/api/qt/clist/get?np=1&fltt=2&invt=2&fields={fields}&pn=1&pz={limit}&fid=f3&po=1&{entity_flag}&ut=f057cbcbce2a86e2866ab8877db1d059&forcect=1&cb=cbCallbackMore&&callback=jQuery34109676853980006124_{now_timestamp() - 1}&_={now_timestamp()}" + resp = requests.get(url, headers=DEFAULT_HEADER) + + resp.raise_for_status() + + result = json_callback_param(resp.text) + resp.close() + data = result["data"]["diff"] + df = pd.DataFrame.from_records(data=data) + + if return_quote: + df = df[ + [ + "f12", + "f14", + "f2", + "f3", + ] + ] + df.columns = ["code", "name", "price", "change_pct"] + else: + if entity_type == TradableType.stock: + df = df[["f12", "f13", "f14", "f20", "f21", "f9", "f23"]] + df.columns = ["code", "exchange", "name", "cap", "cap1", "pe", "pb"] + df[["cap", "cap1", "pe", "pb"]] = df[["cap", "cap1", "pe", "pb"]].apply(pd.to_numeric, errors="coerce") + else: + df = df[["f12", "f13", "f14"]] + df.columns = ["code", "exchange", "name"] + if exchange: + df["exchange"] = exchange.value + df["entity_type"] = entity_type.value + df["id"] = df[["entity_type", "exchange", "code"]].apply(lambda x: "_".join(x.astype(str)), axis=1) + df["entity_id"] = df["id"] + + return df + + +def get_top_stocks(limit=100): + # 沪深和北交所 + entity_flag = "fs=m:0+t:6+f:!2,m:0+t:13+f:!2,m:0+t:80+f:!2,m:1+t:2+f:!2,m:1+t:23+f:!2,m:0+t:81+s:2048" + fields = "f2,f3,f12,f14" + return get_top_tradable_list( + entity_type=TradableType.stock, fields=fields, limit=limit, entity_flag=entity_flag, return_quote=True + ) + + +def get_top_stockhks(limit=20): + entity_flag = "fs=b:DLMK0144,b:DLMK0146" + fields = "f2,f3,f12,f14" + return get_top_tradable_list( + entity_type=TradableType.stockhk, fields=fields, limit=limit, entity_flag=entity_flag, return_quote=True + ) + + def get_tradable_list( entity_type: Union[TradableType, str] = "stock", exchange: Union[Exchange, str] = None, @@ -665,31 +719,15 @@ def get_tradable_list( else: assert False + # f2, f3, f4, f12, f13, f14, f19, f111, f148 fields = "f1,f2,f3,f4,f12,f13,f14" if entity_type == TradableType.stock: # 市值,流通市值,pe,pb fields = fields + ",f20,f21,f9,f23" - url = f"https://push2.eastmoney.com/api/qt/clist/get?np=1&fltt=2&invt=2&fields={fields}&pn=1&pz={limit}&fid=f3&po=1&{entity_flag}&ut=f057cbcbce2a86e2866ab8877db1d059&forcect=1&cb=cbCallbackMore&&callback=jQuery34109676853980006124_{now_timestamp() - 1}&_={now_timestamp()}" - resp = requests.get(url, headers=DEFAULT_HEADER) - - resp.raise_for_status() - - result = json_callback_param(resp.text) - resp.close() - data = result["data"]["diff"] - df = pd.DataFrame.from_records(data=data) - if entity_type == TradableType.stock: - df = df[["f12", "f13", "f14", "f20", "f21", "f9", "f23"]] - df.columns = ["code", "exchange", "name", "cap", "cap1", "pe", "pb"] - df[["cap", "cap1", "pe", "pb"]] = df[["cap", "cap1", "pe", "pb"]].apply(pd.to_numeric, errors="coerce") - else: - df = df[["f12", "f13", "f14"]] - df.columns = ["code", "exchange", "name"] - df["exchange"] = exchange.value - df["entity_type"] = entity_type.value - df["id"] = df[["entity_type", "exchange", "code"]].apply(lambda x: "_".join(x.astype(str)), axis=1) - df["entity_id"] = df["id"] + df = get_top_tradable_list( + entity_type=entity_type, fields=fields, limit=limit, entity_flag=entity_flag, exchange=exchange + ) if entity_type == TradableType.block: df["category"] = block_category.value @@ -996,11 +1034,14 @@ def to_zvt_code(code): # print(df) # print(len(df)) # df = get_tradable_list(entity_type="block") + # print(df) # df = get_tradable_list(entity_type="indexus") + # print(df) # df = get_tradable_list(entity_type="currency") + # print(df) # df = get_tradable_list(entity_type="index") - # df = get_kdata(entity_id="index_us_SPX", level="1d") # print(df) + # df = get_kdata(entity_id="index_us_SPX", level="1d") # df = get_treasury_yield(pn=1, ps=50, fetch_all=False) # print(df) # df = get_future_list() @@ -1022,10 +1063,12 @@ def to_zvt_code(code): # print(events) # print(get_hot_topic()) # record_hot_topic() - df = StockHotTopic.query_data( - filters=[func.json_extract(StockHotTopic.entity_ids, "$").contains("stock_sh_600809")], - ) - print(df) + # df = StockHotTopic.query_data( + # filters=[func.json_extract(StockHotTopic.entity_ids, "$").contains("stock_sh_600809")], + # ) + # print(df) + print(get_top_stocks()) + print(get_top_stockhks()) # the __all__ is generated diff --git a/src/zvt/tag/tag_service.py b/src/zvt/tag/tag_service.py index 3910fb88..1f5b1bc0 100644 --- a/src/zvt/tag/tag_service.py +++ b/src/zvt/tag/tag_service.py @@ -651,6 +651,7 @@ def activate_sub_tags(activate_sub_tags_model: ActivateSubTagsModel): # entity_ids = df["entity_id"].tolist() entity_ids = None + # stock_tag with sub_tag but not set to related main_tag yet stock_tags = StockTags.query_data( session=session, entity_ids=entity_ids, diff --git a/src/zvt/tasks/qmt_tick_runner.py b/src/zvt/tasks/qmt_tick_runner.py index 341f4cbc..bd9e867b 100644 --- a/src/zvt/tasks/qmt_tick_runner.py +++ b/src/zvt/tasks/qmt_tick_runner.py @@ -1,7 +1,9 @@ # -*- coding: utf-8 -*- +from zvt import init_log from zvt.broker.qmt.qmt_quote import record_tick if __name__ == "__main__": + init_log("qmt_tick_runner.log") from apscheduler.schedulers.background import BackgroundScheduler sched = BackgroundScheduler() diff --git a/src/zvt/tasks/stock_pool_runner.py b/src/zvt/tasks/stock_pool_runner.py index 9e090930..e1649cdd 100644 --- a/src/zvt/tasks/stock_pool_runner.py +++ b/src/zvt/tasks/stock_pool_runner.py @@ -3,7 +3,7 @@ from apscheduler.schedulers.background import BackgroundScheduler -from zvt import zvt_config +from zvt import zvt_config, init_log from zvt.api.selector import get_entity_ids_by_filter from zvt.domain import ( Stock, @@ -125,6 +125,7 @@ def record_data_and_build_stock_pools(): if __name__ == "__main__": + init_log("sotck_pool_runner.log") record_data_and_build_stock_pools() sched.add_job(func=record_data_and_build_stock_pools, trigger="cron", hour=16, minute=00, day_of_week="mon-fri") sched.start() diff --git a/src/zvt/tasks/today_shoot_runner.py b/src/zvt/tasks/today_shoot_runner.py new file mode 100644 index 00000000..d7918eba --- /dev/null +++ b/src/zvt/tasks/today_shoot_runner.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +import logging +import time + +import eastmoneypy +from apscheduler.schedulers.background import BackgroundScheduler + +from zvt import init_log +from zvt.api.selector import get_shoot_today +from zvt.domain import Stock +from zvt.informer.inform_utils import add_to_eastmoney +from zvt.tag.common import InsertMode +from zvt.tag.tag_stats import build_stock_pool_and_tag_stats +from zvt.utils.time_utils import now_pd_timestamp, current_date + +logger = logging.getLogger(__name__) + + +sched = BackgroundScheduler() + + +def calculate_top(): + try: + eastmoneypy.del_group("今日异动") + except: + pass + while True: + current_timestamp = now_pd_timestamp() + + if not Stock.in_trading_time(): + logger.info(f"calculate shoots finished at: {current_timestamp}") + break + + if Stock.in_trading_time() and not Stock.in_real_trading_time(): + logger.info(f"Sleeping time......") + time.sleep(60 * 1) + continue + + target_date = current_date() + shoot_up, shoot_down = get_shoot_today() + + shoots = shoot_up + shoot_down + if shoots: + build_stock_pool_and_tag_stats( + entity_ids=shoots, + stock_pool_name="今日异动", + insert_mode=InsertMode.append, + target_date=target_date, + provider="qmt", + ) + add_to_eastmoney(codes=[entity_id.split("_")[2] for entity_id in shoots], group="今日异动", over_write=False) + + logger.info(f"Sleep 1 minutes to compute {target_date} shoots tag stats") + time.sleep(60 * 1) + + +if __name__ == "__main__": + init_log("today_shoot_runner.log") + calculate_top() + sched.add_job(func=calculate_top, trigger="cron", hour=9, minute=30, day_of_week="mon-fri") + sched.start() + sched._thread.join() diff --git a/src/zvt/tasks/today_top_runner.py b/src/zvt/tasks/today_top_runner.py new file mode 100644 index 00000000..90d4249d --- /dev/null +++ b/src/zvt/tasks/today_top_runner.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +import logging +import time +from typing import List + +import eastmoneypy +from apscheduler.schedulers.background import BackgroundScheduler + +from zvt import init_log +from zvt.api.selector import get_top_up_today, get_top_down_today, get_top_vol +from zvt.domain import Stock +from zvt.informer.inform_utils import add_to_eastmoney +from zvt.recorders.em.em_api import record_hot_topic +from zvt.tag.common import InsertMode +from zvt.tag.tag_schemas import StockPools +from zvt.tag.tag_stats import build_stock_pool_and_tag_stats +from zvt.utils.time_utils import now_pd_timestamp, current_date + +logger = logging.getLogger(__name__) + + +sched = BackgroundScheduler() + + +def calculate_top(clear_em=True): + if clear_em: + try: + eastmoneypy.del_group("今日强势") + except: + pass + + seed = 0 + add_all_to_em = False + while True: + current_timestamp = now_pd_timestamp() + + if not Stock.in_trading_time(): + logger.info(f"calculate top finished at: {current_timestamp}") + break + + if Stock.in_trading_time() and not Stock.in_real_trading_time(): + logger.info(f"Sleeping time......") + time.sleep(60 * 1) + continue + + if seed == 0: + record_hot_topic() + seed = seed + 1 + if seed == 5: + seed = 0 + target_date = current_date() + top_up_entity_ids = get_top_up_today() + if top_up_entity_ids: + build_stock_pool_and_tag_stats( + entity_ids=top_up_entity_ids, + stock_pool_name="今日强势", + insert_mode=InsertMode.append, + target_date=target_date, + provider="qmt", + ) + try: + to_added = top_up_entity_ids + if add_all_to_em: + stock_pools: List[StockPools] = StockPools.query_data( + filters=[StockPools.stock_pool_name == "今日强势"], + order=StockPools.timestamp.desc(), + limit=1, + return_type="domain", + ) + if stock_pools: + to_added = stock_pools[0].entity_ids + if len(to_added) > 500: + to_added = get_top_vol(entity_ids=to_added, limit=500) + + add_to_eastmoney( + codes=[entity_id.split("_")[2] for entity_id in to_added], group="今日强势", over_write=False + ) + add_all_to_em = False + except Exception as e: + logger.error(e) + add_all_to_em = True + + top_down_entity_ids = get_top_down_today() + if top_down_entity_ids: + build_stock_pool_and_tag_stats( + entity_ids=top_down_entity_ids, + stock_pool_name="今日弱势", + insert_mode=InsertMode.append, + target_date=target_date, + provider="qmt", + ) + + logger.info(f"Sleep 2 minutes to compute {target_date} top stock tag stats") + time.sleep(60 * 2) + + +if __name__ == "__main__": + init_log("today_top_runner.log") + calculate_top(clear_em=False) + sched.add_job(func=calculate_top, trigger="cron", hour=9, minute=26, day_of_week="mon-fri") + sched.start() + sched._thread.join()