From ca359cd5be768f0ac052607dfb1ce58c93b39cf6 Mon Sep 17 00:00:00 2001 From: laizn Date: Sun, 3 Nov 2024 03:20:36 +0800 Subject: [PATCH 1/3] fix: fix list fetch on 10jqka apis --- src/zvt/recorders/jqka/jqka_api.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/zvt/recorders/jqka/jqka_api.py b/src/zvt/recorders/jqka/jqka_api.py index b75168a3..6f936ee9 100644 --- a/src/zvt/recorders/jqka/jqka_api.py +++ b/src/zvt/recorders/jqka/jqka_api.py @@ -57,27 +57,30 @@ def get_limit_up(date: str): def get_limit_down(date: str): date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) - url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?page=1&limit=15&field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}" + url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}" return get_jkqa_data(url=url) def get_jkqa_data(url, pn=1, ps=200, fetch_all=True, headers=_JKQA_HEADER): - url = url + f"&page={pn}&limit={ps}&_={now_timestamp()}" - print(url) - resp = requests.get(url, headers=headers) + requesting_url = url + f"&page={pn}&limit={ps}&_={now_timestamp()}" + print(requesting_url) + resp = requests.get(requesting_url, headers=headers) if resp.status_code == 200: json_result = resp.json() if json_result and json_result["data"]: data: list = json_result["data"]["info"] if fetch_all: - if pn < json_result["data"]["page"]["page"]: + if pn < json_result["data"]["page"]["count"]: next_data = get_jkqa_data( pn=pn + 1, ps=ps, + url=url, fetch_all=fetch_all, ) if next_data: data = data + next_data + if pn == 1 and len(data) != json_result["data"]["page"]["total"]: + raise RuntimeError(f"Assertion failed, the total length of data should be {json_result['data']['page']['total']}, only {len(data)} fetched") return data else: return data From b06ba511f7aa57e9ae4fbf3d97619522a6969c15 Mon Sep 17 00:00:00 2001 From: laizn Date: Sun, 3 Nov 2024 04:29:28 +0800 Subject: [PATCH 2/3] fix: keep order while requesting pagerized data --- src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py | 4 ++-- src/zvt/recorders/jqka/jqka_api.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py b/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py index 4af10635..8c72c45f 100644 --- a/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py +++ b/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py @@ -113,7 +113,7 @@ def record(self, entity, start, end, size, timestamps): limit_downs = jqka_api.get_limit_down(date=the_date) if limit_downs: records = [] - for data in limit_downs: + for idx, data in enumerate(limit_downs): entity_id = china_stock_code_to_id(code=data["code"]) record = { "id": "{}_{}".format(entity_id, the_date), @@ -205,7 +205,7 @@ def record(self, entity, start, end, size, timestamps): if __name__ == "__main__": - JqkaLimitUpRecorder().run() + JqkaLimitDownRecorder().run() # the __all__ is generated diff --git a/src/zvt/recorders/jqka/jqka_api.py b/src/zvt/recorders/jqka/jqka_api.py index 6f936ee9..295b561c 100644 --- a/src/zvt/recorders/jqka/jqka_api.py +++ b/src/zvt/recorders/jqka/jqka_api.py @@ -51,13 +51,13 @@ def get_limit_stats(date: str): def get_limit_up(date: str): date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) - url = f"https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}" + url = f"https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=199112&order_type=0&date={date_str}" return get_jkqa_data(url=url) def get_limit_down(date: str): date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1) - url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}" + url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=199112&order_type=0&date={date_str}" return get_jkqa_data(url=url) From 16c87564ed8ef15f86f16b68b6308778609f3fef Mon Sep 17 00:00:00 2001 From: laizn Date: Sun, 3 Nov 2024 04:51:46 +0800 Subject: [PATCH 3/3] feat: allow force update for JqkaEmotionRecorder --- src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py b/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py index 8c72c45f..5141e40c 100644 --- a/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py +++ b/src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py @@ -36,7 +36,7 @@ def init_timestamps(self, entity_item) -> List[pd.Timestamp]: latest_infos = LimitUpInfo.query_data( provider=self.provider, order=LimitUpInfo.timestamp.desc(), limit=1, return_type="domain" ) - if latest_infos: + if latest_infos and not self.force_update: start_date = latest_infos[0].timestamp else: # 最近一年的数据 @@ -99,7 +99,7 @@ def init_timestamps(self, entity_item) -> List[pd.Timestamp]: latest_infos = LimitDownInfo.query_data( provider=self.provider, order=LimitDownInfo.timestamp.desc(), limit=1, return_type="domain" ) - if latest_infos: + if latest_infos and not self.force_update: start_date = latest_infos[0].timestamp else: # 最近一年的数据 @@ -163,7 +163,7 @@ def init_timestamps(self, entity_item) -> List[pd.Timestamp]: latest_infos = Emotion.query_data( provider=self.provider, order=Emotion.timestamp.desc(), limit=1, return_type="domain" ) - if latest_infos: + if latest_infos and not self.force_update: start_date = latest_infos[0].timestamp else: # 最近一年的数据 @@ -205,8 +205,8 @@ def record(self, entity, start, end, size, timestamps): if __name__ == "__main__": - JqkaLimitDownRecorder().run() - + # JqkaLimitDownRecorder().run() + LimitDownInfo.record_data(start_timestamp="2024-02-02", end_timestamp="2024-02-16", force_update=True) # the __all__ is generated __all__ = ["JqkaLimitUpRecorder", "JqkaLimitDownRecorder", "JqkaEmotionRecorder"]