Merge pull request #225 from lonelam/fix/jqka_api

fix: fix list fetch on 10jqka apis

src/zvt/recorders/jqka/emotion/JqkaEmotionRecorder.py

@@ -36,7 +36,7 @@ def init_timestamps(self, entity_item) -> List[pd.Timestamp]:
         latest_infos = LimitUpInfo.query_data(
             provider=self.provider, order=LimitUpInfo.timestamp.desc(), limit=1, return_type="domain"
         )
-        if latest_infos:
+        if latest_infos and not self.force_update:
             start_date = latest_infos[0].timestamp
         else:
             # 最近一年的数据
@@ -99,7 +99,7 @@ def init_timestamps(self, entity_item) -> List[pd.Timestamp]:
         latest_infos = LimitDownInfo.query_data(
             provider=self.provider, order=LimitDownInfo.timestamp.desc(), limit=1, return_type="domain"
         )
-        if latest_infos:
+        if latest_infos  and not self.force_update:
             start_date = latest_infos[0].timestamp
         else:
             # 最近一年的数据
@@ -113,7 +113,7 @@ def record(self, entity, start, end, size, timestamps):
             limit_downs = jqka_api.get_limit_down(date=the_date)
             if limit_downs:
                 records = []
-                for data in limit_downs:
+                for idx, data in enumerate(limit_downs):
                     entity_id = china_stock_code_to_id(code=data["code"])
                     record = {
                         "id": "{}_{}".format(entity_id, the_date),
@@ -163,7 +163,7 @@ def init_timestamps(self, entity_item) -> List[pd.Timestamp]:
         latest_infos = Emotion.query_data(
             provider=self.provider, order=Emotion.timestamp.desc(), limit=1, return_type="domain"
         )
-        if latest_infos:
+        if latest_infos and not self.force_update:
             start_date = latest_infos[0].timestamp
         else:
             # 最近一年的数据
@@ -205,8 +205,8 @@ def record(self, entity, start, end, size, timestamps):
 
 
 if __name__ == "__main__":
-    JqkaLimitUpRecorder().run()
-
+    # JqkaLimitDownRecorder().run()
+    LimitDownInfo.record_data(start_timestamp="2024-02-02", end_timestamp="2024-02-16", force_update=True)
 
 # the __all__ is generated
 __all__ = ["JqkaLimitUpRecorder", "JqkaLimitDownRecorder", "JqkaEmotionRecorder"]

src/zvt/recorders/jqka/jqka_api.py

@@ -51,33 +51,36 @@ def get_limit_stats(date: str):
 
 def get_limit_up(date: str):
     date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1)
-    url = f"https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}"
+    url = f"https://data.10jqka.com.cn/dataapi/limit_up/limit_up_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=199112&order_type=0&date={date_str}"
     return get_jkqa_data(url=url)
 
 
 def get_limit_down(date: str):
     date_str = to_time_str(the_time=date, fmt=TIME_FORMAT_DAY1)
-    url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?page=1&limit=15&field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=330324&order_type=0&date={date_str}"
+    url = f"https://data.10jqka.com.cn/dataapi/limit_up/lower_limit_pool?field=199112,10,9001,330323,330324,330325,9002,330329,133971,133970,1968584,3475914,9003,9004&filter=HS,GEM2STAR&order_field=199112&order_type=0&date={date_str}"
     return get_jkqa_data(url=url)
 
 
 def get_jkqa_data(url, pn=1, ps=200, fetch_all=True, headers=_JKQA_HEADER):
-    url = url + f"&page={pn}&limit={ps}&_={now_timestamp()}"
-    print(url)
-    resp = requests.get(url, headers=headers)
+    requesting_url = url + f"&page={pn}&limit={ps}&_={now_timestamp()}"
+    print(requesting_url)
+    resp = requests.get(requesting_url, headers=headers)
     if resp.status_code == 200:
         json_result = resp.json()
         if json_result and json_result["data"]:
             data: list = json_result["data"]["info"]
             if fetch_all:
-                if pn < json_result["data"]["page"]["page"]:
+                if pn < json_result["data"]["page"]["count"]:
                     next_data = get_jkqa_data(
                         pn=pn + 1,
                         ps=ps,
+                        url=url,
                         fetch_all=fetch_all,
                     )
                     if next_data:
                         data = data + next_data
+                        if pn == 1 and len(data) != json_result["data"]["page"]["total"]:
+                            raise RuntimeError(f"Assertion failed, the total length of data should be {json_result['data']['page']['total']}, only {len(data)} fetched")
                         return data
                     else:
                         return data