Use asyncio to greatly improve performance

.github/workflows/updater.yaml

@@ -9,13 +9,13 @@ jobs:
   update:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           token: ${{ secrets.UPDATER }}
           # otherwise, you will failed to push refs to dest repo
           fetch-depth: 0
 
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: "3.x"
 
@@ -32,4 +32,4 @@ jobs:
         with:
           commit_message: "auto updated plugins"
           push_options: --force
-          branch: plugin_api_v2
+          branch: asyncio-revision

ci/envs/requirements-updater.txt

@@ -1,2 +1,2 @@
-requests
+aiohttp
 tqdm

ci/src/_utils.py

@@ -63,7 +63,7 @@ def clean(string: str, flag="-") -> str:
 
 def version_tuple(version: str) -> tuple:
     version = clean(version, "v")
-    return tuple(map(int, (version.split("."))))
+    return tuple(version.split("."))
 
 def check_url(url: str) -> bool:
     regex = re.compile(

ci/src/discord.py

@@ -1,11 +1,11 @@
-import requests
+import aiohttp
 
 from _utils import *
 
 MAX_BODY_LEN = 1024
 
 
-def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) -> None:
+async def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) -> None:
     embed = {
         "content": None,
         "embeds": [
@@ -41,7 +41,9 @@ def update_hook(webhook_url: str, info: dict, latest_ver: str, release: dict) ->
     release_notes = release.get('body')
     if release_notes and release_notes.strip():
         embed['embeds'][0]['fields'].append({"name": "Release Notes", "value": truncate_release_notes(release['html_url'], release.get('body', ""))})
-    requests.post(webhook_url, json=embed)
+
+    async with aiohttp.ClientSession() as session:
+        session.post(webhook_url, json=embed)
 
 def truncate_release_notes(url: str, release_notes: str, length: int = MAX_BODY_LEN) -> str:
     if len(release_notes) <= length:

ci/src/updater.py

@@ -1,18 +1,20 @@
 # -*-coding: utf-8 -*-
-from http.client import responses
+import asyncio
 from typing import List
 from unicodedata import name
 from sys import argv
 import traceback
+import aiohttp
 
-import requests
-from tqdm import tqdm
+from tqdm.asyncio import tqdm
 
 from _utils import *
 from discord import update_hook
 
 
-def batch_github_plugin_info(info: P, tags: ETagsType, webhook_url: str = None) -> P:
+async def batch_github_plugin_info(
+    info: P, tags: ETagsType, webhook_url: str = None
+) -> P:
     try:
         headers = None
         if "github.com" not in info[url_download]:
@@ -26,70 +28,94 @@ def batch_github_plugin_info(info: P, tags: ETagsType, webhook_url: str = None)
         tag: str = tags.get(info[id_name], info.get(etag, ""))
 
         if release_date in info.keys():
-            headers = {"If-None-Match": tag}
-        res = requests.get(
-            url_release.format(repo=repo),
-            headers=headers,
-        )
-        if res.status_code in (403, 304):
-            return info
-
-        latest_rel = res.json()
-        assets = latest_rel.get("assets")
-        if info.get(release_date, '') != latest_rel.get('published_at'):
-            info[release_date] = latest_rel.get('published_at')
-        if assets:
-            info[url_download] = assets[0]["browser_download_url"]
-            send_notification(info, clean(
-                latest_rel["tag_name"], "v"), latest_rel, webhook_url)
-            info[version] = clean(latest_rel["tag_name"], "v")
-
-        tags[info[id_name]] = res.headers.get(etag, "")
-
-        return info
+            headers = {
+                "If-None-Match": tag
+            }
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(
+                url_release.format(repo=repo),
+                headers=headers,
+            ) as res:
+                if res.status in (403, 304):
+                    return info
+
+                latest_rel = await res.json()
+
+                assets = latest_rel.get("assets")
+
+                tqdm.write(str(latest_rel))
+
+                if info.get(release_date, "") != latest_rel["published_at"]:
+                    info[release_date] = latest_rel.get("published_at")
+                if assets:
+                    info[url_download] = assets[0]["browser_download_url"]
+                    await send_notification(
+                        info,
+                        clean(latest_rel["tag_name"], "v"),
+                        latest_rel,
+                        webhook_url,
+                    )
+                    info[version] = clean(latest_rel["tag_name"], "v")
+
+                tags[info[id_name]] = res.headers.get(etag, "")
+
+                return info
     except Exception as e:
         tb = traceback.format_exc()
         print(f"Error when processing plugin {info[plugin_name]}:\n{e} {tb}")
         return info
 
 
-def batch_plugin_infos(plugin_infos: Ps, tags: ETagsType, webhook_url: str = None) -> Ps:
-    return [batch_github_plugin_info(info, tags, webhook_url) for info in tqdm(plugin_infos)]
+async def batch_plugin_infos(
+    plugin_infos: Ps, tags: ETagsType, webhook_url: str = None
+) -> Ps:
+    return await tqdm.gather(
+        *[
+            batch_github_plugin_info(info, tags, webhook_url)
+            for info in tqdm(plugin_infos)
+        ]
+    )
 
 
 def remove_unused_etags(plugin_infos: Ps, etags: ETagsType) -> ETagsType:
     etags_updated = {}
     plugin_ids = [info.get("ID") for info in plugin_infos]
-    
+
     for id, tag in etags.items():
-
         if id not in plugin_ids:
-            print(f"Plugin with ID {id} has been removed. The associated ETag will be also removed now.")
+            print(
+                f"Plugin with ID {id} has been removed. The associated ETag will be also removed now."
+            )
             continue
-        
+
         etags_updated[id] = tag
-    
+
     return etags_updated
 
 
-def send_notification(info: P, latest_ver, release, webhook_url: str = None) -> None:
+async def send_notification(info: P, latest_ver, release, webhook_url: str = None) -> None:
     if version_tuple(info[version]) != version_tuple(latest_ver):
         tqdm.write(f"Update detected: {info[plugin_name]} {latest_ver}")
         try:
-            update_hook(webhook_url, info, latest_ver, release)
+            await update_hook(webhook_url, info, latest_ver, release)
         except Exception as e:
-            tqdm.write(e)
+            tqdm.write(str(e))
 
 
-if __name__ == "__main__":
+async def main():
     webhook_url = None
     if len(argv) > 1:
         webhook_url = argv[1]
     plugin_infos = plugin_reader()
     etags = etag_reader()
-    
-    plugin_infos_new = batch_plugin_infos(plugin_infos, etags, webhook_url)
+
+    plugin_infos_new = await batch_plugin_infos(plugin_infos, etags, webhook_url)
     plugin_writer(plugin_infos_new)
-    
+
     etags_new = remove_unused_etags(plugin_infos_new, etags)
     etags_writer(etags_new)
+
+
+if __name__ == "__main__":
+    asyncio.run(main())