Merge pull request #122 from Deriq-Qian-Dong/main

add scheduled google scholar crawler
THUIR · Nov 23, 2023 · 72176bd · 72176bd
2 parents a6a26b7 + 7670862
commit 72176bd
Showing 1 changed file with 39 additions and 0 deletions.
diff --git a/.github/workflows/scheduled_google_scholar_crawler.yaml b/.github/workflows/scheduled_google_scholar_crawler.yaml
@@ -0,0 +1,39 @@
+name: Scheduled Auto Cite by Crawler
+
+on:
+  schedule:
+    - cron: '0 0 * * *' 
+
+jobs:
+  update_research:
+    name: Auto Cite by Crawler
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: Install packages
+        run: python -m pip install --upgrade --requirement ./auto-cite/requirements.txt
+      - name: Installed package list
+        run: apt list --installed
+      - name: Remove Chrome
+        run: sudo apt purge google-chrome-stable
+      - name: Remove default Chromium
+        run: sudo apt purge chromium-browser
+      - name: Install a new Chromium
+        run: sudo apt install -y chromium-browser
+      - name: Install all necessary packages
+        run: pip install requests beautifulsoup4 pandas webdriver-manager selenium
+      - name: Build updated citations
+        run: python ./auto-cite/auto-cite-by-crawler.py
+        timeout-minutes: 60
+      - name: Commit updated citations
+        uses: stefanzweifel/git-auto-commit-action@v4
+        with:
+          file_pattern: "_data/citations.yaml"
+          commit_message: "Generate citations by crawler"
+          push_options: --force