SearchRepos #9068
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: SearchRepos | |
| # Controls when the workflow will run | |
| on: | |
| schedule: | |
| # * is a special character in YAML so you have to quote this string | |
| - cron: '26 4/12 * * *' | |
| # Allows you to run this workflow manually from the Actions tab | |
| workflow_dispatch: | |
| jobs: | |
| search: | |
| name: Search | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 10 | |
| fail-fast: false | |
| matrix: | |
| search_str: | |
| # comments indicate a snapshot of how many results there were | |
| # for a given search around the time it was added to the list | |
| # 106 | |
| - vuln poc | |
| # 110 | |
| - heap overflow | |
| # 119 | |
| - cnvd-c OR cnvd-2 OR cnnvd-2 | |
| # 126 | |
| - zeroday | |
| # 191 | |
| - rce poc | |
| # 219 | |
| - attack poc | |
| # 2652 | |
| - metasploit module OR payload | |
| # 279 | |
| - command injection | |
| # 383 | |
| - 0day | |
| # 441 | |
| - vulnerability poc | |
| # 526 | |
| - remote code execution | |
| # 960 | |
| - sploit | |
| # 1114 | |
| - cve poc | |
| # 2021 | |
| - shellcode | |
| # 2267 | |
| - rce | |
| # 4626 | |
| - cve-2 OR cve_2 | |
| # 18176 | |
| - exploit | |
| # 17 | |
| - UEFI bootkit | |
| # 313 | |
| - uac bypass | |
| # 12 | |
| - secure boot bypass | |
| # 35 | |
| - exploitation persistence | |
| # # 2944 | |
| # - '"privilege escalation" OR privesc' | |
| # # 1504 | |
| # - buffer overflow | |
| # # 4383 (noisy) | |
| # - zero-day | |
| # # 21863 (very noisy) | |
| # - 0-day | |
| # # 502 (noisy) | |
| # - 1day | |
| steps: | |
| - name: Get today and yesterday dates | |
| id: date | |
| run: | | |
| echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: setup env | |
| uses: ./.github/actions/setup_my_env | |
| - name: search github | |
| uses: ./.github/actions/single_search | |
| with: | |
| start_date: ${{ steps.date.outputs.yesterday }} | |
| end_date: ${{ steps.date.outputs.today }} | |
| search_str: ${{ matrix.search_str }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: add and commit | |
| uses: ./.github/actions/commit_results | |
| with: | |
| commit_msg: Search for ${{ steps.date.outputs.yesterday }}..${{ steps.date.outputs.today }} ${{ matrix.search_str }} | |
| - name: push results | |
| uses: ./.github/actions/pull_rebase_push_retry | |
| update_summaries: | |
| name: Update Summaries | |
| needs: search | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get dates | |
| id: date | |
| run: | | |
| echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "this_month=$(date +'%Y-%m')" >> $GITHUB_OUTPUT | |
| echo "this_year=$(date +'%Y')" >> $GITHUB_OUTPUT | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| token: ${{ secrets.VUL_DRILLER_TOKEN }} | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: setup env | |
| uses: ./.github/actions/setup_my_env | |
| - name: git pull | |
| run: git pull | |
| - name: summarize yesterday | |
| uses: ./.github/actions/generate_summaries | |
| with: | |
| ymd_option: '--day' | |
| ymd_value: ${{ steps.date.outputs.yesterday }} | |
| - name: add and commit | |
| uses: ./.github/actions/commit_results | |
| with: | |
| commit_msg: Update Summaries for day ${{ steps.date.outputs.yesterday }} | |
| - name: summarize today | |
| uses: ./.github/actions/generate_summaries | |
| with: | |
| ymd_option: '--day' | |
| ymd_value: ${{ steps.date.outputs.today }} | |
| - name: add and commit | |
| uses: ./.github/actions/commit_results | |
| with: | |
| commit_msg: Update Summaries for day ${{ steps.date.outputs.today }} | |
| - name: summarize this month | |
| uses: ./.github/actions/generate_summaries | |
| with: | |
| ymd_option: '--month' | |
| ymd_value: ${{ steps.date.outputs.this_month }} | |
| - name: add and commit | |
| uses: ./.github/actions/commit_results | |
| with: | |
| commit_msg: Update Summaries for month ${{ steps.date.outputs.this_month }} | |
| - name: push results | |
| uses: ./.github/actions/pull_rebase_push_retry | |
| deep_dive: | |
| name: Deep Dive | |
| needs: update_summaries | |
| runs-on: ubuntu-latest | |
| strategy: | |
| max-parallel: 11 | |
| fail-fast: false | |
| matrix: | |
| mod: [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ] | |
| steps: | |
| - name: Get dates | |
| id: date | |
| run: | | |
| echo "today=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "yesterday=$(date -d yesterday '+%Y-%m-%d')" >> $GITHUB_OUTPUT | |
| echo "last_day=$(date -d yesterday +'%d')" >> $GITHUB_OUTPUT | |
| echo "this_day=$(date +'%d')" >> $GITHUB_OUTPUT | |
| echo "this_month=$(date +'%m')" >> $GITHUB_OUTPUT | |
| echo "this_year=$(date +'%Y')" >> $GITHUB_OUTPUT | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: setup env | |
| uses: ./.github/actions/setup_my_env | |
| # in case things have changed since we spawned | |
| - name: git pull | |
| run: git pull | |
| - name: do_deep_dive | |
| uses: ./.github/actions/deep_dive | |
| with: | |
| in_dir: results/${{ steps.date.outputs.this_year }}/${{ steps.date.outputs.this_month }}/${{ steps.date.outputs.last_day }} | |
| mod: ${{ matrix.mod }} | |
| divisor: ${{ 10 }} | |
| token: ${{ secrets.VUL_DRILLER_TOKEN }} | |
| max_age: 7200 | |
| - name: do_deep_dive | |
| uses: ./.github/actions/deep_dive | |
| with: | |
| in_dir: results/${{ steps.date.outputs.this_year }}/${{ steps.date.outputs.this_month }}/${{ steps.date.outputs.this_day }} | |
| mod: ${{ matrix.mod }} | |
| divisor: ${{ 10 }} | |
| token: ${{ secrets.VUL_DRILLER_TOKEN }} | |
| max_age: 3600 | |
| - name: add and commit | |
| uses: ./.github/actions/commit_results | |
| with: | |
| results_dir: data | |
| commit_msg: Deep Dive for ${{ steps.date.outputs.yesterday }}..${{ steps.date.outputs.today }} chunk ${{ matrix.mod }}/10 | |
| - name: push result | |
| uses: ./.github/actions/pull_rebase_push_retry | |
| repo2vulid: | |
| name: Repo to Vul IDs | |
| needs: deep_dive | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: checkout | |
| uses: actions/checkout@v4 | |
| - name: setup python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.9' | |
| - name: setup env | |
| uses: ./.github/actions/setup_my_env | |
| # in case things have changed since we spawned | |
| - name: git pull | |
| run: git pull | |
| - name: extract vul ids | |
| run: repo_to_vul_id --verbose --input_dir data/repo_id | |
| - name: add and commit | |
| uses: ./.github/actions/commit_results | |
| with: | |
| results_dir: data/vul_id | |
| commit_msg: Update vulnerability ID data | |
| - name: push result | |
| uses: ./.github/actions/pull_rebase_push_retry |