Skip to content

Commit

Permalink
feat(workflows): optimize self-hosted runner alarm strategy
Browse files Browse the repository at this point in the history
  • Loading branch information
medns authored and hippy-actions[bot] committed Feb 1, 2023
1 parent 07e7c70 commit 899aa9f
Showing 1 changed file with 63 additions and 19 deletions.
82 changes: 63 additions & 19 deletions .github/workflows/gh_runners_monitor.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,27 +3,32 @@ name: '[gh] self-hosted runners monitor'
on:
workflow_dispatch:
schedule:
# Run every 12th hour.
- cron: '0 */12 * * *'
# Run every 30th minute.
- cron: '*/30 * * * *'

jobs:
runners_monitor:
if: github.repository == 'Tencent/Hippy'
runs-on: ubuntu-latest
env:
cache_file: /tmp/gh_runners_monitor.data
cache_key: gh_runners_monitor
steps:
- name: Token
uses: navikt/github-app-token-generator@v1
id: get-token
with:
private-key: ${{ secrets.BOT_APP_KEY }}
app-id: ${{ secrets.BOT_APP_ID }}
- name: Monitor
- name: Check
id: check
uses: actions/[email protected]
with:
github-token: ${{ steps.get-token.outputs.token }}
script: |
const { actions } = github.rest;
const os = require('os');
const fs = require('fs');
const runners = await github.paginate(actions.listSelfHostedRunnersForRepo, {
per_page: 100,
Expand All @@ -32,20 +37,59 @@ jobs:
const offline_runners = runners.filter(runner => runner.status === 'offline');
if (offline_runners.length > 0) {
const content = [];
content.push(`<font color="warning">${offline_runners.length}/${runners.length}</font> self-hosted runners are offline.`);
content.push(...offline_runners.map(runner => `> ${runner.name}`));
await github.request("POST ${{ secrets.WECHAT_WORK_BOT_WEBHOOK }}", {
headers: {
"content-type": "application/json"
},
data: {
chatid: "${{ secrets.WECHAT_WORK_ADMIN_CHAT_ID }}",
msgtype: "markdown",
markdown: {
content: content.join(os.EOL)
}
}
});
fs.appendFileSync(process.env.GITHUB_OUTPUT, `total_runners=${runners.length}${os.EOL}`, { encoding: 'utf8' });
fs.appendFileSync(process.env.GITHUB_OUTPUT, `offline_runners=${JSON.stringify(offline_runners)}${os.EOL}`, { encoding: 'utf8' });
}
- name: Cache
if: steps.check.outputs.offline_runners
uses: actions/cache@v3
env:
prefix: gh_runners_monitor
with:
path: ${{ env.cache_file }}
key: ${{ env.prefix }}-${{ github.run_id }}
restore-keys: ${{ env.prefix }}
- name: Notification
if: steps.check.outputs.offline_runners
uses: actions/[email protected]
env:
offline_runners: ${{ steps.check.outputs.offline_runners }}
with:
script: |
const fs = require('fs');
const os = require('os');
const offline_runners = JSON.parse(process.env.offline_runners);
if (offline_runners.length === 0) {
throw new Error('offline_runners is empty');
}
const message = [
`<font color="warning">${offline_runners.length}/${{ steps.check.outputs.total_runners }}</font> self-hosted runners are offline.`,
...offline_runners.map(runner => `> ${runner.name}`)
].join(os.EOL);
try {
const stored_data = JSON.parse(fs.readFileSync(process.env.cache_file, { encoding: 'utf8'}));
if (stored_data.message === message && Date.now() - stored_data.timestamp < 6_60_60_000 /* every 6th hour */) {
return;
}
} catch(e) {}
await github.request("POST ${{ secrets.WECHAT_WORK_BOT_WEBHOOK }}", {
headers: {
"content-type": "application/json"
},
data: {
chatid: "${{ secrets.WECHAT_WORK_ADMIN_CHAT_ID }}",
msgtype: "markdown",
markdown: {
content: message
}
}
});
fs.writeFileSync(process.env.cache_file, JSON.stringify({
message,
timestamp: Date.now()
}), { encoding: 'utf8' });

0 comments on commit 899aa9f

Please sign in to comment.