Skip to content
This repository has been archived by the owner on Sep 21, 2024. It is now read-only.

Commit

Permalink
chore: Report flakes in CI
Browse files Browse the repository at this point in the history
  • Loading branch information
@cdata
cdata committed Oct 24, 2023
1 parent 3a17db9 commit 264567c
Show file tree
Hide file tree
Showing 2 changed files with 256 additions and 57 deletions.
83 changes: 83 additions & 0 deletions .github/workflows/report_test_flakes.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
on:
workflow_call:
inputs:
target:
required: true
type: string

name: 'Workflow Analysis'

jobs:
report-test-flakes:
name: 'Report test flakes'
runs-on: ubuntu-latest
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/download-artifact@v3
- name: Parse test results
run: |
RESULTS_PATH="./test-results-${{ inputs.target }}/test-results.log"
csplit -q "$RESULTS_PATH" %^------------%
if [[ -f "./xx00" ]]; then
SUMMARY=`tail ./xx00 -n+2`
if [[ "$SUMMARY" =~ 'FLAKY' ]]; then
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
{
echo "summary<<EOF"
echo "$SUMMARY"
echo "EOF"
} >> $GITHUB_OUTPUT
fi
fi
- name: Report flakes
uses: actions/github-script@v6
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const flakeyTestsHeader = 'Test flake analysis: ${{ inputs.target }}';
const existingComment = comments.find(comment => {
return comment.user.type === 'Bot' && comment.body.includes(flakeyTestsHeader)
});
let body;
if (`${{ steps['Parse test results'].outputs.summary }}`) {
body = `${flakeyTestsHeader}
\`\`\`shell
${{ steps['Parse test results'].outputs.summary }}
\`\`\`
`;
} else {
body = `${flakeyTestsHeader}
No flakes detected!
`;
}
if (existingComment) {
github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existingComment.id,
body
})
} else {
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body
})
}
230 changes: 173 additions & 57 deletions .github/workflows/run_test_suite.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,86 +56,202 @@ jobs:
- name: 'Run Linter'
run: cargo clippy --all -- -D warnings

run-test-suite-windows:
runs-on: windows-latest
run-full-test-suite:
strategy:
matrix:
features: ['test-kubo,headers', 'test-kubo,headers,rocksdb']
platform: ['windows-latest', 'ubuntu-latest']
runs-on: ${{ matrix.platform }}
steps:
- uses: actions/checkout@v3
- uses: Swatinem/rust-cache@v2
- name: 'Setup Rust'
run: |
curl -sSf https://sh.rustup.rs | sh -s -- -y
- name: 'Install environment packages'
- name: 'Install environment packages (Windows)'
if: ${{ matrix.platform == 'windows-latest' }}
run: |
choco install -y cmake protoc openssl
shell: sh
- name: Install cargo-binstall
uses: cargo-bins/[email protected]
- name: Install binaries from cargo
run: |
cargo binstall cargo-nextest --no-confirm
- name: 'Install IPFS Kubo'
uses: ibnesayeed/setup-ipfs@master
with:
ipfs_version: v0.17.0
run_daemon: true
- name: 'Run Rust native target tests'
run: cargo nextest run --features test-kubo,headers --retries 5 --color always
env:
NOOSPHERE_LOG: academic

run-test-suite-linux:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: Swatinem/rust-cache@v2
- name: 'Setup Rust'
run: |
curl -sSf https://sh.rustup.rs | sh -s -- -y
- name: 'Install environment packages'
- name: 'Install environment packages (Linux)'
if: ${{ matrix.platform == 'ubuntu-latest' }}
run: |
sudo apt-get update -qqy
sudo apt-get install jq protobuf-compiler cmake
- name: Install cargo-binstall
uses: cargo-bins/[email protected]
- name: Install binaries from cargo
run: |
cargo binstall cargo-nextest --no-confirm
- name: 'Install IPFS Kubo'
uses: ibnesayeed/setup-ipfs@master
with:
ipfs_version: v0.17.0
run_daemon: true
- name: 'Run Rust native target tests'
run: cargo nextest run --features test-kubo,headers --retries 5 --color always
env:
NOOSPHERE_LOG: academic

run-test-suite-linux-rocksdb:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: Swatinem/rust-cache@v2
- name: 'Setup Rust'
run: |
curl -sSf https://sh.rustup.rs | sh -s -- -y
- name: 'Install environment packages'
run: |
sudo apt-get update -qqy
sudo apt-get install jq protobuf-compiler cmake libclang-dev
- name: Install cargo-binstall
uses: cargo-bins/[email protected]
- name: Install binaries from cargo
run: |
cargo binstall cargo-nextest --no-confirm
- name: 'Install IPFS Kubo'
uses: ibnesayeed/setup-ipfs@master
with:
ipfs_version: v0.17.0
run_daemon: true
- name: 'Run Rust native target tests (RocksDB)'
run: cargo nextest run --features rocksdb,test-kubo,headers --retries 5 --color always
cargo binstall cargo-nextest --no-confirm --force
- name: 'Run Rust tests'
run: cargo nextest run --features ${{matrix.features}} --retries 5 --color always 2>&1 | tee test-results.log
env:
NOOSPHERE_LOG: academic
- name: Parse test results
run: |
RESULTS_PATH="./test-results.log"
csplit -q "$RESULTS_PATH" %^------------%
if [[ -f "./xx00" ]]; then
SUMMARY=`tail ./xx00 -n+2`
if [[ "$SUMMARY" =~ 'FLAKY' ]]; then
# See https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
{
echo "summary<<EOF"
echo "$SUMMARY"
echo "EOF"
} >> $GITHUB_OUTPUT
fi
fi
- name: Report test flakes
uses: actions/github-script@v6
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
const { data: comments } = await github.rest.issues.listComments({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: context.issue.number,
});
const flakeyTestsHeader = 'Test flake analysis for `${{ matrix.platform }}` (`${{ matrix.features }}`)';
const existingComment = comments.find(comment => {
return comment.user.type === 'Bot' && comment.body.includes(flakeyTestsHeader)
});
let body;
if (`${{ steps['Parse test results'].outputs.summary }}`) {
body = `#### 🟡 ${flakeyTestsHeader}
Test flakes detected:
\`\`\`shell
${{ steps['Parse test results'].outputs.summary }}
\`\`\`
`;
} else {
body = `#### 🟢 ${flakeyTestsHeader}
No flakes detected 🎉
`;
}
if (existingComment) {
github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: existingComment.id,
body
})
} else {
github.rest.issues.createComment({
issue_number: context.issue.number,
owner: context.repo.owner,
repo: context.repo.repo,
body
})
}
# run-test-suite-linux:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - uses: Swatinem/rust-cache@v2
# - name: 'Setup Rust'
# run: |
# curl -sSf https://sh.rustup.rs | sh -s -- -y
# - name: 'Install environment packages'
# run: |
# sudo apt-get update -qqy
# sudo apt-get install jq protobuf-compiler cmake
# - name: Install cargo-binstall
# uses: cargo-bins/[email protected]
# - name: Install binaries from cargo
# run: |
# cargo binstall cargo-nextest --no-confirm --force
# - name: 'Install IPFS Kubo'
# uses: ibnesayeed/setup-ipfs@master
# with:
# ipfs_version: v0.17.0
# run_daemon: true
# - name: 'Run Rust native target tests'
# run: cargo nextest run --features test-kubo,headers --retries 5 --color always 2>&1 | tee test-results.log
# env:
# NOOSPHERE_LOG: academic
# - uses: actions/upload-artifact@v3
# with:
# name: test-results-linux
# path: ./test-results.log

# report-test-flakes:
# name: 'Report test flakes (Linux)'
# needs: ['run-test-suite-linux']
# uses: ./.github/workflows/report_test_flakes.yaml
# secrets: inherit
# with:
# target: 'linux'

# run-test-suite-windows:
# runs-on: windows-latest
# steps:
# - uses: actions/checkout@v3
# - uses: Swatinem/rust-cache@v2
# - name: 'Setup Rust'
# run: |
# curl -sSf https://sh.rustup.rs | sh -s -- -y
# - name: 'Install environment packages'
# run: |
# choco install -y cmake protoc openssl
# shell: sh
# - name: Install cargo-binstall
# uses: cargo-bins/[email protected]
# - name: Install binaries from cargo
# run: |
# cargo binstall cargo-nextest --no-confirm
# - name: 'Install IPFS Kubo'
# uses: ibnesayeed/setup-ipfs@master
# with:
# ipfs_version: v0.17.0
# run_daemon: true
# - name: 'Run Rust native target tests'
# run: cargo nextest run --features test-kubo,headers --retries 5 --color always 2>&1 | tee test-results.log
# env:
# NOOSPHERE_LOG: academic

# run-test-suite-linux-rocksdb:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - uses: Swatinem/rust-cache@v2
# - name: 'Setup Rust'
# run: |
# curl -sSf https://sh.rustup.rs | sh -s -- -y
# - name: 'Install environment packages'
# run: |
# sudo apt-get update -qqy
# sudo apt-get install jq protobuf-compiler cmake libclang-dev
# - name: Install cargo-binstall
# uses: cargo-bins/[email protected]
# - name: Install binaries from cargo
# run: |
# cargo binstall cargo-nextest --no-confirm --force
# - name: 'Install IPFS Kubo'
# uses: ibnesayeed/setup-ipfs@master
# with:
# ipfs_version: v0.17.0
# run_daemon: true
# - name: 'Run Rust native target tests (RocksDB)'
# run: cargo nextest run --features rocksdb,test-kubo,headers --retries 5 --color always
# env:
# NOOSPHERE_LOG: academic

run-test-suite-linux-c:
runs-on: ubuntu-latest
Expand Down

0 comments on commit 264567c

Please sign in to comment.