Skip to content

CI Report(#11789567972) #62427

CI Report(#11789567972)

CI Report(#11789567972) #62427

Workflow file for this run

name: CI Report
run-name: CI Report(#${{ github.event.workflow_run.id }})
on:
workflow_run:
workflows: [ "CI PIPELINE", "CI PIPELINE - BRANCH" ]
types:
- completed
permissions:
checks: write
actions: write
contents: write
deployments: write
discussions: write
issues: write
packages: write
pages: write
pull-requests: write
repository-projects: write
security-events: write
statuses: write
env:
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
REPO: ${{ github.repository }}
jobs:
INFO:
runs-on: [ self-hosted, quick ]
if: >
github.event.workflow_run.event == 'pull_request' &&
(github.event.workflow_run.conclusion == 'success' || github.event.workflow_run.conclusion == 'failure')
outputs:
PR_NUMBER: ${{ steps.pr_details.outputs.PR_NUMBER }}
BASE_REF: ${{ steps.pr_details.outputs.BASE_REF }}
PR_STATE: ${{ steps.pr_details.outputs.PR_STATE }}
BUCKET_PREFIX: ${{ steps.bucket_info.outputs.bucket_prefix }}
steps:
- run: |
sleep 10
- name: INFO
id: bucket_info
run: |
bucket_prefix=`echo ${REPO%/*} | tr '[:upper:]' '[:lower:]'`
echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT
- name: Download workflow artifact - PR
uses: dawidd6/action-download-artifact@v3
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ github.event.workflow_run.id }}
name: 'pr_num'
- name: Read the pr num file
id: pr_num_reader
uses: juliangruber/read-file-action@v1
with:
path: ./pr_num.txt
- name: PR DETAILS
id: pr_details
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
PR_NUMBER: ${{ steps.pr_num_reader.outputs.content }}
REPO: ${{ github.repository }}
run: |
PR_NUMBER=`echo ${PR_NUMBER}`
pr_info=`gh pr view ${PR_NUMBER} -R ${REPO} --json baseRefName,state`
base_ref=$(echo ${pr_info} | jq -r .baseRefName)
pr_state=$(echo ${pr_info} | jq -r .state)
echo "BASE_REF=${base_ref}" >> $GITHUB_OUTPUT
echo "PR_STATE=${pr_state}" >> $GITHUB_OUTPUT
echo "PR_NUMBER=${PR_NUMBER}" >> $GITHUB_OUTPUT
- name: INFO
id: info
run: |
echo ${{ steps.pr_details.outputs.PR_NUMBER }}
echo ${{ steps.pr_details.outputs.BASE_REF }}
FE-REPORT:
runs-on: [ self-hosted, quick ]
needs: INFO
if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED'
env:
PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
bucket_prefix: ${{ needs.INFO.outputs.BUCKET_PREFIX }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Download FE UT XML
id: download-ut-xml
run: |
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/fe_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
echo "size=${size}" >> $GITHUB_OUTPUT
if [[ "$size" != "0" ]]; then
mkdir fe && cd fe
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/ . --recursive
tar zxf fe_ut_report.tar.gz
elif [[ "${BASE_REF}" == "main" ]]; then
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover /var/local/env/empty_cov_result --github-token ${{ secrets.GITHUB_TOKEN }} \
--repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco
fi
- name: Publish UT Report
uses: mikepenz/action-junit-report@v4
id: publish_report
if: steps.download-ut-xml.outputs.size != '0'
env:
github_token: ${{ secrets.GITHUB_TOKEN }}
token: ${{ secrets.GITHUB_TOKEN }}
with:
check_name: 'FE UT Report'
detailed_summary: true
fail_on_failure: true
commit: ${{ github.event.workflow_run.head_sha }}
report_paths: ./fe/fe-core/target/surefire-reports/*.xml
- name: Merge FE Coverage
id: merge_report
if: steps.publish_report.outcome == 'success' && env.BASE_REF == 'main'
env:
fe_path: ${{ github.workspace }}/fe
run: |
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
if [[ -e ${fe_path}/test.exec ]]; then
java -jar package/jacococli.jar merge ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/test.exec --destfile ${fe_path}/merge.exec
else
cp ${fe_path}/fe-core/target/jacoco.exec ${fe_path}/merge.exec
fi
- name: Generate XML Report
id: generate-xml-report
env:
package_path: ${{ github.workspace }}/ci-tool/package
fe_path: ${{ github.workspace }}/fe
fe_core_path: ${{ github.workspace }}/fe/fe-core
if: steps.merge_report.outcome == 'success'
run: |
rm -rf result
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
cd ${fe_core_path}/target/classes;
rm -rf org;
cd com/starrocks && rm -rf thrift proto sql/parser builtins common/Version.class;
cd ${{ github.workspace }};
java -jar $package_path/jacococli.jar report ${fe_path}/merge.exec --classfiles ${fe_core_path}/target/classes/ \
--html ./result --xml ${{ github.workspace }}/coverage.xml \
--sourcefiles ${fe_core_path}/src/main/java/ --encoding utf-8 --name fe-coverage
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}/
ossutil64 --config-file ~/.ossutilconfig cp coverage.xml ${oss_path} -f
# Incremental Total Coverage
- name: Publish Incremental Coverage Report - Total
if: steps.generate-xml-report.outcome == 'success'
env:
fe_path: ${{ github.workspace }}/fe
run: |
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
if [[ -e "${fe_path}/diff.txt" ]]; then
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \
--repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
-d ${fe_path}/diff.txt -dt file
else
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover ${{ github.workspace }}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \
--repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco
fi
rm -rf coverchecker
rm -rf ci-tools
# total coverage
- name: Coverage Report
uses: codecov/codecov-action@v3
if: steps.generate-xml-report.outcome == 'success'
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: ${{ github.workspace }}/coverage.xml
dry_run: false
name: fe-total
flags: fe-total
fail_ci_if_error: false
verbose: true
override_pr: ${{ env.PR_NUMBER }}
override_branch: ${{ env.BASE_REF }}
override_commit: ${{ env.HEAD_SHA }}
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
BE-REPORT:
runs-on: [ self-hosted, quick ]
needs: INFO
if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED'
env:
GH_TOKEN: ${{ github.token }}
REPO: ${{ github.repository }}
PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
bucket_prefix: ${{ needs.INFO.outputs.BUCKET_PREFIX }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Download BE UT XML
id: download-ut-xml
run: |
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
be_ut_res_path=${oss_path}/flag/be_ut_res
size=$(ossutil64 --config-file ~/.ossutilconfig ls ${be_ut_res_path} | grep "Object Number is" | awk '{print $NF}')
echo "size=${size}" >> $GITHUB_OUTPUT
if [[ "$size" != "0" ]]; then
ossutil64 --config-file ~/.ossutilconfig cp ${be_ut_res_path} . >/dev/null
res=`cat be_ut_res`
if [[ "$res" != "0" ]]; then
echo "::error::BE UT failed!"
exit 1
fi
cd be
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/be_ut_coverage.xml . -f 1>/dev/null
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/base_version.txt . -f 1>/dev/null
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/diff.txt . -f 1>/dev/null
echo "base_version=`cat base_version.txt`" >> $GITHUB_OUTPUT
else
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover /var/local/env/be_empty_coverage.xml --github-token ${{ secrets.GITHUB_TOKEN }} --pr ${PR_NUMBER} \
--repo ${{ github.repository }} --threshold 80 --github-url api.github.com -type cobertura --module BE
fi
- name: Merge BE Total Coverage
id: merge
if: github.repository == 'StarRocks/starrocks' && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success'
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
export ECI_ID=`./bin/create_eci.sh ${BASE_REF} ${REPO}`
export TOTAL_COV_XML=/var/local/env/be_total_coverage_${PR_NUMBER}.xml
export COV_LOG=/var/local/env/be_total_coverage_${PR_NUMBER}.log
export RES_FILE=/var/local/env/be_total_coverage_res_${PR_NUMBER}.log
echo ${ECI_ID}
echo "ECI_ID=${ECI_ID}" >> $GITHUB_OUTPUT
echo "COV_XML=${TOTAL_COV_XML}" >> $GITHUB_OUTPUT
echo "COV_LOG=${COV_LOG}" >> $GITHUB_OUTPUT
echo "RES_FILE=${RES_FILE}" >> $GITHUB_OUTPUT
./bin/gen_be_cov.sh ${REPO} ${BASE_REF} ${PR_NUMBER} ${HEAD_SHA}
# Incremental Total Coverage
- name: Publish Incremental Coverage Report - Total
if: always() && steps.download-ut-xml.outputs.size != '0' && steps.download-ut-xml.outcome == 'success'
env:
be_path: ${{ github.workspace }}/be
merge_outcome: ${{ steps.merge.outcome }}
total_xml: ${{ steps.merge.outputs.COV_XML }}
run: |
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
if [[ "${{ github.repository }}" != "StarRocks/starrocks" ]] || [[ "${merge_outcome}" == "failure" ]]; then
total_xml=${be_path}/be_ut_coverage.xml
fi
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover ${total_xml} --github-token ${{ secrets.GITHUB_TOKEN }} --repo ${{ github.repository }} \
--threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type cobertura --module BE \
-d ${be_path}/diff.txt -dt file
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
eci rm ${{ steps.merge.outputs.ECI_ID }} || true
rm -rf ${{ steps.merge.outputs.COV_LOG }} ${{ steps.merge.outputs.COV_XML }} ${{ steps.merge.outputs.RES_FILE }}
JAVA-EXTENSION-REPORT:
runs-on: [ self-hosted, quick ]
needs: INFO
if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED'
env:
PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
bucket_prefix: ${{ needs.INFO.outputs.BUCKET_PREFIX }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Download Java Extension UT XML
id: download-ut-xml
run: |
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/UT-Report/${PR_NUMBER}
size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path}/java_extension_ut_report.tar.gz | grep "Object Number is" | awk '{print $NF}')
echo "size=${size}" >> $GITHUB_OUTPUT
if [[ "$size" != "0" ]]; then
mkdir java-extensions && cd java-extensions
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path}/java_extension_ut_report.tar.gz . --recursive
tar zxf java_extension_ut_report.tar.gz
elif [[ "${BASE_REF}" == "main" ]]; then
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover /var/local/env/java_extension_empty_coverage --github-token ${{ secrets.GITHUB_TOKEN }} \
--repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
--module "Java-Extensions"
fi
- name: Publish UT Report
uses: mikepenz/action-junit-report@v4
id: publish_report
if: steps.download-ut-xml.outputs.size != '0'
env:
github_token: ${{ secrets.GITHUB_TOKEN }}
token: ${{ secrets.GITHUB_TOKEN }}
with:
check_name: 'Java Extensions UT Report'
detailed_summary: true
fail_on_failure: true
commit: ${{ github.event.workflow_run.head_sha }}
report_paths: ./java-extensions/xml_dir/*.xml
# Incremental Total Coverage
- name: Publish Incremental Coverage Report - Total
if: steps.publish_report.outcome == 'success' && env.BASE_REF == 'main'
env:
code_path: ${{ github.workspace }}/java-extensions
run: |
rm -rf ./coverchecker && ln -s /var/local/env/coverchecker ./coverchecker && cd coverchecker && git pull
export JAVA_HOME=/var/local/env/jdk1.8.0_202;
export PATH=$JAVA_HOME/bin:$PATH;
java -jar cover-checker-console/target/cover-checker-console-1.4.0-jar-with-dependencies.jar \
--cover ${code_path}/result/ --github-token ${{ secrets.GITHUB_TOKEN }} \
--repo ${{ github.repository }} --threshold 80 --github-url api.github.com --pr ${PR_NUMBER} -type jacoco \
-d ${code_path}/diff.txt -dt file --module "Java-Extensions"
rm -rf coverchecker
rm -rf ci-tools
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*
SQL-Tester-REPORT:
runs-on: [ self-hosted, quick ]
needs: INFO
if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED'
env:
PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: INFO
id: info
run: |
repo="${{ github.repository }}"
bucket_prefix=`echo ${repo%/*} | tr '[:upper:]' '[:lower:]'`
echo "bucket_prefix=${bucket_prefix}" >> $GITHUB_OUTPUT
- name: Download SQL-Tester XML
id: download-SQL-Tester-xml
env:
bucket_prefix: ${{ steps.info.outputs.bucket_prefix }}
run: |
mkdir sql-tester-result && cd sql-tester-result
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/SQL-Tester-XML/${PR_NUMBER}/
size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
echo "size=${size}" >> $GITHUB_OUTPUT
if [[ "$size" != "0" ]]; then
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive
else
cp /var/local/env/sqltester_empty_coverage.xml .
fi
- name: Prepare Tools
id: prepare-tools
run: |
mkdir -p .actions/nose-report-action
cd .actions/nose-report-action
git clone https://github.com/StarRocks/action-junit-report.git .
- name: Publish SQL-Tester Report
uses: ./.actions/nose-report-action
if: steps.prepare-tools.outcome == 'success'
with:
check_name: 'SQL-Tester Report'
fail_on_failure: true
detailed_summary: true
token: ${{ secrets.GITHUB_TOKEN }}
commit: ${{ env.HEAD_SHA }}
report_paths: 'sql-tester-result/*.xml'
Admit-REPORT:
runs-on: [ self-hosted, quick ]
needs: INFO
if: needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.BASE_REF == 'main' && needs.INFO.outputs.PR_STATE != 'MERGED'
env:
PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
BASE_REF: ${{ needs.INFO.outputs.BASE_REF }}
bucket_prefix: ${{ needs.INFO.outputs.bucket_prefix }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: Download Admit XML
id: download-admit-xml
run: |
mkdir admit-result && cd admit-result
oss_path=oss://${bucket_prefix}-ci-release/$BASE_REF/Release/pr/Admit-XML/${PR_NUMBER}/
size=$(ossutil64 --config-file ~/.ossutilconfig ls ${oss_path} | grep "Object Number is" | awk '{print $NF}')
echo "size=${size}" >> $GITHUB_OUTPUT
if [[ "$size" != "0" ]]; then
ossutil64 --config-file ~/.ossutilconfig cp ${oss_path} . --recursive
fi
- name: Prepare Tools
if: steps.download-admit-xml.outputs.size != '0'
id: prepare-tools
run: |
mkdir -p .actions/nose-report-action
cd .actions/nose-report-action
git clone https://github.com/StarRocks/action-junit-report.git .
- name: Publish Admit Report
uses: ./.actions/nose-report-action
if: steps.prepare-tools.outcome == 'success'
with:
check_name: 'Admit Report'
fail_on_failure: true
detailed_summary: true
commit: ${{ env.HEAD_SHA }}
token: ${{ secrets.GITHUB_TOKEN }}
report_paths: 'admit-result/*.xml'
NOTIFICATION:
runs-on: [ self-hosted, quick ]
needs: [ INFO, FE-REPORT, Admit-REPORT, BE-REPORT, SQL-Tester-REPORT, JAVA-EXTENSION-REPORT ]
if: always() && needs.INFO.outputs.PR_NUMBER != '' && needs.INFO.outputs.PR_STATE != 'MERGED'
env:
WORKFLOW_RUN_ID: ${{ github.event.workflow_run.id }}
FEISHU_APP_ID: ${{ secrets.FEISHU_APP_ID }}
FEISHU_APP_SECRET: ${{ secrets.FEISHU_APP_SECRET }}
PR_NUMBER: ${{ needs.INFO.outputs.PR_NUMBER }}
steps:
- name: CLEAN
run: |
rm -rf ${{ github.workspace }} && mkdir -p ${{ github.workspace }}
- name: NOTIFY
run: |
rm -rf ./ci-tool && cp -rf /var/lib/ci-tool ./ci-tool && cd ci-tool && git pull && source lib/init.sh
./bin/notify.sh
- name: Clean ENV
if: always()
run: |
rm -rf ${{ github.workspace }}/*