Merge pull request #582 from userpj/master #1687
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # This workflow will install Python dependencies, run tests and lint with a variety of Python versions | |
| # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python | |
| name: Python package | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: [ "master" ] | |
| pull_request_target: | |
| branches: [ "master" ] | |
| jobs: | |
| Pylint: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.12.5"] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| python -m pip install flake8 pytest | |
| python -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9 | |
| if [ -f requirements.txt ]; then pip install -r requirements.txt; fi | |
| - name: Lint with flake8 | |
| run: | | |
| # stop the build if there are Python syntax errors or undefined names | |
| flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics | |
| # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide | |
| flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics | |
| Linux-CI-Tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} | |
| APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} | |
| BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} | |
| INSTANCE_ID: ${{ secrets.INSTANCE_ID }} | |
| DATASET_ID: ${{ secrets.DATASET_ID }} | |
| APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| python-version: ["3.9","3.12"] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v3 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Print ENV | |
| run: | | |
| env | |
| ls | |
| pwd | |
| - name: Git Clone And Checkout Branch | |
| run: | | |
| fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" | |
| mkdir cicd | |
| cd cicd | |
| git clone $fork_repo | |
| cd app-builder | |
| git checkout $GITHUB_HEAD_REF | |
| git remote add upstream https://github.com/baidubce/app-builder.git | |
| git fetch upstream | |
| git remote -v | |
| git status | |
| # 找到当前分支与 upstream/master 的共同祖先提交 | |
| merge_base=$(git merge-base HEAD upstream/master) | |
| echo "merge_base=$merge_base" | |
| # 比较当前分支与 merge_base 之间的差异 | |
| changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) | |
| changed_files_py_sh=$(git diff --name-only --diff-filter=ACMRT $merge_base -- $(find python -type f \( -name '*.py' -o -name '*.sh' \))) | |
| echo "发生更改的文件为:" | |
| echo "$changed_files" | |
| echo "发生更改的py/sh文件为:" | |
| if [ -n "$changed_files_py_sh" ]; then | |
| export APPBUILDER_PYTHON_TESTS=True | |
| echo "$changed_files_py_sh" | |
| else | |
| export APPBUILDER_PYTHON_TESTS=False | |
| echo "没有检测到Python或Shell文件被更改" | |
| fi | |
| echo "APPBUILDER_PYTHON_TESTS=$APPBUILDER_PYTHON_TESTS" >> $GITHUB_ENV | |
| pwd | |
| - name: Install dependencies | |
| run: | | |
| echo $APPBUILDER_PYTHON_TESTS | |
| if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then | |
| echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Install dependencies部分。" | |
| else | |
| echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Install dependencies部分..." | |
| cd cicd/app-builder | |
| pwd | |
| sudo apt-get install ffmpeg libavcodec-extra | |
| python3 -m ensurepip --upgrade | |
| python3 -m pip install --upgrade pip | |
| python3 -m pip install --upgrade setuptools | |
| python3 -m pip install wheel | |
| python3 -m pip install coverage | |
| python3 -m pip install diff-cover | |
| python3 -m pip install pydub | |
| python3 -m pip install SQLAlchemy==2.0.31 | |
| python3 -m pip install chainlit~=1.0.200 flask~=2.3.2 flask-restful==0.3.9 | |
| python3 -m pip install opentelemetry-exporter-otlp==1.23.0 opentelemetry-instrumentation==0.44b0 opentelemetry-sdk==1.23.0 opentelemetry-api==1.23.0 | |
| python3 -m pip install pandas==2.2.2 | |
| fi | |
| - name: Build whl | |
| run: | | |
| echo $APPBUILDER_PYTHON_TESTS | |
| if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then | |
| echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Build whl部分。" | |
| else | |
| echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Build whl部分..." | |
| cd cicd/app-builder | |
| pwd | |
| python3 setup.py bdist_wheel | |
| python3 -m pip uninstall -y appbuilder-sdk | |
| python3 -m pip install dist/*.whl | |
| fi | |
| - name: Test with unittest | |
| run: | | |
| echo $APPBUILDER_PYTHON_TESTS | |
| if [ "$APPBUILDER_PYTHON_TESTS" = "False" ]; then | |
| echo "环境变量APPBUILDER_PYTHON_TESTS为False,没有检测到Python或Shell文件被更改,跳过Test with unittest部分。" | |
| else | |
| echo "检测到Python或Shell文件被更改(根据环境变量APPBUILDER_PYTHON_TESTS),准备启动单元测试Test with unittest部分..." | |
| cd cicd/app-builder | |
| pwd | |
| sh python/tests/run_python_test.sh | |
| fi | |
| Go-Test-CI: | |
| runs-on: ubuntu-latest | |
| env: | |
| APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} | |
| APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} | |
| BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} | |
| INSTANCE_ID: ${{ secrets.INSTANCE_ID }} | |
| DATASET_ID: ${{ secrets.DATASET_ID }} | |
| APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} | |
| APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }} | |
| DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }} | |
| DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| go-version: ['1.18'] | |
| max-parallel: 2 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up Go | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: ${{ matrix.go-version }} | |
| - name: Git Clone And Checkout Branch | |
| run: | | |
| fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" | |
| mkdir cicd | |
| cd cicd | |
| git clone $fork_repo | |
| cd app-builder | |
| echo "github_head的值为：" | |
| echo $GITHUB_HEAD_REF | |
| git checkout $GITHUB_HEAD_REF | |
| git remote add upstream https://github.com/baidubce/app-builder.git | |
| git fetch upstream | |
| git remote -v | |
| git status | |
| # 找到当前分支与 upstream/master 的共同祖先提交 | |
| merge_base=$(git merge-base HEAD upstream/master) | |
| echo "merge_base=$merge_base" | |
| # 比较当前分支与 merge_base 之间的差异 | |
| changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) | |
| changed_files_go=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'go/**') | |
| echo "发生更改的文件为:" | |
| echo "$changed_files" | |
| echo "发生更改的Go相关文件为:" | |
| if [ -n "$changed_files_go" ]; then | |
| export APPBUILDER_GO_TESTS=True | |
| echo "$changed_files_go" | |
| else | |
| export APPBUILDER_GO_TESTS=False | |
| echo "没有检测到Go或Shell文件被更改" | |
| fi | |
| echo "APPBUILDER_GO_TESTS=$APPBUILDER_GO_TESTS" >> $GITHUB_ENV | |
| - name: Install dependencies and tools | |
| run: | | |
| echo $APPBUILDER_GO_TESTS | |
| if [ "$APPBUILDER_GO_TESTS" = "False" ]; then | |
| echo "环境变量APPBUILDER_GO_TESTS为False,没有检测到Go文件被更改，跳过Install dependencies and tools部分。" | |
| else | |
| echo "检测到Go文件被更改，准备启动Install dependencies and tools部分..." | |
| cd cicd/app-builder/go/appbuilder | |
| go mod tidy | |
| go get github.com/axw/gocov/gocov # 安装增量覆盖率工具 | |
| # 安装 golangci-lint | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.53.3 | |
| echo "$GOPATH/bin" >> $GITHUB_PATH | |
| fi | |
| - name: Run tests and calculate incremental coverage | |
| run: | | |
| echo $APPBUILDER_GO_TESTS | |
| if [ "$APPBUILDER_GO_TESTS" = "False" ]; then | |
| echo "环境变量APPBUILDER_GO_TESTS为False，没有检测到Go文件被更改，跳过Run tests部分。" | |
| else | |
| echo "检测到Go文件被更改，准备启动Run tests部分..." | |
| cd cicd/app-builder/go | |
| # 运行测试并生成覆盖率报告 | |
| chmod +x run_go_test.sh | |
| ./run_go_test.sh | |
| fi | |
| shell: bash | |
| Java-CI-Tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| APPBUILDER_TOKEN: ${{ secrets.APPBUILDER_TOKEN }} | |
| APPBUILDER_TOKEN_V2: ${{ secrets.APPBUILDER_TOKEN_V2 }} | |
| BAIDU_VDB_API_KEY: ${{ secrets.BAIDU_VDB_API_KEY }} | |
| INSTANCE_ID: ${{ secrets.INSTANCE_ID }} | |
| DATASET_ID: ${{ secrets.DATASET_ID }} | |
| APPBUILDER_TOKEN_DOC_FORMAT: ${{ secrets.APPBUILDER_TOKEN_DOC_FORMAT }} | |
| APPBUILDER_TOKEN_V3: ${{ secrets.APPBUILDER_TOKEN_V3 }} | |
| DATASET_ID_V3: ${{ secrets.DATASET_ID_V3 }} | |
| DOCUMENT_ID_V3: ${{ secrets.DOCUMENT_ID_V3 }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| java-version: ['8'] # 使用 Java 11 和 17 作为示例 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| - name: Set up JDK ${{ matrix.java-version }} | |
| uses: actions/setup-java@v3 | |
| with: | |
| java-version: ${{ matrix.java-version }} | |
| distribution: 'temurin' | |
| - name: Print ENV | |
| run: | | |
| env | |
| ls | |
| pwd | |
| - name: Git Clone And Checkout Branch | |
| run: | | |
| fork_repo="https://github.com/$GITHUB_ACTOR/app-builder.git" | |
| mkdir cicd | |
| cd cicd | |
| git clone $fork_repo | |
| cd app-builder | |
| git checkout $GITHUB_HEAD_REF | |
| git remote add upstream https://github.com/baidubce/app-builder.git | |
| git fetch upstream | |
| git remote -v | |
| git status | |
| # 找到当前分支与 upstream/master 的共同祖先提交 | |
| merge_base=$(git merge-base HEAD upstream/master) | |
| echo "merge_base=$merge_base" | |
| # 比较当前分支与 merge_base 之间的差异 | |
| changed_files=$(git diff --name-only --diff-filter=ACMRT $merge_base) | |
| changed_files_java=$(git diff --name-only --diff-filter=ACMRT $merge_base -- 'java/**') | |
| echo "发生更改的文件为:" | |
| echo "$changed_files" | |
| echo "发生更改的Java相关文件为:" | |
| if [ -n "$changed_files_java" ]; then | |
| export APPBUILDER_JAVA_TESTS=True | |
| echo "$changed_files_java" | |
| else | |
| export APPBUILDER_JAVA_TESTS=False | |
| echo "没有检测到Java文件被更改" | |
| fi | |
| echo "APPBUILDER_JAVA_TESTS=$APPBUILDER_JAVA_TESTS" >> $GITHUB_ENV | |
| pwd | |
| - name: Install Python dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install gitpython lxml | |
| - name: Install dependencies | |
| run: | | |
| echo $APPBUILDER_JAVA_TESTS | |
| if [ "$APPBUILDER_JAVA_TESTS" = "False" ]; then | |
| echo "环境变量APPBUILDER_JAVA_TESTS为False,没有检测到Java文件被更改,跳过Install dependencies部分。" | |
| else | |
| echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动单元测试Install dependencies部分..." | |
| cd cicd/app-builder/java | |
| mvn clean install -DskipTests | |
| fi | |
| - name: Run Java tests with Jacoco | |
| if: env.APPBUILDER_JAVA_TESTS == 'True' | |
| run: | | |
| echo "检测到Java文件被更改(根据环境变量APPBUILDER_JAVA_TESTS),准备启动Run Java tests with Jacoco部分..." | |
| cd cicd/app-builder/java | |
| bash print_coverage.sh | |