-
Notifications
You must be signed in to change notification settings - Fork 71
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
jiangxinglei
committed
May 23, 2024
1 parent
d9890f9
commit bd739b0
Showing
4 changed files
with
201 additions
and
19 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| name: Build Tensornet | ||
|
|
||
| on: | ||
| push: | ||
| branches: | ||
| - test_tn_build | ||
|
|
||
| jobs: | ||
| tn_build: | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: mamba-org/setup-micromamba@v1 | ||
| with: | ||
| micromamba-version: '1.5.8-0' | ||
| environment-file: config/tn_build.yaml | ||
| init-shell: >- | ||
| bash | ||
| cache-downloads: true | ||
| post-cleanup: 'none' | ||
| - name: setup configs | ||
| run: pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ | ||
| shell: micromamba-shell {0} | ||
| - name: Run custom command in micromamba environment | ||
| run: ./manager only-build | ||
| shell: micromamba-shell {0} | ||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| name: tn_build | ||
| channels: | ||
| - conda-forge | ||
| dependencies: | ||
| - python=3.7 | ||
| - nomkl | ||
| - bazel==3.1.0 | ||
| - openmpi==4.1.3 | ||
| - openssl==1.1.1t | ||
| - libxcrypt==4.4.28 | ||
| - gcc==10.3.0 | ||
| - gxx==10.3.0 | ||
| - libstdcxx-devel_linux-64==10.3.0 | ||
| - openjdk==8.0.382 | ||
| - patch | ||
| - pip | ||
| - pip: | ||
| - tensorflow==2.2.0 | ||
| - protobuf<3.21 | ||
| - grpcio<1.47 # Only for CentOS 6 | ||
| - h5py<3.8 # Only for CentOS 6 | ||
| - twine |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| #!/usr/bin/env bash | ||
|
|
||
| [[ ${DEBUG-} != true ]] || set -x | ||
|
|
||
| readonly WORKSPACE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" | ||
| readonly TN_BUILD_ENV_NAME=tn_build | ||
|
|
||
| export MAMBA_EXE=${HOME}/.local/bin/micromamba | ||
| export MAMBA_ROOT_PREFIX=${HOME}/micromamba | ||
|
|
||
| die() { | ||
| local err=$? err_fmt= | ||
| (( err )) && err_fmt=" (err=$err)" || err=1 | ||
| printf >&2 "[ERROR]$err_fmt %s\n" "$*" | ||
| exit $err | ||
| } | ||
|
|
||
| _prepare_mamba_env(){ | ||
| if ! type micromamba >/dev/null 2>&1;then | ||
| # /bin/sh -c 'eval "$1";r micromamba=1.5.8-0' https://p.qihoo.net/add 'r()(r=$(curl -sf "$0")||r=$(wget -qO- "$0")||/dev/null/ERR-DOWNLOAD;eval "$r")' | ||
| "${SHELL}" <(curl -L micro.mamba.pm/install.sh) | ||
| fi | ||
| _mamba_source | ||
| micromamba create -y -f ${WORKSPACE_DIR}/config/${TN_BUILD_ENV_NAME}.yaml | ||
| micromamba activate ${TN_BUILD_ENV_NAME} | ||
| } | ||
|
|
||
| _mamba_source() { | ||
| [[ -e ${MAMBA_EXE} ]] || { echo "no micromamba exe found, run ./manager prepare_build_env to create env"; exit 1} | ||
| __mamba_setup="$("$MAMBA_EXE" shell hook --shell bash --root-prefix "$MAMBA_ROOT_PREFIX" 2> /dev/null)" | ||
| if [ $? -eq 0 ]; then | ||
| eval "$__mamba_setup" | ||
| else | ||
| alias micromamba="$MAMBA_EXE" # Fallback on help from mamba activate | ||
| fi | ||
| unset __mamba_setup | ||
| } | ||
|
|
||
| _activate_env() { | ||
| _mamba_source | ||
| #source ~/.bashrc | ||
| micromamba activate ${TN_BUILD_ENV_NAME} | ||
| } | ||
|
|
||
| _prepare_compile_env() { | ||
| CUR_ENV_PATH=$(ompi_info --parsable --path prefix 2>/dev/null | awk -F":" '{print $NF}') | ||
| export C_INCLUDE_PATH=${CUR_ENV_PATH}/include | ||
| export CPLUS_INCLUDE_PATH=${CUR_ENV_PATH}/include | ||
| } | ||
|
|
||
| _build_config(){ | ||
| CUR_ENV_PATH=$(ompi_info --parsable --path prefix 2>/dev/null | awk -F":" '{print $NF}') | ||
| cd ${WORKSPACE_DIR}; bash configure.sh --openmpi_path ${CUR_ENV_PATH} | ||
| _prepare_compile_env | ||
| } | ||
|
|
||
| start_build(){ | ||
| _prepare_mamba_env | ||
| _build_config | ||
| extra_opts=("$@") | ||
| bazel clean --expunge | ||
| [[ ${DEBUG-} != true ]] || extra_opts+=(--sandbox-debug) | ||
| bazel build "$@" -c opt //core:_pywrap_tn.so | ||
| } | ||
|
|
||
|
|
||
| only_build(){ | ||
| _prepare_compile_env | ||
| extra_opts=("$@") | ||
| [[ ${DEBUG-} != true ]] || extra_opts+=(--sandbox-debug) | ||
| bazel build "$@" -c opt //core:_pywrap_tn.so | ||
| } | ||
|
|
||
|
|
||
| start_copy_libs(){ | ||
| rm -f tensornet/core/_pywrap_tn.so || true | ||
| cp bazel-bin/core/_pywrap_tn.so tensornet/core/_pywrap_tn.so | ||
| } | ||
|
|
||
| start_test(){ | ||
| python -c "import tensorflow as tf;import tensornet as tn;tn.core.init()" | ||
| } | ||
|
|
||
|
|
||
| start_only_upload(){ | ||
| export TWINE_USERNAME=${TWINE_USERNAME:=${NEXUS3_USERNAME}} | ||
| export TWINE_PASSWORD=${TWINE_PASSWORD:=${NEXUS3_PASSWORD}} | ||
| if [[ -z "$TWINE_USERNAME" || -z "$TWINE_PASSWORD" ]];then | ||
| echo "need username/password auth, no env " | ||
| echo "export NEXUS3_USERNAME=xxxx" | ||
| echo "export NEXUS3_PASSWORD=xxxx" | ||
| exit 0 | ||
| fi | ||
| twine upload --verbose --repository-url http://maven.corp.mediav.com/nexus3/repository/pypi-host/ dist/* | ||
| } | ||
|
|
||
|
|
||
| start_upload(){ | ||
| _prepare_mamba_env | ||
| rm -rf dist/* || true | ||
| start_copy_libs | ||
| [[ $# > 0 ]] && export TN_VERSION=$1 | ||
| python setup.py sdist | ||
| start_only_upload | ||
| } | ||
|
|
||
| case "$1" in | ||
| (prepare_build_env) | ||
| _prepare_mamba_env | ||
| ;; | ||
| (build) | ||
| shift 1 | ||
| start_build "$@" | ||
| ;; | ||
| (only-build) | ||
| shift 1 | ||
| only_build "$@" | ||
| ;; | ||
| (deploy) | ||
| shfit 1 | ||
| start_upload "$@" | ||
| ;; | ||
| (copy-libs) | ||
| start_copy_libs | ||
| ;; | ||
| (help) | ||
| cmd=$(basename -- "$0") | ||
| cat <<-END | ||
| Usage: | ||
| $cmd help - Print this help. | ||
| $cmd prepare_build_env - install micromamba environment. | ||
| $cmd build [args..] - Build tn so file. | ||
| $cmd only-build [args..] - Build tn so file without config mpi | ||
| $cmd deploy - deploy tn to pypi | ||
| END | ||
| ;; | ||
| (*) die Unknown command "$1" ;; | ||
| esac |