Add the example code of how to accelerate the data sharing with vineyard on fluid platform. #5868

Workflow file for this run

.github/workflows/build-test.yml at 0659dfe

	# Copyright 2020-2023 Alibaba Group Holding Limited.
	#
	# Licensed under the Apache License, Version 2.0 (the "License");
	# you may not use this file except in compliance with the License.
	# You may obtain a copy of the License at
	#
	# http://www.apache.org/licenses/LICENSE-2.0
	#
	# Unless required by applicable law or agreed to in writing, software
	# distributed under the License is distributed on an "AS IS" BASIS,
	# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	# See the License for the specific language governing permissions and
	# limitations under the License.

	name: Vineyard CI

	on:
	push:
	paths-ignore:
	- 'docs/'
	- '*/.rst'
	- '*/.md'
	- '*/.txt'
	pull_request:
	paths-ignore:
	- 'docs/'
	- '*/.rst'
	- '*/.md'
	- '*/.txt'
	workflow_dispatch:

	concurrency:
	group: ${{ github.repository }}-${{ github.event.number \|\| github.head_ref \|\| github.sha }}-${{ github.workflow }}
	cancel-in-progress: true

	env:
	CMAKE_C_COMPILER_LAUNCHER: ccache
	CMAKE_CXX_COMPILER_LAUNCHER: ccache

	jobs:
	ci:
	runs-on: ${{ matrix.os }}
	if: ${{ github.repository == 'v6d-io/v6d' }}
	strategy:
	matrix:
	os: [ubuntu-20.04]
	metadata: [local, etcd, redis] # disable redis for now as it seems has some nondeterministic bugs
	exclude:
	- metadata: local
	- metadata: redis
	env:
	RUNNER_ARGS: "--meta=${{ matrix.metadata }}"
	steps:
	- uses: actions/checkout@v3
	with:
	submodules: true

	- name: Free disk space
	if: false
	run: \|
	bash .github/free-disk-space.sh

	- uses: dorny/paths-filter@v2
	id: changes
	with:
	filters: \|
	mimalloc:
	- 'modules/malloc/**'
	- 'src/common/memory/**'
	- 'src/server/memory/**'
	- 'thirdparty/mimalloc/**'

	- name: Generate Summary for Submodules
	run: \|
	git submodule > git-modules.txt
	cat git-modules.txt

	- name: Cache for ccache
	uses: actions/cache@v3
	with:
	path: ~/.ccache
	key: ${{ runner.os }}-${{ matrix.metadata }}-ccache-${{ hashFiles('**/git-modules.txt') }}
	restore-keys: \|
	${{ runner.os }}-${{ matrix.metadata }}-ccache-

	- name: Install Dependencies for Linux
	if: runner.os == 'Linux'
	run: \|
	export PATH=/usr/lib/ccache:$PATH

	sudo apt update -y
	sudo apt install -y ca-certificates \
	ccache \
	cmake \
	doxygen \
	fuse3 \
	libboost-all-dev \
	libcurl4-openssl-dev \
	libfuse3-dev \
	libgflags-dev \
	libgoogle-glog-dev \
	libgmock-dev \
	libgrpc-dev \
	libgrpc++-dev \
	libkrb5-dev \
	libmpich-dev \
	libprotobuf-dev \
	librdkafka-dev \
	libgsasl7-dev \
	librdkafka-dev \
	libssl-dev \
	libunwind-dev \
	libuv1-dev \
	libxml2-dev \
	libz-dev \
	lsb-release \
	redis-server \
	pandoc \
	protobuf-compiler-grpc \
	python3-pip \
	uuid-dev \
	wget

	# install apache-arrow
	wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short \| tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
	sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
	sudo apt update
	sudo apt install -y libarrow-dev=14.0.1-1 \
	libarrow-dataset-dev=14.0.1-1 \
	libarrow-acero-dev=14.0.1-1 \
	libarrow-flight-dev=14.0.1-1 \
	libgandiva-dev=14.0.1-1 \
	libparquet-dev=14.0.1-1

	# install deps for java
	sudo apt install -y default-jdk-headless maven

	# install clang-format
	sudo curl -L https://github.com/muttleyxd/clang-tools-static-binaries/releases/download/master-1d7ec53d/clang-format-11_linux-amd64 --output /usr/bin/clang-format
	sudo chmod +x /usr/bin/clang-format

	# avoid possible permission errors
	ccache -o cache_dir=~/.ccache
	sudo mkdir -p ~/.ccache
	sudo chmod -R a+wrx ~/.ccache

	# install python packages for codegen, and io adaptors
	sudo pip3 install -U "Pygments>=2.4.1"
	sudo pip3 install -r requirements-setup.txt -r requirements.txt -r requirements-dev.txt

	# install linters
	sudo pip3 install auditwheel black isort flake8 twine

	- name: Setup tmate session
	if: false
	uses: mxschmitt/action-tmate@v3

	- name: Setup SSH environment
	run: \|
	# avoid access node by DNS hostnames
	r=`cat /etc/hosts \| grep $(hostname) \|\| true`
	if [ -z "${r}" ];then export hn=$(hostname); sudo -E bash -c 'echo "127.0.0.1 ${hn}" >> /etc/hosts'; fi
	cat /etc/hosts

	# setup SSH server to allow login without password
	cat > sshd_config <<EOF
	SyslogFacility AUTHPRIV
	PermitRootLogin yes
	AuthorizedKeysFile .ssh/authorized_keys
	PasswordAuthentication yes
	ChallengeResponseAuthentication no
	UsePAM yes
	UseDNS no
	X11Forwarding no
	PrintMotd no
	EOF
	sudo mv sshd_config /etc/ssh/sshd_config
	sudo systemctl restart ssh

	# generate proper SSH key
	cat /dev/zero \| ssh-keygen -t rsa -b 4096 -N '' -f ~/.ssh/id_rsa

	# add key to authorized_keys file
	cat ~/.ssh/id_rsa.pub \| tee -a ~/.ssh/authorized_keys

	# ensure the permissions of SSH files
	chmod 600 ~/.ssh/authorized_keys
	chmod 700 ~/.ssh
	sudo chmod -c 0755 ~/
	ls -la ~/.ssh

	# test SSH login, without password
	ssh -vvv -i ~/.ssh/id_rsa -o BatchMode=yes -o StrictHostKeyChecking=no $(whoami)@localhost

	- name: Setup tmate session
	if: false
	uses: mxschmitt/action-tmate@v3

	- name: CMake
	run: \|
	export PATH=/usr/lib/ccache:$PATH

	mkdir build
	cd build
	cmake .. -DCMAKE_BUILD_TYPE=Debug \
	-DBUILD_SHARED_LIBS=ON \
	-DUSE_ASAN=ON \
	-DBUILD_VINEYARD_COVERAGE=ON \
	-DBUILD_VINEYARD_PYTHON_BINDINGS=ON \
	-DBUILD_VINEYARD_BASIC=ON \
	-DBUILD_VINEYARD_IO=ON \
	-DBUILD_VINEYARD_IO_KAFKA=ON \
	-DBUILD_VINEYARD_MALLOC=ON \
	-DBUILD_VINEYARD_HOSSEINMOEIN_DATAFRAME=ON \
	-DBUILD_VINEYARD_FUSE=ON \
	-DBUILD_VINEYARD_FUSE_PARQUET=ON \
	-DBUILD_VINEYARD_GRAPH=OFF \
	-DBUILD_VINEYARD_GRAPH_WITH_GAR=OFF \
	-DBUILD_VINEYARD_TESTS_ALL=ON

	if [ "${{ matrix.metadata }}" == "redis" ]; then
	cmake .. -DBUILD_VINEYARD_SERVER_REDIS=ON
	fi

	- name: Cpp format and lint
	run: \|
	cd build

	function prepend() { while read line; do echo "${1}${line}"; done; }

	make vineyard_clformat
	GIT_DIFF=$(git diff --ignore-submodules)
	if [[ -n $GIT_DIFF ]]; then
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo "\| clang-format failures found!"
	echo "\|"
	echo "$GIT_DIFF" \| prepend "\| "
	echo "\|"
	echo "\| Run: "
	echo "\|"
	echo "\| make vineyard_clformat"
	echo "\|"
	echo "\| to fix this error."
	echo "\|"
	echo "\| Ensure you are working with clang-format-11, which can be obtained from"
	echo "\|"
	echo "\| https://github.com/muttleyxd/clang-tools-static-binaries/releases "
	echo "\|"
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	exit -1
	fi

	function ec() { [[ "$1" == "-h" ]] && { shift && eval $* > /dev/null 2>&1; ec=$?; echo $ec; } \|\| eval $*; ec=$?; }

	ec make vineyard_cpplint
	if [[ "$ec" != "0" ]]; then
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo "\| cpplint failures found! Run: "
	echo "\|"
	echo "\| make vineyard_cpplint"
	echo "\|"
	echo "\| to fix this error."
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	exit -1
	fi

	- name: Python format and lint
	run: \|
	function prepend() { while read line; do echo "${1}${line}"; done; }

	python3 setup.py lint --inplace=true
	GIT_DIFF=$(git diff --ignore-submodules)
	if [[ -n $GIT_DIFF ]]; then
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	echo "\| black (python format) failures found!"
	echo "\|"
	echo "$GIT_DIFF" \| prepend "\| "
	echo "\|"
	echo "\|Run: "
	echo "\|"
	echo "\| python3 setup.py lint --inplace=true"
	echo "\|"
	echo "\| to fix this error."
	echo "\|"
	echo "\| The tool black can be installed by"
	echo "\|"
	echo "\| pip3 install black isort flake8"
	echo "\|"
	echo "~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"
	exit -1
	fi

	- name: Build
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64

	pushd build
	make -j`nproc`
	make vineyard_tests -j`nproc`
	make vineyard_client_python -j`nproc`
	make ccache-stats
	sudo make install
	popd

	# not used anymore
	rm -rf build/lib*

	# build & install vineyard for python
	rm -rf build/lib* build/bdist.*
	python3 setup.py bdist_wheel

	for pkg in airflow bdist dask pyspark io kedro ml; do
	rm -rf build/lib* build/bdist.*
	python3 setup_${pkg}.py bdist_wheel
	done

	# install python packages
	twine check dist/*.whl
	sudo pip3 install dist/*.whl -U

	# don't upload incomplete vineyard_bdist to release
	rm -rf dist/vineyard_bdist* \|\| true

	- name: Get test data
	run: \|
	git clone https://github.com/GraphScope/gstest.git --depth=1

	- name: Setup tmate session
	if: false
	uses: mxschmitt/action-tmate@v3

	- name: Run cpp tests
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-cpp

	- name: Run python tests
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-python

	- name: Run deployment tests
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-deployment --with-migration

	- name: Run contrib/thirdparty tests
	if: false # lack of heavy dependencies like torch/tensorflow in CI
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"
	export VINEYARD_SPARK_PATH=`pwd`/java/spark/target/vineyard-spark_2.12-0.1-SNAPSHOT-jar-with-dependencies.jar

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-contrib

	- name: Run I/O tests
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-io

	- name: Run FUSE tests
	run: \|
	export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/lib64:/usr/local/lib/x86_64-linux-gnu
	export VINEYARD_DATA_DIR=`pwd`/gstest
	export TMPDIR="${TMPDIR:-$(dirname $(mktemp))}"

	rm -rf default.etcd
	rm -rf /dev/shm/etcd*
	python3 test/runner.py $RUNNER_ARGS --with-fuse

	- name: Run CMake tests
	run: \|
	cmake -S test/vineyard-cmake-example -B build/vineyard-cmake-example
	cmake --build build/vineyard-cmake-example

	- name: Upload Coverage
	if: runner.os == 'Linux'
	uses: codecov/codecov-action@v3
	with:
	verbose: true

	- name: Extract tag name
	id: tag
	if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') }}
	run: echo "TAG=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT

	- name: Upload vineyard contrib wheels to tagged release
	uses: svenstaro/upload-release-action@v2
	if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'v6d-io/v6d' }}
	with:
	repo_token: ${{ secrets.GITHUB_TOKEN }}
	file: dist/vineyard.whl
	file_glob: true
	tag: ${{ steps.tag.outputs.TAG }}
	prerelease: false
	overwrite: true
	body: "vineyard-contrib ${{ steps.tag.outputs.TAG }}"

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Add the example code of how to accelerate the data sharing with vineyard on fluid platform. #5868

Workflow file

Add the example code of how to accelerate the data sharing with vineyard on fluid platform. #5868

Jobs

Run details

Workflow file for this run