Skip to content

Commit eb13366

Browse files
yongtangyongtang
authored
Split gs plugin out as an independent wheel (#1359)
1 parent fcb0757 commit eb13366

27 files changed

+305
-75
lines changed

‎.github/workflows/build.bazel.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# ==============================================================================
1616
set -e -x
1717

18-
export TENSORFLOW_INSTALL="$(python3 setup.py --package-version)"
18+
export TENSORFLOW_INSTALL="$(python3 setup.py --install-require)"
1919

2020
export BAZEL_OS=$(uname | tr '[:upper:]' '[:lower:]')
2121
curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
@@ -41,10 +41,11 @@ bazel build \
4141
--noshow_loading_progress \
4242
--verbose_failures \
4343
--test_output=errors \
44-
-- //tensorflow_io/...
44+
-- //tensorflow_io/... //tensorflow_io_plugin_gs/...
4545

4646
rm -rf build && mkdir -p build
4747

4848
cp -r bazel-bin/tensorflow_io build/tensorflow_io
49+
cp -r bazel-bin/tensorflow_io_plugin_gs build/tensorflow_io_plugin_gs
4950

5051
exit 0

‎.github/workflows/build.gpu.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@ export CUDNN_INSTALL_PATH="/usr/lib/x86_64-linux-gnu"
6969
python3 --version
7070
python3 -m pip --version
7171

72-
python3 -m pip install $(python3 setup.py --package-version)
72+
python3 -m pip install $(python3 setup.py --install-require)
7373
python3 tools/build/configure.py --cuda
7474

7575
cat .bazelrc

‎.github/workflows/build.wheel.sh

+1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ export TF_USE_MODULAR_FILESYSTEM=1
66
run_test() {
77
entry=$1
88
CPYTHON_VERSION=$($entry -c 'import sys; print(str(sys.version_info[0])+str(sys.version_info[1]))')
9+
(cd wheelhouse && $entry -m pip install tensorflow_io_plugin_gs-*-cp${CPYTHON_VERSION}-*.whl)
910
(cd wheelhouse && $entry -m pip install tensorflow_io-*-cp${CPYTHON_VERSION}-*.whl)
1011
$entry -m pip install -q pytest pytest-benchmark boto3 fastavro avro-python3 scikit-image pandas pyarrow==3.0.0 google-cloud-pubsub==2.1.0 google-cloud-bigtable==1.6.0 google-cloud-bigquery-storage==1.1.0 google-cloud-bigquery==2.3.1 google-cloud-storage==1.32.0 PyYAML==5.3.1
1112
(cd tests && $entry -m pytest --benchmark-disable -v --import-mode=append $(find . -type f \( -iname "test_*_v1.py" \)))

‎.github/workflows/build.yml

+54-33
Original file line numberDiff line numberDiff line change
@@ -111,7 +111,7 @@ jobs:
111111
BAZEL_VERSION=$(cat .bazelversion)
112112
curl -sSOL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
113113
sudo bash -e bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh
114-
sudo python3 -m pip install $(python3 setup.py --package-version)
114+
sudo python3 -m pip install $(python3 setup.py --install-require)
115115
python3 tools/build/configure.py
116116
bazel build \
117117
${BAZEL_OPTIMIZATION} \
@@ -120,11 +120,16 @@ jobs:
120120
--noshow_loading_progress \
121121
--verbose_failures \
122122
--test_output=errors \
123-
//tensorflow_io/...
124-
- uses: actions/upload-artifact@v1
123+
//tensorflow_io/... //tensorflow_io_plugin_gs/...
124+
mkdir build
125+
cp -r bazel-bin/tensorflow_io build
126+
cp -r bazel-bin/tensorflow_io_plugin_gs build
127+
- uses: actions/upload-artifact@v2
125128
with:
126129
name: ${{ runner.os }}-bazel-bin
127-
path: bazel-bin/tensorflow_io
130+
path: |
131+
build/tensorflow_io
132+
build/tensorflow_io_plugin_gs
128133
129134
macos-wheel:
130135
name: Wheel ${{ matrix.python }} macOS
@@ -138,7 +143,7 @@ jobs:
138143
- uses: actions/download-artifact@v1
139144
with:
140145
name: ${{ runner.os }}-bazel-bin
141-
path: bazel-bin/tensorflow_io
146+
path: bazel-bin
142147
- uses: actions/setup-python@v1
143148
with:
144149
python-version: ${{ matrix.python }}
@@ -147,7 +152,8 @@ jobs:
147152
set -x -e
148153
python -m pip install -U wheel setuptools
149154
python --version
150-
python setup.py --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64
155+
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64
156+
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64
151157
- name: Auditwheel ${{ matrix.python }} macOS
152158
run: |
153159
set -x -e
@@ -158,7 +164,7 @@ jobs:
158164
delocate-wheel -w wheelhouse $f
159165
done
160166
ls wheelhouse/*
161-
- uses: actions/upload-artifact@v1
167+
- uses: actions/upload-artifact@v2
162168
with:
163169
name: ${{ runner.os }}-${{ matrix.python }}-wheel
164170
path: wheelhouse
@@ -195,7 +201,8 @@ jobs:
195201
set -x -e
196202
python --version
197203
df -h
198-
(cd wheelhouse && python -m pip install *.whl)
204+
(cd wheelhouse && python -m pip install tensorflow_io_plugin_gs-*.whl)
205+
(cd wheelhouse && python -m pip install tensorflow_io-*.whl)
199206
- name: Test ${{ matrix.python }} macOS
200207
run: |
201208
set -x -e
@@ -225,11 +232,14 @@ jobs:
225232
gcr.io/tensorflow-testing/nosla-ubuntu16.04-manylinux2010@sha256:3a9b4820021801b1fa7d0592c1738483ac7abc209fc6ee8c9ef06cf2eab2d170 /v/.github/workflows/build.bazel.sh
226233
sudo chown -R $(id -nu):$(id -ng) .
227234
sudo find build/tensorflow_io -name '*runfiles*' | sudo xargs rm -rf
235+
sudo find build/tensorflow_io_plugin_gs -name '*runfiles*' | sudo xargs rm -rf
228236
sudo cp .bazelrc build/tensorflow_io/
229-
- uses: actions/upload-artifact@v1
237+
- uses: actions/upload-artifact@v2
230238
with:
231239
name: ${{ runner.os }}-bazel-bin
232-
path: build/tensorflow_io
240+
path: |
241+
build/tensorflow_io
242+
build/tensorflow_io_plugin_gs
233243
234244
linux-wheel:
235245
name: Wheel ${{ matrix.python }} Linux
@@ -243,12 +253,13 @@ jobs:
243253
- uses: actions/download-artifact@v1
244254
with:
245255
name: ${{ runner.os }}-bazel-bin
246-
path: bazel-bin/tensorflow_io
256+
path: bazel-bin
247257
- name: Wheel ${{ matrix.python }} Linux
248258
run: |
249259
set -x -e
250260
mv bazel-bin/tensorflow_io/.bazelrc .
251-
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --data bazel-bin -q bdist_wheel
261+
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel
262+
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel
252263
- name: Auditwheel ${{ matrix.python }} Linux
253264
run: |
254265
set -x -e
@@ -258,7 +269,7 @@ jobs:
258269
done
259270
sudo chown -R $(id -nu):$(id -ng) .
260271
ls wheelhouse/*
261-
- uses: actions/upload-artifact@v1
272+
- uses: actions/upload-artifact@v2
262273
with:
263274
name: ${{ runner.os }}-${{ matrix.python }}-wheel
264275
path: wheelhouse
@@ -334,14 +345,19 @@ jobs:
334345
python3 --version
335346
python3 -m pip install wheel setuptools
336347
python3 -m pip --version
337-
python3 setup.py --package-version | xargs python3 -m pip install
348+
python3 setup.py --install-require | xargs python3 -m pip install
338349
python3 tools/build/configure.py
339350
cat .bazelrc
340-
bazel build -s --verbose_failures %BAZEL_OPTIMIZATION% //tensorflow_io/core:python/ops/libtensorflow_io.so //tensorflow_io/core:python/ops/libtensorflow_io_plugins.so
341-
- uses: actions/upload-artifact@v1
351+
bazel build -s --verbose_failures %BAZEL_OPTIMIZATION% //tensorflow_io/core:python/ops/libtensorflow_io.so //tensorflow_io/core:python/ops/libtensorflow_io_plugins.so //tensorflow_io_plugin_gs/...
352+
mkdir build
353+
cp -r bazel-bin/tensorflow_io build
354+
cp -r bazel-bin/tensorflow_io_plugin_gs build
355+
- uses: actions/upload-artifact@v2
342356
with:
343357
name: ${{ runner.os }}-bazel-bin
344-
path: bazel-bin/tensorflow_io
358+
path: |
359+
build/tensorflow_io
360+
build/tensorflow_io_plugin_gs
345361
346362
windows-wheel:
347363
name: Wheel ${{ matrix.python }} Windows
@@ -355,7 +371,7 @@ jobs:
355371
- uses: actions/download-artifact@v1
356372
with:
357373
name: ${{ runner.os }}-bazel-bin
358-
path: bazel-bin/tensorflow_io
374+
path: bazel-bin
359375
- uses: actions/setup-python@v1
360376
with:
361377
python-version: ${{ matrix.python }}
@@ -365,9 +381,10 @@ jobs:
365381
@echo on
366382
python --version
367383
python -m pip install -U wheel setuptools
368-
python setup.py --data bazel-bin -q bdist_wheel
384+
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel
385+
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel
369386
ls -la dist
370-
- uses: actions/upload-artifact@v1
387+
- uses: actions/upload-artifact@v2
371388
with:
372389
name: ${{ runner.os }}-${{ matrix.python }}-wheel
373390
path: dist
@@ -401,14 +418,15 @@ jobs:
401418
run: |
402419
@echo on
403420
python --version
404-
(cd wheel && ls *.whl | xargs python -m pip install)
421+
(cd wheel && ls tensorflow_io_plugin_gs-*.whl | xargs python -m pip install && cd ..)
422+
(cd wheel && ls tensorflow_io-*.whl | xargs python -m pip install && cd ..)
405423
- name: Test ${{ matrix.python }} Windows
406424
shell: cmd
407425
run: |
408426
@echo on
409427
python --version
410428
python -m pip install -U pytest-benchmark
411-
rm -rf tensorflow_io
429+
rm -rf tensorflow_io tensorflow_io_plugin_gs
412430
(cd tests && python -m pytest -s -v test_lmdb.py)
413431
(python -m pytest -s -v test_image.py -k "webp or ppm or bmp or bounding or exif or hdr or openexr or tiff or avif")
414432
(python -m pytest -s -v test_serialization.py)
@@ -489,7 +507,7 @@ jobs:
489507
cp Windows-3.9-wheel/*.whl wheelhouse/
490508
ls -la wheelhouse/
491509
sha256sum wheelhouse/*.whl
492-
- uses: actions/upload-artifact@v1
510+
- uses: actions/upload-artifact@v2
493511
with:
494512
name: tensorflow-io-release
495513
path: wheelhouse
@@ -525,7 +543,7 @@ jobs:
525543
set -e -x
526544
BUILD_NUMBER=$(date "+%Y%m%d%H%M%S")
527545
echo ${BUILD_NUMBER} > BUILD_NUMBER
528-
- uses: actions/upload-artifact@v1
546+
- uses: actions/upload-artifact@v2
529547
with:
530548
name: BUILD_NUMBER
531549
path: BUILD_NUMBER
@@ -548,7 +566,7 @@ jobs:
548566
- uses: actions/download-artifact@v1
549567
with:
550568
name: ${{ runner.os }}-bazel-bin
551-
path: bazel-bin/tensorflow_io
569+
path: bazel-bin
552570
- uses: actions/setup-python@v1
553571
with:
554572
python-version: ${{ matrix.python }}
@@ -557,7 +575,8 @@ jobs:
557575
set -x -e
558576
python -m pip install -U wheel setuptools
559577
python --version
560-
python setup.py --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64 --nightly $BUILD_NUMBER
578+
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64 --nightly $BUILD_NUMBER
579+
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --plat-name macosx_10_14_x86_64 --nightly $BUILD_NUMBER
561580
- name: Auditwheel ${{ matrix.python }} macOS
562581
run: |
563582
set -x -e
@@ -568,7 +587,7 @@ jobs:
568587
delocate-wheel -w wheelhouse $f
569588
done
570589
ls wheelhouse/*
571-
- uses: actions/upload-artifact@v1
590+
- uses: actions/upload-artifact@v2
572591
with:
573592
name: ${{ runner.os }}-${{ matrix.python }}-nightly
574593
path: wheelhouse
@@ -591,12 +610,13 @@ jobs:
591610
- uses: actions/download-artifact@v1
592611
with:
593612
name: ${{ runner.os }}-bazel-bin
594-
path: bazel-bin/tensorflow_io
613+
path: bazel-bin
595614
- name: Wheel ${{ matrix.python }} Linux
596615
run: |
597616
set -x -e
598617
mv bazel-bin/tensorflow_io/.bazelrc .
599-
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --data bazel-bin -q bdist_wheel --nightly $BUILD_NUMBER
618+
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --nightly $BUILD_NUMBER
619+
docker run -i --rm --user $(id -u):$(id -g) -v /etc/password:/etc/password -v $PWD:/v -w /v --net=host python:${{ matrix.python }}-slim python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --nightly $BUILD_NUMBER
600620
- name: Auditwheel ${{ matrix.python }} Linux
601621
run: |
602622
set -x -e
@@ -606,7 +626,7 @@ jobs:
606626
done
607627
sudo chown -R $(id -nu):$(id -ng) .
608628
ls wheelhouse/*
609-
- uses: actions/upload-artifact@v1
629+
- uses: actions/upload-artifact@v2
610630
with:
611631
name: ${{ runner.os }}-${{ matrix.python }}-nightly
612632
path: wheelhouse
@@ -629,7 +649,7 @@ jobs:
629649
- uses: actions/download-artifact@v1
630650
with:
631651
name: ${{ runner.os }}-bazel-bin
632-
path: bazel-bin/tensorflow_io
652+
path: bazel-bin
633653
- uses: actions/setup-python@v1
634654
with:
635655
python-version: ${{ matrix.python }}
@@ -639,9 +659,10 @@ jobs:
639659
@echo on
640660
python --version
641661
python -m pip install -U wheel setuptools
642-
python setup.py --data bazel-bin -q bdist_wheel --nightly %BUILD_NUMBER%
662+
python setup.py --project tensorflow-io --data bazel-bin -q bdist_wheel --nightly %BUILD_NUMBER%
663+
python setup.py --project tensorflow-io-plugin-gs --data bazel-bin -q bdist_wheel --nightly %BUILD_NUMBER%
643664
ls -la dist
644-
- uses: actions/upload-artifact@v1
665+
- uses: actions/upload-artifact@v2
645666
with:
646667
name: ${{ runner.os }}-${{ matrix.python }}-nightly
647668
path: dist

‎configure.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@ fi
2424
if $PYTHON -c "import tensorflow as tf" &> /dev/null; then
2525
echo 'using installed tensorflow'
2626
else
27-
$PYTHON -m pip install $($PYTHON setup.py --package-version)
27+
$PYTHON -m pip install $($PYTHON setup.py --install-require)
2828
fi
2929
$PYTHON tools/build/configure.py

‎docs/development.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ sudo ./configure.sh
8282
# export BAZEL_OPTIMIZATION=
8383

8484
# Build shared libraries
85-
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/...
85+
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_plugin_gs/...
8686

8787
# Once build is complete, shared libraries will be available in
8888
# `bazel-bin/tensorflow_io/core/python/ops/` and it is possible
@@ -141,7 +141,7 @@ sudo ./configure.sh
141141
# export BAZEL_OPTIMIZATION=
142142

143143
# Build shared libraries
144-
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/...
144+
bazel build -s --verbose_failures $BAZEL_OPTIMIZATION //tensorflow_io/... //tensorflow_io_plugin_gs/...
145145

146146
# Once build is complete, shared libraries will be available in
147147
# `bazel-bin/tensorflow_io/core/python/ops/` and it is possible

0 commit comments

Comments
 (0)