Skip to content

Commit eab9a50

Browse files
mckaymattmckaymatt
committed
use manylinux container
1 parent 854f817 commit eab9a50

10 files changed

+203
-63
lines changed

‎.travis.yml

+66-35
Original file line numberDiff line numberDiff line change
@@ -13,56 +13,77 @@ env:
1313
- CRATE_NAME=trust_pypi_example
1414

1515
matrix:
16-
# TODO These are all the build jobs. Adjust as necessary. Comment out what you
17-
# don't need
1816
include:
19-
# Linux
20-
# - env: TARGET=i686-unknown-linux-gnu PYENV=2.7.13
21-
# - env: TARGET=i686-unknown-linux-musl PYENV=2.7.13
22-
# - env: TARGET=x86_64-unknown-linux-gnu PYENV=2.7.13
23-
- env: TARGET=x86_64-unknown-linux-musl PYENV=2.7.13
17+
# TARGET is the rust target triplet
18+
# WHEELPLATFORM is the wheel platform target
19+
# PYENV is the Python version to build against. Not relevent if using "manylinux"
20+
2421

25-
# - env: TARGET=i686-unknown-linux-gnu PYENV=3.6.1
26-
# - env: TARGET=i686-unknown-linux-musl PYENV=3.6.1
27-
- env: TARGET=x86_64-unknown-linux-gnu PYENV=3.6.1
28-
# - env: TARGET=x86_64-unknown-linux-musl PYENV=3.6.1
22+
# Linux
23+
# we are using Rustup instead of cross when WHEELPLATFORM="*manylinux*". This is because both
24+
# manylinux and Cross depend on Docker but the shared libs in the Cross contains are too new
25+
# to make valid manylinux containers.
26+
#
27+
# You can lean more about the differnt Rustup toolchains here:
28+
# https://github.com/rust-lang-nursery/rustup.rs#toolchain-specification
29+
#
30+
# More on manylinuyx at https://github.com/pypa/manylinux
31+
# and the pep https://www.python.org/dev/peps/pep-0513
32+
#
33+
# one other thing about manylinux. Since the container contains several versions of python,
34+
# we build/test against all of them at the same time. I have it set up to run against
35+
# cp27-cp27m, cp27-cp27mu, cp33-cp33m, cp34-cp34m, cp35-cp35m, cp36-cp36m
36+
# PYENV versions isn't relevent here so just keep it at 3.6.1
37+
- env: TARGET=i686-unknown-linux-gnu PYENV=3.6.1 WHEELPLATFORM=manylinux1_i686
38+
- env: TARGET=stable-x86_64-unknown-linux-gnu PYENV=3.6.1 WHEELPLATFORM=manylinux1_x86_64
39+
40+
# Note that musl isn't covered by manylinux and I had trouble building
41+
# dylibs with it as well
42+
# - env: TARGET=i686-unknown-linux-musl PYENV=2.7.13 WHEELPLATFORM=
43+
# - env: TARGET=i686-unknown-linux-musl PYENV=3.6.1 WHEELPLATFORM=
44+
# - env: TARGET=x86_64-unknown-linux-musl PYENV=2.7.13 WHEELPLATFORM=
45+
# - env: TARGET=x86_64-unknown-linux-musl PYENV=3.6.1 WHEELPLATFORM=
2946

3047
# OSX
31-
- env: TARGET=i686-apple-darwin PYENV=2.7.13
48+
# See versions https://docs.travis-ci.com/user/osx-ci-environment/#OS-X-Version
49+
# current default is osx_image: xcode7.3 OS X 10.11
50+
#
51+
- env: TARGET=x86_64-apple-darwin PYENV=2.7.13 WHEELPLATFORM=macosx_10_11_x86_64
3252
os: osx
33-
# - env: TARGET=x86_64-apple-darwin PYENV=2.7.13
53+
osx_image: xcode7.3
54+
- env: TARGET=x86_64-apple-darwin PYENV=3.6.1 WHEELPLATFORM=macosx_10_10_x86_64
55+
os: osx
56+
osx_image: xcode6.4
57+
# - env: TARGET=i686-apple-darwin PYENV=2.7.13 WHEELPLATFORM=
3458
# os: osx
35-
36-
# - env: TARGET=i686-apple-darwin PYENV=3.6.1
59+
# - env: TARGET=i686-apple-darwin PYENV=3.6.1 WHEELPLATFORM=
3760
# os: osx
38-
- env: TARGET=x86_64-apple-darwin PYENV=3.6.1
39-
os: osx
40-
4161

4262
# # *BSD
43-
# - env: TARGET=i686-unknown-freebsd DISABLE_TESTS=1
44-
# - env: TARGET=x86_64-unknown-freebsd DISABLE_TESTS=1
45-
# - env: TARGET=x86_64-unknown-netbsd DISABLE_TESTS=1
63+
# - env: TARGET=i686-unknown-freebsd DISABLE_TESTS=1 WHEELPLATFORM=
64+
# - env: TARGET=x86_64-unknown-freebsd DISABLE_TESTS=1 WHEELPLATFORM=
65+
# - env: TARGET=x86_64-unknown-netbsd DISABLE_TESTS=1 WHEELPLATFORM=
4666

4767
# # Other architectures
48-
# - env: TARGET=aarch64-unknown-linux-gnu
49-
# - env: TARGET=armv7-unknown-linux-gnueabihf
50-
# - env: TARGET=mips-unknown-linux-gnu
51-
# - env: TARGET=mips64-unknown-linux-gnuabi64
52-
# - env: TARGET=mips64el-unknown-linux-gnuabi64
53-
# - env: TARGET=mipsel-unknown-linux-gnu
54-
# - env: TARGET=powerpc-unknown-linux-gnu
55-
# - env: TARGET=powerpc64-unknown-linux-gnu
56-
# - env: TARGET=powerpc64le-unknown-linux-gnu
57-
# - env: TARGET=s390x-unknown-linux-gnu DISABLE_TESTS=1
68+
# - env: TARGET=aarch64-unknown-linux-gnu WHEELPLATFORM=
69+
# - env: TARGET=armv7-unknown-linux-gnueabihf WHEELPLATFORM=
70+
# - env: TARGET=mips-unknown-linux-gnu WHEELPLATFORM=
71+
# - env: TARGET=mips64-unknown-linux-gnuabi64 WHEELPLATFORM=
72+
# - env: TARGET=mips64el-unknown-linux-gnuabi64 WHEELPLATFORM=
73+
# - env: TARGET=mipsel-unknown-linux-gnu WHEELPLATFORM=
74+
# - env: TARGET=powerpc-unknown-linux-gnu WHEELPLATFORM=
75+
# - env: TARGET=powerpc64-unknown-linux-gnu WHEELPLATFORM=
76+
# - env: TARGET=powerpc64le-unknown-linux-gnu WHEELPLATFORM=
77+
# - env: TARGET=s390x-unknown-linux-gnu DISABLE_TESTS=1 WHEELPLATFORM=
5878

5979
# # Testing other channels
60-
# - env: TARGET=x86_64-unknown-linux-gnu
80+
# - env: TARGET=x86_64-unknown-linux-gnu WHEELPLATFORM=
6181
# rust: nightly
62-
# - env: TARGET=x86_64-apple-darwin
82+
# - env: TARGET=x86_64-apple-darwin WHEELPLATFORM=
6383
# os: osx
6484
# rust: nightly
6585

86+
6687
before_install: set -e
6788

6889
install:
@@ -98,7 +119,7 @@ deploy:
98119
api_key:
99120
secure: "K4ttQbVILc4kq90ZiHOydhIwm09zHxfwUMnxwnBGMkAFdRL8oSjfXNd0O1Q/OA5XUVaFkp9Uluc6oy3AywOyjsLLu9xRm1qpgxMIak0nKGVQ8fzMrFILcTU3mXsOFX6VEfLqIR0bbmV31p//0Ytn6lH5PdOPe8Jm9AGHSIlrv0+Adzn01CZGgTZFi55p4hrzlP7zQhPliJFhtVd8dYga65t4aubEK7ZVQ030Z1pida/IaMyWQwbD63h1DrmyFBdnTl1gnk0fluWCeVnXQHgld0/P4QKZCPnPcVpn1eOdhju7L9r8Tc+AVirvv5jkCLFBtFDAinTkv2LUu8RsSKgD5Jzg8oYVXdebCpeXQ7f+3JoqyunF8NncHc4dszylNRGuJeMpFy5QTYqPkPDNQczMY9GMbsshzlDp8rKPGMPXpwwh+EeiKijm77eVZJ4UqSmoAt9yLe+Dq8I7ZB1GPUHBaVAlesWvCJ9FfzvDXbD0Piv85Wb0BCAOLuYouCtPeSaD69DeS+Y1rzEt5tSwWx3s8uTV7WPEVlROjlW2TB1ELIUQBETCw/yqEkfVm25XrQSpVlzaHxfh7RuEVyi5/Aj/q/yjeHeJYU+qN53FWWNKazeZJUeqwyDwx3Cjx09ACzKEfcmvcZiwv2mGZxhHqM3VARWvvpDDVcOmh9vQhErZzic="
100121
file_glob: true
101-
file: dist/*.whl
122+
file: wheelhouse/trust_pypi_example*.whl
102123
# file: $CRATE_NAME-$TRAVIS_TAG-$TARGET.*
103124
on:
104125
# TODO Here you can pick which targets will generate binary releases
@@ -114,13 +135,23 @@ before_cache:
114135
- chmod -R a+r $HOME/.cache/pip
115136
- chmod -R a+r $HOME/.cache/pyenv
116137
- chmod -R a+r $HOME/.cargo
138+
- chmod -R a+r $HOME/.rustup
139+
- chmod -R a+r $HOME/.manylinux_pip_cache
140+
- chmod -R a+r $HOME/.manylinux_rustup_cache
141+
- chmod -R a+r $HOME/.manylinux_cargo_cache
142+
117143

118144
cache:
119145
# There is a "cache: cargo" option but it's fails if target/ isn't in the build dir
120146
directories:
121147
- $HOME/.cache/pip
122148
- $HOME/.cache/pyenv
123-
- $HOME/.cargo ## this may not be worth it depending on the time it takes to push/pull from S3
149+
- $HOME/.cargo
150+
- $HOME/.rustup
151+
- $HOME/.manylinux_pip_cache
152+
- $HOME/.manylinux_rustup_cache
153+
- $HOME/.manylinux_cargo_cache
154+
124155

125156
branches:
126157
only:

‎HISTORY.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
History
33
=======
44

5-
0.1.0 (2017-05-11)
5+
0.1.0 (2017-05-16)
66
------------------
77

88
* First release on PyPI.

‎MANIFEST.in

+1-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ include HISTORY.rst
66
include LICENSE
77
include README.rst
88

9-
recursive-include tests *
9+
recursive-include trust_pypi_example/tests *
1010
recursive-exclude * __pycache__
1111
recursive-exclude * *.py[co]
1212

‎Makefile

+2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
.DEFAULT_GOAL := help
33
define BROWSER_PYSCRIPT
44
import os, webbrowser, sys
5+
if os.environ.get('TRAVIS_BUILD_NUMBER'):
6+
exit(0)
57
try:
68
from urllib import pathname2url
79
except:

‎ci/install.sh

+8-2
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,19 @@
11
set -ex
22

33
main() {
4-
local target=
4+
if [[ $WHEELPLATFORM == *"manylinux"* ]]; then
5+
echo "Manylinx does not require Cross - WHEELPLATFORM=$WHEELPLATFORM"
6+
return
7+
fi
58
if [ ! -z ${SKIPCROSS+x} ]; then
69
# This is for local testing. It skips the install.
710
echo "SKIPCROSS is set. This must be a local test"
811
return
9-
elif [ $TRAVIS_OS_NAME = linux ]; then
12+
fi
13+
local target=
14+
if [ $TRAVIS_OS_NAME = linux ]; then
1015
target=x86_64-unknown-linux-musl
16+
# target=x86_64-unknown-linux-gnu
1117
sort=sort
1218
else
1319
target=x86_64-apple-darwin

‎ci/manylinux_build_wheel.sh

+87
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/bin/bash
2+
# Runs in the manylinx Docker container
3+
# https://github.com/pypa/manylinux
4+
# https://www.python.org/dev/peps/pep-0513
5+
6+
set -e -x
7+
8+
function RETURNOWNERSHIP {
9+
# We need to change perms back to user in the end since we run as root in the manylinux container
10+
chown -Rf --reference=/io/setup.py /io /root/.cache/pip /root/.cargo /root/.rustup
11+
}
12+
trap RETURNOWNERSHIP EXIT
13+
14+
export USER='root'
15+
chown -fR root:root /root/.cache/pip /root/.cargo /root/.rustup
16+
17+
function rust_build() {
18+
# We build our Rust lib here so it will be compatible with old versions of common shared libs
19+
20+
# install rustup inside the manylinux container
21+
curl https://sh.rustup.rs -sSf > /tmp/rustup.sh
22+
sh /tmp/rustup.sh -y
23+
source $HOME/.cargo/env
24+
rustup install "nightly-$TARGET"
25+
rustup default "nightly-$TARGET"
26+
rustup update
27+
28+
cd /io/trust_pypi_example/rust/
29+
cargo build --target $TARGET --release
30+
31+
# disable tests
32+
if [ ! -z $DISABLE_TESTS ]; then
33+
echo "Rust Tests Disabled"
34+
else
35+
cargo test --target $TARGET --release
36+
fi
37+
# uncomment if creating a bin
38+
# cargo run --target $TARGET
39+
# cargo run --target $TARGET --release
40+
41+
if [ ! -d "target/$TARGET/release" ]; then
42+
echo "Cannot find release dir at target/$TARGET/release"
43+
exit 2
44+
else
45+
pushd target
46+
mv release release.bak || true
47+
cp -r $TARGET/release release
48+
popd
49+
fi
50+
51+
}
52+
53+
function manylinux_test_and_bundle() {
54+
cd /io
55+
rm -rf /io/wheelhouse
56+
57+
# Skips 2.6
58+
for PYBIN in /opt/python/{cp27*,cp3*}/bin/; do
59+
"${PYBIN}/pip" install -U pip virtualenv
60+
"${PYBIN}/virtualenv" /tmp/$PYBIN
61+
source "/tmp${PYBIN}/bin/activate"
62+
pip install -q -U pip
63+
pip install -q -r /io/requirements_dev.txt
64+
pip wheel /io/ -w /io/wheelhouse/
65+
deactivate
66+
done
67+
68+
# Bundle external shared libraries into the wheels
69+
# for whl in /io/wheelhouse/*.whl; do
70+
# auditwheel show "$whl"
71+
# auditwheel repair "$whl" -w /io/wheelhouse/
72+
# done
73+
74+
# Install packages and test
75+
for PYBIN in /opt/python/{cp27*,cp3*}/bin/; do
76+
source "/tmp${PYBIN}/bin/activate"
77+
"${PYBIN}/pip" install trust_pypi_example --no-index -f /io/wheelhouse
78+
py.test /io/tests/
79+
80+
deactivate
81+
done
82+
}
83+
84+
85+
rust_build
86+
manylinux_test_and_bundle
87+
RETURNOWNERSHIP

‎ci/pyinstall.sh

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
set -ex
2-
2+
# Set up pyinstall and the virtualenv
33
PATH="$HOME/.pyenv/shims:$HOME/.pyenv/bin:$PATH"
44
pyenv --version || true
55
if [ -d "$HOME/.pyenv/.git" ]; then
@@ -39,5 +39,5 @@ pip install -U pip || true
3939
pip install -U virtualenv || true
4040
python -m venv .venv || python -m virtualenv .venv
4141
source .venv/bin/activate
42-
pip install -U pip
43-
pip install -r requirements_dev.txt
42+
pip install -q -U pip
43+
pip install -q -r requirements_dev.txt

‎ci/script.sh

+32-18
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,30 @@
22

33
set -ex
44

5-
main() {
6-
# move into rust project subdir
5+
manylinux_linux() {
6+
# cache pip packages used in docker
7+
mkdir -p "$HOME/.manylinux_pip_cache"
8+
mkdir -p "$HOME/.manylinux_cargo_cache"
9+
mkdir -p "$HOME/.manylinux_rustup_cache"
10+
11+
CACHES="-v $HOME/.manylinux_pip_cache:/root/.cache/pip \
12+
-v $HOME/.manylinux_cargo_cache:/root/.cargo \
13+
-v $HOME/.manylinux_rustup_cache:/root/.rustup "
14+
# SUB='-v /home/matt/workspace/auditwheel/auditwheel/wheeltools.py:/opt/python/cp36-cp36m/lib/python3.6/site-packages/auditwheel/wheeltools.py'
15+
docker run --rm -e TARGET="${TARGET}" $CACHES -v `pwd`:/io quay.io/pypa/manylinux1_x86_64 /io/ci/manylinux_build_wheel.sh
16+
17+
}
18+
19+
pyenv_build_test_bundle() {
20+
# This uses cross and Pyenv
21+
722
pushd trust_pypi_example/rust/
8-
cross build --target $TARGET
923
cross build --target $TARGET --release
1024

1125
# disable tests
1226
if [ ! -z $DISABLE_TESTS ]; then
1327
echo "Rust Tests Disabled"
1428
else
15-
cross test --target $TARGET
1629
cross test --target $TARGET --release
1730
fi
1831
# uncomment if creating a bin
@@ -28,22 +41,22 @@ main() {
2841
else
2942
pushd target
3043
mv release release.bak || true
31-
ln -s $TARGET/release release
44+
cp -r $TARGET/release release
3245
popd
3346
fi
3447
popd
35-
}
36-
37-
pymain() {
48+
rm -rf wheelhouse
49+
mkdir -p wheelhouse
3850
source .venv/bin/activate
39-
python setup.py bdist_wheel
51+
pip install -q -r requirements_dev.txt
52+
python setup.py bdist_wheel --plat-name="$WHEELPLATFORM"
53+
cp dist/*.whl wheelhouse
54+
pip install wheelhouse/*.whl
55+
py.test tests/
56+
4057

41-
make test
42-
make lint
43-
make coverage
44-
make docs
45-
}
4658

59+
}
4760

4861
if [ -z ${TARGET+x} ]; then
4962
if [ ! -z ${TRAVIS_BUILD_NUMBER+x} ]; then
@@ -59,7 +72,8 @@ else
5972
fi
6073

6174

62-
63-
64-
main
65-
pymain
75+
if [ $TRAVIS_OS_NAME = "osx" ]; then
76+
pyenv_build_test_bundle
77+
else
78+
manylinux_linux
79+
fi

‎requirements_dev.txt

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,11 @@
11
pip>=8.1.2
2+
cffi>=1.0.0
23
bumpversion>=0.5.3
34
wheel==0.29.0
45
watchdog>=0.8.3
56
flake8>=2.6.0
67
tox>=2.3.1
78
coverage>=4.1
89
Sphinx>=1.4.8
9-
cryptography>=1.7
10-
PyYAML>=3.11
1110
pytest>=2.9.2
1211
Click>=6.0
13-
cffi>=1.0.0

‎requirements_travis.txt

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
cryptography>=1.7
2+
PyYAML>=3.11

0 commit comments

Comments
 (0)