[wip] Wheel builder and all new versions

.gitignore

@@ -3,3 +3,5 @@ data
 logs
 docker-compose-aws-s3-storage-test.yml
 docker-compose-aws-s3-loader-test.yml
+wheelhouse
+test.yml

.travis.yml

@@ -4,10 +4,7 @@ services:
 - docker
 script:
 - ls -lah
-- docker build -f thumbor/Dockerfile -t apsl/thumbor thumbor/
-- docker build -f thumbor-multiprocess/Dockerfile -t apsl/thumbor-multiprocess thumbor-multiprocess/
-- docker build -f nginx/Dockerfile -t apsl/thumbor-nginx nginx/
-- docker build -f remotecv/Dockerfile -t apsl/remotecv remotecv/
+- builder
 - docker-compose -f docker-compose-travis.yml up -d
 - docker-compose -f docker-compose-travis.yml ps
 - wget http://localhost:8001/unsafe/200x100/www.apsl.net/static/apslweb/img/content/slider/rs-slider4-img15.png
@@ -16,7 +13,7 @@ notifications:
   - [email protected]
 env:
   global:
-  - THUMBOR_VERSION=5.2.1e
+  - THUMBOR_VERSION=6.0.1
   - secure: LSNYgDagNJrEBIZtAMXgGNaza66WniJNWVYpeZ00bxrqF2AJ/NLt9FGGvySDwJHcgqgD0SbsxD7EAefuZlBNq/68qvHFtnNZb5FoU6gjXW9Kvdlvtt6r5zcTvrPDwAlaGUi6iDnTxNd1YK+dQ5M5QPk6Vr3yAU2Pd/SjQtDTZU9TIUPIU2PHkzf/3mOV2VXrwfN4SR1XDe15Dk/dofDc0GWfvSQUedUZzEYGOT4yg5CNa1Zps10tD5rEsGaYoiBfJyTNaaXPufGbytqrJ0OyLXtkKevcVGQcCZn2RrBqwMatuOMdlsx5UxHbxSCHngtwQgIE7nKsuSQupW4R8MK9RmSWPy34rbaBAb3qWYBrmEQxBSeVFDShML+wunMWwjJ2Spugc4ER71PX4O97GtQB/4H9EyEFuDO9MnEIHX6GpRCS21NHVXUVAdrAHXfTGZBIvrOcAHh9y1idHeCsF92X0qBNkC+MNaAxWV+gvCZ+sZpDKrfoUMDI+nxff5B/Cb33++9iv4gWz01GJDVmK6YyTT0/6+KzHNU0mp7ji7wgvlTUDYCSTR1+11R9Aoalt07KfMJS1gGG8HuMFcSCV7z1btxWZUWKpC6WydXwJuVyQZg7g6HYJEEHxtpP8z81FWLhGQWk3qMpi5kxZrqDQ7IES8ibK1o1tyFMUCn0Jdm6Uyo=
   - secure: JHQ1g7ZWuO5IQhPd915so8pBPfMNp00rRRvNmewHaUXO33UnC6g1pFkTKsx5QpDi0QxdH0o0DYOXB1nvd1Y/cO3spW5P2dV0kfftr2GUTGKf6WTakZPfqowvZN6YKA+rI7eFaGnhrVLhzKhK5T5Gu9UHklPlndo6LvQxqQBuYI2uggSTK4M4zD5XWqEe6o0RJ89y6xX7DyqLKIkXF3N/RcoHLh7GSfNZNdaN/u/Nb1fQKTYIOiMVIPYYNm1DU4c/jsIxxJhxC60DgavC5rwCLQg5FXT5vQ6Ky41JRR0U3a23US3E0wMaHP1LvHP9o4jfFeA6pLG7zgceGkY+IKlsWPT2pr3nR71xCHEtZe2tPvAlwvgwIngliM06SIGrk0s5yuhD6Qvug9/DfAe96gD557WYEXXnT++ByW/+gGq3kiWBQlMqg9Z8N15WIA0qe+LUrkmw9MiuWkJyR3hP0Kidl/pv0AAM0oqqGuLiruPAsetk24lT2zh77t4wOI1jyGUwaqVHUlRHOegflXcRh9eAsBHFmXDJEkqLDs4yJ+oeMUnLyoLy24xJ1hvx41oVGbIQL0qTIikkN5i9qAU0hX8ZoBOwQN+WoffNC2+tEX0cDhTugKIg3YzHkHqYDmplCsElP8PAuCzxl0/P8W4CtHqn9F0FO+CgW3HLit9jJfHFYeA=
   - secure: b1+J/WngVLdn37ubp7iJLGHY3X5j95Ewk60mYOy4FAfdb6ofK6dtOUXTGHNjqMNsZDoZShjuFvB3+Qrz5VSzx2JT/R+hgqdr08Ft8powZYUPJUOBih+mZAo1ACgxhn/jHQsp/OUkD4Qe0PGRVqva10HQP55+SINH605b9/HGQdQwGhI++kvsqLaQZVcdKtFyetF+RkY70rXAsite53zd++2FYScvsjuf0ie5BTmPTyoOs1Cq8w1/+xVSb911F5m1KIJg8Z3O1zXL1XJ2IXAHoVGxuoCeGNw61RKEMp3VGZPeX/TBvFSbQ5alShmxgS6GPAcZFsWX/Lv6CdYOWoXjDKCTfrgvoElcn8AvBoZiNDf9aegnHZooHskoA4Bi+Tj83TDoZvB1qQvLWhD/ldZl+/9gBj/f3zO8FECK+RTDeN5MNgKWs/eg8lU9D1JRYw2JkN+AADUZlJXGZiZHGQDh03rsSfb0C4LpwulZwEWCTB5s1y22WwI+1Ec+S4TajqPInojn45qUIyH0sf/JPVYpCYAbbQWugwz+rrvuv90rPYNI8m73Fv2jk5a57P5t4+UBGS6UNfq8Kyj5d/b/hRznT4lvE3hvPwRjo07C4m+hPAJgPuRnFCZhaJ4E6b9zgdgKitaz/xGiosn2O4Y2apT5mNcgbHEq1v0j+P19CxDqDHg=

Dockerfile.build

@@ -0,0 +1,21 @@
+FROM python:2
+
+MAINTAINER Edu Herraiz <[email protected]>
+
+# Things required for a python/pip environment
+COPY system-requirements-build.txt /system-requirements-build.txt
+RUN  \
+    apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y autoremove && \
+    xargs apt-get -y -q install < /system-requirements-build.txt && \
+    apt-get clean
+
+ENV WHEELHOUSE=/wheelhouse
+ENV PIP_WHEEL_DIR=/wheelhouse
+ENV PIP_FIND_LINKS=/wheelhouse
+VOLUME /wheelhouse
+
+RUN pip install wheel
+COPY requirements-build.txt /requirements.txt
+CMD pip wheel -r /requirements.txt

builder

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Build wheels for all projects
+docker build -t builder -f Dockerfile.build .
+mkdir -p wheelhouse
+docker run --rm -v "$(pwd)"/wheelhouse:/wheelhouse builder
+
+mv wheelhouse thumbor/wheelhouse
+docker build -f thumbor/Dockerfile -t apsl/thumbor thumbor/
+mv thumbor/wheelhouse thumbor-multiprocess/wheelhouse
+docker build -f thumbor-multiprocess/Dockerfile -t apsl/thumbor-multiprocess thumbor-multiprocess/
+docker build -f nginx/Dockerfile -t apsl/thumbor-nginx nginx/
+mv thumbor-multiprocess/wheelhouse remotecv/wheelhouse
+docker build -f remotecv/Dockerfile -t apsl/remotecv remotecv/
+mv remotecv/wheelhouse wheelhouse

docker-compose-examples/detector.yml

@@ -2,7 +2,6 @@ thumbor:
   image: apsl/thumbor:latest
   volumes:
     - data:/data
-    - logs:/logs
   ports:
     - "80:8000" # thumbor port
   environment:

docker-compose-test.yml

@@ -1,8 +1,7 @@
 thumbor:
-  image: apsl/thumbor-multiprocess:latest
+  image: apsl/thumbor:latest
   volumes:
     - ./data:/data
-    - ./logs:/logs
   #ports:
     #- "80:8000" # thumbor port
   environment:

remotecv/Dockerfile

@@ -2,7 +2,6 @@ FROM python:2
 
 MAINTAINER Edu Herraiz <[email protected]>
 
-VOLUME /logs
 VOLUME /data
 
 # Things required for a python/pip environment
@@ -19,13 +18,21 @@ ENV SHELL bash
 ENV WORKON_HOME /usr/src/app
 WORKDIR /usr/src/app
 
+ENV WHEELHOUSE=/wheelhouse
+ENV PIP_WHEEL_DIR=/wheelhouse
+ENV PIP_FIND_LINKS=/wheelhouse
+
+COPY wheelhouse /wheelhouse
+
 COPY requirements.txt /usr/src/app/requirements.txt
-RUN pip install --no-cache-dir -r /usr/src/app/requirements.txt
+RUN pip install --no-cache-dir --use-wheel --find-links /wheelhouse -r /usr/src/app/requirements.txt
 
 RUN \ 
     ln /usr/lib/python2.7/dist-packages/cv2.x86_64-linux-gnu.so /usr/local/lib/python2.7/cv2.so && \
     ln /usr/lib/python2.7/dist-packages/cv.py /usr/local/lib/python2.7/cv.py
 
+RUN rm -Rf /wheelhouse
+
 COPY docker-entrypoint.sh /
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["remotecv"]

remotecv/requirements.txt

@@ -1,11 +1,11 @@
 argparse==1.2.1
 envtpl==0.4.1
+boto==2.39.0
 dateutils==0.6.6
-numpy==1.9.2
+numpy==1.11.0
 pyremotecv==0.5.0
-remotecv==2.1.0
+remotecv==2.2.1
 opencv-engine==1.0.1
-redis==2.4.12
-thumbor==5.2.1
-raven==5.10.1
+thumbor==6.0.1
+raven==5.15.0
 remotecv_aws==0.1

remotecv/system-requirements.txt

@@ -2,12 +2,5 @@ python-numpy
 python-opencv
 git
 curl
-build-essential
-libcurl4-openssl-dev
 libdc1394-22
-libdc1394-22-dev
-libjpeg-dev
-libpng12-dev
-libtiff5-dev
-libjasper-dev
 libjpeg-turbo-progs

requirements-build.txt

@@ -0,0 +1,23 @@
+argparse==1.2.1
+envtpl==0.4.1
+boto==2.39.0
+dateutils==0.6.6
+numpy==1.11.0
+pyremotecv==0.5.0
+remotecv==2.2.1
+opencv-engine==1.0.1
+thumbor==6.0.1
+tc-aws==6.0.1
+tc-core==0.3.0
+tc-shortener==0.2.2
+raven==5.15.0
+thumbor-memcached==5.1.0
+tc-mongodb==5.1.0
+tc-redis==1.0.1
+
+## Multiprocess
+circus==0.11.1
+circus-web==0.5
+
+# Remotecv
+remotecv-aws==0.1

system-requirements-build.txt

@@ -0,0 +1,14 @@
+python-numpy
+python-opencv
+git
+curl
+build-essential
+libcurl4-openssl-dev
+libdc1394-22
+libdc1394-22-dev
+libjpeg-dev
+libpng12-dev
+libtiff5-dev
+libjasper-dev
+libmemcached-dev
+libjpeg-turbo-progs

thumbor-multiprocess/Dockerfile

@@ -2,13 +2,21 @@ FROM apsl/thumbor
 
 MAINTAINER Edu Herraiz <[email protected]>
 
+ENV WHEELHOUSE=/wheelhouse
+ENV PIP_WHEEL_DIR=/wheelhouse
+ENV PIP_FIND_LINKS=/wheelhouse
+
+COPY wheelhouse /wheelhouse
+
 COPY requirements.txt /usr/src/app/requirements.txt
-RUN pip install --no-cache-dir -r /usr/src/app/requirements.txt
+RUN pip install --no-cache-dir --use-wheel --find-links /wheelhouse -r /usr/src/app/requirements.txt
 
 ADD conf/circus.ini.tpl /etc/
 RUN mkdir  /etc/circus.d /etc/setup.d
 ADD conf/thumbor.ini.tpl /etc/circus.d/
 
+RUN rm -Rf /wheelhouse
+
 COPY docker-entrypoint.sh /
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["circus"]

thumbor/Dockerfile

@@ -2,7 +2,6 @@ FROM python:2
 
 MAINTAINER Edu Herraiz <[email protected]>
 
-VOLUME /logs
 VOLUME /data
 
 # Things required for a python/pip environment
@@ -18,16 +17,22 @@ ENV HOME /usr/src/app
 ENV SHELL bash
 ENV WORKON_HOME /usr/src/app
 WORKDIR /usr/src/app
+ENV WHEELHOUSE=/wheelhouse
+ENV PIP_WHEEL_DIR=/wheelhouse
+ENV PIP_FIND_LINKS=/wheelhouse
 
+COPY wheelhouse /wheelhouse
 COPY requirements.txt /usr/src/app/requirements.txt
-RUN pip install --no-cache-dir -r /usr/src/app/requirements.txt
+RUN pip install --no-cache-dir --use-wheel --find-links /wheelhouse -r /usr/src/app/requirements.txt
 
 COPY conf/thumbor.conf.tpl /usr/src/app/thumbor.conf.tpl
 
 RUN \ 
     ln /usr/lib/python2.7/dist-packages/cv2.x86_64-linux-gnu.so /usr/local/lib/python2.7/cv2.so && \
     ln /usr/lib/python2.7/dist-packages/cv.py /usr/local/lib/python2.7/cv.py
 
+RUN rm -Rf /wheelhouse
+
 COPY docker-entrypoint.sh /
 ENTRYPOINT ["/docker-entrypoint.sh"]
 CMD ["thumbor"]

thumbor/requirements.txt

@@ -2,13 +2,15 @@ argparse==1.2.1
 envtpl==0.4.1
 boto==2.39.0
 dateutils==0.6.6
-numpy==1.9.2
+numpy==1.11.0
 pyremotecv==0.5.0
-remotecv==2.1.0
+remotecv==2.2.1
 opencv-engine==1.0.1
-redis==2.4.12
-thumbor==5.2.1
-tc-aws==2.1.2
+thumbor==6.0.1
+tc-aws==6.0.1
 tc-core==0.3.0
 tc-shortener==0.2.2
-raven==5.10.1
+raven==5.15.0
+thumbor-memcached==5.1.0
+tc-mongodb==5.1.0
+tc-redis==1.0.1

thumbor/system-requirements.txt

@@ -2,12 +2,5 @@ python-numpy
 python-opencv
 git
 curl
-build-essential
-libcurl4-openssl-dev
 libdc1394-22
-libdc1394-22-dev
-libjpeg-dev
-libpng12-dev
-libtiff5-dev
-libjasper-dev
 libjpeg-turbo-progs