From ce216eed5ba6bda238da1714093d4aa9d679c987 Mon Sep 17 00:00:00 2001 From: Borys Kabakov Date: Mon, 12 Oct 2020 22:06:19 +0300 Subject: [PATCH] New version: 4.5.0 + 2021.1 * upd new version * OpenBLAS faster than JIT * wheel already manylinux and should not be used for win * upd * upd * 4.4.0 ready * submodules test (#11) * Update TODO.md * tensorflow-cpu==2.2.0 -> tensorflow-cpu==2.2.1 * preparations for new version * all set. waiting for opencv-4.5.0 * upd README.md * opencv 4.5.0 is here --- .gitmodules | 12 +++++++++++ README.md | 22 ++++++++++++------- TODO.md | 5 ++--- build/dldt/dldt_setup.sh | 1 - build/opencv/opencv_setup.sh | 6 ++++-- create_wheel/setup.py | 4 ++-- dldt | 1 + dldt/.gitkeep | 0 download_all_stuff.sh | 39 ++++------------------------------ ffmpeg | 1 + ffmpeg/.gitkeep | 0 openblas | 1 + openblas/.gitkeep | 0 opencv | 1 + opencv/.gitkeep | 0 tests/prepare_and_run_tests.sh | 8 ++++--- tests/requirements.txt | 3 ++- 17 files changed, 49 insertions(+), 55 deletions(-) create mode 100644 .gitmodules create mode 160000 dldt delete mode 100644 dldt/.gitkeep create mode 160000 ffmpeg delete mode 100644 ffmpeg/.gitkeep create mode 160000 openblas delete mode 100644 openblas/.gitkeep create mode 160000 opencv delete mode 100644 opencv/.gitkeep diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..91b55a9 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,12 @@ +[submodule "dldt"] + path = dldt + url = https://github.com/openvinotoolkit/openvino +[submodule "opencv"] + path = opencv + url = https://github.com/opencv/opencv +[submodule "openblas"] + path = openblas + url = https://github.com/xianyi/OpenBLAS +[submodule "ffmpeg"] + path = ffmpeg + url = https://github.com/FFmpeg/FFmpeg diff --git a/README.md b/README.md index e990d6d..b93d44c 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ pip3 install opencv-python-inference-engine ## Why I needed an ability to fast deploy a small package that able to run models from [Intel's model zoo](https://github.com/opencv/open_model_zoo/) and use [Movidius NCS](https://software.intel.com/en-us/neural-compute-stick). -Wellr-known [opencv-python](https://github.com/skvark/opencv-python) can't do this. +Well-known [opencv-python](https://github.com/skvark/opencv-python) can't do this. The official way is to use OpenVINO, but it is big and clumsy (just try to use it with python venv or fast download it on cloud instance). @@ -68,7 +68,7 @@ Better to find a model description [here](https://github.com/opencv/open_model_z ## Compiling from source -You will need ~6GB RAM and ~10GB disk space +You will need ~7GB RAM and ~10GB disk space I am using Ubuntu 18.04 [multipass](https://multipass.run/) instance: `multipass launch -c 6 -d 10G -m 7G`. @@ -91,11 +91,12 @@ From [opencv](https://docs.opencv.org/master/d7/d9f/tutorial_linux_install.html) ```bash # We need newer `cmake` for dldt (commands from ) -wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | sudo apt-key add - -sudo apt-add-repository 'deb https://apt.kitware.com/ubuntu/ bionic main' +sudo apt remove --purge cmake +hash -r +sudo snap install cmake --classic sudo apt-get update -sudo apt install build-essential cmake git pkg-config python3-dev nasm python3 virtualenv libusb-1.0-0-dev chrpath autoconf libtool-bin +sudo apt install build-essential git pkg-config python3-dev nasm python3 virtualenv libusb-1.0-0-dev chrpath autoconf libtool-bin # for ngraph # the `dldt/_deps/ext_onnx-src/onnx/gen_proto.py` has `#!/usr/bin/env python` string and will throw an error otherwise @@ -104,9 +105,11 @@ sudo ln -s /usr/bin/python3 /usr/bin/python ### Preparing -1. `git clone https://github.com/banderlog/opencv-python-inference-engine` -2. `cd opencv-python-inference-engine` -3. run `download_all_stuff.sh` (refer for script code for details) +```bash +git clone https://github.com/banderlog/opencv-python-inference-engine +cd opencv-python-inference-engine +./download_all_stuff.sh +``` ### Compilation @@ -126,6 +129,8 @@ cd ../dldt ./dldt_setup.sh && make -j6 +# NB: check `-D INF_ENGINE_RELEASE` value +# should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") cd ../opencv ./opencv_setup.sh && make -j6 @@ -152,6 +157,7 @@ cd create_wheel for i in cv2/*.so; do chrpath -r '$ORIGIN' $i; done # final .whl will be in /create_wheel/dist/ +# NB: check versions in the `setup.py` ../venv/bin/python3 setup.py bdist_wheel ``` diff --git a/TODO.md b/TODO.md index d1bf5f8..2eb7874 100644 --- a/TODO.md +++ b/TODO.md @@ -1,5 +1,4 @@ # TODO list -+ Auto value for `-D INF_ENGINE_RELEASE` -+ Try to submodule things instead of downloading archives of them -+ check tar.gz integrity otherwise \ No newline at end of file ++ Auto value for `-D INF_ENGINE_RELEASE`: https://github.com/openvinotoolkit/openvino/issues/1435 + diff --git a/build/dldt/dldt_setup.sh b/build/dldt/dldt_setup.sh index 321f33a..1cf3d04 100755 --- a/build/dldt/dldt_setup.sh +++ b/build/dldt/dldt_setup.sh @@ -39,7 +39,6 @@ cmake -D CMAKE_BUILD_TYPE=Release \ -D NGRAPH_UNIT_TEST_ENABLE=OFF \ -D NGRAPH_TEST_UTIL_ENABLE=OFF \ -D NGRAPH_ONNX_IMPORT_ENABLE=ON \ - -D NGRAPH_JSON_ENABLE=ON \ -D BLAS_LIBRARIES="$BLAS_LIB" \ -D BLAS_INCLUDE_DIRS="$BLAS_INC" \ -D ENABLE_CLDNN=OFF \ diff --git a/build/opencv/opencv_setup.sh b/build/opencv/opencv_setup.sh index cddeaf3..9590442 100755 --- a/build/opencv/opencv_setup.sh +++ b/build/opencv/opencv_setup.sh @@ -5,6 +5,8 @@ # # -DINF_ENGINE_RELEASE= should match dldt version # See +# From +# "Force IE version, should be in form YYYYAABBCC (e.g. 2020.1.0.2 -> 2020010002)") tmp=$(pwd) ABS_PORTION=${tmp%%"/build/opencv"} @@ -73,8 +75,8 @@ cmake -D CMAKE_BUILD_TYPE=RELEASE \ -D WITH_NGRAPH=ON \ -D ngraph_DIR=$ABS_PORTION/build/dldt/ngraph \ -D WITH_INF_ENGINE=ON \ - -D INF_ENGINE_RELEASE=2020040000 \ + -D INF_ENGINE_RELEASE=2021010000 \ -D INF_ENGINE_INCLUDE_DIRS=$ABS_PORTION/dldt/inference-engine/include \ -D INF_ENGINE_LIB_DIRS=$ABS_PORTION/dldt/bin/intel64/Release/lib \ -D CPU_BASELINE=SSE4_2 \ - -D CPU_DISPATCH=AVX,AVX2,FP16,AVX512 ../../opencv + -D CPU_DISPATCH=AVX,AVX2,FP16,AVX512 ../../opencv \ No newline at end of file diff --git a/create_wheel/setup.py b/create_wheel/setup.py index 54c9986..e71bf95 100644 --- a/create_wheel/setup.py +++ b/create_wheel/setup.py @@ -15,11 +15,11 @@ def __len__(self): setuptools.setup( name="opencv-python-inference-engine", - version="4.4.0.0", + version="4.5.0.0", url="https://github.com/banderlog/opencv-python-inference-engine", maintainer="Kabakov Borys", license='MIT, BSD, Intel Simplified Software License', - description="Wrapper package for OpenCV 4.4.0 with Inference Engine 2020.4 python bindings", + description="Wrapper package for OpenCV 4.5.0 with Inference Engine 2021.1 python bindings", long_description=long_description, long_description_content_type="text/markdown", ext_modules=EmptyListWithLength(), diff --git a/dldt b/dldt new file mode 160000 index 0000000..f557dca --- /dev/null +++ b/dldt @@ -0,0 +1 @@ +Subproject commit f557dca475cb54dcfc9026fbaad0d93ddb85015c diff --git a/dldt/.gitkeep b/dldt/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/download_all_stuff.sh b/download_all_stuff.sh index 02f87ad..4eacfea 100755 --- a/download_all_stuff.sh +++ b/download_all_stuff.sh @@ -17,18 +17,9 @@ red () { ROOT_DIR=$(pwd) -green "DOWNLOAD ALL ARCHIVES" -wget -c https://github.com/opencv/opencv/archive/4.4.0.tar.gz -O opencv.tar.gz -wget -c https://github.com/openvinotoolkit/openvino/archive/2020.4.tar.gz -O dldt.tar.gz -wget -c https://github.com/FFmpeg/FFmpeg/archive/n4.3.1.tar.gz -O ffmpeg.tar.gz -wget -c https://github.com/xianyi/OpenBLAS/archive/v0.3.10.tar.gz -O openblas.tar.gz - - -green "CLEAN LIB DIRS" -rm -drf ./dldt/* -rm -drf ./ffmpeg/* -rm -drf ./opencv/* -rm -drf ./openblas/* +green "RESET GIT SUBMODULES" +# use `git pull && git checkout tags/` for update +git submodule update --init --recursive green "CLEAN BUILD DIRS" find build/dldt/ -mindepth 1 -not -name 'dldt_setup.sh' -delete @@ -42,32 +33,10 @@ rm -drf create_wheel/build rm -drf create_wheel/dist rm -drf create_wheel/*egg-info -green "UNZIP ALL STUFF" -tar -xf opencv.tar.gz --strip-components=1 -C ./opencv/ -tar -xf dldt.tar.gz --strip-components=1 -C ./dldt/ -tar -xf ffmpeg.tar.gz --strip-components=1 -C ./ffmpeg/ -tar -xf openblas.tar.gz --strip-components=1 -C ./openblas/ - -green "GIT RESET FOR ade" -cd ./dldt/inference-engine/thirdparty/ade -git clone https://github.com/opencv/ade/ ./ -git reset --hard cbe2db6 - -green "GIT RESET FOR mkl-dnn" -cd ../mkl-dnn -git clone https://github.com/openvinotoolkit/oneDNN/ ./ -git reset --hard 2706f56 - -green "GIT RESET FOR IE samples gflags" -cd ../../samples/thirdparty/gflags/ -git clone https://github.com/gflags/gflags ./ -git reset --hard 46f73f8 - - green "CREATE VENV" cd $ROOT_DIR if [[ ! -d ./venv ]]; then virtualenv --clear --always-copy -p /usr/bin/python3 ./venv ./venv/bin/pip3 install numpy -fi +fi \ No newline at end of file diff --git a/ffmpeg b/ffmpeg new file mode 160000 index 0000000..6b6b9e5 --- /dev/null +++ b/ffmpeg @@ -0,0 +1 @@ +Subproject commit 6b6b9e593dd4d3aaf75f48d40a13ef03bdef9fdb diff --git a/ffmpeg/.gitkeep b/ffmpeg/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/openblas b/openblas new file mode 160000 index 0000000..63b03ef --- /dev/null +++ b/openblas @@ -0,0 +1 @@ +Subproject commit 63b03efc2af332c88b86d4fd8079d00f4b439adf diff --git a/openblas/.gitkeep b/openblas/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/opencv b/opencv new file mode 160000 index 0000000..d5fd2f0 --- /dev/null +++ b/opencv @@ -0,0 +1 @@ +Subproject commit d5fd2f0155ffad366f9ac912dfd6d189a7a6a98e diff --git a/opencv/.gitkeep b/opencv/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/tests/prepare_and_run_tests.sh b/tests/prepare_and_run_tests.sh index c93aec9..46bb544 100755 --- a/tests/prepare_and_run_tests.sh +++ b/tests/prepare_and_run_tests.sh @@ -12,6 +12,7 @@ red () { echo -e "${red}${1}${end}" } + # check if (no ARG and no some appropriate files are compiled) or # (some args provided but arg1 is not existing file) # of course, you could shoot your leg here in different ways @@ -33,13 +34,14 @@ green "INSTALLING DEPENDENCIES" if [ $1 ]; then # install ARGV1 green "Installing from provided path" - ./venv_t/bin/pip3 install --force-reinstall "$1" + WHEEL="$1" else # install compiled wheel green "Installing from default path" - ./venv_t/bin/pip3 install --force-reinstall ../create_wheel/dist/opencv_python_inference_engine*.whl + WHEEL=$(realpath ../create_wheel/dist/opencv_python_inference_engine*.whl) fi +./venv_t/bin/pip3 install --force-reinstall "$WHEEL" ./venv_t/bin/pip3 install -r requirements.txt @@ -88,7 +90,7 @@ for i in "${!se_net[@]}"; do fi done -green "For \"$1\"" +green "For \"$WHEEL\"" green "RUN TESTS with ./venv_t/bin/python ./tests.py" ./venv_t/bin/python ./tests.py green "RUN TESTS with ./venv_t/bin/python ./speed_test.py" diff --git a/tests/requirements.txt b/tests/requirements.txt index b959118..c7193f5 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,4 +1,5 @@ +numpy>=1.16.0,<1.19.0 tensorflow-cpu==2.2.1 scipy==1.4.1 scikit-image -ipython +ipython \ No newline at end of file