diff --git a/buildscripts/include/depinfo.sh b/buildscripts/include/depinfo.sh index ea2cf001c..00773242b 100755 --- a/buildscripts/include/depinfo.sh +++ b/buildscripts/include/depinfo.sh @@ -31,7 +31,9 @@ dep_libass=(freetype2 fribidi harfbuzz unibreak) dep_lua=() dep_libplacebo=() dep_mpv=(ffmpeg libass lua libplacebo) -dep_mpv_android=(mpv) +dep_openssl=() +dep_python=(openssl) +dep_mpv_android=(mpv python) ## Travis-related diff --git a/buildscripts/include/download-deps.sh b/buildscripts/include/download-deps.sh index 030a73bdf..47a4a8966 100755 --- a/buildscripts/include/download-deps.sh +++ b/buildscripts/include/download-deps.sh @@ -63,6 +63,33 @@ fi # mpv [ ! -d mpv ] && git clone https://github.com/mpv-player/mpv +# openssl +if [ ! -d openssl ]; then + mkdir openssl + $WGET https://www.openssl.org/source/openssl-$v_openssl.tar.gz -O - | \ + tar -xz -C openssl --strip-components=1 +fi + +# python +if [ ! -d python ]; then + mkdir python + $WGET https://www.python.org/ftp/python/$v_python/Python-$v_python.tar.xz -O- | \ + tar -xJ -C python --strip-components=1 + + cd python + for name in inplace static_modules; do + patch -p0 --verbose <../../include/py/$name.patch + done + # Enables all modules *except* these + python3 ../../include/py/uncomment.py Modules/Setup \ + 'readline|_test|spwd|grp|_crypt|nis|termios|resource|audio|_md5|_sha[125]|_tkinter|syslog|_curses|_g?dbm|_(multibyte)?codec' + # SSL path is not used + sed 's|^SSL=.*|SSL=/var/empty|' -i Modules/Setup + # hashlib via openssl + echo '_hashlib _hashopenssl.c -lcrypto' >>Modules/Setup + cd .. +fi + cd .. # youtube-dl diff --git a/buildscripts/include/py/inplace.patch b/buildscripts/include/py/inplace.patch new file mode 100644 index 000000000..679a4a956 --- /dev/null +++ b/buildscripts/include/py/inplace.patch @@ -0,0 +1,11 @@ +--- Modules/getpath.c.orig 2021-03-11 11:47:42.769657798 +0100 ++++ Modules/getpath.c 2021-03-11 11:52:26.920159439 +0100 +@@ -1293,7 +1293,7 @@ + PyStatus res; + + /* Path: / "pythonXY.zip" */ +- wchar_t *path = joinpath2(calculate->platlibdir, ++ wchar_t *path = _PyMem_RawWcsdup( + L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION) + L".zip"); + if (path == NULL) { diff --git a/buildscripts/include/py/static_modules.patch b/buildscripts/include/py/static_modules.patch new file mode 100644 index 000000000..1097aa1fb --- /dev/null +++ b/buildscripts/include/py/static_modules.patch @@ -0,0 +1,10 @@ +--- setup.py 2019-03-25 21:21:05.000000000 +0100 ++++ setup.py 2019-06-21 16:52:23.151217709 +0200 +@@ -379,6 +379,7 @@ + print() + + def build_extension(self, ext): ++ return print('WARNING: not building extension "%s"' % ext.name) + + if ext.name == '_ctypes': + if not self.configure_ctypes(ext): diff --git a/buildscripts/include/py/uncomment.py b/buildscripts/include/py/uncomment.py new file mode 100644 index 000000000..d12802d8f --- /dev/null +++ b/buildscripts/include/py/uncomment.py @@ -0,0 +1,13 @@ +import sys, re +with open(sys.argv[1], "r") as f: + lines = f.readlines() +for i, l in enumerate(lines): + if not l.startswith("#") or l[1] in " \t\n" or re.match(sys.argv[2], l[1:]): continue + elif l.startswith("#*shared*"): continue + elif l.startswith("#*disabled*"): break + lines[i] = l.lstrip("#") + while lines[i].strip().endswith("\\"): + i += 1 + lines[i] = lines[i].lstrip("#") +with open(sys.argv[1], "w") as f: + f.writelines(lines) diff --git a/buildscripts/scripts/openssl.sh b/buildscripts/scripts/openssl.sh new file mode 100755 index 000000000..79ba6fcf8 --- /dev/null +++ b/buildscripts/scripts/openssl.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e + +. ../../include/path.sh + +if [ "$1" == "build" ]; then + true +elif [ "$1" == "clean" ]; then + rm -rf _build$ndk_suffix + exit 0 +else + exit 255 +fi + +export CFLAGS="-Os" + +mkdir -p _build$ndk_suffix +cd _build$ndk_suffix + +case "$ndk_triple" in + arm*) + target=linux-armv4 + ;; + aarch64*) + target=linux-aarch64 + ;; + i686*) + target=linux-x86-clang + ;; + x86_64*) + target=linux-x86_64-clang + ;; +esac + +../Configure $target no-shared +make -j$cores +make DESTDIR="$prefix_dir" install_sw diff --git a/buildscripts/scripts/python.sh b/buildscripts/scripts/python.sh new file mode 100755 index 000000000..a3d13812d --- /dev/null +++ b/buildscripts/scripts/python.sh @@ -0,0 +1,85 @@ +#!/bin/bash -e + +. ../../include/path.sh +. ../../include/depinfo.sh + +if [ "$1" == "build" ]; then + true +elif [ "$1" == "clean" ]; then + rm -rf _build$ndk_suffix + exit 0 +else + exit 255 +fi + +# TODO figure this out +if [[ -z "$DOIT" && "$ndk_triple" != "arm"* ]]; then + echo "Skipping build for $ndk_triple, only supposed to run on ARM (for now)" + echo "To build anyway set DOIT=1 env variable" + exit 0 +fi +### + +hostpy=python${v_python:0:3} +if ! command -v $hostpy; then + echo "compatible Python ($hostpy) is required to build" + exit 1 +fi + +recompile_py () { + find . -name '*.pyc' -delete + $hostpy -OO -m compileall -b -j4 . + # leave only the legacy locations (*.pyc next to *.py) + find . -name "__pycache__" -print0 | xargs -0 -- rm -rf +} + +prune_stdlib () { + local delete=( + pydoc_data turtledemo # docs + test unittest/test # unittests + tkinter sqlite3 venv ensurepip # doesn't work anyway + lib2to3 idlelib distutils multiprocessing # not used by ytdl + ) + rm -r "${delete[@]}" + # ytdl tries to import this: + rm -rf ctypes && mkdir -p ctypes + cat >ctypes/__init__.py <<"FILE" +class cdll(): + @staticmethod + def LoadLibrary(lib): + raise OSError +FILE +} + +export READELF=llvm-readelf +export CFLAGS="-Os -I$prefix_dir/include" +export LDFLAGS="-L$prefix_dir/lib" + +mkdir -p _build$ndk_suffix +cd _build$ndk_suffix + +# build +ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no \ +../configure --host=$ndk_triple --build=${ndk_triple%%-*} \ + --enable-ipv6 --disable-shared --without-ensurepip +make -j$cores + +# install to temporary location +rm -rf dest +make DESTDIR="$PWD/dest" install +inst=$PWD/dest/usr/local + +out=$(realpath ../../../../app/src/main/assets/ytdl) +mkdir -p $out +rm -f $out/python* + +# copy & strip executable +cp -v python $out/python3 +llvm-strip -s $out/python3 + +# package standard library +pushd $inst/lib/python3.* +prune_stdlib +recompile_py +zip -9 $out/python3${v_python:2:1}.zip -R '*.pyc' +popd