python build script

buildscripts/include/depinfo.sh

@@ -31,7 +31,9 @@ dep_libass=(freetype2 fribidi harfbuzz unibreak)
 dep_lua=()
 dep_libplacebo=()
 dep_mpv=(ffmpeg libass lua libplacebo)
-dep_mpv_android=(mpv)
+dep_openssl=()
+dep_python=(openssl)
+dep_mpv_android=(mpv python)
 
 
 ## Travis-related

buildscripts/include/download-deps.sh

@@ -63,6 +63,33 @@ fi
 # mpv
 [ ! -d mpv ] && git clone https://github.com/mpv-player/mpv
 
+# openssl
+if [ ! -d openssl ]; then
+	mkdir openssl
+	$WGET https://www.openssl.org/source/openssl-$v_openssl.tar.gz -O - | \
+		tar -xz -C openssl --strip-components=1
+fi
+
+# python
+if [ ! -d python ]; then
+	mkdir python
+	$WGET https://www.python.org/ftp/python/$v_python/Python-$v_python.tar.xz -O- | \
+		tar -xJ -C python --strip-components=1
+
+	cd python
+	for name in inplace static_modules; do
+		patch -p0 --verbose <../../include/py/$name.patch
+	done
+	# Enables all modules *except* these
+	python3 ../../include/py/uncomment.py Modules/Setup \
+		'readline|_test|spwd|grp|_crypt|nis|termios|resource|audio|_md5|_sha[125]|_tkinter|syslog|_curses|_g?dbm|_(multibyte)?codec'
+	# SSL path is not used
+	sed 's|^SSL=.*|SSL=/var/empty|' -i Modules/Setup
+	# hashlib via openssl
+	echo '_hashlib _hashopenssl.c -lcrypto' >>Modules/Setup
+	cd ..
+fi
+
 cd ..
 
 # youtube-dl

buildscripts/include/py/inplace.patch

@@ -0,0 +1,11 @@
+--- Modules/getpath.c.orig	2021-03-11 11:47:42.769657798 +0100
++++ Modules/getpath.c	2021-03-11 11:52:26.920159439 +0100
+@@ -1293,7 +1293,7 @@
+     PyStatus res;
+
+     /* Path: <platlibdir> / "pythonXY.zip" */
+-    wchar_t *path = joinpath2(calculate->platlibdir,
++    wchar_t *path = _PyMem_RawWcsdup(
+                               L"python" Py_STRINGIFY(PY_MAJOR_VERSION) Py_STRINGIFY(PY_MINOR_VERSION)
+                               L".zip");
+     if (path == NULL) {

buildscripts/include/py/static_modules.patch

@@ -0,0 +1,10 @@
+--- setup.py	2019-03-25 21:21:05.000000000 +0100
++++ setup.py	2019-06-21 16:52:23.151217709 +0200
+@@ -379,6 +379,7 @@
+             print()
+
+     def build_extension(self, ext):
++        return print('WARNING: not building extension "%s"' % ext.name)
+
+         if ext.name == '_ctypes':
+             if not self.configure_ctypes(ext):

buildscripts/include/py/uncomment.py

@@ -0,0 +1,13 @@
+import sys, re
+with open(sys.argv[1], "r") as f:
+	lines = f.readlines()
+for i, l in enumerate(lines):
+	if not l.startswith("#") or l[1] in " \t\n" or re.match(sys.argv[2], l[1:]): continue
+	elif l.startswith("#*shared*"): continue
+	elif l.startswith("#*disabled*"): break
+	lines[i] = l.lstrip("#")
+	while lines[i].strip().endswith("\\"):
+		i += 1
+		lines[i] = lines[i].lstrip("#")
+with open(sys.argv[1], "w") as f:
+	f.writelines(lines)

buildscripts/scripts/openssl.sh

@@ -0,0 +1,36 @@
+#!/bin/bash -e
+
+. ../../include/path.sh
+
+if [ "$1" == "build" ]; then
+	true
+elif [ "$1" == "clean" ]; then
+	rm -rf _build$ndk_suffix
+	exit 0
+else
+	exit 255
+fi
+
+export CFLAGS="-Os"
+
+mkdir -p _build$ndk_suffix
+cd _build$ndk_suffix
+
+case "$ndk_triple" in
+	arm*)
+	target=linux-armv4
+	;;
+	aarch64*)
+	target=linux-aarch64
+	;;
+	i686*)
+	target=linux-x86-clang
+	;;
+	x86_64*)
+	target=linux-x86_64-clang
+	;;
+esac
+
+../Configure $target no-shared
+make -j$cores
+make DESTDIR="$prefix_dir" install_sw

buildscripts/scripts/python.sh

@@ -0,0 +1,85 @@
+#!/bin/bash -e
+
+. ../../include/path.sh
+. ../../include/depinfo.sh
+
+if [ "$1" == "build" ]; then
+	true
+elif [ "$1" == "clean" ]; then
+	rm -rf _build$ndk_suffix
+	exit 0
+else
+	exit 255
+fi
+
+# TODO figure this out
+if [[ -z "$DOIT" && "$ndk_triple" != "arm"* ]]; then
+	echo "Skipping build for $ndk_triple, only supposed to run on ARM (for now)"
+	echo "To build anyway set DOIT=1 env variable"
+	exit 0
+fi
+###
+
+hostpy=python${v_python:0:3}
+if ! command -v $hostpy; then
+	echo "compatible Python ($hostpy) is required to build"
+	exit 1
+fi
+
+recompile_py () {
+	find . -name '*.pyc' -delete
+	$hostpy -OO -m compileall -b -j4 .
+	# leave only the legacy locations (*.pyc next to *.py)
+	find . -name "__pycache__" -print0 | xargs -0 -- rm -rf
+}
+
+prune_stdlib () {
+	local delete=(
+		pydoc_data turtledemo # docs
+		test unittest/test # unittests
+		tkinter sqlite3 venv ensurepip # doesn't work anyway
+		lib2to3 idlelib distutils multiprocessing # not used by ytdl
+	)
+	rm -r "${delete[@]}"
+	# ytdl tries to import this:
+	rm -rf ctypes && mkdir -p ctypes
+	cat >ctypes/__init__.py <<"FILE"
+class cdll():
+  @staticmethod
+  def LoadLibrary(lib):
+    raise OSError
+FILE
+}
+
+export READELF=llvm-readelf
+export CFLAGS="-Os -I$prefix_dir/include"
+export LDFLAGS="-L$prefix_dir/lib"
+
+mkdir -p _build$ndk_suffix
+cd _build$ndk_suffix
+
+# build
+ac_cv_file__dev_ptmx=no ac_cv_file__dev_ptc=no \
+../configure --host=$ndk_triple --build=${ndk_triple%%-*} \
+	--enable-ipv6 --disable-shared --without-ensurepip
+make -j$cores
+
+# install to temporary location
+rm -rf dest
+make DESTDIR="$PWD/dest" install
+inst=$PWD/dest/usr/local
+
+out=$(realpath ../../../../app/src/main/assets/ytdl)
+mkdir -p $out
+rm -f $out/python*
+
+# copy & strip executable
+cp -v python $out/python3
+llvm-strip -s $out/python3
+
+# package standard library
+pushd $inst/lib/python3.*
+prune_stdlib
+recompile_py
+zip -9 $out/python3${v_python:2:1}.zip -R '*.pyc'
+popd