Update Python version to 3.10, and update to Ubuntu 22.04 for our bui…

…ld scripts and Dockerfiles. Update other software versions as needed.

PiperOrigin-RevId: 613741059

Dockerfile

@@ -11,9 +11,9 @@
 # $ sudo docker build --build-arg=FROM_IMAGE=nvidia/cuda:11.3.1-cudnn8-devel-ubuntu20.04 --build-arg=DV_GPU_BUILD=1 -t deepvariant_gpu .
 
 
-ARG FROM_IMAGE=ubuntu:20.04
+ARG FROM_IMAGE=ubuntu:22.04
 # PYTHON_VERSION is also set in settings.sh.
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.10
 ARG DV_GPU_BUILD=0
 ARG VERSION=1.6.0
 ARG TF_ENABLE_ONEDNN_OPTS=1

Dockerfile.deepsomatic

@@ -11,9 +11,9 @@
 # $ sudo docker build -f Dockerfile.deepsomatic --build-arg=FROM_IMAGE=nvidia/cuda:11.3.1-cudnn8-devel-ubuntu20.04 --build-arg=DV_GPU_BUILD=1 -t deepsomatic_gpu .
 
 
-ARG FROM_IMAGE=ubuntu:20.04
+ARG FROM_IMAGE=ubuntu:22.04
 # PYTHON_VERSION is also set in settings.sh.
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.10
 ARG DV_GPU_BUILD=0
 ARG VERSION_DEEPSOMATIC=1.6.0
 ARG TF_ENABLE_ONEDNN_OPTS=1

Dockerfile.deeptrio

@@ -11,9 +11,9 @@
 # $ sudo docker build -f Dockerfile.deeptrio --build-arg=FROM_IMAGE=nvidia/cuda:11.3.1-cudnn8-devel-ubuntu20.04 --build-arg=DV_GPU_BUILD=1 -t deeptrio_gpu .
 
 
-ARG FROM_IMAGE=ubuntu:20.04
+ARG FROM_IMAGE=ubuntu:22.04
 # PYTHON_VERSION is also set in settings.sh.
-ARG PYTHON_VERSION=3.8
+ARG PYTHON_VERSION=3.10
 ARG DV_GPU_BUILD=0
 ARG VERSION_DEEPTRIO=1.6.0
 ARG TF_ENABLE_ONEDNN_OPTS=1

WORKSPACE

@@ -65,6 +65,8 @@ http_archive(
 # That BUILD file must be kept in sync with the version of protobuf used.
 http_archive(
     name = "com_google_protobuf",
+    patch_args = ["-p1"],
+    patches = ["//:third_party/protobuf.patch"],
     build_file = "//:third_party/protobuf.BUILD",
     sha256 = "cfcba2df10feec52a84208693937c17a4b5df7775e1635c1e3baffc487b24c9b",
     # This protobuf release is based on protobuf 3.9.2.

build-prereq.sh

@@ -51,7 +51,7 @@ note_build_stage "build-prereq.sh: Install development packages"
 
 # Need to wait for dpkg lock (see internal)
 wait_for_dpkg_lock
-sudo -H apt-get -qq -y install pkg-config zip g++ zlib1g-dev unzip curl git wget > /dev/null
+sudo -H NEEDRESTART_MODE=a apt-get -qq -y install pkg-config zip g++ zlib1g-dev unzip curl git wget > /dev/null
 
 
 ################################################################################
@@ -134,8 +134,31 @@ wget https://raw.githubusercontent.com/tensorflow/tensorflow/r2.13/third_party/a
 wget https://raw.githubusercontent.com/tensorflow/tensorflow/r2.13/third_party/absl/absl_designated_initializers.patch -O ../tensorflow/third_party/absl/absl_design\
 ated_initializers.patch
 
-note_build_stage "Set pyparsing to 2.2.0 for CLIF."
+# Inspired by part of https://raw.githubusercontent.com/tensorflow/tensorflow/r2.11/third_party/protobuf/protobuf.patch.
+# This is necessary for Python 3.10.
+cat > third_party/protobuf.patch <<- EOM
+diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
+index 3530a9b37..c31fa8fcc 100644
+--- a/python/google/protobuf/pyext/message.cc
++++ b/python/google/protobuf/pyext/message.cc
+@@ -2991,8 +2991,12 @@ bool InitProto2MessageModule(PyObject *m) {
+         reinterpret_cast<PyObject*>(
+             &RepeatedCompositeContainer_Type));
+ 
+-    // Register them as collections.Sequence
++    // Register them as MutableSequence.
++#if PY_MAJOR_VERSION >= 3
++    ScopedPyObjectPtr collections(PyImport_ImportModule("collections.abc"));
++#else
+     ScopedPyObjectPtr collections(PyImport_ImportModule("collections"));
++#endif
+     if (collections == NULL) {
+       return false;
+     }
+EOM
+
+note_build_stage "Set pyparsing to 2.2.2 for CLIF."
 export PATH="$HOME/.local/bin":$PATH
-pip3 uninstall -y pyparsing && pip3 install -Iv 'pyparsing==2.2.0'
+pip3 uninstall -y pyparsing && pip3 install -Iv 'pyparsing==2.2.2'
 
 note_build_stage "build-prereq.sh complete"

deepvariant/call_variants.py

@@ -739,6 +739,8 @@ def call_variants(
 
 
 def main(argv=()):
+  env = os.environ.copy()
+  logging.info('call_variants: env = %s', env)
   with errors.clean_commandline_error_exit():
     if len(argv) > 1:
       errors.log_and_raise(

run-prereq.sh

@@ -82,10 +82,6 @@ sudo -H apt-get install "${APT_ARGS[@]}" python3-distutils > /dev/null
 
 note_build_stage "Install python3 packaging infrastructure"
 
-sudo -H apt-get install "${APT_ARGS[@]}" "python${PYTHON_VERSION}-dev"
-sudo update-alternatives --install /usr/bin/python3 python3 "/usr/bin/python${PYTHON_VERSION}" 0
-sudo update-alternatives --install /usr/bin/python python "/usr/bin/python${PYTHON_VERSION}" 0
-
 # Avoid issue with pip's dependency resolver not accounting for all installed
 # packages.
 sudo -H apt-get install "${APT_ARGS[@]}" "python3-testresources"
@@ -143,7 +139,7 @@ pip3 install "${PIP_ARGS[@]}" 'six>=1.11.0'
 pip3 install "${PIP_ARGS[@]}" joblib
 pip3 install "${PIP_ARGS[@]}" psutil
 pip3 install "${PIP_ARGS[@]}" --upgrade google-api-python-client
-pip3 install "${PIP_ARGS[@]}" 'pandas==1.2.4'
+pip3 install "${PIP_ARGS[@]}" 'pandas==1.3.4'
 # We manually install jsonschema here to pin it to v3.2.0, since
 # the latest v4.0.1 has issues with Altair v4.1.0.
 # See https://github.com/altair-viz/altair/issues/2496
@@ -155,7 +151,9 @@ pip3 install "${PIP_ARGS[@]}" 'Pillow==9.5.0'
 pip3 install "${PIP_ARGS[@]}" 'ipython>=7.9.0'
 pip3 install "${PIP_ARGS[@]}" 'pysam==0.20.0'
 pip3 install "${PIP_ARGS[@]}" 'tensorflow-addons==0.21.0'
-pip3 install "${PIP_ARGS[@]}"  "tf-models-official==${DV_GCP_OPTIMIZED_TF_WHL_VERSION}"
+# This is to avoid ERROR: No matching distribution found for opencv-python-headless==4.5.2.52.
+# TODO: Make this the same as ${DV_GCP_OPTIMIZED_TF_WHL_VERSION}" later
+pip3 install "${PIP_ARGS[@]}"  "tf-models-official==2.13.1"
 
 ################################################################################
 # TensorFlow
@@ -212,6 +210,7 @@ note_build_stage "Install CUDA"
 # See https://www.tensorflow.org/install/source#gpu for versions required.
 if [[ "${DV_GPU_BUILD}" = "1" ]]; then
   if [[ "${DV_INSTALL_GPU_DRIVERS}" = "1" ]]; then
+    # TODO: This will be updated in internal.
     # This script is only maintained for Ubuntu 20.04.
     UBUNTU_VERSION="2004"
     # https://developer.nvidia.com/cuda-downloads?target_os=Linux&target_arch=x86_64&Distribution=Ubuntu&target_version=20.04&target_type=deb_local
@@ -291,10 +290,10 @@ fi
 note_build_stage "Install other packages"
 
 # for htslib
-sudo -H apt-get install "${APT_ARGS[@]}" libssl-dev libcurl4-openssl-dev liblz-dev libbz2-dev liblzma-dev > /dev/null
+sudo -H NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" libssl-dev libcurl4-openssl-dev liblz-dev libbz2-dev liblzma-dev > /dev/null
 
 # for the debruijn graph
-sudo -H apt-get install "${APT_ARGS[@]}" libboost-graph-dev > /dev/null
+sudo -H NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" libboost-graph-dev > /dev/null
 
 # Just being safe, pin protobuf's version one more time.
 pip3 install "${PIP_ARGS[@]}" 'protobuf==3.13.0'

scripts/run_deepsomatic.py

@@ -682,10 +682,7 @@ def main(_):
       'in docker. ****\n'.format(intermediate_results_dir)
   )
   env = os.environ.copy()
-  included_env = None
-  if 'TF_ENABLE_ONEDNN_OPTS' in env:
-    included_env = {}
-    included_env['TF_ENABLE_ONEDNN_OPTS'] = env['TF_ENABLE_ONEDNN_OPTS']
+  logging.info('env = %s', env)
   for command, logfile in commands_logfiles:
     print('\n***** Running the command:*****\n{}\n'.format(command))
     if not _DRY_RUN.value:
@@ -698,7 +695,7 @@ def main(_):
           shell=True,
           executable='/bin/bash',
           universal_newlines=True,
-          env=included_env,
+          env=env,
       ) as proc:
         for line in proc.stdout:
           print(line, end='')

scripts/run_deeptrio.py

@@ -1014,17 +1014,14 @@ def run_commands(
   """
 
   env = os.environ.copy()
-  included_env = None
-  if 'TF_ENABLE_ONEDNN_OPTS' in env:
-    included_env = {}
-    included_env['TF_ENABLE_ONEDNN_OPTS'] = env['TF_ENABLE_ONEDNN_OPTS']
+  logging.info('env = %s', env)
   if sequential:
     for command in commands:
       print('\n***** Running the command:*****\n{}\n'.format(command))
       if not dry_run:
         try:
           subprocess.check_call(
-              command, shell=True, executable='/bin/bash', env=included_env
+              command, shell=True, executable='/bin/bash', env=env
           )
         except subprocess.CalledProcessError as e:
           logging.info(e.output)
@@ -1034,9 +1031,7 @@ def run_commands(
       print('\n***** Running the command:*****\n{}\n'.format(command))
     if not _DRY_RUN.value:
       tasks = [
-          subprocess.Popen(
-              command, shell=True, executable='/bin/bash', env=included_env
-          )
+          subprocess.Popen(command, shell=True, executable='/bin/bash', env=env)
           for command in commands
       ]
       failed_task_indices = [

scripts/run_deepvariant.py

@@ -695,10 +695,7 @@ def main(_):
       'in docker. ****\n'.format(intermediate_results_dir)
   )
   env = os.environ.copy()
-  included_env = None
-  if 'TF_ENABLE_ONEDNN_OPTS' in env:
-    included_env = {}
-    included_env['TF_ENABLE_ONEDNN_OPTS'] = env['TF_ENABLE_ONEDNN_OPTS']
+  logging.info('env = %s', env)
   for command, logfile in commands_logfiles:
     print('\n***** Running the command:*****\n{}\n'.format(command))
     if not _DRY_RUN.value:
@@ -711,7 +708,7 @@ def main(_):
           shell=True,
           executable='/bin/bash',
           universal_newlines=True,
-          env=included_env,
+          env=env,
       ) as proc:
         for line in proc.stdout:
           print(line, end='')

settings.sh

@@ -86,14 +86,14 @@ export DV_GPU_BUILD="${DV_GPU_BUILD:-0}"
 export GCP_OPTIMIZED_TF_WHL_FILENAME="tensorflow-${DV_GCP_OPTIMIZED_TF_WHL_VERSION}.deepvariant_gcp-cp27-none-linux_x86_64.whl"
 export GCP_OPTIMIZED_TF_WHL_PATH="${DV_PACKAGE_BUCKET_PATH}/tensorflow"
 export GCP_OPTIMIZED_TF_WHL_CURL_PATH="${DV_PACKAGE_CURL_PATH}/tensorflow"
-export DV_TF_NUMPY_VERSION="1.19.2"  # To match GCP_OPTIMIZED_TF_WHL_FILENAME
+export DV_TF_NUMPY_VERSION="1.21.2"  # Python 3.10 requires >= 1.21.2
 
 # Set this to 1 to make our prereq scripts install the CUDA libraries.
 # If you already have CUDA installed, such as on a properly provisioned
 # Docker image, it shouldn't be necessary.
 export DV_INSTALL_GPU_DRIVERS="${DV_INSTALL_GPU_DRIVERS:-0}"
 
-export PYTHON_VERSION=3.8
+export PYTHON_VERSION=3.10
 # shellcheck disable=SC2155
 export PYTHON_BIN_PATH="$(which python${PYTHON_VERSION})"
 export PYTHON_LIB_PATH="/usr/local/lib/python${PYTHON_VERSION}/dist-packages"

tools/build_clif.sh

@@ -36,10 +36,9 @@ echo ========== This script has been tested on Ubuntu18.04 and Ubuntu20.04.
 echo ========== See https://github.com/google/clif for how to build on different Unix distributions.
 echo ========== Run this script in root mode.
 
-CLIF_UBUNTU_VERSION="${CLIF_UBUNTU_VERSION-20.04}"
 ABSL_PIN="${ABSL_PIN-29bf8085f3bf17b84d30e34b3d7ff8248fda404e}"
 PROTOBUF_VERSION=3.13.0
-CLIF_PYTHON_VERSION="${CLIF_PYTHON_VERSION-3.8}"
+CLIF_PYTHON_VERSION="${CLIF_PYTHON_VERSION-3.10}"
 # CLIF_PIN can be set to a specific commit hash on
 # https://github.com/google/clif/commits/main.
 # If not set, the default is to checkout the latest commit.
@@ -51,7 +50,7 @@ APT_ARGS=(
 
 
 apt-get update  "${APT_ARGS[@]}"
-apt-get install "${APT_ARGS[@]}" --no-install-recommends \
+NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" --no-install-recommends \
     autoconf \
     automake \
     cmake \
@@ -65,13 +64,9 @@ apt-get install "${APT_ARGS[@]}" --no-install-recommends \
     wget \
     unzip
 
-# Configure LLVM 11 apt repository
-wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key |  apt-key add - && \
-  add-apt-repository "deb http://apt.llvm.org/$(lsb_release -sc)/ llvm-toolchain-$(lsb_release -sc)-11 main"
-
 # Install CLIF dependencies
 apt-get update "${APT_ARGS[@]}"
-apt-get install  "${APT_ARGS[@]}" \
+NEEDRESTART_MODE=a apt-get install "${APT_ARGS[@]}" \
     clang-11 \
     libclang-11-dev \
     libgoogle-glog-dev \
@@ -83,24 +78,6 @@ apt-get install  "${APT_ARGS[@]}" \
     python3-dev \
     zlib1g-dev
 
-# Uninstall an older version of libclang so that cmake uses the correct one.
-apt-get remove "${APT_ARGS[@]}" libclang-common-9-dev
-
-# Configure deadsnakes PPA with the more recent versions of python packaged for
-# Ubuntu. See https://launchpad.net/~deadsnakes/+archive/ubuntu/ppa
-apt-get update "${APT_ARGS[@]}" && \
-  apt-get install "${APT_ARGS[@]}" \
-    "python$CLIF_PYTHON_VERSION-dev" \
-    "python$CLIF_PYTHON_VERSION-distutils"
-
-# Install latest version of pip since the version on ubuntu could be outdated.
-# Uninstall setuptools before installation of pip according to this suggestion:
-# https://github.com/google/deepvariant/commit/5469fec47e57febba751c30f1eb587cbffea8d89#commitcomment-64556350
-curl -Ss -o get-pip.py https://bootstrap.pypa.io/get-pip.py && \
-    pip uninstall -y setuptools &&
-    "python$CLIF_PYTHON_VERSION" get-pip.py --force-reinstall && \
-    rm get-pip.py
-
 # Compile and install absl-cpp from source
 git clone https://github.com/abseil/abseil-cpp.git
 cd abseil-cpp
@@ -123,21 +100,27 @@ wget "https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOBUF_V
     make -j"$(nproc)" && \
     make install && \
     ldconfig && \
-    rm -rf "/protobuf-$PROTOBUF_VERSION" "/protobuf-cpp-$PROTOBUF_VERSION.tar.gz"
+    cd .. && \
+    rm -rf "protobuf-$PROTOBUF_VERSION" "protobuf-cpp-$PROTOBUF_VERSION.tar.gz"
 
 # Install googletest
 cd /usr/src/googletest && \
     cmake . && \
     make install
 
+curl -o get-pip.py https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py --force-reinstall --user
+rm -f get-pip.py
+
+export PATH="$HOME/.local/bin":$PATH
+echo "$(pip3 --version)"
+
 # Install python runtime and test dependencies
-"python$CLIF_PYTHON_VERSION" -m pip install \
+pip3 install \
     absl-py \
     parameterized \
     protobuf=="$PROTOBUF_VERSION" \
-    pyparsing==2.2.0
-
-DV_PLATFORM="ubuntu-${CLIF_UBUNTU_VERSION}"
+    pyparsing==2.2.2
 
 ln -sf /usr/bin/python$CLIF_PYTHON_VERSION /usr/local/bin/python3