Upgrade Python to 3.8 as a minimum. Also simplified Tracker implement…

…ation. (#3646)

Co-authored-by: Nicholas Nezis <[email protected]>
Co-authored-by: Saad Ur Rahman <[email protected]>
Co-authored-by: Nicholas Nezis nanezis <[email protected]>

.travis.yml

@@ -1,8 +1,6 @@
 ---
 
-group: edge
-
-dist: bionic
+dist: focal
 
 language: java
 
@@ -17,19 +15,16 @@ addons:
       - libtool-bin
       - libcppunit-dev
       - pkg-config
-      - python3-dev
-      - python3-pip
-      - python3-setuptools
-      - python3-wheel
-      - python3-venv
+      - python3.8-dev
+      - python3.8-venv
       - wget
       - zip
       - zlib1g-dev
       - google-perftools
       - libgoogle-perftools-dev
 
 env:
-  - BAZEL_VERSION=4.2.2 ENABLE_HEAPCHECK=1
+  - BAZEL_VERSION=4.2.2 ENABLE_HEAPCHECK=1 LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
 
 before_install:
   # download and install bazel

WORKSPACE

@@ -175,9 +175,8 @@ jar_jar_repositories()
 
 http_archive(
     name = "rules_python",
-    sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c",
-    strip_prefix = "rules_python-0.0.2",
-    url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz",
+    sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0",
+    url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz",
 )
 
 load("@rules_python//python:repositories.bzl", "py_repositories")
@@ -188,58 +187,85 @@ py_repositories()
 # pip_repositories()
 
 # for pex repos
-PEX_WHEEL = "https://pypi.python.org/packages/fa/c4/5dbdce75117b60b6ffec65bc92ac25ee873b84158a55cfbffa1d49db6eb1/pex-2.1.54-py2.py3-none-any.whl"
+PEX_PKG = "https://files.pythonhosted.org/packages/d4/73/4c76e06824baadba81b39125721c97fb22e201b35fcd17b32b5a5fa77c59/pex-2.1.62-py2.py3-none-any.whl"
 
-PY_WHEEL = "https://pypi.python.org/packages/53/67/9620edf7803ab867b175e4fd23c7b8bd8eba11cb761514dcd2e726ef07da/py-1.4.34-py2.py3-none-any.whl"
+PYTEST_PKG = "https://files.pythonhosted.org/packages/40/76/86f886e750b81a4357b6ed606b2bcf0ce6d6c27ad3c09ebf63ed674fc86e/pytest-6.2.5-py3-none-any.whl"
 
-PYTEST_WHEEL = "https://pypi.python.org/packages/fd/3e/d326a05d083481746a769fc051ae8d25f574ef140ad4fe7f809a2b63c0f0/pytest-3.1.3-py2.py3-none-any.whl"
+REQUESTS_PKG = "https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl"
 
-REQUESTS_SRC = "https://pypi.python.org/packages/d9/03/155b3e67fe35fe5b6f4227a8d9e96a14fda828b18199800d161bcefc1359/requests-2.12.3.tar.gz"
+SETUPTOOLS_PKG = "https://files.pythonhosted.org/packages/3d/f2/1489d3b6c72d68bf79cd0fba6b6c7497df4ebf7d40970e2d7eceb8d0ea9c/setuptools-51.0.0-py3-none-any.whl"
 
-SETUPTOOLS_WHEEL = "https://pypi.python.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl"
+WHEEL_PKG = "https://files.pythonhosted.org/packages/d4/cf/732e05dce1e37b63d54d1836160b6e24fb36eeff2313e93315ad047c7d90/wheel-0.36.1.tar.gz"
 
-WHEEL_SRC = "https://pypi.python.org/packages/c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/wheel-0.29.0.tar.gz"
+CHARSET_PKG = "https://files.pythonhosted.org/packages/84/3e/1037abe6498e65d645ce7a22d3402605d49a3b2c7f20c3abb027760da4f0/charset_normalizer-2.0.10-py3-none-any.whl"
+
+IDNA_PKG = "https://files.pythonhosted.org/packages/04/a2/d918dcd22354d8958fe113e1a3630137e0fc8b44859ade3063982eacd2a4/idna-3.3-py3-none-any.whl"
+
+CERTIFI_PKG = "https://files.pythonhosted.org/packages/37/45/946c02767aabb873146011e665728b680884cd8fe70dde973c640e45b775/certifi-2021.10.8-py2.py3-none-any.whl"
+
+URLLIB3_PKG = "https://files.pythonhosted.org/packages/4e/b8/f5a25b22e803f0578e668daa33ba3701bb37858ec80e08a150bd7d2cf1b1/urllib3-1.26.8-py2.py3-none-any.whl"
+
+http_file(
+    name = "urllib3_pkg",
+    downloaded_file_path = "urllib3-1.26.8-py2.py3-none-any.whl",
+    sha256 = "000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed",
+    urls = [URLLIB3_PKG],
+)
+
+http_file(
+    name = "certifi_pkg",
+    downloaded_file_path = "certifi-2021.10.8-py2.py3-none-any.whl",
+    sha256 = "d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569",
+    urls = [CERTIFI_PKG],
+)
+
+http_file(
+    name = "idna_pkg",
+    downloaded_file_path = "idna-3.3-py2.py3-none-any.whl",
+    sha256 = "84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
+    urls = [IDNA_PKG],
+)
 
 http_file(
-    name = "pytest_whl",
-    downloaded_file_path = "pytest-3.1.3-py2.py3-none-any.whl",
-    sha256 = "2a4f483468954621fcc8f74784f3b42531e5b5008d49fc609b37bc4dbc6dead1",
-    urls = [PYTEST_WHEEL],
+    name = "charset_pkg",
+    downloaded_file_path = "charset_normalizer-2.0.10-py3-none-any.whl",
+    sha256 = "cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455",
+    urls = [CHARSET_PKG],
 )
 
 http_file(
-    name = "py_whl",
-    downloaded_file_path = "py-1.4.34-py2.py3-none-any.whl",
-    sha256 = "2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a",
-    urls = [PY_WHEEL],
+    name = "pytest_pkg",
+    downloaded_file_path = "pytest-6.2.5-py3-none-any.whl",
+    sha256 = "7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134",
+    urls = [PYTEST_PKG],
 )
 
 http_file(
-    name = "wheel_src",
-    downloaded_file_path = "wheel-0.29.0.tar.gz",
-    sha256 = "1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648",
-    urls = [WHEEL_SRC],
+    name = "wheel_pkg",
+    downloaded_file_path = "wheel-0.36.1.tar.gz",
+    sha256 = "aaef9b8c36db72f8bf7f1e54f85f875c4d466819940863ca0b3f3f77f0a1646f",
+    urls = [WHEEL_PKG],
 )
 
 http_file(
-    name = "pex_src",
-    downloaded_file_path = "pex-2.1.54-py2.py3-none-any.whl",
-    sha256 = "e60b006abe8abfd3c3377128e22c33f30cc6dea89e2beb463cf8360e3626db62",
-    urls = [PEX_WHEEL],
+    name = "pex_pkg",
+    downloaded_file_path = "pex-2.1.62-py2.py3-none-any.whl",
+    sha256 = "7667c6c6d7a9b07c3ff3c3125c1928bd5279dfc077dd5cf4cc0440f40427c484",
+    urls = [PEX_PKG],
 )
 
 http_file(
-    name = "requests_src",
-    downloaded_file_path = "requests-2.12.3.tar.gz",
-    sha256 = "de5d266953875e9647e37ef7bfe6ef1a46ff8ddfe61b5b3652edf7ea717ee2b2",
-    urls = [REQUESTS_SRC],
+    name = "requests_pkg",
+    downloaded_file_path = "requests-2.27.1-py2.py3-none-any.whl",
+    sha256 = "f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d",
+    urls = [REQUESTS_PKG],
 )
 
 http_file(
-    name = "setuptools_wheel",
-    downloaded_file_path = "setuptools-46.1.3-py3-none-any.whl",
-    sha256 = "4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee",
-    urls = [SETUPTOOLS_WHEEL],
+    name = "setuptools_pkg",
+    downloaded_file_path = "setuptools-51.0.0-py3-none-any.whl",
+    sha256 = "8c177936215945c9a37ef809ada0fab365191952f7a123618432bbfac353c529",
+    urls = [SETUPTOOLS_PKG],
 )
 
 # end pex repos
@@ -373,13 +399,12 @@ http_archive(
 # end helm
 
 # for docker image building
-DOCKER_RULES_VERSION = "0.14.4"
 
 http_archive(
     name = "io_bazel_rules_docker",
-    sha256 = "4521794f0fba2e20f3bf15846ab5e01d5332e587e9ce81629c7f96c793bb7036",
-    strip_prefix = "rules_docker-%s" % DOCKER_RULES_VERSION,
-    urls = ["https://github.com/bazelbuild/rules_docker/archive/v%s.tar.gz" % DOCKER_RULES_VERSION],
+    sha256 = "59536e6ae64359b716ba9c46c39183403b01eabfbd57578e84398b4829ca499a",
+    strip_prefix = "rules_docker-0.22.0",
+    urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.22.0/rules_docker-v0.22.0.tar.gz"],
 )
 
 load(
@@ -392,9 +417,9 @@ load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps")
 
 container_deps()
 
-load("@io_bazel_rules_docker//repositories:pip_repositories.bzl", "pip_deps")
+load("@io_bazel_rules_docker//repositories:py_repositories.bzl", "py_deps")
 
-pip_deps()
+py_deps()
 
 load(
     "@io_bazel_rules_docker//container:container.bzl",

bazel_configure.py

@@ -25,9 +25,8 @@
 # on a mac. Then verify the other environments by doing this:
 #
 #  cd docker
-#  ./build-artifacts.sh ubuntu15.10 0.12.0 .
-#  ./build-artifacts.sh ubuntu14.04 0.12.0 .
-#  ./build-artifacts.sh centos7 0.12.0 .
+#  ./build-artifacts.sh ubuntu20.04 0.12.0 .
+#  ./build-artifacts.sh centos8 0.12.0 .
 #
 import os
 import re
@@ -419,7 +418,7 @@ def main():
   env_map['AUTOMAKE'] = discover_tool('automake', 'Automake', 'AUTOMAKE', '1.9.6')
   env_map['AUTOCONF'] = discover_tool('autoconf', 'Autoconf', 'AUTOCONF', '2.6.3')
   env_map['MAKE'] = discover_tool('make', 'Make', 'MAKE', '3.81')
-  env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.6')
+  env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.8')
   test_venv()
 
   if platform == 'Darwin':

docker/Readme.md

@@ -22,11 +22,11 @@
 Make sure enough resources are configured in Docker settings: 2 CPU, 4G RAM and 128G disk.
 ```
 ./docker/scripts/build-artifacts.sh <platform> <version_string> [source-tarball] <output-directory>
-# e.g.  ./docker/scripts/build-artifacts.sh ubuntu14.04 testbuild ~/heron-release
+# e.g.  ./docker/scripts/build-artifacts.sh ubuntu20.04 testbuild ~/heron-release
 ```
 
 ## To build docker containers for running heron daemons:
 ```
 ./docker/scripts/build-docker.sh <platform> <version_string> <output-directory>
-# e.g. ./docker/scripts/build-docker.sh ubuntu14.04 testbuild ~/heron-release
+# e.g. ./docker/scripts/build-docker.sh ubuntu20.04 testbuild ~/heron-release
 ```

docker/base/Dockerfile.base.debian9 → docker/base/Dockerfile.base.debian10

@@ -15,7 +15,7 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-FROM openjdk:11-jdk-slim-stretch
+FROM openjdk:11.0.6-jdk-buster
 
 RUN apt-get -y update && apt-get -y install \
     ant \

docker/compile/Dockerfile.centos7 → docker/compile/Dockerfile.centos8

@@ -15,10 +15,10 @@
 #  specific language governing permissions and limitations
 #  under the License.
 
-FROM centos:centos7
+FROM centos:centos8
 
 # This is passed to the heron build command via the --config flag
-ENV TARGET_PLATFORM centos
+ENV TARGET_PLATFORM linux
 
 RUN yum -y upgrade
 RUN yum -y install \
@@ -35,7 +35,7 @@ RUN yum -y install \
       libtool \
       make \
       patch \
-      python3-devel \
+      python39-devel \
       zip \
       unzip \
       wget \
@@ -44,6 +44,8 @@ RUN yum -y install \
       java-11-openjdk \
       java-11-openjdk-devel
 
+RUN update-alternatives --set python /usr/bin/python3.9
+
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk
 
 ENV bazelVersion 4.2.2

docker/compile/Dockerfile.debian10

@@ -18,7 +18,7 @@
 FROM openjdk:11.0.6-jdk-buster
 
 # This is passed to the heron build command via the --config flag
-ENV TARGET_PLATFORM debian
+ENV TARGET_PLATFORM linux
 
 RUN apt-get update && apt-get -y install \
       ant \

docker/compile/Dockerfile.ubuntu18.04

@@ -18,7 +18,7 @@
 FROM ubuntu:18.04
 
 # This is passed to the heron build command via the --config flag
-ENV TARGET_PLATFORM ubuntu
+ENV TARGET_PLATFORM linux
 
 RUN apt-get update && apt-get -y install \
       ant \
@@ -28,8 +28,9 @@ RUN apt-get update && apt-get -y install \
       libtool-bin \
       libunwind8 \
       patch \
-      python3-dev \
-      python3-venv \
+      python3.8-dev \
+      python3.8-venv \
+      python3.8-distutil \
       pkg-config \
       wget \
       zip \
@@ -39,7 +40,8 @@ RUN apt-get update && apt-get -y install \
       tree \
       openjdk-11-jdk-headless
 
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10
 
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 

docker/compile/Dockerfile.ubuntu20.04

@@ -18,7 +18,7 @@
 FROM ubuntu:20.04
 
 # This is passed to the heron build command via the --config flag
-ENV TARGET_PLATFORM ubuntu
+ENV TARGET_PLATFORM linux
 
 ARG DEBIAN_FRONTEND=noninteractive
 

docker/dist/Dockerfile.dist.centos7 → docker/dist/Dockerfile.dist.centos8

@@ -16,7 +16,7 @@
 #  under the License.
 
 #syntax=docker/dockerfile:1.2
-FROM centos:centos7
+FROM centos:centos8
 
 ENV LC_ALL en_US.utf8
 
@@ -26,14 +26,16 @@ RUN yum -y install epel-release \
     java-11-openjdk-headless \
     supervisor \
     nmap-ncat \
-    python3 \
-    python3-setuptools \
+    python39 \
+    python39-setuptools \
     unzip \
     which \
     && yum clean all
 
 ENV JAVA_HOME /usr/
 
+RUN update-alternatives --set python /usr/bin/python3.9
+
 # run Heron installer
 RUN --mount=type=bind,source=artifacts,target=/tmp/heron /tmp/heron/heron-install.sh \
     && rm -f /usr/local/heron/dist/heron-core.tar.gz

docker/dist/Dockerfile.dist.ubuntu18.04

@@ -25,12 +25,15 @@ RUN apt-get -y update \
     curl \
     netcat-openbsd \
     openjdk-11-jdk-headless \
-    python3 \
+    python3.8 \
     python3-distutils \
     supervisor \
     unzip \
     && apt-get clean
 
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10
+
 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64
 RUN update-ca-certificates -f
 

docker/scripts/build-artifacts.sh

@@ -166,10 +166,10 @@ case $# in
     echo "  "
     echo "Script to build heron artifacts for different platforms"
     echo "  "
-    echo "Platforms Supported: darwin, debian9, ubuntu14.04, ubuntu16.04, ubuntu18.04, centos7"
+    echo "Platforms Supported: darwin, debian10, ubuntu20.04, centos8"
     echo "  "
     echo "Example:"
-    echo "  ./build-artifacts.sh ubuntu14.04 0.12.0 ."
+    echo "  ./build-artifacts.sh ubuntu20.04 0.12.0 ."
     echo "  "
     echo "NOTE: If running on OSX, the output directory will need to "
     echo "      be under /Users so virtualbox has access to."

docker/scripts/build-base.sh

@@ -59,7 +59,7 @@ run_build() {
   DOCKER_TAG="heron/base:$HERON_VERSION"
   DOCKER_LATEST_TAG="heron/base:latest"
 
-  if [ "$TARGET_PLATFORM" == "debian9" ]; then
+  if [ "$TARGET_PLATFORM" == "debian10" ]; then
     DOCKER_TAG="apache/heron:$HERON_VERSION"
     DOCKER_LATEST_TAG="apache/heron:latest"
     DOCKER_IMAGE_FILE="$OUTPUT_DIRECTORY/base-$HERON_VERSION.tar"
@@ -91,10 +91,10 @@ case $# in
     echo "  "
     echo "Usage: $0 <platform> <version_string> <output-directory>"
     echo "  "
-    echo "Platforms Supported: darwin, debian9, ubuntu14.04, ubuntu16.04, ubuntu18.04, centos7"
+    echo "Platforms Supported: darwin, debian10, ubuntu20.04, centos8"
     echo "  "
     echo "Example:"
-    echo "  ./build-base.sh ubuntu14.04 0.12.0 ~/ubuntu"
+    echo "  ./build-base.sh ubuntu20.04 0.12.0 ~/ubuntu"
     echo "  "
     exit 1
     ;;