From abb2767e3df3ca6eba009f46efe1f1e83695617a Mon Sep 17 00:00:00 2001 From: Oliver Bristow Date: Tue, 25 Jan 2022 16:37:11 +0000 Subject: [PATCH] Upgrade Python to 3.8 as a minimum. Also simplified Tracker implementation. (#3646) Co-authored-by: Nicholas Nezis Co-authored-by: Saad Ur Rahman Co-authored-by: Nicholas Nezis nanezis --- .travis.yml | 13 +- WORKSPACE | 103 +- bazel_configure.py | 7 +- docker/Readme.md | 4 +- ....base.debian9 => Dockerfile.base.debian10} | 2 +- ...{Dockerfile.centos7 => Dockerfile.centos8} | 8 +- docker/compile/Dockerfile.debian10 | 2 +- docker/compile/Dockerfile.ubuntu18.04 | 10 +- docker/compile/Dockerfile.ubuntu20.04 | 2 +- ...e.dist.centos7 => Dockerfile.dist.centos8} | 8 +- docker/dist/Dockerfile.dist.ubuntu18.04 | 5 +- docker/scripts/build-artifacts.sh | 4 +- docker/scripts/build-base.sh | 6 +- docker/scripts/build-docker.sh | 4 +- docker/scripts/build-exec-docker.sh | 4 +- docker/scripts/ci-docker.sh | 10 +- docker/scripts/compile-platform.sh | 16 +- docker/scripts/dev-env-create.sh | 6 +- docker/scripts/test-platform.sh | 14 +- docker/scripts/test-unittest.sh | 4 +- ...{Dockerfile.centos7 => Dockerfile.centos8} | 10 +- docker/test/Dockerfile.ubuntu18.04 | 2 +- heron/common/tests/python/pex_loader/BUILD | 2 +- heron/executor/src/python/BUILD | 2 +- heron/executor/src/python/heron_executor.py | 4 +- heron/executor/tests/python/BUILD | 2 +- heron/instance/src/python/BUILD | 2 +- heron/instance/src/python/instance.py | 2 +- heron/instance/tests/python/BUILD | 2 +- heron/instance/tests/python/network/BUILD | 14 +- heron/instance/tests/python/utils/BUILD | 20 +- heron/proto/BUILD | 4 +- heron/shell/src/python/BUILD | 4 +- heron/statemgrs/src/python/BUILD | 2 +- heron/statemgrs/src/python/configloader.py | 2 +- heron/statemgrs/tests/python/BUILD | 6 +- heron/tools/cli/src/python/BUILD | 4 +- heron/tools/cli/src/python/cliconfig.py | 2 +- heron/tools/cli/tests/python/BUILD | 4 +- heron/tools/common/src/python/BUILD | 2 +- .../common/src/python/clients/tracker.py | 4 +- heron/tools/common/src/python/utils/config.py | 6 +- heron/tools/explorer/src/python/BUILD | 3 +- heron/tools/explorer/tests/python/BUILD | 2 +- heron/tools/tracker/src/python/BUILD | 8 +- heron/tools/tracker/src/python/app.py | 142 + heron/tools/tracker/src/python/config.py | 45 +- heron/tools/tracker/src/python/constants.py | 2 +- .../tracker/src/python/handlers/__init__.py | 45 - .../src/python/handlers/basehandler.py | 273 -- .../src/python/handlers/clustershandler.py | 43 - .../python/handlers/containerfilehandler.py | 183 - .../src/python/handlers/defaulthandler.py | 40 - .../src/python/handlers/exceptionhandler.py | 133 - .../handlers/exceptionsummaryhandler.py | 129 - .../python/handlers/executionstatehandler.py | 60 - .../src/python/handlers/jmaphandler.py | 94 - .../src/python/handlers/jstackhandler.py | 93 - .../src/python/handlers/logicalplanhandler.py | 120 - .../src/python/handlers/machineshandler.py | 103 - .../src/python/handlers/mainhandler.py | 33 - .../python/handlers/memoryhistogramhandler.py | 94 - .../src/python/handlers/metadatahandler.py | 76 - .../src/python/handlers/metricshandler.py | 171 - .../python/handlers/metricsqueryhandler.py | 120 - .../python/handlers/metricstimelinehandler.py | 79 - .../src/python/handlers/packingplanhandler.py | 61 - .../python/handlers/physicalplanhandler.py | 61 - .../tracker/src/python/handlers/pidhandler.py | 90 - .../python/handlers/runtimestatehandler.py | 124 - .../handlers/schedulerlocationhandler.py | 61 - .../src/python/handlers/stateshandler.py | 99 - .../src/python/handlers/topologieshandler.py | 108 - .../python/handlers/topologyconfighandler.py | 61 - .../src/python/handlers/topologyhandler.py | 61 - heron/tools/tracker/src/python/main.py | 109 +- .../tracker/src/python/metricstimeline.py | 95 +- heron/tools/tracker/src/python/query.py | 247 +- .../tracker/src/python/query_operators.py | 103 +- .../tracker/src/python/routers/container.py | 326 ++ .../tracker/src/python/routers/metrics.py | 186 + .../tracker/src/python/routers/topologies.py | 179 + heron/tools/tracker/src/python/state.py | 8 + heron/tools/tracker/src/python/topology.py | 744 +++- heron/tools/tracker/src/python/tracker.py | 579 +-- heron/tools/tracker/src/python/utils.py | 125 +- heron/tools/tracker/tests/python/BUILD | 26 +- .../tracker/tests/python/app_unittest.py | 52 + .../tests/python/query_operator_unittest.py | 3266 ++++++++--------- .../tracker/tests/python/query_unittest.py | 248 +- .../tracker/tests/python/topology_unittest.py | 256 +- .../tracker/tests/python/tracker_unittest.py | 423 +-- heron/tools/ui/src/python/BUILD | 4 +- heron/tools/ui/src/python/main.py | 1 - heronpy/api/tests/python/BUILD | 10 +- heronpy/proto/BUILD | 8 +- integration_test/src/python/http_server/BUILD | 2 +- scripts/ci/README.md | 6 +- scripts/ci/build_docker_image.sh | 2 +- scripts/ci/build_release_packages.sh | 2 +- scripts/detect_os_type.sh | 5 +- scripts/images/BUILD | 2 +- scripts/release_check/README.md | 2 +- scripts/release_check/build_docker.sh | 2 +- scripts/release_check/full_release_check.sh | 2 +- scripts/shutils/common.sh | 17 +- tools/bazel.rc | 96 +- tools/docker/bazel.rc | 2 +- tools/rules/pex/BUILD | 30 +- tools/rules/pex/pex_rules.bzl | 3 +- vagrant/init.sh | 2 +- website2/docs/compiling-docker.md | 19 +- website2/docs/compiling-linux.md | 12 +- .../docs/getting-started-local-single-node.md | 2 +- website2/docs/user-manuals-tracker-rest.md | 648 +--- website2/website/pages/en/download.js | 12 +- website2/website/scripts/python-doc-gen.sh | 2 +- 117 files changed, 4335 insertions(+), 6681 deletions(-) rename docker/base/{Dockerfile.base.debian9 => Dockerfile.base.debian10} (97%) rename docker/compile/{Dockerfile.centos7 => Dockerfile.centos8} (92%) rename docker/dist/{Dockerfile.dist.centos7 => Dockerfile.dist.centos8} (94%) rename docker/test/{Dockerfile.centos7 => Dockerfile.centos8} (92%) create mode 100644 heron/tools/tracker/src/python/app.py delete mode 100644 heron/tools/tracker/src/python/handlers/__init__.py delete mode 100644 heron/tools/tracker/src/python/handlers/basehandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/clustershandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/containerfilehandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/defaulthandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/exceptionhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/executionstatehandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/jmaphandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/jstackhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/logicalplanhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/machineshandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/mainhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/metadatahandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/metricshandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/metricsqueryhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/metricstimelinehandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/packingplanhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/physicalplanhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/pidhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/runtimestatehandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/stateshandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/topologieshandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/topologyconfighandler.py delete mode 100644 heron/tools/tracker/src/python/handlers/topologyhandler.py create mode 100644 heron/tools/tracker/src/python/routers/container.py create mode 100644 heron/tools/tracker/src/python/routers/metrics.py create mode 100644 heron/tools/tracker/src/python/routers/topologies.py create mode 100644 heron/tools/tracker/src/python/state.py create mode 100644 heron/tools/tracker/tests/python/app_unittest.py diff --git a/.travis.yml b/.travis.yml index 3848fc1221b..a9190ad2c02 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,6 @@ --- -group: edge - -dist: bionic +dist: focal language: java @@ -17,11 +15,8 @@ addons: - libtool-bin - libcppunit-dev - pkg-config - - python3-dev - - python3-pip - - python3-setuptools - - python3-wheel - - python3-venv + - python3.8-dev + - python3.8-venv - wget - zip - zlib1g-dev @@ -29,7 +24,7 @@ addons: - libgoogle-perftools-dev env: - - BAZEL_VERSION=4.2.2 ENABLE_HEAPCHECK=1 + - BAZEL_VERSION=4.2.2 ENABLE_HEAPCHECK=1 LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 before_install: # download and install bazel diff --git a/WORKSPACE b/WORKSPACE index 7b941b211ae..65370cfbcc7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -175,9 +175,8 @@ jar_jar_repositories() http_archive( name = "rules_python", - sha256 = "b5668cde8bb6e3515057ef465a35ad712214962f0b3a314e551204266c7be90c", - strip_prefix = "rules_python-0.0.2", - url = "https://github.com/bazelbuild/rules_python/releases/download/0.0.2/rules_python-0.0.2.tar.gz", + sha256 = "b6d46438523a3ec0f3cead544190ee13223a52f6a6765a29eae7b7cc24cc83a0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.1.0/rules_python-0.1.0.tar.gz", ) load("@rules_python//python:repositories.bzl", "py_repositories") @@ -188,58 +187,85 @@ py_repositories() # pip_repositories() # for pex repos -PEX_WHEEL = "https://pypi.python.org/packages/fa/c4/5dbdce75117b60b6ffec65bc92ac25ee873b84158a55cfbffa1d49db6eb1/pex-2.1.54-py2.py3-none-any.whl" +PEX_PKG = "https://files.pythonhosted.org/packages/d4/73/4c76e06824baadba81b39125721c97fb22e201b35fcd17b32b5a5fa77c59/pex-2.1.62-py2.py3-none-any.whl" -PY_WHEEL = "https://pypi.python.org/packages/53/67/9620edf7803ab867b175e4fd23c7b8bd8eba11cb761514dcd2e726ef07da/py-1.4.34-py2.py3-none-any.whl" +PYTEST_PKG = "https://files.pythonhosted.org/packages/40/76/86f886e750b81a4357b6ed606b2bcf0ce6d6c27ad3c09ebf63ed674fc86e/pytest-6.2.5-py3-none-any.whl" -PYTEST_WHEEL = "https://pypi.python.org/packages/fd/3e/d326a05d083481746a769fc051ae8d25f574ef140ad4fe7f809a2b63c0f0/pytest-3.1.3-py2.py3-none-any.whl" +REQUESTS_PKG = "https://files.pythonhosted.org/packages/2d/61/08076519c80041bc0ffa1a8af0cbd3bf3e2b62af10435d269a9d0f40564d/requests-2.27.1-py2.py3-none-any.whl" -REQUESTS_SRC = "https://pypi.python.org/packages/d9/03/155b3e67fe35fe5b6f4227a8d9e96a14fda828b18199800d161bcefc1359/requests-2.12.3.tar.gz" +SETUPTOOLS_PKG = "https://files.pythonhosted.org/packages/3d/f2/1489d3b6c72d68bf79cd0fba6b6c7497df4ebf7d40970e2d7eceb8d0ea9c/setuptools-51.0.0-py3-none-any.whl" -SETUPTOOLS_WHEEL = "https://pypi.python.org/packages/a0/df/635cdb901ee4a8a42ec68e480c49f85f4c59e8816effbf57d9e6ee8b3588/setuptools-46.1.3-py3-none-any.whl" +WHEEL_PKG = "https://files.pythonhosted.org/packages/d4/cf/732e05dce1e37b63d54d1836160b6e24fb36eeff2313e93315ad047c7d90/wheel-0.36.1.tar.gz" -WHEEL_SRC = "https://pypi.python.org/packages/c9/1d/bd19e691fd4cfe908c76c429fe6e4436c9e83583c4414b54f6c85471954a/wheel-0.29.0.tar.gz" +CHARSET_PKG = "https://files.pythonhosted.org/packages/84/3e/1037abe6498e65d645ce7a22d3402605d49a3b2c7f20c3abb027760da4f0/charset_normalizer-2.0.10-py3-none-any.whl" + +IDNA_PKG = "https://files.pythonhosted.org/packages/04/a2/d918dcd22354d8958fe113e1a3630137e0fc8b44859ade3063982eacd2a4/idna-3.3-py3-none-any.whl" + +CERTIFI_PKG = "https://files.pythonhosted.org/packages/37/45/946c02767aabb873146011e665728b680884cd8fe70dde973c640e45b775/certifi-2021.10.8-py2.py3-none-any.whl" + +URLLIB3_PKG = "https://files.pythonhosted.org/packages/4e/b8/f5a25b22e803f0578e668daa33ba3701bb37858ec80e08a150bd7d2cf1b1/urllib3-1.26.8-py2.py3-none-any.whl" + +http_file( + name = "urllib3_pkg", + downloaded_file_path = "urllib3-1.26.8-py2.py3-none-any.whl", + sha256 = "000ca7f471a233c2251c6c7023ee85305721bfdf18621ebff4fd17a8653427ed", + urls = [URLLIB3_PKG], +) + +http_file( + name = "certifi_pkg", + downloaded_file_path = "certifi-2021.10.8-py2.py3-none-any.whl", + sha256 = "d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569", + urls = [CERTIFI_PKG], +) + +http_file( + name = "idna_pkg", + downloaded_file_path = "idna-3.3-py2.py3-none-any.whl", + sha256 = "84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff", + urls = [IDNA_PKG], +) http_file( - name = "pytest_whl", - downloaded_file_path = "pytest-3.1.3-py2.py3-none-any.whl", - sha256 = "2a4f483468954621fcc8f74784f3b42531e5b5008d49fc609b37bc4dbc6dead1", - urls = [PYTEST_WHEEL], + name = "charset_pkg", + downloaded_file_path = "charset_normalizer-2.0.10-py3-none-any.whl", + sha256 = "cb957888737fc0bbcd78e3df769addb41fd1ff8cf950dc9e7ad7793f1bf44455", + urls = [CHARSET_PKG], ) http_file( - name = "py_whl", - downloaded_file_path = "py-1.4.34-py2.py3-none-any.whl", - sha256 = "2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a", - urls = [PY_WHEEL], + name = "pytest_pkg", + downloaded_file_path = "pytest-6.2.5-py3-none-any.whl", + sha256 = "7310f8d27bc79ced999e760ca304d69f6ba6c6649c0b60fb0e04a4a77cacc134", + urls = [PYTEST_PKG], ) http_file( - name = "wheel_src", - downloaded_file_path = "wheel-0.29.0.tar.gz", - sha256 = "1ebb8ad7e26b448e9caa4773d2357849bf80ff9e313964bcaf79cbf0201a1648", - urls = [WHEEL_SRC], + name = "wheel_pkg", + downloaded_file_path = "wheel-0.36.1.tar.gz", + sha256 = "aaef9b8c36db72f8bf7f1e54f85f875c4d466819940863ca0b3f3f77f0a1646f", + urls = [WHEEL_PKG], ) http_file( - name = "pex_src", - downloaded_file_path = "pex-2.1.54-py2.py3-none-any.whl", - sha256 = "e60b006abe8abfd3c3377128e22c33f30cc6dea89e2beb463cf8360e3626db62", - urls = [PEX_WHEEL], + name = "pex_pkg", + downloaded_file_path = "pex-2.1.62-py2.py3-none-any.whl", + sha256 = "7667c6c6d7a9b07c3ff3c3125c1928bd5279dfc077dd5cf4cc0440f40427c484", + urls = [PEX_PKG], ) http_file( - name = "requests_src", - downloaded_file_path = "requests-2.12.3.tar.gz", - sha256 = "de5d266953875e9647e37ef7bfe6ef1a46ff8ddfe61b5b3652edf7ea717ee2b2", - urls = [REQUESTS_SRC], + name = "requests_pkg", + downloaded_file_path = "requests-2.27.1-py2.py3-none-any.whl", + sha256 = "f22fa1e554c9ddfd16e6e41ac79759e17be9e492b3587efa038054674760e72d", + urls = [REQUESTS_PKG], ) http_file( - name = "setuptools_wheel", - downloaded_file_path = "setuptools-46.1.3-py3-none-any.whl", - sha256 = "4fe404eec2738c20ab5841fa2d791902d2a645f32318a7850ef26f8d7215a8ee", - urls = [SETUPTOOLS_WHEEL], + name = "setuptools_pkg", + downloaded_file_path = "setuptools-51.0.0-py3-none-any.whl", + sha256 = "8c177936215945c9a37ef809ada0fab365191952f7a123618432bbfac353c529", + urls = [SETUPTOOLS_PKG], ) # end pex repos @@ -373,13 +399,12 @@ http_archive( # end helm # for docker image building -DOCKER_RULES_VERSION = "0.14.4" http_archive( name = "io_bazel_rules_docker", - sha256 = "4521794f0fba2e20f3bf15846ab5e01d5332e587e9ce81629c7f96c793bb7036", - strip_prefix = "rules_docker-%s" % DOCKER_RULES_VERSION, - urls = ["https://github.com/bazelbuild/rules_docker/archive/v%s.tar.gz" % DOCKER_RULES_VERSION], + sha256 = "59536e6ae64359b716ba9c46c39183403b01eabfbd57578e84398b4829ca499a", + strip_prefix = "rules_docker-0.22.0", + urls = ["https://github.com/bazelbuild/rules_docker/releases/download/v0.22.0/rules_docker-v0.22.0.tar.gz"], ) load( @@ -392,9 +417,9 @@ load("@io_bazel_rules_docker//repositories:deps.bzl", container_deps = "deps") container_deps() -load("@io_bazel_rules_docker//repositories:pip_repositories.bzl", "pip_deps") +load("@io_bazel_rules_docker//repositories:py_repositories.bzl", "py_deps") -pip_deps() +py_deps() load( "@io_bazel_rules_docker//container:container.bzl", diff --git a/bazel_configure.py b/bazel_configure.py index 95c83047453..95e3ee4b59e 100755 --- a/bazel_configure.py +++ b/bazel_configure.py @@ -25,9 +25,8 @@ # on a mac. Then verify the other environments by doing this: # # cd docker -# ./build-artifacts.sh ubuntu15.10 0.12.0 . -# ./build-artifacts.sh ubuntu14.04 0.12.0 . -# ./build-artifacts.sh centos7 0.12.0 . +# ./build-artifacts.sh ubuntu20.04 0.12.0 . +# ./build-artifacts.sh centos8 0.12.0 . # import os import re @@ -419,7 +418,7 @@ def main(): env_map['AUTOMAKE'] = discover_tool('automake', 'Automake', 'AUTOMAKE', '1.9.6') env_map['AUTOCONF'] = discover_tool('autoconf', 'Autoconf', 'AUTOCONF', '2.6.3') env_map['MAKE'] = discover_tool('make', 'Make', 'MAKE', '3.81') - env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.6') + env_map['PYTHON3'] = discover_tool('python3', 'Python3', 'PYTHON3', '3.8') test_venv() if platform == 'Darwin': diff --git a/docker/Readme.md b/docker/Readme.md index eaa6fcc3630..41355ff8981 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -22,11 +22,11 @@ Make sure enough resources are configured in Docker settings: 2 CPU, 4G RAM and 128G disk. ``` ./docker/scripts/build-artifacts.sh [source-tarball] -# e.g. ./docker/scripts/build-artifacts.sh ubuntu14.04 testbuild ~/heron-release +# e.g. ./docker/scripts/build-artifacts.sh ubuntu20.04 testbuild ~/heron-release ``` ## To build docker containers for running heron daemons: ``` ./docker/scripts/build-docker.sh -# e.g. ./docker/scripts/build-docker.sh ubuntu14.04 testbuild ~/heron-release +# e.g. ./docker/scripts/build-docker.sh ubuntu20.04 testbuild ~/heron-release ``` diff --git a/docker/base/Dockerfile.base.debian9 b/docker/base/Dockerfile.base.debian10 similarity index 97% rename from docker/base/Dockerfile.base.debian9 rename to docker/base/Dockerfile.base.debian10 index 840d112e9ab..d55e641c27d 100644 --- a/docker/base/Dockerfile.base.debian9 +++ b/docker/base/Dockerfile.base.debian10 @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -FROM openjdk:11-jdk-slim-stretch +FROM openjdk:11.0.6-jdk-buster RUN apt-get -y update && apt-get -y install \ ant \ diff --git a/docker/compile/Dockerfile.centos7 b/docker/compile/Dockerfile.centos8 similarity index 92% rename from docker/compile/Dockerfile.centos7 rename to docker/compile/Dockerfile.centos8 index e4fadfa5570..02ced5c923d 100644 --- a/docker/compile/Dockerfile.centos7 +++ b/docker/compile/Dockerfile.centos8 @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -FROM centos:centos7 +FROM centos:centos8 # This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM centos +ENV TARGET_PLATFORM linux RUN yum -y upgrade RUN yum -y install \ @@ -35,7 +35,7 @@ RUN yum -y install \ libtool \ make \ patch \ - python3-devel \ + python39-devel \ zip \ unzip \ wget \ @@ -44,6 +44,8 @@ RUN yum -y install \ java-11-openjdk \ java-11-openjdk-devel +RUN update-alternatives --set python /usr/bin/python3.9 + ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk ENV bazelVersion 4.2.2 diff --git a/docker/compile/Dockerfile.debian10 b/docker/compile/Dockerfile.debian10 index 1c6c11eeb21..9076ffdd8c9 100644 --- a/docker/compile/Dockerfile.debian10 +++ b/docker/compile/Dockerfile.debian10 @@ -18,7 +18,7 @@ FROM openjdk:11.0.6-jdk-buster # This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM debian +ENV TARGET_PLATFORM linux RUN apt-get update && apt-get -y install \ ant \ diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 index 937087242f1..e574aeeb055 100644 --- a/docker/compile/Dockerfile.ubuntu18.04 +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -18,7 +18,7 @@ FROM ubuntu:18.04 # This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM ubuntu +ENV TARGET_PLATFORM linux RUN apt-get update && apt-get -y install \ ant \ @@ -28,8 +28,9 @@ RUN apt-get update && apt-get -y install \ libtool-bin \ libunwind8 \ patch \ - python3-dev \ - python3-venv \ + python3.8-dev \ + python3.8-venv \ + python3.8-distutil \ pkg-config \ wget \ zip \ @@ -39,7 +40,8 @@ RUN apt-get update && apt-get -y install \ tree \ openjdk-11-jdk-headless -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 diff --git a/docker/compile/Dockerfile.ubuntu20.04 b/docker/compile/Dockerfile.ubuntu20.04 index 4131eb2fbd0..4bdd63ead3f 100644 --- a/docker/compile/Dockerfile.ubuntu20.04 +++ b/docker/compile/Dockerfile.ubuntu20.04 @@ -18,7 +18,7 @@ FROM ubuntu:20.04 # This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM ubuntu +ENV TARGET_PLATFORM linux ARG DEBIAN_FRONTEND=noninteractive diff --git a/docker/dist/Dockerfile.dist.centos7 b/docker/dist/Dockerfile.dist.centos8 similarity index 94% rename from docker/dist/Dockerfile.dist.centos7 rename to docker/dist/Dockerfile.dist.centos8 index 70ee110cbd4..3532a751821 100644 --- a/docker/dist/Dockerfile.dist.centos7 +++ b/docker/dist/Dockerfile.dist.centos8 @@ -16,7 +16,7 @@ # under the License. #syntax=docker/dockerfile:1.2 -FROM centos:centos7 +FROM centos:centos8 ENV LC_ALL en_US.utf8 @@ -26,14 +26,16 @@ RUN yum -y install epel-release \ java-11-openjdk-headless \ supervisor \ nmap-ncat \ - python3 \ - python3-setuptools \ + python39 \ + python39-setuptools \ unzip \ which \ && yum clean all ENV JAVA_HOME /usr/ +RUN update-alternatives --set python /usr/bin/python3.9 + # run Heron installer RUN --mount=type=bind,source=artifacts,target=/tmp/heron /tmp/heron/heron-install.sh \ && rm -f /usr/local/heron/dist/heron-core.tar.gz diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04 index d4ab6d0ebe6..e270f635009 100644 --- a/docker/dist/Dockerfile.dist.ubuntu18.04 +++ b/docker/dist/Dockerfile.dist.ubuntu18.04 @@ -25,12 +25,15 @@ RUN apt-get -y update \ curl \ netcat-openbsd \ openjdk-11-jdk-headless \ - python3 \ + python3.8 \ python3-distutils \ supervisor \ unzip \ && apt-get clean +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10 +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 10 + ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk-amd64 RUN update-ca-certificates -f diff --git a/docker/scripts/build-artifacts.sh b/docker/scripts/build-artifacts.sh index f7773f029b6..8aa361382df 100755 --- a/docker/scripts/build-artifacts.sh +++ b/docker/scripts/build-artifacts.sh @@ -166,10 +166,10 @@ case $# in echo " " echo "Script to build heron artifacts for different platforms" echo " " - echo "Platforms Supported: darwin, debian9, ubuntu14.04, ubuntu16.04, ubuntu18.04, centos7" + echo "Platforms Supported: darwin, debian10, ubuntu20.04, centos8" echo " " echo "Example:" - echo " ./build-artifacts.sh ubuntu14.04 0.12.0 ." + echo " ./build-artifacts.sh ubuntu20.04 0.12.0 ." echo " " echo "NOTE: If running on OSX, the output directory will need to " echo " be under /Users so virtualbox has access to." diff --git a/docker/scripts/build-base.sh b/docker/scripts/build-base.sh index 97450624aba..b0d3eb7fd20 100755 --- a/docker/scripts/build-base.sh +++ b/docker/scripts/build-base.sh @@ -59,7 +59,7 @@ run_build() { DOCKER_TAG="heron/base:$HERON_VERSION" DOCKER_LATEST_TAG="heron/base:latest" - if [ "$TARGET_PLATFORM" == "debian9" ]; then + if [ "$TARGET_PLATFORM" == "debian10" ]; then DOCKER_TAG="apache/heron:$HERON_VERSION" DOCKER_LATEST_TAG="apache/heron:latest" DOCKER_IMAGE_FILE="$OUTPUT_DIRECTORY/base-$HERON_VERSION.tar" @@ -91,10 +91,10 @@ case $# in echo " " echo "Usage: $0 " echo " " - echo "Platforms Supported: darwin, debian9, ubuntu14.04, ubuntu16.04, ubuntu18.04, centos7" + echo "Platforms Supported: darwin, debian10, ubuntu20.04, centos8" echo " " echo "Example:" - echo " ./build-base.sh ubuntu14.04 0.12.0 ~/ubuntu" + echo " ./build-base.sh ubuntu20.04 0.12.0 ~/ubuntu" echo " " exit 1 ;; diff --git a/docker/scripts/build-docker.sh b/docker/scripts/build-docker.sh index aed0bff5676..37f157a46a2 100755 --- a/docker/scripts/build-docker.sh +++ b/docker/scripts/build-docker.sh @@ -96,13 +96,13 @@ case $# in echo "Usage: $0 [-s|--squash]" echo " " echo "Argument options:" - echo " : darwin, debian9, debian10, ubuntu14.04, ubuntu18.04, centos7" + echo " : darwin, debian10, ubuntu20.04, centos8" echo " : Version of Heron build, e.g. v0.17.5.1-rc" echo " : Location of compiled Heron artifact" echo " [-s|--squash]: Enables using Docker experimental feature --squash" echo " " echo "Example:" - echo " ./build-docker.sh ubuntu18.04 0.12.0 ~/ubuntu" + echo " ./build-docker.sh ubuntu20.04 0.12.0 ~/ubuntu" echo " " exit 1 ;; diff --git a/docker/scripts/build-exec-docker.sh b/docker/scripts/build-exec-docker.sh index 066f192f82d..9532c251974 100755 --- a/docker/scripts/build-exec-docker.sh +++ b/docker/scripts/build-exec-docker.sh @@ -86,10 +86,10 @@ case $# in *) echo "Usage: $0 " echo " " - echo "Platforms Supported: darwin, debian9, ubuntu14.04, ubuntu16.04, ubuntu18.04, centos7" + echo "Platforms Supported: darwin, debian10, ubuntu20.04, centos8" echo " " echo "Example:" - echo " ./build-exec-docker.sh ubuntu14.04 0.12.0 ." + echo " ./build-exec-docker.sh ubuntu20.04 0.12.0 ." echo " " exit 1 ;; diff --git a/docker/scripts/ci-docker.sh b/docker/scripts/ci-docker.sh index e57392945c8..4307cffb517 100755 --- a/docker/scripts/ci-docker.sh +++ b/docker/scripts/ci-docker.sh @@ -50,7 +50,7 @@ build_exec_image() { OUTPUT_DIRECTORY=$(realpath $4) if [ "$INPUT_TARGET_PLATFORM" == "latest" ]; then - TARGET_PLATFORM="ubuntu14.04" + TARGET_PLATFORM="ubuntu20.04" DOCKER_TAG="$DOCKER_TAG_PREFIX/heron:$HERON_VERSION" DOCKER_LATEST_TAG="$DOCKER_TAG_PREFIX/heron:latest" DOCKER_IMAGE_FILE="$OUTPUT_DIRECTORY/heron-$HERON_VERSION.tar" @@ -97,7 +97,7 @@ publish_exec_image() { INPUT_DIRECTORY=$(realpath $4) if [ "$INPUT_TARGET_PLATFORM" == "latest" ]; then - TARGET_PLATFORM="ubuntu14.04" + TARGET_PLATFORM="ubuntu20.04" DOCKER_TAG="$DOCKER_TAG_PREFIX/heron:$HERON_VERSION" DOCKER_LATEST_TAG="$DOCKER_TAG_PREFIX/heron:latest" DOCKER_IMAGE_FILE="$INPUT_DIRECTORY/heron-$HERON_VERSION.tar" @@ -139,11 +139,11 @@ case $# in *) echo "Usage: $0 " echo " " - echo "Platforms Supported: latest, ubuntu14.04, ubuntu16.04, centos7, debian9" + echo "Platforms Supported: latest, ubuntu20.04, centos8, debian10" echo " " echo "Example:" - echo " $0 build ubuntu14.04 0.12.0 heron ." - echo " $0 publish ubuntu14.04 0.12.0 streamlio ~/ubuntu" + echo " $0 build ubuntu20.04 0.12.0 heron ." + echo " $0 publish ubuntu20.04 0.12.0 streamlio ~/ubuntu" echo " " exit 1 ;; diff --git a/docker/scripts/compile-platform.sh b/docker/scripts/compile-platform.sh index 34c44037859..e75f3df7548 100755 --- a/docker/scripts/compile-platform.sh +++ b/docker/scripts/compile-platform.sh @@ -37,21 +37,13 @@ echo "Extracting source" tar -C . -xzf $SOURCE_TARBALL if [[ "$TARGET_PLATFORM" =~ "ubuntu" ]]; then - CONFIG_PLATFORM=ubuntu_nostyle + CONFIG_PLATFORM=linux_nostyle elif [[ "$TARGET_PLATFORM" =~ "centos" ]]; then - CONFIG_PLATFORM=centos_nostyle -elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then - CONFIG_PLATFORM=darwin_nostyle + CONFIG_PLATFORM=linux_nostyle elif [[ "$TARGET_PLATFORM" =~ "debian" ]]; then - CONFIG_PLATFORM=debian_nostyle -elif [[ "$TARGET_PLATFORM" =~ "ubuntu_nostyle" ]]; then - CONFIG_PLATFORM=ubuntu_nostyle -elif [[ "$TARGET_PLATFORM" =~ "centos_nostyle" ]]; then - CONFIG_PLATFORM=centos_nostyle -elif [[ "$TARGET_PLATFORM" =~ "darwin_nostyle" ]]; then + CONFIG_PLATFORM=linux_nostyle +elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then CONFIG_PLATFORM=darwin_nostyle -elif [[ "$TARGET_PLATFORM" =~ "debian_nostyle" ]]; then - CONFIG_PLATFORM=debian_nostyle else echo "Unknown platform: $TARGET_PLATFORM" exit 1 diff --git a/docker/scripts/dev-env-create.sh b/docker/scripts/dev-env-create.sh index e94be511385..760a3378c2a 100755 --- a/docker/scripts/dev-env-create.sh +++ b/docker/scripts/dev-env-create.sh @@ -28,8 +28,8 @@ # After the container is started, you can build Heron with bazel # (ubuntu config is used in the example): # ./bazel_configure.py -# bazel build --config=ubuntu heron/... -# bazel build --config=ubuntu scripts/packages:binpkgs +# bazel build --config=linux heron/... +# bazel build --config=linux scripts/packages:binpkgs set -o nounset set -o errexit @@ -43,7 +43,7 @@ case $# in esac # Default platform is ubuntu18.04. Other available platforms -# include centos7, debian9, debian10, ubuntu18.04 +# include centos8, debian10, ubuntu18.04 TARGET_PLATFORM=${2:-"ubuntu18.04"} SCRATCH_DIR="$HOME/.heron-docker" REPOSITORY="heron-dev" diff --git a/docker/scripts/test-platform.sh b/docker/scripts/test-platform.sh index 85c5c06b6a3..70505b4cc16 100755 --- a/docker/scripts/test-platform.sh +++ b/docker/scripts/test-platform.sh @@ -37,21 +37,13 @@ echo "Extracting source" tar -C . -xzf $SOURCE_TARBALL if [[ "$TARGET_PLATFORM" =~ "ubuntu" ]]; then - CONFIG_PLATFORM=ubuntu + CONFIG_PLATFORM=linux_nostyle elif [[ "$TARGET_PLATFORM" =~ "centos" ]]; then - CONFIG_PLATFORM=centos + CONFIG_PLATFORM=linux_nostyle elif [[ "$TARGET_PLATFORM" =~ "darwin" ]]; then CONFIG_PLATFORM=darwin elif [[ "$TARGET_PLATFORM" =~ "debian" ]]; then - CONFIG_PLATFORM=debian -elif [[ "$TARGET_PLATFORM" =~ "ubuntu_nostyle" ]]; then - CONFIG_PLATFORM=ubuntu -elif [[ "$TARGET_PLATFORM" =~ "centos_nostyle" ]]; then - CONFIG_PLATFORM=centos -elif [[ "$TARGET_PLATFORM" =~ "darwin_nostyle" ]]; then - CONFIG_PLATFORM=darwin -elif [[ "$TARGET_PLATFORM" =~ "debian_nostyle" ]]; then - CONFIG_PLATFORM=debian + CONFIG_PLATFORM=linux_nostyle else echo "Unknown platform: $TARGET_PLATFORM" exit 1 diff --git a/docker/scripts/test-unittest.sh b/docker/scripts/test-unittest.sh index 0192a09fee7..aca54b06d60 100755 --- a/docker/scripts/test-unittest.sh +++ b/docker/scripts/test-unittest.sh @@ -163,10 +163,10 @@ case $# in echo " " echo "Script to test heron artifacts for different platforms" echo " " - echo "Platforms Supported: darwin, ubuntu18.04, centos7" + echo "Platforms Supported: darwin, ubuntu18.04, centos8" echo " " echo "Example:" - echo " ./test-unittest.sh centos7 0.20.2" + echo " ./test-unittest.sh centos8 0.20.2" echo " " echo "NOTE: If running on OSX, the output directory will need to " echo " be under /Users so virtualbox has access to." diff --git a/docker/test/Dockerfile.centos7 b/docker/test/Dockerfile.centos8 similarity index 92% rename from docker/test/Dockerfile.centos7 rename to docker/test/Dockerfile.centos8 index f08f1561637..6c68648113a 100644 --- a/docker/test/Dockerfile.centos7 +++ b/docker/test/Dockerfile.centos8 @@ -15,10 +15,10 @@ # specific language governing permissions and limitations # under the License. -FROM centos:centos7 +FROM centos:centos8 # This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM centos +ENV TARGET_PLATFORM linux RUN yum -y upgrade RUN yum -y install \ @@ -34,9 +34,7 @@ RUN yum -y install \ libtool \ make \ patch \ - python3-devel \ - python3-devel \ - python3-setuptools \ + python39-devel \ zip \ unzip \ wget \ @@ -44,6 +42,8 @@ RUN yum -y install \ tree \ java-11-openjdk-devel +RUN update-alternatives --set python /usr/bin/python3.9 + ENV JAVA_HOME /usr/lib/jvm/java-11-openjdk ENV bazelVersion 4.2.2 diff --git a/docker/test/Dockerfile.ubuntu18.04 b/docker/test/Dockerfile.ubuntu18.04 index f48689d721a..874b8673bbc 100644 --- a/docker/test/Dockerfile.ubuntu18.04 +++ b/docker/test/Dockerfile.ubuntu18.04 @@ -18,7 +18,7 @@ FROM ubuntu:18.04 # This is passed to the heron build command via the --config flag -ENV TARGET_PLATFORM ubuntu +ENV TARGET_PLATFORM linux RUN apt-get update && apt-get -y install \ g++ \ diff --git a/heron/common/tests/python/pex_loader/BUILD b/heron/common/tests/python/pex_loader/BUILD index fe071904307..d43a8c38627 100644 --- a/heron/common/tests/python/pex_loader/BUILD +++ b/heron/common/tests/python/pex_loader/BUILD @@ -8,7 +8,7 @@ pex_pytest( "pex_loader_unittest.py", ], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/common/src/python:common-py", diff --git a/heron/executor/src/python/BUILD b/heron/executor/src/python/BUILD index 392326cc986..c6796781c2f 100644 --- a/heron/executor/src/python/BUILD +++ b/heron/executor/src/python/BUILD @@ -3,7 +3,7 @@ package(default_visibility = ["//visibility:public"]) pex_library( name = "executor-py", srcs = ["heron_executor.py"], - reqs = ["PyYAML==3.13", "click==7.1.2"], + reqs = ["PyYAML==5.4.1", "click==7.1.2"], deps = [ "//heron/common/src/python:common-py", "//heron/statemgrs/src/python:statemgr-py", diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index fd6ba2d532c..1f060c1134c 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -412,7 +412,7 @@ def update_packing_plan(self, new_packing_plan): # pylint: disable=no-self-use def _load_logging_dir(self, heron_internals_config_file): with open(heron_internals_config_file, 'r') as stream: - heron_internals_config = yaml.load(stream) + heron_internals_config = yaml.safe_load(stream) return heron_internals_config['heron.logging.directory'] def _get_metricsmgr_cmd(self, metricsManagerId, sink_config_file, port): @@ -1058,7 +1058,7 @@ def start_state_manager_watches(self): Log.info("Start state manager watches") with open(self.override_config_file, 'r') as stream: - overrides = yaml.load(stream) + overrides = yaml.safe_load(stream) if overrides is None: overrides = {} overrides["heron.statemgr.connection.string"] = self.state_manager_connection diff --git a/heron/executor/tests/python/BUILD b/heron/executor/tests/python/BUILD index 480ece694ff..a09e81084fb 100644 --- a/heron/executor/tests/python/BUILD +++ b/heron/executor/tests/python/BUILD @@ -5,7 +5,7 @@ pex_pytest( size = "small", srcs = ["heron_executor_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/executor/src/python:executor-py", diff --git a/heron/instance/src/python/BUILD b/heron/instance/src/python/BUILD index 05f50222374..58be0bff00c 100644 --- a/heron/instance/src/python/BUILD +++ b/heron/instance/src/python/BUILD @@ -17,7 +17,7 @@ pex_binary( name = "heron-python-instance", srcs = ["instance.py"], reqs = [ - "PyYAML==3.13", + "PyYAML==5.4.1", "click==7.1.2", "colorlog==2.6.1", ], diff --git a/heron/instance/src/python/instance.py b/heron/instance/src/python/instance.py index 137eafc7e96..cb9b268c835 100644 --- a/heron/instance/src/python/instance.py +++ b/heron/instance/src/python/instance.py @@ -322,7 +322,7 @@ def yaml_config_reader(config_path): raise ValueError("Config file not yaml") with open(config_path, 'r') as f: - config = yaml.load(f) + config = yaml.safe_load(f) return config diff --git a/heron/instance/tests/python/BUILD b/heron/instance/tests/python/BUILD index f01bee9e424..31a2891bf4d 100644 --- a/heron/instance/tests/python/BUILD +++ b/heron/instance/tests/python/BUILD @@ -4,7 +4,7 @@ pex_library( name = "instance-tests-py", srcs = ["mock_protobuf.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/proto:proto-py", diff --git a/heron/instance/tests/python/network/BUILD b/heron/instance/tests/python/network/BUILD index 7cbb6e6b136..b3afe0a600c 100644 --- a/heron/instance/tests/python/network/BUILD +++ b/heron/instance/tests/python/network/BUILD @@ -25,7 +25,7 @@ pex_pytest( size = "small", srcs = ["st_stmgr_client_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ ":instance-network-mock", @@ -39,7 +39,7 @@ pex_pytest( size = "small", srcs = ["metricsmgr_client_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ ":pytest-network-py", @@ -52,7 +52,7 @@ pex_library( name = "pytest-network-py", srcs = ["mock_generator.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ ":instance-network-mock-client", @@ -64,7 +64,7 @@ pex_pytest( size = "small", srcs = ["protocol_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ ":pytest-network-py", @@ -78,7 +78,7 @@ pex_pytest( size = "small", srcs = ["heron_client_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ ":pytest-network-py", @@ -92,7 +92,7 @@ pex_pytest( size = "small", srcs = ["gateway_looper_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/src/python:instance-py", @@ -104,7 +104,7 @@ pex_pytest( size = "small", srcs = ["event_looper_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/src/python:instance-py", diff --git a/heron/instance/tests/python/utils/BUILD b/heron/instance/tests/python/utils/BUILD index c24cb901bb3..d90857ba776 100644 --- a/heron/instance/tests/python/utils/BUILD +++ b/heron/instance/tests/python/utils/BUILD @@ -16,7 +16,7 @@ pex_pytest( size = "small", srcs = ["communicator_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -28,7 +28,7 @@ pex_pytest( size = "small", srcs = ["custom_grouping_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -40,7 +40,7 @@ pex_pytest( size = "small", srcs = ["metrics_helper_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -52,7 +52,7 @@ pex_pytest( size = "small", srcs = ["outgoing_tuple_helper_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -64,7 +64,7 @@ pex_pytest( size = "small", srcs = ["pplan_helper_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -76,7 +76,7 @@ pex_pytest( size = "small", srcs = ["topology_context_impl_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -88,7 +88,7 @@ pex_pytest( size = "small", srcs = ["tuple_helper_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -100,7 +100,7 @@ pex_pytest( size = "small", srcs = ["global_metrics_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ ":common-utils-mock", @@ -113,7 +113,7 @@ pex_pytest( size = "small", srcs = ["py_metrics_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", @@ -125,7 +125,7 @@ pex_pytest( size = "small", srcs = ["log_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/instance/tests/python/utils:common-utils-mock", diff --git a/heron/proto/BUILD b/heron/proto/BUILD index 822756f2fda..eee9fc8dbc9 100644 --- a/heron/proto/BUILD +++ b/heron/proto/BUILD @@ -181,8 +181,8 @@ java_library( pex_library( name = "proto-py", reqs = [ - "protobuf==3.8.0", - "setuptools==46.1.3", + "protobuf==3.14.0", + "setuptools==51.0.0", ], deps = [ ":proto_ckptmgr_py", diff --git a/heron/shell/src/python/BUILD b/heron/shell/src/python/BUILD index 106c0a375aa..2915ee003fe 100644 --- a/heron/shell/src/python/BUILD +++ b/heron/shell/src/python/BUILD @@ -7,8 +7,8 @@ pex_library( ), reqs = [ "logging-formatter-anticrlf==1.2", - "requests==2.12.3", - "tornado==4.5.3", + "requests==2.27.1", + "tornado==6.1", ], deps = [ "//heron/common/src/python:common-py", diff --git a/heron/statemgrs/src/python/BUILD b/heron/statemgrs/src/python/BUILD index 8616230c3b1..dfd930a8e09 100644 --- a/heron/statemgrs/src/python/BUILD +++ b/heron/statemgrs/src/python/BUILD @@ -4,7 +4,7 @@ pex_library( name = "statemgr-py", srcs = glob(["**/*.py"]), reqs = [ - "PyYAML==3.13", + "PyYAML==5.4.1", "kazoo==2.8.0", "zope.interface==4.0.5", ], diff --git a/heron/statemgrs/src/python/configloader.py b/heron/statemgrs/src/python/configloader.py index 9fe98f2c5d0..9c9c74a11fc 100644 --- a/heron/statemgrs/src/python/configloader.py +++ b/heron/statemgrs/src/python/configloader.py @@ -33,7 +33,7 @@ def load_state_manager_locations(cluster, state_manager_config_file='heron-conf/ locations. Handles a subset of config wildcard substitution supported in the substitute method in org.apache.heron.spi.common.Misc.java""" with open(state_manager_config_file, 'r') as stream: - config = yaml.load(stream) + config = yaml.safe_load(stream) home_dir = os.path.expanduser("~") wildcards = { diff --git a/heron/statemgrs/tests/python/BUILD b/heron/statemgrs/tests/python/BUILD index d28568a8ed0..b6735bc49a0 100644 --- a/heron/statemgrs/tests/python/BUILD +++ b/heron/statemgrs/tests/python/BUILD @@ -10,7 +10,7 @@ pex_pytest( "//heron/config/src/yaml:conf-yaml", ], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/statemgrs/src/python:statemgr-py", @@ -24,7 +24,7 @@ pex_pytest( "zkstatemanager_unittest.py", ], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/statemgrs/src/python:statemgr-py", @@ -38,7 +38,7 @@ pex_pytest( "statemanagerfactory_unittest.py", ], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/statemgrs/src/python:statemgr-py", diff --git a/heron/tools/cli/src/python/BUILD b/heron/tools/cli/src/python/BUILD index e1c45947757..7e61ccc110c 100644 --- a/heron/tools/cli/src/python/BUILD +++ b/heron/tools/cli/src/python/BUILD @@ -6,8 +6,8 @@ pex_library( ["**/*.py"], ), reqs = [ - "PyYAML==3.13", - "requests==2.12.3", + "PyYAML==5.4.1", + "requests==2.27.1", "netifaces==0.10.6", ], deps = [ diff --git a/heron/tools/cli/src/python/cliconfig.py b/heron/tools/cli/src/python/cliconfig.py index f88fd70ebc6..af67c4baea6 100644 --- a/heron/tools/cli/src/python/cliconfig.py +++ b/heron/tools/cli/src/python/cliconfig.py @@ -94,6 +94,6 @@ def _cluster_config(cluster): cluster_config_file = get_cluster_config_file(cluster) if os.path.isfile(cluster_config_file): with open(cluster_config_file, 'r') as cf: - config = yaml.load(cf) + config = yaml.safe_load(cf) return config diff --git a/heron/tools/cli/tests/python/BUILD b/heron/tools/cli/tests/python/BUILD index 5e763d9f8f6..7ee66b3c7df 100644 --- a/heron/tools/cli/tests/python/BUILD +++ b/heron/tools/cli/tests/python/BUILD @@ -5,7 +5,7 @@ pex_pytest( size = "small", srcs = ["opts_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/tools/cli/src/python:cli-py", @@ -17,7 +17,7 @@ pex_pytest( size = "small", srcs = ["client_command_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/tools/cli/src/python:cli-py", diff --git a/heron/tools/common/src/python/BUILD b/heron/tools/common/src/python/BUILD index 8ce984afa2f..afb3cc96e2c 100644 --- a/heron/tools/common/src/python/BUILD +++ b/heron/tools/common/src/python/BUILD @@ -12,7 +12,7 @@ pex_library( exclude = ["clients"], exclude_directories = 1, ), - reqs = ["PyYAML==3.13"], + reqs = ["PyYAML==5.4.1"], deps = [ "//heron/common/src/python:common-py", ], diff --git a/heron/tools/common/src/python/clients/tracker.py b/heron/tools/common/src/python/clients/tracker.py index 43454d417bb..7a034cde8b5 100644 --- a/heron/tools/common/src/python/clients/tracker.py +++ b/heron/tools/common/src/python/clients/tracker.py @@ -133,8 +133,8 @@ def api_get(url: str, params=None) -> dict: return None end = time.time() data = response.json() - if "result" not in data: - Log.error(f"Empty response from {url}") + if data["status"] != "success": + Log.error("error from tracker: %s", data["message"]) return None execution = data["executiontime"] * 1000 diff --git a/heron/tools/common/src/python/utils/config.py b/heron/tools/common/src/python/utils/config.py index dee4adda903..28b388804f7 100644 --- a/heron/tools/common/src/python/utils/config.py +++ b/heron/tools/common/src/python/utils/config.py @@ -265,7 +265,7 @@ def parse_cluster_role_env(cluster_role_env, config_path): else: cli_confs = {} with open(cli_conf_file, 'r') as conf_file: - tmp_confs = yaml.load(conf_file) + tmp_confs = yaml.safe_load(conf_file) # the return value of yaml.load can be None if conf_file is an empty file if tmp_confs is not None: cli_confs = tmp_confs @@ -321,7 +321,7 @@ def direct_mode_cluster_role_env(cluster_role_env, config_path): client_confs = {} with open(cli_conf_file, 'r') as conf_file: - client_confs = yaml.load(conf_file) + client_confs = yaml.safe_load(conf_file) # the return value of yaml.load can be None if conf_file is an empty file if not client_confs: @@ -440,7 +440,7 @@ def print_build_info(): release_file = get_heron_release_file() with open(release_file) as release_info: - release_map = yaml.load(release_info) + release_map = yaml.safe_load(release_info) release_items = sorted(list(release_map.items()), key=lambda tup: tup[0]) for key, value in release_items: print("%s : %s" % (key, value)) diff --git a/heron/tools/explorer/src/python/BUILD b/heron/tools/explorer/src/python/BUILD index 5964c23558f..a2eecafe3c0 100644 --- a/heron/tools/explorer/src/python/BUILD +++ b/heron/tools/explorer/src/python/BUILD @@ -4,10 +4,9 @@ pex_library( name = "explorer-py", srcs = glob(["**/*.py"]), reqs = [ - "tornado==4.5.3", "tabulate==0.7.4", "click==7.1.2", - "requests==2.12.3", + "requests==2.27.1", ], deps = [ "//heron/common/src/python:common-py", diff --git a/heron/tools/explorer/tests/python/BUILD b/heron/tools/explorer/tests/python/BUILD index 313b785a98f..ddbb99a3c6c 100644 --- a/heron/tools/explorer/tests/python/BUILD +++ b/heron/tools/explorer/tests/python/BUILD @@ -5,7 +5,7 @@ pex_pytest( size = "small", srcs = ["explorer_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heron/common/src/python:common-py", diff --git a/heron/tools/tracker/src/python/BUILD b/heron/tools/tracker/src/python/BUILD index 14ed3bde775..6a2daaaf907 100644 --- a/heron/tools/tracker/src/python/BUILD +++ b/heron/tools/tracker/src/python/BUILD @@ -8,10 +8,12 @@ pex_library( ), reqs = [ "click==7.1.2", + "fastapi==0.62.0", + "httpx==0.16.1", "javaobj-py3==0.4.1", - "networkx==2.4", - "protobuf==3.8.0", - "tornado==4.0.2", + "networkx==2.5", + "protobuf==3.14.0", + "uvicorn==0.11.7", ], deps = [ "//heron/common/src/python:common-py", diff --git a/heron/tools/tracker/src/python/app.py b/heron/tools/tracker/src/python/app.py new file mode 100644 index 00000000000..572854a5d8a --- /dev/null +++ b/heron/tools/tracker/src/python/app.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This service uses the configured state manager to retrieve notifications about +running topologies, and uses data from that to communicate with topology managers +when prompted to. + +""" +from typing import Dict, List, Optional + +from heron.tools.tracker.src.python import constants, state, query +from heron.tools.tracker.src.python.utils import ResponseEnvelope +from heron.tools.tracker.src.python.routers import topologies, container, metrics + +from fastapi import FastAPI, Query +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from starlette.exceptions import HTTPException as StarletteHTTPException + + +openapi_tags = [ + {"name": "metrics", "description": query.__doc__}, + {"name": "container", "description": container.__doc__}, + {"name": "topologies", "description": topologies.__doc__}, +] + +app = FastAPI( + title="Heron Tracker", + redoc_url="/", + description=__doc__, + version=constants.API_VERSION, + openapi_tags=openapi_tags, + externalDocs={ + "description": "Heron home page", + "url": "https://heron.incubator.apache.org/", + }, + info={ + "license": { + "name": "Apache 2.0", + "url": "https://www.apache.org/licenses/LICENSE-2.0.html" + }, + }, + **{ + "x-logo": { + "url": "https://heron.incubator.apache.org/img/HeronTextLogo-small.png", + "href": "https://heron.incubator.apache.org/", + "backgroundColor": "#263238", + } + }, +) +app.include_router(container.router, prefix="/topologies", tags=["container"]) +app.include_router(metrics.router, prefix="/topologies", tags=["metrics"]) +app.include_router(topologies.router, prefix="/topologies", tags=["topologies"]) + + +@app.on_event("startup") +async def startup_event(): + """Start recieving topology updates.""" + state.tracker.sync_topologies() + +@app.on_event("shutdown") +async def shutdown_event(): + """Stop recieving topology updates.""" + state.tracker.stop_sync() + + +@app.exception_handler(Exception) +async def handle_exception(_, exc: Exception): + payload = ResponseEnvelope[str]( + result=None, + execution_time=0.0, + message=f"request failed: {exc}", + status=constants.RESPONSE_STATUS_FAILURE + ) + status_code = 500 + if isinstance(exc, StarletteHTTPException): + status_code = exc.status_code + if isinstance(exc, RequestValidationError): + status_code = 400 + return JSONResponse(content=payload.dict(), status_code=status_code) + + +@app.get("/clusters", response_model=ResponseEnvelope[List[str]]) +async def clusters() -> List[str]: + return ResponseEnvelope[List[str]]( + execution_time=0.0, + message="ok", + status="success", + result=[s.name for s in state.tracker.state_managers], + ) + +@app.get( + "/machines", + response_model=ResponseEnvelope[Dict[str, Dict[str, Dict[str, List[str]]]]], +) +async def get_machines( + cluster_names: Optional[List[str]] = Query(None, alias="cluster"), + environ_names: Optional[List[str]] = Query(None, alias="environ"), + topology_names: Optional[List[str]] = Query(None, alias="topology"), +): + """ + Return a map of topology (cluster, environ, name) to a list of machines found in the + physical plans plans of maching topologies. + + If no names are provided, then all topologies matching the other filters are returned. + + """ + # if topology names then clusters and environs needed + if topology_names and not (cluster_names and environ_names): + raise ValueError( + "If topology names are provided then cluster and environ names must be provided" + ) + + response: Dict[str, Dict[str, Dict[str, List[str]]]] = {} + for topology in state.tracker.filtered_topologies(cluster_names, environ_names, topology_names): + response.setdefault(topology.cluster, {}).setdefault(topology.environ, {})[ + topology.name + ] = topology.get_machines() + + return ResponseEnvelope[Dict[str, Dict[str, Dict[str, List[str]]]]]( + execution_time=0.0, + result=response, + status="success", + message="ok", + ) diff --git a/heron/tools/tracker/src/python/config.py b/heron/tools/tracker/src/python/config.py index 9190f0fe1ad..6652a7ff36b 100644 --- a/heron/tools/tracker/src/python/config.py +++ b/heron/tools/tracker/src/python/config.py @@ -39,27 +39,23 @@ class Config: def __init__(self, configs): self.configs = configs self.statemgr_config = StateMgrConfig() - self.extra_links = [] + self.statemgr_config.set_state_locations(configs[STATEMGRS_KEY]) - self.load_configs() + self.extra_links = configs.get(EXTRA_LINKS_KEY, []) + for link in self.extra_links: + self.validate_extra_link(link) - def load_configs(self): - """load config files""" - self.statemgr_config.set_state_locations(self.configs[STATEMGRS_KEY]) - if EXTRA_LINKS_KEY in self.configs: - for extra_link in self.configs[EXTRA_LINKS_KEY]: - self.extra_links.append(self.validate_extra_link(extra_link)) - - def validate_extra_link(self, extra_link: dict): + @classmethod + def validate_extra_link(cls, extra_link: dict) -> None: """validate extra link""" if EXTRA_LINK_NAME_KEY not in extra_link or EXTRA_LINK_FORMATTER_KEY not in extra_link: raise Exception("Invalid extra.links format. " + "Extra link must include a 'name' and 'formatter' field") - self.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY]) - return extra_link + cls.validated_formatter(extra_link[EXTRA_LINK_FORMATTER_KEY]) - def validated_formatter(self, url_format: str) -> None: + @classmethod + def validated_formatter(cls, url_format: str) -> None: """Check visualization url format has no unrecongnised parameters.""" # collect the parameters which would be interpolated formatter_variables = set() @@ -70,30 +66,11 @@ def __getitem__(self, key): string.Template(url_format).safe_substitute(ValidationHelper()) - if not formatter_variables <= self.FORMATTER_PARAMETERS: + if not formatter_variables <= cls.FORMATTER_PARAMETERS: raise Exception(f"Invalid viz.url.format: {url_format!r}") - @staticmethod - def get_formatted_url(formatter: str, execution_state: dict) -> str: - """ - Format a url string using values from the execution state. - - """ - - subs = { - var: execution_state[prop] - for prop, var in ( - ("cluster", "CLUSTER"), - ("environ", "ENVIRON"), - ("jobname", "TOPOLOGY"), - ("role", "ROLE"), - ("submission_user", "USER")) - if prop in execution_state - } - return string.Template(formatter).substitute(subs) - def __str__(self): - return "".join((self.config_str(c) for c in self.configs[STATEMGRS_KEY])) + return "".join(self.config_str(c) for c in self.configs[STATEMGRS_KEY]) @staticmethod def config_str(config): diff --git a/heron/tools/tracker/src/python/constants.py b/heron/tools/tracker/src/python/constants.py index 39efbbdcea3..dfa22129cb9 100644 --- a/heron/tools/tracker/src/python/constants.py +++ b/heron/tools/tracker/src/python/constants.py @@ -30,7 +30,7 @@ try: API_VERSION = common_config.get_version_number() except: - API_VERSION = "" + API_VERSION = "0.1.0" # Handler Constants diff --git a/heron/tools/tracker/src/python/handlers/__init__.py b/heron/tools/tracker/src/python/handlers/__init__.py deleted file mode 100644 index 53052463ead..00000000000 --- a/heron/tools/tracker/src/python/handlers/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -''' __init__ ''' -from .basehandler import BaseHandler -from .clustershandler import ClustersHandler -from .containerfilehandler import ContainerFileDataHandler -from .containerfilehandler import ContainerFileDownloadHandler -from .containerfilehandler import ContainerFileStatsHandler -from .defaulthandler import DefaultHandler -from .exceptionhandler import ExceptionHandler -from .exceptionsummaryhandler import ExceptionSummaryHandler -from .executionstatehandler import ExecutionStateHandler -from .jmaphandler import JmapHandler -from .jstackhandler import JstackHandler -from .logicalplanhandler import LogicalPlanHandler -from .machineshandler import MachinesHandler -from .mainhandler import MainHandler -from .memoryhistogramhandler import MemoryHistogramHandler -from .metadatahandler import MetaDataHandler -from .metricshandler import MetricsHandler -from .metricsqueryhandler import MetricsQueryHandler -from .metricstimelinehandler import MetricsTimelineHandler -from .physicalplanhandler import PhysicalPlanHandler -from .packingplanhandler import PackingPlanHandler -from .pidhandler import PidHandler -from .runtimestatehandler import RuntimeStateHandler -from .schedulerlocationhandler import SchedulerLocationHandler -from .stateshandler import StatesHandler -from .topologieshandler import TopologiesHandler -from .topologyconfighandler import TopologyConfigHandler -from .topologyhandler import TopologyHandler diff --git a/heron/tools/tracker/src/python/handlers/basehandler.py b/heron/tools/tracker/src/python/handlers/basehandler.py deleted file mode 100644 index 49c2892dded..00000000000 --- a/heron/tools/tracker/src/python/handlers/basehandler.py +++ /dev/null @@ -1,273 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' basehandlers.py ''' -import time -import tornado.escape -import tornado.web - -from heron.tools.tracker.src.python import constants - -# pylint: disable=too-many-public-methods -# pylint: disable=abstract-method -class BaseHandler(tornado.web.RequestHandler): - """ - Base Handler. All the other handlers derive from - this class. This exposes some base methods, - like making response for success and failure cases, - and measuring times. - """ - - def set_default_headers(self): - ''' Allow any domain to make queries to tracker. ''' - self.set_header("Access-Control-Allow-Origin", "*") - - # pylint: disable=attribute-defined-outside-init - def prepare(self): - """ - Used for timing. Sets the basehandler_starttime to current time, and - is used when writing the response back. - Subclasses of BaseHandler must never use self.write, but instead use - self.write_error or self.write_result methods to correctly include - the timing. - """ - self.basehandler_starttime = time.time() - - def write_success_response(self, result): - """ - Result may be a python dictionary, array or a primitive type - that can be converted to JSON for writing back the result. - """ - response = self.make_success_response(result) - now = time.time() - spent = now - self.basehandler_starttime - response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent - self.write_json_response(response) - - def write_error_response(self, message): - """ - Writes the message as part of the response and sets 404 status. - """ - self.set_status(404) - response = self.make_error_response(str(message)) - now = time.time() - spent = now - self.basehandler_starttime - response[constants.RESPONSE_KEY_EXECUTION_TIME] = spent - self.write_json_response(response) - - def write_json_response(self, response): - """ write back json response """ - self.write(tornado.escape.json_encode(response)) - self.set_header("Content-Type", "application/json") - - # pylint: disable=no-self-use - def make_response(self, status): - """ - Makes the base dict for the response. - The status is the string value for - the key "status" of the response. This - should be "success" or "failure". - """ - response = { - constants.RESPONSE_KEY_STATUS: status, - constants.RESPONSE_KEY_VERSION: constants.API_VERSION, - constants.RESPONSE_KEY_EXECUTION_TIME: 0, - constants.RESPONSE_KEY_MESSAGE: "", - } - return response - - def make_success_response(self, result): - """ - Makes the python dict corresponding to the - JSON that needs to be sent for a successful - response. Result is the actual payload - that gets sent. - """ - response = self.make_response(constants.RESPONSE_STATUS_SUCCESS) - response[constants.RESPONSE_KEY_RESULT] = result - return response - - def make_error_response(self, message): - """ - Makes the python dict corresponding to the - JSON that needs to be sent for a failed - response. Message is the message that is - sent as the reason for failure. - """ - response = self.make_response(constants.RESPONSE_STATUS_FAILURE) - response[constants.RESPONSE_KEY_MESSAGE] = message - return response - - def get_argument_cluster(self): - """ - Helper function to get request argument. - Raises exception if argument is missing. - Returns the cluster argument. - """ - try: - return self.get_argument(constants.PARAM_CLUSTER) - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_role(self): - """ - Helper function to get request argument. - Raises exception if argument is missing. - Returns the role argument. - """ - try: - return self.get_argument(constants.PARAM_ROLE, default=None) - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - - def get_argument_environ(self): - """ - Helper function to get request argument. - Raises exception if argument is missing. - Returns the environ argument. - """ - try: - return self.get_argument(constants.PARAM_ENVIRON) - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_topology(self): - """ - Helper function to get topology argument. - Raises exception if argument is missing. - Returns the topology argument. - """ - try: - topology = self.get_argument(constants.PARAM_TOPOLOGY) - return topology - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_component(self): - """ - Helper function to get component argument. - Raises exception if argument is missing. - Returns the component argument. - """ - try: - component = self.get_argument(constants.PARAM_COMPONENT) - return component - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_instance(self): - """ - Helper function to get instance argument. - Raises exception if argument is missing. - Returns the instance argument. - """ - try: - instance = self.get_argument(constants.PARAM_INSTANCE) - return instance - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_starttime(self): - """ - Helper function to get starttime argument. - Raises exception if argument is missing. - Returns the starttime argument. - """ - try: - starttime = self.get_argument(constants.PARAM_STARTTIME) - return starttime - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_endtime(self): - """ - Helper function to get endtime argument. - Raises exception if argument is missing. - Returns the endtime argument. - """ - try: - endtime = self.get_argument(constants.PARAM_ENDTIME) - return endtime - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_query(self): - """ - Helper function to get query argument. - Raises exception if argument is missing. - Returns the query argument. - """ - try: - query = self.get_argument(constants.PARAM_QUERY) - return query - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_offset(self): - """ - Helper function to get offset argument. - Raises exception if argument is missing. - Returns the offset argument. - """ - try: - offset = self.get_argument(constants.PARAM_OFFSET) - return offset - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_argument_length(self): - """ - Helper function to get length argument. - Raises exception if argument is missing. - Returns the length argument. - """ - try: - length = self.get_argument(constants.PARAM_LENGTH) - return length - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def get_required_arguments_metricnames(self): - """ - Helper function to get metricname arguments. - Notice that it is get_argument"s" variation, which means that this can be repeated. - Raises exception if argument is missing. - Returns a list of metricname arguments - """ - try: - metricnames = self.get_arguments(constants.PARAM_METRICNAME) - if not metricnames: - raise tornado.web.MissingArgumentError(constants.PARAM_METRICNAME) - return metricnames - except tornado.web.MissingArgumentError as e: - raise Exception(e.log_message) - - def validateInterval(self, startTime, endTime): - """ - Helper function to validate interval. - An interval is valid if starttime and endtime are integrals, - and starttime is less than the endtime. - Raises exception if interval is not valid. - """ - start = int(startTime) - end = int(endTime) - if start > end: - raise Exception("starttime is greater than endtime.") diff --git a/heron/tools/tracker/src/python/handlers/clustershandler.py b/heron/tools/tracker/src/python/handlers/clustershandler.py deleted file mode 100644 index 73af25539d1..00000000000 --- a/heron/tools/tracker/src/python/handlers/clustershandler.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' clustershandler.py ''' -import tornado.gen - -from heron.tools.tracker.src.python.handlers import BaseHandler - -# pylint: disable=attribute-defined-outside-init -class ClustersHandler(BaseHandler): - """ - URL - /clusters - - The response JSON is a list of clusters - """ - - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - clusters = [statemgr.name for statemgr in self.tracker.state_managers] - - self.write_success_response(clusters) diff --git a/heron/tools/tracker/src/python/handlers/containerfilehandler.py b/heron/tools/tracker/src/python/handlers/containerfilehandler.py deleted file mode 100644 index 623a4ade0a6..00000000000 --- a/heron/tools/tracker/src/python/handlers/containerfilehandler.py +++ /dev/null @@ -1,183 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' containerfilehandler.py ''' -import json -import traceback -import tornado.gen - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python import utils - - -# pylint: disable=attribute-defined-outside-init -class ContainerFileDataHandler(BaseHandler): - """ - URL - /topologies/containerfiledata?cluster=&topology= \ - &environ=&container= - Parameters: - - cluster - Name of cluster. - - environ - Running environment. - - role - (optional) Role used to submit the topology. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - container - Container number - - path - Relative path to the file - - offset - From which to read the file - - length - How much data to read - - Get the data from the file for the given topology, container - and path. The data being read is based on offset and length. - """ - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - container = self.get_argument(constants.PARAM_CONTAINER) - path = self.get_argument(constants.PARAM_PATH) - offset = self.get_argument_offset() - length = self.get_argument_length() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - - stmgr_id = "stmgr-" + container - stmgr = topology_info["physical_plan"]["stmgrs"][stmgr_id] - host = stmgr["host"] - shell_port = stmgr["shell_port"] - file_data_url = "http://%s:%d/filedata/%s?offset=%s&length=%s" % \ - (host, shell_port, path, offset, length) - - http_client = tornado.httpclient.AsyncHTTPClient() - response = yield http_client.fetch(file_data_url) - self.write_success_response(json.loads(response.body)) - self.finish() - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - -# pylint: disable=attribute-defined-outside-init -class ContainerFileDownloadHandler(BaseHandler): - """ - URL - /topologies/containerfiledownload?cluster=&topology= \ - &environ=&container= - Parameters: - - cluster - Name of cluster. - - environ - Running environment. - - role - (optional) Role used to submit the topology. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - container - Container number - - path - Relative path to the file - - Download the file for the given topology, container and path. - """ - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """Serve a GET request.""" - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - container = self.get_argument(constants.PARAM_CONTAINER) - path = self.get_argument(constants.PARAM_PATH) - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - - stmgr_id = "stmgr-" + container - stmgr = topology_info["physical_plan"]["stmgrs"][stmgr_id] - host = stmgr["host"] - shell_port = stmgr["shell_port"] - file_download_url = "http://%s:%d/download/%s" % (host, shell_port, path) - Log.debug("download file url: %s", file_download_url) - - path = self.get_argument("path") - filename = path.split("/")[-1] - self.set_header("Content-Disposition", "attachment; filename=%s" % filename) - - def streaming_callback(chunk): - self.write(chunk) - self.flush() - - http_client = tornado.httpclient.AsyncHTTPClient() - yield http_client.fetch(file_download_url, streaming_callback=streaming_callback) - self.finish() - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - -class ContainerFileStatsHandler(BaseHandler): - """ - URL - /topologies/containerfilestats?cluster=&topology= \ - &environ=&container= - Parameters: - - cluster - Name of cluster. - - environ - Running environment. - - role - (optional) Role used to submit the topology. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - container - Container number - - path - Relative path to the directory - - Get the dir stats for the given topology, container and path. - """ - - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - container = self.get_argument(constants.PARAM_CONTAINER) - path = self.get_argument(constants.PARAM_PATH, default=".") - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - - stmgr_id = "stmgr-" + str(container) - stmgr = topology_info["physical_plan"]["stmgrs"][stmgr_id] - host = stmgr["host"] - shell_port = stmgr["shell_port"] - filestats_url = utils.make_shell_filestats_url(host, shell_port, path) - - http_client = tornado.httpclient.AsyncHTTPClient() - response = yield http_client.fetch(filestats_url) - self.write_success_response(json.loads(response.body)) - self.finish() - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/defaulthandler.py b/heron/tools/tracker/src/python/handlers/defaulthandler.py deleted file mode 100644 index a30443be48f..00000000000 --- a/heron/tools/tracker/src/python/handlers/defaulthandler.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" defaulthandler.py """ -import tornado.gen - -from heron.tools.tracker.src.python.handlers import BaseHandler - -class DefaultHandler(BaseHandler): - """ - URL - anything that is not supported - - This is the default case in the regular expression - matching for the URLs. If nothin matched before this, - then this is the URL that is not supported by the API. - - Sends back a "failure" response to the client. - """ - - @tornado.gen.coroutine - def get(self, url): - """ get method """ - self.write_error_response("URL not supported: " + url) diff --git a/heron/tools/tracker/src/python/handlers/exceptionhandler.py b/heron/tools/tracker/src/python/handlers/exceptionhandler.py deleted file mode 100644 index 63930e185eb..00000000000 --- a/heron/tools/tracker/src/python/handlers/exceptionhandler.py +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" exceptionhandler.py """ -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.proto import common_pb2 -from heron.proto import tmanager_pb2 -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python.handlers import BaseHandler - - -# pylint: disable=attribute-defined-outside-init -class ExceptionHandler(BaseHandler): - """ - URL - /topologies/exceptions?cluster=&topology= \ - &environ=&component= - Parameters: - - cluster - Name of cluster. - - environ - Running environment. - - role - (optional) Role used to submit the topology. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - component - Component name - - instance - (optional, repeated) - - Returns all exceptions for the component of the topology. - """ - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - environ = self.get_argument_environ() - role = self.get_argument_role() - topName = self.get_argument_topology() - component = self.get_argument_component() - topology = self.tracker.get_topology( - cluster, role, environ, topName) - instances = self.get_arguments(constants.PARAM_INSTANCE) - exceptions_logs = yield tornado.gen.Task(self.getComponentException, - topology.tmanager, component, instances) - self.write_success_response(exceptions_logs) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=bad-option-value, dangerous-default-value, no-self-use, - # pylint: disable=unused-argument - @tornado.gen.coroutine - def getComponentException(self, tmanager, component_name, instances=[], callback=None): - """ - Get all (last 1000) exceptions for 'component_name' of the topology. - Returns an Array of exception logs on success. - Returns json with message on failure. - """ - if not tmanager or not tmanager.host or not tmanager.stats_port: - return - - exception_request = tmanager_pb2.ExceptionLogRequest() - exception_request.component_name = component_name - if len(instances) > 0: - exception_request.instances.extend(instances) - request_str = exception_request.SerializeToString() - port = str(tmanager.stats_port) - host = tmanager.host - url = "http://{0}:{1}/exceptions".format(host, port) - request = tornado.httpclient.HTTPRequest(url, - body=request_str, - method='POST', - request_timeout=5) - Log.debug('Making HTTP call to fetch exceptions url: %s', url) - try: - client = tornado.httpclient.AsyncHTTPClient() - result = yield client.fetch(request) - Log.debug("HTTP call complete.") - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) - - # Check the response code - error if it is in 400s or 500s - responseCode = result.code - if responseCode >= 400: - message = "Error in getting exceptions from Tmanager, code: " + responseCode - Log.error(message) - raise tornado.gen.Return({ - "message": message - }) - - # Parse the response from tmanager. - exception_response = tmanager_pb2.ExceptionLogResponse() - exception_response.ParseFromString(result.body) - - if exception_response.status.status == common_pb2.NOTOK: - if exception_response.status.HasField("message"): - raise tornado.gen.Return({ - "message": exception_response.status.message - }) - - # Send response - ret = [] - for exception_log in exception_response.exceptions: - ret.append({'hostname': exception_log.hostname, - 'instance_id': exception_log.instance_id, - 'stack_trace': exception_log.stacktrace, - 'lasttime': exception_log.lasttime, - 'firsttime': exception_log.firsttime, - 'count': str(exception_log.count), - 'logging': exception_log.logging}) - raise tornado.gen.Return(ret) diff --git a/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py b/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py deleted file mode 100644 index 900327faa75..00000000000 --- a/heron/tools/tracker/src/python/handlers/exceptionsummaryhandler.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' execeptionsummaryhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.proto import common_pb2 -from heron.proto import tmanager_pb2 -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python.handlers import BaseHandler - -# pylint: disable=attribute-defined-outside-init -class ExceptionSummaryHandler(BaseHandler): - """ - URL - /topologies/exceptionsummary?cluster=&topology= \ - &environ=&component= - Parameters: - - cluster - Name of cluster. - - environ - Running environment. - - role - (optional) Role used to submit the topology. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - component - Component name - - instance - (optional, repeated) - - Returns summary of the exceptions for the component of the topology. - Duplicated exceptions are combined together and shows the number of - occurances, first occurance time and latest occurance time. - """ - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get """ - try: - cluster = self.get_argument_cluster() - environ = self.get_argument_environ() - role = self.get_argument_role() - topology_name = self.get_argument_topology() - component = self.get_argument_component() - topology = self.tracker.get_topology( - cluster, role, environ, topology_name) - instances = self.get_arguments(constants.PARAM_INSTANCE) - exceptions_summary = yield tornado.gen.Task(self.getComponentExceptionSummary, - topology.tmanager, component, instances) - self.write_success_response(exceptions_summary) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=dangerous-default-value, no-self-use, unused-argument - @tornado.gen.coroutine - def getComponentExceptionSummary(self, tmanager, component_name, instances=[], callback=None): - """ - Get the summary of exceptions for component_name and list of instances. - Empty instance list will fetch all exceptions. - """ - if not tmanager or not tmanager.host or not tmanager.stats_port: - return - exception_request = tmanager_pb2.ExceptionLogRequest() - exception_request.component_name = component_name - if len(instances) > 0: - exception_request.instances.extend(instances) - request_str = exception_request.SerializeToString() - port = str(tmanager.stats_port) - host = tmanager.host - url = "http://{0}:{1}/exceptionsummary".format(host, port) - Log.debug("Creating request object.") - request = tornado.httpclient.HTTPRequest(url, - body=request_str, - method='POST', - request_timeout=5) - Log.debug('Making HTTP call to fetch exceptionsummary url: %s', url) - try: - client = tornado.httpclient.AsyncHTTPClient() - result = yield client.fetch(request) - Log.debug("HTTP call complete.") - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) - - # Check the response code - error if it is in 400s or 500s - responseCode = result.code - if responseCode >= 400: - message = "Error in getting exceptions from Tmanager, code: " + responseCode - Log.error(message) - raise tornado.gen.Return({ - "message": message - }) - - # Parse the response from tmanager. - exception_response = tmanager_pb2.ExceptionLogResponse() - exception_response.ParseFromString(result.body) - - if exception_response.status.status == common_pb2.NOTOK: - if exception_response.status.HasField("message"): - raise tornado.gen.Return({ - "message": exception_response.status.message - }) - - # Send response - ret = [] - for exception_log in exception_response.exceptions: - ret.append({'class_name': exception_log.stacktrace, - 'lasttime': exception_log.lasttime, - 'firsttime': exception_log.firsttime, - 'count': str(exception_log.count)}) - raise tornado.gen.Return(ret) diff --git a/heron/tools/tracker/src/python/handlers/executionstatehandler.py b/heron/tools/tracker/src/python/handlers/executionstatehandler.py deleted file mode 100644 index 2cfedcea6db..00000000000 --- a/heron/tools/tracker/src/python/handlers/executionstatehandler.py +++ /dev/null @@ -1,60 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' executionstatehandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - -# pylint: disable=attribute-defined-outside-init -class ExecutionStateHandler(BaseHandler): - """ - URL - /topologies/executionstate - Parameters: - - cluster (required) - - environ (required) - - role - (optional) Role used to submit the topology. - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - information of execution state of the topology. - """ - - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - execution_state = topology_info["execution_state"] - self.write_success_response(execution_state) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/jmaphandler.py b/heron/tools/tracker/src/python/handlers/jmaphandler.py deleted file mode 100644 index 577c2ca2d9b..00000000000 --- a/heron/tools/tracker/src/python/handlers/jmaphandler.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" jmaphandler.py """ -import json -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python import utils -from heron.tools.tracker.src.python.handlers import BaseHandler -from heron.tools.tracker.src.python.handlers.pidhandler import getInstancePid - -class JmapHandler(BaseHandler): - """ - URL - /topologies/jmap?cluster=&topology= \ - &environ=&instance= - Parameters: - - cluster - Name of cluster. - - role - (optional) Role used to submit the topology. - - environ - Running environment. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - instance - Instance Id - - Issue a jmap for instance and save the result in a file like - /tmp/heap.bin - The response JSON is a dict with following format: - { - 'command': Full command executed at server. - 'stdout': Text on stdout of executing the command. - 'stderr': Text on stderr. - } - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - instance = self.get_argument_instance() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - ret = yield self.runInstanceJmap(topology_info, instance) - self.write_success_response(ret) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=no-self-use - @tornado.gen.coroutine - def runInstanceJmap(self, topology_info, instance_id): - """ - Fetches Instance jstack from heron-shell. - """ - pid_response = yield getInstancePid(topology_info, instance_id) - try: - http_client = tornado.httpclient.AsyncHTTPClient() - pid_json = json.loads(pid_response) - pid = pid_json['stdout'].strip() - if pid == '': - raise Exception('Failed to get pid') - endpoint = utils.make_shell_endpoint(topology_info, instance_id) - url = "%s/jmap/%s" % (endpoint, pid) - response = yield http_client.fetch(url) - Log.debug("HTTP call for url: %s", url) - raise tornado.gen.Return(response.body) - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) diff --git a/heron/tools/tracker/src/python/handlers/jstackhandler.py b/heron/tools/tracker/src/python/handlers/jstackhandler.py deleted file mode 100644 index 56f0adb40aa..00000000000 --- a/heron/tools/tracker/src/python/handlers/jstackhandler.py +++ /dev/null @@ -1,93 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' jstackhandler.py ''' -import json -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python import utils -from heron.tools.tracker.src.python.handlers import BaseHandler -from heron.tools.tracker.src.python.handlers.pidhandler import getInstancePid - -# pylint: disable=attribute-defined-outside-init -class JstackHandler(BaseHandler): - """ - URL - /topologies/jstack?cluster=&topology= \ - &environ=&instance= - Parameters: - - cluster - Name of cluster. - - role - (optional) Role used to submit the topology. - - environ - Running environment. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - instance - Instance Id - - Resturns a thread dump of the instance on stdout. - The response JSON is a dict with following format: - { - 'command': Full command executed at server. - 'stdout': Text on stdout of executing the command. - 'stderr': Text on stderr. - } - """ - - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - instance = self.get_argument_instance() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - ret = yield self.getInstanceJstack(topology_info, instance) - self.write_success_response(ret) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=no-self-use - @tornado.gen.coroutine - def getInstanceJstack(self, topology_info, instance_id): - """ - Fetches Instance jstack from heron-shell. - """ - pid_response = yield getInstancePid(topology_info, instance_id) - try: - http_client = tornado.httpclient.AsyncHTTPClient() - pid_json = json.loads(pid_response) - pid = pid_json['stdout'].strip() - if pid == '': - raise Exception('Failed to get pid') - endpoint = utils.make_shell_endpoint(topology_info, instance_id) - url = "%s/jstack/%s" % (endpoint, pid) - response = yield http_client.fetch(url) - Log.debug("HTTP call for url: %s", url) - raise tornado.gen.Return(response.body) - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) diff --git a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py b/heron/tools/tracker/src/python/handlers/logicalplanhandler.py deleted file mode 100644 index ddabbe80040..00000000000 --- a/heron/tools/tracker/src/python/handlers/logicalplanhandler.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Logical plan objects have the shape: - { - 'spouts': { - spout_name: { - 'outputs': [{'stream_name': stream_name}], - } - }, - 'bolts': { - bolt_name: { - 'outputs': [{'stream_name': stream_name}], - 'inputs': [{ - 'stream_name': stream_name, - 'component_name': component_name, - 'grouping': grouping_type, - }] - } - } - } - -""" -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - -import networkx - - -def topology_stages(logical_plan): - """Return the number of stages in a logical plan.""" - graph = networkx.DiGraph( - (input_info["component_name"], bolt_name) - for bolt_name, bolt_info in logical_plan.get("bolts", {}).items() - for input_info in bolt_info["inputs"] - ) - # this is is the same as "diameter" if treating the topology as an undirected graph - return networkx.dag_longest_path_length(graph) - - - -class LogicalPlanHandler(BaseHandler): - """ - URL - /topologies/logicalplan - Parameters: - - cluster (required) - - role - (role) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - information of logical plan of the topology. - """ - # pylint: disable=missing-docstring, attribute-defined-outside-init - def initialize(self, tracker): - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - lplan = topology_info["logical_plan"] - - # format the logical plan as required by the web (because of Ambrose) - # first, spouts followed by bolts - spouts_map = dict() - for name, value in list(lplan['spouts'].items()): - spouts_map[name] = dict( - config=value.get("config", dict()), - outputs=value["outputs"], - spout_type=value["type"], - spout_source=value["source"], - extra_links=value["extra_links"], - ) - - bolts_map = dict() - for name, value in list(lplan['bolts'].items()): - bolts_map[name] = dict( - config=value.get("config", dict()), - inputComponents=[i['component_name'] for i in value['inputs']], - inputs=value["inputs"], - outputs=value["outputs"] - ) - - result = dict( - stages=topology_stages(lplan), - spouts=spouts_map, - bolts=bolts_map - ) - - self.write_success_response(result) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/machineshandler.py b/heron/tools/tracker/src/python/handlers/machineshandler.py deleted file mode 100644 index 6dfa8385162..00000000000 --- a/heron/tools/tracker/src/python/handlers/machineshandler.py +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" machienshandler.py """ -import tornado.gen - -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python.handlers import BaseHandler - -class MachinesHandler(BaseHandler): - """ - URL - /machines - Parameters: - - cluster (optional) - - environ (optional) - - topology (optional, repeated - both 'cluster' and 'environ' are required - if topology is present) - - The response JSON is a dict with following format: - { - : { - : { - : [machine1, machine2,..], - : [...], - ... - }, - : {...}, - ... - }, - : {...} - } - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - clusters = self.get_arguments(constants.PARAM_CLUSTER) - environs = self.get_arguments(constants.PARAM_ENVIRON) - topology_names = self.get_arguments(constants.PARAM_TOPOLOGY) - - ret = {} - - if len(topology_names) > 1: - if not clusters: - message = "Missing argument" + constants.PARAM_CLUSTER - self.write_error_response(message) - return - - if not environs: - message = "Missing argument" + constants.PARAM_ENVIRON - self.write_error_response(message) - return - - ret = {} - topologies = self.tracker.topologies - for topology in topologies: - cluster = topology.cluster - environ = topology.environ - topology_name = topology.name - if not cluster or not environ: - continue - - # This cluster is not asked for. - if clusters and cluster not in clusters: - continue - - # This environ is not asked for. - if environs and environ not in environs: - continue - - if topology_names and topology_name not in topology_names: - continue - - if cluster not in ret: - ret[cluster] = {} - if environ not in ret[cluster]: - ret[cluster][environ] = {} - ret[cluster][environ][topology_name] = topology.get_machines() - - self.write_success_response(ret) diff --git a/heron/tools/tracker/src/python/handlers/mainhandler.py b/heron/tools/tracker/src/python/handlers/mainhandler.py deleted file mode 100644 index 1b09e08c3f3..00000000000 --- a/heron/tools/tracker/src/python/handlers/mainhandler.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" mainhandler.py """ -from heron.tools.tracker.src.python.handlers import BaseHandler - -class MainHandler(BaseHandler): - """ - URL - / - - The response JSON is a list of all the API URLs - that are supported by tracker. - """ - def get(self): - """ get method """ - self.redirect("/topologies") diff --git a/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py b/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py deleted file mode 100644 index 360c0d73573..00000000000 --- a/heron/tools/tracker/src/python/handlers/memoryhistogramhandler.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" memoryhistogramhandler.py """ -import json -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python import utils -from heron.tools.tracker.src.python.handlers import BaseHandler -from heron.tools.tracker.src.python.handlers.pidhandler import getInstancePid - - -class MemoryHistogramHandler(BaseHandler): - """ - URL - /topologies/histo?cluster=&topology= \ - &environ=&instance= - Parameters: - - cluster - Name of the cluster. - - role - (optional) Role used to submit the topology. - - environ - Running environment. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - instance - Instance Id - - Resturns a histogram of top in memory object. - The response JSON is a dict with following format: - { - 'command': Full command executed at server. - 'stdout': Text on stdout of executing the command. - 'stderr': Text on stderr. - } - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - instance = self.get_argument_instance() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - ret = yield self.getInstanceMemoryHistogram(topology_info, instance) - self.write_success_response(ret) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=no-self-use - @tornado.gen.coroutine - def getInstanceMemoryHistogram(self, topology_info, instance_id): - """ - Fetches Instance top memory item as histogram. - """ - pid_response = yield getInstancePid(topology_info, instance_id) - try: - http_client = tornado.httpclient.AsyncHTTPClient() - pid_json = json.loads(pid_response) - pid = pid_json['stdout'].strip() - if pid == '': - raise Exception('Failed to get pid') - endpoint = utils.make_shell_endpoint(topology_info, instance_id) - url = "%s/histo/%s" % (endpoint, pid) - response = yield http_client.fetch(url) - Log.debug("HTTP call for url: %s", url) - raise tornado.gen.Return(response.body) - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) diff --git a/heron/tools/tracker/src/python/handlers/metadatahandler.py b/heron/tools/tracker/src/python/handlers/metadatahandler.py deleted file mode 100644 index a6b442c8ead..00000000000 --- a/heron/tools/tracker/src/python/handlers/metadatahandler.py +++ /dev/null @@ -1,76 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' metadatahandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - -# pylint: disable=attribute-defined-outside-init -class MetaDataHandler(BaseHandler): - """ - URL - /topologies/metadata - Parameters: - - cluster (required) - - environ (required) - - role - (optional) Role used to submit the topology. - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - static properties of a topology. Runtime information - is available from /topologies/runtimestate. - - Example JSON response: - { - release_version: "foo/bar", - cluster: "local", - release_tag: "", - environ: "default", - submission_user: "foo", - release_username: "foo", - submission_time: 1489523952, - viz: "", - role: "foo", - jobname: "EX" - } - """ - - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - metadata = topology_info["metadata"] - self.write_success_response(metadata) - except Exception as e: - Log.error("Exception when handling GET request '/topologies/metadata'") - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/metricshandler.py b/heron/tools/tracker/src/python/handlers/metricshandler.py deleted file mode 100644 index e2074a7477a..00000000000 --- a/heron/tools/tracker/src/python/handlers/metricshandler.py +++ /dev/null @@ -1,171 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" metrichandler.py """ -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.proto import common_pb2 -from heron.proto import tmanager_pb2 -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python.handlers import BaseHandler - -class MetricsHandler(BaseHandler): - """ - URL - /topologies/metrics - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - component (required) - - metricname (required, repeated) - - interval (optional) - - instance (optional, repeated) - - The response JSON is a map of all the requested - (or if nothing is mentioned, all) components - of the topology, to the metrics that are reported - by that component. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - component = self.get_argument_component() - metric_names = self.get_required_arguments_metricnames() - - topology = self.tracker.get_topology( - cluster, role, environ, topology_name) - - interval = int(self.get_argument(constants.PARAM_INTERVAL, default=-1)) - instances = self.get_arguments(constants.PARAM_INSTANCE) - - metrics = yield tornado.gen.Task( - self.getComponentMetrics, - topology.tmanager, component, metric_names, instances, interval) - - self.write_success_response(metrics) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=too-many-locals, no-self-use, unused-argument - @tornado.gen.coroutine - def getComponentMetrics(self, - tmanager, - componentName, - metric_names, - instances, - interval, - callback=None): - """ - Get the specified metrics for the given component name of this topology. - Returns the following dict on success: - { - "metrics": { - : { - : , - : , - ... - }, ... - }, - "interval": , - "component": "..." - } - - Raises exception on failure. - """ - if not tmanager or not tmanager.host or not tmanager.stats_port: - raise Exception("No Tmanager found") - - host = tmanager.host - port = tmanager.stats_port - - metricRequest = tmanager_pb2.MetricRequest() - metricRequest.component_name = componentName - if len(instances) > 0: - for instance in instances: - metricRequest.instance_id.append(instance) - for metric_name in metric_names: - metricRequest.metric.append(metric_name) - metricRequest.interval = interval - - # Serialize the metricRequest to send as a payload - # with the HTTP request. - metricRequestString = metricRequest.SerializeToString() - - url = "http://{0}:{1}/stats".format(host, port) - request = tornado.httpclient.HTTPRequest(url, - body=metricRequestString, - method='POST', - request_timeout=5) - - Log.debug("Making HTTP call to fetch metrics") - Log.debug("url: " + url) - try: - client = tornado.httpclient.AsyncHTTPClient() - result = yield client.fetch(request) - Log.debug("HTTP call complete.") - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) - - # Check the response code - error if it is in 400s or 500s - responseCode = result.code - if responseCode >= 400: - message = "Error in getting metrics from Tmanager, code: " + responseCode - Log.error(message) - raise Exception(message) - - # Parse the response from tmanager. - metricResponse = tmanager_pb2.MetricResponse() - metricResponse.ParseFromString(result.body) - - if metricResponse.status.status == common_pb2.NOTOK: - if metricResponse.status.HasField("message"): - Log.warn("Received response from Tmanager: %s", metricResponse.status.message) - - # Form the response. - ret = {} - ret["interval"] = metricResponse.interval - ret["component"] = componentName - ret["metrics"] = {} - for metric in metricResponse.metric: - instance = metric.instance_id - for im in metric.metric: - metricname = im.name - value = im.value - if metricname not in ret["metrics"]: - ret["metrics"][metricname] = {} - ret["metrics"][metricname][instance] = value - - raise tornado.gen.Return(ret) diff --git a/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py b/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py deleted file mode 100644 index 741ef70a214..00000000000 --- a/heron/tools/tracker/src/python/handlers/metricsqueryhandler.py +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' metricsqueryhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler -from heron.tools.tracker.src.python.query import Query - - -class MetricsQueryHandler(BaseHandler): - """ - URL - /topologies/metricsquery - Parameters: - - cluster (required) - - role - (role) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - starttime (required) - - endtime (required) - - query (required) - - The response JSON is a list of timelines - asked in the query for this topology - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology = self.tracker.get_topology( - cluster, role, environ, topology_name) - - start_time = self.get_argument_starttime() - end_time = self.get_argument_endtime() - self.validateInterval(start_time, end_time) - - query = self.get_argument_query() - metrics = yield tornado.gen.Task(self.executeMetricsQuery, - topology.tmanager, query, int(start_time), int(end_time)) - self.write_success_response(metrics) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) - - # pylint: disable=unused-argument - @tornado.gen.coroutine - def executeMetricsQuery(self, tmanager, queryString, start_time, end_time, callback=None): - """ - Get the specified metrics for the given query in this topology. - Returns the following dict on success: - { - "timeline": [{ - "instance": , - "data": { - : , - : , - ... - } - }, { - ... - }, ... - "starttime": , - "endtime": , - }, - - Returns the following dict on failure: - { - "message": "..." - } - """ - - query = Query(self.tracker) - metrics = yield query.execute_query(tmanager, queryString, start_time, end_time) - - # Parse the response - ret = {} - ret["starttime"] = start_time - ret["endtime"] = end_time - ret["timeline"] = [] - - for metric in metrics: - tl = { - "data": metric.timeline - } - if metric.instance: - tl["instance"] = metric.instance - ret["timeline"].append(tl) - - raise tornado.gen.Return(ret) diff --git a/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py b/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py deleted file mode 100644 index 1f1348907b8..00000000000 --- a/heron/tools/tracker/src/python/handlers/metricstimelinehandler.py +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' metricstimelinehandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python import metricstimeline -from heron.tools.tracker.src.python.handlers import BaseHandler - -class MetricsTimelineHandler(BaseHandler): - """ - URL - /topologies/metricstimeline - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - component (required) - - metricname (required, repeated) - - starttime (required) - - endtime (required) - - instance (optional, repeated) - - The response JSON is a map of all the requested - (or if nothing is mentioned, all) components - of the topology, to the metrics that are reported - by that component. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - component = self.get_argument_component() - metric_names = self.get_required_arguments_metricnames() - start_time = self.get_argument_starttime() - end_time = self.get_argument_endtime() - self.validateInterval(start_time, end_time) - instances = self.get_arguments(constants.PARAM_INSTANCE) - - topology = self.tracker.get_topology( - cluster, role, environ, topology_name) - metrics = yield tornado.gen.Task(metricstimeline.get_metrics_timeline, - topology.tmanager, component, metric_names, - instances, int(start_time), int(end_time)) - self.write_success_response(metrics) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/packingplanhandler.py b/heron/tools/tracker/src/python/handlers/packingplanhandler.py deleted file mode 100644 index a71e08cba78..00000000000 --- a/heron/tools/tracker/src/python/handlers/packingplanhandler.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' packingplanhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class PackingPlanHandler(BaseHandler): - """ - URL - /topologies/packingplan - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - information of packing plan of the topology. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """initialize""" - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """get method""" - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - packing_plan = topology_info["packing_plan"] - self.write_success_response(packing_plan) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/physicalplanhandler.py b/heron/tools/tracker/src/python/handlers/physicalplanhandler.py deleted file mode 100644 index 0372e42d221..00000000000 --- a/heron/tools/tracker/src/python/handlers/physicalplanhandler.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' physicalplanhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class PhysicalPlanHandler(BaseHandler): - """ - URL - /topologies/physicalplan - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - information of physical plan of the topology. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """initialize""" - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """get method""" - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - physical_plan = topology_info["physical_plan"] - self.write_success_response(physical_plan) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/pidhandler.py b/heron/tools/tracker/src/python/handlers/pidhandler.py deleted file mode 100644 index d911a1d88ed..00000000000 --- a/heron/tools/tracker/src/python/handlers/pidhandler.py +++ /dev/null @@ -1,90 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' pidhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python import utils -from heron.tools.tracker.src.python.handlers import BaseHandler - - -@tornado.gen.coroutine -def getInstancePid(topology_info, instance_id): - """ - This method is used by other modules, and so it - is not a part of the class. - Fetches Instance pid from heron-shell. - """ - try: - http_client = tornado.httpclient.AsyncHTTPClient() - endpoint = utils.make_shell_endpoint(topology_info, instance_id) - url = "%s/pid/%s" % (endpoint, instance_id) - Log.debug("HTTP call for url: %s", url) - response = yield http_client.fetch(url) - raise tornado.gen.Return(response.body) - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) - - -class PidHandler(BaseHandler): - """ - URL - /topologies/jmap?cluster=&topology= \ - &environ=&instance= - Parameters: - - cluster - Name of the cluster. - - role - (optional) Role used to submit the topology. - - environ - Running environment. - - topology - Name of topology (Note: Case sensitive. Can only - include [a-zA-Z0-9-_]+) - - instance - Instance Id - - If successfule returns the pid of instance. May include training - spaces and/or linefeed before/after. - The response JSON is a dict with following format: - { - 'command': Full command executed at server. - 'stdout': Text on stdout of executing the command. - 'stderr': Text on stderr. - } - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - instance = self.get_argument_instance() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - result = yield getInstancePid(topology_info, instance) - self.write_success_response(result) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py b/heron/tools/tracker/src/python/handlers/runtimestatehandler.py deleted file mode 100644 index 2e9d098e0d9..00000000000 --- a/heron/tools/tracker/src/python/handlers/runtimestatehandler.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' runtimestatehandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.proto import tmanager_pb2 -from heron.tools.tracker.src.python.handlers import BaseHandler - -# pylint: disable=attribute-defined-outside-init -class RuntimeStateHandler(BaseHandler): - """ - URL - /topologies/runtimestate - Parameters: - - cluster (required) - - environ (required) - - role - (optional) Role used to submit the topology. - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - runtime information of a topology. Static properties - is availble from /topologies/metadata. - - Example JSON response: - { - has_tmanager_location: true, - stmgrs_reg_summary: { - registered_stmgrs: [ - "stmgr-1", - "stmgr-2" - ], - absent_stmgrs: [ ] - }, - has_scheduler_location: true, - has_physical_plan: true - } - """ - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - # pylint: disable=dangerous-default-value, no-self-use, unused-argument - @tornado.gen.coroutine - def getStmgrsRegSummary(self, tmanager, callback=None): - """ - Get summary of stream managers registration summary - """ - if not tmanager or not tmanager.host or not tmanager.stats_port: - return - reg_request = tmanager_pb2.StmgrsRegistrationSummaryRequest() - request_str = reg_request.SerializeToString() - port = str(tmanager.stats_port) - host = tmanager.host - url = "http://{0}:{1}/stmgrsregistrationsummary".format(host, port) - request = tornado.httpclient.HTTPRequest(url, - body=request_str, - method='POST', - request_timeout=5) - Log.debug('Making HTTP call to fetch stmgrsregistrationsummary url: %s', url) - try: - client = tornado.httpclient.AsyncHTTPClient() - result = yield client.fetch(request) - Log.debug("HTTP call complete.") - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) - # Check the response code - error if it is in 400s or 500s - responseCode = result.code - if responseCode >= 400: - message = "Error in getting exceptions from Tmanager, code: " + responseCode - Log.error(message) - raise tornado.gen.Return({ - "message": message - }) - # Parse the response from tmanager. - reg_response = tmanager_pb2.StmgrsRegistrationSummaryResponse() - reg_response.ParseFromString(result.body) - # Send response - ret = {} - for stmgr in reg_response.registered_stmgrs: - ret[stmgr] = True - for stmgr in reg_response.absent_stmgrs: - ret[stmgr] = False - raise tornado.gen.Return(ret) - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - runtime_state = topology_info["runtime_state"] - runtime_state["topology_version"] = topology_info["metadata"]["release_version"] - topology = self.tracker.get_topology( - cluster, role, environ, topology_name) - reg_summary = yield tornado.gen.Task(self.getStmgrsRegSummary, topology.tmanager) - for stmgr, reg in list(reg_summary.items()): - runtime_state["stmgrs"].setdefault(stmgr, {})["is_registered"] = reg - self.write_success_response(runtime_state) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py b/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py deleted file mode 100644 index 9d38634b326..00000000000 --- a/heron/tools/tracker/src/python/handlers/schedulerlocationhandler.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' schedulerlocationhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class SchedulerLocationHandler(BaseHandler): - """ - URL - /topologies/schedulerlocation - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - The response JSON is a dictionary with all the - information of scheduler location of the topology. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - scheduler_location = topology_info["scheduler_location"] - self.write_success_response(scheduler_location) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/stateshandler.py b/heron/tools/tracker/src/python/handlers/stateshandler.py deleted file mode 100644 index f1513f73893..00000000000 --- a/heron/tools/tracker/src/python/handlers/stateshandler.py +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' statehandler.py ''' -import tornado.gen - -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class StatesHandler(BaseHandler): - """ - URL - /topologies/states - Parameters: - - cluster (optional) - - environ (optional) - - The response JSON is a dict with following format: - { - : { - : { - : { - - }, - : {...} - ... - }, - : {...}, - ... - }, - : {...} - } - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - # Get all the values for parameter "cluster". - clusters = self.get_arguments(constants.PARAM_CLUSTER) - - # Get all the values for parameter "environ". - environs = self.get_arguments(constants.PARAM_ENVIRON) - - role = self.get_argument_role() - - ret = {} - topologies = self.tracker.topologies - for topology in topologies: - cluster = topology.cluster - environ = topology.environ - if not cluster or not environ: - continue - - # This cluster is not asked for. - # Note that "if not clusters", then - # we show for all the clusters. - if clusters and cluster not in clusters: - continue - - # This environ is not asked for. - # Note that "if not environs", then - # we show for all the environs. - if environs and environ not in environs: - continue - - if cluster not in ret: - ret[cluster] = {} - if environ not in ret[cluster]: - ret[cluster][environ] = {} - try: - topology_info = self.tracker.get_topology_info(topology.name, cluster, role, environ) - if topology_info and "execution_state" in topology_info: - ret[cluster][environ][topology.name] = topology_info["execution_state"] - except Exception: - # Do nothing - pass - self.write_success_response(ret) diff --git a/heron/tools/tracker/src/python/handlers/topologieshandler.py b/heron/tools/tracker/src/python/handlers/topologieshandler.py deleted file mode 100644 index cbb9e1b38c3..00000000000 --- a/heron/tools/tracker/src/python/handlers/topologieshandler.py +++ /dev/null @@ -1,108 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' topologieshandler.py ''' -import tornado.gen - -from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class TopologiesHandler(BaseHandler): - """ - URL - /topologies - Parameters: - - cluster (optional) - - tag (optional) - - The response JSON is a dict with following format: - { - : { - : [ - topology1, - topology2, - ... - ], - : [ - topology1, - topology2, - ... - ], - : [...], - ... - }, - : {...} - } - """ - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - # Get all the values for parameter "cluster". - clusters = self.get_arguments(constants.PARAM_CLUSTER) - # Get all the values for parameter "environ". - environs = self.get_arguments(constants.PARAM_ENVIRON) - # Get role - role = self.get_argument_role() - - ret = {} - topologies = self.tracker.topologies - for topology in topologies: - cluster = topology.cluster - environ = topology.environ - execution_state = topology.execution_state - - if not cluster or not execution_state or not environ: - continue - - topo_role = execution_state.role - if not topo_role: - continue - - # This cluster is not asked for. - # Note that "if not clusters", then - # we show for all the clusters. - if clusters and cluster not in clusters: - continue - - # This environ is not asked for. - # Note that "if not environs", then - # we show for all the environs. - if environs and environ not in environs: - continue - - # This role is not asked for. - # Note that "if not role", then - # we show for all the roles. - if role and role != topo_role: - continue - - if cluster not in ret: - ret[cluster] = {} - if topo_role not in ret[cluster]: - ret[cluster][topo_role] = {} - if environ not in ret[cluster][topo_role]: - ret[cluster][topo_role][environ] = [] - ret[cluster][topo_role][environ].append(topology.name) - self.write_success_response(ret) diff --git a/heron/tools/tracker/src/python/handlers/topologyconfighandler.py b/heron/tools/tracker/src/python/handlers/topologyconfighandler.py deleted file mode 100644 index 127e9304cdb..00000000000 --- a/heron/tools/tracker/src/python/handlers/topologyconfighandler.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' topologyhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class TopologyConfigHandler(BaseHandler): - """ - url - /topologies/config - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - the response json is a dictionary with all the - configuration for the topology. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - config = topology_info["physical_plan"]["config"] - self.write_success_response(config) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/handlers/topologyhandler.py b/heron/tools/tracker/src/python/handlers/topologyhandler.py deleted file mode 100644 index c700086d04b..00000000000 --- a/heron/tools/tracker/src/python/handlers/topologyhandler.py +++ /dev/null @@ -1,61 +0,0 @@ -#!/usr/bin/env python3 -# -*- encoding: utf-8 -*- - -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -''' topologyhandler.py ''' -import traceback -import tornado.gen -import tornado.web - -from heron.common.src.python.utils.log import Log -from heron.tools.tracker.src.python.handlers import BaseHandler - - -class TopologyHandler(BaseHandler): - """ - url - /topologies/info - Parameters: - - cluster (required) - - role - (optional) Role used to submit the topology. - - environ (required) - - topology (required) name of the requested topology - - the response json is a dictionary with all the - information of the topology, including its - logical and physical plan. - """ - - # pylint: disable=attribute-defined-outside-init - def initialize(self, tracker): - """ initialize """ - self.tracker = tracker - - @tornado.gen.coroutine - def get(self): - """ get method """ - try: - cluster = self.get_argument_cluster() - role = self.get_argument_role() - environ = self.get_argument_environ() - topology_name = self.get_argument_topology() - topology_info = self.tracker.get_topology_info(topology_name, cluster, role, environ) - self.write_success_response(topology_info) - except Exception as e: - Log.debug(traceback.format_exc()) - self.write_error_response(e) diff --git a/heron/tools/tracker/src/python/main.py b/heron/tools/tracker/src/python/main.py index 345ee55437a..a0560810209 100644 --- a/heron/tools/tracker/src/python/main.py +++ b/heron/tools/tracker/src/python/main.py @@ -19,88 +19,25 @@ # under the License. ''' main.py ''' +import signal import logging import os -import signal import sys -import tornado.httpserver -import tornado.ioloop -import tornado.web -from tornado.options import define -from tornado.httpclient import AsyncHTTPClient from heron.tools.common.src.python.utils import config as common_config from heron.common.src.python.utils import log from heron.tools.tracker.src.python import constants -from heron.tools.tracker.src.python import handlers from heron.tools.tracker.src.python import utils from heron.tools.tracker.src.python.config import Config, STATEMGRS_KEY from heron.tools.tracker.src.python.tracker import Tracker +from heron.tools.tracker.src.python.app import app +from heron.tools.tracker.src.python import state import click +import uvicorn Log = log.Log -class Application(tornado.web.Application): - """ Tornado server application """ - def __init__(self, config): - - AsyncHTTPClient.configure(None, defaults=dict(request_timeout=120.0)) - self.tracker = Tracker(config) - self.tracker.synch_topologies() - tornadoHandlers = [ - (r"/", handlers.MainHandler), - (r"/clusters", handlers.ClustersHandler, {"tracker":self.tracker}), - (r"/topologies", handlers.TopologiesHandler, {"tracker":self.tracker}), - (r"/topologies/states", handlers.StatesHandler, {"tracker":self.tracker}), - (r"/topologies/info", handlers.TopologyHandler, {"tracker":self.tracker}), - (r"/topologies/logicalplan", handlers.LogicalPlanHandler, {"tracker":self.tracker}), - (r"/topologies/config", handlers.TopologyConfigHandler, {"tracker":self.tracker}), - (r"/topologies/containerfiledata", handlers.ContainerFileDataHandler, - {"tracker":self.tracker}), - (r"/topologies/containerfiledownload", handlers.ContainerFileDownloadHandler, - {"tracker":self.tracker}), - (r"/topologies/containerfilestats", - handlers.ContainerFileStatsHandler, {"tracker":self.tracker}), - (r"/topologies/physicalplan", handlers.PhysicalPlanHandler, {"tracker":self.tracker}), - (r"/topologies/packingplan", handlers.PackingPlanHandler, {"tracker":self.tracker}), - # Deprecated. See https://github.com/apache/incubator-heron/issues/1754 - (r"/topologies/executionstate", handlers.ExecutionStateHandler, {"tracker":self.tracker}), - (r"/topologies/schedulerlocation", handlers.SchedulerLocationHandler, - {"tracker":self.tracker}), - (r"/topologies/metadata", handlers.MetaDataHandler, {"tracker":self.tracker}), - (r"/topologies/runtimestate", handlers.RuntimeStateHandler, {"tracker":self.tracker}), - (r"/topologies/metrics", handlers.MetricsHandler, {"tracker":self.tracker}), - (r"/topologies/metricstimeline", handlers.MetricsTimelineHandler, {"tracker":self.tracker}), - (r"/topologies/metricsquery", handlers.MetricsQueryHandler, {"tracker":self.tracker}), - (r"/topologies/exceptions", handlers.ExceptionHandler, {"tracker":self.tracker}), - (r"/topologies/exceptionsummary", handlers.ExceptionSummaryHandler, - {"tracker":self.tracker}), - (r"/machines", handlers.MachinesHandler, {"tracker":self.tracker}), - (r"/topologies/pid", handlers.PidHandler, {"tracker":self.tracker}), - (r"/topologies/jstack", handlers.JstackHandler, {"tracker":self.tracker}), - (r"/topologies/jmap", handlers.JmapHandler, {"tracker":self.tracker}), - (r"/topologies/histo", handlers.MemoryHistogramHandler, {"tracker":self.tracker}), - (r"(.*)", handlers.DefaultHandler), - ] - - settings = dict( - debug=True, - serve_traceback=True, - static_path=os.path.dirname(__file__) - ) - tornado.web.Application.__init__(self, tornadoHandlers, **settings) - Log.info("Tracker has started") - - def stop(self): - self.tracker.stop_sync() - - -def define_options(port: int, config_file: str) -> None: - """ define Tornado global variables """ - define("port", default=port) - define("config_file", default=config_file) - def create_tracker_config(config_file: str, stmgr_override: dict) -> dict: # try to parse the config file if we find one @@ -186,10 +123,8 @@ def cli( """ - log.configure(logging.DEBUG if verbose else logging.INFO) - - # set Tornado global option - define_options(port, config_file) + log_level = logging.DEBUG if verbose else logging.INFO + log.configure(log_level) stmgr_override = { "type": stmgr_type, @@ -200,32 +135,16 @@ def cli( } config = Config(create_tracker_config(config_file, stmgr_override)) - # create Tornado application - application = Application(config) - - # pylint: disable=unused-argument - # SIGINT handler: - # 1. stop all the running zkstatemanager and filestatemanagers - # 2. stop the Tornado IO loop - def signal_handler(signum, frame): - # start a new line after ^C character because this looks nice - print('\n', end='') - application.stop() - tornado.ioloop.IOLoop.instance().stop() - - # associate SIGINT and SIGTERM with a handler - signal.signal(signal.SIGINT, signal_handler) - signal.signal(signal.SIGTERM, signal_handler) - - Log.info("Running on port: %d", port) - if config_file: - Log.info("Using config file: %s", config_file) - Log.info(f"Using state manager:\n{config}") + state.tracker = Tracker(config) + state.tracker.sync_topologies() + # this only returns when interrupted + uvicorn.run(app, host="0.0.0.0", port=port, log_level=log_level) + state.tracker.stop_sync() - http_server = tornado.httpserver.HTTPServer(application) - http_server.listen(port) + # non-daemon threads linger and stop the process for quitting, so signal + # for cleaning up + os.kill(os.getpid(), signal.SIGKILL) - tornado.ioloop.IOLoop.instance().start() if __name__ == "__main__": cli() # pylint: disable=no-value-for-parameter diff --git a/heron/tools/tracker/src/python/metricstimeline.py b/heron/tools/tracker/src/python/metricstimeline.py index 40e30c95eda..75b33c8f1b8 100644 --- a/heron/tools/tracker/src/python/metricstimeline.py +++ b/heron/tools/tracker/src/python/metricstimeline.py @@ -19,17 +19,28 @@ # under the License. """ metricstimeline.py """ -from typing import List - -import tornado.gen +from typing import Dict, List from heron.common.src.python.utils.log import Log from heron.proto import common_pb2 from heron.proto import tmanager_pb2 +import httpx + +from pydantic import BaseModel, Field + + +class MetricsTimeline(BaseModel): + component: str + start_time: int = Field(..., alias="starttime") + end_time: int = Field(..., alias="enddtime") + timeline: Dict[str, Dict[str, Dict[int, int]]] = Field( + ..., + description="map of (metric name, instance, start) to metric value", + ) + # pylint: disable=too-many-locals, too-many-branches, unused-argument -@tornado.gen.coroutine -def get_metrics_timeline( +async def get_metrics_timeline( tmanager: tmanager_pb2.TManagerLocation, component_name: str, metric_names: List[str], @@ -37,40 +48,17 @@ def get_metrics_timeline( start_time: int, end_time: int, callback=None, -) -> dict: +) -> MetricsTimeline: """ Get the specified metrics for the given component name of this topology. - Returns the following dict on success: - { - "timeline": { - : { - : { - : , - : , - ... - } - ... - }, ... - }, - "starttime": , - "endtime": , - "component": "..." - } - - Returns the following dict on failure: - { - "message": "..." - } + """ + # Tmanager is the proto object and must have host and port for stats. if not tmanager or not tmanager.host or not tmanager.stats_port: raise Exception("No Tmanager found") - host = tmanager.host - port = tmanager.stats_port - # Create the proto request object to get metrics. - request_parameters = tmanager_pb2.MetricRequest() request_parameters.component_name = component_name @@ -84,21 +72,9 @@ def get_metrics_timeline( request_parameters.minutely = True # Form and send the http request. - url = f"http://{host}:{port}/stats" - request = tornado.httpclient.HTTPRequest(url, - body=request_parameters.SerializeToString(), - method='POST', - request_timeout=5) - - Log.debug("Making HTTP call to fetch metrics") - Log.debug("url: " + url) - try: - client = tornado.httpclient.AsyncHTTPClient() - result = yield client.fetch(request) - Log.debug("HTTP call complete.") - except tornado.httpclient.HTTPError as e: - raise Exception(str(e)) - + url = f"http://{tmanager.host}:{tmanager.stats_port}/stats" + with httpx.AsyncClient() as client: + result = await client.post(url, data=request_parameters.SerializeToString()) # Check the response code - error if it is in 400s or 500s if result.code >= 400: @@ -107,19 +83,13 @@ def get_metrics_timeline( # Parse the response from tmanager. response_data = tmanager_pb2.MetricResponse() - response_data.ParseFromString(result.body) + response_data.ParseFromString(result.content) if response_data.status.status == common_pb2.NOTOK: if response_data.status.HasField("message"): Log.warn("Received response from Tmanager: %s", response_data.status.message) - # Form the response. - ret = {} - ret["starttime"] = start_time - ret["endtime"] = end_time - ret["component"] = component_name - ret["timeline"] = {} - + timeline = {} # Loop through all the metrics # One instance corresponds to one metric, which can have # multiple IndividualMetrics for each metricname requested. @@ -129,15 +99,18 @@ def get_metrics_timeline( # Loop through all individual metrics. for im in metric.metric: metricname = im.name - if metricname not in ret["timeline"]: - ret["timeline"][metricname] = {} - if instance not in ret["timeline"][metricname]: - ret["timeline"][metricname][instance] = {} + if instance not in timeline[metricname]: + timeline.setdefault(metricname, {})[instance] = {} # We get minutely metrics. # Interval-values correspond to the minutely mark for which # this metric value corresponds to. for interval_value in im.interval_values: - ret["timeline"][metricname][instance][interval_value.interval.start] = interval_value.value - - raise tornado.gen.Return(ret) + timeline[metricname][instance][interval_value.interval.start] = interval_value.value + + return MetricsTimeline( + starttime=start_time, + endtime=end_time, + component=component_name, + timeline=timeline, + ) diff --git a/heron/tools/tracker/src/python/query.py b/heron/tools/tracker/src/python/query.py index 6143b11d176..6a54ba3ffd4 100644 --- a/heron/tools/tracker/src/python/query.py +++ b/heron/tools/tracker/src/python/query.py @@ -17,16 +17,247 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +""" +### Metrics Query Language -""" query.py """ -from typing import Any, List, Optional +Metrics queries are useful when some kind of aggregated values are required. For example, +to find the total number of tuples emitted by a spout, `SUM` operator can be used, instead +of fetching metrics for all the instances of the corresponding component, and then summing them. + +#### Terminology + +1. Univariate Timeseries --- A timeseries is called univariate if there is only one set +of minutely data. For example, a timeseries representing the sums of a number of timeseries +would be a univariate timeseries. +2. Multivariate Timeseries --- A set of multiple timeseries is collectively called multivariate. +Note that these timeseries are associated with their instances. + +#### Operators + +##### TS + +```text +TS(componentName, instance, metricName) +``` + +Example: + +```text +TS(component1, *, __emit-count/stream1) +``` + +Time Series Operator. This is the basic operator that is responsible for getting +metrics from TManager. +Accepts a list of 3 elements: + +1. componentName +2. instance - can be "*" for all instances, or a single instance ID +3. metricName - Full metric name with stream id if applicable + +Returns a univariate time series in case of a single instance id given, otherwise returns +a multivariate time series. + +--- + +##### DEFAULT + +```text +DEFAULT(0, TS(component1, *, __emit-count/stream1)) +``` +If the second operator returns more than one timeline, so will the +DEFAULT operator. + +```text +DEFAULT(100.0, SUM(TS(component2, *, __emit-count/default))) <-- +``` +Second operator can be any operator + +Default Operator. This operator is responsible for filling missing values in the metrics timeline. +Must have 2 arguments + +1. First argument is a numeric constant representing the number to fill the missing values with +2. Second one must be one of the operators, that return the metrics timeline + +Returns a univariate or multivariate time series, based on what the second operator is. + +--- + +##### SUM + +```text +SUM(TS(component1, instance1, metric1), DEFAULT(0, TS(component1, *, metric2))) +``` + +Sum Operator. This operator is used to take sum of all argument time series. It can have +any number of arguments, +each of which must be one of the following two types: + +1. Numeric constants, which will fill in the missing values as well, or +2. Operator, which returns one or more timelines + +Returns only a single timeline representing the sum of all time series for each timestamp. +Note that "instance" attribute is not there in the result. + +--- + +##### MAX + +```text +MAX(100, TS(component1, *, metric1)) +``` + +Max Operator. This operator is used to find max of all argument operators for each +individual timestamp. +Each argument must be one of the following types: + +1. Numeric constants, which will fill in the missing values as well, or +2. Operator, which returns one or more timelines + +Returns only a single timeline representing the max of all the time series for each timestamp. +Note that "instance" attribute is not included in the result. + +--- + +##### PERCENTILE + +```text +PERCENTILE(99, TS(component1, *, metric1)) +``` -import tornado.httpclient -import tornado.gen +Percentile Operator. This operator is used to find a quantile of all timelines retuned by +the arguments, for each timestamp. +This is a more general type of query similar to MAX. Note that `PERCENTILE(100, TS...)` is +equivalent to `Max(TS...)`. +Each argument must be either constant or Operators. +First argument must always be the required Quantile. + +1. Quantile (first argument) - Required quantile. 100 percentile = max, 0 percentile = min. +2. Numeric constants will fill in the missing values as well, +3. Operator - which returns one or more timelines + +Returns only a single timeline representing the quantile of all the time series +for each timestamp. Note that "instance" attribute is not there in the result. + +--- + +##### DIVIDE + +```text +DIVIDE(TS(component1, *, metrics1), 100) +``` + +Divide Operator. Accepts two arguments, both can be univariate or multivariate. +Each can be of one of the following types: + +1. Numeric constant will be considered as a constant time series for all applicable + timestamps, they will not fill the missing values +2. Operator - returns one or more timelines + +Three main cases are: + +1. When both operands are multivariate + 1. Divide operation will be done on matching data, that is, with same instance id. + 2. If the instances in both the operands do not match, error is thrown. + 3. Returns multivariate time series, each representing the result of division on the two + corresponding time series. +2. When one operand is univariate, and other is multivariate + 1. This includes division by constants as well. + 2. The univariate operand will participate with all time series in multivariate. + 3. The instance information of the multivariate time series will be preserved in the result. + 4. Returns multivariate time series. +3. When both operands are univariate + 1. Instance information is ignored in this case + 2. Returns univariate time series which is the result of division operation. + +--- + +##### MULTIPLY + +```text +MULTIPLY(10, TS(component1, *, metrics1)) +``` + +Multiply Operator. Has same conditions as division operator. This is to keep the API simple. +Accepts two arguments, both can be univariate or multivariate. Each can be of one of +the following types: + +1. Numeric constant will be considered as a constant time series for all applicable + timestamps, they will not fill the missing values +2. Operator - returns one or more timelines + +Three main cases are: + +1. When both operands are multivariate + 1. Multiply operation will be done on matching data, that is, with same instance id. + 2. If the instances in both the operands do not match, error is thrown. + 3. Returns multivariate time series, each representing the result of multiplication + on the two corresponding time series. +2. When one operand is univariate, and other is multivariate + 1. This includes multiplication by constants as well. + 2. The univariate operand will participate with all time series in multivariate. + 3. The instance information of the multivariate time series will be preserved in the result. + 4. Returns multivariate timeseries. +3. When both operands are univariate + 1. Instance information is ignored in this case + 2. Returns univariate timeseries which is the result of multiplication operation. + +--- + +##### SUBTRACT + +```text +SUBTRACT(TS(component1, instance1, metrics1), TS(componet1, instance1, metrics2)) + +SUBTRACT(TS(component1, instance1, metrics1), 100) +``` + +Subtract Operator. Has same conditions as division operator. This is to keep the API simple. +Accepts two arguments, both can be univariate or multivariate. Each can be of one of +the following types: + +1. Numeric constant will be considered as a constant time series for all applicable + timestamps, they will not fill the missing values +2. Operator - returns one or more timelines + +Three main cases are: + +1. When both operands are multivariate + 1. Subtract operation will be done on matching data, that is, with same instance id. + 2. If the instances in both the operands do not match, error is thrown. + 3. Returns multivariate time series, each representing the result of subtraction + on the two corresponding time series. +2. When one operand is univariate, and other is multivariate + 1. This includes subtraction by constants as well. + 2. The univariate operand will participate with all time series in multivariate. + 3. The instance information of the multivariate time series will be preserved in the result. + 4. Returns multivariate time series. +3. When both operands are univariate + 1. Instance information is ignored in this case + 2. Returns univariate time series which is the result of subtraction operation. + +--- + +##### RATE + +```text +RATE(SUM(TS(component1, *, metrics1))) + +RATE(TS(component1, *, metrics2)) +``` + +Rate Operator. This operator is used to find rate of change for all timeseries. +Accepts a only a single argument, which must be an Operators which returns +univariate or multivariate time series. +Returns univariate or multivariate time series based on the argument, with each +timestamp value corresponding to the rate of change for that timestamp. +""" + +from typing import Any, List, Optional from heron.proto.tmanager_pb2 import TManagerLocation from heron.tools.tracker.src.python.query_operators import * + #################################################################### # Parsing and executing the query string. #################################################################### @@ -55,8 +286,7 @@ def __init__(self, tracker): # pylint: disable=attribute-defined-outside-init, no-member - @tornado.gen.coroutine - def execute_query( + async def execute_query( self, tmanager: TManagerLocation, query_string: str, @@ -68,8 +298,7 @@ def execute_query( raise Exception("No tmanager found") self.tmanager = tmanager root = self.parse_query_string(query_string) - metrics = yield root.execute(self.tracker, self.tmanager, start, end) - raise tornado.gen.Return(metrics) + return await root.execute(self.tracker, self.tmanager, start, end) def find_closing_braces(self, query: str) -> int: """Find the index of the closing braces for the opening braces @@ -128,7 +357,7 @@ def parse_query_string(self, query: str) -> Optional[Operator]: constant = float(query) return constant except ValueError: - raise Exception("Invalid syntax") + raise Exception("Invalid syntax") from ValueError token = query[:start_index].rstrip() operator_cls = self.operators.get(token) if operator_cls is None: diff --git a/heron/tools/tracker/src/python/query_operators.py b/heron/tools/tracker/src/python/query_operators.py index 3b2dc1b85c1..8475e123f5d 100644 --- a/heron/tools/tracker/src/python/query_operators.py +++ b/heron/tools/tracker/src/python/query_operators.py @@ -19,6 +19,7 @@ # under the License. ''' query_operators.py ''' +import asyncio import math from typing import Any, Dict, List, Optional, Union @@ -26,9 +27,6 @@ from heron.proto.tmanager_pb2 import TManagerLocation from heron.tools.tracker.src.python.metricstimeline import get_metrics_timeline -import tornado.httpclient -import tornado.gen - ##################################################################### # Data Structure for fetched Metrics @@ -105,8 +103,7 @@ def __init__(self, _): raise Exception("Not implemented exception") # pylint: disable=unused-argument - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: + async def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: """ execute """ raise Exception("Not implemented exception") @@ -134,12 +131,17 @@ def __init__(self, children): if instance != "*": self.instances.append(instance) - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Optional[Any]: + async def execute( + self, + tracker, + tmanager: TManagerLocation, + start: int, + end: int, + ) -> Optional[Any]: # Fetch metrics for start-60 to end+60 because the minute mark # may be a little skewed. By getting a couple more values, # we can then truncate based on the interval needed. - metrics = yield get_metrics_timeline( + metrics = await get_metrics_timeline( tmanager, self.component, [self.metric_name], self.instances, start - 60, end + 60) if not metrics: @@ -161,7 +163,7 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> }) for instance, timeline in timelines.items() ] - raise tornado.gen.Return(all_metrics) + return all_metrics class Default(Operator): @@ -189,14 +191,13 @@ def __init__(self, children): self.constant = default self.timeseries = timeseries - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: - all_metrics = yield self.timeseries.execute(tracker, tmanager, start, end) + async def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: + all_metrics = await self.timeseries.execute(tracker, tmanager, start, end) if isinstance(all_metrics, str): raise Exception(all_metrics) for metric in all_metrics: metric.setDefault(self.constant, start, end) - raise tornado.gen.Return(all_metrics) + return all_metrics class Sum(Operator): """ @@ -211,24 +212,23 @@ class Sum(Operator): """ # pylint: disable=super-init-not-called def __init__(self, children) -> None: - self.timeSeriesList = children + self.time_series_list = children - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: + async def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: # Initialize the metric to be returned with sum of all the constants. result = Metrics(None, None, None, start, end, {}) - constant_sum = sum(ts for ts in self.timeSeriesList if isinstance(ts, float)) + constant_sum = sum(ts for ts in self.time_series_list if isinstance(ts, float)) result.setDefault(constant_sum, start, end) futureMetrics = [ ts.execute(tracker, tmanager, start, end) - for ts in self.timeSeriesList if isinstance(ts, Operator) + for ts in self.time_series_list if isinstance(ts, Operator) ] - metrics = yield futureMetrics # Get all the timeseries metrics all_metrics = [] - for met in metrics: + for met_f in asyncio.as_completed(futureMetrics): + met = await met_f if isinstance(met, str): raise Exception(met) all_metrics.extend(met) @@ -240,7 +240,7 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> # result.timeline.update(timeline) which uses C if timestamp in result.timeline: result.timeline[timestamp] += value - raise tornado.gen.Return([result]) + return [result] class Max(Operator): """Max Operator. This operator is used to find max of all children timeseries @@ -255,25 +255,24 @@ class Max(Operator): def __init__(self, children): if len(children) < 1: raise Exception("MAX expects at least one operand.") - self.timeSeriesList = children + self.time_series_list = children - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: + async def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: # Initialize the metric to be returned with max of all the constants. result = Metrics(None, None, None, start, end, {}) - constants = [ts for ts in self.timeSeriesList if isinstance(ts, float)] + constants = [ts for ts in self.time_series_list if isinstance(ts, float)] if constants: result.setDefault(max(constants), start, end) futureMetrics = [ ts.execute(tracker, tmanager, start, end) - for ts in self.timeSeriesList if isinstance(ts, Operator) + for ts in self.time_series_list if isinstance(ts, Operator) ] - metrics = yield futureMetrics # Get all the timeseries metrics all_metrics = [] - for met in metrics: + for met_f in asyncio.as_completed(futureMetrics): + met = await met_f if isinstance(met, str): raise Exception(met) all_metrics.extend(met) @@ -285,7 +284,7 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> if timestamp not in result.timeline: result.timeline[timestamp] = value result.timeline[timestamp] = max(value, result.timeline[timestamp]) - raise tornado.gen.Return([result]) + return [result] class Percentile(Operator): """ @@ -313,19 +312,18 @@ def __init__(self, children): if not 0 <= quantile <= 100: raise Exception("Quantile must be between 0 and 100 inclusive.") self.quantile = quantile - self.timeSeriesList = timeseries_list + self.time_series_list = timeseries_list - @tornado.gen.coroutine - def execute(self, tracker, tmanager, start, end): + async def execute(self, tracker, tmanager, start, end): futureMetrics = [ ts.execute(tracker, tmanager, start, end) - for ts in self.timeSeriesList if isinstance(ts, Operator) + for ts in self.time_series_list if isinstance(ts, Operator) ] - metrics = yield futureMetrics # Get all the timeseries metrics all_metrics = [] - for met in metrics: + for met_f in asyncio.as_completed(futureMetrics): + met = await met_f if isinstance(met, str): raise Exception(met) all_metrics.extend(met) @@ -345,7 +343,7 @@ def execute(self, tracker, tmanager, start, end): index = int(self.quantile * 1.0 * (len(values) - 1) / 100.0) timeline[timestamp] = sorted(values)[index] result = Metrics(None, None, None, start, end, timeline) - raise tornado.gen.Return([result]) + return [result] class _SimpleArithmaticOperator(Operator): @@ -384,8 +382,7 @@ def __init__(self, children: list) -> None: self.operand2: Union[float, Operator] = children[1] @classmethod - @tornado.gen.coroutine - def _get_metrics( + async def _get_metrics( cls, operand: Union[float, Operator], tracker, @@ -399,7 +396,7 @@ def _get_metrics( met.setDefault(operand, start, end) result[""] = met else: - met = yield operand.execute(tracker, tmanager, start, end) + met = await operand.execute(tracker, tmanager, start, end) if not met: pass elif len(met) == 1 and not met[0].instance: @@ -410,7 +407,7 @@ def _get_metrics( if not m.instance: raise Exception(f"{cls.NAME} with multivariate requires instance based timeseries") result[m.instance] = m - raise tornado.gen.Return(result) + return result @staticmethod def _is_multivariate(metrics: dict) -> bool: @@ -427,17 +424,16 @@ def _f(self, lhs: float, rhs: Optional[float]) -> None: raise NotImplementedError() # pylint: disable=too-many-branches - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: + async def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: """ Return _f applied over all values in [start, end] of the two operands. Scalars are expanded a timeseries with all points equal to the scalar. """ - metrics, metrics2 = yield [ + metrics, metrics2 = await asyncio.gather( self._get_metrics(self.operand1, tracker, tmanager, start, end), self._get_metrics(self.operand2, tracker, tmanager, start, end), - ] + ) # In case both are multivariate, only equal instances will get operated if self._is_multivariate(metrics) and self._is_multivariate(metrics2): @@ -456,7 +452,7 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> else: met.timeline[timestamp] = value all_metrics.append(met) - raise tornado.gen.Return(all_metrics) + return all_metrics # If first is univariate if not self._is_multivariate(metrics): @@ -472,7 +468,7 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> else: met.timeline[timestamp] = v all_metrics.append(met) - raise tornado.gen.Return(all_metrics) + return all_metrics # If second is univariate all_metrics = [] @@ -486,7 +482,7 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> else: met.timeline[timestamp] = v all_metrics.append(met) - raise tornado.gen.Return(all_metrics) + return all_metrics class Multiply(_SimpleArithmaticOperator): @@ -525,15 +521,14 @@ class Rate(Operator): def __init__(self, children) -> None: if len(children) != 1: raise Exception("RATE requires exactly one argument.") - timeSeries, = children - if not isinstance(timeSeries, Operator): + time_series, = children + if not isinstance(time_series, Operator): raise Exception("RATE requires a timeseries, not constant.") - self.timeSeries = timeSeries + self.time_series = time_series - @tornado.gen.coroutine - def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: + async def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> Any: # Get 1 previous data point to be able to apply rate on the first data - metrics = yield self.timeSeries.execute(tracker, tmanager, start-60, end) + metrics = await self.time_series.execute(tracker, tmanager, start-60, end) # Apply rate on all of them for metric in metrics: @@ -544,4 +539,4 @@ def execute(self, tracker, tmanager: TManagerLocation, start: int, end: int) -> if start <= timestamp <= end and timestamp - prev == 60: timeline[timestamp] = metric.timeline[timestamp] - metric.timeline[prev] metric.timeline = timeline - raise tornado.gen.Return(metrics) + return metrics diff --git a/heron/tools/tracker/src/python/routers/container.py b/heron/tools/tracker/src/python/routers/container.py new file mode 100644 index 00000000000..284ccbf50ce --- /dev/null +++ b/heron/tools/tracker/src/python/routers/container.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +These methods provide information and data about the state of a running +topology, particularly data about heron containers. + +""" +from typing import List, Optional + +from heron.proto import common_pb2, tmanager_pb2 +from heron.tools.tracker.src.python import state, utils +from heron.tools.tracker.src.python.utils import EnvelopingAPIRouter + +import httpx + +from fastapi import Query +from pydantic import BaseModel, Field +from starlette.responses import StreamingResponse + +router = EnvelopingAPIRouter() + + +@router.get("/containerfiledata") +async def get_container_file_slice( # pylint: disable=too-many-arguments + cluster: str, + environ: str, + role: Optional[str], + container: str, + path: str, + offset: int, + length: int, + topology_name: str = Query(..., alias="topology"), +): + """ + Return a range of bytes for the given file wrapped in JSON. + + Usually used to retrieve a log file chunk. + + """ + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + stmgr = state.tracker.pb2_to_api(topology)["physical_plan"]["stmgrs"][f"stmgr-{container}"] + url = f"http://{stmgr['host']}:{stmgr['shell_port']}/filedata/{path}" + params = {"offset": offset, "length": length} + + with httpx.AsyncClient() as client: + response = await client.get(url, params=params) + return response.json() + + +@router.get("/containerfiledownload", response_class=StreamingResponse) +async def get_container_file( # pylint: disable=too-many-arguments + cluster: str, + environ: str, + role: Optional[str], + container: str, + path: str, + topology_name: str = Query(..., alias="topology"), +): + """Return a given raw file.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + stmgr = state.tracker.pb2_to_api(topology)["physical_plan"]["stmgrs"][f"stmgr-{container}"] + url = f"http://{stmgr['host']}:{stmgr['shell_port']}/download/{path}" + + _, _, filename = path.rpartition("/") + with httpx.stream("GET", url) as response: + return StreamingResponse( + content=response.iter_bytes(), + headers={"Content-Disposition": f"attachment; filename={filename}"}, + ) + + +@router.get("/containerfilestats") +async def get_container_file_listing( # pylint: disable=too-many-arguments + cluster: str, + environ: str, + role: Optional[str], + container: str, + path: str, + topology_name: str = Query(..., alias="topology"), +): + """Return the stats for a given directory.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + stmgr = state.tracker.pb2_to_api(topology)["physical_plan"]["stmgrs"][f"stmgr-{container}"] + url = utils.make_shell_filestats_url(stmgr["host"], stmgr["shell_port"], path) + with httpx.AsyncClient() as client: + response = await client.get(url) + return response.json() + + +@router.get("/runtimestate") +async def get_container_runtime_state( + cluster: str, + role: Optional[str], + environ: str, + topology_name: str = Query(..., alias="topology"), +): + """Return the runtime state.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + topology_info = topology.info + tmanager = topology.tmanager + + # find out what is registed + if not (tmanager and tmanager.host and tmanager.stats_port): + raise ValueError("TManager not set yet") + url = f"http://{tmanager.host}:{tmanager.stats_port}/stmgrsregistrationsummary" + with httpx.AsyncClient() as client: + response = await client.post( + url, + data=tmanager_pb2.StmgrsRegistrationSummaryRequest().SerializeToString(), + ) + response.raise_for_status() + reg = tmanager_pb2.StmgrsRegistrationSummaryResponse() + reg.ParseFromString(response.content) + + # update the result with registration status + runtime_state = topology_info.runtime_state.copy() + for stmgr, is_registered in ( + (reg.registered_stmgrs, True), + (reg.absent_stmgrs, False), + ): + runtime_state.stmgrs[stmgr] = {"is_registered": is_registered} + + return runtime_state + +class ExceptionLog(BaseModel): + hostname: str + instance_id: str + stack_trace: str = Field(..., alias="stacktrace") + last_time: int = Field(..., alias="lasttime") + first_time: int = Field(..., alias="firsttime") + count: str = Field(..., description="number of occurances during collection interval") + logging: str = Field(..., description="additional text logged with exception") + +async def _get_exception_log_response( + cluster: str, + role: Optional[str], + environ: str, + component: str, + instances: List[str] = Query(..., alias="instance"), + topology_name: str = Query(..., alias="topology"), + summary: bool = False, +) -> List[tmanager_pb2.ExceptionLogResponse]: + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + tmanager = topology.tmanager + + if not (tmanager and tmanager.host and tmanager.stats_port): + raise ValueError("TManager not set yet") + exception_request = tmanager_pb2.ExceptionLogRequest() + exception_request.component_name = component + exception_request.instances.extend(instances) + url_suffix = "ummary" if summary else "" + url = f"http://{tmanager.host}:{tmanager.stats_port}/exceptions{url_suffix}" + with httpx.AsyncClient() as client: + response = await client.post(url, data=exception_request.SerializeToString()) + response.raise_for_status() + + exception_response = tmanager_pb2.ExceptionLogResponse() + exception_response.ParseFromString(response.content) + + if exception_response.status.status == common_pb2.NOTOK: + raise RuntimeError( + exception_response.status.message + if exception_response.status.HasField("message") + else "an error occurred" + ) + return exception_response + + +@router.get("/exceptions", response_model=List[ExceptionLog]) +async def get_exceptions( # pylint: disable=too-many-arguments + cluster: str, + role: Optional[str], + environ: str, + component: str, + instances: List[str] = Query(..., alias="instance"), + topology_name: str = Query(..., alias="topology"), +): + """Return info about exceptions that have occurred per instance.""" + exception_response = await _get_exception_log_response( + cluster, role, environ, component, instances, topology_name, summary=False + ) + + return [ + ExceptionLog( + hostname=exception_log.hostname, + instance_id=exception_log.instance_id, + stack_trace=exception_log.stacktrace, + lasttime=exception_log.lasttime, + firsttime=exception_log.firsttime, + count=str(exception_log.count), + logging=exception_log.logging, + ) + for exception_log in exception_response.exceptions + ] + + +class ExceptionSummaryItem(BaseModel): + class_name: str + last_time: int = Field(..., alias="lasttime") + first_time: int = Field(..., alias="firsttime") + count: str + +@router.get("/exceptionsummary", response_model=List[ExceptionSummaryItem]) +async def get_exceptions_summary( # pylint: disable=too-many-arguments + cluster: str, + role: Optional[str], + environ: str, + component: str, + instances: List[str] = Query(..., alias="instance"), + topology_name: str = Query(..., alias="topology"), +): + """Return info about exceptions that have occurred.""" + exception_response = await _get_exception_log_response( + cluster, role, environ, component, instances, topology_name, summary=False + ) + + return [ + ExceptionSummaryItem( + class_name=exception_log.stacktrace, + last_time=exception_log.lasttime, + first_time=exception_log.firsttime, + count=str(exception_log.count), + ) + for exception_log in exception_response.exceptions + ] + + +class ShellResponse(BaseModel): # pylint: disable=too-few-public-methods + """Response from heron-shell when executing remote commands.""" + + command: str = Field(..., description="full command executed at server") + stdout: str = Field(..., description="text on stdout") + stderr: Optional[str] = Field(None, description="text on stderr") + + +@router.get("/pid", response_model=ShellResponse) +async def get_container_heron_pid( + cluster: str, + role: Optional[str], + environ: str, + instance: str, + topology_name: str = Query(..., alias="topology"), +): + """Get the PId of the heron process.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + base_url = utils.make_shell_endpoint(state.tracker.pb2_to_api(topology), instance) + url = f"{base_url}/pid/{instance}" + with httpx.AsyncClient() as client: + return await client.get(url).json() + + +@router.get("/jstack", response_model=ShellResponse) +async def get_container_heron_jstack( + cluster: str, + role: Optional[str], + environ: str, + instance: str, + topology_name: str = Query(..., alias="topology"), +): + """Get jstack output for the heron process.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + + pid_response = await get_container_heron_pid(cluster, role, environ, instance, topology_name) + pid = pid_response["stdout"].strip() + + base_url = utils.make_shell_endpoint(state.tracker.pb2_to_api(topology), instance) + url = f"{base_url}/jstack/{pid}" + with httpx.AsyncClient() as client: + return await client.get(url).json() + + +@router.get("/jmap", response_model=ShellResponse) +async def get_container_heron_jmap( + cluster: str, + role: Optional[str], + environ: str, + instance: str, + topology_name: str = Query(..., alias="topology"), +): + """Get jmap output for the heron process.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + + pid_response = await get_container_heron_pid(cluster, role, environ, instance, topology_name) + pid = pid_response["stdout"].strip() + + base_url = utils.make_shell_endpoint(state.tracker.pb2_to_api(topology), instance) + url = f"{base_url}/jmap/{pid}" + with httpx.AsyncClient() as client: + return await client.get(url).json() + + +@router.get("/histo", response_model=ShellResponse) +async def get_container_heron_memory_histogram( + cluster: str, + role: Optional[str], + environ: str, + instance: str, + topology_name: str = Query(..., alias="topology"), +): + """Get memory usage histogram the heron process. This uses the ouput of the last jmap run.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + + pid_response = await get_container_heron_pid(cluster, role, environ, instance, topology_name) + pid = pid_response["stdout"].strip() + + base_url = utils.make_shell_endpoint(state.tracker.pb2_to_api(topology), instance) + url = f"{base_url}/histo/{pid}" + with httpx.AsyncClient() as client: + return await client.get(url).json() diff --git a/heron/tools/tracker/src/python/routers/metrics.py b/heron/tools/tracker/src/python/routers/metrics.py new file mode 100644 index 00000000000..59abed6bc63 --- /dev/null +++ b/heron/tools/tracker/src/python/routers/metrics.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Views on Heron metrics. + +""" +from typing import Dict, List, Optional + +from heron.common.src.python.utils.log import Log +from heron.proto import common_pb2 +from heron.proto import tmanager_pb2 +from heron.tools.tracker.src.python import metricstimeline, state +from heron.tools.tracker.src.python.query import Query as TManagerQuery +from heron.tools.tracker.src.python.utils import EnvelopingAPIRouter, BadRequest + +import httpx + +from fastapi import Query +from pydantic import BaseModel, Field + +router = EnvelopingAPIRouter() + +class ComponentMetrics(BaseModel): + interval: int + component: str + metrics: Dict[str, Dict[str, str]] = Field( + ..., + description="a map of (metric, instance) to value" + ) + + +async def get_component_metrics( + tmanager, + component: str, + metric_names: List[str], + instances: List[str], + interval: int, +) -> ComponentMetrics: + """ + Return metrics from the Tmanager over the given interval. + + The metrics property is keyed with (metric, instance) to metric values. + + Metrics not included in `metric_names` will be truncated. + + """ + if not (tmanager and tmanager.host and tmanager.stats_port): + raise Exception("No Tmanager found") + + metric_request = tmanager_pb2.MetricRequest() + metric_request.component_name = component + if instances: + metric_request.instance_id.extend(instances) + metric_request.metric.extend(metric_names) + metric_request.interval = interval + url = f"http://{tmanager.host}:{tmanager.stats_port}/stats" + async with httpx.AsyncClient() as client: + response = await client.post(url, data=metric_request.SerializeToString()) + metric_response = tmanager_pb2.MetricResponse() + metric_response.ParseFromString(response.content) + + if metric_response.status.status == common_pb2.NOTOK: + if metric_response.status.HasField("message"): + Log.warn( + "Recieved response from Tmanager: %s", metric_response.status.message + ) + + metrics = {} + for metric in metric_response.metric: + instance = metric.instance_id + for instance_metric in metric.metric: + metrics.setdefault(instance_metric.name, {})[ + instance + ] = instance_metric.value + + return ComponentMetrics( + interval=metric_response.interval, + component=component, + metrics=metrics, + ) + + +@router.get("/metrics", response_model=ComponentMetrics) +async def get_metrics( # pylint: disable=too-many-arguments + cluster: str, + role: Optional[str], + environ: str, + component: str, + topology_name: str = Query(..., alias="topology"), + metric_names: Optional[List[str]] = Query(None, alias="metricname"), + instances: Optional[List[str]] = Query(None, alias="instance"), + interval: int = -1, +): + """ + Return metrics over the given interval. Metrics not included in `metric_names` + will be truncated. + + """ + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + return await get_component_metrics( + topology.tmanager, component, metric_names, instances, interval + ) + + +@router.get("/metricstimeline", response_model=metricstimeline.MetricsTimeline) +async def get_metrics_timeline( # pylint: disable=too-many-arguments + cluster: str, + role: Optional[str], + environ: str, + component: str, + start_time: int, + end_time: int, + topology_name: str = Query(..., alias="topology"), + metric_names: Optional[List[str]] = Query(None, alias="metricname"), + instances: Optional[List[str]] = Query(None, alias="instance"), +): + """Return metrics over the given interval.""" + if start_time > end_time: + raise BadRequest("start_time > end_time") + topology = state.tracker.get_toplogy(cluster, role, environ, topology_name) + return await metricstimeline.get_metrics_timeline( + topology.tmanager, component, metric_names, instances, start_time, end_time + ) + + +class TimelinePoint(BaseModel): # pylint: disable=too-few-public-methods + """A metric at discrete points in time.""" + instance: Optional[str] = Field( + None, + description="name of the instance the metrics applies to if not an aggregate", + ) + data: Dict[int, int] = Field(..., description="map of start times to metric values") + + +class MetricsQueryResponse(BaseModel): # pylint: disable=too-few-public-methods + """A metrics timeline over an interval.""" + start_time: int = Field(..., alias="starttime") + end_time: int = Field(..., alias="endtime") + timeline: List[TimelinePoint] = Field( + ..., description="list of timeline point objects", + ) + + +@router.get("/metricsquery", response_model=MetricsQueryResponse) +async def get_metrics_query( # pylint: disable=too-many-arguments + cluster: str, + role: Optional[str], + environ: str, + query: str, + start_time: int = Query(..., alias="starttime"), + end_time: int = Query(..., alias="endtime"), + topology_name: str = Query(..., alias="topology"), +) -> MetricsQueryResponse: + """Run a metrics query against a particular toplogy.""" + topology = state.tracker.get_topology(cluster, role, environ, topology_name) + metrics = await TManagerQuery(state.tracker).execute_query( + topology.tmanager, query, start_time, end_time + ) + + timeline = [ + TimelinePoint(data=metric.timeline, instance=metric.instance) + for metric in metrics + ] + + return MetricsQueryResponse( + startime=start_time, + endtime=end_time, + timeline=timeline, + ) diff --git a/heron/tools/tracker/src/python/routers/topologies.py b/heron/tools/tracker/src/python/routers/topologies.py new file mode 100644 index 00000000000..c96bbdb1e24 --- /dev/null +++ b/heron/tools/tracker/src/python/routers/topologies.py @@ -0,0 +1,179 @@ +#!/usr/bin/env python3 +# -*- encoding: utf-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +These methods provide information about topologies based on information +from the state manager. + +Some information may not be available for a topology due until the state +manager has recieved more information from the state manager. + +""" +from typing import List, Optional, Dict, Union + +from heron.tools.tracker.src.python import state +from heron.tools.tracker.src.python.topology import ( + TopologyInfo, + TopologyInfoExecutionState, + TopologyInfoLogicalPlan, + TopologyInfoMetadata, + TopologyInfoPhysicalPlan, + TopologyInfoSchedulerLocation, +) +from heron.tools.tracker.src.python.utils import EnvelopingAPIRouter + +from fastapi import Query + +router = EnvelopingAPIRouter() + + +@router.get("", response_model=Dict[str, Dict[str, Dict[str, List[str]]]]) +async def get_topologies( + role: Optional[str] = Query(None, deprecated=True), + cluster_names: List[str] = Query(None, alias="cluster"), + environ_names: List[str] = Query(None, alias="environ"), +): + """ + Return a map of (cluster, role, environ) to a list of topology names. + + """ + result = {} + for topology in state.tracker.filtered_topologies( + cluster_names, + environ_names, + (), + roles={role} if role else (), + ): + if topology.execution_state: + t_role = topology.execution_state.role + else: + t_role = None + result.setdefault(topology.cluster, {})\ + .setdefault(t_role, {})\ + .setdefault(topology.environ, [])\ + .append(topology.name) + return result + + +@router.get( + "/states", + response_model=Dict[str, Dict[str, Dict[str, TopologyInfoExecutionState]]], +) +async def get_topologies_state( + cluster_names: Optional[List[str]] = Query(None, alias="cluster"), + environ_names: Optional[List[str]] = Query(None, alias="environ"), + role: Optional[str] = Query(None, deprecated=True), +): + """Return the execution states for topologies. Keyed by (cluster, environ, topology).""" + result = {} + + for topology in state.tracker.filtered_topologies(cluster_names, environ_names, {}, {role}): + topology_info = topology.info + if topology_info is not None: + result.setdefault(topology.cluster, {}).setdefault(topology.environ, {})[ + topology.name + ] = topology_info.execution_state + return result + + +@router.get("/info", response_model=TopologyInfo) +async def get_topology_info( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + topology = state.tracker.get_topology(cluster, role, environ, topology) + return topology.info + + +@router.get("/config", response_model=Dict[str, Union[int, str]]) +async def get_topology_config( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + topology = state.tracker.get_topology(cluster, role, environ, topology) + topology_info = topology.info + return topology_info.physical_plan.config + + +@router.get("/physicalplan", response_model=TopologyInfoPhysicalPlan) +async def get_topology_physical_plan( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + topology = state.tracker.get_topology(cluster, role, environ, topology) + return topology.info.physical_plan + + +# Deprecated. See https://github.com/apache/incubator-heron/issues/1754 +@router.get("/executionstate", response_model=TopologyInfoExecutionState) +async def get_topology_execution_state( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + topology = state.tracker.get_topology(cluster, role, environ, topology) + return topology.info.execution_state + + +@router.get("/schedulerlocation", response_model=TopologyInfoSchedulerLocation) +async def get_topology_scheduler_location( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + topology = state.tracker.get_topology(cluster, role, environ, topology) + return topology.info.scheduler_location + + +@router.get("/metadata", response_model=TopologyInfoMetadata) +async def get_topology_metadata( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + topology = state.tracker.get_topology(cluster, role, environ, topology) + return topology.info.metadata + + +@router.get("/logicalplan", response_model=TopologyInfoLogicalPlan) +async def get_topology_logical_plan( + cluster: str, + environ: str, + topology: str, + role: Optional[str] = Query(None, deprecated=True), +): + """ + This returns a transformed version of the logical plan, it probably + shouldn't, especially with the renaming. The types should be fixed + upstream, and the number of topology stages could find somewhere else + to live. + + """ + topology = state.tracker.get_topology(cluster, role, environ, topology) + topology_info = topology.info + return topology_info.logical_plan diff --git a/heron/tools/tracker/src/python/state.py b/heron/tools/tracker/src/python/state.py new file mode 100644 index 00000000000..566fbc78d8e --- /dev/null +++ b/heron/tools/tracker/src/python/state.py @@ -0,0 +1,8 @@ +""" +This module holds global state which is initialised on entry. + +""" +from heron.tools.tracker.src.python.tracker import Tracker + +# this is populated on entry into the application +tracker: Tracker = None diff --git a/heron/tools/tracker/src/python/topology.py b/heron/tools/tracker/src/python/topology.py index 59f1743a33c..cd71d2830be 100644 --- a/heron/tools/tracker/src/python/topology.py +++ b/heron/tools/tracker/src/python/topology.py @@ -19,15 +19,205 @@ # under the License. ''' topology.py ''' -import uuid - -from typing import Callable, Dict, List, Optional +import dataclasses +import json +import string + +from copy import deepcopy +from typing import Any, Dict, List, Optional + +from heron.proto import topology_pb2 +from heron.proto.execution_state_pb2 import ExecutionState as ExecutionState_pb +from heron.proto.packing_plan_pb2 import PackingPlan as PackingPlan_pb +from heron.proto.physical_plan_pb2 import PhysicalPlan as PhysicalPlan_pb +from heron.proto.scheduler_pb2 import SchedulerLocation as SchedulerLocation_pb +from heron.proto.tmanager_pb2 import TManagerLocation as TManagerLocation_pb +from heron.tools.tracker.src.python.config import ( + Config, + EXTRA_LINK_FORMATTER_KEY, + EXTRA_LINK_URL_KEY, +) +from heron.tools.tracker.src.python import utils + +import networkx + +from pydantic import BaseModel, Field + + +class TopologyInfoMetadata(BaseModel): + cluster: str + environ: str + role: str + jobname: str + submission_time: int + submission_user: str + release_username: str + release_tag: str + release_version: str + extra_links: List[Dict[str, str]] + +class TopologyInfoExecutionState(TopologyInfoMetadata): + """ + This model is a superset of the "metadata". -from heronpy.api import api_constants -from heron.common.src.python.utils.log import Log -from heron.proto.packing_plan_pb2 import PackingPlan -from heron.proto.physical_plan_pb2 import PhysicalPlan -from heron.proto.execution_state_pb2 import ExecutionState + Note: this may be a symptom of a bad pattern, the presence of these + things could be determined by making their respective objects + optional rather than empty + """ + has_physical_plan: bool + has_packing_plan: bool + has_tmanager_location: bool + has_scheduler_location: bool + +class RuntimeStateStatemanager(BaseModel): + is_registered: bool + +class TopologyInfoRuntimeState(BaseModel): + has_physical_plan: bool + has_packing_plan: bool + has_tmanager_location: bool + has_scheduler_location: bool + stmgrs: Dict[str, RuntimeStateStatemanager] = Field( + ..., + deprecated=True, + description="this is only populated by the /topologies/runtimestate endpoint", + ) + + +class TopologyInfoSchedulerLocation(BaseModel): + name: Optional[str] + http_endpoint: Optional[str] + job_page_link: Optional[str] = Field(None, description="may be empty") + +class TopologyInfoTmanagerLocation(BaseModel): + name: Optional[str] + id: Optional[str] + host: Optional[str] + controller_port: Optional[int] + server_port: Optional[int] + stats_port: Optional[int] + +class PackingPlanRequired(BaseModel): + cpu: float + ram: int + disk: int + +class PackingPlanScheduled(BaseModel): + cpu: Optional[float] + ram: Optional[int] + disk: Optional[int] + +class PackingPlanInstance(BaseModel): + component_name: str + task_id: int + component_index: int + instance_resources: PackingPlanRequired + +class PackingPlanContainer(BaseModel): + id: int + instances: List[PackingPlanInstance] + required_resources: PackingPlanRequired + scheduled_resources: PackingPlanScheduled + +class TopologyInfoPackingPlan(BaseModel): + id: str + container_plans: List[PackingPlanContainer] + +class PhysicalPlanStmgr(BaseModel): + id: str + host: str + port: int + shell_port: int + cwd: str + pid: int + joburl: str + logfiles: str = Field(..., description="URL to retrieve logs") + instance_ids: List[str] + +class PhysicalPlanInstance(BaseModel): + id: str + name: str + stmgr_id: str + logfile: str = Field(..., description="URL to retrieve log") + +class PhysicalPlanComponent(BaseModel): + config: Dict[str, Any] + +class TopologyInfoPhysicalPlan(BaseModel): + instances: Dict[str, PhysicalPlanInstance] + # instance_id is in the form ___ + # the container is the "group" + instance_groups: Dict[str, List[str]] = Field( + ..., + description="map of instance group name to instance ids", + ) + stmgrs: Dict[str, PhysicalPlanStmgr] = Field(..., description="map of stmgr id to stmgr info") + spouts: Dict[str, List[str]] = Field(..., description="map of name to instance ids") + bolts: Dict[str, List[str]] = Field(..., description="map of name to instance ids") + config: Dict[str, Any] + components: Dict[str, PhysicalPlanComponent] = Field( + ..., + description="map of bolt/spout name to info", + ) + +class LogicalPlanStream(BaseModel): + name: str = Field(..., alias="stream_name") + +class LogicalPlanBoltInput(BaseModel): + stream_name: str + component_name: str + grouping: str + +class LogicalPlanBolt(BaseModel): + config: Dict[str, Any] + outputs: List[LogicalPlanStream] + inputs: List[LogicalPlanBoltInput] + input_components: List[str] = Field(..., alias="inputComponents", deprecated=True) + +class LogicalPlanSpout(BaseModel): + config: Dict[str, Any] + type: str = Field(..., alias="spout_type") + source: str = Field(..., alias="spout_source") + version: str + outputs: List[LogicalPlanStream] + extra_links: List[Dict[str, Any]] + +class TopologyInfoLogicalPlan(BaseModel): + bolts: Dict[str, LogicalPlanBolt] + spouts: Dict[str, LogicalPlanSpout] + stages: int = Field(..., description="number of components in longest path") + +class TopologyInfo(BaseModel): + execution_state: TopologyInfoExecutionState + id: Optional[str] + logical_plan: TopologyInfoLogicalPlan + metadata: TopologyInfoMetadata + name: str + packing_plan: TopologyInfoPackingPlan + physical_plan: TopologyInfoPhysicalPlan + runtime_state: TopologyInfoRuntimeState + scheduler_location: TopologyInfoSchedulerLocation + tmanager_location: TopologyInfoTmanagerLocation + + +def topology_stages(logical_plan: TopologyInfoLogicalPlan) -> int: + """Return the number of stages in a logical plan.""" + graph = networkx.DiGraph( + (input_info.component_name, bolt_name) + for bolt_name, bolt_info in logical_plan.bolts.items() + for input_info in bolt_info.inputs + ) + # this is is the same as "diameter" if treating the topology as an undirected graph + return networkx.dag_longest_path_length(graph) + +@dataclasses.dataclass(frozen=True) +class TopologyState: + """Collection of state accumulated for tracker from state manager.""" + tmanager: Optional[TManagerLocation_pb] + scheduler_location: Optional[SchedulerLocation_pb] + physical_plan: Optional[PhysicalPlan_pb] + packing_plan: Optional[PackingPlan_pb] + execution_state: Optional[ExecutionState_pb] # pylint: disable=too-many-instance-attributes class Topology: @@ -38,108 +228,411 @@ class Topology: All this info is fetched from state manager in one go. - The watches are the callbacks that are called - when there is any change in the topology - instance using set_physical_plan, set_execution_state, - set_tmanager, and set_scheduler_location. Any other means of changing will - not call the watches. - """ - def __init__(self, name: str, state_manager_name: str) -> None: + def __init__(self, name: str, state_manager_name: str, tracker_config: Config) -> None: self.name = name self.state_manager_name = state_manager_name - self.physical_plan: PhysicalPlan = None - self.packing_plan: PackingPlan = None - self.execution_state: Optional[str] = None + self.physical_plan: Optional[PhysicalPlan_pb] = None + self.packing_plan: Optional[PackingPlan_pb] = None + self.tmanager: Optional[TManagerLocation_pb] = None + self.scheduler_location: Optional[SchedulerLocation_pb] = None + self.execution_state: Optional[ExecutionState_pb] = None self.id: Optional[int] = None - self.tmanager = None - self.scheduler_location = None - self.watches: Dict[uuid.UUID, Callable[[Topology], None]] = {} - - def register_watch(self, callback: Callable[["Topology"], None]) -> Optional[uuid.UUID]: - """ - Returns the UUID with which the watch is - registered. This UUID can be used to unregister - the watch. - Returns None if watch could not be registered. - - The argument 'callback' must be a function that takes - exactly one argument, the topology on which - the watch was triggered. - Note that the watch will be unregistered in case - it raises any Exception the first time. - - This callback is also called at the time of registration. - - """ - # maybe this should use a counter - # Generate a random UUID. - uid = uuid.uuid4() - if uid in self.watches: - raise ValueError("Time to buy a lottery ticket") - Log.info(f"Registering a watch with uid: {uid}") - try: - callback(self) - except Exception as e: - Log.error(f"Caught exception while triggering callback: {e}") - Log.debug("source of error:", exc_info=True) + self.tracker_config: Config = tracker_config + # this maps pb2 structs to structures returned via API endpoints + # it is repopulated every time one of the pb2 roperties is updated + self.info: Optional[TopologyInfo] = None + + @staticmethod + def _render_extra_links(extra_links, topology, execution_state: ExecutionState_pb) -> None: + """Render links in place.""" + subs = { + "cluster": execution_state.cluster, + "environ": execution_state.environ, + "role": execution_state.role, + "jobname": topology.name, + "submission_user": execution_state.submission_user, + } + for link in extra_links: + link[EXTRA_LINK_URL_KEY] = string.Template(link[EXTRA_LINK_FORMATTER_KEY]).substitute(subs) + + def _rebuild_info(self, t_state: TopologyState) -> Optional[TopologyInfo]: + # Execution state is the most basic info. If returnecution state, just return + # as the rest of the things don't matter. + execution_state = t_state.execution_state + if not execution_state: return None - self.watches[uid] = callback - return uid - - def unregister_watch(self, uid) -> None: - """ - Unregister the watch with the given UUID. - """ - # Do not raise an error if UUID is not present in the watches - Log.info(f"Unregister a watch with uid: {uid}") - self.watches.pop(uid, None) - - def trigger_watches(self) -> None: - """ - Call all the callbacks. + # take references to instances to reduce inconsistency risk, which would + # be a problem if the topology is updated in the middle of a call to this + + topology = self + packing_plan = t_state.packing_plan + physical_plan = t_state.physical_plan + tmanager = t_state.tmanager + scheduler_location = t_state.scheduler_location + tracker_config = self.tracker_config # assuming this is never updated + return TopologyInfo( + id=topology.id, + logical_plan=self._build_logical_plan(topology, execution_state, physical_plan), + metadata=self._build_metadata(topology, execution_state, tracker_config), + name=topology.name, # was self.name + packing_plan=self._build_packing_plan(packing_plan), + physical_plan=self._build_physical_plan(physical_plan), + runtime_state=self._build_runtime_state( + physical_plan=physical_plan, + packing_plan=packing_plan, + tmanager=tmanager, + scheduler_location=scheduler_location, + execution_state=execution_state, + ), + execution_state=self._build_execution_state( + topology=topology, + execution_state=execution_state, + physical_plan=physical_plan, + packing_plan=packing_plan, + tmanager=tmanager, + scheduler_location=scheduler_location, + tracker_config=tracker_config, + ), + scheduler_location=self._build_scheduler_location(scheduler_location), + tmanager_location=self._build_tmanager_location(tmanager), + ) + + @staticmethod + def _build_execution_state( + topology, + execution_state, + physical_plan, + packing_plan, + tmanager, + scheduler_location, + tracker_config, + ) -> TopologyInfoExecutionState: + status = { + topology_pb2.RUNNING: "Running", + topology_pb2.PAUSED: "Paused", + topology_pb2.KILLED: "Killed", + }.get(physical_plan.topology.state if physical_plan else None, "Unknown") + metadata = Topology._build_metadata(topology, execution_state, tracker_config) + return TopologyInfoExecutionState( + has_physical_plan=bool(physical_plan), + has_packing_plan=bool(packing_plan), + has_tmanager_location=bool(tmanager), + has_scheduler_location=bool(scheduler_location), + status=status, + **metadata.dict(), + ) + + @staticmethod + def _build_logical_plan( + topology: "Topology", + execution_state: ExecutionState_pb, + physical_plan: Optional[PhysicalPlan_pb], + ) -> TopologyInfoLogicalPlan: + if not physical_plan: + return TopologyInfoLogicalPlan(spouts={}, bolts={}, stages=0) + spouts = {} + for spout in physical_plan.topology.spouts: + config = utils.convert_pb_kvs(spout.comp.config.kvs, include_non_primitives=False) + extra_links = json.loads(config.get("extra.links", "[]")) + Topology._render_extra_links(extra_links, topology, execution_state) + spouts[spout.comp.name] = LogicalPlanSpout( + config=config, + spout_type=config.get("spout.type", "default"), + spout_source=config.get("spout.source", "NA"), + version=config.get("spout.version", "NA"), + extra_links=extra_links, + outputs=[ + LogicalPlanStream(stream_name=output.stream.id) + for output in spout.outputs + ], + ) + + info = TopologyInfoLogicalPlan( + stages=0, + spouts=spouts, + bolts={ + bolt.comp.name: LogicalPlanBolt( + config=utils.convert_pb_kvs(bolt.comp.config.kvs, include_non_primitives=False), + outputs=[ + LogicalPlanStream(stream_name=output.stream.id) + for output in bolt.outputs + ], + inputs=[ + LogicalPlanBoltInput( + stream_name=input_.stream.id, + component_name=input_.stream.component_name, + grouping=topology_pb2.Grouping.Name(input_.gtype), + ) + for input_ in bolt.inputs + ], + inputComponents=[ + input_.stream.component_name + for input_ in bolt.inputs + ], + ) + for bolt in physical_plan.topology.bolts + }, + ) + info.stages = topology_stages(info) + return info + + @staticmethod + def _build_metadata(topology, execution_state, tracker_config) -> TopologyInfoMetadata: + if not execution_state: + return TopologyInfoMetadata() + metadata = { + "cluster": execution_state.cluster, + "environ": execution_state.environ, + "role": execution_state.role, + "jobname": topology.name, + "submission_time": execution_state.submission_time, + "submission_user": execution_state.submission_user, + "release_username": execution_state.release_state.release_username, + "release_tag": execution_state.release_state.release_tag, + "release_version": execution_state.release_state.release_version, + } + extra_links = deepcopy(tracker_config.extra_links) + Topology._render_extra_links(extra_links, topology, execution_state) + return TopologyInfoMetadata( + extra_links=extra_links, + **metadata, + ) + + @staticmethod + def _build_packing_plan(packing_plan) -> TopologyInfoPackingPlan: + if not packing_plan: + return TopologyInfoPackingPlan(id="", container_plans=[]) + return TopologyInfoPackingPlan( + id=packing_plan.id, + container_plans=[ + PackingPlanContainer( + id=container.id, + instances=[ + PackingPlanInstance( + component_name=instance.component_name, + task_id=instance.task_id, + component_index=instance.component_index, + instance_resources=PackingPlanRequired( + cpu=instance.resource.cpu, + ram=instance.resource.ram, + disk=instance.resource.disk, + ), + ) + for instance in container.instance_plans + ], + required_resources=PackingPlanRequired( + cpu=container.requiredResource.cpu, + ram=container.requiredResource.ram, + disk=container.requiredResource.disk, + ), + scheduled_resources=( + PackingPlanScheduled( + cpu=container.scheduledResource.cpu, + ram=container.scheduledResource.ram, + disk=container.scheduledResource.ram, + ) + if container.scheduledResource else + PackingPlanScheduled() + ), + ) + for container in packing_plan.container_plans + ], + ) + + @staticmethod + def _build_physical_plan(physical_plan) -> TopologyInfoPhysicalPlan: + if not physical_plan: + return TopologyInfoPhysicalPlan( + instances={}, + instance_groups={}, + stmgrs={}, + spouts={}, + bolts={}, + config={}, + components={}, + ) + config = {} + if physical_plan.topology.topology_config: + config = utils.convert_pb_kvs(physical_plan.topology.topology_config.kvs) + + components = {} + spouts = {} + bolts = {} + for spout in physical_plan.topology.spouts: + name = spout.comp.name + spouts[name] = [] + if name not in components: + components[name] = PhysicalPlanComponent( + config=utils.convert_pb_kvs(spout.comp.config.kvs), + ) + for bolt in physical_plan.topology.bolts: + name = bolt.comp.name + bolts[name] = [] + if name not in components: + components[name] = PhysicalPlanComponent( + config=utils.convert_pb_kvs(bolt.comp.config.kvs), + ) + + stmgrs = {} + for stmgr in physical_plan.stmgrs: + shell_port = stmgr.shell_port if stmgr.HasField("shell_port") else None + stmgrs[stmgr.id] = PhysicalPlanStmgr( + id=stmgr.id, + host=stmgr.host_name, + port=stmgr.data_port, + shell_port=shell_port, + cwd=stmgr.cwd, + pid=stmgr.pid, + joburl=utils.make_shell_job_url(stmgr.host_name, shell_port, stmgr.cwd), + logfiles=utils.make_shell_logfiles_url(stmgr.host_name, stmgr.shell_port, stmgr.cwd), + instance_ids=[], + ) + + instances = {} + instance_groups = {} + for instance in physical_plan.instances: + component_name = instance.info.component_name + instance_id = instance.instance_id + if component_name in spouts: + spouts[component_name].append(instance_id) + else: + bolts[component_name].append(instance_id) + + stmgr = stmgrs[instance.stmgr_id] + stmgr.instance_ids.append(instance_id) + instances[instance_id] = PhysicalPlanInstance( + id=instance_id, + name=component_name, + stmgr_id=instance.stmgr_id, + logfile=utils.make_shell_logfiles_url( + stmgr.host, + stmgr.shell_port, + stmgr.cwd, + instance_id, + ), + ) + + # instance_id example: container_1_component_1 + # group name would be: container_1 + group_name = instance_id.rsplit("_", 2)[0] + instance_groups.setdefault(group_name, []).append(instance_id) + + return TopologyInfoPhysicalPlan( + instances=instances, + instance_groups=instance_groups, + stmgrs=stmgrs, + spouts=spouts, + bolts=bolts, + components=components, + config=config, + ) + + @staticmethod + def _build_runtime_state( + physical_plan, + packing_plan, + tmanager, + scheduler_location, + execution_state, + ) -> TopologyInfoRuntimeState: + return TopologyInfoRuntimeState( + has_physical_plan=bool(physical_plan), + has_packing_plan=bool(packing_plan), + has_tmanager_location=bool(tmanager), + has_scheduler_location=bool(scheduler_location), + release_version=execution_state.release_state.release_version, + stmgrs={}, + ) + + @staticmethod + def _build_scheduler_location(scheduler_location) -> TopologyInfoSchedulerLocation: + if not scheduler_location: + return TopologyInfoSchedulerLocation(name=None, http_endpoint=None, job_page_link=None) + return TopologyInfoSchedulerLocation( + name=scheduler_location.topology_name, + http_endpoint=scheduler_location.http_endpoint, + job_page_link=( + scheduler_location.job_page_link[0] + if scheduler_location.job_page_link + else "" + ), + ) + + @staticmethod + def _build_tmanager_location(tmanager) -> TopologyInfoTmanagerLocation: + if not tmanager: + return TopologyInfoTmanagerLocation( + name=None, + id=None, + host=None, + controller_port=None, + server_port=None, + stats_port=None, + ) + return TopologyInfoTmanagerLocation( + name=tmanager.topology_name, + id=tmanager.topology_id, + host=tmanager.host, + controller_port=tmanager.controller_port, + server_port=tmanager.server_port, + status_port=tmanager.stats_port, + ) + + def _update( + self, + physical_plan=..., + packing_plan=..., + execution_state=..., + tmanager=..., + scheduler_location=..., + ) -> None: + """Atomically update this instance to avoid inconsistent reads/writes from other threads.""" + t_state = TopologyState( + physical_plan=self.physical_plan if physical_plan is ... else physical_plan, + packing_plan=self.packing_plan if packing_plan is ... else packing_plan, + execution_state=self.execution_state if execution_state is ... else execution_state, + tmanager=self.tmanager if tmanager is ... else tmanager, + scheduler_location=( + self.scheduler_location + if scheduler_location is ... else + scheduler_location + ), + ) + if t_state.physical_plan: + id_ = t_state.physical_plan.topology.id + elif t_state.packing_plan: + id_ = t_state.packing_plan.id + else: + id_ = None - If any callback raises an Exception, unregister the corresponding watch. + info = self._rebuild_info(t_state) + update = dataclasses.asdict(t_state) + update["info"] = info + update["id"] = id_ + # atomic update using python GIL + self.__dict__.update(update) - """ - to_remove = [] - # the list() is in case the callbacks modify the watches - for uid, callback in list(self.watches.items()): - try: - callback(self) - except Exception as e: - Log.error(f"Caught exception while triggering callback: {e}") - Log.debug("source of error:", exc_info=True) - to_remove.append(uid) - - for uid in to_remove: - self.unregister_watch(uid) - - def set_physical_plan(self, physical_plan: PhysicalPlan) -> None: + def set_physical_plan(self, physical_plan: PhysicalPlan_pb) -> None: """ set physical plan """ - if not physical_plan: - self.physical_plan = None - self.id = None - else: - self.physical_plan = physical_plan - self.id = physical_plan.topology.id - self.trigger_watches() + self._update(physical_plan=physical_plan) - def set_packing_plan(self, packing_plan: PackingPlan) -> None: + def set_packing_plan(self, packing_plan: PackingPlan_pb) -> None: """ set packing plan """ - if not packing_plan: - self.packing_plan = None - self.id = None - else: - self.packing_plan = packing_plan - self.id = packing_plan.id - self.trigger_watches() + self._update(packing_plan=packing_plan) + + def set_execution_state(self, execution_state: ExecutionState_pb) -> None: + """ set exectuion state """ + self._update(execution_state=execution_state) - def set_execution_state(self, execution_state: ExecutionState) -> None: + def set_tmanager(self, tmanager: TManagerLocation_pb) -> None: """ set exectuion state """ - self.execution_state = execution_state - self.trigger_watches() + self._update(tmanager=tmanager) + + def set_scheduler_location(self, scheduler_location: SchedulerLocation_pb) -> None: + """ set exectuion state """ + self._update(scheduler_location=scheduler_location) @property def cluster(self) -> Optional[str]: @@ -153,40 +646,12 @@ def environ(self) -> Optional[str]: return self.execution_state.environ return None - def set_tmanager(self, tmanager) -> None: - """ set exectuion state """ - self.tmanager = tmanager - self.trigger_watches() - - def set_scheduler_location(self, scheduler_location) -> None: - """ set exectuion state """ - self.scheduler_location = scheduler_location - self.trigger_watches() - - def num_instances(self) -> int: - """ - Number of spouts + bolts - """ - num = 0 - - # Get all the components - components = self.spouts() + self.bolts() - - # Get instances for each worker - for component in components: - config = component.comp.config - for kvs in config.kvs: - if kvs.key == api_constants.TOPOLOGY_COMPONENT_PARALLELISM: - num += int(kvs.value) - break - - return num - def spouts(self): """ Returns a list of Spout (proto) messages """ - if self.physical_plan: + physical_plan = self.physical_plan + if physical_plan: return list(self.physical_plan.topology.spouts) return [] @@ -200,7 +665,8 @@ def bolts(self): """ Returns a list of Bolt (proto) messages """ - if self.physical_plan: + physical_plan = self.physical_plan + if physical_plan: return list(self.physical_plan.topology.bolts) return [] @@ -215,23 +681,7 @@ def get_machines(self) -> List[str]: Get all the machines that this topology is running on. These are the hosts of all the stmgrs. """ - if self.physical_plan: - return [s.host_name for s in self.physical_plan.stmgrs] + physical_plan = self.physical_plan + if physical_plan: + return [s.host_name for s in physical_plan.stmgrs] return [] - - def get_status(self) -> str: - """ - Get the current state of this topology. - The state values are from the topology.proto - RUNNING = 1, PAUSED = 2, KILLED = 3 - if the state is None "Unknown" is returned. - """ - status = None - if self.physical_plan and self.physical_plan.topology: - status = self.physical_plan.topology.state - - return { - 1: "Running", - 2: "Paused", - 3: "Killed", - }.get(status, "Unknown") diff --git a/heron/tools/tracker/src/python/tracker.py b/heron/tools/tracker/src/python/tracker.py index 4a3f36f8ea9..5fb2fce4167 100644 --- a/heron/tools/tracker/src/python/tracker.py +++ b/heron/tools/tracker/src/python/tracker.py @@ -19,74 +19,16 @@ # under the License. ''' tracker.py ''' -import json import sys -import collections from functools import partial -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Container, List, Optional from heron.common.src.python.utils.log import Log -from heron.proto import topology_pb2 from heron.statemgrs.src.python import statemanagerfactory -from heron.tools.tracker.src.python.config import EXTRA_LINK_FORMATTER_KEY, EXTRA_LINK_URL_KEY +from heron.statemgrs.src.python.statemanager import StateManager +from heron.tools.tracker.src.python.config import Config from heron.tools.tracker.src.python.topology import Topology -from heron.tools.tracker.src.python import utils - -import javaobj.v1 as javaobj - -def convert_pb_kvs(kvs, include_non_primitives=True) -> dict: - """ - converts pb kvs to dict - """ - config = {} - for kv in kvs: - if kv.value: - config[kv.key] = kv.value - elif kv.serialized_value: - # add serialized_value support for python values (fixme) - - # is this a serialized java object - if topology_pb2.JAVA_SERIALIZED_VALUE == kv.type: - jv = _convert_java_value(kv, include_non_primitives=include_non_primitives) - if jv is not None: - config[kv.key] = jv - else: - config[kv.key] = _raw_value(kv) - return config - - -def _convert_java_value(kv, include_non_primitives=True): - try: - pobj = javaobj.loads(kv.serialized_value) - if isinstance(pobj, str): - return pobj - - if isinstance(pobj, javaobj.transformers.DefaultObjectTransformer.JavaPrimitiveClass): - return pobj.value - - if include_non_primitives: - # java objects that are not strings return value and encoded value - # Hexadecimal byte array for Serialized objects that - return { - 'value' : json.dumps(pobj, - default=lambda custom_field: custom_field.__dict__, - sort_keys=True, - indent=2), - 'raw' : kv.serialized_value.hex()} - - return None - except Exception: - Log.exception("Failed to parse data as java object") - if include_non_primitives: - return _raw_value(kv) - return None - -def _raw_value(kv): - return { - # The value should be a valid json object - 'value' : '{}', - 'raw' : kv.serialized_value.hex()} class Tracker: @@ -100,21 +42,14 @@ class Tracker: by handlers. """ - def __init__(self, config): + __slots__ = ["topologies", "config", "state_managers"] + + def __init__(self, config: Config): self.config = config - self.topologies = [] + self.topologies: List[Topology] = [] self.state_managers = [] - # A map from a tuple of form - # (topology_name, state_manager_name) to its - # info, which is its representation - # exposed through the APIs. - # The state_manager_name help when we - # want to remove the topology, - # since other info can not be relied upon. - self.topology_infos: Dict[Tuple[str, str], Any] = {} - - def synch_topologies(self) -> None: + def sync_topologies(self) -> None: """ Sync the topologies with the statemgrs. """ @@ -126,27 +61,24 @@ def synch_topologies(self) -> None: Log.critical(f"Found exception while initializing state managers: {e}", exc_info=True) sys.exit(1) - def on_topologies_watch(state_manager, topologies) -> None: + def on_topologies_watch(state_manager: StateManager, topologies: List[str]) -> None: """watch topologies""" + topologies = set(topologies) Log.info("State watch triggered for topologies.") - Log.debug("Topologies: " + str(topologies)) - cached_names = [t.name for t in self.get_stmgr_topologies(state_manager.name)] - Log.debug(f"Existing topologies: {cached_names}") - for name in cached_names: - if name not in topologies: - Log.info("Removing topology: %s in rootpath: %s", - name, state_manager.rootpath) - self.remove_topology(name, state_manager.name) + Log.debug("Topologies: %s", topologies) + cached_names = {t.name for t in self.get_stmgr_topologies(state_manager.name)} + Log.debug("Existing topologies: %s", cached_names) + for name in cached_names - topologies: + Log.info("Removing topology: %s in rootpath: %s", + name, state_manager.rootpath) + self.remove_topology(name, state_manager.name) - for name in topologies: - if name not in cached_names: - self.add_new_topology(state_manager, name) + for name in topologies - cached_names: + self.add_new_topology(state_manager, name) for state_manager in self.state_managers: - # The callback function with the bound - # state_manager as first variable. - onTopologiesWatch = partial(on_topologies_watch, state_manager) - state_manager.get_topologies(onTopologiesWatch) + # The callback function with the bound state_manager as first variable + state_manager.get_topologies(partial(on_topologies_watch, state_manager)) def stop_sync(self) -> None: for state_manager in self.state_managers: @@ -170,9 +102,9 @@ def get_topology( and t.environ == environ] if len(topologies) != 1: if role is not None: - raise Exception("Topology not found for {0}, {1}, {2}, {3}".format( + raise KeyError("Topology not found for {0}, {1}, {2}, {3}".format( cluster, role, environ, topology_name)) - raise Exception("Topology not found for {0}, {1}, {2}".format( + raise KeyError("Topology not found for {0}, {1}, {2}".format( cluster, environ, topology_name)) # There is only one topology which is returned. @@ -189,15 +121,13 @@ def add_new_topology(self, state_manager, topology_name: str) -> None: Adds a topology in the local cache, and sets a watch on any changes on the topology. """ - topology = Topology(topology_name, state_manager.name) + topology = Topology(topology_name, state_manager.name, self.config) Log.info("Adding new topology: %s, state_manager: %s", topology_name, state_manager.name) + # populate the cache before making it addressable in the topologies to + # avoid races due to concurrent execution self.topologies.append(topology) - # Register a watch on topology and change - # the topology_info on any new change. - topology.register_watch(self.set_topology_info) - # Set watches on the pplan, execution_state, tmanager and scheduler_location. state_manager.get_pplan(topology_name, topology.set_physical_plan) state_manager.get_packing_plan(topology_name, topology.set_packing_plan) @@ -209,434 +139,35 @@ def remove_topology(self, topology_name: str, state_manager_name: str) -> None: """ Removes the topology from the local cache. """ - topologies = [] - for top in self.topologies: - if (top.name == topology_name and - top.state_manager_name == state_manager_name): - # Remove topology_info - if (topology_name, state_manager_name) in self.topology_infos: - self.topology_infos.pop((topology_name, state_manager_name)) - else: - topologies.append(top) - - self.topologies = topologies - - def extract_execution_state(self, topology) -> dict: - """ - Returns the repesentation of execution state that will - be returned from Tracker. - - It looks like this has been replaced with extract_metadata and - extract_runtime_state. - - """ - result = self.extract_metadata(topology) - result.update({ - "has_physical_plan": None, - "has_tmanager_location": None, - "has_scheduler_location": None, - }) - return result - - def extract_metadata(self, topology) -> dict: - """ - Returns metadata that will be returned from Tracker. - """ - execution_state = topology.execution_state - metadata = { - "cluster": execution_state.cluster, - "environ": execution_state.environ, - "role": execution_state.role, - "jobname": topology.name, - "submission_time": execution_state.submission_time, - "submission_user": execution_state.submission_user, - "release_username": execution_state.release_state.release_username, - "release_tag": execution_state.release_state.release_tag, - "release_version": execution_state.release_state.release_version, - "extra_links": [], - } - - for extra_link in self.config.extra_links: - link = extra_link.copy() - link[EXTRA_LINK_URL_KEY] = self.config.get_formatted_url(link[EXTRA_LINK_FORMATTER_KEY], - metadata) - metadata["extra_links"].append(link) - return metadata - - @staticmethod - def extract_runtime_state(topology): - # "stmgrs" listed runtime state for each stream manager - # however it is possible that physical plan is not complete - # yet and we do not know how many stmgrs there are. That said, - # we should not set any key below (stream manager name) - return { - "has_physical_plan": bool(topology.physical_plan), - "has_packing_plan": bool(topology.packing_plan), - "has_tmanager_location": bool(topology.tmanager), - "has_scheduler_location": bool(topology.scheduler_location), - "stmgrs": {}, - } - - # pylint: disable=no-self-use - def extract_scheduler_location(self, topology) -> dict: - """ - Returns the representation of scheduler location that will - be returned from Tracker. - """ - schedulerLocation = { - "name": None, - "http_endpoint": None, - "job_page_link": None, - } - - if topology.scheduler_location: - schedulerLocation["name"] = topology.scheduler_location.topology_name - schedulerLocation["http_endpoint"] = topology.scheduler_location.http_endpoint - schedulerLocation["job_page_link"] = \ - topology.scheduler_location.job_page_link[0] \ - if topology.scheduler_location.job_page_link else "" - - return schedulerLocation - - def extract_tmanager(self, topology) -> dict: - """ - Returns the representation of tmanager that will - be returned from Tracker. - """ - tmanagerLocation = { - "name": None, - "id": None, - "host": None, - "controller_port": None, - "server_port": None, - "stats_port": None, - } - if topology.tmanager: - tmanagerLocation["name"] = topology.tmanager.topology_name - tmanagerLocation["id"] = topology.tmanager.topology_id - tmanagerLocation["host"] = topology.tmanager.host - tmanagerLocation["controller_port"] = topology.tmanager.controller_port - tmanagerLocation["server_port"] = topology.tmanager.server_port - tmanagerLocation["stats_port"] = topology.tmanager.stats_port - - return tmanagerLocation - - # pylint: disable=too-many-locals - def extract_logical_plan(self, topology): - """ - Returns the representation of logical plan that will - be returned from Tracker. - """ - logicalPlan = { - "spouts": {}, - "bolts": {}, - } - - # Add spouts. - for spout in topology.spouts(): - spoutType = "default" - spoutSource = "NA" - spoutVersion = "NA" - spoutConfigs = spout.comp.config.kvs - spoutExtraLinks = [] - for kvs in spoutConfigs: - if kvs.key == "spout.type": - spoutType = javaobj.loads(kvs.serialized_value) - elif kvs.key == "spout.source": - spoutSource = javaobj.loads(kvs.serialized_value) - elif kvs.key == "spout.version": - spoutVersion = javaobj.loads(kvs.serialized_value) - elif kvs.key == "extra.links": - spoutExtraLinks = json.loads(javaobj.loads(kvs.serialized_value)) - - spoutPlan = { - "config": convert_pb_kvs(spoutConfigs, include_non_primitives=False), - "type": spoutType, - "source": spoutSource, - "version": spoutVersion, - "outputs": [ - {"stream_name": outputStream.stream.id} - for outputStream in spout.outputs - ], - "extra_links": spoutExtraLinks, - } - logicalPlan["spouts"][spout.comp.name] = spoutPlan - - # render component extra links with general params - execution_state = { - "cluster": topology.execution_state.cluster, - "environ": topology.execution_state.environ, - "role": topology.execution_state.role, - "jobname": topology.name, - "submission_user": topology.execution_state.submission_user, - } - - for link in spoutPlan["extra_links"]: - link[EXTRA_LINK_URL_KEY] = self.config.get_formatted_url(link[EXTRA_LINK_FORMATTER_KEY], - execution_state) - - # Add bolts. - for bolt in topology.bolts(): - boltName = bolt.comp.name - logicalPlan["bolts"][boltName] = { - "config": convert_pb_kvs(bolt.comp.config.kvs, include_non_primitives=False), - "outputs": [ - {"stream_name": outputStream.stream.id} - for outputStream in bolt.outputs - ], - "inputs": [ - { - "stream_name": inputStream.stream.id, - "component_name": inputStream.stream.component_name, - "grouping": topology_pb2.Grouping.Name(inputStream.gtype), - } - for inputStream in bolt.inputs - ] - } - - - return logicalPlan - - # pylint: disable=too-many-locals - def extract_physical_plan(self, topology): - """ - Returns the representation of physical plan that will - be returned from Tracker. - """ - physicalPlan = { - "instances": {}, - "instance_groups": {}, - "stmgrs": {}, - "spouts": {}, - "bolts": {}, - "config": {}, - "components": {} - } - - if not topology.physical_plan: - return physicalPlan - - spouts = topology.spouts() - bolts = topology.bolts() - stmgrs = None - instances = None - - # Physical Plan - stmgrs = list(topology.physical_plan.stmgrs) - instances = list(topology.physical_plan.instances) - - # Configs - if topology.physical_plan.topology.topology_config: - physicalPlan["config"] = convert_pb_kvs(topology.physical_plan.topology.topology_config.kvs) - - for spout in spouts: - spout_name = spout.comp.name - physicalPlan["spouts"][spout_name] = [] - if spout_name not in physicalPlan["components"]: - physicalPlan["components"][spout_name] = { - "config": convert_pb_kvs(spout.comp.config.kvs) - } - for bolt in bolts: - bolt_name = bolt.comp.name - physicalPlan["bolts"][bolt_name] = [] - if bolt_name not in physicalPlan["components"]: - physicalPlan["components"][bolt_name] = { - "config": convert_pb_kvs(bolt.comp.config.kvs) - } - - for stmgr in stmgrs: - host = stmgr.host_name - cwd = stmgr.cwd - shell_port = stmgr.shell_port if stmgr.HasField("shell_port") else None - physicalPlan["stmgrs"][stmgr.id] = { - "id": stmgr.id, - "host": host, - "port": stmgr.data_port, - "shell_port": shell_port, - "cwd": cwd, - "pid": stmgr.pid, - "joburl": utils.make_shell_job_url(host, shell_port, cwd), - "logfiles": utils.make_shell_logfiles_url(host, shell_port, cwd), - "instance_ids": [] - } - - instance_groups = collections.OrderedDict() - for instance in instances: - instance_id = instance.instance_id - stmgrId = instance.stmgr_id - name = instance.info.component_name - stmgrInfo = physicalPlan["stmgrs"][stmgrId] - host = stmgrInfo["host"] - cwd = stmgrInfo["cwd"] - shell_port = stmgrInfo["shell_port"] - - - # instance_id format container__component_1 - # group name is container_ - group_name = instance_id.rsplit("_", 2)[0] - igroup = instance_groups.get(group_name, list()) - igroup.append(instance_id) - instance_groups[group_name] = igroup - - physicalPlan["instances"][instance_id] = { - "id": instance_id, - "name": name, - "stmgrId": stmgrId, - "logfile": utils.make_shell_logfiles_url(host, shell_port, cwd, instance.instance_id), - } - physicalPlan["stmgrs"][stmgrId]["instance_ids"].append(instance_id) - if name in physicalPlan["spouts"]: - physicalPlan["spouts"][name].append(instance_id) - else: - physicalPlan["bolts"][name].append(instance_id) - - physicalPlan["instance_groups"] = instance_groups - - return physicalPlan - - # pylint: disable=too-many-locals - def extract_packing_plan(self, topology): - """ - Returns the representation of packing plan that will be returned from Tracker. - - """ - packingPlan = { - "id": "", - "container_plans": [] - } - - if not topology.packing_plan: - return packingPlan - - packingPlan["id"] = topology.packing_plan.id - packingPlan["container_plans"] = [ - { - "id": container_plan.id, - "instances": [ - { - "component_name" : instance_plan.component_name, - "task_id" : instance_plan.task_id, - "component_index": instance_plan.component_index, - "instance_resources": { - "cpu": instance_plan.resource.cpu, - "ram": instance_plan.resource.ram, - "disk": instance_plan.resource.disk, - }, - } - for instance_plan in container_plan.instance_plans - ], - "required_resources": { - "cpu": container_plan.requiredResource.cpu, - "ram": container_plan.requiredResource.ram, - "disk": container_plan.requiredResource.disk, - }, - "scheduled_resources": ( - {} - if not container_plan else - { - "cpu": container_plan.scheduledResource.cpu, - "ram": container_plan.scheduledResource.ram, - "disk": container_plan.scheduledResource.disk, - } - ), - } - for container_plan in topology.packing_plan.container_plans + self.topologies = [ + topology + for topology in self.topologies + if not ( + topology.name == topology_name + and topology.state_manager_name == state_manager_name + ) ] - return packingPlan - - def set_topology_info(self, topology) -> Optional[dict]: - """ - Extracts info from the stored proto states and - convert it into representation that is exposed using - the API. - This method is called on any change for the topology. - For example, when a container moves and its host or some - port changes. All the information is parsed all over - again and cache is updated. - """ - # Execution state is the most basic info. - # If there is no execution state, just return - # as the rest of the things don't matter. - if not topology.execution_state: - Log.info("No execution state found for: " + topology.name) - return - - Log.info("Setting topology info for topology: " + topology.name) - has_physical_plan = True - if not topology.physical_plan: - has_physical_plan = False - - Log.info("Setting topology info for topology: " + topology.name) - has_packing_plan = True - if not topology.packing_plan: - has_packing_plan = False - - has_tmanager_location = True - if not topology.tmanager: - has_tmanager_location = False - - has_scheduler_location = True - if not topology.scheduler_location: - has_scheduler_location = False - - topology_info = { - "name": topology.name, - "id": topology.id, - "logical_plan": None, - "physical_plan": None, - "packing_plan": None, - "execution_state": None, - "tmanager_location": None, - "scheduler_location": None, - } - - execution_state = self.extract_execution_state(topology) - execution_state["has_physical_plan"] = has_physical_plan - execution_state["has_packing_plan"] = has_packing_plan - execution_state["has_tmanager_location"] = has_tmanager_location - execution_state["has_scheduler_location"] = has_scheduler_location - execution_state["status"] = topology.get_status() - - topology_info["metadata"] = self.extract_metadata(topology) - topology_info["runtime_state"] = self.extract_runtime_state(topology) - - topology_info["execution_state"] = execution_state - topology_info["logical_plan"] = self.extract_logical_plan(topology) - topology_info["physical_plan"] = self.extract_physical_plan(topology) - topology_info["packing_plan"] = self.extract_packing_plan(topology) - topology_info["tmanager_location"] = self.extract_tmanager(topology) - topology_info["scheduler_location"] = self.extract_scheduler_location(topology) - - self.topology_infos[(topology.name, topology.state_manager_name)] = topology_info - - # topology_name should be at the end to follow the trend - def get_topology_info( + def filtered_topologies( self, - topology_name: str, - cluster: str, - role: Optional[str], - environ: str, - ) -> str: - """ - Returns the JSON representation of a topology - by its name, cluster, environ, and an optional role parameter. - Raises exception if no such topology is found. - - """ - # Iterate over the values to filter the desired topology. - for (tn, _), topology_info in self.topology_infos.items(): - execution_state = topology_info["execution_state"] - if (tn == topology_name and - cluster == execution_state["cluster"] and - environ == execution_state["environ"] and - (not role or execution_state.get("role") == role) - ): - return topology_info - - Log.info( - f"Count not find topology info for cluster={cluster!r}," - f" role={role!r}, environ={environ!r}, role={role!r}," - f" topology={topology_name!r}" - ) - raise Exception("No topology found") + clusters: Container[str], + environs: Container[str], + names: Container[str], + roles: Container[str], # should deprecate? + ) -> List[Topology]: + """ + Return a filtered copy of the topologies which have the given properties. + + If a filter is falsy (i.e. empty) then all topologies will match on that property. + + """ + return [ + topology + for topology in self.topologies[:] + if ( + (not clusters or topology.cluster in clusters) + and (not environs or topology.environ in environs) + and (not names or topology.name in names) + and (not roles or (topology.execution_state and topology.execution_state.role)) + ) + ] diff --git a/heron/tools/tracker/src/python/utils.py b/heron/tools/tracker/src/python/utils.py index 1d3808d2878..2db2b7644e0 100644 --- a/heron/tools/tracker/src/python/utils.py +++ b/heron/tools/tracker/src/python/utils.py @@ -23,21 +23,87 @@ Contains utility functions used by tracker. ''' +import json import os import sys import subprocess +from asyncio import iscoroutinefunction +from functools import wraps from pathlib import Path -from typing import Any, Optional +from typing import Any, Generic, Literal, Optional, TypeVar +from heron.common.src.python.utils.log import Log +from heron.tools.tracker.src.python import constants +from heron.proto import topology_pb2 + +import javaobj.v1 as javaobj import yaml +from fastapi import APIRouter, HTTPException +from pydantic import Field +from pydantic.generics import GenericModel + # directories for heron tools distribution BIN_DIR = "bin" CONF_DIR = "conf" LIB_DIR = "lib" +ResultType = TypeVar("ResultType") + + +class ResponseEnvelope(GenericModel, Generic[ResultType]): + execution_time: float = Field(0.0, alias="executiontime") + message: str + result: Optional[ResultType] = None + status: Literal[ + constants.RESPONSE_STATUS_FAILURE, constants.RESPONSE_STATUS_SUCCESS + ] + tracker_version: str = constants.API_VERSION + +class BadRequest(HTTPException): + """Raised when bad input is recieved.""" + def __init__(self, detail: str = None) -> None: + super().__init__(400, detail) + +class EnvelopingAPIRouter(APIRouter): + """Router which wraps response_models with ResponseEnvelope.""" + + def api_route(self, response_model=None, **kwargs): + """This provides the decorator used by router..""" + if not response_model: + return super().api_route(response_model=response_model, **kwargs) + + wrapped_response_model = ResponseEnvelope[response_model] + decorator = super().api_route(response_model=wrapped_response_model, **kwargs) + + @wraps(decorator) + def new_decorator(f): + if iscoroutinefunction(f): + @wraps(f) + async def envelope(*args, **kwargs): + result = await f(*args, **kwargs) + return wrapped_response_model( + result=result, + execution_time=0.0, + message="ok", + status="success", + ) + else: + @wraps(f) + def envelope(*args, **kwargs): + result = f(*args, **kwargs) + return wrapped_response_model( + result=result, + execution_time=0.0, + message="ok", + status="success", + ) + return decorator(envelope) + + return new_decorator + def make_shell_endpoint(topology_info: dict, instance_id: int) -> str: """ @@ -174,4 +240,59 @@ def parse_config_file(config_file: str) -> Optional[str]: # Read the configuration file with open(expanded_config_file_path, 'r') as f: - return yaml.load(f) + return yaml.safe_load(f) + +################################################################################ +# utils for parsing protobuf key-value pairs from the API +################################################################################ +def convert_pb_kvs(kvs, include_non_primitives=True) -> dict: + """ + converts pb kvs to dict + """ + config = {} + for kv in kvs: + if kv.value: + config[kv.key] = kv.value + elif kv.serialized_value: + # add serialized_value support for python values (fixme) + + # is this a serialized java object + if topology_pb2.JAVA_SERIALIZED_VALUE == kv.type: + jv = _convert_java_value(kv, include_non_primitives=include_non_primitives) + if jv is not None: + config[kv.key] = jv + else: + config[kv.key] = _raw_value(kv) + return config + +def _convert_java_value(kv, include_non_primitives=True): + try: + pobj = javaobj.loads(kv.serialized_value) + if isinstance(pobj, str): + return pobj + + if isinstance(pobj, javaobj.transformers.DefaultObjectTransformer.JavaPrimitiveClass): + return pobj.value + + if include_non_primitives: + # java objects that are not strings return value and encoded value + # Hexadecimal byte array for Serialized objects that + return { + 'value' : json.dumps(pobj, + default=lambda custom_field: custom_field.__dict__, + sort_keys=True, + indent=2), + 'raw' : kv.serialized_value.hex()} + + return None + except Exception: + Log.exception("Failed to parse data as java object") + if include_non_primitives: + return _raw_value(kv) + return None + +def _raw_value(kv): + return { + # The value should be a valid json object + 'value' : '{}', + 'raw' : kv.serialized_value.hex()} diff --git a/heron/tools/tracker/tests/python/BUILD b/heron/tools/tracker/tests/python/BUILD index d411458a75f..0b42de9f185 100644 --- a/heron/tools/tracker/tests/python/BUILD +++ b/heron/tools/tracker/tests/python/BUILD @@ -17,7 +17,8 @@ pex_pytest( "topology_unittest.py", ], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", + "pytest-asyncio==0.14.0", ], deps = [ "//heron/proto:proto-py", @@ -31,7 +32,22 @@ pex_pytest( size = "small", srcs = ["query_operator_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", + "pytest-asyncio==0.14.0", + ], + deps = [ + "//heron/tools/tracker/src/python:tracker-py", + ], +) + +pex_pytest( + name = "app_unittest", + size = "small", + srcs = ["app_unittest.py"], + reqs = [ + "pytest==6.1.2", + "pytest-asyncio==0.14.0", + "requests==2.27.1", ], deps = [ "//heron/tools/tracker/src/python:tracker-py", @@ -43,7 +59,8 @@ pex_pytest( size = "small", srcs = ["query_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", + "pytest-asyncio==0.14.0", ], deps = [ "//heron/tools/tracker/src/python:tracker-py", @@ -58,7 +75,8 @@ pex_pytest( "tracker_unittest.py", ], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", + "pytest-asyncio==0.14.0", ], deps = [ "//heron/proto:proto-py", diff --git a/heron/tools/tracker/tests/python/app_unittest.py b/heron/tools/tracker/tests/python/app_unittest.py new file mode 100644 index 00000000000..9060250504e --- /dev/null +++ b/heron/tools/tracker/tests/python/app_unittest.py @@ -0,0 +1,52 @@ +from unittest.mock import MagicMock + +from heron.tools.tracker.src.python.app import app +from heron.tools.tracker.src.python.tracker import Tracker +from heron.tools.tracker.src.python import state, constants + +import pytest + +from fastapi.testclient import TestClient + +def ok(result) -> dict: + return { + "executiontime": 0.0, + "result": result, + "message": "ok", + "status": constants.RESPONSE_STATUS_SUCCESS, + "tracker_version": constants.API_VERSION, + } + +@pytest.fixture +def tracker(monkeypatch): + mock = MagicMock(Tracker) + monkeypatch.setattr(state, "tracker", mock) + return mock + +@pytest.fixture +def client(tracker): + return TestClient(app) + +def test_clusters(client, tracker): + c1, c2 = MagicMock(), MagicMock() + c1.configure_mock(name="c1") + c2.configure_mock(name="c2") + + tracker.state_managers = [c1, c2] + response = client.get("/clusters") + assert response.json() == ok(["c1", "c2"]) + assert response.status_code == 200 + +def test_machines(client): + response = client.get("/machines", json={ + "cluster": ["c1", "c3"], + "environ": ["e1", "e3"], + }) + assert response.json() == ok({}) + +def test_topologies(client): + response = client.get("/topologies", json={ + "cluster": [], + "environ": [], + }) + assert response.json() == ok({}) diff --git a/heron/tools/tracker/tests/python/query_operator_unittest.py b/heron/tools/tracker/tests/python/query_operator_unittest.py index 286dbe5cb9d..0396a29ca69 100644 --- a/heron/tools/tracker/tests/python/query_operator_unittest.py +++ b/heron/tools/tracker/tests/python/query_operator_unittest.py @@ -19,1685 +19,1639 @@ # over 500 bad indentation errors so disable # pylint: disable=bad-continuation # pylint: disable=unused-argument, unused-variable -import tornado.concurrent -import tornado.gen -import tornado.testing - from unittest.mock import patch, Mock from heron.tools.tracker.src.python.query_operators import * -class QueryOperatorTests(tornado.testing.AsyncTestCase): - @tornado.testing.gen_test - def test_TS_execute(self): - ts = TS(["a", "b", "c"]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def getMetricTimelineSideEffect(*args): - self.assertEqual((tmanager, "a", ["c"], ["b"], 40, 360), args) - raise tornado.gen.Return({ - "starttime": 40, - "endtime": 360, - "component": "a", - "timeline": { - "c": { - "b": { - 40: "1.0", - 100: "1.0", - 160: "1.0", - 220: "1.0", - 280: "1.0", - 340: "1.0" - } - } - } - }) - +import pytest + + +@pytest.mark.asyncio +async def test_TS_execute(): + ts = TS(["a", "b", "c"]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def getMetricTimelineSideEffect(*args): + assert (tmanager, "a", ["c"], ["b"], 40, 360) == args + return ({ + "starttime": 40, + "endtime": 360, + "component": "a", + "timeline": { + "c": { + "b": { + 40: "1.0", + 100: "1.0", + 160: "1.0", + 220: "1.0", + 280: "1.0", + 340: "1.0" + } + } + } + }) + + with patch("heron.tools.tracker.src.python.query_operators.get_metrics_timeline", + side_effect=getMetricTimelineSideEffect): + metrics = await ts.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "b" == metrics[0].instance + assert "c" == metrics[0].metric_name + assert "a" == metrics[0].component_name + assert { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_TS_execute_when_no_timeline(): + ts = TS(["a", "b", "c"]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # If no timeline is returned + def getMetricTimelineSideEffect(*args): + assert (tmanager, "a", ["c"], ["b"], 40, 360) == args + return ({ + "message": "some_exception" + }) + + # pylint: disable=unused-variable + with pytest.raises(Exception): with patch("heron.tools.tracker.src.python.query_operators.get_metrics_timeline", side_effect=getMetricTimelineSideEffect): - metrics = yield ts.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("b", metrics[0].instance) - self.assertEqual("c", metrics[0].metric_name) - self.assertEqual("a", metrics[0].component_name) - self.assertDictEqual({ - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_TS_execute_when_no_timeline(self): - ts = TS(["a", "b", "c"]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # If no timeline is returned - @tornado.gen.coroutine - def getMetricTimelineSideEffect(*args): - self.assertEqual((tmanager, "a", ["c"], ["b"], 40, 360), args) - raise tornado.gen.Return({ - "message": "some_exception" - }) - - # pylint: disable=unused-variable - with self.assertRaises(Exception): - with patch("heron.tools.tracker.src.python.query_operators.get_metrics_timeline", - side_effect=getMetricTimelineSideEffect): - metrics = yield ts.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_TS_execute_with_multiple_instances(self): - ts = TS(["a", "b", "c"]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # With multiple instances - @tornado.gen.coroutine - def getMetricTimelineSideEffect(*args): - self.assertEqual((tmanager, "a", ["c"], [], 40, 360), args) - raise tornado.gen.Return({ - "starttime": 40, - "endtime": 360, - "component": "a", - "timeline": { - "c": { - "b": { - 40: "1.0", - 100: "1.0", + metrics = await ts.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_TS_execute_with_multiple_instances(): + ts = TS(["a", "b", "c"]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # With multiple instances + def getMetricTimelineSideEffect(*args): + assert (tmanager, "a", ["c"], [], 40, 360) == args + return ({ + "starttime": 40, + "endtime": 360, + "component": "a", + "timeline": { + "c": { + "b": { + 40: "1.0", + 100: "1.0", # 160: "1.0", # This value is missing - 220: "1.0", - 280: "1.0", - 340: "1.0" - }, - "d": { - 40: "2.0", - 100: "2.0", - 160: "2.0", - 220: "2.0", - 280: "2.0", - 340: "2.0" - } - } - } - }) - - # pylint: disable=unused-variable - with patch("heron.tools.tracker.src.python.query_operators.get_metrics_timeline", - side_effect=getMetricTimelineSideEffect): - ts = TS(["a", "*", "c"]) - metrics = yield ts.execute(tracker, tmanager, start, end) - self.assertEqual(2, len(metrics)) - metric1 = metrics[0] - metric2 = metrics[1] - for metric in metrics: - if metric.instance == "b": - self.assertEqual("c", metric.metric_name) - self.assertEqual("a", metric.component_name) - self.assertDictEqual({ + 220: "1.0", + 280: "1.0", + 340: "1.0" + }, + "d": { + 40: "2.0", + 100: "2.0", + 160: "2.0", + 220: "2.0", + 280: "2.0", + 340: "2.0" + } + } + } + }) + + # pylint: disable=unused-variable + with patch("heron.tools.tracker.src.python.query_operators.get_metrics_timeline", + side_effect=getMetricTimelineSideEffect): + ts = TS(["a", "*", "c"]) + metrics = await ts.execute(tracker, tmanager, start, end) + assert 2 == len(metrics) + metric1 = metrics[0] + metric2 = metrics[1] + for metric in metrics: + if metric.instance == "b": + assert "c" == metric.metric_name + assert "a" == metric.component_name + assert { # 120: 1.0, # Missing value is not reported - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metric.timeline) - elif metric.instance == "d": - self.assertEqual("c", metric.metric_name) - self.assertEqual("a", metric.component_name) - self.assertDictEqual({ - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0 - }, metric.timeline) - else: - self.fail("Wrong metrics generated by TS.execute") - - @tornado.testing.gen_test - def test_DEFAULT_execute(self): - ts = Mock(TS) - default = Default([float(0), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, 180: 1.0, 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield default.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertEqual("metric_name", metrics[0].metric_name) - self.assertEqual("component", metrics[0].component_name) - self.assertDictEqual({ - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_DEFAULT_execute_when_exception(self): - ts = Mock(TS) - default = Default([float(0), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield default.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_DEFAULT_execute_when_missing_value(self): - ts = Mock(TS) - default = Default([float(0), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { + 300: 1.0 + } == metric.timeline + elif metric.instance == "d": + assert "c" == metric.metric_name + assert "a" == metric.component_name + assert { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0 + } == metric.timeline + else: + pytest.fail("Wrong metrics generated by TS.execute") + +@pytest.mark.asyncio +async def test_DEFAULT_execute(): + ts = Mock(TS) + default = Default([float(0), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, 180: 1.0, 240: 1.0, 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield default.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertEqual("metric_name", metrics[0].metric_name) - self.assertEqual("component", metrics[0].component_name) - self.assertDictEqual({ - 120: 0, # Missing value filled + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await default.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert "metric_name" == metrics[0].metric_name + assert "component" == metrics[0].component_name + assert { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_DEFAULT_execute_when_exception(): + ts = Mock(TS) + default = Default([float(0), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await default.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_DEFAULT_execute_when_missing_value(): + ts = Mock(TS) + default = Default([float(0), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await default.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert "metric_name" == metrics[0].metric_name + assert "component" == metrics[0].component_name + assert { + 120: 0, # Missing value filled 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_DEFAULT_execute_with_multiple_ts(self): - ts = Mock(TS) - default = Default([float(0), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines missing some values - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_DEFAULT_execute_with_multiple_ts(): + ts = Mock(TS) + default = Default([float(0), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines missing some values + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { # 120: 1.0, # Missing - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, # 180: 2.0, # Missing - 240: 2.0, + 240: 2.0, # 300: 2.0, # Missing - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield default.execute(tracker, tmanager, start, end) - self.assertEqual(2, len(metrics)) - for metric in metrics: - if metric.instance == "instance": - self.assertEqual("instance", metric.instance) - self.assertEqual("metric_name", metric.metric_name) - self.assertEqual("component", metric.component_name) - self.assertDictEqual({ - 120: 0, # Filled + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await default.execute(tracker, tmanager, start, end) + assert 2 == len(metrics) + for metric in metrics: + if metric.instance == "instance": + assert "instance" == metric.instance + assert "metric_name" == metric.metric_name + assert "component" == metric.component_name + assert { + 120: 0, # Filled 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metric.timeline) - elif metric.instance == "instance2": - self.assertEqual("instance2", metric.instance) - self.assertEqual("metric_name", metric.metric_name) - self.assertEqual("component", metric.component_name) - self.assertDictEqual({ - 120: 2.0, - 180: 0, # Filled + 240: 1.0, + 300: 1.0 + } == metric.timeline + elif metric.instance == "instance2": + assert "instance2" == metric.instance + assert "metric_name" == metric.metric_name + assert "component" == metric.component_name + assert { + 120: 2.0, + 180: 0, # Filled 240: 2.0, - 300: 0 # Filled - }, metric.timeline) - else: - self.fail("Wrong metrics generated by TS.execute") - - @tornado.testing.gen_test - def test_SUM_execute(self): - ts = Mock(TS) - operator = Sum([float(10), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 11.0, - 180: 11.0, - 240: 11.0, - 300: 11.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_SUM_execute_when_exception(self): - ts = Mock(TS) - operator = Sum([float(10), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_SUM_execute_when_missing_value(self): - ts = Mock(TS) - operator = Sum([float(10), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 180: 1.0, - 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 10, # Missing value filled + 300: 0 # Filled + } == metric.timeline + else: + pytest.fail("Wrong metrics generated by TS.execute") + +@pytest.mark.asyncio +async def test_SUM_execute(): + ts = Mock(TS) + operator = Sum([float(10), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 11.0, + 180: 11.0, + 240: 11.0, + 300: 11.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_SUM_execute_when_exception(): + ts = Mock(TS) + operator = Sum([float(10), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_SUM_execute_when_missing_value(): + ts = Mock(TS) + operator = Sum([float(10), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 10, # Missing value filled 180: 11.0, - 240: 11.0, - 300: 11.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_SUM_execute_with_multiple_ts(self): - ts = Mock(TS) - operator = Sum([float(10), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines missing some values - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { + 240: 11.0, + 300: 11.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_SUM_execute_with_multiple_ts(): + ts = Mock(TS) + operator = Sum([float(10), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines missing some values + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { # 120: 1.0, # Missing - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 12.0, - 180: 13.0, - 240: 13.0, - 300: 13.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_MAX_execute(self): - ts = Mock(TS) - operator = Max([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_MAX_execute_when_exception(self): - ts = Mock(TS) - operator = Max([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_MAX_execute_when_missing_values(self): - ts = Mock(TS) - operator = Max([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 180: 1.0, - 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_MAX_execute_with_multiple_ts(self): - ts = Mock(TS) - operator = Max([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines missing some values - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 12.0, + 180: 13.0, + 240: 13.0, + 300: 13.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_MAX_execute(): + ts = Mock(TS) + operator = Max([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_MAX_execute_when_exception(): + ts = Mock(TS) + operator = Max([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_MAX_execute_when_missing_values(): + ts = Mock(TS) + operator = Max([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 180: 1.0, + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_MAX_execute_with_multiple_ts(): + ts = Mock(TS) + operator = Max([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines missing some values + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { # 120: 1.0, # Missing - 180: 1.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 0.0, - 240: 2.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 2.0, - 180: 1.0, - 240: 3.0, - 300: 5.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_PERCENTILE_execute(self): - ts = Mock(TS) - operator = Percentile([float(90), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_PERCENTILE_execute_when_exception(self): - ts = Mock(TS) - operator = Percentile([float(90), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_PERCENTILE_execute_when_missing_values(self): - ts = Mock(TS) - operator = Percentile([float(90), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 180: 1.0, - 240: 1.0, - 300: 1.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 + 180: 1.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 0.0, + 240: 2.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 2.0, + 180: 1.0, + 240: 3.0, + 300: 5.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_PERCENTILE_execute(): + ts = Mock(TS) + operator = Percentile([float(90), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_PERCENTILE_execute_when_exception(): + ts = Mock(TS) + operator = Percentile([float(90), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_PERCENTILE_execute_when_missing_values(): + ts = Mock(TS) + operator = Percentile([float(90), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 1.0, + 240: 1.0, + 300: 1.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 180: 1.0, + 240: 1.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_PERCENTILE_execute_with_multiple_ts(): + ts = Mock(TS) + operator = Percentile([float(90), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines missing some values + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 4.0, + 240: 6.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 4.0, + 180: 6.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 3.0, + 180: 7.0, + 240: 3.0, + 300: 6.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 8.0, + 240: 2.0, + 300: 7.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert { + 120: 4.0, + 180: 7.0, + 240: 5.0, + 300: 6.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_DIVIDE_execute(): + ts = Mock(TS) + operator = Divide([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 120: 100.0, + 180: 50.0, + 240: 25.0, + 300: 20.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_DIVIDE_execute_when_exception(): + ts = Mock(TS) + operator = Divide([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_DIVIDE_execute_when_missing_values(): + ts = Mock(TS) + operator = Divide([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 180: 50.0, + 240: 25.0, + 300: 20.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_DIVIDE_execute_with_multiple_ts(): + ts = Mock(TS) + ts2 = Mock(TS) + operator = Divide([ts, ts2]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + def ts_side_effect4(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 6.0, + 180: 6.0, + 240: 6.0, + 300: 6.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 8.0, + 180: 8.0, + 240: 8.0, + 300: 8.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 10.0, + 180: 10.0, + 240: 10.0, + 300: 10.0, + }) + ]) + ts2.execute.side_effect = ts_side_effect4 - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 180: 1.0, - 240: 1.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_PERCENTILE_execute_with_multiple_ts(self): - ts = Mock(TS) - operator = Percentile([float(90), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines missing some values - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 4.0, - 240: 6.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 4.0, - 180: 6.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 3.0, - 180: 7.0, - 240: 3.0, - 300: 6.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 8.0, - 240: 2.0, - 300: 7.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertDictEqual({ - 120: 4.0, - 180: 7.0, - 240: 5.0, - 300: 6.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_DIVIDE_execute(self): - ts = Mock(TS) - operator = Divide([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 120: 100.0, - 180: 50.0, - 240: 25.0, - 300: 20.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_DIVIDE_execute_when_exception(self): - ts = Mock(TS) - operator = Divide([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_DIVIDE_execute_when_missing_values(self): - ts = Mock(TS) - operator = Divide([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 180: 50.0, - 240: 25.0, - 300: 20.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_DIVIDE_execute_with_multiple_ts(self): - ts = Mock(TS) - ts2 = Mock(TS) - operator = Divide([ts, ts2]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - @tornado.gen.coroutine - def ts_side_effect4(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 6.0, - 180: 6.0, - 240: 6.0, - 300: 6.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 8.0, - 180: 8.0, - 240: 8.0, - 300: 8.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 10.0, - 180: 10.0, - 240: 10.0, - 300: 10.0, - }) - ]) - ts2.execute.side_effect = ts_side_effect4 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(5, len(metrics)) - for metric in metrics: + metrics = await operator.execute(tracker, tmanager, start, end) + assert 5 == len(metrics) + for metric in metrics: # All should have same value - 0.5 - self.assertDictEqual({ - 120: 0.5, - 180: 0.5, - 240: 0.5, - 300: 0.5 - }, metric.timeline) - - @tornado.testing.gen_test - def test_DIVIDE_execute_with_mulitiple_ts_when_instances_do_not_match(self): - ts = Mock(TS) - ts2 = Mock(TS) - operator = Divide([ts, ts2]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - # When instances do not match - @tornado.gen.coroutine - def ts_side_effect4(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }) - ]) - ts2.execute.side_effect = ts_side_effect4 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(2, len(metrics)) - instances = [] - for metric in metrics: - instances.append(metric.instance) - self.assertDictEqual({ - 120: 0.5, - 180: 0.5, - 240: 0.5, - 300: 0.5 - }, metric.timeline) - self.assertTrue("instance" in instances and "instance2" in instances) - - @tornado.testing.gen_test - def test_MULTIPLY_execute(self): - ts = Mock(TS) - operator = Multiply([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 120: 100.0, - 180: 200.0, - 240: 400.0, - 300: 500.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_MULTIPLY_execute_when_exception(self): - ts = Mock(TS) - operator = Multiply([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_MULTIPLY_execute_when_missing_values(self): - ts = Mock(TS) - operator = Multiply([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 180: 200.0, - 240: 400.0, - 300: 500.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_MULTIPLY_execute_with_multiple_ts(self): - ts = Mock(TS) - ts2 = Mock(TS) - operator = Multiply([ts, ts2]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - @tornado.gen.coroutine - def ts_side_effect4(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 6.0, - 180: 6.0, - 240: 6.0, - 300: 6.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 8.0, - 180: 8.0, - 240: 8.0, - 300: 8.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 10.0, - 180: 10.0, - 240: 10.0, - 300: 10.0, - }) - ]) - ts2.execute.side_effect = ts_side_effect4 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(5, len(metrics)) - for metric in metrics: - if metric.instance == "instance": - self.assertDictEqual({ - 120: 2, - 180: 2, - 240: 2, - 300: 2 - }, metric.timeline) - elif metric.instance == "instance2": - self.assertDictEqual({ - 120: 8, - 180: 8, - 240: 8, - 300: 8 - }, metric.timeline) - elif metric.instance == "instance3": - self.assertDictEqual({ - 120: 18, - 180: 18, - 240: 18, - 300: 18 - }, metric.timeline) - elif metric.instance == "instance4": - self.assertDictEqual({ - 120: 32, - 180: 32, - 240: 32, - 300: 32 - }, metric.timeline) - elif metric.instance == "instance5": - self.assertDictEqual({ - 120: 50, - 180: 50, - 240: 50, - 300: 50 - }, metric.timeline) - - @tornado.testing.gen_test - def test_MULTIPLY_execute_with_multiple_ts_when_instances_do_not_match(self): - ts = Mock(TS) - ts2 = Mock(TS) - operator = Multiply([ts, ts2]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - # When instances do not match - @tornado.gen.coroutine - def ts_side_effect4(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }) - ]) - ts2.execute.side_effect = ts_side_effect4 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(2, len(metrics)) - instances = [] - for metric in metrics: - instances.append(metric.instance) - if metric.instance == "instance": - self.assertDictEqual({ - 120: 2, - 180: 2, - 240: 2, - 300: 2 - }, metric.timeline) - elif metric.instance == "instance2": - self.assertDictEqual({ - 120: 8, - 180: 8, - 240: 8, - 300: 8 - }, metric.timeline) - self.assertTrue("instance" in instances and "instance2" in instances) - - @tornado.testing.gen_test - def test_SUBTRACT_execute(self): - ts = Mock(TS) - operator = Subtract([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 120: 99.0, - 180: 98.0, - 240: 96.0, - 300: 95.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_SUBTRACT_execute_when_exception(self): - ts = Mock(TS) - operator = Subtract([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_SUBTRACT_execute_when_missing_values(self): - ts = Mock(TS) - operator = Subtract([float(100), ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 180: 98.0, - 240: 96.0, - 300: 95.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_SUBTRACT_execute_with_multiple_ts(self): - ts = Mock(TS) - ts2 = Mock(TS) - operator = Subtract([ts, ts2]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - @tornado.gen.coroutine - def ts_side_effect4(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 6.0, - 180: 6.0, - 240: 6.0, - 300: 6.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 8.0, - 180: 8.0, - 240: 8.0, - 300: 8.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 10.0, - 180: 10.0, - 240: 10.0, - 300: 10.0, - }) - ]) - ts2.execute.side_effect = ts_side_effect4 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(5, len(metrics)) - for metric in metrics: - if metric.instance == "instance": - self.assertDictEqual({ - 120: -1, - 180: -1, - 240: -1, - 300: -1 - }, metric.timeline) - elif metric.instance == "instance2": - self.assertDictEqual({ - 120: -2, - 180: -2, - 240: -2, - 300: -2 - }, metric.timeline) - elif metric.instance == "instance3": - self.assertDictEqual({ - 120: -3, - 180: -3, - 240: -3, - 300: -3 - }, metric.timeline) - elif metric.instance == "instance4": - self.assertDictEqual({ - 120: -4, - 180: -4, - 240: -4, - 300: -4 - }, metric.timeline) - elif metric.instance == "instance5": - self.assertDictEqual({ - 120: -5, - 180: -5, - 240: -5, - 300: -5 - }, metric.timeline) - - @tornado.testing.gen_test - def test_SUBTRACT_execute_with_multiple_ts_when_instances_do_not_match(self): - ts = Mock(TS) - ts2 = Mock(TS) - operator = Subtract([ts, ts2]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start, end, { - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start, end, { - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - # When instances do not match - @tornado.gen.coroutine - def ts_side_effect4(*args): - self.assertEqual((tracker, tmanager, 100, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start, end, { - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance2", start, end, { - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }) - ]) - ts2.execute.side_effect = ts_side_effect4 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(2, len(metrics)) - instances = [] - for metric in metrics: - instances.append(metric.instance) - if metric.instance == "instance": - self.assertDictEqual({ - 120: -1, - 180: -1, - 240: -1, - 300: -1 - }, metric.timeline) - elif metric.instance == "instance2": - self.assertDictEqual({ - 120: -2, - 180: -2, - 240: -2, - 300: -2 - }, metric.timeline) - self.assertTrue("instance" in instances and "instance2" in instances) - - @tornado.testing.gen_test - def test_RATE_execute(self): - ts = Mock(TS) - operator = Rate([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Return mocked timeline - @tornado.gen.coroutine - def ts_side_effect(*args): - self.assertEqual((tracker, tmanager, 40, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start-60, end, { - 60: 0.0, - 120: 1.0, - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ - 120: 1.0, - 180: 1.0, - 240: 2.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_RATE_execute_when_exception(self): - ts = Mock(TS) - operator = Rate([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # In case of exception - @tornado.gen.coroutine - def ts_side_effect2(*args): - raise Exception("some_exception") - ts.execute.side_effect = ts_side_effect2 - - with self.assertRaises(Exception): - metrics = yield operator.execute(tracker, tmanager, start, end) - - @tornado.testing.gen_test - def test_RATE_execute_when_missing_values(self): - ts = Mock(TS) - operator = Rate([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # When missing a value - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 40, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start-60, end, { - 60: 0.0, + assert { + 120: 0.5, + 180: 0.5, + 240: 0.5, + 300: 0.5 + } == metric.timeline + +@pytest.mark.asyncio +async def test_DIVIDE_execute_with_mulitiple_ts_when_instances_do_not_match(): + ts = Mock(TS) + ts2 = Mock(TS) + operator = Divide([ts, ts2]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + # When instances do not match + def ts_side_effect4(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }) + ]) + ts2.execute.side_effect = ts_side_effect4 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 2 == len(metrics) + instances = [] + for metric in metrics: + instances.append(metric.instance) + assert { + 120: 0.5, + 180: 0.5, + 240: 0.5, + 300: 0.5 + } == metric.timeline + assert "instance" in instances and "instance2" in instances + +@pytest.mark.asyncio +async def test_MULTIPLY_execute(): + ts = Mock(TS) + operator = Multiply([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 120: 100.0, + 180: 200.0, + 240: 400.0, + 300: 500.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_MULTIPLY_execute_when_exception(): + ts = Mock(TS) + operator = Multiply([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_MULTIPLY_execute_when_missing_values(): + ts = Mock(TS) + operator = Multiply([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 180: 200.0, + 240: 400.0, + 300: 500.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_MULTIPLY_execute_with_multiple_ts(): + ts = Mock(TS) + ts2 = Mock(TS) + operator = Multiply([ts, ts2]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + def ts_side_effect4(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 6.0, + 180: 6.0, + 240: 6.0, + 300: 6.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 8.0, + 180: 8.0, + 240: 8.0, + 300: 8.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 10.0, + 180: 10.0, + 240: 10.0, + 300: 10.0, + }) + ]) + ts2.execute.side_effect = ts_side_effect4 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 5 == len(metrics) + for metric in metrics: + if metric.instance == "instance": + assert { + 120: 2, + 180: 2, + 240: 2, + 300: 2 + } == metric.timeline + elif metric.instance == "instance2": + assert { + 120: 8, + 180: 8, + 240: 8, + 300: 8 + } == metric.timeline + elif metric.instance == "instance3": + assert { + 120: 18, + 180: 18, + 240: 18, + 300: 18 + } == metric.timeline + elif metric.instance == "instance4": + assert { + 120: 32, + 180: 32, + 240: 32, + 300: 32 + } == metric.timeline + elif metric.instance == "instance5": + assert { + 120: 50, + 180: 50, + 240: 50, + 300: 50 + } == metric.timeline + +@pytest.mark.asyncio +async def test_MULTIPLY_execute_with_multiple_ts_when_instances_do_not_match(): + ts = Mock(TS) + ts2 = Mock(TS) + operator = Multiply([ts, ts2]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + # When instances do not match + def ts_side_effect4(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }) + ]) + ts2.execute.side_effect = ts_side_effect4 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 2 == len(metrics) + instances = [] + for metric in metrics: + instances.append(metric.instance) + if metric.instance == "instance": + assert { + 120: 2, + 180: 2, + 240: 2, + 300: 2 + } == metric.timeline + elif metric.instance == "instance2": + assert { + 120: 8, + 180: 8, + 240: 8, + 300: 8 + } == metric.timeline + assert "instance" in instances and "instance2" in instances + +@pytest.mark.asyncio +async def test_SUBTRACT_execute(): + ts = Mock(TS) + operator = Subtract([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 120: 99.0, + 180: 98.0, + 240: 96.0, + 300: 95.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_SUBTRACT_execute_when_exception(): + ts = Mock(TS) + operator = Subtract([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_SUBTRACT_execute_when_missing_values(): + ts = Mock(TS) + operator = Subtract([float(100), ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 180: 98.0, + 240: 96.0, + 300: 95.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_SUBTRACT_execute_with_multiple_ts(): + ts = Mock(TS) + ts2 = Mock(TS) + operator = Subtract([ts, ts2]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + def ts_side_effect4(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 6.0, + 180: 6.0, + 240: 6.0, + 300: 6.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 8.0, + 180: 8.0, + 240: 8.0, + 300: 8.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 10.0, + 180: 10.0, + 240: 10.0, + 300: 10.0, + }) + ]) + ts2.execute.side_effect = ts_side_effect4 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 5 == len(metrics) + for metric in metrics: + if metric.instance == "instance": + assert { + 120: -1, + 180: -1, + 240: -1, + 300: -1 + } == metric.timeline + elif metric.instance == "instance2": + assert { + 120: -2, + 180: -2, + 240: -2, + 300: -2 + } == metric.timeline + elif metric.instance == "instance3": + assert { + 120: -3, + 180: -3, + 240: -3, + 300: -3 + } == metric.timeline + elif metric.instance == "instance4": + assert { + 120: -4, + 180: -4, + 240: -4, + 300: -4 + } == metric.timeline + elif metric.instance == "instance5": + assert { + 120: -5, + 180: -5, + 240: -5, + 300: -5 + } == metric.timeline + +@pytest.mark.asyncio +async def test_SUBTRACT_execute_with_multiple_ts_when_instances_do_not_match(): + ts = Mock(TS) + ts2 = Mock(TS) + operator = Subtract([ts, ts2]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start, end, { + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start, end, { + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + # When instances do not match + def ts_side_effect4(*args): + assert (tracker, tmanager, 100, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start, end, { + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance2", start, end, { + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }) + ]) + ts2.execute.side_effect = ts_side_effect4 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 2 == len(metrics) + instances = [] + for metric in metrics: + instances.append(metric.instance) + if metric.instance == "instance": + assert { + 120: -1, + 180: -1, + 240: -1, + 300: -1 + } == metric.timeline + elif metric.instance == "instance2": + assert { + 120: -2, + 180: -2, + 240: -2, + 300: -2 + } == metric.timeline + assert "instance" in instances and "instance2" in instances + +@pytest.mark.asyncio +async def test_RATE_execute(): + ts = Mock(TS) + operator = Rate([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Return mocked timeline + def ts_side_effect(*args): + assert (tracker, tmanager, 40, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start-60, end, { + 60: 0.0, + 120: 1.0, + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { + 120: 1.0, + 180: 1.0, + 240: 2.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_RATE_execute_when_exception(): + ts = Mock(TS) + operator = Rate([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # In case of exception + def ts_side_effect2(*args): + raise Exception("some_exception") + ts.execute.side_effect = ts_side_effect2 + + with pytest.raises(Exception): + metrics = await operator.execute(tracker, tmanager, start, end) + +@pytest.mark.asyncio +async def test_RATE_execute_when_missing_values(): + ts = Mock(TS) + operator = Rate([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # When missing a value + def ts_side_effect3(*args): + assert (tracker, tmanager, 40, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start-60, end, { + 60: 0.0, # 120: 1.0, # Missing - 180: 2.0, - 240: 4.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(1, len(metrics)) - self.assertEqual("instance", metrics[0].instance) - self.assertDictEqual({ + 180: 2.0, + 240: 4.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 1 == len(metrics) + assert "instance" == metrics[0].instance + assert { # 180: 2.0, # Won't be there since 120 is missing - 240: 2.0, - 300: 1.0 - }, metrics[0].timeline) - - @tornado.testing.gen_test - def test_RATE_execute_with_multiple_ts(self): - ts = Mock(TS) - operator = Rate([ts]) - tmanager = Mock() - tracker = Mock() - start = 100 - end = 300 - - # Multiple timelines - @tornado.gen.coroutine - def ts_side_effect3(*args): - self.assertEqual((tracker, tmanager, 40, 300), args) - raise tornado.gen.Return([ - Metrics("component", "metric_name", "instance", start-60, end, { - 60: 0.0, - 120: 1.0, - 180: 1.0, - 240: 1.0, - 300: 1.0, - }), - Metrics("component", "metric_name", "instance2", start-60, end, { - 60: 0.0, - 120: 2.0, - 180: 2.0, - 240: 2.0, - 300: 2.0, - }), - Metrics("component", "metric_name", "instance3", start-60, end, { - 60: 0.0, - 120: 3.0, - 180: 3.0, - 240: 3.0, - 300: 3.0, - }), - Metrics("component", "metric_name", "instance4", start-60, end, { - 60: 0.0, - 120: 4.0, - 180: 4.0, - 240: 4.0, - 300: 4.0, - }), - Metrics("component", "metric_name", "instance5", start-60, end, { - 60: 0.0, - 120: 5.0, - 180: 5.0, - 240: 5.0, - 300: 5.0, - }) - ]) - ts.execute.side_effect = ts_side_effect3 - - metrics = yield operator.execute(tracker, tmanager, start, end) - self.assertEqual(5, len(metrics)) - for metric in metrics: - if metric.instance == "instance": - self.assertDictEqual({ - 120: 1, - 180: 0, - 240: 0, - 300: 0 - }, metric.timeline) - elif metric.instance == "instance2": - self.assertDictEqual({ - 120: 2, - 180: 0, - 240: 0, - 300: 0 - }, metric.timeline) - elif metric.instance == "instance3": - self.assertDictEqual({ - 120: 3, - 180: 0, - 240: 0, - 300: 0 - }, metric.timeline) - elif metric.instance == "instance4": - self.assertDictEqual({ - 120: 4, - 180: 0, - 240: 0, - 300: 0 - }, metric.timeline) - elif metric.instance == "instance5": - self.assertDictEqual({ - 120: 5, - 180: 0, - 240: 0, - 300: 0 - }, metric.timeline) + 240: 2.0, + 300: 1.0 + } == metrics[0].timeline + +@pytest.mark.asyncio +async def test_RATE_execute_with_multiple_ts(): + ts = Mock(TS) + operator = Rate([ts]) + tmanager = Mock() + tracker = Mock() + start = 100 + end = 300 + + # Multiple timelines + def ts_side_effect3(*args): + assert (tracker, tmanager, 40, 300) == args + return ([ + Metrics("component", "metric_name", "instance", start-60, end, { + 60: 0.0, + 120: 1.0, + 180: 1.0, + 240: 1.0, + 300: 1.0, + }), + Metrics("component", "metric_name", "instance2", start-60, end, { + 60: 0.0, + 120: 2.0, + 180: 2.0, + 240: 2.0, + 300: 2.0, + }), + Metrics("component", "metric_name", "instance3", start-60, end, { + 60: 0.0, + 120: 3.0, + 180: 3.0, + 240: 3.0, + 300: 3.0, + }), + Metrics("component", "metric_name", "instance4", start-60, end, { + 60: 0.0, + 120: 4.0, + 180: 4.0, + 240: 4.0, + 300: 4.0, + }), + Metrics("component", "metric_name", "instance5", start-60, end, { + 60: 0.0, + 120: 5.0, + 180: 5.0, + 240: 5.0, + 300: 5.0, + }) + ]) + ts.execute.side_effect = ts_side_effect3 + + metrics = await operator.execute(tracker, tmanager, start, end) + assert 5 == len(metrics) + for metric in metrics: + if metric.instance == "instance": + assert { + 120: 1, + 180: 0, + 240: 0, + 300: 0 + } == metric.timeline + elif metric.instance == "instance2": + assert { + 120: 2, + 180: 0, + 240: 0, + 300: 0 + } == metric.timeline + elif metric.instance == "instance3": + assert { + 120: 3, + 180: 0, + 240: 0, + 300: 0 + } == metric.timeline + elif metric.instance == "instance4": + assert { + 120: 4, + 180: 0, + 240: 0, + 300: 0 + } == metric.timeline + elif metric.instance == "instance5": + assert { + 120: 5, + 180: 0, + 240: 0, + 300: 0 + } == metric.timeline diff --git a/heron/tools/tracker/tests/python/query_unittest.py b/heron/tools/tracker/tests/python/query_unittest.py index 6d34cc0185c..b482b47d674 100644 --- a/heron/tools/tracker/tests/python/query_unittest.py +++ b/heron/tools/tracker/tests/python/query_unittest.py @@ -16,158 +16,160 @@ # under the License. ''' query_unittest.py ''' # pylint: disable=missing-docstring, undefined-variable -import unittest -from unittest.mock import Mock +from unittest.mock import MagicMock from heron.tools.tracker.src.python.query import * +from heron.tools.tracker.src.python.tracker import Tracker -class QueryTest(unittest.TestCase): - def setUp(self): - self.tracker = Mock() - self.query = Query(self.tracker) +import pytest - def test_find_closing_braces(self): - query = "(())" - self.assertEqual(3, self.query.find_closing_braces(query)) +@pytest.fixture +def mock_query(): + tracker = MagicMock(Tracker) + return Query(tracker) - query = "hello()" - with self.assertRaises(Exception): - self.query.find_closing_braces(query) +def test_find_closing_braces(mock_query): + query = "(())" + assert 3 == mock_query.find_closing_braces(query) - query = "(hello)" - self.assertEqual(6, self.query.find_closing_braces(query)) + query = "hello()" + with pytest.raises(Exception): + mock_query.find_closing_braces(query) - query = "(no closing braces" - with self.assertRaises(Exception): - self.query.find_closing_braces(query) + query = "(hello)" + assert 6 == mock_query.find_closing_braces(query) - query = "()()" - self.assertEqual(1, self.query.find_closing_braces(query)) + query = "(no closing braces" + with pytest.raises(Exception): + mock_query.find_closing_braces(query) - def test_get_sub_parts(self): - query = "abc, def, xyz" - self.assertEqual(["abc", "def", "xyz"], self.query.get_sub_parts(query)) + query = "()()" + assert 1 == mock_query.find_closing_braces(query) - query = "(abc, xyz)" - self.assertEqual(["(abc, xyz)"], self.query.get_sub_parts(query)) +def test_get_sub_parts(mock_query): + query = "abc, def, xyz" + assert ["abc", "def", "xyz"] == mock_query.get_sub_parts(query) - query = "a(x, y), b(p, q)" - self.assertEqual(["a(x, y)", "b(p, q)"], self.query.get_sub_parts(query)) + query = "(abc, xyz)" + assert ["(abc, xyz)"] == mock_query.get_sub_parts(query) - query = ",," - self.assertEqual(["", "", ""], self.query.get_sub_parts(query)) + query = "a(x, y), b(p, q)" + assert ["a(x, y)", "b(p, q)"] == mock_query.get_sub_parts(query) - query = "())" - with self.assertRaises(Exception): - self.query.get_sub_parts(query) + query = ",," + assert ["", "", ""] == mock_query.get_sub_parts(query) + + query = "())" + with pytest.raises(Exception): + mock_query.get_sub_parts(query) # pylint: disable=too-many-statements - def test_parse_query_string(self): - query = "TS(a, b, c)" - root = self.query.parse_query_string(query) - self.assertEqual("a", root.component) - self.assertEqual(["b"], root.instances) - self.assertEqual("c", root.metric_name) - - query = "TS(a, *, m)" - root = self.query.parse_query_string(query) - self.assertEqual("a", root.component) - self.assertEqual([], root.instances) - self.assertEqual("m", root.metric_name) - - query = "DEFAULT(0, TS(a, b, c))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Default) - self.assertIsInstance(root.constant, float) - self.assertEqual(root.constant, 0) - self.assertIsInstance(root.timeseries, TS) - - query = "DEFAULT(0, SUM(TS(a, a, a), TS(b, b, b)))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Default) - self.assertIsInstance(root.constant, float) - self.assertIsInstance(root.timeseries, Sum) - self.assertEqual(2, len(root.timeseries.timeSeriesList)) - self.assertIsInstance(root.timeseries.timeSeriesList[0], TS) - self.assertIsInstance(root.timeseries.timeSeriesList[1], TS) - - query = "MAX(1, TS(a, a, a))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Max) - self.assertIsInstance(root.timeSeriesList[0], float) - self.assertIsInstance(root.timeSeriesList[1], TS) - - query = "PERCENTILE(90, TS(a, a, a))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Percentile) - self.assertIsInstance(root.quantile, float) - self.assertIsInstance(root.timeSeriesList[0], TS) - - query = "PERCENTILE(TS(a, a, a), 90)" - with self.assertRaises(Exception): - self.query.parse_query_string(query) - - query = "DIVIDE(TS(a, a, a), TS(b, b, b))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Divide) - self.assertIsInstance(root.operand1, TS) - self.assertIsInstance(root.operand2, TS) +def test_parse_query_string(mock_query): + query = "TS(a, b, c)" + root = mock_query.parse_query_string(query) + assert "a" == root.component + assert ["b"] == root.instances + assert "c" == root.metric_name + + query = "TS(a, *, m)" + root = mock_query.parse_query_string(query) + assert "a" == root.component + assert [] == root.instances + assert "m" == root.metric_name + + query = "DEFAULT(0, TS(a, b, c))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Default) + assert isinstance(root.constant, float) + assert root.constant == 0 + assert isinstance(root.timeseries, TS) + + query = "DEFAULT(0, SUM(TS(a, a, a), TS(b, b, b)))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Default) + assert isinstance(root.constant, float) + assert isinstance(root.timeseries, Sum) + assert 2 == len(root.timeseries.time_series_list) + assert isinstance(root.timeseries.time_series_list[0], TS) + assert isinstance(root.timeseries.time_series_list[1], TS) + + query = "MAX(1, TS(a, a, a))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Max) + assert isinstance(root.time_series_list[0], float) + assert isinstance(root.time_series_list[1], TS) + + query = "PERCENTILE(90, TS(a, a, a))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Percentile) + assert isinstance(root.quantile, float) + assert isinstance(root.time_series_list[0], TS) + + query = "PERCENTILE(TS(a, a, a), 90)" + with pytest.raises(Exception): + mock_query.parse_query_string(query) + + query = "DIVIDE(TS(a, a, a), TS(b, b, b))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Divide) + assert isinstance(root.operand1, TS) + assert isinstance(root.operand2, TS) # Dividing by a constant is fine - query = "DIVIDE(TS(a, a, a), 90)" - self.query.parse_query_string(query) - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Divide) - self.assertIsInstance(root.operand1, TS) - self.assertIsInstance(root.operand2, float) + query = "DIVIDE(TS(a, a, a), 90)" + mock_query.parse_query_string(query) + root = mock_query.parse_query_string(query) + assert isinstance(root, Divide) + assert isinstance(root.operand1, TS) + assert isinstance(root.operand2, float) # Must have two operands - query = "DIVIDE(TS(a, a, a))" - with self.assertRaises(Exception): - self.query.parse_query_string(query) + query = "DIVIDE(TS(a, a, a))" + with pytest.raises(Exception): + mock_query.parse_query_string(query) - query = "MULTIPLY(TS(a, a, a), TS(b, b, b))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Multiply) - self.assertIsInstance(root.operand1, TS) - self.assertIsInstance(root.operand2, TS) + query = "MULTIPLY(TS(a, a, a), TS(b, b, b))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Multiply) + assert isinstance(root.operand1, TS) + assert isinstance(root.operand2, TS) # Multiplying with a constant is fine. - query = "MULTIPLY(TS(a, a, a), 10)" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Multiply) - self.assertIsInstance(root.operand1, TS) - self.assertIsInstance(root.operand2, float) + query = "MULTIPLY(TS(a, a, a), 10)" + root = mock_query.parse_query_string(query) + assert isinstance(root, Multiply) + assert isinstance(root.operand1, TS) + assert isinstance(root.operand2, float) # Must have two operands - query = "MULTIPLY(TS(a, a, a))" - with self.assertRaises(Exception): - self.query.parse_query_string(query) + query = "MULTIPLY(TS(a, a, a))" + with pytest.raises(Exception): + mock_query.parse_query_string(query) - query = "SUBTRACT(TS(a, a, a), TS(b, b, b))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Subtract) - self.assertIsInstance(root.operand1, TS) - self.assertIsInstance(root.operand2, TS) + query = "SUBTRACT(TS(a, a, a), TS(b, b, b))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Subtract) + assert isinstance(root.operand1, TS) + assert isinstance(root.operand2, TS) # Multiplying with a constant is fine. - query = "SUBTRACT(TS(a, a, a), 10)" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Subtract) - self.assertIsInstance(root.operand1, TS) - self.assertIsInstance(root.operand2, float) + query = "SUBTRACT(TS(a, a, a), 10)" + root = mock_query.parse_query_string(query) + assert isinstance(root, Subtract) + assert isinstance(root.operand1, TS) + assert isinstance(root.operand2, float) # Must have two operands - query = "SUBTRACT(TS(a, a, a))" - with self.assertRaises(Exception): - self.query.parse_query_string(query) + query = "SUBTRACT(TS(a, a, a))" + with pytest.raises(Exception): + mock_query.parse_query_string(query) - query = "RATE(TS(a, a, a))" - root = self.query.parse_query_string(query) - self.assertIsInstance(root, Rate) - self.assertIsInstance(root.timeSeries, TS) + query = "RATE(TS(a, a, a))" + root = mock_query.parse_query_string(query) + assert isinstance(root, Rate) + assert isinstance(root.time_series, TS) # Must have one operand only - query = "RATE(TS(a, a, a), TS(b, b, b))" - with self.assertRaises(Exception): - self.query.parse_query_string(query) + query = "RATE(TS(a, a, a), TS(b, b, b))" + with pytest.raises(Exception): + mock_query.parse_query_string(query) diff --git a/heron/tools/tracker/tests/python/topology_unittest.py b/heron/tools/tracker/tests/python/topology_unittest.py index 8bf3f3219d2..a03ea4aeb36 100644 --- a/heron/tools/tracker/tests/python/topology_unittest.py +++ b/heron/tools/tracker/tests/python/topology_unittest.py @@ -18,167 +18,101 @@ # pylint: disable=missing-docstring import unittest +from unittest.mock import MagicMock + +from heron.tools.tracker.src.python.tracker import Tracker from heron.tools.tracker.src.python.topology import Topology from mock_proto import MockProto -class TopologyTest(unittest.TestCase): - def setUp(self): - self.state_manager_name = "test_state_manager_name" - self.topology = Topology(MockProto.topology_name, - self.state_manager_name) - - def test_set_physical_plan(self): - # Set it to None - self.topology.set_physical_plan(None) - self.assertIsNone(self.topology.id) - self.assertIsNone(self.topology.physical_plan) - - physical_plan = MockProto().create_mock_simple_physical_plan() - self.topology.set_physical_plan(physical_plan) - self.assertEqual(MockProto.topology_id, self.topology.id) - self.assertEqual(physical_plan, self.topology.physical_plan) - - def test_set_packing_plan(self): - # Set it to None - self.topology.set_packing_plan(None) - self.assertIsNone(self.topology.id) - self.assertIsNone(self.topology.packing_plan) - - packing_plan = MockProto().create_mock_simple_packing_plan() - self.topology.set_packing_plan(packing_plan) - self.assertEqual(packing_plan, self.topology.packing_plan) - - # testing with a packing plan with scheduled resources - self.topology.set_packing_plan(None) - self.assertIsNone(self.topology.id) - self.assertIsNone(self.topology.packing_plan) - - packing_plan = MockProto().create_mock_simple_packing_plan2() - self.topology.set_packing_plan(packing_plan) - self.assertEqual(packing_plan, self.topology.packing_plan) - - def test_set_execution_state(self): - # Set it to None - self.topology.set_execution_state(None) - self.assertIsNone(self.topology.execution_state) - self.assertIsNone(self.topology.cluster) - self.assertIsNone(self.topology.environ) - - estate = MockProto().create_mock_execution_state() - self.topology.set_execution_state(estate) - self.assertEqual(estate, self.topology.execution_state) - self.assertEqual(MockProto.cluster, self.topology.cluster) - self.assertEqual(MockProto.environ, self.topology.environ) - - def test_set_tmanager(self): - # Set it to None - self.topology.set_tmanager(None) - self.assertIsNone(self.topology.tmanager) - - tmanager = MockProto().create_mock_tmanager() - self.topology.set_tmanager(tmanager) - self.assertEqual(tmanager, self.topology.tmanager) - - def test_spouts(self): - # When pplan is not set - self.assertEqual(0, len(self.topology.spouts())) - - # Set pplan now - pplan = MockProto().create_mock_simple_physical_plan() - self.topology.set_physical_plan(pplan) - - spouts = self.topology.spouts() - self.assertEqual(1, len(spouts)) - self.assertEqual("mock_spout", spouts[0].comp.name) - self.assertEqual(["mock_spout"], self.topology.spout_names()) - - def test_bolts(self): - # When pplan is not set - self.assertEqual(0, len(self.topology.bolts())) - - # Set pplan - pplan = MockProto().create_mock_medium_physical_plan() - self.topology.set_physical_plan(pplan) - - bolts = self.topology.bolts() - self.assertEqual(3, len(bolts)) - self.assertEqual(["mock_bolt1", "mock_bolt2", "mock_bolt3"], - self.topology.bolt_names()) - - def test_num_instances(self): - # When pplan is not set - self.assertEqual(0, self.topology.num_instances()) - - pplan = MockProto().create_mock_medium_physical_plan(1, 2, 3, 4) - self.topology.set_physical_plan(pplan) - - self.assertEqual(10, self.topology.num_instances()) - - def test_trigger_watches(self): - # Workaround - scope = { - "is_called": False - } - # pylint: disable=unused-argument, unused-variable - def callback(something): - scope["is_called"] = True - uid = self.topology.register_watch(callback) - self.assertTrue(scope["is_called"]) - - scope["is_called"] = False - self.assertFalse(scope["is_called"]) - print(scope) - self.topology.set_physical_plan(None) - print(scope) - self.assertTrue(scope["is_called"]) - print(scope) - - scope["is_called"] = False - self.assertFalse(scope["is_called"]) - self.topology.set_execution_state(None) - self.assertTrue(scope["is_called"]) - - scope["is_called"] = False - self.assertFalse(scope["is_called"]) - self.topology.set_tmanager(None) - self.assertTrue(scope["is_called"]) - - def test_unregister_watch(self): - # Workaround - scope = { - "is_called": False - } - # pylint: disable=unused-argument - def callback(something): - scope["is_called"] = True - uid = self.topology.register_watch(callback) - scope["is_called"] = False - self.assertFalse(scope["is_called"]) - self.topology.set_physical_plan(None) - self.assertTrue(scope["is_called"]) - - self.topology.unregister_watch(uid) - scope["is_called"] = False - self.assertFalse(scope["is_called"]) - self.topology.set_physical_plan(None) - self.assertFalse(scope["is_called"]) - - def test_bad_watch(self): - # Workaround - scope = { - "is_called": False - } - # pylint: disable=unused-argument, unused-variable - def callback(something): - scope["is_called"] = True - raise Exception("Test Bad Trigger Exception") - - uid = self.topology.register_watch(callback) - # is called the first time because of registeration - self.assertTrue(scope["is_called"]) - - # But should no longer be called - scope["is_called"] = False - self.assertFalse(scope["is_called"]) - self.topology.set_physical_plan(None) - self.assertFalse(scope["is_called"]) +import pytest + + +@pytest.fixture +def tracker(): + mock = MagicMock(Tracker) + mock.config.extra_links = [] + return mock + +@pytest.fixture +def topology(tracker): + return Topology(MockProto.topology_name, + "test_state_manager_name", + tracker.config) + +def test_set_physical_plan(topology): + # Set it to None + topology.set_physical_plan(None) + assert topology.id is None + assert topology.physical_plan is None + + physical_plan = MockProto().create_mock_simple_physical_plan() + topology.set_physical_plan(physical_plan) + assert MockProto.topology_id == topology.id + assert physical_plan == topology.physical_plan + +def test_set_packing_plan(topology): + # Set it to None + topology.set_packing_plan(None) + assert topology.id is None + assert topology.packing_plan is None + + packing_plan = MockProto().create_mock_simple_packing_plan() + topology.set_packing_plan(packing_plan) + assert packing_plan == topology.packing_plan + + # testing with a packing plan with scheduled resources + topology.set_packing_plan(None) + assert topology.id is None + assert topology.packing_plan is None + + packing_plan = MockProto().create_mock_simple_packing_plan2() + topology.set_packing_plan(packing_plan) + assert packing_plan == topology.packing_plan + +def test_set_execution_state(topology): + # Set it to None + topology.set_execution_state(None) + assert topology.execution_state is None + assert topology.cluster is None + assert topology.environ is None + + estate = MockProto().create_mock_execution_state() + topology.set_execution_state(estate) + assert estate == topology.execution_state + assert MockProto.cluster == topology.cluster + assert MockProto.environ == topology.environ + +def test_set_tmanager(topology): + # Set it to None + topology.set_tmanager(None) + assert topology.tmanager is None + + tmanager = MockProto().create_mock_tmanager() + topology.set_tmanager(tmanager) + assert tmanager == topology.tmanager + +def test_spouts(topology): + # When pplan is not set + assert 0 == len(topology.spouts()) + + # Set pplan now + pplan = MockProto().create_mock_simple_physical_plan() + topology.set_physical_plan(pplan) + + spouts = topology.spouts() + assert 1 == len(spouts) + assert "mock_spout" == spouts[0].comp.name + assert ["mock_spout"] == topology.spout_names() + +def test_bolts(topology): + # When pplan is not set + assert 0 == len(topology.bolts()) + + # Set pplan + pplan = MockProto().create_mock_medium_physical_plan() + topology.set_physical_plan(pplan) + + bolts = topology.bolts() + assert 3 == len(bolts) + assert ["mock_bolt1", "mock_bolt2", "mock_bolt3"] == \ + topology.bolt_names() diff --git a/heron/tools/tracker/tests/python/tracker_unittest.py b/heron/tools/tracker/tests/python/tracker_unittest.py index 3a4df67f3ef..cdd482e372a 100644 --- a/heron/tools/tracker/tests/python/tracker_unittest.py +++ b/heron/tools/tracker/tests/python/tracker_unittest.py @@ -17,249 +17,196 @@ ''' tracker_unittest.py ''' # pylint: disable=missing-docstring, attribute-defined-outside-init -import unittest +from functools import partial from unittest.mock import call, patch, Mock import heron.proto.execution_state_pb2 as protoEState + from heron.statemgrs.src.python import statemanagerfactory from heron.tools.tracker.src.python.topology import Topology from heron.tools.tracker.src.python.tracker import Tracker -from mock_proto import MockProto - -class TrackerTest(unittest.TestCase): - def setUp(self): - mock_config = Mock() - mock_config.validate.return_value = True - self.tracker = Tracker(mock_config) - - # pylint: disable=unused-argument - @patch.object(Tracker, 'get_stmgr_topologies') - @patch.object(Tracker, 'remove_topology') - @patch.object(Tracker, 'add_new_topology') - @patch.object(statemanagerfactory, 'get_all_state_managers') - def test_first_synch_topologies( - self, mock_get_all_state_managers, - mock_add_new_topology, mock_remove_topology, - mock_get_topologies_for_state_location): - mock_state_manager_1 = Mock() - mock_state_manager_1.name = 'mock_name1' - - mock_state_manager_2 = Mock() - mock_state_manager_2.name = 'mock_name2' - - watches = {} - mock_get_all_state_managers.return_value = [mock_state_manager_1, mock_state_manager_2] - - mock_get_topologies_for_state_location.return_value = [] - def side_effect1(on_topologies_watch): - watches["1"] = on_topologies_watch - on_topologies_watch(['top_name1', 'top_name2']) - mock_state_manager_1.get_topologies = side_effect1 - - def side_effect2(on_topologies_watch): - watches["2"] = on_topologies_watch - on_topologies_watch(['top_name3', 'top_name4']) - mock_state_manager_2.get_topologies = side_effect2 - - self.tracker.synch_topologies() - mock_get_topologies_for_state_location.assert_has_calls( - [call("mock_name2"), - call("mock_name1")], - any_order=True) - mock_add_new_topology.assert_has_calls([call(mock_state_manager_1, 'top_name1'), - call(mock_state_manager_1, 'top_name2'), - call(mock_state_manager_2, 'top_name3'), - call(mock_state_manager_2, 'top_name4')], - any_order=True) - - @patch.object(Tracker, 'get_stmgr_topologies') - @patch.object(Tracker, 'remove_topology') - @patch.object(Tracker, 'add_new_topology') - @patch.object(statemanagerfactory, 'get_all_state_managers') - def test_synch_topologies_leading_with_add_and_remove_topologies( - self, mock_get_all_state_managers, - mock_add_new_topology, mock_remove_topology, - mock_get_topologies_for_state_location): - mock_state_manager_1 = Mock() - mock_state_manager_1.name = 'mock_name1' - - mock_state_manager_2 = Mock() - mock_state_manager_2.name = 'mock_name2' - - watches = {} - mock_get_all_state_managers.return_value = [mock_state_manager_1, mock_state_manager_2] - mock_get_topologies_for_state_location.return_value = [] - - def side_effect1(on_topologies_watch): - watches["1"] = on_topologies_watch - on_topologies_watch(['top_name1', 'top_name2']) - mock_state_manager_1.get_topologies = side_effect1 - - def side_effect2(on_topologies_watch): - watches["2"] = on_topologies_watch - on_topologies_watch(['top_name3', 'top_name4']) - mock_state_manager_2.get_topologies = side_effect2 - - self.tracker.synch_topologies() - mock_get_topologies_for_state_location.assert_has_calls( - [call("mock_name2"), - call("mock_name1")], - any_order=True) - mock_add_new_topology.assert_has_calls([call(mock_state_manager_1, 'top_name1'), - call(mock_state_manager_1, 'top_name2'), - call(mock_state_manager_2, 'top_name3'), - call(mock_state_manager_2, 'top_name4')], - any_order=True) - self.assertEqual(4, mock_add_new_topology.call_count) - self.assertEqual(0, mock_remove_topology.call_count) - mock_get_topologies_for_state_location.reset_mock() - mock_add_new_topology.reset_mock() - mock_remove_topology.reset_mock() - - def get_topologies_for_state_location_side_effect(name): - if name == 'mock_name1': - return [Topology('top_name1', 'mock_name1'), - Topology('top_name2', 'mock_name1')] - if name == 'mock_name2': - return [Topology('top_name3', 'mock_name2'), - Topology('top_name4', 'mock_name2')] - return [] - - # pylint: disable=line-too-long - mock_get_topologies_for_state_location.side_effect = get_topologies_for_state_location_side_effect - - watches["1"](['top_name1', 'top_name3']) - watches["2"](['top_name5', 'top_name6']) - mock_add_new_topology.assert_has_calls([call(mock_state_manager_1, 'top_name3'), - call(mock_state_manager_2, 'top_name5'), - call(mock_state_manager_2, 'top_name6')], - any_order=True) - mock_remove_topology.assert_has_calls([call('top_name2', 'mock_name1'), - call('top_name3', 'mock_name2'), - call('top_name4', 'mock_name2')], - any_order=False) - self.assertEqual(3, mock_add_new_topology.call_count) - self.assertEqual(3, mock_remove_topology.call_count) - - def fill_tracker_topologies(self): - - def create_mock_execution_state(cluster, role, environ): - estate = protoEState.ExecutionState() - estate.cluster = cluster - estate.role = role - estate.environ = environ - return estate - - self.topology1 = Topology('top_name1', 'mock_name1') - self.topology1.execution_state = create_mock_execution_state('cluster1', 'mark', 'env1') - - self.topology2 = Topology('top_name2', 'mock_name1') - self.topology2.execution_state = create_mock_execution_state('cluster1', 'bob', 'env1') - - self.topology3 = Topology('top_name3', 'mock_name1') - self.topology3.execution_state = create_mock_execution_state('cluster1', 'tom', 'env2') - - self.topology4 = Topology('top_name4', 'mock_name2') - self.topology4.execution_state = create_mock_execution_state('cluster2', 'x', 'env1') - - self.topology5 = Topology('top_name5', 'mock_name2') - self.topology5.execution_state = create_mock_execution_state('cluster2', 'x', 'env2') - self.tracker.topologies = [ - self.topology1, - self.topology2, - self.topology3, - self.topology4, - self.topology5] + +import pytest + +# just for convenience in testing +Topology = partial(Topology, tracker_config={}) + +@pytest.fixture +def tracker(): + mock_config = Mock() + mock_config.validate.return_value = True + return Tracker(mock_config) + +@pytest.fixture +def mock_tracker(tracker): + # this wouldn't be so ugly with Python3.9+ + with patch.object(Tracker, 'get_stmgr_topologies'), patch.object(Tracker, 'remove_topology'), patch.object(Tracker, 'add_new_topology'), patch.object(statemanagerfactory, 'get_all_state_managers'): + yield tracker + +# pylint: disable=unused-argument +def test_first_sync_topologies(mock_tracker): + mock_state_manager_1 = Mock() + mock_state_manager_1.name = 'mock_name1' + + mock_state_manager_2 = Mock() + mock_state_manager_2.name = 'mock_name2' + + watches = {} + statemanagerfactory.get_all_state_managers.return_value = [mock_state_manager_1, mock_state_manager_2] + + mock_tracker.get_stmgr_topologies.return_value = [] + def side_effect1(on_topologies_watch): + watches["1"] = on_topologies_watch + on_topologies_watch(['top_name1', 'top_name2']) + mock_state_manager_1.get_topologies = side_effect1 + + def side_effect2(on_topologies_watch): + watches["2"] = on_topologies_watch + on_topologies_watch(['top_name3', 'top_name4']) + mock_state_manager_2.get_topologies = side_effect2 + + mock_tracker.sync_topologies() + mock_tracker.get_stmgr_topologies.assert_has_calls( + [call("mock_name2"), + call("mock_name1")], + any_order=True) + mock_tracker.add_new_topology.assert_has_calls([call(mock_state_manager_1, 'top_name1'), + call(mock_state_manager_1, 'top_name2'), + call(mock_state_manager_2, 'top_name3'), + call(mock_state_manager_2, 'top_name4')], + any_order=True) + +def test_sync_topologies_leading_with_add_and_remove_topologies(mock_tracker): + mock_state_manager_1 = Mock() + mock_state_manager_1.name = 'mock_name1' + + mock_state_manager_2 = Mock() + mock_state_manager_2.name = 'mock_name2' + + watches = {} + statemanagerfactory.get_all_state_managers.return_value = [mock_state_manager_1, mock_state_manager_2] + mock_tracker.get_stmgr_topologies.return_value = [] + + def side_effect1(on_topologies_watch): + watches["1"] = on_topologies_watch + on_topologies_watch(['top_name1', 'top_name2']) + mock_state_manager_1.get_topologies = side_effect1 + + def side_effect2(on_topologies_watch): + watches["2"] = on_topologies_watch + on_topologies_watch(['top_name3', 'top_name4']) + mock_state_manager_2.get_topologies = side_effect2 + + mock_tracker.sync_topologies() + mock_tracker.get_stmgr_topologies.assert_has_calls( + [call("mock_name2"), + call("mock_name1")], + any_order=True) + mock_tracker.add_new_topology.assert_has_calls([call(mock_state_manager_1, 'top_name1'), + call(mock_state_manager_1, 'top_name2'), + call(mock_state_manager_2, 'top_name3'), + call(mock_state_manager_2, 'top_name4')], + any_order=True) + assert 4 == mock_tracker.add_new_topology.call_count + assert 0 == mock_tracker.remove_topology.call_count + mock_tracker.get_stmgr_topologies.reset_mock() + mock_tracker.add_new_topology.reset_mock() + mock_tracker.remove_topology.reset_mock() + + def get_topologies_for_state_location_side_effect(name): + if name == 'mock_name1': + return [Topology('top_name1', 'mock_name1'), + Topology('top_name2', 'mock_name1')] + if name == 'mock_name2': + return [Topology('top_name3', 'mock_name2'), + Topology('top_name4', 'mock_name2')] + return [] # pylint: disable=line-too-long - def test_get_topology_by_cluster_environ_and_name(self): - self.fill_tracker_topologies() - self.assertEqual(self.topology1, self.tracker.get_topology('cluster1', 'mark', 'env1', 'top_name1')) - self.assertEqual(self.topology1, self.tracker.get_topology('cluster1', None, 'env1', 'top_name1')) - self.assertEqual(self.topology2, self.tracker.get_topology('cluster1', 'bob', 'env1', 'top_name2')) - self.assertEqual(self.topology2, self.tracker.get_topology('cluster1', None, 'env1', 'top_name2')) - self.assertEqual(self.topology3, self.tracker.get_topology('cluster1', 'tom', 'env2', 'top_name3')) - self.assertEqual(self.topology3, self.tracker.get_topology('cluster1', None, 'env2', 'top_name3')) - self.assertEqual(self.topology4, self.tracker.get_topology('cluster2', None, 'env1', 'top_name4')) - self.assertEqual(self.topology5, self.tracker.get_topology('cluster2', None, 'env2', 'top_name5')) - - def test_get_topolies_for_state_location(self): - self.fill_tracker_topologies() - self.assertCountEqual( - [self.topology1, self.topology2, self.topology3], - self.tracker.get_stmgr_topologies('mock_name1')) - self.assertCountEqual( - [self.topology4, self.topology5], - self.tracker.get_stmgr_topologies('mock_name2')) - - def test_add_new_topology(self): - self.assertCountEqual([], self.tracker.topologies) - mock_state_manager_1 = Mock() - mock_state_manager_1.name = 'mock_name1' - - self.tracker.add_new_topology(mock_state_manager_1, 'top_name1') - self.assertCountEqual( - ['top_name1'], - [t.name for t in self.tracker.topologies]) - - self.tracker.add_new_topology(mock_state_manager_1, 'top_name2') - self.assertCountEqual( - ['top_name1', 'top_name2'], - [t.name for t in self.tracker.topologies]) - - self.assertEqual(2, mock_state_manager_1.get_pplan.call_count) - self.assertEqual(2, mock_state_manager_1.get_execution_state.call_count) - self.assertEqual(2, mock_state_manager_1.get_tmanager.call_count) - - def test_remove_topology(self): - self.fill_tracker_topologies() - self.tracker.remove_topology('top_name1', 'mock_name1') - self.assertCountEqual([self.topology2, self.topology3, self.topology4, self.topology5], - self.tracker.topologies) - self.tracker.remove_topology('top_name2', 'mock_name1') - self.assertCountEqual([self.topology3, self.topology4, self.topology5], - self.tracker.topologies) - # Removing one that is not there should not have any affect - self.tracker.remove_topology('top_name8', 'mock_name1') - self.assertCountEqual([self.topology3, self.topology4, self.topology5], - self.tracker.topologies) - self.tracker.remove_topology('top_name4', 'mock_name2') - self.assertCountEqual([self.topology3, self.topology5], - self.tracker.topologies) - - def test_extract_physical_plan(self): - # Create topology - pb_pplan = MockProto().create_mock_simple_physical_plan() - topology = Topology('topology_name', 'state_manager') - topology.set_physical_plan(pb_pplan) - # Extract physical plan - pplan = self.tracker.extract_physical_plan(topology) - # Mock topology doesn't have topology config and instances - self.assertEqual(pplan['config'], {}) - self.assertEqual(pplan['bolts'], {'mock_bolt': []}) - self.assertEqual(pplan['spouts'], {'mock_spout': []}) - self.assertEqual(pplan['components']['mock_bolt']['config'], - {'topology.component.parallelism': '1'}) - self.assertEqual(pplan['components']['mock_spout']['config'], - {'topology.component.parallelism': '1'}) - self.assertEqual(pplan['instances'], {}) - self.assertEqual(pplan['stmgrs'], {}) - - def test_extract_packing_plan(self): - # Create topology - pb_pplan = MockProto().create_mock_simple_packing_plan() - topology = Topology('topology_name', 'ExclamationTopology') - topology.set_packing_plan(pb_pplan) - # Extract packing plan - packing_plan = self.tracker.extract_packing_plan(topology) - self.assertEqual(packing_plan['id'], 'ExclamationTopology') - self.assertEqual(packing_plan['container_plans'][0]['id'], 1) - self.assertEqual(packing_plan['container_plans'][0]['required_resources'], - {'disk': 2048, 'ram': 1024, 'cpu': 1.0}) - self.assertEqual(packing_plan['container_plans'][0]['instances'][0], - { - 'component_index': 1, - 'component_name': 'word', - 'instance_resources': {'cpu': 1.0, 'disk': 2048, 'ram': 1024}, - 'task_id': 1 - }) + mock_tracker.get_stmgr_topologies.side_effect = get_topologies_for_state_location_side_effect + + watches["1"](['top_name1', 'top_name3']) + watches["2"](['top_name5', 'top_name6']) + mock_tracker.add_new_topology.assert_has_calls([call(mock_state_manager_1, 'top_name3'), + call(mock_state_manager_2, 'top_name5'), + call(mock_state_manager_2, 'top_name6')], + any_order=True) + mock_tracker.remove_topology.assert_has_calls([call('top_name2', 'mock_name1'), + call('top_name3', 'mock_name2'), + call('top_name4', 'mock_name2')], + any_order=True) + assert 3 == mock_tracker.add_new_topology.call_count + assert 3 == mock_tracker.remove_topology.call_count + +@pytest.fixture +def topologies(tracker): + + def create_mock_execution_state(cluster, role, environ): + estate = protoEState.ExecutionState() + estate.cluster = cluster + estate.role = role + estate.environ = environ + return estate + + topology1 = Topology('top_name1', 'mock_name1') + topology1.execution_state = create_mock_execution_state('cluster1', 'mark', 'env1') + + topology2 = Topology('top_name2', 'mock_name1') + topology2.execution_state = create_mock_execution_state('cluster1', 'bob', 'env1') + + topology3 = Topology('top_name3', 'mock_name1') + topology3.execution_state = create_mock_execution_state('cluster1', 'tom', 'env2') + + topology4 = Topology('top_name4', 'mock_name2') + topology4.execution_state = create_mock_execution_state('cluster2', 'x', 'env1') + + topology5 = Topology('top_name5', 'mock_name2') + topology5.execution_state = create_mock_execution_state('cluster2', 'x', 'env2') + tracker.topologies = [ + topology1, + topology2, + topology3, + topology4, + topology5] + return tracker.topologies[:] + +# pylint: disable=line-too-long +def test_get_topology_by_cluster_environ_and_name(tracker, topologies): + assert topologies[0] == tracker.get_topology('cluster1', 'mark', 'env1', 'top_name1') + assert topologies[0] == tracker.get_topology('cluster1', None, 'env1', 'top_name1') + assert topologies[1] == tracker.get_topology('cluster1', 'bob', 'env1', 'top_name2') + assert topologies[1] == tracker.get_topology('cluster1', None, 'env1', 'top_name2') + assert topologies[2] == tracker.get_topology('cluster1', 'tom', 'env2', 'top_name3') + assert topologies[2] == tracker.get_topology('cluster1', None, 'env2', 'top_name3') + assert topologies[3] == tracker.get_topology('cluster2', None, 'env1', 'top_name4') + assert topologies[4] == tracker.get_topology('cluster2', None, 'env2', 'top_name5') + +def test_get_topolies_for_state_location(tracker, topologies): + assert [topologies[0], topologies[1], topologies[2]] == tracker.get_stmgr_topologies('mock_name1') + assert [topologies[3], topologies[4]] == tracker.get_stmgr_topologies('mock_name2') + +def test_add_new_topology(tracker): + assert [] == tracker.topologies + mock_state_manager_1 = Mock() + mock_state_manager_1.name = 'mock_name1' + + tracker.add_new_topology(mock_state_manager_1, 'top_name1') + assert ['top_name1'] == [t.name for t in tracker.topologies] + + tracker.add_new_topology(mock_state_manager_1, 'top_name2') + assert ['top_name1', 'top_name2'] == [t.name for t in tracker.topologies] + + assert 2 == mock_state_manager_1.get_pplan.call_count + assert 2 == mock_state_manager_1.get_execution_state.call_count + assert 2 == mock_state_manager_1.get_tmanager.call_count + +def test_remove_topology(tracker, topologies): + tracker.remove_topology('top_name1', 'mock_name1') + assert [topologies[1], topologies[2], topologies[3], topologies[4]] == tracker.topologies + tracker.remove_topology('top_name2', 'mock_name1') + assert [topologies[2], topologies[3], topologies[4]] == tracker.topologies + # Removing one that is not there should not have any affect + tracker.remove_topology('top_name8', 'mock_name1') + assert [topologies[2], topologies[3], topologies[4]] == tracker.topologies + tracker.remove_topology('top_name4', 'mock_name2') + assert [topologies[2], topologies[4]] == tracker.topologies diff --git a/heron/tools/ui/src/python/BUILD b/heron/tools/ui/src/python/BUILD index 29422a6b4db..c0c31fed10d 100644 --- a/heron/tools/ui/src/python/BUILD +++ b/heron/tools/ui/src/python/BUILD @@ -7,13 +7,13 @@ pex_library( exclude = ["main.py"], ), reqs = [ - "requests==2.12.3", + "requests==2.27.1", "click==7.1.2", "fastapi==0.60.1", "jinja2==2.11.2", "aiofiles==0.5.0", "uvicorn==0.11.7", - "uvloop==0.14.0", + "uvloop==0.16.0", ], deps = [ "//heron/common/src/python:common-py", diff --git a/heron/tools/ui/src/python/main.py b/heron/tools/ui/src/python/main.py index 8e345bced3d..6cee080ea4e 100644 --- a/heron/tools/ui/src/python/main.py +++ b/heron/tools/ui/src/python/main.py @@ -189,7 +189,6 @@ def metrics( return result[component] return result -# should factor out the tornado based access module query_handler = tracker.HeronQueryHandler() @topologies_router.get("/metrics/timeline") def timeline( diff --git a/heronpy/api/tests/python/BUILD b/heronpy/api/tests/python/BUILD index 7049c09c2a0..c47c2a82dbd 100644 --- a/heronpy/api/tests/python/BUILD +++ b/heronpy/api/tests/python/BUILD @@ -5,7 +5,7 @@ pex_pytest( size = "small", srcs = ["component_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heronpy/api:heron-python-py", @@ -17,7 +17,7 @@ pex_pytest( size = "small", srcs = ["stream_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heronpy/api:heron-python-py", @@ -29,7 +29,7 @@ pex_pytest( size = "small", srcs = ["topology_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heronpy/api:heron-python-py", @@ -41,7 +41,7 @@ pex_pytest( size = "small", srcs = ["serializer_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heronpy/api:heron-python-py", @@ -53,7 +53,7 @@ pex_pytest( size = "small", srcs = ["metrics_unittest.py"], reqs = [ - "pytest==3.2.2", + "pytest==6.1.2", ], deps = [ "//heronpy/api:heron-python-py", diff --git a/heronpy/proto/BUILD b/heronpy/proto/BUILD index ef9e679f09a..3579939ee32 100644 --- a/heronpy/proto/BUILD +++ b/heronpy/proto/BUILD @@ -27,8 +27,8 @@ pex_library( name = "proto-py", srcs = glob(["**/*.py"]), reqs = [ - "protobuf==3.8.0", - "setuptools==46.1.3", + "protobuf==3.14.0", + "setuptools==51.0.0", ], deps = [ ":proto_ckptmgr_py", @@ -50,8 +50,8 @@ pex_binary( name = "proto-py-package", srcs = glob(["**/*.py"]), reqs = [ - "protobuf==3.8.0", - "setuptools==18.8.1", + "protobuf==3.14.0", + "setuptools==51.0.0", ], deps = [ ":proto_ckptmgr_py", diff --git a/integration_test/src/python/http_server/BUILD b/integration_test/src/python/http_server/BUILD index 7833e0da3f8..eddce00fb52 100644 --- a/integration_test/src/python/http_server/BUILD +++ b/integration_test/src/python/http_server/BUILD @@ -7,7 +7,7 @@ pex_binary( ], main = "main.py", reqs = [ - "tornado==4.5.3", + "tornado==6.1", "werkzeug==2.0.2", ], deps = [ diff --git a/scripts/ci/README.md b/scripts/ci/README.md index c16cf14aabe..03cd84798e0 100644 --- a/scripts/ci/README.md +++ b/scripts/ci/README.md @@ -32,9 +32,9 @@ set -o pipefail # Install bazel (linux build) because CI hosts may not have it installed bash scripts/ci/setup_bazel.sh linux -# Build v0.20.1-incubating packages for centos7 and put in artifacts folder +# Build v0.20.1-incubating packages for centos8 and put in artifacts folder HERON_BUILD_USER=release-agent -bash scripts/ci/build_release_packages.sh v0.20.1-incubating centos7 artifacts +bash scripts/ci/build_release_packages.sh v0.20.1-incubating centos8 artifacts ``` @@ -72,6 +72,6 @@ bash scripts/ci/setup_bazel.sh linux # Build v0.20.1-incubating artifacts and put in artifacts folder HERON_BUILD_USER=release-agent -bash scripts/ci/build_docker_image.sh v0.20.1-incubating debian9 artifacts +bash scripts/ci/build_docker_image.sh v0.20.1-incubating 10 artifacts ``` diff --git a/scripts/ci/build_docker_image.sh b/scripts/ci/build_docker_image.sh index c43fa902d13..e4b45eb1dcb 100644 --- a/scripts/ci/build_docker_image.sh +++ b/scripts/ci/build_docker_image.sh @@ -19,7 +19,7 @@ # Build docker image to be released # parameters: # 1. version tag, e.g. v0.20.1-incubating -# 2. build os, e.g. debian9, centos7 +# 2. build os, e.g. debian10, centos8 # 3. output dir # Related environment variables diff --git a/scripts/ci/build_release_packages.sh b/scripts/ci/build_release_packages.sh index 1e38cf07ae0..7bca7d14a9f 100644 --- a/scripts/ci/build_release_packages.sh +++ b/scripts/ci/build_release_packages.sh @@ -19,7 +19,7 @@ # Build packages to be released # parameters: # 1. version tag, e.g. v0.20.1-incubating -# 2. build os, e.g. darwin, centos7 +# 2. build os, e.g. darwin, centos8 # 3. output dir # Related environment variables diff --git a/scripts/detect_os_type.sh b/scripts/detect_os_type.sh index 4462bee7289..63fb3ec452d 100755 --- a/scripts/detect_os_type.sh +++ b/scripts/detect_os_type.sh @@ -20,10 +20,7 @@ function platform { PLATFORM=darwin ;; linux*) - PLATFORM=ubuntu - if [ -f /etc/redhat-release ] ; then - PLATFORM=centos - fi + PLATFORM=linux ;; *) echo "WARNING: Your platform is not currently supported!" >&2 diff --git a/scripts/images/BUILD b/scripts/images/BUILD index d42013621cd..f5b136b9fb1 100644 --- a/scripts/images/BUILD +++ b/scripts/images/BUILD @@ -8,7 +8,7 @@ container_image( "-n", ], directory = "/heron", - stamp = 1, + stamp = "@io_bazel_rules_docker//stamp:always", symlinks = { "/usr/local/bin/heron": "/heron/heron-tools/bin/heron", "/usr/local/bin/heron-explorer": "/heron/heron-tools/bin/heron-explorer", diff --git a/scripts/release_check/README.md b/scripts/release_check/README.md index 868dfd91784..ab486195ffe 100644 --- a/scripts/release_check/README.md +++ b/scripts/release_check/README.md @@ -46,7 +46,7 @@ sh ./scripts/release_check/build.sh sh ./scripts/release_check/run_test_topology.sh ``` -### To compile source into a Heron docker image (host OS: MacOS, target OS: Debian9). +### To compile source into a Heron docker image (host OS: MacOS, target OS: Debian10). ``` sh ./scripts/release_check/build_docker.sh ``` diff --git a/scripts/release_check/build_docker.sh b/scripts/release_check/build_docker.sh index e2f15d14180..2bbfc65c2b0 100644 --- a/scripts/release_check/build_docker.sh +++ b/scripts/release_check/build_docker.sh @@ -18,7 +18,7 @@ set -o errexit -BUILD_OS=debian9 +BUILD_OS=debian10 VERSION_TAG=test_build TEMP_RELEASE_DIR=artifacts diff --git a/scripts/release_check/full_release_check.sh b/scripts/release_check/full_release_check.sh index cc279b5091b..fe8dff8476c 100644 --- a/scripts/release_check/full_release_check.sh +++ b/scripts/release_check/full_release_check.sh @@ -39,5 +39,5 @@ sh ./scripts/release_check/build.sh echo "Run a test topology locally..." sh ./scripts/release_check/run_test_topology.sh -echo "Build debian9 docker image..." +echo "Build debian10 docker image..." sh ./scripts/release_check/build_docker.sh diff --git a/scripts/shutils/common.sh b/scripts/shutils/common.sh index 3c4857bca7b..236ab73a895 100755 --- a/scripts/shutils/common.sh +++ b/scripts/shutils/common.sh @@ -92,18 +92,15 @@ function print_timer_summary { # Discover the platform that we are running on function discover_platform { - discover="${PLATFORM-$(python3 -mplatform)}" - if [[ $discover =~ ^.*centos.*$ ]]; then - echo "centos" - elif [[ $discover =~ ^.*Ubuntu.*$ ]]; then - echo "ubuntu" - elif [[ $discover =~ ^.*debian.*$ ]]; then - echo "debian" - elif [[ $discover =~ ^Darwin.*$ ]]; then + platform='unknown' + unamestr=$(uname) + if [[ "$unamestr" == 'Linux' ]]; then + echo "linux" + elif [[ "$unamestr" == 'Darwin' ]]; then echo "darwin" else - mysterious=`echo $discover | awk -F- '{print $6}'` - echo "$mysterious platform not supported" + mysterious=`echo $unamestr | awk -F- '{print $6}'` + echo "$unamestr platform not supported" exit 1 fi } diff --git a/tools/bazel.rc b/tools/bazel.rc index 6c625477283..bdb85374b07 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -22,33 +22,19 @@ build --ignore_unsupported_sandboxing build --spawn_strategy=standalone build --workspace_status_command scripts/release/status.sh -# For centos -# To use it: bazel build --config=centos -build:centos --experimental_action_listener=tools/cpp:compile_cpp -build:centos --experimental_action_listener=tools/java:compile_java -build:centos --experimental_action_listener=tools/python:compile_python -build:centos --genrule_strategy=standalone -build:centos --ignore_unsupported_sandboxing -build:centos --linkopt -lm -build:centos --linkopt -lpthread -build:centos --linkopt -lrt -build:centos --spawn_strategy=standalone -build:centos --workspace_status_command scripts/release/status.sh -build:centos --copt=-O3 - -# For debian -# To use it: bazel build --config=debian -build:debian --experimental_action_listener=tools/cpp:compile_cpp -build:debian --experimental_action_listener=tools/java:compile_java -build:debian --experimental_action_listener=tools/python:compile_python -build:debian --genrule_strategy=standalone -build:debian --ignore_unsupported_sandboxing -build:debian --linkopt -lm -build:debian --linkopt -lpthread -build:debian --linkopt -lrt -build:debian --spawn_strategy=standalone -build:debian --workspace_status_command scripts/release/status.sh -build:debian --copt=-O3 +# For Linux +# To use it: bazel build --config=linux +build:linux --experimental_action_listener=tools/cpp:compile_cpp +build:linux --experimental_action_listener=tools/java:compile_java +build:linux --experimental_action_listener=tools/python:compile_python +build:linux --genrule_strategy=standalone +build:linux --ignore_unsupported_sandboxing +build:linux --linkopt -lm +build:linux --linkopt -lpthread +build:linux --linkopt -lrt +build:linux --spawn_strategy=standalone +build:linux --workspace_status_command scripts/release/status.sh +build:linux --copt=-O3 # For Mac # To use it: bazel build --config=darwin @@ -61,43 +47,18 @@ build:darwin --spawn_strategy=standalone build:darwin --workspace_status_command scripts/release/status.sh build:darwin --copt=-O3 -# For Ubuntu -# To use it: bazel build --config=ubuntu -build:ubuntu --experimental_action_listener=tools/java:compile_java -build:ubuntu --experimental_action_listener=tools/cpp:compile_cpp -build:ubuntu --experimental_action_listener=tools/python:compile_python -build:ubuntu --genrule_strategy=standalone -build:ubuntu --ignore_unsupported_sandboxing -build:ubuntu --linkopt -lm -build:ubuntu --linkopt -lpthread -build:ubuntu --linkopt -lrt -build:ubuntu --spawn_strategy=standalone -build:ubuntu --workspace_status_command scripts/release/status.sh -build:ubuntu --copt=-O3 - ### Disabled checkstyle -# For centos -# To use it: bazel build --config=centos_nostyle -build:centos_nostyle --genrule_strategy=standalone -build:centos_nostyle --ignore_unsupported_sandboxing -build:centos_nostyle --linkopt -lm -build:centos_nostyle --linkopt -lpthread -build:centos_nostyle --linkopt -lrt -build:centos_nostyle --spawn_strategy=standalone -build:centos_nostyle --workspace_status_command scripts/release/status.sh -build:centos_nostyle --copt=-O3 - -# For debian -# To use it: bazel build --config=debian_nostyle -build:debian_nostyle --genrule_strategy=standalone -build:debian_nostyle --ignore_unsupported_sandboxing -build:debian_nostyle --linkopt -lm -build:debian_nostyle --linkopt -lpthread -build:debian_nostyle --linkopt -lrt -build:debian_nostyle --spawn_strategy=standalone -build:debian_nostyle --workspace_status_command scripts/release/status.sh -build:debian_nostyle --copt=-O3 +# For Linux +# To use it: bazel build --config=linux_nostyle +build:linux_nostyle --genrule_strategy=standalone +build:linux_nostyle --ignore_unsupported_sandboxing +build:linux_nostyle --linkopt -lm +build:linux_nostyle --linkopt -lpthread +build:linux_nostyle --linkopt -lrt +build:linux_nostyle --spawn_strategy=standalone +build:linux_nostyle --workspace_status_command scripts/release/status.sh +build:linux_nostyle --copt=-O3 # For Mac # To use it: bazel build --config=darwin_nostyle @@ -106,14 +67,3 @@ build:darwin_nostyle --ignore_unsupported_sandboxing build:darwin_nostyle --spawn_strategy=standalone build:darwin_nostyle --workspace_status_command scripts/release/status.sh build:darwin_nostyle --copt=-O3 - -# For Ubuntu -# To use it: bazel build --config=ubuntu_nostyle -build:ubuntu_nostyle --genrule_strategy=standalone -build:ubuntu_nostyle --ignore_unsupported_sandboxing -build:ubuntu_nostyle --linkopt -lm -build:ubuntu_nostyle --linkopt -lpthread -build:ubuntu_nostyle --linkopt -lrt -build:ubuntu_nostyle --spawn_strategy=standalone -build:ubuntu_nostyle --workspace_status_command scripts/release/status.sh -build:ubuntu_nostyle --copt=-O3 diff --git a/tools/docker/bazel.rc b/tools/docker/bazel.rc index 05a67d78cde..ef31f17d34e 100644 --- a/tools/docker/bazel.rc +++ b/tools/docker/bazel.rc @@ -30,7 +30,7 @@ test --test_strategy=standalone # Bazel doesn't calculate the memory ceiling correctly when running under Docker. # Limit Bazel to consuming 4G ram and 2 cores. build --local_ram_resources=4096 -build --local_cpu_resources=2 +# build --local_cpu_resources=2 # Echo all the configuration settings and their source build --announce_rc diff --git a/tools/rules/pex/BUILD b/tools/rules/pex/BUILD index 58d3d3e06e4..b2aa361a7f5 100644 --- a/tools/rules/pex/BUILD +++ b/tools/rules/pex/BUILD @@ -31,9 +31,9 @@ POST_EXECUTE = [ 'TEMP="$(@D)/pexbuild"', 'pip install pex \ --quiet --no-cache-dir --no-index \ - --find-links $$(dirname $(location @pex_src//file)) \ - --find-links $$(dirname $(location @wheel_src//file)) \ - --find-links $$(dirname $(location @setuptools_wheel//file))', + --find-links $$(dirname $(location @pex_pkg//file)) \ + --find-links $$(dirname $(location @wheel_pkg//file)) \ + --find-links $$(dirname $(location @setuptools_pkg//file))', '# Work around setuptools insistance on writing to the source directory,', '# which is discouraged by Bazel (and annoying)', @@ -44,10 +44,14 @@ POST_EXECUTE = [ --disable-cache --no-index \ --entry-point=pex_wrapper \ --output-file=$@ \ - --find-links $$(dirname $(location @pex_src//file)) \ - --find-links $$(dirname $(location @setuptools_wheel//file)) \ - --find-links $$(dirname $(location @requests_src//file)) \ - --find-links $$(dirname $(location @wheel_src//file))', + --find-links $$(dirname $(location @pex_pkg//file)) \ + --find-links $$(dirname $(location @setuptools_pkg//file)) \ + --find-links $$(dirname $(location @requests_pkg//file)) \ + --find-links $$(dirname $(location @charset_pkg//file)) \ + --find-links $$(dirname $(location @idna_pkg//file)) \ + --find-links $$(dirname $(location @urllib3_pkg//file)) \ + --find-links $$(dirname $(location @certifi_pkg//file)) \ + --find-links $$(dirname $(location @wheel_pkg//file))', ] genrule( @@ -56,10 +60,14 @@ genrule( "wrapper/setup.py", "wrapper/pex_wrapper.py", "wrapper/README", - "@setuptools_wheel//file", - "@wheel_src//file", - "@pex_src//file", - "@requests_src//file", + "@setuptools_pkg//file", + "@wheel_pkg//file", + "@pex_pkg//file", + "@requests_pkg//file", + "@charset_pkg//file", + "@idna_pkg//file", + "@urllib3_pkg//file", + "@certifi_pkg//file", ], outs = ["pex_wrapper.pex"], cmd = select({ diff --git a/tools/rules/pex/pex_rules.bzl b/tools/rules/pex/pex_rules.bzl index d93fb609939..29815fedc79 100644 --- a/tools/rules/pex/pex_rules.bzl +++ b/tools/rules/pex/pex_rules.bzl @@ -455,8 +455,7 @@ def pex_pytest( deps = deps, data = data, eggs = eggs + [ - "@pytest_whl//file", - "@py_whl//file", + "@pytest_pkg//file", ], entrypoint = "pytest", testonly = True, diff --git a/vagrant/init.sh b/vagrant/init.sh index 18754bac9e1..963ac255627 100644 --- a/vagrant/init.sh +++ b/vagrant/init.sh @@ -59,7 +59,7 @@ build_heron() { pushd /vagrant bazel clean ./bazel_configure.py - bazel --bazelrc=tools/travis/bazel.rc build --config=ubuntu heron/... + bazel --bazelrc=tools/travis/bazel.rc build --config=linux heron/... popd } diff --git a/website2/docs/compiling-docker.md b/website2/docs/compiling-docker.md index 409325b00dc..f6d63452b59 100644 --- a/website2/docs/compiling-docker.md +++ b/website2/docs/compiling-docker.md @@ -24,7 +24,7 @@ For developing Heron, you will need to compile it for the environment that you want to use it in. If you'd like to use Docker to create that build environment, Heron provides a convenient script to make that process easier. -Currently Debian10 and Ubuntu 18.04 are actively being supported. There is also limited support for Ubuntu 14.04, Debian9, and CentOS 7. If you +Currently Debian10 and Ubuntu 20.04 are actively being supported. There is also limited support for Ubuntu 18.04, and CentOS 8. If you need another platform there are instructions for adding new ones [below](#contributing-new-environments). @@ -65,13 +65,13 @@ Script to build heron docker image for different platforms Usage: ./docker/scripts/build-docker.sh [-s|--squash] Argument options: - : darwin, debian9, debian10, ubuntu14.04, ubuntu18.04, centos7 + : darwin, debian10, ubuntu20.04, centos8 : Version of Heron build, e.g. v0.17.5.1-rc : Location of compiled Heron artifact [-s|--squash]: Enables using Docker experimental feature --squash Example: - ./build-docker.sh ubuntu18.04 0.12.0 ~/ubuntu + ./build-docker.sh ubuntu20.04 0.12.0 ~/ubuntu NOTE: If running on OSX, the output directory will need to be under /Users so virtualbox has access to. @@ -79,15 +79,14 @@ NOTE: If running on OSX, the output directory will need to The following arguments are required: -* `platform` --- Currently we are focused on supporting the `debian10` and `ubuntu18.04` platforms. +* `platform` --- Currently we are focused on supporting the `debian10` and `ubuntu20.04` platforms. We also support building Heron locally on OSX. You can specify this as listing `darwin` as the platform. All options are: - - `centos7` + - `centos8` - `darwin` - - `debian9` - `debian10` - - `ubuntu14.04` - `ubuntu18.04` + - `ubuntu20.04` You can add other platforms using the [instructions @@ -148,8 +147,8 @@ After the commands, a new docker container is started with all the libraries and installed. The operation system is Ubuntu 18.04 by default. Now you can build Heron like: ```bash -\# bazel build --config=debian scripts/packages:binpkgs -\# bazel build --config=debian scripts/packages:tarpkgs +\# bazel build --config=linux scripts/packages:binpkgs +\# bazel build --config=linux scripts/packages:tarpkgs ``` The current folder is mapped to the '/heron' directory in the container and any changes @@ -185,7 +184,7 @@ following: Here's an example: ```dockerfile -FROM centos:centos7 +FROM centos:centos8 ``` ### Step 2 --- A `TARGET_PLATFORM` environment variable using the [`ENV`](https://docs.docker.com/engine/reference/builder/#env) instruction. diff --git a/website2/docs/compiling-linux.md b/website2/docs/compiling-linux.md index b3ce9332bad..cfc7665771c 100644 --- a/website2/docs/compiling-linux.md +++ b/website2/docs/compiling-linux.md @@ -96,14 +96,14 @@ $ ./bazel_configure.py ### Step 10 --- Build the project ```bash -$ bazel build --config=ubuntu heron/... +$ bazel build --config=linux heron/... ``` ### Step 11 --- Build the packages ```bash -$ bazel build --config=ubuntu scripts/packages:binpkgs -$ bazel build --config=ubuntu scripts/packages:tarpkgs +$ bazel build --config=linux scripts/packages:binpkgs +$ bazel build --config=linux scripts/packages:tarpkgs ``` This will install Heron packages in the `bazel-bin/scripts/packages/` directory. @@ -198,14 +198,14 @@ bazelVersion %}}). ```bash $ git clone https://github.com/apache/incubator-heron.git && cd heron $ ./bazel_configure.py -$ bazel build --config=centos heron/... +$ bazel build --config=linux heron/... ``` ### Step 7 --- Build the binary packages ```bash -$ bazel build --config=centos scripts/packages:binpkgs -$ bazel build --config=centos scripts/packages:tarpkgs +$ bazel build --config=linux scripts/packages:binpkgs +$ bazel build --config=linux scripts/packages:tarpkgs ``` This will install Heron packages in the `bazel-bin/scripts/packages/` directory. diff --git a/website2/docs/getting-started-local-single-node.md b/website2/docs/getting-started-local-single-node.md index 6e0c1535bd5..4c56c538bac 100644 --- a/website2/docs/getting-started-local-single-node.md +++ b/website2/docs/getting-started-local-single-node.md @@ -28,7 +28,7 @@ For other platforms, you need to build from source. Please refer to the [guide t Heron tools can be installed using [installation scripts](#using-installation-scripts). > Note: As of version 0.20.4-incubating, there is a python compatibility on OSX. -> The supported platforms are CentOS7, Debian10, and Ubuntu18.04. +> The supported platforms are centos8, Debian10, and Ubuntu18.04. ## Using installation scripts diff --git a/website2/docs/user-manuals-tracker-rest.md b/website2/docs/user-manuals-tracker-rest.md index 6055592862a..7261de0ca6f 100644 --- a/website2/docs/user-manuals-tracker-rest.md +++ b/website2/docs/user-manuals-tracker-rest.md @@ -20,646 +20,10 @@ sidebar_label: Heron Tracker REST API under the License. --> -### JSON Interface - -All Heron Tracker endpoints return a JSON object with the following information: - -* `status` --- One of the following: `success`, `failure`. -* `executiontime` --- The time taken to return the HTTP result, in seconds. -* `message` --- Some endpoints return special messages in this field for certain - requests. Often, this field will be an empty string. A `failure` status will - always have a message. -* `result` --- The result payload of the request. The contents will depend on - the endpoint. -* `version` --- The Tracker API version. - -### Endpoints - -* `/` (redirects to `/topologies`) -* [`/clusters`](#clusters) -* [`/topologies`](#topologies) -* [`/topologies/states`](#topologies_states) -* [`/topologies/info`](#topologies_info) -* [`/topologies/logicalplan`](#topologies_logicalplan) -* [`/topologies/physicalplan`](#topologies_physicalplan) -* [`/topologies/executionstate`](#topologies_executionstate) -* [`/topologies/schedulerlocation`](#topologies_schedulerlocation) -* [`/topologies/metrics`](#topologies_metrics) -* [`/topologies/metricstimeline`](#topologies_metricstimeline) -* [`/topologies/metricsquery`](#topologies_metricsquery) -* [`/topologies/containerfiledata`](#topologies_containerfiledata) -* [`/topologies/containerfilestats`](#topologies_containerfilestats) -* [`/topologies/exceptions`](#topologies_exceptions) -* [`/topologies/exceptionsummary`](#topologies_exceptionsummary) -* [`/topologies/pid`](#topologies_pid) -* [`/topologies/jstack`](#topologies_jstack) -* [`/topologies/jmap`](#topologies_jmap) -* [`/topologies/histo`](#topologies_histo) -* [`/machines`](#machines) - -All of these endpoints are documented in the sections below. - ---- - -### /clusters - -Returns JSON list of all the clusters. - ---- - -### /topologies - -Returns JSON describing all currently available topologies - -```bash -$ curl "http://heron-tracker-url/topologies?cluster=cluster1&environ=devel" -``` - -#### Parameters - -* `cluster` (optional) --- The cluster parameter can be used to filter - topologies that are running in this cluster. -* `environ` (optional) --- The environment parameter can be used to filter - topologies that are running in this environment. - ---- - -### /topologies/logicalplan - -Returns a JSON representation of the [logical plan](heron-topology-concepts#logical-plan) of a topology. - -```bash -$ curl "http://heron-tracker-url/topologies/logicalplan?cluster=cluster1&environ=devel&topology=topologyName" +### Heron Tracker REST API +The API's documentation is within the application and is served +automatically when started. Example of accessing documentation: +```shell +# OpenAPI will be available at http://localhost:8080/ +heron-tracker --port=8080 ``` - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology - -The resulting JSON contains the following - -* `spouts` --- A set of JSON objects representing each spout in the topology. - The following information is listed for each spout: - * `source` --- The source of tuples for the spout. - * `type` --- The type of the spout, e.g. `kafka`, `kestrel`, etc. - * `outputs` --- A list of streams to which the spout outputs tuples. -* `bolts` --- A set of JSON objects representing each bolt in the topology. - * `outputs` --- A list of streams to which the bolt outputs tuples. - * `inputs` --- A list of inputs for the bolt. An input is represented by - JSON dictionary containing following information. - * `component_name` --- Name of the component this bolt is receiving tuples from. - * `stream_name` --- Name of the stream from which the tuples are received. - * `grouping` --- Type of grouping used to receive tuples, example `SHUFFLE` or `FIELDS`. - ---- - -### /topologies/physicalplan - -Returns a JSON representation of the [physical plan](heron-topology-concepts#physical-plan) of a topology. - -```bash -$ curl "http://heron-tracker-url/topologies/physicalplan?cluster=datacenter1&environ=prod&topology=topologyName" -``` - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology - -The resulting JSON contains following information - -* All spout and bolt components, with lists of their instances. -* `stmgrs` --- A list of JSON dictionary, containing following information of each stream manager. - * `host` --- Hostname of the machine this container is running on. - * `pid` --- Process ID of the stream manager. - * `cwd` --- Absolute path to the directory from where container was launched. - * `joburl` --- URL to browse the `cwd` through `heron-shell`. - * `shell_port` --- Port to access `heron-shell`. - * `logfiles` --- URL to browse instance log files through `heron-shell`. - * `id` --- ID for this stream manager. - * `port` --- Port at which this stream manager accepts connections from other stream managers. - * `instance_ids` --- List of instance IDs that constitute this container. -* `instances` --- A list of JSON dictionaries containing following information for each instance - * `id` --- Instance ID. - * `name` --- Component name of this instance. - * `logfile` --- Link to log file for this instance, that can be read through `heron-shell`. - * `stmgrId` --- Its stream manager's ID. -* `config` --- Various topology configs. Some of the examples are: - * `topology.message.timeout.secs` --- Time after which a tuple should be considered as failed. - * `topology.acking` --- Whether acking is enabled or not. - ---- - -### /topologies/schedulerlocation - -Returns a JSON representation of the scheduler location of the topology. - -```bash -$ curl "http://heron-tracker-url/topologies/schedulerlocation?cluster=datacenter1&environ=prod&topology=topologyName" -``` - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology - -The `SchedulerLocation` mainly contains the link to the job on the scheduler, -for example, the Aurora page for the job. - ---- - -### /topologies/executionstate - -Returns a JSON representation of the execution state of the topology. - -```bash -$ curl "http://heron-tracker-url/topologies/executionstate?cluster=datacenter1&environ=prod&topology=topologyName" -``` - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology - -Each execution state object lists the following: - -* `cluster` --- The cluster in which the topology is running -* `environ` --- The environment in which the topology is running -* `role` --- The role with which the topology was launched -* `jobname` --- Same as topology name -* `submission_time` --- The time at which the topology was submitted -* `submission_user` --- The user that submitted the topology (can be same as `role`) -* `release_username` --- The user that generated the Heron release for the - topology -* `release_version` --- Release version -* `has_physical_plan` --- Whether the topology has a physical plan -* `has_tmanager_location` --- Whether the topology has a Topology Manager Location -* `has_scheduler_location` --- Whether the topology has a Scheduler Location -* `viz` --- Metric visualization UI URL for the topology if it was [configured](user-manuals-heron-tracker-runbook) - ---- - -### /topologies/states - -Returns a JSON list of execution states of topologies in all the cluster. - -```bash -$ curl "http://heron-tracker-url/topologies/states?cluster=cluster1&environ=devel" -``` - -#### Parameters - -* `cluster` (optional) --- The cluster parameter can be used to filter - topologies that are running in this cluster. -* `environ` (optional) --- The environment parameter can be used to filter - topologies that are running in this environment. - ---- - -### /topologies/info - -Returns a JSON representation of a dictionary containing logical plan, physical plan, -execution state, scheduler location and TManager location for a topology, as described above. -`TManagerLocation` is the location of the TManager, including its host, -port, and the heron-shell port that it exposes. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology - ---- - -### /topologies/containerfilestats - -Returns the file stats for a container. This is the output of the command `ls -lh` when run -in the directory where the heron-controller launched all the processes. - -This endpoint is mainly used by ui for exploring files in a container. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `container` (required) --- Container ID -* `path` (optional) --- Path relative to the directory where heron-controller is launched. - Paths are not allowed to start with a `/` or contain a `..`. - ---- - -### /topologies/containerfiledata - -Returns the file data for a file of a container. - -This endpoint is mainly used by ui for exploring files in a container. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `container` (required) --- Container ID -* `path` (required) --- Path to the file relative to the directory where heron-controller is launched. - Paths are not allowed to start with a `/` or contain a `..`. -* `offset` (required) --- Offset from the beggining of the file. -* `length` (required) --- Number of bytes to be returned. - ---- - -### /topologies/metrics - -Returns a JSON map of instances of the topology to their respective metrics. -To filter instances returned use the `instance` parameter discussed below. - - -Note that these metrics come from TManager, which only holds metrics -for last 3 hours minutely data, as well as cumulative values. If the `interval` -is greater than `10800` seconds, the values will be for all-time metrics. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `component` (required) --- Component name -* `metricname` (required, repeated) --- Names of metrics to fetch -* `interval` (optional) --- For how many seconds, the metrics should be fetched for (max 10800 seconds) -* `instance` (optional) --- IDs of the instances. If not present, return for all the instances. - ---- - -### /topologies/metricstimeline - -Returns a JSON map of instances of the topology to their respective metrics timeline. -To filter instances returned use the `instance` parameter discussed below. - -The difference between this and `/metrics` endpoint above, is that `/metrics` will report -cumulative value over the period of `interval` provided. On the other hand, `/metricstimeline` -endpoint will report minutely values for each metricname for each instance. - -Note that these metrics come from TManager, which only holds metrics -for last 3 hours minutely data, as well as cumulative all-time values. If the starttime -is older than 3 hours ago, those minutes would not be part of the response. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `component` (required) --- Component name -* `metricname` (required, repeated) --- Names of metrics to fetch -* `starttime` (required) --- Start time for the metrics (must be within last 3 hours) -* `endtime` (required) --- End time for the metrics (must be within last 3 hours, - and greater than `starttime`) -* `instance` (optional) --- IDs of the instances. If not present, return for all the instances. - -### /topologies/metricsquery - -Executes the metrics query for the topology and returns the result in form of minutely timeseries. -A detailed description of query language is given [below](#metricsquery). - -Note that these metrics come from TManager, which only holds metrics -for last 3 hours minutely data, as well as cumulative all-time values. If the starttime -is older than 3 hours ago, those minutes would not be part of the response. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `starttime` (required) --- Start time for the metrics (must be within last 3 hours) -* `endtime` (required) --- End time for the metrics (must be within last 3 hours, - and greater than `starttime`) -* `query` (required) --- Query to be executed - ---- - -### /topologies/exceptionsummary - -Returns summary of the exceptions for the component of the topology. -Duplicated exceptions are combined together and includes the number of -occurances, first occurance time and latest occurance time. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `component` (required) --- Component name -* `instance` (optional) --- IDs of the instances. If not present, return for all the instances. - ---- - -### /topologies/exceptions - -Returns all exceptions for the component of the topology. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `component` (required) --- Component name -* `instance` (optional) --- IDs of the instances. If not present, return for all the instances. - ---- - -### /topologies/pid - -Returns the PID of the instance JVM process. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `instance` (required) --- Instance ID - ---- - -### /topologies/jstack - -Returns the thread dump of the instance JVM process. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `instance` (required) --- Instance ID - ---- - -### /topologies/jmap - -Issues the `jmap` command for the instance, and saves the result in a file. -Returns the path to the file that can be downloaded externally. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `instance` (required) --- Instance ID - ---- - -### /topologies/histo - -Returns histogram for the instance JVM process. - -#### Parameters - -* `cluster` (required) --- The cluster in which the topology is running -* `environ` (required) --- The environment in which the topology is running -* `topology` (required) --- The name of the topology -* `instance` (required) --- Instance ID - ---- - -### /machines - -Returns JSON describing all machines that topologies are running on. - -```bash -$ curl "http://heron-tracker-url/machines?topology=mytopology1&cluster=cluster1&environ=prod" -``` - -#### Parameters - -* `cluster` (optional) --- The cluster parameter can be used to filter - machines that are running the topologies in this cluster only. -* `environ` (optional) --- The environment parameter can be used to filter - machines that are running the topologies in this environment only. -* `topology` (optional, repeated) --- Name of the topology. Both `cluster` - and `environ` are required if the `topology` parameter is present - ---- - -### Metrics Query Language - -Metrics queries are useful when some kind of aggregated values are required. For example, -to find the total number of tuples emitted by a spout, `SUM` operator can be used, instead -of fetching metrics for all the instances of the corresponding component, and then summing them. - -#### Terminology - -1. Univariate Timeseries --- A timeseries is called univariate if there is only one set -of minutely data. For example, a timeseries representing the sums of a number of timeseries -would be a univariate timeseries. -2. Multivariate Timeseries --- A set of multiple timeseries is collectively called multivariate. -Note that these timeseries are associated with their instances. - -#### Operators - -##### TS - -```text -TS(componentName, instance, metricName) -``` - -Example: - -```text -TS(component1, *, __emit-count/stream1) -``` - -Time Series Operator. This is the basic operator that is responsible for getting metrics from TManager. -Accepts a list of 3 elements: - -1. componentName -2. instance - can be "*" for all instances, or a single instance ID -3. metricName - Full metric name with stream id if applicable - -Returns a univariate time series in case of a single instance id given, otherwise returns -a multivariate time series. - ---- - -##### DEFAULT - -```text -DEFAULT(0, TS(component1, *, __emit-count/stream1)) -``` -If the second operator returns more than one timeline, so will the -DEFAULT operator. - -```text -DEFAULT(100.0, SUM(TS(component2, *, __emit-count/default))) <-- -``` -Second operator can be any operator - -Default Operator. This operator is responsible for filling missing values in the metrics timeline. -Must have 2 arguments - -1. First argument is a numeric constant representing the number to fill the missing values with -2. Second one must be one of the operators, that return the metrics timeline - -Returns a univariate or multivariate time series, based on what the second operator is. - ---- - -##### SUM - -```text -SUM(TS(component1, instance1, metric1), DEFAULT(0, TS(component1, *, metric2))) -``` - -Sum Operator. This operator is used to take sum of all argument time series. It can have any number of arguments, -each of which must be one of the following two types: - -1. Numeric constants, which will fill in the missing values as well, or -2. Operator, which returns one or more timelines - -Returns only a single timeline representing the sum of all time series for each timestamp. -Note that "instance" attribute is not there in the result. - ---- - -##### MAX - -```text -MAX(100, TS(component1, *, metric1)) -``` - -Max Operator. This operator is used to find max of all argument operators for each individual timestamp. -Each argument must be one of the following types: - -1. Numeric constants, which will fill in the missing values as well, or -2. Operator, which returns one or more timelines - -Returns only a single timeline representing the max of all the time series for each timestamp. -Note that "instance" attribute is not included in the result. - ---- - -##### PERCENTILE - -```text -PERCENTILE(99, TS(component1, *, metric1)) -``` - -Percentile Operator. This operator is used to find a quantile of all timelines retuned by the arguments, for each timestamp. -This is a more general type of query similar to MAX. Note that `PERCENTILE(100, TS...)` is equivalent to `Max(TS...)`. -Each argument must be either constant or Operators. -First argument must always be the required Quantile. - -1. Quantile (first argument) - Required quantile. 100 percentile = max, 0 percentile = min. -2. Numeric constants will fill in the missing values as well, -3. Operator - which returns one or more timelines - -Returns only a single timeline representing the quantile of all the time series -for each timestamp. Note that "instance" attribute is not there in the result. - ---- - -##### DIVIDE - -```text -DIVIDE(TS(component1, *, metrics1), 100) -``` - -Divide Operator. Accepts two arguments, both can be univariate or multivariate. -Each can be of one of the following types: - -1. Numeric constant will be considered as a constant time series for all applicable timestamps, they will not fill the missing values -2. Operator - returns one or more timelines - -Three main cases are: - -1. When both operands are multivariate - 1. Divide operation will be done on matching data, that is, with same instance id. - 2. If the instances in both the operands do not match, error is thrown. - 3. Returns multivariate time series, each representing the result of division on the two corresponding time series. -2. When one operand is univariate, and other is multivariate - 1. This includes division by constants as well. - 2. The univariate operand will participate with all time series in multivariate. - 3. The instance information of the multivariate time series will be preserved in the result. - 4. Returns multivariate time series. -3. When both operands are univariate - 1. Instance information is ignored in this case - 2. Returns univariate time series which is the result of division operation. - ---- - -##### MULTIPLY - -```text -MULTIPLY(10, TS(component1, *, metrics1)) -``` - -Multiply Operator. Has same conditions as division operator. This is to keep the API simple. -Accepts two arguments, both can be univariate or multivariate. Each can be of one of the following types: - -1. Numeric constant will be considered as a constant time series for all applicable timestamps, they will not fill the missing values -2. Operator - returns one or more timelines - -Three main cases are: - -1. When both operands are multivariate - 1. Multiply operation will be done on matching data, that is, with same instance id. - 2. If the instances in both the operands do not match, error is thrown. - 3. Returns multivariate time series, each representing the result of multiplication - on the two corresponding time series. -2. When one operand is univariate, and other is multivariate - 1. This includes multiplication by constants as well. - 2. The univariate operand will participate with all time series in multivariate. - 3. The instance information of the multivariate time series will be preserved in the result. - 4. Returns multivariate timeseries. -3. When both operands are univariate - 1. Instance information is ignored in this case - 2. Returns univariate timeseries which is the result of multiplication operation. - ---- - -##### SUBTRACT - -```text -SUBTRACT(TS(component1, instance1, metrics1), TS(componet1, instance1, metrics2)) - -SUBTRACT(TS(component1, instance1, metrics1), 100) -``` - -Subtract Operator. Has same conditions as division operator. This is to keep the API simple. -Accepts two arguments, both can be univariate or multivariate. Each can be of one of the following types: - -1. Numeric constant will be considered as a constant time series for all applicable timestamps, they will not fill the missing values -2. Operator - returns one or more timelines - -Three main cases are: - -1. When both operands are multivariate - 1. Subtract operation will be done on matching data, that is, with same instance id. - 2. If the instances in both the operands do not match, error is thrown. - 3. Returns multivariate time series, each representing the result of subtraction - on the two corresponding time series. -2. When one operand is univariate, and other is multivariate - 1. This includes subtraction by constants as well. - 2. The univariate operand will participate with all time series in multivariate. - 3. The instance information of the multivariate time series will be preserved in the result. - 4. Returns multivariate time series. -3. When both operands are univariate - 1. Instance information is ignored in this case - 2. Returns univariate time series which is the result of subtraction operation. - ---- - -##### RATE - -```text -RATE(SUM(TS(component1, *, metrics1))) - -RATE(TS(component1, *, metrics2)) -``` - -Rate Operator. This operator is used to find rate of change for all timeseries. -Accepts a only a single argument, which must be an Operators which returns univariate or multivariate time series. -Returns univariate or multivariate time series based on the argument, with each timestamp value -corresponding to the rate of change for that timestamp. diff --git a/website2/website/pages/en/download.js b/website2/website/pages/en/download.js index af29c6682a3..019da050f63 100644 --- a/website2/website/pages/en/download.js +++ b/website2/website/pages/en/download.js @@ -61,8 +61,8 @@ class Download extends React.Component { const latestDebian10TarUrl = getTarUrl(latestHeronVersion, "debian10"); const latestArchiveUrl = distUrl(latestHeronVersion, 'bin'); const latestSrcArchiveUrl = distUrl(latestHeronVersion, 'src') - const centos7InstallUrl = getInstallScriptMirrorUrl(latestHeronVersion, "centos7") - const centos7InstallCryptoUrl = getInstallScriptCryptoUrl(latestHeronVersion, "centos7") + const centos8InstallUrl = getInstallScriptMirrorUrl(latestHeronVersion, "centos8") + const centos8InstallCryptoUrl = getInstallScriptCryptoUrl(latestHeronVersion, "centos8") const debian10InstallUrl = getInstallScriptMirrorUrl(latestHeronVersion, "debian10") const debian10InstallCryptoUrl = getInstallScriptCryptoUrl(latestHeronVersion, "debian10") const ubuntu1804InstallUrl = getInstallScriptMirrorUrl(latestHeronVersion, "ubuntu18.04") @@ -149,13 +149,13 @@ class Download extends React.Component { - CentOS7 + centos8 - heron-install-0.20.4-incubating-centos7.sh + heron-install-0.20.4-incubating-centos8.sh - asc,  - sha512 + asc,  + sha512 diff --git a/website2/website/scripts/python-doc-gen.sh b/website2/website/scripts/python-doc-gen.sh index 09f9a27d304..506abf370a7 100755 --- a/website2/website/scripts/python-doc-gen.sh +++ b/website2/website/scripts/python-doc-gen.sh @@ -25,7 +25,7 @@ cd ${HERON_ROOT_DIR} ./bazel_configure.py # Generate python whl packages, packages will be generated in ${HERON_ROOT_DIR}/bazel-bin/scripts/packages/ -bazel build --config=ubuntu scripts/packages:pypkgs +bazel build --config=linux_nostyle scripts/packages:pypkgs cd website2/website/ mkdir -p ./tmp/