From 289a8e8ee94f62a5c48563a206952dde6cfe0ac7 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:47:34 -0800 Subject: [PATCH 01/23] fix: fetch_job must return a counterfactual to failure --- gantry/routes/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gantry/routes/collection.py b/gantry/routes/collection.py index f8fd695..a5690b1 100644 --- a/gantry/routes/collection.py +++ b/gantry/routes/collection.py @@ -70,7 +70,7 @@ async def fetch_job( logger.error(f"{e} job={job.gl_id}") return - await db.insert_job( + job_id = await db.insert_job( db_conn, { "node": node_id, @@ -89,7 +89,7 @@ async def fetch_job( # we don't accidentally commit a node without a job await db_conn.commit() - return + return job_id async def fetch_node( From 0a674ed6e9614828d2be64ecca631544ab2a08fb Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:53:45 -0800 Subject: [PATCH 02/23] fix: remove ghost job db check they were not being inserted anyways --- gantry/db/get.py | 12 ------------ gantry/routes/collection.py | 2 +- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/gantry/db/get.py b/gantry/db/get.py index 362829c..735f1a8 100644 --- a/gantry/db/get.py +++ b/gantry/db/get.py @@ -26,15 +26,3 @@ async def job_exists(db: aiosqlite.Connection, gl_id: int) -> bool: return True return False - - -async def ghost_exists(db: aiosqlite.Connection, gl_id: int) -> bool: - """return if the ghost job exists in the database""" - - async with db.execute( - "select id from ghost_jobs where gitlab_id = ?", (gl_id,) - ) as cursor: - if await cursor.fetchone(): - return True - - return False diff --git a/gantry/routes/collection.py b/gantry/routes/collection.py index a5690b1..5724f9e 100644 --- a/gantry/routes/collection.py +++ b/gantry/routes/collection.py @@ -48,7 +48,6 @@ async def fetch_job( # uo runners are not in Prometheus or payload["runner"]["description"].startswith("uo") or await db.job_exists(db_conn, job.gl_id) # job already in the database - or await db.ghost_exists(db_conn, job.gl_id) # ghost already in db ): return @@ -56,6 +55,7 @@ async def fetch_job( job_log = await gitlab.job_log(job.gl_id) is_ghost = "No need to rebuild" in job_log if is_ghost: + logging.warning(f"job {job.gl_id} is a ghost, skipping") return try: From bffb3744e1f37f1db4804387689949b4b133caf0 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:57:19 -0800 Subject: [PATCH 03/23] fix: database insertion logic previously, we would rely on `lastrowid` to provide the pk of the inserted row, but since we're using `INSERT OR IGNORE`, we ran the risk of reusing an old `lastrowid` entry. Now, we will only use it if we've confirmed a row was inserted due to the statement. --- gantry/db/insert.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/gantry/db/insert.py b/gantry/db/insert.py index 7564ad9..2829256 100644 --- a/gantry/db/insert.py +++ b/gantry/db/insert.py @@ -1,3 +1,5 @@ +import logging + import aiosqlite from gantry.db.get import get_node @@ -35,15 +37,20 @@ async def insert_node(db: aiosqlite.Connection, node: dict) -> int: "nodes", node, # deal with races + # this also ignores the not-null constraint + # so we need to make sure the node is valid before inserting ignore=True, ) ) as cursor: - pk = cursor.lastrowid + # this check ensures that something was inserted + # and not relying on lastrowid, which could be anything + if cursor.rowcount > 0: + pk = cursor.lastrowid + + pk = await get_node(db, node["uuid"]) - if pk == 0: - # the ignore part of the query was triggered, some other call - # must have inserted the node before this one - pk = await get_node(db, node["uuid"]) + if pk is None: + logging.error(f"node not inserted: {node}. data is likely missing") return pk @@ -60,4 +67,8 @@ async def insert_job(db: aiosqlite.Connection, job: dict) -> int: ignore=True, ) ) as cursor: - return cursor.lastrowid + if cursor.rowcount > 0: + return cursor.lastrowid + + logging.error(f"job not inserted: {job}. data is likely missing") + return None From 35f6e81618dba5e5d846766b04ff43a0adfe2b33 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:24:18 -0800 Subject: [PATCH 04/23] enable foreign keys by default in database --- db/schema.sql | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/db/schema.sql b/db/schema.sql index bba3549..a9dc104 100644 --- a/db/schema.sql +++ b/db/schema.sql @@ -1,3 +1,5 @@ +PRAGMA foreign_keys = ON; + CREATE TABLE nodes ( id INTEGER PRIMARY KEY, uuid TEXT NOT NULL UNIQUE, @@ -27,7 +29,7 @@ CREATE TABLE jobs ( stack TEXT NOT NULL, build_jobs INTEGER NOT NULL, cpu_request REAL NOT NULL, - cpu_limit REAL, -- this can be null becasue it's currently not set + cpu_limit REAL, -- this can be null because it's currently not set cpu_mean REAL NOT NULL, cpu_median REAL NOT NULL, cpu_max REAL NOT NULL, From 4cd60639a6c95bb66f78e8ee0a0cf2ed7da89bd2 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:27:54 -0800 Subject: [PATCH 05/23] fix: reorder calling of _query within prometheus client to better accomodate test mocking --- gantry/clients/prometheus/prometheus.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gantry/clients/prometheus/prometheus.py b/gantry/clients/prometheus/prometheus.py index 1d92503..424d1b1 100644 --- a/gantry/clients/prometheus/prometheus.py +++ b/gantry/clients/prometheus/prometheus.py @@ -39,7 +39,7 @@ async def query_single(self, query: str | dict, time: int) -> list: query = util.process_query(query) url = f"{self.base_url}/query?query={query}&time={time}" - return await self._query(url) + return self.prettify_res(await self._query(url)) async def query_range(self, query: str | dict, start: int, end: int) -> list: """Query Prometheus for a range of values @@ -64,7 +64,7 @@ async def query_range(self, query: str | dict, start: int, end: int) -> list: f"end={end}&" f"step={step}s" ) - return await self._query(url) + return self.prettify_res(await self._query(url)) async def _query(self, url: str) -> list: """Query Prometheus with a query string""" @@ -72,7 +72,7 @@ async def _query(self, url: str) -> list: # submit cookie with request async with session.get(url, cookies=self.cookies) as resp: try: - return self.prettify_res(await resp.json()) + return await resp.json() except aiohttp.ContentTypeError: logger.error( """Prometheus query failed with unexpected response. @@ -81,7 +81,7 @@ async def _query(self, url: str) -> list: return {} def prettify_res(self, response: dict) -> list: - """Process Prometheus response into an arrray of dicts with {label: value}""" + """Process Prometheus response into a list of dicts with {label: value}""" result_type = response.get("data", {}).get("resultType") values_dict = { "matrix": "values", From c21395d91d11ca63b7670760013c02d741b34084 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:29:39 -0800 Subject: [PATCH 06/23] add tests: spec variant parsing --- gantry/tests/test_spec.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 gantry/tests/test_spec.py diff --git a/gantry/tests/test_spec.py b/gantry/tests/test_spec.py new file mode 100644 index 0000000..b91ca86 --- /dev/null +++ b/gantry/tests/test_spec.py @@ -0,0 +1,21 @@ +from gantry.util.spec import spec_variants + + +def test_valid_spec(): + """Test a valid spec string to be parsed""" + assert spec_variants( + "+adios2~advanced_debug patches=02253c7,acb3805,b724e6a use_vtkm=on" + ) == { + "adios2": True, + "advanced_debug": False, + "patches": ["02253c7", "acb3805", "b724e6a"], + "use_vtkm": "on", + } + + +def test_invalid_spec(): + """Test some invalid specs""" + assert spec_variants("fefj!@#$%^&eifejifeifeij---5893843$%^&*()") == {} + assert spec_variants("fifheife") == {} + assert spec_variants("++++++") == {} + assert spec_variants("+~~++") == {} From bbded465ec7324c06a422bef4275369ee90d1d76 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:31:08 -0800 Subject: [PATCH 07/23] add tests: prometheus client --- gantry/tests/defs/__init__.py | 0 gantry/tests/defs/prometheus.py | 12 ++++++++++++ gantry/tests/test_prometheus.py | 25 +++++++++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 gantry/tests/defs/__init__.py create mode 100644 gantry/tests/defs/prometheus.py create mode 100644 gantry/tests/test_prometheus.py diff --git a/gantry/tests/defs/__init__.py b/gantry/tests/defs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/gantry/tests/defs/prometheus.py b/gantry/tests/defs/prometheus.py new file mode 100644 index 0000000..eeec4d3 --- /dev/null +++ b/gantry/tests/defs/prometheus.py @@ -0,0 +1,12 @@ +# flake8: noqa +# fmt: off + +QUERY_DICT = query={"metric": "kube_pod_annotations","filters": {"annotation_gitlab_ci_job_id": 1}} +QUERY_STR = "rate(container_cpu_usage_seconds_total{pod='1', container='build'}[90s])" + +# encoded versions of the above that were put through the original version of process_query +ENCODED_QUERY_DICT = "kube_pod_annotations%7Bannotation_gitlab_ci_job_id%3D%221%22%7D" +ENCODED_QUERY_STR = "rate%28container_cpu_usage_seconds_total%7Bpod%3D%271%27%2C%20container%3D%27build%27%7D%5B90s%5D%29" + +# this will not be parsed as a query +INVALID_QUERY = 1 diff --git a/gantry/tests/test_prometheus.py b/gantry/tests/test_prometheus.py new file mode 100644 index 0000000..6350863 --- /dev/null +++ b/gantry/tests/test_prometheus.py @@ -0,0 +1,25 @@ +import pytest + +from gantry.clients.prometheus import util +from gantry.clients.prometheus.prometheus import PrometheusClient +from gantry.tests.defs.prometheus import ( + ENCODED_QUERY_DICT, + ENCODED_QUERY_STR, + INVALID_QUERY, + QUERY_DICT, + QUERY_STR, +) + + +def test_cookie_set(): + """Test that a cookie is set when specified""" + p = PrometheusClient("", "cookie") + assert p.cookies == {"_oauth2_proxy": "cookie"} + + +def test_process_query(): + """Test that a query is parsed and encoded properly, from both dict and string""" + assert util.process_query(QUERY_DICT) == ENCODED_QUERY_DICT + assert util.process_query(QUERY_STR) == ENCODED_QUERY_STR + with pytest.raises(ValueError): + util.process_query(INVALID_QUERY) From 5b9deae6dfbf2267fd3c02daad2a2854b0d9d6aa Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:36:15 -0800 Subject: [PATCH 08/23] add tests: collection API --- gantry/tests/conftest.py | 16 ++++ gantry/tests/defs/collection.py | 37 +++++++ gantry/tests/sql/insert_job.sql | 1 + gantry/tests/sql/insert_node.sql | 2 + gantry/tests/test_collect.py | 160 +++++++++++++++++++++++++++++++ 5 files changed, 216 insertions(+) create mode 100644 gantry/tests/conftest.py create mode 100644 gantry/tests/defs/collection.py create mode 100644 gantry/tests/sql/insert_job.sql create mode 100644 gantry/tests/sql/insert_node.sql create mode 100644 gantry/tests/test_collect.py diff --git a/gantry/tests/conftest.py b/gantry/tests/conftest.py new file mode 100644 index 0000000..5c1bdc2 --- /dev/null +++ b/gantry/tests/conftest.py @@ -0,0 +1,16 @@ +# fixtures shared among all tests + +import aiosqlite +import pytest + + +@pytest.fixture +async def db_conn(): + """ + In-memory sqlite connection ensures that the database is clean for each test + """ + db = await aiosqlite.connect(":memory:") + with open("db/schema.sql") as f: + await db.executescript(f.read()) + yield db + await db.close() diff --git a/gantry/tests/defs/collection.py b/gantry/tests/defs/collection.py new file mode 100644 index 0000000..d0db895 --- /dev/null +++ b/gantry/tests/defs/collection.py @@ -0,0 +1,37 @@ +# flake8: noqa +# fmt: off + +INVALID_JOB_NAME = "invalid job name" +# uo runners are not supported +INVALID_RUNNER = {"description": "uo-blabla1821"} +INVALID_JOB_STATUS = "failure" +GHOST_JOB_LOG = "No need to rebuild" +VALID_JOB_LOG = "some log" + +VALID_JOB = { + "build_status": "success", + "build_name": "gmsh@4.8.4 /jcchwaj %gcc@11.4.0 arch=linux-ubuntu20.04-x86_64_v3 E4S", + "build_id": 9892514, # not used in testing unless it already exists in the db + "build_started_at": "2024-01-24T17:24:06.000Z", + "build_finished_at": "2024-01-24T17:47:00.000Z", + "ref": "pr42264_bugfix/mathomp4/hdf5-appleclang15", + "runner": {"description": "aws"}, +} + +# used to compare successful insertions +# run SELECT * FROM table_name WHERE id = 1; from python sqlite api and grab fetchone() result +INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux-ubuntu20.04-x86_64_v3', 'e4s', 16, 0.75, None, 4.125322866945405, 3.158058646775204, 11.603810729464888, 0.2483743618267752, 3.348888803394752, 2000000000.0, 48000000000.0, 1649868862.7258806, 999763968.0, 5679742976.0, 2785280.0, 1378705563.2101867) +INSERTED_NODE = (1, 'ec253b04-b1dc-f08b-acac-e23df83b3602', 'ip-192-168-86-107.ec2.internal', 24.0, 196608000000.0, 'amd64', 'linux', 'i3en.6xlarge') + +# these were obtained by executing the respective queries to Prometheus and capturing the JSON output +# or the raw output of PrometheusClient._query +VALID_ANNOTATIONS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_annotations', 'annotation_gitlab_ci_job_id': '9892514', 'annotation_metrics_spack_ci_stack_name': 'e4s', 'annotation_metrics_spack_job_spec_arch': 'linux-ubuntu20.04-x86_64_v3', 'annotation_metrics_spack_job_spec_compiler_name': 'gcc', 'annotation_metrics_spack_job_spec_compiler_version': '11.4.0', 'annotation_metrics_spack_job_spec_pkg_name': 'gmsh', 'annotation_metrics_spack_job_spec_pkg_version': '4.8.4', 'annotation_metrics_spack_job_spec_variants': '+alglib~cairo+cgns+compression~eigen~external+fltk+gmp~hdf5~ipo+med+metis+mmg+mpi+netgen+oce~opencascade~openmp~petsc~privateapi+shared~slepc+tetgen+voropp build_system=cmake build_type=Release generator=make', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0'}, 'value': [1706117733, '1']}]}} +VALID_RESOURCE_REQUESTS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_resource_requests', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'cpu', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'core'}, 'value': [1706117733, '0.75']}, {'metric': {'__name__': 'kube_pod_container_resource_requests', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'memory', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'byte'}, 'value': [1706117733, '2000000000']}]}} +VALID_RESOURCE_LIMITS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_resource_limits', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'memory', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'byte'}, 'value': [1706117733, '48000000000']}]}} +VALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '2785280'], [1706117116, '2785280'], [1706117117, '2785280'], [1706117118, '2785280'], [1706117119, '2785280'], [1706117120, '2785280'], [1706117121, '2785280'], [1706117122, '2785280'], [1706117123, '2785280'], [1706117124, '2785280'], [1706117125, '2785280'], [1706117126, '2785280'], [1706117127, '2785280'], [1706117128, '2785280'], [1706117129, '2785280'], [1706117130, '2785280'], [1706117131, '2785280'], [1706117132, '2785280'], [1706117133, '2785280'], [1706117134, '2785280'], [1706117135, '2785280'], [1706117136, '2785280'], [1706117137, '2785280'], [1706117138, '2785280'], [1706117139, '2785280'], [1706117140, '2785280'], [1706117141, '2785280'], [1706117142, '2785280'], [1706117143, '2785280'], [1706117144, '2785280'], [1706117145, '463159296'], [1706117146, '463159296'], [1706117147, '463159296'], [1706117148, '463159296'], [1706117149, '463159296'], [1706117150, '463159296'], [1706117151, '463159296'], [1706117152, '463159296'], [1706117153, '463159296'], [1706117154, '463159296'], [1706117155, '463159296'], [1706117156, '463159296'], [1706117157, '463159296'], [1706117158, '463159296'], [1706117159, '463159296'], [1706117160, '463159296'], [1706117161, '463159296'], [1706117162, '463159296'], [1706117163, '463159296'], [1706117164, '463159296'], [1706117165, '463159296'], [1706117166, '463159296'], [1706117167, '463159296'], [1706117168, '463159296'], [1706117169, '463159296'], [1706117170, '463159296'], [1706117171, '463159296'], [1706117172, '463159296'], [1706117173, '463159296'], [1706117174, '463159296'], [1706117175, '463159296'], [1706117176, '814882816'], [1706117177, '814882816'], [1706117178, '814882816'], [1706117179, '814882816'], [1706117180, '814882816'], [1706117181, '814882816'], [1706117182, '814882816'], [1706117183, '814882816'], [1706117184, '814882816'], [1706117185, '814882816'], [1706117186, '814882816'], [1706117187, '814882816'], [1706117188, '814882816'], [1706117189, '814882816'], [1706117190, '814882816'], [1706117191, '814882816'], [1706117192, '814882816'], [1706117193, '814882816'], [1706117194, '814882816'], [1706117195, '814882816'], [1706117196, '814882816'], [1706117197, '814882816'], [1706117198, '814882816'], [1706117199, '814882816'], [1706117200, '814882816'], [1706117201, '814882816'], [1706117202, '814882816'], [1706117203, '814882816'], [1706117204, '814882816'], [1706117205, '814882816'], [1706117206, '144236544'], [1706117207, '144236544'], [1706117208, '144236544'], [1706117209, '144236544'], [1706117210, '144236544'], [1706117211, '144236544'], [1706117212, '144236544'], [1706117213, '144236544'], [1706117214, '144236544'], [1706117215, '144236544'], [1706117216, '144236544'], [1706117217, '144236544'], [1706117218, '144236544'], [1706117219, '144236544'], [1706117220, '144236544'], [1706117221, '144236544'], [1706117222, '144236544'], [1706117223, '144236544'], [1706117224, '144236544'], [1706117225, '280522752'], [1706117226, '280522752'], [1706117227, '280522752'], [1706117228, '280522752'], [1706117229, '280522752'], [1706117230, '280522752'], [1706117231, '280522752'], [1706117232, '280522752'], [1706117233, '280522752'], [1706117234, '280522752'], [1706117235, '280522752'], [1706117236, '280522752'], [1706117237, '280522752'], [1706117238, '280522752'], [1706117239, '280522752'], [1706117240, '280522752'], [1706117241, '280522752'], [1706117242, '280522752'], [1706117243, '280522752'], [1706117244, '280522752'], [1706117245, '280522752'], [1706117246, '280522752'], [1706117247, '280522752'], [1706117248, '280522752'], [1706117249, '280522752'], [1706117250, '280522752'], [1706117251, '280522752'], [1706117252, '280522752'], [1706117253, '280522752'], [1706117254, '280522752'], [1706117255, '280522752'], [1706117256, '280522752'], [1706117257, '280522752'], [1706117258, '280522752'], [1706117259, '280522752'], [1706117260, '280522752'], [1706117261, '280522752'], [1706117262, '280522752'], [1706117263, '280522752'], [1706117264, '280522752'], [1706117265, '280522752'], [1706117266, '280522752'], [1706117267, '280522752'], [1706117268, '280522752'], [1706117269, '280522752'], [1706117270, '280522752'], [1706117271, '280522752'], [1706117272, '5402374144'], [1706117273, '5402374144'], [1706117274, '5402374144'], [1706117275, '5402374144'], [1706117276, '5402374144'], [1706117277, '5402374144'], [1706117278, '5402374144'], [1706117279, '5402374144'], [1706117280, '5402374144'], [1706117281, '5402374144'], [1706117282, '5402374144'], [1706117283, '5402374144'], [1706117284, '5402374144'], [1706117285, '5402374144'], [1706117286, '5402374144'], [1706117287, '5402374144'], [1706117288, '5402374144'], [1706117289, '5402374144'], [1706117290, '5402374144'], [1706117291, '5402374144'], [1706117292, '5402374144'], [1706117293, '5402374144'], [1706117294, '5402374144'], [1706117295, '5402374144'], [1706117296, '5402374144'], [1706117297, '5402374144'], [1706117298, '5402374144'], [1706117299, '5402374144'], [1706117300, '5679742976'], [1706117301, '5679742976'], [1706117302, '5679742976'], [1706117303, '5679742976'], [1706117304, '5679742976'], [1706117305, '5679742976'], [1706117306, '5679742976'], [1706117307, '5679742976'], [1706117308, '5679742976'], [1706117309, '5679742976'], [1706117310, '5679742976'], [1706117311, '5679742976'], [1706117312, '5679742976'], [1706117313, '5679742976'], [1706117314, '5679742976'], [1706117315, '5679742976'], [1706117316, '5679742976'], [1706117317, '5679742976'], [1706117318, '5679742976'], [1706117319, '5679742976'], [1706117320, '5679742976'], [1706117321, '5679742976'], [1706117322, '5679742976'], [1706117323, '5679742976'], [1706117324, '5679742976'], [1706117325, '5679742976'], [1706117326, '4521721856'], [1706117327, '4521721856'], [1706117328, '4521721856'], [1706117329, '4521721856'], [1706117330, '4521721856'], [1706117331, '4521721856'], [1706117332, '4521721856'], [1706117333, '4521721856'], [1706117334, '4521721856'], [1706117335, '4521721856'], [1706117336, '4521721856'], [1706117337, '4521721856'], [1706117338, '4521721856'], [1706117339, '4521721856'], [1706117340, '4521721856'], [1706117341, '4521721856'], [1706117342, '4521721856'], [1706117343, '4521721856'], [1706117344, '4521721856'], [1706117345, '4521721856'], [1706117346, '4521721856'], [1706117347, '4521721856'], [1706117348, '4521721856'], [1706117349, '4521721856'], [1706117350, '4521721856'], [1706117351, '4521721856'], [1706117352, '4521721856'], [1706117353, '4521721856'], [1706117354, '4521721856'], [1706117355, '4521721856'], [1706117356, '4426862592'], [1706117357, '4426862592'], [1706117358, '4426862592'], [1706117359, '4426862592'], [1706117360, '4426862592'], [1706117361, '4426862592'], [1706117362, '4426862592'], [1706117363, '4426862592'], [1706117364, '4426862592'], [1706117365, '4426862592'], [1706117366, '4426862592'], [1706117367, '4426862592'], [1706117368, '4426862592'], [1706117369, '4426862592'], [1706117370, '4426862592'], [1706117371, '4426862592'], [1706117372, '4426862592'], [1706117373, '4426862592'], [1706117374, '4426862592'], [1706117375, '4426862592'], [1706117376, '4426862592'], [1706117377, '4426862592'], [1706117378, '4426862592'], [1706117379, '4426862592'], [1706117380, '4426862592'], [1706117381, '4426862592'], [1706117382, '4426862592'], [1706117383, '4426862592'], [1706117384, '4426862592'], [1706117385, '4426862592'], [1706117386, '4426862592'], [1706117387, '4426862592'], [1706117388, '4426862592'], [1706117389, '4426862592'], [1706117390, '4426862592'], [1706117391, '2534088704'], [1706117392, '2534088704'], [1706117393, '2534088704'], [1706117394, '2534088704'], [1706117395, '2534088704'], [1706117396, '2534088704'], [1706117397, '2534088704'], [1706117398, '2534088704'], [1706117399, '2534088704'], [1706117400, '2534088704'], [1706117401, '2534088704'], [1706117402, '2534088704'], [1706117403, '2534088704'], [1706117404, '2534088704'], [1706117405, '2534088704'], [1706117406, '2534088704'], [1706117407, '2721067008'], [1706117408, '2721067008'], [1706117409, '2721067008'], [1706117410, '2721067008'], [1706117411, '2721067008'], [1706117412, '2721067008'], [1706117413, '2721067008'], [1706117414, '2721067008'], [1706117415, '2721067008'], [1706117416, '2721067008'], [1706117417, '2721067008'], [1706117418, '2721067008'], [1706117419, '2721067008'], [1706117420, '2721067008'], [1706117421, '2721067008'], [1706117422, '2721067008'], [1706117423, '2721067008'], [1706117424, '2721067008'], [1706117425, '2721067008'], [1706117426, '2721067008'], [1706117427, '2721067008'], [1706117428, '2721067008'], [1706117429, '2721067008'], [1706117430, '2721067008'], [1706117431, '2721067008'], [1706117432, '2721067008'], [1706117433, '2721067008'], [1706117434, '2721067008'], [1706117435, '2721067008'], [1706117436, '2721067008'], [1706117437, '2721067008'], [1706117438, '2721067008'], [1706117439, '2721067008'], [1706117440, '2262188032'], [1706117441, '2262188032'], [1706117442, '2262188032'], [1706117443, '2262188032'], [1706117444, '2262188032'], [1706117445, '2262188032'], [1706117446, '2262188032'], [1706117447, '2262188032'], [1706117448, '2262188032'], [1706117449, '2262188032'], [1706117450, '2262188032'], [1706117451, '2262188032'], [1706117452, '2262188032'], [1706117453, '2262188032'], [1706117454, '2262188032'], [1706117455, '2262188032'], [1706117456, '2262188032'], [1706117457, '2262188032'], [1706117458, '2262188032'], [1706117459, '2262188032'], [1706117460, '2262188032'], [1706117461, '2262188032'], [1706117462, '2262188032'], [1706117463, '2262188032'], [1706117464, '2262188032'], [1706117465, '2262188032'], [1706117466, '2262188032'], [1706117467, '2262188032'], [1706117468, '2262188032'], [1706117469, '2262188032'], [1706117470, '2262188032'], [1706117471, '2262188032'], [1706117472, '2262188032'], [1706117473, '2262188032'], [1706117474, '2262188032'], [1706117475, '2262188032'], [1706117476, '2262188032'], [1706117477, '2262188032'], [1706117478, '2262188032'], [1706117479, '1202192384'], [1706117480, '1202192384'], [1706117481, '1202192384'], [1706117482, '1202192384'], [1706117483, '1202192384'], [1706117484, '1202192384'], [1706117485, '1202192384'], [1706117486, '1202192384'], [1706117487, '1202192384'], [1706117488, '1202192384'], [1706117489, '1202192384'], [1706117490, '1202192384'], [1706117491, '1202192384'], [1706117492, '1202192384'], [1706117493, '1202192384'], [1706117494, '1202192384'], [1706117495, '1202192384'], [1706117496, '1377042432'], [1706117497, '1377042432'], [1706117498, '1377042432'], [1706117499, '1377042432'], [1706117500, '1377042432'], [1706117501, '1377042432'], [1706117502, '1377042432'], [1706117503, '1377042432'], [1706117504, '1377042432'], [1706117505, '1377042432'], [1706117506, '1377042432'], [1706117507, '1377042432'], [1706117508, '1377042432'], [1706117509, '1377042432'], [1706117510, '1377042432'], [1706117511, '1377042432'], [1706117512, '1377042432'], [1706117513, '1377042432'], [1706117514, '1377042432'], [1706117515, '1377042432'], [1706117516, '1377042432'], [1706117517, '1377042432'], [1706117518, '1377042432'], [1706117519, '1377042432'], [1706117520, '1377042432'], [1706117521, '1377042432'], [1706117522, '1377042432'], [1706117523, '1377042432'], [1706117524, '1377042432'], [1706117525, '1377042432'], [1706117526, '1377042432'], [1706117527, '1377042432'], [1706117528, '937562112'], [1706117529, '937562112'], [1706117530, '937562112'], [1706117531, '937562112'], [1706117532, '937562112'], [1706117533, '937562112'], [1706117534, '937562112'], [1706117535, '937562112'], [1706117536, '937562112'], [1706117537, '937562112'], [1706117538, '937562112'], [1706117539, '937562112'], [1706117540, '937562112'], [1706117541, '937562112'], [1706117542, '937562112'], [1706117543, '937562112'], [1706117544, '937562112'], [1706117545, '937562112'], [1706117546, '937562112'], [1706117547, '937562112'], [1706117548, '937562112'], [1706117549, '937562112'], [1706117550, '937562112'], [1706117551, '937562112'], [1706117552, '937562112'], [1706117553, '937562112'], [1706117554, '937562112'], [1706117555, '937562112'], [1706117556, '937562112'], [1706117557, '937562112'], [1706117558, '937562112'], [1706117559, '937562112'], [1706117560, '937562112'], [1706117561, '937562112'], [1706117562, '937562112'], [1706117563, '937562112'], [1706117564, '937562112'], [1706117565, '937562112'], [1706117566, '999763968'], [1706117567, '999763968'], [1706117568, '999763968'], [1706117569, '999763968'], [1706117570, '999763968'], [1706117571, '999763968'], [1706117572, '999763968'], [1706117573, '999763968'], [1706117574, '999763968'], [1706117575, '999763968'], [1706117576, '999763968'], [1706117577, '999763968'], [1706117578, '999763968'], [1706117579, '999763968'], [1706117580, '999763968'], [1706117581, '999763968'], [1706117582, '999763968'], [1706117583, '999763968'], [1706117584, '999763968'], [1706117585, '999763968'], [1706117586, '999763968'], [1706117587, '999763968'], [1706117588, '999763968'], [1706117589, '999763968'], [1706117590, '999763968'], [1706117591, '999763968'], [1706117592, '999763968'], [1706117593, '999763968'], [1706117594, '999763968'], [1706117595, '999763968'], [1706117596, '999763968'], [1706117597, '999763968'], [1706117598, '566054912'], [1706117599, '566054912'], [1706117600, '566054912'], [1706117601, '566054912'], [1706117602, '566054912'], [1706117603, '566054912'], [1706117604, '566054912'], [1706117605, '566054912'], [1706117606, '566054912'], [1706117607, '566054912'], [1706117608, '566054912'], [1706117609, '566054912'], [1706117610, '566054912'], [1706117611, '566054912'], [1706117612, '566054912'], [1706117613, '566054912'], [1706117614, '566054912'], [1706117615, '566054912'], [1706117616, '566054912'], [1706117617, '566054912'], [1706117618, '566054912'], [1706117619, '566054912'], [1706117620, '566054912'], [1706117621, '566054912'], [1706117622, '566054912'], [1706117623, '566054912'], [1706117624, '566054912'], [1706117625, '566054912'], [1706117626, '566054912'], [1706117627, '566054912'], [1706117628, '756076544'], [1706117629, '756076544'], [1706117630, '756076544'], [1706117631, '756076544'], [1706117632, '756076544'], [1706117633, '756076544'], [1706117634, '756076544'], [1706117635, '756076544'], [1706117636, '756076544'], [1706117637, '756076544'], [1706117638, '756076544'], [1706117639, '756076544'], [1706117640, '756076544'], [1706117641, '756076544'], [1706117642, '756076544'], [1706117643, '756076544'], [1706117644, '756076544'], [1706117645, '756076544'], [1706117646, '756076544'], [1706117647, '756076544'], [1706117648, '756076544'], [1706117649, '756076544'], [1706117650, '756076544'], [1706117651, '756076544'], [1706117652, '756076544'], [1706117653, '756076544'], [1706117654, '756076544'], [1706117655, '756076544'], [1706117656, '756076544'], [1706117657, '756076544'], [1706117658, '756076544'], [1706117659, '756076544'], [1706117660, '826408960'], [1706117661, '826408960'], [1706117662, '826408960'], [1706117663, '826408960'], [1706117664, '826408960'], [1706117665, '826408960'], [1706117666, '826408960'], [1706117667, '826408960'], [1706117668, '826408960'], [1706117669, '826408960'], [1706117670, '826408960'], [1706117671, '826408960'], [1706117672, '826408960'], [1706117673, '826408960'], [1706117674, '826408960'], [1706117675, '826408960'], [1706117676, '826408960'], [1706117677, '826408960'], [1706117678, '826408960'], [1706117679, '826408960'], [1706117680, '826408960'], [1706117681, '826408960'], [1706117682, '826408960'], [1706117683, '826408960'], [1706117684, '826408960'], [1706117685, '826408960'], [1706117686, '826408960'], [1706117687, '826408960'], [1706117688, '826408960'], [1706117689, '779374592'], [1706117690, '779374592'], [1706117691, '779374592'], [1706117692, '779374592'], [1706117693, '779374592'], [1706117694, '779374592'], [1706117695, '779374592'], [1706117696, '779374592'], [1706117697, '779374592'], [1706117698, '779374592'], [1706117699, '779374592'], [1706117700, '779374592'], [1706117701, '779374592'], [1706117702, '779374592'], [1706117703, '779374592'], [1706117704, '779374592'], [1706117705, '779374592'], [1706117706, '779374592'], [1706117707, '779374592'], [1706117708, '515768320'], [1706117709, '515768320'], [1706117710, '515768320'], [1706117711, '515768320'], [1706117712, '515768320'], [1706117713, '515768320'], [1706117714, '515768320'], [1706117715, '515768320'], [1706117716, '515768320'], [1706117717, '515768320'], [1706117718, '515768320'], [1706117719, '515768320'], [1706117720, '515768320'], [1706117721, '515768320'], [1706117722, '515768320'], [1706117723, '515768320'], [1706117724, '515768320'], [1706117725, '515768320'], [1706117726, '515768320'], [1706117727, '515768320'], [1706117728, '515768320'], [1706117729, '515768320'], [1706117730, '515768320'], [1706117731, '515768320'], [1706117732, '515768320'], [1706117733, '515768320'], [1706117734, '515768320'], [1706117735, '515768320'], [1706117736, '515768320'], [1706117737, '515768320'], [1706117738, '515768320'], [1706117739, '515768320'], [1706117740, '515768320'], [1706117741, '515768320'], [1706117742, '515768320'], [1706117743, '515768320'], [1706117744, '515768320'], [1706117745, '515768320'], [1706117746, '515768320'], [1706117747, '515768320'], [1706117748, '515768320'], [1706117749, '515768320'], [1706117750, '518909952'], [1706117751, '518909952'], [1706117752, '518909952'], [1706117753, '518909952'], [1706117754, '518909952'], [1706117755, '518909952'], [1706117756, '518909952'], [1706117757, '518909952'], [1706117758, '518909952'], [1706117759, '518909952'], [1706117760, '518909952'], [1706117761, '518909952'], [1706117762, '518909952'], [1706117763, '518909952'], [1706117764, '518909952'], [1706117765, '518909952'], [1706117766, '518909952'], [1706117767, '518909952'], [1706117768, '518909952'], [1706117769, '518909952'], [1706117770, '518909952'], [1706117771, '518909952'], [1706117772, '518909952'], [1706117773, '518909952'], [1706117774, '518909952'], [1706117775, '518909952'], [1706117776, '518909952'], [1706117777, '527507456'], [1706117778, '527507456'], [1706117779, '527507456'], [1706117780, '527507456'], [1706117781, '527507456'], [1706117782, '527507456'], [1706117783, '527507456'], [1706117784, '527507456'], [1706117785, '527507456'], [1706117786, '527507456'], [1706117787, '527507456'], [1706117788, '527507456'], [1706117789, '527507456'], [1706117790, '527507456'], [1706117791, '527507456'], [1706117792, '527507456'], [1706117793, '527507456'], [1706117794, '527507456'], [1706117795, '527507456'], [1706117796, '527507456'], [1706117797, '527507456'], [1706117798, '527507456'], [1706117799, '527507456'], [1706117800, '527507456'], [1706117801, '527507456'], [1706117802, '527507456'], [1706117803, '527507456'], [1706117804, '527507456'], [1706117805, '527507456'], [1706117806, '527507456'], [1706117807, '550576128'], [1706117808, '550576128'], [1706117809, '550576128'], [1706117810, '550576128'], [1706117811, '550576128'], [1706117812, '550576128'], [1706117813, '550576128'], [1706117814, '550576128'], [1706117815, '550576128'], [1706117816, '550576128'], [1706117817, '550576128'], [1706117818, '550576128'], [1706117819, '550576128'], [1706117820, '550576128'], [1706117821, '550576128'], [1706117822, '550576128'], [1706117823, '550576128'], [1706117824, '550576128'], [1706117825, '550576128'], [1706117826, '550576128'], [1706117827, '550576128'], [1706117828, '550576128'], [1706117829, '550576128'], [1706117830, '550576128'], [1706117831, '550576128'], [1706117832, '550576128'], [1706117833, '550576128'], [1706117834, '550576128'], [1706117835, '707067904'], [1706117836, '707067904'], [1706117837, '707067904'], [1706117838, '707067904'], [1706117839, '707067904'], [1706117840, '707067904'], [1706117841, '707067904'], [1706117842, '707067904'], [1706117843, '707067904'], [1706117844, '707067904'], [1706117845, '707067904'], [1706117846, '707067904'], [1706117847, '707067904'], [1706117848, '707067904'], [1706117849, '707067904'], [1706117850, '707067904'], [1706117851, '707067904'], [1706117852, '707067904'], [1706117853, '707067904'], [1706117854, '707067904'], [1706117855, '707067904'], [1706117856, '707067904'], [1706117857, '707067904'], [1706117858, '707067904'], [1706117859, '707067904'], [1706117860, '707067904'], [1706117861, '707067904'], [1706117862, '707067904'], [1706117863, '640978944'], [1706117864, '640978944'], [1706117865, '640978944'], [1706117866, '640978944'], [1706117867, '640978944'], [1706117868, '640978944'], [1706117869, '640978944'], [1706117870, '640978944'], [1706117871, '640978944'], [1706117872, '640978944'], [1706117873, '640978944'], [1706117874, '640978944'], [1706117875, '640978944'], [1706117876, '640978944'], [1706117877, '640978944'], [1706117878, '640978944'], [1706117879, '640978944'], [1706117880, '640978944'], [1706117881, '640978944'], [1706117882, '640978944'], [1706117883, '640978944'], [1706117884, '640978944'], [1706117885, '640978944'], [1706117886, '640978944'], [1706117887, '640978944'], [1706117888, '640978944'], [1706117889, '640978944'], [1706117890, '640978944'], [1706117891, '640978944'], [1706117892, '640978944'], [1706117893, '611074048'], [1706117894, '611074048'], [1706117895, '611074048'], [1706117896, '611074048'], [1706117897, '611074048'], [1706117898, '611074048'], [1706117899, '611074048'], [1706117900, '611074048'], [1706117901, '611074048'], [1706117902, '611074048'], [1706117903, '611074048'], [1706117904, '611074048'], [1706117905, '611074048'], [1706117906, '611074048'], [1706117907, '611074048'], [1706117908, '611074048'], [1706117909, '611074048'], [1706117910, '611074048'], [1706117911, '611074048'], [1706117912, '611074048'], [1706117913, '611074048'], [1706117914, '611074048'], [1706117915, '611074048'], [1706117916, '611074048'], [1706117917, '611074048'], [1706117918, '611074048'], [1706117919, '495898624'], [1706117920, '495898624'], [1706117921, '495898624'], [1706117922, '495898624'], [1706117923, '495898624'], [1706117924, '495898624'], [1706117925, '495898624'], [1706117926, '495898624'], [1706117927, '495898624'], [1706117928, '495898624'], [1706117929, '495898624'], [1706117930, '495898624'], [1706117931, '495898624'], [1706117932, '495898624'], [1706117933, '495898624'], [1706117934, '495898624'], [1706117935, '495898624'], [1706117936, '495898624'], [1706117937, '495898624'], [1706117938, '495898624'], [1706117939, '495898624'], [1706117940, '495898624'], [1706117941, '495898624'], [1706117942, '495898624'], [1706117943, '495898624'], [1706117944, '495898624'], [1706117945, '495898624'], [1706117946, '495898624'], [1706117947, '2094526464'], [1706117948, '2094526464'], [1706117949, '2094526464'], [1706117950, '2094526464'], [1706117951, '2094526464'], [1706117952, '2094526464'], [1706117953, '2094526464'], [1706117954, '2094526464'], [1706117955, '2094526464'], [1706117956, '2094526464'], [1706117957, '2094526464'], [1706117958, '2094526464'], [1706117959, '2094526464'], [1706117960, '2094526464'], [1706117961, '2094526464'], [1706117962, '2094526464'], [1706117963, '2094526464'], [1706117964, '2094526464'], [1706117965, '2094526464'], [1706117966, '2094526464'], [1706117967, '2094526464'], [1706117968, '2094526464'], [1706117969, '2094526464'], [1706117970, '2094526464'], [1706117971, '2094526464'], [1706117972, '2094526464'], [1706117973, '2094526464'], [1706117974, '2094526464'], [1706117975, '2094526464'], [1706117976, '2094526464'], [1706117977, '2094526464'], [1706117978, '2094526464'], [1706117979, '2094526464'], [1706117980, '2094526464'], [1706117981, '2094526464'], [1706117982, '2043138048'], [1706117983, '2043138048'], [1706117984, '2043138048'], [1706117985, '2043138048'], [1706117986, '2043138048'], [1706117987, '2043138048'], [1706117988, '2043138048'], [1706117989, '2043138048'], [1706117990, '2043138048'], [1706117991, '2043138048'], [1706117992, '2043138048'], [1706117993, '2043138048'], [1706117994, '2043138048'], [1706117995, '2043138048'], [1706117996, '2043138048'], [1706117997, '2043138048'], [1706117998, '2043138048'], [1706117999, '2043138048'], [1706118000, '2043138048'], [1706118001, '2043138048'], [1706118002, '2043138048'], [1706118003, '2043138048'], [1706118004, '2043138048'], [1706118005, '2043138048'], [1706118006, '2043138048'], [1706118007, '2043138048'], [1706118008, '2043138048'], [1706118009, '2043138048'], [1706118010, '2043138048'], [1706118011, '2043138048'], [1706118012, '2043138048'], [1706118013, '2043138048'], [1706118014, '2043138048'], [1706118015, '2043138048'], [1706118016, '2043138048'], [1706118017, '2043138048'], [1706118018, '2152861696'], [1706118019, '2152861696'], [1706118020, '2152861696'], [1706118021, '2152861696'], [1706118022, '2152861696'], [1706118023, '2152861696'], [1706118024, '2152861696'], [1706118025, '2152861696'], [1706118026, '2152861696'], [1706118027, '2152861696'], [1706118028, '2152861696'], [1706118029, '2152861696'], [1706118030, '2152861696'], [1706118031, '2152861696'], [1706118032, '2152861696'], [1706118033, '2152861696'], [1706118034, '2152861696'], [1706118035, '2152861696'], [1706118036, '2152861696'], [1706118037, '2152861696'], [1706118038, '2152861696'], [1706118039, '2152861696'], [1706118040, '2152861696'], [1706118041, '2152861696'], [1706118042, '2152861696'], [1706118043, '2152861696'], [1706118044, '2152861696'], [1706118045, '2152861696'], [1706118046, '2674098176'], [1706118047, '2674098176'], [1706118048, '2674098176'], [1706118049, '2674098176'], [1706118050, '2674098176'], [1706118051, '2674098176'], [1706118052, '2674098176'], [1706118053, '2674098176'], [1706118054, '2674098176'], [1706118055, '2674098176'], [1706118056, '2674098176'], [1706118057, '2674098176'], [1706118058, '2674098176'], [1706118059, '2674098176'], [1706118060, '2674098176'], [1706118061, '2674098176'], [1706118062, '2674098176'], [1706118063, '2674098176'], [1706118064, '2674098176'], [1706118065, '2674098176'], [1706118066, '2674098176'], [1706118067, '2674098176'], [1706118068, '2674098176'], [1706118069, '2674098176'], [1706118070, '2674098176'], [1706118071, '2674098176'], [1706118072, '2674098176'], [1706118073, '2674098176'], [1706118074, '2674098176'], [1706118075, '2674098176'], [1706118076, '2674098176'], [1706118077, '2736074752'], [1706118078, '2736074752'], [1706118079, '2736074752'], [1706118080, '2736074752'], [1706118081, '2736074752'], [1706118082, '2736074752'], [1706118083, '2736074752'], [1706118084, '2736074752'], [1706118085, '2736074752'], [1706118086, '2736074752'], [1706118087, '2736074752'], [1706118088, '2736074752'], [1706118089, '2736074752'], [1706118090, '2736074752'], [1706118091, '2736074752'], [1706118092, '2736074752'], [1706118093, '2736074752'], [1706118094, '2736074752'], [1706118095, '2736074752'], [1706118096, '2736074752'], [1706118097, '2736074752'], [1706118098, '2736074752'], [1706118099, '2736074752'], [1706118100, '2736074752'], [1706118101, '2736074752'], [1706118102, '2736074752'], [1706118103, '2736074752'], [1706118104, '2736074752'], [1706118105, '2736074752'], [1706118106, '2736074752'], [1706118107, '2736074752'], [1706118108, '2736074752'], [1706118109, '2736074752'], [1706118110, '2736074752'], [1706118111, '2736074752'], [1706118112, '2736074752'], [1706118113, '2943303680'], [1706118114, '2943303680'], [1706118115, '2943303680'], [1706118116, '2943303680'], [1706118117, '2943303680'], [1706118118, '2943303680'], [1706118119, '2943303680'], [1706118120, '2943303680'], [1706118121, '2943303680'], [1706118122, '2943303680'], [1706118123, '2943303680'], [1706118124, '2943303680'], [1706118125, '2943303680'], [1706118126, '2943303680'], [1706118127, '2943303680'], [1706118128, '2943303680'], [1706118129, '2943303680'], [1706118130, '2943303680'], [1706118131, '2943303680'], [1706118132, '2943303680'], [1706118133, '2943303680'], [1706118134, '2943303680'], [1706118135, '2943303680'], [1706118136, '2943303680'], [1706118137, '2943303680'], [1706118138, '2943303680'], [1706118139, '2943303680'], [1706118140, '2943303680'], [1706118141, '2952359936'], [1706118142, '2952359936'], [1706118143, '2952359936'], [1706118144, '2952359936'], [1706118145, '2952359936'], [1706118146, '2952359936'], [1706118147, '2952359936'], [1706118148, '2952359936'], [1706118149, '2952359936'], [1706118150, '2952359936'], [1706118151, '2952359936'], [1706118152, '2952359936'], [1706118153, '2952359936'], [1706118154, '2952359936'], [1706118155, '2952359936'], [1706118156, '2952359936'], [1706118157, '2952359936'], [1706118158, '2952359936'], [1706118159, '2952359936'], [1706118160, '2952359936'], [1706118161, '2952359936'], [1706118162, '2952359936'], [1706118163, '2952359936'], [1706118164, '2952359936'], [1706118165, '2952359936'], [1706118166, '2952359936'], [1706118167, '2952359936'], [1706118168, '2952359936'], [1706118169, '2952359936'], [1706118170, '2952359936'], [1706118171, '1859096576'], [1706118172, '1859096576'], [1706118173, '1859096576'], [1706118174, '1859096576'], [1706118175, '1859096576'], [1706118176, '1859096576'], [1706118177, '1859096576'], [1706118178, '1859096576'], [1706118179, '1859096576'], [1706118180, '1859096576'], [1706118181, '1859096576'], [1706118182, '1859096576'], [1706118183, '1859096576'], [1706118184, '1859096576'], [1706118185, '1859096576'], [1706118186, '1859096576'], [1706118187, '1859096576'], [1706118188, '1859096576'], [1706118189, '1859096576'], [1706118190, '1859096576'], [1706118191, '1859096576'], [1706118192, '1859096576'], [1706118193, '1859096576'], [1706118194, '1859096576'], [1706118195, '1859096576'], [1706118196, '1859096576'], [1706118197, '1859096576'], [1706118198, '1859096576'], [1706118199, '1859096576'], [1706118200, '1859096576'], [1706118201, '1859096576'], [1706118202, '1611554816'], [1706118203, '1611554816'], [1706118204, '1611554816'], [1706118205, '1611554816'], [1706118206, '1611554816'], [1706118207, '1611554816'], [1706118208, '1611554816'], [1706118209, '1611554816'], [1706118210, '1611554816'], [1706118211, '1611554816'], [1706118212, '1611554816'], [1706118213, '1611554816'], [1706118214, '1611554816'], [1706118215, '1611554816'], [1706118216, '1611554816'], [1706118217, '1611554816'], [1706118218, '1611554816'], [1706118219, '1611554816'], [1706118220, '1611554816'], [1706118221, '3208851456'], [1706118222, '3208851456'], [1706118223, '3208851456'], [1706118224, '3208851456'], [1706118225, '3208851456'], [1706118226, '3208851456'], [1706118227, '3208851456'], [1706118228, '3208851456'], [1706118229, '3208851456'], [1706118230, '3208851456'], [1706118231, '3208851456'], [1706118232, '3208851456'], [1706118233, '3208851456'], [1706118234, '3208851456'], [1706118235, '3208851456'], [1706118236, '3208851456'], [1706118237, '3208851456'], [1706118238, '3208851456'], [1706118239, '3208851456'], [1706118240, '3208851456'], [1706118241, '3208851456'], [1706118242, '3208851456'], [1706118243, '3208851456'], [1706118244, '3208851456'], [1706118245, '3208851456'], [1706118246, '3208851456'], [1706118247, '3208851456'], [1706118248, '3208851456'], [1706118249, '3208851456'], [1706118250, '3208851456'], [1706118251, '3208851456'], [1706118252, '3208851456'], [1706118253, '3208851456'], [1706118254, '3208851456'], [1706118255, '3208851456'], [1706118256, '3208851456'], [1706118257, '3208851456'], [1706118258, '3208851456'], [1706118259, '3208851456'], [1706118260, '3208851456'], [1706118261, '3208851456'], [1706118262, '3208851456'], [1706118263, '2387591168'], [1706118264, '2387591168'], [1706118265, '2387591168'], [1706118266, '2387591168'], [1706118267, '2387591168'], [1706118268, '2387591168'], [1706118269, '2387591168'], [1706118270, '2387591168'], [1706118271, '2387591168'], [1706118272, '2387591168'], [1706118273, '2387591168'], [1706118274, '2387591168'], [1706118275, '2387591168'], [1706118276, '2387591168'], [1706118277, '2387591168'], [1706118278, '860766208'], [1706118279, '860766208'], [1706118280, '860766208'], [1706118281, '860766208'], [1706118282, '860766208'], [1706118283, '860766208'], [1706118284, '860766208'], [1706118285, '860766208'], [1706118286, '860766208'], [1706118287, '860766208'], [1706118288, '860766208'], [1706118289, '860766208'], [1706118290, '860766208'], [1706118291, '860766208'], [1706118292, '860766208'], [1706118293, '860766208'], [1706118294, '860766208'], [1706118295, '860766208'], [1706118296, '860766208'], [1706118297, '860766208'], [1706118298, '860766208'], [1706118299, '860766208'], [1706118300, '860766208'], [1706118301, '860766208'], [1706118302, '860766208'], [1706118303, '860766208'], [1706118304, '860766208'], [1706118305, '860766208'], [1706118306, '860766208'], [1706118307, '860766208'], [1706118308, '860766208'], [1706118309, '860766208'], [1706118310, '860766208'], [1706118311, '860766208'], [1706118312, '860766208'], [1706118313, '860766208'], [1706118314, '860766208'], [1706118315, '860766208'], [1706118316, '860766208'], [1706118317, '860766208'], [1706118318, '860766208'], [1706118319, '860766208'], [1706118320, '860766208'], [1706118321, '860766208'], [1706118322, '1179983872'], [1706118323, '1179983872'], [1706118324, '1179983872'], [1706118325, '1179983872'], [1706118326, '1179983872'], [1706118327, '1179983872'], [1706118328, '1179983872'], [1706118329, '1179983872'], [1706118330, '1179983872'], [1706118331, '1179983872'], [1706118332, '1179983872'], [1706118333, '1179983872'], [1706118334, '1179983872'], [1706118335, '821743616'], [1706118336, '821743616'], [1706118337, '821743616'], [1706118338, '821743616'], [1706118339, '821743616'], [1706118340, '821743616'], [1706118341, '821743616'], [1706118342, '821743616'], [1706118343, '821743616'], [1706118344, '821743616'], [1706118345, '821743616'], [1706118346, '821743616'], [1706118347, '821743616'], [1706118348, '821743616'], [1706118349, '821743616'], [1706118350, '821743616'], [1706118351, '821743616'], [1706118352, '821743616'], [1706118353, '821743616'], [1706118354, '821743616'], [1706118355, '821743616'], [1706118356, '821743616'], [1706118357, '821743616'], [1706118358, '821743616'], [1706118359, '821743616'], [1706118360, '821743616'], [1706118361, '821743616'], [1706118362, '821743616'], [1706118363, '821743616'], [1706118364, '821743616'], [1706118365, '821743616'], [1706118366, '821743616'], [1706118367, '821743616'], [1706118368, '821743616'], [1706118369, '821743616'], [1706118370, '821743616'], [1706118371, '821743616'], [1706118372, '821743616'], [1706118373, '821743616'], [1706118374, '821743616'], [1706118375, '821743616'], [1706118376, '821743616'], [1706118377, '821743616'], [1706118378, '821743616'], [1706118379, '821743616'], [1706118380, '821743616'], [1706118381, '1029611520'], [1706118382, '1029611520'], [1706118383, '1029611520'], [1706118384, '1029611520'], [1706118385, '1029611520'], [1706118386, '1029611520'], [1706118387, '1029611520'], [1706118388, '1029611520'], [1706118389, '1029611520'], [1706118390, '1029611520'], [1706118391, '1029611520'], [1706118392, '1029611520'], [1706118393, '1029611520'], [1706118394, '1029611520'], [1706118395, '1029611520'], [1706118396, '1029611520'], [1706118397, '1029611520'], [1706118398, '1029611520'], [1706118399, '1029611520'], [1706118400, '1029611520'], [1706118401, '1029611520'], [1706118402, '1029611520'], [1706118403, '1029611520'], [1706118404, '1029611520'], [1706118405, '1029611520'], [1706118406, '594620416'], [1706118407, '594620416'], [1706118408, '594620416'], [1706118409, '594620416'], [1706118410, '594620416'], [1706118411, '594620416'], [1706118412, '594620416'], [1706118413, '594620416'], [1706118414, '594620416'], [1706118415, '594620416'], [1706118416, '594620416'], [1706118417, '594620416'], [1706118418, '594620416'], [1706118419, '594620416'], [1706118420, '594620416']]}]}} +VALID_CPU_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'container': 'build', 'cpu': 'total', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117145, '0.2483743618267752'], [1706117146, '0.25650526138466395'], [1706117147, '0.26463616094255266'], [1706117148, '0.2727670605004414'], [1706117149, '0.28089796005833007'], [1706117150, '0.2890288596162188'], [1706117151, '0.2971597591741076'], [1706117152, '0.30529065873199623'], [1706117153, '0.313421558289885'], [1706117154, '0.3215524578477737'], [1706117155, '0.3296833574056624'], [1706117156, '0.3378142569635511'], [1706117157, '0.3459451565214398'], [1706117158, '0.3540760560793285'], [1706117159, '0.3622069556372173'], [1706117160, '0.370337855195106'], [1706117161, '0.3784687547529946'], [1706117162, '0.38659965431088333'], [1706117163, '0.3947305538687721'], [1706117164, '0.4028614534266608'], [1706117165, '0.41099235298454945'], [1706117166, '0.4191232525424382'], [1706117167, '0.4272541521003269'], [1706117168, '0.4353850516582156'], [1706117169, '0.4435159512161044'], [1706117170, '0.45164685077399314'], [1706117171, '0.4597777503318818'], [1706117172, '0.4679086498897705'], [1706117173, '0.47603954944765925'], [1706117174, '0.48417044900554795'], [1706117175, '0.4923013485634366'], [1706117176, '0.5941941747822107'], [1706117177, '0.6038490390156116'], [1706117178, '0.6135039032490125'], [1706117179, '0.6231587674824133'], [1706117180, '0.6328136317158142'], [1706117181, '0.6424684959492151'], [1706117182, '0.652123360182616'], [1706117183, '0.6617782244160167'], [1706117184, '0.6714330886494176'], [1706117185, '0.6810879528828185'], [1706117186, '0.6907428171162194'], [1706117187, '0.7003976813496203'], [1706117188, '0.7100525455830212'], [1706117189, '0.7197074098164219'], [1706117190, '0.7293622740498229'], [1706117191, '0.7390171382832237'], [1706117192, '0.7486720025166246'], [1706117193, '0.7583268667500255'], [1706117194, '0.7679817309834264'], [1706117195, '0.7776365952168272'], [1706117196, '0.787291459450228'], [1706117197, '0.7969463236836289'], [1706117198, '0.8066011879170297'], [1706117199, '0.8162560521504307'], [1706117200, '0.8259109163838315'], [1706117201, '0.8355657806172325'], [1706117202, '0.8452206448506332'], [1706117203, '0.8548755090840342'], [1706117204, '0.8645303733174351'], [1706117205, '0.9164274605130325'], [1706117206, '0.9253149079610642'], [1706117207, '0.9363825414782355'], [1706117208, '0.947450174995407'], [1706117209, '0.9585178085125783'], [1706117210, '0.9695854420297495'], [1706117211, '0.9806530755469209'], [1706117212, '0.9917207090640923'], [1706117213, '0.9960870165454214'], [1706117214, '0.9960870165454214'], [1706117215, '0.9960870165454214'], [1706117216, '0.9960870165454214'], [1706117217, '0.9960870165454214'], [1706117218, '0.9960870165454214'], [1706117219, '0.9960870165454214'], [1706117220, '0.9960870165454214'], [1706117221, '0.9960870165454214'], [1706117222, '0.9960870165454214'], [1706117223, '0.9960870165454214'], [1706117224, '0.9960870165454214'], [1706117225, '0.9965632246557574'], [1706117226, '0.9965632246557574'], [1706117227, '0.9965632246557574'], [1706117228, '0.9965632246557574'], [1706117229, '0.9965632246557574'], [1706117230, '0.9965632246557574'], [1706117231, '0.9965632246557574'], [1706117232, '0.9965632246557574'], [1706117233, '0.9965632246557574'], [1706117234, '0.9965632246557574'], [1706117235, '0.7884960963099941'], [1706117236, '0.799549211752331'], [1706117237, '0.810602327194668'], [1706117238, '0.821655442637005'], [1706117239, '0.8327085580793421'], [1706117240, '0.9947803898103293'], [1706117241, '0.9947803898103293'], [1706117242, '0.9947803898103293'], [1706117243, '0.9947803898103293'], [1706117244, '0.9947803898103293'], [1706117245, '0.9947803898103293'], [1706117246, '0.9947803898103293'], [1706117247, '0.9947803898103293'], [1706117248, '0.9947803898103293'], [1706117249, '0.9947803898103293'], [1706117250, '0.9947803898103293'], [1706117251, '0.9947803898103293'], [1706117252, '0.8260766888139397'], [1706117253, '0.8150235733716028'], [1706117254, '0.8039704579292658'], [1706117255, '0.7929173424869287'], [1706117256, '0.7818642270445917'], [1706117257, '0.7708111116022549'], [1706117258, '0.7597579961599177'], [1706117259, '0.7487048807175807'], [1706117260, '0.7376517652752437'], [1706117261, '0.7265986498329068'], [1706117262, '0.7155455343905697'], [1706117263, '0.7044924189482329'], [1706117264, '0.6934393035058959'], [1706117265, '0.6823861880635589'], [1706117266, '0.43370360155555576'], [1706117267, '0.43370360155555576'], [1706117268, '0.43370360155555576'], [1706117269, '0.43370360155555576'], [1706117270, '0.43370360155555576'], [1706117271, '0.43370360155555576'], [1706117272, '5.566624524200288'], [1706117273, '5.6360656043509785'], [1706117274, '5.705506684501669'], [1706117275, '5.77494776465236'], [1706117276, '5.844388844803051'], [1706117277, '5.913829924953742'], [1706117278, '5.9832710051044335'], [1706117279, '6.052712085255124'], [1706117280, '6.122153165405815'], [1706117281, '6.191594245556507'], [1706117282, '6.249697213562188'], [1706117283, '6.249697213562188'], [1706117284, '6.249697213562188'], [1706117285, '6.249697213562188'], [1706117286, '6.249697213562188'], [1706117287, '6.249697213562188'], [1706117288, '6.249697213562188'], [1706117289, '6.249697213562188'], [1706117290, '6.249697213562188'], [1706117291, '6.249697213562188'], [1706117292, '6.249697213562188'], [1706117293, '6.249697213562188'], [1706117294, '6.249697213562188'], [1706117295, '6.249697213562188'], [1706117296, '7.8382055254153675'], [1706117297, '7.932086901178828'], [1706117298, '8.025968276942288'], [1706117299, '8.11984965270575'], [1706117300, '6.795147485867362'], [1706117301, '6.795147485867362'], [1706117302, '6.795147485867362'], [1706117303, '6.795147485867362'], [1706117304, '6.795147485867362'], [1706117305, '6.795147485867362'], [1706117306, '6.795147485867362'], [1706117307, '6.795147485867362'], [1706117308, '6.795147485867362'], [1706117309, '6.795147485867362'], [1706117310, '6.795147485867362'], [1706117311, '6.795147485867362'], [1706117312, '6.795147485867362'], [1706117313, '6.795147485867362'], [1706117314, '6.795147485867362'], [1706117315, '2.5720481747280175'], [1706117316, '2.616665769460188'], [1706117317, '2.661283364192357'], [1706117318, '2.705900958924526'], [1706117319, '2.750518553656696'], [1706117320, '2.7951361483888655'], [1706117321, '2.8397537431210353'], [1706117322, '2.8843713378532048'], [1706117323, '2.928988932585375'], [1706117324, '2.9736065273175445'], [1706117325, '3.0182241220497144'], [1706117326, '2.480837658629061'], [1706117327, '2.517210244112489'], [1706117328, '2.5535828295959164'], [1706117329, '2.5899554150793445'], [1706117330, '2.626328000562773'], [1706117331, '2.662700586046201'], [1706117332, '3.2735326935085207'], [1706117333, '3.2735326935085207'], [1706117334, '3.2735326935085207'], [1706117335, '3.2735326935085207'], [1706117336, '3.2735326935085207'], [1706117337, '3.2735326935085207'], [1706117338, '3.2735326935085207'], [1706117339, '3.2735326935085207'], [1706117340, '3.2735326935085207'], [1706117341, '3.2735326935085207'], [1706117342, '3.2735326935085207'], [1706117343, '3.2735326935085207'], [1706117344, '3.2735326935085207'], [1706117345, '3.2735326935085207'], [1706117346, '3.2735326935085207'], [1706117347, '3.2735326935085207'], [1706117348, '3.2735326935085207'], [1706117349, '3.2735326935085207'], [1706117350, '3.2735326935085207'], [1706117351, '3.2735326935085207'], [1706117352, '3.2735326935085207'], [1706117353, '3.2735326935085207'], [1706117354, '3.2735326935085207'], [1706117355, '2.673648734276713'], [1706117356, '3.7319005481816236'], [1706117357, '3.7319005481816236'], [1706117358, '3.7319005481816236'], [1706117359, '3.7319005481816236'], [1706117360, '3.7319005481816245'], [1706117361, '3.7319005481816245'], [1706117362, '3.592445628138214'], [1706117363, '3.592445628138214'], [1706117364, '3.592445628138214'], [1706117365, '3.592445628138214'], [1706117366, '3.592445628138214'], [1706117367, '3.592445628138214'], [1706117368, '3.592445628138214'], [1706117369, '3.592445628138214'], [1706117370, '3.592445628138214'], [1706117371, '3.592445628138214'], [1706117372, '3.592445628138214'], [1706117373, '3.592445628138214'], [1706117374, '3.592445628138214'], [1706117375, '3.592445628138214'], [1706117376, '3.592445628138214'], [1706117377, '3.592445628138214'], [1706117378, '3.592445628138214'], [1706117379, '3.592445628138214'], [1706117380, '3.592445628138214'], [1706117381, '3.592445628138214'], [1706117382, '3.592445628138214'], [1706117383, '3.592445628138214'], [1706117384, '3.592445628138214'], [1706117385, '3.592445628138214'], [1706117386, '3.592445628138214'], [1706117387, '2.9011892361749854'], [1706117388, '2.8612731736401162'], [1706117389, '2.8213571111052476'], [1706117390, '3.57200022999401'], [1706117391, '3.2099744324773454'], [1706117392, '3.2099744324773454'], [1706117393, '3.2099744324773454'], [1706117394, '3.2099744324773454'], [1706117395, '3.2099744324773454'], [1706117396, '3.2099744324773454'], [1706117397, '3.2099744324773454'], [1706117398, '3.2099744324773454'], [1706117399, '3.2099744324773454'], [1706117400, '3.2099744324773454'], [1706117401, '3.2099744324773454'], [1706117402, '3.2099744324773454'], [1706117403, '3.2099744324773454'], [1706117404, '3.2099744324773454'], [1706117405, '3.2099744324773454'], [1706117406, '3.2099744324773454'], [1706117407, '3.5208871213381445'], [1706117408, '3.5208871213381445'], [1706117409, '3.5208871213381445'], [1706117410, '3.5208871213381445'], [1706117411, '3.5208871213381445'], [1706117412, '3.5208871213381445'], [1706117413, '3.5208871213381445'], [1706117414, '3.5208871213381445'], [1706117415, '3.5208871213381445'], [1706117416, '2.3722076083852355'], [1706117417, '2.4045871696667267'], [1706117418, '2.914160515334191'], [1706117419, '2.914160515334191'], [1706117420, '2.914160515334191'], [1706117421, '2.914160515334191'], [1706117422, '2.914160515334191'], [1706117423, '2.914160515334191'], [1706117424, '2.914160515334191'], [1706117425, '2.914160515334191'], [1706117426, '2.914160515334191'], [1706117427, '2.914160515334191'], [1706117428, '2.914160515334191'], [1706117429, '2.914160515334191'], [1706117430, '2.914160515334191'], [1706117431, '2.914160515334191'], [1706117432, '2.914160515334191'], [1706117433, '2.914160515334191'], [1706117434, '2.914160515334191'], [1706117435, '2.914160515334191'], [1706117436, '2.383929009569135'], [1706117437, '2.351549448287644'], [1706117438, '2.319169887006153'], [1706117439, '2.286790325724662'], [1706117440, '3.312709442835095'], [1706117441, '3.312709442835095'], [1706117442, '3.312709442835095'], [1706117443, '3.312709442835095'], [1706117444, '3.312709442835095'], [1706117445, '3.312709442835095'], [1706117446, '3.191276505935721'], [1706117447, '3.237942487846151'], [1706117448, '3.2846084697565816'], [1706117449, '3.331274451667012'], [1706117450, '3.377940433577442'], [1706117451, '3.4246064154878724'], [1706117452, '3.471272397398302'], [1706117453, '4.199938371938713'], [1706117454, '4.199938371938713'], [1706117455, '4.199938371938713'], [1706117456, '4.199938371938713'], [1706117457, '4.199938371938713'], [1706117458, '4.199938371938713'], [1706117459, '4.199938371938713'], [1706117460, '4.199938371938713'], [1706117461, '4.199938371938713'], [1706117462, '4.199938371938713'], [1706117463, '4.199938371938713'], [1706117464, '4.199938371938713'], [1706117465, '4.199938371938713'], [1706117466, '4.199938371938713'], [1706117467, '4.199938371938713'], [1706117468, '3.4633391804735285'], [1706117469, '3.4166731985630983'], [1706117470, '3.370007216652668'], [1706117471, '3.3233412347422386'], [1706117472, '3.2766752528318084'], [1706117473, '3.230009270921378'], [1706117474, '3.183343289010948'], [1706117475, '3.1366773071005176'], [1706117476, '3.0900113251900874'], [1706117477, '3.043345343279658'], [1706117478, '2.9966793613692277'], [1706117479, '3.611361640487743'], [1706117480, '3.611361640487743'], [1706117481, '3.348981988671299'], [1706117482, '3.348981988671299'], [1706117483, '3.348981988671299'], [1706117484, '3.348981988671299'], [1706117485, '3.348981988671299'], [1706117486, '3.348981988671299'], [1706117487, '3.348981988671299'], [1706117488, '3.348981988671299'], [1706117489, '3.348981988671299'], [1706117490, '3.348981988671299'], [1706117491, '3.348981988671299'], [1706117492, '3.348981988671299'], [1706117493, '3.348981988671299'], [1706117494, '3.348981988671299'], [1706117495, '3.348981988671299'], [1706117496, '3.501373337123475'], [1706117497, '2.56696583274599'], [1706117498, '2.6030391162067383'], [1706117499, '2.639112399667487'], [1706117500, '3.246595511467355'], [1706117501, '3.246595511467355'], [1706117502, '3.246595511467355'], [1706117503, '3.246595511467355'], [1706117504, '3.246595511467355'], [1706117505, '3.246595511467355'], [1706117506, '3.246595511467355'], [1706117507, '3.246595511467355'], [1706117508, '3.246595511467355'], [1706117509, '3.246595511467355'], [1706117510, '3.246595511467355'], [1706117511, '3.246595511467355'], [1706117512, '3.246595511467355'], [1706117513, '3.246595511467355'], [1706117514, '3.246595511467355'], [1706117515, '3.246595511467355'], [1706117516, '3.246595511467355'], [1706117517, '3.246595511467355'], [1706117518, '3.246595511467355'], [1706117519, '3.246595511467355'], [1706117520, '3.246595511467355'], [1706117521, '3.246595511467355'], [1706117522, '3.246595511467355'], [1706117523, '3.246595511467355'], [1706117524, '3.246595511467355'], [1706117525, '3.246595511467355'], [1706117526, '3.246595511467355'], [1706117527, '2.6149793730322464'], [1706117528, '3.0222591350532837'], [1706117529, '3.0222591350532837'], [1706117530, '2.2579154362078415'], [1706117531, '2.293004976727566'], [1706117532, '2.3280945172472904'], [1706117533, '2.363184057767015'], [1706117534, '2.3982735982867394'], [1706117535, '2.4333631388064636'], [1706117536, '2.4684526793261883'], [1706117537, '2.503542219845913'], [1706117538, '2.5386317603656376'], [1706117539, '2.5737213008853623'], [1706117540, '2.608810841405086'], [1706117541, '3.158058646775204'], [1706117542, '3.158058646775204'], [1706117543, '3.158058646775204'], [1706117544, '3.158058646775204'], [1706117545, '3.158058646775204'], [1706117546, '3.158058646775204'], [1706117547, '3.158058646775204'], [1706117548, '3.158058646775204'], [1706117549, '3.158058646775204'], [1706117550, '3.158058646775204'], [1706117551, '3.158058646775204'], [1706117552, '3.158058646775204'], [1706117553, '3.158058646775204'], [1706117554, '3.158058646775204'], [1706117555, '2.631934848607585'], [1706117556, '2.59684530808786'], [1706117557, '2.5617557675681355'], [1706117558, '2.5266662270484117'], [1706117559, '2.491576686528687'], [1706117560, '2.456487146008963'], [1706117561, '2.421397605489238'], [1706117562, '2.386308064969514'], [1706117563, '2.351218524449789'], [1706117564, '2.3161289839300645'], [1706117565, '2.2810394434103403'], [1706117566, '3.155680548976084'], [1706117567, '3.155680548976084'], [1706117568, '3.155680548976084'], [1706117569, '2.915342155760893'], [1706117570, '2.915342155760893'], [1706117571, '2.915342155760893'], [1706117572, '2.915342155760893'], [1706117573, '2.915342155760893'], [1706117574, '2.915342155760893'], [1706117575, '2.915342155760893'], [1706117576, '2.915342155760893'], [1706117577, '2.915342155760893'], [1706117578, '2.915342155760893'], [1706117579, '2.915342155760893'], [1706117580, '2.915342155760893'], [1706117581, '2.915342155760893'], [1706117582, '2.915342155760893'], [1706117583, '2.915342155760893'], [1706117584, '2.915342155760893'], [1706117585, '2.915342155760893'], [1706117586, '3.1526011359717856'], [1706117587, '3.1526011359717856'], [1706117588, '3.1526011359717856'], [1706117589, '3.1526011359717856'], [1706117590, '3.1526011359717856'], [1706117591, '3.1526011359717856'], [1706117592, '3.1526011359717856'], [1706117593, '3.1526011359717856'], [1706117594, '3.1526011359717856'], [1706117595, '3.1526011359717856'], [1706117596, '3.1526011359717856'], [1706117597, '3.1526011359717856'], [1706117598, '2.884683472923464'], [1706117599, '2.884683472923464'], [1706117600, '2.884683472923464'], [1706117601, '2.884683472923464'], [1706117602, '2.884683472923464'], [1706117603, '2.884683472923464'], [1706117604, '2.884683472923464'], [1706117605, '2.884683472923464'], [1706117606, '2.884683472923464'], [1706117607, '2.884683472923464'], [1706117608, '2.884683472923464'], [1706117609, '2.884683472923464'], [1706117610, '2.884683472923464'], [1706117611, '2.884683472923464'], [1706117612, '2.884683472923464'], [1706117613, '2.884683472923464'], [1706117614, '2.884683472923464'], [1706117615, '2.884683472923464'], [1706117616, '2.884683472923464'], [1706117617, '2.884683472923464'], [1706117618, '1.9315586770085436'], [1706117619, '1.9600041099286738'], [1706117620, '1.9884495428488045'], [1706117621, '2.0168949757689347'], [1706117622, '2.5600889628117476'], [1706117623, '2.5600889628117476'], [1706117624, '2.5600889628117476'], [1706117625, '2.5600889628117476'], [1706117626, '2.5600889628117476'], [1706117627, '2.5600889628117476'], [1706117628, '1.8256912498379172'], [1706117629, '1.8256912498379172'], [1706117630, '1.8256912498379172'], [1706117631, '1.8256912498379172'], [1706117632, '1.8256912498379172'], [1706117633, '1.8256912498379172'], [1706117634, '1.8256912498379172'], [1706117635, '1.8256912498379172'], [1706117636, '1.8256912498379172'], [1706117637, '1.8256912498379172'], [1706117638, '1.8256912498379172'], [1706117639, '1.8256912498379172'], [1706117640, '1.8256912498379172'], [1706117641, '1.8256912498379172'], [1706117642, '1.8256912498379172'], [1706117643, '1.8256912498379172'], [1706117644, '1.8256912498379172'], [1706117645, '1.8256912498379172'], [1706117646, '1.8256912498379172'], [1706117647, '1.8256912498379172'], [1706117648, '1.8256912498379172'], [1706117649, '1.8256912498379172'], [1706117650, '1.8256912498379172'], [1706117651, '1.8256912498379172'], [1706117652, '1.8256912498379172'], [1706117653, '1.8256912498379172'], [1706117654, '1.8256912498379172'], [1706117655, '1.8256912498379172'], [1706117656, '1.0547932385714336'], [1706117657, '1.0547932385714336'], [1706117658, '1.0547932385714336'], [1706117659, '1.0547932385714336'], [1706117660, '1.7007816403976095'], [1706117661, '1.7007816403976095'], [1706117662, '1.7007816403976097'], [1706117663, '1.7007816403976097'], [1706117664, '1.7007816403976097'], [1706117665, '1.7007816403976097'], [1706117666, '1.7007816403976097'], [1706117667, '1.7007816403976097'], [1706117668, '1.7007816403976097'], [1706117669, '1.7007816403976097'], [1706117670, '1.7007816403976097'], [1706117671, '1.7007816403976097'], [1706117672, '1.7007816403976097'], [1706117673, '1.7007816403976097'], [1706117674, '1.7007816403976097'], [1706117675, '1.7007816403976097'], [1706117676, '1.7007816403976097'], [1706117677, '1.7007816403976097'], [1706117678, '1.7007816403976097'], [1706117679, '1.7007816403976097'], [1706117680, '1.7007816403976097'], [1706117681, '1.7007816403976097'], [1706117682, '1.7007816403976097'], [1706117683, '1.7007816403976097'], [1706117684, '1.7007816403976097'], [1706117685, '1.7007816403976097'], [1706117686, '1.7007816403976097'], [1706117687, '1.7007816403976097'], [1706117688, '2.318687160130923'], [1706117689, '2.0919816211073194'], [1706117690, '2.0919816211073194'], [1706117691, '2.0919816211073194'], [1706117692, '2.0919816211073194'], [1706117693, '2.0919816211073194'], [1706117694, '2.0919816211073194'], [1706117695, '2.0919816211073194'], [1706117696, '2.0919816211073194'], [1706117697, '2.0919816211073194'], [1706117698, '2.0919816211073194'], [1706117699, '2.0919816211073194'], [1706117700, '2.0919816211073194'], [1706117701, '2.0919816211073194'], [1706117702, '2.0919816211073194'], [1706117703, '2.0919816211073194'], [1706117704, '2.0919816211073194'], [1706117705, '2.0919816211073194'], [1706117706, '2.0919816211073194'], [1706117707, '2.0919816211073194'], [1706117708, '1.9021084940746573'], [1706117709, '1.9021084940746573'], [1706117710, '1.9021084940746573'], [1706117711, '1.9021084940746573'], [1706117712, '1.9021084940746573'], [1706117713, '1.9021084940746573'], [1706117714, '1.9021084940746573'], [1706117715, '1.9021084940746573'], [1706117716, '1.9021084940746573'], [1706117717, '1.9021084940746573'], [1706117718, '1.2880315559128186'], [1706117719, '1.3061593772274451'], [1706117720, '1.3242871985420714'], [1706117721, '1.342415019856698'], [1706117722, '1.3605428411713243'], [1706117723, '1.6315039183163735'], [1706117724, '1.6315039183163735'], [1706117725, '1.6315039183163735'], [1706117726, '1.6315039183163735'], [1706117727, '1.6315039183163735'], [1706117728, '1.6315039183163735'], [1706117729, '1.6315039183163735'], [1706117730, '1.6315039183163735'], [1706117731, '1.6315039183163735'], [1706117732, '1.6315039183163735'], [1706117733, '1.6315039183163735'], [1706117734, '1.6315039183163735'], [1706117735, '1.3525484719715741'], [1706117736, '1.3344206506569478'], [1706117737, '1.3162928293423213'], [1706117738, '1.2981650080276952'], [1706117739, '1.2800371867130684'], [1706117740, '1.2619093653984426'], [1706117741, '1.243781544083816'], [1706117742, '1.2256537227691895'], [1706117743, '1.2075259014545632'], [1706117744, '1.1893980801399369'], [1706117745, '1.1712702588253103'], [1706117746, '1.153142437510684'], [1706117747, '1.1350146161960577'], [1706117748, '1.1168867948814314'], [1706117749, '1.0987589735668049'], [1706117750, '1.0898148498594316'], [1706117751, '1.0898148498594316'], [1706117752, '1.0898148498594316'], [1706117753, '1.0898148498594316'], [1706117754, '1.0898148498594316'], [1706117755, '1.0898148498594316'], [1706117756, '1.0898148498594316'], [1706117757, '1.0898148498594316'], [1706117758, '1.0898148498594316'], [1706117759, '1.0898148498594316'], [1706117760, '1.0898148498594316'], [1706117761, '1.0898148498594316'], [1706117762, '1.0898148498594316'], [1706117763, '1.0898148498594316'], [1706117764, '1.0898148498594316'], [1706117765, '1.0898148498594316'], [1706117766, '1.0898148498594316'], [1706117767, '1.0898148498594316'], [1706117768, '1.0898148498594316'], [1706117769, '1.0898148498594316'], [1706117770, '1.0898148498594316'], [1706117771, '1.0898148498594316'], [1706117772, '1.0898148498594316'], [1706117773, '1.0898148498594316'], [1706117774, '1.0898148498594316'], [1706117775, '1.0898148498594316'], [1706117776, '1.0898148498594316'], [1706117777, '1.0622381643896492'], [1706117778, '1.0622381643896492'], [1706117779, '0.9999662272661242'], [1706117780, '0.9999662272661242'], [1706117781, '0.9999662272661242'], [1706117782, '0.9999662272661242'], [1706117783, '0.9999662272661242'], [1706117784, '0.9999662272661242'], [1706117785, '0.9999662272661242'], [1706117786, '0.9999662272661242'], [1706117787, '0.9999662272661242'], [1706117788, '0.9999662272661242'], [1706117789, '0.9999662272661242'], [1706117790, '0.9999662272661242'], [1706117791, '0.9999662272661242'], [1706117792, '0.9999662272661242'], [1706117793, '0.9999662272661242'], [1706117794, '0.9999662272661242'], [1706117795, '0.9999662272661242'], [1706117796, '0.9999662272661242'], [1706117797, '0.9999662272661242'], [1706117798, '0.6893592223628728'], [1706117799, '0.7004700344053406'], [1706117800, '0.7115808464478085'], [1706117801, '0.7226916584902763'], [1706117802, '0.7338024705327442'], [1706117803, '0.7449132825752118'], [1706117804, '0.7560240946176797'], [1706117805, '0.7671349066601475'], [1706117806, '0.5986061095999958'], [1706117807, '0.7985815694614367'], [1706117808, '0.9996883117547429'], [1706117809, '0.9996883117547429'], [1706117810, '0.9996883117547429'], [1706117811, '0.9996883117547429'], [1706117812, '0.9996883117547429'], [1706117813, '0.9996883117547429'], [1706117814, '0.9996883117547429'], [1706117815, '0.9996883117547427'], [1706117816, '0.9996883117547427'], [1706117817, '0.9996883117547427'], [1706117818, '0.9996883117547427'], [1706117819, '0.9996883117547427'], [1706117820, '0.9996883117547427'], [1706117821, '0.9996883117547427'], [1706117822, '0.9996883117547427'], [1706117823, '0.9996883117547427'], [1706117824, '0.9996883117547427'], [1706117825, '0.9996883117547427'], [1706117826, '0.9996883117547427'], [1706117827, '0.9996883117547427'], [1706117828, '0.9996883117547427'], [1706117829, '0.9996883117547427'], [1706117830, '0.9996883117547427'], [1706117831, '0.9996883117547427'], [1706117832, '0.9996883117547427'], [1706117833, '0.9996883117547427'], [1706117834, '0.9996883117547427'], [1706117835, '0.9993381495398761'], [1706117836, '0.9993381495398761'], [1706117837, '0.9993381495398761'], [1706117838, '0.9993381495398761'], [1706117839, '0.9993381495398761'], [1706117840, '0.9990423476012606'], [1706117841, '0.9990423476012606'], [1706117842, '0.9990423476012606'], [1706117843, '0.9990423476012606'], [1706117844, '0.9990423476012606'], [1706117845, '0.9990423476012606'], [1706117846, '0.9990423476012606'], [1706117847, '0.9990423476012606'], [1706117848, '0.9990423476012606'], [1706117849, '0.9990423476012606'], [1706117850, '0.9990423476012606'], [1706117851, '0.9990423476012606'], [1706117852, '0.9990423476012606'], [1706117853, '0.9990423476012606'], [1706117854, '0.9990423476012606'], [1706117855, '0.9990423476012606'], [1706117856, '0.9990423476012606'], [1706117857, '0.9990423476012606'], [1706117858, '0.9990423476012606'], [1706117859, '0.9990423476012606'], [1706117860, '0.9990423476012606'], [1706117861, '0.9990423476012606'], [1706117862, '0.9990423476012606'], [1706117863, '0.9889619143389414'], [1706117864, '0.9889619143389414'], [1706117865, '0.9889619143389414'], [1706117866, '0.9889619143389414'], [1706117867, '0.9832407125162015'], [1706117868, '0.9832407125162015'], [1706117869, '0.9832407125162015'], [1706117870, '0.9832407125162015'], [1706117871, '0.9832407125162015'], [1706117872, '0.9832407125162015'], [1706117873, '0.9832407125162015'], [1706117874, '0.9832407125162015'], [1706117875, '0.9832407125162015'], [1706117876, '0.9832407125162015'], [1706117877, '0.9832407125162015'], [1706117878, '0.9832407125162015'], [1706117879, '0.9832407125162015'], [1706117880, '0.9832407125162015'], [1706117881, '0.9832407125162015'], [1706117882, '0.9832407125162015'], [1706117883, '0.9832407125162015'], [1706117884, '0.9832407125162015'], [1706117885, '0.9832407125162015'], [1706117886, '0.9832407125162015'], [1706117887, '0.9832407125162015'], [1706117888, '0.9832407125162015'], [1706117889, '0.9832407125162017'], [1706117890, '0.9832407125162017'], [1706117891, '0.9832407125162017'], [1706117892, '0.9832407125162017'], [1706117893, '1.0642434901894462'], [1706117894, '1.0642434901894462'], [1706117895, '1.0642434901894462'], [1706117896, '1.0642434901894462'], [1706117897, '1.0952123724224387'], [1706117898, '1.0952123724224387'], [1706117899, '1.0952123724224387'], [1706117900, '1.0952123724224387'], [1706117901, '1.0952123724224387'], [1706117902, '1.0952123724224387'], [1706117903, '1.0952123724224387'], [1706117904, '1.0952123724224387'], [1706117905, '1.0952123724224387'], [1706117906, '1.0952123724224387'], [1706117907, '1.0952123724224387'], [1706117908, '1.0952123724224387'], [1706117909, '1.0952123724224387'], [1706117910, '1.0952123724224387'], [1706117911, '1.0952123724224387'], [1706117912, '1.0952123724224387'], [1706117913, '1.0952123724224387'], [1706117914, '1.0952123724224387'], [1706117915, '1.0952123724224387'], [1706117916, '1.0952123724224387'], [1706117917, '1.0952123724224387'], [1706117918, '1.0952123724224387'], [1706117919, '1.1850365840831112'], [1706117920, '1.1850365840831112'], [1706117921, '1.1850365840831112'], [1706117922, '1.1850365840831112'], [1706117923, '1.1850365840831112'], [1706117924, '1.1850365840831112'], [1706117925, '1.2922856899894286'], [1706117926, '1.2922856899894286'], [1706117927, '1.2922856899894286'], [1706117928, '1.2922856899894286'], [1706117929, '1.2922856899894286'], [1706117930, '1.2922856899894286'], [1706117931, '1.2922856899894286'], [1706117932, '1.2922856899894286'], [1706117933, '1.2922856899894286'], [1706117934, '1.2922856899894286'], [1706117935, '1.2922856899894286'], [1706117936, '1.2922856899894286'], [1706117937, '1.2922856899894286'], [1706117938, '1.2922856899894286'], [1706117939, '1.2922856899894286'], [1706117940, '1.2922856899894286'], [1706117941, '1.2922856899894286'], [1706117942, '1.2922856899894286'], [1706117943, '1.2922856899894286'], [1706117944, '1.2922856899894286'], [1706117945, '1.2922856899894286'], [1706117946, '1.2922856899894286'], [1706117947, '2.0539061765637032'], [1706117948, '2.0539061765637032'], [1706117949, '2.0539061765637032'], [1706117950, '2.0539061765637032'], [1706117951, '2.0539061765637032'], [1706117952, '2.0539061765637032'], [1706117953, '2.5196023966957344'], [1706117954, '2.5196023966957344'], [1706117955, '2.5196023966957344'], [1706117956, '2.5196023966957344'], [1706117957, '2.5196023966957344'], [1706117958, '2.5196023966957344'], [1706117959, '2.5196023966957344'], [1706117960, '2.5196023966957344'], [1706117961, '2.5196023966957344'], [1706117962, '2.5196023966957344'], [1706117963, '2.5196023966957344'], [1706117964, '2.5196023966957344'], [1706117965, '2.5196023966957344'], [1706117966, '2.5196023966957344'], [1706117967, '2.5196023966957344'], [1706117968, '2.5196023966957344'], [1706117969, '2.5196023966957344'], [1706117970, '2.5196023966957344'], [1706117971, '2.5196023966957344'], [1706117972, '2.5196023966957344'], [1706117973, '2.5196023966957344'], [1706117974, '2.5196023966957344'], [1706117975, '2.5196023966957344'], [1706117976, '2.5196023966957344'], [1706117977, '2.058284194547385'], [1706117978, '2.0302886123618764'], [1706117979, '2.0022930301763684'], [1706117980, '1.9742974479908604'], [1706117981, '1.9463018658053521'], [1706117982, '5.694601278566396'], [1706117983, '7.517450980584689'], [1706117984, '7.517450980584689'], [1706117985, '7.517450980584689'], [1706117986, '7.517450980584689'], [1706117987, '7.517450980584689'], [1706117988, '7.517450980584689'], [1706117989, '7.517450980584689'], [1706117990, '7.517450980584689'], [1706117991, '7.517450980584689'], [1706117992, '7.517450980584689'], [1706117993, '7.517450980584689'], [1706117994, '7.517450980584689'], [1706117995, '7.517450980584689'], [1706117996, '7.517450980584689'], [1706117997, '7.517450980584689'], [1706117998, '7.517450980584689'], [1706117999, '7.517450980584689'], [1706118000, '7.517450980584689'], [1706118001, '7.517450980584689'], [1706118002, '7.517450980584689'], [1706118003, '7.517450980584689'], [1706118004, '7.517450980584689'], [1706118005, '7.517450980584689'], [1706118006, '7.517450980584689'], [1706118007, '7.517450980584689'], [1706118008, '7.517450980584689'], [1706118009, '10.647297690281198'], [1706118010, '10.647297690281198'], [1706118011, '10.647297690281198'], [1706118012, '10.647297690281198'], [1706118013, '10.647297690281198'], [1706118014, '10.647297690281198'], [1706118015, '10.647297690281198'], [1706118016, '10.647297690281198'], [1706118017, '10.647297690281198'], [1706118018, '10.63733514331065'], [1706118019, '10.63733514331065'], [1706118020, '10.63733514331065'], [1706118021, '10.637335143310652'], [1706118022, '10.637335143310652'], [1706118023, '10.637335143310652'], [1706118024, '10.637335143310652'], [1706118025, '10.637335143310652'], [1706118026, '10.637335143310652'], [1706118027, '10.637335143310652'], [1706118028, '10.637335143310652'], [1706118029, '10.637335143310652'], [1706118030, '10.637335143310652'], [1706118031, '10.637335143310652'], [1706118032, '10.637335143310652'], [1706118033, '10.637335143310652'], [1706118034, '10.63733514331065'], [1706118035, '10.63733514331065'], [1706118036, '10.63733514331065'], [1706118037, '10.627572753016432'], [1706118038, '10.627572753016432'], [1706118039, '10.627572753016432'], [1706118040, '10.627572753016432'], [1706118041, '10.627572753016432'], [1706118042, '10.627572753016432'], [1706118043, '10.627572753016432'], [1706118044, '10.627572753016432'], [1706118045, '10.627572753016432'], [1706118046, '8.369048497709652'], [1706118047, '8.369048497709652'], [1706118048, '8.369048497709652'], [1706118049, '8.369048497709652'], [1706118050, '8.369048497709652'], [1706118051, '8.369048497709652'], [1706118052, '8.369048497709652'], [1706118053, '8.369048497709652'], [1706118054, '8.369048497709652'], [1706118055, '8.369048497709652'], [1706118056, '8.369048497709652'], [1706118057, '8.369048497709652'], [1706118058, '8.369048497709652'], [1706118059, '8.369048497709652'], [1706118060, '8.369048497709652'], [1706118061, '8.369048497709652'], [1706118062, '8.369048497709652'], [1706118063, '8.369048497709652'], [1706118064, '8.369048497709652'], [1706118065, '8.369048497709652'], [1706118066, '8.369048497709652'], [1706118067, '8.369048497709652'], [1706118068, '8.369048497709652'], [1706118069, '8.369048497709652'], [1706118070, '8.369048497709652'], [1706118071, '8.369048497709652'], [1706118072, '4.195845830049149'], [1706118073, '4.257017788907441'], [1706118074, '4.318189747765734'], [1706118075, '4.379361706624026'], [1706118076, '4.440533665482319'], [1706118077, '4.169829008752811'], [1706118078, '4.169829008752811'], [1706118079, '4.169829008752811'], [1706118080, '4.169829008752811'], [1706118081, '4.169829008752811'], [1706118082, '4.169829008752811'], [1706118083, '4.169829008752811'], [1706118084, '4.169829008752811'], [1706118085, '4.169829008752811'], [1706118086, '4.169829008752811'], [1706118087, '4.169829008752811'], [1706118088, '4.169829008752811'], [1706118089, '4.169829008752811'], [1706118090, '4.169829008752811'], [1706118091, '4.169829008752811'], [1706118092, '4.169829008752811'], [1706118093, '4.169829008752811'], [1706118094, '4.169829008752811'], [1706118095, '4.169829008752811'], [1706118096, '4.169829008752811'], [1706118097, '4.169829008752811'], [1706118098, '4.169829008752811'], [1706118099, '4.169829008752811'], [1706118100, '4.169829008752811'], [1706118101, '4.169829008752811'], [1706118102, '4.169829008752811'], [1706118103, '4.169829008752811'], [1706118104, '4.169829008752811'], [1706118105, '4.169829008752811'], [1706118106, '4.169829008752811'], [1706118107, '4.169829008752811'], [1706118108, '2.9435952099229175'], [1706118109, '2.9435952099229175'], [1706118110, '2.338359328149322'], [1706118111, '2.3056527147057344'], [1706118112, '2.2729461012621464'], [1706118113, '3.2768392754600044'], [1706118114, '3.2768392754600044'], [1706118115, '3.2768392754600044'], [1706118116, '3.2768392754600044'], [1706118117, '3.2768392754600044'], [1706118118, '3.2768392754600044'], [1706118119, '3.2768392754600044'], [1706118120, '3.2768392754600044'], [1706118121, '3.2768392754600044'], [1706118122, '3.2768392754600044'], [1706118123, '3.2768392754600044'], [1706118124, '3.2768392754600044'], [1706118125, '3.2768392754600044'], [1706118126, '3.2768392754600044'], [1706118127, '3.2768392754600044'], [1706118128, '3.2768392754600044'], [1706118129, '3.2768392754600044'], [1706118130, '3.2768392754600044'], [1706118131, '3.2768392754600044'], [1706118132, '3.2768392754600044'], [1706118133, '3.2768392754600044'], [1706118134, '3.2768392754600044'], [1706118135, '3.2768392754600044'], [1706118136, '3.5614226988089612'], [1706118137, '3.5614226988089612'], [1706118138, '3.5614226988089612'], [1706118139, '3.5614226988089612'], [1706118140, '3.5614226988089612'], [1706118141, '5.393769501363211'], [1706118142, '5.393769501363211'], [1706118143, '5.393769501363211'], [1706118144, '5.393769501363211'], [1706118145, '5.393769501363211'], [1706118146, '5.393769501363211'], [1706118147, '5.393769501363211'], [1706118148, '5.393769501363211'], [1706118149, '5.393769501363211'], [1706118150, '5.393769501363211'], [1706118151, '5.393769501363211'], [1706118152, '5.393769501363211'], [1706118153, '5.393769501363211'], [1706118154, '5.393769501363211'], [1706118155, '5.393769501363211'], [1706118156, '5.393769501363211'], [1706118157, '5.393769501363211'], [1706118158, '5.393769501363211'], [1706118159, '5.393769501363211'], [1706118160, '5.393769501363211'], [1706118161, '5.393769501363211'], [1706118162, '5.393769501363211'], [1706118163, '5.393769501363211'], [1706118164, '5.393769501363211'], [1706118165, '5.393769501363211'], [1706118166, '5.393769501363211'], [1706118167, '5.9313093024429815'], [1706118168, '6.0173370094686165'], [1706118169, '6.103364716494252'], [1706118170, '6.189392423519888'], [1706118171, '7.7582975831536505'], [1706118172, '7.7582975831536505'], [1706118173, '7.7582975831536505'], [1706118174, '7.7582975831536505'], [1706118175, '7.7582975831536505'], [1706118176, '7.7582975831536505'], [1706118177, '7.7582975831536505'], [1706118178, '7.7582975831536505'], [1706118179, '7.7582975831536505'], [1706118180, '7.7582975831536505'], [1706118181, '7.7582975831536505'], [1706118182, '7.7582975831536505'], [1706118183, '7.7582975831536505'], [1706118184, '7.7582975831536505'], [1706118185, '7.7582975831536505'], [1706118186, '7.7582975831536505'], [1706118187, '7.7582975831536505'], [1706118188, '7.7582975831536505'], [1706118189, '7.7582975831536505'], [1706118190, '7.7582975831536505'], [1706118191, '7.7582975831536505'], [1706118192, '7.7582975831536505'], [1706118193, '7.7582975831536505'], [1706118194, '7.7582975831536505'], [1706118195, '7.7582975831536505'], [1706118196, '7.7582975831536505'], [1706118197, '7.7582975831536505'], [1706118198, '7.7582975831536505'], [1706118199, '7.7582975831536505'], [1706118200, '7.7582975831536505'], [1706118201, '7.7582975831536505'], [1706118202, '8.375670614675203'], [1706118203, '8.664708104798436'], [1706118204, '8.664708104798436'], [1706118205, '8.664708104798436'], [1706118206, '8.664708104798436'], [1706118207, '8.664708104798436'], [1706118208, '8.664708104798436'], [1706118209, '8.664708104798436'], [1706118210, '8.664708104798436'], [1706118211, '8.664708104798436'], [1706118212, '8.664708104798436'], [1706118213, '8.664708104798436'], [1706118214, '8.664708104798436'], [1706118215, '8.664708104798436'], [1706118216, '8.664708104798436'], [1706118217, '8.664708104798436'], [1706118218, '8.664708104798436'], [1706118219, '8.664708104798436'], [1706118220, '8.664708104798436'], [1706118221, '9.24542095074573'], [1706118222, '9.24542095074573'], [1706118223, '9.24542095074573'], [1706118224, '9.24542095074573'], [1706118225, '9.24542095074573'], [1706118226, '9.24542095074573'], [1706118227, '9.24542095074573'], [1706118228, '9.24542095074573'], [1706118229, '9.24542095074573'], [1706118230, '9.24542095074573'], [1706118231, '8.274444419487123'], [1706118232, '8.38693809827044'], [1706118233, '10.124431090498456'], [1706118234, '10.124431090498456'], [1706118235, '10.124431090498456'], [1706118236, '10.124431090498456'], [1706118237, '10.124431090498456'], [1706118238, '10.124431090498456'], [1706118239, '10.124431090498456'], [1706118240, '10.124431090498456'], [1706118241, '10.124431090498456'], [1706118242, '10.124431090498456'], [1706118243, '10.124431090498456'], [1706118244, '10.124431090498456'], [1706118245, '10.124431090498456'], [1706118246, '10.124431090498456'], [1706118247, '10.124431090498454'], [1706118248, '8.417648872578287'], [1706118249, '8.30515519379497'], [1706118250, '8.192661515011656'], [1706118251, '8.080167836228338'], [1706118252, '7.967674157445021'], [1706118253, '7.855180478661706'], [1706118254, '7.74268679987839'], [1706118255, '7.630193121095074'], [1706118256, '7.5176994423117565'], [1706118257, '7.405205763528442'], [1706118258, '7.292712084745125'], [1706118259, '7.180218405961807'], [1706118260, '7.067724727178492'], [1706118261, '4.691624081399995'], [1706118262, '4.691624081399995'], [1706118263, '11.46968159166063'], [1706118264, '11.46968159166063'], [1706118265, '11.46968159166063'], [1706118266, '11.46968159166063'], [1706118267, '11.46968159166063'], [1706118268, '11.46968159166063'], [1706118269, '11.46968159166063'], [1706118270, '11.46968159166063'], [1706118271, '11.46968159166063'], [1706118272, '11.46968159166063'], [1706118273, '11.46968159166063'], [1706118274, '11.46968159166063'], [1706118275, '11.46968159166063'], [1706118276, '11.469681591660628'], [1706118277, '11.469681591660628'], [1706118278, '11.48298965769115'], [1706118279, '11.48298965769115'], [1706118280, '11.48298965769115'], [1706118281, '11.48298965769115'], [1706118282, '11.48298965769115'], [1706118283, '11.48298965769115'], [1706118284, '11.48298965769115'], [1706118285, '11.48298965769115'], [1706118286, '11.48298965769115'], [1706118287, '11.48298965769115'], [1706118288, '11.48298965769115'], [1706118289, '11.48298965769115'], [1706118290, '11.48298965769115'], [1706118291, '11.48298965769115'], [1706118292, '11.603810729464888'], [1706118293, '11.603810729464888'], [1706118294, '11.603810729464888'], [1706118295, '11.603810729464888'], [1706118296, '11.603810729464888'], [1706118297, '11.603810729464888'], [1706118298, '11.603810729464888'], [1706118299, '11.603810729464888'], [1706118300, '11.603810729464888'], [1706118301, '11.603810729464888'], [1706118302, '11.603810729464888'], [1706118303, '11.603810729464888'], [1706118304, '11.603810729464888'], [1706118305, '11.603810729464888'], [1706118306, '11.603810729464888'], [1706118307, '11.603810729464888'], [1706118308, '11.603810729464888'], [1706118309, '9.372849185494934'], [1706118310, '9.243917955167548'], [1706118311, '3.949989036377782'], [1706118312, '3.949989036377782'], [1706118313, '3.949989036377782'], [1706118314, '3.949989036377782'], [1706118315, '3.949989036377782'], [1706118316, '3.949989036377782'], [1706118317, '3.949989036377782'], [1706118318, '3.949989036377782'], [1706118319, '3.949989036377782'], [1706118320, '3.949989036377782'], [1706118321, '3.949989036377782'], [1706118322, '11.531323979311384'], [1706118323, '11.531323979311384'], [1706118324, '11.531323979311384'], [1706118325, '11.531323979311384'], [1706118326, '11.531323979311384'], [1706118327, '11.531323979311384'], [1706118328, '11.531323979311384'], [1706118329, '11.531323979311384'], [1706118330, '11.531323979311384'], [1706118331, '11.531323979311384'], [1706118332, '11.531323979311384'], [1706118333, '11.531323979311384'], [1706118334, '11.531323979311384'], [1706118335, '11.526537466242566'], [1706118336, '11.526537466242566'], [1706118337, '11.526537466242566'], [1706118338, '11.526537466242566'], [1706118339, '11.526537466242566'], [1706118340, '11.526537466242566'], [1706118341, '11.526537466242566'], [1706118342, '11.526537466242566'], [1706118343, '11.526537466242566'], [1706118344, '11.526537466242566'], [1706118345, '11.526537466242566'], [1706118346, '11.526537466242566'], [1706118347, '11.526537466242566'], [1706118348, '11.526537466242566'], [1706118349, '11.526537466242566'], [1706118350, '11.526537466242566'], [1706118351, '11.526537466242566'], [1706118352, '11.526537466242566'], [1706118353, '11.524128179592621'], [1706118354, '11.524128179592621'], [1706118355, '11.524128179592621'], [1706118356, '11.524128179592621'], [1706118357, '11.524128179592621'], [1706118358, '11.524128179592621'], [1706118359, '11.524128179592621'], [1706118360, '11.524128179592621'], [1706118361, '11.524128179592621'], [1706118362, '11.524128179592621'], [1706118363, '11.524128179592621'], [1706118364, '11.524128179592621'], [1706118365, '11.524128179592621'], [1706118366, '9.28796714074417'], [1706118367, '9.159921272082029'], [1706118368, '3.249779503666671'], [1706118369, '3.249779503666671'], [1706118370, '3.249779503666671'], [1706118371, '3.249779503666671'], [1706118372, '3.249779503666671'], [1706118373, '3.249779503666671'], [1706118374, '3.249779503666671'], [1706118375, '3.249779503666671'], [1706118376, '3.249779503666671'], [1706118377, '3.249779503666671'], [1706118378, '3.249779503666671'], [1706118379, '3.249779503666671'], [1706118380, '3.249779503666671'], [1706118381, '7.037526331434339'], [1706118382, '7.037526331434339'], [1706118383, '7.037526331434339'], [1706118384, '7.037526331434339'], [1706118385, '7.037526331434339'], [1706118386, '7.037526331434339'], [1706118387, '7.037526331434339'], [1706118388, '7.037526331434339'], [1706118389, '7.037526331434339'], [1706118390, '7.037526331434339'], [1706118391, '7.037526331434339'], [1706118392, '7.037526331434339'], [1706118393, '7.037526331434339'], [1706118394, '7.037526331434339'], [1706118395, '7.037526331434339'], [1706118396, '7.037526331434339'], [1706118397, '7.037526331434339'], [1706118398, '7.037526331434339'], [1706118399, '7.037526331434339'], [1706118400, '7.037526331434339'], [1706118401, '7.037526331434339'], [1706118402, '7.037526331434339'], [1706118403, '7.037526331434339'], [1706118404, '7.037526331434339'], [1706118405, '7.037526331434339'], [1706118406, '5.248563086647389'], [1706118407, '5.248563086647389'], [1706118408, '5.248563086647389'], [1706118409, '5.248563086647389'], [1706118410, '5.248563086647389'], [1706118411, '5.248563086647389'], [1706118412, '4.128116379389054'], [1706118413, '4.128116379389054'], [1706118414, '4.128116379389054'], [1706118415, '4.128116379389054'], [1706118416, '4.128116379389054'], [1706118417, '4.128116379389054'], [1706118418, '4.128116379389054'], [1706118419, '4.128116379389054'], [1706118420, '4.128116379389054']]}]}} +VALID_NODE_INFO = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_info', 'container': 'kube-state-metrics', 'container_runtime_version': 'containerd://1.7.2', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'internal_ip': '192.168.86.107', 'job': 'kube-state-metrics', 'kernel_version': '5.10.205-195.804.amzn2.x86_64', 'kubelet_version': 'v1.27.9-eks-5e0fdde', 'kubeproxy_version': 'v1.27.9-eks-5e0fdde', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'os_image': 'Amazon Linux 2', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'provider_id': 'aws:///us-east-1c/i-0fe9d9c99fdb3631d', 'service': 'kube-prometheus-stack-kube-state-metrics', 'system_uuid': 'ec253b04-b1dc-f08b-acac-e23df83b3602'}, 'value': [1706117733, '1']}]}} +VALID_NODE_LABELS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_labels', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'label_beta_kubernetes_io_arch': 'amd64', 'label_beta_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_beta_kubernetes_io_os': 'linux', 'label_failure_domain_beta_kubernetes_io_region': 'us-east-1', 'label_failure_domain_beta_kubernetes_io_zone': 'us-east-1c', 'label_k8s_io_cloud_provider_aws': 'ceb9f9cc8e47252a6f7fe7d6bded2655', 'label_karpenter_k8s_aws_instance_category': 'i', 'label_karpenter_k8s_aws_instance_cpu': '24', 'label_karpenter_k8s_aws_instance_encryption_in_transit_supported': 'true', 'label_karpenter_k8s_aws_instance_family': 'i3en', 'label_karpenter_k8s_aws_instance_generation': '3', 'label_karpenter_k8s_aws_instance_hypervisor': 'nitro', 'label_karpenter_k8s_aws_instance_local_nvme': '15000', 'label_karpenter_k8s_aws_instance_memory': '196608', 'label_karpenter_k8s_aws_instance_network_bandwidth': '25000', 'label_karpenter_k8s_aws_instance_pods': '234', 'label_karpenter_k8s_aws_instance_size': '6xlarge', 'label_karpenter_sh_capacity_type': 'spot', 'label_karpenter_sh_initialized': 'true', 'label_karpenter_sh_provisioner_name': 'glr-x86-64-v4', 'label_kubernetes_io_arch': 'amd64', 'label_kubernetes_io_hostname': 'ip-192-168-86-107.ec2.internal', 'label_kubernetes_io_os': 'linux', 'label_node_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_spack_io_pipeline': 'true', 'label_spack_io_x86_64': 'v4', 'label_topology_ebs_csi_aws_com_zone': 'us-east-1c', 'label_topology_kubernetes_io_region': 'us-east-1', 'label_topology_kubernetes_io_zone': 'us-east-1c', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'service': 'kube-prometheus-stack-kube-state-metrics'}, 'value': [1706117733, '1']}]}} + +# modified version of VALID_MEMORY_USAGE to make the mean/stddev 0 +INVALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '0']]}]}} diff --git a/gantry/tests/sql/insert_job.sql b/gantry/tests/sql/insert_job.sql new file mode 100644 index 0000000..3008da6 --- /dev/null +++ b/gantry/tests/sql/insert_job.sql @@ -0,0 +1 @@ +INSERT INTO jobs VALUES(1,'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z',2,1706117046,1706118420,9892514,'success','pr42264_bugfix/mathomp4/hdf5-appleclang15','gmsh','4.8.4','{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',16,0.75,NULL,4.12532286694540495,3.15805864677520409,11.6038107294648877,0.248374361826775191,3.34888880339475214,2000000000.0,48000000000.0,1649868862.72588062,999763968.0,5679742976.0,2785280.0,1378705563.21018671); \ No newline at end of file diff --git a/gantry/tests/sql/insert_node.sql b/gantry/tests/sql/insert_node.sql new file mode 100644 index 0000000..cad50ee --- /dev/null +++ b/gantry/tests/sql/insert_node.sql @@ -0,0 +1,2 @@ +--- primary key is set to 2 to set up the test that checks for race conditions +INSERT INTO nodes VALUES(2,'ec253b04-b1dc-f08b-acac-e23df83b3602','ip-192-168-86-107.ec2.internal',24.0,196608000000.0,'amd64','linux','i3en.6xlarge'); \ No newline at end of file diff --git a/gantry/tests/test_collect.py b/gantry/tests/test_collect.py new file mode 100644 index 0000000..dab580c --- /dev/null +++ b/gantry/tests/test_collect.py @@ -0,0 +1,160 @@ +import pytest + +from gantry.clients.gitlab import GitlabClient +from gantry.clients.prometheus import PrometheusClient +from gantry.routes.collection import fetch_job, fetch_node +from gantry.tests.defs.collection import ( + GHOST_JOB_LOG, + INSERTED_JOB, + INSERTED_NODE, + INVALID_JOB_NAME, + INVALID_JOB_STATUS, + INVALID_MEMORY_USAGE, + INVALID_RUNNER, + VALID_ANNOTATIONS, + VALID_CPU_USAGE, + VALID_JOB, + VALID_JOB_LOG, + VALID_MEMORY_USAGE, + VALID_NODE_INFO, + VALID_NODE_LABELS, + VALID_RESOURCE_LIMITS, + VALID_RESOURCE_REQUESTS, +) + +# mapping of prometheus request shortcuts +# to raw values that would be returned by resp.json() + +# note: the ordering of this dict indicated the order of the calls +# if the order in which PrometheusClient._query is called changes, +# this dict must be updated +PROMETHEUS_REQS = { + "job_annotations": VALID_ANNOTATIONS, + "job_resources": VALID_RESOURCE_REQUESTS, + "job_limits": VALID_RESOURCE_LIMITS, + "job_memory_usage": VALID_MEMORY_USAGE, + "job_cpu_usage": VALID_CPU_USAGE, + "node_info": VALID_NODE_INFO, + "node_labels": VALID_NODE_LABELS, +} + + +@pytest.fixture +async def gitlab(mocker): + """Returns GitlabClient with some default (mocked) behavior""" + + # mock the request to the gitlab api + # default is to return normal log that wouldn't be detected as a ghost job + mocker.patch.object(GitlabClient, "_request", return_value=VALID_JOB_LOG) + return GitlabClient("", "") + + +@pytest.fixture +async def prometheus(mocker): + """Returns PrometheusClient with some default (mocked) behavior""" + + # use dict value iterable to mock multiple calls + mocker.patch.object( + PrometheusClient, "_query", side_effect=PROMETHEUS_REQS.values() + ) + return PrometheusClient("", "") + + +@pytest.mark.parametrize( + "key, value", + [ + ("build_status", INVALID_JOB_STATUS), + ("build_name", INVALID_JOB_NAME), + ("runner", INVALID_RUNNER), + ], +) +async def test_invalid_gitlab_fields(db_conn, gitlab, prometheus, key, value): + """Tests behavior when invalid data from Gitlab is passed to fetch_job""" + payload = VALID_JOB.copy() + payload[key] = value + + assert await fetch_job(payload, db_conn, gitlab, prometheus) is None + + +async def test_job_exists(db_conn): + """ + Tests that fetch_job returns None when the job already exists in the database. + The return value of fetch_job is only used to indicate when a job is inserted, + not if it's found in the database. + """ + # node must be inserted before job to avoid foreign key constraint + with open("gantry/tests/sql/insert_node.sql") as f: + await db_conn.executescript(f.read()) + with open("gantry/tests/sql/insert_job.sql") as f: + await db_conn.executescript(f.read()) + + assert await fetch_job(VALID_JOB, db_conn, None, None) is None + + +async def test_ghost_job(db_conn, gitlab, mocker): + """Tests that a ghost job is detected""" + + mocker.patch.object(gitlab, "_request", return_value=GHOST_JOB_LOG) + assert await fetch_job(VALID_JOB, db_conn, gitlab, None) is None + + +@pytest.mark.parametrize( + "req", + [ + "job_annotations", + "job_resources", + "job_limits", + "job_memory_usage", + "job_cpu_usage", + "node_info", + "node_labels", + ], +) +async def test_missing_data(db_conn, gitlab, prometheus, req): + """Tests behavior when Prometheus data is missing for certain requests""" + + p = PROMETHEUS_REQS.copy() + # for each req in PROMETHEUS_REQS, set it to an empty dict + p[req] = {} + prometheus._query.side_effect = p.values() + assert await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) is None + + +async def test_invalid_usage(db_conn, gitlab, prometheus): + """Test that when resource usage is invalid (eg mean=0), the job is not inserted""" + + p = PROMETHEUS_REQS.copy() + # could also be cpu usage + p["job_memory_usage"] = INVALID_MEMORY_USAGE + prometheus._query.side_effect = p.values() + assert await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) is None + + +async def test_job_node_inserted(db_conn, gitlab, prometheus): + """Tests that the job and node are inserted into the database after a successful fetch_job call""" + + await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) + # as the first records in the database, the ids should be 1 + async with db_conn.execute("SELECT * FROM jobs WHERE id=?", (1,)) as cursor: + job = await cursor.fetchone() + async with db_conn.execute("SELECT * FROM nodes WHERE id=?", (1,)) as cursor: + node = await cursor.fetchone() + assert job == INSERTED_JOB + assert node == INSERTED_NODE + + +async def test_node_exists(db_conn, prometheus): + """Tests that fetch_node returns the existing node id when the node already in database""" + + # when fetch_node is called, only two prometheus requests are made (see comment above PROMETHEUS_REQS) + prometheus._query.side_effect = [ + PROMETHEUS_REQS["node_info"], + PROMETHEUS_REQS["node_labels"], + ] + + # in the inserted row, the node id is 2 because if if the fetch_node call inserts a new node, + # the id would be set to 1 + with open("gantry/tests/sql/insert_node.sql") as f: + await db_conn.executescript(f.read()) + + assert await fetch_node(db_conn, prometheus, None, None) == 2 From d2173190c132bf47b5defdf20970e04112a6106b Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 20:37:45 -0800 Subject: [PATCH 09/23] add tests: db client --- gantry/tests/defs/db.py | 5 +++++ gantry/tests/test_db.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 gantry/tests/defs/db.py create mode 100644 gantry/tests/test_db.py diff --git a/gantry/tests/defs/db.py b/gantry/tests/defs/db.py new file mode 100644 index 0000000..d4e5290 --- /dev/null +++ b/gantry/tests/defs/db.py @@ -0,0 +1,5 @@ +# flake8: noqa +# fmt: off + +# valid input into insert_node +NODE_INSERT_DICT = {"uuid": "ec253b04-b1dc-f08b-acac-e23df83b3602", "hostname": "ip-192-168-86-107.ec2.internal", "cores": 24.0, "mem": 196608000000.0, "arch": "amd64", "os": "linux", "instance_type": "i3en.6xlarge"} diff --git a/gantry/tests/test_db.py b/gantry/tests/test_db.py new file mode 100644 index 0000000..8083299 --- /dev/null +++ b/gantry/tests/test_db.py @@ -0,0 +1,30 @@ +from gantry.db.insert import insert_job, insert_node +from gantry.tests.defs.db import NODE_INSERT_DICT + + +async def test_node_insert_race(db_conn): + """ + Tests the situation where two identical jobs are inserted around the same time. + The first insert should insert the node and the second should get the id of the node. + We differentiate the two inserts by their id, but are identical otherwise. + """ + # the id of this row is 1 + with open("gantry/tests/sql/insert_node.sql") as f: + await db_conn.executescript(f.read()) + + # the id of NODE_INSERT_DICT is 2, but the output should be 1 + assert await insert_node(db_conn, NODE_INSERT_DICT) == 2 + + +async def test_insert_node_incomplete(db_conn): + """ + Tests that when missing data is passed to the insert_node function, it returns None. + Issues around using lastrowid to get the id of the inserted row were returning + old inserted rows, so this ensures that the function returns None when it should. + """ + assert await insert_node(db_conn, {"uuid": None}) is None + + +async def test_insert_job_incomplete(db_conn): + """See test_insert_node_incomplete""" + assert await insert_job(db_conn, {"node": None}) is None From 3b2597cb79e1d0abccefee6b2bcc99bf45e24b6e Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 21:23:15 -0800 Subject: [PATCH 10/23] add config and dependencies for tests --- .gitignore | 2 ++ pyproject.toml | 8 ++++++++ spack.yaml | 4 +++- 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 372e265..944f826 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__ spack.lock .spack-env db/*.db +.coverage +htmlcov \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 4620ee1..fe739ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,3 +2,11 @@ profile = "black" skip_gitignore = true color_output = true + +[tool.pytest.ini_options] +asyncio_mode = "auto" + +[tool.coverage.run] +include = [ + "gantry/*" +] diff --git a/spack.yaml b/spack.yaml index 44863c0..a36b502 100644 --- a/spack.yaml +++ b/spack.yaml @@ -3,7 +3,9 @@ spack: - python - py-aiohttp - py-pytest - - py-pytest-asyncio + - py-pytest-aiohttp + - py-pytest-mock + - py-coverage - py-flake8 - py-black - py-isort From 285b1ca00cf70bb110aff223c49a2cf568197dd5 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 21:42:05 -0800 Subject: [PATCH 11/23] test_collect -> test_collection --- gantry/tests/{test_collect.py => test_collection.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename gantry/tests/{test_collect.py => test_collection.py} (100%) diff --git a/gantry/tests/test_collect.py b/gantry/tests/test_collection.py similarity index 100% rename from gantry/tests/test_collect.py rename to gantry/tests/test_collection.py From f7e9205a7ba87c45a25b812074da39b910b327e1 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 21:47:11 -0800 Subject: [PATCH 12/23] fix: flake8 compliance --- gantry/tests/test_collection.py | 12 +++++++----- gantry/tests/test_db.py | 4 ++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/gantry/tests/test_collection.py b/gantry/tests/test_collection.py index dab580c..149fe97 100644 --- a/gantry/tests/test_collection.py +++ b/gantry/tests/test_collection.py @@ -131,7 +131,7 @@ async def test_invalid_usage(db_conn, gitlab, prometheus): async def test_job_node_inserted(db_conn, gitlab, prometheus): - """Tests that the job and node are inserted into the database after a successful fetch_job call""" + """Tests that the job and node are in the database after calling fetch_node""" await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) # as the first records in the database, the ids should be 1 @@ -144,16 +144,18 @@ async def test_job_node_inserted(db_conn, gitlab, prometheus): async def test_node_exists(db_conn, prometheus): - """Tests that fetch_node returns the existing node id when the node already in database""" + """Tests that fetch_node returns the existing node id when the node + is already in the database""" - # when fetch_node is called, only two prometheus requests are made (see comment above PROMETHEUS_REQS) + # when fetch_node is called, only two prometheus requests are made + # (see comment above PROMETHEUS_REQS) prometheus._query.side_effect = [ PROMETHEUS_REQS["node_info"], PROMETHEUS_REQS["node_labels"], ] - # in the inserted row, the node id is 2 because if if the fetch_node call inserts a new node, - # the id would be set to 1 + # in the inserted row, the node id is 2 because if the fetch_node call + # inserts a new node, the id would be set to 1 with open("gantry/tests/sql/insert_node.sql") as f: await db_conn.executescript(f.read()) diff --git a/gantry/tests/test_db.py b/gantry/tests/test_db.py index 8083299..3dd3736 100644 --- a/gantry/tests/test_db.py +++ b/gantry/tests/test_db.py @@ -5,8 +5,8 @@ async def test_node_insert_race(db_conn): """ Tests the situation where two identical jobs are inserted around the same time. - The first insert should insert the node and the second should get the id of the node. - We differentiate the two inserts by their id, but are identical otherwise. + The first call should insert the node and the second should return the id + of the first. """ # the id of this row is 1 with open("gantry/tests/sql/insert_node.sql") as f: From c59c6bde8ceaf19af2656f93906ebe09b25b012a Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 22:00:44 -0800 Subject: [PATCH 13/23] update github actions deps --- .github/workflows/requirements/unit-tests.txt | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/requirements/unit-tests.txt b/.github/workflows/requirements/unit-tests.txt index 1393afd..39069ed 100644 --- a/.github/workflows/requirements/unit-tests.txt +++ b/.github/workflows/requirements/unit-tests.txt @@ -1,2 +1,5 @@ -pytest==7.4.3 -pytest-asyncio==0.23.2 +pytest==8.0.1 +pytest-aiohttp==1.0.5 +pytest-mock==3.12.0 +aiosqlite==0.20.0 +aiohttp==3.9.3 \ No newline at end of file From a2dd31e493582260dc00cf9d6ff9aeeefe72ed1f Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 22:04:08 -0800 Subject: [PATCH 14/23] update github actions instructions for unit tests --- .github/workflows/unit-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 551403a..2808827 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -22,4 +22,4 @@ jobs: pip install -r .github/workflows/requirements/unit-tests.txt - name: Run Unit Tests with Pytest run: | - pytest + python -m pytest gantry From feee58000494dd67978aad1d36fe7c2ee4452c36 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 22:08:47 -0800 Subject: [PATCH 15/23] python3.11 requirement for unit tests --- .github/workflows/unit-tests.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 2808827..c40fd10 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,19 +7,20 @@ on: jobs: ubuntu: runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.8', '3.11'] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c + + - name: Set up Python 3.11 + uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c with: - python-version: ${{ matrix.python-version }} + python-version: '3.11' cache: 'pip' cache-dependency-path: '.github/workflows/requirements/unit-tests.txt' + - name: Install Python dependencies run: | pip install -r .github/workflows/requirements/unit-tests.txt + - name: Run Unit Tests with Pytest run: | python -m pytest gantry From 235bf5b7721f1c10fbefeda2f91adbfbb759946f Mon Sep 17 00:00:00 2001 From: caetano melone Date: Wed, 21 Feb 2024 22:23:04 -0800 Subject: [PATCH 16/23] fix: avoid extraneous select statement in insert_node --- gantry/db/insert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gantry/db/insert.py b/gantry/db/insert.py index 2829256..e4f0923 100644 --- a/gantry/db/insert.py +++ b/gantry/db/insert.py @@ -45,7 +45,7 @@ async def insert_node(db: aiosqlite.Connection, node: dict) -> int: # this check ensures that something was inserted # and not relying on lastrowid, which could be anything if cursor.rowcount > 0: - pk = cursor.lastrowid + return cursor.lastrowid pk = await get_node(db, node["uuid"]) From f817534f616acf063699e3a49ed621a2930b6a0f Mon Sep 17 00:00:00 2001 From: caetano melone Date: Thu, 22 Feb 2024 14:10:41 -0800 Subject: [PATCH 17/23] reinstate python matrix for unit tests --- .github/workflows/unit-tests.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index c40fd10..df93f09 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -7,13 +7,14 @@ on: jobs: ubuntu: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.11'] steps: - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 - - - name: Set up Python 3.11 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c with: - python-version: '3.11' + python-version: ${{ matrix.python-version }} cache: 'pip' cache-dependency-path: '.github/workflows/requirements/unit-tests.txt' From 92dbe4c9daa34fa28df1ae710ab851cabc1e0520 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Sun, 25 Feb 2024 21:37:09 -0800 Subject: [PATCH 18/23] fix: handle jobs that haven't started or finished --- gantry/models/job.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gantry/models/job.py b/gantry/models/job.py index 3c3a794..5dd8649 100644 --- a/gantry/models/job.py +++ b/gantry/models/job.py @@ -15,8 +15,11 @@ def __init__( self.status = status self.name = name self.gl_id = gl_id - self.start = datetime.fromisoformat(start).timestamp() - self.end = datetime.fromisoformat(end).timestamp() + # handle jobs that haven't started or finished + if start: + self.start = datetime.fromisoformat(start).timestamp() + if end: + self.end = datetime.fromisoformat(end).timestamp() self.ref = ref @property From 306f64d7d520cfaa5162d4dc3f31fcd27d00993b Mon Sep 17 00:00:00 2001 From: caetano melone Date: Sun, 25 Feb 2024 21:54:06 -0800 Subject: [PATCH 19/23] fix: handle jobs with no runners --- gantry/routes/collection.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gantry/routes/collection.py b/gantry/routes/collection.py index 5724f9e..f5c528f 100644 --- a/gantry/routes/collection.py +++ b/gantry/routes/collection.py @@ -45,6 +45,8 @@ async def fetch_job( if ( job.status != "success" or not job.valid_build_name # is not a build job + # some jobs don't have runners..? + or payload["runner"] is None # uo runners are not in Prometheus or payload["runner"]["description"].startswith("uo") or await db.job_exists(db_conn, job.gl_id) # job already in the database From a2d29c21ffb8a2a3a07702b19dd1ebcd487c93f4 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Tue, 27 Feb 2024 01:16:21 -0800 Subject: [PATCH 20/23] test data definitions no longer imported line-by-line --- gantry/tests/test_collection.py | 61 ++++++++++++--------------------- gantry/tests/test_db.py | 4 +-- gantry/tests/test_prometheus.py | 14 +++----- 3 files changed, 28 insertions(+), 51 deletions(-) diff --git a/gantry/tests/test_collection.py b/gantry/tests/test_collection.py index 149fe97..7e35d11 100644 --- a/gantry/tests/test_collection.py +++ b/gantry/tests/test_collection.py @@ -3,24 +3,7 @@ from gantry.clients.gitlab import GitlabClient from gantry.clients.prometheus import PrometheusClient from gantry.routes.collection import fetch_job, fetch_node -from gantry.tests.defs.collection import ( - GHOST_JOB_LOG, - INSERTED_JOB, - INSERTED_NODE, - INVALID_JOB_NAME, - INVALID_JOB_STATUS, - INVALID_MEMORY_USAGE, - INVALID_RUNNER, - VALID_ANNOTATIONS, - VALID_CPU_USAGE, - VALID_JOB, - VALID_JOB_LOG, - VALID_MEMORY_USAGE, - VALID_NODE_INFO, - VALID_NODE_LABELS, - VALID_RESOURCE_LIMITS, - VALID_RESOURCE_REQUESTS, -) +from gantry.tests.defs import collection as defs # mapping of prometheus request shortcuts # to raw values that would be returned by resp.json() @@ -29,13 +12,13 @@ # if the order in which PrometheusClient._query is called changes, # this dict must be updated PROMETHEUS_REQS = { - "job_annotations": VALID_ANNOTATIONS, - "job_resources": VALID_RESOURCE_REQUESTS, - "job_limits": VALID_RESOURCE_LIMITS, - "job_memory_usage": VALID_MEMORY_USAGE, - "job_cpu_usage": VALID_CPU_USAGE, - "node_info": VALID_NODE_INFO, - "node_labels": VALID_NODE_LABELS, + "job_annotations": defs.VALID_ANNOTATIONS, + "job_resources": defs.VALID_RESOURCE_REQUESTS, + "job_limits": defs.VALID_RESOURCE_LIMITS, + "job_memory_usage": defs.VALID_MEMORY_USAGE, + "job_cpu_usage": defs.VALID_CPU_USAGE, + "node_info": defs.VALID_NODE_INFO, + "node_labels": defs.VALID_NODE_LABELS, } @@ -45,7 +28,7 @@ async def gitlab(mocker): # mock the request to the gitlab api # default is to return normal log that wouldn't be detected as a ghost job - mocker.patch.object(GitlabClient, "_request", return_value=VALID_JOB_LOG) + mocker.patch.object(GitlabClient, "_request", return_value=defs.VALID_JOB_LOG) return GitlabClient("", "") @@ -63,14 +46,14 @@ async def prometheus(mocker): @pytest.mark.parametrize( "key, value", [ - ("build_status", INVALID_JOB_STATUS), - ("build_name", INVALID_JOB_NAME), - ("runner", INVALID_RUNNER), + ("build_status", defs.INVALID_JOB_STATUS), + ("build_name", defs.INVALID_JOB_NAME), + ("runner", defs.INVALID_RUNNER), ], ) async def test_invalid_gitlab_fields(db_conn, gitlab, prometheus, key, value): """Tests behavior when invalid data from Gitlab is passed to fetch_job""" - payload = VALID_JOB.copy() + payload = defs.VALID_JOB.copy() payload[key] = value assert await fetch_job(payload, db_conn, gitlab, prometheus) is None @@ -88,14 +71,14 @@ async def test_job_exists(db_conn): with open("gantry/tests/sql/insert_job.sql") as f: await db_conn.executescript(f.read()) - assert await fetch_job(VALID_JOB, db_conn, None, None) is None + assert await fetch_job(defs.VALID_JOB, db_conn, None, None) is None async def test_ghost_job(db_conn, gitlab, mocker): """Tests that a ghost job is detected""" - mocker.patch.object(gitlab, "_request", return_value=GHOST_JOB_LOG) - assert await fetch_job(VALID_JOB, db_conn, gitlab, None) is None + mocker.patch.object(gitlab, "_request", return_value=defs.GHOST_JOB_LOG) + assert await fetch_job(defs.VALID_JOB, db_conn, gitlab, None) is None @pytest.mark.parametrize( @@ -117,7 +100,7 @@ async def test_missing_data(db_conn, gitlab, prometheus, req): # for each req in PROMETHEUS_REQS, set it to an empty dict p[req] = {} prometheus._query.side_effect = p.values() - assert await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) is None + assert await fetch_job(defs.VALID_JOB, db_conn, gitlab, prometheus) is None async def test_invalid_usage(db_conn, gitlab, prometheus): @@ -125,22 +108,22 @@ async def test_invalid_usage(db_conn, gitlab, prometheus): p = PROMETHEUS_REQS.copy() # could also be cpu usage - p["job_memory_usage"] = INVALID_MEMORY_USAGE + p["job_memory_usage"] = defs.INVALID_MEMORY_USAGE prometheus._query.side_effect = p.values() - assert await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) is None + assert await fetch_job(defs.VALID_JOB, db_conn, gitlab, prometheus) is None async def test_job_node_inserted(db_conn, gitlab, prometheus): """Tests that the job and node are in the database after calling fetch_node""" - await fetch_job(VALID_JOB, db_conn, gitlab, prometheus) + await fetch_job(defs.VALID_JOB, db_conn, gitlab, prometheus) # as the first records in the database, the ids should be 1 async with db_conn.execute("SELECT * FROM jobs WHERE id=?", (1,)) as cursor: job = await cursor.fetchone() async with db_conn.execute("SELECT * FROM nodes WHERE id=?", (1,)) as cursor: node = await cursor.fetchone() - assert job == INSERTED_JOB - assert node == INSERTED_NODE + assert job == defs.INSERTED_JOB + assert node == defs.INSERTED_NODE async def test_node_exists(db_conn, prometheus): diff --git a/gantry/tests/test_db.py b/gantry/tests/test_db.py index 3dd3736..8d265c5 100644 --- a/gantry/tests/test_db.py +++ b/gantry/tests/test_db.py @@ -1,5 +1,5 @@ from gantry.db.insert import insert_job, insert_node -from gantry.tests.defs.db import NODE_INSERT_DICT +from gantry.tests.defs import db as defs async def test_node_insert_race(db_conn): @@ -13,7 +13,7 @@ async def test_node_insert_race(db_conn): await db_conn.executescript(f.read()) # the id of NODE_INSERT_DICT is 2, but the output should be 1 - assert await insert_node(db_conn, NODE_INSERT_DICT) == 2 + assert await insert_node(db_conn, defs.NODE_INSERT_DICT) == 2 async def test_insert_node_incomplete(db_conn): diff --git a/gantry/tests/test_prometheus.py b/gantry/tests/test_prometheus.py index 6350863..ecf82e2 100644 --- a/gantry/tests/test_prometheus.py +++ b/gantry/tests/test_prometheus.py @@ -2,13 +2,7 @@ from gantry.clients.prometheus import util from gantry.clients.prometheus.prometheus import PrometheusClient -from gantry.tests.defs.prometheus import ( - ENCODED_QUERY_DICT, - ENCODED_QUERY_STR, - INVALID_QUERY, - QUERY_DICT, - QUERY_STR, -) +from gantry.tests.defs import prometheus as defs def test_cookie_set(): @@ -19,7 +13,7 @@ def test_cookie_set(): def test_process_query(): """Test that a query is parsed and encoded properly, from both dict and string""" - assert util.process_query(QUERY_DICT) == ENCODED_QUERY_DICT - assert util.process_query(QUERY_STR) == ENCODED_QUERY_STR + assert util.process_query(defs.QUERY_DICT) == defs.ENCODED_QUERY_DICT + assert util.process_query(defs.QUERY_STR) == defs.ENCODED_QUERY_STR with pytest.raises(ValueError): - util.process_query(INVALID_QUERY) + util.process_query(defs.INVALID_QUERY) From 218c2fc04eab5bbb0dc1a94adb9b8e57737e6997 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 4 Mar 2024 09:59:33 -0800 Subject: [PATCH 21/23] logging -> logger --- gantry/db/insert.py | 6 ++++-- gantry/routes/collection.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gantry/db/insert.py b/gantry/db/insert.py index e4f0923..c000d4b 100644 --- a/gantry/db/insert.py +++ b/gantry/db/insert.py @@ -4,6 +4,8 @@ from gantry.db.get import get_node +logger = logging.getLogger(__name__) + def insert_dict(table: str, input: dict, ignore=False) -> tuple[str, tuple]: """ @@ -50,7 +52,7 @@ async def insert_node(db: aiosqlite.Connection, node: dict) -> int: pk = await get_node(db, node["uuid"]) if pk is None: - logging.error(f"node not inserted: {node}. data is likely missing") + logger.error(f"node not inserted: {node}. data is likely missing") return pk @@ -70,5 +72,5 @@ async def insert_job(db: aiosqlite.Connection, job: dict) -> int: if cursor.rowcount > 0: return cursor.lastrowid - logging.error(f"job not inserted: {job}. data is likely missing") + logger.error(f"job not inserted: {job}. data is likely missing") return None diff --git a/gantry/routes/collection.py b/gantry/routes/collection.py index f5c528f..aeb6602 100644 --- a/gantry/routes/collection.py +++ b/gantry/routes/collection.py @@ -57,7 +57,7 @@ async def fetch_job( job_log = await gitlab.job_log(job.gl_id) is_ghost = "No need to rebuild" in job_log if is_ghost: - logging.warning(f"job {job.gl_id} is a ghost, skipping") + logger.warning(f"job {job.gl_id} is a ghost, skipping") return try: From 837ba548ca80fb01f30c845a68e9b904c0c18158 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 4 Mar 2024 10:04:29 -0800 Subject: [PATCH 22/23] use top-level requirements.txt for unit test dependency listing --- .github/workflows/requirements/unit-tests.txt | 4 +--- .github/workflows/unit-tests.yml | 5 ++++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/requirements/unit-tests.txt b/.github/workflows/requirements/unit-tests.txt index 39069ed..2de0d46 100644 --- a/.github/workflows/requirements/unit-tests.txt +++ b/.github/workflows/requirements/unit-tests.txt @@ -1,5 +1,3 @@ pytest==8.0.1 pytest-aiohttp==1.0.5 -pytest-mock==3.12.0 -aiosqlite==0.20.0 -aiohttp==3.9.3 \ No newline at end of file +pytest-mock==3.12.0 \ No newline at end of file diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index df93f09..c48dfed 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -16,10 +16,13 @@ jobs: with: python-version: ${{ matrix.python-version }} cache: 'pip' - cache-dependency-path: '.github/workflows/requirements/unit-tests.txt' + cache-dependency-path: | + 'requirements.txt' + '.github/workflows/requirements/unit-tests.txt' - name: Install Python dependencies run: | + pip install -r requirements.txt pip install -r .github/workflows/requirements/unit-tests.txt - name: Run Unit Tests with Pytest From 5b268b0f29e7490e264a3896488a648cfabce247 Mon Sep 17 00:00:00 2001 From: caetano melone Date: Mon, 4 Mar 2024 10:47:13 -0800 Subject: [PATCH 23/23] =?UTF-8?q?fix:=20shrink=20mock=20of=20usage=20data?= =?UTF-8?q?=20=F0=9F=A4=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- gantry/tests/defs/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/gantry/tests/defs/collection.py b/gantry/tests/defs/collection.py index d0db895..92fcab8 100644 --- a/gantry/tests/defs/collection.py +++ b/gantry/tests/defs/collection.py @@ -20,7 +20,7 @@ # used to compare successful insertions # run SELECT * FROM table_name WHERE id = 1; from python sqlite api and grab fetchone() result -INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux-ubuntu20.04-x86_64_v3', 'e4s', 16, 0.75, None, 4.125322866945405, 3.158058646775204, 11.603810729464888, 0.2483743618267752, 3.348888803394752, 2000000000.0, 48000000000.0, 1649868862.7258806, 999763968.0, 5679742976.0, 2785280.0, 1378705563.2101867) +INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux-ubuntu20.04-x86_64_v3', 'e4s', 16, 0.75, None, 1.899768349523097, 0.2971597591741076, 4.128116379389054, 0.2483743618267752, 1.7602635378120381, 2000000000.0, 48000000000.0, 143698407.6190476, 2785280.0, 594620416.0, 2785280.0, 252073065.82263485) INSERTED_NODE = (1, 'ec253b04-b1dc-f08b-acac-e23df83b3602', 'ip-192-168-86-107.ec2.internal', 24.0, 196608000000.0, 'amd64', 'linux', 'i3en.6xlarge') # these were obtained by executing the respective queries to Prometheus and capturing the JSON output @@ -28,8 +28,8 @@ VALID_ANNOTATIONS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_annotations', 'annotation_gitlab_ci_job_id': '9892514', 'annotation_metrics_spack_ci_stack_name': 'e4s', 'annotation_metrics_spack_job_spec_arch': 'linux-ubuntu20.04-x86_64_v3', 'annotation_metrics_spack_job_spec_compiler_name': 'gcc', 'annotation_metrics_spack_job_spec_compiler_version': '11.4.0', 'annotation_metrics_spack_job_spec_pkg_name': 'gmsh', 'annotation_metrics_spack_job_spec_pkg_version': '4.8.4', 'annotation_metrics_spack_job_spec_variants': '+alglib~cairo+cgns+compression~eigen~external+fltk+gmp~hdf5~ipo+med+metis+mmg+mpi+netgen+oce~opencascade~openmp~petsc~privateapi+shared~slepc+tetgen+voropp build_system=cmake build_type=Release generator=make', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0'}, 'value': [1706117733, '1']}]}} VALID_RESOURCE_REQUESTS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_resource_requests', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'cpu', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'core'}, 'value': [1706117733, '0.75']}, {'metric': {'__name__': 'kube_pod_container_resource_requests', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'memory', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'byte'}, 'value': [1706117733, '2000000000']}]}} VALID_RESOURCE_LIMITS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_pod_container_resource_limits', 'container': 'build', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'resource': 'memory', 'service': 'kube-prometheus-stack-kube-state-metrics', 'uid': 'd7aa13e0-998c-4f21-b1d6-62781f4980b0', 'unit': 'byte'}, 'value': [1706117733, '48000000000']}]}} -VALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '2785280'], [1706117116, '2785280'], [1706117117, '2785280'], [1706117118, '2785280'], [1706117119, '2785280'], [1706117120, '2785280'], [1706117121, '2785280'], [1706117122, '2785280'], [1706117123, '2785280'], [1706117124, '2785280'], [1706117125, '2785280'], [1706117126, '2785280'], [1706117127, '2785280'], [1706117128, '2785280'], [1706117129, '2785280'], [1706117130, '2785280'], [1706117131, '2785280'], [1706117132, '2785280'], [1706117133, '2785280'], [1706117134, '2785280'], [1706117135, '2785280'], [1706117136, '2785280'], [1706117137, '2785280'], [1706117138, '2785280'], [1706117139, '2785280'], [1706117140, '2785280'], [1706117141, '2785280'], [1706117142, '2785280'], [1706117143, '2785280'], [1706117144, '2785280'], [1706117145, '463159296'], [1706117146, '463159296'], [1706117147, '463159296'], [1706117148, '463159296'], [1706117149, '463159296'], [1706117150, '463159296'], [1706117151, '463159296'], [1706117152, '463159296'], [1706117153, '463159296'], [1706117154, '463159296'], [1706117155, '463159296'], [1706117156, '463159296'], [1706117157, '463159296'], [1706117158, '463159296'], [1706117159, '463159296'], [1706117160, '463159296'], [1706117161, '463159296'], [1706117162, '463159296'], [1706117163, '463159296'], [1706117164, '463159296'], [1706117165, '463159296'], [1706117166, '463159296'], [1706117167, '463159296'], [1706117168, '463159296'], [1706117169, '463159296'], [1706117170, '463159296'], [1706117171, '463159296'], [1706117172, '463159296'], [1706117173, '463159296'], [1706117174, '463159296'], [1706117175, '463159296'], [1706117176, '814882816'], [1706117177, '814882816'], [1706117178, '814882816'], [1706117179, '814882816'], [1706117180, '814882816'], [1706117181, '814882816'], [1706117182, '814882816'], [1706117183, '814882816'], [1706117184, '814882816'], [1706117185, '814882816'], [1706117186, '814882816'], [1706117187, '814882816'], [1706117188, '814882816'], [1706117189, '814882816'], [1706117190, '814882816'], [1706117191, '814882816'], [1706117192, '814882816'], [1706117193, '814882816'], [1706117194, '814882816'], [1706117195, '814882816'], [1706117196, '814882816'], [1706117197, '814882816'], [1706117198, '814882816'], [1706117199, '814882816'], [1706117200, '814882816'], [1706117201, '814882816'], [1706117202, '814882816'], [1706117203, '814882816'], [1706117204, '814882816'], [1706117205, '814882816'], [1706117206, '144236544'], [1706117207, '144236544'], [1706117208, '144236544'], [1706117209, '144236544'], [1706117210, '144236544'], [1706117211, '144236544'], [1706117212, '144236544'], [1706117213, '144236544'], [1706117214, '144236544'], [1706117215, '144236544'], [1706117216, '144236544'], [1706117217, '144236544'], [1706117218, '144236544'], [1706117219, '144236544'], [1706117220, '144236544'], [1706117221, '144236544'], [1706117222, '144236544'], [1706117223, '144236544'], [1706117224, '144236544'], [1706117225, '280522752'], [1706117226, '280522752'], [1706117227, '280522752'], [1706117228, '280522752'], [1706117229, '280522752'], [1706117230, '280522752'], [1706117231, '280522752'], [1706117232, '280522752'], [1706117233, '280522752'], [1706117234, '280522752'], [1706117235, '280522752'], [1706117236, '280522752'], [1706117237, '280522752'], [1706117238, '280522752'], [1706117239, '280522752'], [1706117240, '280522752'], [1706117241, '280522752'], [1706117242, '280522752'], [1706117243, '280522752'], [1706117244, '280522752'], [1706117245, '280522752'], [1706117246, '280522752'], [1706117247, '280522752'], [1706117248, '280522752'], [1706117249, '280522752'], [1706117250, '280522752'], [1706117251, '280522752'], [1706117252, '280522752'], [1706117253, '280522752'], [1706117254, '280522752'], [1706117255, '280522752'], [1706117256, '280522752'], [1706117257, '280522752'], [1706117258, '280522752'], [1706117259, '280522752'], [1706117260, '280522752'], [1706117261, '280522752'], [1706117262, '280522752'], [1706117263, '280522752'], [1706117264, '280522752'], [1706117265, '280522752'], [1706117266, '280522752'], [1706117267, '280522752'], [1706117268, '280522752'], [1706117269, '280522752'], [1706117270, '280522752'], [1706117271, '280522752'], [1706117272, '5402374144'], [1706117273, '5402374144'], [1706117274, '5402374144'], [1706117275, '5402374144'], [1706117276, '5402374144'], [1706117277, '5402374144'], [1706117278, '5402374144'], [1706117279, '5402374144'], [1706117280, '5402374144'], [1706117281, '5402374144'], [1706117282, '5402374144'], [1706117283, '5402374144'], [1706117284, '5402374144'], [1706117285, '5402374144'], [1706117286, '5402374144'], [1706117287, '5402374144'], [1706117288, '5402374144'], [1706117289, '5402374144'], [1706117290, '5402374144'], [1706117291, '5402374144'], [1706117292, '5402374144'], [1706117293, '5402374144'], [1706117294, '5402374144'], [1706117295, '5402374144'], [1706117296, '5402374144'], [1706117297, '5402374144'], [1706117298, '5402374144'], [1706117299, '5402374144'], [1706117300, '5679742976'], [1706117301, '5679742976'], [1706117302, '5679742976'], [1706117303, '5679742976'], [1706117304, '5679742976'], [1706117305, '5679742976'], [1706117306, '5679742976'], [1706117307, '5679742976'], [1706117308, '5679742976'], [1706117309, '5679742976'], [1706117310, '5679742976'], [1706117311, '5679742976'], [1706117312, '5679742976'], [1706117313, '5679742976'], [1706117314, '5679742976'], [1706117315, '5679742976'], [1706117316, '5679742976'], [1706117317, '5679742976'], [1706117318, '5679742976'], [1706117319, '5679742976'], [1706117320, '5679742976'], [1706117321, '5679742976'], [1706117322, '5679742976'], [1706117323, '5679742976'], [1706117324, '5679742976'], [1706117325, '5679742976'], [1706117326, '4521721856'], [1706117327, '4521721856'], [1706117328, '4521721856'], [1706117329, '4521721856'], [1706117330, '4521721856'], [1706117331, '4521721856'], [1706117332, '4521721856'], [1706117333, '4521721856'], [1706117334, '4521721856'], [1706117335, '4521721856'], [1706117336, '4521721856'], [1706117337, '4521721856'], [1706117338, '4521721856'], [1706117339, '4521721856'], [1706117340, '4521721856'], [1706117341, '4521721856'], [1706117342, '4521721856'], [1706117343, '4521721856'], [1706117344, '4521721856'], [1706117345, '4521721856'], [1706117346, '4521721856'], [1706117347, '4521721856'], [1706117348, '4521721856'], [1706117349, '4521721856'], [1706117350, '4521721856'], [1706117351, '4521721856'], [1706117352, '4521721856'], [1706117353, '4521721856'], [1706117354, '4521721856'], [1706117355, '4521721856'], [1706117356, '4426862592'], [1706117357, '4426862592'], [1706117358, '4426862592'], [1706117359, '4426862592'], [1706117360, '4426862592'], [1706117361, '4426862592'], [1706117362, '4426862592'], [1706117363, '4426862592'], [1706117364, '4426862592'], [1706117365, '4426862592'], [1706117366, '4426862592'], [1706117367, '4426862592'], [1706117368, '4426862592'], [1706117369, '4426862592'], [1706117370, '4426862592'], [1706117371, '4426862592'], [1706117372, '4426862592'], [1706117373, '4426862592'], [1706117374, '4426862592'], [1706117375, '4426862592'], [1706117376, '4426862592'], [1706117377, '4426862592'], [1706117378, '4426862592'], [1706117379, '4426862592'], [1706117380, '4426862592'], [1706117381, '4426862592'], [1706117382, '4426862592'], [1706117383, '4426862592'], [1706117384, '4426862592'], [1706117385, '4426862592'], [1706117386, '4426862592'], [1706117387, '4426862592'], [1706117388, '4426862592'], [1706117389, '4426862592'], [1706117390, '4426862592'], [1706117391, '2534088704'], [1706117392, '2534088704'], [1706117393, '2534088704'], [1706117394, '2534088704'], [1706117395, '2534088704'], [1706117396, '2534088704'], [1706117397, '2534088704'], [1706117398, '2534088704'], [1706117399, '2534088704'], [1706117400, '2534088704'], [1706117401, '2534088704'], [1706117402, '2534088704'], [1706117403, '2534088704'], [1706117404, '2534088704'], [1706117405, '2534088704'], [1706117406, '2534088704'], [1706117407, '2721067008'], [1706117408, '2721067008'], [1706117409, '2721067008'], [1706117410, '2721067008'], [1706117411, '2721067008'], [1706117412, '2721067008'], [1706117413, '2721067008'], [1706117414, '2721067008'], [1706117415, '2721067008'], [1706117416, '2721067008'], [1706117417, '2721067008'], [1706117418, '2721067008'], [1706117419, '2721067008'], [1706117420, '2721067008'], [1706117421, '2721067008'], [1706117422, '2721067008'], [1706117423, '2721067008'], [1706117424, '2721067008'], [1706117425, '2721067008'], [1706117426, '2721067008'], [1706117427, '2721067008'], [1706117428, '2721067008'], [1706117429, '2721067008'], [1706117430, '2721067008'], [1706117431, '2721067008'], [1706117432, '2721067008'], [1706117433, '2721067008'], [1706117434, '2721067008'], [1706117435, '2721067008'], [1706117436, '2721067008'], [1706117437, '2721067008'], [1706117438, '2721067008'], [1706117439, '2721067008'], [1706117440, '2262188032'], [1706117441, '2262188032'], [1706117442, '2262188032'], [1706117443, '2262188032'], [1706117444, '2262188032'], [1706117445, '2262188032'], [1706117446, '2262188032'], [1706117447, '2262188032'], [1706117448, '2262188032'], [1706117449, '2262188032'], [1706117450, '2262188032'], [1706117451, '2262188032'], [1706117452, '2262188032'], [1706117453, '2262188032'], [1706117454, '2262188032'], [1706117455, '2262188032'], [1706117456, '2262188032'], [1706117457, '2262188032'], [1706117458, '2262188032'], [1706117459, '2262188032'], [1706117460, '2262188032'], [1706117461, '2262188032'], [1706117462, '2262188032'], [1706117463, '2262188032'], [1706117464, '2262188032'], [1706117465, '2262188032'], [1706117466, '2262188032'], [1706117467, '2262188032'], [1706117468, '2262188032'], [1706117469, '2262188032'], [1706117470, '2262188032'], [1706117471, '2262188032'], [1706117472, '2262188032'], [1706117473, '2262188032'], [1706117474, '2262188032'], [1706117475, '2262188032'], [1706117476, '2262188032'], [1706117477, '2262188032'], [1706117478, '2262188032'], [1706117479, '1202192384'], [1706117480, '1202192384'], [1706117481, '1202192384'], [1706117482, '1202192384'], [1706117483, '1202192384'], [1706117484, '1202192384'], [1706117485, '1202192384'], [1706117486, '1202192384'], [1706117487, '1202192384'], [1706117488, '1202192384'], [1706117489, '1202192384'], [1706117490, '1202192384'], [1706117491, '1202192384'], [1706117492, '1202192384'], [1706117493, '1202192384'], [1706117494, '1202192384'], [1706117495, '1202192384'], [1706117496, '1377042432'], [1706117497, '1377042432'], [1706117498, '1377042432'], [1706117499, '1377042432'], [1706117500, '1377042432'], [1706117501, '1377042432'], [1706117502, '1377042432'], [1706117503, '1377042432'], [1706117504, '1377042432'], [1706117505, '1377042432'], [1706117506, '1377042432'], [1706117507, '1377042432'], [1706117508, '1377042432'], [1706117509, '1377042432'], [1706117510, '1377042432'], [1706117511, '1377042432'], [1706117512, '1377042432'], [1706117513, '1377042432'], [1706117514, '1377042432'], [1706117515, '1377042432'], [1706117516, '1377042432'], [1706117517, '1377042432'], [1706117518, '1377042432'], [1706117519, '1377042432'], [1706117520, '1377042432'], [1706117521, '1377042432'], [1706117522, '1377042432'], [1706117523, '1377042432'], [1706117524, '1377042432'], [1706117525, '1377042432'], [1706117526, '1377042432'], [1706117527, '1377042432'], [1706117528, '937562112'], [1706117529, '937562112'], [1706117530, '937562112'], [1706117531, '937562112'], [1706117532, '937562112'], [1706117533, '937562112'], [1706117534, '937562112'], [1706117535, '937562112'], [1706117536, '937562112'], [1706117537, '937562112'], [1706117538, '937562112'], [1706117539, '937562112'], [1706117540, '937562112'], [1706117541, '937562112'], [1706117542, '937562112'], [1706117543, '937562112'], [1706117544, '937562112'], [1706117545, '937562112'], [1706117546, '937562112'], [1706117547, '937562112'], [1706117548, '937562112'], [1706117549, '937562112'], [1706117550, '937562112'], [1706117551, '937562112'], [1706117552, '937562112'], [1706117553, '937562112'], [1706117554, '937562112'], [1706117555, '937562112'], [1706117556, '937562112'], [1706117557, '937562112'], [1706117558, '937562112'], [1706117559, '937562112'], [1706117560, '937562112'], [1706117561, '937562112'], [1706117562, '937562112'], [1706117563, '937562112'], [1706117564, '937562112'], [1706117565, '937562112'], [1706117566, '999763968'], [1706117567, '999763968'], [1706117568, '999763968'], [1706117569, '999763968'], [1706117570, '999763968'], [1706117571, '999763968'], [1706117572, '999763968'], [1706117573, '999763968'], [1706117574, '999763968'], [1706117575, '999763968'], [1706117576, '999763968'], [1706117577, '999763968'], [1706117578, '999763968'], [1706117579, '999763968'], [1706117580, '999763968'], [1706117581, '999763968'], [1706117582, '999763968'], [1706117583, '999763968'], [1706117584, '999763968'], [1706117585, '999763968'], [1706117586, '999763968'], [1706117587, '999763968'], [1706117588, '999763968'], [1706117589, '999763968'], [1706117590, '999763968'], [1706117591, '999763968'], [1706117592, '999763968'], [1706117593, '999763968'], [1706117594, '999763968'], [1706117595, '999763968'], [1706117596, '999763968'], [1706117597, '999763968'], [1706117598, '566054912'], [1706117599, '566054912'], [1706117600, '566054912'], [1706117601, '566054912'], [1706117602, '566054912'], [1706117603, '566054912'], [1706117604, '566054912'], [1706117605, '566054912'], [1706117606, '566054912'], [1706117607, '566054912'], [1706117608, '566054912'], [1706117609, '566054912'], [1706117610, '566054912'], [1706117611, '566054912'], [1706117612, '566054912'], [1706117613, '566054912'], [1706117614, '566054912'], [1706117615, '566054912'], [1706117616, '566054912'], [1706117617, '566054912'], [1706117618, '566054912'], [1706117619, '566054912'], [1706117620, '566054912'], [1706117621, '566054912'], [1706117622, '566054912'], [1706117623, '566054912'], [1706117624, '566054912'], [1706117625, '566054912'], [1706117626, '566054912'], [1706117627, '566054912'], [1706117628, '756076544'], [1706117629, '756076544'], [1706117630, '756076544'], [1706117631, '756076544'], [1706117632, '756076544'], [1706117633, '756076544'], [1706117634, '756076544'], [1706117635, '756076544'], [1706117636, '756076544'], [1706117637, '756076544'], [1706117638, '756076544'], [1706117639, '756076544'], [1706117640, '756076544'], [1706117641, '756076544'], [1706117642, '756076544'], [1706117643, '756076544'], [1706117644, '756076544'], [1706117645, '756076544'], [1706117646, '756076544'], [1706117647, '756076544'], [1706117648, '756076544'], [1706117649, '756076544'], [1706117650, '756076544'], [1706117651, '756076544'], [1706117652, '756076544'], [1706117653, '756076544'], [1706117654, '756076544'], [1706117655, '756076544'], [1706117656, '756076544'], [1706117657, '756076544'], [1706117658, '756076544'], [1706117659, '756076544'], [1706117660, '826408960'], [1706117661, '826408960'], [1706117662, '826408960'], [1706117663, '826408960'], [1706117664, '826408960'], [1706117665, '826408960'], [1706117666, '826408960'], [1706117667, '826408960'], [1706117668, '826408960'], [1706117669, '826408960'], [1706117670, '826408960'], [1706117671, '826408960'], [1706117672, '826408960'], [1706117673, '826408960'], [1706117674, '826408960'], [1706117675, '826408960'], [1706117676, '826408960'], [1706117677, '826408960'], [1706117678, '826408960'], [1706117679, '826408960'], [1706117680, '826408960'], [1706117681, '826408960'], [1706117682, '826408960'], [1706117683, '826408960'], [1706117684, '826408960'], [1706117685, '826408960'], [1706117686, '826408960'], [1706117687, '826408960'], [1706117688, '826408960'], [1706117689, '779374592'], [1706117690, '779374592'], [1706117691, '779374592'], [1706117692, '779374592'], [1706117693, '779374592'], [1706117694, '779374592'], [1706117695, '779374592'], [1706117696, '779374592'], [1706117697, '779374592'], [1706117698, '779374592'], [1706117699, '779374592'], [1706117700, '779374592'], [1706117701, '779374592'], [1706117702, '779374592'], [1706117703, '779374592'], [1706117704, '779374592'], [1706117705, '779374592'], [1706117706, '779374592'], [1706117707, '779374592'], [1706117708, '515768320'], [1706117709, '515768320'], [1706117710, '515768320'], [1706117711, '515768320'], [1706117712, '515768320'], [1706117713, '515768320'], [1706117714, '515768320'], [1706117715, '515768320'], [1706117716, '515768320'], [1706117717, '515768320'], [1706117718, '515768320'], [1706117719, '515768320'], [1706117720, '515768320'], [1706117721, '515768320'], [1706117722, '515768320'], [1706117723, '515768320'], [1706117724, '515768320'], [1706117725, '515768320'], [1706117726, '515768320'], [1706117727, '515768320'], [1706117728, '515768320'], [1706117729, '515768320'], [1706117730, '515768320'], [1706117731, '515768320'], [1706117732, '515768320'], [1706117733, '515768320'], [1706117734, '515768320'], [1706117735, '515768320'], [1706117736, '515768320'], [1706117737, '515768320'], [1706117738, '515768320'], [1706117739, '515768320'], [1706117740, '515768320'], [1706117741, '515768320'], [1706117742, '515768320'], [1706117743, '515768320'], [1706117744, '515768320'], [1706117745, '515768320'], [1706117746, '515768320'], [1706117747, '515768320'], [1706117748, '515768320'], [1706117749, '515768320'], [1706117750, '518909952'], [1706117751, '518909952'], [1706117752, '518909952'], [1706117753, '518909952'], [1706117754, '518909952'], [1706117755, '518909952'], [1706117756, '518909952'], [1706117757, '518909952'], [1706117758, '518909952'], [1706117759, '518909952'], [1706117760, '518909952'], [1706117761, '518909952'], [1706117762, '518909952'], [1706117763, '518909952'], [1706117764, '518909952'], [1706117765, '518909952'], [1706117766, '518909952'], [1706117767, '518909952'], [1706117768, '518909952'], [1706117769, '518909952'], [1706117770, '518909952'], [1706117771, '518909952'], [1706117772, '518909952'], [1706117773, '518909952'], [1706117774, '518909952'], [1706117775, '518909952'], [1706117776, '518909952'], [1706117777, '527507456'], [1706117778, '527507456'], [1706117779, '527507456'], [1706117780, '527507456'], [1706117781, '527507456'], [1706117782, '527507456'], [1706117783, '527507456'], [1706117784, '527507456'], [1706117785, '527507456'], [1706117786, '527507456'], [1706117787, '527507456'], [1706117788, '527507456'], [1706117789, '527507456'], [1706117790, '527507456'], [1706117791, '527507456'], [1706117792, '527507456'], [1706117793, '527507456'], [1706117794, '527507456'], [1706117795, '527507456'], [1706117796, '527507456'], [1706117797, '527507456'], [1706117798, '527507456'], [1706117799, '527507456'], [1706117800, '527507456'], [1706117801, '527507456'], [1706117802, '527507456'], [1706117803, '527507456'], [1706117804, '527507456'], [1706117805, '527507456'], [1706117806, '527507456'], [1706117807, '550576128'], [1706117808, '550576128'], [1706117809, '550576128'], [1706117810, '550576128'], [1706117811, '550576128'], [1706117812, '550576128'], [1706117813, '550576128'], [1706117814, '550576128'], [1706117815, '550576128'], [1706117816, '550576128'], [1706117817, '550576128'], [1706117818, '550576128'], [1706117819, '550576128'], [1706117820, '550576128'], [1706117821, '550576128'], [1706117822, '550576128'], [1706117823, '550576128'], [1706117824, '550576128'], [1706117825, '550576128'], [1706117826, '550576128'], [1706117827, '550576128'], [1706117828, '550576128'], [1706117829, '550576128'], [1706117830, '550576128'], [1706117831, '550576128'], [1706117832, '550576128'], [1706117833, '550576128'], [1706117834, '550576128'], [1706117835, '707067904'], [1706117836, '707067904'], [1706117837, '707067904'], [1706117838, '707067904'], [1706117839, '707067904'], [1706117840, '707067904'], [1706117841, '707067904'], [1706117842, '707067904'], [1706117843, '707067904'], [1706117844, '707067904'], [1706117845, '707067904'], [1706117846, '707067904'], [1706117847, '707067904'], [1706117848, '707067904'], [1706117849, '707067904'], [1706117850, '707067904'], [1706117851, '707067904'], [1706117852, '707067904'], [1706117853, '707067904'], [1706117854, '707067904'], [1706117855, '707067904'], [1706117856, '707067904'], [1706117857, '707067904'], [1706117858, '707067904'], [1706117859, '707067904'], [1706117860, '707067904'], [1706117861, '707067904'], [1706117862, '707067904'], [1706117863, '640978944'], [1706117864, '640978944'], [1706117865, '640978944'], [1706117866, '640978944'], [1706117867, '640978944'], [1706117868, '640978944'], [1706117869, '640978944'], [1706117870, '640978944'], [1706117871, '640978944'], [1706117872, '640978944'], [1706117873, '640978944'], [1706117874, '640978944'], [1706117875, '640978944'], [1706117876, '640978944'], [1706117877, '640978944'], [1706117878, '640978944'], [1706117879, '640978944'], [1706117880, '640978944'], [1706117881, '640978944'], [1706117882, '640978944'], [1706117883, '640978944'], [1706117884, '640978944'], [1706117885, '640978944'], [1706117886, '640978944'], [1706117887, '640978944'], [1706117888, '640978944'], [1706117889, '640978944'], [1706117890, '640978944'], [1706117891, '640978944'], [1706117892, '640978944'], [1706117893, '611074048'], [1706117894, '611074048'], [1706117895, '611074048'], [1706117896, '611074048'], [1706117897, '611074048'], [1706117898, '611074048'], [1706117899, '611074048'], [1706117900, '611074048'], [1706117901, '611074048'], [1706117902, '611074048'], [1706117903, '611074048'], [1706117904, '611074048'], [1706117905, '611074048'], [1706117906, '611074048'], [1706117907, '611074048'], [1706117908, '611074048'], [1706117909, '611074048'], [1706117910, '611074048'], [1706117911, '611074048'], [1706117912, '611074048'], [1706117913, '611074048'], [1706117914, '611074048'], [1706117915, '611074048'], [1706117916, '611074048'], [1706117917, '611074048'], [1706117918, '611074048'], [1706117919, '495898624'], [1706117920, '495898624'], [1706117921, '495898624'], [1706117922, '495898624'], [1706117923, '495898624'], [1706117924, '495898624'], [1706117925, '495898624'], [1706117926, '495898624'], [1706117927, '495898624'], [1706117928, '495898624'], [1706117929, '495898624'], [1706117930, '495898624'], [1706117931, '495898624'], [1706117932, '495898624'], [1706117933, '495898624'], [1706117934, '495898624'], [1706117935, '495898624'], [1706117936, '495898624'], [1706117937, '495898624'], [1706117938, '495898624'], [1706117939, '495898624'], [1706117940, '495898624'], [1706117941, '495898624'], [1706117942, '495898624'], [1706117943, '495898624'], [1706117944, '495898624'], [1706117945, '495898624'], [1706117946, '495898624'], [1706117947, '2094526464'], [1706117948, '2094526464'], [1706117949, '2094526464'], [1706117950, '2094526464'], [1706117951, '2094526464'], [1706117952, '2094526464'], [1706117953, '2094526464'], [1706117954, '2094526464'], [1706117955, '2094526464'], [1706117956, '2094526464'], [1706117957, '2094526464'], [1706117958, '2094526464'], [1706117959, '2094526464'], [1706117960, '2094526464'], [1706117961, '2094526464'], [1706117962, '2094526464'], [1706117963, '2094526464'], [1706117964, '2094526464'], [1706117965, '2094526464'], [1706117966, '2094526464'], [1706117967, '2094526464'], [1706117968, '2094526464'], [1706117969, '2094526464'], [1706117970, '2094526464'], [1706117971, '2094526464'], [1706117972, '2094526464'], [1706117973, '2094526464'], [1706117974, '2094526464'], [1706117975, '2094526464'], [1706117976, '2094526464'], [1706117977, '2094526464'], [1706117978, '2094526464'], [1706117979, '2094526464'], [1706117980, '2094526464'], [1706117981, '2094526464'], [1706117982, '2043138048'], [1706117983, '2043138048'], [1706117984, '2043138048'], [1706117985, '2043138048'], [1706117986, '2043138048'], [1706117987, '2043138048'], [1706117988, '2043138048'], [1706117989, '2043138048'], [1706117990, '2043138048'], [1706117991, '2043138048'], [1706117992, '2043138048'], [1706117993, '2043138048'], [1706117994, '2043138048'], [1706117995, '2043138048'], [1706117996, '2043138048'], [1706117997, '2043138048'], [1706117998, '2043138048'], [1706117999, '2043138048'], [1706118000, '2043138048'], [1706118001, '2043138048'], [1706118002, '2043138048'], [1706118003, '2043138048'], [1706118004, '2043138048'], [1706118005, '2043138048'], [1706118006, '2043138048'], [1706118007, '2043138048'], [1706118008, '2043138048'], [1706118009, '2043138048'], [1706118010, '2043138048'], [1706118011, '2043138048'], [1706118012, '2043138048'], [1706118013, '2043138048'], [1706118014, '2043138048'], [1706118015, '2043138048'], [1706118016, '2043138048'], [1706118017, '2043138048'], [1706118018, '2152861696'], [1706118019, '2152861696'], [1706118020, '2152861696'], [1706118021, '2152861696'], [1706118022, '2152861696'], [1706118023, '2152861696'], [1706118024, '2152861696'], [1706118025, '2152861696'], [1706118026, '2152861696'], [1706118027, '2152861696'], [1706118028, '2152861696'], [1706118029, '2152861696'], [1706118030, '2152861696'], [1706118031, '2152861696'], [1706118032, '2152861696'], [1706118033, '2152861696'], [1706118034, '2152861696'], [1706118035, '2152861696'], [1706118036, '2152861696'], [1706118037, '2152861696'], [1706118038, '2152861696'], [1706118039, '2152861696'], [1706118040, '2152861696'], [1706118041, '2152861696'], [1706118042, '2152861696'], [1706118043, '2152861696'], [1706118044, '2152861696'], [1706118045, '2152861696'], [1706118046, '2674098176'], [1706118047, '2674098176'], [1706118048, '2674098176'], [1706118049, '2674098176'], [1706118050, '2674098176'], [1706118051, '2674098176'], [1706118052, '2674098176'], [1706118053, '2674098176'], [1706118054, '2674098176'], [1706118055, '2674098176'], [1706118056, '2674098176'], [1706118057, '2674098176'], [1706118058, '2674098176'], [1706118059, '2674098176'], [1706118060, '2674098176'], [1706118061, '2674098176'], [1706118062, '2674098176'], [1706118063, '2674098176'], [1706118064, '2674098176'], [1706118065, '2674098176'], [1706118066, '2674098176'], [1706118067, '2674098176'], [1706118068, '2674098176'], [1706118069, '2674098176'], [1706118070, '2674098176'], [1706118071, '2674098176'], [1706118072, '2674098176'], [1706118073, '2674098176'], [1706118074, '2674098176'], [1706118075, '2674098176'], [1706118076, '2674098176'], [1706118077, '2736074752'], [1706118078, '2736074752'], [1706118079, '2736074752'], [1706118080, '2736074752'], [1706118081, '2736074752'], [1706118082, '2736074752'], [1706118083, '2736074752'], [1706118084, '2736074752'], [1706118085, '2736074752'], [1706118086, '2736074752'], [1706118087, '2736074752'], [1706118088, '2736074752'], [1706118089, '2736074752'], [1706118090, '2736074752'], [1706118091, '2736074752'], [1706118092, '2736074752'], [1706118093, '2736074752'], [1706118094, '2736074752'], [1706118095, '2736074752'], [1706118096, '2736074752'], [1706118097, '2736074752'], [1706118098, '2736074752'], [1706118099, '2736074752'], [1706118100, '2736074752'], [1706118101, '2736074752'], [1706118102, '2736074752'], [1706118103, '2736074752'], [1706118104, '2736074752'], [1706118105, '2736074752'], [1706118106, '2736074752'], [1706118107, '2736074752'], [1706118108, '2736074752'], [1706118109, '2736074752'], [1706118110, '2736074752'], [1706118111, '2736074752'], [1706118112, '2736074752'], [1706118113, '2943303680'], [1706118114, '2943303680'], [1706118115, '2943303680'], [1706118116, '2943303680'], [1706118117, '2943303680'], [1706118118, '2943303680'], [1706118119, '2943303680'], [1706118120, '2943303680'], [1706118121, '2943303680'], [1706118122, '2943303680'], [1706118123, '2943303680'], [1706118124, '2943303680'], [1706118125, '2943303680'], [1706118126, '2943303680'], [1706118127, '2943303680'], [1706118128, '2943303680'], [1706118129, '2943303680'], [1706118130, '2943303680'], [1706118131, '2943303680'], [1706118132, '2943303680'], [1706118133, '2943303680'], [1706118134, '2943303680'], [1706118135, '2943303680'], [1706118136, '2943303680'], [1706118137, '2943303680'], [1706118138, '2943303680'], [1706118139, '2943303680'], [1706118140, '2943303680'], [1706118141, '2952359936'], [1706118142, '2952359936'], [1706118143, '2952359936'], [1706118144, '2952359936'], [1706118145, '2952359936'], [1706118146, '2952359936'], [1706118147, '2952359936'], [1706118148, '2952359936'], [1706118149, '2952359936'], [1706118150, '2952359936'], [1706118151, '2952359936'], [1706118152, '2952359936'], [1706118153, '2952359936'], [1706118154, '2952359936'], [1706118155, '2952359936'], [1706118156, '2952359936'], [1706118157, '2952359936'], [1706118158, '2952359936'], [1706118159, '2952359936'], [1706118160, '2952359936'], [1706118161, '2952359936'], [1706118162, '2952359936'], [1706118163, '2952359936'], [1706118164, '2952359936'], [1706118165, '2952359936'], [1706118166, '2952359936'], [1706118167, '2952359936'], [1706118168, '2952359936'], [1706118169, '2952359936'], [1706118170, '2952359936'], [1706118171, '1859096576'], [1706118172, '1859096576'], [1706118173, '1859096576'], [1706118174, '1859096576'], [1706118175, '1859096576'], [1706118176, '1859096576'], [1706118177, '1859096576'], [1706118178, '1859096576'], [1706118179, '1859096576'], [1706118180, '1859096576'], [1706118181, '1859096576'], [1706118182, '1859096576'], [1706118183, '1859096576'], [1706118184, '1859096576'], [1706118185, '1859096576'], [1706118186, '1859096576'], [1706118187, '1859096576'], [1706118188, '1859096576'], [1706118189, '1859096576'], [1706118190, '1859096576'], [1706118191, '1859096576'], [1706118192, '1859096576'], [1706118193, '1859096576'], [1706118194, '1859096576'], [1706118195, '1859096576'], [1706118196, '1859096576'], [1706118197, '1859096576'], [1706118198, '1859096576'], [1706118199, '1859096576'], [1706118200, '1859096576'], [1706118201, '1859096576'], [1706118202, '1611554816'], [1706118203, '1611554816'], [1706118204, '1611554816'], [1706118205, '1611554816'], [1706118206, '1611554816'], [1706118207, '1611554816'], [1706118208, '1611554816'], [1706118209, '1611554816'], [1706118210, '1611554816'], [1706118211, '1611554816'], [1706118212, '1611554816'], [1706118213, '1611554816'], [1706118214, '1611554816'], [1706118215, '1611554816'], [1706118216, '1611554816'], [1706118217, '1611554816'], [1706118218, '1611554816'], [1706118219, '1611554816'], [1706118220, '1611554816'], [1706118221, '3208851456'], [1706118222, '3208851456'], [1706118223, '3208851456'], [1706118224, '3208851456'], [1706118225, '3208851456'], [1706118226, '3208851456'], [1706118227, '3208851456'], [1706118228, '3208851456'], [1706118229, '3208851456'], [1706118230, '3208851456'], [1706118231, '3208851456'], [1706118232, '3208851456'], [1706118233, '3208851456'], [1706118234, '3208851456'], [1706118235, '3208851456'], [1706118236, '3208851456'], [1706118237, '3208851456'], [1706118238, '3208851456'], [1706118239, '3208851456'], [1706118240, '3208851456'], [1706118241, '3208851456'], [1706118242, '3208851456'], [1706118243, '3208851456'], [1706118244, '3208851456'], [1706118245, '3208851456'], [1706118246, '3208851456'], [1706118247, '3208851456'], [1706118248, '3208851456'], [1706118249, '3208851456'], [1706118250, '3208851456'], [1706118251, '3208851456'], [1706118252, '3208851456'], [1706118253, '3208851456'], [1706118254, '3208851456'], [1706118255, '3208851456'], [1706118256, '3208851456'], [1706118257, '3208851456'], [1706118258, '3208851456'], [1706118259, '3208851456'], [1706118260, '3208851456'], [1706118261, '3208851456'], [1706118262, '3208851456'], [1706118263, '2387591168'], [1706118264, '2387591168'], [1706118265, '2387591168'], [1706118266, '2387591168'], [1706118267, '2387591168'], [1706118268, '2387591168'], [1706118269, '2387591168'], [1706118270, '2387591168'], [1706118271, '2387591168'], [1706118272, '2387591168'], [1706118273, '2387591168'], [1706118274, '2387591168'], [1706118275, '2387591168'], [1706118276, '2387591168'], [1706118277, '2387591168'], [1706118278, '860766208'], [1706118279, '860766208'], [1706118280, '860766208'], [1706118281, '860766208'], [1706118282, '860766208'], [1706118283, '860766208'], [1706118284, '860766208'], [1706118285, '860766208'], [1706118286, '860766208'], [1706118287, '860766208'], [1706118288, '860766208'], [1706118289, '860766208'], [1706118290, '860766208'], [1706118291, '860766208'], [1706118292, '860766208'], [1706118293, '860766208'], [1706118294, '860766208'], [1706118295, '860766208'], [1706118296, '860766208'], [1706118297, '860766208'], [1706118298, '860766208'], [1706118299, '860766208'], [1706118300, '860766208'], [1706118301, '860766208'], [1706118302, '860766208'], [1706118303, '860766208'], [1706118304, '860766208'], [1706118305, '860766208'], [1706118306, '860766208'], [1706118307, '860766208'], [1706118308, '860766208'], [1706118309, '860766208'], [1706118310, '860766208'], [1706118311, '860766208'], [1706118312, '860766208'], [1706118313, '860766208'], [1706118314, '860766208'], [1706118315, '860766208'], [1706118316, '860766208'], [1706118317, '860766208'], [1706118318, '860766208'], [1706118319, '860766208'], [1706118320, '860766208'], [1706118321, '860766208'], [1706118322, '1179983872'], [1706118323, '1179983872'], [1706118324, '1179983872'], [1706118325, '1179983872'], [1706118326, '1179983872'], [1706118327, '1179983872'], [1706118328, '1179983872'], [1706118329, '1179983872'], [1706118330, '1179983872'], [1706118331, '1179983872'], [1706118332, '1179983872'], [1706118333, '1179983872'], [1706118334, '1179983872'], [1706118335, '821743616'], [1706118336, '821743616'], [1706118337, '821743616'], [1706118338, '821743616'], [1706118339, '821743616'], [1706118340, '821743616'], [1706118341, '821743616'], [1706118342, '821743616'], [1706118343, '821743616'], [1706118344, '821743616'], [1706118345, '821743616'], [1706118346, '821743616'], [1706118347, '821743616'], [1706118348, '821743616'], [1706118349, '821743616'], [1706118350, '821743616'], [1706118351, '821743616'], [1706118352, '821743616'], [1706118353, '821743616'], [1706118354, '821743616'], [1706118355, '821743616'], [1706118356, '821743616'], [1706118357, '821743616'], [1706118358, '821743616'], [1706118359, '821743616'], [1706118360, '821743616'], [1706118361, '821743616'], [1706118362, '821743616'], [1706118363, '821743616'], [1706118364, '821743616'], [1706118365, '821743616'], [1706118366, '821743616'], [1706118367, '821743616'], [1706118368, '821743616'], [1706118369, '821743616'], [1706118370, '821743616'], [1706118371, '821743616'], [1706118372, '821743616'], [1706118373, '821743616'], [1706118374, '821743616'], [1706118375, '821743616'], [1706118376, '821743616'], [1706118377, '821743616'], [1706118378, '821743616'], [1706118379, '821743616'], [1706118380, '821743616'], [1706118381, '1029611520'], [1706118382, '1029611520'], [1706118383, '1029611520'], [1706118384, '1029611520'], [1706118385, '1029611520'], [1706118386, '1029611520'], [1706118387, '1029611520'], [1706118388, '1029611520'], [1706118389, '1029611520'], [1706118390, '1029611520'], [1706118391, '1029611520'], [1706118392, '1029611520'], [1706118393, '1029611520'], [1706118394, '1029611520'], [1706118395, '1029611520'], [1706118396, '1029611520'], [1706118397, '1029611520'], [1706118398, '1029611520'], [1706118399, '1029611520'], [1706118400, '1029611520'], [1706118401, '1029611520'], [1706118402, '1029611520'], [1706118403, '1029611520'], [1706118404, '1029611520'], [1706118405, '1029611520'], [1706118406, '594620416'], [1706118407, '594620416'], [1706118408, '594620416'], [1706118409, '594620416'], [1706118410, '594620416'], [1706118411, '594620416'], [1706118412, '594620416'], [1706118413, '594620416'], [1706118414, '594620416'], [1706118415, '594620416'], [1706118416, '594620416'], [1706118417, '594620416'], [1706118418, '594620416'], [1706118419, '594620416'], [1706118420, '594620416']]}]}} -VALID_CPU_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'container': 'build', 'cpu': 'total', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117145, '0.2483743618267752'], [1706117146, '0.25650526138466395'], [1706117147, '0.26463616094255266'], [1706117148, '0.2727670605004414'], [1706117149, '0.28089796005833007'], [1706117150, '0.2890288596162188'], [1706117151, '0.2971597591741076'], [1706117152, '0.30529065873199623'], [1706117153, '0.313421558289885'], [1706117154, '0.3215524578477737'], [1706117155, '0.3296833574056624'], [1706117156, '0.3378142569635511'], [1706117157, '0.3459451565214398'], [1706117158, '0.3540760560793285'], [1706117159, '0.3622069556372173'], [1706117160, '0.370337855195106'], [1706117161, '0.3784687547529946'], [1706117162, '0.38659965431088333'], [1706117163, '0.3947305538687721'], [1706117164, '0.4028614534266608'], [1706117165, '0.41099235298454945'], [1706117166, '0.4191232525424382'], [1706117167, '0.4272541521003269'], [1706117168, '0.4353850516582156'], [1706117169, '0.4435159512161044'], [1706117170, '0.45164685077399314'], [1706117171, '0.4597777503318818'], [1706117172, '0.4679086498897705'], [1706117173, '0.47603954944765925'], [1706117174, '0.48417044900554795'], [1706117175, '0.4923013485634366'], [1706117176, '0.5941941747822107'], [1706117177, '0.6038490390156116'], [1706117178, '0.6135039032490125'], [1706117179, '0.6231587674824133'], [1706117180, '0.6328136317158142'], [1706117181, '0.6424684959492151'], [1706117182, '0.652123360182616'], [1706117183, '0.6617782244160167'], [1706117184, '0.6714330886494176'], [1706117185, '0.6810879528828185'], [1706117186, '0.6907428171162194'], [1706117187, '0.7003976813496203'], [1706117188, '0.7100525455830212'], [1706117189, '0.7197074098164219'], [1706117190, '0.7293622740498229'], [1706117191, '0.7390171382832237'], [1706117192, '0.7486720025166246'], [1706117193, '0.7583268667500255'], [1706117194, '0.7679817309834264'], [1706117195, '0.7776365952168272'], [1706117196, '0.787291459450228'], [1706117197, '0.7969463236836289'], [1706117198, '0.8066011879170297'], [1706117199, '0.8162560521504307'], [1706117200, '0.8259109163838315'], [1706117201, '0.8355657806172325'], [1706117202, '0.8452206448506332'], [1706117203, '0.8548755090840342'], [1706117204, '0.8645303733174351'], [1706117205, '0.9164274605130325'], [1706117206, '0.9253149079610642'], [1706117207, '0.9363825414782355'], [1706117208, '0.947450174995407'], [1706117209, '0.9585178085125783'], [1706117210, '0.9695854420297495'], [1706117211, '0.9806530755469209'], [1706117212, '0.9917207090640923'], [1706117213, '0.9960870165454214'], [1706117214, '0.9960870165454214'], [1706117215, '0.9960870165454214'], [1706117216, '0.9960870165454214'], [1706117217, '0.9960870165454214'], [1706117218, '0.9960870165454214'], [1706117219, '0.9960870165454214'], [1706117220, '0.9960870165454214'], [1706117221, '0.9960870165454214'], [1706117222, '0.9960870165454214'], [1706117223, '0.9960870165454214'], [1706117224, '0.9960870165454214'], [1706117225, '0.9965632246557574'], [1706117226, '0.9965632246557574'], [1706117227, '0.9965632246557574'], [1706117228, '0.9965632246557574'], [1706117229, '0.9965632246557574'], [1706117230, '0.9965632246557574'], [1706117231, '0.9965632246557574'], [1706117232, '0.9965632246557574'], [1706117233, '0.9965632246557574'], [1706117234, '0.9965632246557574'], [1706117235, '0.7884960963099941'], [1706117236, '0.799549211752331'], [1706117237, '0.810602327194668'], [1706117238, '0.821655442637005'], [1706117239, '0.8327085580793421'], [1706117240, '0.9947803898103293'], [1706117241, '0.9947803898103293'], [1706117242, '0.9947803898103293'], [1706117243, '0.9947803898103293'], [1706117244, '0.9947803898103293'], [1706117245, '0.9947803898103293'], [1706117246, '0.9947803898103293'], [1706117247, '0.9947803898103293'], [1706117248, '0.9947803898103293'], [1706117249, '0.9947803898103293'], [1706117250, '0.9947803898103293'], [1706117251, '0.9947803898103293'], [1706117252, '0.8260766888139397'], [1706117253, '0.8150235733716028'], [1706117254, '0.8039704579292658'], [1706117255, '0.7929173424869287'], [1706117256, '0.7818642270445917'], [1706117257, '0.7708111116022549'], [1706117258, '0.7597579961599177'], [1706117259, '0.7487048807175807'], [1706117260, '0.7376517652752437'], [1706117261, '0.7265986498329068'], [1706117262, '0.7155455343905697'], [1706117263, '0.7044924189482329'], [1706117264, '0.6934393035058959'], [1706117265, '0.6823861880635589'], [1706117266, '0.43370360155555576'], [1706117267, '0.43370360155555576'], [1706117268, '0.43370360155555576'], [1706117269, '0.43370360155555576'], [1706117270, '0.43370360155555576'], [1706117271, '0.43370360155555576'], [1706117272, '5.566624524200288'], [1706117273, '5.6360656043509785'], [1706117274, '5.705506684501669'], [1706117275, '5.77494776465236'], [1706117276, '5.844388844803051'], [1706117277, '5.913829924953742'], [1706117278, '5.9832710051044335'], [1706117279, '6.052712085255124'], [1706117280, '6.122153165405815'], [1706117281, '6.191594245556507'], [1706117282, '6.249697213562188'], [1706117283, '6.249697213562188'], [1706117284, '6.249697213562188'], [1706117285, '6.249697213562188'], [1706117286, '6.249697213562188'], [1706117287, '6.249697213562188'], [1706117288, '6.249697213562188'], [1706117289, '6.249697213562188'], [1706117290, '6.249697213562188'], [1706117291, '6.249697213562188'], [1706117292, '6.249697213562188'], [1706117293, '6.249697213562188'], [1706117294, '6.249697213562188'], [1706117295, '6.249697213562188'], [1706117296, '7.8382055254153675'], [1706117297, '7.932086901178828'], [1706117298, '8.025968276942288'], [1706117299, '8.11984965270575'], [1706117300, '6.795147485867362'], [1706117301, '6.795147485867362'], [1706117302, '6.795147485867362'], [1706117303, '6.795147485867362'], [1706117304, '6.795147485867362'], [1706117305, '6.795147485867362'], [1706117306, '6.795147485867362'], [1706117307, '6.795147485867362'], [1706117308, '6.795147485867362'], [1706117309, '6.795147485867362'], [1706117310, '6.795147485867362'], [1706117311, '6.795147485867362'], [1706117312, '6.795147485867362'], [1706117313, '6.795147485867362'], [1706117314, '6.795147485867362'], [1706117315, '2.5720481747280175'], [1706117316, '2.616665769460188'], [1706117317, '2.661283364192357'], [1706117318, '2.705900958924526'], [1706117319, '2.750518553656696'], [1706117320, '2.7951361483888655'], [1706117321, '2.8397537431210353'], [1706117322, '2.8843713378532048'], [1706117323, '2.928988932585375'], [1706117324, '2.9736065273175445'], [1706117325, '3.0182241220497144'], [1706117326, '2.480837658629061'], [1706117327, '2.517210244112489'], [1706117328, '2.5535828295959164'], [1706117329, '2.5899554150793445'], [1706117330, '2.626328000562773'], [1706117331, '2.662700586046201'], [1706117332, '3.2735326935085207'], [1706117333, '3.2735326935085207'], [1706117334, '3.2735326935085207'], [1706117335, '3.2735326935085207'], [1706117336, '3.2735326935085207'], [1706117337, '3.2735326935085207'], [1706117338, '3.2735326935085207'], [1706117339, '3.2735326935085207'], [1706117340, '3.2735326935085207'], [1706117341, '3.2735326935085207'], [1706117342, '3.2735326935085207'], [1706117343, '3.2735326935085207'], [1706117344, '3.2735326935085207'], [1706117345, '3.2735326935085207'], [1706117346, '3.2735326935085207'], [1706117347, '3.2735326935085207'], [1706117348, '3.2735326935085207'], [1706117349, '3.2735326935085207'], [1706117350, '3.2735326935085207'], [1706117351, '3.2735326935085207'], [1706117352, '3.2735326935085207'], [1706117353, '3.2735326935085207'], [1706117354, '3.2735326935085207'], [1706117355, '2.673648734276713'], [1706117356, '3.7319005481816236'], [1706117357, '3.7319005481816236'], [1706117358, '3.7319005481816236'], [1706117359, '3.7319005481816236'], [1706117360, '3.7319005481816245'], [1706117361, '3.7319005481816245'], [1706117362, '3.592445628138214'], [1706117363, '3.592445628138214'], [1706117364, '3.592445628138214'], [1706117365, '3.592445628138214'], [1706117366, '3.592445628138214'], [1706117367, '3.592445628138214'], [1706117368, '3.592445628138214'], [1706117369, '3.592445628138214'], [1706117370, '3.592445628138214'], [1706117371, '3.592445628138214'], [1706117372, '3.592445628138214'], [1706117373, '3.592445628138214'], [1706117374, '3.592445628138214'], [1706117375, '3.592445628138214'], [1706117376, '3.592445628138214'], [1706117377, '3.592445628138214'], [1706117378, '3.592445628138214'], [1706117379, '3.592445628138214'], [1706117380, '3.592445628138214'], [1706117381, '3.592445628138214'], [1706117382, '3.592445628138214'], [1706117383, '3.592445628138214'], [1706117384, '3.592445628138214'], [1706117385, '3.592445628138214'], [1706117386, '3.592445628138214'], [1706117387, '2.9011892361749854'], [1706117388, '2.8612731736401162'], [1706117389, '2.8213571111052476'], [1706117390, '3.57200022999401'], [1706117391, '3.2099744324773454'], [1706117392, '3.2099744324773454'], [1706117393, '3.2099744324773454'], [1706117394, '3.2099744324773454'], [1706117395, '3.2099744324773454'], [1706117396, '3.2099744324773454'], [1706117397, '3.2099744324773454'], [1706117398, '3.2099744324773454'], [1706117399, '3.2099744324773454'], [1706117400, '3.2099744324773454'], [1706117401, '3.2099744324773454'], [1706117402, '3.2099744324773454'], [1706117403, '3.2099744324773454'], [1706117404, '3.2099744324773454'], [1706117405, '3.2099744324773454'], [1706117406, '3.2099744324773454'], [1706117407, '3.5208871213381445'], [1706117408, '3.5208871213381445'], [1706117409, '3.5208871213381445'], [1706117410, '3.5208871213381445'], [1706117411, '3.5208871213381445'], [1706117412, '3.5208871213381445'], [1706117413, '3.5208871213381445'], [1706117414, '3.5208871213381445'], [1706117415, '3.5208871213381445'], [1706117416, '2.3722076083852355'], [1706117417, '2.4045871696667267'], [1706117418, '2.914160515334191'], [1706117419, '2.914160515334191'], [1706117420, '2.914160515334191'], [1706117421, '2.914160515334191'], [1706117422, '2.914160515334191'], [1706117423, '2.914160515334191'], [1706117424, '2.914160515334191'], [1706117425, '2.914160515334191'], [1706117426, '2.914160515334191'], [1706117427, '2.914160515334191'], [1706117428, '2.914160515334191'], [1706117429, '2.914160515334191'], [1706117430, '2.914160515334191'], [1706117431, '2.914160515334191'], [1706117432, '2.914160515334191'], [1706117433, '2.914160515334191'], [1706117434, '2.914160515334191'], [1706117435, '2.914160515334191'], [1706117436, '2.383929009569135'], [1706117437, '2.351549448287644'], [1706117438, '2.319169887006153'], [1706117439, '2.286790325724662'], [1706117440, '3.312709442835095'], [1706117441, '3.312709442835095'], [1706117442, '3.312709442835095'], [1706117443, '3.312709442835095'], [1706117444, '3.312709442835095'], [1706117445, '3.312709442835095'], [1706117446, '3.191276505935721'], [1706117447, '3.237942487846151'], [1706117448, '3.2846084697565816'], [1706117449, '3.331274451667012'], [1706117450, '3.377940433577442'], [1706117451, '3.4246064154878724'], [1706117452, '3.471272397398302'], [1706117453, '4.199938371938713'], [1706117454, '4.199938371938713'], [1706117455, '4.199938371938713'], [1706117456, '4.199938371938713'], [1706117457, '4.199938371938713'], [1706117458, '4.199938371938713'], [1706117459, '4.199938371938713'], [1706117460, '4.199938371938713'], [1706117461, '4.199938371938713'], [1706117462, '4.199938371938713'], [1706117463, '4.199938371938713'], [1706117464, '4.199938371938713'], [1706117465, '4.199938371938713'], [1706117466, '4.199938371938713'], [1706117467, '4.199938371938713'], [1706117468, '3.4633391804735285'], [1706117469, '3.4166731985630983'], [1706117470, '3.370007216652668'], [1706117471, '3.3233412347422386'], [1706117472, '3.2766752528318084'], [1706117473, '3.230009270921378'], [1706117474, '3.183343289010948'], [1706117475, '3.1366773071005176'], [1706117476, '3.0900113251900874'], [1706117477, '3.043345343279658'], [1706117478, '2.9966793613692277'], [1706117479, '3.611361640487743'], [1706117480, '3.611361640487743'], [1706117481, '3.348981988671299'], [1706117482, '3.348981988671299'], [1706117483, '3.348981988671299'], [1706117484, '3.348981988671299'], [1706117485, '3.348981988671299'], [1706117486, '3.348981988671299'], [1706117487, '3.348981988671299'], [1706117488, '3.348981988671299'], [1706117489, '3.348981988671299'], [1706117490, '3.348981988671299'], [1706117491, '3.348981988671299'], [1706117492, '3.348981988671299'], [1706117493, '3.348981988671299'], [1706117494, '3.348981988671299'], [1706117495, '3.348981988671299'], [1706117496, '3.501373337123475'], [1706117497, '2.56696583274599'], [1706117498, '2.6030391162067383'], [1706117499, '2.639112399667487'], [1706117500, '3.246595511467355'], [1706117501, '3.246595511467355'], [1706117502, '3.246595511467355'], [1706117503, '3.246595511467355'], [1706117504, '3.246595511467355'], [1706117505, '3.246595511467355'], [1706117506, '3.246595511467355'], [1706117507, '3.246595511467355'], [1706117508, '3.246595511467355'], [1706117509, '3.246595511467355'], [1706117510, '3.246595511467355'], [1706117511, '3.246595511467355'], [1706117512, '3.246595511467355'], [1706117513, '3.246595511467355'], [1706117514, '3.246595511467355'], [1706117515, '3.246595511467355'], [1706117516, '3.246595511467355'], [1706117517, '3.246595511467355'], [1706117518, '3.246595511467355'], [1706117519, '3.246595511467355'], [1706117520, '3.246595511467355'], [1706117521, '3.246595511467355'], [1706117522, '3.246595511467355'], [1706117523, '3.246595511467355'], [1706117524, '3.246595511467355'], [1706117525, '3.246595511467355'], [1706117526, '3.246595511467355'], [1706117527, '2.6149793730322464'], [1706117528, '3.0222591350532837'], [1706117529, '3.0222591350532837'], [1706117530, '2.2579154362078415'], [1706117531, '2.293004976727566'], [1706117532, '2.3280945172472904'], [1706117533, '2.363184057767015'], [1706117534, '2.3982735982867394'], [1706117535, '2.4333631388064636'], [1706117536, '2.4684526793261883'], [1706117537, '2.503542219845913'], [1706117538, '2.5386317603656376'], [1706117539, '2.5737213008853623'], [1706117540, '2.608810841405086'], [1706117541, '3.158058646775204'], [1706117542, '3.158058646775204'], [1706117543, '3.158058646775204'], [1706117544, '3.158058646775204'], [1706117545, '3.158058646775204'], [1706117546, '3.158058646775204'], [1706117547, '3.158058646775204'], [1706117548, '3.158058646775204'], [1706117549, '3.158058646775204'], [1706117550, '3.158058646775204'], [1706117551, '3.158058646775204'], [1706117552, '3.158058646775204'], [1706117553, '3.158058646775204'], [1706117554, '3.158058646775204'], [1706117555, '2.631934848607585'], [1706117556, '2.59684530808786'], [1706117557, '2.5617557675681355'], [1706117558, '2.5266662270484117'], [1706117559, '2.491576686528687'], [1706117560, '2.456487146008963'], [1706117561, '2.421397605489238'], [1706117562, '2.386308064969514'], [1706117563, '2.351218524449789'], [1706117564, '2.3161289839300645'], [1706117565, '2.2810394434103403'], [1706117566, '3.155680548976084'], [1706117567, '3.155680548976084'], [1706117568, '3.155680548976084'], [1706117569, '2.915342155760893'], [1706117570, '2.915342155760893'], [1706117571, '2.915342155760893'], [1706117572, '2.915342155760893'], [1706117573, '2.915342155760893'], [1706117574, '2.915342155760893'], [1706117575, '2.915342155760893'], [1706117576, '2.915342155760893'], [1706117577, '2.915342155760893'], [1706117578, '2.915342155760893'], [1706117579, '2.915342155760893'], [1706117580, '2.915342155760893'], [1706117581, '2.915342155760893'], [1706117582, '2.915342155760893'], [1706117583, '2.915342155760893'], [1706117584, '2.915342155760893'], [1706117585, '2.915342155760893'], [1706117586, '3.1526011359717856'], [1706117587, '3.1526011359717856'], [1706117588, '3.1526011359717856'], [1706117589, '3.1526011359717856'], [1706117590, '3.1526011359717856'], [1706117591, '3.1526011359717856'], [1706117592, '3.1526011359717856'], [1706117593, '3.1526011359717856'], [1706117594, '3.1526011359717856'], [1706117595, '3.1526011359717856'], [1706117596, '3.1526011359717856'], [1706117597, '3.1526011359717856'], [1706117598, '2.884683472923464'], [1706117599, '2.884683472923464'], [1706117600, '2.884683472923464'], [1706117601, '2.884683472923464'], [1706117602, '2.884683472923464'], [1706117603, '2.884683472923464'], [1706117604, '2.884683472923464'], [1706117605, '2.884683472923464'], [1706117606, '2.884683472923464'], [1706117607, '2.884683472923464'], [1706117608, '2.884683472923464'], [1706117609, '2.884683472923464'], [1706117610, '2.884683472923464'], [1706117611, '2.884683472923464'], [1706117612, '2.884683472923464'], [1706117613, '2.884683472923464'], [1706117614, '2.884683472923464'], [1706117615, '2.884683472923464'], [1706117616, '2.884683472923464'], [1706117617, '2.884683472923464'], [1706117618, '1.9315586770085436'], [1706117619, '1.9600041099286738'], [1706117620, '1.9884495428488045'], [1706117621, '2.0168949757689347'], [1706117622, '2.5600889628117476'], [1706117623, '2.5600889628117476'], [1706117624, '2.5600889628117476'], [1706117625, '2.5600889628117476'], [1706117626, '2.5600889628117476'], [1706117627, '2.5600889628117476'], [1706117628, '1.8256912498379172'], [1706117629, '1.8256912498379172'], [1706117630, '1.8256912498379172'], [1706117631, '1.8256912498379172'], [1706117632, '1.8256912498379172'], [1706117633, '1.8256912498379172'], [1706117634, '1.8256912498379172'], [1706117635, '1.8256912498379172'], [1706117636, '1.8256912498379172'], [1706117637, '1.8256912498379172'], [1706117638, '1.8256912498379172'], [1706117639, '1.8256912498379172'], [1706117640, '1.8256912498379172'], [1706117641, '1.8256912498379172'], [1706117642, '1.8256912498379172'], [1706117643, '1.8256912498379172'], [1706117644, '1.8256912498379172'], [1706117645, '1.8256912498379172'], [1706117646, '1.8256912498379172'], [1706117647, '1.8256912498379172'], [1706117648, '1.8256912498379172'], [1706117649, '1.8256912498379172'], [1706117650, '1.8256912498379172'], [1706117651, '1.8256912498379172'], [1706117652, '1.8256912498379172'], [1706117653, '1.8256912498379172'], [1706117654, '1.8256912498379172'], [1706117655, '1.8256912498379172'], [1706117656, '1.0547932385714336'], [1706117657, '1.0547932385714336'], [1706117658, '1.0547932385714336'], [1706117659, '1.0547932385714336'], [1706117660, '1.7007816403976095'], [1706117661, '1.7007816403976095'], [1706117662, '1.7007816403976097'], [1706117663, '1.7007816403976097'], [1706117664, '1.7007816403976097'], [1706117665, '1.7007816403976097'], [1706117666, '1.7007816403976097'], [1706117667, '1.7007816403976097'], [1706117668, '1.7007816403976097'], [1706117669, '1.7007816403976097'], [1706117670, '1.7007816403976097'], [1706117671, '1.7007816403976097'], [1706117672, '1.7007816403976097'], [1706117673, '1.7007816403976097'], [1706117674, '1.7007816403976097'], [1706117675, '1.7007816403976097'], [1706117676, '1.7007816403976097'], [1706117677, '1.7007816403976097'], [1706117678, '1.7007816403976097'], [1706117679, '1.7007816403976097'], [1706117680, '1.7007816403976097'], [1706117681, '1.7007816403976097'], [1706117682, '1.7007816403976097'], [1706117683, '1.7007816403976097'], [1706117684, '1.7007816403976097'], [1706117685, '1.7007816403976097'], [1706117686, '1.7007816403976097'], [1706117687, '1.7007816403976097'], [1706117688, '2.318687160130923'], [1706117689, '2.0919816211073194'], [1706117690, '2.0919816211073194'], [1706117691, '2.0919816211073194'], [1706117692, '2.0919816211073194'], [1706117693, '2.0919816211073194'], [1706117694, '2.0919816211073194'], [1706117695, '2.0919816211073194'], [1706117696, '2.0919816211073194'], [1706117697, '2.0919816211073194'], [1706117698, '2.0919816211073194'], [1706117699, '2.0919816211073194'], [1706117700, '2.0919816211073194'], [1706117701, '2.0919816211073194'], [1706117702, '2.0919816211073194'], [1706117703, '2.0919816211073194'], [1706117704, '2.0919816211073194'], [1706117705, '2.0919816211073194'], [1706117706, '2.0919816211073194'], [1706117707, '2.0919816211073194'], [1706117708, '1.9021084940746573'], [1706117709, '1.9021084940746573'], [1706117710, '1.9021084940746573'], [1706117711, '1.9021084940746573'], [1706117712, '1.9021084940746573'], [1706117713, '1.9021084940746573'], [1706117714, '1.9021084940746573'], [1706117715, '1.9021084940746573'], [1706117716, '1.9021084940746573'], [1706117717, '1.9021084940746573'], [1706117718, '1.2880315559128186'], [1706117719, '1.3061593772274451'], [1706117720, '1.3242871985420714'], [1706117721, '1.342415019856698'], [1706117722, '1.3605428411713243'], [1706117723, '1.6315039183163735'], [1706117724, '1.6315039183163735'], [1706117725, '1.6315039183163735'], [1706117726, '1.6315039183163735'], [1706117727, '1.6315039183163735'], [1706117728, '1.6315039183163735'], [1706117729, '1.6315039183163735'], [1706117730, '1.6315039183163735'], [1706117731, '1.6315039183163735'], [1706117732, '1.6315039183163735'], [1706117733, '1.6315039183163735'], [1706117734, '1.6315039183163735'], [1706117735, '1.3525484719715741'], [1706117736, '1.3344206506569478'], [1706117737, '1.3162928293423213'], [1706117738, '1.2981650080276952'], [1706117739, '1.2800371867130684'], [1706117740, '1.2619093653984426'], [1706117741, '1.243781544083816'], [1706117742, '1.2256537227691895'], [1706117743, '1.2075259014545632'], [1706117744, '1.1893980801399369'], [1706117745, '1.1712702588253103'], [1706117746, '1.153142437510684'], [1706117747, '1.1350146161960577'], [1706117748, '1.1168867948814314'], [1706117749, '1.0987589735668049'], [1706117750, '1.0898148498594316'], [1706117751, '1.0898148498594316'], [1706117752, '1.0898148498594316'], [1706117753, '1.0898148498594316'], [1706117754, '1.0898148498594316'], [1706117755, '1.0898148498594316'], [1706117756, '1.0898148498594316'], [1706117757, '1.0898148498594316'], [1706117758, '1.0898148498594316'], [1706117759, '1.0898148498594316'], [1706117760, '1.0898148498594316'], [1706117761, '1.0898148498594316'], [1706117762, '1.0898148498594316'], [1706117763, '1.0898148498594316'], [1706117764, '1.0898148498594316'], [1706117765, '1.0898148498594316'], [1706117766, '1.0898148498594316'], [1706117767, '1.0898148498594316'], [1706117768, '1.0898148498594316'], [1706117769, '1.0898148498594316'], [1706117770, '1.0898148498594316'], [1706117771, '1.0898148498594316'], [1706117772, '1.0898148498594316'], [1706117773, '1.0898148498594316'], [1706117774, '1.0898148498594316'], [1706117775, '1.0898148498594316'], [1706117776, '1.0898148498594316'], [1706117777, '1.0622381643896492'], [1706117778, '1.0622381643896492'], [1706117779, '0.9999662272661242'], [1706117780, '0.9999662272661242'], [1706117781, '0.9999662272661242'], [1706117782, '0.9999662272661242'], [1706117783, '0.9999662272661242'], [1706117784, '0.9999662272661242'], [1706117785, '0.9999662272661242'], [1706117786, '0.9999662272661242'], [1706117787, '0.9999662272661242'], [1706117788, '0.9999662272661242'], [1706117789, '0.9999662272661242'], [1706117790, '0.9999662272661242'], [1706117791, '0.9999662272661242'], [1706117792, '0.9999662272661242'], [1706117793, '0.9999662272661242'], [1706117794, '0.9999662272661242'], [1706117795, '0.9999662272661242'], [1706117796, '0.9999662272661242'], [1706117797, '0.9999662272661242'], [1706117798, '0.6893592223628728'], [1706117799, '0.7004700344053406'], [1706117800, '0.7115808464478085'], [1706117801, '0.7226916584902763'], [1706117802, '0.7338024705327442'], [1706117803, '0.7449132825752118'], [1706117804, '0.7560240946176797'], [1706117805, '0.7671349066601475'], [1706117806, '0.5986061095999958'], [1706117807, '0.7985815694614367'], [1706117808, '0.9996883117547429'], [1706117809, '0.9996883117547429'], [1706117810, '0.9996883117547429'], [1706117811, '0.9996883117547429'], [1706117812, '0.9996883117547429'], [1706117813, '0.9996883117547429'], [1706117814, '0.9996883117547429'], [1706117815, '0.9996883117547427'], [1706117816, '0.9996883117547427'], [1706117817, '0.9996883117547427'], [1706117818, '0.9996883117547427'], [1706117819, '0.9996883117547427'], [1706117820, '0.9996883117547427'], [1706117821, '0.9996883117547427'], [1706117822, '0.9996883117547427'], [1706117823, '0.9996883117547427'], [1706117824, '0.9996883117547427'], [1706117825, '0.9996883117547427'], [1706117826, '0.9996883117547427'], [1706117827, '0.9996883117547427'], [1706117828, '0.9996883117547427'], [1706117829, '0.9996883117547427'], [1706117830, '0.9996883117547427'], [1706117831, '0.9996883117547427'], [1706117832, '0.9996883117547427'], [1706117833, '0.9996883117547427'], [1706117834, '0.9996883117547427'], [1706117835, '0.9993381495398761'], [1706117836, '0.9993381495398761'], [1706117837, '0.9993381495398761'], [1706117838, '0.9993381495398761'], [1706117839, '0.9993381495398761'], [1706117840, '0.9990423476012606'], [1706117841, '0.9990423476012606'], [1706117842, '0.9990423476012606'], [1706117843, '0.9990423476012606'], [1706117844, '0.9990423476012606'], [1706117845, '0.9990423476012606'], [1706117846, '0.9990423476012606'], [1706117847, '0.9990423476012606'], [1706117848, '0.9990423476012606'], [1706117849, '0.9990423476012606'], [1706117850, '0.9990423476012606'], [1706117851, '0.9990423476012606'], [1706117852, '0.9990423476012606'], [1706117853, '0.9990423476012606'], [1706117854, '0.9990423476012606'], [1706117855, '0.9990423476012606'], [1706117856, '0.9990423476012606'], [1706117857, '0.9990423476012606'], [1706117858, '0.9990423476012606'], [1706117859, '0.9990423476012606'], [1706117860, '0.9990423476012606'], [1706117861, '0.9990423476012606'], [1706117862, '0.9990423476012606'], [1706117863, '0.9889619143389414'], [1706117864, '0.9889619143389414'], [1706117865, '0.9889619143389414'], [1706117866, '0.9889619143389414'], [1706117867, '0.9832407125162015'], [1706117868, '0.9832407125162015'], [1706117869, '0.9832407125162015'], [1706117870, '0.9832407125162015'], [1706117871, '0.9832407125162015'], [1706117872, '0.9832407125162015'], [1706117873, '0.9832407125162015'], [1706117874, '0.9832407125162015'], [1706117875, '0.9832407125162015'], [1706117876, '0.9832407125162015'], [1706117877, '0.9832407125162015'], [1706117878, '0.9832407125162015'], [1706117879, '0.9832407125162015'], [1706117880, '0.9832407125162015'], [1706117881, '0.9832407125162015'], [1706117882, '0.9832407125162015'], [1706117883, '0.9832407125162015'], [1706117884, '0.9832407125162015'], [1706117885, '0.9832407125162015'], [1706117886, '0.9832407125162015'], [1706117887, '0.9832407125162015'], [1706117888, '0.9832407125162015'], [1706117889, '0.9832407125162017'], [1706117890, '0.9832407125162017'], [1706117891, '0.9832407125162017'], [1706117892, '0.9832407125162017'], [1706117893, '1.0642434901894462'], [1706117894, '1.0642434901894462'], [1706117895, '1.0642434901894462'], [1706117896, '1.0642434901894462'], [1706117897, '1.0952123724224387'], [1706117898, '1.0952123724224387'], [1706117899, '1.0952123724224387'], [1706117900, '1.0952123724224387'], [1706117901, '1.0952123724224387'], [1706117902, '1.0952123724224387'], [1706117903, '1.0952123724224387'], [1706117904, '1.0952123724224387'], [1706117905, '1.0952123724224387'], [1706117906, '1.0952123724224387'], [1706117907, '1.0952123724224387'], [1706117908, '1.0952123724224387'], [1706117909, '1.0952123724224387'], [1706117910, '1.0952123724224387'], [1706117911, '1.0952123724224387'], [1706117912, '1.0952123724224387'], [1706117913, '1.0952123724224387'], [1706117914, '1.0952123724224387'], [1706117915, '1.0952123724224387'], [1706117916, '1.0952123724224387'], [1706117917, '1.0952123724224387'], [1706117918, '1.0952123724224387'], [1706117919, '1.1850365840831112'], [1706117920, '1.1850365840831112'], [1706117921, '1.1850365840831112'], [1706117922, '1.1850365840831112'], [1706117923, '1.1850365840831112'], [1706117924, '1.1850365840831112'], [1706117925, '1.2922856899894286'], [1706117926, '1.2922856899894286'], [1706117927, '1.2922856899894286'], [1706117928, '1.2922856899894286'], [1706117929, '1.2922856899894286'], [1706117930, '1.2922856899894286'], [1706117931, '1.2922856899894286'], [1706117932, '1.2922856899894286'], [1706117933, '1.2922856899894286'], [1706117934, '1.2922856899894286'], [1706117935, '1.2922856899894286'], [1706117936, '1.2922856899894286'], [1706117937, '1.2922856899894286'], [1706117938, '1.2922856899894286'], [1706117939, '1.2922856899894286'], [1706117940, '1.2922856899894286'], [1706117941, '1.2922856899894286'], [1706117942, '1.2922856899894286'], [1706117943, '1.2922856899894286'], [1706117944, '1.2922856899894286'], [1706117945, '1.2922856899894286'], [1706117946, '1.2922856899894286'], [1706117947, '2.0539061765637032'], [1706117948, '2.0539061765637032'], [1706117949, '2.0539061765637032'], [1706117950, '2.0539061765637032'], [1706117951, '2.0539061765637032'], [1706117952, '2.0539061765637032'], [1706117953, '2.5196023966957344'], [1706117954, '2.5196023966957344'], [1706117955, '2.5196023966957344'], [1706117956, '2.5196023966957344'], [1706117957, '2.5196023966957344'], [1706117958, '2.5196023966957344'], [1706117959, '2.5196023966957344'], [1706117960, '2.5196023966957344'], [1706117961, '2.5196023966957344'], [1706117962, '2.5196023966957344'], [1706117963, '2.5196023966957344'], [1706117964, '2.5196023966957344'], [1706117965, '2.5196023966957344'], [1706117966, '2.5196023966957344'], [1706117967, '2.5196023966957344'], [1706117968, '2.5196023966957344'], [1706117969, '2.5196023966957344'], [1706117970, '2.5196023966957344'], [1706117971, '2.5196023966957344'], [1706117972, '2.5196023966957344'], [1706117973, '2.5196023966957344'], [1706117974, '2.5196023966957344'], [1706117975, '2.5196023966957344'], [1706117976, '2.5196023966957344'], [1706117977, '2.058284194547385'], [1706117978, '2.0302886123618764'], [1706117979, '2.0022930301763684'], [1706117980, '1.9742974479908604'], [1706117981, '1.9463018658053521'], [1706117982, '5.694601278566396'], [1706117983, '7.517450980584689'], [1706117984, '7.517450980584689'], [1706117985, '7.517450980584689'], [1706117986, '7.517450980584689'], [1706117987, '7.517450980584689'], [1706117988, '7.517450980584689'], [1706117989, '7.517450980584689'], [1706117990, '7.517450980584689'], [1706117991, '7.517450980584689'], [1706117992, '7.517450980584689'], [1706117993, '7.517450980584689'], [1706117994, '7.517450980584689'], [1706117995, '7.517450980584689'], [1706117996, '7.517450980584689'], [1706117997, '7.517450980584689'], [1706117998, '7.517450980584689'], [1706117999, '7.517450980584689'], [1706118000, '7.517450980584689'], [1706118001, '7.517450980584689'], [1706118002, '7.517450980584689'], [1706118003, '7.517450980584689'], [1706118004, '7.517450980584689'], [1706118005, '7.517450980584689'], [1706118006, '7.517450980584689'], [1706118007, '7.517450980584689'], [1706118008, '7.517450980584689'], [1706118009, '10.647297690281198'], [1706118010, '10.647297690281198'], [1706118011, '10.647297690281198'], [1706118012, '10.647297690281198'], [1706118013, '10.647297690281198'], [1706118014, '10.647297690281198'], [1706118015, '10.647297690281198'], [1706118016, '10.647297690281198'], [1706118017, '10.647297690281198'], [1706118018, '10.63733514331065'], [1706118019, '10.63733514331065'], [1706118020, '10.63733514331065'], [1706118021, '10.637335143310652'], [1706118022, '10.637335143310652'], [1706118023, '10.637335143310652'], [1706118024, '10.637335143310652'], [1706118025, '10.637335143310652'], [1706118026, '10.637335143310652'], [1706118027, '10.637335143310652'], [1706118028, '10.637335143310652'], [1706118029, '10.637335143310652'], [1706118030, '10.637335143310652'], [1706118031, '10.637335143310652'], [1706118032, '10.637335143310652'], [1706118033, '10.637335143310652'], [1706118034, '10.63733514331065'], [1706118035, '10.63733514331065'], [1706118036, '10.63733514331065'], [1706118037, '10.627572753016432'], [1706118038, '10.627572753016432'], [1706118039, '10.627572753016432'], [1706118040, '10.627572753016432'], [1706118041, '10.627572753016432'], [1706118042, '10.627572753016432'], [1706118043, '10.627572753016432'], [1706118044, '10.627572753016432'], [1706118045, '10.627572753016432'], [1706118046, '8.369048497709652'], [1706118047, '8.369048497709652'], [1706118048, '8.369048497709652'], [1706118049, '8.369048497709652'], [1706118050, '8.369048497709652'], [1706118051, '8.369048497709652'], [1706118052, '8.369048497709652'], [1706118053, '8.369048497709652'], [1706118054, '8.369048497709652'], [1706118055, '8.369048497709652'], [1706118056, '8.369048497709652'], [1706118057, '8.369048497709652'], [1706118058, '8.369048497709652'], [1706118059, '8.369048497709652'], [1706118060, '8.369048497709652'], [1706118061, '8.369048497709652'], [1706118062, '8.369048497709652'], [1706118063, '8.369048497709652'], [1706118064, '8.369048497709652'], [1706118065, '8.369048497709652'], [1706118066, '8.369048497709652'], [1706118067, '8.369048497709652'], [1706118068, '8.369048497709652'], [1706118069, '8.369048497709652'], [1706118070, '8.369048497709652'], [1706118071, '8.369048497709652'], [1706118072, '4.195845830049149'], [1706118073, '4.257017788907441'], [1706118074, '4.318189747765734'], [1706118075, '4.379361706624026'], [1706118076, '4.440533665482319'], [1706118077, '4.169829008752811'], [1706118078, '4.169829008752811'], [1706118079, '4.169829008752811'], [1706118080, '4.169829008752811'], [1706118081, '4.169829008752811'], [1706118082, '4.169829008752811'], [1706118083, '4.169829008752811'], [1706118084, '4.169829008752811'], [1706118085, '4.169829008752811'], [1706118086, '4.169829008752811'], [1706118087, '4.169829008752811'], [1706118088, '4.169829008752811'], [1706118089, '4.169829008752811'], [1706118090, '4.169829008752811'], [1706118091, '4.169829008752811'], [1706118092, '4.169829008752811'], [1706118093, '4.169829008752811'], [1706118094, '4.169829008752811'], [1706118095, '4.169829008752811'], [1706118096, '4.169829008752811'], [1706118097, '4.169829008752811'], [1706118098, '4.169829008752811'], [1706118099, '4.169829008752811'], [1706118100, '4.169829008752811'], [1706118101, '4.169829008752811'], [1706118102, '4.169829008752811'], [1706118103, '4.169829008752811'], [1706118104, '4.169829008752811'], [1706118105, '4.169829008752811'], [1706118106, '4.169829008752811'], [1706118107, '4.169829008752811'], [1706118108, '2.9435952099229175'], [1706118109, '2.9435952099229175'], [1706118110, '2.338359328149322'], [1706118111, '2.3056527147057344'], [1706118112, '2.2729461012621464'], [1706118113, '3.2768392754600044'], [1706118114, '3.2768392754600044'], [1706118115, '3.2768392754600044'], [1706118116, '3.2768392754600044'], [1706118117, '3.2768392754600044'], [1706118118, '3.2768392754600044'], [1706118119, '3.2768392754600044'], [1706118120, '3.2768392754600044'], [1706118121, '3.2768392754600044'], [1706118122, '3.2768392754600044'], [1706118123, '3.2768392754600044'], [1706118124, '3.2768392754600044'], [1706118125, '3.2768392754600044'], [1706118126, '3.2768392754600044'], [1706118127, '3.2768392754600044'], [1706118128, '3.2768392754600044'], [1706118129, '3.2768392754600044'], [1706118130, '3.2768392754600044'], [1706118131, '3.2768392754600044'], [1706118132, '3.2768392754600044'], [1706118133, '3.2768392754600044'], [1706118134, '3.2768392754600044'], [1706118135, '3.2768392754600044'], [1706118136, '3.5614226988089612'], [1706118137, '3.5614226988089612'], [1706118138, '3.5614226988089612'], [1706118139, '3.5614226988089612'], [1706118140, '3.5614226988089612'], [1706118141, '5.393769501363211'], [1706118142, '5.393769501363211'], [1706118143, '5.393769501363211'], [1706118144, '5.393769501363211'], [1706118145, '5.393769501363211'], [1706118146, '5.393769501363211'], [1706118147, '5.393769501363211'], [1706118148, '5.393769501363211'], [1706118149, '5.393769501363211'], [1706118150, '5.393769501363211'], [1706118151, '5.393769501363211'], [1706118152, '5.393769501363211'], [1706118153, '5.393769501363211'], [1706118154, '5.393769501363211'], [1706118155, '5.393769501363211'], [1706118156, '5.393769501363211'], [1706118157, '5.393769501363211'], [1706118158, '5.393769501363211'], [1706118159, '5.393769501363211'], [1706118160, '5.393769501363211'], [1706118161, '5.393769501363211'], [1706118162, '5.393769501363211'], [1706118163, '5.393769501363211'], [1706118164, '5.393769501363211'], [1706118165, '5.393769501363211'], [1706118166, '5.393769501363211'], [1706118167, '5.9313093024429815'], [1706118168, '6.0173370094686165'], [1706118169, '6.103364716494252'], [1706118170, '6.189392423519888'], [1706118171, '7.7582975831536505'], [1706118172, '7.7582975831536505'], [1706118173, '7.7582975831536505'], [1706118174, '7.7582975831536505'], [1706118175, '7.7582975831536505'], [1706118176, '7.7582975831536505'], [1706118177, '7.7582975831536505'], [1706118178, '7.7582975831536505'], [1706118179, '7.7582975831536505'], [1706118180, '7.7582975831536505'], [1706118181, '7.7582975831536505'], [1706118182, '7.7582975831536505'], [1706118183, '7.7582975831536505'], [1706118184, '7.7582975831536505'], [1706118185, '7.7582975831536505'], [1706118186, '7.7582975831536505'], [1706118187, '7.7582975831536505'], [1706118188, '7.7582975831536505'], [1706118189, '7.7582975831536505'], [1706118190, '7.7582975831536505'], [1706118191, '7.7582975831536505'], [1706118192, '7.7582975831536505'], [1706118193, '7.7582975831536505'], [1706118194, '7.7582975831536505'], [1706118195, '7.7582975831536505'], [1706118196, '7.7582975831536505'], [1706118197, '7.7582975831536505'], [1706118198, '7.7582975831536505'], [1706118199, '7.7582975831536505'], [1706118200, '7.7582975831536505'], [1706118201, '7.7582975831536505'], [1706118202, '8.375670614675203'], [1706118203, '8.664708104798436'], [1706118204, '8.664708104798436'], [1706118205, '8.664708104798436'], [1706118206, '8.664708104798436'], [1706118207, '8.664708104798436'], [1706118208, '8.664708104798436'], [1706118209, '8.664708104798436'], [1706118210, '8.664708104798436'], [1706118211, '8.664708104798436'], [1706118212, '8.664708104798436'], [1706118213, '8.664708104798436'], [1706118214, '8.664708104798436'], [1706118215, '8.664708104798436'], [1706118216, '8.664708104798436'], [1706118217, '8.664708104798436'], [1706118218, '8.664708104798436'], [1706118219, '8.664708104798436'], [1706118220, '8.664708104798436'], [1706118221, '9.24542095074573'], [1706118222, '9.24542095074573'], [1706118223, '9.24542095074573'], [1706118224, '9.24542095074573'], [1706118225, '9.24542095074573'], [1706118226, '9.24542095074573'], [1706118227, '9.24542095074573'], [1706118228, '9.24542095074573'], [1706118229, '9.24542095074573'], [1706118230, '9.24542095074573'], [1706118231, '8.274444419487123'], [1706118232, '8.38693809827044'], [1706118233, '10.124431090498456'], [1706118234, '10.124431090498456'], [1706118235, '10.124431090498456'], [1706118236, '10.124431090498456'], [1706118237, '10.124431090498456'], [1706118238, '10.124431090498456'], [1706118239, '10.124431090498456'], [1706118240, '10.124431090498456'], [1706118241, '10.124431090498456'], [1706118242, '10.124431090498456'], [1706118243, '10.124431090498456'], [1706118244, '10.124431090498456'], [1706118245, '10.124431090498456'], [1706118246, '10.124431090498456'], [1706118247, '10.124431090498454'], [1706118248, '8.417648872578287'], [1706118249, '8.30515519379497'], [1706118250, '8.192661515011656'], [1706118251, '8.080167836228338'], [1706118252, '7.967674157445021'], [1706118253, '7.855180478661706'], [1706118254, '7.74268679987839'], [1706118255, '7.630193121095074'], [1706118256, '7.5176994423117565'], [1706118257, '7.405205763528442'], [1706118258, '7.292712084745125'], [1706118259, '7.180218405961807'], [1706118260, '7.067724727178492'], [1706118261, '4.691624081399995'], [1706118262, '4.691624081399995'], [1706118263, '11.46968159166063'], [1706118264, '11.46968159166063'], [1706118265, '11.46968159166063'], [1706118266, '11.46968159166063'], [1706118267, '11.46968159166063'], [1706118268, '11.46968159166063'], [1706118269, '11.46968159166063'], [1706118270, '11.46968159166063'], [1706118271, '11.46968159166063'], [1706118272, '11.46968159166063'], [1706118273, '11.46968159166063'], [1706118274, '11.46968159166063'], [1706118275, '11.46968159166063'], [1706118276, '11.469681591660628'], [1706118277, '11.469681591660628'], [1706118278, '11.48298965769115'], [1706118279, '11.48298965769115'], [1706118280, '11.48298965769115'], [1706118281, '11.48298965769115'], [1706118282, '11.48298965769115'], [1706118283, '11.48298965769115'], [1706118284, '11.48298965769115'], [1706118285, '11.48298965769115'], [1706118286, '11.48298965769115'], [1706118287, '11.48298965769115'], [1706118288, '11.48298965769115'], [1706118289, '11.48298965769115'], [1706118290, '11.48298965769115'], [1706118291, '11.48298965769115'], [1706118292, '11.603810729464888'], [1706118293, '11.603810729464888'], [1706118294, '11.603810729464888'], [1706118295, '11.603810729464888'], [1706118296, '11.603810729464888'], [1706118297, '11.603810729464888'], [1706118298, '11.603810729464888'], [1706118299, '11.603810729464888'], [1706118300, '11.603810729464888'], [1706118301, '11.603810729464888'], [1706118302, '11.603810729464888'], [1706118303, '11.603810729464888'], [1706118304, '11.603810729464888'], [1706118305, '11.603810729464888'], [1706118306, '11.603810729464888'], [1706118307, '11.603810729464888'], [1706118308, '11.603810729464888'], [1706118309, '9.372849185494934'], [1706118310, '9.243917955167548'], [1706118311, '3.949989036377782'], [1706118312, '3.949989036377782'], [1706118313, '3.949989036377782'], [1706118314, '3.949989036377782'], [1706118315, '3.949989036377782'], [1706118316, '3.949989036377782'], [1706118317, '3.949989036377782'], [1706118318, '3.949989036377782'], [1706118319, '3.949989036377782'], [1706118320, '3.949989036377782'], [1706118321, '3.949989036377782'], [1706118322, '11.531323979311384'], [1706118323, '11.531323979311384'], [1706118324, '11.531323979311384'], [1706118325, '11.531323979311384'], [1706118326, '11.531323979311384'], [1706118327, '11.531323979311384'], [1706118328, '11.531323979311384'], [1706118329, '11.531323979311384'], [1706118330, '11.531323979311384'], [1706118331, '11.531323979311384'], [1706118332, '11.531323979311384'], [1706118333, '11.531323979311384'], [1706118334, '11.531323979311384'], [1706118335, '11.526537466242566'], [1706118336, '11.526537466242566'], [1706118337, '11.526537466242566'], [1706118338, '11.526537466242566'], [1706118339, '11.526537466242566'], [1706118340, '11.526537466242566'], [1706118341, '11.526537466242566'], [1706118342, '11.526537466242566'], [1706118343, '11.526537466242566'], [1706118344, '11.526537466242566'], [1706118345, '11.526537466242566'], [1706118346, '11.526537466242566'], [1706118347, '11.526537466242566'], [1706118348, '11.526537466242566'], [1706118349, '11.526537466242566'], [1706118350, '11.526537466242566'], [1706118351, '11.526537466242566'], [1706118352, '11.526537466242566'], [1706118353, '11.524128179592621'], [1706118354, '11.524128179592621'], [1706118355, '11.524128179592621'], [1706118356, '11.524128179592621'], [1706118357, '11.524128179592621'], [1706118358, '11.524128179592621'], [1706118359, '11.524128179592621'], [1706118360, '11.524128179592621'], [1706118361, '11.524128179592621'], [1706118362, '11.524128179592621'], [1706118363, '11.524128179592621'], [1706118364, '11.524128179592621'], [1706118365, '11.524128179592621'], [1706118366, '9.28796714074417'], [1706118367, '9.159921272082029'], [1706118368, '3.249779503666671'], [1706118369, '3.249779503666671'], [1706118370, '3.249779503666671'], [1706118371, '3.249779503666671'], [1706118372, '3.249779503666671'], [1706118373, '3.249779503666671'], [1706118374, '3.249779503666671'], [1706118375, '3.249779503666671'], [1706118376, '3.249779503666671'], [1706118377, '3.249779503666671'], [1706118378, '3.249779503666671'], [1706118379, '3.249779503666671'], [1706118380, '3.249779503666671'], [1706118381, '7.037526331434339'], [1706118382, '7.037526331434339'], [1706118383, '7.037526331434339'], [1706118384, '7.037526331434339'], [1706118385, '7.037526331434339'], [1706118386, '7.037526331434339'], [1706118387, '7.037526331434339'], [1706118388, '7.037526331434339'], [1706118389, '7.037526331434339'], [1706118390, '7.037526331434339'], [1706118391, '7.037526331434339'], [1706118392, '7.037526331434339'], [1706118393, '7.037526331434339'], [1706118394, '7.037526331434339'], [1706118395, '7.037526331434339'], [1706118396, '7.037526331434339'], [1706118397, '7.037526331434339'], [1706118398, '7.037526331434339'], [1706118399, '7.037526331434339'], [1706118400, '7.037526331434339'], [1706118401, '7.037526331434339'], [1706118402, '7.037526331434339'], [1706118403, '7.037526331434339'], [1706118404, '7.037526331434339'], [1706118405, '7.037526331434339'], [1706118406, '5.248563086647389'], [1706118407, '5.248563086647389'], [1706118408, '5.248563086647389'], [1706118409, '5.248563086647389'], [1706118410, '5.248563086647389'], [1706118411, '5.248563086647389'], [1706118412, '4.128116379389054'], [1706118413, '4.128116379389054'], [1706118414, '4.128116379389054'], [1706118415, '4.128116379389054'], [1706118416, '4.128116379389054'], [1706118417, '4.128116379389054'], [1706118418, '4.128116379389054'], [1706118419, '4.128116379389054'], [1706118420, '4.128116379389054']]}]}} +VALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '2785280'], [1706117116, '2785280'], [1706117117, '2785280'], [1706117118, '2785280'], [1706117119, '2785280'], [1706117120, '2785280'], [1706117121, '2785280'], [1706117122, '2785280'], [1706117123, '2785280'], [1706117124, '2785280'], [1706117125, '2785280'], [1706117126, '2785280'], [1706117127, '2785280'], [1706117128, '2785280'], [1706117129, '2785280'], [1706117130, '2785280'], [1706118416, '594620416'], [1706118417, '594620416'], [1706118418, '594620416'], [1706118419, '594620416'], [1706118420, '594620416']]}]}} +VALID_CPU_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'container': 'build', 'cpu': 'total', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117145, '0.2483743618267752'], [1706117146, '0.25650526138466395'], [1706117147, '0.26463616094255266'], [1706117148, '0.2727670605004414'], [1706117149, '0.28089796005833007'], [1706117150, '0.2890288596162188'], [1706117151, '0.2971597591741076'], [1706117357, '3.7319005481816236'], [1706117358, '3.7319005481816236'], [1706117359, '3.7319005481816236'], [1706117360, '3.7319005481816245'], [1706117361, '3.7319005481816245'], [1706118420, '4.128116379389054']]}]}} VALID_NODE_INFO = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_info', 'container': 'kube-state-metrics', 'container_runtime_version': 'containerd://1.7.2', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'internal_ip': '192.168.86.107', 'job': 'kube-state-metrics', 'kernel_version': '5.10.205-195.804.amzn2.x86_64', 'kubelet_version': 'v1.27.9-eks-5e0fdde', 'kubeproxy_version': 'v1.27.9-eks-5e0fdde', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'os_image': 'Amazon Linux 2', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'provider_id': 'aws:///us-east-1c/i-0fe9d9c99fdb3631d', 'service': 'kube-prometheus-stack-kube-state-metrics', 'system_uuid': 'ec253b04-b1dc-f08b-acac-e23df83b3602'}, 'value': [1706117733, '1']}]}} VALID_NODE_LABELS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_labels', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'label_beta_kubernetes_io_arch': 'amd64', 'label_beta_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_beta_kubernetes_io_os': 'linux', 'label_failure_domain_beta_kubernetes_io_region': 'us-east-1', 'label_failure_domain_beta_kubernetes_io_zone': 'us-east-1c', 'label_k8s_io_cloud_provider_aws': 'ceb9f9cc8e47252a6f7fe7d6bded2655', 'label_karpenter_k8s_aws_instance_category': 'i', 'label_karpenter_k8s_aws_instance_cpu': '24', 'label_karpenter_k8s_aws_instance_encryption_in_transit_supported': 'true', 'label_karpenter_k8s_aws_instance_family': 'i3en', 'label_karpenter_k8s_aws_instance_generation': '3', 'label_karpenter_k8s_aws_instance_hypervisor': 'nitro', 'label_karpenter_k8s_aws_instance_local_nvme': '15000', 'label_karpenter_k8s_aws_instance_memory': '196608', 'label_karpenter_k8s_aws_instance_network_bandwidth': '25000', 'label_karpenter_k8s_aws_instance_pods': '234', 'label_karpenter_k8s_aws_instance_size': '6xlarge', 'label_karpenter_sh_capacity_type': 'spot', 'label_karpenter_sh_initialized': 'true', 'label_karpenter_sh_provisioner_name': 'glr-x86-64-v4', 'label_kubernetes_io_arch': 'amd64', 'label_kubernetes_io_hostname': 'ip-192-168-86-107.ec2.internal', 'label_kubernetes_io_os': 'linux', 'label_node_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_spack_io_pipeline': 'true', 'label_spack_io_x86_64': 'v4', 'label_topology_ebs_csi_aws_com_zone': 'us-east-1c', 'label_topology_kubernetes_io_region': 'us-east-1', 'label_topology_kubernetes_io_zone': 'us-east-1c', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'service': 'kube-prometheus-stack-kube-state-metrics'}, 'value': [1706117733, '1']}]}}