diff --git a/gantry/__main__.py b/gantry/__main__.py index 68acc60..ea7b1e2 100644 --- a/gantry/__main__.py +++ b/gantry/__main__.py @@ -28,6 +28,7 @@ async def apply_migrations(db: aiosqlite.Connection): # and not inadvertently added to the migrations folder ("001_initial.sql", 1), ("002_spec_index.sql", 2), + ("003_job_cost.sql", 3), ] # apply migrations that have not been applied @@ -45,6 +46,8 @@ async def apply_migrations(db: aiosqlite.Connection): async def init_db(app: web.Application): db = await aiosqlite.connect(os.environ["DB_FILE"]) await apply_migrations(db) + # ensure foreign key constraints are enabled + await db.execute("PRAGMA foreign_keys = ON") app["db"] = db yield await db.close() diff --git a/gantry/clients/prometheus/job.py b/gantry/clients/prometheus/job.py index 9f9d7ed..2e4ce3c 100644 --- a/gantry/clients/prometheus/job.py +++ b/gantry/clients/prometheus/job.py @@ -1,5 +1,7 @@ import json +import aiosqlite + from gantry.clients.prometheus import util from gantry.util.spec import spec_variants @@ -152,3 +154,68 @@ async def get_usage(self, pod: str, start: float, end: float) -> dict: "mem_min": mem_usage["min"], "mem_stddev": mem_usage["stddev"], } + + async def get_costs( + self, + db: aiosqlite.Connection, + start: float, + end: float, + node_id: int, + ) -> dict: + """ + Calculates the costs associated with a job. + + args: + db: a database connection + resources: job requests and limits + usage: job memory and cpu usage + start: job start time + end: job end time + node_id: the node that the job ran on + + returns: + dict of: job_cost_instance (cost of the instance over the job's lifetime) + """ + costs = {} + async with db.execute( + """ + select capacity_type, instance_type, zone + from nodes where id = ? + """, + (node_id,), + ) as cursor: + node = await cursor.fetchone() + + if not node: + # this is a temporary condition that will happen during the transition + # to collecting + raise util.IncompleteData( + f"node instance metadata is missing from db. node={node_id}" + ) + + capacity_type, instance_type, zone = node + + # spot instance prices can change, so we avg the cost over the job's runtime + instance_costs = await self.client.query_range( + query={ + "metric": "karpenter_cloudprovider_instance_type_offering_price_estimate", # noqa: E501 + "filters": { + "capacity_type": capacity_type, + "instance_type": instance_type, + "zone": zone, + }, + }, + start=start, + end=end, + ) + + if not instance_costs: + raise util.IncompleteData(f"node cost is missing. node={node_id}") + + instance_costs = [float(value) for _, value in instance_costs[0]["values"]] + # average hourly cost of the instance over the job's lifetime + instance_cost = sum(instance_costs) / len(instance_costs) + # compute cost: hourly instance cost * job runtime in hours + node_cost = instance_cost * ((end - start) / 3600) # seconds to hours + costs["job_cost_instance"] = node_cost + return costs diff --git a/gantry/clients/prometheus/node.py b/gantry/clients/prometheus/node.py index abfb217..4df7260 100644 --- a/gantry/clients/prometheus/node.py +++ b/gantry/clients/prometheus/node.py @@ -53,4 +53,6 @@ async def get_labels(self, hostname: str, time: float) -> dict: "arch": labels["label_kubernetes_io_arch"], "os": labels["label_kubernetes_io_os"], "instance_type": labels["label_node_kubernetes_io_instance_type"], + "capacity_type": labels["label_karpenter_sh_capacity_type"], + "zone": labels["label_topology_kubernetes_io_zone"], } diff --git a/gantry/routes/collection.py b/gantry/routes/collection.py index 7908672..51ef128 100644 --- a/gantry/routes/collection.py +++ b/gantry/routes/collection.py @@ -73,6 +73,8 @@ async def fetch_job( ) usage = await prometheus.job.get_usage(annotations["pod"], job.start, job.end) node_id = await fetch_node(db_conn, prometheus, node_hostname, job.midpoint) + costs = await prometheus.job.get_costs(db_conn, job.start, job.end, node_id) + except aiohttp.ClientError as e: logger.error(f"Request failed: {e}") return @@ -93,6 +95,7 @@ async def fetch_job( **annotations, **resources, **usage, + **costs, }, ) @@ -139,5 +142,7 @@ async def fetch_node( "arch": node_labels["arch"], "os": node_labels["os"], "instance_type": node_labels["instance_type"], + "capacity_type": node_labels["capacity_type"], + "zone": node_labels["zone"], }, ) diff --git a/gantry/tests/defs/collection.py b/gantry/tests/defs/collection.py index d419d79..76bd9d6 100644 --- a/gantry/tests/defs/collection.py +++ b/gantry/tests/defs/collection.py @@ -20,8 +20,8 @@ # used to compare successful insertions # run SELECT * FROM table_name WHERE id = 1; from python sqlite api and grab fetchone() result -INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux-ubuntu20.04-x86_64_v3', 'e4s', 16, 0.75, None, 1.899768349523097, 0.2971597591741076, 4.128116379389054, 0.2483743618267752, 1.7602635378120381, 2000000000.0, 48000000000.0, 143698407.6190476, 2785280.0, 594620416.0, 2785280.0, 252073065.82263485) -INSERTED_NODE = (1, 'ec253b04-b1dc-f08b-acac-e23df83b3602', 'ip-192-168-86-107.ec2.internal', 24.0, 196608000000.0, 'amd64', 'linux', 'i3en.6xlarge') +INSERTED_JOB = (1, 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 1, 1706117046, 1706118420, 9892514, 'success', 'pr42264_bugfix/mathomp4/hdf5-appleclang15', 'gmsh', '4.8.4', '{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}', 'gcc', '11.4.0', 'linux-ubuntu20.04-x86_64_v3', 'e4s', 16, 0.75, None, 1.899768349523097, 0.2971597591741076, 4.128116379389054, 0.2483743618267752, 1.7602635378120381, 2000000000.0, 48000000000.0, 143698407.6190476, 2785280.0, 594620416.0, 2785280.0, 252073065.82263485, 0.19083333333333333) +INSERTED_NODE = (1, 'ec253b04-b1dc-f08b-acac-e23df83b3602', 'ip-192-168-86-107.ec2.internal', 24.0, 196608000000.0, 'amd64', 'linux', 'i3en.6xlarge', 'us-east-1c', 'spot') # these were obtained by executing the respective queries to Prometheus and capturing the JSON output # or the raw output of PrometheusClient._query @@ -32,6 +32,7 @@ VALID_CPU_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'container': 'build', 'cpu': 'total', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117145, '0.2483743618267752'], [1706117146, '0.25650526138466395'], [1706117147, '0.26463616094255266'], [1706117148, '0.2727670605004414'], [1706117149, '0.28089796005833007'], [1706117150, '0.2890288596162188'], [1706117151, '0.2971597591741076'], [1706117357, '3.7319005481816236'], [1706117358, '3.7319005481816236'], [1706117359, '3.7319005481816236'], [1706117360, '3.7319005481816245'], [1706117361, '3.7319005481816245'], [1706118420, '4.128116379389054']]}]}} VALID_NODE_INFO = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_info', 'container': 'kube-state-metrics', 'container_runtime_version': 'containerd://1.7.2', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'internal_ip': '192.168.86.107', 'job': 'kube-state-metrics', 'kernel_version': '5.10.205-195.804.amzn2.x86_64', 'kubelet_version': 'v1.27.9-eks-5e0fdde', 'kubeproxy_version': 'v1.27.9-eks-5e0fdde', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'os_image': 'Amazon Linux 2', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'provider_id': 'aws:///us-east-1c/i-0fe9d9c99fdb3631d', 'service': 'kube-prometheus-stack-kube-state-metrics', 'system_uuid': 'ec253b04-b1dc-f08b-acac-e23df83b3602'}, 'value': [1706117733, '1']}]}} VALID_NODE_LABELS = {'status': 'success', 'data': {'resultType': 'vector', 'result': [{'metric': {'__name__': 'kube_node_labels', 'container': 'kube-state-metrics', 'endpoint': 'http', 'instance': '192.168.164.84:8080', 'job': 'kube-state-metrics', 'label_beta_kubernetes_io_arch': 'amd64', 'label_beta_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_beta_kubernetes_io_os': 'linux', 'label_failure_domain_beta_kubernetes_io_region': 'us-east-1', 'label_failure_domain_beta_kubernetes_io_zone': 'us-east-1c', 'label_k8s_io_cloud_provider_aws': 'ceb9f9cc8e47252a6f7fe7d6bded2655', 'label_karpenter_k8s_aws_instance_category': 'i', 'label_karpenter_k8s_aws_instance_cpu': '24', 'label_karpenter_k8s_aws_instance_encryption_in_transit_supported': 'true', 'label_karpenter_k8s_aws_instance_family': 'i3en', 'label_karpenter_k8s_aws_instance_generation': '3', 'label_karpenter_k8s_aws_instance_hypervisor': 'nitro', 'label_karpenter_k8s_aws_instance_local_nvme': '15000', 'label_karpenter_k8s_aws_instance_memory': '196608', 'label_karpenter_k8s_aws_instance_network_bandwidth': '25000', 'label_karpenter_k8s_aws_instance_pods': '234', 'label_karpenter_k8s_aws_instance_size': '6xlarge', 'label_karpenter_sh_capacity_type': 'spot', 'label_karpenter_sh_initialized': 'true', 'label_karpenter_sh_provisioner_name': 'glr-x86-64-v4', 'label_kubernetes_io_arch': 'amd64', 'label_kubernetes_io_hostname': 'ip-192-168-86-107.ec2.internal', 'label_kubernetes_io_os': 'linux', 'label_node_kubernetes_io_instance_type': 'i3en.6xlarge', 'label_spack_io_pipeline': 'true', 'label_spack_io_x86_64': 'v4', 'label_topology_ebs_csi_aws_com_zone': 'us-east-1c', 'label_topology_kubernetes_io_region': 'us-east-1', 'label_topology_kubernetes_io_zone': 'us-east-1c', 'namespace': 'monitoring', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'kube-prometheus-stack-kube-state-metrics-dbd66d8c7-6ftw8', 'service': 'kube-prometheus-stack-kube-state-metrics'}, 'value': [1706117733, '1']}]}} +VALID_NODE_COST = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'karpenter_cloudprovider_instance_type_offering_price_estimate', 'capacity_type': 'spot', 'container': 'controller', 'endpoint': 'http-metrics', 'instance': '192.168.240.113:8000', 'instance_type': 'i3en.6xlarge', 'job': 'karpenter', 'namespace': 'karpenter', 'pod': 'karpenter-8488f7f6dc-ml7q8', 'region': 'us-east-1', 'service': 'karpenter', 'zone': 'us-east-1c'}, 'values': [[1723838829, '0.5']]}]}} # modified version of VALID_MEMORY_USAGE to make the mean/stddev 0 INVALID_MEMORY_USAGE = {'status': 'success', 'data': {'resultType': 'matrix', 'result': [{'metric': {'__name__': 'container_memory_working_set_bytes', 'container': 'build', 'endpoint': 'https-metrics', 'id': '/kubepods.slice/kubepods-burstable.slice/kubepods-burstable-podd7aa13e0_998c_4f21_b1d6_62781f4980b0.slice/cri-containerd-48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1.scope', 'image': 'ghcr.io/spack/ubuntu20.04-runner-amd64-gcc-11.4:2023.08.01', 'instance': '192.168.86.107:10250', 'job': 'kubelet', 'metrics_path': '/metrics/cadvisor', 'name': '48a5e9e7d46655e73ba119fa16b65fa94ceed23c55157db8269b0b12f18f55d1', 'namespace': 'pipeline', 'node': 'ip-192-168-86-107.ec2.internal', 'pod': 'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z', 'service': 'kube-prometheus-stack-kubelet'}, 'values': [[1706117115, '0']]}]}} diff --git a/gantry/tests/defs/db.py b/gantry/tests/defs/db.py index d4e5290..6842e36 100644 --- a/gantry/tests/defs/db.py +++ b/gantry/tests/defs/db.py @@ -2,4 +2,4 @@ # fmt: off # valid input into insert_node -NODE_INSERT_DICT = {"uuid": "ec253b04-b1dc-f08b-acac-e23df83b3602", "hostname": "ip-192-168-86-107.ec2.internal", "cores": 24.0, "mem": 196608000000.0, "arch": "amd64", "os": "linux", "instance_type": "i3en.6xlarge"} +NODE_INSERT_DICT = {"uuid": "ec253b04-b1dc-f08b-acac-e23df83b3602", "hostname": "ip-192-168-86-107.ec2.internal", "cores": 24.0, "mem": 196608000000.0, "arch": "amd64", "os": "linux", "instance_type": "i3en.6xlarge", "zone": "us-east-1c", "capacity_type": "spot"} diff --git a/gantry/tests/sql/insert_job.sql b/gantry/tests/sql/insert_job.sql index 3008da6..39b18b5 100644 --- a/gantry/tests/sql/insert_job.sql +++ b/gantry/tests/sql/insert_job.sql @@ -1 +1 @@ -INSERT INTO jobs VALUES(1,'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z',2,1706117046,1706118420,9892514,'success','pr42264_bugfix/mathomp4/hdf5-appleclang15','gmsh','4.8.4','{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',16,0.75,NULL,4.12532286694540495,3.15805864677520409,11.6038107294648877,0.248374361826775191,3.34888880339475214,2000000000.0,48000000000.0,1649868862.72588062,999763968.0,5679742976.0,2785280.0,1378705563.21018671); \ No newline at end of file +INSERT INTO jobs VALUES(1,'runner-hwwb-i3u-project-2-concurrent-1-s10tq41z',2,1706117046,1706118420,9892514,'success','pr42264_bugfix/mathomp4/hdf5-appleclang15','gmsh','4.8.4','{"alglib": true, "cairo": false, "cgns": true, "compression": true, "eigen": false, "external": false, "fltk": true, "gmp": true, "hdf5": false, "ipo": false, "med": true, "metis": true, "mmg": true, "mpi": true, "netgen": true, "oce": true, "opencascade": false, "openmp": false, "petsc": false, "privateapi": false, "shared": true, "slepc": false, "tetgen": true, "voropp": true, "build_system": "cmake", "build_type": "Release", "generator": "make"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',16,0.75,NULL,4.12532286694540495,3.15805864677520409,11.6038107294648877,0.248374361826775191,3.34888880339475214,2000000000.0,48000000000.0,1649868862.72588062,999763968.0,5679742976.0,2785280.0,1378705563.21018671,0.19083333333333333); \ No newline at end of file diff --git a/gantry/tests/sql/insert_node.sql b/gantry/tests/sql/insert_node.sql index cad50ee..784ae17 100644 --- a/gantry/tests/sql/insert_node.sql +++ b/gantry/tests/sql/insert_node.sql @@ -1,2 +1,2 @@ --- primary key is set to 2 to set up the test that checks for race conditions -INSERT INTO nodes VALUES(2,'ec253b04-b1dc-f08b-acac-e23df83b3602','ip-192-168-86-107.ec2.internal',24.0,196608000000.0,'amd64','linux','i3en.6xlarge'); \ No newline at end of file +INSERT INTO nodes VALUES(2,'ec253b04-b1dc-f08b-acac-e23df83b3602','ip-192-168-86-107.ec2.internal',24.0,196608000000.0,'amd64','linux','i3en.6xlarge','us-east-1c','spot'); \ No newline at end of file diff --git a/gantry/tests/sql/insert_samples.sql b/gantry/tests/sql/insert_samples.sql index d017ebe..059a787 100644 --- a/gantry/tests/sql/insert_samples.sql +++ b/gantry/tests/sql/insert_samples.sql @@ -1,6 +1,6 @@ -INSERT INTO nodes VALUES(6789,'ec2c47a0-7e9b-cfa3-9ad4-ac227ade598d','ip-192-168-202-150.ec2.internal',32.0,131072000000.0,'amd64','linux','m5.8xlarge'); -INSERT INTO jobs VALUES(6781,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi1',6789,1708919572.983000041,1708924744.811000108,101502092,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,9.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9652098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419); -INSERT INTO jobs VALUES(6782,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi2',6789,1708919572.983000041,1708924744.811000108,101502093,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,10.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9958098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419); -INSERT INTO jobs VALUES(6783,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi3',6789,1708919572.983000041,1708924744.811000108,101502094,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,11.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9158098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419); -INSERT INTO jobs VALUES(6784,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi4',6789,1708919572.983000041,1708924744.811000108,101502095,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,12.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9758098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419); -INSERT INTO jobs VALUES(6785,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi5',6789,1708919572.983000041,1708924744.811000108,101502096,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,13.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9358098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419); +INSERT INTO nodes VALUES(6789,'ec2c47a0-7e9b-cfa3-9ad4-ac227ade598d','ip-192-168-202-150.ec2.internal',32.0,131072000000.0,'amd64','linux','m5.8xlarge','us-east-1c','spot'); +INSERT INTO jobs VALUES(6781,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi1',6789,1708919572.983000041,1708924744.811000108,101502092,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,9.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9652098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0.19083333333333333); +INSERT INTO jobs VALUES(6782,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi2',6789,1708919572.983000041,1708924744.811000108,101502093,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,10.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9958098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0.19083333333333333); +INSERT INTO jobs VALUES(6783,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi3',6789,1708919572.983000041,1708924744.811000108,101502094,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,11.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9158098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0.19083333333333333); +INSERT INTO jobs VALUES(6784,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi4',6789,1708919572.983000041,1708924744.811000108,101502095,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,12.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9758098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0.19083333333333333); +INSERT INTO jobs VALUES(6785,'runner-2j2ndhxu-project-2-concurrent-0-nbogpypi5',6789,1708919572.983000041,1708924744.811000108,101502096,'success','develop','py-torch','2.2.1','{"caffe2": false, "cuda": true, "cudnn": true, "debug": false, "distributed": true, "fbgemm": true, "gloo": true, "kineto": true, "magma": false, "metal": false, "mkldnn": true, "mpi": true, "nccl": false, "nnpack": true, "numa": true, "numpy": true, "onnx_ml": true, "openmp": true, "qnnpack": true, "rocm": false, "tensorpipe": true, "test": false, "valgrind": true, "xnnpack": true, "build_system": "python_pip", "cuda_arch": "80"}','gcc','11.4.0','linux-ubuntu20.04-x86_64_v3','e4s',12,12.0,NULL,13.77948152336477605,11.98751586519425772,12.00060520666194109,0.3736576704015182604,3.811106184376615414,48000000000.0,64000000000.0,9358098890.24199867,7399608320.0,41186873344.0,85508096.0,8707419891.779100419, 0.19083333333333333); diff --git a/gantry/tests/test_collection.py b/gantry/tests/test_collection.py index 926a161..c274f6b 100644 --- a/gantry/tests/test_collection.py +++ b/gantry/tests/test_collection.py @@ -19,6 +19,7 @@ "job_cpu_usage": defs.VALID_CPU_USAGE, "node_info": defs.VALID_NODE_INFO, "node_labels": defs.VALID_NODE_LABELS, + "node_cost": defs.VALID_NODE_COST, } diff --git a/migrations/003_job_cost.sql b/migrations/003_job_cost.sql new file mode 100644 index 0000000..0b70637 --- /dev/null +++ b/migrations/003_job_cost.sql @@ -0,0 +1,76 @@ +-- temporarily disable foreign key constraints +PRAGMA foreign_keys = OFF; + +-- this approach is needed because we want to add new columns +-- with a not null constraint, but also to add default values +-- this isn't directly supported by ALTER TABLE + +-- create tmp table for nodes +CREATE TABLE nodes_tmp ( + id INTEGER PRIMARY KEY, + uuid TEXT NOT NULL UNIQUE, + hostname TEXT NOT NULL, + cores REAL NOT NULL, + mem REAL NOT NULL, + arch TEXT NOT NULL, + os TEXT NOT NULL, + instance_type TEXT NOT NULL, + -- new columns below + zone TEXT NOT NULL, + capacity_type TEXT NOT NULL +); + +-- copy data from nodes to nodes_tmp +-- '', and '' are the default values for zone, and capacity_type +-- all columns need to be individually selected because we are adding new blank columns +INSERT INTO nodes_tmp SELECT id, uuid, hostname, cores, mem, arch, os, instance_type, '', '' FROM nodes; +DROP TABLE nodes; +ALTER TABLE nodes_tmp RENAME TO nodes; + +---- create tmp table for jobs + +CREATE TABLE IF NOT EXISTS jobs_tmp ( + id INTEGER PRIMARY KEY, + pod TEXT NOT NULL UNIQUE, + node INTEGER NOT NULL, + start INTEGER NOT NULL, + end INTEGER NOT NULL, + gitlab_id INTEGER NOT NULL UNIQUE, + job_status TEXT NOT NULL, + ref TEXT NOT NULL, + pkg_name TEXT NOT NULL, + pkg_version TEXT NOT NULL, + pkg_variants TEXT NOT NULL, + compiler_name TEXT NOT NULL, + compiler_version TEXT NOT NULL, + arch TEXT NOT NULL, + stack TEXT NOT NULL, + build_jobs INTEGER NOT NULL, + cpu_request REAL NOT NULL, + cpu_limit REAL, -- this can be null because it's currently not set + cpu_mean REAL NOT NULL, + cpu_median REAL NOT NULL, + cpu_max REAL NOT NULL, + cpu_min REAL NOT NULL, + cpu_stddev REAL NOT NULL, + mem_request REAL NOT NULL, + mem_limit REAL NOT NULL, + mem_mean REAL NOT NULL, + mem_median REAL NOT NULL, + mem_max REAL NOT NULL, + mem_min REAL NOT NULL, + mem_stddev REAL NOT NULL, + job_cost_instance REAL NOT NULL, + FOREIGN KEY (node) + REFERENCES nodes (id) + ON UPDATE CASCADE + ON DELETE CASCADE +); + +-- copy data from jobs to jobs_tmp +-- all old columns will have cost values set to 0 +INSERT INTO jobs_tmp select id, pod, node, start, end, gitlab_id, job_status, ref, pkg_name, pkg_version, pkg_variants, compiler_name, compiler_version, arch, stack, build_jobs, cpu_request, cpu_limit, cpu_mean, cpu_median, cpu_max, cpu_min, cpu_stddev, mem_request, mem_limit, mem_mean, mem_median, mem_max, mem_min, mem_stddev, 0 FROM jobs; +DROP TABLE jobs; +ALTER TABLE jobs_tmp RENAME TO jobs; + +PRAGMA foreign_keys = ON;