From 844fc5c227ccc1874dc1927a06c222fea019546b Mon Sep 17 00:00:00 2001 From: Lucas Bacciotti Date: Mon, 2 Sep 2024 11:54:18 +0100 Subject: [PATCH 1/5] Adding effective price to generator --- nise/__init__.py | 2 +- nise/generators/gcp/compute_engine_generator.py | 2 ++ nise/generators/gcp/gcp_generator.py | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/nise/__init__.py b/nise/__init__.py index 5e6604d0..723f7215 100644 --- a/nise/__init__.py +++ b/nise/__init__.py @@ -1,4 +1,4 @@ -__version__ = "4.6.9" +__version__ = "4.7.0" VERSION = __version__.split(".") diff --git a/nise/generators/gcp/compute_engine_generator.py b/nise/generators/gcp/compute_engine_generator.py index 28202377..1c1b842a 100644 --- a/nise/generators/gcp/compute_engine_generator.py +++ b/nise/generators/gcp/compute_engine_generator.py @@ -89,6 +89,8 @@ def __init__(self, start_date, end_date, currency, project, attributes=None): # self._resource_name = self.attributes.get("resource.name") if self.attributes.get("resource.global_name"): self._resource_global_name = self.attributes.get("resource.global_name") + if self.attributes.get("price.effective_price"): + self._resource_global_name = self.attributes.get("price.effective_price") def _update_data(self, row): # noqa: C901 """Update a data row with compute values.""" diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index bb89c2e0..40f9fd6a 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -56,6 +56,7 @@ "invoice.month", "cost_type", "partition_date", + "price.effective_price", ) GCP_RESOURCE_COLUMNS = ("resource.name", "resource.global_name") From 99527fbf47b31552a8aab0aa78431f6076963019 Mon Sep 17 00:00:00 2001 From: Lucas Bacciotti Date: Mon, 2 Sep 2024 12:16:25 +0100 Subject: [PATCH 2/5] Unit tests for effective_price --- tests/test_gcp_generator.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_gcp_generator.py b/tests/test_gcp_generator.py index 372c6a28..4cd492f0 100644 --- a/tests/test_gcp_generator.py +++ b/tests/test_gcp_generator.py @@ -66,6 +66,7 @@ def setUp(self): "resource.name": "resource-name", "resource.global_name": "global-name", "resource_level": True, + "price.effective_price": 0.04524, } self.now = datetime.now().replace(microsecond=0, second=0, minute=0) self.yesterday = self.now - timedelta(days=1) @@ -126,6 +127,7 @@ def test_compute_engine_init_with_usage_attributes(self): generated_data = generator.generate_data() list_data = list(generated_data) self.assertEqual(list_data[0]["cost"], self.usage_attributes["usage.amount"] * self.usage_attributes["price"]) + self.assertIn("price.effective_price", list_data[0]) def test_compute_engine_init_with_sku_attributes(self): """Test the init with sku attributes for Compute Engine.""" From b88134ae0fb04ba5fcc211c08c11314072ab62c3 Mon Sep 17 00:00:00 2001 From: Lucas Bacciotti Date: Mon, 2 Sep 2024 15:57:44 +0100 Subject: [PATCH 3/5] Removing price.effective_price if. --- nise/generators/gcp/compute_engine_generator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nise/generators/gcp/compute_engine_generator.py b/nise/generators/gcp/compute_engine_generator.py index 1c1b842a..28202377 100644 --- a/nise/generators/gcp/compute_engine_generator.py +++ b/nise/generators/gcp/compute_engine_generator.py @@ -89,8 +89,6 @@ def __init__(self, start_date, end_date, currency, project, attributes=None): # self._resource_name = self.attributes.get("resource.name") if self.attributes.get("resource.global_name"): self._resource_global_name = self.attributes.get("resource.global_name") - if self.attributes.get("price.effective_price"): - self._resource_global_name = self.attributes.get("price.effective_price") def _update_data(self, row): # noqa: C901 """Update a data row with compute values.""" From 264072a5ed481aa133eed61a10c85dd970395d7e Mon Sep 17 00:00:00 2001 From: Lucas Bacciotti Date: Mon, 9 Sep 2024 16:41:56 +0100 Subject: [PATCH 4/5] Adding disk_size. --- nise/generators/gcp/compute_engine_generator.py | 9 +++++++++ nise/generators/gcp/gcp_generator.py | 12 ++++++++++++ nise/upload.py | 7 +++++++ 3 files changed, 28 insertions(+) diff --git a/nise/generators/gcp/compute_engine_generator.py b/nise/generators/gcp/compute_engine_generator.py index 28202377..ab39d8a8 100644 --- a/nise/generators/gcp/compute_engine_generator.py +++ b/nise/generators/gcp/compute_engine_generator.py @@ -58,6 +58,8 @@ class ComputeEngineGenerator(GCPGenerator): LABELS = (([{"key": "vm_key_proj2", "value": "vm_label_proj2"}]), ([])) + EFFECTIVE_PRICE = ("0.0114", "0.0417", "0.1922", "0.0452") + def __init__(self, start_date, end_date, currency, project, attributes=None): # noqa: C901 """Initialize the cloud storage generator.""" super().__init__(start_date, end_date, currency, project, attributes) @@ -123,6 +125,9 @@ def _update_data(self, row): # noqa: C901 row["currency"] = self._currency row["labels"] = self.determine_labels(self.LABELS) row["system_labels"] = self.determine_system_labels(sku[3]) + row["price.effective_price"] = choice(self.EFFECTIVE_PRICE) + row["disk_size"] = self._generate_disk_size(row["cost"], row["price.effective_price"]) + if self.resource_level: resource = self._generate_resource( self._resource_name, self._resource_global_name, self.project.get("region") @@ -183,6 +188,10 @@ def _update_data(self, row): # noqa: C901 month = datetime.strptime(row.get("usage_start_time")[:7], "%Y-%m").month invoice["month"] = f"{year}{month:02d}" row["invoice"] = invoice + effective_price = choice(self.EFFECTIVE_PRICE) + row["price"] = {"effective_price": effective_price} + row["disk_size"] = self._generate_disk_size(row["cost"], row["price"]["effective_price"]) + if self.resource_level: resource = self._generate_resource() row["resource"] = resource diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index 40f9fd6a..ce5b9047 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -57,6 +57,7 @@ "cost_type", "partition_date", "price.effective_price", + "disk_size", ) GCP_RESOURCE_COLUMNS = ("resource.name", "resource.global_name") @@ -320,3 +321,14 @@ def _generate_hourly_data(self, **kwargs): row = self._init_data_row(start, end) row = self._update_data(row) yield row + + def _generate_disk_size(self, cost, effective_price): + """Generate disk size.""" + hours_in_month = 744 + disk_size = 0 + + if cost and effective_price: + rate = float(effective_price) / hours_in_month + disk_size = float(cost) / rate + + return disk_size diff --git a/nise/upload.py b/nise/upload.py index c1f54f21..746cf183 100644 --- a/nise/upload.py +++ b/nise/upload.py @@ -269,6 +269,13 @@ def gcp_bucket_to_dataset(gcp_bucket_name, file_name, dataset_name, table_name, "mode": "NULLABLE", }, {"name": "cost_type", "type": "STRING", "mode": "NULLABLE"}, + { + "name": "price", + "type": "RECORD", + "fields": [{"name": "effective_price", "type": "STRING", "mode": "NULLABLE"}], + "mode": "NULLABLE", + }, + {"name": "disk_size", "type": "STRING", "mode": "NULLABLE"}, { "name": "adjustment_info", "type": "RECORD", From f9285f06f2f5ded14b33c03dc3a0d6dc44539b0b Mon Sep 17 00:00:00 2001 From: Lucas Bacciotti Date: Wed, 11 Sep 2024 10:09:18 +0100 Subject: [PATCH 5/5] Removing. disk_size. --- nise/generators/gcp/compute_engine_generator.py | 2 -- nise/generators/gcp/gcp_generator.py | 12 ------------ nise/upload.py | 1 - 3 files changed, 15 deletions(-) diff --git a/nise/generators/gcp/compute_engine_generator.py b/nise/generators/gcp/compute_engine_generator.py index ab39d8a8..76164c81 100644 --- a/nise/generators/gcp/compute_engine_generator.py +++ b/nise/generators/gcp/compute_engine_generator.py @@ -126,7 +126,6 @@ def _update_data(self, row): # noqa: C901 row["labels"] = self.determine_labels(self.LABELS) row["system_labels"] = self.determine_system_labels(sku[3]) row["price.effective_price"] = choice(self.EFFECTIVE_PRICE) - row["disk_size"] = self._generate_disk_size(row["cost"], row["price.effective_price"]) if self.resource_level: resource = self._generate_resource( @@ -190,7 +189,6 @@ def _update_data(self, row): # noqa: C901 row["invoice"] = invoice effective_price = choice(self.EFFECTIVE_PRICE) row["price"] = {"effective_price": effective_price} - row["disk_size"] = self._generate_disk_size(row["cost"], row["price"]["effective_price"]) if self.resource_level: resource = self._generate_resource() diff --git a/nise/generators/gcp/gcp_generator.py b/nise/generators/gcp/gcp_generator.py index ce5b9047..40f9fd6a 100644 --- a/nise/generators/gcp/gcp_generator.py +++ b/nise/generators/gcp/gcp_generator.py @@ -57,7 +57,6 @@ "cost_type", "partition_date", "price.effective_price", - "disk_size", ) GCP_RESOURCE_COLUMNS = ("resource.name", "resource.global_name") @@ -321,14 +320,3 @@ def _generate_hourly_data(self, **kwargs): row = self._init_data_row(start, end) row = self._update_data(row) yield row - - def _generate_disk_size(self, cost, effective_price): - """Generate disk size.""" - hours_in_month = 744 - disk_size = 0 - - if cost and effective_price: - rate = float(effective_price) / hours_in_month - disk_size = float(cost) / rate - - return disk_size diff --git a/nise/upload.py b/nise/upload.py index 746cf183..74a14943 100644 --- a/nise/upload.py +++ b/nise/upload.py @@ -275,7 +275,6 @@ def gcp_bucket_to_dataset(gcp_bucket_name, file_name, dataset_name, table_name, "fields": [{"name": "effective_price", "type": "STRING", "mode": "NULLABLE"}], "mode": "NULLABLE", }, - {"name": "disk_size", "type": "STRING", "mode": "NULLABLE"}, { "name": "adjustment_info", "type": "RECORD",