tweaks to update testing

Signed-off-by: vsoch <[email protected]>

cloudselect/cloud/google/instance.py

@@ -126,7 +126,21 @@ def add_instance_prices(self, prices):
         if not prices.data:
             return
 
-        # I'm writing this out stupidly / in detail so the logic is clear.
+        print(
+            "⚠️ WARNING: prices are experimental, and often slightly (cents) low (compared to the web UI)"
+        )
+
+        # Note from here https://cloud.google.com/compute/disks-image-pricing
+        # standard provisioned space (for us regions) is 0.04 / gb per month
+        # the default machine comes with 20, so let's assume that
+        # 4 cents per GB per month x 20 / 730 hours per month
+        # Trivial, but let's add it
+        disk_cost = 0.04 * 20 / 730
+
+        # Keep a subset of data for the base network
+        # Not sure if total network vm bandwidth is considered in the web UI
+        # network = [x for x in prices.data if "bandwidth" in x['description']]
+
         # Filter prices down to compute (and then separate on demand from spot) for each of CPU and memory
         data = [x for x in prices.data if x["category"]["resourceFamily"] == "Compute"]
 
@@ -193,20 +207,16 @@ def add_instance_prices(self, prices):
             # This would be the price for the instance type per hour
             # Just try for both
             try:
-                spot_price = (
-                    actual_cpu * preemptible_lookup[prefix]["cpu"][region]
-                    + actual_mem_gb * preemptible_lookup[prefix]["mem"][region]
-                )
-                item["spot_price"] = spot_price
+                cpu_price = actual_cpu * preemptible_lookup[prefix]["cpu"][region]
+                mem_price = actual_mem_gb * preemptible_lookup[prefix]["mem"][region]
+                item["spot_price"] = cpu_price + mem_price + disk_cost
             except Exception:
                 pass
 
             try:
-                demand_price = (
-                    actual_cpu * on_demand_lookup[prefix]["cpu"][region]
-                    + actual_mem_gb * on_demand_lookup[prefix]["mem"][region]
-                )
-                item["price"] = demand_price
+                cpu_price = actual_cpu * on_demand_lookup[prefix]["cpu"][region]
+                mem_price = actual_mem_gb * on_demand_lookup[prefix]["mem"][region]
+                item["price"] = cpu_price + mem_price + disk_cost
             except Exception:
                 pass
 
@@ -240,9 +250,11 @@ def add_instance(instance_name, item):
         for region in item["serviceRegions"]:
             # See https://cloud.google.com/recommender/docs/reference/rest/Shared.Types/Money
             # We divide by 10^9 to convert nanos to USD/hour (per unit like cpu)
-            nanos = item["pricingInfo"][0]["pricingExpression"]["tieredRates"][0][
-                "unitPrice"
-            ]["nanos"]
+            info = item["pricingInfo"]
+            assert len(info) == 1
+            rates = info[0]["pricingExpression"]["tieredRates"]
+            assert len(rates) == 1
+            nanos = rates[0]["unitPrice"]["nanos"]
 
             # This is USD / unit (cpu or mem) / hour
             lookup[instance_name][key][region] = nanos / math.pow(10, 9)

cloudselect/main/client.py

@@ -111,6 +111,11 @@ def load_cache(self, key):
         returns data we have available.
         """
         items = {}
+
+        # If directed not to use a cache, return early
+        if not self.use_cache:
+            return items
+
         for cloud in self.get_clouds():
             # Assume we don't find data
             data = None

cloudselect/main/schemas.py

@@ -92,18 +92,30 @@
         "description": "Maximum number of vcpus available to the instance type. If min not set, is 0.",
     },
     # It seems unlikely to find an exact price, but might as well be consistent!
-    "price-per-hour": {
+    "price": {
         "type": "number",
         "description": "Price/hour in dollars (e.g., 0.09) (sets min and max to the same value)",
     },
-    "price-per-hour-min": {
+    "price-min": {
         "type": "number",
         "description": "Minimum price/hour in dollars. If max not set, is infinity.",
     },
-    "price-per-hour-max": {
+    "price-max": {
         "type": "number",
         "description": "Maximum price/hour in dollars. If min not set, is 0.",
     },
+    "spot-price": {
+        "type": "number",
+        "description": "Price/hour for spot in dollars (e.g., 0.09) (sets min and max to the same value)",
+    },
+    "spot-price-min": {
+        "type": "number",
+        "description": "Minimum spot price/hour in dollars. If max not set, is infinity.",
+    },
+    "spot-price-max": {
+        "type": "number",
+        "description": "Maximum spot price/hour in dollars. If min not set, is 0.",
+    },
     "free-tier": {"type": "boolean", "description": "Free tier only."},
     "ipv6": {"type": "boolean", "description": "Instance Types that support IPv6"},
     "cpu-arch": {
@@ -138,24 +150,6 @@
     },
 }
 
-# These are not added yet (and available for AWS) if we want to add them
-#      --network-encryption                             Instance Types that support automatic network encryption in-transit
-#      --network-interfaces int                         Number of network interfaces (ENIs) that can be attached to the instance (sets --network-interfaces-min and -max to the same value)
-#      --network-interfaces-max int                     Maximum Number of network interfaces (ENIs) that can be attached to the instance If --network-interfaces-min is not specified, the lower bound will be 0
-#      --network-interfaces-min int                     Minimum Number of network interfaces (ENIs) that can be attached to the instance If --network-interfaces-max is not specified, the upper bound will be infinity
-#      --network-performance int                        Bandwidth in Gib/s of network performance (Example: 100) (sets --network-performance-min and -max to the same value)
-#      --network-performance-max int                    Maximum Bandwidth in Gib/s of network performance (Example: 100) If --network-performance-min is not specified, the lower bound will be 0
-#      --network-performance-min int                    Minimum Bandwidth in Gib/s of network performance (Example: 100) If --network-performance-max is not specified, the upper bound will be infinity
-#      --nvme                                           EBS or local instance storage where NVME is supported or required
-#      --placement-group-strategy string                Placement group strategy: [cluster, partition, spread]
-#  -u, --usage-class string                             Usage class: [spot or on-demand]
-#      --virtualization-type string                     Virtualization Type supported: [hvm or pv]
-#  -o, --output string           Specify the output format (table, table-wide, one-line, interactive)
-#      --profile string          AWS CLI profile to use for credentials and config
-#  -r, --region string           AWS Region to use for API requests (NOTE: if not passed in, uses AWS SDK default precedence)
-#      --sort-by string          Specify the field to sort by. Quantity flags present in this CLI (memory, gpus, etc.) or a JSON path to the appropriate instance type field (Ex: ".MemoryInfo.SizeInMiB") is acceptable. (default ".InstanceType")
-#      --sort-direction string   Specify the direction to sort in (ascending, asc, descending, desc) (default "ascending")
-
 ## Settings.yml (loads as json)
 
 cloud_properties = {"regions": {"type": "array", "items": {"type": "string"}}}

cloudselect/main/solve/properties.py

@@ -17,7 +17,8 @@ class Properties:
         "instance_storage",
         "memory",
         "gpu_memory",
-        "price_per_hour",
+        "spot_price",
+        "price",
     ]
 
     def __init__(self, properties, **kwargs):

cloudselect/tests/test_instance.py

@@ -157,7 +157,6 @@ def reset_data():
         "gpu-model",
         "hypervisor",
         "instance-storage",
-        "price-per-hour",
         "cpu-arch",
         "cpu-vendor",
         "gpu-vendor",
@@ -236,19 +235,19 @@ def reset_data():
             assert attr in result
 
     # Note that --max-results is handled by the client directly
-    args = parse_args("instance --price-per-hour-min 1.0")
+    args = parse_args("instance --price-min 1.0")
     results = client.instance_select(**args.__dict__)
     assert len(results) == 9
     for result in results:
         assert result["price"] >= 1.0
 
-    args = parse_args("instance --price-per-hour-max 2.0")
+    args = parse_args("instance --price-max 2.0")
     results = client.instance_select(**args.__dict__)
     assert len(results) == 13
     for result in results:
         assert result["price"] <= 2.0
 
-    args = parse_args("instance --price-per-hour-min 1.0 --price-per-hour-max 2.0")
+    args = parse_args("instance --price-min 1.0 --price-max 2.0")
     results = client.instance_select(**args.__dict__)
     assert len(results) == 3
     for result in results:
@@ -326,7 +325,6 @@ def reset_data():
         "memory",
         "memory_bytes",
         "price",
-        "price_per_hour",
         "region",
         "gpu",
         "gpu_memory",

docs/settings.md

@@ -115,9 +115,12 @@ select on the fly, on the command line.
 | cpus |number |Number of vcpus available to the instance type. Sets min and -max to the same value. | unset |
 | cpus-min | number | Minimum number of vcpus available to the instance type. If max not set, is infinity. | unset |
 | cpus-max | number | Maximum number of vcpus available to the instance type. If min not set, is 0. | unset |
-| price-per-hour | number | Price/hour in dollars (e.g., 0.09) (sets min and max to the same value) | unset |
-| price-per-hour-min | number | Minimum price/hour in dollars. If max not set, is infinity. | unset |
-| price-per-hour-max | number |Maximum price/hour in dollars. If min not set, is 0." | unset |
+| price | number | Price/hour in dollars (e.g., 0.09) (sets min and max to the same value) | unset |
+| price-min | number | Minimum price/hour in dollars. If max not set, is infinity. | unset |
+| price-max | number | Maximum price/hour in dollars. If min not set, is 0." | unset |
+| spot-price | number | Spot Price/hour in dollars (e.g., 0.09) (sets min and max to the same value) | unset |
+| spot-price-min | number | Minimum spot price/hour in dollars. If max not set, is infinity. | unset |
+| spot-price-max | number | Maximum spot price/hour in dollars. If min not set, is 0." | unset |
 | free-tier | boolean | Instance Types that support IPv6 | unset |
 | cpu-arch | string | architecture of the CPU | unset, can be one of "x86_64/amd64", "x86_64_mac", "i386", "arm64" |
 | cpu-vendor | string | manufacturer of CPU | unset can be one of "amd", "intel", "aws" |

docs/tutorial.md

@@ -157,14 +157,14 @@ $ cloud-select --max-results 100 instance
 Or try filtering by price:
 
 ```bash
-$ cloud-select instance --price-per-hour-min 1.0
+$ cloud-select instance --price-min 1.0
 ```
 
 And also sorting, of course, either default or ascending:
 
 ```bash
-$ cloud-select instance --price-per-hour-min 1.0 --sort-by price
-$ cloud-select instance --price-per-hour-min 1.0 --sort-by price --asc
+$ cloud-select instance --price-min 1.0 --sort-by price
+$ cloud-select instance --price-min 1.0 --sort-by price --asc
 ```
 
 ![img/cloud-select-prices.png](img/cloud-select-prices.png)
@@ -273,7 +273,6 @@ CREATE TABLE IF NOT EXISTS instances (
     value_bool number NULLABLE,
     value_number number NULLABLE
 );
-Google Cloud instance prices derived from the web are limited to Iowa (us-centra-1)
 ```
 
 This will prepare the same database for you as used in the command line
@@ -317,7 +316,7 @@ What attributes are available for query (aside from exact fields shown above)? L
  ('memory',),
  ('memory_bytes',),
  ('price',),
- ('price_per_hour',),
+ ('spot_price',),
  ('region',),
  ('zone',)]
 ```
@@ -348,7 +347,6 @@ out for the remainder of these examples. Here is querying for an instance name:
 ```
 ```console
  (23252, ..., 'google', 'n2d-highcpu-224', 'price', None, None, 6.986112),
- (23253, ..., 'google', 'n2d-highcpu-224', 'price_per_hour', None, None, 6.986112),
  (23254, ..., 'google', 'n2d-highcpu-224', 'region', 'us-west1-a', None, None),
  (23255, ..., 'google', 'n2d-highcpu-224', 'zone', 'us-west1-a', None, None)]
 ```