train on new upgrades

scripts/build_feature_store.py

@@ -1403,10 +1403,7 @@ def transform_weather_features() -> DataFrame:
 
 # DBTITLE 1,Write out building metadata feature store
 table_name = "ml.surrogate_model.building_features"
-# TODO: remove this drop statement before retraining-- this is just a temp (yes hacky i know back off) solution to not break dohyo dowstream
-df = building_metadata_applicable_upgrades_with_weather_file_city_index.drop(
-    "heat_pump_sizing_methodology"
-)
+df = building_metadata_applicable_upgrades_with_weather_file_city_index
 if spark.catalog.tableExists(table_name):
     fe.write_table(name=table_name, df=df, mode="merge")
 else:

src/datagen.py

@@ -41,7 +41,6 @@ class DataGenerator(tf.keras.utils.Sequence):
     - fe (databricks.feature_engineering.client.FeatureEngineeringClient: client for interacting with the
                                                                             Databricks Feature Engineering in Unity Catalog
 
-    TODO: modify this be more flexible to use this at inference time only (e.g, don't load various feature tables into mem)
     """
 
     # init FeatureEngineering client
@@ -70,6 +69,7 @@ class DataGenerator(tf.keras.utils.Sequence):
         # heating
         "heating_fuel",
         "heating_appliance_type",
+        "heat_pump_sizing_methodology",
         "has_ducted_heating",
         "heating_efficiency_nominal_percentage",
         "heating_setpoint_degrees_f",
@@ -155,8 +155,7 @@ class DataGenerator(tf.keras.utils.Sequence):
         "propane": ["propane__total"],
     }
 
-    # TODO: add 13.01 and 11.05 before training new model
-    supported_upgrade_ids = [0.0, 1.0, 3.0, 4.0, 6.0, 9.0]
+    supported_upgrade_ids = [0.0, 1.0, 3.0, 4.0, 6.0, 9.0, 13.01, 11.05]
 
     def __init__(
         self,