diff --git a/app/dashboard/views.py b/app/dashboard/views.py index 6e1a2d3ce..29c23add1 100644 --- a/app/dashboard/views.py +++ b/app/dashboard/views.py @@ -755,7 +755,7 @@ def view_asset_parameters(request, scen_id, asset_type_name, asset_uuid): proj_id=scenario.project.id, ) input_timeseries_data = ( - existing_asset.input_timeseries if existing_asset.input_timeseries else "" + existing_asset.input_timeseries.values if existing_asset.input_timeseries else "" ) context.update( diff --git a/app/projects/dtos.py b/app/projects/dtos.py index 1ed456f49..1ce555a54 100644 --- a/app/projects/dtos.py +++ b/app/projects/dtos.py @@ -663,7 +663,7 @@ def to_timeseries_data(model_obj, field_name): value_type = ValueType.objects.filter(type=field_name).first() unit = value_type.unit if value_type is not None else None value_list = ( - json.loads(getattr(model_obj, field_name)) + getattr(model_obj, field_name).values if getattr(model_obj, field_name) is not None else None ) diff --git a/app/projects/forms.py b/app/projects/forms.py index 0f359d7d1..506be30b2 100644 --- a/app/projects/forms.py +++ b/app/projects/forms.py @@ -4,6 +4,7 @@ import json import io import csv +from django.db.models import Q from openpyxl import load_workbook import numpy as np @@ -33,6 +34,7 @@ PARAMETERS, DualNumberField, parse_input_timeseries, + TimeseriesField, ) @@ -664,16 +666,19 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) # which fields exists in the form are decided upon AssetType saved in the db - asset_type = AssetType.objects.get(asset_type=self.asset_type_name) + self.asset_type = AssetType.objects.get(asset_type=self.asset_type_name) + + # remove the fields not needed for the AssetType [ self.fields.pop(field) for field in list(self.fields) - if field not in asset_type.visible_fields + if field not in self.asset_type.visible_fields ] self.timestamps = None if self.existing_asset is not None: self.timestamps = self.existing_asset.timestamps + self.user = self.existing_asset.scenario.project.user elif scenario_id is not None: qs = Scenario.objects.filter(id=scenario_id) if qs.exists(): @@ -688,6 +693,14 @@ def __init__(self, *args, **kwargs): currency = qs.values_list("economic_data__currency", flat=True).get() currency = CURRENCY_SYMBOLS[currency] # TODO use mapping to display currency symbol + self.user = qs.get().user + + # set the custom timeseries field for timeseries + # the qs_ts selects timeseries of the corresponding MVS type that either belong to the user or are open source + if "input_timeseries" in self.fields: + self.fields["input_timeseries"] = TimeseriesField(qs_ts=Timeseries.objects.filter( + Q(ts_type=self.asset_type.mvs_type) & (Q(open_source=True) | Q(user=self.user)), + )) self.fields["inputs"] = forms.CharField( widget=forms.HiddenInput(), required=False @@ -799,7 +812,7 @@ def __init__(self, *args, **kwargs): ) if ":unit:" in self.fields[field].label: self.fields[field].label = self.fields[field].label.replace( - ":unit:", asset_type.unit + ":unit:", self.asset_type.unit ) """ ----------------------------------------------------- """ @@ -810,11 +823,11 @@ def is_input_timeseries_empty(self): else: return True - def clean_input_timeseries(self): + def clean_input_timeseries_old(self): """Override built-in Form method which is called upon form validation""" try: input_timeseries_values = [] - timeseries_file = self.files.get("input_timeseries", None) + timeseries_file = self.files.get("input_timeseries_file", None) # read the timeseries from file if any if timeseries_file is not None: input_timeseries_values = parse_input_timeseries(timeseries_file) @@ -899,8 +912,32 @@ def clean(self): self.timeseries_same_as_timestamps(feedin_tariff, "feedin_tariff") self.timeseries_same_as_timestamps(energy_price, "energy_price") + if "input_timeseries" in cleaned_data: + ts_data = json.loads(cleaned_data["input_timeseries"]) + input_method = ts_data["input_method"]["type"] + if input_method == "upload" or input_method == "manual": + # replace the dict with a new timeseries instance + cleaned_data["input_timeseries"] = self.create_timeseries_from_input( + ts_data) + if input_method == "select": + # return the timeseries instance + timeseries_id = ts_data["input_method"]["extra_info"] + cleaned_data["input_timeseries"] = Timeseries.objects.get(id=timeseries_id) + return cleaned_data + def create_timeseries_from_input(self, input_timeseries): + timeseries_name = input_timeseries["input_method"]["extra_info"] + timeseries_values = input_timeseries["values"] + ts_instance = Timeseries.objects.create(user=self.user, + name=timeseries_name, + ts_type=self.asset_type.mvs_type, + values=timeseries_values, + open_source=False + ) + + return ts_instance + def timeseries_same_as_timestamps(self, ts, param): if isinstance(ts, np.ndarray): ts = np.squeeze(ts).tolist() @@ -949,14 +986,11 @@ class Meta: "lifetime": forms.NumberInput( attrs={"placeholder": "e.g. 10 years", "min": "0", "step": "1"} ), - # TODO: Try changing this to FileInput - "input_timeseries": forms.FileInput( + "input_timeseries_old": forms.FileInput( attrs={ "onchange": "plot_file_trace(obj=this.files, plot_id='timeseries_trace')" } ), - # 'input_timeseries': forms.Textarea(attrs={'placeholder': 'e.g. [4,3,2,5,3,...]', - # 'style': 'font-weight:400; font-size:13px;'}), "crate": forms.NumberInput( attrs={ "placeholder": "factor of total capacity (kWh), e.g. 0.7", diff --git a/app/projects/helpers.py b/app/projects/helpers.py index cd08667f6..c492a6537 100644 --- a/app/projects/helpers.py +++ b/app/projects/helpers.py @@ -355,19 +355,8 @@ def use_required_attribute(self, initial): return False def decompress(self, value): - - answer = [self.default, "", None] - if value is not None: - value = value.replace("'", '"') - value = json.loads(value) - input_method = value["input_method"]["type"] - ts_values = value["values"] - if len(ts_values) == 1: - ts_values = ts_values[0] - if input_method == "select": - answer = [ts_values, value["input_method"]["extra_info"], None] - else: - answer = [ts_values, "", None] + # TODO update handling here - value corresponds to the pk of the timeseries + answer = ["", "", value] return answer @@ -427,13 +416,13 @@ def clean(self, values): self.set_widget_error() raise ValidationError( _( - "Please provide either a number within %(boundaries) s or upload a timeseries from a file" + "Please provide either a number within %(boundaries) s, select a timeseries or upload a timeseries from a file" ), code="required", params={"boundaries": self.boundaries}, ) else: - input_dict = dict(type="manuel") + input_dict = dict(type="manual") self.check_boundaries(answer) return json.dumps(dict(values=answer, input_method=input_dict)) diff --git a/app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py b/app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py new file mode 100644 index 000000000..bb1437c01 --- /dev/null +++ b/app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py @@ -0,0 +1,16 @@ +# Generated by Django 4.2.4 on 2024-10-10 14:37 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0019_timeseries")] + + operations = [ + migrations.RenameField( + model_name="asset", + old_name="input_timeseries", + new_name="input_timeseries_old", + ) + ] diff --git a/app/projects/migrations/0021_asset_input_timeseries.py b/app/projects/migrations/0021_asset_input_timeseries.py new file mode 100644 index 000000000..3d6cce2b4 --- /dev/null +++ b/app/projects/migrations/0021_asset_input_timeseries.py @@ -0,0 +1,23 @@ +# Generated by Django 4.2.4 on 2024-10-10 14:38 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("projects", "0020_rename_input_timeseries_asset_input_timeseries_old") + ] + + operations = [ + migrations.AddField( + model_name="asset", + name="input_timeseries", + field=models.ForeignKey( + null=True, + on_delete=django.db.models.deletion.CASCADE, + to="projects.timeseries", + ), + ) + ] diff --git a/app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py b/app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py new file mode 100644 index 000000000..ac47990d3 --- /dev/null +++ b/app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py @@ -0,0 +1,17 @@ +# Generated by Django 4.2.4 on 2024-10-14 15:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0021_asset_input_timeseries")] + + operations = [ + migrations.RenameField( + model_name="timeseries", old_name="end_time", new_name="end_date" + ), + migrations.RenameField( + model_name="timeseries", old_name="start_time", new_name="start_date" + ), + ] diff --git a/app/projects/migrations/0023_migrate_timeseries_to_model.py b/app/projects/migrations/0023_migrate_timeseries_to_model.py new file mode 100644 index 000000000..0b267e756 --- /dev/null +++ b/app/projects/migrations/0023_migrate_timeseries_to_model.py @@ -0,0 +1,82 @@ +from django.db import migrations +from django.db.models import Q +import json +from datetime import timedelta + + +def convert_timeseries_to_model(apps, schema_editor): + """ + Forward migration: Convert timeseries_old to Timeseries instance + """ + # Get historical models + Asset = apps.get_model('projects', 'Asset') + Timeseries = apps.get_model('projects', 'Timeseries') + db_alias = schema_editor.connection.alias + + # Iterate through all assets with timeseries_old data + for asset in Asset.objects.using(db_alias).exclude(Q(input_timeseries_old__isnull=True) | Q(input_timeseries_old=[])): + try: + # Calculate end time from asset start date and duration + duration = asset.scenario.evaluated_period + total_duration = timedelta(hours=asset.scenario.time_step) * duration + end_date = asset.scenario.start_date + total_duration + + # Create new Timeseries instance + timeseries = Timeseries.objects.using(db_alias).create( + name=f"{asset.name}_migration", + user=asset.scenario.project.user, + scenario=asset.scenario, + values=json.loads(asset.input_timeseries_old), + ts_type=asset.asset_type.mvs_type, + open_source=False, + start_date=asset.scenario.start_date, + time_step=asset.scenario.time_step, + end_date=end_date + ) + + # Update asset to point to new timeseries + asset.input_timeseries = timeseries + asset.save() + + except Exception as e: + print(f"Error migrating asset {asset.id} timeseries: {str(e)}") + raise e + + +def reverse_timeseries_conversion(apps, schema_editor): + """ + Reverse migration: Delete created Timeseries instances and restore old data + """ + Asset = apps.get_model('projects', 'Asset') + Timeseries = apps.get_model('projects', 'Timeseries') + db_alias = schema_editor.connection.alias + + try: + # Find all timeseries created by this migration + migration_timeseries = Timeseries.objects.using(db_alias).filter(name__contains="_migration") + + # Update assets to remove reference to timeseries + Asset.objects.using(db_alias).filter(input_timeseries__in=migration_timeseries).update( + input_timeseries=None + ) + + # Delete the timeseries instances + migration_timeseries.delete() + + except Exception as e: + print(f"Error deleting migrated timeseries: {str(e)}") + raise e + + +class Migration(migrations.Migration): + + dependencies = [("projects", "0022_rename_end_time_timeseries_end_date_and_more")] + + operations = [ + # Run the timeseries migration + migrations.RunPython( + convert_timeseries_to_model, + reverse_timeseries_conversion + ), + ] + diff --git a/app/projects/models/base_models.py b/app/projects/models/base_models.py index 94213cf6f..db70c5f92 100644 --- a/app/projects/models/base_models.py +++ b/app/projects/models/base_models.py @@ -314,9 +314,9 @@ class Timeseries(models.Model): settings.AUTH_USER_MODEL, on_delete=models.CASCADE, null=True, blank=True ) # TODO check that if both a user and scenario are provided the scenario belongs to the user - # scenario = models.ForeignKey( - # Scenario, on_delete=models.CASCADE, null=True, blank=True - # ) + scenario = models.ForeignKey( + Scenario, on_delete=models.CASCADE, null=True, blank=True + ) ts_type = models.CharField(max_length=12, choices=MVS_TYPE, blank=True, null=True) open_source = models.BooleanField( @@ -324,19 +324,18 @@ class Timeseries(models.Model): ) # get this from the scenario - # TODO rename with _date instead of _time - start_time = models.DateTimeField(blank=True, default=None, null=True) - end_time = models.DateTimeField(blank=True, default=None, null=True) + start_date = models.DateTimeField(blank=True, default=None, null=True) + end_date = models.DateTimeField(blank=True, default=None, null=True) time_step = models.IntegerField( blank=True, default=None, null=True, validators=[MinValueValidator(1)] ) + def __str__(self): + return f"{self.name} ({self.pk})" + def save(self, *args, **kwargs): - n = len(self.values) - if n == 1: - self.ts_type = "scalar" - elif n > 1: - self.ts_type = "vector" + # set time attributes + self.set_date_attributes_from_scenario() super().save(*args, **kwargs) @property @@ -350,8 +349,18 @@ def get_values(self): def compute_time_attribute_from_timestamps(self, timestamps): pass - def compute_end_time_from_duration(self, duration): - pass + def compute_end_date_from_duration(self): + duration = self.scenario.evaluated_period + total_duration = timedelta(hours=self.time_step) * duration + end_time = self.start_date + total_duration + return end_time + + def set_date_attributes_from_scenario(self): + if self.scenario is not None: + self.start_date = self.scenario.start_date + self.time_step = self.scenario.time_step + self.end_date = self.compute_end_date_from_duration() + class AssetType(models.Model): @@ -435,9 +444,10 @@ def save(self, *args, **kwargs): lifetime = models.IntegerField( null=True, blank=False, validators=[MinValueValidator(0)] ) - input_timeseries = models.TextField( + input_timeseries_old = models.TextField( null=True, blank=False ) # , validators=[validate_timeseries]) + input_timeseries = models.ForeignKey(Timeseries, on_delete=models.CASCADE, null=True, blank=False) crate = models.FloatField( null=True, blank=False, default=1, validators=[MinValueValidator(0.0)] ) @@ -513,7 +523,7 @@ def get_field_value(self, field_name): "efficiency_multiple", "energy_price", "feedin_tariff", - "input_timeseries", + "input_timeseries_old", ): try: answer = float(answer) diff --git a/app/projects/scenario_topology_helpers.py b/app/projects/scenario_topology_helpers.py index 31268c772..c8b185420 100644 --- a/app/projects/scenario_topology_helpers.py +++ b/app/projects/scenario_topology_helpers.py @@ -113,7 +113,13 @@ def track_asset_changes(scenario, param, form, existing_asset, new_value=None): old_value = pi.old_value if pi.parameter_type == "vector": old_value = (old_value, None) - if new_value == form.fields[pi.name].clean(old_value): + + if pi.name == "input_timeseries": + new_value = str(new_value) + else: + old_value = form.fields[pi.name].clean(old_value) + + if new_value == old_value: pi.delete() else: qs_param.update(new_value=new_value) diff --git a/app/projects/views.py b/app/projects/views.py index 96defa353..0f7860c8a 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -1587,7 +1587,7 @@ def get_asset_create_form(request, scen_id=0, asset_type_name="", asset_uuid=Non proj_id=scenario.project.id, ) input_timeseries_data = ( - existing_asset.input_timeseries + existing_asset.input_timeseries.values if existing_asset.input_timeseries else "" )