Replace old Timeseries field with Timeseries model

app/dashboard/views.py

@@ -755,7 +755,7 @@ def view_asset_parameters(request, scen_id, asset_type_name, asset_uuid):
             proj_id=scenario.project.id,
         )
         input_timeseries_data = (
-            existing_asset.input_timeseries if existing_asset.input_timeseries else ""
+            existing_asset.input_timeseries.values if existing_asset.input_timeseries else ""
         )
 
         context.update(

app/projects/dtos.py

@@ -663,7 +663,7 @@ def to_timeseries_data(model_obj, field_name):
     value_type = ValueType.objects.filter(type=field_name).first()
     unit = value_type.unit if value_type is not None else None
     value_list = (
-        json.loads(getattr(model_obj, field_name))
+        getattr(model_obj, field_name).values
         if getattr(model_obj, field_name) is not None
         else None
     )

app/projects/forms.py

@@ -4,6 +4,7 @@
 import json
 import io
 import csv
+from django.db.models import Q
 from openpyxl import load_workbook
 import numpy as np
 
@@ -33,6 +34,7 @@
     PARAMETERS,
     DualNumberField,
     parse_input_timeseries,
+    TimeseriesField,
 )
 
 
@@ -664,16 +666,19 @@ def __init__(self, *args, **kwargs):
 
         super().__init__(*args, **kwargs)
         # which fields exists in the form are decided upon AssetType saved in the db
-        asset_type = AssetType.objects.get(asset_type=self.asset_type_name)
+        self.asset_type = AssetType.objects.get(asset_type=self.asset_type_name)
+
+        # remove the fields not needed for the AssetType
         [
             self.fields.pop(field)
             for field in list(self.fields)
-            if field not in asset_type.visible_fields
+            if field not in self.asset_type.visible_fields
         ]
 
         self.timestamps = None
         if self.existing_asset is not None:
             self.timestamps = self.existing_asset.timestamps
+            self.user = self.existing_asset.scenario.project.user
         elif scenario_id is not None:
             qs = Scenario.objects.filter(id=scenario_id)
             if qs.exists():
@@ -688,6 +693,14 @@ def __init__(self, *args, **kwargs):
                 currency = qs.values_list("economic_data__currency", flat=True).get()
                 currency = CURRENCY_SYMBOLS[currency]
                 # TODO use mapping to display currency symbol
+                self.user = qs.get().user
+
+        # set the custom timeseries field for timeseries
+        # the qs_ts selects timeseries of the corresponding MVS type that either belong to the user or are open source
+        if "input_timeseries" in self.fields:
+            self.fields["input_timeseries"] = TimeseriesField(qs_ts=Timeseries.objects.filter(
+                Q(ts_type=self.asset_type.mvs_type) & (Q(open_source=True) | Q(user=self.user)),
+            ))
 
         self.fields["inputs"] = forms.CharField(
             widget=forms.HiddenInput(), required=False
@@ -799,7 +812,7 @@ def __init__(self, *args, **kwargs):
                     )
                 if ":unit:" in self.fields[field].label:
                     self.fields[field].label = self.fields[field].label.replace(
-                        ":unit:", asset_type.unit
+                        ":unit:", self.asset_type.unit
                     )
 
         """ ----------------------------------------------------- """
@@ -810,11 +823,11 @@ def is_input_timeseries_empty(self):
         else:
             return True
 
-    def clean_input_timeseries(self):
+    def clean_input_timeseries_old(self):
         """Override built-in Form method which is called upon form validation"""
         try:
             input_timeseries_values = []
-            timeseries_file = self.files.get("input_timeseries", None)
+            timeseries_file = self.files.get("input_timeseries_file", None)
             # read the timeseries from file if any
             if timeseries_file is not None:
                 input_timeseries_values = parse_input_timeseries(timeseries_file)
@@ -899,8 +912,32 @@ def clean(self):
                 self.timeseries_same_as_timestamps(feedin_tariff, "feedin_tariff")
                 self.timeseries_same_as_timestamps(energy_price, "energy_price")
 
+        if "input_timeseries" in cleaned_data:
+            ts_data = json.loads(cleaned_data["input_timeseries"])
+            input_method = ts_data["input_method"]["type"]
+            if input_method == "upload" or input_method == "manual":
+                # replace the dict with a new timeseries instance
+                cleaned_data["input_timeseries"] = self.create_timeseries_from_input(
+                    ts_data)
+            if input_method == "select":
+                # return the timeseries instance
+                timeseries_id = ts_data["input_method"]["extra_info"]
+                cleaned_data["input_timeseries"] = Timeseries.objects.get(id=timeseries_id)
+
         return cleaned_data
 
+    def create_timeseries_from_input(self, input_timeseries):
+        timeseries_name = input_timeseries["input_method"]["extra_info"]
+        timeseries_values = input_timeseries["values"]
+        ts_instance = Timeseries.objects.create(user=self.user,
+                                                name=timeseries_name,
+                                                ts_type=self.asset_type.mvs_type,
+                                                values=timeseries_values,
+                                                open_source=False
+                                                )
+
+        return ts_instance
+
     def timeseries_same_as_timestamps(self, ts, param):
         if isinstance(ts, np.ndarray):
             ts = np.squeeze(ts).tolist()
@@ -949,14 +986,11 @@ class Meta:
             "lifetime": forms.NumberInput(
                 attrs={"placeholder": "e.g. 10 years", "min": "0", "step": "1"}
             ),
-            # TODO: Try changing this to FileInput
-            "input_timeseries": forms.FileInput(
+            "input_timeseries_old": forms.FileInput(
                 attrs={
                     "onchange": "plot_file_trace(obj=this.files, plot_id='timeseries_trace')"
                 }
             ),
-            # 'input_timeseries': forms.Textarea(attrs={'placeholder': 'e.g. [4,3,2,5,3,...]',
-            #                                           'style': 'font-weight:400; font-size:13px;'}),
             "crate": forms.NumberInput(
                 attrs={
                     "placeholder": "factor of total capacity (kWh), e.g. 0.7",

app/projects/helpers.py

@@ -355,19 +355,8 @@ def use_required_attribute(self, initial):
         return False
 
     def decompress(self, value):
-
-        answer = [self.default, "", None]
-        if value is not None:
-            value = value.replace("'", '"')
-            value = json.loads(value)
-            input_method = value["input_method"]["type"]
-            ts_values = value["values"]
-            if len(ts_values) == 1:
-                ts_values = ts_values[0]
-            if input_method == "select":
-                answer = [ts_values, value["input_method"]["extra_info"], None]
-            else:
-                answer = [ts_values, "", None]
+        # TODO update handling here - value corresponds to the pk of the timeseries
+        answer = ["", "", value]
 
         return answer
 
@@ -427,13 +416,13 @@ def clean(self, values):
                 self.set_widget_error()
                 raise ValidationError(
                     _(
-                        "Please provide either a number within %(boundaries) s or upload a timeseries from a file"
+                        "Please provide either a number within %(boundaries) s, select a timeseries or upload a timeseries from a file"
                     ),
                     code="required",
                     params={"boundaries": self.boundaries},
                 )
             else:
-                input_dict = dict(type="manuel")
+                input_dict = dict(type="manual")
 
         self.check_boundaries(answer)
         return json.dumps(dict(values=answer, input_method=input_dict))

app/projects/migrations/0020_rename_input_timeseries_asset_input_timeseries_old.py

@@ -0,0 +1,16 @@
+# Generated by Django 4.2.4 on 2024-10-10 14:37
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("projects", "0019_timeseries")]
+
+    operations = [
+        migrations.RenameField(
+            model_name="asset",
+            old_name="input_timeseries",
+            new_name="input_timeseries_old",
+        )
+    ]

app/projects/migrations/0021_asset_input_timeseries.py

@@ -0,0 +1,23 @@
+# Generated by Django 4.2.4 on 2024-10-10 14:38
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("projects", "0020_rename_input_timeseries_asset_input_timeseries_old")
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="asset",
+            name="input_timeseries",
+            field=models.ForeignKey(
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                to="projects.timeseries",
+            ),
+        )
+    ]

app/projects/migrations/0022_rename_end_time_timeseries_end_date_and_more.py

@@ -0,0 +1,17 @@
+# Generated by Django 4.2.4 on 2024-10-14 15:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("projects", "0021_asset_input_timeseries")]
+
+    operations = [
+        migrations.RenameField(
+            model_name="timeseries", old_name="end_time", new_name="end_date"
+        ),
+        migrations.RenameField(
+            model_name="timeseries", old_name="start_time", new_name="start_date"
+        ),
+    ]

app/projects/migrations/0023_migrate_timeseries_to_model.py

@@ -0,0 +1,82 @@
+from django.db import migrations
+from django.db.models import Q
+import json
+from datetime import timedelta
+
+
+def convert_timeseries_to_model(apps, schema_editor):
+    """
+    Forward migration: Convert timeseries_old to Timeseries instance
+    """
+    # Get historical models
+    Asset = apps.get_model('projects', 'Asset')
+    Timeseries = apps.get_model('projects', 'Timeseries')
+    db_alias = schema_editor.connection.alias
+
+    # Iterate through all assets with timeseries_old data
+    for asset in Asset.objects.using(db_alias).exclude(Q(input_timeseries_old__isnull=True) | Q(input_timeseries_old=[])):
+        try:
+            # Calculate end time from asset start date and duration
+            duration = asset.scenario.evaluated_period
+            total_duration = timedelta(hours=asset.scenario.time_step) * duration
+            end_date = asset.scenario.start_date + total_duration
+
+            # Create new Timeseries instance
+            timeseries = Timeseries.objects.using(db_alias).create(
+                name=f"{asset.name}_migration",
+                user=asset.scenario.project.user,
+                scenario=asset.scenario,
+                values=json.loads(asset.input_timeseries_old),
+                ts_type=asset.asset_type.mvs_type,
+                open_source=False,
+                start_date=asset.scenario.start_date,
+                time_step=asset.scenario.time_step,
+                end_date=end_date
+            )
+
+            # Update asset to point to new timeseries
+            asset.input_timeseries = timeseries
+            asset.save()
+
+        except Exception as e:
+            print(f"Error migrating asset {asset.id} timeseries: {str(e)}")
+            raise e
+
+
+def reverse_timeseries_conversion(apps, schema_editor):
+    """
+    Reverse migration: Delete created Timeseries instances and restore old data
+    """
+    Asset = apps.get_model('projects', 'Asset')
+    Timeseries = apps.get_model('projects', 'Timeseries')
+    db_alias = schema_editor.connection.alias
+
+    try:
+        # Find all timeseries created by this migration
+        migration_timeseries = Timeseries.objects.using(db_alias).filter(name__contains="_migration")
+
+        # Update assets to remove reference to timeseries
+        Asset.objects.using(db_alias).filter(input_timeseries__in=migration_timeseries).update(
+            input_timeseries=None
+        )
+
+        # Delete the timeseries instances
+        migration_timeseries.delete()
+
+    except Exception as e:
+        print(f"Error deleting migrated timeseries: {str(e)}")
+        raise e
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [("projects", "0022_rename_end_time_timeseries_end_date_and_more")]
+
+    operations = [
+        # Run the timeseries migration
+        migrations.RunPython(
+            convert_timeseries_to_model,
+            reverse_timeseries_conversion
+        ),
+    ]
+