Merge branch 'master' into fix_number_of_units

.github/workflows/CompatHelper.yml

@@ -0,0 +1,45 @@
+name: CompatHelper
+on:
+  schedule:
+    - cron: 0 0 * * *
+  workflow_dispatch:
+permissions:
+  contents: write
+  pull-requests: write
+jobs:
+  CompatHelper:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check if Julia is already available in the PATH
+        id: julia_in_path
+        run: which julia
+        continue-on-error: true
+      - name: Install Julia, but only if it is not already available in the PATH
+        uses: julia-actions/setup-julia@v2
+        with:
+          version: '1'
+          arch: ${{ runner.arch }}
+        if: steps.julia_in_path.outcome != 'success'
+      - name: "Add the General registry via Git"
+        run: |
+          import Pkg
+          ENV["JULIA_PKG_SERVER"] = ""
+          Pkg.Registry.add("General")
+        shell: julia --color=yes {0}
+      - name: "Install CompatHelper"
+        run: |
+          import Pkg
+          name = "CompatHelper"
+          uuid = "aa819f21-2bde-4658-8897-bab36330d9b7"
+          version = "3"
+          Pkg.add(; name, uuid, version)
+        shell: julia --color=yes {0}
+      - name: "Run CompatHelper"
+        run: |
+          import CompatHelper
+          CompatHelper.main()
+        shell: julia --color=yes {0}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          COMPATHELPER_PRIV: ${{ secrets.DOCUMENTER_KEY }}
+          # COMPATHELPER_PRIV: ${{ secrets.COMPATHELPER_PRIV }}

.github/workflows/TagBot.yml

@@ -0,0 +1,31 @@
+name: TagBot
+on:
+    issue_comment:
+        types:
+            - created
+    workflow_dispatch:
+        inputs:
+            lookback:
+                default: "3"
+permissions:
+    actions: read
+    checks: read
+    contents: write
+    deployments: read
+    issues: read
+    discussions: read
+    packages: read
+    pages: read
+    pull-requests: read
+    repository-projects: read
+    security-events: read
+    statuses: read
+jobs:
+    TagBot:
+        if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
+        runs-on: ubuntu-latest
+        steps:
+            - uses: JuliaRegistries/TagBot@v1
+              with:
+                  token: ${{ secrets.GITHUB_TOKEN }}
+                  ssh: ${{ secrets.DOCUMENTER_KEY }}

Project.toml

@@ -1,7 +1,7 @@
 name = "SpineOpt"
 uuid = "0d8fc150-4032-4b6e-9540-20efcb304861"
 authors = ["Spine Project consortium <[email protected]>"]
-version = "0.8.3"
+version = "0.9.1"
 
 [deps]
 Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
@@ -20,6 +20,13 @@ SpineInterface = "0cda1612-498a-11e9-3c92-77fa82595a4f"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 
 [compat]
-JuMP = "^1.20"
-SpineInterface = "0.14"
-julia = "^1.8"
+Arrow = "2"
+DataStructures = "0.18"
+Distributions = "0.25"
+HiGHS = "1"
+JSON = "0.21"
+JuMP = "^1.23.5"
+Requires = "1"
+SpineInterface = "0.15.1"
+TOML = "1"
+julia = "^1.8"

docs/make.jl

@@ -34,11 +34,11 @@ pages = [
     "Introduction" => "index.md",
     "Getting Started" => Any[
         "Installation" => joinpath("getting_started", "installation.md"),
-        "Recommended workflow" => joinpath("getting_started", "recommended_workflow.md"),
+        "Verify installation" => joinpath("getting_started", "recommended_workflow.md"),
         "Troubleshooting" => joinpath("getting_started", "troubleshooting.md"),
     ],
     "Tutorials" => Any[
-        "Webinars and examples" => joinpath("tutorial", "webinars.md"),
+        "Webinars" => joinpath("tutorial", "webinars.md"),
         "Simple system" => joinpath("tutorial", "simple_system.md"),
         "Temporal resolution" => joinpath("tutorial", "temporal_resolution.md"),
         "Stochastic structure" => joinpath("tutorial", "stochastic_system.md"),
@@ -49,6 +49,7 @@ pages = [
         "Unit Commitment" => joinpath("tutorial", "unit_commitment.md"),
     ],
     "How to" => [],
+    "Example gallery" => joinpath("gallery", "gallery.md"),    
     "Database structure" => Any[
         "Basics of the data structure" => joinpath("concept_reference", "the_basics.md"),
         "Archetypes" => joinpath("concept_reference", "archetypes.md"),
@@ -60,6 +61,7 @@ pages = [
     "Standard model features" => Any[
         "Unit Commitment" => joinpath("advanced_concepts", "unit_commitment.md"),
         "Investment Optimization" => joinpath("advanced_concepts", "investment_optimization.md"),
+        "Multi-year Investments" => joinpath("advanced_concepts", "multi-year.md"),
         "Reserves" => joinpath("advanced_concepts", "reserves.md"),
         "Ramping" => joinpath("advanced_concepts", "ramping.md"),
         "Lossless nodal DC power flows" => joinpath("advanced_concepts", "Lossless_DC_power_flow.md"),

docs/src/advanced_concepts/multi-year.md

@@ -0,0 +1,53 @@
+# Multi-year investments
+
+Multi-year investments refer to making investment decisions at different points in time, such that a pathway of investments can be modeled. This is particularly useful when long-term scenarios are modeled, but modeling each year is not practical. Or in a business case, investment decisions are supposed to be made in different years which has an impact on the cash flow.
+
+There are two tutorials related to multi-year investments: [Capacity planning Tutorial](https://spine-tools.github.io/SpineOpt.jl/latest/tutorial/capacity_planning/) and [Multi-year investments](https://spine-tools.github.io/SpineOpt.jl/latest/tutorial/capacity_planning/#Multi-year-investments). This section covers the concepts of multi-year investments in SpineOpt, but we highly recommend checking out these tutorial for a more thorough understanding of how the model is set up.
+
+## Basic idea
+
+SpineOpt offers flexibility to the users so that different things can be modeled given specific set-ups and inputs of the model. This flexibility can be greatly illustrated by the multi-year investments modeling. We apply the same mathematical formulation for any capacity planning exercises, as shown in [Capacity planning Tutorial](https://spine-tools.github.io/SpineOpt.jl/latest/tutorial/capacity_planning/). For the multi-year model, what you would need differently from a single-year model is mainly the specification of the temporal structure, i.e., the investment period and operational period, and the rest works very much similarly to a single-year model.
+
+## Economic representation
+
+### Parameters
+It can be tricky to put the correct cost parameters into the model since factors like discounting and end-of-lifetime effects have to be taken into account. For that purpose, SpineOpt has incorporated some dedicated parameters for economic representation. Set `use_economic_representation` to `true` will activate these paramters.
+
+**Discounted annuities**
+
+This factor translates the overnight costs of investment into discounted (to the discount_year) annual payments, distributed over the total lifetime of the investment. Investment payments are assumed to increase linearly over the lead-time, and decrease linearly towards the end of the economic lifetime. This is also illustrated here:
+
+![image](figs_multi-year/discounted_annuities.png)
+
+For this purpose, we first calculate first the fraction of payment per year (e.g. something like 0.25, 0.5, 0.75,1 over the lead time; 1 for the economic lifetime minus the lead time, and 0.75, 0.5, 0.25 and 0 for the remaining economic lifetime). Each payment fraction is then multiplied by the discounting factor of the payment year with respect to the discounting year (e.g. start of optimization).
+
+**Salvage fraction**
+
+As we consider (discounted and annuitized) overnight costs in the objective function, it can happen that the lifetime of a unit exceeds the model horizon. In such cases, the salvage fraction needs to be deducted from the objective function. In principle, this means, that the annuities "already paid", which exceed the modelling horizon, are recuperated.
+
+**Discounted durations**
+
+The discounted duration is used to discount operational costs within a certain investment period to the discount year (e.g. beginning of the optimization). If milestone years are used for investments, the discounted duration is calculated for each investment period as defined by investment temporal blocks, otherwise, it will be calculated on a yearly basis.
+
+**Technology specific discount factors**
+
+The technology specific discount factor can be used, if e.g. investments in a certain technology are particularly risky. The default value for this parameter is 1.
+
+### Adaptions objective terms
+
+When `use_economic_representation` is set to `true`:
+
+- Investment costs are multiplied with discounted annuties conversion factor and the technological discount factor and (1 - salvage fraction).
+
+- Operational cost terms are multiplied with the discounted duration factor.
+
+### Additional information
+
+More information can be found in the following files.
+
+- [Economic representation in SpineOpt](https://github.com/Spine-project/SpineOpt.jl/files/9130471/Spine_invest.3.pdf) contains the details of the calculation of the economic prameters. Note that this document covers more concepts than what is currently available in SpineOpt (the available ones are the parameters listed above and the adaptions in the objective), the rest is under development.
+
+- [Economic parameters calculation tool](https://github.com/Spine-project/SpineOpt.jl/files/9130469/parameter_illustration.2.xlsx) is an excel tool that you can use to calculate the economic parameters on your own.
+
+!!! warning
+    Please also note that the use of economic representation in SpineOpt does not support rolling horizon and Benders' decomposition, which warrants future improvements.

docs/src/gallery/gallery.md

@@ -0,0 +1,22 @@
+# Example gallery
+
+## How to run the examples
+
+To run the examples, we recommend to follow the instructions [here](https://spine-tools.github.io/SpineOpt.jl/latest/getting_started/recommended_workflow/) to set up the basic SpineToolbox workflow, loading the JSON example file in your input data store depending on the example you want to run. We recommend creating a new SpineToolbox project for each example.
+
+## Examples
+
+Each example has a link to the JSON file with the input data.
+
+- [**Simple system**](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/simple_system.json): Example with two nodes and units, including the relationships among them for a temporal block with duration of one day.
+- [**Stochastic model**](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/stochastic.json): Setup of stochastic structures for three forecast scenarios.
+- [**Capacity planning**](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/capacity_planning.json): This example shows the definition of investment and operational temporal blocks in SpineOpt for one target year.
+- **Multi-year investment examples**: The following examples have a setup for a multi-year investment case study (e.g., for pathway planning) in a five-year time horizon with investments every five years (e.g., at the beginning and end of the time horizon) with a 4-month operational temporal block duration. Here, we have different options in SpineOpt:
+  - [*Example without economic parameters calculation*](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/multi-year_investment_without_econ_params.json): No extra manipulation of the input data from SpineOpt (`use_economic_represention=false`). So, the user input data needs to account for the value of money over time according to discount rate and year.
+  - [*Example with economic parameters calculation without milestone years*](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/multi-year_investment_with_econ_params_without_milestones.json): SpineOpt internally-calculates the parameters for discounting investment and operation costs given a discount rate, discount year, and lifetime information (`use_economic_represention=true`).
+  - [*Example with economic parameters calculation with milestone years*](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/multi-year_investment_with_econ_params_with_milestones.json): Since solving five years of operation might be computational intensive, SpineOpt offers the option of having milestone years (`use_milestone_years=true`). In that case, only two years are solved in the model. Still, one represents the operation of the non-milestone years using a calculated weight, including the discounted operation costs.
+- [**Reserves constraints**](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/reserves.json): Extension of the simple system including operating reserve constraints.
+- [**Unit commitment constraints**](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/unit_commitment.json): Extension of the sinple system including unit commitment constraints.
+- [**Rolling horizon**](https://github.com/spine-tools/SpineOpt.jl/blob/master/examples/rolling_horizon.json): This is an Example with a total time horizon of one week, an optimization window of one day, and rolling forward one day at a time.
+
+Some archived examples are also in the spine tools repository, e.g. [case study A5](https://github.com/spine-tools/spine-cs-a5).

docs/src/getting_started/installation.md

@@ -102,9 +102,7 @@ Pkg.activate("jenv") # activate the virtual environment in the current working d
 To install SpineOpt, execute the following commands in the Julia REPL:
 ```julia
 import Pkg # not needed if you are still in the same REPL session
-Pkg.Registry.add("General")
-Pkg.Registry.add(Pkg.RegistrySpec(url = "https://github.com/spine-tools/SpineJuliaRegistry")) # Add SpineJuliaRegistry as an available registry for your Julia
-Pkg.add("SpineOpt") # Install SpineOpt from the SpineJuliaRegistry
+Pkg.add("SpineOpt")
 ```
 
 The SpineOpt package is now available and ready to use in your julia scripts. If you have installed SpineOpt in a virtual environment, don't forget to activate the virtual environment when you try to run your scripts.

docs/src/getting_started/recommended_workflow.md

@@ -1,4 +1,4 @@
-# [Recommended Workflow](@id recommended_workflow)
+# [Verify installation](@id recommended_workflow)
 
 Now that we've installed Spine Toolbox and SpineOpt, let's make sure that everything truly works by running an example. We'll be using an existing example to ensure that any issues we may encounter at this point are related to the installation. If you indeed encounter any problems, check the [troubleshooting](@ref troubleshooting) section. On the other hand, if you are able to successfully complete this example, you can continue to a first hands on experience with the tutorials.
 

docs/src/getting_started/troubleshooting.md

@@ -8,6 +8,11 @@ Have you created a new project? File > New project
 ## I get an error that the 'model' object is not defined in the database
 That tends to happen when you accidentally switched your input and output in the Run SpineOpt tool.
 
+## Solver throws error `Name too long`
+Variables and constraints names in SpineOpt sometimes can be longer than the maximum name length the solvers allow. If you get this error, we recommend using the solver parameters to run it ingnoring the names. For instance, Gurobi has the parameter `IgnoreNames` which allows to run without the names but returning the solution with the original names in the model.
+
+To setup the solver parameters you can look at the [How-to section](@ref how-to-change-solver) guide change the solver and define its parameters.
+
 ## SpineOpt and SpineInterface are out of sync
 Some of the development of SpineOpt depends on the development of SpineInterface and vice versa. At some points in time that can create an incompatibility between the two.
 

docs/src/how_to/change_the_solver.md

@@ -1,4 +1,4 @@
-# How to change the solver
+# [How to change the solver](@id how-to-change-solver)
 
 If you want to change the solver for your optimization problem in SpineOpt, here is some guidance:
 - You can change the solvers in your input datastore using the `db_lp_solver` and `db_mip_solver` parameter values of the `model` object.

docs/src/tutorial/webinars.md

@@ -1,10 +1,8 @@
-# Webinars and examples
+# Webinars
 
 [SpinOpt tutorial](https://www.youtube.com/watch?v=brf6rlzNDR0) covers the entire process from installing Spine Toolbox and SpineOpt, creating and running a model with these tools and manipulating databases.
 
-[Spine Toolbox: Case study of Unit Commitment](https://www.youtube.com/watch?v=IYIEDyBsSvk) is another nice example by Alireza Zakariazadeh ([email protected]).
-
-There are also some archived examples in the spine tools repository, e.g. [case study A5](https://github.com/spine-tools/spine-cs-a5).
+[Spine Toolbox: Case study of Unit Commitment](https://www.youtube.com/watch?v=IYIEDyBsSvk) is another nice webinar by Alireza Zakariazadeh (`[email protected]`).
 
 !!! tip
-    Looking for the Hydro tutorial? It is now part of the how to section.
+    Looking for the Hydro tutorial? It is now part of the how to section.