Merge branch 'nexus-upd' of https://github.com/iiasa/message-ix-models into nexus-upd

Author: awais307

.gitattributes

@@ -1,6 +1,7 @@
 # Reduce the number of merge conflicts
 doc/whatsnew.rst merge=union
 # Git LFS
-*.csv.gz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
 *.xlsx filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text

.github/workflows/pytest.yaml

@@ -62,6 +62,10 @@ jobs:
         version: 25.1.1
         license: ${{ secrets.GAMS_LICENSE }}
 
+    - uses: ts-graphviz/setup-graphviz@v1
+      with:
+        macos-skip-brew-update: true
+
     - name: Install packages and dependencies
       # By default, install:
       # - ixmp, message_ix: from GitHub branches/tags per matrix.upstream-version (above)
@@ -93,12 +97,6 @@ jobs:
           --local-cache --jvmargs="-Xmx6G"
       shell: bash
 
-    - name: Test documentation build using Sphinx
-      if: startsWith(matrix.version.os, 'ubuntu')
-      env:
-        RTD_TOKEN_MESSAGE_DATA: ${{ secrets.RTD_TOKEN_MESSAGE_DATA }}
-      run: make --directory=doc SPHINXOPTS="-n --color" html
-
     - name: Upload test coverage to Codecov.io
       uses: codecov/codecov-action@v3
 
@@ -110,6 +108,7 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v4
+      with: { python-version: "3.11" }
 
     - name: Force recreation of pre-commit virtual environment for mypy
       if: github.event_name == 'schedule'  # Comment this line to run on a PR

.gitignore

@@ -38,20 +38,21 @@ pip-log.txt
 pip-delete-this-directory.txt
 
 # Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
 .benchmarks
+.cache
 .coverage
 .coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
 .hypothesis/
+.nox/
 .pytest_cache/
 .ruff_cache
+.tox/
+*.cover
+*.py,cover
+coverage.xml
+htmlcov/
+nosetests.xml
+/prof/
 
 # Translations
 *.mo
@@ -71,7 +72,7 @@ instance/
 .scrapy
 
 # Sphinx documentation
-doc/_autosummary/
+doc/**/_autosummary/
 doc/_build/
 doc/*.inv
 
@@ -138,6 +139,8 @@ dmypy.json
 # Generated and temporary data files
 debug/
 cache/
+# LibreOffice lock files
+.~lock*#
 
 # VSCode settings
 .vscode

.pre-commit-config.yaml

@@ -10,19 +10,18 @@ repos:
     language: python
     entry: bash -c ". ${PRE_COMMIT_MYPY_VENV:-/dev/null}/bin/activate 2>/dev/null; mypy $0 $@"
     additional_dependencies:
-    - "mypy <1.5"
+    - mypy >= 1.8.0
+    - plotnine
     - pytest
     - sdmx1
     - types-PyYAML
     - types-tqdm
     - "ixmp @ git+https://github.com/iiasa/ixmp.git@main"
     - "message-ix @ git+https://github.com/iiasa/message_ix.git@main"
     args: ["."]
-- repo: https://github.com/psf/black-pre-commit-mirror
-  rev: 23.7.0
-  hooks:
-  - id: black
 - repo: https://github.com/astral-sh/ruff-pre-commit
-  rev: v0.0.287
+  rev: v0.1.13
   hooks:
   - id: ruff
+  - id: ruff-format
+    args: [ --check ]

MANIFEST.in

@@ -1 +1,6 @@
+prune message_ix_models/data/test/advance
+prune message_ix_models/data/test/iea
 prune message_ix_models/data/test/MESSAGEix-GLOBIOM_1.1_R11_no-policy_baseline
+prune message_ix_models/data/test/ssp
+prune message_ix_models/data/water/*
+exclude message_ix_models/data/water/*.tar.xz

README.rst

@@ -25,6 +25,6 @@ See the `online documentation <https://docs.messageix.org/projects/models/>`_ fo
 License
 =======
 
-Copyright © 2020–2023 IIASA Energy, Climate, and Environment (ECE) program
+Copyright © 2020–2024 IIASA Energy, Climate, and Environment (ECE) program
 
 Licensed under the Apache License, version 2.0.

doc/Makefile

@@ -12,9 +12,12 @@ BUILDDIR      = _build
 help:
 	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
 
-.PHONY: help Makefile
+clean-autosummary:
+	find . -name "$(BUILDDIR)" -prune -o -iname _autosummary -print0 | xargs -0 rm -rf
+
+.PHONY: help clean-autosummary Makefile
 
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
+.DEFAULT: Makefile
 	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

doc/api/data-sources.rst

@@ -0,0 +1,141 @@
+Tools for specific data sources
+*******************************
+
+.. _tools-iea:
+
+International Energy Agency (IEA) (:mod:`.tools.iea`)
+=====================================================
+
+The IEA publishes many kinds of data.
+Each distinct data source is handled by a separate submodule of :mod:`message_ix_models.tools.iea`.
+
+Documentation for all module contents:
+
+.. currentmodule:: message_ix_models.tools
+
+.. autosummary::
+   :toctree: _autosummary
+   :template: autosummary-module.rst
+   :recursive:
+
+   iea
+
+.. _tools-iea-web:
+
+(Extended) World Energy Balances (:mod:`.tools.iea.web`)
+--------------------------------------------------------
+
+.. contents::
+   :local:
+   :backlinks: none
+
+.. note:: These data are **proprietary** and require a paid subscription.
+
+The approach to handling proprietary data is the same as in :mod:`.project.advance` and :mod:`.project.ssp`:
+
+- Copies of the data are stored in the (private) :mod:`message_data` repository using Git LFS.
+  This respository is accessible only to users who have a license for the data.
+- :mod:`message_ix_models` contains only a ‘fuzzed’ version of the data (same structure, random values) for testing purposes.
+- Non-IIASA users must obtain their own license to access and use the data; obtain the data themselves; and place it on the system where they use :mod:`message_ix_models`.
+
+The module :mod:`message_ix_models.tools.iea.web` attempts to detect and support both the providers/formats described below.
+The code supports using data from any of the above locations and formats, in multiple ways:
+
+- Use :func:`.tools.iea.web.load_data` to load data as :class:`pandas.DataFrame` and apply further pandas processing.
+- Use :class:`.IEA_EWEB` via :func:`.tools.exo_data.prepare_computer` to use the data in :mod:`genno` structured calculations.
+
+The **documentation** for the `2023 edition <https://iea.blob.core.windows.net/assets/0acb1453-1221-421b-9131-632ce71a4c1a/WORLDBAL_Documentation.pdf>`__ of the IEA source/format is publicly available.
+
+Structure
+~~~~~~~~~
+
+The data have the following conceptual dimensions, each enumerated by a different list of codes:
+
+- ``FLOW``, ``PRODUCT``: for both of these, the lists of codes appearing in the data are the same from 2021 and 2023 inclusive.
+- ``COUNTRY``: The data provided by IEA directly contain codes that are all caps, abbreviated country names, for instance "DOMINICANR".
+  The data provided by the OECD contain ISO 3166-1 alpha-3 codes, for instance "DOM".
+  In both cases, there are additional labels denoting country groupings; these are defined in the documentation linked above.
+
+  Changes visible in these lists include:
+
+  - 2022 → 2023:
+
+    - New codes: ASEAN, BFA, GREENLAND, MALI, MRT, PSE, TCD.
+    - Removed: MASEAN.
+
+  - 2021 → 2022:
+
+    - New codes: GNQ, MDG, MKD, RWA, SWZ, UGA.
+    - Removed: EQGUINEA, GREENLAND, MALI, MBURKINAFA, MCHAD, MMADAGASCA, MMAURITANI, MPALESTINE, MRWANDA, MUGANDA, NORTHMACED.
+
+- TIME: always a year.
+- MEASURE: unit of measurement, either "TJ" or "ktoe".
+
+:mod:`message_ix_models` is packaged with SDMX structure data (stored in :file:`message_ix_models/data/sdmx/`) comprising code lists extracted from the raw data for the COUNTRY, FLOW, and PRODUCT dimensions.
+These can be used with other package utilities, for instance:
+
+.. code-block:: python
+
+   >>> from message_ix_models.util.sdmx import read
+
+   # Read a code list from file: codes used in the
+   # 2022 edition data from the OECD provider
+   >>> cl = read("IEA:PRODUCT_OECD(2022)")
+
+   # Show some of its elements
+   >>> print("\n".join(sorted(cl.items[:5])))
+   ADDITIVE
+   ANTCOAL
+   AVGAS
+   BIODIESEL
+   BIOGASES
+
+The documentation linked above has full descriptions of each code.
+
+IEA provider/format
+~~~~~~~~~~~~~~~~~~~
+
+From 2023 (or earlier), the data are provided directly on the IEA website at https://www.iea.org/data-and-statistics/data-product/world-energy-balances.
+These data are available in two formats; ‘IVT’ or “Beyond 20/20” format (not supported by this module) or fixed-width text files.
+The latter are characterized by:
+
+- Multiple ZIP archives with names like :file:`WBIG[12].zip`, each containing a portion of the data and typically 110–130 MiB compressed size
+- …each containing a single, fixed-with TXT file with a name like :file:`WORLDBIG[12].TXT`, typically 3–4 GiB uncompressed,
+- …with no column headers, but data resembling::
+
+    WORLD  HARDCOAL  1960  INDPROD  KTOE ..
+
+  …that appear to correspond to, respectively, the COUNTRY, PRODUCT, TIME, FLOW, and MEASURE dimensions and "Value" column of the above data, respectively.
+
+OECD provider/format
+~~~~~~~~~~~~~~~~~~~~
+
+Up until 2023, the EWEB data were available from the OECD iLibrary with DOI `10.1787/enestats-data-en <https://doi.org/10.1787/enestats-data-en>`__.
+These files were characterized by:
+
+- Single ZIP archives with names like :file:`cac5fa90-en.zip`; typically ~850 MiB compressed size,
+- …containing a single CSV file with a name like :file:`WBIG_2022-2022-1-EN-20230406T100006.csv`, typically >20 GiB uncompressed,
+- …with a particular list of columns like: "MEASURE", "Unit", "COUNTRY", "Country", "PRODUCT", "Product", "FLOW", "Flow", "TIME", "Time", "Value", "Flag Codes", "Flags",
+- …with contents that duplicated code IDs—for instance, in the "FLOW" column—with human-readable labels—for instance in the "Flow" column:
+
+   ============ ===
+   Column name  Example value
+   ============ ===
+   MEASURE [1]_                   KTOE
+   Unit                           ktoe
+   COUNTRY                         WLD
+   Country                       World
+   PRODUCT                        COAL
+   Product      Coal and coal products
+   FLOW                        INDPROD
+   Flow                     Production
+   TIME                           2012
+   Time                           2012
+   Value                     1234.5678
+   Flag Codes                        M
+   Flags        Missing value; data cannot exist
+   ============ ===
+
+   .. [1] the column is sometimes labelled "UNIT", but the contents appear to be the same.
+
+This source is discontinued and will not publish subsequent editions of the data.

doc/api/model-snapshot.rst

@@ -10,7 +10,7 @@ Usage
 
 From the command line, download data for a single snapshot::
 
-    $ mix-models snapshot fetch 0
+    $ mix-models fetch snapshot-0
 
 …where :program:`0` is the ID of a snapshot; see :data:`.SNAPSHOTS`.
 
@@ -25,7 +25,7 @@ In code, use :func:`.snapshot.load`:
 
     snapshot.load(scenario, 0)
 
-.. note:: For snapshot 0, contrary to the `description of the Zenodo item <https://10.5281/zenodo.5793870>`__, the file cannot be loaded using :meth:`.Scenario.read_excel`.
+.. note:: For snapshot 0, contrary to the `description of the Zenodo item <https://doi.org/10.5281/zenodo.5793870>`__, the file cannot be loaded using :meth:`.Scenario.read_excel`.
    This limitation will be fixed in subsequent snapshots.
 
 Code reference

doc/api/project.rst

@@ -1,16 +1,73 @@
 .. currentmodule:: message_ix_models.project
 
-Specific research projects (:mod:`.project`)
-********************************************
+Specific research projects (:mod:`~message_ix_models.project`)
+**************************************************************
+
+.. contents::
+   :local:
+   :backlinks: none
 
 .. automodule:: message_ix_models.project
    :members:
 
+.. currentmodule:: message_ix_models.project.advance
+
+ADVANCE (:mod:`.project.advance`)
+=================================
+
+.. automodule:: message_ix_models.project.advance
+
+.. automodule:: message_ix_models.project.advance.data
+   :members:
+
+   Although free of charge, the ADVANCE data can not be downloaded automatically.
+   This source requires that users first submit personal information to register before being able to retrieve the data.
+   :mod:`message_ix_models` does not circumvent this requirement.
+   Thus:
+
+   - A copy of the data are stored in :mod:`message_data`.
+   - :mod:`message_ix_models` contains only a ‘fuzzed’ version of the data (same structure, random values) for testing purposes.
 
 .. currentmodule:: message_ix_models.project.ssp
 
 Shared Socioeconomic Pathways (:mod:`.project.ssp`)
 ===================================================
 
+Structure
+---------
+
+The enumerations :obj:`SSP_2017` and :obj:`SSP_2024` contain one member from the corresponding SDMX code lists.
+These can be used to uniquely identify both an SSP narrative *and* the set in which it occurs, in applications where this distinction is meaningful:
+
+.. code-block:: py
+
+   >>> from message_ix_models.project.ssp import SSP_2017, SSP_2024
+   >>> x = SSP_2017["2"]
+   >>> y = SSP_2024["2"]
+   >>> str(y)
+   "ICONICS:SSP(2024).2"
+   >>> x == y
+   False
+
 .. automodule:: message_ix_models.project.ssp
    :members:
+
+Data
+----
+
+.. automodule:: message_ix_models.project.ssp.data
+   :members:
+
+   Although free of charge, neither the 2017 or 2024 SSP data can be downloaded automatically.
+   Both sources require that users first submit personal information to register before being able to retrieve the data.
+   :mod:`message_ix_models` does not circumvent this requirement.
+   Thus:
+
+   - A copy of the data are stored in :mod:`message_data`.
+   - :mod:`message_ix_models` contains only a ‘fuzzed’ version of the data (same structure, random values) for testing purposes.
+
+   .. todo:: Allow users without access to :mod:`message_data` to read a local copy of this data from a :attr:`.Config.local_data` subdirectory.
+
+   .. autosummary::
+      SSPOriginal
+      SSPUpdate