apply configure_logging more comprehensively (#1508)

* apply configure_logging more comprehensively

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* make module level imports at top

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

scripts/build_ammonia_production.py

@@ -21,9 +21,13 @@
  (July 2024) published by the US Geological Survey (USGS) and the National Minerals Information Center and extracts the annual ammonia production per country in ktonN/a. The data is converted to ktonNH3/a.
 """
 
+import logging
+
 import country_converter as coco
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 cc = coco.CountryConverter()
 
@@ -34,6 +38,7 @@
 
         snakemake = mock_snakemake("build_ammonia_production")
 
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     ammonia = pd.read_excel(

scripts/build_biomass_transport_costs.py

@@ -16,7 +16,12 @@
 @author: bw0928
 """
 
+import logging
+
 import pandas as pd
+from _helpers import configure_logging
+
+logger = logging.getLogger(__name__)
 
 ENERGY_CONTENT = 4.8  # unit MWh/t (wood pellets)
 
@@ -117,5 +122,6 @@ def build_biomass_transport_costs():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_biomass_transport_costs")
+    configure_logging(snakemake)
 
     build_biomass_transport_costs()

scripts/build_clustered_population_layouts.py

@@ -6,18 +6,23 @@
 split by urban and rural population.
 """
 
+import logging
+
 import atlite
 import geopandas as gpd
 import pandas as pd
 import xarray as xr
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_clustered_population_layouts", clusters=48)
 
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     cutout = atlite.Cutout(snakemake.input.cutout)

scripts/build_daily_heat_demand.py

@@ -43,13 +43,17 @@
         default_cutout``:
 """
 
+import logging
+
 import atlite
 import geopandas as gpd
 import numpy as np
 import xarray as xr
-from _helpers import get_snapshots, set_scenario_config
+from _helpers import configure_logging, get_snapshots, set_scenario_config
 from dask.distributed import Client, LocalCluster
 
+logger = logging.getLogger(__name__)
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
@@ -59,6 +63,7 @@
             scope="total",
             clusters=48,
         )
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     nprocesses = int(snakemake.threads)

scripts/build_direct_heat_source_utilisation_profiles.py

@@ -24,8 +24,12 @@
 - `resources/<run_name>/direct_heat_source_utilisation_profiles_base_s_{clusters}_{planning_horizons}.nc`: Direct heat source utilisation profiles
 """
 
+import logging
+
 import xarray as xr
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 
 def get_source_temperature(heat_source_key: str):
@@ -89,7 +93,7 @@ def get_profile(
             "build_cop_profiles",
             clusters=48,
         )
-
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     direct_utilisation_heat_sources: list[str] = (

scripts/build_egs_potentials.py

@@ -22,6 +22,7 @@
 import numpy as np
 import pandas as pd
 import xarray as xr
+from _helpers import configure_logging, set_scenario_config
 from shapely.geometry import Polygon
 
 logger = logging.getLogger(__name__)
@@ -199,6 +200,9 @@ def get_capacity_factors(network_regions_file, air_temperatures_file):
             clusters=37,
         )
 
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
+
     egs_config = snakemake.params["sector"]["enhanced_geothermal"]
     costs_config = snakemake.params["costs"]
 

scripts/build_existing_heating_distribution.py

@@ -38,10 +38,14 @@
 - "Mapping and analyses of the current and future (2020 - 2030) heating/cooling fuel deployment (fossil/renewables)" (https://energy.ec.europa.eu/publications/mapping-and-analyses-current-and-future-2020-2030-heatingcooling-fuel-deployment-fossilrenewables-1_en)
 """
 
+import logging
+
 import country_converter as coco
 import numpy as np
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 cc = coco.CountryConverter()
 
@@ -171,6 +175,7 @@ def build_existing_heating():
             clusters=48,
             planning_horizons=2050,
         )
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     build_existing_heating()

scripts/build_hac_features.py

@@ -9,7 +9,7 @@
 
 import atlite
 import geopandas as gpd
-from _helpers import get_snapshots, set_scenario_config
+from _helpers import configure_logging, get_snapshots, set_scenario_config
 from atlite.aggregate import aggregate_matrix
 from dask.distributed import Client
 
@@ -20,6 +20,7 @@
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_hac_features")
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     params = snakemake.params

scripts/build_heat_totals.py

@@ -16,11 +16,15 @@
 - `resources/<run_name>/heat_totals.csv`: Approximated annual heat demand for each country.
 """
 
+import logging
 from itertools import product
 
 import pandas as pd
+from _helpers import configure_logging
 from numpy.polynomial import Polynomial
 
+logger = logging.getLogger(__name__)
+
 idx = pd.IndexSlice
 
 
@@ -104,6 +108,8 @@ def approximate_heat_demand(energy_totals: pd.DataFrame, hdd: pd.DataFrame):
 
         snakemake = mock_snakemake("build_heat_totals")
 
+    configure_logging(snakemake)
+
     hdd = pd.read_csv(snakemake.input.hdd, index_col=0).T
     hdd.index = hdd.index.astype(int)
 

scripts/build_hourly_heat_demand.py

@@ -29,11 +29,19 @@
 - ``resources/hourly_heat_demand_total_base_s<simpl>_<clusters>.nc``:
 """
 
+import logging
 from itertools import product
 
 import pandas as pd
 import xarray as xr
-from _helpers import generate_periodic_profiles, get_snapshots, set_scenario_config
+from _helpers import (
+    configure_logging,
+    generate_periodic_profiles,
+    get_snapshots,
+    set_scenario_config,
+)
+
+logger = logging.getLogger(__name__)
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -44,6 +52,7 @@
             scope="total",
             clusters=5,
         )
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     snapshots = get_snapshots(

scripts/build_industrial_energy_demand_per_country_today.py

@@ -59,14 +59,17 @@
 - waste
 """
 
+import logging
 import multiprocessing as mp
 from functools import partial
 
 import country_converter as coco
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
 from tqdm import tqdm
 
+logger = logging.getLogger(__name__)
+
 cc = coco.CountryConverter()
 
 ktoe_to_twh = 0.011630
@@ -287,6 +290,7 @@ def add_coke_ovens(demand, fn, year, factor=0.75):
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_industrial_energy_demand_per_country_today")
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     params = snakemake.params.industry

scripts/build_industrial_energy_demand_per_node.py

@@ -35,8 +35,12 @@
 which can later be used as values for the industry load.
 """
 
+import logging
+
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 if __name__ == "__main__":
     if "snakemake" not in globals():
@@ -47,6 +51,7 @@
             clusters=48,
             planning_horizons=2030,
         )
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     # import ratios

scripts/build_industrial_energy_demand_per_node_today.py

@@ -24,11 +24,14 @@
 The unit of the energy demand is TWh/a.
 """
 
+import logging
 from itertools import product
 
 import numpy as np
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 # map JRC/our sectors to hotmaps sector, where mapping exist
 sector_mapping = {
@@ -93,6 +96,7 @@ def build_nodal_industrial_energy_demand():
             "build_industrial_energy_demand_per_node_today",
             clusters=48,
         )
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     build_nodal_industrial_energy_demand()

scripts/build_industrial_production_per_country_tomorrow.py

@@ -58,15 +58,20 @@
 The unit of the production is kt/a.
 """
 
+import logging
+
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
 from prepare_sector_network import get
 
+logger = logging.getLogger(__name__)
+
 if __name__ == "__main__":
     if "snakemake" not in globals():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_industrial_production_per_country_tomorrow")
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     params = snakemake.params.industry

scripts/build_industrial_production_per_node.py

@@ -24,10 +24,13 @@
 The unit of the production is kt/a.
 """
 
+import logging
 from itertools import product
 
 import pandas as pd
-from _helpers import set_scenario_config
+from _helpers import configure_logging, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 # map JRC/our sectors to hotmaps sector, where mapping exist
 sector_mapping = {
@@ -87,6 +90,7 @@ def build_nodal_industrial_production():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_industrial_production_per_node", clusters=48)
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     build_nodal_industrial_production()

scripts/build_industry_sector_ratios.py

@@ -47,9 +47,13 @@
 The unit of the specific energy consumption is MWh/t material and tCO2/t material for process emissions.
 """
 
+import logging
+
 import country_converter as coco
 import pandas as pd
-from _helpers import mute_print, set_scenario_config
+from _helpers import configure_logging, mute_print, set_scenario_config
+
+logger = logging.getLogger(__name__)
 
 cc = coco.CountryConverter()
 
@@ -1530,6 +1534,7 @@ def other_industrial_sectors():
         from _helpers import mock_snakemake
 
         snakemake = mock_snakemake("build_industry_sector_ratios")
+    configure_logging(snakemake)
     set_scenario_config(snakemake)
 
     params = snakemake.params.industry

scripts/build_industry_sector_ratios_intermediate.py

@@ -78,10 +78,15 @@
 Unit of the output file is MWh/t.
 """
 
+import logging
+
 import numpy as np
 import pandas as pd
+from _helpers import configure_logging, set_scenario_config
 from prepare_sector_network import get
 
+logger = logging.getLogger(__name__)
+
 
 def build_industry_sector_ratios_intermediate():
     # in TWh/a
@@ -147,6 +152,8 @@ def build_industry_sector_ratios_intermediate():
             "build_industry_sector_ratios_intermediate",
             planning_horizons="2030",
         )
+    configure_logging(snakemake)
+    set_scenario_config(snakemake)
 
     year = int(snakemake.wildcards.planning_horizons)