Skip to content

Commit

Permalink
Preserve uncertainty info when regionalizing datasets.
Browse files Browse the repository at this point in the history
Distinguish between preserving ecoinvent and imports uncertainty.
  • Loading branch information
romainsacchi authored and romainsacchi committed Aug 5, 2024
1 parent 5d9288e commit 16dc032
Show file tree
Hide file tree
Showing 8 changed files with 92 additions and 37 deletions.
Binary file modified premise/data/additional_inventories/lci-battery-capacity.xlsx
Binary file not shown.
2 changes: 1 addition & 1 deletion premise/energy.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ def import_heating_inventories(
version_out=self.version,
path=LCI_HEAT,
system_model=self.system_model,
keep_uncertainty_data=False,
keep_uncertainty_data=True,
)
datasets = inventory.merge_inventory()
self.database.extend(datasets)
Expand Down
5 changes: 1 addition & 4 deletions premise/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -961,7 +961,7 @@ def check_geographical_linking(scenario, original_database):


def prepare_db_for_export(
scenario, name, original_database, keep_uncertainty_data=False, biosphere_name=None
scenario, name, original_database, biosphere_name=None
):
"""
Prepare a database for export.
Expand All @@ -979,7 +979,6 @@ def prepare_db_for_export(
original_database=original_database,
database=scenario["database"],
db_name=name,
keep_uncertainty_data=keep_uncertainty_data,
biosphere_name=biosphere_name,
)
validator.run_all_checks()
Expand All @@ -991,15 +990,13 @@ def _prepare_database(
scenario,
db_name,
original_database,
keep_uncertainty_data,
biosphere_name,
):

scenario["database"] = prepare_db_for_export(
scenario,
name=db_name,
original_database=original_database,
keep_uncertainty_data=keep_uncertainty_data,
biosphere_name=biosphere_name,
)

Expand Down
2 changes: 1 addition & 1 deletion premise/inventory_imports.py
Original file line number Diff line number Diff line change
Expand Up @@ -876,7 +876,7 @@ def prepare_inventory(self) -> None:
# Remove uncertainty data
if not self.keep_uncertainty_data:
print("Remove uncertainty data.")
self.database = remove_uncertainty(self.database)
self.import_db.data = remove_uncertainty(self.import_db.data)
else:
check_uncertainty_data(self.import_db.data, filename=Path(self.path).stem)

Expand Down
21 changes: 8 additions & 13 deletions premise/new_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -514,7 +514,8 @@ def __init__(
use_cached_database: bool = True,
external_scenarios: list = None,
quiet=False,
keep_uncertainty_data=False,
keep_imports_uncertainty=False,
keep_source_db_uncertainty=False,
gains_scenario="CLE",
use_absolute_efficiency=False,
biosphere_name: str = "biosphere3",
Expand All @@ -525,7 +526,8 @@ def __init__(
self.system_model = check_system_model(system_model)
self.system_model_args = system_args
self.use_absolute_efficiency = use_absolute_efficiency
self.keep_uncertainty_data = keep_uncertainty_data
self.keep_imports_uncertainty = keep_imports_uncertainty
self.keep_source_db_uncertainty = keep_source_db_uncertainty
self.biosphere_name = check_presence_biosphere_database(biosphere_name)

# if version is anything other than 3.8 or 3.9
Expand Down Expand Up @@ -630,7 +632,7 @@ def __find_cached_db(self, db_name: str) -> List[dict]:
db_name = f"ecospold_{self.system_model}_{self.version}"

uncertainty_data = (
"w_uncertainty" if self.keep_uncertainty_data is True else "wo_uncertainty"
"w_uncertainty" if self.keep_source_db_uncertainty is True else "wo_uncertainty"
)

file_name = (
Expand Down Expand Up @@ -663,7 +665,7 @@ def __find_cached_inventories(self, db_name: str) -> Union[None, List[dict]]:
db_name = f"ecospold_{self.system_model}_{self.version}"

uncertainty_data = (
"w_uncertainty" if self.keep_uncertainty_data is True else "wo_uncertainty"
"w_uncertainty" if self.keep_imports_uncertainty is True else "wo_uncertainty"
)

file_name = (
Expand Down Expand Up @@ -696,7 +698,7 @@ def __clean_database(self) -> List[dict]:
"""
return DatabaseCleaner(
self.source, self.source_type, self.source_file_path, self.version
).prepare_datasets(self.keep_uncertainty_data)
).prepare_datasets(self.keep_source_db_uncertainty)

def __import_inventories(self) -> List[dict]:
"""
Expand Down Expand Up @@ -808,7 +810,7 @@ def __import_inventories(self) -> List[dict]:
version_out=self.version,
path=filepath[0],
system_model=self.system_model,
keep_uncertainty_data=self.keep_uncertainty_data,
keep_uncertainty_data=self.keep_imports_uncertainty,
)
datasets = inventory.merge_inventory()
data.extend(datasets)
Expand Down Expand Up @@ -1009,7 +1011,6 @@ def write_superstructure_db_to_brightway(
scenario=scenario,
db_name=name,
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)

Expand All @@ -1033,7 +1034,6 @@ def write_superstructure_db_to_brightway(
scenario=tmp_scenario,
name="database",
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)

Expand Down Expand Up @@ -1101,7 +1101,6 @@ def write_db_to_brightway(self, name: [str, List[str]] = None):
scenario=scenario,
db_name=name[s],
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)
write_brightway_database(
Expand Down Expand Up @@ -1176,7 +1175,6 @@ def scenario_name(scenario):
scenario=scenario,
db_name="database",
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)
Export(scenario, filepath[s], self.version).export_db_to_matrices()
Expand Down Expand Up @@ -1217,7 +1215,6 @@ def write_db_to_simapro(self, filepath: str = None):
scenario=scenario,
db_name="database",
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)
export = Export(scenario, filepath, self.version)
Expand Down Expand Up @@ -1264,7 +1261,6 @@ def write_db_to_olca(self, filepath: str = None):
scenario=scenario,
db_name="database",
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)
Export(scenario, filepath, self.version).export_db_to_simapro(
Expand Down Expand Up @@ -1305,7 +1301,6 @@ def write_datapackage(
scenario=scenario,
db_name=name,
original_database=self.database,
keep_uncertainty_data=self.keep_uncertainty_data,
biosphere_name=self.biosphere_name,
)

Expand Down
72 changes: 69 additions & 3 deletions premise/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -1057,6 +1057,19 @@ def relink_datasets(self, excludes_datasets=None, alt_names=None):
if len(excs_to_relink) == 0:
continue

old_uncertainty = {}

for exc in excs_to_relink:
if exc["type"] == "technosphere":
if exc.get("uncertainty type", 0) != 0:
old_uncertainty[(exc["name"], exc.get("product"), exc["unit"])] = {
"uncertainty type": exc.get("uncertainty type", 0),
"loc": exc.get("loc", 0) / exc["amount"] if exc.get("loc") else None,
"scale": exc.get("scale", 0) / exc["amount"] if exc.get("scale") else None,
"minimum": exc.get("minimum", 0) / exc["amount"] if exc.get("minimum") else None,
"maximum": exc.get("maximum", 0) / exc["amount"] if exc.get("maximum") else None,
}

# make a dictionary with the names and amounts
# of the technosphere exchanges to relink
# to compare with the new exchanges
Expand Down Expand Up @@ -1085,6 +1098,17 @@ def relink_datasets(self, excludes_datasets=None, alt_names=None):
act, unique_excs_to_relink, alt_names
)

# apply uncertainties, if any
if old_uncertainty:
for exc in new_exchanges:
key = (exc["name"], exc["product"], exc["unit"])
if key in old_uncertainty:
exc["uncertainty type"] = old_uncertainty[key]["uncertainty type"]
for k, v in old_uncertainty[key].items():
if k != "uncertainty type":
if v is not None:
exc[k] = v * exc["amount"]

# Update act["exchanges"] by removing the exchanges to relink
act["exchanges"] = [e for e in act["exchanges"] if e not in excs_to_relink]
# Update act["exchanges"] by adding new exchanges
Expand Down Expand Up @@ -1574,6 +1598,12 @@ def process_cached_exchange(
"location": i[2],
"type": "technosphere",
"amount": exchange["amount"] * i[-1],
"uncertainty type": exchange.get("uncertainty type", 0),
"loc": exchange.get("loc", 0) * i[-1],
"scale": exchange.get("scale", 0) * i[-1],
"negative": exchange.get("negative", False),
"minimum": exchange.get("minimum", 0) * i[-1],
"maximum": exchange.get("maximum", 0) * i[-1],
}
for i in exchanges
]
Expand Down Expand Up @@ -1634,6 +1664,12 @@ def process_uncached_exchange(
"location": dataset["location"],
"type": "technosphere",
"amount": exchange["amount"],
"uncertainty type": exchange.get("uncertainty type", 0),
"loc": exchange.get("loc", None),
"scale": exchange.get("scale", None),
"negative": exchange.get("negative", False),
"minimum": exchange.get("minimum", None),
"maximum": exchange.get("maximum", None),
}
]

Expand Down Expand Up @@ -1679,6 +1715,12 @@ def new_exchange(self, exchange, location, amount_multiplier):
"location": location,
"type": "technosphere",
"amount": exchange["amount"] * amount_multiplier,
"uncertainty type": exchange.get("uncertainty type", 0),
"loc": exchange.get("loc", None),
"scale": exchange.get("scale", None),
"negative": exchange.get("negative", False),
"minimum": exchange.get("minimum", None),
"maximum": exchange.get("maximum", None),
}

def handle_multiple_possible_datasets(
Expand Down Expand Up @@ -1923,6 +1965,19 @@ def relink_technosphere_exchanges(
if exc["type"] == "technosphere":
exchanges_before[exc["product"]] += exc["amount"]

old_uncertainty = {}

for exc in dataset["exchanges"]:
if exc["type"] == "technosphere":
if exc.get("uncertainty type", 0) != 0:
old_uncertainty[(exc["name"], exc.get("product"), exc["unit"])] = {
"uncertainty type": exc.get("uncertainty type", 0),
"loc": exc.get("loc", 0) / exc["amount"] if exc.get("loc") else None,
"scale": exc.get("scale", 0) / exc["amount"] if exc.get("scale") else None,
"minimum": exc.get("minimum", 0) / exc["amount"] if exc.get("minimum") else None,
"maximum": exc.get("maximum", 0) / exc["amount"] if exc.get("maximum") else None,
}

new_exchanges = self.find_candidates(
dataset,
exclusive=exclusive,
Expand All @@ -1943,14 +1998,25 @@ def relink_technosphere_exchanges(
"type": "technosphere",
"amount": sum(exc["amount"] for exc in exchanges),
}
for (name, prod, location, unit), exchanges in groupby(
for (name, prod, location, unit, ), exchanges in groupby(
sorted(
new_exchanges, key=itemgetter("name", "product", "location", "unit")
new_exchanges, key=itemgetter("name", "product", "location", "unit",)
),
key=itemgetter("name", "product", "location", "unit"),
key=itemgetter("name", "product", "location", "unit",),
)
]

# apply uncertainties, if any
if old_uncertainty:
for exc in new_exchanges:
key = (exc["name"], exc["product"], exc["unit"])
if key in old_uncertainty:
exc["uncertainty type"] = old_uncertainty[key]["uncertainty type"]
for k, v in old_uncertainty[key].items():
if k != "uncertainty type":
if v is not None:
exc[k] = v * exc["amount"]

dataset["exchanges"] = [
exc for exc in dataset["exchanges"] if exc["type"] != "technosphere"
] + new_exchanges
Expand Down
2 changes: 1 addition & 1 deletion premise/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ def hide_messages():
"""

print("Keep uncertainty data?")
print("NewDatabase(..., keep_uncertainty_data=True)")
print("NewDatabase(..., keep_source_db_uncertainty=True), keep_imports_uncertainty=True)")
print("")
print("Hide these messages?")
print("NewDatabase(..., quiet=True)")
Expand Down
25 changes: 11 additions & 14 deletions premise/validation.py
Original file line number Diff line number Diff line change
Expand Up @@ -151,7 +151,6 @@ def __init__(
database,
original_database=None,
db_name=None,
keep_uncertainty_data=False,
biosphere_name=None,
):
self.original_database = original_database
Expand All @@ -164,7 +163,6 @@ def __init__(
self.geo = Geomap(model)
self.minor_issues_log = []
self.major_issues_log = []
self.keep_uncertainty_data = keep_uncertainty_data
self.biosphere_name = biosphere_name

def check_matrix_squareness(self):
Expand Down Expand Up @@ -202,18 +200,17 @@ def check_uncertainty(self):
12: {"loc", "scale", "shape"},
}

if self.keep_uncertainty_data is True:
for ds in self.database:
for exc in ds["exchanges"]:
if int(exc.get("uncertainty type", 0)) not in [0, 1]:
if not all(
f in exc
for f in MANDATORY_UNCERTAINTY_FIELDS[
int(exc["uncertainty type"])
]
):
message = f"Exchange {exc['name']} has incomplete uncertainty data."
self.log_issue(ds, "incomplete uncertainty data", message)
for ds in self.database:
for exc in ds["exchanges"]:
if int(exc.get("uncertainty type", 0)) not in [0, 1]:
if not all(
f in exc
for f in MANDATORY_UNCERTAINTY_FIELDS[
int(exc["uncertainty type"])
]
):
message = f"Exchange {exc['name']} has incomplete uncertainty data."
self.log_issue(ds, "incomplete uncertainty data", message)

def check_datasets_integrity(self):
# Verify no unintended loss of datasets
Expand Down

0 comments on commit 16dc032

Please sign in to comment.