diff --git a/climaf/driver.py b/climaf/driver.py index 2214aad2..7719927d 100644 --- a/climaf/driver.py +++ b/climaf/driver.py @@ -825,10 +825,10 @@ def ceval_script(scriptCall, deep, recurse_list=[]): # # Discard selection parameters if selection already occurred for first operand # TBD : manage the cases where other operands didn't undergo selection - if cache.hasExactObject(scriptCall.operands[0]) : + if cache.hasExactObject(scriptCall.operands[0]): # for key in ["period","period_iso","var","domain","missing","alias","units"]: for key in ["period", "period_iso", "var", "domain", "missing", "alias"]: - if key in subdict : + if key in subdict and not key in scriptCall.parameters: subdict.pop(key) # # print("subdict="+`subdict`) @@ -845,17 +845,20 @@ def ceval_script(scriptCall, deep, recurse_list=[]): subdict.pop('member_label') # # Substitute all args + clogger.debug("Final dictionary to fill template:\n%s" % str(subdict)) template = template.safe_substitute(subdict) + clogger.debug("Template obtained:\n%s" % template) # # Allowing for some formal parameters to be missing in the actual call: # # Discard remaining substrings looking like : # some_word='"${some_keyword}"' , or simply : '"${some_keyword}"' - template = re.sub(r'(\w*=)?(\'\")?\$\{\w*\}(\"\')?', r"", template) + template = re.sub(r'((--)?\w*=)?(\'\")?\$\{\w*\}(\"\')?', r"", template) # # Discard remaining substrings looking like : # some_word=${some_keyword} or simply : ${some_keyword} - template = re.sub(r"(\w*=)?\$\{\w*\}", r"", template) + template = re.sub(r"((--)?\w*=)?\$\{\w*\}", r"", template) + clogger.debug("Final template obtained:\n%s" % template) # # Link the fixed fields needed by the script/operator if script.fixedfields is not None: diff --git a/climaf/functions.py b/climaf/functions.py index 78acf1a9..e4424886 100644 --- a/climaf/functions.py +++ b/climaf/functions.py @@ -101,6 +101,37 @@ def fmul(dat1, dat2): return ccdo2(dat1, dat2, operator='mul') +def xfmul(*dats, **kwargs): + """ + Multiplication of several CliMAF objects using xarray. + :param dats: CliMAF objects to be multiplied + :param constant: Constant with which the field should be multiplied + :param mask: Mask to be used + :param mask_variable: The mask's variable + :return: the CliMAF object which contains the result of the operation. + """ + if len(dats) > 5: + raise ValueError("Could not multiply more than five datasets") + kwargs["var"] = dats[0].variable + is_cens = [isinstance(dat, climaf.classes.cens) for dat in dats] + if any(is_cens): + cens_index = is_cens.index(True) + cens_order = [dats[i].getattr("order") for i in cens_index] + if all([od == cens_order[0] for od in cens_order]): + res_dict = dict() + for (subdats) in zip(*dats): + res_dict[subdats[cens_index[0]]] = xmul(*subdats, **kwargs) + return cens(res_dict, order=cens_order[0]) + else: + cens_err = ["%s:%s" % (index, order) for (index, order) in enumerate(cens_index, cens_order)] + raise climaf.Climaf_Error("Your CliMAF ensembles do not have the same members: %s \n" + "use ensemble_intersection to get ensembles with only their common members" + % (" ; ".join(cens_err))) + else: + return xmul(*dats, **kwargs) + + + def fdiv(dat1, dat2): """ Division of two CliMAF object, or multiplication of the CliMAF object given as first argument diff --git a/climaf/standard_operators.py b/climaf/standard_operators.py index d587fed4..4f27e5a1 100644 --- a/climaf/standard_operators.py +++ b/climaf/standard_operators.py @@ -63,6 +63,11 @@ def load_standard_operators(): cscript('multiply', 'cdo mul ${in_1} ${in_2} ${out}', commuteWithTimeConcatenation=True, commuteWithSpaceConcatenation=True) # + cscript("xmul", scriptpath + "xmul.py ${in_1} ${in_2} ${in_3} ${in_4} ${in_5} --field_variable=${var} " + "--constant=${constant} --mask_file=${mask_file} --mask_variable=${mask_variable} " + "--output_file=${out}", + commuteWithTimeConcatenation=True, commuteWithSpaceConcatenation=True) + # cscript('divide', 'cdo div ${in_1} ${in_2} ${out}', commuteWithTimeConcatenation=True, commuteWithSpaceConcatenation=True) # diff --git a/scripts/xmul.py b/scripts/xmul.py new file mode 100755 index 00000000..88b1d5f0 --- /dev/null +++ b/scripts/xmul.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +""" +This script aims at plotting different fields on the same map. +""" + +from __future__ import division, print_function, unicode_literals, absolute_import + +import os +import shutil +import argparse +import xarray as xr +import numpy as np +import six + + +def check_none_or_other(value): + if value is None or value in ["", "none", "None"]: + return None + elif isinstance(value, six.string_types): + return value.strip() + else: + return str(value).strip() + + +parser = argparse.ArgumentParser() +parser.add_argument("input_files", nargs="+", help="Input files") +parser.add_argument("--field_variable", required=True, help="Variable to be considered in file") +parser.add_argument("--mask_file", type=check_none_or_other, help="Mask to be applied") +parser.add_argument("--mask_variable", default="mask", type=check_none_or_other, help="Variable of the mask file") +parser.add_argument("--constant", type=float, help="Constant with which the field should be multiplied") +parser.add_argument("--output_file", required=True, help="Output file") + +args = parser.parse_args() + +# The attributes of the resulting file are the ones of the first input file +# Copy it to output file +main_input_file = args.input_files[0] +if args.output_file != main_input_file: + if os.path.isfile(args.output_file): + os.remove(args.output_file) +else: + shutil.copy(main_input_file, main_input_file + ".tmp") + os.remove(main_input_file) + main_input_file += ".tmp" + +# Open the first file +with xr.open_dataset(main_input_file) as out_fic: + if args.field_variable not in out_fic.variables: + raise ValueError("Could not find variable %s in file %s" % (args.field_variable, args.input_files[0])) + data_var_main = out_fic.variables[args.field_variable].data + if len(args.input_files) > 1: + for f in args.input_files[1:]: + with xr.open_dataset(f) as in_fic: + if args.field_variable not in in_fic.variables: + raise ValueError("Could not find variable %s in file %s" % (args.field_variable, f)) + data_var_main *= in_fic.variables[args.field_variable].data + if args.mask_file is not None: + with xr.open_dataset(args.mask_file) as in_fic: + if args.mask_variable not in in_fic.variables: + raise ValueError("Could not find variable %s in file %s" % (args.field_variable, args.mask_file)) + data_var_main *= in_fic.variables[args.mask_variable].data + if args.constant is not None: + data_var_main *= args.constant + out_fic.variables[args.field_variable].data = data_var_main + out_fic.to_netcdf(args.output_file) + diff --git a/tests/test_functions.py b/tests/test_functions.py index a99e4e2b..28fc94d1 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -10,7 +10,7 @@ import os import unittest -from tests.tools_for_tests import remove_dir_and_content +from tests.tools_for_tests import remove_dir_and_content, compare_netcdf_files from climaf.cache import setNewUniqueCache from climaf.driver import cfile @@ -18,8 +18,13 @@ from climaf.functions import cscalar, apply_scale_offset, fmul, fdiv, fadd, fsub, iplot, getLevs, vertical_average, \ implot, diff_regrid, diff_regridn, tableau, annual_cycle, clim_average, clim_average_fast, summary, projects, \ lonlatvert_interpolation, zonmean_interpolation, zonmean, diff_zonmean, convert_list_to_string,\ - ts_plot, iplot_members + ts_plot, iplot_members, xfmul from env.environment import * +from env.clogging import clog + +from climaf import __path__ as cpath +if not isinstance(cpath, list): + cpath = cpath.split(os.sep) class CscalarTests(unittest.TestCase): @@ -45,6 +50,20 @@ def test_fmul(self): pass +class XfmulTests(unittest.TestCase): + + def setUp(self): + self.data_1 = ds(project='example', simulation="AMIPV6ALB2G", variable="ta", period="1980") + self.constant = 8. + self.reference_directory = os.sep.join(cpath + ["..", "tests", "reference_data", "test_functions"]) + + def test_xfmul_constant(self): + data = xfmul(self.data_1, constant=self.constant) + compare_netcdf_files(data, os.sep.join([self.reference_directory, "xfmul-1.nc"])) + data = xfmul(self.data_1, self.data_1) + compare_netcdf_files(data, os.sep.join([self.reference_directory, "xfmul-2.nc"])) + + class FdivTests(unittest.TestCase): @unittest.skipUnless(False, "Test not implemented") diff --git a/tests/tools_for_tests.py b/tests/tools_for_tests.py index 01814511..4455232a 100644 --- a/tests/tools_for_tests.py +++ b/tests/tools_for_tests.py @@ -105,18 +105,22 @@ def compare_text_files(file_test, file_ref, **kwargs): def compare_netcdf_files(file_test, file_ref, display=False): # Todo: Check the metadata of the files - if not os.path.exists(file_test) or not os.path.exists(file_ref): - raise OSError("Check files existence: %s - %s" % (file_test, file_ref)) + if not (isinstance(file_test, six.string_types) and os.path.isfile(file_test)): + fic_test = cfile(file_test) + else: + fic_test = file_test + if not os.path.exists(fic_test) or not os.path.exists(file_ref): + raise OSError("Check files existence: %s - %s" % (fic_test, file_ref)) if file_ref.split(".")[-1] != "nc": raise ValueError("This function only apply to netcdf files.") - if file_test.split(".")[-1] != file_ref.split(".")[-1]: - raise ValueError("Files have different formats: %s / %s" % (os.path.basename(file_test), + if fic_test.split(".")[-1] != file_ref.split(".")[-1]: + raise ValueError("Files have different formats: %s / %s" % (os.path.basename(fic_test), os.path.basename(file_ref))) if display: - ncview(file_test) - rep = subprocess.check_output("cdo diffn {} {}".format(file_test, file_ref), shell=True) + ncview(fic_test) + rep = subprocess.check_output("cdo diffn {} {}".format(fic_test, file_ref), shell=True) if len(str(rep).split("\n")) > 1: - raise ValueError("The content of files %s and %s are different" % (file_test, file_ref)) + raise ValueError("The content of files %s and %s are different" % (fic_test, file_ref)) def compare_picture_files(object_test, fic_ref, display=False, display_error=True):