Add function xfmul and script xmul

climaf/driver.py

@@ -825,10 +825,10 @@ def ceval_script(scriptCall, deep, recurse_list=[]):
     #
     # Discard selection parameters if selection already occurred for first operand
     # TBD : manage the cases where other operands didn't undergo selection
-    if cache.hasExactObject(scriptCall.operands[0]) :
+    if cache.hasExactObject(scriptCall.operands[0]):
         # for key in ["period","period_iso","var","domain","missing","alias","units"]:
         for key in ["period", "period_iso", "var", "domain", "missing", "alias"]:
-            if key in subdict :
+            if key in subdict and not key in scriptCall.parameters:
                 subdict.pop(key)
     #
     # print("subdict="+`subdict`)
@@ -845,17 +845,20 @@ def ceval_script(scriptCall, deep, recurse_list=[]):
             subdict.pop('member_label')
     #
     # Substitute all args
+    clogger.debug("Final dictionary to fill template:\n%s" % str(subdict))
     template = template.safe_substitute(subdict)
+    clogger.debug("Template obtained:\n%s" % template)
     #
     # Allowing for some formal parameters to be missing in the actual call:
     #
     # Discard remaining substrings looking like :
     #  some_word='"${some_keyword}"' , or simply : '"${some_keyword}"'
-    template = re.sub(r'(\w*=)?(\'\")?\$\{\w*\}(\"\')?', r"", template)
+    template = re.sub(r'((--)?\w*=)?(\'\")?\$\{\w*\}(\"\')?', r"", template)
     #
     # Discard remaining substrings looking like :
     #  some_word=${some_keyword}  or  simply : ${some_keyword}
-    template = re.sub(r"(\w*=)?\$\{\w*\}", r"", template)
+    template = re.sub(r"((--)?\w*=)?\$\{\w*\}", r"", template)
+    clogger.debug("Final template obtained:\n%s" % template)
     #
     # Link the fixed fields needed by the script/operator
     if script.fixedfields is not None:

climaf/functions.py

@@ -101,6 +101,37 @@ def fmul(dat1, dat2):
         return ccdo2(dat1, dat2, operator='mul')
 
 
+def xfmul(*dats, **kwargs):
+    """
+    Multiplication of several CliMAF objects using xarray.
+    :param dats: CliMAF objects to be multiplied
+    :param constant: Constant with which the field should be multiplied
+    :param mask: Mask to be used
+    :param mask_variable: The mask's variable
+    :return: the CliMAF object which contains the result of the operation.
+    """
+    if len(dats) > 5:
+        raise ValueError("Could not multiply more than five datasets")
+    kwargs["var"] = dats[0].variable
+    is_cens = [isinstance(dat, climaf.classes.cens) for dat in dats]
+    if any(is_cens):
+        cens_index = is_cens.index(True)
+        cens_order = [dats[i].getattr("order") for i in cens_index]
+        if all([od == cens_order[0] for od in cens_order]):
+            res_dict = dict()
+            for (subdats) in zip(*dats):
+                res_dict[subdats[cens_index[0]]] = xmul(*subdats, **kwargs)
+            return cens(res_dict, order=cens_order[0])
+        else:
+            cens_err = ["%s:%s" % (index, order) for (index, order) in enumerate(cens_index, cens_order)]
+            raise climaf.Climaf_Error("Your CliMAF ensembles do not have the same members: %s \n"
+                                      "use ensemble_intersection to get ensembles with only their common members"
+                                      % (" ; ".join(cens_err)))
+    else:
+        return xmul(*dats, **kwargs)
+
+
+
 def fdiv(dat1, dat2):
     """
     Division of two CliMAF object, or multiplication of the CliMAF object given as first argument

climaf/standard_operators.py

@@ -63,6 +63,11 @@ def load_standard_operators():
     cscript('multiply', 'cdo mul ${in_1} ${in_2} ${out}',
             commuteWithTimeConcatenation=True, commuteWithSpaceConcatenation=True)
     #
+    cscript("xmul", scriptpath + "xmul.py ${in_1} ${in_2} ${in_3} ${in_4} ${in_5} --field_variable=${var} "
+                                 "--constant=${constant} --mask_file=${mask_file} --mask_variable=${mask_variable} "
+                                 "--output_file=${out}",
+            commuteWithTimeConcatenation=True, commuteWithSpaceConcatenation=True)
+    #
     cscript('divide', 'cdo div ${in_1} ${in_2} ${out}',
             commuteWithTimeConcatenation=True, commuteWithSpaceConcatenation=True)
     #

scripts/xmul.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+"""
+This script aims at plotting different fields on the same map.
+"""
+
+from __future__ import division, print_function, unicode_literals, absolute_import
+
+import os
+import shutil
+import argparse
+import xarray as xr
+import numpy as np
+import six
+
+
+def check_none_or_other(value):
+    if value is None or value in ["", "none", "None"]:
+        return None
+    elif isinstance(value, six.string_types):
+        return value.strip()
+    else:
+        return str(value).strip()
+
+
+parser = argparse.ArgumentParser()
+parser.add_argument("input_files", nargs="+", help="Input files")
+parser.add_argument("--field_variable", required=True, help="Variable to be considered in file")
+parser.add_argument("--mask_file", type=check_none_or_other, help="Mask to be applied")
+parser.add_argument("--mask_variable", default="mask", type=check_none_or_other, help="Variable of the mask file")
+parser.add_argument("--constant", type=float, help="Constant with which the field should be multiplied")
+parser.add_argument("--output_file", required=True, help="Output file")
+
+args = parser.parse_args()
+
+# The attributes of the resulting file are the ones of the first input file
+# Copy it to output file
+main_input_file = args.input_files[0]
+if args.output_file != main_input_file:
+    if os.path.isfile(args.output_file):
+        os.remove(args.output_file)
+else:
+    shutil.copy(main_input_file, main_input_file + ".tmp")
+    os.remove(main_input_file)
+    main_input_file += ".tmp"
+
+# Open the first file
+with xr.open_dataset(main_input_file) as out_fic:
+    if args.field_variable not in out_fic.variables:
+        raise ValueError("Could not find variable %s in file %s" % (args.field_variable, args.input_files[0]))
+    data_var_main = out_fic.variables[args.field_variable].data
+    if len(args.input_files) > 1:
+        for f in args.input_files[1:]:
+            with xr.open_dataset(f) as in_fic:
+                if args.field_variable not in in_fic.variables:
+                    raise ValueError("Could not find variable %s in file %s" % (args.field_variable, f))
+                data_var_main *= in_fic.variables[args.field_variable].data
+    if args.mask_file is not None:
+        with xr.open_dataset(args.mask_file) as in_fic:
+            if args.mask_variable not in in_fic.variables:
+                raise ValueError("Could not find variable %s in file %s" % (args.field_variable, args.mask_file))
+            data_var_main *= in_fic.variables[args.mask_variable].data
+    if args.constant is not None:
+        data_var_main *= args.constant
+    out_fic.variables[args.field_variable].data = data_var_main
+    out_fic.to_netcdf(args.output_file)
+

tests/test_functions.py

@@ -10,16 +10,21 @@
 import os
 import unittest
 
-from tests.tools_for_tests import remove_dir_and_content
+from tests.tools_for_tests import remove_dir_and_content, compare_netcdf_files
 
 from climaf.cache import setNewUniqueCache
 from climaf.driver import cfile
 from climaf.classes import ds
 from climaf.functions import cscalar, apply_scale_offset, fmul, fdiv, fadd, fsub, iplot, getLevs, vertical_average, \
     implot, diff_regrid, diff_regridn, tableau, annual_cycle, clim_average, clim_average_fast, summary, projects, \
     lonlatvert_interpolation, zonmean_interpolation, zonmean, diff_zonmean, convert_list_to_string,\
-    ts_plot, iplot_members
+    ts_plot, iplot_members, xfmul
 from env.environment import *
+from env.clogging import clog
+
+from climaf import __path__ as cpath
+if not isinstance(cpath, list):
+    cpath = cpath.split(os.sep)
 
 
 class CscalarTests(unittest.TestCase):
@@ -45,6 +50,20 @@ def test_fmul(self):
         pass
 
 
+class XfmulTests(unittest.TestCase):
+
+    def setUp(self):
+        self.data_1 = ds(project='example', simulation="AMIPV6ALB2G", variable="ta", period="1980")
+        self.constant = 8.
+        self.reference_directory = os.sep.join(cpath + ["..", "tests", "reference_data", "test_functions"])
+
+    def test_xfmul_constant(self):
+        data = xfmul(self.data_1, constant=self.constant)
+        compare_netcdf_files(data, os.sep.join([self.reference_directory, "xfmul-1.nc"]))
+        data = xfmul(self.data_1, self.data_1)
+        compare_netcdf_files(data, os.sep.join([self.reference_directory, "xfmul-2.nc"]))
+
+
 class FdivTests(unittest.TestCase):
 
     @unittest.skipUnless(False, "Test not implemented")

tests/tools_for_tests.py

@@ -105,18 +105,22 @@ def compare_text_files(file_test, file_ref, **kwargs):
 
 def compare_netcdf_files(file_test, file_ref, display=False):
     # Todo: Check the metadata of the files
-    if not os.path.exists(file_test) or not os.path.exists(file_ref):
-        raise OSError("Check files existence: %s - %s" % (file_test, file_ref))
+    if not (isinstance(file_test, six.string_types) and os.path.isfile(file_test)):
+        fic_test = cfile(file_test)
+    else:
+        fic_test = file_test
+    if not os.path.exists(fic_test) or not os.path.exists(file_ref):
+        raise OSError("Check files existence: %s - %s" % (fic_test, file_ref))
     if file_ref.split(".")[-1] != "nc":
         raise ValueError("This function only apply to netcdf files.")
-    if file_test.split(".")[-1] != file_ref.split(".")[-1]:
-        raise ValueError("Files have different formats: %s / %s" % (os.path.basename(file_test),
+    if fic_test.split(".")[-1] != file_ref.split(".")[-1]:
+        raise ValueError("Files have different formats: %s / %s" % (os.path.basename(fic_test),
                                                                     os.path.basename(file_ref)))
     if display:
-        ncview(file_test)
-    rep = subprocess.check_output("cdo diffn {} {}".format(file_test, file_ref), shell=True)
+        ncview(fic_test)
+    rep = subprocess.check_output("cdo diffn {} {}".format(fic_test, file_ref), shell=True)
     if len(str(rep).split("\n")) > 1:
-        raise ValueError("The content of files %s and %s are different" % (file_test, file_ref))
+        raise ValueError("The content of files %s and %s are different" % (fic_test, file_ref))
 
 
 def compare_picture_files(object_test, fic_ref, display=False, display_error=True):