From b281b9d5b68b1bbbd3ce4e54dde9edab4023b0a3 Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Sat, 1 Aug 2020 01:15:12 +0200 Subject: [PATCH 01/11] include evolutionary-keras --- conda-recipe/meta.yaml | 1 + n3fit/src/n3fit/backends/keras_backend/MetaModel.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 87be5c5633..7ca100c2c1 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -45,6 +45,7 @@ requirements: - sphinx # documentation - recommonmark - sphinx_rtd_theme + - evolutionary_keras test: requires: diff --git a/n3fit/src/n3fit/backends/keras_backend/MetaModel.py b/n3fit/src/n3fit/backends/keras_backend/MetaModel.py index f9e80a5834..4363f51dd5 100644 --- a/n3fit/src/n3fit/backends/keras_backend/MetaModel.py +++ b/n3fit/src/n3fit/backends/keras_backend/MetaModel.py @@ -7,9 +7,11 @@ import numpy as np import tensorflow as tf -from tensorflow.keras.models import Model from tensorflow.keras import optimizers as Kopt from n3fit.backends.keras_backend.operations import numpy_to_tensor +import evolutionary_keras.optimizers as Evolutionary_optimizers +from evolutionary_keras.models import EvolModel + # Check the TF version to check if legacy-mode is needed (TF < 2.2) tf_version = tf.__version__.split('.') @@ -28,6 +30,8 @@ "Adamax": (Kopt.Adamax, {}), "Nadam": (Kopt.Nadam, {}), "Amsgrad": (Kopt.Adam, {"learning_rate": 0.01, "amsgrad": True}), + "NGA": (Evolutionary_optimizers.NGA, {}), + "CMA": (Evolutionary_optimizers.CMA, {}) } # Some keys need to work for everyone @@ -62,7 +66,7 @@ def _fill_placeholders(original_input, new_input=None): return x -class MetaModel(Model): +class MetaModel(EvolModel): """ The `MetaModel` behaves as the tensorflow.keras.model.Model class, with the addition of `tensor_content`: From 409cf1468ad9c02eb99cc71b2745438184e7ac6e Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Thu, 20 Aug 2020 14:24:12 +0200 Subject: [PATCH 02/11] require evolutionary-keras=>2.0 --- conda-recipe/meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 7ca100c2c1..9011f91944 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -45,7 +45,7 @@ requirements: - sphinx # documentation - recommonmark - sphinx_rtd_theme - - evolutionary_keras + - evolutionary_keras >=2.0 test: requires: From 850dcb530d88d29e234ab2e4a336ff506393a5ea Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 10 Sep 2020 18:46:21 +0200 Subject: [PATCH 03/11] ignore the existence of other backends by default --- conda-recipe/meta.yaml | 1 - n3fit/runcards/Basic_nga.yml | 141 +++++++ n3fit/setup.py | 5 + n3fit/src/n3fit/backends/__init__.py | 25 +- .../n3fit/backends/ga_backend/MetaModel.py | 368 ++++++++++++++++++ .../src/n3fit/backends/ga_backend/__init__.py | 0 .../n3fit/backends/keras_backend/MetaModel.py | 7 +- n3fit/src/n3fit/checks.py | 16 +- n3fit/src/n3fit/performfit.py | 3 +- 9 files changed, 556 insertions(+), 10 deletions(-) create mode 100644 n3fit/runcards/Basic_nga.yml create mode 100644 n3fit/src/n3fit/backends/ga_backend/MetaModel.py create mode 100644 n3fit/src/n3fit/backends/ga_backend/__init__.py diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 9011f91944..87be5c5633 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -45,7 +45,6 @@ requirements: - sphinx # documentation - recommonmark - sphinx_rtd_theme - - evolutionary_keras >=2.0 test: requires: diff --git a/n3fit/runcards/Basic_nga.yml b/n3fit/runcards/Basic_nga.yml new file mode 100644 index 0000000000..4fd5fc00a5 --- /dev/null +++ b/n3fit/runcards/Basic_nga.yml @@ -0,0 +1,141 @@ +# +# Configuration file for n3fit +# + +############################################################ +description: Basic runcard + +############################################################ +# frac: training fraction +# ewk: apply ewk k-factors +# sys: systematics treatment (see systypes) +experiments: + - experiment: ALL + datasets: + - { dataset: SLACP, frac: 0.5} + - { dataset: NMCPD, frac: 0.5 } + - { dataset: CMSJETS11, frac: 0.5, sys: 10 } + +############################################################ +datacuts: + t0pdfset : NNPDF31_nlo_as_0118 # PDF set to generate t0 covmat + q2min : 3.49 # Q2 minimum + w2min : 12.5 # W2 minimum + combocuts : NNPDF31 # NNPDF3.0 final kin. cuts + jetptcut_tev : 0 # jet pt cut for tevatron + jetptcut_lhc : 0 # jet pt cut for lhc + wptcut_lhc : 30.0 # Minimum pT for W pT diff distributions + jetycut_tev : 1e30 # jet rap. cut for tevatron + jetycut_lhc : 1e30 # jet rap. cut for lhc + dymasscut_min: 0 # dy inv.mass. min cut + dymasscut_max: 1e30 # dy inv.mass. max cut + jetcfactcut : 1e30 # jet cfact. cut + +############################################################ +theory: + theoryid: 53 # database id + +############################################################ +fitting: + trvlseed: 1 + nnseed: 2 + mcseed: 3 + epochs: 900 + save: False + savefile: 'weights.hd5' + load: False + loadfile: 'weights.hd5' + plot: False + + seed : 9453862133528 # set the seed for the random generator + genrep : True # on = generate MC replicas, off = use real data + rngalgo : 0 # 0 = ranlux, 1 = cmrg, see randomgenerator.cc + + backend: 'evolutionary_keras' + + parameters: # This defines the parameter dictionary that is passed to the Model Trainer + nodes_per_layer: [15, 10, 8] + activation_per_layer: ['sigmoid', 'sigmoid', 'linear'] + initializer: 'glorot_normal' + optimizer: + optimizer_name: 'RMSprop' + learning_rate: 0.01 + clipnorm: 1.0 + epochs: 900 + positivity: + multiplier: 1.05 # When any of the multiplier and/or the initial is not set + initial: # the poslambda will be used instead to compute these values per dataset + stopping_patience: 0.30 # percentage of the number of epochs + layer_type: 'dense' + dropout: 0.0 + threshold_chi2: 5.0 + + # NN23(QED) = sng=0,g=1,v=2,t3=3,ds=4,sp=5,sm=6,(pht=7) + # EVOL(QED) = sng=0,g=1,v=2,v3=3,v8=4,t3=5,t8=6,(pht=7) + # EVOLS(QED)= sng=0,g=1,v=2,v8=4,t3=4,t8=5,ds=6,(pht=7) + # FLVR(QED) = g=0, u=1, ubar=2, d=3, dbar=4, s=5, sbar=6, (pht=7) + fitbasis: NN31IC # EVOL (7), EVOLQED (8), etc. + basis: + # remeber to change the name of PDF accordingly with fitbasis + # pos: True for NN squared + # mutsize: mutation size + # mutprob: mutation probability + # smallx, largex: preprocessing ranges + - { fl: sng, pos: False, mutsize: [15], mutprob: [0.05], smallx: [1.05,1.19], largex: [1.47,2.70], trainable: False } + - { fl: g, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.94,1.25], largex: [0.11,5.87], trainable: False } + - { fl: v, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.54,0.75], largex: [1.15,2.76], trainable: False } + - { fl: v3, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.21,0.57], largex: [1.35,3.08] } + - { fl: v8, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.52,0.76], largex: [0.77,3.56], trainable: True } + - { fl: t3, pos: False, mutsize: [15], mutprob: [0.05], smallx: [-0.37,1.52], largex: [1.74,3.39] } + - { fl: t8, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.56,1.29], largex: [1.45,3.03] } + - { fl: cp, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.12,1.19], largex: [1.83,6.70] } + +############################################################ +stopping: + stopmethod: LOOKBACK # Stopping method + lbdelta : 0 # Delta for look-back stopping + mingen : 0 # Minimum number of generations + window : 500 # Window for moving average + minchi2 : 3.5 # Minimum chi2 + minchi2exp: 6.0 # Minimum chi2 for experiments + nsmear : 200 # Smear for stopping + deltasm : 200 # Delta smear for stopping + rv : 2 # Ratio for validation stopping + rt : 0.5 # Ratio for training stopping + epsilon : 1e-6 # Gradient epsilon + +############################################################ +positivity: + posdatasets: + - { dataset: POSF2U, poslambda: 1e6 } # Positivity Lagrange Multiplier + - { dataset: POSFLL, poslambda: 1e4 } + +############################################################ +integrability: + integdatasets: + - {dataset: INTEGXT3, poslambda: 1e2} + +############################################################ +closuretest: + filterseed : 0 # Random seed to be used in filtering data partitions + fakedata : False # on = to use FAKEPDF to generate pseudo-data + fakepdf : MSTW2008nlo68cl # Theory input for pseudo-data + errorsize : 1.0 # uncertainties rescaling + fakenoise : False # on = to add random fluctuations to pseudo-data + rancutprob : 1.0 # Fraction of data to be included in the fit + rancutmethod: 0 # Method to select rancutprob data fraction + rancuttrnval: False # 0(1) to output training(valiation) chi2 in report + printpdf4gen: False # To print info on PDFs during minimization + +############################################################ +lhagrid: + nx : 150 + xmin: 1e-9 + xmed: 0.1 + xmax: 1.0 + nq : 50 + qmax: 1e5 + +############################################################ +debug: true +maxcores: 8 diff --git a/n3fit/setup.py b/n3fit/setup.py index 4ffeab3d95..fc66cd74ec 100644 --- a/n3fit/setup.py +++ b/n3fit/setup.py @@ -10,6 +10,11 @@ '':['*.fitinfo', '*.yml'], 'tests/regressions': ['*'], }, + extras_require={ + 'ga' : [ + 'evolutionary-keras', + ], + }, entry_points = {'console_scripts': ['n3fit = n3fit.scripts.n3fit_exec:main', diff --git a/n3fit/src/n3fit/backends/__init__.py b/n3fit/src/n3fit/backends/__init__.py index 3370ae75a8..cda6e198b9 100644 --- a/n3fit/src/n3fit/backends/__init__.py +++ b/n3fit/src/n3fit/backends/__init__.py @@ -3,7 +3,7 @@ clear_backend_state, ) from n3fit.backends.keras_backend.MetaLayer import MetaLayer -from n3fit.backends.keras_backend.MetaModel import MetaModel + from n3fit.backends.keras_backend.base_layers import ( Input, concatenate, @@ -17,4 +17,25 @@ from n3fit.backends.keras_backend import constraints from n3fit.backends.keras_backend import callbacks -print("Using Keras backend") +# Don't import the Model until it needs to be imported +class _MetaModel: + def __init__(self, backend="tensorflow"): + self.backend = backend + from n3fit.backends.keras_backend.MetaModel import MetaModel + + self.meta_model = MetaModel + + def enable_ga(self): + try: + from n3fit.backends.ga_backend.MetaModel import MetaModel + + self.meta_model = MetaModel + self.backend = "evolutionary_keras" + except ModuleNotFoundError: + raise ModuleNotFoundError("Install `evolutionary_keras` to use this backend") + + def __call__(self, *args, **kwargs): + return self.meta_model(*args, **kwargs) + + +MetaModel = _MetaModel() diff --git a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py new file mode 100644 index 0000000000..466d2110b2 --- /dev/null +++ b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py @@ -0,0 +1,368 @@ +""" + MetaModel class + + Extension of the backend Model class containing some wrappers in order to absorb other + backend-dependent calls. +""" + +import tensorflow as tf +from tensorflow.keras import optimizers as Kopt +from n3fit.backends.keras_backend.operations import numpy_to_tensor +import evolutionary_keras.optimizers as Evolutionary_optimizers +from evolutionary_keras.models import EvolModel + + +# Check the TF version to check if legacy-mode is needed (TF < 2.2) +tf_version = tf.__version__.split('.') +if int(tf_version[0]) == 2 and int(tf_version[1]) < 2: + LEGACY = True +else: + LEGACY = False + +# Define in this dictionary new optimizers as well as the arguments they accept +# (with default values if needed be) +optimizers = { + "RMSprop": (Kopt.RMSprop, {"learning_rate": 0.01}), + "Adam": (Kopt.Adam, {"learning_rate": 0.01}), + "Adagrad": (Kopt.Adagrad, {}), + "Adadelta": (Kopt.Adadelta, {"learning_rate": 1.0}), + "Adamax": (Kopt.Adamax, {}), + "Nadam": (Kopt.Nadam, {}), + "Amsgrad": (Kopt.Adam, {"learning_rate": 0.01, "amsgrad": True}), + "NGA": (Evolutionary_optimizers.NGA, {}), + "CMA": (Evolutionary_optimizers.CMA, {}) +} + +# Some keys need to work for everyone +for k, v in optimizers.items(): + v[1]["clipnorm"] = 1.0 + + +def _fill_placeholders(original_input, new_input=None): + """ + Fills the placeholders of the original input with a new set of input + + Parameters + ---------- + original_input: dictionary + dictionary of input layers, can contain None + new_input: list or dictionary + list or dictionary of layers to substitute the None with + """ + if new_input is None: + return original_input + x = {} + i = 0 + for key, value in original_input.items(): + if value is None: + try: + x[key] = new_input[key] + except TypeError: + x[key] = new_input[i] + i += 1 + else: + x[key] = value + return x + + +class MetaModel(EvolModel): + """ + The `MetaModel` behaves as the tensorflow.keras.model.Model class, + with the addition of `tensor_content`: + + - tensor_content: + Sometimes when fitting a network the input is fixed, in this case the input can be given + together with the input_tensors by setting a `tensor_content` equal to the input value. + This is done automatically when using the `numpy_to_input` function from + `n3fit.backends.keras_backend.operations` + + Parameters + ---------- + input_tensors: tensorflow.keras.layers.Input + Input layer + output_tensors: tensorflow.keras.layers.Layer + Output layer + **kwargs: + keyword arguments to pass directly to Model + """ + + accepted_optimizers = optimizers + + def __init__(self, input_tensors, output_tensors, **kwargs): + self.has_dataset = False + + input_list = input_tensors + output_list = output_tensors + + if isinstance(input_list, dict): + # if this is a dictionary, convert it to a list for now + input_list = input_tensors.values() + elif not isinstance(input_list, list): + # if it is not a dict but also not a list, make it into a 1-element list and pray + input_list = [input_list] + + if isinstance(output_list, dict): + # if this is a dictionary, convert it to a list for now + output_list = output_tensors.values() + elif not isinstance(output_list, list): + # if it is not a dict but also not a list, make it into a 1-element list and pray + output_list = [output_list] + + super(MetaModel, self).__init__(input_list, output_list, **kwargs) + self.x_in = {} + self.tensors_in = {} + for input_tensor in input_list: + # If the input contains a tensor_content, store it to use at predict/fit/eval times + # otherwise, put a placeholder None as it will come from the outside + name = input_tensor.op.name + try: + self.x_in[name] = numpy_to_tensor(input_tensor.tensor_content) + self.tensors_in[name] = input_tensor + except AttributeError: + self.x_in[name] = None + self.tensors_in[name] = None + + self.all_inputs = input_list + self.all_outputs = output_list + self.target_tensors = None + self.eval_fun = None + + def _parse_input(self, extra_input=None, pass_content=True): + """ Returns the input tensors the model was compiled with. + Introduces the extra_input in the places asigned to the + placeholders. + + If ``pass_content`` is set to ``False``, pass the tensor object. + """ + if pass_content: + return _fill_placeholders(self.x_in, extra_input) + else: + return _fill_placeholders(self.tensors_in, extra_input) + + def perform_fit(self, x=None, y=None, epochs=1, **kwargs): + """ + Performs forward (and backwards) propagation for the model for a given number of epochs. + + The output of this function consists on a dictionary that maps the names of the metrics + of the model (the loss functions) to the partial losses. + + If the model was compiled with input and output data, they will not be passed through. + In this case by default the number of `epochs` will be set to 1 + + ex: + {'loss': [100], 'dataset_a_loss1' : [67], 'dataset_2_loss': [33]} + + Returns + ------- + loss_dict: dict + a dictionary with all partial losses of the model + """ + x = self._parse_input(self.x_in) + if y is None: + y = self.target_tensors + history = super().fit(x=x, y=y, epochs=epochs, **kwargs,) + loss_dict = history.history + return loss_dict + + def predict(self, x=None, **kwargs): + """ Call super().predict with the right input arguments """ + x = self._parse_input(x) + result = super().predict(x=x, **kwargs) + return result + + def compute_losses(self): + """ + This function is the fast-equivalent to the model ``evaluate(x,y)`` method. + + On first call it calls ``.evaluate(return_dict=True, verbose=0)`` to force + the initialization of the test function. + Subsequent calls of this method will (when applicable) + directly call the internal evaluation function ``eval_fun``. + This bypasses the pre- and post- evaluation steps, resulting in a ~10% speed up + with respect to ``.evaluate(...)`` + + Returns + ------- + dict + a dictionary with all partial losses of the model + """ + if self.eval_fun is None: + # We still need to perform some initialization + if LEGACY: + # For TF < 2.2 we need to generate the test_function ourselves + self.make_test_function() + else: + return self.evaluate(return_dict=True, verbose=False) + if LEGACY: + # For tF < 2.2 we need to force the output to be a float + ret = self.eval_fun() + ret['loss'] = ret['loss'].numpy() + return ret + else: + return self.eval_fun() + + def evaluate(self, x=None, y=None, **kwargs): + """ + Wrapper around evaluate to take into account the case in which the data is already known + when the model is compiled. + """ + x = self._parse_input(self.x_in) + if LEGACY and y is None: + y = self.target_tensors + result = super().evaluate(x=x, y=y, **kwargs) + return result + + def compile( + self, + optimizer_name="RMSprop", + learning_rate=None, + loss=None, + target_output=None, + clipnorm=None, + **kwargs, + ): + """ + Compile the model given an optimizer and a list of loss functions. + The optimizer must be one of those implemented in the `optimizer` attribute of this class. + + Options: + - A learning rate and a list of target outpout can be defined. + These will be passed down to the optimizer. + - A ``target_output`` can be defined. If done in this way + (for instance because we know the target data will be the same for the whole fit) + the data will be compiled together with the model and won't be necessary to + input it again when calling the ``perform_fit`` or ``compute_losses`` methods. + + Parameters + ---------- + optimizer_name: str + string defining the optimizer to be used + learning_rate: float + learning rate of of the optimizer + (if accepted as an argument, if not it will be ignored) + loss: list + list of loss functions to be pass to the model + target_output: list + list of outputs to compare the results to during fitting/evaluation + if given further calls to fit/evaluate must be done with y = None. + """ + try: + opt_tuple = optimizers[optimizer_name] + except KeyError as e: + raise NotImplementedError( + f"[MetaModel.select_initializer] optimizer not implemented: {optimizer_name}" + ) from e + + opt_function = opt_tuple[0] + opt_args = opt_tuple[1] + + user_selected_args = {"learning_rate": learning_rate, "clipnorm": clipnorm} + + # Override defaults with user provided values + for key, value in user_selected_args.items(): + if key in opt_args.keys() and value is not None: + opt_args[key] = value + + # Instantiate the optimizer + opt = opt_function(**opt_args) + + # If given target output, compile it together with the model for better performance + if target_output is not None: + if not isinstance(target_output, list): + target_output = [target_output] + # Tensorize + self.target_tensors = target_output + + # Reset the evaluation function (if any) + self.eval_fun = None + + super(MetaModel, self).compile(optimizer=opt, loss=loss) + + def make_test_function(self): + """ If the model has been compiled with target data, it creates + a specific evaluate function with the target data already evaluated. + Otherwise return the normal tensorflow behaviour. + """ + if self.eval_fun is not None: + return self.eval_fun + + if self.target_tensors is None: + return super().make_test_function() + + # Recover the target tensors and their lengths, we cannot rely + # directly on the output from the model as we might have target_tensors + # with 0 data points (if the tr/vl mask covers the whole set) + lens = [] + tt = [] + for target in self.target_tensors: + lens.append(target.size) + tt.append(numpy_to_tensor(target)) + # Save target_tensors as tensors, as it might be useful for LEGACY + self.target_tensors = tt + + # Get the name of the output layer + # and add the suffix _loss to match TF behaviour + out_names = [f"{i}_loss" for i in self.output_names] + out_names.insert(0, "loss") + + @tf.function + def eval_fun(*args): + predictions = self(self._parse_input(None)) + # Concatenate the output to split them again as a list + ypred = tf.concat(predictions, axis=-1) + predspl = tf.split(ypred, lens, axis=-1) + loss_list = [lfun(target, pred) for target, pred, lfun in zip(tt, predspl, self.loss)] + ret = [tf.reduce_sum(loss_list)] + loss_list + return dict(zip(out_names, ret)) + + # Save the function so we don't go through this again + self.eval_fun = eval_fun + + return eval_fun + + def set_masks_to(self, names, val=0.0): + """ Set all mask value to the selected value + Masks in MetaModel should be named {name}_mask + + Mask are layers with one single weight (shape=(1,)) that multiplies the input + + Parameters + ---------- + names: list + list of masks to look for + val: float + selected value of the mask + """ + mask_val = [val] + for name in names: + mask_name = f"{name}_mask" + mask_w = self.get_layer(mask_name).weights[0] + mask_w.assign(mask_val) + + def multiply_weights(self, layer_names, multipliers): + """ Multiply all weights for the given layers by some scalar + + Parameters + ---------- + layer_names: list + list of names of the layers to update weights + multipliers: list(float) + list of scalar multiplier to apply to each layer + """ + for layer_name, multiplier in zip(layer_names, multipliers): + layer = self.get_layer(layer_name) + w_val = layer.get_weights() + w_ref = layer.weights + for val, tensor in zip(w_val, w_ref): + tensor.assign(val * multiplier) + + def apply_as_layer(self, x): + """ Apply the model as a layer """ + x = self._parse_input(x, pass_content=False) + try: + return super().__call__(x) + except ValueError: + # TF < 2.1 + # TF 2.0 seems to fail with ValueError when passing a dictionary as an input + y = x.values() + return super().__call__(y) diff --git a/n3fit/src/n3fit/backends/ga_backend/__init__.py b/n3fit/src/n3fit/backends/ga_backend/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/n3fit/src/n3fit/backends/keras_backend/MetaModel.py b/n3fit/src/n3fit/backends/keras_backend/MetaModel.py index 4363f51dd5..6494c4a8fc 100644 --- a/n3fit/src/n3fit/backends/keras_backend/MetaModel.py +++ b/n3fit/src/n3fit/backends/keras_backend/MetaModel.py @@ -7,10 +7,9 @@ import numpy as np import tensorflow as tf +from tensorflow.keras.models import Model from tensorflow.keras import optimizers as Kopt from n3fit.backends.keras_backend.operations import numpy_to_tensor -import evolutionary_keras.optimizers as Evolutionary_optimizers -from evolutionary_keras.models import EvolModel # Check the TF version to check if legacy-mode is needed (TF < 2.2) @@ -30,8 +29,6 @@ "Adamax": (Kopt.Adamax, {}), "Nadam": (Kopt.Nadam, {}), "Amsgrad": (Kopt.Adam, {"learning_rate": 0.01, "amsgrad": True}), - "NGA": (Evolutionary_optimizers.NGA, {}), - "CMA": (Evolutionary_optimizers.CMA, {}) } # Some keys need to work for everyone @@ -66,7 +63,7 @@ def _fill_placeholders(original_input, new_input=None): return x -class MetaModel(EvolModel): +class MetaModel(Model): """ The `MetaModel` behaves as the tensorflow.keras.model.Model class, with the addition of `tensor_content`: diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index 61294e1403..ad42fe226c 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -100,6 +100,19 @@ def check_tensorboard(tensorboard): if weight_freq < 0: raise CheckError(f"The frequency at which weights are saved must be greater than 0, received {weight_freq}") +def check_backend(backend): + """ Checks whether the selected backend is available """ + if backend in [None, "tensorflow", "tf", "keras"]: + try: + import tensorflow + except ModuleNotFoundError: + raise CheckError(f"Tensorflow not available") + elif backend == "evolutionary_keras": + try: + import evolutionary_keras + except ModuleNotFoundError: + raise CheckError(f"evolutionary_keras not available") + @make_argcheck def wrapper_check_NN(fitting): """ Wrapper function for all NN-related checks """ @@ -111,7 +124,8 @@ def wrapper_check_NN(fitting): check_stopping(parameters) check_dropout(parameters) # Checks that need to import the backend (and thus take longer) should be done last - check_optimizer(parameters["optimizer"]) + check_backend(fitting.get("backend")) + check_optimizer(parameters["optimizer"]) check_initializer(parameters["initializer"]) diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index fb29ad1a26..42c6bc5ca5 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -141,7 +141,6 @@ def performfit( maxcores: int maximum number of (logical) cores that the backend should be aware of """ - if debug: # If debug is active, fix the initial state this should make the run reproducible # (important to avoid non-deterministic multithread or hidden states) @@ -158,6 +157,8 @@ def performfit( from n3fit.ModelTrainer import ModelTrainer from n3fit.io.writer import WriterWrapper from n3fit.backends import MetaModel, operations + if fitting.get("backend") == "evolutionary_keras": + MetaModel.enable_ga() import n3fit.io.reader as reader # Loading t0set from LHAPDF From f2d5a460cf74ec797c9470e69bbf326e3b4a683b Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Thu, 10 Sep 2020 22:12:16 +0200 Subject: [PATCH 04/11] set optimizer parameters in ga backend's MetaModel.py --- .../n3fit/backends/ga_backend/MetaModel.py | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py index 466d2110b2..a4fd6e34f3 100644 --- a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py +++ b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py @@ -13,7 +13,7 @@ # Check the TF version to check if legacy-mode is needed (TF < 2.2) -tf_version = tf.__version__.split('.') +tf_version = tf.__version__.split(".") if int(tf_version[0]) == 2 and int(tf_version[1]) < 2: LEGACY = True else: @@ -29,8 +29,14 @@ "Adamax": (Kopt.Adamax, {}), "Nadam": (Kopt.Nadam, {}), "Amsgrad": (Kopt.Adam, {"learning_rate": 0.01, "amsgrad": True}), - "NGA": (Evolutionary_optimizers.NGA, {}), - "CMA": (Evolutionary_optimizers.CMA, {}) + "NGA": ( + Evolutionary_optimizers.NGA, + {"sigma_init": 15, "population_size": 80, "mutation_rate": 0.05}, + ), + "CMA": ( + Evolutionary_optimizers.CMA, + {"sigma_init": 0.3, "population_size": None, "max_evaluations": None}, + ), } # Some keys need to work for everyone @@ -196,7 +202,7 @@ def compute_losses(self): if LEGACY: # For tF < 2.2 we need to force the output to be a float ret = self.eval_fun() - ret['loss'] = ret['loss'].numpy() + ret["loss"] = ret["loss"].numpy() return ret else: return self.eval_fun() @@ -215,10 +221,12 @@ def evaluate(self, x=None, y=None, **kwargs): def compile( self, optimizer_name="RMSprop", - learning_rate=None, + sigma_init=None, + population_size=None, + mutation_rate=None, + max_evaluations=None, loss=None, target_output=None, - clipnorm=None, **kwargs, ): """ @@ -256,7 +264,12 @@ def compile( opt_function = opt_tuple[0] opt_args = opt_tuple[1] - user_selected_args = {"learning_rate": learning_rate, "clipnorm": clipnorm} + user_selected_args = { + "sigma_init": sigma_init, + "population_size": population_size, + "mutation_rate": mutation_rate, + "max_evaluations": max_evaluations, + } # Override defaults with user provided values for key, value in user_selected_args.items(): From 05571d973421fe1a208b71bd1753fe477fcbf33e Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Thu, 10 Sep 2020 22:14:19 +0200 Subject: [PATCH 05/11] set optimizer in Basic_nga.yml runcard --- n3fit/runcards/Basic_nga.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/n3fit/runcards/Basic_nga.yml b/n3fit/runcards/Basic_nga.yml index 4fd5fc00a5..8e5439569f 100644 --- a/n3fit/runcards/Basic_nga.yml +++ b/n3fit/runcards/Basic_nga.yml @@ -58,9 +58,10 @@ fitting: activation_per_layer: ['sigmoid', 'sigmoid', 'linear'] initializer: 'glorot_normal' optimizer: - optimizer_name: 'RMSprop' - learning_rate: 0.01 - clipnorm: 1.0 + optimizer_name: 'NGA' + sigma_init: 15 + population_size: 80 + mutation_rate: 0.05 epochs: 900 positivity: multiplier: 1.05 # When any of the multiplier and/or the initial is not set From 065b5691d187e298b8e21814cabe859a64469b16 Mon Sep 17 00:00:00 2001 From: RoyStegeman Date: Thu, 10 Sep 2020 22:58:50 +0200 Subject: [PATCH 06/11] set trainable=False in nga runcard --- n3fit/runcards/Basic_nga.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/n3fit/runcards/Basic_nga.yml b/n3fit/runcards/Basic_nga.yml index 8e5439569f..ec10d20d19 100644 --- a/n3fit/runcards/Basic_nga.yml +++ b/n3fit/runcards/Basic_nga.yml @@ -85,11 +85,11 @@ fitting: - { fl: sng, pos: False, mutsize: [15], mutprob: [0.05], smallx: [1.05,1.19], largex: [1.47,2.70], trainable: False } - { fl: g, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.94,1.25], largex: [0.11,5.87], trainable: False } - { fl: v, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.54,0.75], largex: [1.15,2.76], trainable: False } - - { fl: v3, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.21,0.57], largex: [1.35,3.08] } - - { fl: v8, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.52,0.76], largex: [0.77,3.56], trainable: True } - - { fl: t3, pos: False, mutsize: [15], mutprob: [0.05], smallx: [-0.37,1.52], largex: [1.74,3.39] } - - { fl: t8, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.56,1.29], largex: [1.45,3.03] } - - { fl: cp, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.12,1.19], largex: [1.83,6.70] } + - { fl: v3, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.21,0.57], largex: [1.35,3.08], trainable: False } + - { fl: v8, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.52,0.76], largex: [0.77,3.56], trainable: False } + - { fl: t3, pos: False, mutsize: [15], mutprob: [0.05], smallx: [-0.37,1.52], largex: [1.74,3.39], trainable: False } + - { fl: t8, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.56,1.29], largex: [1.45,3.03], trainable: False } + - { fl: cp, pos: False, mutsize: [15], mutprob: [0.05], smallx: [0.12,1.19], largex: [1.83,6.70], trainable: False } ############################################################ stopping: From 3b7cd2b007094e9a051dada12e28d06b51274e1e Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 17 Sep 2020 10:32:15 +0200 Subject: [PATCH 07/11] add the select_backend function --- n3fit/src/n3fit/backends/__init__.py | 43 ++++++++++--------- .../backends/ga_backend/internal_state.py | 13 ++++++ .../backends/keras_backend/internal_state.py | 7 +-- n3fit/src/n3fit/checks.py | 30 +++++++------ n3fit/src/n3fit/performfit.py | 7 +-- 5 files changed, 61 insertions(+), 39 deletions(-) create mode 100644 n3fit/src/n3fit/backends/ga_backend/internal_state.py diff --git a/n3fit/src/n3fit/backends/__init__.py b/n3fit/src/n3fit/backends/__init__.py index cda6e198b9..96ffb5a6ec 100644 --- a/n3fit/src/n3fit/backends/__init__.py +++ b/n3fit/src/n3fit/backends/__init__.py @@ -1,12 +1,21 @@ +# Make the chosen backend available from n3fit.backends +# These includes: +# set_initial_state, clear_backend_state +# meta classes: MetaLayer, MetaModel +# base_layers: Input, Lambda, base_layer_selector, regularizer_selector, Concatenate +# modules: losses, operations, constraints, callbacks +# + + from n3fit.backends.keras_backend.internal_state import ( set_initial_state, clear_backend_state, ) from n3fit.backends.keras_backend.MetaLayer import MetaLayer +from n3fit.backends.keras_backend.MetaModel import MetaModel from n3fit.backends.keras_backend.base_layers import ( Input, - concatenate, Lambda, base_layer_selector, regularizer_selector, @@ -17,25 +26,19 @@ from n3fit.backends.keras_backend import constraints from n3fit.backends.keras_backend import callbacks -# Don't import the Model until it needs to be imported -class _MetaModel: - def __init__(self, backend="tensorflow"): - self.backend = backend - from n3fit.backends.keras_backend.MetaModel import MetaModel - - self.meta_model = MetaModel - - def enable_ga(self): - try: - from n3fit.backends.ga_backend.MetaModel import MetaModel - - self.meta_model = MetaModel - self.backend = "evolutionary_keras" - except ModuleNotFoundError: - raise ModuleNotFoundError("Install `evolutionary_keras` to use this backend") - def __call__(self, *args, **kwargs): - return self.meta_model(*args, **kwargs) +def select_backend(backend_name): + """ nuke the module from orbit """ + import sys + backends_module = sys.modules[__name__] + # Now depenidng on the backend, we need to load the backend importer function + if backend_name == "evolutionary_keras": + from n3fit.backends.ga_backend.internal_state import set_backend -MetaModel = _MetaModel() + set_backend(backends_module) + elif backend_name in ["tf", "tensorflow", "keras"]: + pass + # from n3fit.backends.keras_backend.internal_state import set_backend + else: + raise ValueError(f"Backend {backend_name} not recognized") diff --git a/n3fit/src/n3fit/backends/ga_backend/internal_state.py b/n3fit/src/n3fit/backends/ga_backend/internal_state.py new file mode 100644 index 0000000000..ca00224560 --- /dev/null +++ b/n3fit/src/n3fit/backends/ga_backend/internal_state.py @@ -0,0 +1,13 @@ +""" + Modify the internal state of the backend +""" +from n3fit.backends.ga_backend.MetaModel import MetaModel + + +def set_backend(module): + """Overwrites the necessary modules and imports from the + backends module. + Receives a reference to the module to overwrite. + """ + # Set the MetaModel and leave all the rest as the Tensorflow Standard + setattr(module, "MetaModel", MetaModel) diff --git a/n3fit/src/n3fit/backends/keras_backend/internal_state.py b/n3fit/src/n3fit/backends/keras_backend/internal_state.py index 5ffdfdd393..40b745e726 100644 --- a/n3fit/src/n3fit/backends/keras_backend/internal_state.py +++ b/n3fit/src/n3fit/backends/keras_backend/internal_state.py @@ -13,6 +13,7 @@ import random as rn import numpy as np import tensorflow as tf + # for (very slow but fine grained) debugging turn eager mode on # tf.config.run_functions_eagerly(True) from tensorflow.keras import backend as K @@ -44,10 +45,10 @@ def set_initial_state(seed=13): def clear_backend_state(max_cores=None): """ - Clears the state of the backend and opens a new session. + Clears the state of the backend and opens a new session. - Note that this function needs to set the TF session, including threads and processes - i.e., this function must NEVER be called after setting the initial state. + Note that this function needs to set the TF session, including threads and processes + i.e., this function must NEVER be called after setting the initial state. """ print("Clearing session") diff --git a/n3fit/src/n3fit/checks.py b/n3fit/src/n3fit/checks.py index ad42fe226c..64843cc77f 100644 --- a/n3fit/src/n3fit/checks.py +++ b/n3fit/src/n3fit/checks.py @@ -94,24 +94,27 @@ def check_dropout(parameters): if dropout is not None and not 0.0 <= dropout <= 1.0: raise CheckError(f"Dropout must be between 0 and 1, got: {dropout}") + def check_tensorboard(tensorboard): if tensorboard is not None: weight_freq = tensorboard.get("weight_freq", 0) if weight_freq < 0: - raise CheckError(f"The frequency at which weights are saved must be greater than 0, received {weight_freq}") + raise CheckError( + f"The frequency at which weights are saved must be greater than 0, received {weight_freq}" + ) + def check_backend(backend): """ Checks whether the selected backend is available """ - if backend in [None, "tensorflow", "tf", "keras"]: - try: - import tensorflow - except ModuleNotFoundError: - raise CheckError(f"Tensorflow not available") - elif backend == "evolutionary_keras": - try: - import evolutionary_keras - except ModuleNotFoundError: - raise CheckError(f"evolutionary_keras not available") + from n3fit.backends import select_backend + + try: + select_backend(backend) + except ModuleNotFoundError: + raise CheckError(f"Backend {backend} is not installed") + except ValueError: + raise CheckError(f"Backend {backend} not recognized") + @make_argcheck def wrapper_check_NN(fitting): @@ -124,8 +127,9 @@ def wrapper_check_NN(fitting): check_stopping(parameters) check_dropout(parameters) # Checks that need to import the backend (and thus take longer) should be done last - check_backend(fitting.get("backend")) - check_optimizer(parameters["optimizer"]) + # and check_backend _must_ always be called first + check_backend(fitting.get("backend")) + check_optimizer(parameters["optimizer"]) check_initializer(parameters["initializer"]) diff --git a/n3fit/src/n3fit/performfit.py b/n3fit/src/n3fit/performfit.py index 42c6bc5ca5..9ce9f80ce6 100644 --- a/n3fit/src/n3fit/performfit.py +++ b/n3fit/src/n3fit/performfit.py @@ -156,9 +156,10 @@ def performfit( # so they can eventually be set from the runcard from n3fit.ModelTrainer import ModelTrainer from n3fit.io.writer import WriterWrapper - from n3fit.backends import MetaModel, operations - if fitting.get("backend") == "evolutionary_keras": - MetaModel.enable_ga() + from n3fit.backends import select_backend, operations + select_backend(fitting.get("backend", "tf")) +# if fitting.get("backend") == "evolutionary_keras": +# MetaModel.enable_ga() import n3fit.io.reader as reader # Loading t0set from LHAPDF From 258f11d601643a5064fd89c53d493a6de1574432 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 17 Sep 2020 10:55:12 +0200 Subject: [PATCH 08/11] set_tf_backend by default as expected, clean up --- n3fit/src/n3fit/backends/__init__.py | 45 +-- .../n3fit/backends/ga_backend/MetaModel.py | 381 +----------------- .../backends/ga_backend/internal_state.py | 6 +- .../backends/keras_backend/internal_state.py | 32 ++ n3fit/src/n3fit/model_gen.py | 23 +- 5 files changed, 74 insertions(+), 413 deletions(-) diff --git a/n3fit/src/n3fit/backends/__init__.py b/n3fit/src/n3fit/backends/__init__.py index 96ffb5a6ec..ee82488161 100644 --- a/n3fit/src/n3fit/backends/__init__.py +++ b/n3fit/src/n3fit/backends/__init__.py @@ -2,43 +2,32 @@ # These includes: # set_initial_state, clear_backend_state # meta classes: MetaLayer, MetaModel -# base_layers: Input, Lambda, base_layer_selector, regularizer_selector, Concatenate -# modules: losses, operations, constraints, callbacks +# modules: backend_layers, losses, operations, constraints, callbacks # +# If no backend has been set the default is the tensorflow backend +from sys import modules as _modules +_backends_module = _modules[__name__] -from n3fit.backends.keras_backend.internal_state import ( - set_initial_state, - clear_backend_state, -) -from n3fit.backends.keras_backend.MetaLayer import MetaLayer -from n3fit.backends.keras_backend.MetaModel import MetaModel - -from n3fit.backends.keras_backend.base_layers import ( - Input, - Lambda, - base_layer_selector, - regularizer_selector, - Concatenate, -) -from n3fit.backends.keras_backend import losses -from n3fit.backends.keras_backend import operations -from n3fit.backends.keras_backend import constraints -from n3fit.backends.keras_backend import callbacks - +from n3fit.backends.keras_backend.internal_state import set_backend as set_tf +set_tf(_backends_module) def select_backend(backend_name): - """ nuke the module from orbit """ - import sys - - backends_module = sys.modules[__name__] + """ Select the appropiate backend by overriding this module + Default is understood as TensorFlow + """ + if backend_name is None: + backend_name = "tf" + try: + backend_name = backend_name.lower() + except AttributeError: + raise ValueError(f"select_backend accepts only strings, received: {backend_name}") # Now depenidng on the backend, we need to load the backend importer function if backend_name == "evolutionary_keras": from n3fit.backends.ga_backend.internal_state import set_backend - set_backend(backends_module) + set_backend(_backends_module) elif backend_name in ["tf", "tensorflow", "keras"]: - pass - # from n3fit.backends.keras_backend.internal_state import set_backend + set_tf(_backends_module) else: raise ValueError(f"Backend {backend_name} not recognized") diff --git a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py index a4fd6e34f3..47dcfae020 100644 --- a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py +++ b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py @@ -1,381 +1,20 @@ """ - MetaModel class + MetaModel class for evolutionary_keras + + Extension of the backend MetaModel class to use evolutionary_keras instead Extension of the backend Model class containing some wrappers in order to absorb other backend-dependent calls. """ -import tensorflow as tf -from tensorflow.keras import optimizers as Kopt -from n3fit.backends.keras_backend.operations import numpy_to_tensor +import n3fit.backends.keras_backend.MetaModel as tf_MetaModel import evolutionary_keras.optimizers as Evolutionary_optimizers from evolutionary_keras.models import EvolModel +# Add the evolutionary algorithms to the list of accepted optimizers +tf_MetaModel.optimizers["NGA"] = (Evolutionary_optimizers.NGA, {"sigma_init": 15, "population_size": 80, "mutation_rate": 0.05}) +tf_MetaModel.optimizers["CMA"] = (Evolutionary_optimizers.CMA,{"sigma_init": 0.3, "population_size": None, "max_evaluations": None}) -# Check the TF version to check if legacy-mode is needed (TF < 2.2) -tf_version = tf.__version__.split(".") -if int(tf_version[0]) == 2 and int(tf_version[1]) < 2: - LEGACY = True -else: - LEGACY = False - -# Define in this dictionary new optimizers as well as the arguments they accept -# (with default values if needed be) -optimizers = { - "RMSprop": (Kopt.RMSprop, {"learning_rate": 0.01}), - "Adam": (Kopt.Adam, {"learning_rate": 0.01}), - "Adagrad": (Kopt.Adagrad, {}), - "Adadelta": (Kopt.Adadelta, {"learning_rate": 1.0}), - "Adamax": (Kopt.Adamax, {}), - "Nadam": (Kopt.Nadam, {}), - "Amsgrad": (Kopt.Adam, {"learning_rate": 0.01, "amsgrad": True}), - "NGA": ( - Evolutionary_optimizers.NGA, - {"sigma_init": 15, "population_size": 80, "mutation_rate": 0.05}, - ), - "CMA": ( - Evolutionary_optimizers.CMA, - {"sigma_init": 0.3, "population_size": None, "max_evaluations": None}, - ), -} - -# Some keys need to work for everyone -for k, v in optimizers.items(): - v[1]["clipnorm"] = 1.0 - - -def _fill_placeholders(original_input, new_input=None): - """ - Fills the placeholders of the original input with a new set of input - - Parameters - ---------- - original_input: dictionary - dictionary of input layers, can contain None - new_input: list or dictionary - list or dictionary of layers to substitute the None with - """ - if new_input is None: - return original_input - x = {} - i = 0 - for key, value in original_input.items(): - if value is None: - try: - x[key] = new_input[key] - except TypeError: - x[key] = new_input[i] - i += 1 - else: - x[key] = value - return x - - -class MetaModel(EvolModel): - """ - The `MetaModel` behaves as the tensorflow.keras.model.Model class, - with the addition of `tensor_content`: - - - tensor_content: - Sometimes when fitting a network the input is fixed, in this case the input can be given - together with the input_tensors by setting a `tensor_content` equal to the input value. - This is done automatically when using the `numpy_to_input` function from - `n3fit.backends.keras_backend.operations` - - Parameters - ---------- - input_tensors: tensorflow.keras.layers.Input - Input layer - output_tensors: tensorflow.keras.layers.Layer - Output layer - **kwargs: - keyword arguments to pass directly to Model - """ - - accepted_optimizers = optimizers - - def __init__(self, input_tensors, output_tensors, **kwargs): - self.has_dataset = False - - input_list = input_tensors - output_list = output_tensors - - if isinstance(input_list, dict): - # if this is a dictionary, convert it to a list for now - input_list = input_tensors.values() - elif not isinstance(input_list, list): - # if it is not a dict but also not a list, make it into a 1-element list and pray - input_list = [input_list] - - if isinstance(output_list, dict): - # if this is a dictionary, convert it to a list for now - output_list = output_tensors.values() - elif not isinstance(output_list, list): - # if it is not a dict but also not a list, make it into a 1-element list and pray - output_list = [output_list] - - super(MetaModel, self).__init__(input_list, output_list, **kwargs) - self.x_in = {} - self.tensors_in = {} - for input_tensor in input_list: - # If the input contains a tensor_content, store it to use at predict/fit/eval times - # otherwise, put a placeholder None as it will come from the outside - name = input_tensor.op.name - try: - self.x_in[name] = numpy_to_tensor(input_tensor.tensor_content) - self.tensors_in[name] = input_tensor - except AttributeError: - self.x_in[name] = None - self.tensors_in[name] = None - - self.all_inputs = input_list - self.all_outputs = output_list - self.target_tensors = None - self.eval_fun = None - - def _parse_input(self, extra_input=None, pass_content=True): - """ Returns the input tensors the model was compiled with. - Introduces the extra_input in the places asigned to the - placeholders. - - If ``pass_content`` is set to ``False``, pass the tensor object. - """ - if pass_content: - return _fill_placeholders(self.x_in, extra_input) - else: - return _fill_placeholders(self.tensors_in, extra_input) - - def perform_fit(self, x=None, y=None, epochs=1, **kwargs): - """ - Performs forward (and backwards) propagation for the model for a given number of epochs. - - The output of this function consists on a dictionary that maps the names of the metrics - of the model (the loss functions) to the partial losses. - - If the model was compiled with input and output data, they will not be passed through. - In this case by default the number of `epochs` will be set to 1 - - ex: - {'loss': [100], 'dataset_a_loss1' : [67], 'dataset_2_loss': [33]} - - Returns - ------- - loss_dict: dict - a dictionary with all partial losses of the model - """ - x = self._parse_input(self.x_in) - if y is None: - y = self.target_tensors - history = super().fit(x=x, y=y, epochs=epochs, **kwargs,) - loss_dict = history.history - return loss_dict - - def predict(self, x=None, **kwargs): - """ Call super().predict with the right input arguments """ - x = self._parse_input(x) - result = super().predict(x=x, **kwargs) - return result - - def compute_losses(self): - """ - This function is the fast-equivalent to the model ``evaluate(x,y)`` method. - - On first call it calls ``.evaluate(return_dict=True, verbose=0)`` to force - the initialization of the test function. - Subsequent calls of this method will (when applicable) - directly call the internal evaluation function ``eval_fun``. - This bypasses the pre- and post- evaluation steps, resulting in a ~10% speed up - with respect to ``.evaluate(...)`` - - Returns - ------- - dict - a dictionary with all partial losses of the model - """ - if self.eval_fun is None: - # We still need to perform some initialization - if LEGACY: - # For TF < 2.2 we need to generate the test_function ourselves - self.make_test_function() - else: - return self.evaluate(return_dict=True, verbose=False) - if LEGACY: - # For tF < 2.2 we need to force the output to be a float - ret = self.eval_fun() - ret["loss"] = ret["loss"].numpy() - return ret - else: - return self.eval_fun() - - def evaluate(self, x=None, y=None, **kwargs): - """ - Wrapper around evaluate to take into account the case in which the data is already known - when the model is compiled. - """ - x = self._parse_input(self.x_in) - if LEGACY and y is None: - y = self.target_tensors - result = super().evaluate(x=x, y=y, **kwargs) - return result - - def compile( - self, - optimizer_name="RMSprop", - sigma_init=None, - population_size=None, - mutation_rate=None, - max_evaluations=None, - loss=None, - target_output=None, - **kwargs, - ): - """ - Compile the model given an optimizer and a list of loss functions. - The optimizer must be one of those implemented in the `optimizer` attribute of this class. - - Options: - - A learning rate and a list of target outpout can be defined. - These will be passed down to the optimizer. - - A ``target_output`` can be defined. If done in this way - (for instance because we know the target data will be the same for the whole fit) - the data will be compiled together with the model and won't be necessary to - input it again when calling the ``perform_fit`` or ``compute_losses`` methods. - - Parameters - ---------- - optimizer_name: str - string defining the optimizer to be used - learning_rate: float - learning rate of of the optimizer - (if accepted as an argument, if not it will be ignored) - loss: list - list of loss functions to be pass to the model - target_output: list - list of outputs to compare the results to during fitting/evaluation - if given further calls to fit/evaluate must be done with y = None. - """ - try: - opt_tuple = optimizers[optimizer_name] - except KeyError as e: - raise NotImplementedError( - f"[MetaModel.select_initializer] optimizer not implemented: {optimizer_name}" - ) from e - - opt_function = opt_tuple[0] - opt_args = opt_tuple[1] - - user_selected_args = { - "sigma_init": sigma_init, - "population_size": population_size, - "mutation_rate": mutation_rate, - "max_evaluations": max_evaluations, - } - - # Override defaults with user provided values - for key, value in user_selected_args.items(): - if key in opt_args.keys() and value is not None: - opt_args[key] = value - - # Instantiate the optimizer - opt = opt_function(**opt_args) - - # If given target output, compile it together with the model for better performance - if target_output is not None: - if not isinstance(target_output, list): - target_output = [target_output] - # Tensorize - self.target_tensors = target_output - - # Reset the evaluation function (if any) - self.eval_fun = None - - super(MetaModel, self).compile(optimizer=opt, loss=loss) - - def make_test_function(self): - """ If the model has been compiled with target data, it creates - a specific evaluate function with the target data already evaluated. - Otherwise return the normal tensorflow behaviour. - """ - if self.eval_fun is not None: - return self.eval_fun - - if self.target_tensors is None: - return super().make_test_function() - - # Recover the target tensors and their lengths, we cannot rely - # directly on the output from the model as we might have target_tensors - # with 0 data points (if the tr/vl mask covers the whole set) - lens = [] - tt = [] - for target in self.target_tensors: - lens.append(target.size) - tt.append(numpy_to_tensor(target)) - # Save target_tensors as tensors, as it might be useful for LEGACY - self.target_tensors = tt - - # Get the name of the output layer - # and add the suffix _loss to match TF behaviour - out_names = [f"{i}_loss" for i in self.output_names] - out_names.insert(0, "loss") - - @tf.function - def eval_fun(*args): - predictions = self(self._parse_input(None)) - # Concatenate the output to split them again as a list - ypred = tf.concat(predictions, axis=-1) - predspl = tf.split(ypred, lens, axis=-1) - loss_list = [lfun(target, pred) for target, pred, lfun in zip(tt, predspl, self.loss)] - ret = [tf.reduce_sum(loss_list)] + loss_list - return dict(zip(out_names, ret)) - - # Save the function so we don't go through this again - self.eval_fun = eval_fun - - return eval_fun - - def set_masks_to(self, names, val=0.0): - """ Set all mask value to the selected value - Masks in MetaModel should be named {name}_mask - - Mask are layers with one single weight (shape=(1,)) that multiplies the input - - Parameters - ---------- - names: list - list of masks to look for - val: float - selected value of the mask - """ - mask_val = [val] - for name in names: - mask_name = f"{name}_mask" - mask_w = self.get_layer(mask_name).weights[0] - mask_w.assign(mask_val) - - def multiply_weights(self, layer_names, multipliers): - """ Multiply all weights for the given layers by some scalar - - Parameters - ---------- - layer_names: list - list of names of the layers to update weights - multipliers: list(float) - list of scalar multiplier to apply to each layer - """ - for layer_name, multiplier in zip(layer_names, multipliers): - layer = self.get_layer(layer_name) - w_val = layer.get_weights() - w_ref = layer.weights - for val, tensor in zip(w_val, w_ref): - tensor.assign(val * multiplier) - - def apply_as_layer(self, x): - """ Apply the model as a layer """ - x = self._parse_input(x, pass_content=False) - try: - return super().__call__(x) - except ValueError: - # TF < 2.1 - # TF 2.0 seems to fail with ValueError when passing a dictionary as an input - y = x.values() - return super().__call__(y) +# Mix inheritance +class MetaModel(tf_MetaModel.MetaModel, EvolModel): + accepted_optimizers = tf_MetaModel.optimizers diff --git a/n3fit/src/n3fit/backends/ga_backend/internal_state.py b/n3fit/src/n3fit/backends/ga_backend/internal_state.py index ca00224560..5aea72f205 100644 --- a/n3fit/src/n3fit/backends/ga_backend/internal_state.py +++ b/n3fit/src/n3fit/backends/ga_backend/internal_state.py @@ -2,6 +2,7 @@ Modify the internal state of the backend """ from n3fit.backends.ga_backend.MetaModel import MetaModel +from n3fit.backends import select_backend def set_backend(module): @@ -9,5 +10,8 @@ def set_backend(module): backends module. Receives a reference to the module to overwrite. """ - # Set the MetaModel and leave all the rest as the Tensorflow Standard + # First ensure the backend is tensorflow + select_backend("Tensorflow") + + # Now set the MetaModel and leave all the rest as the Tensorflow Standard setattr(module, "MetaModel", MetaModel) diff --git a/n3fit/src/n3fit/backends/keras_backend/internal_state.py b/n3fit/src/n3fit/backends/keras_backend/internal_state.py index 40b745e726..6c68ffebab 100644 --- a/n3fit/src/n3fit/backends/keras_backend/internal_state.py +++ b/n3fit/src/n3fit/backends/keras_backend/internal_state.py @@ -19,6 +19,38 @@ from tensorflow.keras import backend as K +def set_backend(module): + """Overwrites the necessary modules and imports from the + backends module. + Receives a reference to the module to overwrite. + """ + from n3fit.backends.keras_backend.internal_state import ( + set_initial_state, + clear_backend_state, + ) + + setattr(module, "set_initial_state", set_initial_state) + setattr(module, "clear_backend_state", clear_backend_state) + + from n3fit.backends.keras_backend.MetaLayer import MetaLayer + from n3fit.backends.keras_backend.MetaModel import MetaModel + + setattr(module, "MetaLayer", MetaLayer) + setattr(module, "MetaModel", MetaModel) + + import n3fit.backends.keras_backend.base_layers as backend_layers + from n3fit.backends.keras_backend import losses + from n3fit.backends.keras_backend import operations + from n3fit.backends.keras_backend import constraints + from n3fit.backends.keras_backend import callbacks + + setattr(module, "backend_layers", backend_layers) + setattr(module, "losses", losses) + setattr(module, "operations", operations) + setattr(module, "constraints", constraints) + setattr(module, "callbacks", callbacks) + + def set_initial_state(seed=13): """ Sets the initial state of the backend diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index b93904f99e..0931cd128b 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -13,13 +13,10 @@ from n3fit.layers import DIS, DY, Mask, ObsRotation from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution -from n3fit.backends import MetaModel, Input +from n3fit.backends import MetaModel, MetaLayer from n3fit.backends import operations from n3fit.backends import losses -from n3fit.backends import MetaLayer, Concatenate, Lambda -from n3fit.backends import base_layer_selector, regularizer_selector - -import tensorflow as tf +from n3fit.backends import backend_layers def observable_generator(spec_dict, positivity_initial=1.0, integrability=False): # pylint: disable=too-many-locals @@ -105,7 +102,7 @@ def observable_generator(spec_dict, positivity_initial=1.0, integrability=False) model_obs.append(obs_layer) # Prepare a concatenation as experiments are one single entity formed by many datasets - concatenator = Concatenate(axis=1, name=f"{spec_name}_full") + concatenator = backend_layers.Concatenate(axis=1, name=f"{spec_name}_full") # creating the experiment as a model turns out to bad for performance def experiment_layer(pdf, datasets_out=None): @@ -237,7 +234,7 @@ def generate_dense_network( for i, (nodes_out, activation) in enumerate(zip(nodes, activations)): # if we have dropout set up, add it to the list if dropout_rate > 0 and i == dropout_layer: - list_of_pdf_layers.append(base_layer_selector("dropout", rate=dropout_rate)) + list_of_pdf_layers.append(backend_layers.base_layer_selector("dropout", rate=dropout_rate)) # select the initializer and move the seed init = MetaLayer.select_initializer(initializer_name, seed=seed + i) @@ -251,7 +248,7 @@ def generate_dense_network( "kernel_regularizer": regularizer, } - layer = base_layer_selector("dense", **arguments) + layer = backend_layers.base_layer_selector("dense", **arguments) list_of_pdf_layers.append(layer) nodes_in = int(nodes_out) @@ -291,11 +288,11 @@ def generate_dense_per_flavour_network( "basis_size": basis_size, } - layer = base_layer_selector("dense_per_flavour", **arguments) + layer = backend_layers.base_layer_selector("dense_per_flavour", **arguments) if i == number_of_layers - 1: # For the last layer, apply concatenate - concatenator = base_layer_selector("concatenate") + concatenator = backend_layers.base_layer_selector("concatenate") def output_layer(ilayer): result = layer(ilayer) @@ -428,7 +425,7 @@ def pdfNN_layer_generator( last_layer_nodes = nodes[-1] if layer_type == "dense": - reg = regularizer_selector(regularizer, **regularizer_args) + reg = backend_layers.regularizer_selector(regularizer, **regularizer_args) list_of_pdf_layers = generate_dense_network( inp, nodes, @@ -449,7 +446,7 @@ def pdfNN_layer_generator( # If the input is of type (x, logx) # create a x --> (x, logx) layer to preppend to everything if inp == 2: - add_log = Lambda(lambda x: operations.concatenate([x, operations.op_log(x)], axis=-1)) + add_log = backend_layers.Lambda(lambda x: operations.concatenate([x, operations.op_log(x)], axis=-1)) def dense_me(x): """ Takes an input tensor `x` and applies all layers @@ -488,7 +485,7 @@ def layer_pdf(x): return layer_evln(layer_fitbasis(x)) # Prepare the input for the PDF model - placeholder_input = Input(shape=(None, 1), batch_size=1) + placeholder_input = backend_layers.Input(shape=(None, 1), batch_size=1) # Impose sumrule if necessary if impose_sumrule: From acb3f68b54ea207865ecb24c92c25a5e001d7710 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 17 Sep 2020 11:05:16 +0200 Subject: [PATCH 09/11] style --- n3fit/src/n3fit/backends/__init__.py | 5 +++- .../n3fit/backends/ga_backend/MetaModel.py | 18 +++++++---- n3fit/src/n3fit/model_gen.py | 30 ++++++++++++------- .../tests/regressions/noval-quickcard.yml | 2 +- n3fit/src/n3fit/tests/test_fit.py | 2 +- 5 files changed, 37 insertions(+), 20 deletions(-) diff --git a/n3fit/src/n3fit/backends/__init__.py b/n3fit/src/n3fit/backends/__init__.py index ee82488161..12439501cd 100644 --- a/n3fit/src/n3fit/backends/__init__.py +++ b/n3fit/src/n3fit/backends/__init__.py @@ -7,13 +7,16 @@ # If no backend has been set the default is the tensorflow backend from sys import modules as _modules + _backends_module = _modules[__name__] from n3fit.backends.keras_backend.internal_state import set_backend as set_tf + set_tf(_backends_module) + def select_backend(backend_name): - """ Select the appropiate backend by overriding this module + """Select the appropiate backend by overriding this module Default is understood as TensorFlow """ if backend_name is None: diff --git a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py index 47dcfae020..5dd103e89b 100644 --- a/n3fit/src/n3fit/backends/ga_backend/MetaModel.py +++ b/n3fit/src/n3fit/backends/ga_backend/MetaModel.py @@ -1,10 +1,9 @@ """ MetaModel class for evolutionary_keras - Extension of the backend MetaModel class to use evolutionary_keras instead - - Extension of the backend Model class containing some wrappers in order to absorb other - backend-dependent calls. + Extension of the backend MetaModel class to use evolutionary_keras. + The MetaModel class provided by this module is a multiple inheritance + of the keras_backend.MetaModel and the evolutionary_keras.EvolModel """ import n3fit.backends.keras_backend.MetaModel as tf_MetaModel @@ -12,9 +11,16 @@ from evolutionary_keras.models import EvolModel # Add the evolutionary algorithms to the list of accepted optimizers -tf_MetaModel.optimizers["NGA"] = (Evolutionary_optimizers.NGA, {"sigma_init": 15, "population_size": 80, "mutation_rate": 0.05}) -tf_MetaModel.optimizers["CMA"] = (Evolutionary_optimizers.CMA,{"sigma_init": 0.3, "population_size": None, "max_evaluations": None}) +tf_MetaModel.optimizers["NGA"] = ( + Evolutionary_optimizers.NGA, + {"sigma_init": 15, "population_size": 80, "mutation_rate": 0.05}, +) +tf_MetaModel.optimizers["CMA"] = ( + Evolutionary_optimizers.CMA, + {"sigma_init": 0.3, "population_size": None, "max_evaluations": None}, +) # Mix inheritance class MetaModel(tf_MetaModel.MetaModel, EvolModel): + accepted_optimizers = tf_MetaModel.optimizers diff --git a/n3fit/src/n3fit/model_gen.py b/n3fit/src/n3fit/model_gen.py index 0931cd128b..70fcb25d43 100644 --- a/n3fit/src/n3fit/model_gen.py +++ b/n3fit/src/n3fit/model_gen.py @@ -7,8 +7,6 @@ # pdfNN_layer_generator: Generates the PDF NN layer to be fitted """ -import numpy as np - import n3fit.msr as msr_constraints from n3fit.layers import DIS, DY, Mask, ObsRotation from n3fit.layers import Preprocessing, FkRotation, FlavourToEvolution @@ -19,7 +17,9 @@ from n3fit.backends import backend_layers -def observable_generator(spec_dict, positivity_initial=1.0, integrability=False): # pylint: disable=too-many-locals +def observable_generator( + spec_dict, positivity_initial=1.0, integrability=False +): # pylint: disable=too-many-locals """ This function generates the observable model for each experiment. These are models which takes as input a PDF tensor (1 x size_of_xgrid x flavours) and outputs @@ -69,7 +69,6 @@ def observable_generator(spec_dict, positivity_initial=1.0, integrability=False) for dataset_dict in spec_dict["datasets"]: # Get the generic information of the dataset dataset_name = dataset_dict["name"] - ndata = dataset_dict["ndata"] # Look at what kind of layer do we need for this dataset if dataset_dict["hadronic"]: @@ -234,7 +233,9 @@ def generate_dense_network( for i, (nodes_out, activation) in enumerate(zip(nodes, activations)): # if we have dropout set up, add it to the list if dropout_rate > 0 and i == dropout_layer: - list_of_pdf_layers.append(backend_layers.base_layer_selector("dropout", rate=dropout_rate)) + list_of_pdf_layers.append( + backend_layers.base_layer_selector("dropout", rate=dropout_rate) + ) # select the initializer and move the seed init = MetaLayer.select_initializer(initializer_name, seed=seed + i) @@ -313,7 +314,7 @@ def pdfNN_layer_generator( initializer_name="glorot_normal", layer_type="dense", flav_info=None, - fitbasis='NN31IC', + fitbasis="NN31IC", out=14, seed=None, dropout=0.0, @@ -440,17 +441,24 @@ def pdfNN_layer_generator( # TODO: this information should come from the basis information # once the basis information is passed to this class list_of_pdf_layers = generate_dense_per_flavour_network( - inp, nodes, activations, initializer_name, seed=seed, basis_size=last_layer_nodes, + inp, + nodes, + activations, + initializer_name, + seed=seed, + basis_size=last_layer_nodes, ) # If the input is of type (x, logx) # create a x --> (x, logx) layer to preppend to everything if inp == 2: - add_log = backend_layers.Lambda(lambda x: operations.concatenate([x, operations.op_log(x)], axis=-1)) + add_log = backend_layers.Lambda( + lambda x: operations.concatenate([x, operations.op_log(x)], axis=-1) + ) def dense_me(x): - """ Takes an input tensor `x` and applies all layers - from the `list_of_pdf_layers` in order """ + """Takes an input tensor `x` and applies all layers + from the `list_of_pdf_layers` in order""" if inp == 1: curr_fun = list_of_pdf_layers[0](x) else: @@ -471,7 +479,7 @@ def dense_me(x): # Basis rotation basis_rotation = FlavourToEvolution(flav_info=flav_info, fitbasis=fitbasis) - + # Apply preprocessing and basis def layer_fitbasis(x): ret = operations.op_multiply([dense_me(x), layer_preproc(x)]) diff --git a/n3fit/src/n3fit/tests/regressions/noval-quickcard.yml b/n3fit/src/n3fit/tests/regressions/noval-quickcard.yml index 0efd5b4efd..ec41b36143 100644 --- a/n3fit/src/n3fit/tests/regressions/noval-quickcard.yml +++ b/n3fit/src/n3fit/tests/regressions/noval-quickcard.yml @@ -51,7 +51,7 @@ fitting: optimizer_name: 'Adadelta' learning_rate: 0.00001 clipnorm: 1.0 - epochs: 20000 + epochs: 200000 # Ensure that it won't end in the alloted time stopping_patience: 1.00 # percentage of the number of epochs layer_type: 'dense_per_flavour' dropout: 0.0 diff --git a/n3fit/src/n3fit/tests/test_fit.py b/n3fit/src/n3fit/tests/test_fit.py index 3b5939be74..59deeba89c 100644 --- a/n3fit/src/n3fit/tests/test_fit.py +++ b/n3fit/src/n3fit/tests/test_fit.py @@ -147,4 +147,4 @@ def test_novalidation(timing=30): tmp_path = pathlib.Path(tmp_name) shutil.copy(quickpath, tmp_path) with pytest.raises(sp.TimeoutExpired): - sp.run(f"{EXE} {quickcard} {REPLICA}".split(), cwd=tmp_path, timeout=timing) + finish = sp.run(f"{EXE} {quickcard} {REPLICA}".split(), cwd=tmp_path, timeout=timing) From fa7a54f7cdd4cdbf12474614c3c8ebfb9441d991 Mon Sep 17 00:00:00 2001 From: juacrumar Date: Thu, 17 Sep 2020 12:13:26 +0200 Subject: [PATCH 10/11] add docs --- doc/sphinx/source/n3fit/backends.md | 36 +++++++++++++++++++++++++++++ doc/sphinx/source/n3fit/index.rst | 1 + 2 files changed, 37 insertions(+) create mode 100644 doc/sphinx/source/n3fit/backends.md diff --git a/doc/sphinx/source/n3fit/backends.md b/doc/sphinx/source/n3fit/backends.md new file mode 100644 index 0000000000..96e839a96a --- /dev/null +++ b/doc/sphinx/source/n3fit/backends.md @@ -0,0 +1,36 @@ +Backend selection +================= + +One of the advantages of ``n3fit`` is the possibility to select different backends +for training the neural network. + +Currently implemented in ``n3fit`` we can find: + +- [Keras-Tensorflow](#keras-tensorflow) +- [Evolutionary Algorithms](#evolutionary-algorithms) + +Keras-Tensorflow +---------------- +The main and default backend for ``n3fit`` is the [TensorFlow](https://www.tensorflow.org/) +library developed by Google. + + +Evolutionary Algorithms +----------------------- +As part of the validation process followed during the development of ``n3fit`` we developed the +[evolutionary-keras](https://evolutionary-keras.readthedocs.io) [[1](https://arxiv.org/abs/2002.06587)] library. + +This library extends [Tensorflow](#keras-tensorflow) for its use with evolutionary algorithms such as the [NGA](https://evolutionary-keras.readthedocs.io/en/latest/optimizers.html#nodal-genetic-algorithm-nga) +used in previous versions of [NNPDF](http://arxiv.org/abs/1410.8849) or the [CMA-ES](https://evolutionary-keras.readthedocs.io/en/latest/optimizers.html#covariance-matrix-adaptation-evolution-strategy-cma-es) used in some [spin-off studies](https://arxiv.org/abs/1706.07049). + +The evolutionary strategies can be used by adding the keyword ``backend: evolutionary_keras`` to the ``fitting`` section in the runcard. + +```yaml +fitting: + backend: evolutionary_keras + optimizer: + optimizer_name: 'NGA' + sigma_init: 15 + population_size: 80 + mutation_rate: 0.05 +``` diff --git a/doc/sphinx/source/n3fit/index.rst b/doc/sphinx/source/n3fit/index.rst index 9f1d542e35..67ac0831d3 100644 --- a/doc/sphinx/source/n3fit/index.rst +++ b/doc/sphinx/source/n3fit/index.rst @@ -6,5 +6,6 @@ n3fit introduction methodology + backends hyperopt runcard_detailed From ed88b301acb3b9b4a05fe2382723266bbd2700ae Mon Sep 17 00:00:00 2001 From: Juacrumar Date: Thu, 17 Sep 2020 15:23:07 +0200 Subject: [PATCH 11/11] Update test_fit.py --- n3fit/src/n3fit/tests/test_fit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/n3fit/src/n3fit/tests/test_fit.py b/n3fit/src/n3fit/tests/test_fit.py index 59deeba89c..3b5939be74 100644 --- a/n3fit/src/n3fit/tests/test_fit.py +++ b/n3fit/src/n3fit/tests/test_fit.py @@ -147,4 +147,4 @@ def test_novalidation(timing=30): tmp_path = pathlib.Path(tmp_name) shutil.copy(quickpath, tmp_path) with pytest.raises(sp.TimeoutExpired): - finish = sp.run(f"{EXE} {quickcard} {REPLICA}".split(), cwd=tmp_path, timeout=timing) + sp.run(f"{EXE} {quickcard} {REPLICA}".split(), cwd=tmp_path, timeout=timing)