From 03830929697464666b58be717ece8328bc6c6965 Mon Sep 17 00:00:00 2001 From: Michal Danilowicz Date: Mon, 16 Sep 2024 13:28:15 +0000 Subject: [PATCH 1/5] [Fix] InferDuplicateStreamsLayer now properly handles forks of multiple-output nodes --- .../fpgadataflow/convert_to_hw_layers.py | 96 +++++++++---------- 1 file changed, 48 insertions(+), 48 deletions(-) diff --git a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py index 25a2032aeb..b02bc89db8 100644 --- a/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py +++ b/src/finn/transformation/fpgadataflow/convert_to_hw_layers.py @@ -585,63 +585,63 @@ def apply(self, model): for node in graph.node: node_ind += 1 - successors = model.find_consumers(node.output[0]) - if successors is not None and len(successors) >= 2: - output_tensor = node.output[0] - n_outputs = len(successors) + for output_tensor in node.output: + successors = model.find_consumers(output_tensor) + if successors is not None and len(successors) >= 2: + n_outputs = len(successors) - dt = model.get_tensor_datatype(output_tensor) + dt = model.get_tensor_datatype(output_tensor) - # skip conversion for layers with float input - if not dt.is_integer(): - continue + # skip conversion for layers with float input + if not dt.is_integer(): + continue - # create clone tensors - out_shape = model.get_tensor_shape(output_tensor) - out_tensor_clones = [] - for i in range(n_outputs): - clone = helper.make_tensor_value_info( - model.make_new_valueinfo_name(), TensorProto.FLOAT, out_shape - ) - model.graph.value_info.append(clone) - out_tensor_clones += [clone.name] + # create clone tensors + out_shape = model.get_tensor_shape(output_tensor) + out_tensor_clones = [] + for i in range(n_outputs): + clone = helper.make_tensor_value_info( + model.make_new_valueinfo_name(), TensorProto.FLOAT, out_shape + ) + model.graph.value_info.append(clone) + out_tensor_clones += [clone.name] - num_ch = int(out_shape[-1]) - vecs = out_shape[:-1] + num_ch = int(out_shape[-1]) + vecs = out_shape[:-1] - # create node with no parallelization first - pe = 1 + # create node with no parallelization first + pe = 1 - dup_node = helper.make_node( - "DuplicateStreams", - [output_tensor], - out_tensor_clones, - domain="finn.custom_op.fpgadataflow", - backend="fpgadataflow", - NumChannels=num_ch, - PE=pe, - inputDataType=dt.name, - numInputVectors=vecs, - NumOutputStreams=n_outputs, - outFIFODepths=[2] * n_outputs, - name="DuplicateStreams_" + node.name, - ) + dup_node = helper.make_node( + "DuplicateStreams", + [output_tensor], + out_tensor_clones, + domain="finn.custom_op.fpgadataflow", + backend="fpgadataflow", + NumChannels=num_ch, + PE=pe, + inputDataType=dt.name, + numInputVectors=vecs, + NumOutputStreams=n_outputs, + outFIFODepths=[2] * n_outputs, + name="DuplicateStreams_" + node.name, + ) - graph.node.insert(node_ind, dup_node) + graph.node.insert(node_ind, dup_node) - # connect successors to out tensor clone - clone_idx = 0 - for successor in successors: - for i, succ_input in enumerate(successor.input): - if succ_input == output_tensor: - successor.input[i] = out_tensor_clones[clone_idx] - clone_idx += 1 - # if one node has multiple connections to the same output - # find_direct_successors will return one node per input - # so break the inner loop will result in correct behaviour - break + # connect successors to out tensor clone + clone_idx = 0 + for successor in successors: + for i, succ_input in enumerate(successor.input): + if succ_input == output_tensor: + successor.input[i] = out_tensor_clones[clone_idx] + clone_idx += 1 + # if one node has multiple connections to the same output + # find_direct_successors will return one node per input + # so break the inner loop will result in correct behaviour + break - graph_modified = True + graph_modified = True if graph_modified: model = model.transform(SortGraph()) From d13aa7e7debb21bd1d75b6dbb6eddc959b4ae8c8 Mon Sep 17 00:00:00 2001 From: Michal Danilowicz Date: Mon, 16 Sep 2024 13:48:43 +0000 Subject: [PATCH 2/5] [Fix] MoveScalarLinearPastInvariants, MakeMaxPoolNHWC, MakeScaleResizeNHWC transformations are checking whether the node to be moved is a fork node, in which case the MoveOpPastFork is called. MoveOpPastFork uses deepcopies of the original node. --- src/finn/transformation/streamline/reorder.py | 30 ++++++++++++++----- 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py index 8ac2d7dad6..9a7e9d0723 100644 --- a/src/finn/transformation/streamline/reorder.py +++ b/src/finn/transformation/streamline/reorder.py @@ -29,6 +29,7 @@ import numpy as np import qonnx.core.data_layout as DataLayout import warnings +from copy import deepcopy from onnx import TensorProto from onnx import helper as oh from qonnx.core.datatype import DataType @@ -641,6 +642,10 @@ def apply(self, model): # if initializer is not scalar, skip if np.prod(init0.shape) != 1: continue + if model.is_fork_node(prod0): + model = model.transform(MoveOpPastFork(prod0.op_type)) + # topology modified, "ask" ModelWrapper to apply this transform again + return (model, True) # Flatten input if required if len(init0.shape) > 0: init0 = init0.flatten()[0] @@ -713,6 +718,12 @@ def apply(self, model): elif producer is not None and producer.op_type == "Transpose": perms = list(get_by_name(producer.attribute, "perm").ints) if perms == [0, 3, 1, 2]: + # check if the producer is a fork node + # (need to move it past the fork before this transform) + if model.is_fork_node(producer): + model = model.transform(MoveTransposePastFork()) + # topology modified, "ask" ModelWrapper to apply this transform again + return (model, True) ceil_mode = get_by_name(n.attribute, "ceil_mode") if ceil_mode is not None: ceil_mode = ceil_mode.i @@ -764,6 +775,12 @@ def apply(self, model): if producer is not None and producer.op_type == "Transpose": perms = list(get_by_name(producer.attribute, "perm").ints) if perms == [0, 3, 1, 2]: + # check if the producer is a fork node + # (need to move it past the fork before this transform) + if model.is_fork_node(producer): + model = model.transform(MoveTransposePastFork()) + # topology modified, "ask" ModelWrapper to apply this transform again + return (model, True) old_value = model.get_initializer(n.input[scales_ind]) new_value = np.array( [old_value[idx] for idx in (0, 2, 3, 1)], @@ -813,10 +830,9 @@ class MoveOpPastFork(Transformation): can be merged with nodes in the branches """ - def __init__(self, op_name_list, get_attrs_fxn=lambda x: {}): + def __init__(self, op_name_list): super().__init__() self.ops_to_move = op_name_list - self.get_attrs_fxn = get_attrs_fxn def apply(self, model): graph = model.graph @@ -859,11 +875,9 @@ def apply(self, model): new_param_name = model.make_new_valueinfo_name() new_inp_list = [n.input[0], new_param_name] model.set_initializer(new_param_name, op_init_param) - attrs = self.get_attrs_fxn(n) - # TODO use copy of original node instead to get attrs? - new_node = oh.make_node( - n.op_type, new_inp_list, [new_output_tensor_name], **attrs - ) + new_node = deepcopy(n) + new_node.input[:] = new_inp_list + new_node.output[:] = [new_output_tensor_name] graph.node.insert(node_ind, new_node) node_ind += 1 @@ -901,7 +915,7 @@ def __init__(self): class MoveTransposePastFork(MoveOpPastFork): def __init__(self): - super().__init__(["Transpose"], lambda x: {"perm": get_by_name(x.attribute, "perm").ints}) + super().__init__(["Transpose"]) class MoveMaxPoolPastMultiThreshold(Transformation): From 6223abe86c7d9aee43788825f3c19545dab0ea54 Mon Sep 17 00:00:00 2001 From: Michal Danilowicz Date: Mon, 16 Sep 2024 13:59:14 +0000 Subject: [PATCH 3/5] [Fix] InsertFIFO transform is fixed for the case of the last node in the graph being a fork node --- src/finn/transformation/fpgadataflow/insert_fifo.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/finn/transformation/fpgadataflow/insert_fifo.py b/src/finn/transformation/fpgadataflow/insert_fifo.py index 21fb843052..9ed0f51cd4 100644 --- a/src/finn/transformation/fpgadataflow/insert_fifo.py +++ b/src/finn/transformation/fpgadataflow/insert_fifo.py @@ -268,7 +268,7 @@ def apply(self, model): fifo_input_tensor = oh.make_tensor_value_info( model.make_new_valueinfo_name(), n0_tensor_dtype, - n0.get_normal_output_shape(), + n0.get_normal_output_shape(out_ind), ) graph.value_info.append(fifo_input_tensor) model.set_tensor_datatype(fifo_input_tensor.name, dtype) @@ -294,7 +294,7 @@ def apply(self, model): graph.node.append(fifo_node) # set fifo output tensor as new input tensor of second node - final_node.output[0] = fifo_input_tensor.name + final_node.output[out_ind] = fifo_input_tensor.name else: warnings.warn( """Output FIFO for %s has depth %d and won't From 39a2efef2fca5f356b9d32017227f1a044a0a0da Mon Sep 17 00:00:00 2001 From: Michal Danilowicz Date: Mon, 16 Sep 2024 15:01:54 +0000 Subject: [PATCH 4/5] [Feature] Moving operations past Split added, MoveIdenticalOpPastJoinOp refactored and derived from by MoveTransposePastJoinAdd, MoveMulPastJoinAdd, MoveAddPastJoinAdd, MoveTransposePastJoinConcat, MoveAffinePastJoinConcat --- src/finn/transformation/streamline/reorder.py | 399 ++++++++++++++++-- .../test_move_identical_op_past_join_add.py | 150 +++++++ ...test_move_identical_op_past_join_concat.py | 183 ++++++++ .../test_move_identical_op_past_join_op.py | 114 ----- .../test_move_identical_op_past_split.py | 145 +++++++ 5 files changed, 839 insertions(+), 152 deletions(-) create mode 100644 tests/transformation/streamline/test_move_identical_op_past_join_add.py create mode 100644 tests/transformation/streamline/test_move_identical_op_past_join_concat.py delete mode 100644 tests/transformation/streamline/test_move_identical_op_past_join_op.py create mode 100644 tests/transformation/streamline/test_move_identical_op_past_split.py diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py index 9a7e9d0723..33751cb4d8 100644 --- a/src/finn/transformation/streamline/reorder.py +++ b/src/finn/transformation/streamline/reorder.py @@ -518,7 +518,9 @@ def apply(self, model): class MoveLinearPastEltwiseAdd(Transformation): - """Move linear operations (mul, add) past elementwise add operations where possible. + """ + DEPRECATED, use MoveAddPastJoinAdd() and MoveMulPastJoinAdd() + Move linear operations (mul, add) past elementwise add operations where possible. Specifically,matches and transforms the following patterns: (x*C) + (y*C) -> (x + y) * C (x+A) + (y+B) -> (x + y) + (A + B) @@ -918,6 +920,121 @@ def __init__(self): super().__init__(["Transpose"]) +def permute_shape(shape, perm): + new_shape = np.zeros(len(shape)) + for i, p in enumerate(perm): + new_shape[i] = shape[p] + return [int(el) for el in new_shape] + + +class MoveScalarLinearPastSplit(Transformation): + """ + Move scalar Mul and Add nodes past channel split operation. + """ + + def __init__(self): + super().__init__() + self.ops_to_move = ["Mul", "Add"] + self.fork_ops = ["Split"] + + def apply(self, model): + graph = model.graph + graph_modified = False + node_ind = 0 + for n in graph.node: + node_ind += 1 + # if n.op_type in self.fork_ops and model.is_fork_node(n): + if n.op_type in self.fork_ops: + producer = model.find_producer(n.input[0]) + if producer is not None and producer.op_type in self.ops_to_move: + linear_param = model.get_initializer(producer.input[1]) + # Check if single input + if len(producer.input) != 2 or linear_param is None: + continue + # Check if scalar + if np.prod(linear_param.shape) != 1: + continue + split_outputs = n.output + for split_output_idx, old_split_output in enumerate(split_outputs): + new_mul_node = deepcopy(producer) + new_split_output = model.make_new_valueinfo_name() + model.set_tensor_datatype( + new_split_output, model.get_tensor_datatype(producer.input[0]) + ) + + model.set_tensor_shape( + new_split_output, model.get_tensor_shape(old_split_output) + ) + + n.output[split_output_idx] = new_split_output + new_mul_node.input[0] = new_split_output + new_mul_node.output[0] = old_split_output + + graph.node.insert(node_ind, new_mul_node) + node_ind += 1 + + # remove the mul node + n.input[0] = producer.input[0] + graph.node.remove(producer) + graph_modified = True + + if graph_modified: + model = model.transform(SortGraph(), make_deepcopy=False, cleanup=False) + + return (model, graph_modified) + + +class MoveTransposePastSplit(Transformation): + def __init__(self): + super().__init__() + self.ops_to_move = ["Transpose"] + self.fork_ops = ["Split"] + + def apply(self, model): + graph = model.graph + graph_modified = False + node_ind = 0 + for n in graph.node: + node_ind += 1 + # if n.op_type in self.fork_ops and model.is_fork_node(n): + if n.op_type in self.fork_ops: + producer = model.find_producer(n.input[0]) + if producer is not None and producer.op_type in self.ops_to_move: + initial_perm = get_by_name(producer.attribute, "perm").ints + reverse_perm = np.argsort(initial_perm) + split_outputs = n.output + for split_output_idx, old_split_output in enumerate(split_outputs): + new_trans_node = deepcopy(producer) + new_split_output = model.make_new_valueinfo_name() + old_split_output_shape = model.get_tensor_shape(old_split_output) + model.set_tensor_datatype( + new_split_output, model.get_tensor_datatype(producer.input[0]) + ) + + model.set_tensor_shape( + new_split_output, permute_shape(old_split_output_shape, reverse_perm) + ) + + n.output[split_output_idx] = new_split_output + new_trans_node.input[0] = new_split_output + new_trans_node.output[0] = old_split_output + + graph.node.insert(node_ind, new_trans_node) + node_ind += 1 + + # remove the transpose node and change the split axis + old_split_axis = get_by_name(n.attribute, "axis").i + get_by_name(n.attribute, "axis").i = initial_perm[old_split_axis] + n.input[0] = producer.input[0] + graph.node.remove(producer) + graph_modified = True + + if graph_modified: + model = model.transform(SortGraph(), make_deepcopy=False, cleanup=False) + + return (model, graph_modified) + + class MoveMaxPoolPastMultiThreshold(Transformation): """Move MaxPool nodes past MultiThreshold nodes on linear segments of the graph.""" @@ -1188,13 +1305,8 @@ def apply(self, model): class MoveIdenticalOpPastJoinOp(Transformation): """ - Move identical operations on different branches past the common join node. - This transformation assumes that the identical operations only change the - data layout. For linear operations, see the transformation MoveLinearPastEltwiseAdd. - Specifically, this transformation matches and transforms the following patterns: - f(x) + f(y) -> f(x + y) - where f(.) is currently only supporting 'Transpose', and an 'Add' node is - the join node. + Move multiple identical operations on different branches past the common join node. + It assumes the shape to be preserved by the join op in the default move_node() method """ def __init__(self, identical_op_list, join_node_list): @@ -1202,52 +1314,77 @@ def __init__(self, identical_op_list, join_node_list): self.ops_to_move = identical_op_list self.join_node_op = join_node_list - def move_node(self, model, n, prod0, prod1): - # Found! move one of the identical_ops to output, remove the other one - identical_op0_in0 = prod0.input[0] - identical_op1_in0 = prod1.input[0] - add_in0 = n.input[0] - add_out = n.output[0] + def move_node(self, model, n, producers): + """ + Should be overwritten for some operations + + Returns: + bool: whether moving the node was successful + """ + identical_ops_inputs = [p.input[0] for p in producers] + # join_in0 = n.input[0] + join_out = n.output[0] - # Rewire - n.input[0] = identical_op0_in0 - n.input[1] = identical_op1_in0 + # Rewire join op inputs + for i in range(len(n.input)): + n.input[i] = identical_ops_inputs[i] # Output tensor of the join node must have the same shape as # its input tensor (original shape is preserved) - new_shape = model.get_tensor_shape(identical_op0_in0) + new_join_output = model.make_new_valueinfo_name() + new_shape = model.get_tensor_shape(identical_ops_inputs[0]) + new_layout = model.get_tensor_layout(identical_ops_inputs[0]) # Set new tensor shape - model.set_tensor_shape(tensor_name=add_in0, tensor_shape=new_shape) - - n.output[0] = add_in0 - prod0.input[0] = add_in0 - prod0.output[0] = add_out - - model.graph.node.remove(prod1) + model.set_tensor_shape(new_join_output, new_shape) + if new_layout: + model.set_tensor_layout(new_join_output, new_layout) + + # Rewire join op outputs (reuse the first join input tensor) + n.output[0] = new_join_output + producers[0].input[0] = new_join_output + producers[0].output[0] = join_out + + for prod in producers[1:]: + model.graph.node.remove(prod) + + return True + + def are_producers_identical(self, model, producers): + """ + Checks only op_types + Should be overwritten for additional checks + """ + op_types = [prod.op_type for prod in producers] + for op in op_types: + if op != op_types[0]: + return False + return True def apply(self, model): graph = model.graph graph_modified = False for n in graph.node: if n.op_type in self.join_node_op and model.is_join_node(n): - in0 = n.input[0] - in1 = n.input[1] - if in0 is None or in1 is None: + inputs = n.input + if None in inputs: continue - prod0 = model.find_producer(in0) - prod1 = model.find_producer(in1) - # Checks if the join node is preceded by - # two different, but identical operations - if prod0 == prod1: + producers = [model.find_producer(inp) for inp in inputs] + if producers[0].op_type not in self.ops_to_move: + continue + identical_ops = self.are_producers_identical(model, producers) + if not identical_ops: + warnings.warn("Producers not identical, skipping") continue - identical_op = prod0.op_type == prod1.op_type - - if identical_op and prod0.op_type in self.ops_to_move: - self.move_node(model, n, prod0, prod1) - graph_modified = True + # check for producers that are fork nodes (need to fork them before our transform) + for prod in producers: + if model.is_fork_node(prod) and not model.is_join_node(prod): + model = model.transform(MoveOpPastFork(self.ops_to_move)) + # topology modified, "ask" ModelWrapper to apply this transform again + return (model, True) + graph_modified = self.move_node(model, n, producers) if graph_modified: model = model.transform(SortGraph(), make_deepcopy=False, cleanup=False) @@ -1258,3 +1395,189 @@ def apply(self, model): class MoveTransposePastJoinAdd(MoveIdenticalOpPastJoinOp): def __init__(self): super().__init__(["Transpose"], ["Add"]) + + def are_producers_identical(self, model, producers): + if not super().are_producers_identical(model, producers): + return False + first_perm = get_by_name(producers[0].attribute, "perm").ints + for producer in producers: + if first_perm != get_by_name(producer.attribute, "perm").ints: + False + return True + + +class MoveMulPastJoinAdd(MoveIdenticalOpPastJoinOp): + def __init__(self): + super().__init__(["Mul"], ["Add"]) + + def are_producers_identical(self, model, producers): + if not super().are_producers_identical(model, producers): + return False + first_mul = model.get_initializer(producers[0].input[1]) + if first_mul is None: + return False + for producer in producers: + if first_mul != model.get_initializer(producer.input[1]): + return False + return True + + +class MoveAddPastJoinAdd(MoveIdenticalOpPastJoinOp): + def __init__(self): + super().__init__(["Add"], ["Add"]) + + def are_producers_identical(self, model, producers): + if not super().are_producers_identical(model, producers): + return False + for producer in producers: + if model.get_initializer(producer.input[1]) is None: + return False + return True + + def move_node(self, model, n, producers): + """ + We use the base move_node method to move the first producer + past the join node (and delete the rest) + """ + add_inits = [model.get_initializer(producer.input[1]) for producer in producers] + new_init = np.sum(add_inits) + model.set_initializer(producers[0].input[1], new_init) + super().move_node(model, n, producers) + + return True + + +class MoveTransposePastJoinConcat(MoveIdenticalOpPastJoinOp): + def __init__(self): + super().__init__(["Transpose"], ["Concat"]) + + def are_producers_identical(self, model, producers): + if not super().are_producers_identical(model, producers): + return False + first_perm = get_by_name(producers[0].attribute, "perm").ints + for producer in producers: + if first_perm != get_by_name(producer.attribute, "perm").ints: + False + return True + + def move_node(self, model, n, producers): + trans_inputs = [prod.input[0] for prod in producers] + # concat_in0 = n.input[0] + concat_out = n.output[0] + # Rewire concat inputs + for i in range(len(n.input)): + n.input[i] = trans_inputs[i] + + new_concat_out = model.make_new_valueinfo_name() # reuse tensor + # reverse the permutation of the concat output + transpose_perm = get_by_name(producers[0].attribute, "perm").ints + reverse_perm = np.argsort(transpose_perm) + new_concat_out_shape = permute_shape(model.get_tensor_shape(concat_out), reverse_perm) + new_concat_out_layout = model.get_tensor_layout(trans_inputs[0]) + # Set tensor layout and shape of the new concatenation output + model.set_tensor_shape(new_concat_out, new_concat_out_shape) + if new_concat_out_layout: + model.set_tensor_layout(new_concat_out, new_concat_out_layout) + # Change concatenation axis + old_concat_axis = get_by_name(n.attribute, "axis").i + get_by_name(n.attribute, "axis").i = transpose_perm[old_concat_axis] + + # Rewire concat output + n.output[0] = new_concat_out + producers[0].input[0] = new_concat_out + producers[0].output[0] = concat_out + + for prod in producers[1:]: + model.graph.node.remove(prod) + + return True + + +class MoveAffinePastJoinConcat(MoveIdenticalOpPastJoinOp): + """ + Applies to scalar linear or channelwise affine ops with the same parameter value + """ + + def __init__(self, linear_ops=["Mul", "Add"]): + super().__init__(linear_ops, ["Concat"]) + + def are_producers_identical_scalar_ops(self, model, producers): + first_param = model.get_initializer(producers[0].input[1]) + for producer in producers: + producer_param = model.get_initializer(producer.input[1]) + if (first_param != producer_param).any() or np.prod(producer_param.shape) != 1: + return False + + return True + + def are_producers_channelwise_ops(self, channel_dim, model, producers): + for producer in producers: + producer_input = producer.input[0] + num_channels = model.get_tensor_shape(producer_input)[channel_dim] + producer_param = model.get_initializer(producer.input[1]) + if ( + len(producer_param.shape) < channel_dim + or producer_param.shape[channel_dim] != num_channels + ): + return False + + return True + + def move_node(self, model, n, producers): + # check if single input + for producer in producers: + producer_init = model.get_initializer(producer.input[1]) + if len(producer.input) != 2 or producer_init is None: + warnings.warn("Producer found that is not single-input, skipping") + return False + + # decide if producers are identical scalar ops or channelwise ops + channelwise_op = False + identical_scalar_op = self.are_producers_identical_scalar_ops(model, producers) + if not identical_scalar_op: + channel_dim = get_by_name(n.attribute, "axis").i + channelwise_op = self.are_producers_channelwise_ops(channel_dim, model, producers) + if not channelwise_op: + warnings.warn( + "Producers are neither identical scalar ops nor channelwise ops, skipping" + ) + return False + + # Rewire concat inputs + producers_inputs = [prod.input[0] for prod in producers] + concat_out = n.output[0] + for i in range(len(n.input)): + n.input[i] = producers_inputs[i] + # Set tensor layout and shape of the new concatenation output + new_concat_out = model.make_new_valueinfo_name() + new_concat_out_layout = model.get_tensor_layout(producers_inputs[0]) + model.set_tensor_shape(new_concat_out, model.get_tensor_shape(concat_out)) + if new_concat_out_layout: + model.set_tensor_layout(new_concat_out, new_concat_out_layout) + model.set_tensor_datatype(new_concat_out, model.get_tensor_datatype(producers_inputs[0])) + + if channelwise_op: + # concatenate op params of producers into one mul tensor + producers_params = [model.get_initializer(prod.input[1]) for prod in producers] + new_mul_tensor = np.concatenate(producers_params, axis=channel_dim) + model.set_initializer(producers[0].input[1], new_mul_tensor) + + # Rewire concat output + n.output[0] = new_concat_out + producers[0].input[0] = new_concat_out + producers[0].output[0] = concat_out + + for prod in producers[1:]: + model.graph.node.remove(prod) + + return True + + +class MoveMulPastJoinConcat(MoveAffinePastJoinConcat): + def __init__(self): + super().__init__(["Mul"]) + + +class MoveAddPastJoinConcat(MoveAffinePastJoinConcat): + def __init__(self): + super().__init__(["Add"]) diff --git a/tests/transformation/streamline/test_move_identical_op_past_join_add.py b/tests/transformation/streamline/test_move_identical_op_past_join_add.py new file mode 100644 index 0000000000..7226d31589 --- /dev/null +++ b/tests/transformation/streamline/test_move_identical_op_past_join_add.py @@ -0,0 +1,150 @@ +# Copyright (c) 2020, Xilinx +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of FINN nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import pytest + +import numpy as np +from onnx import TensorProto +from onnx import helper as oh +from qonnx.core.modelwrapper import ModelWrapper +from qonnx.util.basic import gen_finn_dt_tensor, qonnx_make_model + +import finn.core.onnx_exec as oxe +from finn.transformation.streamline.reorder import ( + MoveAddPastJoinAdd, + MoveMulPastJoinAdd, + MoveTransposePastJoinAdd, +) + + +def create_add_model(identical_op): + perm = None + if "Transpose" in identical_op: + perm = identical_op.split("_")[1] + identical_op = identical_op.split("_")[0] + perm = [int(char) for char in perm] + if perm == [0, 2, 3, 1]: + in_shape = [1, 64, 10, 9] + out_shape = [1, 10, 9, 64] + elif perm == [0, 3, 1, 2]: + in_shape = [1, 10, 9, 64] + out_shape = [1, 64, 10, 9] + else: + in_shape = [1, 64, 10, 9] + out_shape = in_shape + op_value = 1.5 + + op1_node = oh.make_node(identical_op, inputs=["in1"], outputs=["op1_out"]) + + op2_node = oh.make_node(identical_op, inputs=["in2"], outputs=["op2_out"]) + + if identical_op == "Transpose": + new_attr = oh.make_attribute("perm", perm) + op1_node.attribute.append(new_attr) + op2_node.attribute.append(new_attr) + elif identical_op == "Mul" or identical_op == "Add": + op1_init = oh.make_tensor_value_info("op1_param", TensorProto.FLOAT, [1]) + op2_init = oh.make_tensor_value_info("op2_param", TensorProto.FLOAT, [1]) + op1_node.input.append(op1_init.name) + op2_node.input.append(op2_init.name) + + add_node = oh.make_node("Add", inputs=["op1_out", "op2_out"], outputs=["out_join1"]) + + in1 = oh.make_tensor_value_info("in1", TensorProto.FLOAT, in_shape) + in2 = oh.make_tensor_value_info("in2", TensorProto.FLOAT, in_shape) + op1_out = oh.make_tensor_value_info("op1_out", TensorProto.FLOAT, out_shape) + op2_out = oh.make_tensor_value_info("op2_out", TensorProto.FLOAT, out_shape) + out_join1 = oh.make_tensor_value_info("out_join1", TensorProto.FLOAT, out_shape) + + graph = oh.make_graph( + nodes=[op1_node, op2_node, add_node], + name="test_graph", + inputs=[in1, in2], + outputs=[out_join1], + value_info=[ + op1_out, + op2_out, + ], + ) + + onnx_model = qonnx_make_model(graph, producer_name="test_model") + model = ModelWrapper(onnx_model) + if identical_op == "Mul" or identical_op == "Add": + model.set_initializer("op1_param", np.array(op_value).astype(np.float32)) + model.set_initializer("op2_param", np.array(op_value).astype(np.float32)) + + return model + + +transform_dict = { + "Transpose_0231": MoveTransposePastJoinAdd(), + "Transpose_0312": MoveTransposePastJoinAdd(), + "Mul": MoveMulPastJoinAdd(), + "Add": MoveAddPastJoinAdd(), +} + + +@pytest.mark.streamline +# Permutation of transpose node +@pytest.mark.parametrize("identical_op", ["Transpose_0231", "Transpose_0312", "Mul", "Add"]) +def test_move_identical_op_past_join_op(identical_op): + model = create_add_model(identical_op) + # build_dir = os.environ["FINN_BUILD_DIR"] + # model.save(join(build_dir, "add_pytest_model_{}.onnx".format(identical_op))) + + # Create input data + input0_tensor_name = model.graph.input[0].name + input1_tensor_name = model.graph.input[1].name + + # Note: it is assumed that both tensors have the same shape and data type + input_shape = model.get_tensor_shape(input0_tensor_name) + input_dtype = model.get_tensor_datatype(input0_tensor_name) + input_val = gen_finn_dt_tensor(input_dtype, input_shape) + input_dict = {} + input_dict[input0_tensor_name] = input_val + input_dict[input1_tensor_name] = input_val + + model_transformed = model.transform(transform_dict[identical_op]) + # model_transformed.save(join(build_dir, "add_pytest_model_{}_trans.onnx".format(identical_op))) + + assert oxe.compare_execution(model, model_transformed, input_dict) + + # Check if order changed + node0_optype_model = model.find_consumers(model.graph.input[0].name)[0].op_type + node1_optype_model = model.find_consumers(model.graph.input[1].name)[0].op_type + node0_optype_model_transformed = model_transformed.find_consumers( + model_transformed.graph.input[0].name + )[0].op_type + node1_optype_model_transformed = model_transformed.find_consumers( + model_transformed.graph.input[1].name + )[0].op_type + last_node_optype_model_transformed = model_transformed.find_producer( + model_transformed.graph.output[0].name + ).op_type + assert node0_optype_model == last_node_optype_model_transformed + assert node1_optype_model == last_node_optype_model_transformed + assert node0_optype_model_transformed == node1_optype_model_transformed == "Add" diff --git a/tests/transformation/streamline/test_move_identical_op_past_join_concat.py b/tests/transformation/streamline/test_move_identical_op_past_join_concat.py new file mode 100644 index 0000000000..2dcf90d10a --- /dev/null +++ b/tests/transformation/streamline/test_move_identical_op_past_join_concat.py @@ -0,0 +1,183 @@ +# Copyright (c) 2020, Xilinx +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of FINN nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import pytest + +import numpy as np +import os +from onnx import TensorProto +from onnx import helper as oh +from os.path import join +from qonnx.core.modelwrapper import ModelWrapper +from qonnx.util.basic import gen_finn_dt_tensor, qonnx_make_model + +import finn.core.onnx_exec as oxe +from finn.transformation.streamline.reorder import ( + MoveAddPastJoinConcat, + MoveMulPastJoinConcat, + MoveTransposePastJoinConcat, +) + + +def create_concat_model(identical_op): + perm = None + channelwise = False + if "Transpose" in identical_op: + perm = identical_op.split("_")[1] + identical_op = identical_op.split("_")[0] + perm = [int(char) for char in perm] + if "channelwise" in identical_op: + channelwise = True + identical_op = identical_op.split("_")[0] + if perm == [0, 2, 3, 1]: + in_shape1 = [1, 64, 10, 9] + in_shape2 = [1, 32, 10, 9] + out_shape1 = [1, 10, 9, 64] + out_shape2 = [1, 10, 9, 32] + out_join_shape = [1, 10, 9, 96] + concat_axis = 3 + elif perm == [0, 3, 1, 2]: + in_shape1 = [1, 10, 9, 64] + in_shape2 = [1, 10, 9, 32] + out_shape1 = [1, 64, 10, 9] + out_shape2 = [1, 32, 10, 9] + out_join_shape = [1, 96, 10, 9] + concat_axis = 1 + else: + in_shape1 = [1, 64, 10, 9] + in_shape2 = [1, 32, 10, 9] + out_shape1 = in_shape1 + out_shape2 = in_shape2 + out_join_shape = [1, 96, 10, 9] + concat_axis = 1 + if channelwise: + op1_param_shape = [1, 64, 1, 1] + op2_param_shape = [1, 32, 1, 1] + op1_param = np.ones((1, 64, 1, 1)) * 2 + op2_param = np.ones((1, 32, 1, 1)) * 3 + else: + op1_param_shape = [1] + op2_param_shape = [1] + op1_param = 1.5 + op2_param = 1.5 + + op1_node = oh.make_node(identical_op, inputs=["in1"], outputs=["op1_out"]) + + op2_node = oh.make_node(identical_op, inputs=["in2"], outputs=["op2_out"]) + + if identical_op == "Transpose": + new_attr = oh.make_attribute("perm", perm) + op1_node.attribute.append(new_attr) + op2_node.attribute.append(new_attr) + elif identical_op == "Mul" or identical_op == "Add": + op1_init = oh.make_tensor_value_info("op1_param", TensorProto.FLOAT, op1_param_shape) + op2_init = oh.make_tensor_value_info("op2_param", TensorProto.FLOAT, op2_param_shape) + op1_node.input.append(op1_init.name) + op2_node.input.append(op2_init.name) + + concat_node = oh.make_node( + "Concat", inputs=["op1_out", "op2_out"], outputs=["out_join1"], axis=concat_axis + ) + + in1 = oh.make_tensor_value_info("in1", TensorProto.FLOAT, in_shape1) + in2 = oh.make_tensor_value_info("in2", TensorProto.FLOAT, in_shape2) + op1_out = oh.make_tensor_value_info("op1_out", TensorProto.FLOAT, out_shape1) + op2_out = oh.make_tensor_value_info("op2_out", TensorProto.FLOAT, out_shape2) + out_join1 = oh.make_tensor_value_info("out_join1", TensorProto.FLOAT, out_join_shape) + + graph = oh.make_graph( + nodes=[op1_node, op2_node, concat_node], + name="test_graph", + inputs=[in1, in2], + outputs=[out_join1], + value_info=[ + op1_out, + op2_out, + ], + ) + + onnx_model = qonnx_make_model(graph, producer_name="test_model") + model = ModelWrapper(onnx_model) + if identical_op == "Mul" or identical_op == "Add": + model.set_initializer("op1_param", np.array(op1_param).astype(np.float32)) + model.set_initializer("op2_param", np.array(op2_param).astype(np.float32)) + + return model + + +transform_dict = { + "Transpose_0231": MoveTransposePastJoinConcat(), + "Transpose_0312": MoveTransposePastJoinConcat(), + "Mul": MoveMulPastJoinConcat(), + "Mul_channelwise": MoveMulPastJoinConcat(), + "Add": MoveAddPastJoinConcat(), + "Add_channelwise": MoveAddPastJoinConcat(), +} + + +@pytest.mark.streamline +# Permutation of transpose node +@pytest.mark.parametrize( + "identical_op", + ["Transpose_0231", "Transpose_0312", "Mul", "Add", "Mul_channelwise", "Add_channelwise"], +) +def test_move_identical_op_past_join_concat(identical_op): + model = create_concat_model(identical_op) + build_dir = os.environ["FINN_BUILD_DIR"] + model.save(join(build_dir, "concat_pytest_model_{}.onnx".format(identical_op))) + + # Create input data + input0_tensor_name = model.graph.input[0].name + input1_tensor_name = model.graph.input[1].name + + # Note: it is assumed that both tensors have the same shape and data type + input_dict = {} + input_dict[input0_tensor_name] = gen_finn_dt_tensor( + model.get_tensor_datatype(input0_tensor_name), model.get_tensor_shape(input0_tensor_name) + ) + input_dict[input1_tensor_name] = gen_finn_dt_tensor( + model.get_tensor_datatype(input1_tensor_name), model.get_tensor_shape(input1_tensor_name) + ) + + model_transformed = model.transform(transform_dict[identical_op]) + model_transformed.save( + join(build_dir, "concat_pytest_model_{}_trans.onnx".format(identical_op)) + ) + + assert oxe.compare_execution(model, model_transformed, input_dict) + + # Check if order changed + node0_input0_model = model.find_consumers(model.graph.input[0].name)[0].op_type + node1_input1_model = model.find_consumers(model.graph.input[1].name)[0].op_type + node0_input0_model_transformed = model_transformed.find_consumers( + model_transformed.graph.input[0].name + )[0].op_type + node1_input1_model_transformed = model_transformed.find_consumers( + model_transformed.graph.input[1].name + )[0].op_type + assert node0_input0_model != node0_input0_model_transformed + assert node1_input1_model != node1_input1_model_transformed diff --git a/tests/transformation/streamline/test_move_identical_op_past_join_op.py b/tests/transformation/streamline/test_move_identical_op_past_join_op.py deleted file mode 100644 index dd83681fc2..0000000000 --- a/tests/transformation/streamline/test_move_identical_op_past_join_op.py +++ /dev/null @@ -1,114 +0,0 @@ -# Copyright (c) 2020, Xilinx -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of FINN nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import pytest - -from onnx import TensorProto -from onnx import helper as oh -from qonnx.core.modelwrapper import ModelWrapper -from qonnx.util.basic import gen_finn_dt_tensor, qonnx_make_model - -import finn.core.onnx_exec as oxe -from finn.transformation.streamline.reorder import MoveTransposePastJoinAdd - - -def create_model(perm): - if perm == [0, 3, 1, 2]: - in_shape = [1, 128, 1, 256] - out_shape = [1, 256, 128, 1] - if perm == [0, 2, 3, 1]: - in_shape = [1, 256, 128, 1] - out_shape = [1, 128, 1, 256] - - Transpose1_node = oh.make_node( - "Transpose", inputs=["in_transpose1"], outputs=["out_transpose1"], perm=perm - ) - - Transpose2_node = oh.make_node( - "Transpose", inputs=["in_transpose2"], outputs=["out_transpose2"], perm=perm - ) - - Join1_node = oh.make_node( - "Add", inputs=["out_transpose1", "out_transpose2"], outputs=["out_join1"] - ) - - in_transpose1 = oh.make_tensor_value_info("in_transpose1", TensorProto.FLOAT, in_shape) - in_transpose2 = oh.make_tensor_value_info("in_transpose2", TensorProto.FLOAT, in_shape) - out_transpose1 = oh.make_tensor_value_info("out_transpose1", TensorProto.FLOAT, out_shape) - out_transpose2 = oh.make_tensor_value_info("out_transpose2", TensorProto.FLOAT, out_shape) - out_join1 = oh.make_tensor_value_info("out_join1", TensorProto.FLOAT, out_shape) - - graph = oh.make_graph( - nodes=[Transpose1_node, Transpose2_node, Join1_node], - name="test_graph", - inputs=[in_transpose1, in_transpose2], - outputs=[out_join1], - value_info=[ - out_transpose1, - out_transpose2, - ], - ) - - onnx_model = qonnx_make_model(graph, producer_name="test_model") - model = ModelWrapper(onnx_model) - - return model - - -@pytest.mark.streamline -# Permutation of transpose node -@pytest.mark.parametrize("perm", [[0, 3, 1, 2], [0, 2, 3, 1]]) -def test_move_identical_op_past_join_op(perm): - model = create_model(perm) - - # Create input data - input0_tensor_name = model.graph.input[0].name - input1_tensor_name = model.graph.input[1].name - - # Note: it is assumed that both tensors have the same shape and data type - input_shape = model.get_tensor_shape(input0_tensor_name) - input_dtype = model.get_tensor_datatype(input0_tensor_name) - input_val = gen_finn_dt_tensor(input_dtype, input_shape) - input_dict = {} - input_dict[input0_tensor_name] = input_val - input_dict[input1_tensor_name] = input_val - - model_transformed = model.transform(MoveTransposePastJoinAdd()) - - assert oxe.compare_execution(model, model_transformed, input_dict) - - # Check if order changed - node0_input0_model = model.find_consumers(model.graph.input[0].name)[0].op_type - node1_input1_model = model.find_consumers(model.graph.input[1].name)[0].op_type - node0_input0_model_transformed = model_transformed.find_consumers( - model_transformed.graph.input[0].name - )[0].op_type - node1_input1_model_transformed = model_transformed.find_consumers( - model_transformed.graph.input[1].name - )[0].op_type - assert node0_input0_model != node0_input0_model_transformed - assert node1_input1_model != node1_input1_model_transformed diff --git a/tests/transformation/streamline/test_move_identical_op_past_split.py b/tests/transformation/streamline/test_move_identical_op_past_split.py new file mode 100644 index 0000000000..a104f179be --- /dev/null +++ b/tests/transformation/streamline/test_move_identical_op_past_split.py @@ -0,0 +1,145 @@ +# Copyright (c) 2020, Xilinx +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of FINN nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import pytest + +import numpy as np +from onnx import TensorProto +from onnx import helper as oh +from qonnx.core.modelwrapper import ModelWrapper +from qonnx.transformation.general import GiveUniqueNodeNames +from qonnx.util.basic import gen_finn_dt_tensor + +import finn.core.onnx_exec as oxe +from finn.transformation.streamline.reorder import ( + MoveScalarLinearPastSplit, + MoveTransposePastSplit, +) + + +def create_split_model(identical_op): + perm = None + if "Transpose" in identical_op: + perm = identical_op.split("_")[1] + identical_op = identical_op.split("_")[0] + perm = [int(char) for char in perm] + if perm == [0, 2, 3, 1]: + in_shape = [1, 96, 10, 9] + out_shape = [1, 10, 9, 96] + out1_split_shape = [1, 10, 9, 32] + out2_split_shape = [1, 10, 9, 64] + split_axis = 3 + elif perm == [0, 3, 1, 2]: + in_shape = [1, 10, 9, 96] + out_shape = [1, 96, 10, 9] + out1_split_shape = [1, 32, 10, 9] + out2_split_shape = [1, 64, 10, 9] + split_axis = 1 + else: + in_shape = [1, 96, 10, 9] + out_shape = in_shape + out1_split_shape = [1, 32, 10, 9] + out2_split_shape = [1, 64, 10, 9] + split_axis = 1 + op_value = 1.5 + split = [32, 64] + + op_node = oh.make_node(identical_op, inputs=["in1"], outputs=["op_out"]) + + if identical_op == "Transpose": + new_attr = oh.make_attribute("perm", perm) + op_node.attribute.append(new_attr) + elif identical_op == "Mul" or identical_op == "Add": + op_init = oh.make_tensor_value_info("op_param", TensorProto.FLOAT, [1]) + op_node.input.append(op_init.name) + + in1 = oh.make_tensor_value_info("in1", TensorProto.FLOAT, in_shape) + op_out = oh.make_tensor_value_info("op_out", TensorProto.FLOAT, out_shape) + out1_split = oh.make_tensor_value_info("out1_split", TensorProto.FLOAT, out1_split_shape) + out2_split = oh.make_tensor_value_info("out2_split", TensorProto.FLOAT, out2_split_shape) + split_init = oh.make_tensor_value_info("split", TensorProto.INT64, [2]) + + split_node = oh.make_node( + "Split", [op_out.name, split_init.name], [out1_split.name, out2_split.name], axis=split_axis + ) + + graph = oh.make_graph( + nodes=[op_node, split_node], + name="test_graph", + inputs=[in1], + outputs=[out1_split, out2_split], + value_info=[op_out], + ) + + model = oh.make_model(graph) + model = ModelWrapper(model) + model.set_initializer(split_init.name, np.array(split, dtype=np.int64)) + if identical_op == "Mul" or identical_op == "Add": + model.set_initializer(op_init.name, np.array(op_value).astype(np.float32)) + model = model.transform(GiveUniqueNodeNames()) + + return model + + +transform_dict = { + "Transpose_0231": MoveTransposePastSplit(), + "Transpose_0312": MoveTransposePastSplit(), + "Mul": MoveScalarLinearPastSplit(), + "Add": MoveScalarLinearPastSplit(), +} + + +@pytest.mark.streamline +# Permutation of transpose node +@pytest.mark.parametrize("identical_op", ["Transpose_0231", "Transpose_0312", "Mul", "Add"]) +def test_move_identical_op_past_join_concat(identical_op): + model = create_split_model(identical_op) + # build_dir = os.environ["FINN_BUILD_DIR"] + # model.save(join(build_dir, "split_pytest_model_{}.onnx".format(identical_op))) + + # Create input data + input0_tensor_name = model.graph.input[0].name + + # Note: it is assumed that both tensors have the same shape and data type + input_dict = {} + input_dict[input0_tensor_name] = gen_finn_dt_tensor( + model.get_tensor_datatype(input0_tensor_name), model.get_tensor_shape(input0_tensor_name) + ) + + model_transformed = model.transform(transform_dict[identical_op]) + # model_transformed.save( + # join(build_dir, "split_pytest_model_{}_trans.onnx".format(identical_op)) + # ) + + assert oxe.compare_execution(model, model_transformed, input_dict) + + # Check if order changed + node0_input0_model = model.find_consumers(model.graph.input[0].name)[0].op_type + node0_input0_model_transformed = model_transformed.find_consumers( + model_transformed.graph.input[0].name + )[0].op_type + assert node0_input0_model != node0_input0_model_transformed From 51a9199858166673893f8d7bea0d3f8805769232 Mon Sep 17 00:00:00 2001 From: Michal Danilowicz Date: Wed, 18 Sep 2024 14:50:31 +0000 Subject: [PATCH 5/5] [Deprecated] MoveLinearPastEltwiseAdd() removed from the codebase --- src/finn/transformation/streamline/reorder.py | 81 -------- .../streamline/test_linear_past_eltwise.py | 192 ------------------ 2 files changed, 273 deletions(-) delete mode 100644 tests/transformation/streamline/test_linear_past_eltwise.py diff --git a/src/finn/transformation/streamline/reorder.py b/src/finn/transformation/streamline/reorder.py index 33751cb4d8..8688145453 100644 --- a/src/finn/transformation/streamline/reorder.py +++ b/src/finn/transformation/streamline/reorder.py @@ -517,87 +517,6 @@ def apply(self, model): return (model, graph_modified) -class MoveLinearPastEltwiseAdd(Transformation): - """ - DEPRECATED, use MoveAddPastJoinAdd() and MoveMulPastJoinAdd() - Move linear operations (mul, add) past elementwise add operations where possible. - Specifically,matches and transforms the following patterns: - (x*C) + (y*C) -> (x + y) * C - (x+A) + (y+B) -> (x + y) + (A + B) - where x and y are dynamic inputs, A, B, C are constant tensors (in general). - """ - - def move_node(self, graph, n, prod0, prod1, node_ind): - # found! move one of the muls to output, remove the other one - lin0_in0 = prod0.input[0] - lin1_in0 = prod1.input[0] - in0 = n.input[0] - out = n.output[0] - # TODO: check shapes don't change through scalar mul or add - # connect the eltwise add inputs to mul inputs - n.input[0] = lin0_in0 - n.input[1] = lin1_in0 - # connect mul0 output to eltwise add output - prod0.output[0] = out - # connect the input of mul0 and output of eltwise add together - n.output[0] = in0 - prod0.input[0] = in0 - # move prod0 node past eltwise add node, and remove prod1 - graph.node.remove(prod1) - graph.node.remove(prod0) - graph.node.insert(node_ind - 2, prod0) - - def apply(self, model): - graph = model.graph - node_ind = 0 - graph_modified = False - nodes = [n for n in graph.node] - for n in nodes: - node_ind += 1 - if n.op_type == "Add": - # check for tensors on both inputs (eltwise add) - # scalar add has an initializer on one input - in0 = n.input[0] - in1 = n.input[1] - if in0 is None or in1 is None: - continue - A = model.get_initializer(in0) - B = model.get_initializer(in1) - if A is not None or B is not None: - continue - # check for mul with same initializer on both inputs - prod0 = model.find_producer(in0) - prod1 = model.find_producer(in1) - # Also check case when both branches are empty and come - # from the same node: (prod0 == prod1) - # Other transform should handle that - if prod0 is None or prod1 is None or (prod0 == prod1): - continue - if len(prod0.input) < 2 or len(prod1.input) < 2: - continue - init0 = model.get_initializer(prod0.input[1]) - init1 = model.get_initializer(prod1.input[1]) - # if either initializer is None, skip - if init0 is None or init1 is None: - continue - if prod0.op_type == "Mul" and prod1.op_type == "Mul": - if np.array_equal(init0, init1): - self.move_node(graph, n, prod0, prod1, node_ind) - node_ind -= 1 - graph_modified = True - elif prod0.op_type == "Add" and prod1.op_type == "Add": - init = init0 + init1 - # update initializer of prod0, which we'll move - model.set_initializer(prod0.input[1], init) - self.move_node(graph, n, prod0, prod1, node_ind) - node_ind -= 1 - graph_modified = True - else: - continue - model = model.transform(InferShapes()) - return (model, graph_modified) - - class MoveScalarLinearPastInvariants(Transformation): """Move scalar linear operations (mul, add) past functions which are invariant to them. Specifically, matches and transforms the following patterns: diff --git a/tests/transformation/streamline/test_linear_past_eltwise.py b/tests/transformation/streamline/test_linear_past_eltwise.py deleted file mode 100644 index 70fc395652..0000000000 --- a/tests/transformation/streamline/test_linear_past_eltwise.py +++ /dev/null @@ -1,192 +0,0 @@ -# Copyright (c) 2020, Xilinx -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of FINN nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import pytest - -import numpy as np -import os -from onnx import TensorProto, helper -from qonnx.core.modelwrapper import ModelWrapper -from qonnx.transformation.fold_constants import FoldConstants -from qonnx.transformation.general import GiveReadableTensorNames, GiveUniqueNodeNames -from qonnx.transformation.infer_shapes import InferShapes -from qonnx.util.basic import qonnx_make_model - -import finn.core.onnx_exec as oxe -from finn.transformation.streamline.reorder import MoveLinearPastEltwiseAdd - -export_onnx_path = "test_linear_past_eltwise.onnx" -np_default_dtype = np.float32 - -# construct a synthetic graph to test: -# topk insertion, topk conversion to hls, add conversion to hls -# graph should just be a sum - - -def make_model(shape): - inp1 = helper.make_tensor_value_info("inp1", TensorProto.FLOAT, shape) - inp2 = helper.make_tensor_value_info("inp2", TensorProto.FLOAT, shape) - inp1_add = helper.make_tensor_value_info("inp1_add", TensorProto.FLOAT, shape) - inp1_add_ct = helper.make_tensor_value_info("inp1_add_ct", TensorProto.FLOAT, [1]) - inp2_add = helper.make_tensor_value_info("inp2_add", TensorProto.FLOAT, shape) - inp2_add_ct = helper.make_tensor_value_info("inp2_add_ct", TensorProto.FLOAT, [1]) - inp1_mul = helper.make_tensor_value_info("inp1_mul", TensorProto.FLOAT, shape) - inp1_mul_ct = helper.make_tensor_value_info("inp1_mul_ct", TensorProto.FLOAT, [1]) - inp2_mul = helper.make_tensor_value_info("inp2_mul", TensorProto.FLOAT, shape) - inp2_mul_ct = helper.make_tensor_value_info("inp2_mul_ct", TensorProto.FLOAT, [1]) - outp = helper.make_tensor_value_info("outp", TensorProto.FLOAT, shape) - - add1_node = helper.make_node("Add", [inp1.name, inp1_add_ct.name], [inp1_add.name]) - add2_node = helper.make_node("Add", [inp2.name, inp2_add_ct.name], [inp2_add.name]) - mul1_node = helper.make_node("Mul", [inp1_add.name, inp1_mul_ct.name], [inp1_mul.name]) - mul2_node = helper.make_node("Mul", [inp2_add.name, inp2_mul_ct.name], [inp2_mul.name]) - eltwise_add_node = helper.make_node("Add", [inp1_mul.name, inp2_mul.name], [outp.name]) - graph = helper.make_graph( - nodes=[add1_node, add2_node, mul1_node, mul2_node, eltwise_add_node], - name="graph", - inputs=[inp1, inp2], - outputs=[outp], - ) - - model = qonnx_make_model(graph, producer_name="add-model") - model = ModelWrapper(model) - - # set initializers for scalar add/mul nodes - model.set_initializer(add1_node.input[1], np.array([7.0], dtype=np_default_dtype)) - model.set_initializer(add2_node.input[1], np.array([8.0], dtype=np_default_dtype)) - model.set_initializer(mul1_node.input[1], np.array([3.0], dtype=np_default_dtype)) - model.set_initializer(mul2_node.input[1], np.array([3.0], dtype=np_default_dtype)) - - return model - - -@pytest.mark.streamline -# channels -@pytest.mark.parametrize("ch", [64]) -# ifmdim -@pytest.mark.parametrize("ifmdim", [-1, 7]) -def test_linear_past_eltwise_add(ch, ifmdim): - # generate test vectors of correct shape - if ifmdim == -1: - input_tensor_shape = (1, ch) - else: - input_tensor_shape = (1, ch, ifmdim, ifmdim) - - model = make_model(input_tensor_shape) - model.save(export_onnx_path) - model = ModelWrapper(export_onnx_path) - model = model.transform(InferShapes()) - model = model.transform(FoldConstants()) - model = model.transform(GiveUniqueNodeNames()) - model = model.transform(GiveReadableTensorNames()) - - x1 = np.random.randn(*input_tensor_shape).astype(np.float32) - x2 = np.random.randn(*input_tensor_shape).astype(np.float32) - - # generate expected value from streamlined net - input_dict = {model.graph.input[0].name: x1, model.graph.input[1].name: x2} - - output_dict = oxe.execute_onnx(model, input_dict, True) - produced_sum = output_dict[model.graph.output[0].name] - expected_sum = 3.0 * ((x1 + x2) + 15.0) - assert np.isclose(expected_sum, produced_sum, atol=1e-3).all() - assert len(model.get_nodes_by_op_type("Add")) == 3 - assert len(model.get_nodes_by_op_type("Mul")) == 2 - - model = model.transform(MoveLinearPastEltwiseAdd()) - - # verify again, to check we didnt break anything - output_dict = oxe.execute_onnx(model, input_dict, True) - produced_sum = output_dict[model.graph.output[0].name] - assert np.isclose(expected_sum, produced_sum, atol=1e-3).all() - assert len(model.get_nodes_by_op_type("Add")) == 2 - assert len(model.get_nodes_by_op_type("Mul")) == 1 - - os.remove(export_onnx_path) - - -@pytest.mark.streamline -@pytest.mark.parametrize("ch", [64, 1]) -# ifmdim -@pytest.mark.parametrize("ifmdim", [-1, 7]) -def test_linear_past_eltwise_add_multiple_forks(ch, ifmdim): - # generate test vectors of correct shape - if ifmdim == -1: - input_shape = (1, ch) - else: - input_shape = (1, ch, ifmdim, ifmdim) - - top_in = helper.make_tensor_value_info("top_in", TensorProto.FLOAT, input_shape) - top_out = helper.make_tensor_value_info("top_out", TensorProto.FLOAT, input_shape) - - num_of_params = 6 - value_info = [] - for i in range(num_of_params): - value_info += [helper.make_tensor_value_info("p" + str(i), TensorProto.FLOAT, input_shape)] - - modelproto = qonnx_make_model( - helper.make_graph( - name="test", - inputs=[top_in], - outputs=[top_out], - value_info=value_info, - nodes=[ - helper.make_node("Add", ["top_in", "p0"], ["fork1"]), - helper.make_node("Mul", ["fork1", "p1"], ["t2"]), - helper.make_node("Mul", ["fork1", "p2"], ["t3"]), - helper.make_node("Add", ["t2", "t3"], ["t4"]), - helper.make_node("Mul", ["t4", "p3"], ["fork2"]), - helper.make_node("Add", ["fork2", "p4"], ["t5"]), - helper.make_node("Add", ["fork2", "p5"], ["t6"]), - helper.make_node("Add", ["t5", "t6"], ["top_out"]), - ], - ) - ) - model = ModelWrapper(modelproto) - model = model.transform(InferShapes()) - - np.random.seed(0) - for i in range(num_of_params): - model.set_initializer("p" + str(i), np.random.rand(*input_shape).astype(np.float32)) - - # need equal mults: - model.set_initializer("p2", model.get_initializer("p1")) - - # Transform - new_model = model.transform(MoveLinearPastEltwiseAdd()) - inp_dict = {"top_in": np.random.rand(*input_shape).astype(np.float32)} - - # Test - assert oxe.compare_execution(model, new_model, inp_dict) - assert new_model.graph.node[0].op_type == "Add" - assert new_model.graph.node[1].op_type == "Add" - assert new_model.graph.node[2].op_type == "Mul" - assert new_model.graph.node[3].op_type == "Mul" - assert new_model.graph.node[4].op_type == "Add" - assert new_model.graph.node[5].op_type == "Add" - assert len(new_model.graph.node) == 6