|
| 1 | +import warnings |
| 2 | + |
| 3 | +import numpy as np |
| 4 | +import tensorflow as tf |
| 5 | +import torch |
| 6 | + |
| 7 | +from interpolator import Interpolator |
| 8 | + |
| 9 | + |
| 10 | +def translate_state_dict(var_dict, state_dict): |
| 11 | + for name, (prev_name, weight) in zip(state_dict, var_dict.items()): |
| 12 | + print('Mapping', prev_name, '->', name) |
| 13 | + weight = torch.from_numpy(weight) |
| 14 | + if 'kernel' in prev_name: |
| 15 | + # Transpose the conv2d kernel weights, since TF uses (H, W, C, K) and PyTorch uses (K, C, H, W) |
| 16 | + weight = weight.permute(3, 2, 0, 1) |
| 17 | + |
| 18 | + assert state_dict[name].shape == weight.shape, f'Shape mismatch {state_dict[name].shape} != {weight.shape}' |
| 19 | + |
| 20 | + state_dict[name] = weight |
| 21 | + |
| 22 | + |
| 23 | +def import_state_dict(interpolator: Interpolator, saved_model): |
| 24 | + variables = saved_model.keras_api.variables |
| 25 | + |
| 26 | + extract_dict = interpolator.extract.state_dict() |
| 27 | + flow_dict = interpolator.predict_flow.state_dict() |
| 28 | + fuse_dict = interpolator.fuse.state_dict() |
| 29 | + |
| 30 | + extract_vars = {} |
| 31 | + _flow_vars = {} |
| 32 | + _fuse_vars = {} |
| 33 | + |
| 34 | + for var in variables: |
| 35 | + name = var.name |
| 36 | + if name.startswith('feat_net'): |
| 37 | + extract_vars[name[9:]] = var.numpy() |
| 38 | + elif name.startswith('predict_flow'): |
| 39 | + _flow_vars[name[13:]] = var.numpy() |
| 40 | + elif name.startswith('fusion'): |
| 41 | + _fuse_vars[name[7:]] = var.numpy() |
| 42 | + |
| 43 | + # reverse order of modules to allow jit export |
| 44 | + # TODO: improve this hack |
| 45 | + flow_vars = dict(sorted(_flow_vars.items(), key=lambda x: x[0].split('/')[0], reverse=True)) |
| 46 | + fuse_vars = dict(sorted(_fuse_vars.items(), key=lambda x: int((x[0].split('/')[0].split('_')[1:] or [0])[0]) // 3, reverse=True)) |
| 47 | + |
| 48 | + assert len(extract_vars) == len(extract_dict), f'{len(extract_vars)} != {len(extract_dict)}' |
| 49 | + assert len(flow_vars) == len(flow_dict), f'{len(flow_vars)} != {len(flow_dict)}' |
| 50 | + assert len(fuse_vars) == len(fuse_dict), f'{len(fuse_vars)} != {len(fuse_dict)}' |
| 51 | + |
| 52 | + for state_dict, var_dict in ((extract_dict, extract_vars), (flow_dict, flow_vars), (fuse_dict, fuse_vars)): |
| 53 | + translate_state_dict(var_dict, state_dict) |
| 54 | + |
| 55 | + interpolator.extract.load_state_dict(extract_dict) |
| 56 | + interpolator.predict_flow.load_state_dict(flow_dict) |
| 57 | + interpolator.fuse.load_state_dict(fuse_dict) |
| 58 | + |
| 59 | + |
| 60 | +def verify_debug_outputs(pt_outputs, tf_outputs): |
| 61 | + max_error = 0 |
| 62 | + for name, predicted in pt_outputs.items(): |
| 63 | + if name == 'image': |
| 64 | + continue |
| 65 | + pred_frfp = [f.permute(0, 2, 3, 1).detach().cpu().numpy() for f in predicted] |
| 66 | + true_frfp = [f.numpy() for f in tf_outputs[name]] |
| 67 | + |
| 68 | + for i, (pred, true) in enumerate(zip(pred_frfp, true_frfp)): |
| 69 | + assert pred.shape == true.shape, f'{name} {i} shape mismatch {pred.shape} != {true.shape}' |
| 70 | + error = np.max(np.abs(pred - true)) |
| 71 | + max_error = max(max_error, error) |
| 72 | + assert error < 1, f'{name} {i} max error: {error}' |
| 73 | + print('Max intermediate error:', max_error) |
| 74 | + |
| 75 | + |
| 76 | +def test_model(interpolator, model, half=False, gpu=False): |
| 77 | + torch.manual_seed(0) |
| 78 | + time = torch.full((1, 1), .5) |
| 79 | + x0 = torch.rand(1, 3, 256, 256) |
| 80 | + x1 = torch.rand(1, 3, 256, 256) |
| 81 | + |
| 82 | + x0_ = tf.convert_to_tensor(x0.permute(0, 2, 3, 1).numpy(), dtype=tf.float32) |
| 83 | + x1_ = tf.convert_to_tensor(x1.permute(0, 2, 3, 1).numpy(), dtype=tf.float32) |
| 84 | + time_ = tf.convert_to_tensor(time.numpy(), dtype=tf.float32) |
| 85 | + tf_outputs = model({'x0': x0_, 'x1': x1_, 'time': time_}, training=False) |
| 86 | + |
| 87 | + if half: |
| 88 | + x0 = x0.half() |
| 89 | + x1 = x1.half() |
| 90 | + time = time.half() |
| 91 | + |
| 92 | + if gpu and torch.cuda.is_available(): |
| 93 | + x0 = x0.cuda() |
| 94 | + x1 = x1.cuda() |
| 95 | + time = time.cuda() |
| 96 | + |
| 97 | + with torch.no_grad(): |
| 98 | + pt_outputs = interpolator.debug_forward(x0, x1, time) |
| 99 | + |
| 100 | + verify_debug_outputs(pt_outputs, tf_outputs) |
| 101 | + |
| 102 | + with torch.no_grad(): |
| 103 | + prediction = interpolator(x0, x1, time) |
| 104 | + output_color = prediction.permute(0, 2, 3, 1).detach().cpu().numpy() |
| 105 | + true_color = tf_outputs['image'].numpy() |
| 106 | + error = np.abs(output_color - true_color).max() |
| 107 | + |
| 108 | + print('Color max error:', error) |
| 109 | + |
| 110 | + |
| 111 | +def main(model_path, save_path, export_to_torchscript=True, use_gpu=False, fp16=True, skiptest=False): |
| 112 | + print(f'Exporting model to FP{["32", "16"][fp16]} {["state_dict", "torchscript"][export_to_torchscript]} ' |
| 113 | + f'using {"CG"[use_gpu]}PU') |
| 114 | + model = tf.compat.v2.saved_model.load(model_path) |
| 115 | + interpolator = Interpolator() |
| 116 | + interpolator.eval() |
| 117 | + import_state_dict(interpolator, model) |
| 118 | + |
| 119 | + if use_gpu and torch.cuda.is_available(): |
| 120 | + interpolator = interpolator.cuda() |
| 121 | + else: |
| 122 | + if fp16 and use_gpu: |
| 123 | + warnings.warn('No GPU is available, using CPU FP32', UserWarning) |
| 124 | + fp16 = False |
| 125 | + |
| 126 | + if fp16: |
| 127 | + interpolator = interpolator.half() |
| 128 | + if export_to_torchscript: |
| 129 | + interpolator = torch.jit.script(interpolator) |
| 130 | + |
| 131 | + if not skiptest: |
| 132 | + test_model(interpolator, model, fp16, use_gpu) |
| 133 | + |
| 134 | + if export_to_torchscript: |
| 135 | + interpolator.save(save_path) |
| 136 | + else: |
| 137 | + torch.save(interpolator.state_dict(), save_path) |
| 138 | + |
| 139 | + |
| 140 | +if __name__ == '__main__': |
| 141 | + import argparse |
| 142 | + |
| 143 | + parser = argparse.ArgumentParser(description='Export frame-interpolator model to PyTorch state dict') |
| 144 | + |
| 145 | + parser.add_argument('model_path', type=str, help='Path to the TF SavedModel') |
| 146 | + parser.add_argument('save_path', type=str, help='Path to save the PyTorch state dict') |
| 147 | + parser.add_argument('--statedict', action='store_true', help='Export to state dict instead of TorchScript') |
| 148 | + parser.add_argument('--fp32', action='store_true', help='Save at full precision') |
| 149 | + parser.add_argument('--skiptest', action='store_true', help='Save at full precision') |
| 150 | + parser.add_argument('--gpu', action='store_true', help='Use GPU') |
| 151 | + |
| 152 | + args = parser.parse_args() |
| 153 | + |
| 154 | + main(args.model_path, args.save_path, not args.statedict, args.gpu, not args.fp32, args.skiptest) |
0 commit comments