From 8137f03319e45161d7d2748e7569daf79d69f97e Mon Sep 17 00:00:00 2001 From: photonshi Date: Fri, 18 Oct 2024 07:06:09 +0000 Subject: [PATCH 01/16] initial push, need to debug --- src/algos/fl_inversionAttack.py | 413 ++++++++++++++++++++ src/inversefed/__init__.py | 20 + src/inversefed/consts.py | 16 + src/inversefed/medianfilt.py | 54 +++ src/inversefed/metrics.py | 106 +++++ src/inversefed/nn/README.md | 1 + src/inversefed/nn/__init__.py | 6 + src/inversefed/nn/densenet.py | 92 +++++ src/inversefed/nn/models.py | 333 ++++++++++++++++ src/inversefed/nn/modules.py | 98 +++++ src/inversefed/nn/revnet.py | 192 +++++++++ src/inversefed/nn/revnet_utils.py | 132 +++++++ src/inversefed/optimization_strategy.py | 78 ++++ src/inversefed/options.py | 52 +++ src/inversefed/reconstruction_algorithms.py | 392 +++++++++++++++++++ src/inversefed/training/README.md | 1 + src/inversefed/training/__init__.py | 5 + src/inversefed/training/scheduler.py | 95 +++++ src/inversefed/training/training_routine.py | 124 ++++++ src/inversefed/utils.py | 70 ++++ 20 files changed, 2280 insertions(+) create mode 100644 src/algos/fl_inversionAttack.py create mode 100644 src/inversefed/__init__.py create mode 100644 src/inversefed/consts.py create mode 100644 src/inversefed/medianfilt.py create mode 100644 src/inversefed/metrics.py create mode 100644 src/inversefed/nn/README.md create mode 100644 src/inversefed/nn/__init__.py create mode 100644 src/inversefed/nn/densenet.py create mode 100644 src/inversefed/nn/models.py create mode 100644 src/inversefed/nn/modules.py create mode 100644 src/inversefed/nn/revnet.py create mode 100644 src/inversefed/nn/revnet_utils.py create mode 100644 src/inversefed/optimization_strategy.py create mode 100644 src/inversefed/options.py create mode 100644 src/inversefed/reconstruction_algorithms.py create mode 100644 src/inversefed/training/README.md create mode 100644 src/inversefed/training/__init__.py create mode 100644 src/inversefed/training/scheduler.py create mode 100644 src/inversefed/training/training_routine.py create mode 100644 src/inversefed/utils.py diff --git a/src/algos/fl_inversionAttack.py b/src/algos/fl_inversionAttack.py new file mode 100644 index 00000000..757afc54 --- /dev/null +++ b/src/algos/fl_inversionAttack.py @@ -0,0 +1,413 @@ +import numpy as np +import networkx as nx +import matplotlib.pyplot as plt +import scipy as sp +from typing import Any, Dict, List +import torch +from fractions import Fraction +import random + +from utils.communication.comm_utils import CommunicationManager +from utils.log_utils import LogUtils +from algos.fl import FedAvgClient, FedAvgServer +from algos.fl_static import FedStaticNode, FedStaticServer + +import inversefed + +def LaplacianGossipMatrix(G): + max_degree = max([G.degree(node) for node in G.nodes()]) + W = np.eye(G.number_of_nodes()) - 1/max_degree * nx.laplacian_matrix(G).toarray() + return W + +def get_non_attackers_neighbors(G, attackers): + """ + G : networkx graph + attackers : list of the nodes considered as attackers + returns : non repetetive list of the neighbors of the attackers + """ + return sorted(set(n for attacker in attackers for n in G.neighbors(attacker)).difference(set(attackers))) + +def GLS(X, y, cov): + """ + Returns the generalized least squares estimator b, such as + Xb = y + e + e being a noise of covariance matrix cov + """ + X_n, X_m = X.shape + y_m = len(y) + s_n = len(cov) + assert s_n == X_n, "Dimension mismatch" + try: + inv_cov = np.linalg.inv(cov) + except Exception as e: + print("WARNING : The covariance matrix is not invertible, using pseudo inverse instead") + inv_cov = np.linalg.pinv(cov) + return np.linalg.inv(X.T@inv_cov@X)@ X.T@inv_cov@y + +class ReconstructOptim(): + def __init__(self, G, n_iter, attackers, gossip_matrix = LaplacianGossipMatrix, targets_only = False): + """ + A class to reconstruct the intial values used in a decentralized parallel gd algorithm + This class depends only on the graph and the attack parameters n_iter and attackers + It doesn't depend on the actual updates of one particular execution + G: networkx graph, we require the nodes to be indexed from 0 to n-1 + n_iter: number of gossip iterations n_iter >= 1 + attackers: indices of the attacker nodes + gossip_matrix: function that returns the gossip matrix of the graph + + same script as https://github.com/AbdellahElmrini/decAttack/tree/master + """ + self.G = G + self.n_iter = n_iter + self.attackers = attackers + self.n_attackers = len(attackers) + self.W = gossip_matrix(self.G) + self.Wt = torch.tensor(self.W, dtype = torch.float64) + self.build_knowledge_matrix_dec() + + def build_knowledge_matrix_dec(self, centralized=False): + """ + Building a simplified knowledge matrix including only the targets as unknowns + This matrix encodes the system of equations that the attackers receive during the learning + We assume that the n_a attackers appear in the beginning of the gossip matrix + returns : + knowledge_matrix : A matrix of shape m * n, where m = self.n_iter*len(neighbors), n = number of targets + """ + if not centralized: + W = self.W + att_matrix = [] + n_targets = len(self.W) - self.n_attackers + for neighbor in get_non_attackers_neighbors(self.G, self.attackers): + att_matrix.append(np.eye(1,n_targets,neighbor-self.n_attackers)[0]) # Shifting the index of the neighbor to start from 0 + + pW_TT = np.identity(n_targets) + + for _ in range(1, self.n_iter): + pW_TT = W[self.n_attackers:,self.n_attackers: ] @ pW_TT + np.identity((n_targets)) + for neighbor in get_non_attackers_neighbors(self.G, self.attackers): + att_matrix.append(pW_TT[neighbor-self.n_attackers]) # Assuming this neighbor is not an attacker + + self.target_knowledge_matrix = np.array(att_matrix) + return self.target_knowledge_matrix + else: + # Simplify for centralized FL: no gossip matrix, direct aggregation from clients + n_targets = len(self.W) - self.n_attackers # Number of clients (non-attackers) + + att_matrix = [] + for client in range(n_targets): + att_matrix.append(np.eye(1, n_targets, client)[0]) # Identity matrix for each client + + self.target_knowledge_matrix = np.array(att_matrix) + return self.target_knowledge_matrix + def build_cov_target_only(self, sigma): # NewName : Build_covariance_matrix + """ + Function to build the covariance matrix of the system of equations received by the attackers + The number of columns corresponds to the number of targets in the system + See the pseudo code at algorithm 6 in the report + return : + cov : a matrix of size m * m, where m = self.n_iter*len(neighbors) + """ + W = self.W + W_TT = W[self.n_attackers:, self.n_attackers:] + neighbors = get_non_attackers_neighbors(self.G, self.attackers) + + m = self.n_iter*len(neighbors) + + cov = np.zeros((m,m)) + # We iteratively fill this matrix line by line in a triangular fashion (as it is a symetric matrix) + i = 0 + + while i < m: + for it1 in range(self.n_iter): + for neighbor1 in neighbors: + j = it1*len(neighbors) + for it2 in range(it1, self.n_iter): + for neighbor2 in neighbors: + s=0 + for t in range(it1+1): + s+=np.linalg.matrix_power(W_TT,it1+it2-2*t)[neighbor1, neighbor2] + cov[i,j] = sigma**2 * s + cov[j,i] = cov[i,j] + j += 1 + i+=1 + return cov + + + + def reconstruct_GLS_target_only(self, v, X_A, sigma): + """ + Function to reconstruct the inital gradients from the values received by the attackers after self.n_iter iterations. + This method uses GLS estimator + v (nd.array) : vector containing the values received by the attackers (in the order defined by the gossip) + sigma : (float) : variance + returns : + x_hat : a vector of shape n * v.shape[1], where n is the number of nodes + """ + cov = self.build_cov_target_only(sigma) + n_targets = len(self.W) - self.n_attackers + neighbors = np.array(get_non_attackers_neighbors(self.G, self.attackers)) + n_neighbors = len(neighbors) + v = v[self.n_attackers:] # v[:self.n_attackers] are the attacker sent updates which are the same as X_A[:self.n_attackers] + d = v[0].shape[0] + W_TA = self.Wt[self.n_attackers:, :self.n_attackers] + W_TT = self.Wt[self.n_attackers:, self.n_attackers:] + pW_TT = np.identity(n_targets, dtype = np.float64) + new_v = [] + B_t = np.zeros((n_targets, d), dtype = np.float64) + for it in range(self.n_iter): + X_A_t = X_A[it*self.n_attackers:(it+1)*self.n_attackers] + pW_TT = W_TT @ pW_TT + np.identity((n_targets), dtype = np.float64) + theta_T_t = v[it*n_neighbors:(it+1)*n_neighbors] + new_v.extend(theta_T_t-B_t[neighbors-self.n_attackers]) + B_t = W_TT @ B_t + W_TA @ X_A_t + v = np.array(new_v) + try: + return GLS(self.target_knowledge_matrix, v, cov) + except Exception as e: + print(e) + print("Building the knowledge matrix failed") + raise + + def reconstruct_LS_target_only(self, v, X_A): + """ + Function to reconstruct the inital gradients from the values received by the attackers after self.n_iter iterations. + This method uses a Least Squares estimator + v (nd.array) : vector containing the values received by the attackers (in the order defined by the gossip) + v looks like (X_A^0, \theta_T^{0+), X_A^1, \theta_T^{1+), ..., X_A^T, \theta_T^{T+)} + where X_A^t are the attacker sent updates at iteration t and \theta_T^{t+)} are the target sent updates at iteration t + X_A (nd.array) : vector of size n_a*self.n_iter, containing the attacker sent updates at each iteration + returns : + x_hat : a vector of shape n_target * v.shape[1], where n_target is the number of target nodes + """ + # Prepossessing v to adapt to the target only knowledge matrix + + n_targets = len(self.W) - self.n_attackers + neighbors = np.array(get_non_attackers_neighbors(self.G, self.attackers)) + n_neighbors = len(neighbors) + v = v[self.n_attackers:] # v[:self.n_attackers] are the attacker sent updates which are the same as X_A[:self.n_attackers] + d = v[0].shape[0] + W_TA = self.Wt[self.n_attackers:, :self.n_attackers] + W_TT = self.Wt[self.n_attackers:, self.n_attackers:] + #pW_TT = np.identity(n_targets, dtype = np.float32) + new_v = [] + B_t = np.zeros((n_targets, d), dtype = np.float64) + for it in range(self.n_iter): + X_A_t = X_A[it*self.n_attackers:(it+1)*self.n_attackers] + #pW_TT = W_TT @ pW_TT + np.identity((n_targets), dtype = np.float64) + theta_T_t = v[it*n_neighbors:(it+1)*n_neighbors] + new_v.extend(theta_T_t-B_t[neighbors-self.n_attackers]) + + B_t = W_TT @ B_t + W_TA @ X_A_t + + v = torch.stack(new_v).numpy() + + try: + return np.linalg.lstsq(self.target_knowledge_matrix, v)[0] + except Exception as e: + print(e) + print("Building the knowledge matrix failed") + raise + +class GradientInversionFedAvgClient(FedAvgClient): + """ + Implements ground truth for evaluating inversion attack + """ + def __init__(self, config: Dict[str, Any], node_id: int, comm: CommunicationManager, log: LogUtils): + super(GradientInversionFedAvgClient, self).__init__(config, node_id, comm, log) + # get ground truth and labels for evaluation + self.ground_truth, self.labels = self.extract_ground_truth(num_images=config["num_images"]) # set reconstruction number + + # TODO somehow get the server to access the ground truth and labels for evaluation + + def extract_ground_truth(self, num_images=10): + """ + Randomly extract a batch of ground truth images and labels from self.dloader for gradient inversion attacks. + + Args: + num_images (int): Number of images to extract. + + Returns: + ground_truth (torch.Tensor): Tensor containing the extracted ground truth images. + labels (torch.Tensor): Tensor containing the corresponding labels. + """ + # Convert the dataset to a list of (image, label) tuples + data = list(self.dloader.dataset) + + # Randomly sample `num_images` images and labels + sampled_data = random.sample(data, num_images) + + # Separate images and labels + ground_truth = [img for img, label in sampled_data] + labels = [torch.as_tensor((label,)) for img, label in sampled_data] + + # Stack into tensors + ground_truth = torch.stack(ground_truth) + labels = torch.cat(labels) + + return ground_truth, labels + +class GradientInversionFedAvgServer(FedAvgServer): + """ + implements gradient inversion attack to reconstruct training images from other nodes + """ + def __init__(self, config: Dict[str, Any], comm: CommunicationManager, log: LogUtils): + super(GradientInversionFedAvgServer, self).__init__(config, comm, log) + + #TODO somehow obtain the client's ground truth and labels for evaluation + self.ground_truth, self.labels = self.obtain_ground_truth() # should be one list per client + + + def obtain_ground_truth(self): + """ + Obtain the ground truth images and labels from the clients for evaluation. + """ + ground_truth, labels = self.comm_utils.receive([i for i in range(self.num_users)]) + return ground_truth, labels + + def inverting_gradients_attack(self): + """ + Setup the inversion attack for the server. + + Based on reconstruction from weight script: + https://github.com/JonasGeiping/invertinggradients/blob/1157b61c6704df42c497ab9eb074c75da5204334/Recovery%20from%20Weight%20Updates.ipynb + """ + if self.dset == "cifar10": + # TODO figure out whehether we actually have the dm and ds values in our codebase + dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None] + ds = torch.as_tensor(inversefed.consts.cifar10_std, **setup)[:, None, None] + + # extract input parameters (this should be the averaged server-side params after a round of FedAVG) + input_params_s = self.comm_utils.all_gather() #[clinet1 param, client2 param, ...] + self.single_round() + input_params_t = self.comm_utils.all_gather() + + # get the param difference for each client [client1 param diff, client2 param diff, ...] + param_diffs = [] + + # Loop over each client's parameters (assumes input_params_s and input_params_t are lists of lists) + for client_params_s, client_params_t in zip(input_params_s, input_params_t): + client_param_diff = [ + param_t - param_s # element-wise difference of the tensors + for param_s, param_t in zip(client_params_s, client_params_t) + ] + param_diffs.append(client_param_diff) + + assert len(param_diffs) == self.num_users == self.ground_truth, "Number of clients does not match number of param differences" + config = dict(signed=True, + boxed=True, + cost_fn='sim', + indices='def', + weights='equal', + lr=0.1, + optim='adam', + restarts=1, + max_iterations=8_000, + total_variation=1e-6, + init='randn', + filter='none', + lr_decay=True, + scoring_choice='loss') + + for client_i in range(self.num_users): + # TODO assume that client i correspond to order of received params + ground_truth_i, labels_i, params_i = self.ground_truth[client_i], self.labels[client_i], param_diffs[client_i] + + local_steps = 1 # number of local steps for client training + local_lr = self.config["model_lr"] # learning rate for client training + use_updates = False + rec_machine = inversefed.FedAvgReconstructor(self.model, (dm, ds), local_steps, local_lr, config, + use_updates=use_updates) + output, stats = rec_machine.reconstruct(input_parameters, labels, img_shape=(3, 32, 32)) # TODO verify img_shape and change it based on dataset + test_mse = (output.detach() - ground_truth).pow(2).mean() + feat_mse = (model(output.detach())- model(ground_truth)).pow(2).mean() + test_psnr = inversefed.metrics.psnr(output, ground_truth, factor=1/ds) + + # optional plotting: + # plot(output) + # plt.title(f"Rec. loss: {stats['opt']:2.4f} | MSE: {test_mse:2.4f} " + # f"| PSNR: {test_psnr:4.2f} | FMSE: {feat_mse:2.4e} |"); +class GradientInversionFedStaticServer(FedStaticServer): + """ + implements gradient inversion attack to reconstruct training images from other nodes + can handle colluding neighbors + + base on method proposed by https://github.com/AbdellahElmrini/decAttack/tree/master' + + reconstruction method uses InvertingGradients by Jonas Geiping: https://github.com/JonasGeiping/invertinggradients + + The order of stacked params is just the keys of attacker / collaborator IDs in ascending order + """ + def __init__(self, config: Dict[str, Any], G: nx.Graph): + # construct graph + # TODO need to recheck this instantiation depend on graph implementation + # TODO keep copy of weights at 0th round (when everyone finishes training + self.G = G + self.neighbors = [i for i in range(self.num_users) if i != self.node_id] # for the server, neighbors are all the clients + self.attackers = [self.node_id] # for the server, the attacker is itself + self.end_round = config["rounds"] + + def get_model_parameters(self, ids_list: List[int]): + """ + returns stacked model parameters + modeled after Decentralized.get_model_params in decAttack codebase + + TODO verify the actual params getting sent + """ + param_from_collaborators = self.comm_utils.receive(ids_list) + params = [[] for p in range(self.model.parameters())] + + for i in range(len(self.neighbors)): + neighbor_id = self.neighbors[i] + for j, param in enumerate(param_from_collaborators[neighbor_id]): + params[j].append(param) + + + for j in range(len(params)): + params[j] = torch.stack(params[j]) + + return params + + def get_node_weights(self): + """ + helper function that obtains the param updates of attackers + uses commProtocol to get the params from the nodes + + TODO double check that neighbors include attacking nodes as well + """ + + # Issue for FL where server is the attacker: attacker gradeint is the averaged gradients from neighbors + return self.get_model_parameters(self.neighbors), self.get_model_parameters(self.attackers) + + + def launch_attack(self): + """ + Main function for performing inversion attack when the server is the attacker. + This should happen after running FedAVG for a single round. + """ + # Build reconstruction class: + R = ReconstructOptim(self.G, n_iter=1, attackers=self.attackers) + + # Initial parameters (before aggregation) + neighbor_params0, attacker_params0 = self.get_node_weights() + + # Run a single round of FedAVG to update server's representation + self.single_round() + + # Get the updated parameters after aggregation + neighbor_params_i, attacker_params_i = self.get_node_weights() + + # Collect the difference in parameters for attack + sent_params = [] + attacker_params = [] + + # In centralized FL, server is the attacker + for i in range(len(neighbor_params0)): # Loop over all clients + # Calculate the difference between the initial and updated parameters for neighbors (clients) + sent_params.append(torch.cat([(neighbor_params_i[j][i] - neighbor_params0[j][i]).flatten().detach() for j in range(self.n_params)]).cpu()) + + # For the server (attacker), compute the difference between its initial and updated parameters + attacker_params.append(torch.cat([(attacker_params_i[j] - attacker_params0[j]).flatten().detach() for j in range(self.n_params)]).cpu()) + + # Use the collected parameters to reconstruct the images + x_hat = R.reconstruct_LS_target_only(sent_params, attacker_params) + diff --git a/src/inversefed/__init__.py b/src/inversefed/__init__.py new file mode 100644 index 00000000..0dacb9c7 --- /dev/null +++ b/src/inversefed/__init__.py @@ -0,0 +1,20 @@ +"""Library of routines.""" + +from inversefed import nn +from inversefed.nn import construct_model, MetaMonkey + +from inversefed.data import construct_dataloaders +from inversefed.training import train +from inversefed import utils + +from .optimization_strategy import training_strategy + + +from .reconstruction_algorithms import GradientReconstructor, FedAvgReconstructor + +from .options import options +from inversefed import metrics + +__all__ = ['train', 'construct_dataloaders', 'construct_model', 'MetaMonkey', + 'training_strategy', 'nn', 'utils', 'options', + 'metrics', 'GradientReconstructor', 'FedAvgReconstructor'] diff --git a/src/inversefed/consts.py b/src/inversefed/consts.py new file mode 100644 index 00000000..0154c38d --- /dev/null +++ b/src/inversefed/consts.py @@ -0,0 +1,16 @@ +"""Setup constants, ymmv.""" + +PIN_MEMORY = True +NON_BLOCKING = False +BENCHMARK = True +MULTITHREAD_DATAPROCESSING = 4 + + +cifar10_mean = [0.4914672374725342, 0.4822617471218109, 0.4467701315879822] +cifar10_std = [0.24703224003314972, 0.24348513782024384, 0.26158785820007324] +cifar100_mean = [0.5071598291397095, 0.4866936206817627, 0.44120192527770996] +cifar100_std = [0.2673342823982239, 0.2564384639263153, 0.2761504650115967] +mnist_mean = (0.13066373765468597,) +mnist_std = (0.30810782313346863,) +imagenet_mean = [0.485, 0.456, 0.406] +imagenet_std = [0.229, 0.224, 0.225] diff --git a/src/inversefed/medianfilt.py b/src/inversefed/medianfilt.py new file mode 100644 index 00000000..f0039da0 --- /dev/null +++ b/src/inversefed/medianfilt.py @@ -0,0 +1,54 @@ +"""This is code for median pooling from https://gist.github.com/rwightman. + +https://gist.github.com/rwightman/f2d3849281624be7c0f11c85c87c1598 +""" +import torch.nn as nn +import torch.nn.functional as F +from torch.nn.modules.utils import _pair, _quadruple + + +class MedianPool2d(nn.Module): + """Median pool (usable as median filter when stride=1) module. + + Args: + kernel_size: size of pooling kernel, int or 2-tuple + stride: pool stride, int or 2-tuple + padding: pool padding, int or 4-tuple (l, r, t, b) as in pytorch F.pad + same: override padding and enforce same padding, boolean + """ + + def __init__(self, kernel_size=3, stride=1, padding=0, same=True): + """Initialize with kernel_size, stride, padding.""" + super().__init__() + self.k = _pair(kernel_size) + self.stride = _pair(stride) + self.padding = _quadruple(padding) # convert to l, r, t, b + self.same = same + + def _padding(self, x): + if self.same: + ih, iw = x.size()[2:] + if ih % self.stride[0] == 0: + ph = max(self.k[0] - self.stride[0], 0) + else: + ph = max(self.k[0] - (ih % self.stride[0]), 0) + if iw % self.stride[1] == 0: + pw = max(self.k[1] - self.stride[1], 0) + else: + pw = max(self.k[1] - (iw % self.stride[1]), 0) + pl = pw // 2 + pr = pw - pl + pt = ph // 2 + pb = ph - pt + padding = (pl, pr, pt, pb) + else: + padding = self.padding + return padding + + def forward(self, x): + # using existing pytorch functions and tensor ops so that we get autograd, + # would likely be more efficient to implement from scratch at C/Cuda level + x = F.pad(x, self._padding(x), mode='reflect') + x = x.unfold(2, self.k[0], self.stride[0]).unfold(3, self.k[1], self.stride[1]) + x = x.contiguous().view(x.size()[:4] + (-1,)).median(dim=-1)[0] + return x diff --git a/src/inversefed/metrics.py b/src/inversefed/metrics.py new file mode 100644 index 00000000..c227dc42 --- /dev/null +++ b/src/inversefed/metrics.py @@ -0,0 +1,106 @@ +"""This is code based on https://sudomake.ai/inception-score-explained/.""" +import torch +import torchvision + +from collections import defaultdict + +class InceptionScore(torch.nn.Module): + """Class that manages and returns the inception score of images.""" + + def __init__(self, batch_size=32, setup=dict(device=torch.device('cpu'), dtype=torch.float)): + """Initialize with setup and target inception batch size.""" + super().__init__() + self.preprocessing = torch.nn.Upsample(size=(299, 299), mode='bilinear', align_corners=False) + self.model = torchvision.models.inception_v3(pretrained=True).to(**setup) + self.model.eval() + self.batch_size = batch_size + + def forward(self, image_batch): + """Image batch should have dimensions BCHW and should be normalized. + + B should be divisible by self.batch_size. + """ + B, C, H, W = image_batch.shape + batches = B // self.batch_size + scores = [] + for batch in range(batches): + input = self.preprocessing(image_batch[batch * self.batch_size: (batch + 1) * self.batch_size]) + scores.append(self.model(input)) + prob_yx = torch.nn.functional.softmax(torch.cat(scores, 0), dim=1) + entropy = torch.where(prob_yx > 0, -prob_yx * prob_yx.log(), torch.zeros_like(prob_yx)) + return entropy.sum() + + +def psnr(img_batch, ref_batch, batched=False, factor=1.0): + """Standard PSNR.""" + def get_psnr(img_in, img_ref): + mse = ((img_in - img_ref)**2).mean() + if mse > 0 and torch.isfinite(mse): + return (10 * torch.log10(factor**2 / mse)) + elif not torch.isfinite(mse): + return img_batch.new_tensor(float('nan')) + else: + return img_batch.new_tensor(float('inf')) + + if batched: + psnr = get_psnr(img_batch.detach(), ref_batch) + else: + [B, C, m, n] = img_batch.shape + psnrs = [] + for sample in range(B): + psnrs.append(get_psnr(img_batch.detach()[sample, :, :, :], ref_batch[sample, :, :, :])) + psnr = torch.stack(psnrs, dim=0).mean() + + return psnr.item() + + +def total_variation(x): + """Anisotropic TV.""" + dx = torch.mean(torch.abs(x[:, :, :, :-1] - x[:, :, :, 1:])) + dy = torch.mean(torch.abs(x[:, :, :-1, :] - x[:, :, 1:, :])) + return dx + dy + + + +def activation_errors(model, x1, x2): + """Compute activation-level error metrics for every module in the network.""" + model.eval() + + device = next(model.parameters()).device + + hooks = [] + data = defaultdict(dict) + inputs = torch.cat((x1, x2), dim=0) + separator = x1.shape[0] + + def check_activations(self, input, output): + module_name = str(*[name for name, mod in model.named_modules() if self is mod]) + try: + layer_inputs = input[0].detach() + residual = (layer_inputs[:separator] - layer_inputs[separator:]).pow(2) + se_error = residual.sum() + mse_error = residual.mean() + sim = torch.nn.functional.cosine_similarity(layer_inputs[:separator].flatten(), + layer_inputs[separator:].flatten(), + dim=0, eps=1e-8).detach() + data['se'][module_name] = se_error.item() + data['mse'][module_name] = mse_error.item() + data['sim'][module_name] = sim.item() + except (KeyboardInterrupt, SystemExit): + raise + except AttributeError: + pass + + for name, module in model.named_modules(): + hooks.append(module.register_forward_hook(check_activations)) + + try: + outputs = model(inputs.to(device)) + for hook in hooks: + hook.remove() + except Exception as e: + for hook in hooks: + hook.remove() + raise + + return data diff --git a/src/inversefed/nn/README.md b/src/inversefed/nn/README.md new file mode 100644 index 00000000..99211199 --- /dev/null +++ b/src/inversefed/nn/README.md @@ -0,0 +1 @@ +# Models and modules are implemented here \ No newline at end of file diff --git a/src/inversefed/nn/__init__.py b/src/inversefed/nn/__init__.py new file mode 100644 index 00000000..2ba9a68c --- /dev/null +++ b/src/inversefed/nn/__init__.py @@ -0,0 +1,6 @@ +"""Experimental modules and unexperimental model hooks.""" + +from .models import construct_model +from .modules import MetaMonkey + +__all__ = ['construct_model', 'MetaMonkey'] diff --git a/src/inversefed/nn/densenet.py b/src/inversefed/nn/densenet.py new file mode 100644 index 00000000..97f5fa16 --- /dev/null +++ b/src/inversefed/nn/densenet.py @@ -0,0 +1,92 @@ +"""DenseNet in PyTorch.""" +"""Adaptation we did with ******.""" +import math + +import torch +import torch.nn as nn +import torch.nn.functional as F + + +class _Bottleneck(nn.Module): + def __init__(self, in_planes, growth_rate): + super().__init__() + self.bn1 = nn.BatchNorm2d(in_planes) + self.conv1 = nn.Conv2d(in_planes, 4 * growth_rate, kernel_size=1, bias=False) + self.bn2 = nn.BatchNorm2d(4 * growth_rate) + self.conv2 = nn.Conv2d(4 * growth_rate, growth_rate, kernel_size=3, padding=1, bias=False) + + def forward(self, x): + out = self.conv1(F.relu(self.bn1(x))) + out = self.conv2(F.relu(self.bn2(out))) + out = torch.cat([out, x], 1) + return out + + +class _Transition(nn.Module): + def __init__(self, in_planes, out_planes): + super().__init__() + self.bn = nn.BatchNorm2d(in_planes) + self.conv = nn.Conv2d(in_planes, out_planes, kernel_size=1, bias=False) + + def forward(self, x): + out = self.conv(F.relu(self.bn(x))) + out = F.avg_pool2d(out, 2) + return out + + +class _DenseNet(nn.Module): + def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_classes=10): + super().__init__() + self.growth_rate = growth_rate + + num_planes = 2 * growth_rate + self.conv1 = nn.Conv2d(3, num_planes, kernel_size=3, padding=1, bias=False) + + self.dense1 = self._make_dense_layers(block, num_planes, nblocks[0]) + num_planes += nblocks[0] * growth_rate + out_planes = int(math.floor(num_planes * reduction)) + self.trans1 = _Transition(num_planes, out_planes) + num_planes = out_planes + + self.dense2 = self._make_dense_layers(block, num_planes, nblocks[1]) + num_planes += nblocks[1] * growth_rate + out_planes = int(math.floor(num_planes * reduction)) + self.trans2 = _Transition(num_planes, out_planes) + num_planes = out_planes + + self.dense3 = self._make_dense_layers(block, num_planes, nblocks[2]) + num_planes += nblocks[2] * growth_rate + out_planes = int(math.floor(num_planes * reduction)) + # self.trans3 = Transition(num_planes, out_planes) + # num_planes = out_planes + + # self.dense4 = self._make_dense_layers(block, num_planes, nblocks[3]) + # num_planes += nblocks[3]*growth_rate + + self.bn = nn.BatchNorm2d(num_planes) + num_planes = 132 * growth_rate // 12 * 2 * 2 + self.linear = nn.Linear(num_planes, num_classes) + + def _make_dense_layers(self, block, in_planes, nblock): + layers = [] + for i in range(nblock): + layers.append(block(in_planes, self.growth_rate)) + in_planes += self.growth_rate + return nn.Sequential(*layers) + + def forward(self, x): + out = self.conv1(x) + out = self.trans1(self.dense1(out)) + out = self.trans2(self.dense2(out)) + out = self.dense3(out) + # out = self.trans3(self.dense3(out)) + # out = self.dense4(out) + out = F.avg_pool2d(F.relu(self.bn(out)), 4) + out = out.view(out.size(0), -1) + out = self.linear(out) + return out + + +def densenet_cifar(num_classes=10): + """Instantiate the smallest DenseNet.""" + return _DenseNet(_Bottleneck, [6, 6, 6, 0], growth_rate=12, num_classes=num_classes) diff --git a/src/inversefed/nn/models.py b/src/inversefed/nn/models.py new file mode 100644 index 00000000..b8909867 --- /dev/null +++ b/src/inversefed/nn/models.py @@ -0,0 +1,333 @@ +"""Define basic models and translate some torchvision stuff.""" +"""Stuff from https://github.com/pytorch/vision/blob/master/torchvision/models/resnet.py.""" +import torch +import torchvision +import torch.nn as nn + +from torchvision.models.resnet import Bottleneck +from .revnet import iRevNet +from .densenet import _DenseNet, _Bottleneck + +from collections import OrderedDict +import numpy as np +from ..utils import set_random_seed + + + + +def construct_model(model, num_classes=10, seed=None, num_channels=3, modelkey=None): + """Return various models.""" + if modelkey is None: + if seed is None: + model_init_seed = np.random.randint(0, 2**32 - 10) + else: + model_init_seed = seed + else: + model_init_seed = modelkey + set_random_seed(model_init_seed) + + if model in ['ConvNet', 'ConvNet64']: + model = ConvNet(width=64, num_channels=num_channels, num_classes=num_classes) + elif model == 'ConvNet8': + model = ConvNet(width=64, num_channels=num_channels, num_classes=num_classes) + elif model == 'ConvNet16': + model = ConvNet(width=64, num_channels=num_channels, num_classes=num_classes) + elif model == 'ConvNet32': + model = ConvNet(width=64, num_channels=num_channels, num_classes=num_classes) + elif model == 'BeyondInferringMNIST': + model = torch.nn.Sequential(OrderedDict([ + ('conv1', torch.nn.Conv2d(1, 32, 3, stride=2, padding=1)), + ('relu0', torch.nn.LeakyReLU()), + ('conv2', torch.nn.Conv2d(32, 64, 3, stride=1, padding=1)), + ('relu1', torch.nn.LeakyReLU()), + ('conv3', torch.nn.Conv2d(64, 128, 3, stride=2, padding=1)), + ('relu2', torch.nn.LeakyReLU()), + ('conv4', torch.nn.Conv2d(128, 256, 3, stride=1, padding=1)), + ('relu3', torch.nn.LeakyReLU()), + ('flatt', torch.nn.Flatten()), + ('linear0', torch.nn.Linear(12544, 12544)), + ('relu4', torch.nn.LeakyReLU()), + ('linear1', torch.nn.Linear(12544, 10)), + ('softmax', torch.nn.Softmax(dim=1)) + ])) + elif model == 'BeyondInferringCifar': + model = torch.nn.Sequential(OrderedDict([ + ('conv1', torch.nn.Conv2d(3, 32, 3, stride=2, padding=1)), + ('relu0', torch.nn.LeakyReLU()), + ('conv2', torch.nn.Conv2d(32, 64, 3, stride=1, padding=1)), + ('relu1', torch.nn.LeakyReLU()), + ('conv3', torch.nn.Conv2d(64, 128, 3, stride=2, padding=1)), + ('relu2', torch.nn.LeakyReLU()), + ('conv4', torch.nn.Conv2d(128, 256, 3, stride=1, padding=1)), + ('relu3', torch.nn.LeakyReLU()), + ('flatt', torch.nn.Flatten()), + ('linear0', torch.nn.Linear(12544, 12544)), + ('relu4', torch.nn.LeakyReLU()), + ('linear1', torch.nn.Linear(12544, 10)), + ('softmax', torch.nn.Softmax(dim=1)) + ])) + elif model == 'MLP': + width = 1024 + model = torch.nn.Sequential(OrderedDict([ + ('flatten', torch.nn.Flatten()), + ('linear0', torch.nn.Linear(3072, width)), + ('relu0', torch.nn.ReLU()), + ('linear1', torch.nn.Linear(width, width)), + ('relu1', torch.nn.ReLU()), + ('linear2', torch.nn.Linear(width, width)), + ('relu2', torch.nn.ReLU()), + ('linear3', torch.nn.Linear(width, num_classes))])) + elif model == 'TwoLP': + width = 2048 + model = torch.nn.Sequential(OrderedDict([ + ('flatten', torch.nn.Flatten()), + ('linear0', torch.nn.Linear(3072, width)), + ('relu0', torch.nn.ReLU()), + ('linear3', torch.nn.Linear(width, num_classes))])) + elif model == 'ResNet20': + model = ResNet(torchvision.models.resnet.BasicBlock, [3, 3, 3], num_classes=num_classes, base_width=16) + elif model == 'ResNet20-nostride': + model = ResNet(torchvision.models.resnet.BasicBlock, [3, 3, 3], num_classes=num_classes, base_width=16, + strides=[1, 1, 1, 1]) + elif model == 'ResNet20-10': + model = ResNet(torchvision.models.resnet.BasicBlock, [3, 3, 3], num_classes=num_classes, base_width=16 * 10) + elif model == 'ResNet20-4': + model = ResNet(torchvision.models.resnet.BasicBlock, [3, 3, 3], num_classes=num_classes, base_width=16 * 4) + elif model == 'ResNet20-4-unpooled': + model = ResNet(torchvision.models.resnet.BasicBlock, [3, 3, 3], num_classes=num_classes, base_width=16 * 4, + pool='max') + elif model == 'ResNet28-10': + model = ResNet(torchvision.models.resnet.BasicBlock, [4, 4, 4], num_classes=num_classes, base_width=16 * 10) + elif model == 'ResNet32': + model = ResNet(torchvision.models.resnet.BasicBlock, [5, 5, 5], num_classes=num_classes, base_width=16) + elif model == 'ResNet32-10': + model = ResNet(torchvision.models.resnet.BasicBlock, [5, 5, 5], num_classes=num_classes, base_width=16 * 10) + elif model == 'ResNet44': + model = ResNet(torchvision.models.resnet.BasicBlock, [7, 7, 7], num_classes=num_classes, base_width=16) + elif model == 'ResNet56': + model = ResNet(torchvision.models.resnet.BasicBlock, [9, 9, 9], num_classes=num_classes, base_width=16) + elif model == 'ResNet110': + model = ResNet(torchvision.models.resnet.BasicBlock, [18, 18, 18], num_classes=num_classes, base_width=16) + elif model == 'ResNet18': + model = ResNet(torchvision.models.resnet.BasicBlock, [2, 2, 2, 2], num_classes=num_classes, base_width=64) + elif model == 'ResNet34': + model = ResNet(torchvision.models.resnet.BasicBlock, [3, 4, 6, 3], num_classes=num_classes, base_width=64) + elif model == 'ResNet50': + model = ResNet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], num_classes=num_classes, base_width=64) + elif model == 'ResNet50-2': + model = ResNet(torchvision.models.resnet.Bottleneck, [3, 4, 6, 3], num_classes=num_classes, base_width=64 * 2) + elif model == 'ResNet101': + model = ResNet(torchvision.models.resnet.Bottleneck, [3, 4, 23, 3], num_classes=num_classes, base_width=64) + elif model == 'ResNet152': + model = ResNet(torchvision.models.resnet.Bottleneck, [3, 8, 36, 3], num_classes=num_classes, base_width=64) + elif model == 'MobileNet': + inverted_residual_setting = [ + # t, c, n, s + [1, 16, 1, 1], + [6, 24, 2, 1], # cifar adaptation, cf.https://github.com/kuangliu/pytorch-cifar/blob/master/models/mobilenetv2.py + [6, 32, 3, 2], + [6, 64, 4, 2], + [6, 96, 3, 1], + [6, 160, 3, 2], + [6, 320, 1, 1], + ] + model = torchvision.models.MobileNetV2(num_classes=num_classes, + inverted_residual_setting=inverted_residual_setting, + width_mult=1.0) + model.features[0] = torchvision.models.mobilenet.ConvBNReLU(num_channels, 32, stride=1) # this is fixed to width=1 + elif model == 'MNASNet': + model = torchvision.models.MNASNet(1.0, num_classes=num_classes, dropout=0.2) + elif model == 'DenseNet121': + model = torchvision.models.DenseNet(growth_rate=32, block_config=(6, 12, 24, 16), + num_init_features=64, bn_size=4, drop_rate=0, num_classes=num_classes, + memory_efficient=False) + elif model == 'DenseNet40': + model = _DenseNet(_Bottleneck, [6, 6, 6, 0], growth_rate=12, num_classes=num_classes) + elif model == 'DenseNet40-4': + model = _DenseNet(_Bottleneck, [6, 6, 6, 0], growth_rate=12 * 4, num_classes=num_classes) + elif model == 'SRNet3': + model = SRNet(upscale_factor=3, num_channels=num_channels) + elif model == 'SRNet1': + model = SRNet(upscale_factor=1, num_channels=num_channels) + elif model == 'iRevNet': + if num_classes <= 100: + in_shape = [num_channels, 32, 32] # only for cifar right now + model = iRevNet(nBlocks=[18, 18, 18], nStrides=[1, 2, 2], + nChannels=[16, 64, 256], nClasses=num_classes, + init_ds=0, dropout_rate=0.1, affineBN=True, + in_shape=in_shape, mult=4) + else: + in_shape = [3, 224, 224] # only for imagenet + model = iRevNet(nBlocks=[6, 16, 72, 6], nStrides=[2, 2, 2, 2], + nChannels=[24, 96, 384, 1536], nClasses=num_classes, + init_ds=2, dropout_rate=0.1, affineBN=True, + in_shape=in_shape, mult=4) + elif model == 'LeNetZhu': + model = LeNetZhu(num_channels=num_channels, num_classes=num_classes) + else: + raise NotImplementedError('Model not implemented.') + + print(f'Model initialized with random key {model_init_seed}.') + return model, model_init_seed + + +class ResNet(torchvision.models.ResNet): + """ResNet generalization for CIFAR thingies.""" + + def __init__(self, block, layers, num_classes=10, zero_init_residual=False, + groups=1, base_width=64, replace_stride_with_dilation=None, + norm_layer=None, strides=[1, 2, 2, 2], pool='avg'): + """Initialize as usual. Layers and strides are scriptable.""" + super(torchvision.models.ResNet, self).__init__() # nn.Module + if norm_layer is None: + norm_layer = nn.BatchNorm2d + self._norm_layer = norm_layer + + + self.dilation = 1 + if replace_stride_with_dilation is None: + # each element in the tuple indicates if we should replace + # the 2x2 stride with a dilated convolution instead + replace_stride_with_dilation = [False, False, False, False] + if len(replace_stride_with_dilation) != 4: + raise ValueError("replace_stride_with_dilation should be None " + "or a 4-element tuple, got {}".format(replace_stride_with_dilation)) + self.groups = groups + + self.inplanes = base_width + self.base_width = 64 # Do this to circumvent BasicBlock errors. The value is not actually used. + self.conv1 = nn.Conv2d(3, self.inplanes, kernel_size=3, stride=1, padding=1, bias=False) + self.bn1 = norm_layer(self.inplanes) + self.relu = nn.ReLU(inplace=True) + + self.layers = torch.nn.ModuleList() + width = self.inplanes + for idx, layer in enumerate(layers): + self.layers.append(self._make_layer(block, width, layer, stride=strides[idx], dilate=replace_stride_with_dilation[idx])) + width *= 2 + + self.pool = nn.AdaptiveAvgPool2d((1, 1)) if pool == 'avg' else nn.AdaptiveMaxPool2d((1, 1)) + self.fc = nn.Linear(width // 2 * block.expansion, num_classes) + + for m in self.modules(): + if isinstance(m, nn.Conv2d): + nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') + elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)): + nn.init.constant_(m.weight, 1) + nn.init.constant_(m.bias, 0) + + # Zero-initialize the last BN in each residual branch, + # so that the residual branch starts with zeros, and each residual block behaves like an identity. + # This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677 + if zero_init_residual: + for m in self.modules(): + if isinstance(m, Bottleneck): + nn.init.constant_(m.bn3.weight, 0) + elif isinstance(m, BasicBlock): + nn.init.constant_(m.bn2.weight, 0) + + + def _forward_impl(self, x): + # See note [TorchScript super()] + x = self.conv1(x) + x = self.bn1(x) + x = self.relu(x) + + for layer in self.layers: + x = layer(x) + + x = self.pool(x) + x = torch.flatten(x, 1) + x = self.fc(x) + + return x + + +class ConvNet(torch.nn.Module): + """ConvNetBN.""" + + def __init__(self, width=32, num_classes=10, num_channels=3): + """Init with width and num classes.""" + super().__init__() + self.model = torch.nn.Sequential(OrderedDict([ + ('conv0', torch.nn.Conv2d(num_channels, 1 * width, kernel_size=3, padding=1)), + ('bn0', torch.nn.BatchNorm2d(1 * width)), + ('relu0', torch.nn.ReLU()), + + ('conv1', torch.nn.Conv2d(1 * width, 2 * width, kernel_size=3, padding=1)), + ('bn1', torch.nn.BatchNorm2d(2 * width)), + ('relu1', torch.nn.ReLU()), + + ('conv2', torch.nn.Conv2d(2 * width, 2 * width, kernel_size=3, padding=1)), + ('bn2', torch.nn.BatchNorm2d(2 * width)), + ('relu2', torch.nn.ReLU()), + + ('conv3', torch.nn.Conv2d(2 * width, 4 * width, kernel_size=3, padding=1)), + ('bn3', torch.nn.BatchNorm2d(4 * width)), + ('relu3', torch.nn.ReLU()), + + ('conv4', torch.nn.Conv2d(4 * width, 4 * width, kernel_size=3, padding=1)), + ('bn4', torch.nn.BatchNorm2d(4 * width)), + ('relu4', torch.nn.ReLU()), + + ('conv5', torch.nn.Conv2d(4 * width, 4 * width, kernel_size=3, padding=1)), + ('bn5', torch.nn.BatchNorm2d(4 * width)), + ('relu5', torch.nn.ReLU()), + + ('pool0', torch.nn.MaxPool2d(3)), + + ('conv6', torch.nn.Conv2d(4 * width, 4 * width, kernel_size=3, padding=1)), + ('bn6', torch.nn.BatchNorm2d(4 * width)), + ('relu6', torch.nn.ReLU()), + + ('conv6', torch.nn.Conv2d(4 * width, 4 * width, kernel_size=3, padding=1)), + ('bn6', torch.nn.BatchNorm2d(4 * width)), + ('relu6', torch.nn.ReLU()), + + ('conv7', torch.nn.Conv2d(4 * width, 4 * width, kernel_size=3, padding=1)), + ('bn7', torch.nn.BatchNorm2d(4 * width)), + ('relu7', torch.nn.ReLU()), + + ('pool1', torch.nn.MaxPool2d(3)), + ('flatten', torch.nn.Flatten()), + ('linear', torch.nn.Linear(36 * width, num_classes)) + ])) + + def forward(self, input): + return self.model(input) + + +class LeNetZhu(nn.Module): + """LeNet variant from https://github.com/mit-han-lab/dlg/blob/master/models/vision.py.""" + + def __init__(self, num_classes=10, num_channels=3): + """3-Layer sigmoid Conv with large linear layer.""" + super().__init__() + act = nn.Sigmoid + self.body = nn.Sequential( + nn.Conv2d(num_channels, 12, kernel_size=5, padding=5 // 2, stride=2), + act(), + nn.Conv2d(12, 12, kernel_size=5, padding=5 // 2, stride=2), + act(), + nn.Conv2d(12, 12, kernel_size=5, padding=5 // 2, stride=1), + act(), + ) + self.fc = nn.Sequential( + nn.Linear(768, num_classes) + ) + for module in self.modules(): + self.weights_init(module) + + @staticmethod + def weights_init(m): + if hasattr(m, "weight"): + m.weight.data.uniform_(-0.5, 0.5) + if hasattr(m, "bias"): + m.bias.data.uniform_(-0.5, 0.5) + + def forward(self, x): + out = self.body(x) + out = out.view(out.size(0), -1) + # print(out.size()) + out = self.fc(out) + return out diff --git a/src/inversefed/nn/modules.py b/src/inversefed/nn/modules.py new file mode 100644 index 00000000..bdc69615 --- /dev/null +++ b/src/inversefed/nn/modules.py @@ -0,0 +1,98 @@ +"""For monkey-patching into meta-learning frameworks.""" +import torch +import torch.nn.functional as F +from collections import OrderedDict +from functools import partial +import warnings + +from ..consts import BENCHMARK +torch.backends.cudnn.benchmark = BENCHMARK + +DEBUG = False # Emit warning messages when patching. Use this to bootstrap new architectures. + +class MetaMonkey(torch.nn.Module): + """Trace a networks and then replace its module calls with functional calls. + + This allows for backpropagation w.r.t to weights for "normal" PyTorch networks. + """ + + def __init__(self, net): + """Init with network.""" + super().__init__() + self.net = net + self.parameters = OrderedDict(net.named_parameters()) + + + def forward(self, inputs, parameters=None): + """Live Patch ... :> ...""" + # If no parameter dictionary is given, everything is normal + if parameters is None: + return self.net(inputs) + + # But if not ... + param_gen = iter(parameters.values()) + method_pile = [] + counter = 0 + + for name, module in self.net.named_modules(): + if isinstance(module, torch.nn.Conv2d): + ext_weight = next(param_gen) + if module.bias is not None: + ext_bias = next(param_gen) + else: + ext_bias = None + + method_pile.append(module.forward) + module.forward = partial(F.conv2d, weight=ext_weight, bias=ext_bias, stride=module.stride, + padding=module.padding, dilation=module.dilation, groups=module.groups) + elif isinstance(module, torch.nn.BatchNorm2d): + if module.momentum is None: + exponential_average_factor = 0.0 + else: + exponential_average_factor = module.momentum + + if module.training and module.track_running_stats: + if module.num_batches_tracked is not None: + module.num_batches_tracked += 1 + if module.momentum is None: # use cumulative moving average + exponential_average_factor = 1.0 / float(module.num_batches_tracked) + else: # use exponential moving average + exponential_average_factor = module.momentum + + ext_weight = next(param_gen) + ext_bias = next(param_gen) + method_pile.append(module.forward) + module.forward = partial(F.batch_norm, running_mean=module.running_mean, running_var=module.running_var, + weight=ext_weight, bias=ext_bias, + training=module.training or not module.track_running_stats, + momentum=exponential_average_factor, eps=module.eps) + + elif isinstance(module, torch.nn.Linear): + lin_weights = next(param_gen) + lin_bias = next(param_gen) + method_pile.append(module.forward) + module.forward = partial(F.linear, weight=lin_weights, bias=lin_bias) + + elif next(module.parameters(), None) is None: + # Pass over modules that do not contain parameters + pass + elif isinstance(module, torch.nn.Sequential): + # Pass containers + pass + else: + # Warn for other containers + if DEBUG: + warnings.warn(f'Patching for module {module.__class__} is not implemented.') + + output = self.net(inputs) + + # Undo Patch + for name, module in self.net.named_modules(): + if isinstance(module, torch.nn.modules.conv.Conv2d): + module.forward = method_pile.pop(0) + elif isinstance(module, torch.nn.BatchNorm2d): + module.forward = method_pile.pop(0) + elif isinstance(module, torch.nn.Linear): + module.forward = method_pile.pop(0) + + return output diff --git a/src/inversefed/nn/revnet.py b/src/inversefed/nn/revnet.py new file mode 100644 index 00000000..841a2cb2 --- /dev/null +++ b/src/inversefed/nn/revnet.py @@ -0,0 +1,192 @@ +"""https://github.com/jhjacobsen/pytorch-i-revnet/blob/master/models/iRevNet.py. + +Code for "i-RevNet: Deep Invertible Networks" +https://openreview.net/pdf?id=HJsjkMb0Z +ICLR, 2018 + + +(c) Joern-Henrik Jacobsen, 2018 +""" + +""" +MIT License + +Copyright (c) 2018 Jörn Jacobsen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import torch +import torch.nn as nn +import torch.nn.functional as F +from .revnet_utils import split, merge, injective_pad, psi + + +class irevnet_block(nn.Module): + """This is an i-revnet block from Jacobsen et al.""" + + def __init__(self, in_ch, out_ch, stride=1, first=False, dropout_rate=0., + affineBN=True, mult=4): + """Build invertible bottleneck block.""" + super(irevnet_block, self).__init__() + self.first = first + self.pad = 2 * out_ch - in_ch + self.stride = stride + self.inj_pad = injective_pad(self.pad) + self.psi = psi(stride) + if self.pad != 0 and stride == 1: + in_ch = out_ch * 2 + print('') + print('| Injective iRevNet |') + print('') + layers = [] + if not first: + layers.append(nn.BatchNorm2d(in_ch // 2, affine=affineBN)) + layers.append(nn.ReLU(inplace=True)) + layers.append(nn.Conv2d(in_ch // 2, int(out_ch // mult), kernel_size=3, + stride=stride, padding=1, bias=False)) + layers.append(nn.BatchNorm2d(int(out_ch // mult), affine=affineBN)) + layers.append(nn.ReLU(inplace=True)) + layers.append(nn.Conv2d(int(out_ch // mult), int(out_ch // mult), + kernel_size=3, padding=1, bias=False)) + layers.append(nn.Dropout(p=dropout_rate)) + layers.append(nn.BatchNorm2d(int(out_ch // mult), affine=affineBN)) + layers.append(nn.ReLU(inplace=True)) + layers.append(nn.Conv2d(int(out_ch // mult), out_ch, kernel_size=3, + padding=1, bias=False)) + self.bottleneck_block = nn.Sequential(*layers) + + def forward(self, x): + """Bijective or injective block forward.""" + if self.pad != 0 and self.stride == 1: + x = merge(x[0], x[1]) + x = self.inj_pad.forward(x) + x1, x2 = split(x) + x = (x1, x2) + x1 = x[0] + x2 = x[1] + Fx2 = self.bottleneck_block(x2) + if self.stride == 2: + x1 = self.psi.forward(x1) + x2 = self.psi.forward(x2) + y1 = Fx2 + x1 + return (x2, y1) + + def inverse(self, x): + """Bijective or injecitve block inverse.""" + x2, y1 = x[0], x[1] + if self.stride == 2: + x2 = self.psi.inverse(x2) + Fx2 = - self.bottleneck_block(x2) + x1 = Fx2 + y1 + if self.stride == 2: + x1 = self.psi.inverse(x1) + if self.pad != 0 and self.stride == 1: + x = merge(x1, x2) + x = self.inj_pad.inverse(x) + x1, x2 = split(x) + x = (x1, x2) + else: + x = (x1, x2) + return x + + +class iRevNet(nn.Module): + """This is an i-revnet from Jacobsen et al.""" + + def __init__(self, nBlocks, nStrides, nClasses, nChannels=None, init_ds=2, + dropout_rate=0., affineBN=True, in_shape=None, mult=4): + """Init with e.g. nBlocks=[18, 18, 18], nStrides = [1, 2, 2].""" + super(iRevNet, self).__init__() + self.ds = in_shape[2] // 2**(nStrides.count(2) + init_ds // 2) + self.init_ds = init_ds + self.in_ch = in_shape[0] * 2**self.init_ds + self.nBlocks = nBlocks + self.first = True + + print('') + print(' == Building iRevNet %d == ' % (sum(nBlocks) * 3 + 1)) + if not nChannels: + nChannels = [self.in_ch // 2, self.in_ch // 2 * 4, + self.in_ch // 2 * 4**2, self.in_ch // 2 * 4**3] + + self.init_psi = psi(self.init_ds) + self.stack = self.irevnet_stack(irevnet_block, nChannels, nBlocks, + nStrides, dropout_rate=dropout_rate, + affineBN=affineBN, in_ch=self.in_ch, + mult=mult) + self.bn1 = nn.BatchNorm2d(nChannels[-1] * 2, momentum=0.9) + self.linear = nn.Linear(nChannels[-1] * 2, nClasses) + + def irevnet_stack(self, _block, nChannels, nBlocks, nStrides, dropout_rate, + affineBN, in_ch, mult): + """Create stack of irevnet blocks.""" + block_list = nn.ModuleList() + strides = [] + channels = [] + for channel, depth, stride in zip(nChannels, nBlocks, nStrides): + strides = strides + ([stride] + [1] * (depth - 1)) + channels = channels + ([channel] * depth) + for channel, stride in zip(channels, strides): + block_list.append(_block(in_ch, channel, stride, + first=self.first, + dropout_rate=dropout_rate, + affineBN=affineBN, mult=mult)) + in_ch = 2 * channel + self.first = False + return block_list + + def forward(self, x, return_bijection=False): + """Irevnet forward.""" + n = self.in_ch // 2 + if self.init_ds != 0: + x = self.init_psi.forward(x) + out = (x[:, :n, :, :], x[:, n:, :, :]) + for block in self.stack: + out = block.forward(out) + out_bij = merge(out[0], out[1]) + out = F.relu(self.bn1(out_bij)) + out = F.avg_pool2d(out, self.ds) + out = out.view(out.size(0), -1) + out = self.linear(out) + if return_bijection: + return out, out_bij + else: + return out + + def inverse(self, out_bij): + """Irevnet inverse.""" + out = split(out_bij) + for i in range(len(self.stack)): + out = self.stack[-1 - i].inverse(out) + out = merge(out[0], out[1]) + if self.init_ds != 0: + x = self.init_psi.inverse(out) + else: + x = out + return x + + +if __name__ == '__main__': + model = iRevNet(nBlocks=[6, 16, 72, 6], nStrides=[2, 2, 2, 2], + nChannels=None, nClasses=1000, init_ds=2, + dropout_rate=0., affineBN=True, in_shape=[3, 224, 224], + mult=4) + y = model(torch.randn(1, 3, 224, 224)) + print(y.size()) diff --git a/src/inversefed/nn/revnet_utils.py b/src/inversefed/nn/revnet_utils.py new file mode 100644 index 00000000..aa3eaafb --- /dev/null +++ b/src/inversefed/nn/revnet_utils.py @@ -0,0 +1,132 @@ +"""https://github.com/jhjacobsen/pytorch-i-revnet/blob/master/models/model_utils.py. + +Code for "i-RevNet: Deep Invertible Networks" +https://openreview.net/pdf?id=HJsjkMb0Z +ICLR, 2018 + + +(c) Joern-Henrik Jacobsen, 2018 +""" + +""" +MIT License + +Copyright (c) 2018 Jörn Jacobsen + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +import torch +import torch.nn as nn + +from torch.nn import Parameter + + +def split(x): + n = int(x.size()[1] / 2) + x1 = x[:, :n, :, :].contiguous() + x2 = x[:, n:, :, :].contiguous() + return x1, x2 + + +def merge(x1, x2): + return torch.cat((x1, x2), 1) + + +class injective_pad(nn.Module): + def __init__(self, pad_size): + super(injective_pad, self).__init__() + self.pad_size = pad_size + self.pad = nn.ZeroPad2d((0, 0, 0, pad_size)) + + def forward(self, x): + x = x.permute(0, 2, 1, 3) + x = self.pad(x) + return x.permute(0, 2, 1, 3) + + def inverse(self, x): + return x[:, :x.size(1) - self.pad_size, :, :] + + +class psi(nn.Module): + def __init__(self, block_size): + super(psi, self).__init__() + self.block_size = block_size + self.block_size_sq = block_size * block_size + + def inverse(self, input): + output = input.permute(0, 2, 3, 1) + (batch_size, d_height, d_width, d_depth) = output.size() + s_depth = int(d_depth / self.block_size_sq) + s_width = int(d_width * self.block_size) + s_height = int(d_height * self.block_size) + t_1 = output.contiguous().view(batch_size, d_height, d_width, self.block_size_sq, s_depth) + spl = t_1.split(self.block_size, 3) + stack = [t_t.contiguous().view(batch_size, d_height, s_width, s_depth) for t_t in spl] + output = torch.stack(stack, 0).transpose(0, 1).permute(0, 2, 1, 3, 4).contiguous().view(batch_size, s_height, s_width, s_depth) + output = output.permute(0, 3, 1, 2) + return output.contiguous() + + def forward(self, input): + output = input.permute(0, 2, 3, 1) + (batch_size, s_height, s_width, s_depth) = output.size() + d_depth = s_depth * self.block_size_sq + d_height = int(s_height / self.block_size) + t_1 = output.split(self.block_size, 2) + stack = [t_t.contiguous().view(batch_size, d_height, d_depth) for t_t in t_1] + output = torch.stack(stack, 1) + output = output.permute(0, 2, 1, 3) + output = output.permute(0, 3, 1, 2) + return output.contiguous() + + +class ListModule(object): + def __init__(self, module, prefix, *args): + self.module = module + self.prefix = prefix + self.num_module = 0 + for new_module in args: + self.append(new_module) + + def append(self, new_module): + if not isinstance(new_module, nn.Module): + raise ValueError('Not a Module') + else: + self.module.add_module(self.prefix + str(self.num_module), new_module) + self.num_module += 1 + + def __len__(self): + return self.num_module + + def __getitem__(self, i): + if i < 0 or i >= self.num_module: + raise IndexError('Out of bound') + return getattr(self.module, self.prefix + str(i)) + + +def get_all_params(var, all_params): + if isinstance(var, Parameter): + all_params[id(var)] = var.nelement() + elif hasattr(var, "creator") and var.creator is not None: + if var.creator.previous_functions is not None: + for j in var.creator.previous_functions: + get_all_params(j[0], all_params) + elif hasattr(var, "previous_functions"): + for j in var.previous_functions: + get_all_params(j[0], all_params) diff --git a/src/inversefed/optimization_strategy.py b/src/inversefed/optimization_strategy.py new file mode 100644 index 00000000..2c73483d --- /dev/null +++ b/src/inversefed/optimization_strategy.py @@ -0,0 +1,78 @@ +"""Optimization setups.""" + +from dataclasses import dataclass + + +def training_strategy(strategy, lr=None, epochs=None, dryrun=False): + """Parse training strategy.""" + if strategy == 'conservative': + defs = ConservativeStrategy(lr, epochs, dryrun) + elif strategy == 'adam': + defs = AdamStrategy(lr, epochs, dryrun) + else: + raise ValueError('Unknown training strategy.') + return defs + + +@dataclass +class Strategy: + """Default usual parameters, not intended for parsing.""" + + epochs : int + batch_size : int + optimizer : str + lr : float + scheduler : str + weight_decay : float + validate : int + warmup: bool + dryrun : bool + dropout : float + augmentations : bool + + def __init__(self, lr=None, epochs=None, dryrun=False): + """Defaulted parameters. Apply overwrites from args.""" + if epochs is not None: + self.epochs = epochs + if lr is not None: + self.lr = lr + if dryrun: + self.dryrun = dryrun + self.validate = 10 + +@dataclass +class ConservativeStrategy(Strategy): + """Default usual parameters, defines a config object.""" + + def __init__(self, lr=None, epochs=None, dryrun=False): + """Initialize training hyperparameters.""" + self.lr = 0.1 + self.epochs = 120 + self.batch_size = 128 + self.optimizer = 'SGD' + self.scheduler = 'linear' + self.warmup = False + self.weight_decay : float = 5e-4 + self.dropout = 0.0 + self.augmentations = True + self.dryrun = False + super().__init__(lr=None, epochs=None, dryrun=False) + + +@dataclass +class AdamStrategy(Strategy): + """Start slowly. Use a tame Adam.""" + + def __init__(self, lr=None, epochs=None, dryrun=False): + """Initialize training hyperparameters.""" + self.lr = 1e-3 / 10 + self.epochs = 120 + self.batch_size = 32 + self.optimizer = 'AdamW' + self.scheduler = 'linear' + self.warmup = True + self.weight_decay : float = 5e-4 + self.dropout = 0.0 + self.augmentations = True + self.dryrun = False + super().__init__(lr=None, epochs=None, dryrun=False) diff --git a/src/inversefed/options.py b/src/inversefed/options.py new file mode 100644 index 00000000..b8bc3ebe --- /dev/null +++ b/src/inversefed/options.py @@ -0,0 +1,52 @@ +"""Parser options.""" + +import argparse + +def options(): + """Construct the central argument parser, filled with useful defaults.""" + parser = argparse.ArgumentParser(description='Reconstruct some image from a trained model.') + + # Central: + parser.add_argument('--model', default='ConvNet', type=str, help='Vision model.') + parser.add_argument('--dataset', default='CIFAR10', type=str) + parser.add_argument('--dtype', default='float', type=str, help='Data type used during reconstruction [Not during training!].') + + + parser.add_argument('--trained_model', action='store_true', help='Use a trained model.') + parser.add_argument('--epochs', default=120, type=int, help='If using a trained model, how many epochs was it trained?') + + parser.add_argument('--accumulation', default=0, type=int, help='Accumulation 0 is rec. from gradient, accumulation > 0 is reconstruction from fed. averaging.') + parser.add_argument('--num_images', default=1, type=int, help='How many images should be recovered from the given gradient.') + parser.add_argument('--target_id', default=None, type=int, help='Cifar validation image used for reconstruction.') + parser.add_argument('--label_flip', action='store_true', help='Dishonest server permuting weights in classification layer.') + + # Rec. parameters + parser.add_argument('--optim', default='ours', type=str, help='Use our reconstruction method or the DLG method.') + + parser.add_argument('--restarts', default=1, type=int, help='How many restarts to run.') + parser.add_argument('--cost_fn', default='sim', type=str, help='Choice of cost function.') + parser.add_argument('--indices', default='def', type=str, help='Choice of indices from the parameter list.') + parser.add_argument('--weights', default='equal', type=str, help='Weigh the parameter list differently.') + + parser.add_argument('--optimizer', default='adam', type=str, help='Weigh the parameter list differently.') + parser.add_argument('--signed', action='store_false', help='Do not used signed gradients.') + parser.add_argument('--boxed', action='store_false', help='Do not used box constraints.') + + parser.add_argument('--scoring_choice', default='loss', type=str, help='How to find the best image between all restarts.') + parser.add_argument('--init', default='randn', type=str, help='Choice of image initialization.') + parser.add_argument('--tv', default=1e-4, type=float, help='Weight of TV penalty.') + + + # Files and folders: + parser.add_argument('--save_image', action='store_true', help='Save the output to a file.') + + parser.add_argument('--image_path', default='images/', type=str) + parser.add_argument('--model_path', default='models/', type=str) + parser.add_argument('--table_path', default='tables/', type=str) + parser.add_argument('--data_path', default='~/data', type=str) + + # Debugging: + parser.add_argument('--name', default='iv', type=str, help='Name tag for the result table and model.') + parser.add_argument('--deterministic', action='store_true', help='Disable CUDNN non-determinism.') + parser.add_argument('--dryrun', action='store_true', help='Run everything for just one step to test functionality.') + return parser diff --git a/src/inversefed/reconstruction_algorithms.py b/src/inversefed/reconstruction_algorithms.py new file mode 100644 index 00000000..63965d85 --- /dev/null +++ b/src/inversefed/reconstruction_algorithms.py @@ -0,0 +1,392 @@ +"""Mechanisms for image reconstruction from parameter gradients.""" + +import torch +from collections import defaultdict, OrderedDict +from inversefed.nn import MetaMonkey +from .metrics import total_variation as TV +from .metrics import InceptionScore +from .medianfilt import MedianPool2d +from copy import deepcopy + +import time + +DEFAULT_CONFIG = dict(signed=False, + boxed=True, + cost_fn='sim', + indices='def', + weights='equal', + lr=0.1, + optim='adam', + restarts=1, + max_iterations=4800, + total_variation=1e-1, + init='randn', + filter='none', + lr_decay=True, + scoring_choice='loss') + +def _label_to_onehot(target, num_classes=100): + target = torch.unsqueeze(target, 1) + onehot_target = torch.zeros(target.size(0), num_classes, device=target.device) + onehot_target.scatter_(1, target, 1) + return onehot_target + +def _validate_config(config): + for key in DEFAULT_CONFIG.keys(): + if config.get(key) is None: + config[key] = DEFAULT_CONFIG[key] + for key in config.keys(): + if DEFAULT_CONFIG.get(key) is None: + raise ValueError(f'Deprecated key in config dict: {key}!') + return config + + +class GradientReconstructor(): + """Instantiate a reconstruction algorithm.""" + + def __init__(self, model, mean_std=(0.0, 1.0), config=DEFAULT_CONFIG, num_images=1): + """Initialize with algorithm setup.""" + self.config = _validate_config(config) + self.model = model + self.setup = dict(device=next(model.parameters()).device, dtype=next(model.parameters()).dtype) + + self.mean_std = mean_std + self.num_images = num_images + + if self.config['scoring_choice'] == 'inception': + self.inception = InceptionScore(batch_size=1, setup=self.setup) + + self.loss_fn = torch.nn.CrossEntropyLoss(reduction='mean') + self.iDLG = True + + def reconstruct(self, input_data, labels, img_shape=(3, 32, 32), dryrun=False, eval=True, tol=None): + """Reconstruct image from gradient.""" + start_time = time.time() + if eval: + self.model.eval() + + + stats = defaultdict(list) + x = self._init_images(img_shape) + scores = torch.zeros(self.config['restarts']) + + if labels is None: + if self.num_images == 1 and self.iDLG: + # iDLG trick: + last_weight_min = torch.argmin(torch.sum(input_data[-2], dim=-1), dim=-1) + labels = last_weight_min.detach().reshape((1,)).requires_grad_(False) + self.reconstruct_label = False + else: + # DLG label recovery + # However this also improves conditioning for some LBFGS cases + self.reconstruct_label = True + + def loss_fn(pred, labels): + labels = torch.nn.functional.softmax(labels, dim=-1) + return torch.mean(torch.sum(- labels * torch.nn.functional.log_softmax(pred, dim=-1), 1)) + self.loss_fn = loss_fn + else: + assert labels.shape[0] == self.num_images + self.reconstruct_label = False + + try: + for trial in range(self.config['restarts']): + x_trial, labels = self._run_trial(x[trial], input_data, labels, dryrun=dryrun) + # Finalize + scores[trial] = self._score_trial(x_trial, input_data, labels) + x[trial] = x_trial + if tol is not None and scores[trial] <= tol: + break + if dryrun: + break + except KeyboardInterrupt: + print('Trial procedure manually interruped.') + pass + + # Choose optimal result: + if self.config['scoring_choice'] in ['pixelmean', 'pixelmedian']: + x_optimal, stats = self._average_trials(x, labels, input_data, stats) + else: + print('Choosing optimal result ...') + scores = scores[torch.isfinite(scores)] # guard against NaN/-Inf scores? + optimal_index = torch.argmin(scores) + print(f'Optimal result score: {scores[optimal_index]:2.4f}') + stats['opt'] = scores[optimal_index].item() + x_optimal = x[optimal_index] + + print(f'Total time: {time.time()-start_time}.') + return x_optimal.detach(), stats + + def _init_images(self, img_shape): + if self.config['init'] == 'randn': + return torch.randn((self.config['restarts'], self.num_images, *img_shape), **self.setup) + elif self.config['init'] == 'rand': + return (torch.rand((self.config['restarts'], self.num_images, *img_shape), **self.setup) - 0.5) * 2 + elif self.config['init'] == 'zeros': + return torch.zeros((self.config['restarts'], self.num_images, *img_shape), **self.setup) + else: + raise ValueError() + + def _run_trial(self, x_trial, input_data, labels, dryrun=False): + x_trial.requires_grad = True + if self.reconstruct_label: + output_test = self.model(x_trial) + labels = torch.randn(output_test.shape[1]).to(**self.setup).requires_grad_(True) + + if self.config['optim'] == 'adam': + optimizer = torch.optim.Adam([x_trial, labels], lr=self.config['lr']) + elif self.config['optim'] == 'sgd': # actually gd + optimizer = torch.optim.SGD([x_trial, labels], lr=0.01, momentum=0.9, nesterov=True) + elif self.config['optim'] == 'LBFGS': + optimizer = torch.optim.LBFGS([x_trial, labels]) + else: + raise ValueError() + else: + if self.config['optim'] == 'adam': + optimizer = torch.optim.Adam([x_trial], lr=self.config['lr']) + elif self.config['optim'] == 'sgd': # actually gd + optimizer = torch.optim.SGD([x_trial], lr=0.01, momentum=0.9, nesterov=True) + elif self.config['optim'] == 'LBFGS': + optimizer = torch.optim.LBFGS([x_trial]) + else: + raise ValueError() + + max_iterations = self.config['max_iterations'] + dm, ds = self.mean_std + if self.config['lr_decay']: + scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, + milestones=[max_iterations // 2.667, max_iterations // 1.6, + + max_iterations // 1.142], gamma=0.1) # 3/8 5/8 7/8 + try: + for iteration in range(max_iterations): + closure = self._gradient_closure(optimizer, x_trial, input_data, labels) + rec_loss = optimizer.step(closure) + if self.config['lr_decay']: + scheduler.step() + + with torch.no_grad(): + # Project into image space + if self.config['boxed']: + x_trial.data = torch.max(torch.min(x_trial, (1 - dm) / ds), -dm / ds) + + if (iteration + 1 == max_iterations) or iteration % 500 == 0: + print(f'It: {iteration}. Rec. loss: {rec_loss.item():2.4f}.') + + if (iteration + 1) % 500 == 0: + if self.config['filter'] == 'none': + pass + elif self.config['filter'] == 'median': + x_trial.data = MedianPool2d(kernel_size=3, stride=1, padding=1, same=False)(x_trial) + else: + raise ValueError() + + if dryrun: + break + except KeyboardInterrupt: + print(f'Recovery interrupted manually in iteration {iteration}!') + pass + return x_trial.detach(), labels + + def _gradient_closure(self, optimizer, x_trial, input_gradient, label): + + def closure(): + optimizer.zero_grad() + self.model.zero_grad() + loss = self.loss_fn(self.model(x_trial), label) + gradient = torch.autograd.grad(loss, self.model.parameters(), create_graph=True) + rec_loss = reconstruction_costs([gradient], input_gradient, + cost_fn=self.config['cost_fn'], indices=self.config['indices'], + weights=self.config['weights']) + + if self.config['total_variation'] > 0: + rec_loss += self.config['total_variation'] * TV(x_trial) + rec_loss.backward() + if self.config['signed']: + x_trial.grad.sign_() + return rec_loss + return closure + + def _score_trial(self, x_trial, input_gradient, label): + if self.config['scoring_choice'] == 'loss': + self.model.zero_grad() + x_trial.grad = None + loss = self.loss_fn(self.model(x_trial), label) + gradient = torch.autograd.grad(loss, self.model.parameters(), create_graph=False) + return reconstruction_costs([gradient], input_gradient, + cost_fn=self.config['cost_fn'], indices=self.config['indices'], + weights=self.config['weights']) + elif self.config['scoring_choice'] == 'tv': + return TV(x_trial) + elif self.config['scoring_choice'] == 'inception': + # We do not care about diversity here! + return self.inception(x_trial) + elif self.config['scoring_choice'] in ['pixelmean', 'pixelmedian']: + return 0.0 + else: + raise ValueError() + + def _average_trials(self, x, labels, input_data, stats): + print(f'Computing a combined result via {self.config["scoring_choice"]} ...') + if self.config['scoring_choice'] == 'pixelmedian': + x_optimal, _ = x.median(dim=0, keepdims=False) + elif self.config['scoring_choice'] == 'pixelmean': + x_optimal = x.mean(dim=0, keepdims=False) + + self.model.zero_grad() + if self.reconstruct_label: + labels = self.model(x_optimal).softmax(dim=1) + loss = self.loss_fn(self.model(x_optimal), labels) + gradient = torch.autograd.grad(loss, self.model.parameters(), create_graph=False) + stats['opt'] = reconstruction_costs([gradient], input_data, + cost_fn=self.config['cost_fn'], + indices=self.config['indices'], + weights=self.config['weights']) + print(f'Optimal result score: {stats["opt"]:2.4f}') + return x_optimal, stats + + + +class FedAvgReconstructor(GradientReconstructor): + """Reconstruct an image from weights after n gradient descent steps.""" + + def __init__(self, model, mean_std=(0.0, 1.0), local_steps=2, local_lr=1e-4, + config=DEFAULT_CONFIG, num_images=1, use_updates=True, batch_size=0): + """Initialize with model, (mean, std) and config.""" + super().__init__(model, mean_std, config, num_images) + self.local_steps = local_steps + self.local_lr = local_lr + self.use_updates = use_updates + self.batch_size = batch_size + + def _gradient_closure(self, optimizer, x_trial, input_parameters, labels): + def closure(): + optimizer.zero_grad() + self.model.zero_grad() + parameters = loss_steps(self.model, x_trial, labels, loss_fn=self.loss_fn, + local_steps=self.local_steps, lr=self.local_lr, + use_updates=self.use_updates, + batch_size=self.batch_size) + rec_loss = reconstruction_costs([parameters], input_parameters, + cost_fn=self.config['cost_fn'], indices=self.config['indices'], + weights=self.config['weights']) + + if self.config['total_variation'] > 0: + rec_loss += self.config['total_variation'] * TV(x_trial) + rec_loss.backward() + if self.config['signed']: + x_trial.grad.sign_() + return rec_loss + return closure + + def _score_trial(self, x_trial, input_parameters, labels): + if self.config['scoring_choice'] == 'loss': + self.model.zero_grad() + parameters = loss_steps(self.model, x_trial, labels, loss_fn=self.loss_fn, + local_steps=self.local_steps, lr=self.local_lr, use_updates=self.use_updates) + return reconstruction_costs([parameters], input_parameters, + cost_fn=self.config['cost_fn'], indices=self.config['indices'], + weights=self.config['weights']) + elif self.config['scoring_choice'] == 'tv': + return TV(x_trial) + elif self.config['scoring_choice'] == 'inception': + # We do not care about diversity here! + return self.inception(x_trial) + + +def loss_steps(model, inputs, labels, loss_fn=torch.nn.CrossEntropyLoss(), lr=1e-4, local_steps=4, use_updates=True, batch_size=0): + """Take a few gradient descent steps to fit the model to the given input.""" + patched_model = MetaMonkey(model) + if use_updates: + patched_model_origin = deepcopy(patched_model) + for i in range(local_steps): + if batch_size == 0: + outputs = patched_model(inputs, patched_model.parameters) + labels_ = labels + else: + idx = i % (inputs.shape[0] // batch_size) + outputs = patched_model(inputs[idx * batch_size:(idx + 1) * batch_size], patched_model.parameters) + labels_ = labels[idx * batch_size:(idx + 1) * batch_size] + loss = loss_fn(outputs, labels_).sum() + grad = torch.autograd.grad(loss, patched_model.parameters.values(), + retain_graph=True, create_graph=True, only_inputs=True) + + patched_model.parameters = OrderedDict((name, param - lr * grad_part) + for ((name, param), grad_part) + in zip(patched_model.parameters.items(), grad)) + + if use_updates: + patched_model.parameters = OrderedDict((name, param - param_origin) + for ((name, param), (name_origin, param_origin)) + in zip(patched_model.parameters.items(), patched_model_origin.parameters.items())) + return list(patched_model.parameters.values()) + + +def reconstruction_costs(gradients, input_gradient, cost_fn='l2', indices='def', weights='equal'): + """Input gradient is given data.""" + if isinstance(indices, list): + pass + elif indices == 'def': + indices = torch.arange(len(input_gradient)) + elif indices == 'batch': + indices = torch.randperm(len(input_gradient))[:8] + elif indices == 'topk-1': + _, indices = torch.topk(torch.stack([p.norm() for p in input_gradient], dim=0), 4) + elif indices == 'top10': + _, indices = torch.topk(torch.stack([p.norm() for p in input_gradient], dim=0), 10) + elif indices == 'top50': + _, indices = torch.topk(torch.stack([p.norm() for p in input_gradient], dim=0), 50) + elif indices in ['first', 'first4']: + indices = torch.arange(0, 4) + elif indices == 'first5': + indices = torch.arange(0, 5) + elif indices == 'first10': + indices = torch.arange(0, 10) + elif indices == 'first50': + indices = torch.arange(0, 50) + elif indices == 'last5': + indices = torch.arange(len(input_gradient))[-5:] + elif indices == 'last10': + indices = torch.arange(len(input_gradient))[-10:] + elif indices == 'last50': + indices = torch.arange(len(input_gradient))[-50:] + else: + raise ValueError() + + ex = input_gradient[0] + if weights == 'linear': + weights = torch.arange(len(input_gradient), 0, -1, dtype=ex.dtype, device=ex.device) / len(input_gradient) + elif weights == 'exp': + weights = torch.arange(len(input_gradient), 0, -1, dtype=ex.dtype, device=ex.device) + weights = weights.softmax(dim=0) + weights = weights / weights[0] + else: + weights = input_gradient[0].new_ones(len(input_gradient)) + + total_costs = 0 + for trial_gradient in gradients: + pnorm = [0, 0] + costs = 0 + if indices == 'topk-2': + _, indices = torch.topk(torch.stack([p.norm().detach() for p in trial_gradient], dim=0), 4) + for i in indices: + if cost_fn == 'l2': + costs += ((trial_gradient[i] - input_gradient[i]).pow(2)).sum() * weights[i] + elif cost_fn == 'l1': + costs += ((trial_gradient[i] - input_gradient[i]).abs()).sum() * weights[i] + elif cost_fn == 'max': + costs += ((trial_gradient[i] - input_gradient[i]).abs()).max() * weights[i] + elif cost_fn == 'sim': + costs -= (trial_gradient[i] * input_gradient[i]).sum() * weights[i] + pnorm[0] += trial_gradient[i].pow(2).sum() * weights[i] + pnorm[1] += input_gradient[i].pow(2).sum() * weights[i] + elif cost_fn == 'simlocal': + costs += 1 - torch.nn.functional.cosine_similarity(trial_gradient[i].flatten(), + input_gradient[i].flatten(), + 0, 1e-10) * weights[i] + if cost_fn == 'sim': + costs = 1 + costs / pnorm[0].sqrt() / pnorm[1].sqrt() + + # Accumulate final costs + total_costs += costs + return total_costs / len(gradients) diff --git a/src/inversefed/training/README.md b/src/inversefed/training/README.md new file mode 100644 index 00000000..5897a788 --- /dev/null +++ b/src/inversefed/training/README.md @@ -0,0 +1 @@ +# Training routines are implemented here \ No newline at end of file diff --git a/src/inversefed/training/__init__.py b/src/inversefed/training/__init__.py new file mode 100644 index 00000000..d1456816 --- /dev/null +++ b/src/inversefed/training/__init__.py @@ -0,0 +1,5 @@ +"""Basic training routines and loss functions.""" + +from .training_routine import train + +__all__ = ['train'] diff --git a/src/inversefed/training/scheduler.py b/src/inversefed/training/scheduler.py new file mode 100644 index 00000000..e5c4a247 --- /dev/null +++ b/src/inversefed/training/scheduler.py @@ -0,0 +1,95 @@ +"""This file is part of https://github.com/ildoonet/pytorch-gradual-warmup-lr. + +MIT License + +Copyright (c) 2019 Ildoo Kim + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +""" + +from torch.optim.lr_scheduler import _LRScheduler +from torch.optim.lr_scheduler import ReduceLROnPlateau + + +class GradualWarmupScheduler(_LRScheduler): + """Gradually warm-up(increasing) learning rate in optimizer. + + Proposed in 'Accurate, Large Minibatch SGD: Training ImageNet in 1 Hour'. + + Args: + optimizer (Optimizer): Wrapped optimizer. + multiplier: target learning rate = base lr * multiplier + total_epoch: target learning rate is reached at total_epoch, gradually + after_scheduler: after target_epoch, use this scheduler(eg. ReduceLROnPlateau) + + """ + + def __init__(self, optimizer, multiplier, total_epoch, after_scheduler=None): + """Initialize the warm-up start. + + Usage: + + scheduler_normal = torch.optim.lr_scheduler.MultiStepLR(optimizer) + scheduler_warmup = GradualWarmupScheduler(optimizer, multiplier=8, total_epoch=10, after_scheduler=scheduler_normal) + """ + self.multiplier = multiplier + if self.multiplier < 1.: + raise ValueError('multiplier should be greater thant or equal to 1.') + self.total_epoch = total_epoch + self.after_scheduler = after_scheduler + self.finished = False + super().__init__(optimizer) + + def get_lr(self): + if self.last_epoch > self.total_epoch: + if self.after_scheduler: + if not self.finished: + self.after_scheduler.base_lrs = [base_lr * self.multiplier for base_lr in self.base_lrs] + self.finished = True + return self.after_scheduler.get_lr() + return [base_lr * self.multiplier for base_lr in self.base_lrs] + + return [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs] + + def step_ReduceLROnPlateau(self, metrics, epoch=None): + if epoch is None: + epoch = self.last_epoch + 1 + self.last_epoch = epoch if epoch != 0 else 1 # ReduceLROnPlateau is called at the end of epoch, whereas others are called at beginning + if self.last_epoch <= self.total_epoch: + warmup_lr = [base_lr * ((self.multiplier - 1.) * self.last_epoch / self.total_epoch + 1.) for base_lr in self.base_lrs] + for param_group, lr in zip(self.optimizer.param_groups, warmup_lr): + param_group['lr'] = lr + else: + if epoch is None: + self.after_scheduler.step(metrics, None) + else: + self.after_scheduler.step(metrics, epoch - self.total_epoch) + + def step(self, epoch=None, metrics=None): + if type(self.after_scheduler) != ReduceLROnPlateau: + if self.finished and self.after_scheduler: + if epoch is None: + self.after_scheduler.step(None) + else: + self.after_scheduler.step(epoch - self.total_epoch) + else: + return super(GradualWarmupScheduler, self).step(epoch) + else: + self.step_ReduceLROnPlateau(metrics, epoch) diff --git a/src/inversefed/training/training_routine.py b/src/inversefed/training/training_routine.py new file mode 100644 index 00000000..8e16db84 --- /dev/null +++ b/src/inversefed/training/training_routine.py @@ -0,0 +1,124 @@ +"""Implement the .train function.""" + +import torch +import numpy as np + +from collections import defaultdict + +from .scheduler import GradualWarmupScheduler + +from ..consts import BENCHMARK, NON_BLOCKING +torch.backends.cudnn.benchmark = BENCHMARK + +def train(model, loss_fn, trainloader, validloader, defs, setup=dict(dtype=torch.float, device=torch.device('cpu'))): + """Run the main interface. Train a network with specifications from the Strategy object.""" + stats = defaultdict(list) + optimizer, scheduler = set_optimizer(model, defs) + + for epoch in range(defs.epochs): + model.train() + step(model, loss_fn, trainloader, optimizer, scheduler, defs, setup, stats) + + if epoch % defs.validate == 0 or epoch == (defs.epochs - 1): + model.eval() + validate(model, loss_fn, validloader, defs, setup, stats) + # Print information about loss and accuracy + print_status(epoch, loss_fn, optimizer, stats) + + if defs.dryrun: + break + if not (np.isfinite(stats['train_losses'][-1])): + print('Loss is NaN/Inf ... terminating early ...') + break + + return stats + +def step(model, loss_fn, dataloader, optimizer, scheduler, defs, setup, stats): + """Step through one epoch.""" + epoch_loss, epoch_metric = 0, 0 + for batch, (inputs, targets) in enumerate(dataloader): + # Prep Mini-Batch + optimizer.zero_grad() + + # Transfer to GPU + inputs = inputs.to(**setup) + targets = targets.to(device=setup['device'], non_blocking=NON_BLOCKING) + + # Get loss + outputs = model(inputs) + loss, _, _ = loss_fn(outputs, targets) + + + epoch_loss += loss.item() + + loss.backward() + optimizer.step() + + metric, name, _ = loss_fn.metric(outputs, targets) + epoch_metric += metric.item() + + if defs.scheduler == 'cyclic': + scheduler.step() + if defs.dryrun: + break + if defs.scheduler == 'linear': + scheduler.step() + + stats['train_losses'].append(epoch_loss / (batch + 1)) + stats['train_' + name].append(epoch_metric / (batch + 1)) + + +def validate(model, loss_fn, dataloader, defs, setup, stats): + """Validate model effectiveness of val dataset.""" + epoch_loss, epoch_metric = 0, 0 + with torch.no_grad(): + for batch, (inputs, targets) in enumerate(dataloader): + # Transfer to GPU + inputs = inputs.to(**setup) + targets = targets.to(device=setup['device'], non_blocking=NON_BLOCKING) + + # Get loss and metric + outputs = model(inputs) + loss, _, _ = loss_fn(outputs, targets) + metric, name, _ = loss_fn.metric(outputs, targets) + + epoch_loss += loss.item() + epoch_metric += metric.item() + + if defs.dryrun: + break + + stats['valid_losses'].append(epoch_loss / (batch + 1)) + stats['valid_' + name].append(epoch_metric / (batch + 1)) + +def set_optimizer(model, defs): + """Build model optimizer and scheduler from defs. + + The linear scheduler drops the learning rate in intervals. + # Example: epochs=160 leads to drops at 60, 100, 140. + """ + if defs.optimizer == 'SGD': + optimizer = torch.optim.SGD(model.parameters(), lr=defs.lr, momentum=0.9, + weight_decay=defs.weight_decay, nesterov=True) + elif defs.optimizer == 'AdamW': + optimizer = torch.optim.AdamW(model.parameters(), lr=defs.lr, weight_decay=defs.weight_decay) + + if defs.scheduler == 'linear': + scheduler = torch.optim.lr_scheduler.MultiStepLR(optimizer, + milestones=[120 // 2.667, 120 // 1.6, + 120 // 1.142], gamma=0.1) + # Scheduler is fixed to 120 epochs so that calls with fewer epochs are equal in lr drops. + + if defs.warmup: + scheduler = GradualWarmupScheduler(optimizer, multiplier=10, total_epoch=10, after_scheduler=scheduler) + + return optimizer, scheduler + + +def print_status(epoch, loss_fn, optimizer, stats): + """Print basic console printout every defs.validation epochs.""" + current_lr = optimizer.param_groups[0]['lr'] + name, format = loss_fn.metric() + print(f'Epoch: {epoch}| lr: {current_lr:.4f} | ' + f'Train loss is {stats["train_losses"][-1]:6.4f}, Train {name}: {stats["train_" + name][-1]:{format}} | ' + f'Val loss is {stats["valid_losses"][-1]:6.4f}, Val {name}: {stats["valid_" + name][-1]:{format}} |') diff --git a/src/inversefed/utils.py b/src/inversefed/utils.py new file mode 100644 index 00000000..cf9c2d9f --- /dev/null +++ b/src/inversefed/utils.py @@ -0,0 +1,70 @@ +"""Various utilities.""" + +import os +import csv + +import torch +import random +import numpy as np + +import socket +import datetime + + +def system_startup(args=None, defs=None): + """Print useful system information.""" + # Choose GPU device and print status information: + device = torch.device('cuda:0') if torch.cuda.is_available() else torch.device('cpu') + setup = dict(device=device, dtype=torch.float) # non_blocking=NON_BLOCKING + print('Currently evaluating -------------------------------:') + print(datetime.datetime.now().strftime("%A, %d. %B %Y %I:%M%p")) + print(f'CPUs: {torch.get_num_threads()}, GPUs: {torch.cuda.device_count()} on {socket.gethostname()}.') + if args is not None: + print(args) + if defs is not None: + print(repr(defs)) + if torch.cuda.is_available(): + print(f'GPU : {torch.cuda.get_device_name(device=device)}') + return setup + +def save_to_table(out_dir, name, dryrun, **kwargs): + """Save keys to .csv files. Function adapted from Micah.""" + # Check for file + if not os.path.isdir(out_dir): + os.makedirs(out_dir) + fname = os.path.join(out_dir, f'table_{name}.csv') + fieldnames = list(kwargs.keys()) + + # Read or write header + try: + with open(fname, 'r') as f: + reader = csv.reader(f, delimiter='\t') + header = [line for line in reader][0] + except Exception as e: + print('Creating a new .csv table...') + with open(fname, 'w') as f: + writer = csv.DictWriter(f, delimiter='\t', fieldnames=fieldnames) + writer.writeheader() + if not dryrun: + # Add row for this experiment + with open(fname, 'a') as f: + writer = csv.DictWriter(f, delimiter='\t', fieldnames=fieldnames) + writer.writerow(kwargs) + print('\nResults saved to ' + fname + '.') + else: + print(f'Would save results to {fname}.') + print(f'Would save these keys: {fieldnames}.') + +def set_random_seed(seed=233): + """233 = 144 + 89 is my favorite number.""" + torch.manual_seed(seed + 1) + torch.cuda.manual_seed(seed + 2) + torch.cuda.manual_seed_all(seed + 3) + np.random.seed(seed + 4) + torch.cuda.manual_seed_all(seed + 5) + random.seed(seed + 6) + +def set_deterministic(): + """Switch pytorch into a deterministic computation mode.""" + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False From 0316813b487656f5c8c86a2b9226848f4981de8b Mon Sep 17 00:00:00 2001 From: photonshi Date: Tue, 22 Oct 2024 04:50:16 +0000 Subject: [PATCH 02/16] first commit - testing grpc --- src/algos/fl_inversionAttack.py | 47 ++++++++++++++++++++++++--- src/configs/algo_config.py | 19 ++++++++++- src/scheduler.py | 2 ++ src/utils/communication/comm_utils.py | 16 ++++++++- src/utils/communication/grpc/main.py | 2 +- 5 files changed, 79 insertions(+), 7 deletions(-) diff --git a/src/algos/fl_inversionAttack.py b/src/algos/fl_inversionAttack.py index 757afc54..aadb7515 100644 --- a/src/algos/fl_inversionAttack.py +++ b/src/algos/fl_inversionAttack.py @@ -271,6 +271,7 @@ def inverting_gradients_attack(self): Based on reconstruction from weight script: https://github.com/JonasGeiping/invertinggradients/blob/1157b61c6704df42c497ab9eb074c75da5204334/Recovery%20from%20Weight%20Updates.ipynb """ + setup = inversefed.utils.system_startup() if self.dset == "cifar10": # TODO figure out whehether we actually have the dm and ds values in our codebase dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None] @@ -317,15 +318,53 @@ def inverting_gradients_attack(self): use_updates = False rec_machine = inversefed.FedAvgReconstructor(self.model, (dm, ds), local_steps, local_lr, config, use_updates=use_updates) - output, stats = rec_machine.reconstruct(input_parameters, labels, img_shape=(3, 32, 32)) # TODO verify img_shape and change it based on dataset - test_mse = (output.detach() - ground_truth).pow(2).mean() - feat_mse = (model(output.detach())- model(ground_truth)).pow(2).mean() - test_psnr = inversefed.metrics.psnr(output, ground_truth, factor=1/ds) + output, stats = rec_machine.reconstruct(params_i, labels_i, img_shape=(3, 32, 32)) # TODO verify img_shape and change it based on dataset + test_mse = (output.detach() - ground_truth_i).pow(2).mean() + feat_mse = (self.model(output.detach())- self.model(ground_truth_i)).pow(2).mean() + test_psnr = inversefed.metrics.psnr(output, ground_truth_i, factor=1/ds) # optional plotting: # plot(output) # plt.title(f"Rec. loss: {stats['opt']:2.4f} | MSE: {test_mse:2.4f} " # f"| PSNR: {test_psnr:4.2f} | FMSE: {feat_mse:2.4e} |"); + return output, test_mse, test_psnr, feat_mse + def run_protocol(self): + """ + basically a carbon copy of fl.py's run protocol. Except attack is launched at the end + """ + self.log_utils.log_console("Starting clients federated averaging") + start_epochs = self.config.get("start_epochs", 0) + total_epochs = self.config["epochs"] + for round in range(start_epochs, total_epochs): + + if round == total_epochs - 1: + self.log_utils.log_console("Launching inversion attack") + output, test_mse, test_psnr, feat_mse = self.inverting_gradients_attack() + self.log_utils.log_console("Inversion attack complete") + self.log_utils.log_summary( + f"Round {round} inversion attack complete. Test MSE: {test_mse}, Test PSNR: {test_psnr}, Feature MSE: {feat_mse}" + ) + # TODO somehow save output? + + self.log_utils.log_console("Starting round {}".format(round)) + self.log_utils.log_summary("Starting round {}".format(round)) + self.single_round() + self.log_utils.log_console("Server testing the model") + loss, acc, time_taken = self.test() + self.log_utils.log_tb(f"test_acc/clients", acc, round) + self.log_utils.log_tb(f"test_loss/clients", loss, round) + self.log_utils.log_console( + "Round: {} test_acc:{:.4f}, test_loss:{:.4f}, time taken {:.2f} seconds".format( + round, acc, loss, time_taken + ) + ) + # self.log_utils.log_summary("Round: {} test_acc:{:.4f}, test_loss:{:.4f}, time taken {:.2f} seconds".format(round, acc, loss, time_taken)) + self.log_utils.log_console("Round {} complete".format(round)) + self.log_utils.log_summary( + "Round {} complete".format( + round, + ) + ) class GradientInversionFedStaticServer(FedStaticServer): """ implements gradient inversion attack to reconstruct training images from other nodes diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index 0cee71f5..987b4b8f 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -55,6 +55,18 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st "malicious_type": "normal", } + +test_fl_inversion: ConfigType = { + "algo": "fedavg_inversion", + "exp_type": "", + # Learning setup + "epochs": 10, + "model": "resnet10", + "model_lr": 3e-4, + "batch_size": 256, + "malicious_type": "normal", +} + malicious_traditional_bad_weights: ConfigType = { **traditional_fl, "malicious_type": "bad_weights", @@ -65,6 +77,10 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st "malicious_type": "sign_flip", } +# malicious_gradient_inversion: ConfigType = { +# **traditional_fl, +# "malicious_type": "gradient_inversion", +# } fedweight: ConfigType = { "algo": "fedweight", @@ -367,4 +383,5 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st malicious_traditional_flip_signs, ] -default_config_list: List[ConfigType] = [traditional_fl] +# default_config_list: List[ConfigType] = [traditional_fl] +default_config_list: List[ConfigType] = [test_fl_inversion] \ No newline at end of file diff --git a/src/scheduler.py b/src/scheduler.py index 07064723..a1169dc6 100644 --- a/src/scheduler.py +++ b/src/scheduler.py @@ -27,6 +27,7 @@ from algos.fl_central import CentralizedCLient, CentralizedServer from algos.fl_data_repr import FedDataRepClient, FedDataRepServer from algos.fl_val import FedValClient, FedValServer +from algos.fl_inversionAttack import GradientInversionFedAvgClient, GradientInversionFedAvgServer from utils.communication.comm_utils import CommunicationManager from utils.config_utils import load_config, process_config @@ -53,6 +54,7 @@ "centralized": [CentralizedServer, CentralizedCLient], "feddatarepr": [FedDataRepServer, FedDataRepClient], "fedval": [FedValServer, FedValClient], + "fedavg_inversion": [GradientInversionFedAvgServer, GradientInversionFedAvgClient], } diff --git a/src/utils/communication/comm_utils.py b/src/utils/communication/comm_utils.py index c0bbec03..63fd9696 100644 --- a/src/utils/communication/comm_utils.py +++ b/src/utils/communication/comm_utils.py @@ -1,9 +1,10 @@ from enum import Enum -from typing import Any, Dict, List +from typing import Any, Dict, List, Tuple from utils.communication.grpc.main import GRPCCommunication from utils.communication.mpi import MPICommUtils +import numpy as np class CommunicationType(Enum): MPI = 1 @@ -50,6 +51,19 @@ def send(self, dest: str | int | List[str | int], data: Any, tag: int = 0): else: print(f"Sending data to {dest}") self.comm.send(dest=int(dest), data=data) + + def send_dummy_data(self, dest: str | int, dims: Tuple[int, int]): + """ + placeholder method for sending images or other data types + """ + # generate random data of given int dimension + data = np.random.rand(*dims) + if isinstance(dest, list): + for d in dest: + self.comm.send(dest=int(d), data=data) + else: + print(f"Sending data to {dest}") + self.comm.send(dest=int(dest), data=data) def receive(self, node_ids: str | int | List[str | int], tag: int = 0) -> Any: """ diff --git a/src/utils/communication/grpc/main.py b/src/utils/communication/grpc/main.py index 739f523a..9c26e973 100644 --- a/src/utils/communication/grpc/main.py +++ b/src/utils/communication/grpc/main.py @@ -5,7 +5,7 @@ import threading import time import socket -from typing import Any, Dict, List, OrderedDict, Union +from typing import Any, Dict, List, OrderedDict, Union, Tuple from urllib.parse import unquote import grpc # type: ignore from utils.communication.grpc.grpc_utils import deserialize_model, serialize_model From 702d4833a4d53ee2d10ba08582888065dbafae9a Mon Sep 17 00:00:00 2001 From: photonshi Date: Sun, 27 Oct 2024 06:02:49 +0000 Subject: [PATCH 03/16] initial commit --- src/algos/fl_inversionAttack.py | 10 +- src/test_inversion.ipynb | 9171 +++++++++++++++++++++++++++++++ src/utils/gias.py | 72 + 3 files changed, 9252 insertions(+), 1 deletion(-) create mode 100644 src/test_inversion.ipynb create mode 100644 src/utils/gias.py diff --git a/src/algos/fl_inversionAttack.py b/src/algos/fl_inversionAttack.py index aadb7515..17e6d95b 100644 --- a/src/algos/fl_inversionAttack.py +++ b/src/algos/fl_inversionAttack.py @@ -218,6 +218,7 @@ def __init__(self, config: Dict[str, Any], node_id: int, comm: CommunicationMana self.ground_truth, self.labels = self.extract_ground_truth(num_images=config["num_images"]) # set reconstruction number # TODO somehow get the server to access the ground truth and labels for evaluation + self.comm_utils.send(0, [self.ground_truth, self.labels]) def extract_ground_truth(self, num_images=10): """ @@ -261,7 +262,14 @@ def obtain_ground_truth(self): """ Obtain the ground truth images and labels from the clients for evaluation. """ - ground_truth, labels = self.comm_utils.receive([i for i in range(self.num_users)]) + ground_truth, labels = [], [] + client_list = self.comm_utils.receive([i for i in range(self.num_users)]) + # TODO 1) sort the received items + # TODO 2) add tag to indicate we are receiving dummy data + for i in range(len(client_list)): + ground_truth_i, labels_i = client_list[i][:10], client_list[i][10:] + ground_truth.append(ground_truth_i) + labels.append(labels_i) return ground_truth, labels def inverting_gradients_attack(self): diff --git a/src/test_inversion.ipynb b/src/test_inversion.ipynb new file mode 100644 index 00000000..c49f41fc --- /dev/null +++ b/src/test_inversion.ipynb @@ -0,0 +1,9171 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "# script to test basic functionality of gia package\n", + "\n", + "import inversefed\n", + "\n", + "import torch\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "from collections import defaultdict\n", + "from PIL import Image\n", + "from torchvision.utils import save_image\n", + "from utils.model_utils import ModelUtils\n", + "from torch.utils.data import DataLoader, Dataset\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "arch = 'ResNet18'\n", + "num_images = 10\n", + "trained_model = False\n", + "device = 'cuda'" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Currently evaluating -------------------------------:\n", + "Sunday, 27. October 2024 04:25AM\n", + "CPUs: 20, GPUs: 4 on matlaber12.\n", + "GPU : NVIDIA GeForce GTX 1080 Ti\n", + "Files already downloaded and verified\n", + "Files already downloaded and verified\n", + "Model initialized with random key 3695468145.\n" + ] + } + ], + "source": [ + "import inversefed\n", + "setup = inversefed.utils.system_startup()\n", + "defs = inversefed.training_strategy('conservative')\n", + "\n", + "loss_fn, trainloader, validloader = inversefed.construct_dataloaders('CIFAR10', defs)\n", + "\n", + "mutils = ModelUtils(device=\"cuda\")\n", + "model_bespoke = mutils.get_model(\"resnet10\", \"cifar10\", \"cuda\")\n", + "\n", + "model_control, _ = inversefed.construct_model(arch, num_classes=10, num_channels=3)\n", + "model_control.to(**setup)\n", + "if trained_model:\n", + " epochs = 120\n", + " file = f'{arch}_{epochs}.pth'\n", + " try:\n", + " model_control.load_state_dict(torch.load(f'models/{file}'))\n", + " except FileNotFoundError:\n", + " inversefed.train(model_control, loss_fn, trainloader, validloader, defs, setup=setup)\n", + " torch.save(model_control.state_dict(), f'models/{file}')\n", + "model_control.eval();" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None]\n", + "ds = torch.as_tensor(inversefed.consts.cifar10_std, **setup)[:, None, None]\n", + "def plot(tensor):\n", + " tensor = tensor.clone().detach()\n", + " tensor.mul_(ds).add_(dm).clamp_(0, 1)\n", + " if tensor.shape[0] == 1:\n", + " return plt.imshow(tensor[0].permute(1, 2, 0).cpu());\n", + " else:\n", + " fig, axes = plt.subplots(1, tensor.shape[0], figsize=(12, tensor.shape[0]*12))\n", + " for i, im in enumerate(tensor):\n", + " axes[i].imshow(im.permute(1, 2, 0).cpu());" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "num_images = 10\n", + "ground_truth, labels = [], []\n", + "\n", + "for idx in range(num_images*3):\n", + " img, label = validloader.dataset[idx]\n", + " if label not in labels:\n", + " labels.append(torch.as_tensor((label,), device=setup['device']))\n", + " ground_truth.append(img.to(**setup))\n", + "\n", + "ground_truth = torch.stack(ground_truth)\n", + "labels = torch.cat(labels)\n", + "\n", + "ground_truth_target = ground_truth\n", + "labels_target = labels" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "metadata": {}, + "outputs": [], + "source": [ + "# num_images = 1\n", + "# ground_truth_target, labels_target = [], []\n", + "# idx = 25 # choosen randomly ... just whatever you want\n", + "# while len(labels_target) < num_images:\n", + "# img, label = validloader.dataset[idx]\n", + "# idx += 1\n", + "# if label not in labels_target:\n", + "# labels_target.append(torch.as_tensor((label,), device=setup['device']))\n", + "# ground_truth_target.append(img.to(**setup))\n", + "# ground_truth_target = torch.stack(ground_truth_target)\n", + "# labels_target = torch.cat(labels_target)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "(torch.Size([10, 3, 32, 32]),\n", + " torch.Size([10]),\n", + " torch.Size([10, 3, 32, 32]),\n", + " torch.Size([10]))" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "ground_truth.shape, labels.shape, ground_truth_target.shape, labels_target.shape" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAB1cAAADHCAYAAABFldqsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAADA9ElEQVR4nOz9d5hkV3X2Da+TKld1jhO6J4+kURwllEYBkETQI0ASwTaSAEtYBMsP4MQFEvkxYIONX4yxjcCGz4+RSCZJIBDKAuU0o8mxe6ZzdVeuOufs9w++mVen7iUoxRrM/bsuLlH37HPODmuvvfY+XbUsY4wRQgghhBBCCCGEEEIIIYQQQgghvxG73RUghBBCCCGEEEIIIYQQQgghhJDfBfhylRBCCCGEEEIIIYQQQgghhBBCWoAvVwkhhBBCCCGEEEIIIYQQQgghpAX4cpUQQgghhBBCCCGEEEIIIYQQQlqAL1cJIYQQQgghhBBCCCGEEEIIIaQF+HKVEEIIIYQQQgghhBBCCCGEEEJagC9XCSGEEEIIIYQQQgghhBBCCCGkBfhylRBCCCGEEEIIIYQQQgghhBBCWoAvVwkhhBBCCCGEEEIIIYQQQgghpAX4cpUQ8pw4++yzZd26db+13K5du8SyLPnqV7/64leKkN/AL37xC7EsS37xi1/8TtyXkJeC+++/X0477TRJp9NiWZY88sgj7a4SIf/jGR0dlde85jW/tZy2vlxxxRUyOjr64lWOkOfB9ddfL5ZlyfT0dLurQojKQRsl5HcN+ldCnhn6dvL7QKvrwOjoqFxxxRXP61lnn322nH322c/rHr8v8OVqmxkfH5frr7+eh5mEEEIIeUlpNBpy6aWXyuzsrHzuc5+T//iP/5CRkZF2V4uQ58QXv/hF/iEX+b3nnnvukeuvv17y+Xy7q0IIIYQQQggh/6Nx212B33fGx8flIx/5iIyOjspxxx3X7uoQ8oIzMjIilUpFPM9rd1XI7zlnnXWWVCoVicVi7a4KIYcF27dvl927d8u//Mu/yDve8Y52V4eQ58UXv/hF6e3tfd5/pXs4wXWLPFvuuece+chHPiJXXHGFdHZ2trs6hBBCCCGEkJeQzZs3i23z+5QvFexpQsiLimVZkkgkxHGcdleF/J5j27YkEonfGmSUy+WXqEaEtJfJyUkRkd96AF8qlV6C2hBCmml13SLk2RKGoVSr1XZXgxBCyGGCMUYqlUq7q0HI7wzcI5PDlXg8/lu/4ET7feHgTv15MDY2Jm9/+9tleHhY4vG4LFu2TP7kT/5E6vW6zM7Oyvvf/345+uijJZPJSC6XkwsvvFAeffTRQ9f/4he/kJNOOklERK688kqxLIu5KclhQ6FQkGuvvVZGR0clHo9Lf3+/vOIVr5CHHnooUm7jxo1yzjnnSCqVkkWLFsmnP/3pyL9rOVevuOIKyWQysmPHDjn//PMlnU7L8PCwfPSjHxVjzEvRPPI/iN27d8s111wja9askWQyKT09PXLppZfKrl27IuW03HUHcwc/+OCDctZZZ0kqlZK//uu/FpH/LyfeT37yEznuuOMkkUjIkUceKd/+9rd/a53uvPNOufTSS2Xp0qUSj8dlyZIl8md/9mewYT04F8bGxuTiiy+WTCYjfX198v73v1+CIIiUDcNQPv/5z8tRRx0liURCBgYG5Oqrr5a5ubnn1nHk95orrrhCNmzYICIil156qViWJWefffYhm9y+fbu86lWvkmw2K3/wB38gIr8OwN/3vvfJkiVLJB6Py5o1a+Szn/0s+O1KpSLvfe97pbe3V7LZrFx00UUyNjYmlmXJ9ddf/1I3lRyGtOq3nyl/0le/+lWxLOtQ+dHRUXnyySfl9ttvPxRPPz1HzI4dO+TSSy+V7u5uSaVScuqpp8oPf/jDyD0PrhHf/OY35SMf+YgsWrRIstmsXHLJJTI/Py+1Wk2uvfZa6e/vl0wmI1deeaXUarXIPXzfl4997GOyYsUKicfjMjo6Kn/9138N5Q7y29aXVnN6c30gIr+eLx/4wAdERGTZsmWH5sLBWPzd7363fOMb35CjjjpK4vG43Hzzzc9oY1r8LiLy1FNPyWWXXSZ9fX2STCZlzZo18sEPfvA31mv37t2ycuVKWbdunUxMTLyQTSbkN3LXXXfJSSedJIlEQlasWCH//M//DGVa9dthGMr1118vw8PDkkql5JxzzpGNGze+IHnNCHk25PP5Q79O0NHRIVdeeWXkj4NbtemDe91bbrlFTjzxREkmk4fmyE9/+lM544wzpLOzUzKZjKxZs+bQHvkgtVpNrrvuOlm5cuWh/e6f//mfP2PMQ8gLRSu+XUTk61//uqxfv16SyaR0d3fLm970Jtm7dy+U++UvfykXXHCBdHR0SCqVkg0bNsjdd98dKXNwT7Jx40Z5y1veIl1dXXLGGWe8KO0j5LcxPT0tl112meRyOenp6ZE//dM/jfzRZHNscnDvfPvtt8s111wj/f39snjx4kP//uUvf1lWrFghyWRSTj75ZLnzzjtfyub8zsOfBX6OjI+Py8knnyz5fF6uuuoqWbt2rYyNjclNN90k5XJZduzYId/97nfl0ksvlWXLlsnExIT88z//s2zYsEE2btwow8PDcsQRR8hHP/pR+fCHPyxXXXWVnHnmmSIictppp7W5dYSIvPOd75SbbrpJ3v3ud8uRRx4pMzMzctddd8mmTZvkhBNOEBGRubk5ueCCC+T1r3+9XHbZZXLTTTfJX/zFX8jRRx8tF1544W+8fxAEcsEFF8ipp54qn/70p+Xmm2+W6667Tnzfl49+9KMvRRPJ/xDuv/9+ueeee+RNb3qTLF68WHbt2iX/9E//JGeffbZs3LhRUqnUb7x+ZmZGLrzwQnnTm94kf/iHfygDAwOH/m3r1q3yxje+Ud75znfK5ZdfLjfccINceumlcvPNN8srXvGKZ7znjTfeKOVyWf7kT/5Eenp65Fe/+pV84QtfkH379smNN94YKRsEgZx//vlyyimnyGc/+1m59dZb5W//9m9lxYoV8id/8ieHyl199dXy1a9+Va688kp573vfKzt37pR//Md/lIcffljuvvtu/vQ2eVZcffXVsmjRIvnkJz8p733ve+Wkk06SgYEB+cY3viG+78v5558vZ5xxhnz2s5+VVColxhi56KKL5LbbbpO3v/3tctxxx8ktt9wiH/jAB2RsbEw+97nPHbr3FVdcId/85jflj/7oj+TUU0+V22+/XV796le3sbXkcOP5+u1mPv/5z8t73vMeyWQyh170HPTlExMTctppp0m5XJb3vve90tPTI1/72tfkoosukptuukle97rXRe71qU99SpLJpPzlX/6lbNu2Tb7whS+I53li27bMzc3J9ddfL/fdd5989atflWXLlsmHP/zhQ9e+4x3vkK997WtyySWXyPve9z755S9/KZ/61Kdk06ZN8p3vfCfynOe6vmhwfSAiIq9//etly5Yt8p//+Z/yuc99Tnp7e0VEpK+vT0REfv7zn8s3v/lNefe73y29vb0yOjr6rHKzPvbYY3LmmWeK53ly1VVXyejoqGzfvl2+//3vyyc+8Qn1mu3bt8u5554r3d3d8tOf/vRQnQh5sXn88cflla98pfT19cn1118vvu/LddddF4nzRVr323/1V38ln/70p+W1r32tnH/++fLoo4/K+eefz2+Ak5ecyy67TJYtWyaf+tSn5KGHHpJ//dd/lf7+fvmbv/kbEXl2scjmzZvlzW9+s1x99dXyx3/8x7JmzRp58skn5TWveY0cc8wx8tGPflTi8bhs27Yt8rIpDEO56KKL5K677pKrrrpKjjjiCHn88cflc5/7nGzZskW++93vvpRdQn6PaNW3f+ITn5APfehDctlll8k73vEOmZqaki984Qty1llnycMPP3zol5t+/vOfy4UXXijr16+X6667TmzblhtuuEHOPfdcufPOO+Xkk0+O3PfSSy+VVatWySc/+Ul+MYS0jcsuu0xGR0flU5/6lNx3333yD//wDzI3Nyf//u///huvu+aaa6Svr08+/OEPH/rm6r/927/J1VdfLaeddppce+21smPHDrnooouku7tblixZ8lI053cfQ54Tb33rW41t2+b++++HfwvD0FSrVRMEQUTfuXOnicfj5qMf/egh7f777zciYm644YYXu8qEPCs6OjrMu971rmf89w0bNhgRMf/+7/9+SKvVamZwcNC84Q1vOKTt3LkTbPzyyy83ImLe8573HNLCMDSvfvWrTSwWM1NTUy9sY8j/aMrlMmj33nsv2Odtt91mRMTcdttth7SDdvylL30J7jEyMmJExHzrW986pM3Pz5uhoSFz/PHH/8b7anX61Kc+ZSzLMrt37z6kHZwLT18XjDHm+OOPN+vXrz/0+c477zQiYr7xjW9Eyt18882qTkgrHLTdG2+88ZB20Cb/8i//MlL2u9/9rhER8/GPfzyiX3LJJcayLLNt2zZjjDEPPvigERFz7bXXRspdccUVRkTMdddd9+I0hvxO0arfvu6664y2XbnhhhuMiJidO3ce0o466iizYcMGKHvttdcaETF33nnnIa1QKJhly5aZ0dHRQ/H6wfmwbt06U6/XD5V985vfbCzLMhdeeGHkvi972cvMyMjIoc+PPPKIERHzjne8I1Lu/e9/vxER8/Of//yQ9nzWl8svvzzyXK4P5Ol85jOfgblhjDEiYmzbNk8++WRE12zMGD1+P+uss0w2m43EMcb8OoY/yME5OzU1ZTZt2mSGh4fNSSedZGZnZ1+Q9hHSKhdffLFJJBIRe924caNxHOfQutKq3z5w4IBxXddcfPHFkXLXX3+9ERFz+eWXv7iNIcT8f/71bW97W0R/3eteZ3p6eowxzy0WufnmmyNlP/e5zx3y48/Ef/zHfxjbtiOxlTHGfOlLXzIiYu6+++7n1EZCfhut+PZdu3YZx3HMJz7xici1jz/+uHFd95AehqFZtWqVOf/88yOxTLlcNsuWLTOveMUrDmkH59+b3/zmF7N5hPxGDtrhRRddFNGvueYaIyLm0UcfNcb82r8/PTY5uHc+44wzjO/7h/R6vW76+/vNcccdZ2q12iH9y1/+shERdW9NEP4s8HMgDEP57ne/K6997WvlxBNPhH+3LEvi8fih/EhBEMjMzMyhn9No/llVQg5HOjs75Ze//KWMj48/Y5lMJiN/+Id/eOhzLBaTk08+WXbs2NHSM9797ncf+v8Hf66sXq/Lrbfe+twrTn7vSCaTh/5/o9GQmZkZWblypXR2drbkb+PxuFx55ZXqvw0PD0e+1ZTL5eStb32rPPzww3LgwIGW6lQqlWR6elpOO+00McbIww8/DOXf+c53Rj6feeaZkXl04403SkdHh7ziFa+Q6enpQ/9bv369ZDIZue22235rOwl5Njz9W9MiIj/60Y/EcRx573vfG9Hf9773iTFGfvzjH4uIyM033ywiv/6ryKfznve850WsLfld4/n67WfDj370Izn55JMjP92VyWTkqquukl27dsnGjRsj5d/61rdGvul5yimniDFG3va2t0XKnXLKKbJ3717xff/Qc0RE/vf//t+Rcu973/tEROBniJ/r+tIM1wfSKhs2bJAjjzzyOV07NTUld9xxh7ztbW+TpUuXRv5N++nuJ554QjZs2CCjo6Ny6623SldX13N6LiHPhSAI5JZbbpGLL744Yq9HHHGEnH/++Yc+t+q3f/azn4nv+4xtyGGBtm+cmZmRhYWFZx2LLFu2LDInROTQN/q+973vSRiGah1uvPFGOeKII2Tt2rWR2OPcc88VEWHsQV4UWvXt3/72tyUMQ7nssssi9jk4OCirVq06ZJ+PPPKIbN26Vd7ylrfIzMzMoXKlUknOO+88ueOOO2AONM8/QtrBu971rsjng/HIwTXgmfjjP/5jcRzn0OcHHnhAJicn5Z3vfKfEYrFD+hVXXCEdHR0vYI3/Z8OXq8+BqakpWVhYkHXr1j1jmTAM5XOf+5ysWrVK4vG49Pb2Sl9fnzz22GMyPz//EtaWkOfGpz/9aXniiSdkyZIlcvLJJ8v1118PL00XL14MBypdXV0t5fiybVuWL18e0VavXi0iAjnXCPlNVCoV+fCHP3woD+RBf5vP51vyt4sWLYoEEk9n5cqVYOOt2OmePXvkiiuukO7u7kN5VA/mt2yuUyKROPSTfQdpnkdbt26V+fl56e/vl76+vsj/isWiTE5O/tZ2EtIqrutGcnCI/Dpn3vDwsGSz2Yh+xBFHHPr3g/+1bVuWLVsWKbdy5coXscbkd43n67efDbt375Y1a9aA3my7B2l+cXRwY9n8s0gdHR0ShuGh+h60/WZbHxwclM7OTnjOc11fmuH6QFql2S8/Gw7uAX7T/vfpvPa1r5VsNiu33HKL5HK55/xcQp4LU1NTUqlUZNWqVfBvT18PWvXbB//bXK67u5t/OEBecprjlIM2ODc396xjEW1deOMb3yinn366vOMd75CBgQF505veJN/85jcjL5m2bt0qTz75JMQdB+MYxh7kxaBV375161YxxsiqVavARjdt2nTIPrdu3SoiIpdffjmU+9d//Vep1WqwL3k+sRQhLxTNc2DFihVi2/Zv3UM22+/BNaH5fp7nwXk9eWaYc/VF4pOf/KR86EMfkre97W3ysY99TLq7u8W2bbn22muf8a+/CDmcuOyyy+TMM8+U73znO/KTn/xEPvOZz8jf/M3fyLe//e1D+VSf/hcvT8cw9wB5CXnPe94jN9xwg1x77bXyspe9TDo6OsSyLHnTm97Ukr99+jeoXgiCIJBXvOIVMjs7K3/xF38ha9eulXQ6LWNjY3LFFVdAnZ5pHj2dMAylv79fvvGNb6j/3vxylpDnw9N/fYOQF4NW/bb2jTiRX/vZF4tn8smtxjzPVOcXC64PpFW0eOfFmmNveMMb5Gtf+5p84xvfkKuvvvp53YuQF5uX2m8T8nxoJR5p1aa1dSGZTModd9wht912m/zwhz+Um2++Wf7rv/5Lzj33XPnJT34ijuNIGIZy9NFHy9/93d+p92WePtJOwjAUy7Lkxz/+sTpfMpnMoXIiIp/5zGfkuOOOU+91sOxBXuizI0JeCJ6PzyfPH75cfQ709fVJLpeTJ5544hnL3HTTTXLOOefIv/3bv0X0fD4vvb29hz4zkCeHM0NDQ3LNNdfINddcI5OTk3LCCSfIJz7xiUMvV58PYRjKjh07Dv11o4jIli1bRERkdHT0ed+f/P5w0003yeWXXy5/+7d/e0irVquSz+ef9723bdsmxpiIr/5tdvr444/Lli1b5Gtf+5q89a1vPaT/9Kc/fc71WLFihdx6661y+umnMyAibWFkZERuvfVWKRQKkW+vPvXUU4f+/eB/wzCUnTt3Rv4Cctu2bS9thclhTat+++C3MfL5/KGfqRPBb5uKPHNMPTIyIps3bwa92XafLwdtf+vWrYe+FSsiMjExIfl8Hp7zXNYXDa4P5Ok8273l0+fY02meYwf/ev037X+fzmc+8xlxXVeuueYayWaz8pa3vOVZ1YuQ50NfX58kk8lD30p6Ok9fD1r12wf/u23btsi3PmZmZlr6xSZCXiqebSzyTNi2Leedd56cd9558nd/93fyyU9+Uj74wQ/KbbfdJi9/+ctlxYoV8uijj8p5553HM03yktGqb1+xYoUYY2TZsmWR88ZmVqxYISK/Ts3x8pe//IWvMCEvElu3bo3EI9u2bZMwDJ/1WfrBNWHr1q2HftZd5Ndpe3bu3CnHHnvsC1Lf/+nwawnPAdu25eKLL5bvf//78sADD8C/G2PEcRz4S/Ybb7xRxsbGIlo6nRYR3NAS0k6CIICfv+jv75fh4WGp1Wov2HP+8R//8dD/N8bIP/7jP4rneXLeeee9YM8g//PR/O0XvvCFF+SbTePj4/Kd73zn0OeFhQX593//dznuuONkcHDwGesjEv3rYWOM/P3f//1zrsdll10mQRDIxz72Mfg33/e5hpAXnVe96lUSBEHEb4uIfO5znxPLsg790c3BfDdf/OIXI+W+8IUvvDQVJb8TtOq3Dx563HHHHYe0UqkkX/va1+Ce6XRa9YWvetWr5Fe/+pXce++9kXt8+ctfltHR0eecg1J7jojI5z//+Yh+8Fsdr371qyP6c1lfNLg+kKfzbPeWIyMj4jhOZI6JoA/v6+uTs846S77yla/Inj17Iv+m/WKNZVny5S9/WS655BK5/PLL5b//+7+fRSsIeX44jiPnn3++fPe7343Y66ZNm+SWW2459LlVv33eeeeJ67ryT//0T5FyzTERIe3m2cYiGrOzs6Ad/FbfwbOgyy67TMbGxuRf/uVfoGylUpFSqfRsqk1IS7Tq21//+teL4zjykY98BGIUY4zMzMyIiMj69etlxYoV8tnPflaKxSI8b2pq6kVqCSHPj//n//l/Ip8PnrU82y9CnXjiidLX1ydf+tKXpF6vH9K/+tWvcg/5LOA3V58jn/zkJ+UnP/mJbNiwQa666io54ogjZP/+/XLjjTfKXXfdJa95zWvkox/9qFx55ZVy2mmnyeOPPy7f+MY34DerV6xYIZ2dnfKlL31JstmspNNpOeWUU/g77qStFAoFWbx4sVxyySVy7LHHSiaTkVtvvVXuv//+yLdMng+JREJuvvlmufzyy+WUU06RH//4x/LDH/5Q/vqv/5o/YUeeFa95zWvkP/7jP6Sjo0OOPPJIuffee+XWW2+Vnp6e533v1atXy9vf/na5//77ZWBgQL7yla/IxMSE3HDDDc94zdq1a2XFihXy/ve/X8bGxiSXy8m3vvWt5/WX7Rs2bJCrr75aPvWpT8kjjzwir3zlK8XzPNm6davceOON8vd///dyySWXPOf7E/LbeO1rXyvnnHOOfPCDH5Rdu3bJscceKz/5yU/ke9/7nlx77bWHXoKtX79e3vCGN8jnP/95mZmZkVNPPVVuv/32Q9/I41+3E5HW/fYrX/lKWbp0qbz97W+XD3zgA+I4jnzlK1+Rvr4+eMGzfv16+ad/+if5+Mc/LitXrpT+/n4599xz5S//8i/lP//zP+XCCy+U9773vdLd3S1f+9rXZOfOnfKtb33rBfsJ7GOPPVYuv/xy+fKXvyz5fF42bNggv/rVr+RrX/uaXHzxxXLOOedEyj+X9UWD6wN5OuvXrxcRkQ9+8IPypje9STzPk9e+9rXPWL6jo0MuvfRS+cIXviCWZcmKFSvkBz/4gZov7x/+4R/kjDPOkBNOOEGuuuoqWbZsmezatUt++MMfyiOPPALlbduWr3/963LxxRfLZZddJj/60Y8ifxVPyIvJRz7yEbn55pvlzDPPlGuuuUZ835cvfOELctRRR8ljjz0mIq377YGBAfnTP/1T+du//Vu56KKL5IILLpBHH31UfvzjH0tvby9jG3LY8GxjEY2PfvSjcscdd8irX/1qGRkZkcnJSfniF78oixcvljPOOENERP7oj/5IvvnNb8o73/lOue222+T000+XIAjkqaeekm9+85tyyy23yIknnvhiN5f8HtKKb1+xYoV8/OMfl7/6q7+SXbt2ycUXXyzZbFZ27twp3/nOd+Sqq66S97///WLbtvzrv/6rXHjhhXLUUUfJlVdeKYsWLZKxsTG57bbbJJfLyfe///02t5gQZOfOnYfikXvvvVe+/vWvy1ve8pZn/U1Tz/Pk4x//uFx99dVy7rnnyhvf+EbZuXOn3HDDDcy5+mww5Dmze/du89a3vtX09fWZeDxuli9fbt71rneZWq1mqtWqed/73meGhoZMMpk0p59+urn33nvNhg0bzIYNGyL3+d73vmeOPPJI47quERFzww03tKU9hBykVquZD3zgA+bYY4812WzWpNNpc+yxx5ovfvGLh8ps2LDBHHXUUXDt5ZdfbkZGRg593rlzJ9j15ZdfbtLptNm+fbt55StfaVKplBkYGDDXXXedCYLgxWwa+R/I3NycufLKK01vb6/JZDLm/PPPN0899ZQZGRkxl19++aFyt912mxERc9tttx3SnsmOjTFmZGTEvPrVrza33HKLOeaYY0w8Hjdr1641N954Y6Scdt+NGzeal7/85SaTyZje3l7zx3/8x+bRRx99xrnQzHXXXWe0JfrLX/6yWb9+vUkmkyabzZqjjz7a/Pmf/7kZHx9vrbMIeRoHbffpNv1MNmmMMYVCwfzZn/2ZGR4eNp7nmVWrVpnPfOYzJgzDSLlSqWTe9a53me7ubpPJZMzFF19sNm/ebETE/J//839e1DaR3w1a9dvGGPPggw+aU045xcRiMbN06VLzd3/3d+aGG24wImJ27tx5qNyBAwfMq1/9apPNZo2IROLt7du3m0suucR0dnaaRCJhTj75ZPODH/wg8hxtPhhjDj3r/vvvj+gH/fTU1NQhrdFomI985CNm2bJlxvM8s2TJEvNXf/VXplqtRq59PutLc5x1EK4P5CAf+9jHzKJFi4xt24fmiYiYd73rXWr5qakp84Y3vMGkUinT1dVlrr76avPEE0+o+9InnnjCvO51rzs0l9asWWM+9KEPHfp3bV6Uy2WzYcMGk8lkzH333feitJkQjdtvv92sX7/exGIxs3z5cvOlL30JYuxW/bbv++ZDH/qQGRwcNMlk0px77rlm06ZNpqenx7zzne98qZtGfg/R/KsxBmKiZxuLNPOzn/3M/K//9b/M8PCwicViZnh42Lz5zW82W7ZsiZSr1+vmb/7mb8xRRx1l4vG46erqMuvXrzcf+chHzPz8/AvbeEKeRiu+3RhjvvWtb5kzzjjDpNNpk06nzdq1a8273vUus3nz5ki5hx9+2Lz+9a83PT09Jh6Pm5GREXPZZZeZn/3sZ4fKPNP8I+Sl5KAdbty40VxyySUmm82arq4u8+53v9tUKpVD5Zr31M+0nz3IF7/4RbNs2TITj8fNiSeeaO644w71/RXRsYxRfseHEEJeRK644gq56aab1J/eIORwYXR0VNatWyc/+MEP2l0VQn7neeSRR+T444+Xr3/96/IHf/AH7a4OIYQQQsjzIp/PS1dXl3z84x+XD37wg+2uDiGEEEIIeYlhzlVCCCGEEPKCUalUQPv85z8vtm3LWWed1YYaEUIIIYQ8d54pthEROfvss1/ayhBCCCGEkMMC5lwlhBBCCCEvGJ/+9KflwQcflHPOOUdc15Uf//jH8uMf/1iuuuoqWbJkSburRwghhBDyrPiv//ov+epXvyqvetWrJJPJyF133SX/+Z//Ka985Svl9NNPb3f1CCGEEEJIG+DLVUIIIYQQ8oJx2mmnyU9/+lP52Mc+JsViUZYuXSrXX389fzKPEEIIIb+THHPMMeK6rnz605+WhYUFGRgYkD/90z+Vj3/84+2uGiGEEEIIaRPMuUoIIYQQQgghhBBCCCGEEEIIIS3AnKuEEEIIIYQQQgghhBBCCCGEENICfLlKCCGEEEIIIYQQQgghhBBCCCEt0HLO1Rv+7HjQLBOCFvM8LGfjO9x6vQqaHzTwfrEYaEEYfa4J8ZeNLTsAzXZAEtPI4LXig+bFsL6O0n2WjXUJQrxfw8e+C0MLK2jhM/wAy9WUa5W7Sdg0ZpaFpep1HIcgUNqqjL8t2O/1EMuVsEukXMdrP/PNnVjwJWJ6eho038eKa314uPCS1E37YXFFU4spf95hlJK2XhCx0NasJs0oM8NS/s7k+fxieqv9rj1jYGDgOT/3+fKv//ZF0DK9q0FLOuiXc1n0pYUazunSwgxotq34w6YBdpV1JOnGQUs4yrKm+GXVQSrFghDboJULlXLNbRARcV2sn60sTq3akGVp61+0P7W66ffCZ8bj2McxGzUxqFkxbFd5ZhNoGy64pKX6vRic+4oTQPOyHaBNzKHdzs7mQasVMFboGsyB5nb3gmZ5TTbuoM03Crg+73nwSdC8DhyPJasGQUu6OOZhA+O4wMdy3X1J0IaW9YDmKDYfBriWuh76lYVZbO/kgUnQGk0x0Gknr4UypobPvOUnt4O2eHQxaEkP+3Ns737QnGQWtFwatVv/789Aeyn52mqs01AG/fBgAudwwsJ+zCbQD3Vm0GYcW4kvrah/sj28rqHEjIUK1rdSw2sDJX5wlPWkofjrOSVYLdWxLlq0EOSWg+avPxG0hdtvA21SmZcTdZwf3aWpyOedszh3/Sz6H8ngej1RLoPWUUUtXiqBVnZwLBxlb/a1GRz/lw4cS19d37He2vZMQws19OA3KpoW76/tp7X6irKW21rQ83y2CFocpDYWsV/AvQlanogotvfUo4+DNpOfBe3UM04HTTuL0OJ3LYayBOfkS8UfnrMUtIriv7wOLGfnMF6IeThfnLACmgmwXOBE19DQwfghpvTf3CzGXr968BHQCiWsh+PiuFktHn0FAVqWUc49QuXcI7AUPxcq65CyNtnK+ppORevsWLguBz7WI5vDdX5hfgG0Wg3r64oS5yvuR/OhswVcN14qVqwaAW3J4iHQ9k+MgTan+IPRJUvw2r1ToGnE4tFx0/xDtYJ9Xy7jPqJUwj5NJhOKlgItUM5XA+V8sVpH55Dw0NYySZxXw/0Y+/d0dYF2YN84aLPT2O8i0ec6yrlDo45zpWaw74xyJqwZc6OOPr2unE16yjnDbBX9z0vFirPeAprtKWcNyl4sHsN5nlT2MckYaqkkjm86lY58Ngb7z1f2f9p6GirreEO5tlTGmLRYyuP9jPJcC+2g+az813XBuREqbRPlXYMWpdhK0Bc2nTFbyvm5FjlZtqK2eE4lylqtrd+mge3ffNeNSm1eGv74na8GbVcR5+Bx568DrbOnEzRTRztwLZwbDQ9tzUpGxylQOt9Tzit9UwDNUc7StMPyULEfR5nz5fl5vFZZvqwaXptMKed/XRi3zZbQfxfncL2699tPgRaUo8992cVroEz3EqzHwnQR75XF8Y9ncCwcC9cSW+n3MMB+/+ylXwAN7vVbSxBCCCGEEEIIIYQQQgghhBBCCOHLVUIIIYQQQgghhBBCCCGEEEIIaQW+XCWEEEIIIYQQQgghhBBCCCGEkBbgy1VCCCGEEEIIIYQQQgghhBBCCGkBzF77DNSVDM3GKMm7leTLccmAZiv3c11MoGxrr3+bctNaHhaqKUnZ/VB5ppIk2FESlbtKPawQE8SLXwPJFmxXqNSlbmFi+sDBBLv1EOtXV5LuWkri6uY6J5S+c5Xk3raLCYGDhtJ+S0kWrrTfKCm5HefwetfvOGpW8t8pLEtLff4SPFexPSW3uoiShD3U0rUbZSyUZN6WjU+xIJG8VhO0PWPUGrdEq/3+fJ7xYhAa9EG+0wVaw0uDFjiKn/fQDkoVTEJuAkwQ73nRzzWD92rYuN5UFWet5JGXeqMKmq3M+UoZ1znNN3jNFRaReh19pG2jZkJcr2zFH8ZimITd95W51tQtlqWtt7iOdHXhWMeTWaybMs9Cbe7FsU+CItpJO3EzSdCSfdgPmRqu7bNzc6B1D2B/Da0YBm2uirYrzb5PGaNytQBaoMQiHbkO0Pr7c6C5Bm1qfl6JWRx8bqY3BVojwGtrFdSCBtp8PK35TbTdRg3b68ai49jTgW0tF+dRWyiDNjk+DVpSmXuOsi5lcp2gae1vN+csRS3nKXGZhzFdsaL4K6UvjI/3qyt7hGo96jtsJfCv+XjdAk5JKTXQD/nKM21l9xMo5ldQtjklLfRXri2XZkDb8aNbQeswypqotMPCbhffjT44k+2FMtsy6JMem5vAegT4zE6lXTElHPMt7GNH2V+1E1+JSwMf7XvLps2g9fb3oTbQD1pD8WuP3Pcr0KqVqGGduuEsKON6aKS+Ul9tz2aU8VDMVmwlVrWUsDRU42Yt3m4t9vW1clrcrMTIUEq7leIvdm/aAtr2HTtAO+HUU0BzlLVYq5vWBu384KXCsbDeQYg2lMBQTZYvGwCttxvjt9LcJGiTB/aDVg6izw1snI++Ehft2L4LtIU5jEdCpe995UzGUfYNtjKHEkncDyXiqNV9nPOW8oxGHftdApyVS5eiD+/MRsexmMc1Q5t56RTGt9UOLDk7lQetXsZ2DS0eAm1BKddOJifQHksljP2GhtG+43GcL4sWLQYtk8D4ctu27aAVC9FxaijnZpobCUPtjEw5L1Fim+FFOEZzedyrlCs4N4IQ50G1ikGQX8f9s+bnggBt3omhsxlYhHuk5q3EgmLzVWVOKWGMBEp/SogF68r6ahS/YpSzq3bSmcMxT6Rx7lvKuYLr4t7GUzTF1NTz3bAp9jBafKLYmXKkJ4ESs4XKeZ3r4SGPsbTzEhxzdS+g7GN9zZaVOamdt4TK5kIxP3HsaDtCZa0OlT5xlGdq+6ggQF8dKmdrWmynnVi0E78L/cixJ6EfSWaVOFqJhmMdaC8NwfMBzeZjXjQusAz2cxCiv/U8fKYWamvexvexbmFNeRdUwLmRcLDvEouV+aK806krbYunsIaFA2gxtTz2XbazJ/J5bjeuN1NjU6D1Lsfzp55sN2iBhWNtKedKlvJKNHiOfv7w2vkSQgghhBBCCCGEEEIIIYQQQshhCl+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtABfrhJCCCGEEEIIIYQQQgghhBBCSAsoaZx1TIgJYcVgcl6jJHy2AiW5cwMT4jpJJRGvkkLZabqdltw55mGyXt+gFjaUuin3833ULCULvW2UNjiYJNg4CdAqAWoHZrCfSnV8brGo9KeSpDqbiLY3ZmH/5lKYJDgZV5Jq2/hMW0m77DQPmIjgSIg0tOzebcQo46tphzMvdH0tS0nurD3DKKnP1UvRNrS/+ag10P5cZY5LoPgLq5U+aE+q9sPNnmyD/RwofiRQ/EZgVUFLKInke0YG8Lnzs6BlyqXI53oV15sggz4z7OgELRtD49Paattoe/Walpge259IYNJ4zfS0MdfmlaZp9fOVuQHVU+ZezMX5k0wmsR6i1FcwHggF7STU/n5L8yFtxO3IgebFcc3O5rKgpWex3ODiHtCS2TRo+XoR69I8JjbOn6BSweuUbk6ncW7UfcXmDY55tTSPWm0BtNDvw3Lz6AdmDsyB5saw7/qWYl1cZe7WSugLksloHyfiaN9BFdebahltuV5Gmx/s6QYtodhEQ7H52d3joLWbYQ/9WqjEuVVl/pfriv9H05I6PkKCBl5baRpObVmsB9ivRWXZLmETRHmkOMqkCRSHXWxguaqv2KRyrV9X5luItruQwEpnQrTVmPKMKStabiyH69CTCyXQds6VQVuu3N+NYz0SSnwXavH74RXePEN1cCyn9x8ALe6gL+4b7Adt756doG1+6EHQnKb7za9bB2UyXR14XQr9+sIkxk+xFPpXV7k21PZsWkcpy7YWo9SruD6JYhteCtfEUI01UAv9ZvtDe3Q9tNvubuzPyb04rsV5XK+SyWHQ9L0PSm3FwnUwnkTbWLoI4/LRflzfjI/ru+Oi/7I70NaKtWi5iuKXx2cxzqhVCqANDWKc5cTQ94XKGCXTWmyNWjyuzBflfKiixAtzs+hf08p+pVLARdLyMSYp5KM+fGEa+2lkCfqjmZlJrIcSG/Z34Xwse7hW9Xfj+VB7ds/PzJHr1oC2+amnQJuZUfxmHP1BYQFj9ZhiayeccAJovb1RO92zZy+UefjhR0CrKftObY3V7HFuFtul7TvjHrY1kcR25WeV4E7ZA8PhrIgUysq5gIf+J5NB+3PC6JzM2XhdMovt7+rtBK3awHVp547doNUa6GscF9tlK1o7SaVwH2trZ68xXA/cGPqDQDlLC5Qz/3qA4xtUomPSaCh22+rZl1KsOXYSERHFf7se2nKgnjOjv9XOWlwLn2spMbMxynmOrdiQ1ramclqZUIm7tPmdUmKseh3bMD+Pc8NSzv3MYebph05YApqTwfY5CSU+VM5MGiH2g7ZPdCzlnQ4oyrswpZQt2jkNzrNagPGEUfaSsRD9QDKlnE26WJe6he1vKOeuBuJvEcdDm1yYxHVTW9c6muxq79Y8lFGWWxlZ1wVaQolt6w2sW6C8qzPKvtZR/Eor8JurhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gJ8uUoIIYQQQgghhBBCCCGEEEIIIS2gZIV+hoIBJrUVR0nOG2Ji6LijJEN3lSSxSgJp21He/zY91tcSVCtJaL1YCrTB0dWgLeSnQZueKeH9XEycqycnxm6umCRom3ZPgWbiPaA1HCVJdQafW5zHpPZjk3ORz5k4JjcPDsyBtnQA29qTxQzDCVdJ+G1w/GPK8AdKAu12YilJwzWtHWjJy9uGOpZKkvcQC/ohJpBu+GgHW3fsAG1gsB+0sI7Jsvu6o0mvE4rNh23qz8PFng7iSxY0W3Duhw6OW804oDmKlnbRb+RSyjMe+kXkc30ak6MPrVsDmjWFvrBmoc/MONj3hQr6+YSShD5u0IbsngxqdVwPtSWtlsI6uw18rqMkZi+k0ebj8/PRey05EsqUOztAC/0qaIGyliZCHC9LmUN2gOWc4PD6m66OPvQjhTyunYkMxg/ZLhzzzqFu0IpK+OTZaEPJWDQuqCv+0a/iGMVieC8rwPGYO1AALaHZYxHLiYV+OeWg3WbT2CdhAx/SUHyfo8QPoY9zyFbmrudF+8Cx0fck41jfwSXDoC1ZMgra0CK0k5oSyOzbtQ+0cgXtqd3M5tFvVEMcp0BZLysh9q242LcLC/OgBYotVJvmR8MosYISaFQM3qus7Ad8JfbwFBtqKDZeU9pa9ZX6WThXTYBaUplvBSX0nW/gc20L/WnFia6ne+tlKBPMoc/oD3Fcu1yM1bNKiOJhsySu+PXAOrx8fWiw4loIpvmOuWncn+17Cm3tjv/+PmjzY+Og9S6K+p2H7r0byqQ6MB476awzQLvntp+DtnINxkZrjjsatIa271LW/GoJY6NYDO1x+9ZtoNXKaH8nnXY6lmugr485uCaMT0R97PTMDJRZvmYFaBNT+0Gb2r0dtAd/egto517yJtAsxU4cLcBT3OVLRcPBh/f34ZrXGcexnNj0KGiB4l/8GmphoMS+8ag9pxI55f4LoMViaKNxJWaOJ/BcZflytINyJQ+aCPqGAxMToGXS+NyeTowNwzoGfZoZ1JX5l5/E+nV1R9vWkVP2ajaOoedhn0xP4V5qoBf3A4sGekErlyugje0/AFo7WTQ8BFq1ivWuKPu9hXm0vyee2ARaf28faEODA6Bt356PfM7P56FMSrFlo8RAgRLTZ7M4hwoFjN8dxQ8sXrKopXISYFxQWMC+C5Xv68zMYX8ODw6Clkgp+4amxx5zHO5jjz32eNAWL10M2u13/gI039wB2uwMnv8WFrA/qzVc09pJEGA8X6ygD3KVvaIbxzMZLTAyio+UBmph02PrvhIbK+cF2nmYp+wJYwbra1lYzlP2LpbynsEYHEujnF2FitZoYB8nM3jeVKmg/wmU86HmbrGVdyW2sk/TYtuuTvQNC8revlhW4hhL8QPavq+N5Aaxn6sNHMtQcI0NRBkP7RxYOesLBcfNalrdYy7GBLaNNhoYnLdafUNlnqWsLtASgn5U289UBWMAV7FvI9j+RoB9rJ3dNCrYT35dmWthtL2hYt+i3L8yj+tSchDHsOZjfGop9m0p+1UrVHxjCxxeO19CCCGEEEIIIYQQQgghhBBCCDlM4ctVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFuDLVUIIIYQQQgghhBBCCCGEEEIIaQHMEPuMYFJly+1ETUlI7SuJlm0bE9HWfUzsG3MwmWwQRJPfNifD/f9XBO/l4bvkU17+CtAevOde0MbzM6CVfOw+P8AEy7v3TYK2c2wfaPHOYdAWDywDzcSzoNVd7Ccv04f1q0aTGM9MjkOZVGc3aPuKB0CrhjiuA1lMfpzyMHFw0MAEw7aSS7qdaAnXW03CfrjQat20dj2Lp4DieDHQAoPlKkVMBp+fL4E2MT0LWjKLc60ni3PDbkpSbSl/U2JZaMsto/Tx4WsRvw2lLQZ9tW0wUXngK4m/HaWvlQTxVQvHzQuj42v19kOZcgHXjMbOLaD5VhK0MAGSlDxlLVH8XKyBba3vRT8nShJ6S1CrZrAyThXLudhcqQ1if1YOROdL1sK1wOroBS1Q2tpQHLNno52Eyjrv2Nif7mHm6OOukthe0foHcX1eqE3jtR7GBdV59HMxW1mzw+h80fxyvY5GoHmv+ek8aMl0CuuWwDHq7OkCLZtFGy0YvLbsV0ELUthWq45+pTKPcUEspvgQD/sllYm2LW7jnM/1Y/uPOO4I0EQZf5PEZ9qKf0slcT6uP+1YfEabGbdw/Swq8XbGwz6r13DcS2XFForYP8ZH31GtR7WqhX3tK+tsXYkpaspkMMq6FmsoexXFr9U1TXmuUmVpKHGFq8xpt4E+o9a3CLR4z2LQ5vdHY3gzOwFlBrFqUrCxbqPKPPVs9F2SxLllF/F+QYhzvJ1ocZkJFVurYpvv/NltoCnbHUk0iqD5pTxoO7YsRD7P7d0PZWI5nKNrjl4Dmikpz6yiL23UMG6rKfaomLxsevQR0HKZHN6viM8tFzCm92vYx6FWP8UXFxcK0fvPFqDM/u17QHvyV/eB1muh3xp76AG8djnuxZeuwrUjl+sETRQ7eamYK1dA84o4Hrvn86BZBTz3WDyIsWStMAfawgLeL5HujHz24wtQxgkxvsll0T+6SlyQzuC+03PRvovzGKOkUjjXkl4HaFMTGPMZJQ4aWYL9VFrAsTD1DGidSqwVa2ravBIrjY3jOCSTeC8/wAm+fRe2K5fCtb/cwPGpKe1vJzt2bgNt/ziui3UlBg2VPVCglCvk0eeM7dur3C9qf46DtuwpZ2T1OvpH7TgnVNZY18V5UKuhzQ/04R5Q2+/u3LEDyyl1qVbRNrr7lHNIvFSKFZwbJ514auTzKaecBmWGhzEmSiTRbhcvXQ7aO96xFrTxcTyb/fzffw60cgXnXzsplfOg1QNcT8VX9lMNtD/Xw0XLtpWzXF8JuN2mMzdHMRblq12hduYa4jP9GtqZZvO2hVqoxOTqeZ0S42slEwn0r709naAdmMT5bCud4DY9Q49ZlZopWjKFY7hQUs5kYljOs5W9W3B4fR+vHqJPcz3cw4SBsj4FmhfCTgyVeMRxlf5q0tSj9xbfH8RiaFMpG2Ntv4TtN8p8NAbnUENZx20l1o4lcA65tuIbLLSNurIeaH3QrAQN5f2gMlcmNmMcG2bw/okuHP9UAmO7ujbWz/E7qIfXTCGEEEIIIYQQQgghhBBCCCGEkMMUvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFsDM6s9ATUmmO1/GpOGBj0mbuzKYnDbnKEmVlUS3oY8JZq2mYkZJ6G47+N64XJ4F7ec/+B5oE3lsw0QR77d7DO+3e/8e0JxEBrTAwf5M5zC5vJfCa91EErS4kkw4YadBm65Hk8YPLV4KZaoVTBK8Y+cB0GbnMZmyY2F9R/uyoHmBklRcTTDdPmwbE02bUE19/oKi5lHHqQFoibFtNas2Eihp08MQx8hR5lW93gBtamYBtAUl+Xalhn6gVFYSv8fR15Qq6BsyKewov0nC9NzPkHz8eaCNxe8CgZL4PVTmqtH+LifEvq8bnNOBi/frKKANmb6ByOdkP/oq36CdSQyXNdM7CFrFw/q6B2bwfg4meS8pPtgM9IDmhdhP1RD7OJ1FX10vlEGrOWhXbhIt2mmaa25PP5SxPByHwMRByyqm7Ahe61uY5N5SEt+LYH+2k8L8PGiW4vv27tkNWtrD/iorvi9oJECLK31TzOcjn+0Ujm3o41zR/HwsjvOgZ2knaOlORcviOi422nLQwDnUKKL/tpRFrTCJ8dP81DRoR520FrTewW6sX1NV4h72eWcO51m6G2OxSoB93FBsvivTidoStIlCsQhau9lv4xiXDNqbmcF1u7qAvqlSRb+mzXTHKD4xiA5ezVJ8k7LkGAttPFTur9mfrwRVgbZsa2u5orlaXZQdluPjmph2cSwSRx8P2nYLbXqqGrXVboO2W5jHedWbQf+ztAPnR0bpT2Njw6o13DdYjcMrpr//0SdBG98zBlpjfB9oczu2gpbr7wKtpwfH0snguE0sROdVuYA+IlRs9Lab/hu0ytgUaHMHJkF7/IHHQPOVfbclOG7je7aDlvLQDrIZ9KelKt5vbMsWfK6PPsTLKbF/MWprpek5KOMqcy8XVEDr78b9acNH//PET24GbWLfOGhnveq1oCUxXHzJ2DOJfTMzXQAtqfilvhTabV8P9o3m6f06lquF0VirOod+KZnBzupJ4GAWFGedUdbjA/txbuRyHaDV67ifDJU+GegfAC0Wx/anU6hVyng/J47xgsRxHa6baFzlenj/iQmMqWoNXL9D7WxDmcsTyhlPzMP1oNVzhpeKuTm0+aFh3AMdGMf+mpvLg6a1z1U0x8E+bPZDto3jFovhWlwooK8KlL14uYzlYjH0o0r4Lvk5bP/I4sWgOUpbSzXNrrBttnLus2x0BLQLL7gAtJ6e6FxLK/P2qa3bQPuv//tfoC1evAi0v/7rvwStWMqD9stf3Qva3XfdDVo7KSrxl7GU9VSxUcdGH6TFx47i543iS6ymQ3pLsQsNY7C+2llaqOwPjLLg2wbLha4S7yhLmnbkqhyPSULxm56n+AZbOU91sV88E9VCpU8ayrmap6wHRjlrChu4zrnaFkc59wqUurQTXzBmTlvoIyxbOZsM0S/FlbF0lH4NDPah50TjFu38PFDGI+7hnqtSx/21ZvN+HH1/XZnzMQvntxUoe/261lbtAFCxvzjGbX6g+BDlHULzHDfKdaEyp0rT2NaeAt4/PYQxvlHOZpXhka4YxnutwG+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtABfrhJCCCGEEEIIIYQQQgghhBBCSAvw5SohhBBCCCGEEEIIIYQQQgghhLQAZu99BqYqmOh2ttEJ2u333A7akatSoJ1zVC9oXY6SxFbJIG03JVq2bUwGH5gGaJbyKnnn7p2gzVYw+a9JdYHmZDJYt64CaMlOTLBcr2Iy5bqSsDjXhcmOcxnUJg8cAG1BSVafjUWHPJHEJMR75qZBi2Uxqe/kgd2gZSaw/YM5HP+khabnhzhm7aRUxmTREqKNukrib6OU05KXa1pzMngRkeac0raSjFnDFi1TOWrFGtqjMViPpJI0vtrA5Nb7ZxZAm5xDLVTq1/DxueUCJi6fnEb73je2H7QjVy2PfF4xuhjKOEqidq39YpR+V7pY7Xbldur4tBOlOloS9lCxb+1PdQLFf3tKwvX4tq2gVR+8M/LZP+lk5ZmKrzbob2IF9C1VwaTxmf150Jw4PiNMYxssgwnigwY+N9vTCZo3NgOaFNHmvQFMzC578Vo3F12bqlOPQRknhetXuPpI0KoxbJetrFUxH43HVeaywUvbSqGEdtCwsZI7H8E+XDSyCLRcGtfnrjTapKlhXfLzTWPuK7ZXQ3+bUWKC5ceOgNa7sgc0x8GJaynB0sTuedD2bNoLWne2G7R1644G7f4nd4GWn0abT2cxfrKVWLFWi87JVCfOlUQcbT6dToCWNFjOCvCZvZ19oD3+5EOgPbVxM2jtZtN+7OtGTfHrjRbjchvnf6jYUaw5mBGRWpNT0NZFW+l/28FyjhJb2spa7igLsrZHcDx1YUMtxOdqYZonOH9NN+4vdir+9L4duF9ZmI3G62t7cG+VNXXQlmHoKWkL6+ZUlbbW0HkZg35Us4l28uPb7gFtemIKtNUxXLfX9qKPzcaxE33FP6c99ONd8Wi/7q1gn1Z9NKD5Rx8HTVtTE0X0YakSPiMI0TbEx71PJoV7xVio2EFlDrRiAftkoVgCrVZArWvJEGi9A1G/u2v7NiiTUNo1Mohr07zSJ3EHzxSc/CRoe8wToNXPfQVokkPppcL1MPaoLuBaXivjHtCr4zzIz+exnLIe+8q+QZr2io5iuJ6PtlJVYusDeTxrKNX2gJbJYRxg2zivKhX0X/EkPtdS9s+WYPuLin3P5rHfHRefUVLOh0wYnZP9OTQqawnGI14c71+t49jMKPvzQHBudOZwbpSKWN924iv7rtlp9EuxGK7Z2vgqwyt+qNiuh32T64ja3/Jly6FMVRnvcnkjaLUa+jRtj63dL6bs46amcO3ryuG60dGJtja7gPGj1HHuDgx1gvbyl6OPXHf0MaBVKtH2joyMQpntO3aB5nrYVu3IYu/YGGjHHIN74BNPOgm0Bx/AOL+d1BV/G4uhn9PiWS2O9hXfbKsBsrLHbxZaPK+0lL2Bdr4a2PjMQImrXeW8zvKU/UGonOEqa0SoxLOWchZSKeHaZJTx0c5xgybfrB05GkX0lBizVkI/6FdQy8TQb9Ut7JO6sidpJ0lrGDQ7wLY0DI6HYynllHinrpwTxl30kaZpLMMQ7VGU/VUjwLoFgrbiehh/Z2KDoFVsjDuqPq4b8VgnaL7BGMAJsVzdx2d4CTxHiSmaUd9vRG3NUdblQPFH9SqO18STOK6JDuy73kHcJ5s6vlPozeF60Ar85iohhBBCCCGEEEIIIYQQQgghhLQAX64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AF+uEkIIIYQQQgghhBBCCCGEEEJIC2DW2Gcq2LEMtPIMvpttxDBJ7GwZEyOX65hgNhfDpLuhUZICN2Umd5wUFKnWUZuq4a2mC3j/VOcQaF19S0ErhZj8t1ewXU4CtbqHba0qSbCrRXzGyEAPaGUlAfBkvQKa1ZT0en4WkzWLkni7XMQExk4M+3hiYQ60/fNYj5FeLak4VqWd5CtoMJlUGjTbxQTKgZLMWs3pruQHV3LLi92U6N1Skq2rGEwebVn4gAP7x0Dr7u4GLZmIgVarog2l4lhusE9JIK10QKlcBS2tJVyvYjlHMaJiLTqOvtJ+y0J7NErfaQOm3E4bVlVUH9FGPAf72Rb032GA/Rza2BhX+fudzBz6En/fOGi5Jl9VGD8AZeqJDtCMYBJ168AkaOlhnMv1nJJsXdDOkkX0DbG84r8FE6770/vx2iquB/7CPGjx2RxojQoalkkuj3zO79yLz0xmQMsOjYDmYHeKsXGNqAn2na/Mq3p4eDn6kuK/6soaWFNikcwwrsXJMA5aUEd7sS2cV5mmWGFqdhbKVJV1acXRo6CNHr8ItJpBO7OUpaQwjra3+Z4nQCsqNp9eq6yHgnXO9Q+AFlfqErdxLWlgF0t2UbTvJmvYd9lMFrR0EuMYN8Rnio92GzSwwtu34Fyb2Ib+p91MzGJcFld8vaOsUZaNPidu47WB4hNCZU0wVrScZpOOsli6SkEtfnKURTqh2JUvOO+Ni+2qeVjO8ZS1U4lbrAAd6nQOffGm/bje7diyCTS3FvVfiQDjrFUO2m66rPg9xa/7VfR7XoBj4YgSEyj21E52b9sF2uxCHrSlfTge/Q62paTsu6we7H9H2T/Fm2x3QIm30114r3S2E58ZQ5vy0tiGRYtxTUgk8Fq/geuEo7TfUsa8UcNr1xqcf6ESEJs62lXDVea9ia4n+57agnVbmAatpviVeRv9f06Zt2kX54tVx7bWi0V8SD9KLxUDig3ZCWWNnse1PCzlQSsoZxLplBJzK/bieVHNVuzH1FFLhGgDccV+5msYq/sFvLZaxXhkoA8HqVbDcjOzM6CVSjjmK1eMgpZOoa3NzGB/5jrwzGihGLW1HVP7oEz/APqQirK3qPvK+uXg5LAsZe5pa0n68PLzg/3DoE2M4R5zQdljaZtyo8QslhLvVGq435Ni1G9YOPUkFcO9qK/Uo6OrC2+v+JtKFWM7N4bxjq2c5+ydmgCtocSA8TjG0tkc7odOPOZloK0aPRK0dAz38t3ZaGcN9uMc7cjgnCoU8qCVKujf7rzzTqzbqhWgLVqk7ItdZY/QRjwPYxHXVRY8ZS1uNNAfKCGeGGVfrJ0nmia/HliKn9fiefe5n8O5Dl6bVNYgP1DOZgNsl6tM1LqvvI9Q/Ga1in6gXsNr3RDr15WJzoN6A69rlLV6YJ/UK8q5Ug3b2tuDfmWqoL0bQKmdxC08CPBDbLOxUNNs3ih2GlN8v+coe8d6NPYIlM7KKL6qUcGxTPrKAUcS27BQwfctC/O410gl0bcGDl5rZ9AeUx7aVTpAewmUOantGRyjvHZsiu8sZS77ytyr11FbOKDEe3NKzLIaY6xSA8sN9eKeqRX4zVVCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpgZZzrq455mTQ9t23GbRMB/4m/skvw2tTzm7Q6iXMPaHls7S86G8lBwZ//znbvwS0Rx7bBlqmE3MELBo5CjRj429ge0re1LCG+TjqSg4RrV2Okp/uyUcfAy0Xx2tTaczZkE5hzp3xA9F8Cr6Sy8TxsK3dOfx96nwef1d+bha1nQcwr8XwAOa1dWNKUtw24ir5IwIl12lDyb0hFv6eu6YFSg4DW8uT2qQZJZeZhpJeQZR0afB78b9+ptIGJZdsZxZtr6El6VDyeaaUHHhazlXLQZu0lBwD8aTiL5oa7Ct52oyWS6DFvhNlLJS0Knoe1sMs6Wpcyd1llHGTUJmrIbbQVrSih/1fPPFY0HLu+sjncgFzpjSUhIBWXFnW6mjLXhLbWgrQp9tKTpFGoORh0HLwxJScgFg7qSg5P8pFJZ+lUueq8ox4Jur7u7NKjgRlDSoq80c87OOkkmtSy2WsDL80DjObT2XQfxWnMV/b4KLFoI2uWA5aVxL7es/2naCN78AYqKcv6g9jSq7S+hDmz1iyFtdTW5lndlXJm+fjIG1/EPOGlmYwt9OaY7H9R5yC+ZT278H7dSgJVteetAY0O4c2mVTiNi8VvV+1nocyE7OY88QSzJ/iKI4+UOZ3oYC5raYm0XZCJc5qN4ES/ltKHmBXybtmKWuelptVa7Wnrb9NJW2ljKfczVXiMW3s/LSSw0XJM5RUcq7ElbxkRVHyJyk5mbV1oqIsAAUfxYkptCPLYHydbfLPQxXMldRvMKYKlPguVGKqmjL+alwZYh9rebvaycQezFO4Y/cu0JasHARtpA9j1UQWc6B73ZhLOrEY1+1kMrruBFr+eGUeaH3qKPOgXkM7sD205ZiSczrbibkbXSX3sBrVKlKgxDdaPOwq+Zi0GG9+aizyuSeLe12/hLkDbSXfmhfHsdFyRblK7jOngHm9tz98H2jDy3Fde6lIKbm/lq1aBVq/so/bs/0pvF9CyXen/pm+NuZRTcv/puXsDZX8hq4SvyaU/YsTx2sHevtAqyl5WDUyyllLVyeuJVUl76WWF1HLWRgEOGbNOVGTGTyTqVbRz5cquGY4MdxPa+3SfHqjrORwVfZX7WR2Og+alrNdlPMXLXdjqB2VKocG1Tr2dU9/NFbduXsP3j9UclcreXE7lZgl24lr0K6duLfQcq5qYWlDWWBKZZwbfQO4zp13zgWgHXvMCaCl4mhrfT04Jzub2qbFe0889ihomu9ZPIx5eE8/7VTQ0mlcD1MJ1LQ1op0obkSCFgOw0MZymm3YSjktVoc8qerhl1IPZR5o+VAV9yhdOYyferVcxnlcsxtKBbW1pFJWfLqSm7XRUPbtynsAjYGmvMITExjHWNq4KlpZyZuqXRt3cT0I6nj+ZKunV+3Dt5UYV5mX9SK2uVieAi3bgTF+3FNihQaus7WmMc8m8YwiVsU1u3z7GGjDBaxHZSnaXvUojEn7evEdnO8rsU2Aa1XKwfXFEpxDgaPkPw2wTxzlXZXjKr6m6d2IUd6FaWflWv5ko+WJb2A9asEB0BYN4lj3dI9iXVqA31wlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFuDLVUIIIYQQQgghhBBCCCGEEEIIaQEla6xOqgOT844sXw1aBXPkytJlK0HrbWDS2byShL1h8IaBH02we/JZF+Mzl58I2rKjd4H24MOYDL0rMwja+OQ0aK7BBPFxDxPnipJ7ulgqgZafw0Tb3Wm8n5aiPFCyj/f2aYmYowmAp+fmoYzl4Dv3bAaTGrsOmk+9iu3avmcfaH2dmNh51eIcaO3kK//+ddAspZ89F8cok8Vk6CuXLQXtpGOOBM1V/uTBND0XEsaLiLGVrPFKlns/xCTQXd1K8u04tsEoid9jMUyG3tOFScWNkgzdjeEcirmKW/KwLlUf25FfmENtPmrjhfk8lGkoiepFSYzd09MJ2qqVy0HzYtgGZcjE0sasjaTTOC/9BI5RI9D6KwDJD0MsFsNnJAc6QFsoRZPVT80X8V5a8voyrhkxJUF6PV8GzTdY33gM5/eC4gcSnmK3NmphiP1UKysJ50Ns23xFSSSvXJpyo+3ILl4CZRxsqoiN7bK0v8FSJEtbmRSjD5U+bifJ7ixoMWVdtBX/lUngmpXMpUBbfsQa0A7sOQDa/omZyOehDPq945U1Y8ngMGgmxEHybZwbW5/cBtrUninQBpZjPHHEKetAy/Zg+yuVKmi5LK4b8YFu0GxP8T+C82BiW7TOS1YPYD189FuuYvNiK88M0VdPT42DNjeNsWLSxj5pN46Nfk3rClePOPFaJTbQ4iVb8R1us2jwXrYSy3iKbaS7cE5Ws9jWQJmnZgbtI6ihvwoCtL9SgHMrdDOg1RIYS+cbim9JdoE2Oop1TvpRX+UGODbzdey7sITrn6vYuK/MtcDC+loGB9aottM+9uzZA1qtiv3w8F7csyxddipoxx13CmixviHQqg20jYWmhduv4kIeBmh7vo/3cpU4yChxvhaPzc8vgDY+/hRoDaUNdSX46OjoBC2dRpufmcI1plFDewlctMmeZFTLFwpQxqrjvbIxbEMizINm15W1M6Hs7RU/cO+dd4J25hsux2tfIo5YjOtgMo1+ZFCJEW3lPKMyvx+00gKueaLsbZymM4NGow5lghj2fc8inFOhj7HRvhmM20LFB5VL6OeNEpcuXrwYNF/Zd2Yy2J+1Gu5Xtm3FMy5X2e8WFd/sOdF2aPG2pfllUEQadWyD62H702nl6jo+o1rG+K6dlMtK/2kxi2ajynmGr2xZLGU/EBrs1+bzEUvwZp2dnaD9wZuvAO3EE08CLZHAeXCn4oNuvOkm0MbHcJ0bWoT+IpPFPdL649eDduaZZ4DW34v3yynrQTyOsVxPb/RcampqAsps2YJrVeCjX5mamgTt8ccfA23ZstGWrtXO4NqJFpNqWIL9rG3otbmh2bxRYvVW+kbze6ESn6QUW6krbV3Io+8f6MWzpriFz3BjylxW4idtj+MofVL1tb7D9npKHzTHd5UynqnHlLEJGxiLBXUl3lHW9HoV54t2JmV52NZ2EorywknbwyrvKnLK/spWDsVqIa7jRonLu1PRWMEXvK6yBd/xpB7CPUnaxT3sQHoRaKU62rfTjdf6dXyusjWXYg39XNzHdVOUWEk5YhXHw/GxlDne/H7DiSk+Rdn/a5qlnJvu24R7jaE1I6CtXHsyaDNTir/EV4QAv7lKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gJKClodJ54BbXxiE2jHrceE6+mOFN6vMAZaoGSNd2OYfHnH3oXI5zO6lkEZaUouLCKSTWOS+4SL7UrGsL4JJcm9hJjod9HwEGgbt28HLRbDJPQLhQXQli1eDdrqtUeCNjs7B1om1wna+IFowmLLxuS/nV3doM0v4P0dB9/NJ1OYJLoSx37fureA18YOr3f9lXIVtHoFNS0peQFzq0tKKRccsRa0qsHk4nZTwud4DBNZaznkA0XUkkB3dPfhM5VyYuMY1ZUE1U5MSQJt4bV4pUioZCTftXsHaGOTmHx7dmYGtEqlEvkc1DBRfb2iJHSvod0uXjIA2tIl6GvSMc21KmMhSh+3EddT5nQW/WGxjMnaXRevDWxss2vhqNuKzYcS1SwHx81V/BcqIo06jmXSw7XFtXHcPBfv6CnPDXzFrqo10HzF6r0k2kEYoBZTxscLFc2P1q9ulMTvSj0SgeZEsF2a2YaKqHl06zCz+YSLduCFWEe/get9GGAfWjZem0zjer/iqDWgPXDHfZHPm8b2QZljzlgHWs3DcfPmsW49ButRkE7Q1q1eBVrvqkF8RhrjolK5BFrfCD4j1oF1qTRAku4kzrXtjxwAbe+eicjnM9YeDWVCG9fvUDF5Y2Nc2AhwUQ8b6FfCQLETC7V2E1emoavEC44211V/oqCIRotJmiSjrBuB4tl9F+2v6GLsMVnAcU+4adDKnhL7d2VByy0dBm1k2QhoQ0swVne6e/C5d90NWm0a6zyxdy9o+zY+GPl8YKATyix4OdDciWnQOhdw7gYGbVeLIW3FZwaWMrnaSK2O67FpoNMZHkXfYS/GWL0U7wDNV5xYqYj96gfRcl3deK+6Ut9GHddjX4ktQ6NE10rc8pObbwHtrjvuBC2r7CerNYzbTj/tdNCOPBLnwT133wdaqYZ9p/mas06Kjs/QUozB3RT6hlQS47s5g890G9h3MSXmrfrY/lklXGonRyzDvtk+huvnxDTu8RMZ3M8nlDjcKP2wsIB7fKspvjaKPXaPYn1P3PAKvH+Afn7Ttl1YNyVGS6aUMykHbWN4GP18VTkD2LMHnxtTzjPWr18PmufiXr5UxljjwP5oLOgX0R4XlIOHQFmEHaXfRXAexFz0/QOD6KcGB3GNbCehcjZnKU3u7kb79jLYvv37ca1Ujr8kEcOYIpGI2trKFcuhzHnnvRy0U045BbT+fjx/6OrENpx22mmgXXbZZaDdccftoM3P50HTdtUD/YtAy2Ywbh4YwrOlbmUt8RUfkkhGfXi9XoEymqadLGntuv/+X4KWTKBf2bt3t/KIwyumN6LtT7WoHMvZ6qkJGritGL2tnOuZpjNBLebX0M6Ue7rQvqfz6OcWFvDsb2ZiP2gNS4mflDMALX7SzptsC9cNS3m14jgYj8TiaGvzC9H3AGGAftk22IZaFdelno5O0FwH21BYwLVEPdvQzoTbiY/24ivjFlaxv1Jx3BMFIfZNvaydHeIa4YZRP18JMJ6SCeznRBXn47yHvjAeom91tfMMC+urrX2h0neeoI1aiv15Lp7dhMo+saML6xyL49wwTWendhzrZrV4fu4r55W1EmqrhnB99Yv4LvGJPXnQLjoax7+Zw+ttFiGEEEIIIYQQQgghhBBCCCGEHKbw5SohhBBCCCGEEEIIIYQQQgghhLQAX64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AGaWfQa8BCb/rVYx6W6tpiS/jaVAS6XxfukElos7mIg240aT+H71y/8GZV77xndjPUoHQIspiXNtG5+5bDkmb5+cHQetWiyBNtjfC9rsAiYdrtWxP5evXAnaipWrQZt/+CHQSgVMnrxQij7XV5JWVyqYJLmzExP4BqYAWkcnJgb369ifWiLmffsnQWsnl73+DaDVyhXQ0skkaFry5WRMSXyO3S8LTQnNRURCPzqvtITSbhI142Im60oD7cyEWDfbVhJeK4nfXeUZnoeJpi1bSUitJEhvGCxXDdGvpHOYLLursxO0oB69NuHgeOVn8qDtG9sN2splOB8dG/suUNrgKG1VirWVWAzHMpbA9oUGE58nPbQ/38K5X1hA+wscfG6iozvyeSCdhTKiJK/X5p6lJD53lL8tcizUYm7LyyRgFP/qC2qBo8wNpW22osVEyVbf1I6asqYpTRU3VOommKjeUmzZUnyIg8XEcQ6vv+kacDDu2FnG9SkI0Ac1aoot+9hfdhzHaPHqUdD274r6nP3TOB7xYfRfMz6uGf3zWI9sgOt4VxL96MpzXg5a93A3aPOVedCK1ixotQDXzdg41i8sYXuLSYyVPAv7c9XxayOfE73oL2Zm5kArN/BeGcUParFoQpl6tjI3ikWMldpNwsL+dzU/aeG81pYt0+K8VlyMhFb0jur9lThjoY5t8Bt4dXrVMaCtPfeVoPUsWgyanUmDFu/A/Qtah4gfoG+ZaaBvWX7yqaCduRRjjSfv+yVo//SreyOf7y5j3JLN4bw/e9mRoJk9O0ALpsdQs7CPbSWY0eKgtqL45kQK/emxJ64HrSOn7IHLim/Kot+xbJxXQZPtjo3hfjIIFB+eQX+txS2OMl8mJiZAu+OOO0A75aSTQVu2fAVoM7Po6wcHB0DrH8A98Bnnng2a6+H+IlAchtvkawK/D8ssxX1yaCv7Fx/XpkCJ/SsT2vhgTDA1i2tiO3FiMdDcFMbq27ZtBm1p/xBqvV2gJYbQRwYeliuVon2T7UD/sOxotL2eJUeA5pTwrGW5Eh8rW1EZ7MF2GWVvGyrXNpRzL185M7GUGGJkZBS0eAz3UkZZAeu1WuRzYR5jil27t4E2tnsraMH8DGjVAt5v9wEsNzOJ50qDi7E/20k6jT4yHcfBTGfQV/sWzpd4HLWBQfRpR687GrREPBoDpNMYE6xZjWtxTlmzO5XzjXgC7afRQL80MjIC2mWXvRG0METby8/h/mJ8HM9TGw1cr4xR4kxP2e8n0Ie4bpOfDzHK0vaTWihaU+boPXfi2nf/ffeC1teH60tDOa9tJ66NfWo5SuyuODVlyyJGOaewlHjOaDGesZrK4IA0fGUjoDA7h37JKHGco5yDFEsYn8WUvbgJtDMj1HJJ7b0F+oYwRNuwBNdcW4kfFmanI5/9Rg3K1BXb09owr6yRGcVfOMp+WospD7dwPgzQvhsBzvOYcuabMDiWpTr2tSjnA2UlZqwHe6L3j+N4T0+hLS9vYD8Hin98dBavtYrYfi+JY24rc8OzsX7iYvs9R4m/bdRqynuerr4e0NLdOBb1ctN8Vt49hIomio9qKPXIdi4DracH9we/vA/jnbrXqTz3t3N4nXISQgghhBBCCCGEEEIIIYQQQshhCl+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtABfrhJCCCGEEEIIIYQQQgghhBBCSAtgNtxnwHI80MpFTJxbLWOiX8/DBMqFGUxILUrSYU/mQRvqjCbn3bppK5QZ37cN718eB2n3vl2gHT94MmiLRgZBG54cAK20bTdo3fFO0LKdvaBt37ETtKHhRaDlFzC5fCPAxL4TU5icN2xKNK4lPC8rid8tG8dLyYEu6QwmpZewG6SYpSSEnjmg3LF9hA3sU0f5ewRMFS2SiWE/JJVE4pUqjmW5gX29a8euyOdYDOfK0mUjoO3cizb/g5t/BlrDxvmdiGOi9pTShnQSE2N35DBZeGdHFrTjjz8GtL7eLtBWLMZ5YLeYhL1ejSbpdm20+Uo/2ujwENZjeNEQaEGA41UuY6L6dBLHTKluW3GVee5YddASynqQn5wFbba4H7Sp/ftA68pi4vN1Rx4d+ewlsP9qYkBrBJjQ3A6xnDaXbRu9mq0kUrcsLGcMPiOw0IfYRvGcSv00D2sr81TUukSf6yr31+aPdn/PwTnvac5faYLt4DMCpY/bSXGuAFqpWARNm6vzc+i/jeIP+pdg/GArfnPdy46NfD66ugLKOA76lso0xkkDMRy3VKD0/Ry29cAOjJ8cB31wzk5huQDbVVPW0thcDTUX7zc9PgfaygyuJTWJtrdawDjGddG+F0oYJ9UM9vFgJ9ZNixHcGK4vw4N9oLWbuOKvjDI3LU0ziu9U/Km2wfAd5blNkys02K9lZezSy9eA1nsMxhTx0eWgTbodoD2+BdemyYlJ0CpzON8KxTxos3Nl0ObKqJ106omgnfa+s0HLnIl9/ODLXhb5/K1f/BjKTC/gOjygrLknL10FWnke/YPdQM0VXHd9xcbaSaOOPqd3YDForrJnrdYxDspmMqBpcYC2Rk9PTEQ/T05AGbFx/Vy5aiVoblrxwza24aH7HwGtXMY+GRjCPrnz7vtAe/SRx0B71YUX4jMa2Cf7lHmlxRCaBQVN+92G0udxZf3rSeN4ZVLopbrS6Bv21XD8Y1XUXBf3Te0kUNayrizaS6ML92wdylpWqGCbGzFc81cfewqWa0T3/dkMjlHXEO5ja0rfl/IYF4Q1PFeIK2cSjTKeXVlxbIM6rxJYbnR0FDRb6btkCu9XVtYD30df2pHrjHyOeXivbAfad3cH7pv2PPEIaPkG9vHQIO5355Qzvu17Fd/VRgYG8WxuyQCeubkexhQTs7gf6D62H7STT8Y1O1T2Wb+871eRz2vXHgVlurqwbr29+EzHwfW/0cD5rdmPti45ylmICTH28pR+yuUwBteem0ygP5zP41nBoDJmItG6jI9jfFZX7NZzlXYZ3JdpbZ04gLZcKaG/KCvzoJ3YynmBq5xdqF+psrEfmmNyERFLsFwQoF01nzNrOwEtTAqV8ZiZxVg7p/jgRBz9vOOgbcSVONX1lXb56AcScWU9MHi/eIgxVb2O+9HiPM4Dz4p2jKPsexJK7K4cG0upgnF6dQG1VBrnckcO15KaEgO3k1qgvAtR6ug4naDNzaNd+YLj5nh4vyBUzhiT0fliW2jzk4qv7tXW3QBtJT8/BlqmOgxaWtm7eMq5XqWuvINpoGbH0LBCQc3zMH50EthPluJXHLdpn9OF9hhUlfdSDby/NjZLVuM52sQ0js9DG9H3H3cSrs2tcJgd7RNCCCGEEEIIIYQQQgghhBBCyOEJX64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AF+uEkIIIYQQQgghhBBCCCGEEEJIC2BG12dCSdTuGEwCPdSLiZZTCUyw+/PHtoPWpSSVXtWtJHOORxPWxlxMdDs1uQu0sDYH2tIVy0BzlPqmcl2g9Q4sBm1mFpNFzy+UQQuU5NP9fZjA3lWSE1frmLC3riT2rVQxObPf9ODmzyIi1RomWPZ9fA/f04v1tSwcr5iF4xO3lETEBpOFt5Pvfv8noIVKQmpbsL8yMWxLNocJn0dXoQ319WAi8Z6hpZHP3UrfJ9KY5D2/aTdoj2/aC1pFyS7fnGNaRMQVLJdTnrty6QhoLzv5BNB6lETqaQfdkrGwLnVlHvgB2nx5Ph/53AhwDJMpbENnZxq0iQOY8Hp6egbvl8bxHxjEMUulcH735rBPXiosCzvaddAQQhv9QaFQAG1q6gBo+TlMzL7lsV+B9tSj90Y+r1x5JJQZXXkEaF29A6CJje3SEp+LwXYppieOrUwOpaSrTCKtj8MQ175QWySUZ0AyeBGYpUaZ35qmYQKsm6/dT7nWUvy8tn61E0uZg0OL0YaqNfQtgbLu1pV1d+7AFGj9o0tA6+rpjnxOz6IvrO0dB21RDNeWhl3Bulno+4aHlWsbOOaNvZOgTTVw1EPFX2TTuKalkx2gubEYaLaNWi6O83R6Zj7yub5rHsqYbvTzKeWZTlL5u0MPY5uaEhePrl0O2vKluM63G1dwnOran1sqrs4KFD+k+E7NS+YVn+A1+bWGhePUsfZo0BojK0D71RSOe37XfaCFsSRoT+zYAdqe7VtBSxn0zX3duG7vn8Y9R02Jkc/asAG0UgnjymS6F6997Rsin+/Z+CSU2bUH91tP7N0DWiyJvsBK4DzNKr6wSxlXbZ1oJ9o629/XB1qjhn7SzWA8GFN8h/YMbQ01Tf4k1YN7zJSH/t9TNGPhGmYp8zE/h3PDdfHawgLuY3fs2Ana/AzGvpbS/nhc8evKOjGtzJdSqQRatRbdU8aSOJf7lf7s7Udb7u3EtalWwT7ePI/23ecrc7RHiT/biG2U/animDuXLALNUdbenz76GGh5dAfyqpFVoA0tie4LO3I4bjXFt+zbshG0iXHFfylnN10JXEvG96Ofd1K4Z1t2xFFYTjmTibv4DMtV1khPidUVH1ko4l6q2a+4yj5ZLOzP5SvXYrmycibjYfvXLhoGbaGK9vTtH+JZSTvJZnAtXr16NWh15Twn2YF9s2Yt7j37+/Gs86mntoDWaIqRjeKXfSWOtiylnK+c11Wxvo7iW21lz27bGIt4LmrxOD43k0F/ON901iIiUm/gfJ6ZwP1Qdw/65mo1Wpd77rkbyhSKuKYFSj/Nz8yClkqgf8suGQStsxPXkq1bMKZqJ7ZiL9quXLMDUc4ktDMTrZztoh+ymg/slEe6ynXaebRY2IZSA31QTLNlpU/SCWXNKeVBG9uPZ6elSSwXz6BteJYyT+voaxzlYDPRdBbZo7wXkDjGLPsnMRZzlLksIdYjUN7lZBK4HjSUfm8nbgz7zwrRrmwPyyWVswDtrMFXzvdtbe11ouWMgzawoMSLMw0sl9LOUlOodXV2gmZ5aN9KSC6OjW3oiOP+shZi/K1s/8VztHNCfPcVKudjcS+6t5oLsJ/CCl7n+GjLKWXtH12F58RPbFoA7cAkriWO1fpr0qfDb64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AF+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtEDLmVo9F9/DdmQwcW5nFjUrxMTnCyYN2vQcZsntzWIV07Fo0uHAzkOZXeO7QBvowoTpIysxUX0Vc+TKrx7cBNrY/jnQslpyaw8TJz+5bQ8+RHnXHSparY79WSxVQOvs7gbNb0qgvX9iEsqks9hProNJxVMpTHgdi8VBkwYm2g6UBOID/ZiIuJ3c//AToCU9THhdr2FiZC+G43bKqSeBtntsL2gz+7Eu6446KvI5lkSbKtcwCbSXwPE44YRjQKsqyaJjHs69VcuXgXbUEWtAG+7tBC2XUhJtV7HOew9MgTY5h3Nt/zSWKxUx+XY+n498rjewrV4M2xqLYx8HPs6DRgPnY6oTbXmdHAVaRweWWz7YB1o7cVwHtEQC+2btmrWgrTxiEWjlAhr4kw89DNrDD9wX+XznHbuhzKaNOEdXH3EcaKvWYELzzq5O0GKKHTgOtl9EyeguStZ4tZxiQyEmtQ+VZO0aoZJdPmjy86HyTK1mrWIZvJ+xsJ9sG/vTD/HadpLoxFgkNo1+M5lD/xVzsX2ug9rc+AHQ+ocGQQuc6Kj4C+irGnNl0CaD1nx/LoPzNuGBJKlsDrRqGf1crVwFzQRoy8ViATUXr3VcpTKOEmf0YJy1pCMa74Qh9sm2zftA6xroB63moS0XK3g/Rwmhk3FsQ93gte3GUmzXMujDHGUOmwB9U2jhtaHiEyp15Vo72mfeihVQZlaJS598HP1/fg7jse7eAdB8ZT8QKDbjKLFcuYD2LEkl9u9A2z1i3fGgnXLeBtCqDZxHbhF95zEnvCzy+ZzzXgVl/uv/9++gGWUf8ehW3OdklTnZp/i4IFTmgrS2hr1UJJS4eZkS09oOro6W8qfIobJu245S0MZxW5qMajkLbcqEOKcqBu22JBhHOsqaPzg0BFpi82bQtBilpNh8qMTD9QbOoUJ+HrT8LM7Tp57aptwPbahWja4ddgxttNSH/bQyg/6oZOE8KAd4P9/g3q9az4M2OzsBWjvxbGxzMqns0wXLjc/OgrZ5Es8uqkp/zRWLoPXbiyOfA+VsZGYC9wf7dm0FLVTOEBJdPaCVZrANCzM4RrEGxjyWwfkdjyt9p8QLyrKp7qWSCYwrtf1V81xzLOxzJ5bBqnlKpL8abd4yWLe+IYyNcv24Px1v2mO3G63/jLJnmZ9HH7RqDe7TV67CeKSwgD4tk0E/3PzUZAJjAmNwjAoL6G9TabS95vMNEZGuLoxFXCXecxTfYCmGm0njHskosWI+j3NN2w9plMp4duM22e7u3bugTEGLxZR1s17HvZS2fvcPoH03lGvjcVwP2olloQ2FSj94nrLHUlyEUQIeSzkLsbU9W5PfDG2sh+spZxLKUYt2WhAqZwi+Enc0fNTyypljdW4cNKeB61cq0wtaXVnDAuWM0VbOljQaTrQ/J/NYD9vF+yvDr55duYpvFFvxP8r6bWtBcBtZKCpn74Lz0rJwzBNxnPs1Zf/neeivvRjac/P6EobY96ESy24r49n2rgDHd3MBxyi3D9fnJVkc31oFz8otG/sp6ShxR4hjXjW49llaSKmcBTkNZc9Qi55plRt4XpZUjlfryjuPdcevxrrFcN/z0KO47zE+nklpcXErHF4zhRBCCCGEEEIIIYQQQgghhBBCDlP4cpUQQgghhBBCCCGEEEIIIYQQQlqAL1cJIYQQQgghhBBCCCGEEEIIIaQF+HKVEEIIIYQQQgghhBBCCCGEEEJaoLUMyyLiKImMB/sHlRtiubCKyXmHFi8D7YHxXaDlLSWRuhNNfN7Ri0lzO3KYZNtLYLL50ZVHgpbp6AHthq/8B2hlpV0LFUzoXq5gonZP6fnBLqxzdXY3aCUlEXNHDvvpqc1bQZuYiCY2Xihg0urOTqxcLp0BzTGYnNmrY1udMiYL70tjIuKOhJKRu41M7cO+7+7qAm3xYkwqfeQxq0Dz4ti+Jx/5FWgDSsLxjBUd88np/VAmnesArSeH97rogrNA0xKVd3Tg/Xp7cG7Mzs6AtnM32t58HpOPL8wXQCsslEHLl9CuZhcwqbavJAz3vOi8isVxntmO0v4cjldnZydoXf3oV+IpTIIeS6JWrGgJtNtHGGLWcNvGhN7G1sphfzkO9nVnz1LQzjh7ALSVK6NrxF23/wLK7Nw5BlrpYcUvL+RBO/qYY0FbsmQJaK6D/jDw0QcHSt+FoQ+aEUzoLgY1y9I0vNSy0XatpnU4VB5pK9cZpR5au7T6Gm3tN1jhQNHaSamE/savox/xlbz2vjK+QYB946aSoJUX0PclOqLrrJtD33La2RtA++VDD4F29wMPg3b0alyXBrrwGYUZjAs6OnE9WDwwBFqlhNfO5DEuqlYwBhAH+25i5gBoqWwMtJGVayKfrSqOzTLFlnfNToLm5oZBK1Wxvru2bgdt55ZNoA2PngFauwkVfy3YZWJrvkO5tG5h39azOdC6+9eAVq1Fr8334d7iwZ17QIu56Ju7u7W4BeuxL8A5XvdxjDMdeK2dwbiqbynuac5dfxJo513wGrx20QjWpYb96SbQ7qu1qP+KdeBe4Oh1uNYd2PokaDMVnLulrm6839Engtan7HPmHsP4tp1oMUpvbydoA4O9oBXLGL+Khdf6BmOD0KBdZZv2dsndD0CZhjKnKsf1gVaPafszbOtTm7fhM3x8hhNT1nLBcr5yhDA+jf403olrzNT0HGiz+Tw+t4FOyWoKaCwl7i/E0HHNzOH9pxeUcc3h+DuJOGhuHft4qoDxRDtJOugPLBfbotV6XwljlDCDgVB1rgLagQP7QFuxYnlUUPZd9QbOFU85a+juQB8sytwrKWMuddx3dWVxzUknsZ9s5fDGMuiXbUeZVy72nRaHB4Gyv2hqWzKG+0kngfsto+wjUjncx9cbWLc923DNXabsn5cuxTOQdlJQYtBHNipx2TDGeatWrwTNVb6G0qihDcUU26hUonNoXJkX9QD3rA0lFmk00N8UCjhHYzG0R8dRbM9C3+ooAZ/n4bV+HJ/RqOE8VZYw6e7uRNFgwWIxGlNoe4ZSEX2Pr8T5VUVrKOefC0X0hCnlTC6pnOe0k1DZk4eKH7HUQwTUlGETbeeubA+UQni3IMS6abjKGuEo1/plHLeaEpOKjTaaVnx1Rwfu2WtKbL1fORgIfcVhaGc8ypysm6hma2dydZwHjoc2qvoBJS5saHVTRlbbC7aTlI17vXQaz+idOq5Z9QL2YRhTYlwLbage5EGLNcXgyrGh1JTYJlT8fDVAbWoOffWuu7Fui1bhmCeyaBuNKs6hA1ObQevowLNZ7dXh3ALG876Pc8jEcC9Qq0XblijjmhbPYj0aosSxDZyj9z+I73J2bcW9UCxAH+IoMWUr8JurhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gJ8uUoIIYQQQgghhBBCCCGEEEIIIS2gpNzVicUwcWyuaxA0P8Bbxl28dvWypaA98CAmul3wMLl8aEWT3Q4swmTFGzfdC9ppG64E7d577gOtVFoArVGfBm3ywF7QtPfVxQZqrmBS7S4bEwIvSmJd5qe2guY7mMR5oB+1IIgmDq5UqlCmWsGkviUPx9APMRl8ozoGWr+H9xvOYNLhmo/l2snY5idBW8hlQHvtK/8EtAsuOA+0W3/+E9D6O9Hm+1PYN0k3moQ8YWHi7YGOHGhZRUukMLm1ryQvj8WVcgE+98BmHPM9kxOg1Rv4DDeBbc1mu0HrT6RAa9RxDml4sah/cBycj5qWzeLY5HKoOQ4miC+W0JYnJtCHVKuKzZ94LGovEZbtgGZb6NNttwaa5+D4BhbezxK0IdvDJOyrVh8T+Rz6OEb7938LtLnpcdC21uZBmxjD5O0rVq0F7YijjgGtf2AINFdZ5/wGtqvhY2L6QEmabrS+s9HWVEy0jy1p7TqjlVNtQnskjr9YWNC2cb1uJ/VKBbR0Cv18Q1mzwwTaclJZI1LpPtCa12IRkTCI2sHY/AyUWZVCH3Ty0SeA9uBDG0Er1/CZyWQHaImYMubKoI+Po5+Px3F8R0ZHQTMh3s/z8LlLiiXQ9ivP3bYp2t7VRx0PZVZ0HwXa7C+nUJtDv9wQrNv0Qh60jq5e0JavWAFauzFG+dvKEO3ZKNNa2zo0kujrZjsxBu1ehTF92Y8+ZNtMAcoMHHE0aHt2bwEtcNGujFXHZyrxw7p1+IwLLrgAtFXLR0FbtGgxaN39uEcKlT3C9CyuT6LMBb+OvuobX70h8vnO7+CaeEz/MtCqyno618DxP/JI7JMzXn4haO4Ezsm7n3gctHZiK2tZOo3+ukOJm4sV3Is1+2sRUdc8RRLLRG3Sr6Kvd2IY90odn2krvtRYOHHLyt6upswDx0Uf3tOPa1h5AeviKTFEsYw+vFjB/WMuh/uBhId1STadR5TrGI9msniv8QKuf0aJvaw6jkVRiROGlH53WtuWvGR4MSWWUQxyvobjMdlAe8n1Y7xgOTi++Xnc71TKTc8Ica/X1Y1a7IgjQXMKeP/ZafSjFWWP5VpoB709ynPjSdB8dJFiK/txFS1uVhZYX4kNm52ItlcTC326umfwcG/vKecO+3fjWVO1gX4wH+B63U727MOzucIC1vt1S0dASyRwHzc/lwctm8b+mlXKVWtRv/HQow9CmZ/fditob3jdRaC5Lo7v3r3Y1nIZbX5kBNuaTeP6YjloV1rMon0zZ2L/AdCKRfQrZ57xMtDSyplgc9vGxvCsqVzGM8x6gE5Y23dreEr7uxTf0JFFP9hO1FhE8y1KP2g+wnIwxg+UZ1gWamHTGU+gnC+Gyrobi+Fa7yprrD+NezazgDZfr+G65GTQckMX/beraMZGLQyxP2Oecmam7K0c5b2Kb6LtDXzcu1hKf3rKM8MazgPPVfZuynoTU87krIZiY20kEcO2OMq+1nKwD33lvYTlYX/5nnI+5GEs7FnRsawHeJ00sL7TZSyn9XJcOZM6sAXjnf2b8fyz90i0s57kKGgJ5Z1R2cdYWLODlPI2sRHic2MJbEe8I2praRfXgp35PGg5ZW8wmcfY5tE9D4AW1HDdiCXQ5l3lfL8V+M1VQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFuDLVUIIIYQQQgghhBBCCCGEEEIIaQG+XCWEEEIIIYQQQgghhBBCCCGEkBZQUtDqaMnGu3p7QfMtvGXVxiSxiUwOtM5OTBC+Zy8mSD/jpKOi9y9iouhUFhNe7x/bB9q2LVtA8wNMfmxjjnMpLWAy4WzPEGjz85g4uSODSXfXrF4H2v2PPgXaQ0/tBO2Ms18FmhfDZPU7tm2LfM4XsG5a8vpqBZPSjwxkQUumk6B1d2M542ICbb+OSbrbSbWMydCPPu5o0M4971zQejp7QDv9lLNAs21sc9bDJNC5pvnnxNB+3Bj2vVHuHwra9/wcJq3OuViPUHAiLF+Ddtu/eDVos3MLoGU7O0FrBFhnS0lS7imTMlSSxler0cTVxRLasgkxQXexjOX27t+P96/gHGqUMVl2EChJwNPYx+3EtjB5t6NqOEYxJe93qDnOEK81is+p16PJ5RcvGYUyo6Oo3T+BY+T7+MypyTxq05gMftOmx0BbtmwlaCtWrAJtYGARaNksrnNiYWL2ah3tJVB8pBfD9dWYaLlQlD5X3K2xcP7oaHMUDUBLBe+oavtwlLakMhnQcj2o1cIaaLEY2vL0PrTJdG83aAvj0XKJGNrFfRsxJjj92JNAe93rXwfavt27QAua5pmISCKLa7Y2bNkMxntBiPcb34dxXExZr0Ifr3WT2AcDi/tAm5+JrtfTB5R4bx7XoKHBUdD2HdgFmsngPBtZMwLazo07QDuwbxq0dmMpA2pbaLsNxU80HPTrc3GcH49XcDyLT24FLdUUB+R6BqHMQgnX2d3KemyUXU1ibg600hzGd+9//xtAe+Ob3wxavYHtMsoaUy5irFWroc9wlT52FV/8o299B7R7v/HNyOfk9CyUqRSwU4YVux9ehD7plDPPAa2/H/c5sTTu6eKduEdsJ0ceOQpaLIad7xvse0eZGybEfYxn4dxQTEPqiWgckBzAmCJw0PcFil0YUTQlbovH8X5aiJZM4v7i5eedDdqeUfSxGWUPuGcvxlW7du7F+ilrsa/sdeabYsh6uYBlEuiPxmz0F4ork1QSxVVp9P9uTIkT0mgTbSWO/VesYH/tmh4DrWDQ5zoJ9CVeGu1qsohnMMVi9LkWukfpG1wMWncn+pa5PbjOxoI9oMVnJ0Gr1dB/i43tqgfYLuMrc83HuCLwFHsxqPm+Zi84d2Pxpr2ih/4oUPa/QQPvXw+U9mPVxFbOaWxlv5tSnttOpibxPCORwL2262Kj9++fAC3moj/IdXZhOWXMm/fU5SKeK/z0pz8F7agj8Qxl/QnHg7Z7927Qdu3aBZp2NrJ4GNfxlNJPWqwuypnM5s2bQcvP5UE7+eT1oPUmMVa4//77I58fe+xxKKOdq2jzQDtddB1sg2Xj3PPrOA8ySTxfbSeWst676tkAXhsG2F+ei32j7Q+0foXuV8/qlAuVVxJ+HePl0n6MJ7qU80pP85GCY6ntLQILy7lx9PMpJc7wlH6SAO9nedhet2kd8pU10hG0eW38bcW+AyVWrCn7GVtZg/wy9lM7qfhoG416BbQwwHpbjjKnY3iuEBj0fZUGxk+NprN2T9AuqkWs24xyViwJjNniylm+tYD2uOseXPtS/QP4iCzWJW7juY+fz+NzFduo5bF+M7/EeLtfOVAInag953qwHrVpXJerPs6Do4ZwThUFz1zzsxifui7ODdvBZ7QCv7lKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gKY+fUZCH1MCNzRjQl7SxVM/loOMIGtoyRaXrpkMWhbntwK2nw5mnQ2k14KZZasAEl2b8HE72PjmHD3ZS87CbRyGZPQZ4cXgdY9vAy0PbNPgVapYeLcWLobtFzfEtCOz2I/TU1hEuNdux8FrVSJZsfOz2Ni5v6+ftA6DCYQH8lgAvr+HCYu9yxMulxvoD2llYTc7WT5EceB9qY/egdo5cADbfM2TL4cWlgukcM51DDYD7P5pnkVKgnYA0xQbSkzPBRMAl5YQDtwJjDJ+fjkJGi1GpYLq5gsPJ1Kg7Zj6z7Qdu7ZA5rlYt919/aAVq9h2+bn5yOfZ6anoYwJ0G/ZNs5RS9HSSUx43pnAtiYScdAqSoLzduJY2D5NEx/HVyy0AzHo+40o91MSpDdfm1CSvGezObyTrdxL8S1GqZtlsF2FObT5h6cPgPbko/eD1t3TBdrgIPr0waFR0BIJTMLe0zMEWt/AIGiWE21vaNC+/VDRDI5NECrjpXVxiGu6CfBao92vjaSU+esH2MCublzvbMX3VetV0CbH0M91ofmJ34j64eQQrsWzHo7bPY8+DNqrz30laKaK/mbP9m2gxZO4LtXqddCGB7FP4nFcdPIFjJ8SMfSHVoD9OTGH/jqIo60l01H/UClh3NGo4bp5+8MYY+4q43qY6cQ1qKMHbWfJGpzfvQMDoLUdJS5Xpr80LLS3sAN909BJLwPt8QmMSwsHsG/r8022FUN72b51E163gHPNODhOPR0YW3tduEZ3dKA97z8wD9psAbVKBeuiLZ1dHbhmZdK4tmlr5+Ag7jmOWXdc5HN5bg7K9C9bBVrv6rWg5fqwn5StmhSKOIZdKWxD2IX3ayeLBlKg5WI4SJ6PcaTl43jUGuivjBJr+Eq8NJ8ejnyeWYa251nKHHXQbmM22rzj4LW5XBa0gf4+0Lq7MW7pyKDdBnXFN4TYJ2evOgW0VUuwvU4Z/UUiiXG+n4raVV1wbXKVPUNfFse/28I1MYbbWNnn47UP3XUraBX7MPubdaU+YzMY0+6axj1+NYZ2Gyp+KRTUdu7dBdrm7Zsjn0cWj0KZTCfamaecjWT60c5sZd/l7Edtci/O+QqakBSLyp5Gmct2gP7CzWC/B0rfNRQfklRiUtuNGmUoWA9f2f8WZnFOzU3g+YQp5EHrzWIcmPVioBWVuLKdBMq+I6fELNPKWcDOnbtAO3LtGtCySt/ElT1+0Hy2oKwP4wdw7m3binG5dkb63//9fdAsZd3YsXMHaCevXw/a4mHcY/b04D4k5uF6v1Wpcz6P8cicEqMkk3i/hYVoDO+6uLeoK/MnVPaYRvFRYajMxwDn1UIB452FAu4v2knznl9ExPJwDXQN+iVbWSOMjYugpWkO+gPTdN5glFhE+25XqNTNcXFOJTpxLs/u3wtaRtmLJgTXcUc5KJ1VxjyXRK0jgb4vUO5XUuzKjWNdmuMWR7HbQIlPtXcUMQ/rESjny1Vlbx82lAWxXkKtjbhK+xLKPPAsjCHFUdqnvBJr1JR1Nob9XypG+9+zcWxrPj7Tt5S5p5RzlFAkk8D6zu9WYpvN6Fs7jsAYo7If97Xl7Rg/SA3nfGEH+vTiHlzXvEDpg6a5MbYdz2RmF9C+04OjoI14+MwDHSOgPTKP7XdtnMvau8pWOMx2AYQQQgghhBBCCCGEEEIIIYQQcnjCl6uEEEIIIYQQQgghhBBCCCGEENICfLlKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSApgN9xkozOwHLelhoulaFZPVWiE+Rku43tuNSYe32JiEfXI2mlR5xsEEvh2ZQdDWrsMk2Dt2YxLsRgCS5BfKoK1atQq1ZStA260kCX7yycdBm5nGBMixeAa0rkwWtH1PPgXa/hl8rmVHExE7iRyUGVq8DLQRzBEtS7OYJDlhY/LnWhXHJwwx0XrDx2vbySVveQtoXYOLQXv0iX2g1euYfboeYj8EggniTagkdZfoAFhakvMA72+UckruehGlXMPH+03PTIDm+5gE2sZLpTPXCVq9jonBZ2eUpOkO9tP0dBW0WgPr4lei5QIlebsTQx+VSmDS7riS3NrxsW71qpJ9XNCxJNM4h9qJZXDglPzwYiwsZxlsn2XQrsRSbqhoXiza/5ViAcocOIDr0vh+1Obnca2KKTaVTaMPTidwjFIu2kYQYPvH9qNv2LoL17Rq9WegNQKsX2/vMGhHH30kaKtWLol87uvrhzK5jl7Q4klcD4woNqr4Ml9ZN8XC+VIXZfzbSLID2xwY7HvbxjVrfPdO0OppbF/oojaxB21j8ehA9F4V9HHdi3AsN977CGjpO+4E7fh1GLNUK0XQYimMO3oHMe6olzHG0Hy6FtuFypwfHz8AWlBXFqw6Xus33S8IFX8bx3m7d3ISNLsH58bs9Bw+M58H7YSzTgdtsHcAtLYTKGuUMonLvWhvp1z2h6Al1p8C2m03fhu04o5p0MKm2M9L4jgVlb5uFBdAi6fQTlOJNGi9A4tAc+JYbmIGn1usYIyihF/SlcM9R02Jcxcm0O4zylp0/NnngBZresa+/WNQxutU6mFwDtkNtImwinufULGdvQfQn01UcHzaSaoT17KODuznbDYJ2kweY8RagPFNvYLxZUPpV8uKrie+hfdH7y9ia+NWzoPmOrhedfegH87lcP0bHMT9czqJ9+vtQ7uansH5vWx0BLST1q0BrbwV98VOBz5jvxed41t37oYyQQPnWajEbQvKGit13EdUShh/Dg2ib2wklZi3jUzlcd3aPoFzdSHAfqgrf39vFJtPKjHyvItxxZ2/uivyeWgI49nTO7tASyj3t5SzEbcT69vh4ZhbrnK/GK4bCwvYhlBZIztSuL9o3rOLiJSKeD/XxXnvaJv0pjOzwKBPKS7MgLb3qSdAa8zNgtYdx/ktnd1YDWWhC7XFr41klTOy/j70aWPjeJ7hKbZx1BG4x7KVMers7MTn9kdjv+07fgVl5ubyoG3ctBG0U089GbTJKYxfN23cBNoTTzwJ2t133gHa8pFR0Pr7MH6NxXCNvPueu0BzHLTvBx54ALT1608A7ZxzovFOsYhx139//79Be3wjttWE6LdC9bwDx7VQQN8/tg/jrHbiKAc1JsQ10LLQH3ouzn1f2aYHSh8q3SVO09mKUc7oLSXeUY5QJLRwL54ZwnPYmhID5afwLCinjLlRzvUaPrbVCdHndjjYx3YS14O4tg4pe5VqI9rx2lrQsLCjanVcvytKLBqPK2uV8ox6FdcqT2l/OzEWttlXzm5cG/vetdF/lcrY5kaAPifp4bglmrYRlRLum+xAiUmVcDFQYgzllYkY7fuRAc6rA4+gHeQmsa229g5hCtfDcgGvTZYwZo4pdjqu9IFTjfpXU8HrGkpHZT0c/5TgnnOwge/HGnPKmfNS9CsxD88EWoHfXCWEEEIIIYQQQgghhBBCCCGEkBbgy1VCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAcx8+wzs2LYDtKWrjgAtYWPi3LCOiW7dBCbJTShaNpsBLZPLRT6vXbsGytz6kx+BVp4/AFqqux+0bfswQfySxUtBW7YGE7DHY9ily5fitfnZOdA2btoKWmgw+e++PCaVXlASAFcD7M+FfDQ5c//gEiizewYTMXcv6QBtRkmMLSGOf15JzmxcrFstrOH92sjDjzwA2mOPPwKaJZgY23Ew+7TrKYnElX4QwWubE8S7Mfy7CG3+eB7eK6aMmx1T2mDw2lysC6+N4xxtOJo9oi0rOeMllkqB1iijbZRLmLi67mM5q9E0X7Tk7QFWJFASkpcKeP+UMuf7OrBP3BSOT0xJUt5WLCXhupJI3PhoQ77BZOih8uc7loPJ5U2I1zoS1R596EEoU5ybAq0niwnI9+5Hn57rQJ/mKfMx9HH9ymUwGbrjOaDFXLRlL471c+wSaDP5edB27XoStPn8XtAeeiBqWLEYtmvJkuWgDQ/hWjU0jGvE8ACWS2fQN1hJNADLVtaNNpLM4BgVqui/dm7eBlppbhq0dKoPtAaahhQrOOaOFx2nHbv2QJmF2QJoi45eCdqPfnYXaIUa+syTjz4atFoVY4yU6r/Q983n86DVK1XQkin0kbaH61A8ib4h6eBz62F0zGrNfl9EasoatGT5CtCKLjrmeRv9YNcAjrUo6+tEdQbLtZnAYHtqDbT7xRteCdpJV7wTtPv37AMt1zcEmpfGeWRMdKwadWW9L6Ddi8ExbtSKoG3dgc9csvJI0Ow4rk1VH2PaumJbyQT69VIB6/KTH/03aI89/hBofYptnf/KV4O2Ys26yGd3YBjKFPK43yjXML6pVVGrY/OlvIDtuvuOX4C2dxxtop1kV+K6VYvhfJ3bj359TrG/+eldoKWVtTaVRtvwktHnuort2Tb6OUeJXzEaETGKGleeMTmJMdQdd9wD2ujIAGhuDNulrRPzm+4H7bt3PwHaniIaW1FZT2bmo/a3UMa1NAxwjhrl78lzHZ2gdWdwHVqRRH957AUngeZ2Yx+3k53T+0GbrqIt+zG0F6PE/qJolo3XDgyj/9q/dyLy+c5f3g5lVoxiLDOo+DRLiQHKRVy/ChVc8yWVA6ncwHKNKvoBMdjWeh3H3Cpi3FJW7LSntwe0DmVvIk3d7lgYVBbnMb4b34sxZH8S69uRyuIzFYIA2+Ur+7d20tmFfaqdyUxPo+/LKP0wNYV20N2JNhSLYb82a81nOb/W0C89+CDud89/5StA6+vtBe0xxZY1P6+dQ5YKaEO1Gt5vfB+ep2pm0KH417/73N+Btu6oo0C74IILIp9nZnAczjnnHNAOTOF+f59SX9vDuaydd9jKmhuE6GvaibI9EUuZq2JwjVU8pASKTw8U32fZaM/NVVEuE+UysZQ2GAsvrijnNB2LRkArNJQYt4z23TA4vl0dOL8TSifHQtzbZjy8tqsT56lJY0w1NR89byqX8Bwoqey7a8p4NZRzzeZzNRGRdBzvV60r8aN9eMU2tSparqu0JbCVs8mKEh8q55WxhHKWb9D+Gia6Z20o7z20VdJRzuh9H0vaSuxaC7GtnjKHSmWc8w88hXvigSzGHT3KmlaxsC5TSufFFJufKWO/F5reVfkVXKtqgmcCttJ3pSq2NRHMgtYd4rluNoNnnZp/awV+c5UQQgghhBBCCCGEEEIIIYQQQlqAL1cJIYQQQgghhBBCCCGEEEIIIaQF+HKVEEIIIYQQQgghhBBCCCGEEEJagC9XCSGEEEIIIYQQQgghhBBCCCGkBTAb7jPwyDZMEL503cmghVICzfKVdNlK0vCFQgG0fB4TmPd0Hxf5/KoLMKH5cceuBe2b3/4O1s3CZLUdHV2gLRpeDFom1wma42P7uwexm4eWYVLf+SQmSX7o0UdB21/ERNNGSaDdMdgDWu+KaMJiR0kMriUt32zSoG07gAmbY46SfLyKCb9Likn44XNLHPxiceftPwWtvJAHLealQEumssodMfmyY9A2jPI3D7YX7Rs3jv2ciONYJhJx0GIJrK+bQltJxDC5ddxWknsrf6JhJZTE90q2+kYNk09XK2gvjQaWCy0lPbjyDFeaNC1BdRzb1ZlGrSON45VJYsLvuId18yyc81aASbrbScPH+tTrSr197AfbwkmtJXA3guUUtyHFYnQ9qFawbmtWHwHaCcedCNqDjz0B2n0P3A9avlgGLfDR9vqHhkE744wzQHOV+bdr926sy333grbuiCNBy3XgnJw4cAC1iYnIZ23+DA4MgbZs2ShoQYCjWCrMg2aa55mIeC6uG1XFntpJ3MUx2j+1F7RdTz0F2jEnrQPNcdG/FJQ+zCpjWa1Ex6mnuxvK7NmL9jO0egS0ZevRfrbt2gfa8tGloK0YwftVixjb+AGOef/gItDG92Gd5xYw3ospHsMP0V7mZvHaeCo6jiZEP2N8JWZR1qrS/Axoi5dhP40cuQK0sbk9oBWrOP/aTdnHdocpjH2TI6tBu+WXD4F2YH4OtM4utN+44hOtJjs6oNhLtYb2F4tr8U0GtJQS03sxvNZ2cO7W6zh2vrKn8ZSw4vvf+y5o//Fv/wKaUdZOSwmsnnzscdCuetefRT6vXoM+yRKs3OzMLGiV0gJojRL6+jtvvQW0R++7G7RuF/1DOzEexnSBUcbcx3ofGBsH7Rc/uhk0T9k/uZ4SLzXNg0wO9wydHZ2gdStrwuDgAGipNK69j9z/AGiF+SJoGWVeLXQkQQuKuE72LEI/GZ9D3zCzdSNos24vPqOBcV/SifZntgevc7Q+t3FOWYrmKuuQq8zRuhIvuwZtrJ3szk+BVtVOfpQYXNnGiFjKxYrvc5PYr0NL+iOfd42hn9+yHe2iuwvPNzTfX1POkHZt3Q7axCyeZ4WBdp6Be7tCAfcIDSWusBQbmp9HX3rMsceA1tWJ61Wiad+ecHEf7wieARSLFdDiAcZUiwf7QXMdHOuYEt9ayh64nSST6Ku0vveVcRMLJ8LMDJ5DVhbj/slXbKhcjtrLipUYM576stNBu/vO20B78MEHQUunMd5RmiCOo32XBte5krIHrlbRXoyytGv+tVjE9aVQyONzS/jck0+OnjE//sSTUGbxItxvdPfgGrlvH+6TleZLEOK8tRTnGISK7bQRWzlT95TzMMvgOub72OaqsrezHFzbbOU8zW/qWBNg/zlxvM5WYjEtgqwqC07o4ZxPdqIdBCH6w/lpXDc6lXgsmUA/Z5QYwDFa/IB9UKigzftN+41KWXmnYrANojwzm0HfUKviM12DtpxR9mmNMl7bTuKCcZ9fQ9vQzvDiNsYU1QauEY6NbY7ZaBuWRPs6VM4tbBtt1LLRLmzFVhwbZ0Jdic+MEi8XF/A8w7JwzBd8PHsv1NDWihW0STuL51mZJO5BwoUx0CqxaL/Uc7hGNpRzFW1vsD+PdZsr5UEbXo7tWnYc7qOC8Ll9B5XfXCWEEEIIIYQQQgghhBBCCCGEkBbgy1VCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAcxW/wxsmcdEvNMBJvU1HibEteuYJNiESjJrJTH28FA/aGeedkLkc8LDxLyjI5jk/NWXvAm0m77zQ9CmD2B9988rCb+r20CLCSYLn62gtm23kly9riSN710DWld/CrRQSfttWZh8PExErw0tTNDdUBKZzwd4r4SH1yZcTMRcsjAhdMPD+xklAXQ7GejHBM37K1OgBUEetFxPD2iuMh4LU7OgFRYwIXMjiCbkDpUE3RLiPFCxsR5eEueZ8bD9voUuw3bxbzRSMfQX6RTabdDAuSGK/UkCn2HF0NYSMaxfsikxe3cGk2wvyaAvWzyEydJTCaxarVoAzTboB10H69uZw35qJ8Zg3xvFt2iSlpjdUf58J7SUMcdLJdlkL2eefZ5yGT7AddAGVh93Mmjr1p8EmpIzXmylcr3K/F6+HJOwuwn0kaOrjgFteCn6+WQSbaOjA+ekNmazs9EE9kGA61d/3yBoWSUpveMqc15J8h6EmFy+odhEaCmd3Ebm8wugFefzoGVTOJZWiLYcj2P7urvQceyfroBWqkf7cHTFUijT0dcF2vat20FbO4L2aLvog+sG15JyFdegXArXjYKPfq7eQC2V6wRtOj8JWmVuDp+r2GTKQ/uzrej615XG+VMIiqClSxifdMbjoHUM4Bo5VcN4oOjjeiAGbafd1H203YQS89z18COg/fe/fQO0Y084DrSVxx4LWlzpW78StftyCeek6yp7hhja87r1p4A2unItaMkkXus4+Axf6SdPiV+nJsdBu/kH3wUtodhudy/64kod7XLH1qdA+95N/zfy+X+9/s1QplBAXzOTnwZNApy799z2U9Ae++XdoMUVP5JMZ/AZbcSzMEZ2HNS6uzAerFawDzuUOKBRxHLFAvoEqxbtr+2bcT+pxQCa7Tla7OWhVlhA/xeEWG5yAvenb0meC9ryLPq1Qn4GtJrBZ8SV2CgmOP8sT9nvhtE1NjQY34iiBQ3U/AaOfxjgXrQygr5xqoExj+RxDrWT2QbaXsPCftDiSMdS1lkbNeVSsZRrY8mmMXex7/fs2w3aKSdi/G4p4xsqZyiFfB60J7c9CVqxhP2UVPaTQYiNrSrPjceUdU4pl78b/fDQYB9oR6w5OvI5VHzZ4PAQaKNrcG8xtht9zZxi8/0Z9D/1Bvp57fypnZTLOAc1/51T9lOBj/26e/ce0LJK3xSV5040+dKMEguvW7cOtAcfuA+0pzbh+t/VjfuBlLJuhMok1eZQrYbj22igbSjTWxzFN2jzxVYOBqamMJbesmVr5PNpLzsVysRiuI7MzOM+4vHHN4KmnW24SgzYp6zzmezhdXZjKf3saH5e2bMaZd0NfbxWOwup15X1rqkPQ2X9D2zlvFKx0VAxNO0MIVTOV72EEn96uBePeegbCgXlXCCLcaGTxvlXqOJ88TyMFQoB9kGpyYeEIfavUXywp8wpTzlzXKjgvkJq2NaUMq+MshdqJzVlrx2L4/g2lHOKWoB9qLgvKReVWDDAGDfV9G5FGjinSso+wK+jXdiOcg6rhLi+4r887TuTAdpBuYp2UFjIg1av457BVvYgaxePgtZQ3klMlfBsybejvtRNae8eFoNWMfiubsrCdcQ9CmOxFSuWgNbZge8G6sr+oBX4zVVCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFnBbLbg5j+9hv3fX46AdN9IL2mAMk8SmPHz00OAgar050FYsb0psazAx8YEpTDj8lf/7Q9AefASTnNeqeD81j7PBPjFKkuQgjm0IbEwI7AomSPctTIrcnPxXRCShjaSSRLxaj9bZ2FjGdTEhtBMqidGr2Cm+YDkvxH5ylCTl9QbWpZ2YBiZe7khjku9CFRNSNwJMXL127Tp8xlA3aJPTaLuT09EkzcV8HsqUy1iPIFCS1weY3DvtYgLptceuAG18Ads1pSTBrtSw7yoVTBqP1i0Sj2Hy6bSSQLszjfOgv7MTtMHhqF9ZuWgAr4tjTYolTPI+O4vJsp0Y2nJKSXKfyWJ9e3qwXDtRx0gZc9dgf9UVP+wLJgP3ffQRQYDlwiafYzB3u/iKfVtKVvq6klh9eOkyvGGIPshSNFvxrTv34Lyt1LHSluJzsx1Yl+b2i4jMzTdAc12cG+ncaFRQ6js7j2M9PjGr1APbELfRD8ZQEiuDC1N1Dv1POykr8zwVxz497eXngLb2iOWg7Z3ZDtq+BZwvla3Y/5Vy1G8WGmjffZke0GbCadA2PfkUaGcddSxovRmMTwozaMu5blyrLB/XnPky+gGx0A5sNG9Jp7OgpRLoNyvKmMXjUQMMLbSzclwZ6zJWZPnQItBmXLzf3Dz2u5fE9cuvoP9pN4GgbVVD9C979u0CzbXRJywUcJ2IKU6hS1mjt4xHn9Hw0YbiqQxoqe4+0HKdOD9KpTJo3d1Yrr+/HzQN18H5vOXJh0Gbn8d51JnD+TY3h+UCxf/nsriXeuLhByOfV68+AsoMLkY/pY3Njs2bQduy8QnQ4srk7ctiu9JJ3Eu0E9ugHUzM7gDNquEc7sxgW7q6MH4rKL7OWLie5NLRsczPoW/S/v45CHDuFYsYb9uC5Wxb2SgqtpxfwH668eZ7QetS/jzbdOBeYmQQ+2mugRfnS3OgNQK0tXoj6k8DJThsNHB9FWVOeUo8duTaVaCNHI37oel6HjRf6fd2UnNwjTZKP4iF/aCuWooY+ih6SlzabGmJOPqghQW0ZaPsN3wlLteOafqGhkDrmd+N185ocSm2q0PZx3VY6Bt8xQycAPtkYvcEaD//6S14cT06d4dGRqFIoYA+pGCwj2fDFGgP7cK97Tpli2TVcW2OKebUTkrFImhG8RGOgzZUKqPv2zc+Dlq1rsSDc7h/mmmKpWtK/43t3QWa30Dbs5R915IluLanM52gLSygbTgu+mAvpezZGthW5QhTtPmiLTmhjfPZVdq2Z8+eyOdVy1dCGcvBB/T2YhynuDexlHo4Lt5vYDH6kMFB1NqJEpJLUMNxsxxlHih24DqK5mF/aVPfh/UF76Wd9mpnDaGyJ1GOxcUo64HrYbwcKOfbnjLmWpw1U8SYIsygzac95cy7gvM+X0M/1Qii7TXKXkj7VpxR4qTQx76zjbJWK37QUcppMWU7qdUxVqj7ip+zce8Yd7HNtRr2tVF8RKk6D1rQ1NemjGtsUFMiFAvHzXbQuYYu3s9y0JY1P+o6eG2ptB+0wgyuc75yNpvtwfd8XhzjojnlXUYpwDo3H+Oa4jYoY2ycy6Vh9PM9Z+DZTaIX22BcJaasKOfaMZxDrcBvrhJCCCGEEEIIIYQQQgghhBBCSAvw5SohhBBCCCGEEEIIIYQQQgghhLQAX64SQgghhBBCCCGEEEIIIYQQQkgLtJxztaj8jvOtD20Bbct2zF9z4fojQVsxjHlZdu7YCtpZJ2GeykRT/sVCHX8n+Zs33w/aQxvx96TLPubWEeU32W3lN9S134e3LSXHpZLTJVByANaU3KQN5feuLQt/A7om+BvdWo4Jt+m39R3ld/VTKRzruJY7Ufmx/UDJNxQovwXvK7nkYtlOvGEbmRkbAy1Q8vhUlN+hLzflihAR6VZ+R70vgb8F7yn5SpNOtA8rSt4EY7SMM1rWHKW+Fcwdd9ZJR4F21BFHg7ZnD+avmclj7iTt9+xFmUOu8hv3SSWhRG9CyYuVVnIsNPXBgWkcm83T+PvzVgLnQa4f87Qlc0qeQCU3WncvXptR8lO1kzvuuA20ef8x0NLK7/8HNcxV01ByOzW0vNQB+rRm/9VQ8kdoflTLmVKtYTktp4al5HbyXLSz7k7MOZBR8tw0AmXdUNJWWEpCGE2zlXyylpK/2m5ar10XbdlWrtPupeW6VdJEiGUpa1VKqVsVczu1k+5BzCU6tGo1aMetHgGtqxfnb64b/XwM3au4GRzfmYmoDw9DzGO5Zzf6qs4U1sPrwxz2kxW83xLFZzpKwrBAyS3u15U834LxU0zJixRTbK2iJLcf6lfaMQmSFEvRtuWVtlaV3MOVPD5zqrIPNNOLubqtOvqkeBrXdDv+3PJ2vJgUlb+trBUw9gj70BaWL10Kmq/4K6PEGskk5mZpzrntKHnXO3owv2rX4Cg+UxvjErZr8eLFoGn+tazkYNN888QE5s5zlVzx6awSL2RwDhaVPLELSk63QiGqbXsKc6QOLR0FzbJwbPbs2gWaX8F6dCqxV0LZS4gS+7eTNZlhFJU4INWJ/rQ6jv2QOAr3pxufwn1xZxfaS9xr8ombN0EZX/GHeqyg5PhSNC1ZWUcHrn+xOMZ3M9N50KYsJR/YJMb+cSVv2iWXvB60f/rnG0Dbtw/Xu7DJdyXSWN/VSt7UlJID+KmNT4I2M4u5omZnMb+Th65M3MPM1YfK3slRMt5Zyly1lPOHmI2NjsfQf6UUPx9PRPs/abBMJqHkFgNFpFbFfIJBc/IuEenvw3WjbzfafN3GeEHLKJhS8quKwXI1xW9qwXR/BvMR796xF7RvTvwg8tnrXAJl9k9goFlXYjRHiccsg/25ew+uaUevxLV/eX8naO2kpuSa1Nb2/FwetIKSOz6j7OfrSu7U+QLer9kmjbJnVaom9QY6kl3KWcspp54K2qIlGNvMP4F+Lq34za7OTtCMln9yBv28ltdUiwBsZY++aBjr3HzIWFZyXvYP45q+ZiXu3zozGJeXK2gnvtLvTz6xEbR9e3CP0E4sxQcZ1XOisfnK+NpKLnY1R61y5ttcLlTOwLWzZ1uzFqPkDdVy+wZaPdBX15TXHkbxh6HSnw2l/Qt1pY8b2tkSrgeBEo81mvyFq6zfnpLIuKLkZhVfybeunLVZAc6DtLIHsx3lGW0kk8B1vKjsYQtV9FULPq5tjSqO+dAwrrNVpQ9rJhozFqZwLU7Y2KfJFMZADR9tKpFE/6XlSi6U8bkT09gntRLuJYtVLOcofmC4H89Cxvbjufq+CXyG/toxauPlAsbftoX5iUtl9P2NOu7dlig5uGdn8BkS4Dmpr5zxtAK/uUoIIYQQQgghhBBCCCGEEEIIIS3Al6uEEEIIIYQQQgghhBBCCCGEENICfLlKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSAlpmWZWe3j7QZucw0fJ+JUH8PY8+BVrQGFGegslk+wYxybnlRJMC/+qBJ6DMD39+L2i1EJO3i6skbdayyysENUzubELsk1BJYG8MlgsMJjH2lMTvlpJgWBzsO1cp5zQl7s5mMUmyq7TfVpKKi8FyoXhYLlCSRA9i0uFsDrV2MjjcBdq+3ZjwOaj5oImF2s7Nm0Gbj6VB06yvFEZtreTjeISBUg9REtVbaGf1WgG0h+7+CWhnp9Fe1in2UunIYv18nAeWj3Wu1jG5+nyAidknZzBx9+6nMEn5dGUhen8P25/sx8ToXYOdoMVz6EOcJM69VEcOr03hWFtOyy74JSHhYfsajtLmEOsdj2ObQwvLBSH6A9vGMTFNthuGaCuWhbZnDNpZqPgvS5RnKj7YstCPKi5dbMH1wHWwzrUa2rKlrTlYFfF9nM+NBj7DcaL3s21sg6X4gVbXvnoR/YVR6lFVlqq4oySSbyOVMvqbfcUx0OoN9C0jy5aBtnigF7Q1w2tAc2ycG8nYbORzrYaGVitgfRfm0b6PWb0atEQK1+f8JI5Hn5sAbd8U+tuxGbzWeOjnlg8OgJZNJUHTYptKXZlXNvrcYpNN+g3sk4FMP2gbS1tBe2LnDtCWj+CalophfzYqOD57d+8Brd1MBjjX6zX0zSXFBk1Ci3Px2moV+0IJkaVWj46Vm0Qb6ugdAm3JyHLQert6QLOUOCiZQBvfv38cNC2mjydw36Cta5aL9uEosXpHZydofjiFWgXXjnIhGt/s2on2vHL/btCKJRybsb1op/Ualmso+5eyj+UkhnO8nazJoR+qVrFPtTXBrc+BNjywFLQncbsry1aMgpaOR23j3jvugDKBsneyFR9pK8GCFrd4MbRb10U/sHw5tksE9+KPbXwUtIQSQ9VK2MdPbEQ7LZXLynPR1kwY9Rfa/jQTx/YHylpiGnj/h371CGiFaRz///2n7wQtnT68Ynoxir0E2OaUhX4pncbYPx3DOD8Zx3JaLJlo8pupbvTB/297Z9Yjx3Vf8Vt7d/Uy3bOSHA43UaRIUXbkSLAtO4ZhI3lwkAcDec9DkO8QBAGSh3yOfIA8xDECy4hjO1AUxJZkydq5jTgkZ8jZe3p6ra41z9XnCG74wT1Azu9tDqq66t773+4tEv/Awz1mQGx0MEFbKTK0g/EAfbmaYVw+H2JsiBO0FwtN2WRkPx4k6AeZQX922HgvYNx8vFse7+7hNlxTkP2Qw86L2FEB2dP1HvZAOzj+Aq+7fYH84PxgtsfmYUTiTU7yfa+H83AcHMz0LmFY9o2UnHkUJM4vtfFMYvc51icHu7ugXTyH6/HowSZoDjltKsg5TXsBz+a6R6egse1jSuqigtTmL1zCnNOqlmNNldRdS+0WaMN+F7QayX0pqdVzdgZA4mWvg8+YJ/FkDJrvY3xNyBlCZpFzQo+ce5DanVGpTtk8OTBxyPmG7ZIzcIecM5MzHo/EPj/FOJ820a9ysr40Vtfx3voS7kssF3Op5+G7VD20ydNR+QygiHFd84LEEBvfNyaxhu5dxviMQYSaY89oAH8gJhbGZd/HPDbuYNLOyeHU4jKe3eQGYwRxDVOple9Nif0kKZ4hVUOMrQU7X7MwZk5I3jjpYF6aRAPQcrKvG5OzyfNt/A7yl2+8Ctr9LdxjPtp8ANrpEMfhTJ8xE/9Oc3zf4y3MaYeP8ewgXMFclcQYG+su1mJZTgq+GdD/XBVCCCGEEEIIIYQQQgghhBBCiBnQx1UhhBBCCCGEEEIIIYQQQgghhJgBfVwVQgghhBBCCCGEEEIIIYQQQogZ0MdVIYQQQgghhBBCCCGEEEIIIYSYAez8+2UXkmbwHmnGnEbYyHlrH5sOT4Z3QfvO126AVm1hs+jTqNzE9613fgPXjEnD5yTFRrpBgE1tc9KAnTW+ZzgWTqlFGncb0hc6cMi9NlkiollBCFq1WgXNdcv3JgnOU384BC0jncwnKc7TQhsbQp87j1q9QppO97GJ8zy59OIl0HoDtOXhzhG5Gxc9yrBJc4fMoU9sKJ6y54w0fKZGxd6smO26hx+9C9p2H31oxUY7K8gzMhv/LceANGHfK7Bx9eYEbXInxUbToxDnrrFxofT32tXLcE2l1QSN+ZlxcAz1OjbBDpsN/DkSLwsaHOZHTuZ0MDwBLXTYWPD3MvLvd5IUY06cYHxN0yk7sPG+okB7TJIYtDzFtUwzYqMp+pVl4RhyYt/EvE1R4HxOojE+l8QG9oyCxOHCsFiQT12D91nE9pg1svdwYpz3lOTXYQv94PwGavPkeA/jd0rs4PN7T0G7uv8MtDe++Tpoyy2MEZeXL4Lm2OU6a7t7ANds3FoF7WAHfXRz8z3QWu1zoDXJ+vbRRM3Tpzug3X+yDdrqEr7fcoh14UprCbQ2icPbuzjvzRBzTmuxVfp7OMSa6LDXAa0zHIB2ekpqEeIvY2Ine482QasSv503PZcELJLf3AhjWFLHcReWB9poiLm8vojrfuHazdLfzeVFuObGS7dAu3njZdAunlsDjQ01CDGHBT6OochJVCTJrlYlOd9Cu2c58fz6Omgra+irn3/8MWijqGy/e3vP4ZoHn+F9gxE6+SGJZzHJ10P273I93CMa/2z9+90xGUt/jPNw0sV4asjw7n6K8/rowT3QPBftoLVQjnW09sBHGpvVjKRGYTU4u7XRwNw0GmFMvHr1CmjLz9GX97bRhu51cd/0yYOHoNnknKEgtjZdu0RDrPne/d/3QcsyrFEWSM753ve/DdpXvnoHtLXLG6CRbfxc8TLMlVUH41w7JPsYj5yPTHDPFg/Rr2xik1lUtslojLnFbaCvTCLcH9ikph2PcJ+4+xzjYTLB5wbER5OI7TnQiQIH58kqcN5zF+8l2xCzf4jj6E6d1WQ+uZE4eG7Q5gtyfmDbOP4ixZz25BB9ufvO5/guc4TtbXwfx8euGwww9jmkLuoOsUZk8dX1yr4W+Fh3hOSs7toGxpYP3sEzmY9/+1vQXr59G7Tl9gJo3Q7ufY5i9I0L61hTVTycEz/AOR4RHx/H6FcWyX/puOz3e89xD/Kd730XtH//8Y9A6x5h7c/OSNn6sL3thMSQecLyPdXI+XaRs8MLlKwC781yzL0mL8+Xw85QyJz6PibPPEWncnKsE2o++T0XB7FYxXPdToh1zDhHG3VCrBUqLfSN9tIF0JaWcE/T6+zju2zulv5OJiQeGczfSYbvm5F6NyPny5mH40oSrItteu48P7rHmIvqIca5gH0fIXWfs4D23R9jjHTImfdSs7yHPTF4btEl5w/ObNtL49jk/IGeJZJ8n+O6ZUQzNvrVaUxq6w8/AO2NGy+Aln7tRdDubu+B1o/L8eE0wr2vs4BxeeU8njXV13AMdeLfeYC1TTLAOs620CZm4WztfIUQQgghhBBCCCGEEEIIIYQQ4oyij6tCCCGEEEIIIYQQQgghhBBCCDED+rgqhBBCCCGEEEIIIYQQQgghhBAzoI+rQgghhBBCCCGEEEIIIYQQQggxA9jZ+EvISbNxU5Am1Q42hI0NNpjdH2Az8A/uPwftByNsSN0vyo3kn51gY/lKvQ5aOsL3iEhT8jDEZsWuh1PF7rVIQ2DbQs1z8fcKm2jk+7cX4BwPElyfOB2CNt3AnTU8n6TYwHcYYVPjemsZtPYKNiKOU7z33r17oHmswfIcabaxAfnKOWygvLuDDa9JP2rD2iJPSIPwhDSSz0w29Te5aEboneSFkzE2yx4eHoBmV9qgOaTR9nODY/3QoA9tusT+GtjAvXaRrM8F0kh+pdxwPqhhc/OYzEpR4HsELvqywzSHXYf+bTszh+A/CNvbn4H2cBf9t+b7oLlFClpGPQHHzJqrTzdm9wOSb3Js3p6Qhu5EMsbCd3NsfIZlo22w5vLGwnsdsr55jnY1idEP8gyvs8hzbQufYZny+uQkzjP7JpfRFUwMmeN2DbT1V26DtoCXzZXRGO27WcEY8eAxxr4nW3ugDXqYd19/A+dhsY1x89zypdLfteoCXPP05DFo+UWc1EEF36M33AYtrWA90c9x1ccrDdBcdwO0k8EAn4HhkBpb76QL2tLaGmjjwSk+97Ss2S7GqGfHmKvf33wE2sofXQPNJ/6982AHtHqIz/ULFoDmTIXkHgvXxBviejYvYj7ukzWOeyegdTrHeOFU/B8P0Hbv370L2t5TtOd6Fet3z8X39aq4TjaJdmzvw647PcL4kCcYW3wf5/3BgweguWQvcXCA8WaSlHNHv4++8e7//DfeF0eojUf4Hg7afVTg+AuyH2RjmCf9Cb53L0KbH+doL9VqE7SVOs7h1164CNrz3ceg3f+svE5ZNtv+JyE2xfI2i39sD9Ige+WtxxgTRxHaxkKrBdrxwSFoYQtzzLk13Cue9nug7e2izY+j6XkndRs5d8hIvbixgXuGv/6bvwKtUg1AS0lhmWIpN1daDu7TGw2Sy0l+m8RkfCRuTMfvL2N6XxRFOFlJjPY9JDEtDLFG6xzug/bF5n3QKuTe1sISaLUA44Bjox2EVZzPSYz7oYjUAb199JftLubcUT5lzyS0FmSPzTb8NonLOYs/pA4sLBz/MYmh82RMzi5SdoZJsMkekJ3XTUbk98ieMpta85TEqmiEsfWrd14G7Wdvvgnao4dYO1y/ehm0q5ewVv/wCP3FynAtj/ewbq5Xcc9Bhk9jf6fbBW3nEeacqxevlP6OybnS7nOswROyn7569QporQXcX9VDzId+gLk0I/v4ucImn5CmGJdsF32a/c8r22H1MTn3mUqCjoPzZ0i8ccnZkE3O0rIcbdQq0DYsUqdXKiuo2bjm7BlhE/fsXoh1YX0Fawqf2FV2jLHfmrIrUn6beELOvRJyJsPWxkG7TRPUPHKeZ5Pz6nmSjvG994b4HakZoJ8XDtpLJWmB5hZ45m9itNPOQbd8SY41i7ExLvVOMPY7DtYdvodxlPlekpCaitiGYeeaxL+jBJ/x0w8+Ae30BOf9G3dugvb1298CLZ7ab3VTfI8nDo5rXMcxtK6hn3UnuK+wEqwBg4Kca7uzxdVp9D9XhRBCCCGEEEIIIYQQQgghhBBiBvRxVQghhBBCCCGEEEIIIYQQQgghZkAfV4UQQgghhBBCCCGEEEIIIYQQYgb0cVUIIYQQQgghhBBCCCGEEEIIIWYAu/J+GaS5symwmTBrxJsXpCG1jddtHfRB++d/wQbu3/vua+X7nmNT6GGG341z8i3Zq2ADW8dHLSRdpf0qNhge94egJaQhcJGSBtIVXA7Hxbljv+eQpt85WbPxaPA7r2G/1Wovgra8dh60w+MOaN2jPdSePATt+rWroM2TaqUGWhBg43fPR9vISINw4kEmtZiK98LN7LYZIb9ujIVNmwfENu7F2Hx7wa/iddE+aJ+l6BvHTWwqvXTpGmjnr2CD+NZ5tMmghs2s7bw8toTFLZfEAQ/X2iWxwbJx7rIMG79bZI5t62z9+xa7IPZNrrNyjFVFgfZisfGRRupWgTHNm4p9jkWeSYzZYe9mYUxjNl8UuG7MYTIiuiRWZ2T8MRlr7qBdFWSe2HgL0oTeTI3DymZbm8JFLSVa88IaaBdfuQGaa6E9dR98Ato8qYaYx00ag2RnOM/7e8eg/fzHb4PWXEDbePGV66CFbrP098XGClwT2GgE9/Md0CxMz8afEJua4FiTSgLa2vIqaKspPmTY6YHWJ8+oF1jvjeIINLeKvlEjefhkKv5s7TyCa+49xrrDhJi/Vtc3QPvorV+D9t3XXgft9T/5Jmhv//Jn+Nw5Y5OcV/FwLoYk1u0+3QRtTNbk2fY90Pb2sTYYnpZtoSD1Nit5HBtjPc2oJNZZDrmXXUfymiGabTCuJ/EEtBcureMzSG47PMLYsn4B/e3u3YPS3zmJXacn+FsFmVGb1KMFq1FJzskdUvOSHDtPcoPvQ+2AaGEVa9UL67gei4tt0C53cF90cFBek+4V/K3xeAxaHOP6DodYW48SXLck/921gjHG3Lr1MmidU4zXO7u4t4sz9IPvf/tboF0mfnDSQTsdjdCH3n3v/dLfT7a24ZrC4LgCkkvu3HkJtCzGee+NcY5dG/O6e8Zq+mawAFqOJmRiYhpxTOIBLq8piF1ZpObOpkKEVaCftRbQz7Ic65HeEO3x2R7a4z6xqWuLLdCsgKwlqelrNTwXYLHBiTEeHh90QXvw9Ai00zGZ5KlcV8TEzkgesUkMZmtDdkimsHHeJxZqnsHxz5M0xflje/I8p6cheG9C9gPkupw8I5vae00yjC2//MUvQPv7v/070P7iz38A2ps/wTPSx5tfgPbGG2+A9vzJFmhsw7tEcppLzgnb7SXQshznZDxC360FaEPL7fL+Jxrh+dPR4QFoEam7Tvv4TIvUmYMI12ehiTGUnefME/Y27B2ZzbssHpDcRo66jMPOKdJyjLBJXGLnzA454LDJyDJy3ThCO/OreB6YkbE2l9Fux120l6VlrNHWSA0Yk+8PXVKD905xr5xMjYPtcQzZa+Qx0diBETk/GPexPo0T9LUwwPppnlR99MvxKc7DIMNzBXuCMWK5gXGuVl8G7XSIOXvYL39bWV67BNcsbeD5Q/8Y7YzWOz38zmUR20gTHFfBT/3xOmIbLM8Nia998BT39Zcu4twVFubSZtgq/b2xjOeLixMc62mBb7fi4zMP9nG9IlJ7Jh6+m+/8fh9bztYuQAghhBBCCCGEEEIIIYQQQgghzij6uCqEEEIIIYQQQgghhBBCCCGEEDOgj6tCCCGEEEIIIYQQQgghhBBCCDED+rgqhBBCCCGEEEIIIYQQQgghhBAzwDolU5ZaLdCiCBvxDsesIWwVtDQljXO9ALS33v0YtK3nz0t/d4fY/LczwKbkKb6aqdWw4XVKGn4HAb6b62Nz50oVG/2yxuCuh/dm5Ft3mmMzXYtoRYHPzRKclzgpT0K1UoFrVpawuXebNPKOSTPhyEeTGpMm2LnngTYkjeTnSZJhY+whsflGC+cwGmJT6YzYVWbhHGasf/KUaOFyz0xRYIP4wsF1G9o4/rfjU9CejPC64xDH5Z7bAO38+ipoV1ewIfXSAtqkTXx3aHDyIqusuS76Y6WC/l0Ja6C5Pq51pRqCFhC/8ojNnzVSEiSzGP0ytnG+2L0mR7siPdhNTuKXPRXn4hyvyUnj95z4WZ6jPfourpGFpmHyAu3bstGHiCublMRgQ3KfRezWdcjLOOy5pFl9UV4Ln7xcmuFvxSHG6sWb10Bbv4K+HO1jQ/sv7r0PWjXBGDpPvBqZG1xy47XRzy+3zoG2c3cPtLf/80PQwibWRWGtbJO1Kr7b6gKuhxdifHxytAlab4T2GJGa5eT0ELR+jFp0gPkgHKFfJfkiaN0K2q0fNECLY7zuZNAB7dmg/C4dD30qa+C7nV/CdT3cegKaS97j0nXMQY57DFqrvgDavPFI/vRdjOs5CdiTCdY3e6dkP5DgdT6ppVculOvL4YjU7yT+u6S25pCiivwe03ISDJiWFpj/cvJ7n376EWgv3bwD2oU1jC1Pnj4ELYqGpb9ZLmG5yTIY/6nEco5H8mlI8ql7tv79rm/h3KRkfHYF82DdRX+xchxzmmLdeG65CVp2bb30dzxBm89StJ/pPZwxxozHEWijIf5etzsA7eMvDkAbDDA3sTjcO8H4z/anH/zmA9A++/AT0Go1fG6tjvNZ8coxyXeJzRf4vjeuXAJttYW/v/sEc2dI9gMhqfNpgTtHiMkbQ+bGctBXK2Tv7pGzi4IcJcWkhnKm6vxmE+P3lctYyzik3n688xy0wsE91q07r4LWIns2Vm+zOr8wOHfD/hFoeYG/d7CPdcvxEdnDkDxsT+9DClwHiwV6EtSTBMfgsHMBi5xZuBh/3ATfd54UBTsj+/21NGPzxWIOySVTS+JYeM3Dz++D9k//+A+gsXjzjddfA63dboF259ZN0Kwf/hC0n775E9DSiOQcUgNlE7Tl4XgEWuCjnbbXML7GU/d2elhj/tuP/hW0w24XtOMe5qqj0xPQUpJzyZLRknKe2CR+5SQXsxe3SdwoMpwHdiZqHPw9xyrng4CcublkThNSA5EQTN8j9DFvWA7aVI+cMzcXsD5rkv3R2jKeTUZ9tKHtZ9ugpROSN7pok5NRed5dn8QZkqt9D3Mw2zNFxG9jcuhcdTC/GJJf50ke4Zy2qyug2R7aX1ZgLexVMH5V0AxMnGO+89zyfFUCtMeNV/E7yu4mnhf1Oj18KPk+5Ni4bjnxDZvYi+djrZ0VuL5JhHuLLMN5Wl8j5/sr66ClcRe0veNyLXcywPHfXsL98A3yTGsHc8T1YzIuF31vHPwKtFHrJdCMuUy0Mmdr5yuEEEIIIYQQQgghhBBCCCGEEGcUfVwVQgghhBBCCCGEEEIIIYQQQogZ0MdVIYQQQgghhBBCCCGEEEIIIYSYAX1cFUIIIYQQQgghhBBCCCGEEEKIGcAOyF9CRJpAB+TT7IQ0uvVIY+QU+wubYrrzuzHGrmI34cfPD8vXkGbZaUKa0qfY/DgizXqHwyG+B3m3IMCmxjUfG+dWq9iE3rHxXYIK/l41xPHHMTYsPux0QMsNXud65XG0m9jwe22xBdq5c4ugdYcT0HpdbO49OO2C1lrE3zs6PAJtniQZjs8hzcXbqziHyRjXMo1Jg3g0A5PkKBZTDceJ+RjLYOd3y0KtIJpx0W5d0nE+CdGXJwu4li8srIHWXsSm8fUmhqB6iP4cVPC6KMX5jA1qhVcem0Mavxs2J0TzfBy/Q+KPR57hkKbihUF7miskLjsezoPj4XvbLkkIOdEK9m968MHTTy0sXFtm3xUff6vdbINmE3/JSDP4LEfNcfDeICB5LsV5Yn6aE5/Pchxvvz8ArSCxIJ/KuacWXuSewzm5fOMGaO32MmjP7m2CdrS5hc8gc1fxzta/6SryEWjdY6wBdp8dgnb761dBi4e45t3jPmi//I/3QEvt8prHN3D+LiSoLTWXQLt57mXQTvqnoB2MMO86JI6GdgjaxG+B9uC3n4O2e7AP2vmL10HrPPoCtJjUnsyHqqvld7l0+yZc0750CbRhhD7FYtnS+VXQiiquRbePttPt4RjmzcTGHFWQeY3JdRVSI55fx/xuVTAm1psNfO5ULn+8hbFkHKGfVqpok46D8d8h8c9K8PeyBPcv0+9mjDHpBK/LctxLJHEM2jDCuvKze/dBs8m8904xBhVTS+aRPFTks9WGZPnpfLqkDvID3Ofkxdmqb6oG/dDz0IcNy1E5GV+K9yZESxOyR83K65tUSPFFYOuWkpyQEBsdDTA21UL0oQ8f7IAWT9Bf1uq4zzEFqcGHXfw9vNNkfRzbiNSfwdThw1ev436jWcM4s7aKedJNiE2kVdCqFtp8hfhocMbqG1YPGwvf2yc+7blkfOTnJsSF+hHanzNV+y+38XyjEeDcd3cx7nV2dkE712iBtrCAdW4eofWx2j8hsX9MYrplE7sl5zTP98kZh0XylYP7cdsqz11BbM+yyBkaiRcZ2W8YorUauBYWyQenh8QA5gg7rytILmIai6+OjWNu1RdAc4m/VKbOCVukdkpj9JXOEdqKu4xnLdEI4/LDowPQPv5oHbQ/fvVV0Dbv3QNtdw99LRqTuFnHmFurYN5kMTInvrZ3Un7uZ5uP8D4St26+chu0agPf7ZicmzJIWUSPjOYJK7Uyojk+5nseg3BeE3KGXvFIjT/lG66Pcb5K6sUiJjU0yc8BiUFry1dAY/VOMkDfGHTR1xYW8Nyje/AUtK3tx6BNSA3YrGEeCjwcR7NV9nHLxnwzGOI+viBnaCy+ZTn+nkPWwjEYk1JyhjlPLA9jho2ScW3yrcbFeOCS8/2E7NeyAm1yOOqW/s47Xbjm6p3LoD37FG2v9/Zd0FJiUwVZD4fkoJzU5HlOgsP0ZtIY45BS4ZXrV0D74be+CdpSA23y2SF+I+pO7Q8CcubzBTkP3erguVKVnL0vkpqyTb49NGrHoDUHmA+N+VOilTlbniKEEEIIIYQQQgghhBBCCCGEEGcUfVwVQgghhBBCCCGEEEIIIYQQQogZ0MdVIYQQQgghhBBCCCGEEEIIIYSYAX1cFUIIIYQQQgghhBBCCCGEEEKIGcCOrl/CZIyNpgMHm9+G5Bdz0pCa9M82ucHOuXlBtKnGzWlMGtVn+G6zNrTPc3ymbeN36JMTbMzbIWNt1mugLbSxMX2TNCeuGGw0neXYYNm1sNmvE+AkT6aaM1dcnCf2W+kIG2inI3yPQRcbAucJNtCuBB5oEWlSPk8cD+emvYiN2eshrltGbDJN0K7SDLXC4HNtu+xYFvl3EbbF7sM5tV281/XwfUMX7200cPxrU83rjTGmThpI13zUfGIHMUpm4OM7jzNs8J1ZxIfc8g/6DgYpz/dBs4k9WiQOsBgSx9gM3vfxfX3SfHueOCn59zYx2mhm0PcLg37uGpxXx+ACW8R28zydugbnmWl5iu8xGuE8s7U0JAcVLAclGCOjhNgL8VOL2ChxeWNIv/nMoF0ZNgdTtttcbcM1KzeugmaT8d9/7x3QogOM8w7xR5fEH5bT50l3H/P43ffvgxYN0a6cCubn5Q2c63iM/vLs4RFovzIflv72qugrvZUOaM1OC7QLq9dAazWWQfM9tMfQQr9dCfHelSshaJcXGqC99evfgLY13APtaLgD2lLrPGjrly6DdvFi+bqNCxv4+8e41gODtS1zvkYD13WSD/HWDOdkdR1tZ96kQQBaQWKz214F7dxFnP9wdR20hIS64XgE2slROZ74NVJTLOJ7eAHaaWFjMPVIbe3mGK8KUvsXpEZLIrSZOEJbiKIBaOQRxvfRZgx5blZg/J/E/dLfLK3ZNq4rKVtMnmNe80i9VCH1ksNifXa27J5sWY1LalCWe01OanpymctqRA+fkU7lyyQhNQqpFVg9YgJSA1RwLRdI/b68ugTanVtXQOv30JajCa5vlpH9Y4p2m5IaitWBLtmvOFP+7DN7JHNeqWDMC0gcDEPcs1eI5pN72bvMk9VFjKVJjOtmE5u3CrKW5CzI9XDMzTrOzXQ87A8wH/vk3TJiZ3WSq6pk35kNMC4XxPZYnLMsfEZQQztIiZ++f+8T0I66fdCcCvokO5pzrLKWE19h/sOw2b6EHL+88iLm+XzQBe39o2czPfcPBd9Psr0dC/RITmqKcYYxrRmizdfbzdLfrRbWkT45a7i4hGeEFslBkxj9cTzAWP1fP/85aOmYnM2CYoxHcnt7AePKhNRFbN4dMo6Rje8STcWCZgN9b0LyzaO7ZP82IL5H6sKCTAAJg8amMzU/bA9tr1LD3O6HWGuSI19Tr+M+LiMxwid5tpg6o3dqLbhmaXEFNDdHnzrcxT2hE+K72SS3T3q4V47J+ZBPxn+w8wB/L8UL/TqOo1FDLaiQ3OTgXqg3lROLCTnvd/HcIU5xcYoMfQpnyZi8wGc0PPTbyDpbZzcd8r3h8jKea3kjcjZFcuDxEX7n6I8PQLOqpFaolOe/gcttwgbe95U/ewG0cYTrdvjkEK87xevqNazFqiS55+QbWZP41a2NG6C9Rs4O2zWcz80tjMNPjp6CNu1WaY6/da/bBc0neblCcn8jwJi32MQ8fH7tAmi1Q6xtXgcF0f9cFUIIIYQQQgghhBBCCCGEEEKIGdDHVSGEEEIIIYQQQgghhBBCCCGEmAF9XBVCCCGEEEIIIYQQQgghhBBCiBnQx1UhhBBCCCGEEEIIIYQQQgghhJgBq5i1o7sQQgghhBBCCCGEEEIIIYQQQvw/Rv9zVQghhBBCCCGEEEIIIYQQQgghZkAfV4UQQgghhBBCCCGEEEIIIYQQYgb0cVUIIYQQQgghhBBCCCGEEEIIIWZAH1eFEEIIIYQQQgghhBBCCCGEEGIG9HFVCCGEEEIIIYQQQgghhBBCCCFmQB9XhRBCCCGEEEIIIYQQQgghhBBiBvRxVQghhBBCCCGEEEIIIYQQQgghZkAfV4UQQgghhBBCCCGEEEIIIYQQYgb0cVUIIYQQQgghhBBCCCGEEEIIIWbg/wA6Ki6tDmqwfwAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "def grid_plot(tensor, labels):\n", + " tensor = tensor.clone().detach()\n", + " tensor.mul_(ds).add_(dm).clamp_(0, 1)\n", + "\n", + " fig, axes = plt.subplots(1, 10, figsize=(24, 24))\n", + " for im, l, ax in zip(tensor, labels, axes.flatten()):\n", + " ax.imshow(im.permute(1, 2, 0).cpu());\n", + " ax.set_title(l)\n", + " ax.axis('off')\n", + "\n", + "grid_plot(ground_truth, [validloader.dataset.classes[l] for l in labels])" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "local_lr = 3e-4\n", + "local_steps = 1\n", + "import pickle\n", + "\n", + "from collections import OrderedDict\n", + "model_bespoke.zero_grad()\n", + "target_loss, _, _ = loss_fn(model_bespoke(ground_truth), labels)\n", + "input_parameters = inversefed.reconstruction_algorithms.loss_steps(model_bespoke, ground_truth, labels, \n", + " lr=local_lr, local_steps=local_steps,\n", + " use_updates=True)\n", + "\n", + "\n", + "\n", + "\n", + "# params_t = OrderedDict((name, param) for (name, param) in params_t_full[0].items() if name in params_s)\n", + "input_parameters = [p.detach() for p in input_parameters]" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [], + "source": [ + "# based_model_names = [name for name, _ in model_bespoke.named_parameters()]\n", + "# with open (\"start_reprs.pkl\", 'rb') as f:\n", + "# params_s_full = pickle.load(f)\n", + "# with open (\"end_reprs.pkl\", 'rb') as f:\n", + "# params_t_full = pickle.load(f)\n", + "model_bespoke = mutils.get_model(\"resnet10\", \"cifar10\", \"cuda\")\n", + "params_s = OrderedDict((name, param) for (name, param) in model_bespoke.named_parameters())\n", + "with open (\"end_reprs.pkl\", 'rb') as f:\n", + " params_t_full = pickle.load(f)\n", + "\n", + "params_t = OrderedDict((name, param) for (name, param) in params_t_full[0].items() if name in params_s)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "param_diff = OrderedDict()\n", + "for (name_s, p_s), (name_t, p_t) in zip(params_s.items(), params_t.items()):\n", + " if name_s == name_t:\n", + " p_t = p_t.to(device)\n", + " p_s = p_s.to(device)\n", + " param_diff[name_s] = p_t - p_s" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([('conv1.weight',\n", + " tensor([[[[-0.0006, 0.0561, 0.0613],\n", + " [-0.0925, -0.1245, -0.1912],\n", + " [ 0.2336, -0.0051, 0.0833]],\n", + " \n", + " [[-0.0461, 0.0431, 0.1182],\n", + " [ 0.0759, 0.0499, -0.0546],\n", + " [-0.0938, -0.0426, 0.0995]],\n", + " \n", + " [[-0.0867, 0.1786, -0.0268],\n", + " [-0.1879, 0.1099, 0.0997],\n", + " [ 0.0216, -0.0517, 0.0542]]],\n", + " \n", + " \n", + " [[[-0.0842, -0.0590, -0.0917],\n", + " [ 0.0528, -0.0941, -0.2365],\n", + " [-0.1343, 0.0125, 0.0060]],\n", + " \n", + " [[-0.1608, 0.0795, 0.1789],\n", + " [ 0.0797, -0.0478, 0.1068],\n", + " [ 0.1370, 0.0267, 0.1973]],\n", + " \n", + " [[-0.0459, 0.1254, 0.1123],\n", + " [-0.1720, 0.1773, 0.1561],\n", + " [ 0.0347, 0.0052, 0.1108]]],\n", + " \n", + " \n", + " [[[ 0.0406, 0.1152, 0.1499],\n", + " [ 0.1664, -0.0313, 0.1316],\n", + " [-0.0111, -0.0099, -0.2676]],\n", + " \n", + " [[-0.1355, 0.0306, 0.0134],\n", + " [ 0.1814, -0.2067, 0.2133],\n", + " [-0.0884, 0.0318, -0.0323]],\n", + " \n", + " [[-0.0081, -0.0779, 0.1160],\n", + " [-0.0263, -0.0354, 0.0638],\n", + " [-0.2263, 0.1726, -0.1936]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 0.1928, 0.1935, -0.0905],\n", + " [ 0.2506, 0.0178, -0.1192],\n", + " [ 0.1269, -0.0953, -0.2048]],\n", + " \n", + " [[ 0.0185, -0.1787, -0.0943],\n", + " [-0.2781, 0.1565, -0.0043],\n", + " [ 0.0247, -0.0955, 0.0705]],\n", + " \n", + " [[ 0.0941, 0.1939, 0.0726],\n", + " [-0.1135, -0.1600, 0.1947],\n", + " [ 0.1036, -0.1385, 0.0691]]],\n", + " \n", + " \n", + " [[[-0.0507, -0.1775, 0.0227],\n", + " [-0.0576, 0.0651, -0.1624],\n", + " [-0.0899, -0.1509, -0.0459]],\n", + " \n", + " [[-0.0317, 0.1149, 0.0944],\n", + " [ 0.1851, -0.0195, -0.0367],\n", + " [ 0.0663, 0.0479, -0.0879]],\n", + " \n", + " [[-0.1262, -0.1743, -0.0682],\n", + " [ 0.1980, -0.0746, 0.0476],\n", + " [-0.0467, 0.0354, -0.0615]]],\n", + " \n", + " \n", + " [[[-0.0031, 0.1585, 0.0819],\n", + " [-0.0030, 0.0041, 0.0633],\n", + " [-0.2265, -0.0170, -0.0711]],\n", + " \n", + " [[-0.0104, 0.0656, -0.0445],\n", + " [-0.1921, 0.1331, -0.0937],\n", + " [-0.1415, -0.0107, -0.1412]],\n", + " \n", + " [[ 0.0894, -0.1457, -0.0566],\n", + " [ 0.1551, 0.0706, -0.0333],\n", + " [-0.0042, 0.0259, 0.0622]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('bn1.weight',\n", + " tensor([ 4.6492e-06, -2.8610e-06, 2.3842e-06, 3.5763e-07, -3.0994e-06,\n", + " -5.3644e-07, -4.7684e-07, 1.3113e-06, -8.1062e-06, 2.1458e-06,\n", + " -9.7156e-06, 3.5763e-07, -1.1921e-07, 7.0333e-06, 5.9605e-07,\n", + " -5.9605e-08, -3.2187e-06, -3.8147e-06, 3.4571e-06, 8.7023e-06,\n", + " 5.2452e-06, -1.1325e-06, -2.5034e-06, -1.1325e-06, 5.9605e-07,\n", + " -1.2517e-06, -3.5763e-06, -9.5367e-07, 2.2650e-06, -2.8014e-06,\n", + " 2.8610e-06, 1.3113e-06, -3.4571e-06, 1.0729e-06, 1.0729e-06,\n", + " -1.1921e-06, -5.9605e-07, -6.2585e-06, -4.4107e-06, -7.8082e-06,\n", + " -7.7486e-07, 4.7684e-06, -4.7088e-06, 1.6689e-06, 4.4107e-06,\n", + " -1.9670e-06, 2.0266e-06, -2.3842e-07, -1.2517e-06, -1.7285e-06,\n", + " 1.7881e-06, 3.5763e-07, -1.0133e-06, 2.8610e-06, 1.6689e-06,\n", + " -3.2187e-06, 2.7418e-06, -3.5763e-07, 3.5763e-07, -8.9407e-07,\n", + " -1.3709e-06, 7.6294e-06, 3.8147e-06, 7.1526e-06], device='cuda:0',\n", + " grad_fn=)),\n", + " ('bn1.bias',\n", + " tensor([ 5.4231e-06, -3.2287e-06, 8.1165e-07, -4.1569e-07, -1.7457e-06,\n", + " -4.1456e-07, -2.2053e-06, 8.3448e-07, -1.0379e-06, 3.1667e-06,\n", + " -2.9718e-06, 1.3673e-06, 7.8012e-07, 2.8126e-06, -2.2484e-06,\n", + " 2.0777e-07, -3.4527e-07, 3.2588e-07, 3.1864e-06, 7.5375e-06,\n", + " 2.8332e-06, -5.6947e-06, -1.0997e-06, -2.4325e-06, 5.9140e-06,\n", + " -1.9417e-06, -2.4949e-06, 1.4172e-06, 1.4224e-06, -5.0170e-06,\n", + " 1.3376e-06, -1.8968e-06, -3.7705e-06, 2.1635e-07, 3.2337e-06,\n", + " -1.4948e-06, 1.7549e-07, -5.6401e-06, 4.3037e-06, -6.3509e-06,\n", + " 1.3941e-06, 4.9555e-06, -6.7065e-06, 4.9904e-07, 5.8545e-06,\n", + " -4.9189e-07, 1.5127e-06, -1.9307e-06, -2.1169e-06, 3.0242e-06,\n", + " 8.7774e-07, 5.8332e-07, -2.6954e-06, -2.2571e-07, 2.0464e-06,\n", + " -5.8163e-07, -1.1077e-06, -1.4233e-06, 1.1387e-06, -1.1688e-07,\n", + " -1.5522e-06, 2.9426e-06, 2.9115e-06, 5.6802e-06], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer1.0.conv1.weight',\n", + " tensor([[[[-2.2570e-02, 4.6579e-02, 4.3270e-04],\n", + " [ 3.9203e-02, 1.5285e-02, -1.8835e-03],\n", + " [ 4.0751e-02, 3.1965e-02, -2.3628e-02]],\n", + " \n", + " [[-2.2898e-02, -5.9936e-03, 9.9707e-03],\n", + " [ 2.8830e-02, 2.9010e-02, -2.1784e-02],\n", + " [ 2.5152e-02, -4.0623e-03, 1.3364e-03]],\n", + " \n", + " [[-2.0373e-02, -1.5970e-02, -2.7016e-02],\n", + " [ 1.4274e-03, -1.9746e-02, -8.7298e-03],\n", + " [-2.5143e-02, -2.3898e-02, 1.5841e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-9.5870e-03, -3.6288e-02, 9.6889e-03],\n", + " [ 4.5948e-03, -2.9685e-02, 4.0744e-02],\n", + " [ 1.5693e-02, -9.7967e-03, 3.2967e-02]],\n", + " \n", + " [[-1.0512e-02, 2.0607e-02, -7.5278e-03],\n", + " [-7.1602e-04, 4.0102e-02, -2.5954e-03],\n", + " [-4.5868e-02, -6.7171e-03, -1.1770e-02]],\n", + " \n", + " [[-1.1169e-03, -1.6320e-03, 4.8017e-02],\n", + " [-2.8562e-02, -2.4703e-02, -3.4483e-02],\n", + " [ 2.7159e-02, -7.1618e-05, -1.6801e-02]]],\n", + " \n", + " \n", + " [[[-2.1106e-02, -2.2973e-02, -6.0225e-03],\n", + " [-5.5912e-02, -4.6143e-02, 3.0123e-02],\n", + " [-5.7831e-02, 3.0423e-02, 2.4413e-02]],\n", + " \n", + " [[-2.9441e-02, -3.2263e-02, 2.9663e-02],\n", + " [ 1.1660e-02, 3.8977e-02, -3.6358e-03],\n", + " [-2.5374e-02, 1.3639e-02, -7.5420e-03]],\n", + " \n", + " [[ 3.6191e-02, -2.0330e-02, 1.4488e-02],\n", + " [ 2.2990e-02, -2.1860e-02, -3.7394e-02],\n", + " [ 4.2311e-02, -1.9836e-02, -3.7274e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.0211e-02, 1.3598e-03, -3.2501e-02],\n", + " [-5.6539e-02, -3.3627e-03, 7.4346e-03],\n", + " [-9.2457e-04, 4.0321e-02, 2.3731e-02]],\n", + " \n", + " [[-7.9937e-03, -3.6999e-02, -8.0297e-03],\n", + " [ 3.8824e-02, -1.2008e-02, -4.5381e-02],\n", + " [-1.3227e-02, 2.8266e-02, 2.2535e-02]],\n", + " \n", + " [[-2.6154e-02, 2.3498e-02, -1.2162e-02],\n", + " [ 2.1689e-02, -5.1684e-02, 1.1600e-02],\n", + " [-2.4643e-02, -1.0527e-03, 6.2798e-04]]],\n", + " \n", + " \n", + " [[[-1.0321e-02, 2.4365e-02, -5.5286e-02],\n", + " [ 1.2955e-02, -2.3997e-02, 5.2910e-02],\n", + " [ 2.6851e-02, -1.7141e-02, -1.5507e-02]],\n", + " \n", + " [[ 1.1109e-02, -4.2064e-02, 4.1614e-02],\n", + " [-2.7240e-02, 2.6202e-03, 4.7014e-02],\n", + " [-5.2148e-02, 2.9868e-02, 3.2549e-02]],\n", + " \n", + " [[-3.2545e-03, 2.7492e-02, -3.8950e-02],\n", + " [ 1.5161e-02, -2.5894e-02, -3.2587e-02],\n", + " [ 1.7659e-02, -2.2911e-02, 1.6492e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.5354e-02, 5.3448e-02, 2.3870e-02],\n", + " [ 1.5850e-02, 1.3058e-02, 1.6393e-02],\n", + " [-1.5742e-02, 1.6519e-02, 2.2951e-03]],\n", + " \n", + " [[-2.2953e-02, -4.3644e-02, 6.5544e-04],\n", + " [-2.1834e-02, -4.3274e-02, 5.7042e-03],\n", + " [ 1.8394e-02, -1.5122e-02, 1.4276e-02]],\n", + " \n", + " [[ 2.8725e-02, -3.2279e-02, 3.5784e-02],\n", + " [-3.5181e-02, -2.3900e-02, 5.7771e-05],\n", + " [ 2.7850e-02, 3.0817e-03, 5.1541e-03]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 3.9989e-02, -1.7403e-02, 5.0907e-02],\n", + " [-7.3910e-03, -3.4016e-02, 7.1424e-03],\n", + " [-3.7204e-02, -1.9108e-02, -3.5431e-02]],\n", + " \n", + " [[ 1.3562e-02, 2.2771e-02, -6.0645e-02],\n", + " [-2.6215e-02, -2.6883e-02, -3.2663e-02],\n", + " [ 8.0954e-03, -3.4466e-02, -1.8052e-02]],\n", + " \n", + " [[-5.5253e-02, 2.6264e-02, -4.6535e-02],\n", + " [ 3.9234e-03, -1.0685e-02, 9.2071e-03],\n", + " [ 1.2697e-02, -2.4685e-02, -1.7718e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.5044e-02, -1.7497e-02, -2.1783e-02],\n", + " [-2.0340e-02, 1.3263e-02, 4.0077e-02],\n", + " [-3.8877e-02, -1.1697e-02, -5.0333e-02]],\n", + " \n", + " [[-7.3433e-02, -2.2334e-02, -4.7794e-02],\n", + " [-1.1918e-02, 2.2436e-02, -1.8381e-02],\n", + " [ 1.9740e-02, 1.6637e-02, 1.9241e-02]],\n", + " \n", + " [[ 3.2566e-02, -4.9752e-02, 2.6469e-02],\n", + " [-2.4228e-02, -3.1309e-04, -4.1350e-02],\n", + " [-7.4496e-03, 1.7813e-02, -3.1136e-02]]],\n", + " \n", + " \n", + " [[[-3.8622e-02, -3.7675e-02, -1.3183e-02],\n", + " [-4.2526e-02, -1.1766e-02, 3.5638e-02],\n", + " [-2.8378e-02, 2.9796e-03, -1.7564e-02]],\n", + " \n", + " [[ 1.8862e-02, -1.6337e-02, 1.3012e-02],\n", + " [-3.1859e-02, 2.2553e-02, 1.5612e-02],\n", + " [ 1.3076e-02, -4.6412e-03, -9.1515e-03]],\n", + " \n", + " [[ 2.0634e-02, 1.1302e-02, 4.9690e-03],\n", + " [ 1.5253e-03, 9.4118e-03, -3.1450e-02],\n", + " [ 2.0302e-02, 5.1324e-02, 2.1210e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.5408e-02, 1.5580e-02, -2.0611e-02],\n", + " [ 1.2426e-02, -2.4775e-02, 3.1102e-02],\n", + " [-4.5320e-02, 1.4904e-02, -2.4320e-03]],\n", + " \n", + " [[-4.4531e-02, -2.4908e-02, -1.1673e-02],\n", + " [-6.1137e-03, -5.2799e-03, -4.8738e-02],\n", + " [-2.8913e-02, 1.7804e-03, 6.6780e-03]],\n", + " \n", + " [[-4.0387e-02, 2.9545e-02, 1.7133e-02],\n", + " [ 6.8722e-03, -2.8609e-02, -2.0323e-04],\n", + " [ 3.9065e-02, 2.2012e-02, -2.0337e-02]]],\n", + " \n", + " \n", + " [[[ 9.0897e-03, 1.3927e-02, -5.3809e-02],\n", + " [ 4.6715e-04, 1.6106e-02, -2.4101e-02],\n", + " [-8.4275e-03, -1.7561e-02, -1.7671e-02]],\n", + " \n", + " [[-1.2119e-03, -2.4723e-02, -4.8516e-02],\n", + " [-2.0067e-02, 4.0879e-04, -1.8960e-02],\n", + " [ 2.2412e-02, -3.6067e-02, 9.5359e-03]],\n", + " \n", + " [[-3.7317e-02, -2.0986e-03, 3.8389e-02],\n", + " [-5.7991e-03, -1.4761e-02, -9.8050e-04],\n", + " [-3.0369e-03, 3.3129e-02, 2.1562e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.4468e-02, 3.0479e-02, 1.8400e-02],\n", + " [ 1.3205e-02, -1.2044e-02, -3.0517e-03],\n", + " [ 1.4351e-02, -2.7222e-02, -1.7896e-02]],\n", + " \n", + " [[ 1.1494e-02, 1.9010e-02, -1.1443e-02],\n", + " [ 3.3070e-04, 4.1223e-02, -4.1695e-02],\n", + " [ 1.3860e-02, -1.1250e-02, 6.9530e-03]],\n", + " \n", + " [[ 2.5772e-02, 2.0700e-02, -2.7200e-03],\n", + " [-6.9283e-03, 2.3300e-02, -9.7905e-03],\n", + " [ 1.9706e-02, -1.8724e-02, -3.7177e-02]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer1.0.bn1.weight',\n", + " tensor([-1.0729e-06, -1.1921e-06, 1.4305e-06, 1.0729e-06, -7.7486e-07,\n", + " -8.9407e-07, -1.9073e-06, 1.0729e-06, 3.6955e-06, 2.6226e-06,\n", + " 9.5367e-07, 1.1921e-06, 2.7418e-06, -1.6093e-06, 1.3113e-06,\n", + " -1.1325e-06, 4.7684e-07, -5.9605e-08, -2.5630e-06, -1.9073e-06,\n", + " 3.5763e-07, -1.1921e-07, 1.7881e-06, -2.0862e-06, -5.9605e-07,\n", + " 2.1458e-06, -3.2187e-06, -3.3379e-06, -3.1590e-06, 3.5763e-07,\n", + " 3.5763e-07, 1.5497e-06, 2.1458e-06, -4.1723e-07, 1.3113e-06,\n", + " 2.0266e-06, -1.0133e-06, -1.7881e-07, -4.0531e-06, 1.1921e-06,\n", + " 5.9605e-07, -3.9339e-06, 2.6226e-06, -2.6226e-06, -1.6093e-06,\n", + " 4.7684e-06, -5.9605e-07, 3.6955e-06, 8.3447e-07, -1.2517e-06,\n", + " 1.7881e-06, 7.1526e-07, 1.1921e-06, -2.3842e-07, -4.7684e-07,\n", + " 8.3447e-07, 3.0994e-06, -1.9073e-06, 2.1458e-06, -1.2517e-06,\n", + " -1.8477e-06, -1.7285e-06, -1.7881e-06, -8.9407e-07], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer1.0.bn1.bias',\n", + " tensor([-1.6646e-06, -8.3373e-07, -7.3852e-07, -1.8242e-06, -1.9939e-06,\n", + " -4.2068e-06, 1.2229e-07, -9.5055e-07, 3.2928e-06, 2.4605e-06,\n", + " -1.0227e-06, 1.9723e-06, -1.5289e-06, 8.2279e-07, -9.1037e-07,\n", + " 1.4940e-06, -2.5393e-06, -1.6297e-06, -3.4093e-06, -1.6058e-06,\n", + " -1.6695e-06, 6.3167e-07, 3.0258e-06, -3.3431e-06, -1.0892e-06,\n", + " 1.1644e-06, 1.8028e-06, -1.6146e-06, -3.2107e-06, 1.2860e-07,\n", + " 1.0780e-08, 2.2232e-06, -6.5523e-07, -2.5341e-06, -2.6235e-06,\n", + " 1.3846e-06, 1.0392e-06, -3.1820e-07, -3.5873e-06, -9.7919e-07,\n", + " -1.1498e-07, -1.5139e-06, 2.9632e-06, -2.7121e-06, 4.9362e-07,\n", + " 2.4185e-06, -2.5546e-06, 4.9935e-06, 4.7802e-07, 1.9509e-06,\n", + " -4.6129e-09, 1.4534e-06, -6.3682e-07, -4.7444e-07, 1.9421e-07,\n", + " 1.0255e-06, 7.3000e-07, -5.0841e-07, -5.6548e-07, -8.5624e-07,\n", + " -1.5565e-06, 1.0536e-06, -5.5264e-07, -7.5711e-07], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer1.0.conv2.weight',\n", + " tensor([[[[ 1.0396e-02, -2.0216e-02, -9.7034e-03],\n", + " [-1.6117e-02, 3.2930e-02, 2.6085e-02],\n", + " [ 1.6743e-02, -1.5415e-02, 3.0183e-02]],\n", + " \n", + " [[-1.2542e-02, -2.0246e-02, -2.1830e-02],\n", + " [-2.3590e-02, 4.3001e-03, 2.3873e-02],\n", + " [ 2.0339e-02, 2.6765e-02, 1.5982e-02]],\n", + " \n", + " [[ 5.9611e-03, 7.7520e-04, 2.5923e-02],\n", + " [-2.4622e-02, -1.0194e-02, 2.2966e-02],\n", + " [-1.7405e-02, -1.7530e-02, -5.4647e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-8.8577e-03, 4.5698e-02, -7.7993e-03],\n", + " [ 3.3063e-02, -3.2623e-02, 4.7993e-02],\n", + " [ 3.9605e-02, 5.0393e-02, -4.3616e-03]],\n", + " \n", + " [[-3.5010e-02, -2.4205e-02, 4.8226e-02],\n", + " [-3.5885e-02, 2.4366e-02, -1.5791e-02],\n", + " [-1.7299e-02, -2.0041e-02, 2.0468e-02]],\n", + " \n", + " [[-1.0454e-02, 1.0696e-02, 2.5080e-02],\n", + " [-2.6918e-03, -2.4078e-02, -2.8075e-02],\n", + " [ 2.1477e-02, 2.0518e-02, -2.3786e-02]]],\n", + " \n", + " \n", + " [[[ 5.0096e-02, 5.8117e-02, -5.5126e-02],\n", + " [ 2.7779e-03, -5.5745e-03, 2.0170e-02],\n", + " [-3.5873e-03, 4.9762e-03, 3.4356e-02]],\n", + " \n", + " [[ 4.4078e-02, 2.8536e-02, -3.0690e-02],\n", + " [-1.9038e-02, -7.6155e-03, -1.2531e-02],\n", + " [-1.2465e-02, -2.8210e-02, 1.4366e-02]],\n", + " \n", + " [[-2.6739e-02, 2.4669e-02, 4.6204e-02],\n", + " [ 2.9311e-02, -3.4707e-02, 2.6172e-02],\n", + " [-5.1166e-02, 3.2466e-02, 1.6315e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.5474e-02, -8.1297e-03, 6.5045e-03],\n", + " [-1.9999e-05, -1.3515e-02, 1.1818e-02],\n", + " [-1.5118e-02, 9.1569e-03, -1.3914e-02]],\n", + " \n", + " [[ 2.1644e-02, 2.0979e-02, 4.0117e-04],\n", + " [ 4.2587e-02, 3.4951e-04, 1.9599e-02],\n", + " [ 1.7381e-02, 3.0407e-03, -2.1914e-03]],\n", + " \n", + " [[ 3.8028e-02, 2.3806e-03, -2.4305e-02],\n", + " [ 7.0412e-04, -2.9877e-02, 9.9972e-03],\n", + " [-3.7057e-02, 2.9808e-02, 2.9682e-02]]],\n", + " \n", + " \n", + " [[[ 1.3419e-02, -1.2343e-02, -8.7604e-04],\n", + " [-4.3507e-03, -9.1235e-03, -5.0805e-03],\n", + " [ 2.2734e-02, 2.4164e-02, 2.5917e-02]],\n", + " \n", + " [[ 7.2083e-03, 2.1416e-02, 1.5793e-03],\n", + " [ 6.6166e-03, 8.8527e-03, -1.3748e-02],\n", + " [ 2.1095e-02, 6.7640e-03, -1.8672e-02]],\n", + " \n", + " [[-4.5202e-02, 1.1592e-02, -2.6447e-02],\n", + " [ 4.8787e-02, -2.6143e-02, -2.1383e-02],\n", + " [-1.3499e-02, -3.1855e-02, 4.2831e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.8935e-02, 2.6718e-02, 6.5139e-02],\n", + " [-1.9350e-02, -7.3472e-03, 1.4973e-02],\n", + " [ 4.9365e-03, 3.4216e-02, -1.3682e-02]],\n", + " \n", + " [[-5.1517e-03, -1.1952e-02, -6.4165e-02],\n", + " [ 6.0639e-03, 1.3290e-02, -1.2138e-03],\n", + " [-2.2491e-02, 2.1337e-02, -1.4244e-02]],\n", + " \n", + " [[-3.3079e-02, 6.3611e-02, 1.0925e-02],\n", + " [-2.7628e-02, 3.0921e-02, 4.1639e-02],\n", + " [-2.7173e-02, -1.0412e-02, 4.0272e-02]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 1.4416e-02, 2.0361e-02, -2.7634e-02],\n", + " [ 1.7212e-02, 4.7874e-02, -4.4004e-02],\n", + " [-6.9799e-03, -1.9941e-03, 4.5099e-02]],\n", + " \n", + " [[-1.7381e-02, -1.7613e-02, -2.4853e-02],\n", + " [ 1.1966e-02, -1.7379e-02, 2.6862e-02],\n", + " [ 1.1788e-02, 1.9911e-02, 1.4908e-02]],\n", + " \n", + " [[-4.0639e-02, -3.3738e-02, -2.7907e-02],\n", + " [ 1.8080e-02, -1.7539e-02, -4.5545e-02],\n", + " [ 1.7268e-02, 2.7446e-02, -3.0459e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1242e-02, -1.4522e-02, -7.9271e-03],\n", + " [ 7.7173e-04, 2.8665e-02, -4.2147e-02],\n", + " [ 4.1924e-02, -6.0923e-03, -2.9036e-02]],\n", + " \n", + " [[ 4.4091e-02, -8.0454e-03, 3.6650e-03],\n", + " [ 1.8156e-02, 3.4450e-02, -2.4242e-02],\n", + " [-2.2434e-02, -1.3157e-02, 3.6239e-02]],\n", + " \n", + " [[ 4.2711e-02, -4.1155e-02, 9.3523e-04],\n", + " [ 2.3412e-02, -1.3124e-02, -4.7054e-03],\n", + " [ 3.0531e-02, -4.1716e-02, 3.7192e-02]]],\n", + " \n", + " \n", + " [[[ 4.1159e-02, -3.3871e-03, 2.4896e-02],\n", + " [-1.5587e-02, -2.0231e-02, 5.4416e-02],\n", + " [ 2.0937e-02, 2.6982e-02, -3.9793e-02]],\n", + " \n", + " [[-9.0131e-03, 2.5295e-03, 2.8721e-02],\n", + " [ 1.7443e-02, -3.7112e-02, -4.8766e-03],\n", + " [ 1.4680e-02, 2.3375e-02, -5.4663e-02]],\n", + " \n", + " [[ 1.0035e-02, 1.2707e-02, 2.0542e-02],\n", + " [ 1.7489e-03, 3.8192e-02, 5.9935e-02],\n", + " [ 1.6425e-03, 9.2595e-04, -2.0362e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.0376e-02, -4.4225e-02, -2.9585e-02],\n", + " [ 4.0246e-02, -1.4116e-02, -3.0059e-02],\n", + " [-2.9061e-02, -1.2602e-02, -3.5743e-02]],\n", + " \n", + " [[ 1.3734e-02, -2.0217e-02, -3.7963e-03],\n", + " [ 2.2904e-02, 3.1865e-02, 3.4065e-02],\n", + " [-1.7687e-02, -8.5194e-03, 1.8948e-03]],\n", + " \n", + " [[-1.6955e-02, -4.8008e-02, -2.5571e-02],\n", + " [-3.5338e-02, -9.7560e-03, 3.9748e-02],\n", + " [ 1.1480e-02, 1.0620e-02, -3.8812e-02]]],\n", + " \n", + " \n", + " [[[-4.1050e-02, 8.1633e-03, -1.3328e-03],\n", + " [-3.5018e-02, 5.9904e-03, 2.6881e-02],\n", + " [ 2.7519e-02, 2.4498e-02, -2.9707e-02]],\n", + " \n", + " [[-1.1786e-03, -3.3742e-02, 1.5864e-02],\n", + " [ 1.7007e-02, -1.7952e-02, -2.8105e-02],\n", + " [ 3.7245e-02, -1.4320e-02, 3.3163e-02]],\n", + " \n", + " [[-4.0185e-02, -2.3607e-02, -4.6880e-04],\n", + " [ 3.4657e-02, -4.1312e-02, -4.4827e-02],\n", + " [-1.0059e-02, -2.4776e-03, 5.1147e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.1438e-02, -6.6742e-02, 2.2961e-02],\n", + " [ 2.5789e-02, -1.0198e-02, -5.3327e-02],\n", + " [ 3.0370e-03, -1.2007e-02, -9.1576e-03]],\n", + " \n", + " [[ 2.0191e-02, -1.2843e-02, 2.2009e-02],\n", + " [ 6.0755e-03, -2.4749e-02, 3.1961e-02],\n", + " [-2.1769e-03, 2.5303e-03, 4.7090e-02]],\n", + " \n", + " [[ 2.5646e-02, -3.6510e-02, -1.1055e-03],\n", + " [-1.7726e-02, 2.3315e-02, -2.3016e-03],\n", + " [-6.9395e-03, -1.9193e-02, 3.1284e-02]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer1.0.bn2.weight',\n", + " tensor([ 5.1260e-06, 2.6226e-06, -2.2054e-06, -9.5367e-07, -1.0729e-06,\n", + " -3.0994e-06, -8.9407e-07, -1.7881e-06, -1.7881e-06, -1.0133e-06,\n", + " -1.9670e-06, -2.6226e-06, 1.4305e-06, 2.1458e-06, 2.3842e-07,\n", + " -1.6093e-06, -5.9605e-07, 4.7684e-07, -6.5565e-07, -1.7881e-07,\n", + " -3.5763e-07, 2.3842e-07, -1.1921e-07, 7.1526e-07, 3.0994e-06,\n", + " 4.7684e-07, 2.3842e-07, 4.7684e-07, -5.9605e-07, 1.5497e-06,\n", + " 3.5763e-07, 1.1921e-06, 1.1921e-06, -8.3447e-07, 1.1921e-06,\n", + " 7.1526e-07, -4.1723e-07, 2.0266e-06, -1.4305e-06, -8.9407e-07,\n", + " -4.8876e-06, -2.4438e-06, -1.4901e-06, -4.7684e-07, -2.8014e-06,\n", + " -5.3644e-07, -1.8477e-06, 4.7684e-07, -2.0266e-06, -7.7486e-07,\n", + " 9.5367e-07, -5.9605e-08, 2.3842e-06, 7.1526e-07, -2.8014e-06,\n", + " 3.3379e-06, -1.0133e-06, 2.3842e-07, 6.5565e-06, 3.5763e-06,\n", + " -9.5367e-07, 1.5497e-06, 1.6689e-06, 8.3447e-07], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer1.0.bn2.bias',\n", + " tensor([ 1.9615e-06, -1.7051e-06, -9.6066e-07, 2.9261e-07, 1.5350e-06,\n", + " -2.4253e-06, 1.4455e-06, -2.3713e-07, -1.0793e-06, -2.7920e-07,\n", + " 6.6946e-10, -1.2152e-07, -1.4768e-06, -1.2127e-07, 1.6777e-07,\n", + " -1.6548e-06, 1.6779e-07, 1.3144e-06, -3.9356e-07, 2.3618e-06,\n", + " 1.1459e-06, 7.5798e-08, -1.4052e-06, 1.9464e-06, 1.8920e-06,\n", + " 1.4963e-06, 7.9997e-07, 1.2816e-06, 1.0315e-06, 1.0541e-06,\n", + " 8.0646e-07, -1.6433e-07, 1.1502e-06, -2.8317e-07, -8.3817e-07,\n", + " -1.4362e-06, -3.3804e-06, -7.9363e-07, -2.2784e-06, -3.2797e-06,\n", + " 3.3910e-07, -1.8120e-06, -6.6409e-07, 1.1106e-06, 3.8630e-07,\n", + " -1.6407e-06, 6.6217e-07, -1.1658e-06, 1.1164e-06, -9.5826e-07,\n", + " -1.5532e-06, 1.0813e-06, -7.4805e-07, 2.6794e-06, -2.8860e-06,\n", + " 1.8326e-07, 6.5250e-07, 1.1221e-06, 6.3122e-07, 2.8292e-06,\n", + " 1.5905e-06, 2.7533e-06, 1.6429e-06, 2.0841e-06], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.conv1.weight',\n", + " tensor([[[[ 3.1222e-02, -2.0293e-02, -2.2732e-02],\n", + " [ 5.1991e-02, -5.2411e-02, -7.1547e-03],\n", + " [-1.3944e-02, -1.4512e-02, 1.5920e-02]],\n", + " \n", + " [[ 1.4411e-03, 2.3664e-02, 5.2533e-02],\n", + " [ 4.9313e-03, 2.8384e-02, 4.0038e-02],\n", + " [ 1.3472e-02, 2.9390e-02, -1.8044e-03]],\n", + " \n", + " [[-2.5809e-02, 2.3759e-02, 3.2983e-03],\n", + " [ 1.6743e-02, -2.7121e-02, 2.0905e-02],\n", + " [-4.8166e-03, 2.8307e-02, 5.3652e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.1179e-02, 1.3626e-02, -3.3159e-02],\n", + " [-3.0455e-02, 7.1659e-03, -2.3647e-02],\n", + " [ 1.5195e-02, -2.5908e-02, 7.0486e-03]],\n", + " \n", + " [[-2.9581e-02, 6.3754e-03, 4.0771e-02],\n", + " [-2.9581e-02, 4.5928e-02, -5.0159e-03],\n", + " [-3.1577e-02, 1.8542e-02, -2.3790e-02]],\n", + " \n", + " [[ 7.9386e-03, -5.5335e-02, -2.3499e-02],\n", + " [ 5.1797e-02, -1.9768e-02, 2.6371e-02],\n", + " [ 2.8223e-02, -3.6843e-03, -2.4914e-02]]],\n", + " \n", + " \n", + " [[[ 3.0091e-02, -1.5946e-02, -2.0017e-02],\n", + " [-7.3625e-03, -1.2496e-02, -1.5988e-02],\n", + " [-3.9418e-03, -1.5774e-02, 3.1421e-02]],\n", + " \n", + " [[-2.3044e-02, 2.6058e-02, 4.0771e-02],\n", + " [ 2.8217e-02, -1.3101e-02, -1.9169e-02],\n", + " [ 1.6696e-02, -2.9469e-02, 3.0725e-02]],\n", + " \n", + " [[-1.7412e-02, 5.9761e-03, 1.4802e-02],\n", + " [-9.6282e-03, -2.5679e-02, -3.2987e-02],\n", + " [-2.8646e-02, 2.0080e-02, -1.1815e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.5162e-02, 3.7288e-02, -5.4347e-02],\n", + " [-4.5577e-03, 8.9547e-05, 1.2762e-02],\n", + " [ 2.4004e-02, 1.5974e-02, -3.9689e-03]],\n", + " \n", + " [[ 2.0893e-02, 3.8545e-02, -4.7115e-04],\n", + " [ 2.0476e-02, -5.5991e-02, 2.3132e-02],\n", + " [-8.4840e-03, 4.0287e-03, -4.3514e-02]],\n", + " \n", + " [[ 4.1755e-03, 3.6891e-02, 2.2836e-02],\n", + " [ 1.5228e-02, -3.1077e-02, -4.8275e-03],\n", + " [ 3.4390e-02, -2.4285e-03, -1.6479e-02]]],\n", + " \n", + " \n", + " [[[ 3.1451e-02, 2.7231e-02, -4.8490e-02],\n", + " [-2.9251e-02, 7.3146e-03, -8.5553e-03],\n", + " [-3.0377e-02, 5.3074e-02, -3.5402e-02]],\n", + " \n", + " [[ 1.6317e-03, 2.9512e-02, 3.7927e-02],\n", + " [ 3.9694e-03, -1.0264e-02, 2.0400e-02],\n", + " [-5.0912e-03, 1.0976e-02, 2.0343e-02]],\n", + " \n", + " [[ 3.3715e-02, 1.8947e-02, 3.7091e-02],\n", + " [ 4.4463e-04, 1.9399e-02, 3.0023e-02],\n", + " [ 4.5544e-02, -2.2597e-02, -3.3840e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 5.0087e-02, 2.4138e-02, -1.6329e-02],\n", + " [-4.9508e-03, 1.4488e-02, -1.2929e-02],\n", + " [-2.0144e-02, 2.3250e-02, -4.7392e-03]],\n", + " \n", + " [[ 3.2129e-02, 2.7853e-02, 1.8916e-02],\n", + " [-1.9839e-02, -4.3275e-02, -2.0834e-03],\n", + " [-1.0927e-02, 4.9429e-03, 1.1458e-02]],\n", + " \n", + " [[-4.0182e-02, 2.0645e-02, 1.3425e-02],\n", + " [ 1.5748e-02, -2.5606e-02, 2.1066e-02],\n", + " [-2.5730e-02, -3.5284e-02, 4.0417e-02]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 3.2400e-02, 2.7387e-02, -4.1836e-02],\n", + " [-3.3519e-02, -3.0823e-02, -3.9949e-02],\n", + " [ 2.9098e-02, 2.3158e-02, -2.2002e-02]],\n", + " \n", + " [[-4.4127e-02, -5.0314e-02, 5.4241e-02],\n", + " [ 4.1223e-02, 2.0904e-02, 2.4865e-02],\n", + " [-7.3921e-03, -2.4277e-02, -3.4978e-02]],\n", + " \n", + " [[ 2.1406e-02, -3.5557e-02, -1.4675e-02],\n", + " [ 1.6927e-03, -2.0588e-02, 2.9574e-03],\n", + " [ 5.3911e-03, 3.2478e-03, -1.5555e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.8776e-02, -2.9488e-02, -2.7173e-03],\n", + " [-1.1154e-02, -1.5518e-02, -1.3146e-02],\n", + " [ 2.7336e-02, -2.0737e-03, -1.8577e-03]],\n", + " \n", + " [[ 2.7353e-02, -5.7622e-02, -1.9810e-02],\n", + " [ 4.4887e-02, -2.4649e-02, -2.8831e-04],\n", + " [-1.7285e-02, 4.4593e-02, 5.0801e-03]],\n", + " \n", + " [[-1.0272e-02, -1.1177e-02, 1.9091e-02],\n", + " [ 2.6784e-02, -1.6878e-02, -2.2759e-02],\n", + " [-3.9895e-02, 1.9132e-02, 3.3437e-02]]],\n", + " \n", + " \n", + " [[[ 2.4377e-03, -3.6880e-02, 8.9704e-03],\n", + " [ 1.8340e-02, -3.8695e-02, -2.9911e-02],\n", + " [ 2.7198e-02, -2.6570e-02, -1.1830e-02]],\n", + " \n", + " [[-3.9071e-03, -9.8153e-03, -4.5207e-02],\n", + " [ 1.4953e-02, -2.0952e-02, 1.6540e-03],\n", + " [ 3.1941e-03, -1.7849e-02, 3.0970e-02]],\n", + " \n", + " [[-5.4628e-03, -1.0248e-02, 1.8554e-02],\n", + " [ 1.4621e-02, 3.0034e-02, -4.1184e-02],\n", + " [-3.1914e-02, -1.6138e-03, 3.5175e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.1999e-02, 3.7370e-02, -6.6605e-03],\n", + " [ 8.5423e-03, -2.3363e-02, 5.3106e-02],\n", + " [-3.8766e-02, -4.6355e-02, -2.1916e-02]],\n", + " \n", + " [[ 4.6545e-02, -1.3589e-02, -9.0086e-04],\n", + " [-2.2744e-02, -7.9690e-03, 4.1620e-02],\n", + " [-1.8275e-02, -1.1720e-02, -1.7068e-02]],\n", + " \n", + " [[ 4.2182e-03, -5.2635e-02, 3.7010e-02],\n", + " [-1.2204e-02, 1.6322e-02, 2.7914e-02],\n", + " [-1.5614e-03, 1.2287e-02, -3.1604e-02]]],\n", + " \n", + " \n", + " [[[ 3.3208e-02, 6.6656e-03, 8.7341e-03],\n", + " [ 1.7176e-02, -2.1905e-02, 3.8850e-02],\n", + " [-2.0100e-02, 1.8937e-03, 5.5718e-04]],\n", + " \n", + " [[-5.3009e-02, 2.7671e-03, -2.5455e-02],\n", + " [-4.1016e-02, 4.6348e-02, 1.0834e-02],\n", + " [-4.8903e-02, 4.2398e-02, 2.0632e-02]],\n", + " \n", + " [[-2.7099e-02, -4.9743e-03, -1.6784e-02],\n", + " [ 1.2828e-02, -6.0318e-04, -4.7372e-02],\n", + " [-3.3223e-02, -5.8787e-02, -6.7474e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 6.1073e-03, -3.3571e-02, 1.1209e-02],\n", + " [-3.4866e-02, -3.0714e-02, 2.1105e-03],\n", + " [ 5.3477e-02, -2.4217e-02, -2.5762e-02]],\n", + " \n", + " [[ 2.4611e-02, -1.8060e-02, -2.9728e-02],\n", + " [ 3.6908e-02, -2.9815e-02, 3.9181e-02],\n", + " [-4.4588e-02, 4.1345e-02, -1.4445e-02]],\n", + " \n", + " [[ 1.0095e-02, 4.4746e-02, -3.2337e-02],\n", + " [-8.9715e-03, -2.4242e-02, -1.0168e-02],\n", + " [ 3.7602e-02, 1.3277e-02, 4.1139e-02]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.bn1.weight',\n", + " tensor([-5.9605e-08, 2.3842e-07, 5.9605e-07, -1.0133e-06, -9.5367e-07,\n", + " 2.2650e-06, -3.5763e-07, 4.7684e-07, 8.3447e-07, -5.9605e-07,\n", + " -4.7684e-07, -9.5367e-07, -2.9802e-07, -1.7881e-07, -5.9605e-08,\n", + " 8.3447e-07, -8.3447e-07, 1.1921e-06, -2.3842e-07, -2.9802e-07,\n", + " 2.6226e-06, -7.7486e-07, -5.9605e-07, 1.1921e-06, 9.5367e-07,\n", + " -5.9605e-07, 7.1526e-07, 3.5763e-07, -9.5367e-07, 4.7684e-07,\n", + " -3.5763e-07, 9.5367e-07, 1.3113e-06, -2.9802e-07, -1.3709e-06,\n", + " -1.4305e-06, 1.5497e-06, 0.0000e+00, -1.7285e-06, 4.7684e-07,\n", + " -5.9605e-07, 7.1526e-07, 1.3113e-06, 5.9605e-07, -8.9407e-07,\n", + " -4.1723e-07, -1.1325e-06, 1.1921e-07, -6.5565e-07, -1.4901e-06,\n", + " 5.9605e-07, -5.3644e-07, -5.9605e-08, 3.5763e-07, 2.3842e-06,\n", + " 4.7684e-07, -7.1526e-07, 1.1921e-06, -1.3709e-06, 1.4305e-06,\n", + " -4.7684e-07, -1.0133e-06, 2.2650e-06, -3.5763e-07, -6.5565e-07,\n", + " -4.7684e-07, -7.7486e-07, 1.3113e-06, -5.9605e-08, -8.3447e-07,\n", + " 5.9605e-07, -1.9670e-06, 5.9605e-07, 1.7881e-06, -6.5565e-07,\n", + " 4.7684e-07, 2.1458e-06, 8.3447e-07, 9.5367e-07, 0.0000e+00,\n", + " -3.5763e-07, -5.9605e-07, 5.9605e-07, 1.4305e-06, -1.1325e-06,\n", + " -5.3644e-07, 4.7684e-07, 1.0729e-06, -1.0729e-06, -2.3842e-07,\n", + " -2.9802e-07, -6.5565e-07, 1.3113e-06, -6.5565e-07, 2.3842e-07,\n", + " 4.7684e-07, -8.3447e-07, 9.5367e-07, 1.1921e-07, -1.1921e-07,\n", + " -5.9605e-07, -1.1325e-06, 1.1921e-07, -1.1325e-06, 5.9605e-07,\n", + " -5.3644e-07, 2.3842e-07, 5.9605e-07, 8.3447e-07, 1.1921e-07,\n", + " 2.3842e-07, -7.1526e-07, -2.8014e-06, 0.0000e+00, -1.6093e-06,\n", + " -4.1723e-07, -1.1921e-07, -9.5367e-07, 5.9605e-07, -4.7684e-07,\n", + " 4.7684e-07, -1.1921e-07, 1.1921e-07, 2.3842e-07, -1.3709e-06,\n", + " -9.5367e-07, 2.3842e-07, 1.1921e-07], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.bn1.bias',\n", + " tensor([-2.1467e-07, -6.6178e-07, 1.7024e-07, 2.0613e-07, -2.4169e-07,\n", + " 1.9816e-06, -1.1832e-06, -1.5366e-07, 5.6462e-07, 1.5284e-08,\n", + " 4.0878e-07, -4.9318e-07, -5.0621e-07, -6.0479e-07, 1.7576e-06,\n", + " 8.4282e-07, -5.4288e-07, 5.3258e-07, 5.2341e-07, 5.0324e-07,\n", + " 1.7585e-06, -8.1943e-07, 6.5351e-07, 5.5759e-07, -1.0581e-06,\n", + " 9.2307e-07, 9.9461e-08, 1.4788e-07, -1.1720e-06, -1.6742e-07,\n", + " 8.3141e-07, 2.1564e-07, -7.3767e-07, 5.7207e-07, -3.0203e-07,\n", + " -1.9044e-06, 4.9593e-08, 1.4940e-06, 3.8589e-08, 9.1724e-08,\n", + " -5.0841e-07, 8.4666e-07, 1.4527e-06, 1.0706e-06, -1.6604e-06,\n", + " -1.1521e-07, -5.2608e-07, 1.7766e-06, 5.5275e-07, -1.6253e-06,\n", + " 4.1272e-07, -9.8920e-07, 5.8334e-07, 1.0732e-06, 1.8116e-06,\n", + " 1.1440e-06, -1.2197e-06, 3.4906e-07, -6.0978e-07, 8.8430e-07,\n", + " -6.6463e-07, -9.4432e-07, 1.4670e-06, -1.3969e-06, -6.5653e-07,\n", + " 7.5088e-07, -1.4769e-06, -4.7031e-07, -3.6269e-07, -2.9390e-08,\n", + " 1.1469e-07, -5.4929e-07, 6.8075e-07, 8.0234e-07, -4.5739e-07,\n", + " -4.1138e-07, 1.3506e-06, 1.2868e-06, -1.2592e-07, -9.3637e-07,\n", + " -7.6653e-07, -1.9897e-07, 6.9171e-07, 3.0005e-08, -2.2339e-06,\n", + " -1.5658e-07, -1.1111e-06, 1.6107e-06, 1.1992e-06, 8.4681e-07,\n", + " 1.3424e-06, 1.3030e-06, 7.1811e-07, 7.8723e-07, 4.0089e-07,\n", + " 5.9138e-07, -8.1441e-07, 7.5653e-07, -2.5908e-07, 4.2406e-07,\n", + " -3.7178e-07, -5.6875e-07, -8.2396e-07, -1.0907e-06, -3.1799e-07,\n", + " -5.7387e-07, 4.5297e-07, 1.1021e-06, 1.4436e-06, -2.8676e-07,\n", + " 4.7926e-08, 7.7351e-07, -1.5734e-06, -2.5014e-07, -9.6145e-07,\n", + " 1.0515e-06, -4.0971e-07, -9.0253e-07, 4.4345e-07, -1.4786e-07,\n", + " -6.9746e-07, 5.1135e-07, -2.9096e-07, -6.8238e-08, -1.8120e-06,\n", + " 8.7449e-07, 4.9817e-07, 1.2301e-06], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.conv2.weight',\n", + " tensor([[[[ 0.0061, 0.0248, 0.0266],\n", + " [-0.0154, -0.0017, -0.0064],\n", + " [ 0.0182, -0.0421, -0.0336]],\n", + " \n", + " [[ 0.0341, -0.0028, -0.0326],\n", + " [-0.0029, -0.0011, -0.0415],\n", + " [-0.0231, -0.0174, -0.0123]],\n", + " \n", + " [[-0.0016, -0.0241, -0.0310],\n", + " [-0.0170, 0.0123, -0.0002],\n", + " [ 0.0171, 0.0016, -0.0373]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0090, 0.0006, 0.0137],\n", + " [-0.0105, -0.0244, 0.0183],\n", + " [-0.0004, 0.0322, 0.0156]],\n", + " \n", + " [[-0.0249, -0.0007, -0.0213],\n", + " [ 0.0010, -0.0180, 0.0122],\n", + " [-0.0211, 0.0167, -0.0023]],\n", + " \n", + " [[-0.0242, 0.0292, 0.0299],\n", + " [ 0.0119, 0.0187, 0.0037],\n", + " [ 0.0297, 0.0022, 0.0109]]],\n", + " \n", + " \n", + " [[[-0.0269, -0.0043, 0.0191],\n", + " [-0.0209, -0.0125, -0.0290],\n", + " [ 0.0195, -0.0040, 0.0271]],\n", + " \n", + " [[-0.0198, -0.0021, -0.0042],\n", + " [-0.0018, 0.0042, -0.0213],\n", + " [-0.0078, -0.0174, -0.0072]],\n", + " \n", + " [[ 0.0318, 0.0153, 0.0026],\n", + " [-0.0350, 0.0107, 0.0009],\n", + " [-0.0130, 0.0201, 0.0011]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0102, -0.0144, -0.0065],\n", + " [-0.0047, 0.0310, -0.0157],\n", + " [ 0.0013, 0.0064, 0.0090]],\n", + " \n", + " [[-0.0095, 0.0098, 0.0033],\n", + " [ 0.0423, 0.0246, 0.0002],\n", + " [-0.0016, -0.0120, 0.0049]],\n", + " \n", + " [[-0.0198, 0.0237, -0.0067],\n", + " [-0.0096, 0.0138, 0.0137],\n", + " [ 0.0183, 0.0119, 0.0224]]],\n", + " \n", + " \n", + " [[[ 0.0297, 0.0352, -0.0161],\n", + " [-0.0172, -0.0186, -0.0146],\n", + " [-0.0212, 0.0170, 0.0156]],\n", + " \n", + " [[ 0.0327, 0.0215, 0.0073],\n", + " [-0.0230, 0.0068, 0.0204],\n", + " [-0.0469, -0.0293, -0.0211]],\n", + " \n", + " [[-0.0200, -0.0304, -0.0019],\n", + " [-0.0006, 0.0144, 0.0267],\n", + " [ 0.0174, -0.0108, -0.0253]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0021, -0.0095, -0.0195],\n", + " [ 0.0152, 0.0130, -0.0070],\n", + " [ 0.0251, 0.0134, 0.0258]],\n", + " \n", + " [[ 0.0183, -0.0031, 0.0047],\n", + " [ 0.0072, -0.0249, -0.0123],\n", + " [ 0.0281, -0.0248, -0.0010]],\n", + " \n", + " [[ 0.0252, 0.0180, -0.0235],\n", + " [-0.0014, 0.0064, -0.0137],\n", + " [ 0.0230, 0.0008, -0.0191]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 0.0282, -0.0075, 0.0432],\n", + " [-0.0325, -0.0211, -0.0156],\n", + " [-0.0047, -0.0331, -0.0143]],\n", + " \n", + " [[-0.0109, 0.0056, -0.0271],\n", + " [ 0.0308, 0.0146, 0.0129],\n", + " [-0.0129, 0.0256, 0.0154]],\n", + " \n", + " [[-0.0321, 0.0118, 0.0130],\n", + " [-0.0057, -0.0097, 0.0054],\n", + " [ 0.0159, 0.0210, 0.0006]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0046, 0.0206, 0.0274],\n", + " [ 0.0140, 0.0397, 0.0243],\n", + " [-0.0147, -0.0160, 0.0216]],\n", + " \n", + " [[ 0.0203, -0.0391, 0.0018],\n", + " [ 0.0050, 0.0139, -0.0256],\n", + " [ 0.0157, 0.0140, 0.0029]],\n", + " \n", + " [[-0.0159, -0.0278, 0.0334],\n", + " [ 0.0070, -0.0179, 0.0059],\n", + " [-0.0117, 0.0188, -0.0005]]],\n", + " \n", + " \n", + " [[[ 0.0179, -0.0143, 0.0172],\n", + " [ 0.0228, 0.0128, -0.0189],\n", + " [-0.0170, -0.0010, 0.0167]],\n", + " \n", + " [[ 0.0291, -0.0274, -0.0327],\n", + " [ 0.0033, -0.0214, 0.0099],\n", + " [ 0.0255, -0.0062, 0.0171]],\n", + " \n", + " [[-0.0347, -0.0177, 0.0207],\n", + " [-0.0179, 0.0280, -0.0178],\n", + " [ 0.0211, 0.0030, -0.0068]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0055, 0.0100, -0.0051],\n", + " [ 0.0262, -0.0121, 0.0355],\n", + " [-0.0250, 0.0059, -0.0248]],\n", + " \n", + " [[ 0.0187, 0.0238, 0.0206],\n", + " [-0.0002, 0.0277, 0.0077],\n", + " [ 0.0068, 0.0034, -0.0409]],\n", + " \n", + " [[-0.0030, 0.0121, 0.0156],\n", + " [ 0.0125, -0.0119, 0.0086],\n", + " [-0.0004, 0.0258, -0.0002]]],\n", + " \n", + " \n", + " [[[ 0.0172, -0.0334, -0.0074],\n", + " [-0.0184, 0.0094, -0.0019],\n", + " [ 0.0407, 0.0041, 0.0084]],\n", + " \n", + " [[ 0.0339, 0.0150, -0.0341],\n", + " [-0.0116, 0.0231, 0.0189],\n", + " [ 0.0049, 0.0024, 0.0197]],\n", + " \n", + " [[ 0.0343, -0.0007, 0.0028],\n", + " [ 0.0260, -0.0005, -0.0192],\n", + " [-0.0197, -0.0036, -0.0064]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0159, 0.0336, 0.0037],\n", + " [-0.0347, -0.0117, -0.0147],\n", + " [-0.0152, 0.0203, -0.0101]],\n", + " \n", + " [[-0.0228, 0.0130, 0.0209],\n", + " [ 0.0134, -0.0121, 0.0251],\n", + " [ 0.0195, 0.0150, 0.0038]],\n", + " \n", + " [[ 0.0225, 0.0109, -0.0084],\n", + " [-0.0076, 0.0192, -0.0155],\n", + " [-0.0044, -0.0217, -0.0129]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.bn2.weight',\n", + " tensor([-2.9802e-07, -9.5367e-07, -1.7285e-06, -5.9605e-07, -1.8477e-06,\n", + " 1.5497e-06, -1.7881e-07, 3.5763e-07, 7.1526e-07, 3.5763e-07,\n", + " 7.1526e-07, -1.3113e-06, -1.7881e-07, 1.9073e-06, -1.3113e-06,\n", + " 3.5763e-07, 4.7684e-07, 2.1458e-06, 5.9605e-07, 3.5763e-07,\n", + " 4.7684e-07, 7.1526e-07, -1.1921e-06, 3.5763e-07, -3.5763e-07,\n", + " 3.5763e-07, 2.3842e-07, -4.7684e-07, 2.3842e-06, 1.1921e-07,\n", + " 1.6689e-06, 1.0729e-06, -1.1921e-07, 1.5497e-06, 4.7684e-07,\n", + " 2.3842e-07, -9.5367e-07, 3.5763e-07, 9.5367e-07, 5.9605e-07,\n", + " 5.9605e-07, -7.1526e-07, 2.3842e-07, 1.1921e-07, -3.5763e-07,\n", + " -1.7881e-07, -1.6093e-06, -4.7684e-07, -2.3842e-06, -1.0729e-06,\n", + " -1.0729e-06, -5.3644e-07, 5.9605e-07, -4.7684e-07, -4.1723e-07,\n", + " -1.7285e-06, 2.3842e-07, 8.3447e-07, -8.9407e-07, 7.1526e-07,\n", + " -4.1723e-07, 7.1526e-07, -6.5565e-07, -3.2187e-06, 3.5763e-07,\n", + " -5.3644e-07, -1.1921e-07, -3.5763e-07, 7.1526e-07, 7.1526e-07,\n", + " 9.5367e-07, -1.9073e-06, 1.0729e-06, -8.3447e-07, -4.1723e-07,\n", + " 0.0000e+00, 7.1526e-07, 4.7684e-07, 3.5763e-07, 3.5763e-07,\n", + " -5.9605e-08, 1.1921e-07, -9.5367e-07, -5.9605e-07, 1.1921e-07,\n", + " 4.7684e-07, 8.3447e-07, -1.3113e-06, 2.2650e-06, 7.1526e-07,\n", + " 0.0000e+00, -5.9605e-07, 4.7684e-07, 3.5763e-07, -5.9605e-08,\n", + " -1.2517e-06, -1.1921e-07, 7.1526e-07, -3.5763e-07, 5.9605e-07,\n", + " -7.7486e-07, 3.2187e-06, 9.5367e-07, -5.9605e-08, -7.1526e-07,\n", + " -6.5565e-07, -7.1526e-07, 1.4305e-06, -2.9802e-07, 8.3447e-07,\n", + " 2.3842e-07, -4.7684e-07, 3.5763e-07, 1.0729e-06, -1.2517e-06,\n", + " -7.7486e-07, 2.5034e-06, -6.5565e-07, -1.7881e-07, -5.9605e-08,\n", + " -5.9605e-08, 2.3842e-07, -2.1458e-06, -1.8477e-06, -2.3246e-06,\n", + " 0.0000e+00, 2.3842e-07, 4.7684e-07], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.bn2.bias',\n", + " tensor([-5.5672e-07, -5.2195e-07, -7.3200e-07, -1.3146e-06, -5.1317e-07,\n", + " 2.8345e-07, 2.3253e-07, 1.2570e-06, 5.3647e-08, 5.4823e-09,\n", + " -1.7705e-07, -2.2771e-06, 4.4279e-08, -4.5075e-08, -7.2442e-07,\n", + " 1.7921e-07, 1.1754e-06, 7.2712e-07, -1.3447e-06, -4.6385e-07,\n", + " 2.7598e-07, 1.0195e-07, -4.0322e-07, 7.0501e-07, 3.4089e-07,\n", + " -4.2139e-07, -4.5342e-07, 1.1544e-06, 2.0046e-06, -2.1043e-07,\n", + " -1.6187e-06, 3.2673e-07, 6.9511e-07, -8.8326e-07, 8.7073e-08,\n", + " 6.0346e-07, -1.2989e-06, -3.6423e-07, -3.2090e-07, 3.3890e-07,\n", + " -2.5060e-07, -1.7383e-06, 5.6231e-07, 1.0725e-06, -4.4510e-07,\n", + " 6.6358e-07, -3.3527e-07, 4.8670e-07, -5.1463e-07, -4.9155e-07,\n", + " -5.0953e-07, 3.5141e-07, 4.6948e-07, 1.5160e-06, 2.2840e-07,\n", + " 4.2011e-07, -1.2933e-06, 7.4209e-07, -1.1028e-06, -1.8627e-07,\n", + " 2.3393e-07, -3.6756e-07, -8.0544e-07, -2.6689e-08, -5.6918e-07,\n", + " -1.3877e-06, 4.1389e-07, -2.4542e-07, 5.1264e-08, -5.3017e-07,\n", + " -1.1562e-06, -8.6156e-07, 1.1412e-06, 5.1820e-07, 8.3670e-07,\n", + " -2.1070e-07, -1.9332e-07, 4.2176e-07, 7.6960e-08, 4.6864e-07,\n", + " 7.6292e-07, 4.0845e-08, -7.3546e-07, 3.2210e-07, 3.4499e-07,\n", + " 4.2282e-07, 2.0984e-07, 4.4692e-07, 2.7307e-06, 8.0059e-07,\n", + " 4.5338e-07, -8.3362e-07, 4.6176e-07, -6.4901e-07, -4.8625e-07,\n", + " 8.6808e-07, -1.1363e-07, 8.4456e-07, 1.3166e-06, 9.8014e-07,\n", + " 2.2637e-07, 8.4249e-07, 2.0628e-07, -5.5369e-08, -1.6014e-06,\n", + " -3.6345e-07, 2.0868e-06, 1.5960e-06, 2.1796e-07, 7.7901e-07,\n", + " 4.0972e-07, -1.4152e-07, -4.3252e-07, 1.0869e-06, -1.2064e-06,\n", + " -2.5503e-07, 6.4234e-07, -2.7155e-06, -6.4880e-07, 6.0029e-07,\n", + " 4.6931e-07, 1.1417e-06, -7.8651e-07, -4.1475e-07, -7.0440e-07,\n", + " 1.5567e-06, -3.1585e-07, -1.4987e-08], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.shortcut.0.weight',\n", + " tensor([[[[ 0.0434]],\n", + " \n", + " [[-0.0352]],\n", + " \n", + " [[-0.0098]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.0883]],\n", + " \n", + " [[-0.0434]],\n", + " \n", + " [[ 0.0117]]],\n", + " \n", + " \n", + " [[[-0.0396]],\n", + " \n", + " [[-0.1196]],\n", + " \n", + " [[-0.0617]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.0264]],\n", + " \n", + " [[-0.0807]],\n", + " \n", + " [[-0.0648]]],\n", + " \n", + " \n", + " [[[ 0.1829]],\n", + " \n", + " [[ 0.0400]],\n", + " \n", + " [[-0.0604]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0866]],\n", + " \n", + " [[ 0.0344]],\n", + " \n", + " [[-0.0640]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-0.0823]],\n", + " \n", + " [[ 0.1039]],\n", + " \n", + " [[ 0.0969]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0864]],\n", + " \n", + " [[ 0.1064]],\n", + " \n", + " [[ 0.0088]]],\n", + " \n", + " \n", + " [[[ 0.0982]],\n", + " \n", + " [[ 0.1408]],\n", + " \n", + " [[-0.0697]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.1474]],\n", + " \n", + " [[-0.0310]],\n", + " \n", + " [[ 0.1249]]],\n", + " \n", + " \n", + " [[[-0.0877]],\n", + " \n", + " [[ 0.0198]],\n", + " \n", + " [[ 0.0845]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.0722]],\n", + " \n", + " [[ 0.0428]],\n", + " \n", + " [[-0.0519]]]], device='cuda:0', grad_fn=)),\n", + " ('layer2.0.shortcut.1.weight',\n", + " tensor([-5.9605e-08, 8.3447e-07, 8.3447e-07, -1.7285e-06, 4.7684e-07,\n", + " 4.7684e-07, 1.1921e-07, 1.4305e-06, -7.1526e-07, -1.6093e-06,\n", + " -1.1921e-07, -3.3975e-06, 2.3842e-07, -1.5497e-06, 1.1921e-07,\n", + " 4.7684e-07, 4.7684e-07, 4.7684e-07, -8.9407e-07, -1.3113e-06,\n", + " 2.3842e-07, -1.1921e-07, -4.7684e-07, -1.1921e-07, 1.0729e-06,\n", + " 0.0000e+00, 4.7684e-07, 2.0266e-06, 5.9605e-07, -4.7684e-07,\n", + " -2.6226e-06, 4.7684e-07, 4.7684e-07, -1.6093e-06, -3.5763e-07,\n", + " -2.1458e-06, -7.7486e-07, 1.0729e-06, -5.3644e-07, -2.9802e-07,\n", + " 4.7684e-07, -1.0729e-06, 2.3842e-07, 2.3842e-07, -6.5565e-07,\n", + " 1.7881e-06, 1.4305e-06, 5.9605e-07, 9.5367e-07, 8.3447e-07,\n", + " 7.1526e-07, 3.5763e-07, -1.1921e-07, 1.1921e-06, -6.5565e-07,\n", + " -1.1921e-07, -2.2054e-06, 1.1921e-06, 3.5763e-07, -6.5565e-07,\n", + " 0.0000e+00, -5.3644e-07, -5.3644e-07, 1.7881e-06, 1.1921e-07,\n", + " 2.3842e-07, -5.9605e-07, -7.7486e-07, -1.3113e-06, -1.2517e-06,\n", + " -1.1325e-06, -1.7881e-07, 7.1526e-07, 4.7684e-07, 8.3447e-07,\n", + " 5.9605e-07, -1.1921e-07, -7.7486e-07, -1.5497e-06, -5.9605e-07,\n", + " 1.3113e-06, 1.1921e-07, -1.1921e-07, 2.3842e-07, 0.0000e+00,\n", + " -7.1526e-07, -4.1723e-07, 1.1921e-07, 5.9605e-07, 1.4305e-06,\n", + " 1.5497e-06, -1.1325e-06, 1.1921e-06, -8.3447e-07, 9.5367e-07,\n", + " 3.5763e-07, 1.1921e-06, 1.6689e-06, 1.0729e-06, 1.0729e-06,\n", + " 1.5497e-06, 0.0000e+00, 4.7684e-07, -1.7881e-07, -1.7285e-06,\n", + " -6.5565e-07, 1.5497e-06, 1.1921e-07, 7.1526e-07, 2.3842e-07,\n", + " -1.1921e-07, 3.5763e-07, -6.5565e-07, 1.1921e-07, -9.5367e-07,\n", + " -3.5763e-07, -5.3644e-07, -2.2054e-06, 1.1921e-07, 8.3447e-07,\n", + " 9.5367e-07, -6.5565e-07, 0.0000e+00, 7.1526e-07, 9.5367e-07,\n", + " 9.5367e-07, -7.1526e-07, -1.1325e-06], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer2.0.shortcut.1.bias',\n", + " tensor([-5.5672e-07, -5.2195e-07, -7.3200e-07, -1.3146e-06, -5.1317e-07,\n", + " 2.8345e-07, 2.3253e-07, 1.2570e-06, 5.3647e-08, 5.4823e-09,\n", + " -1.7705e-07, -2.2771e-06, 4.4279e-08, -4.5075e-08, -7.2442e-07,\n", + " 1.7921e-07, 1.1754e-06, 7.2712e-07, -1.3447e-06, -4.6385e-07,\n", + " 2.7598e-07, 1.0195e-07, -4.0322e-07, 7.0501e-07, 3.4089e-07,\n", + " -4.2139e-07, -4.5342e-07, 1.1544e-06, 2.0046e-06, -2.1043e-07,\n", + " -1.6187e-06, 3.2673e-07, 6.9511e-07, -8.8326e-07, 8.7073e-08,\n", + " 6.0346e-07, -1.2989e-06, -3.6423e-07, -3.2090e-07, 3.3890e-07,\n", + " -2.5060e-07, -1.7383e-06, 5.6231e-07, 1.0725e-06, -4.4510e-07,\n", + " 6.6358e-07, -3.3527e-07, 4.8670e-07, -5.1463e-07, -4.9155e-07,\n", + " -5.0953e-07, 3.5141e-07, 4.6948e-07, 1.5160e-06, 2.2840e-07,\n", + " 4.2011e-07, -1.2933e-06, 7.4209e-07, -1.1028e-06, -1.8627e-07,\n", + " 2.3393e-07, -3.6756e-07, -8.0544e-07, -2.6689e-08, -5.6918e-07,\n", + " -1.3877e-06, 4.1389e-07, -2.4542e-07, 5.1264e-08, -5.3017e-07,\n", + " -1.1562e-06, -8.6156e-07, 1.1412e-06, 5.1820e-07, 8.3670e-07,\n", + " -2.1070e-07, -1.9332e-07, 4.2176e-07, 7.6960e-08, 4.6864e-07,\n", + " 7.6292e-07, 4.0845e-08, -7.3546e-07, 3.2210e-07, 3.4499e-07,\n", + " 4.2282e-07, 2.0984e-07, 4.4692e-07, 2.7307e-06, 8.0059e-07,\n", + " 4.5338e-07, -8.3362e-07, 4.6176e-07, -6.4901e-07, -4.8625e-07,\n", + " 8.6808e-07, -1.1363e-07, 8.4456e-07, 1.3166e-06, 9.8014e-07,\n", + " 2.2637e-07, 8.4249e-07, 2.0628e-07, -5.5369e-08, -1.6014e-06,\n", + " -3.6345e-07, 2.0868e-06, 1.5960e-06, 2.1796e-07, 7.7901e-07,\n", + " 4.0972e-07, -1.4152e-07, -4.3252e-07, 1.0869e-06, -1.2064e-06,\n", + " -2.5503e-07, 6.4234e-07, -2.7155e-06, -6.4880e-07, 6.0029e-07,\n", + " 4.6931e-07, 1.1417e-06, -7.8651e-07, -4.1475e-07, -7.0440e-07,\n", + " 1.5567e-06, -3.1585e-07, -1.4987e-08], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer3.0.conv1.weight',\n", + " tensor([[[[-3.5847e-02, 2.0367e-02, -1.9664e-02],\n", + " [ 2.8290e-02, -2.4266e-02, -3.1215e-02],\n", + " [-2.2953e-02, 1.5775e-02, 1.7100e-02]],\n", + " \n", + " [[-1.1806e-02, 1.9726e-02, 7.9145e-03],\n", + " [ 1.7040e-02, 2.5175e-02, 4.0052e-03],\n", + " [-1.6612e-02, -1.9356e-02, -2.1757e-03]],\n", + " \n", + " [[ 1.7233e-02, -2.8255e-03, 1.0205e-02],\n", + " [-3.3721e-02, -4.3411e-02, -7.9912e-03],\n", + " [ 2.8045e-02, 1.4699e-03, -3.7339e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.1966e-02, -2.5432e-02, -9.9482e-03],\n", + " [-1.7659e-02, 8.5453e-03, 2.9822e-02],\n", + " [-2.4194e-02, 9.3559e-03, -1.0396e-02]],\n", + " \n", + " [[-1.3652e-03, -3.2642e-03, -1.3523e-02],\n", + " [ 1.2295e-02, -4.4702e-03, 7.8963e-03],\n", + " [-7.2810e-03, 8.4824e-03, -2.1397e-02]],\n", + " \n", + " [[-4.8522e-03, 9.7668e-03, 2.1152e-02],\n", + " [-1.5838e-02, -2.8412e-02, -2.3352e-02],\n", + " [ 6.0608e-03, 1.9728e-03, 3.4929e-02]]],\n", + " \n", + " \n", + " [[[-1.5575e-02, -2.0735e-02, -1.5452e-04],\n", + " [-1.3825e-02, -1.6600e-02, -5.9313e-03],\n", + " [-9.0447e-03, -2.0076e-03, 1.5479e-02]],\n", + " \n", + " [[ 1.7221e-02, -7.0439e-03, 3.1504e-02],\n", + " [ 8.7690e-03, 7.2981e-03, -1.0521e-02],\n", + " [-3.9003e-02, 3.0051e-02, -1.5717e-02]],\n", + " \n", + " [[-1.7256e-02, -2.6381e-02, 4.3414e-03],\n", + " [-8.3730e-03, -9.9747e-04, -1.9681e-02],\n", + " [ 5.4823e-03, 2.9394e-03, 2.0712e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.4154e-03, -2.6488e-02, -6.9511e-03],\n", + " [-5.0294e-03, 1.2300e-02, 3.5730e-02],\n", + " [-2.6664e-02, -9.6863e-03, -3.2638e-02]],\n", + " \n", + " [[ 9.2684e-03, -1.0054e-02, 2.4518e-02],\n", + " [-2.5752e-02, -1.2214e-02, -2.8972e-02],\n", + " [ 1.0317e-02, 9.6287e-03, -1.0533e-02]],\n", + " \n", + " [[ 1.3457e-02, -3.5893e-03, -1.4567e-03],\n", + " [-3.4104e-04, -1.3895e-02, 1.2368e-02],\n", + " [-1.8869e-02, -8.0909e-03, 1.4769e-02]]],\n", + " \n", + " \n", + " [[[ 2.2298e-02, -9.7045e-03, -1.3930e-02],\n", + " [ 2.9282e-03, 2.1107e-02, -8.0453e-03],\n", + " [ 2.2023e-02, 1.5977e-02, -1.6384e-02]],\n", + " \n", + " [[ 1.2560e-02, -8.2418e-03, -2.8852e-02],\n", + " [-1.1178e-02, -1.7865e-02, 3.0926e-02],\n", + " [-2.5572e-02, 2.7698e-02, 1.7171e-02]],\n", + " \n", + " [[-1.1127e-02, -1.9849e-02, -2.7101e-02],\n", + " [-1.0094e-02, -7.5767e-04, -2.7074e-05],\n", + " [ 1.2436e-02, -4.1406e-02, 7.5492e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.6541e-02, 3.7938e-02, -7.3941e-03],\n", + " [-2.7031e-02, -3.6088e-02, 2.0318e-02],\n", + " [ 1.1700e-02, -1.7650e-02, -4.3927e-03]],\n", + " \n", + " [[-3.3291e-02, -1.3972e-02, 4.2048e-02],\n", + " [-6.9396e-03, -6.5420e-04, -2.3174e-02],\n", + " [-1.5313e-02, -2.7187e-02, 1.1052e-02]],\n", + " \n", + " [[-2.7701e-02, 1.3175e-02, 3.0270e-02],\n", + " [-1.7060e-03, -1.9307e-02, 6.8763e-03],\n", + " [ 9.6039e-04, -1.3544e-02, -2.7310e-02]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 2.7231e-03, 2.1136e-02, 1.9296e-02],\n", + " [ 4.1838e-02, -1.8260e-02, -9.6020e-03],\n", + " [ 2.2156e-03, 5.0148e-04, 2.9776e-02]],\n", + " \n", + " [[ 2.8787e-02, -1.0886e-02, -1.5779e-02],\n", + " [-1.9401e-03, 3.2563e-02, 2.4798e-02],\n", + " [ 3.6458e-02, 2.2999e-02, 1.1850e-02]],\n", + " \n", + " [[-2.3522e-02, 1.5405e-02, 1.8846e-02],\n", + " [ 3.0848e-02, -2.2616e-02, -3.8844e-02],\n", + " [ 3.5323e-03, 2.6513e-02, 2.0811e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.2667e-03, -1.7920e-02, 9.5128e-03],\n", + " [-5.5903e-03, 1.9465e-03, 2.2540e-02],\n", + " [-9.2771e-03, -2.9740e-02, 1.1990e-02]],\n", + " \n", + " [[-1.9649e-02, -1.6388e-02, 9.8337e-03],\n", + " [ 2.1826e-02, -1.1939e-02, -2.0516e-03],\n", + " [ 2.5273e-02, 1.4294e-02, 2.3242e-03]],\n", + " \n", + " [[ 2.6544e-03, -1.2401e-03, -5.3836e-03],\n", + " [-1.0099e-02, -7.2049e-03, -3.8413e-02],\n", + " [ 8.2304e-03, -1.8288e-02, -2.1143e-02]]],\n", + " \n", + " \n", + " [[[-3.7608e-02, 1.7813e-02, 3.2610e-02],\n", + " [-1.3644e-02, -1.5045e-02, -1.8056e-02],\n", + " [ 2.1015e-02, 2.5689e-02, -1.7453e-02]],\n", + " \n", + " [[-3.7034e-03, -1.6591e-02, 1.2074e-02],\n", + " [ 2.5513e-03, 1.6834e-02, -8.4531e-03],\n", + " [ 2.7908e-03, 2.4256e-02, 1.1538e-02]],\n", + " \n", + " [[ 2.4028e-02, -9.6117e-03, 9.3481e-03],\n", + " [-1.5518e-02, -2.1593e-02, 7.6084e-03],\n", + " [-1.1305e-02, 2.6390e-03, -2.6524e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.0536e-02, -1.1165e-02, -3.6322e-02],\n", + " [-8.2758e-03, 2.4712e-02, -5.8127e-03],\n", + " [-2.0468e-04, -1.3327e-02, -2.2236e-02]],\n", + " \n", + " [[-1.1215e-02, -3.1569e-03, -2.3264e-02],\n", + " [-2.2216e-02, 2.6291e-02, 1.5805e-02],\n", + " [ 1.9801e-02, 6.9040e-03, -1.2441e-02]],\n", + " \n", + " [[ 5.4119e-03, -2.0331e-02, -1.8075e-02],\n", + " [ 1.8203e-02, -1.1281e-02, 1.0496e-02],\n", + " [ 4.4981e-03, 2.6457e-02, 1.5682e-02]]],\n", + " \n", + " \n", + " [[[ 6.7285e-03, -1.0132e-02, 2.6872e-03],\n", + " [-3.8748e-03, 2.0391e-02, 1.6657e-02],\n", + " [-1.4085e-02, -3.3888e-02, -1.0991e-02]],\n", + " \n", + " [[-1.2712e-02, -3.6912e-02, 2.5412e-03],\n", + " [ 3.5808e-03, -2.4707e-02, 1.4955e-02],\n", + " [-1.0587e-02, 2.8549e-03, 1.1967e-02]],\n", + " \n", + " [[ 2.8115e-02, 1.6836e-02, -2.1065e-02],\n", + " [-9.3984e-03, -2.6535e-02, -6.5040e-03],\n", + " [-1.5094e-02, -7.1796e-03, 1.4032e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.6607e-02, -2.3249e-02, 2.2395e-02],\n", + " [ 1.1159e-02, -2.7459e-03, 6.7371e-03],\n", + " [ 1.3695e-02, -2.3425e-02, 1.2643e-02]],\n", + " \n", + " [[-2.2071e-02, 1.2437e-02, 1.6717e-02],\n", + " [-2.1038e-03, -3.6188e-03, 2.1535e-02],\n", + " [-4.1022e-02, -1.8896e-02, -9.2582e-03]],\n", + " \n", + " [[-2.1602e-02, -1.3935e-02, -1.4913e-02],\n", + " [-3.8020e-03, 1.6783e-02, -4.2063e-02],\n", + " [ 3.1403e-02, -9.7103e-04, -2.7294e-03]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer3.0.bn1.weight',\n", + " tensor([ 1.1921e-07, 3.5763e-07, 8.3447e-07, 3.5763e-07, 3.5763e-07,\n", + " -2.3842e-07, -8.9407e-07, 1.1921e-07, -8.9407e-07, 4.7684e-07,\n", + " 1.1921e-06, 1.1921e-07, -3.5763e-07, 3.5763e-07, -8.9407e-07,\n", + " 0.0000e+00, -5.9605e-07, -2.3842e-07, 5.9605e-07, 5.9605e-07,\n", + " 1.4305e-06, -5.9605e-07, -1.1921e-07, 1.1921e-07, 9.5367e-07,\n", + " 1.1921e-07, 5.9605e-07, 4.7684e-07, -2.3842e-07, 1.1921e-07,\n", + " 9.5367e-07, -2.9802e-07, -1.7881e-07, 0.0000e+00, -2.9802e-07,\n", + " -6.5565e-07, -5.9605e-07, -1.1921e-07, -1.1921e-07, -3.5763e-07,\n", + " -6.5565e-07, -2.3842e-07, 5.9605e-07, 1.1921e-07, 0.0000e+00,\n", + " -7.7486e-07, 7.1526e-07, 2.3842e-07, 5.9605e-07, 0.0000e+00,\n", + " 0.0000e+00, -4.1723e-07, -8.9407e-07, -1.1921e-07, 3.5763e-07,\n", + " -5.9605e-08, -5.9605e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", + " 3.5763e-07, -1.1921e-07, 3.5763e-07, -3.5763e-07, 2.3842e-07,\n", + " -7.7486e-07, 1.3113e-06, -7.7486e-07, -1.7881e-07, -6.5565e-07,\n", + " 1.1921e-07, -1.7881e-07, 3.5763e-07, 1.1921e-07, -2.3842e-07,\n", + " -1.2517e-06, 5.9605e-07, -1.1921e-07, -1.3113e-06, -1.0133e-06,\n", + " -1.7881e-07, -5.3644e-07, 1.0729e-06, -2.9802e-07, -8.3447e-07,\n", + " 2.3842e-07, -2.3842e-07, 8.3447e-07, 2.3842e-07, -7.1526e-07,\n", + " 3.5763e-07, -2.3842e-07, -2.9802e-07, -1.1921e-06, 5.9605e-07,\n", + " 8.3447e-07, -9.5367e-07, 9.5367e-07, 2.3842e-07, -1.2517e-06,\n", + " -8.3447e-07, -5.9605e-07, 1.9073e-06, 1.1921e-07, 2.3842e-07,\n", + " -5.9605e-08, -3.5763e-07, -1.2517e-06, 2.3842e-07, 1.1921e-06,\n", + " -2.3842e-07, 3.5763e-07, 8.3447e-07, 3.5763e-07, -1.1921e-07,\n", + " -4.7684e-07, 1.1921e-06, 3.5763e-07, -1.7881e-06, 8.3447e-07,\n", + " 3.5763e-07, -4.7684e-07, 4.7684e-07, 8.3447e-07, -4.7684e-07,\n", + " -2.3842e-07, -3.5763e-07, 1.1921e-07, -4.1723e-07, -6.5565e-07,\n", + " -1.1325e-06, 7.1526e-07, 1.0729e-06, -9.5367e-07, 2.3842e-07,\n", + " 1.4305e-06, 1.0729e-06, 1.1921e-07, 0.0000e+00, 9.5367e-07,\n", + " 4.7684e-07, -5.9605e-08, 7.1526e-07, -1.0729e-06, -4.1723e-07,\n", + " -6.5565e-07, -7.7486e-07, -1.7881e-07, 8.3447e-07, 4.7684e-07,\n", + " -1.1921e-07, -7.1526e-07, 0.0000e+00, -4.7684e-07, -2.9802e-07,\n", + " -2.9802e-07, -5.3644e-07, 7.1526e-07, 2.3842e-07, -6.5565e-07,\n", + " -7.7486e-07, -1.1921e-07, -2.3842e-07, -7.7486e-07, -1.1921e-07,\n", + " 0.0000e+00, 2.3842e-07, 1.1921e-07, 4.7684e-07, -1.5497e-06,\n", + " -4.7684e-07, 3.5763e-07, 1.4305e-06, 0.0000e+00, 1.1921e-06,\n", + " -5.9605e-08, -5.9605e-08, 1.1921e-07, -3.5763e-07, -5.3644e-07,\n", + " 7.1526e-07, -5.9605e-08, -7.1526e-07, 4.7684e-07, 5.9605e-07,\n", + " -6.5565e-07, 1.1921e-07, 1.4305e-06, 5.9605e-07, -1.6093e-06,\n", + " -8.9407e-07, 3.5763e-07, 1.3113e-06, -5.3644e-07, -7.7486e-07,\n", + " 1.1921e-07, -1.7881e-07, -1.7881e-07, -4.7684e-07, -1.7881e-07,\n", + " 3.5763e-07, 1.1921e-07, -9.5367e-07, -5.9605e-08, -8.3447e-07,\n", + " -1.1921e-06, 1.1921e-07, 0.0000e+00, 7.1526e-07, -4.7684e-07,\n", + " 2.3842e-07, -5.3644e-07, -2.9802e-07, -1.1921e-07, 2.3842e-07,\n", + " 2.3842e-07, 1.3113e-06, 1.0729e-06, 2.3842e-07, 1.0729e-06,\n", + " 1.0729e-06, -8.3447e-07, -5.9605e-08, -2.3842e-07, 2.3842e-07,\n", + " 1.0729e-06, 1.1921e-07, -9.5367e-07, 1.0729e-06, -2.3842e-07,\n", + " -6.5565e-07, 2.3842e-07, 3.5763e-07, -5.3644e-07, -8.3447e-07,\n", + " -8.3447e-07, -1.2517e-06, -1.1921e-07, -2.9802e-07, -2.3842e-07,\n", + " -3.5763e-07, -2.3842e-07, 2.3842e-07, -4.7684e-07, 1.0729e-06,\n", + " -1.7881e-07, 1.6689e-06, 4.7684e-07, 5.9605e-07, 3.5763e-07,\n", + " -4.7684e-07, 1.1921e-07, 1.1921e-07, 5.9605e-07, -5.9605e-08,\n", + " 5.9605e-07], device='cuda:0', grad_fn=)),\n", + " ('layer3.0.bn1.bias',\n", + " tensor([ 1.0567e-07, -2.2608e-07, 7.7980e-07, 6.0618e-07, 7.4346e-08,\n", + " 7.4604e-08, 5.1290e-08, 5.6766e-07, -4.3678e-07, -1.2464e-07,\n", + " 6.4503e-07, -1.6041e-07, -6.4229e-07, 2.4704e-08, -1.0346e-06,\n", + " 1.8932e-08, -1.6404e-07, -5.9393e-07, 6.4292e-07, 6.3990e-07,\n", + " 5.6315e-07, -7.0395e-07, 1.6254e-07, -1.5116e-07, 1.1146e-06,\n", + " 6.2241e-08, 2.4521e-07, 4.1210e-07, 2.9143e-07, 4.8081e-07,\n", + " 4.0388e-07, 2.9603e-07, -4.0308e-08, -3.8228e-07, 9.7999e-08,\n", + " -4.1132e-07, -8.2195e-07, -4.1296e-07, -4.3110e-07, 1.7718e-08,\n", + " -3.7168e-07, 9.9837e-08, -3.3698e-07, -5.6538e-07, -5.1895e-07,\n", + " -5.0196e-07, 7.5443e-07, 5.2117e-07, -1.0318e-07, -3.8180e-08,\n", + " 4.2558e-07, -1.1005e-06, -4.8846e-07, -1.0395e-07, -4.9117e-08,\n", + " -4.2353e-07, -3.2792e-08, -2.4488e-07, 1.3006e-07, -2.3050e-07,\n", + " 2.1416e-07, 1.0950e-06, 4.0173e-07, -6.7249e-07, 1.1099e-06,\n", + " -2.5061e-07, 1.0650e-06, -2.3371e-08, -1.3088e-06, -3.3970e-07,\n", + " -2.0291e-07, -4.7117e-07, 3.8934e-07, 4.4948e-08, 4.9929e-07,\n", + " -9.4986e-07, 3.0889e-08, 2.5346e-08, -1.1193e-06, -6.0021e-07,\n", + " 3.7868e-07, -4.5946e-07, 1.1235e-06, -1.3853e-07, -7.3375e-07,\n", + " 8.9059e-08, -6.3013e-08, 3.5381e-07, -5.6841e-07, -2.3535e-07,\n", + " -8.4209e-08, -5.2336e-07, 2.7097e-07, -8.5456e-07, 9.0027e-07,\n", + " 6.0925e-07, -1.9199e-07, 2.8452e-07, -2.1114e-07, -2.6626e-07,\n", + " -2.6247e-07, -6.5535e-07, 1.1454e-06, -3.0620e-07, 4.0436e-07,\n", + " -2.2470e-07, -1.3247e-08, -1.7177e-07, 3.6733e-07, 7.3840e-07,\n", + " 1.8200e-07, -7.2507e-07, 4.0633e-07, -1.0148e-07, -7.2527e-07,\n", + " -1.3658e-07, 6.1720e-07, 6.5810e-07, -1.2376e-06, 5.8190e-07,\n", + " 2.8525e-08, -3.0644e-07, -3.4234e-07, 4.1653e-08, -2.9781e-07,\n", + " -2.4274e-07, -2.7573e-07, -1.0545e-07, 1.9631e-07, -4.2902e-07,\n", + " -1.0430e-06, -6.5661e-08, 5.7729e-07, -1.3148e-06, 5.0728e-07,\n", + " 9.7028e-07, 1.2680e-06, -4.9404e-07, -1.8790e-07, 4.8792e-07,\n", + " 2.9958e-07, 2.3991e-07, 7.8558e-07, -1.2066e-06, -1.9705e-07,\n", + " 2.7108e-07, -8.7893e-07, -8.4956e-08, 3.7687e-07, 5.8561e-07,\n", + " -1.1985e-06, -1.4008e-07, -3.7536e-07, -8.7938e-08, -1.9926e-07,\n", + " -5.6485e-07, -1.0832e-06, 1.0691e-06, 4.7958e-08, -3.2534e-07,\n", + " -1.9102e-07, -6.1334e-07, -1.2687e-07, -4.5103e-07, 2.8810e-07,\n", + " 2.6147e-07, 1.3258e-08, -1.7984e-07, 7.3758e-07, -1.0172e-06,\n", + " 3.7485e-07, 4.0969e-08, 1.3691e-06, 3.3062e-07, 3.2045e-07,\n", + " -2.2348e-07, -2.4110e-07, 7.5167e-07, -1.0377e-07, -7.7313e-07,\n", + " 5.8024e-07, -2.8523e-07, -6.8930e-07, 1.2996e-07, 4.2186e-07,\n", + " 4.0243e-07, 1.6612e-07, 1.4744e-06, 4.6616e-07, -1.1064e-06,\n", + " -1.2318e-06, 4.7393e-07, 7.5768e-07, -5.7123e-07, -5.3793e-08,\n", + " 2.6284e-07, 9.8255e-08, 1.1614e-08, -5.5923e-07, 1.3437e-07,\n", + " 4.5761e-07, 3.6241e-07, -3.8616e-07, 7.1338e-08, 3.5419e-07,\n", + " -5.7918e-07, 3.1612e-07, -3.0651e-07, 1.2589e-07, -6.9323e-08,\n", + " 1.7585e-07, -8.9432e-08, 6.7227e-08, 3.3740e-07, 8.9268e-07,\n", + " 3.2936e-07, 7.7259e-07, 1.3288e-06, 5.2479e-07, 5.2387e-07,\n", + " 8.7761e-07, -7.5338e-07, -4.1550e-07, -1.1366e-07, -3.5931e-07,\n", + " 8.6762e-07, 5.2379e-07, -3.9887e-07, 4.7742e-07, -5.2923e-07,\n", + " -1.3261e-06, 2.3211e-08, 2.0264e-07, -8.8497e-08, -9.1244e-07,\n", + " -1.2321e-07, -6.4311e-07, -1.0426e-07, -2.5215e-08, -2.3498e-07,\n", + " -6.8606e-07, -2.0956e-07, 5.3289e-08, 1.9906e-07, 5.2144e-07,\n", + " 5.2902e-07, 1.6739e-06, -1.7456e-07, 4.8457e-07, 2.7094e-07,\n", + " -8.8920e-08, 6.4603e-07, -6.6939e-09, -4.6188e-07, -2.8319e-08,\n", + " 1.2474e-09], device='cuda:0', grad_fn=)),\n", + " ('layer3.0.conv2.weight',\n", + " tensor([[[[ 1.0662e-02, 2.3105e-02, 2.0944e-02],\n", + " [-1.6962e-03, -1.7286e-02, -1.3265e-02],\n", + " [-7.9107e-04, -1.2152e-02, 1.1042e-02]],\n", + " \n", + " [[-2.1019e-02, -3.9375e-03, 2.7401e-02],\n", + " [ 2.6706e-02, 1.9458e-02, -1.6592e-03],\n", + " [ 1.2585e-02, -5.9605e-03, 1.2421e-02]],\n", + " \n", + " [[-6.9274e-03, 8.3360e-03, -9.4914e-03],\n", + " [ 5.3120e-03, 2.5441e-03, -1.9309e-02],\n", + " [-1.2729e-02, 1.2098e-03, -1.3140e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.5503e-02, -2.0459e-02, -1.7660e-02],\n", + " [ 2.1041e-03, -4.7642e-03, -1.6214e-03],\n", + " [-8.4515e-03, -8.6028e-03, 7.7011e-03]],\n", + " \n", + " [[ 1.9754e-03, 1.2156e-02, 1.5698e-02],\n", + " [-1.4284e-02, 7.9187e-03, 5.6066e-03],\n", + " [-1.4685e-02, 5.0906e-03, 2.0291e-02]],\n", + " \n", + " [[ 1.0820e-03, 2.5782e-02, -8.0276e-03],\n", + " [-1.5261e-02, -1.0020e-02, -7.2103e-03],\n", + " [-1.4226e-02, -8.4334e-03, 6.6771e-03]]],\n", + " \n", + " \n", + " [[[ 1.5926e-02, -6.3484e-03, -2.1075e-03],\n", + " [-1.5315e-03, -5.5726e-03, 1.7136e-02],\n", + " [ 1.4945e-03, -1.7555e-02, 4.3468e-03]],\n", + " \n", + " [[-2.1769e-02, 3.3361e-03, 2.4591e-02],\n", + " [ 1.5109e-02, -2.8429e-02, 1.0172e-03],\n", + " [-1.6435e-02, 2.6428e-03, 1.5481e-02]],\n", + " \n", + " [[-1.4824e-02, -8.5155e-03, 1.8834e-02],\n", + " [ 1.2652e-02, 6.7154e-03, -2.6716e-03],\n", + " [-8.6145e-03, 1.0106e-02, -2.3724e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.6900e-02, -5.1725e-03, -1.7768e-03],\n", + " [-1.0207e-02, -3.7643e-03, -9.7340e-03],\n", + " [ 2.5798e-02, -1.9137e-02, 2.0954e-02]],\n", + " \n", + " [[ 1.7942e-02, 1.1663e-02, 1.3949e-02],\n", + " [ 7.6405e-03, 1.2597e-02, 1.8279e-02],\n", + " [-5.5239e-03, -1.8590e-02, -1.8047e-03]],\n", + " \n", + " [[-1.4462e-02, 7.7165e-03, 2.2021e-03],\n", + " [-1.0820e-02, -1.0834e-03, -3.6798e-03],\n", + " [-1.9548e-02, -1.3020e-03, -1.1262e-02]]],\n", + " \n", + " \n", + " [[[ 1.2946e-02, 1.1081e-02, 1.1393e-02],\n", + " [ 4.8879e-04, -1.1928e-02, 1.0031e-02],\n", + " [-1.7902e-02, -1.7419e-02, -4.2769e-03]],\n", + " \n", + " [[-6.6826e-04, 3.5241e-03, -2.0011e-02],\n", + " [ 8.4268e-03, -1.5843e-02, 5.7051e-03],\n", + " [ 5.3919e-03, -1.1616e-03, -1.6102e-02]],\n", + " \n", + " [[ 5.3986e-03, 2.4112e-02, -8.9773e-03],\n", + " [ 1.3202e-02, -2.2026e-03, -2.1816e-02],\n", + " [ 5.5018e-03, 8.3577e-05, 1.2664e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.1547e-02, 6.8494e-03, 2.1623e-02],\n", + " [ 2.6007e-02, 4.2709e-03, -1.6149e-03],\n", + " [ 2.8532e-02, -1.6667e-02, -2.0934e-02]],\n", + " \n", + " [[ 1.0923e-02, 2.3803e-02, -2.7859e-03],\n", + " [-1.5858e-02, 1.5274e-02, 1.4692e-02],\n", + " [ 8.9732e-03, 6.4990e-03, 6.1606e-03]],\n", + " \n", + " [[ 2.8232e-03, 6.6697e-03, -2.7311e-03],\n", + " [-2.2336e-02, -1.6776e-03, -2.1136e-03],\n", + " [-7.1193e-03, -9.1132e-03, 1.7332e-02]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 5.0451e-03, -1.4421e-02, 1.0128e-02],\n", + " [ 2.3629e-02, -2.6037e-03, -6.2489e-03],\n", + " [-2.4079e-02, 1.1019e-04, 1.8441e-02]],\n", + " \n", + " [[-1.6919e-02, 4.9657e-03, -1.1922e-02],\n", + " [-4.4497e-03, 2.1552e-02, -6.9888e-03],\n", + " [ 5.7468e-03, 1.0028e-02, -2.9763e-03]],\n", + " \n", + " [[-1.1208e-02, -4.8582e-04, 9.0071e-03],\n", + " [ 4.2691e-03, -1.6680e-02, 1.5524e-03],\n", + " [-2.0489e-03, -1.5415e-03, 9.5223e-04]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.2714e-02, -8.5548e-03, 1.7332e-02],\n", + " [-7.8999e-03, 1.6427e-02, -1.6550e-02],\n", + " [-1.2743e-02, -2.4785e-02, -5.7847e-03]],\n", + " \n", + " [[-1.8290e-03, 3.9545e-03, 1.5218e-02],\n", + " [-1.5010e-02, 8.9211e-03, 7.1442e-03],\n", + " [-1.5832e-02, -1.0943e-02, 1.2260e-04]],\n", + " \n", + " [[ 2.5532e-03, 1.7381e-02, -8.9490e-03],\n", + " [-1.6582e-02, -1.3242e-02, -5.1941e-03],\n", + " [-1.5262e-04, 1.1208e-03, 6.8951e-03]]],\n", + " \n", + " \n", + " [[[ 1.3280e-02, 1.4718e-02, 1.1648e-03],\n", + " [ 5.4112e-03, -1.0257e-03, -1.9862e-02],\n", + " [ 4.9593e-03, 1.6201e-02, -6.7019e-03]],\n", + " \n", + " [[-7.3088e-03, -1.2836e-02, -1.6779e-02],\n", + " [ 7.5643e-03, -7.6154e-03, -1.7013e-02],\n", + " [-7.5895e-04, 2.4402e-02, -3.3536e-02]],\n", + " \n", + " [[ 2.2493e-02, -1.3170e-02, 1.9583e-02],\n", + " [-7.0033e-03, 1.5615e-02, 2.2892e-02],\n", + " [-2.1505e-04, -2.7462e-04, 5.6131e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 9.3670e-03, 1.7157e-02, -3.8728e-03],\n", + " [-5.8528e-03, -9.1498e-03, -7.8514e-03],\n", + " [-7.9125e-03, 1.5890e-02, -2.6130e-03]],\n", + " \n", + " [[-8.3766e-04, -3.3567e-03, 2.4494e-02],\n", + " [ 2.6621e-02, 7.9730e-03, 2.1429e-02],\n", + " [-6.8606e-03, 1.3426e-02, -6.1734e-03]],\n", + " \n", + " [[-5.1905e-04, 1.5170e-02, 1.0361e-02],\n", + " [-4.1423e-04, -2.1263e-03, 2.0374e-02],\n", + " [-2.1371e-02, -1.8019e-02, 1.7098e-02]]],\n", + " \n", + " \n", + " [[[ 1.7300e-02, -2.7703e-02, 1.1888e-02],\n", + " [-2.5120e-02, 8.9563e-03, -1.3949e-03],\n", + " [ 4.1365e-03, -5.1395e-03, 2.7497e-03]],\n", + " \n", + " [[-8.7912e-03, -3.8518e-06, -2.4544e-02],\n", + " [-2.5744e-02, -2.0509e-02, 7.0818e-03],\n", + " [-2.0765e-02, -2.0426e-02, -2.6527e-03]],\n", + " \n", + " [[ 1.0005e-02, -6.5160e-03, 2.3553e-02],\n", + " [ 2.4463e-02, 1.2743e-03, -2.6316e-02],\n", + " [-7.5289e-03, -3.1372e-03, 2.5079e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.1344e-02, 2.5635e-02, -2.7420e-02],\n", + " [ 1.5453e-02, -1.3559e-02, 7.9082e-03],\n", + " [ 1.8416e-02, -2.1079e-02, 3.1051e-04]],\n", + " \n", + " [[ 1.6002e-02, 4.8267e-03, 1.7036e-02],\n", + " [-2.0450e-02, 1.6423e-02, 1.5697e-02],\n", + " [-5.6105e-03, -1.2805e-02, -7.7957e-04]],\n", + " \n", + " [[ 1.4429e-02, -4.5842e-03, -1.0417e-02],\n", + " [ 7.9961e-03, 8.3452e-03, -1.1142e-02],\n", + " [-1.6436e-02, 2.8955e-02, -1.7532e-02]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer3.0.bn2.weight',\n", + " tensor([ 3.5763e-07, 5.9605e-07, 4.7684e-07, -4.1723e-07, -2.3842e-07,\n", + " 8.3447e-07, 1.1921e-07, 4.7684e-07, 2.3842e-07, -3.5763e-07,\n", + " -3.5763e-07, -1.0729e-06, -1.7881e-07, 3.5763e-07, 0.0000e+00,\n", + " 1.0729e-06, 2.3842e-07, -1.0729e-06, 0.0000e+00, 1.1921e-06,\n", + " -5.9605e-07, -1.1921e-07, -7.1526e-07, 4.7684e-07, 5.9605e-07,\n", + " 2.3842e-07, 1.1921e-07, -4.1723e-07, -1.1921e-07, -4.7684e-07,\n", + " -1.2517e-06, 5.9605e-07, 4.7684e-07, 3.5763e-07, 1.1921e-07,\n", + " 1.1921e-07, 4.7684e-07, -2.3842e-07, 1.1921e-07, 3.5763e-07,\n", + " -5.3644e-07, -1.7881e-07, 1.0729e-06, -1.0133e-06, -5.3644e-07,\n", + " 7.1526e-07, -1.7881e-07, 3.5763e-07, 2.3842e-07, -1.1921e-07,\n", + " -5.9605e-07, -4.7684e-07, 1.1921e-07, 2.3842e-07, -7.7486e-07,\n", + " -2.3842e-07, -6.5565e-07, -3.5763e-07, 4.7684e-07, 3.5763e-07,\n", + " 0.0000e+00, -2.3842e-07, 2.3842e-07, -6.5565e-07, 2.3842e-07,\n", + " -2.9802e-07, 2.3842e-07, -1.1921e-07, 0.0000e+00, 2.3842e-07,\n", + " -8.3447e-07, -1.1921e-07, 2.3842e-07, 5.9605e-07, 9.5367e-07,\n", + " -7.7486e-07, -4.7684e-07, 4.7684e-07, 2.3842e-07, 1.1921e-07,\n", + " 5.9605e-07, -5.3644e-07, -7.7486e-07, 3.5763e-07, -4.1723e-07,\n", + " -3.5763e-07, -8.3447e-07, -8.9407e-07, -1.1921e-07, -1.0133e-06,\n", + " -3.5763e-07, 1.0729e-06, -5.9605e-08, -6.5565e-07, -1.7881e-07,\n", + " -9.5367e-07, 0.0000e+00, -2.3842e-07, -5.9605e-08, 1.0729e-06,\n", + " 7.1526e-07, -8.3447e-07, 1.1921e-06, 1.1921e-07, 8.3447e-07,\n", + " 3.5763e-07, -2.9802e-07, 1.1921e-07, 5.9605e-07, 5.9605e-07,\n", + " -1.2517e-06, 3.5763e-07, -1.7881e-07, 4.7684e-07, -1.1921e-07,\n", + " 7.1526e-07, -7.7486e-07, 7.1526e-07, 1.1921e-07, 3.5763e-07,\n", + " 0.0000e+00, 1.1921e-07, 4.7684e-07, -1.7881e-07, 3.5763e-07,\n", + " 5.9605e-07, 1.1921e-07, -1.7881e-07, 0.0000e+00, -5.3644e-07,\n", + " -8.3447e-07, -2.9802e-07, -2.3842e-07, -1.1921e-07, -2.9802e-07,\n", + " 1.1921e-06, -6.5565e-07, -2.3842e-07, 4.7684e-07, 8.3447e-07,\n", + " -4.1723e-07, 1.0729e-06, 4.7684e-07, 1.1921e-07, 1.1921e-07,\n", + " -6.5565e-07, -1.0133e-06, 1.0729e-06, 4.7684e-07, -5.9605e-08,\n", + " 3.5763e-07, 2.3842e-07, 1.1921e-07, -4.7684e-07, 2.3842e-07,\n", + " 3.5763e-07, -1.1921e-07, 2.3842e-07, 4.7684e-07, -5.9605e-08,\n", + " 3.5763e-07, -2.9802e-07, -2.3842e-07, -3.5763e-07, 7.1526e-07,\n", + " 3.5763e-07, -4.1723e-07, -3.5763e-07, 0.0000e+00, -8.3447e-07,\n", + " 9.5367e-07, -5.9605e-08, -5.9605e-07, -5.9605e-07, 7.1526e-07,\n", + " 7.1526e-07, 1.1921e-07, 3.5763e-07, -3.5763e-07, -4.1723e-07,\n", + " 0.0000e+00, 2.3842e-07, 2.3842e-07, -4.1723e-07, 9.5367e-07,\n", + " 2.3842e-07, 5.9605e-07, 3.5763e-07, -3.5763e-07, -1.0133e-06,\n", + " -2.3842e-07, -1.7881e-07, -1.0729e-06, -5.3644e-07, -7.1526e-07,\n", + " -7.7486e-07, 8.3447e-07, 2.3842e-07, -5.3644e-07, -5.3644e-07,\n", + " -1.1921e-07, -6.5565e-07, -7.7486e-07, -4.1723e-07, 3.5763e-07,\n", + " -6.5565e-07, -5.9605e-08, -9.5367e-07, -5.3644e-07, 1.1921e-06,\n", + " -1.1921e-06, -1.1921e-07, 5.9605e-07, 7.1526e-07, 5.9605e-07,\n", + " -2.3842e-07, -1.1921e-07, 0.0000e+00, -3.5763e-07, 3.5763e-07,\n", + " -2.9802e-07, -2.9802e-07, 0.0000e+00, -2.9802e-07, -1.1921e-07,\n", + " -7.1526e-07, -1.4901e-06, 1.1921e-07, 3.5763e-07, 1.5497e-06,\n", + " -4.1723e-07, 1.1921e-07, -2.9802e-07, 4.7684e-07, 7.1526e-07,\n", + " 8.3447e-07, -2.3842e-07, 9.5367e-07, -7.1526e-07, 2.3842e-07,\n", + " 1.1921e-07, 3.5763e-07, 3.5763e-07, -5.9605e-07, -3.5763e-07,\n", + " 4.7684e-07, 2.3842e-07, -1.7881e-07, 7.1526e-07, 8.3447e-07,\n", + " 8.3447e-07, 0.0000e+00, 5.9605e-07, 4.7684e-07, -1.7881e-07,\n", + " 0.0000e+00], device='cuda:0', grad_fn=)),\n", + " ('layer3.0.bn2.bias',\n", + " tensor([ 2.3680e-07, 3.5135e-07, 3.4183e-07, 8.8436e-08, 2.0979e-08,\n", + " 2.5290e-07, 6.5370e-07, -2.6026e-07, -3.0810e-08, -5.5992e-08,\n", + " -2.3866e-07, -3.7466e-07, -7.0554e-07, 1.1698e-07, -4.4040e-07,\n", + " 6.7360e-07, -1.4584e-09, 5.1592e-09, -7.2081e-07, 5.7207e-07,\n", + " -2.4412e-07, 5.0197e-08, -6.8681e-07, -2.0246e-07, 6.1333e-07,\n", + " -9.8468e-08, 3.9486e-07, -2.1139e-07, -3.0886e-07, -7.6842e-07,\n", + " -6.5191e-07, 2.6826e-07, 4.6805e-07, -1.5186e-07, -1.0934e-07,\n", + " -2.5448e-07, -8.2063e-08, -1.9279e-07, 5.8344e-07, 4.2441e-07,\n", + " -8.0338e-07, -3.2880e-07, 3.8258e-07, 1.5075e-07, 4.8517e-07,\n", + " 1.1028e-07, 3.3404e-07, 2.7634e-07, -1.3250e-07, 7.7396e-08,\n", + " 1.8827e-08, 1.7996e-08, 3.3748e-07, -4.5710e-07, -9.9208e-07,\n", + " -4.3114e-07, 4.0201e-08, 3.1458e-07, 1.9742e-07, -2.1169e-07,\n", + " -1.6180e-07, 9.9652e-08, 1.1093e-07, -5.2305e-07, 2.8458e-07,\n", + " 6.2370e-07, 1.5391e-07, -2.4377e-07, -1.0629e-08, -4.5090e-07,\n", + " -6.9124e-07, 1.6198e-07, -3.7297e-08, 7.4784e-07, 5.0719e-07,\n", + " -5.0355e-07, -6.1532e-07, 3.8803e-07, 3.1920e-07, 2.7575e-08,\n", + " 6.2666e-07, -1.0726e-07, 1.9611e-07, -5.7010e-07, -3.4205e-07,\n", + " 1.5303e-07, -8.2835e-07, 8.1646e-07, 7.0774e-07, -4.7583e-07,\n", + " -2.7194e-07, 8.6398e-08, 9.4109e-08, -1.7007e-07, -3.2332e-07,\n", + " 6.5095e-08, -8.3583e-07, 1.9435e-07, 4.7392e-07, 6.5855e-07,\n", + " 2.7783e-07, -4.3500e-07, 1.5604e-07, -4.1401e-07, 8.9819e-08,\n", + " 2.7929e-07, -3.7298e-07, -8.6467e-08, 7.3443e-07, -2.6574e-07,\n", + " 8.4724e-08, -2.7784e-07, -2.2111e-07, 8.4979e-07, 3.4278e-08,\n", + " 3.4340e-07, -1.5229e-08, 1.2894e-06, 4.1525e-07, 4.1044e-07,\n", + " -1.7110e-07, 1.1602e-06, -8.1511e-08, -1.3142e-07, -2.8796e-07,\n", + " -4.2405e-07, -2.3917e-07, -4.5406e-07, -4.2457e-07, -5.5305e-08,\n", + " -4.0831e-07, -7.1161e-07, -7.1909e-07, 4.3880e-08, -2.1519e-07,\n", + " 6.9777e-07, 4.6607e-07, 3.3010e-07, -2.1112e-07, 4.3409e-07,\n", + " -4.4896e-08, 4.9483e-07, -3.5638e-07, 3.2437e-07, 3.2455e-07,\n", + " 3.6626e-08, -2.3136e-08, 3.0788e-07, 9.0411e-07, -7.1682e-07,\n", + " 1.3247e-07, 6.7468e-07, 2.1672e-07, -3.4881e-07, -1.6487e-07,\n", + " 2.6893e-07, -4.5407e-07, 1.7996e-09, -9.0014e-08, 3.7263e-08,\n", + " -1.5918e-07, 1.6091e-07, -6.2528e-07, -2.8585e-07, 2.4054e-07,\n", + " -1.8991e-07, 4.6312e-07, -3.8197e-07, 6.5382e-08, -6.0868e-07,\n", + " 1.7277e-07, 1.5869e-07, 2.4873e-07, 1.4232e-08, 7.9809e-07,\n", + " 9.4013e-07, 3.8632e-07, -4.7660e-07, -4.2201e-08, -9.8884e-07,\n", + " -4.1836e-07, 8.7761e-07, -3.5395e-07, -1.2057e-07, -8.8166e-08,\n", + " 2.7069e-07, 2.6314e-07, -8.3510e-08, -1.0514e-07, 1.0384e-07,\n", + " 6.7777e-07, -8.5769e-07, -8.8564e-07, 1.9361e-07, -5.7802e-07,\n", + " -1.2372e-07, 2.6111e-07, 3.3398e-07, -3.8925e-07, -1.3571e-07,\n", + " 3.4758e-07, -4.1377e-07, -8.7289e-07, -3.4511e-07, 3.8844e-07,\n", + " -3.9513e-07, -8.4503e-09, -1.2836e-07, -8.8299e-07, 9.3458e-07,\n", + " -8.7290e-07, -2.2613e-07, 2.4439e-07, 4.7279e-07, 4.2166e-07,\n", + " 3.5165e-08, 3.3261e-08, 1.3420e-09, 8.2721e-08, -1.3073e-08,\n", + " -6.5133e-07, 2.2716e-07, 1.0948e-07, 8.1573e-08, -2.5343e-07,\n", + " -5.5931e-07, -1.7230e-07, -3.9210e-07, -3.8520e-07, -3.1067e-07,\n", + " -9.0795e-07, -2.2396e-08, 4.6810e-07, 3.8014e-07, 2.1777e-09,\n", + " 8.1825e-07, -2.2735e-07, -1.5970e-07, -2.4650e-07, -6.7152e-07,\n", + " -2.7729e-07, -1.0032e-07, 5.6624e-08, -5.8656e-07, -2.4579e-08,\n", + " 1.2047e-07, 2.9561e-07, -5.2755e-07, 6.3744e-07, 6.7626e-07,\n", + " 1.7435e-07, 4.1876e-07, 8.1455e-07, -3.8383e-07, 2.2717e-07,\n", + " -1.2689e-07], device='cuda:0', grad_fn=)),\n", + " ('layer3.0.shortcut.0.weight',\n", + " tensor([[[[-1.2532e-01]],\n", + " \n", + " [[ 1.7072e-02]],\n", + " \n", + " [[ 5.5834e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.5657e-02]],\n", + " \n", + " [[ 1.2111e-02]],\n", + " \n", + " [[-7.3992e-05]]],\n", + " \n", + " \n", + " [[[-5.7674e-02]],\n", + " \n", + " [[-1.4630e-03]],\n", + " \n", + " [[-3.7302e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.3707e-02]],\n", + " \n", + " [[ 3.9356e-02]],\n", + " \n", + " [[-1.1301e-01]]],\n", + " \n", + " \n", + " [[[ 2.2812e-02]],\n", + " \n", + " [[-6.2792e-02]],\n", + " \n", + " [[-8.5218e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.4610e-02]],\n", + " \n", + " [[ 1.0115e-01]],\n", + " \n", + " [[-2.1533e-02]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 2.8550e-02]],\n", + " \n", + " [[ 6.4809e-03]],\n", + " \n", + " [[-6.6245e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.0575e-01]],\n", + " \n", + " [[-4.9512e-02]],\n", + " \n", + " [[ 3.9289e-02]]],\n", + " \n", + " \n", + " [[[-4.6138e-02]],\n", + " \n", + " [[-7.4739e-02]],\n", + " \n", + " [[ 5.0019e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-9.5460e-04]],\n", + " \n", + " [[ 5.4519e-02]],\n", + " \n", + " [[-2.4941e-02]]],\n", + " \n", + " \n", + " [[[-6.3402e-02]],\n", + " \n", + " [[-3.0870e-02]],\n", + " \n", + " [[ 1.0218e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.0487e-03]],\n", + " \n", + " [[-4.1583e-02]],\n", + " \n", + " [[-8.3212e-03]]]], device='cuda:0', grad_fn=)),\n", + " ('layer3.0.shortcut.1.weight',\n", + " tensor([-5.9605e-08, 5.9605e-07, 3.5763e-07, -4.1723e-07, 3.5763e-07,\n", + " 1.1921e-07, 9.5367e-07, 0.0000e+00, -2.9802e-07, 5.9605e-07,\n", + " 5.9605e-07, 1.1921e-07, -8.9407e-07, 0.0000e+00, -5.3644e-07,\n", + " -2.3842e-07, 4.7684e-07, 5.9605e-07, -9.5367e-07, 2.3842e-07,\n", + " 1.1921e-07, 5.9605e-07, -5.3644e-07, 1.1921e-07, 2.3842e-07,\n", + " 2.3842e-07, 0.0000e+00, 2.3842e-07, -8.9407e-07, -4.7684e-07,\n", + " -1.7881e-07, -1.1921e-07, -5.9605e-08, -5.3644e-07, 4.7684e-07,\n", + " -5.9605e-08, -5.9605e-08, -3.5763e-07, 5.9605e-07, 4.7684e-07,\n", + " -2.3842e-07, -4.7684e-07, -3.5763e-07, 0.0000e+00, 7.1526e-07,\n", + " 2.3842e-07, -1.1921e-07, 3.5763e-07, -4.1723e-07, -5.9605e-08,\n", + " 5.9605e-07, 5.9605e-07, 5.9605e-07, -1.1921e-06, -6.5565e-07,\n", + " -4.7684e-07, 5.9605e-07, -1.7881e-07, -7.1526e-07, -1.7881e-07,\n", + " -6.5565e-07, 3.5763e-07, -2.3842e-07, 5.9605e-07, -1.1921e-07,\n", + " 2.3842e-07, -1.1921e-07, 2.3842e-07, -5.3644e-07, -8.9407e-07,\n", + " 2.3842e-07, -1.7881e-07, -4.7684e-07, 1.1921e-07, -2.9802e-07,\n", + " 1.1921e-07, -5.9605e-07, 2.3842e-07, 2.3842e-07, 2.3842e-07,\n", + " 8.3447e-07, 5.9605e-07, 8.3447e-07, -1.4305e-06, -6.5565e-07,\n", + " -5.9605e-08, -5.3644e-07, 1.5497e-06, 9.5367e-07, 1.1921e-07,\n", + " -2.9802e-07, -5.9605e-07, -2.3842e-07, -2.3842e-07, -7.7486e-07,\n", + " 1.1921e-07, -5.9605e-07, 0.0000e+00, 1.0729e-06, -5.9605e-08,\n", + " 1.1921e-07, -8.3447e-07, 3.5763e-07, -2.9802e-07, -4.1723e-07,\n", + " 0.0000e+00, -3.5763e-07, 3.5763e-07, -4.1723e-07, -9.5367e-07,\n", + " 1.1921e-06, 2.3842e-07, 1.0729e-06, 1.0729e-06, 1.1921e-07,\n", + " 0.0000e+00, 4.7684e-07, 1.0729e-06, 2.3842e-07, -2.3842e-07,\n", + " -1.1921e-07, 4.7684e-07, 2.3842e-07, 5.9605e-07, -5.9605e-07,\n", + " 0.0000e+00, -2.9802e-07, -1.1921e-07, -5.9605e-08, 2.3842e-07,\n", + " -8.3447e-07, -2.9802e-07, -1.4305e-06, -4.1723e-07, 1.1921e-07,\n", + " 1.0729e-06, -4.7684e-07, 1.1921e-07, 3.5763e-07, -5.3644e-07,\n", + " 1.1921e-07, -3.5763e-07, -2.3842e-07, 3.5763e-07, 8.3447e-07,\n", + " 9.5367e-07, 1.0729e-06, 4.7684e-07, 4.7684e-07, -2.3842e-07,\n", + " -1.7881e-07, 5.9605e-07, 2.3842e-07, -6.5565e-07, -5.9605e-07,\n", + " -7.1526e-07, -5.9605e-08, 1.1921e-07, 7.1526e-07, -5.9605e-07,\n", + " -1.1921e-07, 8.3447e-07, -5.9605e-07, 2.3842e-07, -3.5763e-07,\n", + " -5.9605e-07, 7.1526e-07, 0.0000e+00, -2.9802e-07, -4.1723e-07,\n", + " 2.3842e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07, 1.0729e-06,\n", + " 3.5763e-07, 2.3842e-07, -3.5763e-07, 8.3447e-07, -8.9407e-07,\n", + " -1.7881e-07, 5.9605e-07, -5.3644e-07, -5.9605e-08, -6.5565e-07,\n", + " 4.7684e-07, 2.3842e-07, -1.1325e-06, 1.1921e-07, 4.7684e-07,\n", + " 5.9605e-07, -2.9802e-07, 2.3842e-07, 2.3842e-07, -3.5763e-07,\n", + " 0.0000e+00, 5.9605e-07, -4.1723e-07, -7.7486e-07, -5.9605e-08,\n", + " 0.0000e+00, -6.5565e-07, -4.7684e-07, -5.9605e-08, 1.1921e-07,\n", + " -3.5763e-07, 1.4305e-06, 2.3842e-07, -1.0729e-06, 1.1921e-07,\n", + " -9.5367e-07, 3.5763e-07, -4.7684e-07, 0.0000e+00, 0.0000e+00,\n", + " 3.5763e-07, 8.3447e-07, 4.7684e-07, -1.1921e-07, 3.5763e-07,\n", + " -4.1723e-07, 7.1526e-07, -3.5763e-07, -1.1921e-07, 2.3842e-07,\n", + " -1.0133e-06, 7.1526e-07, -6.5565e-07, -1.0729e-06, -1.4305e-06,\n", + " -7.7486e-07, 1.1921e-07, 5.9605e-07, 7.1526e-07, 5.9605e-07,\n", + " 3.5763e-07, 2.3842e-07, -1.0133e-06, 9.5367e-07, -1.2517e-06,\n", + " -4.7684e-07, -2.3842e-07, -5.9605e-07, 3.5763e-07, -5.3644e-07,\n", + " -1.1921e-07, 5.9605e-07, -1.0729e-06, 1.1921e-07, 1.1921e-07,\n", + " -2.9802e-07, 3.5763e-07, 5.9605e-07, -4.1723e-07, 4.7684e-07,\n", + " 5.9605e-07], device='cuda:0', grad_fn=)),\n", + " ('layer3.0.shortcut.1.bias',\n", + " tensor([ 2.3680e-07, 3.5135e-07, 3.4183e-07, 8.8436e-08, 2.0979e-08,\n", + " 2.5290e-07, 6.5370e-07, -2.6026e-07, -3.0810e-08, -5.5992e-08,\n", + " -2.3866e-07, -3.7466e-07, -7.0554e-07, 1.1698e-07, -4.4040e-07,\n", + " 6.7360e-07, -1.4584e-09, 5.1592e-09, -7.2081e-07, 5.7207e-07,\n", + " -2.4412e-07, 5.0197e-08, -6.8681e-07, -2.0246e-07, 6.1333e-07,\n", + " -9.8468e-08, 3.9486e-07, -2.1139e-07, -3.0886e-07, -7.6842e-07,\n", + " -6.5191e-07, 2.6826e-07, 4.6805e-07, -1.5186e-07, -1.0934e-07,\n", + " -2.5448e-07, -8.2063e-08, -1.9279e-07, 5.8344e-07, 4.2441e-07,\n", + " -8.0338e-07, -3.2880e-07, 3.8258e-07, 1.5075e-07, 4.8517e-07,\n", + " 1.1028e-07, 3.3404e-07, 2.7634e-07, -1.3250e-07, 7.7396e-08,\n", + " 1.8827e-08, 1.7996e-08, 3.3748e-07, -4.5710e-07, -9.9208e-07,\n", + " -4.3114e-07, 4.0201e-08, 3.1458e-07, 1.9742e-07, -2.1169e-07,\n", + " -1.6180e-07, 9.9652e-08, 1.1093e-07, -5.2305e-07, 2.8458e-07,\n", + " 6.2370e-07, 1.5391e-07, -2.4377e-07, -1.0629e-08, -4.5090e-07,\n", + " -6.9124e-07, 1.6198e-07, -3.7297e-08, 7.4784e-07, 5.0719e-07,\n", + " -5.0355e-07, -6.1532e-07, 3.8803e-07, 3.1920e-07, 2.7575e-08,\n", + " 6.2666e-07, -1.0726e-07, 1.9611e-07, -5.7010e-07, -3.4205e-07,\n", + " 1.5303e-07, -8.2835e-07, 8.1646e-07, 7.0774e-07, -4.7583e-07,\n", + " -2.7194e-07, 8.6398e-08, 9.4109e-08, -1.7007e-07, -3.2332e-07,\n", + " 6.5095e-08, -8.3583e-07, 1.9435e-07, 4.7392e-07, 6.5855e-07,\n", + " 2.7783e-07, -4.3500e-07, 1.5604e-07, -4.1401e-07, 8.9819e-08,\n", + " 2.7929e-07, -3.7298e-07, -8.6467e-08, 7.3443e-07, -2.6574e-07,\n", + " 8.4724e-08, -2.7784e-07, -2.2111e-07, 8.4979e-07, 3.4278e-08,\n", + " 3.4340e-07, -1.5229e-08, 1.2894e-06, 4.1525e-07, 4.1044e-07,\n", + " -1.7110e-07, 1.1602e-06, -8.1511e-08, -1.3142e-07, -2.8796e-07,\n", + " -4.2405e-07, -2.3917e-07, -4.5406e-07, -4.2457e-07, -5.5305e-08,\n", + " -4.0831e-07, -7.1161e-07, -7.1909e-07, 4.3880e-08, -2.1519e-07,\n", + " 6.9777e-07, 4.6607e-07, 3.3010e-07, -2.1112e-07, 4.3409e-07,\n", + " -4.4896e-08, 4.9483e-07, -3.5638e-07, 3.2437e-07, 3.2455e-07,\n", + " 3.6626e-08, -2.3136e-08, 3.0788e-07, 9.0411e-07, -7.1682e-07,\n", + " 1.3247e-07, 6.7468e-07, 2.1672e-07, -3.4881e-07, -1.6487e-07,\n", + " 2.6893e-07, -4.5407e-07, 1.7996e-09, -9.0014e-08, 3.7263e-08,\n", + " -1.5918e-07, 1.6091e-07, -6.2528e-07, -2.8585e-07, 2.4054e-07,\n", + " -1.8991e-07, 4.6312e-07, -3.8197e-07, 6.5382e-08, -6.0868e-07,\n", + " 1.7277e-07, 1.5869e-07, 2.4873e-07, 1.4232e-08, 7.9809e-07,\n", + " 9.4013e-07, 3.8632e-07, -4.7660e-07, -4.2201e-08, -9.8884e-07,\n", + " -4.1836e-07, 8.7761e-07, -3.5395e-07, -1.2057e-07, -8.8166e-08,\n", + " 2.7069e-07, 2.6314e-07, -8.3510e-08, -1.0514e-07, 1.0384e-07,\n", + " 6.7777e-07, -8.5769e-07, -8.8564e-07, 1.9361e-07, -5.7802e-07,\n", + " -1.2372e-07, 2.6111e-07, 3.3398e-07, -3.8925e-07, -1.3571e-07,\n", + " 3.4758e-07, -4.1377e-07, -8.7289e-07, -3.4511e-07, 3.8844e-07,\n", + " -3.9513e-07, -8.4503e-09, -1.2836e-07, -8.8299e-07, 9.3458e-07,\n", + " -8.7290e-07, -2.2613e-07, 2.4439e-07, 4.7279e-07, 4.2166e-07,\n", + " 3.5165e-08, 3.3261e-08, 1.3420e-09, 8.2721e-08, -1.3073e-08,\n", + " -6.5133e-07, 2.2716e-07, 1.0948e-07, 8.1573e-08, -2.5343e-07,\n", + " -5.5931e-07, -1.7230e-07, -3.9210e-07, -3.8520e-07, -3.1067e-07,\n", + " -9.0795e-07, -2.2396e-08, 4.6810e-07, 3.8014e-07, 2.1777e-09,\n", + " 8.1825e-07, -2.2735e-07, -1.5970e-07, -2.4650e-07, -6.7152e-07,\n", + " -2.7729e-07, -1.0032e-07, 5.6624e-08, -5.8656e-07, -2.4579e-08,\n", + " 1.2047e-07, 2.9561e-07, -5.2755e-07, 6.3744e-07, 6.7626e-07,\n", + " 1.7435e-07, 4.1876e-07, 8.1455e-07, -3.8383e-07, 2.2717e-07,\n", + " -1.2689e-07], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.conv1.weight',\n", + " tensor([[[[ 1.8114e-02, 5.2051e-04, -1.4573e-02],\n", + " [ 1.0840e-02, 3.5196e-03, -2.3358e-02],\n", + " [-3.9562e-03, -1.5493e-02, -7.0367e-03]],\n", + " \n", + " [[-6.0210e-03, 5.6906e-03, -1.5362e-02],\n", + " [ 2.8613e-03, -8.9222e-03, 1.2015e-02],\n", + " [-6.4128e-03, 2.4453e-02, 1.6543e-03]],\n", + " \n", + " [[ 1.7152e-02, -1.7592e-02, -1.1884e-03],\n", + " [-1.1144e-02, -1.7280e-02, -6.4692e-03],\n", + " [-1.9362e-04, -1.3540e-02, 1.7617e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1642e-02, 2.2848e-03, 1.3447e-02],\n", + " [ 1.3338e-03, 1.2532e-02, -4.2178e-03],\n", + " [ 6.1783e-03, 1.0295e-02, -3.5218e-03]],\n", + " \n", + " [[-4.4384e-03, 2.5541e-03, 6.1089e-03],\n", + " [-1.8793e-02, -5.2627e-03, 2.2046e-03],\n", + " [ 1.0186e-02, 5.0196e-03, -1.5322e-02]],\n", + " \n", + " [[-1.3257e-02, 1.6731e-02, 9.1186e-03],\n", + " [ 5.3671e-03, -2.3662e-02, 1.7017e-02],\n", + " [-9.3166e-03, -2.1477e-02, 1.6195e-02]]],\n", + " \n", + " \n", + " [[[ 8.3378e-03, -1.8744e-02, 6.6716e-05],\n", + " [ 1.0186e-02, 3.5303e-03, 2.3636e-02],\n", + " [-8.1820e-03, 1.4936e-03, 2.1595e-02]],\n", + " \n", + " [[-7.4387e-03, 1.0811e-02, -4.1400e-03],\n", + " [-1.0415e-02, -1.0622e-02, 1.4807e-02],\n", + " [-1.7862e-02, -1.2328e-02, 2.1008e-02]],\n", + " \n", + " [[ 1.3586e-02, -4.4893e-04, -1.4608e-02],\n", + " [ 2.0508e-02, -1.1871e-02, -1.7351e-03],\n", + " [-9.2368e-03, -1.1429e-02, 2.0413e-04]],\n", + " \n", + " ...,\n", + " \n", + " [[ 9.7317e-03, 6.8197e-03, 2.1716e-02],\n", + " [-1.5704e-02, -1.3467e-02, -1.0175e-02],\n", + " [-1.1507e-02, 9.9187e-03, 6.8689e-03]],\n", + " \n", + " [[-2.3748e-03, -6.8333e-03, -2.1954e-02],\n", + " [ 2.7879e-02, -2.3770e-03, -2.4249e-03],\n", + " [ 1.3316e-03, -2.1276e-03, 3.5368e-03]],\n", + " \n", + " [[ 4.1176e-03, 9.7428e-03, 5.2396e-03],\n", + " [-2.5708e-02, 2.1975e-02, 6.3758e-03],\n", + " [ 1.2244e-02, 8.3727e-03, 2.9957e-03]]],\n", + " \n", + " \n", + " [[[ 1.4465e-02, -1.4984e-02, -7.4286e-03],\n", + " [-1.6736e-02, -9.9520e-03, 3.2938e-02],\n", + " [-6.5588e-03, -1.0266e-02, 9.0892e-03]],\n", + " \n", + " [[ 7.4289e-03, -1.4877e-02, -8.2701e-03],\n", + " [-1.5337e-02, 1.7032e-02, -1.0953e-02],\n", + " [-1.0200e-02, 9.5796e-03, 2.4805e-04]],\n", + " \n", + " [[-4.5040e-03, 1.1648e-02, 6.5312e-03],\n", + " [ 3.9309e-03, 2.0466e-03, 5.2178e-03],\n", + " [-1.7061e-02, -2.7747e-03, 3.4949e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.8357e-02, -1.6988e-02, 1.2481e-02],\n", + " [-2.2475e-02, 1.0150e-02, -1.6380e-02],\n", + " [-2.5348e-03, 5.4042e-03, 1.7112e-03]],\n", + " \n", + " [[ 1.9661e-02, 2.7684e-02, -1.7579e-02],\n", + " [-1.7852e-02, 2.8303e-03, 5.3348e-03],\n", + " [-5.2718e-03, 7.7013e-03, 7.2804e-03]],\n", + " \n", + " [[ 9.1490e-03, 7.7883e-04, -1.7632e-02],\n", + " [-1.2583e-02, 8.7818e-03, 5.1676e-03],\n", + " [-3.6902e-04, -1.7178e-02, 1.3540e-02]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 8.1712e-04, 9.0576e-03, -1.9149e-02],\n", + " [-4.0932e-03, -2.1350e-03, 2.4558e-02],\n", + " [-1.8063e-03, 1.5787e-02, 2.4794e-02]],\n", + " \n", + " [[-5.6834e-03, 1.4906e-02, 1.7886e-02],\n", + " [-9.9328e-03, -1.4857e-03, 1.4721e-03],\n", + " [ 5.4608e-03, -1.5380e-02, 1.0360e-02]],\n", + " \n", + " [[ 2.0395e-02, -3.1558e-03, -4.9012e-03],\n", + " [ 1.2573e-02, 1.1591e-02, -8.4176e-03],\n", + " [-1.1168e-02, -1.3468e-02, 2.1639e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.7053e-02, -1.8436e-02, 2.2686e-02],\n", + " [-6.1383e-03, 1.5574e-02, -4.3106e-03],\n", + " [ 1.0139e-02, 2.5462e-04, 1.0165e-02]],\n", + " \n", + " [[ 1.4791e-02, 2.1350e-02, -1.8956e-03],\n", + " [ 2.2326e-02, 1.0025e-02, 1.4650e-02],\n", + " [ 2.1074e-02, -4.2362e-03, -6.1958e-03]],\n", + " \n", + " [[-2.2347e-02, 3.0744e-03, -1.0679e-02],\n", + " [ 8.9897e-04, -1.8132e-02, -7.3476e-03],\n", + " [ 1.2704e-02, -9.0810e-03, 7.2003e-03]]],\n", + " \n", + " \n", + " [[[-1.4371e-02, -2.7548e-03, 1.4248e-02],\n", + " [-7.4112e-03, -1.0907e-02, -2.5385e-02],\n", + " [-2.5373e-02, 2.7448e-03, 5.8521e-03]],\n", + " \n", + " [[-7.9273e-03, -2.7029e-03, 1.6600e-02],\n", + " [-1.2491e-02, 3.4288e-03, -7.3017e-03],\n", + " [ 1.8565e-03, 1.5789e-02, -5.2022e-03]],\n", + " \n", + " [[-2.9924e-03, -1.0030e-02, -5.9783e-03],\n", + " [ 2.2283e-02, -1.6772e-02, 1.1581e-02],\n", + " [ 2.6474e-02, -8.5180e-03, -3.0422e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1342e-03, 2.0211e-03, -1.0525e-02],\n", + " [-5.8896e-03, 2.5927e-02, -1.1140e-02],\n", + " [ 1.1061e-02, 1.7324e-02, -1.2695e-02]],\n", + " \n", + " [[-2.6018e-02, 2.3288e-03, 1.0441e-02],\n", + " [-1.5647e-02, -1.6925e-03, 1.2329e-02],\n", + " [ 3.0129e-03, -1.7816e-02, 1.0631e-02]],\n", + " \n", + " [[-1.5661e-02, -1.9536e-03, 5.5617e-03],\n", + " [ 6.3089e-04, 2.1462e-02, 1.8862e-02],\n", + " [-2.0530e-02, -1.4116e-02, 3.1988e-04]]],\n", + " \n", + " \n", + " [[[-1.4379e-02, 1.0504e-03, -1.8726e-03],\n", + " [ 1.1856e-02, -5.4930e-03, 3.6104e-03],\n", + " [ 9.3104e-03, 3.9445e-03, -6.1743e-03]],\n", + " \n", + " [[ 9.6196e-04, -1.0889e-02, 3.9305e-03],\n", + " [ 1.3188e-02, -4.7789e-03, -3.3596e-03],\n", + " [-2.0044e-03, 7.3119e-03, -1.8782e-02]],\n", + " \n", + " [[ 1.5632e-02, 1.0031e-02, 1.4872e-02],\n", + " [ 1.2690e-02, 2.0029e-03, 5.7593e-03],\n", + " [ 1.9563e-02, -1.9929e-02, 1.0187e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.3114e-02, 1.6794e-02, 6.3443e-03],\n", + " [-2.9880e-03, -6.8296e-04, -2.6330e-03],\n", + " [-8.2479e-03, -2.5342e-02, 2.0142e-02]],\n", + " \n", + " [[ 1.1973e-02, 9.4697e-03, 3.1806e-03],\n", + " [ 5.7394e-03, 8.0790e-03, 7.6858e-03],\n", + " [-1.7608e-03, 1.4017e-02, 1.1223e-02]],\n", + " \n", + " [[-1.4254e-03, -2.6417e-02, -1.7212e-02],\n", + " [ 9.6490e-03, 9.6998e-03, 1.3176e-02],\n", + " [-1.8310e-02, -1.2498e-02, -2.0969e-03]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer4.0.bn1.weight',\n", + " tensor([-5.9605e-08, 0.0000e+00, 1.1921e-07, 1.1921e-07, 0.0000e+00,\n", + " -1.7881e-07, -5.9605e-08, -1.7881e-07, 2.3842e-07, 1.1921e-07,\n", + " 2.3842e-07, -4.1723e-07, -2.3842e-07, -1.7881e-07, 0.0000e+00,\n", + " -3.5763e-07, -1.7881e-07, -4.1723e-07, 5.9605e-07, 2.3842e-07,\n", + " -2.3842e-07, 0.0000e+00, 7.1526e-07, 7.1526e-07, -1.7881e-07,\n", + " -4.1723e-07, 1.1921e-07, -5.9605e-08, -7.7486e-07, -5.3644e-07,\n", + " 2.3842e-07, -5.9605e-08, 2.3842e-07, -1.7881e-07, 1.1921e-07,\n", + " 7.1526e-07, 0.0000e+00, -5.3644e-07, 0.0000e+00, -1.1921e-07,\n", + " 3.5763e-07, -5.9605e-08, -2.3842e-07, 2.3842e-07, -1.7881e-07,\n", + " -3.5763e-07, -4.1723e-07, 1.1921e-07, -2.3842e-07, 0.0000e+00,\n", + " 1.1921e-07, -2.3842e-07, 3.5763e-07, -4.1723e-07, 0.0000e+00,\n", + " 8.3447e-07, 0.0000e+00, -6.5565e-07, -4.7684e-07, 3.5763e-07,\n", + " -2.3842e-07, 0.0000e+00, 3.5763e-07, -2.3842e-07, 1.1921e-07,\n", + " -1.7881e-07, 1.1921e-07, 1.1921e-07, 0.0000e+00, 1.1921e-07,\n", + " -5.3644e-07, -4.7684e-07, 0.0000e+00, -1.7881e-07, 0.0000e+00,\n", + " -5.3644e-07, -2.9802e-07, -1.1921e-07, 1.1921e-07, -5.9605e-08,\n", + " -1.1921e-07, 1.1921e-07, 3.5763e-07, 5.9605e-07, -2.9802e-07,\n", + " -5.9605e-08, 5.9605e-07, -5.9605e-08, 1.1921e-07, 2.3842e-07,\n", + " 2.3842e-07, 3.5763e-07, 3.5763e-07, -3.5763e-07, -5.9605e-08,\n", + " 2.3842e-07, 3.5763e-07, 0.0000e+00, -2.9802e-07, -2.9802e-07,\n", + " -5.9605e-08, 1.1921e-07, 4.7684e-07, -4.1723e-07, -1.7881e-07,\n", + " -1.7881e-07, -1.7881e-07, -3.5763e-07, 0.0000e+00, -5.3644e-07,\n", + " 3.5763e-07, -1.1921e-07, -4.1723e-07, -2.3842e-07, -1.7881e-07,\n", + " 4.7684e-07, -2.9802e-07, 2.3842e-07, -1.7881e-07, -2.3842e-07,\n", + " 4.7684e-07, -3.5763e-07, -1.7881e-07, -2.3842e-07, 2.3842e-07,\n", + " -1.1921e-07, -5.3644e-07, 2.3842e-07, -2.9802e-07, -1.7881e-07,\n", + " -4.1723e-07, -2.3842e-07, -5.9605e-07, -5.9605e-08, 0.0000e+00,\n", + " -5.9605e-08, -5.9605e-08, 0.0000e+00, -1.1921e-07, -4.1723e-07,\n", + " 1.1921e-07, -2.3842e-07, 0.0000e+00, 1.1921e-07, 1.1921e-07,\n", + " 0.0000e+00, -5.9605e-07, 3.5763e-07, -2.3842e-07, 2.3842e-07,\n", + " -3.5763e-07, 0.0000e+00, 2.3842e-07, -4.1723e-07, 0.0000e+00,\n", + " -3.5763e-07, -1.1921e-07, 0.0000e+00, 2.3842e-07, 3.5763e-07,\n", + " -2.3842e-07, -2.9802e-07, -5.9605e-08, 7.1526e-07, -5.9605e-08,\n", + " 2.3842e-07, -1.7881e-07, -1.7881e-07, -1.1921e-07, -2.3842e-07,\n", + " 1.1921e-07, -5.9605e-08, 3.5763e-07, -2.9802e-07, 1.1921e-07,\n", + " -2.9802e-07, 1.1921e-07, -1.7881e-07, -5.9605e-07, 4.7684e-07,\n", + " -4.7684e-07, 1.1921e-07, 1.1921e-07, -2.9802e-07, -1.7881e-07,\n", + " 1.1921e-07, -3.5763e-07, -4.1723e-07, 0.0000e+00, 7.1526e-07,\n", + " -1.1921e-07, 0.0000e+00, -5.9605e-08, 2.3842e-07, 3.5763e-07,\n", + " 2.3842e-07, 2.3842e-07, -3.5763e-07, 2.3842e-07, 2.3842e-07,\n", + " -2.3842e-07, 2.3842e-07, 0.0000e+00, -2.9802e-07, 1.1921e-07,\n", + " 3.5763e-07, -3.5763e-07, 2.3842e-07, -5.3644e-07, -5.9605e-08,\n", + " 1.1921e-07, 0.0000e+00, 4.7684e-07, -5.9605e-08, 2.3842e-07,\n", + " -5.9605e-08, -1.7881e-07, 0.0000e+00, 4.7684e-07, 5.9605e-07,\n", + " -5.9605e-08, 4.7684e-07, -4.1723e-07, -5.9605e-08, 5.9605e-07,\n", + " -1.1921e-07, -4.7684e-07, 2.3842e-07, -2.9802e-07, 1.1921e-07,\n", + " -1.7881e-07, -1.7881e-07, -1.1921e-07, 4.7684e-07, 0.0000e+00,\n", + " 4.7684e-07, 0.0000e+00, -5.9605e-08, 1.1921e-07, 1.1921e-07,\n", + " 5.9605e-07, 2.3842e-07, -1.1921e-07, -1.1921e-07, 0.0000e+00,\n", + " 3.5763e-07, -2.3842e-07, -3.5763e-07, -5.9605e-08, 1.1921e-07,\n", + " 2.3842e-07, 4.7684e-07, 0.0000e+00, 2.3842e-07, -5.9605e-08,\n", + " -5.3644e-07, -2.3842e-07, 4.7684e-07, 5.9605e-07, -3.5763e-07,\n", + " 4.7684e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07, 0.0000e+00,\n", + " 2.3842e-07, 4.7684e-07, -1.7881e-07, -1.7881e-07, -5.9605e-08,\n", + " 2.3842e-07, 1.1921e-07, 0.0000e+00, -2.3842e-07, -4.7684e-07,\n", + " 2.3842e-07, 0.0000e+00, -1.7881e-07, 1.1921e-07, 2.3842e-07,\n", + " 1.1921e-07, 1.1921e-07, 5.9605e-07, -1.7881e-07, 1.1921e-07,\n", + " 0.0000e+00, 0.0000e+00, 0.0000e+00, -3.5763e-07, 1.1921e-07,\n", + " 2.3842e-07, -2.9802e-07, -5.9605e-08, -5.9605e-08, -4.7684e-07,\n", + " 2.3842e-07, 0.0000e+00, -5.9605e-08, -1.7881e-07, 3.5763e-07,\n", + " 2.3842e-07, 1.1921e-07, 0.0000e+00, -2.9802e-07, -6.5565e-07,\n", + " 1.1921e-07, -4.1723e-07, -5.9605e-08, -1.1921e-07, -5.9605e-08,\n", + " 5.9605e-07, -5.9605e-08, 2.3842e-07, 1.1921e-07, -4.7684e-07,\n", + " 2.3842e-07, 0.0000e+00, 2.3842e-07, 0.0000e+00, -3.5763e-07,\n", + " 3.5763e-07, 0.0000e+00, 2.3842e-07, 4.7684e-07, 0.0000e+00,\n", + " 1.1921e-07, -5.9605e-08, 3.5763e-07, 1.1921e-07, 4.7684e-07,\n", + " -2.3842e-07, -4.7684e-07, -3.5763e-07, 1.1921e-07, 4.7684e-07,\n", + " 3.5763e-07, -1.1921e-07, -2.9802e-07, 3.5763e-07, 0.0000e+00,\n", + " 3.5763e-07, 3.5763e-07, -1.1921e-07, -4.1723e-07, -5.9605e-07,\n", + " 4.7684e-07, 1.1921e-07, 7.1526e-07, 2.3842e-07, -5.3644e-07,\n", + " -4.1723e-07, 0.0000e+00, 3.5763e-07, -4.1723e-07, -5.9605e-08,\n", + " 2.3842e-07, -3.5763e-07, -1.7881e-07, -2.3842e-07, -1.7881e-07,\n", + " -5.9605e-08, -4.1723e-07, -5.9605e-08, -4.1723e-07, -3.5763e-07,\n", + " 3.5763e-07, 2.3842e-07, -2.3842e-07, 2.3842e-07, -5.3644e-07,\n", + " -2.3842e-07, -6.5565e-07, 3.5763e-07, 3.5763e-07, -1.7881e-07,\n", + " -1.7881e-07, 4.7684e-07, 0.0000e+00, -5.9605e-08, 0.0000e+00,\n", + " 2.3842e-07, -4.7684e-07, -5.9605e-08, 7.1526e-07, 1.1921e-07,\n", + " -2.3842e-07, -1.1921e-07, 4.7684e-07, 4.7684e-07, 7.1526e-07,\n", + " 2.3842e-07, -5.9605e-08, 2.3842e-07, 5.9605e-07, -1.7881e-07,\n", + " -1.1921e-07, 0.0000e+00, -4.1723e-07, -1.7881e-07, 0.0000e+00,\n", + " 1.1921e-07, -4.7684e-07, -4.1723e-07, 2.3842e-07, 3.5763e-07,\n", + " 0.0000e+00, -2.3842e-07, 0.0000e+00, 1.1921e-07, 1.1921e-07,\n", + " -2.3842e-07, 5.9605e-07, -3.5763e-07, -1.1921e-07, -1.7881e-07,\n", + " -5.9605e-08, 3.5763e-07, 0.0000e+00, -2.9802e-07, 0.0000e+00,\n", + " 0.0000e+00, 0.0000e+00, -5.9605e-08, 5.9605e-07, 0.0000e+00,\n", + " 0.0000e+00, 9.5367e-07, 0.0000e+00, -1.7881e-07, -6.5565e-07,\n", + " 3.5763e-07, -1.7881e-07, -1.1921e-07, -1.1921e-07, 4.7684e-07,\n", + " -2.9802e-07, -1.1921e-07, -2.3842e-07, 8.3447e-07, -7.1526e-07,\n", + " 0.0000e+00, -1.1921e-07, -2.9802e-07, 3.5763e-07, 4.7684e-07,\n", + " 2.3842e-07, -2.3842e-07, -1.1921e-07, 1.1921e-07, 4.7684e-07,\n", + " -5.9605e-08, 2.3842e-07, 0.0000e+00, 7.1526e-07, 2.3842e-07,\n", + " -5.9605e-08, 4.7684e-07, 0.0000e+00, 2.3842e-07, 1.1921e-07,\n", + " 2.3842e-07, -3.5763e-07, -1.7881e-07, 3.5763e-07, 4.7684e-07,\n", + " 4.7684e-07, -3.5763e-07, -4.1723e-07, -5.3644e-07, 3.5763e-07,\n", + " -4.1723e-07, -8.3447e-07, -4.7684e-07, 0.0000e+00, 0.0000e+00,\n", + " -5.9605e-08, -5.3644e-07, -2.3842e-07, -3.5763e-07, 1.1921e-07,\n", + " -2.9802e-07, -6.5565e-07, 3.5763e-07, -1.1921e-07, 3.5763e-07,\n", + " 1.1921e-07, -1.1921e-07, 5.9605e-07, 8.3447e-07, 2.3842e-07,\n", + " 1.1921e-07, 0.0000e+00, 1.1921e-07, -1.1921e-07, -4.1723e-07,\n", + " 0.0000e+00, -2.9802e-07, -2.3842e-07, -1.1921e-07, -2.3842e-07,\n", + " 3.5763e-07, 5.9605e-07, 0.0000e+00, -5.9605e-08, -1.1921e-07,\n", + " 0.0000e+00, 2.3842e-07, -4.1723e-07, 1.1921e-07, -1.1921e-07,\n", + " -1.7881e-07, 7.1526e-07], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.bn1.bias',\n", + " tensor([ 3.1726e-08, -1.7731e-07, 9.9779e-09, -6.4998e-08, -1.0859e-07,\n", + " 1.0166e-07, 2.5536e-07, -1.3389e-08, 8.9986e-08, 1.9249e-08,\n", + " -4.1351e-07, -2.6128e-07, 8.3407e-08, -1.9537e-07, -1.3523e-07,\n", + " -4.3386e-07, -1.1622e-07, -2.2359e-07, -5.8473e-08, 3.5024e-07,\n", + " -1.8368e-07, -1.7182e-08, 5.2150e-07, 4.4955e-07, -1.5387e-07,\n", + " -2.8533e-07, 1.5760e-07, 8.1673e-08, -4.0682e-07, -1.9296e-07,\n", + " 2.1261e-07, -1.4812e-08, 2.2860e-07, -8.1006e-08, 1.6673e-07,\n", + " 4.5937e-07, -4.7240e-08, -2.4283e-07, -4.9272e-09, -3.4337e-07,\n", + " 2.5559e-07, -1.3976e-08, 2.6116e-07, 8.7624e-08, -1.7782e-07,\n", + " -1.2930e-07, -1.2506e-07, 6.7839e-08, -1.2276e-07, 4.3874e-08,\n", + " 9.7849e-08, 2.1440e-08, -8.0827e-08, -5.4108e-07, -8.0169e-08,\n", + " 3.6907e-07, 2.6496e-08, -2.6881e-07, -3.3249e-07, 4.5289e-07,\n", + " -1.3388e-07, 7.2097e-08, -4.8439e-08, 3.0130e-08, 1.9556e-07,\n", + " -4.3262e-07, 1.6216e-07, -2.1796e-07, 2.3822e-07, -1.6489e-07,\n", + " -4.5521e-07, -2.6305e-07, 1.9379e-07, -6.8674e-09, -1.1466e-07,\n", + " 1.2936e-07, 1.0731e-07, -3.0371e-07, 2.8924e-07, -4.0951e-08,\n", + " -5.1409e-07, 6.2814e-07, 3.0467e-07, 3.7954e-07, 1.2780e-07,\n", + " -2.8808e-07, 2.2043e-07, 2.3640e-07, -2.3760e-07, -5.5569e-08,\n", + " -7.0060e-08, 1.5659e-07, 1.9410e-07, -4.0949e-08, -3.2050e-08,\n", + " 2.8438e-07, 2.2002e-07, -8.4607e-09, -5.1829e-07, -3.3730e-07,\n", + " -2.8925e-07, 1.3643e-07, 3.6955e-07, 1.8995e-07, 3.9761e-08,\n", + " -1.3284e-07, -2.5212e-07, -1.9680e-07, -9.3926e-08, -2.3736e-07,\n", + " 5.6697e-08, -3.6571e-07, -4.4511e-07, -1.1081e-07, -1.4395e-07,\n", + " -1.1706e-07, -3.7293e-07, -7.7388e-08, -1.8207e-07, -3.4861e-07,\n", + " 1.9085e-07, -2.4633e-07, -1.3726e-07, -7.7572e-08, 1.3992e-07,\n", + " 1.5841e-07, -4.8458e-07, 6.7845e-07, -2.3140e-07, 2.3232e-07,\n", + " -4.2101e-07, -2.0065e-07, -4.1939e-07, 1.1340e-07, -4.8612e-08,\n", + " -1.0231e-07, 1.8115e-08, -4.2669e-07, 1.2405e-07, 1.4445e-07,\n", + " 8.9523e-09, -1.4322e-07, 1.4196e-07, 1.0362e-07, 4.5899e-08,\n", + " 2.0909e-08, -1.4452e-07, -2.2955e-07, -1.4234e-07, -3.3185e-08,\n", + " -1.1197e-07, 2.2181e-08, -1.3756e-07, -2.6207e-07, 9.9575e-08,\n", + " -3.1685e-07, 2.4888e-07, 2.5448e-07, 1.3901e-07, 1.4122e-07,\n", + " -2.6080e-07, -4.4499e-08, 1.7342e-07, 2.7159e-07, -1.2643e-07,\n", + " 1.7036e-07, -6.3905e-07, -1.9381e-07, 1.4063e-07, -3.4839e-07,\n", + " 9.9977e-08, 4.2366e-07, 1.0504e-07, -3.6718e-07, 1.6732e-07,\n", + " -2.7617e-07, 5.5355e-08, -3.3797e-07, -5.1699e-07, 1.2824e-07,\n", + " -1.9716e-07, 2.4884e-07, 6.7651e-08, -1.8893e-07, -1.5658e-07,\n", + " 4.7686e-07, -1.4786e-07, -1.3890e-07, -2.4762e-07, 1.0687e-07,\n", + " -1.2919e-07, 8.3134e-08, -1.0985e-07, 2.2108e-07, 3.3113e-07,\n", + " 1.3139e-07, 3.1832e-07, 5.4610e-09, 2.8666e-07, 2.2507e-07,\n", + " -1.3751e-07, 6.0405e-08, 2.8325e-07, -1.8085e-07, -3.8834e-08,\n", + " 5.2421e-07, -1.6108e-07, -1.5174e-07, -1.1893e-07, -2.0988e-07,\n", + " -2.3623e-08, -1.2304e-08, 3.3319e-07, -1.0587e-07, 2.7317e-07,\n", + " -6.2978e-08, -4.8378e-07, 1.2936e-08, 2.5831e-07, 2.5507e-09,\n", + " 2.8688e-07, 9.7795e-08, -1.4314e-07, -3.9753e-07, 3.5461e-07,\n", + " -4.0676e-07, -3.3493e-07, -3.1406e-08, -1.3037e-07, -1.8069e-07,\n", + " -3.4912e-07, -1.9761e-07, -3.6452e-08, 3.9209e-08, -4.1246e-08,\n", + " 2.9505e-07, 6.6562e-08, -8.9539e-09, -9.8791e-08, -2.1552e-07,\n", + " 4.6567e-07, -1.8645e-07, 1.2557e-07, -5.9367e-08, 1.6528e-07,\n", + " 3.6556e-07, -1.9133e-07, -6.1086e-08, -1.9499e-07, 1.2546e-07,\n", + " 2.4251e-07, 4.1227e-07, -7.8549e-08, 7.9225e-09, 7.4521e-08,\n", + " -2.0761e-07, -3.0494e-07, 4.0810e-07, 4.0615e-07, -1.6896e-07,\n", + " 3.9878e-07, 1.1604e-07, -8.1415e-09, 8.2245e-08, 2.8860e-08,\n", + " 1.8276e-07, 9.5420e-08, -3.5587e-08, -1.3545e-07, -1.6446e-07,\n", + " -3.7396e-08, 8.8892e-09, -2.4477e-07, 1.9488e-07, -1.8547e-07,\n", + " -2.4438e-08, 9.3581e-08, -5.6522e-07, 5.0627e-09, -1.8690e-07,\n", + " 4.0675e-07, 1.6759e-07, 4.4454e-07, 1.6735e-07, 3.4398e-08,\n", + " 1.1073e-07, -1.0062e-07, 1.4148e-07, -5.2365e-07, 8.8247e-08,\n", + " 2.4630e-07, -3.7311e-07, -2.7261e-08, 1.5593e-07, -5.6472e-07,\n", + " 1.7655e-07, 1.6060e-07, -2.1312e-07, -8.2568e-08, -1.9649e-07,\n", + " -1.4583e-07, 1.7356e-07, -2.3207e-07, -1.1145e-07, -3.8427e-07,\n", + " 2.9967e-07, -3.9306e-08, -6.6338e-08, -2.4694e-07, 4.0503e-08,\n", + " 2.0625e-07, 3.3832e-08, 1.4000e-07, 9.9012e-08, -1.4842e-07,\n", + " 6.9324e-08, 5.4214e-08, 3.3105e-07, -2.3108e-07, -2.9612e-07,\n", + " -2.5485e-08, -2.7959e-07, 1.1182e-07, 2.3020e-07, -1.0539e-07,\n", + " 4.2965e-07, -3.2165e-07, 2.5231e-07, 1.2803e-07, -2.2698e-08,\n", + " -2.8631e-07, -4.0032e-07, -4.7990e-07, 3.8790e-07, 5.5012e-08,\n", + " 2.1181e-07, -3.4345e-07, -1.0273e-07, 3.3141e-07, 3.9423e-08,\n", + " 3.9785e-07, 3.5342e-07, -1.7269e-07, -3.2614e-07, -4.4307e-07,\n", + " 5.1917e-07, 1.4295e-07, 3.0101e-07, 2.0181e-07, -4.3180e-07,\n", + " -7.5416e-09, -2.3724e-07, 2.5669e-07, -3.0930e-07, -2.7942e-08,\n", + " -2.2161e-07, -2.3573e-07, -5.0254e-07, -4.3599e-09, -2.7798e-08,\n", + " 2.5370e-07, -1.5972e-07, -4.3477e-08, -3.2825e-07, -8.7036e-08,\n", + " 1.0273e-07, 5.6967e-08, -2.0524e-07, 2.6476e-07, -1.1129e-08,\n", + " -2.2975e-07, -2.2688e-07, 2.2536e-07, 9.0293e-08, 5.6805e-08,\n", + " -5.3037e-07, 6.6778e-07, 2.4106e-07, -2.1485e-07, -7.2225e-08,\n", + " 3.0555e-07, 1.1304e-07, 2.7074e-07, 5.3919e-07, 2.7431e-07,\n", + " -6.5594e-09, 7.0458e-08, 2.2255e-07, 4.8071e-07, 2.7403e-07,\n", + " 1.5821e-07, 2.5005e-07, 3.4690e-07, 3.0483e-07, 3.1065e-07,\n", + " -2.5807e-07, -1.0820e-07, -1.7822e-07, -1.4796e-07, 9.4314e-08,\n", + " 1.9424e-07, -4.5983e-07, -3.7128e-08, 4.3416e-07, 2.7088e-07,\n", + " 1.7964e-07, -2.7266e-07, 2.3749e-07, 3.9147e-07, 3.3245e-07,\n", + " -3.2510e-07, 1.8839e-07, -1.8541e-07, -2.7306e-07, -2.7169e-07,\n", + " -2.6862e-07, 2.5923e-07, -5.9224e-09, 1.8698e-07, -1.0680e-07,\n", + " -3.2318e-07, -8.9712e-08, -1.6405e-07, 5.8242e-07, 2.0582e-07,\n", + " 2.5597e-07, 6.6531e-07, 3.3725e-07, 1.6997e-07, -5.5700e-07,\n", + " 1.1019e-07, -2.9223e-07, -1.1995e-07, 1.1060e-08, 2.2936e-07,\n", + " -4.6518e-07, -5.1757e-08, -2.9770e-08, 3.7894e-07, -1.8620e-07,\n", + " -4.7999e-07, -7.0762e-08, -3.0505e-08, -1.0269e-07, -4.2373e-08,\n", + " 9.0745e-08, 2.0844e-07, -4.4592e-07, 1.2248e-07, 3.2833e-07,\n", + " -1.6091e-07, -5.9777e-08, 2.4343e-07, 2.8965e-07, 2.7996e-07,\n", + " -2.8026e-07, 1.2297e-07, -1.3367e-07, 1.1679e-07, -3.0673e-08,\n", + " 2.9914e-08, -3.7007e-07, 9.5431e-08, 2.8178e-07, 4.1163e-07,\n", + " 3.4943e-07, -1.7463e-07, -1.4960e-07, -3.9748e-07, 3.0402e-07,\n", + " 6.0561e-10, -2.5993e-07, -5.3542e-07, 8.5851e-08, 3.7541e-08,\n", + " -5.6760e-08, -3.6608e-07, -1.1721e-07, -2.7867e-07, 4.2660e-07,\n", + " -3.0838e-09, -5.5235e-07, 2.5093e-07, -1.1648e-07, 1.9175e-08,\n", + " -4.4914e-07, -1.0806e-08, 3.2415e-07, 7.5054e-07, -8.4961e-08,\n", + " 1.1927e-07, 5.3004e-09, 1.1822e-07, -6.2535e-08, -1.9777e-07,\n", + " 1.4558e-07, -1.3232e-07, -2.5719e-07, 1.0912e-07, -4.6794e-07,\n", + " 3.1465e-07, 3.9155e-07, -2.1163e-07, -6.1621e-08, 4.9127e-08,\n", + " 1.4371e-08, 5.8589e-07, -3.0203e-07, 3.2204e-08, -3.6141e-07,\n", + " -2.1699e-07, 5.8263e-07], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.conv2.weight',\n", + " tensor([[[[-8.8919e-03, -5.5261e-03, -1.1649e-02],\n", + " [-3.2865e-03, -3.2163e-03, 2.1970e-03],\n", + " [-9.5255e-03, -1.4290e-02, 1.0239e-02]],\n", + " \n", + " [[-8.4998e-03, 6.0042e-03, -1.1939e-02],\n", + " [ 1.4830e-02, 3.7390e-03, 1.3846e-02],\n", + " [ 1.1486e-02, 4.7125e-03, -1.0030e-02]],\n", + " \n", + " [[-8.3829e-03, 7.1931e-03, 3.8290e-03],\n", + " [-1.4467e-02, 1.2313e-02, 3.8561e-05],\n", + " [-1.3615e-02, -4.5642e-03, 1.5877e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.8610e-03, -1.3229e-02, 5.3278e-03],\n", + " [-1.2192e-02, 5.5769e-04, -2.9691e-04],\n", + " [ 2.3656e-03, -4.9448e-03, -3.2270e-03]],\n", + " \n", + " [[-7.9613e-04, 8.4969e-03, 5.2952e-03],\n", + " [ 1.5548e-02, 8.6120e-03, -1.1546e-02],\n", + " [-5.0554e-03, -4.9455e-03, -1.2913e-02]],\n", + " \n", + " [[ 2.3799e-03, 6.1107e-03, 1.0281e-02],\n", + " [ 1.4803e-03, -7.1119e-03, -1.9499e-02],\n", + " [-1.1962e-02, 2.1622e-02, 1.0925e-02]]],\n", + " \n", + " \n", + " [[[ 1.4340e-02, 7.1889e-03, 7.5618e-03],\n", + " [-1.3251e-02, 3.6634e-03, 3.7104e-04],\n", + " [ 4.6030e-03, -2.0175e-02, -6.0965e-03]],\n", + " \n", + " [[ 1.2104e-02, -9.1580e-03, 8.1003e-03],\n", + " [-1.2511e-02, -1.5410e-02, 6.9187e-03],\n", + " [ 3.7470e-03, -2.3987e-03, -7.4581e-03]],\n", + " \n", + " [[-5.2357e-03, -1.8020e-03, -1.4156e-02],\n", + " [ 7.3226e-03, -2.2034e-04, 7.5318e-04],\n", + " [-9.4075e-03, -4.4618e-03, -1.6356e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 8.0691e-04, -1.6836e-02, 1.4165e-02],\n", + " [-1.4672e-02, -3.1906e-03, -1.0278e-03],\n", + " [ 4.1163e-03, -3.3362e-03, 1.3807e-02]],\n", + " \n", + " [[-2.7671e-03, -5.6578e-03, -1.0477e-02],\n", + " [ 6.5509e-03, 1.2876e-02, -1.5242e-02],\n", + " [-3.4014e-03, -1.0206e-02, -9.1380e-03]],\n", + " \n", + " [[ 1.4649e-02, 1.1887e-02, -3.7188e-03],\n", + " [ 1.2284e-02, -1.1505e-02, -1.2666e-02],\n", + " [-2.5857e-03, -8.7228e-03, -2.0777e-03]]],\n", + " \n", + " \n", + " [[[ 1.2072e-02, -2.3061e-03, -6.7005e-03],\n", + " [-8.0823e-03, -8.3503e-03, 6.9081e-03],\n", + " [-9.8445e-03, -2.6542e-03, 2.6641e-03]],\n", + " \n", + " [[ 2.7775e-03, -2.5929e-03, 1.9189e-02],\n", + " [-2.7051e-03, 5.8931e-03, 3.4024e-04],\n", + " [-1.0870e-02, -1.3271e-02, 1.2677e-02]],\n", + " \n", + " [[-9.7288e-04, 4.6576e-04, 6.6519e-04],\n", + " [-1.8496e-03, 1.5163e-02, -1.3952e-02],\n", + " [-6.4673e-03, 4.5288e-03, 5.0748e-04]],\n", + " \n", + " ...,\n", + " \n", + " [[ 8.5364e-03, 9.1448e-03, 8.0631e-04],\n", + " [-2.2104e-03, -5.2943e-03, -1.3910e-02],\n", + " [-1.2475e-02, 4.3112e-03, 9.1181e-03]],\n", + " \n", + " [[-4.0816e-03, -2.2592e-03, 1.3574e-02],\n", + " [ 1.5750e-02, -1.5506e-02, -1.4866e-03],\n", + " [-1.6357e-02, 5.1772e-03, 9.1530e-03]],\n", + " \n", + " [[ 7.5529e-03, 2.9393e-03, -7.6150e-03],\n", + " [ 4.8107e-03, -1.7729e-02, -2.4929e-03],\n", + " [-3.1347e-04, -5.6910e-04, 4.8326e-03]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 3.1950e-03, 1.2405e-02, -4.2770e-03],\n", + " [ 8.9297e-03, 2.9347e-03, 5.5196e-03],\n", + " [-8.4849e-03, 4.7235e-03, 1.4928e-02]],\n", + " \n", + " [[-1.9742e-02, -3.4240e-03, 8.5688e-03],\n", + " [-9.5304e-03, 6.9871e-03, -4.9104e-03],\n", + " [-1.1383e-02, -1.4184e-03, 8.3761e-03]],\n", + " \n", + " [[ 1.1749e-02, -1.3858e-02, 7.7950e-03],\n", + " [-1.4731e-02, -1.3514e-03, 1.0907e-02],\n", + " [-7.3442e-03, 4.8755e-03, 7.4672e-03]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.2847e-02, 1.5669e-02, 1.3149e-02],\n", + " [-2.7236e-03, -5.3160e-03, -2.6364e-03],\n", + " [ 4.9001e-03, -7.4731e-03, 1.0864e-02]],\n", + " \n", + " [[ 6.3456e-03, 1.4196e-02, 3.2071e-03],\n", + " [ 1.7465e-03, -4.9140e-03, -9.9985e-03],\n", + " [-5.4781e-03, 3.3783e-03, 2.3971e-05]],\n", + " \n", + " [[-8.7421e-03, -4.3363e-03, -7.4800e-03],\n", + " [ 1.8229e-02, -1.3717e-03, 1.2374e-02],\n", + " [-7.2042e-03, 1.1575e-02, 1.8360e-03]]],\n", + " \n", + " \n", + " [[[ 8.4327e-03, 3.6422e-03, 2.9160e-03],\n", + " [-1.1756e-02, -9.3760e-03, 1.2245e-02],\n", + " [ 7.0258e-03, 4.5228e-03, -9.3750e-04]],\n", + " \n", + " [[ 7.4205e-03, 2.8433e-03, 1.7987e-03],\n", + " [ 3.5197e-03, -1.4728e-02, -6.8848e-03],\n", + " [ 4.6512e-03, 4.1685e-03, 4.2135e-04]],\n", + " \n", + " [[-1.8019e-02, 1.2760e-02, 1.1175e-02],\n", + " [-2.3896e-03, 1.3814e-02, 1.6097e-02],\n", + " [ 6.7313e-03, -5.1544e-03, -1.9122e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.5587e-02, 6.6637e-03, -9.2753e-03],\n", + " [-1.5219e-02, -1.4307e-02, 1.1896e-03],\n", + " [-1.3813e-02, 4.6334e-03, 2.9915e-03]],\n", + " \n", + " [[-1.4803e-02, -7.3671e-03, 1.1902e-02],\n", + " [ 5.0613e-03, 4.7833e-03, 3.3237e-03],\n", + " [ 7.2514e-03, 2.7958e-04, 1.5791e-02]],\n", + " \n", + " [[ 1.8511e-02, 5.9318e-03, -1.2440e-02],\n", + " [-6.7291e-03, -8.8253e-04, 4.5097e-04],\n", + " [ 5.8221e-03, 1.0105e-02, 2.6565e-03]]],\n", + " \n", + " \n", + " [[[-5.3214e-03, -3.0500e-03, 1.3938e-02],\n", + " [ 1.5274e-02, 1.0179e-02, 1.9662e-02],\n", + " [-9.6668e-05, 4.1659e-03, -6.4923e-03]],\n", + " \n", + " [[-1.1835e-02, -8.5019e-03, 1.4460e-03],\n", + " [ 1.6145e-03, 5.6016e-03, 5.1102e-03],\n", + " [-1.6813e-02, 1.4814e-02, 4.8468e-04]],\n", + " \n", + " [[-1.2867e-02, -1.7569e-03, -1.4131e-02],\n", + " [-4.2748e-03, 1.0950e-02, -2.5505e-03],\n", + " [-8.8524e-03, -7.2978e-03, -1.0411e-02]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.5008e-02, -3.9596e-03, -8.1293e-03],\n", + " [-1.4422e-02, 9.5751e-03, 1.0249e-02],\n", + " [-1.0301e-02, 3.5868e-03, -8.3649e-03]],\n", + " \n", + " [[ 8.8733e-04, 1.7633e-02, -1.1830e-02],\n", + " [-1.5473e-02, 9.8210e-03, -8.5470e-03],\n", + " [-6.6058e-03, 3.0551e-04, -5.6332e-03]],\n", + " \n", + " [[-4.1049e-03, 3.9279e-03, -1.3175e-02],\n", + " [-1.0068e-02, -2.7920e-03, 1.9161e-02],\n", + " [ 1.5485e-02, -6.6603e-03, -1.1391e-02]]]], device='cuda:0',\n", + " grad_fn=)),\n", + " ('layer4.0.bn2.weight',\n", + " tensor([-4.7684e-07, 2.3842e-07, -3.5763e-07, -1.7881e-07, 2.3842e-07,\n", + " -7.1526e-07, -2.9802e-07, 2.3842e-07, -3.5763e-07, -5.9605e-07,\n", + " 2.3842e-07, -1.7881e-07, 3.5763e-07, -5.9605e-08, 1.1921e-07,\n", + " -2.3842e-07, -1.7881e-07, 0.0000e+00, -4.1723e-07, -7.7486e-07,\n", + " 0.0000e+00, -6.5565e-07, -2.3842e-07, 2.3842e-07, -7.7486e-07,\n", + " 3.5763e-07, 2.3842e-07, -2.3842e-07, 4.7684e-07, -2.3842e-07,\n", + " -5.9605e-07, 0.0000e+00, -3.5763e-07, -5.9605e-08, 0.0000e+00,\n", + " 1.1921e-07, 0.0000e+00, 5.9605e-07, 3.5763e-07, -5.9605e-07,\n", + " 1.1921e-07, 1.1921e-07, -1.1921e-07, 8.3447e-07, 3.5763e-07,\n", + " 1.1921e-07, 3.5763e-07, 1.1921e-07, -3.5763e-07, 4.7684e-07,\n", + " -2.9802e-07, -4.1723e-07, -5.9605e-08, -5.3644e-07, 5.9605e-07,\n", + " -6.5565e-07, -2.3842e-07, -4.1723e-07, 7.1526e-07, -1.7881e-07,\n", + " 0.0000e+00, 1.1921e-07, 4.7684e-07, -5.9605e-08, -1.1921e-07,\n", + " -5.9605e-07, -3.5763e-07, 0.0000e+00, -1.1921e-07, 0.0000e+00,\n", + " -5.9605e-08, -9.5367e-07, -4.1723e-07, -2.9802e-07, 2.3842e-07,\n", + " -7.7486e-07, 2.3842e-07, -5.9605e-08, -3.5763e-07, 4.7684e-07,\n", + " 0.0000e+00, 4.7684e-07, -1.1921e-07, -5.9605e-08, 1.1921e-07,\n", + " -5.3644e-07, 1.1921e-07, 2.3842e-07, -7.7486e-07, -2.9802e-07,\n", + " -1.7881e-07, 0.0000e+00, -2.3842e-07, -1.7881e-07, -1.7881e-07,\n", + " 1.1921e-06, -1.7881e-07, 3.5763e-07, 2.3842e-07, 1.1921e-07,\n", + " -2.3842e-07, -5.9605e-07, -3.5763e-07, -1.0729e-06, -1.1921e-07,\n", + " 2.3842e-07, 3.5763e-07, -3.5763e-07, 4.7684e-07, -1.7881e-07,\n", + " 0.0000e+00, -7.7486e-07, -6.5565e-07, -2.3842e-07, 0.0000e+00,\n", + " 4.7684e-07, -2.9802e-07, 2.3842e-07, 1.1921e-07, -5.3644e-07,\n", + " 0.0000e+00, -1.1921e-07, -4.1723e-07, -4.1723e-07, 0.0000e+00,\n", + " -6.5565e-07, -4.1723e-07, 0.0000e+00, -1.7881e-07, 1.1921e-07,\n", + " 0.0000e+00, -5.9605e-07, 5.9605e-07, 1.1921e-07, 5.9605e-07,\n", + " -1.1921e-07, 1.1921e-07, -1.1921e-07, -4.7684e-07, -5.3644e-07,\n", + " -5.3644e-07, -8.3447e-07, -1.1921e-07, -5.9605e-08, -5.9605e-08,\n", + " 3.5763e-07, -7.1526e-07, 0.0000e+00, 1.0729e-06, 0.0000e+00,\n", + " 0.0000e+00, 2.3842e-07, -4.1723e-07, 4.7684e-07, -5.9605e-08,\n", + " 0.0000e+00, -5.3644e-07, 2.3842e-07, -5.9605e-07, 2.3842e-07,\n", + " -4.7684e-07, -1.1921e-07, -4.1723e-07, -2.3842e-07, -2.3842e-07,\n", + " 4.7684e-07, 4.7684e-07, -2.9802e-07, 0.0000e+00, 2.3842e-07,\n", + " -1.1921e-07, -2.3842e-07, -5.9605e-08, 2.3842e-07, 4.7684e-07,\n", + " 5.9605e-07, -2.3842e-07, 3.5763e-07, -2.3842e-07, 0.0000e+00,\n", + " -1.7881e-07, 1.1921e-07, -2.3842e-07, 2.3842e-07, 7.1526e-07,\n", + " 7.1526e-07, 1.1921e-07, 1.1921e-07, -2.9802e-07, -2.3842e-07,\n", + " 0.0000e+00, 4.7684e-07, -2.3842e-07, -1.1325e-06, -5.9605e-08,\n", + " -5.9605e-08, -3.5763e-07, -5.9605e-08, -1.7881e-07, -4.1723e-07,\n", + " 5.9605e-07, -3.5763e-07, 4.7684e-07, 1.1921e-07, -8.3447e-07,\n", + " 1.1921e-07, -1.0729e-06, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", + " 1.1921e-07, -2.3842e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07,\n", + " -5.3644e-07, 0.0000e+00, 4.7684e-07, -5.9605e-08, 2.3842e-07,\n", + " 0.0000e+00, -5.9605e-08, -1.1921e-07, -1.1921e-07, 0.0000e+00,\n", + " 3.5763e-07, -5.9605e-08, -4.1723e-07, 1.1921e-07, -4.1723e-07,\n", + " 2.3842e-07, -2.9802e-07, 1.1921e-07, 0.0000e+00, 3.5763e-07,\n", + " -1.1921e-07, -3.5763e-07, 1.1921e-07, 2.3842e-07, -1.7881e-07,\n", + " -8.3447e-07, 2.3842e-07, -2.9802e-07, 2.3842e-07, -5.9605e-08,\n", + " 2.3842e-07, -5.9605e-08, 0.0000e+00, 2.3842e-07, 2.3842e-07,\n", + " 5.9605e-07, 1.1921e-07, 2.3842e-07, -5.9605e-07, -1.7881e-07,\n", + " 0.0000e+00, 0.0000e+00, 2.3842e-07, 4.7684e-07, 0.0000e+00,\n", + " -4.1723e-07, 5.9605e-07, 2.3842e-07, 3.5763e-07, -4.1723e-07,\n", + " 0.0000e+00, 3.5763e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07,\n", + " 2.3842e-07, -5.9605e-07, 7.1526e-07, -1.1325e-06, 5.9605e-07,\n", + " -1.7881e-07, -5.9605e-08, -2.9802e-07, -5.9605e-08, 1.1921e-07,\n", + " 2.3842e-07, -2.9802e-07, -2.9802e-07, -8.3447e-07, 7.1526e-07,\n", + " 3.5763e-07, 2.3842e-07, 3.5763e-07, -2.9802e-07, -4.1723e-07,\n", + " -1.1921e-07, -5.9605e-07, -1.1921e-07, -2.9802e-07, 3.5763e-07,\n", + " -3.5763e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07, -4.1723e-07,\n", + " -4.7684e-07, -4.1723e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", + " 3.5763e-07, 1.1921e-07, 3.5763e-07, 1.1921e-07, -1.7881e-07,\n", + " -3.5763e-07, -4.7684e-07, 1.1921e-07, 3.5763e-07, -2.9802e-07,\n", + " 0.0000e+00, 4.7684e-07, 2.3842e-07, -5.9605e-08, -2.3842e-07,\n", + " 4.7684e-07, 7.1526e-07, 0.0000e+00, -2.3842e-07, 0.0000e+00,\n", + " -1.7881e-07, 1.1921e-07, -1.1921e-07, 0.0000e+00, -1.7881e-07,\n", + " 1.1921e-07, -2.9802e-07, -4.7684e-07, 5.9605e-07, 1.1921e-07,\n", + " 4.7684e-07, -7.1526e-07, -5.9605e-08, 2.3842e-07, 2.3842e-07,\n", + " 0.0000e+00, -1.7881e-07, 2.3842e-07, 0.0000e+00, 1.1921e-07,\n", + " 1.1921e-07, 1.1921e-07, -1.1921e-07, 2.3842e-07, 3.5763e-07,\n", + " 2.3842e-07, 4.7684e-07, 2.3842e-07, -1.1921e-07, -1.1921e-07,\n", + " 5.9605e-07, -1.7881e-07, 0.0000e+00, -2.3842e-07, -3.5763e-07,\n", + " 1.1921e-07, 3.5763e-07, -1.1921e-07, 0.0000e+00, -5.9605e-08,\n", + " -2.3842e-07, 5.9605e-07, 1.1921e-07, -1.7881e-07, 1.1921e-07,\n", + " -1.7881e-07, -1.3113e-06, 1.1921e-07, -3.5763e-07, -5.9605e-07,\n", + " -2.9802e-07, 1.1921e-07, 4.7684e-07, 3.5763e-07, -2.3842e-07,\n", + " 2.3842e-07, 1.1921e-07, 7.1526e-07, 3.5763e-07, -5.9605e-08,\n", + " -2.3842e-07, 2.3842e-07, 3.5763e-07, -5.9605e-08, -3.5763e-07,\n", + " 4.7684e-07, -5.9605e-08, 5.9605e-07, -9.5367e-07, -1.1921e-07,\n", + " 2.3842e-07, -2.9802e-07, 1.1921e-07, -5.3644e-07, 1.1921e-07,\n", + " -4.7684e-07, 3.5763e-07, -3.5763e-07, 2.3842e-07, 1.1921e-07,\n", + " -5.3644e-07, -2.3842e-07, 3.5763e-07, -7.7486e-07, -4.7684e-07,\n", + " -4.7684e-07, -1.1921e-07, -1.1921e-07, -5.9605e-07, 1.1921e-07,\n", + " -2.9802e-07, -9.5367e-07, -3.5763e-07, -2.3842e-07, 0.0000e+00,\n", + " 2.3842e-07, 5.9605e-07, -3.5763e-07, -5.3644e-07, -1.7881e-07,\n", + " -5.3644e-07, 3.5763e-07, 2.3842e-07, -1.1921e-07, -1.7881e-07,\n", + " -3.5763e-07, -1.1921e-07, 4.7684e-07, 4.7684e-07, 3.5763e-07,\n", + " 1.1921e-07, 1.1921e-07, 3.5763e-07, -4.1723e-07, -4.7684e-07,\n", + " 1.1921e-07, -2.3842e-07, 2.3842e-07, 3.5763e-07, 2.3842e-07,\n", + " 0.0000e+00, 1.1921e-07, 5.9605e-07, -1.2517e-06, 0.0000e+00,\n", + " 4.7684e-07, -2.9802e-07, 4.7684e-07, -1.7881e-07, 4.7684e-07,\n", + " 1.1921e-07, 3.5763e-07, -2.3842e-07, -5.3644e-07, 3.5763e-07,\n", + " 1.1921e-07, 3.5763e-07, -1.1921e-07, -2.9802e-07, -2.3842e-07,\n", + " 1.1921e-07, -1.7881e-07, -8.9407e-07, 1.1921e-07, 3.5763e-07,\n", + " -4.7684e-07, -2.3842e-07, 1.1921e-07, 1.1921e-07, -1.1921e-07,\n", + " 1.1921e-07, 1.1921e-07, -1.1921e-07, 5.9605e-07, -5.9605e-08,\n", + " 4.7684e-07, 1.1921e-07, 0.0000e+00, 2.3842e-07, 1.1921e-07,\n", + " -2.3842e-07, 2.3842e-07, 3.5763e-07, -1.1921e-07, 0.0000e+00,\n", + " -3.5763e-07, -2.9802e-07, 4.7684e-07, -5.9605e-08, 1.1921e-07,\n", + " 2.3842e-07, 1.1921e-07, -4.7684e-07, -5.9605e-08, 2.3842e-07,\n", + " 1.1921e-07, -2.9802e-07, -5.9605e-08, 5.9605e-07, 0.0000e+00,\n", + " 3.5763e-07, -7.7486e-07, 0.0000e+00, 1.1921e-07, 2.3842e-07,\n", + " -6.5565e-07, 3.5763e-07], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.bn2.bias',\n", + " tensor([-2.5952e-07, -5.2718e-08, -3.8981e-07, -1.2332e-07, 2.4942e-07,\n", + " -4.1001e-07, -1.8458e-07, 2.2059e-07, 3.3934e-11, -5.6743e-08,\n", + " 3.8876e-07, -2.2358e-07, 1.5773e-07, -3.4872e-07, -2.7905e-07,\n", + " -2.3630e-08, -3.0215e-08, -2.4326e-07, -2.1305e-07, -2.9157e-07,\n", + " -4.5004e-07, -3.3447e-07, -1.6505e-07, -2.8462e-07, -1.1983e-07,\n", + " 3.0670e-07, -1.7117e-07, 2.2260e-07, -1.8211e-07, -3.3789e-07,\n", + " -6.4035e-07, -3.4069e-07, -2.1152e-07, 6.1553e-08, -1.1388e-07,\n", + " 3.9278e-08, 6.4294e-07, 5.3870e-07, -2.0622e-07, -5.0324e-07,\n", + " -8.4042e-08, 1.2858e-07, -3.1810e-07, 8.5574e-07, 2.0792e-07,\n", + " -3.2931e-07, 4.0081e-07, 1.2134e-07, 1.0071e-07, 9.3611e-07,\n", + " -3.1569e-07, -1.4190e-07, -9.8398e-08, -4.4183e-07, 3.8909e-07,\n", + " -2.3117e-07, -3.3583e-07, -4.5050e-07, 3.9275e-07, 5.3557e-07,\n", + " -8.5330e-07, -4.7482e-08, -7.7420e-08, 4.5063e-08, 5.6890e-08,\n", + " -6.9403e-07, -6.6087e-07, -3.2312e-07, -1.5117e-07, -5.0290e-07,\n", + " -8.3522e-08, -6.2873e-07, -1.9139e-07, -4.7217e-08, -1.4439e-07,\n", + " -3.0978e-07, -3.0010e-07, -4.4070e-07, 6.1973e-08, 4.8330e-07,\n", + " -1.5466e-07, -6.7563e-08, -6.4472e-07, -3.3934e-08, -3.2509e-07,\n", + " 3.2013e-09, -1.0098e-08, 6.1731e-08, -9.7081e-08, 6.2362e-08,\n", + " 2.2826e-07, -8.8843e-07, -7.5161e-07, -4.3784e-07, 5.1198e-08,\n", + " 7.3383e-07, -8.6796e-08, 1.3520e-07, 2.7753e-07, -7.6281e-08,\n", + " 3.2856e-07, -8.5474e-07, 1.4127e-07, 1.6500e-07, -1.1475e-07,\n", + " -9.0230e-08, -3.4021e-08, -4.5122e-07, 1.0504e-07, -3.4202e-08,\n", + " -6.6387e-09, -4.7148e-07, -5.1771e-07, -3.5938e-07, 1.1671e-07,\n", + " 7.6556e-08, 1.0592e-08, 2.3111e-07, -1.5009e-07, -2.9490e-07,\n", + " -2.6290e-07, -2.8363e-07, -4.4743e-07, -4.2879e-07, 1.9960e-07,\n", + " -2.2103e-07, 5.0751e-07, 2.3843e-08, -2.6756e-07, -1.8564e-07,\n", + " -2.7111e-07, -1.7106e-07, 8.1063e-07, -4.9894e-07, 1.3639e-07,\n", + " -4.6950e-07, 1.0927e-07, -1.2955e-07, -4.4637e-07, -8.5186e-08,\n", + " -6.1976e-08, -5.0219e-08, -6.6460e-07, -1.8259e-07, -2.3600e-07,\n", + " -1.2455e-07, 1.3833e-07, 2.3173e-07, 1.3480e-07, 2.6061e-07,\n", + " -2.2656e-07, -2.9893e-07, -3.5553e-07, 4.9369e-07, 2.8315e-07,\n", + " -1.6586e-07, -7.3051e-07, 3.2315e-07, -2.5779e-07, 1.5562e-07,\n", + " -3.1320e-07, -5.5600e-08, 6.4858e-08, -4.3620e-07, -3.0921e-08,\n", + " 1.7959e-07, 4.7084e-07, -2.9651e-07, 1.8505e-08, 4.9029e-07,\n", + " -2.0760e-07, -5.8565e-07, 1.7540e-07, -2.5290e-08, 5.8386e-08,\n", + " 2.4527e-07, 8.4265e-08, 3.2661e-08, -2.2252e-08, -2.5374e-07,\n", + " -7.2610e-08, 1.9237e-07, -3.9457e-07, -8.6376e-08, 9.7661e-08,\n", + " 8.1748e-08, -1.8011e-07, 5.4112e-08, -2.4380e-07, 4.2298e-08,\n", + " -2.2257e-07, 2.4752e-07, -8.2461e-07, -4.9300e-07, -2.8506e-07,\n", + " 1.4784e-07, 4.9734e-08, -3.3205e-07, -1.1681e-07, -4.5991e-07,\n", + " -1.0016e-07, -4.8842e-07, 3.3371e-07, -4.3462e-07, -7.9114e-07,\n", + " -6.8228e-08, -4.7129e-07, -3.3589e-07, 2.6814e-07, 1.9327e-07,\n", + " 1.4206e-07, 3.3498e-08, -5.5501e-08, 1.7940e-07, 4.2684e-07,\n", + " 1.7147e-07, 5.6309e-08, -7.1572e-09, -3.8246e-08, -2.8099e-07,\n", + " 9.5294e-08, -4.6339e-07, 5.0270e-07, 1.0481e-08, 1.9457e-07,\n", + " -2.0168e-08, 8.2068e-08, -1.6883e-09, 2.6224e-07, -3.9644e-07,\n", + " 4.4707e-07, -8.3741e-08, -1.7601e-07, -1.7361e-07, 7.8584e-07,\n", + " 7.0257e-11, -1.2827e-07, 3.2551e-07, 6.4153e-07, -3.4501e-08,\n", + " -7.0563e-07, 1.2502e-07, -5.3899e-07, 6.2128e-07, 1.6744e-07,\n", + " 3.0381e-08, 3.1756e-07, 1.4472e-07, 3.9837e-07, -6.7176e-08,\n", + " 7.3281e-07, -3.6448e-07, -2.5766e-08, 9.8277e-09, -5.1608e-07,\n", + " -2.1131e-07, -4.9822e-07, 6.0511e-07, -2.0183e-08, 2.1366e-07,\n", + " -1.2711e-07, 3.8626e-07, 3.8279e-07, 7.5298e-07, -6.0340e-07,\n", + " -4.7458e-08, -7.7033e-08, -2.6878e-07, 1.7127e-07, -4.7343e-07,\n", + " -5.9129e-09, -6.3804e-07, 3.3628e-07, -2.4381e-07, 3.1775e-07,\n", + " -6.1137e-07, -4.6261e-07, -4.1723e-07, 5.3470e-07, 7.1795e-09,\n", + " -2.3043e-08, -1.2530e-07, -5.4000e-07, -6.3562e-07, 1.3268e-07,\n", + " 2.2244e-07, -1.7262e-07, 4.3062e-07, -1.3121e-07, -2.1163e-07,\n", + " 6.3334e-08, -6.1981e-07, 6.5072e-08, -2.1843e-07, 1.8716e-08,\n", + " 7.4588e-08, 4.0129e-07, -8.5405e-08, 6.3721e-08, -4.6985e-07,\n", + " -2.2629e-07, -4.9808e-07, 6.7194e-07, 1.3299e-07, -4.3199e-09,\n", + " 2.7960e-07, -3.2597e-07, -2.8505e-08, -1.2399e-07, -2.7097e-07,\n", + " 8.7115e-08, -6.0820e-07, -1.7309e-07, 3.7368e-07, -2.4287e-07,\n", + " -1.1012e-07, 1.2617e-08, 1.7739e-07, -2.6112e-07, -3.0125e-07,\n", + " -1.2478e-07, 2.5956e-07, -1.0944e-07, 1.7735e-07, -5.8064e-08,\n", + " -2.0758e-07, -2.8337e-07, -2.6004e-07, 3.2003e-08, -3.9736e-07,\n", + " 2.9050e-07, -2.3287e-08, -2.2765e-07, 4.0771e-07, 4.0802e-07,\n", + " 7.0819e-08, 1.4434e-07, -1.8711e-07, -2.4609e-08, 2.6868e-07,\n", + " -1.8774e-08, -2.0145e-07, 1.3446e-07, 5.9894e-07, -5.2298e-08,\n", + " 2.3941e-07, 2.3950e-07, -3.1209e-07, 3.9168e-07, -3.1503e-07,\n", + " -5.3224e-07, 4.3975e-07, -9.7231e-09, -3.8906e-07, -3.3958e-07,\n", + " 1.7475e-07, -1.2887e-07, -7.8835e-07, 6.8751e-08, -4.2049e-07,\n", + " 1.2126e-07, 8.8542e-08, -5.7822e-07, -4.5029e-07, -1.0936e-07,\n", + " -2.1707e-07, 3.0449e-07, 2.9369e-07, 1.6307e-07, -2.1861e-07,\n", + " -4.1288e-07, -9.9163e-07, 3.3039e-07, -1.5236e-07, -1.9777e-08,\n", + " 3.7243e-07, -3.4649e-07, -3.2635e-07, -2.5982e-07, -2.4515e-07,\n", + " 7.5304e-08, -6.0096e-08, 2.8725e-07, 9.5974e-08, 1.0231e-07,\n", + " -6.2956e-09, 9.3015e-08, 5.1241e-07, -3.7457e-07, -5.8680e-07,\n", + " 5.1277e-08, 1.5735e-07, 1.5098e-07, -4.7827e-07, 1.5020e-07,\n", + " 3.2007e-07, -1.1792e-07, 4.1907e-07, -5.4599e-07, 3.5650e-07,\n", + " -7.2770e-07, 2.4135e-07, -5.4777e-08, -1.6205e-07, 1.0242e-07,\n", + " -8.1226e-07, -2.6392e-07, -3.5215e-07, -5.7353e-07, -2.3653e-07,\n", + " -7.6270e-07, -1.0341e-07, -1.5335e-07, -4.0521e-07, 5.1136e-08,\n", + " -2.1407e-07, -1.1061e-06, -1.1568e-07, -1.8562e-07, -2.2229e-07,\n", + " 1.7121e-07, 4.0764e-07, -5.0914e-07, -1.5899e-07, 1.9196e-07,\n", + " -4.7060e-07, 1.8851e-07, -5.7792e-08, -4.6329e-07, -3.9686e-07,\n", + " -1.8117e-08, -1.4392e-07, 4.1560e-07, 4.7124e-07, -2.5345e-08,\n", + " -2.1026e-07, -1.2185e-07, -2.1304e-07, 1.5910e-09, -3.9998e-07,\n", + " -1.5126e-07, -2.4613e-07, -1.0463e-07, 1.4167e-07, 2.5327e-08,\n", + " 3.7949e-07, -5.3259e-07, 7.2235e-07, -1.0368e-06, -2.0211e-07,\n", + " 4.1569e-07, 5.9008e-09, 2.3589e-07, -1.1660e-07, 3.8977e-08,\n", + " -2.6139e-07, 2.5065e-07, -3.0728e-07, -2.3161e-07, 5.0300e-08,\n", + " 4.0945e-08, -3.4996e-08, -2.7710e-07, -4.3644e-09, -1.2612e-07,\n", + " -3.5672e-07, -3.0295e-07, -3.4237e-07, -1.6870e-07, 3.2721e-07,\n", + " -2.5081e-07, 1.2698e-07, 2.5012e-07, 4.2570e-07, -3.8390e-07,\n", + " 1.6247e-07, 1.5678e-07, -1.8048e-07, 2.3226e-07, 2.6242e-07,\n", + " 1.4829e-07, -1.9754e-07, -2.4445e-07, -1.5111e-07, -1.8443e-07,\n", + " -5.3174e-07, -4.4128e-08, 2.1757e-07, -4.6750e-07, -7.1977e-08,\n", + " -1.7918e-07, -3.1625e-07, 2.4026e-07, 7.8625e-08, -4.3738e-08,\n", + " -2.1923e-09, -5.7170e-07, -4.3710e-07, -2.2763e-07, 5.7661e-07,\n", + " -3.9948e-07, 1.6238e-07, 2.4817e-07, 4.2508e-07, -5.4188e-07,\n", + " 8.4595e-08, -2.3122e-07, 2.6185e-07, -7.8099e-08, -1.5002e-07,\n", + " -4.9218e-08, 2.9320e-07], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.shortcut.0.weight',\n", + " tensor([[[[-0.0539]],\n", + " \n", + " [[ 0.0183]],\n", + " \n", + " [[ 0.0007]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0457]],\n", + " \n", + " [[-0.0235]],\n", + " \n", + " [[ 0.0321]]],\n", + " \n", + " \n", + " [[[ 0.0683]],\n", + " \n", + " [[ 0.0342]],\n", + " \n", + " [[-0.0583]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.0131]],\n", + " \n", + " [[-0.0058]],\n", + " \n", + " [[ 0.0132]]],\n", + " \n", + " \n", + " [[[ 0.0510]],\n", + " \n", + " [[ 0.0250]],\n", + " \n", + " [[ 0.0626]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0634]],\n", + " \n", + " [[-0.0507]],\n", + " \n", + " [[-0.0434]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 0.0283]],\n", + " \n", + " [[-0.0284]],\n", + " \n", + " [[-0.0425]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.0560]],\n", + " \n", + " [[ 0.0462]],\n", + " \n", + " [[-0.0382]]],\n", + " \n", + " \n", + " [[[ 0.0125]],\n", + " \n", + " [[-0.0113]],\n", + " \n", + " [[-0.0371]],\n", + " \n", + " ...,\n", + " \n", + " [[ 0.0561]],\n", + " \n", + " [[-0.0112]],\n", + " \n", + " [[-0.0145]]],\n", + " \n", + " \n", + " [[[ 0.0360]],\n", + " \n", + " [[-0.0027]],\n", + " \n", + " [[ 0.0408]],\n", + " \n", + " ...,\n", + " \n", + " [[-0.0422]],\n", + " \n", + " [[-0.0196]],\n", + " \n", + " [[ 0.0299]]]], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.shortcut.1.weight',\n", + " tensor([-4.7684e-07, 1.1921e-07, -1.7881e-07, -5.9605e-08, 1.1921e-07,\n", + " 0.0000e+00, 3.5763e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07,\n", + " -5.9605e-08, 0.0000e+00, -4.7684e-07, -2.9802e-07, 1.1921e-07,\n", + " 3.5763e-07, -2.9802e-07, -2.3842e-07, 0.0000e+00, 2.3842e-07,\n", + " -8.9407e-07, 1.1921e-07, 0.0000e+00, -3.5763e-07, 2.3842e-07,\n", + " 0.0000e+00, 0.0000e+00, 0.0000e+00, -6.5565e-07, -2.3842e-07,\n", + " -2.9802e-07, 2.3842e-07, -5.9605e-07, -1.1921e-07, 1.1921e-07,\n", + " -2.9802e-07, 5.9605e-07, 2.3842e-07, -8.3447e-07, -4.1723e-07,\n", + " 0.0000e+00, 5.9605e-07, -2.9802e-07, 7.1526e-07, 2.3842e-07,\n", + " -2.3842e-07, 4.7684e-07, 4.7684e-07, 7.1526e-07, 7.1526e-07,\n", + " -2.9802e-07, 2.3842e-07, 3.5763e-07, -2.9802e-07, 1.1921e-07,\n", + " -6.5565e-07, -2.9802e-07, -7.1526e-07, 1.1921e-07, 7.1526e-07,\n", + " -8.9407e-07, 4.7684e-07, -1.7881e-07, 1.1921e-07, -1.1921e-07,\n", + " -5.9605e-07, -4.1723e-07, -5.9605e-08, 4.7684e-07, -5.3644e-07,\n", + " -2.3842e-07, -2.3842e-07, 2.3842e-07, 1.1921e-07, -2.3842e-07,\n", + " -1.7881e-07, -4.1723e-07, -1.1921e-07, 5.9605e-07, 3.5763e-07,\n", + " -5.3644e-07, -5.9605e-08, -5.3644e-07, -5.9605e-08, -5.3644e-07,\n", + " 4.7684e-07, 0.0000e+00, 5.9605e-07, 4.7684e-07, 0.0000e+00,\n", + " 2.3842e-07, -9.5367e-07, -4.1723e-07, -5.9605e-08, 0.0000e+00,\n", + " 7.1526e-07, 1.1921e-07, -5.9605e-08, 2.3842e-07, -5.3644e-07,\n", + " 3.5763e-07, -1.0133e-06, 3.5763e-07, 1.0729e-06, 2.3842e-07,\n", + " -2.9802e-07, -2.3842e-07, -5.9605e-07, -4.1723e-07, -5.9605e-08,\n", + " 0.0000e+00, -2.3842e-07, -1.7881e-07, 2.3842e-07, 1.1921e-07,\n", + " 3.5763e-07, 4.7684e-07, -2.9802e-07, -5.3644e-07, 2.3842e-07,\n", + " -4.7684e-07, -3.5763e-07, -5.9605e-07, -1.1921e-07, -2.3842e-07,\n", + " 1.1921e-07, 1.3113e-06, -2.9802e-07, -5.9605e-08, 0.0000e+00,\n", + " -4.1723e-07, -1.7881e-07, 7.1526e-07, -4.1723e-07, 4.7684e-07,\n", + " -1.1921e-07, 3.5763e-07, 3.5763e-07, -5.3644e-07, -5.9605e-08,\n", + " 2.3842e-07, 2.3842e-07, -2.9802e-07, -2.3842e-07, -1.1921e-07,\n", + " -1.1921e-07, 9.5367e-07, 0.0000e+00, -7.1526e-07, 5.9605e-07,\n", + " -1.1921e-07, -1.7881e-07, 1.1921e-07, 3.5763e-07, -1.7881e-07,\n", + " -5.9605e-08, -1.0729e-06, 3.5763e-07, -1.1921e-07, -2.9802e-07,\n", + " -1.1921e-07, 0.0000e+00, 3.5763e-07, -1.1921e-07, 1.1921e-07,\n", + " -1.7881e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07, 4.7684e-07,\n", + " -3.5763e-07, -1.7881e-07, 2.3842e-07, 1.1921e-07, 3.5763e-07,\n", + " -4.7684e-07, 4.7684e-07, -1.1921e-07, 3.5763e-07, -6.5565e-07,\n", + " 4.7684e-07, -5.9605e-08, -2.3842e-07, -4.7684e-07, -3.5763e-07,\n", + " 1.1921e-07, 3.5763e-07, -5.9605e-08, -1.1921e-07, 3.5763e-07,\n", + " -3.5763e-07, 3.5763e-07, -7.7486e-07, 1.1921e-07, -7.1526e-07,\n", + " -5.9605e-08, 7.1526e-07, -1.1921e-07, 0.0000e+00, -5.9605e-08,\n", + " -2.3842e-07, -7.7486e-07, -5.9605e-08, -2.9802e-07, -5.9605e-07,\n", + " -5.9605e-08, -1.1921e-07, -4.1723e-07, -4.1723e-07, 3.5763e-07,\n", + " 1.1921e-07, -2.3842e-07, 1.1921e-07, 2.3842e-07, 1.1921e-07,\n", + " 5.9605e-07, -3.5763e-07, -1.7881e-07, -3.5763e-07, -5.9605e-07,\n", + " -1.1921e-07, -3.5763e-07, 4.7684e-07, 4.7684e-07, 4.7684e-07,\n", + " 3.5763e-07, -6.5565e-07, -2.3842e-07, 3.5763e-07, 3.5763e-07,\n", + " 4.7684e-07, -1.1921e-07, -4.1723e-07, -2.9802e-07, 3.5763e-07,\n", + " 1.1921e-07, 0.0000e+00, -5.9605e-08, 7.1526e-07, 2.3842e-07,\n", + " -5.9605e-07, 0.0000e+00, -2.9802e-07, 7.1526e-07, 4.7684e-07,\n", + " -5.3644e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07, -2.3842e-07,\n", + " 4.7684e-07, -6.5565e-07, -5.9605e-08, 1.1921e-07, 0.0000e+00,\n", + " -5.9605e-08, -2.3842e-07, 1.1921e-06, -3.5763e-07, 1.1921e-07,\n", + " -1.1921e-07, -1.1921e-07, -5.9605e-08, 4.7684e-07, -4.1723e-07,\n", + " 0.0000e+00, -5.9605e-08, 0.0000e+00, 0.0000e+00, -3.5763e-07,\n", + " 0.0000e+00, -5.9605e-07, 1.1921e-07, 2.3842e-07, 0.0000e+00,\n", + " -8.3447e-07, -7.7486e-07, -4.7684e-07, 8.3447e-07, 0.0000e+00,\n", + " -2.3842e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07,\n", + " 4.7684e-07, -7.1526e-07, 2.3842e-07, -4.7684e-07, -1.7881e-07,\n", + " -1.1921e-07, -1.0133e-06, 0.0000e+00, -5.9605e-08, -2.9802e-07,\n", + " 0.0000e+00, 1.1921e-07, -3.5763e-07, 0.0000e+00, -5.3644e-07,\n", + " 1.1921e-07, -2.9802e-07, 3.5763e-07, 1.1921e-07, -2.9802e-07,\n", + " 1.1921e-07, 0.0000e+00, -7.7486e-07, -1.7881e-07, 2.3842e-07,\n", + " 1.1921e-07, -3.5763e-07, -1.1921e-07, 1.1921e-07, -5.9605e-08,\n", + " -2.3842e-07, 0.0000e+00, -1.1921e-07, -1.7881e-07, 0.0000e+00,\n", + " -3.5763e-07, -2.3842e-07, -2.3842e-07, 2.3842e-07, 4.7684e-07,\n", + " -1.7881e-07, 0.0000e+00, 1.1921e-07, -2.3842e-07, -2.3842e-07,\n", + " 2.3842e-07, -1.1921e-07, 0.0000e+00, 4.7684e-07, -5.9605e-08,\n", + " 0.0000e+00, 1.1921e-07, -3.5763e-07, -5.9605e-08, 0.0000e+00,\n", + " 3.5763e-07, 0.0000e+00, 0.0000e+00, 7.1526e-07, 2.3842e-07,\n", + " 3.5763e-07, -5.3644e-07, -1.1921e-07, -2.3842e-07, -4.1723e-07,\n", + " -6.5565e-07, 1.1921e-07, 0.0000e+00, -2.9802e-07, -5.9605e-08,\n", + " 1.1921e-07, 0.0000e+00, -8.3447e-07, 0.0000e+00, -2.3842e-07,\n", + " 0.0000e+00, -2.3842e-07, -7.1526e-07, -5.9605e-07, -2.9802e-07,\n", + " -1.1921e-07, 3.5763e-07, -4.1723e-07, 4.7684e-07, -3.5763e-07,\n", + " -5.9605e-08, -2.3842e-07, 7.1526e-07, -3.5763e-07, 1.1921e-07,\n", + " 5.9605e-07, -4.7684e-07, -4.1723e-07, -2.9802e-07, -4.1723e-07,\n", + " 2.3842e-07, -2.9802e-07, 2.3842e-07, 3.5763e-07, 4.7684e-07,\n", + " 1.1921e-07, -2.9802e-07, 4.7684e-07, 0.0000e+00, -6.5565e-07,\n", + " 4.7684e-07, 2.3842e-07, 3.5763e-07, 0.0000e+00, 2.3842e-07,\n", + " 0.0000e+00, -1.1921e-07, 3.5763e-07, -2.9802e-07, -1.1921e-07,\n", + " -5.9605e-07, 0.0000e+00, 5.9605e-07, -5.3644e-07, -2.3842e-07,\n", + " -7.1526e-07, -5.9605e-08, -4.1723e-07, -2.9802e-07, -5.9605e-08,\n", + " -2.9802e-07, -1.1921e-07, -4.1723e-07, 2.3842e-07, 2.3842e-07,\n", + " 2.3842e-07, -5.9605e-08, 0.0000e+00, -5.9605e-08, -6.5565e-07,\n", + " -1.7881e-07, 2.3842e-07, -6.5565e-07, 1.1921e-07, 4.7684e-07,\n", + " -4.7684e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07, 0.0000e+00,\n", + " 0.0000e+00, 1.1921e-07, 4.7684e-07, 2.3842e-07, -4.1723e-07,\n", + " -4.1723e-07, -5.9605e-07, -4.7684e-07, 1.1921e-07, -2.3842e-07,\n", + " -4.1723e-07, -5.3644e-07, -7.7486e-07, -8.3447e-07, 0.0000e+00,\n", + " 3.5763e-07, -4.1723e-07, 2.3842e-07, -7.7486e-07, -1.7881e-07,\n", + " 2.3842e-07, 2.3842e-07, -2.9802e-07, 2.3842e-07, -5.3644e-07,\n", + " 2.3842e-07, 0.0000e+00, -7.1526e-07, 1.1921e-07, -5.9605e-08,\n", + " 0.0000e+00, 1.1921e-07, -5.9605e-08, 0.0000e+00, 2.3842e-07,\n", + " -4.7684e-07, -1.7881e-07, -2.9802e-07, -3.5763e-07, -5.9605e-08,\n", + " 2.3842e-07, 2.3842e-07, 0.0000e+00, 4.7684e-07, -1.7881e-07,\n", + " 4.7684e-07, 2.3842e-07, -2.9802e-07, -4.7684e-07, 3.5763e-07,\n", + " 0.0000e+00, -1.7881e-07, -6.5565e-07, -4.7684e-07, -4.7684e-07,\n", + " -2.3842e-07, 0.0000e+00, -2.3842e-07, -4.1723e-07, -5.9605e-08,\n", + " 1.1921e-07, -7.7486e-07, 0.0000e+00, -4.1723e-07, 1.1921e-07,\n", + " -1.1921e-07, -8.9407e-07, -2.3842e-07, -1.7881e-07, 8.3447e-07,\n", + " -7.7486e-07, 2.3842e-07, -5.9605e-08, -5.9605e-08, -2.3842e-07,\n", + " -2.3842e-07, 2.3842e-07, 1.1921e-07, 0.0000e+00, -5.9605e-08,\n", + " -5.9605e-08, -5.9605e-08], device='cuda:0', grad_fn=)),\n", + " ('layer4.0.shortcut.1.bias',\n", + " tensor([-2.5952e-07, -5.2718e-08, -3.8981e-07, -1.2332e-07, 2.4942e-07,\n", + " -4.1001e-07, -1.8458e-07, 2.2059e-07, 3.3934e-11, -5.6743e-08,\n", + " 3.8876e-07, -2.2358e-07, 1.5773e-07, -3.4872e-07, -2.7905e-07,\n", + " -2.3630e-08, -3.0215e-08, -2.4326e-07, -2.1305e-07, -2.9157e-07,\n", + " -4.5004e-07, -3.3447e-07, -1.6505e-07, -2.8462e-07, -1.1983e-07,\n", + " 3.0670e-07, -1.7117e-07, 2.2260e-07, -1.8211e-07, -3.3789e-07,\n", + " -6.4035e-07, -3.4069e-07, -2.1152e-07, 6.1553e-08, -1.1388e-07,\n", + " 3.9278e-08, 6.4294e-07, 5.3870e-07, -2.0622e-07, -5.0324e-07,\n", + " -8.4042e-08, 1.2858e-07, -3.1810e-07, 8.5574e-07, 2.0792e-07,\n", + " -3.2931e-07, 4.0081e-07, 1.2134e-07, 1.0071e-07, 9.3611e-07,\n", + " -3.1569e-07, -1.4190e-07, -9.8398e-08, -4.4183e-07, 3.8909e-07,\n", + " -2.3117e-07, -3.3583e-07, -4.5050e-07, 3.9275e-07, 5.3557e-07,\n", + " -8.5330e-07, -4.7482e-08, -7.7420e-08, 4.5063e-08, 5.6890e-08,\n", + " -6.9403e-07, -6.6087e-07, -3.2312e-07, -1.5117e-07, -5.0290e-07,\n", + " -8.3522e-08, -6.2873e-07, -1.9139e-07, -4.7217e-08, -1.4439e-07,\n", + " -3.0978e-07, -3.0010e-07, -4.4070e-07, 6.1973e-08, 4.8330e-07,\n", + " -1.5466e-07, -6.7563e-08, -6.4472e-07, -3.3934e-08, -3.2509e-07,\n", + " 3.2013e-09, -1.0098e-08, 6.1731e-08, -9.7081e-08, 6.2362e-08,\n", + " 2.2826e-07, -8.8843e-07, -7.5161e-07, -4.3784e-07, 5.1198e-08,\n", + " 7.3383e-07, -8.6796e-08, 1.3520e-07, 2.7753e-07, -7.6281e-08,\n", + " 3.2856e-07, -8.5474e-07, 1.4127e-07, 1.6500e-07, -1.1475e-07,\n", + " -9.0230e-08, -3.4021e-08, -4.5122e-07, 1.0504e-07, -3.4202e-08,\n", + " -6.6387e-09, -4.7148e-07, -5.1771e-07, -3.5938e-07, 1.1671e-07,\n", + " 7.6556e-08, 1.0592e-08, 2.3111e-07, -1.5009e-07, -2.9490e-07,\n", + " -2.6290e-07, -2.8363e-07, -4.4743e-07, -4.2879e-07, 1.9960e-07,\n", + " -2.2103e-07, 5.0751e-07, 2.3843e-08, -2.6756e-07, -1.8564e-07,\n", + " -2.7111e-07, -1.7106e-07, 8.1063e-07, -4.9894e-07, 1.3639e-07,\n", + " -4.6950e-07, 1.0927e-07, -1.2955e-07, -4.4637e-07, -8.5186e-08,\n", + " -6.1976e-08, -5.0219e-08, -6.6460e-07, -1.8259e-07, -2.3600e-07,\n", + " -1.2455e-07, 1.3833e-07, 2.3173e-07, 1.3480e-07, 2.6061e-07,\n", + " -2.2656e-07, -2.9893e-07, -3.5553e-07, 4.9369e-07, 2.8315e-07,\n", + " -1.6586e-07, -7.3051e-07, 3.2315e-07, -2.5779e-07, 1.5562e-07,\n", + " -3.1320e-07, -5.5600e-08, 6.4858e-08, -4.3620e-07, -3.0921e-08,\n", + " 1.7959e-07, 4.7084e-07, -2.9651e-07, 1.8505e-08, 4.9029e-07,\n", + " -2.0760e-07, -5.8565e-07, 1.7540e-07, -2.5290e-08, 5.8386e-08,\n", + " 2.4527e-07, 8.4265e-08, 3.2661e-08, -2.2252e-08, -2.5374e-07,\n", + " -7.2610e-08, 1.9237e-07, -3.9457e-07, -8.6376e-08, 9.7661e-08,\n", + " 8.1748e-08, -1.8011e-07, 5.4112e-08, -2.4380e-07, 4.2298e-08,\n", + " -2.2257e-07, 2.4752e-07, -8.2461e-07, -4.9300e-07, -2.8506e-07,\n", + " 1.4784e-07, 4.9734e-08, -3.3205e-07, -1.1681e-07, -4.5991e-07,\n", + " -1.0016e-07, -4.8842e-07, 3.3371e-07, -4.3462e-07, -7.9114e-07,\n", + " -6.8228e-08, -4.7129e-07, -3.3589e-07, 2.6814e-07, 1.9327e-07,\n", + " 1.4206e-07, 3.3498e-08, -5.5501e-08, 1.7940e-07, 4.2684e-07,\n", + " 1.7147e-07, 5.6309e-08, -7.1572e-09, -3.8246e-08, -2.8099e-07,\n", + " 9.5294e-08, -4.6339e-07, 5.0270e-07, 1.0481e-08, 1.9457e-07,\n", + " -2.0168e-08, 8.2068e-08, -1.6883e-09, 2.6224e-07, -3.9644e-07,\n", + " 4.4707e-07, -8.3741e-08, -1.7601e-07, -1.7361e-07, 7.8584e-07,\n", + " 7.0257e-11, -1.2827e-07, 3.2551e-07, 6.4153e-07, -3.4501e-08,\n", + " -7.0563e-07, 1.2502e-07, -5.3899e-07, 6.2128e-07, 1.6744e-07,\n", + " 3.0381e-08, 3.1756e-07, 1.4472e-07, 3.9837e-07, -6.7176e-08,\n", + " 7.3281e-07, -3.6448e-07, -2.5766e-08, 9.8277e-09, -5.1608e-07,\n", + " -2.1131e-07, -4.9822e-07, 6.0511e-07, -2.0183e-08, 2.1366e-07,\n", + " -1.2711e-07, 3.8626e-07, 3.8279e-07, 7.5298e-07, -6.0340e-07,\n", + " -4.7458e-08, -7.7033e-08, -2.6878e-07, 1.7127e-07, -4.7343e-07,\n", + " -5.9129e-09, -6.3804e-07, 3.3628e-07, -2.4381e-07, 3.1775e-07,\n", + " -6.1137e-07, -4.6261e-07, -4.1723e-07, 5.3470e-07, 7.1795e-09,\n", + " -2.3043e-08, -1.2530e-07, -5.4000e-07, -6.3562e-07, 1.3268e-07,\n", + " 2.2244e-07, -1.7262e-07, 4.3062e-07, -1.3121e-07, -2.1163e-07,\n", + " 6.3334e-08, -6.1981e-07, 6.5072e-08, -2.1843e-07, 1.8716e-08,\n", + " 7.4588e-08, 4.0129e-07, -8.5405e-08, 6.3721e-08, -4.6985e-07,\n", + " -2.2629e-07, -4.9808e-07, 6.7194e-07, 1.3299e-07, -4.3199e-09,\n", + " 2.7960e-07, -3.2597e-07, -2.8505e-08, -1.2399e-07, -2.7097e-07,\n", + " 8.7115e-08, -6.0820e-07, -1.7309e-07, 3.7368e-07, -2.4287e-07,\n", + " -1.1012e-07, 1.2617e-08, 1.7739e-07, -2.6112e-07, -3.0125e-07,\n", + " -1.2478e-07, 2.5956e-07, -1.0944e-07, 1.7735e-07, -5.8064e-08,\n", + " -2.0758e-07, -2.8337e-07, -2.6004e-07, 3.2003e-08, -3.9736e-07,\n", + " 2.9050e-07, -2.3287e-08, -2.2765e-07, 4.0771e-07, 4.0802e-07,\n", + " 7.0819e-08, 1.4434e-07, -1.8711e-07, -2.4609e-08, 2.6868e-07,\n", + " -1.8774e-08, -2.0145e-07, 1.3446e-07, 5.9894e-07, -5.2298e-08,\n", + " 2.3941e-07, 2.3950e-07, -3.1209e-07, 3.9168e-07, -3.1503e-07,\n", + " -5.3224e-07, 4.3975e-07, -9.7231e-09, -3.8906e-07, -3.3958e-07,\n", + " 1.7475e-07, -1.2887e-07, -7.8835e-07, 6.8751e-08, -4.2049e-07,\n", + " 1.2126e-07, 8.8542e-08, -5.7822e-07, -4.5029e-07, -1.0936e-07,\n", + " -2.1707e-07, 3.0449e-07, 2.9369e-07, 1.6307e-07, -2.1861e-07,\n", + " -4.1288e-07, -9.9163e-07, 3.3039e-07, -1.5236e-07, -1.9777e-08,\n", + " 3.7243e-07, -3.4649e-07, -3.2635e-07, -2.5982e-07, -2.4515e-07,\n", + " 7.5304e-08, -6.0096e-08, 2.8725e-07, 9.5974e-08, 1.0231e-07,\n", + " -6.2956e-09, 9.3015e-08, 5.1241e-07, -3.7457e-07, -5.8680e-07,\n", + " 5.1277e-08, 1.5735e-07, 1.5098e-07, -4.7827e-07, 1.5020e-07,\n", + " 3.2007e-07, -1.1792e-07, 4.1907e-07, -5.4599e-07, 3.5650e-07,\n", + " -7.2770e-07, 2.4135e-07, -5.4777e-08, -1.6205e-07, 1.0242e-07,\n", + " -8.1226e-07, -2.6392e-07, -3.5215e-07, -5.7353e-07, -2.3653e-07,\n", + " -7.6270e-07, -1.0341e-07, -1.5335e-07, -4.0521e-07, 5.1136e-08,\n", + " -2.1407e-07, -1.1061e-06, -1.1568e-07, -1.8562e-07, -2.2229e-07,\n", + " 1.7121e-07, 4.0764e-07, -5.0914e-07, -1.5899e-07, 1.9196e-07,\n", + " -4.7060e-07, 1.8851e-07, -5.7792e-08, -4.6329e-07, -3.9686e-07,\n", + " -1.8117e-08, -1.4392e-07, 4.1560e-07, 4.7124e-07, -2.5345e-08,\n", + " -2.1026e-07, -1.2185e-07, -2.1304e-07, 1.5910e-09, -3.9998e-07,\n", + " -1.5126e-07, -2.4613e-07, -1.0463e-07, 1.4167e-07, 2.5327e-08,\n", + " 3.7949e-07, -5.3259e-07, 7.2235e-07, -1.0368e-06, -2.0211e-07,\n", + " 4.1569e-07, 5.9008e-09, 2.3589e-07, -1.1660e-07, 3.8977e-08,\n", + " -2.6139e-07, 2.5065e-07, -3.0728e-07, -2.3161e-07, 5.0300e-08,\n", + " 4.0945e-08, -3.4996e-08, -2.7710e-07, -4.3644e-09, -1.2612e-07,\n", + " -3.5672e-07, -3.0295e-07, -3.4237e-07, -1.6870e-07, 3.2721e-07,\n", + " -2.5081e-07, 1.2698e-07, 2.5012e-07, 4.2570e-07, -3.8390e-07,\n", + " 1.6247e-07, 1.5678e-07, -1.8048e-07, 2.3226e-07, 2.6242e-07,\n", + " 1.4829e-07, -1.9754e-07, -2.4445e-07, -1.5111e-07, -1.8443e-07,\n", + " -5.3174e-07, -4.4128e-08, 2.1757e-07, -4.6750e-07, -7.1977e-08,\n", + " -1.7918e-07, -3.1625e-07, 2.4026e-07, 7.8625e-08, -4.3738e-08,\n", + " -2.1923e-09, -5.7170e-07, -4.3710e-07, -2.2763e-07, 5.7661e-07,\n", + " -3.9948e-07, 1.6238e-07, 2.4817e-07, 4.2508e-07, -5.4188e-07,\n", + " 8.4595e-08, -2.3122e-07, 2.6185e-07, -7.8099e-08, -1.5002e-07,\n", + " -4.9218e-08, 2.9320e-07], device='cuda:0', grad_fn=)),\n", + " ('linear.weight',\n", + " tensor([[ 0.0289, 0.0068, -0.0082, ..., 0.0602, -0.0364, 0.0396],\n", + " [ 0.0019, -0.0095, 0.0242, ..., 0.0193, -0.0087, 0.0316],\n", + " [-0.0057, -0.0301, 0.0333, ..., -0.0258, 0.0058, 0.0334],\n", + " ...,\n", + " [-0.0019, -0.0217, -0.0481, ..., -0.0218, -0.0286, -0.0146],\n", + " [ 0.0179, 0.0245, -0.0157, ..., 0.0186, 0.0383, -0.0235],\n", + " [-0.0072, -0.0170, -0.0242, ..., 0.0236, -0.0367, 0.0263]],\n", + " device='cuda:0', grad_fn=)),\n", + " ('linear.bias',\n", + " tensor([ 0.0288, -0.0477, 0.0047, 0.0126, -0.0182, 0.0702, -0.0292, -0.0152,\n", + " 0.0034, -0.0235], device='cuda:0', grad_fn=))])" + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "param_diff" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Starting training...\n", + "Batch: 0/1, Loss: 2.3967, Acc: 10.00%\n", + "Epoch: 1, Loss: 2.3967, Accuracy: 0.1000\n" + ] + } + ], + "source": [ + "import torch\n", + "from torch import nn\n", + "from collections import OrderedDict\n", + "from torch.utils.data import Dataset, DataLoader\n", + "from typing import Tuple, Any\n", + "\n", + "class CustomImageDataset(Dataset):\n", + " def __init__(self, images, labels):\n", + " self.images = images\n", + " self.labels = labels\n", + " \n", + " def __len__(self):\n", + " return len(self.labels)\n", + " \n", + " def __getitem__(self, idx):\n", + " image = self.images[idx]\n", + " label = self.labels[idx]\n", + " return image, label\n", + "\n", + "def train_one_epoch(\n", + " model: nn.Module,\n", + " optimizer: torch.optim.Optimizer,\n", + " dataloader: DataLoader,\n", + " criterion: nn.Module,\n", + " device: str\n", + ") -> Tuple[float, float]:\n", + " model.train()\n", + " total_loss = 0\n", + " correct = 0\n", + " total = 0\n", + "\n", + " for batch_idx, (data, target) in enumerate(dataloader):\n", + " # Move data to device\n", + " data, target = data.to(device), target.to(device)\n", + " \n", + " # Zero gradients\n", + " optimizer.zero_grad()\n", + " \n", + " # Forward pass\n", + " outputs = model(data)\n", + " \n", + " # Calculate loss\n", + " loss = criterion(outputs, target)\n", + " \n", + " # Backward pass\n", + " loss.backward()\n", + " \n", + " # Update weights\n", + " optimizer.step()\n", + " \n", + " # Track statistics\n", + " total_loss += loss.item()\n", + " _, predicted = outputs.max(1)\n", + " total += target.size(0)\n", + " correct += predicted.eq(target).sum().item()\n", + " \n", + " # Print batch statistics\n", + " if batch_idx % 10 == 0:\n", + " print(f'Batch: {batch_idx}/{len(dataloader)}, Loss: {loss.item():.4f}, '\n", + " f'Acc: {100.*correct/total:.2f}%')\n", + " \n", + " return total_loss / len(dataloader), correct / total\n", + "\n", + "# Main training setup\n", + "def main():\n", + " # Model setup\n", + " model_bespoke = mutils.get_model(\"resnet10\", \"cifar10\", \"cuda\")\n", + " model_bespoke.train()\n", + " \n", + " # Save initial parameters\n", + " params_initial = OrderedDict((name, param.clone().detach()) \n", + " for (name, param) in model_bespoke.named_parameters())\n", + " \n", + " # Optimizer setup\n", + " optimizer = torch.optim.SGD(model_bespoke.parameters(), lr=3e-4, weight_decay=0)\n", + " criterion = nn.CrossEntropyLoss()\n", + " \n", + " # Dataset setup (assuming ground_truth and labels are defined)\n", + " dataset = CustomImageDataset(ground_truth, labels)\n", + " dataloader = DataLoader(dataset, batch_size=10, shuffle=True, num_workers=0)\n", + " \n", + " # Training loop\n", + " print(\"Starting training...\")\n", + " for epoch in range(1):\n", + " loss, acc = train_one_epoch(model_bespoke, optimizer, dataloader, criterion, \"cuda\")\n", + " print(f'Epoch: {epoch+1}, Loss: {loss:.4f}, Accuracy: {acc:.4f}')\n", + " \n", + " # Save final parameters and calculate differences\n", + " params_final = OrderedDict((name, param.clone().detach()) \n", + " for (name, param) in model_bespoke.named_parameters())\n", + " \n", + " # Calculate and print parameter differences\n", + " param_diff = OrderedDict()\n", + " for (name, p_initial), (_, p_final) in zip(params_initial.items(), params_final.items()):\n", + " param_diff[name] = (p_final - p_initial)\n", + " \n", + " return model_bespoke, param_diff\n", + "\n", + "if __name__ == \"__main__\":\n", + " model, param_diff = main()" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "OrderedDict([('conv1.weight',\n", + " tensor([[[[ 9.9093e-07, -4.6194e-07, 4.0047e-06],\n", + " [-1.0729e-06, -8.2254e-06, -2.9691e-06],\n", + " [-9.6262e-06, -1.4752e-05, -4.8131e-06]],\n", + " \n", + " [[-4.6864e-06, -4.3809e-06, -1.7285e-06],\n", + " [-5.7071e-06, -1.0679e-05, -7.5102e-06],\n", + " [-1.4395e-05, -1.7047e-05, -8.5291e-06]],\n", + " \n", + " [[-6.2361e-06, -5.7817e-06, -3.9898e-06],\n", + " [-6.8545e-06, -1.1079e-05, -7.0771e-06],\n", + " [-1.3404e-05, -1.5400e-05, -6.4522e-06]]],\n", + " \n", + " \n", + " [[[ 1.8328e-06, 6.0797e-06, 2.1458e-06],\n", + " [-1.0148e-05, -6.2585e-06, -2.6934e-06],\n", + " [-1.3158e-05, -9.8050e-06, -9.6485e-06]],\n", + " \n", + " [[ 1.0267e-05, 1.5482e-05, 9.8795e-06],\n", + " [-2.4140e-06, -6.2585e-07, 1.1474e-06],\n", + " [-5.7593e-06, -3.9488e-06, -6.4485e-06]],\n", + " \n", + " [[ 1.5780e-05, 1.7703e-05, 1.1794e-05],\n", + " [ 4.9025e-06, 5.3421e-06, 5.1409e-06],\n", + " [-2.2352e-07, 1.6503e-06, -2.5034e-06]]],\n", + " \n", + " \n", + " [[[ 1.3784e-06, -2.9132e-06, 8.7917e-07],\n", + " [-2.5034e-06, -1.2890e-06, -1.8179e-06],\n", + " [-5.1185e-06, 2.5630e-06, -1.0058e-06]],\n", + " \n", + " [[ 1.0371e-05, 8.0764e-06, 1.0230e-05],\n", + " [ 7.8678e-06, 1.1332e-05, 8.4192e-06],\n", + " [ 5.2005e-06, 1.3366e-05, 8.1956e-06]],\n", + " \n", + " [[ 6.7428e-06, 5.0217e-06, 7.5144e-06],\n", + " [ 5.5060e-06, 7.9572e-06, 6.5565e-06],\n", + " [ 4.6194e-06, 1.2025e-05, 8.2552e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-1.1228e-05, -7.6741e-06, 6.9439e-06],\n", + " [-1.0118e-05, -1.7919e-06, 1.2919e-05],\n", + " [-1.6190e-05, -3.1982e-06, 8.8289e-06]],\n", + " \n", + " [[-1.6570e-05, -1.0796e-05, 4.9919e-06],\n", + " [-1.6170e-05, -6.9663e-06, 7.4562e-06],\n", + " [-2.0325e-05, -8.1211e-06, 5.6177e-06]],\n", + " \n", + " [[-1.2562e-05, -7.4804e-06, 8.5384e-06],\n", + " [-1.6130e-05, -6.4820e-06, 8.5905e-06],\n", + " [-2.1935e-05, -8.5533e-06, 4.5151e-06]]],\n", + " \n", + " \n", + " [[[ 6.1095e-06, 9.4026e-06, 6.8843e-06],\n", + " [ 6.2920e-06, 7.5996e-06, 5.8860e-06],\n", + " [ 6.4075e-06, 7.2122e-06, 4.0829e-06]],\n", + " \n", + " [[ 9.7807e-06, 1.2234e-05, 8.0429e-06],\n", + " [ 9.1195e-06, 8.1584e-06, 4.3991e-06],\n", + " [ 9.1344e-06, 8.0355e-06, 3.4571e-06]],\n", + " \n", + " [[ 4.0382e-06, 4.8876e-06, 1.0412e-06],\n", + " [ 2.1011e-06, -3.1292e-07, -4.2915e-06],\n", + " [ 3.5092e-06, -5.2154e-07, -4.8578e-06]]],\n", + " \n", + " \n", + " [[[ 6.3144e-06, 7.4543e-06, 1.1399e-05],\n", + " [ 4.6194e-07, -1.1176e-08, 5.0813e-06],\n", + " [ 2.1160e-06, 5.0664e-06, 6.1244e-06]],\n", + " \n", + " [[ 3.7402e-06, 4.3111e-06, 7.0184e-06],\n", + " [-1.3188e-06, -1.5944e-06, 3.5614e-06],\n", + " [-1.1176e-06, 3.1441e-06, 3.6871e-06]],\n", + " \n", + " [[ 5.3048e-06, 7.3910e-06, 1.0658e-05],\n", + " [ 1.6242e-06, 2.0862e-06, 7.3463e-06],\n", + " [ 1.2368e-06, 5.9903e-06, 7.5251e-06]]]], device='cuda:0')),\n", + " ('bn1.weight',\n", + " tensor([ 3.0994e-06, 3.5763e-07, -7.3314e-06, 1.1921e-06, -4.3511e-06,\n", + " 4.7684e-07, 9.5367e-07, 1.7881e-06, -4.0531e-06, 2.8610e-06,\n", + " -4.5300e-06, 1.0729e-06, -2.7418e-06, -5.0068e-06, 3.3379e-06,\n", + " 4.8876e-06, -5.8413e-06, -4.4703e-06, 4.0531e-06, 3.9339e-06,\n", + " -3.8147e-06, 4.6492e-06, -6.6757e-06, 4.2915e-06, 3.5763e-06,\n", + " -1.1921e-06, 9.5367e-07, 1.5497e-06, 9.5367e-07, -1.1921e-06,\n", + " 3.4571e-06, 4.7684e-07, 3.0994e-06, 7.1526e-07, 3.6955e-06,\n", + " 3.5763e-06, 1.0729e-06, 8.3447e-07, -3.5763e-07, 2.7418e-06,\n", + " 6.6757e-06, -1.8477e-06, 8.2254e-06, -4.7684e-07, 7.2718e-06,\n", + " 2.2650e-06, 5.1260e-06, -7.3314e-06, -1.7285e-06, 1.7881e-06,\n", + " 3.3379e-06, 1.4305e-06, -4.0531e-06, -1.9073e-06, 3.6955e-06,\n", + " -8.2850e-06, 4.1723e-06, 6.3181e-06, -3.8147e-06, 9.5367e-07,\n", + " 3.6955e-06, -3.9339e-06, -1.0610e-05, 4.1723e-06], device='cuda:0')),\n", + " ('bn1.bias',\n", + " tensor([ 4.2480e-06, 5.9006e-06, -4.8113e-06, 2.8228e-06, 8.5083e-07,\n", + " 3.2780e-06, 5.0684e-06, 4.3948e-06, -3.4867e-06, 1.5374e-06,\n", + " -1.8370e-06, 1.9890e-06, -2.3541e-06, -5.6038e-06, 5.9714e-06,\n", + " 3.8110e-06, -5.7142e-06, 5.9350e-07, 6.3278e-06, 1.9870e-06,\n", + " -6.3236e-06, 6.7887e-06, -3.1113e-06, -3.9099e-07, -5.9868e-08,\n", + " -5.4377e-06, 1.9105e-06, 4.1676e-06, 6.8520e-07, 4.0158e-06,\n", + " 1.2476e-06, 1.1877e-08, -1.6836e-06, -3.9817e-06, 5.0061e-07,\n", + " -1.8907e-06, 4.0654e-06, -2.6084e-07, 8.4628e-07, 7.6336e-07,\n", + " 8.8257e-06, -4.3922e-06, 3.6792e-06, -6.5806e-06, 1.0206e-05,\n", + " 3.6491e-06, 6.8027e-06, -4.4466e-06, -8.1502e-07, -1.8807e-06,\n", + " 7.4160e-06, 2.7364e-06, -4.5183e-06, -1.0573e-06, 5.4573e-06,\n", + " 2.6168e-07, -2.8168e-06, 3.9376e-06, 1.5581e-06, 8.1046e-08,\n", + " -3.9231e-07, 1.5138e-06, -1.0100e-05, 1.9233e-06], device='cuda:0')),\n", + " ('layer1.0.conv1.weight',\n", + " tensor([[[[ 5.3197e-06, 2.5735e-06, 4.9677e-06],\n", + " [ 2.2687e-06, 1.3215e-06, 6.7260e-06],\n", + " [-7.5065e-07, 1.2163e-06, 7.1549e-06]],\n", + " \n", + " [[ 3.6657e-06, -1.5572e-06, 2.5872e-06],\n", + " [ 8.9761e-06, 3.2061e-06, 1.9390e-06],\n", + " [ 9.6913e-06, 3.4496e-06, 3.3490e-06]],\n", + " \n", + " [[ 4.6026e-06, 4.1500e-06, 6.6273e-06],\n", + " [ 4.5188e-06, 5.9195e-06, 8.9146e-06],\n", + " [ 2.5216e-06, 6.0610e-06, 7.4264e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 7.8343e-06, 3.2730e-06, 2.7278e-06],\n", + " [ 3.6531e-06, 5.8208e-06, 5.4631e-06],\n", + " [ 4.6603e-06, 3.2969e-06, 5.4352e-06]],\n", + " \n", + " [[ 8.9929e-06, 9.3840e-06, 4.5430e-06],\n", + " [ 7.3332e-06, 5.3700e-06, 1.1288e-06],\n", + " [ 4.8243e-06, 2.8703e-06, 6.5018e-07]],\n", + " \n", + " [[ 5.6121e-06, 6.1393e-06, 6.9626e-06],\n", + " [ 4.0215e-06, 5.4743e-06, 6.6962e-06],\n", + " [ 1.0822e-06, 2.9709e-06, 3.7886e-06]]],\n", + " \n", + " \n", + " [[[-9.5088e-07, 1.7602e-06, 3.9407e-06],\n", + " [ 2.0731e-06, 1.4435e-06, 5.0133e-06],\n", + " [ 4.6305e-06, 2.6263e-06, 4.7162e-06]],\n", + " \n", + " [[-3.6489e-06, 2.9802e-08, 4.2648e-06],\n", + " [ 1.4901e-07, -1.1828e-07, 3.9935e-06],\n", + " [ 2.0228e-06, 8.7958e-07, 3.7327e-06]],\n", + " \n", + " [[-1.0431e-07, 2.4671e-06, 3.8389e-06],\n", + " [ 1.3597e-06, 1.6810e-06, 2.6971e-06],\n", + " [ 2.0014e-06, 9.4436e-07, 2.0210e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.3267e-06, 1.5609e-06, 1.4380e-06],\n", + " [ 2.3637e-06, -9.4622e-07, -3.7998e-07],\n", + " [ 2.9150e-07, -1.0962e-06, -1.0170e-06]],\n", + " \n", + " [[ 3.9730e-06, 3.7905e-06, 1.3942e-06],\n", + " [ 2.7791e-06, 2.3413e-06, 4.2561e-07],\n", + " [ 2.6338e-06, 2.1085e-06, 2.1569e-06]],\n", + " \n", + " [[-1.8850e-06, 9.8720e-08, 2.0146e-07],\n", + " [-9.3877e-07, -8.7544e-07, -6.8918e-08],\n", + " [-1.7975e-07, -4.6659e-07, -5.9046e-07]]],\n", + " \n", + " \n", + " [[[ 1.2033e-06, -1.3299e-06, -2.8559e-06],\n", + " [-5.4762e-06, -7.9861e-06, -3.8221e-06],\n", + " [-2.9244e-06, -5.6196e-06, -2.2016e-06]],\n", + " \n", + " [[ 2.4727e-07, 7.0781e-07, 3.1963e-06],\n", + " [-2.4950e-06, 1.9707e-06, 4.5709e-06],\n", + " [-5.0031e-06, 2.9728e-06, 9.2518e-06]],\n", + " \n", + " [[-1.8207e-06, -4.3288e-06, 4.6268e-06],\n", + " [-6.0368e-06, -2.5928e-06, 3.3807e-06],\n", + " [ 2.1979e-07, 2.8408e-06, 4.3400e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.1656e-06, -3.1851e-07, -2.1411e-06],\n", + " [ 2.7530e-06, -4.5057e-06, -3.3136e-06],\n", + " [ 4.9341e-06, -3.8240e-06, -6.9663e-07]],\n", + " \n", + " [[ 2.8424e-06, 7.0184e-06, 2.2314e-06],\n", + " [ 4.1723e-07, 1.1735e-07, -5.0850e-06],\n", + " [-5.3681e-06, -3.4831e-06, -7.8762e-06]],\n", + " \n", + " [[-8.8662e-07, 9.2108e-07, 2.4959e-07],\n", + " [-1.1809e-06, 1.1176e-08, -1.7062e-06],\n", + " [ 7.2177e-07, -9.7975e-07, -2.2464e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 1.2767e-05, 1.0841e-05, 3.4422e-06],\n", + " [ 1.4702e-05, 1.1206e-05, 8.2599e-06],\n", + " [ 1.3908e-05, 1.2608e-05, 1.0829e-05]],\n", + " \n", + " [[ 2.8610e-06, -4.5970e-06, -1.7211e-06],\n", + " [ 5.1465e-06, 5.0291e-08, -2.1150e-06],\n", + " [ 7.2159e-06, 1.5320e-06, -8.7242e-07]],\n", + " \n", + " [[ 3.5076e-06, 3.9490e-06, 2.9262e-06],\n", + " [ 6.1616e-06, 4.7190e-06, 5.3495e-06],\n", + " [ 4.4499e-06, 5.6326e-06, 3.7383e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.7400e-06, 6.7540e-06, 3.9362e-06],\n", + " [ 1.3877e-06, 3.7868e-06, 3.9116e-06],\n", + " [ 2.0789e-06, 5.8301e-07, -3.5833e-07]],\n", + " \n", + " [[ 7.3574e-06, 5.7472e-06, 7.1190e-06],\n", + " [ 1.1409e-05, 7.6070e-06, 6.3684e-06],\n", + " [ 9.0655e-06, 6.0052e-06, 6.5975e-06]],\n", + " \n", + " [[ 5.5323e-06, 2.6356e-06, 1.1306e-06],\n", + " [ 6.8489e-06, 4.5933e-06, 4.3800e-06],\n", + " [ 7.4692e-06, 6.4746e-06, 5.5432e-06]]],\n", + " \n", + " \n", + " [[[-7.0045e-06, -7.1097e-06, -1.3830e-05],\n", + " [-1.5358e-05, -1.5994e-05, -2.4969e-05],\n", + " [-1.2049e-05, -1.9913e-05, -2.9732e-05]],\n", + " \n", + " [[ 1.3510e-05, 9.2760e-06, 4.4014e-06],\n", + " [ 1.3236e-05, 2.1923e-06, -2.6240e-06],\n", + " [ 1.4646e-05, 3.0799e-06, -8.1831e-06]],\n", + " \n", + " [[ 3.0398e-06, -8.8508e-06, -1.7661e-05],\n", + " [-5.3140e-06, -1.6805e-05, -2.5857e-05],\n", + " [-4.6920e-06, -2.2337e-05, -2.4119e-05]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.4177e-06, -5.0031e-06, -7.2159e-06],\n", + " [-3.1861e-06, -3.9972e-06, -9.8143e-06],\n", + " [-5.7630e-06, -7.0781e-08, -3.1888e-06]],\n", + " \n", + " [[-1.6071e-05, -2.0384e-05, -1.2822e-05],\n", + " [-2.5393e-05, -2.8031e-05, -2.0761e-05],\n", + " [-2.4607e-05, -2.7642e-05, -2.1874e-05]],\n", + " \n", + " [[-4.1854e-06, -8.3670e-06, -1.2558e-05],\n", + " [-9.9316e-06, -1.8132e-05, -2.3715e-05],\n", + " [-1.3510e-05, -2.1283e-05, -2.6420e-05]]],\n", + " \n", + " \n", + " [[[ 6.6161e-06, 1.1180e-05, 1.1187e-05],\n", + " [ 5.1903e-06, 1.2271e-05, 8.8448e-06],\n", + " [ 6.6292e-06, 8.8671e-06, 1.0509e-05]],\n", + " \n", + " [[-1.5578e-05, -7.0557e-06, 8.2366e-06],\n", + " [-9.8622e-06, 1.9372e-06, 8.7224e-06],\n", + " [-1.0390e-05, 7.1414e-06, 6.0899e-06]],\n", + " \n", + " [[ 4.7917e-06, 1.1917e-05, 1.2315e-05],\n", + " [ 7.0594e-06, 1.1899e-05, 9.0422e-06],\n", + " [ 7.0352e-06, 9.6485e-06, 7.6331e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.2502e-05, 7.2082e-06, 3.5334e-06],\n", + " [ 1.0693e-05, 5.3262e-06, 6.6422e-06],\n", + " [ 4.2715e-06, 3.0487e-06, 4.9546e-06]],\n", + " \n", + " [[ 1.1127e-05, 7.5912e-06, 2.5816e-06],\n", + " [ 1.0917e-05, 8.5382e-06, 3.7998e-06],\n", + " [ 1.1211e-05, 7.8902e-06, 3.1926e-06]],\n", + " \n", + " [[ 5.7612e-06, 8.8997e-06, 8.2571e-06],\n", + " [ 5.9661e-06, 8.7135e-06, 7.1120e-06],\n", + " [ 6.4522e-06, 7.1619e-06, 5.1055e-06]]]], device='cuda:0')),\n", + " ('layer1.0.bn1.weight',\n", + " tensor([ 5.1260e-06, -5.3644e-07, -5.3048e-06, -4.0531e-06, 2.7418e-06,\n", + " -1.9670e-06, 1.1921e-06, -2.9802e-07, -2.9802e-06, 1.0729e-06,\n", + " 2.3842e-07, -1.3113e-06, 8.3447e-07, 4.8876e-06, -5.1260e-06,\n", + " -8.9407e-07, -1.9670e-06, 2.9802e-06, -5.3644e-07, 2.1458e-06,\n", + " 3.3379e-06, 5.1260e-06, -1.9670e-06, 4.4107e-06, 1.7881e-06,\n", + " -2.7418e-06, -2.5630e-06, 2.5034e-06, 1.6689e-06, -1.6689e-06,\n", + " -1.0133e-06, -4.8876e-06, 0.0000e+00, 2.3842e-07, 0.0000e+00,\n", + " -3.3975e-06, -5.3048e-06, 5.9605e-07, -3.0994e-06, 4.0531e-06,\n", + " -1.8477e-06, 1.4305e-06, -3.3379e-06, 1.9073e-06, 1.1921e-06,\n", + " -1.3113e-06, -2.3246e-06, -3.0398e-06, -1.1921e-07, 5.7220e-06,\n", + " 2.3842e-07, 3.5763e-06, -1.4305e-06, -3.6359e-06, 3.4571e-06,\n", + " -2.1458e-06, -1.2517e-06, 4.7684e-07, -3.2187e-06, -1.3113e-06,\n", + " 4.0531e-06, 5.2452e-06, 3.0994e-06, 1.1921e-06], device='cuda:0')),\n", + " ('layer1.0.bn1.bias',\n", + " tensor([ 1.5239e-06, 4.9805e-07, -3.0667e-06, -1.6546e-06, 3.0882e-08,\n", + " 2.4659e-06, 1.3029e-06, 4.9287e-07, -2.1967e-06, -3.9831e-07,\n", + " -2.6451e-06, -5.8212e-06, -4.2027e-07, 3.1355e-06, -1.7615e-06,\n", + " 1.4869e-06, -3.5126e-06, 8.0605e-07, -1.4975e-06, 8.3291e-07,\n", + " -1.3617e-06, 1.6682e-06, -5.2246e-07, -4.0934e-07, 2.7161e-06,\n", + " -1.3181e-06, 8.2625e-07, 3.2450e-08, -3.6535e-06, -9.6101e-07,\n", + " -1.2502e-06, -2.3344e-06, 1.6987e-06, 2.2037e-06, -3.1528e-06,\n", + " -9.9399e-07, -4.4102e-06, -5.3685e-07, 9.5601e-07, 6.0543e-06,\n", + " -3.2821e-06, 6.2275e-07, 2.8308e-06, 2.4347e-07, -1.2094e-06,\n", + " -7.0944e-07, -1.9430e-06, -1.6659e-06, -2.4504e-06, 2.8594e-06,\n", + " 1.8393e-06, 4.1009e-06, -2.6293e-06, -2.9461e-06, 2.8180e-06,\n", + " -5.8443e-06, -7.1812e-07, -4.7377e-07, -3.6096e-06, 1.3539e-06,\n", + " 4.7317e-06, 1.0842e-06, 3.4641e-06, -1.2135e-06], device='cuda:0')),\n", + " ('layer1.0.conv2.weight',\n", + " tensor([[[[ 1.3895e-06, -3.3528e-08, -4.2841e-07],\n", + " [ 1.9986e-06, -9.4064e-08, 3.5092e-06],\n", + " [ 3.0175e-07, -2.3097e-07, 1.2908e-06]],\n", + " \n", + " [[ 2.6971e-06, 4.4099e-06, 2.1607e-07],\n", + " [ 1.7956e-06, 5.1707e-06, 6.4960e-07],\n", + " [ 1.4529e-07, 3.0976e-06, 2.1374e-06]],\n", + " \n", + " [[-6.7800e-07, 7.6741e-07, 1.6876e-06],\n", + " [-4.0978e-07, 9.8348e-07, 1.5469e-06],\n", + " [ 3.4273e-07, 5.9232e-07, 2.0955e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.4887e-06, 3.8017e-06, 3.4627e-06],\n", + " [ 1.4147e-06, 4.8801e-06, -1.4827e-06],\n", + " [-2.4028e-06, 5.4948e-08, -4.2375e-07]],\n", + " \n", + " [[ 6.7800e-07, -1.1288e-06, -1.1846e-06],\n", + " [ 3.8184e-07, -2.4587e-07, 1.1362e-07],\n", + " [ 1.7248e-06, -6.7055e-07, -3.7327e-06]],\n", + " \n", + " [[ 1.7509e-07, -3.7486e-07, -3.2037e-07],\n", + " [ 3.6955e-06, 6.4634e-07, 1.1455e-07],\n", + " [ 2.5462e-06, -7.6089e-07, -4.3586e-07]]],\n", + " \n", + " \n", + " [[[ 6.3553e-06, 5.3002e-06, 5.2452e-06],\n", + " [ 7.3832e-06, 6.2138e-06, 1.1921e-07],\n", + " [ 5.2825e-06, 4.7777e-07, -7.9004e-06]],\n", + " \n", + " [[ 1.6578e-06, 2.6762e-06, 5.0580e-06],\n", + " [-1.2908e-06, -3.6135e-07, 6.6636e-06],\n", + " [-1.0408e-06, -5.8289e-07, 4.3102e-06]],\n", + " \n", + " [[-8.1286e-06, -1.1109e-05, -1.3784e-05],\n", + " [-6.4163e-06, -7.4115e-06, -1.2979e-05],\n", + " [-9.3095e-06, -7.6890e-06, -6.4038e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 5.2415e-06, -1.0207e-06, 3.7961e-06],\n", + " [ 3.5949e-06, -3.6508e-07, 6.2585e-06],\n", + " [ 7.6168e-06, -2.5192e-06, -9.6299e-07]],\n", + " \n", + " [[ 4.4964e-06, -4.7013e-06, -9.3598e-07],\n", + " [ 4.0308e-06, -6.9039e-06, -9.1176e-06],\n", + " [-4.5365e-06, -3.0026e-06, -1.3849e-06]],\n", + " \n", + " [[ 7.5623e-07, -1.5087e-06, 1.3788e-06],\n", + " [-2.3041e-06, 1.5795e-06, 4.9621e-06],\n", + " [-1.7043e-06, 4.3698e-06, 3.5278e-06]]],\n", + " \n", + " \n", + " [[[-1.1008e-06, 1.9707e-06, -6.1467e-07],\n", + " [-2.0806e-06, -3.7812e-07, 1.9372e-06],\n", + " [-5.2229e-06, -5.2806e-07, 1.8217e-06]],\n", + " \n", + " [[-7.3283e-07, -3.6582e-06, 6.3330e-08],\n", + " [-2.1700e-07, -2.0089e-06, -2.1532e-06],\n", + " [-2.6785e-06, -3.9823e-06, -2.2296e-06]],\n", + " \n", + " [[ 1.0082e-05, 9.0534e-06, 9.5973e-06],\n", + " [ 1.2001e-05, 1.0580e-05, 9.1903e-06],\n", + " [ 1.0397e-05, 9.9298e-06, 7.8939e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-8.7917e-07, -3.6135e-07, -1.3039e-08],\n", + " [-4.3614e-06, 1.5292e-06, -3.7029e-06],\n", + " [-5.6848e-06, -2.7260e-06, 4.5169e-08]],\n", + " \n", + " [[ 3.4794e-06, 4.3632e-06, 4.2007e-06],\n", + " [ 1.5423e-06, 7.4264e-06, 1.1304e-05],\n", + " [ 1.2834e-06, 5.1297e-06, 3.8533e-06]],\n", + " \n", + " [[-1.0051e-05, -1.1733e-05, -4.7721e-06],\n", + " [-1.1101e-05, -1.0878e-05, -3.0957e-06],\n", + " [-1.3369e-05, -1.7643e-05, -6.2836e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-1.1660e-06, -2.5872e-06, -1.9646e-06],\n", + " [ 5.0291e-06, 1.0729e-06, 2.1732e-06],\n", + " [ 1.0667e-05, 8.1407e-06, 5.6550e-06]],\n", + " \n", + " [[-5.3532e-06, -6.5267e-06, -3.6673e-06],\n", + " [-3.4217e-06, -4.2040e-06, -5.0038e-06],\n", + " [-1.1306e-06, 3.8464e-07, -2.1746e-06]],\n", + " \n", + " [[ 4.5984e-06, 3.9041e-06, 4.9509e-06],\n", + " [ 9.3877e-07, 6.5751e-07, 1.6717e-06],\n", + " [-2.3693e-06, -2.2501e-06, -2.1830e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.7567e-06, 7.0781e-08, -5.0766e-06],\n", + " [ 1.0319e-06, 2.7232e-06, -2.2780e-06],\n", + " [ 4.1490e-06, 3.7365e-06, -5.4389e-07]],\n", + " \n", + " [[ 5.7966e-06, 5.7369e-07, 2.8405e-06],\n", + " [ 8.0094e-07, -3.5502e-06, -2.0489e-06],\n", + " [-1.8403e-06, -4.5821e-06, -4.9067e-06]],\n", + " \n", + " [[-2.1290e-06, -4.4480e-06, -1.9595e-06],\n", + " [-7.9069e-07, -5.0232e-06, -6.1356e-06],\n", + " [ 1.4706e-06, 3.2596e-06, 4.5076e-07]]],\n", + " \n", + " \n", + " [[[-5.4576e-07, 3.9525e-06, 3.7681e-06],\n", + " [-3.3192e-06, -1.7975e-07, 1.1645e-06],\n", + " [-5.7463e-06, -1.2405e-06, 7.0781e-07]],\n", + " \n", + " [[ 1.5423e-06, 4.5206e-06, 3.3192e-06],\n", + " [ 2.4252e-06, 4.7460e-06, 2.8666e-06],\n", + " [ 2.0682e-06, 2.6338e-06, 3.3146e-06]],\n", + " \n", + " [[ 9.0497e-06, 4.4964e-06, 5.3942e-06],\n", + " [ 6.2836e-06, 4.9816e-06, 3.9216e-06],\n", + " [ 8.8476e-07, 7.0100e-08, -1.4864e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.7877e-06, 2.3507e-06, 9.4641e-06],\n", + " [-2.4065e-06, -1.6689e-06, 1.6689e-06],\n", + " [ 4.8913e-06, 6.5602e-06, 9.1582e-06]],\n", + " \n", + " [[ 8.3260e-07, 5.4054e-06, 9.8087e-06],\n", + " [ 2.7306e-06, 5.0971e-06, 5.7071e-06],\n", + " [-1.0947e-05, -1.7285e-06, 6.8583e-06]],\n", + " \n", + " [[-2.0545e-06, 5.8860e-07, -5.7369e-07],\n", + " [-7.1805e-07, -6.4075e-07, -2.0452e-06],\n", + " [ 5.8413e-06, 3.0590e-06, 5.4822e-06]]],\n", + " \n", + " \n", + " [[[ 1.8328e-06, 3.6284e-06, 3.0231e-06],\n", + " [-4.5858e-06, -2.7753e-06, -1.2238e-06],\n", + " [-3.0724e-06, -6.3702e-07, 1.8217e-06]],\n", + " \n", + " [[-1.2824e-06, 4.4703e-08, 5.2080e-06],\n", + " [ 0.0000e+00, -7.9535e-07, 3.9994e-06],\n", + " [ 1.2526e-06, 1.8626e-06, 6.1337e-06]],\n", + " \n", + " [[-4.1947e-06, -7.7859e-06, -1.0822e-05],\n", + " [-3.0547e-06, -4.3726e-06, -8.1249e-06],\n", + " [-3.7402e-06, -4.1574e-06, -8.7796e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.6508e-06, -2.3860e-06, 2.8871e-08],\n", + " [-7.7672e-07, 6.6496e-07, 8.0466e-07],\n", + " [-3.4682e-06, -3.6177e-06, -2.9251e-06]],\n", + " \n", + " [[-6.0499e-06, -5.2142e-06, -6.7689e-06],\n", + " [-1.3324e-05, -7.8473e-06, -1.1117e-05],\n", + " [-1.5919e-05, -1.0084e-05, -1.3931e-05]],\n", + " \n", + " [[ 6.3349e-06, 4.5877e-06, 4.8876e-06],\n", + " [ 6.6347e-06, 8.4620e-06, 9.3040e-06],\n", + " [ 6.8210e-06, 5.1488e-06, 7.3761e-06]]]], device='cuda:0')),\n", + " ('layer1.0.bn2.weight',\n", + " tensor([-3.9339e-06, 5.7220e-06, -1.0729e-06, -1.6689e-06, -3.0994e-06,\n", + " 7.1526e-07, -1.3113e-06, 2.0266e-06, -3.5763e-06, -2.7418e-06,\n", + " 4.0531e-06, -3.2783e-06, 2.1458e-06, -1.3709e-06, -8.9407e-07,\n", + " -1.1325e-06, -4.0531e-06, 4.2915e-06, 4.7684e-06, -1.5497e-06,\n", + " -7.1526e-07, -1.3113e-06, -1.8477e-06, -4.2319e-06, -1.7285e-06,\n", + " 2.1458e-06, -6.5565e-07, 2.0266e-06, 3.0994e-06, 4.1723e-06,\n", + " -1.0729e-06, 1.9073e-06, -2.6822e-06, 3.0994e-06, 2.8610e-06,\n", + " -2.0266e-06, -3.0994e-06, -2.9802e-07, -2.0266e-06, -1.1921e-07,\n", + " -1.9670e-06, -5.3048e-06, -8.9407e-07, -2.5034e-06, -1.6689e-06,\n", + " 1.0729e-06, -3.3379e-06, 2.5034e-06, -2.9802e-06, -5.9009e-06,\n", + " 1.6689e-06, 2.9802e-06, -2.9802e-07, 1.1921e-06, -3.3379e-06,\n", + " -2.3842e-07, -9.5367e-07, 1.7881e-06, -2.3246e-06, -1.6093e-06,\n", + " -1.1921e-07, 3.8147e-06, -2.3246e-06, 2.0266e-06], device='cuda:0')),\n", + " ('layer1.0.bn2.bias',\n", + " tensor([ 1.5868e-07, 9.3977e-07, -1.1009e-06, -3.2320e-06, 1.3682e-06,\n", + " -4.9855e-07, 2.4686e-06, 2.8355e-06, 5.1637e-07, -2.9283e-07,\n", + " -9.8881e-07, 1.4233e-06, 1.7544e-06, -1.6418e-06, 2.7256e-06,\n", + " 1.7330e-07, -2.3644e-06, 3.8721e-07, 2.2418e-06, -5.5773e-07,\n", + " -1.0968e-06, 2.1732e-06, -1.0681e-06, 4.4223e-06, 8.2948e-08,\n", + " -1.1339e-06, 2.2975e-06, 1.7483e-06, 1.5275e-06, 2.7966e-06,\n", + " -1.4331e-06, 5.2383e-06, 1.9282e-07, -8.2710e-08, -1.4253e-06,\n", + " -5.4586e-08, -2.7365e-06, -9.0235e-07, 1.1680e-06, 2.5383e-06,\n", + " 7.6398e-07, -1.3603e-06, 1.1668e-06, -9.5251e-07, 4.4751e-06,\n", + " 1.6577e-06, 3.6312e-07, 1.1681e-06, -4.1789e-06, 7.3535e-08,\n", + " 3.8469e-06, 3.4083e-06, -1.3044e-06, 9.2400e-07, -1.1203e-06,\n", + " -3.5916e-06, -1.6950e-06, -9.8079e-07, -1.2194e-06, -6.2152e-07,\n", + " 6.9892e-07, 2.2628e-06, -2.1050e-06, 6.5259e-07], device='cuda:0')),\n", + " ('layer2.0.conv1.weight',\n", + " tensor([[[[ 1.5902e-06, 1.3839e-06, 5.2154e-08],\n", + " [ 1.7993e-06, -5.0105e-07, -1.0366e-06],\n", + " [-1.5534e-06, -2.4866e-06, -3.2764e-06]],\n", + " \n", + " [[-7.5586e-06, -3.6433e-06, -2.8238e-06],\n", + " [-4.4589e-06, -4.8950e-06, -9.1996e-06],\n", + " [ 5.7463e-07, -3.7588e-06, -6.7241e-06]],\n", + " \n", + " [[-1.9744e-07, 6.6124e-08, -4.3390e-06],\n", + " [ 1.7229e-07, -4.2313e-06, -3.4543e-06],\n", + " [ 2.0191e-06, 8.6403e-07, -1.3607e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 6.5677e-06, 2.8005e-06, 2.8703e-06],\n", + " [ 2.7921e-06, 2.2464e-06, 4.3809e-06],\n", + " [ 1.1988e-06, 1.4380e-06, 2.6226e-06]],\n", + " \n", + " [[-4.7684e-06, -3.1590e-06, -2.2165e-06],\n", + " [-3.4925e-06, -1.7174e-06, -1.8254e-07],\n", + " [-5.8524e-06, -6.5900e-06, -3.0585e-06]],\n", + " \n", + " [[-8.0280e-06, -5.0226e-06, -3.7458e-06],\n", + " [-2.5481e-06, 1.8463e-07, 1.3430e-06],\n", + " [-4.3772e-07, 8.0094e-08, 2.3153e-06]]],\n", + " \n", + " \n", + " [[[-7.4436e-07, -4.0173e-06, -5.8236e-06],\n", + " [-6.0350e-07, -2.9355e-06, -4.2031e-06],\n", + " [ 1.5460e-06, -2.3264e-06, -1.8524e-06]],\n", + " \n", + " [[-9.7975e-07, -2.8834e-06, -2.8163e-06],\n", + " [-7.4878e-07, -1.0943e-06, -6.7474e-07],\n", + " [-4.4862e-06, -3.2000e-06, 8.1118e-07]],\n", + " \n", + " [[-7.4226e-07, 5.9046e-07, 5.6997e-07],\n", + " [-3.7439e-07, -6.3679e-07, 1.1697e-06],\n", + " [-1.7844e-06, -1.1744e-06, -2.1379e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.9339e-06, -2.0631e-06, -1.2554e-06],\n", + " [ 7.0781e-08, -2.6843e-06, -2.2678e-06],\n", + " [ 3.7067e-06, -1.6356e-06, -1.4366e-06]],\n", + " \n", + " [[-1.1008e-06, 2.7190e-06, 5.2266e-06],\n", + " [ 1.7560e-06, 6.4149e-06, 5.0552e-06],\n", + " [-2.5071e-06, 5.1111e-06, 1.0952e-06]],\n", + " \n", + " [[-4.3288e-06, -4.2375e-08, -6.1188e-07],\n", + " [-5.4818e-06, -5.6252e-07, 1.9334e-06],\n", + " [-3.3206e-06, -8.2329e-07, 1.2852e-06]]],\n", + " \n", + " \n", + " [[[-2.8536e-06, -1.1292e-06, -9.2294e-07],\n", + " [-5.5768e-06, -1.2964e-06, -1.2219e-06],\n", + " [-4.1910e-06, -1.2228e-06, 9.2015e-07]],\n", + " \n", + " [[ 3.3304e-06, 4.1276e-06, 2.2016e-06],\n", + " [ 7.6313e-06, 9.5479e-06, 4.8429e-06],\n", + " [ 7.2848e-06, 9.1661e-06, 5.1521e-06]],\n", + " \n", + " [[ 3.7234e-06, 3.9302e-06, -1.7323e-06],\n", + " [ 1.8487e-06, 1.8633e-06, -2.1700e-06],\n", + " [-1.0543e-06, 8.4564e-07, -2.5992e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.2522e-06, -6.2212e-07, -1.0729e-06],\n", + " [-5.3775e-06, 8.4564e-07, 1.0915e-06],\n", + " [-2.3246e-06, 9.8348e-07, 1.4007e-06]],\n", + " \n", + " [[ 5.3765e-06, -1.1027e-06, 3.7532e-06],\n", + " [-1.6410e-06, -6.9682e-06, -1.2703e-06],\n", + " [-5.6522e-06, -5.5939e-06, -7.5437e-07]],\n", + " \n", + " [[ 6.2548e-06, -7.1898e-07, -2.9057e-06],\n", + " [ 3.6843e-06, -4.1574e-06, -5.1297e-06],\n", + " [ 4.4182e-06, -7.5880e-07, -3.2485e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-4.8848e-07, -2.2370e-06, -7.3761e-07],\n", + " [-2.6040e-06, -2.7157e-06, 6.4448e-07],\n", + " [-4.6734e-06, -5.6233e-06, -1.1399e-06]],\n", + " \n", + " [[ 1.5744e-06, 4.2841e-07, 2.7604e-06],\n", + " [ 4.1211e-07, -1.1679e-06, 1.5814e-06],\n", + " [ 4.0978e-08, 6.5006e-07, 3.0883e-06]],\n", + " \n", + " [[ 1.4529e-07, -3.7104e-06, 1.3374e-06],\n", + " [-2.7436e-06, -5.1670e-06, -7.4878e-07],\n", + " [-1.9148e-06, -5.0664e-07, 3.9861e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.8871e-08, -1.3355e-06, 2.4214e-06],\n", + " [ 1.5683e-06, 3.0920e-07, 2.9989e-06],\n", + " [ 8.9779e-07, -4.2096e-07, 2.5444e-06]],\n", + " \n", + " [[-6.6869e-07, -6.1095e-07, -9.7975e-07],\n", + " [-2.5481e-06, -4.0948e-06, -3.0454e-06],\n", + " [-8.5682e-07, -1.7434e-06, -2.9467e-06]],\n", + " \n", + " [[ 5.7369e-07, -2.5863e-06, -2.7101e-06],\n", + " [ 4.2096e-07, -1.4342e-06, -3.4403e-06],\n", + " [-2.2538e-06, -3.6289e-06, -3.2485e-06]]],\n", + " \n", + " \n", + " [[[ 3.2373e-06, 1.9334e-06, 1.7742e-07],\n", + " [ 3.5670e-06, 3.5502e-06, 3.4412e-06],\n", + " [ 4.6641e-06, 3.3602e-06, 3.5083e-06]],\n", + " \n", + " [[ 1.0449e-06, -1.8813e-07, 1.7798e-06],\n", + " [ 2.2855e-06, -6.5751e-07, 1.9576e-06],\n", + " [ 1.7090e-06, -2.1979e-06, -1.7575e-06]],\n", + " \n", + " [[ 1.3299e-06, 1.1865e-06, 3.3714e-07],\n", + " [-7.8231e-07, -2.0489e-06, 4.3232e-06],\n", + " [-2.2352e-08, -5.4110e-06, -2.6571e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.4752e-06, -1.1646e-06, -1.8552e-06],\n", + " [ 1.1590e-06, -5.8860e-07, -8.1956e-07],\n", + " [ 2.2389e-06, -8.1956e-08, -3.5157e-07]],\n", + " \n", + " [[ 1.4435e-06, 1.6987e-06, 3.6778e-06],\n", + " [ 1.5367e-06, 3.9674e-07, 5.6811e-07],\n", + " [-1.2927e-06, -1.4920e-06, -3.2373e-06]],\n", + " \n", + " [[ 2.4848e-06, 3.2182e-06, 2.7558e-06],\n", + " [ 1.3527e-06, 5.2899e-07, 6.3516e-07],\n", + " [ 2.4633e-06, 7.4506e-07, 5.3272e-07]]],\n", + " \n", + " \n", + " [[[-1.1139e-06, -5.5134e-07, 1.3784e-06],\n", + " [-1.5832e-07, 2.1681e-06, 2.5332e-06],\n", + " [ 8.7172e-07, 1.8976e-06, 1.9995e-06]],\n", + " \n", + " [[ 9.8720e-07, -7.5344e-07, -1.0161e-06],\n", + " [-1.2331e-06, -6.7614e-07, -8.3819e-07],\n", + " [-1.3132e-07, 1.3988e-06, 2.1905e-06]],\n", + " \n", + " [[ 3.2950e-06, 1.7965e-06, 1.4603e-06],\n", + " [ 3.5167e-06, 1.9483e-06, 3.4384e-06],\n", + " [ 4.1015e-06, 5.1074e-06, 4.8671e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.3027e-07, 1.9632e-06, 2.3972e-06],\n", + " [ 1.1725e-06, -2.5332e-07, 5.9977e-07],\n", + " [ 2.3171e-06, -9.8720e-07, 3.0175e-07]],\n", + " \n", + " [[-1.1092e-06, 2.0117e-07, -2.1085e-06],\n", + " [ 7.3016e-07, 2.3767e-06, 2.7288e-06],\n", + " [ 8.9779e-07, 1.3616e-06, 5.2061e-07]],\n", + " \n", + " [[ 6.9663e-07, -2.1420e-07, -1.7919e-06],\n", + " [ 2.5257e-06, 3.1330e-06, -4.6194e-07],\n", + " [ 2.1383e-06, 2.7437e-06, 1.3290e-06]]]], device='cuda:0')),\n", + " ('layer2.0.bn1.weight',\n", + " tensor([ 3.0994e-06, -2.3246e-06, 4.6492e-06, 7.1526e-07, 3.3379e-06,\n", + " -1.4305e-06, -1.6689e-06, 1.1921e-06, -4.1723e-07, 7.1526e-07,\n", + " 1.3113e-06, 1.3113e-06, 1.0729e-06, -1.7285e-06, -5.3644e-07,\n", + " -2.9802e-07, -4.7684e-07, -2.0266e-06, 2.3842e-07, -5.3644e-07,\n", + " 1.6689e-06, 4.7684e-07, -5.3644e-07, 2.3842e-07, 0.0000e+00,\n", + " -5.9605e-08, -7.7486e-07, -6.5565e-07, 1.9073e-06, -2.6226e-06,\n", + " -7.1526e-07, 2.3842e-07, -1.5497e-06, -2.3842e-07, -2.4438e-06,\n", + " 5.9605e-07, -3.5763e-07, 1.1921e-06, -3.2783e-06, 2.5034e-06,\n", + " -1.6689e-06, 2.0266e-06, 2.8610e-06, 2.0266e-06, -1.0729e-06,\n", + " 1.1921e-07, -1.6689e-06, 9.5367e-07, 9.5367e-07, 5.9605e-07,\n", + " -1.7881e-06, -1.0729e-06, -1.4305e-06, 4.7684e-07, 2.0266e-06,\n", + " -1.1921e-06, 2.3842e-07, 1.1921e-06, 1.6689e-06, -2.4438e-06,\n", + " -1.9670e-06, -1.7881e-06, 2.3842e-07, 1.5497e-06, 1.6689e-06,\n", + " -1.7881e-07, 2.3842e-07, -1.5497e-06, 1.1921e-06, 1.5497e-06,\n", + " -1.6093e-06, -2.2054e-06, -2.0862e-06, -1.6689e-06, -1.4305e-06,\n", + " 1.4305e-06, -8.3447e-07, -2.6226e-06, -1.1921e-06, 2.1458e-06,\n", + " -2.7418e-06, 3.5763e-07, -1.1921e-07, -1.1325e-06, -1.4901e-06,\n", + " 3.6955e-06, -1.7285e-06, -2.4438e-06, -6.5565e-07, -3.5763e-07,\n", + " 1.3113e-06, -4.7684e-07, -9.5367e-07, -5.9605e-07, 1.6689e-06,\n", + " 4.7684e-07, 4.7684e-07, 7.1526e-07, -2.3246e-06, 1.1921e-06,\n", + " -4.1723e-07, 2.3842e-07, 0.0000e+00, 2.6226e-06, -2.0862e-06,\n", + " 5.9605e-07, 2.3842e-06, 1.1921e-06, 1.5497e-06, -1.4901e-06,\n", + " 9.5367e-07, -2.3842e-07, 1.1921e-06, 2.5034e-06, 8.3447e-07,\n", + " 7.1526e-07, -2.9802e-06, 1.1921e-07, 1.4305e-06, 1.4305e-06,\n", + " 3.5763e-07, 1.3113e-06, 2.3842e-06, -1.2517e-06, -2.3842e-06,\n", + " -3.1590e-06, -3.5763e-07, 2.1458e-06], device='cuda:0')),\n", + " ('layer2.0.bn1.bias',\n", + " tensor([ 1.0771e-06, -1.8999e-06, 2.3541e-06, 9.5966e-07, 8.2739e-07,\n", + " 6.1912e-07, -1.2996e-06, -1.6184e-08, 4.1995e-07, -1.6756e-06,\n", + " 1.3479e-06, 1.0882e-06, 1.9268e-06, -4.1985e-07, -1.1347e-06,\n", + " 5.0927e-07, 2.6343e-07, 4.0396e-07, 1.0203e-07, -8.1831e-07,\n", + " -9.6027e-07, 6.5029e-07, -3.9052e-07, -9.8193e-09, -2.3134e-07,\n", + " -1.4903e-06, 1.7103e-07, -1.4026e-06, 7.8846e-07, -1.4667e-06,\n", + " 5.2717e-07, -1.0885e-06, -1.0228e-06, -1.1879e-06, -1.7712e-06,\n", + " -9.4132e-07, 1.8629e-06, 1.4127e-06, -3.9082e-06, 4.5633e-06,\n", + " 2.9374e-07, 4.0827e-06, 2.5944e-06, -1.6692e-07, 2.0453e-07,\n", + " -7.2985e-07, -2.6793e-06, 2.5667e-06, -1.1951e-07, 1.1597e-06,\n", + " -2.8109e-06, -7.0437e-07, -2.7890e-06, -9.6153e-07, 8.7216e-07,\n", + " -1.2926e-06, 4.4592e-07, 1.8612e-06, 1.7681e-06, -1.9762e-06,\n", + " -5.1246e-07, -1.2139e-06, -6.3102e-07, 5.6782e-07, 1.6684e-06,\n", + " -5.8652e-10, 1.3252e-06, -3.3027e-07, 5.1575e-07, 1.8013e-06,\n", + " -1.7359e-06, -1.2953e-06, -1.7131e-06, -9.5531e-07, -2.4773e-06,\n", + " 9.2660e-07, -6.0866e-07, 6.1627e-08, -1.3812e-06, 2.8934e-06,\n", + " -3.2839e-06, 6.3595e-07, 6.5303e-07, 5.2513e-07, -1.0461e-06,\n", + " 1.2402e-06, -1.7747e-06, -2.3992e-06, -1.1695e-06, -2.2699e-07,\n", + " 1.1653e-06, -3.9334e-07, -1.7928e-07, -2.2050e-06, 2.8558e-06,\n", + " -4.4786e-08, 1.0607e-06, 9.8461e-07, -3.5410e-07, 4.3437e-07,\n", + " -9.9145e-07, -5.3667e-07, -2.7750e-07, 7.5311e-07, -1.4992e-06,\n", + " -8.5458e-08, 2.2051e-06, 1.2309e-06, -4.6171e-07, -1.5314e-06,\n", + " 1.3039e-06, 3.8103e-07, 1.0440e-06, 7.3113e-07, 3.6085e-07,\n", + " 1.0573e-06, -1.3022e-06, -8.5626e-07, -8.9232e-07, 1.5499e-06,\n", + " -2.2795e-06, 2.6757e-08, 4.8539e-07, 1.0232e-06, -1.1481e-06,\n", + " -2.9484e-06, 3.6970e-07, 2.1285e-06], device='cuda:0')),\n", + " ('layer2.0.conv2.weight',\n", + " tensor([[[[-2.1514e-06, -4.4517e-07, -2.9746e-06],\n", + " [ 1.1344e-06, 3.4533e-06, -6.4541e-07],\n", + " [-2.0098e-06, -8.0187e-07, 9.8906e-07]],\n", + " \n", + " [[-3.3062e-06, 1.6661e-06, -5.7537e-06],\n", + " [-5.1633e-06, -3.9916e-06, -4.7423e-06],\n", + " [-1.5590e-06, -5.8301e-07, -1.4924e-06]],\n", + " \n", + " [[ 6.4896e-07, 2.2817e-06, 2.6803e-06],\n", + " [ 2.0345e-06, 1.3877e-06, 7.6555e-07],\n", + " [-7.0175e-07, 2.4941e-06, -2.8219e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.3395e-06, -1.1553e-06, -1.6065e-06],\n", + " [-7.0222e-07, -1.7192e-06, -1.4841e-06],\n", + " [ 7.2271e-07, -6.1467e-07, -2.3991e-06]],\n", + " \n", + " [[ 1.8273e-06, 4.4936e-06, 3.6415e-06],\n", + " [ 4.7404e-07, -1.6363e-06, -2.8387e-06],\n", + " [ 1.9139e-07, 3.1106e-07, 9.4343e-07]],\n", + " \n", + " [[-1.4259e-06, 1.3872e-06, -1.6890e-06],\n", + " [ 1.8533e-07, -1.7621e-06, -1.7360e-06],\n", + " [ 9.9931e-07, 5.2154e-07, -8.0717e-07]]],\n", + " \n", + " \n", + " [[[ 1.7267e-06, 1.5832e-06, -2.2966e-06],\n", + " [ 3.2969e-07, -1.9427e-06, -1.2852e-07],\n", + " [-7.7300e-08, 3.0920e-07, -1.5665e-06]],\n", + " \n", + " [[-2.5411e-06, -2.0552e-06, -3.6787e-06],\n", + " [-2.5965e-06, 9.0152e-07, 7.1153e-07],\n", + " [ 1.2773e-06, -4.9733e-07, 1.8459e-06]],\n", + " \n", + " [[ 5.9893e-06, 5.8422e-06, 4.9574e-06],\n", + " [-8.1677e-07, 4.0879e-06, 1.6466e-06],\n", + " [-3.8650e-06, 6.7134e-06, 1.2349e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.0419e-06, -4.0117e-07, -2.0582e-06],\n", + " [-3.5390e-06, 2.6021e-06, 2.2091e-06],\n", + " [ 6.0376e-07, 1.8068e-06, 8.2888e-06]],\n", + " \n", + " [[-3.5036e-06, -5.3332e-06, -3.3472e-06],\n", + " [ 2.8405e-06, 4.3381e-06, 7.2783e-07],\n", + " [ 1.2168e-06, 2.5146e-06, 8.4937e-07]],\n", + " \n", + " [[-5.7928e-06, 6.7428e-07, -1.7097e-06],\n", + " [-3.8249e-06, 3.7663e-06, -1.0133e-06],\n", + " [-4.5784e-06, 2.1448e-06, 5.2340e-07]]],\n", + " \n", + " \n", + " [[[-1.0058e-07, 9.3319e-07, 6.0908e-07],\n", + " [-2.2706e-06, -1.8161e-06, 1.3597e-07],\n", + " [ 1.9195e-06, -5.4017e-08, 7.3528e-07]],\n", + " \n", + " [[-3.7653e-06, -1.3839e-06, -1.8850e-06],\n", + " [ 2.2743e-06, -3.7043e-06, 2.1104e-06],\n", + " [ 2.0303e-07, -2.9318e-06, -1.9697e-06]],\n", + " \n", + " [[ 5.2527e-07, -2.2016e-06, -1.6796e-06],\n", + " [-4.0112e-06, -3.6266e-06, -2.6468e-06],\n", + " [-2.9904e-06, -3.6880e-06, -2.9420e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.5777e-06, 1.3664e-06, 2.5253e-06],\n", + " [-1.0487e-06, -1.3374e-06, 2.2696e-06],\n", + " [-3.2429e-06, -2.6189e-06, -8.7358e-07]],\n", + " \n", + " [[ 2.0545e-06, -1.5879e-07, 6.7288e-07],\n", + " [ 2.9467e-06, 2.0145e-06, 1.0217e-06],\n", + " [-3.0193e-06, -2.2622e-06, 4.6939e-07]],\n", + " \n", + " [[-3.4696e-06, 8.2515e-07, 1.4845e-06],\n", + " [-8.0746e-07, 1.3467e-06, 4.7041e-06],\n", + " [-1.9581e-06, 5.1195e-06, 2.6107e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 2.8685e-07, 1.9372e-07, -9.9884e-07],\n", + " [ 2.5108e-06, -6.0175e-07, 1.4156e-07],\n", + " [-2.3656e-07, -9.0338e-08, 1.5652e-06]],\n", + " \n", + " [[-3.6918e-06, 5.6997e-07, -3.8557e-07],\n", + " [-1.2922e-07, -2.5872e-06, 8.2515e-07],\n", + " [ 7.7114e-07, 4.8988e-07, -3.9302e-07]],\n", + " \n", + " [[-3.8408e-06, -1.8626e-08, -2.6049e-06],\n", + " [-3.6149e-06, -4.4340e-06, -2.1178e-06],\n", + " [-3.2354e-06, -3.5204e-07, -2.5909e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.1048e-06, 3.4748e-06, 2.1970e-06],\n", + " [-8.9407e-08, -1.5143e-06, -6.6869e-07],\n", + " [-1.8440e-07, 4.0280e-07, 1.4566e-06]],\n", + " \n", + " [[ 1.6391e-06, 1.5753e-06, 5.0291e-08],\n", + " [-2.1998e-06, -1.4798e-06, -3.2410e-07],\n", + " [-1.6149e-06, -1.6035e-06, -1.8626e-07]],\n", + " \n", + " [[ 1.3895e-06, -6.9290e-07, 1.0058e-06],\n", + " [ 9.7603e-07, 1.8254e-07, 2.1122e-06],\n", + " [ 1.1222e-06, -1.3644e-06, 1.4249e-06]]],\n", + " \n", + " \n", + " [[[ 9.8883e-07, 1.2954e-06, 1.1176e-08],\n", + " [ 1.9465e-06, 1.8999e-07, 9.9652e-07],\n", + " [ 1.0934e-06, 3.8464e-06, 3.0501e-06]],\n", + " \n", + " [[ 2.5611e-06, 5.6438e-07, 2.6748e-06],\n", + " [-5.2934e-06, -3.7625e-07, 1.7006e-06],\n", + " [-2.6617e-06, -4.5151e-06, -6.0908e-07]],\n", + " \n", + " [[ 5.6410e-06, 4.3306e-06, 4.9236e-06],\n", + " [ 1.6531e-06, 5.6252e-07, 1.8831e-06],\n", + " [-1.4342e-06, 1.5864e-06, 1.9833e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.6678e-06, 1.4324e-06, 4.2813e-06],\n", + " [-4.9733e-07, -3.1106e-07, 5.8115e-07],\n", + " [ 1.6531e-07, -2.5555e-06, -8.4937e-07]],\n", + " \n", + " [[ 2.5015e-06, 1.5032e-06, 1.0561e-06],\n", + " [ 3.4347e-06, 9.9372e-07, 2.5593e-06],\n", + " [-3.6322e-07, -3.2689e-07, -3.8557e-07]],\n", + " \n", + " [[-8.7707e-07, -1.1334e-06, -2.5295e-06],\n", + " [ 5.0152e-07, -3.6117e-06, 1.3619e-06],\n", + " [ 1.2880e-06, -1.3337e-06, 1.8906e-06]]],\n", + " \n", + " \n", + " [[[ 2.3991e-06, -1.5143e-06, 7.7672e-07],\n", + " [-5.1153e-07, -5.9344e-06, 6.1095e-07],\n", + " [ 2.1607e-06, 3.4459e-07, -1.2433e-06]],\n", + " \n", + " [[ 8.2888e-07, 3.4738e-07, 3.4310e-06],\n", + " [-2.8498e-07, 8.0653e-07, 1.4659e-06],\n", + " [ 1.7006e-06, 1.2480e-07, 2.5472e-07]],\n", + " \n", + " [[-3.4552e-06, -4.0336e-06, -4.9975e-06],\n", + " [-2.2650e-06, -6.0592e-06, -1.8515e-06],\n", + " [-9.5461e-07, -1.8887e-06, -6.9030e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.2055e-06, 5.0664e-07, -1.9995e-06],\n", + " [-4.9919e-07, -1.3830e-06, 4.9779e-07],\n", + " [-4.6324e-06, -1.8235e-06, 1.4249e-06]],\n", + " \n", + " [[-2.1365e-06, -2.4159e-06, -2.1439e-06],\n", + " [-1.3076e-06, -4.5174e-06, -5.5879e-09],\n", + " [-1.0505e-06, -4.1085e-06, -3.5774e-06]],\n", + " \n", + " [[ 2.1681e-06, 4.9733e-07, 1.4934e-06],\n", + " [ 4.3348e-06, 2.0023e-06, 4.0140e-06],\n", + " [ 2.2836e-06, 5.8487e-07, 2.8322e-06]]]], device='cuda:0')),\n", + " ('layer2.0.bn2.weight',\n", + " tensor([-8.9407e-07, -1.1921e-07, 1.1921e-07, 2.0266e-06, 2.3842e-06,\n", + " -5.9605e-08, -2.0862e-06, -1.4901e-06, -1.4305e-06, -5.9605e-07,\n", + " 1.1921e-07, -1.0729e-06, -1.8477e-06, 1.9073e-06, 1.1921e-07,\n", + " -1.4305e-06, -1.0133e-06, -2.6822e-06, 0.0000e+00, 2.3842e-07,\n", + " -1.0133e-06, 5.9605e-07, -4.7684e-07, -1.1921e-07, -1.1921e-06,\n", + " -1.6093e-06, 1.1921e-07, -7.7486e-07, 1.9073e-06, -1.7285e-06,\n", + " -1.0133e-06, -1.7881e-06, 1.5497e-06, 1.4305e-06, 2.3842e-07,\n", + " 1.1921e-07, 1.0729e-06, 1.9073e-06, -2.2054e-06, -1.3709e-06,\n", + " -2.0862e-06, -3.5763e-07, 4.7684e-07, -1.1921e-07, 1.1921e-06,\n", + " -1.4305e-06, -1.3113e-06, -1.9670e-06, -4.1723e-07, 1.6689e-06,\n", + " 4.2915e-06, 1.0729e-06, 9.5367e-07, 9.5367e-07, -2.0266e-06,\n", + " 0.0000e+00, -1.1325e-06, 1.0729e-06, -2.4438e-06, 1.1921e-07,\n", + " 1.1921e-07, 5.9605e-07, 1.1921e-06, 8.3447e-07, -2.9802e-07,\n", + " -1.4901e-06, -1.0729e-06, 4.7684e-07, -7.1526e-07, 2.3842e-06,\n", + " -9.5367e-07, 7.1526e-07, 2.1458e-06, -1.6689e-06, -1.1921e-07,\n", + " -8.3447e-07, -3.5763e-07, 1.1921e-07, -1.9670e-06, -7.1526e-07,\n", + " 4.7684e-07, -1.1921e-07, 1.1921e-07, 2.5034e-06, 4.0531e-06,\n", + " -2.0266e-06, 2.3842e-06, 1.1921e-07, 3.5763e-06, -5.9605e-08,\n", + " -1.6689e-06, 9.5367e-07, 0.0000e+00, 8.3447e-07, 1.1921e-07,\n", + " 5.9605e-07, 1.3113e-06, -1.0729e-06, -4.7684e-07, 1.9073e-06,\n", + " -1.4305e-06, -5.9605e-08, 1.1921e-07, -4.1723e-07, 1.4305e-06,\n", + " 2.3842e-07, -1.3709e-06, -3.3975e-06, 5.9605e-07, -1.8477e-06,\n", + " -7.1526e-07, 9.5367e-07, -1.2517e-06, -2.0862e-06, 1.5497e-06,\n", + " 1.6689e-06, 7.1526e-07, 1.6689e-06, -8.9407e-07, 1.7881e-06,\n", + " -2.3842e-07, -1.6093e-06, 9.5367e-07, 1.1921e-07, -5.4836e-06,\n", + " -1.1921e-06, -3.5763e-07, 1.1921e-07], device='cuda:0')),\n", + " ('layer2.0.bn2.bias',\n", + " tensor([-1.9510e-06, 1.9011e-07, -9.3751e-07, 1.4218e-06, 1.5925e-06,\n", + " -4.4937e-07, 1.6194e-07, 1.3984e-06, -9.2120e-07, -1.6821e-07,\n", + " -1.0528e-06, -8.1123e-07, 5.3874e-07, 9.9689e-10, -7.2899e-07,\n", + " -1.2392e-06, 5.1126e-07, -7.3782e-07, -1.4009e-06, -3.6747e-07,\n", + " -1.5048e-07, 3.9299e-07, 8.8879e-07, 1.2054e-06, 4.6736e-07,\n", + " -4.8633e-07, 6.5945e-07, 4.0367e-07, 1.0216e-06, -5.3070e-08,\n", + " 6.0140e-07, -1.6197e-06, 1.6229e-06, -1.6424e-08, -5.1029e-07,\n", + " -4.7327e-07, -2.4663e-07, 1.1023e-06, -6.4749e-07, -7.0314e-07,\n", + " -1.1032e-06, -1.1533e-06, -1.3615e-07, -5.3711e-07, 1.8676e-06,\n", + " -1.2163e-06, -3.2784e-07, -5.9893e-08, 1.2751e-06, 7.8705e-07,\n", + " 9.8065e-07, -1.1253e-06, 3.8889e-07, -1.3021e-06, -1.0771e-06,\n", + " -9.6265e-08, 1.0677e-06, 1.4190e-06, -2.3022e-07, 2.1243e-07,\n", + " -2.4294e-06, 1.4054e-06, -2.2632e-07, 2.3604e-06, 8.6605e-07,\n", + " 5.1579e-07, -3.7519e-07, 2.3580e-08, -1.0342e-07, 1.8895e-06,\n", + " 4.3311e-07, 2.1627e-06, 1.2310e-06, -1.4375e-06, -1.8110e-06,\n", + " -6.7435e-07, -1.1090e-06, 3.9569e-08, -7.6939e-07, -7.7326e-07,\n", + " -5.9887e-07, -1.9408e-07, -2.5970e-07, 2.3944e-06, 2.1859e-06,\n", + " -1.1776e-06, 9.0604e-07, 3.0137e-07, 1.8005e-07, 1.5975e-06,\n", + " 5.8386e-07, 2.5003e-07, -2.1950e-07, 1.0011e-06, -7.0608e-07,\n", + " 4.6388e-07, 1.3524e-06, 1.2713e-06, 1.4605e-07, -7.1791e-07,\n", + " 5.6460e-07, 4.2521e-07, -5.2877e-07, -4.5762e-07, -4.2111e-07,\n", + " -7.0937e-07, 3.3107e-07, -5.2286e-07, -3.0880e-07, -1.5176e-06,\n", + " -8.7055e-07, 6.5832e-07, 1.8420e-07, -1.2434e-06, -1.6331e-08,\n", + " -4.1791e-07, 3.5488e-07, 6.8525e-07, -4.0871e-07, 5.5506e-07,\n", + " -6.4726e-07, 4.8202e-07, -1.8036e-06, 1.2732e-06, -2.8784e-06,\n", + " -2.3360e-06, -1.4952e-06, -1.5416e-06], device='cuda:0')),\n", + " ('layer2.0.shortcut.0.weight',\n", + " tensor([[[[-2.5854e-06]],\n", + " \n", + " [[-1.9930e-06]],\n", + " \n", + " [[ 1.7509e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.1681e-06]],\n", + " \n", + " [[ 2.6748e-06]],\n", + " \n", + " [[ 1.9744e-07]]],\n", + " \n", + " \n", + " [[[ 9.6671e-07]],\n", + " \n", + " [[ 3.7253e-07]],\n", + " \n", + " [[ 2.2352e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.0978e-07]],\n", + " \n", + " [[ 4.0457e-06]],\n", + " \n", + " [[-3.8883e-08]]],\n", + " \n", + " \n", + " [[[-2.4810e-06]],\n", + " \n", + " [[ 8.9962e-07]],\n", + " \n", + " [[ 2.9579e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.8701e-06]],\n", + " \n", + " [[-8.6427e-07]],\n", + " \n", + " [[-9.2480e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-5.1484e-06]],\n", + " \n", + " [[-4.0270e-06]],\n", + " \n", + " [[-3.0454e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 9.8348e-07]],\n", + " \n", + " [[-1.9073e-06]],\n", + " \n", + " [[-1.9446e-06]]],\n", + " \n", + " \n", + " [[[-3.1665e-07]],\n", + " \n", + " [[ 4.6939e-07]],\n", + " \n", + " [[-3.2857e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 5.1782e-07]],\n", + " \n", + " [[ 9.4250e-07]],\n", + " \n", + " [[ 4.9919e-07]]],\n", + " \n", + " \n", + " [[[-1.2517e-06]],\n", + " \n", + " [[-2.1607e-07]],\n", + " \n", + " [[-1.9586e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.1963e-06]],\n", + " \n", + " [[-3.1367e-06]],\n", + " \n", + " [[-3.6154e-06]]]], device='cuda:0')),\n", + " ('layer2.0.shortcut.1.weight',\n", + " tensor([-1.7881e-07, -5.9605e-08, -8.3447e-07, 1.5497e-06, -2.9802e-07,\n", + " 2.3842e-07, -7.1526e-07, 2.7418e-06, -8.9407e-07, 5.9605e-07,\n", + " -1.1325e-06, -1.4305e-06, 3.5763e-07, 1.1921e-07, -9.5367e-07,\n", + " -1.1921e-06, 1.6689e-06, 4.7684e-07, -4.7684e-07, -5.9605e-07,\n", + " -3.5763e-07, -6.5565e-07, -1.1921e-07, 1.0729e-06, 2.3842e-06,\n", + " 1.4305e-06, 1.3113e-06, -2.3842e-07, -1.3709e-06, 2.1458e-06,\n", + " -5.9605e-08, -2.0862e-06, 2.5034e-06, -1.7881e-06, -5.3644e-07,\n", + " 9.5367e-07, -1.2517e-06, -1.6689e-06, 7.1526e-07, 5.9605e-07,\n", + " 2.3842e-07, 9.5367e-07, -5.9605e-08, 1.3113e-06, 1.4305e-06,\n", + " 1.6689e-06, 5.9605e-07, -4.1723e-07, 2.2650e-06, -2.9802e-07,\n", + " -2.9802e-07, -1.0729e-06, 4.7684e-07, -2.9802e-06, -1.0729e-06,\n", + " -2.3842e-07, 4.7684e-07, -2.9802e-07, 9.5367e-07, -2.2054e-06,\n", + " -1.5497e-06, 1.1921e-07, -4.1723e-07, 1.6689e-06, 1.1921e-07,\n", + " 1.7881e-06, 1.4305e-06, -5.9605e-08, 1.1921e-06, -3.5763e-07,\n", + " -1.7881e-07, 4.0531e-06, 8.3447e-07, -1.3113e-06, -1.5497e-06,\n", + " 1.1921e-06, -3.5763e-07, -9.5367e-07, -1.7881e-07, -1.3113e-06,\n", + " -1.3113e-06, 0.0000e+00, 2.3842e-07, 8.3447e-07, 2.9802e-06,\n", + " 7.1526e-07, -1.7881e-06, 2.1458e-06, -1.9670e-06, 1.5497e-06,\n", + " 1.0729e-06, -8.3447e-07, -7.1526e-07, 1.6689e-06, 4.7684e-07,\n", + " -2.3842e-07, 5.9605e-07, 4.7684e-07, 1.9073e-06, -5.3644e-07,\n", + " 2.0266e-06, -5.9605e-08, -8.3447e-07, 8.3447e-07, -4.0531e-06,\n", + " 1.1921e-07, 2.3842e-06, 1.3113e-06, -1.3709e-06, 1.1921e-07,\n", + " -1.7881e-07, -8.3447e-07, 1.9073e-06, 2.5034e-06, -4.7684e-07,\n", + " -9.5367e-07, 3.5763e-07, 3.5763e-07, 1.1921e-06, -1.7881e-06,\n", + " -8.3447e-07, 5.9605e-07, -1.4305e-06, -1.1921e-07, 1.4305e-06,\n", + " -1.8477e-06, -1.1325e-06, -2.4438e-06], device='cuda:0')),\n", + " ('layer2.0.shortcut.1.bias',\n", + " tensor([-1.9510e-06, 1.9011e-07, -9.3751e-07, 1.4218e-06, 1.5925e-06,\n", + " -4.4937e-07, 1.6194e-07, 1.3984e-06, -9.2120e-07, -1.6821e-07,\n", + " -1.0528e-06, -8.1123e-07, 5.3874e-07, 9.9689e-10, -7.2899e-07,\n", + " -1.2392e-06, 5.1126e-07, -7.3782e-07, -1.4009e-06, -3.6747e-07,\n", + " -1.5048e-07, 3.9299e-07, 8.8879e-07, 1.2054e-06, 4.6736e-07,\n", + " -4.8633e-07, 6.5945e-07, 4.0367e-07, 1.0216e-06, -5.3070e-08,\n", + " 6.0140e-07, -1.6197e-06, 1.6229e-06, -1.6424e-08, -5.1029e-07,\n", + " -4.7327e-07, -2.4663e-07, 1.1023e-06, -6.4749e-07, -7.0314e-07,\n", + " -1.1032e-06, -1.1533e-06, -1.3615e-07, -5.3711e-07, 1.8676e-06,\n", + " -1.2163e-06, -3.2784e-07, -5.9893e-08, 1.2751e-06, 7.8705e-07,\n", + " 9.8065e-07, -1.1253e-06, 3.8889e-07, -1.3021e-06, -1.0771e-06,\n", + " -9.6265e-08, 1.0677e-06, 1.4190e-06, -2.3022e-07, 2.1243e-07,\n", + " -2.4294e-06, 1.4054e-06, -2.2632e-07, 2.3604e-06, 8.6605e-07,\n", + " 5.1579e-07, -3.7519e-07, 2.3580e-08, -1.0342e-07, 1.8895e-06,\n", + " 4.3311e-07, 2.1627e-06, 1.2310e-06, -1.4375e-06, -1.8110e-06,\n", + " -6.7435e-07, -1.1090e-06, 3.9569e-08, -7.6939e-07, -7.7326e-07,\n", + " -5.9887e-07, -1.9408e-07, -2.5970e-07, 2.3944e-06, 2.1859e-06,\n", + " -1.1776e-06, 9.0604e-07, 3.0137e-07, 1.8005e-07, 1.5975e-06,\n", + " 5.8386e-07, 2.5003e-07, -2.1950e-07, 1.0011e-06, -7.0608e-07,\n", + " 4.6388e-07, 1.3524e-06, 1.2713e-06, 1.4605e-07, -7.1791e-07,\n", + " 5.6460e-07, 4.2521e-07, -5.2877e-07, -4.5762e-07, -4.2111e-07,\n", + " -7.0937e-07, 3.3107e-07, -5.2286e-07, -3.0880e-07, -1.5176e-06,\n", + " -8.7055e-07, 6.5832e-07, 1.8420e-07, -1.2434e-06, -1.6331e-08,\n", + " -4.1791e-07, 3.5488e-07, 6.8525e-07, -4.0871e-07, 5.5506e-07,\n", + " -6.4726e-07, 4.8202e-07, -1.8036e-06, 1.2732e-06, -2.8784e-06,\n", + " -2.3360e-06, -1.4952e-06, -1.5416e-06], device='cuda:0')),\n", + " ('layer3.0.conv1.weight',\n", + " tensor([[[[ 5.6066e-07, 3.1535e-06, -6.3330e-08],\n", + " [ 2.7567e-07, 2.3115e-06, -4.9919e-07],\n", + " [ 1.1055e-06, 3.4086e-07, -1.7099e-06]],\n", + " \n", + " [[ 7.2829e-07, -9.6858e-08, 2.2147e-06],\n", + " [-4.3772e-07, -1.8394e-06, 7.1712e-07],\n", + " [ 5.1595e-07, -2.5462e-06, 2.2170e-06]],\n", + " \n", + " [[ 7.7486e-07, 3.1106e-07, 8.1607e-08],\n", + " [-4.1351e-07, 2.9430e-07, 2.8312e-07],\n", + " [ 7.6741e-07, 2.6692e-06, -6.5565e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.8629e-06, 3.5446e-06, 1.9521e-06],\n", + " [ 1.9660e-06, 2.7940e-08, -1.6373e-06],\n", + " [ 1.1846e-06, -1.9739e-06, -4.0876e-06]],\n", + " \n", + " [[ 1.7420e-06, -8.7731e-07, -3.0389e-06],\n", + " [ 4.0838e-07, 3.3341e-07, -2.4578e-06],\n", + " [ 2.0117e-06, -2.5751e-07, 4.4145e-07]],\n", + " \n", + " [[ 9.3598e-07, -3.6322e-08, -1.7881e-07],\n", + " [-1.0561e-06, 2.5891e-07, 2.0489e-07],\n", + " [ 2.3227e-06, 8.8196e-07, 4.3586e-07]]],\n", + " \n", + " \n", + " [[[ 1.9260e-06, 5.9232e-07, 1.6717e-06],\n", + " [-9.0571e-08, 1.9991e-06, -2.2165e-07],\n", + " [ 6.3377e-07, -2.0675e-07, -1.4231e-06]],\n", + " \n", + " [[ 3.2317e-07, -3.0757e-07, -5.6578e-08],\n", + " [-6.3470e-07, -1.9409e-06, -1.7677e-06],\n", + " [ 5.7626e-07, -2.5439e-06, -1.5665e-06]],\n", + " \n", + " [[-2.1514e-07, -1.3020e-06, -3.8417e-07],\n", + " [ 1.2033e-06, 9.5926e-07, 1.2582e-06],\n", + " [-6.8918e-08, 2.0675e-07, 8.0559e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.0473e-06, -3.7812e-07, 1.7732e-06],\n", + " [ 7.0455e-07, -3.6322e-07, -1.1548e-07],\n", + " [-9.6997e-07, -9.4622e-07, -1.6037e-06]],\n", + " \n", + " [[ 1.9846e-06, 1.6354e-06, 2.0321e-06],\n", + " [-1.5739e-07, 2.3842e-07, 1.6857e-07],\n", + " [-3.8370e-07, 1.9930e-07, 1.7909e-06]],\n", + " \n", + " [[ 1.1469e-06, 9.3691e-07, -1.8883e-06],\n", + " [-2.6161e-06, 2.0303e-07, -1.5078e-06],\n", + " [-2.8755e-07, -2.2445e-06, -8.0466e-07]]],\n", + " \n", + " \n", + " [[[-2.3879e-06, -2.2701e-06, -9.4809e-07],\n", + " [-2.3656e-07, -6.1840e-07, -1.6708e-06],\n", + " [ 1.5316e-06, 2.4587e-06, -4.0233e-07]],\n", + " \n", + " [[ 1.4883e-06, 7.1991e-07, 4.5635e-07],\n", + " [ 2.4773e-07, -7.0408e-07, 2.4373e-06],\n", + " [ 1.0128e-07, 1.0291e-07, 2.0806e-06]],\n", + " \n", + " [[ 2.2622e-06, 2.5705e-07, -4.8988e-07],\n", + " [ 2.1104e-06, -2.2743e-06, 3.8464e-07],\n", + " [-3.8501e-06, -1.9502e-06, -1.0375e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.3560e-06, 1.4572e-06, -9.3528e-07],\n", + " [ 2.5779e-06, 9.0711e-07, -1.7043e-07],\n", + " [ 1.2387e-06, -1.3411e-07, 1.0864e-06]],\n", + " \n", + " [[-3.1479e-06, 2.8871e-07, 2.9197e-07],\n", + " [ 1.7229e-07, -5.1595e-07, 6.3702e-07],\n", + " [ 7.8045e-07, 1.8599e-06, 3.9558e-06]],\n", + " \n", + " [[-2.0117e-07, 1.1567e-06, 2.1909e-06],\n", + " [ 9.1083e-07, 1.3397e-06, -4.1351e-07],\n", + " [-9.0711e-07, -3.1628e-06, -2.3060e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 5.0478e-07, 2.3898e-06, 3.1013e-06],\n", + " [ 5.4340e-07, 2.3097e-07, 1.8235e-06],\n", + " [ 8.7987e-07, 1.2647e-06, 3.5305e-06]],\n", + " \n", + " [[ 1.9222e-06, 1.0282e-06, 5.4948e-07],\n", + " [ 1.0799e-06, 1.8217e-06, -1.2759e-07],\n", + " [ 4.8056e-07, 8.9779e-07, 7.1060e-07]],\n", + " \n", + " [[-9.6555e-07, 4.3772e-08, -6.3144e-07],\n", + " [-2.0200e-06, -4.2655e-07, -2.7381e-07],\n", + " [-5.6438e-07, -1.7178e-06, 4.3213e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1306e-06, 2.8070e-06, 1.0971e-06],\n", + " [ 2.2715e-06, 4.5449e-07, 2.7929e-06],\n", + " [ 6.0257e-07, 3.2345e-06, 3.9386e-06]],\n", + " \n", + " [[ 1.8943e-06, 1.2610e-06, 4.7777e-07],\n", + " [-1.2708e-06, -1.0971e-06, 1.0752e-06],\n", + " [-5.0850e-07, -2.9197e-06, -4.2655e-07]],\n", + " \n", + " [[-6.5804e-07, -1.7099e-06, -1.2871e-06],\n", + " [ 1.7509e-06, -2.1532e-06, -4.8708e-07],\n", + " [ 1.5008e-06, -8.8662e-07, 7.5344e-07]]],\n", + " \n", + " \n", + " [[[-6.4820e-07, -1.7136e-07, 3.1060e-07],\n", + " [-1.0151e-06, -8.1956e-08, 2.6170e-06],\n", + " [-1.3318e-06, -8.2422e-07, 1.6224e-06]],\n", + " \n", + " [[-5.6438e-07, 1.0938e-06, -1.3020e-06],\n", + " [-1.5167e-06, 4.1979e-07, -2.2158e-06],\n", + " [-6.5751e-07, -6.7055e-08, -2.1867e-06]],\n", + " \n", + " [[ 2.0750e-06, 1.0412e-06, -1.3839e-06],\n", + " [ 2.7334e-06, 2.9076e-06, -1.4063e-06],\n", + " [ 4.3653e-06, 3.5036e-06, -2.4121e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.3253e-06, -6.4075e-07, -3.8097e-07],\n", + " [ 2.1569e-06, 1.7346e-06, -7.0222e-07],\n", + " [ 1.2806e-06, -7.6555e-07, -3.7532e-07]],\n", + " \n", + " [[ 6.8918e-08, -1.6484e-06, -4.5700e-06],\n", + " [ 1.8589e-06, -2.8498e-06, -2.6142e-06],\n", + " [-5.8580e-07, -2.4866e-06, -3.8752e-06]],\n", + " \n", + " [[ 2.5444e-06, 7.1339e-07, -1.8440e-07],\n", + " [ 1.7388e-06, -8.7079e-07, 1.6373e-06],\n", + " [ 1.3169e-06, 2.7567e-07, -5.7556e-07]]],\n", + " \n", + " \n", + " [[[ 1.1427e-06, 2.1700e-06, 2.7027e-06],\n", + " [ 6.2957e-07, -7.4692e-07, 3.0883e-06],\n", + " [ 3.2410e-07, 2.7688e-06, 1.8831e-06]],\n", + " \n", + " [[ 1.0086e-06, 6.1840e-07, 1.3001e-06],\n", + " [-3.5577e-07, -7.1013e-07, 1.1241e-06],\n", + " [-6.8732e-07, -3.0296e-06, 1.6298e-06]],\n", + " \n", + " [[ 2.0759e-06, 3.9022e-07, 1.2827e-06],\n", + " [ 2.2352e-08, -8.7731e-07, 1.2331e-06],\n", + " [-4.2282e-07, -9.7416e-07, -2.5705e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.6280e-06, 3.1050e-06, 2.8405e-06],\n", + " [ 1.2685e-06, 7.7672e-07, 2.3125e-06],\n", + " [ 2.2021e-06, -4.9639e-07, 4.2832e-06]],\n", + " \n", + " [[-2.0508e-06, 6.3100e-07, 1.4380e-06],\n", + " [-2.2063e-06, 1.2754e-06, 2.7139e-06],\n", + " [ 1.3579e-06, 8.0094e-07, 2.4997e-06]],\n", + " \n", + " [[ 1.4678e-06, 2.4959e-07, -1.5739e-07],\n", + " [ 8.9873e-07, 5.2992e-07, -4.3213e-07],\n", + " [-6.1374e-07, -8.3819e-08, 1.2051e-06]]]], device='cuda:0')),\n", + " ('layer3.0.bn1.weight',\n", + " tensor([ 7.1526e-07, -1.1921e-06, -2.9802e-07, 2.3842e-07, 1.1921e-07,\n", + " 8.3447e-07, 5.9605e-07, -1.7881e-07, -2.3842e-07, 4.7684e-07,\n", + " -5.3644e-07, 7.1526e-07, -1.0729e-06, -1.0729e-06, -2.3842e-07,\n", + " 1.1921e-06, 0.0000e+00, -1.4901e-06, 3.5763e-07, 2.3842e-07,\n", + " 5.9605e-07, 1.4305e-06, 3.5763e-07, -7.1526e-07, 1.4305e-06,\n", + " 4.7684e-07, -4.1723e-07, 1.1921e-07, 1.1921e-06, -7.7486e-07,\n", + " -1.0729e-06, -1.1921e-06, -5.9605e-08, -1.7881e-07, -1.1921e-06,\n", + " -5.3644e-07, -8.9407e-07, -1.8477e-06, 8.3447e-07, 7.1526e-07,\n", + " -1.1921e-07, -4.7684e-07, -6.5565e-07, 8.3447e-07, 0.0000e+00,\n", + " 3.5763e-07, 1.4305e-06, 9.5367e-07, 5.9605e-07, 8.3447e-07,\n", + " 9.5367e-07, 1.1921e-07, -1.0729e-06, -1.1921e-07, 1.1921e-06,\n", + " 1.1921e-07, 1.1921e-06, 7.1526e-07, -8.3447e-07, 7.1526e-07,\n", + " 5.9605e-07, -4.7684e-07, -1.8477e-06, 1.0729e-06, 1.1921e-06,\n", + " 9.5367e-07, 4.7684e-07, 1.0729e-06, -8.3447e-07, 4.7684e-07,\n", + " -2.1458e-06, -1.5497e-06, 0.0000e+00, 3.5763e-07, 1.1921e-06,\n", + " -1.1325e-06, -4.1723e-07, 1.1921e-07, 4.7684e-07, -1.0729e-06,\n", + " 1.1921e-06, 1.0729e-06, 2.3842e-07, -1.1921e-07, 1.5497e-06,\n", + " -5.3644e-07, -2.3842e-07, 5.9605e-07, 4.7684e-07, -4.7684e-07,\n", + " 8.3447e-07, -5.9605e-08, -5.9605e-07, -4.7684e-07, -7.7486e-07,\n", + " 3.5763e-07, -1.1921e-06, 0.0000e+00, 5.9605e-07, -1.7881e-07,\n", + " -1.1921e-06, -6.5565e-07, -4.7684e-07, 0.0000e+00, 9.5367e-07,\n", + " 0.0000e+00, -5.9605e-07, -4.1723e-07, 7.1526e-07, -6.5565e-07,\n", + " -1.0729e-06, 8.3447e-07, 7.1526e-07, 7.1526e-07, 4.7684e-07,\n", + " 4.7684e-07, -8.9407e-07, -1.6093e-06, -4.1723e-07, 4.7684e-07,\n", + " 8.3447e-07, 1.1921e-07, 8.3447e-07, -3.5763e-07, 4.7684e-07,\n", + " 1.1921e-06, 1.1921e-07, -5.9605e-08, -1.4305e-06, -3.5763e-07,\n", + " -2.3842e-07, 1.1921e-07, -4.1723e-07, 1.9073e-06, 3.5763e-07,\n", + " -5.9605e-07, -4.1723e-07, -4.7684e-07, 4.7684e-07, -8.3447e-07,\n", + " 0.0000e+00, 1.1921e-07, -1.7881e-07, -4.1723e-07, -1.1921e-07,\n", + " -3.5763e-07, -8.3447e-07, -1.9670e-06, 4.7684e-07, -7.7486e-07,\n", + " -1.7881e-07, -2.4438e-06, 9.5367e-07, -3.5763e-07, -1.3709e-06,\n", + " -5.3644e-07, -6.5565e-07, 1.3113e-06, 7.1526e-07, -1.0729e-06,\n", + " 3.5763e-07, 9.5367e-07, -3.5763e-07, 4.7684e-07, 1.1921e-07,\n", + " -1.0133e-06, 2.3842e-07, -7.7486e-07, 2.3842e-07, -1.0133e-06,\n", + " 1.1921e-07, -1.0133e-06, 1.4305e-06, 1.1921e-06, -2.9802e-07,\n", + " 8.3447e-07, 3.5763e-07, -2.3842e-07, 1.1921e-07, -1.7881e-07,\n", + " 1.5497e-06, -5.9605e-07, -5.9605e-08, -5.9605e-08, 5.9605e-07,\n", + " -2.3842e-07, 1.1921e-07, -6.5565e-07, 7.1526e-07, 7.1526e-07,\n", + " -1.1921e-07, -1.1325e-06, 7.1526e-07, -4.1723e-07, 2.3842e-07,\n", + " -5.9605e-07, -1.3709e-06, 3.5763e-07, 4.7684e-07, 1.1921e-07,\n", + " 1.4305e-06, -2.3842e-07, -3.5763e-07, 9.5367e-07, 5.9605e-07,\n", + " 4.7684e-07, -9.5367e-07, -1.0729e-06, -4.7684e-07, 1.1921e-07,\n", + " -1.0133e-06, 1.0729e-06, -1.1921e-07, -2.9802e-07, -1.7881e-07,\n", + " 5.9605e-07, 5.9605e-07, -5.9605e-07, 9.5367e-07, 9.5367e-07,\n", + " 3.5763e-07, 9.5367e-07, 8.3447e-07, -7.7486e-07, 9.5367e-07,\n", + " 7.1526e-07, -1.0133e-06, 0.0000e+00, -5.9605e-07, 4.7684e-07,\n", + " -2.9802e-07, -1.1921e-07, -1.1921e-07, -7.7486e-07, -1.9670e-06,\n", + " 1.3113e-06, -5.9605e-08, -1.1325e-06, 1.1921e-07, -8.9407e-07,\n", + " -4.1723e-07, -4.1723e-07, 9.5367e-07, 2.3842e-07, 2.3842e-07,\n", + " -2.9802e-07, 1.0729e-06, -4.1723e-07, -4.1723e-07, 7.1526e-07,\n", + " -1.4305e-06, -1.1921e-07, -5.9605e-08, 1.9073e-06, 1.1921e-07,\n", + " -5.9605e-08], device='cuda:0')),\n", + " ('layer3.0.bn1.bias',\n", + " tensor([-5.6823e-07, -1.3981e-06, -5.8143e-07, -4.9844e-07, 3.0359e-07,\n", + " -6.4545e-07, 2.8438e-07, -4.5231e-07, -5.9321e-07, -4.6126e-08,\n", + " -6.9217e-07, 9.4221e-07, -1.1141e-06, -9.0594e-07, 2.9411e-07,\n", + " 8.6175e-07, -2.3770e-07, -1.5887e-06, 1.0951e-06, 1.2469e-06,\n", + " 1.2919e-06, 6.5640e-07, 9.5573e-07, -5.9106e-07, 4.3798e-07,\n", + " 8.4446e-07, -6.2349e-07, -1.8349e-07, 8.4023e-07, 1.1595e-06,\n", + " -5.6597e-07, -4.5581e-07, 2.9411e-07, 3.7765e-07, -5.3188e-07,\n", + " -4.4759e-07, 1.0989e-06, -7.0273e-07, 8.3838e-07, -1.4067e-09,\n", + " -4.8365e-08, -3.3166e-07, -1.3072e-06, 3.4386e-07, 6.9363e-07,\n", + " 7.2602e-08, 1.1142e-06, 5.8377e-07, 5.1305e-08, 3.5936e-07,\n", + " 3.3200e-08, -1.0829e-07, -5.9911e-07, 1.2177e-07, 2.4382e-07,\n", + " 1.9582e-07, 4.1886e-07, 2.5329e-07, -6.9319e-07, -4.8527e-07,\n", + " 4.5209e-07, -4.1583e-07, -1.0734e-06, 1.5452e-06, 5.3113e-07,\n", + " 1.1329e-06, 5.7745e-07, 9.0927e-07, 1.2965e-07, 1.1920e-06,\n", + " -1.2917e-06, -1.7132e-06, -7.9898e-07, -9.2034e-07, -1.8442e-07,\n", + " -5.5113e-07, 3.1931e-07, 9.5560e-07, 3.0754e-07, -1.7962e-07,\n", + " 2.0756e-07, 7.9398e-07, 2.0676e-07, -6.0307e-07, 1.2794e-06,\n", + " -1.0505e-06, -1.2550e-06, 1.4746e-07, 1.7321e-07, -2.9260e-07,\n", + " 5.8848e-07, 1.0114e-07, -8.7763e-08, -8.6689e-07, 1.7805e-08,\n", + " -7.3110e-07, -3.7368e-07, -4.9181e-07, 1.4132e-07, -7.5466e-07,\n", + " -1.0650e-06, -2.4837e-07, -7.2920e-07, -2.0560e-07, 7.5436e-07,\n", + " 2.4887e-07, -5.0478e-07, -1.1000e-07, 5.7459e-07, -9.5710e-07,\n", + " -1.9314e-07, 9.2695e-07, 1.0267e-06, 4.8308e-08, 7.6856e-07,\n", + " 9.2601e-07, -6.8997e-07, -5.1764e-07, -4.9746e-07, 2.0167e-08,\n", + " -3.0086e-07, -1.5695e-07, 4.7365e-07, 7.3920e-08, 2.1957e-07,\n", + " -1.7095e-07, -3.8337e-07, -5.4026e-09, -5.5063e-08, -2.7068e-07,\n", + " 2.8007e-07, 1.1806e-06, -1.2740e-07, 9.5515e-07, 4.3170e-07,\n", + " -6.3234e-07, -1.5391e-07, 1.8739e-07, 4.3259e-07, -4.1224e-07,\n", + " 1.3033e-07, 4.5940e-07, -4.5620e-07, -9.0492e-08, -4.6963e-07,\n", + " -3.5890e-07, -1.9067e-07, -1.1707e-06, 5.3332e-07, -7.1210e-07,\n", + " -7.5044e-07, -7.9760e-07, -2.2486e-07, -3.4558e-07, -2.8345e-07,\n", + " -2.7052e-07, -7.1833e-08, 1.9516e-06, -7.1396e-07, -2.0790e-07,\n", + " -1.0211e-08, 5.9483e-07, -5.8415e-07, -4.6524e-07, 1.1732e-07,\n", + " -3.6851e-07, 1.5508e-06, -1.3852e-07, 6.8619e-07, 1.9448e-07,\n", + " -8.8853e-07, -9.2429e-07, 6.2866e-07, 6.6644e-07, 1.0236e-08,\n", + " 1.3283e-06, 2.6774e-07, -1.3880e-06, -2.8537e-07, -2.9740e-07,\n", + " 1.2232e-06, -5.8244e-07, -2.9546e-07, -7.8462e-07, 1.0469e-06,\n", + " -1.4055e-06, -2.9223e-08, -8.9813e-07, 2.5954e-07, 5.1834e-07,\n", + " -7.1301e-08, -1.7958e-06, 6.8665e-07, -3.1676e-08, 4.7086e-08,\n", + " -3.7147e-07, -3.8185e-07, -6.4055e-07, 8.1996e-07, 8.1394e-07,\n", + " 1.1889e-06, -4.3364e-07, -3.6164e-07, 1.0069e-06, 1.0859e-06,\n", + " 1.0522e-06, -1.4343e-06, -1.1262e-06, -4.3589e-07, -1.4687e-06,\n", + " -1.1378e-06, 1.0889e-06, -8.0180e-07, 2.5179e-07, -8.2007e-08,\n", + " 6.6869e-07, 5.4022e-07, -1.0662e-06, 1.6516e-06, -1.4696e-07,\n", + " 8.2590e-08, 2.9557e-07, 9.4435e-07, -4.5838e-07, 3.2872e-07,\n", + " 1.1526e-06, -2.2169e-07, 2.3443e-07, 5.5351e-08, 5.1754e-08,\n", + " 2.7610e-07, -5.2608e-08, -8.8920e-07, -8.1640e-07, -1.1482e-06,\n", + " 1.3194e-06, -3.1359e-07, -1.2565e-06, 2.8645e-07, -1.3402e-06,\n", + " -2.6665e-08, 5.1468e-08, 2.1430e-07, -9.3682e-08, 2.3872e-07,\n", + " -7.2445e-07, 1.1124e-06, -5.9129e-07, -1.2152e-07, 8.0874e-07,\n", + " 1.2747e-07, -3.5549e-07, -6.1583e-07, 7.4368e-07, 2.9648e-07,\n", + " -8.8238e-07], device='cuda:0')),\n", + " ('layer3.0.conv2.weight',\n", + " tensor([[[[ 1.3057e-06, -2.6138e-06, 1.9912e-06],\n", + " [ 1.9046e-06, 1.1588e-06, 5.1130e-07],\n", + " [ 1.7826e-06, 1.3243e-06, 3.1153e-06]],\n", + " \n", + " [[-1.0901e-06, 2.5947e-06, 1.5767e-06],\n", + " [ 4.8853e-06, 3.7169e-06, 3.2913e-06],\n", + " [ 2.1281e-06, 2.4103e-06, 2.5703e-06]],\n", + " \n", + " [[ 5.8091e-07, -6.4028e-07, 4.1351e-07],\n", + " [ 2.3320e-06, 3.2922e-06, 1.8850e-06],\n", + " [ 2.3283e-06, 6.0885e-07, 1.3956e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.0098e-06, 1.6801e-06, 2.7046e-06],\n", + " [ 4.1922e-06, 4.2524e-06, 2.1830e-06],\n", + " [ 2.0792e-06, 2.4531e-06, 3.1613e-06]],\n", + " \n", + " [[ 9.7230e-07, 2.8196e-06, 6.6590e-07],\n", + " [ 2.5225e-06, 3.7211e-06, 2.7816e-06],\n", + " [-2.6450e-07, 1.2089e-06, 1.3248e-06]],\n", + " \n", + " [[-1.7732e-06, -3.1665e-08, -1.5106e-06],\n", + " [ 2.4301e-06, -1.0282e-06, 1.5609e-06],\n", + " [ 4.2096e-07, 1.3942e-06, 1.0987e-06]]],\n", + " \n", + " \n", + " [[[ 2.0228e-06, 1.1697e-06, 9.9489e-07],\n", + " [-1.8440e-07, 4.6566e-08, 4.1910e-08],\n", + " [ 8.5915e-07, 3.8370e-07, 3.8799e-06]],\n", + " \n", + " [[ 4.0047e-08, 1.1567e-06, -1.9008e-06],\n", + " [ 2.0806e-06, -6.4448e-07, -2.5351e-06],\n", + " [ 4.2282e-07, 2.2575e-06, -2.5406e-06]],\n", + " \n", + " [[ 2.5420e-06, 9.2760e-07, -8.0653e-07],\n", + " [ 1.1194e-06, -1.2890e-06, -1.6810e-06],\n", + " [ 2.6245e-06, -7.5251e-07, 1.3150e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.4110e-06, -1.4915e-06, -9.0525e-07],\n", + " [ 3.6077e-07, 6.1281e-07, -2.5015e-06],\n", + " [ 2.5220e-06, 1.0338e-06, -1.2182e-06]],\n", + " \n", + " [[ 4.1211e-07, 1.1772e-06, 2.7008e-07],\n", + " [ 2.6310e-06, 1.1353e-06, 1.1409e-06],\n", + " [ 6.2585e-07, 2.7374e-06, 2.7195e-07]],\n", + " \n", + " [[ 6.1002e-08, 1.1306e-06, -1.3048e-06],\n", + " [ 1.6908e-06, -1.6326e-06, -4.8539e-07],\n", + " [ 1.6429e-06, 2.6822e-06, -7.9628e-07]]],\n", + " \n", + " \n", + " [[[ 8.1398e-07, 2.9895e-07, 1.5097e-06],\n", + " [ 2.3581e-06, 3.0650e-06, 3.2019e-06],\n", + " [ 1.3504e-07, -1.1511e-06, -8.7373e-07]],\n", + " \n", + " [[-1.0077e-06, 1.7782e-06, 3.5483e-07],\n", + " [ 1.0589e-06, 1.5870e-06, 9.5740e-07],\n", + " [-9.8534e-07, 3.2410e-07, -1.3839e-06]],\n", + " \n", + " [[-1.3541e-06, -1.3374e-06, 2.5705e-07],\n", + " [-4.9639e-07, 7.8138e-07, 7.6881e-07],\n", + " [ 6.2515e-08, -1.2070e-06, -2.8368e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.6543e-07, -9.0431e-07, -5.1782e-07],\n", + " [-8.4285e-07, -1.1786e-06, 9.2760e-07],\n", + " [ 1.0291e-07, -2.4131e-06, -8.0001e-07]],\n", + " \n", + " [[ 5.3272e-07, 7.6741e-07, -1.5385e-06],\n", + " [-1.0151e-06, -8.0466e-07, 2.8685e-07],\n", + " [-1.2936e-06, -1.3478e-06, -7.0175e-07]],\n", + " \n", + " [[ 6.9477e-07, -3.8464e-07, -4.6566e-09],\n", + " [ 1.1316e-07, 6.7987e-08, 2.3232e-06],\n", + " [ 2.3935e-07, 3.9290e-07, -2.0061e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 1.9018e-06, 5.8487e-07, 1.3602e-06],\n", + " [ 1.2463e-06, 4.4145e-07, 9.3412e-07],\n", + " [ 3.9488e-07, 8.8476e-07, 2.7129e-06]],\n", + " \n", + " [[-2.0042e-06, 2.1264e-06, -1.4435e-06],\n", + " [ 9.9931e-07, 7.2829e-07, -1.8035e-06],\n", + " [-2.5611e-07, 5.6624e-07, -9.7742e-07]],\n", + " \n", + " [[ 4.4517e-07, 1.6938e-07, -1.4994e-06],\n", + " [-6.1467e-07, -5.4250e-07, 5.8324e-07],\n", + " [-9.5367e-07, -3.1362e-07, -2.3283e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.5874e-06, -5.1316e-07, 9.7416e-07],\n", + " [-1.8254e-07, -3.1437e-06, -1.0012e-06],\n", + " [-9.0152e-07, -1.1763e-06, -2.3171e-06]],\n", + " \n", + " [[ 1.0794e-06, 1.2871e-06, 8.8476e-07],\n", + " [ 1.0924e-06, -5.8953e-07, -1.1618e-07],\n", + " [ 2.2538e-06, -1.0384e-06, -1.1181e-06]],\n", + " \n", + " [[-1.5525e-06, -5.6718e-07, -3.1292e-07],\n", + " [-3.0547e-07, 6.4634e-07, 1.2238e-06],\n", + " [ 1.3532e-06, -2.0093e-07, 5.9605e-07]]],\n", + " \n", + " \n", + " [[[-2.0294e-06, 1.7039e-06, -4.8801e-07],\n", + " [-5.4808e-07, -1.6736e-06, -6.3330e-07],\n", + " [ 2.0573e-06, 1.9413e-06, -4.1164e-07]],\n", + " \n", + " [[ 6.5938e-07, 2.2445e-06, 1.6289e-06],\n", + " [ 2.7893e-07, 8.6799e-07, 2.7772e-06],\n", + " [-2.9756e-07, 1.1092e-06, 2.1541e-06]],\n", + " \n", + " [[ 5.6671e-07, 3.3563e-07, -1.7071e-06],\n", + " [-3.0734e-08, 6.7800e-07, 8.7963e-07],\n", + " [ 7.9721e-07, 4.6566e-08, -1.3597e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 7.5018e-07, 3.3262e-06, 2.9728e-06],\n", + " [ 7.2084e-07, 2.6328e-06, 2.7940e-08],\n", + " [-5.4762e-07, -2.0005e-06, -1.2787e-06]],\n", + " \n", + " [[ 8.8010e-07, -3.7812e-07, -1.5274e-06],\n", + " [ 1.7690e-06, -1.1227e-06, 9.1270e-07],\n", + " [-7.5437e-07, 6.6031e-07, -3.7253e-08]],\n", + " \n", + " [[ 4.0885e-07, 6.3283e-07, 6.4331e-07],\n", + " [-3.5763e-07, 9.0618e-07, 2.5611e-07],\n", + " [-1.5600e-06, -2.4587e-07, -1.3071e-06]]],\n", + " \n", + " \n", + " [[[-2.8191e-06, -1.8105e-06, -5.6392e-07],\n", + " [-1.3543e-06, 1.2156e-06, 2.4505e-07],\n", + " [-2.7474e-06, -1.3094e-06, -8.6147e-08]],\n", + " \n", + " [[-1.3690e-06, -2.5544e-06, -7.1153e-07],\n", + " [-3.3993e-07, -5.9186e-07, 1.5670e-07],\n", + " [ 9.4250e-07, -1.3281e-06, -2.0331e-06]],\n", + " \n", + " [[-1.7378e-06, -7.6758e-07, 2.8498e-07],\n", + " [ 8.6986e-07, -3.1674e-06, -9.3319e-07],\n", + " [-2.7036e-06, -8.8476e-08, 2.6077e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.4273e-07, -1.8748e-06, 1.0058e-06],\n", + " [-1.3680e-06, -1.5926e-06, -4.2096e-07],\n", + " [-1.4417e-06, 6.4727e-08, -6.8918e-07]],\n", + " \n", + " [[-7.8278e-07, -7.5623e-07, 5.7276e-07],\n", + " [-5.6624e-07, -1.4994e-06, -1.5888e-06],\n", + " [-1.0417e-06, 6.1467e-07, 4.5588e-07]],\n", + " \n", + " [[-1.6233e-06, -1.0841e-06, 5.7742e-07],\n", + " [-1.0729e-06, 9.9186e-07, -1.1884e-06],\n", + " [-3.8091e-07, -1.7569e-06, 1.3858e-06]]]], device='cuda:0')),\n", + " ('layer3.0.bn2.weight',\n", + " tensor([ 2.3842e-06, 0.0000e+00, -8.3447e-07, 8.3447e-07, 7.1526e-07,\n", + " -1.0133e-06, 5.9605e-07, 3.5763e-07, 2.3842e-07, 0.0000e+00,\n", + " 7.1526e-07, 7.1526e-07, 1.0729e-06, -1.1921e-07, -5.3644e-07,\n", + " 1.1921e-07, -5.3644e-07, 1.0729e-06, 1.5497e-06, 0.0000e+00,\n", + " 3.5763e-07, -1.2517e-06, -1.0729e-06, -1.7881e-07, 9.5367e-07,\n", + " 3.5763e-07, -8.9407e-07, -5.3644e-07, -1.9073e-06, 4.7684e-07,\n", + " 8.3447e-07, 7.1526e-07, -2.3842e-07, 7.1526e-07, 2.3842e-07,\n", + " -4.7684e-07, -7.7486e-07, 3.5763e-07, -1.7881e-07, -8.9407e-07,\n", + " -2.3842e-07, 1.0729e-06, -4.7684e-07, -1.1921e-07, -5.3644e-07,\n", + " 4.7684e-07, -1.5497e-06, -2.9802e-07, 5.9605e-07, -5.9605e-07,\n", + " -1.0133e-06, 8.3447e-07, 5.9605e-07, -1.7881e-07, -5.9605e-08,\n", + " -7.7486e-07, 2.0266e-06, -5.3644e-07, 5.9605e-07, -1.1921e-07,\n", + " 5.9605e-07, 1.6689e-06, 1.4305e-06, -2.9802e-07, 3.5763e-07,\n", + " -4.7684e-07, 2.3842e-07, -8.3447e-07, -6.5565e-07, -1.0133e-06,\n", + " 5.9605e-07, 1.1921e-07, 9.5367e-07, 8.3447e-07, 0.0000e+00,\n", + " 3.5763e-07, 1.1921e-07, 1.1921e-07, -1.0729e-06, 1.1921e-07,\n", + " 1.1921e-07, 1.1921e-07, 5.9605e-07, 7.1526e-07, 1.1921e-07,\n", + " -1.1325e-06, -5.9605e-08, 1.1921e-07, -4.7684e-07, 9.5367e-07,\n", + " -5.9605e-07, 4.7684e-07, -5.3644e-07, 0.0000e+00, -8.9407e-07,\n", + " -5.9605e-07, -1.9073e-06, -2.3842e-07, -4.1723e-07, -1.4901e-06,\n", + " 2.3842e-07, 4.7684e-07, 5.9605e-07, -1.6689e-06, 1.1921e-07,\n", + " -1.7881e-07, 0.0000e+00, 1.4305e-06, 1.3113e-06, -6.5565e-07,\n", + " -2.9802e-07, -1.0729e-06, -5.3644e-07, 5.9605e-07, -4.7684e-07,\n", + " -8.9407e-07, -1.1921e-07, -2.3842e-07, 2.3842e-07, -1.7881e-06,\n", + " 3.5763e-07, -2.3842e-07, 3.5763e-07, 2.3842e-07, -1.1921e-07,\n", + " -1.1325e-06, -6.5565e-07, 0.0000e+00, 3.5763e-07, 8.3447e-07,\n", + " -6.5565e-07, -6.5565e-07, -2.2650e-06, 2.3842e-07, 9.5367e-07,\n", + " 7.1526e-07, -7.1526e-07, -7.7486e-07, -3.5763e-07, 1.4305e-06,\n", + " 4.7684e-07, -1.8477e-06, 4.7684e-07, 8.3447e-07, 1.5497e-06,\n", + " -5.9605e-07, 8.3447e-07, -5.3644e-07, 8.3447e-07, -2.2054e-06,\n", + " 0.0000e+00, 0.0000e+00, -1.7881e-07, 4.7684e-07, -1.2517e-06,\n", + " 2.3842e-07, 4.7684e-07, 2.3842e-07, -1.1921e-07, -4.7684e-07,\n", + " 5.9605e-07, 3.5763e-07, -1.1921e-06, 1.3113e-06, -1.7881e-07,\n", + " 7.1526e-07, 8.3447e-07, -4.7684e-07, 3.5763e-07, -1.0133e-06,\n", + " -4.7684e-07, 5.9605e-07, 1.1921e-07, 1.1921e-07, 8.3447e-07,\n", + " 0.0000e+00, 0.0000e+00, -3.5763e-07, 1.1921e-07, 4.7684e-07,\n", + " 1.1921e-07, -6.5565e-07, 4.7684e-07, 0.0000e+00, 9.5367e-07,\n", + " 3.5763e-07, 1.3113e-06, 2.3842e-07, -5.9605e-08, 1.1921e-07,\n", + " 0.0000e+00, -5.3644e-07, -5.9605e-08, 8.3447e-07, 2.3842e-07,\n", + " -2.9802e-07, -5.9605e-08, 2.3842e-07, -6.5565e-07, -1.1325e-06,\n", + " 2.3842e-07, 5.9605e-07, -1.1325e-06, 1.3113e-06, 4.7684e-07,\n", + " 1.1921e-07, -2.3842e-07, -4.7684e-07, -5.3644e-07, 1.1921e-07,\n", + " 2.3842e-07, 7.1526e-07, 0.0000e+00, 2.3842e-07, -1.1921e-07,\n", + " 1.1921e-06, 1.5497e-06, 0.0000e+00, 1.1921e-07, 8.3447e-07,\n", + " -5.9605e-07, 2.3842e-07, 4.7684e-07, 3.5763e-07, -7.1526e-07,\n", + " 1.0729e-06, 3.5763e-07, 1.0729e-06, 8.3447e-07, -4.1723e-07,\n", + " -1.2517e-06, -9.5367e-07, 1.1921e-07, 8.3447e-07, -1.1921e-06,\n", + " -9.5367e-07, 1.4305e-06, 3.5763e-07, -1.7881e-07, 1.0729e-06,\n", + " -1.1921e-06, 3.5763e-07, 5.9605e-07, -2.3842e-07, 0.0000e+00,\n", + " 3.5763e-07, 7.1526e-07, -5.9605e-08, -5.9605e-08, -8.9407e-07,\n", + " 1.3113e-06, 8.3447e-07, 3.5763e-07, -2.9802e-07, -7.7486e-07,\n", + " 2.3842e-07], device='cuda:0')),\n", + " ('layer3.0.bn2.bias',\n", + " tensor([ 1.2397e-06, 6.3916e-07, -1.5730e-07, 7.7664e-07, 6.6675e-07,\n", + " 7.5460e-07, 3.2428e-07, -4.9460e-07, -7.5864e-09, -1.1655e-07,\n", + " 4.3920e-07, 5.2803e-07, 3.4970e-07, -5.1197e-07, -2.4610e-07,\n", + " 6.8364e-07, 2.1216e-08, 5.2061e-07, 1.2123e-06, -3.2477e-07,\n", + " -7.3147e-08, -3.9508e-07, 2.2369e-07, -5.3686e-07, 1.8602e-08,\n", + " 1.3056e-06, -9.8787e-07, 1.3210e-06, -1.8658e-06, 2.4755e-07,\n", + " 4.4678e-07, -5.5527e-07, -4.8989e-07, 6.6215e-07, 8.1737e-07,\n", + " 1.0583e-06, -8.2283e-08, 2.7286e-07, 9.8735e-08, -4.9545e-07,\n", + " -1.0303e-07, 5.3118e-07, -7.7630e-07, -2.2175e-07, -8.3009e-07,\n", + " -3.2777e-07, -4.4338e-07, -8.1797e-07, -3.8200e-07, -5.3417e-07,\n", + " 3.6573e-07, 1.2024e-06, 1.0901e-06, -8.4797e-07, 2.9635e-07,\n", + " -4.1304e-07, 9.2186e-07, -5.6299e-07, 4.1419e-07, 1.4131e-07,\n", + " -2.4994e-08, 1.0638e-06, 7.4731e-07, 3.4836e-07, -6.1722e-07,\n", + " -3.2058e-07, 4.1246e-07, -2.0439e-07, -7.5934e-07, 2.0549e-07,\n", + " -4.7065e-07, 2.8008e-07, 2.7104e-07, -7.3181e-07, 9.1271e-07,\n", + " 8.7888e-08, -1.5313e-07, -3.1190e-08, -4.8686e-07, -2.8495e-07,\n", + " 8.0396e-07, -5.6101e-07, -1.4228e-07, 3.0207e-07, -4.3237e-07,\n", + " 3.8904e-07, 2.6641e-07, 2.0103e-07, -5.5700e-07, 2.1933e-07,\n", + " -7.6924e-07, 2.4533e-07, -2.0562e-07, -6.3526e-07, -7.7532e-07,\n", + " -1.0781e-08, -6.0451e-08, 2.9524e-07, -2.9730e-07, -7.1229e-07,\n", + " -5.2639e-07, -1.0158e-06, -1.8155e-07, -9.5364e-07, -7.7976e-08,\n", + " 4.2866e-07, 1.0097e-06, -2.3944e-07, 4.3388e-07, -7.4387e-07,\n", + " 3.8048e-07, -7.9571e-07, -9.8154e-07, -4.4025e-07, 2.4142e-07,\n", + " 4.3612e-07, -1.4472e-07, -4.2033e-07, -1.6008e-07, 6.7188e-09,\n", + " 5.8529e-07, -2.9896e-07, 7.1821e-07, 5.8884e-07, -3.3722e-08,\n", + " -8.7960e-07, -1.7185e-07, 3.9410e-09, 2.6008e-07, -7.1072e-07,\n", + " -1.2491e-08, -5.7730e-07, -1.2584e-06, -4.9229e-07, 3.5214e-07,\n", + " 1.3187e-06, -5.0259e-07, 1.9477e-07, -2.3074e-07, 8.2709e-07,\n", + " 5.3013e-07, -1.0832e-06, 2.9348e-07, -2.3056e-07, 9.2496e-07,\n", + " -3.6641e-07, -2.7478e-07, -7.4791e-07, 9.0510e-07, -3.4336e-07,\n", + " 1.4417e-06, 4.0621e-07, 3.4944e-07, 1.1388e-06, -4.8168e-08,\n", + " -4.2881e-08, 1.2046e-07, 1.9531e-07, -4.9289e-07, -4.9626e-07,\n", + " -2.5901e-07, 5.8413e-07, -1.1101e-06, 3.8676e-07, -2.1239e-07,\n", + " 3.7457e-07, -3.3742e-07, 1.3505e-07, 2.8487e-07, -2.7694e-07,\n", + " 7.8956e-07, 5.4949e-07, 5.5654e-07, 5.7680e-07, 2.9649e-07,\n", + " 3.8036e-07, -4.4860e-07, -3.1570e-07, -9.8473e-07, -7.6646e-07,\n", + " -8.1532e-08, 1.9141e-07, 8.7107e-07, 5.5474e-09, -6.6521e-07,\n", + " -2.2067e-07, -9.4595e-08, 6.9143e-07, 1.0157e-07, -5.9210e-07,\n", + " 1.1940e-08, 4.6116e-07, 1.2312e-06, 1.6454e-07, 4.2485e-07,\n", + " -4.9396e-07, -3.4643e-07, 1.1768e-07, 4.5359e-09, -4.0335e-07,\n", + " 1.4072e-07, 2.7469e-07, 6.8554e-07, -8.4500e-08, 3.7428e-07,\n", + " 5.9899e-07, 4.1019e-07, -8.1789e-07, -5.4011e-07, -3.5031e-07,\n", + " -5.5258e-07, -3.5712e-07, 7.5301e-07, 5.9972e-07, 4.0490e-07,\n", + " 2.6592e-07, 4.8271e-07, 4.1183e-07, 7.3446e-07, -5.6122e-07,\n", + " -3.2023e-08, -6.2278e-08, 1.9076e-07, -8.7138e-07, 1.1447e-06,\n", + " 8.4598e-07, -2.0409e-07, 1.0238e-06, 6.4743e-07, 3.9771e-07,\n", + " -8.7026e-07, 2.1872e-07, -5.9180e-07, 7.0975e-07, -1.5564e-07,\n", + " 6.7422e-07, 1.3821e-06, 6.2665e-07, -4.7970e-07, -3.3708e-07,\n", + " -7.5254e-08, -6.1423e-07, 6.8763e-07, 1.4221e-07, 1.8319e-06,\n", + " 1.1302e-06, 3.3478e-07, -3.1522e-07, 1.8243e-07, -7.6020e-08,\n", + " 1.0345e-06, 3.5484e-07, 6.0152e-07, -3.3911e-07, -3.1310e-07,\n", + " -6.6216e-07], device='cuda:0')),\n", + " ('layer3.0.shortcut.0.weight',\n", + " tensor([[[[ 9.5367e-07]],\n", + " \n", + " [[-4.9733e-07]],\n", + " \n", + " [[-1.4445e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.9116e-07]],\n", + " \n", + " [[ 3.7253e-07]],\n", + " \n", + " [[ 1.4603e-06]]],\n", + " \n", + " \n", + " [[[ 2.1819e-06]],\n", + " \n", + " [[ 4.3772e-08]],\n", + " \n", + " [[ 8.0094e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.2787e-06]],\n", + " \n", + " [[ 3.2783e-07]],\n", + " \n", + " [[ 1.4249e-07]]],\n", + " \n", + " \n", + " [[[-6.7055e-08]],\n", + " \n", + " [[-8.5682e-08]],\n", + " \n", + " [[-3.5219e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.9954e-07]],\n", + " \n", + " [[-6.5379e-07]],\n", + " \n", + " [[ 4.9919e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 2.8312e-07]],\n", + " \n", + " [[ 1.6484e-06]],\n", + " \n", + " [[-1.3635e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-5.5949e-07]],\n", + " \n", + " [[ 1.7285e-06]],\n", + " \n", + " [[-1.2666e-06]]],\n", + " \n", + " \n", + " [[[-5.6624e-07]],\n", + " \n", + " [[ 9.1270e-07]],\n", + " \n", + " [[-9.5367e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.5705e-07]],\n", + " \n", + " [[ 2.5611e-07]],\n", + " \n", + " [[-3.6508e-07]]],\n", + " \n", + " \n", + " [[[-1.9297e-06]],\n", + " \n", + " [[ 3.3528e-08]],\n", + " \n", + " [[-5.2154e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.0319e-06]],\n", + " \n", + " [[-4.3213e-07]],\n", + " \n", + " [[-3.2857e-06]]]], device='cuda:0')),\n", + " ('layer3.0.shortcut.1.weight',\n", + " tensor([-5.9605e-07, 1.7881e-06, 8.3447e-07, 8.3447e-07, 5.9605e-07,\n", + " -2.9802e-07, -6.5565e-07, -7.7486e-07, -5.9605e-07, 0.0000e+00,\n", + " 0.0000e+00, 1.3113e-06, -1.3113e-06, -4.1723e-07, -2.9802e-07,\n", + " 3.5763e-07, -6.5565e-07, -8.9407e-07, 1.0729e-06, -8.3447e-07,\n", + " -2.3842e-07, -1.3113e-06, 0.0000e+00, 0.0000e+00, -1.1325e-06,\n", + " 9.5367e-07, 3.5763e-07, 1.3113e-06, -1.4305e-06, -5.3644e-07,\n", + " -5.9605e-08, -1.0133e-06, 0.0000e+00, -5.9605e-07, 7.1526e-07,\n", + " 1.0729e-06, 7.1526e-07, -1.1921e-07, -6.5565e-07, -1.7881e-07,\n", + " 4.7684e-07, 1.3113e-06, -9.5367e-07, -2.3842e-07, -1.1325e-06,\n", + " -2.9802e-07, 2.3842e-07, 4.7684e-07, -4.1723e-07, -4.1723e-07,\n", + " 4.7684e-07, 1.0729e-06, 7.1526e-07, -1.7285e-06, 2.3842e-07,\n", + " -6.5565e-07, 4.7684e-07, -1.0729e-06, 2.3842e-07, -4.1723e-07,\n", + " -8.3447e-07, 3.5763e-07, -8.3447e-07, 7.1526e-07, 0.0000e+00,\n", + " 8.3447e-07, 5.9605e-07, -4.7684e-07, -1.1325e-06, 1.3113e-06,\n", + " -5.9605e-07, -1.7881e-07, -1.0729e-06, -7.1526e-07, 1.1921e-06,\n", + " -1.7881e-06, -5.9605e-08, 0.0000e+00, -6.5565e-07, -9.5367e-07,\n", + " 3.5763e-07, -6.5565e-07, 3.5763e-07, -5.3644e-07, -2.3842e-07,\n", + " 3.5763e-07, -4.1723e-07, 3.5763e-07, 1.1921e-07, -1.7881e-07,\n", + " -3.5763e-07, 7.1526e-07, -2.9802e-07, 0.0000e+00, 2.3842e-07,\n", + " 3.5763e-07, 7.1526e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", + " -4.7684e-07, -9.5367e-07, -4.1723e-07, -8.9407e-07, -1.1921e-07,\n", + " -1.7285e-06, 3.5763e-07, -1.1325e-06, -3.5763e-07, -9.5367e-07,\n", + " 1.3113e-06, 3.5763e-07, -5.9605e-07, -1.9670e-06, 9.5367e-07,\n", + " 1.3113e-06, -1.0133e-06, 1.1921e-06, -2.9802e-07, 1.0729e-06,\n", + " 9.5367e-07, 1.1921e-06, 1.4305e-06, 5.9605e-07, 0.0000e+00,\n", + " -1.7881e-07, 9.5367e-07, -1.4305e-06, -1.1921e-07, -7.1526e-07,\n", + " 0.0000e+00, 1.1921e-07, 0.0000e+00, -2.3842e-07, -8.3447e-07,\n", + " 4.7684e-07, -3.5763e-07, 5.9605e-07, 0.0000e+00, 4.7684e-07,\n", + " -1.7881e-07, 8.3447e-07, -2.0862e-06, 0.0000e+00, -6.5565e-07,\n", + " 0.0000e+00, -4.7684e-07, -1.4901e-06, 5.9605e-07, 2.3842e-07,\n", + " 1.4305e-06, -1.1921e-07, -5.3644e-07, 8.3447e-07, 5.9605e-07,\n", + " -8.9407e-07, 9.5367e-07, -5.9605e-07, -4.7684e-07, 8.3447e-07,\n", + " -1.1325e-06, -1.7881e-07, -1.6689e-06, -7.1526e-07, 0.0000e+00,\n", + " 5.9605e-07, -9.5367e-07, 0.0000e+00, 1.0729e-06, 1.4305e-06,\n", + " 3.5763e-07, 4.7684e-07, -3.5763e-07, 5.9605e-07, -2.3842e-07,\n", + " 5.9605e-07, -1.1325e-06, -5.3644e-07, -8.9407e-07, -1.9670e-06,\n", + " 2.3842e-07, 5.9605e-07, -5.9605e-08, 2.3842e-07, -1.0729e-06,\n", + " -1.3113e-06, -1.0133e-06, 2.3842e-07, -1.0133e-06, -2.9802e-07,\n", + " -3.5763e-07, -1.1921e-07, 2.0266e-06, -5.9605e-07, 1.3113e-06,\n", + " -6.5565e-07, 0.0000e+00, 1.1921e-07, 1.3113e-06, 2.3842e-07,\n", + " -5.3644e-07, -2.9802e-07, 1.6689e-06, 3.5763e-07, 4.7684e-07,\n", + " 2.3842e-07, 7.1526e-07, -2.9802e-07, 7.1526e-07, 1.0729e-06,\n", + " -1.8477e-06, 2.3842e-07, 1.1921e-07, 7.1526e-07, 0.0000e+00,\n", + " 0.0000e+00, -1.4305e-06, 5.9605e-07, -7.1526e-07, -5.3644e-07,\n", + " 5.9605e-07, 3.5763e-07, 1.1921e-07, -1.0729e-06, 2.3842e-07,\n", + " -1.7881e-07, -6.5565e-07, 1.1921e-06, -5.3644e-07, 1.3113e-06,\n", + " 3.5763e-07, 5.9605e-07, -1.0133e-06, 8.3447e-07, 1.0729e-06,\n", + " 7.1526e-07, 7.1526e-07, 8.3447e-07, -5.9605e-08, -1.0133e-06,\n", + " 2.3842e-07, -2.0266e-06, 8.3447e-07, 1.1921e-07, 1.1921e-06,\n", + " 3.5763e-07, -1.0729e-06, 2.3842e-07, 0.0000e+00, 7.1526e-07,\n", + " -1.1921e-07, 8.3447e-07, 1.1921e-07, -1.1921e-07, 2.3842e-07,\n", + " -1.0133e-06], device='cuda:0')),\n", + " ('layer3.0.shortcut.1.bias',\n", + " tensor([ 1.2397e-06, 6.3916e-07, -1.5730e-07, 7.7664e-07, 6.6675e-07,\n", + " 7.5460e-07, 3.2428e-07, -4.9460e-07, -7.5864e-09, -1.1655e-07,\n", + " 4.3920e-07, 5.2803e-07, 3.4970e-07, -5.1197e-07, -2.4610e-07,\n", + " 6.8364e-07, 2.1216e-08, 5.2061e-07, 1.2123e-06, -3.2477e-07,\n", + " -7.3147e-08, -3.9508e-07, 2.2369e-07, -5.3686e-07, 1.8602e-08,\n", + " 1.3056e-06, -9.8787e-07, 1.3210e-06, -1.8658e-06, 2.4755e-07,\n", + " 4.4678e-07, -5.5527e-07, -4.8989e-07, 6.6215e-07, 8.1737e-07,\n", + " 1.0583e-06, -8.2283e-08, 2.7286e-07, 9.8735e-08, -4.9545e-07,\n", + " -1.0303e-07, 5.3118e-07, -7.7630e-07, -2.2175e-07, -8.3009e-07,\n", + " -3.2777e-07, -4.4338e-07, -8.1797e-07, -3.8200e-07, -5.3417e-07,\n", + " 3.6573e-07, 1.2024e-06, 1.0901e-06, -8.4797e-07, 2.9635e-07,\n", + " -4.1304e-07, 9.2186e-07, -5.6299e-07, 4.1419e-07, 1.4131e-07,\n", + " -2.4994e-08, 1.0638e-06, 7.4731e-07, 3.4836e-07, -6.1722e-07,\n", + " -3.2058e-07, 4.1246e-07, -2.0439e-07, -7.5934e-07, 2.0549e-07,\n", + " -4.7065e-07, 2.8008e-07, 2.7104e-07, -7.3181e-07, 9.1271e-07,\n", + " 8.7888e-08, -1.5313e-07, -3.1190e-08, -4.8686e-07, -2.8495e-07,\n", + " 8.0396e-07, -5.6101e-07, -1.4228e-07, 3.0207e-07, -4.3237e-07,\n", + " 3.8904e-07, 2.6641e-07, 2.0103e-07, -5.5700e-07, 2.1933e-07,\n", + " -7.6924e-07, 2.4533e-07, -2.0562e-07, -6.3526e-07, -7.7532e-07,\n", + " -1.0781e-08, -6.0451e-08, 2.9524e-07, -2.9730e-07, -7.1229e-07,\n", + " -5.2639e-07, -1.0158e-06, -1.8155e-07, -9.5364e-07, -7.7976e-08,\n", + " 4.2866e-07, 1.0097e-06, -2.3944e-07, 4.3388e-07, -7.4387e-07,\n", + " 3.8048e-07, -7.9571e-07, -9.8154e-07, -4.4025e-07, 2.4142e-07,\n", + " 4.3612e-07, -1.4472e-07, -4.2033e-07, -1.6008e-07, 6.7188e-09,\n", + " 5.8529e-07, -2.9896e-07, 7.1821e-07, 5.8884e-07, -3.3722e-08,\n", + " -8.7960e-07, -1.7185e-07, 3.9410e-09, 2.6008e-07, -7.1072e-07,\n", + " -1.2491e-08, -5.7730e-07, -1.2584e-06, -4.9229e-07, 3.5214e-07,\n", + " 1.3187e-06, -5.0259e-07, 1.9477e-07, -2.3074e-07, 8.2709e-07,\n", + " 5.3013e-07, -1.0832e-06, 2.9348e-07, -2.3056e-07, 9.2496e-07,\n", + " -3.6641e-07, -2.7478e-07, -7.4791e-07, 9.0510e-07, -3.4336e-07,\n", + " 1.4417e-06, 4.0621e-07, 3.4944e-07, 1.1388e-06, -4.8168e-08,\n", + " -4.2881e-08, 1.2046e-07, 1.9531e-07, -4.9289e-07, -4.9626e-07,\n", + " -2.5901e-07, 5.8413e-07, -1.1101e-06, 3.8676e-07, -2.1239e-07,\n", + " 3.7457e-07, -3.3742e-07, 1.3505e-07, 2.8487e-07, -2.7694e-07,\n", + " 7.8956e-07, 5.4949e-07, 5.5654e-07, 5.7680e-07, 2.9649e-07,\n", + " 3.8036e-07, -4.4860e-07, -3.1570e-07, -9.8473e-07, -7.6646e-07,\n", + " -8.1532e-08, 1.9141e-07, 8.7107e-07, 5.5474e-09, -6.6521e-07,\n", + " -2.2067e-07, -9.4595e-08, 6.9143e-07, 1.0157e-07, -5.9210e-07,\n", + " 1.1940e-08, 4.6116e-07, 1.2312e-06, 1.6454e-07, 4.2485e-07,\n", + " -4.9396e-07, -3.4643e-07, 1.1768e-07, 4.5359e-09, -4.0335e-07,\n", + " 1.4072e-07, 2.7469e-07, 6.8554e-07, -8.4500e-08, 3.7428e-07,\n", + " 5.9899e-07, 4.1019e-07, -8.1789e-07, -5.4011e-07, -3.5031e-07,\n", + " -5.5258e-07, -3.5712e-07, 7.5301e-07, 5.9972e-07, 4.0490e-07,\n", + " 2.6592e-07, 4.8271e-07, 4.1183e-07, 7.3446e-07, -5.6122e-07,\n", + " -3.2023e-08, -6.2278e-08, 1.9076e-07, -8.7138e-07, 1.1447e-06,\n", + " 8.4598e-07, -2.0409e-07, 1.0238e-06, 6.4743e-07, 3.9771e-07,\n", + " -8.7026e-07, 2.1872e-07, -5.9180e-07, 7.0975e-07, -1.5564e-07,\n", + " 6.7422e-07, 1.3821e-06, 6.2665e-07, -4.7970e-07, -3.3708e-07,\n", + " -7.5254e-08, -6.1423e-07, 6.8763e-07, 1.4221e-07, 1.8319e-06,\n", + " 1.1302e-06, 3.3478e-07, -3.1522e-07, 1.8243e-07, -7.6020e-08,\n", + " 1.0345e-06, 3.5484e-07, 6.0152e-07, -3.3911e-07, -3.1310e-07,\n", + " -6.6216e-07], device='cuda:0')),\n", + " ('layer4.0.conv1.weight',\n", + " tensor([[[[-6.5658e-07, -3.7719e-08, -2.9663e-07],\n", + " [-4.8429e-08, -2.5332e-07, -1.2014e-07],\n", + " [ 4.7125e-07, -5.1968e-07, 5.4017e-07]],\n", + " \n", + " [[ 2.0396e-06, -3.2212e-07, 2.5299e-06],\n", + " [-1.3877e-07, 7.3831e-07, 1.0414e-06],\n", + " [ 5.2899e-07, 1.7328e-07, 1.5325e-06]],\n", + " \n", + " [[-4.4890e-07, 6.5938e-07, -9.2853e-07],\n", + " [-1.3839e-06, -1.4184e-06, -1.1837e-06],\n", + " [-1.3523e-06, -7.1991e-07, -1.0827e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.0920e-07, 2.1793e-07, 7.3388e-07],\n", + " [ 1.1707e-06, -3.0018e-07, 1.1865e-06],\n", + " [ 4.1351e-07, -2.1013e-07, 1.1569e-06]],\n", + " \n", + " [[ 2.6543e-07, -1.2666e-07, -6.8825e-07],\n", + " [-1.1735e-07, 1.9097e-06, 6.3935e-07],\n", + " [-6.5679e-07, 4.5449e-07, 2.9802e-08]],\n", + " \n", + " [[ 1.4207e-06, -8.2888e-07, 1.7975e-06],\n", + " [ 1.8091e-06, 4.5635e-07, 1.3406e-06],\n", + " [ 1.6857e-06, -3.1013e-07, 8.6613e-07]]],\n", + " \n", + " \n", + " [[[ 2.2002e-07, 9.5833e-07, 4.0652e-07],\n", + " [-5.3830e-07, -4.3027e-07, 2.3842e-07],\n", + " [ 9.0199e-07, 8.8662e-07, -9.8790e-07]],\n", + " \n", + " [[ 5.3830e-07, 1.1232e-06, 3.0827e-07],\n", + " [ 5.1875e-07, 9.0897e-07, 1.1828e-06],\n", + " [-4.0978e-07, 5.5810e-07, -4.8988e-07]],\n", + " \n", + " [[ 4.9081e-07, -1.2154e-07, 3.5390e-08],\n", + " [-2.1048e-07, -5.1782e-07, 4.5868e-07],\n", + " [ 2.3190e-07, 2.6426e-07, -5.2270e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.9989e-07, -4.8336e-07, 1.5208e-06],\n", + " [-6.8266e-07, -8.3214e-07, -7.0082e-07],\n", + " [ 3.0920e-07, -1.4110e-07, 5.4017e-07]],\n", + " \n", + " [[ 6.5123e-07, 1.0990e-07, 3.7439e-07],\n", + " [ 1.2461e-06, 3.9581e-07, 1.0966e-07],\n", + " [-8.0967e-07, 1.8561e-06, 7.7463e-07]],\n", + " \n", + " [[-3.7160e-07, -2.9057e-07, -4.2841e-07],\n", + " [-5.7276e-07, 4.5635e-08, -9.2760e-07],\n", + " [-4.7637e-07, 3.2876e-07, 5.8860e-07]]],\n", + " \n", + " \n", + " [[[-9.1828e-07, 1.2137e-06, -2.3949e-06],\n", + " [-1.2591e-06, -4.2468e-07, -1.0675e-07],\n", + " [-1.5423e-06, 3.9022e-07, -2.0843e-06]],\n", + " \n", + " [[ 6.0163e-07, 1.4980e-06, 2.7362e-06],\n", + " [ 1.5371e-06, 1.3086e-06, 3.1629e-06],\n", + " [ 9.4902e-07, 2.4084e-06, 2.9653e-06]],\n", + " \n", + " [[ 7.2224e-07, 2.5891e-07, 1.6082e-06],\n", + " [-5.0175e-08, -5.1875e-07, -2.2561e-07],\n", + " [ 8.0466e-07, 2.0210e-07, 1.5367e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.1735e-07, -1.5814e-06, 4.2748e-07],\n", + " [ 4.9824e-07, -1.3281e-06, 1.1828e-06],\n", + " [ 1.1455e-07, 9.7649e-07, 1.0896e-07]],\n", + " \n", + " [[ 3.3528e-07, -1.2049e-06, 7.2643e-07],\n", + " [ 2.1793e-07, 1.5032e-06, -4.7963e-07],\n", + " [ 2.7940e-08, 5.8580e-07, 1.3281e-06]],\n", + " \n", + " [[ 1.9185e-07, 2.9057e-07, 4.7404e-07],\n", + " [-8.3633e-07, -1.9590e-06, -1.0915e-06],\n", + " [ 2.5146e-07, 4.7730e-07, 1.0227e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-7.5810e-07, -1.2480e-07, -1.9427e-06],\n", + " [-1.5055e-06, -1.2676e-06, -2.9870e-06],\n", + " [-1.4789e-06, -3.5646e-07, -1.4715e-06]],\n", + " \n", + " [[-8.5495e-07, 3.9279e-07, -6.1467e-07],\n", + " [ 2.4051e-07, -1.4603e-06, -8.8196e-07],\n", + " [ 4.9779e-07, 1.4668e-07, 1.8626e-07]],\n", + " \n", + " [[ 4.3493e-07, -1.2247e-06, -4.5542e-07],\n", + " [ 3.5041e-07, -2.5146e-08, -4.5635e-08],\n", + " [-4.7311e-07, 1.9651e-07, -2.6077e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.9430e-07, 1.9930e-07, -1.0431e-07],\n", + " [-5.9605e-07, 1.8850e-06, 2.1327e-07],\n", + " [-7.6077e-07, 1.2624e-06, -3.9116e-08]],\n", + " \n", + " [[-3.6974e-07, -6.9849e-07, -8.7731e-07],\n", + " [ 8.5682e-07, -9.7789e-07, -2.1234e-07],\n", + " [-7.4506e-07, -1.7444e-06, -3.2596e-07]],\n", + " \n", + " [[ 2.1625e-06, 5.9232e-07, -2.6077e-08],\n", + " [ 1.7248e-06, 1.7649e-07, 1.8496e-06],\n", + " [ 1.3439e-06, 1.5330e-06, 6.8173e-07]]],\n", + " \n", + " \n", + " [[[ 1.6000e-06, 5.0478e-07, 1.2908e-06],\n", + " [ 1.5376e-06, 1.5721e-06, 1.4119e-06],\n", + " [ 1.5273e-06, 2.9244e-07, 1.4193e-06]],\n", + " \n", + " [[ 8.0373e-07, 1.6098e-06, 2.6496e-07],\n", + " [ 7.0967e-07, -2.9593e-07, -9.0525e-07],\n", + " [ 7.0618e-07, -9.5926e-08, -4.2003e-07]],\n", + " \n", + " [[-4.3213e-07, 2.5705e-07, -4.4983e-07],\n", + " [-4.2096e-07, -4.0606e-07, -1.0002e-06],\n", + " [-3.9022e-07, 9.8720e-07, -2.5984e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 8.2701e-07, 1.3579e-06, -9.7789e-07],\n", + " [ 2.5146e-08, -5.1904e-07, -5.4017e-07],\n", + " [ 3.4831e-07, -8.9407e-07, -2.9244e-07]],\n", + " \n", + " [[ 5.7090e-07, 9.4157e-07, 1.5087e-07],\n", + " [ 3.1199e-07, -6.3889e-07, 4.5542e-07],\n", + " [-1.7975e-07, -2.3581e-06, -6.7893e-07]],\n", + " \n", + " [[ 3.1851e-07, -5.7835e-07, -5.1921e-08],\n", + " [-6.1467e-08, -8.4843e-07, -1.4808e-07],\n", + " [ 4.3213e-07, -1.0869e-06, -9.1176e-07]]],\n", + " \n", + " \n", + " [[[-9.7789e-08, -1.2685e-06, -3.0734e-07],\n", + " [-4.3772e-07, -7.9021e-07, -1.9558e-07],\n", + " [-1.3858e-06, -1.0114e-06, -4.6566e-09]],\n", + " \n", + " [[-1.9465e-07, 1.3970e-07, -2.5518e-07],\n", + " [-1.2610e-06, 5.5321e-07, -1.4156e-07],\n", + " [ 7.4506e-09, -1.0431e-07, -9.9780e-07]],\n", + " \n", + " [[-1.8813e-07, -9.2754e-08, -1.7043e-07],\n", + " [-8.9034e-07, 2.3097e-07, -4.9919e-07],\n", + " [-1.7090e-06, 3.1479e-07, 3.8277e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.5297e-07, -4.0699e-07, -7.6136e-07],\n", + " [ 3.0734e-07, -1.9372e-07, -6.2305e-07],\n", + " [-3.4645e-07, 2.3376e-06, -1.1618e-07]],\n", + " \n", + " [[ 9.9558e-07, 7.0501e-07, 4.0699e-07],\n", + " [-6.5751e-07, -8.3912e-07, -1.0272e-06],\n", + " [-6.4448e-07, -2.7008e-07, -7.0781e-07]],\n", + " \n", + " [[-3.0873e-07, -3.8370e-07, -1.4324e-06],\n", + " [ 2.2273e-06, 5.0152e-07, 1.1120e-06],\n", + " [ 3.8836e-07, 5.5414e-07, 1.1791e-06]]]], device='cuda:0')),\n", + " ('layer4.0.bn1.weight',\n", + " tensor([ 4.7684e-07, 1.1921e-07, 4.7684e-07, 2.3842e-07, 1.1921e-07,\n", + " 4.7684e-07, -1.1921e-07, 0.0000e+00, -1.7881e-07, 4.7684e-07,\n", + " 1.1921e-07, -2.9802e-07, 2.3842e-07, 3.5763e-07, -4.1723e-07,\n", + " 7.1526e-07, 4.7684e-07, -2.9802e-07, 0.0000e+00, 3.5763e-07,\n", + " 4.7684e-07, -1.7881e-07, -1.7881e-07, 4.7684e-07, -2.3842e-07,\n", + " -5.9605e-08, -5.9605e-08, -5.9605e-08, -5.3644e-07, -1.7881e-07,\n", + " 4.7684e-07, -3.5763e-07, 0.0000e+00, -5.3644e-07, 0.0000e+00,\n", + " -1.7881e-07, 1.1921e-07, 1.3113e-06, -1.1921e-07, -2.9802e-07,\n", + " -7.7486e-07, 2.3842e-07, -7.1526e-07, 5.9605e-07, -8.3447e-07,\n", + " 4.7684e-07, 0.0000e+00, -4.7684e-07, -4.7684e-07, -2.9802e-07,\n", + " -7.1526e-07, 1.1921e-07, -5.3644e-07, 7.1526e-07, -4.1723e-07,\n", + " 3.5763e-07, -4.7684e-07, 1.1921e-07, 0.0000e+00, 3.5763e-07,\n", + " -7.7486e-07, -5.9605e-08, 1.5497e-06, -6.5565e-07, -2.3842e-07,\n", + " 9.5367e-07, 0.0000e+00, -5.9605e-08, 3.5763e-07, -2.3842e-07,\n", + " -8.3447e-07, 1.1921e-07, -1.1921e-07, 7.1526e-07, -2.9802e-07,\n", + " -1.0133e-06, 5.9605e-07, -2.9802e-07, 1.1921e-07, 3.5763e-07,\n", + " -3.5763e-07, 4.7684e-07, -2.3842e-07, 1.1921e-07, 1.1921e-07,\n", + " 0.0000e+00, -4.1723e-07, -1.1921e-07, 2.3842e-07, 3.5763e-07,\n", + " 0.0000e+00, -4.7684e-07, 2.3842e-07, 1.1921e-07, -2.9802e-07,\n", + " 8.3447e-07, -1.1921e-07, -1.7881e-07, -2.3842e-07, -2.3842e-07,\n", + " 7.1526e-07, -2.3842e-07, -2.9802e-07, 4.7684e-07, 4.7684e-07,\n", + " -5.9605e-08, -3.5763e-07, 5.9605e-07, 1.1921e-07, -3.5763e-07,\n", + " -3.5763e-07, -5.9605e-07, 0.0000e+00, -2.9802e-07, 9.5367e-07,\n", + " 2.3842e-07, 2.3842e-07, 2.3842e-07, -1.1921e-07, -7.7486e-07,\n", + " -2.3842e-07, -1.7881e-07, -2.9802e-07, -3.5763e-07, -5.9605e-07,\n", + " -6.5565e-07, -2.9802e-07, 4.7684e-07, -5.9605e-07, 1.1921e-07,\n", + " 1.1921e-07, 2.3842e-07, 0.0000e+00, 2.3842e-07, 4.7684e-07,\n", + " -1.7881e-07, 3.5763e-07, -4.7684e-07, -1.7881e-07, -9.5367e-07,\n", + " -2.3842e-07, -1.7881e-07, 3.5763e-07, 4.7684e-07, 3.5763e-07,\n", + " 1.0729e-06, 2.3842e-07, -4.7684e-07, 2.3842e-07, 1.1921e-07,\n", + " -2.9802e-07, -1.1921e-07, -7.7486e-07, -1.1921e-07, 1.1921e-07,\n", + " -2.3842e-07, -1.7881e-07, 1.1921e-07, 2.3842e-07, -5.9605e-08,\n", + " 9.5367e-07, 2.3842e-07, 7.1526e-07, -1.1921e-07, -1.1921e-07,\n", + " -7.1526e-07, 5.9605e-07, -1.7881e-07, -2.9802e-07, 8.3447e-07,\n", + " -4.7684e-07, 1.1921e-07, -6.5565e-07, 1.1921e-07, 5.9605e-07,\n", + " -2.3842e-07, 0.0000e+00, 0.0000e+00, 4.7684e-07, 1.1921e-07,\n", + " 1.1921e-07, -6.5565e-07, 9.5367e-07, 5.9605e-07, 1.1921e-07,\n", + " -5.9605e-08, -6.5565e-07, -3.5763e-07, -1.1921e-07, -6.5565e-07,\n", + " 5.9605e-07, -5.3644e-07, -8.9407e-07, 3.5763e-07, -8.3447e-07,\n", + " 5.9605e-07, 4.7684e-07, 3.5763e-07, 8.3447e-07, 0.0000e+00,\n", + " 1.1921e-07, 0.0000e+00, -4.1723e-07, 2.3842e-07, 1.1921e-07,\n", + " 4.7684e-07, -1.7881e-07, -4.7684e-07, -5.9605e-08, -2.9802e-07,\n", + " 2.3842e-07, 1.1921e-07, 2.3842e-07, -2.9802e-07, -1.7881e-07,\n", + " -3.5763e-07, 0.0000e+00, -1.7881e-07, -7.1526e-07, -2.9802e-07,\n", + " 5.9605e-07, 2.3842e-07, 1.1921e-07, -2.9802e-07, 2.3842e-07,\n", + " 1.1921e-07, 3.5763e-07, 1.1921e-06, 2.3842e-07, -2.3842e-07,\n", + " -4.1723e-07, 2.3842e-07, 1.1921e-07, -1.7881e-07, -6.5565e-07,\n", + " -1.0729e-06, -5.9605e-08, -2.3842e-07, 0.0000e+00, 4.7684e-07,\n", + " -5.9605e-08, 4.7684e-07, 3.5763e-07, -4.1723e-07, 0.0000e+00,\n", + " -6.5565e-07, 1.1921e-07, 1.1921e-07, 0.0000e+00, -5.3644e-07,\n", + " 0.0000e+00, 1.1921e-07, 1.1921e-07, 9.5367e-07, 1.1921e-07,\n", + " 3.5763e-07, -1.2517e-06, 1.1921e-07, -4.7684e-07, 1.1921e-06,\n", + " -3.5763e-07, 1.1921e-07, 1.1921e-07, -7.1526e-07, -8.9407e-07,\n", + " 1.1921e-07, 3.5763e-07, 0.0000e+00, -4.1723e-07, 2.3842e-07,\n", + " -8.3447e-07, 3.5763e-07, 1.1921e-07, -3.5763e-07, -5.3644e-07,\n", + " 3.5763e-07, 0.0000e+00, -2.9802e-07, -5.9605e-08, 3.5763e-07,\n", + " -1.7881e-07, 4.7684e-07, -2.9802e-07, 7.1526e-07, -5.3644e-07,\n", + " -7.7486e-07, 1.3113e-06, 1.1921e-07, -1.7881e-07, -3.5763e-07,\n", + " -1.1921e-07, -1.1921e-07, -2.9802e-07, 2.3842e-07, 4.7684e-07,\n", + " -5.3644e-07, -2.3842e-07, 4.7684e-07, 3.5763e-07, 1.1921e-07,\n", + " 7.1526e-07, -1.7881e-07, 5.9605e-07, -5.3644e-07, -1.7881e-07,\n", + " 9.5367e-07, -4.7684e-07, -5.9605e-08, 3.5763e-07, -4.7684e-07,\n", + " -7.7486e-07, -1.7881e-07, 1.1921e-07, -5.9605e-08, 0.0000e+00,\n", + " 2.3842e-07, 0.0000e+00, -5.3644e-07, 8.3447e-07, 0.0000e+00,\n", + " 3.5763e-07, -2.9802e-07, -5.9605e-08, 0.0000e+00, 4.7684e-07,\n", + " -2.9802e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07, 0.0000e+00,\n", + " -2.9802e-07, -4.7684e-07, 2.3842e-07, 0.0000e+00, -5.3644e-07,\n", + " -3.5763e-07, -5.9605e-08, 4.7684e-07, 1.0729e-06, -1.7881e-07,\n", + " 3.5763e-07, 7.1526e-07, 1.1921e-07, 9.5367e-07, 4.7684e-07,\n", + " 8.3447e-07, -4.1723e-07, 2.3842e-07, -3.5763e-07, -5.3644e-07,\n", + " 0.0000e+00, -1.7881e-07, 2.3842e-07, 0.0000e+00, 3.5763e-07,\n", + " -8.9407e-07, 1.1921e-07, 1.1921e-07, 7.1526e-07, 4.7684e-07,\n", + " -3.5763e-07, 4.7684e-07, -5.3644e-07, 3.5763e-07, -4.7684e-07,\n", + " 1.1921e-07, -2.9802e-07, 1.1921e-07, 7.1526e-07, 0.0000e+00,\n", + " -1.7881e-07, 4.7684e-07, -9.5367e-07, -1.1325e-06, -1.7881e-07,\n", + " -5.9605e-07, -1.1921e-07, -2.3842e-07, -1.7881e-07, 3.5763e-07,\n", + " 1.1921e-07, -7.1526e-07, -2.3842e-07, 2.3842e-07, -4.7684e-07,\n", + " -1.1921e-07, -1.7881e-07, 1.1921e-07, 4.7684e-07, -2.3842e-07,\n", + " -5.3644e-07, -5.9605e-08, -1.7881e-07, 1.0729e-06, 2.3842e-07,\n", + " 1.1921e-07, 1.4305e-06, 1.3113e-06, 1.0729e-06, 3.5763e-07,\n", + " 0.0000e+00, -4.1723e-07, -5.9605e-07, -1.7881e-07, 1.1921e-07,\n", + " -7.1526e-07, 0.0000e+00, 2.3842e-07, -7.1526e-07, 1.1921e-07,\n", + " 4.7684e-07, -4.1723e-07, -4.1723e-07, -1.7881e-07, -5.3644e-07,\n", + " 2.3842e-07, 1.1921e-07, -1.7881e-07, 1.1921e-07, 1.0729e-06,\n", + " -2.9802e-07, -2.9802e-07, 2.3842e-07, 1.4305e-06, 4.7684e-07,\n", + " -4.1723e-07, -5.9605e-08, 0.0000e+00, 0.0000e+00, 1.1921e-07,\n", + " 1.1921e-07, 0.0000e+00, -3.5763e-07, -4.1723e-07, -6.5565e-07,\n", + " -3.5763e-07, -2.9802e-07, 0.0000e+00, -2.9802e-07, 2.3842e-07,\n", + " -7.7486e-07, 7.1526e-07, -3.5763e-07, 3.5763e-07, -6.5565e-07,\n", + " 2.3842e-07, -1.7881e-07, -3.5763e-07, 2.3842e-07, 1.1921e-07,\n", + " 1.1921e-07, 2.3842e-07, -1.7881e-07, -4.1723e-07, -1.1921e-07,\n", + " -5.9605e-07, 3.5763e-07, -2.3842e-07, 2.3842e-07, -1.7881e-07,\n", + " 5.9605e-07, -1.1325e-06, -3.5763e-07, -4.1723e-07, -3.5763e-07,\n", + " 2.3842e-07, -1.1921e-07, -5.9605e-07, 7.1526e-07, -5.9605e-08,\n", + " -2.3842e-07, 3.5763e-07, -7.1526e-07, -5.9605e-08, -5.3644e-07,\n", + " 7.1526e-07, -2.9802e-07, -4.1723e-07, -1.1921e-07, 3.5763e-07,\n", + " 1.0729e-06, 2.3842e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07,\n", + " 4.7684e-07, -2.9802e-07, -2.3842e-07, -2.9802e-07, 5.9605e-07,\n", + " -6.5565e-07, -8.3447e-07, -1.7881e-07, 0.0000e+00, -6.5565e-07,\n", + " 3.5763e-07, 7.1526e-07, -2.9802e-07, 7.1526e-07, -1.7881e-07,\n", + " -5.9605e-08, 0.0000e+00, -1.1921e-07, 0.0000e+00, -1.1921e-07,\n", + " 2.3842e-07, 3.5763e-07, 2.3842e-07, 4.7684e-07, 3.5763e-07,\n", + " -6.5565e-07, 1.1921e-07], device='cuda:0')),\n", + " ('layer4.0.bn1.bias',\n", + " tensor([ 2.6659e-07, -2.0982e-07, 4.8005e-07, -1.1635e-07, 3.8215e-07,\n", + " 6.8295e-07, -1.7284e-07, 2.8059e-07, -1.3988e-07, 1.1298e-08,\n", + " -1.2147e-07, 3.9384e-09, 1.0461e-07, 2.2419e-07, -2.6699e-07,\n", + " 7.8134e-08, 7.8973e-07, -2.7129e-07, -4.5257e-07, 2.7466e-07,\n", + " 1.7832e-07, -2.4214e-07, -2.9081e-07, 1.8024e-07, -4.9308e-07,\n", + " -5.6225e-08, -4.9937e-09, 7.0436e-08, -6.0486e-07, -3.4290e-07,\n", + " 3.8399e-07, 5.2644e-08, -4.3564e-07, -7.4487e-07, 2.6390e-07,\n", + " -1.8460e-07, -1.3691e-07, 8.6185e-07, 1.0023e-07, -1.2655e-07,\n", + " -8.7669e-07, 1.7654e-08, -1.9440e-07, 3.6522e-07, -3.4770e-07,\n", + " 7.3103e-07, 5.0401e-07, -4.6698e-07, -1.1129e-07, 4.6539e-09,\n", + " -6.6089e-07, 2.4661e-07, -4.3374e-07, 1.4014e-07, 5.5946e-07,\n", + " 3.3049e-07, -3.6001e-07, 2.8588e-07, 3.3717e-07, -5.1532e-09,\n", + " -6.4083e-07, -5.5091e-07, 6.9731e-07, -5.2641e-07, -4.6825e-07,\n", + " 6.0111e-07, 3.3230e-07, 7.3765e-08, 3.7736e-07, -2.2260e-07,\n", + " -8.5065e-07, 5.1249e-09, 2.4908e-08, 2.0588e-07, -8.0888e-07,\n", + " -8.7111e-07, 5.2081e-07, -1.1161e-07, 3.7942e-07, -5.3236e-08,\n", + " -4.2491e-07, 5.6897e-07, -1.9099e-07, -5.0960e-08, 3.1389e-07,\n", + " 2.5616e-07, 6.1289e-09, -2.5935e-07, 2.5534e-07, 7.3285e-07,\n", + " -2.0531e-07, 2.5075e-07, 5.8549e-07, -3.4469e-07, -2.5699e-07,\n", + " 2.5711e-07, -1.7227e-07, 2.5903e-08, -2.9787e-07, -2.6435e-07,\n", + " 6.9626e-07, -3.2891e-07, -4.7151e-07, -4.9839e-08, 5.5711e-07,\n", + " 1.8523e-07, 1.5311e-08, -6.6544e-08, -2.2489e-07, 2.5704e-07,\n", + " -1.6068e-07, -2.5091e-07, -3.2175e-07, -2.8687e-07, 3.1285e-07,\n", + " 4.7909e-08, 3.8899e-07, 1.6728e-07, -1.8015e-07, -2.1716e-07,\n", + " 6.1003e-08, -2.6587e-07, -1.5374e-07, -2.8845e-07, 8.9849e-10,\n", + " -8.4853e-07, -3.3859e-08, 1.5536e-07, 1.1042e-07, 2.5031e-07,\n", + " 2.6351e-07, 1.7642e-07, -5.4142e-08, 7.8362e-07, 6.5737e-07,\n", + " 7.1140e-08, 3.5251e-07, -3.2149e-07, -1.6892e-07, -8.2579e-07,\n", + " -4.2970e-07, -1.9542e-07, 2.1797e-07, 3.0029e-07, 2.6878e-07,\n", + " 6.2129e-07, 2.6288e-07, -5.4413e-07, -1.4614e-08, -1.6254e-07,\n", + " 2.2507e-07, -3.1475e-07, -5.3041e-07, -6.0969e-08, 3.2513e-07,\n", + " 5.7300e-08, 1.0650e-07, -4.1601e-07, 4.1368e-07, -2.2369e-07,\n", + " 1.1620e-06, 3.3273e-07, 5.0645e-07, -3.6037e-07, -6.9913e-07,\n", + " -5.1527e-07, 5.1412e-07, -3.7701e-07, -5.4504e-08, 2.0858e-07,\n", + " -2.5934e-08, 5.5633e-07, -4.4394e-07, 9.8369e-08, 2.2145e-07,\n", + " -8.7645e-09, 1.3667e-07, 1.7740e-07, 6.7056e-07, -5.0873e-07,\n", + " -1.0292e-07, -2.5302e-07, 1.1807e-06, -1.4492e-08, 1.6166e-07,\n", + " -1.9679e-07, -7.0686e-08, -2.7642e-07, 2.6622e-08, -4.0368e-07,\n", + " 2.3976e-07, -6.0022e-07, -1.4709e-07, 7.6702e-07, -5.0405e-07,\n", + " 8.8996e-07, 3.3934e-07, 1.3966e-07, 4.7994e-07, 8.3614e-08,\n", + " 4.5502e-07, 2.5827e-07, -4.7942e-07, 1.5037e-07, -3.9148e-07,\n", + " 3.0440e-07, 5.3337e-08, -3.9186e-07, -2.7119e-08, -1.5321e-07,\n", + " 2.5038e-07, 6.7230e-07, -3.2854e-07, -3.6195e-08, -3.2283e-07,\n", + " 8.6197e-08, 2.7312e-07, 1.3391e-07, -6.2963e-07, -4.4744e-07,\n", + " 3.5468e-07, 5.9017e-07, -1.0100e-07, -2.4813e-07, 2.6591e-07,\n", + " 4.4953e-07, 6.9342e-07, 1.0524e-06, -1.4089e-07, -5.0600e-07,\n", + " 2.6431e-07, -1.5005e-07, 2.6835e-07, -6.6012e-07, -5.5946e-07,\n", + " -9.7378e-07, 2.1921e-07, -2.6048e-07, 1.4275e-07, 4.5729e-07,\n", + " 2.6154e-07, 5.4520e-07, 4.6810e-07, 1.3848e-09, -2.5565e-07,\n", + " -3.8275e-07, -6.9606e-07, -1.2039e-07, -1.1958e-07, -3.8735e-07,\n", + " 5.1144e-07, 2.9636e-07, 4.2048e-07, 2.8232e-07, 7.2826e-08,\n", + " 4.0116e-07, -7.1807e-07, 3.7533e-07, 2.8475e-07, 7.3259e-07,\n", + " -3.1961e-08, 9.2208e-08, 1.4817e-07, -4.8989e-07, -3.4522e-07,\n", + " 3.8085e-07, -2.1879e-07, -6.7025e-07, 1.8295e-07, 4.9678e-07,\n", + " -6.0239e-07, 7.2177e-07, 2.1635e-07, -4.3453e-07, -3.5543e-07,\n", + " -2.0856e-07, 2.0068e-08, 1.8934e-07, -2.0346e-07, 9.0863e-08,\n", + " 3.3822e-07, 1.9965e-07, -3.4879e-07, 7.0592e-07, 2.8215e-07,\n", + " -6.1477e-07, 8.4427e-07, 3.2009e-07, 4.2358e-07, -3.9903e-07,\n", + " -4.8077e-07, 1.7988e-07, 2.2444e-07, 3.7915e-07, -2.2638e-08,\n", + " -5.8419e-07, 4.5737e-08, 2.1573e-07, -1.0285e-08, -2.2151e-07,\n", + " 9.2880e-07, -3.6823e-07, 5.7077e-07, -5.1109e-07, -2.2646e-07,\n", + " 4.5903e-07, -2.8698e-07, 2.7114e-07, 4.5595e-07, -6.2474e-07,\n", + " -8.6035e-07, 1.2112e-07, 2.1300e-07, -6.6163e-07, 1.6430e-07,\n", + " 3.5742e-07, 2.3956e-08, -7.2227e-07, 5.4798e-07, 2.2492e-07,\n", + " 5.2444e-07, -3.5310e-07, -2.0026e-07, -1.9205e-07, 1.9381e-07,\n", + " -1.4635e-07, 3.7263e-07, -1.9696e-07, 6.0810e-08, -2.2472e-08,\n", + " -5.9005e-07, -2.4705e-07, 9.6962e-07, 1.8417e-07, -1.0311e-07,\n", + " -9.2609e-07, -1.8569e-07, 6.1638e-07, 8.0491e-07, -1.5014e-07,\n", + " 3.7104e-07, 1.5524e-07, -2.4523e-07, 4.3940e-07, 6.8601e-08,\n", + " 2.8458e-08, -1.3479e-07, 2.9886e-07, 2.7438e-07, -4.5789e-07,\n", + " -2.9986e-07, 3.2887e-07, 2.6367e-07, 6.5678e-08, -5.2845e-08,\n", + " -1.8448e-07, 5.9895e-08, 3.8829e-07, 6.2424e-08, 3.1400e-07,\n", + " 1.6121e-07, 2.3031e-07, -4.7892e-07, 5.0512e-07, -6.7946e-07,\n", + " -9.6831e-08, 8.1650e-08, 4.8874e-07, 6.9909e-07, -2.6260e-07,\n", + " -3.3464e-07, 5.1337e-07, -1.1322e-06, -1.9760e-07, -3.6737e-07,\n", + " -6.7565e-07, 5.9541e-08, -1.2394e-07, -5.0226e-07, 5.8358e-07,\n", + " -1.0674e-07, -2.2724e-07, -9.9859e-08, 5.2597e-07, -1.4322e-07,\n", + " -1.8635e-07, -2.4558e-07, 3.3967e-07, -7.3067e-09, -2.4298e-07,\n", + " -4.4757e-07, 1.3672e-07, -1.9951e-07, 9.4763e-07, 2.6936e-07,\n", + " -2.9640e-07, 1.0396e-06, 6.7169e-07, 2.4742e-07, 6.5884e-07,\n", + " 2.0972e-07, -4.0602e-07, -1.2101e-07, -8.6610e-08, 1.1707e-07,\n", + " -1.2173e-06, 7.6500e-08, 6.1672e-07, -2.3657e-07, -1.2074e-07,\n", + " 6.9198e-08, -1.4256e-07, 3.3578e-07, -6.9027e-08, -3.0768e-07,\n", + " 2.3745e-07, -3.5318e-07, -1.9281e-07, -2.4589e-07, 8.3489e-07,\n", + " 6.0610e-08, -1.0983e-07, 3.5077e-07, 6.0623e-07, 2.3406e-07,\n", + " -2.1303e-07, -5.7148e-08, 7.9081e-08, 1.5781e-07, 1.1813e-07,\n", + " -3.1826e-08, -5.1946e-07, -4.5346e-07, -2.7249e-07, -2.1805e-07,\n", + " -6.7970e-07, 1.0566e-07, 9.6752e-08, -7.1769e-07, 4.2578e-08,\n", + " -3.0736e-07, 6.7938e-07, -2.6022e-07, 1.7747e-07, -7.5835e-08,\n", + " 2.4098e-07, -2.0173e-07, -4.1996e-07, -1.0586e-07, -1.1475e-07,\n", + " 2.8858e-07, 2.3780e-07, -2.6370e-07, -2.1977e-07, -2.3373e-07,\n", + " -3.5201e-07, 8.5626e-08, -2.5834e-07, 5.5657e-07, -6.2127e-07,\n", + " 1.9869e-08, -8.6794e-07, 1.0717e-07, -2.8302e-07, -5.1231e-07,\n", + " 3.6485e-07, -5.1699e-07, -2.2456e-07, 5.7222e-07, 1.9374e-08,\n", + " 8.9344e-08, -3.1100e-08, -7.3690e-07, -1.1705e-07, -1.8583e-08,\n", + " -2.2865e-07, -2.9119e-07, 8.9379e-08, 7.7748e-09, 1.1452e-07,\n", + " 9.1766e-07, -2.4583e-07, 1.1647e-07, -1.7874e-07, 3.4206e-07,\n", + " 4.5904e-07, 1.5186e-07, 2.3652e-07, 4.3211e-07, 4.8214e-07,\n", + " -5.7052e-07, -1.2741e-06, 5.1995e-07, 1.3815e-07, 1.9537e-07,\n", + " 4.0341e-07, 2.6584e-07, -1.6991e-07, 5.8831e-07, -2.2692e-07,\n", + " -3.5686e-07, 2.1014e-09, -2.0653e-07, 2.9375e-07, -3.1663e-07,\n", + " 8.5626e-08, -3.6269e-08, 3.6752e-07, 8.9283e-08, 3.4088e-07,\n", + " -1.3519e-07, -2.0703e-07], device='cuda:0')),\n", + " ('layer4.0.conv2.weight',\n", + " tensor([[[[ 1.3984e-07, -1.1567e-06, -2.1700e-07],\n", + " [-1.3681e-06, -1.4883e-06, -6.7987e-08],\n", + " [-1.6829e-06, -1.1120e-06, -8.1770e-07]],\n", + " \n", + " [[-1.5032e-06, -1.1045e-06, -9.5926e-07],\n", + " [-1.5246e-06, -2.1507e-06, -3.9116e-07],\n", + " [-6.9663e-07, -1.2419e-06, 2.8848e-07]],\n", + " \n", + " [[ 1.6647e-07, -1.4069e-06, -1.4831e-06],\n", + " [-6.8080e-07, -1.2489e-06, -1.8366e-06],\n", + " [-9.1299e-07, -1.2685e-06, -9.4716e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.6566e-08, -6.4261e-08, -5.6297e-07],\n", + " [-1.3597e-07, -1.3390e-06, -4.7707e-07],\n", + " [-7.4779e-07, -7.2876e-08, -5.7986e-07]],\n", + " \n", + " [[-1.0699e-06, -9.5135e-07, -4.7195e-07],\n", + " [-2.4019e-06, -2.0582e-06, -1.5553e-06],\n", + " [-1.3486e-06, -1.3709e-06, -7.2643e-08]],\n", + " \n", + " [[-3.9426e-07, -1.1846e-06, -1.9018e-06],\n", + " [-1.9325e-06, -9.3691e-07, -1.7730e-06],\n", + " [-7.6089e-07, -2.7004e-06, -2.6114e-06]]],\n", + " \n", + " \n", + " [[[-1.3763e-07, -3.3714e-07, 1.3958e-07],\n", + " [-6.2166e-07, 3.6554e-07, -8.5495e-07],\n", + " [-2.8126e-07, -7.5158e-07, -2.6263e-07]],\n", + " \n", + " [[-3.7346e-07, 2.5146e-07, 1.4249e-07],\n", + " [-6.5344e-07, -2.7299e-07, -2.2675e-07],\n", + " [-3.5390e-08, -6.1002e-07, 2.4354e-07]],\n", + " \n", + " [[-4.4703e-08, -8.4774e-07, -6.9477e-07],\n", + " [-7.2829e-07, -2.6822e-07, -2.1579e-06],\n", + " [-7.9675e-07, -2.4051e-07, -7.5530e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-8.7544e-08, 7.0594e-07, 8.0094e-08],\n", + " [-2.0070e-07, -2.1327e-07, -1.6019e-07],\n", + " [-2.9360e-07, 3.0780e-07, -4.1910e-08]],\n", + " \n", + " [[-1.8300e-07, -1.2405e-06, -5.7300e-07],\n", + " [-1.1781e-06, -9.0618e-07, -1.2463e-06],\n", + " [ 1.6764e-08, -6.0920e-07, -4.1910e-08]],\n", + " \n", + " [[ 7.1153e-07, 3.0361e-07, 2.9802e-08],\n", + " [ 1.2715e-06, 8.6147e-07, -2.9476e-07],\n", + " [ 3.6135e-07, 6.3051e-07, 8.8010e-08]]],\n", + " \n", + " \n", + " [[[ 3.2212e-07, 6.5286e-07, 1.6717e-06],\n", + " [-6.6590e-08, 1.2433e-06, 2.1770e-06],\n", + " [ 1.0803e-07, 5.1223e-08, 4.3865e-07]],\n", + " \n", + " [[ 8.2981e-07, 1.8002e-06, 1.7667e-06],\n", + " [ 5.6811e-07, 2.3339e-06, 1.7919e-06],\n", + " [ 6.3423e-07, 1.5423e-06, 1.7118e-06]],\n", + " \n", + " [[ 8.5682e-08, 3.7905e-07, 1.9318e-06],\n", + " [-1.4342e-07, 2.6310e-07, 1.1101e-06],\n", + " [ 1.3877e-07, 3.7160e-07, 1.1367e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.4983e-07, 7.7952e-07, 7.0682e-07],\n", + " [ 6.9384e-07, -9.5111e-08, 1.4193e-06],\n", + " [ 2.0489e-07, 5.7649e-07, 1.6885e-06]],\n", + " \n", + " [[ 1.4435e-08, -6.0536e-08, 9.9838e-07],\n", + " [-3.5390e-07, 5.4203e-07, 2.4368e-06],\n", + " [-4.5355e-07, -7.3016e-07, 9.5088e-07]],\n", + " \n", + " [[ 4.6380e-07, 5.5414e-07, 1.8190e-06],\n", + " [-1.2852e-07, 1.0037e-06, 1.9865e-06],\n", + " [ 1.8626e-09, -2.7567e-07, 1.2647e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 1.0058e-07, -1.0545e-06, -6.7614e-07],\n", + " [-4.5262e-07, -2.2724e-07, -1.1139e-06],\n", + " [ 6.3679e-08, -1.6205e-06, -9.4995e-07]],\n", + " \n", + " [[-9.8022e-07, -2.5099e-06, -2.1181e-06],\n", + " [-2.1691e-06, -1.7202e-06, -2.0822e-06],\n", + " [-1.3243e-06, -2.5127e-06, -1.0971e-06]],\n", + " \n", + " [[-2.1700e-07, -6.1840e-07, -1.1986e-06],\n", + " [-5.6997e-07, -9.7509e-07, -2.4782e-06],\n", + " [-5.9139e-08, -5.2061e-07, -1.9332e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 9.7789e-08, 6.8653e-07, -6.8452e-07],\n", + " [ 4.1444e-07, -7.3109e-07, -1.1846e-06],\n", + " [-1.5926e-07, -3.3248e-07, -5.9791e-07]],\n", + " \n", + " [[ 6.1467e-08, -2.6060e-07, -1.9872e-06],\n", + " [-5.6240e-07, -3.2410e-07, -3.6461e-07],\n", + " [ 1.1006e-06, -1.1129e-06, -1.2780e-06]],\n", + " \n", + " [[ 6.1747e-07, 6.4075e-07, 1.0617e-07],\n", + " [ 6.3889e-07, 1.1101e-06, -1.1446e-06],\n", + " [ 1.5320e-07, -1.1092e-06, -3.2084e-07]]],\n", + " \n", + " \n", + " [[[ 7.3202e-07, 3.2663e-07, 3.1479e-07],\n", + " [ 1.9558e-06, 1.3807e-06, 1.1679e-06],\n", + " [ 1.5089e-06, 1.4845e-06, 1.2149e-06]],\n", + " \n", + " [[ 1.7742e-07, 5.4538e-07, 7.8883e-07],\n", + " [ 7.1526e-07, 1.1381e-06, 1.5292e-06],\n", + " [ 3.4086e-07, -3.9069e-07, 6.4634e-07]],\n", + " \n", + " [[ 7.3458e-07, 1.7518e-06, 6.3237e-07],\n", + " [ 1.4721e-06, 1.3504e-06, 1.3513e-06],\n", + " [ 8.4005e-07, 1.2638e-06, 2.5239e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.9267e-07, 1.0887e-06, 4.6100e-07],\n", + " [ 1.4110e-07, 1.1190e-06, 7.4157e-08],\n", + " [ 8.6753e-07, 8.3540e-07, -8.5216e-07]],\n", + " \n", + " [[ 1.0123e-06, 1.5162e-06, 1.6415e-07],\n", + " [ 6.9849e-07, 1.6196e-06, 2.5891e-07],\n", + " [ 6.0536e-07, 3.2270e-07, 2.2491e-07]],\n", + " \n", + " [[ 2.1888e-07, -1.9744e-07, -6.3609e-07],\n", + " [ 2.0117e-07, 6.2725e-07, -1.2498e-06],\n", + " [-2.5798e-07, 5.3179e-07, 1.2666e-07]]],\n", + " \n", + " \n", + " [[[ 1.1087e-06, 1.0915e-06, -8.3074e-07],\n", + " [ 2.1579e-06, 1.8310e-06, 9.8546e-08],\n", + " [ 2.5469e-06, 1.4603e-06, 7.7952e-07]],\n", + " \n", + " [[-3.2037e-07, -1.3858e-06, -1.2787e-06],\n", + " [-3.6601e-07, -5.2433e-07, -5.4832e-07],\n", + " [-6.8836e-07, -4.7521e-07, -5.7090e-07]],\n", + " \n", + " [[-3.7812e-07, 4.7684e-07, -4.1910e-07],\n", + " [ 2.5425e-07, 8.4890e-07, 1.3979e-06],\n", + " [ 7.3342e-07, 7.9628e-07, 6.0722e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.5050e-06, 2.3693e-06, 9.5088e-07],\n", + " [ 1.9956e-06, 1.7863e-06, 1.3666e-06],\n", + " [ 8.2050e-07, 7.5577e-07, 9.3877e-07]],\n", + " \n", + " [[ 8.1025e-08, 4.7173e-07, 3.0315e-07],\n", + " [ 5.5385e-07, 5.8487e-07, 7.5344e-07],\n", + " [ 1.7136e-07, 4.1816e-07, -2.8126e-07]],\n", + " \n", + " [[ 1.1921e-07, 1.9243e-07, 1.1115e-06],\n", + " [-1.2410e-07, 2.0694e-06, 1.9688e-06],\n", + " [ 2.0768e-07, 1.0170e-06, 1.2182e-06]]]], device='cuda:0')),\n", + " ('layer4.0.bn2.weight',\n", + " tensor([ 1.1921e-07, 2.3842e-07, -2.9802e-07, -2.9802e-07, 9.5367e-07,\n", + " -1.1921e-07, 5.9605e-07, 2.3842e-07, 1.1921e-07, 2.3842e-07,\n", + " 4.7684e-07, -5.3644e-07, -4.1723e-07, -4.1723e-07, 2.3842e-07,\n", + " -6.5565e-07, 1.1921e-07, 0.0000e+00, -2.9802e-07, 2.3842e-07,\n", + " -5.9605e-08, -1.6093e-06, 2.3842e-07, -3.5763e-07, 1.1921e-07,\n", + " -7.7486e-07, -3.5763e-07, -1.1921e-07, 1.1921e-07, -5.3644e-07,\n", + " -7.1526e-07, -1.0729e-06, -5.3644e-07, -2.3842e-07, -4.7684e-07,\n", + " -2.9802e-07, 5.9605e-07, 8.3447e-07, 2.3842e-07, -2.9802e-07,\n", + " 1.1921e-07, 3.5763e-07, 3.5763e-07, -2.3842e-07, -2.3842e-07,\n", + " 3.5763e-07, 7.1526e-07, -1.7881e-07, -4.7684e-07, 5.9605e-07,\n", + " -5.9605e-08, 3.5763e-07, -2.9802e-07, -8.9407e-07, 2.3842e-07,\n", + " 3.5763e-07, -1.1921e-06, -5.9605e-08, -8.9407e-07, 1.1921e-07,\n", + " 0.0000e+00, -2.9802e-07, 2.3842e-07, 1.1921e-07, -6.5565e-07,\n", + " 1.1921e-07, 5.9605e-07, 0.0000e+00, -5.9605e-07, -5.9605e-08,\n", + " 8.3447e-07, 5.9605e-07, 3.5763e-07, -2.3842e-07, 9.5367e-07,\n", + " 5.9605e-07, -3.5763e-07, 1.1921e-07, 2.3842e-07, 1.1921e-07,\n", + " -1.3709e-06, 8.3447e-07, -4.1723e-07, -1.4305e-06, -3.5763e-07,\n", + " -9.5367e-07, 1.1921e-07, -5.9605e-08, 7.1526e-07, -7.7486e-07,\n", + " -2.3842e-07, -2.9802e-07, 7.1526e-07, -5.3644e-07, 0.0000e+00,\n", + " 1.1921e-07, 9.5367e-07, 1.1921e-06, 5.9605e-07, 0.0000e+00,\n", + " -7.1526e-07, 4.7684e-07, -2.3842e-07, -1.1921e-06, 4.7684e-07,\n", + " 5.9605e-07, -5.9605e-08, 3.5763e-07, -1.7881e-07, 2.3842e-07,\n", + " 1.1921e-07, -5.9605e-08, 1.1921e-07, -4.7684e-07, 7.1526e-07,\n", + " 4.7684e-07, 8.3447e-07, -5.9605e-08, -3.5763e-07, 8.3447e-07,\n", + " 4.7684e-07, -1.7881e-07, 4.7684e-07, -3.5763e-07, 4.7684e-07,\n", + " 4.7684e-07, -5.3644e-07, -2.9802e-07, -7.1526e-07, -3.5763e-07,\n", + " -1.1921e-07, -2.9802e-07, 4.7684e-07, -1.0133e-06, -2.3842e-07,\n", + " 2.3842e-07, 0.0000e+00, 5.9605e-07, -9.5367e-07, -1.1921e-07,\n", + " -3.5763e-07, -7.7486e-07, -3.5763e-07, -7.7486e-07, 8.3447e-07,\n", + " -4.7684e-07, -2.3842e-07, 0.0000e+00, -8.9407e-07, 4.7684e-07,\n", + " -2.9802e-07, 7.1526e-07, 3.5763e-07, -5.9605e-08, -7.1526e-07,\n", + " 0.0000e+00, -5.9605e-08, 2.3842e-07, -5.9605e-08, -1.1325e-06,\n", + " -4.1723e-07, -1.0133e-06, -7.1526e-07, -1.7881e-07, -5.9605e-08,\n", + " -1.7881e-07, -5.9605e-07, -2.3842e-07, -1.7881e-07, -4.1723e-07,\n", + " 1.0729e-06, 3.5763e-07, 1.1921e-07, 1.1921e-07, -2.3842e-07,\n", + " 5.9605e-07, -2.9802e-07, 8.3447e-07, 4.7684e-07, 0.0000e+00,\n", + " -6.5565e-07, 5.9605e-07, -3.5763e-07, 5.9605e-07, -7.7486e-07,\n", + " 1.1921e-07, -3.5763e-07, -1.7881e-07, 4.7684e-07, 3.5763e-07,\n", + " -5.9605e-08, 0.0000e+00, 4.7684e-07, -1.7881e-07, -9.5367e-07,\n", + " 4.7684e-07, 1.1921e-07, -5.3644e-07, -1.7881e-07, -3.5763e-07,\n", + " 1.1921e-07, 3.5763e-07, 7.1526e-07, -1.1921e-07, -2.3842e-07,\n", + " -5.9605e-07, -4.7684e-07, 4.7684e-07, 1.5497e-06, 0.0000e+00,\n", + " 3.5763e-07, -5.9605e-07, -1.1921e-07, 2.3842e-07, -1.0133e-06,\n", + " -1.7881e-07, -7.1526e-07, -4.7684e-07, 1.1921e-07, 2.3842e-07,\n", + " 1.1921e-07, 2.3842e-07, -1.0133e-06, 4.7684e-07, 5.9605e-07,\n", + " 5.9605e-07, 4.7684e-07, 2.3842e-07, 2.3842e-07, 7.1526e-07,\n", + " 3.5763e-07, 1.0729e-06, -4.1723e-07, 2.3842e-07, 0.0000e+00,\n", + " 2.3842e-07, 2.3842e-07, -2.3842e-07, -1.1921e-07, -1.0133e-06,\n", + " -5.3644e-07, 2.3842e-07, 5.9605e-07, -2.3842e-07, -6.5565e-07,\n", + " -5.9605e-08, 1.1921e-07, 2.3842e-07, 1.1921e-07, 1.1921e-06,\n", + " -1.1921e-06, -5.9605e-07, 3.5763e-07, -1.1921e-07, 1.1921e-07,\n", + " 2.3842e-07, 2.3842e-07, 1.0729e-06, 3.5763e-07, 5.9605e-07,\n", + " 2.3842e-07, -5.9605e-08, -4.1723e-07, -1.1921e-06, -1.1325e-06,\n", + " -4.1723e-07, -2.9802e-07, 8.3447e-07, 2.3842e-07, 2.3842e-07,\n", + " 0.0000e+00, -2.3842e-07, 1.1921e-07, -4.1723e-07, 5.9605e-07,\n", + " 0.0000e+00, -9.5367e-07, 0.0000e+00, -2.9802e-07, 4.7684e-07,\n", + " 1.0729e-06, -2.3842e-07, 7.1526e-07, -2.9802e-07, -4.1723e-07,\n", + " 0.0000e+00, -3.5763e-07, 3.5763e-07, -8.9407e-07, 2.3842e-07,\n", + " -1.6689e-06, 3.5763e-07, 0.0000e+00, 2.3842e-07, -5.9605e-07,\n", + " 2.3842e-07, -5.9605e-08, -4.1723e-07, -1.7881e-07, 9.5367e-07,\n", + " -8.3447e-07, 0.0000e+00, -1.0133e-06, 4.7684e-07, -6.5565e-07,\n", + " -2.9802e-07, -3.5763e-07, 2.3842e-07, -5.9605e-07, -4.1723e-07,\n", + " -1.9073e-06, -7.1526e-07, 1.1921e-07, 2.3842e-07, -4.1723e-07,\n", + " 1.1921e-07, -5.9605e-08, -5.9605e-07, -4.1723e-07, 2.3842e-07,\n", + " 1.1921e-07, -5.9605e-08, 3.5763e-07, 4.7684e-07, -4.7684e-07,\n", + " 9.5367e-07, -5.3644e-07, 5.9605e-07, 3.5763e-07, -8.9407e-07,\n", + " -1.1921e-07, -5.9605e-08, 2.3842e-07, 8.3447e-07, 1.1921e-07,\n", + " -7.1526e-07, -5.9605e-08, 2.3842e-07, 4.7684e-07, 0.0000e+00,\n", + " 4.7684e-07, 2.3842e-07, -2.3842e-07, -3.5763e-07, -1.1921e-07,\n", + " -2.9802e-07, 5.9605e-07, 1.1921e-07, 0.0000e+00, -1.1921e-07,\n", + " -5.3644e-07, -4.1723e-07, 1.1921e-07, 3.5763e-07, -3.5763e-07,\n", + " -5.3644e-07, -4.1723e-07, -6.5565e-07, 2.3842e-07, -2.3842e-07,\n", + " -1.7881e-07, -2.3842e-07, -1.7881e-07, -2.3842e-07, 5.9605e-07,\n", + " -1.7881e-07, -2.9802e-07, 8.3447e-07, -6.5565e-07, -1.7881e-07,\n", + " 1.1921e-07, -2.3842e-07, -7.1526e-07, -4.1723e-07, 4.7684e-07,\n", + " 0.0000e+00, -4.1723e-07, -7.1526e-07, 3.5763e-07, -9.5367e-07,\n", + " 2.3842e-07, 1.1921e-07, 1.1921e-07, 4.7684e-07, 1.1921e-07,\n", + " 0.0000e+00, 4.7684e-07, 0.0000e+00, -4.1723e-07, 7.1526e-07,\n", + " 8.3447e-07, -4.1723e-07, -2.9802e-07, 3.5763e-07, 0.0000e+00,\n", + " -9.5367e-07, -2.3842e-07, -2.9802e-07, -7.7486e-07, -1.7881e-07,\n", + " -7.1526e-07, -3.5763e-07, 3.5763e-07, -2.3842e-07, -4.1723e-07,\n", + " 1.1921e-07, 2.3842e-07, -4.1723e-07, -7.1526e-07, 1.1921e-07,\n", + " -7.7486e-07, 3.5763e-07, -1.1921e-07, -5.9605e-08, -8.3447e-07,\n", + " -2.3842e-07, -1.1921e-07, 5.9605e-07, -5.3644e-07, -5.9605e-07,\n", + " -4.1723e-07, -5.9605e-08, 5.9605e-07, 0.0000e+00, 4.7684e-07,\n", + " -2.9802e-07, 4.7684e-07, -3.5763e-07, 0.0000e+00, -6.5565e-07,\n", + " -3.5763e-07, -3.5763e-07, 3.5763e-07, 2.3842e-07, -4.7684e-07,\n", + " -1.7881e-07, 3.5763e-07, 2.3842e-07, -4.1723e-07, -2.3842e-07,\n", + " 8.3447e-07, 2.3842e-07, 0.0000e+00, -2.9802e-07, -2.3842e-07,\n", + " -1.4305e-06, 5.9605e-07, 0.0000e+00, -4.7684e-07, 0.0000e+00,\n", + " 1.1921e-07, 1.1921e-07, 9.5367e-07, -7.1526e-07, 1.0729e-06,\n", + " -2.3842e-07, 1.1921e-07, 5.9605e-07, 2.3842e-07, 1.1921e-07,\n", + " -1.1921e-07, -1.0729e-06, -1.1921e-07, 5.9605e-07, -5.9605e-07,\n", + " -4.7684e-07, 1.1921e-07, 0.0000e+00, -1.0133e-06, 7.1526e-07,\n", + " -6.5565e-07, -1.2517e-06, 3.5763e-07, 1.1921e-07, 1.0729e-06,\n", + " 2.3842e-07, -7.7486e-07, 4.7684e-07, -5.9605e-07, -1.1921e-07,\n", + " -5.9605e-08, -8.9407e-07, -5.9605e-08, 1.1921e-07, -9.5367e-07,\n", + " -5.9605e-08, -5.9605e-07, -1.7881e-07, 4.7684e-07, -2.3842e-07,\n", + " -4.7684e-07, 2.3842e-07, -1.0133e-06, 7.1526e-07, -2.3842e-07,\n", + " -5.9605e-07, -4.1723e-07, 2.3842e-07, -5.9605e-08, 2.3842e-07,\n", + " -3.5763e-07, 1.1921e-07, -1.7881e-07, -8.9407e-07, -1.3709e-06,\n", + " -4.1723e-07, -5.9605e-08, 3.5763e-07, 0.0000e+00, 2.3842e-07,\n", + " 3.5763e-07, -5.9605e-08], device='cuda:0')),\n", + " ('layer4.0.bn2.bias',\n", + " tensor([ 1.2267e-07, 5.2131e-07, 4.9048e-07, -6.6727e-08, 5.8954e-07,\n", + " 2.0044e-07, 3.3354e-07, 1.0084e-07, 3.4111e-07, 1.0555e-07,\n", + " 4.8835e-07, -1.1029e-06, 2.0666e-08, 2.0148e-07, 2.1696e-07,\n", + " -7.4097e-07, 3.7880e-08, 6.4895e-08, -8.4100e-07, -3.1968e-07,\n", + " 1.0362e-07, -9.7019e-07, -3.3352e-07, -4.1362e-07, 4.1598e-08,\n", + " 2.2223e-07, 1.6852e-07, -1.0521e-07, 2.1346e-07, -1.2921e-06,\n", + " 3.7748e-07, -5.9054e-07, -1.5242e-08, -5.5715e-07, -2.1764e-07,\n", + " 2.1038e-07, 7.7662e-07, 9.8997e-07, 2.4168e-07, 8.6350e-08,\n", + " 1.8026e-07, 9.3270e-08, 2.4967e-07, -5.6322e-07, -3.2481e-07,\n", + " 5.5609e-09, 2.3633e-07, -4.5999e-07, 6.0220e-08, 6.4592e-07,\n", + " 2.2764e-07, -1.9812e-07, -9.3918e-07, -3.2255e-07, 2.1820e-07,\n", + " 1.5435e-06, -1.6022e-06, 9.6016e-08, -2.3592e-07, 7.2372e-07,\n", + " -4.9385e-07, 1.8043e-07, -4.0969e-07, 2.5857e-07, -5.5846e-07,\n", + " 3.3289e-07, 3.6472e-07, -9.9066e-07, -6.4836e-07, -5.9628e-08,\n", + " 7.9305e-07, 5.1530e-07, 3.1524e-07, -3.6323e-07, 3.4015e-07,\n", + " 1.8533e-07, -2.7069e-07, 2.8576e-07, -5.2353e-07, -1.3846e-07,\n", + " -1.7000e-06, 3.0305e-07, -8.1954e-07, -9.5841e-07, 3.0598e-07,\n", + " -8.9424e-07, 2.0526e-07, -5.2114e-07, 3.8126e-07, -8.6022e-07,\n", + " 9.4818e-07, -1.4355e-07, 2.4742e-07, -4.0967e-07, 1.2586e-07,\n", + " -9.7055e-07, 1.4553e-06, 1.1442e-07, 8.6497e-07, 1.3247e-07,\n", + " -5.0140e-07, -2.5456e-07, -2.6590e-08, -3.5772e-07, 8.2593e-07,\n", + " 5.3295e-07, -1.0410e-07, 4.3498e-07, 2.3294e-07, 4.7580e-07,\n", + " -5.4222e-07, 4.6807e-07, 3.8665e-07, -5.5240e-07, 4.0680e-07,\n", + " 4.8383e-07, -3.1379e-07, 2.7812e-07, -8.6844e-07, -4.2003e-08,\n", + " 3.9490e-07, -2.1611e-07, 4.1244e-08, -7.7546e-07, -2.5390e-07,\n", + " 7.5168e-07, -1.0881e-06, -3.0041e-09, -1.2022e-06, -1.0697e-06,\n", + " 2.1438e-07, -2.7698e-07, 1.0052e-07, -2.8831e-07, -7.9248e-07,\n", + " -1.1695e-08, 3.7242e-07, 3.6402e-07, -4.4123e-07, 6.7511e-08,\n", + " 6.2402e-07, -7.6874e-07, -7.7209e-07, -4.6478e-07, 8.0412e-07,\n", + " -1.1367e-06, 7.2712e-07, -1.3130e-07, -5.0524e-07, 2.0572e-07,\n", + " -4.2604e-07, 1.5546e-07, 9.0354e-07, -5.7759e-07, -6.8648e-07,\n", + " 3.3436e-07, 4.4954e-07, 3.9408e-07, -6.5580e-07, -9.3324e-07,\n", + " -6.8870e-07, -6.4934e-07, -1.5007e-07, -5.6741e-08, -9.6720e-07,\n", + " 4.3615e-07, -1.5925e-07, -8.6850e-07, 4.7006e-07, -6.9503e-07,\n", + " 8.5696e-07, 4.0076e-07, 5.0084e-07, 3.3838e-07, 1.2934e-07,\n", + " 1.0382e-06, -4.2320e-07, 3.1105e-07, 9.0592e-07, 5.3915e-07,\n", + " -2.8505e-07, 3.0121e-07, -3.5367e-07, 3.1758e-07, -9.3912e-07,\n", + " 7.7104e-07, 4.0988e-07, 4.6698e-08, 3.6261e-07, -2.4688e-07,\n", + " -5.6081e-07, 5.2616e-07, 3.9207e-07, 3.3400e-07, 1.3801e-07,\n", + " 1.5755e-07, 1.3671e-07, -2.9247e-07, -3.3614e-08, 2.7387e-07,\n", + " 4.1058e-08, 1.0102e-07, 5.9427e-07, 6.9601e-07, 5.1401e-07,\n", + " -9.0142e-07, -7.8599e-07, 8.5890e-07, 8.0110e-07, -2.6069e-08,\n", + " -4.7613e-07, -1.1880e-07, 6.0977e-07, -3.2545e-08, -8.0672e-07,\n", + " -4.4460e-07, 4.9622e-08, -1.0001e-06, -3.8140e-07, 2.0876e-07,\n", + " -7.7196e-07, 7.1734e-07, -4.6298e-07, 6.7959e-07, 1.0245e-06,\n", + " 4.1416e-07, 5.4496e-07, 7.8721e-07, -4.2891e-07, 1.6825e-06,\n", + " 1.4942e-07, -1.6828e-07, 1.7415e-07, 4.0402e-08, -5.3157e-07,\n", + " 1.0582e-07, -2.0883e-07, 4.7203e-07, -1.3684e-07, -1.3398e-06,\n", + " -5.9472e-07, 3.9554e-07, 2.3206e-07, -2.5863e-07, -4.5267e-07,\n", + " 1.0301e-06, -5.3752e-07, -7.6298e-08, -5.4183e-07, 2.8119e-07,\n", + " 7.9946e-08, -1.5766e-07, 1.8332e-07, -8.2711e-07, -3.7360e-07,\n", + " -1.5190e-07, -3.5914e-08, 3.6115e-07, -1.4778e-07, 8.6465e-07,\n", + " -2.9969e-08, 2.9304e-07, 2.5676e-07, -2.8295e-07, -8.0901e-07,\n", + " -3.3888e-07, -1.1907e-07, 2.6471e-07, 1.6580e-07, -6.6758e-08,\n", + " 2.1079e-07, 3.7526e-07, 3.1104e-07, 8.7153e-07, 3.5282e-07,\n", + " 4.8555e-07, -2.3091e-07, -5.6911e-07, 1.3365e-07, 3.0886e-07,\n", + " 2.5100e-07, -2.3340e-07, 4.3298e-07, -3.9339e-07, -6.6168e-07,\n", + " 2.4621e-07, 1.2055e-07, 5.7209e-08, -5.5472e-07, 8.4360e-07,\n", + " -8.6571e-07, 1.9837e-07, 5.0580e-08, 4.3711e-07, 4.6089e-07,\n", + " 1.3192e-07, 1.8432e-08, -4.0834e-07, 2.2995e-07, 9.0939e-07,\n", + " -6.0561e-07, 4.8501e-07, -6.9802e-07, 4.8374e-07, -3.7478e-07,\n", + " -3.0589e-07, -2.6858e-07, -4.6554e-08, -1.0507e-07, -2.8908e-07,\n", + " -1.0337e-06, -1.0129e-06, 2.2487e-07, 1.7116e-07, -4.1344e-07,\n", + " -4.2085e-07, -3.7954e-08, -5.6785e-07, -5.8464e-07, 1.2217e-08,\n", + " -9.5578e-08, -2.8373e-07, 2.6618e-07, 7.6734e-07, -8.6086e-07,\n", + " 3.6636e-07, -9.9598e-07, -1.5478e-07, 1.7445e-07, -5.5791e-07,\n", + " -7.7129e-07, -2.5449e-07, 2.9259e-07, 2.6276e-07, -7.1697e-08,\n", + " -5.8335e-07, 1.1469e-08, 2.1788e-07, 1.0519e-06, 5.5815e-08,\n", + " 4.8869e-07, -6.1105e-08, 2.8824e-08, 5.1238e-07, -2.2381e-08,\n", + " -1.9842e-07, 1.5346e-07, 2.2163e-07, -3.7323e-07, 2.4546e-07,\n", + " -5.5839e-07, -1.2237e-06, 7.6679e-07, 1.1058e-06, -6.1683e-07,\n", + " -5.6177e-07, -4.2822e-07, -2.7411e-07, 2.8429e-09, -1.7565e-07,\n", + " -6.2889e-07, -5.5862e-07, -5.8493e-07, -4.7978e-07, 6.6172e-07,\n", + " 6.3017e-08, -5.3413e-07, 8.2590e-07, -2.3073e-07, -2.7192e-07,\n", + " 5.1345e-08, -4.7095e-07, -7.3116e-07, -7.4617e-07, 1.2349e-06,\n", + " -5.5501e-08, -3.8916e-07, 4.9881e-07, -5.8128e-07, -1.0835e-06,\n", + " 8.9368e-08, -2.4767e-08, -1.7907e-07, 7.5381e-07, -9.3912e-07,\n", + " 1.6275e-07, 8.3961e-07, 1.9004e-07, -6.1507e-07, 1.7484e-07,\n", + " 6.1234e-07, -3.3452e-07, 1.7429e-07, 6.1249e-07, -4.0872e-07,\n", + " -8.5998e-07, -2.6819e-08, 4.6977e-07, -4.8661e-07, -3.7181e-07,\n", + " -4.3231e-07, -4.4359e-07, 1.0288e-07, -4.6875e-07, 1.2184e-07,\n", + " 5.4193e-07, 1.6957e-07, 1.2930e-08, -8.3377e-07, 2.0160e-07,\n", + " -1.1283e-06, 4.6074e-08, 7.9631e-08, -1.9956e-07, -2.8262e-07,\n", + " -2.7352e-07, -3.9198e-07, 9.0809e-07, -6.9796e-07, -7.7386e-07,\n", + " -6.3717e-07, 1.9364e-07, 4.2499e-07, -2.1963e-07, 3.3570e-07,\n", + " -1.1842e-06, -1.3149e-07, -2.0294e-07, -8.4020e-08, -1.8796e-07,\n", + " -8.5904e-08, 1.0188e-06, 1.1152e-07, -7.9846e-07, -1.3746e-07,\n", + " 3.0789e-07, 1.1175e-07, -3.5673e-07, -3.4911e-08, -2.8830e-08,\n", + " -2.7811e-07, 6.8873e-08, 6.6728e-07, -2.9486e-07, 6.0918e-07,\n", + " -1.2829e-06, 6.8836e-07, 3.0841e-07, 4.4511e-07, 3.0244e-07,\n", + " 4.3634e-07, 3.6050e-08, 5.0360e-07, -5.9110e-07, 9.7646e-07,\n", + " -9.2563e-07, -4.3873e-07, 1.1422e-07, -1.3625e-07, -6.4917e-07,\n", + " 5.3954e-08, -1.5143e-06, -5.9071e-07, -2.4121e-08, -1.0838e-06,\n", + " -4.0250e-07, 5.5546e-07, -7.1030e-07, -1.0235e-06, -9.1596e-08,\n", + " 1.7611e-08, -1.6254e-06, 5.3340e-07, 5.0056e-07, 2.9434e-07,\n", + " 3.0389e-07, -6.4017e-07, 3.4055e-07, -2.9545e-07, 1.7299e-07,\n", + " -3.5783e-08, 3.9323e-07, 1.4520e-08, 2.4158e-09, -8.8659e-07,\n", + " -2.5928e-07, -4.6700e-07, -4.3284e-07, 7.8656e-08, 2.4758e-07,\n", + " 7.3693e-08, 2.9827e-08, -5.8056e-07, -9.8295e-08, -2.7469e-08,\n", + " -6.7635e-07, 5.9764e-08, 9.8274e-08, 1.2037e-08, -3.1155e-07,\n", + " -2.1793e-07, 6.5858e-08, 4.3835e-08, 1.8843e-07, 1.4375e-07,\n", + " 7.0740e-08, 9.6125e-08, 1.4013e-07, 6.2235e-07, 1.1469e-06,\n", + " -4.6511e-09, 2.3411e-07], device='cuda:0')),\n", + " ('layer4.0.shortcut.0.weight',\n", + " tensor([[[[ 3.1851e-07]],\n", + " \n", + " [[ 3.7812e-07]],\n", + " \n", + " [[ 1.4585e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-5.5134e-07]],\n", + " \n", + " [[-1.2666e-07]],\n", + " \n", + " [[ 4.6939e-07]]],\n", + " \n", + " \n", + " [[[-6.9104e-07]],\n", + " \n", + " [[-5.1036e-07]],\n", + " \n", + " [[ 9.2760e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.1665e-08]],\n", + " \n", + " [[ 3.7067e-07]],\n", + " \n", + " [[ 1.5087e-07]]],\n", + " \n", + " \n", + " [[[ 9.3132e-09]],\n", + " \n", + " [[ 5.0245e-07]],\n", + " \n", + " [[-5.2154e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-8.9407e-08]],\n", + " \n", + " [[ 4.8429e-07]],\n", + " \n", + " [[-9.7920e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-1.2740e-06]],\n", + " \n", + " [[-1.5795e-06]],\n", + " \n", + " [[ 1.6820e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.7998e-07]],\n", + " \n", + " [[-4.6939e-07]],\n", + " \n", + " [[ 2.9681e-06]]],\n", + " \n", + " \n", + " [[[ 7.5996e-07]],\n", + " \n", + " [[ 1.3264e-06]],\n", + " \n", + " [[-4.4098e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-7.2643e-08]],\n", + " \n", + " [[ 1.1958e-06]],\n", + " \n", + " [[ 1.5832e-07]]],\n", + " \n", + " \n", + " [[[-6.0722e-07]],\n", + " \n", + " [[-9.4622e-07]],\n", + " \n", + " [[ 8.3633e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-9.1642e-07]],\n", + " \n", + " [[ 1.2033e-06]],\n", + " \n", + " [[ 8.2329e-07]]]], device='cuda:0')),\n", + " ('layer4.0.shortcut.1.weight',\n", + " tensor([-4.1723e-07, 1.1921e-07, 7.1526e-07, 3.5763e-07, -2.9802e-07,\n", + " 2.3842e-07, -2.3842e-07, -1.1921e-07, 2.3842e-07, -5.9605e-08,\n", + " -1.1921e-07, -7.1526e-07, 2.3842e-07, 5.9605e-07, 1.1921e-07,\n", + " 1.1921e-07, 0.0000e+00, -2.3842e-07, -8.3447e-07, -1.1325e-06,\n", + " -2.3842e-07, -7.7486e-07, -1.7881e-07, 2.3842e-07, 0.0000e+00,\n", + " 7.1526e-07, 4.7684e-07, 0.0000e+00, 4.7684e-07, -1.0729e-06,\n", + " 2.3842e-07, 1.1921e-07, 3.5763e-07, -1.2517e-06, -5.9605e-08,\n", + " 8.3447e-07, 1.5497e-06, 3.5763e-07, 3.5763e-07, 1.1921e-07,\n", + " 1.1921e-07, -3.5763e-07, -1.1921e-07, -9.5367e-07, -5.3644e-07,\n", + " -5.3644e-07, -8.3447e-07, -9.5367e-07, 2.3842e-07, 0.0000e+00,\n", + " 1.1921e-07, -1.3709e-06, -3.5763e-07, -4.1723e-07, 8.3447e-07,\n", + " 1.7881e-06, -2.5630e-06, 3.5763e-07, 1.1921e-07, 1.1921e-07,\n", + " -7.1526e-07, 5.9605e-07, 0.0000e+00, 2.3842e-07, -5.9605e-07,\n", + " -6.5565e-07, -5.3644e-07, -4.7684e-07, 1.1921e-07, -5.9605e-08,\n", + " 3.5763e-07, 1.0729e-06, -3.5763e-07, -2.3842e-07, -2.9802e-07,\n", + " -1.7881e-07, 3.5763e-07, -2.3842e-07, -1.1921e-06, -1.1921e-07,\n", + " -1.3113e-06, -4.7684e-07, -3.5763e-07, -5.9605e-07, 1.1921e-06,\n", + " -8.3447e-07, 1.1921e-07, -1.0729e-06, -2.9802e-07, -3.5763e-07,\n", + " 7.1526e-07, -2.9802e-07, 2.3842e-07, -5.9605e-07, 9.5367e-07,\n", + " -8.9407e-07, 4.7684e-07, -3.5763e-07, 1.1921e-07, 3.5763e-07,\n", + " 0.0000e+00, -6.5565e-07, -4.7684e-07, -2.3842e-07, 1.1921e-07,\n", + " 1.1921e-07, 2.3842e-07, 2.3842e-07, -5.3644e-07, 3.5763e-07,\n", + " -9.5367e-07, 0.0000e+00, -1.1921e-07, -5.9605e-08, -1.1921e-07,\n", + " 8.3447e-07, 4.7684e-07, 3.5763e-07, -3.5763e-07, -1.7881e-07,\n", + " -5.9605e-08, -5.9605e-08, -5.9605e-08, -2.9802e-07, -4.7684e-07,\n", + " 1.1921e-07, -8.3447e-07, -5.9605e-07, -6.5565e-07, -1.9073e-06,\n", + " 7.1526e-07, 3.5763e-07, -3.5763e-07, 8.3447e-07, -1.3113e-06,\n", + " 3.5763e-07, -2.3842e-07, -2.3842e-07, -3.5763e-07, 4.7684e-07,\n", + " 1.1921e-07, -3.5763e-07, -4.1723e-07, 1.1921e-07, 5.9605e-07,\n", + " -5.9605e-07, 1.4305e-06, 3.5763e-07, -2.3842e-07, -2.9802e-07,\n", + " -2.3842e-07, -4.1723e-07, 1.0729e-06, -7.7486e-07, -5.3644e-07,\n", + " 3.5763e-07, 4.7684e-07, -3.5763e-07, -3.5763e-07, -7.1526e-07,\n", + " -2.3842e-07, -7.7486e-07, 4.7684e-07, -1.1921e-07, -8.3447e-07,\n", + " 2.3842e-07, -5.9605e-08, -5.9605e-08, 7.1526e-07, -8.3447e-07,\n", + " 3.5763e-07, 7.1526e-07, 3.5763e-07, 2.3842e-07, 0.0000e+00,\n", + " 1.5497e-06, -1.7881e-07, 0.0000e+00, 1.1921e-07, 9.5367e-07,\n", + " 5.9605e-07, 1.1921e-07, 3.5763e-07, 1.1921e-07, -1.0729e-06,\n", + " 8.3447e-07, 2.3842e-07, -5.3644e-07, -1.7881e-07, -3.5763e-07,\n", + " -7.1526e-07, 8.3447e-07, 7.1526e-07, 3.5763e-07, 8.3447e-07,\n", + " 5.9605e-07, 1.1921e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", + " -5.9605e-08, -2.3842e-07, -2.9802e-07, 8.3447e-07, 9.5367e-07,\n", + " -1.1325e-06, -7.7486e-07, 9.5367e-07, 2.3842e-07, 7.1526e-07,\n", + " -8.3447e-07, -5.9605e-08, 1.4305e-06, -2.3842e-07, -3.5763e-07,\n", + " -7.1526e-07, 9.5367e-07, -1.6093e-06, -4.7684e-07, 2.3842e-07,\n", + " -2.0266e-06, 5.9605e-07, -4.7684e-07, 3.5763e-07, 1.0729e-06,\n", + " 9.5367e-07, 2.3842e-07, 8.3447e-07, -1.0729e-06, 1.1921e-06,\n", + " -2.9802e-07, -4.1723e-07, -1.7881e-07, -3.5763e-07, -9.5367e-07,\n", + " 8.3447e-07, -2.3842e-07, 9.5367e-07, -1.1921e-07, -8.3447e-07,\n", + " -7.1526e-07, -3.5763e-07, -2.3842e-07, -1.7881e-07, -3.5763e-07,\n", + " 1.3113e-06, -1.0133e-06, -4.7684e-07, -9.5367e-07, -8.9407e-07,\n", + " 1.0729e-06, -7.1526e-07, 1.1921e-07, -1.0729e-06, -5.3644e-07,\n", + " -2.9802e-07, -5.9605e-08, -7.1526e-07, -3.5763e-07, 1.1921e-07,\n", + " -4.1723e-07, 7.1526e-07, 1.1921e-06, 4.7684e-07, -1.0133e-06,\n", + " -4.1723e-07, 1.1921e-07, 3.5763e-07, -1.7881e-07, -7.7486e-07,\n", + " 1.1921e-07, 4.7684e-07, 4.7684e-07, 8.3447e-07, 5.9605e-07,\n", + " 1.0729e-06, -2.3842e-07, -8.3447e-07, 5.9605e-07, 0.0000e+00,\n", + " 1.1921e-07, 3.5763e-07, 7.1526e-07, -2.3842e-07, -5.3644e-07,\n", + " 0.0000e+00, 3.5763e-07, -5.9605e-08, -5.9605e-08, 4.7684e-07,\n", + " 1.1921e-07, -5.9605e-07, 2.3842e-07, 0.0000e+00, 9.5367e-07,\n", + " -4.7684e-07, 1.1921e-07, -4.7684e-07, 9.5367e-07, 7.1526e-07,\n", + " 0.0000e+00, 1.1921e-07, -1.7881e-07, 4.7684e-07, -2.3842e-07,\n", + " 1.1921e-07, -1.1921e-07, -5.3644e-07, -2.9802e-07, -9.5367e-07,\n", + " -1.1921e-07, -8.9407e-07, 8.3447e-07, 3.5763e-07, 2.3842e-07,\n", + " -4.1723e-07, -5.9605e-08, -3.5763e-07, -7.1526e-07, 3.5763e-07,\n", + " 0.0000e+00, -4.7684e-07, 0.0000e+00, 8.3447e-07, -1.1921e-07,\n", + " -2.9802e-07, -1.1325e-06, -5.9605e-07, -2.3842e-07, 3.5763e-07,\n", + " -2.9802e-07, -7.7486e-07, 3.5763e-07, 9.5367e-07, -4.7684e-07,\n", + " -5.9605e-08, -5.9605e-08, -2.9802e-07, 7.1526e-07, 7.1526e-07,\n", + " 5.9605e-07, -2.3842e-07, 2.3842e-07, 8.3447e-07, 3.5763e-07,\n", + " 1.1921e-07, -2.3842e-07, 4.7684e-07, -4.1723e-07, 0.0000e+00,\n", + " -5.9605e-07, -1.0133e-06, 4.7684e-07, 1.6689e-06, -7.7486e-07,\n", + " -2.9802e-07, 3.5763e-07, 4.7684e-07, 2.3842e-07, 4.7684e-07,\n", + " -4.1723e-07, -7.7486e-07, -1.1921e-07, -5.9605e-08, 5.9605e-07,\n", + " 1.1921e-07, -5.9605e-07, 3.5763e-07, 1.1921e-07, -1.7881e-07,\n", + " 3.5763e-07, -2.3842e-07, -3.5763e-07, -2.3842e-07, 1.4305e-06,\n", + " -2.3842e-07, -2.3842e-07, 8.3447e-07, -2.9802e-07, -3.5763e-07,\n", + " -3.5763e-07, -2.3842e-07, -5.9605e-07, 0.0000e+00, -1.4305e-06,\n", + " 2.3842e-07, 3.5763e-07, 3.5763e-07, -6.5565e-07, 4.7684e-07,\n", + " 1.1921e-07, 1.1921e-07, 1.1921e-07, 2.3842e-07, -8.3447e-07,\n", + " -8.3447e-07, 1.1921e-07, 1.1921e-07, 3.5763e-07, -5.3644e-07,\n", + " 3.5763e-07, -3.5763e-07, -7.7486e-07, -3.5763e-07, 7.1526e-07,\n", + " 9.5367e-07, -2.9802e-07, 2.3842e-07, -7.7486e-07, 4.7684e-07,\n", + " -1.1921e-07, -4.1723e-07, 4.7684e-07, 1.1921e-07, -2.9802e-07,\n", + " -4.7684e-07, 0.0000e+00, 5.9605e-07, -5.9605e-08, -6.5565e-07,\n", + " -3.5763e-07, 9.5367e-07, 8.3447e-07, -5.3644e-07, 7.1526e-07,\n", + " -1.7285e-06, 1.1921e-07, -2.9802e-07, 1.1921e-07, 1.1921e-07,\n", + " 3.5763e-07, 1.1921e-06, 1.1921e-07, -1.0729e-06, -1.7881e-07,\n", + " 3.5763e-07, 2.3842e-07, -1.0729e-06, -1.1921e-07, 0.0000e+00,\n", + " 1.1921e-07, -7.1526e-07, 4.7684e-07, -2.3842e-07, -2.9802e-07,\n", + " -2.9802e-07, 3.5763e-07, 9.5367e-07, 4.7684e-07, -2.9802e-07,\n", + " 4.7684e-07, -4.7684e-07, -5.9605e-07, 3.5763e-07, 3.5763e-07,\n", + " -4.1723e-07, -9.5367e-07, -5.9605e-08, -7.7486e-07, -7.7486e-07,\n", + " 4.7684e-07, -1.6689e-06, -4.1723e-07, -2.9802e-07, 3.5763e-07,\n", + " -4.7684e-07, 2.3842e-07, -6.5565e-07, -6.5565e-07, -6.5565e-07,\n", + " 2.3842e-07, -1.1921e-06, 2.3842e-07, 3.5763e-07, 4.7684e-07,\n", + " 4.7684e-07, -1.1921e-07, 2.3842e-07, 1.1921e-07, -1.1921e-07,\n", + " 1.1921e-07, 7.1526e-07, 3.5763e-07, -4.1723e-07, 2.3842e-07,\n", + " 0.0000e+00, 4.7684e-07, -8.9407e-07, -5.9605e-07, 4.7684e-07,\n", + " 3.5763e-07, 0.0000e+00, -1.1921e-07, -2.9802e-07, -1.7881e-07,\n", + " -5.9605e-07, 3.5763e-07, -1.1921e-07, 8.3447e-07, -7.7486e-07,\n", + " -6.5565e-07, 0.0000e+00, -2.9802e-07, -1.1921e-07, 5.9605e-07,\n", + " 5.9605e-07, 0.0000e+00, 1.1921e-07, -1.7881e-07, 7.1526e-07,\n", + " -4.7684e-07, 9.5367e-07], device='cuda:0')),\n", + " ('layer4.0.shortcut.1.bias',\n", + " tensor([ 1.2267e-07, 5.2131e-07, 4.9048e-07, -6.6727e-08, 5.8954e-07,\n", + " 2.0044e-07, 3.3354e-07, 1.0084e-07, 3.4111e-07, 1.0555e-07,\n", + " 4.8835e-07, -1.1029e-06, 2.0666e-08, 2.0148e-07, 2.1696e-07,\n", + " -7.4097e-07, 3.7880e-08, 6.4895e-08, -8.4100e-07, -3.1968e-07,\n", + " 1.0362e-07, -9.7019e-07, -3.3352e-07, -4.1362e-07, 4.1598e-08,\n", + " 2.2223e-07, 1.6852e-07, -1.0521e-07, 2.1346e-07, -1.2921e-06,\n", + " 3.7748e-07, -5.9054e-07, -1.5242e-08, -5.5715e-07, -2.1764e-07,\n", + " 2.1038e-07, 7.7662e-07, 9.8997e-07, 2.4168e-07, 8.6350e-08,\n", + " 1.8026e-07, 9.3270e-08, 2.4967e-07, -5.6322e-07, -3.2481e-07,\n", + " 5.5609e-09, 2.3633e-07, -4.5999e-07, 6.0220e-08, 6.4592e-07,\n", + " 2.2764e-07, -1.9812e-07, -9.3918e-07, -3.2255e-07, 2.1820e-07,\n", + " 1.5435e-06, -1.6022e-06, 9.6016e-08, -2.3592e-07, 7.2372e-07,\n", + " -4.9385e-07, 1.8043e-07, -4.0969e-07, 2.5857e-07, -5.5846e-07,\n", + " 3.3289e-07, 3.6472e-07, -9.9066e-07, -6.4836e-07, -5.9628e-08,\n", + " 7.9305e-07, 5.1530e-07, 3.1524e-07, -3.6323e-07, 3.4015e-07,\n", + " 1.8533e-07, -2.7069e-07, 2.8576e-07, -5.2353e-07, -1.3846e-07,\n", + " -1.7000e-06, 3.0305e-07, -8.1954e-07, -9.5841e-07, 3.0598e-07,\n", + " -8.9424e-07, 2.0526e-07, -5.2114e-07, 3.8126e-07, -8.6022e-07,\n", + " 9.4818e-07, -1.4355e-07, 2.4742e-07, -4.0967e-07, 1.2586e-07,\n", + " -9.7055e-07, 1.4553e-06, 1.1442e-07, 8.6497e-07, 1.3247e-07,\n", + " -5.0140e-07, -2.5456e-07, -2.6590e-08, -3.5772e-07, 8.2593e-07,\n", + " 5.3295e-07, -1.0410e-07, 4.3498e-07, 2.3294e-07, 4.7580e-07,\n", + " -5.4222e-07, 4.6807e-07, 3.8665e-07, -5.5240e-07, 4.0680e-07,\n", + " 4.8383e-07, -3.1379e-07, 2.7812e-07, -8.6844e-07, -4.2003e-08,\n", + " 3.9490e-07, -2.1611e-07, 4.1244e-08, -7.7546e-07, -2.5390e-07,\n", + " 7.5168e-07, -1.0881e-06, -3.0041e-09, -1.2022e-06, -1.0697e-06,\n", + " 2.1438e-07, -2.7698e-07, 1.0052e-07, -2.8831e-07, -7.9248e-07,\n", + " -1.1695e-08, 3.7242e-07, 3.6402e-07, -4.4123e-07, 6.7511e-08,\n", + " 6.2402e-07, -7.6874e-07, -7.7209e-07, -4.6478e-07, 8.0412e-07,\n", + " -1.1367e-06, 7.2712e-07, -1.3130e-07, -5.0524e-07, 2.0572e-07,\n", + " -4.2604e-07, 1.5546e-07, 9.0354e-07, -5.7759e-07, -6.8648e-07,\n", + " 3.3436e-07, 4.4954e-07, 3.9408e-07, -6.5580e-07, -9.3324e-07,\n", + " -6.8870e-07, -6.4934e-07, -1.5007e-07, -5.6741e-08, -9.6720e-07,\n", + " 4.3615e-07, -1.5925e-07, -8.6850e-07, 4.7006e-07, -6.9503e-07,\n", + " 8.5696e-07, 4.0076e-07, 5.0084e-07, 3.3838e-07, 1.2934e-07,\n", + " 1.0382e-06, -4.2320e-07, 3.1105e-07, 9.0592e-07, 5.3915e-07,\n", + " -2.8505e-07, 3.0121e-07, -3.5367e-07, 3.1758e-07, -9.3912e-07,\n", + " 7.7104e-07, 4.0988e-07, 4.6698e-08, 3.6261e-07, -2.4688e-07,\n", + " -5.6081e-07, 5.2616e-07, 3.9207e-07, 3.3400e-07, 1.3801e-07,\n", + " 1.5755e-07, 1.3671e-07, -2.9247e-07, -3.3614e-08, 2.7387e-07,\n", + " 4.1058e-08, 1.0102e-07, 5.9427e-07, 6.9601e-07, 5.1401e-07,\n", + " -9.0142e-07, -7.8599e-07, 8.5890e-07, 8.0110e-07, -2.6069e-08,\n", + " -4.7613e-07, -1.1880e-07, 6.0977e-07, -3.2545e-08, -8.0672e-07,\n", + " -4.4460e-07, 4.9622e-08, -1.0001e-06, -3.8140e-07, 2.0876e-07,\n", + " -7.7196e-07, 7.1734e-07, -4.6298e-07, 6.7959e-07, 1.0245e-06,\n", + " 4.1416e-07, 5.4496e-07, 7.8721e-07, -4.2891e-07, 1.6825e-06,\n", + " 1.4942e-07, -1.6828e-07, 1.7415e-07, 4.0402e-08, -5.3157e-07,\n", + " 1.0582e-07, -2.0883e-07, 4.7203e-07, -1.3684e-07, -1.3398e-06,\n", + " -5.9472e-07, 3.9554e-07, 2.3206e-07, -2.5863e-07, -4.5267e-07,\n", + " 1.0301e-06, -5.3752e-07, -7.6298e-08, -5.4183e-07, 2.8119e-07,\n", + " 7.9946e-08, -1.5766e-07, 1.8332e-07, -8.2711e-07, -3.7360e-07,\n", + " -1.5190e-07, -3.5914e-08, 3.6115e-07, -1.4778e-07, 8.6465e-07,\n", + " -2.9969e-08, 2.9304e-07, 2.5676e-07, -2.8295e-07, -8.0901e-07,\n", + " -3.3888e-07, -1.1907e-07, 2.6471e-07, 1.6580e-07, -6.6758e-08,\n", + " 2.1079e-07, 3.7526e-07, 3.1104e-07, 8.7153e-07, 3.5282e-07,\n", + " 4.8555e-07, -2.3091e-07, -5.6911e-07, 1.3365e-07, 3.0886e-07,\n", + " 2.5100e-07, -2.3340e-07, 4.3298e-07, -3.9339e-07, -6.6168e-07,\n", + " 2.4621e-07, 1.2055e-07, 5.7209e-08, -5.5472e-07, 8.4360e-07,\n", + " -8.6571e-07, 1.9837e-07, 5.0580e-08, 4.3711e-07, 4.6089e-07,\n", + " 1.3192e-07, 1.8432e-08, -4.0834e-07, 2.2995e-07, 9.0939e-07,\n", + " -6.0561e-07, 4.8501e-07, -6.9802e-07, 4.8374e-07, -3.7478e-07,\n", + " -3.0589e-07, -2.6858e-07, -4.6554e-08, -1.0507e-07, -2.8908e-07,\n", + " -1.0337e-06, -1.0129e-06, 2.2487e-07, 1.7116e-07, -4.1344e-07,\n", + " -4.2085e-07, -3.7954e-08, -5.6785e-07, -5.8464e-07, 1.2217e-08,\n", + " -9.5578e-08, -2.8373e-07, 2.6618e-07, 7.6734e-07, -8.6086e-07,\n", + " 3.6636e-07, -9.9598e-07, -1.5478e-07, 1.7445e-07, -5.5791e-07,\n", + " -7.7129e-07, -2.5449e-07, 2.9259e-07, 2.6276e-07, -7.1697e-08,\n", + " -5.8335e-07, 1.1469e-08, 2.1788e-07, 1.0519e-06, 5.5815e-08,\n", + " 4.8869e-07, -6.1105e-08, 2.8824e-08, 5.1238e-07, -2.2381e-08,\n", + " -1.9842e-07, 1.5346e-07, 2.2163e-07, -3.7323e-07, 2.4546e-07,\n", + " -5.5839e-07, -1.2237e-06, 7.6679e-07, 1.1058e-06, -6.1683e-07,\n", + " -5.6177e-07, -4.2822e-07, -2.7411e-07, 2.8429e-09, -1.7565e-07,\n", + " -6.2889e-07, -5.5862e-07, -5.8493e-07, -4.7978e-07, 6.6172e-07,\n", + " 6.3017e-08, -5.3413e-07, 8.2590e-07, -2.3073e-07, -2.7192e-07,\n", + " 5.1345e-08, -4.7095e-07, -7.3116e-07, -7.4617e-07, 1.2349e-06,\n", + " -5.5501e-08, -3.8916e-07, 4.9881e-07, -5.8128e-07, -1.0835e-06,\n", + " 8.9368e-08, -2.4767e-08, -1.7907e-07, 7.5381e-07, -9.3912e-07,\n", + " 1.6275e-07, 8.3961e-07, 1.9004e-07, -6.1507e-07, 1.7484e-07,\n", + " 6.1234e-07, -3.3452e-07, 1.7429e-07, 6.1249e-07, -4.0872e-07,\n", + " -8.5998e-07, -2.6819e-08, 4.6977e-07, -4.8661e-07, -3.7181e-07,\n", + " -4.3231e-07, -4.4359e-07, 1.0288e-07, -4.6875e-07, 1.2184e-07,\n", + " 5.4193e-07, 1.6957e-07, 1.2930e-08, -8.3377e-07, 2.0160e-07,\n", + " -1.1283e-06, 4.6074e-08, 7.9631e-08, -1.9956e-07, -2.8262e-07,\n", + " -2.7352e-07, -3.9198e-07, 9.0809e-07, -6.9796e-07, -7.7386e-07,\n", + " -6.3717e-07, 1.9364e-07, 4.2499e-07, -2.1963e-07, 3.3570e-07,\n", + " -1.1842e-06, -1.3149e-07, -2.0294e-07, -8.4020e-08, -1.8796e-07,\n", + " -8.5904e-08, 1.0188e-06, 1.1152e-07, -7.9846e-07, -1.3746e-07,\n", + " 3.0789e-07, 1.1175e-07, -3.5673e-07, -3.4911e-08, -2.8830e-08,\n", + " -2.7811e-07, 6.8873e-08, 6.6728e-07, -2.9486e-07, 6.0918e-07,\n", + " -1.2829e-06, 6.8836e-07, 3.0841e-07, 4.4511e-07, 3.0244e-07,\n", + " 4.3634e-07, 3.6050e-08, 5.0360e-07, -5.9110e-07, 9.7646e-07,\n", + " -9.2563e-07, -4.3873e-07, 1.1422e-07, -1.3625e-07, -6.4917e-07,\n", + " 5.3954e-08, -1.5143e-06, -5.9071e-07, -2.4121e-08, -1.0838e-06,\n", + " -4.0250e-07, 5.5546e-07, -7.1030e-07, -1.0235e-06, -9.1596e-08,\n", + " 1.7611e-08, -1.6254e-06, 5.3340e-07, 5.0056e-07, 2.9434e-07,\n", + " 3.0389e-07, -6.4017e-07, 3.4055e-07, -2.9545e-07, 1.7299e-07,\n", + " -3.5783e-08, 3.9323e-07, 1.4520e-08, 2.4158e-09, -8.8659e-07,\n", + " -2.5928e-07, -4.6700e-07, -4.3284e-07, 7.8656e-08, 2.4758e-07,\n", + " 7.3693e-08, 2.9827e-08, -5.8056e-07, -9.8295e-08, -2.7469e-08,\n", + " -6.7635e-07, 5.9764e-08, 9.8274e-08, 1.2037e-08, -3.1155e-07,\n", + " -2.1793e-07, 6.5858e-08, 4.3835e-08, 1.8843e-07, 1.4375e-07,\n", + " 7.0740e-08, 9.6125e-08, 1.4013e-07, 6.2235e-07, 1.1469e-06,\n", + " -4.6511e-09, 2.3411e-07], device='cuda:0')),\n", + " ('linear.weight',\n", + " tensor([[-1.0904e-05, 9.6373e-06, -3.4794e-06, ..., 1.5184e-05,\n", + " -4.0233e-07, 8.1211e-07],\n", + " [ 2.1834e-05, 6.3516e-06, 9.2201e-06, ..., -2.7325e-06,\n", + " 8.5123e-06, 1.3521e-05],\n", + " [ 4.4946e-06, 2.2314e-06, 4.9248e-06, ..., -5.9907e-06,\n", + " 6.3404e-06, 8.9928e-06],\n", + " ...,\n", + " [ 2.6450e-07, -9.1344e-06, -2.0359e-06, ..., -5.2564e-06,\n", + " -6.4522e-06, 2.2016e-06],\n", + " [ 1.2709e-05, -6.8583e-06, -2.1979e-06, ..., 7.8660e-06,\n", + " 1.2752e-05, 1.5881e-05],\n", + " [ 2.4494e-07, -1.4435e-07, 2.5369e-06, ..., 4.0978e-07,\n", + " 1.2223e-05, -1.4249e-07]], device='cuda:0')),\n", + " ('linear.bias',\n", + " tensor([ 4.1835e-06, 1.4639e-05, 1.2632e-05, 8.4601e-06, -2.2002e-05,\n", + " -9.7230e-06, 7.1898e-07, -1.4666e-05, 6.3330e-06, -5.7463e-07],\n", + " device='cuda:0'))])" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "param_diff" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[tensor([[[[ 1.3094e-06, 2.2575e-06, 9.5367e-07],\n", + " [-3.5763e-07, -5.7369e-07, -1.7434e-06],\n", + " [-1.3709e-06, -3.0920e-06, -3.6210e-06]],\n", + " \n", + " [[ 5.0340e-06, 5.3197e-06, 4.6045e-06],\n", + " [ 2.2957e-06, 1.7583e-06, 7.5251e-07],\n", + " [ 1.6615e-06, -4.3213e-07, -8.0466e-07]],\n", + " \n", + " [[ 1.4380e-06, 1.3784e-06, 1.5944e-06],\n", + " [-1.5199e-06, -2.6003e-06, -2.5854e-06],\n", + " [-1.8328e-06, -4.6492e-06, -4.3511e-06]]],\n", + " \n", + " \n", + " [[[-4.0978e-06, -9.0785e-06, -7.7784e-06],\n", + " [-1.8328e-06, -3.0994e-06, -4.8429e-07],\n", + " [-1.9227e-06, 3.5604e-06, 6.3498e-06]],\n", + " \n", + " [[-3.7700e-06, -8.8876e-06, -8.1509e-06],\n", + " [-5.2527e-07, -2.9840e-06, -2.8312e-07],\n", + " [-3.0547e-06, 1.2964e-06, 3.2708e-06]],\n", + " \n", + " [[-3.4943e-06, -9.6448e-06, -7.7486e-06],\n", + " [ 6.5565e-07, -3.5614e-06, -4.3213e-07],\n", + " [-2.4363e-06, -7.0967e-07, 1.7583e-06]]],\n", + " \n", + " \n", + " [[[ 2.6673e-05, 2.1972e-05, 1.6913e-05],\n", + " [ 2.3261e-05, 2.2944e-05, 2.2389e-05],\n", + " [ 2.5196e-05, 2.5854e-05, 2.5735e-05]],\n", + " \n", + " [[ 3.0279e-05, 2.6144e-05, 2.2963e-05],\n", + " [ 3.0980e-05, 2.7031e-05, 2.6032e-05],\n", + " [ 3.0085e-05, 2.8834e-05, 2.9545e-05]],\n", + " \n", + " [[ 2.9221e-05, 2.6941e-05, 2.4391e-05],\n", + " [ 3.1814e-05, 3.2485e-05, 3.1255e-05],\n", + " [ 3.1918e-05, 3.4809e-05, 3.3238e-05]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 2.8461e-06, 1.1273e-05, 7.7039e-06],\n", + " [ 1.2405e-06, 6.9886e-06, 5.4389e-06],\n", + " [ 2.8685e-06, 5.6028e-06, 7.4003e-06]],\n", + " \n", + " [[ 3.9190e-06, 1.0349e-05, 5.9307e-06],\n", + " [ 7.4506e-09, 3.5614e-06, 1.1474e-06],\n", + " [-5.8115e-07, 1.0058e-07, 1.2144e-06]],\n", + " \n", + " [[ 3.8743e-06, 9.0078e-06, 5.7509e-06],\n", + " [-1.3225e-06, 1.2368e-06, 1.7881e-07],\n", + " [-3.2336e-06, -3.4049e-06, -1.4156e-06]]],\n", + " \n", + " \n", + " [[[ 8.3447e-07, 4.5300e-06, 7.3425e-06],\n", + " [ 4.7684e-06, 4.2021e-06, 6.7465e-06],\n", + " [ 5.5358e-06, 7.1377e-06, 8.0615e-06]],\n", + " \n", + " [[-6.8042e-06, -6.2436e-06, -2.8461e-06],\n", + " [ 1.4007e-06, -2.7195e-06, -6.1840e-07],\n", + " [ 1.8179e-06, 1.0058e-06, 2.3842e-06]],\n", + " \n", + " [[-6.6310e-06, -6.3963e-06, -3.5614e-06],\n", + " [ 5.5134e-07, -3.4869e-06, -9.1735e-07],\n", + " [ 3.1143e-06, 1.7732e-06, 3.7588e-06]]],\n", + " \n", + " \n", + " [[[ 4.8280e-06, 2.9951e-06, 6.4448e-06],\n", + " [-7.3258e-06, -5.4464e-06, -1.7434e-06],\n", + " [-1.1042e-05, -4.9919e-06, -6.2287e-06]],\n", + " \n", + " [[ 1.0684e-05, 1.1906e-05, 1.2517e-05],\n", + " [-2.6673e-06, 2.7418e-06, 4.4033e-06],\n", + " [-8.4639e-06, 3.8743e-07, -1.0952e-06]],\n", + " \n", + " [[ 1.0833e-05, 1.4074e-05, 1.5259e-05],\n", + " [-4.4703e-07, 6.6757e-06, 8.8885e-06],\n", + " [-5.3793e-06, 4.7088e-06, 3.7588e-06]]]], device='cuda:0'),\n", + " tensor([ 5.7220e-06, 7.1526e-07, -5.3644e-07, 6.4373e-06, -4.9472e-06,\n", + " 3.2187e-06, -1.9670e-06, -6.4373e-06, 9.5367e-07, 4.7684e-07,\n", + " -3.0398e-06, 5.3644e-06, -2.6226e-06, -1.3113e-06, 0.0000e+00,\n", + " -7.8082e-06, -3.3975e-06, 1.6689e-06, -2.2054e-06, -3.6955e-06,\n", + " -1.3113e-06, 3.6955e-06, -3.3975e-06, 3.6955e-06, 4.4107e-06,\n", + " -9.4771e-06, -3.5167e-06, 9.7752e-06, -2.4438e-06, -8.3447e-07,\n", + " -4.8280e-06, 1.0729e-06, -3.1590e-06, -4.2915e-06, -2.4438e-06,\n", + " -3.0994e-06, -1.0848e-05, 5.1260e-06, 2.9802e-06, -1.5497e-06,\n", + " -1.6689e-06, -4.4703e-06, 4.4107e-06, 1.0729e-06, -2.9802e-07,\n", + " 8.8215e-06, -4.5896e-06, 6.0797e-06, 9.7752e-06, 5.2452e-06,\n", + " 1.6689e-06, -4.1723e-07, 2.2650e-06, -6.5565e-06, 8.3447e-07,\n", + " 6.3181e-06, -4.7088e-06, -4.3511e-06, -4.2915e-06, 4.6492e-06,\n", + " -1.7881e-07, 1.9073e-06, 1.4305e-06, -8.3447e-07], device='cuda:0'),\n", + " tensor([ 5.1800e-06, -5.8434e-07, 1.9089e-06, 4.9003e-06, -1.9461e-06,\n", + " -3.6183e-06, -1.1956e-06, -1.8275e-06, -2.3727e-06, 2.0175e-06,\n", + " -9.0375e-07, 5.2483e-06, -4.0264e-06, -1.9131e-06, 6.7097e-07,\n", + " -1.0581e-05, -3.0417e-06, 1.6963e-06, 2.1104e-06, -2.6472e-06,\n", + " -3.2058e-06, 3.0026e-06, 1.8405e-06, 1.6015e-07, 3.5247e-07,\n", + " -3.5816e-06, 1.2254e-06, 3.6254e-06, -3.3752e-06, -1.8590e-06,\n", + " -4.0213e-06, 1.7876e-06, -5.8218e-06, -7.0407e-06, -3.8052e-06,\n", + " -4.0191e-06, -6.2535e-06, 4.4571e-06, 1.0564e-06, 4.3555e-07,\n", + " -5.3791e-06, -8.4146e-07, 4.8352e-06, 1.5922e-06, -1.8305e-06,\n", + " 6.3709e-06, -1.9585e-06, 4.7529e-06, -6.0682e-06, 3.9466e-06,\n", + " -2.0051e-06, -2.7619e-06, 4.9631e-06, -5.6562e-06, -1.1060e-06,\n", + " 1.8016e-06, -2.9148e-06, -5.2561e-07, -3.8957e-06, 6.7037e-06,\n", + " 1.4417e-06, 4.2739e-07, -5.6229e-06, 2.4895e-06], device='cuda:0'),\n", + " tensor([[[[-1.1077e-05, -1.3351e-05, -1.3445e-05],\n", + " [-1.1542e-05, -1.2651e-05, -1.2247e-05],\n", + " [-8.0392e-06, -8.5840e-06, -9.3412e-06]],\n", + " \n", + " [[ 9.7696e-07, 2.9877e-06, 5.6960e-06],\n", + " [-2.4959e-07, 2.9840e-06, 3.9032e-06],\n", + " [ 1.7248e-06, 1.4092e-06, 3.3677e-06]],\n", + " \n", + " [[ 1.6801e-06, 1.3839e-06, 4.0382e-06],\n", + " [-8.3447e-07, -1.2778e-06, 9.8720e-07],\n", + " [-7.5519e-06, -6.0269e-06, -4.8466e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 6.1188e-07, 1.4915e-06, 3.4235e-06],\n", + " [ 8.2608e-07, 1.5702e-06, 3.3225e-06],\n", + " [-1.4435e-07, 5.6624e-07, 8.0466e-07]],\n", + " \n", + " [[-5.9982e-07, 2.5332e-07, 3.5102e-06],\n", + " [-2.6356e-07, 2.7977e-06, 3.7896e-06],\n", + " [ 3.4086e-07, 1.5418e-06, 1.4640e-06]],\n", + " \n", + " [[ 3.3891e-06, 3.9116e-07, 1.1139e-06],\n", + " [ 6.0275e-06, 3.4408e-06, 1.3205e-06],\n", + " [ 8.8597e-06, 4.2245e-06, 3.3457e-06]]],\n", + " \n", + " \n", + " [[[ 5.8608e-06, 6.4773e-06, 7.9181e-06],\n", + " [ 2.2501e-06, 1.9427e-06, 3.5204e-06],\n", + " [ 4.7162e-06, 4.3027e-06, 4.3828e-06]],\n", + " \n", + " [[-7.2327e-06, -8.4508e-06, -4.1667e-06],\n", + " [-5.6550e-06, -4.2372e-06, -6.7614e-07],\n", + " [-3.0911e-06, -8.1584e-07, -5.8832e-06]],\n", + " \n", + " [[ 5.8059e-06, 8.8625e-06, 2.9579e-06],\n", + " [-6.4736e-06, -8.4308e-07, 3.4049e-06],\n", + " [-8.2515e-07, 4.6254e-06, 8.2117e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.2107e-06, 2.6133e-06, 3.4831e-07],\n", + " [-1.0639e-05, -5.5619e-06, -5.5544e-06],\n", + " [-9.5721e-06, -5.6596e-06, -3.7905e-06]],\n", + " \n", + " [[-1.8440e-06, 2.0619e-06, 1.4417e-06],\n", + " [-1.1794e-05, -2.8946e-06, -5.0736e-06],\n", + " [-1.1954e-05, -5.2908e-06, -5.1484e-06]],\n", + " \n", + " [[-6.2555e-06, -1.2591e-06, -2.0694e-06],\n", + " [-8.8848e-07, -7.8976e-07, -1.4305e-06],\n", + " [ 6.2720e-06, 3.3956e-06, 9.1093e-06]]],\n", + " \n", + " \n", + " [[[-3.9963e-06, -8.3866e-06, -3.3099e-06],\n", + " [-3.6014e-06, -8.0690e-06, -5.1260e-06],\n", + " [-4.9341e-06, -6.4764e-06, -3.4515e-06]],\n", + " \n", + " [[ 1.1835e-05, 1.0084e-05, 8.2310e-06],\n", + " [ 1.4350e-05, 7.1451e-06, 6.6385e-06],\n", + " [ 1.6532e-05, 1.2064e-05, 9.2201e-06]],\n", + " \n", + " [[ 1.3463e-05, 1.0386e-05, 5.9130e-06],\n", + " [ 9.9950e-06, 8.2487e-06, 7.7151e-06],\n", + " [-6.3702e-07, -3.2855e-06, 5.9232e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.5248e-05, 1.1347e-05, 8.3297e-06],\n", + " [ 1.8368e-05, 1.1865e-05, 9.6178e-06],\n", + " [ 1.6540e-05, 1.2219e-05, 1.1163e-05]],\n", + " \n", + " [[ 1.4391e-05, 6.0964e-06, 7.6713e-06],\n", + " [ 1.8489e-05, 9.6224e-06, 8.3763e-06],\n", + " [ 1.7405e-05, 1.1558e-05, 8.1407e-06]],\n", + " \n", + " [[-3.3677e-06, 1.7807e-06, -5.1223e-06],\n", + " [-5.9060e-06, 5.2527e-07, -2.8340e-06],\n", + " [-8.7004e-06, -5.4650e-06, -4.6156e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-1.7360e-06, 5.4762e-07, 2.7195e-06],\n", + " [-2.4382e-06, -2.2650e-06, -1.0524e-06],\n", + " [-2.7567e-07, -1.2275e-06, -1.7919e-06]],\n", + " \n", + " [[-5.0822e-06, -7.7039e-06, -9.2110e-06],\n", + " [-1.3849e-06, -3.0864e-06, -3.4422e-06],\n", + " [-2.2687e-06, -1.2293e-06, -9.7230e-07]],\n", + " \n", + " [[ 8.6054e-07, 8.1025e-07, 1.1176e-06],\n", + " [-2.0377e-06, -2.6319e-06, 3.0063e-06],\n", + " [ 6.4448e-07, 2.4252e-06, 8.2143e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.4145e-06, -5.3197e-06, -6.9588e-06],\n", + " [-4.0792e-06, -5.8394e-06, -5.6103e-06],\n", + " [-2.9318e-06, -1.2163e-06, -1.6280e-06]],\n", + " \n", + " [[-3.3230e-06, -4.0093e-06, -7.9703e-06],\n", + " [-2.6223e-06, -4.0159e-06, -7.2364e-06],\n", + " [-1.2740e-06, -6.3330e-08, -3.6899e-06]],\n", + " \n", + " [[-2.3842e-06, -5.4110e-07, -1.3970e-06],\n", + " [-4.8056e-06, -3.3015e-07, -6.0019e-06],\n", + " [-1.3257e-06, 1.7844e-06, -3.7998e-07]]],\n", + " \n", + " \n", + " [[[ 3.2345e-06, 4.0606e-06, 4.6827e-06],\n", + " [ 3.9581e-06, 6.2026e-06, 6.0778e-06],\n", + " [ 3.8482e-06, 7.9684e-06, 6.9216e-06]],\n", + " \n", + " [[-2.4848e-06, -7.5959e-06, -9.3263e-06],\n", + " [-1.1595e-06, -5.9567e-06, -8.2850e-06],\n", + " [-3.3425e-06, -6.2436e-06, -1.0731e-05]],\n", + " \n", + " [[ 7.7710e-06, 1.1087e-05, 2.1253e-06],\n", + " [ 2.3516e-06, 2.1737e-06, -5.2582e-06],\n", + " [-5.0201e-06, -4.3120e-06, -2.9020e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.8254e-07, -3.2522e-06, -5.4492e-06],\n", + " [ 1.1362e-06, -4.9453e-06, -6.3721e-06],\n", + " [ 9.0897e-07, -4.7460e-06, -7.0799e-06]],\n", + " \n", + " [[-2.5658e-07, -2.7902e-06, -2.6394e-06],\n", + " [ 1.7732e-06, -4.1220e-06, -5.3123e-06],\n", + " [ 3.0876e-06, -2.0824e-06, -5.3644e-06]],\n", + " \n", + " [[ 4.7442e-06, 2.9951e-06, 1.1277e-05],\n", + " [ 9.6490e-06, 7.3425e-06, 1.3027e-05],\n", + " [-1.1586e-06, -1.8999e-07, 6.1467e-06]]],\n", + " \n", + " \n", + " [[[-6.4075e-06, -6.9048e-06, -8.5756e-06],\n", + " [-6.4103e-06, -6.2175e-06, -6.0480e-06],\n", + " [-7.4264e-06, -6.6143e-06, -6.4950e-06]],\n", + " \n", + " [[ 5.3104e-06, 3.9414e-06, 8.6054e-06],\n", + " [-3.3397e-06, 4.0606e-07, 5.6159e-06],\n", + " [-4.9863e-06, -2.5965e-06, 5.1418e-06]],\n", + " \n", + " [[-7.7859e-06, -1.3482e-05, -6.9756e-06],\n", + " [ 1.9577e-05, 1.4581e-05, 1.3802e-05],\n", + " [ 1.1358e-05, 1.0703e-05, 1.2726e-05]],\n", + " \n", + " ...,\n", + " \n", + " [[ 8.5440e-06, 7.8101e-06, 9.2369e-06],\n", + " [ 7.8743e-06, 7.1749e-06, 6.1598e-06],\n", + " [ 3.6135e-06, 2.7828e-06, 8.0112e-06]],\n", + " \n", + " [[-1.7043e-06, 1.6764e-08, 2.8984e-06],\n", + " [ 1.2182e-06, 6.9477e-07, -1.3039e-06],\n", + " [ 3.5912e-06, -5.1605e-06, -2.6915e-06]],\n", + " \n", + " [[-1.9607e-06, -4.8131e-06, -6.8620e-06],\n", + " [ 5.2191e-06, -6.1467e-07, -5.7463e-06],\n", + " [-2.8685e-07, 1.7951e-07, -8.0280e-07]]]], device='cuda:0'),\n", + " tensor([ 9.5367e-07, 0.0000e+00, -1.0729e-06, -8.2254e-06, 1.3113e-06,\n", + " 7.6294e-06, -1.1921e-06, -7.7486e-06, -3.5167e-06, -1.9670e-06,\n", + " 4.6492e-06, -3.9339e-06, -5.0068e-06, 1.7881e-06, -3.8743e-06,\n", + " -3.0994e-06, 2.2650e-06, 3.5763e-07, 2.3842e-07, 8.3447e-07,\n", + " 6.6757e-06, -4.0531e-06, -9.5367e-07, 1.1921e-07, -8.9407e-07,\n", + " -1.4901e-06, -3.0398e-06, -1.7881e-06, 1.4305e-06, -7.7486e-06,\n", + " 2.3842e-07, 4.6492e-06, 0.0000e+00, -1.1921e-07, 2.9802e-06,\n", + " -1.8477e-06, 2.3842e-06, -2.2054e-06, 1.6689e-06, 9.5367e-07,\n", + " 2.3842e-06, 1.0729e-06, 6.4373e-06, 3.6955e-06, -5.3644e-07,\n", + " -1.6093e-06, -8.9407e-07, -2.0266e-06, 5.7220e-06, 2.3842e-07,\n", + " 9.5367e-07, 5.9605e-07, 7.8678e-06, -5.9605e-08, -2.2054e-06,\n", + " -3.7551e-06, -7.1526e-07, 2.3842e-06, 1.6689e-06, 2.2650e-06,\n", + " 2.1458e-06, -1.4901e-06, 2.2650e-06, -3.1590e-06], device='cuda:0'),\n", + " tensor([ 2.6768e-06, 1.9124e-06, -1.3289e-07, -8.0001e-06, 1.6482e-06,\n", + " 4.9723e-06, 2.5025e-06, -3.4393e-06, -1.3543e-06, -1.6179e-07,\n", + " 3.2728e-06, -2.7219e-06, -4.3160e-06, 1.7468e-06, -1.1131e-06,\n", + " -2.0409e-06, 2.2392e-06, -1.4375e-06, 8.6258e-07, 2.6158e-06,\n", + " 5.8557e-06, -1.5091e-06, -3.1794e-07, 3.2463e-06, 2.5008e-06,\n", + " -4.0393e-06, -8.9229e-07, -3.1561e-06, 3.7993e-07, -6.5335e-06,\n", + " 2.5505e-06, 2.9680e-06, 7.9633e-07, -3.2913e-06, 1.1366e-06,\n", + " -1.5826e-06, 2.8537e-07, 3.2345e-06, 2.9700e-06, -1.7639e-06,\n", + " 3.0974e-06, -7.6699e-07, 3.8670e-06, -9.4418e-07, 1.4069e-06,\n", + " -2.0534e-06, 5.8089e-07, -3.6296e-06, 8.4546e-07, 6.6837e-07,\n", + " 8.0962e-07, 2.2233e-06, 4.3749e-06, 2.1475e-06, -7.4168e-07,\n", + " -2.2586e-06, 5.3124e-07, 4.4179e-06, -1.9280e-06, 5.8637e-07,\n", + " 4.8589e-07, -1.7005e-06, 1.1679e-06, -4.1782e-06], device='cuda:0'),\n", + " tensor([[[[ 6.8545e-06, -2.7642e-06, 7.6927e-07],\n", + " [ 1.2424e-06, -4.1537e-06, -4.5188e-06],\n", + " [-2.1346e-06, -3.7439e-07, -4.5253e-06]],\n", + " \n", + " [[ 1.0302e-05, 1.4732e-05, 1.1850e-05],\n", + " [ 7.7300e-06, 1.0770e-05, 1.0885e-05],\n", + " [ 8.0653e-06, 1.0647e-05, 1.2951e-05]],\n", + " \n", + " [[-1.4819e-06, -2.3656e-06, -1.1160e-05],\n", + " [ 6.9290e-07, 1.1502e-07, -8.7265e-06],\n", + " [ 3.6135e-06, 5.2182e-06, -9.1409e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 9.9167e-06, 6.6161e-06, 7.1656e-06],\n", + " [ 7.6666e-06, 7.4543e-06, 3.8240e-06],\n", + " [ 1.9455e-06, 1.6578e-06, -3.4273e-06]],\n", + " \n", + " [[ 8.7172e-07, -1.5176e-06, -8.2329e-07],\n", + " [ 5.5740e-07, -2.4755e-06, -1.2703e-06],\n", + " [-1.9423e-06, -4.0457e-06, 2.4149e-06]],\n", + " \n", + " [[-7.4450e-06, -3.3551e-06, -5.1595e-06],\n", + " [ 7.1898e-07, 7.2116e-06, 3.7476e-06],\n", + " [ 4.2431e-06, 6.8992e-06, 4.1407e-06]]],\n", + " \n", + " \n", + " [[[-9.7672e-07, -1.1250e-06, -4.3670e-06],\n", + " [ 1.4119e-06, 2.3879e-06, -1.7881e-07],\n", + " [ 3.6471e-06, 1.4715e-06, 2.5146e-06]],\n", + " \n", + " [[-2.1979e-07, 8.6194e-07, -9.3132e-08],\n", + " [-1.0896e-07, 1.7686e-06, 3.8389e-06],\n", + " [ 9.4250e-07, -1.2834e-06, 1.5097e-06]],\n", + " \n", + " [[-6.5397e-06, -2.5074e-06, -5.4762e-07],\n", + " [-5.4091e-06, -3.7719e-07, -7.5344e-07],\n", + " [-4.9137e-06, 2.4606e-06, 6.5845e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.2897e-06, -7.5996e-07, 1.0803e-06],\n", + " [ 3.2000e-06, 1.3162e-06, -9.7975e-07],\n", + " [ 1.0123e-06, -3.9414e-06, -9.1726e-06]],\n", + " \n", + " [[-3.4384e-06, 1.0692e-06, 7.7486e-07],\n", + " [-5.3057e-06, -1.6671e-07, 2.1765e-06],\n", + " [-2.6673e-06, 4.4983e-07, 3.2634e-06]],\n", + " \n", + " [[-1.9129e-06, 4.4238e-07, 3.2447e-06],\n", + " [-6.3479e-06, 1.0803e-07, 3.5837e-06],\n", + " [-3.8035e-06, 2.3972e-06, 2.4568e-06]]],\n", + " \n", + " \n", + " [[[ 8.4192e-07, -1.3798e-06, 1.4901e-07],\n", + " [ 2.6077e-07, -1.9146e-06, -3.6638e-06],\n", + " [ 1.5544e-06, -4.7134e-06, -5.4822e-06]],\n", + " \n", + " [[ 6.4727e-08, 4.8503e-06, -3.9488e-07],\n", + " [-2.8275e-06, -7.8976e-07, -3.7272e-06],\n", + " [ 2.4363e-06, 1.9111e-06, -3.1069e-06]],\n", + " \n", + " [[ 4.8932e-06, 6.0722e-07, 3.9460e-06],\n", + " [ 8.0299e-06, 5.1351e-06, 1.1583e-05],\n", + " [ 3.1609e-06, 1.5479e-06, 4.0233e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.5709e-06, 3.6675e-06, 8.6054e-07],\n", + " [ 4.0866e-06, 7.2345e-06, 8.8215e-06],\n", + " [ 6.9551e-06, 1.0017e-05, 1.3247e-05]],\n", + " \n", + " [[-2.5816e-06, -2.7604e-06, -1.3150e-06],\n", + " [-3.9279e-06, -8.3931e-06, -5.7220e-06],\n", + " [ 2.4009e-06, -8.4937e-07, -3.4794e-06]],\n", + " \n", + " [[-2.2277e-06, 2.2072e-06, 2.2706e-06],\n", + " [ 9.3877e-06, 7.9777e-06, 6.1207e-06],\n", + " [ 6.4953e-06, 3.3351e-06, 6.1579e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 4.6939e-06, 3.7625e-07, 8.5309e-07],\n", + " [ 3.8845e-06, 2.5900e-06, 4.2645e-06],\n", + " [ 4.0168e-06, 2.0415e-06, 2.6468e-06]],\n", + " \n", + " [[-2.2938e-06, -4.3958e-06, -4.1798e-06],\n", + " [-3.8892e-06, -5.9064e-06, -6.6822e-06],\n", + " [-6.4662e-06, -9.3188e-06, -9.6066e-06]],\n", + " \n", + " [[ 5.9549e-06, 1.3746e-06, -1.5218e-06],\n", + " [ 1.6745e-06, -3.0203e-06, -4.0531e-06],\n", + " [ 1.2647e-06, 3.9525e-06, 6.7502e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.3013e-06, -4.0922e-06, -8.0075e-06],\n", + " [-5.6997e-06, -4.4364e-06, -8.7935e-06],\n", + " [-6.7540e-06, -6.8489e-06, -9.5926e-06]],\n", + " \n", + " [[ 4.8894e-06, 4.2394e-06, 6.2529e-06],\n", + " [ 1.3988e-06, 3.6657e-06, 5.0766e-06],\n", + " [-1.3574e-06, -1.4156e-07, -1.6112e-07]],\n", + " \n", + " [[-5.0478e-06, -9.2871e-06, -1.1311e-05],\n", + " [-5.6028e-06, -5.2424e-06, -5.3756e-06],\n", + " [-4.7777e-07, 4.3213e-07, -2.4918e-06]]],\n", + " \n", + " \n", + " [[[ 7.4701e-06, 7.1507e-06, 8.0178e-06],\n", + " [ 9.0674e-06, 9.0469e-06, 1.2387e-05],\n", + " [ 1.1539e-05, 1.2392e-05, 1.3046e-05]],\n", + " \n", + " [[ 4.3814e-06, 1.9222e-06, -7.3537e-06],\n", + " [-7.3761e-07, 4.0121e-06, -5.2750e-06],\n", + " [ 2.7195e-06, 2.8834e-06, -2.5444e-06]],\n", + " \n", + " [[-6.5621e-06, -2.8647e-06, -8.0653e-07],\n", + " [-6.2957e-07, 3.0268e-07, -2.6077e-08],\n", + " [-1.0600e-05, -1.1221e-05, -1.0191e-05]],\n", + " \n", + " ...,\n", + " \n", + " [[ 6.1952e-06, 9.4734e-06, 6.4820e-06],\n", + " [ 5.0750e-06, 7.7318e-06, 6.7148e-06],\n", + " [ 4.1593e-06, 5.5018e-06, 2.7809e-06]],\n", + " \n", + " [[ 1.4715e-06, 3.4086e-06, -1.9111e-06],\n", + " [-1.3251e-05, -8.5169e-06, -8.4005e-06],\n", + " [-8.0657e-06, -4.1851e-06, 3.4366e-07]],\n", + " \n", + " [[ 3.8259e-06, -2.9833e-06, -2.4363e-06],\n", + " [ 4.2394e-06, 1.9409e-06, -4.7944e-06],\n", + " [ 6.5044e-06, 5.3942e-06, 4.4182e-06]]],\n", + " \n", + " \n", + " [[[-2.6450e-07, -1.3839e-06, 2.3320e-06],\n", + " [ 1.5227e-06, 7.0408e-07, 1.7062e-06],\n", + " [ 7.8231e-08, 2.5351e-06, 2.8536e-06]],\n", + " \n", + " [[-1.3914e-06, -1.3160e-06, -2.5891e-06],\n", + " [-1.6969e-06, -3.2715e-06, -6.1244e-06],\n", + " [-2.7679e-06, 2.2212e-07, -2.8824e-06]],\n", + " \n", + " [[-8.6986e-07, 2.3621e-07, -2.0363e-06],\n", + " [ 1.3188e-06, 9.1437e-06, 6.4708e-06],\n", + " [ 3.9823e-06, 3.4459e-06, -3.5204e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 8.3633e-07, 4.0755e-06, -1.2666e-07],\n", + " [ 2.4959e-07, 5.6513e-06, 4.0494e-06],\n", + " [ 2.1867e-06, 5.2229e-06, 5.8115e-06]],\n", + " \n", + " [[-2.2780e-06, 3.0920e-07, 2.5649e-06],\n", + " [-4.4983e-06, -6.8564e-06, -2.4103e-06],\n", + " [-4.7367e-06, -7.6070e-06, -5.9633e-06]],\n", + " \n", + " [[-8.4396e-06, -3.3081e-06, 9.9093e-07],\n", + " [-9.4809e-07, 4.1630e-06, -4.8522e-07],\n", + " [-2.9858e-06, 4.8093e-06, -1.0617e-07]]]], device='cuda:0'),\n", + " tensor([ 3.4571e-06, -8.9407e-07, 1.5497e-06, -4.2915e-06, 3.5763e-07,\n", + " 1.0729e-06, 8.3447e-07, -4.7684e-06, -1.0133e-06, 7.1526e-07,\n", + " 3.6955e-06, 4.7684e-07, 9.5367e-07, -6.5565e-07, 2.5034e-06,\n", + " 9.5367e-07, 9.5367e-07, 0.0000e+00, -5.6028e-06, -3.3379e-06,\n", + " -1.4305e-06, 1.3113e-06, -1.0133e-06, 7.1526e-07, 9.5367e-07,\n", + " 2.9802e-06, -1.9073e-06, -1.9073e-06, 1.5497e-06, 2.3842e-07,\n", + " -9.5367e-07, 4.2915e-06, -3.4571e-06, 2.3842e-06, -3.6955e-06,\n", + " 2.5034e-06, 6.7949e-06, -1.9073e-06, -6.5565e-07, -8.9407e-07,\n", + " 3.2187e-06, 8.3447e-07, -8.9407e-07, -1.2517e-06, -5.9605e-07,\n", + " 1.5497e-06, 1.1921e-06, 4.0531e-06, 2.3842e-06, -1.4305e-06,\n", + " 4.7684e-07, -1.8477e-06, -4.0531e-06, 3.5763e-07, -2.0862e-06,\n", + " 2.9802e-06, 1.5497e-06, -1.1921e-07, 2.1458e-06, -4.8280e-06,\n", + " 9.5367e-07, 3.4571e-06, 7.1526e-07, 0.0000e+00], device='cuda:0'),\n", + " tensor([ 1.9688e-06, -6.2133e-07, -1.2541e-06, -3.2627e-06, 4.4054e-07,\n", + " -2.1605e-06, 6.9847e-07, 1.0742e-06, 3.2157e-07, -6.0075e-07,\n", + " 9.1122e-08, 1.5464e-06, -2.7741e-07, 2.3383e-06, -1.9613e-06,\n", + " 1.3622e-06, 2.9484e-06, 1.3717e-06, 5.5811e-07, 8.9026e-07,\n", + " 1.6398e-07, -9.3554e-07, -7.5569e-08, -5.7897e-07, 1.9944e-06,\n", + " -8.7225e-09, 9.2347e-08, -2.3372e-06, 1.5234e-06, -3.1647e-06,\n", + " 3.8301e-08, 1.9056e-06, -5.5854e-08, -6.9428e-08, -4.9286e-07,\n", + " -3.7826e-06, 2.9969e-06, 3.6108e-06, 1.9197e-06, 3.2961e-07,\n", + " -9.1902e-07, -1.1679e-07, -8.4645e-07, -1.3577e-06, -1.1488e-06,\n", + " 2.4970e-06, -1.8767e-06, 1.5445e-06, 1.0123e-08, 7.8204e-07,\n", + " 1.1457e-06, 2.8526e-07, 1.4468e-06, -1.2701e-06, -1.0599e-08,\n", + " 4.3910e-06, 2.6940e-06, 1.2874e-06, -1.9065e-07, 7.5306e-07,\n", + " 4.4734e-07, -7.6610e-07, -2.6200e-06, 1.1360e-06], device='cuda:0'),\n", + " tensor([[[[ 1.3500e-06, 1.2200e-07, 3.0696e-06],\n", + " [ 2.9905e-06, 5.2340e-07, 2.3963e-06],\n", + " [ 8.2888e-07, -5.2084e-07, -6.1002e-07]],\n", + " \n", + " [[ 3.9884e-06, 2.9132e-06, 1.0747e-06],\n", + " [ 2.7344e-06, 2.6971e-06, 2.7195e-06],\n", + " [ 2.9132e-06, 3.6694e-06, -6.4285e-07]],\n", + " \n", + " [[-1.7346e-06, 4.1723e-07, 4.0587e-06],\n", + " [-6.7614e-07, -1.5125e-06, -6.2585e-07],\n", + " [ 7.1153e-07, 2.2687e-06, 2.1700e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.9116e-07, 1.4678e-06, 6.1374e-07],\n", + " [-1.2890e-06, 1.2275e-06, 3.4124e-06],\n", + " [ 2.3432e-06, 3.4329e-06, 2.3385e-06]],\n", + " \n", + " [[-1.4752e-06, 1.1772e-06, -2.5276e-06],\n", + " [ 5.6624e-07, 1.1695e-06, 1.7881e-07],\n", + " [ 1.2927e-06, 1.1288e-06, -1.4286e-06]],\n", + " \n", + " [[ 1.1679e-06, 1.4435e-07, 3.9265e-06],\n", + " [ 2.1663e-06, -1.0460e-06, 4.5449e-07],\n", + " [-4.1910e-07, -7.6974e-07, 1.1176e-07]]],\n", + " \n", + " \n", + " [[[-2.2613e-06, 4.1984e-06, 7.7821e-06],\n", + " [-8.1845e-06, -2.6878e-06, 5.1968e-07],\n", + " [-1.8049e-06, -1.2293e-06, -1.1222e-06]],\n", + " \n", + " [[ 3.8184e-07, -2.9802e-08, -1.6913e-06],\n", + " [ 5.6252e-07, 4.9546e-07, 3.0547e-07],\n", + " [ 3.8184e-08, -3.3155e-07, -2.2724e-07]],\n", + " \n", + " [[ 2.3842e-07, 1.1660e-06, 3.8650e-07],\n", + " [ 2.4214e-08, -3.2783e-06, 1.6671e-07],\n", + " [ 1.2955e-06, 2.0154e-06, 3.7756e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 9.8255e-07, 8.3819e-07, -5.6066e-07],\n", + " [ 1.3076e-06, 1.5222e-06, -2.8312e-07],\n", + " [-1.8626e-07, 5.9418e-07, -1.0990e-06]],\n", + " \n", + " [[ 6.8098e-06, 5.5621e-06, 7.4308e-06],\n", + " [ 3.7542e-06, 5.0794e-06, 4.3251e-06],\n", + " [ 5.2527e-07, 3.4962e-06, 1.3225e-06]],\n", + " \n", + " [[ 2.9802e-07, -2.0154e-06, -7.6368e-08],\n", + " [-3.1954e-06, -5.3719e-06, -5.5507e-07],\n", + " [-3.7104e-06, -4.5672e-06, -9.2573e-07]]],\n", + " \n", + " \n", + " [[[ 3.9330e-06, 8.7973e-06, 7.0743e-06],\n", + " [ 1.7071e-06, 5.0981e-06, 4.6827e-06],\n", + " [ 8.5682e-08, 1.8757e-06, 4.1388e-06]],\n", + " \n", + " [[ 6.1234e-06, 5.3542e-06, 2.7977e-06],\n", + " [ 3.6303e-06, 1.1027e-06, 5.7684e-07],\n", + " [ 2.7050e-06, 1.7211e-06, -8.3866e-07]],\n", + " \n", + " [[-2.6878e-06, 1.0747e-06, 1.2014e-06],\n", + " [-1.4827e-06, 1.5181e-06, 3.2000e-06],\n", + " [-3.1441e-06, -1.6037e-06, 2.7474e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 5.6326e-06, 6.5546e-06, 1.6820e-06],\n", + " [ 7.3407e-06, 5.3002e-06, 5.2759e-06],\n", + " [ 1.5609e-06, 7.1991e-07, 3.4682e-06]],\n", + " \n", + " [[-2.6142e-06, 5.8115e-07, -1.1213e-06],\n", + " [-7.7300e-07, 3.6694e-07, -2.5667e-06],\n", + " [ 1.0878e-06, -2.0014e-06, 1.0952e-06]],\n", + " \n", + " [[-5.3458e-06, -8.8662e-06, -2.7549e-06],\n", + " [-8.3745e-06, -1.0052e-05, -6.9044e-06],\n", + " [-5.1633e-06, -6.5006e-06, -4.2170e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 5.0134e-07, 1.0005e-06, -2.4773e-06],\n", + " [-6.4261e-07, -2.1439e-06, -3.3600e-06],\n", + " [-1.1194e-06, -4.9695e-06, -2.9542e-06]],\n", + " \n", + " [[-2.8610e-06, -3.3081e-06, -4.3809e-06],\n", + " [-3.2871e-06, -3.3900e-06, -2.3693e-06],\n", + " [-2.8647e-06, -2.7027e-06, 1.9558e-08]],\n", + " \n", + " [[ 7.8604e-07, 1.6727e-06, 1.3497e-06],\n", + " [ 2.8741e-06, 2.9523e-06, 3.1702e-06],\n", + " [ 1.6764e-07, -1.0077e-06, -2.3884e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.7162e-06, -5.6177e-06, -5.4459e-06],\n", + " [-3.8911e-06, -4.2245e-06, -4.3474e-06],\n", + " [-2.5928e-06, -2.6263e-06, -2.4382e-06]],\n", + " \n", + " [[ 2.7148e-06, -2.7232e-06, -2.0026e-06],\n", + " [-6.5565e-07, -1.7043e-06, -1.5572e-06],\n", + " [-1.8515e-06, 1.4100e-06, 5.3877e-07]],\n", + " \n", + " [[ 1.3523e-06, 9.9745e-07, -7.4506e-09],\n", + " [ 2.1365e-06, 4.2152e-06, 2.2575e-06],\n", + " [ 3.7495e-06, 1.8664e-06, 6.2212e-07]]],\n", + " \n", + " \n", + " [[[-8.6799e-07, -3.1628e-06, -4.0811e-06],\n", + " [ 1.2852e-06, -1.0431e-06, -9.0711e-07],\n", + " [ 3.3863e-06, 3.0175e-06, 3.0771e-06]],\n", + " \n", + " [[-6.7987e-07, 1.7863e-06, 1.1083e-06],\n", + " [-2.3972e-06, 3.3528e-07, -3.6508e-07],\n", + " [-2.0787e-06, -1.5255e-06, -2.6133e-06]],\n", + " \n", + " [[ 2.2016e-06, -2.4773e-07, 2.3283e-07],\n", + " [ 1.2103e-06, -9.0897e-07, 8.4937e-07],\n", + " [ 2.2613e-06, 1.6391e-06, 1.3802e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.3188e-06, 2.7479e-06, 2.4363e-06],\n", + " [ 1.2405e-06, 9.2946e-07, 1.3141e-06],\n", + " [ 1.1548e-07, 8.8802e-07, -1.2480e-07]],\n", + " \n", + " [[ 2.5444e-06, 1.1791e-06, -1.6605e-06],\n", + " [-2.9057e-07, 9.2015e-07, -2.0973e-06],\n", + " [-3.9134e-06, -2.4214e-07, 6.1002e-07]],\n", + " \n", + " [[ 2.8210e-06, 5.8301e-07, 7.4506e-07],\n", + " [ 1.6801e-06, 3.4980e-06, 1.7053e-06],\n", + " [ 2.5257e-06, 4.4499e-06, 2.0787e-06]]],\n", + " \n", + " \n", + " [[[ 7.7486e-07, 4.3772e-07, -6.1095e-07],\n", + " [ 4.6249e-06, 2.3022e-06, 2.0750e-06],\n", + " [ 4.8594e-06, 8.4843e-07, 1.3150e-06]],\n", + " \n", + " [[ 1.2554e-06, 1.5832e-06, 1.3420e-06],\n", + " [ 1.8775e-06, 2.5481e-06, 1.8846e-06],\n", + " [ 5.6103e-06, 3.4468e-06, 3.1404e-06]],\n", + " \n", + " [[ 2.4829e-06, 3.9060e-06, 4.3884e-06],\n", + " [-2.6338e-06, -8.9966e-07, 2.4522e-06],\n", + " [-1.1064e-06, -7.6788e-07, 2.1420e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.7374e-06, 7.2364e-07, 5.4017e-08],\n", + " [ 1.1660e-06, 2.5034e-06, 2.1076e-06],\n", + " [ 1.8803e-06, -7.2457e-07, -2.2762e-06]],\n", + " \n", + " [[ 3.2075e-06, 2.1812e-06, -2.8126e-06],\n", + " [ 4.7423e-06, 2.4922e-06, -2.6934e-06],\n", + " [ 2.1067e-06, -1.0394e-06, 2.6403e-07]],\n", + " \n", + " [[ 1.1399e-06, -2.1365e-06, -1.2193e-06],\n", + " [ 5.6333e-07, -1.8980e-06, 5.1130e-07],\n", + " [ 1.0626e-06, 2.8266e-07, -6.1840e-07]]]], device='cuda:0'),\n", + " tensor([ 9.5367e-07, 7.1526e-07, -1.1325e-06, 1.1921e-06, -7.7486e-07,\n", + " 1.1921e-07, 1.3113e-06, 1.1921e-07, -1.1921e-07, 4.7684e-07,\n", + " -1.4901e-06, 2.8610e-06, -7.1526e-07, -1.1921e-07, -1.7881e-06,\n", + " 2.0266e-06, 2.3842e-07, -3.4571e-06, -4.7684e-07, -1.0729e-06,\n", + " -1.0133e-06, -2.2650e-06, -1.1921e-06, 0.0000e+00, -1.0729e-06,\n", + " -7.7486e-07, -8.3447e-07, -4.1723e-07, 1.5497e-06, 1.1921e-07,\n", + " -1.3113e-06, 3.5763e-07, -7.7486e-07, -1.6689e-06, 5.9605e-07,\n", + " -5.9605e-07, -9.5367e-07, 4.4107e-06, 1.1921e-07, 2.9802e-06,\n", + " 9.5367e-07, 1.1921e-06, -2.9802e-07, -1.5497e-06, -2.1458e-06,\n", + " -1.4901e-06, -1.6689e-06, -1.7285e-06, 5.9605e-07, -2.9802e-07,\n", + " 3.5763e-07, -2.3842e-07, 1.4305e-06, -1.7285e-06, -8.3447e-07,\n", + " -7.1526e-07, -5.3644e-07, 1.4305e-06, -7.1526e-07, 0.0000e+00,\n", + " 1.6689e-06, -1.2517e-06, -4.8280e-06, 1.6689e-06, 2.5034e-06,\n", + " 1.1921e-07, 1.6689e-06, 3.2187e-06, -4.7684e-07, -7.7486e-07,\n", + " 1.0729e-06, -1.6093e-06, 1.3113e-06, -4.1723e-07, -2.6226e-06,\n", + " 1.3113e-06, -1.5497e-06, 2.3842e-07, -1.6093e-06, 3.5763e-07,\n", + " 5.9605e-07, 5.9605e-07, -5.3644e-07, 1.1921e-07, 1.1921e-06,\n", + " 9.5367e-07, 1.9073e-06, -6.5565e-07, 1.9073e-06, -4.7684e-07,\n", + " 8.3447e-07, 9.5367e-07, -5.9605e-08, -2.3842e-07, -5.3644e-07,\n", + " 4.1723e-06, -1.2517e-06, -1.7881e-07, -4.7684e-07, 8.3447e-07,\n", + " 1.6689e-06, -5.9605e-08, 2.9802e-06, 3.5763e-07, 3.4571e-06,\n", + " -2.9802e-07, 8.3447e-07, 1.6689e-06, -3.0398e-06, -7.1526e-07,\n", + " 1.7881e-06, -7.7486e-07, 1.0729e-06, 1.3113e-06, -1.7881e-07,\n", + " -2.0862e-06, 3.0994e-06, -1.6689e-06, -1.0729e-06, 4.7684e-07,\n", + " -8.9407e-07, -3.5763e-06, 5.9605e-07, -2.8014e-06, 1.4305e-06,\n", + " -5.3644e-07, 7.1526e-07, -1.5497e-06], device='cuda:0'),\n", + " tensor([ 1.9497e-06, 1.1289e-06, -1.5738e-06, 1.1510e-06, -7.4326e-08,\n", + " 3.8126e-07, 9.6900e-07, -3.7402e-07, 1.4467e-07, 1.2582e-06,\n", + " -7.9345e-07, 1.5984e-06, -6.9576e-07, 4.6673e-07, -1.3461e-06,\n", + " 1.0145e-06, 3.3077e-07, -1.5204e-06, -1.5039e-06, -4.2089e-07,\n", + " 1.8937e-07, -1.3914e-06, -1.0388e-07, 1.1274e-06, -1.5774e-06,\n", + " 6.0853e-07, 9.5481e-08, -8.7090e-07, 1.2509e-06, 2.7946e-07,\n", + " -2.0569e-06, -1.3727e-07, -1.4220e-06, -1.2110e-06, 3.7401e-07,\n", + " 2.3480e-07, 1.2070e-06, 1.2229e-06, -2.7928e-07, -4.0546e-07,\n", + " -2.9544e-07, -5.3795e-07, 5.3077e-07, -9.8826e-07, -9.3950e-07,\n", + " -2.1373e-07, -2.0103e-06, -6.8640e-07, 1.4187e-06, 1.6758e-06,\n", + " 3.7920e-07, 3.1582e-07, 1.3419e-06, -2.7509e-06, -1.6658e-06,\n", + " -6.2629e-08, 2.2001e-06, 3.7348e-08, -2.7134e-07, -3.5544e-07,\n", + " -6.9813e-07, -3.3215e-07, -3.3120e-06, 1.3260e-06, 2.2269e-06,\n", + " -4.2576e-07, 1.1992e-06, 1.6864e-06, -2.6903e-07, 6.8447e-07,\n", + " -4.4472e-07, -6.7873e-07, 1.7654e-06, -5.0672e-07, -1.7452e-06,\n", + " 6.8183e-07, -9.6769e-07, 8.3157e-08, -2.2955e-06, 3.7853e-07,\n", + " 1.3297e-06, -1.2752e-06, -2.1363e-06, 1.3164e-06, 1.2520e-06,\n", + " 1.6037e-06, 1.8909e-07, -2.1382e-06, 2.2903e-06, -1.8987e-06,\n", + " -1.6115e-07, -3.2639e-07, -4.6660e-07, -4.4684e-08, -5.2856e-07,\n", + " 1.3377e-06, -1.8117e-06, 1.6422e-07, -1.4720e-06, 5.7671e-07,\n", + " 8.7481e-07, -4.3880e-08, 3.1518e-06, 2.3125e-07, 1.3741e-06,\n", + " 3.3694e-07, 8.4926e-07, 7.3630e-07, -1.2645e-06, -1.1614e-06,\n", + " 9.5084e-07, 4.2168e-07, -8.2645e-08, 1.5115e-06, -6.6022e-07,\n", + " -2.0173e-06, 2.9759e-06, -5.2929e-07, -3.9942e-07, -1.1476e-06,\n", + " 1.3308e-06, -1.7744e-06, 1.3992e-06, -1.0328e-06, 7.9348e-07,\n", + " -1.5275e-07, 1.2827e-06, -3.9466e-08], device='cuda:0'),\n", + " tensor([[[[ 2.4680e-07, 4.6026e-06, 2.6245e-06],\n", + " [-2.0801e-06, 1.1120e-06, -1.1115e-06],\n", + " [-1.0098e-06, -1.0179e-06, -2.1886e-07]],\n", + " \n", + " [[ 2.8926e-06, 2.9979e-06, -2.0643e-06],\n", + " [-1.0729e-06, 3.1553e-06, -1.5851e-06],\n", + " [ 3.2037e-07, -1.2163e-06, 2.9933e-06]],\n", + " \n", + " [[ 3.0659e-06, 1.5637e-06, 3.5544e-06],\n", + " [-1.6931e-06, -3.0175e-07, 1.8347e-06],\n", + " [ 7.9279e-07, 1.6730e-06, 4.8093e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.2526e-06, 1.8943e-06, 4.0680e-06],\n", + " [ 2.8886e-06, 3.4068e-06, 3.6228e-06],\n", + " [ 2.1234e-07, -3.3248e-07, -3.8184e-07]],\n", + " \n", + " [[-1.9120e-06, -1.6149e-06, -2.4028e-06],\n", + " [-7.1675e-06, -6.6552e-06, -5.9269e-06],\n", + " [-6.1579e-06, -5.2690e-06, -4.8615e-06]],\n", + " \n", + " [[ 1.5516e-06, 5.7819e-06, 2.6170e-06],\n", + " [-2.7278e-06, 2.7381e-07, -3.9116e-08],\n", + " [ 2.0303e-07, 2.9979e-06, 1.3260e-06]]],\n", + " \n", + " \n", + " [[[ 4.7218e-07, 1.6131e-06, 2.5639e-06],\n", + " [ 1.8110e-06, -9.3691e-07, 2.1365e-06],\n", + " [ 1.7509e-06, -1.2740e-06, 1.6037e-06]],\n", + " \n", + " [[ 1.8775e-06, 1.7210e-06, -6.8918e-07],\n", + " [ 4.1164e-07, 3.7253e-09, 2.5341e-06],\n", + " [ 1.9781e-06, 4.6599e-06, -6.6776e-07]],\n", + " \n", + " [[-4.1351e-07, -8.7544e-07, 2.8964e-06],\n", + " [ 3.0212e-06, -1.5534e-06, -2.7008e-07],\n", + " [-1.7309e-06, -4.2263e-06, -1.5777e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.2815e-06, 8.4874e-09, 1.0431e-06],\n", + " [ 2.1663e-06, 1.8775e-06, -3.3388e-07],\n", + " [ 1.5525e-06, 2.8163e-06, 2.3199e-06]],\n", + " \n", + " [[ 2.3884e-06, 1.1921e-07, -3.2308e-06],\n", + " [-2.3060e-06, -4.2263e-06, -5.4501e-06],\n", + " [-1.4587e-07, -3.3388e-06, -3.0622e-06]],\n", + " \n", + " [[ 2.0054e-06, 1.7574e-06, 4.5728e-06],\n", + " [-2.2352e-06, 7.8976e-07, -3.8370e-07],\n", + " [-8.8010e-07, -1.9819e-06, 8.9407e-08]]],\n", + " \n", + " \n", + " [[[ 2.3488e-06, -9.7603e-07, -1.6429e-06],\n", + " [-7.7672e-07, -6.8918e-08, -3.3714e-07],\n", + " [ 2.3842e-07, 1.9353e-06, 3.1004e-06]],\n", + " \n", + " [[ 3.0510e-06, -1.1161e-06, 6.8452e-07],\n", + " [ 1.9185e-06, 2.9234e-06, 8.4098e-07],\n", + " [-5.9791e-07, -2.1700e-07, -4.7460e-06]],\n", + " \n", + " [[ 1.0878e-06, 2.7600e-06, 3.9805e-06],\n", + " [ 2.1881e-06, -1.3951e-06, 6.2445e-07],\n", + " [-7.3016e-07, -1.1176e-07, 2.2575e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.1814e-06, 1.2321e-06, -5.3924e-07],\n", + " [ 2.7753e-06, 2.8284e-06, 2.1269e-06],\n", + " [ 2.4736e-06, 2.1348e-06, 2.4103e-06]],\n", + " \n", + " [[-1.5609e-06, 1.8626e-06, -7.6368e-08],\n", + " [ 4.3688e-06, 3.3472e-06, 4.1262e-06],\n", + " [ 5.0850e-07, -3.8820e-06, -2.6021e-06]],\n", + " \n", + " [[ 1.6261e-06, -1.0626e-06, 2.7325e-06],\n", + " [ 2.5965e-06, 2.0191e-06, 2.8480e-06],\n", + " [ 5.8357e-06, 9.9838e-07, 4.4331e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 3.0976e-06, 4.6566e-08, 1.1427e-06],\n", + " [ 6.5118e-06, 4.4201e-06, -7.0222e-07],\n", + " [ 2.4866e-06, -1.8473e-06, 3.3900e-07]],\n", + " \n", + " [[-1.9744e-06, -2.1625e-06, -2.9802e-07],\n", + " [ 1.2815e-06, -2.1625e-06, -7.7859e-07],\n", + " [-1.4482e-06, -2.7530e-06, 1.8626e-07]],\n", + " \n", + " [[ 6.0536e-07, 9.3132e-07, 2.7763e-06],\n", + " [ 2.0862e-06, 2.4028e-07, 2.0564e-06],\n", + " [ 1.1496e-06, -3.9954e-06, 5.9186e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.1162e-06, -4.3074e-08, 1.2554e-06],\n", + " [ 2.6943e-06, 3.1991e-06, 1.4352e-06],\n", + " [ 3.5805e-06, 2.6580e-06, 3.9460e-06]],\n", + " \n", + " [[-1.7509e-07, 1.7509e-06, -6.3051e-07],\n", + " [-3.9740e-06, -2.1160e-06, -3.6712e-06],\n", + " [-3.3956e-06, -2.1229e-06, -2.9244e-06]],\n", + " \n", + " [[ 3.5875e-06, 1.1255e-06, -1.6745e-06],\n", + " [ 4.9621e-06, 3.9906e-06, 2.7381e-06],\n", + " [ 2.0429e-06, 2.0508e-06, 4.7341e-06]]],\n", + " \n", + " \n", + " [[[-2.6524e-06, -1.4914e-06, 2.1979e-07],\n", + " [-5.1348e-06, -4.7088e-06, 4.2282e-07],\n", + " [-4.5355e-06, -1.0580e-06, -1.6559e-06]],\n", + " \n", + " [[ 6.8732e-07, 4.2096e-07, -5.9791e-07],\n", + " [ 6.1095e-07, 1.9041e-06, 1.8552e-06],\n", + " [-1.0226e-06, 4.6175e-06, -4.5542e-07]],\n", + " \n", + " [[ 4.9528e-06, 1.2238e-06, 1.1399e-06],\n", + " [ 4.1332e-06, 1.4482e-06, -2.1122e-06],\n", + " [ 4.2338e-06, 4.3996e-06, 2.0410e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.0361e-07, 5.1782e-07, 4.7870e-07],\n", + " [-6.4261e-07, -2.7008e-06, -1.0822e-06],\n", + " [-4.6659e-07, -3.1348e-06, -2.0862e-06]],\n", + " \n", + " [[-1.3588e-06, -1.0896e-06, -1.3774e-06],\n", + " [ 1.6324e-06, -5.7928e-07, -8.7079e-08],\n", + " [-2.2040e-06, -1.5069e-06, -4.3772e-06]],\n", + " \n", + " [[ 2.3583e-06, -5.2713e-07, 3.3895e-06],\n", + " [-8.3144e-07, -4.1337e-06, -8.1211e-07],\n", + " [-1.9405e-06, -7.7821e-06, -1.9437e-06]]],\n", + " \n", + " \n", + " [[[-3.9143e-06, -2.2687e-06, -2.1812e-06],\n", + " [-3.8780e-06, -1.6596e-06, -4.6194e-06],\n", + " [-1.9256e-06, -1.9111e-06, -3.1497e-06]],\n", + " \n", + " [[ 3.3490e-06, 2.8852e-06, 3.9299e-06],\n", + " [-4.9733e-07, 1.5199e-06, 3.6880e-06],\n", + " [-1.0580e-06, 3.9656e-06, 1.1660e-06]],\n", + " \n", + " [[ 5.7556e-07, -2.6373e-06, -3.8473e-06],\n", + " [ 4.1127e-06, 2.2128e-06, 3.1032e-06],\n", + " [ 2.3693e-06, -1.9651e-07, -1.1791e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.6354e-06, -9.6299e-07, 1.5609e-06],\n", + " [-2.6338e-06, -2.8769e-06, -2.6245e-06],\n", + " [-1.0693e-06, -1.5106e-06, -4.6603e-06]],\n", + " \n", + " [[ 3.0603e-06, 1.5721e-06, -1.9213e-06],\n", + " [ 1.6578e-07, -2.5935e-06, -3.3109e-06],\n", + " [-8.9407e-07, -1.7101e-07, -1.2284e-06]],\n", + " \n", + " [[-2.3362e-06, 3.8184e-07, -3.1351e-06],\n", + " [ 3.7476e-06, -1.2461e-06, -1.1781e-07],\n", + " [-1.2573e-06, -2.1867e-06, -2.1167e-06]]]], device='cuda:0'),\n", + " tensor([-1.9073e-06, -1.1325e-06, -5.3644e-07, 8.3447e-07, 3.5763e-07,\n", + " -1.3709e-06, 3.5763e-07, 9.5367e-07, 8.3447e-07, -5.9605e-08,\n", + " -2.2054e-06, -3.2783e-06, 2.7418e-06, -8.3447e-07, -1.3709e-06,\n", + " -8.9407e-07, -1.4901e-06, 2.3842e-07, 3.5763e-07, 2.7418e-06,\n", + " 1.3113e-06, -1.7881e-07, 5.9605e-07, 4.7684e-07, -1.0729e-06,\n", + " -1.1921e-07, 1.1921e-06, 0.0000e+00, 2.9802e-06, -2.9802e-07,\n", + " -2.2054e-06, -1.1921e-06, 9.5367e-07, -2.0862e-06, 1.6689e-06,\n", + " -1.3709e-06, -1.1921e-07, 0.0000e+00, 7.1526e-07, -7.1526e-07,\n", + " 1.0729e-06, 8.3447e-07, 1.7881e-06, -1.0133e-06, -1.1921e-06,\n", + " -2.6226e-06, 5.9605e-07, 9.5367e-07, 4.7684e-07, 2.0266e-06,\n", + " -5.9605e-07, -2.0266e-06, -6.5565e-07, -8.3447e-07, -5.3644e-07,\n", + " 3.5763e-07, -3.5763e-06, 1.1921e-06, -1.3709e-06, 4.7684e-07,\n", + " 1.9073e-06, -2.2054e-06, -7.1526e-07, -1.0729e-06, 7.1526e-07,\n", + " 1.7881e-06, -5.9605e-07, -2.9802e-07, 2.3842e-07, 2.3842e-06,\n", + " -3.3975e-06, -1.3113e-06, 2.3842e-07, 0.0000e+00, 3.5763e-07,\n", + " 1.1921e-07, -9.5367e-07, -6.5565e-07, 1.5497e-06, -1.1921e-07,\n", + " 3.5763e-07, -1.4901e-06, 1.1921e-07, -1.2517e-06, 1.9073e-06,\n", + " 1.5497e-06, 1.0729e-06, 1.1921e-06, -5.9605e-07, -1.4901e-06,\n", + " 2.0266e-06, 7.1526e-07, 1.0729e-06, -4.1723e-07, -1.1921e-07,\n", + " 8.3447e-07, 1.4305e-06, -9.5367e-07, 1.4305e-06, -6.5565e-07,\n", + " -8.3447e-07, 7.1526e-07, -1.1921e-06, -1.9073e-06, 1.1921e-07,\n", + " 1.5497e-06, 1.3113e-06, -1.3709e-06, -1.0729e-06, 8.3447e-07,\n", + " -5.9605e-07, 7.1526e-07, 1.7881e-06, 1.7881e-06, 1.6689e-06,\n", + " 1.1921e-06, -2.3842e-07, 2.1458e-06, -1.7881e-06, 4.7684e-07,\n", + " -5.9605e-07, 4.7684e-07, 3.5763e-07, -1.1921e-07, 4.7684e-07,\n", + " 2.0266e-06, 1.1921e-06, 1.6689e-06], device='cuda:0'),\n", + " tensor([-9.1369e-07, -3.0748e-07, -9.0601e-07, 4.1690e-07, -4.7604e-07,\n", + " 8.1218e-07, 2.0112e-06, -8.5575e-07, -4.3124e-07, -9.8559e-07,\n", + " -6.4047e-07, 7.4814e-08, 9.7423e-07, 7.2637e-07, 4.6106e-09,\n", + " -9.8934e-07, -1.1821e-06, 3.3636e-07, 5.9197e-07, -8.3250e-09,\n", + " 4.7365e-07, 3.8908e-07, -7.9900e-07, 1.1213e-07, -5.1611e-07,\n", + " 5.7080e-07, 1.7410e-07, -6.2933e-07, 1.2636e-06, 1.1473e-06,\n", + " -9.1808e-07, -1.5391e-06, -4.5448e-08, 1.3961e-06, -1.2846e-06,\n", + " -6.6418e-07, -7.5253e-08, -6.9444e-07, 1.1273e-06, -3.1001e-07,\n", + " -1.9286e-06, 1.0208e-06, -1.8318e-07, 1.2594e-07, -6.4955e-07,\n", + " -1.6561e-06, -7.6606e-07, -6.6829e-07, -6.2103e-07, 1.2311e-07,\n", + " -1.3618e-06, -1.5188e-06, -1.2815e-06, 6.3298e-07, -1.3043e-06,\n", + " 7.3002e-08, 1.0671e-06, -6.9393e-07, -8.1839e-07, 7.7284e-07,\n", + " 4.2962e-07, 6.4041e-07, -6.6803e-07, -8.9368e-07, -4.7253e-08,\n", + " 6.4242e-07, 1.4340e-07, 2.3820e-07, -2.3032e-07, 1.3455e-07,\n", + " -2.4945e-06, -7.8601e-07, 8.3551e-07, 8.6485e-07, 2.4023e-07,\n", + " 1.7556e-06, -5.7290e-07, 1.5098e-07, 9.2456e-07, 1.1759e-06,\n", + " 2.8615e-07, -2.2824e-06, -2.1893e-06, 2.0725e-06, 9.3870e-07,\n", + " 1.6482e-06, -7.7833e-07, -6.6209e-07, -1.5313e-06, 1.1172e-06,\n", + " 6.2416e-07, 9.9326e-07, 8.7520e-07, 2.3982e-07, -1.4602e-07,\n", + " 2.0822e-06, -7.0897e-07, 1.0941e-07, -1.1862e-06, -1.6575e-06,\n", + " -1.6125e-06, 1.6521e-06, 1.0219e-06, -9.5518e-07, 1.8921e-06,\n", + " 1.2553e-06, 2.3294e-06, -1.0472e-06, -3.4637e-08, 5.1041e-07,\n", + " 1.6033e-06, 1.1330e-06, -2.9018e-07, -8.9494e-07, -6.7316e-07,\n", + " 7.9159e-07, 2.6595e-06, -9.3374e-07, 9.3966e-07, 2.0426e-06,\n", + " -4.1359e-07, -1.1283e-06, 1.1768e-06, -2.5566e-07, -4.6953e-07,\n", + " 9.1875e-07, -6.7674e-08, -3.9833e-07], device='cuda:0'),\n", + " tensor([[[[-4.4843e-06]],\n", + " \n", + " [[-2.4885e-06]],\n", + " \n", + " [[-3.1590e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.0212e-06]],\n", + " \n", + " [[ 1.1697e-06]],\n", + " \n", + " [[ 7.3016e-07]]],\n", + " \n", + " \n", + " [[[-6.4448e-06]],\n", + " \n", + " [[ 3.0175e-06]],\n", + " \n", + " [[ 1.0319e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.3362e-06]],\n", + " \n", + " [[ 3.5390e-06]],\n", + " \n", + " [[ 2.2054e-06]]],\n", + " \n", + " \n", + " [[[-5.7369e-07]],\n", + " \n", + " [[-1.5087e-06]],\n", + " \n", + " [[ 1.6466e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.8056e-07]],\n", + " \n", + " [[ 1.8757e-06]],\n", + " \n", + " [[ 2.0862e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-1.9595e-06]],\n", + " \n", + " [[ 8.1584e-07]],\n", + " \n", + " [[ 4.3865e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.0755e-06]],\n", + " \n", + " [[ 1.7136e-06]],\n", + " \n", + " [[-2.2640e-06]]],\n", + " \n", + " \n", + " [[[-6.8806e-06]],\n", + " \n", + " [[-3.4105e-06]],\n", + " \n", + " [[-5.0068e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.5449e-06]],\n", + " \n", + " [[-4.9472e-06]],\n", + " \n", + " [[ 2.5444e-06]]],\n", + " \n", + " \n", + " [[[-7.2736e-06]],\n", + " \n", + " [[-3.5763e-06]],\n", + " \n", + " [[ 1.6093e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.9600e-06]],\n", + " \n", + " [[-1.2573e-06]],\n", + " \n", + " [[ 1.4901e-07]]]], device='cuda:0'),\n", + " tensor([ 0.0000e+00, -1.0729e-06, -2.9802e-07, 1.3113e-06, -1.0133e-06,\n", + " 3.5763e-07, 6.3181e-06, -1.5497e-06, 3.5763e-07, -1.4305e-06,\n", + " 1.5497e-06, 1.3113e-06, -1.3709e-06, 1.4305e-06, -5.3644e-07,\n", + " 1.1921e-07, -2.5630e-06, -1.7881e-07, 2.2650e-06, -1.3709e-06,\n", + " 1.3113e-06, 1.6689e-06, -5.9605e-08, -2.9802e-07, -1.0133e-06,\n", + " 1.5497e-06, -8.3447e-07, -1.4901e-06, 1.1921e-07, 2.3842e-06,\n", + " -7.7486e-07, -9.5367e-07, -6.5565e-07, 2.9802e-06, -2.9802e-06,\n", + " 1.9073e-06, -1.6093e-06, 7.1526e-07, 9.5367e-07, -1.0729e-06,\n", + " -2.8014e-06, 0.0000e+00, -9.5367e-07, -5.9605e-07, -1.4305e-06,\n", + " 3.5763e-07, -8.9407e-07, -1.0729e-06, -1.0133e-06, 7.1526e-07,\n", + " -1.7881e-06, -3.3975e-06, -2.3842e-07, 4.1723e-06, -5.3644e-07,\n", + " 2.3842e-07, 2.0266e-06, -3.0994e-06, -5.9605e-08, -6.5565e-07,\n", + " -7.1526e-07, 1.6689e-06, 9.5367e-07, 8.3447e-07, -4.7684e-07,\n", + " -4.1723e-07, -2.3842e-07, 4.7684e-07, 8.3447e-07, -1.4305e-06,\n", + " -4.2915e-06, -4.7684e-07, -2.9802e-07, 2.3842e-07, 9.5367e-07,\n", + " 3.5763e-07, 1.1921e-07, 3.5763e-07, 4.7684e-07, 8.3447e-07,\n", + " -1.4305e-06, 1.1921e-06, -1.9073e-06, 5.9605e-07, -3.5763e-07,\n", + " 1.5497e-06, -2.6226e-06, -6.5565e-07, -2.3842e-06, 2.7418e-06,\n", + " -2.6822e-06, 1.1921e-07, 8.3447e-07, 1.1921e-06, -2.3842e-06,\n", + " 2.3842e-07, -1.6093e-06, -1.7881e-07, 2.7418e-06, -2.6226e-06,\n", + " 3.5763e-07, 1.4305e-06, 1.5497e-06, 5.9605e-07, -4.1723e-07,\n", + " 7.1526e-07, 2.3842e-06, -5.9605e-08, -5.9605e-08, 9.5367e-07,\n", + " -7.1526e-07, -1.1921e-06, -4.1723e-07, -7.1526e-07, 9.5367e-07,\n", + " 9.5367e-07, 3.3379e-06, -3.3975e-06, 2.0266e-06, 1.1921e-06,\n", + " 4.7684e-07, 1.3113e-06, 1.1921e-07, -7.1526e-07, -2.1458e-06,\n", + " 9.5367e-07, -1.4901e-06, -5.3644e-07], device='cuda:0'),\n", + " tensor([-9.1369e-07, -3.0748e-07, -9.0601e-07, 4.1690e-07, -4.7604e-07,\n", + " 8.1218e-07, 2.0112e-06, -8.5575e-07, -4.3124e-07, -9.8559e-07,\n", + " -6.4047e-07, 7.4814e-08, 9.7423e-07, 7.2637e-07, 4.6106e-09,\n", + " -9.8934e-07, -1.1821e-06, 3.3636e-07, 5.9197e-07, -8.3250e-09,\n", + " 4.7365e-07, 3.8908e-07, -7.9900e-07, 1.1213e-07, -5.1611e-07,\n", + " 5.7080e-07, 1.7410e-07, -6.2933e-07, 1.2636e-06, 1.1473e-06,\n", + " -9.1808e-07, -1.5391e-06, -4.5448e-08, 1.3961e-06, -1.2846e-06,\n", + " -6.6418e-07, -7.5253e-08, -6.9444e-07, 1.1273e-06, -3.1001e-07,\n", + " -1.9286e-06, 1.0208e-06, -1.8318e-07, 1.2594e-07, -6.4955e-07,\n", + " -1.6561e-06, -7.6606e-07, -6.6829e-07, -6.2103e-07, 1.2311e-07,\n", + " -1.3618e-06, -1.5188e-06, -1.2815e-06, 6.3298e-07, -1.3043e-06,\n", + " 7.3002e-08, 1.0671e-06, -6.9393e-07, -8.1839e-07, 7.7284e-07,\n", + " 4.2962e-07, 6.4041e-07, -6.6803e-07, -8.9368e-07, -4.7253e-08,\n", + " 6.4242e-07, 1.4340e-07, 2.3820e-07, -2.3032e-07, 1.3455e-07,\n", + " -2.4945e-06, -7.8601e-07, 8.3551e-07, 8.6485e-07, 2.4023e-07,\n", + " 1.7556e-06, -5.7290e-07, 1.5098e-07, 9.2456e-07, 1.1759e-06,\n", + " 2.8615e-07, -2.2824e-06, -2.1893e-06, 2.0725e-06, 9.3870e-07,\n", + " 1.6482e-06, -7.7833e-07, -6.6209e-07, -1.5313e-06, 1.1172e-06,\n", + " 6.2416e-07, 9.9326e-07, 8.7520e-07, 2.3982e-07, -1.4602e-07,\n", + " 2.0822e-06, -7.0897e-07, 1.0941e-07, -1.1862e-06, -1.6575e-06,\n", + " -1.6125e-06, 1.6521e-06, 1.0219e-06, -9.5518e-07, 1.8921e-06,\n", + " 1.2553e-06, 2.3294e-06, -1.0472e-06, -3.4637e-08, 5.1041e-07,\n", + " 1.6033e-06, 1.1330e-06, -2.9018e-07, -8.9494e-07, -6.7316e-07,\n", + " 7.9159e-07, 2.6595e-06, -9.3374e-07, 9.3966e-07, 2.0426e-06,\n", + " -4.1359e-07, -1.1283e-06, 1.1768e-06, -2.5566e-07, -4.6953e-07,\n", + " 9.1875e-07, -6.7674e-08, -3.9833e-07], device='cuda:0'),\n", + " tensor([[[[-1.8403e-06, -1.3132e-07, 4.1910e-07],\n", + " [-2.9635e-06, -1.3411e-06, -2.2529e-06],\n", + " [-8.4285e-07, 1.4529e-07, 7.4506e-07]],\n", + " \n", + " [[ 5.9167e-06, 8.3577e-06, 8.6566e-06],\n", + " [ 7.1782e-06, 6.8275e-06, 7.5828e-06],\n", + " [ 6.5733e-06, 4.0652e-06, 3.1553e-06]],\n", + " \n", + " [[-6.9290e-07, 3.1311e-06, 1.1884e-06],\n", + " [ 9.6578e-07, 3.6461e-06, -6.1095e-07],\n", + " [ 1.5199e-06, 6.4112e-06, -1.3048e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.5272e-06, 5.5972e-06, 4.6393e-06],\n", + " [ 5.5131e-06, 4.0464e-06, 2.6692e-06],\n", + " [ 2.1489e-06, 2.7381e-06, 3.7728e-06]],\n", + " \n", + " [[ 1.0245e-07, 3.1572e-06, 2.6356e-06],\n", + " [-1.1148e-06, -9.5367e-07, 2.2911e-06],\n", + " [ 6.4075e-07, 1.5096e-06, 1.5600e-06]],\n", + " \n", + " [[-1.0058e-06, 3.6694e-07, 1.0990e-07],\n", + " [-1.3374e-06, -9.0338e-07, 9.1083e-07],\n", + " [-2.3581e-06, 4.9733e-07, 1.0328e-06]]],\n", + " \n", + " \n", + " [[[ 2.5518e-07, -1.4063e-06, 4.6077e-07],\n", + " [ 3.9674e-07, 8.0466e-07, 1.3337e-06],\n", + " [ 2.9057e-07, -7.0222e-07, -8.3703e-07]],\n", + " \n", + " [[ 1.6950e-06, 1.7611e-06, 1.2666e-07],\n", + " [ 1.8440e-07, 1.3290e-06, 1.8720e-06],\n", + " [-1.5106e-06, -8.8336e-07, 8.0466e-07]],\n", + " \n", + " [[ 1.8068e-07, 2.8946e-06, 1.7304e-06],\n", + " [ 4.0978e-08, -1.3970e-07, -6.2585e-07],\n", + " [-1.1232e-06, -1.4445e-06, -2.1141e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1257e-06, 2.9248e-06, 1.3709e-06],\n", + " [ 9.3691e-07, 6.5798e-07, 2.1011e-06],\n", + " [ 1.4137e-06, 4.0419e-07, 1.0543e-06]],\n", + " \n", + " [[ 1.6298e-06, 1.3770e-06, -1.2852e-07],\n", + " [ 1.7555e-06, 1.1814e-06, -1.8202e-06],\n", + " [ 2.0955e-08, 2.1816e-06, -3.9674e-07]],\n", + " \n", + " [[ 2.2273e-06, 1.3923e-06, -2.3004e-07],\n", + " [-8.5682e-08, -5.4250e-07, 2.1141e-07],\n", + " [ 6.2212e-07, 3.2601e-06, -1.9444e-06]]],\n", + " \n", + " \n", + " [[[-2.2650e-06, -1.8813e-06, -1.7062e-06],\n", + " [-1.2228e-06, -3.1311e-06, -4.2953e-06],\n", + " [ 5.6671e-07, -8.5495e-07, 1.1511e-06]],\n", + " \n", + " [[ 1.8273e-06, 1.5022e-06, 3.2187e-06],\n", + " [-4.1351e-07, -1.2494e-06, 2.5043e-06],\n", + " [-2.4159e-06, -8.0280e-07, -4.9360e-08]],\n", + " \n", + " [[ 7.5251e-07, 1.2759e-06, 1.2228e-06],\n", + " [-3.0324e-06, -2.2631e-07, -2.0005e-06],\n", + " [-1.4156e-06, -9.0525e-07, -3.3826e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.6473e-07, -1.5423e-06, -4.3958e-07],\n", + " [-5.7556e-07, -5.9884e-07, 1.1805e-06],\n", + " [ 1.7841e-06, 1.5851e-06, 6.0163e-07]],\n", + " \n", + " [[-8.3516e-07, -1.1735e-06, -1.9008e-06],\n", + " [-1.9185e-06, -4.0419e-06, -3.7476e-06],\n", + " [-4.4126e-06, -2.8610e-06, -4.2629e-06]],\n", + " \n", + " [[ 1.1767e-06, 2.2892e-06, 8.2795e-07],\n", + " [ 2.2175e-06, 1.0375e-06, -9.6858e-07],\n", + " [ 8.3540e-07, 1.0722e-06, -3.3528e-08]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 1.0282e-06, -5.2713e-07, -2.0633e-06],\n", + " [-1.7202e-06, 2.4233e-06, 1.0896e-06],\n", + " [-8.0280e-07, -1.8077e-06, -2.0942e-07]],\n", + " \n", + " [[-9.2480e-07, -1.5087e-06, -4.6194e-07],\n", + " [ 8.7358e-07, -1.5018e-06, -3.2689e-06],\n", + " [-1.3970e-08, -2.0396e-07, -9.6671e-07]],\n", + " \n", + " [[ 1.3923e-07, 3.7272e-06, 3.3574e-06],\n", + " [ 2.0675e-06, 2.3858e-06, 2.0601e-06],\n", + " [ 9.0152e-07, 1.4515e-06, 4.9919e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.3900e-07, 1.4659e-06, 1.1316e-06],\n", + " [-1.2023e-06, -2.5779e-06, -5.6531e-07],\n", + " [ 6.5472e-07, -1.6801e-06, -1.8135e-06]],\n", + " \n", + " [[-2.1812e-06, -2.6133e-06, 1.5832e-07],\n", + " [-7.8045e-07, -6.9570e-07, -2.2082e-06],\n", + " [-3.7625e-07, -1.0543e-06, -6.4261e-07]],\n", + " \n", + " [[ 7.0059e-07, 8.5309e-07, 4.3586e-07],\n", + " [-1.8883e-06, 1.5269e-06, 3.0082e-06],\n", + " [-1.2715e-06, -7.6368e-08, 8.0094e-07]]],\n", + " \n", + " \n", + " [[[-2.2464e-06, -1.6782e-06, 6.5565e-07],\n", + " [ 9.0338e-07, 1.6345e-07, 6.5099e-07],\n", + " [ 1.4156e-07, 7.2457e-07, 8.1491e-07]],\n", + " \n", + " [[ 1.9912e-06, 4.3586e-07, 1.6659e-07],\n", + " [ 5.0478e-07, 8.0094e-07, -6.9849e-07],\n", + " [ 1.3271e-06, 3.7439e-07, -6.8918e-08]],\n", + " \n", + " [[ 7.0035e-07, 2.6077e-07, 1.3076e-06],\n", + " [ 2.0647e-06, 1.6429e-06, 2.1001e-06],\n", + " [-4.7870e-07, -3.5018e-07, 8.9128e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.3001e-06, 1.3644e-06, -1.8068e-06],\n", + " [ 2.2352e-07, 8.8289e-07, -1.1027e-06],\n", + " [-1.5078e-06, -3.0547e-07, -2.3167e-06]],\n", + " \n", + " [[ 1.9278e-07, 7.2271e-07, 6.4075e-07],\n", + " [-7.1587e-07, -1.4715e-06, -1.0505e-06],\n", + " [-1.5064e-06, -2.9635e-06, -2.0145e-06]],\n", + " \n", + " [[-8.6427e-07, -8.6240e-07, -1.4901e-07],\n", + " [ 1.2480e-07, -9.2946e-07, 4.0047e-07],\n", + " [-1.8906e-06, -1.4938e-06, 5.2899e-07]]],\n", + " \n", + " \n", + " [[[-1.7451e-07, -7.4692e-07, -6.7428e-07],\n", + " [-6.5193e-08, -6.6846e-07, -1.1828e-06],\n", + " [ 7.8697e-08, -9.3225e-07, -2.2352e-08]],\n", + " \n", + " [[ 5.9158e-06, 7.6722e-06, 6.1896e-06],\n", + " [ 2.8582e-06, 3.6582e-06, 5.8301e-06],\n", + " [ 4.4638e-06, 4.3232e-06, 5.6196e-06]],\n", + " \n", + " [[ 5.6624e-07, -1.4286e-06, 5.8115e-07],\n", + " [-6.2399e-08, 6.4820e-07, 8.1956e-07],\n", + " [ 8.2888e-07, -2.3814e-06, -1.1809e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.2815e-06, 3.4645e-07, 1.5977e-06],\n", + " [ 1.6801e-06, 1.0859e-06, 1.1623e-06],\n", + " [ 2.6748e-06, 1.4342e-06, 6.4727e-07]],\n", + " \n", + " [[ 3.0268e-07, 2.2948e-06, 1.1520e-06],\n", + " [ 4.3609e-07, 3.1302e-06, 2.8396e-06],\n", + " [ 6.3051e-07, 1.8162e-07, 1.6438e-06]],\n", + " \n", + " [[-4.7986e-07, -1.3672e-06, -2.6599e-06],\n", + " [ 1.7416e-06, 8.6427e-07, -1.7863e-06],\n", + " [-6.8115e-07, -1.4611e-06, -2.6692e-06]]]], device='cuda:0'),\n", + " tensor([-3.5763e-07, -4.1723e-07, -1.1921e-07, 0.0000e+00, -2.3842e-06,\n", + " 1.1921e-07, -4.7684e-07, -1.1921e-07, 4.7684e-07, 1.1921e-06,\n", + " 0.0000e+00, 8.3447e-07, -4.1723e-07, 1.0729e-06, 4.7684e-07,\n", + " -1.7881e-07, -1.0729e-06, -5.9605e-07, -5.3644e-07, 1.1921e-07,\n", + " -7.1526e-07, -1.1921e-06, -5.9605e-07, 1.1921e-06, 7.1526e-07,\n", + " -4.1723e-07, 5.9605e-07, 2.3842e-07, 8.3447e-07, -1.0133e-06,\n", + " 4.7684e-07, 3.5763e-07, 2.3842e-07, -7.1526e-07, -2.9802e-07,\n", + " -7.7486e-07, -6.5565e-07, -1.5497e-06, -8.3447e-07, -4.1723e-07,\n", + " 8.3447e-07, -1.0729e-06, -2.9802e-07, 7.1526e-07, -1.0133e-06,\n", + " 1.4305e-06, -5.9605e-07, 4.7684e-07, -1.6689e-06, 7.1526e-07,\n", + " -7.7486e-07, 4.7684e-07, 8.3447e-07, 0.0000e+00, 7.1526e-07,\n", + " -2.9802e-07, 1.1921e-07, 4.7684e-07, -2.9802e-07, -2.3842e-07,\n", + " 1.0729e-06, 3.5763e-07, -1.9073e-06, -7.1526e-07, -1.7881e-07,\n", + " -1.6093e-06, 7.1526e-07, -2.3842e-07, -1.0133e-06, 2.3842e-07,\n", + " 9.5367e-07, 3.5763e-07, -5.3644e-07, 1.1921e-06, -5.9605e-07,\n", + " 1.7881e-06, -2.3842e-07, -1.1921e-07, -1.0133e-06, 3.3379e-06,\n", + " 2.3842e-07, -5.9605e-07, 8.3447e-07, -1.3113e-06, 1.1921e-07,\n", + " -5.9605e-08, -8.9407e-07, 7.1526e-07, -1.2517e-06, -7.7486e-07,\n", + " -1.0729e-06, 1.1921e-07, -5.9605e-07, -5.3644e-07, -1.1921e-07,\n", + " -1.3113e-06, -1.0729e-06, -4.1723e-07, 4.7684e-07, -8.3447e-07,\n", + " 2.1458e-06, -3.5763e-07, -8.3447e-07, 2.3842e-07, -8.3447e-07,\n", + " -1.7285e-06, 7.1526e-07, -7.7486e-07, 2.3842e-07, 8.3447e-07,\n", + " 4.7684e-07, -8.3447e-07, 4.7684e-07, 0.0000e+00, 1.6689e-06,\n", + " 1.9073e-06, 5.9605e-07, -8.3447e-07, -1.0133e-06, -8.3447e-07,\n", + " -4.7684e-07, 0.0000e+00, 9.5367e-07, -1.1325e-06, -2.9802e-07,\n", + " -1.0133e-06, -7.7486e-07, -5.9605e-07, 5.9605e-07, -8.3447e-07,\n", + " 4.7684e-07, 1.1921e-06, 1.3113e-06, -2.3842e-06, -2.9802e-07,\n", + " -5.9605e-07, 3.5763e-07, -1.2517e-06, 4.7684e-07, 3.5763e-07,\n", + " 2.3842e-07, 1.1921e-07, 1.4305e-06, 8.3447e-07, 2.3842e-07,\n", + " -2.3842e-07, -8.3447e-07, 7.1526e-07, -1.1921e-07, 1.5497e-06,\n", + " -4.7684e-07, 1.0729e-06, -5.3644e-07, 7.1526e-07, 2.3842e-06,\n", + " -5.9605e-07, 0.0000e+00, 8.3447e-07, 1.1921e-07, 4.7684e-07,\n", + " 5.9605e-07, 4.7684e-07, -1.1325e-06, 2.3842e-07, 1.0729e-06,\n", + " -3.3379e-06, 5.9605e-07, 8.3447e-07, 7.1526e-07, 1.9073e-06,\n", + " 2.5034e-06, 1.1921e-07, -5.3644e-07, 8.3447e-07, 8.3447e-07,\n", + " -5.9605e-08, -3.5763e-07, -7.1526e-07, -4.1723e-07, 3.5763e-07,\n", + " 1.3113e-06, 8.3447e-07, -2.3842e-07, 1.1921e-06, -1.7881e-06,\n", + " 1.4305e-06, -1.0133e-06, 3.5763e-07, -2.9802e-07, -5.9605e-08,\n", + " 1.0729e-06, -7.7486e-07, -1.9670e-06, 4.7684e-07, -1.1921e-06,\n", + " 5.9605e-07, 9.5367e-07, 3.5763e-07, -1.1921e-06, 1.1921e-07,\n", + " 8.3447e-07, -5.9605e-07, 9.5367e-07, 2.3842e-07, 0.0000e+00,\n", + " 3.5763e-07, -1.7881e-07, -2.9802e-07, 8.3447e-07, -5.9605e-07,\n", + " 8.3447e-07, -1.5497e-06, 7.1526e-07, 1.1921e-07, -1.3113e-06,\n", + " 7.1526e-07, -2.3842e-07, -1.1325e-06, -7.7486e-07, 0.0000e+00,\n", + " -4.1723e-07, 2.3842e-07, 1.3113e-06, 1.4305e-06, 1.1921e-07,\n", + " -8.3447e-07, -1.1325e-06, 2.3842e-07, 1.6689e-06, 1.3113e-06,\n", + " -5.9605e-08, -6.5565e-07, 1.0729e-06, 2.3842e-07, 4.7684e-07,\n", + " 2.3842e-07, 0.0000e+00, -5.9605e-08, -6.5565e-07, 9.5367e-07,\n", + " -4.7684e-07, 4.7684e-07, -7.1526e-07, 1.1921e-07, -1.2517e-06,\n", + " 7.1526e-07, 1.1921e-07, -1.1921e-07, -2.3842e-07, 2.3842e-07,\n", + " -1.0729e-06, 2.3842e-07, 8.3447e-07, 5.9605e-07, -5.9605e-08,\n", + " -2.9802e-07], device='cuda:0'),\n", + " tensor([-2.9877e-07, -6.0828e-07, 4.5930e-07, 4.1853e-07, -1.7363e-06,\n", + " 3.8803e-07, -1.8305e-07, 1.8413e-08, 6.1509e-07, 1.2918e-06,\n", + " 2.5392e-07, 3.7740e-07, -6.4319e-07, 5.2017e-07, 2.6308e-07,\n", + " 2.9436e-07, -1.0043e-07, -8.2840e-07, -1.1444e-08, -3.9894e-07,\n", + " -3.8904e-07, -3.0451e-07, -8.8293e-07, 8.9173e-07, 2.6468e-07,\n", + " 3.3383e-07, 8.0802e-07, -1.1930e-06, 1.2235e-06, -7.9587e-07,\n", + " -3.4577e-07, 2.3863e-07, 9.7938e-08, -4.0020e-07, 1.0080e-06,\n", + " -5.2720e-07, -8.3278e-07, -1.1856e-06, -6.2362e-07, -4.3046e-07,\n", + " -4.5586e-08, -3.4174e-07, -8.9879e-07, 7.3194e-07, -1.3142e-06,\n", + " 1.3045e-06, -1.7480e-07, 3.7845e-07, -1.3178e-06, 6.6896e-07,\n", + " -4.2099e-07, -5.4902e-07, 2.7925e-07, 2.6155e-07, 6.6702e-07,\n", + " -6.1461e-07, 4.0904e-07, 8.1861e-07, -7.2904e-07, -2.8011e-07,\n", + " 1.2763e-06, -4.8832e-07, -5.9228e-07, 9.3926e-07, 9.4038e-07,\n", + " -3.8900e-07, 3.5983e-07, -1.0416e-06, -8.7933e-07, 2.8657e-07,\n", + " -4.3863e-07, -1.3785e-07, -5.3276e-07, 1.2505e-06, -3.4932e-07,\n", + " 7.9656e-07, -2.3599e-07, 1.0419e-07, -3.8420e-07, 2.1569e-06,\n", + " -3.4136e-07, -2.7307e-07, -5.2092e-07, -3.3845e-07, 8.4514e-07,\n", + " 1.1931e-07, -6.6160e-08, 1.5253e-07, -1.3723e-06, -3.0826e-07,\n", + " -1.1393e-06, 4.4033e-07, -9.8301e-07, -6.6679e-07, -1.9473e-07,\n", + " -1.8548e-06, -6.1208e-07, -6.5729e-07, -1.1443e-07, -4.5872e-07,\n", + " -9.5973e-08, -8.9015e-08, 1.6086e-07, 6.8209e-07, -5.6067e-08,\n", + " 5.7320e-07, 1.4644e-07, -5.2722e-07, -1.4234e-07, 4.5176e-08,\n", + " -2.5149e-07, -1.8360e-06, 4.3521e-07, 5.5334e-07, 1.5618e-06,\n", + " 1.8674e-06, -1.2565e-07, -9.0837e-07, -1.2934e-07, 2.4103e-07,\n", + " 4.5534e-07, -5.2275e-07, 6.2046e-07, -4.7477e-07, 1.4448e-08,\n", + " -3.7181e-07, -6.9485e-07, -3.0114e-07, 4.4949e-07, -7.8480e-07,\n", + " -4.3108e-07, -8.2225e-08, 1.0328e-06, -1.2272e-06, 3.3779e-07,\n", + " -3.3428e-07, 5.8373e-07, -5.0156e-07, 3.4294e-07, 4.4207e-07,\n", + " -1.8404e-08, -1.2264e-07, 1.0217e-06, 9.7197e-07, 2.6029e-07,\n", + " -3.3480e-07, -1.0682e-06, 2.0034e-07, 9.8647e-07, 8.0695e-07,\n", + " -7.1272e-08, 3.6538e-07, -1.3156e-06, 9.3369e-08, 8.5994e-07,\n", + " -1.0821e-06, -9.3545e-07, -4.1727e-08, 3.5602e-07, -2.2246e-07,\n", + " 1.7382e-07, 5.9968e-07, -6.1940e-07, -1.5544e-07, 6.1496e-08,\n", + " -1.9020e-06, 4.2236e-07, -3.4969e-08, -6.4763e-07, 1.2352e-06,\n", + " 1.8911e-07, 1.0038e-06, -3.9493e-07, -4.2307e-07, 4.3479e-07,\n", + " 1.0595e-06, 1.4161e-07, -3.4526e-07, 3.8437e-07, -1.0566e-06,\n", + " 8.1219e-07, 5.8152e-07, 3.3569e-07, -4.2393e-08, -2.0979e-06,\n", + " 2.4898e-07, -1.4319e-06, 8.7014e-07, -5.4009e-07, 5.1806e-07,\n", + " 1.5672e-07, -4.4967e-07, -7.4723e-07, 3.3632e-08, -1.7880e-07,\n", + " -4.2434e-08, 1.1680e-06, 4.3179e-07, 1.2370e-07, 1.4654e-07,\n", + " 2.0227e-07, -4.9486e-07, 7.2001e-07, -2.8281e-07, 1.2026e-07,\n", + " -1.2374e-07, 1.5187e-07, -6.7956e-07, 8.9595e-07, -7.0767e-07,\n", + " 1.2359e-06, -9.4191e-07, 5.9228e-07, 2.2014e-07, 2.4551e-07,\n", + " 7.0375e-07, -7.5531e-09, -4.2107e-07, -5.5966e-07, -2.7207e-07,\n", + " -1.2696e-06, 5.3629e-07, 1.1028e-06, 1.3448e-06, 9.1707e-07,\n", + " -4.1077e-07, -1.4194e-06, 8.2703e-08, 1.3841e-06, 8.4550e-07,\n", + " -1.0788e-06, -7.5913e-07, 6.7956e-07, 1.5473e-07, 6.0747e-08,\n", + " 5.2321e-07, 7.4032e-07, -2.2953e-07, -4.8046e-07, 1.3302e-06,\n", + " -6.8801e-09, 8.0393e-07, -1.7242e-06, -1.0591e-06, -3.9076e-07,\n", + " -7.7495e-08, -2.8139e-07, 3.3762e-07, -1.9094e-07, -4.8483e-07,\n", + " -9.5804e-07, 9.5378e-07, 8.9375e-07, 1.8582e-07, -6.7819e-08,\n", + " 5.9106e-07], device='cuda:0'),\n", + " tensor([[[[ 1.8924e-06, -8.8848e-07, -6.9756e-07],\n", + " [ 2.0321e-06, -5.2154e-08, 1.7174e-06],\n", + " [ 1.9046e-06, -2.4214e-07, -5.1362e-07]],\n", + " \n", + " [[-8.1770e-07, -9.0804e-07, 1.6792e-06],\n", + " [-7.1526e-07, 2.7437e-06, 1.2583e-06],\n", + " [-2.1048e-07, 2.6822e-07, 6.1002e-07]],\n", + " \n", + " [[ 1.8813e-07, 7.5344e-07, 2.2731e-06],\n", + " [ 1.1399e-06, 1.7229e-06, 2.0880e-06],\n", + " [ 2.2715e-06, 1.0692e-06, 9.1083e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-9.5088e-07, 5.3085e-07, 4.8615e-07],\n", + " [ 1.9465e-06, 3.2410e-06, 1.0207e-06],\n", + " [-2.2911e-07, 7.3481e-07, 8.9034e-07]],\n", + " \n", + " [[-5.0338e-07, -1.2694e-06, 2.4308e-07],\n", + " [ 1.6391e-06, 1.7183e-06, 1.9586e-06],\n", + " [ 2.1234e-07, -1.9076e-06, 1.4808e-07]],\n", + " \n", + " [[ 2.6822e-07, -7.9116e-07, -1.9558e-07],\n", + " [ 1.2629e-06, -7.3109e-07, 1.3076e-06],\n", + " [ 5.2340e-07, 3.4645e-07, 8.6799e-07]]],\n", + " \n", + " \n", + " [[[-9.3738e-07, 4.5635e-07, 1.8347e-07],\n", + " [-3.1665e-08, -2.0592e-06, -9.0897e-07],\n", + " [ 4.5076e-07, 9.8906e-07, -1.8621e-07]],\n", + " \n", + " [[-3.2969e-07, -8.5012e-07, -1.5867e-06],\n", + " [-2.5984e-06, -3.7861e-06, -1.3746e-06],\n", + " [-1.7323e-06, -8.7544e-07, -7.6462e-07]],\n", + " \n", + " [[ 6.5006e-07, -1.0738e-06, 2.9430e-07],\n", + " [-5.8301e-07, -7.0920e-07, 7.2177e-07],\n", + " [-2.1085e-06, -1.2824e-06, -3.7812e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1791e-06, 8.0094e-07, 1.2927e-06],\n", + " [-6.7055e-07, -1.2852e-07, 2.6636e-07],\n", + " [ 8.4285e-07, -1.6782e-06, 8.2888e-08]],\n", + " \n", + " [[ 1.0561e-06, 1.0077e-06, -1.1828e-06],\n", + " [-6.2026e-07, -2.3935e-07, -1.1495e-06],\n", + " [ 6.6124e-07, 8.2160e-08, 6.1095e-07]],\n", + " \n", + " [[ 4.2398e-07, 5.4948e-08, -1.0375e-06],\n", + " [-1.5311e-06, -1.1986e-06, -2.0696e-06],\n", + " [ 1.2759e-07, -1.2014e-06, -1.6992e-06]]],\n", + " \n", + " \n", + " [[[ 4.4797e-07, -6.5984e-07, 4.2701e-07],\n", + " [-5.8673e-07, -5.5134e-07, 8.2795e-07],\n", + " [-1.3448e-06, -6.1933e-07, -8.1025e-08]],\n", + " \n", + " [[ 7.6368e-07, 1.7779e-06, 1.9949e-06],\n", + " [-1.6456e-06, 1.6810e-07, 6.7521e-07],\n", + " [-7.8371e-07, 6.6683e-07, -7.1898e-07]],\n", + " \n", + " [[ 1.4314e-06, 9.5484e-07, 1.6317e-06],\n", + " [ 3.8184e-07, -9.0338e-08, 1.7053e-06],\n", + " [ 1.4249e-07, -1.6019e-07, 1.7006e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-1.5274e-07, 9.5507e-07, 4.4657e-07],\n", + " [ 2.3050e-07, 1.2331e-06, 9.2038e-07],\n", + " [-5.9232e-07, -2.2352e-07, -1.6708e-06]],\n", + " \n", + " [[ 1.3737e-07, -1.7229e-08, 9.8301e-07],\n", + " [ 1.3694e-06, 1.8813e-07, 1.3094e-06],\n", + " [-1.1995e-06, 7.7579e-07, 8.4471e-07]],\n", + " \n", + " [[ 7.0687e-07, 5.5879e-09, 1.3420e-06],\n", + " [ 1.1032e-06, -2.1537e-07, 2.0638e-06],\n", + " [-3.4459e-07, -1.2299e-06, 8.5682e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 4.8429e-08, 1.3467e-06, 1.5162e-06],\n", + " [-1.5004e-06, -1.0617e-06, -3.9302e-07],\n", + " [-1.3690e-06, 4.7497e-07, -1.7416e-07]],\n", + " \n", + " [[-1.1288e-06, -2.2575e-06, -4.2561e-07],\n", + " [ 9.6206e-07, -3.1646e-06, 3.7812e-07],\n", + " [ 1.8626e-09, -1.3215e-06, -2.0005e-06]],\n", + " \n", + " [[-1.8980e-06, -2.8023e-06, -1.6885e-06],\n", + " [-1.5572e-06, -7.3621e-07, -1.4361e-06],\n", + " [-9.2667e-07, -1.1539e-06, -2.5744e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.6058e-06, -3.1519e-06, -1.7546e-06],\n", + " [-2.3330e-06, -1.4864e-06, -1.1800e-06],\n", + " [-3.3323e-06, -1.5590e-06, 3.8557e-07]],\n", + " \n", + " [[ 6.4727e-07, -1.7192e-06, -2.8405e-07],\n", + " [ 4.4168e-07, -2.2091e-06, -1.8645e-06],\n", + " [-1.4142e-06, -2.0384e-06, -2.7249e-06]],\n", + " \n", + " [[ 9.7789e-07, -1.5218e-06, -2.3060e-06],\n", + " [ 3.3472e-06, -8.0280e-07, -3.2596e-08],\n", + " [-6.8499e-07, -2.5388e-06, 3.5204e-07]]],\n", + " \n", + " \n", + " [[[-4.9506e-07, 2.0664e-06, -1.8831e-06],\n", + " [ 4.5262e-07, -1.2107e-07, -9.0338e-07],\n", + " [-1.4901e-07, -1.0990e-07, -1.2470e-06]],\n", + " \n", + " [[-3.2792e-06, -4.2459e-06, -2.5975e-06],\n", + " [-1.8068e-07, -2.0990e-06, -4.2114e-06],\n", + " [-8.0653e-07, -2.4959e-06, -3.7458e-06]],\n", + " \n", + " [[-1.0636e-06, -6.0070e-07, 7.0059e-07],\n", + " [-6.5193e-09, -4.8382e-07, -6.5740e-07],\n", + " [ 3.6304e-07, 2.8266e-07, 1.8920e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.6217e-06, -2.4289e-06, -2.6701e-06],\n", + " [-2.5239e-06, -3.6713e-06, -2.7586e-06],\n", + " [ 4.9127e-08, -1.2238e-06, -7.3528e-07]],\n", + " \n", + " [[-3.7253e-08, -1.2801e-06, -9.4436e-07],\n", + " [ 1.4137e-06, -2.3469e-07, -2.5518e-07],\n", + " [-5.2154e-08, -4.3400e-07, -3.4645e-07]],\n", + " \n", + " [[-9.7416e-07, 4.1840e-07, -8.9966e-07],\n", + " [-8.3307e-07, 2.0070e-06, -8.3260e-07],\n", + " [ 6.0117e-07, 2.4308e-06, 2.2436e-06]]],\n", + " \n", + " \n", + " [[[ 4.3772e-07, 2.3330e-07, 8.8103e-07],\n", + " [-1.8489e-06, -1.2983e-06, -2.5369e-06],\n", + " [ 9.0525e-07, 3.2037e-07, 8.2515e-07]],\n", + " \n", + " [[ 1.4659e-06, -6.3190e-07, -1.6205e-07],\n", + " [-7.7207e-07, 4.3400e-07, 2.8871e-08],\n", + " [-7.7975e-07, -1.8943e-06, -1.9125e-06]],\n", + " \n", + " [[ 8.5030e-07, -1.5274e-07, -3.2596e-07],\n", + " [ 1.1744e-06, 1.0678e-06, 1.1483e-06],\n", + " [-1.6536e-06, -1.4342e-07, -1.9744e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.9116e-07, 9.9186e-07, 1.1157e-06],\n", + " [ 1.5814e-06, 1.0254e-06, 2.9150e-07],\n", + " [-5.4482e-08, -1.2480e-07, 7.8510e-07]],\n", + " \n", + " [[ 2.1756e-06, 8.7777e-07, -1.6391e-07],\n", + " [-1.0292e-06, -2.0433e-06, -9.5181e-07],\n", + " [ 9.1549e-07, 2.3469e-07, -6.8895e-07]],\n", + " \n", + " [[ 6.8359e-07, 8.0140e-07, -8.3633e-07],\n", + " [-1.7288e-07, -1.3168e-06, -5.7183e-07],\n", + " [ 6.8592e-07, -9.3132e-07, 7.6601e-07]]]], device='cuda:0'),\n", + " tensor([ 8.3447e-07, -9.5367e-07, 1.0729e-06, 1.0729e-06, 1.1921e-07,\n", + " -7.7486e-07, -1.0133e-06, -6.5565e-07, -1.1921e-07, -7.7486e-07,\n", + " 4.7684e-07, 7.1526e-07, 2.3842e-07, 7.1526e-07, -8.3447e-07,\n", + " 4.7684e-07, 5.9605e-07, -2.3842e-07, -5.9605e-08, 9.5367e-07,\n", + " -5.9605e-07, 3.5763e-07, 1.1921e-07, -4.1723e-07, 0.0000e+00,\n", + " 7.1526e-07, -5.9605e-07, 1.6689e-06, -7.1526e-07, 4.7684e-07,\n", + " -4.7684e-07, -7.1526e-07, 1.1921e-06, 0.0000e+00, -2.3842e-07,\n", + " 0.0000e+00, -5.9605e-07, -7.1526e-07, 0.0000e+00, 0.0000e+00,\n", + " -9.5367e-07, -1.7881e-07, -4.7684e-07, 1.1921e-07, 3.5763e-07,\n", + " -5.3644e-07, 8.3447e-07, -1.1325e-06, 2.2650e-06, 5.9605e-07,\n", + " -1.1921e-07, -8.9407e-07, -4.1723e-07, 7.1526e-07, -4.1723e-07,\n", + " 1.0729e-06, -5.9605e-08, -2.0266e-06, -1.1921e-06, 7.1526e-07,\n", + " 1.5497e-06, 3.5763e-07, -1.1921e-07, -1.7881e-07, 5.9605e-07,\n", + " -4.1723e-07, 2.3842e-07, -4.7684e-07, 1.0729e-06, 2.3842e-07,\n", + " -1.2517e-06, 4.7684e-07, 8.3447e-07, 9.5367e-07, -1.0729e-06,\n", + " 1.0729e-06, 5.9605e-07, 9.5367e-07, -1.1921e-07, -1.1921e-07,\n", + " -5.9605e-08, -1.0133e-06, -4.1723e-07, 4.7684e-07, 2.3842e-07,\n", + " 8.3447e-07, -8.9407e-07, -1.7881e-06, -2.3842e-07, -4.1723e-07,\n", + " 0.0000e+00, -5.9605e-08, 1.4305e-06, 2.3842e-07, -1.4305e-06,\n", + " 9.5367e-07, 1.4305e-06, 5.9605e-07, -7.1526e-07, 7.1526e-07,\n", + " -7.7486e-07, -5.9605e-07, -5.9605e-08, 2.1458e-06, 0.0000e+00,\n", + " -6.5565e-07, 9.5367e-07, -7.1526e-07, -1.7881e-07, 3.5763e-07,\n", + " 1.6689e-06, -8.9407e-07, 9.5367e-07, 9.5367e-07, -3.5763e-07,\n", + " 1.1921e-06, 0.0000e+00, 2.3842e-07, 0.0000e+00, -4.7684e-07,\n", + " 8.3447e-07, -6.5565e-07, 1.1921e-07, -7.7486e-07, 7.1526e-07,\n", + " -5.9605e-07, 8.3447e-07, -1.7881e-07, -5.9605e-08, 1.1921e-07,\n", + " 1.1921e-06, -1.1325e-06, 0.0000e+00, -1.3113e-06, 4.7684e-07,\n", + " 4.7684e-07, -7.7486e-07, -2.9802e-07, 1.9073e-06, 1.1921e-07,\n", + " -1.1921e-06, -1.8477e-06, 3.5763e-07, -2.3842e-07, 1.1921e-06,\n", + " 8.3447e-07, 3.5763e-07, -2.3842e-06, 5.9605e-07, 4.7684e-07,\n", + " -1.8477e-06, 0.0000e+00, -8.3447e-07, -7.7486e-07, 1.4305e-06,\n", + " 5.9605e-07, 3.5763e-07, 7.1526e-07, 8.3447e-07, 2.3842e-07,\n", + " -1.1921e-07, -1.0133e-06, -8.9407e-07, -1.6689e-06, -6.5565e-07,\n", + " -5.3644e-07, -4.7684e-07, -2.9802e-07, 4.7684e-07, 7.1526e-07,\n", + " -2.3842e-07, -8.3447e-07, -6.5565e-07, 3.5763e-07, 0.0000e+00,\n", + " 1.1921e-07, -8.3447e-07, -1.0133e-06, -8.9407e-07, 5.9605e-07,\n", + " -1.7881e-07, 9.5367e-07, 5.9605e-07, 3.5763e-07, 4.7684e-07,\n", + " 1.5497e-06, 1.1921e-06, -2.9802e-07, 0.0000e+00, -1.1921e-07,\n", + " 1.9073e-06, -6.5565e-07, 1.1921e-07, -5.3644e-07, -5.3644e-07,\n", + " -2.9802e-07, 1.0729e-06, -3.5763e-07, 9.5367e-07, -7.7486e-07,\n", + " 0.0000e+00, 1.1921e-07, -7.1526e-07, 3.5763e-07, -1.0729e-06,\n", + " -2.4438e-06, 3.5763e-07, 3.5763e-07, -8.9407e-07, 0.0000e+00,\n", + " 2.3842e-07, -5.9605e-08, 0.0000e+00, 0.0000e+00, -1.6093e-06,\n", + " 1.1921e-07, -1.4305e-06, -5.3644e-07, -1.7881e-07, -2.3842e-07,\n", + " 9.5367e-07, 1.3113e-06, 8.3447e-07, 1.0729e-06, 1.3113e-06,\n", + " -3.5763e-07, -2.9802e-07, 0.0000e+00, 2.3842e-07, -5.3644e-07,\n", + " 3.5763e-07, -1.0133e-06, 9.5367e-07, 0.0000e+00, 1.0729e-06,\n", + " -7.7486e-07, 1.5497e-06, 1.1921e-07, -5.9605e-08, 8.3447e-07,\n", + " -1.7881e-07, -3.5763e-07, -1.5497e-06, 4.7684e-07, -1.7881e-07,\n", + " -1.1921e-07, -2.3842e-07, -9.5367e-07, 3.5763e-07, -6.5565e-07,\n", + " 3.5763e-07, 9.5367e-07, -7.7486e-07, -1.3709e-06, -1.9073e-06,\n", + " -4.1723e-07], device='cuda:0'),\n", + " tensor([ 1.4996e-06, -9.7831e-07, 2.7066e-07, 4.9675e-07, -1.3047e-07,\n", + " -6.1084e-07, -3.3956e-07, 2.3697e-07, -5.8405e-08, -1.8422e-06,\n", + " 1.0136e-06, 1.1218e-06, -1.0392e-06, 1.8002e-07, -1.3222e-06,\n", + " 4.3368e-07, -1.2230e-07, -2.7726e-07, -3.3027e-08, 8.2742e-07,\n", + " -6.0166e-08, -2.9004e-07, 1.9976e-07, -4.1473e-07, -8.4476e-07,\n", + " -2.9888e-08, 4.1221e-07, 2.7511e-07, -7.9906e-07, 1.1749e-06,\n", + " -9.4270e-07, 1.3504e-07, 8.1365e-07, -1.1818e-06, -7.4986e-07,\n", + " 1.1759e-06, -1.1395e-07, -3.5590e-07, -4.6699e-08, 4.7032e-07,\n", + " -9.2778e-08, 1.9660e-07, -6.9121e-07, 6.0014e-07, -5.8659e-08,\n", + " 2.6431e-08, -5.4318e-08, -1.3243e-06, 5.5120e-07, 7.7604e-07,\n", + " 8.4065e-08, -1.0963e-06, -6.6890e-08, 5.7401e-07, -8.2514e-07,\n", + " 6.3638e-07, 4.8604e-08, -1.1413e-06, -1.1278e-06, 3.3778e-07,\n", + " 8.9280e-07, -8.0010e-07, -1.2865e-06, -3.0982e-07, -2.5761e-07,\n", + " 9.0459e-07, -1.4777e-07, 6.9124e-07, 3.8644e-07, 8.2839e-07,\n", + " -6.2080e-07, 5.4476e-07, -1.6914e-07, 1.6066e-07, -5.3019e-07,\n", + " 8.8009e-07, -6.6119e-07, 1.1239e-07, -3.1296e-07, 1.6062e-07,\n", + " -5.4466e-07, -9.9712e-07, 1.3982e-08, 8.6614e-07, -8.1877e-07,\n", + " 1.0593e-06, -6.8200e-07, -2.3642e-07, -1.4596e-07, 3.5521e-07,\n", + " -1.9287e-07, -5.2644e-08, 3.5540e-07, 6.3538e-07, -9.7509e-07,\n", + " 8.7442e-09, 2.9551e-07, -9.9604e-08, -1.4247e-06, 2.6556e-07,\n", + " -5.0887e-07, -7.7480e-07, 1.7260e-07, 7.8099e-07, 4.0190e-07,\n", + " -2.4187e-07, -2.1173e-07, -8.1731e-07, -4.1487e-07, 2.3282e-07,\n", + " 1.2374e-06, -1.0589e-07, -7.2642e-07, -4.7979e-07, -7.0970e-07,\n", + " 4.4455e-07, -4.0603e-08, 6.6465e-07, -1.0499e-06, -1.2124e-07,\n", + " 5.0759e-07, -3.3925e-07, -3.8467e-07, -3.1564e-07, -4.3414e-09,\n", + " -5.4672e-07, 6.3713e-07, -3.8460e-07, -3.7613e-07, 4.9192e-07,\n", + " 1.1413e-07, 6.7237e-07, 6.8360e-07, -4.4353e-07, 2.2006e-07,\n", + " 7.9770e-07, -6.9525e-07, 4.9770e-07, 1.2657e-06, 9.7913e-07,\n", + " 1.8810e-07, 3.0461e-07, -2.4066e-07, -5.0781e-07, 7.8403e-07,\n", + " 1.7687e-07, 4.7308e-07, -1.8610e-06, 9.0300e-07, -3.3854e-07,\n", + " -7.2396e-07, 5.2172e-07, -8.8731e-07, -5.0957e-07, 2.6942e-07,\n", + " -2.1702e-07, -9.7588e-08, 5.1437e-07, 8.0258e-07, -3.9910e-08,\n", + " 3.0799e-07, -1.0998e-06, -9.4656e-07, 9.5955e-08, 6.1174e-07,\n", + " 8.9107e-08, 7.4862e-07, 2.4129e-07, 1.7060e-07, -8.3884e-08,\n", + " -1.2298e-06, 6.1371e-07, -3.2114e-07, 2.4378e-07, 4.1363e-07,\n", + " 4.4874e-08, -1.3397e-07, -2.1337e-07, -9.1465e-07, -1.3507e-06,\n", + " 6.7283e-08, -1.3089e-07, -5.7291e-07, 3.9417e-07, 6.9060e-07,\n", + " 1.2198e-06, 4.8669e-07, 2.6102e-07, 6.2616e-07, -1.1010e-06,\n", + " 7.1556e-07, 2.0051e-07, -1.2558e-06, -6.5537e-07, -7.2214e-07,\n", + " -2.6249e-07, 7.3104e-07, -7.9916e-08, 5.6357e-07, -2.2323e-07,\n", + " -9.2495e-08, 7.5343e-07, -7.5690e-07, 3.1329e-07, -1.1467e-07,\n", + " -7.7100e-07, 8.4661e-08, 7.9552e-07, -9.5221e-07, -4.5532e-07,\n", + " 1.0231e-06, -1.1224e-06, 8.0024e-07, 4.1700e-07, -7.5773e-07,\n", + " -3.2492e-07, 2.5452e-09, 8.2564e-07, 1.9540e-07, -1.0915e-06,\n", + " 8.0702e-07, 6.7307e-07, 6.1853e-07, -5.3461e-08, 9.1746e-07,\n", + " 5.0425e-07, -4.3953e-07, 8.6081e-07, -6.9142e-07, -3.0641e-07,\n", + " -7.6885e-08, 1.2958e-07, 1.8386e-07, -2.7983e-07, 6.6745e-07,\n", + " -5.0889e-07, 1.3176e-06, -7.9853e-08, -4.5977e-07, 2.9240e-08,\n", + " -1.6256e-06, -2.2015e-07, -4.1468e-08, -2.0701e-07, 2.5846e-08,\n", + " 3.0667e-08, 2.3528e-08, -2.0532e-06, -1.8726e-07, -2.9575e-07,\n", + " 1.2827e-06, 1.0446e-07, -6.4511e-07, 4.5967e-07, 2.4757e-07,\n", + " -1.0032e-06], device='cuda:0'),\n", + " tensor([[[[-2.2799e-06]],\n", + " \n", + " [[-6.4820e-07]],\n", + " \n", + " [[ 2.3972e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-5.5134e-07]],\n", + " \n", + " [[-5.8720e-07]],\n", + " \n", + " [[ 4.1351e-07]]],\n", + " \n", + " \n", + " [[[-3.0156e-06]],\n", + " \n", + " [[ 1.9148e-06]],\n", + " \n", + " [[-1.9222e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-7.7300e-08]],\n", + " \n", + " [[-9.1642e-07]],\n", + " \n", + " [[-1.5050e-06]]],\n", + " \n", + " \n", + " [[[-1.6913e-06]],\n", + " \n", + " [[ 1.0021e-06]],\n", + " \n", + " [[-2.2352e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.4715e-06]],\n", + " \n", + " [[-7.4506e-07]],\n", + " \n", + " [[-2.9877e-06]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 4.4703e-07]],\n", + " \n", + " [[-2.5332e-06]],\n", + " \n", + " [[-1.9819e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-2.7604e-06]],\n", + " \n", + " [[ 2.8312e-07]],\n", + " \n", + " [[-3.7998e-07]]],\n", + " \n", + " \n", + " [[[-3.4645e-07]],\n", + " \n", + " [[-2.6375e-06]],\n", + " \n", + " [[-4.0606e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-9.1642e-07]],\n", + " \n", + " [[ 2.7148e-06]],\n", + " \n", + " [[ 2.7791e-06]]],\n", + " \n", + " \n", + " [[[ 2.8312e-06]],\n", + " \n", + " [[ 3.3658e-06]],\n", + " \n", + " [[-2.3805e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.4273e-06]],\n", + " \n", + " [[ 4.0196e-06]],\n", + " \n", + " [[-1.3411e-07]]]], device='cuda:0'),\n", + " tensor([ 5.9605e-07, -1.6093e-06, -4.7684e-07, 4.7684e-07, -4.1723e-07,\n", + " -7.1526e-07, 2.3842e-07, 1.1921e-07, -1.7881e-07, -1.7285e-06,\n", + " 1.4305e-06, 1.3113e-06, -1.1325e-06, -1.0729e-06, -2.2054e-06,\n", + " 5.9605e-07, 3.5763e-07, 3.5763e-07, 2.3842e-07, 2.3842e-07,\n", + " 1.7881e-06, -4.1723e-07, -5.3644e-07, -1.1921e-07, -2.9802e-07,\n", + " -4.1723e-07, 1.0729e-06, 3.5763e-07, -2.3842e-07, 8.3447e-07,\n", + " -7.1526e-07, 9.5367e-07, -5.9605e-08, -9.5367e-07, -4.1723e-07,\n", + " -1.7881e-07, 2.3842e-07, -5.9605e-08, 5.9605e-07, 1.0729e-06,\n", + " 1.7881e-06, 4.7684e-07, -7.1526e-07, -2.3842e-07, -1.7881e-07,\n", + " 4.7684e-07, -1.3709e-06, 2.3842e-07, -1.1921e-06, 0.0000e+00,\n", + " 2.3842e-07, -1.4901e-06, 1.1921e-07, 9.5367e-07, -1.0133e-06,\n", + " -2.3842e-07, -9.5367e-07, 9.5367e-07, 1.3113e-06, 8.3447e-07,\n", + " 4.7684e-07, 0.0000e+00, -1.6689e-06, 3.5763e-07, -2.3842e-07,\n", + " 1.5497e-06, -2.3842e-07, 1.5497e-06, 4.7684e-07, 3.5763e-07,\n", + " -1.6093e-06, -7.7486e-07, -7.1526e-07, 1.1921e-07, -4.1723e-07,\n", + " 9.5367e-07, -1.7881e-07, 1.1921e-07, -5.9605e-07, -1.1921e-07,\n", + " -8.9407e-07, -4.7684e-07, -1.1921e-06, 1.4305e-06, -8.9407e-07,\n", + " -4.7684e-07, -6.5565e-07, 8.3447e-07, -4.7684e-07, 8.3447e-07,\n", + " 3.5763e-07, -2.9802e-07, -1.0133e-06, 2.8610e-06, 4.7684e-07,\n", + " -7.7486e-07, 3.5763e-07, -3.5763e-07, -1.6093e-06, 4.7684e-07,\n", + " 1.1921e-06, 1.1921e-07, 0.0000e+00, -5.9605e-07, 2.3842e-07,\n", + " 1.1921e-06, -5.3644e-07, 4.7684e-07, 3.5763e-07, -3.5763e-07,\n", + " 8.3447e-07, -7.1526e-07, -5.9605e-08, -1.2517e-06, -7.1526e-07,\n", + " 4.7684e-07, 9.5367e-07, 2.3842e-07, -5.9605e-07, 8.3447e-07,\n", + " 7.1526e-07, 1.1921e-07, -5.3644e-07, 2.3842e-07, 3.5763e-07,\n", + " -4.1723e-07, 4.7684e-07, 3.5763e-07, -2.3842e-07, 7.1526e-07,\n", + " -6.5565e-07, 1.1921e-06, 8.3447e-07, 1.1921e-07, -3.5763e-07,\n", + " 3.5763e-07, -8.3447e-07, -4.1723e-07, 1.1921e-07, 9.5367e-07,\n", + " 2.3842e-07, 1.0729e-06, -4.1723e-07, 1.1921e-07, 9.5367e-07,\n", + " -5.9605e-07, 8.3447e-07, 3.5763e-07, 5.9605e-07, -7.1526e-07,\n", + " 4.7684e-07, 1.0729e-06, -7.1526e-07, -2.9802e-07, -1.1921e-07,\n", + " -1.1325e-06, -1.4305e-06, -4.7684e-07, -5.9605e-07, 2.3842e-07,\n", + " -5.9605e-07, -4.1723e-07, -8.3447e-07, 1.1921e-07, 1.5497e-06,\n", + " 7.1526e-07, 1.4305e-06, -6.5565e-07, 5.9605e-07, 4.7684e-07,\n", + " -2.1458e-06, 1.6689e-06, -8.9407e-07, -2.9802e-07, 1.1921e-07,\n", + " -7.7486e-07, 1.1921e-06, 9.5367e-07, -4.7684e-07, -1.3113e-06,\n", + " -5.9605e-08, -2.3842e-07, 0.0000e+00, -5.3644e-07, -5.9605e-07,\n", + " 7.1526e-07, -8.3447e-07, 9.5367e-07, 3.5763e-07, -1.1325e-06,\n", + " -7.7486e-07, 0.0000e+00, -8.3447e-07, -1.0729e-06, 3.5763e-07,\n", + " 8.3447e-07, 0.0000e+00, 4.7684e-07, 7.1526e-07, 4.7684e-07,\n", + " 1.1921e-07, 7.1526e-07, -2.3842e-07, 2.3842e-07, 7.1526e-07,\n", + " 1.9073e-06, -5.3644e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07,\n", + " 1.0729e-06, -1.1921e-06, 1.3113e-06, -6.5565e-07, -5.3644e-07,\n", + " -4.7684e-07, 1.0729e-06, 1.4305e-06, 4.7684e-07, -1.7285e-06,\n", + " -2.9802e-07, 0.0000e+00, 2.3842e-07, -9.5367e-07, 9.5367e-07,\n", + " 7.1526e-07, -2.3842e-07, 5.9605e-07, -6.5565e-07, -1.1921e-06,\n", + " -5.9605e-07, 1.1921e-06, -4.1723e-07, 1.0729e-06, 0.0000e+00,\n", + " -5.3644e-07, 1.1921e-06, -1.1921e-07, -1.0729e-06, -1.1921e-07,\n", + " -3.8147e-06, 0.0000e+00, 1.6689e-06, 1.1921e-07, -7.1526e-07,\n", + " -5.3644e-07, 0.0000e+00, -2.3842e-06, 2.3842e-07, 3.5763e-07,\n", + " 1.1921e-07, -1.7881e-07, -1.7881e-07, 1.3113e-06, 1.0729e-06,\n", + " -8.9407e-07], device='cuda:0'),\n", + " tensor([ 1.4996e-06, -9.7831e-07, 2.7066e-07, 4.9675e-07, -1.3047e-07,\n", + " -6.1084e-07, -3.3956e-07, 2.3697e-07, -5.8405e-08, -1.8422e-06,\n", + " 1.0136e-06, 1.1218e-06, -1.0392e-06, 1.8002e-07, -1.3222e-06,\n", + " 4.3368e-07, -1.2230e-07, -2.7726e-07, -3.3027e-08, 8.2742e-07,\n", + " -6.0166e-08, -2.9004e-07, 1.9976e-07, -4.1473e-07, -8.4476e-07,\n", + " -2.9888e-08, 4.1221e-07, 2.7511e-07, -7.9906e-07, 1.1749e-06,\n", + " -9.4270e-07, 1.3504e-07, 8.1365e-07, -1.1818e-06, -7.4986e-07,\n", + " 1.1759e-06, -1.1395e-07, -3.5590e-07, -4.6699e-08, 4.7032e-07,\n", + " -9.2778e-08, 1.9660e-07, -6.9121e-07, 6.0014e-07, -5.8659e-08,\n", + " 2.6431e-08, -5.4318e-08, -1.3243e-06, 5.5120e-07, 7.7604e-07,\n", + " 8.4065e-08, -1.0963e-06, -6.6890e-08, 5.7401e-07, -8.2514e-07,\n", + " 6.3638e-07, 4.8604e-08, -1.1413e-06, -1.1278e-06, 3.3778e-07,\n", + " 8.9280e-07, -8.0010e-07, -1.2865e-06, -3.0982e-07, -2.5761e-07,\n", + " 9.0459e-07, -1.4777e-07, 6.9124e-07, 3.8644e-07, 8.2839e-07,\n", + " -6.2080e-07, 5.4476e-07, -1.6914e-07, 1.6066e-07, -5.3019e-07,\n", + " 8.8009e-07, -6.6119e-07, 1.1239e-07, -3.1296e-07, 1.6062e-07,\n", + " -5.4466e-07, -9.9712e-07, 1.3982e-08, 8.6614e-07, -8.1877e-07,\n", + " 1.0593e-06, -6.8200e-07, -2.3642e-07, -1.4596e-07, 3.5521e-07,\n", + " -1.9287e-07, -5.2644e-08, 3.5540e-07, 6.3538e-07, -9.7509e-07,\n", + " 8.7442e-09, 2.9551e-07, -9.9604e-08, -1.4247e-06, 2.6556e-07,\n", + " -5.0887e-07, -7.7480e-07, 1.7260e-07, 7.8099e-07, 4.0190e-07,\n", + " -2.4187e-07, -2.1173e-07, -8.1731e-07, -4.1487e-07, 2.3282e-07,\n", + " 1.2374e-06, -1.0589e-07, -7.2642e-07, -4.7979e-07, -7.0970e-07,\n", + " 4.4455e-07, -4.0603e-08, 6.6465e-07, -1.0499e-06, -1.2124e-07,\n", + " 5.0759e-07, -3.3925e-07, -3.8467e-07, -3.1564e-07, -4.3414e-09,\n", + " -5.4672e-07, 6.3713e-07, -3.8460e-07, -3.7613e-07, 4.9192e-07,\n", + " 1.1413e-07, 6.7237e-07, 6.8360e-07, -4.4353e-07, 2.2006e-07,\n", + " 7.9770e-07, -6.9525e-07, 4.9770e-07, 1.2657e-06, 9.7913e-07,\n", + " 1.8810e-07, 3.0461e-07, -2.4066e-07, -5.0781e-07, 7.8403e-07,\n", + " 1.7687e-07, 4.7308e-07, -1.8610e-06, 9.0300e-07, -3.3854e-07,\n", + " -7.2396e-07, 5.2172e-07, -8.8731e-07, -5.0957e-07, 2.6942e-07,\n", + " -2.1702e-07, -9.7588e-08, 5.1437e-07, 8.0258e-07, -3.9910e-08,\n", + " 3.0799e-07, -1.0998e-06, -9.4656e-07, 9.5955e-08, 6.1174e-07,\n", + " 8.9107e-08, 7.4862e-07, 2.4129e-07, 1.7060e-07, -8.3884e-08,\n", + " -1.2298e-06, 6.1371e-07, -3.2114e-07, 2.4378e-07, 4.1363e-07,\n", + " 4.4874e-08, -1.3397e-07, -2.1337e-07, -9.1465e-07, -1.3507e-06,\n", + " 6.7283e-08, -1.3089e-07, -5.7291e-07, 3.9417e-07, 6.9060e-07,\n", + " 1.2198e-06, 4.8669e-07, 2.6102e-07, 6.2616e-07, -1.1010e-06,\n", + " 7.1556e-07, 2.0051e-07, -1.2558e-06, -6.5537e-07, -7.2214e-07,\n", + " -2.6249e-07, 7.3104e-07, -7.9916e-08, 5.6357e-07, -2.2323e-07,\n", + " -9.2495e-08, 7.5343e-07, -7.5690e-07, 3.1329e-07, -1.1467e-07,\n", + " -7.7100e-07, 8.4661e-08, 7.9552e-07, -9.5221e-07, -4.5532e-07,\n", + " 1.0231e-06, -1.1224e-06, 8.0024e-07, 4.1700e-07, -7.5773e-07,\n", + " -3.2492e-07, 2.5452e-09, 8.2564e-07, 1.9540e-07, -1.0915e-06,\n", + " 8.0702e-07, 6.7307e-07, 6.1853e-07, -5.3461e-08, 9.1746e-07,\n", + " 5.0425e-07, -4.3953e-07, 8.6081e-07, -6.9142e-07, -3.0641e-07,\n", + " -7.6885e-08, 1.2958e-07, 1.8386e-07, -2.7983e-07, 6.6745e-07,\n", + " -5.0889e-07, 1.3176e-06, -7.9853e-08, -4.5977e-07, 2.9240e-08,\n", + " -1.6256e-06, -2.2015e-07, -4.1468e-08, -2.0701e-07, 2.5846e-08,\n", + " 3.0667e-08, 2.3528e-08, -2.0532e-06, -1.8726e-07, -2.9575e-07,\n", + " 1.2827e-06, 1.0446e-07, -6.4511e-07, 4.5967e-07, 2.4757e-07,\n", + " -1.0032e-06], device='cuda:0'),\n", + " tensor([[[[-6.1654e-07, -4.9314e-07, 5.4017e-08],\n", + " [-1.4529e-07, 8.9779e-07, 5.5090e-07],\n", + " [-1.3281e-06, -1.1828e-07, -8.6369e-07]],\n", + " \n", + " [[-6.3330e-08, 4.1444e-08, -4.6939e-07],\n", + " [-3.6322e-07, 9.8161e-07, -1.7136e-07],\n", + " [-1.9995e-06, -2.0228e-06, -3.5297e-06]],\n", + " \n", + " [[ 1.7993e-06, 1.2540e-06, -6.5938e-07],\n", + " [-1.2177e-07, -2.1346e-06, -8.2050e-07],\n", + " [-1.4994e-07, 2.8219e-07, 1.2014e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.4878e-07, 1.0729e-06, 1.4375e-06],\n", + " [-1.1940e-06, 2.0489e-08, 2.0833e-07],\n", + " [ 2.2724e-07, 3.8347e-07, -6.7637e-08]],\n", + " \n", + " [[-5.9605e-07, -7.2364e-07, -5.2806e-07],\n", + " [-9.4250e-07, 5.8953e-07, -7.1479e-07],\n", + " [-2.1886e-07, -5.7742e-08, 3.5949e-07]],\n", + " \n", + " [[ 3.2410e-07, -2.7008e-07, -4.6333e-08],\n", + " [-1.1334e-06, 1.0040e-06, -2.6636e-07],\n", + " [-4.7125e-07, -5.1502e-07, -8.8848e-07]]],\n", + " \n", + " \n", + " [[[-5.1409e-07, -2.0303e-07, -2.5751e-07],\n", + " [-1.5739e-07, -6.2305e-07, 1.1222e-07],\n", + " [ 3.1292e-07, -7.5065e-07, -2.5012e-07]],\n", + " \n", + " [[ 5.4459e-07, 7.3574e-07, 8.7265e-07],\n", + " [ 7.3016e-07, 1.1828e-06, 1.2054e-06],\n", + " [ 6.1654e-07, 4.5309e-07, 1.0747e-06]],\n", + " \n", + " [[-5.1304e-07, 1.8199e-07, -2.0675e-07],\n", + " [-2.7753e-07, -1.3737e-07, -6.6496e-07],\n", + " [-2.3469e-07, 2.4587e-07, -8.7358e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 6.5845e-07, 4.1933e-07, -2.5891e-07],\n", + " [-1.4901e-07, 7.9721e-07, 3.3528e-08],\n", + " [ 1.8813e-07, 8.4937e-07, 1.1865e-06]],\n", + " \n", + " [[-9.9838e-07, -6.5193e-08, -8.0746e-07],\n", + " [ 5.0850e-07, -2.4633e-07, -4.3027e-07],\n", + " [-9.6858e-07, -6.5933e-07, -2.1514e-07]],\n", + " \n", + " [[ 1.3085e-06, 1.3881e-06, 2.0396e-06],\n", + " [ 2.0061e-06, 1.4156e-06, 1.9734e-06],\n", + " [ 7.3854e-07, 2.2110e-06, 7.4517e-07]]],\n", + " \n", + " \n", + " [[[ 3.5390e-07, 1.2008e-07, -5.3179e-07],\n", + " [ 1.4558e-07, -8.4518e-08, 8.2608e-07],\n", + " [ 1.4156e-07, -3.4831e-07, -2.9244e-07]],\n", + " \n", + " [[-7.6368e-08, 6.7241e-07, -1.9907e-07],\n", + " [ 1.2573e-07, -1.5013e-06, 5.6624e-07],\n", + " [-1.4901e-08, -2.6729e-07, 3.7719e-07]],\n", + " \n", + " [[ 1.4156e-07, 1.8242e-07, 5.4063e-07],\n", + " [-3.9581e-07, 2.4261e-07, -5.5972e-07],\n", + " [ 4.8894e-07, 3.0780e-07, 4.6566e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[-7.0967e-07, -1.3793e-06, -9.8627e-07],\n", + " [-1.6415e-07, -5.4631e-07, -1.0971e-06],\n", + " [ 6.5193e-09, -3.4273e-07, -8.3400e-07]],\n", + " \n", + " [[-2.8498e-07, -1.5674e-06, -1.4754e-06],\n", + " [ 2.4214e-07, -7.8417e-07, -2.9519e-06],\n", + " [ 3.8836e-07, -1.0817e-06, -8.5123e-07]],\n", + " \n", + " [[ 5.1595e-07, -2.0117e-07, 2.5285e-07],\n", + " [ 7.2643e-08, -7.3574e-08, 1.8720e-07],\n", + " [-2.9430e-07, 5.3458e-07, 1.6298e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 2.1607e-07, 1.0896e-07, 7.0222e-07],\n", + " [-1.3970e-07, 3.8091e-07, 8.1619e-07],\n", + " [-7.3668e-07, -1.7695e-07, -3.9302e-07]],\n", + " \n", + " [[-9.7509e-07, -1.2200e-06, -7.5577e-07],\n", + " [-1.1199e-07, -1.6950e-07, 8.1956e-08],\n", + " [ 1.4994e-07, -9.1363e-07, -8.5682e-07]],\n", + " \n", + " [[ 1.4069e-06, 1.5609e-06, 1.0058e-06],\n", + " [-2.5705e-07, -3.2131e-07, 1.0179e-06],\n", + " [-1.2545e-06, -3.0082e-07, -1.4236e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-5.1083e-07, -4.7963e-08, 7.4785e-07],\n", + " [ 2.9942e-07, 5.4389e-07, -4.2841e-07],\n", + " [ 5.8673e-07, 7.4599e-07, 2.8871e-07]],\n", + " \n", + " [[-6.2911e-07, -4.0047e-07, -5.7789e-07],\n", + " [-1.0766e-06, 6.2212e-07, -1.6764e-07],\n", + " [-3.1944e-07, 6.7055e-08, -8.1770e-07]],\n", + " \n", + " [[-4.7032e-07, 3.5670e-07, -2.1863e-07],\n", + " [-1.4901e-07, -1.1520e-06, -6.8452e-08],\n", + " [-1.1965e-06, -1.6887e-06, -4.8056e-07]]],\n", + " \n", + " \n", + " [[[-3.7812e-07, 4.5268e-07, 1.4221e-06],\n", + " [-9.9652e-07, 8.6613e-08, 2.9989e-07],\n", + " [-9.8161e-07, -3.2876e-07, 7.2736e-07]],\n", + " \n", + " [[ 6.9535e-07, -9.7789e-08, 4.3935e-07],\n", + " [ 2.5332e-07, -1.6351e-07, -1.3113e-06],\n", + " [-2.9802e-08, -4.0606e-07, -8.0373e-07]],\n", + " \n", + " [[-5.3272e-07, -8.8522e-07, -1.8626e-06],\n", + " [ 5.4156e-07, -1.1586e-06, 4.2841e-08],\n", + " [-1.8962e-06, -9.6112e-07, -5.7928e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 3.9209e-07, 1.1269e-07, -9.7137e-07],\n", + " [-2.3190e-07, 1.1921e-07, -1.2564e-06],\n", + " [ 8.1211e-07, -6.6450e-07, -1.5199e-06]],\n", + " \n", + " [[-3.1013e-07, -3.4319e-07, 5.9605e-08],\n", + " [-6.1281e-07, 5.7369e-07, -1.2470e-06],\n", + " [-5.0361e-07, 7.2643e-08, -1.6140e-06]],\n", + " \n", + " [[ 1.6233e-06, -7.0862e-07, 3.6042e-07],\n", + " [ 1.2033e-06, 3.8836e-07, 1.0934e-06],\n", + " [ 1.0617e-06, -8.4192e-07, 9.0525e-07]]],\n", + " \n", + " \n", + " [[[ 9.9465e-07, 2.3458e-07, 1.2764e-06],\n", + " [-4.8243e-07, -2.9337e-07, -5.8860e-07],\n", + " [ 5.6718e-07, 1.7229e-06, 1.4063e-07]],\n", + " \n", + " [[ 1.0654e-06, 1.1083e-06, -5.3877e-07],\n", + " [-8.4285e-07, 4.8801e-07, -9.4296e-07],\n", + " [ 9.2946e-07, 1.8626e-09, -3.4785e-07]],\n", + " \n", + " [[-1.7835e-07, -1.5274e-07, -3.1572e-07],\n", + " [ 1.2014e-06, -1.1344e-06, 1.2740e-06],\n", + " [ 3.8790e-07, 3.5204e-07, 6.6310e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.1921e-07, -5.2713e-07, -1.5167e-06],\n", + " [ 2.4820e-07, 1.3327e-06, -5.0012e-07],\n", + " [-2.4773e-07, -2.1528e-06, -7.0385e-07]],\n", + " \n", + " [[ 2.0768e-07, -5.4087e-07, 4.3865e-07],\n", + " [-7.7859e-07, 1.7709e-06, -5.0850e-07],\n", + " [-1.5590e-06, -4.3958e-07, -1.8477e-06]],\n", + " \n", + " [[ 2.1979e-06, -1.0855e-06, 1.7428e-06],\n", + " [ 3.2224e-07, -7.2177e-07, -2.8126e-07],\n", + " [-3.7998e-07, 1.0114e-06, 9.7230e-07]]]], device='cuda:0'),\n", + " tensor([-4.7684e-07, 7.1526e-07, -5.9605e-08, 2.3842e-07, -2.3842e-07,\n", + " -4.7684e-07, 4.7684e-07, -8.9407e-07, 7.1526e-07, 1.1921e-07,\n", + " 4.7684e-07, -5.9605e-07, 1.1921e-06, 2.3842e-07, 2.3842e-07,\n", + " -2.3842e-07, 8.3447e-07, -1.1921e-07, -2.9802e-07, 4.7684e-07,\n", + " -1.7881e-07, -2.9802e-07, -7.1526e-07, -3.5763e-07, 0.0000e+00,\n", + " -5.9605e-08, 3.5763e-07, 9.5367e-07, 3.5763e-07, 7.1526e-07,\n", + " -5.3644e-07, -1.1921e-07, -5.9605e-08, 3.5763e-07, -1.7881e-07,\n", + " 1.1921e-07, 3.5763e-07, 2.3842e-07, -3.5763e-07, -4.1723e-07,\n", + " 1.1921e-07, 2.3842e-07, -2.9802e-07, 3.5763e-07, -3.5763e-07,\n", + " -5.9605e-07, 1.1921e-07, 4.7684e-07, 0.0000e+00, 4.7684e-07,\n", + " 0.0000e+00, -3.5763e-07, -6.5565e-07, -4.1723e-07, -5.3644e-07,\n", + " 3.5763e-07, -5.9605e-08, -4.1723e-07, -1.1921e-07, -5.3644e-07,\n", + " -1.7881e-07, 3.5763e-07, -2.3842e-07, -1.7881e-07, -1.7285e-06,\n", + " 1.1921e-07, -2.9802e-07, -1.1921e-07, 2.3842e-07, 8.3447e-07,\n", + " 1.1921e-07, -3.5763e-07, -2.3842e-07, 1.1921e-07, 1.1921e-07,\n", + " 1.1921e-07, 1.1921e-07, 2.3842e-07, 1.1921e-07, -7.1526e-07,\n", + " -2.3842e-07, 3.5763e-07, 4.7684e-07, -8.3447e-07, 2.3842e-07,\n", + " -5.9605e-08, 0.0000e+00, -2.3842e-07, -2.9802e-07, 4.7684e-07,\n", + " -5.9605e-08, -4.1723e-07, 2.3842e-07, 0.0000e+00, 8.3447e-07,\n", + " -4.1723e-07, -1.1921e-07, 0.0000e+00, 0.0000e+00, 4.7684e-07,\n", + " -7.7486e-07, 5.9605e-07, -5.9605e-07, 3.5763e-07, -8.3447e-07,\n", + " 8.3447e-07, -2.9802e-07, 0.0000e+00, -7.7486e-07, 2.3842e-07,\n", + " 0.0000e+00, 4.7684e-07, 5.9605e-07, 4.7684e-07, 2.3842e-07,\n", + " 5.9605e-07, 1.1921e-07, -2.3842e-07, 9.5367e-07, 9.5367e-07,\n", + " -4.7684e-07, 5.9605e-07, -4.1723e-07, 5.9605e-07, 0.0000e+00,\n", + " -3.5763e-07, -5.9605e-07, 0.0000e+00, -1.7881e-07, 1.1921e-07,\n", + " 0.0000e+00, -1.1325e-06, 1.1921e-07, -2.9802e-07, 0.0000e+00,\n", + " 1.1921e-07, 4.7684e-07, 4.7684e-07, 2.3842e-07, 7.1526e-07,\n", + " 0.0000e+00, 1.1921e-07, 2.3842e-07, -5.9605e-08, -3.5763e-07,\n", + " 3.5763e-07, -8.3447e-07, 2.3842e-07, -7.1526e-07, 9.5367e-07,\n", + " 3.5763e-07, 7.1526e-07, 4.7684e-07, -4.1723e-07, 0.0000e+00,\n", + " -5.9605e-08, -4.1723e-07, -7.7486e-07, -2.3842e-07, 1.4305e-06,\n", + " 2.3842e-07, 0.0000e+00, -1.1921e-07, -8.9407e-07, -4.7684e-07,\n", + " 1.1921e-07, -8.3447e-07, 1.1921e-07, -1.0133e-06, -1.7881e-07,\n", + " 1.1921e-07, -5.9605e-07, -3.5763e-07, 1.1921e-07, -5.9605e-08,\n", + " -1.7881e-07, 0.0000e+00, 1.1921e-06, 2.3842e-07, -8.9407e-07,\n", + " 4.7684e-07, -5.9605e-07, 4.7684e-07, -5.3644e-07, 7.1526e-07,\n", + " -2.9802e-07, 2.3842e-07, -5.9605e-08, 2.3842e-07, 1.1921e-07,\n", + " -1.1921e-07, -4.1723e-07, 7.1526e-07, -5.9605e-08, 4.7684e-07,\n", + " -1.1921e-07, -7.1526e-07, 1.1921e-07, 9.5367e-07, -2.9802e-07,\n", + " 4.7684e-07, -5.9605e-08, 5.9605e-07, -8.3447e-07, -5.9605e-07,\n", + " 2.3842e-07, 2.3842e-07, 4.7684e-07, 1.0729e-06, 1.1921e-06,\n", + " 5.9605e-07, -1.1325e-06, -7.1526e-07, 1.7881e-06, -4.1723e-07,\n", + " 1.1921e-07, 1.1921e-07, -4.7684e-07, -1.1921e-07, -2.3842e-07,\n", + " -1.1921e-07, 0.0000e+00, -1.7881e-07, -2.9802e-07, -9.5367e-07,\n", + " 1.1921e-07, 0.0000e+00, 0.0000e+00, -7.1526e-07, 2.3842e-07,\n", + " 0.0000e+00, 7.1526e-07, 1.1921e-06, 3.5763e-07, -5.9605e-08,\n", + " 5.9605e-07, -1.1921e-07, -5.9605e-08, -7.1526e-07, -5.3644e-07,\n", + " 3.5763e-07, 4.7684e-07, -1.7881e-07, 3.5763e-07, -7.1526e-07,\n", + " 1.1921e-07, -1.1921e-07, -2.3842e-07, 0.0000e+00, 1.1921e-07,\n", + " 1.1921e-07, -3.5763e-07, 3.5763e-07, -2.9802e-07, 3.5763e-07,\n", + " 1.1921e-07, 2.3842e-07, 3.5763e-07, 4.7684e-07, -5.9605e-07,\n", + " 1.1921e-07, -1.1921e-07, 4.7684e-07, 1.4305e-06, 1.1921e-07,\n", + " -5.9605e-07, -1.1921e-07, 0.0000e+00, -3.5763e-07, 0.0000e+00,\n", + " -2.3842e-07, 8.3447e-07, -5.9605e-08, 3.5763e-07, -1.4901e-06,\n", + " 2.3842e-07, -1.1921e-07, 8.3447e-07, 2.3842e-07, 0.0000e+00,\n", + " -3.5763e-07, -6.5565e-07, -2.3842e-07, 3.5763e-07, -4.1723e-07,\n", + " -7.1526e-07, 2.3842e-07, -1.7881e-07, 1.0729e-06, -7.7486e-07,\n", + " -4.7684e-07, 2.3842e-07, -1.7881e-07, 3.5763e-07, -5.3644e-07,\n", + " 0.0000e+00, 4.7684e-07, 4.7684e-07, -1.1921e-07, -3.5763e-07,\n", + " -1.0133e-06, 0.0000e+00, 7.1526e-07, 1.1921e-07, -2.3842e-07,\n", + " -8.9407e-07, -7.1526e-07, 2.3842e-07, -8.9407e-07, -1.1921e-07,\n", + " -2.3842e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07, 8.3447e-07,\n", + " 0.0000e+00, 2.3842e-07, -5.3644e-07, 1.3113e-06, 9.5367e-07,\n", + " -1.1921e-07, -1.1921e-07, 1.1921e-07, 4.7684e-07, 1.1921e-07,\n", + " 5.9605e-07, -2.9802e-07, 2.3842e-07, 0.0000e+00, 1.0729e-06,\n", + " 2.3842e-07, 2.3842e-07, -2.3842e-07, -2.9802e-07, -7.7486e-07,\n", + " 1.1921e-06, 2.3842e-07, 0.0000e+00, 3.5763e-07, 5.9605e-07,\n", + " -3.5763e-07, -1.1921e-07, -7.7486e-07, -1.3709e-06, -5.9605e-07,\n", + " -3.5763e-07, -4.7684e-07, -1.1921e-07, 0.0000e+00, 1.1921e-07,\n", + " 1.1921e-07, 1.1921e-07, -2.9802e-07, -3.5763e-07, -1.1921e-07,\n", + " -2.3842e-07, -4.1723e-07, 1.1921e-07, -3.5763e-07, -2.9802e-07,\n", + " 5.9605e-07, 3.5763e-07, 2.3842e-07, -9.5367e-07, -1.7881e-07,\n", + " -1.7881e-07, -2.3842e-07, -2.3842e-07, 0.0000e+00, -6.5565e-07,\n", + " -4.7684e-07, -1.1921e-07, -4.1723e-07, 5.9605e-07, -4.1723e-07,\n", + " -1.1921e-07, 1.1921e-07, -2.9802e-07, 3.5763e-07, -2.3842e-07,\n", + " 5.9605e-07, -5.9605e-08, 1.1921e-07, -4.7684e-07, 2.3842e-07,\n", + " 7.1526e-07, -1.7881e-07, -4.7684e-07, 4.7684e-07, 5.9605e-07,\n", + " 1.1921e-07, -4.1723e-07, -5.9605e-07, 3.5763e-07, 2.3842e-07,\n", + " -1.1921e-07, -1.1921e-07, 1.1921e-07, -8.9407e-07, -4.1723e-07,\n", + " -1.7881e-07, 2.3842e-07, -5.9605e-08, -1.7881e-07, -1.7881e-07,\n", + " 9.5367e-07, -3.5763e-07, -1.1921e-07, 5.9605e-07, -2.3842e-07,\n", + " -1.0729e-06, 5.9605e-07, 0.0000e+00, -1.1921e-07, -1.1921e-07,\n", + " -2.3842e-07, 4.7684e-07, 4.7684e-07, 2.3842e-07, -5.9605e-07,\n", + " -1.7881e-07, -5.9605e-07, 1.1921e-07, 3.5763e-07, 1.1921e-07,\n", + " -4.1723e-07, 1.1921e-07, 2.3842e-07, 5.9605e-07, 4.7684e-07,\n", + " 0.0000e+00, 5.9605e-07, 4.7684e-07, 3.5763e-07, -5.9605e-08,\n", + " -5.9605e-08, 4.7684e-07, 5.9605e-07, 2.3842e-07, -2.9802e-07,\n", + " 7.1526e-07, -5.9605e-08, 3.5763e-07, -2.9802e-07, 3.5763e-07,\n", + " -4.1723e-07, 2.3842e-07, 3.5763e-07, -4.7684e-07, -1.7881e-07,\n", + " -1.3113e-06, -1.1921e-07, -2.9802e-07, -2.1458e-06, -2.9802e-07,\n", + " 5.9605e-07, 1.1921e-07, 0.0000e+00, 3.5763e-07, -7.7486e-07,\n", + " 3.5763e-07, -2.9802e-07, 5.9605e-07, 5.9605e-07, 1.1921e-07,\n", + " -2.9802e-07, -1.1921e-07, 4.7684e-07, 2.3842e-07, -4.7684e-07,\n", + " -5.9605e-08, -5.9605e-08, 0.0000e+00, 2.3842e-07, -4.1723e-07,\n", + " 1.1921e-07, 4.7684e-07, 3.5763e-07, 8.3447e-07, -6.5565e-07,\n", + " 1.1921e-07, 2.3842e-07, -4.1723e-07, -5.9605e-08, -8.3447e-07,\n", + " -4.7684e-07, 5.9605e-07, 0.0000e+00, 2.3842e-07, 1.1921e-07,\n", + " 4.7684e-07, 2.3842e-07, 3.5763e-07, 2.3842e-07, -1.7881e-07,\n", + " 0.0000e+00, -2.3842e-07, -6.5565e-07, 5.9605e-07, 9.5367e-07,\n", + " -3.5763e-07, -2.3842e-07, 2.3842e-07, -2.3842e-07, 1.1921e-07,\n", + " 0.0000e+00, -5.9605e-08, -5.9605e-08, 2.3842e-07, -5.3644e-07,\n", + " -1.9670e-06, -3.5763e-07], device='cuda:0'),\n", + " tensor([-7.0471e-07, -1.8349e-08, 2.0926e-07, -1.9718e-07, -5.0470e-07,\n", + " -2.0335e-07, 2.5020e-07, -5.5646e-07, -1.8064e-07, -6.6849e-07,\n", + " 1.7477e-08, -3.9154e-07, 6.1181e-07, 3.2980e-07, 1.0190e-07,\n", + " 9.8190e-08, 4.5841e-07, -2.9208e-07, 2.5231e-07, 1.0851e-07,\n", + " -6.3703e-08, -1.1971e-07, -4.7798e-08, -3.7517e-07, 2.3964e-07,\n", + " -5.2821e-07, 4.5428e-07, 3.5899e-07, 7.5338e-07, 2.2175e-07,\n", + " 2.6582e-07, -1.2163e-07, -1.1361e-07, 4.1826e-07, -1.9857e-07,\n", + " 3.4440e-07, 1.8137e-07, -2.9222e-07, -4.6063e-08, -1.2579e-07,\n", + " 1.7651e-07, 4.1248e-09, -5.7320e-07, 1.9714e-07, -5.0703e-07,\n", + " -9.1181e-07, 3.4195e-07, -1.5289e-08, -2.8127e-07, -1.0984e-07,\n", + " -4.4231e-07, 3.9074e-07, -4.6410e-07, -2.5822e-07, -6.3460e-07,\n", + " 2.1609e-08, 1.9698e-07, -4.0347e-07, 3.2517e-07, -1.9259e-07,\n", + " -1.9836e-07, 2.8106e-07, -1.2027e-07, -3.3226e-07, -1.5892e-06,\n", + " 3.3092e-07, -3.0162e-07, -4.1899e-08, -3.4353e-07, 2.8133e-07,\n", + " 3.5431e-07, -1.0125e-07, -2.1538e-07, 8.3516e-08, 2.1812e-07,\n", + " 5.0520e-07, -1.1525e-07, 3.1269e-07, -2.6864e-07, -9.7003e-09,\n", + " -2.8103e-07, 6.9125e-07, 2.8932e-07, -5.0337e-07, 3.0293e-07,\n", + " -3.8707e-07, -4.1809e-08, -2.5323e-07, 1.8370e-07, 6.4680e-07,\n", + " 5.6331e-08, -4.5315e-07, 2.2719e-07, -5.8242e-08, 2.2758e-07,\n", + " -4.7488e-07, -3.2628e-07, -1.2880e-07, 5.6331e-08, 5.5098e-07,\n", + " -9.7950e-07, 6.2894e-07, -6.5212e-07, 5.4815e-07, -5.8033e-07,\n", + " 6.9457e-08, -6.1899e-07, 4.1434e-07, -8.2837e-07, 7.0188e-08,\n", + " 6.4545e-08, 1.5691e-07, -4.9377e-09, 2.3902e-07, 2.8643e-07,\n", + " 8.0544e-07, -1.5877e-07, -5.0277e-07, 1.3618e-07, 5.5041e-07,\n", + " -6.3429e-07, 5.8772e-07, -3.7449e-07, 4.5133e-07, 3.4375e-07,\n", + " -6.1737e-07, -3.8223e-07, 1.5771e-07, -1.7511e-07, 3.0276e-09,\n", + " -2.9674e-07, -7.2252e-07, -6.7949e-09, -3.4926e-07, 4.3451e-08,\n", + " 1.0547e-07, 3.6982e-07, -2.0552e-07, 3.6342e-08, -4.7638e-08,\n", + " 1.4293e-07, 4.3917e-07, -7.5869e-08, -9.9415e-08, 4.5105e-08,\n", + " -1.0068e-08, -8.3892e-07, -7.9137e-08, -7.0750e-07, 1.9178e-07,\n", + " 3.8754e-07, 8.4444e-08, 2.0464e-07, 1.0252e-07, 7.5644e-08,\n", + " -6.1341e-08, -3.8856e-07, -3.5416e-07, -3.2500e-08, 1.5496e-06,\n", + " 9.9053e-09, 2.1292e-07, -3.9759e-07, 1.7524e-08, -7.2150e-07,\n", + " -2.6113e-07, -5.0575e-07, -5.3997e-08, -5.2743e-07, -1.2711e-07,\n", + " 6.4650e-09, -6.3542e-07, -1.7246e-07, 2.9385e-11, -6.1737e-07,\n", + " -3.0933e-07, -2.7813e-07, 6.1660e-07, -2.8017e-08, -1.2472e-06,\n", + " 3.9303e-07, -7.5703e-07, 4.1201e-07, -7.2459e-07, 2.2319e-07,\n", + " 2.6729e-07, 2.2822e-07, 3.2175e-07, -2.8173e-07, -3.4105e-07,\n", + " 2.9246e-07, -2.6755e-07, 5.5486e-07, 4.6074e-07, 7.9531e-07,\n", + " 1.4455e-07, -3.7617e-07, 1.2965e-07, 3.5979e-07, -7.1783e-08,\n", + " 7.6971e-07, -3.0106e-07, 4.4186e-07, -5.5656e-07, -3.6725e-07,\n", + " 3.2615e-07, 3.9505e-07, 2.8579e-07, 3.9586e-07, 9.8934e-07,\n", + " 6.1876e-07, -1.5795e-07, -1.0104e-06, 8.5002e-07, 8.5860e-08,\n", + " 3.2351e-07, -3.0993e-07, -3.4158e-07, 1.5116e-07, 9.9469e-08,\n", + " 6.0069e-08, -1.5719e-07, 8.0823e-08, 5.0363e-09, -6.6124e-07,\n", + " 2.8697e-07, -2.8923e-07, -3.3104e-07, -2.6741e-07, 2.1849e-07,\n", + " 2.2162e-07, 3.4295e-07, 7.9886e-07, 2.3284e-07, -6.5059e-08,\n", + " 2.3726e-07, -5.0067e-07, 2.1284e-07, -4.0939e-07, -3.2488e-07,\n", + " 7.8382e-08, 4.8203e-07, 1.9867e-07, -7.4483e-08, -3.1353e-07,\n", + " 5.8297e-08, -3.2046e-07, -1.1322e-07, -1.7365e-07, 3.7144e-07,\n", + " -7.3813e-08, -1.8838e-07, 2.8666e-07, -1.7835e-07, 5.0580e-07,\n", + " 2.9507e-07, 5.4638e-07, 2.3299e-07, 2.9693e-07, -6.9876e-07,\n", + " 8.6258e-08, -2.5660e-07, 5.0152e-07, 4.6366e-07, -2.0910e-08,\n", + " 1.8724e-08, -2.8327e-07, -2.6789e-07, -1.1106e-07, 1.1285e-08,\n", + " -1.3813e-07, 5.0809e-07, -6.2693e-08, -1.2473e-07, -2.0929e-07,\n", + " 3.2090e-07, 1.5514e-07, 6.5293e-07, -2.9884e-07, -1.3903e-07,\n", + " -2.4482e-07, 5.9293e-08, -3.9810e-07, 4.6002e-07, -7.9899e-07,\n", + " -5.8540e-07, 3.9786e-07, -3.2218e-07, 1.7169e-07, -4.9423e-07,\n", + " 1.0056e-07, 1.6145e-07, -9.8824e-08, 1.5913e-07, -4.6710e-07,\n", + " 1.5865e-07, 1.9589e-07, 3.9636e-07, 8.4565e-08, -6.8108e-07,\n", + " -6.6097e-07, 2.5134e-07, 6.7006e-07, 8.2307e-08, -7.5262e-08,\n", + " -3.2836e-07, -1.3761e-07, 1.0036e-07, -4.3035e-07, 4.0706e-07,\n", + " -1.3434e-07, -1.8913e-07, 4.9829e-07, -4.5375e-08, 5.6738e-07,\n", + " 4.0693e-07, -5.6909e-08, -5.6851e-07, 9.1993e-07, 4.2265e-07,\n", + " -1.1494e-07, -7.5589e-08, 1.8950e-07, -1.6677e-07, 2.5244e-08,\n", + " 5.4369e-07, -3.2131e-07, 6.1759e-07, -1.5090e-07, -3.5649e-08,\n", + " -1.8636e-07, -6.5008e-07, -1.6549e-07, -1.0451e-07, -5.9369e-07,\n", + " 8.2756e-07, 1.3586e-07, 6.7147e-08, 2.9688e-07, 3.7646e-07,\n", + " -1.6771e-07, -1.4768e-07, -2.5235e-07, -6.7719e-07, -4.6095e-07,\n", + " -2.8132e-07, -6.6112e-07, -8.3085e-07, -2.6775e-07, 2.5259e-07,\n", + " 7.4928e-07, -1.5687e-07, -4.7067e-07, -3.1492e-07, -2.0967e-07,\n", + " 9.5000e-08, -4.0278e-07, -1.6959e-07, -7.5251e-08, -2.5161e-07,\n", + " -2.2986e-07, 1.5039e-07, 2.9913e-07, -6.2833e-07, 8.5704e-08,\n", + " -2.2223e-08, -3.4837e-07, 1.6446e-07, 9.9939e-08, -6.5905e-07,\n", + " -4.8044e-07, -2.2191e-07, -2.0167e-07, 4.7854e-08, -4.6355e-07,\n", + " 2.7731e-07, -1.7976e-07, 2.6123e-07, 7.7950e-08, -4.4389e-07,\n", + " 4.6500e-07, -7.5943e-07, 1.8375e-07, -1.8547e-07, 3.4531e-07,\n", + " 2.7376e-07, -2.6321e-07, 3.0851e-07, 3.6956e-07, 3.3230e-07,\n", + " -2.0266e-07, -5.1608e-07, -3.1862e-07, -2.3650e-07, 2.8463e-07,\n", + " 4.2497e-08, 3.3198e-07, 2.9593e-07, -5.1724e-07, -1.7267e-07,\n", + " -2.0332e-07, 3.9231e-08, -1.3391e-08, -1.3861e-07, -2.3822e-07,\n", + " 2.4600e-07, -2.2045e-07, -1.1891e-07, 3.1210e-07, -1.2919e-07,\n", + " -8.2457e-07, 7.9238e-08, 2.3087e-07, -3.4217e-07, 2.6637e-07,\n", + " -3.7788e-07, 3.4308e-07, 6.8147e-08, 7.7669e-09, 4.1843e-07,\n", + " -4.6620e-07, -5.8995e-07, -1.9276e-07, 4.6281e-07, -1.7721e-07,\n", + " -4.2817e-07, 4.8755e-08, 1.8626e-07, 3.1048e-07, -3.5857e-07,\n", + " -1.2703e-07, 6.3200e-07, 1.3583e-07, 1.9140e-07, -8.6435e-08,\n", + " -1.2847e-08, 4.4478e-08, 4.1731e-07, -3.6239e-07, -2.0481e-07,\n", + " 4.8974e-07, -1.8279e-07, 2.0065e-07, 5.7776e-07, -2.9647e-07,\n", + " -4.1236e-07, -1.0541e-07, 2.4803e-07, -4.6571e-08, 1.3400e-07,\n", + " -7.4622e-07, -4.9445e-07, -9.1950e-08, -1.6310e-06, -5.4975e-07,\n", + " 1.1868e-06, 2.4416e-08, -1.0423e-07, 7.2068e-07, -3.8962e-07,\n", + " 1.3652e-07, 1.3550e-07, 6.9641e-07, 3.6471e-07, 5.4914e-08,\n", + " -4.5800e-08, 4.8776e-07, 5.0718e-07, 1.3634e-07, -2.0882e-07,\n", + " 4.4605e-07, 5.6278e-08, -5.4570e-08, 1.6894e-07, -3.2997e-07,\n", + " -3.0133e-07, 5.0226e-07, 8.4537e-08, 9.8503e-07, -2.7918e-08,\n", + " 1.2293e-07, 4.9457e-07, -9.1373e-08, 9.6624e-08, -5.7357e-07,\n", + " -2.6009e-07, 3.5024e-07, 1.7546e-07, 4.5867e-07, -4.8926e-07,\n", + " 1.1235e-07, -5.2832e-08, 6.5561e-07, -1.5632e-08, -1.9907e-07,\n", + " -2.0202e-08, -4.4126e-07, -4.0808e-07, 6.8809e-07, 9.7743e-07,\n", + " 3.4900e-07, 3.0799e-07, -1.1393e-07, 2.6764e-07, 3.8836e-08,\n", + " 1.1816e-07, -2.1617e-07, -2.2566e-07, 1.4733e-07, -8.5993e-08,\n", + " -8.8775e-07, -9.8369e-08], device='cuda:0'),\n", + " tensor([[[[-2.2165e-07, 2.0303e-07, -6.1677e-07],\n", + " [-6.1281e-07, -1.0803e-07, -1.0654e-06],\n", + " [ 8.0559e-08, 9.6671e-07, 4.0745e-07]],\n", + " \n", + " [[-1.6764e-07, -3.0175e-07, 3.3155e-07],\n", + " [-3.2131e-07, 4.1910e-07, 9.8068e-07],\n", + " [-6.7288e-08, 1.1176e-08, 2.0489e-08]],\n", + " \n", + " [[ 3.4925e-07, 1.0359e-06, 2.6543e-07],\n", + " [ 8.0699e-07, 1.3872e-06, 1.0617e-06],\n", + " [ 1.3690e-07, 9.4529e-07, 3.4831e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.3481e-07, 2.9150e-07, 2.3283e-08],\n", + " [-3.7393e-07, -1.5474e-06, -9.4692e-07],\n", + " [ 6.2678e-07, 6.4913e-07, 3.6112e-07]],\n", + " \n", + " [[-4.2934e-07, -9.1922e-07, -9.2387e-07],\n", + " [-7.3854e-07, -1.7500e-06, -1.1366e-06],\n", + " [-5.5600e-07, -7.6881e-07, -8.1677e-07]],\n", + " \n", + " [[-7.3900e-07, -8.7754e-07, -7.6485e-07],\n", + " [-3.6322e-07, -6.5938e-07, -1.6764e-07],\n", + " [ 2.1700e-07, -5.4840e-07, 3.9954e-07]]],\n", + " \n", + " \n", + " [[[-4.7870e-07, -2.8592e-07, 3.8743e-07],\n", + " [-1.2480e-07, -6.2399e-08, 1.7975e-07],\n", + " [ 1.3504e-08, -1.5134e-07, 1.8626e-09]],\n", + " \n", + " [[-1.9232e-07, -5.8580e-07, -4.8615e-07],\n", + " [-1.5832e-08, -8.5682e-08, -6.5193e-08],\n", + " [ 1.2666e-07, 5.0291e-08, 8.4750e-08]],\n", + " \n", + " [[-8.2701e-07, -1.0785e-06, -3.9348e-07],\n", + " [-9.0618e-07, -1.1120e-06, -8.2981e-07],\n", + " [-1.1176e-07, -4.5542e-07, -2.0768e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.7160e-07, -4.8429e-07, -3.3993e-07],\n", + " [ 1.4901e-08, -5.0478e-07, 1.7788e-07],\n", + " [ 3.8843e-08, 3.3434e-07, -9.6858e-08]],\n", + " \n", + " [[-2.2072e-07, 4.7963e-07, -4.8021e-08],\n", + " [ 3.1989e-07, 2.6217e-07, 3.2736e-07],\n", + " [-5.9605e-08, 1.9651e-07, 4.3586e-07]],\n", + " \n", + " [[ 1.1539e-06, 1.0452e-06, 6.4378e-07],\n", + " [ 3.7416e-07, 6.8918e-07, 8.1491e-07],\n", + " [ 3.1095e-07, 8.1595e-07, 6.6403e-07]]],\n", + " \n", + " \n", + " [[[ 6.9238e-07, 2.6543e-07, -3.1618e-07],\n", + " [ 1.3020e-06, 9.6485e-07, 5.2620e-07],\n", + " [ 9.3179e-07, 5.4948e-07, -9.3132e-10]],\n", + " \n", + " [[-1.0533e-06, -2.3115e-06, -5.2340e-07],\n", + " [-1.0659e-06, -1.6280e-06, -1.0915e-06],\n", + " [-4.7591e-07, -4.7754e-07, -3.6196e-07]],\n", + " \n", + " [[ 6.3385e-07, 7.9814e-07, 7.7020e-07],\n", + " [ 6.5332e-07, 4.4331e-07, 6.0163e-07],\n", + " [ 4.2340e-07, 3.6228e-07, 4.4424e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-9.7090e-07, -2.0806e-06, -1.5544e-06],\n", + " [-1.9232e-07, -2.1662e-06, -1.9390e-06],\n", + " [-4.0513e-07, -9.7416e-07, -1.3700e-06]],\n", + " \n", + " [[-5.5414e-07, -1.2098e-06, -1.4147e-06],\n", + " [-6.9407e-07, -1.4040e-06, -1.8161e-06],\n", + " [-7.6079e-07, -4.7218e-07, -2.5006e-07]],\n", + " \n", + " [[ 1.8300e-07, 6.2957e-07, 2.3027e-07],\n", + " [ 2.1514e-07, 2.6240e-07, 1.2740e-06],\n", + " [ 4.6566e-09, 2.3586e-07, 5.8953e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[ 5.2247e-07, 1.2461e-06, 2.1979e-07],\n", + " [-1.7649e-07, 3.9861e-07, -4.1863e-07],\n", + " [ 1.8626e-08, 1.2740e-06, 5.3085e-08]],\n", + " \n", + " [[ 1.5367e-06, -8.8196e-07, 3.3248e-07],\n", + " [ 7.5158e-07, 4.9314e-07, 1.0021e-06],\n", + " [-1.9278e-07, 4.9500e-07, 7.5088e-07]],\n", + " \n", + " [[-2.5658e-07, -5.8790e-07, 8.9919e-07],\n", + " [ 3.9954e-07, -3.3545e-07, 6.2259e-07],\n", + " [ 3.8929e-07, 7.3574e-07, 5.8487e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.8915e-06, 1.6885e-06, 1.0710e-06],\n", + " [ 7.8429e-07, 3.8184e-07, -1.9744e-07],\n", + " [ 7.2224e-07, 1.5968e-06, 2.2436e-06]],\n", + " \n", + " [[-7.7859e-07, -1.6014e-06, -2.0557e-06],\n", + " [-1.3784e-07, -1.1573e-06, -1.0915e-06],\n", + " [ 3.5670e-07, 1.4063e-07, -7.3574e-07]],\n", + " \n", + " [[ 5.0943e-07, -2.4890e-07, -4.6473e-07],\n", + " [-5.3784e-07, -1.0850e-06, -8.6334e-07],\n", + " [-8.3886e-07, -1.1204e-06, -9.3084e-07]]],\n", + " \n", + " \n", + " [[[ 5.8766e-07, 1.6061e-06, -3.1432e-08],\n", + " [-3.5856e-07, -9.9000e-07, -5.1223e-08],\n", + " [ 6.2864e-07, 4.1490e-07, 3.6744e-07]],\n", + " \n", + " [[-1.9530e-06, -1.0896e-06, -7.9721e-07],\n", + " [-1.5832e-06, -1.4724e-06, -1.4277e-06],\n", + " [-6.9430e-07, -7.6258e-07, -1.2978e-06]],\n", + " \n", + " [[ 6.4168e-07, 4.7969e-07, 1.4501e-06],\n", + " [ 1.8075e-06, -3.8091e-07, 3.1712e-07],\n", + " [ 9.0525e-07, 9.8301e-07, 1.5246e-06]],\n", + " \n", + " ...,\n", + " \n", + " [[-4.5868e-07, 2.2026e-07, 1.2293e-07],\n", + " [-1.0002e-06, -1.7951e-07, -7.7835e-07],\n", + " [-2.3097e-07, -5.4669e-07, -5.0990e-07]],\n", + " \n", + " [[ 2.5705e-07, 4.8149e-07, 4.6613e-07],\n", + " [ 3.8184e-07, 1.0817e-06, 2.4680e-08],\n", + " [ 9.2667e-07, 1.0850e-06, 1.3742e-06]],\n", + " \n", + " [[-7.5763e-07, -8.2329e-07, 2.7381e-07],\n", + " [-9.4250e-07, -8.2445e-07, -7.4506e-09],\n", + " [-3.8929e-07, -3.0390e-07, -5.6066e-07]]],\n", + " \n", + " \n", + " [[[-2.7078e-07, 1.6624e-07, 5.3272e-07],\n", + " [ 2.5315e-07, 4.3679e-07, 1.8068e-07],\n", + " [ 1.8068e-07, -3.3714e-07, -3.2596e-09]],\n", + " \n", + " [[-8.0187e-07, -1.2629e-06, -8.6147e-07],\n", + " [-6.3705e-07, -6.8382e-07, -1.0277e-06],\n", + " [-2.5611e-07, -1.2852e-07, 4.2142e-07]],\n", + " \n", + " [[-6.1281e-07, 5.6298e-07, -6.1467e-07],\n", + " [-6.4680e-07, 7.9628e-07, -3.4645e-07],\n", + " [-2.0023e-07, 4.4703e-08, -5.2899e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.6787e-07, -4.0513e-07, 1.7043e-07],\n", + " [-3.1851e-07, -2.0163e-07, 3.3528e-08],\n", + " [-4.8149e-07, -1.8207e-07, -7.6601e-08]],\n", + " \n", + " [[-3.6275e-07, -4.3306e-08, 9.2736e-07],\n", + " [ 3.5577e-07, 8.0373e-07, 2.0396e-06],\n", + " [-2.5146e-08, 9.7509e-07, 1.7257e-06]],\n", + " \n", + " [[-1.4913e-06, -8.7731e-07, -1.8626e-07],\n", + " [-6.0396e-07, -3.6787e-07, -6.5751e-07],\n", + " [ 1.0910e-06, 8.3167e-07, 6.7987e-08]]]], device='cuda:0'),\n", + " tensor([-9.5367e-07, 4.7684e-07, -1.7881e-07, -1.1921e-07, -7.1526e-07,\n", + " -3.5763e-07, -2.3842e-07, 0.0000e+00, -4.7684e-07, 2.3842e-07,\n", + " 0.0000e+00, -1.1921e-07, 3.5763e-07, -1.1921e-07, -1.7881e-07,\n", + " -1.1921e-06, 2.3842e-07, 2.3842e-07, -7.7486e-07, 5.9605e-07,\n", + " 3.5763e-07, -8.3447e-07, 1.1921e-07, 0.0000e+00, -5.9605e-07,\n", + " 3.5763e-07, -1.2517e-06, -2.9802e-07, -8.3447e-07, 1.1921e-06,\n", + " -2.3842e-07, 2.3842e-07, 3.5763e-07, 5.9605e-07, 4.7684e-07,\n", + " 8.3447e-07, -4.7684e-07, -3.5763e-07, -2.9802e-07, -1.4305e-06,\n", + " 5.9605e-07, 0.0000e+00, -5.9605e-08, -1.1921e-07, 0.0000e+00,\n", + " -5.9605e-08, -1.1921e-07, -5.9605e-08, -4.1723e-07, 5.9605e-07,\n", + " -4.7684e-07, 1.1921e-07, -5.9605e-08, -2.9802e-07, -4.7684e-07,\n", + " 0.0000e+00, 5.9605e-07, -7.7486e-07, -2.3842e-07, -1.0133e-06,\n", + " 3.5763e-07, -1.1921e-07, -4.7684e-07, -1.1921e-07, -3.5763e-07,\n", + " -1.1921e-07, -9.5367e-07, -5.9605e-08, -2.3842e-07, 3.5763e-07,\n", + " -5.9605e-08, -1.7881e-07, -5.9605e-08, -1.1325e-06, 7.1526e-07,\n", + " -2.9802e-07, 1.4305e-06, 2.3842e-07, -1.7881e-07, -1.7881e-07,\n", + " 2.3842e-07, -1.0729e-06, 0.0000e+00, 4.7684e-07, -3.5763e-07,\n", + " 2.3842e-07, -5.3644e-07, -4.7684e-07, 3.5763e-07, -2.3842e-07,\n", + " 7.1526e-07, 4.7684e-07, -1.1921e-07, -6.5565e-07, -2.3842e-07,\n", + " -1.7881e-07, 3.5763e-07, -2.9802e-07, -3.5763e-07, -3.5763e-07,\n", + " -7.1526e-07, -1.1921e-07, 3.5763e-07, -8.3447e-07, -1.7881e-07,\n", + " 3.5763e-07, 2.3842e-07, -5.9605e-07, 2.3842e-07, -1.1921e-07,\n", + " 0.0000e+00, 7.1526e-07, 3.5763e-07, -7.7486e-07, 1.1921e-07,\n", + " -5.9605e-07, -1.1921e-07, -1.3113e-06, -8.9407e-07, 4.7684e-07,\n", + " -1.1921e-07, -4.7684e-07, 0.0000e+00, -4.1723e-07, -1.1921e-07,\n", + " -5.9605e-08, -5.9605e-08, 2.3842e-07, 1.1921e-07, 1.1921e-07,\n", + " -1.7881e-07, -7.1526e-07, -6.5565e-07, -2.9802e-07, 3.5763e-07,\n", + " -2.3842e-07, -1.7881e-07, 8.3447e-07, -7.1526e-07, -2.3842e-07,\n", + " 7.1526e-07, 0.0000e+00, -4.1723e-07, -5.3644e-07, 1.1921e-07,\n", + " -7.7486e-07, -1.1921e-07, 2.3842e-07, 0.0000e+00, -5.9605e-08,\n", + " 0.0000e+00, -5.9605e-08, -4.1723e-07, 3.5763e-07, 5.9605e-07,\n", + " 1.1921e-07, -7.7486e-07, -2.9802e-07, -2.9802e-07, 0.0000e+00,\n", + " -3.5763e-07, 0.0000e+00, 0.0000e+00, -5.9605e-08, -5.9605e-08,\n", + " -5.9605e-08, 8.3447e-07, -4.1723e-07, 0.0000e+00, 3.5763e-07,\n", + " -1.7881e-07, -3.5763e-07, -4.1723e-07, -5.9605e-07, -6.5565e-07,\n", + " 1.1921e-07, 1.1921e-07, -5.9605e-07, -5.3644e-07, -1.4305e-06,\n", + " 2.3842e-07, -6.5565e-07, 1.1921e-07, -5.9605e-08, 8.3447e-07,\n", + " 5.9605e-07, -1.7881e-07, 5.9605e-07, -2.9802e-07, -4.1723e-07,\n", + " 0.0000e+00, -5.3644e-07, -4.1723e-07, -1.5497e-06, 1.1921e-07,\n", + " 2.3842e-07, 3.5763e-07, -4.7684e-07, -4.1723e-07, 7.1526e-07,\n", + " 1.1921e-07, 4.7684e-07, -2.3842e-07, 5.9605e-07, 2.3842e-07,\n", + " -2.3842e-07, 3.5763e-07, 7.1526e-07, -5.3644e-07, 1.1921e-07,\n", + " 1.1921e-07, 2.3842e-07, 1.1921e-07, 1.1921e-07, 3.5763e-07,\n", + " 7.1526e-07, 2.3842e-07, 0.0000e+00, 8.3447e-07, 8.3447e-07,\n", + " 7.1526e-07, 3.5763e-07, -4.7684e-07, -5.9605e-08, 4.7684e-07,\n", + " -1.1921e-07, 1.1921e-07, 8.3447e-07, 4.7684e-07, -8.3447e-07,\n", + " 2.3842e-07, 0.0000e+00, 3.5763e-07, -5.9605e-08, 1.1921e-07,\n", + " -5.9605e-07, 1.1921e-07, -3.5763e-07, 7.1526e-07, -1.7881e-07,\n", + " -1.0133e-06, -7.7486e-07, 0.0000e+00, 8.3447e-07, 4.7684e-07,\n", + " -3.5763e-07, 0.0000e+00, 7.1526e-07, -8.3447e-07, -1.1921e-07,\n", + " -5.3644e-07, -4.1723e-07, 0.0000e+00, 3.5763e-07, 7.1526e-07,\n", + " -1.0729e-06, 1.1921e-07, 1.1921e-07, 3.5763e-07, -4.1723e-07,\n", + " -3.5763e-07, -2.3842e-07, -1.7881e-07, -2.3842e-07, 2.3842e-07,\n", + " 1.1921e-07, 0.0000e+00, -4.7684e-07, 4.7684e-07, -5.3644e-07,\n", + " -5.3644e-07, -5.9605e-07, -5.9605e-08, 3.5763e-07, -4.1723e-07,\n", + " -1.7881e-07, 5.9605e-07, -2.3842e-07, 4.7684e-07, -5.3644e-07,\n", + " -2.3842e-07, 7.1526e-07, 8.3447e-07, -2.9802e-07, -1.1921e-07,\n", + " -3.5763e-07, 1.1921e-07, 9.5367e-07, 3.5763e-07, 1.1921e-06,\n", + " 1.4305e-06, 7.1526e-07, -1.7881e-07, -5.9605e-08, -4.1723e-07,\n", + " 1.0729e-06, -5.3644e-07, -1.1921e-07, 3.5763e-07, -8.9407e-07,\n", + " 4.7684e-07, -3.5763e-07, -5.9605e-07, 0.0000e+00, -7.7486e-07,\n", + " 1.1921e-07, 3.5763e-07, -2.9802e-07, -3.5763e-07, -1.2517e-06,\n", + " -1.1921e-07, -7.1526e-07, -2.3842e-07, 7.1526e-07, -6.5565e-07,\n", + " 1.1921e-07, 5.9605e-07, -3.5763e-07, -1.1921e-07, 1.1921e-07,\n", + " -3.5763e-07, 2.3842e-07, -4.7684e-07, -1.1921e-07, -5.9605e-08,\n", + " 2.3842e-07, -5.9605e-08, -2.9802e-07, 3.5763e-07, -5.3644e-07,\n", + " 5.9605e-07, -1.7881e-07, -1.1921e-07, 3.5763e-07, 3.5763e-07,\n", + " -8.3447e-07, -1.3709e-06, -1.7881e-07, 1.0729e-06, 1.4305e-06,\n", + " 0.0000e+00, 0.0000e+00, 2.3842e-07, -2.3842e-07, 2.3842e-07,\n", + " 1.3113e-06, 8.3447e-07, -1.0133e-06, -5.9605e-07, -1.1921e-07,\n", + " 2.3842e-07, -8.3447e-07, 7.1526e-07, -1.0133e-06, 3.5763e-07,\n", + " -5.3644e-07, -3.5763e-07, -5.9605e-07, 8.3447e-07, 1.1921e-07,\n", + " -1.1921e-07, 1.1921e-07, -5.9605e-08, -1.1921e-07, 5.9605e-07,\n", + " 9.5367e-07, 5.9605e-07, 4.7684e-07, 1.1921e-07, 0.0000e+00,\n", + " 7.1526e-07, 5.9605e-07, -3.5763e-07, 5.9605e-07, -7.1526e-07,\n", + " 5.9605e-07, -5.9605e-08, -4.7684e-07, -5.9605e-07, -4.1723e-07,\n", + " -2.3842e-07, 1.1921e-07, 2.3842e-07, -4.1723e-07, -2.3842e-07,\n", + " -2.9802e-07, 1.1921e-07, 0.0000e+00, 1.1921e-07, 4.7684e-07,\n", + " 0.0000e+00, 0.0000e+00, -3.5763e-07, -2.9802e-07, 2.3842e-07,\n", + " -7.7486e-07, 8.3447e-07, -2.3842e-07, -8.3447e-07, -1.0133e-06,\n", + " -1.1921e-07, -3.5763e-07, -3.5763e-07, -1.7881e-07, 7.1526e-07,\n", + " 2.3842e-07, -5.9605e-07, 8.3447e-07, 8.3447e-07, -7.7486e-07,\n", + " -5.3644e-07, 0.0000e+00, -5.9605e-08, 5.9605e-07, -1.6093e-06,\n", + " 8.3447e-07, 3.5763e-07, -5.9605e-07, 2.3842e-07, 4.7684e-07,\n", + " -5.3644e-07, 5.9605e-07, -5.9605e-08, 3.5763e-07, 5.9605e-07,\n", + " -8.3447e-07, 3.5763e-07, 1.1921e-07, 3.5763e-07, -3.5763e-07,\n", + " 1.1921e-07, -7.1526e-07, 2.3842e-07, -7.1526e-07, -2.3842e-07,\n", + " 1.1921e-07, -5.9605e-07, 1.1921e-07, -5.9605e-08, -5.3644e-07,\n", + " 1.1921e-07, -1.1325e-06, -5.9605e-07, -1.7881e-07, 3.5763e-07,\n", + " 1.1921e-07, 5.9605e-07, -1.7881e-07, 1.0729e-06, 1.1921e-07,\n", + " 8.3447e-07, -2.9802e-07, 1.1921e-07, 8.3447e-07, 0.0000e+00,\n", + " 1.7881e-06, 1.1921e-06, 1.1921e-07, -1.0729e-06, 4.7684e-07,\n", + " -8.9407e-07, -7.1526e-07, -2.9802e-07, 2.3842e-07, -4.1723e-07,\n", + " -5.3644e-07, -3.5763e-07, -2.3842e-07, 0.0000e+00, 3.5763e-07,\n", + " -4.7684e-07, -1.0133e-06, -4.7684e-07, -2.3842e-07, 0.0000e+00,\n", + " -1.8477e-06, 2.3842e-07, 2.3842e-07, -3.5763e-07, 9.5367e-07,\n", + " -7.1526e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07, 3.5763e-07,\n", + " 2.3842e-07, 2.3842e-07, 5.9605e-07, -6.5565e-07, 1.1921e-07,\n", + " -4.1723e-07, -2.3842e-07, 4.7684e-07, -1.1921e-07, 4.7684e-07,\n", + " 8.3447e-07, 5.9605e-07, -4.1723e-07, 5.9605e-07, -1.0133e-06,\n", + " 1.1921e-06, -1.1921e-06, -1.1921e-07, 9.5367e-07, 2.3842e-07,\n", + " -1.1325e-06, 1.1921e-07, 0.0000e+00, -1.7881e-07, 2.3842e-07,\n", + " 5.9605e-07, 3.5763e-07], device='cuda:0'),\n", + " tensor([ 1.4361e-07, 3.7875e-07, -8.3476e-07, 9.4134e-07, -6.7843e-07,\n", + " 3.1072e-07, 1.5542e-07, 1.5177e-07, 5.9782e-07, 5.7706e-07,\n", + " 3.6449e-07, -7.4512e-08, 1.9858e-07, -4.9469e-07, 7.5448e-07,\n", + " -5.9229e-07, 5.5775e-07, 7.6487e-07, -1.4062e-06, 7.2411e-07,\n", + " 4.1560e-07, -1.3103e-06, -6.2538e-07, -3.8220e-09, -1.0456e-06,\n", + " 1.7845e-07, -7.8648e-07, -4.8804e-07, -1.0593e-06, 1.1402e-06,\n", + " -2.8512e-07, 8.6257e-07, 5.7358e-07, -2.7734e-08, -1.3981e-07,\n", + " 6.7581e-07, -1.1391e-06, 3.5621e-08, -1.1710e-07, -5.2912e-07,\n", + " 1.0740e-06, 1.8467e-07, -6.4858e-08, -7.4553e-07, -4.4954e-07,\n", + " -2.0085e-08, -2.6388e-07, 1.0682e-07, -4.3396e-07, 5.0373e-07,\n", + " -3.1702e-09, 2.1772e-07, -3.6832e-07, -7.9641e-07, -5.5592e-07,\n", + " -8.9712e-08, -2.3330e-07, -5.7435e-07, 2.1003e-07, -3.4901e-07,\n", + " -1.2920e-07, -7.1090e-08, -2.8965e-07, -6.8484e-08, 1.1578e-07,\n", + " 1.8122e-07, -8.2851e-07, -5.4816e-07, -5.4788e-07, 6.4785e-07,\n", + " -6.7243e-07, -2.9483e-07, 3.0059e-07, -6.2761e-07, -1.3943e-07,\n", + " -4.6666e-08, 8.2269e-07, 5.2700e-07, -2.0402e-07, -1.8630e-07,\n", + " -1.4299e-07, -5.8553e-07, -4.3453e-08, 1.8669e-07, 4.3835e-08,\n", + " -1.5051e-07, 1.0361e-08, -8.9533e-07, -7.6743e-08, -2.0186e-07,\n", + " 5.8199e-07, 4.8817e-07, 5.3219e-07, 1.5891e-07, -5.4972e-07,\n", + " -4.3597e-08, 6.6097e-07, -4.5730e-10, -5.9889e-08, -5.4971e-07,\n", + " 1.5863e-08, 4.2603e-07, 4.4973e-07, -3.9109e-07, 3.3712e-07,\n", + " 4.1975e-07, 7.8129e-07, -4.5257e-07, -2.4825e-07, -6.3345e-07,\n", + " -7.7735e-07, 9.1151e-07, 5.8416e-08, -2.9035e-07, -1.3461e-06,\n", + " -2.6095e-07, -3.0684e-07, -1.6109e-06, -1.3722e-06, 2.2558e-08,\n", + " -4.3157e-07, -1.2280e-07, -1.1057e-07, -5.6610e-07, -2.7475e-07,\n", + " 3.3783e-07, -4.9134e-07, 5.5273e-07, 3.0693e-09, -2.3143e-07,\n", + " -8.8538e-07, -1.6646e-07, -1.4726e-07, -3.3940e-07, 1.1079e-06,\n", + " -3.8600e-07, 1.9262e-07, 8.6771e-09, -6.1051e-07, 2.0421e-07,\n", + " 6.8559e-07, 1.6167e-07, 6.7790e-07, -6.8239e-07, 1.2664e-07,\n", + " -9.9949e-09, 3.2615e-07, -6.1828e-07, -3.7162e-07, -3.7594e-07,\n", + " -1.4025e-09, -6.1806e-07, -3.8709e-07, -1.8537e-08, 5.6683e-07,\n", + " -1.8389e-07, -2.1760e-06, -2.1463e-07, -3.7027e-07, -1.2480e-07,\n", + " -5.7935e-07, 3.2740e-07, -3.9764e-07, -8.7763e-07, -6.9524e-07,\n", + " 1.8890e-07, 6.0723e-07, 2.8119e-07, 1.9862e-07, -4.5653e-07,\n", + " -3.2686e-07, -6.4305e-07, 3.0504e-07, -7.7494e-07, -1.0217e-06,\n", + " 1.7248e-07, 3.5446e-07, 3.2754e-07, -8.2340e-07, -2.8620e-07,\n", + " -1.2324e-07, 1.4035e-08, -4.2806e-07, 2.2246e-07, 5.9490e-07,\n", + " 9.5043e-07, 1.6535e-07, -8.7346e-07, -7.8100e-07, -5.2794e-09,\n", + " -7.6545e-07, -5.1436e-07, -4.0977e-07, -1.1618e-06, 6.8537e-08,\n", + " -1.4605e-07, 5.8257e-08, -8.0188e-07, -2.0639e-07, 2.1339e-07,\n", + " 1.5732e-07, 9.9957e-07, 4.9952e-07, 1.3014e-06, 2.9355e-07,\n", + " -3.4369e-07, 5.1924e-07, 5.3847e-07, -9.4620e-07, 5.6011e-07,\n", + " 7.9131e-07, -5.2138e-09, -1.4767e-06, -6.3886e-07, 1.4941e-07,\n", + " 9.8527e-07, -3.7774e-07, 3.2528e-07, 9.4757e-07, 2.1744e-07,\n", + " 5.9661e-07, -1.1091e-07, -6.2895e-07, 4.1626e-07, 9.3376e-07,\n", + " -2.4913e-07, 2.1312e-08, 7.0954e-07, 1.7708e-07, -1.0764e-06,\n", + " -1.6039e-07, -2.0194e-07, 3.0868e-07, -2.1210e-07, 2.8911e-07,\n", + " -3.3863e-07, 3.0655e-07, -8.2397e-07, 6.2707e-07, 4.0511e-07,\n", + " -6.2630e-07, -5.2816e-07, 6.9402e-07, 3.1391e-07, 6.8661e-07,\n", + " 1.2054e-07, -2.9753e-07, 2.2532e-09, -5.5803e-07, -2.0193e-07,\n", + " -5.2379e-07, -4.5617e-07, 1.9949e-08, 9.5923e-07, 6.3150e-07,\n", + " -3.8950e-07, -8.7060e-07, 3.2801e-07, -1.5988e-07, -2.0283e-07,\n", + " 2.0498e-08, -6.2763e-08, 1.2942e-07, 4.8690e-07, 2.9747e-08,\n", + " -5.9033e-07, -2.3964e-07, -7.6579e-07, -5.2635e-07, -1.1527e-06,\n", + " 6.4606e-07, -8.1474e-07, -4.2017e-07, 6.5501e-07, 3.6491e-07,\n", + " 5.6719e-07, -6.4178e-08, 8.4156e-07, 1.8924e-07, 5.0438e-07,\n", + " -8.7186e-07, 5.7862e-07, 7.9785e-07, -1.1940e-07, -2.8611e-07,\n", + " -3.7413e-07, 6.9951e-07, 1.1857e-06, -7.0392e-07, 2.6504e-07,\n", + " 6.2547e-07, 4.0341e-07, -6.8973e-07, 1.0543e-06, -3.9495e-07,\n", + " 6.2045e-07, -5.2668e-07, 3.1209e-07, 2.9866e-08, -1.3103e-06,\n", + " -7.4060e-07, -2.1091e-07, -1.1769e-06, 4.9558e-07, -1.5905e-06,\n", + " 2.1223e-07, 3.8217e-07, 2.4839e-07, -9.2843e-07, -3.9248e-07,\n", + " 5.8080e-08, -9.3520e-07, 2.0115e-08, 8.0949e-07, -6.3440e-07,\n", + " -6.9958e-07, 3.6014e-08, 1.2903e-08, -7.1608e-08, -1.1944e-06,\n", + " -9.8625e-07, 4.3675e-07, -5.9806e-07, 7.8947e-07, -8.5338e-07,\n", + " -3.3651e-07, -5.3144e-07, -6.9125e-07, 1.4479e-07, -8.7782e-07,\n", + " 2.1694e-07, 6.6267e-07, -1.7303e-07, 5.9009e-07, 1.9341e-07,\n", + " -1.2323e-06, -3.1903e-07, 1.7893e-07, 8.5715e-07, 1.6794e-06,\n", + " 4.4822e-07, 8.3358e-07, 7.1812e-08, 1.7047e-07, -1.3995e-07,\n", + " 1.5727e-06, -7.6507e-07, -1.1366e-06, -6.4408e-07, -3.2520e-07,\n", + " 3.6125e-08, -5.6142e-07, 1.0516e-06, -1.0970e-06, -4.1031e-07,\n", + " 1.9087e-07, -9.4430e-08, -4.1627e-08, 7.3488e-07, 2.9984e-07,\n", + " -4.2442e-07, 1.1619e-06, -9.3843e-07, -3.9253e-07, 7.6044e-07,\n", + " 8.9230e-07, -6.1505e-07, 2.8537e-07, 1.9979e-07, 1.1293e-07,\n", + " 9.3129e-09, -4.7411e-07, -3.3502e-07, 6.4352e-07, -8.2211e-07,\n", + " 2.8376e-07, -2.1078e-07, -4.7676e-07, -3.2840e-07, -1.7811e-07,\n", + " -1.0372e-07, -1.5150e-07, -2.9330e-07, -5.0891e-08, 4.6162e-07,\n", + " -4.4931e-07, -1.1251e-07, -2.1810e-08, 3.6239e-07, 4.1384e-07,\n", + " -1.9962e-07, -2.2854e-07, 4.1248e-07, 1.3120e-07, 2.2709e-07,\n", + " -6.5830e-07, 2.7621e-07, -2.8515e-07, -7.1727e-07, -2.2612e-07,\n", + " 2.9839e-07, -1.8185e-07, -7.0273e-07, -2.9024e-07, 5.5723e-07,\n", + " -4.9760e-07, -7.6321e-07, 2.0529e-07, 8.0877e-07, -8.9062e-08,\n", + " -6.0039e-07, 2.3167e-07, -1.9675e-07, 6.2634e-08, -5.7431e-07,\n", + " 7.7656e-07, 4.2352e-07, -5.7160e-07, -1.6343e-08, 6.0179e-07,\n", + " -6.7309e-07, 3.5918e-07, 4.8857e-08, 1.7525e-07, -5.2018e-07,\n", + " -2.2266e-07, -1.4090e-07, 1.4820e-07, 5.7232e-07, -6.3711e-07,\n", + " 3.2627e-07, -1.3462e-06, -6.5572e-08, -6.0948e-07, -5.2886e-07,\n", + " -4.4639e-07, -5.9024e-07, -7.8491e-07, -1.6743e-07, 2.8212e-09,\n", + " -1.1809e-07, -6.3132e-07, 4.9501e-07, -5.7799e-07, 6.2618e-07,\n", + " 4.2153e-07, 3.3870e-07, 2.6234e-07, 1.1589e-06, -6.3834e-07,\n", + " -1.7349e-07, -2.8091e-07, 1.3085e-07, 2.7772e-07, 8.0872e-08,\n", + " 2.6623e-07, 9.5979e-07, 2.8653e-07, -1.1525e-06, 4.1606e-07,\n", + " -8.0760e-07, -9.3070e-07, -4.5792e-07, 3.1079e-07, 3.9923e-07,\n", + " -6.9447e-07, 4.0161e-08, 1.7961e-07, -2.0261e-07, 5.0900e-08,\n", + " -5.9011e-07, -8.9109e-07, -5.2656e-07, -1.3622e-07, 5.2093e-07,\n", + " -1.5905e-06, 1.5041e-06, 2.2004e-07, -3.0572e-07, 5.7887e-07,\n", + " 1.7697e-07, -2.7734e-08, 5.3296e-07, 9.1149e-08, -3.8058e-07,\n", + " 3.5024e-07, 3.7948e-07, -1.5867e-08, -8.8103e-07, 7.0423e-08,\n", + " -8.5883e-07, -3.3453e-07, 6.0323e-07, -3.6401e-07, 3.0336e-07,\n", + " 4.4667e-07, 8.7888e-07, -1.7145e-07, 6.6095e-07, -7.5378e-07,\n", + " 3.2432e-07, -1.0764e-06, -9.0861e-08, 7.4299e-07, 2.8685e-07,\n", + " -8.3725e-07, 3.7188e-07, 1.8325e-07, 6.5297e-07, 1.3390e-07,\n", + " 6.0233e-07, 3.0721e-07], device='cuda:0'),\n", + " tensor([[[[-4.5076e-07]],\n", + " \n", + " [[-5.0198e-07]],\n", + " \n", + " [[-4.6566e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.9802e-07]],\n", + " \n", + " [[-2.2165e-06]],\n", + " \n", + " [[-5.5134e-07]]],\n", + " \n", + " \n", + " [[[ 5.2154e-07]],\n", + " \n", + " [[ 1.3039e-06]],\n", + " \n", + " [[ 5.3085e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.8626e-08]],\n", + " \n", + " [[ 6.5938e-07]],\n", + " \n", + " [[ 1.2065e-06]]],\n", + " \n", + " \n", + " [[[ 8.9034e-07]],\n", + " \n", + " [[ 1.3784e-07]],\n", + " \n", + " [[ 2.5705e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[-3.3155e-07]],\n", + " \n", + " [[-5.2527e-07]],\n", + " \n", + " [[-7.6927e-07]]],\n", + " \n", + " \n", + " ...,\n", + " \n", + " \n", + " [[[-4.3958e-07]],\n", + " \n", + " [[-2.1048e-06]],\n", + " \n", + " [[-2.6077e-08]],\n", + " \n", + " ...,\n", + " \n", + " [[ 1.2107e-06]],\n", + " \n", + " [[ 1.5274e-07]],\n", + " \n", + " [[-6.1840e-07]]],\n", + " \n", + " \n", + " [[[-1.0449e-06]],\n", + " \n", + " [[-2.3097e-07]],\n", + " \n", + " [[ 3.6508e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 2.8498e-07]],\n", + " \n", + " [[ 3.6182e-07]],\n", + " \n", + " [[ 6.2399e-08]]],\n", + " \n", + " \n", + " [[[-2.6077e-08]],\n", + " \n", + " [[ 1.4901e-08]],\n", + " \n", + " [[-3.2037e-07]],\n", + " \n", + " ...,\n", + " \n", + " [[ 4.4703e-08]],\n", + " \n", + " [[-8.7544e-08]],\n", + " \n", + " [[-2.8126e-07]]]], device='cuda:0'),\n", + " tensor([ 1.1921e-07, 1.1921e-07, -1.7285e-06, 7.1526e-07, 7.1526e-07,\n", + " -1.1921e-07, 3.5763e-07, 4.7684e-07, 8.3447e-07, 0.0000e+00,\n", + " 4.7684e-07, 2.3842e-07, -4.1723e-07, -5.9605e-08, 5.9605e-07,\n", + " 0.0000e+00, 5.9605e-07, 7.1526e-07, -7.1526e-07, 2.3842e-07,\n", + " 2.3842e-07, -7.7486e-07, -2.9802e-07, -1.7881e-07, -8.3447e-07,\n", + " -6.5565e-07, -5.3644e-07, -2.3842e-07, -2.9802e-07, -9.5367e-07,\n", + " -4.1723e-07, 1.0729e-06, 5.9605e-07, 0.0000e+00, -5.9605e-08,\n", + " 1.1921e-07, -8.9407e-07, 3.5763e-07, 4.7684e-07, -1.7881e-07,\n", + " 4.7684e-07, 0.0000e+00, -2.9802e-07, -8.9407e-07, -6.5565e-07,\n", + " -5.9605e-08, -1.1921e-07, -2.9802e-07, -2.3842e-07, -1.7881e-07,\n", + " -5.9605e-08, 5.9605e-07, 2.3842e-07, -2.3842e-07, 3.5763e-07,\n", + " 2.3842e-07, -5.9605e-08, 2.3842e-07, 7.1526e-07, 7.1526e-07,\n", + " 1.1921e-07, 9.5367e-07, 0.0000e+00, 1.1921e-07, 3.5763e-07,\n", + " 5.9605e-07, 5.9605e-07, -1.1921e-07, -5.9605e-08, 4.7684e-07,\n", + " -5.9605e-07, -2.3842e-07, 4.7684e-07, -9.5367e-07, 1.1921e-07,\n", + " 3.5763e-07, 1.0729e-06, 1.1921e-07, 1.1921e-07, -5.3644e-07,\n", + " -2.3842e-07, -1.1921e-07, -5.9605e-08, -2.9802e-07, -4.7684e-07,\n", + " -6.5565e-07, 3.5763e-07, -4.7684e-07, 4.7684e-07, 0.0000e+00,\n", + " -3.5763e-07, 1.1921e-07, 3.5763e-07, 8.3447e-07, -2.9802e-07,\n", + " -1.1921e-07, 3.5763e-07, -7.1526e-07, -1.7881e-07, 1.1921e-07,\n", + " 1.1921e-07, 1.1921e-07, 1.1921e-07, 5.9605e-07, 5.9605e-07,\n", + " 1.1921e-07, -3.5763e-07, -7.1526e-07, -3.5763e-07, -4.1723e-07,\n", + " -9.5367e-07, 8.3447e-07, -1.1921e-07, -5.9605e-08, -1.3709e-06,\n", + " -5.9605e-08, -2.9802e-07, -8.9407e-07, -1.5497e-06, -2.9802e-07,\n", + " -9.5367e-07, -5.9605e-08, 1.1921e-07, -1.1921e-07, -5.9605e-07,\n", + " -2.3842e-07, -5.3644e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07,\n", + " -4.7684e-07, -5.9605e-08, 1.1921e-07, 1.1921e-07, 1.1921e-06,\n", + " -4.7684e-07, 1.3113e-06, -5.9605e-07, 0.0000e+00, -7.1526e-07,\n", + " 8.3447e-07, -5.9605e-08, 7.1526e-07, -4.7684e-07, 2.3842e-07,\n", + " -8.9407e-07, 0.0000e+00, -5.9605e-07, -4.1723e-07, -5.9605e-08,\n", + " 1.1921e-07, -7.7486e-07, -8.9407e-07, -3.5763e-07, -4.1723e-07,\n", + " -3.5763e-07, -2.1458e-06, -5.9605e-08, -5.3644e-07, 2.3842e-07,\n", + " -5.9605e-07, 0.0000e+00, -8.3447e-07, -2.3842e-07, -7.1526e-07,\n", + " -3.5763e-07, -2.9802e-07, 4.7684e-07, 0.0000e+00, -4.7684e-07,\n", + " -3.5763e-07, -9.5367e-07, 1.1921e-06, -5.9605e-07, -3.5763e-07,\n", + " 3.5763e-07, -2.3842e-07, 4.7684e-07, -4.1723e-07, 5.9605e-07,\n", + " 0.0000e+00, 4.7684e-07, 0.0000e+00, 1.1921e-07, 5.9605e-07,\n", + " 7.1526e-07, 1.1921e-07, -1.2517e-06, -4.7684e-07, 4.7684e-07,\n", + " -1.1921e-07, 2.3842e-07, 1.1921e-07, -1.0133e-06, -1.7881e-07,\n", + " -1.7881e-07, -5.9605e-08, -5.3644e-07, 4.7684e-07, -1.1921e-07,\n", + " -5.9605e-08, 4.7684e-07, 3.5763e-07, 3.5763e-07, 4.7684e-07,\n", + " 4.7684e-07, 7.1526e-07, 9.5367e-07, -4.7684e-07, 2.3842e-07,\n", + " 7.1526e-07, -3.5763e-07, -6.5565e-07, -8.9407e-07, -1.7881e-07,\n", + " 2.3842e-07, 9.5367e-07, 3.5763e-07, 1.6689e-06, 1.1921e-07,\n", + " 1.1921e-07, -9.5367e-07, -7.1526e-07, 2.3842e-07, 1.0729e-06,\n", + " -5.9605e-07, 4.7684e-07, 5.9605e-07, 1.0729e-06, -1.1921e-06,\n", + " -4.1723e-07, 5.9605e-07, 3.5763e-07, -4.7684e-07, 3.5763e-07,\n", + " 0.0000e+00, 4.7684e-07, -4.1723e-07, 2.3842e-07, 2.3842e-07,\n", + " -3.5763e-07, -2.9802e-07, 2.3842e-07, 3.5763e-07, 3.5763e-07,\n", + " 2.3842e-07, -5.3644e-07, 0.0000e+00, -4.1723e-07, -1.7881e-07,\n", + " -2.9802e-07, -1.5497e-06, -5.9605e-08, 4.7684e-07, 4.7684e-07,\n", + " 1.1921e-07, -5.9605e-07, 1.1921e-06, 2.3842e-07, 2.3842e-07,\n", + " 1.1921e-07, -5.9605e-08, 0.0000e+00, 5.9605e-07, 0.0000e+00,\n", + " -1.1921e-07, -1.1921e-07, -2.9802e-07, -4.1723e-07, 3.5763e-07,\n", + " 9.5367e-07, -5.9605e-08, -2.3842e-07, 2.3842e-07, 5.9605e-07,\n", + " -1.7881e-07, 2.3842e-07, 5.9605e-07, 3.5763e-07, 1.3113e-06,\n", + " -1.6689e-06, 1.1921e-07, 4.7684e-07, -4.7684e-07, -5.9605e-07,\n", + " -5.9605e-08, -1.1921e-07, 1.6689e-06, -1.3709e-06, 1.1921e-07,\n", + " -2.3842e-07, 7.1526e-07, -1.1325e-06, 4.7684e-07, 1.1921e-07,\n", + " -4.1723e-07, -2.9802e-07, -1.1921e-07, -4.7684e-07, -4.1723e-07,\n", + " -5.3644e-07, 1.1921e-07, -8.9407e-07, 4.7684e-07, -1.3113e-06,\n", + " 8.3447e-07, 0.0000e+00, 8.3447e-07, -9.5367e-07, 1.1921e-07,\n", + " 2.3842e-07, -7.7486e-07, 0.0000e+00, 5.9605e-07, -8.3447e-07,\n", + " -1.0133e-06, 0.0000e+00, -1.7881e-07, 2.3842e-07, -1.4305e-06,\n", + " -1.8477e-06, 5.9605e-07, -5.9605e-08, 8.3447e-07, -1.0133e-06,\n", + " 8.3447e-07, -7.1526e-07, -1.0729e-06, 0.0000e+00, -3.5763e-07,\n", + " -5.3644e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07, 8.3447e-07,\n", + " -1.6689e-06, 8.3447e-07, 3.5763e-07, 1.3113e-06, 8.3447e-07,\n", + " 3.5763e-07, 3.5763e-07, 3.5763e-07, -1.1921e-07, -2.9802e-07,\n", + " 9.5367e-07, -7.1526e-07, -8.3447e-07, -6.5565e-07, 1.1921e-07,\n", + " -2.9802e-07, -1.1921e-07, 5.9605e-07, -7.7486e-07, -5.9605e-07,\n", + " 4.7684e-07, 5.9605e-07, 1.1921e-07, 2.3842e-07, -2.9802e-07,\n", + " -4.7684e-07, 1.0729e-06, -1.1921e-06, 3.5763e-07, -5.9605e-08,\n", + " 5.9605e-07, -9.5367e-07, 4.7684e-07, 7.1526e-07, -5.9605e-08,\n", + " -3.5763e-07, -6.5565e-07, -2.9802e-07, -1.7881e-07, -9.5367e-07,\n", + " -5.3644e-07, -2.3842e-07, -5.9605e-08, -2.9802e-07, 3.5763e-07,\n", + " -1.1921e-07, 1.1921e-07, -5.9605e-08, 3.5763e-07, 7.1526e-07,\n", + " -1.7881e-07, -5.9605e-08, -5.9605e-08, -2.9802e-07, 5.9605e-07,\n", + " 3.5763e-07, -1.1921e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07,\n", + " -7.7486e-07, -8.3447e-07, -5.9605e-07, 1.1921e-07, 7.1526e-07,\n", + " 4.7684e-07, -1.7881e-07, -1.1325e-06, -2.9802e-07, 4.7684e-07,\n", + " -5.3644e-07, -8.3447e-07, 3.5763e-07, 4.7684e-07, 3.5763e-07,\n", + " -4.1723e-07, 2.3842e-07, -5.3644e-07, -1.7881e-07, -1.7881e-07,\n", + " 2.3842e-07, 2.3842e-07, -1.7881e-07, 0.0000e+00, 4.7684e-07,\n", + " -1.7881e-07, -5.9605e-07, -1.7881e-07, 3.5763e-07, -1.3113e-06,\n", + " -3.5763e-07, 1.1921e-07, -5.3644e-07, 4.7684e-07, -9.5367e-07,\n", + " -1.1921e-07, -5.3644e-07, -1.7881e-07, -1.1921e-07, 0.0000e+00,\n", + " -8.3447e-07, 1.1921e-07, -1.1325e-06, -3.5763e-07, 5.9605e-07,\n", + " 3.5763e-07, -5.3644e-07, 7.1526e-07, -4.1723e-07, 2.3842e-07,\n", + " 7.1526e-07, 8.3447e-07, 1.3113e-06, 1.1921e-07, -1.0133e-06,\n", + " -1.2517e-06, 1.1921e-07, 3.5763e-07, 0.0000e+00, -4.1723e-07,\n", + " -8.3447e-07, 3.5763e-07, 3.5763e-07, -5.9605e-07, 5.9605e-07,\n", + " -8.3447e-07, -7.1526e-07, -2.9802e-07, -2.3842e-07, -7.1526e-07,\n", + " -5.3644e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07, -1.7881e-07,\n", + " 2.3842e-07, 2.3842e-07, -1.7881e-07, -2.3842e-07, 2.3842e-07,\n", + " -6.5565e-07, 1.3113e-06, -5.9605e-08, -2.9802e-07, 7.1526e-07,\n", + " 1.3113e-06, 1.1921e-07, 5.9605e-07, -2.3842e-07, -2.9802e-07,\n", + " 1.1921e-07, -1.7881e-07, -2.9802e-07, 0.0000e+00, -3.5763e-07,\n", + " -5.3644e-07, -7.1526e-07, 5.9605e-07, 1.1921e-07, 1.1921e-07,\n", + " -7.1526e-07, 7.1526e-07, 5.9605e-07, 7.1526e-07, 5.9605e-07,\n", + " 5.9605e-07, -7.7486e-07, -3.5763e-07, -5.9605e-08, -1.1921e-07,\n", + " 4.7684e-07, 2.3842e-07, 4.7684e-07, 1.3113e-06, 3.5763e-07,\n", + " -4.7684e-07, 2.3842e-07], device='cuda:0'),\n", + " tensor([ 1.4361e-07, 3.7875e-07, -8.3476e-07, 9.4134e-07, -6.7843e-07,\n", + " 3.1072e-07, 1.5542e-07, 1.5177e-07, 5.9782e-07, 5.7706e-07,\n", + " 3.6449e-07, -7.4512e-08, 1.9858e-07, -4.9469e-07, 7.5448e-07,\n", + " -5.9229e-07, 5.5775e-07, 7.6487e-07, -1.4062e-06, 7.2411e-07,\n", + " 4.1560e-07, -1.3103e-06, -6.2538e-07, -3.8220e-09, -1.0456e-06,\n", + " 1.7845e-07, -7.8648e-07, -4.8804e-07, -1.0593e-06, 1.1402e-06,\n", + " -2.8512e-07, 8.6257e-07, 5.7358e-07, -2.7734e-08, -1.3981e-07,\n", + " 6.7581e-07, -1.1391e-06, 3.5621e-08, -1.1710e-07, -5.2912e-07,\n", + " 1.0740e-06, 1.8467e-07, -6.4858e-08, -7.4553e-07, -4.4954e-07,\n", + " -2.0085e-08, -2.6388e-07, 1.0682e-07, -4.3396e-07, 5.0373e-07,\n", + " -3.1702e-09, 2.1772e-07, -3.6832e-07, -7.9641e-07, -5.5592e-07,\n", + " -8.9712e-08, -2.3330e-07, -5.7435e-07, 2.1003e-07, -3.4901e-07,\n", + " -1.2920e-07, -7.1090e-08, -2.8965e-07, -6.8484e-08, 1.1578e-07,\n", + " 1.8122e-07, -8.2851e-07, -5.4816e-07, -5.4788e-07, 6.4785e-07,\n", + " -6.7243e-07, -2.9483e-07, 3.0059e-07, -6.2761e-07, -1.3943e-07,\n", + " -4.6666e-08, 8.2269e-07, 5.2700e-07, -2.0402e-07, -1.8630e-07,\n", + " -1.4299e-07, -5.8553e-07, -4.3453e-08, 1.8669e-07, 4.3835e-08,\n", + " -1.5051e-07, 1.0361e-08, -8.9533e-07, -7.6743e-08, -2.0186e-07,\n", + " 5.8199e-07, 4.8817e-07, 5.3219e-07, 1.5891e-07, -5.4972e-07,\n", + " -4.3597e-08, 6.6097e-07, -4.5730e-10, -5.9889e-08, -5.4971e-07,\n", + " 1.5863e-08, 4.2603e-07, 4.4973e-07, -3.9109e-07, 3.3712e-07,\n", + " 4.1975e-07, 7.8129e-07, -4.5257e-07, -2.4825e-07, -6.3345e-07,\n", + " -7.7735e-07, 9.1151e-07, 5.8416e-08, -2.9035e-07, -1.3461e-06,\n", + " -2.6095e-07, -3.0684e-07, -1.6109e-06, -1.3722e-06, 2.2558e-08,\n", + " -4.3157e-07, -1.2280e-07, -1.1057e-07, -5.6610e-07, -2.7475e-07,\n", + " 3.3783e-07, -4.9134e-07, 5.5273e-07, 3.0693e-09, -2.3143e-07,\n", + " -8.8538e-07, -1.6646e-07, -1.4726e-07, -3.3940e-07, 1.1079e-06,\n", + " -3.8600e-07, 1.9262e-07, 8.6771e-09, -6.1051e-07, 2.0421e-07,\n", + " 6.8559e-07, 1.6167e-07, 6.7790e-07, -6.8239e-07, 1.2664e-07,\n", + " -9.9949e-09, 3.2615e-07, -6.1828e-07, -3.7162e-07, -3.7594e-07,\n", + " -1.4025e-09, -6.1806e-07, -3.8709e-07, -1.8537e-08, 5.6683e-07,\n", + " -1.8389e-07, -2.1760e-06, -2.1463e-07, -3.7027e-07, -1.2480e-07,\n", + " -5.7935e-07, 3.2740e-07, -3.9764e-07, -8.7763e-07, -6.9524e-07,\n", + " 1.8890e-07, 6.0723e-07, 2.8119e-07, 1.9862e-07, -4.5653e-07,\n", + " -3.2686e-07, -6.4305e-07, 3.0504e-07, -7.7494e-07, -1.0217e-06,\n", + " 1.7248e-07, 3.5446e-07, 3.2754e-07, -8.2340e-07, -2.8620e-07,\n", + " -1.2324e-07, 1.4035e-08, -4.2806e-07, 2.2246e-07, 5.9490e-07,\n", + " 9.5043e-07, 1.6535e-07, -8.7346e-07, -7.8100e-07, -5.2794e-09,\n", + " -7.6545e-07, -5.1436e-07, -4.0977e-07, -1.1618e-06, 6.8537e-08,\n", + " -1.4605e-07, 5.8257e-08, -8.0188e-07, -2.0639e-07, 2.1339e-07,\n", + " 1.5732e-07, 9.9957e-07, 4.9952e-07, 1.3014e-06, 2.9355e-07,\n", + " -3.4369e-07, 5.1924e-07, 5.3847e-07, -9.4620e-07, 5.6011e-07,\n", + " 7.9131e-07, -5.2138e-09, -1.4767e-06, -6.3886e-07, 1.4941e-07,\n", + " 9.8527e-07, -3.7774e-07, 3.2528e-07, 9.4757e-07, 2.1744e-07,\n", + " 5.9661e-07, -1.1091e-07, -6.2895e-07, 4.1626e-07, 9.3376e-07,\n", + " -2.4913e-07, 2.1312e-08, 7.0954e-07, 1.7708e-07, -1.0764e-06,\n", + " -1.6039e-07, -2.0194e-07, 3.0868e-07, -2.1210e-07, 2.8911e-07,\n", + " -3.3863e-07, 3.0655e-07, -8.2397e-07, 6.2707e-07, 4.0511e-07,\n", + " -6.2630e-07, -5.2816e-07, 6.9402e-07, 3.1391e-07, 6.8661e-07,\n", + " 1.2054e-07, -2.9753e-07, 2.2532e-09, -5.5803e-07, -2.0193e-07,\n", + " -5.2379e-07, -4.5617e-07, 1.9949e-08, 9.5923e-07, 6.3150e-07,\n", + " -3.8950e-07, -8.7060e-07, 3.2801e-07, -1.5988e-07, -2.0283e-07,\n", + " 2.0498e-08, -6.2763e-08, 1.2942e-07, 4.8690e-07, 2.9747e-08,\n", + " -5.9033e-07, -2.3964e-07, -7.6579e-07, -5.2635e-07, -1.1527e-06,\n", + " 6.4606e-07, -8.1474e-07, -4.2017e-07, 6.5501e-07, 3.6491e-07,\n", + " 5.6719e-07, -6.4178e-08, 8.4156e-07, 1.8924e-07, 5.0438e-07,\n", + " -8.7186e-07, 5.7862e-07, 7.9785e-07, -1.1940e-07, -2.8611e-07,\n", + " -3.7413e-07, 6.9951e-07, 1.1857e-06, -7.0392e-07, 2.6504e-07,\n", + " 6.2547e-07, 4.0341e-07, -6.8973e-07, 1.0543e-06, -3.9495e-07,\n", + " 6.2045e-07, -5.2668e-07, 3.1209e-07, 2.9866e-08, -1.3103e-06,\n", + " -7.4060e-07, -2.1091e-07, -1.1769e-06, 4.9558e-07, -1.5905e-06,\n", + " 2.1223e-07, 3.8217e-07, 2.4839e-07, -9.2843e-07, -3.9248e-07,\n", + " 5.8080e-08, -9.3520e-07, 2.0115e-08, 8.0949e-07, -6.3440e-07,\n", + " -6.9958e-07, 3.6014e-08, 1.2903e-08, -7.1608e-08, -1.1944e-06,\n", + " -9.8625e-07, 4.3675e-07, -5.9806e-07, 7.8947e-07, -8.5338e-07,\n", + " -3.3651e-07, -5.3144e-07, -6.9125e-07, 1.4479e-07, -8.7782e-07,\n", + " 2.1694e-07, 6.6267e-07, -1.7303e-07, 5.9009e-07, 1.9341e-07,\n", + " -1.2323e-06, -3.1903e-07, 1.7893e-07, 8.5715e-07, 1.6794e-06,\n", + " 4.4822e-07, 8.3358e-07, 7.1812e-08, 1.7047e-07, -1.3995e-07,\n", + " 1.5727e-06, -7.6507e-07, -1.1366e-06, -6.4408e-07, -3.2520e-07,\n", + " 3.6125e-08, -5.6142e-07, 1.0516e-06, -1.0970e-06, -4.1031e-07,\n", + " 1.9087e-07, -9.4430e-08, -4.1627e-08, 7.3488e-07, 2.9984e-07,\n", + " -4.2442e-07, 1.1619e-06, -9.3843e-07, -3.9253e-07, 7.6044e-07,\n", + " 8.9230e-07, -6.1505e-07, 2.8537e-07, 1.9979e-07, 1.1293e-07,\n", + " 9.3129e-09, -4.7411e-07, -3.3502e-07, 6.4352e-07, -8.2211e-07,\n", + " 2.8376e-07, -2.1078e-07, -4.7676e-07, -3.2840e-07, -1.7811e-07,\n", + " -1.0372e-07, -1.5150e-07, -2.9330e-07, -5.0891e-08, 4.6162e-07,\n", + " -4.4931e-07, -1.1251e-07, -2.1810e-08, 3.6239e-07, 4.1384e-07,\n", + " -1.9962e-07, -2.2854e-07, 4.1248e-07, 1.3120e-07, 2.2709e-07,\n", + " -6.5830e-07, 2.7621e-07, -2.8515e-07, -7.1727e-07, -2.2612e-07,\n", + " 2.9839e-07, -1.8185e-07, -7.0273e-07, -2.9024e-07, 5.5723e-07,\n", + " -4.9760e-07, -7.6321e-07, 2.0529e-07, 8.0877e-07, -8.9062e-08,\n", + " -6.0039e-07, 2.3167e-07, -1.9675e-07, 6.2634e-08, -5.7431e-07,\n", + " 7.7656e-07, 4.2352e-07, -5.7160e-07, -1.6343e-08, 6.0179e-07,\n", + " -6.7309e-07, 3.5918e-07, 4.8857e-08, 1.7525e-07, -5.2018e-07,\n", + " -2.2266e-07, -1.4090e-07, 1.4820e-07, 5.7232e-07, -6.3711e-07,\n", + " 3.2627e-07, -1.3462e-06, -6.5572e-08, -6.0948e-07, -5.2886e-07,\n", + " -4.4639e-07, -5.9024e-07, -7.8491e-07, -1.6743e-07, 2.8212e-09,\n", + " -1.1809e-07, -6.3132e-07, 4.9501e-07, -5.7799e-07, 6.2618e-07,\n", + " 4.2153e-07, 3.3870e-07, 2.6234e-07, 1.1589e-06, -6.3834e-07,\n", + " -1.7349e-07, -2.8091e-07, 1.3085e-07, 2.7772e-07, 8.0872e-08,\n", + " 2.6623e-07, 9.5979e-07, 2.8653e-07, -1.1525e-06, 4.1606e-07,\n", + " -8.0760e-07, -9.3070e-07, -4.5792e-07, 3.1079e-07, 3.9923e-07,\n", + " -6.9447e-07, 4.0161e-08, 1.7961e-07, -2.0261e-07, 5.0900e-08,\n", + " -5.9011e-07, -8.9109e-07, -5.2656e-07, -1.3622e-07, 5.2093e-07,\n", + " -1.5905e-06, 1.5041e-06, 2.2004e-07, -3.0572e-07, 5.7887e-07,\n", + " 1.7697e-07, -2.7734e-08, 5.3296e-07, 9.1149e-08, -3.8058e-07,\n", + " 3.5024e-07, 3.7948e-07, -1.5867e-08, -8.8103e-07, 7.0423e-08,\n", + " -8.5883e-07, -3.3453e-07, 6.0323e-07, -3.6401e-07, 3.0336e-07,\n", + " 4.4667e-07, 8.7888e-07, -1.7145e-07, 6.6095e-07, -7.5378e-07,\n", + " 3.2432e-07, -1.0764e-06, -9.0861e-08, 7.4299e-07, 2.8685e-07,\n", + " -8.3725e-07, 3.7188e-07, 1.8325e-07, 6.5297e-07, 1.3390e-07,\n", + " 6.0233e-07, 3.0721e-07], device='cuda:0'),\n", + " tensor([[ 5.8622e-06, 6.0122e-06, 5.8827e-06, ..., 5.6075e-06,\n", + " 1.0453e-05, -2.4978e-06],\n", + " [-8.0243e-06, -3.6648e-07, 7.9758e-06, ..., 9.5163e-06,\n", + " 9.7416e-06, 8.0727e-06],\n", + " [ 3.6899e-06, -4.5393e-06, -4.8149e-06, ..., -4.9956e-06,\n", + " -8.8997e-06, -4.2990e-06],\n", + " ...,\n", + " [ 1.6269e-05, 3.3859e-05, 1.9226e-05, ..., 1.8030e-05,\n", + " 3.7677e-06, 6.0163e-06],\n", + " [ 4.7535e-06, 8.4136e-06, 2.0556e-05, ..., 7.8417e-07,\n", + " 2.7270e-05, 2.2111e-05],\n", + " [-1.1865e-06, -4.3474e-06, -1.7798e-06, ..., 2.7046e-06,\n", + " 8.9593e-06, 1.0707e-05]], device='cuda:0'),\n", + " tensor([ 1.4883e-05, -1.2254e-05, 1.7649e-06, 5.1819e-06, -3.3751e-06,\n", + " -2.6276e-06, -1.3839e-05, 1.2923e-05, 8.4192e-06, -1.1077e-05],\n", + " device='cuda:0')]" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "input_parameters" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "config = dict(signed=True,\n", + " boxed=True,\n", + " cost_fn='sim',\n", + " indices='def',\n", + " weights='equal',\n", + " lr=0.1,\n", + " optim='adam',\n", + " restarts=1,\n", + " max_iterations=8_000,\n", + " total_variation=1e-6,\n", + " init='randn',\n", + " filter='none',\n", + " lr_decay=True,\n", + " scoring_choice='loss')\n", + "\n", + "rec_machine = inversefed.FedAvgReconstructor(model_bespoke, (dm, ds), 1, 3e-4, config,\n", + " use_updates=True, num_images=num_images)\n", + "# output, stats = rec_machine.reconstruct(input_parameters, labels_target, img_shape=(3, 32, 32))\n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "param_diff = [p.detach() for p in param_diff.values()]" + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "tensor([3, 8, 0, 6, 1, 9, 5, 7, 4, 2], device='cuda:0')" + ] + }, + "execution_count": 32, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels_target" + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "labels = torch.cat([torch.as_tensor((i,), device=setup['device']) for i in range(10)])\n", + "labels = labels.to(dtype=torch.float32)" + ] + }, + { + "cell_type": "code", + "execution_count": 34, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "It: 0. Rec. loss: 1.0003.\n", + "It: 500. Rec. loss: 0.9900.\n", + "It: 1000. Rec. loss: 0.9899.\n", + "It: 1500. Rec. loss: 0.9898.\n", + "It: 2000. Rec. loss: 0.9898.\n", + "It: 2500. Rec. loss: 0.9897.\n", + "It: 3000. Rec. loss: 0.9900.\n", + "It: 3500. Rec. loss: 0.9892.\n", + "It: 4000. Rec. loss: 0.9890.\n", + "It: 4500. Rec. loss: 0.9890.\n", + "It: 5000. Rec. loss: 0.9891.\n", + "It: 5500. Rec. loss: 0.9889.\n", + "It: 6000. Rec. loss: 0.9889.\n", + "It: 6500. Rec. loss: 0.9890.\n", + "It: 7000. Rec. loss: 0.9890.\n", + "It: 7500. Rec. loss: 0.9890.\n", + "It: 7999. Rec. loss: 0.9890.\n", + "Choosing optimal result ...\n", + "Optimal result score: 0.9889\n", + "Total time: 616.5541167259216.\n" + ] + } + ], + "source": [ + "\n", + "output, stats = rec_machine.reconstruct(param_diff, labels=labels, img_shape=(3, 32, 32))" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAACBsAAADHCAYAAABS68lEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAD8WklEQVR4nOyddVgWW9u3f0OnYoCKAYgodndgu21U7ACxsHXbuu3u7u7uDuzu7m5FFEREcn1/7I/7dfzdbsFtPM/7XudxeOw9J2tm1sysuNaauWc0pZSCIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCAjH53RkQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBOG/C3nYQBAEQRAEQRAEQRAEQRAEQRAEQRAEQRCERCEPGwiCIAiCIAiCIAiCIAiCIAiCIAiCIAiCkCjkYQNBEARBEARBEARBEARBEARBEARBEARBEBKFPGwgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCEKikIcNBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEFIFPKwgSAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIiUIeNhAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEIVHIwwaCIAiCIAiCIAiCIAiCIAiCIAiCIAiCICQKedhAEITvonTp0siRI8c30z18+BCapmHRokU/P1OC8A8cPHgQmqbh4MGD/xXbFYRfwZkzZ1CsWDHY2tpC0zRcvHjxd2dJEP7X4+rqimrVqn0znbH+xc/PD66urj8vc4LwLxg0aBA0TcObN29+d1YEwSjxZVQQ/tuQ9lX4T+Z3x6fSticcV1dXDBo06HdnQ/hJLFq0SOrCfxHx4/2HDx/+7qwIgo74+3mJvdchDxv8Zp4/f45BgwbJ5L4gCIIgCL+U6Oho1K1bF2/fvsXEiROxdOlSuLi4/O5sCcJ3MWPGDHmwUfg/z/HjxzFo0CCEhIT87qwIgiAIgvADiL95GP/PzMwMadOmhZ+fH549e/a7s/dfSWRkJHr16gVnZ2dYW1ujcOHC2Lt3b4LXX7VqFfLlywcrKys4OjqiRYsWRh8ECg0NRc+ePeHh4QFra2u4uLigRYsWePz4MaXdt28fypQpg5QpU8LBwQGFChXC0qVLKd2rV6/QvHlzODk5wdraGvny5cPatWsTdwISQPwDHCYmJnjy5An9/f3797C2toamaejQoYPub0FBQejcuTM8PT1hbW0NJycnFCpUCL169cKHDx8M6fz8/HRl+/N/VlZW35Xv58+fo0mTJsiSJQvs7e0N53Lx4sVQSn1z/fibv8b+nTx5ktJHRUVhxIgR8PT0hJWVFVKlSoWqVavi6dOnhjTXrl1D3bp1kTFjRtjY2CBlypQoVaoUtm7dStubO3cuvLy8kCpVKlhaWsLNzQ3Nmzf/KTejPz82ExMTODs7o2LFinRzMSoqCpMnT0bevHmRJEkSODg4IHv27GjdujVu3rxpSBffVllZWRltm4z9aNHV1VWXD1tbWxQqVAhLliz518cXFxeHmTNnIk+ePLC2tkaKFClQtmxZXLp06Zvrfpmv+H8BAQH/uF6rVq2gaVqCfliQWL6WJ03T8OnTJwD6/uLo0aO0DaUU0qdPbzSPHz58wMCBA5EjRw7Y2toiRYoUyJMnDzp37oznz58b0sW3DV/79/Lly0Qf25MnTzB48GAUKlQIyZIlQ8qUKVG6dGns27cvUdu5d+8eGjVqZGgfPTw80K9fP8Pf4+LisGjRItSoUQPp06eHra0tcuTIgWHDhhnO4ecktr1dvXo1ihYtCltbWzg4OKBYsWLYv39/oo4hIWzZssXQB2XIkAEDBw5ETEyMLk1gYCD8/f2ROXNm2NjYIGPGjGjZsiVevHjxw/PzOWY/devCN3n+/DkGDx4MV1dX5MmT53dnRxB+OC4uLoiIiIC5ufnvzorwf5xSpUohIiICFhYWvzsrgvAfwb179/Do0SPMnTsXLVu2/N3ZEYR/xYwZM5AyZUr4+fn97qz8MKTfEhLL8ePHMXjwYPj5+cHBweF3Z0cQBEEQhB/EkCFD4Obmhk+fPuHkyZNYtGgRjh49iqtXr373jdn/q/j5+WHdunXo0qULPDw8sGjRIlSpUgUHDhxAiRIl/nHdmTNnol27dihXrhwmTJiAp0+fYvLkyTh79ixOnTpluBZxcXGoUKECrl+/jnbt2iFz5sy4e/cuZsyYgd27d+PGjRuwt7cH8PeNI29vbxQtWtRwI2/NmjVo1qwZ3rx5g65duwL4+wZ/iRIl8OrVK3Tu3BmpU6fGmjVrUK9ePSxfvhyNGjX64efK0tISK1euRM+ePXV+w4YNRtO/ffsWBQoUwPv37+Hv7w9PT08EBwfj8uXLmDlzJtq2bQs7Ozvd9ufNm0fbMTU1/a78vnnzBk+fPoWPjw8yZMiA6Oho7N27F35+frh16xZGjBiRoO106tQJBQsW1LlMmTLplqOjo1G1alUcP34crVq1Qq5cufDu3TucOnUKoaGhSJcuHQDg0aNHCAsLg6+vL5ydnfHx40esX78eNWrUwOzZs9G6dWvDNi9cuAA3NzfUqFEDyZIlw4MHDzB37lxs27YNly5dgrOz83edl69RoUIFNGvWDEopPHjwADNmzEDZsmWxfft2VK5cGQBQp04d7Ny5Ew0bNkSrVq0QHR2NmzdvYtu2bShWrBg8PT1124yMjMSoUaMwderUBOUhT5486NatGwDgxYsXmDdvHnx9fREZGYlWrVp997H5+/tj+fLlaNasGTp06IDw8HBcuHABr1+/TnS+4smcOfNX0589exaLFi36qe2xsTwBoPkCKysrrFixgtqzQ4cO4enTp7C0tNT56OholCpVCjdv3oSvry86duyIDx8+4Nq1a1ixYgVq1apFZW/mzJm6uhzP94xBN2/ejNGjR8Pb2xu+vr6IiYnBkiVLUKFCBSxYsADNmzf/5jYuXryI0qVLI23atOjWrRtSpEiBx48f6x6W+vjxI5o3b44iRYogICAATk5OOHHiBAYOHIjAwEDs37/f8GaQxLa3gwYNwpAhQ+Dj4wM/Pz9ER0fj6tWrP/yhwJ07d8Lb2xulS5fG1KlTceXKFQwbNgyvX7/GzJkzDel69eqFt2/fom7duvDw8MD9+/cxbdo0bNu2DRcvXkTq1Kl/aL4MKOG3cubMGQVALVy48HdnRRAShZeXl8qePfvvzoYg/HDCw8MTlf7AgQMKgDpw4MDPyZAg/CQOHTqkAKi1a9f+Y7oPHz78ohwJwveTPXt25eXl9buzkSBcXFxU1apVv2tdX19f5eLi8mMzJPyvYezYsQqAevDgwT+mi42NVRERET98/wMHDlQAVFBQ0A/ftiD8COLLqCD8t/Gz29e4uDj18ePHn7Jt4d+xcOFCBUCdOXNG53v16qUAqNWrV/+mnP0Pvzs+TUzbfurUKQVAjR071uAiIiKUu7u7Klq06D+uGxkZqRwcHFSpUqVUWFiYwW/dulUBUFOmTDG4Y8eOKQBq2rRpum0sWLBAAVAbNmwwuAoVKihnZ2f16dMng4uOjlbu7u4qV65cBjdmzBgFQAUGBhpcbGysKliwoEqdOrWKjIz85vG7uLiogQMHfjNd/DmtXbu2ypMnD/29QoUKqk6dOgqAat++PeXx2LFjtE5oaKgu/vT19VW2trbfzMuPoFq1asrW1lbFxMT8Y7r4+b1vzZEopdTo0aOVubm5OnXqVKLzExMTo3Lnzq2yZMnyzbRnz55VANTIkSO/mTa+vUgIX147pZS6fPmyAqAqVqyolFLq9OnTCoAaPny40WN48+YN7TtPnjzK0tJSPXv2TJfe2H0EY+Pi169fKzs7O5U1a9YEHYcxVq9eTfUsMSR2vB4XF6eKFi2q/P39E7VufHn71tgtoXmKvwa1a9dWKVOmVNHR0bq/t2rVSuXPn5+2tWbNGgVALV++nLYZERGhQkNDDcs/Ix65evUqbe/Tp0/K09NTpUuX7pvrx8bGqhw5cqjChQv/YywTGRlptG0aPHiwAqD27t1rcIlpb0+cOKE0TVMTJkz4Zl7/LdmyZVO5c+fWXdt+/fopTdPUjRs3DO7QoUMqNjZWt278HHC/fv2+uZ8HDx58170O+YzCv+DZs2do0aIFnJ2dDa+2adu2LaKiovD27Vt0794dOXPmhJ2dHZIkSYLKlSvrXtVy8OBBw1NyzZs3N7xuRF4BK/wnEBYWhi5dusDV1RWWlpZwcnJChQoVcP78eV2669evo0yZMrCxsUHatGkxZswY3d/jv/Hyebn28/ODnZ0d7t+/j0qVKsHW1hbOzs4YMmRIgl5rJQif8+jRI7Rr1w5ZsmQxvBqrbt269JoxY9++jn+N17lz51CqVCnY2Nigb9++AP7nm9p79uxBnjx5YGVlhWzZsn31Ce7POXLkCOrWrYsMGTLA0tIS6dOnR9euXREREaFLF18Xnj17Bm9vb9jZ2cHR0RHdu3dHbGysLm1cXBwmTZqE7NmzG17P1qZNG7x79+77Tpzwfxo/Pz94eXkBAOrWrQtN01C6dGlDmbx37x6qVKkCe3t7NG7cGAAQHh6Obt26IX369LC0tESWLFkwbtw4arcjIiLQqVMnpEyZEvb29qhRowaePXsGTdPk+5ACgIS321/7/mr86wHj07u6uuLatWs4dOiQIZ4uXbq0If39+/dRt25dJE+eHDY2NihSpAi2b9+u22Z8H7FmzRoMHjwYadOmhb29PXx8fBAaGorIyEh06dIFTk5OsLOzQ/PmzREZGanbRkxMDIYOHQp3d3dYWlrC1dUVffv2pXTxfKt/MdZvGUP6BwH4u7706NEDAODm5maoC/GxeIcOHbB8+XJkz54dlpaW2LVr11fLmLH4HQBu3ryJevXqwdHREdbW1siSJYvu1ZTGePToETJlyoQcOXLg1atXP/KQBeEfOXr0KAoWLAgrKyu4u7tj9uzZlCah7XZcXBwGDRoEZ2dn2NjYoEyZMrh+/TpcXV3/V71RR/jPJyQkxPD2mqRJk6J58+b4+PGj4e8JLdPxY93du3ejQIECsLa2NtSRvXv3okSJEnBwcICdnR2yZMliGCPHExkZiYEDByJTpkyG8W7Pnj2/GvMIP56SJUsC+PttdZ9z8+ZN+Pj4IHny5LCyskKBAgWwZcsWWj8kJARdu3Y1zPmlS5fO8Ev6H0FCx44JKW9Tp05F9uzZYWNjA3t7e9ja2sLc3Jza9s8/T7Bs2TLkz58f1tbWSJ48ORo0aIAnT55g3bp1MDU1Nfya/NSpU/D29sbTp09x4sQJFC5cGMeOHdPtP35MsmXLFoSEhODMmTOG8w8A1apVg52dHVatWmVw79+/BwCkSpVKt600adIAAKytrXVpkyVLpvu1sZmZGVKmTKlLd+TIETg6OqJs2bIGZ2Jignr16uHly5c4dOiQ0Wvxb2jUqBEuXryoe13+y5cvsX//fqNvUrh37x5MTU1RpEgR+luSJEm++1ff9+7do7KeGFxdXfHx40dERUUleJ2wsDB6NXk8cXFxmDx5MmrVqoVChQohJiZG1xZ/C1NTU6RPnz5Bnz5zdXUFgF/ymbScOXMiZcqUePDgAYD/aV+KFy9OaU1NTZEiRQryffv2RWxsLEaNGvVdeXB0dISnpydd79DQUNy8eROhoaHf3MaECRNQqFAh1KpVC3FxcQgPD/+uvERFRSVo3aVLl+Lq1asYPnz4d+3nR9OwYUMEBwfrPg8TFRWFdevWfbXeAsavs5WVFZIkSfJd+Xj8+LGu7fga2bNnR8qUKXXO0tISVapUwdOnTxEWFvaP6+/ZswdXr17FwIEDYW1tjY8fP9KcOvD3GyCKFStGvlatWgCAGzduGFxi2ttJkyYhderU6Ny5M5RSus/FfMm/iZ+uX7+O69evo3Xr1jAz+58PFrRr1w5KKaxbt87gSpUqBRMT/a3/UqVKIXny5Lrj/NHIwwbfyfPnz1GoUCGsWrUK9evXx5QpU9C0aVMcOnQIHz9+xP3797Fp0yZUq1YNEyZMQI8ePXDlyhV4eXkZvnOSNWtWDBkyBADQunVrLF26FEuXLkWpUqV+56EJAgAgICAAM2fORJ06dTBjxgx0794d1tbWugbp3bt3+OOPP5A7d26MHz8enp6e6NWrF3bu3PnN7cfGxuKPP/5AqlSpMGbMGOTPnx8DBw7EwIEDf+ZhCf8LOXPmDI4fP44GDRpgypQpCAgIQGBgIEqXLp2gYD84OBiVK1dGnjx5MGnSJJQpU8bwtzt37qB+/fqoXLkyRo4cCTMzM9StW/eb3/Nbu3YtPn78iLZt22Lq1KmoVKkSpk6dimbNmlHa2NhYVKpUCSlSpMC4cePg5eWF8ePHY86cObp0bdq0QY8ePVC8eHFMnjwZzZs3x/Lly1GpUiVER0cn8GwJwt+0adPGMInTqVMnLF261HDDKCYmBpUqVYKTkxPGjRuHOnXqQCmFGjVqYOLEifjjjz8wYcIEZMmSBT169MCff/6p27afnx+mTp2KKlWqYPTo0bC2tkbVqlV/+TEK/7n823b7SyZNmoR06dLB09PTEE/Hl+dXr16hWLFi2L17N9q1a4fhw4fj06dPqFGjBjZu3EjbGjlyJHbv3o3evXvD398fGzZsQEBAAPz9/XH79m0MGjQItWvXxqJFizB69Gjdui1btsSAAQOQL18+TJw4EV5eXhg5ciQaNGhA+/ne/sUY0j8IAFC7dm00bNgQADBx4kRDXXB0dAQA7N+/H127dkX9+vUxefJkw8RlQrl8+TIKFy6M/fv3o1WrVpg8eTK8vb2Nfm82nnv37qFUqVKwt7fHwYMHacJdEH4WV65cQcWKFfH69WsMGjQIzZs3x8CBA6ndT2i73adPHwwePBgFChTA2LFj4eHhgUqVKn335LUgfC/16tVDWFgYRo4ciXr16mHRokUYPHiw4e+JiUVu3bqFhg0bokKFCpg8eTLy5MmDa9euoVq1aoiMjMSQIUMwfvx41KhRQ3fzNS4uDjVq1MC4ceNQvXp1TJ06Fd7e3pg4cSLq16//S86DAMNDt8mSJTO4a9euoUiRIrhx4wZ69+6N8ePHw9bWFt7e3rr278OHDyhZsiSmTp2KihUrYvLkyQgICMDNmzd135r/XhI6dkxIeZs7dy46deqEbNmyoUePHoiMjISmaShUqBC17fHzLcOHD0ezZs3g4eGBCRMmoEuXLggMDESpUqVw5swZZM6cGUmSJMH+/ftRqlQpvH//3vDg2OvXr1G2bFmcPn2ajiv+oU4fHx96vbu1tTUuXLiAuLg4AECBAgVga2uL/v37Y//+/Xj27BkOHTqEnj17omDBgihfvrxh3dKlS+PatWvo378/7t69i3v37mHo0KE4e/as7vMFkZGRuocP4rGxsQEAnDt3LuEXKYGUKlUK6dKlw4oVKwxu9erVsLOzMzrGd3FxQWxsLJYuXZrgfbx584b+xT+sEU+5cuVQrly5BG8zIiICb968wcOHD7F48WIsXLgQRYsWNXr+jNG8eXPDwxFlypTB2bNndX+/fv06nj9/jly5cqF169awtbWFra0tcuXKhQMHDhjdZnh4ON68eYN79+5h4sSJ2Llz51ePKTg4GK9fv8bZs2cNr5BPzPF/L+/evcO7d+8MDxG4uLgAAJYvX/7VBy++xM3NDc2aNcPcuXMN98ASQ0xMDJ4+fapr2wBg48aNyJo1q9Ex/Oe8f/8ep0+fRsGCBdG3b18kTZoUdnZ2yJgxI9asWZPgfOzfvx82Njaws7ODq6srJk+ebDRdWFgYevXqhb59+/68V9P/f6Kjo6muGJs/cXV1RdGiRbFy5UqD27lzJ0JDQ43GA/HXecmSJQn+Eejbt28pL18+ENOsWTNkzZo1EUeo5+XLl7CxsTG0cV9j3759AP5+QCG+7bWxsUGDBg3w9u3bBO0HgO6Bh8S0t4GBgShYsCCmTJkCR0dH2NvbI02aNJg2bZpu3X8bP124cAHA3/3L5zg7OyNdunSGv3+NDx8+4MOHD/Rgxw8lUe9BEAw0a9ZMmZiY0GuslPr71SmfPn2iV1U8ePBAWVpaqiFDhhicfEZB+E8ladKk9Dqlz/Hy8lIA1JIlSwwuMjJSpU6dWtWpU8fg4l+78nkZ9/X1VQBUx44dDS4uLk5VrVpVWVhYyKtXhURh7BVJJ06coPJp7HMH8eV41qxZtA0XFxcFQK1fv97gQkNDVZo0aVTevHn/cbvG8jRy5EilaZp69OiRwcXXhc/7BaWUyps3r8qfP79h+ciRI0ZfabVr166vvupKEL6FsVcExpfJ3r1769Ju2rRJAVDDhg3TeR8fH6Vpmrp7965SSqlz584pAKpLly66dH5+fgpAgl7ZKPzvJ6Ht9tdeiRr/esDPXzf4tc8odOnSRQFQR44cMbiwsDDl5uamXF1dDfF6fH3IkSOHioqKMqRt2LCh0jRNVa5cWbfdokWL6l4Xe/HiRQVAtWzZUpeue/fuCoDav3+/wf2b/uXL19RK/yB8ztc+owBAmZiYqGvXrun81z4FZSx+L1WqlLK3t9fFMUr9HcPH8/lrNW/cuKGcnZ1VwYIF1du3b3/I8QlCQvH29lZWVla68nr9+nVlampq6FcS2m6/fPlSmZmZKW9vb126QYMGKQDK19f35x6MIKj/aV/9/f11vlatWipFihRKqe+LRXbt2qVLO3HixG++Hnnp0qXKxMREF1sppdSsWbO++vp04fuJj3v37dungoKC1JMnT9S6deuUo6OjsrS0VE+ePDGkLVeunMqZM6fudfxxcXGqWLFiysPDw+AGDBjw1VeMf96vJ5Qv49OEjh0TUt5q1qxpePX6t9p2Ly8v9fDhQ2Vqakqvfb9y5YoyMzNTTk5OqmzZsiouLk55eHioSpUqqbi4OHXt2jXDpxDc3NxUhQoVDOt+/kkBTdNUixYtdNu+efOmAqAA6F4rv23bNpUmTRrD3wCoSpUq6T7BoNTfny2sV6+e0jTNkM7GxkZt2rRJl65jx47KxMREPXz4UOcbNGigAKgOHTp89TzGk9jPKAQFBanu3burTJkyGf5WsGBB1bx5c6UUv4r/5cuXytHRUQFQnp6eKiAgQK1YsUKFhITQPuLnHoz9q1SpEuU7MZ/qGDlypG575cqVU48fP/7meseOHVN16tRR8+fPV5s3b1YjR45UKVKkUFZWVur8+fOGdBs2bFAAVIoUKZSHh4dauHChWrhwofLw8FAWFhbq0qVLtO02bdoY8mNiYqJ8fHy+GiNbWloa0qZIkUL3iY5/IrGfUWjRooUKCgpSr1+/VqdOnVLlypVTANT48eOVUn+3B/HzpqlSpVINGzZU06dPp/HA5/s+c+aMunfvnjIzM1OdOnUy/P1rn1GoWLGiCgoKUkFBQerKlSuqadOmRj/xEL/9b91DO3/+vOG8pUqVSs2YMUMtX75cFSpUSGmapnbu3PnNc1O9enU1evRotWnTJjV//nxVsmRJBUD17NmT0nbv3l25ubkZ2t2f+RkFY3Xl8/r8+TWYNm2asre3N8y91K1bV5UpU8ZoHj9+/KiyZMmiACgXFxfl5+en5s+fr169ekX5iG8bjP378rMg8WXne7hz546ysrJSTZs2/WbaGjVqGK5548aN1bp161T//v2VmZmZKlas2Df7tfLly6skSZKod+/eGVxC29u3b98a9m1nZ6fGjh2rVq9erf744w+63/Bv46f4sb6xtqxgwYKqSJEi/7j+0KFD6dMQX0M+o/ALiYuLw6ZNm1C9enV6kgQANE2DpaWl4VUVsbGxCA4ONrwO6svX0AvCfyIODg44derUPz6FaGdnhyZNmhiWLSwsUKhQIdy/fz9B++jQoYPh/+Nf7xoVFWV4Ik0QEsLnTxpGR0cjODgYmTJlgoODQ4LaW0tLS8OTwl/i7OxseJ0S8Pdr35o1a4YLFy4Ynnz8Vp7in14uVqwYlFJGnzQMCAjQLZcsWVJXj9auXYukSZOiQoUKuqdG8+fPDzs7u68+OS0I30vbtm11yzt27ICpqSk6deqk8926dYNSyvBGm127dgH4+zVen9OxY8efmFvhv41/224nhh07dqBQoUIoUaKEwdnZ2aF169Z4+PAhrl+/rkvfrFkzmJubG5YLFy4MpRT8/f116QoXLownT54YfuGxY8cOAKA3fXTr1g0A6LMN39u/fIn0D0JC8fLyQrZs2b5r3aCgIBw+fBj+/v7IkCGD7m/GPnVy9epVeHl5wdXVFfv27aNfJgnCzyQ2Nha7d++Gt7e3rrxmzZoVlSpVMiwntN0ODAxETEyMxDbCfwTGxo3BwcF4//59omMRNzc3XZ0A/p4HAoDNmzcbfqH9JWvXrkXWrFnh6empiz3iXzUsscfPoXz58nB0dET69Onh4+MDW1tbbNmyBenSpQPw969M9+/fb3j7Rfx1CQ4ORqVKlXDnzh08e/YMALB+/Xrkzp1bF4vGY6xfTywJHTsmpLw5ODjg6dOnOHny5Dfb9oMHD2LDhg2Ii4tDvXr1dOUzderU8PDwQFhYGCwtLXHx4kXcuXMHjRo1QnBwsOHXwaGhoShXrhwOHz5MeercuTPq1auHxYsXY/z48bh//z6OHDmC+vXrG8YPn38609HREXnz5sXw4cOxadMmDBo0CEeOHKH5J0tLS2TOnBk+Pj5YuXIlli1bhgIFCqBJkyY4efKkIV3Lli1hamqKevXq4fjx47h37x5Gjhxp+KX3l5/t/FE0atQId+/exZkzZwz/NfYqduDvz0ZcunQJAQEBePfuHWbNmoVGjRrByckJQ4cOpV9NW1lZYe/evfTvy1fwP3z4kD659080bNgQe/fuxYoVKwx5Tcj5KVasGNatWwd/f3/UqFEDvXv3xsmTJ6FpGvr06WNIF/+a9LCwMAQGBsLPzw9+fn7Yt28flFL0iWEA6NKlC/bu3YvFixejcuXKiI2N/epnHXbu3IkdO3Zg/PjxyJAhw097k9L8+fPh6OgIJycnwydE/vzzT3Tp0gXA3+3B7t27MWzYMCRLlgwrV65E+/bt4eLigvr163/10w4ZM2ZE06ZNMWfOHLx48eIf87Bnzx44OjrC0dEROXPmxNKlS9G8eXOMHTtWl87Pzw9KqW9+vir+2gQHB2Pz5s1o27YtGjVqhMDAQKRIkQLDhg375nnZsmULevbsiZo1a8Lf3x+HDh1CpUqVMGHCBN3bX27fvo3Jkydj7Nixus+g/CwKFy5MdcXYG3SBv9+EFBERgW3btiEsLAzbtm37ar21trbGqVOnDG9vWbRoEVq0aIE0adKgY8eORl/xv379esrLwoULdWkOHjz4XZ/L/vjxI+rWrQtra+sEfY4j/poXLFgQy5YtQ506dTBkyBAMHToUx48fR2Bg4FfXHTFiBPbt24dRo0YZ+iQg4e3t5+Vt3rx56N69O+rVq4ft27cjW7ZsuvL2b+On+H0aK2tWVlb/2MYdPnwYgwcPRr169XSfhvjRmH07ifAlQUFBeP/+PXLkyPHVNPHf7pkxYwYePHig+06Ise/ZCMJ/GmPGjIGvry/Sp0+P/Pnzo0qVKmjWrBkyZsxoSJMuXToaiCRLlgyXL1/+5vZNTEx02wKAzJkzA0CiAkhBiIiIwMiRI7Fw4UI8e/ZMF8gk5FteadOmhYWFhdG/ZcqUicr45+X0a6/Ievz4MQYMGIAtW7bQN7O/zJOVlZXhFcfxJEuWTLfenTt3EBoaCicnJ6P7e/36tVEvCN+DmZmZYdIqnkePHsHZ2Rn29vY6H/9KtEePHhn+a2JiAjc3N126TJky/cQcC/9t/Nt2OzE8evQIhQsXJv952f08pv/yRmrSpEkBAOnTpycfFxeH0NBQpEiRwlD2vyzrqVOnhoODg6GOxPO9/cuXSP8gJJQv2+XEEP8A5D+Nfz+nevXqSJUqFXbv3g07O7vv3q8gfA9BQUGIiIiAh4cH/S1LliyGG7IJbbfj//tluuTJk8uDNMIv58s4Jb4Mvnv3LtGxiLF+oX79+pg3bx5atmyJ3r17o1y5cqhduzZ8fHwMP6i6c+cObty4QWPYeCT2+DlMnz4dmTNnRmhoKBYsWIDDhw/rbjjcvXsXSin0798f/fv3N7qN169fI23atLh37x7q1Knz0/Ka0LFjQspbr169sG/fPhQtWhTA38d57Ngx3bfFP2/b79y5A6WU0T4A+PsmTWRkJO7cuQMA8PX11f3983MXGhqqa+fd3Nwwe/ZsREREoHv37ujevTsAoEmTJnB3d8eGDRsMcc/9+/dRpkwZLFmyxHCua9asCVdXV/j5+WHnzp2oXLkygL9/iHXy5EmcP3/ecNz16tVD9uzZ0blzZ5w6dQoAkCtXLqxYsQIBAQGG40+dOjUmTZqEtm3b/rSYK2/evPD09MSKFSvg4OCA1KlT/+PNqjRp0mDmzJmYMWMG7ty5g927d2P06NEYMGAA0qRJg5YtWxrSmpqa6j4p8aNwcXExvBq+YcOGaN26NcqXL49bt24l+FMK8WTKlAk1a9bEhg0bEBsbC1NTU8M2ihcvrhsnZsiQASVKlMDx48dpO56envD09ATw9wPuFStWRPXq1XHq1CkaF8Z/3rVy5cqoWbMmcuTIATs7O92P9n4ENWvWRIcOHaBpGuzt7ZE9e3bY2trq0lhaWqJfv37o168fXrx4gUOHDmHy5MlYs2YNzM3NsWzZMqPb/uuvv7B06VKMGjXqq58gAP6+gT5s2DDExsbi6tWrGDZsGN69e/fVedpvEX9t3NzcdPMAdnZ2qF69OpYtW4aYmBjdN++/haZp6Nq1K3bv3o2DBw8afnjZuXNnFCtW7Ke2p5+TMmXKBNcXR0dHlC9fHitWrMDHjx8RGxsLHx+fr6ZPmjQpxowZgzFjxuDRo0cIDAzEuHHjMG3aNCRNmpQe0ihVqtRPeR1/bGwsGjRogOvXr2Pnzp1wdnb+5jrx1zz+k4LxNGrUCH369MHx48eNnrfVq1fjr7/+QosWLegHVwltb+P3bW5urju/JiYmqF+/PgYOHIjHjx8jQ4YMCY6f3r59q3sQydraGkmTJjXsy9jDH58+ffpq23bz5k3UqlULOXLkwLx584ym+VHIwwY/iREjRqB///7w9/fH0KFDkTx5cpiYmKBLly5ffVpTEP6TqFevHkqWLImNGzdiz549GDt2LEaPHo0NGzYYAmJTU1Oj637PU2uC8L107NgRCxcuRJcuXVC0aFEkTZoUmqahQYMGCWpvEzvQ+BaxsbGoUKEC3r59i169esHT0xO2trZ49uwZ/Pz8KE9fq0efExcXBycnJyxfvtzo378WqAjC9/D525kE4WeQ0Hb7a7+s+vwh3h/N19rkhMY8P+LXYIlB+gchoRiLd35WHatTpw4WL16M5cuXo02bNv9qW4Lws/nV7bYg/BsSEo8ktEwb6xesra1x+PBhHDhwANu3b8euXbuwevVqlC1bFnv27IGpqSni4uKQM2dOTJgwweh2v3xAU/gxFCpUyPB2XW9vb5QoUQKNGjXCrVu3YGdnZ4ihu3fvTm+siOc/7QHwhJS3rFmz4tatW1i2bBkCAgJw48YNlChRAgMGDMDgwYNpm3FxcdA0DTt37jRaX/r27YsXL14YztfYsWORJ08enD9/Hr169cKQIUMMDzZ8efM+/obP5s2b8fjxYzx8+NBwU7tYsWJwdHQ0/DJ20aJF+PTpE6pVq6bbRo0aNQAAx44dQ+XKlREVFYX58+ejZ8+eujG4ubk5KleujGnTpiEqKspw49XHxwc1atTApUuXEBsbi3z58uHgwYMA/ufB5Z9Bo0aNMHPmTNjb26N+/foJmi/QNA2ZM2dG5syZUbVqVXh4eGD58uW6hw1+FT4+Ppg7dy4OHz781frxT6RPnx5RUVEIDw9HkiRJDDdAU6VKRWmdnJy++e30+Dy1adMGt2/fRpYsWb6azt3dHXnz5sXy5ct/+MMG6dKlS9TDHmnSpEGDBg1Qp04dZM+eHWvWrMGiRYuM3rjPmDEjmjRpgjlz5qB3795f3ebnN9ArVaoET09PVKtWDZMnT6Y39SSEb12b6OhohIeHG35UkFDi+7a3b98CAPbv349du3Zhw4YNuh9NxsTEICIiAg8fPkTy5MmRJEmSRB/Dj6JRo0Zo1aoVXr58icqVK+t+uf9PuLi4wN/fH7Vq1ULGjBmxfPnyBL0R4kfQqlUrbNu2DcuXL0/wL/C/ds3jf5Dx5Y8AARjeClG1alXMmjXL6HYT0t4mT54cVlZWcHBwoD7n8/1nyJAhwfFT7dq1cejQIYP39fXFokWLkCZNGgDAixcvKNZ68eIFChUqRNt88uQJKlasiKRJk2LHjh30EOCPRh42+A4cHR2RJEkSXL169atp1q1bhzJlymD+/Pk6HxISonvqRwa2wn8yadKkQbt27dCuXTu8fv0a+fLlw/Dhww0PG/wb4uLicP/+fV0wfPv2bQCAq6vrv96+8H+HdevWwdfXF+PHjze4T58+ffV1Xokh/tcBn7fV3yqnV65cwe3bt7F48WLd66z27t373flwd3fHvn37ULx48R/+cIQgJAQXFxfs27cPYWFhuuD05s2bhr/H/zcuLg4PHjzQ/Zrk7t27vzbDwn80CW23439JFBISohsYf/nLPODrMbWLiwtu3bpF/suy+2+JL/t37twx/GoLAF69eoWQkBDaz/f0L8aQ/kH4nMSOLT+vY5/zZR2LfxvZP41/P2fs2LEwMzNDu3btYG9v/9VXdgrCz8DR0RHW1taGX61+zuf9QULb7fj/3r17V/dL8ODgYKOTl4Lwu0hsLPI1TExMUK5cOZQrVw4TJkzAiBEj0K9fPxw4cADly5eHu7s7Ll26hHLlysmc5m/C1NQUI0eORJkyZTBt2jT07t3b0Febm5t/8+ahu7t7gvv07yGhY0fg2+UNAGxtbdGyZUt07doVVapUwcePHzF8+HD06dMHVlZWurbd3d0dSim4ubkZvfm+e/duTJw40XBTKkmSJChfvjxOnz4N4O9XtSfkgZkMGTIY3jQSEhKCc+fO6X7d/OrVKyil6AHO6OhoADB8ii04OBgxMTFGH/SMjo5GXFwc/c3CwgIFCxY0LMd/ivZnvCEgnkaNGmHAgAF48eIFli5dmuj1M2bMiGTJkn3zlfo/i/jXi3/vW/Tu378PKysrwwMoOXPmhLm5ueHTJJ/z/PnzBD3wnZg8RUREGP018+/C3NwcuXLlwp07dwyfKTHGX3/9hWXLlmH06NEJ3nbVqlXh5eWFESNGoE2bNvSmhW/h7OyM1KlTf/XaWFlZfdcN1/g3vcVf28ePHwP4+8bwlzx79gxubm6YOHGi4ZMUv4NatWqhTZs2OHnyJFavXp3o9ZMlS/bT+4vP6dGjBxYuXIhJkybRWwr+ifz582Pu3Ll0zeM/Df5lfTx16hRq1aqFAgUKYM2aNf/4lotvtbcmJibIkycPzpw5o3swzNj+Exo/jR8/XjfGiH+YIk+ePACAs2fP6h4seP78OZ4+fYrWrVvrthMcHIyKFSsiMjISgYGBhocVfibys7XvwMTEBN7e3ti6dSvOnj1Lf1dKwdTUlH7ptHbtWir08Q3mj7gpJgg/itjYWAp2nJyc4Ozs/EODm2nTphn+XymFadOmwdzcHOXKlfth+xD+92OsvZ06deoP+eXr8+fPDd9jAoD3799jyZIlyJMnz1eD6fgnGT/Pk1LqH18b9i3q1auH2NhYDB06lP4WExMjfYjw06lSpQpiY2N17TYATJw4EZqmGR5Ci/+VwIwZM3Tppk6d+msyKvxXkNB2293dHcDf35eLJzw8HIsXL6Zt2traGm0Lq1SpgtOnT+PEiRO6bcyZMweurq7f/Q17Y/sBgEmTJul8/FPrVatW1fnv6V+MIf2D8DmJHVu6uLjA1NRUV8cAbsMdHR1RqlQpLFiwwDCxFo+xN5ppmoY5c+bAx8cHvr6+2LJlSyKOQhD+HaampqhUqRI2bdqkK683btzA7t27DcsJbbfLlSsHMzMzzJw5U5fuy5hIEH43iY1FjBH/i83PiZ/cjp8LqlevHp49e4a5c+dS2oiIiJ/2bXFBT+nSpVGoUCFMmjQJnz59gpOTE0qXLo3Zs2cbvaEbFBRk+P86derg0qVLulg0ns/79Zs3b1K/nxASOnZMSHkLDg4G8D9t+9atW5EuXToopRAdHa1r2x8/fozatWvD1NQUgwcPphhFKYUKFSogNjYWZ8+ehbu7O8aNG4fg4GAsXLgQhQsXNjxoEBQUhMePHxsekPgn+vTpg5iYGHTt2tXgMmfODKUU1qxZo0u7cuVKAH9/mgD4e67VwcEBGzdu1L02+8OHD9i6dSs8PT3/8YHiO3fuYNasWahWrdpPfbOBu7s7Jk2ahJEjRxr99Ww8p06dMtoGnD59GsHBwf/4C/5/4t69e7h37943031ezj9n/vz50DQN+fLlM7g3b97g5s2b+Pjx4z+uf+nSJWzZsgUVK1Y0vNHB3t4eVapUwfHjx3Vl5MaNGzh+/DgqVKhgcMY+LRMdHY0lS5bA2traMB6NiYkx+hDj6dOnceXKFcObTX4ld+7cMdoGhISE4MSJE0iWLNk/Pljh7u6OJk2aYPbs2Xj58mWC99urVy8EBwfr+pnQ0FDcvHkzQQ9n1K9fH0+ePNH96OvNmzfYvHkzypYta7iO0dHRuHnzpq7NfPv2rdGHhEaNGgULCwvDJy7Kli2LjRs30j9HR0cUKFAAGzduRPXq1RN8zD8DOzs7zJw5E4MGDfrHvFy6dAlv3rwh/+jRI1y/fv27621C21Dg74fVx40bh759+6Jz585fTWesHNSsWROWlpZYuHCh7k2Z8Z8M+Lw+3rhxA1WrVoWrqyu2bduWqB9sfK29rV+/PmJjY3XzVJ8+fcLy5cuRLVs2w8MCCY2f8ufPj/Llyxv+xbcR2bNnh6enJ+bMmaMrozNnzoSmabrPOISHh6NKlSp49uwZduzY8dVPC/1o5M0G38mIESOwZ88eeHl5oXXr1siaNStevHiBtWvX4ujRo6hWrRqGDBmC5s2bo1ixYrhy5QqWL19O36h3d3eHg4MDZs2aBXt7e9ja2qJw4cL/6puagvBvCQsLQ7p06eDj44PcuXPDzs4O+/btw5kzZ3S/Qvw3WFlZYdeuXfD19UXhwoWxc+dObN++HX379pVX/gqJolq1ali6dCmSJk2KbNmy4cSJE9i3bx9SpEjxr7edOXNmtGjRAmfOnEGqVKmwYMECvHr1CgsXLvzqOp6ennB3d0f37t3x7NkzJEmSBOvXr/9Xv3zy8vJCmzZtMHLkSFy8eBEVK1aEubk57ty5g7Vr12Ly5Mn/+O0tQfi3VK9eHWXKlEG/fv3w8OFD5M6dG3v27MHmzZvRpUsXw03h/Pnzo06dOpg0aRKCg4NRpEgRHDp0yPCLbfn1kwAkvN2uWLEiMmTIgBYtWqBHjx4wNTXFggUL4OjoSJMe+fPnx8yZMzFs2DBkypQJTk5OKFu2LHr37o2VK1eicuXK6NSpE5InT47FixfjwYMHWL9+/Q/7ZEju3Lnh6+uLOXPmICQkBF5eXjh9+jQWL14Mb29vw6REPN/TvxhD+gfhc/Lnzw8A6NevHxo0aABzc/N/nFhKmjQp6tati6lTp0LTNLi7u2Pbtm1GJ0WnTJmCEiVKIF++fGjdujXc3Nzw8OFDbN++HRcvXqT0JiYmWLZsGby9vVGvXj3s2LEjwa/CFIR/y+DBg7Fr1y6ULFkS7dq1Q0xMDKZOnYrs2bPj8uXLABLebqdKlQqdO3fG+PHjUaNGDfzxxx+4dOkSdu7ciZQpU0psI/zHkNhYxBhDhgzB4cOHUbVqVbi4uOD169eYMWMG0qVLhxIlSgAAmjZtijVr1iAgIAAHDhxA8eLFERsbi5s3b2LNmjXYvXv3b7kp9n+RHj16oG7duli0aBECAgIwffp0lChRAjlz5kSrVq2QMWNGvHr1CidOnMDTp09x6dIlw3rr1q1D3bp14e/vj/z58+Pt27fYsmULZs2ahdy5cwMAsmbNCi8vL8NroxNKQseOCSlvFStWROrUqVG8eHHkzp0bW7duxezZs5E1a1ZMmTJF17Y3a9YMBw8exLBhw9CnTx88fPgQ3t7esLe3x4MHD7Bx40a0bt0adevWRd++fVGnTh2sX78ezs7OiImJgbe3NwYNGoQDBw4gSZIkCAsLw6FDhzBw4EDDsY0aNQpXr15F4cKFYWZmhk2bNmHPnj0YNmyY7tevfn5+GDduHNq0aYMLFy4ge/bsOH/+PObNm4fs2bOjVq1aAP5+iKJ79+7466+/UKRIETRr1gyxsbGYP38+nj59imXLlunObbZs2VC3bl1kyJABDx48wMyZM5E8efKvvgb8R/JPNwDjWbp0KZYvX45atWohf/78sLCwwI0bN7BgwQJYWVmhb9++uvQxMTF0jPHUqlXL8CBt/I/SPn9dvTGGDx+OY8eO4Y8//kCGDBnw9u1brF+/HmfOnEHHjh11nxKZNm0aBg8ejAMHDqB06dIA/r5paG1tjWLFisHJyQnXr1/HnDlzYGNjg1GjRun2NWLECAQGBqJs2bLo1KkTgL/j5eTJk+uOs02bNnj//j1KlSqFtGnT4uXLl1i+fDlu3ryJ8ePHG96W8OHDB6RPnx7169dH9uzZYWtriytXrmDhwoVImjQp+vfv/42z/+O5dOkSGjVqhMqVK6NkyZJInjw5nj17hsWLF+P58+eYNGnSNz8N269fPyxduhS3bt1C9uzZE7TfypUrI0eOHJgwYQLat28Pc3NzbNy4Ec2bN8fChQvh5+f3j+v36dMHa9asQZ06dfDnn38iadKkmDVrFqKjozFixAhDumfPniFr1qyG19QDwJYtWzBs2DD4+PjAzc0Nb9++xYoVK3D16lWMGDHC8MOAz99u8jldunRBqlSp4O3tnaBj/dn4+vp+M83evXsxcOBA1KhRA0WKFIGdnR3u37+PBQsWIDIyEoMGDaJ11q1bR5+aAf6+sR//5phmzZrh0KFD3/zc9saNG9GzZ094eHgga9as1CZ8vk1j5SB16tTo168fBgwYgD/++APe3t64dOkS5s6di4YNGxra5rCwMFSqVAnv3r1Djx49sH37dt1+3N3dDZ/RARLe3rZp0wbz5s1D+/btcfv2bWTIkAFLly7Fo0ePsHXrVkO6HxE/jR07FjVq1EDFihXRoEEDXL16FdOmTUPLli11b7Vq3LgxTp8+DX9/f9y4cQM3btww/M3Ozu7nlU8lfDePHj1SzZo1U46OjsrS0lJlzJhRtW/fXkVGRqpPnz6pbt26qTRp0ihra2tVvHhxdeLECeXl5aW8vLx029m8ebPKli2bMjMzUwDUwoULf8vxCEI8kZGRqkePHip37tzK3t5e2draqty5c6sZM2YY0nh5eans2bPTur6+vsrFxcWw/ODBAyrXvr6+ytbWVt27d09VrFhR2djYqFSpUqmBAweq2NjYn3lowv9C3r17p5o3b65Spkyp7OzsVKVKldTNmzeVi4uL8vX1NaQ7cOCAAqAOHDhgcF8rx0op5eLioqpWrap2796tcuXKpSwtLZWnp6dau3atLp2x7V6/fl2VL19e2dnZqZQpU6pWrVqpS5cufbUufMnAgQOVsS56zpw5Kn/+/Mra2lrZ29urnDlzqp49e6rnz58n7GQJwmfEl93Py/TXyqRSSoWFhamuXbsqZ2dnZW5urjw8PNTYsWNVXFycLl14eLhq3769Sp48ubKzs1Pe3t7q1q1bCoAaNWrUTz0m4b+DhLbbSil17tw5VbhwYWVhYaEyZMigJkyYoBYuXKgAqAcPHhjSvXz5UlWtWlXZ29srALp4+969e8rHx0c5ODgoKysrVahQIbVt2zbdfozVB6WUYV9nzpzR+fh2OigoyOCio6PV4MGDlZubmzI3N1fp06dXffr0UZ8+fdKt+2/6ly/jrHikfxDiGTp0qEqbNq0yMTEx1BMAqn379kbTBwUFqTp16igbGxuVLFky1aZNG3X16lWj49KrV6+qWrVqGepSlixZVP/+/Q1/N1YvPn78qLy8vJSdnZ06efLkTzlmQTDGoUOHVP78+ZWFhYXKmDGjmjVrFsXYCW23Y2JiVP/+/VXq1KmVtbW1Klu2rLpx44ZKkSKFCggI+NWHJvwfxFj7qpSimCixsciXBAYGqpo1aypnZ2dlYWGhnJ2dVcOGDdXt27d16aKiotTo0aNV9uzZlaWlpUqWLJnKnz+/Gjx4sAoNDf2xB/9/nK/FokopFRsbq9zd3ZW7u7uKiYlRSv0d9zZr1kylTp1amZubq7Rp06pq1aqpdevW6dYNDg5WHTp0UGnTplUWFhYqXbp0ytfXV71588aQ5suY+msYi08TMnZMSHmbPXu2KlWqlEqRIoWytLRUadOmValSpVLm5ubUtn+e1/Xr16sSJUooW1tbZWtrqzw9PVX79u3VrVu3VEREhOrevbtKnTq1srCwUMmSJVP29vbK0tJSubi4qHr16qnAwEDl5eWlAOjq37Zt21ShQoWUvb29srGxUUWKFFFr1qwxel6ePn2q/P39lZubm7KwsFBp0qRRrVq1onqslFLLly9XhQoVUg4ODsra2loVLlyYrplSSjVo0EClT5/ecL4CAgLUq1evvnmN4nFxcVEDBw78ZrqvtTlf8mWMefnyZdWjRw+VL18+lTx5cmVmZqbSpEmj6tatq86fP69b19fXVwH46r/Px3ouLi5Gx0BfsmfPHlWtWjVDubO3t1fFixdXCxcupHmL+GP8fKw1efJkVahQIV3emzRpou7cuWN0f+fOnVPly5dXtra2yt7eXtWsWZPay5UrV6ry5curVKlSKTMzM5UsWTJVvnx5tXnzZl26yMhI1blzZ5UrVy6VJEkSZW5urlxcXFSLFi105+KfiG8vEsI/jQ/iefXqlRo1apTy8vJSadKkMeS/bNmyVD7/qa2Kv9Zfzr9+rS9SSqlFixbpxiPx20/ofbN79+6pWrVqqSRJkhhit9OnT+vSxI+TPp+DOHv2rKpevbqhbbSzs1MlSpT4aj3/kn86pi+JH+8n5PomZLv/dA3+aVv3799XAwYMUEWKFFFOTk7KzMxMOTo6qqpVq6r9+/fr1o2vN1/79+Wce0LKY2K2+bVyEBcXp6ZOnaoyZ85siH/++usvFRUVZUgTf72/9u/LuajEtLevXr1Svr6+Knny5MrS0lIVLlxY7dq1i9L9iPhp48aNKk+ePMrS0lKlS5eOjlOpv6/x144zIW1p/Ln6/NwnBE2pbzxaIgiC8IPx8/PDunXr8OHDh9+dFUH4Kq6ursiRIwe2bdv2u7MiCP/1XLx4EXnz5sWyZcvQuHHj350dQRAEQRCEf0VISAiSJUuGYcOGoV+/fr87O4IgCILwTVxdXeHn52f0l8rCfz+LFi1C8+bNv/lLcuE/g4MHD6JMmTJ48OABXF1df3d2BMHAw4cP4ebmpnvzS0L4Me8NFQRBEARBEAT8/a2xL5k0aRJMTExQqlSp35AjQRAEQRCE7+drsQ2ARE3ACYIgCIIgCIIg/G/E7HdnQBAEQRAEQfjfw5gxY3Du3DmUKVMGZmZm2LlzJ3bu3InWrVsjffr0vzt7giAIgiAIiWL16tVYtGgRqlSpAjs7Oxw9ehQrV65ExYoVUbx48d+dPUEQBEEQBEEQhN+KPGwgCIIgCIIg/DCKFSuGvXv3YujQofjw4QMyZMiAQYMGySuGBUEQBEH4ryRXrlwwMzPDmDFj8P79e6RKlQqdO3fGsGHDfnfWBEEQBEEQBEEQfjuako+4CIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQCEx+dwYEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQfjvQh42EARBEARBEARBEARBEARBEARBEARBEAQhUZglNKEGjVykxl9gsEzH6956Opqcw7mx5PZtCyY3vtorcvfKltMth9a+QmlMF3B+Y418MeJ2ytPkAt8UJhdgX5Rc+fDS5MLPjST3uLEtuefv15Crn6sKuYy7SGHEbD62YW3ek1sAe3L3VQ7d8lbtHaWpgefk5ht5LMU/jp0xYsHnPepBFLmcTSzJ3T36+77yoT0LJ3crHV/LLFN43Q+d+BoF54kl5zLJlNyxdN3I/RE5XrcclonLrbIoRE7TSpCzxkJyK+MmkPtkMp1c/Yd8XAjLwvvNfpvzZ1KfXLa4w+RuFOHy59yeC2C1VdvIuW30IDeyl769eJ8rG+dtyn5yNpe4jIbjDLnXsXzep5qGkbsaxcc6w+INOWfVjNyvQivI7k8+ZEwc1Jzcq0Fcrhy78LF0mjyT3JTCXK+qXtaXtR0fO1Eav2qTyS3iYgE05z4CdzeRurGZy57nnMfkoqy4r7LoepK3V4zrd7aOMZyXBlyvWveyJjc1/Sfeb4dy5GaMCNQtt6+2ltKoI3U5H9z0AJGsxhhJVkxFkFt2wYrcrDlcTjCTy9OvglsM4E7x1OS048YKVgEy7y5zv5vsAscU5ZYEkut2y0e3HPmU445k4La/9MdV5N7YkEKKbBzyNcm9g9zAYV7kdr1ISq5TieO8k0n5eb/duc7vjwkgl+s09yUxhW6RM/c6yPu9UUa//KoyJXnVeye5/C15U4/blCe3NNCdnK/NbHJqGm8v17C+5C7fH84JfyGakfgt8wGOt5NUuUDuRBSfHzNlTq6xkf1muc4ubSn9css3RuK+u6tJjcnEexi1lNvX4KbcvmrHz5GzKsZld5CR/q83N/9QjTleOpqiHbmPd/KQq3rrErmXObm+pbj5iJzm5qDPB2cNfpXSkjPZ/YLcQo1j1Ibv+dytsF9JTh3kfjLZyMHkQnZzXP2rSAnuU99o3G9pRrL41EgZSmukK9e2sIsxMvYMnajf3uQJmynNkB5cp+bW4HGiT2befrJozu+s1P7kDl7itvlTYx6LbQzgvlzz4esbYM4nYFb0UXLRTzh/DukbcDrFZa3pF9Vl4B0eH0yuxQ1cyTt8DNtNOaaam8zIOMfxKaky4Hq1FgfIpVBlyP0qUlRlV6kCB3orzMaTe98hmpyvTRdyn3JyTPehMPcbFYt46pb3NOZy+9CWY5Qcm7ii7axACsjK5UC7Xp2c/VtO16/pJnK9tw4gd92kD7lsf2YipyZyTPZ+a0lyQYNIIWpXLnKdF1/WLe/t3prSjDnNcX7GfffIPforObkevTgfsT34+px7yq5dJe6rTj3nOalfRRoOaTG/ahC5VSm4vYkJ5ri84yVuD4pd5vFoKh+Oix6rGrrlajbcPvRRHOOWedSenEN3rgeh0/KQK5cqB7l9KZaRg6uRds6L4wJtIs8jQXHspTTuD9yMzB3bwpVcWSPjx7T99fHo+6GcjYsuXB4PbK5DbryWm1y7PFxHm9hwgDrlDu+3a0VuWBdd3c4JfxFuRtyDXinJaROMzM0V5/Yge9UR5EJz87i4zvnL5KaEftEeTORykfY0l4tnrzjGf1mxHrk7itetRgYIOMjjrtG+PEePJ3zdtCscZ6nZd8nlGM9tfxtzjtNdmvMc46lyvckdnK5viGefbkRpysXNIpdB8fzqbM2TXN6TRup8Xq5D9rl4TiZLs0Xkzvb9jXP04D5Q2c8h1zjsBLl8e5uQazvrPjnbxbxfZcfn0HKi/jxE9uP1NJMMLD9wWbk1j5NlbsnzBa/teC6o3gceRxysb+SaJzOSvwsZyW04xeek9lu+5ssnpiE3uelLcqczZya365i+TfJ+VIrSRDQ6RA5G+pZMu7KSq9qZjyH5Le5LB/aeSC5pQZ4nCK2zyEheBEH4GvJmA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoU8bCAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIQqKQhw0EQRAEQRAEQRAEQRAEQRAEQRAEQRAEQUgU8rCBIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAiJwiyhCZudZGeZy5Kc1i6SXOb5buRy5r1KLktIGnINmoeT63ngim5Z5bXifGj8HMXGXqRgla4QuQCvPOQ2LzpObq+PB7nS6WPJPTuokbuTIge5VclXkfuklpLTPrQiN6m1IzlztOB1H+nPeyNFSaC0v8ile1uOnH/6suTsQniDJ03ykeuIDuTuZeS8/E5ypK1ILktOLgdIwsdsq7FzWbOZ173LqpjFBHLv973Wi41PKU1hcLkAhpIZqXqS63fpCblc4HK7KzW7hW25GVF2nBNt02pO94nP09tTweQaVuhPburequQeRWYh1ztopF7483nSztdil4bzNtB8J7kxs1+RM1dcN44e3c/roi65SWhG7lex5xTnZ6ozN/6XmnJ5eViNt9doO1/LvRjHCQMHktqe/4vzH25LabRtU8gpi3rk9hxcQ65iGm9et8QmclOah5Lr7GFPbruJOblb5dLxPkoPJ4c05Ulp2yLIReU3Uv8+XCbXLra7bjlyelZKY7aHy3dsFc6axUauzLccncnV6MZl3vrZPXLa6szk1MzmvONfRFOcJVcw9UtyW9p+IJdixhFyz7TZ5FLBj1xqL85L5adftP22QyjN0eBT5NTc3byxKpVI9bgXQ255nk7kDqfnvi9OcSwWNXYFuRldOB7b/OI9uVxpON7Tzq8kd08Z6TfwJzloD74Q3Llqo7l/+Og/kdPtH0PujXaFnIq7QC5ga15yix9wLPa7CbzCbUm56pyuSEduJ8xGcjpUG0Rqx23ut4fWWEvOPW993bIyEntoHG6jR33O24lU93ndP+PIbfo4nVzNcAveSZMoUo17cLLSzdqTezehIbnj2iVyp225jGf8wP3YezMuv6XL689BzL7TlKaob2Fy9TvxuVvgzOe9ZP4T5HA7BanDbhxDdd4TwOv+Rjq/+ETO1EhZW1WQz03alRznbloRTS7flhLkUlznfYSOGa9bHvGS64XqvIxcpU41yBXNuYAcjMSvQduOkpsO7mNS7DtPrv2GO5w/xQ3BtU825JJYpeX8HeX8dXzO5+mQLedl/kdv3fIca97WyLu8rYAs7C778jFs/YvH8dVMbpFrHLePXOb7HMtxFPzrCJ7Gx2xXjuvljgcPyZ1px3M32yNGkLM8xX1tlqlcTsO/qFf7P3J8eLcq17PMfEqBqL2kNIuC5Ewj+Vq+T85xRssdIeTUpe28j3zW5F4oLn+3R3EclDn0I7mXyXkuyHx9a3JbY/X9y8Wb2SlN8vJ8ojI78ThiUW6O72YWu0HucjKOjbJu70qukJH47neSo0oDclWxgVzoIJ5zG7KK60vZ3LyPebY8d9iy8iZyqcrqY4oz9jyHlGlZP3LawF3kUozm9hup3Unty8fubTAHd8mNzFNpLZ/zPhTXU3X8LbnRi/ncvXjlTS7nMT6O1nt4t1mG6ftr86FG5npH8T6V/xze2PlBpFaqZ+SepeRVk6fNxW4wjwd+J+Xe8fis6N43nDCE+6wTN/3J2W5sR87BiscxZ+dwu4HxnrrFPZ98KIkPTyEgahzHvFuMzFHHaDw+ty2dityOA3z81Sry/MvmCZyZ9Kt5eyFrOBa50rEMOS1/EnI+rblOjm/C8/vD+1bQLeeYuYTS3LN9QS7jbZ6n25eK531KXeax0Bou3mh1cAu5Qy053e9kgppLrscHngtxjDtHrlUZPmiTigfImY/9cl4BcPHbRC7FBX1fefkul9GrjjznkV2rQw6K+yrNyLRpNp6Swofbo3jdLFyJ6qVnZ/8mjJwn+D5f95Lcvja+zuPOKX4ZyJ3qd5tclb1jdcsRlzg+gYWROYHq08gVPsRtzaEqW8m9XepAbuYorsvBnbguw8glEwTh68ibDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIlCHjYQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBCFRyMMGgiAIgiAIgiAIgiAIgiAIgiAIgiAIgiAkCrOEJpxyqCS5xZd3kGsXcIrchHKlyF3J6UJOu8b7tVZu5NRuTS9MUlEaj6qW5Lp/uR6Al3luk/uwxJ5ceGtet2+PIuQOdefnN8IyuZKrcDsnuaCHS8nt08aRG5F9LbnF6SaRu90igByK649jOR8WpsyyJtcx6XBOGLWLlHcgb7B4xRHkDpVrQS7PvhDex2/kCo6T292bz+mf9fmY33nz9vKo2eTa14kj19Caz/WqjI66ZdXckdKcGjCUnHaiBLnOGtfHps+7kXtZfQC5EW1CyfXYk5TcuFiuB8rIsW6y4XNXa+xhcmn2cj091r01ubIbuN7jgdLnIyCCkmQZMYfckWv5yTntPM/b38/H4HhzHbmbM0+SCwnjNvR3UmHuU3bP3pPTTPiYc9ry9g4pdjnTc7dzlZscWG7fq1u+dzyc0ih/zkdc9BpyHvs5I5oLr6tucL+0ozPX2721epJb/zg3uS7l/ySHIb6kTI7+xXlpxPkr2a0xOef6nJdeKpduudVzzkbHfOzs2nBCLfUHcm5chaAmxJAzV+bk7tnt4ZV/I34oQG7Aek6nFbYjt0HjuKiHfQ9yr8K57RubiQt9zK50umW38S8ozfklD8j160AKazuyvKvSktunBZIb4FKO3PH0nK79vabkXoLP08SGnTiDz7istQrvRy405XRyWr+35CYrfbvS6Q3X+cWKr7XNemNtcFtWxZKQ6mayhPPWgPc700g7+LsZEsbucu4p5DoM5D418/Yocpe28fYcxnB7+iLcjxPuNRKIfomR7WslOVaPK8HlL8ebl+RSrzPSJ9Rcwe4mxwZvTHKRO4j05DpNzkxuoeL9doAHubD79ckpE143NLC/btlsUktK03our+eiOEbLmGcTuaPTi5HTrnB8g5bLSXVRgzkdfl9lGMDhMFSz5uTsDywg9yxXNLmWuQuTe+HJ8VJTr1jeR0/9+a/Z00gdUBxb7x66hdz6czfI5bPgcjvMdBS5eZOGkSuvOC8TNb5uBw6eIdeuJI9Zb5pxeVlvysf2IIoDi5haeTh/Pcbolk14KA6TCxyQLnDm8YtFhS7kqpfkxubPmunILSvmR25hiv+sxt6pGrvzRn5nEjaU2+pBppPIKefJ5D5oqcldszhHrsgkffua535ZSnPW/hO5cVVI4cbrCpy3p9wfD7FaRi5dFRtyxY5lIBeWL5jcYiNDzK4ed8lt2MOx+ouu3M5nMecx1470hcgtza2PK/1SfqQ0bhFG2pB9+0mtq3WInPukY+TmreAx9pmrvI9+qjzvF3uNuF/DVrcQcs8PWZGz9eL2YJyR/ikafMwPwzkGUim531haUb+cwWQMpYHfalIHIiuSK2PG8dl9zCSnFeK6HDSWA77wtX3IdfWpQ+6MkTZtU/Bccpvrc5nc9Yzr2g6Vg1xOU87f+6H6a2Y+nY8r3IXn5Kpd6EguxTZuV85V4+v68Q33X6qBBbnQAf9Z7fzxZNwXNzxXmtzJRhwE3Vn1mFzZ20Hk0mfh8rLoIcdFi1brz2toRR4nvn/emZxFBSPtV6OxpBq35jgmdA7PtWzrcoXckrl8v8Bk6ztyG4I4tlv4sh65mLo8L3XanWOP9OMLkuu66zQ5fNGGp2vD5cyjcVZylo2HkLtpyW1N0vzc96dZuJtczgE8xr48lcdHv5NWn3hsYmdk3j7fPZ4vX+HB5/WMJwcaK0vy/MDVcjyuG3Rbf92GmHHZW5+MFHC6FSm1imOCZj14DnxwjxByc2MbkhvYl+uVxaks5EzGVCf3h8ZzQY+ytSEXpjh271gxktyfoXnJZXXPpBfezpQGPnzyVI6+5Oy2ZCNXcRaP8S5d5fkci/3cf8VM9iFnBp7DFQTh68ibDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIlCHjYQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBCFRyMMGgiAIgiAIgiAIgiAIgiAIgiAIgiAIgiAkCrOEJhyW7gi5sbAjNxMbyFlNmUXueKFYciYXeb8pq2ssk/bWLWqP+lCSkPStyCXRipPr2nEbuYE50pJbb3qDXMFNfAy4Y8379enK6crzuXMJnU+uaNUV5M6MykPuao4AcnHwIPf2tn45e0Q6SnPd8gU52L4j1SvSgVyNio/JLSvxhLe3T7EzcqlhJNmvQjtckNyRtVyWrzfm8xWTsy+57hccyb06xweducg6covW9NeLbVMoDT50IpViHlfxKLwmNzHbAnIB+8LJLVs2jNy2GM7KQu05ucu5UpGrhgheuVMpUs+0MeTGqnnkno5bRO5Jef2xzeFmADdNHchpTbjwbVvB7UrZME53ybcLuSLdJpML5mYAv7PQawFlyMUFnCDXvB+vm2nYSXJL0Jvc4Bk3yTnb1yUX9VFfD7IZOS0XsrLr22EVuSzLjTQurqxe/xFELvhYD3IV/qxJTqVbTq5tx2bkbjtxmzs6bXNysaNXk6tZ0YXcinO5yXX+suqOoyTARFZV27B7tC4jubNtOR7AVe43owdxspQLOL9vuCr/Mly0CeQURpFzKHyWXLpDoeR2v+OTncyX29w5O++Rm1JQ//xnhav2lMYR3M4HRRUh12r1dHLRRqrBgkNu5BaXLExuwmBed2Yl3qCaVZ3cxINR5PIEjSD3usAOcoV37OQdX7YkFTJcv/w8dT1K02z6e3Kd1+8m93IVNzaWPY21y3yeZhS5Qq5/4FVyw8pxvfqVHNrD8WHO9B3IXfsjmNySGUnJlUj+J7n3f8wmV3Alt3X4IoZwrp2N0/TiOFpdG0su1m0fuQtnuH6cbVGW99HlEilXrSU5s5RcnrVSRoZTlbgteDd8P7lpjwqRi9M2k1uBC+QaYYtuuUIXjtE2dq1C7olZL3LWFQeQO+MWR849215yBQ5y2ZkYweuCh0i/jN4cguFRE459P7hwR/inC/cT92K5XgcsuEZu01RTcpeLFtMtN1aZKM3FJVvJIbUnKRNndrHzTpGrN600ucyduA0v25F3G3TLitysvzgOuFnaSLARvJGU6bwKvL2KvI/YPLc4Xe4v6mTUQEpTu9kczsekj6Ty7J5G7vD+7uQsd3D7djGc+4SW+ETOH3xcv4qg65xHdw+OPTb330XOerY7b9CH25LaedeTG8FTKzBrrZ/jaHi/MaVRYVyWtZ3cfis3rsztUnL8kEtxvPk4F+ft0bmU5Ma/5b7vbvLx5FYNTsH5m/+I3N5hOTldlU3kNKs35EIr6+PrmR2OUZrmwdy4Nv3UgNwSG46N4HGd3RQ+hk+qNjlrzCXHvdCvw/oBl2VzG55XmQILcgGoRc5PcZ3OG8kTHzbZeAz8sYZ+kJrmPscxhV/x9dhhupacfUUOwkMcO5PDoqaksjXi+CRrLO/jipGxz9u3XG7hE0aqZgs+fst13P5Eb+LYBh7cR9o3PaQX3q6Uxta6PW8L3H7jNMcisdU5b6bKyJjmII9VOAL+vQRpPHfTH9wvhljw/FrvQ4vJvZi4hNw4I+FcNR/e3vTn+nJ19ISRMfZibm9i3m4nlyuoKrmcPKxA7zk8KZVjMpepF9l4fGZuzmPAmEUHeSeFuZ9zXJeXXIFAjj3yluU+4riRSe846PvEKks4TXPfQ+TmLOMO1+ExjzUW8PQBShR4S87DgvugD424P7RryNv7VdjbHCe3AXwtL0zKQK5RCM9Rv7/Bcw0hBy+Ts+zwklzcG31bYjKsNKWJ/ovbm8iCfH0LVOSx2cVue8g1Gs9x5fAlPM9+yY/328SrHbl0Y3g8XbE2x70ad6/Yu4wns7u5cR8xcQrfI6xTRx/zOfbNTGne1OF7UNjB8UlM1uTkTMP5fOJNdlJ1Hc+Qs1KNeF1BEBKFvNlAEARBEARBEARBEARBEARBEARBEARBEIREIQ8bCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQKORhA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoU8bCAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIQqIwS2jCsQPiyGnISc4Z7uTKZFtG7lb4M3I3y7cl5+Hfj/e7f7hu2T/Dfkqj2h3j9c7z4fZKQQqTXGLI5Xjektzz1B3JbTFpQq5np3HkHIf4kKue04ncds+N5DLUa0RuTfms5Hbk3kxOe5xNtxybpSCl2fziCblNyUkBJe1JNdz9iVzYggbkfNGanFmHYkZ2MseI+zV0XXqe3Mg0vcilV6nIOcYNIXfLdDy5II9X5IbdOUouv9Kf604oSWmGal7kYucdJHf6HSkM3TCfnHV+W3IR7t7kMrbi7T1+wPu1da1PLtsWft5JTe1Lzr1oRXIj6sSSM90QSq5ks0x6YcHtRevyvN60WqSQ9dY8cna27GJPD2NXiNuGdkuW8k64afxlXMs0ktyQw+wWFue6327Ye3IzNS5D0+s0JZcsSCNX9nVNvbjL5y9oL/cZq15xOdNWpCVXMCe3c7lncdt6gpOheRVTciOxj9yaKXvJFZ7Kx3HJh11gt/LkqnUrTq77fCN982h9vWrIm8LTiaXJHRnejNzDftxWa1f5fPaJVeSCCvB+39S4zxIZjbhfxOlupOZ04GRDKqcj1+CFBbn7B3ndMxm5b7d9ze31oKb6c2gzuC6lGdllELm5JgPJaXX5egyuz/1G4Yfc32TyakxuRieuozNcSEGryvst6MP9XEdvXtepdxC5kHUzySXN25X3222wbvnPsiGUZs7a6bz9MjvJmR7iY/XNTwqTO7qSa9OFz6fpRCPluxyrX4nKf4aclp374w7l+XpOsOPtnXvHbbG20Jqc+eYwcj7RO3TLW8atoTTq9Ure6dmFpMpYcn0+nIcD2GKtDvA+Fq4lt8g/E7mkR5ZwXtYWYdeiOim/otz/Na9kJFYfwuML5VeJnMlV/fZsNd5+6I4K5DIezkbulvspclY4QW7YEN7evWQ89oO5kfPExemXsbt6T3KjXnFdz36eM2mSg8ei5QM5ftt58DK5WpfKkNN6VNYtq+kbKM2LHJ7kytUhhWMvDpGzfT2UnKctB+trknCba14gKblZypzc0XMTyI09x+cusOsucjZLOMCeYs7tz8dqvL0FT/XLl9yfU5rIgC7kdp/luHWqC/eJC3Nw43yq5TVOZ8mx//ukHC9BcYz2q1BG4tL8rt3JVb3Pdb9Va0tyq4qtIOe0kcfKBTYu5swc0MeXagDXPc3mLjmTeYd5W/c5NuqkRZB7e5L7JZMrvLmM42+RU05cXjT8Sc5pB88ZpVnWg9yQDNyH7e/L9UAt5okpy3frdMuld/GYOINrMLkHqXlO5sJ27vtjTbaQK3B2Kjmr+tx/q7XryeH3FXkc03jnHazOkQsozWPbiwesyOXTuB6gLqdTRuZCtH1flKt6B3lTpbmtvq/1J7cs42lyWfZwHVJx3H5fNuG5tE2dz5I7dMuB3AnwPg6v53a5yERvck0ded618vEQcnXMua5NdNfHaH9qgyiNUrXJaZPSkKs/jRRMs7CL3faAt/eS9wGtD7vfGNu4X+b4q2iuUeTaRbUnNyUr93c5Z94jd2kMz6svcchHbtB6/fz29Uc8v3Hahed786sqvH2NT+r1Sb7kOikelOzXeLwXbqRdsvHn2N2sHZf54VaB5NrV53p6pRyX5dJXdpCzasuZsZ2rjzMH8HQoeq3muYNT4Xw/JmkTnnPef5Gv66G0fD5NzKLIDbcNJ8ct6K+jfGMe/9VaVopc7hIcu55eyXN4E0y4b6tzkq/vhPocMzt4HtEtq5iDlGZLDM+b1UjD5btW4Gpy5c058K/2kMtP9Byeyy69nuPjqLrpyfVqdoPc6MeO5JIW5fntts/+IpfypA05tTglOdfp+nPw5o6RvtWHj7Wc4vGlicNccs4TeE7mubsfubW1XpODxvcLoIzcHBAE4avImw0EQRAEQRAEQRAEQRAEQRAEQRAEQRAEQUgU8rCBIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAiJQh42EARBEARBEARBEARBEARBEARBEARBEAQhUcjDBoIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgJApNKaUSlDKGVXSVI+Tq7DlP7sD15uTCmu0n1yu2NrlMf14i13r0a91ytmsOlOZaSBVyUalfk4uO4HS2QR/JIacXKY/+g8g1H6yR62u+h9zF3RXJdRjVmtf1nUPuSpZZ5HpNDSBXfGwYueN/TNIth45fzHnLfpdcKRdSCO9RlVxB023kbuTmvJnu52NY1P8FuSYqDe/4F7Eoris5P69F5DKGvCO39kozcvkjgngnhW+S6mb9gNyELi10y6r+fN7W/FOkjrwbRG7z9erk0s5pR66rJV+3/hhL7lSO8uT2Xp1IzvXlWnJdunchN21KY3I3S/I53nvtOrnYRnvJLVlhpltejTKUptTzOuRc06TnbWESOa0UN6EqNB+nu8xtoyo7ghwC+7L7ZRh5/iz9E0711IJc3OUD5E75csOx/kIRcmMHcXlJuXCybjnoEdcL7C5JKqgit18PJ3DbUsD/HLmkybzJhTV7RE4t5g7R7uZAcv1SLifnHfKYXNZF3M41b1+Q3EKnreR6HCUFkzm3dcv9u2fmfXIRRe49XJZNXbhPe1X4Frn317KQu+EcxTvpyGVHTU1YGPIzUBhJ7m4KPmaP4FbksiE3uaWxz8jZOI0j9+CeL7n3Q/V9aoOzeSnNy0NcbreVbEvu2JEb5PZ1uUxu8tr75GqfqElubj4+Brtg7iOP35tMbqo79zmT7icjlzwjxw9N2weT06ZzeSnUe5VuOWzUJEqTJTv3kZuiuJ6tvM7nuJF5JDnlWZTzduMMuYzPuDzdS/v7yjwATJrFeRrD3SwG+TwlVywFx6pRmTzINVdcFi6FXCBn2aSjbvnjCm7rTd9yP/TiLvdX1V+4kTvb0owcnvK1szBfQi56HMfv6FaZ1O1RseQye5YlV77WPnJ7U4wiV7kFxwF3G3YgF1avl255TB9KghP+PAabAM7HKWVHruZ4Hkfc787nPeX7OHKdHbmMTfr0G8t9OOcn/zXOz7FdvKrVDF43d0dOd6nkQnLdvf4g5++TWrf8dt1hSlNimgO5xWe5z/Er3YNcUI165KKf8jV/tGApuSJFV5JDfR4rNg/aTO7hgRzk2ntxOu9ULciZNz1GTmXnmF6boG+LL1u4U5pcZwfztlJxjKbAbb3J7RPkxrcJJ+dxYCi5tBrPKeRTa8j9MrZxuY1ry31+2ycZyWXEAHK99nJMN3YZt1/Ri7le9S30Rbt+wpXSYAXn91YzLnuucX7kGq3xJ7f+7RtyJut5/kEFciwTZbKKnMWt7ryuHY/jPn7iPqe/WxJybcDl6mMcn7sAf/156b6Yt1V3XXJyFcdGkMvwmGPZm8+4LG/STpNLWZ8Upqbk8XOHadxf/zKumrJbxP2z5XGecwy4vYFc8QJNyBUI4rFdkpd83VJ6eeqWH858Tmku3Awh9zEjO9+VKcjdXs7lzO0kX/NZ4HTtIjm2RmeuL0W6ZyLXrTaf4+dXOb7upArzPoxMM1TW+Bx3KZFHt5wvmOteRBluL1ymFuId+CQlVXN9AXLN/SeQ854/ibd3lzt/lSma0/0iYtRLcma9d5OLXcV95bNHPOedATyxYFl6BbmGB6+Qc8lqo1s+f537m1fR/cidtuD4JAd4PvAq+Bq9zsDX3Ckjj8W0uTx3qjxasgPP/5nc5z7y7ocx5Fxb8rUwTcNzp9FH+Rx3+mJ4ZLk/A6WZ2Pwi5w0819bCmvvv+S5G+q97NuTwbBqps5U2kitwnuekfhkaz3vMVjPJveMmAm0VjxOTduXYDZM+kHp+muMR57L6MbHmyP0NHPjc33XmsXTHbVyndnThcVjgoezkOpzneP6VCc9xaabryblcqUSu1vwQcm7jOE5/E851Y7zdPXJPFN+Hm6o56Zan8VQCytzmOjW7BvffsVs5PjUZzveqEO5MSivB1wK1uLNSv3MMKwj/hcibDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIlCHjYQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBCFRyMMGgiAIgiAIgiAIgiAIgiAIgiAIgiAIgiAkCk0ppRKSsKvWhtwkzOaEWaNJ7RtmSq5cbXZlt9Ykl3zfKnLl7ax1y22H8yEkb6eRi5vJ6Ryrc7rBGx3J3TStx+kwipx2sDs5dKxG6rB5FXIlzxt79uMCK3c+T9r9WHIzQ56RC2iiP16Vn49/xpA4ch3iHpJTTdkdnF2G3OGUpDAvwo5cftcP5DY+SFDx/Cl4Fefre+gYHx+0wqQOPOVk7mXNyJ2b+IZc0SpJyTUqoy8bzQ/yeVl1nPe5rRg7vuIAxrNt3fYdubDAYbzfauPJDd/M1/fwkYHkdqUeynnp3tpIBreSsQWfg/S4Re6klky37KCCePObnUl9yPCanF0HXnXqcW6jTqIZueRFuG28dHYTucPRXL9/FZ5GzukRjcuGk5F1VSC3XyHFeXvJ5uYjt6rCcHLTs1TWLdefWI7SBC7l6zb6whJyc663J3fM3Y/cLGcuj153/MmFJKtFrlwHbi+W1XMg52LKfWnUWVJABz7vISO57Ob/aye5e+b68qey83XQPrBLe8+CXFj4GXKhs5by9rZzO4D9jdjlXUFKnedkv46eRtwfZOysw8gVi+BysCddEnL7ut8kV74L99lZX+n7kpup2lIalao/uQXL25EbX346uXOPeHuw4LZKqxlJziqwNLnkqe6Sq72W63dQtVByLYdw+a52jrO39kIBcgMec1/yqnp13fLJ5Gkozc3dg8nZm/Uj55xlCjnXwIbkspIB7pf9yDJwspGUvY24X4kHmZiLd8iZ/9mRnNo/jdw6cLu2fU4Rcos6GAlUPvTVLVomcaMkn9oaKRzLkpOq/4bXrb6jD7mGrcLJXdySnlyB/D3Inav6hFzRS83JnX6yl1zuJRyXP/BtQs6tuZG6uoDHISm+iOjetn/I683nsrvU6gq5QiEcg+dnBc+OfI7PDOf9DkzDY4nBRgPQ30i4D6kjtqfJlazgR65ySY4XyncKINctGceSedVG3fKbnnyuno7hgcSHUenI3eXijdxpHpPb/NyWXM1aKXjljTwu0UbyQO5eNi6PHiVjyM2fvJGc325u198O4FgjsPwscpcs9HVj+GiOZW5eHkCu/gqObw6/4/40STJSqFuOr8+6nVyYVVcbXnmakT7hVzHPhd3SF6Rmn+Y8VhlUiVz/9TwWWzSTy9VqG74mPbLqx89JN5yiNMfbcz4yBZLC6+F87gtM5HRXb3N9mfmc45ZS9azIuRsZD6mZfD7fBjwg18aE57jWGgkY3gVeIpcszStyTf7Qx5+V6tahNE1XGJkv2c8qD3jcdDEHlwkjIS9mHubY0M+M2zc483n/Zbhz2RifJAO57rd4zKrOPOTt1WalLY4g936lNblcKfT9y+m8aymNY3VuC3uiAbmx2jFyamwFzlxPLmhVL58gV3fGJHIxM3mMWcBIscpjpB8/AR7bFb3HsccWI9entJG65rJH379ElclGaT6acUyvjMxQaBqXUWVkn0jP82+Yx31aSEWOHx14zV+G//Dt5FKf4fZ15KZP5NSc2+QW2lUnV2c8x/1JZnF5zpx/j245WyiPTzcl60VO056TO7Y4M7liu7jPxqaZ7Gbx3LvWjNuBIRq3/W9H7yJXeRaPZ0oHcbmy+Mjnzrogx/gRJ4tz/h58MWlbjOMO9fIhufVpOCavc/clOdiU5n0O53qLfjdIjQ1uTK67kfDxV/FGe08uVnH9TaVxOYDaxqobJzPp/BenO8dzRha19PM+0We43VOFVpLTcI3cG1WRXHh+ns+YuZKPq+gjHgx0qZCH3AON5672qnnkKoRyLFJ7OM/XThvD22t+hdPt3paLnMqkb1eePOTx9d0eB8mVqcj3Y+z28FzTs5M8J2d+7TC5apO3kGtyeRw5nhEWBOGfkDcbCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQKORhA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoU8bCAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIQqKQhw0EQRAEQRAEQRAEQRAEQRAEQRAEQRAEQUgUZglNOEnNIfeh2Uxy15f0JueNi+Tmv8xB7sDuzLzfnlbk6qfWL8c6aJSmfUgrcntmcrpKW2+Qa2jmSU5rxuu6lhpLLmL0UHKmuZzInVj3jhxik5F6/ZqT7RrJDtVYBTydSi56ttItmycxcu582pDbas/XX7NR5MzinpGLiWhGTqXcSy5fclNyv5P11fqQG7eEj7kX+ILEpV1K7pHpW3INt84jt2vyIs7LrlW65c5j+Lpt2EUK01mhuHIg52qknEW3fkFuVadO5JTDcnI16qYmF4oe5Cqv5/3uKjuO3MssfJ5K2J0ml67TYXLXVYhuuZhmzjt14fp4t80wcnm29yeXPSlvrqNWlpxWez+5+9FGKu5vpBfqkrPqvI5c2KTN5LQDfuS6FS7HO7k5iNSx+bXIHc4SqRdLLSlN+1L+5Ez6ct2IU1Hk9qznPmiy921yIYv4+FXvPeQQ1YDUpX5ryN3v9IZcujWO5N5qfC3OruB080a1Jme1T99OvfIqSWkO4QG5kv0iyaV8Uocc/pxFSo3pSC69aRdyT+ql4O0h2Ij7VXC7hN0nSX3Y4E3OtATX6TNPue6b9EvD6cK4L8n4rohuuUi5vyjNtMu83gxubnHDyHFZnwwnp3JXIDdp1nNOZx1CTuuVnNzcalvItcZicnatkpArcuQ9uWNoTm7EG44z3qbUt+Fa4Q2U5syLFuQ6aNPInbLlmAV82vFuegZyrXZxWZ46jeNiqw68vV/JLtwhNx9ryV1fkJ6cmTaB3I6dXcldaLOTnC2MjCUs7HTLFdtSEkyPm0uu+Rtuh2p+4HULvk9FrumzTeRWpHpFTsXyhV9doCm5yJsc02oat6fderqQS6547NMvaUNyNtn4OI520y+fuMTn99ynY+SaNOb+pajGAVncHWtyg59w35EmLdfJpuDxG9Qndr+IyUbcVTtuw0e4+ZCbtJ/HdmVdeXtxR3zJaSorOaW665ZDx/L4J+lYrqMaeFth6iY5CyPtVXAyLht4x2NlaNxH+9jdInc0jOObWO0gueeK+4ltf80ntz+UY7fw13wgs9Ppz9Wff3lQmrBoL3LTx3GZT5KsOLmGRtrG1hd5XmB4z27kTk7nUVcR7mJ+GaM2PCbXuzKf0zKH+dynL1uZXOmAtORuOWQi12oOl+enqfXxequIg5TmwIsYcq+q7iOXNGMcud5OfAw+Rdn5XeCxBKbxOVEdeV1jpGjH7TeseHvu1yuSy67lJjfeSLCxLHy7brnbXR7HppvD+R3MOcMjHoIYq/JY+CkluT9v3yXnbpuNHI84fiFGYqtu2iNybbpyxQzo+pBctVtGfpc1kftF+8kbyc25Ul23PCJXYUrT1/MpOXfTxuSaVh1CLrLDJnJWp0jhXQ4uU9fW9iPnwNUKuQ5lJHe48n1yJcZz/6LcGpHrxGEMYg/zPNq7LPr+hUfnQOYc3KZr9znuLKGMTHIhF6viPAbR2i0jp3i6FjhjxP0iTrbnOZT9m3jeo309vpZj+3I8NzSY53iOvOa2f1ZBbvsrfHLVLU/n7h8m4DI/HDyXWHwj36YIGcHzcNotHjQkbWYkng9fyOtG8xhT9SrA6e7bkAPced24EHI2vjwfr57vYOf2RT1ox7Eoug0n5WNjwenerODt2xiZc7Quw255KKkeTXgc311xvPurSKl4nK5p3M6lieT5nHBsI/d+NO+jYmF7cn/E8FzfmFr6+LhL4yy8sYPnSR24xveHUt7nfaoHpci9z8rzkCZxA8nds+L7XOeKcBAQO7YduT6rOaa4cSCa3B9Hx5CzK8nn3aoVz6f699F3WAtMp1Ca9Bl4fvGvQO6/w2tw5/fXFu5wphXl86my8nmKAfe5wAAjThCEryFvNhAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEIVHIwwaCIAiCIAiCIAiCIAiCIAiCIAiCIAiCICQKedhAEARBEARBEARBEARBEARBEARBEARBEIREoSmljHzZ0UjCMHZZ7Vlerx/CCbcVZfecv5Fe0YG/h7TnQz1yMbYzdMvmD/jbaehVhNTUbOwyD95OrlwTZ3Jmy4x83+VadXZG9rEN/J3k6l1IQfGnk5GHP/WLgbM7kwucMYncjIqleR++B3XL+RV/j+f8ES4S4Xz4+BDG6zrZ8XctQzX+nujVzPztoqjbL8iVUVV4x78I7cIictPLcFluH8rfZuKvTgLR54ycr/wLyL1fyt8PQxP9uhP4c/P405y/jXfo/jByG5tzPdu61I1c+bf1yQ1W/L2zOnn4uDpf4gLTweIouYad+AOBU8Zy+Wt7kb+2OyOvkW8ptTTy7fc/9d+29NvDV+dwF65T5oq/y+Wj8bUeUYad6svfqsIp/r73I1v+Hp1LF171V6GBr6XSrpBrqLgArsz9kjd4mb9lmg38ne3risuuyS4H3bJHHv5eYJ003PYPV7bkNI37ERTnbxA7HvMntxn8jcP9z9ntPcdlL6g1l2U+UuCvgxHkoqrwN6I33MtPbo3iPG+A/ruT5aryPgvu4P5rdFH+HjS8+VtsR/gzwKj8gPvND1cPk/O4y994vG2eoDDkp6CN4TLv0IvbqhdL+NvOVjXLkgt8xm3J+2xvydWaYeR7g4FftAdtuM241pRPfrZGXJaDjnN9dDRy4Rpb8DchN03jmCVcO0EObbgMaQX426vIsJRU082jyP2xmb8Z39h7BjlVgfuwiYv1H1Et7RxCaZal5+9QNnzSh1yWoXPJ2U/k7xSO6LSEXI++3H9Xs4gltzthofdPQ6vH5X7FWv7GaXXlSM7OyLehG7TmfdTQkpBrtOMcJ6zyRLc4V5WjJK01buttu3wkV2gSb775Yf42cTNv/pCses/fYA+O5uNPofH3Lms4cBzQ4x3Xy5Im/B3suFzct228xHXQ5/BBchki0+mW37wIoDT7unCf+4yrHy625ut60kg5DdxnpC3I0ZNUdBqO0cxUaV73F6FtZVfWyCdsM3Tjfmv0OU7oqPF3X00OXSRXUPH3f08v0vcnK3n4g0bRRtqm0Bbk0h/mPtXuFrdNrZfuI1dlBvdh2h7+ruqwU3ys/Xg4AFWMv626uyl/r3jvB17XLB2XtVFbOB1K6Y+3R6ZdlGS0F9dlkwUe5J6ajyeXg5t6fIzZQC4qoxcndDZnd4K/wfurWD6J6/QyE/4e/K7Oz8hpisdnM8EF1V+7Q273JI65q3XRlwNN7aU00Faza83fD1dz+JrXj+D5gj+t+dznDuL44Xoh7vvy8TQVDk/nGGpQ4xTkzIyMQ4rGcptuY8b1alzdmuSCblvpRR/+RrdTg4fkQpCG3HBnjrPqe3AcuO0gKew/mZNczuZryQ28YeSb1b8IDdzHKo2/N26dlwdGEReMbNDIPNk4VihrpK80e65PWJCnZDD9Ha8Xkf8BuY7ZcvHKruVJrc6ejly99w3J7T7F/XP3XrvJFeOhHYZy9lBneGVyB2vuJOdwjcvfs+q1ySWdoj930TzcwDZbPne1nPgY+jzgMdgoI6PxEzV5e3k2mZKz9uG2Ua37fTF9Mh6mI2R9JXKbi3chV3Mit5vKvCW5xVHzyPm7lCDXc41+/JyUp8WxHTy5PQAclObozec0zcik5DKZdyfXKKYQufZTeK4ldaf75Iq59ycX0JlP8l63DuTGVOc8pzYytwYuVhiyUN8AbW7SldKcNeHjb4MQcnNwkJzPa55TWO80hdyaMdxe1u25iRx+4zjWyBk1SlqN27nSKhk5b43nvH3MOHZHjC+pwVP1DfugEXMozd4Xs8ktNjtLbulpHiOrovPJNY5cTq6O4jmOyxqfqR6VeK7Pv+11cuPecRtp7VeanONMIxMAPBTA60U8X5n8kf68nO3Hsc0ijYPy8u2iyB2Y0YPcdLWK3BiN52u5JwVCdwSSK1OZ5ycEQfg68mYDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShTxsIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCopCHDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCInCLKEJlf0BcppWltykw4pc223PyFk5cLrzbQuTm2ZXl9yAKyn1ecsxmNIsaZKMnO/HruQC1Qhy7x59IIfcbUh1rFaFXNT9B+QmPnMkN2XyenKd8/iQG9KIz1OdloGcv/UaqRwW3chpvR10y6peSU4z0pfcA+fFvP3kTch1X+NOruTsY+SqDz9MbgzqkCtD5hfSujmpT7aDON2dnqTeOFqTS8WXCCs3+5M7E8P7Lbj3tW7ZLpTL1IpDDck1zuhGLvL5WXKTKiwj1/y1DbmW93qRO+7HZbRcl0hyx6YVJOfhcpnc5LmkoLXmfcysfIec4tMOLZs+LyUQS2m891qQmxg7iVy3Q3wRl5XivI2cbUfuVFwXctsHZiUX3aUxuV9FlRLsPNbkJLccfMw4l5LdRG5fPHs2SlBe4ky768VJLj9rN/C50vq/ITf5owe5zuDykyMJrzvSLQm51jf2ketfnfdRo8UMcosGtSM3PL0nuV3dH5FrMHIeuZ1oT24q9P3aId4lArezG32BnRo9hty5QXz9l4Q/J1d76XByt7cYKTvc9P86wliNGcllNGdTvh47cZBcuSQcK82ryG3OCiNtmiqvj5WaLktLaTpMrkjuwMdd5Mb24oZ0Zmx5chcmsqs+NA25dSO4Lvv4cvsdfSEXuXzp+Fgrln1K7t2pPOTU1ZXk3LO3JFemdZBu+U9c4211iSE3uRvHO879+VjtVQS5vlozctrC3OTS4BK534332qTkZpxJRa6hZskrp/pEalUbrsQOczhGbNSL20nTL4YiUbFcXpzUdHLeGsc8f71fQm5okr7kYMXjgcy3uTyHhWchd7nOZHITRg4ll6kFH3+F9+fIDbLn8uGT2kj7UNKKXGyc/tgGdeTYa7sdb2v4Ko5lihnp1wPzViPneqEouZdv7pPTOLwxFjn8MgIv3iJXpgb3vSY4Si4jXpHrb6RNaKvtJhcSV5xc9uaZdMuvcJfSzFOryGX9g9sc6zOkkPfaMHJHup8gp/a0Jrc/ciq5OZYLydVVHPNdAJ/PZ9pecuNVZs5LqDnnxXkRubFfnKudRprXNBVukss7sxi5lgW4/6tVkPO7MJbri/a4Abm3tWeT49mIX0f1B9x+70jF11f5JOeVeQgI7SO7Ngglt6Qzj22rl3TSLa9/04HSrL4zmtxAf25vYv34og/l5hEvxr8n55mCr+WR3Dt45U2VSTlt59jowOv85BTqkdPq2fI+UJrMIV8+8VpVfR8Z/p4D1+GKr3Xrh9zi9ih0j1zHKZxuB7geNK7O8efANVznf2dLX3FhMLkp+W6QG9OnKrnmXYzU/XHryC0fzPFOB/C8lnVa/dyKAs9pxk7m8miW2Ztcx+xTyG3tx2V058CO5OoEcLp39lxvr3J4AlWT27lhQ7hvOlrNSAygGZlPXcMxcu+zceTuxBzXLWfCIEpTwsjmT7XhPjhgDs/1jlRcRm9qfC2sNR6HfRzE89q/k3ep+pBbfeoguRoLeayo1CR2E2qRq2/D8w++ETzHsXugfs7oD/A5naqyk6sQzdfjFU/NQTPjuXJEc7+xu2Q4ucEdeR+vO3C5ddSukpu7iccbS8B9mI0qQI5bJKAptpDrUc5Pt3ygLk/KmazjeZV1zjyeQ3G+1uuNjMVLIQ+5egN4jG0ktPut8fybK9zhZ83F4x+/DpzLYXE8N778zCJy0wryeLJQ99LkBnXQt1+WlqcpzaT2PL5+F8DtXngeLj/RPA2HOTW5r3qgOMbwMXKVBl2yJ9fHay05l6QryJlE8ZxMXFvO3yUj8465ivH1ubhF314U1bitVpF8u7Jmdr42BUvxvRFN49p3oN9qcvmH1yeXsQqPD+4Z6TcEQfg68mYDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShTxsIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCopCHDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCInCLKEJz4SXIfdi3GBy2U/xul3P7CRXKvs8csX28sresCP3eks63XKznJyPOUhFbsKV6uQq5WlDrpN6RS6XzSZyR++W5O09qEbOJiwfuc4vfcg1TaXIdT2fmdyAPrfJZW/H6+a/k4ncoaGhevGhI6WBJ29/arFb5PxGdyE3rtZkcmPxjFyn5LzulPvrOS84bMT9Is7wOX1q24jTrePja9KOz/1eE95e1aHHyWln/uJ9aKd1i+qjHyVpZXOIXI3pG8j91f4quVRJB5K7EepFrqI6SG77ykXkyt3j/Hl0GEBO8+RrPgx5yOVW28hN0bbw9v46QA4vi+sWizQ+ymkG5ySlKmjkbm43Jfc49gK5vlk+kAsq1YNcyuaNOS+/kd0r55NTk16TK+zFbfXVVOXJPerZnNzGd7zfY+BzXaKivr7E5KtBaep+4PJT9XYEuXPDYsl1NrLPg+A6und2NLkKp/KQU2G8vbT2pPB8/ixe98Ujcg6T5pILvdeKNxjFaskhfZ6PTuI0vUdakKubNxu5tpmfk/PMysfapSH3uWrmS3KlXDgvh+uw+2UMYeVf+zG51t6cLnhzMXLF1xwjd+sDP9eZzPQFueYe7rrl8JG8T/Oo3eSsYEXuT1wk1wMPyW3fkYZcw8dcD/K4tSTnY87t5nGN69+jUbPJobc1qT7vfMmdOR9HrnH2DORMMx3RLce51uJ9uhlpZypxf1h+dwPe519zyKWqEExOxaQgt/gR16HfTetYPsagAS2MpCxIJni3kbbOjfvGO47ryCXrwOs6P9XXD7P9/SmNqtiO3JSuf5DLWXUFufO3uVx1fjWNXA7XJORaaZHkbNaPIec06ya5Mgs4llYLuH4omJMbf3Q1OQ1NyMFllH5bj8tRkiZluT3LnqwzueYZOW61XtyB3NVCnA284euq7htJZ6SP/VVs6D+BXI3TU8hd3FqXXB5we13GyOX4qLgeeOIauZVZl+mW1ZERvDEjoao9N//40JHP6cVp3L+XQl9yzfwvkktjUYTc8iscj0zVqpDL92YBuXoqLTlEcR16MDaGnN9eB3LK44s+QblRmhfHP5FrhPbk6tzj+A7vuCzXfjKI3PxGTckla8ZtyO/k/WQ+D8sz2ZDL2szIb09mcNwMh9OkNCPtl6alI7fcX9+W1l7QkNLUSR1ILnmWG+RyXdpDLkDjtj+F4nUfcciDLC+5LH+sy+P9zOU5psUm3i86cznY1mMjuYInz5BzqsblVIN+HGaz/SClaVWhNbnxFTlr5jcvk9uRoj65+RWCyA3hKSng5e9r041RtSG3fXvP8Ph7W92u5AahAm8wisfpF0/xHJ5NbR4Xl0MW3fLm55y3ms48v4qVHHcszsD9c7m0K8mNjVpLbkApbtNGDGUHHp5C4yku3K+9ilz5tUYGvA/yklrl1pbcFB8uqLYZ9uqzlsNI56fxdSioeF7AoiC3Rw+e85xUQVwnpzryuBj9/7PKPHpz31l/N49ZJq3oRe6CVS5yi59w3U+bjufh1rzkdS/E6gOjjZl4W94at0FXjBQfWyPzNEkdOF3oPm6YTh0/zwk1Hp87ehbnZDcXknOaxrHNEq5q0GKPkMuSldv5+fl4jrV8Q325Kr2d5xfh4kDqRHkujyU+clxk2WEGucO7uG4ggvMbfOMEp/uN7MzJbeR7I9VyuHaX3MBpD8g1vpmMVw7g8ZpW30i7ad5Mt/gpejGv576PXM1PHHcV4KkRjPrEE6eVUnO7VPdVOK/8gPN7zY3HODlKcgxQdgvHuM8y8nl/l5Irb24Tvh8GJ46VWs7V34pUxbiN+mMRH8Pr7bz5mtFcAFyX8bplhucm92EGH7/t1f+wdl4Q/guRNxsIgiAIgiAIgiAIgiAIgiAIgiAIgiAIgpAo5GEDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShTxsIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCopCHDQRBEARBEARBEARBEARBEARBEARBEARBSBRmCU1Y6I9e5NS5MeTe2g4kF9p9I+9YZSUX+ekKuU75TpIzH5lZt2zVWqM0S5eRAvw3kDpX4ia5yk0Ok3vdvCQ510fNyI1ybU3O6WADcn/cDSPX1YYvh8mfJ8jdXbqEXKbpW8hFT3tOLvmdhrplK9d6lGb3CkXuD06GOIwiVyLNOXLpX8SSu7mCy8lqJw9y9Xm3v47pGUlN8CxHLupxbnI9cJXcyZgIckXSFic3WBtHbsOpF7plzboxpalditcrX30luQb91pFzGtuQ3LMZbcnt4aqGT/Wqkzs31UhCS1ZQB0kd1NzJXUzCZX7QVT53cU5cdjXfG/rl1eacjRRHeT3H/OQOeRhZ1zQVOVR+SSps2lhyH1ryqnZ8CL8Mh2aTyHU+OJTc6fFDyGXfOIvc+nxNyC317U+u+EjOS5iLvgyZ7XSlNCqVI7kd4HbE40YBclWTPuDtOR9i15zbqshjj8gVs+D+cGTunuRCep4nNyztanI7bnJfMs+jB7k/Ne6vChQurVteNfodpdnuze3yAzOut16Z+ZnEmZ6XyeVfmZPcnhIzyAWGtCcH/MZCr3G9vHasGrnSzWPIFQYX3NfVOVZwGLCW3PFuHBcVzXlNtzwnObeFW1OEkquZnutUwC5u05DqFan2F/jcH7NMTm4LuAxVVPPJhVnxbt/bcP07tpzzsqYn52VrOS6TQ7Q4ckFfPDtre5HLbcQezhsK9yOldpcg12dEd3J/We0l1/Ij78IXaYzs+Pfyuhq3174mWciV31KK3KY8vL0717i82XpPItd4Al/jafZddMvvIm5QmrkIJ9epdzS5RxNqkPszL5eh+qsek8va0Uj7P82LHBTHHmfb8HHln21k3S4WpDyXOpAbacJ9DOpznqe/1y9rGpdJ+48ZyIXV5bitSizHtwGPOPbKvrcT5w0c8wBGYqPfyPSnc8id3szXrfuSI+SK+HH87hu8jVw6R+4nTHNw+av2RRFX1bldfxnD60Us5UB6Ss4J5PJUJIUyfbjOjzWyjxVG+uPsFauQ86jFfRjuLyC1OmUfdmoar5uV48UG1Y6TKz2xg2654zmuFwu9+DqEh+8jp15xjFLDgbM2LBXn7ZR3C3KL8x4g58ub+2UMO8vXt/sbvr73B3Kc27kCd+Y3Qj7xTl7cJVXoMM/nbE+nr0ONJ86kNBYxD8lV9C5DbvalvuSW1Pcjd700x0ajgkih7U52Q/7g+aeBrzlegBPnb2prngu56h9Frkw1vj5xXjyWiE2rj8m0KqaUJjW4Tmm3eRwfnZsnyB4rPoYgXy7LmLuVXSVW4CmFX0bnAI4VqpWoRe7P1Tzn9vINx6VLJhk5r8UXk3NswPHIvgIhuuWOabiP7TjKyHkezXU0LadC8/U8nloe1ZWcVmoEufAYH3J5p54ll3PxX+Q2zLvNmWlZjNSaMnwt5hsZ7h1HNnIeZ/VlVxvP/ZepkZ/MVdDsye2EG7kYfCAXBFdyaSdzhk9pXG/T/cZx7LLZPDdQuDbHX+Erub/DdG4PPo0bTe7WMB4X357LY9Rhc4fp91mc52naanXJnb5VllzeK37kcposIndkZBdytR1tyaUL4ljp2kZu+65lnUhu8zmuQ7mmcLka48/zPocv8n2A8jFcXr7sSp08uH7fdn1PLnc5Pse97/Bc08d0HPffDOZK5FaLx7sqlPslJGX1q+jVhV13I3W1vxZJzvwCz2uO3NeZXLdZfG76cxiJxjb69lDbkJ3SxH3kca1J0x3ktCg+Bm9tM7nsaZ+Rm9iI+7kKrty21onhIGjDVp5X2e/Gc32+dbnu5vqDx1Z9FvGYIaoQH9vcL6Y6z3bltvVOPu797o/ga1jHyPxTaid2r8BtjdWKPOSSHblIjmfCBEH4J+TNBoIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgJAp52EAQBEEQBEEQBEEQBEEQBEEQBEEQBEEQhEQhDxsIgiAIgiAIgiAIgiAIgiAIgiAIgiAIgpAo5GEDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShVlCE94+WpBcIYwkZ/2hN7mrrQ+RKzi1ODn725vIFTk8lpxXzvm6ZeeuGqVJZaPITez6hpw2+DC5eUW7kzuNceSGue8iN8kkH7nlr6+Q25IihNyZG0/I5b3uTK6vxuf9+tpT5Kp3KkVOUyt0y3eyrqQ0D9fz+YxUfD7fYC+5Fto1ch6NppDrW8mK3OSnjcjBltWvQrV5QG5fh6LkJl5LTc5ctSd3tO8OckUvxJA7NjEzuYHbLujzVjgppUGzIqS2ZuhGztdqEDlV25PcgA3e5DrP49326NaLXNFsXF6KRfG6ajM7DVyvHmarTG7OkTPkBi3k7XW5oD+fRYIOcqIU/UjZBu0jdyJuJ7n7WllySd1vkrvS8h25HpvtOS8Jb5Z/OLkKc1vV9aA3ub6aF7mBIVnI9RjN5aDJfBt26cLJpcuir/zaiI+URuuVjRxGPyV1/TknW/CUy62G0uRUuCW5JOkiyb1K34Nc7yfchsyYuJ7clLxHyHVz5HOXNeoiuRxXcpKLzL5ftxwWVJrSjHjAfV/d6mXIPdkyhFyWzRvI3c7K+dhYvyG5oS1ukRvCp/iXsVzj/j43uA+MTTOR3CBVjFxoX96HdqMwuVkruc0t1tBDt6zmcPnOeMqB99mE9xnGoRjmHuHyiLXvSU2rsoecFsExIDR/UjvUfHIdNC7L6S7wPjI1vkRumAtfCycj1+dpttm65RalWlOaqe9nkHuXtis5TNhEatSfnEx9rEBu7VxOd+jPYHLTwlJwwl9I9zyLyBVcyv3W02eTyHXmy4ny65rxPlpcJGexhted/l7fZh/TuN0ovWYUucghu8m1P8jt6zjf0uTShqQnd2DzJ3JaCXYxDbn8TbQghb1zeOyT7Qyvu9tI1XKtk4tc9JHJ5Mzfv9ItK43bGhi5Xlpba3JpO3HeWu6txet6FOJd3OF0H7NyLGdzIwln5hexOsNAcgXjON3pIVxIJ8dxjOLd0o5csj+nkTvkwBdAmxugWx7XvQulSeLGY4awdrx9T7PF5Jx78lgU19g19ea8LS3HqwbN5Ly0SMXxcIGC3CZ+qsYxxLb9fOJnh08ll/EsV6yqiwfrln26BlCaHsqU3LQm98j1SfKCXDFtFrlm74aRc/jTnFyFs0vJ+RZoSu5XMWsQj096OnG6iNCr5BZ1504v4BXPNSxcc4Jc9waB5EYE6WNklZ3zcWHmBHKeEZXIlR7Bcdus3m7kTvpzfHP+4DpyScM4L9f3cHw3oByXA3RlV9JIO//Gleua7RsjjXMEK9z5oqxtaEtJXtZsQ06Zcj7AYRaKHOZ25cVIHlCfe7GEXL6y1XmDv5PGPIbe2oPr5TmNy8vAixxzzzTj+M33Nk+GvGrTkZw254tx68YLlGbHGS4DrXh4irj7HNSvnrCIXI1pXIA2ZuxD7qKpN7nunXi+bkLvluRuz01DbuLgLuTqWj8mV68p1z+17BW5cUpfdqu+4na5XSy3R2dXcWxXOj/PpV7tOJPcm4yPyD2z5+vz6q2RemskBvxVzG42h9x0Dk9wIpTbjdf1eSxidb0euX4vue7PaMFzQRHvMuiWM/q3oDRvZ8WSm3stlFzeirXJ1ezGfVXV3Tzw2mHOx9oHPD64lPUkuWzneD7VwUhondeW5w7HJue29LAbl29N4zmj9uq4brmj51BK42HDfXpEY55Tn54zA7kXuZKTM3nEsdgMF1KI4yk4YJMR94soPrkGuWEHjfR387aTaubsSK6HKV/zts9Kkxven3eRZoE+aLa5MpzSlHXlIEN9NHKTY0h9UlY2IeSuleZ7Rl69eOwcAm9yD9NyYxXTgI//QU+uGw4uXJavt7hNLmrkeHKqEI9hixXQn5f0XUdTmnshVcm1fcgBZEMjc0M7NFdyGZfxfF7zrdXIHahLClhrxAmC8FXkzQaCIAiCIAiCIAiCIAiCIAiCIAiCIAiCICQKedhAEARBEARBEARBEARBEARBEARBEARBEIREIQ8bCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQKORhA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoWmlFIJSxrJK2MAOdd0seRyPOV0W/GCd1E5C7uk7qSeb1ipW04bXYjSKLWbXPT9LuSs7tcml7H8JXK90u4jd8KnLrl5V5eQ0xbwMx2vXWqSy4Cm5D7V9yYX5peJnPXoh+TMkvOl1Xw13XLaJXkoTaU2F8kt8A8jl+2KPblHyXORa4DL5A6F8fkM2LiAXLemy8n9KjZDI1cTncn1s5hMrvODsuSc+gaSWxUaSi67bRJyJ1bry9C+w5QEKwezM919kJyGxuR2rb9JbmAdO3J1u1uRKzsrL7m84Sd4v4rPZygcyRWp/JqcdS9SOHf6NrmFqzOT8z+v32/+xpt4Y8vSkjqL/OSmozC5bd4Hye3cfJZc5bZFyPV6Zkmu9OYENsk/g6D7pNY7PSWXv7cXObfqnG/1mHcRPIfLwXNn7l9yNdCXNZXGm9JMmX+KXNjMC+RmqnTkxmvW5EqGb+Z8XJ9HrlWhReT8QszJZXlgSs7F8Q9yc6vdIldxvgu5W225rrlGupGzutxWt1yr5zVKs24At0emNty3KFWVnElMALkb6WaRy/iK+w0zl1LktId8zX4V23CaXNwVjilm5ORyuytyCrnYqHHkticZRu6d+kjOT9NfJ4VUlGb9qn7khhUIJ3dh5CRyj9143aSf+LgWDePtLUnPMdu5fNwfYFMNdtu3knpVdTy792bkZi94RG5G1w+8D9VQt3hTa0JJ8vBamDq3G7nG/blu2Nw7Ti6yAvcRlsd6k5urNSPX8jc28wAwfg33ed2C48idf8bX3W5YQXJZqnP8GvFiEjnrS9zHDDfXtzt9T96jNA+rcTySbjWXA/N8EeRUTS73WjWupxPm9yH38motcgHRfJ7cavxJ7u76YrzuKI61qg6YSa4znMgVUBxXJbXNoFsuEcbnqZMJj0scl/UgN6MJ91fTNB7THQX3nY6qDbnWHhwbz7zz+wr+rdQ8FjvYZQe5gBmVyU14zPnmCBQou+ogub9KlyaXY69++VJTjhmPoh25Ye98yOVySE6uo8l6cpG905MbN2o1uawjOK5q3pX7osdHuf2vcYvPZ+VMHC+mO9CWXN1RvO4GjfvJhwfS6Jb/LMNt1KCUpDB75EVyL1vl4YTD+Vqf69eVXP4X08jdXcBl3r0vX8dfB49ZYCTmacxhALwXPidX149j6Yd9eUzp+pDL5L5eD3TLFfJwvN1EtSK3FPnITVhfndwCF87bqkn1yIXX6kmuiG99cpXD/cntiLtLLlpbSK6FC5/QJe85NkDNDOwWFyCVw1ff5/yRjGPKcRM5zkC9I6S0davIhT7gcXeS/ByrfxzN28vMUzx4Ovk3Bjg5eTxVbZsnuaSuHcmlGBlDbpI/z/s8NOG5G09HjptfmOnbKp+YcpRmeBRvv5gFX6O3daPJnWq/llyq0jzuqrnoELkW5bgMDQrktlobsILco2q25Fb25zK/Kg2Xg9f3+pKb16Q/uRMn3uqWzcExa25j83TbeJ9PqnmQS4875N6vG0PuSAceI1R5WZQcFLerv4pInCFnWYfnIbGe5xC0c8XJBfU+Ss53+AhykydzbN15xQHd8nJwvPx4L8+XPCq+hlyNLd7k3kzmOfpuQRyTds/AdbnQlhvkcu/jOnSy0gZyipsVqNzdyZns7sDp6nI/5Ixj5A56ZtUtPzrD5ewNuD423sxxesWam8hlgTe5vrm5P0w9iNMdrsVxYUnFcdEvI5bHXI/M+N6Ky+PSvG4HbiNqRnOZb7pxErkt+deRW3NF3w4NMzFy/2VnFXJ//hFC7sn7xeQyHMhNDjw1hxnmHC+3N5JQ9eF7MIdceR62uinn770Lj01RphKpvVvykKtQm+dMrtvpx9jZkr/hzT/mgH7C4XPkAhvx9m895WuxhQwQGcPz8aNMo8gF4DdP3gjCfxnyZgNBEARBEARBEARBEARBEARBEARBEARBEBKFPGwgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCEKikIcNBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEFIFPKwgSAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIicIsoQnrpbYkp7qMIhd7WyOnpWrA69qsJpf7oSK3eu8zcu9XZdItW2mbKA169SVlPqodO/eO5O64ZOXtcTbQ8qwLOa+PJcjld+V1A4tvJtfk2CZyZjtTkos58Ybcq2VXyPku5f3COZ9u8dmGc5Rkfoqb7HbxuYOdke2/uExKS72BnNpTntM15evfzcgufhWu823JtXhZjtz86HvkTl+rTK7qeK4bGUcVJtdw/ElyYQ+r6ZYPFdxKaUybcFlBRi9Sr1pWJ+fUJhm5PxBN7tTdSHLFSnN+j/OpwyZbvr5nPvI5uRHNLvJjEnJa1vfkRvccSm7+fv2yf1lvztz7A7x9R87Huf58jsts5YPNgi3kbge4k9thcZXz8hvJt6couaPgMmQ3iq8l+nEfEVwsDbkUDXjdFLjL21ujT9dq1lFKMjdJEK9X2InULI23X78ut9+wKU2qewF2faK58fNPz8elveAyhDzzSQ25kJqcl5H6F9tjO7nRdXaQS/LFc4T17mekNJlsNpLrCm6rr9avQG7f7FnkPPN1JVc7MAu5VY8fkDMn8+uopnEbjDGc7wq5jFzfEL6+yi4vuZol+5M7l5+vCdQp3WLuv/icXmrAfezQSrnJ7b9gSk67yuuWPXCB3Nvl1uS6pMpEzrS8kXNSKge5fo25HRjyeAK5zS425O4jnNxGI82P94fk+nzAitIo3whyWiveVhYVy+su5b5Fe/qR0/X35Q2qZux+M922cBsWm+sQuSTF9pMrijPknLdyXbDi0B+bzrLzhz6G6j+YE8U9yU+ucDFuSx8pLrtNt3KBUY2NtM0XFpHqo5XidNouUnkH1CJ39kJPcnP6G4kDb0wnt2nVMHJ/xpUmVzp8gG45/WGOUYem/X/t/VV018r7vw9PWgotxd3d3d3d3Z2N28Z14+7u7s7G3d3dvbhDcap5jsOVz2/tHnzL81/rdZ3lWpNkMpm5555p4L0SzjRdC9XBJYdMcp71TZ6Hlzuwli+71O0acHN4ariR9hv7QYb+zNW/r+A4iGy79BfLpV/FTQK3LEEjuJONfzmOXxftx8uf4DizzRS4ate47gxg+DdVf7C+O+zRcMk84sL9XPELzvvlBri9xZrAZe7EtruVgfNJIfMMrvIPjudBwc55oudS5jLnWjCrGLogG5xxmUuMy7su/g+vdzRmJbjC7UrAvXa5RbhhnaMqyVzdmOowh+6UgPMxnBurDuFmw8FI/nBZj890HDOKGnMttC7cS0/uSXRLx/XUtruMwfUib4Jr83093MiInHNOf+e689OoSXDXmN6Z5Rb3aUxoX6g8rcfAzezMcXDjeTLHsRXCjjvxtUtn3uASo0xalnv0hO5fxp91XRmnzlzmXuAfZfQ0qJ0pOAoPxua661A/xm+P0FE891ZSuGf2PbgCmws7jt9+WYAyhb1qw1nmJdyuDYngSuSvB2fW8Ln8au+DO264BrG+sC4BFZgrdt7JcfBtVjDcZbsC72GGwVU6xdw/g4nqOH5scc/nlBkCV7nKRrj4rbkHYAdynTehDmPDOFMD7r3hPf4kkSLko3TJAaJax+AGuMx3cfYwyfO8VRSuWnVe7/oqZ06a0zAPvmY1hUthmJPeYupkoubLC7c09V0W7MK16E/D9XmVO4yRZ2oM5/UM86IaAz/D7duZEm4Mt4nN/QDmFFFs55zjt4m5e7nDJeCaV+e+yhibfxdIeI9zpNXCZd7oxX3Nou+4Fv+TfI7A/arkhvsK35NBmdD97PNbXR6v3ELmlm+vsFP+vuzvdZzXr/P4H7htphdcugyMS2YV+5SJ8BWqZUH+TatTPO4bWiUPwNk3eYuvfuwHNS9xbzJ6mahwn+q0hyt1+wrcvAzO434W846ASczjclbnC8tZLRDO+HGfZmFp/p3vSfEtcNGPu/zxz2UpKIT43+h/NhBCCCGEEEIIIYQQQgghhBBCCCFEmNDHBkIIIYQQQgghhBBCCCGEEEIIIYQIE/rYQAghhBBCCCGEEEIIIYQQQgghhBBhQh8bCCGEEEIIIYQQQgghhBBCCCGEECJMRPivBf3fuMgH1aD+8d0O1yCEt3nxi5e76WXBZdwyiQVbxXQcpt1aE0VWu9Q3vcffcAErbsAtaMJvMNJv84YbmWYU3LbvNtzuwVHgJg/7BmdlaQSXshrvYfpTXU26GK5etSlwe25sdRw3NWxznwVzeINWBaB6BnrBjYzHUzdVqgWX+ybbKU+qlDz58GO6cCJz5+9wi54mgbPm0ZlyE6Dalb0C9+/9f+BysmnMYVPbcbzqHdve5Oc7OjO5Dty0k5Xg1lzoCZfsoCfc0y3r4E6aenCdI3SFqx7C2DAoLpQxkV5CfX6dgOUSdIEqbQbC2aHrncK6zmv5zIPaEzgSbsHRg3BzGX6MmcvY2C8bX+yvV59ZFZf3H14M7fUWzqfFerh77xg3rn94Che7aTK4gC0t4WbmZlvb6286jrv3zowyj0r4wG3wZt1mu7Sp25d2oZ5r4PoXOwn3My/j99ORY+DsQrzxpbQu903Ptht1sCFc8mSch7YdCIQ7+tt9f4WyTe6N4jjYsH4oXJavDOrzzgbA2W2/wGWtGQRnXjH+/Eled7oFl6B3RrjPn9iG1fryelYDl/hyPBhqaO4HcK2Mc6yV7TIOZd6MHA43ctQKuFKrp7MeDXtBTd7XB66aGQ13vC/bKZ9HD7iIvl/h7K8f4Ob3KwIX50cqOF8ftnu82qFwfTd9dBzf7zMDZf7pAGXyLrsNV8FkgiuZlmP+7vlhcLXiM17kHZkBrr99h5UJR0atugv3D8OpSdOXOcp7+wLcxW1M6ht0Z7y65s97fEjgjBNNKixEmWx2bF6rTiu45D3p3k0qBPdzMGNz/FK8b+x7jHVjEjGIH4hyH84atg9u1yiOy0+LuTYZ/I055LRPx+ES9azgOB6xLDvKHDSV4Uqk5BiPdo2x62se5kb9WsyCm/qW97UbRoYz9p9LcE4lYbyyz3KNaUWPynIu15vlsn46WuYY3A2zEu7lYeda7PDx/C735F0fpeLiNns89tF7uSPBFWnFNWZLazVczau8r1WROZrZyHz7XKeYcIdd0oCayVm/0xbHeFmXNggNfuU4Hrw9Ico0jNsM7kP1KnBueeArvlZTvTcforj3VrjTrffy5AWcT8KLhUwtzedAxup5i3bC9V7A9Z4ZGw1qRCRer7Rhe9n5nXNO1TVs6EBPTtLPI3NPwuNuNrjTZhOvtwDKFGRIMwWaMQ8KcZkPtryCMpWiMRGMxKWdecYpwlzexr577u0I1s/ccxyfaOzSScvRZc7KYreuuUSzVUugMo6PAzfzGtc5PawacOv/4DrWTJ8NZS9mhXz+Yh7zMxfn55Xr2a/SjlwGl20d5+cT6525QtrhtVFmW3PuZWw1rFvFDIwjdkPe0yTmfDC7DYsV6cbc98oX7i1FbM815oLZ7GsxXfaCjNkDY61uAbcyE9u4UcHUjuP6U/gQrUtMg7tsmHdE5NRiXnbh4PjSjHthH1Ychguqw/1Bl125cOPeRDrfblwnnYvM93bIPwVc7X3sfw0usK2bH9sNV7K+My+4bnhexDhH4Mq/Z7yNtYpr1mWGwXVt4jxweXaeh7PT+cEFJWWb/G2/hzMu+cm9S2ynspc46ZSzeI/vbXiuPWyss8yz8igzNSPbs0dX5oUTDdfAJi3joH2axZLanPu/uEw50f5gnL/6yA+u2CXmBS8Zco13Mu6vRb3D/b/9vtzgtVtx3W9aFHccWkX4bhsO4N9z/mlaDO7mNe7Dmjh8v4Us7qkfsJnRHt7Dl7TrqMszxC4LdW8qz/3XcI7sG8A57Fgo84cIzdiJni5x3mNSCZY51bMtXIXm/LvX4TzcawtcynNtbjUZK1VFuJ/tucYxLvmjEOJ/o//ZQAghhBBCCCGEEEIIIYQQQgghhBBhQh8bCCGEEEIIIYQQQgghhBBCCCGEECJM6GMDIYQQQgghhBBCCCGEEEIIIYQQQoQJfWwghBBCCCGEEEIIIYQQQgghhBBCiDBh2bZt/7eSFlStlrfgNi+55HJyY5gYo1iq3z/54WZ+Pgv3rMJvdWnNa1l9vODeVg+E81/E50r7N6/X8yKbabLNc+1TPNf0pxowlu3ksWsrXLG9wXCf9o2Ea3A7KtyvkK9w3p6/1flJE1Yu+WAoq2tauH7TIsCNth/zXIv1MI0zQ2Va1Qfupj2O54YTlv0Obm7jALj2a5LA2W2nwl0JqgP3YwnH0JOd5eAaXnS+N2tQU5Q545MFbu4vtmmTTlCmQH/27yiJD8JdfDAJLleaPXA/rJhwASEf4WL93h+NMZ9ZPTPUqyNccNAEuOn2aJ486KXz+peWoMjiXexncb6VhIsRJR9cpdRp4JKnygH3PGgG6/bR5Xuvq/HowgkriHHEjlgLLml6b7j7AfPgvHPz/VoH2Nfs+i6VSfhbIE7H9mvZmO1X89xyuGo1S/CeKxOzbiVH0AUO4rm5a8CZH4zf1pzCcP3LHoC7bzheNoQUgCsXgeO5th0Jrp055zguZMaizDNrPNzzzi7BYWYQ1D9Bv+BOez2BK27Sww22a/IeZrOLCx+sK+yjM6vth+v8rCzcT8OY5p0qFu/hcl97McfBpifO91tnGN/R6UcV4ba2YwzuOC8X3CSbecc0y2U8doYyUbryKbq4hLRJZ3i9wedSwHWOwv4S4zr7t5WG+cin4BdwPpnyOI4j3fqOMvErct58u2UunN+tiHDJVzPWWC8qwwXO2AVXPnFVuEO/tsGFJzlX831Gacw6nbDZZq3O+8PVz8t7JLE4/kPmroXL0s7ZV6dZGVBmq3HJe+1WcG0tlwzi12GoH8VYD+8IU+Eyn/KBe2zzHmutaHA1WBOz+xbbvWKmdXCW4SIp2OV5yxT/x3HcwCMFyow9ch3OzzCXtZNwzrGrc64vPJNtcnjBIbgsbTl33rc5x4YXlkskPhuNefP8imfgFt5aCPfsui/c/ZjMVWsOeQb3uU4Cx/HsDMxBv39n/pDGVIKrWoNtv7YX12LjS3LuvdbbEy7gaWm4SGV4j/gtGOvfRGkHt74n1w1VFwyH25JrJlzDbT/h7KzOPNU6xjVmhNi8/gHD/pjQZcVxrXIMuIU7ocwYl/Q9V+gHSps5QXhhvWafD03I9xbMpY3xOsRz9x5iP6jZJDZc+rbZ4BYddOaXES4tQJnsFufj4OZ/wXluiw+3ucFbuC1zvsEtC74NZzJ1odvBOJArXVy4wBzMNTpdoeswh7ewJp6DG/aAudvgyr/F8B2MDQU8GNPP1gjhTV226RbPccnvmk2D2/qhK1zxGlnhImy+xpuEE6Vm81lqduK6sEuDh3AH1iSAK2OxHSaZq3DVP3CfLFkEZ9tE3OMSSFpQWec4Lx3KylhYstMXnjyL8XtRW+Y7VbYyP5v6+i7cmIhc2zfu+wlufkXuCf7IMgYu133GyBMTGCND1jnXV6mSMG5VeM74ncU+AjfxVHU4824ZlBXnB1zr91yH1erBfKDiQ7Z7eGEZJuCbrclwNfMWhbPPu1xwAsfQt4YV4I4lng7XrFU6x3GxxbVR5lKuHnBHVzC2eGRg/Po8tBBc8xH8W0HZlDvgxj7i+F4RYwpc08/MyUNNA7geSfkHg/ED3sCZkRwvQQ9Ww92f5tx3zNGP+VQccx/ufQmuic2RVXSR+kL17s5xVSiY7R64sS1c/ceLeI9wwnrCPjqKXdSMnNoMbmsFjv3cLuuDePFuwgW/SQXXxjjXRAsSs02tvMXgilzl2iBtdv4hadDGRHAZkk+FS/KiG9zDLdzzvlKjJVxOwyQwXzrOfTfvcR3xrPwruDP7oExFm7HULP0t5p4ZhiJ15rE9N25yyW2K8O8xJmFynhuSFK7sJK65o8efxes1YZsIIf43+p8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBMR/mvBNOYiXMEs6eDs1fF4cqNxUAOW9YXz/3IM7kYOCy7RA+fxy+82yoQ0KA7XPtJnuPnt8sKlz3UO7uoMKLM9bnY4y1xhwTGbocqP8YJbau2CizjvPFyyaUPh4ifnffN5sA2iGmdbfT3B9rVmtIMb3Dkt3M2/guFeN9wEF2FaJLjg6FAmaNV4SsO+E150r8++fKPVSLg5K3vx5JYToHKsZrnVdZ7A5asUF67z9kuOY/tqc5Sxfq6As/t2hruZfB3c9iQt4UzEMlBpZqbifZumhhtlPsINCOwJF5yCYzfCW1bFjpsUbvyTWXBNTRe4dCN3/GYmocygfbHgQruzbifXX4br9DQHXM1eUeD8CySEi3ktDpxt3sGFFytc3nnWqSPgnndNBOdp3vOCVg66w/mhBkdcCzfidDPH8dP601EmpGEhuITfG8M9Tcbv6hKdC4VLagbB2V414azrr+HGRWR/CSzjC+dhRYM7bQfyHjmgTCKb/W9DzZxwQzY7x3i33FdRpurX5HDpqzL2b87OOSK3F5/V284Bdz4yn2uI1RHO5uXCjQ1eVeAyDX4KZ0dnJRvXqwW3Pnt3uKT2ZLjmVUPgur675Tje1Iwx/b5L/xnbkX15TEzmU1MnMCeYnCg+XKaOL+GO9r4Jl2dLJri8gQ/hBj5aDWfSs/95HE8M124BY3PM9C6TxL0vdL+zm+rnYrpILtUdlLge3KjVzNn2ZOf7ORzw52L6/+JKm/Rwue3YcLblDxepyly4Rds/wH3rxf42oQJjTF3j7FtVL6REmdP7feAOBUWFM2YPzMnTJeC8t9LlSXAI7v50jvGsb4bANS/B3ONJi09wN04wjlSNtBQu7RnGbDvncLixz746jgvYAShTwOYclsNwXWKd4bgPSPgIrtnSX3DeLms123DN9SdJ4JLmBt9lHMqekPP2O5e1SFLrDtwnuzXcF8Oc2/ptLWZc4pBdj7EvjSkJF6FPbrgmhekaBvEdXW/D/pitxUFWpjnXKobLEGNCOOYPvi/PU4dVhev/wmW9N4LzicdNZ55W9SPjVojNObx2K46p64tzwJVKzzbZ0W0qXK6KnOsDQlmXSL+/6/CkDVWB5FyAn+t9Fs6+OgruXoGvcD9OteBNMvlBbSvnjF/VDNdwnhuY00+s/R3ukscDuPizuKX1bUcQ3NdeP+GimZ1wwQm4jh/4Ywpc40iMh4+e34db0JL7KLnrcWws8OgHt2xcRcfxw08/UCazPRjubDJvOHsB67u1Avvo3dAYcIkPQJn75a7D8UnDj6+duMb/60ozuPrp+MyHTV2454kY51/4cy2WpCtj5LwVzjnQZz3H3vtkq+DMdK5j36Xh3H50IvfXqsxlAvttHl/c5gPH4R68LgZn/V0Armth5o++JbnO8S3GfYFV/TiGahfvAXfx0W/vZx/nrwlnuA9Z13BNZ4/n+uBpdObvAx6uh+sxg/0kDkPIn4zyxmy5ALXR8F2ac+xD7y0/uDi92f8a9mY/jW2YZ/gUdL6TzYbnDS80D27Jbu53p5jJ+paYw7F3YQbzb4/O5eDOf+ZbinWH95gXnfuw61+xTaJH5P5nRY8UcHE7cW1xe+ItuKCXzkVqZZvr+PENOA7KeqyE6zI0BtyVk3z+T6MZrWtW5vz66Ocf7eHArlcWzuq1H65+t2VwZX8x9hnve1CdSzG+WiYzz63rnMcXcillvm5jTnAt9BVcgsqM1bnrbofzzlUQ7lHEJXBWEBc+9hb+rWagX224Zte3wKW1uP9p2dzPMpE60R3h3kHXBc59pGonXGJZdPZ5rx3sj0ENxsAds1vATajA69XdOhtunzfXDGWbQAkh/h/ofzYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBP62EAIIYQQQgghhBBCCCGEEEIIIYQQYUIfGwghhBBCCCGEEEIIIYQQQgghhBAiTFi2bdv/qWQCqjHTKsNdbrALbkPuo7zxxzZwdrZYcLuqn4ar+DnIKWpF5LWS74Nb8bUsXMGoFtzVX1BmuPdwuOsPB8M189sBtywx2ynZsm6sy8jicJM9a8I9TlICrmj+V3B2/0xwUfJscRx/M3z91lW2yaDXLJeiYny4JaF8FzH+2g+3Y3EOuE2GDV/rv/XO/xOsn7coc2aGGlmYlWxksQ1T5nJ5mFCqa7Xouvea4Dg+uLoPykxe2RluQ5OZcGssl3eZkPVd8NYf7v2NGHDLMyaHu20mw1nmK1xe8xzuvPkHruAr1u9Uwhy8x/NycCbJeMfh5dZ8/hzVeP3Q5XHhPNe85vXXunyzVY/KXlgM7lfvK3De37/w5HCizOf5cGeTMpaW+doVrkSC3HBdU/aDs05vhTt4oB3ctvXOfjCVYdQ8qsZ3Of8a3+XY7LHhLvVbD5d7bCm4qD94veL5IsOdP/IR7k2Gb3DR3rEu2V1Cw3GL/cDDKgm3/Ox2uCNFnPPmgiAf3iB5DDq/vlArJhyHa5a4I1zHfLPhZqVNyHvYnKv+JJZVx8VWgrENA3NaOwZcbpfYv94lRCQOYbz2s5zxxdPm+3hqbYJLXnAMXMjpuXAeOxlbrdvXeb1bLeGenNwDN7jQbrg2X87BFfnbH65d8Tlwqc0ZuPpD8sBFHfoE7otJ8pt5hjI+5Tg/fFlxG86rOd+hdZyD1P5ZH86EMq4YuwydxbwoPLlWms+Y3YM5aKIDteFe+LK/Rd/KOeFzMZ7b0CsN3OkZzrr4vUcREyd5f7iEKZln3CjxA66eywAc2CAOXDY7GVzuan5w+Rt+gpvNbmpMujtQdT3Tw62fnQ0u7vvGcO8H9IIbvz+C43hKmnUoEy8l+6mndRLu4lnOf1b+tnDmJ9/hJp95cMW+3YSL48vLhRsusTlpnOxwz6Ocgqvqxzl/2/b8vEXVs3B3XOb3DlYqx/GhFktQJtqM8nAhUXPCpRg+Cc5vcA24H2nesSIBC6DOPGsNl//O7/HVGKvvCzjPrYfhyrxdBLf3yEo4r37n4YJGuAysxs73aNscy1ZqjuWBE5nTF6vFd13Orgt3JNIEuOLFmcuZMS4vm6lxuHHeSguX1zyCs+yBcGciLIcrEMz3a3IkhrqakXsB5cY484pXKdjnLyZibP18OyZcg+it4L5bOeDSZ+4Cl/UG31Eul9jQDcaYTiO4F9Qk2iC4QpwOjbEfQnW0GEuPG7bL40POGP71COORyeyyn1M/Kdwkl3V3T5v5XVmTEe7wkm1wQS2Zy/5JrG18l6ZKDbrNjJszy/DFnX3HuXhFmgZwPVz20yb5/JZ7pHfbc+MYNdMYg59s5Xpg2QmeGvMRc/9OE9rzvtOGws2uPxouz0nmzZNe74RbO8nl2eK65NLzY8DZhznu51iRHMfN13MvuVHSgnAv2rAe5wf6wbWdyli2IA73cEOXzYC7n50xL92zP7dhaRnmCtMac+9mzGrmaT0vMseLt5HvrR7TfhPJZZ0V1ThjzpTq3Ntts88bbvE3vo95VQ/BnRvLGJy5wEu4mym511DrJvOJTSn43losSAe3qDLrdzUp+0H9hwFwD+ZxP94ewvkgYwXnXs3tpXx+63QhuNfToMx2TyYekatzn6ZFXc43UwO5Pl8aCcqc+4N79Jl/bYF76F0ArvdT/gErX3LmgtVs5rP1dzGP+VWRdZno4RwvbfYwPlaoz30lvzEcZ3PXcLMzenper9BJvqNlMTvArTgAZXpE5ovjSsiYI+MqwMWqtReugcvW+xqmmaZUF+57+Mxy9vmdxQrzxKMuczqrYayB0+FWtmO7N2nPOd2ewf0n04nKhLg4IcT/RP+zgRBCCCGEEEIIIYQQQgghhBBCCCHChD42EEIIIYQQQgghhBBCCCGEEEIIIUSY0McGQgghhBBCCCGEEEIIIYQQQgghhAgT+thACCGEEEIIIYQQQgghhBBCCCGEEGEiwn8taF2xKBPadPZNqPbrv8PNrdaOp/5IDucfl7fo8y6i43hCLJZpmLg23JrnJ+Aa5+O5q0pHprRz0Q0Mgmp7egbcjHqN4Jbs+QJXekxKuAdZeNsX14/AbU1QAO5r2W1w38wzx7GVawhv8JbvdUS5iCx3uhpUp/xT4J6uTgr3004A520e8x6GbRJeZNo9DK7YXXa2LHfYD1JO5Yvb/XQtXMWGbOs0C9PB7Sjfx3G80oxEmaZNB8J5JQ9k3Z5Egot4bTpc6WUP4HZlhDKfP9aD+2cPx1/pDBxDI3LtgStkR4ObkGaSS10+wY1uPAJusHnrOI471CWWpeF78Pz1E+5DgdxwCZdcgbvU9D7cBcOxsbQWrzeTtQs3Xr9jXK7sy9iyLn4HuISD28D9m5D9KvJexsgtZdrCTbc3OY6t55dQ5uWtVXDjpiSEC5leEm7SrTlwpgnj8pec3eHO3E0CV3CLN9zhzZxiS/YOhjv2i+M0NHsJuAv1LsPly+8D16yqc/yd3O6FMreaMIZYg47DRR3J8RJic7x4WHyHZtMrqIELP8ONbB2d54YT/UI573zzuA130pyE2x+XsapCKOOc7cFEw99qAPfO/HIcNzjD8xb7toT7fJpR49i/4+COV/oIt7pLTri609LDZWnIb1P7l4EyWV3C65O6dP3mnoZL0I51adqZ+cPyQRx/5rfpwHrbCkV89/O9xu/IeHS8WBe4L6FVeE8/f6ggw7HRtslduCUMXeFK9h4usv9gqB7pmN88Os9x/aXcerjlZ/vCrak+HK761t9EDVbtzbBRcBGsayx41BMqyv04cD3KvIdbe4jjLXaEd3Bl6/N6GzyYj2SqyJxvw7U8cBHij4ULaf8vnD2I7XnltjO296lRH2UKjGKf3Hx/BZxV4x7cpO3J4Pw55ZhasZkTbI7CcjVdlo3hhXWI+btHqat0jX3h0i5gvD63YT/ci+BucL6JS8MdMk2cdZtYAmV+pWZjefsyCd+3/xRcsapv4UZvZ05b3/Bl+pVmEC/BKcE8fLgMbuu1zHDdmy2He7mb4yDOm9lwQY0Xw80r5FzHzjCcI0Y92gW34NQFuHchz+Gupt0JNy3wGFzxjlPhlp9lvGyWm/lXeJHPZlzqYDVhwQZLofKHcL50W6GkyJ0XLtti9t1ddb46jgfEXIIyY+/E4C1jcB1hOpaCij7iKNyVQbvhLluH4DYsLwSXpCnnl+cW50irIV13purmxl814YJ+sk9e9+HYuF2qm+O4uV0BZZZbL+ASv+e6pFDs7XDTqkyDy36Sc+k+/x1wPYt3hpuUCir8qPYIyl7FCg1YzXVHpzpsm8P2Q7hNidmvSqyrxLp8dOYAX30YW1MaxsdK3lxPdwrg/F8vmGu2W905V1meLgE82R2onms51vZ1YK6wfm4muA8P08At65oDrogX2+C5xXm4ZkPnuti3bkGUqZOf662s112SjJO8553Tf8HZFucSk4njattr3qMXzww/vu+DurWadXxtuEd2NTf7/Fmb/aVUVPaXJNx2NN/2ONdKVj3utQx5w9zp1RLGoO7z1sBNar0Irmi61nCZknGdsvYGx218w75ROiv3kbZ047rz1fN+cA8eMJ+3p3EvyOrMdaZ935k/WJc5L+VaxvfaeAMXlK+acJ4f8yE/XKDl9jePDVAde1ZnOcN3EV7cesH5tEAtlhtZ/AacbXFd18Llbx8LY2aA87RcNj4Ov3QcHknJ8fO+wlO4f/dwHp8bjfN/7QWcYz+0qAg34vkzuAk+vIft0ueHuuxdxOzLv2U867MXLulj5lRrLeZotikLZ3Vw5sfWbObQf1eICTe9HPdr7Yd/wz2+zH12U4Zrl6JXisMND0kMV9Ks5vWEEP8T/c8GQgghhBBCCCGEEEIIIYQQQgghhAgT+thACCGEEEIIIYQQQgghhBBCCCGEEGFCHxsIIYQQQgghhBBCCCGEEEIIIYQQIkzoYwMhhBBCCCGEEEIIIYQQQgghhBBChAnLtm37v5X0gUqQLATu6NNAuA8udxh8lG5ikc5wOWL58b7H2jiO32SvgTKvY+eFG/OxENzU7DvgrIc8d93pTXD1/l0P13xwJ7iDP3PClR6xE673GDZUul0W3O2oVeBGrp0Lt2bmbTjLyuoUQy6jjGd6KGMaVuC1dkaB8228GW7I56Fwzww7wOTRXnAR+v+37vl/A9v+FJXpEfsn3F+ju8Ol7sd3FORyvQof6bztZ47jO4bj7HIl9uWaJ97yYssGQCXdz7o9j98Tzv76GM5KnALOv0cxuOi+x+GGBa2AG1skOtw/CUfCnVh2Dq5l/bVwDfrXdxxXLosiZodLm//r8v5rL/WDe9aiIVzS5xxXfZJugTsX2g/usMVzw4tu62bALW7A/vKl2RO4am3jw22vHQyXf98ruDbZEsCtKuc83r+P72Pm2uZw7XIsg/N5DWVilqQrcG4O3KT27eFKJWdsfRU9I5y9lPeY+4tujJ0P7mnkf+CirZ4FF1JjO9wv70iO4yQucSaZ9QnuRL5YcB9LHYFLOqM43LdvnNP6Ra4Mt+47K/PY/Lk4/3RzTLjKwezL8etFhUt8cwSv97Ii3JHSieG+ejyFizrX+f3n2bEXUKZEpDRwae8yP/vX3gj3KhOdac96eE0Lgivw8AjPNbFhbBMRrm0bzoe9O8xn/fLwvhdzNYK7U+473NpDPRzHn889QhnzvSlU9dqcv9fP5qkZMm2BW/KDedFgwzloS9cWcLGm8R7hSY/y/nCRctIdHJcK7nz7m3CbirSBq72Y/c0+dAkusu18B3G3X0SZJ9XGwCXMzFx4yN04cJ1cxnOs74w5hXzHw5259xzurc3xfI/hwYxzCbz9XzDGJn7pB1c5XzS4g8Gsc8p5znuM38YydSaxHokzMP6/nML5JXmPoXAeNvPPxy75UrTqUObLVrrwwmUaNDa7mrHy8B2ZbL5Ql6uxb0S+zzx86boacGP8EjqOR6Y4gzL/2CngfI6woy378BmubRHmXp+zMOb+asZx692zG9zcxOzzG1uzfgfm1oNb94Z5Zb2bnHete73gjndi8lbUcq6pT02/gjKZa3N/Itp6l39f0e0gVP+0LeC2lUkJ16YEc7Tu27fA2SuZV4YXywz3M1pY7EOD7ENwN0xtuH8918AVCKkFd8bbZW+l2zfHsX2D+cPiHYx7My2uYzeHcjSniN4YzrxZDWWfYj5ilZ7EcpHYXz4Gcn5Z6/USrlPuDaxLRdYvTeVQuMfnuR4Iifyv47hR/hQosyr9dbjvETPBPQpkTnWiPZ+rY6bpcHZXKGO+uDiX+TC8sMwouFAfLvyTGMaILl9Lw/XzY44YGol5jEcy7vVZT5w5RZGWfVHmeLrBcD3nROa1KnO/YNouKPMiB+eDeFc4f3nZ3H+JlogT9IeK7AdWtKRwQVOuwV04ynYv+NdZuNBhyeE8qjnfRd0l3F/t1rUOr/+BYyp56Gi4pwkHwmVjlzDX3Jan5V3kXpdy4YRlfYCb34RjuuhK1jt/IHOALyXYX76sZbtGScaFjMcAT6cYzfOqp2Uf3eKXH+6ey15TrRzj4K7PYX5ileD8Zc3ivLE+L/828O5DErgJEVbBPY7IvU5z7TTvu5055f0azD1ylHauUT9dYkyP9OEGrx81C1z9VXzWlfOGwSX8OQSuOKdvs6Ebnb3gz+3djAxkvzqRlfthK+4mg2txOhfczkLecP1qlIEbu4U5RdTfJrxvh6+izJXvOeBKVmFfuRS7INy6Dzy37226nYWXwFXJsRjuQ+q/4GIv5n7qm35s43gV+sMNLJoZrmIo42vh7dwzs6o594Qb+iVCmdUfx8L1OsM2LtGRa9Nd5/m3gjl5+beMroWZs838sBAu+M4BOCHE/0b/s4EQQgghhBBCCCGEEEIIIYQQQgghwoQ+NhBCCCGEEEIIIYQQQgghhBBCCCFEmNDHBkIIIYQQQgghhBBCCCGEEEIIIYQIE/rYQAghhBBCCCGEEEIIIYQQQgghhBBhIsJ/LbjP/gX3ul5EuFTGghs75RzcwT7P4D7/aAxnlasJFyX7Tsex7fIU1gTes0a3hC4FqezOT+BaL/WFq/a6D5xvqhtwRe6zLpMaFIeLGasw3OuKrF+ovR1ufG9/OC8rBZydNJPjuODQDygTj7c0ezrYcDXmeME18ioHV3c5z932Dxveay7va/d3qUx4UakbVJYnB+GWn+sBFz//HLi3t0vCpX1SH67be7bXr7w+juNPryKjzBpv9r3j0/fBTSjPvvxs+ES4lxXY+FkmJYd7PzItXAv7PlwdGGOGfGM/iNAhC9ykldfhHo7MBxcrwXPe5N9kjsN7LTugyF+Gz7rE8D2MqMMYldTik4V4nIJLeewwnF+VK3BmJ1V4MbV+F7h9Z/vCBddZCJen8Gu47XZGuLPGky4O6zK4yFLH8UmTG2WyNuA7erD+JtzygB9w14Inw11vsBRu0aPMcM9vF4W7/Ow8XK7R7KN2pAJwfw3kuf+YqHBfVnEePt9wPdzd1c0cx008GOdPtswD13galKm7h3PV+W+z4eLZ4+HeRuAENs5lznUZauFGspr+cGl7NITL15zPHHMf+1C9FnyYw996w1kV+a3ntXbOxsnaDkWMl3UV7or9Bi7tMeY7p4rVhms7YyPc8JSb4ZpYfNYKwey3tidjdZJFvN7a+VHgWocyLyxmBcKtvLgKbn6A01nT2edtH16rzG4o0zlpZ7gvvn/B3Q7+Bnf8yQC4n9Nb8ybTgunCkbPHY8GdHH+HBTP/AzW5KeN6gzon4O7sZX5z0Yvx6kdq55j5sphzqqdpBJfl6mq4Dp6V4SKmXQr3zuL4W1Gb+U3ARro9QW/hgga75EYhHINda7yE67+FefNBt+6Rg+f6/Zby1PnARLp5s5xwhYOi8fqTBkJtt6vCZY/ZC86qyvhgb+Ut/mSwt13mHqslc3Vjs71+mR1wH4wPXKLnieESrwuCa5Kpp+N4kEmPMgPNaVZtNmNY6vWF4CLe5cPaoyfBWaNYrkKKWXBdAhfBZYzAsXbA4rOWft8TzqRkPmwfP8Nyt75C1R1xxXFc9G/2qU01osPF6PYZbu/sMnD3UvB6vaqEwP11m7nspe3Mof8kzZdwjs4+jLlMAYvr+as2Y+T97ZzLzqRyyW9+sa0rxYrhOG76KTvKPOqzF+5i+aZwHoOZA4RE5Hvz9OAckdnwHX2x9sBtasPrVV7A8VK0D+c5M5L7Yz5p18HdX5KL5/pdg7LMJsfxajMdZQp/4lql3yY+w1fDNvmnDp+rd88ZcGu5zWAaZGU+YPxu0YUTdqItcNZt5ieNpvOd+8xj3mh9L8h7ROS+nnnFeaNeIuf6uZsP42iqeswtmTEbk2MX71l8/Ca4eC5T2hVzEm5KkmZwnaq34PXGukycn+9CJfNJDTf3J+eSJQ3Wwnk24fw3znLGlZ2G+8FZu3If9lb3rHDPunONZFxyrGuWN9zpaVx3LzxRA26R2cILhhO2fQzuSPlacClc1vjz/bmf0/ZUS7gOHRhzByffwro8CHVevwfzmMAih+CS1OJ8U3gR1wIJKv8Nd/wWc+iIt5bCLZ7Pvc7VJbg/1PrqVLgNk9nnTWfG78AIAXC2j0tnm821Yu8MJRzH6Wt9RJlmidm/l41hnE9whXu9+V8NhotaiPPXY99/4QYvdNmQX0AVXpQpw/zr4L1QuHgBzHdSu4wDs/MnVMnKvN4xl3+ne/67M0Y2W87LV83Pd7TAXgyXhE1vul7kPrvJyD4fOJKx+stA5k9RI3OvxQ5mHDAHuP+7vMgjuCfnX8B9jMK1kN8Q9me7emmnyNceZU4eHQu3NhfXCxO/co288TLnSDvfBrjc67heb72F6w8hRNjQ/2wghBBCCCGEEEIIIYQQQgghhBBCiDChjw2EEEIIIYQQQgghhBBCCCGEEEIIESb0sYEQQgghhBBCCCGEEEIIIYQQQgghwoQ+NhBCCCGEEEIIIYQQQgghhBBCCCFEmLBs27b/W9FCMLZ9Gs5jHC9n75rOy9lnoRJvYbGXsXPCXbR6OY6jJx2CMtmevoP7YWbDTdwRDS7zxyRwFZvfgptsh8D1sBbBmR3t6CrvhvpspYH7FEgXubcFd24q270B72pCzW/nHuZ53xvz+h0HZoKb3WE83PRHD+FOne0Kt666Sz/xTQFnjJ+LCydmN4dKNvw9XO03S+AeBcSH28ruYp6emgiXvExJuLnXfzmO22cpiDK2xfd25U5buKFP/4VrkawcXKsMa+A+zhkFF+fyALh3q/hcplhvqEd7WCy1zfFnm1A4L/MS7s3dT3BR0sd0HFfKwXumX8X+6JuZ7TkhK8+1bz+l7B4BavCEhHBXC56E23qqMK8XXnTgM1tzWWxIxatw3RPwmaNlYtx4dT8xXKJ57FeWcY61rb3Yf6rtj8PKXePYM2VuQL2ex77inSQ63WjOJT5vEvAeS/+GsvNz7jvlEnMX/MN2jzuatxgf4vJsV15DFUjkvN7ZdD9QpsE39se1kT7CrUzsCVfhPGN/nEp8P+ZsNqjC9jW4EzwzHKkFU/FdabhT8TrDPQnm1WIcpPtQju/8ieE7yd3kt7FR3x9lNlWLDPdqWQm4znNywdlnRsIt3826Le8LZc5dHQz39fpwuNBsLjEkUU+6Fxvg/A1jaSyXb2KfGU6miUwW5/W/XkeZi1F5rUQWn/8yjDEVO1aCKzR7Hdzawcwpxw5fDjfbbupyl3Ak9Xy6Xy65qsXWiHuQefmb6fvhso7YDBcSaxZcQcsZd5Z4uvShE7HgZp++DTerx0a4GxmnwUUqdQiu9jRO8OsjlIXrZvisk8Y3gjN9OL9Xfcs+7huH+c1Al3bPGDk33N83Uzvr8ZzXt5sEwIV4cQx5P+Qc/u4Ac8OUpTnnxD7JuB7fJZW5SBVuWJvo7Dot4RbYS+FW1+S5h1Pt4j1sxomASVHgblulHMc5PLaxbuyixiTiemNS+l5wPb9xfjkVhWuLXTafoaQv84BenTiG7o4vCvfTegNXtQr3D7btTAlncalo8v5Dd/63tG/od2+UmeDLNfa3zN/gAm+VgPMyfF+JWA3zegHd0ed0xYa6nBxOZK3PWLpsfQq4XC5r7d7/8r1NyH0KznJZP9n9mRxFfevlOC4cn3Nl3G9f4BbfYbkh15lvdqvIsRcvalw448nct1yIF9zRLS6nvuKzZp7CcufvxIbz+8U1a8pskeDe1eP14kzM5zi+Vus8yvSekx1ub5IrvJgbhdlP5p5k3dragXD+X9kmsaL+x63F/wOCTH64CFvPwY2uHhMu+yjGkionRsA9WtMKLlXu6nAZHzoXBBut7yiTef9dOFOW8bFNdPbRBZ+5l7ooqADcZC9e79ahJ3ABz5gXRPrInO1ed66RLtsZ4dx6weOWtP0ms5wV87c+WYhrmgc5L8Glnl2ZF/vOZzWXuS/QpTDn0qZtudcdPQ1zu/R9/lyfN5+ZoLxIxA22vz/8hMvhy7E/iNOnscbuozzCnDTxGmfsexE1Lc/r73KDa2xT274Hl/0094v6j+e6uM4l5jZBu8bC3czEdspncW0RapWAO1CX+U4ZhgZT2mXP++A2rrd6vHeuyybPY15tem6HepWW+7AJuSVlbh7geInV0QcujsX11uv+3PdKmvjP9XmrQQs4eynjvEnFd7k7Letd6SgT7uQdGeciMi0yGZ91cBwvzsO/BcUu+wpuTAjbeUAE/t3HLODfVnp/5d5cgOG4epepPFzT8syFKx3jWLOL54BbEo3xon5OxoEKRy7AcbYyJsAccBxH3FYGZeYlzwN3uSWvX+lyCdaj81E4a6ZLRdbcgdqVIQZcxRwu+7BCiP+J/mcDIYQQQgghhBBCCCGEEEIIIYQQQoQJfWwghBBCCCGEEEIIIYQQQgghhBBCiDChjw2EEEIIIYQQQgghhBBCCCGEEEIIESb0sYEQQgghhBBCCCGEEEIIIYQQQgghwkSE/1zSOgUVMPItXIGBXeGuHD0Al7N4A1Ymz2C4kDcsdz5XdsfxkdOeKDPMXITrbSy476N47jOX68Wr1Raue5z5cN7+LJcueju455cqwn092QYuRZJ/4ezTheCq2Hy2b7OD4Xy373Uc79/L84z9FapHpJ1whTvFhjthV4GLmjoEbq31lPd95EeXkirc6LQA6mn7CSw3Nz5dOba9yVgPKtncAJa7Q9Uu2PnO25Vi21cd+wEuTe0bcE+GJIJLf3oN3IfVrMfhBkPgSn4YCGd/TQ13ryyvd+HNHMqLvaCS//sdLqg2n8Oky0NnN3ccHjArUOQKzzI59zCWmfLTXEomo4pHtTqCDTeo9HGX6xV2ceHEbNbx9pwNcDk8MsPlvT0SrnT6MXC57V9wr0OD4OygiI7jp41RxMSbyH47v0FpuJR9GOf2vG8JdzH1brgNoewH9ojirMyv6XTjrkHN8IgMF8FmDOmTvimc9/WJcNUOjoM708k5X4+Mw3sOPPsZ7lYtxq18d2vBxf7iCzfvjA9cic5D4dLX2AVntlSiCy+mco5d94lzVnRzEy5fXcave3/xFrEt9r8jhmPt7h1nrpRuu1s8W0KXNDdU5/HsP8v689SlFaLB1ajAHOBAszdw41eUgLOK87kWHKXzMZPgFpnTcKHt2aC+86CMsQc5DqfG47e0fYM5t6QwnNMWthvO689i/7Znse2MF68XuX0zljN8P+FKHeal70bwPR19cguuUrpRcNYSf7joSRl3TrdgVe7YsRzHF+oxHobcZN0S7RoPl+nzILiiLjlVyTTZ4PpZj+BGB++H6+Qynu19fNbux5hrTLnnD/fwPvPmhvcZJ3f/YBvMbPbecWxtYG647dd6uJIrmHt6fmc/jVU6FG55Ir6f5q+gzAtOMca4DJnwwh7ltt5hm55NeATO8xvz6zNfOW89cbnvBsMx1DjGNsexx0T2vZE968ANuuSSe5R/DzXAdxZcVrsM3IWKXNvlZbptLu8fBnc9KfPAE804T+7ayOtZb5PAxYj7Eq5CLfa/pYed76xSC77Xvn+/hst7l4Hg/K6qrFtF7jvYiTbBebThGmRXFj+4YkOhwo3rB73hAuzHcFZUtmG8WpngDn/jXlCkXzz3q9kL9yWu8729K8bz4kdJBRfa9CHc6uWL4c687QhnRWYu89RlPN53C0yPY0BNGMDnb3ydiVXdyMylC6yJxHu8ZBvEycCYZDoNdRzeu1oZRfam5no/1MyGm+DXCa7fWt4yxwnOEYM8tsA9s7vwZJdHCC8C4+SEG/yeud9Yi3syQztz0ynP7tZwy5v78b7t9sDdNLcdxxlHcm/kQNOMcEljMe4FcwoyB88x72jtsrPb2C7FulVaCHf3x0c4OwX3U40pCvPkPV96mTg8s9FS9vnjX3iuPd+Zy3w+fQllulXi89eOxgnMjsyxUc1lq8V+xTX2tFQc82WXvuDJfajCC7sc9yYDv2+GOxCQC64zw6bpUKk83D/7ysGNOtQK7rn5LV6/SYMyT2ZzPk3+bgBcH8O1+MXnHEOPPbmv5PWkA9zJpYz976dwDyX0YVq4rSYKXP/1J+BuhbIjnPjKPv/Yaxvc6HlNHMexhx5DmXcLOZZ3r+H1/+oJZbJxOWOCuew2Vi+Ox6TlOG+4bIuEG+drcNI6M5H94MZP7le1acv2GuIyaQ2Lxr/L2OkKwL2Yc9Vx3LhsOpSxRnBtenBWEbhCNvOdSRXvwRXI4nKPCcz7S3/lXFLB4n7O+uwP4Fp6JIf79qUkXItfdeFujOZcGuDJdW3wNWdeXmAX32sU/4Z0MMZU/Oso3JV8KVjQ9oPKaS3j9QzXPX8ytxHi/4vofzYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBP62EAIIYQQQgghhBBCCCGEEEIIIYQQYcLll73cKXKFv29zLMciuOS7+ZvmF9tmgYtn87eOs7flfT3mVmNdljh/g+74Jf7gVMEC/I7iSyT+0MrbX/zN142Gv9kVEm8B3O0F8+HeuPwuU9qTUGZzZf7WeHOX31G8vMrlB6Crsc5FPrn8ZuJLvt4pv/106/A9Y1GmZBT+lvC+SPwdJPsCf5/OCuLvp6V5wd8WvP6Tv6k3MT3b0+MJf4s0vOh3gr+vOLDlBbiXXfgbSelm8Pca+x/kb5SN/cZ3FJiJv6Ucc5/z9+v9D7n8xr3L7wi7/biQvY6/EfVzE393bFOTpHB1avD3m0LL3oeLcHk3XIjh7zpO+MB+4OXP39t7Eou/GZghUg64O7047mOPc7bLxxNsp2/1asJl60F3rfI5OCuYz+r9bjDcTzsQ7rPl8iPqI/rRhRP30jGWbnjN3/wMSMM2rPydv2P2wfC3nV+34+8G23n525lXCzvjXI4lnDP6leWP0tX4kBDOrGVMz5Gav9FqW/wd2M7P+N52P+DvzW90+bHmDhv5u+ATbnGiSxrA2Le5J39j9FyWlXAx7vflPa4538/wxM9RZuAD9tvr7+rDRe3P3xW0Ej6Fs8s/g9u/Oy5crNDocMb4u7jwoVC0vHCruq+CsxdwHIRUp1sdl2Ojsct9zxiO83Txnb8daffmb8VavTn/B5iWcN+LRYSLnJ/zUkB1zkF5XH6TzrqSjHIkfzMwZCB/O7SV6Q3XxiSAs0vmgSt9hL/VF912SV2rrHMcdr3E3+LemOkA3PJ6Z1mP29t5fbsiXQr+pnXJYP6Gabm5Lg3KcBauWOOYW9oFWc+rZ/iMy9Lkh2vfPTdchsbr4W7P3sHKTHW+qzuXWeTUhqVw/9qMfeU4nM23zD/geizzhMtWl79jadZFhTpRlHlzlrg94EpUus7rxaoDtXBvIbgqC5inxb7rB9dyhfN3MW3DMWQZxvAvPRjjcsblnFijLn93c27Va3A/ejEG/Srp8nuXF13iSDhhXeLvhdqd2TZLXnMtFuLj8vuoW/l7wnYN/j5xY5t5RSfPoc7r9/RHmRyH4vH6lYbA3SzONcNPl35wy89ljbCbucz5LnPh9s7gqU/fM1cfmS49XNtL7C9H+02F83eZKYd3fcsbX3I+xzSrCYrUNg3ggm3mWaZzZ6gR5WfCWUuY4NlbuT9h+JPnfxTrvcs8a7HPG5ux/43FNaDJxHLWl9hwd4ZxH+GIj/NdludrMz1O8feK769guYfLOX91fctf8Z07hGM0/pxGcIl/foF72SgDnJ2Cv2V+/SnvO+s+f9M+Hn/63fTq5pLjMYQYO51zH+Udt2RMWm+u32ono2tX7DZc63rs8xPNarihdg1ej1UxS11ceBHlHedO27j8pnR19uUhU7nuMt25fhy2ijlF/JARcKPnOmPV3Ue8vDWQ8bZPDq6nejTbCZfl7hS46g+KwxX5yDxuaCjng6EeQ+E23+H81c3w3CcpeY+ara7AbXH5d27T+DPdxm71wXEc7RdzsaXeX+G6RvOGy5FyJNzGtwPhrH10A36ycpdu+8BlMsztwotaZ2/CbfBIC/fFYp/PFsp9uMHTuWY7cXQLXPVqzAEe+cVxHDdx2QMv3py5e9A31q1+Pfaz+QPpOt1+AWdbV3hjlz2ZIe+Zpzc8wvXMiRzL4W6V5fWuntkLlzUH90JSnijNcrav4/hDdLf51mWPKxdVp4ncp7EeZITz2PQNzs7OZzVbH9D9QfIWPATnsiQ0g2NfhbvehHvKWVJwUvX15b5mynzMH64nTOI4jhnKfb5MdTjv9hr8Du6aD2NLga9855YXlLGXbOb1qrGffarBnHzGlulwx33/gVv1iwmZT+tycAFtuN/0eT7r0nDPK8fxX6nLoky9iz/hvDjMzHhWwxw3R+Fsw3VojQX8+4bVbozLuS77OUKI/4n+ZwMhhBBCCCGEEEIIIYQQQgghhBBChAl9bCCEEEIIIYQQQgghhBBCCCGEEEKIMKGPDYQQQgghhBBCCCGEEEIIIYQQQggRJvSxgRBCCCGEEEIIIYQQQgghhBBCCCHChGXbtv2fCloWXPPDaeFSvu8CN7TOBbhrBQvBfa32DK5Z//FwBzyCHMe/Hm1Ameur6sL1+BAC93xSXjjz8hSUfdgbrk7TcXCbB/eBCx12Hu6v0KtwWXO0hltZ/hfcxYmsiwmiMl58Z8ae5jyu0R1FYsQ7DbdlXj64yQ2Sw91c+RTuTsRQVs3OAjfF2gfXzU4MF15YccrBVR7NOu6IeoiueSu4nUGP4SoE8L5lHlaA+zdDRMdxPg8O20pBNeDsk6zH4+LpWG4Ex9ntQYFwK8fVgzvYrzPc3bgz4fwjQJloLdg3/Ed7wrl175/P6WLsYrvYHvOdIsVglCmfjXFgX/yRcDvNZd7TbgFX+C+XsbdkKlSTIt3gVhz/TyH5/wTLYgy2tySDSzWadXwctSXPPbiEN+nfFCrFpR08d8hHx/GTvGxTv6ib4Gb41oQrMT8GXLP+X+CSN+c9fNpBmdOx2E6fM/eHa/rPIrjmjffAFZrpA5coYUm4TQ+iwdUu+QNuScYVjuOWj6ajzPBJy+AG784IN3VRebhu1ffDufXaV9k5R9pV1sElGnnR5ezwwS23Sf4kJtzqfB/hWr3m2LjRinNguZ4H4AJdYvOtNk8cx57mDc97cBtu24TRcN2/B8NdWpUNbs789XDtqzDiFgh+B5d1Gdvu7sBdcBVNLDj7XgG4AUPZi67shTIDPuyGO9CorOP4bVnOt6NbvoQ7YkrAsYcaMziEdTs3wBfu9CyOx40u83zboD8X540xJuJBvruSZb7D7bMjwwWU4bl9DvJ5SnI5YNpN53og5tk8juM7TFGMHfEtXAnveHBHdsyD672JQbzpwOJw39Kzbutstsn0Ry71S9UCznrFGBuUkOfe+sm2ex+Zz1bqMsdgqVfOAfKrEvPWAf534KrEiA13zdSBy2Y3hnu1vi3c+HrsE1Mslz7+B7u9VYt1XLiZ/eWqzefLtIDnRnfp30E1+YCb1/DcLdZkx3GCxHy3r54zrq+PkguufoZrcAUuMv6ffAFlvKMxzwisdBguaGp8uIgJOTa+3mVdom4ZAtfhENcceV5NhDv2PArcskjDfjN+KNMwEeeIQK8YcBsjc61ibnPN7pMwAdzh18wDC1Tm/Gx2ZKALJ/K2Zd8LWnAG7tyU/HBfi06Fi52nG1w8bpmYGAW5B/PYWu44Ds7MvGjMjWpwSVJxTDXNwOfKuYtr1hQmIty/e7fBHf/G+2apXRYuZq6qcKcvMRAUC+Y6J8hrBZz14wNc4qRcZ774kN1xPK8Zx0WWZcPhava+AXdoItdbmUNdAjNDkomaim3y9Tvfo/lvW4v/J0Sd9A9cgV6/xwxjJl7lpsTa7OxXITZzul8PuA83Iy1jrpnz2nFoF0mKIhGz8J6BZi1cD+sk3FTfqXCh386yHlZlqE03GdPrZOYkEXsP18pTi+2Ee1CM+cPQSz/hvv3L6tWKkRvuXAnnurCQSx99Y7iOClnC3OZSas4tHsXnwCUN7gXX6BznucGzoYzvyj+4d2PYh9Jb7KN3f/F95PVuDnfeHgOXJTJzhe8vksA9ivnbuNrE/YJ8tbnGrGa49zfQ9oKrmpI56aBPzIvy3+e+hx1/LFw9l7z3YQgntVF9+XeLSN24jj2ZmOuNHp50vvyThEn+227nE5c9uRPLGEMKZ/kGdyE354g8+9/znmUKww0ZcBcu/RieW9jmWAsval9kn0+Yh/twxQ//DVdvLBdiifYegWuWk+vElS953+eTnMf2cpdYsI/9cU5O5g5/J+bftKpd4t77v0+Y7zQ8xHxndc+FcLFucHL/dZT7i7N3dILLWbQb3KBq0+C2XWUbBGdl23lVcZbz2cV9r2/2UrhncaPCJc/A+cBrH/ttsDfjlts/v/6PfyIVQvw/0P9sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJiI8F8LRrVtuGWWBXdozw0424yGs7Z1hxsZj+7hgFFwQ+1Qx3HbL+dRpt6+nXA1jhaE+7LlElyOGnPhrKZ8VnO0D11NlrNjs5j1N525/ovnDuvNc4ewXDIvvp8nlXmLucZ543ZbuqJMdisf3F0q83Ip3+vDtdfgAk0TONsMgYttqsF1Mxd543AiWYP9cI/asp2NVRrq+MtvcAMSxeU9VraHi3N3L9z7cVscx+VCk6HMnAj14CqMbgXXsPZ4OONdFKrbXB+4Hx2ewJ3deAyufp7ZcGu7doKzRveHK2Y4htaZALg3Sb3gzCcqa31bx7FdoC3K7B7M8zwH8fnPjfgJV+wzz/VbnAou+RL2iZcnXPrTn+SzN1Ryn0C4pFXZR0+NWQp3+ntkuEJrVsK9fRwMF8/797Zhv3iXchjctKEsN6BmJrj3DK3GM2QiXLvt9+HGtRgE133PDLjtOSLBbYv7DK5K5yRweW8mgDtXJx7cWYsxN//Qpo7jSt7HUWbwcvZl05dtl/ke42DO63xfVnZPOPsKb7HGug7XcCTLhRcFb/AdnR7CQFJ4+kK42Fdd3uWiKXAlLOY2gw1j6aOYzvZPWQdFjJWmHVyxRvvgLhZnbMlW6Cbc87ZP4aKU+Ar3/aFLXH7Kfpvc9IXL9O4M3LiyjOkTUvMeU+MPhHv5aQTcjzXO9+iz+h3KjE8TAufx8BTcCVMIbqX9D5zZ+R1qwVs+Q1/fNzz3D3O0dE+4Qnl8WdBiP7ps6Lzb8NQaszjnHRydB25mHmeb3fJ0mcj9MkNde/Iebn0N1m2hiQg3IZTzmpX9KO+bsDGU/Xoc3MOJXHMYX8bYdR0fw+ULrQpXKkZWuGHfKsBFXlXecZzN3oUyRWNXguvoERPOLygO3O5ETeFMFOZ8Ji9Vp4kcC2lc+k540X0aXesGjKfGygl1z3CeTbs5EU+tycvFv+zyzI+dbfPK5vxpBX3keXUuQ1U5HBWuts22L1+SeWnQkYe8r5kD9zRXbbglG1mu5dctcGtXloFrMHAWnJ2DnehI7AZw5rbz2ezXLLL1zQa4Gs855yZI1BrO9y/mt81es40LjJ4JZw3IAPcns/zzHeisCFyzPuvOWJ3adIPbmZj96mZBPmG81YwvtvmtXW9wsFiFoYx53BFqzJDUcF2s6HBFmL6acxW4Fi1TYQVc8JFGcBkvM5e+mY73yHV/KpzHTbZdrPdsuz0Ny8JV2+M8bjuB97w0sCTc24lb6Uw0uOzPmKPlSF4A7mtdjpfzGzhHuEwH4Ubcnl3gDhzOAed1jrF0bEPG9O7f/OBmPOGa0jbZ4UrtW+4U7bl3M8glQix1matOmnO8Zz72eesp83fbcH6p3WQMXN4HneEe3WGe23R3LLjG5ZjvjAlln5/H9N2cveAPd/y3NX+dtkwyby7gHkOWaRwcX69xbR8jIfffFleia7nQZf+37P+f7d2YljDruqyFyzFqC9y5tHXhdhqOg7QnGPu6d2W+6bU8l+M4URDHxdek6+HK1GwIZ6YHQe3wY7HlPnwf7eJzoz2i4f5T4Gu+XytkN9y9jjHg1qTxh0v/i9eL6pLK2RZzG8t2bkzV+X3ONMYU/sEB9OwV40re86Fwqc/z35g+Hc+2S7+fz1D413A4Y1wCVTjRLxHrnc9wX3jmX/xbRVqXte69bSfgrGrMY86a3LxvKue6Lue/KGKuDOCeh5lWju6fuzy3Xn24xaX497bEx7jWDb7OPSm/D/zbyp2WieHyp2M8fOjBuiSxmIRbLdmHbu9kf0m2y7mJ3sBegzIeMZbC1fDn+Gm7jXvYJb6wn1zi9r7ZyeWMMc9d+nwSlz8YCCH+J/qfDYQQQgghhBBCCCGEEEIIIYQQQggRJvSxgRBCCCGEEEIIIYQQQgghhBBCCCHChD42EEIIIYQQQgghhBBCCCGEEEIIIUSY0McGQgghhBBCCCGEEEIIIYQQQgghhAgTEf5rwa+b+V3C29Yl4Z7P/wuuXvlPvGDXFVDXhw6C+zx0FVy0mZbj2NpXiecd/wBXs1wwXMdk/eBilzkKV8g0hmt1wIJLHOsz3MevP+GMlQCqj+kEF9l0gRvRk89rDY3Ie5gqMK1+O3ZpJhMrwWA436nD4S60bgL3bONquP37xsHVsO7CFfM6z8r8QQJnlYDLOqsZ3LFCheFWXogNVyWoD1xAhPpwMWv6wPX/4BxrzeN4o0x68wAue5EQuIp9q8ENMS/g9jbpDXdv0EK4RC+uwa2wN8OZLS+h7NEJ4Z5eXgRXZLEX3GPjy3sk/A41a4VznMZrYqPM20eTWLemv+Can+OYvxqDz78l0VO47EnZd648fgJnTHIXF05ELwU1zZ4G18yzPNzpkv3hau3ZBGffjAuX3mMZy310jitreQaUubgWynh1XAx3p9oCuBIP48HdtwLh5rf1h5v4vC5cxEFT4IIM+1q6fUfgUpxNCvd8kCfc01DOYUeDc8LZnrMcx1aTrCjzwcoDF+vgQLiUL1rAjfrI+Nawiz9cpIlQJqAT5/4/SdosAXBrXN5bivGH4P5tkAuu+MHucBFMDJc7M1dIWcd5X6sS4429ay5cuZG8+v53aeEmd4oD5+vJ8X2sNJ8r02HG4LhmD9zX+GPgvO48hJvWIh3cwUzMKZu0jw63IhaUMSlmOI/f8TzzsBuU/W0qXLPInL8aRawBN67kcrhUPhXh+hrGmj9NcYtzHmdPYyLbMSkt5vlBoybDRXx6Eq70j5twdogzFj334j2TNovBegRtgGJGZUzEfZHhbnc7B5etWHG4Tl+2w+VZzRz8Qi/Oa6EfM8Pl6DAM7prZAWf3Y7401HsG3OkFFRzHMw3rFuMn71kolHl+8ki34eYGcw5vZ7hwuLLkPdzeLmz3NDDhxxZOs8ZORjfIVIZL+7IbXOaEj+BahyyBW+DB9ZMxzhzRMqxIs3tcE/vVZj8LjsvrH6rDNXv+lYyv+xdnZ9W+F4XqXGc2XL/LUeB6P+wBt3HaUrgK67nO8e/O+eRdpwZwC7x+mydzc570DG0EV2fTKLg82crBjWrPNcj8JRx78wcwz7eZOvxRrFwp4EoaP7iDhpPqWpt7NzOica8hngmCsxtzz8QER3McMgIbM9lmjttjD3Phros59pLtLwhXbMcPuPw9XOqb/R1cy+ZcD0w3NeFeJfwIN+okc+QK8bhgGd07BVzUmaPhHtnO9VViwz5funhbODOS+zQlXfLb41cYq5tn5pzut6E9HGcvY/K6uPBivGcJuBpJuOcUoRH7sn3nCJwVhblkzAEuN17ADOpQ62zOa7mcZlvr4O7YMeDq+7NV/YYwj2u50u0mnJ+tDZznqqWpATfeh7UukZ7zS9cUoXDTL6WHe7XzDutiMc+yZ3R0lunMOSjBeChj5eZ73bniXxZs5A+V93FfOP+rzGNjuGzr/lGGbIQqN4Nrtg7r+c6/5+OcXeVGVbh7pTlH7PLnQFg519nnhzVk33tiesI9bN0QzmKaYDyqc/8lZgquBY7d5b7SlkzsMDPa8mXap5mPNbrPfOd6Ft432+ZMcCExue4J9NgG99w4195JpudGmbk7uXczaMdhuAizuTc9ugPXFXVvQRkrmGO5kxfjwCyY8CNfQtanqC/f24uj3Hu9l5QxPdhwz6RsXP49KN+756xMQeea6Eor1i10G+ddj5n8u0D5Qyy39yJv2eo4Y9WXiuzfXiHMY8x+xkiuLIzxNcy9qo0vBHfzNc9tFD8HXIYEXeGe2H87jq0v3EPbw+0ycy36ELgOS7lPfORvvosb69jG0Uox739V/Pe/mhmTkNtZQoj/B/qfDYQQQgghhBBCCCGEEEIIIYQQQggRJvSxgRBCCCGEEEIIIYQQQgghhBBCCCHChD42EEIIIYQQQgghhBBCCCGEEEIIIUSY0McGQgghhBBCCCGEEEIIIYQQQgghhAgTEf5rwZ+9H8PdeZACrrdVAc7X8xNcnWW8xznvEnDRfCqxYJcujsNb9ioUiWPlgSu0zxNujJkAd79KEFzHQC+4sUd/sG4jokPZg6bDFchqw40qfg5uZB+2p9fAUN532t9QxQ6w2PG0luN4ayXW48h8tsmaAAsuvcVz+5hucEe3s77XKqaH27/7EJwxZVxc+HBl+2i4PnkLwU1ulwju7f1/4YrlTwpnv80M9/5pFrhVcWI6joMvx0KZx8v84SoUYZ/P3RzKTFnGug3PFAPOXnsXLtnaNHBPG9WEs0p94PVcXnm2am/hTkcrBZcpOcffoCdj4Dq1eeY4XrKa9fDf2gPuS582cMuqfYGL0Jdj3vK/Apev7hw4E599zLynCjeiX4fKZH7BfXUJQbX28VkCPJrB5SiVEa5737/g0i5xxpzbtXnPDInYv5e2XwdX6NduOHsN+1nKkYxzdqGpvHGph3S+G6Dqzv4Gl7kZv/HbXJKXszbPh9t1KRhufZ1XcOm/rXYcfzg4G2Vir+A9V0boCffYxIQbXZMx72Z71qPXgca8yaxRdDML0oUTE0wLuOFT2a+6xs0Bd816DmezC5lYwf5wdwpxgn5yy3ly8V1lUaZp/H1wNd+w3L4tJ+F698gOV29WAbgUA0PgvA0f7FN05gBLv0yEu5qeuY3VLCvcIy/mO2mnvIYLbsa69I6R0nF8Oh6DVJ5BHI9Rgpg/fu1YA86y/oFbv28A62ZdhjMmnYu75+LCjyCb746ZrzFlTD+47/fGwf0dpR7clz3l4UKjM4ewvvRxHNtxWDd7KPPS9DvqwHUJ5fXTpvOHy8RwZV7mvQKX+AzjVajNudyKw/5cLhbzwGvTmGuNfc3n9ejLecIewD7dP285x/HGIShiUvwYDHfSJbU+vr89XLGTAXBpih6BKz3CBy7u33yuP8llw1hnVp2GGvmUxWqNYMNmfpcazi96K7hHK5kjpvrlzLUWx2dbtXxbEa7JjGdwGWcdhhuavTRcqDdzigODIsHVMEvhun7dCNf30064E1/HwxUZyfVz56qMDTHjMXezK7DtAhI683DvAjFQpqLNfYfYFp/fjBkINTkj4/VX0x3OO9JXuLWneIsGXDaGI9y7mX72AlzWfMz9U1st4XLcXgJXIDgZXAubfWOZVdRxHGr1R5l5cZh7HWRXMT5NOfaWR2bf+2sJ62HHYV++ULM33JILieFmTjsKd7Er67cs7ne4Tgvrsy7N6sJdHc/Y/8rq4Dh+YZi3WFMawF2+Hg8umMsBUyj1FLgT65jfjF/C9zNrMtfKf5KsIVx3eJV1ycuKV4WyG3Fi/Go+wkWt1w7Oaj2X97X8HcefmFqbX5eYv9cwfIYFdf3hRs7nOy/VjnsjVu2fcK/PXISLnzUXK3jdZVOiJHOK6fOzweW0fFmXStxTsFtV4z26OPeR7K6MyyGxmEd/ScGAGzM796Q+Puc6ImuSvKxHqm1QL9ZxrZzI9OK54YRd3B+u7FTObSl/cC72rcsYmfEi85F0iaLABQxhO4z54fzTwr1Q7lvkuXcf7mKncnChv5gvp1jHvRbLZX0a/HoGXIT4vO9C4w23KSdzkTXpXDZqJvO+L0LZdtYU7h3arXfBxbVKOM+zp6FM+1hd4PLcZT3qNToPV78g5/nLmbn/OeZwVLhZPgvgTMCfy/HvG947zVO2gxWrA1yi79wAexllGFzBCkngMgRyT+JZsLMudlHOu+2Lca4/3JLPMPIRn2FTT47beoYxM8purhHtyy7/rrgIY5qVmmv4H0HMezs8Zp0/TmKdfxqXOffiHqghEfI5jl9M5z5VitArcInnuezXHnsE92J5Mbi5FznmhxXkHOnrtikihAgT+p8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBMR/mtB74dZ4eLcbgB3IKQXnP2D13sZxYJLTGVMEhvq4fMYjuOMphzKpF/SFi7VPwvhWv7L6y+97cV6BAZD+bbns0Yxt+Fqdu0CdzrkMO8xvSTrYv0L9/jHCbi2l7fArWkzGy5J3D6O4zPHZqBML78+cPd2JYerO5AvbMwItufqAix3cheUWWdVg+thyrBgOOFTsyDcsuB3LHgvLlTF+ZXgAuKzbYzPBKiSIePgyg556Dhuk7MDrzVuPpS9OgSuaBue+9Bm/7bfbYMbEP883Pji7I+BPT7BHagQxHv0Zt8YtJ3tlNHeynPHu4yhPv2hRsR2jufuAazHlpKsR9K5oXAzzFS44Ku1WY+SHMvW3Hxw8z7wvpZx6SfhRL3PjKUjWuyDs71Z71iJGOi/ds8I1+sXv3F7sWci3LwKznaI5jI/zLW/wbWv95gFl0WCqtSb42VbWZ5qHWlDGac7lN/1p3Bbzr+ECzEr4ZKfGAaXr+JfcEVar4DrvzURq2dmOo47L+mGMibwLVRCOwbLubT7y2g94B6W55xb3xyC2+xywZp/sM/Hs5fAzbTozFbWu8hh1rtLaZ66zn8uXPpzXVnw3TTHYRuL/XapYT/r8MgPzqrJeaRfpDdwS36xGrOXelLO4LNGuP0cLlPiAzz37QCoFoZ9PpUZBGfv6QgX6MkcLaJxjvtmLv3s5gj2+XHD6dLOZb5zbinjfN4WJ+EqWlNd6lYALhAmfGnF8GLOjac7EIdzeYR07Auz33G+bHugHpy1ivFqQ9V4juOStfegzJUBjeAqjfoJ12flcbifk4fAVay6Hy6h1QnunOclXq/9PTg7Ocf4mfds0HIW56fWy1mXvhd4vZefPsLdPOg891hpzk19mOabjzHprnscgxv+/W+40iYhT85AlWR5Uji7xzMWDCeiJz4DV59TtCnNZjZNYzFeVXS5x3qb8Tne7Flwb306O45b9t2MMluHxYBb2WU33Akv5vRXT8+By2xawJ05OQUu+H4OuCstT8GtDr0Jt8olX0hWoTFclYYseLcyz7WqMsczkaM4Dt/drI8iMSyXDnn8LlTrovnh3nS9xXPNGBjvgC8sVtilAew/l98Yw7nydn6ugaIZrncfuuxn/FjTDS7PsJZwSz3ZDks9nO2QMoTrNT+zEW7RuTpwWYIZ03MuYT+b6JcSLsDjClzMaIxLzavyva1YzTlnWx4o87kp54iYy3PCHTQcz2UCP8Nt/e3fBgVf/Y4yD7Kxz6cO2guXecR0OPtBO7iL6WLAzW7KsbbK4rtu/Adz+vQueWT0v132+jaw3pG6sS+vf81ydhb208dWYbjRWZ15UcwiLu1yijmWXZJ7Q6FHbsD1NVfgirZgP0scmbftTGU62UXgpljMC7ZM5pxmTWH/y2VGwl1q4gPXexXbJaZ55DgeEHIdZW63qQGXafYROJ+I9+HuTJoH1z4W1/tpo+eGS2Bzf8zY3BMOL5aXzgb39tM1uN5TXPacorPtn1ThfvmzmwngIg7aADfk9/1Ej8ko43+C+cm9A2Phkp/m3tAnlz1ll600s9eKRZm4AlSWF9yvDWnHPceBTMlNmqFsuxZMFYzdrS5c1EXMIOea6I7j7hav7/ankssu7kZqxpUZH/h3m26d/HmPWJyXDuxeAOey3RFupLH+gXv4jO1lL+bGXnOXcgdt/r3luOF61bK4JvKImcp5XIRvqcZj7q+WCE0Gd+Up67bPDoALOe4Nl8xKAXfEXIDzdNm7+BiBm0HNBzD2lzZ94Za+5Hw1eyj3OqvmZu6xfbaz7cwm7pecWJQdLp+ny4CcEgWqqNUeblsI96GHd+X7rzqRuee2P5jbCPH/RfQ/GwghhBBCCCGEEEIIIYQQQgghhBAiTOhjAyGEEEIIIYQQQgghhBBCCCGEEEKECX1sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEmLNu27f9ScIWL2x78Da7avqhwTSvx3C+1isCl2Hwc7qNh9SqYso7j3WY7ylSsGxnOewPr8aYf3TmX+n7q6XJuh4hwbertgDv8pRzc4rIt4FrdTAu3vOlPuHxXRsD9vM76LbbrwX2yNjqOH18ORZkTQ9vCRdn6FG672QOX31hwe1K2g6vVYx5cjSEx4LZ8+AQXXlgP6ew26ykP1ee5fdm/Q8enh0sVfyHczDdsw8rmvOM4s8nDehhvmJgmAC4Lu6Px3Uc3ynAgDIrhD9fS/yTc5eF8hguDo8PFmNwLrlbvc3BRgjnWUjbYxLqsD4ZbaJ47jjM2nIIyt4uwASLUvgXXPX5DuMlJ18ANeBYDrqP9GS5qa8a36OwS4cZ3iwHR12Z8PRm6Ga7UlRJwh3K9giv2KhAuU6LncCerF3QcR9v0GmVMzg9Q1rXYLJd0OlSL59XhlsRdBOexkvE2DkOk6b2L5f6++xecd61EcHaedHBvvt2Hq1mZ8bp7DFam3qCJTtGT48dOlxTOtHNpT7MALqlpA/c09Vee+zkG3LX3bPesNsdyeFHEbIS7PdsT7kOnmnARXOa7tze6wCXOwv7XyOXclmOcfbx+vzQocziA83o0b/bb8cmhzN1YvOeayy4xKBLL2TVZzprfB+5XVM7ZkUJ7wJX0yAh32OoK93cyxusZazk3eZXa6xShrG/gryA405xjw0q6DM4exbklxOV73VOdhsPFmP0YLut/S73/7+C0bZolSQy3LsVLuECTCa7wVc6XF/J7wVn9mUPVHOLMEdfYb1DGNkPhGj8vAbc6RXO4JyGcE5LZl+FuwxiTcUMpuIhNuFZ5H8B5zdt05wWnHYaKlKMx3JlWw+AKPPwOF2qKOo6vHWIsrZEgLly8THfhzhmOyTYu/XSBYS5rmzt0GxlHrDp/rt8n/5f1+fmQc+rbXhzXltun+e27QdlzGa8y1GbPurPpieP4YISKKFN6G+emdRU5RhuEuCTw0Weybpnewe05HwWuks31XjyLsf7J7ZhwkTJMYF22ctERI3tZuEE9e8PdSMEYUnOyM45XPzQHZeySxVgPw7FcMgbzyrX+nP/jH+Z7Nd5/Q+Wfy45ydhn7WHixw2yBy/69KVwm3xJw33LyvdnF+MyeTG/M4kqt4NLuzO843l+aeeSQg0XhzLgLUO0GsNjhdr/gKi3junjaj7dwtokGx2hhjI9N+8NyiWntqdY/YLl6B1i/lTbX7U3SO/vV99eLUcb3MecRu6TLHHytKtzmqFwP1Kw1G+79siRwMXtx/eY54c/F+Vf9J8MlCmXfsPcybk4MOAr3/CrvMTXSEbhqcZkrfR5U2XGcwZ6EMvMOcxzcGcX3kTFnSzjP9lxjrnzIMXRqZwG4suY0XDWmXsaO5/IuL9+EsryY2/zYthLOpxnntQ8JksE1/y2l3JmdOZbdjrF60lzmO9Utrm2jnWesXpmWz/oyOts4tyfHX8OQP9fnYwyKBzf950i4iCuewDX+ORouuj/XniNiMC6FfuWc2uWbc/9h/46OKFO2BvvPe++dcEUN84RVNtdTsTKGwOVPAWV2jOI7GpCL5aZc4PyVsjDX2acDP8K1NrHg/DxccuFQl/m1kDM2pznJuSqC4b7pnbI3eP23heFWXOPzN/Fh3U4xfTSFEtKZin+uz1vRT8BV/cz9l/sZrsD15nLF/HWK7zLWKr7LjyX96EZddByPu1IbZRq5NFX25qvgWu/fCrewN9fN0bmtYupH5t7I/B8cV70+c39xYnH2q+i9Z8F9acwbP9nJP7AlqwxlrHF09rprzjIXs7KMWzbmsoexpRkbuTZDvwn5TOkdGgduQfotcE3vcQ9TCPG/0f9sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJiI8F8LNrUqwj2Ltweuyds3cINaJoUrU34j3Eff03A/6pSEi1l9luO4vWGZ3Z2hjOkaGSpvkh9w29tPhIu+gU1lZ05D13ItXK0yN+A23cwC91eWr3BHZkaFS9XjPdyVa5HgppsacObyeudxzgcokty0gntp8sNFN7yn2ct75i4/D65Kbxsu5c9KvN4fxC7HvmEt6wQXauVmObs23fhRcKV3FYBbGMK6lIqV0HF8M01DFuodDPV9/AG4KFYZOJuvw1hpA+E8HpyCW2TPgLttPYFr7Jsc7ltlDtRo3bfBhf7dnPW7Pw0ufyjbYIhx3rdPFb6HXh3Pw73vaMEluJYObnJKxrfRCT/DmSyfoJZ1DYVr/ge/AUtsj4X7vHgdXO1WFeAC7Vdw+acVhTvV1Qsu+tZUcNGqOWNO9gYoYq5+qgxnbzoLN/gZzx2WlWPj0s39cKFD2sHVmDEXzovFzN6X11i/ApxLR426BTepFNtpzf1DcOXSxoCbkMPZBr3Kl0eZARH7wQ2LPxVuwfs2cK1rNoVbXpFz1eVWnKuyvooN9yc54VsI7kDbxHCWWc6T7dVQses1cyk3HWrRfRbrn+6t4/jFBOYE5kMjOsO8a7xffLi9DGkmuhkHZ//ihHBtF8+1Y3AeuteSeVHaRewb95e6VGYzbzI7HuehD8vZLtMuOes8dtc5lBnuUZjXd5n7ngT3pTSMbzVS8Bm2p+UFF3xhrpTV5Q7hycvezMGXn35Jl3AMnPWDScoJX+Z5lrcPXPVuM+ECBv/Wf926Rut9cLUXfoFbFcDcY0ExXrCN1QcubdB43njfMagDo7rBFSqWD+7GKcaH7M9YP+ORF+rRX5yfavTi3GZFdNaldISjKPO9ch04Pzs93PkUHFc/A3zhfNKzPUOfMPfaMY6TYjWY8KNdRI7NAX16smCfm3RDM0MlsC/C7XHpvPE3uVQmhXMcfO7Bum1kqmDqeV2A6xo4BO717NdwVvMEcN9+8Z1vnXES7lm29XBLM25lBQtzPtnUlnHXP2EvuFzjmFe0T8N5rOgk53GIS7wIycMIO/Ah2+SIP0+OFbEFXPOgrnDLvv0Nd3Ykc/o/SZXBNeCskd/gHsXYAffqo1uuwbkx8zSuG5pHcHkpxhkPC5drixIJq8yHa7vzONx8wz5qzx4JZ91LBjfwQHe4oIMcf7nKMFpdtLbDGZMSJrRATrh6c9kmlifXj6EW38/PF871Y+SzXBPXYxgw6x/xuRZN5pjKfp2xwXKJW7bhYirRhIdwzCTCj1ylGdPtJYxLd1scgYuxk/sPvfoNhLPsYnCPLK7dU3Zxjo251nCU8e/IHDcDX4exb1anrMQ1dv/7GeHi+fKNVGY3M7bFvc7l+abAVTzH2Gfbt+EOZme5Mo+S8NxFzEd22Hcdx/W+cU1c/S7P65aUe7gBrTn24uXh2OhpJWLdzFS4yo+XwLnsyoUbXiM5dzYzzL/6uSy/T4zrD1dwymg4i9tk5msGtusHb+eeaLmhzB2+T2KfitOwPZzfshxwueZehUu6g3nqm0VQxuTiujCvxfklVmbGdN9AnluwFfN+v0Vskzg+CeGM4Vxn3qVwHN7uzhwzxZRocNaBWHDtbK41mmbhfm0TbneYQh85v9x2OZeRJvwYtYDvqL91Bc6yY8AVtfzhGi2fDPdxFv/2YSKwXWMFO9eiGSzGluwW90FuNOW1bp+Zw3vm5Rj64rJ3kbOgS1A/zb3nEVZauAmbGUM+3ygF52O4eZXMn9Evt+HfC4r39WP9Qpz7BNFq8W8vdzfzNB/DNVnvrhx7pT+t4Mmp+KyrPTlv1Nx7mecal3lYCPE/0f9sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJiwbNu2/3QlhBBCCCGEEEIIIYQQQgghhBBCCPH/HfQ/GwghhBBCCCGEEEIIIYQQQgghhBAiTOhjAyGEEEIIIYQQQgghhBBCCCGEEEKECX1sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBP62EAIIYQQQgghhBBCCCGEEEIIIYQQYUIfGwghhBBCCCGEEEIIIYQQQgghhBAiTPz/AL8updm1BrGHAAAAAElFTkSuQmCC", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "test_mse = (output.detach() - ground_truth_target).pow(2).mean()\n", + "feat_mse = (model_bespoke(output.detach())- model_bespoke(ground_truth_target)).pow(2).mean() \n", + "test_psnr = inversefed.metrics.psnr(output, ground_truth_target, factor=1/ds)\n", + "\n", + "grid_plot(output, [validloader.dataset.classes[l] for l in labels_target])\n", + "plt.title(f\"Rec. loss: {stats['opt']:2.4f} | MSE: {test_mse:2.4f} \"\n", + " f\"| PSNR: {test_psnr:4.2f} | FMSE: {feat_mse:2.4e} |\");" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/src/utils/gias.py b/src/utils/gias.py new file mode 100644 index 00000000..df86d88a --- /dev/null +++ b/src/utils/gias.py @@ -0,0 +1,72 @@ +# /////////////// Gradient Inversion Helpers /////////////// + +import inversefed +from collections import OrderedDict +import matplotlib.pyplot as plt + +# based on InvertingGradients code by Jonas Geiping +# code found in https://github.com/JonasGeiping/invertinggradients/tree/1157b61c6704df42c497ab9eb074c75da5204334 + +def compute_param_delta(param_s, param_t, basic_params): + """ + Generates the input value for reconstruction + Assumes param_s and param_t are from the same client. + + basic_params: list of names present in model params + """ + return [(param_t[name] - param_s[name]).detach() for name in basic_params if name in param_s and name in param_t] + +def reconstruct_gradient(param_diff, target_labels, lr, local_steps, model, client_id=0): + """ + Reconstructs the gradient following the Geiping InvertingGradients technique + """ + + + dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None] + ds = torch.as_tensor(inversefed.consts.cifar10_std, **setup)[:, None, None] + + config = dict(signed=True, + boxed=True, + cost_fn='sim', + indices='def', + weights='equal', + lr=0.1, + optim='adam', + restarts=1, + max_iterations=8_000, + total_variation=1e-6, + init='randn', + filter='none', + lr_decay=True, + scoring_choice='loss') + + rec_machine = inversefed.FedAvgReconstructor(model, (dm, ds), local_steps, lr, config, + use_updates=True, num_images=len(target_labels)) + + output, stats = rec_machine.reconstruct(param_diff, target_labels, img_shape=(3, 32, 32)) + + grid_plot(output, target_labels, ds, dm, save_path=f"gias_output_client_{client_id}.png") + return output, stats + +def grid_plot(tensor, labels, ds, dm, save_path=None): + tensor = tensor.clone().detach() + tensor.mul_(ds).add_(dm).clamp_(0, 1) + + fig, axes = plt.subplots(1, 10, figsize=(24, 24)) + for im, l, ax in zip(tensor, labels, axes.flatten()): + ax.imshow(im.permute(1, 2, 0).cpu()) + ax.set_title(l) + ax.axis('off') + + if save_path: + plt.savefig(save_path, bbox_inches='tight') # Save the figure if save_path is provided + # plt.show() # Show the plot after saving + +def gia_main(param_s, param_t, model): + """ + Main function for Gradient Inversion Attack + """ + params = model.parameters().keys() + param_diff = compute_param_delta(param_s, param_t, params) + output, stats = reconstruct_gradient(param_diff, 3e-4 , 1, model) + return output, stats \ No newline at end of file From 9ddcbb1d57d3b994ea8ff27debe09a288c7d8972 Mon Sep 17 00:00:00 2001 From: photonshi Date: Sun, 27 Oct 2024 06:10:29 +0000 Subject: [PATCH 04/16] adding support for 10 image dataset --- src/algos/base_class.py | 39 ++++++++++++--- src/algos/fl.py | 20 ++++++-- src/utils/data_utils.py | 106 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 153 insertions(+), 12 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 3cbbf8f8..04c8e09e 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -19,6 +19,8 @@ get_dataset, non_iid_balanced, balanced_subset, + gia_client_dataset, + gia_server_testset, CacheDataset, TransformDataset, CorruptDataset, @@ -101,13 +103,14 @@ def set_model_parameters(self, config: Dict[str, Any]) -> None: optim = torch.optim.SGD else: raise ValueError(f"Unknown optimizer: {optim_name}.") + if "gia" in config: + optim = torch.optim.SGD num_classes = self.dset_obj.num_cls num_channels = self.dset_obj.num_channels self.model = self.model_utils.get_model( - config["model"], - self.dset, - self.device, - self.device_ids, + model_name=config["model"], + dset=self.dset, + device=self.device, num_classes=num_classes, num_channels=num_channels, pretrained=config.get("pretrained", False), @@ -230,8 +233,25 @@ def set_data_parameters(self, config: Dict[str, Any]) -> None: train_dset = self.dset_obj.train_dset test_dset = self.dset_obj.test_dset - # print("num train", len(train_dset)) - # print("num test", len(test_dset)) + # Handle GIA case first, before any other modifications + if "gia" in config: + # Select 10 random labels and exactly one image per label for both train and test + train_dset, test_dset, classes, train_indices = gia_client_dataset( + train_dset, test_dset, num_labels=10 + ) + + assert len(train_dset) == 10, "GIA should have exactly 10 samples in train set" + assert len(test_dset) == 10, "GIA should have exactly 10 samples in test set" + + # Set up the dataloaders with batch_size equal to dataset size for single-pass training + self.classes_of_interest = classes + self.train_indices = train_indices + self.train_dset = train_dset + self.dloader = DataLoader(train_dset, batch_size=len(train_dset), shuffle=True) + self._test_loader = DataLoader(test_dset, batch_size=len(test_dset)) + print("Using GIA data setup") + # Skip the rest of the function since we don't want other modifications + return if config.get("test_samples_per_class", None) is not None: test_dset, _ = balanced_subset(test_dset, config["test_samples_per_class"]) @@ -476,8 +496,11 @@ def set_data_parameters(self, config: Dict[str, Any]) -> None: """Add docstring here""" test_dset = self.dset_obj.test_dset batch_size = config["batch_size"] - self._test_loader = DataLoader(test_dset, batch_size=batch_size) - + if "gia" not in config: + self._test_loader = DataLoader(test_dset, batch_size=batch_size) + else: + test_data, labels, indices = gia_server_testset(test_dset) + self._test_loader = DataLoader(test_data, batch_size=10) def aggregate( self, representation_list: List[OrderedDict[str, Tensor]], **kwargs: Any ) -> OrderedDict[str, Tensor]: diff --git a/src/algos/fl.py b/src/algos/fl.py index c8a515a0..44852fad 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -13,11 +13,14 @@ from algos.attack_bad_weights import BadWeightsAttack from algos.attack_sign_flip import SignFlipAttack +import pickle + class FedAvgClient(BaseClient): def __init__( self, config: Dict[str, Any], comm_utils: CommunicationManager ) -> None: super().__init__(config, comm_utils) + print("WE ARE IN FEDAVG CLIENT") self.config = config try: @@ -47,8 +50,8 @@ def local_train(self, round: int, **kwargs: Any): time_taken = end_time - start_time self.client_log_utils.log_console( - "Client {} finished training with loss {:.4f}, accuracy {:.4f}, time taken {:.2f} seconds".format( - self.node_id, avg_loss, avg_accuracy, time_taken + "Round {} Client {} finished training with loss {:.4f}, accuracy {:.4f}, time taken {:.2f} seconds".format( + round, self.node_id, avg_loss, avg_accuracy, time_taken ) ) self.client_log_utils.log_summary( @@ -214,7 +217,7 @@ def test(self, **kwargs: Any) -> List[float]: self.model_utils.save_model(self.model, self.model_save_path) return [test_loss, test_acc, time_taken] - def single_round(self): + def single_round(self, dump_file_name: str = ""): """ Runs the whole training procedure """ @@ -222,6 +225,9 @@ def single_round(self): # self.log_utils.log_console("Server waiting for all clients to finish") reprs = self.comm_utils.all_gather() # self.log_utils.log_console("Server received all clients done signal") + if len(dump_file_name) > 0: + with open(f"{dump_file_name}.pkl", "wb") as f: + pickle.dump(reprs, f) avg_wts = self.aggregate(reprs) self.set_representation(avg_wts) # Remove the signal file after confirming that all client paths have been created @@ -235,8 +241,14 @@ def run_protocol(self): for round in range(start_epochs, total_epochs): self.log_utils.log_console("Starting round {}".format(round)) self.log_utils.log_summary("Starting round {}".format(round)) - self.single_round() + dump_file_name = "" + if round == 0: + dump_file_name = "/u/yshi23/sonar/src/start_reprs" + elif round == 1: + dump_file_name = "/u/yshi23/sonar/src/end_reprs" + self.single_round(dump_file_name=dump_file_name) self.log_utils.log_console("Server testing the model") + self.log_utils.log_console(f"server test loader length is {len(self._test_loader.dataset)}") loss, acc, time_taken = self.test() self.log_utils.log_tb(f"test_acc/clients", acc, round) self.log_utils.log_tb(f"test_loss/clients", loss, round) diff --git a/src/utils/data_utils.py b/src/utils/data_utils.py index 0dde7183..dc89db0e 100644 --- a/src/utils/data_utils.py +++ b/src/utils/data_utils.py @@ -365,3 +365,109 @@ def non_iid_balanced( clnt_y = np.asarray(clnt_y) return clnt_y, clnt_idx, cls_priors + +def gia_client_dataset(train_dataset, test_dataset, num_labels=10): + """ + Select random labels and exactly one random image per selected label from both train and test datasets. + + Args: + train_dataset: Training dataset object with __getitem__ returning (image, label) tuples + test_dataset: Test dataset object with __getitem__ returning (image, label) tuples + num_labels (int): Number of unique labels to select + + Returns: + filtered_train_dataset: Subset of training dataset with one image per selected label + filtered_test_dataset: Subset of test dataset with one image per selected label + selected_labels: List of selected label indices + train_indices: List of indices for the selected training images + """ + # Get all unique labels from the training dataset + all_labels = list(set([train_dataset[i][1] for i in range(len(train_dataset))])) + + # Randomly select labels + selected_labels = sorted(np.random.choice(all_labels, size=num_labels, replace=False)) + + # Process training dataset + temp_train_dataset, train_all_indices = filter_by_class(train_dataset, selected_labels) + train_label_to_indices = {} + for idx in range(len(temp_train_dataset)): + label = temp_train_dataset[idx][1] + if label not in train_label_to_indices: + train_label_to_indices[label] = [] + train_label_to_indices[label].append(train_all_indices[idx]) + + # Process test dataset + temp_test_dataset, test_all_indices = filter_by_class(test_dataset, selected_labels) + test_label_to_indices = {} + for idx in range(len(temp_test_dataset)): + label = temp_test_dataset[idx][1] + if label not in test_label_to_indices: + test_label_to_indices[label] = [] + test_label_to_indices[label].append(test_all_indices[idx]) + + # Select one random image per label for both datasets + final_train_indices = [] + final_test_indices = [] + for label in selected_labels: + # Training dataset + train_label_indices = train_label_to_indices[label] + selected_train_idx = np.random.choice(train_label_indices, size=1)[0] + final_train_indices.append(selected_train_idx) + + # Test dataset + test_label_indices = test_label_to_indices[label] + selected_test_idx = np.random.choice(test_label_indices, size=1)[0] + final_test_indices.append(selected_test_idx) + + # Create final datasets with exactly one image per label + filtered_train_dataset = Subset(train_dataset, final_train_indices) + filtered_test_dataset = Subset(test_dataset, final_test_indices) + + return filtered_train_dataset, filtered_test_dataset, selected_labels, final_train_indices + + +def gia_server_testset(test_dataset, num_labels=10, num_images_per_label=4): + """ + Select random labels and exactly four random images per selected label from the test dataset. + + Args: + test_dataset: Test dataset object with __getitem__ returning (image, label) tuples + num_labels (int): Number of unique labels to select + num_images_per_label (int): Number of images to select per label + + Returns: + filtered_test_dataset: Subset of test dataset with four images per selected label + selected_labels: List of selected label indices + test_indices: List of indices for the selected test images + """ + # Get all unique labels from the test dataset + all_labels = list(set([test_dataset[i][1] for i in range(len(test_dataset))])) + + # Randomly select labels + selected_labels = sorted(np.random.choice(all_labels, size=num_labels, replace=False)) + + # Process test dataset + temp_test_dataset, test_all_indices = filter_by_class(test_dataset, selected_labels) + test_label_to_indices = {} + for idx in range(len(temp_test_dataset)): + label = temp_test_dataset[idx][1] + if label not in test_label_to_indices: + test_label_to_indices[label] = [] + test_label_to_indices[label].append(test_all_indices[idx]) + + # Select four random images per label for the test dataset + final_test_indices = [] + for label in selected_labels: + test_label_indices = test_label_to_indices[label] + + # Ensure there are at least 'num_images_per_label' images per label + if len(test_label_indices) >= num_images_per_label: + selected_test_indices = np.random.choice(test_label_indices, size=num_images_per_label, replace=False) + final_test_indices.extend(selected_test_indices) + else: + raise ValueError(f"Not enough images in class {label} to select {num_images_per_label} images.") + + # Create final dataset with exactly four images per label + filtered_test_dataset = Subset(test_dataset, final_test_indices) + + return filtered_test_dataset, selected_labels, final_test_indices \ No newline at end of file From ee9b24abe1d384678439dfac9d84613d0c4c8277 Mon Sep 17 00:00:00 2001 From: photonshi Date: Mon, 28 Oct 2024 01:20:00 +0000 Subject: [PATCH 05/16] added gia for fl --- src/algos/base_class.py | 323 ++++++++++++++------------ src/algos/fl.py | 37 ++- src/configs/algo_config.py | 34 +-- src/configs/sys_config.py | 29 ++- src/utils/communication/comm_utils.py | 7 +- src/utils/communication/grpc/main.py | 5 +- src/utils/gias.py | 23 +- src/utils/model_utils.py | 1 - 8 files changed, 253 insertions(+), 206 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 57aa9da0..ebec2ef8 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -254,6 +254,10 @@ def get_model_weights(self) -> Dict[str, Tensor]: Share the model weights """ message = {"sender": self.node_id, "round": self.round, "model": self.model.state_dict()} + if "gia" in self.config: + # also stream image and labels + message["images"] = self.images + message["labels"] = self.labels # Move to CPU before sending for key in message["model"].keys(): @@ -404,176 +408,189 @@ def set_data_parameters(self, config: ConfigType) -> None: assert len(train_dset) == 10, "GIA should have exactly 10 samples in train set" assert len(test_dset) == 10, "GIA should have exactly 10 samples in test set" + # Store the images and labels in tensors, matching the format from your example + self.images = [] + self.labels = [] + + # Collect images and labels in order + for idx in range(len(train_dset)): + img, label = train_dset[idx] + self.images.append(img) + self.labels.append(torch.tensor([label])) + + # Stack/concatenate into final tensors + self.images = torch.stack(self.images) # Shape: [10, C, H, W] + self.labels = torch.cat(self.labels) # Shape: [10] + # Set up the dataloaders with batch_size equal to dataset size for single-pass training self.classes_of_interest = classes self.train_indices = train_indices self.train_dset = train_dset - self.dloader = DataLoader(train_dset, batch_size=len(train_dset), shuffle=True) - self._test_loader = DataLoader(test_dset, batch_size=len(test_dset)) + self.dloader = DataLoader(train_dset, batch_size=len(train_dset), shuffle=False) + self._test_loader = DataLoader(test_dset, batch_size=len(test_dset), shuffle=False) print("Using GIA data setup") - # Skip the rest of the function since we don't want other modifications - return - - if config.get("test_samples_per_class", None) is not None: - test_dset, _ = balanced_subset(test_dset, config["test_samples_per_class"]) - - samples_per_user = config["samples_per_user"] - batch_size: int = config["batch_size"] # type: ignore - print(f"samples per user: {samples_per_user}, batch size: {batch_size}") - - # Support user specific dataset - if isinstance(config["dset"], dict): - - def is_same_dest(dset): - # Consider all variations of cifar10 as the same dataset - # To avoid having exactly same original dataset (without - # considering transformation) on multiple users - if self.dset == "cifar10" or self.dset.startswith("cifar10_"): - return dset == "cifar10" or dset.startswith("cifar10_") - else: - return dset == self.dset - - users_with_same_dset = sorted( - [int(k) for k, v in config["dset"].items() if is_same_dest(v)] - ) + print(self.labels) else: - users_with_same_dset = list(range(1, config["num_users"] + 1)) - user_idx = users_with_same_dset.index(self.node_id) - - cls_prior = None - # If iid, each user has random samples from the whole dataset (no - # overlap between users) - if config["train_label_distribution"] == "iid": - indices = np.random.permutation(len(train_dset)) - train_indices = indices[ - user_idx * samples_per_user : (user_idx + 1) * samples_per_user - ] - train_dset = Subset(train_dset, train_indices) - classes = list(set([train_dset[i][1] for i in range(len(train_dset))])) - # If non_iid, each user get random samples from its support classes - # (mulitple users might have same images) - elif config["train_label_distribution"] == "support": - classes = config["support"][str(self.node_id)] - support_classes_dataset, indices = filter_by_class(train_dset, classes) - train_dset, sel_indices = random_samples( - support_classes_dataset, samples_per_user - ) - train_indices = [indices[i] for i in sel_indices] - elif config["train_label_distribution"].endswith("non_iid"): - alpha = config.get("alpha_data", 0.4) - if config["train_label_distribution"] == "inter_domain_non_iid": - # Hack to get the same class prior for all users with the same dataset - # While keeping the same random state for all users - if isinstance(config["dset"], dict) and isinstance( - config["dset"], dict - ): - cls_priors = [] - dsets = list(config["dset"].values()) - for _ in dsets: - n_cls = self.dset_obj.num_cls - cls_priors.append( - np.random.dirichlet( - alpha=[alpha] * n_cls, size=len(users_with_same_dset) + if config.get("test_samples_per_class", None) is not None: + test_dset, _ = balanced_subset(test_dset, config["test_samples_per_class"]) + + samples_per_user = config["samples_per_user"] + batch_size: int = config["batch_size"] # type: ignore + print(f"samples per user: {samples_per_user}, batch size: {batch_size}") + + # Support user specific dataset + if isinstance(config["dset"], dict): + + def is_same_dest(dset): + # Consider all variations of cifar10 as the same dataset + # To avoid having exactly same original dataset (without + # considering transformation) on multiple users + if self.dset == "cifar10" or self.dset.startswith("cifar10_"): + return dset == "cifar10" or dset.startswith("cifar10_") + else: + return dset == self.dset + + users_with_same_dset = sorted( + [int(k) for k, v in config["dset"].items() if is_same_dest(v)] + ) + else: + users_with_same_dset = list(range(1, config["num_users"] + 1)) + user_idx = users_with_same_dset.index(self.node_id) + + cls_prior = None + # If iid, each user has random samples from the whole dataset (no + # overlap between users) + if config["train_label_distribution"] == "iid": + indices = np.random.permutation(len(train_dset)) + train_indices = indices[ + user_idx * samples_per_user : (user_idx + 1) * samples_per_user + ] + train_dset = Subset(train_dset, train_indices) + classes = list(set([train_dset[i][1] for i in range(len(train_dset))])) + # If non_iid, each user get random samples from its support classes + # (mulitple users might have same images) + elif config["train_label_distribution"] == "support": + classes = config["support"][str(self.node_id)] + support_classes_dataset, indices = filter_by_class(train_dset, classes) + train_dset, sel_indices = random_samples( + support_classes_dataset, samples_per_user + ) + train_indices = [indices[i] for i in sel_indices] + elif config["train_label_distribution"].endswith("non_iid"): + alpha = config.get("alpha_data", 0.4) + if config["train_label_distribution"] == "inter_domain_non_iid": + # Hack to get the same class prior for all users with the same dataset + # While keeping the same random state for all users + if isinstance(config["dset"], dict) and isinstance( + config["dset"], dict + ): + cls_priors = [] + dsets = list(config["dset"].values()) + for _ in dsets: + n_cls = self.dset_obj.num_cls + cls_priors.append( + np.random.dirichlet( + alpha=[alpha] * n_cls, size=len(users_with_same_dset) + ) ) - ) - cls_prior = cls_priors[dsets.index(self.dset)] - train_y, train_idx_split, cls_prior = non_iid_balanced( - self.dset_obj, - len(users_with_same_dset), - samples_per_user, - alpha, - cls_priors=cls_prior, - is_train=True, - ) - train_indices = train_idx_split[self.node_id - 1] - train_dset = Subset(train_dset, train_indices) - classes = np.unique(train_y[user_idx]).tolist() - # One plot per dataset - # if user_idx == 0: - # print("using non_iid_balanced", alpha) - # self.plot_utils.plot_training_distribution(train_y, - # self.dset, users_with_same_dset) - elif config["train_label_distribution"] == "shard": - raise NotImplementedError - # classes_per_user = config["shards"]["classes_per_user"] - # samples_per_shard = samples_per_user // classes_per_user - # train_dset = build_shards_dataset(train_dset, samples_per_shard, - # classes_per_user, self.node_id) - else: - raise ValueError( - "Unknown train label distribution: {}.".format( - config["train_label_distribution"] + cls_prior = cls_priors[dsets.index(self.dset)] + train_y, train_idx_split, cls_prior = non_iid_balanced( + self.dset_obj, + len(users_with_same_dset), + samples_per_user, + alpha, + cls_priors=cls_prior, + is_train=True, + ) + train_indices = train_idx_split[self.node_id - 1] + train_dset = Subset(train_dset, train_indices) + classes = np.unique(train_y[user_idx]).tolist() + # One plot per dataset + # if user_idx == 0: + # print("using non_iid_balanced", alpha) + # self.plot_utils.plot_training_distribution(train_y, + # self.dset, users_with_same_dset) + elif config["train_label_distribution"] == "shard": + raise NotImplementedError + # classes_per_user = config["shards"]["classes_per_user"] + # samples_per_shard = samples_per_user // classes_per_user + # train_dset = build_shards_dataset(train_dset, samples_per_shard, + # classes_per_user, self.node_id) + else: + raise ValueError( + "Unknown train label distribution: {}.".format( + config["train_label_distribution"] + ) ) - ) - if self.dset.startswith("domainnet"): - train_transform = T.Compose( - [ - T.RandomResizedCrop(32, scale=(0.75, 1)), - T.RandomHorizontalFlip(), - # T.ToTensor() - ] - ) + if self.dset.startswith("domainnet"): + train_transform = T.Compose( + [ + T.RandomResizedCrop(32, scale=(0.75, 1)), + T.RandomHorizontalFlip(), + # T.ToTensor() + ] + ) - # Cache before transform to preserve transform randomness - train_dset = TransformDataset(CacheDataset(train_dset), train_transform) + # Cache before transform to preserve transform randomness + train_dset = TransformDataset(CacheDataset(train_dset), train_transform) - if config.get("malicious_type", None) == "corrupt_data": - corruption_fn_name = config.get("corruption_fn", "gaussian_noise") - severity = config.get("corrupt_severity", 1) - train_dset = CorruptDataset(CacheDataset(train_dset), corruption_fn_name, severity) - print("created train dataset with corruption function: ", corruption_fn_name) + if config.get("malicious_type", None) == "corrupt_data": + corruption_fn_name = config.get("corruption_fn", "gaussian_noise") + severity = config.get("corrupt_severity", 1) + train_dset = CorruptDataset(CacheDataset(train_dset), corruption_fn_name, severity) + print("created train dataset with corruption function: ", corruption_fn_name) - self.classes_of_interest = classes + self.classes_of_interest = classes - val_prop = config.get("validation_prop", 0) - val_dset = None - if val_prop > 0: - val_size = int(val_prop * len(train_dset)) - train_size = len(train_dset) - val_size - train_dset, val_dset = torch.utils.data.random_split( - train_dset, [train_size, val_size] - ) - # self.val_dloader = DataLoader(val_dset, batch_size=batch_size*len(self.device_ids), - # shuffle=True) - self.val_dloader = DataLoader(val_dset, batch_size=batch_size, shuffle=True) - - assert isinstance(train_dset, torch.utils.data.Dataset), "train_dset must be a Dataset" - self.train_indices = train_indices - self.train_dset = train_dset - self.dloader = DataLoader(train_dset, batch_size=batch_size, shuffle=True) # type: ignore - - if config["test_label_distribution"] == "iid": - pass - # If non_iid, each users ge the whole test set for each of its - # support classes - elif config["test_label_distribution"] == "support": - classes = config["support"][str(self.node_id)] - test_dset, _ = filter_by_class(test_dset, classes) - elif config["test_label_distribution"] == "non_iid": - - test_y, test_idx_split, _ = non_iid_balanced( - self.dset_obj, - len(users_with_same_dset), - config["test_samples_per_user"], - is_train=False, - ) + val_prop = config.get("validation_prop", 0) + val_dset = None + if val_prop > 0: + val_size = int(val_prop * len(train_dset)) + train_size = len(train_dset) - val_size + train_dset, val_dset = torch.utils.data.random_split( + train_dset, [train_size, val_size] + ) + # self.val_dloader = DataLoader(val_dset, batch_size=batch_size*len(self.device_ids), + # shuffle=True) + self.val_dloader = DataLoader(val_dset, batch_size=batch_size, shuffle=True) - train_indices = test_idx_split[self.node_id - 1] - test_dset = Subset(test_dset, train_indices) - else: - raise ValueError( - "Unknown test label distribution: {}.".format( - config["test_label_distribution"] + assert isinstance(train_dset, torch.utils.data.Dataset), "train_dset must be a Dataset" + self.train_indices = train_indices + self.train_dset = train_dset + self.dloader = DataLoader(train_dset, batch_size=batch_size, shuffle=True) # type: ignore + + if config["test_label_distribution"] == "iid": + pass + # If non_iid, each users ge the whole test set for each of its + # support classes + elif config["test_label_distribution"] == "support": + classes = config["support"][str(self.node_id)] + test_dset, _ = filter_by_class(test_dset, classes) + elif config["test_label_distribution"] == "non_iid": + + test_y, test_idx_split, _ = non_iid_balanced( + self.dset_obj, + len(users_with_same_dset), + config["test_samples_per_user"], + is_train=False, ) - ) - if self.dset.startswith("domainnet"): - test_dset = CacheDataset(test_dset) + train_indices = test_idx_split[self.node_id - 1] + test_dset = Subset(test_dset, train_indices) + else: + raise ValueError( + "Unknown test label distribution: {}.".format( + config["test_label_distribution"] + ) + ) - self._test_loader = DataLoader(test_dset, batch_size=batch_size) - # TODO: fix print_data_summary - # self.print_data_summary(train_dset, test_dset, val_dset=val_dset) + if self.dset.startswith("domainnet"): + test_dset = CacheDataset(test_dset) + + self._test_loader = DataLoader(test_dset, batch_size=batch_size) + # TODO: fix print_data_summary + # self.print_data_summary(train_dset, test_dset, val_dset=val_dset) def local_train(self, round: int, epochs: int = 1, **kwargs: Any) -> Tuple[float, float, float]: """ diff --git a/src/algos/fl.py b/src/algos/fl.py index c4840994..33f2d7d0 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -11,6 +11,8 @@ from algos.attack_bad_weights import BadWeightsAttack from algos.attack_sign_flip import SignFlipAttack +from utils.gias import gia_main + import pickle class FedAvgClient(BaseClient): @@ -101,6 +103,10 @@ def __init__( self.model_save_path = "{}/saved_models/node_{}.pt".format( self.config["results_path"], self.node_id ) + if "gia" in self.config: + # to store param differences for GIA attack + self.params_s = list() + self.params_t = list() def fed_avg(self, model_wts: List[OrderedDict[str, Tensor]]): num_users = len(model_wts) @@ -155,11 +161,40 @@ def test(self, **kwargs: Any) -> List[float]: self.model_utils.save_model(self.model, self.model_save_path) return [test_loss, test_acc, time_taken] - def receive_and_aggregate(self, dump_file_name: str = ""): + def receive_and_aggregate(self, round:int, dump_file_name: str = ""): reprs = self.comm_utils.all_gather() + if len(dump_file_name) > 0: with open(f"{dump_file_name}.pkl", "wb") as f: pickle.dump(reprs, f) + + # Handle GIA-specific logic + if "gia" in self.config: + base_params = [key for key in self.model.parameters()] + + for rep in reprs: + model_state_dict = rep["model"] + + # Extract relevant model parameters + model_params = OrderedDict( + (key, value) for key, value in model_state_dict.items() + if key in base_params + ) + + # Store parameters based on round + if round == 0: + self.params_s.append(model_params) + elif round == 1: + self.params_t.append(model_params) + images = rep["images"] + labels = rep["labels"] + + # Launch GIA attack + for client_id in range(len(self.params_s)): + p_s = self.params_s[client_id] # Fixed: now using params_s instead of params_t + p_t = self.params_t[client_id] + gia_main(p_s, p_t, base_params, self.model, labels, images, client_id) + avg_wts = self.aggregate(reprs) self.set_representation(avg_wts) diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index e2a07844..f2350c3a 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -39,36 +39,17 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st "batch_size": 256, } -<<<<<<< HEAD - test_fl_inversion: ConfigType = { - "algo": "fedavg_inversion", - "exp_type": "", - # Learning setup - "epochs": 10, + # Collaboration setup + "algo": "fedavg", + "rounds": 2, + + # Model parameters "model": "resnet10", "model_lr": 3e-4, "batch_size": 256, - "malicious_type": "normal", } -malicious_traditional_bad_weights: ConfigType = { - **traditional_fl, - "malicious_type": "bad_weights", -} - -malicious_traditional_flip_signs: ConfigType = { - **traditional_fl, - "malicious_type": "sign_flip", -} - -# malicious_gradient_inversion: ConfigType = { -# **traditional_fl, -# "malicious_type": "gradient_inversion", -# } - -======= ->>>>>>> main fedweight: ConfigType = { "algo": "fedweight", "num_rep": 1, @@ -357,10 +338,5 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st malicious_traditional_model_update_attack, ] -<<<<<<< HEAD # default_config_list: List[ConfigType] = [traditional_fl] default_config_list: List[ConfigType] = [test_fl_inversion] -======= - -default_config_list: List[ConfigType] = [traditional_fl] ->>>>>>> main diff --git a/src/configs/sys_config.py b/src/configs/sys_config.py index 4a60e20f..e1488008 100644 --- a/src/configs/sys_config.py +++ b/src/configs/sys_config.py @@ -156,7 +156,7 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): CIAR10_DPATH = "./datasets/imgs/cifar10/" NUM_COLLABORATORS = 1 -DUMP_DIR = "/mas/camera/Experiments/SONAR/abhi/" +DUMP_DIR = "/mas/camera/Experiments/SONAR/yshi23/" mpi_system_config: ConfigType = { "exp_id": "", @@ -318,7 +318,7 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): "exp_keys": [], } -num_users = 9 +num_users = 4 dropout_dict = { "distribution_dict": { # leave dict empty to disable dropout @@ -333,7 +333,7 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): for i in range(1, num_users + 1): dropout_dicts[f"node_{i}"] = dropout_dict -gpu_ids = [2, 3, 5, 6] +gpu_ids = [0,1,2,3] grpc_system_config: ConfigType = { "exp_id": "static", "num_users": num_users, @@ -353,5 +353,26 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): "dropout_dicts": dropout_dicts, } -current_config = grpc_system_config +grpc_system_config_gia: ConfigType = { + "exp_id": "static", + "num_users": num_users, + "num_collaborators": NUM_COLLABORATORS, + "comm": {"type": "GRPC", "synchronous": True, "peer_ids": ["localhost:50048"]}, # The super-node + "dset": CIFAR10_DSET, + "dump_dir": DUMP_DIR, + "dpath": CIAR10_DPATH, + "seed": 2, + "device_ids": get_device_ids(num_users, gpu_ids), + # "algos": get_algo_configs(num_users=num_users, algo_configs=default_config_list), # type: ignore + "algos": get_algo_configs(num_users=num_users, algo_configs=[traditional_fl]), # type: ignore + "samples_per_user": 50000 // num_users, # distributed equally + "train_label_distribution": "iid", + "test_label_distribution": "iid", + "exp_keys": [], + "dropout_dicts": dropout_dicts, + "gia":True +} + +current_config = grpc_system_config_gia +# current_config = grpc_system_config # current_config = mpi_system_config diff --git a/src/utils/communication/comm_utils.py b/src/utils/communication/comm_utils.py index 0423a2c4..54566916 100644 --- a/src/utils/communication/comm_utils.py +++ b/src/utils/communication/comm_utils.py @@ -1,11 +1,6 @@ from enum import Enum -<<<<<<< HEAD -from typing import Any, Dict, List, Tuple - -======= ->>>>>>> main from utils.communication.grpc.main import GRPCCommunication -from typing import Any, Dict, List, TYPE_CHECKING +from typing import Any, Dict, List, Tuple, TYPE_CHECKING # from utils.communication.mpi import MPICommUtils if TYPE_CHECKING: diff --git a/src/utils/communication/grpc/main.py b/src/utils/communication/grpc/main.py index 4941cf6c..f400d77c 100644 --- a/src/utils/communication/grpc/main.py +++ b/src/utils/communication/grpc/main.py @@ -6,12 +6,8 @@ import threading import time import socket -<<<<<<< HEAD -from typing import Any, Dict, List, OrderedDict, Union, Tuple -======= import functools from typing import Any, Callable, Dict, List, OrderedDict, Union, TYPE_CHECKING ->>>>>>> main from urllib.parse import unquote import grpc # type: ignore from torch import Tensor @@ -408,6 +404,7 @@ def callback_fn(stub: comm_pb2_grpc.CommunicationServerStub) -> int: # 2. Tensor data - Tensors # 3. Metadata - JSON format def receive(self, node_ids: List[int]) -> List[Any]: + print("ALL PARTICIPATING NODES", node_ids) if self.synchronous: for id in node_ids: self.wait_until_rounds_match(id) diff --git a/src/utils/gias.py b/src/utils/gias.py index df86d88a..d7679b52 100644 --- a/src/utils/gias.py +++ b/src/utils/gias.py @@ -1,9 +1,9 @@ # /////////////// Gradient Inversion Helpers /////////////// import inversefed -from collections import OrderedDict import matplotlib.pyplot as plt + # based on InvertingGradients code by Jonas Geiping # code found in https://github.com/JonasGeiping/invertinggradients/tree/1157b61c6704df42c497ab9eb074c75da5204334 @@ -16,7 +16,7 @@ def compute_param_delta(param_s, param_t, basic_params): """ return [(param_t[name] - param_s[name]).detach() for name in basic_params if name in param_s and name in param_t] -def reconstruct_gradient(param_diff, target_labels, lr, local_steps, model, client_id=0): +def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_steps, model, client_id=0): """ Reconstructs the gradient following the Geiping InvertingGradients technique """ @@ -45,10 +45,16 @@ def reconstruct_gradient(param_diff, target_labels, lr, local_steps, model, clie output, stats = rec_machine.reconstruct(param_diff, target_labels, img_shape=(3, 32, 32)) - grid_plot(output, target_labels, ds, dm, save_path=f"gias_output_client_{client_id}.png") + # compute reconstruction acccuracy + test_mse = (output.detach() - target_images).pow(2).mean() + feat_mse = (model(output.detach())- model(target_images)).pow(2).mean() + test_psnr = inversefed.metrics.psnr(output, target_images, factor=1/ds) + print(f"Client {client_id} Test MSE: {test_mse:.2e}, Test PSNR: {test_psnr:.2f}, Feature MSE: {feat_mse:.2e}") + + grid_plot(output, target_labels, ds, dm, stats, test_mse, feat_mse, test_psnr, save_path=f"gias_output_client_{client_id}.png") return output, stats -def grid_plot(tensor, labels, ds, dm, save_path=None): +def grid_plot(tensor, labels, ds, dm, stats, test_mse, feat_mse, test_psnr, save_path=None): tensor = tensor.clone().detach() tensor.mul_(ds).add_(dm).clamp_(0, 1) @@ -57,16 +63,17 @@ def grid_plot(tensor, labels, ds, dm, save_path=None): ax.imshow(im.permute(1, 2, 0).cpu()) ax.set_title(l) ax.axis('off') + plt.title(f"Rec. loss: {stats['opt']:2.4f} | MSE: {test_mse:2.4f} " + f"| PSNR: {test_psnr:4.2f} | FMSE: {feat_mse:2.4e} |"); if save_path: plt.savefig(save_path, bbox_inches='tight') # Save the figure if save_path is provided # plt.show() # Show the plot after saving -def gia_main(param_s, param_t, model): +def gia_main(param_s, param_t, base_params, model, target_labels, target_images, client_id): """ Main function for Gradient Inversion Attack """ - params = model.parameters().keys() - param_diff = compute_param_delta(param_s, param_t, params) - output, stats = reconstruct_gradient(param_diff, 3e-4 , 1, model) + param_diff = compute_param_delta(param_s, param_t, base_params) + output, stats = reconstruct_gradient(param_diff, target_labels, target_images, 3e-4 , 1, model, client_id) return output, stats \ No newline at end of file diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index c3134fde..510ec27f 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -32,7 +32,6 @@ def get_model( model_name: str, dset: str, device: torch.device, - device_ids: List[int], pretrained: bool = False, **kwargs: Any, ): From d0b347f4df8582dad24d46e3c7fa20a673281916 Mon Sep 17 00:00:00 2001 From: photonshi Date: Mon, 28 Oct 2024 19:15:02 +0000 Subject: [PATCH 06/16] debug commit --- src/algos/base_class.py | 3 ++- src/algos/fl.py | 31 ++++++++++++++++++++-------- src/configs/algo_config.py | 1 + src/configs/sys_config.py | 7 ++++--- src/utils/communication/grpc/main.py | 1 - 5 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index ebec2ef8..53f78a52 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -254,7 +254,8 @@ def get_model_weights(self) -> Dict[str, Tensor]: Share the model weights """ message = {"sender": self.node_id, "round": self.round, "model": self.model.state_dict()} - if "gia" in self.config: + + if "gia" in self.config and hasattr(self, 'images') and hasattr(self, 'labels'): # also stream image and labels message["images"] = self.images message["labels"] = self.labels diff --git a/src/algos/fl.py b/src/algos/fl.py index 33f2d7d0..79e705d0 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -161,8 +161,9 @@ def test(self, **kwargs: Any) -> List[float]: self.model_utils.save_model(self.model, self.model_save_path) return [test_loss, test_acc, time_taken] - def receive_and_aggregate(self, round:int, dump_file_name: str = ""): + def receive_and_aggregate_gia(self, round:int, dump_file_name: str = ""): reprs = self.comm_utils.all_gather() + if len(dump_file_name) > 0: with open(f"{dump_file_name}.pkl", "wb") as f: @@ -170,9 +171,11 @@ def receive_and_aggregate(self, round:int, dump_file_name: str = ""): # Handle GIA-specific logic if "gia" in self.config: + print("Server Running GIA attack") base_params = [key for key in self.model.parameters()] for rep in reprs: + assert "images" and "labels" in rep, "Images and labels not found in representation" model_state_dict = rep["model"] # Extract relevant model parameters @@ -198,11 +201,26 @@ def receive_and_aggregate(self, round:int, dump_file_name: str = ""): avg_wts = self.aggregate(reprs) self.set_representation(avg_wts) - def single_round(self, fp: str = ""): + def receive_and_aggregate(self): + reprs = self.comm_utils.all_gather() + avg_wts = self.aggregate(reprs) + self.set_representation(avg_wts) + + def single_round(self, round:int): """ Runs the whole training procedure """ - self.receive_and_aggregate(fp) + if round > 1: + self.receive_and_aggregate() + else: + dump_file_name = "" + if round == 0: + dump_file_name = "/u/yshi23/sonar/src/start_reprs" + elif round == 1: + dump_file_name = "/u/yshi23/sonar/src/end_reprs" + + print(f"in round {round}, about to prepare for GIA") + self.receive_and_aggregate_gia(round, dump_file_name) def run_protocol(self): stats: Dict[str, Any] = {} @@ -210,14 +228,9 @@ def run_protocol(self): start_rounds = self.config.get("start_rounds", 0) total_rounds = self.config["rounds"] for round in range(start_rounds, total_rounds): - dump_file_name = "" - if round == 0: - dump_file_name = "/u/yshi23/sonar/src/start_reprs" - elif round == 1: - dump_file_name = "/u/yshi23/sonar/src/end_reprs" self.local_round_done() - self.single_round(dump_file_name) + self.single_round(round) stats["bytes_received"], stats["bytes_sent"] = self.comm_utils.get_comm_cost() stats["test_loss"], stats["test_acc"], stats["test_time"] = self.test() self.log_metrics(stats=stats, iteration=round) diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index f2350c3a..905da0ff 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -48,6 +48,7 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st "model": "resnet10", "model_lr": 3e-4, "batch_size": 256, + "gia": True, } fedweight: ConfigType = { diff --git a/src/configs/sys_config.py b/src/configs/sys_config.py index e1488008..12b0ff33 100644 --- a/src/configs/sys_config.py +++ b/src/configs/sys_config.py @@ -10,7 +10,8 @@ malicious_algo_config_list, default_config_list, fedstatic, - traditional_fl + traditional_fl, + test_fl_inversion, ) sliding_window_8c_4cpc_support = { @@ -156,7 +157,7 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): CIAR10_DPATH = "./datasets/imgs/cifar10/" NUM_COLLABORATORS = 1 -DUMP_DIR = "/mas/camera/Experiments/SONAR/yshi23/" +DUMP_DIR = "/u/yshi23/sonar/src/expt_dump/test/" mpi_system_config: ConfigType = { "exp_id": "", @@ -364,7 +365,7 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): "seed": 2, "device_ids": get_device_ids(num_users, gpu_ids), # "algos": get_algo_configs(num_users=num_users, algo_configs=default_config_list), # type: ignore - "algos": get_algo_configs(num_users=num_users, algo_configs=[traditional_fl]), # type: ignore + "algos": get_algo_configs(num_users=num_users, algo_configs=[test_fl_inversion]), # type: ignore "samples_per_user": 50000 // num_users, # distributed equally "train_label_distribution": "iid", "test_label_distribution": "iid", diff --git a/src/utils/communication/grpc/main.py b/src/utils/communication/grpc/main.py index f400d77c..47a04e43 100644 --- a/src/utils/communication/grpc/main.py +++ b/src/utils/communication/grpc/main.py @@ -404,7 +404,6 @@ def callback_fn(stub: comm_pb2_grpc.CommunicationServerStub) -> int: # 2. Tensor data - Tensors # 3. Metadata - JSON format def receive(self, node_ids: List[int]) -> List[Any]: - print("ALL PARTICIPATING NODES", node_ids) if self.synchronous: for id in node_ids: self.wait_until_rounds_match(id) From 200ee0b1e174ed77cf94ed0d178b9916e5ace5b3 Mon Sep 17 00:00:00 2001 From: photonshi Date: Mon, 28 Oct 2024 22:13:18 +0000 Subject: [PATCH 07/16] debugging training --- src/algos/base_class.py | 2 +- src/algos/fl.py | 58 ++++++++++++++++++++++---------------- src/configs/algo_config.py | 4 +-- src/utils/gias.py | 14 ++++++++- src/utils/model_utils.py | 3 ++ 5 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 53f78a52..5873d128 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -605,7 +605,7 @@ def local_train(self, round: int, epochs: int = 1, **kwargs: Any) -> Tuple[float avg_loss, avg_acc = 0, 0 for _ in range(epochs): tr_loss, tr_acc = self.model_utils.train( - self.model, self.optim, self.dloader, self.loss_fn, self.device, malicious_type=self.config.get("malicious_type", "normal"), config=self.config, + self.model, self.optim, self.dloader, self.loss_fn, self.device, malicious_type=self.config.get("malicious_type", "normal"), config=self.config ) avg_loss += tr_loss avg_acc += tr_acc diff --git a/src/algos/fl.py b/src/algos/fl.py index 79e705d0..6d9b5da4 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -70,6 +70,12 @@ def get_model_weights(self, **kwargs: Any) -> Dict[str, Any]: # move the model to cpu before sending for key in message["model"].keys(): message["model"][key] = message["model"][key].to("cpu") + + # assert hasattr(self, 'images') and hasattr(self, 'labels'), "Images and labels not found" + if "gia" in self.config and hasattr(self, 'images') and hasattr(self, 'labels'): + # also stream image and labels + message["images"] = self.images.to("cpu") + message["labels"] = self.labels.to("cpu") return message # type: ignore @@ -105,8 +111,8 @@ def __init__( ) if "gia" in self.config: # to store param differences for GIA attack - self.params_s = list() - self.params_t = list() + self.params_s = [None for i in range(4)] + self.params_t = [None for i in range(4)] def fed_avg(self, model_wts: List[OrderedDict[str, Tensor]]): num_users = len(model_wts) @@ -164,18 +170,15 @@ def test(self, **kwargs: Any) -> List[float]: def receive_and_aggregate_gia(self, round:int, dump_file_name: str = ""): reprs = self.comm_utils.all_gather() - - if len(dump_file_name) > 0: - with open(f"{dump_file_name}.pkl", "wb") as f: - pickle.dump(reprs, f) - # Handle GIA-specific logic if "gia" in self.config: print("Server Running GIA attack") - base_params = [key for key in self.model.parameters()] + base_params = [key for key, _ in self.model.named_parameters()] + print(base_params) for rep in reprs: - assert "images" and "labels" in rep, "Images and labels not found in representation" + client_id = rep["sender"] + assert "images" in rep and "labels" in rep, "Images and labels not found in representation" model_state_dict = rep["model"] # Extract relevant model parameters @@ -186,16 +189,20 @@ def receive_and_aggregate_gia(self, round:int, dump_file_name: str = ""): # Store parameters based on round if round == 0: - self.params_s.append(model_params) + self.params_s[client_id-1] = model_params elif round == 1: - self.params_t.append(model_params) + self.params_t[client_id-1] = model_params images = rep["images"] labels = rep["labels"] + with open(f"params_t_{client_id}.pkl", "wb") as f: + pickle.dump(model_params, f) + with open(f"params_s_{client_id}.pkl", "wb") as f: + pickle.dump(self.params_s[client_id-1], f) # Launch GIA attack - for client_id in range(len(self.params_s)): - p_s = self.params_s[client_id] # Fixed: now using params_s instead of params_t - p_t = self.params_t[client_id] + for idx in range(len(self.params_s)): + p_s = self.params_s[idx] # Fixed: now using params_s instead of params_t + p_t = self.params_t[idx] gia_main(p_s, p_t, base_params, self.model, labels, images, client_id) avg_wts = self.aggregate(reprs) @@ -210,17 +217,18 @@ def single_round(self, round:int): """ Runs the whole training procedure """ - if round > 1: - self.receive_and_aggregate() - else: - dump_file_name = "" - if round == 0: - dump_file_name = "/u/yshi23/sonar/src/start_reprs" - elif round == 1: - dump_file_name = "/u/yshi23/sonar/src/end_reprs" - - print(f"in round {round}, about to prepare for GIA") - self.receive_and_aggregate_gia(round, dump_file_name) + self.receive_and_aggregate() + # if round > 1: + # self.receive_and_aggregate() + # else: + # dump_file_name = "" + # if round == 0: + # dump_file_name = "/u/yshi23/sonar/src/start_reprs" + # elif round == 1: + # dump_file_name = "/u/yshi23/sonar/src/end_reprs" + + # print(f"in round {round}, about to prepare for GIA") + # self.receive_and_aggregate_gia(round, dump_file_name) def run_protocol(self): stats: Dict[str, Any] = {} diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index 905da0ff..f66e91bf 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -31,7 +31,7 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st traditional_fl: ConfigType = { # Collaboration setup "algo": "fedavg", - "rounds": 2, + "rounds": 5, # Model parameters "model": "resnet10", @@ -42,7 +42,7 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st test_fl_inversion: ConfigType = { # Collaboration setup "algo": "fedavg", - "rounds": 2, + "rounds": 5, # Model parameters "model": "resnet10", diff --git a/src/utils/gias.py b/src/utils/gias.py index d7679b52..5a66522e 100644 --- a/src/utils/gias.py +++ b/src/utils/gias.py @@ -3,6 +3,8 @@ import inversefed import matplotlib.pyplot as plt +import torch +import pickle # based on InvertingGradients code by Jonas Geiping # code found in https://github.com/JonasGeiping/invertinggradients/tree/1157b61c6704df42c497ab9eb074c75da5204334 @@ -20,7 +22,15 @@ def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_ste """ Reconstructs the gradient following the Geiping InvertingGradients technique """ - + print("length of param diff: ", len(param_diff)) + with open("param_diff.pkl", "wb") as f: + pickle.dump(param_diff, f) + setup = inversefed.utils.system_startup() + for p in range(len(param_diff)): + param_diff[p] = param_diff[p].to(setup['device']) + # param_diff = param_diff.to(setup['device']) + target_labels = target_labels.to(setup['device']) + target_images = target_images.to(setup['device']) dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None] ds = torch.as_tensor(inversefed.consts.cifar10_std, **setup)[:, None, None] @@ -40,6 +50,8 @@ def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_ste lr_decay=True, scoring_choice='loss') + assert len(param_diff) == 38 + rec_machine = inversefed.FedAvgReconstructor(model, (dm, ds), local_steps, lr, config, use_updates=True, num_images=len(target_labels)) diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index 510ec27f..8aa47fcb 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -191,6 +191,9 @@ def train_classification( output = nn.functional.log_softmax(output, dim=1) # type: ignore loss = loss_fn(output, target) + # if kwargs.get("gia", True): + # print("you are here!") + # loss = loss_fn(output, target).sum() loss.backward() optim.step() train_loss += loss.item() From 4523b3b6357ea16707227f392bff6dc88926a60a Mon Sep 17 00:00:00 2001 From: photonshi Date: Tue, 29 Oct 2024 15:50:03 +0000 Subject: [PATCH 08/16] working now but accuracy is very low --- src/algos/base_class.py | 6 ++-- src/algos/fl.py | 74 +++++++++++++++++++++++---------------- src/data_loaders/cifar.py | 4 +-- src/test_inversion.ipynb | 7 ++++ src/utils/data_utils.py | 65 +++++++++++++++------------------- src/utils/gias.py | 16 ++++++--- src/utils/model_utils.py | 56 +++++++++++++++++++++++++++-- 7 files changed, 150 insertions(+), 78 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 5873d128..a63e2103 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -24,7 +24,6 @@ non_iid_balanced, balanced_subset, gia_client_dataset, - gia_server_testset, CacheDataset, TransformDataset, CorruptDataset, @@ -189,6 +188,7 @@ def set_model_parameters(self, config: Dict[str, Any]) -> None: else: raise ValueError(f"Unknown optimizer: {optim_name}.") if "gia" in config: + print("setting optim to gia") optim = torch.optim.SGD num_classes = self.dset_obj.num_cls num_channels = self.dset_obj.num_channels @@ -605,7 +605,7 @@ def local_train(self, round: int, epochs: int = 1, **kwargs: Any) -> Tuple[float avg_loss, avg_acc = 0, 0 for _ in range(epochs): tr_loss, tr_acc = self.model_utils.train( - self.model, self.optim, self.dloader, self.loss_fn, self.device, malicious_type=self.config.get("malicious_type", "normal"), config=self.config + self.model, self.optim, self.dloader, self.loss_fn, self.device, malicious_type=self.config.get("malicious_type", "normal"), config=self.config, node_id=self.node_id, gia=self.config.get("gia", False) ) avg_loss += tr_loss avg_acc += tr_acc @@ -722,7 +722,7 @@ def set_data_parameters(self, config: Dict[str, Any]) -> None: if "gia" not in config: self._test_loader = DataLoader(test_dset, batch_size=batch_size) else: - test_data, labels, indices = gia_server_testset(test_dset) + _, test_data, labels, indices = gia_client_dataset(self.dset_obj.train_dset, test_dset) self._test_loader = DataLoader(test_data, batch_size=10) def aggregate( self, representation_list: List[OrderedDict[str, Any]], **kwargs: Any diff --git a/src/algos/fl.py b/src/algos/fl.py index 6d9b5da4..5a05d135 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -114,6 +114,9 @@ def __init__( self.params_s = [None for i in range(4)] self.params_t = [None for i in range(4)] + # save randomly initialized parameters + self.random_params = self.model.state_dict() + def fed_avg(self, model_wts: List[OrderedDict[str, Tensor]]): num_users = len(model_wts) coeff = 1 / num_users @@ -167,68 +170,79 @@ def test(self, **kwargs: Any) -> List[float]: self.model_utils.save_model(self.model, self.model_save_path) return [test_loss, test_acc, time_taken] - def receive_and_aggregate_gia(self, round:int, dump_file_name: str = ""): + def receive_and_aggregate_gia(self, round: int, attack_start_round: int, attack_end_round: int, dump_file_name: str = ""): reprs = self.comm_utils.all_gather() + with open(dump_file_name, "wb") as f: + pickle.dump(reprs, f) + # Handle GIA-specific logic if "gia" in self.config: print("Server Running GIA attack") base_params = [key for key, _ in self.model.named_parameters()] print(base_params) - + for rep in reprs: client_id = rep["sender"] assert "images" in rep and "labels" in rep, "Images and labels not found in representation" model_state_dict = rep["model"] - + # Extract relevant model parameters model_params = OrderedDict( (key, value) for key, value in model_state_dict.items() if key in base_params ) - - # Store parameters based on round - if round == 0: - self.params_s[client_id-1] = model_params - elif round == 1: - self.params_t[client_id-1] = model_params + + # Store parameters based on attack start and end rounds + if round == attack_start_round: + self.params_s[client_id - 1] = model_params + elif round == attack_end_round: + self.params_t[client_id - 1] = model_params images = rep["images"] labels = rep["labels"] - + with open(f"params_t_{client_id}.pkl", "wb") as f: pickle.dump(model_params, f) with open(f"params_s_{client_id}.pkl", "wb") as f: - pickle.dump(self.params_s[client_id-1], f) + pickle.dump(self.params_s[client_id - 1], f) + # Launch GIA attack - for idx in range(len(self.params_s)): - p_s = self.params_s[idx] # Fixed: now using params_s instead of params_t - p_t = self.params_t[idx] - gia_main(p_s, p_t, base_params, self.model, labels, images, client_id) - + # p_s, p_t = self.params_s[client_id - 1], self.params_t[client_id - 1] + p_s, p_t = self.random_params, self.params_s[client_id - 1] + gia_main(p_s, p_t, base_params, self.model, labels, images, client_id) + avg_wts = self.aggregate(reprs) self.set_representation(avg_wts) + def receive_and_aggregate(self): reprs = self.comm_utils.all_gather() avg_wts = self.aggregate(reprs) self.set_representation(avg_wts) - def single_round(self, round:int): + def single_round(self, round: int, attack_start_round: int = 0, attack_end_round: int = 1): """ - Runs the whole training procedure + Runs the whole training procedure. + + Parameters: + round (int): Current round of training. + attack_start_round (int): The starting round to initiate the attack. + attack_end_round (int): The last round for the attack to be performed. """ - self.receive_and_aggregate() - # if round > 1: - # self.receive_and_aggregate() - # else: - # dump_file_name = "" - # if round == 0: - # dump_file_name = "/u/yshi23/sonar/src/start_reprs" - # elif round == 1: - # dump_file_name = "/u/yshi23/sonar/src/end_reprs" - - # print(f"in round {round}, about to prepare for GIA") - # self.receive_and_aggregate_gia(round, dump_file_name) + # Normal training when outside the attack range + if round < attack_start_round or round > attack_end_round: + self.receive_and_aggregate() + else: + # Set file name based on start or end of attack range + dump_file_name = "" + if round == attack_start_round: + dump_file_name = "/u/yshi23/sonar/src/start_reprs" + elif round == attack_end_round: + dump_file_name = "/u/yshi23/sonar/src/end_reprs" + + print(f"In round {round}, preparing for GIA with file: {dump_file_name}") + self.receive_and_aggregate_gia(round, attack_start_round, attack_end_round, dump_file_name) + def run_protocol(self): stats: Dict[str, Any] = {} diff --git a/src/data_loaders/cifar.py b/src/data_loaders/cifar.py index 80405f57..4e474630 100644 --- a/src/data_loaders/cifar.py +++ b/src/data_loaders/cifar.py @@ -18,8 +18,8 @@ def __init__(self, dpath: str, rot_angle: int = 0) -> None: self.train_transform = T.Compose( [ - T.RandomCrop(32, padding=4), - T.RandomHorizontalFlip(), + # T.RandomCrop(32, padding=4), + # T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(self.mean, self.std), ] diff --git a/src/test_inversion.ipynb b/src/test_inversion.ipynb index c49f41fc..e65f9557 100644 --- a/src/test_inversion.ipynb +++ b/src/test_inversion.ipynb @@ -20,6 +20,13 @@ "from torch.utils.data import DataLoader, Dataset\n" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, { "cell_type": "code", "execution_count": 2, diff --git a/src/utils/data_utils.py b/src/utils/data_utils.py index b04c2eba..0ca55451 100644 --- a/src/utils/data_utils.py +++ b/src/utils/data_utils.py @@ -381,51 +381,42 @@ def gia_client_dataset(train_dataset, test_dataset, num_labels=10): selected_labels: List of selected label indices train_indices: List of indices for the selected training images """ - # Get all unique labels from the training dataset - all_labels = list(set([train_dataset[i][1] for i in range(len(train_dataset))])) - # Randomly select labels - selected_labels = sorted(np.random.choice(all_labels, size=num_labels, replace=False)) - - # Process training dataset - temp_train_dataset, train_all_indices = filter_by_class(train_dataset, selected_labels) - train_label_to_indices = {} - for idx in range(len(temp_train_dataset)): - label = temp_train_dataset[idx][1] - if label not in train_label_to_indices: - train_label_to_indices[label] = [] - train_label_to_indices[label].append(train_all_indices[idx]) - - # Process test dataset - temp_test_dataset, test_all_indices = filter_by_class(test_dataset, selected_labels) - test_label_to_indices = {} - for idx in range(len(temp_test_dataset)): - label = temp_test_dataset[idx][1] - if label not in test_label_to_indices: - test_label_to_indices[label] = [] - test_label_to_indices[label].append(test_all_indices[idx]) - - # Select one random image per label for both datasets - final_train_indices = [] - final_test_indices = [] - for label in selected_labels: - # Training dataset - train_label_indices = train_label_to_indices[label] - selected_train_idx = np.random.choice(train_label_indices, size=1)[0] - final_train_indices.append(selected_train_idx) + def get_ordered_indices(dataset): + label_to_indices = {i: [] for i in range(num_labels)} + for idx in range(len(dataset)): + label = dataset[idx][1] + if label < num_labels: + label_to_indices[label].append(idx) - # Test dataset - test_label_indices = test_label_to_indices[label] - selected_test_idx = np.random.choice(test_label_indices, size=1)[0] - final_test_indices.append(selected_test_idx) + ordered_indices = [] + for label in range(num_labels): + # Shuffle indices for each label to randomize selection + np.random.seed(None) + np.random.shuffle(label_to_indices[label]) + random_idx = label_to_indices[label][0] # Select the first random index after shuffling + ordered_indices.append(random_idx) + + return ordered_indices + + # Get ordered indices for both datasets + final_train_indices = get_ordered_indices(train_dataset) + final_test_indices = get_ordered_indices(test_dataset) - # Create final datasets with exactly one image per label + # Create the subsets filtered_train_dataset = Subset(train_dataset, final_train_indices) filtered_test_dataset = Subset(test_dataset, final_test_indices) + # Create selected_labels in ascending order + selected_labels = list(range(num_labels)) + + # Verify ordering + for i in range(num_labels): + assert filtered_train_dataset[i][1] == i, f"Train label at position {i} is not {i}" + assert filtered_test_dataset[i][1] == i, f"Test label at position {i} is not {i}" + return filtered_train_dataset, filtered_test_dataset, selected_labels, final_train_indices - def gia_server_testset(test_dataset, num_labels=10, num_images_per_label=4): """ Select random labels and exactly four random images per selected label from the test dataset. diff --git a/src/utils/gias.py b/src/utils/gias.py index 5a66522e..ce8a38c8 100644 --- a/src/utils/gias.py +++ b/src/utils/gias.py @@ -16,7 +16,8 @@ def compute_param_delta(param_s, param_t, basic_params): basic_params: list of names present in model params """ - return [(param_t[name] - param_s[name]).detach() for name in basic_params if name in param_s and name in param_t] + assert len(param_s) != 0 and len(param_t) != 0, "Empty parameters" + return [(param_t[name].to("cuda") - param_s[name].to("cuda")).detach() for name in basic_params if name in param_s and name in param_t] def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_steps, model, client_id=0): """ @@ -32,8 +33,9 @@ def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_ste target_labels = target_labels.to(setup['device']) target_images = target_images.to(setup['device']) - dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None] - ds = torch.as_tensor(inversefed.consts.cifar10_std, **setup)[:, None, None] + mean, std = [0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010] + dm = torch.as_tensor(mean, **setup)[:, None, None] + ds = torch.as_tensor(std, **setup)[:, None, None] config = dict(signed=True, boxed=True, @@ -87,5 +89,11 @@ def gia_main(param_s, param_t, base_params, model, target_labels, target_images, Main function for Gradient Inversion Attack """ param_diff = compute_param_delta(param_s, param_t, base_params) - output, stats = reconstruct_gradient(param_diff, target_labels, target_images, 3e-4 , 1, model, client_id) + + # Check if all elements in para_diff are zero tensors + if all((diff == 0).all() for diff in param_diff): + print("Parameter differences contain only zeros for client ", client_id) + return None # or return an empty list, depending on your needs + + output, stats = reconstruct_gradient(param_diff, target_labels, target_images, 3e-4, 1, model, client_id) return output, stats \ No newline at end of file diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index 8aa47fcb..c8de81a3 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -10,6 +10,8 @@ import resnet import resnet_in +import matplotlib.pyplot as plt + import yolo from utils.types import ConfigType @@ -176,6 +178,7 @@ def train_classification( test_loader: DataLoader[Any] | None = None, **kwargs: Any, ) -> Tuple[float, float]: + model.train() correct = 0 train_loss = 0 for batch_idx, (data, target) in enumerate(dloader): @@ -188,10 +191,14 @@ def train_classification( output = model(data, position=position) if kwargs.get("apply_softmax", False): + print("here, applying softmax") output = nn.functional.log_softmax(output, dim=1) # type: ignore - loss = loss_fn(output, target) - # if kwargs.get("gia", True): + if kwargs.get("gia", True): + # print(data.shape, target.shape) + node_id = kwargs.get("node_id") + plot_and_save(data, target, filename=f"data_target_plot_{node_id}.png") + # print("you are here!") # loss = loss_fn(output, target).sum() loss.backward() @@ -511,3 +518,48 @@ def get_memory_usage(self): Get the memory usage """ return torch.cuda.memory_allocated(self.device) + +def plot_and_save(data, + target, + dm=torch.as_tensor([0.4914, 0.4822, 0.4465])[:, None, None], + ds=torch.as_tensor([0.2023, 0.1994, 0.2010])[:, None, None], + filename="plot.png"): + """ + Plots a grid of images from `data` with corresponding labels from `target`, and saves the plot. + + Args: + data (torch.Tensor): The data tensor with shape (batch, channels, height, width). + target (torch.Tensor): The target labels tensor with shape (batch,). + dm (torch.Tensor): The mean of the dataset used for normalization, with shape (3, 1, 1). + ds (torch.Tensor): The standard deviation of the dataset used for normalization, with shape (3, 1, 1). + filename (str): The filename to save the plot as an image. + """ + # Move data and target to CPU if they are on a GPU, and detach from the computation graph + data = data.cpu().detach() + target = target.cpu().detach() + + # Normalize and clamp the data to the valid range [0, 1] + data = data.mul(ds).add(dm) + data.clamp_(0, 1) + + # Set up grid size for plotting (e.g., 2 rows of 5 images if batch size is 10) + batch_size = data.size(0) + rows = 1 + cols = batch_size // rows if batch_size % rows == 0 else batch_size + + fig, axes = plt.subplots(rows, cols, figsize=(12, 6)) + axes = axes.flatten() + + # Loop over each image and label in the batch + for i in range(batch_size): + # Convert image to numpy format for plotting + img = data[i].permute(1, 2, 0).numpy() + + # Plot the image and label + axes[i].imshow(img) + axes[i].set_title(f"Label: {target[i].item()}") + axes[i].axis("off") + + plt.tight_layout() + plt.savefig(filename) + plt.show() \ No newline at end of file From 8f916732aa4e170902b27d9229c917011d8cbcf4 Mon Sep 17 00:00:00 2001 From: photonshi Date: Tue, 29 Oct 2024 21:32:23 +0000 Subject: [PATCH 09/16] rewrote using loss steps for much better performance --- src/algos/base_class.py | 6 +-- src/algos/fl.py | 26 +++++++++---- src/configs/algo_config.py | 5 +-- src/utils/data_utils.py | 36 ++++++++++-------- src/utils/gias.py | 5 ++- src/utils/model_utils.py | 75 ++++++++++++++++++++++++++++++-------- 6 files changed, 109 insertions(+), 44 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index a63e2103..070ecf9e 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -187,9 +187,9 @@ def set_model_parameters(self, config: Dict[str, Any]) -> None: optim = torch.optim.SGD else: raise ValueError(f"Unknown optimizer: {optim_name}.") - if "gia" in config: - print("setting optim to gia") - optim = torch.optim.SGD + # if "gia" in config: + # print("setting optim to gia") + # optim = torch.optim.SGD num_classes = self.dset_obj.num_cls num_channels = self.dset_obj.num_channels self.model = self.model_utils.get_model( diff --git a/src/algos/fl.py b/src/algos/fl.py index 5a05d135..3f8257a9 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -20,8 +20,8 @@ def __init__( self, config: Dict[str, Any], comm_utils: CommunicationManager ) -> None: super().__init__(config, comm_utils) - print("WE ARE IN FEDAVG CLIENT") self.config = config + self.random_params = self.model.state_dict() def local_test(self, **kwargs: Any): """ @@ -76,6 +76,10 @@ def get_model_weights(self, **kwargs: Any) -> Dict[str, Any]: # also stream image and labels message["images"] = self.images.to("cpu") message["labels"] = self.labels.to("cpu") + + message["random_params"] = self.random_params + for key in message["random_params"].keys(): + message["random_params"][key] = message["random_params"][key].to("cpu") return message # type: ignore @@ -193,6 +197,12 @@ def receive_and_aggregate_gia(self, round: int, attack_start_round: int, attack_ if key in base_params ) + random_params = rep["random_params"] + random_params = OrderedDict( + (key, value) for key, value in random_params.items() + if key in base_params + ) + # Store parameters based on attack start and end rounds if round == attack_start_round: self.params_s[client_id - 1] = model_params @@ -201,14 +211,15 @@ def receive_and_aggregate_gia(self, round: int, attack_start_round: int, attack_ images = rep["images"] labels = rep["labels"] - with open(f"params_t_{client_id}.pkl", "wb") as f: - pickle.dump(model_params, f) - with open(f"params_s_{client_id}.pkl", "wb") as f: - pickle.dump(self.params_s[client_id - 1], f) + # with open(f"params_t_{client_id}.pkl", "wb") as f: + # pickle.dump(model_params, f) + # with open(f"params_s_{client_id}.pkl", "wb") as f: + # pickle.dump(self.params_s[client_id - 1], f) + # with open(f"random_params_{client_id}.pkl", "wb") as f: + # pickle.dump(random_params, f) # Launch GIA attack - # p_s, p_t = self.params_s[client_id - 1], self.params_t[client_id - 1] - p_s, p_t = self.random_params, self.params_s[client_id - 1] + p_s, p_t = self.params_s[client_id - 1], self.params_t[client_id - 1] gia_main(p_s, p_t, base_params, self.model, labels, images, client_id) avg_wts = self.aggregate(reprs) @@ -230,6 +241,7 @@ def single_round(self, round: int, attack_start_round: int = 0, attack_end_round attack_end_round (int): The last round for the attack to be performed. """ # Normal training when outside the attack range + if round < attack_start_round or round > attack_end_round: self.receive_and_aggregate() else: diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index f66e91bf..62211ff4 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -32,7 +32,6 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st # Collaboration setup "algo": "fedavg", "rounds": 5, - # Model parameters "model": "resnet10", "model_lr": 3e-4, @@ -43,11 +42,11 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st # Collaboration setup "algo": "fedavg", "rounds": 5, - + "optimizer": "sgd", # Model parameters "model": "resnet10", "model_lr": 3e-4, - "batch_size": 256, + # "batch_size": 256, "gia": True, } diff --git a/src/utils/data_utils.py b/src/utils/data_utils.py index 0ca55451..28c3040d 100644 --- a/src/utils/data_utils.py +++ b/src/utils/data_utils.py @@ -366,18 +366,19 @@ def non_iid_balanced( clnt_y = np.asarray(clnt_y) return clnt_y, clnt_idx, cls_priors -def gia_client_dataset(train_dataset, test_dataset, num_labels=10): +def gia_client_dataset(train_dataset, test_dataset, num_labels=10, n=1): """ - Select random labels and exactly one random image per selected label from both train and test datasets. + Select random labels and n random images per selected label from both train and test datasets. Args: train_dataset: Training dataset object with __getitem__ returning (image, label) tuples test_dataset: Test dataset object with __getitem__ returning (image, label) tuples num_labels (int): Number of unique labels to select + n (int): Number of images to select per unique label (default is 1) Returns: - filtered_train_dataset: Subset of training dataset with one image per selected label - filtered_test_dataset: Subset of test dataset with one image per selected label + filtered_train_dataset: Subset of training dataset with n images per selected label + filtered_test_dataset: Subset of test dataset with n images per selected label selected_labels: List of selected label indices train_indices: List of indices for the selected training images """ @@ -390,30 +391,35 @@ def get_ordered_indices(dataset): label_to_indices[label].append(idx) ordered_indices = [] + selected_labels = [] for label in range(num_labels): # Shuffle indices for each label to randomize selection np.random.seed(None) np.random.shuffle(label_to_indices[label]) - random_idx = label_to_indices[label][0] # Select the first random index after shuffling - ordered_indices.append(random_idx) - - return ordered_indices + for i in range(n): + if i < len(label_to_indices[label]): + random_idx = label_to_indices[label][i] + ordered_indices.append(random_idx) + selected_labels.append(label) + + return ordered_indices, selected_labels - # Get ordered indices for both datasets - final_train_indices = get_ordered_indices(train_dataset) - final_test_indices = get_ordered_indices(test_dataset) + # Get ordered indices and selected labels for both datasets + final_train_indices, train_selected_labels = get_ordered_indices(train_dataset) + final_test_indices, test_selected_labels = get_ordered_indices(test_dataset) # Create the subsets filtered_train_dataset = Subset(train_dataset, final_train_indices) filtered_test_dataset = Subset(test_dataset, final_test_indices) # Create selected_labels in ascending order - selected_labels = list(range(num_labels)) + selected_labels = sorted(set(train_selected_labels)) # Verify ordering - for i in range(num_labels): - assert filtered_train_dataset[i][1] == i, f"Train label at position {i} is not {i}" - assert filtered_test_dataset[i][1] == i, f"Test label at position {i} is not {i}" + for i in range(len(final_train_indices)): + assert filtered_train_dataset[i][1] == train_selected_labels[i], f"Train label at position {i} is not {train_selected_labels[i]}" + for i in range(len(final_test_indices)): + assert filtered_test_dataset[i][1] == test_selected_labels[i], f"Test label at position {i} is not {test_selected_labels[i]}" return filtered_train_dataset, filtered_test_dataset, selected_labels, final_train_indices diff --git a/src/utils/gias.py b/src/utils/gias.py index ce8a38c8..2fdcde8b 100644 --- a/src/utils/gias.py +++ b/src/utils/gias.py @@ -24,7 +24,7 @@ def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_ste Reconstructs the gradient following the Geiping InvertingGradients technique """ print("length of param diff: ", len(param_diff)) - with open("param_diff.pkl", "wb") as f: + with open(f"param_diff_{client_id}.pkl", "wb") as f: pickle.dump(param_diff, f) setup = inversefed.utils.system_startup() for p in range(len(param_diff)): @@ -90,6 +90,9 @@ def gia_main(param_s, param_t, base_params, model, target_labels, target_images, """ param_diff = compute_param_delta(param_s, param_t, base_params) + # with open(f"param_diff_after_one_round_{client_id}.pkl", "wb") as f: + # pickle.dump(param_diff, f) + # Check if all elements in para_diff are zero tensors if all((diff == 0).all() for diff in param_diff): print("Parameter differences contain only zeros for client ", client_id) diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index c8de81a3..3bd46c5b 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -15,6 +15,8 @@ import yolo from utils.types import ConfigType +from inversefed.reconstruction_algorithms import loss_steps +import pickle class ModelUtils: def __init__(self, device: torch.device, config: ConfigType) -> None: @@ -193,29 +195,72 @@ def train_classification( if kwargs.get("apply_softmax", False): print("here, applying softmax") output = nn.functional.log_softmax(output, dim=1) # type: ignore - loss = loss_fn(output, target) - if kwargs.get("gia", True): + if kwargs.get("gia", False): # print(data.shape, target.shape) node_id = kwargs.get("node_id") plot_and_save(data, target, filename=f"data_target_plot_{node_id}.png") + # Sum the loss and create gradient graph like in loss_steps + # Use modified loss_steps function that returns loss + model.eval() + param_updates = loss_steps( + model, + data, + target, + loss_fn=loss_fn, + lr=3e-4, + local_steps=1, + use_updates=True, # Must be True to get parameter differences + batch_size=10 + ) + + # save parameter update for sanity check + # with open(f"param_updates_{node_id}.pkl", "wb") as f: + # pickle.dump(param_updates, f) + model.train() + + # Apply the updates to the model parameters + with torch.no_grad(): + for param, update in zip(model.parameters(), param_updates): + param.data.add_(update) # Directly add the update differences + + # Compute loss for tracking (without gradients since we've already updated) + with torch.no_grad(): + position = kwargs.get("position", 0) + output = model(data, position=position) + if kwargs.get("apply_softmax", False): + output = nn.functional.log_softmax(output, dim=1) + loss = loss_fn(output, target) + train_loss += loss.item() + + else: + # Standard training procedure + optim.zero_grad() + position = kwargs.get("position", 0) + output = model(data, position=position) + + if kwargs.get("apply_softmax", False): + output = nn.functional.log_softmax(output, dim=1) + + loss = loss_fn(output, target) + loss.backward() + optim.step() + train_loss += loss.item() - # print("you are here!") - # loss = loss_fn(output, target).sum() - loss.backward() - optim.step() - train_loss += loss.item() - pred = output.argmax(dim=1, keepdim=True) - # view_as() is used to make sure the shape of pred and target are - # the same - if len(target.size()) > 1: - target = target.argmax(dim=1, keepdim=True) - correct += pred.eq(target.view_as(pred)).sum().item() + # Compute accuracy + with torch.no_grad(): + output = model(data, position=position) + pred = output.argmax(dim=1, keepdim=True) + if len(target.size()) > 1: + target = target.argmax(dim=1, keepdim=True) + correct += pred.eq(target.view_as(pred)).sum().item() if test_loader is not None: - # TODO: implement test loader for pascal test_loss, test_acc = self.test(model, test_loader, loss_fn, device) print( - f"Train Loss: {train_loss/(batch_idx+1):.6f} | Train Acc: {correct/((batch_idx+1)*len(data)):.6f} | Test Loss: {test_loss:.6f} | Test Acc: {test_acc:.6f}" + f"Train Loss: {train_loss/(batch_idx+1):.6f} | " + f"Train Acc: {correct/((batch_idx+1)*len(data)):.6f} | " + f"Test Loss: {test_loss:.6f} | " + f"Test Acc: {test_acc:.6f}" ) acc = correct / len(dloader.dataset) From bcde3d1ad17c45d3d296b1535c85899bc0f6a311 Mon Sep 17 00:00:00 2001 From: photonshi Date: Tue, 29 Oct 2024 23:08:18 +0000 Subject: [PATCH 10/16] gia for flStatic - need to add support for multiple attackers --- src/algos/fl_static.py | 43 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/src/algos/fl_static.py b/src/algos/fl_static.py index af0d764a..ea0c83af 100644 --- a/src/algos/fl_static.py +++ b/src/algos/fl_static.py @@ -9,6 +9,7 @@ from algos.base_class import BaseFedAvgClient from algos.topologies.collections import select_topology +from utils.gias import gia_main class FedStaticNode(BaseFedAvgClient): """ @@ -21,12 +22,41 @@ def __init__( super().__init__(config, comm_utils) self.topology = select_topology(config, self.node_id) self.topology.initialize() + if "gia" in config: + self.params_s = dict() + self.params_t = dict() + + self.attack_start_round = 0 + self.attack_end_round = 1 + + self.base_params = [key for key, _ in self.model.named_parameters()] def get_representation(self, **kwargs: Any) -> OrderedDict[str, torch.Tensor]: """ Returns the model weights as representation. """ return self.get_model_weights() + + def receive_attack_and_aggregate(self, neighbors: List]) -> None: + """ + Receives updates, launch GIA attack, and aggregate + """ + for neighbor_info in neighbors: + # Receive the attack from the neighbors + neighbor_id = neighbor_info["sender"] + neighbor_model = neighbor_info["model"] + neighbor_images = neighbor_info["images"] + neighbor_labels = neighbor_info["labels"] + + if round == self.attack_start_round: + self.params_s[neighbor_id] = neighbor_model + elif round == self.attack_end_round: + self.params_t[neighbor_id] = neighbor_model + + # attack + p_s, p_t = self.params_s[neighbor_id], self.params_t[neighbor_id] + gia_main(p_s, p_t, self.base_params, self.model, neighbor_labels, neighbor_images, self.node_id) + def run_protocol(self) -> None: """ @@ -41,6 +71,7 @@ def run_protocol(self) -> None: ) total_rounds = self.config["rounds"] epochs_per_round = self.config.get("epochs_per_round", 1) + for it in range(start_round, total_rounds): # Train locally and send the representation to the server stats["train_loss"], stats["train_acc"], stats["train_time"] = self.local_train( @@ -50,10 +81,18 @@ def run_protocol(self) -> None: # Collect the representations from all other nodes from the server neighbors = self.topology.sample_neighbours(self.num_collaborators) + # TODO: Log the neighbors stats["neighbors"] = neighbors - self.receive_and_aggregate(neighbors) + if "gia" in self.config: + if it < self.attack_start_round or it > self.attack_end_round: + self.receive_and_aggregate(neighbors) + + else: + self.receive_attack_and_aggregate(neighbors) + else: + self.receive_and_aggregate(neighbors) stats["bytes_received"], stats["bytes_sent"] = self.comm_utils.get_comm_cost() @@ -62,8 +101,6 @@ def run_protocol(self) -> None: stats["test_loss"], stats["test_acc"] = self.local_test() self.log_metrics(stats=stats, iteration=it) - - class FedStaticServer(BaseFedAvgClient): """ Federated Static Server Class. It does not do anything. From 9f813f6c904fb1b715601be4c6accc56f18bbb51 Mon Sep 17 00:00:00 2001 From: photonshi Date: Wed, 30 Oct 2024 06:34:16 +0000 Subject: [PATCH 11/16] fl static - need to optimize for attacker to track num rounds --- src/algos/base_class.py | 4 ++ src/algos/fl_static.py | 103 ++++++++++++++++++++++++------------- src/configs/algo_config.py | 3 +- src/configs/sys_config.py | 5 +- src/utils/gias.py | 34 ++++++++++-- 5 files changed, 106 insertions(+), 43 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 070ecf9e..6adba86a 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -120,6 +120,10 @@ def __init__( dropout_rng = random.Random(dropout_seed) self.dropout = NodeDropout(self.node_id, config["dropout_dicts"], dropout_rng) + # TODO set self.gia_attacker true if gia and node_id matches + if "gia" in config and self.node_id in config["gia_attackers"]: + self.gia_attacker = True + def set_constants(self) -> None: """Add docstring here""" self.best_acc = 0.0 diff --git a/src/algos/fl_static.py b/src/algos/fl_static.py index ea0c83af..57e2d1a9 100644 --- a/src/algos/fl_static.py +++ b/src/algos/fl_static.py @@ -1,7 +1,9 @@ """ Module for FedStaticClient and FedStaticServer in Federated Learning. """ -from typing import Any, Dict, OrderedDict +from typing import Any, Dict, OrderedDict, List +from collections import OrderedDict, defaultdict + from utils.communication.comm_utils import CommunicationManager import torch import time @@ -22,12 +24,15 @@ def __init__( super().__init__(config, comm_utils) self.topology = select_topology(config, self.node_id) self.topology.initialize() - if "gia" in config: + if "gia" in config: + if int(self.node_id) in self.config["gia_attackers"]: + self.gia_attacker = True self.params_s = dict() self.params_t = dict() - - self.attack_start_round = 0 - self.attack_end_round = 1 + # Track neighbor updates with a dictionary mapping neighbor_id to their updates + self.neighbor_updates = defaultdict(list) + # Track which neighbors we've already attacked + self.attacked_neighbors = set() self.base_params = [key for key, _ in self.model.named_parameters()] @@ -37,26 +42,60 @@ def get_representation(self, **kwargs: Any) -> OrderedDict[str, torch.Tensor]: """ return self.get_model_weights() - def receive_attack_and_aggregate(self, neighbors: List]) -> None: + def receive_attack_and_aggregate(self, neighbors: List[int], round: int, num_neighbors: int) -> None: """ - Receives updates, launch GIA attack, and aggregate + Receives updates, launches GIA attack when second update is seen from a neighbor """ - for neighbor_info in neighbors: - # Receive the attack from the neighbors - neighbor_id = neighbor_info["sender"] - neighbor_model = neighbor_info["model"] - neighbor_images = neighbor_info["images"] - neighbor_labels = neighbor_info["labels"] - - if round == self.attack_start_round: - self.params_s[neighbor_id] = neighbor_model - elif round == self.attack_end_round: - self.params_t[neighbor_id] = neighbor_model - - # attack - p_s, p_t = self.params_s[neighbor_id], self.params_t[neighbor_id] - gia_main(p_s, p_t, self.base_params, self.model, neighbor_labels, neighbor_images, self.node_id) - + print("CLIENT RECEIVING ATTACK AND AGGREGATING") + if self.is_working: + # Receive the model updates from the neighbors + model_updates = self.comm_utils.receive(node_ids=neighbors) + assert len(model_updates) == num_neighbors + + for neighbor_info in model_updates: + neighbor_id = neighbor_info["sender"] + neighbor_model = neighbor_info["model"] + neighbor_model = OrderedDict( + (key, value) for key, value in neighbor_model.items() + if key in self.base_params + ) + + neighbor_images = neighbor_info["images"] + neighbor_labels = neighbor_info["labels"] + + # Store this update + self.neighbor_updates[neighbor_id].append({ + "model": neighbor_model, + "images": neighbor_images, + "labels": neighbor_labels + }) + + # Check if we have 2 updates from this neighbor and haven't attacked them yet + if len(self.neighbor_updates[neighbor_id]) == 2 and neighbor_id not in self.attacked_neighbors: + print(f"Client {self.node_id} attacking {neighbor_id}!") + + # Get the two parameter sets for the attack + p_s = self.neighbor_updates[neighbor_id][0]["model"] + p_t = self.neighbor_updates[neighbor_id][1]["model"] + + # Launch the attack + gia_main( + p_s, + p_t, + self.base_params, + self.model, + neighbor_labels, + neighbor_images, + self.node_id + ) + + # Mark this neighbor as attacked + self.attacked_neighbors.add(neighbor_id) + + # Optionally, clear the stored updates to save memory + del self.neighbor_updates[neighbor_id] + + self.aggregate(model_updates, keys_to_ignore=self.model_keys_to_ignore) def run_protocol(self) -> None: """ @@ -75,29 +114,21 @@ def run_protocol(self) -> None: for it in range(start_round, total_rounds): # Train locally and send the representation to the server stats["train_loss"], stats["train_acc"], stats["train_time"] = self.local_train( - it, epochs_per_round - ) + it, epochs_per_round + ) self.local_round_done() # Collect the representations from all other nodes from the server neighbors = self.topology.sample_neighbours(self.num_collaborators) - - # TODO: Log the neighbors stats["neighbors"] = neighbors - if "gia" in self.config: - if it < self.attack_start_round or it > self.attack_end_round: - self.receive_and_aggregate(neighbors) - - else: - self.receive_attack_and_aggregate(neighbors) + if hasattr(self, "gia_attacker"): + print(f"Client {self.node_id} is a GIA attacker!") + self.receive_attack_and_aggregate(neighbors, it, len(neighbors)) else: self.receive_and_aggregate(neighbors) stats["bytes_received"], stats["bytes_sent"] = self.comm_utils.get_comm_cost() - - # evaluate the model on the test data - # Inside FedStaticNode.run_protocol() stats["test_loss"], stats["test_acc"] = self.local_test() self.log_metrics(stats=stats, iteration=it) diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index 62211ff4..4a54487a 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -203,9 +203,10 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st # Collaboration setup "algo": "fedstatic", "topology": {"name": "watts_strogatz", "k": 3, "p": 0.2}, # type: ignore - "rounds": 20, + "rounds": 3, # Model parameters + "optimizer": "sgd", # TODO comment out for real training "model": "resnet10", "model_lr": 3e-4, "batch_size": 256, diff --git a/src/configs/sys_config.py b/src/configs/sys_config.py index 12b0ff33..30775bb6 100644 --- a/src/configs/sys_config.py +++ b/src/configs/sys_config.py @@ -365,13 +365,14 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): "seed": 2, "device_ids": get_device_ids(num_users, gpu_ids), # "algos": get_algo_configs(num_users=num_users, algo_configs=default_config_list), # type: ignore - "algos": get_algo_configs(num_users=num_users, algo_configs=[test_fl_inversion]), # type: ignore + "algos": get_algo_configs(num_users=num_users, algo_configs=[fedstatic]), # type: ignore "samples_per_user": 50000 // num_users, # distributed equally "train_label_distribution": "iid", "test_label_distribution": "iid", "exp_keys": [], "dropout_dicts": dropout_dicts, - "gia":True + "gia":True, + "gia_attackers":[1] } current_config = grpc_system_config_gia diff --git a/src/utils/gias.py b/src/utils/gias.py index 2fdcde8b..7b914f57 100644 --- a/src/utils/gias.py +++ b/src/utils/gias.py @@ -36,7 +36,7 @@ def reconstruct_gradient(param_diff, target_labels, target_images, lr, local_ste mean, std = [0.4914, 0.4822, 0.4465], [0.2023, 0.1994, 0.2010] dm = torch.as_tensor(mean, **setup)[:, None, None] ds = torch.as_tensor(std, **setup)[:, None, None] - + model = model.to(setup['device']) config = dict(signed=True, boxed=True, cost_fn='sim', @@ -87,16 +87,42 @@ def grid_plot(tensor, labels, ds, dm, stats, test_mse, feat_mse, test_psnr, save def gia_main(param_s, param_t, base_params, model, target_labels, target_images, client_id): """ Main function for Gradient Inversion Attack + Returns results moved back to their original devices """ + # Store original devices + model_device = next(model.parameters()).device + target_labels_device = target_labels.device + target_images_device = target_images.device + + # Store original parameter devices + param_s_devices = {name: param_s[name].device for name in base_params if name in param_s} + param_t_devices = {name: param_t[name].device for name in base_params if name in param_t} + param_diff = compute_param_delta(param_s, param_t, base_params) - # with open(f"param_diff_after_one_round_{client_id}.pkl", "wb") as f: - # pickle.dump(param_diff, f) - # Check if all elements in para_diff are zero tensors if all((diff == 0).all() for diff in param_diff): print("Parameter differences contain only zeros for client ", client_id) return None # or return an empty list, depending on your needs output, stats = reconstruct_gradient(param_diff, target_labels, target_images, 3e-4, 1, model, client_id) + + # Move output back to target_images device (since it's a reconstruction of the images) + if output is not None: + output = output.to(target_images_device) + + # Move model back to original device + model.to(model_device) + + # Move parameters back to their original devices + for name in base_params: + if name in param_s: + param_s[name] = param_s[name].to(param_s_devices[name]) + if name in param_t: + param_t[name] = param_t[name].to(param_t_devices[name]) + + # Move labels and images back to their original devices + target_labels = target_labels.to(target_labels_device) + target_images = target_images.to(target_images_device) + return output, stats \ No newline at end of file From 410eaa7cb533546b3f75665b3c0e86a4b82d0437 Mon Sep 17 00:00:00 2001 From: photonshi Date: Fri, 8 Nov 2024 07:14:54 +0000 Subject: [PATCH 12/16] fixed PR changes --- src/algos/base_class.py | 7 + src/configs/algo_config.py | 3 +- src/configs/sys_config.py | 5 +- src/data_loaders/cifar.py | 4 +- src/test_inversion.ipynb | 9178 ------------------------- src/utils/communication/comm_utils.py | 13 - src/utils/log_utils.py | 44 + src/utils/model_utils.py | 50 +- 8 files changed, 57 insertions(+), 9247 deletions(-) delete mode 100644 src/test_inversion.ipynb diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 6adba86a..4da52be5 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -397,6 +397,13 @@ def set_parameters(self, config: Dict[str, Any]) -> None: self.set_shared_exp_parameters(config) self.set_data_parameters(config) + # after setting data loaders, save client dataset + # TODO verify this .data and .labels fields are correct + if "gia" in config: + self.log_utils.log_gia_target(self.train_dset.data, + self.train_dset.labels, + self.node_id) + def set_data_parameters(self, config: ConfigType) -> None: # Train set and test set from original dataset diff --git a/src/configs/algo_config.py b/src/configs/algo_config.py index 4a54487a..ed931fb7 100644 --- a/src/configs/algo_config.py +++ b/src/configs/algo_config.py @@ -339,5 +339,4 @@ def get_malicious_types(malicious_config_list: List[ConfigType]) -> Dict[str, st malicious_traditional_model_update_attack, ] -# default_config_list: List[ConfigType] = [traditional_fl] -default_config_list: List[ConfigType] = [test_fl_inversion] +default_config_list: List[ConfigType] = [traditional_fl] diff --git a/src/configs/sys_config.py b/src/configs/sys_config.py index 30775bb6..b8d75330 100644 --- a/src/configs/sys_config.py +++ b/src/configs/sys_config.py @@ -157,7 +157,7 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): CIAR10_DPATH = "./datasets/imgs/cifar10/" NUM_COLLABORATORS = 1 -DUMP_DIR = "/u/yshi23/sonar/src/expt_dump/test/" +DUMP_DIR = "/mas/camera/Experiments/SONAR/abhi/" mpi_system_config: ConfigType = { "exp_id": "", @@ -375,6 +375,5 @@ def get_digit_five_support(num_users: int, domains: List[str] = DIGIT_FIVE): "gia_attackers":[1] } -current_config = grpc_system_config_gia -# current_config = grpc_system_config +current_config = grpc_system_config # current_config = mpi_system_config diff --git a/src/data_loaders/cifar.py b/src/data_loaders/cifar.py index 4e474630..80405f57 100644 --- a/src/data_loaders/cifar.py +++ b/src/data_loaders/cifar.py @@ -18,8 +18,8 @@ def __init__(self, dpath: str, rot_angle: int = 0) -> None: self.train_transform = T.Compose( [ - # T.RandomCrop(32, padding=4), - # T.RandomHorizontalFlip(), + T.RandomCrop(32, padding=4), + T.RandomHorizontalFlip(), T.ToTensor(), T.Normalize(self.mean, self.std), ] diff --git a/src/test_inversion.ipynb b/src/test_inversion.ipynb deleted file mode 100644 index e65f9557..00000000 --- a/src/test_inversion.ipynb +++ /dev/null @@ -1,9178 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [], - "source": [ - "# script to test basic functionality of gia package\n", - "\n", - "import inversefed\n", - "\n", - "import torch\n", - "import numpy as np\n", - "import matplotlib.pyplot as plt\n", - "from collections import defaultdict\n", - "from PIL import Image\n", - "from torchvision.utils import save_image\n", - "from utils.model_utils import ModelUtils\n", - "from torch.utils.data import DataLoader, Dataset\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "arch = 'ResNet18'\n", - "num_images = 10\n", - "trained_model = False\n", - "device = 'cuda'" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Currently evaluating -------------------------------:\n", - "Sunday, 27. October 2024 04:25AM\n", - "CPUs: 20, GPUs: 4 on matlaber12.\n", - "GPU : NVIDIA GeForce GTX 1080 Ti\n", - "Files already downloaded and verified\n", - "Files already downloaded and verified\n", - "Model initialized with random key 3695468145.\n" - ] - } - ], - "source": [ - "import inversefed\n", - "setup = inversefed.utils.system_startup()\n", - "defs = inversefed.training_strategy('conservative')\n", - "\n", - "loss_fn, trainloader, validloader = inversefed.construct_dataloaders('CIFAR10', defs)\n", - "\n", - "mutils = ModelUtils(device=\"cuda\")\n", - "model_bespoke = mutils.get_model(\"resnet10\", \"cifar10\", \"cuda\")\n", - "\n", - "model_control, _ = inversefed.construct_model(arch, num_classes=10, num_channels=3)\n", - "model_control.to(**setup)\n", - "if trained_model:\n", - " epochs = 120\n", - " file = f'{arch}_{epochs}.pth'\n", - " try:\n", - " model_control.load_state_dict(torch.load(f'models/{file}'))\n", - " except FileNotFoundError:\n", - " inversefed.train(model_control, loss_fn, trainloader, validloader, defs, setup=setup)\n", - " torch.save(model_control.state_dict(), f'models/{file}')\n", - "model_control.eval();" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "dm = torch.as_tensor(inversefed.consts.cifar10_mean, **setup)[:, None, None]\n", - "ds = torch.as_tensor(inversefed.consts.cifar10_std, **setup)[:, None, None]\n", - "def plot(tensor):\n", - " tensor = tensor.clone().detach()\n", - " tensor.mul_(ds).add_(dm).clamp_(0, 1)\n", - " if tensor.shape[0] == 1:\n", - " return plt.imshow(tensor[0].permute(1, 2, 0).cpu());\n", - " else:\n", - " fig, axes = plt.subplots(1, tensor.shape[0], figsize=(12, tensor.shape[0]*12))\n", - " for i, im in enumerate(tensor):\n", - " axes[i].imshow(im.permute(1, 2, 0).cpu());" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "num_images = 10\n", - "ground_truth, labels = [], []\n", - "\n", - "for idx in range(num_images*3):\n", - " img, label = validloader.dataset[idx]\n", - " if label not in labels:\n", - " labels.append(torch.as_tensor((label,), device=setup['device']))\n", - " ground_truth.append(img.to(**setup))\n", - "\n", - "ground_truth = torch.stack(ground_truth)\n", - "labels = torch.cat(labels)\n", - "\n", - "ground_truth_target = ground_truth\n", - "labels_target = labels" - ] - }, - { - "cell_type": "code", - "execution_count": 54, - "metadata": {}, - "outputs": [], - "source": [ - "# num_images = 1\n", - "# ground_truth_target, labels_target = [], []\n", - "# idx = 25 # choosen randomly ... just whatever you want\n", - "# while len(labels_target) < num_images:\n", - "# img, label = validloader.dataset[idx]\n", - "# idx += 1\n", - "# if label not in labels_target:\n", - "# labels_target.append(torch.as_tensor((label,), device=setup['device']))\n", - "# ground_truth_target.append(img.to(**setup))\n", - "# ground_truth_target = torch.stack(ground_truth_target)\n", - "# labels_target = torch.cat(labels_target)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "(torch.Size([10, 3, 32, 32]),\n", - " torch.Size([10]),\n", - " torch.Size([10, 3, 32, 32]),\n", - " torch.Size([10]))" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "ground_truth.shape, labels.shape, ground_truth_target.shape, labels_target.shape" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAB1cAAADHCAYAAABFldqsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAADA9ElEQVR4nOz9d5hkV3X2Da+TKld1jhO6J4+kURwllEYBkETQI0ASwTaSAEtYBMsP4MQFEvkxYIONX4yxjcCGz4+RSCZJIBDKAuU0o8mxe6ZzdVeuOufs9w++mVen7iUoxRrM/bsuLlH37HPODmuvvfY+XbUsY4wRQgghhBBCCCGEEEIIIYQQQgghvxG73RUghBBCCCGEEEIIIYQQQgghhJDfBfhylRBCCCGEEEIIIYQQQgghhBBCWoAvVwkhhBBCCCGEEEIIIYQQQgghpAX4cpUQQgghhBBCCCGEEEIIIYQQQlqAL1cJIYQQQgghhBBCCCGEEEIIIaQF+HKVEEIIIYQQQgghhBBCCCGEEEJagC9XCSGEEEIIIYQQQgghhBBCCCGkBfhylRBCCCGEEEIIIYQQQgghhBBCWoAvVwkhhBBCCCGEEEIIIYQQQgghpAX4cpUQ8pw4++yzZd26db+13K5du8SyLPnqV7/64leKkN/AL37xC7EsS37xi1/8TtyXkJeC+++/X0477TRJp9NiWZY88sgj7a4SIf/jGR0dlde85jW/tZy2vlxxxRUyOjr64lWOkOfB9ddfL5ZlyfT0dLurQojKQRsl5HcN+ldCnhn6dvL7QKvrwOjoqFxxxRXP61lnn322nH322c/rHr8v8OVqmxkfH5frr7+eh5mEEEIIeUlpNBpy6aWXyuzsrHzuc5+T//iP/5CRkZF2V4uQ58QXv/hF/iEX+b3nnnvukeuvv17y+Xy7q0IIIYQQQggh/6Nx212B33fGx8flIx/5iIyOjspxxx3X7uoQ8oIzMjIilUpFPM9rd1XI7zlnnXWWVCoVicVi7a4KIYcF27dvl927d8u//Mu/yDve8Y52V4eQ58UXv/hF6e3tfd5/pXs4wXWLPFvuuece+chHPiJXXHGFdHZ2trs6hBBCCCGEkJeQzZs3i23z+5QvFexpQsiLimVZkkgkxHGcdleF/J5j27YkEonfGmSUy+WXqEaEtJfJyUkRkd96AF8qlV6C2hBCmml13SLk2RKGoVSr1XZXgxBCyGGCMUYqlUq7q0HI7wzcI5PDlXg8/lu/4ET7feHgTv15MDY2Jm9/+9tleHhY4vG4LFu2TP7kT/5E6vW6zM7Oyvvf/345+uijJZPJSC6XkwsvvFAeffTRQ9f/4he/kJNOOklERK688kqxLIu5KclhQ6FQkGuvvVZGR0clHo9Lf3+/vOIVr5CHHnooUm7jxo1yzjnnSCqVkkWLFsmnP/3pyL9rOVevuOIKyWQysmPHDjn//PMlnU7L8PCwfPSjHxVjzEvRPPI/iN27d8s111wja9askWQyKT09PXLppZfKrl27IuW03HUHcwc/+OCDctZZZ0kqlZK//uu/FpH/LyfeT37yEznuuOMkkUjIkUceKd/+9rd/a53uvPNOufTSS2Xp0qUSj8dlyZIl8md/9mewYT04F8bGxuTiiy+WTCYjfX198v73v1+CIIiUDcNQPv/5z8tRRx0liURCBgYG5Oqrr5a5ubnn1nHk95orrrhCNmzYICIil156qViWJWefffYhm9y+fbu86lWvkmw2K3/wB38gIr8OwN/3vvfJkiVLJB6Py5o1a+Szn/0s+O1KpSLvfe97pbe3V7LZrFx00UUyNjYmlmXJ9ddf/1I3lRyGtOq3nyl/0le/+lWxLOtQ+dHRUXnyySfl9ttvPxRPPz1HzI4dO+TSSy+V7u5uSaVScuqpp8oPf/jDyD0PrhHf/OY35SMf+YgsWrRIstmsXHLJJTI/Py+1Wk2uvfZa6e/vl0wmI1deeaXUarXIPXzfl4997GOyYsUKicfjMjo6Kn/9138N5Q7y29aXVnN6c30gIr+eLx/4wAdERGTZsmWH5sLBWPzd7363fOMb35CjjjpK4vG43Hzzzc9oY1r8LiLy1FNPyWWXXSZ9fX2STCZlzZo18sEPfvA31mv37t2ycuVKWbdunUxMTLyQTSbkN3LXXXfJSSedJIlEQlasWCH//M//DGVa9dthGMr1118vw8PDkkql5JxzzpGNGze+IHnNCHk25PP5Q79O0NHRIVdeeWXkj4NbtemDe91bbrlFTjzxREkmk4fmyE9/+lM544wzpLOzUzKZjKxZs+bQHvkgtVpNrrvuOlm5cuWh/e6f//mfP2PMQ8gLRSu+XUTk61//uqxfv16SyaR0d3fLm970Jtm7dy+U++UvfykXXHCBdHR0SCqVkg0bNsjdd98dKXNwT7Jx40Z5y1veIl1dXXLGGWe8KO0j5LcxPT0tl112meRyOenp6ZE//dM/jfzRZHNscnDvfPvtt8s111wj/f39snjx4kP//uUvf1lWrFghyWRSTj75ZLnzzjtfyub8zsOfBX6OjI+Py8knnyz5fF6uuuoqWbt2rYyNjclNN90k5XJZduzYId/97nfl0ksvlWXLlsnExIT88z//s2zYsEE2btwow8PDcsQRR8hHP/pR+fCHPyxXXXWVnHnmmSIictppp7W5dYSIvPOd75SbbrpJ3v3ud8uRRx4pMzMzctddd8mmTZvkhBNOEBGRubk5ueCCC+T1r3+9XHbZZXLTTTfJX/zFX8jRRx8tF1544W+8fxAEcsEFF8ipp54qn/70p+Xmm2+W6667Tnzfl49+9KMvRRPJ/xDuv/9+ueeee+RNb3qTLF68WHbt2iX/9E//JGeffbZs3LhRUqnUb7x+ZmZGLrzwQnnTm94kf/iHfygDAwOH/m3r1q3yxje+Ud75znfK5ZdfLjfccINceumlcvPNN8srXvGKZ7znjTfeKOVyWf7kT/5Eenp65Fe/+pV84QtfkH379smNN94YKRsEgZx//vlyyimnyGc/+1m59dZb5W//9m9lxYoV8id/8ieHyl199dXy1a9+Va688kp573vfKzt37pR//Md/lIcffljuvvtu/vQ2eVZcffXVsmjRIvnkJz8p733ve+Wkk06SgYEB+cY3viG+78v5558vZ5xxhnz2s5+VVColxhi56KKL5LbbbpO3v/3tctxxx8ktt9wiH/jAB2RsbEw+97nPHbr3FVdcId/85jflj/7oj+TUU0+V22+/XV796le3sbXkcOP5+u1mPv/5z8t73vMeyWQyh170HPTlExMTctppp0m5XJb3vve90tPTI1/72tfkoosukptuukle97rXRe71qU99SpLJpPzlX/6lbNu2Tb7whS+I53li27bMzc3J9ddfL/fdd5989atflWXLlsmHP/zhQ9e+4x3vkK997WtyySWXyPve9z755S9/KZ/61Kdk06ZN8p3vfCfynOe6vmhwfSAiIq9//etly5Yt8p//+Z/yuc99Tnp7e0VEpK+vT0REfv7zn8s3v/lNefe73y29vb0yOjr6rHKzPvbYY3LmmWeK53ly1VVXyejoqGzfvl2+//3vyyc+8Qn1mu3bt8u5554r3d3d8tOf/vRQnQh5sXn88cflla98pfT19cn1118vvu/LddddF4nzRVr323/1V38ln/70p+W1r32tnH/++fLoo4/K+eefz2+Ak5ecyy67TJYtWyaf+tSn5KGHHpJ//dd/lf7+fvmbv/kbEXl2scjmzZvlzW9+s1x99dXyx3/8x7JmzRp58skn5TWveY0cc8wx8tGPflTi8bhs27Yt8rIpDEO56KKL5K677pKrrrpKjjjiCHn88cflc5/7nGzZskW++93vvpRdQn6PaNW3f+ITn5APfehDctlll8k73vEOmZqaki984Qty1llnycMPP3zol5t+/vOfy4UXXijr16+X6667TmzblhtuuEHOPfdcufPOO+Xkk0+O3PfSSy+VVatWySc/+Ul+MYS0jcsuu0xGR0flU5/6lNx3333yD//wDzI3Nyf//u///huvu+aaa6Svr08+/OEPH/rm6r/927/J1VdfLaeddppce+21smPHDrnooouku7tblixZ8lI053cfQ54Tb33rW41t2+b++++HfwvD0FSrVRMEQUTfuXOnicfj5qMf/egh7f777zciYm644YYXu8qEPCs6OjrMu971rmf89w0bNhgRMf/+7/9+SKvVamZwcNC84Q1vOKTt3LkTbPzyyy83ImLe8573HNLCMDSvfvWrTSwWM1NTUy9sY8j/aMrlMmj33nsv2Odtt91mRMTcdttth7SDdvylL30J7jEyMmJExHzrW986pM3Pz5uhoSFz/PHH/8b7anX61Kc+ZSzLMrt37z6kHZwLT18XjDHm+OOPN+vXrz/0+c477zQiYr7xjW9Eyt18882qTkgrHLTdG2+88ZB20Cb/8i//MlL2u9/9rhER8/GPfzyiX3LJJcayLLNt2zZjjDEPPvigERFz7bXXRspdccUVRkTMdddd9+I0hvxO0arfvu6664y2XbnhhhuMiJidO3ce0o466iizYcMGKHvttdcaETF33nnnIa1QKJhly5aZ0dHRQ/H6wfmwbt06U6/XD5V985vfbCzLMhdeeGHkvi972cvMyMjIoc+PPPKIERHzjne8I1Lu/e9/vxER8/Of//yQ9nzWl8svvzzyXK4P5Ol85jOfgblhjDEiYmzbNk8++WRE12zMGD1+P+uss0w2m43EMcb8OoY/yME5OzU1ZTZt2mSGh4fNSSedZGZnZ1+Q9hHSKhdffLFJJBIRe924caNxHOfQutKq3z5w4IBxXddcfPHFkXLXX3+9ERFz+eWXv7iNIcT8f/71bW97W0R/3eteZ3p6eowxzy0WufnmmyNlP/e5zx3y48/Ef/zHfxjbtiOxlTHGfOlLXzIiYu6+++7n1EZCfhut+PZdu3YZx3HMJz7xici1jz/+uHFd95AehqFZtWqVOf/88yOxTLlcNsuWLTOveMUrDmkH59+b3/zmF7N5hPxGDtrhRRddFNGvueYaIyLm0UcfNcb82r8/PTY5uHc+44wzjO/7h/R6vW76+/vNcccdZ2q12iH9y1/+shERdW9NEP4s8HMgDEP57ne/K6997WvlxBNPhH+3LEvi8fih/EhBEMjMzMyhn9No/llVQg5HOjs75Ze//KWMj48/Y5lMJiN/+Id/eOhzLBaTk08+WXbs2NHSM9797ncf+v8Hf66sXq/Lrbfe+twrTn7vSCaTh/5/o9GQmZkZWblypXR2drbkb+PxuFx55ZXqvw0PD0e+1ZTL5eStb32rPPzww3LgwIGW6lQqlWR6elpOO+00McbIww8/DOXf+c53Rj6feeaZkXl04403SkdHh7ziFa+Q6enpQ/9bv369ZDIZue22235rOwl5Njz9W9MiIj/60Y/EcRx573vfG9Hf9773iTFGfvzjH4uIyM033ywiv/6ryKfznve850WsLfld4/n67WfDj370Izn55JMjP92VyWTkqquukl27dsnGjRsj5d/61rdGvul5yimniDFG3va2t0XKnXLKKbJ3717xff/Qc0RE/vf//t+Rcu973/tEROBniJ/r+tIM1wfSKhs2bJAjjzzyOV07NTUld9xxh7ztbW+TpUuXRv5N++nuJ554QjZs2CCjo6Ny6623SldX13N6LiHPhSAI5JZbbpGLL744Yq9HHHGEnH/++Yc+t+q3f/azn4nv+4xtyGGBtm+cmZmRhYWFZx2LLFu2LDInROTQN/q+973vSRiGah1uvPFGOeKII2Tt2rWR2OPcc88VEWHsQV4UWvXt3/72tyUMQ7nssssi9jk4OCirVq06ZJ+PPPKIbN26Vd7ylrfIzMzMoXKlUknOO+88ueOOO2AONM8/QtrBu971rsjng/HIwTXgmfjjP/5jcRzn0OcHHnhAJicn5Z3vfKfEYrFD+hVXXCEdHR0vYI3/Z8OXq8+BqakpWVhYkHXr1j1jmTAM5XOf+5ysWrVK4vG49Pb2Sl9fnzz22GMyPz//EtaWkOfGpz/9aXniiSdkyZIlcvLJJ8v1118PL00XL14MBypdXV0t5fiybVuWL18e0VavXi0iAjnXCPlNVCoV+fCHP3woD+RBf5vP51vyt4sWLYoEEk9n5cqVYOOt2OmePXvkiiuukO7u7kN5VA/mt2yuUyKROPSTfQdpnkdbt26V+fl56e/vl76+vsj/isWiTE5O/tZ2EtIqrutGcnCI/Dpn3vDwsGSz2Yh+xBFHHPr3g/+1bVuWLVsWKbdy5coXscbkd43n67efDbt375Y1a9aA3my7B2l+cXRwY9n8s0gdHR0ShuGh+h60/WZbHxwclM7OTnjOc11fmuH6QFql2S8/Gw7uAX7T/vfpvPa1r5VsNiu33HKL5HK55/xcQp4LU1NTUqlUZNWqVfBvT18PWvXbB//bXK67u5t/OEBecprjlIM2ODc396xjEW1deOMb3yinn366vOMd75CBgQF505veJN/85jcjL5m2bt0qTz75JMQdB+MYxh7kxaBV375161YxxsiqVavARjdt2nTIPrdu3SoiIpdffjmU+9d//Vep1WqwL3k+sRQhLxTNc2DFihVi2/Zv3UM22+/BNaH5fp7nwXk9eWaYc/VF4pOf/KR86EMfkre97W3ysY99TLq7u8W2bbn22muf8a+/CDmcuOyyy+TMM8+U73znO/KTn/xEPvOZz8jf/M3fyLe//e1D+VSf/hcvT8cw9wB5CXnPe94jN9xwg1x77bXyspe9TDo6OsSyLHnTm97Ukr99+jeoXgiCIJBXvOIVMjs7K3/xF38ha9eulXQ6LWNjY3LFFVdAnZ5pHj2dMAylv79fvvGNb6j/3vxylpDnw9N/fYOQF4NW/bb2jTiRX/vZF4tn8smtxjzPVOcXC64PpFW0eOfFmmNveMMb5Gtf+5p84xvfkKuvvvp53YuQF5uX2m8T8nxoJR5p1aa1dSGZTModd9wht912m/zwhz+Um2++Wf7rv/5Lzj33XPnJT34ijuNIGIZy9NFHy9/93d+p92WePtJOwjAUy7Lkxz/+sTpfMpnMoXIiIp/5zGfkuOOOU+91sOxBXuizI0JeCJ6PzyfPH75cfQ709fVJLpeTJ5544hnL3HTTTXLOOefIv/3bv0X0fD4vvb29hz4zkCeHM0NDQ3LNNdfINddcI5OTk3LCCSfIJz7xiUMvV58PYRjKjh07Dv11o4jIli1bRERkdHT0ed+f/P5w0003yeWXXy5/+7d/e0irVquSz+ef9723bdsmxpiIr/5tdvr444/Lli1b5Gtf+5q89a1vPaT/9Kc/fc71WLFihdx6661y+umnMyAibWFkZERuvfVWKRQKkW+vPvXUU4f+/eB/wzCUnTt3Rv4Cctu2bS9thclhTat+++C3MfL5/KGfqRPBb5uKPHNMPTIyIps3bwa92XafLwdtf+vWrYe+FSsiMjExIfl8Hp7zXNYXDa4P5Ok8273l0+fY02meYwf/ev037X+fzmc+8xlxXVeuueYayWaz8pa3vOVZ1YuQ50NfX58kk8lD30p6Ok9fD1r12wf/u23btsi3PmZmZlr6xSZCXiqebSzyTNi2Leedd56cd9558nd/93fyyU9+Uj74wQ/KbbfdJi9/+ctlxYoV8uijj8p5553HM03yktGqb1+xYoUYY2TZsmWR88ZmVqxYISK/Ts3x8pe//IWvMCEvElu3bo3EI9u2bZMwDJ/1WfrBNWHr1q2HftZd5Ndpe3bu3CnHHnvsC1Lf/+nwawnPAdu25eKLL5bvf//78sADD8C/G2PEcRz4S/Ybb7xRxsbGIlo6nRYR3NAS0k6CIICfv+jv75fh4WGp1Wov2HP+8R//8dD/N8bIP/7jP4rneXLeeee9YM8g//PR/O0XvvCFF+SbTePj4/Kd73zn0OeFhQX593//dznuuONkcHDwGesjEv3rYWOM/P3f//1zrsdll10mQRDIxz72Mfg33/e5hpAXnVe96lUSBEHEb4uIfO5znxPLsg790c3BfDdf/OIXI+W+8IUvvDQVJb8TtOq3Dx563HHHHYe0UqkkX/va1+Ce6XRa9YWvetWr5Fe/+pXce++9kXt8+ctfltHR0eecg1J7jojI5z//+Yh+8Fsdr371qyP6c1lfNLg+kKfzbPeWIyMj4jhOZI6JoA/v6+uTs846S77yla/Inj17Iv+m/WKNZVny5S9/WS655BK5/PLL5b//+7+fRSsIeX44jiPnn3++fPe7343Y66ZNm+SWW2459LlVv33eeeeJ67ryT//0T5FyzTERIe3m2cYiGrOzs6Ad/FbfwbOgyy67TMbGxuRf/uVfoGylUpFSqfRsqk1IS7Tq21//+teL4zjykY98BGIUY4zMzMyIiMj69etlxYoV8tnPflaKxSI8b2pq6kVqCSHPj//n//l/Ip8PnrU82y9CnXjiidLX1ydf+tKXpF6vH9K/+tWvcg/5LOA3V58jn/zkJ+UnP/mJbNiwQa666io54ogjZP/+/XLjjTfKXXfdJa95zWvkox/9qFx55ZVy2mmnyeOPPy7f+MY34DerV6xYIZ2dnfKlL31JstmspNNpOeWUU/g77qStFAoFWbx4sVxyySVy7LHHSiaTkVtvvVXuv//+yLdMng+JREJuvvlmufzyy+WUU06RH//4x/LDH/5Q/vqv/5o/YUeeFa95zWvkP/7jP6Sjo0OOPPJIuffee+XWW2+Vnp6e533v1atXy9vf/na5//77ZWBgQL7yla/IxMSE3HDDDc94zdq1a2XFihXy/ve/X8bGxiSXy8m3vvWt5/WX7Rs2bJCrr75aPvWpT8kjjzwir3zlK8XzPNm6davceOON8vd///dyySWXPOf7E/LbeO1rXyvnnHOOfPCDH5Rdu3bJscceKz/5yU/ke9/7nlx77bWHXoKtX79e3vCGN8jnP/95mZmZkVNPPVVuv/32Q9/I41+3E5HW/fYrX/lKWbp0qbz97W+XD3zgA+I4jnzlK1+Rvr4+eMGzfv16+ad/+if5+Mc/LitXrpT+/n4599xz5S//8i/lP//zP+XCCy+U9773vdLd3S1f+9rXZOfOnfKtb33rBfsJ7GOPPVYuv/xy+fKXvyz5fF42bNggv/rVr+RrX/uaXHzxxXLOOedEyj+X9UWD6wN5OuvXrxcRkQ9+8IPypje9STzPk9e+9rXPWL6jo0MuvfRS+cIXviCWZcmKFSvkBz/4gZov7x/+4R/kjDPOkBNOOEGuuuoqWbZsmezatUt++MMfyiOPPALlbduWr3/963LxxRfLZZddJj/60Y8ifxVPyIvJRz7yEbn55pvlzDPPlGuuuUZ835cvfOELctRRR8ljjz0mIq377YGBAfnTP/1T+du//Vu56KKL5IILLpBHH31UfvzjH0tvby9jG3LY8GxjEY2PfvSjcscdd8irX/1qGRkZkcnJSfniF78oixcvljPOOENERP7oj/5IvvnNb8o73/lOue222+T000+XIAjkqaeekm9+85tyyy23yIknnvhiN5f8HtKKb1+xYoV8/OMfl7/6q7+SXbt2ycUXXyzZbFZ27twp3/nOd+Sqq66S97///WLbtvzrv/6rXHjhhXLUUUfJlVdeKYsWLZKxsTG57bbbJJfLyfe///02t5gQZOfOnYfikXvvvVe+/vWvy1ve8pZn/U1Tz/Pk4x//uFx99dVy7rnnyhvf+EbZuXOn3HDDDcy5+mww5Dmze/du89a3vtX09fWZeDxuli9fbt71rneZWq1mqtWqed/73meGhoZMMpk0p59+urn33nvNhg0bzIYNGyL3+d73vmeOPPJI47quERFzww03tKU9hBykVquZD3zgA+bYY4812WzWpNNpc+yxx5ovfvGLh8ps2LDBHHXUUXDt5ZdfbkZGRg593rlzJ9j15ZdfbtLptNm+fbt55StfaVKplBkYGDDXXXedCYLgxWwa+R/I3NycufLKK01vb6/JZDLm/PPPN0899ZQZGRkxl19++aFyt912mxERc9tttx3SnsmOjTFmZGTEvPrVrza33HKLOeaYY0w8Hjdr1641N954Y6Scdt+NGzeal7/85SaTyZje3l7zx3/8x+bRRx99xrnQzHXXXWe0JfrLX/6yWb9+vUkmkyabzZqjjz7a/Pmf/7kZHx9vrbMIeRoHbffpNv1MNmmMMYVCwfzZn/2ZGR4eNp7nmVWrVpnPfOYzJgzDSLlSqWTe9a53me7ubpPJZMzFF19sNm/ebETE/J//839e1DaR3w1a9dvGGPPggw+aU045xcRiMbN06VLzd3/3d+aGG24wImJ27tx5qNyBAwfMq1/9apPNZo2IROLt7du3m0suucR0dnaaRCJhTj75ZPODH/wg8hxtPhhjDj3r/vvvj+gH/fTU1NQhrdFomI985CNm2bJlxvM8s2TJEvNXf/VXplqtRq59PutLc5x1EK4P5CAf+9jHzKJFi4xt24fmiYiYd73rXWr5qakp84Y3vMGkUinT1dVlrr76avPEE0+o+9InnnjCvO51rzs0l9asWWM+9KEPHfp3bV6Uy2WzYcMGk8lkzH333feitJkQjdtvv92sX7/exGIxs3z5cvOlL30JYuxW/bbv++ZDH/qQGRwcNMlk0px77rlm06ZNpqenx7zzne98qZtGfg/R/KsxBmKiZxuLNPOzn/3M/K//9b/M8PCwicViZnh42Lz5zW82W7ZsiZSr1+vmb/7mb8xRRx1l4vG46erqMuvXrzcf+chHzPz8/AvbeEKeRiu+3RhjvvWtb5kzzjjDpNNpk06nzdq1a8273vUus3nz5ki5hx9+2Lz+9a83PT09Jh6Pm5GREXPZZZeZn/3sZ4fKPNP8I+Sl5KAdbty40VxyySUmm82arq4u8+53v9tUKpVD5Zr31M+0nz3IF7/4RbNs2TITj8fNiSeeaO644w71/RXRsYxRfseHEEJeRK644gq56aab1J/eIORwYXR0VNatWyc/+MEP2l0VQn7neeSRR+T444+Xr3/96/IHf/AH7a4OIYQQQsjzIp/PS1dXl3z84x+XD37wg+2uDiGEEEIIeYlhzlVCCCGEEPKCUalUQPv85z8vtm3LWWed1YYaEUIIIYQ8d54pthEROfvss1/ayhBCCCGEkMMC5lwlhBBCCCEvGJ/+9KflwQcflHPOOUdc15Uf//jH8uMf/1iuuuoqWbJkSburRwghhBDyrPiv//ov+epXvyqvetWrJJPJyF133SX/+Z//Ka985Svl9NNPb3f1CCGEEEJIG+DLVUIIIYQQ8oJx2mmnyU9/+lP52Mc+JsViUZYuXSrXX389fzKPEEIIIb+THHPMMeK6rnz605+WhYUFGRgYkD/90z+Vj3/84+2uGiGEEEIIaRPMuUoIIYQQQgghhBBCCCGEEEIIIS3AnKuEEEIIIYQQQgghhBBCCCGEENICfLlKCCGEEEIIIYQQQgghhBBCCCEt0HLO1Rv+7HjQLBOCFvM8LGfjO9x6vQqaHzTwfrEYaEEYfa4J8ZeNLTsAzXZAEtPI4LXig+bFsL6O0n2WjXUJQrxfw8e+C0MLK2jhM/wAy9WUa5W7Sdg0ZpaFpep1HIcgUNqqjL8t2O/1EMuVsEukXMdrP/PNnVjwJWJ6eho038eKa314uPCS1E37YXFFU4spf95hlJK2XhCx0NasJs0oM8NS/s7k+fxieqv9rj1jYGDgOT/3+fKv//ZF0DK9q0FLOuiXc1n0pYUazunSwgxotq34w6YBdpV1JOnGQUs4yrKm+GXVQSrFghDboJULlXLNbRARcV2sn60sTq3akGVp61+0P7W66ffCZ8bj2McxGzUxqFkxbFd5ZhNoGy64pKX6vRic+4oTQPOyHaBNzKHdzs7mQasVMFboGsyB5nb3gmZ5TTbuoM03Crg+73nwSdC8DhyPJasGQUu6OOZhA+O4wMdy3X1J0IaW9YDmKDYfBriWuh76lYVZbO/kgUnQGk0x0Gknr4UypobPvOUnt4O2eHQxaEkP+3Ns737QnGQWtFwatVv/789Aeyn52mqs01AG/fBgAudwwsJ+zCbQD3Vm0GYcW4kvrah/sj28rqHEjIUK1rdSw2sDJX5wlPWkofjrOSVYLdWxLlq0EOSWg+avPxG0hdtvA21SmZcTdZwf3aWpyOedszh3/Sz6H8ngej1RLoPWUUUtXiqBVnZwLBxlb/a1GRz/lw4cS19d37He2vZMQws19OA3KpoW76/tp7X6irKW21rQ83y2CFocpDYWsV/AvQlanogotvfUo4+DNpOfBe3UM04HTTuL0OJ3LYayBOfkS8UfnrMUtIriv7wOLGfnMF6IeThfnLACmgmwXOBE19DQwfghpvTf3CzGXr968BHQCiWsh+PiuFktHn0FAVqWUc49QuXcI7AUPxcq65CyNtnK+ppORevsWLguBz7WI5vDdX5hfgG0Wg3r64oS5yvuR/OhswVcN14qVqwaAW3J4iHQ9k+MgTan+IPRJUvw2r1ToGnE4tFx0/xDtYJ9Xy7jPqJUwj5NJhOKlgItUM5XA+V8sVpH55Dw0NYySZxXw/0Y+/d0dYF2YN84aLPT2O8i0ec6yrlDo45zpWaw74xyJqwZc6OOPr2unE16yjnDbBX9z0vFirPeAprtKWcNyl4sHsN5nlT2MckYaqkkjm86lY58Ngb7z1f2f9p6GirreEO5tlTGmLRYyuP9jPJcC+2g+az813XBuREqbRPlXYMWpdhK0Bc2nTFbyvm5FjlZtqK2eE4lylqtrd+mge3ffNeNSm1eGv74na8GbVcR5+Bx568DrbOnEzRTRztwLZwbDQ9tzUpGxylQOt9Tzit9UwDNUc7StMPyULEfR5nz5fl5vFZZvqwaXptMKed/XRi3zZbQfxfncL2699tPgRaUo8992cVroEz3EqzHwnQR75XF8Y9ncCwcC9cSW+n3MMB+/+ylXwAN7vVbSxBCCCGEEEIIIYQQQgghhBBCCOHLVUIIIYQQQgghhBBCCCGEEEIIaQW+XCWEEEIIIYQQQgghhBBCCCGEkBbgy1VCCCGEEEIIIYQQQgghhBBCCGkBzF77DNSVDM3GKMm7leTLccmAZiv3c11MoGxrr3+bctNaHhaqKUnZ/VB5ppIk2FESlbtKPawQE8SLXwPJFmxXqNSlbmFi+sDBBLv1EOtXV5LuWkri6uY6J5S+c5Xk3raLCYGDhtJ+S0kWrrTfKCm5HefwetfvOGpW8t8pLEtLff4SPFexPSW3uoiShD3U0rUbZSyUZN6WjU+xIJG8VhO0PWPUGrdEq/3+fJ7xYhAa9EG+0wVaw0uDFjiKn/fQDkoVTEJuAkwQ73nRzzWD92rYuN5UFWet5JGXeqMKmq3M+UoZ1znNN3jNFRaReh19pG2jZkJcr2zFH8ZimITd95W51tQtlqWtt7iOdHXhWMeTWaybMs9Cbe7FsU+CItpJO3EzSdCSfdgPmRqu7bNzc6B1D2B/Da0YBm2uirYrzb5PGaNytQBaoMQiHbkO0Pr7c6C5Bm1qfl6JWRx8bqY3BVojwGtrFdSCBtp8PK35TbTdRg3b68ai49jTgW0tF+dRWyiDNjk+DVpSmXuOsi5lcp2gae1vN+csRS3nKXGZhzFdsaL4K6UvjI/3qyt7hGo96jtsJfCv+XjdAk5JKTXQD/nKM21l9xMo5ldQtjklLfRXri2XZkDb8aNbQeswypqotMPCbhffjT44k+2FMtsy6JMem5vAegT4zE6lXTElHPMt7GNH2V+1E1+JSwMf7XvLps2g9fb3oTbQD1pD8WuP3Pcr0KqVqGGduuEsKON6aKS+Ul9tz2aU8VDMVmwlVrWUsDRU42Yt3m4t9vW1clrcrMTIUEq7leIvdm/aAtr2HTtAO+HUU0BzlLVYq5vWBu384KXCsbDeQYg2lMBQTZYvGwCttxvjt9LcJGiTB/aDVg6izw1snI++Ehft2L4LtIU5jEdCpe995UzGUfYNtjKHEkncDyXiqNV9nPOW8oxGHftdApyVS5eiD+/MRsexmMc1Q5t56RTGt9UOLDk7lQetXsZ2DS0eAm1BKddOJifQHksljP2GhtG+43GcL4sWLQYtk8D4ctu27aAVC9FxaijnZpobCUPtjEw5L1Fim+FFOEZzedyrlCs4N4IQ50G1ikGQX8f9s+bnggBt3omhsxlYhHuk5q3EgmLzVWVOKWGMBEp/SogF68r6ahS/YpSzq3bSmcMxT6Rx7lvKuYLr4t7GUzTF1NTz3bAp9jBafKLYmXKkJ4ESs4XKeZ3r4SGPsbTzEhxzdS+g7GN9zZaVOamdt4TK5kIxP3HsaDtCZa0OlT5xlGdq+6ggQF8dKmdrWmynnVi0E78L/cixJ6EfSWaVOFqJhmMdaC8NwfMBzeZjXjQusAz2cxCiv/U8fKYWamvexvexbmFNeRdUwLmRcLDvEouV+aK806krbYunsIaFA2gxtTz2XbazJ/J5bjeuN1NjU6D1Lsfzp55sN2iBhWNtKedKlvJKNHiOfv7w2vkSQgghhBBCCCGEEEIIIYQQQshhCl+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtABfrhJCCCGEEEIIIYQQQgghhBBCSAsoaZx1TIgJYcVgcl6jJHy2AiW5cwMT4jpJJRGvkkLZabqdltw55mGyXt+gFjaUuin3833ULCULvW2UNjiYJNg4CdAqAWoHZrCfSnV8brGo9KeSpDqbiLY3ZmH/5lKYJDgZV5Jq2/hMW0m77DQPmIjgSIg0tOzebcQo46tphzMvdH0tS0nurD3DKKnP1UvRNrS/+ag10P5cZY5LoPgLq5U+aE+q9sPNnmyD/RwofiRQ/EZgVUFLKInke0YG8Lnzs6BlyqXI53oV15sggz4z7OgELRtD49Paattoe/Walpge259IYNJ4zfS0MdfmlaZp9fOVuQHVU+ZezMX5k0wmsR6i1FcwHggF7STU/n5L8yFtxO3IgebFcc3O5rKgpWex3ODiHtCS2TRo+XoR69I8JjbOn6BSweuUbk6ncW7UfcXmDY55tTSPWm0BtNDvw3Lz6AdmDsyB5saw7/qWYl1cZe7WSugLksloHyfiaN9BFdebahltuV5Gmx/s6QYtodhEQ7H52d3joLWbYQ/9WqjEuVVl/pfriv9H05I6PkKCBl5baRpObVmsB9ivRWXZLmETRHmkOMqkCRSHXWxguaqv2KRyrV9X5luItruQwEpnQrTVmPKMKStabiyH69CTCyXQds6VQVuu3N+NYz0SSnwXavH74RXePEN1cCyn9x8ALe6gL+4b7Adt756doG1+6EHQnKb7za9bB2UyXR14XQr9+sIkxk+xFPpXV7k21PZsWkcpy7YWo9SruD6JYhteCtfEUI01UAv9ZvtDe3Q9tNvubuzPyb04rsV5XK+SyWHQ9L0PSm3FwnUwnkTbWLoI4/LRflzfjI/ru+Oi/7I70NaKtWi5iuKXx2cxzqhVCqANDWKc5cTQ94XKGCXTWmyNWjyuzBflfKiixAtzs+hf08p+pVLARdLyMSYp5KM+fGEa+2lkCfqjmZlJrIcSG/Z34Xwse7hW9Xfj+VB7ds/PzJHr1oC2+amnQJuZUfxmHP1BYQFj9ZhiayeccAJovb1RO92zZy+UefjhR0CrKftObY3V7HFuFtul7TvjHrY1kcR25WeV4E7ZA8PhrIgUysq5gIf+J5NB+3PC6JzM2XhdMovt7+rtBK3awHVp547doNUa6GscF9tlK1o7SaVwH2trZ68xXA/cGPqDQDlLC5Qz/3qA4xtUomPSaCh22+rZl1KsOXYSERHFf7se2nKgnjOjv9XOWlwLn2spMbMxynmOrdiQ1ramclqZUIm7tPmdUmKseh3bMD+Pc8NSzv3MYebph05YApqTwfY5CSU+VM5MGiH2g7ZPdCzlnQ4oyrswpZQt2jkNzrNagPGEUfaSsRD9QDKlnE26WJe6he1vKOeuBuJvEcdDm1yYxHVTW9c6muxq79Y8lFGWWxlZ1wVaQolt6w2sW6C8qzPKvtZR/Eor8JurhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gJ8uUoIIYQQQgghhBBCCCGEEEIIIS2gZIV+hoIBJrUVR0nOG2Ji6LijJEN3lSSxSgJp21He/zY91tcSVCtJaL1YCrTB0dWgLeSnQZueKeH9XEycqycnxm6umCRom3ZPgWbiPaA1HCVJdQafW5zHpPZjk3ORz5k4JjcPDsyBtnQA29qTxQzDCVdJ+G1w/GPK8AdKAu12YilJwzWtHWjJy9uGOpZKkvcQC/ohJpBu+GgHW3fsAG1gsB+0sI7Jsvu6o0mvE4rNh23qz8PFng7iSxY0W3Duhw6OW804oDmKlnbRb+RSyjMe+kXkc30ak6MPrVsDmjWFvrBmoc/MONj3hQr6+YSShD5u0IbsngxqdVwPtSWtlsI6uw18rqMkZi+k0ebj8/PRey05EsqUOztAC/0qaIGyliZCHC9LmUN2gOWc4PD6m66OPvQjhTyunYkMxg/ZLhzzzqFu0IpK+OTZaEPJWDQuqCv+0a/iGMVieC8rwPGYO1AALaHZYxHLiYV+OeWg3WbT2CdhAx/SUHyfo8QPoY9zyFbmrudF+8Cx0fck41jfwSXDoC1ZMgra0CK0k5oSyOzbtQ+0cgXtqd3M5tFvVEMcp0BZLysh9q242LcLC/OgBYotVJvmR8MosYISaFQM3qus7Ad8JfbwFBtqKDZeU9pa9ZX6WThXTYBaUplvBSX0nW/gc20L/WnFia6ne+tlKBPMoc/oD3Fcu1yM1bNKiOJhsySu+PXAOrx8fWiw4loIpvmOuWncn+17Cm3tjv/+PmjzY+Og9S6K+p2H7r0byqQ6MB476awzQLvntp+DtnINxkZrjjsatIa271LW/GoJY6NYDO1x+9ZtoNXKaH8nnXY6lmugr485uCaMT0R97PTMDJRZvmYFaBNT+0Gb2r0dtAd/egto517yJtAsxU4cLcBT3OVLRcPBh/f34ZrXGcexnNj0KGiB4l/8GmphoMS+8ag9pxI55f4LoMViaKNxJWaOJ/BcZflytINyJQ+aCPqGAxMToGXS+NyeTowNwzoGfZoZ1JX5l5/E+nV1R9vWkVP2ajaOoedhn0xP4V5qoBf3A4sGekErlyugje0/AFo7WTQ8BFq1ivWuKPu9hXm0vyee2ARaf28faEODA6Bt356PfM7P56FMSrFlo8RAgRLTZ7M4hwoFjN8dxQ8sXrKopXISYFxQWMC+C5Xv68zMYX8ODw6Clkgp+4amxx5zHO5jjz32eNAWL10M2u13/gI039wB2uwMnv8WFrA/qzVc09pJEGA8X6ygD3KVvaIbxzMZLTAyio+UBmph02PrvhIbK+cF2nmYp+wJYwbra1lYzlP2LpbynsEYHEujnF2FitZoYB8nM3jeVKmg/wmU86HmbrGVdyW2sk/TYtuuTvQNC8revlhW4hhL8QPavq+N5Aaxn6sNHMtQcI0NRBkP7RxYOesLBcfNalrdYy7GBLaNNhoYnLdafUNlnqWsLtASgn5U289UBWMAV7FvI9j+RoB9rJ3dNCrYT35dmWthtL2hYt+i3L8yj+tSchDHsOZjfGop9m0p+1UrVHxjCxxeO19CCCGEEEIIIYQQQgghhBBCCDlM4ctVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFuDLVUIIIYQQQgghhBBCCCGEEEIIaQHMEPuMYFJly+1ETUlI7SuJlm0bE9HWfUzsG3MwmWwQRJPfNifD/f9XBO/l4bvkU17+CtAevOde0MbzM6CVfOw+P8AEy7v3TYK2c2wfaPHOYdAWDywDzcSzoNVd7Ccv04f1q0aTGM9MjkOZVGc3aPuKB0CrhjiuA1lMfpzyMHFw0MAEw7aSS7qdaAnXW03CfrjQat20dj2Lp4DieDHQAoPlKkVMBp+fL4E2MT0LWjKLc60ni3PDbkpSbSl/U2JZaMsto/Tx4WsRvw2lLQZ9tW0wUXngK4m/HaWvlQTxVQvHzQuj42v19kOZcgHXjMbOLaD5VhK0MAGSlDxlLVH8XKyBba3vRT8nShJ6S1CrZrAyThXLudhcqQ1if1YOROdL1sK1wOroBS1Q2tpQHLNno52Eyjrv2Nif7mHm6OOukthe0foHcX1eqE3jtR7GBdV59HMxW1mzw+h80fxyvY5GoHmv+ek8aMl0CuuWwDHq7OkCLZtFGy0YvLbsV0ELUthWq45+pTKPcUEspvgQD/sllYm2LW7jnM/1Y/uPOO4I0EQZf5PEZ9qKf0slcT6uP+1YfEabGbdw/Swq8XbGwz6r13DcS2XFForYP8ZH31GtR7WqhX3tK+tsXYkpaspkMMq6FmsoexXFr9U1TXmuUmVpKHGFq8xpt4E+o9a3CLR4z2LQ5vdHY3gzOwFlBrFqUrCxbqPKPPVs9F2SxLllF/F+QYhzvJ1ocZkJFVurYpvv/NltoCnbHUk0iqD5pTxoO7YsRD7P7d0PZWI5nKNrjl4Dmikpz6yiL23UMG6rKfaomLxsevQR0HKZHN6viM8tFzCm92vYx6FWP8UXFxcK0fvPFqDM/u17QHvyV/eB1muh3xp76AG8djnuxZeuwrUjl+sETRQ7eamYK1dA84o4Hrvn86BZBTz3WDyIsWStMAfawgLeL5HujHz24wtQxgkxvsll0T+6SlyQzuC+03PRvovzGKOkUjjXkl4HaFMTGPMZJQ4aWYL9VFrAsTD1DGidSqwVa2ravBIrjY3jOCSTeC8/wAm+fRe2K5fCtb/cwPGpKe1vJzt2bgNt/ziui3UlBg2VPVCglCvk0eeM7dur3C9qf46DtuwpZ2T1OvpH7TgnVNZY18V5UKuhzQ/04R5Q2+/u3LEDyyl1qVbRNrr7lHNIvFSKFZwbJ514auTzKaecBmWGhzEmSiTRbhcvXQ7aO96xFrTxcTyb/fzffw60cgXnXzsplfOg1QNcT8VX9lMNtD/Xw0XLtpWzXF8JuN2mMzdHMRblq12hduYa4jP9GtqZZvO2hVqoxOTqeZ0S42slEwn0r709naAdmMT5bCud4DY9Q49ZlZopWjKFY7hQUs5kYljOs5W9W3B4fR+vHqJPcz3cw4SBsj4FmhfCTgyVeMRxlf5q0tSj9xbfH8RiaFMpG2Ntv4TtN8p8NAbnUENZx20l1o4lcA65tuIbLLSNurIeaH3QrAQN5f2gMlcmNmMcG2bw/okuHP9UAmO7ujbWz/E7qIfXTCGEEEIIIYQQQgghhBBCCCGEkMMUvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFsDM6s9ATUmmO1/GpOGBj0mbuzKYnDbnKEmVlUS3oY8JZq2mYkZJ6G47+N64XJ4F7ec/+B5oE3lsw0QR77d7DO+3e/8e0JxEBrTAwf5M5zC5vJfCa91EErS4kkw4YadBm65Hk8YPLV4KZaoVTBK8Y+cB0GbnMZmyY2F9R/uyoHmBklRcTTDdPmwbE02bUE19/oKi5lHHqQFoibFtNas2Eihp08MQx8hR5lW93gBtamYBtAUl+Xalhn6gVFYSv8fR15Qq6BsyKewov0nC9NzPkHz8eaCNxe8CgZL4PVTmqtH+LifEvq8bnNOBi/frKKANmb6ByOdkP/oq36CdSQyXNdM7CFrFw/q6B2bwfg4meS8pPtgM9IDmhdhP1RD7OJ1FX10vlEGrOWhXbhIt2mmaa25PP5SxPByHwMRByyqm7Ahe61uY5N5SEt+LYH+2k8L8PGiW4vv27tkNWtrD/iorvi9oJECLK31TzOcjn+0Ujm3o41zR/HwsjvOgZ2knaOlORcviOi422nLQwDnUKKL/tpRFrTCJ8dP81DRoR520FrTewW6sX1NV4h72eWcO51m6G2OxSoB93FBsvivTidoStIlCsQhau9lv4xiXDNqbmcF1u7qAvqlSRb+mzXTHKD4xiA5ezVJ8k7LkGAttPFTur9mfrwRVgbZsa2u5orlaXZQdluPjmph2cSwSRx8P2nYLbXqqGrXVboO2W5jHedWbQf+ztAPnR0bpT2Njw6o13DdYjcMrpr//0SdBG98zBlpjfB9oczu2gpbr7wKtpwfH0snguE0sROdVuYA+IlRs9Lab/hu0ytgUaHMHJkF7/IHHQPOVfbclOG7je7aDlvLQDrIZ9KelKt5vbMsWfK6PPsTLKbF/MWprpek5KOMqcy8XVEDr78b9acNH//PET24GbWLfOGhnveq1oCUxXHzJ2DOJfTMzXQAtqfilvhTabV8P9o3m6f06lquF0VirOod+KZnBzupJ4GAWFGedUdbjA/txbuRyHaDV67ifDJU+GegfAC0Wx/anU6hVyng/J47xgsRxHa6baFzlenj/iQmMqWoNXL9D7WxDmcsTyhlPzMP1oNVzhpeKuTm0+aFh3AMdGMf+mpvLg6a1z1U0x8E+bPZDto3jFovhWlwooK8KlL14uYzlYjH0o0r4Lvk5bP/I4sWgOUpbSzXNrrBttnLus2x0BLQLL7gAtJ6e6FxLK/P2qa3bQPuv//tfoC1evAi0v/7rvwStWMqD9stf3Qva3XfdDVo7KSrxl7GU9VSxUcdGH6TFx47i543iS6ymQ3pLsQsNY7C+2llaqOwPjLLg2wbLha4S7yhLmnbkqhyPSULxm56n+AZbOU91sV88E9VCpU8ayrmap6wHRjlrChu4zrnaFkc59wqUurQTXzBmTlvoIyxbOZsM0S/FlbF0lH4NDPah50TjFu38PFDGI+7hnqtSx/21ZvN+HH1/XZnzMQvntxUoe/261lbtAFCxvzjGbX6g+BDlHULzHDfKdaEyp0rT2NaeAt4/PYQxvlHOZpXhka4YxnutwG+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtABfrhJCCCGEEEIIIYQQQgghhBBCSAvw5SohhBBCCCGEEEIIIYQQQgghhLQAZu99BqYqmOh2ttEJ2u333A7akatSoJ1zVC9oXY6SxFbJIG03JVq2bUwGH5gGaJbyKnnn7p2gzVYw+a9JdYHmZDJYt64CaMlOTLBcr2Iy5bqSsDjXhcmOcxnUJg8cAG1BSVafjUWHPJHEJMR75qZBi2Uxqe/kgd2gZSaw/YM5HP+khabnhzhm7aRUxmTREqKNukrib6OU05KXa1pzMngRkeac0raSjFnDFi1TOWrFGtqjMViPpJI0vtrA5Nb7ZxZAm5xDLVTq1/DxueUCJi6fnEb73je2H7QjVy2PfF4xuhjKOEqidq39YpR+V7pY7Xbldur4tBOlOloS9lCxb+1PdQLFf3tKwvX4tq2gVR+8M/LZP+lk5ZmKrzbob2IF9C1VwaTxmf150Jw4PiNMYxssgwnigwY+N9vTCZo3NgOaFNHmvQFMzC578Vo3F12bqlOPQRknhetXuPpI0KoxbJetrFUxH43HVeaywUvbSqGEdtCwsZI7H8E+XDSyCLRcGtfnrjTapKlhXfLzTWPuK7ZXQ3+bUWKC5ceOgNa7sgc0x8GJaynB0sTuedD2bNoLWne2G7R1644G7f4nd4GWn0abT2cxfrKVWLFWi87JVCfOlUQcbT6dToCWNFjOCvCZvZ19oD3+5EOgPbVxM2jtZtN+7OtGTfHrjRbjchvnf6jYUaw5mBGRWpNT0NZFW+l/28FyjhJb2spa7igLsrZHcDx1YUMtxOdqYZonOH9NN+4vdir+9L4duF9ZmI3G62t7cG+VNXXQlmHoKWkL6+ZUlbbW0HkZg35Us4l28uPb7gFtemIKtNUxXLfX9qKPzcaxE33FP6c99ONd8Wi/7q1gn1Z9NKD5Rx8HTVtTE0X0YakSPiMI0TbEx71PJoV7xVio2EFlDrRiAftkoVgCrVZArWvJEGi9A1G/u2v7NiiTUNo1Mohr07zSJ3EHzxSc/CRoe8wToNXPfQVokkPppcL1MPaoLuBaXivjHtCr4zzIz+exnLIe+8q+QZr2io5iuJ6PtlJVYusDeTxrKNX2gJbJYRxg2zivKhX0X/EkPtdS9s+WYPuLin3P5rHfHRefUVLOh0wYnZP9OTQqawnGI14c71+t49jMKPvzQHBudOZwbpSKWN924iv7rtlp9EuxGK7Z2vgqwyt+qNiuh32T64ja3/Jly6FMVRnvcnkjaLUa+jRtj63dL6bs46amcO3ryuG60dGJtja7gPGj1HHuDgx1gvbyl6OPXHf0MaBVKtH2joyMQpntO3aB5nrYVu3IYu/YGGjHHIN74BNPOgm0Bx/AOL+d1BV/G4uhn9PiWS2O9hXfbKsBsrLHbxZaPK+0lL2Bdr4a2PjMQImrXeW8zvKU/UGonOEqa0SoxLOWchZSKeHaZJTx0c5xgybfrB05GkX0lBizVkI/6FdQy8TQb9Ut7JO6sidpJ0lrGDQ7wLY0DI6HYynllHinrpwTxl30kaZpLMMQ7VGU/VUjwLoFgrbiehh/Z2KDoFVsjDuqPq4b8VgnaL7BGMAJsVzdx2d4CTxHiSmaUd9vRG3NUdblQPFH9SqO18STOK6JDuy73kHcJ5s6vlPozeF60Ar85iohhBBCCCGEEEIIIYQQQgghhLQAX64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AF+uEkIIIYQQQgghhBBCCCGEEEJIC2DW2Gcq2LEMtPIMvpttxDBJ7GwZEyOX65hgNhfDpLuhUZICN2Umd5wUFKnWUZuq4a2mC3j/VOcQaF19S0ErhZj8t1ewXU4CtbqHba0qSbCrRXzGyEAPaGUlAfBkvQKa1ZT0en4WkzWLkni7XMQExk4M+3hiYQ60/fNYj5FeLak4VqWd5CtoMJlUGjTbxQTKgZLMWs3pruQHV3LLi92U6N1Skq2rGEwebVn4gAP7x0Dr7u4GLZmIgVarog2l4lhusE9JIK10QKlcBS2tJVyvYjlHMaJiLTqOvtJ+y0J7NErfaQOm3E4bVlVUH9FGPAf72Rb032GA/Rza2BhX+fudzBz6En/fOGi5Jl9VGD8AZeqJDtCMYBJ168AkaOlhnMv1nJJsXdDOkkX0DbG84r8FE6770/vx2iquB/7CPGjx2RxojQoalkkuj3zO79yLz0xmQMsOjYDmYHeKsXGNqAn2na/Mq3p4eDn6kuK/6soaWFNikcwwrsXJMA5aUEd7sS2cV5mmWGFqdhbKVJV1acXRo6CNHr8ItJpBO7OUpaQwjra3+Z4nQCsqNp9eq6yHgnXO9Q+AFlfqErdxLWlgF0t2UbTvJmvYd9lMFrR0EuMYN8Rnio92GzSwwtu34Fyb2Ib+p91MzGJcFld8vaOsUZaNPidu47WB4hNCZU0wVrScZpOOsli6SkEtfnKURTqh2JUvOO+Ni+2qeVjO8ZS1U4lbrAAd6nQOffGm/bje7diyCTS3FvVfiQDjrFUO2m66rPg9xa/7VfR7XoBj4YgSEyj21E52b9sF2uxCHrSlfTge/Q62paTsu6we7H9H2T/Fm2x3QIm30114r3S2E58ZQ5vy0tiGRYtxTUgk8Fq/geuEo7TfUsa8UcNr1xqcf6ESEJs62lXDVea9ia4n+57agnVbmAatpviVeRv9f06Zt2kX54tVx7bWi0V8SD9KLxUDig3ZCWWNnse1PCzlQSsoZxLplBJzK/bieVHNVuzH1FFLhGgDccV+5msYq/sFvLZaxXhkoA8HqVbDcjOzM6CVSjjmK1eMgpZOoa3NzGB/5jrwzGihGLW1HVP7oEz/APqQirK3qPvK+uXg5LAsZe5pa0n68PLzg/3DoE2M4R5zQdljaZtyo8QslhLvVGq435Ni1G9YOPUkFcO9qK/Uo6OrC2+v+JtKFWM7N4bxjq2c5+ydmgCtocSA8TjG0tkc7odOPOZloK0aPRK0dAz38t3ZaGcN9uMc7cjgnCoU8qCVKujf7rzzTqzbqhWgLVqk7ItdZY/QRjwPYxHXVRY8ZS1uNNAfKCGeGGVfrJ0nmia/HliKn9fiefe5n8O5Dl6bVNYgP1DOZgNsl6tM1LqvvI9Q/Ga1in6gXsNr3RDr15WJzoN6A69rlLV6YJ/UK8q5Ug3b2tuDfmWqoL0bQKmdxC08CPBDbLOxUNNs3ih2GlN8v+coe8d6NPYIlM7KKL6qUcGxTPrKAUcS27BQwfctC/O410gl0bcGDl5rZ9AeUx7aVTpAewmUOantGRyjvHZsiu8sZS77ytyr11FbOKDEe3NKzLIaY6xSA8sN9eKeqRX4zVVCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpgZZzrq455mTQ9t23GbRMB/4m/skvw2tTzm7Q6iXMPaHls7S86G8lBwZ//znbvwS0Rx7bBlqmE3MELBo5CjRj429ge0re1LCG+TjqSg4RrV2Okp/uyUcfAy0Xx2tTaczZkE5hzp3xA9F8Cr6Sy8TxsK3dOfx96nwef1d+bha1nQcwr8XwAOa1dWNKUtw24ir5IwIl12lDyb0hFv6eu6YFSg4DW8uT2qQZJZeZhpJeQZR0afB78b9+ptIGJZdsZxZtr6El6VDyeaaUHHhazlXLQZu0lBwD8aTiL5oa7Ct52oyWS6DFvhNlLJS0Knoe1sMs6Wpcyd1llHGTUJmrIbbQVrSih/1fPPFY0HLu+sjncgFzpjSUhIBWXFnW6mjLXhLbWgrQp9tKTpFGoORh0HLwxJScgFg7qSg5P8pFJZ+lUueq8ox4Jur7u7NKjgRlDSoq80c87OOkkmtSy2WsDL80DjObT2XQfxWnMV/b4KLFoI2uWA5aVxL7es/2naCN78AYqKcv6g9jSq7S+hDmz1iyFtdTW5lndlXJm+fjIG1/EPOGlmYwt9OaY7H9R5yC+ZT278H7dSgJVteetAY0O4c2mVTiNi8VvV+1nocyE7OY88QSzJ/iKI4+UOZ3oYC5raYm0XZCJc5qN4ES/ltKHmBXybtmKWuelptVa7Wnrb9NJW2ljKfczVXiMW3s/LSSw0XJM5RUcq7ElbxkRVHyJyk5mbV1oqIsAAUfxYkptCPLYHydbfLPQxXMldRvMKYKlPguVGKqmjL+alwZYh9rebvaycQezFO4Y/cu0JasHARtpA9j1UQWc6B73ZhLOrEY1+1kMrruBFr+eGUeaH3qKPOgXkM7sD205ZiSczrbibkbXSX3sBrVKlKgxDdaPOwq+Zi0GG9+aizyuSeLe12/hLkDbSXfmhfHsdFyRblK7jOngHm9tz98H2jDy3Fde6lIKbm/lq1aBVq/so/bs/0pvF9CyXen/pm+NuZRTcv/puXsDZX8hq4SvyaU/YsTx2sHevtAqyl5WDUyyllLVyeuJVUl76WWF1HLWRgEOGbNOVGTGTyTqVbRz5cquGY4MdxPa+3SfHqjrORwVfZX7WR2Og+alrNdlPMXLXdjqB2VKocG1Tr2dU9/NFbduXsP3j9UclcreXE7lZgl24lr0K6duLfQcq5qYWlDWWBKZZwbfQO4zp13zgWgHXvMCaCl4mhrfT04Jzub2qbFe0889ihomu9ZPIx5eE8/7VTQ0mlcD1MJ1LQ1op0obkSCFgOw0MZymm3YSjktVoc8qerhl1IPZR5o+VAV9yhdOYyferVcxnlcsxtKBbW1pFJWfLqSm7XRUPbtynsAjYGmvMITExjHWNq4KlpZyZuqXRt3cT0I6nj+ZKunV+3Dt5UYV5mX9SK2uVieAi3bgTF+3FNihQaus7WmMc8m8YwiVsU1u3z7GGjDBaxHZSnaXvUojEn7evEdnO8rsU2Aa1XKwfXFEpxDgaPkPw2wTxzlXZXjKr6m6d2IUd6FaWflWv5ko+WJb2A9asEB0BYN4lj3dI9iXVqA31wlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFuDLVUIIIYQQQgghhBBCCCGEEEIIaQEla6xOqgOT844sXw1aBXPkytJlK0HrbWDS2byShL1h8IaBH02we/JZF+Mzl58I2rKjd4H24MOYDL0rMwja+OQ0aK7BBPFxDxPnipJ7ulgqgZafw0Tb3Wm8n5aiPFCyj/f2aYmYowmAp+fmoYzl4Dv3bAaTGrsOmk+9iu3avmcfaH2dmNh51eIcaO3kK//+ddAspZ89F8cok8Vk6CuXLQXtpGOOBM1V/uTBND0XEsaLiLGVrPFKlns/xCTQXd1K8u04tsEoid9jMUyG3tOFScWNkgzdjeEcirmKW/KwLlUf25FfmENtPmrjhfk8lGkoiepFSYzd09MJ2qqVy0HzYtgGZcjE0sasjaTTOC/9BI5RI9D6KwDJD0MsFsNnJAc6QFsoRZPVT80X8V5a8voyrhkxJUF6PV8GzTdY33gM5/eC4gcSnmK3NmphiP1UKysJ50Ns23xFSSSvXJpyo+3ILl4CZRxsqoiN7bK0v8FSJEtbmRSjD5U+bifJ7ixoMWVdtBX/lUngmpXMpUBbfsQa0A7sOQDa/omZyOehDPq945U1Y8ngMGgmxEHybZwbW5/cBtrUninQBpZjPHHEKetAy/Zg+yuVKmi5LK4b8YFu0GxP8T+C82BiW7TOS1YPYD189FuuYvNiK88M0VdPT42DNjeNsWLSxj5pN46Nfk3rClePOPFaJTbQ4iVb8R1us2jwXrYSy3iKbaS7cE5Ws9jWQJmnZgbtI6ihvwoCtL9SgHMrdDOg1RIYS+cbim9JdoE2Oop1TvpRX+UGODbzdey7sITrn6vYuK/MtcDC+loGB9aottM+9uzZA1qtiv3w8F7csyxddipoxx13CmixviHQqg20jYWmhduv4kIeBmh7vo/3cpU4yChxvhaPzc8vgDY+/hRoDaUNdSX46OjoBC2dRpufmcI1plFDewlctMmeZFTLFwpQxqrjvbIxbEMizINm15W1M6Hs7RU/cO+dd4J25hsux2tfIo5YjOtgMo1+ZFCJEW3lPKMyvx+00gKueaLsbZymM4NGow5lghj2fc8inFOhj7HRvhmM20LFB5VL6OeNEpcuXrwYNF/Zd2Yy2J+1Gu5Xtm3FMy5X2e8WFd/sOdF2aPG2pfllUEQadWyD62H702nl6jo+o1rG+K6dlMtK/2kxi2ajynmGr2xZLGU/EBrs1+bzEUvwZp2dnaD9wZuvAO3EE08CLZHAeXCn4oNuvOkm0MbHcJ0bWoT+IpPFPdL649eDduaZZ4DW34v3yynrQTyOsVxPb/RcampqAsps2YJrVeCjX5mamgTt8ccfA23ZstGWrtXO4NqJFpNqWIL9rG3otbmh2bxRYvVW+kbze6ESn6QUW6krbV3Io+8f6MWzpriFz3BjylxW4idtj+MofVL1tb7D9npKHzTHd5UynqnHlLEJGxiLBXUl3lHW9HoV54t2JmV52NZ2EorywknbwyrvKnLK/spWDsVqIa7jRonLu1PRWMEXvK6yBd/xpB7CPUnaxT3sQHoRaKU62rfTjdf6dXyusjWXYg39XNzHdVOUWEk5YhXHw/GxlDne/H7DiSk+Rdn/a5qlnJvu24R7jaE1I6CtXHsyaDNTir/EV4QAv7lKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gJKClodJ54BbXxiE2jHrceE6+mOFN6vMAZaoGSNd2OYfHnH3oXI5zO6lkEZaUouLCKSTWOS+4SL7UrGsL4JJcm9hJjod9HwEGgbt28HLRbDJPQLhQXQli1eDdrqtUeCNjs7B1om1wna+IFowmLLxuS/nV3doM0v4P0dB9/NJ1OYJLoSx37fureA18YOr3f9lXIVtHoFNS0peQFzq0tKKRccsRa0qsHk4nZTwud4DBNZaznkA0XUkkB3dPfhM5VyYuMY1ZUE1U5MSQJt4bV4pUioZCTftXsHaGOTmHx7dmYGtEqlEvkc1DBRfb2iJHSvod0uXjIA2tIl6GvSMc21KmMhSh+3EddT5nQW/WGxjMnaXRevDWxss2vhqNuKzYcS1SwHx81V/BcqIo06jmXSw7XFtXHcPBfv6CnPDXzFrqo10HzF6r0k2kEYoBZTxscLFc2P1q9ulMTvSj0SgeZEsF2a2YaKqHl06zCz+YSLduCFWEe/get9GGAfWjZem0zjer/iqDWgPXDHfZHPm8b2QZljzlgHWs3DcfPmsW49ButRkE7Q1q1eBVrvqkF8RhrjolK5BFrfCD4j1oF1qTRAku4kzrXtjxwAbe+eicjnM9YeDWVCG9fvUDF5Y2Nc2AhwUQ8b6FfCQLETC7V2E1emoavEC44211V/oqCIRotJmiSjrBuB4tl9F+2v6GLsMVnAcU+4adDKnhL7d2VByy0dBm1k2QhoQ0swVne6e/C5d90NWm0a6zyxdy9o+zY+GPl8YKATyix4OdDciWnQOhdw7gYGbVeLIW3FZwaWMrnaSK2O67FpoNMZHkXfYS/GWL0U7wDNV5xYqYj96gfRcl3deK+6Ut9GHddjX4ktQ6NE10rc8pObbwHtrjvuBC2r7CerNYzbTj/tdNCOPBLnwT133wdaqYZ9p/mas06Kjs/QUozB3RT6hlQS47s5g890G9h3MSXmrfrY/lklXGonRyzDvtk+huvnxDTu8RMZ3M8nlDjcKP2wsIB7fKspvjaKPXaPYn1P3PAKvH+Afn7Ttl1YNyVGS6aUMykHbWN4GP18VTkD2LMHnxtTzjPWr18PmufiXr5UxljjwP5oLOgX0R4XlIOHQFmEHaXfRXAexFz0/QOD6KcGB3GNbCehcjZnKU3u7kb79jLYvv37ca1Ujr8kEcOYIpGI2trKFcuhzHnnvRy0U045BbT+fjx/6OrENpx22mmgXXbZZaDdccftoM3P50HTdtUD/YtAy2Ywbh4YwrOlbmUt8RUfkkhGfXi9XoEymqadLGntuv/+X4KWTKBf2bt3t/KIwyumN6LtT7WoHMvZ6qkJGritGL2tnOuZpjNBLebX0M6Ue7rQvqfz6OcWFvDsb2ZiP2gNS4mflDMALX7SzptsC9cNS3m14jgYj8TiaGvzC9H3AGGAftk22IZaFdelno5O0FwH21BYwLVEPdvQzoTbiY/24ivjFlaxv1Jx3BMFIfZNvaydHeIa4YZRP18JMJ6SCeznRBXn47yHvjAeom91tfMMC+urrX2h0neeoI1aiv15Lp7dhMo+saML6xyL49wwTWendhzrZrV4fu4r55W1EmqrhnB99Yv4LvGJPXnQLjoax7+Zw+ttFiGEEEIIIYQQQgghhBBCCCGEHKbw5SohhBBCCCGEEEIIIYQQQgghhLQAX64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AGaWfQa8BCb/rVYx6W6tpiS/jaVAS6XxfukElos7mIg240aT+H71y/8GZV77xndjPUoHQIspiXNtG5+5bDkmb5+cHQetWiyBNtjfC9rsAiYdrtWxP5evXAnaipWrQZt/+CHQSgVMnrxQij7XV5JWVyqYJLmzExP4BqYAWkcnJgb369ifWiLmffsnQWsnl73+DaDVyhXQ0skkaFry5WRMSXyO3S8LTQnNRURCPzqvtITSbhI142Im60oD7cyEWDfbVhJeK4nfXeUZnoeJpi1bSUitJEhvGCxXDdGvpHOYLLursxO0oB69NuHgeOVn8qDtG9sN2splOB8dG/suUNrgKG1VirWVWAzHMpbA9oUGE58nPbQ/38K5X1hA+wscfG6iozvyeSCdhTKiJK/X5p6lJD53lL8tcizUYm7LyyRgFP/qC2qBo8wNpW22osVEyVbf1I6asqYpTRU3VOommKjeUmzZUnyIg8XEcQ6vv+kacDDu2FnG9SkI0Ac1aoot+9hfdhzHaPHqUdD274r6nP3TOB7xYfRfMz6uGf3zWI9sgOt4VxL96MpzXg5a93A3aPOVedCK1ixotQDXzdg41i8sYXuLSYyVPAv7c9XxayOfE73oL2Zm5kArN/BeGcUParFoQpl6tjI3ikWMldpNwsL+dzU/aeG81pYt0+K8VlyMhFb0jur9lThjoY5t8Bt4dXrVMaCtPfeVoPUsWgyanUmDFu/A/Qtah4gfoG+ZaaBvWX7yqaCduRRjjSfv+yVo//SreyOf7y5j3JLN4bw/e9mRoJk9O0ALpsdQs7CPbSWY0eKgtqL45kQK/emxJ64HrSOn7IHLim/Kot+xbJxXQZPtjo3hfjIIFB+eQX+txS2OMl8mJiZAu+OOO0A75aSTQVu2fAVoM7Po6wcHB0DrH8A98Bnnng2a6+H+IlAchtvkawK/D8ssxX1yaCv7Fx/XpkCJ/SsT2vhgTDA1i2tiO3FiMdDcFMbq27ZtBm1p/xBqvV2gJYbQRwYeliuVon2T7UD/sOxotL2eJUeA5pTwrGW5Eh8rW1EZ7MF2GWVvGyrXNpRzL185M7GUGGJkZBS0eAz3UkZZAeu1WuRzYR5jil27t4E2tnsraMH8DGjVAt5v9wEsNzOJ50qDi7E/20k6jT4yHcfBTGfQV/sWzpd4HLWBQfRpR687GrREPBoDpNMYE6xZjWtxTlmzO5XzjXgC7afRQL80MjIC2mWXvRG0METby8/h/mJ8HM9TGw1cr4xR4kxP2e8n0Ie4bpOfDzHK0vaTWihaU+boPXfi2nf/ffeC1teH60tDOa9tJ66NfWo5SuyuODVlyyJGOaewlHjOaDGesZrK4IA0fGUjoDA7h37JKHGco5yDFEsYn8WUvbgJtDMj1HJJ7b0F+oYwRNuwBNdcW4kfFmanI5/9Rg3K1BXb09owr6yRGcVfOMp+WospD7dwPgzQvhsBzvOYcuabMDiWpTr2tSjnA2UlZqwHe6L3j+N4T0+hLS9vYD8Hin98dBavtYrYfi+JY24rc8OzsX7iYvs9R4m/bdRqynuerr4e0NLdOBb1ctN8Vt49hIomio9qKPXIdi4DracH9we/vA/jnbrXqTz3t3N4nXISQgghhBBCCCGEEEIIIYQQQshhCl+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtABfrhJCCCGEEEIIIYQQQgghhBBCSAtgNtxnwHI80MpFTJxbLWOiX8/DBMqFGUxILUrSYU/mQRvqjCbn3bppK5QZ37cN718eB2n3vl2gHT94MmiLRgZBG54cAK20bTdo3fFO0LKdvaBt37ETtKHhRaDlFzC5fCPAxL4TU5icN2xKNK4lPC8rid8tG8dLyYEu6QwmpZewG6SYpSSEnjmg3LF9hA3sU0f5ewRMFS2SiWE/JJVE4pUqjmW5gX29a8euyOdYDOfK0mUjoO3cizb/g5t/BlrDxvmdiGOi9pTShnQSE2N35DBZeGdHFrTjjz8GtL7eLtBWLMZ5YLeYhL1ejSbpdm20+Uo/2ujwENZjeNEQaEGA41UuY6L6dBLHTKluW3GVee5YddASynqQn5wFbba4H7Sp/ftA68pi4vN1Rx4d+ewlsP9qYkBrBJjQ3A6xnDaXbRu9mq0kUrcsLGcMPiOw0IfYRvGcSv00D2sr81TUukSf6yr31+aPdn/PwTnvac5faYLt4DMCpY/bSXGuAFqpWARNm6vzc+i/jeIP+pdg/GArfnPdy46NfD66ugLKOA76lso0xkkDMRy3VKD0/Ry29cAOjJ8cB31wzk5huQDbVVPW0thcDTUX7zc9PgfaygyuJTWJtrdawDjGddG+F0oYJ9UM9vFgJ9ZNixHcGK4vw4N9oLWbuOKvjDI3LU0ziu9U/Km2wfAd5blNkys02K9lZezSy9eA1nsMxhTx0eWgTbodoD2+BdemyYlJ0CpzON8KxTxos3Nl0ObKqJ106omgnfa+s0HLnIl9/ODLXhb5/K1f/BjKTC/gOjygrLknL10FWnke/YPdQM0VXHd9xcbaSaOOPqd3YDForrJnrdYxDspmMqBpcYC2Rk9PTEQ/T05AGbFx/Vy5aiVoblrxwza24aH7HwGtXMY+GRjCPrnz7vtAe/SRx0B71YUX4jMa2Cf7lHmlxRCaBQVN+92G0udxZf3rSeN4ZVLopbrS6Bv21XD8Y1XUXBf3Te0kUNayrizaS6ML92wdylpWqGCbGzFc81cfewqWa0T3/dkMjlHXEO5ja0rfl/IYF4Q1PFeIK2cSjTKeXVlxbIM6rxJYbnR0FDRb6btkCu9XVtYD30df2pHrjHyOeXivbAfad3cH7pv2PPEIaPkG9vHQIO5355Qzvu17Fd/VRgYG8WxuyQCeubkexhQTs7gf6D62H7STT8Y1O1T2Wb+871eRz2vXHgVlurqwbr29+EzHwfW/0cD5rdmPti45ylmICTH28pR+yuUwBteem0ygP5zP41nBoDJmItG6jI9jfFZX7NZzlXYZ3JdpbZ04gLZcKaG/KCvzoJ3YynmBq5xdqF+psrEfmmNyERFLsFwQoF01nzNrOwEtTAqV8ZiZxVg7p/jgRBz9vOOgbcSVONX1lXb56AcScWU9MHi/eIgxVb2O+9HiPM4Dz4p2jKPsexJK7K4cG0upgnF6dQG1VBrnckcO15KaEgO3k1qgvAtR6ug4naDNzaNd+YLj5nh4vyBUzhiT0fliW2jzk4qv7tXW3QBtJT8/BlqmOgxaWtm7eMq5XqWuvINpoGbH0LBCQc3zMH50EthPluJXHLdpn9OF9hhUlfdSDby/NjZLVuM52sQ0js9DG9H3H3cSrs2tcJgd7RNCCCGEEEIIIYQQQgghhBBCyOEJX64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AF+uEkIIIYQQQgghhBBCCCGEEEJIC2BG12dCSdTuGEwCPdSLiZZTCUyw+/PHtoPWpSSVXtWtJHOORxPWxlxMdDs1uQu0sDYH2tIVy0BzlPqmcl2g9Q4sBm1mFpNFzy+UQQuU5NP9fZjA3lWSE1frmLC3riT2rVQxObPf9ODmzyIi1RomWPZ9fA/f04v1tSwcr5iF4xO3lETEBpOFt5Pvfv8noIVKQmpbsL8yMWxLNocJn0dXoQ319WAi8Z6hpZHP3UrfJ9KY5D2/aTdoj2/aC1pFyS7fnGNaRMQVLJdTnrty6QhoLzv5BNB6lETqaQfdkrGwLnVlHvgB2nx5Ph/53AhwDJMpbENnZxq0iQOY8Hp6egbvl8bxHxjEMUulcH735rBPXiosCzvaddAQQhv9QaFQAG1q6gBo+TlMzL7lsV+B9tSj90Y+r1x5JJQZXXkEaF29A6CJje3SEp+LwXYppieOrUwOpaSrTCKtj8MQ175QWySUZ0AyeBGYpUaZ35qmYQKsm6/dT7nWUvy8tn61E0uZg0OL0YaqNfQtgbLu1pV1d+7AFGj9o0tA6+rpjnxOz6IvrO0dB21RDNeWhl3Bulno+4aHlWsbOOaNvZOgTTVw1EPFX2TTuKalkx2gubEYaLaNWi6O83R6Zj7yub5rHsqYbvTzKeWZTlL5u0MPY5uaEhePrl0O2vKluM63G1dwnOran1sqrs4KFD+k+E7NS+YVn+A1+bWGhePUsfZo0BojK0D71RSOe37XfaCFsSRoT+zYAdqe7VtBSxn0zX3duG7vn8Y9R02Jkc/asAG0UgnjymS6F6997Rsin+/Z+CSU2bUH91tP7N0DWiyJvsBK4DzNKr6wSxlXbZ1oJ9o629/XB1qjhn7SzWA8GFN8h/YMbQ01Tf4k1YN7zJSH/t9TNGPhGmYp8zE/h3PDdfHawgLuY3fs2Ana/AzGvpbS/nhc8evKOjGtzJdSqQRatRbdU8aSOJf7lf7s7Udb7u3EtalWwT7ePI/23ecrc7RHiT/biG2U/animDuXLALNUdbenz76GGh5dAfyqpFVoA0tie4LO3I4bjXFt+zbshG0iXHFfylnN10JXEvG96Ofd1K4Z1t2xFFYTjmTibv4DMtV1khPidUVH1ko4l6q2a+4yj5ZLOzP5SvXYrmycibjYfvXLhoGbaGK9vTtH+JZSTvJZnAtXr16NWh15Twn2YF9s2Yt7j37+/Gs86mntoDWaIqRjeKXfSWOtiylnK+c11Wxvo7iW21lz27bGIt4LmrxOD43k0F/ON901iIiUm/gfJ6ZwP1Qdw/65mo1Wpd77rkbyhSKuKYFSj/Nz8yClkqgf8suGQStsxPXkq1bMKZqJ7ZiL9quXLMDUc4ktDMTrZztoh+ymg/slEe6ynXaebRY2IZSA31QTLNlpU/SCWXNKeVBG9uPZ6elSSwXz6BteJYyT+voaxzlYDPRdBbZo7wXkDjGLPsnMRZzlLksIdYjUN7lZBK4HjSUfm8nbgz7zwrRrmwPyyWVswDtrMFXzvdtbe11ouWMgzawoMSLMw0sl9LOUlOodXV2gmZ5aN9KSC6OjW3oiOP+shZi/K1s/8VztHNCfPcVKudjcS+6t5oLsJ/CCl7n+GjLKWXtH12F58RPbFoA7cAkriWO1fpr0qfDb64SQgghhBBCCCGEEEIIIYQQQkgL8OUqIYQQQgghhBBCCCGEEEIIIYS0AF+uEkIIIYQQQgghhBBCCCGEEEJIC/DlKiGEEEIIIYQQQgghhBBCCCGEtEDLmVo9F9/DdmQwcW5nFjUrxMTnCyYN2vQcZsntzWIV07Fo0uHAzkOZXeO7QBvowoTpIysxUX0Vc+TKrx7cBNrY/jnQslpyaw8TJz+5bQ8+RHnXHSparY79WSxVQOvs7gbNb0qgvX9iEsqks9hProNJxVMpTHgdi8VBkwYm2g6UBOID/ZiIuJ3c//AToCU9THhdr2FiZC+G43bKqSeBtntsL2gz+7Eu6446KvI5lkSbKtcwCbSXwPE44YRjQKsqyaJjHs69VcuXgXbUEWtAG+7tBC2XUhJtV7HOew9MgTY5h3Nt/zSWKxUx+XY+n498rjewrV4M2xqLYx8HPs6DRgPnY6oTbXmdHAVaRweWWz7YB1o7cVwHtEQC+2btmrWgrTxiEWjlAhr4kw89DNrDD9wX+XznHbuhzKaNOEdXH3EcaKvWYELzzq5O0GKKHTgOtl9EyeguStZ4tZxiQyEmtQ+VZO0aoZJdPmjy86HyTK1mrWIZvJ+xsJ9sG/vTD/HadpLoxFgkNo1+M5lD/xVzsX2ug9rc+AHQ+ocGQQuc6Kj4C+irGnNl0CaD1nx/LoPzNuGBJKlsDrRqGf1crVwFzQRoy8ViATUXr3VcpTKOEmf0YJy1pCMa74Qh9sm2zftA6xroB63moS0XK3g/Rwmhk3FsQ93gte3GUmzXMujDHGUOmwB9U2jhtaHiEyp15Vo72mfeihVQZlaJS598HP1/fg7jse7eAdB8ZT8QKDbjKLFcuYD2LEkl9u9A2z1i3fGgnXLeBtCqDZxHbhF95zEnvCzy+ZzzXgVl/uv/9++gGWUf8ehW3OdklTnZp/i4IFTmgrS2hr1UJJS4eZkS09oOro6W8qfIobJu245S0MZxW5qMajkLbcqEOKcqBu22JBhHOsqaPzg0BFpi82bQtBilpNh8qMTD9QbOoUJ+HrT8LM7Tp57aptwPbahWja4ddgxttNSH/bQyg/6oZOE8KAd4P9/g3q9az4M2OzsBWjvxbGxzMqns0wXLjc/OgrZ5Es8uqkp/zRWLoPXbiyOfA+VsZGYC9wf7dm0FLVTOEBJdPaCVZrANCzM4RrEGxjyWwfkdjyt9p8QLyrKp7qWSCYwrtf1V81xzLOxzJ5bBqnlKpL8abd4yWLe+IYyNcv24Px1v2mO3G63/jLJnmZ9HH7RqDe7TV67CeKSwgD4tk0E/3PzUZAJjAmNwjAoL6G9TabS95vMNEZGuLoxFXCXecxTfYCmGm0njHskosWI+j3NN2w9plMp4duM22e7u3bugTEGLxZR1s17HvZS2fvcPoH03lGvjcVwP2olloQ2FSj94nrLHUlyEUQIeSzkLsbU9W5PfDG2sh+spZxLKUYt2WhAqZwi+Enc0fNTyypljdW4cNKeB61cq0wtaXVnDAuWM0VbOljQaTrQ/J/NYD9vF+yvDr55duYpvFFvxP8r6bWtBcBtZKCpn74Lz0rJwzBNxnPs1Zf/neeivvRjac/P6EobY96ESy24r49n2rgDHd3MBxyi3D9fnJVkc31oFz8otG/sp6ShxR4hjXjW49llaSKmcBTkNZc9Qi55plRt4XpZUjlfryjuPdcevxrrFcN/z0KO47zE+nklpcXErHF4zhRBCCCGEEEIIIYQQQgghhBBCDlP4cpUQQgghhBBCCCGEEEIIIYQQQlqAL1cJIYQQQgghhBBCCCGEEEIIIaQF+HKVEEIIIYQQQgghhBBCCCGEEEJaoLUMyyLiKImMB/sHlRtiubCKyXmHFi8D7YHxXaDlLSWRuhNNfN7Ri0lzO3KYZNtLYLL50ZVHgpbp6AHthq/8B2hlpV0LFUzoXq5gonZP6fnBLqxzdXY3aCUlEXNHDvvpqc1bQZuYiCY2Xihg0urOTqxcLp0BzTGYnNmrY1udMiYL70tjIuKOhJKRu41M7cO+7+7qAm3xYkwqfeQxq0Dz4ti+Jx/5FWgDSsLxjBUd88np/VAmnesArSeH97rogrNA0xKVd3Tg/Xp7cG7Mzs6AtnM32t58HpOPL8wXQCsslEHLl9CuZhcwqbavJAz3vOi8isVxntmO0v4cjldnZydoXf3oV+IpTIIeS6JWrGgJtNtHGGLWcNvGhN7G1sphfzkO9nVnz1LQzjh7ALSVK6NrxF23/wLK7Nw5BlrpYcUvL+RBO/qYY0FbsmQJaK6D/jDw0QcHSt+FoQ+aEUzoLgY1y9I0vNSy0XatpnU4VB5pK9cZpR5au7T6Gm3tN1jhQNHaSamE/savox/xlbz2vjK+QYB946aSoJUX0PclOqLrrJtD33La2RtA++VDD4F29wMPg3b0alyXBrrwGYUZjAs6OnE9WDwwBFqlhNfO5DEuqlYwBhAH+25i5gBoqWwMtJGVayKfrSqOzTLFlnfNToLm5oZBK1Wxvru2bgdt55ZNoA2PngFauwkVfy3YZWJrvkO5tG5h39azOdC6+9eAVq1Fr8334d7iwZ17QIu56Ju7u7W4BeuxL8A5XvdxjDMdeK2dwbiqbynuac5dfxJo513wGrx20QjWpYb96SbQ7qu1qP+KdeBe4Oh1uNYd2PokaDMVnLulrm6839Engtan7HPmHsP4tp1oMUpvbydoA4O9oBXLGL+Khdf6BmOD0KBdZZv2dsndD0CZhjKnKsf1gVaPafszbOtTm7fhM3x8hhNT1nLBcr5yhDA+jf403olrzNT0HGiz+Tw+t4FOyWoKaCwl7i/E0HHNzOH9pxeUcc3h+DuJOGhuHft4qoDxRDtJOugPLBfbotV6XwljlDCDgVB1rgLagQP7QFuxYnlUUPZd9QbOFU85a+juQB8sytwrKWMuddx3dWVxzUknsZ9s5fDGMuiXbUeZVy72nRaHB4Gyv2hqWzKG+0kngfsto+wjUjncx9cbWLc923DNXabsn5cuxTOQdlJQYtBHNipx2TDGeatWrwTNVb6G0qihDcUU26hUonNoXJkX9QD3rA0lFmk00N8UCjhHYzG0R8dRbM9C3+ooAZ/n4bV+HJ/RqOE8VZYw6e7uRNFgwWIxGlNoe4ZSEX2Pr8T5VUVrKOefC0X0hCnlTC6pnOe0k1DZk4eKH7HUQwTUlGETbeeubA+UQni3IMS6abjKGuEo1/plHLeaEpOKjTaaVnx1Rwfu2WtKbL1fORgIfcVhaGc8ypysm6hma2dydZwHjoc2qvoBJS5saHVTRlbbC7aTlI17vXQaz+idOq5Z9QL2YRhTYlwLbage5EGLNcXgyrGh1JTYJlT8fDVAbWoOffWuu7Fui1bhmCeyaBuNKs6hA1ObQevowLNZ7dXh3ALG876Pc8jEcC9Qq0XblijjmhbPYj0aosSxDZyj9z+I73J2bcW9UCxAH+IoMWUr8JurhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gJ8uUoIIYQQQgghhBBCCCGEEEIIIS2gpNzVicUwcWyuaxA0P8Bbxl28dvWypaA98CAmul3wMLl8aEWT3Q4swmTFGzfdC9ppG64E7d577gOtVFoArVGfBm3ywF7QtPfVxQZqrmBS7S4bEwIvSmJd5qe2guY7mMR5oB+1IIgmDq5UqlCmWsGkviUPx9APMRl8ozoGWr+H9xvOYNLhmo/l2snY5idBW8hlQHvtK/8EtAsuOA+0W3/+E9D6O9Hm+1PYN0k3moQ8YWHi7YGOHGhZRUukMLm1ryQvj8WVcgE+98BmHPM9kxOg1Rv4DDeBbc1mu0HrT6RAa9RxDml4sah/cBycj5qWzeLY5HKoOQ4miC+W0JYnJtCHVKuKzZ94LGovEZbtgGZb6NNttwaa5+D4BhbezxK0IdvDJOyrVh8T+Rz6OEb7938LtLnpcdC21uZBmxjD5O0rVq0F7YijjgGtf2AINFdZ5/wGtqvhY2L6QEmabrS+s9HWVEy0jy1p7TqjlVNtQnskjr9YWNC2cb1uJ/VKBbR0Cv18Q1mzwwTaclJZI1LpPtCa12IRkTCI2sHY/AyUWZVCH3Ty0SeA9uBDG0Er1/CZyWQHaImYMubKoI+Po5+Px3F8R0ZHQTMh3s/z8LlLiiXQ9ivP3bYp2t7VRx0PZVZ0HwXa7C+nUJtDv9wQrNv0Qh60jq5e0JavWAFauzFG+dvKEO3ZKNNa2zo0kujrZjsxBu1ehTF92Y8+ZNtMAcoMHHE0aHt2bwEtcNGujFXHZyrxw7p1+IwLLrgAtFXLR0FbtGgxaN39uEcKlT3C9CyuT6LMBb+OvuobX70h8vnO7+CaeEz/MtCqyno618DxP/JI7JMzXn4haO4Ezsm7n3gctHZiK2tZOo3+ukOJm4sV3Is1+2sRUdc8RRLLRG3Sr6Kvd2IY90odn2krvtRYOHHLyt6upswDx0Uf3tOPa1h5AeviKTFEsYw+vFjB/WMuh/uBhId1STadR5TrGI9msniv8QKuf0aJvaw6jkVRiROGlH53WtuWvGR4MSWWUQxyvobjMdlAe8n1Y7xgOTi++Xnc71TKTc8Ica/X1Y1a7IgjQXMKeP/ZafSjFWWP5VpoB709ynPjSdB8dJFiK/txFS1uVhZYX4kNm52ItlcTC326umfwcG/vKecO+3fjWVO1gX4wH+B63U727MOzucIC1vt1S0dASyRwHzc/lwctm8b+mlXKVWtRv/HQow9CmZ/fditob3jdRaC5Lo7v3r3Y1nIZbX5kBNuaTeP6YjloV1rMon0zZ2L/AdCKRfQrZ57xMtDSyplgc9vGxvCsqVzGM8x6gE5Y23dreEr7uxTf0JFFP9hO1FhE8y1KP2g+wnIwxg+UZ1gWamHTGU+gnC+Gyrobi+Fa7yprrD+NezazgDZfr+G65GTQckMX/beraMZGLQyxP2Oecmam7K0c5b2Kb6LtDXzcu1hKf3rKM8MazgPPVfZuynoTU87krIZiY20kEcO2OMq+1nKwD33lvYTlYX/5nnI+5GEs7FnRsawHeJ00sL7TZSyn9XJcOZM6sAXjnf2b8fyz90i0s57kKGgJ5Z1R2cdYWLODlPI2sRHic2MJbEe8I2praRfXgp35PGg5ZW8wmcfY5tE9D4AW1HDdiCXQ5l3lfL8V+M1VQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFuDLVUIIIYQQQgghhBBCCCGEEEIIaQG+XCWEEEIIIYQQQgghhBBCCCGEkBZQUtDqaMnGu3p7QfMtvGXVxiSxiUwOtM5OTBC+Zy8mSD/jpKOi9y9iouhUFhNe7x/bB9q2LVtA8wNMfmxjjnMpLWAy4WzPEGjz85g4uSODSXfXrF4H2v2PPgXaQ0/tBO2Ms18FmhfDZPU7tm2LfM4XsG5a8vpqBZPSjwxkQUumk6B1d2M542ICbb+OSbrbSbWMydCPPu5o0M4971zQejp7QDv9lLNAs21sc9bDJNC5pvnnxNB+3Bj2vVHuHwra9/wcJq3OuViPUHAiLF+Ddtu/eDVos3MLoGU7O0FrBFhnS0lS7imTMlSSxler0cTVxRLasgkxQXexjOX27t+P96/gHGqUMVl2EChJwNPYx+3EtjB5t6NqOEYxJe93qDnOEK81is+p16PJ5RcvGYUyo6Oo3T+BY+T7+MypyTxq05gMftOmx0BbtmwlaCtWrAJtYGARaNksrnNiYWL2ah3tJVB8pBfD9dWYaLlQlD5X3K2xcP7oaHMUDUBLBe+oavtwlLakMhnQcj2o1cIaaLEY2vL0PrTJdG83aAvj0XKJGNrFfRsxJjj92JNAe93rXwfavt27QAua5pmISCKLa7Y2bNkMxntBiPcb34dxXExZr0Ifr3WT2AcDi/tAm5+JrtfTB5R4bx7XoKHBUdD2HdgFmsngPBtZMwLazo07QDuwbxq0dmMpA2pbaLsNxU80HPTrc3GcH49XcDyLT24FLdUUB+R6BqHMQgnX2d3KemyUXU1ibg600hzGd+9//xtAe+Ob3wxavYHtMsoaUy5irFWroc9wlT52FV/8o299B7R7v/HNyOfk9CyUqRSwU4YVux9ehD7plDPPAa2/H/c5sTTu6eKduEdsJ0ceOQpaLIad7xvse0eZGybEfYxn4dxQTEPqiWgckBzAmCJw0PcFil0YUTQlbovH8X5aiJZM4v7i5eedDdqeUfSxGWUPuGcvxlW7du7F+ilrsa/sdeabYsh6uYBlEuiPxmz0F4ork1QSxVVp9P9uTIkT0mgTbSWO/VesYH/tmh4DrWDQ5zoJ9CVeGu1qsohnMMVi9LkWukfpG1wMWncn+pa5PbjOxoI9oMVnJ0Gr1dB/i43tqgfYLuMrc83HuCLwFHsxqPm+Zi84d2Pxpr2ih/4oUPa/QQPvXw+U9mPVxFbOaWxlv5tSnttOpibxPCORwL2262Kj9++fAC3moj/IdXZhOWXMm/fU5SKeK/z0pz8F7agj8Qxl/QnHg7Z7927Qdu3aBZp2NrJ4GNfxlNJPWqwuypnM5s2bQcvP5UE7+eT1oPUmMVa4//77I58fe+xxKKOdq2jzQDtddB1sg2Xj3PPrOA8ySTxfbSeWst676tkAXhsG2F+ei32j7Q+0foXuV8/qlAuVVxJ+HePl0n6MJ7qU80pP85GCY6ntLQILy7lx9PMpJc7wlH6SAO9nedhet2kd8pU10hG0eW38bcW+AyVWrCn7GVtZg/wy9lM7qfhoG416BbQwwHpbjjKnY3iuEBj0fZUGxk+NprN2T9AuqkWs24xyViwJjNniylm+tYD2uOseXPtS/QP4iCzWJW7juY+fz+NzFduo5bF+M7/EeLtfOVAInag953qwHrVpXJerPs6Do4ZwThUFz1zzsxifui7ODdvBZ7QCv7lKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSAny5SgghhBBCCCGEEEIIIYQQQgghLcCXq4QQQgghhBBCCCGEEEIIIYQQ0gKY+fUZCH1MCNzRjQl7SxVM/loOMIGtoyRaXrpkMWhbntwK2nw5mnQ2k14KZZasAEl2b8HE72PjmHD3ZS87CbRyGZPQZ4cXgdY9vAy0PbNPgVapYeLcWLobtFzfEtCOz2I/TU1hEuNdux8FrVSJZsfOz2Ni5v6+ftA6DCYQH8lgAvr+HCYu9yxMulxvoD2llYTc7WT5EceB9qY/egdo5cADbfM2TL4cWlgukcM51DDYD7P5pnkVKgnYA0xQbSkzPBRMAl5YQDtwJjDJ+fjkJGi1GpYLq5gsPJ1Kg7Zj6z7Qdu7ZA5rlYt919/aAVq9h2+bn5yOfZ6anoYwJ0G/ZNs5RS9HSSUx43pnAtiYScdAqSoLzduJY2D5NEx/HVyy0AzHo+40o91MSpDdfm1CSvGezObyTrdxL8S1GqZtlsF2FObT5h6cPgPbko/eD1t3TBdrgIPr0waFR0BIJTMLe0zMEWt/AIGiWE21vaNC+/VDRDI5NECrjpXVxiGu6CfBao92vjaSU+esH2MCublzvbMX3VetV0CbH0M91ofmJ34j64eQQrsWzHo7bPY8+DNqrz30laKaK/mbP9m2gxZO4LtXqddCGB7FP4nFcdPIFjJ8SMfSHVoD9OTGH/jqIo60l01H/UClh3NGo4bp5+8MYY+4q43qY6cQ1qKMHbWfJGpzfvQMDoLUdJS5Xpr80LLS3sAN909BJLwPt8QmMSwsHsG/r8022FUN72b51E163gHPNODhOPR0YW3tduEZ3dKA97z8wD9psAbVKBeuiLZ1dHbhmZdK4tmlr5+Ag7jmOWXdc5HN5bg7K9C9bBVrv6rWg5fqwn5StmhSKOIZdKWxD2IX3ayeLBlKg5WI4SJ6PcaTl43jUGuivjBJr+Eq8NJ8ejnyeWYa251nKHHXQbmM22rzj4LW5XBa0gf4+0Lq7MW7pyKDdBnXFN4TYJ2evOgW0VUuwvU4Z/UUiiXG+n4raVV1wbXKVPUNfFse/28I1MYbbWNnn47UP3XUraBX7MPubdaU+YzMY0+6axj1+NYZ2Gyp+KRTUdu7dBdrm7Zsjn0cWj0KZTCfamaecjWT60c5sZd/l7Edtci/O+QqakBSLyp5Gmct2gP7CzWC/B0rfNRQfklRiUtuNGmUoWA9f2f8WZnFOzU3g+YQp5EHrzWIcmPVioBWVuLKdBMq+I6fELNPKWcDOnbtAO3LtGtCySt/ElT1+0Hy2oKwP4wdw7m3binG5dkb63//9fdAsZd3YsXMHaCevXw/a4mHcY/b04D4k5uF6v1Wpcz6P8cicEqMkk3i/hYVoDO+6uLeoK/MnVPaYRvFRYajMxwDn1UIB452FAu4v2knznl9ExPJwDXQN+iVbWSOMjYugpWkO+gPTdN5glFhE+25XqNTNcXFOJTpxLs/u3wtaRtmLJgTXcUc5KJ1VxjyXRK0jgb4vUO5XUuzKjWNdmuMWR7HbQIlPtXcUMQ/rESjny1Vlbx82lAWxXkKtjbhK+xLKPPAsjCHFUdqnvBJr1JR1Nob9XypG+9+zcWxrPj7Tt5S5p5RzlFAkk8D6zu9WYpvN6Fs7jsAYo7If97Xl7Rg/SA3nfGEH+vTiHlzXvEDpg6a5MbYdz2RmF9C+04OjoI14+MwDHSOgPTKP7XdtnMvau8pWOMx2AYQQQgghhBBCCCGEEEIIIYQQcnjCl6uEEEIIIYQQQgghhBBCCCGEENICfLlKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSApgN9xkozOwHLelhoulaFZPVWiE+Rku43tuNSYe32JiEfXI2mlR5xsEEvh2ZQdDWrsMk2Dt2YxLsRgCS5BfKoK1atQq1ZStA260kCX7yycdBm5nGBMixeAa0rkwWtH1PPgXa/hl8rmVHExE7iRyUGVq8DLQRzBEtS7OYJDlhY/LnWhXHJwwx0XrDx2vbySVveQtoXYOLQXv0iX2g1euYfboeYj8EggniTagkdZfoAFhakvMA72+UckruehGlXMPH+03PTIDm+5gE2sZLpTPXCVq9jonBZ2eUpOkO9tP0dBW0WgPr4lei5QIlebsTQx+VSmDS7riS3NrxsW71qpJ9XNCxJNM4h9qJZXDglPzwYiwsZxlsn2XQrsRSbqhoXiza/5ViAcocOIDr0vh+1Obnca2KKTaVTaMPTidwjFIu2kYQYPvH9qNv2LoL17Rq9WegNQKsX2/vMGhHH30kaKtWLol87uvrhzK5jl7Q4klcD4woNqr4Ml9ZN8XC+VIXZfzbSLID2xwY7HvbxjVrfPdO0OppbF/oojaxB21j8ehA9F4V9HHdi3AsN977CGjpO+4E7fh1GLNUK0XQYimMO3oHMe6olzHG0Hy6FtuFypwfHz8AWlBXFqw6Xus33S8IFX8bx3m7d3ISNLsH58bs9Bw+M58H7YSzTgdtsHcAtLYTKGuUMonLvWhvp1z2h6Al1p8C2m03fhu04o5p0MKm2M9L4jgVlb5uFBdAi6fQTlOJNGi9A4tAc+JYbmIGn1usYIyihF/SlcM9R02Jcxcm0O4zylp0/NnngBZresa+/WNQxutU6mFwDtkNtImwinufULGdvQfQn01UcHzaSaoT17KODuznbDYJ2kweY8RagPFNvYLxZUPpV8uKrie+hfdH7y9ia+NWzoPmOrhedfegH87lcP0bHMT9czqJ9+vtQ7uansH5vWx0BLST1q0BrbwV98VOBz5jvxed41t37oYyQQPnWajEbQvKGit13EdUShh/Dg2ib2wklZi3jUzlcd3aPoFzdSHAfqgrf39vFJtPKjHyvItxxZ2/uivyeWgI49nTO7tASyj3t5SzEbcT69vh4ZhbrnK/GK4bCwvYhlBZIztSuL9o3rOLiJSKeD/XxXnvaJv0pjOzwKBPKS7MgLb3qSdAa8zNgtYdx/ktnd1YDWWhC7XFr41klTOy/j70aWPjeJ7hKbZx1BG4x7KVMers7MTn9kdjv+07fgVl5ubyoG3ctBG0U089GbTJKYxfN23cBNoTTzwJ2t133gHa8pFR0Pr7MH6NxXCNvPueu0BzHLTvBx54ALT1608A7ZxzovFOsYhx139//79Be3wjttWE6LdC9bwDx7VQQN8/tg/jrHbiKAc1JsQ10LLQH3ouzn1f2aYHSh8q3SVO09mKUc7oLSXeUY5QJLRwL54ZwnPYmhID5afwLCinjLlRzvUaPrbVCdHndjjYx3YS14O4tg4pe5VqI9rx2lrQsLCjanVcvytKLBqPK2uV8ox6FdcqT2l/OzEWttlXzm5cG/vetdF/lcrY5kaAPifp4bglmrYRlRLum+xAiUmVcDFQYgzllYkY7fuRAc6rA4+gHeQmsa229g5hCtfDcgGvTZYwZo4pdjqu9IFTjfpXU8HrGkpHZT0c/5TgnnOwge/HGnPKmfNS9CsxD88EWoHfXCWEEEIIIYQQQgghhBBCCCGEkBbgy1VCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAcx8+wzs2LYDtKWrjgAtYWPi3LCOiW7dBCbJTShaNpsBLZPLRT6vXbsGytz6kx+BVp4/AFqqux+0bfswQfySxUtBW7YGE7DHY9ily5fitfnZOdA2btoKWmgw+e++PCaVXlASAFcD7M+FfDQ5c//gEiizewYTMXcv6QBtRkmMLSGOf15JzmxcrFstrOH92sjDjzwA2mOPPwKaJZgY23Ew+7TrKYnElX4QwWubE8S7Mfy7CG3+eB7eK6aMmx1T2mDw2lysC6+N4xxtOJo9oi0rOeMllkqB1iijbZRLmLi67mM5q9E0X7Tk7QFWJFASkpcKeP+UMuf7OrBP3BSOT0xJUt5WLCXhupJI3PhoQ77BZOih8uc7loPJ5U2I1zoS1R596EEoU5ybAq0niwnI9+5Hn57rQJ/mKfMx9HH9ymUwGbrjOaDFXLRlL471c+wSaDP5edB27XoStPn8XtAeeiBqWLEYtmvJkuWgDQ/hWjU0jGvE8ACWS2fQN1hJNADLVtaNNpLM4BgVqui/dm7eBlppbhq0dKoPtAaahhQrOOaOFx2nHbv2QJmF2QJoi45eCdqPfnYXaIUa+syTjz4atFoVY4yU6r/Q983n86DVK1XQkin0kbaH61A8ib4h6eBz62F0zGrNfl9EasoatGT5CtCKLjrmeRv9YNcAjrUo6+tEdQbLtZnAYHtqDbT7xRteCdpJV7wTtPv37AMt1zcEmpfGeWRMdKwadWW9L6Ddi8ExbtSKoG3dgc9csvJI0Ow4rk1VH2PaumJbyQT69VIB6/KTH/03aI89/hBofYptnf/KV4O2Ys26yGd3YBjKFPK43yjXML6pVVGrY/OlvIDtuvuOX4C2dxxtop1kV+K6VYvhfJ3bj359TrG/+eldoKWVtTaVRtvwktHnuort2Tb6OUeJXzEaETGKGleeMTmJMdQdd9wD2ujIAGhuDNulrRPzm+4H7bt3PwHaniIaW1FZT2bmo/a3UMa1NAxwjhrl78lzHZ2gdWdwHVqRRH957AUngeZ2Yx+3k53T+0GbrqIt+zG0F6PE/qJolo3XDgyj/9q/dyLy+c5f3g5lVoxiLDOo+DRLiQHKRVy/ChVc8yWVA6ncwHKNKvoBMdjWeh3H3Cpi3FJW7LSntwe0DmVvIk3d7lgYVBbnMb4b34sxZH8S69uRyuIzFYIA2+Ur+7d20tmFfaqdyUxPo+/LKP0wNYV20N2JNhSLYb82a81nOb/W0C89+CDud89/5StA6+vtBe0xxZY1P6+dQ5YKaEO1Gt5vfB+ep2pm0KH417/73N+Btu6oo0C74IILIp9nZnAczjnnHNAOTOF+f59SX9vDuaydd9jKmhuE6GvaibI9EUuZq2JwjVU8pASKTw8U32fZaM/NVVEuE+UysZQ2GAsvrijnNB2LRkArNJQYt4z23TA4vl0dOL8TSifHQtzbZjy8tqsT56lJY0w1NR89byqX8Bwoqey7a8p4NZRzzeZzNRGRdBzvV60r8aN9eMU2tSparqu0JbCVs8mKEh8q55WxhHKWb9D+Gia6Z20o7z20VdJRzuh9H0vaSuxaC7GtnjKHSmWc8w88hXvigSzGHT3KmlaxsC5TSufFFJufKWO/F5reVfkVXKtqgmcCttJ3pSq2NRHMgtYd4rluNoNnnZp/awV+c5UQQgghhBBCCCGEEEIIIYQQQlqAL1cJIYQQQgghhBBCCCGEEEIIIaQF+HKVEEIIIYQQQgghhBBCCCGEEEJagC9XCSGEEEIIIYQQQgghhBBCCCGkBTAb7jPwyDZMEL503cmghVICzfKVdNlK0vCFQgG0fB4TmPd0Hxf5/KoLMKH5cceuBe2b3/4O1s3CZLUdHV2gLRpeDFom1wma42P7uwexm4eWYVLf+SQmSX7o0UdB21/ERNNGSaDdMdgDWu+KaMJiR0kMriUt32zSoG07gAmbY46SfLyKCb9Likn44XNLHPxiceftPwWtvJAHLealQEumssodMfmyY9A2jPI3D7YX7Rs3jv2ciONYJhJx0GIJrK+bQltJxDC5ddxWknsrf6JhJZTE90q2+kYNk09XK2gvjQaWCy0lPbjyDFeaNC1BdRzb1ZlGrSON45VJYsLvuId18yyc81aASbrbScPH+tTrSr197AfbwkmtJXA3guUUtyHFYnQ9qFawbmtWHwHaCcedCNqDjz0B2n0P3A9avlgGLfDR9vqHhkE744wzQHOV+bdr926sy333grbuiCNBy3XgnJw4cAC1iYnIZ23+DA4MgbZs2ShoQYCjWCrMg2aa55mIeC6uG1XFntpJ3MUx2j+1F7RdTz0F2jEnrQPNcdG/FJQ+zCpjWa1Ex6mnuxvK7NmL9jO0egS0ZevRfrbt2gfa8tGloK0YwftVixjb+AGOef/gItDG92Gd5xYw3ospHsMP0V7mZvHaeCo6jiZEP2N8JWZR1qrS/Axoi5dhP40cuQK0sbk9oBWrOP/aTdnHdocpjH2TI6tBu+WXD4F2YH4OtM4utN+44hOtJjs6oNhLtYb2F4tr8U0GtJQS03sxvNZ2cO7W6zh2vrKn8ZSw4vvf+y5o//Fv/wKaUdZOSwmsnnzscdCuetefRT6vXoM+yRKs3OzMLGiV0gJojRL6+jtvvQW0R++7G7RuF/1DOzEexnSBUcbcx3ofGBsH7Rc/uhk0T9k/uZ4SLzXNg0wO9wydHZ2gdStrwuDgAGipNK69j9z/AGiF+SJoGWVeLXQkQQuKuE72LEI/GZ9D3zCzdSNos24vPqOBcV/SifZntgevc7Q+t3FOWYrmKuuQq8zRuhIvuwZtrJ3szk+BVtVOfpQYXNnGiFjKxYrvc5PYr0NL+iOfd42hn9+yHe2iuwvPNzTfX1POkHZt3Q7axCyeZ4WBdp6Be7tCAfcIDSWusBQbmp9HX3rMsceA1tWJ61Wiad+ecHEf7wieARSLFdDiAcZUiwf7QXMdHOuYEt9ayh64nSST6Ku0vveVcRMLJ8LMDJ5DVhbj/slXbKhcjtrLipUYM576stNBu/vO20B78MEHQUunMd5RmiCOo32XBte5krIHrlbRXoyytGv+tVjE9aVQyONzS/jck0+OnjE//sSTUGbxItxvdPfgGrlvH+6TleZLEOK8tRTnGISK7bQRWzlT95TzMMvgOub72OaqsrezHFzbbOU8zW/qWBNg/zlxvM5WYjEtgqwqC07o4ZxPdqIdBCH6w/lpXDc6lXgsmUA/Z5QYwDFa/IB9UKigzftN+41KWXmnYrANojwzm0HfUKviM12DtpxR9mmNMl7bTuKCcZ9fQ9vQzvDiNsYU1QauEY6NbY7ZaBuWRPs6VM4tbBtt1LLRLmzFVhwbZ0Jdic+MEi8XF/A8w7JwzBd8PHsv1NDWihW0STuL51mZJO5BwoUx0CqxaL/Uc7hGNpRzFW1vsD+PdZsr5UEbXo7tWnYc7qOC8Ll9B5XfXCWEEEIIIYQQQgghhBBCCCGEkBbgy1VCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAcxW/wxsmcdEvNMBJvU1HibEteuYJNiESjJrJTH28FA/aGeedkLkc8LDxLyjI5jk/NWXvAm0m77zQ9CmD2B9988rCb+r20CLCSYLn62gtm23kly9riSN710DWld/CrRQSfttWZh8PExErw0tTNDdUBKZzwd4r4SH1yZcTMRcsjAhdMPD+xklAXQ7GejHBM37K1OgBUEetFxPD2iuMh4LU7OgFRYwIXMjiCbkDpUE3RLiPFCxsR5eEueZ8bD9voUuw3bxbzRSMfQX6RTabdDAuSGK/UkCn2HF0NYSMaxfsikxe3cGk2wvyaAvWzyEydJTCaxarVoAzTboB10H69uZw35qJ8Zg3xvFt2iSlpjdUf58J7SUMcdLJdlkL2eefZ5yGT7AddAGVh93Mmjr1p8EmpIzXmylcr3K/F6+HJOwuwn0kaOrjgFteCn6+WQSbaOjA+ekNmazs9EE9kGA61d/3yBoWSUpveMqc15J8h6EmFy+odhEaCmd3Ebm8wugFefzoGVTOJZWiLYcj2P7urvQceyfroBWqkf7cHTFUijT0dcF2vat20FbO4L2aLvog+sG15JyFdegXArXjYKPfq7eQC2V6wRtOj8JWmVuDp+r2GTKQ/uzrej615XG+VMIiqClSxifdMbjoHUM4Bo5VcN4oOjjeiAGbafd1H203YQS89z18COg/fe/fQO0Y084DrSVxx4LWlzpW78StftyCeek6yp7hhja87r1p4A2unItaMkkXus4+Axf6SdPiV+nJsdBu/kH3wUtodhudy/64kod7XLH1qdA+95N/zfy+X+9/s1QplBAXzOTnwZNApy799z2U9Ae++XdoMUVP5JMZ/AZbcSzMEZ2HNS6uzAerFawDzuUOKBRxHLFAvoEqxbtr+2bcT+pxQCa7Tla7OWhVlhA/xeEWG5yAvenb0meC9ryLPq1Qn4GtJrBZ8SV2CgmOP8sT9nvhtE1NjQY34iiBQ3U/AaOfxjgXrQygr5xqoExj+RxDrWT2QbaXsPCftDiSMdS1lkbNeVSsZRrY8mmMXex7/fs2w3aKSdi/G4p4xsqZyiFfB60J7c9CVqxhP2UVPaTQYiNrSrPjceUdU4pl78b/fDQYB9oR6w5OvI5VHzZ4PAQaKNrcG8xtht9zZxi8/0Z9D/1Bvp57fypnZTLOAc1/51T9lOBj/26e/ce0LJK3xSV5040+dKMEguvW7cOtAcfuA+0pzbh+t/VjfuBlLJuhMok1eZQrYbj22igbSjTWxzFN2jzxVYOBqamMJbesmVr5PNpLzsVysRiuI7MzOM+4vHHN4KmnW24SgzYp6zzmezhdXZjKf3saH5e2bMaZd0NfbxWOwup15X1rqkPQ2X9D2zlvFKx0VAxNO0MIVTOV72EEn96uBePeegbCgXlXCCLcaGTxvlXqOJ88TyMFQoB9kGpyYeEIfavUXywp8wpTzlzXKjgvkJq2NaUMq+MshdqJzVlrx2L4/g2lHOKWoB9qLgvKReVWDDAGDfV9G5FGjinSso+wK+jXdiOcg6rhLi+4r887TuTAdpBuYp2UFjIg1av457BVvYgaxePgtZQ3klMlfBsybejvtRNae8eFoNWMfiubsrCdcQ9CmOxFSuWgNbZge8G6sr+oBX4zVVCCCGEEEIIIYQQQgghhBBCCGkBvlwlhBBCCCGEEEIIIYQQQgghhJAW4MtVQgghhBBCCCGEEEIIIYQQQghpAb5cJYQQQgghhBBCCCGEEEIIIYSQFnBbLbg5j+9hv3fX46AdN9IL2mAMk8SmPHz00OAgar050FYsb0psazAx8YEpTDj8lf/7Q9AefASTnNeqeD81j7PBPjFKkuQgjm0IbEwI7AomSPctTIrcnPxXRCShjaSSRLxaj9bZ2FjGdTEhtBMqidGr2Cm+YDkvxH5ylCTl9QbWpZ2YBiZe7khjku9CFRNSNwJMXL127Tp8xlA3aJPTaLuT09EkzcV8HsqUy1iPIFCS1weY3DvtYgLptceuAG18Ads1pSTBrtSw7yoVTBqP1i0Sj2Hy6bSSQLszjfOgv7MTtMHhqF9ZuWgAr4tjTYolTPI+O4vJsp0Y2nJKSXKfyWJ9e3qwXDtRx0gZc9dgf9UVP+wLJgP3ffQRQYDlwiafYzB3u/iKfVtKVvq6klh9eOkyvGGIPshSNFvxrTv34Lyt1LHSluJzsx1Yl+b2i4jMzTdAc12cG+ncaFRQ6js7j2M9PjGr1APbELfRD8ZQEiuDC1N1Dv1POykr8zwVxz497eXngLb2iOWg7Z3ZDtq+BZwvla3Y/5Vy1G8WGmjffZke0GbCadA2PfkUaGcddSxovRmMTwozaMu5blyrLB/XnPky+gGx0A5sNG9Jp7OgpRLoNyvKmMXjUQMMLbSzclwZ6zJWZPnQItBmXLzf3Dz2u5fE9cuvoP9pN4GgbVVD9C979u0CzbXRJywUcJ2IKU6hS1mjt4xHn9Hw0YbiqQxoqe4+0HKdOD9KpTJo3d1Yrr+/HzQN18H5vOXJh0Gbn8d51JnD+TY3h+UCxf/nsriXeuLhByOfV68+AsoMLkY/pY3Njs2bQduy8QnQ4srk7ctiu9JJ3Eu0E9ugHUzM7gDNquEc7sxgW7q6MH4rKL7OWLie5NLRsczPoW/S/v45CHDuFYsYb9uC5Wxb2SgqtpxfwH668eZ7QetS/jzbdOBeYmQQ+2mugRfnS3OgNQK0tXoj6k8DJThsNHB9FWVOeUo8duTaVaCNHI37oel6HjRf6fd2UnNwjTZKP4iF/aCuWooY+ih6SlzabGmJOPqghQW0ZaPsN3wlLteOafqGhkDrmd+N185ocSm2q0PZx3VY6Bt8xQycAPtkYvcEaD//6S14cT06d4dGRqFIoYA+pGCwj2fDFGgP7cK97Tpli2TVcW2OKebUTkrFImhG8RGOgzZUKqPv2zc+Dlq1rsSDc7h/mmmKpWtK/43t3QWa30Dbs5R915IluLanM52gLSygbTgu+mAvpezZGthW5QhTtPmiLTmhjfPZVdq2Z8+eyOdVy1dCGcvBB/T2YhynuDexlHo4Lt5vYDH6kMFB1NqJEpJLUMNxsxxlHih24DqK5mF/aVPfh/UF76Wd9mpnDaGyJ1GOxcUo64HrYbwcKOfbnjLmWpw1U8SYIsygzac95cy7gvM+X0M/1Qii7TXKXkj7VpxR4qTQx76zjbJWK37QUcppMWU7qdUxVqj7ip+zce8Yd7HNtRr2tVF8RKk6D1rQ1NemjGtsUFMiFAvHzXbQuYYu3s9y0JY1P+o6eG2ptB+0wgyuc75yNpvtwfd8XhzjojnlXUYpwDo3H+Oa4jYoY2ycy6Vh9PM9Z+DZTaIX22BcJaasKOfaMZxDrcBvrhJCCCGEEEIIIYQQQgghhBBCSAvw5SohhBBCCCGEEEIIIYQQQgghhLQAX64SQgghhBBCCCGEEEIIIYQQQkgLtJxztaj8jvOtD20Bbct2zF9z4fojQVsxjHlZdu7YCtpZJ2GeykRT/sVCHX8n+Zs33w/aQxvx96TLPubWEeU32W3lN9S134e3LSXHpZLTJVByANaU3KQN5feuLQt/A7om+BvdWo4Jt+m39R3ld/VTKRzruJY7Ufmx/UDJNxQovwXvK7nkYtlOvGEbmRkbAy1Q8vhUlN+hLzflihAR6VZ+R70vgb8F7yn5SpNOtA8rSt4EY7SMM1rWHKW+Fcwdd9ZJR4F21BFHg7ZnD+avmclj7iTt9+xFmUOu8hv3SSWhRG9CyYuVVnIsNPXBgWkcm83T+PvzVgLnQa4f87Qlc0qeQCU3WncvXptR8lO1kzvuuA20ef8x0NLK7/8HNcxV01ByOzW0vNQB+rRm/9VQ8kdoflTLmVKtYTktp4al5HbyXLSz7k7MOZBR8tw0AmXdUNJWWEpCGE2zlXyylpK/2m5ar10XbdlWrtPupeW6VdJEiGUpa1VKqVsVczu1k+5BzCU6tGo1aMetHgGtqxfnb64b/XwM3au4GRzfmYmoDw9DzGO5Zzf6qs4U1sPrwxz2kxW83xLFZzpKwrBAyS3u15U834LxU0zJixRTbK2iJLcf6lfaMQmSFEvRtuWVtlaV3MOVPD5zqrIPNNOLubqtOvqkeBrXdDv+3PJ2vJgUlb+trBUw9gj70BaWL10Kmq/4K6PEGskk5mZpzrntKHnXO3owv2rX4Cg+UxvjErZr8eLFoGn+tazkYNN888QE5s5zlVzx6awSL2RwDhaVPLELSk63QiGqbXsKc6QOLR0FzbJwbPbs2gWaX8F6dCqxV0LZS4gS+7eTNZlhFJU4INWJ/rQ6jv2QOAr3pxufwn1xZxfaS9xr8ombN0EZX/GHeqyg5PhSNC1ZWUcHrn+xOMZ3M9N50KYsJR/YJMb+cSVv2iWXvB60f/rnG0Dbtw/Xu7DJdyXSWN/VSt7UlJID+KmNT4I2M4u5omZnMb+Th65M3MPM1YfK3slRMt5Zyly1lPOHmI2NjsfQf6UUPx9PRPs/abBMJqHkFgNFpFbFfIJBc/IuEenvw3WjbzfafN3GeEHLKJhS8quKwXI1xW9qwXR/BvMR796xF7RvTvwg8tnrXAJl9k9goFlXYjRHiccsg/25ew+uaUevxLV/eX8naO2kpuSa1Nb2/FwetIKSOz6j7OfrSu7U+QLer9kmjbJnVaom9QY6kl3KWcspp54K2qIlGNvMP4F+Lq34za7OTtCMln9yBv28ltdUiwBsZY++aBjr3HzIWFZyXvYP45q+ZiXu3zozGJeXK2gnvtLvTz6xEbR9e3CP0E4sxQcZ1XOisfnK+NpKLnY1R61y5ttcLlTOwLWzZ1uzFqPkDdVy+wZaPdBX15TXHkbxh6HSnw2l/Qt1pY8b2tkSrgeBEo81mvyFq6zfnpLIuKLkZhVfybeunLVZAc6DtLIHsx3lGW0kk8B1vKjsYQtV9FULPq5tjSqO+dAwrrNVpQ9rJhozFqZwLU7Y2KfJFMZADR9tKpFE/6XlSi6U8bkT09gntRLuJYtVLOcofmC4H89Cxvbjufq+CXyG/toxauPlAsbftoX5iUtl9P2NOu7dlig5uGdn8BkS4Dmpr5zxtAK/uUoIIYQQQgghhBBCCCGEEEIIIS3Al6uEEEIIIYQQQgghhBBCCCGEENICfLlKCCGEEEIIIYQQQgghhBBCCCEtwJerhBBCCCGEEEIIIYQQQgghhBDSAlpmWZWe3j7QZucw0fJ+JUH8PY8+BVrQGFGegslk+wYxybnlRJMC/+qBJ6DMD39+L2i1EJO3i6skbdayyysENUzubELsk1BJYG8MlgsMJjH2lMTvlpJgWBzsO1cp5zQl7s5mMUmyq7TfVpKKi8FyoXhYLlCSRA9i0uFsDrV2MjjcBdq+3ZjwOaj5oImF2s7Nm0Gbj6VB06yvFEZtreTjeISBUg9REtVbaGf1WgG0h+7+CWhnp9Fe1in2UunIYv18nAeWj3Wu1jG5+nyAidknZzBx9+6nMEn5dGUhen8P25/sx8ToXYOdoMVz6EOcJM69VEcOr03hWFtOyy74JSHhYfsajtLmEOsdj2ObQwvLBSH6A9vGMTFNthuGaCuWhbZnDNpZqPgvS5RnKj7YstCPKi5dbMH1wHWwzrUa2rKlrTlYFfF9nM+NBj7DcaL3s21sg6X4gVbXvnoR/YVR6lFVlqq4oySSbyOVMvqbfcUx0OoN9C0jy5aBtnigF7Q1w2tAc2ycG8nYbORzrYaGVitgfRfm0b6PWb0atEQK1+f8JI5Hn5sAbd8U+tuxGbzWeOjnlg8OgJZNJUHTYptKXZlXNvrcYpNN+g3sk4FMP2gbS1tBe2LnDtCWj+CalophfzYqOD57d+8Brd1MBjjX6zX0zSXFBk1Ci3Px2moV+0IJkaVWj46Vm0Qb6ugdAm3JyHLQert6QLOUOCiZQBvfv38cNC2mjydw36Cta5aL9uEosXpHZydofjiFWgXXjnIhGt/s2on2vHL/btCKJRybsb1op/Ualmso+5eyj+UkhnO8nazJoR+qVrFPtTXBrc+BNjywFLQncbsry1aMgpaOR23j3jvugDKBsneyFR9pK8GCFrd4MbRb10U/sHw5tksE9+KPbXwUtIQSQ9VK2MdPbEQ7LZXLynPR1kwY9Rfa/jQTx/YHylpiGnj/h371CGiFaRz///2n7wQtnT68Ynoxir0E2OaUhX4pncbYPx3DOD8Zx3JaLJlo8pupbvTB/297Z9Yjx3Vf8Vt7d/Uy3bOSHA43UaRIUXbkSLAtO4ZhI3lwkAcDec9DkO8QBAGSh3yOfIA8xDECy4hjO1AUxJZkydq5jTgkZ8jZe3p6ra41z9XnCG74wT1Azu9tDqq66t773+4tEv/Awz1mQGx0MEFbKTK0g/EAfbmaYVw+H2JsiBO0FwtN2WRkPx4k6AeZQX922HgvYNx8vFse7+7hNlxTkP2Qw86L2FEB2dP1HvZAOzj+Aq+7fYH84PxgtsfmYUTiTU7yfa+H83AcHMz0LmFY9o2UnHkUJM4vtfFMYvc51icHu7ugXTyH6/HowSZoDjltKsg5TXsBz+a6R6egse1jSuqigtTmL1zCnNOqlmNNldRdS+0WaMN+F7QayX0pqdVzdgZA4mWvg8+YJ/FkDJrvY3xNyBlCZpFzQo+ce5DanVGpTtk8OTBxyPmG7ZIzcIecM5MzHo/EPj/FOJ820a9ysr40Vtfx3voS7kssF3Op5+G7VD20ydNR+QygiHFd84LEEBvfNyaxhu5dxviMQYSaY89oAH8gJhbGZd/HPDbuYNLOyeHU4jKe3eQGYwRxDVOple9Nif0kKZ4hVUOMrQU7X7MwZk5I3jjpYF6aRAPQcrKvG5OzyfNt/A7yl2+8Ctr9LdxjPtp8ANrpEMfhTJ8xE/9Oc3zf4y3MaYeP8ewgXMFclcQYG+su1mJZTgq+GdD/XBVCCCGEEEIIIYQQQgghhBBCiBnQx1UhhBBCCCGEEEIIIYQQQgghhJgBfVwVQgghhBBCCCGEEEIIIYQQQogZ0MdVIYQQQgghhBBCCCGEEEIIIYSYAez8+2UXkmbwHmnGnEbYyHlrH5sOT4Z3QfvO126AVm1hs+jTqNzE9613fgPXjEnD5yTFRrpBgE1tc9KAnTW+ZzgWTqlFGncb0hc6cMi9NlkiollBCFq1WgXNdcv3JgnOU384BC0jncwnKc7TQhsbQp87j1q9QppO97GJ8zy59OIl0HoDtOXhzhG5Gxc9yrBJc4fMoU9sKJ6y54w0fKZGxd6smO26hx+9C9p2H31oxUY7K8gzMhv/LceANGHfK7Bx9eYEbXInxUbToxDnrrFxofT32tXLcE2l1QSN+ZlxcAz1OjbBDpsN/DkSLwsaHOZHTuZ0MDwBLXTYWPD3MvLvd5IUY06cYHxN0yk7sPG+okB7TJIYtDzFtUwzYqMp+pVl4RhyYt/EvE1R4HxOojE+l8QG9oyCxOHCsFiQT12D91nE9pg1svdwYpz3lOTXYQv94PwGavPkeA/jd0rs4PN7T0G7uv8MtDe++Tpoyy2MEZeXL4Lm2OU6a7t7ANds3FoF7WAHfXRz8z3QWu1zoDXJ+vbRRM3Tpzug3X+yDdrqEr7fcoh14UprCbQ2icPbuzjvzRBzTmuxVfp7OMSa6LDXAa0zHIB2ekpqEeIvY2Ine482QasSv503PZcELJLf3AhjWFLHcReWB9poiLm8vojrfuHazdLfzeVFuObGS7dAu3njZdAunlsDjQ01CDGHBT6OochJVCTJrlYlOd9Cu2c58fz6Omgra+irn3/8MWijqGy/e3vP4ZoHn+F9gxE6+SGJZzHJ10P273I93CMa/2z9+90xGUt/jPNw0sV4asjw7n6K8/rowT3QPBftoLVQjnW09sBHGpvVjKRGYTU4u7XRwNw0GmFMvHr1CmjLz9GX97bRhu51cd/0yYOHoNnknKEgtjZdu0RDrPne/d/3QcsyrFEWSM753ve/DdpXvnoHtLXLG6CRbfxc8TLMlVUH41w7JPsYj5yPTHDPFg/Rr2xik1lUtslojLnFbaCvTCLcH9ikph2PcJ+4+xzjYTLB5wbER5OI7TnQiQIH58kqcN5zF+8l2xCzf4jj6E6d1WQ+uZE4eG7Q5gtyfmDbOP4ixZz25BB9ufvO5/guc4TtbXwfx8euGwww9jmkLuoOsUZk8dX1yr4W+Fh3hOSs7toGxpYP3sEzmY9/+1vQXr59G7Tl9gJo3Q7ufY5i9I0L61hTVTycEz/AOR4RHx/H6FcWyX/puOz3e89xD/Kd730XtH//8Y9A6x5h7c/OSNn6sL3thMSQecLyPdXI+XaRs8MLlKwC781yzL0mL8+Xw85QyJz6PibPPEWncnKsE2o++T0XB7FYxXPdToh1zDhHG3VCrBUqLfSN9tIF0JaWcE/T6+zju2zulv5OJiQeGczfSYbvm5F6NyPny5mH40oSrItteu48P7rHmIvqIca5gH0fIXWfs4D23R9jjHTImfdSs7yHPTF4btEl5w/ObNtL49jk/IGeJZJ8n+O6ZUQzNvrVaUxq6w8/AO2NGy+Aln7tRdDubu+B1o/L8eE0wr2vs4BxeeU8njXV13AMdeLfeYC1TTLAOs620CZm4WztfIUQQgghhBBCCCGEEEIIIYQQ4oyij6tCCCGEEEIIIYQQQgghhBBCCDED+rgqhBBCCCGEEEIIIYQQQgghhBAzoI+rQgghhBBCCCGEEEIIIYQQQggxA9jZ+EvISbNxU5Am1Q42hI0NNpjdH2Az8A/uPwftByNsSN0vyo3kn51gY/lKvQ5aOsL3iEhT8jDEZsWuh1PF7rVIQ2DbQs1z8fcKm2jk+7cX4BwPElyfOB2CNt3AnTU8n6TYwHcYYVPjemsZtPYKNiKOU7z33r17oHmswfIcabaxAfnKOWygvLuDDa9JP2rD2iJPSIPwhDSSz0w29Te5aEboneSFkzE2yx4eHoBmV9qgOaTR9nODY/3QoA9tusT+GtjAvXaRrM8F0kh+pdxwPqhhc/OYzEpR4HsELvqywzSHXYf+bTszh+A/CNvbn4H2cBf9t+b7oLlFClpGPQHHzJqrTzdm9wOSb3Js3p6Qhu5EMsbCd3NsfIZlo22w5vLGwnsdsr55jnY1idEP8gyvs8hzbQufYZny+uQkzjP7JpfRFUwMmeN2DbT1V26DtoCXzZXRGO27WcEY8eAxxr4nW3ugDXqYd19/A+dhsY1x89zypdLfteoCXPP05DFo+UWc1EEF36M33AYtrWA90c9x1ccrDdBcdwO0k8EAn4HhkBpb76QL2tLaGmjjwSk+97Ss2S7GqGfHmKvf33wE2sofXQPNJ/6982AHtHqIz/ULFoDmTIXkHgvXxBviejYvYj7ukzWOeyegdTrHeOFU/B8P0Hbv370L2t5TtOd6Fet3z8X39aq4TjaJdmzvw647PcL4kCcYW3wf5/3BgweguWQvcXCA8WaSlHNHv4++8e7//DfeF0eojUf4Hg7afVTg+AuyH2RjmCf9Cb53L0KbH+doL9VqE7SVOs7h1164CNrz3ceg3f+svE5ZNtv+JyE2xfI2i39sD9Ige+WtxxgTRxHaxkKrBdrxwSFoYQtzzLk13Cue9nug7e2izY+j6XkndRs5d8hIvbixgXuGv/6bvwKtUg1AS0lhmWIpN1daDu7TGw2Sy0l+m8RkfCRuTMfvL2N6XxRFOFlJjPY9JDEtDLFG6xzug/bF5n3QKuTe1sISaLUA44Bjox2EVZzPSYz7oYjUAb199JftLubcUT5lzyS0FmSPzTb8NonLOYs/pA4sLBz/MYmh82RMzi5SdoZJsMkekJ3XTUbk98ieMpta85TEqmiEsfWrd14G7Wdvvgnao4dYO1y/ehm0q5ewVv/wCP3FynAtj/ewbq5Xcc9Bhk9jf6fbBW3nEeacqxevlP6OybnS7nOswROyn7569QporQXcX9VDzId+gLk0I/v4ucImn5CmGJdsF32a/c8r22H1MTn3mUqCjoPzZ0i8ccnZkE3O0rIcbdQq0DYsUqdXKiuo2bjm7BlhE/fsXoh1YX0Fawqf2FV2jLHfmrIrUn6beELOvRJyJsPWxkG7TRPUPHKeZ5Pz6nmSjvG994b4HakZoJ8XDtpLJWmB5hZ45m9itNPOQbd8SY41i7ExLvVOMPY7DtYdvodxlPlekpCaitiGYeeaxL+jBJ/x0w8+Ae30BOf9G3dugvb1298CLZ7ab3VTfI8nDo5rXMcxtK6hn3UnuK+wEqwBg4Kca7uzxdVp9D9XhRBCCCGEEEIIIYQQQgghhBBiBvRxVQghhBBCCCGEEEIIIYQQQgghZkAfV4UQQgghhBBCCCGEEEIIIYQQYgb0cVUIIYQQQgghhBBCCCGEEEIIIWYAu/J+GaS5symwmTBrxJsXpCG1jddtHfRB++d/wQbu3/vua+X7nmNT6GGG341z8i3Zq2ADW8dHLSRdpf0qNhge94egJaQhcJGSBtIVXA7Hxbljv+eQpt85WbPxaPA7r2G/1Wovgra8dh60w+MOaN2jPdSePATt+rWroM2TaqUGWhBg43fPR9vISINw4kEmtZiK98LN7LYZIb9ujIVNmwfENu7F2Hx7wa/iddE+aJ+l6BvHTWwqvXTpGmjnr2CD+NZ5tMmghs2s7bw8toTFLZfEAQ/X2iWxwbJx7rIMG79bZI5t62z9+xa7IPZNrrNyjFVFgfZisfGRRupWgTHNm4p9jkWeSYzZYe9mYUxjNl8UuG7MYTIiuiRWZ2T8MRlr7qBdFWSe2HgL0oTeTI3DymZbm8JFLSVa88IaaBdfuQGaa6E9dR98Ato8qYaYx00ag2RnOM/7e8eg/fzHb4PWXEDbePGV66CFbrP098XGClwT2GgE9/Md0CxMz8afEJua4FiTSgLa2vIqaKspPmTY6YHWJ8+oF1jvjeIINLeKvlEjefhkKv5s7TyCa+49xrrDhJi/Vtc3QPvorV+D9t3XXgft9T/5Jmhv//Jn+Nw5Y5OcV/FwLoYk1u0+3QRtTNbk2fY90Pb2sTYYnpZtoSD1Nit5HBtjPc2oJNZZDrmXXUfymiGabTCuJ/EEtBcureMzSG47PMLYsn4B/e3u3YPS3zmJXacn+FsFmVGb1KMFq1FJzskdUvOSHDtPcoPvQ+2AaGEVa9UL67gei4tt0C53cF90cFBek+4V/K3xeAxaHOP6DodYW48SXLck/921gjHG3Lr1MmidU4zXO7u4t4sz9IPvf/tboF0mfnDSQTsdjdCH3n3v/dLfT7a24ZrC4LgCkkvu3HkJtCzGee+NcY5dG/O6e8Zq+mawAFqOJmRiYhpxTOIBLq8piF1ZpObOpkKEVaCftRbQz7Ic65HeEO3x2R7a4z6xqWuLLdCsgKwlqelrNTwXYLHBiTEeHh90QXvw9Ai00zGZ5KlcV8TEzkgesUkMZmtDdkimsHHeJxZqnsHxz5M0xflje/I8p6cheG9C9gPkupw8I5vae00yjC2//MUvQPv7v/070P7iz38A2ps/wTPSx5tfgPbGG2+A9vzJFmhsw7tEcppLzgnb7SXQshznZDxC360FaEPL7fL+Jxrh+dPR4QFoEam7Tvv4TIvUmYMI12ehiTGUnefME/Y27B2ZzbssHpDcRo66jMPOKdJyjLBJXGLnzA454LDJyDJy3ThCO/OreB6YkbE2l9Fux120l6VlrNHWSA0Yk+8PXVKD905xr5xMjYPtcQzZa+Qx0diBETk/GPexPo0T9LUwwPppnlR99MvxKc7DIMNzBXuCMWK5gXGuVl8G7XSIOXvYL39bWV67BNcsbeD5Q/8Y7YzWOz38zmUR20gTHFfBT/3xOmIbLM8Nia998BT39Zcu4twVFubSZtgq/b2xjOeLixMc62mBb7fi4zMP9nG9IlJ7Jh6+m+/8fh9bztYuQAghhBBCCCGEEEIIIYQQQgghzij6uCqEEEIIIYQQQgghhBBCCCGEEDOgj6tCCCGEEEIIIYQQQgghhBBCCDED+rgqhBBCCCGEEEIIIYQQQgghhBAzwDolU5ZaLdCiCBvxDsesIWwVtDQljXO9ALS33v0YtK3nz0t/d4fY/LczwKbkKb6aqdWw4XVKGn4HAb6b62Nz50oVG/2yxuCuh/dm5Ft3mmMzXYtoRYHPzRKclzgpT0K1UoFrVpawuXebNPKOSTPhyEeTGpMm2LnngTYkjeTnSZJhY+whsflGC+cwGmJT6YzYVWbhHGasf/KUaOFyz0xRYIP4wsF1G9o4/rfjU9CejPC64xDH5Z7bAO38+ipoV1ewIfXSAtqkTXx3aHDyIqusuS76Y6WC/l0Ja6C5Pq51pRqCFhC/8ojNnzVSEiSzGP0ytnG+2L0mR7siPdhNTuKXPRXn4hyvyUnj95z4WZ6jPfourpGFpmHyAu3bstGHiCublMRgQ3KfRezWdcjLOOy5pFl9UV4Ln7xcmuFvxSHG6sWb10Bbv4K+HO1jQ/sv7r0PWjXBGDpPvBqZG1xy47XRzy+3zoG2c3cPtLf/80PQwibWRWGtbJO1Kr7b6gKuhxdifHxytAlab4T2GJGa5eT0ELR+jFp0gPkgHKFfJfkiaN0K2q0fNECLY7zuZNAB7dmg/C4dD30qa+C7nV/CdT3cegKaS97j0nXMQY57DFqrvgDavPFI/vRdjOs5CdiTCdY3e6dkP5DgdT6ppVculOvL4YjU7yT+u6S25pCiivwe03ISDJiWFpj/cvJ7n376EWgv3bwD2oU1jC1Pnj4ELYqGpb9ZLmG5yTIY/6nEco5H8mlI8ql7tv79rm/h3KRkfHYF82DdRX+xchxzmmLdeG65CVp2bb30dzxBm89StJ/pPZwxxozHEWijIf5etzsA7eMvDkAbDDA3sTjcO8H4z/anH/zmA9A++/AT0Go1fG6tjvNZ8coxyXeJzRf4vjeuXAJttYW/v/sEc2dI9gMhqfNpgTtHiMkbQ+bGctBXK2Tv7pGzi4IcJcWkhnKm6vxmE+P3lctYyzik3n688xy0wsE91q07r4LWIns2Vm+zOr8wOHfD/hFoeYG/d7CPdcvxEdnDkDxsT+9DClwHiwV6EtSTBMfgsHMBi5xZuBh/3ATfd54UBTsj+/21NGPzxWIOySVTS+JYeM3Dz++D9k//+A+gsXjzjddfA63dboF259ZN0Kwf/hC0n775E9DSiOQcUgNlE7Tl4XgEWuCjnbbXML7GU/d2elhj/tuP/hW0w24XtOMe5qqj0xPQUpJzyZLRknKe2CR+5SQXsxe3SdwoMpwHdiZqHPw9xyrng4CcublkThNSA5EQTN8j9DFvWA7aVI+cMzcXsD5rkv3R2jKeTUZ9tKHtZ9ugpROSN7pok5NRed5dn8QZkqt9D3Mw2zNFxG9jcuhcdTC/GJJf50ke4Zy2qyug2R7aX1ZgLexVMH5V0AxMnGO+89zyfFUCtMeNV/E7yu4mnhf1Oj18KPk+5Ni4bjnxDZvYi+djrZ0VuL5JhHuLLMN5Wl8j5/sr66ClcRe0veNyLXcywPHfXsL98A3yTGsHc8T1YzIuF31vHPwKtFHrJdCMuUy0Mmdr5yuEEEIIIYQQQgghhBBCCCGEEGcUfVwVQgghhBBCCCGEEEIIIYQQQogZ0MdVIYQQQgghhBBCCCGEEEIIIYSYAX1cFUIIIYQQQgghhBBCCCGEEEKIGcAOyF9CRJpAB+TT7IQ0uvVIY+QU+wubYrrzuzHGrmI34cfPD8vXkGbZaUKa0qfY/DgizXqHwyG+B3m3IMCmxjUfG+dWq9iE3rHxXYIK/l41xPHHMTYsPux0QMsNXud65XG0m9jwe22xBdq5c4ugdYcT0HpdbO49OO2C1lrE3zs6PAJtniQZjs8hzcXbqziHyRjXMo1Jg3g0A5PkKBZTDceJ+RjLYOd3y0KtIJpx0W5d0nE+CdGXJwu4li8srIHWXsSm8fUmhqB6iP4cVPC6KMX5jA1qhVcem0Mavxs2J0TzfBy/Q+KPR57hkKbihUF7miskLjsezoPj4XvbLkkIOdEK9m968MHTTy0sXFtm3xUff6vdbINmE3/JSDP4LEfNcfDeICB5LsV5Yn6aE5/Pchxvvz8ArSCxIJ/KuacWXuSewzm5fOMGaO32MmjP7m2CdrS5hc8gc1fxzta/6SryEWjdY6wBdp8dgnb761dBi4e45t3jPmi//I/3QEvt8prHN3D+LiSoLTWXQLt57mXQTvqnoB2MMO86JI6GdgjaxG+B9uC3n4O2e7AP2vmL10HrPPoCtJjUnsyHqqvld7l0+yZc0750CbRhhD7FYtnS+VXQiiquRbePttPt4RjmzcTGHFWQeY3JdRVSI55fx/xuVTAm1psNfO5ULn+8hbFkHKGfVqpok46D8d8h8c9K8PeyBPcv0+9mjDHpBK/LctxLJHEM2jDCuvKze/dBs8m8904xBhVTS+aRPFTks9WGZPnpfLqkDvID3Ofkxdmqb6oG/dDz0IcNy1E5GV+K9yZESxOyR83K65tUSPFFYOuWkpyQEBsdDTA21UL0oQ8f7IAWT9Bf1uq4zzEFqcGHXfw9vNNkfRzbiNSfwdThw1ev436jWcM4s7aKedJNiE2kVdCqFtp8hfhocMbqG1YPGwvf2yc+7blkfOTnJsSF+hHanzNV+y+38XyjEeDcd3cx7nV2dkE712iBtrCAdW4eofWx2j8hsX9MYrplE7sl5zTP98kZh0XylYP7cdsqz11BbM+yyBkaiRcZ2W8YorUauBYWyQenh8QA5gg7rytILmIai6+OjWNu1RdAc4m/VKbOCVukdkpj9JXOEdqKu4xnLdEI4/LDowPQPv5oHbQ/fvVV0Dbv3QNtdw99LRqTuFnHmFurYN5kMTInvrZ3Un7uZ5uP8D4St26+chu0agPf7ZicmzJIWUSPjOYJK7Uyojk+5nseg3BeE3KGXvFIjT/lG66Pcb5K6sUiJjU0yc8BiUFry1dAY/VOMkDfGHTR1xYW8Nyje/AUtK3tx6BNSA3YrGEeCjwcR7NV9nHLxnwzGOI+viBnaCy+ZTn+nkPWwjEYk1JyhjlPLA9jho2ScW3yrcbFeOCS8/2E7NeyAm1yOOqW/s47Xbjm6p3LoD37FG2v9/Zd0FJiUwVZD4fkoJzU5HlOgsP0ZtIY45BS4ZXrV0D74be+CdpSA23y2SF+I+pO7Q8CcubzBTkP3erguVKVnL0vkpqyTb49NGrHoDUHmA+N+VOilTlbniKEEEIIIYQQQgghhBBCCCGEEGcUfVwVQgghhBBCCCGEEEIIIYQQQogZ0MdVIYQQQgghhBBCCCGEEEIIIYSYAX1cFUIIIYQQQgghhBBCCCGEEEKIGcCOrl/CZIyNpgMHm9+G5Bdz0pCa9M82ucHOuXlBtKnGzWlMGtVn+G6zNrTPc3ymbeN36JMTbMzbIWNt1mugLbSxMX2TNCeuGGw0neXYYNm1sNmvE+AkT6aaM1dcnCf2W+kIG2inI3yPQRcbAucJNtCuBB5oEWlSPk8cD+emvYiN2eshrltGbDJN0K7SDLXC4HNtu+xYFvl3EbbF7sM5tV281/XwfUMX7200cPxrU83rjTGmThpI13zUfGIHMUpm4OM7jzNs8J1ZxIfc8g/6DgYpz/dBs4k9WiQOsBgSx9gM3vfxfX3SfHueOCn59zYx2mhm0PcLg37uGpxXx+ACW8R28zydugbnmWl5iu8xGuE8s7U0JAcVLAclGCOjhNgL8VOL2ChxeWNIv/nMoF0ZNgdTtttcbcM1KzeugmaT8d9/7x3QogOM8w7xR5fEH5bT50l3H/P43ffvgxYN0a6cCubn5Q2c63iM/vLs4RFovzIflv72qugrvZUOaM1OC7QLq9dAazWWQfM9tMfQQr9dCfHelSshaJcXGqC99evfgLY13APtaLgD2lLrPGjrly6DdvFi+bqNCxv4+8e41gODtS1zvkYD13WSD/HWDOdkdR1tZ96kQQBaQWKz214F7dxFnP9wdR20hIS64XgE2slROZ74NVJTLOJ7eAHaaWFjMPVIbe3mGK8KUvsXpEZLIrSZOEJbiKIBaOQRxvfRZgx5blZg/J/E/dLfLK3ZNq4rKVtMnmNe80i9VCH1ksNifXa27J5sWY1LalCWe01OanpymctqRA+fkU7lyyQhNQqpFVg9YgJSA1RwLRdI/b68ugTanVtXQOv30JajCa5vlpH9Y4p2m5IaitWBLtmvOFP+7DN7JHNeqWDMC0gcDEPcs1eI5pN72bvMk9VFjKVJjOtmE5u3CrKW5CzI9XDMzTrOzXQ87A8wH/vk3TJiZ3WSq6pk35kNMC4XxPZYnLMsfEZQQztIiZ++f+8T0I66fdCcCvokO5pzrLKWE19h/sOw2b6EHL+88iLm+XzQBe39o2czPfcPBd9Psr0dC/RITmqKcYYxrRmizdfbzdLfrRbWkT45a7i4hGeEFslBkxj9cTzAWP1fP/85aOmYnM2CYoxHcnt7AePKhNRFbN4dMo6Rje8STcWCZgN9b0LyzaO7ZP82IL5H6sKCTAAJg8amMzU/bA9tr1LD3O6HWGuSI19Tr+M+LiMxwid5tpg6o3dqLbhmaXEFNDdHnzrcxT2hE+K72SS3T3q4V47J+ZBPxn+w8wB/L8UL/TqOo1FDLaiQ3OTgXqg3lROLCTnvd/HcIU5xcYoMfQpnyZi8wGc0PPTbyDpbZzcd8r3h8jKea3kjcjZFcuDxEX7n6I8PQLOqpFaolOe/gcttwgbe95U/ewG0cYTrdvjkEK87xevqNazFqiS55+QbWZP41a2NG6C9Rs4O2zWcz80tjMNPjp6CNu1WaY6/da/bBc0neblCcn8jwJi32MQ8fH7tAmi1Q6xtXgcF0f9cFUIIIYQQQgghhBBCCCGEEEKIGdDHVSGEEEIIIYQQQgghhBBCCCGEmAF9XBVCCCGEEEIIIYQQQgghhBBCiBnQx1UhhBBCCCGEEEIIIYQQQgghhJgBq5i1o7sQQgghhBBCCCGEEEIIIYQQQvw/Rv9zVQghhBBCCCGEEEIIIYQQQgghZkAfV4UQQgghhBBCCCGEEEIIIYQQYgb0cVUIIYQQQgghhBBCCCGEEEIIIWZAH1eFEEIIIYQQQgghhBBCCCGEEGIG9HFVCCGEEEIIIYQQQgghhBBCCCFmQB9XhRBCCCGEEEIIIYQQQgghhBBiBvRxVQghhBBCCCGEEEIIIYQQQgghZkAfV4UQQgghhBBCCCGEEEIIIYQQYgb0cVUIIYQQQgghhBBCCCGEEEIIIWbg/wA6Ki6tDmqwfwAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "def grid_plot(tensor, labels):\n", - " tensor = tensor.clone().detach()\n", - " tensor.mul_(ds).add_(dm).clamp_(0, 1)\n", - "\n", - " fig, axes = plt.subplots(1, 10, figsize=(24, 24))\n", - " for im, l, ax in zip(tensor, labels, axes.flatten()):\n", - " ax.imshow(im.permute(1, 2, 0).cpu());\n", - " ax.set_title(l)\n", - " ax.axis('off')\n", - "\n", - "grid_plot(ground_truth, [validloader.dataset.classes[l] for l in labels])" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "local_lr = 3e-4\n", - "local_steps = 1\n", - "import pickle\n", - "\n", - "from collections import OrderedDict\n", - "model_bespoke.zero_grad()\n", - "target_loss, _, _ = loss_fn(model_bespoke(ground_truth), labels)\n", - "input_parameters = inversefed.reconstruction_algorithms.loss_steps(model_bespoke, ground_truth, labels, \n", - " lr=local_lr, local_steps=local_steps,\n", - " use_updates=True)\n", - "\n", - "\n", - "\n", - "\n", - "# params_t = OrderedDict((name, param) for (name, param) in params_t_full[0].items() if name in params_s)\n", - "input_parameters = [p.detach() for p in input_parameters]" - ] - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "# based_model_names = [name for name, _ in model_bespoke.named_parameters()]\n", - "# with open (\"start_reprs.pkl\", 'rb') as f:\n", - "# params_s_full = pickle.load(f)\n", - "# with open (\"end_reprs.pkl\", 'rb') as f:\n", - "# params_t_full = pickle.load(f)\n", - "model_bespoke = mutils.get_model(\"resnet10\", \"cifar10\", \"cuda\")\n", - "params_s = OrderedDict((name, param) for (name, param) in model_bespoke.named_parameters())\n", - "with open (\"end_reprs.pkl\", 'rb') as f:\n", - " params_t_full = pickle.load(f)\n", - "\n", - "params_t = OrderedDict((name, param) for (name, param) in params_t_full[0].items() if name in params_s)" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [], - "source": [ - "param_diff = OrderedDict()\n", - "for (name_s, p_s), (name_t, p_t) in zip(params_s.items(), params_t.items()):\n", - " if name_s == name_t:\n", - " p_t = p_t.to(device)\n", - " p_s = p_s.to(device)\n", - " param_diff[name_s] = p_t - p_s" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "OrderedDict([('conv1.weight',\n", - " tensor([[[[-0.0006, 0.0561, 0.0613],\n", - " [-0.0925, -0.1245, -0.1912],\n", - " [ 0.2336, -0.0051, 0.0833]],\n", - " \n", - " [[-0.0461, 0.0431, 0.1182],\n", - " [ 0.0759, 0.0499, -0.0546],\n", - " [-0.0938, -0.0426, 0.0995]],\n", - " \n", - " [[-0.0867, 0.1786, -0.0268],\n", - " [-0.1879, 0.1099, 0.0997],\n", - " [ 0.0216, -0.0517, 0.0542]]],\n", - " \n", - " \n", - " [[[-0.0842, -0.0590, -0.0917],\n", - " [ 0.0528, -0.0941, -0.2365],\n", - " [-0.1343, 0.0125, 0.0060]],\n", - " \n", - " [[-0.1608, 0.0795, 0.1789],\n", - " [ 0.0797, -0.0478, 0.1068],\n", - " [ 0.1370, 0.0267, 0.1973]],\n", - " \n", - " [[-0.0459, 0.1254, 0.1123],\n", - " [-0.1720, 0.1773, 0.1561],\n", - " [ 0.0347, 0.0052, 0.1108]]],\n", - " \n", - " \n", - " [[[ 0.0406, 0.1152, 0.1499],\n", - " [ 0.1664, -0.0313, 0.1316],\n", - " [-0.0111, -0.0099, -0.2676]],\n", - " \n", - " [[-0.1355, 0.0306, 0.0134],\n", - " [ 0.1814, -0.2067, 0.2133],\n", - " [-0.0884, 0.0318, -0.0323]],\n", - " \n", - " [[-0.0081, -0.0779, 0.1160],\n", - " [-0.0263, -0.0354, 0.0638],\n", - " [-0.2263, 0.1726, -0.1936]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 0.1928, 0.1935, -0.0905],\n", - " [ 0.2506, 0.0178, -0.1192],\n", - " [ 0.1269, -0.0953, -0.2048]],\n", - " \n", - " [[ 0.0185, -0.1787, -0.0943],\n", - " [-0.2781, 0.1565, -0.0043],\n", - " [ 0.0247, -0.0955, 0.0705]],\n", - " \n", - " [[ 0.0941, 0.1939, 0.0726],\n", - " [-0.1135, -0.1600, 0.1947],\n", - " [ 0.1036, -0.1385, 0.0691]]],\n", - " \n", - " \n", - " [[[-0.0507, -0.1775, 0.0227],\n", - " [-0.0576, 0.0651, -0.1624],\n", - " [-0.0899, -0.1509, -0.0459]],\n", - " \n", - " [[-0.0317, 0.1149, 0.0944],\n", - " [ 0.1851, -0.0195, -0.0367],\n", - " [ 0.0663, 0.0479, -0.0879]],\n", - " \n", - " [[-0.1262, -0.1743, -0.0682],\n", - " [ 0.1980, -0.0746, 0.0476],\n", - " [-0.0467, 0.0354, -0.0615]]],\n", - " \n", - " \n", - " [[[-0.0031, 0.1585, 0.0819],\n", - " [-0.0030, 0.0041, 0.0633],\n", - " [-0.2265, -0.0170, -0.0711]],\n", - " \n", - " [[-0.0104, 0.0656, -0.0445],\n", - " [-0.1921, 0.1331, -0.0937],\n", - " [-0.1415, -0.0107, -0.1412]],\n", - " \n", - " [[ 0.0894, -0.1457, -0.0566],\n", - " [ 0.1551, 0.0706, -0.0333],\n", - " [-0.0042, 0.0259, 0.0622]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('bn1.weight',\n", - " tensor([ 4.6492e-06, -2.8610e-06, 2.3842e-06, 3.5763e-07, -3.0994e-06,\n", - " -5.3644e-07, -4.7684e-07, 1.3113e-06, -8.1062e-06, 2.1458e-06,\n", - " -9.7156e-06, 3.5763e-07, -1.1921e-07, 7.0333e-06, 5.9605e-07,\n", - " -5.9605e-08, -3.2187e-06, -3.8147e-06, 3.4571e-06, 8.7023e-06,\n", - " 5.2452e-06, -1.1325e-06, -2.5034e-06, -1.1325e-06, 5.9605e-07,\n", - " -1.2517e-06, -3.5763e-06, -9.5367e-07, 2.2650e-06, -2.8014e-06,\n", - " 2.8610e-06, 1.3113e-06, -3.4571e-06, 1.0729e-06, 1.0729e-06,\n", - " -1.1921e-06, -5.9605e-07, -6.2585e-06, -4.4107e-06, -7.8082e-06,\n", - " -7.7486e-07, 4.7684e-06, -4.7088e-06, 1.6689e-06, 4.4107e-06,\n", - " -1.9670e-06, 2.0266e-06, -2.3842e-07, -1.2517e-06, -1.7285e-06,\n", - " 1.7881e-06, 3.5763e-07, -1.0133e-06, 2.8610e-06, 1.6689e-06,\n", - " -3.2187e-06, 2.7418e-06, -3.5763e-07, 3.5763e-07, -8.9407e-07,\n", - " -1.3709e-06, 7.6294e-06, 3.8147e-06, 7.1526e-06], device='cuda:0',\n", - " grad_fn=)),\n", - " ('bn1.bias',\n", - " tensor([ 5.4231e-06, -3.2287e-06, 8.1165e-07, -4.1569e-07, -1.7457e-06,\n", - " -4.1456e-07, -2.2053e-06, 8.3448e-07, -1.0379e-06, 3.1667e-06,\n", - " -2.9718e-06, 1.3673e-06, 7.8012e-07, 2.8126e-06, -2.2484e-06,\n", - " 2.0777e-07, -3.4527e-07, 3.2588e-07, 3.1864e-06, 7.5375e-06,\n", - " 2.8332e-06, -5.6947e-06, -1.0997e-06, -2.4325e-06, 5.9140e-06,\n", - " -1.9417e-06, -2.4949e-06, 1.4172e-06, 1.4224e-06, -5.0170e-06,\n", - " 1.3376e-06, -1.8968e-06, -3.7705e-06, 2.1635e-07, 3.2337e-06,\n", - " -1.4948e-06, 1.7549e-07, -5.6401e-06, 4.3037e-06, -6.3509e-06,\n", - " 1.3941e-06, 4.9555e-06, -6.7065e-06, 4.9904e-07, 5.8545e-06,\n", - " -4.9189e-07, 1.5127e-06, -1.9307e-06, -2.1169e-06, 3.0242e-06,\n", - " 8.7774e-07, 5.8332e-07, -2.6954e-06, -2.2571e-07, 2.0464e-06,\n", - " -5.8163e-07, -1.1077e-06, -1.4233e-06, 1.1387e-06, -1.1688e-07,\n", - " -1.5522e-06, 2.9426e-06, 2.9115e-06, 5.6802e-06], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer1.0.conv1.weight',\n", - " tensor([[[[-2.2570e-02, 4.6579e-02, 4.3270e-04],\n", - " [ 3.9203e-02, 1.5285e-02, -1.8835e-03],\n", - " [ 4.0751e-02, 3.1965e-02, -2.3628e-02]],\n", - " \n", - " [[-2.2898e-02, -5.9936e-03, 9.9707e-03],\n", - " [ 2.8830e-02, 2.9010e-02, -2.1784e-02],\n", - " [ 2.5152e-02, -4.0623e-03, 1.3364e-03]],\n", - " \n", - " [[-2.0373e-02, -1.5970e-02, -2.7016e-02],\n", - " [ 1.4274e-03, -1.9746e-02, -8.7298e-03],\n", - " [-2.5143e-02, -2.3898e-02, 1.5841e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-9.5870e-03, -3.6288e-02, 9.6889e-03],\n", - " [ 4.5948e-03, -2.9685e-02, 4.0744e-02],\n", - " [ 1.5693e-02, -9.7967e-03, 3.2967e-02]],\n", - " \n", - " [[-1.0512e-02, 2.0607e-02, -7.5278e-03],\n", - " [-7.1602e-04, 4.0102e-02, -2.5954e-03],\n", - " [-4.5868e-02, -6.7171e-03, -1.1770e-02]],\n", - " \n", - " [[-1.1169e-03, -1.6320e-03, 4.8017e-02],\n", - " [-2.8562e-02, -2.4703e-02, -3.4483e-02],\n", - " [ 2.7159e-02, -7.1618e-05, -1.6801e-02]]],\n", - " \n", - " \n", - " [[[-2.1106e-02, -2.2973e-02, -6.0225e-03],\n", - " [-5.5912e-02, -4.6143e-02, 3.0123e-02],\n", - " [-5.7831e-02, 3.0423e-02, 2.4413e-02]],\n", - " \n", - " [[-2.9441e-02, -3.2263e-02, 2.9663e-02],\n", - " [ 1.1660e-02, 3.8977e-02, -3.6358e-03],\n", - " [-2.5374e-02, 1.3639e-02, -7.5420e-03]],\n", - " \n", - " [[ 3.6191e-02, -2.0330e-02, 1.4488e-02],\n", - " [ 2.2990e-02, -2.1860e-02, -3.7394e-02],\n", - " [ 4.2311e-02, -1.9836e-02, -3.7274e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.0211e-02, 1.3598e-03, -3.2501e-02],\n", - " [-5.6539e-02, -3.3627e-03, 7.4346e-03],\n", - " [-9.2457e-04, 4.0321e-02, 2.3731e-02]],\n", - " \n", - " [[-7.9937e-03, -3.6999e-02, -8.0297e-03],\n", - " [ 3.8824e-02, -1.2008e-02, -4.5381e-02],\n", - " [-1.3227e-02, 2.8266e-02, 2.2535e-02]],\n", - " \n", - " [[-2.6154e-02, 2.3498e-02, -1.2162e-02],\n", - " [ 2.1689e-02, -5.1684e-02, 1.1600e-02],\n", - " [-2.4643e-02, -1.0527e-03, 6.2798e-04]]],\n", - " \n", - " \n", - " [[[-1.0321e-02, 2.4365e-02, -5.5286e-02],\n", - " [ 1.2955e-02, -2.3997e-02, 5.2910e-02],\n", - " [ 2.6851e-02, -1.7141e-02, -1.5507e-02]],\n", - " \n", - " [[ 1.1109e-02, -4.2064e-02, 4.1614e-02],\n", - " [-2.7240e-02, 2.6202e-03, 4.7014e-02],\n", - " [-5.2148e-02, 2.9868e-02, 3.2549e-02]],\n", - " \n", - " [[-3.2545e-03, 2.7492e-02, -3.8950e-02],\n", - " [ 1.5161e-02, -2.5894e-02, -3.2587e-02],\n", - " [ 1.7659e-02, -2.2911e-02, 1.6492e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.5354e-02, 5.3448e-02, 2.3870e-02],\n", - " [ 1.5850e-02, 1.3058e-02, 1.6393e-02],\n", - " [-1.5742e-02, 1.6519e-02, 2.2951e-03]],\n", - " \n", - " [[-2.2953e-02, -4.3644e-02, 6.5544e-04],\n", - " [-2.1834e-02, -4.3274e-02, 5.7042e-03],\n", - " [ 1.8394e-02, -1.5122e-02, 1.4276e-02]],\n", - " \n", - " [[ 2.8725e-02, -3.2279e-02, 3.5784e-02],\n", - " [-3.5181e-02, -2.3900e-02, 5.7771e-05],\n", - " [ 2.7850e-02, 3.0817e-03, 5.1541e-03]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 3.9989e-02, -1.7403e-02, 5.0907e-02],\n", - " [-7.3910e-03, -3.4016e-02, 7.1424e-03],\n", - " [-3.7204e-02, -1.9108e-02, -3.5431e-02]],\n", - " \n", - " [[ 1.3562e-02, 2.2771e-02, -6.0645e-02],\n", - " [-2.6215e-02, -2.6883e-02, -3.2663e-02],\n", - " [ 8.0954e-03, -3.4466e-02, -1.8052e-02]],\n", - " \n", - " [[-5.5253e-02, 2.6264e-02, -4.6535e-02],\n", - " [ 3.9234e-03, -1.0685e-02, 9.2071e-03],\n", - " [ 1.2697e-02, -2.4685e-02, -1.7718e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.5044e-02, -1.7497e-02, -2.1783e-02],\n", - " [-2.0340e-02, 1.3263e-02, 4.0077e-02],\n", - " [-3.8877e-02, -1.1697e-02, -5.0333e-02]],\n", - " \n", - " [[-7.3433e-02, -2.2334e-02, -4.7794e-02],\n", - " [-1.1918e-02, 2.2436e-02, -1.8381e-02],\n", - " [ 1.9740e-02, 1.6637e-02, 1.9241e-02]],\n", - " \n", - " [[ 3.2566e-02, -4.9752e-02, 2.6469e-02],\n", - " [-2.4228e-02, -3.1309e-04, -4.1350e-02],\n", - " [-7.4496e-03, 1.7813e-02, -3.1136e-02]]],\n", - " \n", - " \n", - " [[[-3.8622e-02, -3.7675e-02, -1.3183e-02],\n", - " [-4.2526e-02, -1.1766e-02, 3.5638e-02],\n", - " [-2.8378e-02, 2.9796e-03, -1.7564e-02]],\n", - " \n", - " [[ 1.8862e-02, -1.6337e-02, 1.3012e-02],\n", - " [-3.1859e-02, 2.2553e-02, 1.5612e-02],\n", - " [ 1.3076e-02, -4.6412e-03, -9.1515e-03]],\n", - " \n", - " [[ 2.0634e-02, 1.1302e-02, 4.9690e-03],\n", - " [ 1.5253e-03, 9.4118e-03, -3.1450e-02],\n", - " [ 2.0302e-02, 5.1324e-02, 2.1210e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.5408e-02, 1.5580e-02, -2.0611e-02],\n", - " [ 1.2426e-02, -2.4775e-02, 3.1102e-02],\n", - " [-4.5320e-02, 1.4904e-02, -2.4320e-03]],\n", - " \n", - " [[-4.4531e-02, -2.4908e-02, -1.1673e-02],\n", - " [-6.1137e-03, -5.2799e-03, -4.8738e-02],\n", - " [-2.8913e-02, 1.7804e-03, 6.6780e-03]],\n", - " \n", - " [[-4.0387e-02, 2.9545e-02, 1.7133e-02],\n", - " [ 6.8722e-03, -2.8609e-02, -2.0323e-04],\n", - " [ 3.9065e-02, 2.2012e-02, -2.0337e-02]]],\n", - " \n", - " \n", - " [[[ 9.0897e-03, 1.3927e-02, -5.3809e-02],\n", - " [ 4.6715e-04, 1.6106e-02, -2.4101e-02],\n", - " [-8.4275e-03, -1.7561e-02, -1.7671e-02]],\n", - " \n", - " [[-1.2119e-03, -2.4723e-02, -4.8516e-02],\n", - " [-2.0067e-02, 4.0879e-04, -1.8960e-02],\n", - " [ 2.2412e-02, -3.6067e-02, 9.5359e-03]],\n", - " \n", - " [[-3.7317e-02, -2.0986e-03, 3.8389e-02],\n", - " [-5.7991e-03, -1.4761e-02, -9.8050e-04],\n", - " [-3.0369e-03, 3.3129e-02, 2.1562e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.4468e-02, 3.0479e-02, 1.8400e-02],\n", - " [ 1.3205e-02, -1.2044e-02, -3.0517e-03],\n", - " [ 1.4351e-02, -2.7222e-02, -1.7896e-02]],\n", - " \n", - " [[ 1.1494e-02, 1.9010e-02, -1.1443e-02],\n", - " [ 3.3070e-04, 4.1223e-02, -4.1695e-02],\n", - " [ 1.3860e-02, -1.1250e-02, 6.9530e-03]],\n", - " \n", - " [[ 2.5772e-02, 2.0700e-02, -2.7200e-03],\n", - " [-6.9283e-03, 2.3300e-02, -9.7905e-03],\n", - " [ 1.9706e-02, -1.8724e-02, -3.7177e-02]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer1.0.bn1.weight',\n", - " tensor([-1.0729e-06, -1.1921e-06, 1.4305e-06, 1.0729e-06, -7.7486e-07,\n", - " -8.9407e-07, -1.9073e-06, 1.0729e-06, 3.6955e-06, 2.6226e-06,\n", - " 9.5367e-07, 1.1921e-06, 2.7418e-06, -1.6093e-06, 1.3113e-06,\n", - " -1.1325e-06, 4.7684e-07, -5.9605e-08, -2.5630e-06, -1.9073e-06,\n", - " 3.5763e-07, -1.1921e-07, 1.7881e-06, -2.0862e-06, -5.9605e-07,\n", - " 2.1458e-06, -3.2187e-06, -3.3379e-06, -3.1590e-06, 3.5763e-07,\n", - " 3.5763e-07, 1.5497e-06, 2.1458e-06, -4.1723e-07, 1.3113e-06,\n", - " 2.0266e-06, -1.0133e-06, -1.7881e-07, -4.0531e-06, 1.1921e-06,\n", - " 5.9605e-07, -3.9339e-06, 2.6226e-06, -2.6226e-06, -1.6093e-06,\n", - " 4.7684e-06, -5.9605e-07, 3.6955e-06, 8.3447e-07, -1.2517e-06,\n", - " 1.7881e-06, 7.1526e-07, 1.1921e-06, -2.3842e-07, -4.7684e-07,\n", - " 8.3447e-07, 3.0994e-06, -1.9073e-06, 2.1458e-06, -1.2517e-06,\n", - " -1.8477e-06, -1.7285e-06, -1.7881e-06, -8.9407e-07], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer1.0.bn1.bias',\n", - " tensor([-1.6646e-06, -8.3373e-07, -7.3852e-07, -1.8242e-06, -1.9939e-06,\n", - " -4.2068e-06, 1.2229e-07, -9.5055e-07, 3.2928e-06, 2.4605e-06,\n", - " -1.0227e-06, 1.9723e-06, -1.5289e-06, 8.2279e-07, -9.1037e-07,\n", - " 1.4940e-06, -2.5393e-06, -1.6297e-06, -3.4093e-06, -1.6058e-06,\n", - " -1.6695e-06, 6.3167e-07, 3.0258e-06, -3.3431e-06, -1.0892e-06,\n", - " 1.1644e-06, 1.8028e-06, -1.6146e-06, -3.2107e-06, 1.2860e-07,\n", - " 1.0780e-08, 2.2232e-06, -6.5523e-07, -2.5341e-06, -2.6235e-06,\n", - " 1.3846e-06, 1.0392e-06, -3.1820e-07, -3.5873e-06, -9.7919e-07,\n", - " -1.1498e-07, -1.5139e-06, 2.9632e-06, -2.7121e-06, 4.9362e-07,\n", - " 2.4185e-06, -2.5546e-06, 4.9935e-06, 4.7802e-07, 1.9509e-06,\n", - " -4.6129e-09, 1.4534e-06, -6.3682e-07, -4.7444e-07, 1.9421e-07,\n", - " 1.0255e-06, 7.3000e-07, -5.0841e-07, -5.6548e-07, -8.5624e-07,\n", - " -1.5565e-06, 1.0536e-06, -5.5264e-07, -7.5711e-07], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer1.0.conv2.weight',\n", - " tensor([[[[ 1.0396e-02, -2.0216e-02, -9.7034e-03],\n", - " [-1.6117e-02, 3.2930e-02, 2.6085e-02],\n", - " [ 1.6743e-02, -1.5415e-02, 3.0183e-02]],\n", - " \n", - " [[-1.2542e-02, -2.0246e-02, -2.1830e-02],\n", - " [-2.3590e-02, 4.3001e-03, 2.3873e-02],\n", - " [ 2.0339e-02, 2.6765e-02, 1.5982e-02]],\n", - " \n", - " [[ 5.9611e-03, 7.7520e-04, 2.5923e-02],\n", - " [-2.4622e-02, -1.0194e-02, 2.2966e-02],\n", - " [-1.7405e-02, -1.7530e-02, -5.4647e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-8.8577e-03, 4.5698e-02, -7.7993e-03],\n", - " [ 3.3063e-02, -3.2623e-02, 4.7993e-02],\n", - " [ 3.9605e-02, 5.0393e-02, -4.3616e-03]],\n", - " \n", - " [[-3.5010e-02, -2.4205e-02, 4.8226e-02],\n", - " [-3.5885e-02, 2.4366e-02, -1.5791e-02],\n", - " [-1.7299e-02, -2.0041e-02, 2.0468e-02]],\n", - " \n", - " [[-1.0454e-02, 1.0696e-02, 2.5080e-02],\n", - " [-2.6918e-03, -2.4078e-02, -2.8075e-02],\n", - " [ 2.1477e-02, 2.0518e-02, -2.3786e-02]]],\n", - " \n", - " \n", - " [[[ 5.0096e-02, 5.8117e-02, -5.5126e-02],\n", - " [ 2.7779e-03, -5.5745e-03, 2.0170e-02],\n", - " [-3.5873e-03, 4.9762e-03, 3.4356e-02]],\n", - " \n", - " [[ 4.4078e-02, 2.8536e-02, -3.0690e-02],\n", - " [-1.9038e-02, -7.6155e-03, -1.2531e-02],\n", - " [-1.2465e-02, -2.8210e-02, 1.4366e-02]],\n", - " \n", - " [[-2.6739e-02, 2.4669e-02, 4.6204e-02],\n", - " [ 2.9311e-02, -3.4707e-02, 2.6172e-02],\n", - " [-5.1166e-02, 3.2466e-02, 1.6315e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.5474e-02, -8.1297e-03, 6.5045e-03],\n", - " [-1.9999e-05, -1.3515e-02, 1.1818e-02],\n", - " [-1.5118e-02, 9.1569e-03, -1.3914e-02]],\n", - " \n", - " [[ 2.1644e-02, 2.0979e-02, 4.0117e-04],\n", - " [ 4.2587e-02, 3.4951e-04, 1.9599e-02],\n", - " [ 1.7381e-02, 3.0407e-03, -2.1914e-03]],\n", - " \n", - " [[ 3.8028e-02, 2.3806e-03, -2.4305e-02],\n", - " [ 7.0412e-04, -2.9877e-02, 9.9972e-03],\n", - " [-3.7057e-02, 2.9808e-02, 2.9682e-02]]],\n", - " \n", - " \n", - " [[[ 1.3419e-02, -1.2343e-02, -8.7604e-04],\n", - " [-4.3507e-03, -9.1235e-03, -5.0805e-03],\n", - " [ 2.2734e-02, 2.4164e-02, 2.5917e-02]],\n", - " \n", - " [[ 7.2083e-03, 2.1416e-02, 1.5793e-03],\n", - " [ 6.6166e-03, 8.8527e-03, -1.3748e-02],\n", - " [ 2.1095e-02, 6.7640e-03, -1.8672e-02]],\n", - " \n", - " [[-4.5202e-02, 1.1592e-02, -2.6447e-02],\n", - " [ 4.8787e-02, -2.6143e-02, -2.1383e-02],\n", - " [-1.3499e-02, -3.1855e-02, 4.2831e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.8935e-02, 2.6718e-02, 6.5139e-02],\n", - " [-1.9350e-02, -7.3472e-03, 1.4973e-02],\n", - " [ 4.9365e-03, 3.4216e-02, -1.3682e-02]],\n", - " \n", - " [[-5.1517e-03, -1.1952e-02, -6.4165e-02],\n", - " [ 6.0639e-03, 1.3290e-02, -1.2138e-03],\n", - " [-2.2491e-02, 2.1337e-02, -1.4244e-02]],\n", - " \n", - " [[-3.3079e-02, 6.3611e-02, 1.0925e-02],\n", - " [-2.7628e-02, 3.0921e-02, 4.1639e-02],\n", - " [-2.7173e-02, -1.0412e-02, 4.0272e-02]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 1.4416e-02, 2.0361e-02, -2.7634e-02],\n", - " [ 1.7212e-02, 4.7874e-02, -4.4004e-02],\n", - " [-6.9799e-03, -1.9941e-03, 4.5099e-02]],\n", - " \n", - " [[-1.7381e-02, -1.7613e-02, -2.4853e-02],\n", - " [ 1.1966e-02, -1.7379e-02, 2.6862e-02],\n", - " [ 1.1788e-02, 1.9911e-02, 1.4908e-02]],\n", - " \n", - " [[-4.0639e-02, -3.3738e-02, -2.7907e-02],\n", - " [ 1.8080e-02, -1.7539e-02, -4.5545e-02],\n", - " [ 1.7268e-02, 2.7446e-02, -3.0459e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1242e-02, -1.4522e-02, -7.9271e-03],\n", - " [ 7.7173e-04, 2.8665e-02, -4.2147e-02],\n", - " [ 4.1924e-02, -6.0923e-03, -2.9036e-02]],\n", - " \n", - " [[ 4.4091e-02, -8.0454e-03, 3.6650e-03],\n", - " [ 1.8156e-02, 3.4450e-02, -2.4242e-02],\n", - " [-2.2434e-02, -1.3157e-02, 3.6239e-02]],\n", - " \n", - " [[ 4.2711e-02, -4.1155e-02, 9.3523e-04],\n", - " [ 2.3412e-02, -1.3124e-02, -4.7054e-03],\n", - " [ 3.0531e-02, -4.1716e-02, 3.7192e-02]]],\n", - " \n", - " \n", - " [[[ 4.1159e-02, -3.3871e-03, 2.4896e-02],\n", - " [-1.5587e-02, -2.0231e-02, 5.4416e-02],\n", - " [ 2.0937e-02, 2.6982e-02, -3.9793e-02]],\n", - " \n", - " [[-9.0131e-03, 2.5295e-03, 2.8721e-02],\n", - " [ 1.7443e-02, -3.7112e-02, -4.8766e-03],\n", - " [ 1.4680e-02, 2.3375e-02, -5.4663e-02]],\n", - " \n", - " [[ 1.0035e-02, 1.2707e-02, 2.0542e-02],\n", - " [ 1.7489e-03, 3.8192e-02, 5.9935e-02],\n", - " [ 1.6425e-03, 9.2595e-04, -2.0362e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.0376e-02, -4.4225e-02, -2.9585e-02],\n", - " [ 4.0246e-02, -1.4116e-02, -3.0059e-02],\n", - " [-2.9061e-02, -1.2602e-02, -3.5743e-02]],\n", - " \n", - " [[ 1.3734e-02, -2.0217e-02, -3.7963e-03],\n", - " [ 2.2904e-02, 3.1865e-02, 3.4065e-02],\n", - " [-1.7687e-02, -8.5194e-03, 1.8948e-03]],\n", - " \n", - " [[-1.6955e-02, -4.8008e-02, -2.5571e-02],\n", - " [-3.5338e-02, -9.7560e-03, 3.9748e-02],\n", - " [ 1.1480e-02, 1.0620e-02, -3.8812e-02]]],\n", - " \n", - " \n", - " [[[-4.1050e-02, 8.1633e-03, -1.3328e-03],\n", - " [-3.5018e-02, 5.9904e-03, 2.6881e-02],\n", - " [ 2.7519e-02, 2.4498e-02, -2.9707e-02]],\n", - " \n", - " [[-1.1786e-03, -3.3742e-02, 1.5864e-02],\n", - " [ 1.7007e-02, -1.7952e-02, -2.8105e-02],\n", - " [ 3.7245e-02, -1.4320e-02, 3.3163e-02]],\n", - " \n", - " [[-4.0185e-02, -2.3607e-02, -4.6880e-04],\n", - " [ 3.4657e-02, -4.1312e-02, -4.4827e-02],\n", - " [-1.0059e-02, -2.4776e-03, 5.1147e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.1438e-02, -6.6742e-02, 2.2961e-02],\n", - " [ 2.5789e-02, -1.0198e-02, -5.3327e-02],\n", - " [ 3.0370e-03, -1.2007e-02, -9.1576e-03]],\n", - " \n", - " [[ 2.0191e-02, -1.2843e-02, 2.2009e-02],\n", - " [ 6.0755e-03, -2.4749e-02, 3.1961e-02],\n", - " [-2.1769e-03, 2.5303e-03, 4.7090e-02]],\n", - " \n", - " [[ 2.5646e-02, -3.6510e-02, -1.1055e-03],\n", - " [-1.7726e-02, 2.3315e-02, -2.3016e-03],\n", - " [-6.9395e-03, -1.9193e-02, 3.1284e-02]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer1.0.bn2.weight',\n", - " tensor([ 5.1260e-06, 2.6226e-06, -2.2054e-06, -9.5367e-07, -1.0729e-06,\n", - " -3.0994e-06, -8.9407e-07, -1.7881e-06, -1.7881e-06, -1.0133e-06,\n", - " -1.9670e-06, -2.6226e-06, 1.4305e-06, 2.1458e-06, 2.3842e-07,\n", - " -1.6093e-06, -5.9605e-07, 4.7684e-07, -6.5565e-07, -1.7881e-07,\n", - " -3.5763e-07, 2.3842e-07, -1.1921e-07, 7.1526e-07, 3.0994e-06,\n", - " 4.7684e-07, 2.3842e-07, 4.7684e-07, -5.9605e-07, 1.5497e-06,\n", - " 3.5763e-07, 1.1921e-06, 1.1921e-06, -8.3447e-07, 1.1921e-06,\n", - " 7.1526e-07, -4.1723e-07, 2.0266e-06, -1.4305e-06, -8.9407e-07,\n", - " -4.8876e-06, -2.4438e-06, -1.4901e-06, -4.7684e-07, -2.8014e-06,\n", - " -5.3644e-07, -1.8477e-06, 4.7684e-07, -2.0266e-06, -7.7486e-07,\n", - " 9.5367e-07, -5.9605e-08, 2.3842e-06, 7.1526e-07, -2.8014e-06,\n", - " 3.3379e-06, -1.0133e-06, 2.3842e-07, 6.5565e-06, 3.5763e-06,\n", - " -9.5367e-07, 1.5497e-06, 1.6689e-06, 8.3447e-07], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer1.0.bn2.bias',\n", - " tensor([ 1.9615e-06, -1.7051e-06, -9.6066e-07, 2.9261e-07, 1.5350e-06,\n", - " -2.4253e-06, 1.4455e-06, -2.3713e-07, -1.0793e-06, -2.7920e-07,\n", - " 6.6946e-10, -1.2152e-07, -1.4768e-06, -1.2127e-07, 1.6777e-07,\n", - " -1.6548e-06, 1.6779e-07, 1.3144e-06, -3.9356e-07, 2.3618e-06,\n", - " 1.1459e-06, 7.5798e-08, -1.4052e-06, 1.9464e-06, 1.8920e-06,\n", - " 1.4963e-06, 7.9997e-07, 1.2816e-06, 1.0315e-06, 1.0541e-06,\n", - " 8.0646e-07, -1.6433e-07, 1.1502e-06, -2.8317e-07, -8.3817e-07,\n", - " -1.4362e-06, -3.3804e-06, -7.9363e-07, -2.2784e-06, -3.2797e-06,\n", - " 3.3910e-07, -1.8120e-06, -6.6409e-07, 1.1106e-06, 3.8630e-07,\n", - " -1.6407e-06, 6.6217e-07, -1.1658e-06, 1.1164e-06, -9.5826e-07,\n", - " -1.5532e-06, 1.0813e-06, -7.4805e-07, 2.6794e-06, -2.8860e-06,\n", - " 1.8326e-07, 6.5250e-07, 1.1221e-06, 6.3122e-07, 2.8292e-06,\n", - " 1.5905e-06, 2.7533e-06, 1.6429e-06, 2.0841e-06], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.conv1.weight',\n", - " tensor([[[[ 3.1222e-02, -2.0293e-02, -2.2732e-02],\n", - " [ 5.1991e-02, -5.2411e-02, -7.1547e-03],\n", - " [-1.3944e-02, -1.4512e-02, 1.5920e-02]],\n", - " \n", - " [[ 1.4411e-03, 2.3664e-02, 5.2533e-02],\n", - " [ 4.9313e-03, 2.8384e-02, 4.0038e-02],\n", - " [ 1.3472e-02, 2.9390e-02, -1.8044e-03]],\n", - " \n", - " [[-2.5809e-02, 2.3759e-02, 3.2983e-03],\n", - " [ 1.6743e-02, -2.7121e-02, 2.0905e-02],\n", - " [-4.8166e-03, 2.8307e-02, 5.3652e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.1179e-02, 1.3626e-02, -3.3159e-02],\n", - " [-3.0455e-02, 7.1659e-03, -2.3647e-02],\n", - " [ 1.5195e-02, -2.5908e-02, 7.0486e-03]],\n", - " \n", - " [[-2.9581e-02, 6.3754e-03, 4.0771e-02],\n", - " [-2.9581e-02, 4.5928e-02, -5.0159e-03],\n", - " [-3.1577e-02, 1.8542e-02, -2.3790e-02]],\n", - " \n", - " [[ 7.9386e-03, -5.5335e-02, -2.3499e-02],\n", - " [ 5.1797e-02, -1.9768e-02, 2.6371e-02],\n", - " [ 2.8223e-02, -3.6843e-03, -2.4914e-02]]],\n", - " \n", - " \n", - " [[[ 3.0091e-02, -1.5946e-02, -2.0017e-02],\n", - " [-7.3625e-03, -1.2496e-02, -1.5988e-02],\n", - " [-3.9418e-03, -1.5774e-02, 3.1421e-02]],\n", - " \n", - " [[-2.3044e-02, 2.6058e-02, 4.0771e-02],\n", - " [ 2.8217e-02, -1.3101e-02, -1.9169e-02],\n", - " [ 1.6696e-02, -2.9469e-02, 3.0725e-02]],\n", - " \n", - " [[-1.7412e-02, 5.9761e-03, 1.4802e-02],\n", - " [-9.6282e-03, -2.5679e-02, -3.2987e-02],\n", - " [-2.8646e-02, 2.0080e-02, -1.1815e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.5162e-02, 3.7288e-02, -5.4347e-02],\n", - " [-4.5577e-03, 8.9547e-05, 1.2762e-02],\n", - " [ 2.4004e-02, 1.5974e-02, -3.9689e-03]],\n", - " \n", - " [[ 2.0893e-02, 3.8545e-02, -4.7115e-04],\n", - " [ 2.0476e-02, -5.5991e-02, 2.3132e-02],\n", - " [-8.4840e-03, 4.0287e-03, -4.3514e-02]],\n", - " \n", - " [[ 4.1755e-03, 3.6891e-02, 2.2836e-02],\n", - " [ 1.5228e-02, -3.1077e-02, -4.8275e-03],\n", - " [ 3.4390e-02, -2.4285e-03, -1.6479e-02]]],\n", - " \n", - " \n", - " [[[ 3.1451e-02, 2.7231e-02, -4.8490e-02],\n", - " [-2.9251e-02, 7.3146e-03, -8.5553e-03],\n", - " [-3.0377e-02, 5.3074e-02, -3.5402e-02]],\n", - " \n", - " [[ 1.6317e-03, 2.9512e-02, 3.7927e-02],\n", - " [ 3.9694e-03, -1.0264e-02, 2.0400e-02],\n", - " [-5.0912e-03, 1.0976e-02, 2.0343e-02]],\n", - " \n", - " [[ 3.3715e-02, 1.8947e-02, 3.7091e-02],\n", - " [ 4.4463e-04, 1.9399e-02, 3.0023e-02],\n", - " [ 4.5544e-02, -2.2597e-02, -3.3840e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 5.0087e-02, 2.4138e-02, -1.6329e-02],\n", - " [-4.9508e-03, 1.4488e-02, -1.2929e-02],\n", - " [-2.0144e-02, 2.3250e-02, -4.7392e-03]],\n", - " \n", - " [[ 3.2129e-02, 2.7853e-02, 1.8916e-02],\n", - " [-1.9839e-02, -4.3275e-02, -2.0834e-03],\n", - " [-1.0927e-02, 4.9429e-03, 1.1458e-02]],\n", - " \n", - " [[-4.0182e-02, 2.0645e-02, 1.3425e-02],\n", - " [ 1.5748e-02, -2.5606e-02, 2.1066e-02],\n", - " [-2.5730e-02, -3.5284e-02, 4.0417e-02]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 3.2400e-02, 2.7387e-02, -4.1836e-02],\n", - " [-3.3519e-02, -3.0823e-02, -3.9949e-02],\n", - " [ 2.9098e-02, 2.3158e-02, -2.2002e-02]],\n", - " \n", - " [[-4.4127e-02, -5.0314e-02, 5.4241e-02],\n", - " [ 4.1223e-02, 2.0904e-02, 2.4865e-02],\n", - " [-7.3921e-03, -2.4277e-02, -3.4978e-02]],\n", - " \n", - " [[ 2.1406e-02, -3.5557e-02, -1.4675e-02],\n", - " [ 1.6927e-03, -2.0588e-02, 2.9574e-03],\n", - " [ 5.3911e-03, 3.2478e-03, -1.5555e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.8776e-02, -2.9488e-02, -2.7173e-03],\n", - " [-1.1154e-02, -1.5518e-02, -1.3146e-02],\n", - " [ 2.7336e-02, -2.0737e-03, -1.8577e-03]],\n", - " \n", - " [[ 2.7353e-02, -5.7622e-02, -1.9810e-02],\n", - " [ 4.4887e-02, -2.4649e-02, -2.8831e-04],\n", - " [-1.7285e-02, 4.4593e-02, 5.0801e-03]],\n", - " \n", - " [[-1.0272e-02, -1.1177e-02, 1.9091e-02],\n", - " [ 2.6784e-02, -1.6878e-02, -2.2759e-02],\n", - " [-3.9895e-02, 1.9132e-02, 3.3437e-02]]],\n", - " \n", - " \n", - " [[[ 2.4377e-03, -3.6880e-02, 8.9704e-03],\n", - " [ 1.8340e-02, -3.8695e-02, -2.9911e-02],\n", - " [ 2.7198e-02, -2.6570e-02, -1.1830e-02]],\n", - " \n", - " [[-3.9071e-03, -9.8153e-03, -4.5207e-02],\n", - " [ 1.4953e-02, -2.0952e-02, 1.6540e-03],\n", - " [ 3.1941e-03, -1.7849e-02, 3.0970e-02]],\n", - " \n", - " [[-5.4628e-03, -1.0248e-02, 1.8554e-02],\n", - " [ 1.4621e-02, 3.0034e-02, -4.1184e-02],\n", - " [-3.1914e-02, -1.6138e-03, 3.5175e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.1999e-02, 3.7370e-02, -6.6605e-03],\n", - " [ 8.5423e-03, -2.3363e-02, 5.3106e-02],\n", - " [-3.8766e-02, -4.6355e-02, -2.1916e-02]],\n", - " \n", - " [[ 4.6545e-02, -1.3589e-02, -9.0086e-04],\n", - " [-2.2744e-02, -7.9690e-03, 4.1620e-02],\n", - " [-1.8275e-02, -1.1720e-02, -1.7068e-02]],\n", - " \n", - " [[ 4.2182e-03, -5.2635e-02, 3.7010e-02],\n", - " [-1.2204e-02, 1.6322e-02, 2.7914e-02],\n", - " [-1.5614e-03, 1.2287e-02, -3.1604e-02]]],\n", - " \n", - " \n", - " [[[ 3.3208e-02, 6.6656e-03, 8.7341e-03],\n", - " [ 1.7176e-02, -2.1905e-02, 3.8850e-02],\n", - " [-2.0100e-02, 1.8937e-03, 5.5718e-04]],\n", - " \n", - " [[-5.3009e-02, 2.7671e-03, -2.5455e-02],\n", - " [-4.1016e-02, 4.6348e-02, 1.0834e-02],\n", - " [-4.8903e-02, 4.2398e-02, 2.0632e-02]],\n", - " \n", - " [[-2.7099e-02, -4.9743e-03, -1.6784e-02],\n", - " [ 1.2828e-02, -6.0318e-04, -4.7372e-02],\n", - " [-3.3223e-02, -5.8787e-02, -6.7474e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 6.1073e-03, -3.3571e-02, 1.1209e-02],\n", - " [-3.4866e-02, -3.0714e-02, 2.1105e-03],\n", - " [ 5.3477e-02, -2.4217e-02, -2.5762e-02]],\n", - " \n", - " [[ 2.4611e-02, -1.8060e-02, -2.9728e-02],\n", - " [ 3.6908e-02, -2.9815e-02, 3.9181e-02],\n", - " [-4.4588e-02, 4.1345e-02, -1.4445e-02]],\n", - " \n", - " [[ 1.0095e-02, 4.4746e-02, -3.2337e-02],\n", - " [-8.9715e-03, -2.4242e-02, -1.0168e-02],\n", - " [ 3.7602e-02, 1.3277e-02, 4.1139e-02]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.bn1.weight',\n", - " tensor([-5.9605e-08, 2.3842e-07, 5.9605e-07, -1.0133e-06, -9.5367e-07,\n", - " 2.2650e-06, -3.5763e-07, 4.7684e-07, 8.3447e-07, -5.9605e-07,\n", - " -4.7684e-07, -9.5367e-07, -2.9802e-07, -1.7881e-07, -5.9605e-08,\n", - " 8.3447e-07, -8.3447e-07, 1.1921e-06, -2.3842e-07, -2.9802e-07,\n", - " 2.6226e-06, -7.7486e-07, -5.9605e-07, 1.1921e-06, 9.5367e-07,\n", - " -5.9605e-07, 7.1526e-07, 3.5763e-07, -9.5367e-07, 4.7684e-07,\n", - " -3.5763e-07, 9.5367e-07, 1.3113e-06, -2.9802e-07, -1.3709e-06,\n", - " -1.4305e-06, 1.5497e-06, 0.0000e+00, -1.7285e-06, 4.7684e-07,\n", - " -5.9605e-07, 7.1526e-07, 1.3113e-06, 5.9605e-07, -8.9407e-07,\n", - " -4.1723e-07, -1.1325e-06, 1.1921e-07, -6.5565e-07, -1.4901e-06,\n", - " 5.9605e-07, -5.3644e-07, -5.9605e-08, 3.5763e-07, 2.3842e-06,\n", - " 4.7684e-07, -7.1526e-07, 1.1921e-06, -1.3709e-06, 1.4305e-06,\n", - " -4.7684e-07, -1.0133e-06, 2.2650e-06, -3.5763e-07, -6.5565e-07,\n", - " -4.7684e-07, -7.7486e-07, 1.3113e-06, -5.9605e-08, -8.3447e-07,\n", - " 5.9605e-07, -1.9670e-06, 5.9605e-07, 1.7881e-06, -6.5565e-07,\n", - " 4.7684e-07, 2.1458e-06, 8.3447e-07, 9.5367e-07, 0.0000e+00,\n", - " -3.5763e-07, -5.9605e-07, 5.9605e-07, 1.4305e-06, -1.1325e-06,\n", - " -5.3644e-07, 4.7684e-07, 1.0729e-06, -1.0729e-06, -2.3842e-07,\n", - " -2.9802e-07, -6.5565e-07, 1.3113e-06, -6.5565e-07, 2.3842e-07,\n", - " 4.7684e-07, -8.3447e-07, 9.5367e-07, 1.1921e-07, -1.1921e-07,\n", - " -5.9605e-07, -1.1325e-06, 1.1921e-07, -1.1325e-06, 5.9605e-07,\n", - " -5.3644e-07, 2.3842e-07, 5.9605e-07, 8.3447e-07, 1.1921e-07,\n", - " 2.3842e-07, -7.1526e-07, -2.8014e-06, 0.0000e+00, -1.6093e-06,\n", - " -4.1723e-07, -1.1921e-07, -9.5367e-07, 5.9605e-07, -4.7684e-07,\n", - " 4.7684e-07, -1.1921e-07, 1.1921e-07, 2.3842e-07, -1.3709e-06,\n", - " -9.5367e-07, 2.3842e-07, 1.1921e-07], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.bn1.bias',\n", - " tensor([-2.1467e-07, -6.6178e-07, 1.7024e-07, 2.0613e-07, -2.4169e-07,\n", - " 1.9816e-06, -1.1832e-06, -1.5366e-07, 5.6462e-07, 1.5284e-08,\n", - " 4.0878e-07, -4.9318e-07, -5.0621e-07, -6.0479e-07, 1.7576e-06,\n", - " 8.4282e-07, -5.4288e-07, 5.3258e-07, 5.2341e-07, 5.0324e-07,\n", - " 1.7585e-06, -8.1943e-07, 6.5351e-07, 5.5759e-07, -1.0581e-06,\n", - " 9.2307e-07, 9.9461e-08, 1.4788e-07, -1.1720e-06, -1.6742e-07,\n", - " 8.3141e-07, 2.1564e-07, -7.3767e-07, 5.7207e-07, -3.0203e-07,\n", - " -1.9044e-06, 4.9593e-08, 1.4940e-06, 3.8589e-08, 9.1724e-08,\n", - " -5.0841e-07, 8.4666e-07, 1.4527e-06, 1.0706e-06, -1.6604e-06,\n", - " -1.1521e-07, -5.2608e-07, 1.7766e-06, 5.5275e-07, -1.6253e-06,\n", - " 4.1272e-07, -9.8920e-07, 5.8334e-07, 1.0732e-06, 1.8116e-06,\n", - " 1.1440e-06, -1.2197e-06, 3.4906e-07, -6.0978e-07, 8.8430e-07,\n", - " -6.6463e-07, -9.4432e-07, 1.4670e-06, -1.3969e-06, -6.5653e-07,\n", - " 7.5088e-07, -1.4769e-06, -4.7031e-07, -3.6269e-07, -2.9390e-08,\n", - " 1.1469e-07, -5.4929e-07, 6.8075e-07, 8.0234e-07, -4.5739e-07,\n", - " -4.1138e-07, 1.3506e-06, 1.2868e-06, -1.2592e-07, -9.3637e-07,\n", - " -7.6653e-07, -1.9897e-07, 6.9171e-07, 3.0005e-08, -2.2339e-06,\n", - " -1.5658e-07, -1.1111e-06, 1.6107e-06, 1.1992e-06, 8.4681e-07,\n", - " 1.3424e-06, 1.3030e-06, 7.1811e-07, 7.8723e-07, 4.0089e-07,\n", - " 5.9138e-07, -8.1441e-07, 7.5653e-07, -2.5908e-07, 4.2406e-07,\n", - " -3.7178e-07, -5.6875e-07, -8.2396e-07, -1.0907e-06, -3.1799e-07,\n", - " -5.7387e-07, 4.5297e-07, 1.1021e-06, 1.4436e-06, -2.8676e-07,\n", - " 4.7926e-08, 7.7351e-07, -1.5734e-06, -2.5014e-07, -9.6145e-07,\n", - " 1.0515e-06, -4.0971e-07, -9.0253e-07, 4.4345e-07, -1.4786e-07,\n", - " -6.9746e-07, 5.1135e-07, -2.9096e-07, -6.8238e-08, -1.8120e-06,\n", - " 8.7449e-07, 4.9817e-07, 1.2301e-06], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.conv2.weight',\n", - " tensor([[[[ 0.0061, 0.0248, 0.0266],\n", - " [-0.0154, -0.0017, -0.0064],\n", - " [ 0.0182, -0.0421, -0.0336]],\n", - " \n", - " [[ 0.0341, -0.0028, -0.0326],\n", - " [-0.0029, -0.0011, -0.0415],\n", - " [-0.0231, -0.0174, -0.0123]],\n", - " \n", - " [[-0.0016, -0.0241, -0.0310],\n", - " [-0.0170, 0.0123, -0.0002],\n", - " [ 0.0171, 0.0016, -0.0373]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0090, 0.0006, 0.0137],\n", - " [-0.0105, -0.0244, 0.0183],\n", - " [-0.0004, 0.0322, 0.0156]],\n", - " \n", - " [[-0.0249, -0.0007, -0.0213],\n", - " [ 0.0010, -0.0180, 0.0122],\n", - " [-0.0211, 0.0167, -0.0023]],\n", - " \n", - " [[-0.0242, 0.0292, 0.0299],\n", - " [ 0.0119, 0.0187, 0.0037],\n", - " [ 0.0297, 0.0022, 0.0109]]],\n", - " \n", - " \n", - " [[[-0.0269, -0.0043, 0.0191],\n", - " [-0.0209, -0.0125, -0.0290],\n", - " [ 0.0195, -0.0040, 0.0271]],\n", - " \n", - " [[-0.0198, -0.0021, -0.0042],\n", - " [-0.0018, 0.0042, -0.0213],\n", - " [-0.0078, -0.0174, -0.0072]],\n", - " \n", - " [[ 0.0318, 0.0153, 0.0026],\n", - " [-0.0350, 0.0107, 0.0009],\n", - " [-0.0130, 0.0201, 0.0011]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0102, -0.0144, -0.0065],\n", - " [-0.0047, 0.0310, -0.0157],\n", - " [ 0.0013, 0.0064, 0.0090]],\n", - " \n", - " [[-0.0095, 0.0098, 0.0033],\n", - " [ 0.0423, 0.0246, 0.0002],\n", - " [-0.0016, -0.0120, 0.0049]],\n", - " \n", - " [[-0.0198, 0.0237, -0.0067],\n", - " [-0.0096, 0.0138, 0.0137],\n", - " [ 0.0183, 0.0119, 0.0224]]],\n", - " \n", - " \n", - " [[[ 0.0297, 0.0352, -0.0161],\n", - " [-0.0172, -0.0186, -0.0146],\n", - " [-0.0212, 0.0170, 0.0156]],\n", - " \n", - " [[ 0.0327, 0.0215, 0.0073],\n", - " [-0.0230, 0.0068, 0.0204],\n", - " [-0.0469, -0.0293, -0.0211]],\n", - " \n", - " [[-0.0200, -0.0304, -0.0019],\n", - " [-0.0006, 0.0144, 0.0267],\n", - " [ 0.0174, -0.0108, -0.0253]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0021, -0.0095, -0.0195],\n", - " [ 0.0152, 0.0130, -0.0070],\n", - " [ 0.0251, 0.0134, 0.0258]],\n", - " \n", - " [[ 0.0183, -0.0031, 0.0047],\n", - " [ 0.0072, -0.0249, -0.0123],\n", - " [ 0.0281, -0.0248, -0.0010]],\n", - " \n", - " [[ 0.0252, 0.0180, -0.0235],\n", - " [-0.0014, 0.0064, -0.0137],\n", - " [ 0.0230, 0.0008, -0.0191]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 0.0282, -0.0075, 0.0432],\n", - " [-0.0325, -0.0211, -0.0156],\n", - " [-0.0047, -0.0331, -0.0143]],\n", - " \n", - " [[-0.0109, 0.0056, -0.0271],\n", - " [ 0.0308, 0.0146, 0.0129],\n", - " [-0.0129, 0.0256, 0.0154]],\n", - " \n", - " [[-0.0321, 0.0118, 0.0130],\n", - " [-0.0057, -0.0097, 0.0054],\n", - " [ 0.0159, 0.0210, 0.0006]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0046, 0.0206, 0.0274],\n", - " [ 0.0140, 0.0397, 0.0243],\n", - " [-0.0147, -0.0160, 0.0216]],\n", - " \n", - " [[ 0.0203, -0.0391, 0.0018],\n", - " [ 0.0050, 0.0139, -0.0256],\n", - " [ 0.0157, 0.0140, 0.0029]],\n", - " \n", - " [[-0.0159, -0.0278, 0.0334],\n", - " [ 0.0070, -0.0179, 0.0059],\n", - " [-0.0117, 0.0188, -0.0005]]],\n", - " \n", - " \n", - " [[[ 0.0179, -0.0143, 0.0172],\n", - " [ 0.0228, 0.0128, -0.0189],\n", - " [-0.0170, -0.0010, 0.0167]],\n", - " \n", - " [[ 0.0291, -0.0274, -0.0327],\n", - " [ 0.0033, -0.0214, 0.0099],\n", - " [ 0.0255, -0.0062, 0.0171]],\n", - " \n", - " [[-0.0347, -0.0177, 0.0207],\n", - " [-0.0179, 0.0280, -0.0178],\n", - " [ 0.0211, 0.0030, -0.0068]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0055, 0.0100, -0.0051],\n", - " [ 0.0262, -0.0121, 0.0355],\n", - " [-0.0250, 0.0059, -0.0248]],\n", - " \n", - " [[ 0.0187, 0.0238, 0.0206],\n", - " [-0.0002, 0.0277, 0.0077],\n", - " [ 0.0068, 0.0034, -0.0409]],\n", - " \n", - " [[-0.0030, 0.0121, 0.0156],\n", - " [ 0.0125, -0.0119, 0.0086],\n", - " [-0.0004, 0.0258, -0.0002]]],\n", - " \n", - " \n", - " [[[ 0.0172, -0.0334, -0.0074],\n", - " [-0.0184, 0.0094, -0.0019],\n", - " [ 0.0407, 0.0041, 0.0084]],\n", - " \n", - " [[ 0.0339, 0.0150, -0.0341],\n", - " [-0.0116, 0.0231, 0.0189],\n", - " [ 0.0049, 0.0024, 0.0197]],\n", - " \n", - " [[ 0.0343, -0.0007, 0.0028],\n", - " [ 0.0260, -0.0005, -0.0192],\n", - " [-0.0197, -0.0036, -0.0064]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0159, 0.0336, 0.0037],\n", - " [-0.0347, -0.0117, -0.0147],\n", - " [-0.0152, 0.0203, -0.0101]],\n", - " \n", - " [[-0.0228, 0.0130, 0.0209],\n", - " [ 0.0134, -0.0121, 0.0251],\n", - " [ 0.0195, 0.0150, 0.0038]],\n", - " \n", - " [[ 0.0225, 0.0109, -0.0084],\n", - " [-0.0076, 0.0192, -0.0155],\n", - " [-0.0044, -0.0217, -0.0129]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.bn2.weight',\n", - " tensor([-2.9802e-07, -9.5367e-07, -1.7285e-06, -5.9605e-07, -1.8477e-06,\n", - " 1.5497e-06, -1.7881e-07, 3.5763e-07, 7.1526e-07, 3.5763e-07,\n", - " 7.1526e-07, -1.3113e-06, -1.7881e-07, 1.9073e-06, -1.3113e-06,\n", - " 3.5763e-07, 4.7684e-07, 2.1458e-06, 5.9605e-07, 3.5763e-07,\n", - " 4.7684e-07, 7.1526e-07, -1.1921e-06, 3.5763e-07, -3.5763e-07,\n", - " 3.5763e-07, 2.3842e-07, -4.7684e-07, 2.3842e-06, 1.1921e-07,\n", - " 1.6689e-06, 1.0729e-06, -1.1921e-07, 1.5497e-06, 4.7684e-07,\n", - " 2.3842e-07, -9.5367e-07, 3.5763e-07, 9.5367e-07, 5.9605e-07,\n", - " 5.9605e-07, -7.1526e-07, 2.3842e-07, 1.1921e-07, -3.5763e-07,\n", - " -1.7881e-07, -1.6093e-06, -4.7684e-07, -2.3842e-06, -1.0729e-06,\n", - " -1.0729e-06, -5.3644e-07, 5.9605e-07, -4.7684e-07, -4.1723e-07,\n", - " -1.7285e-06, 2.3842e-07, 8.3447e-07, -8.9407e-07, 7.1526e-07,\n", - " -4.1723e-07, 7.1526e-07, -6.5565e-07, -3.2187e-06, 3.5763e-07,\n", - " -5.3644e-07, -1.1921e-07, -3.5763e-07, 7.1526e-07, 7.1526e-07,\n", - " 9.5367e-07, -1.9073e-06, 1.0729e-06, -8.3447e-07, -4.1723e-07,\n", - " 0.0000e+00, 7.1526e-07, 4.7684e-07, 3.5763e-07, 3.5763e-07,\n", - " -5.9605e-08, 1.1921e-07, -9.5367e-07, -5.9605e-07, 1.1921e-07,\n", - " 4.7684e-07, 8.3447e-07, -1.3113e-06, 2.2650e-06, 7.1526e-07,\n", - " 0.0000e+00, -5.9605e-07, 4.7684e-07, 3.5763e-07, -5.9605e-08,\n", - " -1.2517e-06, -1.1921e-07, 7.1526e-07, -3.5763e-07, 5.9605e-07,\n", - " -7.7486e-07, 3.2187e-06, 9.5367e-07, -5.9605e-08, -7.1526e-07,\n", - " -6.5565e-07, -7.1526e-07, 1.4305e-06, -2.9802e-07, 8.3447e-07,\n", - " 2.3842e-07, -4.7684e-07, 3.5763e-07, 1.0729e-06, -1.2517e-06,\n", - " -7.7486e-07, 2.5034e-06, -6.5565e-07, -1.7881e-07, -5.9605e-08,\n", - " -5.9605e-08, 2.3842e-07, -2.1458e-06, -1.8477e-06, -2.3246e-06,\n", - " 0.0000e+00, 2.3842e-07, 4.7684e-07], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.bn2.bias',\n", - " tensor([-5.5672e-07, -5.2195e-07, -7.3200e-07, -1.3146e-06, -5.1317e-07,\n", - " 2.8345e-07, 2.3253e-07, 1.2570e-06, 5.3647e-08, 5.4823e-09,\n", - " -1.7705e-07, -2.2771e-06, 4.4279e-08, -4.5075e-08, -7.2442e-07,\n", - " 1.7921e-07, 1.1754e-06, 7.2712e-07, -1.3447e-06, -4.6385e-07,\n", - " 2.7598e-07, 1.0195e-07, -4.0322e-07, 7.0501e-07, 3.4089e-07,\n", - " -4.2139e-07, -4.5342e-07, 1.1544e-06, 2.0046e-06, -2.1043e-07,\n", - " -1.6187e-06, 3.2673e-07, 6.9511e-07, -8.8326e-07, 8.7073e-08,\n", - " 6.0346e-07, -1.2989e-06, -3.6423e-07, -3.2090e-07, 3.3890e-07,\n", - " -2.5060e-07, -1.7383e-06, 5.6231e-07, 1.0725e-06, -4.4510e-07,\n", - " 6.6358e-07, -3.3527e-07, 4.8670e-07, -5.1463e-07, -4.9155e-07,\n", - " -5.0953e-07, 3.5141e-07, 4.6948e-07, 1.5160e-06, 2.2840e-07,\n", - " 4.2011e-07, -1.2933e-06, 7.4209e-07, -1.1028e-06, -1.8627e-07,\n", - " 2.3393e-07, -3.6756e-07, -8.0544e-07, -2.6689e-08, -5.6918e-07,\n", - " -1.3877e-06, 4.1389e-07, -2.4542e-07, 5.1264e-08, -5.3017e-07,\n", - " -1.1562e-06, -8.6156e-07, 1.1412e-06, 5.1820e-07, 8.3670e-07,\n", - " -2.1070e-07, -1.9332e-07, 4.2176e-07, 7.6960e-08, 4.6864e-07,\n", - " 7.6292e-07, 4.0845e-08, -7.3546e-07, 3.2210e-07, 3.4499e-07,\n", - " 4.2282e-07, 2.0984e-07, 4.4692e-07, 2.7307e-06, 8.0059e-07,\n", - " 4.5338e-07, -8.3362e-07, 4.6176e-07, -6.4901e-07, -4.8625e-07,\n", - " 8.6808e-07, -1.1363e-07, 8.4456e-07, 1.3166e-06, 9.8014e-07,\n", - " 2.2637e-07, 8.4249e-07, 2.0628e-07, -5.5369e-08, -1.6014e-06,\n", - " -3.6345e-07, 2.0868e-06, 1.5960e-06, 2.1796e-07, 7.7901e-07,\n", - " 4.0972e-07, -1.4152e-07, -4.3252e-07, 1.0869e-06, -1.2064e-06,\n", - " -2.5503e-07, 6.4234e-07, -2.7155e-06, -6.4880e-07, 6.0029e-07,\n", - " 4.6931e-07, 1.1417e-06, -7.8651e-07, -4.1475e-07, -7.0440e-07,\n", - " 1.5567e-06, -3.1585e-07, -1.4987e-08], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.shortcut.0.weight',\n", - " tensor([[[[ 0.0434]],\n", - " \n", - " [[-0.0352]],\n", - " \n", - " [[-0.0098]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.0883]],\n", - " \n", - " [[-0.0434]],\n", - " \n", - " [[ 0.0117]]],\n", - " \n", - " \n", - " [[[-0.0396]],\n", - " \n", - " [[-0.1196]],\n", - " \n", - " [[-0.0617]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.0264]],\n", - " \n", - " [[-0.0807]],\n", - " \n", - " [[-0.0648]]],\n", - " \n", - " \n", - " [[[ 0.1829]],\n", - " \n", - " [[ 0.0400]],\n", - " \n", - " [[-0.0604]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0866]],\n", - " \n", - " [[ 0.0344]],\n", - " \n", - " [[-0.0640]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-0.0823]],\n", - " \n", - " [[ 0.1039]],\n", - " \n", - " [[ 0.0969]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0864]],\n", - " \n", - " [[ 0.1064]],\n", - " \n", - " [[ 0.0088]]],\n", - " \n", - " \n", - " [[[ 0.0982]],\n", - " \n", - " [[ 0.1408]],\n", - " \n", - " [[-0.0697]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.1474]],\n", - " \n", - " [[-0.0310]],\n", - " \n", - " [[ 0.1249]]],\n", - " \n", - " \n", - " [[[-0.0877]],\n", - " \n", - " [[ 0.0198]],\n", - " \n", - " [[ 0.0845]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.0722]],\n", - " \n", - " [[ 0.0428]],\n", - " \n", - " [[-0.0519]]]], device='cuda:0', grad_fn=)),\n", - " ('layer2.0.shortcut.1.weight',\n", - " tensor([-5.9605e-08, 8.3447e-07, 8.3447e-07, -1.7285e-06, 4.7684e-07,\n", - " 4.7684e-07, 1.1921e-07, 1.4305e-06, -7.1526e-07, -1.6093e-06,\n", - " -1.1921e-07, -3.3975e-06, 2.3842e-07, -1.5497e-06, 1.1921e-07,\n", - " 4.7684e-07, 4.7684e-07, 4.7684e-07, -8.9407e-07, -1.3113e-06,\n", - " 2.3842e-07, -1.1921e-07, -4.7684e-07, -1.1921e-07, 1.0729e-06,\n", - " 0.0000e+00, 4.7684e-07, 2.0266e-06, 5.9605e-07, -4.7684e-07,\n", - " -2.6226e-06, 4.7684e-07, 4.7684e-07, -1.6093e-06, -3.5763e-07,\n", - " -2.1458e-06, -7.7486e-07, 1.0729e-06, -5.3644e-07, -2.9802e-07,\n", - " 4.7684e-07, -1.0729e-06, 2.3842e-07, 2.3842e-07, -6.5565e-07,\n", - " 1.7881e-06, 1.4305e-06, 5.9605e-07, 9.5367e-07, 8.3447e-07,\n", - " 7.1526e-07, 3.5763e-07, -1.1921e-07, 1.1921e-06, -6.5565e-07,\n", - " -1.1921e-07, -2.2054e-06, 1.1921e-06, 3.5763e-07, -6.5565e-07,\n", - " 0.0000e+00, -5.3644e-07, -5.3644e-07, 1.7881e-06, 1.1921e-07,\n", - " 2.3842e-07, -5.9605e-07, -7.7486e-07, -1.3113e-06, -1.2517e-06,\n", - " -1.1325e-06, -1.7881e-07, 7.1526e-07, 4.7684e-07, 8.3447e-07,\n", - " 5.9605e-07, -1.1921e-07, -7.7486e-07, -1.5497e-06, -5.9605e-07,\n", - " 1.3113e-06, 1.1921e-07, -1.1921e-07, 2.3842e-07, 0.0000e+00,\n", - " -7.1526e-07, -4.1723e-07, 1.1921e-07, 5.9605e-07, 1.4305e-06,\n", - " 1.5497e-06, -1.1325e-06, 1.1921e-06, -8.3447e-07, 9.5367e-07,\n", - " 3.5763e-07, 1.1921e-06, 1.6689e-06, 1.0729e-06, 1.0729e-06,\n", - " 1.5497e-06, 0.0000e+00, 4.7684e-07, -1.7881e-07, -1.7285e-06,\n", - " -6.5565e-07, 1.5497e-06, 1.1921e-07, 7.1526e-07, 2.3842e-07,\n", - " -1.1921e-07, 3.5763e-07, -6.5565e-07, 1.1921e-07, -9.5367e-07,\n", - " -3.5763e-07, -5.3644e-07, -2.2054e-06, 1.1921e-07, 8.3447e-07,\n", - " 9.5367e-07, -6.5565e-07, 0.0000e+00, 7.1526e-07, 9.5367e-07,\n", - " 9.5367e-07, -7.1526e-07, -1.1325e-06], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer2.0.shortcut.1.bias',\n", - " tensor([-5.5672e-07, -5.2195e-07, -7.3200e-07, -1.3146e-06, -5.1317e-07,\n", - " 2.8345e-07, 2.3253e-07, 1.2570e-06, 5.3647e-08, 5.4823e-09,\n", - " -1.7705e-07, -2.2771e-06, 4.4279e-08, -4.5075e-08, -7.2442e-07,\n", - " 1.7921e-07, 1.1754e-06, 7.2712e-07, -1.3447e-06, -4.6385e-07,\n", - " 2.7598e-07, 1.0195e-07, -4.0322e-07, 7.0501e-07, 3.4089e-07,\n", - " -4.2139e-07, -4.5342e-07, 1.1544e-06, 2.0046e-06, -2.1043e-07,\n", - " -1.6187e-06, 3.2673e-07, 6.9511e-07, -8.8326e-07, 8.7073e-08,\n", - " 6.0346e-07, -1.2989e-06, -3.6423e-07, -3.2090e-07, 3.3890e-07,\n", - " -2.5060e-07, -1.7383e-06, 5.6231e-07, 1.0725e-06, -4.4510e-07,\n", - " 6.6358e-07, -3.3527e-07, 4.8670e-07, -5.1463e-07, -4.9155e-07,\n", - " -5.0953e-07, 3.5141e-07, 4.6948e-07, 1.5160e-06, 2.2840e-07,\n", - " 4.2011e-07, -1.2933e-06, 7.4209e-07, -1.1028e-06, -1.8627e-07,\n", - " 2.3393e-07, -3.6756e-07, -8.0544e-07, -2.6689e-08, -5.6918e-07,\n", - " -1.3877e-06, 4.1389e-07, -2.4542e-07, 5.1264e-08, -5.3017e-07,\n", - " -1.1562e-06, -8.6156e-07, 1.1412e-06, 5.1820e-07, 8.3670e-07,\n", - " -2.1070e-07, -1.9332e-07, 4.2176e-07, 7.6960e-08, 4.6864e-07,\n", - " 7.6292e-07, 4.0845e-08, -7.3546e-07, 3.2210e-07, 3.4499e-07,\n", - " 4.2282e-07, 2.0984e-07, 4.4692e-07, 2.7307e-06, 8.0059e-07,\n", - " 4.5338e-07, -8.3362e-07, 4.6176e-07, -6.4901e-07, -4.8625e-07,\n", - " 8.6808e-07, -1.1363e-07, 8.4456e-07, 1.3166e-06, 9.8014e-07,\n", - " 2.2637e-07, 8.4249e-07, 2.0628e-07, -5.5369e-08, -1.6014e-06,\n", - " -3.6345e-07, 2.0868e-06, 1.5960e-06, 2.1796e-07, 7.7901e-07,\n", - " 4.0972e-07, -1.4152e-07, -4.3252e-07, 1.0869e-06, -1.2064e-06,\n", - " -2.5503e-07, 6.4234e-07, -2.7155e-06, -6.4880e-07, 6.0029e-07,\n", - " 4.6931e-07, 1.1417e-06, -7.8651e-07, -4.1475e-07, -7.0440e-07,\n", - " 1.5567e-06, -3.1585e-07, -1.4987e-08], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer3.0.conv1.weight',\n", - " tensor([[[[-3.5847e-02, 2.0367e-02, -1.9664e-02],\n", - " [ 2.8290e-02, -2.4266e-02, -3.1215e-02],\n", - " [-2.2953e-02, 1.5775e-02, 1.7100e-02]],\n", - " \n", - " [[-1.1806e-02, 1.9726e-02, 7.9145e-03],\n", - " [ 1.7040e-02, 2.5175e-02, 4.0052e-03],\n", - " [-1.6612e-02, -1.9356e-02, -2.1757e-03]],\n", - " \n", - " [[ 1.7233e-02, -2.8255e-03, 1.0205e-02],\n", - " [-3.3721e-02, -4.3411e-02, -7.9912e-03],\n", - " [ 2.8045e-02, 1.4699e-03, -3.7339e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.1966e-02, -2.5432e-02, -9.9482e-03],\n", - " [-1.7659e-02, 8.5453e-03, 2.9822e-02],\n", - " [-2.4194e-02, 9.3559e-03, -1.0396e-02]],\n", - " \n", - " [[-1.3652e-03, -3.2642e-03, -1.3523e-02],\n", - " [ 1.2295e-02, -4.4702e-03, 7.8963e-03],\n", - " [-7.2810e-03, 8.4824e-03, -2.1397e-02]],\n", - " \n", - " [[-4.8522e-03, 9.7668e-03, 2.1152e-02],\n", - " [-1.5838e-02, -2.8412e-02, -2.3352e-02],\n", - " [ 6.0608e-03, 1.9728e-03, 3.4929e-02]]],\n", - " \n", - " \n", - " [[[-1.5575e-02, -2.0735e-02, -1.5452e-04],\n", - " [-1.3825e-02, -1.6600e-02, -5.9313e-03],\n", - " [-9.0447e-03, -2.0076e-03, 1.5479e-02]],\n", - " \n", - " [[ 1.7221e-02, -7.0439e-03, 3.1504e-02],\n", - " [ 8.7690e-03, 7.2981e-03, -1.0521e-02],\n", - " [-3.9003e-02, 3.0051e-02, -1.5717e-02]],\n", - " \n", - " [[-1.7256e-02, -2.6381e-02, 4.3414e-03],\n", - " [-8.3730e-03, -9.9747e-04, -1.9681e-02],\n", - " [ 5.4823e-03, 2.9394e-03, 2.0712e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.4154e-03, -2.6488e-02, -6.9511e-03],\n", - " [-5.0294e-03, 1.2300e-02, 3.5730e-02],\n", - " [-2.6664e-02, -9.6863e-03, -3.2638e-02]],\n", - " \n", - " [[ 9.2684e-03, -1.0054e-02, 2.4518e-02],\n", - " [-2.5752e-02, -1.2214e-02, -2.8972e-02],\n", - " [ 1.0317e-02, 9.6287e-03, -1.0533e-02]],\n", - " \n", - " [[ 1.3457e-02, -3.5893e-03, -1.4567e-03],\n", - " [-3.4104e-04, -1.3895e-02, 1.2368e-02],\n", - " [-1.8869e-02, -8.0909e-03, 1.4769e-02]]],\n", - " \n", - " \n", - " [[[ 2.2298e-02, -9.7045e-03, -1.3930e-02],\n", - " [ 2.9282e-03, 2.1107e-02, -8.0453e-03],\n", - " [ 2.2023e-02, 1.5977e-02, -1.6384e-02]],\n", - " \n", - " [[ 1.2560e-02, -8.2418e-03, -2.8852e-02],\n", - " [-1.1178e-02, -1.7865e-02, 3.0926e-02],\n", - " [-2.5572e-02, 2.7698e-02, 1.7171e-02]],\n", - " \n", - " [[-1.1127e-02, -1.9849e-02, -2.7101e-02],\n", - " [-1.0094e-02, -7.5767e-04, -2.7074e-05],\n", - " [ 1.2436e-02, -4.1406e-02, 7.5492e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.6541e-02, 3.7938e-02, -7.3941e-03],\n", - " [-2.7031e-02, -3.6088e-02, 2.0318e-02],\n", - " [ 1.1700e-02, -1.7650e-02, -4.3927e-03]],\n", - " \n", - " [[-3.3291e-02, -1.3972e-02, 4.2048e-02],\n", - " [-6.9396e-03, -6.5420e-04, -2.3174e-02],\n", - " [-1.5313e-02, -2.7187e-02, 1.1052e-02]],\n", - " \n", - " [[-2.7701e-02, 1.3175e-02, 3.0270e-02],\n", - " [-1.7060e-03, -1.9307e-02, 6.8763e-03],\n", - " [ 9.6039e-04, -1.3544e-02, -2.7310e-02]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 2.7231e-03, 2.1136e-02, 1.9296e-02],\n", - " [ 4.1838e-02, -1.8260e-02, -9.6020e-03],\n", - " [ 2.2156e-03, 5.0148e-04, 2.9776e-02]],\n", - " \n", - " [[ 2.8787e-02, -1.0886e-02, -1.5779e-02],\n", - " [-1.9401e-03, 3.2563e-02, 2.4798e-02],\n", - " [ 3.6458e-02, 2.2999e-02, 1.1850e-02]],\n", - " \n", - " [[-2.3522e-02, 1.5405e-02, 1.8846e-02],\n", - " [ 3.0848e-02, -2.2616e-02, -3.8844e-02],\n", - " [ 3.5323e-03, 2.6513e-02, 2.0811e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.2667e-03, -1.7920e-02, 9.5128e-03],\n", - " [-5.5903e-03, 1.9465e-03, 2.2540e-02],\n", - " [-9.2771e-03, -2.9740e-02, 1.1990e-02]],\n", - " \n", - " [[-1.9649e-02, -1.6388e-02, 9.8337e-03],\n", - " [ 2.1826e-02, -1.1939e-02, -2.0516e-03],\n", - " [ 2.5273e-02, 1.4294e-02, 2.3242e-03]],\n", - " \n", - " [[ 2.6544e-03, -1.2401e-03, -5.3836e-03],\n", - " [-1.0099e-02, -7.2049e-03, -3.8413e-02],\n", - " [ 8.2304e-03, -1.8288e-02, -2.1143e-02]]],\n", - " \n", - " \n", - " [[[-3.7608e-02, 1.7813e-02, 3.2610e-02],\n", - " [-1.3644e-02, -1.5045e-02, -1.8056e-02],\n", - " [ 2.1015e-02, 2.5689e-02, -1.7453e-02]],\n", - " \n", - " [[-3.7034e-03, -1.6591e-02, 1.2074e-02],\n", - " [ 2.5513e-03, 1.6834e-02, -8.4531e-03],\n", - " [ 2.7908e-03, 2.4256e-02, 1.1538e-02]],\n", - " \n", - " [[ 2.4028e-02, -9.6117e-03, 9.3481e-03],\n", - " [-1.5518e-02, -2.1593e-02, 7.6084e-03],\n", - " [-1.1305e-02, 2.6390e-03, -2.6524e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.0536e-02, -1.1165e-02, -3.6322e-02],\n", - " [-8.2758e-03, 2.4712e-02, -5.8127e-03],\n", - " [-2.0468e-04, -1.3327e-02, -2.2236e-02]],\n", - " \n", - " [[-1.1215e-02, -3.1569e-03, -2.3264e-02],\n", - " [-2.2216e-02, 2.6291e-02, 1.5805e-02],\n", - " [ 1.9801e-02, 6.9040e-03, -1.2441e-02]],\n", - " \n", - " [[ 5.4119e-03, -2.0331e-02, -1.8075e-02],\n", - " [ 1.8203e-02, -1.1281e-02, 1.0496e-02],\n", - " [ 4.4981e-03, 2.6457e-02, 1.5682e-02]]],\n", - " \n", - " \n", - " [[[ 6.7285e-03, -1.0132e-02, 2.6872e-03],\n", - " [-3.8748e-03, 2.0391e-02, 1.6657e-02],\n", - " [-1.4085e-02, -3.3888e-02, -1.0991e-02]],\n", - " \n", - " [[-1.2712e-02, -3.6912e-02, 2.5412e-03],\n", - " [ 3.5808e-03, -2.4707e-02, 1.4955e-02],\n", - " [-1.0587e-02, 2.8549e-03, 1.1967e-02]],\n", - " \n", - " [[ 2.8115e-02, 1.6836e-02, -2.1065e-02],\n", - " [-9.3984e-03, -2.6535e-02, -6.5040e-03],\n", - " [-1.5094e-02, -7.1796e-03, 1.4032e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.6607e-02, -2.3249e-02, 2.2395e-02],\n", - " [ 1.1159e-02, -2.7459e-03, 6.7371e-03],\n", - " [ 1.3695e-02, -2.3425e-02, 1.2643e-02]],\n", - " \n", - " [[-2.2071e-02, 1.2437e-02, 1.6717e-02],\n", - " [-2.1038e-03, -3.6188e-03, 2.1535e-02],\n", - " [-4.1022e-02, -1.8896e-02, -9.2582e-03]],\n", - " \n", - " [[-2.1602e-02, -1.3935e-02, -1.4913e-02],\n", - " [-3.8020e-03, 1.6783e-02, -4.2063e-02],\n", - " [ 3.1403e-02, -9.7103e-04, -2.7294e-03]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer3.0.bn1.weight',\n", - " tensor([ 1.1921e-07, 3.5763e-07, 8.3447e-07, 3.5763e-07, 3.5763e-07,\n", - " -2.3842e-07, -8.9407e-07, 1.1921e-07, -8.9407e-07, 4.7684e-07,\n", - " 1.1921e-06, 1.1921e-07, -3.5763e-07, 3.5763e-07, -8.9407e-07,\n", - " 0.0000e+00, -5.9605e-07, -2.3842e-07, 5.9605e-07, 5.9605e-07,\n", - " 1.4305e-06, -5.9605e-07, -1.1921e-07, 1.1921e-07, 9.5367e-07,\n", - " 1.1921e-07, 5.9605e-07, 4.7684e-07, -2.3842e-07, 1.1921e-07,\n", - " 9.5367e-07, -2.9802e-07, -1.7881e-07, 0.0000e+00, -2.9802e-07,\n", - " -6.5565e-07, -5.9605e-07, -1.1921e-07, -1.1921e-07, -3.5763e-07,\n", - " -6.5565e-07, -2.3842e-07, 5.9605e-07, 1.1921e-07, 0.0000e+00,\n", - " -7.7486e-07, 7.1526e-07, 2.3842e-07, 5.9605e-07, 0.0000e+00,\n", - " 0.0000e+00, -4.1723e-07, -8.9407e-07, -1.1921e-07, 3.5763e-07,\n", - " -5.9605e-08, -5.9605e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", - " 3.5763e-07, -1.1921e-07, 3.5763e-07, -3.5763e-07, 2.3842e-07,\n", - " -7.7486e-07, 1.3113e-06, -7.7486e-07, -1.7881e-07, -6.5565e-07,\n", - " 1.1921e-07, -1.7881e-07, 3.5763e-07, 1.1921e-07, -2.3842e-07,\n", - " -1.2517e-06, 5.9605e-07, -1.1921e-07, -1.3113e-06, -1.0133e-06,\n", - " -1.7881e-07, -5.3644e-07, 1.0729e-06, -2.9802e-07, -8.3447e-07,\n", - " 2.3842e-07, -2.3842e-07, 8.3447e-07, 2.3842e-07, -7.1526e-07,\n", - " 3.5763e-07, -2.3842e-07, -2.9802e-07, -1.1921e-06, 5.9605e-07,\n", - " 8.3447e-07, -9.5367e-07, 9.5367e-07, 2.3842e-07, -1.2517e-06,\n", - " -8.3447e-07, -5.9605e-07, 1.9073e-06, 1.1921e-07, 2.3842e-07,\n", - " -5.9605e-08, -3.5763e-07, -1.2517e-06, 2.3842e-07, 1.1921e-06,\n", - " -2.3842e-07, 3.5763e-07, 8.3447e-07, 3.5763e-07, -1.1921e-07,\n", - " -4.7684e-07, 1.1921e-06, 3.5763e-07, -1.7881e-06, 8.3447e-07,\n", - " 3.5763e-07, -4.7684e-07, 4.7684e-07, 8.3447e-07, -4.7684e-07,\n", - " -2.3842e-07, -3.5763e-07, 1.1921e-07, -4.1723e-07, -6.5565e-07,\n", - " -1.1325e-06, 7.1526e-07, 1.0729e-06, -9.5367e-07, 2.3842e-07,\n", - " 1.4305e-06, 1.0729e-06, 1.1921e-07, 0.0000e+00, 9.5367e-07,\n", - " 4.7684e-07, -5.9605e-08, 7.1526e-07, -1.0729e-06, -4.1723e-07,\n", - " -6.5565e-07, -7.7486e-07, -1.7881e-07, 8.3447e-07, 4.7684e-07,\n", - " -1.1921e-07, -7.1526e-07, 0.0000e+00, -4.7684e-07, -2.9802e-07,\n", - " -2.9802e-07, -5.3644e-07, 7.1526e-07, 2.3842e-07, -6.5565e-07,\n", - " -7.7486e-07, -1.1921e-07, -2.3842e-07, -7.7486e-07, -1.1921e-07,\n", - " 0.0000e+00, 2.3842e-07, 1.1921e-07, 4.7684e-07, -1.5497e-06,\n", - " -4.7684e-07, 3.5763e-07, 1.4305e-06, 0.0000e+00, 1.1921e-06,\n", - " -5.9605e-08, -5.9605e-08, 1.1921e-07, -3.5763e-07, -5.3644e-07,\n", - " 7.1526e-07, -5.9605e-08, -7.1526e-07, 4.7684e-07, 5.9605e-07,\n", - " -6.5565e-07, 1.1921e-07, 1.4305e-06, 5.9605e-07, -1.6093e-06,\n", - " -8.9407e-07, 3.5763e-07, 1.3113e-06, -5.3644e-07, -7.7486e-07,\n", - " 1.1921e-07, -1.7881e-07, -1.7881e-07, -4.7684e-07, -1.7881e-07,\n", - " 3.5763e-07, 1.1921e-07, -9.5367e-07, -5.9605e-08, -8.3447e-07,\n", - " -1.1921e-06, 1.1921e-07, 0.0000e+00, 7.1526e-07, -4.7684e-07,\n", - " 2.3842e-07, -5.3644e-07, -2.9802e-07, -1.1921e-07, 2.3842e-07,\n", - " 2.3842e-07, 1.3113e-06, 1.0729e-06, 2.3842e-07, 1.0729e-06,\n", - " 1.0729e-06, -8.3447e-07, -5.9605e-08, -2.3842e-07, 2.3842e-07,\n", - " 1.0729e-06, 1.1921e-07, -9.5367e-07, 1.0729e-06, -2.3842e-07,\n", - " -6.5565e-07, 2.3842e-07, 3.5763e-07, -5.3644e-07, -8.3447e-07,\n", - " -8.3447e-07, -1.2517e-06, -1.1921e-07, -2.9802e-07, -2.3842e-07,\n", - " -3.5763e-07, -2.3842e-07, 2.3842e-07, -4.7684e-07, 1.0729e-06,\n", - " -1.7881e-07, 1.6689e-06, 4.7684e-07, 5.9605e-07, 3.5763e-07,\n", - " -4.7684e-07, 1.1921e-07, 1.1921e-07, 5.9605e-07, -5.9605e-08,\n", - " 5.9605e-07], device='cuda:0', grad_fn=)),\n", - " ('layer3.0.bn1.bias',\n", - " tensor([ 1.0567e-07, -2.2608e-07, 7.7980e-07, 6.0618e-07, 7.4346e-08,\n", - " 7.4604e-08, 5.1290e-08, 5.6766e-07, -4.3678e-07, -1.2464e-07,\n", - " 6.4503e-07, -1.6041e-07, -6.4229e-07, 2.4704e-08, -1.0346e-06,\n", - " 1.8932e-08, -1.6404e-07, -5.9393e-07, 6.4292e-07, 6.3990e-07,\n", - " 5.6315e-07, -7.0395e-07, 1.6254e-07, -1.5116e-07, 1.1146e-06,\n", - " 6.2241e-08, 2.4521e-07, 4.1210e-07, 2.9143e-07, 4.8081e-07,\n", - " 4.0388e-07, 2.9603e-07, -4.0308e-08, -3.8228e-07, 9.7999e-08,\n", - " -4.1132e-07, -8.2195e-07, -4.1296e-07, -4.3110e-07, 1.7718e-08,\n", - " -3.7168e-07, 9.9837e-08, -3.3698e-07, -5.6538e-07, -5.1895e-07,\n", - " -5.0196e-07, 7.5443e-07, 5.2117e-07, -1.0318e-07, -3.8180e-08,\n", - " 4.2558e-07, -1.1005e-06, -4.8846e-07, -1.0395e-07, -4.9117e-08,\n", - " -4.2353e-07, -3.2792e-08, -2.4488e-07, 1.3006e-07, -2.3050e-07,\n", - " 2.1416e-07, 1.0950e-06, 4.0173e-07, -6.7249e-07, 1.1099e-06,\n", - " -2.5061e-07, 1.0650e-06, -2.3371e-08, -1.3088e-06, -3.3970e-07,\n", - " -2.0291e-07, -4.7117e-07, 3.8934e-07, 4.4948e-08, 4.9929e-07,\n", - " -9.4986e-07, 3.0889e-08, 2.5346e-08, -1.1193e-06, -6.0021e-07,\n", - " 3.7868e-07, -4.5946e-07, 1.1235e-06, -1.3853e-07, -7.3375e-07,\n", - " 8.9059e-08, -6.3013e-08, 3.5381e-07, -5.6841e-07, -2.3535e-07,\n", - " -8.4209e-08, -5.2336e-07, 2.7097e-07, -8.5456e-07, 9.0027e-07,\n", - " 6.0925e-07, -1.9199e-07, 2.8452e-07, -2.1114e-07, -2.6626e-07,\n", - " -2.6247e-07, -6.5535e-07, 1.1454e-06, -3.0620e-07, 4.0436e-07,\n", - " -2.2470e-07, -1.3247e-08, -1.7177e-07, 3.6733e-07, 7.3840e-07,\n", - " 1.8200e-07, -7.2507e-07, 4.0633e-07, -1.0148e-07, -7.2527e-07,\n", - " -1.3658e-07, 6.1720e-07, 6.5810e-07, -1.2376e-06, 5.8190e-07,\n", - " 2.8525e-08, -3.0644e-07, -3.4234e-07, 4.1653e-08, -2.9781e-07,\n", - " -2.4274e-07, -2.7573e-07, -1.0545e-07, 1.9631e-07, -4.2902e-07,\n", - " -1.0430e-06, -6.5661e-08, 5.7729e-07, -1.3148e-06, 5.0728e-07,\n", - " 9.7028e-07, 1.2680e-06, -4.9404e-07, -1.8790e-07, 4.8792e-07,\n", - " 2.9958e-07, 2.3991e-07, 7.8558e-07, -1.2066e-06, -1.9705e-07,\n", - " 2.7108e-07, -8.7893e-07, -8.4956e-08, 3.7687e-07, 5.8561e-07,\n", - " -1.1985e-06, -1.4008e-07, -3.7536e-07, -8.7938e-08, -1.9926e-07,\n", - " -5.6485e-07, -1.0832e-06, 1.0691e-06, 4.7958e-08, -3.2534e-07,\n", - " -1.9102e-07, -6.1334e-07, -1.2687e-07, -4.5103e-07, 2.8810e-07,\n", - " 2.6147e-07, 1.3258e-08, -1.7984e-07, 7.3758e-07, -1.0172e-06,\n", - " 3.7485e-07, 4.0969e-08, 1.3691e-06, 3.3062e-07, 3.2045e-07,\n", - " -2.2348e-07, -2.4110e-07, 7.5167e-07, -1.0377e-07, -7.7313e-07,\n", - " 5.8024e-07, -2.8523e-07, -6.8930e-07, 1.2996e-07, 4.2186e-07,\n", - " 4.0243e-07, 1.6612e-07, 1.4744e-06, 4.6616e-07, -1.1064e-06,\n", - " -1.2318e-06, 4.7393e-07, 7.5768e-07, -5.7123e-07, -5.3793e-08,\n", - " 2.6284e-07, 9.8255e-08, 1.1614e-08, -5.5923e-07, 1.3437e-07,\n", - " 4.5761e-07, 3.6241e-07, -3.8616e-07, 7.1338e-08, 3.5419e-07,\n", - " -5.7918e-07, 3.1612e-07, -3.0651e-07, 1.2589e-07, -6.9323e-08,\n", - " 1.7585e-07, -8.9432e-08, 6.7227e-08, 3.3740e-07, 8.9268e-07,\n", - " 3.2936e-07, 7.7259e-07, 1.3288e-06, 5.2479e-07, 5.2387e-07,\n", - " 8.7761e-07, -7.5338e-07, -4.1550e-07, -1.1366e-07, -3.5931e-07,\n", - " 8.6762e-07, 5.2379e-07, -3.9887e-07, 4.7742e-07, -5.2923e-07,\n", - " -1.3261e-06, 2.3211e-08, 2.0264e-07, -8.8497e-08, -9.1244e-07,\n", - " -1.2321e-07, -6.4311e-07, -1.0426e-07, -2.5215e-08, -2.3498e-07,\n", - " -6.8606e-07, -2.0956e-07, 5.3289e-08, 1.9906e-07, 5.2144e-07,\n", - " 5.2902e-07, 1.6739e-06, -1.7456e-07, 4.8457e-07, 2.7094e-07,\n", - " -8.8920e-08, 6.4603e-07, -6.6939e-09, -4.6188e-07, -2.8319e-08,\n", - " 1.2474e-09], device='cuda:0', grad_fn=)),\n", - " ('layer3.0.conv2.weight',\n", - " tensor([[[[ 1.0662e-02, 2.3105e-02, 2.0944e-02],\n", - " [-1.6962e-03, -1.7286e-02, -1.3265e-02],\n", - " [-7.9107e-04, -1.2152e-02, 1.1042e-02]],\n", - " \n", - " [[-2.1019e-02, -3.9375e-03, 2.7401e-02],\n", - " [ 2.6706e-02, 1.9458e-02, -1.6592e-03],\n", - " [ 1.2585e-02, -5.9605e-03, 1.2421e-02]],\n", - " \n", - " [[-6.9274e-03, 8.3360e-03, -9.4914e-03],\n", - " [ 5.3120e-03, 2.5441e-03, -1.9309e-02],\n", - " [-1.2729e-02, 1.2098e-03, -1.3140e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.5503e-02, -2.0459e-02, -1.7660e-02],\n", - " [ 2.1041e-03, -4.7642e-03, -1.6214e-03],\n", - " [-8.4515e-03, -8.6028e-03, 7.7011e-03]],\n", - " \n", - " [[ 1.9754e-03, 1.2156e-02, 1.5698e-02],\n", - " [-1.4284e-02, 7.9187e-03, 5.6066e-03],\n", - " [-1.4685e-02, 5.0906e-03, 2.0291e-02]],\n", - " \n", - " [[ 1.0820e-03, 2.5782e-02, -8.0276e-03],\n", - " [-1.5261e-02, -1.0020e-02, -7.2103e-03],\n", - " [-1.4226e-02, -8.4334e-03, 6.6771e-03]]],\n", - " \n", - " \n", - " [[[ 1.5926e-02, -6.3484e-03, -2.1075e-03],\n", - " [-1.5315e-03, -5.5726e-03, 1.7136e-02],\n", - " [ 1.4945e-03, -1.7555e-02, 4.3468e-03]],\n", - " \n", - " [[-2.1769e-02, 3.3361e-03, 2.4591e-02],\n", - " [ 1.5109e-02, -2.8429e-02, 1.0172e-03],\n", - " [-1.6435e-02, 2.6428e-03, 1.5481e-02]],\n", - " \n", - " [[-1.4824e-02, -8.5155e-03, 1.8834e-02],\n", - " [ 1.2652e-02, 6.7154e-03, -2.6716e-03],\n", - " [-8.6145e-03, 1.0106e-02, -2.3724e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.6900e-02, -5.1725e-03, -1.7768e-03],\n", - " [-1.0207e-02, -3.7643e-03, -9.7340e-03],\n", - " [ 2.5798e-02, -1.9137e-02, 2.0954e-02]],\n", - " \n", - " [[ 1.7942e-02, 1.1663e-02, 1.3949e-02],\n", - " [ 7.6405e-03, 1.2597e-02, 1.8279e-02],\n", - " [-5.5239e-03, -1.8590e-02, -1.8047e-03]],\n", - " \n", - " [[-1.4462e-02, 7.7165e-03, 2.2021e-03],\n", - " [-1.0820e-02, -1.0834e-03, -3.6798e-03],\n", - " [-1.9548e-02, -1.3020e-03, -1.1262e-02]]],\n", - " \n", - " \n", - " [[[ 1.2946e-02, 1.1081e-02, 1.1393e-02],\n", - " [ 4.8879e-04, -1.1928e-02, 1.0031e-02],\n", - " [-1.7902e-02, -1.7419e-02, -4.2769e-03]],\n", - " \n", - " [[-6.6826e-04, 3.5241e-03, -2.0011e-02],\n", - " [ 8.4268e-03, -1.5843e-02, 5.7051e-03],\n", - " [ 5.3919e-03, -1.1616e-03, -1.6102e-02]],\n", - " \n", - " [[ 5.3986e-03, 2.4112e-02, -8.9773e-03],\n", - " [ 1.3202e-02, -2.2026e-03, -2.1816e-02],\n", - " [ 5.5018e-03, 8.3577e-05, 1.2664e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.1547e-02, 6.8494e-03, 2.1623e-02],\n", - " [ 2.6007e-02, 4.2709e-03, -1.6149e-03],\n", - " [ 2.8532e-02, -1.6667e-02, -2.0934e-02]],\n", - " \n", - " [[ 1.0923e-02, 2.3803e-02, -2.7859e-03],\n", - " [-1.5858e-02, 1.5274e-02, 1.4692e-02],\n", - " [ 8.9732e-03, 6.4990e-03, 6.1606e-03]],\n", - " \n", - " [[ 2.8232e-03, 6.6697e-03, -2.7311e-03],\n", - " [-2.2336e-02, -1.6776e-03, -2.1136e-03],\n", - " [-7.1193e-03, -9.1132e-03, 1.7332e-02]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 5.0451e-03, -1.4421e-02, 1.0128e-02],\n", - " [ 2.3629e-02, -2.6037e-03, -6.2489e-03],\n", - " [-2.4079e-02, 1.1019e-04, 1.8441e-02]],\n", - " \n", - " [[-1.6919e-02, 4.9657e-03, -1.1922e-02],\n", - " [-4.4497e-03, 2.1552e-02, -6.9888e-03],\n", - " [ 5.7468e-03, 1.0028e-02, -2.9763e-03]],\n", - " \n", - " [[-1.1208e-02, -4.8582e-04, 9.0071e-03],\n", - " [ 4.2691e-03, -1.6680e-02, 1.5524e-03],\n", - " [-2.0489e-03, -1.5415e-03, 9.5223e-04]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.2714e-02, -8.5548e-03, 1.7332e-02],\n", - " [-7.8999e-03, 1.6427e-02, -1.6550e-02],\n", - " [-1.2743e-02, -2.4785e-02, -5.7847e-03]],\n", - " \n", - " [[-1.8290e-03, 3.9545e-03, 1.5218e-02],\n", - " [-1.5010e-02, 8.9211e-03, 7.1442e-03],\n", - " [-1.5832e-02, -1.0943e-02, 1.2260e-04]],\n", - " \n", - " [[ 2.5532e-03, 1.7381e-02, -8.9490e-03],\n", - " [-1.6582e-02, -1.3242e-02, -5.1941e-03],\n", - " [-1.5262e-04, 1.1208e-03, 6.8951e-03]]],\n", - " \n", - " \n", - " [[[ 1.3280e-02, 1.4718e-02, 1.1648e-03],\n", - " [ 5.4112e-03, -1.0257e-03, -1.9862e-02],\n", - " [ 4.9593e-03, 1.6201e-02, -6.7019e-03]],\n", - " \n", - " [[-7.3088e-03, -1.2836e-02, -1.6779e-02],\n", - " [ 7.5643e-03, -7.6154e-03, -1.7013e-02],\n", - " [-7.5895e-04, 2.4402e-02, -3.3536e-02]],\n", - " \n", - " [[ 2.2493e-02, -1.3170e-02, 1.9583e-02],\n", - " [-7.0033e-03, 1.5615e-02, 2.2892e-02],\n", - " [-2.1505e-04, -2.7462e-04, 5.6131e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 9.3670e-03, 1.7157e-02, -3.8728e-03],\n", - " [-5.8528e-03, -9.1498e-03, -7.8514e-03],\n", - " [-7.9125e-03, 1.5890e-02, -2.6130e-03]],\n", - " \n", - " [[-8.3766e-04, -3.3567e-03, 2.4494e-02],\n", - " [ 2.6621e-02, 7.9730e-03, 2.1429e-02],\n", - " [-6.8606e-03, 1.3426e-02, -6.1734e-03]],\n", - " \n", - " [[-5.1905e-04, 1.5170e-02, 1.0361e-02],\n", - " [-4.1423e-04, -2.1263e-03, 2.0374e-02],\n", - " [-2.1371e-02, -1.8019e-02, 1.7098e-02]]],\n", - " \n", - " \n", - " [[[ 1.7300e-02, -2.7703e-02, 1.1888e-02],\n", - " [-2.5120e-02, 8.9563e-03, -1.3949e-03],\n", - " [ 4.1365e-03, -5.1395e-03, 2.7497e-03]],\n", - " \n", - " [[-8.7912e-03, -3.8518e-06, -2.4544e-02],\n", - " [-2.5744e-02, -2.0509e-02, 7.0818e-03],\n", - " [-2.0765e-02, -2.0426e-02, -2.6527e-03]],\n", - " \n", - " [[ 1.0005e-02, -6.5160e-03, 2.3553e-02],\n", - " [ 2.4463e-02, 1.2743e-03, -2.6316e-02],\n", - " [-7.5289e-03, -3.1372e-03, 2.5079e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.1344e-02, 2.5635e-02, -2.7420e-02],\n", - " [ 1.5453e-02, -1.3559e-02, 7.9082e-03],\n", - " [ 1.8416e-02, -2.1079e-02, 3.1051e-04]],\n", - " \n", - " [[ 1.6002e-02, 4.8267e-03, 1.7036e-02],\n", - " [-2.0450e-02, 1.6423e-02, 1.5697e-02],\n", - " [-5.6105e-03, -1.2805e-02, -7.7957e-04]],\n", - " \n", - " [[ 1.4429e-02, -4.5842e-03, -1.0417e-02],\n", - " [ 7.9961e-03, 8.3452e-03, -1.1142e-02],\n", - " [-1.6436e-02, 2.8955e-02, -1.7532e-02]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer3.0.bn2.weight',\n", - " tensor([ 3.5763e-07, 5.9605e-07, 4.7684e-07, -4.1723e-07, -2.3842e-07,\n", - " 8.3447e-07, 1.1921e-07, 4.7684e-07, 2.3842e-07, -3.5763e-07,\n", - " -3.5763e-07, -1.0729e-06, -1.7881e-07, 3.5763e-07, 0.0000e+00,\n", - " 1.0729e-06, 2.3842e-07, -1.0729e-06, 0.0000e+00, 1.1921e-06,\n", - " -5.9605e-07, -1.1921e-07, -7.1526e-07, 4.7684e-07, 5.9605e-07,\n", - " 2.3842e-07, 1.1921e-07, -4.1723e-07, -1.1921e-07, -4.7684e-07,\n", - " -1.2517e-06, 5.9605e-07, 4.7684e-07, 3.5763e-07, 1.1921e-07,\n", - " 1.1921e-07, 4.7684e-07, -2.3842e-07, 1.1921e-07, 3.5763e-07,\n", - " -5.3644e-07, -1.7881e-07, 1.0729e-06, -1.0133e-06, -5.3644e-07,\n", - " 7.1526e-07, -1.7881e-07, 3.5763e-07, 2.3842e-07, -1.1921e-07,\n", - " -5.9605e-07, -4.7684e-07, 1.1921e-07, 2.3842e-07, -7.7486e-07,\n", - " -2.3842e-07, -6.5565e-07, -3.5763e-07, 4.7684e-07, 3.5763e-07,\n", - " 0.0000e+00, -2.3842e-07, 2.3842e-07, -6.5565e-07, 2.3842e-07,\n", - " -2.9802e-07, 2.3842e-07, -1.1921e-07, 0.0000e+00, 2.3842e-07,\n", - " -8.3447e-07, -1.1921e-07, 2.3842e-07, 5.9605e-07, 9.5367e-07,\n", - " -7.7486e-07, -4.7684e-07, 4.7684e-07, 2.3842e-07, 1.1921e-07,\n", - " 5.9605e-07, -5.3644e-07, -7.7486e-07, 3.5763e-07, -4.1723e-07,\n", - " -3.5763e-07, -8.3447e-07, -8.9407e-07, -1.1921e-07, -1.0133e-06,\n", - " -3.5763e-07, 1.0729e-06, -5.9605e-08, -6.5565e-07, -1.7881e-07,\n", - " -9.5367e-07, 0.0000e+00, -2.3842e-07, -5.9605e-08, 1.0729e-06,\n", - " 7.1526e-07, -8.3447e-07, 1.1921e-06, 1.1921e-07, 8.3447e-07,\n", - " 3.5763e-07, -2.9802e-07, 1.1921e-07, 5.9605e-07, 5.9605e-07,\n", - " -1.2517e-06, 3.5763e-07, -1.7881e-07, 4.7684e-07, -1.1921e-07,\n", - " 7.1526e-07, -7.7486e-07, 7.1526e-07, 1.1921e-07, 3.5763e-07,\n", - " 0.0000e+00, 1.1921e-07, 4.7684e-07, -1.7881e-07, 3.5763e-07,\n", - " 5.9605e-07, 1.1921e-07, -1.7881e-07, 0.0000e+00, -5.3644e-07,\n", - " -8.3447e-07, -2.9802e-07, -2.3842e-07, -1.1921e-07, -2.9802e-07,\n", - " 1.1921e-06, -6.5565e-07, -2.3842e-07, 4.7684e-07, 8.3447e-07,\n", - " -4.1723e-07, 1.0729e-06, 4.7684e-07, 1.1921e-07, 1.1921e-07,\n", - " -6.5565e-07, -1.0133e-06, 1.0729e-06, 4.7684e-07, -5.9605e-08,\n", - " 3.5763e-07, 2.3842e-07, 1.1921e-07, -4.7684e-07, 2.3842e-07,\n", - " 3.5763e-07, -1.1921e-07, 2.3842e-07, 4.7684e-07, -5.9605e-08,\n", - " 3.5763e-07, -2.9802e-07, -2.3842e-07, -3.5763e-07, 7.1526e-07,\n", - " 3.5763e-07, -4.1723e-07, -3.5763e-07, 0.0000e+00, -8.3447e-07,\n", - " 9.5367e-07, -5.9605e-08, -5.9605e-07, -5.9605e-07, 7.1526e-07,\n", - " 7.1526e-07, 1.1921e-07, 3.5763e-07, -3.5763e-07, -4.1723e-07,\n", - " 0.0000e+00, 2.3842e-07, 2.3842e-07, -4.1723e-07, 9.5367e-07,\n", - " 2.3842e-07, 5.9605e-07, 3.5763e-07, -3.5763e-07, -1.0133e-06,\n", - " -2.3842e-07, -1.7881e-07, -1.0729e-06, -5.3644e-07, -7.1526e-07,\n", - " -7.7486e-07, 8.3447e-07, 2.3842e-07, -5.3644e-07, -5.3644e-07,\n", - " -1.1921e-07, -6.5565e-07, -7.7486e-07, -4.1723e-07, 3.5763e-07,\n", - " -6.5565e-07, -5.9605e-08, -9.5367e-07, -5.3644e-07, 1.1921e-06,\n", - " -1.1921e-06, -1.1921e-07, 5.9605e-07, 7.1526e-07, 5.9605e-07,\n", - " -2.3842e-07, -1.1921e-07, 0.0000e+00, -3.5763e-07, 3.5763e-07,\n", - " -2.9802e-07, -2.9802e-07, 0.0000e+00, -2.9802e-07, -1.1921e-07,\n", - " -7.1526e-07, -1.4901e-06, 1.1921e-07, 3.5763e-07, 1.5497e-06,\n", - " -4.1723e-07, 1.1921e-07, -2.9802e-07, 4.7684e-07, 7.1526e-07,\n", - " 8.3447e-07, -2.3842e-07, 9.5367e-07, -7.1526e-07, 2.3842e-07,\n", - " 1.1921e-07, 3.5763e-07, 3.5763e-07, -5.9605e-07, -3.5763e-07,\n", - " 4.7684e-07, 2.3842e-07, -1.7881e-07, 7.1526e-07, 8.3447e-07,\n", - " 8.3447e-07, 0.0000e+00, 5.9605e-07, 4.7684e-07, -1.7881e-07,\n", - " 0.0000e+00], device='cuda:0', grad_fn=)),\n", - " ('layer3.0.bn2.bias',\n", - " tensor([ 2.3680e-07, 3.5135e-07, 3.4183e-07, 8.8436e-08, 2.0979e-08,\n", - " 2.5290e-07, 6.5370e-07, -2.6026e-07, -3.0810e-08, -5.5992e-08,\n", - " -2.3866e-07, -3.7466e-07, -7.0554e-07, 1.1698e-07, -4.4040e-07,\n", - " 6.7360e-07, -1.4584e-09, 5.1592e-09, -7.2081e-07, 5.7207e-07,\n", - " -2.4412e-07, 5.0197e-08, -6.8681e-07, -2.0246e-07, 6.1333e-07,\n", - " -9.8468e-08, 3.9486e-07, -2.1139e-07, -3.0886e-07, -7.6842e-07,\n", - " -6.5191e-07, 2.6826e-07, 4.6805e-07, -1.5186e-07, -1.0934e-07,\n", - " -2.5448e-07, -8.2063e-08, -1.9279e-07, 5.8344e-07, 4.2441e-07,\n", - " -8.0338e-07, -3.2880e-07, 3.8258e-07, 1.5075e-07, 4.8517e-07,\n", - " 1.1028e-07, 3.3404e-07, 2.7634e-07, -1.3250e-07, 7.7396e-08,\n", - " 1.8827e-08, 1.7996e-08, 3.3748e-07, -4.5710e-07, -9.9208e-07,\n", - " -4.3114e-07, 4.0201e-08, 3.1458e-07, 1.9742e-07, -2.1169e-07,\n", - " -1.6180e-07, 9.9652e-08, 1.1093e-07, -5.2305e-07, 2.8458e-07,\n", - " 6.2370e-07, 1.5391e-07, -2.4377e-07, -1.0629e-08, -4.5090e-07,\n", - " -6.9124e-07, 1.6198e-07, -3.7297e-08, 7.4784e-07, 5.0719e-07,\n", - " -5.0355e-07, -6.1532e-07, 3.8803e-07, 3.1920e-07, 2.7575e-08,\n", - " 6.2666e-07, -1.0726e-07, 1.9611e-07, -5.7010e-07, -3.4205e-07,\n", - " 1.5303e-07, -8.2835e-07, 8.1646e-07, 7.0774e-07, -4.7583e-07,\n", - " -2.7194e-07, 8.6398e-08, 9.4109e-08, -1.7007e-07, -3.2332e-07,\n", - " 6.5095e-08, -8.3583e-07, 1.9435e-07, 4.7392e-07, 6.5855e-07,\n", - " 2.7783e-07, -4.3500e-07, 1.5604e-07, -4.1401e-07, 8.9819e-08,\n", - " 2.7929e-07, -3.7298e-07, -8.6467e-08, 7.3443e-07, -2.6574e-07,\n", - " 8.4724e-08, -2.7784e-07, -2.2111e-07, 8.4979e-07, 3.4278e-08,\n", - " 3.4340e-07, -1.5229e-08, 1.2894e-06, 4.1525e-07, 4.1044e-07,\n", - " -1.7110e-07, 1.1602e-06, -8.1511e-08, -1.3142e-07, -2.8796e-07,\n", - " -4.2405e-07, -2.3917e-07, -4.5406e-07, -4.2457e-07, -5.5305e-08,\n", - " -4.0831e-07, -7.1161e-07, -7.1909e-07, 4.3880e-08, -2.1519e-07,\n", - " 6.9777e-07, 4.6607e-07, 3.3010e-07, -2.1112e-07, 4.3409e-07,\n", - " -4.4896e-08, 4.9483e-07, -3.5638e-07, 3.2437e-07, 3.2455e-07,\n", - " 3.6626e-08, -2.3136e-08, 3.0788e-07, 9.0411e-07, -7.1682e-07,\n", - " 1.3247e-07, 6.7468e-07, 2.1672e-07, -3.4881e-07, -1.6487e-07,\n", - " 2.6893e-07, -4.5407e-07, 1.7996e-09, -9.0014e-08, 3.7263e-08,\n", - " -1.5918e-07, 1.6091e-07, -6.2528e-07, -2.8585e-07, 2.4054e-07,\n", - " -1.8991e-07, 4.6312e-07, -3.8197e-07, 6.5382e-08, -6.0868e-07,\n", - " 1.7277e-07, 1.5869e-07, 2.4873e-07, 1.4232e-08, 7.9809e-07,\n", - " 9.4013e-07, 3.8632e-07, -4.7660e-07, -4.2201e-08, -9.8884e-07,\n", - " -4.1836e-07, 8.7761e-07, -3.5395e-07, -1.2057e-07, -8.8166e-08,\n", - " 2.7069e-07, 2.6314e-07, -8.3510e-08, -1.0514e-07, 1.0384e-07,\n", - " 6.7777e-07, -8.5769e-07, -8.8564e-07, 1.9361e-07, -5.7802e-07,\n", - " -1.2372e-07, 2.6111e-07, 3.3398e-07, -3.8925e-07, -1.3571e-07,\n", - " 3.4758e-07, -4.1377e-07, -8.7289e-07, -3.4511e-07, 3.8844e-07,\n", - " -3.9513e-07, -8.4503e-09, -1.2836e-07, -8.8299e-07, 9.3458e-07,\n", - " -8.7290e-07, -2.2613e-07, 2.4439e-07, 4.7279e-07, 4.2166e-07,\n", - " 3.5165e-08, 3.3261e-08, 1.3420e-09, 8.2721e-08, -1.3073e-08,\n", - " -6.5133e-07, 2.2716e-07, 1.0948e-07, 8.1573e-08, -2.5343e-07,\n", - " -5.5931e-07, -1.7230e-07, -3.9210e-07, -3.8520e-07, -3.1067e-07,\n", - " -9.0795e-07, -2.2396e-08, 4.6810e-07, 3.8014e-07, 2.1777e-09,\n", - " 8.1825e-07, -2.2735e-07, -1.5970e-07, -2.4650e-07, -6.7152e-07,\n", - " -2.7729e-07, -1.0032e-07, 5.6624e-08, -5.8656e-07, -2.4579e-08,\n", - " 1.2047e-07, 2.9561e-07, -5.2755e-07, 6.3744e-07, 6.7626e-07,\n", - " 1.7435e-07, 4.1876e-07, 8.1455e-07, -3.8383e-07, 2.2717e-07,\n", - " -1.2689e-07], device='cuda:0', grad_fn=)),\n", - " ('layer3.0.shortcut.0.weight',\n", - " tensor([[[[-1.2532e-01]],\n", - " \n", - " [[ 1.7072e-02]],\n", - " \n", - " [[ 5.5834e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.5657e-02]],\n", - " \n", - " [[ 1.2111e-02]],\n", - " \n", - " [[-7.3992e-05]]],\n", - " \n", - " \n", - " [[[-5.7674e-02]],\n", - " \n", - " [[-1.4630e-03]],\n", - " \n", - " [[-3.7302e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.3707e-02]],\n", - " \n", - " [[ 3.9356e-02]],\n", - " \n", - " [[-1.1301e-01]]],\n", - " \n", - " \n", - " [[[ 2.2812e-02]],\n", - " \n", - " [[-6.2792e-02]],\n", - " \n", - " [[-8.5218e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.4610e-02]],\n", - " \n", - " [[ 1.0115e-01]],\n", - " \n", - " [[-2.1533e-02]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 2.8550e-02]],\n", - " \n", - " [[ 6.4809e-03]],\n", - " \n", - " [[-6.6245e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.0575e-01]],\n", - " \n", - " [[-4.9512e-02]],\n", - " \n", - " [[ 3.9289e-02]]],\n", - " \n", - " \n", - " [[[-4.6138e-02]],\n", - " \n", - " [[-7.4739e-02]],\n", - " \n", - " [[ 5.0019e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-9.5460e-04]],\n", - " \n", - " [[ 5.4519e-02]],\n", - " \n", - " [[-2.4941e-02]]],\n", - " \n", - " \n", - " [[[-6.3402e-02]],\n", - " \n", - " [[-3.0870e-02]],\n", - " \n", - " [[ 1.0218e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.0487e-03]],\n", - " \n", - " [[-4.1583e-02]],\n", - " \n", - " [[-8.3212e-03]]]], device='cuda:0', grad_fn=)),\n", - " ('layer3.0.shortcut.1.weight',\n", - " tensor([-5.9605e-08, 5.9605e-07, 3.5763e-07, -4.1723e-07, 3.5763e-07,\n", - " 1.1921e-07, 9.5367e-07, 0.0000e+00, -2.9802e-07, 5.9605e-07,\n", - " 5.9605e-07, 1.1921e-07, -8.9407e-07, 0.0000e+00, -5.3644e-07,\n", - " -2.3842e-07, 4.7684e-07, 5.9605e-07, -9.5367e-07, 2.3842e-07,\n", - " 1.1921e-07, 5.9605e-07, -5.3644e-07, 1.1921e-07, 2.3842e-07,\n", - " 2.3842e-07, 0.0000e+00, 2.3842e-07, -8.9407e-07, -4.7684e-07,\n", - " -1.7881e-07, -1.1921e-07, -5.9605e-08, -5.3644e-07, 4.7684e-07,\n", - " -5.9605e-08, -5.9605e-08, -3.5763e-07, 5.9605e-07, 4.7684e-07,\n", - " -2.3842e-07, -4.7684e-07, -3.5763e-07, 0.0000e+00, 7.1526e-07,\n", - " 2.3842e-07, -1.1921e-07, 3.5763e-07, -4.1723e-07, -5.9605e-08,\n", - " 5.9605e-07, 5.9605e-07, 5.9605e-07, -1.1921e-06, -6.5565e-07,\n", - " -4.7684e-07, 5.9605e-07, -1.7881e-07, -7.1526e-07, -1.7881e-07,\n", - " -6.5565e-07, 3.5763e-07, -2.3842e-07, 5.9605e-07, -1.1921e-07,\n", - " 2.3842e-07, -1.1921e-07, 2.3842e-07, -5.3644e-07, -8.9407e-07,\n", - " 2.3842e-07, -1.7881e-07, -4.7684e-07, 1.1921e-07, -2.9802e-07,\n", - " 1.1921e-07, -5.9605e-07, 2.3842e-07, 2.3842e-07, 2.3842e-07,\n", - " 8.3447e-07, 5.9605e-07, 8.3447e-07, -1.4305e-06, -6.5565e-07,\n", - " -5.9605e-08, -5.3644e-07, 1.5497e-06, 9.5367e-07, 1.1921e-07,\n", - " -2.9802e-07, -5.9605e-07, -2.3842e-07, -2.3842e-07, -7.7486e-07,\n", - " 1.1921e-07, -5.9605e-07, 0.0000e+00, 1.0729e-06, -5.9605e-08,\n", - " 1.1921e-07, -8.3447e-07, 3.5763e-07, -2.9802e-07, -4.1723e-07,\n", - " 0.0000e+00, -3.5763e-07, 3.5763e-07, -4.1723e-07, -9.5367e-07,\n", - " 1.1921e-06, 2.3842e-07, 1.0729e-06, 1.0729e-06, 1.1921e-07,\n", - " 0.0000e+00, 4.7684e-07, 1.0729e-06, 2.3842e-07, -2.3842e-07,\n", - " -1.1921e-07, 4.7684e-07, 2.3842e-07, 5.9605e-07, -5.9605e-07,\n", - " 0.0000e+00, -2.9802e-07, -1.1921e-07, -5.9605e-08, 2.3842e-07,\n", - " -8.3447e-07, -2.9802e-07, -1.4305e-06, -4.1723e-07, 1.1921e-07,\n", - " 1.0729e-06, -4.7684e-07, 1.1921e-07, 3.5763e-07, -5.3644e-07,\n", - " 1.1921e-07, -3.5763e-07, -2.3842e-07, 3.5763e-07, 8.3447e-07,\n", - " 9.5367e-07, 1.0729e-06, 4.7684e-07, 4.7684e-07, -2.3842e-07,\n", - " -1.7881e-07, 5.9605e-07, 2.3842e-07, -6.5565e-07, -5.9605e-07,\n", - " -7.1526e-07, -5.9605e-08, 1.1921e-07, 7.1526e-07, -5.9605e-07,\n", - " -1.1921e-07, 8.3447e-07, -5.9605e-07, 2.3842e-07, -3.5763e-07,\n", - " -5.9605e-07, 7.1526e-07, 0.0000e+00, -2.9802e-07, -4.1723e-07,\n", - " 2.3842e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07, 1.0729e-06,\n", - " 3.5763e-07, 2.3842e-07, -3.5763e-07, 8.3447e-07, -8.9407e-07,\n", - " -1.7881e-07, 5.9605e-07, -5.3644e-07, -5.9605e-08, -6.5565e-07,\n", - " 4.7684e-07, 2.3842e-07, -1.1325e-06, 1.1921e-07, 4.7684e-07,\n", - " 5.9605e-07, -2.9802e-07, 2.3842e-07, 2.3842e-07, -3.5763e-07,\n", - " 0.0000e+00, 5.9605e-07, -4.1723e-07, -7.7486e-07, -5.9605e-08,\n", - " 0.0000e+00, -6.5565e-07, -4.7684e-07, -5.9605e-08, 1.1921e-07,\n", - " -3.5763e-07, 1.4305e-06, 2.3842e-07, -1.0729e-06, 1.1921e-07,\n", - " -9.5367e-07, 3.5763e-07, -4.7684e-07, 0.0000e+00, 0.0000e+00,\n", - " 3.5763e-07, 8.3447e-07, 4.7684e-07, -1.1921e-07, 3.5763e-07,\n", - " -4.1723e-07, 7.1526e-07, -3.5763e-07, -1.1921e-07, 2.3842e-07,\n", - " -1.0133e-06, 7.1526e-07, -6.5565e-07, -1.0729e-06, -1.4305e-06,\n", - " -7.7486e-07, 1.1921e-07, 5.9605e-07, 7.1526e-07, 5.9605e-07,\n", - " 3.5763e-07, 2.3842e-07, -1.0133e-06, 9.5367e-07, -1.2517e-06,\n", - " -4.7684e-07, -2.3842e-07, -5.9605e-07, 3.5763e-07, -5.3644e-07,\n", - " -1.1921e-07, 5.9605e-07, -1.0729e-06, 1.1921e-07, 1.1921e-07,\n", - " -2.9802e-07, 3.5763e-07, 5.9605e-07, -4.1723e-07, 4.7684e-07,\n", - " 5.9605e-07], device='cuda:0', grad_fn=)),\n", - " ('layer3.0.shortcut.1.bias',\n", - " tensor([ 2.3680e-07, 3.5135e-07, 3.4183e-07, 8.8436e-08, 2.0979e-08,\n", - " 2.5290e-07, 6.5370e-07, -2.6026e-07, -3.0810e-08, -5.5992e-08,\n", - " -2.3866e-07, -3.7466e-07, -7.0554e-07, 1.1698e-07, -4.4040e-07,\n", - " 6.7360e-07, -1.4584e-09, 5.1592e-09, -7.2081e-07, 5.7207e-07,\n", - " -2.4412e-07, 5.0197e-08, -6.8681e-07, -2.0246e-07, 6.1333e-07,\n", - " -9.8468e-08, 3.9486e-07, -2.1139e-07, -3.0886e-07, -7.6842e-07,\n", - " -6.5191e-07, 2.6826e-07, 4.6805e-07, -1.5186e-07, -1.0934e-07,\n", - " -2.5448e-07, -8.2063e-08, -1.9279e-07, 5.8344e-07, 4.2441e-07,\n", - " -8.0338e-07, -3.2880e-07, 3.8258e-07, 1.5075e-07, 4.8517e-07,\n", - " 1.1028e-07, 3.3404e-07, 2.7634e-07, -1.3250e-07, 7.7396e-08,\n", - " 1.8827e-08, 1.7996e-08, 3.3748e-07, -4.5710e-07, -9.9208e-07,\n", - " -4.3114e-07, 4.0201e-08, 3.1458e-07, 1.9742e-07, -2.1169e-07,\n", - " -1.6180e-07, 9.9652e-08, 1.1093e-07, -5.2305e-07, 2.8458e-07,\n", - " 6.2370e-07, 1.5391e-07, -2.4377e-07, -1.0629e-08, -4.5090e-07,\n", - " -6.9124e-07, 1.6198e-07, -3.7297e-08, 7.4784e-07, 5.0719e-07,\n", - " -5.0355e-07, -6.1532e-07, 3.8803e-07, 3.1920e-07, 2.7575e-08,\n", - " 6.2666e-07, -1.0726e-07, 1.9611e-07, -5.7010e-07, -3.4205e-07,\n", - " 1.5303e-07, -8.2835e-07, 8.1646e-07, 7.0774e-07, -4.7583e-07,\n", - " -2.7194e-07, 8.6398e-08, 9.4109e-08, -1.7007e-07, -3.2332e-07,\n", - " 6.5095e-08, -8.3583e-07, 1.9435e-07, 4.7392e-07, 6.5855e-07,\n", - " 2.7783e-07, -4.3500e-07, 1.5604e-07, -4.1401e-07, 8.9819e-08,\n", - " 2.7929e-07, -3.7298e-07, -8.6467e-08, 7.3443e-07, -2.6574e-07,\n", - " 8.4724e-08, -2.7784e-07, -2.2111e-07, 8.4979e-07, 3.4278e-08,\n", - " 3.4340e-07, -1.5229e-08, 1.2894e-06, 4.1525e-07, 4.1044e-07,\n", - " -1.7110e-07, 1.1602e-06, -8.1511e-08, -1.3142e-07, -2.8796e-07,\n", - " -4.2405e-07, -2.3917e-07, -4.5406e-07, -4.2457e-07, -5.5305e-08,\n", - " -4.0831e-07, -7.1161e-07, -7.1909e-07, 4.3880e-08, -2.1519e-07,\n", - " 6.9777e-07, 4.6607e-07, 3.3010e-07, -2.1112e-07, 4.3409e-07,\n", - " -4.4896e-08, 4.9483e-07, -3.5638e-07, 3.2437e-07, 3.2455e-07,\n", - " 3.6626e-08, -2.3136e-08, 3.0788e-07, 9.0411e-07, -7.1682e-07,\n", - " 1.3247e-07, 6.7468e-07, 2.1672e-07, -3.4881e-07, -1.6487e-07,\n", - " 2.6893e-07, -4.5407e-07, 1.7996e-09, -9.0014e-08, 3.7263e-08,\n", - " -1.5918e-07, 1.6091e-07, -6.2528e-07, -2.8585e-07, 2.4054e-07,\n", - " -1.8991e-07, 4.6312e-07, -3.8197e-07, 6.5382e-08, -6.0868e-07,\n", - " 1.7277e-07, 1.5869e-07, 2.4873e-07, 1.4232e-08, 7.9809e-07,\n", - " 9.4013e-07, 3.8632e-07, -4.7660e-07, -4.2201e-08, -9.8884e-07,\n", - " -4.1836e-07, 8.7761e-07, -3.5395e-07, -1.2057e-07, -8.8166e-08,\n", - " 2.7069e-07, 2.6314e-07, -8.3510e-08, -1.0514e-07, 1.0384e-07,\n", - " 6.7777e-07, -8.5769e-07, -8.8564e-07, 1.9361e-07, -5.7802e-07,\n", - " -1.2372e-07, 2.6111e-07, 3.3398e-07, -3.8925e-07, -1.3571e-07,\n", - " 3.4758e-07, -4.1377e-07, -8.7289e-07, -3.4511e-07, 3.8844e-07,\n", - " -3.9513e-07, -8.4503e-09, -1.2836e-07, -8.8299e-07, 9.3458e-07,\n", - " -8.7290e-07, -2.2613e-07, 2.4439e-07, 4.7279e-07, 4.2166e-07,\n", - " 3.5165e-08, 3.3261e-08, 1.3420e-09, 8.2721e-08, -1.3073e-08,\n", - " -6.5133e-07, 2.2716e-07, 1.0948e-07, 8.1573e-08, -2.5343e-07,\n", - " -5.5931e-07, -1.7230e-07, -3.9210e-07, -3.8520e-07, -3.1067e-07,\n", - " -9.0795e-07, -2.2396e-08, 4.6810e-07, 3.8014e-07, 2.1777e-09,\n", - " 8.1825e-07, -2.2735e-07, -1.5970e-07, -2.4650e-07, -6.7152e-07,\n", - " -2.7729e-07, -1.0032e-07, 5.6624e-08, -5.8656e-07, -2.4579e-08,\n", - " 1.2047e-07, 2.9561e-07, -5.2755e-07, 6.3744e-07, 6.7626e-07,\n", - " 1.7435e-07, 4.1876e-07, 8.1455e-07, -3.8383e-07, 2.2717e-07,\n", - " -1.2689e-07], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.conv1.weight',\n", - " tensor([[[[ 1.8114e-02, 5.2051e-04, -1.4573e-02],\n", - " [ 1.0840e-02, 3.5196e-03, -2.3358e-02],\n", - " [-3.9562e-03, -1.5493e-02, -7.0367e-03]],\n", - " \n", - " [[-6.0210e-03, 5.6906e-03, -1.5362e-02],\n", - " [ 2.8613e-03, -8.9222e-03, 1.2015e-02],\n", - " [-6.4128e-03, 2.4453e-02, 1.6543e-03]],\n", - " \n", - " [[ 1.7152e-02, -1.7592e-02, -1.1884e-03],\n", - " [-1.1144e-02, -1.7280e-02, -6.4692e-03],\n", - " [-1.9362e-04, -1.3540e-02, 1.7617e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1642e-02, 2.2848e-03, 1.3447e-02],\n", - " [ 1.3338e-03, 1.2532e-02, -4.2178e-03],\n", - " [ 6.1783e-03, 1.0295e-02, -3.5218e-03]],\n", - " \n", - " [[-4.4384e-03, 2.5541e-03, 6.1089e-03],\n", - " [-1.8793e-02, -5.2627e-03, 2.2046e-03],\n", - " [ 1.0186e-02, 5.0196e-03, -1.5322e-02]],\n", - " \n", - " [[-1.3257e-02, 1.6731e-02, 9.1186e-03],\n", - " [ 5.3671e-03, -2.3662e-02, 1.7017e-02],\n", - " [-9.3166e-03, -2.1477e-02, 1.6195e-02]]],\n", - " \n", - " \n", - " [[[ 8.3378e-03, -1.8744e-02, 6.6716e-05],\n", - " [ 1.0186e-02, 3.5303e-03, 2.3636e-02],\n", - " [-8.1820e-03, 1.4936e-03, 2.1595e-02]],\n", - " \n", - " [[-7.4387e-03, 1.0811e-02, -4.1400e-03],\n", - " [-1.0415e-02, -1.0622e-02, 1.4807e-02],\n", - " [-1.7862e-02, -1.2328e-02, 2.1008e-02]],\n", - " \n", - " [[ 1.3586e-02, -4.4893e-04, -1.4608e-02],\n", - " [ 2.0508e-02, -1.1871e-02, -1.7351e-03],\n", - " [-9.2368e-03, -1.1429e-02, 2.0413e-04]],\n", - " \n", - " ...,\n", - " \n", - " [[ 9.7317e-03, 6.8197e-03, 2.1716e-02],\n", - " [-1.5704e-02, -1.3467e-02, -1.0175e-02],\n", - " [-1.1507e-02, 9.9187e-03, 6.8689e-03]],\n", - " \n", - " [[-2.3748e-03, -6.8333e-03, -2.1954e-02],\n", - " [ 2.7879e-02, -2.3770e-03, -2.4249e-03],\n", - " [ 1.3316e-03, -2.1276e-03, 3.5368e-03]],\n", - " \n", - " [[ 4.1176e-03, 9.7428e-03, 5.2396e-03],\n", - " [-2.5708e-02, 2.1975e-02, 6.3758e-03],\n", - " [ 1.2244e-02, 8.3727e-03, 2.9957e-03]]],\n", - " \n", - " \n", - " [[[ 1.4465e-02, -1.4984e-02, -7.4286e-03],\n", - " [-1.6736e-02, -9.9520e-03, 3.2938e-02],\n", - " [-6.5588e-03, -1.0266e-02, 9.0892e-03]],\n", - " \n", - " [[ 7.4289e-03, -1.4877e-02, -8.2701e-03],\n", - " [-1.5337e-02, 1.7032e-02, -1.0953e-02],\n", - " [-1.0200e-02, 9.5796e-03, 2.4805e-04]],\n", - " \n", - " [[-4.5040e-03, 1.1648e-02, 6.5312e-03],\n", - " [ 3.9309e-03, 2.0466e-03, 5.2178e-03],\n", - " [-1.7061e-02, -2.7747e-03, 3.4949e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.8357e-02, -1.6988e-02, 1.2481e-02],\n", - " [-2.2475e-02, 1.0150e-02, -1.6380e-02],\n", - " [-2.5348e-03, 5.4042e-03, 1.7112e-03]],\n", - " \n", - " [[ 1.9661e-02, 2.7684e-02, -1.7579e-02],\n", - " [-1.7852e-02, 2.8303e-03, 5.3348e-03],\n", - " [-5.2718e-03, 7.7013e-03, 7.2804e-03]],\n", - " \n", - " [[ 9.1490e-03, 7.7883e-04, -1.7632e-02],\n", - " [-1.2583e-02, 8.7818e-03, 5.1676e-03],\n", - " [-3.6902e-04, -1.7178e-02, 1.3540e-02]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 8.1712e-04, 9.0576e-03, -1.9149e-02],\n", - " [-4.0932e-03, -2.1350e-03, 2.4558e-02],\n", - " [-1.8063e-03, 1.5787e-02, 2.4794e-02]],\n", - " \n", - " [[-5.6834e-03, 1.4906e-02, 1.7886e-02],\n", - " [-9.9328e-03, -1.4857e-03, 1.4721e-03],\n", - " [ 5.4608e-03, -1.5380e-02, 1.0360e-02]],\n", - " \n", - " [[ 2.0395e-02, -3.1558e-03, -4.9012e-03],\n", - " [ 1.2573e-02, 1.1591e-02, -8.4176e-03],\n", - " [-1.1168e-02, -1.3468e-02, 2.1639e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.7053e-02, -1.8436e-02, 2.2686e-02],\n", - " [-6.1383e-03, 1.5574e-02, -4.3106e-03],\n", - " [ 1.0139e-02, 2.5462e-04, 1.0165e-02]],\n", - " \n", - " [[ 1.4791e-02, 2.1350e-02, -1.8956e-03],\n", - " [ 2.2326e-02, 1.0025e-02, 1.4650e-02],\n", - " [ 2.1074e-02, -4.2362e-03, -6.1958e-03]],\n", - " \n", - " [[-2.2347e-02, 3.0744e-03, -1.0679e-02],\n", - " [ 8.9897e-04, -1.8132e-02, -7.3476e-03],\n", - " [ 1.2704e-02, -9.0810e-03, 7.2003e-03]]],\n", - " \n", - " \n", - " [[[-1.4371e-02, -2.7548e-03, 1.4248e-02],\n", - " [-7.4112e-03, -1.0907e-02, -2.5385e-02],\n", - " [-2.5373e-02, 2.7448e-03, 5.8521e-03]],\n", - " \n", - " [[-7.9273e-03, -2.7029e-03, 1.6600e-02],\n", - " [-1.2491e-02, 3.4288e-03, -7.3017e-03],\n", - " [ 1.8565e-03, 1.5789e-02, -5.2022e-03]],\n", - " \n", - " [[-2.9924e-03, -1.0030e-02, -5.9783e-03],\n", - " [ 2.2283e-02, -1.6772e-02, 1.1581e-02],\n", - " [ 2.6474e-02, -8.5180e-03, -3.0422e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1342e-03, 2.0211e-03, -1.0525e-02],\n", - " [-5.8896e-03, 2.5927e-02, -1.1140e-02],\n", - " [ 1.1061e-02, 1.7324e-02, -1.2695e-02]],\n", - " \n", - " [[-2.6018e-02, 2.3288e-03, 1.0441e-02],\n", - " [-1.5647e-02, -1.6925e-03, 1.2329e-02],\n", - " [ 3.0129e-03, -1.7816e-02, 1.0631e-02]],\n", - " \n", - " [[-1.5661e-02, -1.9536e-03, 5.5617e-03],\n", - " [ 6.3089e-04, 2.1462e-02, 1.8862e-02],\n", - " [-2.0530e-02, -1.4116e-02, 3.1988e-04]]],\n", - " \n", - " \n", - " [[[-1.4379e-02, 1.0504e-03, -1.8726e-03],\n", - " [ 1.1856e-02, -5.4930e-03, 3.6104e-03],\n", - " [ 9.3104e-03, 3.9445e-03, -6.1743e-03]],\n", - " \n", - " [[ 9.6196e-04, -1.0889e-02, 3.9305e-03],\n", - " [ 1.3188e-02, -4.7789e-03, -3.3596e-03],\n", - " [-2.0044e-03, 7.3119e-03, -1.8782e-02]],\n", - " \n", - " [[ 1.5632e-02, 1.0031e-02, 1.4872e-02],\n", - " [ 1.2690e-02, 2.0029e-03, 5.7593e-03],\n", - " [ 1.9563e-02, -1.9929e-02, 1.0187e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.3114e-02, 1.6794e-02, 6.3443e-03],\n", - " [-2.9880e-03, -6.8296e-04, -2.6330e-03],\n", - " [-8.2479e-03, -2.5342e-02, 2.0142e-02]],\n", - " \n", - " [[ 1.1973e-02, 9.4697e-03, 3.1806e-03],\n", - " [ 5.7394e-03, 8.0790e-03, 7.6858e-03],\n", - " [-1.7608e-03, 1.4017e-02, 1.1223e-02]],\n", - " \n", - " [[-1.4254e-03, -2.6417e-02, -1.7212e-02],\n", - " [ 9.6490e-03, 9.6998e-03, 1.3176e-02],\n", - " [-1.8310e-02, -1.2498e-02, -2.0969e-03]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer4.0.bn1.weight',\n", - " tensor([-5.9605e-08, 0.0000e+00, 1.1921e-07, 1.1921e-07, 0.0000e+00,\n", - " -1.7881e-07, -5.9605e-08, -1.7881e-07, 2.3842e-07, 1.1921e-07,\n", - " 2.3842e-07, -4.1723e-07, -2.3842e-07, -1.7881e-07, 0.0000e+00,\n", - " -3.5763e-07, -1.7881e-07, -4.1723e-07, 5.9605e-07, 2.3842e-07,\n", - " -2.3842e-07, 0.0000e+00, 7.1526e-07, 7.1526e-07, -1.7881e-07,\n", - " -4.1723e-07, 1.1921e-07, -5.9605e-08, -7.7486e-07, -5.3644e-07,\n", - " 2.3842e-07, -5.9605e-08, 2.3842e-07, -1.7881e-07, 1.1921e-07,\n", - " 7.1526e-07, 0.0000e+00, -5.3644e-07, 0.0000e+00, -1.1921e-07,\n", - " 3.5763e-07, -5.9605e-08, -2.3842e-07, 2.3842e-07, -1.7881e-07,\n", - " -3.5763e-07, -4.1723e-07, 1.1921e-07, -2.3842e-07, 0.0000e+00,\n", - " 1.1921e-07, -2.3842e-07, 3.5763e-07, -4.1723e-07, 0.0000e+00,\n", - " 8.3447e-07, 0.0000e+00, -6.5565e-07, -4.7684e-07, 3.5763e-07,\n", - " -2.3842e-07, 0.0000e+00, 3.5763e-07, -2.3842e-07, 1.1921e-07,\n", - " -1.7881e-07, 1.1921e-07, 1.1921e-07, 0.0000e+00, 1.1921e-07,\n", - " -5.3644e-07, -4.7684e-07, 0.0000e+00, -1.7881e-07, 0.0000e+00,\n", - " -5.3644e-07, -2.9802e-07, -1.1921e-07, 1.1921e-07, -5.9605e-08,\n", - " -1.1921e-07, 1.1921e-07, 3.5763e-07, 5.9605e-07, -2.9802e-07,\n", - " -5.9605e-08, 5.9605e-07, -5.9605e-08, 1.1921e-07, 2.3842e-07,\n", - " 2.3842e-07, 3.5763e-07, 3.5763e-07, -3.5763e-07, -5.9605e-08,\n", - " 2.3842e-07, 3.5763e-07, 0.0000e+00, -2.9802e-07, -2.9802e-07,\n", - " -5.9605e-08, 1.1921e-07, 4.7684e-07, -4.1723e-07, -1.7881e-07,\n", - " -1.7881e-07, -1.7881e-07, -3.5763e-07, 0.0000e+00, -5.3644e-07,\n", - " 3.5763e-07, -1.1921e-07, -4.1723e-07, -2.3842e-07, -1.7881e-07,\n", - " 4.7684e-07, -2.9802e-07, 2.3842e-07, -1.7881e-07, -2.3842e-07,\n", - " 4.7684e-07, -3.5763e-07, -1.7881e-07, -2.3842e-07, 2.3842e-07,\n", - " -1.1921e-07, -5.3644e-07, 2.3842e-07, -2.9802e-07, -1.7881e-07,\n", - " -4.1723e-07, -2.3842e-07, -5.9605e-07, -5.9605e-08, 0.0000e+00,\n", - " -5.9605e-08, -5.9605e-08, 0.0000e+00, -1.1921e-07, -4.1723e-07,\n", - " 1.1921e-07, -2.3842e-07, 0.0000e+00, 1.1921e-07, 1.1921e-07,\n", - " 0.0000e+00, -5.9605e-07, 3.5763e-07, -2.3842e-07, 2.3842e-07,\n", - " -3.5763e-07, 0.0000e+00, 2.3842e-07, -4.1723e-07, 0.0000e+00,\n", - " -3.5763e-07, -1.1921e-07, 0.0000e+00, 2.3842e-07, 3.5763e-07,\n", - " -2.3842e-07, -2.9802e-07, -5.9605e-08, 7.1526e-07, -5.9605e-08,\n", - " 2.3842e-07, -1.7881e-07, -1.7881e-07, -1.1921e-07, -2.3842e-07,\n", - " 1.1921e-07, -5.9605e-08, 3.5763e-07, -2.9802e-07, 1.1921e-07,\n", - " -2.9802e-07, 1.1921e-07, -1.7881e-07, -5.9605e-07, 4.7684e-07,\n", - " -4.7684e-07, 1.1921e-07, 1.1921e-07, -2.9802e-07, -1.7881e-07,\n", - " 1.1921e-07, -3.5763e-07, -4.1723e-07, 0.0000e+00, 7.1526e-07,\n", - " -1.1921e-07, 0.0000e+00, -5.9605e-08, 2.3842e-07, 3.5763e-07,\n", - " 2.3842e-07, 2.3842e-07, -3.5763e-07, 2.3842e-07, 2.3842e-07,\n", - " -2.3842e-07, 2.3842e-07, 0.0000e+00, -2.9802e-07, 1.1921e-07,\n", - " 3.5763e-07, -3.5763e-07, 2.3842e-07, -5.3644e-07, -5.9605e-08,\n", - " 1.1921e-07, 0.0000e+00, 4.7684e-07, -5.9605e-08, 2.3842e-07,\n", - " -5.9605e-08, -1.7881e-07, 0.0000e+00, 4.7684e-07, 5.9605e-07,\n", - " -5.9605e-08, 4.7684e-07, -4.1723e-07, -5.9605e-08, 5.9605e-07,\n", - " -1.1921e-07, -4.7684e-07, 2.3842e-07, -2.9802e-07, 1.1921e-07,\n", - " -1.7881e-07, -1.7881e-07, -1.1921e-07, 4.7684e-07, 0.0000e+00,\n", - " 4.7684e-07, 0.0000e+00, -5.9605e-08, 1.1921e-07, 1.1921e-07,\n", - " 5.9605e-07, 2.3842e-07, -1.1921e-07, -1.1921e-07, 0.0000e+00,\n", - " 3.5763e-07, -2.3842e-07, -3.5763e-07, -5.9605e-08, 1.1921e-07,\n", - " 2.3842e-07, 4.7684e-07, 0.0000e+00, 2.3842e-07, -5.9605e-08,\n", - " -5.3644e-07, -2.3842e-07, 4.7684e-07, 5.9605e-07, -3.5763e-07,\n", - " 4.7684e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07, 0.0000e+00,\n", - " 2.3842e-07, 4.7684e-07, -1.7881e-07, -1.7881e-07, -5.9605e-08,\n", - " 2.3842e-07, 1.1921e-07, 0.0000e+00, -2.3842e-07, -4.7684e-07,\n", - " 2.3842e-07, 0.0000e+00, -1.7881e-07, 1.1921e-07, 2.3842e-07,\n", - " 1.1921e-07, 1.1921e-07, 5.9605e-07, -1.7881e-07, 1.1921e-07,\n", - " 0.0000e+00, 0.0000e+00, 0.0000e+00, -3.5763e-07, 1.1921e-07,\n", - " 2.3842e-07, -2.9802e-07, -5.9605e-08, -5.9605e-08, -4.7684e-07,\n", - " 2.3842e-07, 0.0000e+00, -5.9605e-08, -1.7881e-07, 3.5763e-07,\n", - " 2.3842e-07, 1.1921e-07, 0.0000e+00, -2.9802e-07, -6.5565e-07,\n", - " 1.1921e-07, -4.1723e-07, -5.9605e-08, -1.1921e-07, -5.9605e-08,\n", - " 5.9605e-07, -5.9605e-08, 2.3842e-07, 1.1921e-07, -4.7684e-07,\n", - " 2.3842e-07, 0.0000e+00, 2.3842e-07, 0.0000e+00, -3.5763e-07,\n", - " 3.5763e-07, 0.0000e+00, 2.3842e-07, 4.7684e-07, 0.0000e+00,\n", - " 1.1921e-07, -5.9605e-08, 3.5763e-07, 1.1921e-07, 4.7684e-07,\n", - " -2.3842e-07, -4.7684e-07, -3.5763e-07, 1.1921e-07, 4.7684e-07,\n", - " 3.5763e-07, -1.1921e-07, -2.9802e-07, 3.5763e-07, 0.0000e+00,\n", - " 3.5763e-07, 3.5763e-07, -1.1921e-07, -4.1723e-07, -5.9605e-07,\n", - " 4.7684e-07, 1.1921e-07, 7.1526e-07, 2.3842e-07, -5.3644e-07,\n", - " -4.1723e-07, 0.0000e+00, 3.5763e-07, -4.1723e-07, -5.9605e-08,\n", - " 2.3842e-07, -3.5763e-07, -1.7881e-07, -2.3842e-07, -1.7881e-07,\n", - " -5.9605e-08, -4.1723e-07, -5.9605e-08, -4.1723e-07, -3.5763e-07,\n", - " 3.5763e-07, 2.3842e-07, -2.3842e-07, 2.3842e-07, -5.3644e-07,\n", - " -2.3842e-07, -6.5565e-07, 3.5763e-07, 3.5763e-07, -1.7881e-07,\n", - " -1.7881e-07, 4.7684e-07, 0.0000e+00, -5.9605e-08, 0.0000e+00,\n", - " 2.3842e-07, -4.7684e-07, -5.9605e-08, 7.1526e-07, 1.1921e-07,\n", - " -2.3842e-07, -1.1921e-07, 4.7684e-07, 4.7684e-07, 7.1526e-07,\n", - " 2.3842e-07, -5.9605e-08, 2.3842e-07, 5.9605e-07, -1.7881e-07,\n", - " -1.1921e-07, 0.0000e+00, -4.1723e-07, -1.7881e-07, 0.0000e+00,\n", - " 1.1921e-07, -4.7684e-07, -4.1723e-07, 2.3842e-07, 3.5763e-07,\n", - " 0.0000e+00, -2.3842e-07, 0.0000e+00, 1.1921e-07, 1.1921e-07,\n", - " -2.3842e-07, 5.9605e-07, -3.5763e-07, -1.1921e-07, -1.7881e-07,\n", - " -5.9605e-08, 3.5763e-07, 0.0000e+00, -2.9802e-07, 0.0000e+00,\n", - " 0.0000e+00, 0.0000e+00, -5.9605e-08, 5.9605e-07, 0.0000e+00,\n", - " 0.0000e+00, 9.5367e-07, 0.0000e+00, -1.7881e-07, -6.5565e-07,\n", - " 3.5763e-07, -1.7881e-07, -1.1921e-07, -1.1921e-07, 4.7684e-07,\n", - " -2.9802e-07, -1.1921e-07, -2.3842e-07, 8.3447e-07, -7.1526e-07,\n", - " 0.0000e+00, -1.1921e-07, -2.9802e-07, 3.5763e-07, 4.7684e-07,\n", - " 2.3842e-07, -2.3842e-07, -1.1921e-07, 1.1921e-07, 4.7684e-07,\n", - " -5.9605e-08, 2.3842e-07, 0.0000e+00, 7.1526e-07, 2.3842e-07,\n", - " -5.9605e-08, 4.7684e-07, 0.0000e+00, 2.3842e-07, 1.1921e-07,\n", - " 2.3842e-07, -3.5763e-07, -1.7881e-07, 3.5763e-07, 4.7684e-07,\n", - " 4.7684e-07, -3.5763e-07, -4.1723e-07, -5.3644e-07, 3.5763e-07,\n", - " -4.1723e-07, -8.3447e-07, -4.7684e-07, 0.0000e+00, 0.0000e+00,\n", - " -5.9605e-08, -5.3644e-07, -2.3842e-07, -3.5763e-07, 1.1921e-07,\n", - " -2.9802e-07, -6.5565e-07, 3.5763e-07, -1.1921e-07, 3.5763e-07,\n", - " 1.1921e-07, -1.1921e-07, 5.9605e-07, 8.3447e-07, 2.3842e-07,\n", - " 1.1921e-07, 0.0000e+00, 1.1921e-07, -1.1921e-07, -4.1723e-07,\n", - " 0.0000e+00, -2.9802e-07, -2.3842e-07, -1.1921e-07, -2.3842e-07,\n", - " 3.5763e-07, 5.9605e-07, 0.0000e+00, -5.9605e-08, -1.1921e-07,\n", - " 0.0000e+00, 2.3842e-07, -4.1723e-07, 1.1921e-07, -1.1921e-07,\n", - " -1.7881e-07, 7.1526e-07], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.bn1.bias',\n", - " tensor([ 3.1726e-08, -1.7731e-07, 9.9779e-09, -6.4998e-08, -1.0859e-07,\n", - " 1.0166e-07, 2.5536e-07, -1.3389e-08, 8.9986e-08, 1.9249e-08,\n", - " -4.1351e-07, -2.6128e-07, 8.3407e-08, -1.9537e-07, -1.3523e-07,\n", - " -4.3386e-07, -1.1622e-07, -2.2359e-07, -5.8473e-08, 3.5024e-07,\n", - " -1.8368e-07, -1.7182e-08, 5.2150e-07, 4.4955e-07, -1.5387e-07,\n", - " -2.8533e-07, 1.5760e-07, 8.1673e-08, -4.0682e-07, -1.9296e-07,\n", - " 2.1261e-07, -1.4812e-08, 2.2860e-07, -8.1006e-08, 1.6673e-07,\n", - " 4.5937e-07, -4.7240e-08, -2.4283e-07, -4.9272e-09, -3.4337e-07,\n", - " 2.5559e-07, -1.3976e-08, 2.6116e-07, 8.7624e-08, -1.7782e-07,\n", - " -1.2930e-07, -1.2506e-07, 6.7839e-08, -1.2276e-07, 4.3874e-08,\n", - " 9.7849e-08, 2.1440e-08, -8.0827e-08, -5.4108e-07, -8.0169e-08,\n", - " 3.6907e-07, 2.6496e-08, -2.6881e-07, -3.3249e-07, 4.5289e-07,\n", - " -1.3388e-07, 7.2097e-08, -4.8439e-08, 3.0130e-08, 1.9556e-07,\n", - " -4.3262e-07, 1.6216e-07, -2.1796e-07, 2.3822e-07, -1.6489e-07,\n", - " -4.5521e-07, -2.6305e-07, 1.9379e-07, -6.8674e-09, -1.1466e-07,\n", - " 1.2936e-07, 1.0731e-07, -3.0371e-07, 2.8924e-07, -4.0951e-08,\n", - " -5.1409e-07, 6.2814e-07, 3.0467e-07, 3.7954e-07, 1.2780e-07,\n", - " -2.8808e-07, 2.2043e-07, 2.3640e-07, -2.3760e-07, -5.5569e-08,\n", - " -7.0060e-08, 1.5659e-07, 1.9410e-07, -4.0949e-08, -3.2050e-08,\n", - " 2.8438e-07, 2.2002e-07, -8.4607e-09, -5.1829e-07, -3.3730e-07,\n", - " -2.8925e-07, 1.3643e-07, 3.6955e-07, 1.8995e-07, 3.9761e-08,\n", - " -1.3284e-07, -2.5212e-07, -1.9680e-07, -9.3926e-08, -2.3736e-07,\n", - " 5.6697e-08, -3.6571e-07, -4.4511e-07, -1.1081e-07, -1.4395e-07,\n", - " -1.1706e-07, -3.7293e-07, -7.7388e-08, -1.8207e-07, -3.4861e-07,\n", - " 1.9085e-07, -2.4633e-07, -1.3726e-07, -7.7572e-08, 1.3992e-07,\n", - " 1.5841e-07, -4.8458e-07, 6.7845e-07, -2.3140e-07, 2.3232e-07,\n", - " -4.2101e-07, -2.0065e-07, -4.1939e-07, 1.1340e-07, -4.8612e-08,\n", - " -1.0231e-07, 1.8115e-08, -4.2669e-07, 1.2405e-07, 1.4445e-07,\n", - " 8.9523e-09, -1.4322e-07, 1.4196e-07, 1.0362e-07, 4.5899e-08,\n", - " 2.0909e-08, -1.4452e-07, -2.2955e-07, -1.4234e-07, -3.3185e-08,\n", - " -1.1197e-07, 2.2181e-08, -1.3756e-07, -2.6207e-07, 9.9575e-08,\n", - " -3.1685e-07, 2.4888e-07, 2.5448e-07, 1.3901e-07, 1.4122e-07,\n", - " -2.6080e-07, -4.4499e-08, 1.7342e-07, 2.7159e-07, -1.2643e-07,\n", - " 1.7036e-07, -6.3905e-07, -1.9381e-07, 1.4063e-07, -3.4839e-07,\n", - " 9.9977e-08, 4.2366e-07, 1.0504e-07, -3.6718e-07, 1.6732e-07,\n", - " -2.7617e-07, 5.5355e-08, -3.3797e-07, -5.1699e-07, 1.2824e-07,\n", - " -1.9716e-07, 2.4884e-07, 6.7651e-08, -1.8893e-07, -1.5658e-07,\n", - " 4.7686e-07, -1.4786e-07, -1.3890e-07, -2.4762e-07, 1.0687e-07,\n", - " -1.2919e-07, 8.3134e-08, -1.0985e-07, 2.2108e-07, 3.3113e-07,\n", - " 1.3139e-07, 3.1832e-07, 5.4610e-09, 2.8666e-07, 2.2507e-07,\n", - " -1.3751e-07, 6.0405e-08, 2.8325e-07, -1.8085e-07, -3.8834e-08,\n", - " 5.2421e-07, -1.6108e-07, -1.5174e-07, -1.1893e-07, -2.0988e-07,\n", - " -2.3623e-08, -1.2304e-08, 3.3319e-07, -1.0587e-07, 2.7317e-07,\n", - " -6.2978e-08, -4.8378e-07, 1.2936e-08, 2.5831e-07, 2.5507e-09,\n", - " 2.8688e-07, 9.7795e-08, -1.4314e-07, -3.9753e-07, 3.5461e-07,\n", - " -4.0676e-07, -3.3493e-07, -3.1406e-08, -1.3037e-07, -1.8069e-07,\n", - " -3.4912e-07, -1.9761e-07, -3.6452e-08, 3.9209e-08, -4.1246e-08,\n", - " 2.9505e-07, 6.6562e-08, -8.9539e-09, -9.8791e-08, -2.1552e-07,\n", - " 4.6567e-07, -1.8645e-07, 1.2557e-07, -5.9367e-08, 1.6528e-07,\n", - " 3.6556e-07, -1.9133e-07, -6.1086e-08, -1.9499e-07, 1.2546e-07,\n", - " 2.4251e-07, 4.1227e-07, -7.8549e-08, 7.9225e-09, 7.4521e-08,\n", - " -2.0761e-07, -3.0494e-07, 4.0810e-07, 4.0615e-07, -1.6896e-07,\n", - " 3.9878e-07, 1.1604e-07, -8.1415e-09, 8.2245e-08, 2.8860e-08,\n", - " 1.8276e-07, 9.5420e-08, -3.5587e-08, -1.3545e-07, -1.6446e-07,\n", - " -3.7396e-08, 8.8892e-09, -2.4477e-07, 1.9488e-07, -1.8547e-07,\n", - " -2.4438e-08, 9.3581e-08, -5.6522e-07, 5.0627e-09, -1.8690e-07,\n", - " 4.0675e-07, 1.6759e-07, 4.4454e-07, 1.6735e-07, 3.4398e-08,\n", - " 1.1073e-07, -1.0062e-07, 1.4148e-07, -5.2365e-07, 8.8247e-08,\n", - " 2.4630e-07, -3.7311e-07, -2.7261e-08, 1.5593e-07, -5.6472e-07,\n", - " 1.7655e-07, 1.6060e-07, -2.1312e-07, -8.2568e-08, -1.9649e-07,\n", - " -1.4583e-07, 1.7356e-07, -2.3207e-07, -1.1145e-07, -3.8427e-07,\n", - " 2.9967e-07, -3.9306e-08, -6.6338e-08, -2.4694e-07, 4.0503e-08,\n", - " 2.0625e-07, 3.3832e-08, 1.4000e-07, 9.9012e-08, -1.4842e-07,\n", - " 6.9324e-08, 5.4214e-08, 3.3105e-07, -2.3108e-07, -2.9612e-07,\n", - " -2.5485e-08, -2.7959e-07, 1.1182e-07, 2.3020e-07, -1.0539e-07,\n", - " 4.2965e-07, -3.2165e-07, 2.5231e-07, 1.2803e-07, -2.2698e-08,\n", - " -2.8631e-07, -4.0032e-07, -4.7990e-07, 3.8790e-07, 5.5012e-08,\n", - " 2.1181e-07, -3.4345e-07, -1.0273e-07, 3.3141e-07, 3.9423e-08,\n", - " 3.9785e-07, 3.5342e-07, -1.7269e-07, -3.2614e-07, -4.4307e-07,\n", - " 5.1917e-07, 1.4295e-07, 3.0101e-07, 2.0181e-07, -4.3180e-07,\n", - " -7.5416e-09, -2.3724e-07, 2.5669e-07, -3.0930e-07, -2.7942e-08,\n", - " -2.2161e-07, -2.3573e-07, -5.0254e-07, -4.3599e-09, -2.7798e-08,\n", - " 2.5370e-07, -1.5972e-07, -4.3477e-08, -3.2825e-07, -8.7036e-08,\n", - " 1.0273e-07, 5.6967e-08, -2.0524e-07, 2.6476e-07, -1.1129e-08,\n", - " -2.2975e-07, -2.2688e-07, 2.2536e-07, 9.0293e-08, 5.6805e-08,\n", - " -5.3037e-07, 6.6778e-07, 2.4106e-07, -2.1485e-07, -7.2225e-08,\n", - " 3.0555e-07, 1.1304e-07, 2.7074e-07, 5.3919e-07, 2.7431e-07,\n", - " -6.5594e-09, 7.0458e-08, 2.2255e-07, 4.8071e-07, 2.7403e-07,\n", - " 1.5821e-07, 2.5005e-07, 3.4690e-07, 3.0483e-07, 3.1065e-07,\n", - " -2.5807e-07, -1.0820e-07, -1.7822e-07, -1.4796e-07, 9.4314e-08,\n", - " 1.9424e-07, -4.5983e-07, -3.7128e-08, 4.3416e-07, 2.7088e-07,\n", - " 1.7964e-07, -2.7266e-07, 2.3749e-07, 3.9147e-07, 3.3245e-07,\n", - " -3.2510e-07, 1.8839e-07, -1.8541e-07, -2.7306e-07, -2.7169e-07,\n", - " -2.6862e-07, 2.5923e-07, -5.9224e-09, 1.8698e-07, -1.0680e-07,\n", - " -3.2318e-07, -8.9712e-08, -1.6405e-07, 5.8242e-07, 2.0582e-07,\n", - " 2.5597e-07, 6.6531e-07, 3.3725e-07, 1.6997e-07, -5.5700e-07,\n", - " 1.1019e-07, -2.9223e-07, -1.1995e-07, 1.1060e-08, 2.2936e-07,\n", - " -4.6518e-07, -5.1757e-08, -2.9770e-08, 3.7894e-07, -1.8620e-07,\n", - " -4.7999e-07, -7.0762e-08, -3.0505e-08, -1.0269e-07, -4.2373e-08,\n", - " 9.0745e-08, 2.0844e-07, -4.4592e-07, 1.2248e-07, 3.2833e-07,\n", - " -1.6091e-07, -5.9777e-08, 2.4343e-07, 2.8965e-07, 2.7996e-07,\n", - " -2.8026e-07, 1.2297e-07, -1.3367e-07, 1.1679e-07, -3.0673e-08,\n", - " 2.9914e-08, -3.7007e-07, 9.5431e-08, 2.8178e-07, 4.1163e-07,\n", - " 3.4943e-07, -1.7463e-07, -1.4960e-07, -3.9748e-07, 3.0402e-07,\n", - " 6.0561e-10, -2.5993e-07, -5.3542e-07, 8.5851e-08, 3.7541e-08,\n", - " -5.6760e-08, -3.6608e-07, -1.1721e-07, -2.7867e-07, 4.2660e-07,\n", - " -3.0838e-09, -5.5235e-07, 2.5093e-07, -1.1648e-07, 1.9175e-08,\n", - " -4.4914e-07, -1.0806e-08, 3.2415e-07, 7.5054e-07, -8.4961e-08,\n", - " 1.1927e-07, 5.3004e-09, 1.1822e-07, -6.2535e-08, -1.9777e-07,\n", - " 1.4558e-07, -1.3232e-07, -2.5719e-07, 1.0912e-07, -4.6794e-07,\n", - " 3.1465e-07, 3.9155e-07, -2.1163e-07, -6.1621e-08, 4.9127e-08,\n", - " 1.4371e-08, 5.8589e-07, -3.0203e-07, 3.2204e-08, -3.6141e-07,\n", - " -2.1699e-07, 5.8263e-07], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.conv2.weight',\n", - " tensor([[[[-8.8919e-03, -5.5261e-03, -1.1649e-02],\n", - " [-3.2865e-03, -3.2163e-03, 2.1970e-03],\n", - " [-9.5255e-03, -1.4290e-02, 1.0239e-02]],\n", - " \n", - " [[-8.4998e-03, 6.0042e-03, -1.1939e-02],\n", - " [ 1.4830e-02, 3.7390e-03, 1.3846e-02],\n", - " [ 1.1486e-02, 4.7125e-03, -1.0030e-02]],\n", - " \n", - " [[-8.3829e-03, 7.1931e-03, 3.8290e-03],\n", - " [-1.4467e-02, 1.2313e-02, 3.8561e-05],\n", - " [-1.3615e-02, -4.5642e-03, 1.5877e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.8610e-03, -1.3229e-02, 5.3278e-03],\n", - " [-1.2192e-02, 5.5769e-04, -2.9691e-04],\n", - " [ 2.3656e-03, -4.9448e-03, -3.2270e-03]],\n", - " \n", - " [[-7.9613e-04, 8.4969e-03, 5.2952e-03],\n", - " [ 1.5548e-02, 8.6120e-03, -1.1546e-02],\n", - " [-5.0554e-03, -4.9455e-03, -1.2913e-02]],\n", - " \n", - " [[ 2.3799e-03, 6.1107e-03, 1.0281e-02],\n", - " [ 1.4803e-03, -7.1119e-03, -1.9499e-02],\n", - " [-1.1962e-02, 2.1622e-02, 1.0925e-02]]],\n", - " \n", - " \n", - " [[[ 1.4340e-02, 7.1889e-03, 7.5618e-03],\n", - " [-1.3251e-02, 3.6634e-03, 3.7104e-04],\n", - " [ 4.6030e-03, -2.0175e-02, -6.0965e-03]],\n", - " \n", - " [[ 1.2104e-02, -9.1580e-03, 8.1003e-03],\n", - " [-1.2511e-02, -1.5410e-02, 6.9187e-03],\n", - " [ 3.7470e-03, -2.3987e-03, -7.4581e-03]],\n", - " \n", - " [[-5.2357e-03, -1.8020e-03, -1.4156e-02],\n", - " [ 7.3226e-03, -2.2034e-04, 7.5318e-04],\n", - " [-9.4075e-03, -4.4618e-03, -1.6356e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 8.0691e-04, -1.6836e-02, 1.4165e-02],\n", - " [-1.4672e-02, -3.1906e-03, -1.0278e-03],\n", - " [ 4.1163e-03, -3.3362e-03, 1.3807e-02]],\n", - " \n", - " [[-2.7671e-03, -5.6578e-03, -1.0477e-02],\n", - " [ 6.5509e-03, 1.2876e-02, -1.5242e-02],\n", - " [-3.4014e-03, -1.0206e-02, -9.1380e-03]],\n", - " \n", - " [[ 1.4649e-02, 1.1887e-02, -3.7188e-03],\n", - " [ 1.2284e-02, -1.1505e-02, -1.2666e-02],\n", - " [-2.5857e-03, -8.7228e-03, -2.0777e-03]]],\n", - " \n", - " \n", - " [[[ 1.2072e-02, -2.3061e-03, -6.7005e-03],\n", - " [-8.0823e-03, -8.3503e-03, 6.9081e-03],\n", - " [-9.8445e-03, -2.6542e-03, 2.6641e-03]],\n", - " \n", - " [[ 2.7775e-03, -2.5929e-03, 1.9189e-02],\n", - " [-2.7051e-03, 5.8931e-03, 3.4024e-04],\n", - " [-1.0870e-02, -1.3271e-02, 1.2677e-02]],\n", - " \n", - " [[-9.7288e-04, 4.6576e-04, 6.6519e-04],\n", - " [-1.8496e-03, 1.5163e-02, -1.3952e-02],\n", - " [-6.4673e-03, 4.5288e-03, 5.0748e-04]],\n", - " \n", - " ...,\n", - " \n", - " [[ 8.5364e-03, 9.1448e-03, 8.0631e-04],\n", - " [-2.2104e-03, -5.2943e-03, -1.3910e-02],\n", - " [-1.2475e-02, 4.3112e-03, 9.1181e-03]],\n", - " \n", - " [[-4.0816e-03, -2.2592e-03, 1.3574e-02],\n", - " [ 1.5750e-02, -1.5506e-02, -1.4866e-03],\n", - " [-1.6357e-02, 5.1772e-03, 9.1530e-03]],\n", - " \n", - " [[ 7.5529e-03, 2.9393e-03, -7.6150e-03],\n", - " [ 4.8107e-03, -1.7729e-02, -2.4929e-03],\n", - " [-3.1347e-04, -5.6910e-04, 4.8326e-03]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 3.1950e-03, 1.2405e-02, -4.2770e-03],\n", - " [ 8.9297e-03, 2.9347e-03, 5.5196e-03],\n", - " [-8.4849e-03, 4.7235e-03, 1.4928e-02]],\n", - " \n", - " [[-1.9742e-02, -3.4240e-03, 8.5688e-03],\n", - " [-9.5304e-03, 6.9871e-03, -4.9104e-03],\n", - " [-1.1383e-02, -1.4184e-03, 8.3761e-03]],\n", - " \n", - " [[ 1.1749e-02, -1.3858e-02, 7.7950e-03],\n", - " [-1.4731e-02, -1.3514e-03, 1.0907e-02],\n", - " [-7.3442e-03, 4.8755e-03, 7.4672e-03]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.2847e-02, 1.5669e-02, 1.3149e-02],\n", - " [-2.7236e-03, -5.3160e-03, -2.6364e-03],\n", - " [ 4.9001e-03, -7.4731e-03, 1.0864e-02]],\n", - " \n", - " [[ 6.3456e-03, 1.4196e-02, 3.2071e-03],\n", - " [ 1.7465e-03, -4.9140e-03, -9.9985e-03],\n", - " [-5.4781e-03, 3.3783e-03, 2.3971e-05]],\n", - " \n", - " [[-8.7421e-03, -4.3363e-03, -7.4800e-03],\n", - " [ 1.8229e-02, -1.3717e-03, 1.2374e-02],\n", - " [-7.2042e-03, 1.1575e-02, 1.8360e-03]]],\n", - " \n", - " \n", - " [[[ 8.4327e-03, 3.6422e-03, 2.9160e-03],\n", - " [-1.1756e-02, -9.3760e-03, 1.2245e-02],\n", - " [ 7.0258e-03, 4.5228e-03, -9.3750e-04]],\n", - " \n", - " [[ 7.4205e-03, 2.8433e-03, 1.7987e-03],\n", - " [ 3.5197e-03, -1.4728e-02, -6.8848e-03],\n", - " [ 4.6512e-03, 4.1685e-03, 4.2135e-04]],\n", - " \n", - " [[-1.8019e-02, 1.2760e-02, 1.1175e-02],\n", - " [-2.3896e-03, 1.3814e-02, 1.6097e-02],\n", - " [ 6.7313e-03, -5.1544e-03, -1.9122e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.5587e-02, 6.6637e-03, -9.2753e-03],\n", - " [-1.5219e-02, -1.4307e-02, 1.1896e-03],\n", - " [-1.3813e-02, 4.6334e-03, 2.9915e-03]],\n", - " \n", - " [[-1.4803e-02, -7.3671e-03, 1.1902e-02],\n", - " [ 5.0613e-03, 4.7833e-03, 3.3237e-03],\n", - " [ 7.2514e-03, 2.7958e-04, 1.5791e-02]],\n", - " \n", - " [[ 1.8511e-02, 5.9318e-03, -1.2440e-02],\n", - " [-6.7291e-03, -8.8253e-04, 4.5097e-04],\n", - " [ 5.8221e-03, 1.0105e-02, 2.6565e-03]]],\n", - " \n", - " \n", - " [[[-5.3214e-03, -3.0500e-03, 1.3938e-02],\n", - " [ 1.5274e-02, 1.0179e-02, 1.9662e-02],\n", - " [-9.6668e-05, 4.1659e-03, -6.4923e-03]],\n", - " \n", - " [[-1.1835e-02, -8.5019e-03, 1.4460e-03],\n", - " [ 1.6145e-03, 5.6016e-03, 5.1102e-03],\n", - " [-1.6813e-02, 1.4814e-02, 4.8468e-04]],\n", - " \n", - " [[-1.2867e-02, -1.7569e-03, -1.4131e-02],\n", - " [-4.2748e-03, 1.0950e-02, -2.5505e-03],\n", - " [-8.8524e-03, -7.2978e-03, -1.0411e-02]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.5008e-02, -3.9596e-03, -8.1293e-03],\n", - " [-1.4422e-02, 9.5751e-03, 1.0249e-02],\n", - " [-1.0301e-02, 3.5868e-03, -8.3649e-03]],\n", - " \n", - " [[ 8.8733e-04, 1.7633e-02, -1.1830e-02],\n", - " [-1.5473e-02, 9.8210e-03, -8.5470e-03],\n", - " [-6.6058e-03, 3.0551e-04, -5.6332e-03]],\n", - " \n", - " [[-4.1049e-03, 3.9279e-03, -1.3175e-02],\n", - " [-1.0068e-02, -2.7920e-03, 1.9161e-02],\n", - " [ 1.5485e-02, -6.6603e-03, -1.1391e-02]]]], device='cuda:0',\n", - " grad_fn=)),\n", - " ('layer4.0.bn2.weight',\n", - " tensor([-4.7684e-07, 2.3842e-07, -3.5763e-07, -1.7881e-07, 2.3842e-07,\n", - " -7.1526e-07, -2.9802e-07, 2.3842e-07, -3.5763e-07, -5.9605e-07,\n", - " 2.3842e-07, -1.7881e-07, 3.5763e-07, -5.9605e-08, 1.1921e-07,\n", - " -2.3842e-07, -1.7881e-07, 0.0000e+00, -4.1723e-07, -7.7486e-07,\n", - " 0.0000e+00, -6.5565e-07, -2.3842e-07, 2.3842e-07, -7.7486e-07,\n", - " 3.5763e-07, 2.3842e-07, -2.3842e-07, 4.7684e-07, -2.3842e-07,\n", - " -5.9605e-07, 0.0000e+00, -3.5763e-07, -5.9605e-08, 0.0000e+00,\n", - " 1.1921e-07, 0.0000e+00, 5.9605e-07, 3.5763e-07, -5.9605e-07,\n", - " 1.1921e-07, 1.1921e-07, -1.1921e-07, 8.3447e-07, 3.5763e-07,\n", - " 1.1921e-07, 3.5763e-07, 1.1921e-07, -3.5763e-07, 4.7684e-07,\n", - " -2.9802e-07, -4.1723e-07, -5.9605e-08, -5.3644e-07, 5.9605e-07,\n", - " -6.5565e-07, -2.3842e-07, -4.1723e-07, 7.1526e-07, -1.7881e-07,\n", - " 0.0000e+00, 1.1921e-07, 4.7684e-07, -5.9605e-08, -1.1921e-07,\n", - " -5.9605e-07, -3.5763e-07, 0.0000e+00, -1.1921e-07, 0.0000e+00,\n", - " -5.9605e-08, -9.5367e-07, -4.1723e-07, -2.9802e-07, 2.3842e-07,\n", - " -7.7486e-07, 2.3842e-07, -5.9605e-08, -3.5763e-07, 4.7684e-07,\n", - " 0.0000e+00, 4.7684e-07, -1.1921e-07, -5.9605e-08, 1.1921e-07,\n", - " -5.3644e-07, 1.1921e-07, 2.3842e-07, -7.7486e-07, -2.9802e-07,\n", - " -1.7881e-07, 0.0000e+00, -2.3842e-07, -1.7881e-07, -1.7881e-07,\n", - " 1.1921e-06, -1.7881e-07, 3.5763e-07, 2.3842e-07, 1.1921e-07,\n", - " -2.3842e-07, -5.9605e-07, -3.5763e-07, -1.0729e-06, -1.1921e-07,\n", - " 2.3842e-07, 3.5763e-07, -3.5763e-07, 4.7684e-07, -1.7881e-07,\n", - " 0.0000e+00, -7.7486e-07, -6.5565e-07, -2.3842e-07, 0.0000e+00,\n", - " 4.7684e-07, -2.9802e-07, 2.3842e-07, 1.1921e-07, -5.3644e-07,\n", - " 0.0000e+00, -1.1921e-07, -4.1723e-07, -4.1723e-07, 0.0000e+00,\n", - " -6.5565e-07, -4.1723e-07, 0.0000e+00, -1.7881e-07, 1.1921e-07,\n", - " 0.0000e+00, -5.9605e-07, 5.9605e-07, 1.1921e-07, 5.9605e-07,\n", - " -1.1921e-07, 1.1921e-07, -1.1921e-07, -4.7684e-07, -5.3644e-07,\n", - " -5.3644e-07, -8.3447e-07, -1.1921e-07, -5.9605e-08, -5.9605e-08,\n", - " 3.5763e-07, -7.1526e-07, 0.0000e+00, 1.0729e-06, 0.0000e+00,\n", - " 0.0000e+00, 2.3842e-07, -4.1723e-07, 4.7684e-07, -5.9605e-08,\n", - " 0.0000e+00, -5.3644e-07, 2.3842e-07, -5.9605e-07, 2.3842e-07,\n", - " -4.7684e-07, -1.1921e-07, -4.1723e-07, -2.3842e-07, -2.3842e-07,\n", - " 4.7684e-07, 4.7684e-07, -2.9802e-07, 0.0000e+00, 2.3842e-07,\n", - " -1.1921e-07, -2.3842e-07, -5.9605e-08, 2.3842e-07, 4.7684e-07,\n", - " 5.9605e-07, -2.3842e-07, 3.5763e-07, -2.3842e-07, 0.0000e+00,\n", - " -1.7881e-07, 1.1921e-07, -2.3842e-07, 2.3842e-07, 7.1526e-07,\n", - " 7.1526e-07, 1.1921e-07, 1.1921e-07, -2.9802e-07, -2.3842e-07,\n", - " 0.0000e+00, 4.7684e-07, -2.3842e-07, -1.1325e-06, -5.9605e-08,\n", - " -5.9605e-08, -3.5763e-07, -5.9605e-08, -1.7881e-07, -4.1723e-07,\n", - " 5.9605e-07, -3.5763e-07, 4.7684e-07, 1.1921e-07, -8.3447e-07,\n", - " 1.1921e-07, -1.0729e-06, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", - " 1.1921e-07, -2.3842e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07,\n", - " -5.3644e-07, 0.0000e+00, 4.7684e-07, -5.9605e-08, 2.3842e-07,\n", - " 0.0000e+00, -5.9605e-08, -1.1921e-07, -1.1921e-07, 0.0000e+00,\n", - " 3.5763e-07, -5.9605e-08, -4.1723e-07, 1.1921e-07, -4.1723e-07,\n", - " 2.3842e-07, -2.9802e-07, 1.1921e-07, 0.0000e+00, 3.5763e-07,\n", - " -1.1921e-07, -3.5763e-07, 1.1921e-07, 2.3842e-07, -1.7881e-07,\n", - " -8.3447e-07, 2.3842e-07, -2.9802e-07, 2.3842e-07, -5.9605e-08,\n", - " 2.3842e-07, -5.9605e-08, 0.0000e+00, 2.3842e-07, 2.3842e-07,\n", - " 5.9605e-07, 1.1921e-07, 2.3842e-07, -5.9605e-07, -1.7881e-07,\n", - " 0.0000e+00, 0.0000e+00, 2.3842e-07, 4.7684e-07, 0.0000e+00,\n", - " -4.1723e-07, 5.9605e-07, 2.3842e-07, 3.5763e-07, -4.1723e-07,\n", - " 0.0000e+00, 3.5763e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07,\n", - " 2.3842e-07, -5.9605e-07, 7.1526e-07, -1.1325e-06, 5.9605e-07,\n", - " -1.7881e-07, -5.9605e-08, -2.9802e-07, -5.9605e-08, 1.1921e-07,\n", - " 2.3842e-07, -2.9802e-07, -2.9802e-07, -8.3447e-07, 7.1526e-07,\n", - " 3.5763e-07, 2.3842e-07, 3.5763e-07, -2.9802e-07, -4.1723e-07,\n", - " -1.1921e-07, -5.9605e-07, -1.1921e-07, -2.9802e-07, 3.5763e-07,\n", - " -3.5763e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07, -4.1723e-07,\n", - " -4.7684e-07, -4.1723e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", - " 3.5763e-07, 1.1921e-07, 3.5763e-07, 1.1921e-07, -1.7881e-07,\n", - " -3.5763e-07, -4.7684e-07, 1.1921e-07, 3.5763e-07, -2.9802e-07,\n", - " 0.0000e+00, 4.7684e-07, 2.3842e-07, -5.9605e-08, -2.3842e-07,\n", - " 4.7684e-07, 7.1526e-07, 0.0000e+00, -2.3842e-07, 0.0000e+00,\n", - " -1.7881e-07, 1.1921e-07, -1.1921e-07, 0.0000e+00, -1.7881e-07,\n", - " 1.1921e-07, -2.9802e-07, -4.7684e-07, 5.9605e-07, 1.1921e-07,\n", - " 4.7684e-07, -7.1526e-07, -5.9605e-08, 2.3842e-07, 2.3842e-07,\n", - " 0.0000e+00, -1.7881e-07, 2.3842e-07, 0.0000e+00, 1.1921e-07,\n", - " 1.1921e-07, 1.1921e-07, -1.1921e-07, 2.3842e-07, 3.5763e-07,\n", - " 2.3842e-07, 4.7684e-07, 2.3842e-07, -1.1921e-07, -1.1921e-07,\n", - " 5.9605e-07, -1.7881e-07, 0.0000e+00, -2.3842e-07, -3.5763e-07,\n", - " 1.1921e-07, 3.5763e-07, -1.1921e-07, 0.0000e+00, -5.9605e-08,\n", - " -2.3842e-07, 5.9605e-07, 1.1921e-07, -1.7881e-07, 1.1921e-07,\n", - " -1.7881e-07, -1.3113e-06, 1.1921e-07, -3.5763e-07, -5.9605e-07,\n", - " -2.9802e-07, 1.1921e-07, 4.7684e-07, 3.5763e-07, -2.3842e-07,\n", - " 2.3842e-07, 1.1921e-07, 7.1526e-07, 3.5763e-07, -5.9605e-08,\n", - " -2.3842e-07, 2.3842e-07, 3.5763e-07, -5.9605e-08, -3.5763e-07,\n", - " 4.7684e-07, -5.9605e-08, 5.9605e-07, -9.5367e-07, -1.1921e-07,\n", - " 2.3842e-07, -2.9802e-07, 1.1921e-07, -5.3644e-07, 1.1921e-07,\n", - " -4.7684e-07, 3.5763e-07, -3.5763e-07, 2.3842e-07, 1.1921e-07,\n", - " -5.3644e-07, -2.3842e-07, 3.5763e-07, -7.7486e-07, -4.7684e-07,\n", - " -4.7684e-07, -1.1921e-07, -1.1921e-07, -5.9605e-07, 1.1921e-07,\n", - " -2.9802e-07, -9.5367e-07, -3.5763e-07, -2.3842e-07, 0.0000e+00,\n", - " 2.3842e-07, 5.9605e-07, -3.5763e-07, -5.3644e-07, -1.7881e-07,\n", - " -5.3644e-07, 3.5763e-07, 2.3842e-07, -1.1921e-07, -1.7881e-07,\n", - " -3.5763e-07, -1.1921e-07, 4.7684e-07, 4.7684e-07, 3.5763e-07,\n", - " 1.1921e-07, 1.1921e-07, 3.5763e-07, -4.1723e-07, -4.7684e-07,\n", - " 1.1921e-07, -2.3842e-07, 2.3842e-07, 3.5763e-07, 2.3842e-07,\n", - " 0.0000e+00, 1.1921e-07, 5.9605e-07, -1.2517e-06, 0.0000e+00,\n", - " 4.7684e-07, -2.9802e-07, 4.7684e-07, -1.7881e-07, 4.7684e-07,\n", - " 1.1921e-07, 3.5763e-07, -2.3842e-07, -5.3644e-07, 3.5763e-07,\n", - " 1.1921e-07, 3.5763e-07, -1.1921e-07, -2.9802e-07, -2.3842e-07,\n", - " 1.1921e-07, -1.7881e-07, -8.9407e-07, 1.1921e-07, 3.5763e-07,\n", - " -4.7684e-07, -2.3842e-07, 1.1921e-07, 1.1921e-07, -1.1921e-07,\n", - " 1.1921e-07, 1.1921e-07, -1.1921e-07, 5.9605e-07, -5.9605e-08,\n", - " 4.7684e-07, 1.1921e-07, 0.0000e+00, 2.3842e-07, 1.1921e-07,\n", - " -2.3842e-07, 2.3842e-07, 3.5763e-07, -1.1921e-07, 0.0000e+00,\n", - " -3.5763e-07, -2.9802e-07, 4.7684e-07, -5.9605e-08, 1.1921e-07,\n", - " 2.3842e-07, 1.1921e-07, -4.7684e-07, -5.9605e-08, 2.3842e-07,\n", - " 1.1921e-07, -2.9802e-07, -5.9605e-08, 5.9605e-07, 0.0000e+00,\n", - " 3.5763e-07, -7.7486e-07, 0.0000e+00, 1.1921e-07, 2.3842e-07,\n", - " -6.5565e-07, 3.5763e-07], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.bn2.bias',\n", - " tensor([-2.5952e-07, -5.2718e-08, -3.8981e-07, -1.2332e-07, 2.4942e-07,\n", - " -4.1001e-07, -1.8458e-07, 2.2059e-07, 3.3934e-11, -5.6743e-08,\n", - " 3.8876e-07, -2.2358e-07, 1.5773e-07, -3.4872e-07, -2.7905e-07,\n", - " -2.3630e-08, -3.0215e-08, -2.4326e-07, -2.1305e-07, -2.9157e-07,\n", - " -4.5004e-07, -3.3447e-07, -1.6505e-07, -2.8462e-07, -1.1983e-07,\n", - " 3.0670e-07, -1.7117e-07, 2.2260e-07, -1.8211e-07, -3.3789e-07,\n", - " -6.4035e-07, -3.4069e-07, -2.1152e-07, 6.1553e-08, -1.1388e-07,\n", - " 3.9278e-08, 6.4294e-07, 5.3870e-07, -2.0622e-07, -5.0324e-07,\n", - " -8.4042e-08, 1.2858e-07, -3.1810e-07, 8.5574e-07, 2.0792e-07,\n", - " -3.2931e-07, 4.0081e-07, 1.2134e-07, 1.0071e-07, 9.3611e-07,\n", - " -3.1569e-07, -1.4190e-07, -9.8398e-08, -4.4183e-07, 3.8909e-07,\n", - " -2.3117e-07, -3.3583e-07, -4.5050e-07, 3.9275e-07, 5.3557e-07,\n", - " -8.5330e-07, -4.7482e-08, -7.7420e-08, 4.5063e-08, 5.6890e-08,\n", - " -6.9403e-07, -6.6087e-07, -3.2312e-07, -1.5117e-07, -5.0290e-07,\n", - " -8.3522e-08, -6.2873e-07, -1.9139e-07, -4.7217e-08, -1.4439e-07,\n", - " -3.0978e-07, -3.0010e-07, -4.4070e-07, 6.1973e-08, 4.8330e-07,\n", - " -1.5466e-07, -6.7563e-08, -6.4472e-07, -3.3934e-08, -3.2509e-07,\n", - " 3.2013e-09, -1.0098e-08, 6.1731e-08, -9.7081e-08, 6.2362e-08,\n", - " 2.2826e-07, -8.8843e-07, -7.5161e-07, -4.3784e-07, 5.1198e-08,\n", - " 7.3383e-07, -8.6796e-08, 1.3520e-07, 2.7753e-07, -7.6281e-08,\n", - " 3.2856e-07, -8.5474e-07, 1.4127e-07, 1.6500e-07, -1.1475e-07,\n", - " -9.0230e-08, -3.4021e-08, -4.5122e-07, 1.0504e-07, -3.4202e-08,\n", - " -6.6387e-09, -4.7148e-07, -5.1771e-07, -3.5938e-07, 1.1671e-07,\n", - " 7.6556e-08, 1.0592e-08, 2.3111e-07, -1.5009e-07, -2.9490e-07,\n", - " -2.6290e-07, -2.8363e-07, -4.4743e-07, -4.2879e-07, 1.9960e-07,\n", - " -2.2103e-07, 5.0751e-07, 2.3843e-08, -2.6756e-07, -1.8564e-07,\n", - " -2.7111e-07, -1.7106e-07, 8.1063e-07, -4.9894e-07, 1.3639e-07,\n", - " -4.6950e-07, 1.0927e-07, -1.2955e-07, -4.4637e-07, -8.5186e-08,\n", - " -6.1976e-08, -5.0219e-08, -6.6460e-07, -1.8259e-07, -2.3600e-07,\n", - " -1.2455e-07, 1.3833e-07, 2.3173e-07, 1.3480e-07, 2.6061e-07,\n", - " -2.2656e-07, -2.9893e-07, -3.5553e-07, 4.9369e-07, 2.8315e-07,\n", - " -1.6586e-07, -7.3051e-07, 3.2315e-07, -2.5779e-07, 1.5562e-07,\n", - " -3.1320e-07, -5.5600e-08, 6.4858e-08, -4.3620e-07, -3.0921e-08,\n", - " 1.7959e-07, 4.7084e-07, -2.9651e-07, 1.8505e-08, 4.9029e-07,\n", - " -2.0760e-07, -5.8565e-07, 1.7540e-07, -2.5290e-08, 5.8386e-08,\n", - " 2.4527e-07, 8.4265e-08, 3.2661e-08, -2.2252e-08, -2.5374e-07,\n", - " -7.2610e-08, 1.9237e-07, -3.9457e-07, -8.6376e-08, 9.7661e-08,\n", - " 8.1748e-08, -1.8011e-07, 5.4112e-08, -2.4380e-07, 4.2298e-08,\n", - " -2.2257e-07, 2.4752e-07, -8.2461e-07, -4.9300e-07, -2.8506e-07,\n", - " 1.4784e-07, 4.9734e-08, -3.3205e-07, -1.1681e-07, -4.5991e-07,\n", - " -1.0016e-07, -4.8842e-07, 3.3371e-07, -4.3462e-07, -7.9114e-07,\n", - " -6.8228e-08, -4.7129e-07, -3.3589e-07, 2.6814e-07, 1.9327e-07,\n", - " 1.4206e-07, 3.3498e-08, -5.5501e-08, 1.7940e-07, 4.2684e-07,\n", - " 1.7147e-07, 5.6309e-08, -7.1572e-09, -3.8246e-08, -2.8099e-07,\n", - " 9.5294e-08, -4.6339e-07, 5.0270e-07, 1.0481e-08, 1.9457e-07,\n", - " -2.0168e-08, 8.2068e-08, -1.6883e-09, 2.6224e-07, -3.9644e-07,\n", - " 4.4707e-07, -8.3741e-08, -1.7601e-07, -1.7361e-07, 7.8584e-07,\n", - " 7.0257e-11, -1.2827e-07, 3.2551e-07, 6.4153e-07, -3.4501e-08,\n", - " -7.0563e-07, 1.2502e-07, -5.3899e-07, 6.2128e-07, 1.6744e-07,\n", - " 3.0381e-08, 3.1756e-07, 1.4472e-07, 3.9837e-07, -6.7176e-08,\n", - " 7.3281e-07, -3.6448e-07, -2.5766e-08, 9.8277e-09, -5.1608e-07,\n", - " -2.1131e-07, -4.9822e-07, 6.0511e-07, -2.0183e-08, 2.1366e-07,\n", - " -1.2711e-07, 3.8626e-07, 3.8279e-07, 7.5298e-07, -6.0340e-07,\n", - " -4.7458e-08, -7.7033e-08, -2.6878e-07, 1.7127e-07, -4.7343e-07,\n", - " -5.9129e-09, -6.3804e-07, 3.3628e-07, -2.4381e-07, 3.1775e-07,\n", - " -6.1137e-07, -4.6261e-07, -4.1723e-07, 5.3470e-07, 7.1795e-09,\n", - " -2.3043e-08, -1.2530e-07, -5.4000e-07, -6.3562e-07, 1.3268e-07,\n", - " 2.2244e-07, -1.7262e-07, 4.3062e-07, -1.3121e-07, -2.1163e-07,\n", - " 6.3334e-08, -6.1981e-07, 6.5072e-08, -2.1843e-07, 1.8716e-08,\n", - " 7.4588e-08, 4.0129e-07, -8.5405e-08, 6.3721e-08, -4.6985e-07,\n", - " -2.2629e-07, -4.9808e-07, 6.7194e-07, 1.3299e-07, -4.3199e-09,\n", - " 2.7960e-07, -3.2597e-07, -2.8505e-08, -1.2399e-07, -2.7097e-07,\n", - " 8.7115e-08, -6.0820e-07, -1.7309e-07, 3.7368e-07, -2.4287e-07,\n", - " -1.1012e-07, 1.2617e-08, 1.7739e-07, -2.6112e-07, -3.0125e-07,\n", - " -1.2478e-07, 2.5956e-07, -1.0944e-07, 1.7735e-07, -5.8064e-08,\n", - " -2.0758e-07, -2.8337e-07, -2.6004e-07, 3.2003e-08, -3.9736e-07,\n", - " 2.9050e-07, -2.3287e-08, -2.2765e-07, 4.0771e-07, 4.0802e-07,\n", - " 7.0819e-08, 1.4434e-07, -1.8711e-07, -2.4609e-08, 2.6868e-07,\n", - " -1.8774e-08, -2.0145e-07, 1.3446e-07, 5.9894e-07, -5.2298e-08,\n", - " 2.3941e-07, 2.3950e-07, -3.1209e-07, 3.9168e-07, -3.1503e-07,\n", - " -5.3224e-07, 4.3975e-07, -9.7231e-09, -3.8906e-07, -3.3958e-07,\n", - " 1.7475e-07, -1.2887e-07, -7.8835e-07, 6.8751e-08, -4.2049e-07,\n", - " 1.2126e-07, 8.8542e-08, -5.7822e-07, -4.5029e-07, -1.0936e-07,\n", - " -2.1707e-07, 3.0449e-07, 2.9369e-07, 1.6307e-07, -2.1861e-07,\n", - " -4.1288e-07, -9.9163e-07, 3.3039e-07, -1.5236e-07, -1.9777e-08,\n", - " 3.7243e-07, -3.4649e-07, -3.2635e-07, -2.5982e-07, -2.4515e-07,\n", - " 7.5304e-08, -6.0096e-08, 2.8725e-07, 9.5974e-08, 1.0231e-07,\n", - " -6.2956e-09, 9.3015e-08, 5.1241e-07, -3.7457e-07, -5.8680e-07,\n", - " 5.1277e-08, 1.5735e-07, 1.5098e-07, -4.7827e-07, 1.5020e-07,\n", - " 3.2007e-07, -1.1792e-07, 4.1907e-07, -5.4599e-07, 3.5650e-07,\n", - " -7.2770e-07, 2.4135e-07, -5.4777e-08, -1.6205e-07, 1.0242e-07,\n", - " -8.1226e-07, -2.6392e-07, -3.5215e-07, -5.7353e-07, -2.3653e-07,\n", - " -7.6270e-07, -1.0341e-07, -1.5335e-07, -4.0521e-07, 5.1136e-08,\n", - " -2.1407e-07, -1.1061e-06, -1.1568e-07, -1.8562e-07, -2.2229e-07,\n", - " 1.7121e-07, 4.0764e-07, -5.0914e-07, -1.5899e-07, 1.9196e-07,\n", - " -4.7060e-07, 1.8851e-07, -5.7792e-08, -4.6329e-07, -3.9686e-07,\n", - " -1.8117e-08, -1.4392e-07, 4.1560e-07, 4.7124e-07, -2.5345e-08,\n", - " -2.1026e-07, -1.2185e-07, -2.1304e-07, 1.5910e-09, -3.9998e-07,\n", - " -1.5126e-07, -2.4613e-07, -1.0463e-07, 1.4167e-07, 2.5327e-08,\n", - " 3.7949e-07, -5.3259e-07, 7.2235e-07, -1.0368e-06, -2.0211e-07,\n", - " 4.1569e-07, 5.9008e-09, 2.3589e-07, -1.1660e-07, 3.8977e-08,\n", - " -2.6139e-07, 2.5065e-07, -3.0728e-07, -2.3161e-07, 5.0300e-08,\n", - " 4.0945e-08, -3.4996e-08, -2.7710e-07, -4.3644e-09, -1.2612e-07,\n", - " -3.5672e-07, -3.0295e-07, -3.4237e-07, -1.6870e-07, 3.2721e-07,\n", - " -2.5081e-07, 1.2698e-07, 2.5012e-07, 4.2570e-07, -3.8390e-07,\n", - " 1.6247e-07, 1.5678e-07, -1.8048e-07, 2.3226e-07, 2.6242e-07,\n", - " 1.4829e-07, -1.9754e-07, -2.4445e-07, -1.5111e-07, -1.8443e-07,\n", - " -5.3174e-07, -4.4128e-08, 2.1757e-07, -4.6750e-07, -7.1977e-08,\n", - " -1.7918e-07, -3.1625e-07, 2.4026e-07, 7.8625e-08, -4.3738e-08,\n", - " -2.1923e-09, -5.7170e-07, -4.3710e-07, -2.2763e-07, 5.7661e-07,\n", - " -3.9948e-07, 1.6238e-07, 2.4817e-07, 4.2508e-07, -5.4188e-07,\n", - " 8.4595e-08, -2.3122e-07, 2.6185e-07, -7.8099e-08, -1.5002e-07,\n", - " -4.9218e-08, 2.9320e-07], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.shortcut.0.weight',\n", - " tensor([[[[-0.0539]],\n", - " \n", - " [[ 0.0183]],\n", - " \n", - " [[ 0.0007]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0457]],\n", - " \n", - " [[-0.0235]],\n", - " \n", - " [[ 0.0321]]],\n", - " \n", - " \n", - " [[[ 0.0683]],\n", - " \n", - " [[ 0.0342]],\n", - " \n", - " [[-0.0583]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.0131]],\n", - " \n", - " [[-0.0058]],\n", - " \n", - " [[ 0.0132]]],\n", - " \n", - " \n", - " [[[ 0.0510]],\n", - " \n", - " [[ 0.0250]],\n", - " \n", - " [[ 0.0626]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0634]],\n", - " \n", - " [[-0.0507]],\n", - " \n", - " [[-0.0434]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 0.0283]],\n", - " \n", - " [[-0.0284]],\n", - " \n", - " [[-0.0425]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.0560]],\n", - " \n", - " [[ 0.0462]],\n", - " \n", - " [[-0.0382]]],\n", - " \n", - " \n", - " [[[ 0.0125]],\n", - " \n", - " [[-0.0113]],\n", - " \n", - " [[-0.0371]],\n", - " \n", - " ...,\n", - " \n", - " [[ 0.0561]],\n", - " \n", - " [[-0.0112]],\n", - " \n", - " [[-0.0145]]],\n", - " \n", - " \n", - " [[[ 0.0360]],\n", - " \n", - " [[-0.0027]],\n", - " \n", - " [[ 0.0408]],\n", - " \n", - " ...,\n", - " \n", - " [[-0.0422]],\n", - " \n", - " [[-0.0196]],\n", - " \n", - " [[ 0.0299]]]], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.shortcut.1.weight',\n", - " tensor([-4.7684e-07, 1.1921e-07, -1.7881e-07, -5.9605e-08, 1.1921e-07,\n", - " 0.0000e+00, 3.5763e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07,\n", - " -5.9605e-08, 0.0000e+00, -4.7684e-07, -2.9802e-07, 1.1921e-07,\n", - " 3.5763e-07, -2.9802e-07, -2.3842e-07, 0.0000e+00, 2.3842e-07,\n", - " -8.9407e-07, 1.1921e-07, 0.0000e+00, -3.5763e-07, 2.3842e-07,\n", - " 0.0000e+00, 0.0000e+00, 0.0000e+00, -6.5565e-07, -2.3842e-07,\n", - " -2.9802e-07, 2.3842e-07, -5.9605e-07, -1.1921e-07, 1.1921e-07,\n", - " -2.9802e-07, 5.9605e-07, 2.3842e-07, -8.3447e-07, -4.1723e-07,\n", - " 0.0000e+00, 5.9605e-07, -2.9802e-07, 7.1526e-07, 2.3842e-07,\n", - " -2.3842e-07, 4.7684e-07, 4.7684e-07, 7.1526e-07, 7.1526e-07,\n", - " -2.9802e-07, 2.3842e-07, 3.5763e-07, -2.9802e-07, 1.1921e-07,\n", - " -6.5565e-07, -2.9802e-07, -7.1526e-07, 1.1921e-07, 7.1526e-07,\n", - " -8.9407e-07, 4.7684e-07, -1.7881e-07, 1.1921e-07, -1.1921e-07,\n", - " -5.9605e-07, -4.1723e-07, -5.9605e-08, 4.7684e-07, -5.3644e-07,\n", - " -2.3842e-07, -2.3842e-07, 2.3842e-07, 1.1921e-07, -2.3842e-07,\n", - " -1.7881e-07, -4.1723e-07, -1.1921e-07, 5.9605e-07, 3.5763e-07,\n", - " -5.3644e-07, -5.9605e-08, -5.3644e-07, -5.9605e-08, -5.3644e-07,\n", - " 4.7684e-07, 0.0000e+00, 5.9605e-07, 4.7684e-07, 0.0000e+00,\n", - " 2.3842e-07, -9.5367e-07, -4.1723e-07, -5.9605e-08, 0.0000e+00,\n", - " 7.1526e-07, 1.1921e-07, -5.9605e-08, 2.3842e-07, -5.3644e-07,\n", - " 3.5763e-07, -1.0133e-06, 3.5763e-07, 1.0729e-06, 2.3842e-07,\n", - " -2.9802e-07, -2.3842e-07, -5.9605e-07, -4.1723e-07, -5.9605e-08,\n", - " 0.0000e+00, -2.3842e-07, -1.7881e-07, 2.3842e-07, 1.1921e-07,\n", - " 3.5763e-07, 4.7684e-07, -2.9802e-07, -5.3644e-07, 2.3842e-07,\n", - " -4.7684e-07, -3.5763e-07, -5.9605e-07, -1.1921e-07, -2.3842e-07,\n", - " 1.1921e-07, 1.3113e-06, -2.9802e-07, -5.9605e-08, 0.0000e+00,\n", - " -4.1723e-07, -1.7881e-07, 7.1526e-07, -4.1723e-07, 4.7684e-07,\n", - " -1.1921e-07, 3.5763e-07, 3.5763e-07, -5.3644e-07, -5.9605e-08,\n", - " 2.3842e-07, 2.3842e-07, -2.9802e-07, -2.3842e-07, -1.1921e-07,\n", - " -1.1921e-07, 9.5367e-07, 0.0000e+00, -7.1526e-07, 5.9605e-07,\n", - " -1.1921e-07, -1.7881e-07, 1.1921e-07, 3.5763e-07, -1.7881e-07,\n", - " -5.9605e-08, -1.0729e-06, 3.5763e-07, -1.1921e-07, -2.9802e-07,\n", - " -1.1921e-07, 0.0000e+00, 3.5763e-07, -1.1921e-07, 1.1921e-07,\n", - " -1.7881e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07, 4.7684e-07,\n", - " -3.5763e-07, -1.7881e-07, 2.3842e-07, 1.1921e-07, 3.5763e-07,\n", - " -4.7684e-07, 4.7684e-07, -1.1921e-07, 3.5763e-07, -6.5565e-07,\n", - " 4.7684e-07, -5.9605e-08, -2.3842e-07, -4.7684e-07, -3.5763e-07,\n", - " 1.1921e-07, 3.5763e-07, -5.9605e-08, -1.1921e-07, 3.5763e-07,\n", - " -3.5763e-07, 3.5763e-07, -7.7486e-07, 1.1921e-07, -7.1526e-07,\n", - " -5.9605e-08, 7.1526e-07, -1.1921e-07, 0.0000e+00, -5.9605e-08,\n", - " -2.3842e-07, -7.7486e-07, -5.9605e-08, -2.9802e-07, -5.9605e-07,\n", - " -5.9605e-08, -1.1921e-07, -4.1723e-07, -4.1723e-07, 3.5763e-07,\n", - " 1.1921e-07, -2.3842e-07, 1.1921e-07, 2.3842e-07, 1.1921e-07,\n", - " 5.9605e-07, -3.5763e-07, -1.7881e-07, -3.5763e-07, -5.9605e-07,\n", - " -1.1921e-07, -3.5763e-07, 4.7684e-07, 4.7684e-07, 4.7684e-07,\n", - " 3.5763e-07, -6.5565e-07, -2.3842e-07, 3.5763e-07, 3.5763e-07,\n", - " 4.7684e-07, -1.1921e-07, -4.1723e-07, -2.9802e-07, 3.5763e-07,\n", - " 1.1921e-07, 0.0000e+00, -5.9605e-08, 7.1526e-07, 2.3842e-07,\n", - " -5.9605e-07, 0.0000e+00, -2.9802e-07, 7.1526e-07, 4.7684e-07,\n", - " -5.3644e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07, -2.3842e-07,\n", - " 4.7684e-07, -6.5565e-07, -5.9605e-08, 1.1921e-07, 0.0000e+00,\n", - " -5.9605e-08, -2.3842e-07, 1.1921e-06, -3.5763e-07, 1.1921e-07,\n", - " -1.1921e-07, -1.1921e-07, -5.9605e-08, 4.7684e-07, -4.1723e-07,\n", - " 0.0000e+00, -5.9605e-08, 0.0000e+00, 0.0000e+00, -3.5763e-07,\n", - " 0.0000e+00, -5.9605e-07, 1.1921e-07, 2.3842e-07, 0.0000e+00,\n", - " -8.3447e-07, -7.7486e-07, -4.7684e-07, 8.3447e-07, 0.0000e+00,\n", - " -2.3842e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07, 1.1921e-07,\n", - " 4.7684e-07, -7.1526e-07, 2.3842e-07, -4.7684e-07, -1.7881e-07,\n", - " -1.1921e-07, -1.0133e-06, 0.0000e+00, -5.9605e-08, -2.9802e-07,\n", - " 0.0000e+00, 1.1921e-07, -3.5763e-07, 0.0000e+00, -5.3644e-07,\n", - " 1.1921e-07, -2.9802e-07, 3.5763e-07, 1.1921e-07, -2.9802e-07,\n", - " 1.1921e-07, 0.0000e+00, -7.7486e-07, -1.7881e-07, 2.3842e-07,\n", - " 1.1921e-07, -3.5763e-07, -1.1921e-07, 1.1921e-07, -5.9605e-08,\n", - " -2.3842e-07, 0.0000e+00, -1.1921e-07, -1.7881e-07, 0.0000e+00,\n", - " -3.5763e-07, -2.3842e-07, -2.3842e-07, 2.3842e-07, 4.7684e-07,\n", - " -1.7881e-07, 0.0000e+00, 1.1921e-07, -2.3842e-07, -2.3842e-07,\n", - " 2.3842e-07, -1.1921e-07, 0.0000e+00, 4.7684e-07, -5.9605e-08,\n", - " 0.0000e+00, 1.1921e-07, -3.5763e-07, -5.9605e-08, 0.0000e+00,\n", - " 3.5763e-07, 0.0000e+00, 0.0000e+00, 7.1526e-07, 2.3842e-07,\n", - " 3.5763e-07, -5.3644e-07, -1.1921e-07, -2.3842e-07, -4.1723e-07,\n", - " -6.5565e-07, 1.1921e-07, 0.0000e+00, -2.9802e-07, -5.9605e-08,\n", - " 1.1921e-07, 0.0000e+00, -8.3447e-07, 0.0000e+00, -2.3842e-07,\n", - " 0.0000e+00, -2.3842e-07, -7.1526e-07, -5.9605e-07, -2.9802e-07,\n", - " -1.1921e-07, 3.5763e-07, -4.1723e-07, 4.7684e-07, -3.5763e-07,\n", - " -5.9605e-08, -2.3842e-07, 7.1526e-07, -3.5763e-07, 1.1921e-07,\n", - " 5.9605e-07, -4.7684e-07, -4.1723e-07, -2.9802e-07, -4.1723e-07,\n", - " 2.3842e-07, -2.9802e-07, 2.3842e-07, 3.5763e-07, 4.7684e-07,\n", - " 1.1921e-07, -2.9802e-07, 4.7684e-07, 0.0000e+00, -6.5565e-07,\n", - " 4.7684e-07, 2.3842e-07, 3.5763e-07, 0.0000e+00, 2.3842e-07,\n", - " 0.0000e+00, -1.1921e-07, 3.5763e-07, -2.9802e-07, -1.1921e-07,\n", - " -5.9605e-07, 0.0000e+00, 5.9605e-07, -5.3644e-07, -2.3842e-07,\n", - " -7.1526e-07, -5.9605e-08, -4.1723e-07, -2.9802e-07, -5.9605e-08,\n", - " -2.9802e-07, -1.1921e-07, -4.1723e-07, 2.3842e-07, 2.3842e-07,\n", - " 2.3842e-07, -5.9605e-08, 0.0000e+00, -5.9605e-08, -6.5565e-07,\n", - " -1.7881e-07, 2.3842e-07, -6.5565e-07, 1.1921e-07, 4.7684e-07,\n", - " -4.7684e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07, 0.0000e+00,\n", - " 0.0000e+00, 1.1921e-07, 4.7684e-07, 2.3842e-07, -4.1723e-07,\n", - " -4.1723e-07, -5.9605e-07, -4.7684e-07, 1.1921e-07, -2.3842e-07,\n", - " -4.1723e-07, -5.3644e-07, -7.7486e-07, -8.3447e-07, 0.0000e+00,\n", - " 3.5763e-07, -4.1723e-07, 2.3842e-07, -7.7486e-07, -1.7881e-07,\n", - " 2.3842e-07, 2.3842e-07, -2.9802e-07, 2.3842e-07, -5.3644e-07,\n", - " 2.3842e-07, 0.0000e+00, -7.1526e-07, 1.1921e-07, -5.9605e-08,\n", - " 0.0000e+00, 1.1921e-07, -5.9605e-08, 0.0000e+00, 2.3842e-07,\n", - " -4.7684e-07, -1.7881e-07, -2.9802e-07, -3.5763e-07, -5.9605e-08,\n", - " 2.3842e-07, 2.3842e-07, 0.0000e+00, 4.7684e-07, -1.7881e-07,\n", - " 4.7684e-07, 2.3842e-07, -2.9802e-07, -4.7684e-07, 3.5763e-07,\n", - " 0.0000e+00, -1.7881e-07, -6.5565e-07, -4.7684e-07, -4.7684e-07,\n", - " -2.3842e-07, 0.0000e+00, -2.3842e-07, -4.1723e-07, -5.9605e-08,\n", - " 1.1921e-07, -7.7486e-07, 0.0000e+00, -4.1723e-07, 1.1921e-07,\n", - " -1.1921e-07, -8.9407e-07, -2.3842e-07, -1.7881e-07, 8.3447e-07,\n", - " -7.7486e-07, 2.3842e-07, -5.9605e-08, -5.9605e-08, -2.3842e-07,\n", - " -2.3842e-07, 2.3842e-07, 1.1921e-07, 0.0000e+00, -5.9605e-08,\n", - " -5.9605e-08, -5.9605e-08], device='cuda:0', grad_fn=)),\n", - " ('layer4.0.shortcut.1.bias',\n", - " tensor([-2.5952e-07, -5.2718e-08, -3.8981e-07, -1.2332e-07, 2.4942e-07,\n", - " -4.1001e-07, -1.8458e-07, 2.2059e-07, 3.3934e-11, -5.6743e-08,\n", - " 3.8876e-07, -2.2358e-07, 1.5773e-07, -3.4872e-07, -2.7905e-07,\n", - " -2.3630e-08, -3.0215e-08, -2.4326e-07, -2.1305e-07, -2.9157e-07,\n", - " -4.5004e-07, -3.3447e-07, -1.6505e-07, -2.8462e-07, -1.1983e-07,\n", - " 3.0670e-07, -1.7117e-07, 2.2260e-07, -1.8211e-07, -3.3789e-07,\n", - " -6.4035e-07, -3.4069e-07, -2.1152e-07, 6.1553e-08, -1.1388e-07,\n", - " 3.9278e-08, 6.4294e-07, 5.3870e-07, -2.0622e-07, -5.0324e-07,\n", - " -8.4042e-08, 1.2858e-07, -3.1810e-07, 8.5574e-07, 2.0792e-07,\n", - " -3.2931e-07, 4.0081e-07, 1.2134e-07, 1.0071e-07, 9.3611e-07,\n", - " -3.1569e-07, -1.4190e-07, -9.8398e-08, -4.4183e-07, 3.8909e-07,\n", - " -2.3117e-07, -3.3583e-07, -4.5050e-07, 3.9275e-07, 5.3557e-07,\n", - " -8.5330e-07, -4.7482e-08, -7.7420e-08, 4.5063e-08, 5.6890e-08,\n", - " -6.9403e-07, -6.6087e-07, -3.2312e-07, -1.5117e-07, -5.0290e-07,\n", - " -8.3522e-08, -6.2873e-07, -1.9139e-07, -4.7217e-08, -1.4439e-07,\n", - " -3.0978e-07, -3.0010e-07, -4.4070e-07, 6.1973e-08, 4.8330e-07,\n", - " -1.5466e-07, -6.7563e-08, -6.4472e-07, -3.3934e-08, -3.2509e-07,\n", - " 3.2013e-09, -1.0098e-08, 6.1731e-08, -9.7081e-08, 6.2362e-08,\n", - " 2.2826e-07, -8.8843e-07, -7.5161e-07, -4.3784e-07, 5.1198e-08,\n", - " 7.3383e-07, -8.6796e-08, 1.3520e-07, 2.7753e-07, -7.6281e-08,\n", - " 3.2856e-07, -8.5474e-07, 1.4127e-07, 1.6500e-07, -1.1475e-07,\n", - " -9.0230e-08, -3.4021e-08, -4.5122e-07, 1.0504e-07, -3.4202e-08,\n", - " -6.6387e-09, -4.7148e-07, -5.1771e-07, -3.5938e-07, 1.1671e-07,\n", - " 7.6556e-08, 1.0592e-08, 2.3111e-07, -1.5009e-07, -2.9490e-07,\n", - " -2.6290e-07, -2.8363e-07, -4.4743e-07, -4.2879e-07, 1.9960e-07,\n", - " -2.2103e-07, 5.0751e-07, 2.3843e-08, -2.6756e-07, -1.8564e-07,\n", - " -2.7111e-07, -1.7106e-07, 8.1063e-07, -4.9894e-07, 1.3639e-07,\n", - " -4.6950e-07, 1.0927e-07, -1.2955e-07, -4.4637e-07, -8.5186e-08,\n", - " -6.1976e-08, -5.0219e-08, -6.6460e-07, -1.8259e-07, -2.3600e-07,\n", - " -1.2455e-07, 1.3833e-07, 2.3173e-07, 1.3480e-07, 2.6061e-07,\n", - " -2.2656e-07, -2.9893e-07, -3.5553e-07, 4.9369e-07, 2.8315e-07,\n", - " -1.6586e-07, -7.3051e-07, 3.2315e-07, -2.5779e-07, 1.5562e-07,\n", - " -3.1320e-07, -5.5600e-08, 6.4858e-08, -4.3620e-07, -3.0921e-08,\n", - " 1.7959e-07, 4.7084e-07, -2.9651e-07, 1.8505e-08, 4.9029e-07,\n", - " -2.0760e-07, -5.8565e-07, 1.7540e-07, -2.5290e-08, 5.8386e-08,\n", - " 2.4527e-07, 8.4265e-08, 3.2661e-08, -2.2252e-08, -2.5374e-07,\n", - " -7.2610e-08, 1.9237e-07, -3.9457e-07, -8.6376e-08, 9.7661e-08,\n", - " 8.1748e-08, -1.8011e-07, 5.4112e-08, -2.4380e-07, 4.2298e-08,\n", - " -2.2257e-07, 2.4752e-07, -8.2461e-07, -4.9300e-07, -2.8506e-07,\n", - " 1.4784e-07, 4.9734e-08, -3.3205e-07, -1.1681e-07, -4.5991e-07,\n", - " -1.0016e-07, -4.8842e-07, 3.3371e-07, -4.3462e-07, -7.9114e-07,\n", - " -6.8228e-08, -4.7129e-07, -3.3589e-07, 2.6814e-07, 1.9327e-07,\n", - " 1.4206e-07, 3.3498e-08, -5.5501e-08, 1.7940e-07, 4.2684e-07,\n", - " 1.7147e-07, 5.6309e-08, -7.1572e-09, -3.8246e-08, -2.8099e-07,\n", - " 9.5294e-08, -4.6339e-07, 5.0270e-07, 1.0481e-08, 1.9457e-07,\n", - " -2.0168e-08, 8.2068e-08, -1.6883e-09, 2.6224e-07, -3.9644e-07,\n", - " 4.4707e-07, -8.3741e-08, -1.7601e-07, -1.7361e-07, 7.8584e-07,\n", - " 7.0257e-11, -1.2827e-07, 3.2551e-07, 6.4153e-07, -3.4501e-08,\n", - " -7.0563e-07, 1.2502e-07, -5.3899e-07, 6.2128e-07, 1.6744e-07,\n", - " 3.0381e-08, 3.1756e-07, 1.4472e-07, 3.9837e-07, -6.7176e-08,\n", - " 7.3281e-07, -3.6448e-07, -2.5766e-08, 9.8277e-09, -5.1608e-07,\n", - " -2.1131e-07, -4.9822e-07, 6.0511e-07, -2.0183e-08, 2.1366e-07,\n", - " -1.2711e-07, 3.8626e-07, 3.8279e-07, 7.5298e-07, -6.0340e-07,\n", - " -4.7458e-08, -7.7033e-08, -2.6878e-07, 1.7127e-07, -4.7343e-07,\n", - " -5.9129e-09, -6.3804e-07, 3.3628e-07, -2.4381e-07, 3.1775e-07,\n", - " -6.1137e-07, -4.6261e-07, -4.1723e-07, 5.3470e-07, 7.1795e-09,\n", - " -2.3043e-08, -1.2530e-07, -5.4000e-07, -6.3562e-07, 1.3268e-07,\n", - " 2.2244e-07, -1.7262e-07, 4.3062e-07, -1.3121e-07, -2.1163e-07,\n", - " 6.3334e-08, -6.1981e-07, 6.5072e-08, -2.1843e-07, 1.8716e-08,\n", - " 7.4588e-08, 4.0129e-07, -8.5405e-08, 6.3721e-08, -4.6985e-07,\n", - " -2.2629e-07, -4.9808e-07, 6.7194e-07, 1.3299e-07, -4.3199e-09,\n", - " 2.7960e-07, -3.2597e-07, -2.8505e-08, -1.2399e-07, -2.7097e-07,\n", - " 8.7115e-08, -6.0820e-07, -1.7309e-07, 3.7368e-07, -2.4287e-07,\n", - " -1.1012e-07, 1.2617e-08, 1.7739e-07, -2.6112e-07, -3.0125e-07,\n", - " -1.2478e-07, 2.5956e-07, -1.0944e-07, 1.7735e-07, -5.8064e-08,\n", - " -2.0758e-07, -2.8337e-07, -2.6004e-07, 3.2003e-08, -3.9736e-07,\n", - " 2.9050e-07, -2.3287e-08, -2.2765e-07, 4.0771e-07, 4.0802e-07,\n", - " 7.0819e-08, 1.4434e-07, -1.8711e-07, -2.4609e-08, 2.6868e-07,\n", - " -1.8774e-08, -2.0145e-07, 1.3446e-07, 5.9894e-07, -5.2298e-08,\n", - " 2.3941e-07, 2.3950e-07, -3.1209e-07, 3.9168e-07, -3.1503e-07,\n", - " -5.3224e-07, 4.3975e-07, -9.7231e-09, -3.8906e-07, -3.3958e-07,\n", - " 1.7475e-07, -1.2887e-07, -7.8835e-07, 6.8751e-08, -4.2049e-07,\n", - " 1.2126e-07, 8.8542e-08, -5.7822e-07, -4.5029e-07, -1.0936e-07,\n", - " -2.1707e-07, 3.0449e-07, 2.9369e-07, 1.6307e-07, -2.1861e-07,\n", - " -4.1288e-07, -9.9163e-07, 3.3039e-07, -1.5236e-07, -1.9777e-08,\n", - " 3.7243e-07, -3.4649e-07, -3.2635e-07, -2.5982e-07, -2.4515e-07,\n", - " 7.5304e-08, -6.0096e-08, 2.8725e-07, 9.5974e-08, 1.0231e-07,\n", - " -6.2956e-09, 9.3015e-08, 5.1241e-07, -3.7457e-07, -5.8680e-07,\n", - " 5.1277e-08, 1.5735e-07, 1.5098e-07, -4.7827e-07, 1.5020e-07,\n", - " 3.2007e-07, -1.1792e-07, 4.1907e-07, -5.4599e-07, 3.5650e-07,\n", - " -7.2770e-07, 2.4135e-07, -5.4777e-08, -1.6205e-07, 1.0242e-07,\n", - " -8.1226e-07, -2.6392e-07, -3.5215e-07, -5.7353e-07, -2.3653e-07,\n", - " -7.6270e-07, -1.0341e-07, -1.5335e-07, -4.0521e-07, 5.1136e-08,\n", - " -2.1407e-07, -1.1061e-06, -1.1568e-07, -1.8562e-07, -2.2229e-07,\n", - " 1.7121e-07, 4.0764e-07, -5.0914e-07, -1.5899e-07, 1.9196e-07,\n", - " -4.7060e-07, 1.8851e-07, -5.7792e-08, -4.6329e-07, -3.9686e-07,\n", - " -1.8117e-08, -1.4392e-07, 4.1560e-07, 4.7124e-07, -2.5345e-08,\n", - " -2.1026e-07, -1.2185e-07, -2.1304e-07, 1.5910e-09, -3.9998e-07,\n", - " -1.5126e-07, -2.4613e-07, -1.0463e-07, 1.4167e-07, 2.5327e-08,\n", - " 3.7949e-07, -5.3259e-07, 7.2235e-07, -1.0368e-06, -2.0211e-07,\n", - " 4.1569e-07, 5.9008e-09, 2.3589e-07, -1.1660e-07, 3.8977e-08,\n", - " -2.6139e-07, 2.5065e-07, -3.0728e-07, -2.3161e-07, 5.0300e-08,\n", - " 4.0945e-08, -3.4996e-08, -2.7710e-07, -4.3644e-09, -1.2612e-07,\n", - " -3.5672e-07, -3.0295e-07, -3.4237e-07, -1.6870e-07, 3.2721e-07,\n", - " -2.5081e-07, 1.2698e-07, 2.5012e-07, 4.2570e-07, -3.8390e-07,\n", - " 1.6247e-07, 1.5678e-07, -1.8048e-07, 2.3226e-07, 2.6242e-07,\n", - " 1.4829e-07, -1.9754e-07, -2.4445e-07, -1.5111e-07, -1.8443e-07,\n", - " -5.3174e-07, -4.4128e-08, 2.1757e-07, -4.6750e-07, -7.1977e-08,\n", - " -1.7918e-07, -3.1625e-07, 2.4026e-07, 7.8625e-08, -4.3738e-08,\n", - " -2.1923e-09, -5.7170e-07, -4.3710e-07, -2.2763e-07, 5.7661e-07,\n", - " -3.9948e-07, 1.6238e-07, 2.4817e-07, 4.2508e-07, -5.4188e-07,\n", - " 8.4595e-08, -2.3122e-07, 2.6185e-07, -7.8099e-08, -1.5002e-07,\n", - " -4.9218e-08, 2.9320e-07], device='cuda:0', grad_fn=)),\n", - " ('linear.weight',\n", - " tensor([[ 0.0289, 0.0068, -0.0082, ..., 0.0602, -0.0364, 0.0396],\n", - " [ 0.0019, -0.0095, 0.0242, ..., 0.0193, -0.0087, 0.0316],\n", - " [-0.0057, -0.0301, 0.0333, ..., -0.0258, 0.0058, 0.0334],\n", - " ...,\n", - " [-0.0019, -0.0217, -0.0481, ..., -0.0218, -0.0286, -0.0146],\n", - " [ 0.0179, 0.0245, -0.0157, ..., 0.0186, 0.0383, -0.0235],\n", - " [-0.0072, -0.0170, -0.0242, ..., 0.0236, -0.0367, 0.0263]],\n", - " device='cuda:0', grad_fn=)),\n", - " ('linear.bias',\n", - " tensor([ 0.0288, -0.0477, 0.0047, 0.0126, -0.0182, 0.0702, -0.0292, -0.0152,\n", - " 0.0034, -0.0235], device='cuda:0', grad_fn=))])" - ] - }, - "execution_count": 22, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "param_diff" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Starting training...\n", - "Batch: 0/1, Loss: 2.3967, Acc: 10.00%\n", - "Epoch: 1, Loss: 2.3967, Accuracy: 0.1000\n" - ] - } - ], - "source": [ - "import torch\n", - "from torch import nn\n", - "from collections import OrderedDict\n", - "from torch.utils.data import Dataset, DataLoader\n", - "from typing import Tuple, Any\n", - "\n", - "class CustomImageDataset(Dataset):\n", - " def __init__(self, images, labels):\n", - " self.images = images\n", - " self.labels = labels\n", - " \n", - " def __len__(self):\n", - " return len(self.labels)\n", - " \n", - " def __getitem__(self, idx):\n", - " image = self.images[idx]\n", - " label = self.labels[idx]\n", - " return image, label\n", - "\n", - "def train_one_epoch(\n", - " model: nn.Module,\n", - " optimizer: torch.optim.Optimizer,\n", - " dataloader: DataLoader,\n", - " criterion: nn.Module,\n", - " device: str\n", - ") -> Tuple[float, float]:\n", - " model.train()\n", - " total_loss = 0\n", - " correct = 0\n", - " total = 0\n", - "\n", - " for batch_idx, (data, target) in enumerate(dataloader):\n", - " # Move data to device\n", - " data, target = data.to(device), target.to(device)\n", - " \n", - " # Zero gradients\n", - " optimizer.zero_grad()\n", - " \n", - " # Forward pass\n", - " outputs = model(data)\n", - " \n", - " # Calculate loss\n", - " loss = criterion(outputs, target)\n", - " \n", - " # Backward pass\n", - " loss.backward()\n", - " \n", - " # Update weights\n", - " optimizer.step()\n", - " \n", - " # Track statistics\n", - " total_loss += loss.item()\n", - " _, predicted = outputs.max(1)\n", - " total += target.size(0)\n", - " correct += predicted.eq(target).sum().item()\n", - " \n", - " # Print batch statistics\n", - " if batch_idx % 10 == 0:\n", - " print(f'Batch: {batch_idx}/{len(dataloader)}, Loss: {loss.item():.4f}, '\n", - " f'Acc: {100.*correct/total:.2f}%')\n", - " \n", - " return total_loss / len(dataloader), correct / total\n", - "\n", - "# Main training setup\n", - "def main():\n", - " # Model setup\n", - " model_bespoke = mutils.get_model(\"resnet10\", \"cifar10\", \"cuda\")\n", - " model_bespoke.train()\n", - " \n", - " # Save initial parameters\n", - " params_initial = OrderedDict((name, param.clone().detach()) \n", - " for (name, param) in model_bespoke.named_parameters())\n", - " \n", - " # Optimizer setup\n", - " optimizer = torch.optim.SGD(model_bespoke.parameters(), lr=3e-4, weight_decay=0)\n", - " criterion = nn.CrossEntropyLoss()\n", - " \n", - " # Dataset setup (assuming ground_truth and labels are defined)\n", - " dataset = CustomImageDataset(ground_truth, labels)\n", - " dataloader = DataLoader(dataset, batch_size=10, shuffle=True, num_workers=0)\n", - " \n", - " # Training loop\n", - " print(\"Starting training...\")\n", - " for epoch in range(1):\n", - " loss, acc = train_one_epoch(model_bespoke, optimizer, dataloader, criterion, \"cuda\")\n", - " print(f'Epoch: {epoch+1}, Loss: {loss:.4f}, Accuracy: {acc:.4f}')\n", - " \n", - " # Save final parameters and calculate differences\n", - " params_final = OrderedDict((name, param.clone().detach()) \n", - " for (name, param) in model_bespoke.named_parameters())\n", - " \n", - " # Calculate and print parameter differences\n", - " param_diff = OrderedDict()\n", - " for (name, p_initial), (_, p_final) in zip(params_initial.items(), params_final.items()):\n", - " param_diff[name] = (p_final - p_initial)\n", - " \n", - " return model_bespoke, param_diff\n", - "\n", - "if __name__ == \"__main__\":\n", - " model, param_diff = main()" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "OrderedDict([('conv1.weight',\n", - " tensor([[[[ 9.9093e-07, -4.6194e-07, 4.0047e-06],\n", - " [-1.0729e-06, -8.2254e-06, -2.9691e-06],\n", - " [-9.6262e-06, -1.4752e-05, -4.8131e-06]],\n", - " \n", - " [[-4.6864e-06, -4.3809e-06, -1.7285e-06],\n", - " [-5.7071e-06, -1.0679e-05, -7.5102e-06],\n", - " [-1.4395e-05, -1.7047e-05, -8.5291e-06]],\n", - " \n", - " [[-6.2361e-06, -5.7817e-06, -3.9898e-06],\n", - " [-6.8545e-06, -1.1079e-05, -7.0771e-06],\n", - " [-1.3404e-05, -1.5400e-05, -6.4522e-06]]],\n", - " \n", - " \n", - " [[[ 1.8328e-06, 6.0797e-06, 2.1458e-06],\n", - " [-1.0148e-05, -6.2585e-06, -2.6934e-06],\n", - " [-1.3158e-05, -9.8050e-06, -9.6485e-06]],\n", - " \n", - " [[ 1.0267e-05, 1.5482e-05, 9.8795e-06],\n", - " [-2.4140e-06, -6.2585e-07, 1.1474e-06],\n", - " [-5.7593e-06, -3.9488e-06, -6.4485e-06]],\n", - " \n", - " [[ 1.5780e-05, 1.7703e-05, 1.1794e-05],\n", - " [ 4.9025e-06, 5.3421e-06, 5.1409e-06],\n", - " [-2.2352e-07, 1.6503e-06, -2.5034e-06]]],\n", - " \n", - " \n", - " [[[ 1.3784e-06, -2.9132e-06, 8.7917e-07],\n", - " [-2.5034e-06, -1.2890e-06, -1.8179e-06],\n", - " [-5.1185e-06, 2.5630e-06, -1.0058e-06]],\n", - " \n", - " [[ 1.0371e-05, 8.0764e-06, 1.0230e-05],\n", - " [ 7.8678e-06, 1.1332e-05, 8.4192e-06],\n", - " [ 5.2005e-06, 1.3366e-05, 8.1956e-06]],\n", - " \n", - " [[ 6.7428e-06, 5.0217e-06, 7.5144e-06],\n", - " [ 5.5060e-06, 7.9572e-06, 6.5565e-06],\n", - " [ 4.6194e-06, 1.2025e-05, 8.2552e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-1.1228e-05, -7.6741e-06, 6.9439e-06],\n", - " [-1.0118e-05, -1.7919e-06, 1.2919e-05],\n", - " [-1.6190e-05, -3.1982e-06, 8.8289e-06]],\n", - " \n", - " [[-1.6570e-05, -1.0796e-05, 4.9919e-06],\n", - " [-1.6170e-05, -6.9663e-06, 7.4562e-06],\n", - " [-2.0325e-05, -8.1211e-06, 5.6177e-06]],\n", - " \n", - " [[-1.2562e-05, -7.4804e-06, 8.5384e-06],\n", - " [-1.6130e-05, -6.4820e-06, 8.5905e-06],\n", - " [-2.1935e-05, -8.5533e-06, 4.5151e-06]]],\n", - " \n", - " \n", - " [[[ 6.1095e-06, 9.4026e-06, 6.8843e-06],\n", - " [ 6.2920e-06, 7.5996e-06, 5.8860e-06],\n", - " [ 6.4075e-06, 7.2122e-06, 4.0829e-06]],\n", - " \n", - " [[ 9.7807e-06, 1.2234e-05, 8.0429e-06],\n", - " [ 9.1195e-06, 8.1584e-06, 4.3991e-06],\n", - " [ 9.1344e-06, 8.0355e-06, 3.4571e-06]],\n", - " \n", - " [[ 4.0382e-06, 4.8876e-06, 1.0412e-06],\n", - " [ 2.1011e-06, -3.1292e-07, -4.2915e-06],\n", - " [ 3.5092e-06, -5.2154e-07, -4.8578e-06]]],\n", - " \n", - " \n", - " [[[ 6.3144e-06, 7.4543e-06, 1.1399e-05],\n", - " [ 4.6194e-07, -1.1176e-08, 5.0813e-06],\n", - " [ 2.1160e-06, 5.0664e-06, 6.1244e-06]],\n", - " \n", - " [[ 3.7402e-06, 4.3111e-06, 7.0184e-06],\n", - " [-1.3188e-06, -1.5944e-06, 3.5614e-06],\n", - " [-1.1176e-06, 3.1441e-06, 3.6871e-06]],\n", - " \n", - " [[ 5.3048e-06, 7.3910e-06, 1.0658e-05],\n", - " [ 1.6242e-06, 2.0862e-06, 7.3463e-06],\n", - " [ 1.2368e-06, 5.9903e-06, 7.5251e-06]]]], device='cuda:0')),\n", - " ('bn1.weight',\n", - " tensor([ 3.0994e-06, 3.5763e-07, -7.3314e-06, 1.1921e-06, -4.3511e-06,\n", - " 4.7684e-07, 9.5367e-07, 1.7881e-06, -4.0531e-06, 2.8610e-06,\n", - " -4.5300e-06, 1.0729e-06, -2.7418e-06, -5.0068e-06, 3.3379e-06,\n", - " 4.8876e-06, -5.8413e-06, -4.4703e-06, 4.0531e-06, 3.9339e-06,\n", - " -3.8147e-06, 4.6492e-06, -6.6757e-06, 4.2915e-06, 3.5763e-06,\n", - " -1.1921e-06, 9.5367e-07, 1.5497e-06, 9.5367e-07, -1.1921e-06,\n", - " 3.4571e-06, 4.7684e-07, 3.0994e-06, 7.1526e-07, 3.6955e-06,\n", - " 3.5763e-06, 1.0729e-06, 8.3447e-07, -3.5763e-07, 2.7418e-06,\n", - " 6.6757e-06, -1.8477e-06, 8.2254e-06, -4.7684e-07, 7.2718e-06,\n", - " 2.2650e-06, 5.1260e-06, -7.3314e-06, -1.7285e-06, 1.7881e-06,\n", - " 3.3379e-06, 1.4305e-06, -4.0531e-06, -1.9073e-06, 3.6955e-06,\n", - " -8.2850e-06, 4.1723e-06, 6.3181e-06, -3.8147e-06, 9.5367e-07,\n", - " 3.6955e-06, -3.9339e-06, -1.0610e-05, 4.1723e-06], device='cuda:0')),\n", - " ('bn1.bias',\n", - " tensor([ 4.2480e-06, 5.9006e-06, -4.8113e-06, 2.8228e-06, 8.5083e-07,\n", - " 3.2780e-06, 5.0684e-06, 4.3948e-06, -3.4867e-06, 1.5374e-06,\n", - " -1.8370e-06, 1.9890e-06, -2.3541e-06, -5.6038e-06, 5.9714e-06,\n", - " 3.8110e-06, -5.7142e-06, 5.9350e-07, 6.3278e-06, 1.9870e-06,\n", - " -6.3236e-06, 6.7887e-06, -3.1113e-06, -3.9099e-07, -5.9868e-08,\n", - " -5.4377e-06, 1.9105e-06, 4.1676e-06, 6.8520e-07, 4.0158e-06,\n", - " 1.2476e-06, 1.1877e-08, -1.6836e-06, -3.9817e-06, 5.0061e-07,\n", - " -1.8907e-06, 4.0654e-06, -2.6084e-07, 8.4628e-07, 7.6336e-07,\n", - " 8.8257e-06, -4.3922e-06, 3.6792e-06, -6.5806e-06, 1.0206e-05,\n", - " 3.6491e-06, 6.8027e-06, -4.4466e-06, -8.1502e-07, -1.8807e-06,\n", - " 7.4160e-06, 2.7364e-06, -4.5183e-06, -1.0573e-06, 5.4573e-06,\n", - " 2.6168e-07, -2.8168e-06, 3.9376e-06, 1.5581e-06, 8.1046e-08,\n", - " -3.9231e-07, 1.5138e-06, -1.0100e-05, 1.9233e-06], device='cuda:0')),\n", - " ('layer1.0.conv1.weight',\n", - " tensor([[[[ 5.3197e-06, 2.5735e-06, 4.9677e-06],\n", - " [ 2.2687e-06, 1.3215e-06, 6.7260e-06],\n", - " [-7.5065e-07, 1.2163e-06, 7.1549e-06]],\n", - " \n", - " [[ 3.6657e-06, -1.5572e-06, 2.5872e-06],\n", - " [ 8.9761e-06, 3.2061e-06, 1.9390e-06],\n", - " [ 9.6913e-06, 3.4496e-06, 3.3490e-06]],\n", - " \n", - " [[ 4.6026e-06, 4.1500e-06, 6.6273e-06],\n", - " [ 4.5188e-06, 5.9195e-06, 8.9146e-06],\n", - " [ 2.5216e-06, 6.0610e-06, 7.4264e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 7.8343e-06, 3.2730e-06, 2.7278e-06],\n", - " [ 3.6531e-06, 5.8208e-06, 5.4631e-06],\n", - " [ 4.6603e-06, 3.2969e-06, 5.4352e-06]],\n", - " \n", - " [[ 8.9929e-06, 9.3840e-06, 4.5430e-06],\n", - " [ 7.3332e-06, 5.3700e-06, 1.1288e-06],\n", - " [ 4.8243e-06, 2.8703e-06, 6.5018e-07]],\n", - " \n", - " [[ 5.6121e-06, 6.1393e-06, 6.9626e-06],\n", - " [ 4.0215e-06, 5.4743e-06, 6.6962e-06],\n", - " [ 1.0822e-06, 2.9709e-06, 3.7886e-06]]],\n", - " \n", - " \n", - " [[[-9.5088e-07, 1.7602e-06, 3.9407e-06],\n", - " [ 2.0731e-06, 1.4435e-06, 5.0133e-06],\n", - " [ 4.6305e-06, 2.6263e-06, 4.7162e-06]],\n", - " \n", - " [[-3.6489e-06, 2.9802e-08, 4.2648e-06],\n", - " [ 1.4901e-07, -1.1828e-07, 3.9935e-06],\n", - " [ 2.0228e-06, 8.7958e-07, 3.7327e-06]],\n", - " \n", - " [[-1.0431e-07, 2.4671e-06, 3.8389e-06],\n", - " [ 1.3597e-06, 1.6810e-06, 2.6971e-06],\n", - " [ 2.0014e-06, 9.4436e-07, 2.0210e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.3267e-06, 1.5609e-06, 1.4380e-06],\n", - " [ 2.3637e-06, -9.4622e-07, -3.7998e-07],\n", - " [ 2.9150e-07, -1.0962e-06, -1.0170e-06]],\n", - " \n", - " [[ 3.9730e-06, 3.7905e-06, 1.3942e-06],\n", - " [ 2.7791e-06, 2.3413e-06, 4.2561e-07],\n", - " [ 2.6338e-06, 2.1085e-06, 2.1569e-06]],\n", - " \n", - " [[-1.8850e-06, 9.8720e-08, 2.0146e-07],\n", - " [-9.3877e-07, -8.7544e-07, -6.8918e-08],\n", - " [-1.7975e-07, -4.6659e-07, -5.9046e-07]]],\n", - " \n", - " \n", - " [[[ 1.2033e-06, -1.3299e-06, -2.8559e-06],\n", - " [-5.4762e-06, -7.9861e-06, -3.8221e-06],\n", - " [-2.9244e-06, -5.6196e-06, -2.2016e-06]],\n", - " \n", - " [[ 2.4727e-07, 7.0781e-07, 3.1963e-06],\n", - " [-2.4950e-06, 1.9707e-06, 4.5709e-06],\n", - " [-5.0031e-06, 2.9728e-06, 9.2518e-06]],\n", - " \n", - " [[-1.8207e-06, -4.3288e-06, 4.6268e-06],\n", - " [-6.0368e-06, -2.5928e-06, 3.3807e-06],\n", - " [ 2.1979e-07, 2.8408e-06, 4.3400e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.1656e-06, -3.1851e-07, -2.1411e-06],\n", - " [ 2.7530e-06, -4.5057e-06, -3.3136e-06],\n", - " [ 4.9341e-06, -3.8240e-06, -6.9663e-07]],\n", - " \n", - " [[ 2.8424e-06, 7.0184e-06, 2.2314e-06],\n", - " [ 4.1723e-07, 1.1735e-07, -5.0850e-06],\n", - " [-5.3681e-06, -3.4831e-06, -7.8762e-06]],\n", - " \n", - " [[-8.8662e-07, 9.2108e-07, 2.4959e-07],\n", - " [-1.1809e-06, 1.1176e-08, -1.7062e-06],\n", - " [ 7.2177e-07, -9.7975e-07, -2.2464e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 1.2767e-05, 1.0841e-05, 3.4422e-06],\n", - " [ 1.4702e-05, 1.1206e-05, 8.2599e-06],\n", - " [ 1.3908e-05, 1.2608e-05, 1.0829e-05]],\n", - " \n", - " [[ 2.8610e-06, -4.5970e-06, -1.7211e-06],\n", - " [ 5.1465e-06, 5.0291e-08, -2.1150e-06],\n", - " [ 7.2159e-06, 1.5320e-06, -8.7242e-07]],\n", - " \n", - " [[ 3.5076e-06, 3.9490e-06, 2.9262e-06],\n", - " [ 6.1616e-06, 4.7190e-06, 5.3495e-06],\n", - " [ 4.4499e-06, 5.6326e-06, 3.7383e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.7400e-06, 6.7540e-06, 3.9362e-06],\n", - " [ 1.3877e-06, 3.7868e-06, 3.9116e-06],\n", - " [ 2.0789e-06, 5.8301e-07, -3.5833e-07]],\n", - " \n", - " [[ 7.3574e-06, 5.7472e-06, 7.1190e-06],\n", - " [ 1.1409e-05, 7.6070e-06, 6.3684e-06],\n", - " [ 9.0655e-06, 6.0052e-06, 6.5975e-06]],\n", - " \n", - " [[ 5.5323e-06, 2.6356e-06, 1.1306e-06],\n", - " [ 6.8489e-06, 4.5933e-06, 4.3800e-06],\n", - " [ 7.4692e-06, 6.4746e-06, 5.5432e-06]]],\n", - " \n", - " \n", - " [[[-7.0045e-06, -7.1097e-06, -1.3830e-05],\n", - " [-1.5358e-05, -1.5994e-05, -2.4969e-05],\n", - " [-1.2049e-05, -1.9913e-05, -2.9732e-05]],\n", - " \n", - " [[ 1.3510e-05, 9.2760e-06, 4.4014e-06],\n", - " [ 1.3236e-05, 2.1923e-06, -2.6240e-06],\n", - " [ 1.4646e-05, 3.0799e-06, -8.1831e-06]],\n", - " \n", - " [[ 3.0398e-06, -8.8508e-06, -1.7661e-05],\n", - " [-5.3140e-06, -1.6805e-05, -2.5857e-05],\n", - " [-4.6920e-06, -2.2337e-05, -2.4119e-05]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.4177e-06, -5.0031e-06, -7.2159e-06],\n", - " [-3.1861e-06, -3.9972e-06, -9.8143e-06],\n", - " [-5.7630e-06, -7.0781e-08, -3.1888e-06]],\n", - " \n", - " [[-1.6071e-05, -2.0384e-05, -1.2822e-05],\n", - " [-2.5393e-05, -2.8031e-05, -2.0761e-05],\n", - " [-2.4607e-05, -2.7642e-05, -2.1874e-05]],\n", - " \n", - " [[-4.1854e-06, -8.3670e-06, -1.2558e-05],\n", - " [-9.9316e-06, -1.8132e-05, -2.3715e-05],\n", - " [-1.3510e-05, -2.1283e-05, -2.6420e-05]]],\n", - " \n", - " \n", - " [[[ 6.6161e-06, 1.1180e-05, 1.1187e-05],\n", - " [ 5.1903e-06, 1.2271e-05, 8.8448e-06],\n", - " [ 6.6292e-06, 8.8671e-06, 1.0509e-05]],\n", - " \n", - " [[-1.5578e-05, -7.0557e-06, 8.2366e-06],\n", - " [-9.8622e-06, 1.9372e-06, 8.7224e-06],\n", - " [-1.0390e-05, 7.1414e-06, 6.0899e-06]],\n", - " \n", - " [[ 4.7917e-06, 1.1917e-05, 1.2315e-05],\n", - " [ 7.0594e-06, 1.1899e-05, 9.0422e-06],\n", - " [ 7.0352e-06, 9.6485e-06, 7.6331e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.2502e-05, 7.2082e-06, 3.5334e-06],\n", - " [ 1.0693e-05, 5.3262e-06, 6.6422e-06],\n", - " [ 4.2715e-06, 3.0487e-06, 4.9546e-06]],\n", - " \n", - " [[ 1.1127e-05, 7.5912e-06, 2.5816e-06],\n", - " [ 1.0917e-05, 8.5382e-06, 3.7998e-06],\n", - " [ 1.1211e-05, 7.8902e-06, 3.1926e-06]],\n", - " \n", - " [[ 5.7612e-06, 8.8997e-06, 8.2571e-06],\n", - " [ 5.9661e-06, 8.7135e-06, 7.1120e-06],\n", - " [ 6.4522e-06, 7.1619e-06, 5.1055e-06]]]], device='cuda:0')),\n", - " ('layer1.0.bn1.weight',\n", - " tensor([ 5.1260e-06, -5.3644e-07, -5.3048e-06, -4.0531e-06, 2.7418e-06,\n", - " -1.9670e-06, 1.1921e-06, -2.9802e-07, -2.9802e-06, 1.0729e-06,\n", - " 2.3842e-07, -1.3113e-06, 8.3447e-07, 4.8876e-06, -5.1260e-06,\n", - " -8.9407e-07, -1.9670e-06, 2.9802e-06, -5.3644e-07, 2.1458e-06,\n", - " 3.3379e-06, 5.1260e-06, -1.9670e-06, 4.4107e-06, 1.7881e-06,\n", - " -2.7418e-06, -2.5630e-06, 2.5034e-06, 1.6689e-06, -1.6689e-06,\n", - " -1.0133e-06, -4.8876e-06, 0.0000e+00, 2.3842e-07, 0.0000e+00,\n", - " -3.3975e-06, -5.3048e-06, 5.9605e-07, -3.0994e-06, 4.0531e-06,\n", - " -1.8477e-06, 1.4305e-06, -3.3379e-06, 1.9073e-06, 1.1921e-06,\n", - " -1.3113e-06, -2.3246e-06, -3.0398e-06, -1.1921e-07, 5.7220e-06,\n", - " 2.3842e-07, 3.5763e-06, -1.4305e-06, -3.6359e-06, 3.4571e-06,\n", - " -2.1458e-06, -1.2517e-06, 4.7684e-07, -3.2187e-06, -1.3113e-06,\n", - " 4.0531e-06, 5.2452e-06, 3.0994e-06, 1.1921e-06], device='cuda:0')),\n", - " ('layer1.0.bn1.bias',\n", - " tensor([ 1.5239e-06, 4.9805e-07, -3.0667e-06, -1.6546e-06, 3.0882e-08,\n", - " 2.4659e-06, 1.3029e-06, 4.9287e-07, -2.1967e-06, -3.9831e-07,\n", - " -2.6451e-06, -5.8212e-06, -4.2027e-07, 3.1355e-06, -1.7615e-06,\n", - " 1.4869e-06, -3.5126e-06, 8.0605e-07, -1.4975e-06, 8.3291e-07,\n", - " -1.3617e-06, 1.6682e-06, -5.2246e-07, -4.0934e-07, 2.7161e-06,\n", - " -1.3181e-06, 8.2625e-07, 3.2450e-08, -3.6535e-06, -9.6101e-07,\n", - " -1.2502e-06, -2.3344e-06, 1.6987e-06, 2.2037e-06, -3.1528e-06,\n", - " -9.9399e-07, -4.4102e-06, -5.3685e-07, 9.5601e-07, 6.0543e-06,\n", - " -3.2821e-06, 6.2275e-07, 2.8308e-06, 2.4347e-07, -1.2094e-06,\n", - " -7.0944e-07, -1.9430e-06, -1.6659e-06, -2.4504e-06, 2.8594e-06,\n", - " 1.8393e-06, 4.1009e-06, -2.6293e-06, -2.9461e-06, 2.8180e-06,\n", - " -5.8443e-06, -7.1812e-07, -4.7377e-07, -3.6096e-06, 1.3539e-06,\n", - " 4.7317e-06, 1.0842e-06, 3.4641e-06, -1.2135e-06], device='cuda:0')),\n", - " ('layer1.0.conv2.weight',\n", - " tensor([[[[ 1.3895e-06, -3.3528e-08, -4.2841e-07],\n", - " [ 1.9986e-06, -9.4064e-08, 3.5092e-06],\n", - " [ 3.0175e-07, -2.3097e-07, 1.2908e-06]],\n", - " \n", - " [[ 2.6971e-06, 4.4099e-06, 2.1607e-07],\n", - " [ 1.7956e-06, 5.1707e-06, 6.4960e-07],\n", - " [ 1.4529e-07, 3.0976e-06, 2.1374e-06]],\n", - " \n", - " [[-6.7800e-07, 7.6741e-07, 1.6876e-06],\n", - " [-4.0978e-07, 9.8348e-07, 1.5469e-06],\n", - " [ 3.4273e-07, 5.9232e-07, 2.0955e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.4887e-06, 3.8017e-06, 3.4627e-06],\n", - " [ 1.4147e-06, 4.8801e-06, -1.4827e-06],\n", - " [-2.4028e-06, 5.4948e-08, -4.2375e-07]],\n", - " \n", - " [[ 6.7800e-07, -1.1288e-06, -1.1846e-06],\n", - " [ 3.8184e-07, -2.4587e-07, 1.1362e-07],\n", - " [ 1.7248e-06, -6.7055e-07, -3.7327e-06]],\n", - " \n", - " [[ 1.7509e-07, -3.7486e-07, -3.2037e-07],\n", - " [ 3.6955e-06, 6.4634e-07, 1.1455e-07],\n", - " [ 2.5462e-06, -7.6089e-07, -4.3586e-07]]],\n", - " \n", - " \n", - " [[[ 6.3553e-06, 5.3002e-06, 5.2452e-06],\n", - " [ 7.3832e-06, 6.2138e-06, 1.1921e-07],\n", - " [ 5.2825e-06, 4.7777e-07, -7.9004e-06]],\n", - " \n", - " [[ 1.6578e-06, 2.6762e-06, 5.0580e-06],\n", - " [-1.2908e-06, -3.6135e-07, 6.6636e-06],\n", - " [-1.0408e-06, -5.8289e-07, 4.3102e-06]],\n", - " \n", - " [[-8.1286e-06, -1.1109e-05, -1.3784e-05],\n", - " [-6.4163e-06, -7.4115e-06, -1.2979e-05],\n", - " [-9.3095e-06, -7.6890e-06, -6.4038e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 5.2415e-06, -1.0207e-06, 3.7961e-06],\n", - " [ 3.5949e-06, -3.6508e-07, 6.2585e-06],\n", - " [ 7.6168e-06, -2.5192e-06, -9.6299e-07]],\n", - " \n", - " [[ 4.4964e-06, -4.7013e-06, -9.3598e-07],\n", - " [ 4.0308e-06, -6.9039e-06, -9.1176e-06],\n", - " [-4.5365e-06, -3.0026e-06, -1.3849e-06]],\n", - " \n", - " [[ 7.5623e-07, -1.5087e-06, 1.3788e-06],\n", - " [-2.3041e-06, 1.5795e-06, 4.9621e-06],\n", - " [-1.7043e-06, 4.3698e-06, 3.5278e-06]]],\n", - " \n", - " \n", - " [[[-1.1008e-06, 1.9707e-06, -6.1467e-07],\n", - " [-2.0806e-06, -3.7812e-07, 1.9372e-06],\n", - " [-5.2229e-06, -5.2806e-07, 1.8217e-06]],\n", - " \n", - " [[-7.3283e-07, -3.6582e-06, 6.3330e-08],\n", - " [-2.1700e-07, -2.0089e-06, -2.1532e-06],\n", - " [-2.6785e-06, -3.9823e-06, -2.2296e-06]],\n", - " \n", - " [[ 1.0082e-05, 9.0534e-06, 9.5973e-06],\n", - " [ 1.2001e-05, 1.0580e-05, 9.1903e-06],\n", - " [ 1.0397e-05, 9.9298e-06, 7.8939e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-8.7917e-07, -3.6135e-07, -1.3039e-08],\n", - " [-4.3614e-06, 1.5292e-06, -3.7029e-06],\n", - " [-5.6848e-06, -2.7260e-06, 4.5169e-08]],\n", - " \n", - " [[ 3.4794e-06, 4.3632e-06, 4.2007e-06],\n", - " [ 1.5423e-06, 7.4264e-06, 1.1304e-05],\n", - " [ 1.2834e-06, 5.1297e-06, 3.8533e-06]],\n", - " \n", - " [[-1.0051e-05, -1.1733e-05, -4.7721e-06],\n", - " [-1.1101e-05, -1.0878e-05, -3.0957e-06],\n", - " [-1.3369e-05, -1.7643e-05, -6.2836e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-1.1660e-06, -2.5872e-06, -1.9646e-06],\n", - " [ 5.0291e-06, 1.0729e-06, 2.1732e-06],\n", - " [ 1.0667e-05, 8.1407e-06, 5.6550e-06]],\n", - " \n", - " [[-5.3532e-06, -6.5267e-06, -3.6673e-06],\n", - " [-3.4217e-06, -4.2040e-06, -5.0038e-06],\n", - " [-1.1306e-06, 3.8464e-07, -2.1746e-06]],\n", - " \n", - " [[ 4.5984e-06, 3.9041e-06, 4.9509e-06],\n", - " [ 9.3877e-07, 6.5751e-07, 1.6717e-06],\n", - " [-2.3693e-06, -2.2501e-06, -2.1830e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.7567e-06, 7.0781e-08, -5.0766e-06],\n", - " [ 1.0319e-06, 2.7232e-06, -2.2780e-06],\n", - " [ 4.1490e-06, 3.7365e-06, -5.4389e-07]],\n", - " \n", - " [[ 5.7966e-06, 5.7369e-07, 2.8405e-06],\n", - " [ 8.0094e-07, -3.5502e-06, -2.0489e-06],\n", - " [-1.8403e-06, -4.5821e-06, -4.9067e-06]],\n", - " \n", - " [[-2.1290e-06, -4.4480e-06, -1.9595e-06],\n", - " [-7.9069e-07, -5.0232e-06, -6.1356e-06],\n", - " [ 1.4706e-06, 3.2596e-06, 4.5076e-07]]],\n", - " \n", - " \n", - " [[[-5.4576e-07, 3.9525e-06, 3.7681e-06],\n", - " [-3.3192e-06, -1.7975e-07, 1.1645e-06],\n", - " [-5.7463e-06, -1.2405e-06, 7.0781e-07]],\n", - " \n", - " [[ 1.5423e-06, 4.5206e-06, 3.3192e-06],\n", - " [ 2.4252e-06, 4.7460e-06, 2.8666e-06],\n", - " [ 2.0682e-06, 2.6338e-06, 3.3146e-06]],\n", - " \n", - " [[ 9.0497e-06, 4.4964e-06, 5.3942e-06],\n", - " [ 6.2836e-06, 4.9816e-06, 3.9216e-06],\n", - " [ 8.8476e-07, 7.0100e-08, -1.4864e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.7877e-06, 2.3507e-06, 9.4641e-06],\n", - " [-2.4065e-06, -1.6689e-06, 1.6689e-06],\n", - " [ 4.8913e-06, 6.5602e-06, 9.1582e-06]],\n", - " \n", - " [[ 8.3260e-07, 5.4054e-06, 9.8087e-06],\n", - " [ 2.7306e-06, 5.0971e-06, 5.7071e-06],\n", - " [-1.0947e-05, -1.7285e-06, 6.8583e-06]],\n", - " \n", - " [[-2.0545e-06, 5.8860e-07, -5.7369e-07],\n", - " [-7.1805e-07, -6.4075e-07, -2.0452e-06],\n", - " [ 5.8413e-06, 3.0590e-06, 5.4822e-06]]],\n", - " \n", - " \n", - " [[[ 1.8328e-06, 3.6284e-06, 3.0231e-06],\n", - " [-4.5858e-06, -2.7753e-06, -1.2238e-06],\n", - " [-3.0724e-06, -6.3702e-07, 1.8217e-06]],\n", - " \n", - " [[-1.2824e-06, 4.4703e-08, 5.2080e-06],\n", - " [ 0.0000e+00, -7.9535e-07, 3.9994e-06],\n", - " [ 1.2526e-06, 1.8626e-06, 6.1337e-06]],\n", - " \n", - " [[-4.1947e-06, -7.7859e-06, -1.0822e-05],\n", - " [-3.0547e-06, -4.3726e-06, -8.1249e-06],\n", - " [-3.7402e-06, -4.1574e-06, -8.7796e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.6508e-06, -2.3860e-06, 2.8871e-08],\n", - " [-7.7672e-07, 6.6496e-07, 8.0466e-07],\n", - " [-3.4682e-06, -3.6177e-06, -2.9251e-06]],\n", - " \n", - " [[-6.0499e-06, -5.2142e-06, -6.7689e-06],\n", - " [-1.3324e-05, -7.8473e-06, -1.1117e-05],\n", - " [-1.5919e-05, -1.0084e-05, -1.3931e-05]],\n", - " \n", - " [[ 6.3349e-06, 4.5877e-06, 4.8876e-06],\n", - " [ 6.6347e-06, 8.4620e-06, 9.3040e-06],\n", - " [ 6.8210e-06, 5.1488e-06, 7.3761e-06]]]], device='cuda:0')),\n", - " ('layer1.0.bn2.weight',\n", - " tensor([-3.9339e-06, 5.7220e-06, -1.0729e-06, -1.6689e-06, -3.0994e-06,\n", - " 7.1526e-07, -1.3113e-06, 2.0266e-06, -3.5763e-06, -2.7418e-06,\n", - " 4.0531e-06, -3.2783e-06, 2.1458e-06, -1.3709e-06, -8.9407e-07,\n", - " -1.1325e-06, -4.0531e-06, 4.2915e-06, 4.7684e-06, -1.5497e-06,\n", - " -7.1526e-07, -1.3113e-06, -1.8477e-06, -4.2319e-06, -1.7285e-06,\n", - " 2.1458e-06, -6.5565e-07, 2.0266e-06, 3.0994e-06, 4.1723e-06,\n", - " -1.0729e-06, 1.9073e-06, -2.6822e-06, 3.0994e-06, 2.8610e-06,\n", - " -2.0266e-06, -3.0994e-06, -2.9802e-07, -2.0266e-06, -1.1921e-07,\n", - " -1.9670e-06, -5.3048e-06, -8.9407e-07, -2.5034e-06, -1.6689e-06,\n", - " 1.0729e-06, -3.3379e-06, 2.5034e-06, -2.9802e-06, -5.9009e-06,\n", - " 1.6689e-06, 2.9802e-06, -2.9802e-07, 1.1921e-06, -3.3379e-06,\n", - " -2.3842e-07, -9.5367e-07, 1.7881e-06, -2.3246e-06, -1.6093e-06,\n", - " -1.1921e-07, 3.8147e-06, -2.3246e-06, 2.0266e-06], device='cuda:0')),\n", - " ('layer1.0.bn2.bias',\n", - " tensor([ 1.5868e-07, 9.3977e-07, -1.1009e-06, -3.2320e-06, 1.3682e-06,\n", - " -4.9855e-07, 2.4686e-06, 2.8355e-06, 5.1637e-07, -2.9283e-07,\n", - " -9.8881e-07, 1.4233e-06, 1.7544e-06, -1.6418e-06, 2.7256e-06,\n", - " 1.7330e-07, -2.3644e-06, 3.8721e-07, 2.2418e-06, -5.5773e-07,\n", - " -1.0968e-06, 2.1732e-06, -1.0681e-06, 4.4223e-06, 8.2948e-08,\n", - " -1.1339e-06, 2.2975e-06, 1.7483e-06, 1.5275e-06, 2.7966e-06,\n", - " -1.4331e-06, 5.2383e-06, 1.9282e-07, -8.2710e-08, -1.4253e-06,\n", - " -5.4586e-08, -2.7365e-06, -9.0235e-07, 1.1680e-06, 2.5383e-06,\n", - " 7.6398e-07, -1.3603e-06, 1.1668e-06, -9.5251e-07, 4.4751e-06,\n", - " 1.6577e-06, 3.6312e-07, 1.1681e-06, -4.1789e-06, 7.3535e-08,\n", - " 3.8469e-06, 3.4083e-06, -1.3044e-06, 9.2400e-07, -1.1203e-06,\n", - " -3.5916e-06, -1.6950e-06, -9.8079e-07, -1.2194e-06, -6.2152e-07,\n", - " 6.9892e-07, 2.2628e-06, -2.1050e-06, 6.5259e-07], device='cuda:0')),\n", - " ('layer2.0.conv1.weight',\n", - " tensor([[[[ 1.5902e-06, 1.3839e-06, 5.2154e-08],\n", - " [ 1.7993e-06, -5.0105e-07, -1.0366e-06],\n", - " [-1.5534e-06, -2.4866e-06, -3.2764e-06]],\n", - " \n", - " [[-7.5586e-06, -3.6433e-06, -2.8238e-06],\n", - " [-4.4589e-06, -4.8950e-06, -9.1996e-06],\n", - " [ 5.7463e-07, -3.7588e-06, -6.7241e-06]],\n", - " \n", - " [[-1.9744e-07, 6.6124e-08, -4.3390e-06],\n", - " [ 1.7229e-07, -4.2313e-06, -3.4543e-06],\n", - " [ 2.0191e-06, 8.6403e-07, -1.3607e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 6.5677e-06, 2.8005e-06, 2.8703e-06],\n", - " [ 2.7921e-06, 2.2464e-06, 4.3809e-06],\n", - " [ 1.1988e-06, 1.4380e-06, 2.6226e-06]],\n", - " \n", - " [[-4.7684e-06, -3.1590e-06, -2.2165e-06],\n", - " [-3.4925e-06, -1.7174e-06, -1.8254e-07],\n", - " [-5.8524e-06, -6.5900e-06, -3.0585e-06]],\n", - " \n", - " [[-8.0280e-06, -5.0226e-06, -3.7458e-06],\n", - " [-2.5481e-06, 1.8463e-07, 1.3430e-06],\n", - " [-4.3772e-07, 8.0094e-08, 2.3153e-06]]],\n", - " \n", - " \n", - " [[[-7.4436e-07, -4.0173e-06, -5.8236e-06],\n", - " [-6.0350e-07, -2.9355e-06, -4.2031e-06],\n", - " [ 1.5460e-06, -2.3264e-06, -1.8524e-06]],\n", - " \n", - " [[-9.7975e-07, -2.8834e-06, -2.8163e-06],\n", - " [-7.4878e-07, -1.0943e-06, -6.7474e-07],\n", - " [-4.4862e-06, -3.2000e-06, 8.1118e-07]],\n", - " \n", - " [[-7.4226e-07, 5.9046e-07, 5.6997e-07],\n", - " [-3.7439e-07, -6.3679e-07, 1.1697e-06],\n", - " [-1.7844e-06, -1.1744e-06, -2.1379e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.9339e-06, -2.0631e-06, -1.2554e-06],\n", - " [ 7.0781e-08, -2.6843e-06, -2.2678e-06],\n", - " [ 3.7067e-06, -1.6356e-06, -1.4366e-06]],\n", - " \n", - " [[-1.1008e-06, 2.7190e-06, 5.2266e-06],\n", - " [ 1.7560e-06, 6.4149e-06, 5.0552e-06],\n", - " [-2.5071e-06, 5.1111e-06, 1.0952e-06]],\n", - " \n", - " [[-4.3288e-06, -4.2375e-08, -6.1188e-07],\n", - " [-5.4818e-06, -5.6252e-07, 1.9334e-06],\n", - " [-3.3206e-06, -8.2329e-07, 1.2852e-06]]],\n", - " \n", - " \n", - " [[[-2.8536e-06, -1.1292e-06, -9.2294e-07],\n", - " [-5.5768e-06, -1.2964e-06, -1.2219e-06],\n", - " [-4.1910e-06, -1.2228e-06, 9.2015e-07]],\n", - " \n", - " [[ 3.3304e-06, 4.1276e-06, 2.2016e-06],\n", - " [ 7.6313e-06, 9.5479e-06, 4.8429e-06],\n", - " [ 7.2848e-06, 9.1661e-06, 5.1521e-06]],\n", - " \n", - " [[ 3.7234e-06, 3.9302e-06, -1.7323e-06],\n", - " [ 1.8487e-06, 1.8633e-06, -2.1700e-06],\n", - " [-1.0543e-06, 8.4564e-07, -2.5992e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.2522e-06, -6.2212e-07, -1.0729e-06],\n", - " [-5.3775e-06, 8.4564e-07, 1.0915e-06],\n", - " [-2.3246e-06, 9.8348e-07, 1.4007e-06]],\n", - " \n", - " [[ 5.3765e-06, -1.1027e-06, 3.7532e-06],\n", - " [-1.6410e-06, -6.9682e-06, -1.2703e-06],\n", - " [-5.6522e-06, -5.5939e-06, -7.5437e-07]],\n", - " \n", - " [[ 6.2548e-06, -7.1898e-07, -2.9057e-06],\n", - " [ 3.6843e-06, -4.1574e-06, -5.1297e-06],\n", - " [ 4.4182e-06, -7.5880e-07, -3.2485e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-4.8848e-07, -2.2370e-06, -7.3761e-07],\n", - " [-2.6040e-06, -2.7157e-06, 6.4448e-07],\n", - " [-4.6734e-06, -5.6233e-06, -1.1399e-06]],\n", - " \n", - " [[ 1.5744e-06, 4.2841e-07, 2.7604e-06],\n", - " [ 4.1211e-07, -1.1679e-06, 1.5814e-06],\n", - " [ 4.0978e-08, 6.5006e-07, 3.0883e-06]],\n", - " \n", - " [[ 1.4529e-07, -3.7104e-06, 1.3374e-06],\n", - " [-2.7436e-06, -5.1670e-06, -7.4878e-07],\n", - " [-1.9148e-06, -5.0664e-07, 3.9861e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.8871e-08, -1.3355e-06, 2.4214e-06],\n", - " [ 1.5683e-06, 3.0920e-07, 2.9989e-06],\n", - " [ 8.9779e-07, -4.2096e-07, 2.5444e-06]],\n", - " \n", - " [[-6.6869e-07, -6.1095e-07, -9.7975e-07],\n", - " [-2.5481e-06, -4.0948e-06, -3.0454e-06],\n", - " [-8.5682e-07, -1.7434e-06, -2.9467e-06]],\n", - " \n", - " [[ 5.7369e-07, -2.5863e-06, -2.7101e-06],\n", - " [ 4.2096e-07, -1.4342e-06, -3.4403e-06],\n", - " [-2.2538e-06, -3.6289e-06, -3.2485e-06]]],\n", - " \n", - " \n", - " [[[ 3.2373e-06, 1.9334e-06, 1.7742e-07],\n", - " [ 3.5670e-06, 3.5502e-06, 3.4412e-06],\n", - " [ 4.6641e-06, 3.3602e-06, 3.5083e-06]],\n", - " \n", - " [[ 1.0449e-06, -1.8813e-07, 1.7798e-06],\n", - " [ 2.2855e-06, -6.5751e-07, 1.9576e-06],\n", - " [ 1.7090e-06, -2.1979e-06, -1.7575e-06]],\n", - " \n", - " [[ 1.3299e-06, 1.1865e-06, 3.3714e-07],\n", - " [-7.8231e-07, -2.0489e-06, 4.3232e-06],\n", - " [-2.2352e-08, -5.4110e-06, -2.6571e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.4752e-06, -1.1646e-06, -1.8552e-06],\n", - " [ 1.1590e-06, -5.8860e-07, -8.1956e-07],\n", - " [ 2.2389e-06, -8.1956e-08, -3.5157e-07]],\n", - " \n", - " [[ 1.4435e-06, 1.6987e-06, 3.6778e-06],\n", - " [ 1.5367e-06, 3.9674e-07, 5.6811e-07],\n", - " [-1.2927e-06, -1.4920e-06, -3.2373e-06]],\n", - " \n", - " [[ 2.4848e-06, 3.2182e-06, 2.7558e-06],\n", - " [ 1.3527e-06, 5.2899e-07, 6.3516e-07],\n", - " [ 2.4633e-06, 7.4506e-07, 5.3272e-07]]],\n", - " \n", - " \n", - " [[[-1.1139e-06, -5.5134e-07, 1.3784e-06],\n", - " [-1.5832e-07, 2.1681e-06, 2.5332e-06],\n", - " [ 8.7172e-07, 1.8976e-06, 1.9995e-06]],\n", - " \n", - " [[ 9.8720e-07, -7.5344e-07, -1.0161e-06],\n", - " [-1.2331e-06, -6.7614e-07, -8.3819e-07],\n", - " [-1.3132e-07, 1.3988e-06, 2.1905e-06]],\n", - " \n", - " [[ 3.2950e-06, 1.7965e-06, 1.4603e-06],\n", - " [ 3.5167e-06, 1.9483e-06, 3.4384e-06],\n", - " [ 4.1015e-06, 5.1074e-06, 4.8671e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.3027e-07, 1.9632e-06, 2.3972e-06],\n", - " [ 1.1725e-06, -2.5332e-07, 5.9977e-07],\n", - " [ 2.3171e-06, -9.8720e-07, 3.0175e-07]],\n", - " \n", - " [[-1.1092e-06, 2.0117e-07, -2.1085e-06],\n", - " [ 7.3016e-07, 2.3767e-06, 2.7288e-06],\n", - " [ 8.9779e-07, 1.3616e-06, 5.2061e-07]],\n", - " \n", - " [[ 6.9663e-07, -2.1420e-07, -1.7919e-06],\n", - " [ 2.5257e-06, 3.1330e-06, -4.6194e-07],\n", - " [ 2.1383e-06, 2.7437e-06, 1.3290e-06]]]], device='cuda:0')),\n", - " ('layer2.0.bn1.weight',\n", - " tensor([ 3.0994e-06, -2.3246e-06, 4.6492e-06, 7.1526e-07, 3.3379e-06,\n", - " -1.4305e-06, -1.6689e-06, 1.1921e-06, -4.1723e-07, 7.1526e-07,\n", - " 1.3113e-06, 1.3113e-06, 1.0729e-06, -1.7285e-06, -5.3644e-07,\n", - " -2.9802e-07, -4.7684e-07, -2.0266e-06, 2.3842e-07, -5.3644e-07,\n", - " 1.6689e-06, 4.7684e-07, -5.3644e-07, 2.3842e-07, 0.0000e+00,\n", - " -5.9605e-08, -7.7486e-07, -6.5565e-07, 1.9073e-06, -2.6226e-06,\n", - " -7.1526e-07, 2.3842e-07, -1.5497e-06, -2.3842e-07, -2.4438e-06,\n", - " 5.9605e-07, -3.5763e-07, 1.1921e-06, -3.2783e-06, 2.5034e-06,\n", - " -1.6689e-06, 2.0266e-06, 2.8610e-06, 2.0266e-06, -1.0729e-06,\n", - " 1.1921e-07, -1.6689e-06, 9.5367e-07, 9.5367e-07, 5.9605e-07,\n", - " -1.7881e-06, -1.0729e-06, -1.4305e-06, 4.7684e-07, 2.0266e-06,\n", - " -1.1921e-06, 2.3842e-07, 1.1921e-06, 1.6689e-06, -2.4438e-06,\n", - " -1.9670e-06, -1.7881e-06, 2.3842e-07, 1.5497e-06, 1.6689e-06,\n", - " -1.7881e-07, 2.3842e-07, -1.5497e-06, 1.1921e-06, 1.5497e-06,\n", - " -1.6093e-06, -2.2054e-06, -2.0862e-06, -1.6689e-06, -1.4305e-06,\n", - " 1.4305e-06, -8.3447e-07, -2.6226e-06, -1.1921e-06, 2.1458e-06,\n", - " -2.7418e-06, 3.5763e-07, -1.1921e-07, -1.1325e-06, -1.4901e-06,\n", - " 3.6955e-06, -1.7285e-06, -2.4438e-06, -6.5565e-07, -3.5763e-07,\n", - " 1.3113e-06, -4.7684e-07, -9.5367e-07, -5.9605e-07, 1.6689e-06,\n", - " 4.7684e-07, 4.7684e-07, 7.1526e-07, -2.3246e-06, 1.1921e-06,\n", - " -4.1723e-07, 2.3842e-07, 0.0000e+00, 2.6226e-06, -2.0862e-06,\n", - " 5.9605e-07, 2.3842e-06, 1.1921e-06, 1.5497e-06, -1.4901e-06,\n", - " 9.5367e-07, -2.3842e-07, 1.1921e-06, 2.5034e-06, 8.3447e-07,\n", - " 7.1526e-07, -2.9802e-06, 1.1921e-07, 1.4305e-06, 1.4305e-06,\n", - " 3.5763e-07, 1.3113e-06, 2.3842e-06, -1.2517e-06, -2.3842e-06,\n", - " -3.1590e-06, -3.5763e-07, 2.1458e-06], device='cuda:0')),\n", - " ('layer2.0.bn1.bias',\n", - " tensor([ 1.0771e-06, -1.8999e-06, 2.3541e-06, 9.5966e-07, 8.2739e-07,\n", - " 6.1912e-07, -1.2996e-06, -1.6184e-08, 4.1995e-07, -1.6756e-06,\n", - " 1.3479e-06, 1.0882e-06, 1.9268e-06, -4.1985e-07, -1.1347e-06,\n", - " 5.0927e-07, 2.6343e-07, 4.0396e-07, 1.0203e-07, -8.1831e-07,\n", - " -9.6027e-07, 6.5029e-07, -3.9052e-07, -9.8193e-09, -2.3134e-07,\n", - " -1.4903e-06, 1.7103e-07, -1.4026e-06, 7.8846e-07, -1.4667e-06,\n", - " 5.2717e-07, -1.0885e-06, -1.0228e-06, -1.1879e-06, -1.7712e-06,\n", - " -9.4132e-07, 1.8629e-06, 1.4127e-06, -3.9082e-06, 4.5633e-06,\n", - " 2.9374e-07, 4.0827e-06, 2.5944e-06, -1.6692e-07, 2.0453e-07,\n", - " -7.2985e-07, -2.6793e-06, 2.5667e-06, -1.1951e-07, 1.1597e-06,\n", - " -2.8109e-06, -7.0437e-07, -2.7890e-06, -9.6153e-07, 8.7216e-07,\n", - " -1.2926e-06, 4.4592e-07, 1.8612e-06, 1.7681e-06, -1.9762e-06,\n", - " -5.1246e-07, -1.2139e-06, -6.3102e-07, 5.6782e-07, 1.6684e-06,\n", - " -5.8652e-10, 1.3252e-06, -3.3027e-07, 5.1575e-07, 1.8013e-06,\n", - " -1.7359e-06, -1.2953e-06, -1.7131e-06, -9.5531e-07, -2.4773e-06,\n", - " 9.2660e-07, -6.0866e-07, 6.1627e-08, -1.3812e-06, 2.8934e-06,\n", - " -3.2839e-06, 6.3595e-07, 6.5303e-07, 5.2513e-07, -1.0461e-06,\n", - " 1.2402e-06, -1.7747e-06, -2.3992e-06, -1.1695e-06, -2.2699e-07,\n", - " 1.1653e-06, -3.9334e-07, -1.7928e-07, -2.2050e-06, 2.8558e-06,\n", - " -4.4786e-08, 1.0607e-06, 9.8461e-07, -3.5410e-07, 4.3437e-07,\n", - " -9.9145e-07, -5.3667e-07, -2.7750e-07, 7.5311e-07, -1.4992e-06,\n", - " -8.5458e-08, 2.2051e-06, 1.2309e-06, -4.6171e-07, -1.5314e-06,\n", - " 1.3039e-06, 3.8103e-07, 1.0440e-06, 7.3113e-07, 3.6085e-07,\n", - " 1.0573e-06, -1.3022e-06, -8.5626e-07, -8.9232e-07, 1.5499e-06,\n", - " -2.2795e-06, 2.6757e-08, 4.8539e-07, 1.0232e-06, -1.1481e-06,\n", - " -2.9484e-06, 3.6970e-07, 2.1285e-06], device='cuda:0')),\n", - " ('layer2.0.conv2.weight',\n", - " tensor([[[[-2.1514e-06, -4.4517e-07, -2.9746e-06],\n", - " [ 1.1344e-06, 3.4533e-06, -6.4541e-07],\n", - " [-2.0098e-06, -8.0187e-07, 9.8906e-07]],\n", - " \n", - " [[-3.3062e-06, 1.6661e-06, -5.7537e-06],\n", - " [-5.1633e-06, -3.9916e-06, -4.7423e-06],\n", - " [-1.5590e-06, -5.8301e-07, -1.4924e-06]],\n", - " \n", - " [[ 6.4896e-07, 2.2817e-06, 2.6803e-06],\n", - " [ 2.0345e-06, 1.3877e-06, 7.6555e-07],\n", - " [-7.0175e-07, 2.4941e-06, -2.8219e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.3395e-06, -1.1553e-06, -1.6065e-06],\n", - " [-7.0222e-07, -1.7192e-06, -1.4841e-06],\n", - " [ 7.2271e-07, -6.1467e-07, -2.3991e-06]],\n", - " \n", - " [[ 1.8273e-06, 4.4936e-06, 3.6415e-06],\n", - " [ 4.7404e-07, -1.6363e-06, -2.8387e-06],\n", - " [ 1.9139e-07, 3.1106e-07, 9.4343e-07]],\n", - " \n", - " [[-1.4259e-06, 1.3872e-06, -1.6890e-06],\n", - " [ 1.8533e-07, -1.7621e-06, -1.7360e-06],\n", - " [ 9.9931e-07, 5.2154e-07, -8.0717e-07]]],\n", - " \n", - " \n", - " [[[ 1.7267e-06, 1.5832e-06, -2.2966e-06],\n", - " [ 3.2969e-07, -1.9427e-06, -1.2852e-07],\n", - " [-7.7300e-08, 3.0920e-07, -1.5665e-06]],\n", - " \n", - " [[-2.5411e-06, -2.0552e-06, -3.6787e-06],\n", - " [-2.5965e-06, 9.0152e-07, 7.1153e-07],\n", - " [ 1.2773e-06, -4.9733e-07, 1.8459e-06]],\n", - " \n", - " [[ 5.9893e-06, 5.8422e-06, 4.9574e-06],\n", - " [-8.1677e-07, 4.0879e-06, 1.6466e-06],\n", - " [-3.8650e-06, 6.7134e-06, 1.2349e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.0419e-06, -4.0117e-07, -2.0582e-06],\n", - " [-3.5390e-06, 2.6021e-06, 2.2091e-06],\n", - " [ 6.0376e-07, 1.8068e-06, 8.2888e-06]],\n", - " \n", - " [[-3.5036e-06, -5.3332e-06, -3.3472e-06],\n", - " [ 2.8405e-06, 4.3381e-06, 7.2783e-07],\n", - " [ 1.2168e-06, 2.5146e-06, 8.4937e-07]],\n", - " \n", - " [[-5.7928e-06, 6.7428e-07, -1.7097e-06],\n", - " [-3.8249e-06, 3.7663e-06, -1.0133e-06],\n", - " [-4.5784e-06, 2.1448e-06, 5.2340e-07]]],\n", - " \n", - " \n", - " [[[-1.0058e-07, 9.3319e-07, 6.0908e-07],\n", - " [-2.2706e-06, -1.8161e-06, 1.3597e-07],\n", - " [ 1.9195e-06, -5.4017e-08, 7.3528e-07]],\n", - " \n", - " [[-3.7653e-06, -1.3839e-06, -1.8850e-06],\n", - " [ 2.2743e-06, -3.7043e-06, 2.1104e-06],\n", - " [ 2.0303e-07, -2.9318e-06, -1.9697e-06]],\n", - " \n", - " [[ 5.2527e-07, -2.2016e-06, -1.6796e-06],\n", - " [-4.0112e-06, -3.6266e-06, -2.6468e-06],\n", - " [-2.9904e-06, -3.6880e-06, -2.9420e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.5777e-06, 1.3664e-06, 2.5253e-06],\n", - " [-1.0487e-06, -1.3374e-06, 2.2696e-06],\n", - " [-3.2429e-06, -2.6189e-06, -8.7358e-07]],\n", - " \n", - " [[ 2.0545e-06, -1.5879e-07, 6.7288e-07],\n", - " [ 2.9467e-06, 2.0145e-06, 1.0217e-06],\n", - " [-3.0193e-06, -2.2622e-06, 4.6939e-07]],\n", - " \n", - " [[-3.4696e-06, 8.2515e-07, 1.4845e-06],\n", - " [-8.0746e-07, 1.3467e-06, 4.7041e-06],\n", - " [-1.9581e-06, 5.1195e-06, 2.6107e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 2.8685e-07, 1.9372e-07, -9.9884e-07],\n", - " [ 2.5108e-06, -6.0175e-07, 1.4156e-07],\n", - " [-2.3656e-07, -9.0338e-08, 1.5652e-06]],\n", - " \n", - " [[-3.6918e-06, 5.6997e-07, -3.8557e-07],\n", - " [-1.2922e-07, -2.5872e-06, 8.2515e-07],\n", - " [ 7.7114e-07, 4.8988e-07, -3.9302e-07]],\n", - " \n", - " [[-3.8408e-06, -1.8626e-08, -2.6049e-06],\n", - " [-3.6149e-06, -4.4340e-06, -2.1178e-06],\n", - " [-3.2354e-06, -3.5204e-07, -2.5909e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.1048e-06, 3.4748e-06, 2.1970e-06],\n", - " [-8.9407e-08, -1.5143e-06, -6.6869e-07],\n", - " [-1.8440e-07, 4.0280e-07, 1.4566e-06]],\n", - " \n", - " [[ 1.6391e-06, 1.5753e-06, 5.0291e-08],\n", - " [-2.1998e-06, -1.4798e-06, -3.2410e-07],\n", - " [-1.6149e-06, -1.6035e-06, -1.8626e-07]],\n", - " \n", - " [[ 1.3895e-06, -6.9290e-07, 1.0058e-06],\n", - " [ 9.7603e-07, 1.8254e-07, 2.1122e-06],\n", - " [ 1.1222e-06, -1.3644e-06, 1.4249e-06]]],\n", - " \n", - " \n", - " [[[ 9.8883e-07, 1.2954e-06, 1.1176e-08],\n", - " [ 1.9465e-06, 1.8999e-07, 9.9652e-07],\n", - " [ 1.0934e-06, 3.8464e-06, 3.0501e-06]],\n", - " \n", - " [[ 2.5611e-06, 5.6438e-07, 2.6748e-06],\n", - " [-5.2934e-06, -3.7625e-07, 1.7006e-06],\n", - " [-2.6617e-06, -4.5151e-06, -6.0908e-07]],\n", - " \n", - " [[ 5.6410e-06, 4.3306e-06, 4.9236e-06],\n", - " [ 1.6531e-06, 5.6252e-07, 1.8831e-06],\n", - " [-1.4342e-06, 1.5864e-06, 1.9833e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.6678e-06, 1.4324e-06, 4.2813e-06],\n", - " [-4.9733e-07, -3.1106e-07, 5.8115e-07],\n", - " [ 1.6531e-07, -2.5555e-06, -8.4937e-07]],\n", - " \n", - " [[ 2.5015e-06, 1.5032e-06, 1.0561e-06],\n", - " [ 3.4347e-06, 9.9372e-07, 2.5593e-06],\n", - " [-3.6322e-07, -3.2689e-07, -3.8557e-07]],\n", - " \n", - " [[-8.7707e-07, -1.1334e-06, -2.5295e-06],\n", - " [ 5.0152e-07, -3.6117e-06, 1.3619e-06],\n", - " [ 1.2880e-06, -1.3337e-06, 1.8906e-06]]],\n", - " \n", - " \n", - " [[[ 2.3991e-06, -1.5143e-06, 7.7672e-07],\n", - " [-5.1153e-07, -5.9344e-06, 6.1095e-07],\n", - " [ 2.1607e-06, 3.4459e-07, -1.2433e-06]],\n", - " \n", - " [[ 8.2888e-07, 3.4738e-07, 3.4310e-06],\n", - " [-2.8498e-07, 8.0653e-07, 1.4659e-06],\n", - " [ 1.7006e-06, 1.2480e-07, 2.5472e-07]],\n", - " \n", - " [[-3.4552e-06, -4.0336e-06, -4.9975e-06],\n", - " [-2.2650e-06, -6.0592e-06, -1.8515e-06],\n", - " [-9.5461e-07, -1.8887e-06, -6.9030e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.2055e-06, 5.0664e-07, -1.9995e-06],\n", - " [-4.9919e-07, -1.3830e-06, 4.9779e-07],\n", - " [-4.6324e-06, -1.8235e-06, 1.4249e-06]],\n", - " \n", - " [[-2.1365e-06, -2.4159e-06, -2.1439e-06],\n", - " [-1.3076e-06, -4.5174e-06, -5.5879e-09],\n", - " [-1.0505e-06, -4.1085e-06, -3.5774e-06]],\n", - " \n", - " [[ 2.1681e-06, 4.9733e-07, 1.4934e-06],\n", - " [ 4.3348e-06, 2.0023e-06, 4.0140e-06],\n", - " [ 2.2836e-06, 5.8487e-07, 2.8322e-06]]]], device='cuda:0')),\n", - " ('layer2.0.bn2.weight',\n", - " tensor([-8.9407e-07, -1.1921e-07, 1.1921e-07, 2.0266e-06, 2.3842e-06,\n", - " -5.9605e-08, -2.0862e-06, -1.4901e-06, -1.4305e-06, -5.9605e-07,\n", - " 1.1921e-07, -1.0729e-06, -1.8477e-06, 1.9073e-06, 1.1921e-07,\n", - " -1.4305e-06, -1.0133e-06, -2.6822e-06, 0.0000e+00, 2.3842e-07,\n", - " -1.0133e-06, 5.9605e-07, -4.7684e-07, -1.1921e-07, -1.1921e-06,\n", - " -1.6093e-06, 1.1921e-07, -7.7486e-07, 1.9073e-06, -1.7285e-06,\n", - " -1.0133e-06, -1.7881e-06, 1.5497e-06, 1.4305e-06, 2.3842e-07,\n", - " 1.1921e-07, 1.0729e-06, 1.9073e-06, -2.2054e-06, -1.3709e-06,\n", - " -2.0862e-06, -3.5763e-07, 4.7684e-07, -1.1921e-07, 1.1921e-06,\n", - " -1.4305e-06, -1.3113e-06, -1.9670e-06, -4.1723e-07, 1.6689e-06,\n", - " 4.2915e-06, 1.0729e-06, 9.5367e-07, 9.5367e-07, -2.0266e-06,\n", - " 0.0000e+00, -1.1325e-06, 1.0729e-06, -2.4438e-06, 1.1921e-07,\n", - " 1.1921e-07, 5.9605e-07, 1.1921e-06, 8.3447e-07, -2.9802e-07,\n", - " -1.4901e-06, -1.0729e-06, 4.7684e-07, -7.1526e-07, 2.3842e-06,\n", - " -9.5367e-07, 7.1526e-07, 2.1458e-06, -1.6689e-06, -1.1921e-07,\n", - " -8.3447e-07, -3.5763e-07, 1.1921e-07, -1.9670e-06, -7.1526e-07,\n", - " 4.7684e-07, -1.1921e-07, 1.1921e-07, 2.5034e-06, 4.0531e-06,\n", - " -2.0266e-06, 2.3842e-06, 1.1921e-07, 3.5763e-06, -5.9605e-08,\n", - " -1.6689e-06, 9.5367e-07, 0.0000e+00, 8.3447e-07, 1.1921e-07,\n", - " 5.9605e-07, 1.3113e-06, -1.0729e-06, -4.7684e-07, 1.9073e-06,\n", - " -1.4305e-06, -5.9605e-08, 1.1921e-07, -4.1723e-07, 1.4305e-06,\n", - " 2.3842e-07, -1.3709e-06, -3.3975e-06, 5.9605e-07, -1.8477e-06,\n", - " -7.1526e-07, 9.5367e-07, -1.2517e-06, -2.0862e-06, 1.5497e-06,\n", - " 1.6689e-06, 7.1526e-07, 1.6689e-06, -8.9407e-07, 1.7881e-06,\n", - " -2.3842e-07, -1.6093e-06, 9.5367e-07, 1.1921e-07, -5.4836e-06,\n", - " -1.1921e-06, -3.5763e-07, 1.1921e-07], device='cuda:0')),\n", - " ('layer2.0.bn2.bias',\n", - " tensor([-1.9510e-06, 1.9011e-07, -9.3751e-07, 1.4218e-06, 1.5925e-06,\n", - " -4.4937e-07, 1.6194e-07, 1.3984e-06, -9.2120e-07, -1.6821e-07,\n", - " -1.0528e-06, -8.1123e-07, 5.3874e-07, 9.9689e-10, -7.2899e-07,\n", - " -1.2392e-06, 5.1126e-07, -7.3782e-07, -1.4009e-06, -3.6747e-07,\n", - " -1.5048e-07, 3.9299e-07, 8.8879e-07, 1.2054e-06, 4.6736e-07,\n", - " -4.8633e-07, 6.5945e-07, 4.0367e-07, 1.0216e-06, -5.3070e-08,\n", - " 6.0140e-07, -1.6197e-06, 1.6229e-06, -1.6424e-08, -5.1029e-07,\n", - " -4.7327e-07, -2.4663e-07, 1.1023e-06, -6.4749e-07, -7.0314e-07,\n", - " -1.1032e-06, -1.1533e-06, -1.3615e-07, -5.3711e-07, 1.8676e-06,\n", - " -1.2163e-06, -3.2784e-07, -5.9893e-08, 1.2751e-06, 7.8705e-07,\n", - " 9.8065e-07, -1.1253e-06, 3.8889e-07, -1.3021e-06, -1.0771e-06,\n", - " -9.6265e-08, 1.0677e-06, 1.4190e-06, -2.3022e-07, 2.1243e-07,\n", - " -2.4294e-06, 1.4054e-06, -2.2632e-07, 2.3604e-06, 8.6605e-07,\n", - " 5.1579e-07, -3.7519e-07, 2.3580e-08, -1.0342e-07, 1.8895e-06,\n", - " 4.3311e-07, 2.1627e-06, 1.2310e-06, -1.4375e-06, -1.8110e-06,\n", - " -6.7435e-07, -1.1090e-06, 3.9569e-08, -7.6939e-07, -7.7326e-07,\n", - " -5.9887e-07, -1.9408e-07, -2.5970e-07, 2.3944e-06, 2.1859e-06,\n", - " -1.1776e-06, 9.0604e-07, 3.0137e-07, 1.8005e-07, 1.5975e-06,\n", - " 5.8386e-07, 2.5003e-07, -2.1950e-07, 1.0011e-06, -7.0608e-07,\n", - " 4.6388e-07, 1.3524e-06, 1.2713e-06, 1.4605e-07, -7.1791e-07,\n", - " 5.6460e-07, 4.2521e-07, -5.2877e-07, -4.5762e-07, -4.2111e-07,\n", - " -7.0937e-07, 3.3107e-07, -5.2286e-07, -3.0880e-07, -1.5176e-06,\n", - " -8.7055e-07, 6.5832e-07, 1.8420e-07, -1.2434e-06, -1.6331e-08,\n", - " -4.1791e-07, 3.5488e-07, 6.8525e-07, -4.0871e-07, 5.5506e-07,\n", - " -6.4726e-07, 4.8202e-07, -1.8036e-06, 1.2732e-06, -2.8784e-06,\n", - " -2.3360e-06, -1.4952e-06, -1.5416e-06], device='cuda:0')),\n", - " ('layer2.0.shortcut.0.weight',\n", - " tensor([[[[-2.5854e-06]],\n", - " \n", - " [[-1.9930e-06]],\n", - " \n", - " [[ 1.7509e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.1681e-06]],\n", - " \n", - " [[ 2.6748e-06]],\n", - " \n", - " [[ 1.9744e-07]]],\n", - " \n", - " \n", - " [[[ 9.6671e-07]],\n", - " \n", - " [[ 3.7253e-07]],\n", - " \n", - " [[ 2.2352e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.0978e-07]],\n", - " \n", - " [[ 4.0457e-06]],\n", - " \n", - " [[-3.8883e-08]]],\n", - " \n", - " \n", - " [[[-2.4810e-06]],\n", - " \n", - " [[ 8.9962e-07]],\n", - " \n", - " [[ 2.9579e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.8701e-06]],\n", - " \n", - " [[-8.6427e-07]],\n", - " \n", - " [[-9.2480e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-5.1484e-06]],\n", - " \n", - " [[-4.0270e-06]],\n", - " \n", - " [[-3.0454e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 9.8348e-07]],\n", - " \n", - " [[-1.9073e-06]],\n", - " \n", - " [[-1.9446e-06]]],\n", - " \n", - " \n", - " [[[-3.1665e-07]],\n", - " \n", - " [[ 4.6939e-07]],\n", - " \n", - " [[-3.2857e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 5.1782e-07]],\n", - " \n", - " [[ 9.4250e-07]],\n", - " \n", - " [[ 4.9919e-07]]],\n", - " \n", - " \n", - " [[[-1.2517e-06]],\n", - " \n", - " [[-2.1607e-07]],\n", - " \n", - " [[-1.9586e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.1963e-06]],\n", - " \n", - " [[-3.1367e-06]],\n", - " \n", - " [[-3.6154e-06]]]], device='cuda:0')),\n", - " ('layer2.0.shortcut.1.weight',\n", - " tensor([-1.7881e-07, -5.9605e-08, -8.3447e-07, 1.5497e-06, -2.9802e-07,\n", - " 2.3842e-07, -7.1526e-07, 2.7418e-06, -8.9407e-07, 5.9605e-07,\n", - " -1.1325e-06, -1.4305e-06, 3.5763e-07, 1.1921e-07, -9.5367e-07,\n", - " -1.1921e-06, 1.6689e-06, 4.7684e-07, -4.7684e-07, -5.9605e-07,\n", - " -3.5763e-07, -6.5565e-07, -1.1921e-07, 1.0729e-06, 2.3842e-06,\n", - " 1.4305e-06, 1.3113e-06, -2.3842e-07, -1.3709e-06, 2.1458e-06,\n", - " -5.9605e-08, -2.0862e-06, 2.5034e-06, -1.7881e-06, -5.3644e-07,\n", - " 9.5367e-07, -1.2517e-06, -1.6689e-06, 7.1526e-07, 5.9605e-07,\n", - " 2.3842e-07, 9.5367e-07, -5.9605e-08, 1.3113e-06, 1.4305e-06,\n", - " 1.6689e-06, 5.9605e-07, -4.1723e-07, 2.2650e-06, -2.9802e-07,\n", - " -2.9802e-07, -1.0729e-06, 4.7684e-07, -2.9802e-06, -1.0729e-06,\n", - " -2.3842e-07, 4.7684e-07, -2.9802e-07, 9.5367e-07, -2.2054e-06,\n", - " -1.5497e-06, 1.1921e-07, -4.1723e-07, 1.6689e-06, 1.1921e-07,\n", - " 1.7881e-06, 1.4305e-06, -5.9605e-08, 1.1921e-06, -3.5763e-07,\n", - " -1.7881e-07, 4.0531e-06, 8.3447e-07, -1.3113e-06, -1.5497e-06,\n", - " 1.1921e-06, -3.5763e-07, -9.5367e-07, -1.7881e-07, -1.3113e-06,\n", - " -1.3113e-06, 0.0000e+00, 2.3842e-07, 8.3447e-07, 2.9802e-06,\n", - " 7.1526e-07, -1.7881e-06, 2.1458e-06, -1.9670e-06, 1.5497e-06,\n", - " 1.0729e-06, -8.3447e-07, -7.1526e-07, 1.6689e-06, 4.7684e-07,\n", - " -2.3842e-07, 5.9605e-07, 4.7684e-07, 1.9073e-06, -5.3644e-07,\n", - " 2.0266e-06, -5.9605e-08, -8.3447e-07, 8.3447e-07, -4.0531e-06,\n", - " 1.1921e-07, 2.3842e-06, 1.3113e-06, -1.3709e-06, 1.1921e-07,\n", - " -1.7881e-07, -8.3447e-07, 1.9073e-06, 2.5034e-06, -4.7684e-07,\n", - " -9.5367e-07, 3.5763e-07, 3.5763e-07, 1.1921e-06, -1.7881e-06,\n", - " -8.3447e-07, 5.9605e-07, -1.4305e-06, -1.1921e-07, 1.4305e-06,\n", - " -1.8477e-06, -1.1325e-06, -2.4438e-06], device='cuda:0')),\n", - " ('layer2.0.shortcut.1.bias',\n", - " tensor([-1.9510e-06, 1.9011e-07, -9.3751e-07, 1.4218e-06, 1.5925e-06,\n", - " -4.4937e-07, 1.6194e-07, 1.3984e-06, -9.2120e-07, -1.6821e-07,\n", - " -1.0528e-06, -8.1123e-07, 5.3874e-07, 9.9689e-10, -7.2899e-07,\n", - " -1.2392e-06, 5.1126e-07, -7.3782e-07, -1.4009e-06, -3.6747e-07,\n", - " -1.5048e-07, 3.9299e-07, 8.8879e-07, 1.2054e-06, 4.6736e-07,\n", - " -4.8633e-07, 6.5945e-07, 4.0367e-07, 1.0216e-06, -5.3070e-08,\n", - " 6.0140e-07, -1.6197e-06, 1.6229e-06, -1.6424e-08, -5.1029e-07,\n", - " -4.7327e-07, -2.4663e-07, 1.1023e-06, -6.4749e-07, -7.0314e-07,\n", - " -1.1032e-06, -1.1533e-06, -1.3615e-07, -5.3711e-07, 1.8676e-06,\n", - " -1.2163e-06, -3.2784e-07, -5.9893e-08, 1.2751e-06, 7.8705e-07,\n", - " 9.8065e-07, -1.1253e-06, 3.8889e-07, -1.3021e-06, -1.0771e-06,\n", - " -9.6265e-08, 1.0677e-06, 1.4190e-06, -2.3022e-07, 2.1243e-07,\n", - " -2.4294e-06, 1.4054e-06, -2.2632e-07, 2.3604e-06, 8.6605e-07,\n", - " 5.1579e-07, -3.7519e-07, 2.3580e-08, -1.0342e-07, 1.8895e-06,\n", - " 4.3311e-07, 2.1627e-06, 1.2310e-06, -1.4375e-06, -1.8110e-06,\n", - " -6.7435e-07, -1.1090e-06, 3.9569e-08, -7.6939e-07, -7.7326e-07,\n", - " -5.9887e-07, -1.9408e-07, -2.5970e-07, 2.3944e-06, 2.1859e-06,\n", - " -1.1776e-06, 9.0604e-07, 3.0137e-07, 1.8005e-07, 1.5975e-06,\n", - " 5.8386e-07, 2.5003e-07, -2.1950e-07, 1.0011e-06, -7.0608e-07,\n", - " 4.6388e-07, 1.3524e-06, 1.2713e-06, 1.4605e-07, -7.1791e-07,\n", - " 5.6460e-07, 4.2521e-07, -5.2877e-07, -4.5762e-07, -4.2111e-07,\n", - " -7.0937e-07, 3.3107e-07, -5.2286e-07, -3.0880e-07, -1.5176e-06,\n", - " -8.7055e-07, 6.5832e-07, 1.8420e-07, -1.2434e-06, -1.6331e-08,\n", - " -4.1791e-07, 3.5488e-07, 6.8525e-07, -4.0871e-07, 5.5506e-07,\n", - " -6.4726e-07, 4.8202e-07, -1.8036e-06, 1.2732e-06, -2.8784e-06,\n", - " -2.3360e-06, -1.4952e-06, -1.5416e-06], device='cuda:0')),\n", - " ('layer3.0.conv1.weight',\n", - " tensor([[[[ 5.6066e-07, 3.1535e-06, -6.3330e-08],\n", - " [ 2.7567e-07, 2.3115e-06, -4.9919e-07],\n", - " [ 1.1055e-06, 3.4086e-07, -1.7099e-06]],\n", - " \n", - " [[ 7.2829e-07, -9.6858e-08, 2.2147e-06],\n", - " [-4.3772e-07, -1.8394e-06, 7.1712e-07],\n", - " [ 5.1595e-07, -2.5462e-06, 2.2170e-06]],\n", - " \n", - " [[ 7.7486e-07, 3.1106e-07, 8.1607e-08],\n", - " [-4.1351e-07, 2.9430e-07, 2.8312e-07],\n", - " [ 7.6741e-07, 2.6692e-06, -6.5565e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.8629e-06, 3.5446e-06, 1.9521e-06],\n", - " [ 1.9660e-06, 2.7940e-08, -1.6373e-06],\n", - " [ 1.1846e-06, -1.9739e-06, -4.0876e-06]],\n", - " \n", - " [[ 1.7420e-06, -8.7731e-07, -3.0389e-06],\n", - " [ 4.0838e-07, 3.3341e-07, -2.4578e-06],\n", - " [ 2.0117e-06, -2.5751e-07, 4.4145e-07]],\n", - " \n", - " [[ 9.3598e-07, -3.6322e-08, -1.7881e-07],\n", - " [-1.0561e-06, 2.5891e-07, 2.0489e-07],\n", - " [ 2.3227e-06, 8.8196e-07, 4.3586e-07]]],\n", - " \n", - " \n", - " [[[ 1.9260e-06, 5.9232e-07, 1.6717e-06],\n", - " [-9.0571e-08, 1.9991e-06, -2.2165e-07],\n", - " [ 6.3377e-07, -2.0675e-07, -1.4231e-06]],\n", - " \n", - " [[ 3.2317e-07, -3.0757e-07, -5.6578e-08],\n", - " [-6.3470e-07, -1.9409e-06, -1.7677e-06],\n", - " [ 5.7626e-07, -2.5439e-06, -1.5665e-06]],\n", - " \n", - " [[-2.1514e-07, -1.3020e-06, -3.8417e-07],\n", - " [ 1.2033e-06, 9.5926e-07, 1.2582e-06],\n", - " [-6.8918e-08, 2.0675e-07, 8.0559e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.0473e-06, -3.7812e-07, 1.7732e-06],\n", - " [ 7.0455e-07, -3.6322e-07, -1.1548e-07],\n", - " [-9.6997e-07, -9.4622e-07, -1.6037e-06]],\n", - " \n", - " [[ 1.9846e-06, 1.6354e-06, 2.0321e-06],\n", - " [-1.5739e-07, 2.3842e-07, 1.6857e-07],\n", - " [-3.8370e-07, 1.9930e-07, 1.7909e-06]],\n", - " \n", - " [[ 1.1469e-06, 9.3691e-07, -1.8883e-06],\n", - " [-2.6161e-06, 2.0303e-07, -1.5078e-06],\n", - " [-2.8755e-07, -2.2445e-06, -8.0466e-07]]],\n", - " \n", - " \n", - " [[[-2.3879e-06, -2.2701e-06, -9.4809e-07],\n", - " [-2.3656e-07, -6.1840e-07, -1.6708e-06],\n", - " [ 1.5316e-06, 2.4587e-06, -4.0233e-07]],\n", - " \n", - " [[ 1.4883e-06, 7.1991e-07, 4.5635e-07],\n", - " [ 2.4773e-07, -7.0408e-07, 2.4373e-06],\n", - " [ 1.0128e-07, 1.0291e-07, 2.0806e-06]],\n", - " \n", - " [[ 2.2622e-06, 2.5705e-07, -4.8988e-07],\n", - " [ 2.1104e-06, -2.2743e-06, 3.8464e-07],\n", - " [-3.8501e-06, -1.9502e-06, -1.0375e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.3560e-06, 1.4572e-06, -9.3528e-07],\n", - " [ 2.5779e-06, 9.0711e-07, -1.7043e-07],\n", - " [ 1.2387e-06, -1.3411e-07, 1.0864e-06]],\n", - " \n", - " [[-3.1479e-06, 2.8871e-07, 2.9197e-07],\n", - " [ 1.7229e-07, -5.1595e-07, 6.3702e-07],\n", - " [ 7.8045e-07, 1.8599e-06, 3.9558e-06]],\n", - " \n", - " [[-2.0117e-07, 1.1567e-06, 2.1909e-06],\n", - " [ 9.1083e-07, 1.3397e-06, -4.1351e-07],\n", - " [-9.0711e-07, -3.1628e-06, -2.3060e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 5.0478e-07, 2.3898e-06, 3.1013e-06],\n", - " [ 5.4340e-07, 2.3097e-07, 1.8235e-06],\n", - " [ 8.7987e-07, 1.2647e-06, 3.5305e-06]],\n", - " \n", - " [[ 1.9222e-06, 1.0282e-06, 5.4948e-07],\n", - " [ 1.0799e-06, 1.8217e-06, -1.2759e-07],\n", - " [ 4.8056e-07, 8.9779e-07, 7.1060e-07]],\n", - " \n", - " [[-9.6555e-07, 4.3772e-08, -6.3144e-07],\n", - " [-2.0200e-06, -4.2655e-07, -2.7381e-07],\n", - " [-5.6438e-07, -1.7178e-06, 4.3213e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1306e-06, 2.8070e-06, 1.0971e-06],\n", - " [ 2.2715e-06, 4.5449e-07, 2.7929e-06],\n", - " [ 6.0257e-07, 3.2345e-06, 3.9386e-06]],\n", - " \n", - " [[ 1.8943e-06, 1.2610e-06, 4.7777e-07],\n", - " [-1.2708e-06, -1.0971e-06, 1.0752e-06],\n", - " [-5.0850e-07, -2.9197e-06, -4.2655e-07]],\n", - " \n", - " [[-6.5804e-07, -1.7099e-06, -1.2871e-06],\n", - " [ 1.7509e-06, -2.1532e-06, -4.8708e-07],\n", - " [ 1.5008e-06, -8.8662e-07, 7.5344e-07]]],\n", - " \n", - " \n", - " [[[-6.4820e-07, -1.7136e-07, 3.1060e-07],\n", - " [-1.0151e-06, -8.1956e-08, 2.6170e-06],\n", - " [-1.3318e-06, -8.2422e-07, 1.6224e-06]],\n", - " \n", - " [[-5.6438e-07, 1.0938e-06, -1.3020e-06],\n", - " [-1.5167e-06, 4.1979e-07, -2.2158e-06],\n", - " [-6.5751e-07, -6.7055e-08, -2.1867e-06]],\n", - " \n", - " [[ 2.0750e-06, 1.0412e-06, -1.3839e-06],\n", - " [ 2.7334e-06, 2.9076e-06, -1.4063e-06],\n", - " [ 4.3653e-06, 3.5036e-06, -2.4121e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.3253e-06, -6.4075e-07, -3.8097e-07],\n", - " [ 2.1569e-06, 1.7346e-06, -7.0222e-07],\n", - " [ 1.2806e-06, -7.6555e-07, -3.7532e-07]],\n", - " \n", - " [[ 6.8918e-08, -1.6484e-06, -4.5700e-06],\n", - " [ 1.8589e-06, -2.8498e-06, -2.6142e-06],\n", - " [-5.8580e-07, -2.4866e-06, -3.8752e-06]],\n", - " \n", - " [[ 2.5444e-06, 7.1339e-07, -1.8440e-07],\n", - " [ 1.7388e-06, -8.7079e-07, 1.6373e-06],\n", - " [ 1.3169e-06, 2.7567e-07, -5.7556e-07]]],\n", - " \n", - " \n", - " [[[ 1.1427e-06, 2.1700e-06, 2.7027e-06],\n", - " [ 6.2957e-07, -7.4692e-07, 3.0883e-06],\n", - " [ 3.2410e-07, 2.7688e-06, 1.8831e-06]],\n", - " \n", - " [[ 1.0086e-06, 6.1840e-07, 1.3001e-06],\n", - " [-3.5577e-07, -7.1013e-07, 1.1241e-06],\n", - " [-6.8732e-07, -3.0296e-06, 1.6298e-06]],\n", - " \n", - " [[ 2.0759e-06, 3.9022e-07, 1.2827e-06],\n", - " [ 2.2352e-08, -8.7731e-07, 1.2331e-06],\n", - " [-4.2282e-07, -9.7416e-07, -2.5705e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.6280e-06, 3.1050e-06, 2.8405e-06],\n", - " [ 1.2685e-06, 7.7672e-07, 2.3125e-06],\n", - " [ 2.2021e-06, -4.9639e-07, 4.2832e-06]],\n", - " \n", - " [[-2.0508e-06, 6.3100e-07, 1.4380e-06],\n", - " [-2.2063e-06, 1.2754e-06, 2.7139e-06],\n", - " [ 1.3579e-06, 8.0094e-07, 2.4997e-06]],\n", - " \n", - " [[ 1.4678e-06, 2.4959e-07, -1.5739e-07],\n", - " [ 8.9873e-07, 5.2992e-07, -4.3213e-07],\n", - " [-6.1374e-07, -8.3819e-08, 1.2051e-06]]]], device='cuda:0')),\n", - " ('layer3.0.bn1.weight',\n", - " tensor([ 7.1526e-07, -1.1921e-06, -2.9802e-07, 2.3842e-07, 1.1921e-07,\n", - " 8.3447e-07, 5.9605e-07, -1.7881e-07, -2.3842e-07, 4.7684e-07,\n", - " -5.3644e-07, 7.1526e-07, -1.0729e-06, -1.0729e-06, -2.3842e-07,\n", - " 1.1921e-06, 0.0000e+00, -1.4901e-06, 3.5763e-07, 2.3842e-07,\n", - " 5.9605e-07, 1.4305e-06, 3.5763e-07, -7.1526e-07, 1.4305e-06,\n", - " 4.7684e-07, -4.1723e-07, 1.1921e-07, 1.1921e-06, -7.7486e-07,\n", - " -1.0729e-06, -1.1921e-06, -5.9605e-08, -1.7881e-07, -1.1921e-06,\n", - " -5.3644e-07, -8.9407e-07, -1.8477e-06, 8.3447e-07, 7.1526e-07,\n", - " -1.1921e-07, -4.7684e-07, -6.5565e-07, 8.3447e-07, 0.0000e+00,\n", - " 3.5763e-07, 1.4305e-06, 9.5367e-07, 5.9605e-07, 8.3447e-07,\n", - " 9.5367e-07, 1.1921e-07, -1.0729e-06, -1.1921e-07, 1.1921e-06,\n", - " 1.1921e-07, 1.1921e-06, 7.1526e-07, -8.3447e-07, 7.1526e-07,\n", - " 5.9605e-07, -4.7684e-07, -1.8477e-06, 1.0729e-06, 1.1921e-06,\n", - " 9.5367e-07, 4.7684e-07, 1.0729e-06, -8.3447e-07, 4.7684e-07,\n", - " -2.1458e-06, -1.5497e-06, 0.0000e+00, 3.5763e-07, 1.1921e-06,\n", - " -1.1325e-06, -4.1723e-07, 1.1921e-07, 4.7684e-07, -1.0729e-06,\n", - " 1.1921e-06, 1.0729e-06, 2.3842e-07, -1.1921e-07, 1.5497e-06,\n", - " -5.3644e-07, -2.3842e-07, 5.9605e-07, 4.7684e-07, -4.7684e-07,\n", - " 8.3447e-07, -5.9605e-08, -5.9605e-07, -4.7684e-07, -7.7486e-07,\n", - " 3.5763e-07, -1.1921e-06, 0.0000e+00, 5.9605e-07, -1.7881e-07,\n", - " -1.1921e-06, -6.5565e-07, -4.7684e-07, 0.0000e+00, 9.5367e-07,\n", - " 0.0000e+00, -5.9605e-07, -4.1723e-07, 7.1526e-07, -6.5565e-07,\n", - " -1.0729e-06, 8.3447e-07, 7.1526e-07, 7.1526e-07, 4.7684e-07,\n", - " 4.7684e-07, -8.9407e-07, -1.6093e-06, -4.1723e-07, 4.7684e-07,\n", - " 8.3447e-07, 1.1921e-07, 8.3447e-07, -3.5763e-07, 4.7684e-07,\n", - " 1.1921e-06, 1.1921e-07, -5.9605e-08, -1.4305e-06, -3.5763e-07,\n", - " -2.3842e-07, 1.1921e-07, -4.1723e-07, 1.9073e-06, 3.5763e-07,\n", - " -5.9605e-07, -4.1723e-07, -4.7684e-07, 4.7684e-07, -8.3447e-07,\n", - " 0.0000e+00, 1.1921e-07, -1.7881e-07, -4.1723e-07, -1.1921e-07,\n", - " -3.5763e-07, -8.3447e-07, -1.9670e-06, 4.7684e-07, -7.7486e-07,\n", - " -1.7881e-07, -2.4438e-06, 9.5367e-07, -3.5763e-07, -1.3709e-06,\n", - " -5.3644e-07, -6.5565e-07, 1.3113e-06, 7.1526e-07, -1.0729e-06,\n", - " 3.5763e-07, 9.5367e-07, -3.5763e-07, 4.7684e-07, 1.1921e-07,\n", - " -1.0133e-06, 2.3842e-07, -7.7486e-07, 2.3842e-07, -1.0133e-06,\n", - " 1.1921e-07, -1.0133e-06, 1.4305e-06, 1.1921e-06, -2.9802e-07,\n", - " 8.3447e-07, 3.5763e-07, -2.3842e-07, 1.1921e-07, -1.7881e-07,\n", - " 1.5497e-06, -5.9605e-07, -5.9605e-08, -5.9605e-08, 5.9605e-07,\n", - " -2.3842e-07, 1.1921e-07, -6.5565e-07, 7.1526e-07, 7.1526e-07,\n", - " -1.1921e-07, -1.1325e-06, 7.1526e-07, -4.1723e-07, 2.3842e-07,\n", - " -5.9605e-07, -1.3709e-06, 3.5763e-07, 4.7684e-07, 1.1921e-07,\n", - " 1.4305e-06, -2.3842e-07, -3.5763e-07, 9.5367e-07, 5.9605e-07,\n", - " 4.7684e-07, -9.5367e-07, -1.0729e-06, -4.7684e-07, 1.1921e-07,\n", - " -1.0133e-06, 1.0729e-06, -1.1921e-07, -2.9802e-07, -1.7881e-07,\n", - " 5.9605e-07, 5.9605e-07, -5.9605e-07, 9.5367e-07, 9.5367e-07,\n", - " 3.5763e-07, 9.5367e-07, 8.3447e-07, -7.7486e-07, 9.5367e-07,\n", - " 7.1526e-07, -1.0133e-06, 0.0000e+00, -5.9605e-07, 4.7684e-07,\n", - " -2.9802e-07, -1.1921e-07, -1.1921e-07, -7.7486e-07, -1.9670e-06,\n", - " 1.3113e-06, -5.9605e-08, -1.1325e-06, 1.1921e-07, -8.9407e-07,\n", - " -4.1723e-07, -4.1723e-07, 9.5367e-07, 2.3842e-07, 2.3842e-07,\n", - " -2.9802e-07, 1.0729e-06, -4.1723e-07, -4.1723e-07, 7.1526e-07,\n", - " -1.4305e-06, -1.1921e-07, -5.9605e-08, 1.9073e-06, 1.1921e-07,\n", - " -5.9605e-08], device='cuda:0')),\n", - " ('layer3.0.bn1.bias',\n", - " tensor([-5.6823e-07, -1.3981e-06, -5.8143e-07, -4.9844e-07, 3.0359e-07,\n", - " -6.4545e-07, 2.8438e-07, -4.5231e-07, -5.9321e-07, -4.6126e-08,\n", - " -6.9217e-07, 9.4221e-07, -1.1141e-06, -9.0594e-07, 2.9411e-07,\n", - " 8.6175e-07, -2.3770e-07, -1.5887e-06, 1.0951e-06, 1.2469e-06,\n", - " 1.2919e-06, 6.5640e-07, 9.5573e-07, -5.9106e-07, 4.3798e-07,\n", - " 8.4446e-07, -6.2349e-07, -1.8349e-07, 8.4023e-07, 1.1595e-06,\n", - " -5.6597e-07, -4.5581e-07, 2.9411e-07, 3.7765e-07, -5.3188e-07,\n", - " -4.4759e-07, 1.0989e-06, -7.0273e-07, 8.3838e-07, -1.4067e-09,\n", - " -4.8365e-08, -3.3166e-07, -1.3072e-06, 3.4386e-07, 6.9363e-07,\n", - " 7.2602e-08, 1.1142e-06, 5.8377e-07, 5.1305e-08, 3.5936e-07,\n", - " 3.3200e-08, -1.0829e-07, -5.9911e-07, 1.2177e-07, 2.4382e-07,\n", - " 1.9582e-07, 4.1886e-07, 2.5329e-07, -6.9319e-07, -4.8527e-07,\n", - " 4.5209e-07, -4.1583e-07, -1.0734e-06, 1.5452e-06, 5.3113e-07,\n", - " 1.1329e-06, 5.7745e-07, 9.0927e-07, 1.2965e-07, 1.1920e-06,\n", - " -1.2917e-06, -1.7132e-06, -7.9898e-07, -9.2034e-07, -1.8442e-07,\n", - " -5.5113e-07, 3.1931e-07, 9.5560e-07, 3.0754e-07, -1.7962e-07,\n", - " 2.0756e-07, 7.9398e-07, 2.0676e-07, -6.0307e-07, 1.2794e-06,\n", - " -1.0505e-06, -1.2550e-06, 1.4746e-07, 1.7321e-07, -2.9260e-07,\n", - " 5.8848e-07, 1.0114e-07, -8.7763e-08, -8.6689e-07, 1.7805e-08,\n", - " -7.3110e-07, -3.7368e-07, -4.9181e-07, 1.4132e-07, -7.5466e-07,\n", - " -1.0650e-06, -2.4837e-07, -7.2920e-07, -2.0560e-07, 7.5436e-07,\n", - " 2.4887e-07, -5.0478e-07, -1.1000e-07, 5.7459e-07, -9.5710e-07,\n", - " -1.9314e-07, 9.2695e-07, 1.0267e-06, 4.8308e-08, 7.6856e-07,\n", - " 9.2601e-07, -6.8997e-07, -5.1764e-07, -4.9746e-07, 2.0167e-08,\n", - " -3.0086e-07, -1.5695e-07, 4.7365e-07, 7.3920e-08, 2.1957e-07,\n", - " -1.7095e-07, -3.8337e-07, -5.4026e-09, -5.5063e-08, -2.7068e-07,\n", - " 2.8007e-07, 1.1806e-06, -1.2740e-07, 9.5515e-07, 4.3170e-07,\n", - " -6.3234e-07, -1.5391e-07, 1.8739e-07, 4.3259e-07, -4.1224e-07,\n", - " 1.3033e-07, 4.5940e-07, -4.5620e-07, -9.0492e-08, -4.6963e-07,\n", - " -3.5890e-07, -1.9067e-07, -1.1707e-06, 5.3332e-07, -7.1210e-07,\n", - " -7.5044e-07, -7.9760e-07, -2.2486e-07, -3.4558e-07, -2.8345e-07,\n", - " -2.7052e-07, -7.1833e-08, 1.9516e-06, -7.1396e-07, -2.0790e-07,\n", - " -1.0211e-08, 5.9483e-07, -5.8415e-07, -4.6524e-07, 1.1732e-07,\n", - " -3.6851e-07, 1.5508e-06, -1.3852e-07, 6.8619e-07, 1.9448e-07,\n", - " -8.8853e-07, -9.2429e-07, 6.2866e-07, 6.6644e-07, 1.0236e-08,\n", - " 1.3283e-06, 2.6774e-07, -1.3880e-06, -2.8537e-07, -2.9740e-07,\n", - " 1.2232e-06, -5.8244e-07, -2.9546e-07, -7.8462e-07, 1.0469e-06,\n", - " -1.4055e-06, -2.9223e-08, -8.9813e-07, 2.5954e-07, 5.1834e-07,\n", - " -7.1301e-08, -1.7958e-06, 6.8665e-07, -3.1676e-08, 4.7086e-08,\n", - " -3.7147e-07, -3.8185e-07, -6.4055e-07, 8.1996e-07, 8.1394e-07,\n", - " 1.1889e-06, -4.3364e-07, -3.6164e-07, 1.0069e-06, 1.0859e-06,\n", - " 1.0522e-06, -1.4343e-06, -1.1262e-06, -4.3589e-07, -1.4687e-06,\n", - " -1.1378e-06, 1.0889e-06, -8.0180e-07, 2.5179e-07, -8.2007e-08,\n", - " 6.6869e-07, 5.4022e-07, -1.0662e-06, 1.6516e-06, -1.4696e-07,\n", - " 8.2590e-08, 2.9557e-07, 9.4435e-07, -4.5838e-07, 3.2872e-07,\n", - " 1.1526e-06, -2.2169e-07, 2.3443e-07, 5.5351e-08, 5.1754e-08,\n", - " 2.7610e-07, -5.2608e-08, -8.8920e-07, -8.1640e-07, -1.1482e-06,\n", - " 1.3194e-06, -3.1359e-07, -1.2565e-06, 2.8645e-07, -1.3402e-06,\n", - " -2.6665e-08, 5.1468e-08, 2.1430e-07, -9.3682e-08, 2.3872e-07,\n", - " -7.2445e-07, 1.1124e-06, -5.9129e-07, -1.2152e-07, 8.0874e-07,\n", - " 1.2747e-07, -3.5549e-07, -6.1583e-07, 7.4368e-07, 2.9648e-07,\n", - " -8.8238e-07], device='cuda:0')),\n", - " ('layer3.0.conv2.weight',\n", - " tensor([[[[ 1.3057e-06, -2.6138e-06, 1.9912e-06],\n", - " [ 1.9046e-06, 1.1588e-06, 5.1130e-07],\n", - " [ 1.7826e-06, 1.3243e-06, 3.1153e-06]],\n", - " \n", - " [[-1.0901e-06, 2.5947e-06, 1.5767e-06],\n", - " [ 4.8853e-06, 3.7169e-06, 3.2913e-06],\n", - " [ 2.1281e-06, 2.4103e-06, 2.5703e-06]],\n", - " \n", - " [[ 5.8091e-07, -6.4028e-07, 4.1351e-07],\n", - " [ 2.3320e-06, 3.2922e-06, 1.8850e-06],\n", - " [ 2.3283e-06, 6.0885e-07, 1.3956e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.0098e-06, 1.6801e-06, 2.7046e-06],\n", - " [ 4.1922e-06, 4.2524e-06, 2.1830e-06],\n", - " [ 2.0792e-06, 2.4531e-06, 3.1613e-06]],\n", - " \n", - " [[ 9.7230e-07, 2.8196e-06, 6.6590e-07],\n", - " [ 2.5225e-06, 3.7211e-06, 2.7816e-06],\n", - " [-2.6450e-07, 1.2089e-06, 1.3248e-06]],\n", - " \n", - " [[-1.7732e-06, -3.1665e-08, -1.5106e-06],\n", - " [ 2.4301e-06, -1.0282e-06, 1.5609e-06],\n", - " [ 4.2096e-07, 1.3942e-06, 1.0987e-06]]],\n", - " \n", - " \n", - " [[[ 2.0228e-06, 1.1697e-06, 9.9489e-07],\n", - " [-1.8440e-07, 4.6566e-08, 4.1910e-08],\n", - " [ 8.5915e-07, 3.8370e-07, 3.8799e-06]],\n", - " \n", - " [[ 4.0047e-08, 1.1567e-06, -1.9008e-06],\n", - " [ 2.0806e-06, -6.4448e-07, -2.5351e-06],\n", - " [ 4.2282e-07, 2.2575e-06, -2.5406e-06]],\n", - " \n", - " [[ 2.5420e-06, 9.2760e-07, -8.0653e-07],\n", - " [ 1.1194e-06, -1.2890e-06, -1.6810e-06],\n", - " [ 2.6245e-06, -7.5251e-07, 1.3150e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.4110e-06, -1.4915e-06, -9.0525e-07],\n", - " [ 3.6077e-07, 6.1281e-07, -2.5015e-06],\n", - " [ 2.5220e-06, 1.0338e-06, -1.2182e-06]],\n", - " \n", - " [[ 4.1211e-07, 1.1772e-06, 2.7008e-07],\n", - " [ 2.6310e-06, 1.1353e-06, 1.1409e-06],\n", - " [ 6.2585e-07, 2.7374e-06, 2.7195e-07]],\n", - " \n", - " [[ 6.1002e-08, 1.1306e-06, -1.3048e-06],\n", - " [ 1.6908e-06, -1.6326e-06, -4.8539e-07],\n", - " [ 1.6429e-06, 2.6822e-06, -7.9628e-07]]],\n", - " \n", - " \n", - " [[[ 8.1398e-07, 2.9895e-07, 1.5097e-06],\n", - " [ 2.3581e-06, 3.0650e-06, 3.2019e-06],\n", - " [ 1.3504e-07, -1.1511e-06, -8.7373e-07]],\n", - " \n", - " [[-1.0077e-06, 1.7782e-06, 3.5483e-07],\n", - " [ 1.0589e-06, 1.5870e-06, 9.5740e-07],\n", - " [-9.8534e-07, 3.2410e-07, -1.3839e-06]],\n", - " \n", - " [[-1.3541e-06, -1.3374e-06, 2.5705e-07],\n", - " [-4.9639e-07, 7.8138e-07, 7.6881e-07],\n", - " [ 6.2515e-08, -1.2070e-06, -2.8368e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.6543e-07, -9.0431e-07, -5.1782e-07],\n", - " [-8.4285e-07, -1.1786e-06, 9.2760e-07],\n", - " [ 1.0291e-07, -2.4131e-06, -8.0001e-07]],\n", - " \n", - " [[ 5.3272e-07, 7.6741e-07, -1.5385e-06],\n", - " [-1.0151e-06, -8.0466e-07, 2.8685e-07],\n", - " [-1.2936e-06, -1.3478e-06, -7.0175e-07]],\n", - " \n", - " [[ 6.9477e-07, -3.8464e-07, -4.6566e-09],\n", - " [ 1.1316e-07, 6.7987e-08, 2.3232e-06],\n", - " [ 2.3935e-07, 3.9290e-07, -2.0061e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 1.9018e-06, 5.8487e-07, 1.3602e-06],\n", - " [ 1.2463e-06, 4.4145e-07, 9.3412e-07],\n", - " [ 3.9488e-07, 8.8476e-07, 2.7129e-06]],\n", - " \n", - " [[-2.0042e-06, 2.1264e-06, -1.4435e-06],\n", - " [ 9.9931e-07, 7.2829e-07, -1.8035e-06],\n", - " [-2.5611e-07, 5.6624e-07, -9.7742e-07]],\n", - " \n", - " [[ 4.4517e-07, 1.6938e-07, -1.4994e-06],\n", - " [-6.1467e-07, -5.4250e-07, 5.8324e-07],\n", - " [-9.5367e-07, -3.1362e-07, -2.3283e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.5874e-06, -5.1316e-07, 9.7416e-07],\n", - " [-1.8254e-07, -3.1437e-06, -1.0012e-06],\n", - " [-9.0152e-07, -1.1763e-06, -2.3171e-06]],\n", - " \n", - " [[ 1.0794e-06, 1.2871e-06, 8.8476e-07],\n", - " [ 1.0924e-06, -5.8953e-07, -1.1618e-07],\n", - " [ 2.2538e-06, -1.0384e-06, -1.1181e-06]],\n", - " \n", - " [[-1.5525e-06, -5.6718e-07, -3.1292e-07],\n", - " [-3.0547e-07, 6.4634e-07, 1.2238e-06],\n", - " [ 1.3532e-06, -2.0093e-07, 5.9605e-07]]],\n", - " \n", - " \n", - " [[[-2.0294e-06, 1.7039e-06, -4.8801e-07],\n", - " [-5.4808e-07, -1.6736e-06, -6.3330e-07],\n", - " [ 2.0573e-06, 1.9413e-06, -4.1164e-07]],\n", - " \n", - " [[ 6.5938e-07, 2.2445e-06, 1.6289e-06],\n", - " [ 2.7893e-07, 8.6799e-07, 2.7772e-06],\n", - " [-2.9756e-07, 1.1092e-06, 2.1541e-06]],\n", - " \n", - " [[ 5.6671e-07, 3.3563e-07, -1.7071e-06],\n", - " [-3.0734e-08, 6.7800e-07, 8.7963e-07],\n", - " [ 7.9721e-07, 4.6566e-08, -1.3597e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 7.5018e-07, 3.3262e-06, 2.9728e-06],\n", - " [ 7.2084e-07, 2.6328e-06, 2.7940e-08],\n", - " [-5.4762e-07, -2.0005e-06, -1.2787e-06]],\n", - " \n", - " [[ 8.8010e-07, -3.7812e-07, -1.5274e-06],\n", - " [ 1.7690e-06, -1.1227e-06, 9.1270e-07],\n", - " [-7.5437e-07, 6.6031e-07, -3.7253e-08]],\n", - " \n", - " [[ 4.0885e-07, 6.3283e-07, 6.4331e-07],\n", - " [-3.5763e-07, 9.0618e-07, 2.5611e-07],\n", - " [-1.5600e-06, -2.4587e-07, -1.3071e-06]]],\n", - " \n", - " \n", - " [[[-2.8191e-06, -1.8105e-06, -5.6392e-07],\n", - " [-1.3543e-06, 1.2156e-06, 2.4505e-07],\n", - " [-2.7474e-06, -1.3094e-06, -8.6147e-08]],\n", - " \n", - " [[-1.3690e-06, -2.5544e-06, -7.1153e-07],\n", - " [-3.3993e-07, -5.9186e-07, 1.5670e-07],\n", - " [ 9.4250e-07, -1.3281e-06, -2.0331e-06]],\n", - " \n", - " [[-1.7378e-06, -7.6758e-07, 2.8498e-07],\n", - " [ 8.6986e-07, -3.1674e-06, -9.3319e-07],\n", - " [-2.7036e-06, -8.8476e-08, 2.6077e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.4273e-07, -1.8748e-06, 1.0058e-06],\n", - " [-1.3680e-06, -1.5926e-06, -4.2096e-07],\n", - " [-1.4417e-06, 6.4727e-08, -6.8918e-07]],\n", - " \n", - " [[-7.8278e-07, -7.5623e-07, 5.7276e-07],\n", - " [-5.6624e-07, -1.4994e-06, -1.5888e-06],\n", - " [-1.0417e-06, 6.1467e-07, 4.5588e-07]],\n", - " \n", - " [[-1.6233e-06, -1.0841e-06, 5.7742e-07],\n", - " [-1.0729e-06, 9.9186e-07, -1.1884e-06],\n", - " [-3.8091e-07, -1.7569e-06, 1.3858e-06]]]], device='cuda:0')),\n", - " ('layer3.0.bn2.weight',\n", - " tensor([ 2.3842e-06, 0.0000e+00, -8.3447e-07, 8.3447e-07, 7.1526e-07,\n", - " -1.0133e-06, 5.9605e-07, 3.5763e-07, 2.3842e-07, 0.0000e+00,\n", - " 7.1526e-07, 7.1526e-07, 1.0729e-06, -1.1921e-07, -5.3644e-07,\n", - " 1.1921e-07, -5.3644e-07, 1.0729e-06, 1.5497e-06, 0.0000e+00,\n", - " 3.5763e-07, -1.2517e-06, -1.0729e-06, -1.7881e-07, 9.5367e-07,\n", - " 3.5763e-07, -8.9407e-07, -5.3644e-07, -1.9073e-06, 4.7684e-07,\n", - " 8.3447e-07, 7.1526e-07, -2.3842e-07, 7.1526e-07, 2.3842e-07,\n", - " -4.7684e-07, -7.7486e-07, 3.5763e-07, -1.7881e-07, -8.9407e-07,\n", - " -2.3842e-07, 1.0729e-06, -4.7684e-07, -1.1921e-07, -5.3644e-07,\n", - " 4.7684e-07, -1.5497e-06, -2.9802e-07, 5.9605e-07, -5.9605e-07,\n", - " -1.0133e-06, 8.3447e-07, 5.9605e-07, -1.7881e-07, -5.9605e-08,\n", - " -7.7486e-07, 2.0266e-06, -5.3644e-07, 5.9605e-07, -1.1921e-07,\n", - " 5.9605e-07, 1.6689e-06, 1.4305e-06, -2.9802e-07, 3.5763e-07,\n", - " -4.7684e-07, 2.3842e-07, -8.3447e-07, -6.5565e-07, -1.0133e-06,\n", - " 5.9605e-07, 1.1921e-07, 9.5367e-07, 8.3447e-07, 0.0000e+00,\n", - " 3.5763e-07, 1.1921e-07, 1.1921e-07, -1.0729e-06, 1.1921e-07,\n", - " 1.1921e-07, 1.1921e-07, 5.9605e-07, 7.1526e-07, 1.1921e-07,\n", - " -1.1325e-06, -5.9605e-08, 1.1921e-07, -4.7684e-07, 9.5367e-07,\n", - " -5.9605e-07, 4.7684e-07, -5.3644e-07, 0.0000e+00, -8.9407e-07,\n", - " -5.9605e-07, -1.9073e-06, -2.3842e-07, -4.1723e-07, -1.4901e-06,\n", - " 2.3842e-07, 4.7684e-07, 5.9605e-07, -1.6689e-06, 1.1921e-07,\n", - " -1.7881e-07, 0.0000e+00, 1.4305e-06, 1.3113e-06, -6.5565e-07,\n", - " -2.9802e-07, -1.0729e-06, -5.3644e-07, 5.9605e-07, -4.7684e-07,\n", - " -8.9407e-07, -1.1921e-07, -2.3842e-07, 2.3842e-07, -1.7881e-06,\n", - " 3.5763e-07, -2.3842e-07, 3.5763e-07, 2.3842e-07, -1.1921e-07,\n", - " -1.1325e-06, -6.5565e-07, 0.0000e+00, 3.5763e-07, 8.3447e-07,\n", - " -6.5565e-07, -6.5565e-07, -2.2650e-06, 2.3842e-07, 9.5367e-07,\n", - " 7.1526e-07, -7.1526e-07, -7.7486e-07, -3.5763e-07, 1.4305e-06,\n", - " 4.7684e-07, -1.8477e-06, 4.7684e-07, 8.3447e-07, 1.5497e-06,\n", - " -5.9605e-07, 8.3447e-07, -5.3644e-07, 8.3447e-07, -2.2054e-06,\n", - " 0.0000e+00, 0.0000e+00, -1.7881e-07, 4.7684e-07, -1.2517e-06,\n", - " 2.3842e-07, 4.7684e-07, 2.3842e-07, -1.1921e-07, -4.7684e-07,\n", - " 5.9605e-07, 3.5763e-07, -1.1921e-06, 1.3113e-06, -1.7881e-07,\n", - " 7.1526e-07, 8.3447e-07, -4.7684e-07, 3.5763e-07, -1.0133e-06,\n", - " -4.7684e-07, 5.9605e-07, 1.1921e-07, 1.1921e-07, 8.3447e-07,\n", - " 0.0000e+00, 0.0000e+00, -3.5763e-07, 1.1921e-07, 4.7684e-07,\n", - " 1.1921e-07, -6.5565e-07, 4.7684e-07, 0.0000e+00, 9.5367e-07,\n", - " 3.5763e-07, 1.3113e-06, 2.3842e-07, -5.9605e-08, 1.1921e-07,\n", - " 0.0000e+00, -5.3644e-07, -5.9605e-08, 8.3447e-07, 2.3842e-07,\n", - " -2.9802e-07, -5.9605e-08, 2.3842e-07, -6.5565e-07, -1.1325e-06,\n", - " 2.3842e-07, 5.9605e-07, -1.1325e-06, 1.3113e-06, 4.7684e-07,\n", - " 1.1921e-07, -2.3842e-07, -4.7684e-07, -5.3644e-07, 1.1921e-07,\n", - " 2.3842e-07, 7.1526e-07, 0.0000e+00, 2.3842e-07, -1.1921e-07,\n", - " 1.1921e-06, 1.5497e-06, 0.0000e+00, 1.1921e-07, 8.3447e-07,\n", - " -5.9605e-07, 2.3842e-07, 4.7684e-07, 3.5763e-07, -7.1526e-07,\n", - " 1.0729e-06, 3.5763e-07, 1.0729e-06, 8.3447e-07, -4.1723e-07,\n", - " -1.2517e-06, -9.5367e-07, 1.1921e-07, 8.3447e-07, -1.1921e-06,\n", - " -9.5367e-07, 1.4305e-06, 3.5763e-07, -1.7881e-07, 1.0729e-06,\n", - " -1.1921e-06, 3.5763e-07, 5.9605e-07, -2.3842e-07, 0.0000e+00,\n", - " 3.5763e-07, 7.1526e-07, -5.9605e-08, -5.9605e-08, -8.9407e-07,\n", - " 1.3113e-06, 8.3447e-07, 3.5763e-07, -2.9802e-07, -7.7486e-07,\n", - " 2.3842e-07], device='cuda:0')),\n", - " ('layer3.0.bn2.bias',\n", - " tensor([ 1.2397e-06, 6.3916e-07, -1.5730e-07, 7.7664e-07, 6.6675e-07,\n", - " 7.5460e-07, 3.2428e-07, -4.9460e-07, -7.5864e-09, -1.1655e-07,\n", - " 4.3920e-07, 5.2803e-07, 3.4970e-07, -5.1197e-07, -2.4610e-07,\n", - " 6.8364e-07, 2.1216e-08, 5.2061e-07, 1.2123e-06, -3.2477e-07,\n", - " -7.3147e-08, -3.9508e-07, 2.2369e-07, -5.3686e-07, 1.8602e-08,\n", - " 1.3056e-06, -9.8787e-07, 1.3210e-06, -1.8658e-06, 2.4755e-07,\n", - " 4.4678e-07, -5.5527e-07, -4.8989e-07, 6.6215e-07, 8.1737e-07,\n", - " 1.0583e-06, -8.2283e-08, 2.7286e-07, 9.8735e-08, -4.9545e-07,\n", - " -1.0303e-07, 5.3118e-07, -7.7630e-07, -2.2175e-07, -8.3009e-07,\n", - " -3.2777e-07, -4.4338e-07, -8.1797e-07, -3.8200e-07, -5.3417e-07,\n", - " 3.6573e-07, 1.2024e-06, 1.0901e-06, -8.4797e-07, 2.9635e-07,\n", - " -4.1304e-07, 9.2186e-07, -5.6299e-07, 4.1419e-07, 1.4131e-07,\n", - " -2.4994e-08, 1.0638e-06, 7.4731e-07, 3.4836e-07, -6.1722e-07,\n", - " -3.2058e-07, 4.1246e-07, -2.0439e-07, -7.5934e-07, 2.0549e-07,\n", - " -4.7065e-07, 2.8008e-07, 2.7104e-07, -7.3181e-07, 9.1271e-07,\n", - " 8.7888e-08, -1.5313e-07, -3.1190e-08, -4.8686e-07, -2.8495e-07,\n", - " 8.0396e-07, -5.6101e-07, -1.4228e-07, 3.0207e-07, -4.3237e-07,\n", - " 3.8904e-07, 2.6641e-07, 2.0103e-07, -5.5700e-07, 2.1933e-07,\n", - " -7.6924e-07, 2.4533e-07, -2.0562e-07, -6.3526e-07, -7.7532e-07,\n", - " -1.0781e-08, -6.0451e-08, 2.9524e-07, -2.9730e-07, -7.1229e-07,\n", - " -5.2639e-07, -1.0158e-06, -1.8155e-07, -9.5364e-07, -7.7976e-08,\n", - " 4.2866e-07, 1.0097e-06, -2.3944e-07, 4.3388e-07, -7.4387e-07,\n", - " 3.8048e-07, -7.9571e-07, -9.8154e-07, -4.4025e-07, 2.4142e-07,\n", - " 4.3612e-07, -1.4472e-07, -4.2033e-07, -1.6008e-07, 6.7188e-09,\n", - " 5.8529e-07, -2.9896e-07, 7.1821e-07, 5.8884e-07, -3.3722e-08,\n", - " -8.7960e-07, -1.7185e-07, 3.9410e-09, 2.6008e-07, -7.1072e-07,\n", - " -1.2491e-08, -5.7730e-07, -1.2584e-06, -4.9229e-07, 3.5214e-07,\n", - " 1.3187e-06, -5.0259e-07, 1.9477e-07, -2.3074e-07, 8.2709e-07,\n", - " 5.3013e-07, -1.0832e-06, 2.9348e-07, -2.3056e-07, 9.2496e-07,\n", - " -3.6641e-07, -2.7478e-07, -7.4791e-07, 9.0510e-07, -3.4336e-07,\n", - " 1.4417e-06, 4.0621e-07, 3.4944e-07, 1.1388e-06, -4.8168e-08,\n", - " -4.2881e-08, 1.2046e-07, 1.9531e-07, -4.9289e-07, -4.9626e-07,\n", - " -2.5901e-07, 5.8413e-07, -1.1101e-06, 3.8676e-07, -2.1239e-07,\n", - " 3.7457e-07, -3.3742e-07, 1.3505e-07, 2.8487e-07, -2.7694e-07,\n", - " 7.8956e-07, 5.4949e-07, 5.5654e-07, 5.7680e-07, 2.9649e-07,\n", - " 3.8036e-07, -4.4860e-07, -3.1570e-07, -9.8473e-07, -7.6646e-07,\n", - " -8.1532e-08, 1.9141e-07, 8.7107e-07, 5.5474e-09, -6.6521e-07,\n", - " -2.2067e-07, -9.4595e-08, 6.9143e-07, 1.0157e-07, -5.9210e-07,\n", - " 1.1940e-08, 4.6116e-07, 1.2312e-06, 1.6454e-07, 4.2485e-07,\n", - " -4.9396e-07, -3.4643e-07, 1.1768e-07, 4.5359e-09, -4.0335e-07,\n", - " 1.4072e-07, 2.7469e-07, 6.8554e-07, -8.4500e-08, 3.7428e-07,\n", - " 5.9899e-07, 4.1019e-07, -8.1789e-07, -5.4011e-07, -3.5031e-07,\n", - " -5.5258e-07, -3.5712e-07, 7.5301e-07, 5.9972e-07, 4.0490e-07,\n", - " 2.6592e-07, 4.8271e-07, 4.1183e-07, 7.3446e-07, -5.6122e-07,\n", - " -3.2023e-08, -6.2278e-08, 1.9076e-07, -8.7138e-07, 1.1447e-06,\n", - " 8.4598e-07, -2.0409e-07, 1.0238e-06, 6.4743e-07, 3.9771e-07,\n", - " -8.7026e-07, 2.1872e-07, -5.9180e-07, 7.0975e-07, -1.5564e-07,\n", - " 6.7422e-07, 1.3821e-06, 6.2665e-07, -4.7970e-07, -3.3708e-07,\n", - " -7.5254e-08, -6.1423e-07, 6.8763e-07, 1.4221e-07, 1.8319e-06,\n", - " 1.1302e-06, 3.3478e-07, -3.1522e-07, 1.8243e-07, -7.6020e-08,\n", - " 1.0345e-06, 3.5484e-07, 6.0152e-07, -3.3911e-07, -3.1310e-07,\n", - " -6.6216e-07], device='cuda:0')),\n", - " ('layer3.0.shortcut.0.weight',\n", - " tensor([[[[ 9.5367e-07]],\n", - " \n", - " [[-4.9733e-07]],\n", - " \n", - " [[-1.4445e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.9116e-07]],\n", - " \n", - " [[ 3.7253e-07]],\n", - " \n", - " [[ 1.4603e-06]]],\n", - " \n", - " \n", - " [[[ 2.1819e-06]],\n", - " \n", - " [[ 4.3772e-08]],\n", - " \n", - " [[ 8.0094e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.2787e-06]],\n", - " \n", - " [[ 3.2783e-07]],\n", - " \n", - " [[ 1.4249e-07]]],\n", - " \n", - " \n", - " [[[-6.7055e-08]],\n", - " \n", - " [[-8.5682e-08]],\n", - " \n", - " [[-3.5219e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.9954e-07]],\n", - " \n", - " [[-6.5379e-07]],\n", - " \n", - " [[ 4.9919e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 2.8312e-07]],\n", - " \n", - " [[ 1.6484e-06]],\n", - " \n", - " [[-1.3635e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-5.5949e-07]],\n", - " \n", - " [[ 1.7285e-06]],\n", - " \n", - " [[-1.2666e-06]]],\n", - " \n", - " \n", - " [[[-5.6624e-07]],\n", - " \n", - " [[ 9.1270e-07]],\n", - " \n", - " [[-9.5367e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.5705e-07]],\n", - " \n", - " [[ 2.5611e-07]],\n", - " \n", - " [[-3.6508e-07]]],\n", - " \n", - " \n", - " [[[-1.9297e-06]],\n", - " \n", - " [[ 3.3528e-08]],\n", - " \n", - " [[-5.2154e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.0319e-06]],\n", - " \n", - " [[-4.3213e-07]],\n", - " \n", - " [[-3.2857e-06]]]], device='cuda:0')),\n", - " ('layer3.0.shortcut.1.weight',\n", - " tensor([-5.9605e-07, 1.7881e-06, 8.3447e-07, 8.3447e-07, 5.9605e-07,\n", - " -2.9802e-07, -6.5565e-07, -7.7486e-07, -5.9605e-07, 0.0000e+00,\n", - " 0.0000e+00, 1.3113e-06, -1.3113e-06, -4.1723e-07, -2.9802e-07,\n", - " 3.5763e-07, -6.5565e-07, -8.9407e-07, 1.0729e-06, -8.3447e-07,\n", - " -2.3842e-07, -1.3113e-06, 0.0000e+00, 0.0000e+00, -1.1325e-06,\n", - " 9.5367e-07, 3.5763e-07, 1.3113e-06, -1.4305e-06, -5.3644e-07,\n", - " -5.9605e-08, -1.0133e-06, 0.0000e+00, -5.9605e-07, 7.1526e-07,\n", - " 1.0729e-06, 7.1526e-07, -1.1921e-07, -6.5565e-07, -1.7881e-07,\n", - " 4.7684e-07, 1.3113e-06, -9.5367e-07, -2.3842e-07, -1.1325e-06,\n", - " -2.9802e-07, 2.3842e-07, 4.7684e-07, -4.1723e-07, -4.1723e-07,\n", - " 4.7684e-07, 1.0729e-06, 7.1526e-07, -1.7285e-06, 2.3842e-07,\n", - " -6.5565e-07, 4.7684e-07, -1.0729e-06, 2.3842e-07, -4.1723e-07,\n", - " -8.3447e-07, 3.5763e-07, -8.3447e-07, 7.1526e-07, 0.0000e+00,\n", - " 8.3447e-07, 5.9605e-07, -4.7684e-07, -1.1325e-06, 1.3113e-06,\n", - " -5.9605e-07, -1.7881e-07, -1.0729e-06, -7.1526e-07, 1.1921e-06,\n", - " -1.7881e-06, -5.9605e-08, 0.0000e+00, -6.5565e-07, -9.5367e-07,\n", - " 3.5763e-07, -6.5565e-07, 3.5763e-07, -5.3644e-07, -2.3842e-07,\n", - " 3.5763e-07, -4.1723e-07, 3.5763e-07, 1.1921e-07, -1.7881e-07,\n", - " -3.5763e-07, 7.1526e-07, -2.9802e-07, 0.0000e+00, 2.3842e-07,\n", - " 3.5763e-07, 7.1526e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", - " -4.7684e-07, -9.5367e-07, -4.1723e-07, -8.9407e-07, -1.1921e-07,\n", - " -1.7285e-06, 3.5763e-07, -1.1325e-06, -3.5763e-07, -9.5367e-07,\n", - " 1.3113e-06, 3.5763e-07, -5.9605e-07, -1.9670e-06, 9.5367e-07,\n", - " 1.3113e-06, -1.0133e-06, 1.1921e-06, -2.9802e-07, 1.0729e-06,\n", - " 9.5367e-07, 1.1921e-06, 1.4305e-06, 5.9605e-07, 0.0000e+00,\n", - " -1.7881e-07, 9.5367e-07, -1.4305e-06, -1.1921e-07, -7.1526e-07,\n", - " 0.0000e+00, 1.1921e-07, 0.0000e+00, -2.3842e-07, -8.3447e-07,\n", - " 4.7684e-07, -3.5763e-07, 5.9605e-07, 0.0000e+00, 4.7684e-07,\n", - " -1.7881e-07, 8.3447e-07, -2.0862e-06, 0.0000e+00, -6.5565e-07,\n", - " 0.0000e+00, -4.7684e-07, -1.4901e-06, 5.9605e-07, 2.3842e-07,\n", - " 1.4305e-06, -1.1921e-07, -5.3644e-07, 8.3447e-07, 5.9605e-07,\n", - " -8.9407e-07, 9.5367e-07, -5.9605e-07, -4.7684e-07, 8.3447e-07,\n", - " -1.1325e-06, -1.7881e-07, -1.6689e-06, -7.1526e-07, 0.0000e+00,\n", - " 5.9605e-07, -9.5367e-07, 0.0000e+00, 1.0729e-06, 1.4305e-06,\n", - " 3.5763e-07, 4.7684e-07, -3.5763e-07, 5.9605e-07, -2.3842e-07,\n", - " 5.9605e-07, -1.1325e-06, -5.3644e-07, -8.9407e-07, -1.9670e-06,\n", - " 2.3842e-07, 5.9605e-07, -5.9605e-08, 2.3842e-07, -1.0729e-06,\n", - " -1.3113e-06, -1.0133e-06, 2.3842e-07, -1.0133e-06, -2.9802e-07,\n", - " -3.5763e-07, -1.1921e-07, 2.0266e-06, -5.9605e-07, 1.3113e-06,\n", - " -6.5565e-07, 0.0000e+00, 1.1921e-07, 1.3113e-06, 2.3842e-07,\n", - " -5.3644e-07, -2.9802e-07, 1.6689e-06, 3.5763e-07, 4.7684e-07,\n", - " 2.3842e-07, 7.1526e-07, -2.9802e-07, 7.1526e-07, 1.0729e-06,\n", - " -1.8477e-06, 2.3842e-07, 1.1921e-07, 7.1526e-07, 0.0000e+00,\n", - " 0.0000e+00, -1.4305e-06, 5.9605e-07, -7.1526e-07, -5.3644e-07,\n", - " 5.9605e-07, 3.5763e-07, 1.1921e-07, -1.0729e-06, 2.3842e-07,\n", - " -1.7881e-07, -6.5565e-07, 1.1921e-06, -5.3644e-07, 1.3113e-06,\n", - " 3.5763e-07, 5.9605e-07, -1.0133e-06, 8.3447e-07, 1.0729e-06,\n", - " 7.1526e-07, 7.1526e-07, 8.3447e-07, -5.9605e-08, -1.0133e-06,\n", - " 2.3842e-07, -2.0266e-06, 8.3447e-07, 1.1921e-07, 1.1921e-06,\n", - " 3.5763e-07, -1.0729e-06, 2.3842e-07, 0.0000e+00, 7.1526e-07,\n", - " -1.1921e-07, 8.3447e-07, 1.1921e-07, -1.1921e-07, 2.3842e-07,\n", - " -1.0133e-06], device='cuda:0')),\n", - " ('layer3.0.shortcut.1.bias',\n", - " tensor([ 1.2397e-06, 6.3916e-07, -1.5730e-07, 7.7664e-07, 6.6675e-07,\n", - " 7.5460e-07, 3.2428e-07, -4.9460e-07, -7.5864e-09, -1.1655e-07,\n", - " 4.3920e-07, 5.2803e-07, 3.4970e-07, -5.1197e-07, -2.4610e-07,\n", - " 6.8364e-07, 2.1216e-08, 5.2061e-07, 1.2123e-06, -3.2477e-07,\n", - " -7.3147e-08, -3.9508e-07, 2.2369e-07, -5.3686e-07, 1.8602e-08,\n", - " 1.3056e-06, -9.8787e-07, 1.3210e-06, -1.8658e-06, 2.4755e-07,\n", - " 4.4678e-07, -5.5527e-07, -4.8989e-07, 6.6215e-07, 8.1737e-07,\n", - " 1.0583e-06, -8.2283e-08, 2.7286e-07, 9.8735e-08, -4.9545e-07,\n", - " -1.0303e-07, 5.3118e-07, -7.7630e-07, -2.2175e-07, -8.3009e-07,\n", - " -3.2777e-07, -4.4338e-07, -8.1797e-07, -3.8200e-07, -5.3417e-07,\n", - " 3.6573e-07, 1.2024e-06, 1.0901e-06, -8.4797e-07, 2.9635e-07,\n", - " -4.1304e-07, 9.2186e-07, -5.6299e-07, 4.1419e-07, 1.4131e-07,\n", - " -2.4994e-08, 1.0638e-06, 7.4731e-07, 3.4836e-07, -6.1722e-07,\n", - " -3.2058e-07, 4.1246e-07, -2.0439e-07, -7.5934e-07, 2.0549e-07,\n", - " -4.7065e-07, 2.8008e-07, 2.7104e-07, -7.3181e-07, 9.1271e-07,\n", - " 8.7888e-08, -1.5313e-07, -3.1190e-08, -4.8686e-07, -2.8495e-07,\n", - " 8.0396e-07, -5.6101e-07, -1.4228e-07, 3.0207e-07, -4.3237e-07,\n", - " 3.8904e-07, 2.6641e-07, 2.0103e-07, -5.5700e-07, 2.1933e-07,\n", - " -7.6924e-07, 2.4533e-07, -2.0562e-07, -6.3526e-07, -7.7532e-07,\n", - " -1.0781e-08, -6.0451e-08, 2.9524e-07, -2.9730e-07, -7.1229e-07,\n", - " -5.2639e-07, -1.0158e-06, -1.8155e-07, -9.5364e-07, -7.7976e-08,\n", - " 4.2866e-07, 1.0097e-06, -2.3944e-07, 4.3388e-07, -7.4387e-07,\n", - " 3.8048e-07, -7.9571e-07, -9.8154e-07, -4.4025e-07, 2.4142e-07,\n", - " 4.3612e-07, -1.4472e-07, -4.2033e-07, -1.6008e-07, 6.7188e-09,\n", - " 5.8529e-07, -2.9896e-07, 7.1821e-07, 5.8884e-07, -3.3722e-08,\n", - " -8.7960e-07, -1.7185e-07, 3.9410e-09, 2.6008e-07, -7.1072e-07,\n", - " -1.2491e-08, -5.7730e-07, -1.2584e-06, -4.9229e-07, 3.5214e-07,\n", - " 1.3187e-06, -5.0259e-07, 1.9477e-07, -2.3074e-07, 8.2709e-07,\n", - " 5.3013e-07, -1.0832e-06, 2.9348e-07, -2.3056e-07, 9.2496e-07,\n", - " -3.6641e-07, -2.7478e-07, -7.4791e-07, 9.0510e-07, -3.4336e-07,\n", - " 1.4417e-06, 4.0621e-07, 3.4944e-07, 1.1388e-06, -4.8168e-08,\n", - " -4.2881e-08, 1.2046e-07, 1.9531e-07, -4.9289e-07, -4.9626e-07,\n", - " -2.5901e-07, 5.8413e-07, -1.1101e-06, 3.8676e-07, -2.1239e-07,\n", - " 3.7457e-07, -3.3742e-07, 1.3505e-07, 2.8487e-07, -2.7694e-07,\n", - " 7.8956e-07, 5.4949e-07, 5.5654e-07, 5.7680e-07, 2.9649e-07,\n", - " 3.8036e-07, -4.4860e-07, -3.1570e-07, -9.8473e-07, -7.6646e-07,\n", - " -8.1532e-08, 1.9141e-07, 8.7107e-07, 5.5474e-09, -6.6521e-07,\n", - " -2.2067e-07, -9.4595e-08, 6.9143e-07, 1.0157e-07, -5.9210e-07,\n", - " 1.1940e-08, 4.6116e-07, 1.2312e-06, 1.6454e-07, 4.2485e-07,\n", - " -4.9396e-07, -3.4643e-07, 1.1768e-07, 4.5359e-09, -4.0335e-07,\n", - " 1.4072e-07, 2.7469e-07, 6.8554e-07, -8.4500e-08, 3.7428e-07,\n", - " 5.9899e-07, 4.1019e-07, -8.1789e-07, -5.4011e-07, -3.5031e-07,\n", - " -5.5258e-07, -3.5712e-07, 7.5301e-07, 5.9972e-07, 4.0490e-07,\n", - " 2.6592e-07, 4.8271e-07, 4.1183e-07, 7.3446e-07, -5.6122e-07,\n", - " -3.2023e-08, -6.2278e-08, 1.9076e-07, -8.7138e-07, 1.1447e-06,\n", - " 8.4598e-07, -2.0409e-07, 1.0238e-06, 6.4743e-07, 3.9771e-07,\n", - " -8.7026e-07, 2.1872e-07, -5.9180e-07, 7.0975e-07, -1.5564e-07,\n", - " 6.7422e-07, 1.3821e-06, 6.2665e-07, -4.7970e-07, -3.3708e-07,\n", - " -7.5254e-08, -6.1423e-07, 6.8763e-07, 1.4221e-07, 1.8319e-06,\n", - " 1.1302e-06, 3.3478e-07, -3.1522e-07, 1.8243e-07, -7.6020e-08,\n", - " 1.0345e-06, 3.5484e-07, 6.0152e-07, -3.3911e-07, -3.1310e-07,\n", - " -6.6216e-07], device='cuda:0')),\n", - " ('layer4.0.conv1.weight',\n", - " tensor([[[[-6.5658e-07, -3.7719e-08, -2.9663e-07],\n", - " [-4.8429e-08, -2.5332e-07, -1.2014e-07],\n", - " [ 4.7125e-07, -5.1968e-07, 5.4017e-07]],\n", - " \n", - " [[ 2.0396e-06, -3.2212e-07, 2.5299e-06],\n", - " [-1.3877e-07, 7.3831e-07, 1.0414e-06],\n", - " [ 5.2899e-07, 1.7328e-07, 1.5325e-06]],\n", - " \n", - " [[-4.4890e-07, 6.5938e-07, -9.2853e-07],\n", - " [-1.3839e-06, -1.4184e-06, -1.1837e-06],\n", - " [-1.3523e-06, -7.1991e-07, -1.0827e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.0920e-07, 2.1793e-07, 7.3388e-07],\n", - " [ 1.1707e-06, -3.0018e-07, 1.1865e-06],\n", - " [ 4.1351e-07, -2.1013e-07, 1.1569e-06]],\n", - " \n", - " [[ 2.6543e-07, -1.2666e-07, -6.8825e-07],\n", - " [-1.1735e-07, 1.9097e-06, 6.3935e-07],\n", - " [-6.5679e-07, 4.5449e-07, 2.9802e-08]],\n", - " \n", - " [[ 1.4207e-06, -8.2888e-07, 1.7975e-06],\n", - " [ 1.8091e-06, 4.5635e-07, 1.3406e-06],\n", - " [ 1.6857e-06, -3.1013e-07, 8.6613e-07]]],\n", - " \n", - " \n", - " [[[ 2.2002e-07, 9.5833e-07, 4.0652e-07],\n", - " [-5.3830e-07, -4.3027e-07, 2.3842e-07],\n", - " [ 9.0199e-07, 8.8662e-07, -9.8790e-07]],\n", - " \n", - " [[ 5.3830e-07, 1.1232e-06, 3.0827e-07],\n", - " [ 5.1875e-07, 9.0897e-07, 1.1828e-06],\n", - " [-4.0978e-07, 5.5810e-07, -4.8988e-07]],\n", - " \n", - " [[ 4.9081e-07, -1.2154e-07, 3.5390e-08],\n", - " [-2.1048e-07, -5.1782e-07, 4.5868e-07],\n", - " [ 2.3190e-07, 2.6426e-07, -5.2270e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.9989e-07, -4.8336e-07, 1.5208e-06],\n", - " [-6.8266e-07, -8.3214e-07, -7.0082e-07],\n", - " [ 3.0920e-07, -1.4110e-07, 5.4017e-07]],\n", - " \n", - " [[ 6.5123e-07, 1.0990e-07, 3.7439e-07],\n", - " [ 1.2461e-06, 3.9581e-07, 1.0966e-07],\n", - " [-8.0967e-07, 1.8561e-06, 7.7463e-07]],\n", - " \n", - " [[-3.7160e-07, -2.9057e-07, -4.2841e-07],\n", - " [-5.7276e-07, 4.5635e-08, -9.2760e-07],\n", - " [-4.7637e-07, 3.2876e-07, 5.8860e-07]]],\n", - " \n", - " \n", - " [[[-9.1828e-07, 1.2137e-06, -2.3949e-06],\n", - " [-1.2591e-06, -4.2468e-07, -1.0675e-07],\n", - " [-1.5423e-06, 3.9022e-07, -2.0843e-06]],\n", - " \n", - " [[ 6.0163e-07, 1.4980e-06, 2.7362e-06],\n", - " [ 1.5371e-06, 1.3086e-06, 3.1629e-06],\n", - " [ 9.4902e-07, 2.4084e-06, 2.9653e-06]],\n", - " \n", - " [[ 7.2224e-07, 2.5891e-07, 1.6082e-06],\n", - " [-5.0175e-08, -5.1875e-07, -2.2561e-07],\n", - " [ 8.0466e-07, 2.0210e-07, 1.5367e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.1735e-07, -1.5814e-06, 4.2748e-07],\n", - " [ 4.9824e-07, -1.3281e-06, 1.1828e-06],\n", - " [ 1.1455e-07, 9.7649e-07, 1.0896e-07]],\n", - " \n", - " [[ 3.3528e-07, -1.2049e-06, 7.2643e-07],\n", - " [ 2.1793e-07, 1.5032e-06, -4.7963e-07],\n", - " [ 2.7940e-08, 5.8580e-07, 1.3281e-06]],\n", - " \n", - " [[ 1.9185e-07, 2.9057e-07, 4.7404e-07],\n", - " [-8.3633e-07, -1.9590e-06, -1.0915e-06],\n", - " [ 2.5146e-07, 4.7730e-07, 1.0227e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-7.5810e-07, -1.2480e-07, -1.9427e-06],\n", - " [-1.5055e-06, -1.2676e-06, -2.9870e-06],\n", - " [-1.4789e-06, -3.5646e-07, -1.4715e-06]],\n", - " \n", - " [[-8.5495e-07, 3.9279e-07, -6.1467e-07],\n", - " [ 2.4051e-07, -1.4603e-06, -8.8196e-07],\n", - " [ 4.9779e-07, 1.4668e-07, 1.8626e-07]],\n", - " \n", - " [[ 4.3493e-07, -1.2247e-06, -4.5542e-07],\n", - " [ 3.5041e-07, -2.5146e-08, -4.5635e-08],\n", - " [-4.7311e-07, 1.9651e-07, -2.6077e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.9430e-07, 1.9930e-07, -1.0431e-07],\n", - " [-5.9605e-07, 1.8850e-06, 2.1327e-07],\n", - " [-7.6077e-07, 1.2624e-06, -3.9116e-08]],\n", - " \n", - " [[-3.6974e-07, -6.9849e-07, -8.7731e-07],\n", - " [ 8.5682e-07, -9.7789e-07, -2.1234e-07],\n", - " [-7.4506e-07, -1.7444e-06, -3.2596e-07]],\n", - " \n", - " [[ 2.1625e-06, 5.9232e-07, -2.6077e-08],\n", - " [ 1.7248e-06, 1.7649e-07, 1.8496e-06],\n", - " [ 1.3439e-06, 1.5330e-06, 6.8173e-07]]],\n", - " \n", - " \n", - " [[[ 1.6000e-06, 5.0478e-07, 1.2908e-06],\n", - " [ 1.5376e-06, 1.5721e-06, 1.4119e-06],\n", - " [ 1.5273e-06, 2.9244e-07, 1.4193e-06]],\n", - " \n", - " [[ 8.0373e-07, 1.6098e-06, 2.6496e-07],\n", - " [ 7.0967e-07, -2.9593e-07, -9.0525e-07],\n", - " [ 7.0618e-07, -9.5926e-08, -4.2003e-07]],\n", - " \n", - " [[-4.3213e-07, 2.5705e-07, -4.4983e-07],\n", - " [-4.2096e-07, -4.0606e-07, -1.0002e-06],\n", - " [-3.9022e-07, 9.8720e-07, -2.5984e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 8.2701e-07, 1.3579e-06, -9.7789e-07],\n", - " [ 2.5146e-08, -5.1904e-07, -5.4017e-07],\n", - " [ 3.4831e-07, -8.9407e-07, -2.9244e-07]],\n", - " \n", - " [[ 5.7090e-07, 9.4157e-07, 1.5087e-07],\n", - " [ 3.1199e-07, -6.3889e-07, 4.5542e-07],\n", - " [-1.7975e-07, -2.3581e-06, -6.7893e-07]],\n", - " \n", - " [[ 3.1851e-07, -5.7835e-07, -5.1921e-08],\n", - " [-6.1467e-08, -8.4843e-07, -1.4808e-07],\n", - " [ 4.3213e-07, -1.0869e-06, -9.1176e-07]]],\n", - " \n", - " \n", - " [[[-9.7789e-08, -1.2685e-06, -3.0734e-07],\n", - " [-4.3772e-07, -7.9021e-07, -1.9558e-07],\n", - " [-1.3858e-06, -1.0114e-06, -4.6566e-09]],\n", - " \n", - " [[-1.9465e-07, 1.3970e-07, -2.5518e-07],\n", - " [-1.2610e-06, 5.5321e-07, -1.4156e-07],\n", - " [ 7.4506e-09, -1.0431e-07, -9.9780e-07]],\n", - " \n", - " [[-1.8813e-07, -9.2754e-08, -1.7043e-07],\n", - " [-8.9034e-07, 2.3097e-07, -4.9919e-07],\n", - " [-1.7090e-06, 3.1479e-07, 3.8277e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.5297e-07, -4.0699e-07, -7.6136e-07],\n", - " [ 3.0734e-07, -1.9372e-07, -6.2305e-07],\n", - " [-3.4645e-07, 2.3376e-06, -1.1618e-07]],\n", - " \n", - " [[ 9.9558e-07, 7.0501e-07, 4.0699e-07],\n", - " [-6.5751e-07, -8.3912e-07, -1.0272e-06],\n", - " [-6.4448e-07, -2.7008e-07, -7.0781e-07]],\n", - " \n", - " [[-3.0873e-07, -3.8370e-07, -1.4324e-06],\n", - " [ 2.2273e-06, 5.0152e-07, 1.1120e-06],\n", - " [ 3.8836e-07, 5.5414e-07, 1.1791e-06]]]], device='cuda:0')),\n", - " ('layer4.0.bn1.weight',\n", - " tensor([ 4.7684e-07, 1.1921e-07, 4.7684e-07, 2.3842e-07, 1.1921e-07,\n", - " 4.7684e-07, -1.1921e-07, 0.0000e+00, -1.7881e-07, 4.7684e-07,\n", - " 1.1921e-07, -2.9802e-07, 2.3842e-07, 3.5763e-07, -4.1723e-07,\n", - " 7.1526e-07, 4.7684e-07, -2.9802e-07, 0.0000e+00, 3.5763e-07,\n", - " 4.7684e-07, -1.7881e-07, -1.7881e-07, 4.7684e-07, -2.3842e-07,\n", - " -5.9605e-08, -5.9605e-08, -5.9605e-08, -5.3644e-07, -1.7881e-07,\n", - " 4.7684e-07, -3.5763e-07, 0.0000e+00, -5.3644e-07, 0.0000e+00,\n", - " -1.7881e-07, 1.1921e-07, 1.3113e-06, -1.1921e-07, -2.9802e-07,\n", - " -7.7486e-07, 2.3842e-07, -7.1526e-07, 5.9605e-07, -8.3447e-07,\n", - " 4.7684e-07, 0.0000e+00, -4.7684e-07, -4.7684e-07, -2.9802e-07,\n", - " -7.1526e-07, 1.1921e-07, -5.3644e-07, 7.1526e-07, -4.1723e-07,\n", - " 3.5763e-07, -4.7684e-07, 1.1921e-07, 0.0000e+00, 3.5763e-07,\n", - " -7.7486e-07, -5.9605e-08, 1.5497e-06, -6.5565e-07, -2.3842e-07,\n", - " 9.5367e-07, 0.0000e+00, -5.9605e-08, 3.5763e-07, -2.3842e-07,\n", - " -8.3447e-07, 1.1921e-07, -1.1921e-07, 7.1526e-07, -2.9802e-07,\n", - " -1.0133e-06, 5.9605e-07, -2.9802e-07, 1.1921e-07, 3.5763e-07,\n", - " -3.5763e-07, 4.7684e-07, -2.3842e-07, 1.1921e-07, 1.1921e-07,\n", - " 0.0000e+00, -4.1723e-07, -1.1921e-07, 2.3842e-07, 3.5763e-07,\n", - " 0.0000e+00, -4.7684e-07, 2.3842e-07, 1.1921e-07, -2.9802e-07,\n", - " 8.3447e-07, -1.1921e-07, -1.7881e-07, -2.3842e-07, -2.3842e-07,\n", - " 7.1526e-07, -2.3842e-07, -2.9802e-07, 4.7684e-07, 4.7684e-07,\n", - " -5.9605e-08, -3.5763e-07, 5.9605e-07, 1.1921e-07, -3.5763e-07,\n", - " -3.5763e-07, -5.9605e-07, 0.0000e+00, -2.9802e-07, 9.5367e-07,\n", - " 2.3842e-07, 2.3842e-07, 2.3842e-07, -1.1921e-07, -7.7486e-07,\n", - " -2.3842e-07, -1.7881e-07, -2.9802e-07, -3.5763e-07, -5.9605e-07,\n", - " -6.5565e-07, -2.9802e-07, 4.7684e-07, -5.9605e-07, 1.1921e-07,\n", - " 1.1921e-07, 2.3842e-07, 0.0000e+00, 2.3842e-07, 4.7684e-07,\n", - " -1.7881e-07, 3.5763e-07, -4.7684e-07, -1.7881e-07, -9.5367e-07,\n", - " -2.3842e-07, -1.7881e-07, 3.5763e-07, 4.7684e-07, 3.5763e-07,\n", - " 1.0729e-06, 2.3842e-07, -4.7684e-07, 2.3842e-07, 1.1921e-07,\n", - " -2.9802e-07, -1.1921e-07, -7.7486e-07, -1.1921e-07, 1.1921e-07,\n", - " -2.3842e-07, -1.7881e-07, 1.1921e-07, 2.3842e-07, -5.9605e-08,\n", - " 9.5367e-07, 2.3842e-07, 7.1526e-07, -1.1921e-07, -1.1921e-07,\n", - " -7.1526e-07, 5.9605e-07, -1.7881e-07, -2.9802e-07, 8.3447e-07,\n", - " -4.7684e-07, 1.1921e-07, -6.5565e-07, 1.1921e-07, 5.9605e-07,\n", - " -2.3842e-07, 0.0000e+00, 0.0000e+00, 4.7684e-07, 1.1921e-07,\n", - " 1.1921e-07, -6.5565e-07, 9.5367e-07, 5.9605e-07, 1.1921e-07,\n", - " -5.9605e-08, -6.5565e-07, -3.5763e-07, -1.1921e-07, -6.5565e-07,\n", - " 5.9605e-07, -5.3644e-07, -8.9407e-07, 3.5763e-07, -8.3447e-07,\n", - " 5.9605e-07, 4.7684e-07, 3.5763e-07, 8.3447e-07, 0.0000e+00,\n", - " 1.1921e-07, 0.0000e+00, -4.1723e-07, 2.3842e-07, 1.1921e-07,\n", - " 4.7684e-07, -1.7881e-07, -4.7684e-07, -5.9605e-08, -2.9802e-07,\n", - " 2.3842e-07, 1.1921e-07, 2.3842e-07, -2.9802e-07, -1.7881e-07,\n", - " -3.5763e-07, 0.0000e+00, -1.7881e-07, -7.1526e-07, -2.9802e-07,\n", - " 5.9605e-07, 2.3842e-07, 1.1921e-07, -2.9802e-07, 2.3842e-07,\n", - " 1.1921e-07, 3.5763e-07, 1.1921e-06, 2.3842e-07, -2.3842e-07,\n", - " -4.1723e-07, 2.3842e-07, 1.1921e-07, -1.7881e-07, -6.5565e-07,\n", - " -1.0729e-06, -5.9605e-08, -2.3842e-07, 0.0000e+00, 4.7684e-07,\n", - " -5.9605e-08, 4.7684e-07, 3.5763e-07, -4.1723e-07, 0.0000e+00,\n", - " -6.5565e-07, 1.1921e-07, 1.1921e-07, 0.0000e+00, -5.3644e-07,\n", - " 0.0000e+00, 1.1921e-07, 1.1921e-07, 9.5367e-07, 1.1921e-07,\n", - " 3.5763e-07, -1.2517e-06, 1.1921e-07, -4.7684e-07, 1.1921e-06,\n", - " -3.5763e-07, 1.1921e-07, 1.1921e-07, -7.1526e-07, -8.9407e-07,\n", - " 1.1921e-07, 3.5763e-07, 0.0000e+00, -4.1723e-07, 2.3842e-07,\n", - " -8.3447e-07, 3.5763e-07, 1.1921e-07, -3.5763e-07, -5.3644e-07,\n", - " 3.5763e-07, 0.0000e+00, -2.9802e-07, -5.9605e-08, 3.5763e-07,\n", - " -1.7881e-07, 4.7684e-07, -2.9802e-07, 7.1526e-07, -5.3644e-07,\n", - " -7.7486e-07, 1.3113e-06, 1.1921e-07, -1.7881e-07, -3.5763e-07,\n", - " -1.1921e-07, -1.1921e-07, -2.9802e-07, 2.3842e-07, 4.7684e-07,\n", - " -5.3644e-07, -2.3842e-07, 4.7684e-07, 3.5763e-07, 1.1921e-07,\n", - " 7.1526e-07, -1.7881e-07, 5.9605e-07, -5.3644e-07, -1.7881e-07,\n", - " 9.5367e-07, -4.7684e-07, -5.9605e-08, 3.5763e-07, -4.7684e-07,\n", - " -7.7486e-07, -1.7881e-07, 1.1921e-07, -5.9605e-08, 0.0000e+00,\n", - " 2.3842e-07, 0.0000e+00, -5.3644e-07, 8.3447e-07, 0.0000e+00,\n", - " 3.5763e-07, -2.9802e-07, -5.9605e-08, 0.0000e+00, 4.7684e-07,\n", - " -2.9802e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07, 0.0000e+00,\n", - " -2.9802e-07, -4.7684e-07, 2.3842e-07, 0.0000e+00, -5.3644e-07,\n", - " -3.5763e-07, -5.9605e-08, 4.7684e-07, 1.0729e-06, -1.7881e-07,\n", - " 3.5763e-07, 7.1526e-07, 1.1921e-07, 9.5367e-07, 4.7684e-07,\n", - " 8.3447e-07, -4.1723e-07, 2.3842e-07, -3.5763e-07, -5.3644e-07,\n", - " 0.0000e+00, -1.7881e-07, 2.3842e-07, 0.0000e+00, 3.5763e-07,\n", - " -8.9407e-07, 1.1921e-07, 1.1921e-07, 7.1526e-07, 4.7684e-07,\n", - " -3.5763e-07, 4.7684e-07, -5.3644e-07, 3.5763e-07, -4.7684e-07,\n", - " 1.1921e-07, -2.9802e-07, 1.1921e-07, 7.1526e-07, 0.0000e+00,\n", - " -1.7881e-07, 4.7684e-07, -9.5367e-07, -1.1325e-06, -1.7881e-07,\n", - " -5.9605e-07, -1.1921e-07, -2.3842e-07, -1.7881e-07, 3.5763e-07,\n", - " 1.1921e-07, -7.1526e-07, -2.3842e-07, 2.3842e-07, -4.7684e-07,\n", - " -1.1921e-07, -1.7881e-07, 1.1921e-07, 4.7684e-07, -2.3842e-07,\n", - " -5.3644e-07, -5.9605e-08, -1.7881e-07, 1.0729e-06, 2.3842e-07,\n", - " 1.1921e-07, 1.4305e-06, 1.3113e-06, 1.0729e-06, 3.5763e-07,\n", - " 0.0000e+00, -4.1723e-07, -5.9605e-07, -1.7881e-07, 1.1921e-07,\n", - " -7.1526e-07, 0.0000e+00, 2.3842e-07, -7.1526e-07, 1.1921e-07,\n", - " 4.7684e-07, -4.1723e-07, -4.1723e-07, -1.7881e-07, -5.3644e-07,\n", - " 2.3842e-07, 1.1921e-07, -1.7881e-07, 1.1921e-07, 1.0729e-06,\n", - " -2.9802e-07, -2.9802e-07, 2.3842e-07, 1.4305e-06, 4.7684e-07,\n", - " -4.1723e-07, -5.9605e-08, 0.0000e+00, 0.0000e+00, 1.1921e-07,\n", - " 1.1921e-07, 0.0000e+00, -3.5763e-07, -4.1723e-07, -6.5565e-07,\n", - " -3.5763e-07, -2.9802e-07, 0.0000e+00, -2.9802e-07, 2.3842e-07,\n", - " -7.7486e-07, 7.1526e-07, -3.5763e-07, 3.5763e-07, -6.5565e-07,\n", - " 2.3842e-07, -1.7881e-07, -3.5763e-07, 2.3842e-07, 1.1921e-07,\n", - " 1.1921e-07, 2.3842e-07, -1.7881e-07, -4.1723e-07, -1.1921e-07,\n", - " -5.9605e-07, 3.5763e-07, -2.3842e-07, 2.3842e-07, -1.7881e-07,\n", - " 5.9605e-07, -1.1325e-06, -3.5763e-07, -4.1723e-07, -3.5763e-07,\n", - " 2.3842e-07, -1.1921e-07, -5.9605e-07, 7.1526e-07, -5.9605e-08,\n", - " -2.3842e-07, 3.5763e-07, -7.1526e-07, -5.9605e-08, -5.3644e-07,\n", - " 7.1526e-07, -2.9802e-07, -4.1723e-07, -1.1921e-07, 3.5763e-07,\n", - " 1.0729e-06, 2.3842e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07,\n", - " 4.7684e-07, -2.9802e-07, -2.3842e-07, -2.9802e-07, 5.9605e-07,\n", - " -6.5565e-07, -8.3447e-07, -1.7881e-07, 0.0000e+00, -6.5565e-07,\n", - " 3.5763e-07, 7.1526e-07, -2.9802e-07, 7.1526e-07, -1.7881e-07,\n", - " -5.9605e-08, 0.0000e+00, -1.1921e-07, 0.0000e+00, -1.1921e-07,\n", - " 2.3842e-07, 3.5763e-07, 2.3842e-07, 4.7684e-07, 3.5763e-07,\n", - " -6.5565e-07, 1.1921e-07], device='cuda:0')),\n", - " ('layer4.0.bn1.bias',\n", - " tensor([ 2.6659e-07, -2.0982e-07, 4.8005e-07, -1.1635e-07, 3.8215e-07,\n", - " 6.8295e-07, -1.7284e-07, 2.8059e-07, -1.3988e-07, 1.1298e-08,\n", - " -1.2147e-07, 3.9384e-09, 1.0461e-07, 2.2419e-07, -2.6699e-07,\n", - " 7.8134e-08, 7.8973e-07, -2.7129e-07, -4.5257e-07, 2.7466e-07,\n", - " 1.7832e-07, -2.4214e-07, -2.9081e-07, 1.8024e-07, -4.9308e-07,\n", - " -5.6225e-08, -4.9937e-09, 7.0436e-08, -6.0486e-07, -3.4290e-07,\n", - " 3.8399e-07, 5.2644e-08, -4.3564e-07, -7.4487e-07, 2.6390e-07,\n", - " -1.8460e-07, -1.3691e-07, 8.6185e-07, 1.0023e-07, -1.2655e-07,\n", - " -8.7669e-07, 1.7654e-08, -1.9440e-07, 3.6522e-07, -3.4770e-07,\n", - " 7.3103e-07, 5.0401e-07, -4.6698e-07, -1.1129e-07, 4.6539e-09,\n", - " -6.6089e-07, 2.4661e-07, -4.3374e-07, 1.4014e-07, 5.5946e-07,\n", - " 3.3049e-07, -3.6001e-07, 2.8588e-07, 3.3717e-07, -5.1532e-09,\n", - " -6.4083e-07, -5.5091e-07, 6.9731e-07, -5.2641e-07, -4.6825e-07,\n", - " 6.0111e-07, 3.3230e-07, 7.3765e-08, 3.7736e-07, -2.2260e-07,\n", - " -8.5065e-07, 5.1249e-09, 2.4908e-08, 2.0588e-07, -8.0888e-07,\n", - " -8.7111e-07, 5.2081e-07, -1.1161e-07, 3.7942e-07, -5.3236e-08,\n", - " -4.2491e-07, 5.6897e-07, -1.9099e-07, -5.0960e-08, 3.1389e-07,\n", - " 2.5616e-07, 6.1289e-09, -2.5935e-07, 2.5534e-07, 7.3285e-07,\n", - " -2.0531e-07, 2.5075e-07, 5.8549e-07, -3.4469e-07, -2.5699e-07,\n", - " 2.5711e-07, -1.7227e-07, 2.5903e-08, -2.9787e-07, -2.6435e-07,\n", - " 6.9626e-07, -3.2891e-07, -4.7151e-07, -4.9839e-08, 5.5711e-07,\n", - " 1.8523e-07, 1.5311e-08, -6.6544e-08, -2.2489e-07, 2.5704e-07,\n", - " -1.6068e-07, -2.5091e-07, -3.2175e-07, -2.8687e-07, 3.1285e-07,\n", - " 4.7909e-08, 3.8899e-07, 1.6728e-07, -1.8015e-07, -2.1716e-07,\n", - " 6.1003e-08, -2.6587e-07, -1.5374e-07, -2.8845e-07, 8.9849e-10,\n", - " -8.4853e-07, -3.3859e-08, 1.5536e-07, 1.1042e-07, 2.5031e-07,\n", - " 2.6351e-07, 1.7642e-07, -5.4142e-08, 7.8362e-07, 6.5737e-07,\n", - " 7.1140e-08, 3.5251e-07, -3.2149e-07, -1.6892e-07, -8.2579e-07,\n", - " -4.2970e-07, -1.9542e-07, 2.1797e-07, 3.0029e-07, 2.6878e-07,\n", - " 6.2129e-07, 2.6288e-07, -5.4413e-07, -1.4614e-08, -1.6254e-07,\n", - " 2.2507e-07, -3.1475e-07, -5.3041e-07, -6.0969e-08, 3.2513e-07,\n", - " 5.7300e-08, 1.0650e-07, -4.1601e-07, 4.1368e-07, -2.2369e-07,\n", - " 1.1620e-06, 3.3273e-07, 5.0645e-07, -3.6037e-07, -6.9913e-07,\n", - " -5.1527e-07, 5.1412e-07, -3.7701e-07, -5.4504e-08, 2.0858e-07,\n", - " -2.5934e-08, 5.5633e-07, -4.4394e-07, 9.8369e-08, 2.2145e-07,\n", - " -8.7645e-09, 1.3667e-07, 1.7740e-07, 6.7056e-07, -5.0873e-07,\n", - " -1.0292e-07, -2.5302e-07, 1.1807e-06, -1.4492e-08, 1.6166e-07,\n", - " -1.9679e-07, -7.0686e-08, -2.7642e-07, 2.6622e-08, -4.0368e-07,\n", - " 2.3976e-07, -6.0022e-07, -1.4709e-07, 7.6702e-07, -5.0405e-07,\n", - " 8.8996e-07, 3.3934e-07, 1.3966e-07, 4.7994e-07, 8.3614e-08,\n", - " 4.5502e-07, 2.5827e-07, -4.7942e-07, 1.5037e-07, -3.9148e-07,\n", - " 3.0440e-07, 5.3337e-08, -3.9186e-07, -2.7119e-08, -1.5321e-07,\n", - " 2.5038e-07, 6.7230e-07, -3.2854e-07, -3.6195e-08, -3.2283e-07,\n", - " 8.6197e-08, 2.7312e-07, 1.3391e-07, -6.2963e-07, -4.4744e-07,\n", - " 3.5468e-07, 5.9017e-07, -1.0100e-07, -2.4813e-07, 2.6591e-07,\n", - " 4.4953e-07, 6.9342e-07, 1.0524e-06, -1.4089e-07, -5.0600e-07,\n", - " 2.6431e-07, -1.5005e-07, 2.6835e-07, -6.6012e-07, -5.5946e-07,\n", - " -9.7378e-07, 2.1921e-07, -2.6048e-07, 1.4275e-07, 4.5729e-07,\n", - " 2.6154e-07, 5.4520e-07, 4.6810e-07, 1.3848e-09, -2.5565e-07,\n", - " -3.8275e-07, -6.9606e-07, -1.2039e-07, -1.1958e-07, -3.8735e-07,\n", - " 5.1144e-07, 2.9636e-07, 4.2048e-07, 2.8232e-07, 7.2826e-08,\n", - " 4.0116e-07, -7.1807e-07, 3.7533e-07, 2.8475e-07, 7.3259e-07,\n", - " -3.1961e-08, 9.2208e-08, 1.4817e-07, -4.8989e-07, -3.4522e-07,\n", - " 3.8085e-07, -2.1879e-07, -6.7025e-07, 1.8295e-07, 4.9678e-07,\n", - " -6.0239e-07, 7.2177e-07, 2.1635e-07, -4.3453e-07, -3.5543e-07,\n", - " -2.0856e-07, 2.0068e-08, 1.8934e-07, -2.0346e-07, 9.0863e-08,\n", - " 3.3822e-07, 1.9965e-07, -3.4879e-07, 7.0592e-07, 2.8215e-07,\n", - " -6.1477e-07, 8.4427e-07, 3.2009e-07, 4.2358e-07, -3.9903e-07,\n", - " -4.8077e-07, 1.7988e-07, 2.2444e-07, 3.7915e-07, -2.2638e-08,\n", - " -5.8419e-07, 4.5737e-08, 2.1573e-07, -1.0285e-08, -2.2151e-07,\n", - " 9.2880e-07, -3.6823e-07, 5.7077e-07, -5.1109e-07, -2.2646e-07,\n", - " 4.5903e-07, -2.8698e-07, 2.7114e-07, 4.5595e-07, -6.2474e-07,\n", - " -8.6035e-07, 1.2112e-07, 2.1300e-07, -6.6163e-07, 1.6430e-07,\n", - " 3.5742e-07, 2.3956e-08, -7.2227e-07, 5.4798e-07, 2.2492e-07,\n", - " 5.2444e-07, -3.5310e-07, -2.0026e-07, -1.9205e-07, 1.9381e-07,\n", - " -1.4635e-07, 3.7263e-07, -1.9696e-07, 6.0810e-08, -2.2472e-08,\n", - " -5.9005e-07, -2.4705e-07, 9.6962e-07, 1.8417e-07, -1.0311e-07,\n", - " -9.2609e-07, -1.8569e-07, 6.1638e-07, 8.0491e-07, -1.5014e-07,\n", - " 3.7104e-07, 1.5524e-07, -2.4523e-07, 4.3940e-07, 6.8601e-08,\n", - " 2.8458e-08, -1.3479e-07, 2.9886e-07, 2.7438e-07, -4.5789e-07,\n", - " -2.9986e-07, 3.2887e-07, 2.6367e-07, 6.5678e-08, -5.2845e-08,\n", - " -1.8448e-07, 5.9895e-08, 3.8829e-07, 6.2424e-08, 3.1400e-07,\n", - " 1.6121e-07, 2.3031e-07, -4.7892e-07, 5.0512e-07, -6.7946e-07,\n", - " -9.6831e-08, 8.1650e-08, 4.8874e-07, 6.9909e-07, -2.6260e-07,\n", - " -3.3464e-07, 5.1337e-07, -1.1322e-06, -1.9760e-07, -3.6737e-07,\n", - " -6.7565e-07, 5.9541e-08, -1.2394e-07, -5.0226e-07, 5.8358e-07,\n", - " -1.0674e-07, -2.2724e-07, -9.9859e-08, 5.2597e-07, -1.4322e-07,\n", - " -1.8635e-07, -2.4558e-07, 3.3967e-07, -7.3067e-09, -2.4298e-07,\n", - " -4.4757e-07, 1.3672e-07, -1.9951e-07, 9.4763e-07, 2.6936e-07,\n", - " -2.9640e-07, 1.0396e-06, 6.7169e-07, 2.4742e-07, 6.5884e-07,\n", - " 2.0972e-07, -4.0602e-07, -1.2101e-07, -8.6610e-08, 1.1707e-07,\n", - " -1.2173e-06, 7.6500e-08, 6.1672e-07, -2.3657e-07, -1.2074e-07,\n", - " 6.9198e-08, -1.4256e-07, 3.3578e-07, -6.9027e-08, -3.0768e-07,\n", - " 2.3745e-07, -3.5318e-07, -1.9281e-07, -2.4589e-07, 8.3489e-07,\n", - " 6.0610e-08, -1.0983e-07, 3.5077e-07, 6.0623e-07, 2.3406e-07,\n", - " -2.1303e-07, -5.7148e-08, 7.9081e-08, 1.5781e-07, 1.1813e-07,\n", - " -3.1826e-08, -5.1946e-07, -4.5346e-07, -2.7249e-07, -2.1805e-07,\n", - " -6.7970e-07, 1.0566e-07, 9.6752e-08, -7.1769e-07, 4.2578e-08,\n", - " -3.0736e-07, 6.7938e-07, -2.6022e-07, 1.7747e-07, -7.5835e-08,\n", - " 2.4098e-07, -2.0173e-07, -4.1996e-07, -1.0586e-07, -1.1475e-07,\n", - " 2.8858e-07, 2.3780e-07, -2.6370e-07, -2.1977e-07, -2.3373e-07,\n", - " -3.5201e-07, 8.5626e-08, -2.5834e-07, 5.5657e-07, -6.2127e-07,\n", - " 1.9869e-08, -8.6794e-07, 1.0717e-07, -2.8302e-07, -5.1231e-07,\n", - " 3.6485e-07, -5.1699e-07, -2.2456e-07, 5.7222e-07, 1.9374e-08,\n", - " 8.9344e-08, -3.1100e-08, -7.3690e-07, -1.1705e-07, -1.8583e-08,\n", - " -2.2865e-07, -2.9119e-07, 8.9379e-08, 7.7748e-09, 1.1452e-07,\n", - " 9.1766e-07, -2.4583e-07, 1.1647e-07, -1.7874e-07, 3.4206e-07,\n", - " 4.5904e-07, 1.5186e-07, 2.3652e-07, 4.3211e-07, 4.8214e-07,\n", - " -5.7052e-07, -1.2741e-06, 5.1995e-07, 1.3815e-07, 1.9537e-07,\n", - " 4.0341e-07, 2.6584e-07, -1.6991e-07, 5.8831e-07, -2.2692e-07,\n", - " -3.5686e-07, 2.1014e-09, -2.0653e-07, 2.9375e-07, -3.1663e-07,\n", - " 8.5626e-08, -3.6269e-08, 3.6752e-07, 8.9283e-08, 3.4088e-07,\n", - " -1.3519e-07, -2.0703e-07], device='cuda:0')),\n", - " ('layer4.0.conv2.weight',\n", - " tensor([[[[ 1.3984e-07, -1.1567e-06, -2.1700e-07],\n", - " [-1.3681e-06, -1.4883e-06, -6.7987e-08],\n", - " [-1.6829e-06, -1.1120e-06, -8.1770e-07]],\n", - " \n", - " [[-1.5032e-06, -1.1045e-06, -9.5926e-07],\n", - " [-1.5246e-06, -2.1507e-06, -3.9116e-07],\n", - " [-6.9663e-07, -1.2419e-06, 2.8848e-07]],\n", - " \n", - " [[ 1.6647e-07, -1.4069e-06, -1.4831e-06],\n", - " [-6.8080e-07, -1.2489e-06, -1.8366e-06],\n", - " [-9.1299e-07, -1.2685e-06, -9.4716e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.6566e-08, -6.4261e-08, -5.6297e-07],\n", - " [-1.3597e-07, -1.3390e-06, -4.7707e-07],\n", - " [-7.4779e-07, -7.2876e-08, -5.7986e-07]],\n", - " \n", - " [[-1.0699e-06, -9.5135e-07, -4.7195e-07],\n", - " [-2.4019e-06, -2.0582e-06, -1.5553e-06],\n", - " [-1.3486e-06, -1.3709e-06, -7.2643e-08]],\n", - " \n", - " [[-3.9426e-07, -1.1846e-06, -1.9018e-06],\n", - " [-1.9325e-06, -9.3691e-07, -1.7730e-06],\n", - " [-7.6089e-07, -2.7004e-06, -2.6114e-06]]],\n", - " \n", - " \n", - " [[[-1.3763e-07, -3.3714e-07, 1.3958e-07],\n", - " [-6.2166e-07, 3.6554e-07, -8.5495e-07],\n", - " [-2.8126e-07, -7.5158e-07, -2.6263e-07]],\n", - " \n", - " [[-3.7346e-07, 2.5146e-07, 1.4249e-07],\n", - " [-6.5344e-07, -2.7299e-07, -2.2675e-07],\n", - " [-3.5390e-08, -6.1002e-07, 2.4354e-07]],\n", - " \n", - " [[-4.4703e-08, -8.4774e-07, -6.9477e-07],\n", - " [-7.2829e-07, -2.6822e-07, -2.1579e-06],\n", - " [-7.9675e-07, -2.4051e-07, -7.5530e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-8.7544e-08, 7.0594e-07, 8.0094e-08],\n", - " [-2.0070e-07, -2.1327e-07, -1.6019e-07],\n", - " [-2.9360e-07, 3.0780e-07, -4.1910e-08]],\n", - " \n", - " [[-1.8300e-07, -1.2405e-06, -5.7300e-07],\n", - " [-1.1781e-06, -9.0618e-07, -1.2463e-06],\n", - " [ 1.6764e-08, -6.0920e-07, -4.1910e-08]],\n", - " \n", - " [[ 7.1153e-07, 3.0361e-07, 2.9802e-08],\n", - " [ 1.2715e-06, 8.6147e-07, -2.9476e-07],\n", - " [ 3.6135e-07, 6.3051e-07, 8.8010e-08]]],\n", - " \n", - " \n", - " [[[ 3.2212e-07, 6.5286e-07, 1.6717e-06],\n", - " [-6.6590e-08, 1.2433e-06, 2.1770e-06],\n", - " [ 1.0803e-07, 5.1223e-08, 4.3865e-07]],\n", - " \n", - " [[ 8.2981e-07, 1.8002e-06, 1.7667e-06],\n", - " [ 5.6811e-07, 2.3339e-06, 1.7919e-06],\n", - " [ 6.3423e-07, 1.5423e-06, 1.7118e-06]],\n", - " \n", - " [[ 8.5682e-08, 3.7905e-07, 1.9318e-06],\n", - " [-1.4342e-07, 2.6310e-07, 1.1101e-06],\n", - " [ 1.3877e-07, 3.7160e-07, 1.1367e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.4983e-07, 7.7952e-07, 7.0682e-07],\n", - " [ 6.9384e-07, -9.5111e-08, 1.4193e-06],\n", - " [ 2.0489e-07, 5.7649e-07, 1.6885e-06]],\n", - " \n", - " [[ 1.4435e-08, -6.0536e-08, 9.9838e-07],\n", - " [-3.5390e-07, 5.4203e-07, 2.4368e-06],\n", - " [-4.5355e-07, -7.3016e-07, 9.5088e-07]],\n", - " \n", - " [[ 4.6380e-07, 5.5414e-07, 1.8190e-06],\n", - " [-1.2852e-07, 1.0037e-06, 1.9865e-06],\n", - " [ 1.8626e-09, -2.7567e-07, 1.2647e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 1.0058e-07, -1.0545e-06, -6.7614e-07],\n", - " [-4.5262e-07, -2.2724e-07, -1.1139e-06],\n", - " [ 6.3679e-08, -1.6205e-06, -9.4995e-07]],\n", - " \n", - " [[-9.8022e-07, -2.5099e-06, -2.1181e-06],\n", - " [-2.1691e-06, -1.7202e-06, -2.0822e-06],\n", - " [-1.3243e-06, -2.5127e-06, -1.0971e-06]],\n", - " \n", - " [[-2.1700e-07, -6.1840e-07, -1.1986e-06],\n", - " [-5.6997e-07, -9.7509e-07, -2.4782e-06],\n", - " [-5.9139e-08, -5.2061e-07, -1.9332e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 9.7789e-08, 6.8653e-07, -6.8452e-07],\n", - " [ 4.1444e-07, -7.3109e-07, -1.1846e-06],\n", - " [-1.5926e-07, -3.3248e-07, -5.9791e-07]],\n", - " \n", - " [[ 6.1467e-08, -2.6060e-07, -1.9872e-06],\n", - " [-5.6240e-07, -3.2410e-07, -3.6461e-07],\n", - " [ 1.1006e-06, -1.1129e-06, -1.2780e-06]],\n", - " \n", - " [[ 6.1747e-07, 6.4075e-07, 1.0617e-07],\n", - " [ 6.3889e-07, 1.1101e-06, -1.1446e-06],\n", - " [ 1.5320e-07, -1.1092e-06, -3.2084e-07]]],\n", - " \n", - " \n", - " [[[ 7.3202e-07, 3.2663e-07, 3.1479e-07],\n", - " [ 1.9558e-06, 1.3807e-06, 1.1679e-06],\n", - " [ 1.5089e-06, 1.4845e-06, 1.2149e-06]],\n", - " \n", - " [[ 1.7742e-07, 5.4538e-07, 7.8883e-07],\n", - " [ 7.1526e-07, 1.1381e-06, 1.5292e-06],\n", - " [ 3.4086e-07, -3.9069e-07, 6.4634e-07]],\n", - " \n", - " [[ 7.3458e-07, 1.7518e-06, 6.3237e-07],\n", - " [ 1.4721e-06, 1.3504e-06, 1.3513e-06],\n", - " [ 8.4005e-07, 1.2638e-06, 2.5239e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.9267e-07, 1.0887e-06, 4.6100e-07],\n", - " [ 1.4110e-07, 1.1190e-06, 7.4157e-08],\n", - " [ 8.6753e-07, 8.3540e-07, -8.5216e-07]],\n", - " \n", - " [[ 1.0123e-06, 1.5162e-06, 1.6415e-07],\n", - " [ 6.9849e-07, 1.6196e-06, 2.5891e-07],\n", - " [ 6.0536e-07, 3.2270e-07, 2.2491e-07]],\n", - " \n", - " [[ 2.1888e-07, -1.9744e-07, -6.3609e-07],\n", - " [ 2.0117e-07, 6.2725e-07, -1.2498e-06],\n", - " [-2.5798e-07, 5.3179e-07, 1.2666e-07]]],\n", - " \n", - " \n", - " [[[ 1.1087e-06, 1.0915e-06, -8.3074e-07],\n", - " [ 2.1579e-06, 1.8310e-06, 9.8546e-08],\n", - " [ 2.5469e-06, 1.4603e-06, 7.7952e-07]],\n", - " \n", - " [[-3.2037e-07, -1.3858e-06, -1.2787e-06],\n", - " [-3.6601e-07, -5.2433e-07, -5.4832e-07],\n", - " [-6.8836e-07, -4.7521e-07, -5.7090e-07]],\n", - " \n", - " [[-3.7812e-07, 4.7684e-07, -4.1910e-07],\n", - " [ 2.5425e-07, 8.4890e-07, 1.3979e-06],\n", - " [ 7.3342e-07, 7.9628e-07, 6.0722e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.5050e-06, 2.3693e-06, 9.5088e-07],\n", - " [ 1.9956e-06, 1.7863e-06, 1.3666e-06],\n", - " [ 8.2050e-07, 7.5577e-07, 9.3877e-07]],\n", - " \n", - " [[ 8.1025e-08, 4.7173e-07, 3.0315e-07],\n", - " [ 5.5385e-07, 5.8487e-07, 7.5344e-07],\n", - " [ 1.7136e-07, 4.1816e-07, -2.8126e-07]],\n", - " \n", - " [[ 1.1921e-07, 1.9243e-07, 1.1115e-06],\n", - " [-1.2410e-07, 2.0694e-06, 1.9688e-06],\n", - " [ 2.0768e-07, 1.0170e-06, 1.2182e-06]]]], device='cuda:0')),\n", - " ('layer4.0.bn2.weight',\n", - " tensor([ 1.1921e-07, 2.3842e-07, -2.9802e-07, -2.9802e-07, 9.5367e-07,\n", - " -1.1921e-07, 5.9605e-07, 2.3842e-07, 1.1921e-07, 2.3842e-07,\n", - " 4.7684e-07, -5.3644e-07, -4.1723e-07, -4.1723e-07, 2.3842e-07,\n", - " -6.5565e-07, 1.1921e-07, 0.0000e+00, -2.9802e-07, 2.3842e-07,\n", - " -5.9605e-08, -1.6093e-06, 2.3842e-07, -3.5763e-07, 1.1921e-07,\n", - " -7.7486e-07, -3.5763e-07, -1.1921e-07, 1.1921e-07, -5.3644e-07,\n", - " -7.1526e-07, -1.0729e-06, -5.3644e-07, -2.3842e-07, -4.7684e-07,\n", - " -2.9802e-07, 5.9605e-07, 8.3447e-07, 2.3842e-07, -2.9802e-07,\n", - " 1.1921e-07, 3.5763e-07, 3.5763e-07, -2.3842e-07, -2.3842e-07,\n", - " 3.5763e-07, 7.1526e-07, -1.7881e-07, -4.7684e-07, 5.9605e-07,\n", - " -5.9605e-08, 3.5763e-07, -2.9802e-07, -8.9407e-07, 2.3842e-07,\n", - " 3.5763e-07, -1.1921e-06, -5.9605e-08, -8.9407e-07, 1.1921e-07,\n", - " 0.0000e+00, -2.9802e-07, 2.3842e-07, 1.1921e-07, -6.5565e-07,\n", - " 1.1921e-07, 5.9605e-07, 0.0000e+00, -5.9605e-07, -5.9605e-08,\n", - " 8.3447e-07, 5.9605e-07, 3.5763e-07, -2.3842e-07, 9.5367e-07,\n", - " 5.9605e-07, -3.5763e-07, 1.1921e-07, 2.3842e-07, 1.1921e-07,\n", - " -1.3709e-06, 8.3447e-07, -4.1723e-07, -1.4305e-06, -3.5763e-07,\n", - " -9.5367e-07, 1.1921e-07, -5.9605e-08, 7.1526e-07, -7.7486e-07,\n", - " -2.3842e-07, -2.9802e-07, 7.1526e-07, -5.3644e-07, 0.0000e+00,\n", - " 1.1921e-07, 9.5367e-07, 1.1921e-06, 5.9605e-07, 0.0000e+00,\n", - " -7.1526e-07, 4.7684e-07, -2.3842e-07, -1.1921e-06, 4.7684e-07,\n", - " 5.9605e-07, -5.9605e-08, 3.5763e-07, -1.7881e-07, 2.3842e-07,\n", - " 1.1921e-07, -5.9605e-08, 1.1921e-07, -4.7684e-07, 7.1526e-07,\n", - " 4.7684e-07, 8.3447e-07, -5.9605e-08, -3.5763e-07, 8.3447e-07,\n", - " 4.7684e-07, -1.7881e-07, 4.7684e-07, -3.5763e-07, 4.7684e-07,\n", - " 4.7684e-07, -5.3644e-07, -2.9802e-07, -7.1526e-07, -3.5763e-07,\n", - " -1.1921e-07, -2.9802e-07, 4.7684e-07, -1.0133e-06, -2.3842e-07,\n", - " 2.3842e-07, 0.0000e+00, 5.9605e-07, -9.5367e-07, -1.1921e-07,\n", - " -3.5763e-07, -7.7486e-07, -3.5763e-07, -7.7486e-07, 8.3447e-07,\n", - " -4.7684e-07, -2.3842e-07, 0.0000e+00, -8.9407e-07, 4.7684e-07,\n", - " -2.9802e-07, 7.1526e-07, 3.5763e-07, -5.9605e-08, -7.1526e-07,\n", - " 0.0000e+00, -5.9605e-08, 2.3842e-07, -5.9605e-08, -1.1325e-06,\n", - " -4.1723e-07, -1.0133e-06, -7.1526e-07, -1.7881e-07, -5.9605e-08,\n", - " -1.7881e-07, -5.9605e-07, -2.3842e-07, -1.7881e-07, -4.1723e-07,\n", - " 1.0729e-06, 3.5763e-07, 1.1921e-07, 1.1921e-07, -2.3842e-07,\n", - " 5.9605e-07, -2.9802e-07, 8.3447e-07, 4.7684e-07, 0.0000e+00,\n", - " -6.5565e-07, 5.9605e-07, -3.5763e-07, 5.9605e-07, -7.7486e-07,\n", - " 1.1921e-07, -3.5763e-07, -1.7881e-07, 4.7684e-07, 3.5763e-07,\n", - " -5.9605e-08, 0.0000e+00, 4.7684e-07, -1.7881e-07, -9.5367e-07,\n", - " 4.7684e-07, 1.1921e-07, -5.3644e-07, -1.7881e-07, -3.5763e-07,\n", - " 1.1921e-07, 3.5763e-07, 7.1526e-07, -1.1921e-07, -2.3842e-07,\n", - " -5.9605e-07, -4.7684e-07, 4.7684e-07, 1.5497e-06, 0.0000e+00,\n", - " 3.5763e-07, -5.9605e-07, -1.1921e-07, 2.3842e-07, -1.0133e-06,\n", - " -1.7881e-07, -7.1526e-07, -4.7684e-07, 1.1921e-07, 2.3842e-07,\n", - " 1.1921e-07, 2.3842e-07, -1.0133e-06, 4.7684e-07, 5.9605e-07,\n", - " 5.9605e-07, 4.7684e-07, 2.3842e-07, 2.3842e-07, 7.1526e-07,\n", - " 3.5763e-07, 1.0729e-06, -4.1723e-07, 2.3842e-07, 0.0000e+00,\n", - " 2.3842e-07, 2.3842e-07, -2.3842e-07, -1.1921e-07, -1.0133e-06,\n", - " -5.3644e-07, 2.3842e-07, 5.9605e-07, -2.3842e-07, -6.5565e-07,\n", - " -5.9605e-08, 1.1921e-07, 2.3842e-07, 1.1921e-07, 1.1921e-06,\n", - " -1.1921e-06, -5.9605e-07, 3.5763e-07, -1.1921e-07, 1.1921e-07,\n", - " 2.3842e-07, 2.3842e-07, 1.0729e-06, 3.5763e-07, 5.9605e-07,\n", - " 2.3842e-07, -5.9605e-08, -4.1723e-07, -1.1921e-06, -1.1325e-06,\n", - " -4.1723e-07, -2.9802e-07, 8.3447e-07, 2.3842e-07, 2.3842e-07,\n", - " 0.0000e+00, -2.3842e-07, 1.1921e-07, -4.1723e-07, 5.9605e-07,\n", - " 0.0000e+00, -9.5367e-07, 0.0000e+00, -2.9802e-07, 4.7684e-07,\n", - " 1.0729e-06, -2.3842e-07, 7.1526e-07, -2.9802e-07, -4.1723e-07,\n", - " 0.0000e+00, -3.5763e-07, 3.5763e-07, -8.9407e-07, 2.3842e-07,\n", - " -1.6689e-06, 3.5763e-07, 0.0000e+00, 2.3842e-07, -5.9605e-07,\n", - " 2.3842e-07, -5.9605e-08, -4.1723e-07, -1.7881e-07, 9.5367e-07,\n", - " -8.3447e-07, 0.0000e+00, -1.0133e-06, 4.7684e-07, -6.5565e-07,\n", - " -2.9802e-07, -3.5763e-07, 2.3842e-07, -5.9605e-07, -4.1723e-07,\n", - " -1.9073e-06, -7.1526e-07, 1.1921e-07, 2.3842e-07, -4.1723e-07,\n", - " 1.1921e-07, -5.9605e-08, -5.9605e-07, -4.1723e-07, 2.3842e-07,\n", - " 1.1921e-07, -5.9605e-08, 3.5763e-07, 4.7684e-07, -4.7684e-07,\n", - " 9.5367e-07, -5.3644e-07, 5.9605e-07, 3.5763e-07, -8.9407e-07,\n", - " -1.1921e-07, -5.9605e-08, 2.3842e-07, 8.3447e-07, 1.1921e-07,\n", - " -7.1526e-07, -5.9605e-08, 2.3842e-07, 4.7684e-07, 0.0000e+00,\n", - " 4.7684e-07, 2.3842e-07, -2.3842e-07, -3.5763e-07, -1.1921e-07,\n", - " -2.9802e-07, 5.9605e-07, 1.1921e-07, 0.0000e+00, -1.1921e-07,\n", - " -5.3644e-07, -4.1723e-07, 1.1921e-07, 3.5763e-07, -3.5763e-07,\n", - " -5.3644e-07, -4.1723e-07, -6.5565e-07, 2.3842e-07, -2.3842e-07,\n", - " -1.7881e-07, -2.3842e-07, -1.7881e-07, -2.3842e-07, 5.9605e-07,\n", - " -1.7881e-07, -2.9802e-07, 8.3447e-07, -6.5565e-07, -1.7881e-07,\n", - " 1.1921e-07, -2.3842e-07, -7.1526e-07, -4.1723e-07, 4.7684e-07,\n", - " 0.0000e+00, -4.1723e-07, -7.1526e-07, 3.5763e-07, -9.5367e-07,\n", - " 2.3842e-07, 1.1921e-07, 1.1921e-07, 4.7684e-07, 1.1921e-07,\n", - " 0.0000e+00, 4.7684e-07, 0.0000e+00, -4.1723e-07, 7.1526e-07,\n", - " 8.3447e-07, -4.1723e-07, -2.9802e-07, 3.5763e-07, 0.0000e+00,\n", - " -9.5367e-07, -2.3842e-07, -2.9802e-07, -7.7486e-07, -1.7881e-07,\n", - " -7.1526e-07, -3.5763e-07, 3.5763e-07, -2.3842e-07, -4.1723e-07,\n", - " 1.1921e-07, 2.3842e-07, -4.1723e-07, -7.1526e-07, 1.1921e-07,\n", - " -7.7486e-07, 3.5763e-07, -1.1921e-07, -5.9605e-08, -8.3447e-07,\n", - " -2.3842e-07, -1.1921e-07, 5.9605e-07, -5.3644e-07, -5.9605e-07,\n", - " -4.1723e-07, -5.9605e-08, 5.9605e-07, 0.0000e+00, 4.7684e-07,\n", - " -2.9802e-07, 4.7684e-07, -3.5763e-07, 0.0000e+00, -6.5565e-07,\n", - " -3.5763e-07, -3.5763e-07, 3.5763e-07, 2.3842e-07, -4.7684e-07,\n", - " -1.7881e-07, 3.5763e-07, 2.3842e-07, -4.1723e-07, -2.3842e-07,\n", - " 8.3447e-07, 2.3842e-07, 0.0000e+00, -2.9802e-07, -2.3842e-07,\n", - " -1.4305e-06, 5.9605e-07, 0.0000e+00, -4.7684e-07, 0.0000e+00,\n", - " 1.1921e-07, 1.1921e-07, 9.5367e-07, -7.1526e-07, 1.0729e-06,\n", - " -2.3842e-07, 1.1921e-07, 5.9605e-07, 2.3842e-07, 1.1921e-07,\n", - " -1.1921e-07, -1.0729e-06, -1.1921e-07, 5.9605e-07, -5.9605e-07,\n", - " -4.7684e-07, 1.1921e-07, 0.0000e+00, -1.0133e-06, 7.1526e-07,\n", - " -6.5565e-07, -1.2517e-06, 3.5763e-07, 1.1921e-07, 1.0729e-06,\n", - " 2.3842e-07, -7.7486e-07, 4.7684e-07, -5.9605e-07, -1.1921e-07,\n", - " -5.9605e-08, -8.9407e-07, -5.9605e-08, 1.1921e-07, -9.5367e-07,\n", - " -5.9605e-08, -5.9605e-07, -1.7881e-07, 4.7684e-07, -2.3842e-07,\n", - " -4.7684e-07, 2.3842e-07, -1.0133e-06, 7.1526e-07, -2.3842e-07,\n", - " -5.9605e-07, -4.1723e-07, 2.3842e-07, -5.9605e-08, 2.3842e-07,\n", - " -3.5763e-07, 1.1921e-07, -1.7881e-07, -8.9407e-07, -1.3709e-06,\n", - " -4.1723e-07, -5.9605e-08, 3.5763e-07, 0.0000e+00, 2.3842e-07,\n", - " 3.5763e-07, -5.9605e-08], device='cuda:0')),\n", - " ('layer4.0.bn2.bias',\n", - " tensor([ 1.2267e-07, 5.2131e-07, 4.9048e-07, -6.6727e-08, 5.8954e-07,\n", - " 2.0044e-07, 3.3354e-07, 1.0084e-07, 3.4111e-07, 1.0555e-07,\n", - " 4.8835e-07, -1.1029e-06, 2.0666e-08, 2.0148e-07, 2.1696e-07,\n", - " -7.4097e-07, 3.7880e-08, 6.4895e-08, -8.4100e-07, -3.1968e-07,\n", - " 1.0362e-07, -9.7019e-07, -3.3352e-07, -4.1362e-07, 4.1598e-08,\n", - " 2.2223e-07, 1.6852e-07, -1.0521e-07, 2.1346e-07, -1.2921e-06,\n", - " 3.7748e-07, -5.9054e-07, -1.5242e-08, -5.5715e-07, -2.1764e-07,\n", - " 2.1038e-07, 7.7662e-07, 9.8997e-07, 2.4168e-07, 8.6350e-08,\n", - " 1.8026e-07, 9.3270e-08, 2.4967e-07, -5.6322e-07, -3.2481e-07,\n", - " 5.5609e-09, 2.3633e-07, -4.5999e-07, 6.0220e-08, 6.4592e-07,\n", - " 2.2764e-07, -1.9812e-07, -9.3918e-07, -3.2255e-07, 2.1820e-07,\n", - " 1.5435e-06, -1.6022e-06, 9.6016e-08, -2.3592e-07, 7.2372e-07,\n", - " -4.9385e-07, 1.8043e-07, -4.0969e-07, 2.5857e-07, -5.5846e-07,\n", - " 3.3289e-07, 3.6472e-07, -9.9066e-07, -6.4836e-07, -5.9628e-08,\n", - " 7.9305e-07, 5.1530e-07, 3.1524e-07, -3.6323e-07, 3.4015e-07,\n", - " 1.8533e-07, -2.7069e-07, 2.8576e-07, -5.2353e-07, -1.3846e-07,\n", - " -1.7000e-06, 3.0305e-07, -8.1954e-07, -9.5841e-07, 3.0598e-07,\n", - " -8.9424e-07, 2.0526e-07, -5.2114e-07, 3.8126e-07, -8.6022e-07,\n", - " 9.4818e-07, -1.4355e-07, 2.4742e-07, -4.0967e-07, 1.2586e-07,\n", - " -9.7055e-07, 1.4553e-06, 1.1442e-07, 8.6497e-07, 1.3247e-07,\n", - " -5.0140e-07, -2.5456e-07, -2.6590e-08, -3.5772e-07, 8.2593e-07,\n", - " 5.3295e-07, -1.0410e-07, 4.3498e-07, 2.3294e-07, 4.7580e-07,\n", - " -5.4222e-07, 4.6807e-07, 3.8665e-07, -5.5240e-07, 4.0680e-07,\n", - " 4.8383e-07, -3.1379e-07, 2.7812e-07, -8.6844e-07, -4.2003e-08,\n", - " 3.9490e-07, -2.1611e-07, 4.1244e-08, -7.7546e-07, -2.5390e-07,\n", - " 7.5168e-07, -1.0881e-06, -3.0041e-09, -1.2022e-06, -1.0697e-06,\n", - " 2.1438e-07, -2.7698e-07, 1.0052e-07, -2.8831e-07, -7.9248e-07,\n", - " -1.1695e-08, 3.7242e-07, 3.6402e-07, -4.4123e-07, 6.7511e-08,\n", - " 6.2402e-07, -7.6874e-07, -7.7209e-07, -4.6478e-07, 8.0412e-07,\n", - " -1.1367e-06, 7.2712e-07, -1.3130e-07, -5.0524e-07, 2.0572e-07,\n", - " -4.2604e-07, 1.5546e-07, 9.0354e-07, -5.7759e-07, -6.8648e-07,\n", - " 3.3436e-07, 4.4954e-07, 3.9408e-07, -6.5580e-07, -9.3324e-07,\n", - " -6.8870e-07, -6.4934e-07, -1.5007e-07, -5.6741e-08, -9.6720e-07,\n", - " 4.3615e-07, -1.5925e-07, -8.6850e-07, 4.7006e-07, -6.9503e-07,\n", - " 8.5696e-07, 4.0076e-07, 5.0084e-07, 3.3838e-07, 1.2934e-07,\n", - " 1.0382e-06, -4.2320e-07, 3.1105e-07, 9.0592e-07, 5.3915e-07,\n", - " -2.8505e-07, 3.0121e-07, -3.5367e-07, 3.1758e-07, -9.3912e-07,\n", - " 7.7104e-07, 4.0988e-07, 4.6698e-08, 3.6261e-07, -2.4688e-07,\n", - " -5.6081e-07, 5.2616e-07, 3.9207e-07, 3.3400e-07, 1.3801e-07,\n", - " 1.5755e-07, 1.3671e-07, -2.9247e-07, -3.3614e-08, 2.7387e-07,\n", - " 4.1058e-08, 1.0102e-07, 5.9427e-07, 6.9601e-07, 5.1401e-07,\n", - " -9.0142e-07, -7.8599e-07, 8.5890e-07, 8.0110e-07, -2.6069e-08,\n", - " -4.7613e-07, -1.1880e-07, 6.0977e-07, -3.2545e-08, -8.0672e-07,\n", - " -4.4460e-07, 4.9622e-08, -1.0001e-06, -3.8140e-07, 2.0876e-07,\n", - " -7.7196e-07, 7.1734e-07, -4.6298e-07, 6.7959e-07, 1.0245e-06,\n", - " 4.1416e-07, 5.4496e-07, 7.8721e-07, -4.2891e-07, 1.6825e-06,\n", - " 1.4942e-07, -1.6828e-07, 1.7415e-07, 4.0402e-08, -5.3157e-07,\n", - " 1.0582e-07, -2.0883e-07, 4.7203e-07, -1.3684e-07, -1.3398e-06,\n", - " -5.9472e-07, 3.9554e-07, 2.3206e-07, -2.5863e-07, -4.5267e-07,\n", - " 1.0301e-06, -5.3752e-07, -7.6298e-08, -5.4183e-07, 2.8119e-07,\n", - " 7.9946e-08, -1.5766e-07, 1.8332e-07, -8.2711e-07, -3.7360e-07,\n", - " -1.5190e-07, -3.5914e-08, 3.6115e-07, -1.4778e-07, 8.6465e-07,\n", - " -2.9969e-08, 2.9304e-07, 2.5676e-07, -2.8295e-07, -8.0901e-07,\n", - " -3.3888e-07, -1.1907e-07, 2.6471e-07, 1.6580e-07, -6.6758e-08,\n", - " 2.1079e-07, 3.7526e-07, 3.1104e-07, 8.7153e-07, 3.5282e-07,\n", - " 4.8555e-07, -2.3091e-07, -5.6911e-07, 1.3365e-07, 3.0886e-07,\n", - " 2.5100e-07, -2.3340e-07, 4.3298e-07, -3.9339e-07, -6.6168e-07,\n", - " 2.4621e-07, 1.2055e-07, 5.7209e-08, -5.5472e-07, 8.4360e-07,\n", - " -8.6571e-07, 1.9837e-07, 5.0580e-08, 4.3711e-07, 4.6089e-07,\n", - " 1.3192e-07, 1.8432e-08, -4.0834e-07, 2.2995e-07, 9.0939e-07,\n", - " -6.0561e-07, 4.8501e-07, -6.9802e-07, 4.8374e-07, -3.7478e-07,\n", - " -3.0589e-07, -2.6858e-07, -4.6554e-08, -1.0507e-07, -2.8908e-07,\n", - " -1.0337e-06, -1.0129e-06, 2.2487e-07, 1.7116e-07, -4.1344e-07,\n", - " -4.2085e-07, -3.7954e-08, -5.6785e-07, -5.8464e-07, 1.2217e-08,\n", - " -9.5578e-08, -2.8373e-07, 2.6618e-07, 7.6734e-07, -8.6086e-07,\n", - " 3.6636e-07, -9.9598e-07, -1.5478e-07, 1.7445e-07, -5.5791e-07,\n", - " -7.7129e-07, -2.5449e-07, 2.9259e-07, 2.6276e-07, -7.1697e-08,\n", - " -5.8335e-07, 1.1469e-08, 2.1788e-07, 1.0519e-06, 5.5815e-08,\n", - " 4.8869e-07, -6.1105e-08, 2.8824e-08, 5.1238e-07, -2.2381e-08,\n", - " -1.9842e-07, 1.5346e-07, 2.2163e-07, -3.7323e-07, 2.4546e-07,\n", - " -5.5839e-07, -1.2237e-06, 7.6679e-07, 1.1058e-06, -6.1683e-07,\n", - " -5.6177e-07, -4.2822e-07, -2.7411e-07, 2.8429e-09, -1.7565e-07,\n", - " -6.2889e-07, -5.5862e-07, -5.8493e-07, -4.7978e-07, 6.6172e-07,\n", - " 6.3017e-08, -5.3413e-07, 8.2590e-07, -2.3073e-07, -2.7192e-07,\n", - " 5.1345e-08, -4.7095e-07, -7.3116e-07, -7.4617e-07, 1.2349e-06,\n", - " -5.5501e-08, -3.8916e-07, 4.9881e-07, -5.8128e-07, -1.0835e-06,\n", - " 8.9368e-08, -2.4767e-08, -1.7907e-07, 7.5381e-07, -9.3912e-07,\n", - " 1.6275e-07, 8.3961e-07, 1.9004e-07, -6.1507e-07, 1.7484e-07,\n", - " 6.1234e-07, -3.3452e-07, 1.7429e-07, 6.1249e-07, -4.0872e-07,\n", - " -8.5998e-07, -2.6819e-08, 4.6977e-07, -4.8661e-07, -3.7181e-07,\n", - " -4.3231e-07, -4.4359e-07, 1.0288e-07, -4.6875e-07, 1.2184e-07,\n", - " 5.4193e-07, 1.6957e-07, 1.2930e-08, -8.3377e-07, 2.0160e-07,\n", - " -1.1283e-06, 4.6074e-08, 7.9631e-08, -1.9956e-07, -2.8262e-07,\n", - " -2.7352e-07, -3.9198e-07, 9.0809e-07, -6.9796e-07, -7.7386e-07,\n", - " -6.3717e-07, 1.9364e-07, 4.2499e-07, -2.1963e-07, 3.3570e-07,\n", - " -1.1842e-06, -1.3149e-07, -2.0294e-07, -8.4020e-08, -1.8796e-07,\n", - " -8.5904e-08, 1.0188e-06, 1.1152e-07, -7.9846e-07, -1.3746e-07,\n", - " 3.0789e-07, 1.1175e-07, -3.5673e-07, -3.4911e-08, -2.8830e-08,\n", - " -2.7811e-07, 6.8873e-08, 6.6728e-07, -2.9486e-07, 6.0918e-07,\n", - " -1.2829e-06, 6.8836e-07, 3.0841e-07, 4.4511e-07, 3.0244e-07,\n", - " 4.3634e-07, 3.6050e-08, 5.0360e-07, -5.9110e-07, 9.7646e-07,\n", - " -9.2563e-07, -4.3873e-07, 1.1422e-07, -1.3625e-07, -6.4917e-07,\n", - " 5.3954e-08, -1.5143e-06, -5.9071e-07, -2.4121e-08, -1.0838e-06,\n", - " -4.0250e-07, 5.5546e-07, -7.1030e-07, -1.0235e-06, -9.1596e-08,\n", - " 1.7611e-08, -1.6254e-06, 5.3340e-07, 5.0056e-07, 2.9434e-07,\n", - " 3.0389e-07, -6.4017e-07, 3.4055e-07, -2.9545e-07, 1.7299e-07,\n", - " -3.5783e-08, 3.9323e-07, 1.4520e-08, 2.4158e-09, -8.8659e-07,\n", - " -2.5928e-07, -4.6700e-07, -4.3284e-07, 7.8656e-08, 2.4758e-07,\n", - " 7.3693e-08, 2.9827e-08, -5.8056e-07, -9.8295e-08, -2.7469e-08,\n", - " -6.7635e-07, 5.9764e-08, 9.8274e-08, 1.2037e-08, -3.1155e-07,\n", - " -2.1793e-07, 6.5858e-08, 4.3835e-08, 1.8843e-07, 1.4375e-07,\n", - " 7.0740e-08, 9.6125e-08, 1.4013e-07, 6.2235e-07, 1.1469e-06,\n", - " -4.6511e-09, 2.3411e-07], device='cuda:0')),\n", - " ('layer4.0.shortcut.0.weight',\n", - " tensor([[[[ 3.1851e-07]],\n", - " \n", - " [[ 3.7812e-07]],\n", - " \n", - " [[ 1.4585e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-5.5134e-07]],\n", - " \n", - " [[-1.2666e-07]],\n", - " \n", - " [[ 4.6939e-07]]],\n", - " \n", - " \n", - " [[[-6.9104e-07]],\n", - " \n", - " [[-5.1036e-07]],\n", - " \n", - " [[ 9.2760e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.1665e-08]],\n", - " \n", - " [[ 3.7067e-07]],\n", - " \n", - " [[ 1.5087e-07]]],\n", - " \n", - " \n", - " [[[ 9.3132e-09]],\n", - " \n", - " [[ 5.0245e-07]],\n", - " \n", - " [[-5.2154e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-8.9407e-08]],\n", - " \n", - " [[ 4.8429e-07]],\n", - " \n", - " [[-9.7920e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-1.2740e-06]],\n", - " \n", - " [[-1.5795e-06]],\n", - " \n", - " [[ 1.6820e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.7998e-07]],\n", - " \n", - " [[-4.6939e-07]],\n", - " \n", - " [[ 2.9681e-06]]],\n", - " \n", - " \n", - " [[[ 7.5996e-07]],\n", - " \n", - " [[ 1.3264e-06]],\n", - " \n", - " [[-4.4098e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-7.2643e-08]],\n", - " \n", - " [[ 1.1958e-06]],\n", - " \n", - " [[ 1.5832e-07]]],\n", - " \n", - " \n", - " [[[-6.0722e-07]],\n", - " \n", - " [[-9.4622e-07]],\n", - " \n", - " [[ 8.3633e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-9.1642e-07]],\n", - " \n", - " [[ 1.2033e-06]],\n", - " \n", - " [[ 8.2329e-07]]]], device='cuda:0')),\n", - " ('layer4.0.shortcut.1.weight',\n", - " tensor([-4.1723e-07, 1.1921e-07, 7.1526e-07, 3.5763e-07, -2.9802e-07,\n", - " 2.3842e-07, -2.3842e-07, -1.1921e-07, 2.3842e-07, -5.9605e-08,\n", - " -1.1921e-07, -7.1526e-07, 2.3842e-07, 5.9605e-07, 1.1921e-07,\n", - " 1.1921e-07, 0.0000e+00, -2.3842e-07, -8.3447e-07, -1.1325e-06,\n", - " -2.3842e-07, -7.7486e-07, -1.7881e-07, 2.3842e-07, 0.0000e+00,\n", - " 7.1526e-07, 4.7684e-07, 0.0000e+00, 4.7684e-07, -1.0729e-06,\n", - " 2.3842e-07, 1.1921e-07, 3.5763e-07, -1.2517e-06, -5.9605e-08,\n", - " 8.3447e-07, 1.5497e-06, 3.5763e-07, 3.5763e-07, 1.1921e-07,\n", - " 1.1921e-07, -3.5763e-07, -1.1921e-07, -9.5367e-07, -5.3644e-07,\n", - " -5.3644e-07, -8.3447e-07, -9.5367e-07, 2.3842e-07, 0.0000e+00,\n", - " 1.1921e-07, -1.3709e-06, -3.5763e-07, -4.1723e-07, 8.3447e-07,\n", - " 1.7881e-06, -2.5630e-06, 3.5763e-07, 1.1921e-07, 1.1921e-07,\n", - " -7.1526e-07, 5.9605e-07, 0.0000e+00, 2.3842e-07, -5.9605e-07,\n", - " -6.5565e-07, -5.3644e-07, -4.7684e-07, 1.1921e-07, -5.9605e-08,\n", - " 3.5763e-07, 1.0729e-06, -3.5763e-07, -2.3842e-07, -2.9802e-07,\n", - " -1.7881e-07, 3.5763e-07, -2.3842e-07, -1.1921e-06, -1.1921e-07,\n", - " -1.3113e-06, -4.7684e-07, -3.5763e-07, -5.9605e-07, 1.1921e-06,\n", - " -8.3447e-07, 1.1921e-07, -1.0729e-06, -2.9802e-07, -3.5763e-07,\n", - " 7.1526e-07, -2.9802e-07, 2.3842e-07, -5.9605e-07, 9.5367e-07,\n", - " -8.9407e-07, 4.7684e-07, -3.5763e-07, 1.1921e-07, 3.5763e-07,\n", - " 0.0000e+00, -6.5565e-07, -4.7684e-07, -2.3842e-07, 1.1921e-07,\n", - " 1.1921e-07, 2.3842e-07, 2.3842e-07, -5.3644e-07, 3.5763e-07,\n", - " -9.5367e-07, 0.0000e+00, -1.1921e-07, -5.9605e-08, -1.1921e-07,\n", - " 8.3447e-07, 4.7684e-07, 3.5763e-07, -3.5763e-07, -1.7881e-07,\n", - " -5.9605e-08, -5.9605e-08, -5.9605e-08, -2.9802e-07, -4.7684e-07,\n", - " 1.1921e-07, -8.3447e-07, -5.9605e-07, -6.5565e-07, -1.9073e-06,\n", - " 7.1526e-07, 3.5763e-07, -3.5763e-07, 8.3447e-07, -1.3113e-06,\n", - " 3.5763e-07, -2.3842e-07, -2.3842e-07, -3.5763e-07, 4.7684e-07,\n", - " 1.1921e-07, -3.5763e-07, -4.1723e-07, 1.1921e-07, 5.9605e-07,\n", - " -5.9605e-07, 1.4305e-06, 3.5763e-07, -2.3842e-07, -2.9802e-07,\n", - " -2.3842e-07, -4.1723e-07, 1.0729e-06, -7.7486e-07, -5.3644e-07,\n", - " 3.5763e-07, 4.7684e-07, -3.5763e-07, -3.5763e-07, -7.1526e-07,\n", - " -2.3842e-07, -7.7486e-07, 4.7684e-07, -1.1921e-07, -8.3447e-07,\n", - " 2.3842e-07, -5.9605e-08, -5.9605e-08, 7.1526e-07, -8.3447e-07,\n", - " 3.5763e-07, 7.1526e-07, 3.5763e-07, 2.3842e-07, 0.0000e+00,\n", - " 1.5497e-06, -1.7881e-07, 0.0000e+00, 1.1921e-07, 9.5367e-07,\n", - " 5.9605e-07, 1.1921e-07, 3.5763e-07, 1.1921e-07, -1.0729e-06,\n", - " 8.3447e-07, 2.3842e-07, -5.3644e-07, -1.7881e-07, -3.5763e-07,\n", - " -7.1526e-07, 8.3447e-07, 7.1526e-07, 3.5763e-07, 8.3447e-07,\n", - " 5.9605e-07, 1.1921e-07, 2.3842e-07, 2.3842e-07, 0.0000e+00,\n", - " -5.9605e-08, -2.3842e-07, -2.9802e-07, 8.3447e-07, 9.5367e-07,\n", - " -1.1325e-06, -7.7486e-07, 9.5367e-07, 2.3842e-07, 7.1526e-07,\n", - " -8.3447e-07, -5.9605e-08, 1.4305e-06, -2.3842e-07, -3.5763e-07,\n", - " -7.1526e-07, 9.5367e-07, -1.6093e-06, -4.7684e-07, 2.3842e-07,\n", - " -2.0266e-06, 5.9605e-07, -4.7684e-07, 3.5763e-07, 1.0729e-06,\n", - " 9.5367e-07, 2.3842e-07, 8.3447e-07, -1.0729e-06, 1.1921e-06,\n", - " -2.9802e-07, -4.1723e-07, -1.7881e-07, -3.5763e-07, -9.5367e-07,\n", - " 8.3447e-07, -2.3842e-07, 9.5367e-07, -1.1921e-07, -8.3447e-07,\n", - " -7.1526e-07, -3.5763e-07, -2.3842e-07, -1.7881e-07, -3.5763e-07,\n", - " 1.3113e-06, -1.0133e-06, -4.7684e-07, -9.5367e-07, -8.9407e-07,\n", - " 1.0729e-06, -7.1526e-07, 1.1921e-07, -1.0729e-06, -5.3644e-07,\n", - " -2.9802e-07, -5.9605e-08, -7.1526e-07, -3.5763e-07, 1.1921e-07,\n", - " -4.1723e-07, 7.1526e-07, 1.1921e-06, 4.7684e-07, -1.0133e-06,\n", - " -4.1723e-07, 1.1921e-07, 3.5763e-07, -1.7881e-07, -7.7486e-07,\n", - " 1.1921e-07, 4.7684e-07, 4.7684e-07, 8.3447e-07, 5.9605e-07,\n", - " 1.0729e-06, -2.3842e-07, -8.3447e-07, 5.9605e-07, 0.0000e+00,\n", - " 1.1921e-07, 3.5763e-07, 7.1526e-07, -2.3842e-07, -5.3644e-07,\n", - " 0.0000e+00, 3.5763e-07, -5.9605e-08, -5.9605e-08, 4.7684e-07,\n", - " 1.1921e-07, -5.9605e-07, 2.3842e-07, 0.0000e+00, 9.5367e-07,\n", - " -4.7684e-07, 1.1921e-07, -4.7684e-07, 9.5367e-07, 7.1526e-07,\n", - " 0.0000e+00, 1.1921e-07, -1.7881e-07, 4.7684e-07, -2.3842e-07,\n", - " 1.1921e-07, -1.1921e-07, -5.3644e-07, -2.9802e-07, -9.5367e-07,\n", - " -1.1921e-07, -8.9407e-07, 8.3447e-07, 3.5763e-07, 2.3842e-07,\n", - " -4.1723e-07, -5.9605e-08, -3.5763e-07, -7.1526e-07, 3.5763e-07,\n", - " 0.0000e+00, -4.7684e-07, 0.0000e+00, 8.3447e-07, -1.1921e-07,\n", - " -2.9802e-07, -1.1325e-06, -5.9605e-07, -2.3842e-07, 3.5763e-07,\n", - " -2.9802e-07, -7.7486e-07, 3.5763e-07, 9.5367e-07, -4.7684e-07,\n", - " -5.9605e-08, -5.9605e-08, -2.9802e-07, 7.1526e-07, 7.1526e-07,\n", - " 5.9605e-07, -2.3842e-07, 2.3842e-07, 8.3447e-07, 3.5763e-07,\n", - " 1.1921e-07, -2.3842e-07, 4.7684e-07, -4.1723e-07, 0.0000e+00,\n", - " -5.9605e-07, -1.0133e-06, 4.7684e-07, 1.6689e-06, -7.7486e-07,\n", - " -2.9802e-07, 3.5763e-07, 4.7684e-07, 2.3842e-07, 4.7684e-07,\n", - " -4.1723e-07, -7.7486e-07, -1.1921e-07, -5.9605e-08, 5.9605e-07,\n", - " 1.1921e-07, -5.9605e-07, 3.5763e-07, 1.1921e-07, -1.7881e-07,\n", - " 3.5763e-07, -2.3842e-07, -3.5763e-07, -2.3842e-07, 1.4305e-06,\n", - " -2.3842e-07, -2.3842e-07, 8.3447e-07, -2.9802e-07, -3.5763e-07,\n", - " -3.5763e-07, -2.3842e-07, -5.9605e-07, 0.0000e+00, -1.4305e-06,\n", - " 2.3842e-07, 3.5763e-07, 3.5763e-07, -6.5565e-07, 4.7684e-07,\n", - " 1.1921e-07, 1.1921e-07, 1.1921e-07, 2.3842e-07, -8.3447e-07,\n", - " -8.3447e-07, 1.1921e-07, 1.1921e-07, 3.5763e-07, -5.3644e-07,\n", - " 3.5763e-07, -3.5763e-07, -7.7486e-07, -3.5763e-07, 7.1526e-07,\n", - " 9.5367e-07, -2.9802e-07, 2.3842e-07, -7.7486e-07, 4.7684e-07,\n", - " -1.1921e-07, -4.1723e-07, 4.7684e-07, 1.1921e-07, -2.9802e-07,\n", - " -4.7684e-07, 0.0000e+00, 5.9605e-07, -5.9605e-08, -6.5565e-07,\n", - " -3.5763e-07, 9.5367e-07, 8.3447e-07, -5.3644e-07, 7.1526e-07,\n", - " -1.7285e-06, 1.1921e-07, -2.9802e-07, 1.1921e-07, 1.1921e-07,\n", - " 3.5763e-07, 1.1921e-06, 1.1921e-07, -1.0729e-06, -1.7881e-07,\n", - " 3.5763e-07, 2.3842e-07, -1.0729e-06, -1.1921e-07, 0.0000e+00,\n", - " 1.1921e-07, -7.1526e-07, 4.7684e-07, -2.3842e-07, -2.9802e-07,\n", - " -2.9802e-07, 3.5763e-07, 9.5367e-07, 4.7684e-07, -2.9802e-07,\n", - " 4.7684e-07, -4.7684e-07, -5.9605e-07, 3.5763e-07, 3.5763e-07,\n", - " -4.1723e-07, -9.5367e-07, -5.9605e-08, -7.7486e-07, -7.7486e-07,\n", - " 4.7684e-07, -1.6689e-06, -4.1723e-07, -2.9802e-07, 3.5763e-07,\n", - " -4.7684e-07, 2.3842e-07, -6.5565e-07, -6.5565e-07, -6.5565e-07,\n", - " 2.3842e-07, -1.1921e-06, 2.3842e-07, 3.5763e-07, 4.7684e-07,\n", - " 4.7684e-07, -1.1921e-07, 2.3842e-07, 1.1921e-07, -1.1921e-07,\n", - " 1.1921e-07, 7.1526e-07, 3.5763e-07, -4.1723e-07, 2.3842e-07,\n", - " 0.0000e+00, 4.7684e-07, -8.9407e-07, -5.9605e-07, 4.7684e-07,\n", - " 3.5763e-07, 0.0000e+00, -1.1921e-07, -2.9802e-07, -1.7881e-07,\n", - " -5.9605e-07, 3.5763e-07, -1.1921e-07, 8.3447e-07, -7.7486e-07,\n", - " -6.5565e-07, 0.0000e+00, -2.9802e-07, -1.1921e-07, 5.9605e-07,\n", - " 5.9605e-07, 0.0000e+00, 1.1921e-07, -1.7881e-07, 7.1526e-07,\n", - " -4.7684e-07, 9.5367e-07], device='cuda:0')),\n", - " ('layer4.0.shortcut.1.bias',\n", - " tensor([ 1.2267e-07, 5.2131e-07, 4.9048e-07, -6.6727e-08, 5.8954e-07,\n", - " 2.0044e-07, 3.3354e-07, 1.0084e-07, 3.4111e-07, 1.0555e-07,\n", - " 4.8835e-07, -1.1029e-06, 2.0666e-08, 2.0148e-07, 2.1696e-07,\n", - " -7.4097e-07, 3.7880e-08, 6.4895e-08, -8.4100e-07, -3.1968e-07,\n", - " 1.0362e-07, -9.7019e-07, -3.3352e-07, -4.1362e-07, 4.1598e-08,\n", - " 2.2223e-07, 1.6852e-07, -1.0521e-07, 2.1346e-07, -1.2921e-06,\n", - " 3.7748e-07, -5.9054e-07, -1.5242e-08, -5.5715e-07, -2.1764e-07,\n", - " 2.1038e-07, 7.7662e-07, 9.8997e-07, 2.4168e-07, 8.6350e-08,\n", - " 1.8026e-07, 9.3270e-08, 2.4967e-07, -5.6322e-07, -3.2481e-07,\n", - " 5.5609e-09, 2.3633e-07, -4.5999e-07, 6.0220e-08, 6.4592e-07,\n", - " 2.2764e-07, -1.9812e-07, -9.3918e-07, -3.2255e-07, 2.1820e-07,\n", - " 1.5435e-06, -1.6022e-06, 9.6016e-08, -2.3592e-07, 7.2372e-07,\n", - " -4.9385e-07, 1.8043e-07, -4.0969e-07, 2.5857e-07, -5.5846e-07,\n", - " 3.3289e-07, 3.6472e-07, -9.9066e-07, -6.4836e-07, -5.9628e-08,\n", - " 7.9305e-07, 5.1530e-07, 3.1524e-07, -3.6323e-07, 3.4015e-07,\n", - " 1.8533e-07, -2.7069e-07, 2.8576e-07, -5.2353e-07, -1.3846e-07,\n", - " -1.7000e-06, 3.0305e-07, -8.1954e-07, -9.5841e-07, 3.0598e-07,\n", - " -8.9424e-07, 2.0526e-07, -5.2114e-07, 3.8126e-07, -8.6022e-07,\n", - " 9.4818e-07, -1.4355e-07, 2.4742e-07, -4.0967e-07, 1.2586e-07,\n", - " -9.7055e-07, 1.4553e-06, 1.1442e-07, 8.6497e-07, 1.3247e-07,\n", - " -5.0140e-07, -2.5456e-07, -2.6590e-08, -3.5772e-07, 8.2593e-07,\n", - " 5.3295e-07, -1.0410e-07, 4.3498e-07, 2.3294e-07, 4.7580e-07,\n", - " -5.4222e-07, 4.6807e-07, 3.8665e-07, -5.5240e-07, 4.0680e-07,\n", - " 4.8383e-07, -3.1379e-07, 2.7812e-07, -8.6844e-07, -4.2003e-08,\n", - " 3.9490e-07, -2.1611e-07, 4.1244e-08, -7.7546e-07, -2.5390e-07,\n", - " 7.5168e-07, -1.0881e-06, -3.0041e-09, -1.2022e-06, -1.0697e-06,\n", - " 2.1438e-07, -2.7698e-07, 1.0052e-07, -2.8831e-07, -7.9248e-07,\n", - " -1.1695e-08, 3.7242e-07, 3.6402e-07, -4.4123e-07, 6.7511e-08,\n", - " 6.2402e-07, -7.6874e-07, -7.7209e-07, -4.6478e-07, 8.0412e-07,\n", - " -1.1367e-06, 7.2712e-07, -1.3130e-07, -5.0524e-07, 2.0572e-07,\n", - " -4.2604e-07, 1.5546e-07, 9.0354e-07, -5.7759e-07, -6.8648e-07,\n", - " 3.3436e-07, 4.4954e-07, 3.9408e-07, -6.5580e-07, -9.3324e-07,\n", - " -6.8870e-07, -6.4934e-07, -1.5007e-07, -5.6741e-08, -9.6720e-07,\n", - " 4.3615e-07, -1.5925e-07, -8.6850e-07, 4.7006e-07, -6.9503e-07,\n", - " 8.5696e-07, 4.0076e-07, 5.0084e-07, 3.3838e-07, 1.2934e-07,\n", - " 1.0382e-06, -4.2320e-07, 3.1105e-07, 9.0592e-07, 5.3915e-07,\n", - " -2.8505e-07, 3.0121e-07, -3.5367e-07, 3.1758e-07, -9.3912e-07,\n", - " 7.7104e-07, 4.0988e-07, 4.6698e-08, 3.6261e-07, -2.4688e-07,\n", - " -5.6081e-07, 5.2616e-07, 3.9207e-07, 3.3400e-07, 1.3801e-07,\n", - " 1.5755e-07, 1.3671e-07, -2.9247e-07, -3.3614e-08, 2.7387e-07,\n", - " 4.1058e-08, 1.0102e-07, 5.9427e-07, 6.9601e-07, 5.1401e-07,\n", - " -9.0142e-07, -7.8599e-07, 8.5890e-07, 8.0110e-07, -2.6069e-08,\n", - " -4.7613e-07, -1.1880e-07, 6.0977e-07, -3.2545e-08, -8.0672e-07,\n", - " -4.4460e-07, 4.9622e-08, -1.0001e-06, -3.8140e-07, 2.0876e-07,\n", - " -7.7196e-07, 7.1734e-07, -4.6298e-07, 6.7959e-07, 1.0245e-06,\n", - " 4.1416e-07, 5.4496e-07, 7.8721e-07, -4.2891e-07, 1.6825e-06,\n", - " 1.4942e-07, -1.6828e-07, 1.7415e-07, 4.0402e-08, -5.3157e-07,\n", - " 1.0582e-07, -2.0883e-07, 4.7203e-07, -1.3684e-07, -1.3398e-06,\n", - " -5.9472e-07, 3.9554e-07, 2.3206e-07, -2.5863e-07, -4.5267e-07,\n", - " 1.0301e-06, -5.3752e-07, -7.6298e-08, -5.4183e-07, 2.8119e-07,\n", - " 7.9946e-08, -1.5766e-07, 1.8332e-07, -8.2711e-07, -3.7360e-07,\n", - " -1.5190e-07, -3.5914e-08, 3.6115e-07, -1.4778e-07, 8.6465e-07,\n", - " -2.9969e-08, 2.9304e-07, 2.5676e-07, -2.8295e-07, -8.0901e-07,\n", - " -3.3888e-07, -1.1907e-07, 2.6471e-07, 1.6580e-07, -6.6758e-08,\n", - " 2.1079e-07, 3.7526e-07, 3.1104e-07, 8.7153e-07, 3.5282e-07,\n", - " 4.8555e-07, -2.3091e-07, -5.6911e-07, 1.3365e-07, 3.0886e-07,\n", - " 2.5100e-07, -2.3340e-07, 4.3298e-07, -3.9339e-07, -6.6168e-07,\n", - " 2.4621e-07, 1.2055e-07, 5.7209e-08, -5.5472e-07, 8.4360e-07,\n", - " -8.6571e-07, 1.9837e-07, 5.0580e-08, 4.3711e-07, 4.6089e-07,\n", - " 1.3192e-07, 1.8432e-08, -4.0834e-07, 2.2995e-07, 9.0939e-07,\n", - " -6.0561e-07, 4.8501e-07, -6.9802e-07, 4.8374e-07, -3.7478e-07,\n", - " -3.0589e-07, -2.6858e-07, -4.6554e-08, -1.0507e-07, -2.8908e-07,\n", - " -1.0337e-06, -1.0129e-06, 2.2487e-07, 1.7116e-07, -4.1344e-07,\n", - " -4.2085e-07, -3.7954e-08, -5.6785e-07, -5.8464e-07, 1.2217e-08,\n", - " -9.5578e-08, -2.8373e-07, 2.6618e-07, 7.6734e-07, -8.6086e-07,\n", - " 3.6636e-07, -9.9598e-07, -1.5478e-07, 1.7445e-07, -5.5791e-07,\n", - " -7.7129e-07, -2.5449e-07, 2.9259e-07, 2.6276e-07, -7.1697e-08,\n", - " -5.8335e-07, 1.1469e-08, 2.1788e-07, 1.0519e-06, 5.5815e-08,\n", - " 4.8869e-07, -6.1105e-08, 2.8824e-08, 5.1238e-07, -2.2381e-08,\n", - " -1.9842e-07, 1.5346e-07, 2.2163e-07, -3.7323e-07, 2.4546e-07,\n", - " -5.5839e-07, -1.2237e-06, 7.6679e-07, 1.1058e-06, -6.1683e-07,\n", - " -5.6177e-07, -4.2822e-07, -2.7411e-07, 2.8429e-09, -1.7565e-07,\n", - " -6.2889e-07, -5.5862e-07, -5.8493e-07, -4.7978e-07, 6.6172e-07,\n", - " 6.3017e-08, -5.3413e-07, 8.2590e-07, -2.3073e-07, -2.7192e-07,\n", - " 5.1345e-08, -4.7095e-07, -7.3116e-07, -7.4617e-07, 1.2349e-06,\n", - " -5.5501e-08, -3.8916e-07, 4.9881e-07, -5.8128e-07, -1.0835e-06,\n", - " 8.9368e-08, -2.4767e-08, -1.7907e-07, 7.5381e-07, -9.3912e-07,\n", - " 1.6275e-07, 8.3961e-07, 1.9004e-07, -6.1507e-07, 1.7484e-07,\n", - " 6.1234e-07, -3.3452e-07, 1.7429e-07, 6.1249e-07, -4.0872e-07,\n", - " -8.5998e-07, -2.6819e-08, 4.6977e-07, -4.8661e-07, -3.7181e-07,\n", - " -4.3231e-07, -4.4359e-07, 1.0288e-07, -4.6875e-07, 1.2184e-07,\n", - " 5.4193e-07, 1.6957e-07, 1.2930e-08, -8.3377e-07, 2.0160e-07,\n", - " -1.1283e-06, 4.6074e-08, 7.9631e-08, -1.9956e-07, -2.8262e-07,\n", - " -2.7352e-07, -3.9198e-07, 9.0809e-07, -6.9796e-07, -7.7386e-07,\n", - " -6.3717e-07, 1.9364e-07, 4.2499e-07, -2.1963e-07, 3.3570e-07,\n", - " -1.1842e-06, -1.3149e-07, -2.0294e-07, -8.4020e-08, -1.8796e-07,\n", - " -8.5904e-08, 1.0188e-06, 1.1152e-07, -7.9846e-07, -1.3746e-07,\n", - " 3.0789e-07, 1.1175e-07, -3.5673e-07, -3.4911e-08, -2.8830e-08,\n", - " -2.7811e-07, 6.8873e-08, 6.6728e-07, -2.9486e-07, 6.0918e-07,\n", - " -1.2829e-06, 6.8836e-07, 3.0841e-07, 4.4511e-07, 3.0244e-07,\n", - " 4.3634e-07, 3.6050e-08, 5.0360e-07, -5.9110e-07, 9.7646e-07,\n", - " -9.2563e-07, -4.3873e-07, 1.1422e-07, -1.3625e-07, -6.4917e-07,\n", - " 5.3954e-08, -1.5143e-06, -5.9071e-07, -2.4121e-08, -1.0838e-06,\n", - " -4.0250e-07, 5.5546e-07, -7.1030e-07, -1.0235e-06, -9.1596e-08,\n", - " 1.7611e-08, -1.6254e-06, 5.3340e-07, 5.0056e-07, 2.9434e-07,\n", - " 3.0389e-07, -6.4017e-07, 3.4055e-07, -2.9545e-07, 1.7299e-07,\n", - " -3.5783e-08, 3.9323e-07, 1.4520e-08, 2.4158e-09, -8.8659e-07,\n", - " -2.5928e-07, -4.6700e-07, -4.3284e-07, 7.8656e-08, 2.4758e-07,\n", - " 7.3693e-08, 2.9827e-08, -5.8056e-07, -9.8295e-08, -2.7469e-08,\n", - " -6.7635e-07, 5.9764e-08, 9.8274e-08, 1.2037e-08, -3.1155e-07,\n", - " -2.1793e-07, 6.5858e-08, 4.3835e-08, 1.8843e-07, 1.4375e-07,\n", - " 7.0740e-08, 9.6125e-08, 1.4013e-07, 6.2235e-07, 1.1469e-06,\n", - " -4.6511e-09, 2.3411e-07], device='cuda:0')),\n", - " ('linear.weight',\n", - " tensor([[-1.0904e-05, 9.6373e-06, -3.4794e-06, ..., 1.5184e-05,\n", - " -4.0233e-07, 8.1211e-07],\n", - " [ 2.1834e-05, 6.3516e-06, 9.2201e-06, ..., -2.7325e-06,\n", - " 8.5123e-06, 1.3521e-05],\n", - " [ 4.4946e-06, 2.2314e-06, 4.9248e-06, ..., -5.9907e-06,\n", - " 6.3404e-06, 8.9928e-06],\n", - " ...,\n", - " [ 2.6450e-07, -9.1344e-06, -2.0359e-06, ..., -5.2564e-06,\n", - " -6.4522e-06, 2.2016e-06],\n", - " [ 1.2709e-05, -6.8583e-06, -2.1979e-06, ..., 7.8660e-06,\n", - " 1.2752e-05, 1.5881e-05],\n", - " [ 2.4494e-07, -1.4435e-07, 2.5369e-06, ..., 4.0978e-07,\n", - " 1.2223e-05, -1.4249e-07]], device='cuda:0')),\n", - " ('linear.bias',\n", - " tensor([ 4.1835e-06, 1.4639e-05, 1.2632e-05, 8.4601e-06, -2.2002e-05,\n", - " -9.7230e-06, 7.1898e-07, -1.4666e-05, 6.3330e-06, -5.7463e-07],\n", - " device='cuda:0'))])" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "param_diff" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "[tensor([[[[ 1.3094e-06, 2.2575e-06, 9.5367e-07],\n", - " [-3.5763e-07, -5.7369e-07, -1.7434e-06],\n", - " [-1.3709e-06, -3.0920e-06, -3.6210e-06]],\n", - " \n", - " [[ 5.0340e-06, 5.3197e-06, 4.6045e-06],\n", - " [ 2.2957e-06, 1.7583e-06, 7.5251e-07],\n", - " [ 1.6615e-06, -4.3213e-07, -8.0466e-07]],\n", - " \n", - " [[ 1.4380e-06, 1.3784e-06, 1.5944e-06],\n", - " [-1.5199e-06, -2.6003e-06, -2.5854e-06],\n", - " [-1.8328e-06, -4.6492e-06, -4.3511e-06]]],\n", - " \n", - " \n", - " [[[-4.0978e-06, -9.0785e-06, -7.7784e-06],\n", - " [-1.8328e-06, -3.0994e-06, -4.8429e-07],\n", - " [-1.9227e-06, 3.5604e-06, 6.3498e-06]],\n", - " \n", - " [[-3.7700e-06, -8.8876e-06, -8.1509e-06],\n", - " [-5.2527e-07, -2.9840e-06, -2.8312e-07],\n", - " [-3.0547e-06, 1.2964e-06, 3.2708e-06]],\n", - " \n", - " [[-3.4943e-06, -9.6448e-06, -7.7486e-06],\n", - " [ 6.5565e-07, -3.5614e-06, -4.3213e-07],\n", - " [-2.4363e-06, -7.0967e-07, 1.7583e-06]]],\n", - " \n", - " \n", - " [[[ 2.6673e-05, 2.1972e-05, 1.6913e-05],\n", - " [ 2.3261e-05, 2.2944e-05, 2.2389e-05],\n", - " [ 2.5196e-05, 2.5854e-05, 2.5735e-05]],\n", - " \n", - " [[ 3.0279e-05, 2.6144e-05, 2.2963e-05],\n", - " [ 3.0980e-05, 2.7031e-05, 2.6032e-05],\n", - " [ 3.0085e-05, 2.8834e-05, 2.9545e-05]],\n", - " \n", - " [[ 2.9221e-05, 2.6941e-05, 2.4391e-05],\n", - " [ 3.1814e-05, 3.2485e-05, 3.1255e-05],\n", - " [ 3.1918e-05, 3.4809e-05, 3.3238e-05]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 2.8461e-06, 1.1273e-05, 7.7039e-06],\n", - " [ 1.2405e-06, 6.9886e-06, 5.4389e-06],\n", - " [ 2.8685e-06, 5.6028e-06, 7.4003e-06]],\n", - " \n", - " [[ 3.9190e-06, 1.0349e-05, 5.9307e-06],\n", - " [ 7.4506e-09, 3.5614e-06, 1.1474e-06],\n", - " [-5.8115e-07, 1.0058e-07, 1.2144e-06]],\n", - " \n", - " [[ 3.8743e-06, 9.0078e-06, 5.7509e-06],\n", - " [-1.3225e-06, 1.2368e-06, 1.7881e-07],\n", - " [-3.2336e-06, -3.4049e-06, -1.4156e-06]]],\n", - " \n", - " \n", - " [[[ 8.3447e-07, 4.5300e-06, 7.3425e-06],\n", - " [ 4.7684e-06, 4.2021e-06, 6.7465e-06],\n", - " [ 5.5358e-06, 7.1377e-06, 8.0615e-06]],\n", - " \n", - " [[-6.8042e-06, -6.2436e-06, -2.8461e-06],\n", - " [ 1.4007e-06, -2.7195e-06, -6.1840e-07],\n", - " [ 1.8179e-06, 1.0058e-06, 2.3842e-06]],\n", - " \n", - " [[-6.6310e-06, -6.3963e-06, -3.5614e-06],\n", - " [ 5.5134e-07, -3.4869e-06, -9.1735e-07],\n", - " [ 3.1143e-06, 1.7732e-06, 3.7588e-06]]],\n", - " \n", - " \n", - " [[[ 4.8280e-06, 2.9951e-06, 6.4448e-06],\n", - " [-7.3258e-06, -5.4464e-06, -1.7434e-06],\n", - " [-1.1042e-05, -4.9919e-06, -6.2287e-06]],\n", - " \n", - " [[ 1.0684e-05, 1.1906e-05, 1.2517e-05],\n", - " [-2.6673e-06, 2.7418e-06, 4.4033e-06],\n", - " [-8.4639e-06, 3.8743e-07, -1.0952e-06]],\n", - " \n", - " [[ 1.0833e-05, 1.4074e-05, 1.5259e-05],\n", - " [-4.4703e-07, 6.6757e-06, 8.8885e-06],\n", - " [-5.3793e-06, 4.7088e-06, 3.7588e-06]]]], device='cuda:0'),\n", - " tensor([ 5.7220e-06, 7.1526e-07, -5.3644e-07, 6.4373e-06, -4.9472e-06,\n", - " 3.2187e-06, -1.9670e-06, -6.4373e-06, 9.5367e-07, 4.7684e-07,\n", - " -3.0398e-06, 5.3644e-06, -2.6226e-06, -1.3113e-06, 0.0000e+00,\n", - " -7.8082e-06, -3.3975e-06, 1.6689e-06, -2.2054e-06, -3.6955e-06,\n", - " -1.3113e-06, 3.6955e-06, -3.3975e-06, 3.6955e-06, 4.4107e-06,\n", - " -9.4771e-06, -3.5167e-06, 9.7752e-06, -2.4438e-06, -8.3447e-07,\n", - " -4.8280e-06, 1.0729e-06, -3.1590e-06, -4.2915e-06, -2.4438e-06,\n", - " -3.0994e-06, -1.0848e-05, 5.1260e-06, 2.9802e-06, -1.5497e-06,\n", - " -1.6689e-06, -4.4703e-06, 4.4107e-06, 1.0729e-06, -2.9802e-07,\n", - " 8.8215e-06, -4.5896e-06, 6.0797e-06, 9.7752e-06, 5.2452e-06,\n", - " 1.6689e-06, -4.1723e-07, 2.2650e-06, -6.5565e-06, 8.3447e-07,\n", - " 6.3181e-06, -4.7088e-06, -4.3511e-06, -4.2915e-06, 4.6492e-06,\n", - " -1.7881e-07, 1.9073e-06, 1.4305e-06, -8.3447e-07], device='cuda:0'),\n", - " tensor([ 5.1800e-06, -5.8434e-07, 1.9089e-06, 4.9003e-06, -1.9461e-06,\n", - " -3.6183e-06, -1.1956e-06, -1.8275e-06, -2.3727e-06, 2.0175e-06,\n", - " -9.0375e-07, 5.2483e-06, -4.0264e-06, -1.9131e-06, 6.7097e-07,\n", - " -1.0581e-05, -3.0417e-06, 1.6963e-06, 2.1104e-06, -2.6472e-06,\n", - " -3.2058e-06, 3.0026e-06, 1.8405e-06, 1.6015e-07, 3.5247e-07,\n", - " -3.5816e-06, 1.2254e-06, 3.6254e-06, -3.3752e-06, -1.8590e-06,\n", - " -4.0213e-06, 1.7876e-06, -5.8218e-06, -7.0407e-06, -3.8052e-06,\n", - " -4.0191e-06, -6.2535e-06, 4.4571e-06, 1.0564e-06, 4.3555e-07,\n", - " -5.3791e-06, -8.4146e-07, 4.8352e-06, 1.5922e-06, -1.8305e-06,\n", - " 6.3709e-06, -1.9585e-06, 4.7529e-06, -6.0682e-06, 3.9466e-06,\n", - " -2.0051e-06, -2.7619e-06, 4.9631e-06, -5.6562e-06, -1.1060e-06,\n", - " 1.8016e-06, -2.9148e-06, -5.2561e-07, -3.8957e-06, 6.7037e-06,\n", - " 1.4417e-06, 4.2739e-07, -5.6229e-06, 2.4895e-06], device='cuda:0'),\n", - " tensor([[[[-1.1077e-05, -1.3351e-05, -1.3445e-05],\n", - " [-1.1542e-05, -1.2651e-05, -1.2247e-05],\n", - " [-8.0392e-06, -8.5840e-06, -9.3412e-06]],\n", - " \n", - " [[ 9.7696e-07, 2.9877e-06, 5.6960e-06],\n", - " [-2.4959e-07, 2.9840e-06, 3.9032e-06],\n", - " [ 1.7248e-06, 1.4092e-06, 3.3677e-06]],\n", - " \n", - " [[ 1.6801e-06, 1.3839e-06, 4.0382e-06],\n", - " [-8.3447e-07, -1.2778e-06, 9.8720e-07],\n", - " [-7.5519e-06, -6.0269e-06, -4.8466e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 6.1188e-07, 1.4915e-06, 3.4235e-06],\n", - " [ 8.2608e-07, 1.5702e-06, 3.3225e-06],\n", - " [-1.4435e-07, 5.6624e-07, 8.0466e-07]],\n", - " \n", - " [[-5.9982e-07, 2.5332e-07, 3.5102e-06],\n", - " [-2.6356e-07, 2.7977e-06, 3.7896e-06],\n", - " [ 3.4086e-07, 1.5418e-06, 1.4640e-06]],\n", - " \n", - " [[ 3.3891e-06, 3.9116e-07, 1.1139e-06],\n", - " [ 6.0275e-06, 3.4408e-06, 1.3205e-06],\n", - " [ 8.8597e-06, 4.2245e-06, 3.3457e-06]]],\n", - " \n", - " \n", - " [[[ 5.8608e-06, 6.4773e-06, 7.9181e-06],\n", - " [ 2.2501e-06, 1.9427e-06, 3.5204e-06],\n", - " [ 4.7162e-06, 4.3027e-06, 4.3828e-06]],\n", - " \n", - " [[-7.2327e-06, -8.4508e-06, -4.1667e-06],\n", - " [-5.6550e-06, -4.2372e-06, -6.7614e-07],\n", - " [-3.0911e-06, -8.1584e-07, -5.8832e-06]],\n", - " \n", - " [[ 5.8059e-06, 8.8625e-06, 2.9579e-06],\n", - " [-6.4736e-06, -8.4308e-07, 3.4049e-06],\n", - " [-8.2515e-07, 4.6254e-06, 8.2117e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.2107e-06, 2.6133e-06, 3.4831e-07],\n", - " [-1.0639e-05, -5.5619e-06, -5.5544e-06],\n", - " [-9.5721e-06, -5.6596e-06, -3.7905e-06]],\n", - " \n", - " [[-1.8440e-06, 2.0619e-06, 1.4417e-06],\n", - " [-1.1794e-05, -2.8946e-06, -5.0736e-06],\n", - " [-1.1954e-05, -5.2908e-06, -5.1484e-06]],\n", - " \n", - " [[-6.2555e-06, -1.2591e-06, -2.0694e-06],\n", - " [-8.8848e-07, -7.8976e-07, -1.4305e-06],\n", - " [ 6.2720e-06, 3.3956e-06, 9.1093e-06]]],\n", - " \n", - " \n", - " [[[-3.9963e-06, -8.3866e-06, -3.3099e-06],\n", - " [-3.6014e-06, -8.0690e-06, -5.1260e-06],\n", - " [-4.9341e-06, -6.4764e-06, -3.4515e-06]],\n", - " \n", - " [[ 1.1835e-05, 1.0084e-05, 8.2310e-06],\n", - " [ 1.4350e-05, 7.1451e-06, 6.6385e-06],\n", - " [ 1.6532e-05, 1.2064e-05, 9.2201e-06]],\n", - " \n", - " [[ 1.3463e-05, 1.0386e-05, 5.9130e-06],\n", - " [ 9.9950e-06, 8.2487e-06, 7.7151e-06],\n", - " [-6.3702e-07, -3.2855e-06, 5.9232e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.5248e-05, 1.1347e-05, 8.3297e-06],\n", - " [ 1.8368e-05, 1.1865e-05, 9.6178e-06],\n", - " [ 1.6540e-05, 1.2219e-05, 1.1163e-05]],\n", - " \n", - " [[ 1.4391e-05, 6.0964e-06, 7.6713e-06],\n", - " [ 1.8489e-05, 9.6224e-06, 8.3763e-06],\n", - " [ 1.7405e-05, 1.1558e-05, 8.1407e-06]],\n", - " \n", - " [[-3.3677e-06, 1.7807e-06, -5.1223e-06],\n", - " [-5.9060e-06, 5.2527e-07, -2.8340e-06],\n", - " [-8.7004e-06, -5.4650e-06, -4.6156e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-1.7360e-06, 5.4762e-07, 2.7195e-06],\n", - " [-2.4382e-06, -2.2650e-06, -1.0524e-06],\n", - " [-2.7567e-07, -1.2275e-06, -1.7919e-06]],\n", - " \n", - " [[-5.0822e-06, -7.7039e-06, -9.2110e-06],\n", - " [-1.3849e-06, -3.0864e-06, -3.4422e-06],\n", - " [-2.2687e-06, -1.2293e-06, -9.7230e-07]],\n", - " \n", - " [[ 8.6054e-07, 8.1025e-07, 1.1176e-06],\n", - " [-2.0377e-06, -2.6319e-06, 3.0063e-06],\n", - " [ 6.4448e-07, 2.4252e-06, 8.2143e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.4145e-06, -5.3197e-06, -6.9588e-06],\n", - " [-4.0792e-06, -5.8394e-06, -5.6103e-06],\n", - " [-2.9318e-06, -1.2163e-06, -1.6280e-06]],\n", - " \n", - " [[-3.3230e-06, -4.0093e-06, -7.9703e-06],\n", - " [-2.6223e-06, -4.0159e-06, -7.2364e-06],\n", - " [-1.2740e-06, -6.3330e-08, -3.6899e-06]],\n", - " \n", - " [[-2.3842e-06, -5.4110e-07, -1.3970e-06],\n", - " [-4.8056e-06, -3.3015e-07, -6.0019e-06],\n", - " [-1.3257e-06, 1.7844e-06, -3.7998e-07]]],\n", - " \n", - " \n", - " [[[ 3.2345e-06, 4.0606e-06, 4.6827e-06],\n", - " [ 3.9581e-06, 6.2026e-06, 6.0778e-06],\n", - " [ 3.8482e-06, 7.9684e-06, 6.9216e-06]],\n", - " \n", - " [[-2.4848e-06, -7.5959e-06, -9.3263e-06],\n", - " [-1.1595e-06, -5.9567e-06, -8.2850e-06],\n", - " [-3.3425e-06, -6.2436e-06, -1.0731e-05]],\n", - " \n", - " [[ 7.7710e-06, 1.1087e-05, 2.1253e-06],\n", - " [ 2.3516e-06, 2.1737e-06, -5.2582e-06],\n", - " [-5.0201e-06, -4.3120e-06, -2.9020e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.8254e-07, -3.2522e-06, -5.4492e-06],\n", - " [ 1.1362e-06, -4.9453e-06, -6.3721e-06],\n", - " [ 9.0897e-07, -4.7460e-06, -7.0799e-06]],\n", - " \n", - " [[-2.5658e-07, -2.7902e-06, -2.6394e-06],\n", - " [ 1.7732e-06, -4.1220e-06, -5.3123e-06],\n", - " [ 3.0876e-06, -2.0824e-06, -5.3644e-06]],\n", - " \n", - " [[ 4.7442e-06, 2.9951e-06, 1.1277e-05],\n", - " [ 9.6490e-06, 7.3425e-06, 1.3027e-05],\n", - " [-1.1586e-06, -1.8999e-07, 6.1467e-06]]],\n", - " \n", - " \n", - " [[[-6.4075e-06, -6.9048e-06, -8.5756e-06],\n", - " [-6.4103e-06, -6.2175e-06, -6.0480e-06],\n", - " [-7.4264e-06, -6.6143e-06, -6.4950e-06]],\n", - " \n", - " [[ 5.3104e-06, 3.9414e-06, 8.6054e-06],\n", - " [-3.3397e-06, 4.0606e-07, 5.6159e-06],\n", - " [-4.9863e-06, -2.5965e-06, 5.1418e-06]],\n", - " \n", - " [[-7.7859e-06, -1.3482e-05, -6.9756e-06],\n", - " [ 1.9577e-05, 1.4581e-05, 1.3802e-05],\n", - " [ 1.1358e-05, 1.0703e-05, 1.2726e-05]],\n", - " \n", - " ...,\n", - " \n", - " [[ 8.5440e-06, 7.8101e-06, 9.2369e-06],\n", - " [ 7.8743e-06, 7.1749e-06, 6.1598e-06],\n", - " [ 3.6135e-06, 2.7828e-06, 8.0112e-06]],\n", - " \n", - " [[-1.7043e-06, 1.6764e-08, 2.8984e-06],\n", - " [ 1.2182e-06, 6.9477e-07, -1.3039e-06],\n", - " [ 3.5912e-06, -5.1605e-06, -2.6915e-06]],\n", - " \n", - " [[-1.9607e-06, -4.8131e-06, -6.8620e-06],\n", - " [ 5.2191e-06, -6.1467e-07, -5.7463e-06],\n", - " [-2.8685e-07, 1.7951e-07, -8.0280e-07]]]], device='cuda:0'),\n", - " tensor([ 9.5367e-07, 0.0000e+00, -1.0729e-06, -8.2254e-06, 1.3113e-06,\n", - " 7.6294e-06, -1.1921e-06, -7.7486e-06, -3.5167e-06, -1.9670e-06,\n", - " 4.6492e-06, -3.9339e-06, -5.0068e-06, 1.7881e-06, -3.8743e-06,\n", - " -3.0994e-06, 2.2650e-06, 3.5763e-07, 2.3842e-07, 8.3447e-07,\n", - " 6.6757e-06, -4.0531e-06, -9.5367e-07, 1.1921e-07, -8.9407e-07,\n", - " -1.4901e-06, -3.0398e-06, -1.7881e-06, 1.4305e-06, -7.7486e-06,\n", - " 2.3842e-07, 4.6492e-06, 0.0000e+00, -1.1921e-07, 2.9802e-06,\n", - " -1.8477e-06, 2.3842e-06, -2.2054e-06, 1.6689e-06, 9.5367e-07,\n", - " 2.3842e-06, 1.0729e-06, 6.4373e-06, 3.6955e-06, -5.3644e-07,\n", - " -1.6093e-06, -8.9407e-07, -2.0266e-06, 5.7220e-06, 2.3842e-07,\n", - " 9.5367e-07, 5.9605e-07, 7.8678e-06, -5.9605e-08, -2.2054e-06,\n", - " -3.7551e-06, -7.1526e-07, 2.3842e-06, 1.6689e-06, 2.2650e-06,\n", - " 2.1458e-06, -1.4901e-06, 2.2650e-06, -3.1590e-06], device='cuda:0'),\n", - " tensor([ 2.6768e-06, 1.9124e-06, -1.3289e-07, -8.0001e-06, 1.6482e-06,\n", - " 4.9723e-06, 2.5025e-06, -3.4393e-06, -1.3543e-06, -1.6179e-07,\n", - " 3.2728e-06, -2.7219e-06, -4.3160e-06, 1.7468e-06, -1.1131e-06,\n", - " -2.0409e-06, 2.2392e-06, -1.4375e-06, 8.6258e-07, 2.6158e-06,\n", - " 5.8557e-06, -1.5091e-06, -3.1794e-07, 3.2463e-06, 2.5008e-06,\n", - " -4.0393e-06, -8.9229e-07, -3.1561e-06, 3.7993e-07, -6.5335e-06,\n", - " 2.5505e-06, 2.9680e-06, 7.9633e-07, -3.2913e-06, 1.1366e-06,\n", - " -1.5826e-06, 2.8537e-07, 3.2345e-06, 2.9700e-06, -1.7639e-06,\n", - " 3.0974e-06, -7.6699e-07, 3.8670e-06, -9.4418e-07, 1.4069e-06,\n", - " -2.0534e-06, 5.8089e-07, -3.6296e-06, 8.4546e-07, 6.6837e-07,\n", - " 8.0962e-07, 2.2233e-06, 4.3749e-06, 2.1475e-06, -7.4168e-07,\n", - " -2.2586e-06, 5.3124e-07, 4.4179e-06, -1.9280e-06, 5.8637e-07,\n", - " 4.8589e-07, -1.7005e-06, 1.1679e-06, -4.1782e-06], device='cuda:0'),\n", - " tensor([[[[ 6.8545e-06, -2.7642e-06, 7.6927e-07],\n", - " [ 1.2424e-06, -4.1537e-06, -4.5188e-06],\n", - " [-2.1346e-06, -3.7439e-07, -4.5253e-06]],\n", - " \n", - " [[ 1.0302e-05, 1.4732e-05, 1.1850e-05],\n", - " [ 7.7300e-06, 1.0770e-05, 1.0885e-05],\n", - " [ 8.0653e-06, 1.0647e-05, 1.2951e-05]],\n", - " \n", - " [[-1.4819e-06, -2.3656e-06, -1.1160e-05],\n", - " [ 6.9290e-07, 1.1502e-07, -8.7265e-06],\n", - " [ 3.6135e-06, 5.2182e-06, -9.1409e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 9.9167e-06, 6.6161e-06, 7.1656e-06],\n", - " [ 7.6666e-06, 7.4543e-06, 3.8240e-06],\n", - " [ 1.9455e-06, 1.6578e-06, -3.4273e-06]],\n", - " \n", - " [[ 8.7172e-07, -1.5176e-06, -8.2329e-07],\n", - " [ 5.5740e-07, -2.4755e-06, -1.2703e-06],\n", - " [-1.9423e-06, -4.0457e-06, 2.4149e-06]],\n", - " \n", - " [[-7.4450e-06, -3.3551e-06, -5.1595e-06],\n", - " [ 7.1898e-07, 7.2116e-06, 3.7476e-06],\n", - " [ 4.2431e-06, 6.8992e-06, 4.1407e-06]]],\n", - " \n", - " \n", - " [[[-9.7672e-07, -1.1250e-06, -4.3670e-06],\n", - " [ 1.4119e-06, 2.3879e-06, -1.7881e-07],\n", - " [ 3.6471e-06, 1.4715e-06, 2.5146e-06]],\n", - " \n", - " [[-2.1979e-07, 8.6194e-07, -9.3132e-08],\n", - " [-1.0896e-07, 1.7686e-06, 3.8389e-06],\n", - " [ 9.4250e-07, -1.2834e-06, 1.5097e-06]],\n", - " \n", - " [[-6.5397e-06, -2.5074e-06, -5.4762e-07],\n", - " [-5.4091e-06, -3.7719e-07, -7.5344e-07],\n", - " [-4.9137e-06, 2.4606e-06, 6.5845e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.2897e-06, -7.5996e-07, 1.0803e-06],\n", - " [ 3.2000e-06, 1.3162e-06, -9.7975e-07],\n", - " [ 1.0123e-06, -3.9414e-06, -9.1726e-06]],\n", - " \n", - " [[-3.4384e-06, 1.0692e-06, 7.7486e-07],\n", - " [-5.3057e-06, -1.6671e-07, 2.1765e-06],\n", - " [-2.6673e-06, 4.4983e-07, 3.2634e-06]],\n", - " \n", - " [[-1.9129e-06, 4.4238e-07, 3.2447e-06],\n", - " [-6.3479e-06, 1.0803e-07, 3.5837e-06],\n", - " [-3.8035e-06, 2.3972e-06, 2.4568e-06]]],\n", - " \n", - " \n", - " [[[ 8.4192e-07, -1.3798e-06, 1.4901e-07],\n", - " [ 2.6077e-07, -1.9146e-06, -3.6638e-06],\n", - " [ 1.5544e-06, -4.7134e-06, -5.4822e-06]],\n", - " \n", - " [[ 6.4727e-08, 4.8503e-06, -3.9488e-07],\n", - " [-2.8275e-06, -7.8976e-07, -3.7272e-06],\n", - " [ 2.4363e-06, 1.9111e-06, -3.1069e-06]],\n", - " \n", - " [[ 4.8932e-06, 6.0722e-07, 3.9460e-06],\n", - " [ 8.0299e-06, 5.1351e-06, 1.1583e-05],\n", - " [ 3.1609e-06, 1.5479e-06, 4.0233e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.5709e-06, 3.6675e-06, 8.6054e-07],\n", - " [ 4.0866e-06, 7.2345e-06, 8.8215e-06],\n", - " [ 6.9551e-06, 1.0017e-05, 1.3247e-05]],\n", - " \n", - " [[-2.5816e-06, -2.7604e-06, -1.3150e-06],\n", - " [-3.9279e-06, -8.3931e-06, -5.7220e-06],\n", - " [ 2.4009e-06, -8.4937e-07, -3.4794e-06]],\n", - " \n", - " [[-2.2277e-06, 2.2072e-06, 2.2706e-06],\n", - " [ 9.3877e-06, 7.9777e-06, 6.1207e-06],\n", - " [ 6.4953e-06, 3.3351e-06, 6.1579e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 4.6939e-06, 3.7625e-07, 8.5309e-07],\n", - " [ 3.8845e-06, 2.5900e-06, 4.2645e-06],\n", - " [ 4.0168e-06, 2.0415e-06, 2.6468e-06]],\n", - " \n", - " [[-2.2938e-06, -4.3958e-06, -4.1798e-06],\n", - " [-3.8892e-06, -5.9064e-06, -6.6822e-06],\n", - " [-6.4662e-06, -9.3188e-06, -9.6066e-06]],\n", - " \n", - " [[ 5.9549e-06, 1.3746e-06, -1.5218e-06],\n", - " [ 1.6745e-06, -3.0203e-06, -4.0531e-06],\n", - " [ 1.2647e-06, 3.9525e-06, 6.7502e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.3013e-06, -4.0922e-06, -8.0075e-06],\n", - " [-5.6997e-06, -4.4364e-06, -8.7935e-06],\n", - " [-6.7540e-06, -6.8489e-06, -9.5926e-06]],\n", - " \n", - " [[ 4.8894e-06, 4.2394e-06, 6.2529e-06],\n", - " [ 1.3988e-06, 3.6657e-06, 5.0766e-06],\n", - " [-1.3574e-06, -1.4156e-07, -1.6112e-07]],\n", - " \n", - " [[-5.0478e-06, -9.2871e-06, -1.1311e-05],\n", - " [-5.6028e-06, -5.2424e-06, -5.3756e-06],\n", - " [-4.7777e-07, 4.3213e-07, -2.4918e-06]]],\n", - " \n", - " \n", - " [[[ 7.4701e-06, 7.1507e-06, 8.0178e-06],\n", - " [ 9.0674e-06, 9.0469e-06, 1.2387e-05],\n", - " [ 1.1539e-05, 1.2392e-05, 1.3046e-05]],\n", - " \n", - " [[ 4.3814e-06, 1.9222e-06, -7.3537e-06],\n", - " [-7.3761e-07, 4.0121e-06, -5.2750e-06],\n", - " [ 2.7195e-06, 2.8834e-06, -2.5444e-06]],\n", - " \n", - " [[-6.5621e-06, -2.8647e-06, -8.0653e-07],\n", - " [-6.2957e-07, 3.0268e-07, -2.6077e-08],\n", - " [-1.0600e-05, -1.1221e-05, -1.0191e-05]],\n", - " \n", - " ...,\n", - " \n", - " [[ 6.1952e-06, 9.4734e-06, 6.4820e-06],\n", - " [ 5.0750e-06, 7.7318e-06, 6.7148e-06],\n", - " [ 4.1593e-06, 5.5018e-06, 2.7809e-06]],\n", - " \n", - " [[ 1.4715e-06, 3.4086e-06, -1.9111e-06],\n", - " [-1.3251e-05, -8.5169e-06, -8.4005e-06],\n", - " [-8.0657e-06, -4.1851e-06, 3.4366e-07]],\n", - " \n", - " [[ 3.8259e-06, -2.9833e-06, -2.4363e-06],\n", - " [ 4.2394e-06, 1.9409e-06, -4.7944e-06],\n", - " [ 6.5044e-06, 5.3942e-06, 4.4182e-06]]],\n", - " \n", - " \n", - " [[[-2.6450e-07, -1.3839e-06, 2.3320e-06],\n", - " [ 1.5227e-06, 7.0408e-07, 1.7062e-06],\n", - " [ 7.8231e-08, 2.5351e-06, 2.8536e-06]],\n", - " \n", - " [[-1.3914e-06, -1.3160e-06, -2.5891e-06],\n", - " [-1.6969e-06, -3.2715e-06, -6.1244e-06],\n", - " [-2.7679e-06, 2.2212e-07, -2.8824e-06]],\n", - " \n", - " [[-8.6986e-07, 2.3621e-07, -2.0363e-06],\n", - " [ 1.3188e-06, 9.1437e-06, 6.4708e-06],\n", - " [ 3.9823e-06, 3.4459e-06, -3.5204e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 8.3633e-07, 4.0755e-06, -1.2666e-07],\n", - " [ 2.4959e-07, 5.6513e-06, 4.0494e-06],\n", - " [ 2.1867e-06, 5.2229e-06, 5.8115e-06]],\n", - " \n", - " [[-2.2780e-06, 3.0920e-07, 2.5649e-06],\n", - " [-4.4983e-06, -6.8564e-06, -2.4103e-06],\n", - " [-4.7367e-06, -7.6070e-06, -5.9633e-06]],\n", - " \n", - " [[-8.4396e-06, -3.3081e-06, 9.9093e-07],\n", - " [-9.4809e-07, 4.1630e-06, -4.8522e-07],\n", - " [-2.9858e-06, 4.8093e-06, -1.0617e-07]]]], device='cuda:0'),\n", - " tensor([ 3.4571e-06, -8.9407e-07, 1.5497e-06, -4.2915e-06, 3.5763e-07,\n", - " 1.0729e-06, 8.3447e-07, -4.7684e-06, -1.0133e-06, 7.1526e-07,\n", - " 3.6955e-06, 4.7684e-07, 9.5367e-07, -6.5565e-07, 2.5034e-06,\n", - " 9.5367e-07, 9.5367e-07, 0.0000e+00, -5.6028e-06, -3.3379e-06,\n", - " -1.4305e-06, 1.3113e-06, -1.0133e-06, 7.1526e-07, 9.5367e-07,\n", - " 2.9802e-06, -1.9073e-06, -1.9073e-06, 1.5497e-06, 2.3842e-07,\n", - " -9.5367e-07, 4.2915e-06, -3.4571e-06, 2.3842e-06, -3.6955e-06,\n", - " 2.5034e-06, 6.7949e-06, -1.9073e-06, -6.5565e-07, -8.9407e-07,\n", - " 3.2187e-06, 8.3447e-07, -8.9407e-07, -1.2517e-06, -5.9605e-07,\n", - " 1.5497e-06, 1.1921e-06, 4.0531e-06, 2.3842e-06, -1.4305e-06,\n", - " 4.7684e-07, -1.8477e-06, -4.0531e-06, 3.5763e-07, -2.0862e-06,\n", - " 2.9802e-06, 1.5497e-06, -1.1921e-07, 2.1458e-06, -4.8280e-06,\n", - " 9.5367e-07, 3.4571e-06, 7.1526e-07, 0.0000e+00], device='cuda:0'),\n", - " tensor([ 1.9688e-06, -6.2133e-07, -1.2541e-06, -3.2627e-06, 4.4054e-07,\n", - " -2.1605e-06, 6.9847e-07, 1.0742e-06, 3.2157e-07, -6.0075e-07,\n", - " 9.1122e-08, 1.5464e-06, -2.7741e-07, 2.3383e-06, -1.9613e-06,\n", - " 1.3622e-06, 2.9484e-06, 1.3717e-06, 5.5811e-07, 8.9026e-07,\n", - " 1.6398e-07, -9.3554e-07, -7.5569e-08, -5.7897e-07, 1.9944e-06,\n", - " -8.7225e-09, 9.2347e-08, -2.3372e-06, 1.5234e-06, -3.1647e-06,\n", - " 3.8301e-08, 1.9056e-06, -5.5854e-08, -6.9428e-08, -4.9286e-07,\n", - " -3.7826e-06, 2.9969e-06, 3.6108e-06, 1.9197e-06, 3.2961e-07,\n", - " -9.1902e-07, -1.1679e-07, -8.4645e-07, -1.3577e-06, -1.1488e-06,\n", - " 2.4970e-06, -1.8767e-06, 1.5445e-06, 1.0123e-08, 7.8204e-07,\n", - " 1.1457e-06, 2.8526e-07, 1.4468e-06, -1.2701e-06, -1.0599e-08,\n", - " 4.3910e-06, 2.6940e-06, 1.2874e-06, -1.9065e-07, 7.5306e-07,\n", - " 4.4734e-07, -7.6610e-07, -2.6200e-06, 1.1360e-06], device='cuda:0'),\n", - " tensor([[[[ 1.3500e-06, 1.2200e-07, 3.0696e-06],\n", - " [ 2.9905e-06, 5.2340e-07, 2.3963e-06],\n", - " [ 8.2888e-07, -5.2084e-07, -6.1002e-07]],\n", - " \n", - " [[ 3.9884e-06, 2.9132e-06, 1.0747e-06],\n", - " [ 2.7344e-06, 2.6971e-06, 2.7195e-06],\n", - " [ 2.9132e-06, 3.6694e-06, -6.4285e-07]],\n", - " \n", - " [[-1.7346e-06, 4.1723e-07, 4.0587e-06],\n", - " [-6.7614e-07, -1.5125e-06, -6.2585e-07],\n", - " [ 7.1153e-07, 2.2687e-06, 2.1700e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.9116e-07, 1.4678e-06, 6.1374e-07],\n", - " [-1.2890e-06, 1.2275e-06, 3.4124e-06],\n", - " [ 2.3432e-06, 3.4329e-06, 2.3385e-06]],\n", - " \n", - " [[-1.4752e-06, 1.1772e-06, -2.5276e-06],\n", - " [ 5.6624e-07, 1.1695e-06, 1.7881e-07],\n", - " [ 1.2927e-06, 1.1288e-06, -1.4286e-06]],\n", - " \n", - " [[ 1.1679e-06, 1.4435e-07, 3.9265e-06],\n", - " [ 2.1663e-06, -1.0460e-06, 4.5449e-07],\n", - " [-4.1910e-07, -7.6974e-07, 1.1176e-07]]],\n", - " \n", - " \n", - " [[[-2.2613e-06, 4.1984e-06, 7.7821e-06],\n", - " [-8.1845e-06, -2.6878e-06, 5.1968e-07],\n", - " [-1.8049e-06, -1.2293e-06, -1.1222e-06]],\n", - " \n", - " [[ 3.8184e-07, -2.9802e-08, -1.6913e-06],\n", - " [ 5.6252e-07, 4.9546e-07, 3.0547e-07],\n", - " [ 3.8184e-08, -3.3155e-07, -2.2724e-07]],\n", - " \n", - " [[ 2.3842e-07, 1.1660e-06, 3.8650e-07],\n", - " [ 2.4214e-08, -3.2783e-06, 1.6671e-07],\n", - " [ 1.2955e-06, 2.0154e-06, 3.7756e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 9.8255e-07, 8.3819e-07, -5.6066e-07],\n", - " [ 1.3076e-06, 1.5222e-06, -2.8312e-07],\n", - " [-1.8626e-07, 5.9418e-07, -1.0990e-06]],\n", - " \n", - " [[ 6.8098e-06, 5.5621e-06, 7.4308e-06],\n", - " [ 3.7542e-06, 5.0794e-06, 4.3251e-06],\n", - " [ 5.2527e-07, 3.4962e-06, 1.3225e-06]],\n", - " \n", - " [[ 2.9802e-07, -2.0154e-06, -7.6368e-08],\n", - " [-3.1954e-06, -5.3719e-06, -5.5507e-07],\n", - " [-3.7104e-06, -4.5672e-06, -9.2573e-07]]],\n", - " \n", - " \n", - " [[[ 3.9330e-06, 8.7973e-06, 7.0743e-06],\n", - " [ 1.7071e-06, 5.0981e-06, 4.6827e-06],\n", - " [ 8.5682e-08, 1.8757e-06, 4.1388e-06]],\n", - " \n", - " [[ 6.1234e-06, 5.3542e-06, 2.7977e-06],\n", - " [ 3.6303e-06, 1.1027e-06, 5.7684e-07],\n", - " [ 2.7050e-06, 1.7211e-06, -8.3866e-07]],\n", - " \n", - " [[-2.6878e-06, 1.0747e-06, 1.2014e-06],\n", - " [-1.4827e-06, 1.5181e-06, 3.2000e-06],\n", - " [-3.1441e-06, -1.6037e-06, 2.7474e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 5.6326e-06, 6.5546e-06, 1.6820e-06],\n", - " [ 7.3407e-06, 5.3002e-06, 5.2759e-06],\n", - " [ 1.5609e-06, 7.1991e-07, 3.4682e-06]],\n", - " \n", - " [[-2.6142e-06, 5.8115e-07, -1.1213e-06],\n", - " [-7.7300e-07, 3.6694e-07, -2.5667e-06],\n", - " [ 1.0878e-06, -2.0014e-06, 1.0952e-06]],\n", - " \n", - " [[-5.3458e-06, -8.8662e-06, -2.7549e-06],\n", - " [-8.3745e-06, -1.0052e-05, -6.9044e-06],\n", - " [-5.1633e-06, -6.5006e-06, -4.2170e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 5.0134e-07, 1.0005e-06, -2.4773e-06],\n", - " [-6.4261e-07, -2.1439e-06, -3.3600e-06],\n", - " [-1.1194e-06, -4.9695e-06, -2.9542e-06]],\n", - " \n", - " [[-2.8610e-06, -3.3081e-06, -4.3809e-06],\n", - " [-3.2871e-06, -3.3900e-06, -2.3693e-06],\n", - " [-2.8647e-06, -2.7027e-06, 1.9558e-08]],\n", - " \n", - " [[ 7.8604e-07, 1.6727e-06, 1.3497e-06],\n", - " [ 2.8741e-06, 2.9523e-06, 3.1702e-06],\n", - " [ 1.6764e-07, -1.0077e-06, -2.3884e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.7162e-06, -5.6177e-06, -5.4459e-06],\n", - " [-3.8911e-06, -4.2245e-06, -4.3474e-06],\n", - " [-2.5928e-06, -2.6263e-06, -2.4382e-06]],\n", - " \n", - " [[ 2.7148e-06, -2.7232e-06, -2.0026e-06],\n", - " [-6.5565e-07, -1.7043e-06, -1.5572e-06],\n", - " [-1.8515e-06, 1.4100e-06, 5.3877e-07]],\n", - " \n", - " [[ 1.3523e-06, 9.9745e-07, -7.4506e-09],\n", - " [ 2.1365e-06, 4.2152e-06, 2.2575e-06],\n", - " [ 3.7495e-06, 1.8664e-06, 6.2212e-07]]],\n", - " \n", - " \n", - " [[[-8.6799e-07, -3.1628e-06, -4.0811e-06],\n", - " [ 1.2852e-06, -1.0431e-06, -9.0711e-07],\n", - " [ 3.3863e-06, 3.0175e-06, 3.0771e-06]],\n", - " \n", - " [[-6.7987e-07, 1.7863e-06, 1.1083e-06],\n", - " [-2.3972e-06, 3.3528e-07, -3.6508e-07],\n", - " [-2.0787e-06, -1.5255e-06, -2.6133e-06]],\n", - " \n", - " [[ 2.2016e-06, -2.4773e-07, 2.3283e-07],\n", - " [ 1.2103e-06, -9.0897e-07, 8.4937e-07],\n", - " [ 2.2613e-06, 1.6391e-06, 1.3802e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.3188e-06, 2.7479e-06, 2.4363e-06],\n", - " [ 1.2405e-06, 9.2946e-07, 1.3141e-06],\n", - " [ 1.1548e-07, 8.8802e-07, -1.2480e-07]],\n", - " \n", - " [[ 2.5444e-06, 1.1791e-06, -1.6605e-06],\n", - " [-2.9057e-07, 9.2015e-07, -2.0973e-06],\n", - " [-3.9134e-06, -2.4214e-07, 6.1002e-07]],\n", - " \n", - " [[ 2.8210e-06, 5.8301e-07, 7.4506e-07],\n", - " [ 1.6801e-06, 3.4980e-06, 1.7053e-06],\n", - " [ 2.5257e-06, 4.4499e-06, 2.0787e-06]]],\n", - " \n", - " \n", - " [[[ 7.7486e-07, 4.3772e-07, -6.1095e-07],\n", - " [ 4.6249e-06, 2.3022e-06, 2.0750e-06],\n", - " [ 4.8594e-06, 8.4843e-07, 1.3150e-06]],\n", - " \n", - " [[ 1.2554e-06, 1.5832e-06, 1.3420e-06],\n", - " [ 1.8775e-06, 2.5481e-06, 1.8846e-06],\n", - " [ 5.6103e-06, 3.4468e-06, 3.1404e-06]],\n", - " \n", - " [[ 2.4829e-06, 3.9060e-06, 4.3884e-06],\n", - " [-2.6338e-06, -8.9966e-07, 2.4522e-06],\n", - " [-1.1064e-06, -7.6788e-07, 2.1420e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.7374e-06, 7.2364e-07, 5.4017e-08],\n", - " [ 1.1660e-06, 2.5034e-06, 2.1076e-06],\n", - " [ 1.8803e-06, -7.2457e-07, -2.2762e-06]],\n", - " \n", - " [[ 3.2075e-06, 2.1812e-06, -2.8126e-06],\n", - " [ 4.7423e-06, 2.4922e-06, -2.6934e-06],\n", - " [ 2.1067e-06, -1.0394e-06, 2.6403e-07]],\n", - " \n", - " [[ 1.1399e-06, -2.1365e-06, -1.2193e-06],\n", - " [ 5.6333e-07, -1.8980e-06, 5.1130e-07],\n", - " [ 1.0626e-06, 2.8266e-07, -6.1840e-07]]]], device='cuda:0'),\n", - " tensor([ 9.5367e-07, 7.1526e-07, -1.1325e-06, 1.1921e-06, -7.7486e-07,\n", - " 1.1921e-07, 1.3113e-06, 1.1921e-07, -1.1921e-07, 4.7684e-07,\n", - " -1.4901e-06, 2.8610e-06, -7.1526e-07, -1.1921e-07, -1.7881e-06,\n", - " 2.0266e-06, 2.3842e-07, -3.4571e-06, -4.7684e-07, -1.0729e-06,\n", - " -1.0133e-06, -2.2650e-06, -1.1921e-06, 0.0000e+00, -1.0729e-06,\n", - " -7.7486e-07, -8.3447e-07, -4.1723e-07, 1.5497e-06, 1.1921e-07,\n", - " -1.3113e-06, 3.5763e-07, -7.7486e-07, -1.6689e-06, 5.9605e-07,\n", - " -5.9605e-07, -9.5367e-07, 4.4107e-06, 1.1921e-07, 2.9802e-06,\n", - " 9.5367e-07, 1.1921e-06, -2.9802e-07, -1.5497e-06, -2.1458e-06,\n", - " -1.4901e-06, -1.6689e-06, -1.7285e-06, 5.9605e-07, -2.9802e-07,\n", - " 3.5763e-07, -2.3842e-07, 1.4305e-06, -1.7285e-06, -8.3447e-07,\n", - " -7.1526e-07, -5.3644e-07, 1.4305e-06, -7.1526e-07, 0.0000e+00,\n", - " 1.6689e-06, -1.2517e-06, -4.8280e-06, 1.6689e-06, 2.5034e-06,\n", - " 1.1921e-07, 1.6689e-06, 3.2187e-06, -4.7684e-07, -7.7486e-07,\n", - " 1.0729e-06, -1.6093e-06, 1.3113e-06, -4.1723e-07, -2.6226e-06,\n", - " 1.3113e-06, -1.5497e-06, 2.3842e-07, -1.6093e-06, 3.5763e-07,\n", - " 5.9605e-07, 5.9605e-07, -5.3644e-07, 1.1921e-07, 1.1921e-06,\n", - " 9.5367e-07, 1.9073e-06, -6.5565e-07, 1.9073e-06, -4.7684e-07,\n", - " 8.3447e-07, 9.5367e-07, -5.9605e-08, -2.3842e-07, -5.3644e-07,\n", - " 4.1723e-06, -1.2517e-06, -1.7881e-07, -4.7684e-07, 8.3447e-07,\n", - " 1.6689e-06, -5.9605e-08, 2.9802e-06, 3.5763e-07, 3.4571e-06,\n", - " -2.9802e-07, 8.3447e-07, 1.6689e-06, -3.0398e-06, -7.1526e-07,\n", - " 1.7881e-06, -7.7486e-07, 1.0729e-06, 1.3113e-06, -1.7881e-07,\n", - " -2.0862e-06, 3.0994e-06, -1.6689e-06, -1.0729e-06, 4.7684e-07,\n", - " -8.9407e-07, -3.5763e-06, 5.9605e-07, -2.8014e-06, 1.4305e-06,\n", - " -5.3644e-07, 7.1526e-07, -1.5497e-06], device='cuda:0'),\n", - " tensor([ 1.9497e-06, 1.1289e-06, -1.5738e-06, 1.1510e-06, -7.4326e-08,\n", - " 3.8126e-07, 9.6900e-07, -3.7402e-07, 1.4467e-07, 1.2582e-06,\n", - " -7.9345e-07, 1.5984e-06, -6.9576e-07, 4.6673e-07, -1.3461e-06,\n", - " 1.0145e-06, 3.3077e-07, -1.5204e-06, -1.5039e-06, -4.2089e-07,\n", - " 1.8937e-07, -1.3914e-06, -1.0388e-07, 1.1274e-06, -1.5774e-06,\n", - " 6.0853e-07, 9.5481e-08, -8.7090e-07, 1.2509e-06, 2.7946e-07,\n", - " -2.0569e-06, -1.3727e-07, -1.4220e-06, -1.2110e-06, 3.7401e-07,\n", - " 2.3480e-07, 1.2070e-06, 1.2229e-06, -2.7928e-07, -4.0546e-07,\n", - " -2.9544e-07, -5.3795e-07, 5.3077e-07, -9.8826e-07, -9.3950e-07,\n", - " -2.1373e-07, -2.0103e-06, -6.8640e-07, 1.4187e-06, 1.6758e-06,\n", - " 3.7920e-07, 3.1582e-07, 1.3419e-06, -2.7509e-06, -1.6658e-06,\n", - " -6.2629e-08, 2.2001e-06, 3.7348e-08, -2.7134e-07, -3.5544e-07,\n", - " -6.9813e-07, -3.3215e-07, -3.3120e-06, 1.3260e-06, 2.2269e-06,\n", - " -4.2576e-07, 1.1992e-06, 1.6864e-06, -2.6903e-07, 6.8447e-07,\n", - " -4.4472e-07, -6.7873e-07, 1.7654e-06, -5.0672e-07, -1.7452e-06,\n", - " 6.8183e-07, -9.6769e-07, 8.3157e-08, -2.2955e-06, 3.7853e-07,\n", - " 1.3297e-06, -1.2752e-06, -2.1363e-06, 1.3164e-06, 1.2520e-06,\n", - " 1.6037e-06, 1.8909e-07, -2.1382e-06, 2.2903e-06, -1.8987e-06,\n", - " -1.6115e-07, -3.2639e-07, -4.6660e-07, -4.4684e-08, -5.2856e-07,\n", - " 1.3377e-06, -1.8117e-06, 1.6422e-07, -1.4720e-06, 5.7671e-07,\n", - " 8.7481e-07, -4.3880e-08, 3.1518e-06, 2.3125e-07, 1.3741e-06,\n", - " 3.3694e-07, 8.4926e-07, 7.3630e-07, -1.2645e-06, -1.1614e-06,\n", - " 9.5084e-07, 4.2168e-07, -8.2645e-08, 1.5115e-06, -6.6022e-07,\n", - " -2.0173e-06, 2.9759e-06, -5.2929e-07, -3.9942e-07, -1.1476e-06,\n", - " 1.3308e-06, -1.7744e-06, 1.3992e-06, -1.0328e-06, 7.9348e-07,\n", - " -1.5275e-07, 1.2827e-06, -3.9466e-08], device='cuda:0'),\n", - " tensor([[[[ 2.4680e-07, 4.6026e-06, 2.6245e-06],\n", - " [-2.0801e-06, 1.1120e-06, -1.1115e-06],\n", - " [-1.0098e-06, -1.0179e-06, -2.1886e-07]],\n", - " \n", - " [[ 2.8926e-06, 2.9979e-06, -2.0643e-06],\n", - " [-1.0729e-06, 3.1553e-06, -1.5851e-06],\n", - " [ 3.2037e-07, -1.2163e-06, 2.9933e-06]],\n", - " \n", - " [[ 3.0659e-06, 1.5637e-06, 3.5544e-06],\n", - " [-1.6931e-06, -3.0175e-07, 1.8347e-06],\n", - " [ 7.9279e-07, 1.6730e-06, 4.8093e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.2526e-06, 1.8943e-06, 4.0680e-06],\n", - " [ 2.8886e-06, 3.4068e-06, 3.6228e-06],\n", - " [ 2.1234e-07, -3.3248e-07, -3.8184e-07]],\n", - " \n", - " [[-1.9120e-06, -1.6149e-06, -2.4028e-06],\n", - " [-7.1675e-06, -6.6552e-06, -5.9269e-06],\n", - " [-6.1579e-06, -5.2690e-06, -4.8615e-06]],\n", - " \n", - " [[ 1.5516e-06, 5.7819e-06, 2.6170e-06],\n", - " [-2.7278e-06, 2.7381e-07, -3.9116e-08],\n", - " [ 2.0303e-07, 2.9979e-06, 1.3260e-06]]],\n", - " \n", - " \n", - " [[[ 4.7218e-07, 1.6131e-06, 2.5639e-06],\n", - " [ 1.8110e-06, -9.3691e-07, 2.1365e-06],\n", - " [ 1.7509e-06, -1.2740e-06, 1.6037e-06]],\n", - " \n", - " [[ 1.8775e-06, 1.7210e-06, -6.8918e-07],\n", - " [ 4.1164e-07, 3.7253e-09, 2.5341e-06],\n", - " [ 1.9781e-06, 4.6599e-06, -6.6776e-07]],\n", - " \n", - " [[-4.1351e-07, -8.7544e-07, 2.8964e-06],\n", - " [ 3.0212e-06, -1.5534e-06, -2.7008e-07],\n", - " [-1.7309e-06, -4.2263e-06, -1.5777e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.2815e-06, 8.4874e-09, 1.0431e-06],\n", - " [ 2.1663e-06, 1.8775e-06, -3.3388e-07],\n", - " [ 1.5525e-06, 2.8163e-06, 2.3199e-06]],\n", - " \n", - " [[ 2.3884e-06, 1.1921e-07, -3.2308e-06],\n", - " [-2.3060e-06, -4.2263e-06, -5.4501e-06],\n", - " [-1.4587e-07, -3.3388e-06, -3.0622e-06]],\n", - " \n", - " [[ 2.0054e-06, 1.7574e-06, 4.5728e-06],\n", - " [-2.2352e-06, 7.8976e-07, -3.8370e-07],\n", - " [-8.8010e-07, -1.9819e-06, 8.9407e-08]]],\n", - " \n", - " \n", - " [[[ 2.3488e-06, -9.7603e-07, -1.6429e-06],\n", - " [-7.7672e-07, -6.8918e-08, -3.3714e-07],\n", - " [ 2.3842e-07, 1.9353e-06, 3.1004e-06]],\n", - " \n", - " [[ 3.0510e-06, -1.1161e-06, 6.8452e-07],\n", - " [ 1.9185e-06, 2.9234e-06, 8.4098e-07],\n", - " [-5.9791e-07, -2.1700e-07, -4.7460e-06]],\n", - " \n", - " [[ 1.0878e-06, 2.7600e-06, 3.9805e-06],\n", - " [ 2.1881e-06, -1.3951e-06, 6.2445e-07],\n", - " [-7.3016e-07, -1.1176e-07, 2.2575e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.1814e-06, 1.2321e-06, -5.3924e-07],\n", - " [ 2.7753e-06, 2.8284e-06, 2.1269e-06],\n", - " [ 2.4736e-06, 2.1348e-06, 2.4103e-06]],\n", - " \n", - " [[-1.5609e-06, 1.8626e-06, -7.6368e-08],\n", - " [ 4.3688e-06, 3.3472e-06, 4.1262e-06],\n", - " [ 5.0850e-07, -3.8820e-06, -2.6021e-06]],\n", - " \n", - " [[ 1.6261e-06, -1.0626e-06, 2.7325e-06],\n", - " [ 2.5965e-06, 2.0191e-06, 2.8480e-06],\n", - " [ 5.8357e-06, 9.9838e-07, 4.4331e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 3.0976e-06, 4.6566e-08, 1.1427e-06],\n", - " [ 6.5118e-06, 4.4201e-06, -7.0222e-07],\n", - " [ 2.4866e-06, -1.8473e-06, 3.3900e-07]],\n", - " \n", - " [[-1.9744e-06, -2.1625e-06, -2.9802e-07],\n", - " [ 1.2815e-06, -2.1625e-06, -7.7859e-07],\n", - " [-1.4482e-06, -2.7530e-06, 1.8626e-07]],\n", - " \n", - " [[ 6.0536e-07, 9.3132e-07, 2.7763e-06],\n", - " [ 2.0862e-06, 2.4028e-07, 2.0564e-06],\n", - " [ 1.1496e-06, -3.9954e-06, 5.9186e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.1162e-06, -4.3074e-08, 1.2554e-06],\n", - " [ 2.6943e-06, 3.1991e-06, 1.4352e-06],\n", - " [ 3.5805e-06, 2.6580e-06, 3.9460e-06]],\n", - " \n", - " [[-1.7509e-07, 1.7509e-06, -6.3051e-07],\n", - " [-3.9740e-06, -2.1160e-06, -3.6712e-06],\n", - " [-3.3956e-06, -2.1229e-06, -2.9244e-06]],\n", - " \n", - " [[ 3.5875e-06, 1.1255e-06, -1.6745e-06],\n", - " [ 4.9621e-06, 3.9906e-06, 2.7381e-06],\n", - " [ 2.0429e-06, 2.0508e-06, 4.7341e-06]]],\n", - " \n", - " \n", - " [[[-2.6524e-06, -1.4914e-06, 2.1979e-07],\n", - " [-5.1348e-06, -4.7088e-06, 4.2282e-07],\n", - " [-4.5355e-06, -1.0580e-06, -1.6559e-06]],\n", - " \n", - " [[ 6.8732e-07, 4.2096e-07, -5.9791e-07],\n", - " [ 6.1095e-07, 1.9041e-06, 1.8552e-06],\n", - " [-1.0226e-06, 4.6175e-06, -4.5542e-07]],\n", - " \n", - " [[ 4.9528e-06, 1.2238e-06, 1.1399e-06],\n", - " [ 4.1332e-06, 1.4482e-06, -2.1122e-06],\n", - " [ 4.2338e-06, 4.3996e-06, 2.0410e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.0361e-07, 5.1782e-07, 4.7870e-07],\n", - " [-6.4261e-07, -2.7008e-06, -1.0822e-06],\n", - " [-4.6659e-07, -3.1348e-06, -2.0862e-06]],\n", - " \n", - " [[-1.3588e-06, -1.0896e-06, -1.3774e-06],\n", - " [ 1.6324e-06, -5.7928e-07, -8.7079e-08],\n", - " [-2.2040e-06, -1.5069e-06, -4.3772e-06]],\n", - " \n", - " [[ 2.3583e-06, -5.2713e-07, 3.3895e-06],\n", - " [-8.3144e-07, -4.1337e-06, -8.1211e-07],\n", - " [-1.9405e-06, -7.7821e-06, -1.9437e-06]]],\n", - " \n", - " \n", - " [[[-3.9143e-06, -2.2687e-06, -2.1812e-06],\n", - " [-3.8780e-06, -1.6596e-06, -4.6194e-06],\n", - " [-1.9256e-06, -1.9111e-06, -3.1497e-06]],\n", - " \n", - " [[ 3.3490e-06, 2.8852e-06, 3.9299e-06],\n", - " [-4.9733e-07, 1.5199e-06, 3.6880e-06],\n", - " [-1.0580e-06, 3.9656e-06, 1.1660e-06]],\n", - " \n", - " [[ 5.7556e-07, -2.6373e-06, -3.8473e-06],\n", - " [ 4.1127e-06, 2.2128e-06, 3.1032e-06],\n", - " [ 2.3693e-06, -1.9651e-07, -1.1791e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.6354e-06, -9.6299e-07, 1.5609e-06],\n", - " [-2.6338e-06, -2.8769e-06, -2.6245e-06],\n", - " [-1.0693e-06, -1.5106e-06, -4.6603e-06]],\n", - " \n", - " [[ 3.0603e-06, 1.5721e-06, -1.9213e-06],\n", - " [ 1.6578e-07, -2.5935e-06, -3.3109e-06],\n", - " [-8.9407e-07, -1.7101e-07, -1.2284e-06]],\n", - " \n", - " [[-2.3362e-06, 3.8184e-07, -3.1351e-06],\n", - " [ 3.7476e-06, -1.2461e-06, -1.1781e-07],\n", - " [-1.2573e-06, -2.1867e-06, -2.1167e-06]]]], device='cuda:0'),\n", - " tensor([-1.9073e-06, -1.1325e-06, -5.3644e-07, 8.3447e-07, 3.5763e-07,\n", - " -1.3709e-06, 3.5763e-07, 9.5367e-07, 8.3447e-07, -5.9605e-08,\n", - " -2.2054e-06, -3.2783e-06, 2.7418e-06, -8.3447e-07, -1.3709e-06,\n", - " -8.9407e-07, -1.4901e-06, 2.3842e-07, 3.5763e-07, 2.7418e-06,\n", - " 1.3113e-06, -1.7881e-07, 5.9605e-07, 4.7684e-07, -1.0729e-06,\n", - " -1.1921e-07, 1.1921e-06, 0.0000e+00, 2.9802e-06, -2.9802e-07,\n", - " -2.2054e-06, -1.1921e-06, 9.5367e-07, -2.0862e-06, 1.6689e-06,\n", - " -1.3709e-06, -1.1921e-07, 0.0000e+00, 7.1526e-07, -7.1526e-07,\n", - " 1.0729e-06, 8.3447e-07, 1.7881e-06, -1.0133e-06, -1.1921e-06,\n", - " -2.6226e-06, 5.9605e-07, 9.5367e-07, 4.7684e-07, 2.0266e-06,\n", - " -5.9605e-07, -2.0266e-06, -6.5565e-07, -8.3447e-07, -5.3644e-07,\n", - " 3.5763e-07, -3.5763e-06, 1.1921e-06, -1.3709e-06, 4.7684e-07,\n", - " 1.9073e-06, -2.2054e-06, -7.1526e-07, -1.0729e-06, 7.1526e-07,\n", - " 1.7881e-06, -5.9605e-07, -2.9802e-07, 2.3842e-07, 2.3842e-06,\n", - " -3.3975e-06, -1.3113e-06, 2.3842e-07, 0.0000e+00, 3.5763e-07,\n", - " 1.1921e-07, -9.5367e-07, -6.5565e-07, 1.5497e-06, -1.1921e-07,\n", - " 3.5763e-07, -1.4901e-06, 1.1921e-07, -1.2517e-06, 1.9073e-06,\n", - " 1.5497e-06, 1.0729e-06, 1.1921e-06, -5.9605e-07, -1.4901e-06,\n", - " 2.0266e-06, 7.1526e-07, 1.0729e-06, -4.1723e-07, -1.1921e-07,\n", - " 8.3447e-07, 1.4305e-06, -9.5367e-07, 1.4305e-06, -6.5565e-07,\n", - " -8.3447e-07, 7.1526e-07, -1.1921e-06, -1.9073e-06, 1.1921e-07,\n", - " 1.5497e-06, 1.3113e-06, -1.3709e-06, -1.0729e-06, 8.3447e-07,\n", - " -5.9605e-07, 7.1526e-07, 1.7881e-06, 1.7881e-06, 1.6689e-06,\n", - " 1.1921e-06, -2.3842e-07, 2.1458e-06, -1.7881e-06, 4.7684e-07,\n", - " -5.9605e-07, 4.7684e-07, 3.5763e-07, -1.1921e-07, 4.7684e-07,\n", - " 2.0266e-06, 1.1921e-06, 1.6689e-06], device='cuda:0'),\n", - " tensor([-9.1369e-07, -3.0748e-07, -9.0601e-07, 4.1690e-07, -4.7604e-07,\n", - " 8.1218e-07, 2.0112e-06, -8.5575e-07, -4.3124e-07, -9.8559e-07,\n", - " -6.4047e-07, 7.4814e-08, 9.7423e-07, 7.2637e-07, 4.6106e-09,\n", - " -9.8934e-07, -1.1821e-06, 3.3636e-07, 5.9197e-07, -8.3250e-09,\n", - " 4.7365e-07, 3.8908e-07, -7.9900e-07, 1.1213e-07, -5.1611e-07,\n", - " 5.7080e-07, 1.7410e-07, -6.2933e-07, 1.2636e-06, 1.1473e-06,\n", - " -9.1808e-07, -1.5391e-06, -4.5448e-08, 1.3961e-06, -1.2846e-06,\n", - " -6.6418e-07, -7.5253e-08, -6.9444e-07, 1.1273e-06, -3.1001e-07,\n", - " -1.9286e-06, 1.0208e-06, -1.8318e-07, 1.2594e-07, -6.4955e-07,\n", - " -1.6561e-06, -7.6606e-07, -6.6829e-07, -6.2103e-07, 1.2311e-07,\n", - " -1.3618e-06, -1.5188e-06, -1.2815e-06, 6.3298e-07, -1.3043e-06,\n", - " 7.3002e-08, 1.0671e-06, -6.9393e-07, -8.1839e-07, 7.7284e-07,\n", - " 4.2962e-07, 6.4041e-07, -6.6803e-07, -8.9368e-07, -4.7253e-08,\n", - " 6.4242e-07, 1.4340e-07, 2.3820e-07, -2.3032e-07, 1.3455e-07,\n", - " -2.4945e-06, -7.8601e-07, 8.3551e-07, 8.6485e-07, 2.4023e-07,\n", - " 1.7556e-06, -5.7290e-07, 1.5098e-07, 9.2456e-07, 1.1759e-06,\n", - " 2.8615e-07, -2.2824e-06, -2.1893e-06, 2.0725e-06, 9.3870e-07,\n", - " 1.6482e-06, -7.7833e-07, -6.6209e-07, -1.5313e-06, 1.1172e-06,\n", - " 6.2416e-07, 9.9326e-07, 8.7520e-07, 2.3982e-07, -1.4602e-07,\n", - " 2.0822e-06, -7.0897e-07, 1.0941e-07, -1.1862e-06, -1.6575e-06,\n", - " -1.6125e-06, 1.6521e-06, 1.0219e-06, -9.5518e-07, 1.8921e-06,\n", - " 1.2553e-06, 2.3294e-06, -1.0472e-06, -3.4637e-08, 5.1041e-07,\n", - " 1.6033e-06, 1.1330e-06, -2.9018e-07, -8.9494e-07, -6.7316e-07,\n", - " 7.9159e-07, 2.6595e-06, -9.3374e-07, 9.3966e-07, 2.0426e-06,\n", - " -4.1359e-07, -1.1283e-06, 1.1768e-06, -2.5566e-07, -4.6953e-07,\n", - " 9.1875e-07, -6.7674e-08, -3.9833e-07], device='cuda:0'),\n", - " tensor([[[[-4.4843e-06]],\n", - " \n", - " [[-2.4885e-06]],\n", - " \n", - " [[-3.1590e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.0212e-06]],\n", - " \n", - " [[ 1.1697e-06]],\n", - " \n", - " [[ 7.3016e-07]]],\n", - " \n", - " \n", - " [[[-6.4448e-06]],\n", - " \n", - " [[ 3.0175e-06]],\n", - " \n", - " [[ 1.0319e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.3362e-06]],\n", - " \n", - " [[ 3.5390e-06]],\n", - " \n", - " [[ 2.2054e-06]]],\n", - " \n", - " \n", - " [[[-5.7369e-07]],\n", - " \n", - " [[-1.5087e-06]],\n", - " \n", - " [[ 1.6466e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.8056e-07]],\n", - " \n", - " [[ 1.8757e-06]],\n", - " \n", - " [[ 2.0862e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-1.9595e-06]],\n", - " \n", - " [[ 8.1584e-07]],\n", - " \n", - " [[ 4.3865e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.0755e-06]],\n", - " \n", - " [[ 1.7136e-06]],\n", - " \n", - " [[-2.2640e-06]]],\n", - " \n", - " \n", - " [[[-6.8806e-06]],\n", - " \n", - " [[-3.4105e-06]],\n", - " \n", - " [[-5.0068e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.5449e-06]],\n", - " \n", - " [[-4.9472e-06]],\n", - " \n", - " [[ 2.5444e-06]]],\n", - " \n", - " \n", - " [[[-7.2736e-06]],\n", - " \n", - " [[-3.5763e-06]],\n", - " \n", - " [[ 1.6093e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.9600e-06]],\n", - " \n", - " [[-1.2573e-06]],\n", - " \n", - " [[ 1.4901e-07]]]], device='cuda:0'),\n", - " tensor([ 0.0000e+00, -1.0729e-06, -2.9802e-07, 1.3113e-06, -1.0133e-06,\n", - " 3.5763e-07, 6.3181e-06, -1.5497e-06, 3.5763e-07, -1.4305e-06,\n", - " 1.5497e-06, 1.3113e-06, -1.3709e-06, 1.4305e-06, -5.3644e-07,\n", - " 1.1921e-07, -2.5630e-06, -1.7881e-07, 2.2650e-06, -1.3709e-06,\n", - " 1.3113e-06, 1.6689e-06, -5.9605e-08, -2.9802e-07, -1.0133e-06,\n", - " 1.5497e-06, -8.3447e-07, -1.4901e-06, 1.1921e-07, 2.3842e-06,\n", - " -7.7486e-07, -9.5367e-07, -6.5565e-07, 2.9802e-06, -2.9802e-06,\n", - " 1.9073e-06, -1.6093e-06, 7.1526e-07, 9.5367e-07, -1.0729e-06,\n", - " -2.8014e-06, 0.0000e+00, -9.5367e-07, -5.9605e-07, -1.4305e-06,\n", - " 3.5763e-07, -8.9407e-07, -1.0729e-06, -1.0133e-06, 7.1526e-07,\n", - " -1.7881e-06, -3.3975e-06, -2.3842e-07, 4.1723e-06, -5.3644e-07,\n", - " 2.3842e-07, 2.0266e-06, -3.0994e-06, -5.9605e-08, -6.5565e-07,\n", - " -7.1526e-07, 1.6689e-06, 9.5367e-07, 8.3447e-07, -4.7684e-07,\n", - " -4.1723e-07, -2.3842e-07, 4.7684e-07, 8.3447e-07, -1.4305e-06,\n", - " -4.2915e-06, -4.7684e-07, -2.9802e-07, 2.3842e-07, 9.5367e-07,\n", - " 3.5763e-07, 1.1921e-07, 3.5763e-07, 4.7684e-07, 8.3447e-07,\n", - " -1.4305e-06, 1.1921e-06, -1.9073e-06, 5.9605e-07, -3.5763e-07,\n", - " 1.5497e-06, -2.6226e-06, -6.5565e-07, -2.3842e-06, 2.7418e-06,\n", - " -2.6822e-06, 1.1921e-07, 8.3447e-07, 1.1921e-06, -2.3842e-06,\n", - " 2.3842e-07, -1.6093e-06, -1.7881e-07, 2.7418e-06, -2.6226e-06,\n", - " 3.5763e-07, 1.4305e-06, 1.5497e-06, 5.9605e-07, -4.1723e-07,\n", - " 7.1526e-07, 2.3842e-06, -5.9605e-08, -5.9605e-08, 9.5367e-07,\n", - " -7.1526e-07, -1.1921e-06, -4.1723e-07, -7.1526e-07, 9.5367e-07,\n", - " 9.5367e-07, 3.3379e-06, -3.3975e-06, 2.0266e-06, 1.1921e-06,\n", - " 4.7684e-07, 1.3113e-06, 1.1921e-07, -7.1526e-07, -2.1458e-06,\n", - " 9.5367e-07, -1.4901e-06, -5.3644e-07], device='cuda:0'),\n", - " tensor([-9.1369e-07, -3.0748e-07, -9.0601e-07, 4.1690e-07, -4.7604e-07,\n", - " 8.1218e-07, 2.0112e-06, -8.5575e-07, -4.3124e-07, -9.8559e-07,\n", - " -6.4047e-07, 7.4814e-08, 9.7423e-07, 7.2637e-07, 4.6106e-09,\n", - " -9.8934e-07, -1.1821e-06, 3.3636e-07, 5.9197e-07, -8.3250e-09,\n", - " 4.7365e-07, 3.8908e-07, -7.9900e-07, 1.1213e-07, -5.1611e-07,\n", - " 5.7080e-07, 1.7410e-07, -6.2933e-07, 1.2636e-06, 1.1473e-06,\n", - " -9.1808e-07, -1.5391e-06, -4.5448e-08, 1.3961e-06, -1.2846e-06,\n", - " -6.6418e-07, -7.5253e-08, -6.9444e-07, 1.1273e-06, -3.1001e-07,\n", - " -1.9286e-06, 1.0208e-06, -1.8318e-07, 1.2594e-07, -6.4955e-07,\n", - " -1.6561e-06, -7.6606e-07, -6.6829e-07, -6.2103e-07, 1.2311e-07,\n", - " -1.3618e-06, -1.5188e-06, -1.2815e-06, 6.3298e-07, -1.3043e-06,\n", - " 7.3002e-08, 1.0671e-06, -6.9393e-07, -8.1839e-07, 7.7284e-07,\n", - " 4.2962e-07, 6.4041e-07, -6.6803e-07, -8.9368e-07, -4.7253e-08,\n", - " 6.4242e-07, 1.4340e-07, 2.3820e-07, -2.3032e-07, 1.3455e-07,\n", - " -2.4945e-06, -7.8601e-07, 8.3551e-07, 8.6485e-07, 2.4023e-07,\n", - " 1.7556e-06, -5.7290e-07, 1.5098e-07, 9.2456e-07, 1.1759e-06,\n", - " 2.8615e-07, -2.2824e-06, -2.1893e-06, 2.0725e-06, 9.3870e-07,\n", - " 1.6482e-06, -7.7833e-07, -6.6209e-07, -1.5313e-06, 1.1172e-06,\n", - " 6.2416e-07, 9.9326e-07, 8.7520e-07, 2.3982e-07, -1.4602e-07,\n", - " 2.0822e-06, -7.0897e-07, 1.0941e-07, -1.1862e-06, -1.6575e-06,\n", - " -1.6125e-06, 1.6521e-06, 1.0219e-06, -9.5518e-07, 1.8921e-06,\n", - " 1.2553e-06, 2.3294e-06, -1.0472e-06, -3.4637e-08, 5.1041e-07,\n", - " 1.6033e-06, 1.1330e-06, -2.9018e-07, -8.9494e-07, -6.7316e-07,\n", - " 7.9159e-07, 2.6595e-06, -9.3374e-07, 9.3966e-07, 2.0426e-06,\n", - " -4.1359e-07, -1.1283e-06, 1.1768e-06, -2.5566e-07, -4.6953e-07,\n", - " 9.1875e-07, -6.7674e-08, -3.9833e-07], device='cuda:0'),\n", - " tensor([[[[-1.8403e-06, -1.3132e-07, 4.1910e-07],\n", - " [-2.9635e-06, -1.3411e-06, -2.2529e-06],\n", - " [-8.4285e-07, 1.4529e-07, 7.4506e-07]],\n", - " \n", - " [[ 5.9167e-06, 8.3577e-06, 8.6566e-06],\n", - " [ 7.1782e-06, 6.8275e-06, 7.5828e-06],\n", - " [ 6.5733e-06, 4.0652e-06, 3.1553e-06]],\n", - " \n", - " [[-6.9290e-07, 3.1311e-06, 1.1884e-06],\n", - " [ 9.6578e-07, 3.6461e-06, -6.1095e-07],\n", - " [ 1.5199e-06, 6.4112e-06, -1.3048e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.5272e-06, 5.5972e-06, 4.6393e-06],\n", - " [ 5.5131e-06, 4.0464e-06, 2.6692e-06],\n", - " [ 2.1489e-06, 2.7381e-06, 3.7728e-06]],\n", - " \n", - " [[ 1.0245e-07, 3.1572e-06, 2.6356e-06],\n", - " [-1.1148e-06, -9.5367e-07, 2.2911e-06],\n", - " [ 6.4075e-07, 1.5096e-06, 1.5600e-06]],\n", - " \n", - " [[-1.0058e-06, 3.6694e-07, 1.0990e-07],\n", - " [-1.3374e-06, -9.0338e-07, 9.1083e-07],\n", - " [-2.3581e-06, 4.9733e-07, 1.0328e-06]]],\n", - " \n", - " \n", - " [[[ 2.5518e-07, -1.4063e-06, 4.6077e-07],\n", - " [ 3.9674e-07, 8.0466e-07, 1.3337e-06],\n", - " [ 2.9057e-07, -7.0222e-07, -8.3703e-07]],\n", - " \n", - " [[ 1.6950e-06, 1.7611e-06, 1.2666e-07],\n", - " [ 1.8440e-07, 1.3290e-06, 1.8720e-06],\n", - " [-1.5106e-06, -8.8336e-07, 8.0466e-07]],\n", - " \n", - " [[ 1.8068e-07, 2.8946e-06, 1.7304e-06],\n", - " [ 4.0978e-08, -1.3970e-07, -6.2585e-07],\n", - " [-1.1232e-06, -1.4445e-06, -2.1141e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1257e-06, 2.9248e-06, 1.3709e-06],\n", - " [ 9.3691e-07, 6.5798e-07, 2.1011e-06],\n", - " [ 1.4137e-06, 4.0419e-07, 1.0543e-06]],\n", - " \n", - " [[ 1.6298e-06, 1.3770e-06, -1.2852e-07],\n", - " [ 1.7555e-06, 1.1814e-06, -1.8202e-06],\n", - " [ 2.0955e-08, 2.1816e-06, -3.9674e-07]],\n", - " \n", - " [[ 2.2273e-06, 1.3923e-06, -2.3004e-07],\n", - " [-8.5682e-08, -5.4250e-07, 2.1141e-07],\n", - " [ 6.2212e-07, 3.2601e-06, -1.9444e-06]]],\n", - " \n", - " \n", - " [[[-2.2650e-06, -1.8813e-06, -1.7062e-06],\n", - " [-1.2228e-06, -3.1311e-06, -4.2953e-06],\n", - " [ 5.6671e-07, -8.5495e-07, 1.1511e-06]],\n", - " \n", - " [[ 1.8273e-06, 1.5022e-06, 3.2187e-06],\n", - " [-4.1351e-07, -1.2494e-06, 2.5043e-06],\n", - " [-2.4159e-06, -8.0280e-07, -4.9360e-08]],\n", - " \n", - " [[ 7.5251e-07, 1.2759e-06, 1.2228e-06],\n", - " [-3.0324e-06, -2.2631e-07, -2.0005e-06],\n", - " [-1.4156e-06, -9.0525e-07, -3.3826e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.6473e-07, -1.5423e-06, -4.3958e-07],\n", - " [-5.7556e-07, -5.9884e-07, 1.1805e-06],\n", - " [ 1.7841e-06, 1.5851e-06, 6.0163e-07]],\n", - " \n", - " [[-8.3516e-07, -1.1735e-06, -1.9008e-06],\n", - " [-1.9185e-06, -4.0419e-06, -3.7476e-06],\n", - " [-4.4126e-06, -2.8610e-06, -4.2629e-06]],\n", - " \n", - " [[ 1.1767e-06, 2.2892e-06, 8.2795e-07],\n", - " [ 2.2175e-06, 1.0375e-06, -9.6858e-07],\n", - " [ 8.3540e-07, 1.0722e-06, -3.3528e-08]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 1.0282e-06, -5.2713e-07, -2.0633e-06],\n", - " [-1.7202e-06, 2.4233e-06, 1.0896e-06],\n", - " [-8.0280e-07, -1.8077e-06, -2.0942e-07]],\n", - " \n", - " [[-9.2480e-07, -1.5087e-06, -4.6194e-07],\n", - " [ 8.7358e-07, -1.5018e-06, -3.2689e-06],\n", - " [-1.3970e-08, -2.0396e-07, -9.6671e-07]],\n", - " \n", - " [[ 1.3923e-07, 3.7272e-06, 3.3574e-06],\n", - " [ 2.0675e-06, 2.3858e-06, 2.0601e-06],\n", - " [ 9.0152e-07, 1.4515e-06, 4.9919e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.3900e-07, 1.4659e-06, 1.1316e-06],\n", - " [-1.2023e-06, -2.5779e-06, -5.6531e-07],\n", - " [ 6.5472e-07, -1.6801e-06, -1.8135e-06]],\n", - " \n", - " [[-2.1812e-06, -2.6133e-06, 1.5832e-07],\n", - " [-7.8045e-07, -6.9570e-07, -2.2082e-06],\n", - " [-3.7625e-07, -1.0543e-06, -6.4261e-07]],\n", - " \n", - " [[ 7.0059e-07, 8.5309e-07, 4.3586e-07],\n", - " [-1.8883e-06, 1.5269e-06, 3.0082e-06],\n", - " [-1.2715e-06, -7.6368e-08, 8.0094e-07]]],\n", - " \n", - " \n", - " [[[-2.2464e-06, -1.6782e-06, 6.5565e-07],\n", - " [ 9.0338e-07, 1.6345e-07, 6.5099e-07],\n", - " [ 1.4156e-07, 7.2457e-07, 8.1491e-07]],\n", - " \n", - " [[ 1.9912e-06, 4.3586e-07, 1.6659e-07],\n", - " [ 5.0478e-07, 8.0094e-07, -6.9849e-07],\n", - " [ 1.3271e-06, 3.7439e-07, -6.8918e-08]],\n", - " \n", - " [[ 7.0035e-07, 2.6077e-07, 1.3076e-06],\n", - " [ 2.0647e-06, 1.6429e-06, 2.1001e-06],\n", - " [-4.7870e-07, -3.5018e-07, 8.9128e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.3001e-06, 1.3644e-06, -1.8068e-06],\n", - " [ 2.2352e-07, 8.8289e-07, -1.1027e-06],\n", - " [-1.5078e-06, -3.0547e-07, -2.3167e-06]],\n", - " \n", - " [[ 1.9278e-07, 7.2271e-07, 6.4075e-07],\n", - " [-7.1587e-07, -1.4715e-06, -1.0505e-06],\n", - " [-1.5064e-06, -2.9635e-06, -2.0145e-06]],\n", - " \n", - " [[-8.6427e-07, -8.6240e-07, -1.4901e-07],\n", - " [ 1.2480e-07, -9.2946e-07, 4.0047e-07],\n", - " [-1.8906e-06, -1.4938e-06, 5.2899e-07]]],\n", - " \n", - " \n", - " [[[-1.7451e-07, -7.4692e-07, -6.7428e-07],\n", - " [-6.5193e-08, -6.6846e-07, -1.1828e-06],\n", - " [ 7.8697e-08, -9.3225e-07, -2.2352e-08]],\n", - " \n", - " [[ 5.9158e-06, 7.6722e-06, 6.1896e-06],\n", - " [ 2.8582e-06, 3.6582e-06, 5.8301e-06],\n", - " [ 4.4638e-06, 4.3232e-06, 5.6196e-06]],\n", - " \n", - " [[ 5.6624e-07, -1.4286e-06, 5.8115e-07],\n", - " [-6.2399e-08, 6.4820e-07, 8.1956e-07],\n", - " [ 8.2888e-07, -2.3814e-06, -1.1809e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.2815e-06, 3.4645e-07, 1.5977e-06],\n", - " [ 1.6801e-06, 1.0859e-06, 1.1623e-06],\n", - " [ 2.6748e-06, 1.4342e-06, 6.4727e-07]],\n", - " \n", - " [[ 3.0268e-07, 2.2948e-06, 1.1520e-06],\n", - " [ 4.3609e-07, 3.1302e-06, 2.8396e-06],\n", - " [ 6.3051e-07, 1.8162e-07, 1.6438e-06]],\n", - " \n", - " [[-4.7986e-07, -1.3672e-06, -2.6599e-06],\n", - " [ 1.7416e-06, 8.6427e-07, -1.7863e-06],\n", - " [-6.8115e-07, -1.4611e-06, -2.6692e-06]]]], device='cuda:0'),\n", - " tensor([-3.5763e-07, -4.1723e-07, -1.1921e-07, 0.0000e+00, -2.3842e-06,\n", - " 1.1921e-07, -4.7684e-07, -1.1921e-07, 4.7684e-07, 1.1921e-06,\n", - " 0.0000e+00, 8.3447e-07, -4.1723e-07, 1.0729e-06, 4.7684e-07,\n", - " -1.7881e-07, -1.0729e-06, -5.9605e-07, -5.3644e-07, 1.1921e-07,\n", - " -7.1526e-07, -1.1921e-06, -5.9605e-07, 1.1921e-06, 7.1526e-07,\n", - " -4.1723e-07, 5.9605e-07, 2.3842e-07, 8.3447e-07, -1.0133e-06,\n", - " 4.7684e-07, 3.5763e-07, 2.3842e-07, -7.1526e-07, -2.9802e-07,\n", - " -7.7486e-07, -6.5565e-07, -1.5497e-06, -8.3447e-07, -4.1723e-07,\n", - " 8.3447e-07, -1.0729e-06, -2.9802e-07, 7.1526e-07, -1.0133e-06,\n", - " 1.4305e-06, -5.9605e-07, 4.7684e-07, -1.6689e-06, 7.1526e-07,\n", - " -7.7486e-07, 4.7684e-07, 8.3447e-07, 0.0000e+00, 7.1526e-07,\n", - " -2.9802e-07, 1.1921e-07, 4.7684e-07, -2.9802e-07, -2.3842e-07,\n", - " 1.0729e-06, 3.5763e-07, -1.9073e-06, -7.1526e-07, -1.7881e-07,\n", - " -1.6093e-06, 7.1526e-07, -2.3842e-07, -1.0133e-06, 2.3842e-07,\n", - " 9.5367e-07, 3.5763e-07, -5.3644e-07, 1.1921e-06, -5.9605e-07,\n", - " 1.7881e-06, -2.3842e-07, -1.1921e-07, -1.0133e-06, 3.3379e-06,\n", - " 2.3842e-07, -5.9605e-07, 8.3447e-07, -1.3113e-06, 1.1921e-07,\n", - " -5.9605e-08, -8.9407e-07, 7.1526e-07, -1.2517e-06, -7.7486e-07,\n", - " -1.0729e-06, 1.1921e-07, -5.9605e-07, -5.3644e-07, -1.1921e-07,\n", - " -1.3113e-06, -1.0729e-06, -4.1723e-07, 4.7684e-07, -8.3447e-07,\n", - " 2.1458e-06, -3.5763e-07, -8.3447e-07, 2.3842e-07, -8.3447e-07,\n", - " -1.7285e-06, 7.1526e-07, -7.7486e-07, 2.3842e-07, 8.3447e-07,\n", - " 4.7684e-07, -8.3447e-07, 4.7684e-07, 0.0000e+00, 1.6689e-06,\n", - " 1.9073e-06, 5.9605e-07, -8.3447e-07, -1.0133e-06, -8.3447e-07,\n", - " -4.7684e-07, 0.0000e+00, 9.5367e-07, -1.1325e-06, -2.9802e-07,\n", - " -1.0133e-06, -7.7486e-07, -5.9605e-07, 5.9605e-07, -8.3447e-07,\n", - " 4.7684e-07, 1.1921e-06, 1.3113e-06, -2.3842e-06, -2.9802e-07,\n", - " -5.9605e-07, 3.5763e-07, -1.2517e-06, 4.7684e-07, 3.5763e-07,\n", - " 2.3842e-07, 1.1921e-07, 1.4305e-06, 8.3447e-07, 2.3842e-07,\n", - " -2.3842e-07, -8.3447e-07, 7.1526e-07, -1.1921e-07, 1.5497e-06,\n", - " -4.7684e-07, 1.0729e-06, -5.3644e-07, 7.1526e-07, 2.3842e-06,\n", - " -5.9605e-07, 0.0000e+00, 8.3447e-07, 1.1921e-07, 4.7684e-07,\n", - " 5.9605e-07, 4.7684e-07, -1.1325e-06, 2.3842e-07, 1.0729e-06,\n", - " -3.3379e-06, 5.9605e-07, 8.3447e-07, 7.1526e-07, 1.9073e-06,\n", - " 2.5034e-06, 1.1921e-07, -5.3644e-07, 8.3447e-07, 8.3447e-07,\n", - " -5.9605e-08, -3.5763e-07, -7.1526e-07, -4.1723e-07, 3.5763e-07,\n", - " 1.3113e-06, 8.3447e-07, -2.3842e-07, 1.1921e-06, -1.7881e-06,\n", - " 1.4305e-06, -1.0133e-06, 3.5763e-07, -2.9802e-07, -5.9605e-08,\n", - " 1.0729e-06, -7.7486e-07, -1.9670e-06, 4.7684e-07, -1.1921e-06,\n", - " 5.9605e-07, 9.5367e-07, 3.5763e-07, -1.1921e-06, 1.1921e-07,\n", - " 8.3447e-07, -5.9605e-07, 9.5367e-07, 2.3842e-07, 0.0000e+00,\n", - " 3.5763e-07, -1.7881e-07, -2.9802e-07, 8.3447e-07, -5.9605e-07,\n", - " 8.3447e-07, -1.5497e-06, 7.1526e-07, 1.1921e-07, -1.3113e-06,\n", - " 7.1526e-07, -2.3842e-07, -1.1325e-06, -7.7486e-07, 0.0000e+00,\n", - " -4.1723e-07, 2.3842e-07, 1.3113e-06, 1.4305e-06, 1.1921e-07,\n", - " -8.3447e-07, -1.1325e-06, 2.3842e-07, 1.6689e-06, 1.3113e-06,\n", - " -5.9605e-08, -6.5565e-07, 1.0729e-06, 2.3842e-07, 4.7684e-07,\n", - " 2.3842e-07, 0.0000e+00, -5.9605e-08, -6.5565e-07, 9.5367e-07,\n", - " -4.7684e-07, 4.7684e-07, -7.1526e-07, 1.1921e-07, -1.2517e-06,\n", - " 7.1526e-07, 1.1921e-07, -1.1921e-07, -2.3842e-07, 2.3842e-07,\n", - " -1.0729e-06, 2.3842e-07, 8.3447e-07, 5.9605e-07, -5.9605e-08,\n", - " -2.9802e-07], device='cuda:0'),\n", - " tensor([-2.9877e-07, -6.0828e-07, 4.5930e-07, 4.1853e-07, -1.7363e-06,\n", - " 3.8803e-07, -1.8305e-07, 1.8413e-08, 6.1509e-07, 1.2918e-06,\n", - " 2.5392e-07, 3.7740e-07, -6.4319e-07, 5.2017e-07, 2.6308e-07,\n", - " 2.9436e-07, -1.0043e-07, -8.2840e-07, -1.1444e-08, -3.9894e-07,\n", - " -3.8904e-07, -3.0451e-07, -8.8293e-07, 8.9173e-07, 2.6468e-07,\n", - " 3.3383e-07, 8.0802e-07, -1.1930e-06, 1.2235e-06, -7.9587e-07,\n", - " -3.4577e-07, 2.3863e-07, 9.7938e-08, -4.0020e-07, 1.0080e-06,\n", - " -5.2720e-07, -8.3278e-07, -1.1856e-06, -6.2362e-07, -4.3046e-07,\n", - " -4.5586e-08, -3.4174e-07, -8.9879e-07, 7.3194e-07, -1.3142e-06,\n", - " 1.3045e-06, -1.7480e-07, 3.7845e-07, -1.3178e-06, 6.6896e-07,\n", - " -4.2099e-07, -5.4902e-07, 2.7925e-07, 2.6155e-07, 6.6702e-07,\n", - " -6.1461e-07, 4.0904e-07, 8.1861e-07, -7.2904e-07, -2.8011e-07,\n", - " 1.2763e-06, -4.8832e-07, -5.9228e-07, 9.3926e-07, 9.4038e-07,\n", - " -3.8900e-07, 3.5983e-07, -1.0416e-06, -8.7933e-07, 2.8657e-07,\n", - " -4.3863e-07, -1.3785e-07, -5.3276e-07, 1.2505e-06, -3.4932e-07,\n", - " 7.9656e-07, -2.3599e-07, 1.0419e-07, -3.8420e-07, 2.1569e-06,\n", - " -3.4136e-07, -2.7307e-07, -5.2092e-07, -3.3845e-07, 8.4514e-07,\n", - " 1.1931e-07, -6.6160e-08, 1.5253e-07, -1.3723e-06, -3.0826e-07,\n", - " -1.1393e-06, 4.4033e-07, -9.8301e-07, -6.6679e-07, -1.9473e-07,\n", - " -1.8548e-06, -6.1208e-07, -6.5729e-07, -1.1443e-07, -4.5872e-07,\n", - " -9.5973e-08, -8.9015e-08, 1.6086e-07, 6.8209e-07, -5.6067e-08,\n", - " 5.7320e-07, 1.4644e-07, -5.2722e-07, -1.4234e-07, 4.5176e-08,\n", - " -2.5149e-07, -1.8360e-06, 4.3521e-07, 5.5334e-07, 1.5618e-06,\n", - " 1.8674e-06, -1.2565e-07, -9.0837e-07, -1.2934e-07, 2.4103e-07,\n", - " 4.5534e-07, -5.2275e-07, 6.2046e-07, -4.7477e-07, 1.4448e-08,\n", - " -3.7181e-07, -6.9485e-07, -3.0114e-07, 4.4949e-07, -7.8480e-07,\n", - " -4.3108e-07, -8.2225e-08, 1.0328e-06, -1.2272e-06, 3.3779e-07,\n", - " -3.3428e-07, 5.8373e-07, -5.0156e-07, 3.4294e-07, 4.4207e-07,\n", - " -1.8404e-08, -1.2264e-07, 1.0217e-06, 9.7197e-07, 2.6029e-07,\n", - " -3.3480e-07, -1.0682e-06, 2.0034e-07, 9.8647e-07, 8.0695e-07,\n", - " -7.1272e-08, 3.6538e-07, -1.3156e-06, 9.3369e-08, 8.5994e-07,\n", - " -1.0821e-06, -9.3545e-07, -4.1727e-08, 3.5602e-07, -2.2246e-07,\n", - " 1.7382e-07, 5.9968e-07, -6.1940e-07, -1.5544e-07, 6.1496e-08,\n", - " -1.9020e-06, 4.2236e-07, -3.4969e-08, -6.4763e-07, 1.2352e-06,\n", - " 1.8911e-07, 1.0038e-06, -3.9493e-07, -4.2307e-07, 4.3479e-07,\n", - " 1.0595e-06, 1.4161e-07, -3.4526e-07, 3.8437e-07, -1.0566e-06,\n", - " 8.1219e-07, 5.8152e-07, 3.3569e-07, -4.2393e-08, -2.0979e-06,\n", - " 2.4898e-07, -1.4319e-06, 8.7014e-07, -5.4009e-07, 5.1806e-07,\n", - " 1.5672e-07, -4.4967e-07, -7.4723e-07, 3.3632e-08, -1.7880e-07,\n", - " -4.2434e-08, 1.1680e-06, 4.3179e-07, 1.2370e-07, 1.4654e-07,\n", - " 2.0227e-07, -4.9486e-07, 7.2001e-07, -2.8281e-07, 1.2026e-07,\n", - " -1.2374e-07, 1.5187e-07, -6.7956e-07, 8.9595e-07, -7.0767e-07,\n", - " 1.2359e-06, -9.4191e-07, 5.9228e-07, 2.2014e-07, 2.4551e-07,\n", - " 7.0375e-07, -7.5531e-09, -4.2107e-07, -5.5966e-07, -2.7207e-07,\n", - " -1.2696e-06, 5.3629e-07, 1.1028e-06, 1.3448e-06, 9.1707e-07,\n", - " -4.1077e-07, -1.4194e-06, 8.2703e-08, 1.3841e-06, 8.4550e-07,\n", - " -1.0788e-06, -7.5913e-07, 6.7956e-07, 1.5473e-07, 6.0747e-08,\n", - " 5.2321e-07, 7.4032e-07, -2.2953e-07, -4.8046e-07, 1.3302e-06,\n", - " -6.8801e-09, 8.0393e-07, -1.7242e-06, -1.0591e-06, -3.9076e-07,\n", - " -7.7495e-08, -2.8139e-07, 3.3762e-07, -1.9094e-07, -4.8483e-07,\n", - " -9.5804e-07, 9.5378e-07, 8.9375e-07, 1.8582e-07, -6.7819e-08,\n", - " 5.9106e-07], device='cuda:0'),\n", - " tensor([[[[ 1.8924e-06, -8.8848e-07, -6.9756e-07],\n", - " [ 2.0321e-06, -5.2154e-08, 1.7174e-06],\n", - " [ 1.9046e-06, -2.4214e-07, -5.1362e-07]],\n", - " \n", - " [[-8.1770e-07, -9.0804e-07, 1.6792e-06],\n", - " [-7.1526e-07, 2.7437e-06, 1.2583e-06],\n", - " [-2.1048e-07, 2.6822e-07, 6.1002e-07]],\n", - " \n", - " [[ 1.8813e-07, 7.5344e-07, 2.2731e-06],\n", - " [ 1.1399e-06, 1.7229e-06, 2.0880e-06],\n", - " [ 2.2715e-06, 1.0692e-06, 9.1083e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-9.5088e-07, 5.3085e-07, 4.8615e-07],\n", - " [ 1.9465e-06, 3.2410e-06, 1.0207e-06],\n", - " [-2.2911e-07, 7.3481e-07, 8.9034e-07]],\n", - " \n", - " [[-5.0338e-07, -1.2694e-06, 2.4308e-07],\n", - " [ 1.6391e-06, 1.7183e-06, 1.9586e-06],\n", - " [ 2.1234e-07, -1.9076e-06, 1.4808e-07]],\n", - " \n", - " [[ 2.6822e-07, -7.9116e-07, -1.9558e-07],\n", - " [ 1.2629e-06, -7.3109e-07, 1.3076e-06],\n", - " [ 5.2340e-07, 3.4645e-07, 8.6799e-07]]],\n", - " \n", - " \n", - " [[[-9.3738e-07, 4.5635e-07, 1.8347e-07],\n", - " [-3.1665e-08, -2.0592e-06, -9.0897e-07],\n", - " [ 4.5076e-07, 9.8906e-07, -1.8621e-07]],\n", - " \n", - " [[-3.2969e-07, -8.5012e-07, -1.5867e-06],\n", - " [-2.5984e-06, -3.7861e-06, -1.3746e-06],\n", - " [-1.7323e-06, -8.7544e-07, -7.6462e-07]],\n", - " \n", - " [[ 6.5006e-07, -1.0738e-06, 2.9430e-07],\n", - " [-5.8301e-07, -7.0920e-07, 7.2177e-07],\n", - " [-2.1085e-06, -1.2824e-06, -3.7812e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1791e-06, 8.0094e-07, 1.2927e-06],\n", - " [-6.7055e-07, -1.2852e-07, 2.6636e-07],\n", - " [ 8.4285e-07, -1.6782e-06, 8.2888e-08]],\n", - " \n", - " [[ 1.0561e-06, 1.0077e-06, -1.1828e-06],\n", - " [-6.2026e-07, -2.3935e-07, -1.1495e-06],\n", - " [ 6.6124e-07, 8.2160e-08, 6.1095e-07]],\n", - " \n", - " [[ 4.2398e-07, 5.4948e-08, -1.0375e-06],\n", - " [-1.5311e-06, -1.1986e-06, -2.0696e-06],\n", - " [ 1.2759e-07, -1.2014e-06, -1.6992e-06]]],\n", - " \n", - " \n", - " [[[ 4.4797e-07, -6.5984e-07, 4.2701e-07],\n", - " [-5.8673e-07, -5.5134e-07, 8.2795e-07],\n", - " [-1.3448e-06, -6.1933e-07, -8.1025e-08]],\n", - " \n", - " [[ 7.6368e-07, 1.7779e-06, 1.9949e-06],\n", - " [-1.6456e-06, 1.6810e-07, 6.7521e-07],\n", - " [-7.8371e-07, 6.6683e-07, -7.1898e-07]],\n", - " \n", - " [[ 1.4314e-06, 9.5484e-07, 1.6317e-06],\n", - " [ 3.8184e-07, -9.0338e-08, 1.7053e-06],\n", - " [ 1.4249e-07, -1.6019e-07, 1.7006e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-1.5274e-07, 9.5507e-07, 4.4657e-07],\n", - " [ 2.3050e-07, 1.2331e-06, 9.2038e-07],\n", - " [-5.9232e-07, -2.2352e-07, -1.6708e-06]],\n", - " \n", - " [[ 1.3737e-07, -1.7229e-08, 9.8301e-07],\n", - " [ 1.3694e-06, 1.8813e-07, 1.3094e-06],\n", - " [-1.1995e-06, 7.7579e-07, 8.4471e-07]],\n", - " \n", - " [[ 7.0687e-07, 5.5879e-09, 1.3420e-06],\n", - " [ 1.1032e-06, -2.1537e-07, 2.0638e-06],\n", - " [-3.4459e-07, -1.2299e-06, 8.5682e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 4.8429e-08, 1.3467e-06, 1.5162e-06],\n", - " [-1.5004e-06, -1.0617e-06, -3.9302e-07],\n", - " [-1.3690e-06, 4.7497e-07, -1.7416e-07]],\n", - " \n", - " [[-1.1288e-06, -2.2575e-06, -4.2561e-07],\n", - " [ 9.6206e-07, -3.1646e-06, 3.7812e-07],\n", - " [ 1.8626e-09, -1.3215e-06, -2.0005e-06]],\n", - " \n", - " [[-1.8980e-06, -2.8023e-06, -1.6885e-06],\n", - " [-1.5572e-06, -7.3621e-07, -1.4361e-06],\n", - " [-9.2667e-07, -1.1539e-06, -2.5744e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.6058e-06, -3.1519e-06, -1.7546e-06],\n", - " [-2.3330e-06, -1.4864e-06, -1.1800e-06],\n", - " [-3.3323e-06, -1.5590e-06, 3.8557e-07]],\n", - " \n", - " [[ 6.4727e-07, -1.7192e-06, -2.8405e-07],\n", - " [ 4.4168e-07, -2.2091e-06, -1.8645e-06],\n", - " [-1.4142e-06, -2.0384e-06, -2.7249e-06]],\n", - " \n", - " [[ 9.7789e-07, -1.5218e-06, -2.3060e-06],\n", - " [ 3.3472e-06, -8.0280e-07, -3.2596e-08],\n", - " [-6.8499e-07, -2.5388e-06, 3.5204e-07]]],\n", - " \n", - " \n", - " [[[-4.9506e-07, 2.0664e-06, -1.8831e-06],\n", - " [ 4.5262e-07, -1.2107e-07, -9.0338e-07],\n", - " [-1.4901e-07, -1.0990e-07, -1.2470e-06]],\n", - " \n", - " [[-3.2792e-06, -4.2459e-06, -2.5975e-06],\n", - " [-1.8068e-07, -2.0990e-06, -4.2114e-06],\n", - " [-8.0653e-07, -2.4959e-06, -3.7458e-06]],\n", - " \n", - " [[-1.0636e-06, -6.0070e-07, 7.0059e-07],\n", - " [-6.5193e-09, -4.8382e-07, -6.5740e-07],\n", - " [ 3.6304e-07, 2.8266e-07, 1.8920e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.6217e-06, -2.4289e-06, -2.6701e-06],\n", - " [-2.5239e-06, -3.6713e-06, -2.7586e-06],\n", - " [ 4.9127e-08, -1.2238e-06, -7.3528e-07]],\n", - " \n", - " [[-3.7253e-08, -1.2801e-06, -9.4436e-07],\n", - " [ 1.4137e-06, -2.3469e-07, -2.5518e-07],\n", - " [-5.2154e-08, -4.3400e-07, -3.4645e-07]],\n", - " \n", - " [[-9.7416e-07, 4.1840e-07, -8.9966e-07],\n", - " [-8.3307e-07, 2.0070e-06, -8.3260e-07],\n", - " [ 6.0117e-07, 2.4308e-06, 2.2436e-06]]],\n", - " \n", - " \n", - " [[[ 4.3772e-07, 2.3330e-07, 8.8103e-07],\n", - " [-1.8489e-06, -1.2983e-06, -2.5369e-06],\n", - " [ 9.0525e-07, 3.2037e-07, 8.2515e-07]],\n", - " \n", - " [[ 1.4659e-06, -6.3190e-07, -1.6205e-07],\n", - " [-7.7207e-07, 4.3400e-07, 2.8871e-08],\n", - " [-7.7975e-07, -1.8943e-06, -1.9125e-06]],\n", - " \n", - " [[ 8.5030e-07, -1.5274e-07, -3.2596e-07],\n", - " [ 1.1744e-06, 1.0678e-06, 1.1483e-06],\n", - " [-1.6536e-06, -1.4342e-07, -1.9744e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.9116e-07, 9.9186e-07, 1.1157e-06],\n", - " [ 1.5814e-06, 1.0254e-06, 2.9150e-07],\n", - " [-5.4482e-08, -1.2480e-07, 7.8510e-07]],\n", - " \n", - " [[ 2.1756e-06, 8.7777e-07, -1.6391e-07],\n", - " [-1.0292e-06, -2.0433e-06, -9.5181e-07],\n", - " [ 9.1549e-07, 2.3469e-07, -6.8895e-07]],\n", - " \n", - " [[ 6.8359e-07, 8.0140e-07, -8.3633e-07],\n", - " [-1.7288e-07, -1.3168e-06, -5.7183e-07],\n", - " [ 6.8592e-07, -9.3132e-07, 7.6601e-07]]]], device='cuda:0'),\n", - " tensor([ 8.3447e-07, -9.5367e-07, 1.0729e-06, 1.0729e-06, 1.1921e-07,\n", - " -7.7486e-07, -1.0133e-06, -6.5565e-07, -1.1921e-07, -7.7486e-07,\n", - " 4.7684e-07, 7.1526e-07, 2.3842e-07, 7.1526e-07, -8.3447e-07,\n", - " 4.7684e-07, 5.9605e-07, -2.3842e-07, -5.9605e-08, 9.5367e-07,\n", - " -5.9605e-07, 3.5763e-07, 1.1921e-07, -4.1723e-07, 0.0000e+00,\n", - " 7.1526e-07, -5.9605e-07, 1.6689e-06, -7.1526e-07, 4.7684e-07,\n", - " -4.7684e-07, -7.1526e-07, 1.1921e-06, 0.0000e+00, -2.3842e-07,\n", - " 0.0000e+00, -5.9605e-07, -7.1526e-07, 0.0000e+00, 0.0000e+00,\n", - " -9.5367e-07, -1.7881e-07, -4.7684e-07, 1.1921e-07, 3.5763e-07,\n", - " -5.3644e-07, 8.3447e-07, -1.1325e-06, 2.2650e-06, 5.9605e-07,\n", - " -1.1921e-07, -8.9407e-07, -4.1723e-07, 7.1526e-07, -4.1723e-07,\n", - " 1.0729e-06, -5.9605e-08, -2.0266e-06, -1.1921e-06, 7.1526e-07,\n", - " 1.5497e-06, 3.5763e-07, -1.1921e-07, -1.7881e-07, 5.9605e-07,\n", - " -4.1723e-07, 2.3842e-07, -4.7684e-07, 1.0729e-06, 2.3842e-07,\n", - " -1.2517e-06, 4.7684e-07, 8.3447e-07, 9.5367e-07, -1.0729e-06,\n", - " 1.0729e-06, 5.9605e-07, 9.5367e-07, -1.1921e-07, -1.1921e-07,\n", - " -5.9605e-08, -1.0133e-06, -4.1723e-07, 4.7684e-07, 2.3842e-07,\n", - " 8.3447e-07, -8.9407e-07, -1.7881e-06, -2.3842e-07, -4.1723e-07,\n", - " 0.0000e+00, -5.9605e-08, 1.4305e-06, 2.3842e-07, -1.4305e-06,\n", - " 9.5367e-07, 1.4305e-06, 5.9605e-07, -7.1526e-07, 7.1526e-07,\n", - " -7.7486e-07, -5.9605e-07, -5.9605e-08, 2.1458e-06, 0.0000e+00,\n", - " -6.5565e-07, 9.5367e-07, -7.1526e-07, -1.7881e-07, 3.5763e-07,\n", - " 1.6689e-06, -8.9407e-07, 9.5367e-07, 9.5367e-07, -3.5763e-07,\n", - " 1.1921e-06, 0.0000e+00, 2.3842e-07, 0.0000e+00, -4.7684e-07,\n", - " 8.3447e-07, -6.5565e-07, 1.1921e-07, -7.7486e-07, 7.1526e-07,\n", - " -5.9605e-07, 8.3447e-07, -1.7881e-07, -5.9605e-08, 1.1921e-07,\n", - " 1.1921e-06, -1.1325e-06, 0.0000e+00, -1.3113e-06, 4.7684e-07,\n", - " 4.7684e-07, -7.7486e-07, -2.9802e-07, 1.9073e-06, 1.1921e-07,\n", - " -1.1921e-06, -1.8477e-06, 3.5763e-07, -2.3842e-07, 1.1921e-06,\n", - " 8.3447e-07, 3.5763e-07, -2.3842e-06, 5.9605e-07, 4.7684e-07,\n", - " -1.8477e-06, 0.0000e+00, -8.3447e-07, -7.7486e-07, 1.4305e-06,\n", - " 5.9605e-07, 3.5763e-07, 7.1526e-07, 8.3447e-07, 2.3842e-07,\n", - " -1.1921e-07, -1.0133e-06, -8.9407e-07, -1.6689e-06, -6.5565e-07,\n", - " -5.3644e-07, -4.7684e-07, -2.9802e-07, 4.7684e-07, 7.1526e-07,\n", - " -2.3842e-07, -8.3447e-07, -6.5565e-07, 3.5763e-07, 0.0000e+00,\n", - " 1.1921e-07, -8.3447e-07, -1.0133e-06, -8.9407e-07, 5.9605e-07,\n", - " -1.7881e-07, 9.5367e-07, 5.9605e-07, 3.5763e-07, 4.7684e-07,\n", - " 1.5497e-06, 1.1921e-06, -2.9802e-07, 0.0000e+00, -1.1921e-07,\n", - " 1.9073e-06, -6.5565e-07, 1.1921e-07, -5.3644e-07, -5.3644e-07,\n", - " -2.9802e-07, 1.0729e-06, -3.5763e-07, 9.5367e-07, -7.7486e-07,\n", - " 0.0000e+00, 1.1921e-07, -7.1526e-07, 3.5763e-07, -1.0729e-06,\n", - " -2.4438e-06, 3.5763e-07, 3.5763e-07, -8.9407e-07, 0.0000e+00,\n", - " 2.3842e-07, -5.9605e-08, 0.0000e+00, 0.0000e+00, -1.6093e-06,\n", - " 1.1921e-07, -1.4305e-06, -5.3644e-07, -1.7881e-07, -2.3842e-07,\n", - " 9.5367e-07, 1.3113e-06, 8.3447e-07, 1.0729e-06, 1.3113e-06,\n", - " -3.5763e-07, -2.9802e-07, 0.0000e+00, 2.3842e-07, -5.3644e-07,\n", - " 3.5763e-07, -1.0133e-06, 9.5367e-07, 0.0000e+00, 1.0729e-06,\n", - " -7.7486e-07, 1.5497e-06, 1.1921e-07, -5.9605e-08, 8.3447e-07,\n", - " -1.7881e-07, -3.5763e-07, -1.5497e-06, 4.7684e-07, -1.7881e-07,\n", - " -1.1921e-07, -2.3842e-07, -9.5367e-07, 3.5763e-07, -6.5565e-07,\n", - " 3.5763e-07, 9.5367e-07, -7.7486e-07, -1.3709e-06, -1.9073e-06,\n", - " -4.1723e-07], device='cuda:0'),\n", - " tensor([ 1.4996e-06, -9.7831e-07, 2.7066e-07, 4.9675e-07, -1.3047e-07,\n", - " -6.1084e-07, -3.3956e-07, 2.3697e-07, -5.8405e-08, -1.8422e-06,\n", - " 1.0136e-06, 1.1218e-06, -1.0392e-06, 1.8002e-07, -1.3222e-06,\n", - " 4.3368e-07, -1.2230e-07, -2.7726e-07, -3.3027e-08, 8.2742e-07,\n", - " -6.0166e-08, -2.9004e-07, 1.9976e-07, -4.1473e-07, -8.4476e-07,\n", - " -2.9888e-08, 4.1221e-07, 2.7511e-07, -7.9906e-07, 1.1749e-06,\n", - " -9.4270e-07, 1.3504e-07, 8.1365e-07, -1.1818e-06, -7.4986e-07,\n", - " 1.1759e-06, -1.1395e-07, -3.5590e-07, -4.6699e-08, 4.7032e-07,\n", - " -9.2778e-08, 1.9660e-07, -6.9121e-07, 6.0014e-07, -5.8659e-08,\n", - " 2.6431e-08, -5.4318e-08, -1.3243e-06, 5.5120e-07, 7.7604e-07,\n", - " 8.4065e-08, -1.0963e-06, -6.6890e-08, 5.7401e-07, -8.2514e-07,\n", - " 6.3638e-07, 4.8604e-08, -1.1413e-06, -1.1278e-06, 3.3778e-07,\n", - " 8.9280e-07, -8.0010e-07, -1.2865e-06, -3.0982e-07, -2.5761e-07,\n", - " 9.0459e-07, -1.4777e-07, 6.9124e-07, 3.8644e-07, 8.2839e-07,\n", - " -6.2080e-07, 5.4476e-07, -1.6914e-07, 1.6066e-07, -5.3019e-07,\n", - " 8.8009e-07, -6.6119e-07, 1.1239e-07, -3.1296e-07, 1.6062e-07,\n", - " -5.4466e-07, -9.9712e-07, 1.3982e-08, 8.6614e-07, -8.1877e-07,\n", - " 1.0593e-06, -6.8200e-07, -2.3642e-07, -1.4596e-07, 3.5521e-07,\n", - " -1.9287e-07, -5.2644e-08, 3.5540e-07, 6.3538e-07, -9.7509e-07,\n", - " 8.7442e-09, 2.9551e-07, -9.9604e-08, -1.4247e-06, 2.6556e-07,\n", - " -5.0887e-07, -7.7480e-07, 1.7260e-07, 7.8099e-07, 4.0190e-07,\n", - " -2.4187e-07, -2.1173e-07, -8.1731e-07, -4.1487e-07, 2.3282e-07,\n", - " 1.2374e-06, -1.0589e-07, -7.2642e-07, -4.7979e-07, -7.0970e-07,\n", - " 4.4455e-07, -4.0603e-08, 6.6465e-07, -1.0499e-06, -1.2124e-07,\n", - " 5.0759e-07, -3.3925e-07, -3.8467e-07, -3.1564e-07, -4.3414e-09,\n", - " -5.4672e-07, 6.3713e-07, -3.8460e-07, -3.7613e-07, 4.9192e-07,\n", - " 1.1413e-07, 6.7237e-07, 6.8360e-07, -4.4353e-07, 2.2006e-07,\n", - " 7.9770e-07, -6.9525e-07, 4.9770e-07, 1.2657e-06, 9.7913e-07,\n", - " 1.8810e-07, 3.0461e-07, -2.4066e-07, -5.0781e-07, 7.8403e-07,\n", - " 1.7687e-07, 4.7308e-07, -1.8610e-06, 9.0300e-07, -3.3854e-07,\n", - " -7.2396e-07, 5.2172e-07, -8.8731e-07, -5.0957e-07, 2.6942e-07,\n", - " -2.1702e-07, -9.7588e-08, 5.1437e-07, 8.0258e-07, -3.9910e-08,\n", - " 3.0799e-07, -1.0998e-06, -9.4656e-07, 9.5955e-08, 6.1174e-07,\n", - " 8.9107e-08, 7.4862e-07, 2.4129e-07, 1.7060e-07, -8.3884e-08,\n", - " -1.2298e-06, 6.1371e-07, -3.2114e-07, 2.4378e-07, 4.1363e-07,\n", - " 4.4874e-08, -1.3397e-07, -2.1337e-07, -9.1465e-07, -1.3507e-06,\n", - " 6.7283e-08, -1.3089e-07, -5.7291e-07, 3.9417e-07, 6.9060e-07,\n", - " 1.2198e-06, 4.8669e-07, 2.6102e-07, 6.2616e-07, -1.1010e-06,\n", - " 7.1556e-07, 2.0051e-07, -1.2558e-06, -6.5537e-07, -7.2214e-07,\n", - " -2.6249e-07, 7.3104e-07, -7.9916e-08, 5.6357e-07, -2.2323e-07,\n", - " -9.2495e-08, 7.5343e-07, -7.5690e-07, 3.1329e-07, -1.1467e-07,\n", - " -7.7100e-07, 8.4661e-08, 7.9552e-07, -9.5221e-07, -4.5532e-07,\n", - " 1.0231e-06, -1.1224e-06, 8.0024e-07, 4.1700e-07, -7.5773e-07,\n", - " -3.2492e-07, 2.5452e-09, 8.2564e-07, 1.9540e-07, -1.0915e-06,\n", - " 8.0702e-07, 6.7307e-07, 6.1853e-07, -5.3461e-08, 9.1746e-07,\n", - " 5.0425e-07, -4.3953e-07, 8.6081e-07, -6.9142e-07, -3.0641e-07,\n", - " -7.6885e-08, 1.2958e-07, 1.8386e-07, -2.7983e-07, 6.6745e-07,\n", - " -5.0889e-07, 1.3176e-06, -7.9853e-08, -4.5977e-07, 2.9240e-08,\n", - " -1.6256e-06, -2.2015e-07, -4.1468e-08, -2.0701e-07, 2.5846e-08,\n", - " 3.0667e-08, 2.3528e-08, -2.0532e-06, -1.8726e-07, -2.9575e-07,\n", - " 1.2827e-06, 1.0446e-07, -6.4511e-07, 4.5967e-07, 2.4757e-07,\n", - " -1.0032e-06], device='cuda:0'),\n", - " tensor([[[[-2.2799e-06]],\n", - " \n", - " [[-6.4820e-07]],\n", - " \n", - " [[ 2.3972e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-5.5134e-07]],\n", - " \n", - " [[-5.8720e-07]],\n", - " \n", - " [[ 4.1351e-07]]],\n", - " \n", - " \n", - " [[[-3.0156e-06]],\n", - " \n", - " [[ 1.9148e-06]],\n", - " \n", - " [[-1.9222e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-7.7300e-08]],\n", - " \n", - " [[-9.1642e-07]],\n", - " \n", - " [[-1.5050e-06]]],\n", - " \n", - " \n", - " [[[-1.6913e-06]],\n", - " \n", - " [[ 1.0021e-06]],\n", - " \n", - " [[-2.2352e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.4715e-06]],\n", - " \n", - " [[-7.4506e-07]],\n", - " \n", - " [[-2.9877e-06]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 4.4703e-07]],\n", - " \n", - " [[-2.5332e-06]],\n", - " \n", - " [[-1.9819e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-2.7604e-06]],\n", - " \n", - " [[ 2.8312e-07]],\n", - " \n", - " [[-3.7998e-07]]],\n", - " \n", - " \n", - " [[[-3.4645e-07]],\n", - " \n", - " [[-2.6375e-06]],\n", - " \n", - " [[-4.0606e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-9.1642e-07]],\n", - " \n", - " [[ 2.7148e-06]],\n", - " \n", - " [[ 2.7791e-06]]],\n", - " \n", - " \n", - " [[[ 2.8312e-06]],\n", - " \n", - " [[ 3.3658e-06]],\n", - " \n", - " [[-2.3805e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.4273e-06]],\n", - " \n", - " [[ 4.0196e-06]],\n", - " \n", - " [[-1.3411e-07]]]], device='cuda:0'),\n", - " tensor([ 5.9605e-07, -1.6093e-06, -4.7684e-07, 4.7684e-07, -4.1723e-07,\n", - " -7.1526e-07, 2.3842e-07, 1.1921e-07, -1.7881e-07, -1.7285e-06,\n", - " 1.4305e-06, 1.3113e-06, -1.1325e-06, -1.0729e-06, -2.2054e-06,\n", - " 5.9605e-07, 3.5763e-07, 3.5763e-07, 2.3842e-07, 2.3842e-07,\n", - " 1.7881e-06, -4.1723e-07, -5.3644e-07, -1.1921e-07, -2.9802e-07,\n", - " -4.1723e-07, 1.0729e-06, 3.5763e-07, -2.3842e-07, 8.3447e-07,\n", - " -7.1526e-07, 9.5367e-07, -5.9605e-08, -9.5367e-07, -4.1723e-07,\n", - " -1.7881e-07, 2.3842e-07, -5.9605e-08, 5.9605e-07, 1.0729e-06,\n", - " 1.7881e-06, 4.7684e-07, -7.1526e-07, -2.3842e-07, -1.7881e-07,\n", - " 4.7684e-07, -1.3709e-06, 2.3842e-07, -1.1921e-06, 0.0000e+00,\n", - " 2.3842e-07, -1.4901e-06, 1.1921e-07, 9.5367e-07, -1.0133e-06,\n", - " -2.3842e-07, -9.5367e-07, 9.5367e-07, 1.3113e-06, 8.3447e-07,\n", - " 4.7684e-07, 0.0000e+00, -1.6689e-06, 3.5763e-07, -2.3842e-07,\n", - " 1.5497e-06, -2.3842e-07, 1.5497e-06, 4.7684e-07, 3.5763e-07,\n", - " -1.6093e-06, -7.7486e-07, -7.1526e-07, 1.1921e-07, -4.1723e-07,\n", - " 9.5367e-07, -1.7881e-07, 1.1921e-07, -5.9605e-07, -1.1921e-07,\n", - " -8.9407e-07, -4.7684e-07, -1.1921e-06, 1.4305e-06, -8.9407e-07,\n", - " -4.7684e-07, -6.5565e-07, 8.3447e-07, -4.7684e-07, 8.3447e-07,\n", - " 3.5763e-07, -2.9802e-07, -1.0133e-06, 2.8610e-06, 4.7684e-07,\n", - " -7.7486e-07, 3.5763e-07, -3.5763e-07, -1.6093e-06, 4.7684e-07,\n", - " 1.1921e-06, 1.1921e-07, 0.0000e+00, -5.9605e-07, 2.3842e-07,\n", - " 1.1921e-06, -5.3644e-07, 4.7684e-07, 3.5763e-07, -3.5763e-07,\n", - " 8.3447e-07, -7.1526e-07, -5.9605e-08, -1.2517e-06, -7.1526e-07,\n", - " 4.7684e-07, 9.5367e-07, 2.3842e-07, -5.9605e-07, 8.3447e-07,\n", - " 7.1526e-07, 1.1921e-07, -5.3644e-07, 2.3842e-07, 3.5763e-07,\n", - " -4.1723e-07, 4.7684e-07, 3.5763e-07, -2.3842e-07, 7.1526e-07,\n", - " -6.5565e-07, 1.1921e-06, 8.3447e-07, 1.1921e-07, -3.5763e-07,\n", - " 3.5763e-07, -8.3447e-07, -4.1723e-07, 1.1921e-07, 9.5367e-07,\n", - " 2.3842e-07, 1.0729e-06, -4.1723e-07, 1.1921e-07, 9.5367e-07,\n", - " -5.9605e-07, 8.3447e-07, 3.5763e-07, 5.9605e-07, -7.1526e-07,\n", - " 4.7684e-07, 1.0729e-06, -7.1526e-07, -2.9802e-07, -1.1921e-07,\n", - " -1.1325e-06, -1.4305e-06, -4.7684e-07, -5.9605e-07, 2.3842e-07,\n", - " -5.9605e-07, -4.1723e-07, -8.3447e-07, 1.1921e-07, 1.5497e-06,\n", - " 7.1526e-07, 1.4305e-06, -6.5565e-07, 5.9605e-07, 4.7684e-07,\n", - " -2.1458e-06, 1.6689e-06, -8.9407e-07, -2.9802e-07, 1.1921e-07,\n", - " -7.7486e-07, 1.1921e-06, 9.5367e-07, -4.7684e-07, -1.3113e-06,\n", - " -5.9605e-08, -2.3842e-07, 0.0000e+00, -5.3644e-07, -5.9605e-07,\n", - " 7.1526e-07, -8.3447e-07, 9.5367e-07, 3.5763e-07, -1.1325e-06,\n", - " -7.7486e-07, 0.0000e+00, -8.3447e-07, -1.0729e-06, 3.5763e-07,\n", - " 8.3447e-07, 0.0000e+00, 4.7684e-07, 7.1526e-07, 4.7684e-07,\n", - " 1.1921e-07, 7.1526e-07, -2.3842e-07, 2.3842e-07, 7.1526e-07,\n", - " 1.9073e-06, -5.3644e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07,\n", - " 1.0729e-06, -1.1921e-06, 1.3113e-06, -6.5565e-07, -5.3644e-07,\n", - " -4.7684e-07, 1.0729e-06, 1.4305e-06, 4.7684e-07, -1.7285e-06,\n", - " -2.9802e-07, 0.0000e+00, 2.3842e-07, -9.5367e-07, 9.5367e-07,\n", - " 7.1526e-07, -2.3842e-07, 5.9605e-07, -6.5565e-07, -1.1921e-06,\n", - " -5.9605e-07, 1.1921e-06, -4.1723e-07, 1.0729e-06, 0.0000e+00,\n", - " -5.3644e-07, 1.1921e-06, -1.1921e-07, -1.0729e-06, -1.1921e-07,\n", - " -3.8147e-06, 0.0000e+00, 1.6689e-06, 1.1921e-07, -7.1526e-07,\n", - " -5.3644e-07, 0.0000e+00, -2.3842e-06, 2.3842e-07, 3.5763e-07,\n", - " 1.1921e-07, -1.7881e-07, -1.7881e-07, 1.3113e-06, 1.0729e-06,\n", - " -8.9407e-07], device='cuda:0'),\n", - " tensor([ 1.4996e-06, -9.7831e-07, 2.7066e-07, 4.9675e-07, -1.3047e-07,\n", - " -6.1084e-07, -3.3956e-07, 2.3697e-07, -5.8405e-08, -1.8422e-06,\n", - " 1.0136e-06, 1.1218e-06, -1.0392e-06, 1.8002e-07, -1.3222e-06,\n", - " 4.3368e-07, -1.2230e-07, -2.7726e-07, -3.3027e-08, 8.2742e-07,\n", - " -6.0166e-08, -2.9004e-07, 1.9976e-07, -4.1473e-07, -8.4476e-07,\n", - " -2.9888e-08, 4.1221e-07, 2.7511e-07, -7.9906e-07, 1.1749e-06,\n", - " -9.4270e-07, 1.3504e-07, 8.1365e-07, -1.1818e-06, -7.4986e-07,\n", - " 1.1759e-06, -1.1395e-07, -3.5590e-07, -4.6699e-08, 4.7032e-07,\n", - " -9.2778e-08, 1.9660e-07, -6.9121e-07, 6.0014e-07, -5.8659e-08,\n", - " 2.6431e-08, -5.4318e-08, -1.3243e-06, 5.5120e-07, 7.7604e-07,\n", - " 8.4065e-08, -1.0963e-06, -6.6890e-08, 5.7401e-07, -8.2514e-07,\n", - " 6.3638e-07, 4.8604e-08, -1.1413e-06, -1.1278e-06, 3.3778e-07,\n", - " 8.9280e-07, -8.0010e-07, -1.2865e-06, -3.0982e-07, -2.5761e-07,\n", - " 9.0459e-07, -1.4777e-07, 6.9124e-07, 3.8644e-07, 8.2839e-07,\n", - " -6.2080e-07, 5.4476e-07, -1.6914e-07, 1.6066e-07, -5.3019e-07,\n", - " 8.8009e-07, -6.6119e-07, 1.1239e-07, -3.1296e-07, 1.6062e-07,\n", - " -5.4466e-07, -9.9712e-07, 1.3982e-08, 8.6614e-07, -8.1877e-07,\n", - " 1.0593e-06, -6.8200e-07, -2.3642e-07, -1.4596e-07, 3.5521e-07,\n", - " -1.9287e-07, -5.2644e-08, 3.5540e-07, 6.3538e-07, -9.7509e-07,\n", - " 8.7442e-09, 2.9551e-07, -9.9604e-08, -1.4247e-06, 2.6556e-07,\n", - " -5.0887e-07, -7.7480e-07, 1.7260e-07, 7.8099e-07, 4.0190e-07,\n", - " -2.4187e-07, -2.1173e-07, -8.1731e-07, -4.1487e-07, 2.3282e-07,\n", - " 1.2374e-06, -1.0589e-07, -7.2642e-07, -4.7979e-07, -7.0970e-07,\n", - " 4.4455e-07, -4.0603e-08, 6.6465e-07, -1.0499e-06, -1.2124e-07,\n", - " 5.0759e-07, -3.3925e-07, -3.8467e-07, -3.1564e-07, -4.3414e-09,\n", - " -5.4672e-07, 6.3713e-07, -3.8460e-07, -3.7613e-07, 4.9192e-07,\n", - " 1.1413e-07, 6.7237e-07, 6.8360e-07, -4.4353e-07, 2.2006e-07,\n", - " 7.9770e-07, -6.9525e-07, 4.9770e-07, 1.2657e-06, 9.7913e-07,\n", - " 1.8810e-07, 3.0461e-07, -2.4066e-07, -5.0781e-07, 7.8403e-07,\n", - " 1.7687e-07, 4.7308e-07, -1.8610e-06, 9.0300e-07, -3.3854e-07,\n", - " -7.2396e-07, 5.2172e-07, -8.8731e-07, -5.0957e-07, 2.6942e-07,\n", - " -2.1702e-07, -9.7588e-08, 5.1437e-07, 8.0258e-07, -3.9910e-08,\n", - " 3.0799e-07, -1.0998e-06, -9.4656e-07, 9.5955e-08, 6.1174e-07,\n", - " 8.9107e-08, 7.4862e-07, 2.4129e-07, 1.7060e-07, -8.3884e-08,\n", - " -1.2298e-06, 6.1371e-07, -3.2114e-07, 2.4378e-07, 4.1363e-07,\n", - " 4.4874e-08, -1.3397e-07, -2.1337e-07, -9.1465e-07, -1.3507e-06,\n", - " 6.7283e-08, -1.3089e-07, -5.7291e-07, 3.9417e-07, 6.9060e-07,\n", - " 1.2198e-06, 4.8669e-07, 2.6102e-07, 6.2616e-07, -1.1010e-06,\n", - " 7.1556e-07, 2.0051e-07, -1.2558e-06, -6.5537e-07, -7.2214e-07,\n", - " -2.6249e-07, 7.3104e-07, -7.9916e-08, 5.6357e-07, -2.2323e-07,\n", - " -9.2495e-08, 7.5343e-07, -7.5690e-07, 3.1329e-07, -1.1467e-07,\n", - " -7.7100e-07, 8.4661e-08, 7.9552e-07, -9.5221e-07, -4.5532e-07,\n", - " 1.0231e-06, -1.1224e-06, 8.0024e-07, 4.1700e-07, -7.5773e-07,\n", - " -3.2492e-07, 2.5452e-09, 8.2564e-07, 1.9540e-07, -1.0915e-06,\n", - " 8.0702e-07, 6.7307e-07, 6.1853e-07, -5.3461e-08, 9.1746e-07,\n", - " 5.0425e-07, -4.3953e-07, 8.6081e-07, -6.9142e-07, -3.0641e-07,\n", - " -7.6885e-08, 1.2958e-07, 1.8386e-07, -2.7983e-07, 6.6745e-07,\n", - " -5.0889e-07, 1.3176e-06, -7.9853e-08, -4.5977e-07, 2.9240e-08,\n", - " -1.6256e-06, -2.2015e-07, -4.1468e-08, -2.0701e-07, 2.5846e-08,\n", - " 3.0667e-08, 2.3528e-08, -2.0532e-06, -1.8726e-07, -2.9575e-07,\n", - " 1.2827e-06, 1.0446e-07, -6.4511e-07, 4.5967e-07, 2.4757e-07,\n", - " -1.0032e-06], device='cuda:0'),\n", - " tensor([[[[-6.1654e-07, -4.9314e-07, 5.4017e-08],\n", - " [-1.4529e-07, 8.9779e-07, 5.5090e-07],\n", - " [-1.3281e-06, -1.1828e-07, -8.6369e-07]],\n", - " \n", - " [[-6.3330e-08, 4.1444e-08, -4.6939e-07],\n", - " [-3.6322e-07, 9.8161e-07, -1.7136e-07],\n", - " [-1.9995e-06, -2.0228e-06, -3.5297e-06]],\n", - " \n", - " [[ 1.7993e-06, 1.2540e-06, -6.5938e-07],\n", - " [-1.2177e-07, -2.1346e-06, -8.2050e-07],\n", - " [-1.4994e-07, 2.8219e-07, 1.2014e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.4878e-07, 1.0729e-06, 1.4375e-06],\n", - " [-1.1940e-06, 2.0489e-08, 2.0833e-07],\n", - " [ 2.2724e-07, 3.8347e-07, -6.7637e-08]],\n", - " \n", - " [[-5.9605e-07, -7.2364e-07, -5.2806e-07],\n", - " [-9.4250e-07, 5.8953e-07, -7.1479e-07],\n", - " [-2.1886e-07, -5.7742e-08, 3.5949e-07]],\n", - " \n", - " [[ 3.2410e-07, -2.7008e-07, -4.6333e-08],\n", - " [-1.1334e-06, 1.0040e-06, -2.6636e-07],\n", - " [-4.7125e-07, -5.1502e-07, -8.8848e-07]]],\n", - " \n", - " \n", - " [[[-5.1409e-07, -2.0303e-07, -2.5751e-07],\n", - " [-1.5739e-07, -6.2305e-07, 1.1222e-07],\n", - " [ 3.1292e-07, -7.5065e-07, -2.5012e-07]],\n", - " \n", - " [[ 5.4459e-07, 7.3574e-07, 8.7265e-07],\n", - " [ 7.3016e-07, 1.1828e-06, 1.2054e-06],\n", - " [ 6.1654e-07, 4.5309e-07, 1.0747e-06]],\n", - " \n", - " [[-5.1304e-07, 1.8199e-07, -2.0675e-07],\n", - " [-2.7753e-07, -1.3737e-07, -6.6496e-07],\n", - " [-2.3469e-07, 2.4587e-07, -8.7358e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 6.5845e-07, 4.1933e-07, -2.5891e-07],\n", - " [-1.4901e-07, 7.9721e-07, 3.3528e-08],\n", - " [ 1.8813e-07, 8.4937e-07, 1.1865e-06]],\n", - " \n", - " [[-9.9838e-07, -6.5193e-08, -8.0746e-07],\n", - " [ 5.0850e-07, -2.4633e-07, -4.3027e-07],\n", - " [-9.6858e-07, -6.5933e-07, -2.1514e-07]],\n", - " \n", - " [[ 1.3085e-06, 1.3881e-06, 2.0396e-06],\n", - " [ 2.0061e-06, 1.4156e-06, 1.9734e-06],\n", - " [ 7.3854e-07, 2.2110e-06, 7.4517e-07]]],\n", - " \n", - " \n", - " [[[ 3.5390e-07, 1.2008e-07, -5.3179e-07],\n", - " [ 1.4558e-07, -8.4518e-08, 8.2608e-07],\n", - " [ 1.4156e-07, -3.4831e-07, -2.9244e-07]],\n", - " \n", - " [[-7.6368e-08, 6.7241e-07, -1.9907e-07],\n", - " [ 1.2573e-07, -1.5013e-06, 5.6624e-07],\n", - " [-1.4901e-08, -2.6729e-07, 3.7719e-07]],\n", - " \n", - " [[ 1.4156e-07, 1.8242e-07, 5.4063e-07],\n", - " [-3.9581e-07, 2.4261e-07, -5.5972e-07],\n", - " [ 4.8894e-07, 3.0780e-07, 4.6566e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[-7.0967e-07, -1.3793e-06, -9.8627e-07],\n", - " [-1.6415e-07, -5.4631e-07, -1.0971e-06],\n", - " [ 6.5193e-09, -3.4273e-07, -8.3400e-07]],\n", - " \n", - " [[-2.8498e-07, -1.5674e-06, -1.4754e-06],\n", - " [ 2.4214e-07, -7.8417e-07, -2.9519e-06],\n", - " [ 3.8836e-07, -1.0817e-06, -8.5123e-07]],\n", - " \n", - " [[ 5.1595e-07, -2.0117e-07, 2.5285e-07],\n", - " [ 7.2643e-08, -7.3574e-08, 1.8720e-07],\n", - " [-2.9430e-07, 5.3458e-07, 1.6298e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 2.1607e-07, 1.0896e-07, 7.0222e-07],\n", - " [-1.3970e-07, 3.8091e-07, 8.1619e-07],\n", - " [-7.3668e-07, -1.7695e-07, -3.9302e-07]],\n", - " \n", - " [[-9.7509e-07, -1.2200e-06, -7.5577e-07],\n", - " [-1.1199e-07, -1.6950e-07, 8.1956e-08],\n", - " [ 1.4994e-07, -9.1363e-07, -8.5682e-07]],\n", - " \n", - " [[ 1.4069e-06, 1.5609e-06, 1.0058e-06],\n", - " [-2.5705e-07, -3.2131e-07, 1.0179e-06],\n", - " [-1.2545e-06, -3.0082e-07, -1.4236e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-5.1083e-07, -4.7963e-08, 7.4785e-07],\n", - " [ 2.9942e-07, 5.4389e-07, -4.2841e-07],\n", - " [ 5.8673e-07, 7.4599e-07, 2.8871e-07]],\n", - " \n", - " [[-6.2911e-07, -4.0047e-07, -5.7789e-07],\n", - " [-1.0766e-06, 6.2212e-07, -1.6764e-07],\n", - " [-3.1944e-07, 6.7055e-08, -8.1770e-07]],\n", - " \n", - " [[-4.7032e-07, 3.5670e-07, -2.1863e-07],\n", - " [-1.4901e-07, -1.1520e-06, -6.8452e-08],\n", - " [-1.1965e-06, -1.6887e-06, -4.8056e-07]]],\n", - " \n", - " \n", - " [[[-3.7812e-07, 4.5268e-07, 1.4221e-06],\n", - " [-9.9652e-07, 8.6613e-08, 2.9989e-07],\n", - " [-9.8161e-07, -3.2876e-07, 7.2736e-07]],\n", - " \n", - " [[ 6.9535e-07, -9.7789e-08, 4.3935e-07],\n", - " [ 2.5332e-07, -1.6351e-07, -1.3113e-06],\n", - " [-2.9802e-08, -4.0606e-07, -8.0373e-07]],\n", - " \n", - " [[-5.3272e-07, -8.8522e-07, -1.8626e-06],\n", - " [ 5.4156e-07, -1.1586e-06, 4.2841e-08],\n", - " [-1.8962e-06, -9.6112e-07, -5.7928e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 3.9209e-07, 1.1269e-07, -9.7137e-07],\n", - " [-2.3190e-07, 1.1921e-07, -1.2564e-06],\n", - " [ 8.1211e-07, -6.6450e-07, -1.5199e-06]],\n", - " \n", - " [[-3.1013e-07, -3.4319e-07, 5.9605e-08],\n", - " [-6.1281e-07, 5.7369e-07, -1.2470e-06],\n", - " [-5.0361e-07, 7.2643e-08, -1.6140e-06]],\n", - " \n", - " [[ 1.6233e-06, -7.0862e-07, 3.6042e-07],\n", - " [ 1.2033e-06, 3.8836e-07, 1.0934e-06],\n", - " [ 1.0617e-06, -8.4192e-07, 9.0525e-07]]],\n", - " \n", - " \n", - " [[[ 9.9465e-07, 2.3458e-07, 1.2764e-06],\n", - " [-4.8243e-07, -2.9337e-07, -5.8860e-07],\n", - " [ 5.6718e-07, 1.7229e-06, 1.4063e-07]],\n", - " \n", - " [[ 1.0654e-06, 1.1083e-06, -5.3877e-07],\n", - " [-8.4285e-07, 4.8801e-07, -9.4296e-07],\n", - " [ 9.2946e-07, 1.8626e-09, -3.4785e-07]],\n", - " \n", - " [[-1.7835e-07, -1.5274e-07, -3.1572e-07],\n", - " [ 1.2014e-06, -1.1344e-06, 1.2740e-06],\n", - " [ 3.8790e-07, 3.5204e-07, 6.6310e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.1921e-07, -5.2713e-07, -1.5167e-06],\n", - " [ 2.4820e-07, 1.3327e-06, -5.0012e-07],\n", - " [-2.4773e-07, -2.1528e-06, -7.0385e-07]],\n", - " \n", - " [[ 2.0768e-07, -5.4087e-07, 4.3865e-07],\n", - " [-7.7859e-07, 1.7709e-06, -5.0850e-07],\n", - " [-1.5590e-06, -4.3958e-07, -1.8477e-06]],\n", - " \n", - " [[ 2.1979e-06, -1.0855e-06, 1.7428e-06],\n", - " [ 3.2224e-07, -7.2177e-07, -2.8126e-07],\n", - " [-3.7998e-07, 1.0114e-06, 9.7230e-07]]]], device='cuda:0'),\n", - " tensor([-4.7684e-07, 7.1526e-07, -5.9605e-08, 2.3842e-07, -2.3842e-07,\n", - " -4.7684e-07, 4.7684e-07, -8.9407e-07, 7.1526e-07, 1.1921e-07,\n", - " 4.7684e-07, -5.9605e-07, 1.1921e-06, 2.3842e-07, 2.3842e-07,\n", - " -2.3842e-07, 8.3447e-07, -1.1921e-07, -2.9802e-07, 4.7684e-07,\n", - " -1.7881e-07, -2.9802e-07, -7.1526e-07, -3.5763e-07, 0.0000e+00,\n", - " -5.9605e-08, 3.5763e-07, 9.5367e-07, 3.5763e-07, 7.1526e-07,\n", - " -5.3644e-07, -1.1921e-07, -5.9605e-08, 3.5763e-07, -1.7881e-07,\n", - " 1.1921e-07, 3.5763e-07, 2.3842e-07, -3.5763e-07, -4.1723e-07,\n", - " 1.1921e-07, 2.3842e-07, -2.9802e-07, 3.5763e-07, -3.5763e-07,\n", - " -5.9605e-07, 1.1921e-07, 4.7684e-07, 0.0000e+00, 4.7684e-07,\n", - " 0.0000e+00, -3.5763e-07, -6.5565e-07, -4.1723e-07, -5.3644e-07,\n", - " 3.5763e-07, -5.9605e-08, -4.1723e-07, -1.1921e-07, -5.3644e-07,\n", - " -1.7881e-07, 3.5763e-07, -2.3842e-07, -1.7881e-07, -1.7285e-06,\n", - " 1.1921e-07, -2.9802e-07, -1.1921e-07, 2.3842e-07, 8.3447e-07,\n", - " 1.1921e-07, -3.5763e-07, -2.3842e-07, 1.1921e-07, 1.1921e-07,\n", - " 1.1921e-07, 1.1921e-07, 2.3842e-07, 1.1921e-07, -7.1526e-07,\n", - " -2.3842e-07, 3.5763e-07, 4.7684e-07, -8.3447e-07, 2.3842e-07,\n", - " -5.9605e-08, 0.0000e+00, -2.3842e-07, -2.9802e-07, 4.7684e-07,\n", - " -5.9605e-08, -4.1723e-07, 2.3842e-07, 0.0000e+00, 8.3447e-07,\n", - " -4.1723e-07, -1.1921e-07, 0.0000e+00, 0.0000e+00, 4.7684e-07,\n", - " -7.7486e-07, 5.9605e-07, -5.9605e-07, 3.5763e-07, -8.3447e-07,\n", - " 8.3447e-07, -2.9802e-07, 0.0000e+00, -7.7486e-07, 2.3842e-07,\n", - " 0.0000e+00, 4.7684e-07, 5.9605e-07, 4.7684e-07, 2.3842e-07,\n", - " 5.9605e-07, 1.1921e-07, -2.3842e-07, 9.5367e-07, 9.5367e-07,\n", - " -4.7684e-07, 5.9605e-07, -4.1723e-07, 5.9605e-07, 0.0000e+00,\n", - " -3.5763e-07, -5.9605e-07, 0.0000e+00, -1.7881e-07, 1.1921e-07,\n", - " 0.0000e+00, -1.1325e-06, 1.1921e-07, -2.9802e-07, 0.0000e+00,\n", - " 1.1921e-07, 4.7684e-07, 4.7684e-07, 2.3842e-07, 7.1526e-07,\n", - " 0.0000e+00, 1.1921e-07, 2.3842e-07, -5.9605e-08, -3.5763e-07,\n", - " 3.5763e-07, -8.3447e-07, 2.3842e-07, -7.1526e-07, 9.5367e-07,\n", - " 3.5763e-07, 7.1526e-07, 4.7684e-07, -4.1723e-07, 0.0000e+00,\n", - " -5.9605e-08, -4.1723e-07, -7.7486e-07, -2.3842e-07, 1.4305e-06,\n", - " 2.3842e-07, 0.0000e+00, -1.1921e-07, -8.9407e-07, -4.7684e-07,\n", - " 1.1921e-07, -8.3447e-07, 1.1921e-07, -1.0133e-06, -1.7881e-07,\n", - " 1.1921e-07, -5.9605e-07, -3.5763e-07, 1.1921e-07, -5.9605e-08,\n", - " -1.7881e-07, 0.0000e+00, 1.1921e-06, 2.3842e-07, -8.9407e-07,\n", - " 4.7684e-07, -5.9605e-07, 4.7684e-07, -5.3644e-07, 7.1526e-07,\n", - " -2.9802e-07, 2.3842e-07, -5.9605e-08, 2.3842e-07, 1.1921e-07,\n", - " -1.1921e-07, -4.1723e-07, 7.1526e-07, -5.9605e-08, 4.7684e-07,\n", - " -1.1921e-07, -7.1526e-07, 1.1921e-07, 9.5367e-07, -2.9802e-07,\n", - " 4.7684e-07, -5.9605e-08, 5.9605e-07, -8.3447e-07, -5.9605e-07,\n", - " 2.3842e-07, 2.3842e-07, 4.7684e-07, 1.0729e-06, 1.1921e-06,\n", - " 5.9605e-07, -1.1325e-06, -7.1526e-07, 1.7881e-06, -4.1723e-07,\n", - " 1.1921e-07, 1.1921e-07, -4.7684e-07, -1.1921e-07, -2.3842e-07,\n", - " -1.1921e-07, 0.0000e+00, -1.7881e-07, -2.9802e-07, -9.5367e-07,\n", - " 1.1921e-07, 0.0000e+00, 0.0000e+00, -7.1526e-07, 2.3842e-07,\n", - " 0.0000e+00, 7.1526e-07, 1.1921e-06, 3.5763e-07, -5.9605e-08,\n", - " 5.9605e-07, -1.1921e-07, -5.9605e-08, -7.1526e-07, -5.3644e-07,\n", - " 3.5763e-07, 4.7684e-07, -1.7881e-07, 3.5763e-07, -7.1526e-07,\n", - " 1.1921e-07, -1.1921e-07, -2.3842e-07, 0.0000e+00, 1.1921e-07,\n", - " 1.1921e-07, -3.5763e-07, 3.5763e-07, -2.9802e-07, 3.5763e-07,\n", - " 1.1921e-07, 2.3842e-07, 3.5763e-07, 4.7684e-07, -5.9605e-07,\n", - " 1.1921e-07, -1.1921e-07, 4.7684e-07, 1.4305e-06, 1.1921e-07,\n", - " -5.9605e-07, -1.1921e-07, 0.0000e+00, -3.5763e-07, 0.0000e+00,\n", - " -2.3842e-07, 8.3447e-07, -5.9605e-08, 3.5763e-07, -1.4901e-06,\n", - " 2.3842e-07, -1.1921e-07, 8.3447e-07, 2.3842e-07, 0.0000e+00,\n", - " -3.5763e-07, -6.5565e-07, -2.3842e-07, 3.5763e-07, -4.1723e-07,\n", - " -7.1526e-07, 2.3842e-07, -1.7881e-07, 1.0729e-06, -7.7486e-07,\n", - " -4.7684e-07, 2.3842e-07, -1.7881e-07, 3.5763e-07, -5.3644e-07,\n", - " 0.0000e+00, 4.7684e-07, 4.7684e-07, -1.1921e-07, -3.5763e-07,\n", - " -1.0133e-06, 0.0000e+00, 7.1526e-07, 1.1921e-07, -2.3842e-07,\n", - " -8.9407e-07, -7.1526e-07, 2.3842e-07, -8.9407e-07, -1.1921e-07,\n", - " -2.3842e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07, 8.3447e-07,\n", - " 0.0000e+00, 2.3842e-07, -5.3644e-07, 1.3113e-06, 9.5367e-07,\n", - " -1.1921e-07, -1.1921e-07, 1.1921e-07, 4.7684e-07, 1.1921e-07,\n", - " 5.9605e-07, -2.9802e-07, 2.3842e-07, 0.0000e+00, 1.0729e-06,\n", - " 2.3842e-07, 2.3842e-07, -2.3842e-07, -2.9802e-07, -7.7486e-07,\n", - " 1.1921e-06, 2.3842e-07, 0.0000e+00, 3.5763e-07, 5.9605e-07,\n", - " -3.5763e-07, -1.1921e-07, -7.7486e-07, -1.3709e-06, -5.9605e-07,\n", - " -3.5763e-07, -4.7684e-07, -1.1921e-07, 0.0000e+00, 1.1921e-07,\n", - " 1.1921e-07, 1.1921e-07, -2.9802e-07, -3.5763e-07, -1.1921e-07,\n", - " -2.3842e-07, -4.1723e-07, 1.1921e-07, -3.5763e-07, -2.9802e-07,\n", - " 5.9605e-07, 3.5763e-07, 2.3842e-07, -9.5367e-07, -1.7881e-07,\n", - " -1.7881e-07, -2.3842e-07, -2.3842e-07, 0.0000e+00, -6.5565e-07,\n", - " -4.7684e-07, -1.1921e-07, -4.1723e-07, 5.9605e-07, -4.1723e-07,\n", - " -1.1921e-07, 1.1921e-07, -2.9802e-07, 3.5763e-07, -2.3842e-07,\n", - " 5.9605e-07, -5.9605e-08, 1.1921e-07, -4.7684e-07, 2.3842e-07,\n", - " 7.1526e-07, -1.7881e-07, -4.7684e-07, 4.7684e-07, 5.9605e-07,\n", - " 1.1921e-07, -4.1723e-07, -5.9605e-07, 3.5763e-07, 2.3842e-07,\n", - " -1.1921e-07, -1.1921e-07, 1.1921e-07, -8.9407e-07, -4.1723e-07,\n", - " -1.7881e-07, 2.3842e-07, -5.9605e-08, -1.7881e-07, -1.7881e-07,\n", - " 9.5367e-07, -3.5763e-07, -1.1921e-07, 5.9605e-07, -2.3842e-07,\n", - " -1.0729e-06, 5.9605e-07, 0.0000e+00, -1.1921e-07, -1.1921e-07,\n", - " -2.3842e-07, 4.7684e-07, 4.7684e-07, 2.3842e-07, -5.9605e-07,\n", - " -1.7881e-07, -5.9605e-07, 1.1921e-07, 3.5763e-07, 1.1921e-07,\n", - " -4.1723e-07, 1.1921e-07, 2.3842e-07, 5.9605e-07, 4.7684e-07,\n", - " 0.0000e+00, 5.9605e-07, 4.7684e-07, 3.5763e-07, -5.9605e-08,\n", - " -5.9605e-08, 4.7684e-07, 5.9605e-07, 2.3842e-07, -2.9802e-07,\n", - " 7.1526e-07, -5.9605e-08, 3.5763e-07, -2.9802e-07, 3.5763e-07,\n", - " -4.1723e-07, 2.3842e-07, 3.5763e-07, -4.7684e-07, -1.7881e-07,\n", - " -1.3113e-06, -1.1921e-07, -2.9802e-07, -2.1458e-06, -2.9802e-07,\n", - " 5.9605e-07, 1.1921e-07, 0.0000e+00, 3.5763e-07, -7.7486e-07,\n", - " 3.5763e-07, -2.9802e-07, 5.9605e-07, 5.9605e-07, 1.1921e-07,\n", - " -2.9802e-07, -1.1921e-07, 4.7684e-07, 2.3842e-07, -4.7684e-07,\n", - " -5.9605e-08, -5.9605e-08, 0.0000e+00, 2.3842e-07, -4.1723e-07,\n", - " 1.1921e-07, 4.7684e-07, 3.5763e-07, 8.3447e-07, -6.5565e-07,\n", - " 1.1921e-07, 2.3842e-07, -4.1723e-07, -5.9605e-08, -8.3447e-07,\n", - " -4.7684e-07, 5.9605e-07, 0.0000e+00, 2.3842e-07, 1.1921e-07,\n", - " 4.7684e-07, 2.3842e-07, 3.5763e-07, 2.3842e-07, -1.7881e-07,\n", - " 0.0000e+00, -2.3842e-07, -6.5565e-07, 5.9605e-07, 9.5367e-07,\n", - " -3.5763e-07, -2.3842e-07, 2.3842e-07, -2.3842e-07, 1.1921e-07,\n", - " 0.0000e+00, -5.9605e-08, -5.9605e-08, 2.3842e-07, -5.3644e-07,\n", - " -1.9670e-06, -3.5763e-07], device='cuda:0'),\n", - " tensor([-7.0471e-07, -1.8349e-08, 2.0926e-07, -1.9718e-07, -5.0470e-07,\n", - " -2.0335e-07, 2.5020e-07, -5.5646e-07, -1.8064e-07, -6.6849e-07,\n", - " 1.7477e-08, -3.9154e-07, 6.1181e-07, 3.2980e-07, 1.0190e-07,\n", - " 9.8190e-08, 4.5841e-07, -2.9208e-07, 2.5231e-07, 1.0851e-07,\n", - " -6.3703e-08, -1.1971e-07, -4.7798e-08, -3.7517e-07, 2.3964e-07,\n", - " -5.2821e-07, 4.5428e-07, 3.5899e-07, 7.5338e-07, 2.2175e-07,\n", - " 2.6582e-07, -1.2163e-07, -1.1361e-07, 4.1826e-07, -1.9857e-07,\n", - " 3.4440e-07, 1.8137e-07, -2.9222e-07, -4.6063e-08, -1.2579e-07,\n", - " 1.7651e-07, 4.1248e-09, -5.7320e-07, 1.9714e-07, -5.0703e-07,\n", - " -9.1181e-07, 3.4195e-07, -1.5289e-08, -2.8127e-07, -1.0984e-07,\n", - " -4.4231e-07, 3.9074e-07, -4.6410e-07, -2.5822e-07, -6.3460e-07,\n", - " 2.1609e-08, 1.9698e-07, -4.0347e-07, 3.2517e-07, -1.9259e-07,\n", - " -1.9836e-07, 2.8106e-07, -1.2027e-07, -3.3226e-07, -1.5892e-06,\n", - " 3.3092e-07, -3.0162e-07, -4.1899e-08, -3.4353e-07, 2.8133e-07,\n", - " 3.5431e-07, -1.0125e-07, -2.1538e-07, 8.3516e-08, 2.1812e-07,\n", - " 5.0520e-07, -1.1525e-07, 3.1269e-07, -2.6864e-07, -9.7003e-09,\n", - " -2.8103e-07, 6.9125e-07, 2.8932e-07, -5.0337e-07, 3.0293e-07,\n", - " -3.8707e-07, -4.1809e-08, -2.5323e-07, 1.8370e-07, 6.4680e-07,\n", - " 5.6331e-08, -4.5315e-07, 2.2719e-07, -5.8242e-08, 2.2758e-07,\n", - " -4.7488e-07, -3.2628e-07, -1.2880e-07, 5.6331e-08, 5.5098e-07,\n", - " -9.7950e-07, 6.2894e-07, -6.5212e-07, 5.4815e-07, -5.8033e-07,\n", - " 6.9457e-08, -6.1899e-07, 4.1434e-07, -8.2837e-07, 7.0188e-08,\n", - " 6.4545e-08, 1.5691e-07, -4.9377e-09, 2.3902e-07, 2.8643e-07,\n", - " 8.0544e-07, -1.5877e-07, -5.0277e-07, 1.3618e-07, 5.5041e-07,\n", - " -6.3429e-07, 5.8772e-07, -3.7449e-07, 4.5133e-07, 3.4375e-07,\n", - " -6.1737e-07, -3.8223e-07, 1.5771e-07, -1.7511e-07, 3.0276e-09,\n", - " -2.9674e-07, -7.2252e-07, -6.7949e-09, -3.4926e-07, 4.3451e-08,\n", - " 1.0547e-07, 3.6982e-07, -2.0552e-07, 3.6342e-08, -4.7638e-08,\n", - " 1.4293e-07, 4.3917e-07, -7.5869e-08, -9.9415e-08, 4.5105e-08,\n", - " -1.0068e-08, -8.3892e-07, -7.9137e-08, -7.0750e-07, 1.9178e-07,\n", - " 3.8754e-07, 8.4444e-08, 2.0464e-07, 1.0252e-07, 7.5644e-08,\n", - " -6.1341e-08, -3.8856e-07, -3.5416e-07, -3.2500e-08, 1.5496e-06,\n", - " 9.9053e-09, 2.1292e-07, -3.9759e-07, 1.7524e-08, -7.2150e-07,\n", - " -2.6113e-07, -5.0575e-07, -5.3997e-08, -5.2743e-07, -1.2711e-07,\n", - " 6.4650e-09, -6.3542e-07, -1.7246e-07, 2.9385e-11, -6.1737e-07,\n", - " -3.0933e-07, -2.7813e-07, 6.1660e-07, -2.8017e-08, -1.2472e-06,\n", - " 3.9303e-07, -7.5703e-07, 4.1201e-07, -7.2459e-07, 2.2319e-07,\n", - " 2.6729e-07, 2.2822e-07, 3.2175e-07, -2.8173e-07, -3.4105e-07,\n", - " 2.9246e-07, -2.6755e-07, 5.5486e-07, 4.6074e-07, 7.9531e-07,\n", - " 1.4455e-07, -3.7617e-07, 1.2965e-07, 3.5979e-07, -7.1783e-08,\n", - " 7.6971e-07, -3.0106e-07, 4.4186e-07, -5.5656e-07, -3.6725e-07,\n", - " 3.2615e-07, 3.9505e-07, 2.8579e-07, 3.9586e-07, 9.8934e-07,\n", - " 6.1876e-07, -1.5795e-07, -1.0104e-06, 8.5002e-07, 8.5860e-08,\n", - " 3.2351e-07, -3.0993e-07, -3.4158e-07, 1.5116e-07, 9.9469e-08,\n", - " 6.0069e-08, -1.5719e-07, 8.0823e-08, 5.0363e-09, -6.6124e-07,\n", - " 2.8697e-07, -2.8923e-07, -3.3104e-07, -2.6741e-07, 2.1849e-07,\n", - " 2.2162e-07, 3.4295e-07, 7.9886e-07, 2.3284e-07, -6.5059e-08,\n", - " 2.3726e-07, -5.0067e-07, 2.1284e-07, -4.0939e-07, -3.2488e-07,\n", - " 7.8382e-08, 4.8203e-07, 1.9867e-07, -7.4483e-08, -3.1353e-07,\n", - " 5.8297e-08, -3.2046e-07, -1.1322e-07, -1.7365e-07, 3.7144e-07,\n", - " -7.3813e-08, -1.8838e-07, 2.8666e-07, -1.7835e-07, 5.0580e-07,\n", - " 2.9507e-07, 5.4638e-07, 2.3299e-07, 2.9693e-07, -6.9876e-07,\n", - " 8.6258e-08, -2.5660e-07, 5.0152e-07, 4.6366e-07, -2.0910e-08,\n", - " 1.8724e-08, -2.8327e-07, -2.6789e-07, -1.1106e-07, 1.1285e-08,\n", - " -1.3813e-07, 5.0809e-07, -6.2693e-08, -1.2473e-07, -2.0929e-07,\n", - " 3.2090e-07, 1.5514e-07, 6.5293e-07, -2.9884e-07, -1.3903e-07,\n", - " -2.4482e-07, 5.9293e-08, -3.9810e-07, 4.6002e-07, -7.9899e-07,\n", - " -5.8540e-07, 3.9786e-07, -3.2218e-07, 1.7169e-07, -4.9423e-07,\n", - " 1.0056e-07, 1.6145e-07, -9.8824e-08, 1.5913e-07, -4.6710e-07,\n", - " 1.5865e-07, 1.9589e-07, 3.9636e-07, 8.4565e-08, -6.8108e-07,\n", - " -6.6097e-07, 2.5134e-07, 6.7006e-07, 8.2307e-08, -7.5262e-08,\n", - " -3.2836e-07, -1.3761e-07, 1.0036e-07, -4.3035e-07, 4.0706e-07,\n", - " -1.3434e-07, -1.8913e-07, 4.9829e-07, -4.5375e-08, 5.6738e-07,\n", - " 4.0693e-07, -5.6909e-08, -5.6851e-07, 9.1993e-07, 4.2265e-07,\n", - " -1.1494e-07, -7.5589e-08, 1.8950e-07, -1.6677e-07, 2.5244e-08,\n", - " 5.4369e-07, -3.2131e-07, 6.1759e-07, -1.5090e-07, -3.5649e-08,\n", - " -1.8636e-07, -6.5008e-07, -1.6549e-07, -1.0451e-07, -5.9369e-07,\n", - " 8.2756e-07, 1.3586e-07, 6.7147e-08, 2.9688e-07, 3.7646e-07,\n", - " -1.6771e-07, -1.4768e-07, -2.5235e-07, -6.7719e-07, -4.6095e-07,\n", - " -2.8132e-07, -6.6112e-07, -8.3085e-07, -2.6775e-07, 2.5259e-07,\n", - " 7.4928e-07, -1.5687e-07, -4.7067e-07, -3.1492e-07, -2.0967e-07,\n", - " 9.5000e-08, -4.0278e-07, -1.6959e-07, -7.5251e-08, -2.5161e-07,\n", - " -2.2986e-07, 1.5039e-07, 2.9913e-07, -6.2833e-07, 8.5704e-08,\n", - " -2.2223e-08, -3.4837e-07, 1.6446e-07, 9.9939e-08, -6.5905e-07,\n", - " -4.8044e-07, -2.2191e-07, -2.0167e-07, 4.7854e-08, -4.6355e-07,\n", - " 2.7731e-07, -1.7976e-07, 2.6123e-07, 7.7950e-08, -4.4389e-07,\n", - " 4.6500e-07, -7.5943e-07, 1.8375e-07, -1.8547e-07, 3.4531e-07,\n", - " 2.7376e-07, -2.6321e-07, 3.0851e-07, 3.6956e-07, 3.3230e-07,\n", - " -2.0266e-07, -5.1608e-07, -3.1862e-07, -2.3650e-07, 2.8463e-07,\n", - " 4.2497e-08, 3.3198e-07, 2.9593e-07, -5.1724e-07, -1.7267e-07,\n", - " -2.0332e-07, 3.9231e-08, -1.3391e-08, -1.3861e-07, -2.3822e-07,\n", - " 2.4600e-07, -2.2045e-07, -1.1891e-07, 3.1210e-07, -1.2919e-07,\n", - " -8.2457e-07, 7.9238e-08, 2.3087e-07, -3.4217e-07, 2.6637e-07,\n", - " -3.7788e-07, 3.4308e-07, 6.8147e-08, 7.7669e-09, 4.1843e-07,\n", - " -4.6620e-07, -5.8995e-07, -1.9276e-07, 4.6281e-07, -1.7721e-07,\n", - " -4.2817e-07, 4.8755e-08, 1.8626e-07, 3.1048e-07, -3.5857e-07,\n", - " -1.2703e-07, 6.3200e-07, 1.3583e-07, 1.9140e-07, -8.6435e-08,\n", - " -1.2847e-08, 4.4478e-08, 4.1731e-07, -3.6239e-07, -2.0481e-07,\n", - " 4.8974e-07, -1.8279e-07, 2.0065e-07, 5.7776e-07, -2.9647e-07,\n", - " -4.1236e-07, -1.0541e-07, 2.4803e-07, -4.6571e-08, 1.3400e-07,\n", - " -7.4622e-07, -4.9445e-07, -9.1950e-08, -1.6310e-06, -5.4975e-07,\n", - " 1.1868e-06, 2.4416e-08, -1.0423e-07, 7.2068e-07, -3.8962e-07,\n", - " 1.3652e-07, 1.3550e-07, 6.9641e-07, 3.6471e-07, 5.4914e-08,\n", - " -4.5800e-08, 4.8776e-07, 5.0718e-07, 1.3634e-07, -2.0882e-07,\n", - " 4.4605e-07, 5.6278e-08, -5.4570e-08, 1.6894e-07, -3.2997e-07,\n", - " -3.0133e-07, 5.0226e-07, 8.4537e-08, 9.8503e-07, -2.7918e-08,\n", - " 1.2293e-07, 4.9457e-07, -9.1373e-08, 9.6624e-08, -5.7357e-07,\n", - " -2.6009e-07, 3.5024e-07, 1.7546e-07, 4.5867e-07, -4.8926e-07,\n", - " 1.1235e-07, -5.2832e-08, 6.5561e-07, -1.5632e-08, -1.9907e-07,\n", - " -2.0202e-08, -4.4126e-07, -4.0808e-07, 6.8809e-07, 9.7743e-07,\n", - " 3.4900e-07, 3.0799e-07, -1.1393e-07, 2.6764e-07, 3.8836e-08,\n", - " 1.1816e-07, -2.1617e-07, -2.2566e-07, 1.4733e-07, -8.5993e-08,\n", - " -8.8775e-07, -9.8369e-08], device='cuda:0'),\n", - " tensor([[[[-2.2165e-07, 2.0303e-07, -6.1677e-07],\n", - " [-6.1281e-07, -1.0803e-07, -1.0654e-06],\n", - " [ 8.0559e-08, 9.6671e-07, 4.0745e-07]],\n", - " \n", - " [[-1.6764e-07, -3.0175e-07, 3.3155e-07],\n", - " [-3.2131e-07, 4.1910e-07, 9.8068e-07],\n", - " [-6.7288e-08, 1.1176e-08, 2.0489e-08]],\n", - " \n", - " [[ 3.4925e-07, 1.0359e-06, 2.6543e-07],\n", - " [ 8.0699e-07, 1.3872e-06, 1.0617e-06],\n", - " [ 1.3690e-07, 9.4529e-07, 3.4831e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.3481e-07, 2.9150e-07, 2.3283e-08],\n", - " [-3.7393e-07, -1.5474e-06, -9.4692e-07],\n", - " [ 6.2678e-07, 6.4913e-07, 3.6112e-07]],\n", - " \n", - " [[-4.2934e-07, -9.1922e-07, -9.2387e-07],\n", - " [-7.3854e-07, -1.7500e-06, -1.1366e-06],\n", - " [-5.5600e-07, -7.6881e-07, -8.1677e-07]],\n", - " \n", - " [[-7.3900e-07, -8.7754e-07, -7.6485e-07],\n", - " [-3.6322e-07, -6.5938e-07, -1.6764e-07],\n", - " [ 2.1700e-07, -5.4840e-07, 3.9954e-07]]],\n", - " \n", - " \n", - " [[[-4.7870e-07, -2.8592e-07, 3.8743e-07],\n", - " [-1.2480e-07, -6.2399e-08, 1.7975e-07],\n", - " [ 1.3504e-08, -1.5134e-07, 1.8626e-09]],\n", - " \n", - " [[-1.9232e-07, -5.8580e-07, -4.8615e-07],\n", - " [-1.5832e-08, -8.5682e-08, -6.5193e-08],\n", - " [ 1.2666e-07, 5.0291e-08, 8.4750e-08]],\n", - " \n", - " [[-8.2701e-07, -1.0785e-06, -3.9348e-07],\n", - " [-9.0618e-07, -1.1120e-06, -8.2981e-07],\n", - " [-1.1176e-07, -4.5542e-07, -2.0768e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.7160e-07, -4.8429e-07, -3.3993e-07],\n", - " [ 1.4901e-08, -5.0478e-07, 1.7788e-07],\n", - " [ 3.8843e-08, 3.3434e-07, -9.6858e-08]],\n", - " \n", - " [[-2.2072e-07, 4.7963e-07, -4.8021e-08],\n", - " [ 3.1989e-07, 2.6217e-07, 3.2736e-07],\n", - " [-5.9605e-08, 1.9651e-07, 4.3586e-07]],\n", - " \n", - " [[ 1.1539e-06, 1.0452e-06, 6.4378e-07],\n", - " [ 3.7416e-07, 6.8918e-07, 8.1491e-07],\n", - " [ 3.1095e-07, 8.1595e-07, 6.6403e-07]]],\n", - " \n", - " \n", - " [[[ 6.9238e-07, 2.6543e-07, -3.1618e-07],\n", - " [ 1.3020e-06, 9.6485e-07, 5.2620e-07],\n", - " [ 9.3179e-07, 5.4948e-07, -9.3132e-10]],\n", - " \n", - " [[-1.0533e-06, -2.3115e-06, -5.2340e-07],\n", - " [-1.0659e-06, -1.6280e-06, -1.0915e-06],\n", - " [-4.7591e-07, -4.7754e-07, -3.6196e-07]],\n", - " \n", - " [[ 6.3385e-07, 7.9814e-07, 7.7020e-07],\n", - " [ 6.5332e-07, 4.4331e-07, 6.0163e-07],\n", - " [ 4.2340e-07, 3.6228e-07, 4.4424e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-9.7090e-07, -2.0806e-06, -1.5544e-06],\n", - " [-1.9232e-07, -2.1662e-06, -1.9390e-06],\n", - " [-4.0513e-07, -9.7416e-07, -1.3700e-06]],\n", - " \n", - " [[-5.5414e-07, -1.2098e-06, -1.4147e-06],\n", - " [-6.9407e-07, -1.4040e-06, -1.8161e-06],\n", - " [-7.6079e-07, -4.7218e-07, -2.5006e-07]],\n", - " \n", - " [[ 1.8300e-07, 6.2957e-07, 2.3027e-07],\n", - " [ 2.1514e-07, 2.6240e-07, 1.2740e-06],\n", - " [ 4.6566e-09, 2.3586e-07, 5.8953e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[ 5.2247e-07, 1.2461e-06, 2.1979e-07],\n", - " [-1.7649e-07, 3.9861e-07, -4.1863e-07],\n", - " [ 1.8626e-08, 1.2740e-06, 5.3085e-08]],\n", - " \n", - " [[ 1.5367e-06, -8.8196e-07, 3.3248e-07],\n", - " [ 7.5158e-07, 4.9314e-07, 1.0021e-06],\n", - " [-1.9278e-07, 4.9500e-07, 7.5088e-07]],\n", - " \n", - " [[-2.5658e-07, -5.8790e-07, 8.9919e-07],\n", - " [ 3.9954e-07, -3.3545e-07, 6.2259e-07],\n", - " [ 3.8929e-07, 7.3574e-07, 5.8487e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.8915e-06, 1.6885e-06, 1.0710e-06],\n", - " [ 7.8429e-07, 3.8184e-07, -1.9744e-07],\n", - " [ 7.2224e-07, 1.5968e-06, 2.2436e-06]],\n", - " \n", - " [[-7.7859e-07, -1.6014e-06, -2.0557e-06],\n", - " [-1.3784e-07, -1.1573e-06, -1.0915e-06],\n", - " [ 3.5670e-07, 1.4063e-07, -7.3574e-07]],\n", - " \n", - " [[ 5.0943e-07, -2.4890e-07, -4.6473e-07],\n", - " [-5.3784e-07, -1.0850e-06, -8.6334e-07],\n", - " [-8.3886e-07, -1.1204e-06, -9.3084e-07]]],\n", - " \n", - " \n", - " [[[ 5.8766e-07, 1.6061e-06, -3.1432e-08],\n", - " [-3.5856e-07, -9.9000e-07, -5.1223e-08],\n", - " [ 6.2864e-07, 4.1490e-07, 3.6744e-07]],\n", - " \n", - " [[-1.9530e-06, -1.0896e-06, -7.9721e-07],\n", - " [-1.5832e-06, -1.4724e-06, -1.4277e-06],\n", - " [-6.9430e-07, -7.6258e-07, -1.2978e-06]],\n", - " \n", - " [[ 6.4168e-07, 4.7969e-07, 1.4501e-06],\n", - " [ 1.8075e-06, -3.8091e-07, 3.1712e-07],\n", - " [ 9.0525e-07, 9.8301e-07, 1.5246e-06]],\n", - " \n", - " ...,\n", - " \n", - " [[-4.5868e-07, 2.2026e-07, 1.2293e-07],\n", - " [-1.0002e-06, -1.7951e-07, -7.7835e-07],\n", - " [-2.3097e-07, -5.4669e-07, -5.0990e-07]],\n", - " \n", - " [[ 2.5705e-07, 4.8149e-07, 4.6613e-07],\n", - " [ 3.8184e-07, 1.0817e-06, 2.4680e-08],\n", - " [ 9.2667e-07, 1.0850e-06, 1.3742e-06]],\n", - " \n", - " [[-7.5763e-07, -8.2329e-07, 2.7381e-07],\n", - " [-9.4250e-07, -8.2445e-07, -7.4506e-09],\n", - " [-3.8929e-07, -3.0390e-07, -5.6066e-07]]],\n", - " \n", - " \n", - " [[[-2.7078e-07, 1.6624e-07, 5.3272e-07],\n", - " [ 2.5315e-07, 4.3679e-07, 1.8068e-07],\n", - " [ 1.8068e-07, -3.3714e-07, -3.2596e-09]],\n", - " \n", - " [[-8.0187e-07, -1.2629e-06, -8.6147e-07],\n", - " [-6.3705e-07, -6.8382e-07, -1.0277e-06],\n", - " [-2.5611e-07, -1.2852e-07, 4.2142e-07]],\n", - " \n", - " [[-6.1281e-07, 5.6298e-07, -6.1467e-07],\n", - " [-6.4680e-07, 7.9628e-07, -3.4645e-07],\n", - " [-2.0023e-07, 4.4703e-08, -5.2899e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.6787e-07, -4.0513e-07, 1.7043e-07],\n", - " [-3.1851e-07, -2.0163e-07, 3.3528e-08],\n", - " [-4.8149e-07, -1.8207e-07, -7.6601e-08]],\n", - " \n", - " [[-3.6275e-07, -4.3306e-08, 9.2736e-07],\n", - " [ 3.5577e-07, 8.0373e-07, 2.0396e-06],\n", - " [-2.5146e-08, 9.7509e-07, 1.7257e-06]],\n", - " \n", - " [[-1.4913e-06, -8.7731e-07, -1.8626e-07],\n", - " [-6.0396e-07, -3.6787e-07, -6.5751e-07],\n", - " [ 1.0910e-06, 8.3167e-07, 6.7987e-08]]]], device='cuda:0'),\n", - " tensor([-9.5367e-07, 4.7684e-07, -1.7881e-07, -1.1921e-07, -7.1526e-07,\n", - " -3.5763e-07, -2.3842e-07, 0.0000e+00, -4.7684e-07, 2.3842e-07,\n", - " 0.0000e+00, -1.1921e-07, 3.5763e-07, -1.1921e-07, -1.7881e-07,\n", - " -1.1921e-06, 2.3842e-07, 2.3842e-07, -7.7486e-07, 5.9605e-07,\n", - " 3.5763e-07, -8.3447e-07, 1.1921e-07, 0.0000e+00, -5.9605e-07,\n", - " 3.5763e-07, -1.2517e-06, -2.9802e-07, -8.3447e-07, 1.1921e-06,\n", - " -2.3842e-07, 2.3842e-07, 3.5763e-07, 5.9605e-07, 4.7684e-07,\n", - " 8.3447e-07, -4.7684e-07, -3.5763e-07, -2.9802e-07, -1.4305e-06,\n", - " 5.9605e-07, 0.0000e+00, -5.9605e-08, -1.1921e-07, 0.0000e+00,\n", - " -5.9605e-08, -1.1921e-07, -5.9605e-08, -4.1723e-07, 5.9605e-07,\n", - " -4.7684e-07, 1.1921e-07, -5.9605e-08, -2.9802e-07, -4.7684e-07,\n", - " 0.0000e+00, 5.9605e-07, -7.7486e-07, -2.3842e-07, -1.0133e-06,\n", - " 3.5763e-07, -1.1921e-07, -4.7684e-07, -1.1921e-07, -3.5763e-07,\n", - " -1.1921e-07, -9.5367e-07, -5.9605e-08, -2.3842e-07, 3.5763e-07,\n", - " -5.9605e-08, -1.7881e-07, -5.9605e-08, -1.1325e-06, 7.1526e-07,\n", - " -2.9802e-07, 1.4305e-06, 2.3842e-07, -1.7881e-07, -1.7881e-07,\n", - " 2.3842e-07, -1.0729e-06, 0.0000e+00, 4.7684e-07, -3.5763e-07,\n", - " 2.3842e-07, -5.3644e-07, -4.7684e-07, 3.5763e-07, -2.3842e-07,\n", - " 7.1526e-07, 4.7684e-07, -1.1921e-07, -6.5565e-07, -2.3842e-07,\n", - " -1.7881e-07, 3.5763e-07, -2.9802e-07, -3.5763e-07, -3.5763e-07,\n", - " -7.1526e-07, -1.1921e-07, 3.5763e-07, -8.3447e-07, -1.7881e-07,\n", - " 3.5763e-07, 2.3842e-07, -5.9605e-07, 2.3842e-07, -1.1921e-07,\n", - " 0.0000e+00, 7.1526e-07, 3.5763e-07, -7.7486e-07, 1.1921e-07,\n", - " -5.9605e-07, -1.1921e-07, -1.3113e-06, -8.9407e-07, 4.7684e-07,\n", - " -1.1921e-07, -4.7684e-07, 0.0000e+00, -4.1723e-07, -1.1921e-07,\n", - " -5.9605e-08, -5.9605e-08, 2.3842e-07, 1.1921e-07, 1.1921e-07,\n", - " -1.7881e-07, -7.1526e-07, -6.5565e-07, -2.9802e-07, 3.5763e-07,\n", - " -2.3842e-07, -1.7881e-07, 8.3447e-07, -7.1526e-07, -2.3842e-07,\n", - " 7.1526e-07, 0.0000e+00, -4.1723e-07, -5.3644e-07, 1.1921e-07,\n", - " -7.7486e-07, -1.1921e-07, 2.3842e-07, 0.0000e+00, -5.9605e-08,\n", - " 0.0000e+00, -5.9605e-08, -4.1723e-07, 3.5763e-07, 5.9605e-07,\n", - " 1.1921e-07, -7.7486e-07, -2.9802e-07, -2.9802e-07, 0.0000e+00,\n", - " -3.5763e-07, 0.0000e+00, 0.0000e+00, -5.9605e-08, -5.9605e-08,\n", - " -5.9605e-08, 8.3447e-07, -4.1723e-07, 0.0000e+00, 3.5763e-07,\n", - " -1.7881e-07, -3.5763e-07, -4.1723e-07, -5.9605e-07, -6.5565e-07,\n", - " 1.1921e-07, 1.1921e-07, -5.9605e-07, -5.3644e-07, -1.4305e-06,\n", - " 2.3842e-07, -6.5565e-07, 1.1921e-07, -5.9605e-08, 8.3447e-07,\n", - " 5.9605e-07, -1.7881e-07, 5.9605e-07, -2.9802e-07, -4.1723e-07,\n", - " 0.0000e+00, -5.3644e-07, -4.1723e-07, -1.5497e-06, 1.1921e-07,\n", - " 2.3842e-07, 3.5763e-07, -4.7684e-07, -4.1723e-07, 7.1526e-07,\n", - " 1.1921e-07, 4.7684e-07, -2.3842e-07, 5.9605e-07, 2.3842e-07,\n", - " -2.3842e-07, 3.5763e-07, 7.1526e-07, -5.3644e-07, 1.1921e-07,\n", - " 1.1921e-07, 2.3842e-07, 1.1921e-07, 1.1921e-07, 3.5763e-07,\n", - " 7.1526e-07, 2.3842e-07, 0.0000e+00, 8.3447e-07, 8.3447e-07,\n", - " 7.1526e-07, 3.5763e-07, -4.7684e-07, -5.9605e-08, 4.7684e-07,\n", - " -1.1921e-07, 1.1921e-07, 8.3447e-07, 4.7684e-07, -8.3447e-07,\n", - " 2.3842e-07, 0.0000e+00, 3.5763e-07, -5.9605e-08, 1.1921e-07,\n", - " -5.9605e-07, 1.1921e-07, -3.5763e-07, 7.1526e-07, -1.7881e-07,\n", - " -1.0133e-06, -7.7486e-07, 0.0000e+00, 8.3447e-07, 4.7684e-07,\n", - " -3.5763e-07, 0.0000e+00, 7.1526e-07, -8.3447e-07, -1.1921e-07,\n", - " -5.3644e-07, -4.1723e-07, 0.0000e+00, 3.5763e-07, 7.1526e-07,\n", - " -1.0729e-06, 1.1921e-07, 1.1921e-07, 3.5763e-07, -4.1723e-07,\n", - " -3.5763e-07, -2.3842e-07, -1.7881e-07, -2.3842e-07, 2.3842e-07,\n", - " 1.1921e-07, 0.0000e+00, -4.7684e-07, 4.7684e-07, -5.3644e-07,\n", - " -5.3644e-07, -5.9605e-07, -5.9605e-08, 3.5763e-07, -4.1723e-07,\n", - " -1.7881e-07, 5.9605e-07, -2.3842e-07, 4.7684e-07, -5.3644e-07,\n", - " -2.3842e-07, 7.1526e-07, 8.3447e-07, -2.9802e-07, -1.1921e-07,\n", - " -3.5763e-07, 1.1921e-07, 9.5367e-07, 3.5763e-07, 1.1921e-06,\n", - " 1.4305e-06, 7.1526e-07, -1.7881e-07, -5.9605e-08, -4.1723e-07,\n", - " 1.0729e-06, -5.3644e-07, -1.1921e-07, 3.5763e-07, -8.9407e-07,\n", - " 4.7684e-07, -3.5763e-07, -5.9605e-07, 0.0000e+00, -7.7486e-07,\n", - " 1.1921e-07, 3.5763e-07, -2.9802e-07, -3.5763e-07, -1.2517e-06,\n", - " -1.1921e-07, -7.1526e-07, -2.3842e-07, 7.1526e-07, -6.5565e-07,\n", - " 1.1921e-07, 5.9605e-07, -3.5763e-07, -1.1921e-07, 1.1921e-07,\n", - " -3.5763e-07, 2.3842e-07, -4.7684e-07, -1.1921e-07, -5.9605e-08,\n", - " 2.3842e-07, -5.9605e-08, -2.9802e-07, 3.5763e-07, -5.3644e-07,\n", - " 5.9605e-07, -1.7881e-07, -1.1921e-07, 3.5763e-07, 3.5763e-07,\n", - " -8.3447e-07, -1.3709e-06, -1.7881e-07, 1.0729e-06, 1.4305e-06,\n", - " 0.0000e+00, 0.0000e+00, 2.3842e-07, -2.3842e-07, 2.3842e-07,\n", - " 1.3113e-06, 8.3447e-07, -1.0133e-06, -5.9605e-07, -1.1921e-07,\n", - " 2.3842e-07, -8.3447e-07, 7.1526e-07, -1.0133e-06, 3.5763e-07,\n", - " -5.3644e-07, -3.5763e-07, -5.9605e-07, 8.3447e-07, 1.1921e-07,\n", - " -1.1921e-07, 1.1921e-07, -5.9605e-08, -1.1921e-07, 5.9605e-07,\n", - " 9.5367e-07, 5.9605e-07, 4.7684e-07, 1.1921e-07, 0.0000e+00,\n", - " 7.1526e-07, 5.9605e-07, -3.5763e-07, 5.9605e-07, -7.1526e-07,\n", - " 5.9605e-07, -5.9605e-08, -4.7684e-07, -5.9605e-07, -4.1723e-07,\n", - " -2.3842e-07, 1.1921e-07, 2.3842e-07, -4.1723e-07, -2.3842e-07,\n", - " -2.9802e-07, 1.1921e-07, 0.0000e+00, 1.1921e-07, 4.7684e-07,\n", - " 0.0000e+00, 0.0000e+00, -3.5763e-07, -2.9802e-07, 2.3842e-07,\n", - " -7.7486e-07, 8.3447e-07, -2.3842e-07, -8.3447e-07, -1.0133e-06,\n", - " -1.1921e-07, -3.5763e-07, -3.5763e-07, -1.7881e-07, 7.1526e-07,\n", - " 2.3842e-07, -5.9605e-07, 8.3447e-07, 8.3447e-07, -7.7486e-07,\n", - " -5.3644e-07, 0.0000e+00, -5.9605e-08, 5.9605e-07, -1.6093e-06,\n", - " 8.3447e-07, 3.5763e-07, -5.9605e-07, 2.3842e-07, 4.7684e-07,\n", - " -5.3644e-07, 5.9605e-07, -5.9605e-08, 3.5763e-07, 5.9605e-07,\n", - " -8.3447e-07, 3.5763e-07, 1.1921e-07, 3.5763e-07, -3.5763e-07,\n", - " 1.1921e-07, -7.1526e-07, 2.3842e-07, -7.1526e-07, -2.3842e-07,\n", - " 1.1921e-07, -5.9605e-07, 1.1921e-07, -5.9605e-08, -5.3644e-07,\n", - " 1.1921e-07, -1.1325e-06, -5.9605e-07, -1.7881e-07, 3.5763e-07,\n", - " 1.1921e-07, 5.9605e-07, -1.7881e-07, 1.0729e-06, 1.1921e-07,\n", - " 8.3447e-07, -2.9802e-07, 1.1921e-07, 8.3447e-07, 0.0000e+00,\n", - " 1.7881e-06, 1.1921e-06, 1.1921e-07, -1.0729e-06, 4.7684e-07,\n", - " -8.9407e-07, -7.1526e-07, -2.9802e-07, 2.3842e-07, -4.1723e-07,\n", - " -5.3644e-07, -3.5763e-07, -2.3842e-07, 0.0000e+00, 3.5763e-07,\n", - " -4.7684e-07, -1.0133e-06, -4.7684e-07, -2.3842e-07, 0.0000e+00,\n", - " -1.8477e-06, 2.3842e-07, 2.3842e-07, -3.5763e-07, 9.5367e-07,\n", - " -7.1526e-07, -1.7881e-07, 1.1921e-07, 1.1921e-07, 3.5763e-07,\n", - " 2.3842e-07, 2.3842e-07, 5.9605e-07, -6.5565e-07, 1.1921e-07,\n", - " -4.1723e-07, -2.3842e-07, 4.7684e-07, -1.1921e-07, 4.7684e-07,\n", - " 8.3447e-07, 5.9605e-07, -4.1723e-07, 5.9605e-07, -1.0133e-06,\n", - " 1.1921e-06, -1.1921e-06, -1.1921e-07, 9.5367e-07, 2.3842e-07,\n", - " -1.1325e-06, 1.1921e-07, 0.0000e+00, -1.7881e-07, 2.3842e-07,\n", - " 5.9605e-07, 3.5763e-07], device='cuda:0'),\n", - " tensor([ 1.4361e-07, 3.7875e-07, -8.3476e-07, 9.4134e-07, -6.7843e-07,\n", - " 3.1072e-07, 1.5542e-07, 1.5177e-07, 5.9782e-07, 5.7706e-07,\n", - " 3.6449e-07, -7.4512e-08, 1.9858e-07, -4.9469e-07, 7.5448e-07,\n", - " -5.9229e-07, 5.5775e-07, 7.6487e-07, -1.4062e-06, 7.2411e-07,\n", - " 4.1560e-07, -1.3103e-06, -6.2538e-07, -3.8220e-09, -1.0456e-06,\n", - " 1.7845e-07, -7.8648e-07, -4.8804e-07, -1.0593e-06, 1.1402e-06,\n", - " -2.8512e-07, 8.6257e-07, 5.7358e-07, -2.7734e-08, -1.3981e-07,\n", - " 6.7581e-07, -1.1391e-06, 3.5621e-08, -1.1710e-07, -5.2912e-07,\n", - " 1.0740e-06, 1.8467e-07, -6.4858e-08, -7.4553e-07, -4.4954e-07,\n", - " -2.0085e-08, -2.6388e-07, 1.0682e-07, -4.3396e-07, 5.0373e-07,\n", - " -3.1702e-09, 2.1772e-07, -3.6832e-07, -7.9641e-07, -5.5592e-07,\n", - " -8.9712e-08, -2.3330e-07, -5.7435e-07, 2.1003e-07, -3.4901e-07,\n", - " -1.2920e-07, -7.1090e-08, -2.8965e-07, -6.8484e-08, 1.1578e-07,\n", - " 1.8122e-07, -8.2851e-07, -5.4816e-07, -5.4788e-07, 6.4785e-07,\n", - " -6.7243e-07, -2.9483e-07, 3.0059e-07, -6.2761e-07, -1.3943e-07,\n", - " -4.6666e-08, 8.2269e-07, 5.2700e-07, -2.0402e-07, -1.8630e-07,\n", - " -1.4299e-07, -5.8553e-07, -4.3453e-08, 1.8669e-07, 4.3835e-08,\n", - " -1.5051e-07, 1.0361e-08, -8.9533e-07, -7.6743e-08, -2.0186e-07,\n", - " 5.8199e-07, 4.8817e-07, 5.3219e-07, 1.5891e-07, -5.4972e-07,\n", - " -4.3597e-08, 6.6097e-07, -4.5730e-10, -5.9889e-08, -5.4971e-07,\n", - " 1.5863e-08, 4.2603e-07, 4.4973e-07, -3.9109e-07, 3.3712e-07,\n", - " 4.1975e-07, 7.8129e-07, -4.5257e-07, -2.4825e-07, -6.3345e-07,\n", - " -7.7735e-07, 9.1151e-07, 5.8416e-08, -2.9035e-07, -1.3461e-06,\n", - " -2.6095e-07, -3.0684e-07, -1.6109e-06, -1.3722e-06, 2.2558e-08,\n", - " -4.3157e-07, -1.2280e-07, -1.1057e-07, -5.6610e-07, -2.7475e-07,\n", - " 3.3783e-07, -4.9134e-07, 5.5273e-07, 3.0693e-09, -2.3143e-07,\n", - " -8.8538e-07, -1.6646e-07, -1.4726e-07, -3.3940e-07, 1.1079e-06,\n", - " -3.8600e-07, 1.9262e-07, 8.6771e-09, -6.1051e-07, 2.0421e-07,\n", - " 6.8559e-07, 1.6167e-07, 6.7790e-07, -6.8239e-07, 1.2664e-07,\n", - " -9.9949e-09, 3.2615e-07, -6.1828e-07, -3.7162e-07, -3.7594e-07,\n", - " -1.4025e-09, -6.1806e-07, -3.8709e-07, -1.8537e-08, 5.6683e-07,\n", - " -1.8389e-07, -2.1760e-06, -2.1463e-07, -3.7027e-07, -1.2480e-07,\n", - " -5.7935e-07, 3.2740e-07, -3.9764e-07, -8.7763e-07, -6.9524e-07,\n", - " 1.8890e-07, 6.0723e-07, 2.8119e-07, 1.9862e-07, -4.5653e-07,\n", - " -3.2686e-07, -6.4305e-07, 3.0504e-07, -7.7494e-07, -1.0217e-06,\n", - " 1.7248e-07, 3.5446e-07, 3.2754e-07, -8.2340e-07, -2.8620e-07,\n", - " -1.2324e-07, 1.4035e-08, -4.2806e-07, 2.2246e-07, 5.9490e-07,\n", - " 9.5043e-07, 1.6535e-07, -8.7346e-07, -7.8100e-07, -5.2794e-09,\n", - " -7.6545e-07, -5.1436e-07, -4.0977e-07, -1.1618e-06, 6.8537e-08,\n", - " -1.4605e-07, 5.8257e-08, -8.0188e-07, -2.0639e-07, 2.1339e-07,\n", - " 1.5732e-07, 9.9957e-07, 4.9952e-07, 1.3014e-06, 2.9355e-07,\n", - " -3.4369e-07, 5.1924e-07, 5.3847e-07, -9.4620e-07, 5.6011e-07,\n", - " 7.9131e-07, -5.2138e-09, -1.4767e-06, -6.3886e-07, 1.4941e-07,\n", - " 9.8527e-07, -3.7774e-07, 3.2528e-07, 9.4757e-07, 2.1744e-07,\n", - " 5.9661e-07, -1.1091e-07, -6.2895e-07, 4.1626e-07, 9.3376e-07,\n", - " -2.4913e-07, 2.1312e-08, 7.0954e-07, 1.7708e-07, -1.0764e-06,\n", - " -1.6039e-07, -2.0194e-07, 3.0868e-07, -2.1210e-07, 2.8911e-07,\n", - " -3.3863e-07, 3.0655e-07, -8.2397e-07, 6.2707e-07, 4.0511e-07,\n", - " -6.2630e-07, -5.2816e-07, 6.9402e-07, 3.1391e-07, 6.8661e-07,\n", - " 1.2054e-07, -2.9753e-07, 2.2532e-09, -5.5803e-07, -2.0193e-07,\n", - " -5.2379e-07, -4.5617e-07, 1.9949e-08, 9.5923e-07, 6.3150e-07,\n", - " -3.8950e-07, -8.7060e-07, 3.2801e-07, -1.5988e-07, -2.0283e-07,\n", - " 2.0498e-08, -6.2763e-08, 1.2942e-07, 4.8690e-07, 2.9747e-08,\n", - " -5.9033e-07, -2.3964e-07, -7.6579e-07, -5.2635e-07, -1.1527e-06,\n", - " 6.4606e-07, -8.1474e-07, -4.2017e-07, 6.5501e-07, 3.6491e-07,\n", - " 5.6719e-07, -6.4178e-08, 8.4156e-07, 1.8924e-07, 5.0438e-07,\n", - " -8.7186e-07, 5.7862e-07, 7.9785e-07, -1.1940e-07, -2.8611e-07,\n", - " -3.7413e-07, 6.9951e-07, 1.1857e-06, -7.0392e-07, 2.6504e-07,\n", - " 6.2547e-07, 4.0341e-07, -6.8973e-07, 1.0543e-06, -3.9495e-07,\n", - " 6.2045e-07, -5.2668e-07, 3.1209e-07, 2.9866e-08, -1.3103e-06,\n", - " -7.4060e-07, -2.1091e-07, -1.1769e-06, 4.9558e-07, -1.5905e-06,\n", - " 2.1223e-07, 3.8217e-07, 2.4839e-07, -9.2843e-07, -3.9248e-07,\n", - " 5.8080e-08, -9.3520e-07, 2.0115e-08, 8.0949e-07, -6.3440e-07,\n", - " -6.9958e-07, 3.6014e-08, 1.2903e-08, -7.1608e-08, -1.1944e-06,\n", - " -9.8625e-07, 4.3675e-07, -5.9806e-07, 7.8947e-07, -8.5338e-07,\n", - " -3.3651e-07, -5.3144e-07, -6.9125e-07, 1.4479e-07, -8.7782e-07,\n", - " 2.1694e-07, 6.6267e-07, -1.7303e-07, 5.9009e-07, 1.9341e-07,\n", - " -1.2323e-06, -3.1903e-07, 1.7893e-07, 8.5715e-07, 1.6794e-06,\n", - " 4.4822e-07, 8.3358e-07, 7.1812e-08, 1.7047e-07, -1.3995e-07,\n", - " 1.5727e-06, -7.6507e-07, -1.1366e-06, -6.4408e-07, -3.2520e-07,\n", - " 3.6125e-08, -5.6142e-07, 1.0516e-06, -1.0970e-06, -4.1031e-07,\n", - " 1.9087e-07, -9.4430e-08, -4.1627e-08, 7.3488e-07, 2.9984e-07,\n", - " -4.2442e-07, 1.1619e-06, -9.3843e-07, -3.9253e-07, 7.6044e-07,\n", - " 8.9230e-07, -6.1505e-07, 2.8537e-07, 1.9979e-07, 1.1293e-07,\n", - " 9.3129e-09, -4.7411e-07, -3.3502e-07, 6.4352e-07, -8.2211e-07,\n", - " 2.8376e-07, -2.1078e-07, -4.7676e-07, -3.2840e-07, -1.7811e-07,\n", - " -1.0372e-07, -1.5150e-07, -2.9330e-07, -5.0891e-08, 4.6162e-07,\n", - " -4.4931e-07, -1.1251e-07, -2.1810e-08, 3.6239e-07, 4.1384e-07,\n", - " -1.9962e-07, -2.2854e-07, 4.1248e-07, 1.3120e-07, 2.2709e-07,\n", - " -6.5830e-07, 2.7621e-07, -2.8515e-07, -7.1727e-07, -2.2612e-07,\n", - " 2.9839e-07, -1.8185e-07, -7.0273e-07, -2.9024e-07, 5.5723e-07,\n", - " -4.9760e-07, -7.6321e-07, 2.0529e-07, 8.0877e-07, -8.9062e-08,\n", - " -6.0039e-07, 2.3167e-07, -1.9675e-07, 6.2634e-08, -5.7431e-07,\n", - " 7.7656e-07, 4.2352e-07, -5.7160e-07, -1.6343e-08, 6.0179e-07,\n", - " -6.7309e-07, 3.5918e-07, 4.8857e-08, 1.7525e-07, -5.2018e-07,\n", - " -2.2266e-07, -1.4090e-07, 1.4820e-07, 5.7232e-07, -6.3711e-07,\n", - " 3.2627e-07, -1.3462e-06, -6.5572e-08, -6.0948e-07, -5.2886e-07,\n", - " -4.4639e-07, -5.9024e-07, -7.8491e-07, -1.6743e-07, 2.8212e-09,\n", - " -1.1809e-07, -6.3132e-07, 4.9501e-07, -5.7799e-07, 6.2618e-07,\n", - " 4.2153e-07, 3.3870e-07, 2.6234e-07, 1.1589e-06, -6.3834e-07,\n", - " -1.7349e-07, -2.8091e-07, 1.3085e-07, 2.7772e-07, 8.0872e-08,\n", - " 2.6623e-07, 9.5979e-07, 2.8653e-07, -1.1525e-06, 4.1606e-07,\n", - " -8.0760e-07, -9.3070e-07, -4.5792e-07, 3.1079e-07, 3.9923e-07,\n", - " -6.9447e-07, 4.0161e-08, 1.7961e-07, -2.0261e-07, 5.0900e-08,\n", - " -5.9011e-07, -8.9109e-07, -5.2656e-07, -1.3622e-07, 5.2093e-07,\n", - " -1.5905e-06, 1.5041e-06, 2.2004e-07, -3.0572e-07, 5.7887e-07,\n", - " 1.7697e-07, -2.7734e-08, 5.3296e-07, 9.1149e-08, -3.8058e-07,\n", - " 3.5024e-07, 3.7948e-07, -1.5867e-08, -8.8103e-07, 7.0423e-08,\n", - " -8.5883e-07, -3.3453e-07, 6.0323e-07, -3.6401e-07, 3.0336e-07,\n", - " 4.4667e-07, 8.7888e-07, -1.7145e-07, 6.6095e-07, -7.5378e-07,\n", - " 3.2432e-07, -1.0764e-06, -9.0861e-08, 7.4299e-07, 2.8685e-07,\n", - " -8.3725e-07, 3.7188e-07, 1.8325e-07, 6.5297e-07, 1.3390e-07,\n", - " 6.0233e-07, 3.0721e-07], device='cuda:0'),\n", - " tensor([[[[-4.5076e-07]],\n", - " \n", - " [[-5.0198e-07]],\n", - " \n", - " [[-4.6566e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.9802e-07]],\n", - " \n", - " [[-2.2165e-06]],\n", - " \n", - " [[-5.5134e-07]]],\n", - " \n", - " \n", - " [[[ 5.2154e-07]],\n", - " \n", - " [[ 1.3039e-06]],\n", - " \n", - " [[ 5.3085e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.8626e-08]],\n", - " \n", - " [[ 6.5938e-07]],\n", - " \n", - " [[ 1.2065e-06]]],\n", - " \n", - " \n", - " [[[ 8.9034e-07]],\n", - " \n", - " [[ 1.3784e-07]],\n", - " \n", - " [[ 2.5705e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[-3.3155e-07]],\n", - " \n", - " [[-5.2527e-07]],\n", - " \n", - " [[-7.6927e-07]]],\n", - " \n", - " \n", - " ...,\n", - " \n", - " \n", - " [[[-4.3958e-07]],\n", - " \n", - " [[-2.1048e-06]],\n", - " \n", - " [[-2.6077e-08]],\n", - " \n", - " ...,\n", - " \n", - " [[ 1.2107e-06]],\n", - " \n", - " [[ 1.5274e-07]],\n", - " \n", - " [[-6.1840e-07]]],\n", - " \n", - " \n", - " [[[-1.0449e-06]],\n", - " \n", - " [[-2.3097e-07]],\n", - " \n", - " [[ 3.6508e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 2.8498e-07]],\n", - " \n", - " [[ 3.6182e-07]],\n", - " \n", - " [[ 6.2399e-08]]],\n", - " \n", - " \n", - " [[[-2.6077e-08]],\n", - " \n", - " [[ 1.4901e-08]],\n", - " \n", - " [[-3.2037e-07]],\n", - " \n", - " ...,\n", - " \n", - " [[ 4.4703e-08]],\n", - " \n", - " [[-8.7544e-08]],\n", - " \n", - " [[-2.8126e-07]]]], device='cuda:0'),\n", - " tensor([ 1.1921e-07, 1.1921e-07, -1.7285e-06, 7.1526e-07, 7.1526e-07,\n", - " -1.1921e-07, 3.5763e-07, 4.7684e-07, 8.3447e-07, 0.0000e+00,\n", - " 4.7684e-07, 2.3842e-07, -4.1723e-07, -5.9605e-08, 5.9605e-07,\n", - " 0.0000e+00, 5.9605e-07, 7.1526e-07, -7.1526e-07, 2.3842e-07,\n", - " 2.3842e-07, -7.7486e-07, -2.9802e-07, -1.7881e-07, -8.3447e-07,\n", - " -6.5565e-07, -5.3644e-07, -2.3842e-07, -2.9802e-07, -9.5367e-07,\n", - " -4.1723e-07, 1.0729e-06, 5.9605e-07, 0.0000e+00, -5.9605e-08,\n", - " 1.1921e-07, -8.9407e-07, 3.5763e-07, 4.7684e-07, -1.7881e-07,\n", - " 4.7684e-07, 0.0000e+00, -2.9802e-07, -8.9407e-07, -6.5565e-07,\n", - " -5.9605e-08, -1.1921e-07, -2.9802e-07, -2.3842e-07, -1.7881e-07,\n", - " -5.9605e-08, 5.9605e-07, 2.3842e-07, -2.3842e-07, 3.5763e-07,\n", - " 2.3842e-07, -5.9605e-08, 2.3842e-07, 7.1526e-07, 7.1526e-07,\n", - " 1.1921e-07, 9.5367e-07, 0.0000e+00, 1.1921e-07, 3.5763e-07,\n", - " 5.9605e-07, 5.9605e-07, -1.1921e-07, -5.9605e-08, 4.7684e-07,\n", - " -5.9605e-07, -2.3842e-07, 4.7684e-07, -9.5367e-07, 1.1921e-07,\n", - " 3.5763e-07, 1.0729e-06, 1.1921e-07, 1.1921e-07, -5.3644e-07,\n", - " -2.3842e-07, -1.1921e-07, -5.9605e-08, -2.9802e-07, -4.7684e-07,\n", - " -6.5565e-07, 3.5763e-07, -4.7684e-07, 4.7684e-07, 0.0000e+00,\n", - " -3.5763e-07, 1.1921e-07, 3.5763e-07, 8.3447e-07, -2.9802e-07,\n", - " -1.1921e-07, 3.5763e-07, -7.1526e-07, -1.7881e-07, 1.1921e-07,\n", - " 1.1921e-07, 1.1921e-07, 1.1921e-07, 5.9605e-07, 5.9605e-07,\n", - " 1.1921e-07, -3.5763e-07, -7.1526e-07, -3.5763e-07, -4.1723e-07,\n", - " -9.5367e-07, 8.3447e-07, -1.1921e-07, -5.9605e-08, -1.3709e-06,\n", - " -5.9605e-08, -2.9802e-07, -8.9407e-07, -1.5497e-06, -2.9802e-07,\n", - " -9.5367e-07, -5.9605e-08, 1.1921e-07, -1.1921e-07, -5.9605e-07,\n", - " -2.3842e-07, -5.3644e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07,\n", - " -4.7684e-07, -5.9605e-08, 1.1921e-07, 1.1921e-07, 1.1921e-06,\n", - " -4.7684e-07, 1.3113e-06, -5.9605e-07, 0.0000e+00, -7.1526e-07,\n", - " 8.3447e-07, -5.9605e-08, 7.1526e-07, -4.7684e-07, 2.3842e-07,\n", - " -8.9407e-07, 0.0000e+00, -5.9605e-07, -4.1723e-07, -5.9605e-08,\n", - " 1.1921e-07, -7.7486e-07, -8.9407e-07, -3.5763e-07, -4.1723e-07,\n", - " -3.5763e-07, -2.1458e-06, -5.9605e-08, -5.3644e-07, 2.3842e-07,\n", - " -5.9605e-07, 0.0000e+00, -8.3447e-07, -2.3842e-07, -7.1526e-07,\n", - " -3.5763e-07, -2.9802e-07, 4.7684e-07, 0.0000e+00, -4.7684e-07,\n", - " -3.5763e-07, -9.5367e-07, 1.1921e-06, -5.9605e-07, -3.5763e-07,\n", - " 3.5763e-07, -2.3842e-07, 4.7684e-07, -4.1723e-07, 5.9605e-07,\n", - " 0.0000e+00, 4.7684e-07, 0.0000e+00, 1.1921e-07, 5.9605e-07,\n", - " 7.1526e-07, 1.1921e-07, -1.2517e-06, -4.7684e-07, 4.7684e-07,\n", - " -1.1921e-07, 2.3842e-07, 1.1921e-07, -1.0133e-06, -1.7881e-07,\n", - " -1.7881e-07, -5.9605e-08, -5.3644e-07, 4.7684e-07, -1.1921e-07,\n", - " -5.9605e-08, 4.7684e-07, 3.5763e-07, 3.5763e-07, 4.7684e-07,\n", - " 4.7684e-07, 7.1526e-07, 9.5367e-07, -4.7684e-07, 2.3842e-07,\n", - " 7.1526e-07, -3.5763e-07, -6.5565e-07, -8.9407e-07, -1.7881e-07,\n", - " 2.3842e-07, 9.5367e-07, 3.5763e-07, 1.6689e-06, 1.1921e-07,\n", - " 1.1921e-07, -9.5367e-07, -7.1526e-07, 2.3842e-07, 1.0729e-06,\n", - " -5.9605e-07, 4.7684e-07, 5.9605e-07, 1.0729e-06, -1.1921e-06,\n", - " -4.1723e-07, 5.9605e-07, 3.5763e-07, -4.7684e-07, 3.5763e-07,\n", - " 0.0000e+00, 4.7684e-07, -4.1723e-07, 2.3842e-07, 2.3842e-07,\n", - " -3.5763e-07, -2.9802e-07, 2.3842e-07, 3.5763e-07, 3.5763e-07,\n", - " 2.3842e-07, -5.3644e-07, 0.0000e+00, -4.1723e-07, -1.7881e-07,\n", - " -2.9802e-07, -1.5497e-06, -5.9605e-08, 4.7684e-07, 4.7684e-07,\n", - " 1.1921e-07, -5.9605e-07, 1.1921e-06, 2.3842e-07, 2.3842e-07,\n", - " 1.1921e-07, -5.9605e-08, 0.0000e+00, 5.9605e-07, 0.0000e+00,\n", - " -1.1921e-07, -1.1921e-07, -2.9802e-07, -4.1723e-07, 3.5763e-07,\n", - " 9.5367e-07, -5.9605e-08, -2.3842e-07, 2.3842e-07, 5.9605e-07,\n", - " -1.7881e-07, 2.3842e-07, 5.9605e-07, 3.5763e-07, 1.3113e-06,\n", - " -1.6689e-06, 1.1921e-07, 4.7684e-07, -4.7684e-07, -5.9605e-07,\n", - " -5.9605e-08, -1.1921e-07, 1.6689e-06, -1.3709e-06, 1.1921e-07,\n", - " -2.3842e-07, 7.1526e-07, -1.1325e-06, 4.7684e-07, 1.1921e-07,\n", - " -4.1723e-07, -2.9802e-07, -1.1921e-07, -4.7684e-07, -4.1723e-07,\n", - " -5.3644e-07, 1.1921e-07, -8.9407e-07, 4.7684e-07, -1.3113e-06,\n", - " 8.3447e-07, 0.0000e+00, 8.3447e-07, -9.5367e-07, 1.1921e-07,\n", - " 2.3842e-07, -7.7486e-07, 0.0000e+00, 5.9605e-07, -8.3447e-07,\n", - " -1.0133e-06, 0.0000e+00, -1.7881e-07, 2.3842e-07, -1.4305e-06,\n", - " -1.8477e-06, 5.9605e-07, -5.9605e-08, 8.3447e-07, -1.0133e-06,\n", - " 8.3447e-07, -7.1526e-07, -1.0729e-06, 0.0000e+00, -3.5763e-07,\n", - " -5.3644e-07, 0.0000e+00, 3.5763e-07, 1.1921e-07, 8.3447e-07,\n", - " -1.6689e-06, 8.3447e-07, 3.5763e-07, 1.3113e-06, 8.3447e-07,\n", - " 3.5763e-07, 3.5763e-07, 3.5763e-07, -1.1921e-07, -2.9802e-07,\n", - " 9.5367e-07, -7.1526e-07, -8.3447e-07, -6.5565e-07, 1.1921e-07,\n", - " -2.9802e-07, -1.1921e-07, 5.9605e-07, -7.7486e-07, -5.9605e-07,\n", - " 4.7684e-07, 5.9605e-07, 1.1921e-07, 2.3842e-07, -2.9802e-07,\n", - " -4.7684e-07, 1.0729e-06, -1.1921e-06, 3.5763e-07, -5.9605e-08,\n", - " 5.9605e-07, -9.5367e-07, 4.7684e-07, 7.1526e-07, -5.9605e-08,\n", - " -3.5763e-07, -6.5565e-07, -2.9802e-07, -1.7881e-07, -9.5367e-07,\n", - " -5.3644e-07, -2.3842e-07, -5.9605e-08, -2.9802e-07, 3.5763e-07,\n", - " -1.1921e-07, 1.1921e-07, -5.9605e-08, 3.5763e-07, 7.1526e-07,\n", - " -1.7881e-07, -5.9605e-08, -5.9605e-08, -2.9802e-07, 5.9605e-07,\n", - " 3.5763e-07, -1.1921e-07, 5.9605e-07, 0.0000e+00, 1.1921e-07,\n", - " -7.7486e-07, -8.3447e-07, -5.9605e-07, 1.1921e-07, 7.1526e-07,\n", - " 4.7684e-07, -1.7881e-07, -1.1325e-06, -2.9802e-07, 4.7684e-07,\n", - " -5.3644e-07, -8.3447e-07, 3.5763e-07, 4.7684e-07, 3.5763e-07,\n", - " -4.1723e-07, 2.3842e-07, -5.3644e-07, -1.7881e-07, -1.7881e-07,\n", - " 2.3842e-07, 2.3842e-07, -1.7881e-07, 0.0000e+00, 4.7684e-07,\n", - " -1.7881e-07, -5.9605e-07, -1.7881e-07, 3.5763e-07, -1.3113e-06,\n", - " -3.5763e-07, 1.1921e-07, -5.3644e-07, 4.7684e-07, -9.5367e-07,\n", - " -1.1921e-07, -5.3644e-07, -1.7881e-07, -1.1921e-07, 0.0000e+00,\n", - " -8.3447e-07, 1.1921e-07, -1.1325e-06, -3.5763e-07, 5.9605e-07,\n", - " 3.5763e-07, -5.3644e-07, 7.1526e-07, -4.1723e-07, 2.3842e-07,\n", - " 7.1526e-07, 8.3447e-07, 1.3113e-06, 1.1921e-07, -1.0133e-06,\n", - " -1.2517e-06, 1.1921e-07, 3.5763e-07, 0.0000e+00, -4.1723e-07,\n", - " -8.3447e-07, 3.5763e-07, 3.5763e-07, -5.9605e-07, 5.9605e-07,\n", - " -8.3447e-07, -7.1526e-07, -2.9802e-07, -2.3842e-07, -7.1526e-07,\n", - " -5.3644e-07, 1.1921e-07, 3.5763e-07, 3.5763e-07, -1.7881e-07,\n", - " 2.3842e-07, 2.3842e-07, -1.7881e-07, -2.3842e-07, 2.3842e-07,\n", - " -6.5565e-07, 1.3113e-06, -5.9605e-08, -2.9802e-07, 7.1526e-07,\n", - " 1.3113e-06, 1.1921e-07, 5.9605e-07, -2.3842e-07, -2.9802e-07,\n", - " 1.1921e-07, -1.7881e-07, -2.9802e-07, 0.0000e+00, -3.5763e-07,\n", - " -5.3644e-07, -7.1526e-07, 5.9605e-07, 1.1921e-07, 1.1921e-07,\n", - " -7.1526e-07, 7.1526e-07, 5.9605e-07, 7.1526e-07, 5.9605e-07,\n", - " 5.9605e-07, -7.7486e-07, -3.5763e-07, -5.9605e-08, -1.1921e-07,\n", - " 4.7684e-07, 2.3842e-07, 4.7684e-07, 1.3113e-06, 3.5763e-07,\n", - " -4.7684e-07, 2.3842e-07], device='cuda:0'),\n", - " tensor([ 1.4361e-07, 3.7875e-07, -8.3476e-07, 9.4134e-07, -6.7843e-07,\n", - " 3.1072e-07, 1.5542e-07, 1.5177e-07, 5.9782e-07, 5.7706e-07,\n", - " 3.6449e-07, -7.4512e-08, 1.9858e-07, -4.9469e-07, 7.5448e-07,\n", - " -5.9229e-07, 5.5775e-07, 7.6487e-07, -1.4062e-06, 7.2411e-07,\n", - " 4.1560e-07, -1.3103e-06, -6.2538e-07, -3.8220e-09, -1.0456e-06,\n", - " 1.7845e-07, -7.8648e-07, -4.8804e-07, -1.0593e-06, 1.1402e-06,\n", - " -2.8512e-07, 8.6257e-07, 5.7358e-07, -2.7734e-08, -1.3981e-07,\n", - " 6.7581e-07, -1.1391e-06, 3.5621e-08, -1.1710e-07, -5.2912e-07,\n", - " 1.0740e-06, 1.8467e-07, -6.4858e-08, -7.4553e-07, -4.4954e-07,\n", - " -2.0085e-08, -2.6388e-07, 1.0682e-07, -4.3396e-07, 5.0373e-07,\n", - " -3.1702e-09, 2.1772e-07, -3.6832e-07, -7.9641e-07, -5.5592e-07,\n", - " -8.9712e-08, -2.3330e-07, -5.7435e-07, 2.1003e-07, -3.4901e-07,\n", - " -1.2920e-07, -7.1090e-08, -2.8965e-07, -6.8484e-08, 1.1578e-07,\n", - " 1.8122e-07, -8.2851e-07, -5.4816e-07, -5.4788e-07, 6.4785e-07,\n", - " -6.7243e-07, -2.9483e-07, 3.0059e-07, -6.2761e-07, -1.3943e-07,\n", - " -4.6666e-08, 8.2269e-07, 5.2700e-07, -2.0402e-07, -1.8630e-07,\n", - " -1.4299e-07, -5.8553e-07, -4.3453e-08, 1.8669e-07, 4.3835e-08,\n", - " -1.5051e-07, 1.0361e-08, -8.9533e-07, -7.6743e-08, -2.0186e-07,\n", - " 5.8199e-07, 4.8817e-07, 5.3219e-07, 1.5891e-07, -5.4972e-07,\n", - " -4.3597e-08, 6.6097e-07, -4.5730e-10, -5.9889e-08, -5.4971e-07,\n", - " 1.5863e-08, 4.2603e-07, 4.4973e-07, -3.9109e-07, 3.3712e-07,\n", - " 4.1975e-07, 7.8129e-07, -4.5257e-07, -2.4825e-07, -6.3345e-07,\n", - " -7.7735e-07, 9.1151e-07, 5.8416e-08, -2.9035e-07, -1.3461e-06,\n", - " -2.6095e-07, -3.0684e-07, -1.6109e-06, -1.3722e-06, 2.2558e-08,\n", - " -4.3157e-07, -1.2280e-07, -1.1057e-07, -5.6610e-07, -2.7475e-07,\n", - " 3.3783e-07, -4.9134e-07, 5.5273e-07, 3.0693e-09, -2.3143e-07,\n", - " -8.8538e-07, -1.6646e-07, -1.4726e-07, -3.3940e-07, 1.1079e-06,\n", - " -3.8600e-07, 1.9262e-07, 8.6771e-09, -6.1051e-07, 2.0421e-07,\n", - " 6.8559e-07, 1.6167e-07, 6.7790e-07, -6.8239e-07, 1.2664e-07,\n", - " -9.9949e-09, 3.2615e-07, -6.1828e-07, -3.7162e-07, -3.7594e-07,\n", - " -1.4025e-09, -6.1806e-07, -3.8709e-07, -1.8537e-08, 5.6683e-07,\n", - " -1.8389e-07, -2.1760e-06, -2.1463e-07, -3.7027e-07, -1.2480e-07,\n", - " -5.7935e-07, 3.2740e-07, -3.9764e-07, -8.7763e-07, -6.9524e-07,\n", - " 1.8890e-07, 6.0723e-07, 2.8119e-07, 1.9862e-07, -4.5653e-07,\n", - " -3.2686e-07, -6.4305e-07, 3.0504e-07, -7.7494e-07, -1.0217e-06,\n", - " 1.7248e-07, 3.5446e-07, 3.2754e-07, -8.2340e-07, -2.8620e-07,\n", - " -1.2324e-07, 1.4035e-08, -4.2806e-07, 2.2246e-07, 5.9490e-07,\n", - " 9.5043e-07, 1.6535e-07, -8.7346e-07, -7.8100e-07, -5.2794e-09,\n", - " -7.6545e-07, -5.1436e-07, -4.0977e-07, -1.1618e-06, 6.8537e-08,\n", - " -1.4605e-07, 5.8257e-08, -8.0188e-07, -2.0639e-07, 2.1339e-07,\n", - " 1.5732e-07, 9.9957e-07, 4.9952e-07, 1.3014e-06, 2.9355e-07,\n", - " -3.4369e-07, 5.1924e-07, 5.3847e-07, -9.4620e-07, 5.6011e-07,\n", - " 7.9131e-07, -5.2138e-09, -1.4767e-06, -6.3886e-07, 1.4941e-07,\n", - " 9.8527e-07, -3.7774e-07, 3.2528e-07, 9.4757e-07, 2.1744e-07,\n", - " 5.9661e-07, -1.1091e-07, -6.2895e-07, 4.1626e-07, 9.3376e-07,\n", - " -2.4913e-07, 2.1312e-08, 7.0954e-07, 1.7708e-07, -1.0764e-06,\n", - " -1.6039e-07, -2.0194e-07, 3.0868e-07, -2.1210e-07, 2.8911e-07,\n", - " -3.3863e-07, 3.0655e-07, -8.2397e-07, 6.2707e-07, 4.0511e-07,\n", - " -6.2630e-07, -5.2816e-07, 6.9402e-07, 3.1391e-07, 6.8661e-07,\n", - " 1.2054e-07, -2.9753e-07, 2.2532e-09, -5.5803e-07, -2.0193e-07,\n", - " -5.2379e-07, -4.5617e-07, 1.9949e-08, 9.5923e-07, 6.3150e-07,\n", - " -3.8950e-07, -8.7060e-07, 3.2801e-07, -1.5988e-07, -2.0283e-07,\n", - " 2.0498e-08, -6.2763e-08, 1.2942e-07, 4.8690e-07, 2.9747e-08,\n", - " -5.9033e-07, -2.3964e-07, -7.6579e-07, -5.2635e-07, -1.1527e-06,\n", - " 6.4606e-07, -8.1474e-07, -4.2017e-07, 6.5501e-07, 3.6491e-07,\n", - " 5.6719e-07, -6.4178e-08, 8.4156e-07, 1.8924e-07, 5.0438e-07,\n", - " -8.7186e-07, 5.7862e-07, 7.9785e-07, -1.1940e-07, -2.8611e-07,\n", - " -3.7413e-07, 6.9951e-07, 1.1857e-06, -7.0392e-07, 2.6504e-07,\n", - " 6.2547e-07, 4.0341e-07, -6.8973e-07, 1.0543e-06, -3.9495e-07,\n", - " 6.2045e-07, -5.2668e-07, 3.1209e-07, 2.9866e-08, -1.3103e-06,\n", - " -7.4060e-07, -2.1091e-07, -1.1769e-06, 4.9558e-07, -1.5905e-06,\n", - " 2.1223e-07, 3.8217e-07, 2.4839e-07, -9.2843e-07, -3.9248e-07,\n", - " 5.8080e-08, -9.3520e-07, 2.0115e-08, 8.0949e-07, -6.3440e-07,\n", - " -6.9958e-07, 3.6014e-08, 1.2903e-08, -7.1608e-08, -1.1944e-06,\n", - " -9.8625e-07, 4.3675e-07, -5.9806e-07, 7.8947e-07, -8.5338e-07,\n", - " -3.3651e-07, -5.3144e-07, -6.9125e-07, 1.4479e-07, -8.7782e-07,\n", - " 2.1694e-07, 6.6267e-07, -1.7303e-07, 5.9009e-07, 1.9341e-07,\n", - " -1.2323e-06, -3.1903e-07, 1.7893e-07, 8.5715e-07, 1.6794e-06,\n", - " 4.4822e-07, 8.3358e-07, 7.1812e-08, 1.7047e-07, -1.3995e-07,\n", - " 1.5727e-06, -7.6507e-07, -1.1366e-06, -6.4408e-07, -3.2520e-07,\n", - " 3.6125e-08, -5.6142e-07, 1.0516e-06, -1.0970e-06, -4.1031e-07,\n", - " 1.9087e-07, -9.4430e-08, -4.1627e-08, 7.3488e-07, 2.9984e-07,\n", - " -4.2442e-07, 1.1619e-06, -9.3843e-07, -3.9253e-07, 7.6044e-07,\n", - " 8.9230e-07, -6.1505e-07, 2.8537e-07, 1.9979e-07, 1.1293e-07,\n", - " 9.3129e-09, -4.7411e-07, -3.3502e-07, 6.4352e-07, -8.2211e-07,\n", - " 2.8376e-07, -2.1078e-07, -4.7676e-07, -3.2840e-07, -1.7811e-07,\n", - " -1.0372e-07, -1.5150e-07, -2.9330e-07, -5.0891e-08, 4.6162e-07,\n", - " -4.4931e-07, -1.1251e-07, -2.1810e-08, 3.6239e-07, 4.1384e-07,\n", - " -1.9962e-07, -2.2854e-07, 4.1248e-07, 1.3120e-07, 2.2709e-07,\n", - " -6.5830e-07, 2.7621e-07, -2.8515e-07, -7.1727e-07, -2.2612e-07,\n", - " 2.9839e-07, -1.8185e-07, -7.0273e-07, -2.9024e-07, 5.5723e-07,\n", - " -4.9760e-07, -7.6321e-07, 2.0529e-07, 8.0877e-07, -8.9062e-08,\n", - " -6.0039e-07, 2.3167e-07, -1.9675e-07, 6.2634e-08, -5.7431e-07,\n", - " 7.7656e-07, 4.2352e-07, -5.7160e-07, -1.6343e-08, 6.0179e-07,\n", - " -6.7309e-07, 3.5918e-07, 4.8857e-08, 1.7525e-07, -5.2018e-07,\n", - " -2.2266e-07, -1.4090e-07, 1.4820e-07, 5.7232e-07, -6.3711e-07,\n", - " 3.2627e-07, -1.3462e-06, -6.5572e-08, -6.0948e-07, -5.2886e-07,\n", - " -4.4639e-07, -5.9024e-07, -7.8491e-07, -1.6743e-07, 2.8212e-09,\n", - " -1.1809e-07, -6.3132e-07, 4.9501e-07, -5.7799e-07, 6.2618e-07,\n", - " 4.2153e-07, 3.3870e-07, 2.6234e-07, 1.1589e-06, -6.3834e-07,\n", - " -1.7349e-07, -2.8091e-07, 1.3085e-07, 2.7772e-07, 8.0872e-08,\n", - " 2.6623e-07, 9.5979e-07, 2.8653e-07, -1.1525e-06, 4.1606e-07,\n", - " -8.0760e-07, -9.3070e-07, -4.5792e-07, 3.1079e-07, 3.9923e-07,\n", - " -6.9447e-07, 4.0161e-08, 1.7961e-07, -2.0261e-07, 5.0900e-08,\n", - " -5.9011e-07, -8.9109e-07, -5.2656e-07, -1.3622e-07, 5.2093e-07,\n", - " -1.5905e-06, 1.5041e-06, 2.2004e-07, -3.0572e-07, 5.7887e-07,\n", - " 1.7697e-07, -2.7734e-08, 5.3296e-07, 9.1149e-08, -3.8058e-07,\n", - " 3.5024e-07, 3.7948e-07, -1.5867e-08, -8.8103e-07, 7.0423e-08,\n", - " -8.5883e-07, -3.3453e-07, 6.0323e-07, -3.6401e-07, 3.0336e-07,\n", - " 4.4667e-07, 8.7888e-07, -1.7145e-07, 6.6095e-07, -7.5378e-07,\n", - " 3.2432e-07, -1.0764e-06, -9.0861e-08, 7.4299e-07, 2.8685e-07,\n", - " -8.3725e-07, 3.7188e-07, 1.8325e-07, 6.5297e-07, 1.3390e-07,\n", - " 6.0233e-07, 3.0721e-07], device='cuda:0'),\n", - " tensor([[ 5.8622e-06, 6.0122e-06, 5.8827e-06, ..., 5.6075e-06,\n", - " 1.0453e-05, -2.4978e-06],\n", - " [-8.0243e-06, -3.6648e-07, 7.9758e-06, ..., 9.5163e-06,\n", - " 9.7416e-06, 8.0727e-06],\n", - " [ 3.6899e-06, -4.5393e-06, -4.8149e-06, ..., -4.9956e-06,\n", - " -8.8997e-06, -4.2990e-06],\n", - " ...,\n", - " [ 1.6269e-05, 3.3859e-05, 1.9226e-05, ..., 1.8030e-05,\n", - " 3.7677e-06, 6.0163e-06],\n", - " [ 4.7535e-06, 8.4136e-06, 2.0556e-05, ..., 7.8417e-07,\n", - " 2.7270e-05, 2.2111e-05],\n", - " [-1.1865e-06, -4.3474e-06, -1.7798e-06, ..., 2.7046e-06,\n", - " 8.9593e-06, 1.0707e-05]], device='cuda:0'),\n", - " tensor([ 1.4883e-05, -1.2254e-05, 1.7649e-06, 5.1819e-06, -3.3751e-06,\n", - " -2.6276e-06, -1.3839e-05, 1.2923e-05, 8.4192e-06, -1.1077e-05],\n", - " device='cuda:0')]" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "input_parameters" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [], - "source": [ - "config = dict(signed=True,\n", - " boxed=True,\n", - " cost_fn='sim',\n", - " indices='def',\n", - " weights='equal',\n", - " lr=0.1,\n", - " optim='adam',\n", - " restarts=1,\n", - " max_iterations=8_000,\n", - " total_variation=1e-6,\n", - " init='randn',\n", - " filter='none',\n", - " lr_decay=True,\n", - " scoring_choice='loss')\n", - "\n", - "rec_machine = inversefed.FedAvgReconstructor(model_bespoke, (dm, ds), 1, 3e-4, config,\n", - " use_updates=True, num_images=num_images)\n", - "# output, stats = rec_machine.reconstruct(input_parameters, labels_target, img_shape=(3, 32, 32))\n" - ] - }, - { - "cell_type": "code", - "execution_count": 23, - "metadata": {}, - "outputs": [], - "source": [ - "param_diff = [p.detach() for p in param_diff.values()]" - ] - }, - { - "cell_type": "code", - "execution_count": 32, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "tensor([3, 8, 0, 6, 1, 9, 5, 7, 4, 2], device='cuda:0')" - ] - }, - "execution_count": 32, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "labels_target" - ] - }, - { - "cell_type": "code", - "execution_count": 33, - "metadata": {}, - "outputs": [], - "source": [ - "labels = torch.cat([torch.as_tensor((i,), device=setup['device']) for i in range(10)])\n", - "labels = labels.to(dtype=torch.float32)" - ] - }, - { - "cell_type": "code", - "execution_count": 34, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "It: 0. Rec. loss: 1.0003.\n", - "It: 500. Rec. loss: 0.9900.\n", - "It: 1000. Rec. loss: 0.9899.\n", - "It: 1500. Rec. loss: 0.9898.\n", - "It: 2000. Rec. loss: 0.9898.\n", - "It: 2500. Rec. loss: 0.9897.\n", - "It: 3000. Rec. loss: 0.9900.\n", - "It: 3500. Rec. loss: 0.9892.\n", - "It: 4000. Rec. loss: 0.9890.\n", - "It: 4500. Rec. loss: 0.9890.\n", - "It: 5000. Rec. loss: 0.9891.\n", - "It: 5500. Rec. loss: 0.9889.\n", - "It: 6000. Rec. loss: 0.9889.\n", - "It: 6500. Rec. loss: 0.9890.\n", - "It: 7000. Rec. loss: 0.9890.\n", - "It: 7500. Rec. loss: 0.9890.\n", - "It: 7999. Rec. loss: 0.9890.\n", - "Choosing optimal result ...\n", - "Optimal result score: 0.9889\n", - "Total time: 616.5541167259216.\n" - ] - } - ], - "source": [ - "\n", - "output, stats = rec_machine.reconstruct(param_diff, labels=labels, img_shape=(3, 32, 32))" - ] - }, - { - "cell_type": "code", - "execution_count": 35, - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAACBsAAADHCAYAAABS68lEAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAD8WklEQVR4nOyddVgWW9u3f0OnYoCKAYgodndgu21U7ACxsHXbuu3u7u7uDuzu7m5FFEREcn1/7I/7dfzdbsFtPM/7XudxeOw9J2tm1sysuNaauWc0pZSCIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCAjH53RkQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBOG/C3nYQBAEQRAEQRAEQRAEQRAEQRAEQRAEQRCERCEPGwiCIAiCIAiCIAiCIAiCIAiCIAiCIAiCkCjkYQNBEARBEARBEARBEARBEARBEARBEARBEBKFPGwgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCEKikIcNBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEFIFPKwgSAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIiUIeNhAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEIVHIwwaCIAiCIAiCIAiCIAiCIAiCIAiCIAiCICQKedhAEITvonTp0siRI8c30z18+BCapmHRokU/P1OC8A8cPHgQmqbh4MGD/xXbFYRfwZkzZ1CsWDHY2tpC0zRcvHjxd2dJEP7X4+rqimrVqn0znbH+xc/PD66urj8vc4LwLxg0aBA0TcObN29+d1YEwSjxZVQQ/tuQ9lX4T+Z3x6fSticcV1dXDBo06HdnQ/hJLFq0SOrCfxHx4/2HDx/+7qwIgo74+3mJvdchDxv8Zp4/f45BgwbJ5L4gCIIgCL+U6Oho1K1bF2/fvsXEiROxdOlSuLi4/O5sCcJ3MWPGDHmwUfg/z/HjxzFo0CCEhIT87qwIgiAIgvADiL95GP/PzMwMadOmhZ+fH549e/a7s/dfSWRkJHr16gVnZ2dYW1ujcOHC2Lt3b4LXX7VqFfLlywcrKys4OjqiRYsWRh8ECg0NRc+ePeHh4QFra2u4uLigRYsWePz4MaXdt28fypQpg5QpU8LBwQGFChXC0qVLKd2rV6/QvHlzODk5wdraGvny5cPatWsTdwISQPwDHCYmJnjy5An9/f3797C2toamaejQoYPub0FBQejcuTM8PT1hbW0NJycnFCpUCL169cKHDx8M6fz8/HRl+/N/VlZW35Xv58+fo0mTJsiSJQvs7e0N53Lx4sVQSn1z/fibv8b+nTx5ktJHRUVhxIgR8PT0hJWVFVKlSoWqVavi6dOnhjTXrl1D3bp1kTFjRtjY2CBlypQoVaoUtm7dStubO3cuvLy8kCpVKlhaWsLNzQ3Nmzf/KTejPz82ExMTODs7o2LFinRzMSoqCpMnT0bevHmRJEkSODg4IHv27GjdujVu3rxpSBffVllZWRltm4z9aNHV1VWXD1tbWxQqVAhLliz518cXFxeHmTNnIk+ePLC2tkaKFClQtmxZXLp06Zvrfpmv+H8BAQH/uF6rVq2gaVqCfliQWL6WJ03T8OnTJwD6/uLo0aO0DaUU0qdPbzSPHz58wMCBA5EjRw7Y2toiRYoUyJMnDzp37oznz58b0sW3DV/79/Lly0Qf25MnTzB48GAUKlQIyZIlQ8qUKVG6dGns27cvUdu5d+8eGjVqZGgfPTw80K9fP8Pf4+LisGjRItSoUQPp06eHra0tcuTIgWHDhhnO4ecktr1dvXo1ihYtCltbWzg4OKBYsWLYv39/oo4hIWzZssXQB2XIkAEDBw5ETEyMLk1gYCD8/f2ROXNm2NjYIGPGjGjZsiVevHjxw/PzOWY/devCN3n+/DkGDx4MV1dX5MmT53dnRxB+OC4uLoiIiIC5ufnvzorwf5xSpUohIiICFhYWvzsrgvAfwb179/Do0SPMnTsXLVu2/N3ZEYR/xYwZM5AyZUr4+fn97qz8MKTfEhLL8ePHMXjwYPj5+cHBweF3Z0cQBEEQhB/EkCFD4Obmhk+fPuHkyZNYtGgRjh49iqtXr373jdn/q/j5+WHdunXo0qULPDw8sGjRIlSpUgUHDhxAiRIl/nHdmTNnol27dihXrhwmTJiAp0+fYvLkyTh79ixOnTpluBZxcXGoUKECrl+/jnbt2iFz5sy4e/cuZsyYgd27d+PGjRuwt7cH8PeNI29vbxQtWtRwI2/NmjVo1qwZ3rx5g65duwL4+wZ/iRIl8OrVK3Tu3BmpU6fGmjVrUK9ePSxfvhyNGjX64efK0tISK1euRM+ePXV+w4YNRtO/ffsWBQoUwPv37+Hv7w9PT08EBwfj8uXLmDlzJtq2bQs7Ozvd9ufNm0fbMTU1/a78vnnzBk+fPoWPjw8yZMiA6Oho7N27F35+frh16xZGjBiRoO106tQJBQsW1LlMmTLplqOjo1G1alUcP34crVq1Qq5cufDu3TucOnUKoaGhSJcuHQDg0aNHCAsLg6+vL5ydnfHx40esX78eNWrUwOzZs9G6dWvDNi9cuAA3NzfUqFEDyZIlw4MHDzB37lxs27YNly5dgrOz83edl69RoUIFNGvWDEopPHjwADNmzEDZsmWxfft2VK5cGQBQp04d7Ny5Ew0bNkSrVq0QHR2NmzdvYtu2bShWrBg8PT1124yMjMSoUaMwderUBOUhT5486NatGwDgxYsXmDdvHnx9fREZGYlWrVp997H5+/tj+fLlaNasGTp06IDw8HBcuHABr1+/TnS+4smcOfNX0589exaLFi36qe2xsTwBoPkCKysrrFixgtqzQ4cO4enTp7C0tNT56OholCpVCjdv3oSvry86duyIDx8+4Nq1a1ixYgVq1apFZW/mzJm6uhzP94xBN2/ejNGjR8Pb2xu+vr6IiYnBkiVLUKFCBSxYsADNmzf/5jYuXryI0qVLI23atOjWrRtSpEiBx48f6x6W+vjxI5o3b44iRYogICAATk5OOHHiBAYOHIjAwEDs37/f8GaQxLa3gwYNwpAhQ+Dj4wM/Pz9ER0fj6tWrP/yhwJ07d8Lb2xulS5fG1KlTceXKFQwbNgyvX7/GzJkzDel69eqFt2/fom7duvDw8MD9+/cxbdo0bNu2DRcvXkTq1Kl/aL4MKOG3cubMGQVALVy48HdnRRAShZeXl8qePfvvzoYg/HDCw8MTlf7AgQMKgDpw4MDPyZAg/CQOHTqkAKi1a9f+Y7oPHz78ohwJwveTPXt25eXl9buzkSBcXFxU1apVv2tdX19f5eLi8mMzJPyvYezYsQqAevDgwT+mi42NVRERET98/wMHDlQAVFBQ0A/ftiD8COLLqCD8t/Gz29e4uDj18ePHn7Jt4d+xcOFCBUCdOXNG53v16qUAqNWrV/+mnP0Pvzs+TUzbfurUKQVAjR071uAiIiKUu7u7Klq06D+uGxkZqRwcHFSpUqVUWFiYwW/dulUBUFOmTDG4Y8eOKQBq2rRpum0sWLBAAVAbNmwwuAoVKihnZ2f16dMng4uOjlbu7u4qV65cBjdmzBgFQAUGBhpcbGysKliwoEqdOrWKjIz85vG7uLiogQMHfjNd/DmtXbu2ypMnD/29QoUKqk6dOgqAat++PeXx2LFjtE5oaKgu/vT19VW2trbfzMuPoFq1asrW1lbFxMT8Y7r4+b1vzZEopdTo0aOVubm5OnXqVKLzExMTo3Lnzq2yZMnyzbRnz55VANTIkSO/mTa+vUgIX147pZS6fPmyAqAqVqyolFLq9OnTCoAaPny40WN48+YN7TtPnjzK0tJSPXv2TJfe2H0EY+Pi169fKzs7O5U1a9YEHYcxVq9eTfUsMSR2vB4XF6eKFi2q/P39E7VufHn71tgtoXmKvwa1a9dWKVOmVNHR0bq/t2rVSuXPn5+2tWbNGgVALV++nLYZERGhQkNDDcs/Ix65evUqbe/Tp0/K09NTpUuX7pvrx8bGqhw5cqjChQv/YywTGRlptG0aPHiwAqD27t1rcIlpb0+cOKE0TVMTJkz4Zl7/LdmyZVO5c+fWXdt+/fopTdPUjRs3DO7QoUMqNjZWt278HHC/fv2+uZ8HDx58170O+YzCv+DZs2do0aIFnJ2dDa+2adu2LaKiovD27Vt0794dOXPmhJ2dHZIkSYLKlSvrXtVy8OBBw1NyzZs3N7xuRF4BK/wnEBYWhi5dusDV1RWWlpZwcnJChQoVcP78eV2669evo0yZMrCxsUHatGkxZswY3d/jv/Hyebn28/ODnZ0d7t+/j0qVKsHW1hbOzs4YMmRIgl5rJQif8+jRI7Rr1w5ZsmQxvBqrbt269JoxY9++jn+N17lz51CqVCnY2Nigb9++AP7nm9p79uxBnjx5YGVlhWzZsn31Ce7POXLkCOrWrYsMGTLA0tIS6dOnR9euXREREaFLF18Xnj17Bm9vb9jZ2cHR0RHdu3dHbGysLm1cXBwmTZqE7NmzG17P1qZNG7x79+77Tpzwfxo/Pz94eXkBAOrWrQtN01C6dGlDmbx37x6qVKkCe3t7NG7cGAAQHh6Obt26IX369LC0tESWLFkwbtw4arcjIiLQqVMnpEyZEvb29qhRowaePXsGTdPk+5ACgIS321/7/mr86wHj07u6uuLatWs4dOiQIZ4uXbq0If39+/dRt25dJE+eHDY2NihSpAi2b9+u22Z8H7FmzRoMHjwYadOmhb29PXx8fBAaGorIyEh06dIFTk5OsLOzQ/PmzREZGanbRkxMDIYOHQp3d3dYWlrC1dUVffv2pXTxfKt/MdZvGUP6BwH4u7706NEDAODm5maoC/GxeIcOHbB8+XJkz54dlpaW2LVr11fLmLH4HQBu3ryJevXqwdHREdbW1siSJYvu1ZTGePToETJlyoQcOXLg1atXP/KQBeEfOXr0KAoWLAgrKyu4u7tj9uzZlCah7XZcXBwGDRoEZ2dn2NjYoEyZMrh+/TpcXV3/V71RR/jPJyQkxPD2mqRJk6J58+b4+PGj4e8JLdPxY93du3ejQIECsLa2NtSRvXv3okSJEnBwcICdnR2yZMliGCPHExkZiYEDByJTpkyG8W7Pnj2/GvMIP56SJUsC+PttdZ9z8+ZN+Pj4IHny5LCyskKBAgWwZcsWWj8kJARdu3Y1zPmlS5fO8Ev6H0FCx44JKW9Tp05F9uzZYWNjA3t7e9ja2sLc3Jza9s8/T7Bs2TLkz58f1tbWSJ48ORo0aIAnT55g3bp1MDU1Nfya/NSpU/D29sbTp09x4sQJFC5cGMeOHdPtP35MsmXLFoSEhODMmTOG8w8A1apVg52dHVatWmVw79+/BwCkSpVKt600adIAAKytrXVpkyVLpvu1sZmZGVKmTKlLd+TIETg6OqJs2bIGZ2Jignr16uHly5c4dOiQ0Wvxb2jUqBEuXryoe13+y5cvsX//fqNvUrh37x5MTU1RpEgR+luSJEm++1ff9+7do7KeGFxdXfHx40dERUUleJ2wsDB6NXk8cXFxmDx5MmrVqoVChQohJiZG1xZ/C1NTU6RPnz5Bnz5zdXUFgF/ymbScOXMiZcqUePDgAYD/aV+KFy9OaU1NTZEiRQryffv2RWxsLEaNGvVdeXB0dISnpydd79DQUNy8eROhoaHf3MaECRNQqFAh1KpVC3FxcQgPD/+uvERFRSVo3aVLl+Lq1asYPnz4d+3nR9OwYUMEBwfrPg8TFRWFdevWfbXeAsavs5WVFZIkSfJd+Xj8+LGu7fga2bNnR8qUKXXO0tISVapUwdOnTxEWFvaP6+/ZswdXr17FwIEDYW1tjY8fP9KcOvD3GyCKFStGvlatWgCAGzduGFxi2ttJkyYhderU6Ny5M5RSus/FfMm/iZ+uX7+O69evo3Xr1jAz+58PFrRr1w5KKaxbt87gSpUqBRMT/a3/UqVKIXny5Lrj/NHIwwbfyfPnz1GoUCGsWrUK9evXx5QpU9C0aVMcOnQIHz9+xP3797Fp0yZUq1YNEyZMQI8ePXDlyhV4eXkZvnOSNWtWDBkyBADQunVrLF26FEuXLkWpUqV+56EJAgAgICAAM2fORJ06dTBjxgx0794d1tbWugbp3bt3+OOPP5A7d26MHz8enp6e6NWrF3bu3PnN7cfGxuKPP/5AqlSpMGbMGOTPnx8DBw7EwIEDf+ZhCf8LOXPmDI4fP44GDRpgypQpCAgIQGBgIEqXLp2gYD84OBiVK1dGnjx5MGnSJJQpU8bwtzt37qB+/fqoXLkyRo4cCTMzM9StW/eb3/Nbu3YtPn78iLZt22Lq1KmoVKkSpk6dimbNmlHa2NhYVKpUCSlSpMC4cePg5eWF8ePHY86cObp0bdq0QY8ePVC8eHFMnjwZzZs3x/Lly1GpUiVER0cn8GwJwt+0adPGMInTqVMnLF261HDDKCYmBpUqVYKTkxPGjRuHOnXqQCmFGjVqYOLEifjjjz8wYcIEZMmSBT169MCff/6p27afnx+mTp2KKlWqYPTo0bC2tkbVqlV/+TEK/7n823b7SyZNmoR06dLB09PTEE/Hl+dXr16hWLFi2L17N9q1a4fhw4fj06dPqFGjBjZu3EjbGjlyJHbv3o3evXvD398fGzZsQEBAAPz9/XH79m0MGjQItWvXxqJFizB69Gjdui1btsSAAQOQL18+TJw4EV5eXhg5ciQaNGhA+/ne/sUY0j8IAFC7dm00bNgQADBx4kRDXXB0dAQA7N+/H127dkX9+vUxefJkw8RlQrl8+TIKFy6M/fv3o1WrVpg8eTK8vb2Nfm82nnv37qFUqVKwt7fHwYMHacJdEH4WV65cQcWKFfH69WsMGjQIzZs3x8CBA6ndT2i73adPHwwePBgFChTA2LFj4eHhgUqVKn335LUgfC/16tVDWFgYRo4ciXr16mHRokUYPHiw4e+JiUVu3bqFhg0bokKFCpg8eTLy5MmDa9euoVq1aoiMjMSQIUMwfvx41KhRQ3fzNS4uDjVq1MC4ceNQvXp1TJ06Fd7e3pg4cSLq16//S86DAMNDt8mSJTO4a9euoUiRIrhx4wZ69+6N8ePHw9bWFt7e3rr278OHDyhZsiSmTp2KihUrYvLkyQgICMDNmzd135r/XhI6dkxIeZs7dy46deqEbNmyoUePHoiMjISmaShUqBC17fHzLcOHD0ezZs3g4eGBCRMmoEuXLggMDESpUqVw5swZZM6cGUmSJMH+/ftRqlQpvH//3vDg2OvXr1G2bFmcPn2ajiv+oU4fHx96vbu1tTUuXLiAuLg4AECBAgVga2uL/v37Y//+/Xj27BkOHTqEnj17omDBgihfvrxh3dKlS+PatWvo378/7t69i3v37mHo0KE4e/as7vMFkZGRuocP4rGxsQEAnDt3LuEXKYGUKlUK6dKlw4oVKwxu9erVsLOzMzrGd3FxQWxsLJYuXZrgfbx584b+xT+sEU+5cuVQrly5BG8zIiICb968wcOHD7F48WIsXLgQRYsWNXr+jNG8eXPDwxFlypTB2bNndX+/fv06nj9/jly5cqF169awtbWFra0tcuXKhQMHDhjdZnh4ON68eYN79+5h4sSJ2Llz51ePKTg4GK9fv8bZs2cNr5BPzPF/L+/evcO7d+8MDxG4uLgAAJYvX/7VBy++xM3NDc2aNcPcuXMN98ASQ0xMDJ4+fapr2wBg48aNyJo1q9Ex/Oe8f/8ep0+fRsGCBdG3b18kTZoUdnZ2yJgxI9asWZPgfOzfvx82Njaws7ODq6srJk+ebDRdWFgYevXqhb59+/68V9P/f6Kjo6muGJs/cXV1RdGiRbFy5UqD27lzJ0JDQ43GA/HXecmSJQn+Eejbt28pL18+ENOsWTNkzZo1EUeo5+XLl7CxsTG0cV9j3759AP5+QCG+7bWxsUGDBg3w9u3bBO0HgO6Bh8S0t4GBgShYsCCmTJkCR0dH2NvbI02aNJg2bZpu3X8bP124cAHA3/3L5zg7OyNdunSGv3+NDx8+4MOHD/Rgxw8lUe9BEAw0a9ZMmZiY0GuslPr71SmfPn2iV1U8ePBAWVpaqiFDhhicfEZB+E8ladKk9Dqlz/Hy8lIA1JIlSwwuMjJSpU6dWtWpU8fg4l+78nkZ9/X1VQBUx44dDS4uLk5VrVpVWVhYyKtXhURh7BVJJ06coPJp7HMH8eV41qxZtA0XFxcFQK1fv97gQkNDVZo0aVTevHn/cbvG8jRy5EilaZp69OiRwcXXhc/7BaWUyps3r8qfP79h+ciRI0ZfabVr166vvupKEL6FsVcExpfJ3r1769Ju2rRJAVDDhg3TeR8fH6Vpmrp7965SSqlz584pAKpLly66dH5+fgpAgl7ZKPzvJ6Ht9tdeiRr/esDPXzf4tc8odOnSRQFQR44cMbiwsDDl5uamXF1dDfF6fH3IkSOHioqKMqRt2LCh0jRNVa5cWbfdokWL6l4Xe/HiRQVAtWzZUpeue/fuCoDav3+/wf2b/uXL19RK/yB8ztc+owBAmZiYqGvXrun81z4FZSx+L1WqlLK3t9fFMUr9HcPH8/lrNW/cuKGcnZ1VwYIF1du3b3/I8QlCQvH29lZWVla68nr9+nVlampq6FcS2m6/fPlSmZmZKW9vb126QYMGKQDK19f35x6MIKj/aV/9/f11vlatWipFihRKqe+LRXbt2qVLO3HixG++Hnnp0qXKxMREF1sppdSsWbO++vp04fuJj3v37dungoKC1JMnT9S6deuUo6OjsrS0VE+ePDGkLVeunMqZM6fudfxxcXGqWLFiysPDw+AGDBjw1VeMf96vJ5Qv49OEjh0TUt5q1qxpePX6t9p2Ly8v9fDhQ2Vqakqvfb9y5YoyMzNTTk5OqmzZsiouLk55eHioSpUqqbi4OHXt2jXDpxDc3NxUhQoVDOt+/kkBTdNUixYtdNu+efOmAqAA6F4rv23bNpUmTRrD3wCoSpUq6T7BoNTfny2sV6+e0jTNkM7GxkZt2rRJl65jx47KxMREPXz4UOcbNGigAKgOHTp89TzGk9jPKAQFBanu3burTJkyGf5WsGBB1bx5c6UUv4r/5cuXytHRUQFQnp6eKiAgQK1YsUKFhITQPuLnHoz9q1SpEuU7MZ/qGDlypG575cqVU48fP/7meseOHVN16tRR8+fPV5s3b1YjR45UKVKkUFZWVur8+fOGdBs2bFAAVIoUKZSHh4dauHChWrhwofLw8FAWFhbq0qVLtO02bdoY8mNiYqJ8fHy+GiNbWloa0qZIkUL3iY5/IrGfUWjRooUKCgpSr1+/VqdOnVLlypVTANT48eOVUn+3B/HzpqlSpVINGzZU06dPp/HA5/s+c+aMunfvnjIzM1OdOnUy/P1rn1GoWLGiCgoKUkFBQerKlSuqadOmRj/xEL/9b91DO3/+vOG8pUqVSs2YMUMtX75cFSpUSGmapnbu3PnNc1O9enU1evRotWnTJjV//nxVsmRJBUD17NmT0nbv3l25ubkZ2t2f+RkFY3Xl8/r8+TWYNm2asre3N8y91K1bV5UpU8ZoHj9+/KiyZMmiACgXFxfl5+en5s+fr169ekX5iG8bjP378rMg8WXne7hz546ysrJSTZs2/WbaGjVqGK5548aN1bp161T//v2VmZmZKlas2Df7tfLly6skSZKod+/eGVxC29u3b98a9m1nZ6fGjh2rVq9erf744w+63/Bv46f4sb6xtqxgwYKqSJEi/7j+0KFD6dMQX0M+o/ALiYuLw6ZNm1C9enV6kgQANE2DpaWl4VUVsbGxCA4ONrwO6svX0AvCfyIODg44derUPz6FaGdnhyZNmhiWLSwsUKhQIdy/fz9B++jQoYPh/+Nf7xoVFWV4Ik0QEsLnTxpGR0cjODgYmTJlgoODQ4LaW0tLS8OTwl/i7OxseJ0S8Pdr35o1a4YLFy4Ynnz8Vp7in14uVqwYlFJGnzQMCAjQLZcsWVJXj9auXYukSZOiQoUKuqdG8+fPDzs7u68+OS0I30vbtm11yzt27ICpqSk6deqk8926dYNSyvBGm127dgH4+zVen9OxY8efmFvhv41/224nhh07dqBQoUIoUaKEwdnZ2aF169Z4+PAhrl+/rkvfrFkzmJubG5YLFy4MpRT8/f116QoXLownT54YfuGxY8cOAKA3fXTr1g0A6LMN39u/fIn0D0JC8fLyQrZs2b5r3aCgIBw+fBj+/v7IkCGD7m/GPnVy9epVeHl5wdXVFfv27aNfJgnCzyQ2Nha7d++Gt7e3rrxmzZoVlSpVMiwntN0ODAxETEyMxDbCfwTGxo3BwcF4//59omMRNzc3XZ0A/p4HAoDNmzcbfqH9JWvXrkXWrFnh6empiz3iXzUsscfPoXz58nB0dET69Onh4+MDW1tbbNmyBenSpQPw969M9+/fb3j7Rfx1CQ4ORqVKlXDnzh08e/YMALB+/Xrkzp1bF4vGY6xfTywJHTsmpLw5ODjg6dOnOHny5Dfb9oMHD2LDhg2Ii4tDvXr1dOUzderU8PDwQFhYGCwtLXHx4kXcuXMHjRo1QnBwsOHXwaGhoShXrhwOHz5MeercuTPq1auHxYsXY/z48bh//z6OHDmC+vXrG8YPn38609HREXnz5sXw4cOxadMmDBo0CEeOHKH5J0tLS2TOnBk+Pj5YuXIlli1bhgIFCqBJkyY4efKkIV3Lli1hamqKevXq4fjx47h37x5Gjhxp+KX3l5/t/FE0atQId+/exZkzZwz/NfYqduDvz0ZcunQJAQEBePfuHWbNmoVGjRrByckJQ4cOpV9NW1lZYe/evfTvy1fwP3z4kD659080bNgQe/fuxYoVKwx5Tcj5KVasGNatWwd/f3/UqFEDvXv3xsmTJ6FpGvr06WNIF/+a9LCwMAQGBsLPzw9+fn7Yt28flFL0iWEA6NKlC/bu3YvFixejcuXKiI2N/epnHXbu3IkdO3Zg/PjxyJAhw097k9L8+fPh6OgIJycnwydE/vzzT3Tp0gXA3+3B7t27MWzYMCRLlgwrV65E+/bt4eLigvr163/10w4ZM2ZE06ZNMWfOHLx48eIf87Bnzx44OjrC0dEROXPmxNKlS9G8eXOMHTtWl87Pzw9KqW9+vir+2gQHB2Pz5s1o27YtGjVqhMDAQKRIkQLDhg375nnZsmULevbsiZo1a8Lf3x+HDh1CpUqVMGHCBN3bX27fvo3Jkydj7Nixus+g/CwKFy5MdcXYG3SBv9+EFBERgW3btiEsLAzbtm37ar21trbGqVOnDG9vWbRoEVq0aIE0adKgY8eORl/xv379esrLwoULdWkOHjz4XZ/L/vjxI+rWrQtra+sEfY4j/poXLFgQy5YtQ506dTBkyBAMHToUx48fR2Bg4FfXHTFiBPbt24dRo0YZ+iQg4e3t5+Vt3rx56N69O+rVq4ft27cjW7ZsuvL2b+On+H0aK2tWVlb/2MYdPnwYgwcPRr169XSfhvjRmH07ifAlQUFBeP/+PXLkyPHVNPHf7pkxYwYePHig+06Ise/ZCMJ/GmPGjIGvry/Sp0+P/Pnzo0qVKmjWrBkyZsxoSJMuXToaiCRLlgyXL1/+5vZNTEx02wKAzJkzA0CiAkhBiIiIwMiRI7Fw4UI8e/ZMF8gk5FteadOmhYWFhdG/ZcqUicr45+X0a6/Ievz4MQYMGIAtW7bQN7O/zJOVlZXhFcfxJEuWTLfenTt3EBoaCicnJ6P7e/36tVEvCN+DmZmZYdIqnkePHsHZ2Rn29vY6H/9KtEePHhn+a2JiAjc3N126TJky/cQcC/9t/Nt2OzE8evQIhQsXJv952f08pv/yRmrSpEkBAOnTpycfFxeH0NBQpEiRwlD2vyzrqVOnhoODg6GOxPO9/cuXSP8gJJQv2+XEEP8A5D+Nfz+nevXqSJUqFXbv3g07O7vv3q8gfA9BQUGIiIiAh4cH/S1LliyGG7IJbbfj//tluuTJk8uDNMIv58s4Jb4Mvnv3LtGxiLF+oX79+pg3bx5atmyJ3r17o1y5cqhduzZ8fHwMP6i6c+cObty4QWPYeCT2+DlMnz4dmTNnRmhoKBYsWIDDhw/rbjjcvXsXSin0798f/fv3N7qN169fI23atLh37x7q1Knz0/Ka0LFjQspbr169sG/fPhQtWhTA38d57Ngx3bfFP2/b79y5A6WU0T4A+PsmTWRkJO7cuQMA8PX11f3983MXGhqqa+fd3Nwwe/ZsREREoHv37ujevTsAoEmTJnB3d8eGDRsMcc/9+/dRpkwZLFmyxHCua9asCVdXV/j5+WHnzp2oXLkygL9/iHXy5EmcP3/ecNz16tVD9uzZ0blzZ5w6dQoAkCtXLqxYsQIBAQGG40+dOjUmTZqEtm3b/rSYK2/evPD09MSKFSvg4OCA1KlT/+PNqjRp0mDmzJmYMWMG7ty5g927d2P06NEYMGAA0qRJg5YtWxrSmpqa6j4p8aNwcXExvBq+YcOGaN26NcqXL49bt24l+FMK8WTKlAk1a9bEhg0bEBsbC1NTU8M2ihcvrhsnZsiQASVKlMDx48dpO56envD09ATw9wPuFStWRPXq1XHq1CkaF8Z/3rVy5cqoWbMmcuTIATs7O92P9n4ENWvWRIcOHaBpGuzt7ZE9e3bY2trq0lhaWqJfv37o168fXrx4gUOHDmHy5MlYs2YNzM3NsWzZMqPb/uuvv7B06VKMGjXqq58gAP6+gT5s2DDExsbi6tWrGDZsGN69e/fVedpvEX9t3NzcdPMAdnZ2qF69OpYtW4aYmBjdN++/haZp6Nq1K3bv3o2DBw8afnjZuXNnFCtW7Ke2p5+TMmXKBNcXR0dHlC9fHitWrMDHjx8RGxsLHx+fr6ZPmjQpxowZgzFjxuDRo0cIDAzEuHHjMG3aNCRNmpQe0ihVqtRPeR1/bGwsGjRogOvXr2Pnzp1wdnb+5jrx1zz+k4LxNGrUCH369MHx48eNnrfVq1fjr7/+QosWLegHVwltb+P3bW5urju/JiYmqF+/PgYOHIjHjx8jQ4YMCY6f3r59q3sQydraGkmTJjXsy9jDH58+ffpq23bz5k3UqlULOXLkwLx584ym+VHIwwY/iREjRqB///7w9/fH0KFDkTx5cpiYmKBLly5ffVpTEP6TqFevHkqWLImNGzdiz549GDt2LEaPHo0NGzYYAmJTU1Oj637PU2uC8L107NgRCxcuRJcuXVC0aFEkTZoUmqahQYMGCWpvEzvQ+BaxsbGoUKEC3r59i169esHT0xO2trZ49uwZ/Pz8KE9fq0efExcXBycnJyxfvtzo378WqAjC9/D525kE4WeQ0Hb7a7+s+vwh3h/N19rkhMY8P+LXYIlB+gchoRiLd35WHatTpw4WL16M5cuXo02bNv9qW4Lws/nV7bYg/BsSEo8ktEwb6xesra1x+PBhHDhwANu3b8euXbuwevVqlC1bFnv27IGpqSni4uKQM2dOTJgwweh2v3xAU/gxFCpUyPB2XW9vb5QoUQKNGjXCrVu3YGdnZ4ihu3fvTm+siOc/7QHwhJS3rFmz4tatW1i2bBkCAgJw48YNlChRAgMGDMDgwYNpm3FxcdA0DTt37jRaX/r27YsXL14YztfYsWORJ08enD9/Hr169cKQIUMMDzZ8efM+/obP5s2b8fjxYzx8+NBwU7tYsWJwdHQ0/DJ20aJF+PTpE6pVq6bbRo0aNQAAx44dQ+XKlREVFYX58+ejZ8+eujG4ubk5KleujGnTpiEqKspw49XHxwc1atTApUuXEBsbi3z58uHgwYMA/ufB5Z9Bo0aNMHPmTNjb26N+/foJmi/QNA2ZM2dG5syZUbVqVXh4eGD58uW6hw1+FT4+Ppg7dy4OHz781frxT6RPnx5RUVEIDw9HkiRJDDdAU6VKRWmdnJy++e30+Dy1adMGt2/fRpYsWb6azt3dHXnz5sXy5ct/+MMG6dKlS9TDHmnSpEGDBg1Qp04dZM+eHWvWrMGiRYuM3rjPmDEjmjRpgjlz5qB3795f3ebnN9ArVaoET09PVKtWDZMnT6Y39SSEb12b6OhohIeHG35UkFDi+7a3b98CAPbv349du3Zhw4YNuh9NxsTEICIiAg8fPkTy5MmRJEmSRB/Dj6JRo0Zo1aoVXr58icqVK+t+uf9PuLi4wN/fH7Vq1ULGjBmxfPnyBL0R4kfQqlUrbNu2DcuXL0/wL/C/ds3jf5Dx5Y8AARjeClG1alXMmjXL6HYT0t4mT54cVlZWcHBwoD7n8/1nyJAhwfFT7dq1cejQIYP39fXFokWLkCZNGgDAixcvKNZ68eIFChUqRNt88uQJKlasiKRJk2LHjh30EOCPRh42+A4cHR2RJEkSXL169atp1q1bhzJlymD+/Pk6HxISonvqRwa2wn8yadKkQbt27dCuXTu8fv0a+fLlw/Dhww0PG/wb4uLicP/+fV0wfPv2bQCAq6vrv96+8H+HdevWwdfXF+PHjze4T58+ffV1Xokh/tcBn7fV3yqnV65cwe3bt7F48WLd66z27t373flwd3fHvn37ULx48R/+cIQgJAQXFxfs27cPYWFhuuD05s2bhr/H/zcuLg4PHjzQ/Zrk7t27vzbDwn80CW23439JFBISohsYf/nLPODrMbWLiwtu3bpF/suy+2+JL/t37twx/GoLAF69eoWQkBDaz/f0L8aQ/kH4nMSOLT+vY5/zZR2LfxvZP41/P2fs2LEwMzNDu3btYG9v/9VXdgrCz8DR0RHW1taGX61+zuf9QULb7fj/3r17V/dL8ODgYKOTl4Lwu0hsLPI1TExMUK5cOZQrVw4TJkzAiBEj0K9fPxw4cADly5eHu7s7Ll26hHLlysmc5m/C1NQUI0eORJkyZTBt2jT07t3b0Febm5t/8+ahu7t7gvv07yGhY0fg2+UNAGxtbdGyZUt07doVVapUwcePHzF8+HD06dMHVlZWurbd3d0dSim4ubkZvfm+e/duTJw40XBTKkmSJChfvjxOnz4N4O9XtSfkgZkMGTIY3jQSEhKCc+fO6X7d/OrVKyil6AHO6OhoADB8ii04OBgxMTFGH/SMjo5GXFwc/c3CwgIFCxY0LMd/ivZnvCEgnkaNGmHAgAF48eIFli5dmuj1M2bMiGTJkn3zlfo/i/jXi3/vW/Tu378PKysrwwMoOXPmhLm5ueHTJJ/z/PnzBD3wnZg8RUREGP018+/C3NwcuXLlwp07dwyfKTHGX3/9hWXLlmH06NEJ3nbVqlXh5eWFESNGoE2bNvSmhW/h7OyM1KlTf/XaWFlZfdcN1/g3vcVf28ePHwP4+8bwlzx79gxubm6YOHGi4ZMUv4NatWqhTZs2OHnyJFavXp3o9ZMlS/bT+4vP6dGjBxYuXIhJkybRWwr+ifz582Pu3Ll0zeM/Df5lfTx16hRq1aqFAgUKYM2aNf/4lotvtbcmJibIkycPzpw5o3swzNj+Exo/jR8/XjfGiH+YIk+ePACAs2fP6h4seP78OZ4+fYrWrVvrthMcHIyKFSsiMjISgYGBhocVfibys7XvwMTEBN7e3ti6dSvOnj1Lf1dKwdTUlH7ptHbtWir08Q3mj7gpJgg/itjYWAp2nJyc4Ozs/EODm2nTphn+XymFadOmwdzcHOXKlfth+xD+92OsvZ06deoP+eXr8+fPDd9jAoD3799jyZIlyJMnz1eD6fgnGT/Pk1LqH18b9i3q1auH2NhYDB06lP4WExMjfYjw06lSpQpiY2N17TYATJw4EZqmGR5Ci/+VwIwZM3Tppk6d+msyKvxXkNB2293dHcDf35eLJzw8HIsXL6Zt2traGm0Lq1SpgtOnT+PEiRO6bcyZMweurq7f/Q17Y/sBgEmTJul8/FPrVatW1fnv6V+MIf2D8DmJHVu6uLjA1NRUV8cAbsMdHR1RqlQpLFiwwDCxFo+xN5ppmoY5c+bAx8cHvr6+2LJlSyKOQhD+HaampqhUqRI2bdqkK683btzA7t27DcsJbbfLlSsHMzMzzJw5U5fuy5hIEH43iY1FjBH/i83PiZ/cjp8LqlevHp49e4a5c+dS2oiIiJ/2bXFBT+nSpVGoUCFMmjQJnz59gpOTE0qXLo3Zs2cbvaEbFBRk+P86derg0qVLulg0ns/79Zs3b1K/nxASOnZMSHkLDg4G8D9t+9atW5EuXToopRAdHa1r2x8/fozatWvD1NQUgwcPphhFKYUKFSogNjYWZ8+ehbu7O8aNG4fg4GAsXLgQhQsXNjxoEBQUhMePHxsekPgn+vTpg5iYGHTt2tXgMmfODKUU1qxZo0u7cuVKAH9/mgD4e67VwcEBGzdu1L02+8OHD9i6dSs8PT3/8YHiO3fuYNasWahWrdpPfbOBu7s7Jk2ahJEjRxr99Ww8p06dMtoGnD59GsHBwf/4C/5/4t69e7h37943031ezj9n/vz50DQN+fLlM7g3b97g5s2b+Pjx4z+uf+nSJWzZsgUVK1Y0vNHB3t4eVapUwfHjx3Vl5MaNGzh+/DgqVKhgcMY+LRMdHY0lS5bA2traMB6NiYkx+hDj6dOnceXKFcObTX4ld+7cMdoGhISE4MSJE0iWLNk/Pljh7u6OJk2aYPbs2Xj58mWC99urVy8EBwfr+pnQ0FDcvHkzQQ9n1K9fH0+ePNH96OvNmzfYvHkzypYta7iO0dHRuHnzpq7NfPv2rdGHhEaNGgULCwvDJy7Kli2LjRs30j9HR0cUKFAAGzduRPXq1RN8zD8DOzs7zJw5E4MGDfrHvFy6dAlv3rwh/+jRI1y/fv27621C21Dg74fVx40bh759+6Jz585fTWesHNSsWROWlpZYuHCh7k2Z8Z8M+Lw+3rhxA1WrVoWrqyu2bduWqB9sfK29rV+/PmJjY3XzVJ8+fcLy5cuRLVs2w8MCCY2f8ufPj/Llyxv+xbcR2bNnh6enJ+bMmaMrozNnzoSmabrPOISHh6NKlSp49uwZduzY8dVPC/1o5M0G38mIESOwZ88eeHl5oXXr1siaNStevHiBtWvX4ujRo6hWrRqGDBmC5s2bo1ixYrhy5QqWL19O36h3d3eHg4MDZs2aBXt7e9ja2qJw4cL/6puagvBvCQsLQ7p06eDj44PcuXPDzs4O+/btw5kzZ3S/Qvw3WFlZYdeuXfD19UXhwoWxc+dObN++HX379pVX/gqJolq1ali6dCmSJk2KbNmy4cSJE9i3bx9SpEjxr7edOXNmtGjRAmfOnEGqVKmwYMECvHr1CgsXLvzqOp6ennB3d0f37t3x7NkzJEmSBOvXr/9Xv3zy8vJCmzZtMHLkSFy8eBEVK1aEubk57ty5g7Vr12Ly5Mn/+O0tQfi3VK9eHWXKlEG/fv3w8OFD5M6dG3v27MHmzZvRpUsXw03h/Pnzo06dOpg0aRKCg4NRpEgRHDp0yPCLbfn1kwAkvN2uWLEiMmTIgBYtWqBHjx4wNTXFggUL4OjoSJMe+fPnx8yZMzFs2DBkypQJTk5OKFu2LHr37o2VK1eicuXK6NSpE5InT47FixfjwYMHWL9+/Q/7ZEju3Lnh6+uLOXPmICQkBF5eXjh9+jQWL14Mb29vw6REPN/TvxhD+gfhc/Lnzw8A6NevHxo0aABzc/N/nFhKmjQp6tati6lTp0LTNLi7u2Pbtm1GJ0WnTJmCEiVKIF++fGjdujXc3Nzw8OFDbN++HRcvXqT0JiYmWLZsGby9vVGvXj3s2LEjwa/CFIR/y+DBg7Fr1y6ULFkS7dq1Q0xMDKZOnYrs2bPj8uXLABLebqdKlQqdO3fG+PHjUaNGDfzxxx+4dOkSdu7ciZQpU0psI/zHkNhYxBhDhgzB4cOHUbVqVbi4uOD169eYMWMG0qVLhxIlSgAAmjZtijVr1iAgIAAHDhxA8eLFERsbi5s3b2LNmjXYvXv3b7kp9n+RHj16oG7duli0aBECAgIwffp0lChRAjlz5kSrVq2QMWNGvHr1CidOnMDTp09x6dIlw3rr1q1D3bp14e/vj/z58+Pt27fYsmULZs2ahdy5cwMAsmbNCi8vL8NroxNKQseOCSlvFStWROrUqVG8eHHkzp0bW7duxezZs5E1a1ZMmTJF17Y3a9YMBw8exLBhw9CnTx88fPgQ3t7esLe3x4MHD7Bx40a0bt0adevWRd++fVGnTh2sX78ezs7OiImJgbe3NwYNGoQDBw4gSZIkCAsLw6FDhzBw4EDDsY0aNQpXr15F4cKFYWZmhk2bNmHPnj0YNmyY7tevfn5+GDduHNq0aYMLFy4ge/bsOH/+PObNm4fs2bOjVq1aAP5+iKJ79+7466+/UKRIETRr1gyxsbGYP38+nj59imXLlunObbZs2VC3bl1kyJABDx48wMyZM5E8efKvvgb8R/JPNwDjWbp0KZYvX45atWohf/78sLCwwI0bN7BgwQJYWVmhb9++uvQxMTF0jPHUqlXL8CBt/I/SPn9dvTGGDx+OY8eO4Y8//kCGDBnw9u1brF+/HmfOnEHHjh11nxKZNm0aBg8ejAMHDqB06dIA/r5paG1tjWLFisHJyQnXr1/HnDlzYGNjg1GjRun2NWLECAQGBqJs2bLo1KkTgL/j5eTJk+uOs02bNnj//j1KlSqFtGnT4uXLl1i+fDlu3ryJ8ePHG96W8OHDB6RPnx7169dH9uzZYWtriytXrmDhwoVImjQp+vfv/42z/+O5dOkSGjVqhMqVK6NkyZJInjw5nj17hsWLF+P58+eYNGnSNz8N269fPyxduhS3bt1C9uzZE7TfypUrI0eOHJgwYQLat28Pc3NzbNy4Ec2bN8fChQvh5+f3j+v36dMHa9asQZ06dfDnn38iadKkmDVrFqKjozFixAhDumfPniFr1qyG19QDwJYtWzBs2DD4+PjAzc0Nb9++xYoVK3D16lWMGDHC8MOAz99u8jldunRBqlSp4O3tnaBj/dn4+vp+M83evXsxcOBA1KhRA0WKFIGdnR3u37+PBQsWIDIyEoMGDaJ11q1bR5+aAf6+sR//5phmzZrh0KFD3/zc9saNG9GzZ094eHgga9as1CZ8vk1j5SB16tTo168fBgwYgD/++APe3t64dOkS5s6di4YNGxra5rCwMFSqVAnv3r1Djx49sH37dt1+3N3dDZ/RARLe3rZp0wbz5s1D+/btcfv2bWTIkAFLly7Fo0ePsHXrVkO6HxE/jR07FjVq1EDFihXRoEEDXL16FdOmTUPLli11b7Vq3LgxTp8+DX9/f9y4cQM3btww/M3Ozu7nlU8lfDePHj1SzZo1U46OjsrS0lJlzJhRtW/fXkVGRqpPnz6pbt26qTRp0ihra2tVvHhxdeLECeXl5aW8vLx029m8ebPKli2bMjMzUwDUwoULf8vxCEI8kZGRqkePHip37tzK3t5e2draqty5c6sZM2YY0nh5eans2bPTur6+vsrFxcWw/ODBAyrXvr6+ytbWVt27d09VrFhR2djYqFSpUqmBAweq2NjYn3lowv9C3r17p5o3b65Spkyp7OzsVKVKldTNmzeVi4uL8vX1NaQ7cOCAAqAOHDhgcF8rx0op5eLioqpWrap2796tcuXKpSwtLZWnp6dau3atLp2x7V6/fl2VL19e2dnZqZQpU6pWrVqpS5cufbUufMnAgQOVsS56zpw5Kn/+/Mra2lrZ29urnDlzqp49e6rnz58n7GQJwmfEl93Py/TXyqRSSoWFhamuXbsqZ2dnZW5urjw8PNTYsWNVXFycLl14eLhq3769Sp48ubKzs1Pe3t7q1q1bCoAaNWrUTz0m4b+DhLbbSil17tw5VbhwYWVhYaEyZMigJkyYoBYuXKgAqAcPHhjSvXz5UlWtWlXZ29srALp4+969e8rHx0c5ODgoKysrVahQIbVt2zbdfozVB6WUYV9nzpzR+fh2OigoyOCio6PV4MGDlZubmzI3N1fp06dXffr0UZ8+fdKt+2/6ly/jrHikfxDiGTp0qEqbNq0yMTEx1BMAqn379kbTBwUFqTp16igbGxuVLFky1aZNG3X16lWj49KrV6+qWrVqGepSlixZVP/+/Q1/N1YvPn78qLy8vJSdnZ06efLkTzlmQTDGoUOHVP78+ZWFhYXKmDGjmjVrFsXYCW23Y2JiVP/+/VXq1KmVtbW1Klu2rLpx44ZKkSKFCggI+NWHJvwfxFj7qpSimCixsciXBAYGqpo1aypnZ2dlYWGhnJ2dVcOGDdXt27d16aKiotTo0aNV9uzZlaWlpUqWLJnKnz+/Gjx4sAoNDf2xB/9/nK/FokopFRsbq9zd3ZW7u7uKiYlRSv0d9zZr1kylTp1amZubq7Rp06pq1aqpdevW6dYNDg5WHTp0UGnTplUWFhYqXbp0ytfXV71588aQ5suY+msYi08TMnZMSHmbPXu2KlWqlEqRIoWytLRUadOmValSpVLm5ubUtn+e1/Xr16sSJUooW1tbZWtrqzw9PVX79u3VrVu3VEREhOrevbtKnTq1srCwUMmSJVP29vbK0tJSubi4qHr16qnAwEDl5eWlAOjq37Zt21ShQoWUvb29srGxUUWKFFFr1qwxel6ePn2q/P39lZubm7KwsFBp0qRRrVq1onqslFLLly9XhQoVUg4ODsra2loVLlyYrplSSjVo0EClT5/ecL4CAgLUq1evvnmN4nFxcVEDBw78ZrqvtTlf8mWMefnyZdWjRw+VL18+lTx5cmVmZqbSpEmj6tatq86fP69b19fXVwH46r/Px3ouLi5Gx0BfsmfPHlWtWjVDubO3t1fFixdXCxcupHmL+GP8fKw1efJkVahQIV3emzRpou7cuWN0f+fOnVPly5dXtra2yt7eXtWsWZPay5UrV6ry5curVKlSKTMzM5UsWTJVvnx5tXnzZl26yMhI1blzZ5UrVy6VJEkSZW5urlxcXFSLFi105+KfiG8vEsI/jQ/iefXqlRo1apTy8vJSadKkMeS/bNmyVD7/qa2Kv9Zfzr9+rS9SSqlFixbpxiPx20/ofbN79+6pWrVqqSRJkhhit9OnT+vSxI+TPp+DOHv2rKpevbqhbbSzs1MlSpT4aj3/kn86pi+JH+8n5PomZLv/dA3+aVv3799XAwYMUEWKFFFOTk7KzMxMOTo6qqpVq6r9+/fr1o2vN1/79+Wce0LKY2K2+bVyEBcXp6ZOnaoyZ85siH/++usvFRUVZUgTf72/9u/LuajEtLevXr1Svr6+Knny5MrS0lIVLlxY7dq1i9L9iPhp48aNKk+ePMrS0lKlS5eOjlOpv6/x144zIW1p/Ln6/NwnBE2pbzxaIgiC8IPx8/PDunXr8OHDh9+dFUH4Kq6ursiRIwe2bdv2u7MiCP/1XLx4EXnz5sWyZcvQuHHj350dQRAEQRCEf0VISAiSJUuGYcOGoV+/fr87O4IgCILwTVxdXeHn52f0l8rCfz+LFi1C8+bNv/lLcuE/g4MHD6JMmTJ48OABXF1df3d2BMHAw4cP4ebmpnvzS0L4Me8NFQRBEARBEAT8/a2xL5k0aRJMTExQqlSp35AjQRAEQRCE7+drsQ2ARE3ACYIgCIIgCIIg/G/E7HdnQBAEQRAEQfjfw5gxY3Du3DmUKVMGZmZm2LlzJ3bu3InWrVsjffr0vzt7giAIgiAIiWL16tVYtGgRqlSpAjs7Oxw9ehQrV65ExYoVUbx48d+dPUEQBEEQBEEQhN+KPGwgCIIgCIIg/DCKFSuGvXv3YujQofjw4QMyZMiAQYMGySuGBUEQBEH4ryRXrlwwMzPDmDFj8P79e6RKlQqdO3fGsGHDfnfWBEEQBEEQBEEQfjuako+4CIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQCEx+dwYEQRAEQRAEQRAEQRAEQRAEQRAEQRAEQfjvQh42EARBEARBEARBEARBEARBEARBEARBEAQhUZglNKEGjVykxl9gsEzH6956Opqcw7mx5PZtCyY3vtorcvfKltMth9a+QmlMF3B+Y418MeJ2ytPkAt8UJhdgX5Rc+fDS5MLPjST3uLEtuefv15Crn6sKuYy7SGHEbD62YW3ek1sAe3L3VQ7d8lbtHaWpgefk5ht5LMU/jp0xYsHnPepBFLmcTSzJ3T36+77yoT0LJ3crHV/LLFN43Q+d+BoF54kl5zLJlNyxdN3I/RE5XrcclonLrbIoRE7TSpCzxkJyK+MmkPtkMp1c/Yd8XAjLwvvNfpvzZ1KfXLa4w+RuFOHy59yeC2C1VdvIuW30IDeyl769eJ8rG+dtyn5yNpe4jIbjDLnXsXzep5qGkbsaxcc6w+INOWfVjNyvQivI7k8+ZEwc1Jzcq0Fcrhy78LF0mjyT3JTCXK+qXtaXtR0fO1Eav2qTyS3iYgE05z4CdzeRurGZy57nnMfkoqy4r7LoepK3V4zrd7aOMZyXBlyvWveyJjc1/Sfeb4dy5GaMCNQtt6+2ltKoI3U5H9z0AJGsxhhJVkxFkFt2wYrcrDlcTjCTy9OvglsM4E7x1OS048YKVgEy7y5zv5vsAscU5ZYEkut2y0e3HPmU445k4La/9MdV5N7YkEKKbBzyNcm9g9zAYV7kdr1ISq5TieO8k0n5eb/duc7vjwkgl+s09yUxhW6RM/c6yPu9UUa//KoyJXnVeye5/C15U4/blCe3NNCdnK/NbHJqGm8v17C+5C7fH84JfyGakfgt8wGOt5NUuUDuRBSfHzNlTq6xkf1muc4ubSn9css3RuK+u6tJjcnEexi1lNvX4KbcvmrHz5GzKsZld5CR/q83N/9QjTleOpqiHbmPd/KQq3rrErmXObm+pbj5iJzm5qDPB2cNfpXSkjPZ/YLcQo1j1Ibv+dytsF9JTh3kfjLZyMHkQnZzXP2rSAnuU99o3G9pRrL41EgZSmukK9e2sIsxMvYMnajf3uQJmynNkB5cp+bW4HGiT2befrJozu+s1P7kDl7itvlTYx6LbQzgvlzz4esbYM4nYFb0UXLRTzh/DukbcDrFZa3pF9Vl4B0eH0yuxQ1cyTt8DNtNOaaam8zIOMfxKaky4Hq1FgfIpVBlyP0qUlRlV6kCB3orzMaTe98hmpyvTRdyn3JyTPehMPcbFYt46pb3NOZy+9CWY5Qcm7ii7axACsjK5UC7Xp2c/VtO16/pJnK9tw4gd92kD7lsf2YipyZyTPZ+a0lyQYNIIWpXLnKdF1/WLe/t3prSjDnNcX7GfffIPforObkevTgfsT34+px7yq5dJe6rTj3nOalfRRoOaTG/ahC5VSm4vYkJ5ri84yVuD4pd5vFoKh+Oix6rGrrlajbcPvRRHOOWedSenEN3rgeh0/KQK5cqB7l9KZaRg6uRds6L4wJtIs8jQXHspTTuD9yMzB3bwpVcWSPjx7T99fHo+6GcjYsuXB4PbK5DbryWm1y7PFxHm9hwgDrlDu+3a0VuWBdd3c4JfxFuRtyDXinJaROMzM0V5/Yge9UR5EJz87i4zvnL5KaEftEeTORykfY0l4tnrzjGf1mxHrk7itetRgYIOMjjrtG+PEePJ3zdtCscZ6nZd8nlGM9tfxtzjtNdmvMc46lyvckdnK5viGefbkRpysXNIpdB8fzqbM2TXN6TRup8Xq5D9rl4TiZLs0Xkzvb9jXP04D5Q2c8h1zjsBLl8e5uQazvrPjnbxbxfZcfn0HKi/jxE9uP1NJMMLD9wWbk1j5NlbsnzBa/teC6o3gceRxysb+SaJzOSvwsZyW04xeek9lu+5ssnpiE3uelLcqczZya365i+TfJ+VIrSRDQ6RA5G+pZMu7KSq9qZjyH5Le5LB/aeSC5pQZ4nCK2zyEheBEH4GvJmA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoU8bCAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIQqKQhw0EQRAEQRAEQRAEQRAEQRAEQRAEQRAEQUgU8rCBIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAiJwiyhCZudZGeZy5Kc1i6SXOb5buRy5r1KLktIGnINmoeT63ngim5Z5bXifGj8HMXGXqRgla4QuQCvPOQ2LzpObq+PB7nS6WPJPTuokbuTIge5VclXkfuklpLTPrQiN6m1IzlztOB1H+nPeyNFSaC0v8ile1uOnH/6suTsQniDJ03ykeuIDuTuZeS8/E5ypK1ILktOLgdIwsdsq7FzWbOZ173LqpjFBHLv973Wi41PKU1hcLkAhpIZqXqS63fpCblc4HK7KzW7hW25GVF2nBNt02pO94nP09tTweQaVuhPburequQeRWYh1ztopF7483nSztdil4bzNtB8J7kxs1+RM1dcN44e3c/roi65SWhG7lex5xTnZ6ozN/6XmnJ5eViNt9doO1/LvRjHCQMHktqe/4vzH25LabRtU8gpi3rk9hxcQ65iGm9et8QmclOah5Lr7GFPbruJOblb5dLxPkoPJ4c05Ulp2yLIReU3Uv8+XCbXLra7bjlyelZKY7aHy3dsFc6axUauzLccncnV6MZl3vrZPXLa6szk1MzmvONfRFOcJVcw9UtyW9p+IJdixhFyz7TZ5FLBj1xqL85L5adftP22QyjN0eBT5NTc3byxKpVI9bgXQ255nk7kDqfnvi9OcSwWNXYFuRldOB7b/OI9uVxpON7Tzq8kd08Z6TfwJzloD74Q3Llqo7l/+Og/kdPtH0PujXaFnIq7QC5ga15yix9wLPa7CbzCbUm56pyuSEduJ8xGcjpUG0Rqx23ut4fWWEvOPW993bIyEntoHG6jR33O24lU93ndP+PIbfo4nVzNcAveSZMoUo17cLLSzdqTezehIbnj2iVyp225jGf8wP3YezMuv6XL689BzL7TlKaob2Fy9TvxuVvgzOe9ZP4T5HA7BanDbhxDdd4TwOv+Rjq/+ETO1EhZW1WQz03alRznbloRTS7flhLkUlznfYSOGa9bHvGS64XqvIxcpU41yBXNuYAcjMSvQduOkpsO7mNS7DtPrv2GO5w/xQ3BtU825JJYpeX8HeX8dXzO5+mQLedl/kdv3fIca97WyLu8rYAs7C778jFs/YvH8dVMbpFrHLePXOb7HMtxFPzrCJ7Gx2xXjuvljgcPyZ1px3M32yNGkLM8xX1tlqlcTsO/qFf7P3J8eLcq17PMfEqBqL2kNIuC5Ewj+Vq+T85xRssdIeTUpe28j3zW5F4oLn+3R3EclDn0I7mXyXkuyHx9a3JbY/X9y8Wb2SlN8vJ8ojI78ThiUW6O72YWu0HucjKOjbJu70qukJH47neSo0oDclWxgVzoIJ5zG7KK60vZ3LyPebY8d9iy8iZyqcrqY4oz9jyHlGlZP3LawF3kUozm9hup3Unty8fubTAHd8mNzFNpLZ/zPhTXU3X8LbnRi/ncvXjlTS7nMT6O1nt4t1mG6ftr86FG5npH8T6V/xze2PlBpFaqZ+SepeRVk6fNxW4wjwd+J+Xe8fis6N43nDCE+6wTN/3J2W5sR87BiscxZ+dwu4HxnrrFPZ98KIkPTyEgahzHvFuMzFHHaDw+ty2dityOA3z81Sry/MvmCZyZ9Kt5eyFrOBa50rEMOS1/EnI+rblOjm/C8/vD+1bQLeeYuYTS3LN9QS7jbZ6n25eK531KXeax0Bou3mh1cAu5Qy053e9kgppLrscHngtxjDtHrlUZPmiTigfImY/9cl4BcPHbRC7FBX1fefkul9GrjjznkV2rQw6K+yrNyLRpNp6Swofbo3jdLFyJ6qVnZ/8mjJwn+D5f95Lcvja+zuPOKX4ZyJ3qd5tclb1jdcsRlzg+gYWROYHq08gVPsRtzaEqW8m9XepAbuYorsvBnbguw8glEwTh68ibDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIlCHjYQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBCFRyMMGgiAIgiAIgiAIgiAIgiAIgiAIgiAIgiAkCrOEJpxyqCS5xZd3kGsXcIrchHKlyF3J6UJOu8b7tVZu5NRuTS9MUlEaj6qW5Lp/uR6Al3luk/uwxJ5ceGtet2+PIuQOdefnN8IyuZKrcDsnuaCHS8nt08aRG5F9LbnF6SaRu90igByK649jOR8WpsyyJtcx6XBOGLWLlHcgb7B4xRHkDpVrQS7PvhDex2/kCo6T292bz+mf9fmY33nz9vKo2eTa14kj19Caz/WqjI66ZdXckdKcGjCUnHaiBLnOGtfHps+7kXtZfQC5EW1CyfXYk5TcuFiuB8rIsW6y4XNXa+xhcmn2cj091r01ubIbuN7jgdLnIyCCkmQZMYfckWv5yTntPM/b38/H4HhzHbmbM0+SCwnjNvR3UmHuU3bP3pPTTPiYc9ry9g4pdjnTc7dzlZscWG7fq1u+dzyc0ih/zkdc9BpyHvs5I5oLr6tucL+0ozPX2721epJb/zg3uS7l/ySHIb6kTI7+xXlpxPkr2a0xOef6nJdeKpduudVzzkbHfOzs2nBCLfUHcm5chaAmxJAzV+bk7tnt4ZV/I34oQG7Aek6nFbYjt0HjuKiHfQ9yr8K57RubiQt9zK50umW38S8ozfklD8j160AKazuyvKvSktunBZIb4FKO3PH0nK79vabkXoLP08SGnTiDz7istQrvRy405XRyWr+35CYrfbvS6Q3X+cWKr7XNemNtcFtWxZKQ6mayhPPWgPc700g7+LsZEsbucu4p5DoM5D418/Yocpe28fYcxnB7+iLcjxPuNRKIfomR7WslOVaPK8HlL8ebl+RSrzPSJ9Rcwe4mxwZvTHKRO4j05DpNzkxuoeL9doAHubD79ckpE143NLC/btlsUktK03our+eiOEbLmGcTuaPTi5HTrnB8g5bLSXVRgzkdfl9lGMDhMFSz5uTsDywg9yxXNLmWuQuTe+HJ8VJTr1jeR0/9+a/Z00gdUBxb7x66hdz6czfI5bPgcjvMdBS5eZOGkSuvOC8TNb5uBw6eIdeuJI9Zb5pxeVlvysf2IIoDi5haeTh/Pcbolk14KA6TCxyQLnDm8YtFhS7kqpfkxubPmunILSvmR25hiv+sxt6pGrvzRn5nEjaU2+pBppPIKefJ5D5oqcldszhHrsgkffua535ZSnPW/hO5cVVI4cbrCpy3p9wfD7FaRi5dFRtyxY5lIBeWL5jcYiNDzK4ed8lt2MOx+ouu3M5nMecx1470hcgtza2PK/1SfqQ0bhFG2pB9+0mtq3WInPukY+TmreAx9pmrvI9+qjzvF3uNuF/DVrcQcs8PWZGz9eL2YJyR/ikafMwPwzkGUim531haUb+cwWQMpYHfalIHIiuSK2PG8dl9zCSnFeK6HDSWA77wtX3IdfWpQ+6MkTZtU/Bccpvrc5nc9Yzr2g6Vg1xOU87f+6H6a2Y+nY8r3IXn5Kpd6EguxTZuV85V4+v68Q33X6qBBbnQAf9Z7fzxZNwXNzxXmtzJRhwE3Vn1mFzZ20Hk0mfh8rLoIcdFi1brz2toRR4nvn/emZxFBSPtV6OxpBq35jgmdA7PtWzrcoXckrl8v8Bk6ztyG4I4tlv4sh65mLo8L3XanWOP9OMLkuu66zQ5fNGGp2vD5cyjcVZylo2HkLtpyW1N0vzc96dZuJtczgE8xr48lcdHv5NWn3hsYmdk3j7fPZ4vX+HB5/WMJwcaK0vy/MDVcjyuG3Rbf92GmHHZW5+MFHC6FSm1imOCZj14DnxwjxByc2MbkhvYl+uVxaks5EzGVCf3h8ZzQY+ytSEXpjh271gxktyfoXnJZXXPpBfezpQGPnzyVI6+5Oy2ZCNXcRaP8S5d5fkci/3cf8VM9iFnBp7DFQTh68ibDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIlCHjYQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBCFRyMMGgiAIgiAIgiAIgiAIgiAIgiAIgiAIgiAkCrOEJhyW7gi5sbAjNxMbyFlNmUXueKFYciYXeb8pq2ssk/bWLWqP+lCSkPStyCXRipPr2nEbuYE50pJbb3qDXMFNfAy4Y8379enK6crzuXMJnU+uaNUV5M6MykPuao4AcnHwIPf2tn45e0Q6SnPd8gU52L4j1SvSgVyNio/JLSvxhLe3T7EzcqlhJNmvQjtckNyRtVyWrzfm8xWTsy+57hccyb06xweducg6covW9NeLbVMoDT50IpViHlfxKLwmNzHbAnIB+8LJLVs2jNy2GM7KQu05ucu5UpGrhgheuVMpUs+0MeTGqnnkno5bRO5Jef2xzeFmADdNHchpTbjwbVvB7UrZME53ybcLuSLdJpML5mYAv7PQawFlyMUFnCDXvB+vm2nYSXJL0Jvc4Bk3yTnb1yUX9VFfD7IZOS0XsrLr22EVuSzLjTQurqxe/xFELvhYD3IV/qxJTqVbTq5tx2bkbjtxmzs6bXNysaNXk6tZ0YXcinO5yXX+suqOoyTARFZV27B7tC4jubNtOR7AVe43owdxspQLOL9vuCr/Mly0CeQURpFzKHyWXLpDoeR2v+OTncyX29w5O++Rm1JQ//xnhav2lMYR3M4HRRUh12r1dHLRRqrBgkNu5BaXLExuwmBed2Yl3qCaVZ3cxINR5PIEjSD3usAOcoV37OQdX7YkFTJcv/w8dT1K02z6e3Kd1+8m93IVNzaWPY21y3yeZhS5Qq5/4FVyw8pxvfqVHNrD8WHO9B3IXfsjmNySGUnJlUj+J7n3f8wmV3Alt3X4IoZwrp2N0/TiOFpdG0su1m0fuQtnuH6cbVGW99HlEilXrSU5s5RcnrVSRoZTlbgteDd8P7lpjwqRi9M2k1uBC+QaYYtuuUIXjtE2dq1C7olZL3LWFQeQO+MWR849215yBQ5y2ZkYweuCh0i/jN4cguFRE459P7hwR/inC/cT92K5XgcsuEZu01RTcpeLFtMtN1aZKM3FJVvJIbUnKRNndrHzTpGrN600ucyduA0v25F3G3TLitysvzgOuFnaSLARvJGU6bwKvL2KvI/YPLc4Xe4v6mTUQEpTu9kczsekj6Ty7J5G7vD+7uQsd3D7djGc+4SW+ETOH3xcv4qg65xHdw+OPTb330XOerY7b9CH25LaedeTG8FTKzBrrZ/jaHi/MaVRYVyWtZ3cfis3rsztUnL8kEtxvPk4F+ft0bmU5Ma/5b7vbvLx5FYNTsH5m/+I3N5hOTldlU3kNKs35EIr6+PrmR2OUZrmwdy4Nv3UgNwSG46N4HGd3RQ+hk+qNjlrzCXHvdCvw/oBl2VzG55XmQILcgGoRc5PcZ3OG8kTHzbZeAz8sYZ+kJrmPscxhV/x9dhhupacfUUOwkMcO5PDoqaksjXi+CRrLO/jipGxz9u3XG7hE0aqZgs+fst13P5Eb+LYBh7cR9o3PaQX3q6Uxta6PW8L3H7jNMcisdU5b6bKyJjmII9VOAL+vQRpPHfTH9wvhljw/FrvQ4vJvZi4hNw4I+FcNR/e3vTn+nJ19ISRMfZibm9i3m4nlyuoKrmcPKxA7zk8KZVjMpepF9l4fGZuzmPAmEUHeSeFuZ9zXJeXXIFAjj3yluU+4riRSe846PvEKks4TXPfQ+TmLOMO1+ExjzUW8PQBShR4S87DgvugD424P7RryNv7VdjbHCe3AXwtL0zKQK5RCM9Rv7/Bcw0hBy+Ts+zwklzcG31bYjKsNKWJ/ovbm8iCfH0LVOSx2cVue8g1Gs9x5fAlPM9+yY/328SrHbl0Y3g8XbE2x70ad6/Yu4wns7u5cR8xcQrfI6xTRx/zOfbNTGne1OF7UNjB8UlM1uTkTMP5fOJNdlJ1Hc+Qs1KNeF1BEBKFvNlAEARBEARBEARBEARBEARBEARBEARBEIREIQ8bCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQKORhA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoU8bCAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIQqIwS2jCsQPiyGnISc4Z7uTKZFtG7lb4M3I3y7cl5+Hfj/e7f7hu2T/Dfkqj2h3j9c7z4fZKQQqTXGLI5Xjektzz1B3JbTFpQq5np3HkHIf4kKue04ncds+N5DLUa0RuTfms5Hbk3kxOe5xNtxybpSCl2fziCblNyUkBJe1JNdz9iVzYggbkfNGanFmHYkZ2MseI+zV0XXqe3Mg0vcilV6nIOcYNIXfLdDy5II9X5IbdOUouv9Kf604oSWmGal7kYucdJHf6HSkM3TCfnHV+W3IR7t7kMrbi7T1+wPu1da1PLtsWft5JTe1Lzr1oRXIj6sSSM90QSq5ks0x6YcHtRevyvN60WqSQ9dY8cna27GJPD2NXiNuGdkuW8k64afxlXMs0ktyQw+wWFue6327Ye3IzNS5D0+s0JZcsSCNX9nVNvbjL5y9oL/cZq15xOdNWpCVXMCe3c7lncdt6gpOheRVTciOxj9yaKXvJFZ7Kx3HJh11gt/LkqnUrTq77fCN982h9vWrIm8LTiaXJHRnejNzDftxWa1f5fPaJVeSCCvB+39S4zxIZjbhfxOlupOZ04GRDKqcj1+CFBbn7B3ndMxm5b7d9ze31oKb6c2gzuC6lGdllELm5JgPJaXX5egyuz/1G4Yfc32TyakxuRieuozNcSEGryvst6MP9XEdvXtepdxC5kHUzySXN25X3222wbvnPsiGUZs7a6bz9MjvJmR7iY/XNTwqTO7qSa9OFz6fpRCPluxyrX4nKf4aclp374w7l+XpOsOPtnXvHbbG20Jqc+eYwcj7RO3TLW8atoTTq9Ure6dmFpMpYcn0+nIcD2GKtDvA+Fq4lt8g/E7mkR5ZwXtYWYdeiOim/otz/Na9kJFYfwuML5VeJnMlV/fZsNd5+6I4K5DIezkbulvspclY4QW7YEN7evWQ89oO5kfPExemXsbt6T3KjXnFdz36eM2mSg8ei5QM5ftt58DK5WpfKkNN6VNYtq+kbKM2LHJ7kytUhhWMvDpGzfT2UnKctB+trknCba14gKblZypzc0XMTyI09x+cusOsucjZLOMCeYs7tz8dqvL0FT/XLl9yfU5rIgC7kdp/luHWqC/eJC3Nw43yq5TVOZ8mx//ukHC9BcYz2q1BG4tL8rt3JVb3Pdb9Va0tyq4qtIOe0kcfKBTYu5swc0MeXagDXPc3mLjmTeYd5W/c5NuqkRZB7e5L7JZMrvLmM42+RU05cXjT8Sc5pB88ZpVnWg9yQDNyH7e/L9UAt5okpy3frdMuld/GYOINrMLkHqXlO5sJ27vtjTbaQK3B2Kjmr+tx/q7XryeH3FXkc03jnHazOkQsozWPbiwesyOXTuB6gLqdTRuZCtH1flKt6B3lTpbmtvq/1J7cs42lyWfZwHVJx3H5fNuG5tE2dz5I7dMuB3AnwPg6v53a5yERvck0ded618vEQcnXMua5NdNfHaH9qgyiNUrXJaZPSkKs/jRRMs7CL3faAt/eS9wGtD7vfGNu4X+b4q2iuUeTaRbUnNyUr93c5Z94jd2kMz6svcchHbtB6/fz29Uc8v3Hahed786sqvH2NT+r1Sb7kOikelOzXeLwXbqRdsvHn2N2sHZf54VaB5NrV53p6pRyX5dJXdpCzasuZsZ2rjzMH8HQoeq3muYNT4Xw/JmkTnnPef5Gv66G0fD5NzKLIDbcNJ8ct6K+jfGMe/9VaVopc7hIcu55eyXN4E0y4b6tzkq/vhPocMzt4HtEtq5iDlGZLDM+b1UjD5btW4Gpy5c058K/2kMtP9Byeyy69nuPjqLrpyfVqdoPc6MeO5JIW5fntts/+IpfypA05tTglOdfp+nPw5o6RvtWHj7Wc4vGlicNccs4TeE7mubsfubW1XpODxvcLoIzcHBAE4avImw0EQRAEQRAEQRAEQRAEQRAEQRAEQRAEQUgU8rCBIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAiJQh42EARBEARBEARBEARBEARBEARBEARBEAQhUcjDBoIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgJApNKaUSlDKGVXSVI+Tq7DlP7sD15uTCmu0n1yu2NrlMf14i13r0a91ytmsOlOZaSBVyUalfk4uO4HS2QR/JIacXKY/+g8g1H6yR62u+h9zF3RXJdRjVmtf1nUPuSpZZ5HpNDSBXfGwYueN/TNIth45fzHnLfpdcKRdSCO9RlVxB023kbuTmvJnu52NY1P8FuSYqDe/4F7Eoris5P69F5DKGvCO39kozcvkjgngnhW+S6mb9gNyELi10y6r+fN7W/FOkjrwbRG7z9erk0s5pR66rJV+3/hhL7lSO8uT2Xp1IzvXlWnJdunchN21KY3I3S/I53nvtOrnYRnvJLVlhpltejTKUptTzOuRc06TnbWESOa0UN6EqNB+nu8xtoyo7ghwC+7L7ZRh5/iz9E0711IJc3OUD5E75csOx/kIRcmMHcXlJuXCybjnoEdcL7C5JKqgit18PJ3DbUsD/HLmkybzJhTV7RE4t5g7R7uZAcv1SLifnHfKYXNZF3M41b1+Q3EKnreR6HCUFkzm3dcv9u2fmfXIRRe49XJZNXbhPe1X4Frn317KQu+EcxTvpyGVHTU1YGPIzUBhJ7m4KPmaP4FbksiE3uaWxz8jZOI0j9+CeL7n3Q/V9aoOzeSnNy0NcbreVbEvu2JEb5PZ1uUxu8tr75GqfqElubj4+Brtg7iOP35tMbqo79zmT7icjlzwjxw9N2weT06ZzeSnUe5VuOWzUJEqTJTv3kZuiuJ6tvM7nuJF5JDnlWZTzduMMuYzPuDzdS/v7yjwATJrFeRrD3SwG+TwlVywFx6pRmTzINVdcFi6FXCBn2aSjbvnjCm7rTd9yP/TiLvdX1V+4kTvb0owcnvK1szBfQi56HMfv6FaZ1O1RseQye5YlV77WPnJ7U4wiV7kFxwF3G3YgF1avl255TB9KghP+PAabAM7HKWVHruZ4Hkfc787nPeX7OHKdHbmMTfr0G8t9OOcn/zXOz7FdvKrVDF43d0dOd6nkQnLdvf4g5++TWrf8dt1hSlNimgO5xWe5z/Er3YNcUI165KKf8jV/tGApuSJFV5JDfR4rNg/aTO7hgRzk2ntxOu9ULciZNz1GTmXnmF6boG+LL1u4U5pcZwfztlJxjKbAbb3J7RPkxrcJJ+dxYCi5tBrPKeRTa8j9MrZxuY1ry31+2ycZyWXEAHK99nJMN3YZt1/Ri7le9S30Rbt+wpXSYAXn91YzLnuucX7kGq3xJ7f+7RtyJut5/kEFciwTZbKKnMWt7ryuHY/jPn7iPqe/WxJybcDl6mMcn7sAf/156b6Yt1V3XXJyFcdGkMvwmGPZm8+4LG/STpNLWZ8Upqbk8XOHadxf/zKumrJbxP2z5XGecwy4vYFc8QJNyBUI4rFdkpd83VJ6eeqWH858Tmku3Awh9zEjO9+VKcjdXs7lzO0kX/NZ4HTtIjm2RmeuL0W6ZyLXrTaf4+dXOb7upArzPoxMM1TW+Bx3KZFHt5wvmOteRBluL1ymFuId+CQlVXN9AXLN/SeQ854/ibd3lzt/lSma0/0iYtRLcma9d5OLXcV95bNHPOedATyxYFl6BbmGB6+Qc8lqo1s+f537m1fR/cidtuD4JAd4PvAq+Bq9zsDX3Ckjj8W0uTx3qjxasgPP/5nc5z7y7ocx5Fxb8rUwTcNzp9FH+Rx3+mJ4ZLk/A6WZ2Pwi5w0819bCmvvv+S5G+q97NuTwbBqps5U2kitwnuekfhkaz3vMVjPJveMmAm0VjxOTduXYDZM+kHp+muMR57L6MbHmyP0NHPjc33XmsXTHbVyndnThcVjgoezkOpzneP6VCc9xaabryblcqUSu1vwQcm7jOE5/E851Y7zdPXJPFN+Hm6o56Zan8VQCytzmOjW7BvffsVs5PjUZzveqEO5MSivB1wK1uLNSv3MMKwj/hcibDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIlCHjYQBEEQBEEQBEEQBEEQBEEQBEEQBEEQBCFRyMMGgiAIgiAIgiAIgiAIgiAIgiAIgiAIgiAkCk0ppRKSsKvWhtwkzOaEWaNJ7RtmSq5cbXZlt9Ykl3zfKnLl7ax1y22H8yEkb6eRi5vJ6Ryrc7rBGx3J3TStx+kwipx2sDs5dKxG6rB5FXIlzxt79uMCK3c+T9r9WHIzQ56RC2iiP16Vn49/xpA4ch3iHpJTTdkdnF2G3OGUpDAvwo5cftcP5DY+SFDx/Cl4Fefre+gYHx+0wqQOPOVk7mXNyJ2b+IZc0SpJyTUqoy8bzQ/yeVl1nPe5rRg7vuIAxrNt3fYdubDAYbzfauPJDd/M1/fwkYHkdqUeynnp3tpIBreSsQWfg/S4Re6klky37KCCePObnUl9yPCanF0HXnXqcW6jTqIZueRFuG28dHYTucPRXL9/FZ5GzukRjcuGk5F1VSC3XyHFeXvJ5uYjt6rCcHLTs1TWLdefWI7SBC7l6zb6whJyc663J3fM3Y/cLGcuj153/MmFJKtFrlwHbi+W1XMg52LKfWnUWVJABz7vISO57Ob/aye5e+b68qey83XQPrBLe8+CXFj4GXKhs5by9rZzO4D9jdjlXUFKnedkv46eRtwfZOysw8gVi+BysCddEnL7ut8kV74L99lZX+n7kpup2lIalao/uQXL25EbX346uXOPeHuw4LZKqxlJziqwNLnkqe6Sq72W63dQtVByLYdw+a52jrO39kIBcgMec1/yqnp13fLJ5Gkozc3dg8nZm/Uj55xlCjnXwIbkspIB7pf9yDJwspGUvY24X4kHmZiLd8iZ/9mRnNo/jdw6cLu2fU4Rcos6GAlUPvTVLVomcaMkn9oaKRzLkpOq/4bXrb6jD7mGrcLJXdySnlyB/D3Inav6hFzRS83JnX6yl1zuJRyXP/BtQs6tuZG6uoDHISm+iOjetn/I683nsrvU6gq5QiEcg+dnBc+OfI7PDOf9DkzDY4nBRgPQ30i4D6kjtqfJlazgR65ySY4XyncKINctGceSedVG3fKbnnyuno7hgcSHUenI3eXijdxpHpPb/NyWXM1aKXjljTwu0UbyQO5eNi6PHiVjyM2fvJGc325u198O4FgjsPwscpcs9HVj+GiOZW5eHkCu/gqObw6/4/40STJSqFuOr8+6nVyYVVcbXnmakT7hVzHPhd3SF6Rmn+Y8VhlUiVz/9TwWWzSTy9VqG74mPbLqx89JN5yiNMfbcz4yBZLC6+F87gtM5HRXb3N9mfmc45ZS9azIuRsZD6mZfD7fBjwg18aE57jWGgkY3gVeIpcszStyTf7Qx5+V6tahNE1XGJkv2c8qD3jcdDEHlwkjIS9mHubY0M+M2zc483n/Zbhz2RifJAO57rd4zKrOPOTt1WalLY4g936lNblcKfT9y+m8aymNY3VuC3uiAbmx2jFyamwFzlxPLmhVL58gV3fGJHIxM3mMWcBIscpjpB8/AR7bFb3HsccWI9entJG65rJH379ElclGaT6acUyvjMxQaBqXUWVkn0jP82+Yx31aSEWOHx14zV+G//Dt5FKf4fZ15KZP5NSc2+QW2lUnV2c8x/1JZnF5zpx/j245WyiPTzcl60VO056TO7Y4M7liu7jPxqaZ7Gbx3LvWjNuBIRq3/W9H7yJXeRaPZ0oHcbmy+Mjnzrogx/gRJ4tz/h58MWlbjOMO9fIhufVpOCavc/clOdiU5n0O53qLfjdIjQ1uTK67kfDxV/FGe08uVnH9TaVxOYDaxqobJzPp/BenO8dzRha19PM+0We43VOFVpLTcI3cG1WRXHh+ns+YuZKPq+gjHgx0qZCH3AON5672qnnkKoRyLFJ7OM/XThvD22t+hdPt3paLnMqkb1eePOTx9d0eB8mVqcj3Y+z28FzTs5M8J2d+7TC5apO3kGtyeRw5nhEWBOGfkDcbCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQKORhA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoU8bCAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIQqKQhw0EQRAEQRAEQRAEQRAEQRAEQRAEQRAEQUgUZglNOEnNIfeh2Uxy15f0JueNi+Tmv8xB7sDuzLzfnlbk6qfWL8c6aJSmfUgrcntmcrpKW2+Qa2jmSU5rxuu6lhpLLmL0UHKmuZzInVj3jhxik5F6/ZqT7RrJDtVYBTydSi56ttItmycxcu582pDbas/XX7NR5MzinpGLiWhGTqXcSy5fclNyv5P11fqQG7eEj7kX+ILEpV1K7pHpW3INt84jt2vyIs7LrlW65c5j+Lpt2EUK01mhuHIg52qknEW3fkFuVadO5JTDcnI16qYmF4oe5Cqv5/3uKjuO3MssfJ5K2J0ml67TYXLXVYhuuZhmzjt14fp4t80wcnm29yeXPSlvrqNWlpxWez+5+9FGKu5vpBfqkrPqvI5c2KTN5LQDfuS6FS7HO7k5iNSx+bXIHc4SqRdLLSlN+1L+5Ez6ct2IU1Hk9qznPmiy921yIYv4+FXvPeQQ1YDUpX5ryN3v9IZcujWO5N5qfC3OruB080a1Jme1T99OvfIqSWkO4QG5kv0iyaV8Uocc/pxFSo3pSC69aRdyT+ql4O0h2Ij7VXC7hN0nSX3Y4E3OtATX6TNPue6b9EvD6cK4L8n4rohuuUi5vyjNtMu83gxubnHDyHFZnwwnp3JXIDdp1nNOZx1CTuuVnNzcalvItcZicnatkpArcuQ9uWNoTm7EG44z3qbUt+Fa4Q2U5syLFuQ6aNPInbLlmAV82vFuegZyrXZxWZ46jeNiqw68vV/JLtwhNx9ryV1fkJ6cmTaB3I6dXcldaLOTnC2MjCUs7HTLFdtSEkyPm0uu+Rtuh2p+4HULvk9FrumzTeRWpHpFTsXyhV9doCm5yJsc02oat6fderqQS6547NMvaUNyNtn4OI520y+fuMTn99ynY+SaNOb+pajGAVncHWtyg59w35EmLdfJpuDxG9Qndr+IyUbcVTtuw0e4+ZCbtJ/HdmVdeXtxR3zJaSorOaW665ZDx/L4J+lYrqMaeFth6iY5CyPtVXAyLht4x2NlaNxH+9jdInc0jOObWO0gueeK+4ltf80ntz+UY7fw13wgs9Ppz9Wff3lQmrBoL3LTx3GZT5KsOLmGRtrG1hd5XmB4z27kTk7nUVcR7mJ+GaM2PCbXuzKf0zKH+dynL1uZXOmAtORuOWQi12oOl+enqfXxequIg5TmwIsYcq+q7iOXNGMcud5OfAw+Rdn5XeCxBKbxOVEdeV1jpGjH7TeseHvu1yuSy67lJjfeSLCxLHy7brnbXR7HppvD+R3MOcMjHoIYq/JY+CkluT9v3yXnbpuNHI84fiFGYqtu2iNybbpyxQzo+pBctVtGfpc1kftF+8kbyc25Ul23PCJXYUrT1/MpOXfTxuSaVh1CLrLDJnJWp0jhXQ4uU9fW9iPnwNUKuQ5lJHe48n1yJcZz/6LcGpHrxGEMYg/zPNq7LPr+hUfnQOYc3KZr9znuLKGMTHIhF6viPAbR2i0jp3i6FjhjxP0iTrbnOZT9m3jeo309vpZj+3I8NzSY53iOvOa2f1ZBbvsrfHLVLU/n7h8m4DI/HDyXWHwj36YIGcHzcNotHjQkbWYkng9fyOtG8xhT9SrA6e7bkAPced24EHI2vjwfr57vYOf2RT1ox7Eoug0n5WNjwenerODt2xiZc7Quw255KKkeTXgc311xvPurSKl4nK5p3M6lieT5nHBsI/d+NO+jYmF7cn/E8FzfmFr6+LhL4yy8sYPnSR24xveHUt7nfaoHpci9z8rzkCZxA8nds+L7XOeKcBAQO7YduT6rOaa4cSCa3B9Hx5CzK8nn3aoVz6f699F3WAtMp1Ca9Bl4fvGvQO6/w2tw5/fXFu5wphXl86my8nmKAfe5wAAjThCEryFvNhAEQRAEQRAEQRAEQRAEQRAEQRAEQRAEIVHIwwaCIAiCIAiCIAiCIAiCIAiCIAiCIAiCICQKedhAEARBEARBEARBEARBEARBEARBEARBEIREoSmljHzZ0UjCMHZZ7Vlerx/CCbcVZfecv5Fe0YG/h7TnQz1yMbYzdMvmD/jbaehVhNTUbOwyD95OrlwTZ3Jmy4x83+VadXZG9rEN/J3k6l1IQfGnk5GHP/WLgbM7kwucMYncjIqleR++B3XL+RV/j+f8ES4S4Xz4+BDG6zrZ8XctQzX+nujVzPztoqjbL8iVUVV4x78I7cIictPLcFluH8rfZuKvTgLR54ycr/wLyL1fyt8PQxP9uhP4c/P405y/jXfo/jByG5tzPdu61I1c+bf1yQ1W/L2zOnn4uDpf4gLTweIouYad+AOBU8Zy+Wt7kb+2OyOvkW8ptTTy7fc/9d+29NvDV+dwF65T5oq/y+Wj8bUeUYad6svfqsIp/r73I1v+Hp1LF171V6GBr6XSrpBrqLgArsz9kjd4mb9lmg38ne3risuuyS4H3bJHHv5eYJ003PYPV7bkNI37ERTnbxA7HvMntxn8jcP9z9ntPcdlL6g1l2U+UuCvgxHkoqrwN6I33MtPbo3iPG+A/ruT5aryPgvu4P5rdFH+HjS8+VtsR/gzwKj8gPvND1cPk/O4y994vG2eoDDkp6CN4TLv0IvbqhdL+NvOVjXLkgt8xm3J+2xvydWaYeR7g4FftAdtuM241pRPfrZGXJaDjnN9dDRy4Rpb8DchN03jmCVcO0EObbgMaQX426vIsJRU082jyP2xmb8Z39h7BjlVgfuwiYv1H1Et7RxCaZal5+9QNnzSh1yWoXPJ2U/k7xSO6LSEXI++3H9Xs4gltzthofdPQ6vH5X7FWv7GaXXlSM7OyLehG7TmfdTQkpBrtOMcJ6zyRLc4V5WjJK01buttu3wkV2gSb775Yf42cTNv/pCses/fYA+O5uNPofH3Lms4cBzQ4x3Xy5Im/B3suFzct228xHXQ5/BBchki0+mW37wIoDT7unCf+4yrHy625ut60kg5DdxnpC3I0ZNUdBqO0cxUaV73F6FtZVfWyCdsM3Tjfmv0OU7oqPF3X00OXSRXUPH3f08v0vcnK3n4g0bRRtqm0Bbk0h/mPtXuFrdNrZfuI1dlBvdh2h7+ruqwU3ys/Xg4AFWMv626uyl/r3jvB17XLB2XtVFbOB1K6Y+3R6ZdlGS0F9dlkwUe5J6ajyeXg5t6fIzZQC4qoxcndDZnd4K/wfurWD6J6/QyE/4e/K7Oz8hpisdnM8EF1V+7Q273JI65q3XRlwNN7aU00Faza83fD1dz+JrXj+D5gj+t+dznDuL44Xoh7vvy8TQVDk/nGGpQ4xTkzIyMQ4rGcptuY8b1alzdmuSCblvpRR/+RrdTg4fkQpCG3HBnjrPqe3AcuO0gKew/mZNczuZryQ28YeSb1b8IDdzHKo2/N26dlwdGEReMbNDIPNk4VihrpK80e65PWJCnZDD9Ha8Xkf8BuY7ZcvHKruVJrc6ejly99w3J7T7F/XP3XrvJFeOhHYZy9lBneGVyB2vuJOdwjcvfs+q1ySWdoj930TzcwDZbPne1nPgY+jzgMdgoI6PxEzV5e3k2mZKz9uG2Ua37fTF9Mh6mI2R9JXKbi3chV3Mit5vKvCW5xVHzyPm7lCDXc41+/JyUp8WxHTy5PQAclObozec0zcik5DKZdyfXKKYQufZTeK4ldaf75Iq59ycX0JlP8l63DuTGVOc8pzYytwYuVhiyUN8AbW7SldKcNeHjb4MQcnNwkJzPa55TWO80hdyaMdxe1u25iRx+4zjWyBk1SlqN27nSKhk5b43nvH3MOHZHjC+pwVP1DfugEXMozd4Xs8ktNjtLbulpHiOrovPJNY5cTq6O4jmOyxqfqR6VeK7Pv+11cuPecRtp7VeanONMIxMAPBTA60U8X5n8kf68nO3Hsc0ijYPy8u2iyB2Y0YPcdLWK3BiN52u5JwVCdwSSK1OZ5ycEQfg68mYDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShTxsIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCopCHDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCInCLKEJlf0BcppWltykw4pc223PyFk5cLrzbQuTm2ZXl9yAKyn1ecsxmNIsaZKMnO/HruQC1Qhy7x59IIfcbUh1rFaFXNT9B+QmPnMkN2XyenKd8/iQG9KIz1OdloGcv/UaqRwW3chpvR10y6peSU4z0pfcA+fFvP3kTch1X+NOruTsY+SqDz9MbgzqkCtD5hfSujmpT7aDON2dnqTeOFqTS8WXCCs3+5M7E8P7Lbj3tW7ZLpTL1IpDDck1zuhGLvL5WXKTKiwj1/y1DbmW93qRO+7HZbRcl0hyx6YVJOfhcpnc5LmkoLXmfcysfIec4tMOLZs+LyUQS2m891qQmxg7iVy3Q3wRl5XivI2cbUfuVFwXctsHZiUX3aUxuV9FlRLsPNbkJLccfMw4l5LdRG5fPHs2SlBe4ky768VJLj9rN/C50vq/ITf5owe5zuDykyMJrzvSLQm51jf2ketfnfdRo8UMcosGtSM3PL0nuV3dH5FrMHIeuZ1oT24q9P3aId4lArezG32BnRo9hty5QXz9l4Q/J1d76XByt7cYKTvc9P86wliNGcllNGdTvh47cZBcuSQcK82ryG3OCiNtmiqvj5WaLktLaTpMrkjuwMdd5Mb24oZ0Zmx5chcmsqs+NA25dSO4Lvv4cvsdfSEXuXzp+Fgrln1K7t2pPOTU1ZXk3LO3JFemdZBu+U9c4211iSE3uRvHO879+VjtVQS5vlozctrC3OTS4BK534332qTkZpxJRa6hZskrp/pEalUbrsQOczhGbNSL20nTL4YiUbFcXpzUdHLeGsc8f71fQm5okr7kYMXjgcy3uTyHhWchd7nOZHITRg4ll6kFH3+F9+fIDbLn8uGT2kj7UNKKXGyc/tgGdeTYa7sdb2v4Ko5lihnp1wPzViPneqEouZdv7pPTOLwxFjn8MgIv3iJXpgb3vSY4Si4jXpHrb6RNaKvtJhcSV5xc9uaZdMuvcJfSzFOryGX9g9sc6zOkkPfaMHJHup8gp/a0Jrc/ciq5OZYLydVVHPNdAJ/PZ9pecuNVZs5LqDnnxXkRubFfnKudRprXNBVukss7sxi5lgW4/6tVkPO7MJbri/a4Abm3tWeT49mIX0f1B9x+70jF11f5JOeVeQgI7SO7Ngglt6Qzj22rl3TSLa9/04HSrL4zmtxAf25vYv34og/l5hEvxr8n55mCr+WR3Dt45U2VSTlt59jowOv85BTqkdPq2fI+UJrMIV8+8VpVfR8Z/p4D1+GKr3Xrh9zi9ih0j1zHKZxuB7geNK7O8efANVznf2dLX3FhMLkp+W6QG9OnKrnmXYzU/XHryC0fzPFOB/C8lnVa/dyKAs9pxk7m8miW2Ztcx+xTyG3tx2V058CO5OoEcLp39lxvr3J4AlWT27lhQ7hvOlrNSAygGZlPXcMxcu+zceTuxBzXLWfCIEpTwsjmT7XhPjhgDs/1jlRcRm9qfC2sNR6HfRzE89q/k3ep+pBbfeoguRoLeayo1CR2E2qRq2/D8w++ETzHsXugfs7oD/A5naqyk6sQzdfjFU/NQTPjuXJEc7+xu2Q4ucEdeR+vO3C5ddSukpu7iccbS8B9mI0qQI5bJKAptpDrUc5Pt3ygLk/KmazjeZV1zjyeQ3G+1uuNjMVLIQ+5egN4jG0ktPut8fybK9zhZ83F4x+/DpzLYXE8N778zCJy0wryeLJQ99LkBnXQt1+WlqcpzaT2PL5+F8DtXngeLj/RPA2HOTW5r3qgOMbwMXKVBl2yJ9fHay05l6QryJlE8ZxMXFvO3yUj8465ivH1ubhF314U1bitVpF8u7Jmdr42BUvxvRFN49p3oN9qcvmH1yeXsQqPD+4Z6TcEQfg68mYDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShTxsIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCopCHDQRBEARBEARBEARBEARBEARBEARBEARBSBTysIEgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCInCLKEJz4SXIfdi3GBy2U/xul3P7CRXKvs8csX28sresCP3eks63XKznJyPOUhFbsKV6uQq5WlDrpN6RS6XzSZyR++W5O09qEbOJiwfuc4vfcg1TaXIdT2fmdyAPrfJZW/H6+a/k4ncoaGhevGhI6WBJ29/arFb5PxGdyE3rtZkcmPxjFyn5LzulPvrOS84bMT9Is7wOX1q24jTrePja9KOz/1eE95e1aHHyWln/uJ9aKd1i+qjHyVpZXOIXI3pG8j91f4quVRJB5K7EepFrqI6SG77ykXkyt3j/Hl0GEBO8+RrPgx5yOVW28hN0bbw9v46QA4vi+sWizQ+ymkG5ySlKmjkbm43Jfc49gK5vlk+kAsq1YNcyuaNOS+/kd0r55NTk16TK+zFbfXVVOXJPerZnNzGd7zfY+BzXaKivr7E5KtBaep+4PJT9XYEuXPDYsl1NrLPg+A6und2NLkKp/KQU2G8vbT2pPB8/ixe98Ujcg6T5pILvdeKNxjFaskhfZ6PTuI0vUdakKubNxu5tpmfk/PMysfapSH3uWrmS3KlXDgvh+uw+2UMYeVf+zG51t6cLnhzMXLF1xwjd+sDP9eZzPQFueYe7rrl8JG8T/Oo3eSsYEXuT1wk1wMPyW3fkYZcw8dcD/K4tSTnY87t5nGN69+jUbPJobc1qT7vfMmdOR9HrnH2DORMMx3RLce51uJ9uhlpZypxf1h+dwPe519zyKWqEExOxaQgt/gR16HfTetYPsagAS2MpCxIJni3kbbOjfvGO47ryCXrwOs6P9XXD7P9/SmNqtiO3JSuf5DLWXUFufO3uVx1fjWNXA7XJORaaZHkbNaPIec06ya5Mgs4llYLuH4omJMbf3Q1OQ1NyMFllH5bj8tRkiZluT3LnqwzueYZOW61XtyB3NVCnA284euq7htJZ6SP/VVs6D+BXI3TU8hd3FqXXB5we13GyOX4qLgeeOIauZVZl+mW1ZERvDEjoao9N//40JHP6cVp3L+XQl9yzfwvkktjUYTc8iscj0zVqpDL92YBuXoqLTlEcR16MDaGnN9eB3LK44s+QblRmhfHP5FrhPbk6tzj+A7vuCzXfjKI3PxGTckla8ZtyO/k/WQ+D8sz2ZDL2szIb09mcNwMh9OkNCPtl6alI7fcX9+W1l7QkNLUSR1ILnmWG+RyXdpDLkDjtj+F4nUfcciDLC+5LH+sy+P9zOU5psUm3i86cznY1mMjuYInz5BzqsblVIN+HGaz/SClaVWhNbnxFTlr5jcvk9uRoj65+RWCyA3hKSng5e9r041RtSG3fXvP8Ph7W92u5AahAm8wisfpF0/xHJ5NbR4Xl0MW3fLm55y3ms48v4qVHHcszsD9c7m0K8mNjVpLbkApbtNGDGUHHp5C4yku3K+9ilz5tUYGvA/yklrl1pbcFB8uqLYZ9uqzlsNI56fxdSioeF7AoiC3Rw+e85xUQVwnpzryuBj9/7PKPHpz31l/N49ZJq3oRe6CVS5yi59w3U+bjufh1rzkdS/E6gOjjZl4W94at0FXjBQfWyPzNEkdOF3oPm6YTh0/zwk1Hp87ehbnZDcXknOaxrHNEq5q0GKPkMuSldv5+fl4jrV8Q325Kr2d5xfh4kDqRHkujyU+clxk2WEGucO7uG4ggvMbfOMEp/uN7MzJbeR7I9VyuHaX3MBpD8g1vpmMVw7g8ZpW30i7ad5Mt/gpejGv576PXM1PHHcV4KkRjPrEE6eVUnO7VPdVOK/8gPN7zY3HODlKcgxQdgvHuM8y8nl/l5Irb24Tvh8GJ46VWs7V34pUxbiN+mMRH8Pr7bz5mtFcAFyX8bplhucm92EGH7/t1f+wdl4Q/guRNxsIgiAIgiAIgiAIgiAIgiAIgiAIgiAIgpAo5GEDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShTxsIAiCIAiCIAiCIAiCIAiCIAiCIAiCIAhCopCHDQRBEARBEARBEARBEARBEARBEARBEARBSBRmCU1Y6I9e5NS5MeTe2g4kF9p9I+9YZSUX+ekKuU75TpIzH5lZt2zVWqM0S5eRAvw3kDpX4ia5yk0Ok3vdvCQ510fNyI1ybU3O6WADcn/cDSPX1YYvh8mfJ8jdXbqEXKbpW8hFT3tOLvmdhrplK9d6lGb3CkXuD06GOIwiVyLNOXLpX8SSu7mCy8lqJw9y9Xm3v47pGUlN8CxHLupxbnI9cJXcyZgIckXSFic3WBtHbsOpF7plzboxpalditcrX30luQb91pFzGtuQ3LMZbcnt4aqGT/Wqkzs31UhCS1ZQB0kd1NzJXUzCZX7QVT53cU5cdjXfG/rl1eacjRRHeT3H/OQOeRhZ1zQVOVR+SSps2lhyH1ryqnZ8CL8Mh2aTyHU+OJTc6fFDyGXfOIvc+nxNyC317U+u+EjOS5iLvgyZ7XSlNCqVI7kd4HbE40YBclWTPuDtOR9i15zbqshjj8gVs+D+cGTunuRCep4nNyztanI7bnJfMs+jB7k/Ne6vChQurVteNfodpdnuze3yAzOut16Z+ZnEmZ6XyeVfmZPcnhIzyAWGtCcH/MZCr3G9vHasGrnSzWPIFQYX3NfVOVZwGLCW3PFuHBcVzXlNtzwnObeFW1OEkquZnutUwC5u05DqFan2F/jcH7NMTm4LuAxVVPPJhVnxbt/bcP07tpzzsqYn52VrOS6TQ7Q4ckFfPDtre5HLbcQezhsK9yOldpcg12dEd3J/We0l1/Ij78IXaYzs+Pfyuhq3174mWciV31KK3KY8vL0717i82XpPItd4Al/jafZddMvvIm5QmrkIJ9epdzS5RxNqkPszL5eh+qsek8va0Uj7P82LHBTHHmfb8HHln21k3S4WpDyXOpAbacJ9DOpznqe/1y9rGpdJ+48ZyIXV5bitSizHtwGPOPbKvrcT5w0c8wBGYqPfyPSnc8id3szXrfuSI+SK+HH87hu8jVw6R+4nTHNw+av2RRFX1bldfxnD60Us5UB6Ss4J5PJUJIUyfbjOjzWyjxVG+uPsFauQ86jFfRjuLyC1OmUfdmoar5uV48UG1Y6TKz2xg2654zmuFwu9+DqEh+8jp15xjFLDgbM2LBXn7ZR3C3KL8x4g58ub+2UMO8vXt/sbvr73B3Kc27kCd+Y3Qj7xTl7cJVXoMM/nbE+nr0ONJ86kNBYxD8lV9C5DbvalvuSW1Pcjd700x0ajgkih7U52Q/7g+aeBrzlegBPnb2prngu56h9Frkw1vj5xXjyWiE2rj8m0KqaUJjW4Tmm3eRwfnZsnyB4rPoYgXy7LmLuVXSVW4CmFX0bnAI4VqpWoRe7P1Tzn9vINx6VLJhk5r8UXk3NswPHIvgIhuuWOabiP7TjKyHkezXU0LadC8/U8nloe1ZWcVmoEufAYH3J5p54ll3PxX+Q2zLvNmWlZjNSaMnwt5hsZ7h1HNnIeZ/VlVxvP/ZepkZ/MVdDsye2EG7kYfCAXBFdyaSdzhk9pXG/T/cZx7LLZPDdQuDbHX+Erub/DdG4PPo0bTe7WMB4X357LY9Rhc4fp91mc52naanXJnb5VllzeK37kcposIndkZBdytR1tyaUL4ljp2kZu+65lnUhu8zmuQ7mmcLka48/zPocv8n2A8jFcXr7sSp08uH7fdn1PLnc5Pse97/Bc08d0HPffDOZK5FaLx7sqlPslJGX1q+jVhV13I3W1vxZJzvwCz2uO3NeZXLdZfG76cxiJxjb69lDbkJ3SxH3kca1J0x3ktCg+Bm9tM7nsaZ+Rm9iI+7kKrty21onhIGjDVp5X2e/Gc32+dbnu5vqDx1Z9FvGYIaoQH9vcL6Y6z3bltvVOPu797o/ga1jHyPxTaid2r8BtjdWKPOSSHblIjmfCBEH4J+TNBoIgCIIgCIIgCIIgCIIgCIIgCIIgCIIgJAp52EAQBEEQBEEQBEEQBEEQBEEQBEEQBEEQhEQhDxsIgiAIgiAIgiAIgiAIgiAIgiAIgiAIgpAo5GEDQRAEQRAEQRAEQRAEQRAEQRAEQRAEQRAShVlCE94+WpBcIYwkZ/2hN7mrrQ+RKzi1ODn725vIFTk8lpxXzvm6ZeeuGqVJZaPITez6hpw2+DC5eUW7kzuNceSGue8iN8kkH7nlr6+Q25IihNyZG0/I5b3uTK6vxuf9+tpT5Kp3KkVOUyt0y3eyrqQ0D9fz+YxUfD7fYC+5Fto1ch6NppDrW8mK3OSnjcjBltWvQrV5QG5fh6LkJl5LTc5ctSd3tO8OckUvxJA7NjEzuYHbLujzVjgppUGzIqS2ZuhGztdqEDlV25PcgA3e5DrP49326NaLXNFsXF6KRfG6ajM7DVyvHmarTG7OkTPkBi3k7XW5oD+fRYIOcqIU/UjZBu0jdyJuJ7n7WllySd1vkrvS8h25HpvtOS8Jb5Z/OLkKc1vV9aA3ub6aF7mBIVnI9RjN5aDJfBt26cLJpcuir/zaiI+URuuVjRxGPyV1/TknW/CUy62G0uRUuCW5JOkiyb1K34Nc7yfchsyYuJ7clLxHyHVz5HOXNeoiuRxXcpKLzL5ftxwWVJrSjHjAfV/d6mXIPdkyhFyWzRvI3c7K+dhYvyG5oS1ukRvCp/iXsVzj/j43uA+MTTOR3CBVjFxoX96HdqMwuVkruc0t1tBDt6zmcPnOeMqB99mE9xnGoRjmHuHyiLXvSU2rsoecFsExIDR/UjvUfHIdNC7L6S7wPjI1vkRumAtfCycj1+dpttm65RalWlOaqe9nkHuXtis5TNhEatSfnEx9rEBu7VxOd+jPYHLTwlJwwl9I9zyLyBVcyv3W02eTyHXmy4ny65rxPlpcJGexhted/l7fZh/TuN0ovWYUucghu8m1P8jt6zjf0uTShqQnd2DzJ3JaCXYxDbn8TbQghb1zeOyT7Qyvu9tI1XKtk4tc9JHJ5Mzfv9ItK43bGhi5Xlpba3JpO3HeWu6txet6FOJd3OF0H7NyLGdzIwln5hexOsNAcgXjON3pIVxIJ8dxjOLd0o5csj+nkTvkwBdAmxugWx7XvQulSeLGY4awdrx9T7PF5Jx78lgU19g19ea8LS3HqwbN5Ly0SMXxcIGC3CZ+qsYxxLb9fOJnh08ll/EsV6yqiwfrln26BlCaHsqU3LQm98j1SfKCXDFtFrlm74aRc/jTnFyFs0vJ+RZoSu5XMWsQj096OnG6iNCr5BZ1504v4BXPNSxcc4Jc9waB5EYE6WNklZ3zcWHmBHKeEZXIlR7Bcdus3m7kTvpzfHP+4DpyScM4L9f3cHw3oByXA3RlV9JIO//Gleua7RsjjXMEK9z5oqxtaEtJXtZsQ06Zcj7AYRaKHOZ25cVIHlCfe7GEXL6y1XmDv5PGPIbe2oPr5TmNy8vAixxzzzTj+M33Nk+GvGrTkZw254tx68YLlGbHGS4DrXh4irj7HNSvnrCIXI1pXIA2ZuxD7qKpN7nunXi+bkLvluRuz01DbuLgLuTqWj8mV68p1z+17BW5cUpfdqu+4na5XSy3R2dXcWxXOj/PpV7tOJPcm4yPyD2z5+vz6q2RemskBvxVzG42h9x0Dk9wIpTbjdf1eSxidb0euX4vue7PaMFzQRHvMuiWM/q3oDRvZ8WSm3stlFzeirXJ1ezGfVXV3Tzw2mHOx9oHPD64lPUkuWzneD7VwUhondeW5w7HJue29LAbl29N4zmj9uq4brmj51BK42HDfXpEY55Tn54zA7kXuZKTM3nEsdgMF1KI4yk4YJMR94soPrkGuWEHjfR387aTaubsSK6HKV/zts9Kkxven3eRZoE+aLa5MpzSlHXlIEN9NHKTY0h9UlY2IeSuleZ7Rl69eOwcAm9yD9NyYxXTgI//QU+uGw4uXJavt7hNLmrkeHKqEI9hixXQn5f0XUdTmnshVcm1fcgBZEMjc0M7NFdyGZfxfF7zrdXIHahLClhrxAmC8FXkzQaCIAiCIAiCIAiCIAiCIAiCIAiCIAiCICQKedhAEARBEARBEARBEARBEARBEARBEARBEIREIQ8bCIIgCIIgCIIgCIIgCIIgCIIgCIIgCIKQKORhA0EQBEEQBEEQBEEQBEEQBEEQBEEQBEEQEoWmlFIJSxrJK2MAOdd0seRyPOV0W/GCd1E5C7uk7qSeb1ipW04bXYjSKLWbXPT9LuSs7tcml7H8JXK90u4jd8KnLrl5V5eQ0xbwMx2vXWqSy4Cm5D7V9yYX5peJnPXoh+TMkvOl1Xw13XLaJXkoTaU2F8kt8A8jl+2KPblHyXORa4DL5A6F8fkM2LiAXLemy8n9KjZDI1cTncn1s5hMrvODsuSc+gaSWxUaSi67bRJyJ1bry9C+w5QEKwezM919kJyGxuR2rb9JbmAdO3J1u1uRKzsrL7m84Sd4v4rPZygcyRWp/JqcdS9SOHf6NrmFqzOT8z+v32/+xpt4Y8vSkjqL/OSmozC5bd4Hye3cfJZc5bZFyPV6Zkmu9OYENsk/g6D7pNY7PSWXv7cXObfqnG/1mHcRPIfLwXNn7l9yNdCXNZXGm9JMmX+KXNjMC+RmqnTkxmvW5EqGb+Z8XJ9HrlWhReT8QszJZXlgSs7F8Q9yc6vdIldxvgu5W225rrlGupGzutxWt1yr5zVKs24At0emNty3KFWVnElMALkb6WaRy/iK+w0zl1LktId8zX4V23CaXNwVjilm5ORyuytyCrnYqHHkticZRu6d+kjOT9NfJ4VUlGb9qn7khhUIJ3dh5CRyj9143aSf+LgWDePtLUnPMdu5fNwfYFMNdtu3knpVdTy792bkZi94RG5G1w+8D9VQt3hTa0JJ8vBamDq3G7nG/blu2Nw7Ti6yAvcRlsd6k5urNSPX8jc28wAwfg33ed2C48idf8bX3W5YQXJZqnP8GvFiEjnrS9zHDDfXtzt9T96jNA+rcTySbjWXA/N8EeRUTS73WjWupxPm9yH38motcgHRfJ7cavxJ7u76YrzuKI61qg6YSa4znMgVUBxXJbXNoFsuEcbnqZMJj0scl/UgN6MJ91fTNB7THQX3nY6qDbnWHhwbz7zz+wr+rdQ8FjvYZQe5gBmVyU14zPnmCBQou+ogub9KlyaXY69++VJTjhmPoh25Ye98yOVySE6uo8l6cpG905MbN2o1uawjOK5q3pX7osdHuf2vcYvPZ+VMHC+mO9CWXN1RvO4GjfvJhwfS6Jb/LMNt1KCUpDB75EVyL1vl4YTD+Vqf69eVXP4X08jdXcBl3r0vX8dfB49ZYCTmacxhALwXPidX149j6Yd9eUzp+pDL5L5eD3TLFfJwvN1EtSK3FPnITVhfndwCF87bqkn1yIXX6kmuiG99cpXD/cntiLtLLlpbSK6FC5/QJe85NkDNDOwWFyCVw1ff5/yRjGPKcRM5zkC9I6S0davIhT7gcXeS/ByrfxzN28vMUzx4Ovk3Bjg5eTxVbZsnuaSuHcmlGBlDbpI/z/s8NOG5G09HjptfmOnbKp+YcpRmeBRvv5gFX6O3daPJnWq/llyq0jzuqrnoELkW5bgMDQrktlobsILco2q25Fb25zK/Kg2Xg9f3+pKb16Q/uRMn3uqWzcExa25j83TbeJ9PqnmQS4875N6vG0PuSAceI1R5WZQcFLerv4pInCFnWYfnIbGe5xC0c8XJBfU+Ss53+AhykydzbN15xQHd8nJwvPx4L8+XPCq+hlyNLd7k3kzmOfpuQRyTds/AdbnQlhvkcu/jOnSy0gZyipsVqNzdyZns7sDp6nI/5Ixj5A56ZtUtPzrD5ewNuD423sxxesWam8hlgTe5vrm5P0w9iNMdrsVxYUnFcdEvI5bHXI/M+N6Ky+PSvG4HbiNqRnOZb7pxErkt+deRW3NF3w4NMzFy/2VnFXJ//hFC7sn7xeQyHMhNDjw1hxnmHC+3N5JQ9eF7MIdceR62uinn770Lj01RphKpvVvykKtQm+dMrtvpx9jZkr/hzT/mgH7C4XPkAhvx9m895WuxhQwQGcPz8aNMo8gF4DdP3gjCfxnyZgNBEARBEARBEARBEARBEARBEARBEARBEBKFPGwgCIIgCIIgCIIgCIIgCIIgCIIgCIIgCEKikIcNBEEQBEEQBEEQBEEQBEEQBEEQBEEQBEFIFPKwgSAIgiAIgiAIgiAIgiAIgiAIgiAIgiAIicIsoQnrpbYkp7qMIhd7WyOnpWrA69qsJpf7oSK3eu8zcu9XZdItW2mbKA169SVlPqodO/eO5O64ZOXtcTbQ8qwLOa+PJcjld+V1A4tvJtfk2CZyZjtTkos58Ybcq2VXyPku5f3COZ9u8dmGc5Rkfoqb7HbxuYOdke2/uExKS72BnNpTntM15evfzcgufhWu823JtXhZjtz86HvkTl+rTK7qeK4bGUcVJtdw/ElyYQ+r6ZYPFdxKaUybcFlBRi9Sr1pWJ+fUJhm5PxBN7tTdSHLFSnN+j/OpwyZbvr5nPvI5uRHNLvJjEnJa1vfkRvccSm7+fv2yf1lvztz7A7x9R87Huf58jsts5YPNgi3kbge4k9thcZXz8hvJt6couaPgMmQ3iq8l+nEfEVwsDbkUDXjdFLjL21ujT9dq1lFKMjdJEK9X2InULI23X78ut9+wKU2qewF2faK58fNPz8elveAyhDzzSQ25kJqcl5H6F9tjO7nRdXaQS/LFc4T17mekNJlsNpLrCm6rr9avQG7f7FnkPPN1JVc7MAu5VY8fkDMn8+uopnEbjDGc7wq5jFzfEL6+yi4vuZol+5M7l5+vCdQp3WLuv/icXmrAfezQSrnJ7b9gSk67yuuWPXCB3Nvl1uS6pMpEzrS8kXNSKge5fo25HRjyeAK5zS425O4jnNxGI82P94fk+nzAitIo3whyWiveVhYVy+su5b5Fe/qR0/X35Q2qZux+M922cBsWm+sQuSTF9pMrijPknLdyXbDi0B+bzrLzhz6G6j+YE8U9yU+ucDFuSx8pLrtNt3KBUY2NtM0XFpHqo5XidNouUnkH1CJ39kJPcnP6G4kDb0wnt2nVMHJ/xpUmVzp8gG45/WGOUYem/X/t/VV018r7vw9PWgotxd3d3d3d3Z2N28Z14+7u7s7G3d3dvbhDcap5jsOVz2/tHnzL81/rdZ3lWpNkMpm5555p4L0SzjRdC9XBJYdMcp71TZ6Hlzuwli+71O0acHN4ariR9hv7QYb+zNW/r+A4iGy79BfLpV/FTQK3LEEjuJONfzmOXxftx8uf4DizzRS4ate47gxg+DdVf7C+O+zRcMk84sL9XPELzvvlBri9xZrAZe7EtruVgfNJIfMMrvIPjudBwc55oudS5jLnWjCrGLogG5xxmUuMy7su/g+vdzRmJbjC7UrAvXa5RbhhnaMqyVzdmOowh+6UgPMxnBurDuFmw8FI/nBZj890HDOKGnMttC7cS0/uSXRLx/XUtruMwfUib4Jr83093MiInHNOf+e689OoSXDXmN6Z5Rb3aUxoX6g8rcfAzezMcXDjeTLHsRXCjjvxtUtn3uASo0xalnv0hO5fxp91XRmnzlzmXuAfZfQ0qJ0pOAoPxua661A/xm+P0FE891ZSuGf2PbgCmws7jt9+WYAyhb1qw1nmJdyuDYngSuSvB2fW8Ln8au+DO264BrG+sC4BFZgrdt7JcfBtVjDcZbsC72GGwVU6xdw/g4nqOH5scc/nlBkCV7nKRrj4rbkHYAdynTehDmPDOFMD7r3hPf4kkSLko3TJAaJax+AGuMx3cfYwyfO8VRSuWnVe7/oqZ06a0zAPvmY1hUthmJPeYupkoubLC7c09V0W7MK16E/D9XmVO4yRZ2oM5/UM86IaAz/D7duZEm4Mt4nN/QDmFFFs55zjt4m5e7nDJeCaV+e+yhibfxdIeI9zpNXCZd7oxX3Nou+4Fv+TfI7A/arkhvsK35NBmdD97PNbXR6v3ELmlm+vsFP+vuzvdZzXr/P4H7htphdcugyMS2YV+5SJ8BWqZUH+TatTPO4bWiUPwNk3eYuvfuwHNS9xbzJ6mahwn+q0hyt1+wrcvAzO434W846ASczjclbnC8tZLRDO+HGfZmFp/p3vSfEtcNGPu/zxz2UpKIT43+h/NhBCCCGEEEIIIYQQQgghhBBCCCFEmNDHBkIIIYQQQgghhBBCCCGEEEIIIYQIE/rYQAghhBBCCCGEEEIIIYQQQgghhBBhQh8bCCGEEEIIIYQQQgghhBBCCCGEECJMRPivBf3fuMgH1aD+8d0O1yCEt3nxi5e76WXBZdwyiQVbxXQcpt1aE0VWu9Q3vcffcAErbsAtaMJvMNJv84YbmWYU3LbvNtzuwVHgJg/7BmdlaQSXshrvYfpTXU26GK5etSlwe25sdRw3NWxznwVzeINWBaB6BnrBjYzHUzdVqgWX+ybbKU+qlDz58GO6cCJz5+9wi54mgbPm0ZlyE6Dalb0C9+/9f+BysmnMYVPbcbzqHdve5Oc7OjO5Dty0k5Xg1lzoCZfsoCfc0y3r4E6aenCdI3SFqx7C2DAoLpQxkV5CfX6dgOUSdIEqbQbC2aHrncK6zmv5zIPaEzgSbsHRg3BzGX6MmcvY2C8bX+yvV59ZFZf3H14M7fUWzqfFerh77xg3rn94Che7aTK4gC0t4WbmZlvb6286jrv3zowyj0r4wG3wZt1mu7Sp25d2oZ5r4PoXOwn3My/j99ORY+DsQrzxpbQu903Ptht1sCFc8mSch7YdCIQ7+tt9f4WyTe6N4jjYsH4oXJavDOrzzgbA2W2/wGWtGQRnXjH+/Eled7oFl6B3RrjPn9iG1fryelYDl/hyPBhqaO4HcK2Mc6yV7TIOZd6MHA43ctQKuFKrp7MeDXtBTd7XB66aGQ13vC/bKZ9HD7iIvl/h7K8f4Ob3KwIX50cqOF8ftnu82qFwfTd9dBzf7zMDZf7pAGXyLrsNV8FkgiuZlmP+7vlhcLXiM17kHZkBrr99h5UJR0atugv3D8OpSdOXOcp7+wLcxW1M6ht0Z7y65s97fEjgjBNNKixEmWx2bF6rTiu45D3p3k0qBPdzMGNz/FK8b+x7jHVjEjGIH4hyH84atg9u1yiOy0+LuTYZ/I055LRPx+ES9azgOB6xLDvKHDSV4Uqk5BiPdo2x62se5kb9WsyCm/qW97UbRoYz9p9LcE4lYbyyz3KNaUWPynIu15vlsn46WuYY3A2zEu7lYeda7PDx/C735F0fpeLiNns89tF7uSPBFWnFNWZLazVczau8r1WROZrZyHz7XKeYcIdd0oCayVm/0xbHeFmXNggNfuU4Hrw9Ico0jNsM7kP1KnBueeArvlZTvTcforj3VrjTrffy5AWcT8KLhUwtzedAxup5i3bC9V7A9Z4ZGw1qRCRer7Rhe9n5nXNO1TVs6EBPTtLPI3NPwuNuNrjTZhOvtwDKFGRIMwWaMQ8KcZkPtryCMpWiMRGMxKWdecYpwlzexr577u0I1s/ccxyfaOzSScvRZc7KYreuuUSzVUugMo6PAzfzGtc5PawacOv/4DrWTJ8NZS9mhXz+Yh7zMxfn55Xr2a/SjlwGl20d5+cT6525QtrhtVFmW3PuZWw1rFvFDIwjdkPe0yTmfDC7DYsV6cbc98oX7i1FbM815oLZ7GsxXfaCjNkDY61uAbcyE9u4UcHUjuP6U/gQrUtMg7tsmHdE5NRiXnbh4PjSjHthH1Ychguqw/1Bl125cOPeRDrfblwnnYvM93bIPwVc7X3sfw0usK2bH9sNV7K+My+4bnhexDhH4Mq/Z7yNtYpr1mWGwXVt4jxweXaeh7PT+cEFJWWb/G2/hzMu+cm9S2ynspc46ZSzeI/vbXiuPWyss8yz8igzNSPbs0dX5oUTDdfAJi3joH2axZLanPu/uEw50f5gnL/6yA+u2CXmBS8Zco13Mu6vRb3D/b/9vtzgtVtx3W9aFHccWkX4bhsO4N9z/mlaDO7mNe7Dmjh8v4Us7qkfsJnRHt7Dl7TrqMszxC4LdW8qz/3XcI7sG8A57Fgo84cIzdiJni5x3mNSCZY51bMtXIXm/LvX4TzcawtcynNtbjUZK1VFuJ/tucYxLvmjEOJ/o//ZQAghhBBCCCGEEEIIIYQQQgghhBBhQh8bCCGEEEIIIYQQQgghhBBCCCGEECJM6GMDIYQQQgghhBBCCCGEEEIIIYQQQoQJfWwghBBCCCGEEEIIIYQQQgghhBBCiDBh2bZt/7eSFlStlrfgNi+55HJyY5gYo1iq3z/54WZ+Pgv3rMJvdWnNa1l9vODeVg+E81/E50r7N6/X8yKbabLNc+1TPNf0pxowlu3ksWsrXLG9wXCf9o2Ea3A7KtyvkK9w3p6/1flJE1Yu+WAoq2tauH7TIsCNth/zXIv1MI0zQ2Va1Qfupj2O54YTlv0Obm7jALj2a5LA2W2nwl0JqgP3YwnH0JOd5eAaXnS+N2tQU5Q545MFbu4vtmmTTlCmQH/27yiJD8JdfDAJLleaPXA/rJhwASEf4WL93h+NMZ9ZPTPUqyNccNAEuOn2aJ486KXz+peWoMjiXexncb6VhIsRJR9cpdRp4JKnygH3PGgG6/bR5Xuvq/HowgkriHHEjlgLLml6b7j7AfPgvHPz/VoH2Nfs+i6VSfhbIE7H9mvZmO1X89xyuGo1S/CeKxOzbiVH0AUO4rm5a8CZH4zf1pzCcP3LHoC7bzheNoQUgCsXgeO5th0Jrp055zguZMaizDNrPNzzzi7BYWYQ1D9Bv+BOez2BK27Sww22a/IeZrOLCx+sK+yjM6vth+v8rCzcT8OY5p0qFu/hcl97McfBpifO91tnGN/R6UcV4ba2YwzuOC8X3CSbecc0y2U8doYyUbryKbq4hLRJZ3i9wedSwHWOwv4S4zr7t5WG+cin4BdwPpnyOI4j3fqOMvErct58u2UunN+tiHDJVzPWWC8qwwXO2AVXPnFVuEO/tsGFJzlX831Gacw6nbDZZq3O+8PVz8t7JLE4/kPmroXL0s7ZV6dZGVBmq3HJe+1WcG0tlwzi12GoH8VYD+8IU+Eyn/KBe2zzHmutaHA1WBOz+xbbvWKmdXCW4SIp2OV5yxT/x3HcwCMFyow9ch3OzzCXtZNwzrGrc64vPJNtcnjBIbgsbTl33rc5x4YXlkskPhuNefP8imfgFt5aCPfsui/c/ZjMVWsOeQb3uU4Cx/HsDMxBv39n/pDGVIKrWoNtv7YX12LjS3LuvdbbEy7gaWm4SGV4j/gtGOvfRGkHt74n1w1VFwyH25JrJlzDbT/h7KzOPNU6xjVmhNi8/gHD/pjQZcVxrXIMuIU7ocwYl/Q9V+gHSps5QXhhvWafD03I9xbMpY3xOsRz9x5iP6jZJDZc+rbZ4BYddOaXES4tQJnsFufj4OZ/wXluiw+3ucFbuC1zvsEtC74NZzJ1odvBOJArXVy4wBzMNTpdoeswh7ewJp6DG/aAudvgyr/F8B2MDQU8GNPP1gjhTV226RbPccnvmk2D2/qhK1zxGlnhImy+xpuEE6Vm81lqduK6sEuDh3AH1iSAK2OxHSaZq3DVP3CfLFkEZ9tE3OMSSFpQWec4Lx3KylhYstMXnjyL8XtRW+Y7VbYyP5v6+i7cmIhc2zfu+wlufkXuCf7IMgYu133GyBMTGCND1jnXV6mSMG5VeM74ncU+AjfxVHU4824ZlBXnB1zr91yH1erBfKDiQ7Z7eGEZJuCbrclwNfMWhbPPu1xwAsfQt4YV4I4lng7XrFU6x3GxxbVR5lKuHnBHVzC2eGRg/Po8tBBc8xH8W0HZlDvgxj7i+F4RYwpc08/MyUNNA7geSfkHg/ED3sCZkRwvQQ9Ww92f5tx3zNGP+VQccx/ufQmuic2RVXSR+kL17s5xVSiY7R64sS1c/ceLeI9wwnrCPjqKXdSMnNoMbmsFjv3cLuuDePFuwgW/SQXXxjjXRAsSs02tvMXgilzl2iBtdv4hadDGRHAZkk+FS/KiG9zDLdzzvlKjJVxOwyQwXzrOfTfvcR3xrPwruDP7oExFm7HULP0t5p4ZhiJ15rE9N25yyW2K8O8xJmFynhuSFK7sJK65o8efxes1YZsIIf43+p8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBMR/mvBNOYiXMEs6eDs1fF4cqNxUAOW9YXz/3IM7kYOCy7RA+fxy+82yoQ0KA7XPtJnuPnt8sKlz3UO7uoMKLM9bnY4y1xhwTGbocqP8YJbau2CizjvPFyyaUPh4ifnffN5sA2iGmdbfT3B9rVmtIMb3Dkt3M2/guFeN9wEF2FaJLjg6FAmaNV4SsO+E150r8++fKPVSLg5K3vx5JYToHKsZrnVdZ7A5asUF67z9kuOY/tqc5Sxfq6As/t2hruZfB3c9iQt4UzEMlBpZqbifZumhhtlPsINCOwJF5yCYzfCW1bFjpsUbvyTWXBNTRe4dCN3/GYmocygfbHgQruzbifXX4br9DQHXM1eUeD8CySEi3ktDpxt3sGFFytc3nnWqSPgnndNBOdp3vOCVg66w/mhBkdcCzfidDPH8dP601EmpGEhuITfG8M9Tcbv6hKdC4VLagbB2V414azrr+HGRWR/CSzjC+dhRYM7bQfyHjmgTCKb/W9DzZxwQzY7x3i33FdRpurX5HDpqzL2b87OOSK3F5/V284Bdz4yn2uI1RHO5uXCjQ1eVeAyDX4KZ0dnJRvXqwW3Pnt3uKT2ZLjmVUPgur675Tje1Iwx/b5L/xnbkX15TEzmU1MnMCeYnCg+XKaOL+GO9r4Jl2dLJri8gQ/hBj5aDWfSs/95HE8M124BY3PM9C6TxL0vdL+zm+rnYrpILtUdlLge3KjVzNn2ZOf7ORzw52L6/+JKm/Rwue3YcLblDxepyly4Rds/wH3rxf42oQJjTF3j7FtVL6REmdP7feAOBUWFM2YPzMnTJeC8t9LlSXAI7v50jvGsb4bANS/B3ONJi09wN04wjlSNtBQu7RnGbDvncLixz746jgvYAShTwOYclsNwXWKd4bgPSPgIrtnSX3DeLms123DN9SdJ4JLmBt9lHMqekPP2O5e1SFLrDtwnuzXcF8Oc2/ptLWZc4pBdj7EvjSkJF6FPbrgmhekaBvEdXW/D/pitxUFWpjnXKobLEGNCOOYPvi/PU4dVhev/wmW9N4LzicdNZ55W9SPjVojNObx2K46p64tzwJVKzzbZ0W0qXK6KnOsDQlmXSL+/6/CkDVWB5FyAn+t9Fs6+OgruXoGvcD9OteBNMvlBbSvnjF/VDNdwnhuY00+s/R3ukscDuPizuKX1bUcQ3NdeP+GimZ1wwQm4jh/4Ywpc40iMh4+e34db0JL7KLnrcWws8OgHt2xcRcfxw08/UCazPRjubDJvOHsB67u1Avvo3dAYcIkPQJn75a7D8UnDj6+duMb/60ozuPrp+MyHTV2454kY51/4cy2WpCtj5LwVzjnQZz3H3vtkq+DMdK5j36Xh3H50IvfXqsxlAvttHl/c5gPH4R68LgZn/V0Armth5o++JbnO8S3GfYFV/TiGahfvAXfx0W/vZx/nrwlnuA9Z13BNZ4/n+uBpdObvAx6uh+sxg/0kDkPIn4zyxmy5ALXR8F2ac+xD7y0/uDi92f8a9mY/jW2YZ/gUdL6TzYbnDS80D27Jbu53p5jJ+paYw7F3YQbzb4/O5eDOf+ZbinWH95gXnfuw61+xTaJH5P5nRY8UcHE7cW1xe+ItuKCXzkVqZZvr+PENOA7KeqyE6zI0BtyVk3z+T6MZrWtW5vz66Ocf7eHArlcWzuq1H65+t2VwZX8x9hnve1CdSzG+WiYzz63rnMcXcillvm5jTnAt9BVcgsqM1bnrbofzzlUQ7lHEJXBWEBc+9hb+rWagX224Zte3wKW1uP9p2dzPMpE60R3h3kHXBc59pGonXGJZdPZ5rx3sj0ENxsAds1vATajA69XdOhtunzfXDGWbQAkh/h/ofzYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBP62EAIIYQQQgghhBBCCCGEEEIIIYQQYUIfGwghhBBCCCGEEEIIIYQQQgghhBAiTFi2bdv/qWQCqjHTKsNdbrALbkPuo7zxxzZwdrZYcLuqn4ar+DnIKWpF5LWS74Nb8bUsXMGoFtzVX1BmuPdwuOsPB8M189sBtywx2ynZsm6sy8jicJM9a8I9TlICrmj+V3B2/0xwUfJscRx/M3z91lW2yaDXLJeiYny4JaF8FzH+2g+3Y3EOuE2GDV/rv/XO/xOsn7coc2aGGlmYlWxksQ1T5nJ5mFCqa7Xouvea4Dg+uLoPykxe2RluQ5OZcGssl3eZkPVd8NYf7v2NGHDLMyaHu20mw1nmK1xe8xzuvPkHruAr1u9Uwhy8x/NycCbJeMfh5dZ8/hzVeP3Q5XHhPNe85vXXunyzVY/KXlgM7lfvK3De37/w5HCizOf5cGeTMpaW+doVrkSC3HBdU/aDs05vhTt4oB3ctvXOfjCVYdQ8qsZ3Of8a3+XY7LHhLvVbD5d7bCm4qD94veL5IsOdP/IR7k2Gb3DR3rEu2V1Cw3GL/cDDKgm3/Ox2uCNFnPPmgiAf3iB5DDq/vlArJhyHa5a4I1zHfLPhZqVNyHvYnKv+JJZVx8VWgrENA3NaOwZcbpfYv94lRCQOYbz2s5zxxdPm+3hqbYJLXnAMXMjpuXAeOxlbrdvXeb1bLeGenNwDN7jQbrg2X87BFfnbH65d8Tlwqc0ZuPpD8sBFHfoE7otJ8pt5hjI+5Tg/fFlxG86rOd+hdZyD1P5ZH86EMq4YuwydxbwoPLlWms+Y3YM5aKIDteFe+LK/Rd/KOeFzMZ7b0CsN3OkZzrr4vUcREyd5f7iEKZln3CjxA66eywAc2CAOXDY7GVzuan5w+Rt+gpvNbmpMujtQdT3Tw62fnQ0u7vvGcO8H9IIbvz+C43hKmnUoEy8l+6mndRLu4lnOf1b+tnDmJ9/hJp95cMW+3YSL48vLhRsusTlpnOxwz6Ocgqvqxzl/2/b8vEXVs3B3XOb3DlYqx/GhFktQJtqM8nAhUXPCpRg+Cc5vcA24H2nesSIBC6DOPGsNl//O7/HVGKvvCzjPrYfhyrxdBLf3yEo4r37n4YJGuAysxs73aNscy1ZqjuWBE5nTF6vFd13Orgt3JNIEuOLFmcuZMS4vm6lxuHHeSguX1zyCs+yBcGciLIcrEMz3a3IkhrqakXsB5cY484pXKdjnLyZibP18OyZcg+it4L5bOeDSZ+4Cl/UG31Eul9jQDcaYTiO4F9Qk2iC4QpwOjbEfQnW0GEuPG7bL40POGP71COORyeyyn1M/Kdwkl3V3T5v5XVmTEe7wkm1wQS2Zy/5JrG18l6ZKDbrNjJszy/DFnX3HuXhFmgZwPVz20yb5/JZ7pHfbc+MYNdMYg59s5Xpg2QmeGvMRc/9OE9rzvtOGws2uPxouz0nmzZNe74RbO8nl2eK65NLzY8DZhznu51iRHMfN13MvuVHSgnAv2rAe5wf6wbWdyli2IA73cEOXzYC7n50xL92zP7dhaRnmCtMac+9mzGrmaT0vMseLt5HvrR7TfhPJZZ0V1ThjzpTq3Ntts88bbvE3vo95VQ/BnRvLGJy5wEu4mym511DrJvOJTSn43losSAe3qDLrdzUp+0H9hwFwD+ZxP94ewvkgYwXnXs3tpXx+63QhuNfToMx2TyYekatzn6ZFXc43UwO5Pl8aCcqc+4N79Jl/bYF76F0ArvdT/gErX3LmgtVs5rP1dzGP+VWRdZno4RwvbfYwPlaoz30lvzEcZ3PXcLMzenper9BJvqNlMTvArTgAZXpE5ovjSsiYI+MqwMWqtReugcvW+xqmmaZUF+57+Mxy9vmdxQrzxKMuczqrYayB0+FWtmO7N2nPOd2ewf0n04nKhLg4IcT/RP+zgRBCCCGEEEIIIYQQQgghhBBCCCHChD42EEIIIYQQQgghhBBCCCGEEEIIIUSY0McGQgghhBBCCCGEEEIIIYQQQgghhAgT+thACCGEEEIIIYQQQgghhBBCCCGEEGEiwn8taF2xKBPadPZNqPbrv8PNrdaOp/5IDucfl7fo8y6i43hCLJZpmLg23JrnJ+Aa5+O5q0pHprRz0Q0Mgmp7egbcjHqN4Jbs+QJXekxKuAdZeNsX14/AbU1QAO5r2W1w38wzx7GVawhv8JbvdUS5iCx3uhpUp/xT4J6uTgr3004A520e8x6GbRJeZNo9DK7YXXa2LHfYD1JO5Yvb/XQtXMWGbOs0C9PB7Sjfx3G80oxEmaZNB8J5JQ9k3Z5Egot4bTpc6WUP4HZlhDKfP9aD+2cPx1/pDBxDI3LtgStkR4ObkGaSS10+wY1uPAJusHnrOI471CWWpeF78Pz1E+5DgdxwCZdcgbvU9D7cBcOxsbQWrzeTtQs3Xr9jXK7sy9iyLn4HuISD28D9m5D9KvJexsgtZdrCTbc3OY6t55dQ5uWtVXDjpiSEC5leEm7SrTlwpgnj8pec3eHO3E0CV3CLN9zhzZxiS/YOhjv2i+M0NHsJuAv1LsPly+8D16yqc/yd3O6FMreaMIZYg47DRR3J8RJic7x4WHyHZtMrqIELP8ONbB2d54YT/UI573zzuA130pyE2x+XsapCKOOc7cFEw99qAPfO/HIcNzjD8xb7toT7fJpR49i/4+COV/oIt7pLTri609LDZWnIb1P7l4EyWV3C65O6dP3mnoZL0I51adqZ+cPyQRx/5rfpwHrbCkV89/O9xu/IeHS8WBe4L6FVeE8/f6ggw7HRtslduCUMXeFK9h4usv9gqB7pmN88Os9x/aXcerjlZ/vCrak+HK761t9EDVbtzbBRcBGsayx41BMqyv04cD3KvIdbe4jjLXaEd3Bl6/N6GzyYj2SqyJxvw7U8cBHij4ULaf8vnD2I7XnltjO296lRH2UKjGKf3Hx/BZxV4x7cpO3J4Pw55ZhasZkTbI7CcjVdlo3hhXWI+btHqat0jX3h0i5gvD63YT/ci+BucL6JS8MdMk2cdZtYAmV+pWZjefsyCd+3/xRcsapv4UZvZ05b3/Bl+pVmEC/BKcE8fLgMbuu1zHDdmy2He7mb4yDOm9lwQY0Xw80r5FzHzjCcI0Y92gW34NQFuHchz+Gupt0JNy3wGFzxjlPhlp9lvGyWm/lXeJHPZlzqYDVhwQZLofKHcL50W6GkyJ0XLtti9t1ddb46jgfEXIIyY+/E4C1jcB1hOpaCij7iKNyVQbvhLluH4DYsLwSXpCnnl+cW50irIV13purmxl814YJ+sk9e9+HYuF2qm+O4uV0BZZZbL+ASv+e6pFDs7XDTqkyDy36Sc+k+/x1wPYt3hpuUCir8qPYIyl7FCg1YzXVHpzpsm8P2Q7hNidmvSqyrxLp8dOYAX30YW1MaxsdK3lxPdwrg/F8vmGu2W905V1meLgE82R2onms51vZ1YK6wfm4muA8P08At65oDrogX2+C5xXm4ZkPnuti3bkGUqZOf662s112SjJO8553Tf8HZFucSk4njattr3qMXzww/vu+DurWadXxtuEd2NTf7/Fmb/aVUVPaXJNx2NN/2ONdKVj3utQx5w9zp1RLGoO7z1sBNar0Irmi61nCZknGdsvYGx218w75ROiv3kbZ047rz1fN+cA8eMJ+3p3EvyOrMdaZ935k/WJc5L+VaxvfaeAMXlK+acJ4f8yE/XKDl9jePDVAde1ZnOcN3EV7cesH5tEAtlhtZ/AacbXFd18Llbx8LY2aA87RcNj4Ov3QcHknJ8fO+wlO4f/dwHp8bjfN/7QWcYz+0qAg34vkzuAk+vIft0ueHuuxdxOzLv2U867MXLulj5lRrLeZotikLZ3Vw5sfWbObQf1eICTe9HPdr7Yd/wz2+zH12U4Zrl6JXisMND0kMV9Ks5vWEEP8T/c8GQgghhBBCCCGEEEIIIYQQQgghhAgT+thACCGEEEIIIYQQQgghhBBCCCGEEGFCHxsIIYQQQgghhBBCCCGEEEIIIYQQIkzoYwMhhBBCCCGEEEIIIYQQQgghhBBChAnLtm37v5X0gUqQLATu6NNAuA8udxh8lG5ikc5wOWL58b7H2jiO32SvgTKvY+eFG/OxENzU7DvgrIc8d93pTXD1/l0P13xwJ7iDP3PClR6xE673GDZUul0W3O2oVeBGrp0Lt2bmbTjLyuoUQy6jjGd6KGMaVuC1dkaB8228GW7I56Fwzww7wOTRXnAR+v+37vl/A9v+FJXpEfsn3F+ju8Ol7sd3FORyvQof6bztZ47jO4bj7HIl9uWaJ97yYssGQCXdz7o9j98Tzv76GM5KnALOv0cxuOi+x+GGBa2AG1skOtw/CUfCnVh2Dq5l/bVwDfrXdxxXLosiZodLm//r8v5rL/WDe9aiIVzS5xxXfZJugTsX2g/usMVzw4tu62bALW7A/vKl2RO4am3jw22vHQyXf98ruDbZEsCtKuc83r+P72Pm2uZw7XIsg/N5DWVilqQrcG4O3KT27eFKJWdsfRU9I5y9lPeY+4tujJ0P7mnkf+CirZ4FF1JjO9wv70iO4yQucSaZ9QnuRL5YcB9LHYFLOqM43LdvnNP6Ra4Mt+47K/PY/Lk4/3RzTLjKwezL8etFhUt8cwSv97Ii3JHSieG+ejyFizrX+f3n2bEXUKZEpDRwae8yP/vX3gj3KhOdac96eE0Lgivw8AjPNbFhbBMRrm0bzoe9O8xn/fLwvhdzNYK7U+473NpDPRzHn889QhnzvSlU9dqcv9fP5qkZMm2BW/KDedFgwzloS9cWcLGm8R7hSY/y/nCRctIdHJcK7nz7m3CbirSBq72Y/c0+dAkusu18B3G3X0SZJ9XGwCXMzFx4yN04cJ1cxnOs74w5hXzHw5259xzurc3xfI/hwYxzCbz9XzDGJn7pB1c5XzS4g8Gsc8p5znuM38YydSaxHokzMP6/nML5JXmPoXAeNvPPxy75UrTqUObLVrrwwmUaNDa7mrHy8B2ZbL5Ql6uxb0S+zzx86boacGP8EjqOR6Y4gzL/2CngfI6woy378BmubRHmXp+zMOb+asZx692zG9zcxOzzG1uzfgfm1oNb94Z5Zb2bnHete73gjndi8lbUcq6pT02/gjKZa3N/Itp6l39f0e0gVP+0LeC2lUkJ16YEc7Tu27fA2SuZV4YXywz3M1pY7EOD7ENwN0xtuH8918AVCKkFd8bbZW+l2zfHsX2D+cPiHYx7My2uYzeHcjSniN4YzrxZDWWfYj5ilZ7EcpHYXz4Gcn5Z6/USrlPuDaxLRdYvTeVQuMfnuR4Iifyv47hR/hQosyr9dbjvETPBPQpkTnWiPZ+rY6bpcHZXKGO+uDiX+TC8sMwouFAfLvyTGMaILl9Lw/XzY44YGol5jEcy7vVZT5w5RZGWfVHmeLrBcD3nROa1KnO/YNouKPMiB+eDeFc4f3nZ3H+JlogT9IeK7AdWtKRwQVOuwV04ynYv+NdZuNBhyeE8qjnfRd0l3F/t1rUOr/+BYyp56Gi4pwkHwmVjlzDX3Jan5V3kXpdy4YRlfYCb34RjuuhK1jt/IHOALyXYX76sZbtGScaFjMcAT6cYzfOqp2Uf3eKXH+6ey15TrRzj4K7PYX5ileD8Zc3ivLE+L/828O5DErgJEVbBPY7IvU5z7TTvu5055f0azD1ylHauUT9dYkyP9OEGrx81C1z9VXzWlfOGwSX8OQSuOKdvs6Ebnb3gz+3djAxkvzqRlfthK+4mg2txOhfczkLecP1qlIEbu4U5RdTfJrxvh6+izJXvOeBKVmFfuRS7INy6Dzy37226nYWXwFXJsRjuQ+q/4GIv5n7qm35s43gV+sMNLJoZrmIo42vh7dwzs6o594Qb+iVCmdUfx8L1OsM2LtGRa9Nd5/m3gjl5+beMroWZs838sBAu+M4BOCHE/0b/s4EQQgghhBBCCCGEEEIIIYQQQgghwoQ+NhBCCCGEEEIIIYQQQgghhBBCCCFEmNDHBkIIIYQQQgghhBBCCCGEEEIIIYQIE/rYQAghhBBCCCGEEEIIIYQQQgghhBBhIsJ/LbjP/gX3ul5EuFTGghs75RzcwT7P4D7/aAxnlasJFyX7Tsex7fIU1gTes0a3hC4FqezOT+BaL/WFq/a6D5xvqhtwRe6zLpMaFIeLGasw3OuKrF+ovR1ufG9/OC8rBZydNJPjuODQDygTj7c0ezrYcDXmeME18ioHV3c5z932Dxveay7va/d3qUx4UakbVJYnB+GWn+sBFz//HLi3t0vCpX1SH67be7bXr7w+juNPryKjzBpv9r3j0/fBTSjPvvxs+ES4lxXY+FkmJYd7PzItXAv7PlwdGGOGfGM/iNAhC9ykldfhHo7MBxcrwXPe5N9kjsN7LTugyF+Gz7rE8D2MqMMYldTik4V4nIJLeewwnF+VK3BmJ1V4MbV+F7h9Z/vCBddZCJen8Gu47XZGuLPGky4O6zK4yFLH8UmTG2WyNuA7erD+JtzygB9w14Inw11vsBRu0aPMcM9vF4W7/Ow8XK7R7KN2pAJwfw3kuf+YqHBfVnEePt9wPdzd1c0cx008GOdPtswD13galKm7h3PV+W+z4eLZ4+HeRuAENs5lznUZauFGspr+cGl7NITL15zPHHMf+1C9FnyYw996w1kV+a3ntXbOxsnaDkWMl3UV7or9Bi7tMeY7p4rVhms7YyPc8JSb4ZpYfNYKwey3tidjdZJFvN7a+VHgWocyLyxmBcKtvLgKbn6A01nT2edtH16rzG4o0zlpZ7gvvn/B3Q7+Bnf8yQC4n9Nb8ybTgunCkbPHY8GdHH+HBTP/AzW5KeN6gzon4O7sZX5z0Yvx6kdq55j5sphzqqdpBJfl6mq4Dp6V4SKmXQr3zuL4W1Gb+U3ARro9QW/hgga75EYhHINda7yE67+FefNBt+6Rg+f6/Zby1PnARLp5s5xwhYOi8fqTBkJtt6vCZY/ZC86qyvhgb+Ut/mSwt13mHqslc3Vjs71+mR1wH4wPXKLnieESrwuCa5Kpp+N4kEmPMgPNaVZtNmNY6vWF4CLe5cPaoyfBWaNYrkKKWXBdAhfBZYzAsXbA4rOWft8TzqRkPmwfP8Nyt75C1R1xxXFc9G/2qU01osPF6PYZbu/sMnD3UvB6vaqEwP11m7nspe3Mof8kzZdwjs4+jLlMAYvr+as2Y+T97ZzLzqRyyW9+sa0rxYrhOG76KTvKPOqzF+5i+aZwHoOZA4RE5Hvz9OAckdnwHX2x9sBtasPrVV7A8VK0D+c5M5L7Yz5p18HdX5KL5/pdg7LMJsfxajMdZQp/4lql3yY+w1fDNvmnDp+rd88ZcGu5zWAaZGU+YPxu0YUTdqItcNZt5ieNpvOd+8xj3mh9L8h7ROS+nnnFeaNeIuf6uZsP42iqeswtmTEbk2MX71l8/Ca4eC5T2hVzEm5KkmZwnaq34PXGukycn+9CJfNJDTf3J+eSJQ3Wwnk24fw3znLGlZ2G+8FZu3If9lb3rHDPunONZFxyrGuWN9zpaVx3LzxRA26R2cILhhO2fQzuSPlacClc1vjz/bmf0/ZUS7gOHRhzByffwro8CHVevwfzmMAih+CS1OJ8U3gR1wIJKv8Nd/wWc+iIt5bCLZ7Pvc7VJbg/1PrqVLgNk9nnTWfG78AIAXC2j0tnm821Yu8MJRzH6Wt9RJlmidm/l41hnE9whXu9+V8NhotaiPPXY99/4QYvdNmQX0AVXpQpw/zr4L1QuHgBzHdSu4wDs/MnVMnKvN4xl3+ne/67M0Y2W87LV83Pd7TAXgyXhE1vul7kPrvJyD4fOJKx+stA5k9RI3OvxQ5mHDAHuP+7vMgjuCfnX8B9jMK1kN8Q9me7emmnyNceZU4eHQu3NhfXCxO/co288TLnSDvfBrjc67heb72F6w8hRNjQ/2wghBBCCCGEEEIIIYQQQgghhBBCiDChjw2EEEIIIYQQQgghhBBCCCGEEEIIESb0sYEQQgghhBBCCCGEEEIIIYQQQgghwoQ+NhBCCCGEEEIIIYQQQgghhBBCCCFEmLBs27b/W9FCMLZ9Gs5jHC9n75rOy9lnoRJvYbGXsXPCXbR6OY6jJx2CMtmevoP7YWbDTdwRDS7zxyRwFZvfgptsh8D1sBbBmR3t6CrvhvpspYH7FEgXubcFd24q270B72pCzW/nHuZ53xvz+h0HZoKb3WE83PRHD+FOne0Kt666Sz/xTQFnjJ+LCydmN4dKNvw9XO03S+AeBcSH28ruYp6emgiXvExJuLnXfzmO22cpiDK2xfd25U5buKFP/4VrkawcXKsMa+A+zhkFF+fyALh3q/hcplhvqEd7WCy1zfFnm1A4L/MS7s3dT3BR0sd0HFfKwXumX8X+6JuZ7TkhK8+1bz+l7B4BavCEhHBXC56E23qqMK8XXnTgM1tzWWxIxatw3RPwmaNlYtx4dT8xXKJ57FeWcY61rb3Yf6rtj8PKXePYM2VuQL2ex77inSQ63WjOJT5vEvAeS/+GsvNz7jvlEnMX/MN2jzuatxgf4vJsV15DFUjkvN7ZdD9QpsE39se1kT7CrUzsCVfhPGN/nEp8P+ZsNqjC9jW4EzwzHKkFU/FdabhT8TrDPQnm1WIcpPtQju/8ieE7yd3kt7FR3x9lNlWLDPdqWQm4znNywdlnRsIt3826Le8LZc5dHQz39fpwuNBsLjEkUU+6Fxvg/A1jaSyXb2KfGU6miUwW5/W/XkeZi1F5rUQWn/8yjDEVO1aCKzR7Hdzawcwpxw5fDjfbbupyl3Ak9Xy6Xy65qsXWiHuQefmb6fvhso7YDBcSaxZcQcsZd5Z4uvShE7HgZp++DTerx0a4GxmnwUUqdQiu9jRO8OsjlIXrZvisk8Y3gjN9OL9Xfcs+7huH+c1Al3bPGDk33N83Uzvr8ZzXt5sEwIV4cQx5P+Qc/u4Ac8OUpTnnxD7JuB7fJZW5SBVuWJvo7Dot4RbYS+FW1+S5h1Pt4j1sxomASVHgblulHMc5PLaxbuyixiTiemNS+l5wPb9xfjkVhWuLXTafoaQv84BenTiG7o4vCvfTegNXtQr3D7btTAlncalo8v5Dd/63tG/od2+UmeDLNfa3zN/gAm+VgPMyfF+JWA3zegHd0ed0xYa6nBxOZK3PWLpsfQq4XC5r7d7/8r1NyH0KznJZP9n9mRxFfevlOC4cn3Nl3G9f4BbfYbkh15lvdqvIsRcvalw448nct1yIF9zRLS6nvuKzZp7CcufvxIbz+8U1a8pskeDe1eP14kzM5zi+Vus8yvSekx1ub5IrvJgbhdlP5p5k3dragXD+X9kmsaL+x63F/wOCTH64CFvPwY2uHhMu+yjGkionRsA9WtMKLlXu6nAZHzoXBBut7yiTef9dOFOW8bFNdPbRBZ+5l7ooqADcZC9e79ahJ3ABz5gXRPrInO1ed66RLtsZ4dx6weOWtP0ms5wV87c+WYhrmgc5L8Glnl2ZF/vOZzWXuS/QpTDn0qZtudcdPQ1zu/R9/lyfN5+ZoLxIxA22vz/8hMvhy7E/iNOnscbuozzCnDTxGmfsexE1Lc/r73KDa2xT274Hl/0094v6j+e6uM4l5jZBu8bC3czEdspncW0RapWAO1CX+U4ZhgZT2mXP++A2rrd6vHeuyybPY15tem6HepWW+7AJuSVlbh7geInV0QcujsX11uv+3PdKmvjP9XmrQQs4eynjvEnFd7k7Letd6SgT7uQdGeciMi0yGZ91cBwvzsO/BcUu+wpuTAjbeUAE/t3HLODfVnp/5d5cgOG4epepPFzT8syFKx3jWLOL54BbEo3xon5OxoEKRy7AcbYyJsAccBxH3FYGZeYlzwN3uSWvX+lyCdaj81E4a6ZLRdbcgdqVIQZcxRwu+7BCiP+J/mcDIYQQQgghhBBCCCGEEEIIIYQQQoQJfWwghBBCCCGEEEIIIYQQQgghhBBCiDChjw2EEEIIIYQQQgghhBBCCCGEEEIIESb0sYEQQgghhBBCCCGEEEIIIYQQQgghwkSE/1zSOgUVMPItXIGBXeGuHD0Al7N4A1Ymz2C4kDcsdz5XdsfxkdOeKDPMXITrbSy476N47jOX68Wr1Raue5z5cN7+LJcueju455cqwn092QYuRZJ/4ezTheCq2Hy2b7OD4Xy373Uc79/L84z9FapHpJ1whTvFhjthV4GLmjoEbq31lPd95EeXkirc6LQA6mn7CSw3Nz5dOba9yVgPKtncAJa7Q9Uu2PnO25Vi21cd+wEuTe0bcE+GJIJLf3oN3IfVrMfhBkPgSn4YCGd/TQ13ryyvd+HNHMqLvaCS//sdLqg2n8Oky0NnN3ccHjArUOQKzzI59zCWmfLTXEomo4pHtTqCDTeo9HGX6xV2ceHEbNbx9pwNcDk8MsPlvT0SrnT6MXC57V9wr0OD4OygiI7jp41RxMSbyH47v0FpuJR9GOf2vG8JdzH1brgNoewH9ojirMyv6XTjrkHN8IgMF8FmDOmTvimc9/WJcNUOjoM708k5X4+Mw3sOPPsZ7lYtxq18d2vBxf7iCzfvjA9cic5D4dLX2AVntlSiCy+mco5d94lzVnRzEy5fXcave3/xFrEt9r8jhmPt7h1nrpRuu1s8W0KXNDdU5/HsP8v689SlFaLB1ajAHOBAszdw41eUgLOK87kWHKXzMZPgFpnTcKHt2aC+86CMsQc5DqfG47e0fYM5t6QwnNMWthvO689i/7Znse2MF68XuX0zljN8P+FKHeal70bwPR19cguuUrpRcNYSf7joSRl3TrdgVe7YsRzHF+oxHobcZN0S7RoPl+nzILiiLjlVyTTZ4PpZj+BGB++H6+Qynu19fNbux5hrTLnnD/fwPvPmhvcZJ3f/YBvMbPbecWxtYG647dd6uJIrmHt6fmc/jVU6FG55Ir6f5q+gzAtOMca4DJnwwh7ltt5hm55NeATO8xvz6zNfOW89cbnvBsMx1DjGNsexx0T2vZE968ANuuSSe5R/DzXAdxZcVrsM3IWKXNvlZbptLu8fBnc9KfPAE804T+7ayOtZb5PAxYj7Eq5CLfa/pYed76xSC77Xvn+/hst7l4Hg/K6qrFtF7jvYiTbBebThGmRXFj+4YkOhwo3rB73hAuzHcFZUtmG8WpngDn/jXlCkXzz3q9kL9yWu8729K8bz4kdJBRfa9CHc6uWL4c687QhnRWYu89RlPN53C0yPY0BNGMDnb3ydiVXdyMylC6yJxHu8ZBvEycCYZDoNdRzeu1oZRfam5no/1MyGm+DXCa7fWt4yxwnOEYM8tsA9s7vwZJdHCC8C4+SEG/yeud9Yi3syQztz0ynP7tZwy5v78b7t9sDdNLcdxxlHcm/kQNOMcEljMe4FcwoyB88x72jtsrPb2C7FulVaCHf3x0c4OwX3U40pCvPkPV96mTg8s9FS9vnjX3iuPd+Zy3w+fQllulXi89eOxgnMjsyxUc1lq8V+xTX2tFQc82WXvuDJfajCC7sc9yYDv2+GOxCQC64zw6bpUKk83D/7ysGNOtQK7rn5LV6/SYMyT2ZzPk3+bgBcH8O1+MXnHEOPPbmv5PWkA9zJpYz976dwDyX0YVq4rSYKXP/1J+BuhbIjnPjKPv/Yaxvc6HlNHMexhx5DmXcLOZZ3r+H1/+oJZbJxOWOCuew2Vi+Ox6TlOG+4bIuEG+drcNI6M5H94MZP7le1acv2GuIyaQ2Lxr/L2OkKwL2Yc9Vx3LhsOpSxRnBtenBWEbhCNvOdSRXvwRXI4nKPCcz7S3/lXFLB4n7O+uwP4Fp6JIf79qUkXItfdeFujOZcGuDJdW3wNWdeXmAX32sU/4Z0MMZU/Oso3JV8KVjQ9oPKaS3j9QzXPX8ytxHi/4vofzYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBP62EAIIYQQQgghhBBCCCGEEEIIIYQQYcLll73cKXKFv29zLMciuOS7+ZvmF9tmgYtn87eOs7flfT3mVmNdljh/g+74Jf7gVMEC/I7iSyT+0MrbX/zN142Gv9kVEm8B3O0F8+HeuPwuU9qTUGZzZf7WeHOX31G8vMrlB6Crsc5FPrn8ZuJLvt4pv/106/A9Y1GmZBT+lvC+SPwdJPsCf5/OCuLvp6V5wd8WvP6Tv6k3MT3b0+MJf4s0vOh3gr+vOLDlBbiXXfgbSelm8Pca+x/kb5SN/cZ3FJiJv6Ucc5/z9+v9D7n8xr3L7wi7/biQvY6/EfVzE393bFOTpHB1avD3m0LL3oeLcHk3XIjh7zpO+MB+4OXP39t7Eou/GZghUg64O7047mOPc7bLxxNsp2/1asJl60F3rfI5OCuYz+r9bjDcTzsQ7rPl8iPqI/rRhRP30jGWbnjN3/wMSMM2rPydv2P2wfC3nV+34+8G23n525lXCzvjXI4lnDP6leWP0tX4kBDOrGVMz5Gav9FqW/wd2M7P+N52P+DvzW90+bHmDhv5u+ATbnGiSxrA2Le5J39j9FyWlXAx7vflPa4538/wxM9RZuAD9tvr7+rDRe3P3xW0Ej6Fs8s/g9u/Oy5crNDocMb4u7jwoVC0vHCruq+CsxdwHIRUp1sdl2Ojsct9zxiO83Txnb8daffmb8VavTn/B5iWcN+LRYSLnJ/zUkB1zkF5XH6TzrqSjHIkfzMwZCB/O7SV6Q3XxiSAs0vmgSt9hL/VF912SV2rrHMcdr3E3+LemOkA3PJ6Z1mP29t5fbsiXQr+pnXJYP6Gabm5Lg3KcBauWOOYW9oFWc+rZ/iMy9Lkh2vfPTdchsbr4W7P3sHKTHW+qzuXWeTUhqVw/9qMfeU4nM23zD/geizzhMtWl79jadZFhTpRlHlzlrg94EpUus7rxaoDtXBvIbgqC5inxb7rB9dyhfN3MW3DMWQZxvAvPRjjcsblnFijLn93c27Va3A/ejEG/Srp8nuXF13iSDhhXeLvhdqd2TZLXnMtFuLj8vuoW/l7wnYN/j5xY5t5RSfPoc7r9/RHmRyH4vH6lYbA3SzONcNPl35wy89ljbCbucz5LnPh9s7gqU/fM1cfmS49XNtL7C9H+02F83eZKYd3fcsbX3I+xzSrCYrUNg3ggm3mWaZzZ6gR5WfCWUuY4NlbuT9h+JPnfxTrvcs8a7HPG5ux/43FNaDJxHLWl9hwd4ZxH+GIj/NdludrMz1O8feK769guYfLOX91fctf8Z07hGM0/pxGcIl/foF72SgDnJ2Cv2V+/SnvO+s+f9M+Hn/63fTq5pLjMYQYO51zH+Udt2RMWm+u32ono2tX7DZc63rs8xPNarihdg1ej1UxS11ceBHlHedO27j8pnR19uUhU7nuMt25fhy2ijlF/JARcKPnOmPV3Ue8vDWQ8bZPDq6nejTbCZfl7hS46g+KwxX5yDxuaCjng6EeQ+E23+H81c3w3CcpeY+ara7AbXH5d27T+DPdxm71wXEc7RdzsaXeX+G6RvOGy5FyJNzGtwPhrH10A36ycpdu+8BlMsztwotaZ2/CbfBIC/fFYp/PFsp9uMHTuWY7cXQLXPVqzAEe+cVxHDdx2QMv3py5e9A31q1+Pfaz+QPpOt1+AWdbV3hjlz2ZIe+Zpzc8wvXMiRzL4W6V5fWuntkLlzUH90JSnijNcrav4/hDdLf51mWPKxdVp4ncp7EeZITz2PQNzs7OZzVbH9D9QfIWPATnsiQ0g2NfhbvehHvKWVJwUvX15b5mynzMH64nTOI4jhnKfb5MdTjv9hr8Du6aD2NLga9855YXlLGXbOb1qrGffarBnHzGlulwx33/gVv1iwmZT+tycAFtuN/0eT7r0nDPK8fxX6nLoky9iz/hvDjMzHhWwxw3R+Fsw3VojQX8+4bVbozLuS77OUKI/4n+ZwMhhBBCCCGEEEIIIYQQQgghhBBChAl9bCCEEEIIIYQQQgghhBBCCCGEEEKIMKGPDYQQQgghhBBCCCGEEEIIIYQQQggRJvSxgRBCCCGEEEIIIYQQQgghhBBCCCHChGXbtv2fCloWXPPDaeFSvu8CN7TOBbhrBQvBfa32DK5Z//FwBzyCHMe/Hm1Ameur6sL1+BAC93xSXjjz8hSUfdgbrk7TcXCbB/eBCx12Hu6v0KtwWXO0hltZ/hfcxYmsiwmiMl58Z8ae5jyu0R1FYsQ7DbdlXj64yQ2Sw91c+RTuTsRQVs3OAjfF2gfXzU4MF15YccrBVR7NOu6IeoiueSu4nUGP4SoE8L5lHlaA+zdDRMdxPg8O20pBNeDsk6zH4+LpWG4Ex9ntQYFwK8fVgzvYrzPc3bgz4fwjQJloLdg3/Ed7wrl175/P6WLsYrvYHvOdIsVglCmfjXFgX/yRcDvNZd7TbgFX+C+XsbdkKlSTIt3gVhz/TyH5/wTLYgy2tySDSzWadXwctSXPPbiEN+nfFCrFpR08d8hHx/GTvGxTv6ib4Gb41oQrMT8GXLP+X+CSN+c9fNpBmdOx2E6fM/eHa/rPIrjmjffAFZrpA5coYUm4TQ+iwdUu+QNuScYVjuOWj6ajzPBJy+AG784IN3VRebhu1ffDufXaV9k5R9pV1sElGnnR5ezwwS23Sf4kJtzqfB/hWr3m2LjRinNguZ4H4AJdYvOtNk8cx57mDc97cBtu24TRcN2/B8NdWpUNbs789XDtqzDiFgh+B5d1Gdvu7sBdcBVNLDj7XgG4AUPZi67shTIDPuyGO9CorOP4bVnOt6NbvoQ7YkrAsYcaMziEdTs3wBfu9CyOx40u83zboD8X540xJuJBvruSZb7D7bMjwwWU4bl9DvJ5SnI5YNpN53og5tk8juM7TFGMHfEtXAnveHBHdsyD672JQbzpwOJw39Kzbutstsn0Ry71S9UCznrFGBuUkOfe+sm2ex+Zz1bqMsdgqVfOAfKrEvPWAf534KrEiA13zdSBy2Y3hnu1vi3c+HrsE1Mslz7+B7u9VYt1XLiZ/eWqzefLtIDnRnfp30E1+YCb1/DcLdZkx3GCxHy3r54zrq+PkguufoZrcAUuMv6ffAFlvKMxzwisdBguaGp8uIgJOTa+3mVdom4ZAtfhENcceV5NhDv2PArcskjDfjN+KNMwEeeIQK8YcBsjc61ibnPN7pMwAdzh18wDC1Tm/Gx2ZKALJ/K2Zd8LWnAG7tyU/HBfi06Fi52nG1w8bpmYGAW5B/PYWu44Ds7MvGjMjWpwSVJxTDXNwOfKuYtr1hQmIty/e7fBHf/G+2apXRYuZq6qcKcvMRAUC+Y6J8hrBZz14wNc4qRcZ774kN1xPK8Zx0WWZcPhava+AXdoItdbmUNdAjNDkomaim3y9Tvfo/lvW4v/J0Sd9A9cgV6/xwxjJl7lpsTa7OxXITZzul8PuA83Iy1jrpnz2nFoF0mKIhGz8J6BZi1cD+sk3FTfqXCh386yHlZlqE03GdPrZOYkEXsP18pTi+2Ee1CM+cPQSz/hvv3L6tWKkRvuXAnnurCQSx99Y7iOClnC3OZSas4tHsXnwCUN7gXX6BznucGzoYzvyj+4d2PYh9Jb7KN3f/F95PVuDnfeHgOXJTJzhe8vksA9ivnbuNrE/YJ8tbnGrGa49zfQ9oKrmpI56aBPzIvy3+e+hx1/LFw9l7z3YQgntVF9+XeLSN24jj2ZmOuNHp50vvyThEn+227nE5c9uRPLGEMKZ/kGdyE354g8+9/znmUKww0ZcBcu/RieW9jmWAsval9kn0+Yh/twxQ//DVdvLBdiifYegWuWk+vElS953+eTnMf2cpdYsI/9cU5O5g5/J+bftKpd4t77v0+Y7zQ8xHxndc+FcLFucHL/dZT7i7N3dILLWbQb3KBq0+C2XWUbBGdl23lVcZbz2cV9r2/2UrhncaPCJc/A+cBrH/ttsDfjlts/v/6PfyIVQvw/0P9sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJiI8F8LRrVtuGWWBXdozw0424yGs7Z1hxsZj+7hgFFwQ+1Qx3HbL+dRpt6+nXA1jhaE+7LlElyOGnPhrKZ8VnO0D11NlrNjs5j1N525/ovnDuvNc4ewXDIvvp8nlXmLucZ543ZbuqJMdisf3F0q83Ip3+vDtdfgAk0TONsMgYttqsF1Mxd543AiWYP9cI/asp2NVRrq+MtvcAMSxeU9VraHi3N3L9z7cVscx+VCk6HMnAj14CqMbgXXsPZ4OONdFKrbXB+4Hx2ewJ3deAyufp7ZcGu7doKzRveHK2Y4htaZALg3Sb3gzCcqa31bx7FdoC3K7B7M8zwH8fnPjfgJV+wzz/VbnAou+RL2iZcnXPrTn+SzN1Ryn0C4pFXZR0+NWQp3+ntkuEJrVsK9fRwMF8/797Zhv3iXchjctKEsN6BmJrj3DK3GM2QiXLvt9+HGtRgE133PDLjtOSLBbYv7DK5K5yRweW8mgDtXJx7cWYsxN//Qpo7jSt7HUWbwcvZl05dtl/ke42DO63xfVnZPOPsKb7HGug7XcCTLhRcFb/AdnR7CQFJ4+kK42Fdd3uWiKXAlLOY2gw1j6aOYzvZPWQdFjJWmHVyxRvvgLhZnbMlW6Cbc87ZP4aKU+Ar3/aFLXH7Kfpvc9IXL9O4M3LiyjOkTUvMeU+MPhHv5aQTcjzXO9+iz+h3KjE8TAufx8BTcCVMIbqX9D5zZ+R1qwVs+Q1/fNzz3D3O0dE+4Qnl8WdBiP7ps6Lzb8NQaszjnHRydB25mHmeb3fJ0mcj9MkNde/Iebn0N1m2hiQg3IZTzmpX9KO+bsDGU/Xoc3MOJXHMYX8bYdR0fw+ULrQpXKkZWuGHfKsBFXlXecZzN3oUyRWNXguvoERPOLygO3O5ETeFMFOZ8Ji9Vp4kcC2lc+k540X0aXesGjKfGygl1z3CeTbs5EU+tycvFv+zyzI+dbfPK5vxpBX3keXUuQ1U5HBWuts22L1+SeWnQkYe8r5kD9zRXbbglG1mu5dctcGtXloFrMHAWnJ2DnehI7AZw5rbz2ezXLLL1zQa4Gs855yZI1BrO9y/mt81es40LjJ4JZw3IAPcns/zzHeisCFyzPuvOWJ3adIPbmZj96mZBPmG81YwvtvmtXW9wsFiFoYx53BFqzJDUcF2s6HBFmL6acxW4Fi1TYQVc8JFGcBkvM5e+mY73yHV/KpzHTbZdrPdsuz0Ny8JV2+M8bjuB97w0sCTc24lb6Uw0uOzPmKPlSF4A7mtdjpfzGzhHuEwH4Ubcnl3gDhzOAed1jrF0bEPG9O7f/OBmPOGa0jbZ4UrtW+4U7bl3M8glQix1matOmnO8Zz72eesp83fbcH6p3WQMXN4HneEe3WGe23R3LLjG5ZjvjAlln5/H9N2cveAPd/y3NX+dtkwyby7gHkOWaRwcX69xbR8jIfffFleia7nQZf+37P+f7d2YljDruqyFyzFqC9y5tHXhdhqOg7QnGPu6d2W+6bU8l+M4URDHxdek6+HK1GwIZ6YHQe3wY7HlPnwf7eJzoz2i4f5T4Gu+XytkN9y9jjHg1qTxh0v/i9eL6pLK2RZzG8t2bkzV+X3ONMYU/sEB9OwV40re86Fwqc/z35g+Hc+2S7+fz1D413A4Y1wCVTjRLxHrnc9wX3jmX/xbRVqXte69bSfgrGrMY86a3LxvKue6Lue/KGKuDOCeh5lWju6fuzy3Xn24xaX497bEx7jWDb7OPSm/D/zbyp2WieHyp2M8fOjBuiSxmIRbLdmHbu9kf0m2y7mJ3sBegzIeMZbC1fDn+Gm7jXvYJb6wn1zi9r7ZyeWMMc9d+nwSlz8YCCH+J/qfDYQQQgghhBBCCCGEEEIIIYQQQggRJvSxgRBCCCGEEEIIIYQQQgghhBBCCCHChD42EEIIIYQQQgghhBBCCCGEEEIIIUSY0McGQgghhBBCCCGEEEIIIYQQQgghhAgTEf5rwa+b+V3C29Yl4Z7P/wuuXvlPvGDXFVDXhw6C+zx0FVy0mZbj2NpXiecd/wBXs1wwXMdk/eBilzkKV8g0hmt1wIJLHOsz3MevP+GMlQCqj+kEF9l0gRvRk89rDY3Ie5gqMK1+O3ZpJhMrwWA436nD4S60bgL3bONquP37xsHVsO7CFfM6z8r8QQJnlYDLOqsZ3LFCheFWXogNVyWoD1xAhPpwMWv6wPX/4BxrzeN4o0x68wAue5EQuIp9q8ENMS/g9jbpDXdv0EK4RC+uwa2wN8OZLS+h7NEJ4Z5eXgRXZLEX3GPjy3sk/A41a4VznMZrYqPM20eTWLemv+Can+OYvxqDz78l0VO47EnZd648fgJnTHIXF05ELwU1zZ4G18yzPNzpkv3hau3ZBGffjAuX3mMZy310jitreQaUubgWynh1XAx3p9oCuBIP48HdtwLh5rf1h5v4vC5cxEFT4IIM+1q6fUfgUpxNCvd8kCfc01DOYUeDc8LZnrMcx1aTrCjzwcoDF+vgQLiUL1rAjfrI+Nawiz9cpIlQJqAT5/4/SdosAXBrXN5bivGH4P5tkAuu+MHucBFMDJc7M1dIWcd5X6sS4429ay5cuZG8+v53aeEmd4oD5+vJ8X2sNJ8r02HG4LhmD9zX+GPgvO48hJvWIh3cwUzMKZu0jw63IhaUMSlmOI/f8TzzsBuU/W0qXLPInL8aRawBN67kcrhUPhXh+hrGmj9NcYtzHmdPYyLbMSkt5vlBoybDRXx6Eq70j5twdogzFj334j2TNovBegRtgGJGZUzEfZHhbnc7B5etWHG4Tl+2w+VZzRz8Qi/Oa6EfM8Pl6DAM7prZAWf3Y7401HsG3OkFFRzHMw3rFuMn71kolHl+8ki34eYGcw5vZ7hwuLLkPdzeLmz3NDDhxxZOs8ZORjfIVIZL+7IbXOaEj+BahyyBW+DB9ZMxzhzRMqxIs3tcE/vVZj8LjsvrH6rDNXv+lYyv+xdnZ9W+F4XqXGc2XL/LUeB6P+wBt3HaUrgK67nO8e/O+eRdpwZwC7x+mydzc570DG0EV2fTKLg82crBjWrPNcj8JRx78wcwz7eZOvxRrFwp4EoaP7iDhpPqWpt7NzOica8hngmCsxtzz8QER3McMgIbM9lmjttjD3Phros59pLtLwhXbMcPuPw9XOqb/R1cy+ZcD0w3NeFeJfwIN+okc+QK8bhgGd07BVzUmaPhHtnO9VViwz5funhbODOS+zQlXfLb41cYq5tn5pzut6E9HGcvY/K6uPBivGcJuBpJuOcUoRH7sn3nCJwVhblkzAEuN17ADOpQ62zOa7mcZlvr4O7YMeDq+7NV/YYwj2u50u0mnJ+tDZznqqWpATfeh7UukZ7zS9cUoXDTL6WHe7XzDutiMc+yZ3R0lunMOSjBeChj5eZ73bniXxZs5A+V93FfOP+rzGNjuGzr/lGGbIQqN4Nrtg7r+c6/5+OcXeVGVbh7pTlH7PLnQFg519nnhzVk33tiesI9bN0QzmKaYDyqc/8lZgquBY7d5b7SlkzsMDPa8mXap5mPNbrPfOd6Ft432+ZMcCExue4J9NgG99w4195JpudGmbk7uXczaMdhuAizuTc9ugPXFXVvQRkrmGO5kxfjwCyY8CNfQtanqC/f24uj3Hu9l5QxPdhwz6RsXP49KN+756xMQeea6Eor1i10G+ddj5n8u0D5Qyy39yJv2eo4Y9WXiuzfXiHMY8x+xkiuLIzxNcy9qo0vBHfzNc9tFD8HXIYEXeGe2H87jq0v3EPbw+0ycy36ELgOS7lPfORvvosb69jG0Uox739V/Pe/mhmTkNtZQoj/B/qfDYQQQgghhBBCCCGEEEIIIYQQQggRJvSxgRBCCCGEEEIIIYQQQgghhBBCCCHChD42EEIIIYQQQgghhBBCCCGEEEIIIUSY0McGQgghhBBCCCGEEEIIIYQQQgghhAgTEf5rwZ+9H8PdeZACrrdVAc7X8xNcnWW8xznvEnDRfCqxYJcujsNb9ioUiWPlgSu0zxNujJkAd79KEFzHQC+4sUd/sG4jokPZg6bDFchqw40qfg5uZB+2p9fAUN532t9QxQ6w2PG0luN4ayXW48h8tsmaAAsuvcVz+5hucEe3s77XKqaH27/7EJwxZVxc+HBl+2i4PnkLwU1ulwju7f1/4YrlTwpnv80M9/5pFrhVcWI6joMvx0KZx8v84SoUYZ/P3RzKTFnGug3PFAPOXnsXLtnaNHBPG9WEs0p94PVcXnm2am/hTkcrBZcpOcffoCdj4Dq1eeY4XrKa9fDf2gPuS582cMuqfYGL0Jdj3vK/Apev7hw4E599zLynCjeiX4fKZH7BfXUJQbX28VkCPJrB5SiVEa5737/g0i5xxpzbtXnPDInYv5e2XwdX6NduOHsN+1nKkYxzdqGpvHGph3S+G6Dqzv4Gl7kZv/HbXJKXszbPh9t1KRhufZ1XcOm/rXYcfzg4G2Vir+A9V0boCffYxIQbXZMx72Z71qPXgca8yaxRdDML0oUTE0wLuOFT2a+6xs0Bd816DmezC5lYwf5wdwpxgn5yy3ly8V1lUaZp/H1wNd+w3L4tJ+F698gOV29WAbgUA0PgvA0f7FN05gBLv0yEu5qeuY3VLCvcIy/mO2mnvIYLbsa69I6R0nF8Oh6DVJ5BHI9Rgpg/fu1YA86y/oFbv28A62ZdhjMmnYu75+LCjyCb746ZrzFlTD+47/fGwf0dpR7clz3l4UKjM4ewvvRxHNtxWDd7KPPS9DvqwHUJ5fXTpvOHy8RwZV7mvQKX+AzjVajNudyKw/5cLhbzwGvTmGuNfc3n9ejLecIewD7dP285x/HGIShiUvwYDHfSJbU+vr89XLGTAXBpih6BKz3CBy7u33yuP8llw1hnVp2GGvmUxWqNYMNmfpcazi96K7hHK5kjpvrlzLUWx2dbtXxbEa7JjGdwGWcdhhuavTRcqDdzigODIsHVMEvhun7dCNf30064E1/HwxUZyfVz56qMDTHjMXezK7DtAhI683DvAjFQpqLNfYfYFp/fjBkINTkj4/VX0x3OO9JXuLWneIsGXDaGI9y7mX72AlzWfMz9U1st4XLcXgJXIDgZXAubfWOZVdRxHGr1R5l5cZh7HWRXMT5NOfaWR2bf+2sJ62HHYV++ULM33JILieFmTjsKd7Er67cs7ne4Tgvrsy7N6sJdHc/Y/8rq4Dh+YZi3WFMawF2+Hg8umMsBUyj1FLgT65jfjF/C9zNrMtfKf5KsIVx3eJV1ycuKV4WyG3Fi/Go+wkWt1w7Oaj2X97X8HcefmFqbX5eYv9cwfIYFdf3hRs7nOy/VjnsjVu2fcK/PXISLnzUXK3jdZVOiJHOK6fOzweW0fFmXStxTsFtV4z26OPeR7K6MyyGxmEd/ScGAGzM796Q+Puc6ImuSvKxHqm1QL9ZxrZzI9OK54YRd3B+u7FTObSl/cC72rcsYmfEi85F0iaLABQxhO4z54fzTwr1Q7lvkuXcf7mKncnChv5gvp1jHvRbLZX0a/HoGXIT4vO9C4w23KSdzkTXpXDZqJvO+L0LZdtYU7h3arXfBxbVKOM+zp6FM+1hd4PLcZT3qNToPV78g5/nLmbn/OeZwVLhZPgvgTMCfy/HvG947zVO2gxWrA1yi79wAexllGFzBCkngMgRyT+JZsLMudlHOu+2Lca4/3JLPMPIRn2FTT47beoYxM8purhHtyy7/rrgIY5qVmmv4H0HMezs8Zp0/TmKdfxqXOffiHqghEfI5jl9M5z5VitArcInnuezXHnsE92J5Mbi5FznmhxXkHOnrtikihAgT+p8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBMR/mtB74dZ4eLcbgB3IKQXnP2D13sZxYJLTGVMEhvq4fMYjuOMphzKpF/SFi7VPwvhWv7L6y+97cV6BAZD+bbns0Yxt+Fqdu0CdzrkMO8xvSTrYv0L9/jHCbi2l7fArWkzGy5J3D6O4zPHZqBML78+cPd2JYerO5AvbMwItufqAix3cheUWWdVg+thyrBgOOFTsyDcsuB3LHgvLlTF+ZXgAuKzbYzPBKiSIePgyg556Dhuk7MDrzVuPpS9OgSuaBue+9Bm/7bfbYMbEP883Pji7I+BPT7BHagQxHv0Zt8YtJ3tlNHeynPHu4yhPv2hRsR2jufuAazHlpKsR9K5oXAzzFS44Ku1WY+SHMvW3Hxw8z7wvpZx6SfhRL3PjKUjWuyDs71Z71iJGOi/ds8I1+sXv3F7sWci3LwKznaI5jI/zLW/wbWv95gFl0WCqtSb42VbWZ5qHWlDGac7lN/1p3Bbzr+ECzEr4ZKfGAaXr+JfcEVar4DrvzURq2dmOo47L+mGMibwLVRCOwbLubT7y2g94B6W55xb3xyC2+xywZp/sM/Hs5fAzbTozFbWu8hh1rtLaZ66zn8uXPpzXVnw3TTHYRuL/XapYT/r8MgPzqrJeaRfpDdwS36xGrOXelLO4LNGuP0cLlPiAzz37QCoFoZ9PpUZBGfv6QgX6MkcLaJxjvtmLv3s5gj2+XHD6dLOZb5zbinjfN4WJ+EqWlNd6lYALhAmfGnF8GLOjac7EIdzeYR07Auz33G+bHugHpy1ivFqQ9V4juOStfegzJUBjeAqjfoJ12flcbifk4fAVay6Hy6h1QnunOclXq/9PTg7Ocf4mfds0HIW56fWy1mXvhd4vZefPsLdPOg891hpzk19mOabjzHprnscgxv+/W+40iYhT85AlWR5Uji7xzMWDCeiJz4DV59TtCnNZjZNYzFeVXS5x3qb8Tne7Flwb306O45b9t2MMluHxYBb2WU33Akv5vRXT8+By2xawJ05OQUu+H4OuCstT8GtDr0Jt8olX0hWoTFclYYseLcyz7WqMsczkaM4Dt/drI8iMSyXDnn8LlTrovnh3nS9xXPNGBjvgC8sVtilAew/l98Yw7nydn6ugaIZrncfuuxn/FjTDS7PsJZwSz3ZDks9nO2QMoTrNT+zEW7RuTpwWYIZ03MuYT+b6JcSLsDjClzMaIxLzavyva1YzTlnWx4o87kp54iYy3PCHTQcz2UCP8Nt/e3fBgVf/Y4yD7Kxz6cO2guXecR0OPtBO7iL6WLAzW7KsbbK4rtu/Adz+vQueWT0v132+jaw3pG6sS+vf81ydhb208dWYbjRWZ15UcwiLu1yijmWXZJ7Q6FHbsD1NVfgirZgP0scmbftTGU62UXgpljMC7ZM5pxmTWH/y2VGwl1q4gPXexXbJaZ55DgeEHIdZW63qQGXafYROJ+I9+HuTJoH1z4W1/tpo+eGS2Bzf8zY3BMOL5aXzgb39tM1uN5TXPacorPtn1ThfvmzmwngIg7aADfk9/1Ej8ko43+C+cm9A2Phkp/m3tAnlz1ll600s9eKRZm4AlSWF9yvDWnHPceBTMlNmqFsuxZMFYzdrS5c1EXMIOea6I7j7hav7/ankssu7kZqxpUZH/h3m26d/HmPWJyXDuxeAOey3RFupLH+gXv4jO1lL+bGXnOXcgdt/r3luOF61bK4JvKImcp5XIRvqcZj7q+WCE0Gd+Up67bPDoALOe4Nl8xKAXfEXIDzdNm7+BiBm0HNBzD2lzZ94Za+5Hw1eyj3OqvmZu6xfbaz7cwm7pecWJQdLp+ny4CcEgWqqNUeblsI96GHd+X7rzqRuee2P5jbCPH/RfQ/GwghhBBCCCGEEEIIIYQQQgghhBAiTOhjAyGEEEIIIYQQQgghhBBCCCGEEEKECX1sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEmLNu27f9ScIWL2x78Da7avqhwTSvx3C+1isCl2Hwc7qNh9SqYso7j3WY7ylSsGxnOewPr8aYf3TmX+n7q6XJuh4hwbertgDv8pRzc4rIt4FrdTAu3vOlPuHxXRsD9vM76LbbrwX2yNjqOH18ORZkTQ9vCRdn6FG672QOX31hwe1K2g6vVYx5cjSEx4LZ8+AQXXlgP6ew26ykP1ee5fdm/Q8enh0sVfyHczDdsw8rmvOM4s8nDehhvmJgmAC4Lu6Px3Uc3ynAgDIrhD9fS/yTc5eF8hguDo8PFmNwLrlbvc3BRgjnWUjbYxLqsD4ZbaJ47jjM2nIIyt4uwASLUvgXXPX5DuMlJ18ANeBYDrqP9GS5qa8a36OwS4cZ3iwHR12Z8PRm6Ga7UlRJwh3K9giv2KhAuU6LncCerF3QcR9v0GmVMzg9Q1rXYLJd0OlSL59XhlsRdBOexkvE2DkOk6b2L5f6++xecd61EcHaedHBvvt2Hq1mZ8bp7DFam3qCJTtGT48dOlxTOtHNpT7MALqlpA/c09Vee+zkG3LX3bPesNsdyeFHEbIS7PdsT7kOnmnARXOa7tze6wCXOwv7XyOXclmOcfbx+vzQocziA83o0b/bb8cmhzN1YvOeayy4xKBLL2TVZzprfB+5XVM7ZkUJ7wJX0yAh32OoK93cyxusZazk3eZXa6xShrG/gryA405xjw0q6DM4exbklxOV73VOdhsPFmP0YLut/S73/7+C0bZolSQy3LsVLuECTCa7wVc6XF/J7wVn9mUPVHOLMEdfYb1DGNkPhGj8vAbc6RXO4JyGcE5LZl+FuwxiTcUMpuIhNuFZ5H8B5zdt05wWnHYaKlKMx3JlWw+AKPPwOF2qKOo6vHWIsrZEgLly8THfhzhmOyTYu/XSBYS5rmzt0GxlHrDp/rt8n/5f1+fmQc+rbXhzXltun+e27QdlzGa8y1GbPurPpieP4YISKKFN6G+emdRU5RhuEuCTw0Weybpnewe05HwWuks31XjyLsf7J7ZhwkTJMYF22ctERI3tZuEE9e8PdSMEYUnOyM45XPzQHZeySxVgPw7FcMgbzyrX+nP/jH+Z7Nd5/Q+Wfy45ydhn7WHixw2yBy/69KVwm3xJw33LyvdnF+MyeTG/M4kqt4NLuzO843l+aeeSQg0XhzLgLUO0GsNjhdr/gKi3junjaj7dwtokGx2hhjI9N+8NyiWntqdY/YLl6B1i/lTbX7U3SO/vV99eLUcb3MecRu6TLHHytKtzmqFwP1Kw1G+79siRwMXtx/eY54c/F+Vf9J8MlCmXfsPcybk4MOAr3/CrvMTXSEbhqcZkrfR5U2XGcwZ6EMvMOcxzcGcX3kTFnSzjP9lxjrnzIMXRqZwG4suY0XDWmXsaO5/IuL9+EsryY2/zYthLOpxnntQ8JksE1/y2l3JmdOZbdjrF60lzmO9Utrm2jnWesXpmWz/oyOts4tyfHX8OQP9fnYwyKBzf950i4iCuewDX+ORouuj/XniNiMC6FfuWc2uWbc/9h/46OKFO2BvvPe++dcEUN84RVNtdTsTKGwOVPAWV2jOI7GpCL5aZc4PyVsjDX2acDP8K1NrHg/DxccuFQl/m1kDM2pznJuSqC4b7pnbI3eP23heFWXOPzN/Fh3U4xfTSFEtKZin+uz1vRT8BV/cz9l/sZrsD15nLF/HWK7zLWKr7LjyX96EZddByPu1IbZRq5NFX25qvgWu/fCrewN9fN0bmtYupH5t7I/B8cV70+c39xYnH2q+i9Z8F9acwbP9nJP7AlqwxlrHF09rprzjIXs7KMWzbmsoexpRkbuTZDvwn5TOkdGgduQfotcE3vcQ9TCPG/0f9sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJiI8F8LNrUqwj2Ltweuyds3cINaJoUrU34j3Eff03A/6pSEi1l9luO4vWGZ3Z2hjOkaGSpvkh9w29tPhIu+gU1lZ05D13ItXK0yN+A23cwC91eWr3BHZkaFS9XjPdyVa5HgppsacObyeudxzgcokty0gntp8sNFN7yn2ct75i4/D65Kbxsu5c9KvN4fxC7HvmEt6wQXauVmObs23fhRcKV3FYBbGMK6lIqV0HF8M01DFuodDPV9/AG4KFYZOJuvw1hpA+E8HpyCW2TPgLttPYFr7Jsc7ltlDtRo3bfBhf7dnPW7Pw0ufyjbYIhx3rdPFb6HXh3Pw73vaMEluJYObnJKxrfRCT/DmSyfoJZ1DYVr/ge/AUtsj4X7vHgdXO1WFeAC7Vdw+acVhTvV1Qsu+tZUcNGqOWNO9gYoYq5+qgxnbzoLN/gZzx2WlWPj0s39cKFD2sHVmDEXzovFzN6X11i/ApxLR426BTepFNtpzf1DcOXSxoCbkMPZBr3Kl0eZARH7wQ2LPxVuwfs2cK1rNoVbXpFz1eVWnKuyvooN9yc54VsI7kDbxHCWWc6T7dVQses1cyk3HWrRfRbrn+6t4/jFBOYE5kMjOsO8a7xffLi9DGkmuhkHZ//ihHBtF8+1Y3AeuteSeVHaRewb95e6VGYzbzI7HuehD8vZLtMuOes8dtc5lBnuUZjXd5n7ngT3pTSMbzVS8Bm2p+UFF3xhrpTV5Q7hycvezMGXn35Jl3AMnPWDScoJX+Z5lrcPXPVuM+ECBv/Wf926Rut9cLUXfoFbFcDcY0ExXrCN1QcubdB43njfMagDo7rBFSqWD+7GKcaH7M9YP+ORF+rRX5yfavTi3GZFdNaldISjKPO9ch04Pzs93PkUHFc/A3zhfNKzPUOfMPfaMY6TYjWY8KNdRI7NAX16smCfm3RDM0MlsC/C7XHpvPE3uVQmhXMcfO7Bum1kqmDqeV2A6xo4BO717NdwVvMEcN9+8Z1vnXES7lm29XBLM25lBQtzPtnUlnHXP2EvuFzjmFe0T8N5rOgk53GIS7wIycMIO/Ah2+SIP0+OFbEFXPOgrnDLvv0Nd3Ykc/o/SZXBNeCskd/gHsXYAffqo1uuwbkx8zSuG5pHcHkpxhkPC5drixIJq8yHa7vzONx8wz5qzx4JZ91LBjfwQHe4oIMcf7nKMFpdtLbDGZMSJrRATrh6c9kmlifXj6EW38/PF871Y+SzXBPXYxgw6x/xuRZN5pjKfp2xwXKJW7bhYirRhIdwzCTCj1ylGdPtJYxLd1scgYuxk/sPvfoNhLPsYnCPLK7dU3Zxjo251nCU8e/IHDcDX4exb1anrMQ1dv/7GeHi+fKNVGY3M7bFvc7l+abAVTzH2Gfbt+EOZme5Mo+S8NxFzEd22Hcdx/W+cU1c/S7P65aUe7gBrTn24uXh2OhpJWLdzFS4yo+XwLnsyoUbXiM5dzYzzL/6uSy/T4zrD1dwymg4i9tk5msGtusHb+eeaLmhzB2+T2KfitOwPZzfshxwueZehUu6g3nqm0VQxuTiujCvxfklVmbGdN9AnluwFfN+v0Vskzg+CeGM4Vxn3qVwHN7uzhwzxZRocNaBWHDtbK41mmbhfm0TbneYQh85v9x2OZeRJvwYtYDvqL91Bc6yY8AVtfzhGi2fDPdxFv/2YSKwXWMFO9eiGSzGluwW90FuNOW1bp+Zw3vm5Rj64rJ3kbOgS1A/zb3nEVZauAmbGUM+3ygF52O4eZXMn9Evt+HfC4r39WP9Qpz7BNFq8W8vdzfzNB/DNVnvrhx7pT+t4Mmp+KyrPTlv1Nx7mecal3lYCPE/0f9sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJiwbNu2/3QlhBBCCCGEEEIIIYQQQgghhBBCCPH/HfQ/GwghhBBCCCGEEEIIIYQQQgghhBAiTOhjAyGEEEIIIYQQQgghhBBCCCGEEEKECX1sIIQQQgghhBBCCCGEEEIIIYQQQogwoY8NhBBCCCGEEEIIIYQQQgghhBBCCBEm9LGBEEIIIYQQQgghhBBCCCGEEEIIIcKEPjYQQgghhBBCCCGEEEIIIYQQQgghRJjQxwZCCCGEEEIIIYQQQgghhBBCCCGECBP62EAIIYQQQgghhBBCCCGEEEIIIYQQYUIfGwghhBBCCCGEEEIIIYQQQgghhBAiTPz/AL8updm1BrGHAAAAAElFTkSuQmCC", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "test_mse = (output.detach() - ground_truth_target).pow(2).mean()\n", - "feat_mse = (model_bespoke(output.detach())- model_bespoke(ground_truth_target)).pow(2).mean() \n", - "test_psnr = inversefed.metrics.psnr(output, ground_truth_target, factor=1/ds)\n", - "\n", - "grid_plot(output, [validloader.dataset.classes[l] for l in labels_target])\n", - "plt.title(f\"Rec. loss: {stats['opt']:2.4f} | MSE: {test_mse:2.4f} \"\n", - " f\"| PSNR: {test_psnr:4.2f} | FMSE: {feat_mse:2.4e} |\");" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.12" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/src/utils/communication/comm_utils.py b/src/utils/communication/comm_utils.py index 54566916..1c6aeaa8 100644 --- a/src/utils/communication/comm_utils.py +++ b/src/utils/communication/comm_utils.py @@ -61,19 +61,6 @@ def send(self, dest: str | int | List[str | int], data: Any, tag: int = 0): print(f"Sending data to {dest}") self.comm.send(dest=int(dest), data=data) - def send_dummy_data(self, dest: str | int, dims: Tuple[int, int]): - """ - placeholder method for sending images or other data types - """ - # generate random data of given int dimension - data = np.random.rand(*dims) - if isinstance(dest, list): - for d in dest: - self.comm.send(dest=int(d), data=data) - else: - print(f"Sending data to {dest}") - self.comm.send(dest=int(dest), data=data) - def receive(self, node_ids: List[int]) -> Any: """ Receive data from the specified node diff --git a/src/utils/log_utils.py b/src/utils/log_utils.py index 62f015aa..0bbece06 100644 --- a/src/utils/log_utils.py +++ b/src/utils/log_utils.py @@ -190,6 +190,50 @@ def log_image(self, imgs: torch.Tensor, key: str, iteration: int): save_image(grid_img, f"{self.log_dir}/{iteration}_{key}.png") self.writer.add_image(key, grid_img.numpy(), iteration) + def log_gia_target(data, + target, + node_id, + dm=torch.as_tensor([0.4914, 0.4822, 0.4465])[:, None, None], + ds=torch.as_tensor([0.2023, 0.1994, 0.2010])[:, None, None]): + """ + Plots a grid of images from `data` with corresponding labels from `target`, and saves the plot. + + Args: + data (torch.Tensor): The data tensor with shape (batch, channels, height, width). + target (torch.Tensor): The target labels tensor with shape (batch,). + node_id (int): The node ID for the client. + dm (torch.Tensor): The mean of the dataset used for normalization, with shape (3, 1, 1). + ds (torch.Tensor): The standard deviation of the dataset used for normalization, with shape (3, 1, 1). + """ + # Move data and target to CPU if they are on a GPU, and detach from the computation graph + data = data.cpu().detach() + target = target.cpu().detach() + + # Normalize and clamp the data to the valid range [0, 1] + data = data.mul(ds).add(dm) + data.clamp_(0, 1) + + # Set up grid size for plotting (e.g., 2 rows of 5 images if batch size is 10) + batch_size = data.size(0) + rows = 1 + cols = batch_size // rows if batch_size % rows == 0 else batch_size + + fig, axes = plt.subplots(rows, cols, figsize=(12, 6)) + axes = axes.flatten() + + # Loop over each image and label in the batch + for i in range(batch_size): + # Convert image to numpy format for plotting + img = data[i].permute(1, 2, 0).numpy() + + # Plot the image and label + axes[i].imshow(img) + axes[i].set_title(f"Label: {target[i].item()}") + axes[i].axis("off") + + plt.tight_layout() + plt.savefig(f"{self.log_dir}/{node_id}_original.png") + def log_console(self, msg: str): """ Log a message to the console. diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index 3bd46c5b..4ee2b9b7 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -196,9 +196,6 @@ def train_classification( print("here, applying softmax") output = nn.functional.log_softmax(output, dim=1) # type: ignore if kwargs.get("gia", False): - # print(data.shape, target.shape) - node_id = kwargs.get("node_id") - plot_and_save(data, target, filename=f"data_target_plot_{node_id}.png") # Sum the loss and create gradient graph like in loss_steps # Use modified loss_steps function that returns loss model.eval() @@ -562,49 +559,4 @@ def get_memory_usage(self): """ Get the memory usage """ - return torch.cuda.memory_allocated(self.device) - -def plot_and_save(data, - target, - dm=torch.as_tensor([0.4914, 0.4822, 0.4465])[:, None, None], - ds=torch.as_tensor([0.2023, 0.1994, 0.2010])[:, None, None], - filename="plot.png"): - """ - Plots a grid of images from `data` with corresponding labels from `target`, and saves the plot. - - Args: - data (torch.Tensor): The data tensor with shape (batch, channels, height, width). - target (torch.Tensor): The target labels tensor with shape (batch,). - dm (torch.Tensor): The mean of the dataset used for normalization, with shape (3, 1, 1). - ds (torch.Tensor): The standard deviation of the dataset used for normalization, with shape (3, 1, 1). - filename (str): The filename to save the plot as an image. - """ - # Move data and target to CPU if they are on a GPU, and detach from the computation graph - data = data.cpu().detach() - target = target.cpu().detach() - - # Normalize and clamp the data to the valid range [0, 1] - data = data.mul(ds).add(dm) - data.clamp_(0, 1) - - # Set up grid size for plotting (e.g., 2 rows of 5 images if batch size is 10) - batch_size = data.size(0) - rows = 1 - cols = batch_size // rows if batch_size % rows == 0 else batch_size - - fig, axes = plt.subplots(rows, cols, figsize=(12, 6)) - axes = axes.flatten() - - # Loop over each image and label in the batch - for i in range(batch_size): - # Convert image to numpy format for plotting - img = data[i].permute(1, 2, 0).numpy() - - # Plot the image and label - axes[i].imshow(img) - axes[i].set_title(f"Label: {target[i].item()}") - axes[i].axis("off") - - plt.tight_layout() - plt.savefig(filename) - plt.show() \ No newline at end of file + return torch.cuda.memory_allocated(self.device) \ No newline at end of file From 8970b3b98df48368a723015bf3e900edd67a6cff Mon Sep 17 00:00:00 2001 From: photonshi Date: Sun, 10 Nov 2024 19:37:46 +0000 Subject: [PATCH 13/16] fixed logging --- src/algos/base_class.py | 10 +++++++--- src/algos/fl_static.py | 25 ++++++++++++++++--------- src/utils/log_utils.py | 24 ++++++++++++++++++------ src/utils/model_utils.py | 3 --- 4 files changed, 41 insertions(+), 21 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index 4da52be5..a0c0625d 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -400,9 +400,13 @@ def set_parameters(self, config: Dict[str, Any]) -> None: # after setting data loaders, save client dataset # TODO verify this .data and .labels fields are correct if "gia" in config: - self.log_utils.log_gia_target(self.train_dset.data, - self.train_dset.labels, - self.node_id) + # Extract data and labels + train_data = torch.stack([data[0] for data in self.train_dset]) + train_labels = torch.tensor([data[1] for data in self.train_dset]) + + self.log_utils.log_gia_image(train_data, + train_labels, + self.node_id) def set_data_parameters(self, config: ConfigType) -> None: diff --git a/src/algos/fl_static.py b/src/algos/fl_static.py index 57e2d1a9..5bdc52ab 100644 --- a/src/algos/fl_static.py +++ b/src/algos/fl_static.py @@ -79,15 +79,22 @@ def receive_attack_and_aggregate(self, neighbors: List[int], round: int, num_nei p_t = self.neighbor_updates[neighbor_id][1]["model"] # Launch the attack - gia_main( - p_s, - p_t, - self.base_params, - self.model, - neighbor_labels, - neighbor_images, - self.node_id - ) + if result := gia_main(p_s, + p_t, + self.base_params, + self.model, + neighbor_labels, + neighbor_images, + self.node_id): + output, stats = result + + # log output and stats as image + self.log_utils.log_gia_image(output, neighbor_labels, neighbor_id, label=f"round_{round}_reconstruction") + self.log_utils.log_summary(f"round {round} gia targeting {neighbor_id} stats: {stats}") + else: + self.log_utils.log_summary(f"Client {self.node_id} failed to attack {neighbor_id} in round {round}!") + print(f"Client {self.node_id} failed to attack {neighbor_id}!") + continue # Mark this neighbor as attacked self.attacked_neighbors.add(neighbor_id) diff --git a/src/utils/log_utils.py b/src/utils/log_utils.py index 0bbece06..39d4ee89 100644 --- a/src/utils/log_utils.py +++ b/src/utils/log_utils.py @@ -17,6 +17,7 @@ import pandas as pd from utils.types import ConfigType import json +import matplotlib.pyplot as plt def deprocess(img: torch.Tensor) -> torch.Tensor: @@ -190,11 +191,13 @@ def log_image(self, imgs: torch.Tensor, key: str, iteration: int): save_image(grid_img, f"{self.log_dir}/{iteration}_{key}.png") self.writer.add_image(key, grid_img.numpy(), iteration) - def log_gia_target(data, - target, - node_id, - dm=torch.as_tensor([0.4914, 0.4822, 0.4465])[:, None, None], - ds=torch.as_tensor([0.2023, 0.1994, 0.2010])[:, None, None]): + def log_gia_image(self, + data, + target, + node_id, + dm=torch.as_tensor([0.4914, 0.4822, 0.4465])[:, None, None], + ds=torch.as_tensor([0.2023, 0.1994, 0.2010])[:, None, None], + label=None): """ Plots a grid of images from `data` with corresponding labels from `target`, and saves the plot. @@ -232,8 +235,17 @@ def log_gia_target(data, axes[i].axis("off") plt.tight_layout() - plt.savefig(f"{self.log_dir}/{node_id}_original.png") + log_lab = "base" if not label else label + + plt.savefig(f"{self.log_dir}/{node_id}_{log_lab}.png") + plt.close() + + # Log images to TensorBoard + grid_img = make_grid(data, normalize=True, scale_each=True) + self.writer.add_image(f"gia_images_node_{node_id}_{log_lab}", grid_img.numpy(), node_id) + self.writer.add_text(f"gia_labels_node_{node_id}_{log_lab}", str(target.tolist()), node_id) + def log_console(self, msg: str): """ Log a message to the console. diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index 4ee2b9b7..6c19dfc3 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -10,13 +10,10 @@ import resnet import resnet_in -import matplotlib.pyplot as plt - import yolo from utils.types import ConfigType from inversefed.reconstruction_algorithms import loss_steps -import pickle class ModelUtils: def __init__(self, device: torch.device, config: ConfigType) -> None: From 8e64e60778bf28ca6ac081dd763cbf12fb577790 Mon Sep 17 00:00:00 2001 From: photonshi Date: Wed, 13 Nov 2024 18:01:23 +0000 Subject: [PATCH 14/16] migrated attack code to basenode --- src/algos/base_class.py | 96 +++++++++++++++++++++++++++++++++++++---- src/algos/fl.py | 26 +---------- src/algos/fl_static.py | 79 +-------------------------------- 3 files changed, 91 insertions(+), 110 deletions(-) diff --git a/src/algos/base_class.py b/src/algos/base_class.py index a0c0625d..20176941 100644 --- a/src/algos/base_class.py +++ b/src/algos/base_class.py @@ -37,6 +37,7 @@ ) from utils.types import ConfigType from utils.dropout_utils import NodeDropout +from utils.gias import gia_main import torchvision.transforms as T # type: ignore import os @@ -382,6 +383,17 @@ def __init__( super().__init__(config, comm_utils) self.server_node = 0 self.set_parameters(config) + if "gia" in config: + if int(self.node_id) in self.config["gia_attackers"]: + self.gia_attacker = True + self.params_s = dict() + self.params_t = dict() + # Track neighbor updates with a dictionary mapping neighbor_id to their updates + self.neighbor_updates = defaultdict(list) + # Track which neighbors we've already attacked + self.attacked_neighbors = set() + + self.base_params = [key for key, _ in self.model.named_parameters()] def set_parameters(self, config: Dict[str, Any]) -> None: """ @@ -672,6 +684,68 @@ def receive_and_aggregate(self): assert "model" in repr, "Model not found in the received message" self.set_model_weights(repr["model"]) + def receive_attack_and_aggregate(self, neighbors: List[int], round: int, num_neighbors: int) -> None: + """ + Receives updates, launches GIA attack when second update is seen from a neighbor + """ + print("CLIENT RECEIVING ATTACK AND AGGREGATING") + if self.is_working: + # Receive the model updates from the neighbors + model_updates = self.comm_utils.receive(node_ids=neighbors) + assert len(model_updates) == num_neighbors + + for neighbor_info in model_updates: + neighbor_id = neighbor_info["sender"] + neighbor_model = neighbor_info["model"] + neighbor_model = OrderedDict( + (key, value) for key, value in neighbor_model.items() + if key in self.base_params + ) + + neighbor_images = neighbor_info["images"] + neighbor_labels = neighbor_info["labels"] + + # Store this update + self.neighbor_updates[neighbor_id].append({ + "model": neighbor_model, + "images": neighbor_images, + "labels": neighbor_labels + }) + + # Check if we have 2 updates from this neighbor and haven't attacked them yet + if len(self.neighbor_updates[neighbor_id]) == 2 and neighbor_id not in self.attacked_neighbors: + print(f"Client {self.node_id} attacking {neighbor_id}!") + + # Get the two parameter sets for the attack + p_s = self.neighbor_updates[neighbor_id][0]["model"] + p_t = self.neighbor_updates[neighbor_id][1]["model"] + + # Launch the attack + if result := gia_main(p_s, + p_t, + self.base_params, + self.model, + neighbor_labels, + neighbor_images, + self.node_id): + output, stats = result + + # log output and stats as image + self.log_utils.log_gia_image(output, neighbor_labels, neighbor_id, label=f"round_{round}_reconstruction") + self.log_utils.log_summary(f"round {round} gia targeting {neighbor_id} stats: {stats}") + else: + self.log_utils.log_summary(f"Client {self.node_id} failed to attack {neighbor_id} in round {round}!") + print(f"Client {self.node_id} failed to attack {neighbor_id}!") + continue + + # Mark this neighbor as attacked + self.attacked_neighbors.add(neighbor_id) + + # Optionally, clear the stored updates to save memory + del self.neighbor_updates[neighbor_id] + + self.aggregate(model_updates, keys_to_ignore=self.model_keys_to_ignore) + def run_protocol(self) -> None: raise NotImplementedError @@ -762,7 +836,6 @@ def get_model(self, **kwargs: Any) -> Any: def run_protocol(self) -> None: raise NotImplementedError - class CommProtocol(object): """ Communication protocol tags for the server and users @@ -800,6 +873,7 @@ def __init__( keys = self.model_utils.get_last_layer_keys(self.get_model_weights()) self.model_keys_to_ignore.extend(keys) + def local_test(self, **kwargs: Any) -> Tuple[float, float]: """ Test the model locally, not to be used in the traditional FedAvg @@ -864,13 +938,19 @@ def aggregate( self.set_model_weights(agg_wts) return None - def receive_and_aggregate(self, neighbors: List[int]) -> None: - if self.is_working: - # Receive the model updates from the neighbors - model_updates = self.comm_utils.receive(node_ids=neighbors) - # Aggregate the representations - self.aggregate(model_updates, keys_to_ignore=self.model_keys_to_ignore) - + def receive_and_aggregate(self, neighbors: List[int], it:int=0) -> None: + """ + Receive the model weights from the collaborators and aggregate + launches GIA attack if self is a GIA attacker + """ + if hasattr(self, "gia_attacker"): + self.receive_attack_and_aggregate(neighbors, it, len(neighbors)) + else: + if self.is_working: + # Receive the model updates from the neighbors + model_updates = self.comm_utils.receive(node_ids=neighbors) + # Aggregate the representations + self.aggregate(model_updates, keys_to_ignore=self.model_keys_to_ignore) def get_collaborator_weights( self, reprs_dict: Dict[int, OrderedDict[int, Tensor]] diff --git a/src/algos/fl.py b/src/algos/fl.py index 3f8257a9..f071a0e2 100644 --- a/src/algos/fl.py +++ b/src/algos/fl.py @@ -113,13 +113,6 @@ def __init__( self.model_save_path = "{}/saved_models/node_{}.pt".format( self.config["results_path"], self.node_id ) - if "gia" in self.config: - # to store param differences for GIA attack - self.params_s = [None for i in range(4)] - self.params_t = [None for i in range(4)] - - # save randomly initialized parameters - self.random_params = self.model.state_dict() def fed_avg(self, model_wts: List[OrderedDict[str, Tensor]]): num_users = len(model_wts) @@ -174,7 +167,7 @@ def test(self, **kwargs: Any) -> List[float]: self.model_utils.save_model(self.model, self.model_save_path) return [test_loss, test_acc, time_taken] - def receive_and_aggregate_gia(self, round: int, attack_start_round: int, attack_end_round: int, dump_file_name: str = ""): + def receive_attack_and_aggregate(self, round: int, attack_start_round: int, attack_end_round: int, dump_file_name: str = ""): reprs = self.comm_utils.all_gather() with open(dump_file_name, "wb") as f: @@ -211,13 +204,6 @@ def receive_and_aggregate_gia(self, round: int, attack_start_round: int, attack_ images = rep["images"] labels = rep["labels"] - # with open(f"params_t_{client_id}.pkl", "wb") as f: - # pickle.dump(model_params, f) - # with open(f"params_s_{client_id}.pkl", "wb") as f: - # pickle.dump(self.params_s[client_id - 1], f) - # with open(f"random_params_{client_id}.pkl", "wb") as f: - # pickle.dump(random_params, f) - # Launch GIA attack p_s, p_t = self.params_s[client_id - 1], self.params_t[client_id - 1] gia_main(p_s, p_t, base_params, self.model, labels, images, client_id) @@ -245,15 +231,7 @@ def single_round(self, round: int, attack_start_round: int = 0, attack_end_round if round < attack_start_round or round > attack_end_round: self.receive_and_aggregate() else: - # Set file name based on start or end of attack range - dump_file_name = "" - if round == attack_start_round: - dump_file_name = "/u/yshi23/sonar/src/start_reprs" - elif round == attack_end_round: - dump_file_name = "/u/yshi23/sonar/src/end_reprs" - - print(f"In round {round}, preparing for GIA with file: {dump_file_name}") - self.receive_and_aggregate_gia(round, attack_start_round, attack_end_round, dump_file_name) + self.receive_attack_and_aggregate(round, attack_start_round, attack_end_round, dump_file_name) def run_protocol(self): diff --git a/src/algos/fl_static.py b/src/algos/fl_static.py index 5bdc52ab..111397d1 100644 --- a/src/algos/fl_static.py +++ b/src/algos/fl_static.py @@ -24,17 +24,6 @@ def __init__( super().__init__(config, comm_utils) self.topology = select_topology(config, self.node_id) self.topology.initialize() - if "gia" in config: - if int(self.node_id) in self.config["gia_attackers"]: - self.gia_attacker = True - self.params_s = dict() - self.params_t = dict() - # Track neighbor updates with a dictionary mapping neighbor_id to their updates - self.neighbor_updates = defaultdict(list) - # Track which neighbors we've already attacked - self.attacked_neighbors = set() - - self.base_params = [key for key, _ in self.model.named_parameters()] def get_representation(self, **kwargs: Any) -> OrderedDict[str, torch.Tensor]: """ @@ -42,68 +31,6 @@ def get_representation(self, **kwargs: Any) -> OrderedDict[str, torch.Tensor]: """ return self.get_model_weights() - def receive_attack_and_aggregate(self, neighbors: List[int], round: int, num_neighbors: int) -> None: - """ - Receives updates, launches GIA attack when second update is seen from a neighbor - """ - print("CLIENT RECEIVING ATTACK AND AGGREGATING") - if self.is_working: - # Receive the model updates from the neighbors - model_updates = self.comm_utils.receive(node_ids=neighbors) - assert len(model_updates) == num_neighbors - - for neighbor_info in model_updates: - neighbor_id = neighbor_info["sender"] - neighbor_model = neighbor_info["model"] - neighbor_model = OrderedDict( - (key, value) for key, value in neighbor_model.items() - if key in self.base_params - ) - - neighbor_images = neighbor_info["images"] - neighbor_labels = neighbor_info["labels"] - - # Store this update - self.neighbor_updates[neighbor_id].append({ - "model": neighbor_model, - "images": neighbor_images, - "labels": neighbor_labels - }) - - # Check if we have 2 updates from this neighbor and haven't attacked them yet - if len(self.neighbor_updates[neighbor_id]) == 2 and neighbor_id not in self.attacked_neighbors: - print(f"Client {self.node_id} attacking {neighbor_id}!") - - # Get the two parameter sets for the attack - p_s = self.neighbor_updates[neighbor_id][0]["model"] - p_t = self.neighbor_updates[neighbor_id][1]["model"] - - # Launch the attack - if result := gia_main(p_s, - p_t, - self.base_params, - self.model, - neighbor_labels, - neighbor_images, - self.node_id): - output, stats = result - - # log output and stats as image - self.log_utils.log_gia_image(output, neighbor_labels, neighbor_id, label=f"round_{round}_reconstruction") - self.log_utils.log_summary(f"round {round} gia targeting {neighbor_id} stats: {stats}") - else: - self.log_utils.log_summary(f"Client {self.node_id} failed to attack {neighbor_id} in round {round}!") - print(f"Client {self.node_id} failed to attack {neighbor_id}!") - continue - - # Mark this neighbor as attacked - self.attacked_neighbors.add(neighbor_id) - - # Optionally, clear the stored updates to save memory - del self.neighbor_updates[neighbor_id] - - self.aggregate(model_updates, keys_to_ignore=self.model_keys_to_ignore) - def run_protocol(self) -> None: """ Runs the federated learning protocol for the client. @@ -129,11 +56,7 @@ def run_protocol(self) -> None: neighbors = self.topology.sample_neighbours(self.num_collaborators) stats["neighbors"] = neighbors - if hasattr(self, "gia_attacker"): - print(f"Client {self.node_id} is a GIA attacker!") - self.receive_attack_and_aggregate(neighbors, it, len(neighbors)) - else: - self.receive_and_aggregate(neighbors) + self.receive_and_aggregate(neighbors) stats["bytes_received"], stats["bytes_sent"] = self.comm_utils.get_comm_cost() stats["test_loss"], stats["test_acc"] = self.local_test() From e1ac62259e6d817c48b945e65c2c3be0e0775ead Mon Sep 17 00:00:00 2001 From: photonshi Date: Mon, 2 Dec 2024 05:45:06 +0000 Subject: [PATCH 15/16] fixed inversefed import error --- src/utils/model_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/model_utils.py b/src/utils/model_utils.py index 3e8b50e5..076cdf93 100644 --- a/src/utils/model_utils.py +++ b/src/utils/model_utils.py @@ -13,8 +13,6 @@ import yolo from utils.types import ConfigType -from inversefed.reconstruction_algorithms import loss_steps - class ModelUtils: def __init__(self, device: torch.device, config: ConfigType) -> None: self.device = device @@ -197,6 +195,8 @@ def train_classification( print("here, applying softmax") output = nn.functional.log_softmax(output, dim=1) # type: ignore if kwargs.get("gia", False): + from inversefed.reconstruction_algorithms import loss_steps + # Sum the loss and create gradient graph like in loss_steps # Use modified loss_steps function that returns loss model.eval() From 212e12b1f831d25855642318f5b08fb149a921eb Mon Sep 17 00:00:00 2001 From: photonshi Date: Mon, 2 Dec 2024 05:51:31 +0000 Subject: [PATCH 16/16] added data subfolder in inversefed --- src/inversefed/data/__init__.py | 0 src/inversefed/data/data.py | 0 src/inversefed/data/data_processing.py | 209 +++++++++++++++++++++++++ src/inversefed/data/datasets.py | 0 src/inversefed/data/loss.py | 114 ++++++++++++++ 5 files changed, 323 insertions(+) create mode 100644 src/inversefed/data/__init__.py create mode 100644 src/inversefed/data/data.py create mode 100644 src/inversefed/data/data_processing.py create mode 100644 src/inversefed/data/datasets.py create mode 100644 src/inversefed/data/loss.py diff --git a/src/inversefed/data/__init__.py b/src/inversefed/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/inversefed/data/data.py b/src/inversefed/data/data.py new file mode 100644 index 00000000..e69de29b diff --git a/src/inversefed/data/data_processing.py b/src/inversefed/data/data_processing.py new file mode 100644 index 00000000..f6e36e3a --- /dev/null +++ b/src/inversefed/data/data_processing.py @@ -0,0 +1,209 @@ +"""Repeatable code parts concerning data loading.""" + + +import torch +import torchvision +import torchvision.transforms as transforms + +import os + +from ..consts import * + +from .data import _build_bsds_sr, _build_bsds_dn +from .loss import Classification, PSNR + + +def construct_dataloaders(dataset, defs, data_path='~/data', shuffle=True, normalize=True): + """Return a dataloader with given dataset and augmentation, normalize data?.""" + path = os.path.expanduser(data_path) + + if dataset == 'CIFAR10': + trainset, validset = _build_cifar10(path, defs.augmentations, normalize) + loss_fn = Classification() + elif dataset == 'CIFAR100': + trainset, validset = _build_cifar100(path, defs.augmentations, normalize) + loss_fn = Classification() + elif dataset == 'MNIST': + trainset, validset = _build_mnist(path, defs.augmentations, normalize) + loss_fn = Classification() + elif dataset == 'MNIST_GRAY': + trainset, validset = _build_mnist_gray(path, defs.augmentations, normalize) + loss_fn = Classification() + elif dataset == 'ImageNet': + trainset, validset = _build_imagenet(path, defs.augmentations, normalize) + loss_fn = Classification() + elif dataset == 'BSDS-SR': + trainset, validset = _build_bsds_sr(path, defs.augmentations, normalize, upscale_factor=3, RGB=True) + loss_fn = PSNR() + elif dataset == 'BSDS-DN': + trainset, validset = _build_bsds_dn(path, defs.augmentations, normalize, noise_level=25 / 255, RGB=False) + loss_fn = PSNR() + elif dataset == 'BSDS-RGB': + trainset, validset = _build_bsds_dn(path, defs.augmentations, normalize, noise_level=25 / 255, RGB=True) + loss_fn = PSNR() + + if MULTITHREAD_DATAPROCESSING: + num_workers = min(torch.get_num_threads(), MULTITHREAD_DATAPROCESSING) if torch.get_num_threads() > 1 else 0 + else: + num_workers = 0 + + trainloader = torch.utils.data.DataLoader(trainset, batch_size=min(defs.batch_size, len(trainset)), + shuffle=shuffle, drop_last=True, num_workers=num_workers, pin_memory=PIN_MEMORY) + validloader = torch.utils.data.DataLoader(validset, batch_size=min(defs.batch_size, len(trainset)), + shuffle=False, drop_last=False, num_workers=num_workers, pin_memory=PIN_MEMORY) + + return loss_fn, trainloader, validloader + + +def _build_cifar10(data_path, augmentations=True, normalize=True): + """Define CIFAR-10 with everything considered.""" + # Load data + trainset = torchvision.datasets.CIFAR10(root=data_path, train=True, download=True, transform=transforms.ToTensor()) + validset = torchvision.datasets.CIFAR10(root=data_path, train=False, download=True, transform=transforms.ToTensor()) + + if cifar10_mean is None: + data_mean, data_std = _get_meanstd(trainset) + else: + data_mean, data_std = cifar10_mean, cifar10_std + + # Organize preprocessing + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(data_mean, data_std) if normalize else transforms.Lambda(lambda x: x)]) + if augmentations: + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transform]) + trainset.transform = transform_train + else: + trainset.transform = transform + validset.transform = transform + + return trainset, validset + +def _build_cifar100(data_path, augmentations=True, normalize=True): + """Define CIFAR-100 with everything considered.""" + # Load data + trainset = torchvision.datasets.CIFAR100(root=data_path, train=True, download=True, transform=transforms.ToTensor()) + validset = torchvision.datasets.CIFAR100(root=data_path, train=False, download=True, transform=transforms.ToTensor()) + + if cifar100_mean is None: + data_mean, data_std = _get_meanstd(trainset) + else: + data_mean, data_std = cifar100_mean, cifar100_std + + # Organize preprocessing + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(data_mean, data_std) if normalize else transforms.Lambda(lambda x: x)]) + if augmentations: + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transform]) + trainset.transform = transform_train + else: + trainset.transform = transform + validset.transform = transform + + return trainset, validset + + +def _build_mnist(data_path, augmentations=True, normalize=True): + """Define MNIST with everything considered.""" + # Load data + trainset = torchvision.datasets.MNIST(root=data_path, train=True, download=True, transform=transforms.ToTensor()) + validset = torchvision.datasets.MNIST(root=data_path, train=False, download=True, transform=transforms.ToTensor()) + + if mnist_mean is None: + cc = torch.cat([trainset[i][0].reshape(-1) for i in range(len(trainset))], dim=0) + data_mean = (torch.mean(cc, dim=0).item(),) + data_std = (torch.std(cc, dim=0).item(),) + else: + data_mean, data_std = mnist_mean, mnist_std + + # Organize preprocessing + transform = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize(data_mean, data_std) if normalize else transforms.Lambda(lambda x: x)]) + if augmentations: + transform_train = transforms.Compose([ + transforms.RandomCrop(28, padding=4), + transforms.RandomHorizontalFlip(), + transform]) + trainset.transform = transform_train + else: + trainset.transform = transform + validset.transform = transform + + return trainset, validset + +def _build_mnist_gray(data_path, augmentations=True, normalize=True): + """Define MNIST with everything considered.""" + # Load data + trainset = torchvision.datasets.MNIST(root=data_path, train=True, download=True, transform=transforms.ToTensor()) + validset = torchvision.datasets.MNIST(root=data_path, train=False, download=True, transform=transforms.ToTensor()) + + if mnist_mean is None: + cc = torch.cat([trainset[i][0].reshape(-1) for i in range(len(trainset))], dim=0) + data_mean = (torch.mean(cc, dim=0).item(),) + data_std = (torch.std(cc, dim=0).item(),) + else: + data_mean, data_std = mnist_mean, mnist_std + + # Organize preprocessing + transform = transforms.Compose([ + transforms.Grayscale(num_output_channels=1), + transforms.ToTensor(), + transforms.Normalize(data_mean, data_std) if normalize else transforms.Lambda(lambda x: x)]) + if augmentations: + transform_train = transforms.Compose([ + transforms.Grayscale(num_output_channels=1), + transforms.RandomCrop(28, padding=4), + transforms.RandomHorizontalFlip(), + transform]) + trainset.transform = transform_train + else: + trainset.transform = transform + validset.transform = transform + + return trainset, validset + + +def _build_imagenet(data_path, augmentations=True, normalize=True): + """Define ImageNet with everything considered.""" + # Load data + trainset = torchvision.datasets.ImageNet(root=data_path, split='train', transform=transforms.ToTensor()) + validset = torchvision.datasets.ImageNet(root=data_path, split='val', transform=transforms.ToTensor()) + + if imagenet_mean is None: + data_mean, data_std = _get_meanstd(trainset) + else: + data_mean, data_std = imagenet_mean, imagenet_std + + # Organize preprocessing + transform = transforms.Compose([ + transforms.Resize(256), + transforms.CenterCrop(224), + transforms.ToTensor(), + transforms.Normalize(data_mean, data_std) if normalize else transforms.Lambda(lambda x : x)]) + if augmentations: + transform_train = transforms.Compose([ + transforms.RandomResizedCrop(224), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize(data_mean, data_std) if normalize else transforms.Lambda(lambda x : x)]) + trainset.transform = transform_train + else: + trainset.transform = transform + validset.transform = transform + + return trainset, validset + + +def _get_meanstd(dataset): + cc = torch.cat([trainset[i][0].reshape(3, -1) for i in range(len(trainset))], dim=1) + data_mean = torch.mean(cc, dim=1).tolist() + data_std = torch.std(cc, dim=1).tolist() + return data_mean, data_std \ No newline at end of file diff --git a/src/inversefed/data/datasets.py b/src/inversefed/data/datasets.py new file mode 100644 index 00000000..e69de29b diff --git a/src/inversefed/data/loss.py b/src/inversefed/data/loss.py new file mode 100644 index 00000000..f43ce933 --- /dev/null +++ b/src/inversefed/data/loss.py @@ -0,0 +1,114 @@ +"""Define various loss functions and bundle them with appropriate metrics.""" + +import torch +import numpy as np + + +class Loss: + """Abstract class, containing necessary methods. + + Abstract class to collect information about the 'higher-level' loss function, used to train an energy-based model + containing the evaluation of the loss function, its gradients w.r.t. to first and second argument and evaluations + of the actual metric that is targeted. + + """ + + def __init__(self): + """Init.""" + pass + + def __call__(self, reference, argmin): + """Return l(x, y).""" + raise NotImplementedError() + return value, name, format + + def metric(self, reference, argmin): + """The actually sought metric.""" + raise NotImplementedError() + return value, name, format + + +class PSNR(Loss): + """A classical MSE target. + + The minimized criterion is MSE Loss, the actual metric is average PSNR. + """ + + def __init__(self): + """Init with torch MSE.""" + self.loss_fn = torch.nn.MSELoss(size_average=None, reduce=None, reduction='mean') + + def __call__(self, x=None, y=None): + """Return l(x, y).""" + name = 'MSE' + format = '.6f' + if x is None: + return name, format + else: + value = 0.5 * self.loss_fn(x, y) + return value, name, format + + def metric(self, x=None, y=None): + """The actually sought metric.""" + name = 'avg PSNR' + format = '.3f' + if x is None: + return name, format + else: + value = self.psnr_compute(x, y) + return value, name, format + + @staticmethod + def psnr_compute(img_batch, ref_batch, batched=False, factor=1.0): + """Standard PSNR.""" + def get_psnr(img_in, img_ref): + mse = ((img_in - img_ref)**2).mean() + if mse > 0 and torch.isfinite(mse): + return (10 * torch.log10(factor**2 / mse)).item() + elif not torch.isfinite(mse): + return float('nan') + else: + return float('inf') + + if batched: + psnr = get_psnr(img_batch.detach(), ref_batch) + else: + [B, C, m, n] = img_batch.shape + psnrs = [] + for sample in range(B): + psnrs.append(get_psnr(img_batch.detach()[sample, :, :, :], ref_batch[sample, :, :, :])) + psnr = np.mean(psnrs) + + return psnr + + +class Classification(Loss): + """A classical NLL loss for classification. Evaluation has the softmax baked in. + + The minimized criterion is cross entropy, the actual metric is total accuracy. + """ + + def __init__(self): + """Init with torch MSE.""" + self.loss_fn = torch.nn.CrossEntropyLoss(weight=None, size_average=None, ignore_index=-100, + reduce=None, reduction='mean') + + def __call__(self, x=None, y=None): + """Return l(x, y).""" + name = 'CrossEntropy' + format = '1.5f' + if x is None: + return name, format + else: + value = self.loss_fn(x, y) + return value, name, format + + def metric(self, x=None, y=None): + """The actually sought metric.""" + name = 'Accuracy' + format = '6.2%' + if x is None: + return name, format + else: + value = (x.data.argmax(dim=1) == y).sum().float() / y.shape[0] + return value.detach(), name, format \ No newline at end of file