From 27f98e4b5a1a95c919ad201b5a16f6db2e641d92 Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Mon, 3 Jul 2023 16:54:47 -0400 Subject: [PATCH 01/18] Cosmetic changes Signed-off-by: Radha Guhane --- .gitignore | 3 ++ benchmarks/spatial/model/amoebanet_run.py | 35 ++++++++------- benchmarks/spatial/model/resnet_model.py | 54 +++++++++++++---------- setup.py | 3 ++ torchgems/parser.py | 15 +++++++ 5 files changed, 71 insertions(+), 39 deletions(-) create mode 100644 .gitignore create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..56028830 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +models/__pycache__/ +torch_gems.egg-info/ +torchgems/__pycache__/ \ No newline at end of file diff --git a/benchmarks/spatial/model/amoebanet_run.py b/benchmarks/spatial/model/amoebanet_run.py index 5ac43868..2bbec157 100644 --- a/benchmarks/spatial/model/amoebanet_run.py +++ b/benchmarks/spatial/model/amoebanet_run.py @@ -92,6 +92,8 @@ def get_depth(version, n): num_spatial_parts = [int(i) for i in temp_num_spatial_parts] num_spatial_parts_list = num_spatial_parts +spatial_part_size = num_spatial_parts_list[0] #Partition size for spatial parallelism + times = 1 num_classes = 1000 LOCAL_DP_LP = args.local_DP @@ -127,12 +129,15 @@ def verify_config(): if args.slice_method == "square": assert isPowerTwo( - int(image_size / math.sqrt(num_spatial_parts)) + int(image_size / math.sqrt(spatial_part_size)) ), "Image size of each partition should be power of Two" else: assert isPowerTwo( - int(image_size / num_spatial_parts) + int(image_size / spatial_part_size) ), "Image size of each partition should be power of Two" + + for each_part_size in num_spatial_parts_list: + assert each_part_size == spatial_part_size, "Size of each SP partition should be same" verify_config() @@ -419,7 +424,7 @@ def verify_config(): if APP == 1: trainset = torchvision.datasets.ImageFolder( - "/usr/workspace/jain8/project/cancer/1024_1024_5/train", + "./train", transform=transform, target_transform=None, ) @@ -468,13 +473,13 @@ def verify_config(): def split_input(inputs): if args.slice_method == "square": - image_height_local = int(image_size / math.sqrt(num_spatial_parts)) - image_width_local = int(image_size / math.sqrt(num_spatial_parts)) + image_height_local = int(image_size / math.sqrt(spatial_part_size)) + image_width_local = int(image_size / math.sqrt(spatial_part_size)) - total_rows = int(math.sqrt(num_spatial_parts)) - total_cols = int(math.sqrt(num_spatial_parts)) + total_rows = int(math.sqrt(spatial_part_size)) + total_cols = int(math.sqrt(spatial_part_size)) - # current position of rank in matrix of math.sqrt(num_spatial_parts) * math.sqrt(num_spatial_parts) + # current position of rank in matrix of math.sqrt(spatial_part_size) * math.sqrt(num_spatial_parts) row = int(local_rank / total_cols) col = int(local_rank % total_cols) @@ -487,13 +492,13 @@ def split_input(inputs): return inputs[:, :, start_top:end_bottom, start_left:end_right] elif args.slice_method == "vertical": - image_height_local = int(image_size / num_spatial_parts) - image_width_local = int(image_size / num_spatial_parts) + image_height_local = int(image_size / spatial_part_size) + image_width_local = int(image_size / spatial_part_size) start_left = local_rank * image_width_local end_right = (local_rank + 1) * image_width_local - if local_rank == num_spatial_parts - 1: + if local_rank == spatial_part_size - 1: # In case of GPU count, partition size will be uneven and last # rank will receive remaining image return inputs[:, :, :, start_left:] @@ -501,13 +506,13 @@ def split_input(inputs): return inputs[:, :, :, start_left:end_right] elif args.slice_method == "horizontal": - image_height_local = int(image_size / num_spatial_parts) - image_width_local = int(image_size / num_spatial_parts) + image_height_local = int(image_size / spatial_part_size) + image_width_local = int(image_size / spatial_part_size) start_top = local_rank * image_height_local end_bottom = (local_rank + 1) * image_height_local - if local_rank == num_spatial_parts - 1: + if local_rank == spatial_part_size - 1: # In case of odd GPU count, partition size will be uneven and last # rank will receive remaining image return inputs[:, :, start_top:, :] @@ -528,7 +533,7 @@ def run_epoch(): break inputs, labels = data - if local_rank < num_spatial_parts_list[0]: + if local_rank < spatial_part_size: x = split_input(inputs) else: x = inputs diff --git a/benchmarks/spatial/model/resnet_model.py b/benchmarks/spatial/model/resnet_model.py index 0a4bd639..f8c84ef2 100644 --- a/benchmarks/spatial/model/resnet_model.py +++ b/benchmarks/spatial/model/resnet_model.py @@ -89,6 +89,8 @@ def get_depth(version, n): num_spatial_parts = [int(i) for i in temp_num_spatial_parts] num_spatial_parts_list = num_spatial_parts +spatial_part_size = num_spatial_parts_list[0] #Partition size for spatial parallelism + times = 1 num_classes = 10 @@ -120,12 +122,16 @@ def verify_config(): if args.slice_method == "square": assert isPowerTwo( - int(image_size / math.sqrt(num_spatial_parts)) + int(image_size / math.sqrt(spatial_part_size)) ), "Image size of each partition should be power of Two" else: assert isPowerTwo( - int(image_size / num_spatial_parts) + int(image_size / spatial_part_size) ), "Image size of each partition should be power of Two" + + for each_part_size in num_spatial_parts_list: + assert each_part_size == spatial_part_size, "Size of each SP partition should be same" + verify_config() @@ -178,12 +184,12 @@ def verify_config(): int( model_gen_seq.shape_list[0][2] * image_size_times - / math.sqrt(num_spatial_parts) + / math.sqrt(spatial_part_size) ), int( model_gen_seq.shape_list[0][3] * image_size_times - / math.sqrt(num_spatial_parts) + / math.sqrt(spatial_part_size) ), ), model_gen_seq.shape_list[1], @@ -195,7 +201,7 @@ def verify_config(): model_gen_seq.shape_list[0][0], model_gen_seq.shape_list[0][1], int(model_gen_seq.shape_list[0][2] * image_size_times / 1), - int(model_gen_seq.shape_list[0][3] * image_size_times / num_spatial_parts), + int(model_gen_seq.shape_list[0][3] * image_size_times / spatial_part_size), ), model_gen_seq.shape_list[1], ] @@ -205,7 +211,7 @@ def verify_config(): ( model_gen_seq.shape_list[0][0], model_gen_seq.shape_list[0][1], - int(model_gen_seq.shape_list[0][2] * image_size_times / num_spatial_parts), + int(model_gen_seq.shape_list[0][2] * image_size_times / spatial_part_size), int(model_gen_seq.shape_list[0][3] * image_size_times / 1), ), model_gen_seq.shape_list[1], @@ -220,7 +226,7 @@ def verify_config(): model, balance = resnet_cifar_torch_spatial.get_resnet_v2( input_shape=(batch_size / parts, 3, image_size, image_size), depth=get_depth(2, 12), - local_rank=local_rank % num_spatial_parts, + local_rank=local_rank % spatial_part_size, mp_size=split_size, balance=balance, spatial_size=spatial_size, @@ -233,7 +239,7 @@ def verify_config(): model = resnet_cifar_torch_spatial.get_resnet_v2( input_shape=(batch_size / parts, 3, image_size, image_size), depth=get_depth(2, 12), - local_rank=local_rank % num_spatial_parts, + local_rank=local_rank % spatial_part_size, mp_size=split_size, balance=balance, spatial_size=spatial_size, @@ -257,7 +263,7 @@ def verify_config(): print("Shape list", resnet_shapes_list) -if local_rank == num_spatial_parts: +if local_rank == spatial_part_size: print(model_gen.models) @@ -337,13 +343,13 @@ def verify_config(): def split_input(inputs): if args.slice_method == "square": - image_height_local = int(image_size / math.sqrt(num_spatial_parts)) - image_width_local = int(image_size / math.sqrt(num_spatial_parts)) + image_height_local = int(image_size / math.sqrt(spatial_part_size)) + image_width_local = int(image_size / math.sqrt(spatial_part_size)) - total_rows = int(math.sqrt(num_spatial_parts)) - total_cols = int(math.sqrt(num_spatial_parts)) + total_rows = int(math.sqrt(spatial_part_size)) + total_cols = int(math.sqrt(spatial_part_size)) - # current position of rank in matrix of math.sqrt(num_spatial_parts) * math.sqrt(num_spatial_parts) + # current position of rank in matrix of math.sqrt(spatial_part_size) * math.sqrt(spatial_part_size) row = int(local_rank / total_cols) col = int(local_rank % total_cols) @@ -356,13 +362,13 @@ def split_input(inputs): return inputs[:, :, start_top:end_bottom, start_left:end_right] elif args.slice_method == "vertical": - image_height_local = int(image_size / num_spatial_parts) - image_width_local = int(image_size / num_spatial_parts) + image_height_local = int(image_size / spatial_part_size) + image_width_local = int(image_size / spatial_part_size) start_left = local_rank * image_width_local end_right = (local_rank + 1) * image_width_local - if local_rank == num_spatial_parts - 1: + if local_rank == spatial_part_size - 1: # In case of GPU count, partition size will be uneven and last # rank will receive remaining image return inputs[:, :, :, start_left:] @@ -370,13 +376,13 @@ def split_input(inputs): return inputs[:, :, :, start_left:end_right] elif args.slice_method == "horizontal": - image_height_local = int(image_size / num_spatial_parts) - image_width_local = int(image_size / num_spatial_parts) + image_height_local = int(image_size / spatial_part_size) + image_width_local = int(image_size / spatial_part_size) start_top = local_rank * image_height_local end_bottom = (local_rank + 1) * image_height_local - if local_rank == num_spatial_parts - 1: + if local_rank == spatial_part_size - 1: # In case of odd GPU count, partition size will be uneven and last # rank will receive remaining image return inputs[:, :, start_top:, :] @@ -397,7 +403,7 @@ def run_epoch(): break inputs, labels = data - if local_rank < num_spatial_parts_list[0]: + if local_rank < spatial_part_size: x = split_input(inputs) else: x = inputs @@ -405,14 +411,14 @@ def run_epoch(): temp_loss, temp_correct = t_s.run_step(x, labels) loss += temp_loss correct += temp_correct - if local_rank < spatial_size * num_spatial_parts: + if local_rank < spatial_size * spatial_part_size: sync_allreduce.apply_allreduce( model_gen, mpi_comm.spatial_allreduce_grp ) torch.cuda.synchronize() t_s.update() - if local_rank == num_spatial_parts: + if local_rank == spatial_part_size: logging.info( f"Step :{i}, LOSS: {temp_loss}, Global loss: {loss/(i+1)} Acc: {temp_correct}" ) @@ -425,7 +431,7 @@ def run_epoch(): perf.append(batch_size / t) t = time.time() - if local_rank == num_spatial_parts: + if local_rank == spatial_part_size: print("epoch", i_e, " Global loss:", loss, " acc", correct / i) diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..291ab5f5 --- /dev/null +++ b/setup.py @@ -0,0 +1,3 @@ +from setuptools import setup, find_packages + +setup(name='torch-gems', version='1.0', packages=find_packages()) diff --git a/torchgems/parser.py b/torchgems/parser.py index a2655733..31e69787 100644 --- a/torchgems/parser.py +++ b/torchgems/parser.py @@ -6,6 +6,14 @@ def get_parser(): description="MP-DP ResNet Script", formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) + + parser.add_argument( + "-v", + "--verbose", + help="Prints performance numbers or logs", + action="store_true", + ) + parser.add_argument( "--fp16-allreduce", action="store_true", @@ -124,5 +132,12 @@ def get_parser(): default="square", help="Slice method (square, vertical, and horizontal) in Spatial parallelism", ) + parser.add_argument( + "--app", + type=int, + default=3, + help="Application type (1.medical, 2.cifar, and synthetic) in Spatial parallelism", + ) + parser.set_defaults(enable_dp=False) return parser From 7ea08b98911f50cba9cc0c9580d1764c5499cf7a Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Mon, 3 Jul 2023 16:58:21 -0400 Subject: [PATCH 02/18] LP for amoebaNet and resNet Signed-off-by: Radha Guhane --- benchmarks/layer_parallelism/amoebanet_lp.py | 175 ++++++++++++++++++ benchmarks/layer_parallelism/resnet_lp.py | 183 +++++++++++++++++++ 2 files changed, 358 insertions(+) create mode 100644 benchmarks/layer_parallelism/amoebanet_lp.py create mode 100644 benchmarks/layer_parallelism/resnet_lp.py diff --git a/benchmarks/layer_parallelism/amoebanet_lp.py b/benchmarks/layer_parallelism/amoebanet_lp.py new file mode 100644 index 00000000..763604f1 --- /dev/null +++ b/benchmarks/layer_parallelism/amoebanet_lp.py @@ -0,0 +1,175 @@ +import torch.nn as nn +import torch.optim as optim +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torchvision.transforms as transforms +import torchvision +import numpy as np +import time +import sys +import math +from torchgems import parser +import time +from torchgems.mp_pipeline import model_generator, train_model +from models import amoebanet +import torchgems.comm as gems_comm + + + +parser_obj = parser.get_parser() +args = parser_obj.parse_args() + +gems_comm.initialize_cuda() + +class Unbuffered(object): + def __init__(self, stream): + self.stream = stream + def write(self, data): + self.stream.write(data) + self.stream.flush() + def writelines(self, datas): + self.stream.writelines(datas) + self.stream.flush() + def __getattr__(self, attr): + return getattr(self.stream, attr) + +sys.stdout = Unbuffered(sys.stdout) + +np.random.seed(seed=1405) +parts =args.parts +batch_size = args.batch_size +resnet_n = 18 +epoch=args.num_epochs +ENABLE_ASYNC=True +ENABLE_APP=False +amoebanet_test = False +image_size = int(args.image_size) +print("image size", image_size) +steps = 100 +num_layers = args.num_layers +num_filters = args.num_filters +balance = args.balance +mp_size = args.split_size +image_size_seq = 512 +times = 1 +num_classes=1000 + +mpi_comm = gems_comm.MPIComm(split_size=mp_size ,ENABLE_MASTER=False) +rank = mpi_comm.rank + +local_rank = rank % mp_size + +if(balance is not None): + balance = [int(i) for i in balance.split(',')] + +model = amoebanet.amoebanetd(num_classes=1000, num_layers=args.num_layers, num_filters=args.num_filters) + +model_gen = model_generator( + model=model, + split_size=mp_size, + input_size=(int(batch_size / parts), 3, image_size_seq, image_size_seq), + balance=balance +) +model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) + + +image_size_times = int(image_size / image_size_seq) +resnet_shapes_list = [] +for output_shape in model_gen.shape_list: + if(isinstance(output_shape, list)): + temp_shape = [] + for shape_tuple in output_shape: + + + x = (shape_tuple[0], shape_tuple[1], int(shape_tuple[2]* image_size_times), int(shape_tuple[3]* image_size_times) ) + temp_shape.append(x) + resnet_shapes_list.append(temp_shape) + else: + + if(len(output_shape) == 2): + resnet_shapes_list.append(output_shape) + else: + + x = (output_shape[0], output_shape[1], int(output_shape[2]* image_size_times), int(output_shape[3]* image_size_times) ) + resnet_shapes_list.append(x) + +model_gen.shape_list = resnet_shapes_list +print("local_ran:",local_rank," Shapes:",model_gen.shape_list) + + +del model_gen +del model +torch.cuda.ipc_collect() + +model = amoebanet.amoebanetd(num_classes=1000, num_layers=args.num_layers, num_filters=args.num_filters) + + +model_gen = model_generator(model=model, split_size=mp_size,input_size=(int(batch_size/parts),3,image_size,image_size),balance=balance, shape_list=resnet_shapes_list) +model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) + +tm = train_model(model_gen, local_rank,batch_size,epoch,criterion=None,optimizer=None,parts=parts,ASYNC=ENABLE_ASYNC) + +#Dataset +transform = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) +torch.manual_seed(0) +if(ENABLE_APP==True): + trainset = torchvision.datasets.ImageFolder("/train", transform=transform, target_transform=None) + my_dataloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, + shuffle=True, num_workers=0,pin_memory=True) +else: + my_dataset = torchvision.datasets.FakeData( + size=10 * batch_size, + image_size=(3, image_size, image_size), + num_classes=num_classes, + transform=transform, + target_transform=None, + random_offset=0, + ) + my_dataloader = torch.utils.data.DataLoader( + my_dataset, + batch_size=batch_size * times, + shuffle=False, + num_workers=0, + pin_memory=True, + ) + size_dataset = 10 * batch_size + +perf = [] + +def run_epoch(): + + for i_e in range(epoch): + loss =0 + t = time.time() + for i, data in enumerate(my_dataloader, 0): + start_event = torch.cuda.Event(enable_timing=True, blocking=True) + end_event = torch.cuda.Event(enable_timing=True, blocking=True) + start_event.record() + + if i > math.floor(size_dataset / (times * batch_size)) - 1: + break + inputs, labels = data + + temp_loss = tm.run_step(inputs, labels) + tm.update() + + end_event.record() + torch.cuda.synchronize() + t = start_event.elapsed_time(end_event) / 1000 + + if(local_rank==mp_size-1): + None + + if(local_rank==0): + print("Epoch: {} images per sec:{}".format(i_e,batch_size/t)) + perf.append(batch_size/t) + + t = time.time() + +run_epoch() + +if(local_rank==0): + print("Mean {} Median {}".format(sum(perf)/len(perf), np.median(perf))) diff --git a/benchmarks/layer_parallelism/resnet_lp.py b/benchmarks/layer_parallelism/resnet_lp.py new file mode 100644 index 00000000..cc5340ac --- /dev/null +++ b/benchmarks/layer_parallelism/resnet_lp.py @@ -0,0 +1,183 @@ +import torch.nn as nn +import torch.optim as optim +import torch +import torch.distributed as dist +import torch.nn.functional as F +import torchvision.transforms as transforms +import torchvision +import numpy as np +import time +import sys +import math +from torchgems import parser +import time +from torchgems.mp_pipeline import model_generator, train_model +from models import resnet_cifar_torch +import torchgems.comm as gems_comm + + + +parser_obj = parser.get_parser() +args = parser_obj.parse_args() + +gems_comm.initialize_cuda() + +class Unbuffered(object): + def __init__(self, stream): + self.stream = stream + def write(self, data): + self.stream.write(data) + self.stream.flush() + def writelines(self, datas): + self.stream.writelines(datas) + self.stream.flush() + def __getattr__(self, attr): + return getattr(self.stream, attr) + +sys.stdout = Unbuffered(sys.stdout) + +np.random.seed(seed=1405) +parts =args.parts +batch_size = args.batch_size +resnet_n = 12 +epoch=args.num_epochs +ENABLE_ASYNC=True +ENABLE_APP=False +amoebanet_test = False +image_size = int(args.image_size) #1024 +print("image size", image_size) +steps = 100 +num_layers = args.num_layers +num_filters = args.num_filters +balance = args.balance +mp_size = args.split_size +image_size_seq = 32 +times = 1 +num_classes=10 + +mpi_comm = gems_comm.MPIComm(split_size=mp_size ,ENABLE_MASTER=False) +rank = mpi_comm.rank + +local_rank = rank % mp_size + +if(balance is not None): + balance = [int(i) for i in balance.split(',')] + +def get_depth(version,n): + if version == 1: + return n * 6 + 2 + elif version == 2: + return n * 9 + 2 + +model = resnet_cifar_torch.get_resnet_v2((int(batch_size / parts), 3, image_size_seq, image_size_seq), depth=get_depth(2, resnet_n)) + +mul_shape = int(args.image_size/image_size_seq) + +model_gen = model_generator( + model=model, + split_size=mp_size, + input_size=(int(batch_size / parts), 3, image_size_seq, image_size_seq), + balance=balance, +) +model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) + + +image_size_times = int(image_size / image_size_seq) +resnet_shapes_list = [] +for output_shape in model_gen.shape_list: + if(isinstance(output_shape, list)): + temp_shape = [] + for shape_tuple in output_shape: + + + x = (shape_tuple[0], shape_tuple[1], int(shape_tuple[2]* image_size_times), int(shape_tuple[3]* image_size_times) ) + temp_shape.append(x) + resnet_shapes_list.append(temp_shape) + else: + + if(len(output_shape) == 2): + resnet_shapes_list.append(output_shape) + else: + + x = (output_shape[0], output_shape[1], int(output_shape[2]* image_size_times), int(output_shape[3]* image_size_times) ) + resnet_shapes_list.append(x) + +model_gen.shape_list = resnet_shapes_list +print("local_ran:",local_rank," Shapes:",model_gen.shape_list) + + +del model_gen +del model +torch.cuda.ipc_collect() + +model = resnet_cifar_torch.get_resnet_v2((int(batch_size/parts),3,image_size,image_size),get_depth(2, resnet_n)) + +model_gen = model_generator(model=model, split_size=mp_size,input_size=(int(batch_size/parts),3,image_size,image_size),balance=balance, shape_list=resnet_shapes_list) +model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) + +tm = train_model(model_gen, local_rank,batch_size,epoch,criterion=None,optimizer=None,parts=parts,ASYNC=ENABLE_ASYNC) + +#Dataset +transform = transforms.Compose( + [transforms.ToTensor(), + transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) +torch.manual_seed(0) +if(ENABLE_APP==True): + trainset = torchvision.datasets.ImageFolder("/usr/workspace/jain8/project/cancer/1024_1024_5/train", transform=transform, target_transform=None) + my_dataloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, + shuffle=True, num_workers=0,pin_memory=True) +else: + my_dataset = torchvision.datasets.FakeData( + size=10 * batch_size, + image_size=(3, image_size, image_size), + num_classes=num_classes, + transform=transform, + target_transform=None, + random_offset=0, + ) + my_dataloader = torch.utils.data.DataLoader( + my_dataset, + batch_size=batch_size * times, + shuffle=False, + num_workers=0, + pin_memory=True, + ) + size_dataset = 10 * batch_size + +perf = [] + +def run_epoch(): + + for i_e in range(epoch): + loss =0 + t = time.time() + for i, data in enumerate(my_dataloader, 0): + start_event = torch.cuda.Event(enable_timing=True, blocking=True) + end_event = torch.cuda.Event(enable_timing=True, blocking=True) + start_event.record() + + if i > math.floor(size_dataset / (times * batch_size)) - 1: + break + + inputs, labels = data + + temp_loss = tm.run_step(inputs, labels) + tm.update() + + end_event.record() + torch.cuda.synchronize() + t = start_event.elapsed_time(end_event) / 1000 + + if(local_rank==mp_size-1): + None + + if(local_rank==0): + print("Epoch: {} images per sec:{}".format(i_e,batch_size/t)) + perf.append(batch_size/t) + + t = time.time() + +run_epoch() + +if(local_rank==0): + print("Mean {} Median {}".format(sum(perf)/len(perf), np.median(perf))) From bbf1711ee41aa8fb12344a7cbcea1134eb087595 Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Mon, 3 Jul 2023 17:20:13 -0400 Subject: [PATCH 03/18] code refactoring Signed-off-by: Radha Guhane --- benchmarks/layer_parallelism/amoebanet_lp.py | 218 +++++++++-------- benchmarks/layer_parallelism/resnet_lp.py | 235 +++++++++++-------- setup.py | 2 +- 3 files changed, 260 insertions(+), 195 deletions(-) diff --git a/benchmarks/layer_parallelism/amoebanet_lp.py b/benchmarks/layer_parallelism/amoebanet_lp.py index 763604f1..9e0171fb 100644 --- a/benchmarks/layer_parallelism/amoebanet_lp.py +++ b/benchmarks/layer_parallelism/amoebanet_lp.py @@ -1,20 +1,14 @@ -import torch.nn as nn -import torch.optim as optim import torch -import torch.distributed as dist -import torch.nn.functional as F import torchvision.transforms as transforms import torchvision import numpy as np -import time import sys import math from torchgems import parser import time from torchgems.mp_pipeline import model_generator, train_model -from models import amoebanet -import torchgems.comm as gems_comm - +from models import amoebanet +import torchgems.comm as gems_comm parser_obj = parser.get_parser() @@ -22,27 +16,32 @@ gems_comm.initialize_cuda() + class Unbuffered(object): - def __init__(self, stream): - self.stream = stream - def write(self, data): - self.stream.write(data) - self.stream.flush() - def writelines(self, datas): - self.stream.writelines(datas) - self.stream.flush() - def __getattr__(self, attr): - return getattr(self.stream, attr) + def __init__(self, stream): + self.stream = stream + + def write(self, data): + self.stream.write(data) + self.stream.flush() + + def writelines(self, datas): + self.stream.writelines(datas) + self.stream.flush() + + def __getattr__(self, attr): + return getattr(self.stream, attr) + sys.stdout = Unbuffered(sys.stdout) np.random.seed(seed=1405) -parts =args.parts +parts = args.parts batch_size = args.batch_size resnet_n = 18 -epoch=args.num_epochs -ENABLE_ASYNC=True -ENABLE_APP=False +epoch = args.num_epochs +ENABLE_ASYNC = True +ENABLE_APP = False amoebanet_test = False image_size = int(args.image_size) print("image size", image_size) @@ -53,23 +52,25 @@ def __getattr__(self, attr): mp_size = args.split_size image_size_seq = 512 times = 1 -num_classes=1000 +num_classes = 1000 -mpi_comm = gems_comm.MPIComm(split_size=mp_size ,ENABLE_MASTER=False) +mpi_comm = gems_comm.MPIComm(split_size=mp_size, ENABLE_MASTER=False) rank = mpi_comm.rank local_rank = rank % mp_size -if(balance is not None): - balance = [int(i) for i in balance.split(',')] +if balance is not None: + balance = [int(i) for i in balance.split(",")] -model = amoebanet.amoebanetd(num_classes=1000, num_layers=args.num_layers, num_filters=args.num_filters) +model = amoebanet.amoebanetd( + num_classes=1000, num_layers=args.num_layers, num_filters=args.num_filters +) model_gen = model_generator( model=model, split_size=mp_size, input_size=(int(batch_size / parts), 3, image_size_seq, image_size_seq), - balance=balance + balance=balance, ) model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) @@ -77,99 +78,128 @@ def __getattr__(self, attr): image_size_times = int(image_size / image_size_seq) resnet_shapes_list = [] for output_shape in model_gen.shape_list: - if(isinstance(output_shape, list)): - temp_shape = [] - for shape_tuple in output_shape: - - - x = (shape_tuple[0], shape_tuple[1], int(shape_tuple[2]* image_size_times), int(shape_tuple[3]* image_size_times) ) - temp_shape.append(x) - resnet_shapes_list.append(temp_shape) - else: - - if(len(output_shape) == 2): - resnet_shapes_list.append(output_shape) - else: - - x = (output_shape[0], output_shape[1], int(output_shape[2]* image_size_times), int(output_shape[3]* image_size_times) ) - resnet_shapes_list.append(x) + if isinstance(output_shape, list): + temp_shape = [] + for shape_tuple in output_shape: + + x = ( + shape_tuple[0], + shape_tuple[1], + int(shape_tuple[2] * image_size_times), + int(shape_tuple[3] * image_size_times), + ) + temp_shape.append(x) + resnet_shapes_list.append(temp_shape) + else: + + if len(output_shape) == 2: + resnet_shapes_list.append(output_shape) + else: + + x = ( + output_shape[0], + output_shape[1], + int(output_shape[2] * image_size_times), + int(output_shape[3] * image_size_times), + ) + resnet_shapes_list.append(x) model_gen.shape_list = resnet_shapes_list -print("local_ran:",local_rank," Shapes:",model_gen.shape_list) +print("local_ran:", local_rank, " Shapes:", model_gen.shape_list) del model_gen del model torch.cuda.ipc_collect() -model = amoebanet.amoebanetd(num_classes=1000, num_layers=args.num_layers, num_filters=args.num_filters) +model = amoebanet.amoebanetd( + num_classes=1000, num_layers=args.num_layers, num_filters=args.num_filters +) -model_gen = model_generator(model=model, split_size=mp_size,input_size=(int(batch_size/parts),3,image_size,image_size),balance=balance, shape_list=resnet_shapes_list) +model_gen = model_generator( + model=model, + split_size=mp_size, + input_size=(int(batch_size / parts), 3, image_size, image_size), + balance=balance, + shape_list=resnet_shapes_list, +) model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) -tm = train_model(model_gen, local_rank,batch_size,epoch,criterion=None,optimizer=None,parts=parts,ASYNC=ENABLE_ASYNC) +tm = train_model( + model_gen, + local_rank, + batch_size, + epoch, + criterion=None, + optimizer=None, + parts=parts, + ASYNC=ENABLE_ASYNC, +) -#Dataset +# Dataset transform = transforms.Compose( - [transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] +) torch.manual_seed(0) -if(ENABLE_APP==True): - trainset = torchvision.datasets.ImageFolder("/train", transform=transform, target_transform=None) - my_dataloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, - shuffle=True, num_workers=0,pin_memory=True) +if ENABLE_APP: + trainset = torchvision.datasets.ImageFolder( + "/train", transform=transform, target_transform=None + ) + my_dataloader = torch.utils.data.DataLoader( + trainset, batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True + ) else: - my_dataset = torchvision.datasets.FakeData( - size=10 * batch_size, - image_size=(3, image_size, image_size), - num_classes=num_classes, - transform=transform, - target_transform=None, - random_offset=0, - ) - my_dataloader = torch.utils.data.DataLoader( - my_dataset, - batch_size=batch_size * times, - shuffle=False, - num_workers=0, - pin_memory=True, - ) - size_dataset = 10 * batch_size + my_dataset = torchvision.datasets.FakeData( + size=10 * batch_size, + image_size=(3, image_size, image_size), + num_classes=num_classes, + transform=transform, + target_transform=None, + random_offset=0, + ) + my_dataloader = torch.utils.data.DataLoader( + my_dataset, + batch_size=batch_size * times, + shuffle=False, + num_workers=0, + pin_memory=True, + ) + size_dataset = 10 * batch_size perf = [] + def run_epoch(): - for i_e in range(epoch): - loss =0 - t = time.time() - for i, data in enumerate(my_dataloader, 0): - start_event = torch.cuda.Event(enable_timing=True, blocking=True) - end_event = torch.cuda.Event(enable_timing=True, blocking=True) - start_event.record() + for i_e in range(epoch): + t = time.time() + for i, data in enumerate(my_dataloader, 0): + start_event = torch.cuda.Event(enable_timing=True, blocking=True) + end_event = torch.cuda.Event(enable_timing=True, blocking=True) + start_event.record() + + if i > math.floor(size_dataset / (times * batch_size)) - 1: + break + inputs, labels = data - if i > math.floor(size_dataset / (times * batch_size)) - 1: - break - inputs, labels = data + tm.update() - temp_loss = tm.run_step(inputs, labels) - tm.update() + end_event.record() + torch.cuda.synchronize() + t = start_event.elapsed_time(end_event) / 1000 - end_event.record() - torch.cuda.synchronize() - t = start_event.elapsed_time(end_event) / 1000 + if local_rank == mp_size - 1: + None - if(local_rank==mp_size-1): - None + if local_rank == 0: + print("Epoch: {} images per sec:{}".format(i_e, batch_size / t)) + perf.append(batch_size / t) - if(local_rank==0): - print("Epoch: {} images per sec:{}".format(i_e,batch_size/t)) - perf.append(batch_size/t) + t = time.time() - t = time.time() run_epoch() -if(local_rank==0): - print("Mean {} Median {}".format(sum(perf)/len(perf), np.median(perf))) +if local_rank == 0: + print("Mean {} Median {}".format(sum(perf) / len(perf), np.median(perf))) diff --git a/benchmarks/layer_parallelism/resnet_lp.py b/benchmarks/layer_parallelism/resnet_lp.py index cc5340ac..1c5f57d2 100644 --- a/benchmarks/layer_parallelism/resnet_lp.py +++ b/benchmarks/layer_parallelism/resnet_lp.py @@ -1,20 +1,14 @@ -import torch.nn as nn -import torch.optim as optim import torch -import torch.distributed as dist -import torch.nn.functional as F import torchvision.transforms as transforms import torchvision import numpy as np -import time import sys import math from torchgems import parser import time from torchgems.mp_pipeline import model_generator, train_model from models import resnet_cifar_torch -import torchgems.comm as gems_comm - +import torchgems.comm as gems_comm parser_obj = parser.get_parser() @@ -22,29 +16,34 @@ gems_comm.initialize_cuda() + class Unbuffered(object): - def __init__(self, stream): - self.stream = stream - def write(self, data): - self.stream.write(data) - self.stream.flush() - def writelines(self, datas): - self.stream.writelines(datas) - self.stream.flush() - def __getattr__(self, attr): - return getattr(self.stream, attr) + def __init__(self, stream): + self.stream = stream + + def write(self, data): + self.stream.write(data) + self.stream.flush() + + def writelines(self, datas): + self.stream.writelines(datas) + self.stream.flush() + + def __getattr__(self, attr): + return getattr(self.stream, attr) + sys.stdout = Unbuffered(sys.stdout) np.random.seed(seed=1405) -parts =args.parts +parts = args.parts batch_size = args.batch_size resnet_n = 12 -epoch=args.num_epochs -ENABLE_ASYNC=True -ENABLE_APP=False +epoch = args.num_epochs +ENABLE_ASYNC = True +ENABLE_APP = False amoebanet_test = False -image_size = int(args.image_size) #1024 +image_size = int(args.image_size) # 1024 print("image size", image_size) steps = 100 num_layers = args.num_layers @@ -53,25 +52,30 @@ def __getattr__(self, attr): mp_size = args.split_size image_size_seq = 32 times = 1 -num_classes=10 +num_classes = 10 -mpi_comm = gems_comm.MPIComm(split_size=mp_size ,ENABLE_MASTER=False) +mpi_comm = gems_comm.MPIComm(split_size=mp_size, ENABLE_MASTER=False) rank = mpi_comm.rank local_rank = rank % mp_size -if(balance is not None): - balance = [int(i) for i in balance.split(',')] +if balance is not None: + balance = [int(i) for i in balance.split(",")] + + +def get_depth(version, n): + if version == 1: + return n * 6 + 2 + elif version == 2: + return n * 9 + 2 -def get_depth(version,n): - if version == 1: - return n * 6 + 2 - elif version == 2: - return n * 9 + 2 -model = resnet_cifar_torch.get_resnet_v2((int(batch_size / parts), 3, image_size_seq, image_size_seq), depth=get_depth(2, resnet_n)) +model = resnet_cifar_torch.get_resnet_v2( + (int(batch_size / parts), 3, image_size_seq, image_size_seq), + depth=get_depth(2, resnet_n), +) -mul_shape = int(args.image_size/image_size_seq) +mul_shape = int(args.image_size / image_size_seq) model_gen = model_generator( model=model, @@ -85,99 +89,130 @@ def get_depth(version,n): image_size_times = int(image_size / image_size_seq) resnet_shapes_list = [] for output_shape in model_gen.shape_list: - if(isinstance(output_shape, list)): - temp_shape = [] - for shape_tuple in output_shape: - - - x = (shape_tuple[0], shape_tuple[1], int(shape_tuple[2]* image_size_times), int(shape_tuple[3]* image_size_times) ) - temp_shape.append(x) - resnet_shapes_list.append(temp_shape) - else: - - if(len(output_shape) == 2): - resnet_shapes_list.append(output_shape) - else: - - x = (output_shape[0], output_shape[1], int(output_shape[2]* image_size_times), int(output_shape[3]* image_size_times) ) - resnet_shapes_list.append(x) + if isinstance(output_shape, list): + temp_shape = [] + for shape_tuple in output_shape: + + x = ( + shape_tuple[0], + shape_tuple[1], + int(shape_tuple[2] * image_size_times), + int(shape_tuple[3] * image_size_times), + ) + temp_shape.append(x) + resnet_shapes_list.append(temp_shape) + else: + + if len(output_shape) == 2: + resnet_shapes_list.append(output_shape) + else: + + x = ( + output_shape[0], + output_shape[1], + int(output_shape[2] * image_size_times), + int(output_shape[3] * image_size_times), + ) + resnet_shapes_list.append(x) model_gen.shape_list = resnet_shapes_list -print("local_ran:",local_rank," Shapes:",model_gen.shape_list) +print("local_ran:", local_rank, " Shapes:", model_gen.shape_list) del model_gen del model torch.cuda.ipc_collect() -model = resnet_cifar_torch.get_resnet_v2((int(batch_size/parts),3,image_size,image_size),get_depth(2, resnet_n)) +model = resnet_cifar_torch.get_resnet_v2( + (int(batch_size / parts), 3, image_size, image_size), get_depth(2, resnet_n) +) -model_gen = model_generator(model=model, split_size=mp_size,input_size=(int(batch_size/parts),3,image_size,image_size),balance=balance, shape_list=resnet_shapes_list) +model_gen = model_generator( + model=model, + split_size=mp_size, + input_size=(int(batch_size / parts), 3, image_size, image_size), + balance=balance, + shape_list=resnet_shapes_list, +) model_gen.ready_model(split_rank=local_rank, GET_SHAPES_ON_CUDA=True) -tm = train_model(model_gen, local_rank,batch_size,epoch,criterion=None,optimizer=None,parts=parts,ASYNC=ENABLE_ASYNC) +tm = train_model( + model_gen, + local_rank, + batch_size, + epoch, + criterion=None, + optimizer=None, + parts=parts, + ASYNC=ENABLE_ASYNC, +) -#Dataset +# Dataset transform = transforms.Compose( - [transforms.ToTensor(), - transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]) + [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] +) torch.manual_seed(0) -if(ENABLE_APP==True): - trainset = torchvision.datasets.ImageFolder("/usr/workspace/jain8/project/cancer/1024_1024_5/train", transform=transform, target_transform=None) - my_dataloader = torch.utils.data.DataLoader(trainset, batch_size=batch_size, - shuffle=True, num_workers=0,pin_memory=True) +if ENABLE_APP: + trainset = torchvision.datasets.ImageFolder( + "/usr/workspace/jain8/project/cancer/1024_1024_5/train", + transform=transform, + target_transform=None, + ) + my_dataloader = torch.utils.data.DataLoader( + trainset, batch_size=batch_size, shuffle=True, num_workers=0, pin_memory=True + ) else: - my_dataset = torchvision.datasets.FakeData( - size=10 * batch_size, - image_size=(3, image_size, image_size), - num_classes=num_classes, - transform=transform, - target_transform=None, - random_offset=0, - ) - my_dataloader = torch.utils.data.DataLoader( - my_dataset, - batch_size=batch_size * times, - shuffle=False, - num_workers=0, - pin_memory=True, - ) - size_dataset = 10 * batch_size + my_dataset = torchvision.datasets.FakeData( + size=10 * batch_size, + image_size=(3, image_size, image_size), + num_classes=num_classes, + transform=transform, + target_transform=None, + random_offset=0, + ) + my_dataloader = torch.utils.data.DataLoader( + my_dataset, + batch_size=batch_size * times, + shuffle=False, + num_workers=0, + pin_memory=True, + ) + size_dataset = 10 * batch_size perf = [] + def run_epoch(): - for i_e in range(epoch): - loss =0 - t = time.time() - for i, data in enumerate(my_dataloader, 0): - start_event = torch.cuda.Event(enable_timing=True, blocking=True) - end_event = torch.cuda.Event(enable_timing=True, blocking=True) - start_event.record() + for i_e in range(epoch): + t = time.time() + for i, data in enumerate(my_dataloader, 0): + start_event = torch.cuda.Event(enable_timing=True, blocking=True) + end_event = torch.cuda.Event(enable_timing=True, blocking=True) + start_event.record() + + if i > math.floor(size_dataset / (times * batch_size)) - 1: + break + + inputs, labels = data - if i > math.floor(size_dataset / (times * batch_size)) - 1: - break + tm.update() - inputs, labels = data - - temp_loss = tm.run_step(inputs, labels) - tm.update() + end_event.record() + torch.cuda.synchronize() + t = start_event.elapsed_time(end_event) / 1000 - end_event.record() - torch.cuda.synchronize() - t = start_event.elapsed_time(end_event) / 1000 + if local_rank == mp_size - 1: + None - if(local_rank==mp_size-1): - None + if local_rank == 0: + print("Epoch: {} images per sec:{}".format(i_e, batch_size / t)) + perf.append(batch_size / t) - if(local_rank==0): - print("Epoch: {} images per sec:{}".format(i_e,batch_size/t)) - perf.append(batch_size/t) + t = time.time() - t = time.time() run_epoch() -if(local_rank==0): - print("Mean {} Median {}".format(sum(perf)/len(perf), np.median(perf))) +if local_rank == 0: + print("Mean {} Median {}".format(sum(perf) / len(perf), np.median(perf))) diff --git a/setup.py b/setup.py index 291ab5f5..07391670 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,3 @@ from setuptools import setup, find_packages -setup(name='torch-gems', version='1.0', packages=find_packages()) +setup(name="torch-gems", version="1.0", packages=find_packages()) From 9bb8d7a36a762a6221bdef96d323af5c6e8c20da Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Mon, 3 Jul 2023 17:38:38 -0400 Subject: [PATCH 04/18] Refactoring and adding version for black Signed-off-by: Radha Guhane --- .pre-commit-config.yaml | 2 +- benchmarks/layer_parallelism/amoebanet_lp.py | 4 ---- benchmarks/layer_parallelism/resnet_lp.py | 4 ---- benchmarks/spatial/model/amoebanet_run.py | 8 +++++--- benchmarks/spatial/model/resnet_model.py | 9 +++++---- 5 files changed, 11 insertions(+), 16 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 07ee79c2..0056c15c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/psf/black - rev: stable + rev: 23.3.0 hooks: - id: black name: black-format-test diff --git a/benchmarks/layer_parallelism/amoebanet_lp.py b/benchmarks/layer_parallelism/amoebanet_lp.py index 9e0171fb..990d48e1 100644 --- a/benchmarks/layer_parallelism/amoebanet_lp.py +++ b/benchmarks/layer_parallelism/amoebanet_lp.py @@ -81,7 +81,6 @@ def __getattr__(self, attr): if isinstance(output_shape, list): temp_shape = [] for shape_tuple in output_shape: - x = ( shape_tuple[0], shape_tuple[1], @@ -91,11 +90,9 @@ def __getattr__(self, attr): temp_shape.append(x) resnet_shapes_list.append(temp_shape) else: - if len(output_shape) == 2: resnet_shapes_list.append(output_shape) else: - x = ( output_shape[0], output_shape[1], @@ -171,7 +168,6 @@ def __getattr__(self, attr): def run_epoch(): - for i_e in range(epoch): t = time.time() for i, data in enumerate(my_dataloader, 0): diff --git a/benchmarks/layer_parallelism/resnet_lp.py b/benchmarks/layer_parallelism/resnet_lp.py index 1c5f57d2..f9c68de8 100644 --- a/benchmarks/layer_parallelism/resnet_lp.py +++ b/benchmarks/layer_parallelism/resnet_lp.py @@ -92,7 +92,6 @@ def get_depth(version, n): if isinstance(output_shape, list): temp_shape = [] for shape_tuple in output_shape: - x = ( shape_tuple[0], shape_tuple[1], @@ -102,11 +101,9 @@ def get_depth(version, n): temp_shape.append(x) resnet_shapes_list.append(temp_shape) else: - if len(output_shape) == 2: resnet_shapes_list.append(output_shape) else: - x = ( output_shape[0], output_shape[1], @@ -183,7 +180,6 @@ def get_depth(version, n): def run_epoch(): - for i_e in range(epoch): t = time.time() for i, data in enumerate(my_dataloader, 0): diff --git a/benchmarks/spatial/model/amoebanet_run.py b/benchmarks/spatial/model/amoebanet_run.py index 2bbec157..11dbd750 100644 --- a/benchmarks/spatial/model/amoebanet_run.py +++ b/benchmarks/spatial/model/amoebanet_run.py @@ -92,7 +92,7 @@ def get_depth(version, n): num_spatial_parts = [int(i) for i in temp_num_spatial_parts] num_spatial_parts_list = num_spatial_parts -spatial_part_size = num_spatial_parts_list[0] #Partition size for spatial parallelism +spatial_part_size = num_spatial_parts_list[0] # Partition size for spatial parallelism times = 1 num_classes = 1000 @@ -135,9 +135,11 @@ def verify_config(): assert isPowerTwo( int(image_size / spatial_part_size) ), "Image size of each partition should be power of Two" - + for each_part_size in num_spatial_parts_list: - assert each_part_size == spatial_part_size, "Size of each SP partition should be same" + assert ( + each_part_size == spatial_part_size + ), "Size of each SP partition should be same" verify_config() diff --git a/benchmarks/spatial/model/resnet_model.py b/benchmarks/spatial/model/resnet_model.py index f8c84ef2..d61c3425 100644 --- a/benchmarks/spatial/model/resnet_model.py +++ b/benchmarks/spatial/model/resnet_model.py @@ -89,7 +89,7 @@ def get_depth(version, n): num_spatial_parts = [int(i) for i in temp_num_spatial_parts] num_spatial_parts_list = num_spatial_parts -spatial_part_size = num_spatial_parts_list[0] #Partition size for spatial parallelism +spatial_part_size = num_spatial_parts_list[0] # Partition size for spatial parallelism times = 1 num_classes = 10 @@ -128,10 +128,11 @@ def verify_config(): assert isPowerTwo( int(image_size / spatial_part_size) ), "Image size of each partition should be power of Two" - - for each_part_size in num_spatial_parts_list: - assert each_part_size == spatial_part_size, "Size of each SP partition should be same" + for each_part_size in num_spatial_parts_list: + assert ( + each_part_size == spatial_part_size + ), "Size of each SP partition should be same" verify_config() From 1615663e115196213e7dcc73cb9bbdf5c246a888 Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Mon, 3 Jul 2023 17:47:06 -0400 Subject: [PATCH 05/18] Fix ResNet halo-D2 bug Signed-off-by: Radha Guhane --- models/resnet_cifar_torch_spatial_d2.py | 11 +++++++++++ torchgems/spatial_new.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/models/resnet_cifar_torch_spatial_d2.py b/models/resnet_cifar_torch_spatial_d2.py index 88452229..9dddd652 100644 --- a/models/resnet_cifar_torch_spatial_d2.py +++ b/models/resnet_cifar_torch_spatial_d2.py @@ -267,6 +267,14 @@ def forward(self, x): return x +def get_balance(num_layers, mp_size): + part_layer = int(num_layers / mp_size) + balance = [part_layer] * mp_size + # add remianing layers to last split if split is uneven + balance[mp_size - 1] = balance[mp_size - 1] + (num_layers - sum(balance)) + return balance + + def get_start_end_layer_index(num_layers, balance, mp_size, local_rank=0): # return the index of start and end layer for the model # based on the size of model parallelism and local rank of the process @@ -572,6 +580,9 @@ def get_resnet_v2( num_layers = num_res_blocks * 3 + 2 + if balance == None: + balance = get_balance(num_layers, mp_size) + _, end_layer = get_start_end_layer_index( num_layers, balance, mp_size, local_rank=spatial_size - 1 ) diff --git a/torchgems/spatial_new.py b/torchgems/spatial_new.py index db90f16b..7c245630 100644 --- a/torchgems/spatial_new.py +++ b/torchgems/spatial_new.py @@ -66,7 +66,7 @@ def __init__( elif self.spatial_local_rank == 1: padding_left, padding_right, padding_top, padding_bottom = ( self.halo_len_width, - padding, + padding[1], padding[0], self.halo_len_height, ) From 5a5f871b1197c3d93dd3da6927a6cdb7cd377d45 Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Fri, 7 Jul 2023 11:58:05 -0400 Subject: [PATCH 06/18] Update README.md Signed-off-by: Radha Guhane --- README.md | 20 ++++++++++++++++++ .../assets/images/AmeobaNet_img_size_1024.png | Bin 0 -> 12767 bytes .../assets/images/AmeobaNet_img_size_2048.png | Bin 0 -> 14147 bytes docs/assets/images/ResNet_img_size_1024.png | Bin 0 -> 12424 bytes docs/assets/images/ResNet_img_size_2048.png | Bin 0 -> 15597 bytes 5 files changed, 20 insertions(+) create mode 100644 docs/assets/images/AmeobaNet_img_size_1024.png create mode 100644 docs/assets/images/AmeobaNet_img_size_2048.png create mode 100644 docs/assets/images/ResNet_img_size_1024.png create mode 100644 docs/assets/images/ResNet_img_size_2048.png diff --git a/README.md b/README.md index 122dc338..4721bca0 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,26 @@ Example to run Amoebanet model with partition size for model as two, spatial par ```bash $MV2_HOME/bin/mpirun_rsh --export-all -np 5 --hostfile {$HOSTFILE} MV2_USE_GDRCOPY=0 MV2_ENABLE_AFFINITY=0 MV2_USE_CUDA=1 LD_PRELOAD=$MV2_HOME/lib/libmpi.so python benchmarks/spatial/model/amoebanet_run.py --image-size 512 --num-spatial-parts 4 --slice-method "vertical" --split-size 2 --spatial-size 1 ``` + +## Experimental Results: + +#### Using Spatial, Model and Pipeline Parallelism, where the model is split into two parts and utilizes spatial parallelism by dividing the image into four parts + +- AmeobaNet Model + +
+ + +
+ +- ResNet Model + +
+ + +
+ + ## References: 1. Arpan Jain, Ammar Ahmad Awan, Asmaa M. Aljuhani, Jahanzeb Maqbool Hashmi, Quentin G. Anthony, Hari Subramoni, Dhableswar K. Panda, Raghu Machiraju, and Anil Parwani. 2020. GEMS: GPU-enabled memory-aware model-parallelism system for distributed DNN training. In Proceedings of the International Conference for High Performance Computing, Networking, Storage and Analysis (SC '20). IEEE Press, Article 45, 1–15. 2. Arpan Jain, Aamir Shafi, Quentin Anthony, Pouya Kousha, Hari Subramoni, and Dhableswar K. Panda. 2022. Hy-Fi: Hybrid Five-Dimensional Parallel DNN Training on High-Performance GPU Clusters. In High Performance Computing: 37th International Conference, ISC High Performance 2022, Hamburg, Germany, May 29 – June 2, 2022, Proceedings. Springer-Verlag, Berlin, Heidelberg, 109–130. https://doi.org/10.1007/978-3-031-07312-0_6 diff --git a/docs/assets/images/AmeobaNet_img_size_1024.png b/docs/assets/images/AmeobaNet_img_size_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..58864657d54ae12ff107fbdf01e1695958c4ec3e GIT binary patch literal 12767 zcmb_@cT|(zx2Aq5f(A)IDH2LR1*I1uAP@*uDS{1(e;@mk=*9XdqPd!aGQbVh6=@~CAnGPlzY(KXSkElo(AiWU>|8up)#)b4K^*p@!2vTfQgnW^5B-#xJA=H?a?>6bsY+Z{KLTyd32)y`iV z^UZ~z+?AD;b4-7j2I(s+TX0=NK_JoRbulW+%GAm*CJ2NT{Lcl2|CbAi{Ey^zpSrbQ z*L(VU|98VYk8g#`Qr~JYn|*(ESE7&N@O9B*4vyZ8)Uy| z%qX`adrAFPSKcYJuu1)TwcFb0>hR|$y^Tet*N7J{%!Mlw8eZ=&(?KkgI|{UwXhb{S zOK7yLDq+6!Os0@mLLP%%5;XM=GPHW$!BuZ`=RQFm*br__Ccuq1KDJrp*{DUpejWaG z^A`)rV`H+)P6j;N?QWus>X~Q#Y6k^o;0fd6wV&<_LWJ-1J!%vx5F2aU$2gab7p#C}hhG;?yn5w$R?hf5H$Ezwuxi9#Iyi;Y%CPydc>U zYk7`LgqIWx9W02 ziZW|<;}CMX)(7mh=zKBSLr)%3ttmdnHo1j1ucAOas;(S+fvAla`dzCOe}A(%&Mt;l z(ygemW@#SGB6Bc?r!xcuOs&3)`m?B4~&;`rVC%=bGx(#hizv zO3Z~ji>VgqvBr73(>g&7L!H$re7GwYc=c@N+g@2V(deZG7K*-`@c-Bn?ijrGKv-{zM@6u_l--4gc(W57ElCS#9>9RCy5q}N-VIY=BqWKLC8@DqGjl2A4>sG<8 zBTYsBCJ-8}RPz{;bL)#L;{_fw3%QR z>?9K`3{J_&gp3pjr3t0{WqbwTv?_0BHR6J za->aeNNSJU*?l9%Io4}lc{sS0RCbG-@mE7_etTWU2e4eBsrs%)B9 zHVooeyPVr{WFGGsZ2j)?C|>zHemTThKov2+32~3x(k5_1g$cFAXqxI2@F!I!IWA2m zRI5(>14`-R87#$04{4A$-oJ|}dmd*a*bUoOgPDMdf!(US0gCVE%*b7zqC^Y|-fZR$ z*mj3*ke!1u@!_PK2#{+GZ&!RM$dvA>J#~f^h~K1hrz2qd3M}D+ z%@!h(EElvJMcy3ZsI|uj`P^9>)P;?bMZlpv+M>(y z(Tx;~bc~S>^cr>U6yd{hb3b|W3rlNoCyKiw3ml5kR2=K6GeR;YUIzlR#ly+CfbLrs z(FXrbA^U$te+yV3T>g0)f|&Z9lS>$}(|hR~iPvu|q!WM!!6;ZDixZvRyK(#J`dWxV zBV8Bo_k*Vf)6}>+jep&+gA9w}&LA1SYbL%MSIelj|2at`7IO^15c8MA$go%3jXmzx z5L9awty515Fl0w-K@dE2y%GtXeF#&1WANoUfI%!Y!(Lz@!!@*p8&(2*LGZmXl+DpX zys7;S_57g~*SZH+8}#qsIE%s7byhg9@2q%9KN|Z+gU}j-uV+IxBQJ|0UynS~yVvxl zxg$(ER1iN$x~1$DH}MsNQRfs01JsErQu{AK8;c!e%1p_~9`dRQv4!_4kEWjq9HF*u z4F;qPJCkzI_?!DZ0_GLrTh=#nx0Ef6h;~?HfJ8%_XN}xuJ3~imml~g>`mWoKtQ>nh z<@NaaBK3eAw>r){{erz=!~ak3eousaxpK*0$5_^7>v0S7ryvOGr8|^z=ngTPw8B*g zX&tjFr)pRGO=XYl@2!`l)%>|K(kMKmD_6B0Fd|Nvd2;fSK=`ucaJ;eFo;;(^elV0? zVp4aiW~ky+Jf$&4(XXO3Af7ZFoSLGyZc(h|t8%@M`skk?H@d#w#Yz1xYTR#@8<~rg z9ex}_s&fM-yKpbiW0JKjUC~HZC=Cv*Z2zatv$x6h z@XPV9T}r0RPJU_OiHRwD?XK3yK;((G8~GEkKR&;Ij(iuMHcG8q;?W|D)B9)29G|>S zE?n3amV0Il-*a{&O(km7zAqAHnOQRRERs2HcwIR3T7E3$67O$shl;1KciNgJdidxY zju8#2;IYAb;$uhkH=7nyinbBERrGrKoClgNCN6CwL2bgG9PT22!yXD4(}3~{hhSc9wpNPZQnWbUfM2-s+zsm6Sw7#^eJ7uZd?4EMcP02rNTvR;~MsU&KPkS!FPPz zNW*0su?&ffQ=NeYzx}gVUt4n=hAg(0*Cu}ENXH|pYHJT@-c2d3(b3Mh;!#?{e{54kHET7Kq<@Mh=;q#!{ks%`cjs7J4%sm)ue!F(-df zGW5&Zoi}k^Dl*D@+o+n86r0A|G!YguG(#pVq9T(MSj5#3M~1x@&0G{;L>6L6*wK5H7`hSvP>N(a#F(XCZB9+U zU<>c1lrvRRUYM>>yxrCr9ZIyNUu7Kf1dX8 zNbR;?H;<4f3SSdyPP!kk=4U6QuohYw@MqnlNbA~IU48GtXW(}|@Q6>d z@u3Log-jpMB*jzLZ|(U{zrZ)Lqo#X$Y^kPt`fPO_td^_3N>%Jka*022xX>us6W%#? zGDYJnXX}7#lpGHKHCEvPau*LMJ_55{*;h@Lij8yqqg8n z3xBA7y5gDz(P%>Do>)lsf?LZ*^*>>#hQWKo-)vRLL%VYmL)8VwSAgC#^*qQN( zv6>LbXk&bQhZ~3P-^3n%x2^R&QZ}~(SBt;?Vrxhy!_L_1FDVtwK@# z&-V}d{KqhtGzRdL-;=BJKMmOe-gl-Sx3pF@O0!9op_;S+t(ZT9#DXDk>nO0ajdr6Gar|$ z25R&-bvtcltlE5SDes;Dk<=*K1wq>SgKmhcS?TrMgj}@3^wJ@BdkP_kFGOlE=C>y? zzH;I!nhDa}N4+2IJoas8AE1CP>kS{^l!m68`??ZmVpZ z4pxR!p6$-7S&bpjI`Aw))Cb>kYgaY`ny{|>NBNS%KMQt72uWm5a(?Xun!>0d#qY)Y zH7sU25N`4n(toUdY=FEVOSFLX|Kc>;qVyd~6II9{Zd1ltmBsLu+QaxHW+M>|`NLfL z;b5!R+nhs(%$%h{X|B%{l`V8vv@m*qA?v2B!L*E=E;MCJxg1_>_0pOEaOniFP{1#2 z1OJV;z2Xbw!fhVy%m5YhaXKi$#|qM>pJer(RaDT15k;(|1tQZmNOajiGC2O7%o^zW zdGztJ(jA5KwQ3ByUm@Yxfv#gGSBQnCsQZ8~E099kS@bO%>)8F6qH zQ}c(k=2R)H$~bz6GDr!N4^dT?3aqwO?ePEkN_y)F3n|zS>3)txrG1dB>nn21TujE z73|i z&k81P4$7l5(Su9^YcvF`-`1c@Yt|ort7k2mI@2swLoX&Vykp(@~H-pPPC^BsMXy6iUU)ndF4TNi*_fDeu zNW+X*Zryexb7ADniJjZf9`6?q-<^Tg)nv{ur%T`dl=|(V_n#sBM&w4;rpe)}O_Ubj z5n=mj@jtdDH8mfDyyft=ZK{L(b3Bq-eE54}WsmYJ-PQ z>EzL8-Y@h%R;Ig|_tQ({-S@>u*iBCrtLAL`_0pU6+gE=(uot**czr(VXe)g>}CQ?F?VxO>){jsK6(Q?rk<^_j(@h?O&BLci88-l103gQ_B1v zKG|jVsL8KLTDau+hexu@{669!bHQSbh5o4(IdXs-Lizqc*_@}6!A9E!7uPmT!AGy zMcOcQjUTf%u&BI{Lmp>Nv(d)haw@hik(x3+nR@qOElU~%rlzF|YH59A_Cv&xMHa+G z4WWA9f_sC~5&gCOx?@pT&6z?I?YU97Cs6;&E{NrbQF7< zo4(?3lzM6ntqhM0U%!Sj;L`?DidO#oFKTHrFMLlj^}h7~JVx@A^!DrymTxB8duB8q zP5qSlHP$66ZgrSQ){kiPI%XWf3dM@(#{NceNl`d^#h$6x7~*;947qpJdbRM+Si=I_ z$RnelCLz1TYX`Tk^|ZV8VJxqu$hc&BH%fBAN_vbCejb7)@ux19w3vmobk67+p&R5< zU-Dpk^JzeS@~tM`bi_2dzgLWtnTkBqJIJA$9C|)(TV%=z>kIry)pq8ov5n zmSi172g8#5uH{O+sOpIULx<204uR|9S8X`sq6{A9HR!*vDjM`NTyyFk6yNy%SVCj4 zQdnbH;Mvf$fU)?ZjlUgOgRpZ+<7W!<4n4oEK+uq>K)B2AAO1ykWv<%(EaC*q>O=_hX7HodJtGW;YUoZ_arN_Ip_FJId$O?HgPwob3?OD;;wZd$J1O}9 znuCS8^Z0*Ubyy-0=+0ZHc7@%u`jY$V0|sNK3t@&cDJ6eozOLYQ?!r~yz>X#fH<5O_LmSBx>=@dg0#(8&PoiN>IS);1i{Jt(Ag z7AtQ8`o8X=RXT4a*y<*CNs~fSXjcYN9cxAxVf6!4n&VYLDPs zngny5z;?uRQF|}SDLVleSs!pCnJKj$47(DV3>!i)p*c++G7!Xk0i;$MybuV#;!|H- zsD=60>cWGViJE5_yk$yBVg>N{KS0402+Uu@MZ`kGDNf%Ko@3tnOa^+!N&JNfC^(~B zY^IhG0W9Jp`tnyZ=R1leqlFTmD?jM|C-`y$YsHs4(g+{Ia5T4%58{8tUxbDva95I8 zah`5gHL~9pYyLqiqT&hQKhq7z<>&CCKzs`a;v1;v>2^Oc@vu zm6>)V#;7v?yFm{4Q$rrM zqy#A})=5vT~rZT$NQdmn0u^Q3{@zvBb|Mt!05 z17=wtH*JLzpu>xup(KF;LRgIvszdAe;3Kf9-nrGR{}DVb-o+Vf49*%Qohdm&{GWOs zYXf-6I`-Q^G$Y- zx1Ee=gdG7U1Au!8faJi`Mb`gi4lkYYw|9f3UIH2VFW@XH_BTxRAD^X{ASR2FTag<% zLwx{uqAKg8SW9ng23{U%w0!{_^*(0{?jD(u~N6u1U zDh&QdNiU&FKMCO1&MK|R9q8qLF83fs$L-!1Js&mB*2J3h&zeZD9@E@GvZX5qVnv#r z{^i`PcFvG++9W9{v!LWADAOr~lMyw_n(Q+rSQlg($$%s8U!`$j!fIs%-TtIa1fS%=OMmN(wC zta~GluxNtNn3Pq>u#{C8JYue&wAqc%DDHJMXC}T(n}c15bnOu1Ckb+F*gTXMdS+!(QjoYbZSDX|KK2y!Xo?k_|hx#9={8Z;5x2ntxEjbB9UJCLI z0C$LoQc;Dqv^7&vBU$1>c=M>2u8s;dl5-Eo)y47bRM@u16QUP%B{SA59{?=BZWGto zC-F)8D)^9y~HzWy^0&MWIJP~zq;;rBmsE4Kw zz0loq^~cXwnRYYD>+#*c4K%mI;Hxi8*5@T3W29I>KGduMIT9=rHL5i=S)q!^dN|C* z2m79Fmi1+`gLE^-G>;$!h>{4-EZ}v^AS-xft=p+d+gxc~l>8t9X)NyTlDvxfG{pXa zZ(F~kEr-ImbP)v>H|^49N7`liD`C0}*wI6`a5+H*OLYT6Xrv~qbDa{zauKo3XB)#i zR#j~O;}*`2QL1&uxmz$5nPNSv6b9(0PsCF{ZK}!Nzp&-bNyWgj7<>YujeI2zH*ay* zzstOkyFUE3P9a8a^4*M4|9X1ck_dQL=mbw=Nm!mcuh==4c?~tmt&H&wPOzdt(+&(A zl^mU?%8(QKSbtJGvtdxpPCsh$7EY9d_%2v-m4TS4i&;8H*~ew%w}}dTC)njHf-wtgx8)tq>YgXecGV0s)SM*9tlx zP55iVIZWg@z${%s{)9n^6ZUuA?&PMFleZED=1jyCd4l;hBEg~Q#5s|VcOkq^Hy!+A zlm2aZouCv?1nCXJj9Uw^kLF(=Zr$^n!!^JBRgMtRM;*>j+LkuRs&li)ecb}V^7}m! z=jJ&=TWFo7rT)IJ6IZ}UP^g741v2?#ZJ05e!=aKGEUo_}rcUJr8qI`;I^mJOOPBY< zB{{%v3m6xWEPNi)94L3_>}O_lly%@iMGD*8+saoJ`lwsvoZOYnOxdzw%@d@>9a|21 z-KBaTs9OMVn4+xvq)%ArFO#UPi#p02>_Tp}DYE@wVq?d*A;LqzGvS&vPMANcim>0b zbbSoi0Az}jEFX?FNdK(;V}<>H+x}s|Rg)NmH}urBWEyRwL&<7%@!YMD_hyLP-+3_Q zTA7smgHNjTP1sVM+WRPnaFQUB{5Hq>#ReKxE3n0xHHbJL(T+HxK)jLi@tR68_5ZyWt4h=orv z)g(L^cpj~0s^bynH~iK)=$s=+QJ~vkl#&_3@%LmTJ_#H`(_iLEc<%H5PW~hP)#Ram zGba*v)Rx>R1ko|}Hy8UVr|?LpIDzbZwd;nkWTtV}>EQx&$IW20#64&I;1l&}y?K&k zj~OZ1icb^SLsk@Ph0Kbd({IhoeL4=q|0jLEC4@D^0g{O(A&-FVXbNL z<{;3>?JzvcCtdn328V`s1@xz3OGfF?rXqUkZ%`h^5{G#L?|kSi?~Lc%GduI;Z0(p& zdV1VNc1wE3s_a|a?a|sa8kN_Z!h*mX<9(-<*PVdc%z&!x;o|3V=)AGL+I`)n@>3|k z^;c>8N^^{7C6AeUi081h3EgCzi9$+M2}OVp=MsE%;RC-+T#G_T$obj$mM#ZS@S6<` z-rcAdlIGgA3F@L@6e7}}w+syh{bELRT{U}^rIl4ws|D|HALd<6Zi_c_ls4rO4)f8@ z7IyYs=7AEN+BqUb4%xdEC;D|g24CKv6#u24!^-@3ZSiqVaiq&(>%eWaW!TD?H(IEPnK_t@hm}4KY8Kc#W9$ z^5sjGt5CS(1RqsX87CFr%lq^Km`|S+hp+o->Ih%R@BOiOVzf7Pdjh<4>YPWlcUjba6hYzRGd8OhxjX0;w*mM&oCf!?h)e0v8Ud?o{6GV4rV+3C@db`E{p^ea9H3 zzyMN@| z??38)hm;iofH-$FWH2LMCd4g?Ns7NOxI>SqE7M zY`b*Isd^j;?mLDvG30{GatJy4i=|Gc*iFTM+&&~4Li-T+4;l!NMy-puTS@k@$ zELR}xgRMtyYWic>v|KO|B45Gybks)w?wBQJ#`1zg@L0)#oP(`NNg^?=c4!9Z93s~{rbC7E(8If#-~u78}64?%UD&zj!1W4VS(yyE11!HlJX z)ZHPm`6sWzfWLKaYfj^^qtwwC zK)+|*q5n@!pg^AVpZiHIT4omx52-Gf{Zu(r49cA40Uc(8xY_7}^ey_hJbi+N{eN0Y zm!$-a(1@BHD&I_hXQ;uaHtZ%H{S0H74H$e6>jnB*rE_M*?GAS0sIAC9m)auO@82GM z28#=iKnjRi1KC&%k?O!}v2$OK2h@aNo!xBOpMIS@L!(ijp@I2+sU;FVn;ryrSlZN$ zgGaQb&pkq%>?}CeLKW}eST9I*F*5TUpb6Lw=OpRh&Tjf8MmeAKc;J0%ODY|KXA^*_ zE>Mxq;VvjDScn~jya6+bf@Z(RtN_g{tiko*n|?|-nordPt2|3-b%b(KCGlNWpd*c` zcBEsV@S0Y_D|#qlXxQx5!rz72oj-u%>O!sfy`mfpqEUQrF3N^OTo9vix}x{NVfw0K z_bks$iu?3!j@gH#o`qj5UKgZ+7CfDFutDoWjpCh%iqZ48V?$?vE?Q&l>|zkN5^(N& zSaE+1D9DPzo*IbM1;RP`=PU=lbE~rWX*f?>%D*WnR^F`R(!`)6*l=YpM`aZBYB9;y zcwun6}&lVmA}G|#uKqDS_R6Jjru{x;K$ z61Fsd-5n=b$lwOi!C3y(x$MO9H`ZLz9cajwK)n$OH0?1WEqNYn%rrGYP&g4hnkQe) zFObKc&kCHjPIeWE_=xr4po0DKmPy-9W3Ap4|9CLBnf)Oq6?BxmBZ6EK^uF2nb(teeWEV% zVh#>SY{ca5r(!GPSqcpVBNd|yQ&^ah671H zwj1x9IQdFL!{QbnuH3$PIXKE{F03Uikx?j+ZjOUGZS%l7+wjJuNYACK&>C4Ko}WBw z5fV*iJuc(pyuK0Jq1_dY>by(rMa*YPix|Fl^u8_@9Yy%y+xiulDhN&PSSK6cXz#Z_;4r!2;!w!U!Tz?xFvy)enS+8fO|?;bqf@79g}2>2iCF8A*n zhDf8hmT3ODQR zsF#==qTwSw5D#e=4plWYF^+N4YvUg~-L%CyPnUcVa>u%qun`Vi>aUr$Dw8wpn6ad_ zcc(8W5_hs65soSZOBo`b$gAj&UJlY9O@UeCybMG^cd6qO@A^%>4^!+r6+obNT;OD) zl^~hN2PiF*xQl6$yH(&33-Fr3gLu5tr$DP4!6}d>@4z>M+Lg?HI~w+cSK`D)#{s_~ z8c|XgTWL8segSpoKE7o^HbXYzNsMLHe*z}|N5t~~f8k@$LHGLdzy9SgB)1n!p+Db2 X-I+&HJI_@W zUtPI!6;JwcQrsY2d2qF#f%JFP^_7bJm4aSY0_o(Mm7Ip$l`F-Ol&5CbN#{46RrOu3 zT%l^Y{9SE#%6@z0is zg5XMF;3(-Jhk~*g)W)dAK@T$z1EJ;lu%z4SfqJNLQUkY&X;?^C@54AmNmqHW^7f== zU!(q`BF+DM#iY~{yO9ETuf%d@IEz@spLk@c&+(41-}$MqsHo_s`3+j6LaAlX3l>p( z#zPS9IwkRneDI5QuY)lwh7|jhgclq@AP};g;%KYv9w3O!d~A@knkL}IC86#Zr4@^lgZLjPUmg!NG|{lt@#EqkV}R%-?=qN zOqOt!(Il62-(;1w<4En*Ly^;fQ^!iJ7z&J+bw%p$;-9+2XVbXbW?TaWMO!O@bgDcY zP|?Am?5~XaCaGd2ZWXROJZLNVAhq0^V#u1iX%HLj+V8lSHj>+!Q5Iubdg3%>nM2p9 zz$iENu90=ek-_rix7*99-j14^S{O(Q(Xh(dj2QsnZ%87926|liKA$dRSnq|{kmR&) zY12vZ!>Ha{NmdnN;9Sj0pechr741Yn60m6gXEZ64M)jpmo*wbBq__YbbYQhCZz_P_ ze@Bt!^-|q=(olwC;X{65I<(71l7UQ(?tMCCE$FVW z%%*~|4)Ccdq&Sr66D1hdUaWu>8%v~I?*&y&g`v7S93GX@!F?b31vu9TcuooPs-;>n zC3mew@3|zKGsg*2`2TDLH!U`@{DdyMcyI8>>!O+59RNXT|Yj25GV|=0gV@eg1 z6f_hJZbN>MMJAKsX=P8Mpk>$JW^_*6sRnckjX#KS@8%vivJ4&wC8|nRe6UpC1G-R& zwwPV#yk79e>I=?5US3_nj>4-k$8ysGot@Ef`m=0Z!CYxNpB)`l z^iKl10`)4s8gt7r2E;puvFB{L$%pYho zemfO6Z%0!-aO=M7U&l4ClHxvl`o!ESM=DXGw{jJ`Q@6Vpc$yJz;n^Nhk!Qr(B@khL zTqFpE3Iy3%n)7z90^XYIPi6b~;9+Cmx~Yy^_ZH%orH4AE^=5t^PP)BJnN?O+elbBY zL*&l;DF~%*EUDwNKSZky7m-yC%djEd0n8^@_l~5+TfC-=E*`Dj?c-M4Zotd z_}F#u!>WvlmbG@|$J1TAXVadn&E<43-*(FoKBS9c-d6GL1vZRlvBeMN8wPKyqvO@Js?Q~X(X$YPm>##Q&^UX%XPElYF6oxa{Wox-enuPX$hV)sd`38%A4sA zce&9+-J%2OS__0(Yb{4Lp|=^U-vEQ=xp3Fo@A^z!7H%=dLN5~}%d1)5s^}Y%83&*G zaEsD%4}aL{EVSX?SCa9RUM7&ujo$>>DE5;zD{?%%z37PWi>-NFD<(<%E@@BttAYc6 zrOkhej(vrkK{AB`@ToU9quVba;Flz#nNH9x)xy(7%4xbz$mKp1z*M%A;r~^-L@=n! zK??)iu+|Yk{0NA&>5$!u4`JICD8(=Z*l>$l5TdMVJ$~deD(};D<-Kbyf!r_QC@f`N ze_SiNf!E{)sh5_-#7YJ~>A%r|Bfpg1j&c=VWS)_Qf&qgjb{R{$VB6FLsu$>}qi)*E zZSUHJvF)deXVM0Wj~gAlxJ#gI(2`U~kW}&tg4vUtE(8z*sFU=bnuLCcp z+N}v$v<@0@a_4u>A8&r!vBpy)1uamJGSw-R2>6BjRakFiLM@Tpw^q*Qd)aGtK!Dn@ zD}=y-HNur>KAb7$x|O=~+Mp~@2ahSc55)Kcu4_r`NG%lUH&5mR(9D-c9Cww%G2!## zKQ`)>m&P1TmKdJ5`}nPSP3xr<2fPvmBttn^aQy$f?f!oS3~$*3s_aXhqMUvLTWm5A zr#GXW7@aO5ghqa#9%n6_Fi_?^*Nm1AJ6hsY(>oojxIH?^A}2<|8mri6VJN0CH(sn~ zvW^W2r+~CU`~l(+4WXUDryQwbPScoc6PNJi%LfKLN-=@3A_XapD!oK94{$F~AO8Hb zke}3G#?ME}3h;FFUY3Gb1;K~6h?sC1p>d0ogzbQEYCDeKi4~X7W>uclA+4U8i(K!Y z6dUVfrJ>M<_T9TCvvnq>@-ekccP>YD_J&>g5*>BJ|BCRx2ZWd?I2_CGr2m#|cL{zF zv5y38(e0Z2MO@sr{MfrAIuq=-6m=oatPDp_E<4h8IqqK-cVSqabh}H~C8T=0j%Gf1 z_NkWu9ZW;RJ4B7t6~x=w!1!`>d4hrfmi9C$MTk`O2W&P_Ji=?S^IB!Bs#wbe`}i2C zAn_m59DagTakwvP`YqWuq&t2 zxIzrsMOhvsK_p@!05&y@GIeG&xh#Y`cpAzrGoAqPsgefxxod$NTpfrR;x+rna5yg7 z#{P`UW<8c1xAMaqw-V}(=IXn61n@j*gC7y?2If=>RD-|z3Ie=lB?+^>GwS4<=U(9L zc|Uco&~usa$(<}=pLx@)xi#65BeOG2=ERFyn+uD^gag0B#w=e~zkG2OuhBPQ3Hr&b zC+y+*)r~7c2gavsN@+Ax=Usji^7dZXoM^&eaX);uLlx{fn*o20zaC&B+Nhn~$0`6;s^q`$%!Fqj1eHqrc-~7sKp3wD~jg;`^huZ|B=# zpbd$GTwP}0k#ZZ4xwNiFraL8$hJA#Ct!QZ@H1ivy8xB)H}{2fp+BocPl6vsn@7V08e#JuNZs|9)piDq^g}|Imjkyuk+w zdcyRU4=aMsBWmLX&iBhMyf36T8c+Si&KXZlm}{1x1KZTHX+cgW^32s6U4)FR_89KZ2^kz#4IY*U=xswz)341!`hCSu0V3KM+0{n6k0oKMQ9YFM-47ci`|Y zA;RbZ!$2{>Zh%-RmN~!GXacX`KCDlIM65r@P*G=TsYD7RQ@ELB%fvqUwS^6;IoMa7 z>7GKsb*HOvynp_@i1m#Wy4o2&WpG(#i5(DNX6;_aKOV_+JEE~(2H?3GlrH8_y@@Xdm5mhSbHtZgb35;}H`5~)L&Q_J}5roI8 zn7B8jvHCf=f{Na3zjSj*pNH>yFzW`tNbBliAPPf|HkZ_iY(Eta^GI0aV6A)}RB+Uo zWuw%7o32!~`yPnpEt<*_7vbo|3Gi}u9KZ0t&@#BNnykARVB9}91lhH3+@gzJDQXv1k`sJDjtucUJqyuSlxi`>Lrhk%G;sorO@Xj1Y8-msjjp7K?yt*(ch2E%SrlKvcj9S&oZ@*NK*^;Bl_d}=(L7}QW|c8K89|u z)|*UIZ^DJcP?dBJ1D%y}VOw|Iqtg;(;BeXF;4~*Vd*~e@<*i{HD__>k?`39KmJq^@$(3=$X^nOQ)}YS2Zf4m zJ1W5mpZXb)M(SEg&;c=yREoPJA@=@{=B%Qqw_D86(SLq%Rr-m9=lf=_yzRzHoSzTl zO*Xg79k7QgeuiNNky~B}DNDcEsZ(>O0(;at9_Ce(k`h?E&`7^k$mY3O(Uul?K5obW zS)bBW=U}UvPu>CFiWn-OI>Yb4eEp+?55h5TI|#WzlmoH ze|n+q71FZ6t}=guk%{iS&0@((3@`R@%P zXw=VpvDCYWbB;XQrn^I|UxL0^gA~NOAvcHdzC^hXvh*3eahVF#CdolgM_mK&79-y6 zSOwT@;{iuk7qAWPpWX?@8tJsH#DB50P2?V}-&@BLl4!U-Pv$PB@~4mXP?T%yp_BJ% z^msXx-=EdQ*eIzSMi)xv#wWhb=})+Ictj=ZVyxg}82MS*@lO(b;dlbwbEb{9XgBfv zMw1$q)GNU6x~c4#8cQYE$OG^_t2PjP)+8)VxxG!JtuK*2Z^j4g-0|Jb$q*#L369Sw z{Feu6x1tvGxW?Sd?!s>Rz>gMP46^!fJNb*dP8GB>WKD_rR+n*onW*2UBy6ZSsyo?x z<_m`t7ToGit-ZFULvbp)WynWXefxe1)4toE%I33FUf>cSoB0Sqv~d?h*5MYo@sU{J zT+GmOxOagJZ@u5)b(@iBceI#T!DEd|w#^=d;C+X~7Q@EG1LMi@&B%reLxR6#L#mr} zv3Py8nYN9W8(GmEPJA#0XV6`|V?Z&k$VE|o^r(pboVfqk9Q=L$&#%K&UA|hels7!7 zJ@VtyX5E~3S}6$i3DSr@vgaydvc`4vVmld)ygGqAxrHkp!HKckdK(2J&}{WI0lXz$_*}R3L2V~oRR(t>LDYJEQ*&P$M*0f8nkh#2w4|G zBwMocVdG~+;9W;?-CLq!9z$(Bqtu5<#@L;(tA%nK7fnua!>(`z-E$=PsMHBlEJB@0=j|tw z`qK4?OfOGoxJlYxRR4|3eNp2lLjw#Pmj8CD#?ls`!$fLCEPoSL&K4#bjMk;J?K3sW zR*I-)-L(7rSUy_6f?89(LZygp=P491+xZHYbd-vaR-LI=~FHVVR}%b_nn&dvpkO-7?5YTza>;cH?vme6Z9t|GoQLzZeN`!aZbCGiNsc(I#)NejV;5n_#1?{LG?Ajg~9p`*Hr^G(Yz6 zIOu3|3IxP4YH{C?Mc?|2li?b}mu2S9^!(n>E^AKGk-p zZdkH76B6kbu?!2#;Vq|!0T*m5NsFTf*7^xAE_3e&MX}ECQ%jna1>`%S_O@$v8MZ_b z%c7M~n&p4S_)}y?V}na+lH@M8Yd+nM0_7CGgEwJ0+`cTdQ003QH;d8Vb_vVkB2%`u z6Y=2a8R*i~Z&<6q%&v%~!S701mj+8=sz!+NuapSX^HAyF3`R;tCM%MsG8objj^b9p z$lvZ!o;K*uWvRQ9oMP1g0n?g(caDc?kgcJw8aB%l z>`P|J`jw{z#JJe(?Xn?F`l8zF=lF!~~|I8ZM$82z4xCS%pSK+mq6*yccP;*4hTnTP%dZ z9AXEVGN~)%C3x2-fx4LB0kOS zh^L^OPE$jpT{pcmARduzxJVi~F199Q?YPn&5;x;(V6^2h`aznv za$U`EM%k!+V6;Da{wvh)M3gi2c1naLt-iy_=k~c|bbdI<68w_Es$@B=KDDqdL51R z5ZeBXV4SA*UL@{M#b%|8Yj5UJnnFx#;Myn4xw;ri=3`@Id#=qKbOa;GHXRT2vDNb0 zY2~+B%r(R9rFYUO74Xe^t(YW%PHB94F$7~y zB&@K+r6RxO#M3Jq%}DF1n;IIesgbsvt8W!QS=(kNhN4Lm?S4ST#-Dmnze}vdxzpFD zo}YXX$-<9!`(}X>7e!LYb?VYY8$^;L!tNa4nwNxqX<^qGIg|tT$L}JM9-WLr#$)wP zeD&Mo?KVX7+JQ=^iC0qmMh^>mCPxa4O!Mf=_X4R0mQ2GlGmyV+#!2Yx>x!2wjO2mO zpG7L;cGqNI$a=lCA{sy{p^q_BaJ$whR^Yv+PwPq!mtmF8DKMNa&BP4gFK;U&9=e z06uw0T8Ki{>!;h!UOTbUtz<7}s^-e~xEbnN5@bbnXJMHSg|zADrrO6cYT+gYi>vdMCC#Vg5TWumR7(2W-QC=QBqql3bR5I#(Dt zVrbl-aw=y^j@^h7MzD&xJwuh3d<9v0wX8|kEw!x4)E%^}$y$)+kmyBQ$V9B6Ed&rx zqCjWg1m$nv9SRA1DM{7H?;hBz|uANPfV|I%uibd2xtuKG_3((6}tHyV%4L&rr zHlS3Qr*Q!vT7@4x;Ki}|lJy4z4>-Hk;6PSS)_+~@#45zVO?s4 zBY&j>T4MFX$v2a1w$+f6^yzHb)#5y4Ey52$?A)W)Er2jg*Hu(ahq8?LjtM0V2FT0UEVAbgS>xj!*)Tyr$EH>FgF_LMb9 zS`uAdXjW#iSgYH0@_5jrfAr#{ZQpKSq|h+>d_Pg7wZbcAEBTmYiwgB(&o_6p30-md zJQb6F@P|6|h-2Md9du*-pLZvxN`ed%MBjstam;>iOTIBgbi z&ph7TGIm~Dg4LGxA-!!6_!iH(4jHQ>JL=^($=~%w{_&cuO`>{9)u~YZh;1>mE+%Ow zZb+>%!n<~|@obnsDRd2fsI_(PP^;RivyIRepP(`mpl7F{=JAV~Ls|7W!jB5MQ6Z-| zIDgEjY{-tU0o;Ld#|ume%NTpj2yJDt$y8-{s0Qmwc>KnwKdC|Z$+#m zjjIKW%dFLiDzlir2Hm>2uWEM>IeQaNg1q2Yb{xv$w7*F)mh338 z4CiOxXzfV|4h^}DMKOMGs2mn%|2cjG6p6E8Cbgq3dx>$M4($~iVQ{S}pR8}|KEyKf zUxwom-v#AaResQYr{h>+HH%RJlCzw~RQv_uVjN57_PiVpO5F15C&&H<$EYe}W(maT zjsCxhfb+|B@eij&|46Js$RlJJe79_|Fu45P_rIxtXQqKw6W-WlF7+@4FokQa1I>qB zFW;P@HRxr3v`Ip`l&g8zV~i6Q3>_q*aY>4Z$L%>NCPP8fE2sGrrE}H`5>%~0;DxKb*s2iuB?<) z*06W5qNr%IAOD#IP@f_-9>-7d#mRq^C#>Qb*3)2|^hy8|iuN)wA!PhNV11DMJ5O>C z&I$H5_YlvKc=w1)uLi71VpnvFqI@{J$9+STWa_emFQ)}W!-aS2(v=6Ir`#`RoT~fL ze}L`EpI@?GUN@04D1UvcPya2R%HL`DzX&dqWMw}XC_~hTWI7D0 z6nO()*7WhZ)p*PA-@lI%LKM8?g{&asMP@BkeHi^Z4;P1V|00^h zD@exiKk;pim!58wp(-Y+x7JZ!3b*zF@&V8|cvZl#!d@P)zzQCR%nxg$rhq}X2XRJU z+%_lmebUsupZx^iFQ~aiBxSlxjFLz+6OA9zpQ$!Jz`;%OQVJ+Q#BBR}wr~xue?@Mb zE@GmnY&xJ3i(s1~Y}Z)~B!ziYy^tK-{RsPUOYJM4gqBG-)#F0tF!A2`KF{M;*Oxk0 zRW+2wD;D0LXQLp-;|lj+n!`XE{mxuku87^tCKMD&WR(x-=!gWH$Ebr546qORF&_@T zC6I+X664Z3m6>;vDa^HC>(S1nV~Q z^e|RZRRJLr5Eb@`D-$_%V;vH)Q0%)`k%P2pG+oS^)k#B|Lsinr6#y3!9HFU5^WLmZ zqe(6~UGF~#KQ|aaA7#ZT z?8D3y{_oZk*zynO6BUfIywn-Hxlcq$?zZrgSs|x4*go8bSV1sOltZ3}+u|qkCvdE{ zcs)~qf@MwB>|R4&?+%zA8$!?tv@cNewnoSn>$2ZqK88tJd>&mnd4bx|1R!wgsOOZL zkro*yodJw2uo`WSq-rYxLklRMo&MOj1R7U$qd?O#YfUYV2(C4rasR-o_TL`ysgtEv zP@CZBKkbkV{b0JYeXf8KDvZFTPy*{mAxw`bJKTn@rxhpn%LYZ7JzMNOTcfDx**kd8 zjy?EH2|{qtrYGOUt+2Jw$=)is#@&|BJK9!uwmDU+7ASrg2^&{O4FUIpgTS^AvD)Sw zn^z~dG&7bz;)=t+HL{KVAK7s{T)vIU)$I zMk*W)htuADqX8+Sxh0ywo7txiig*lbX^Y3n&I2?PNqX^5xSE!&&uXw-@Q2XzXXgET zzb30)`-JaiwcOKT<-qQZ#p%m#*1*L%$0fAZt{T367;&|hmOg;qFCvHGrAMa{Alq>P z3eXhI($=y1VcwcgN%SJr^pzdFr^r?x4Se#(eHthX*13ETOUrStzD!yn^a%80o#x8K z!~*8^{?|f_(H$^5dukrnWEq{YXN2ECe1`fB(t>-wcS0LT%~jJx5J!1D}g zRUfOTcVQu0>P_;K+>>+3ZToLHBC4eIP+at|zqB}<`~QkaXZe2CjNZg)P9)Ybo{Abp z6nI>i4ga$mVzNcx_FA!iO8wnTOzqjK?;H}OU3jQ23Mg=|O^NZ6yI&}#j|$I`(Bk;l zp9gz7w{ieg>3uzDTB#mL0~I-zXYa`lB^}f}+(_-Hpn)pv38uz%mOCqK2Db=qJrQq< zWGkiT`I!^7{oH8a)i_qO>X+M_=<9#p8N&2wZiM)E-WpWOqsx+$Lf8lB>OYqpwz;iK z4jX&KYth!A$3Rj(I51e&aFLiGnH7g2`7nbi%~6%jIR$@|_;|5vog%!YQM-Kha{45q zi`T<2qq+)D=3)V7=I5Haof)Pt-N3%FS8M;hlB_`ED!IGkG+vgIEo)*Y&w4M_)xbeb z64bhPQ_a)-1jGlS{IIa5$J;PI@|GRYS*Rp}?+_K}{g7`k8sMdQJdbBJTALiU+(&I3 zMc+8j18D%-ijMYcPN&{*Y=VTpou&_LZKsQK zc?7X*O(=NRmP~u-=7y%jtHq!6`Civ~sL#um^->vS1*rjU)5?l`g*zME3u^7q1_R=Y zcK99-om&a~L35{z(dsJ;)D{CT;AZIRC+PS`MZ@;~v>Hyx7)lQz9Loz`bo=a%%=1d` z_B@t|i+p2lL}K$9?W-~lZ~!fa2fkSlZsOGu$F(F)Gq3t~IzJGY1#)g0`mp`nd{HZ0 z&N5lmoMpTRY5Caiyfa+SGt!dqpc<5x0x$T+oZ!%G`J+rXn{f?Gb%!8A-EQT)A#Z7g3 zyo_ASH}lb2_pJm7)6t;22z!%@z1}xA?(l5=Bd}aGX}gaQMMhS%gF^r=d`_j?i0j_E zg(k)^aqpH997hFD(#H@2YFxug>bd&JdS%f!PGchBF=LD^Z96epv}=#iu_LCk?4~!l zI)KOv&D(N-#%5tU{3Dv<>iD#A@|QE6fXFXGQ`fRD7uQ#ghAE!=3wDwML*Q?>NN{9p1ZY= z*k-OySU9cUgQYB-^El-x%+@v!(b`+gc^aFtT0~mBuZv)B?B(&+XC7oi`rzXm9`Gmc zY>F8%&=Yi-in8nts=M1f&djGSdmmz6o?mx-kf95SPQBZQK` zrC4*oBWRX%^Nm!HUZoGuSuxsKx7Z@ zyxXGXIP^R(S4WaWajdE{`JsG}bLSZ!ON8s-bwv8fKdsu*Z6)o(6bgFfS5Il5HP5~t zIa3yGTT{~4c8tyKR&LF*KL)mf?sT$FJgc2~UdSn1r_f3$c0cT7B2Z<7nTHiFJE290Co6 zYKc+_1+HgEmEVK4(tEI9@;K&l^7^%jh*QA4t*_OEO7?2N(=6>Bz zeDP+e|Hp2tEYQ_4{Wv~k?%)llE%n(w{3wvdWc?%OH5<#BAbU}iuoX~JG_+4ZPmyu5wyMkP7jV>@b!?~1QR3|K(UDd{wJfq9JR z`^UEWnV1J4uAWox@W9c%Ys~du1u~;EO6SF!BO}xtdkr2PjFGMsAoot0)nruzTP0<`>DK+?)N-i++)Nj?_Xd#tJu0?!Ax^86RRE zHxj!E%9!(Th^60fs#VVUYq9fjW~TDp}-o-n`)BC((CUW;<}Dv3aI!=$JPt&gpDR^Psz;=PphN;}6O zs4ap4^SE=AcsJq+tNXZFTZFF5QSj}BmIA4bfhYC;HHBKZVG32t~iFKzEL?N zLnkoK@s1;cy{bgTPkGz0g%`G?M^W?ATmUy z%19~$kjzC*yW7h_wHVcLSD@{FwYZz%4QmUAG}8A6;ArK9J8iE+{z_1fof)N~D1ZNY zMgmQ{t7qLKuqKwG#R{S>*+))O9ztOvM8Q=Y?h@D=;@EYiXPwiL6(CmH6wgC)m~rMb zl!^*4_IG~Q(n(RaXX3#xUtD)a}hg%!dTl9ht4VT@u9Gu7VzZzBoPJ z4Sjd#X;+-!1r{;)=k0eS_bWl8ND0>^^&bv2?!$TdO8QkY85tS#R$&Q@^e`cZZ{|AW zUEC@E!iH~O`yTs9T^Yqq>DikqEy-_O12>>3UQKw{lO$HS$ab&SWy2saA)5wDKYr~P z#?nrr)yKzVvyff3(h;Z!5NT%)PjT&)2xMfxy@Ttx@7LSq)R!t9NW&Eba%2KY?{KlY z47scsmDVMOK=|6feW5H~eK7oCHQei)%z{eY3dZi4-hf;Y(Md-o0JjobhEL1&=uvei zeORJo5*D9f%G;r9A!JFi5Itoaw}254aqe3zfNZcP*c!agsDcFf=m$fW!)51`?y-J9 zIwC$MhKnwifyExZ-1IPhy&R=yI;h4)k{Op<%qYW(5ml(%Joat#+3 zS1Sg*%cwK+JZ;wpkI#k@uyw6)TG)_~MP^t4=VJSRh5U%)Ny5q;=6|X3}&miAxr>fBOX3MZfm%Ef=YXh5fs0 zV!>7FV$oJejSPp*+=OlAj?g0H%P#fnq#0X`xoIhwf=U6UkYZrPrHfhsk3Eb(+B`hP zbK1t8^Bb1ChmpSV5wnG-2v?*e(?QQ!U)tR+liC=FNdB^(XasTDN1!1<(hT>Ny8 zMC!R^`Vi|Wm6cW0FVO5F(AKn=;)eu|ys!p;@1%9%3%(ZPmEN#DUxTgDPN5YG(mF$M zVzFIpb)V#uCt6>-h53hiK0^YnG)1HT2lV6rK`8IP0gzPh;g^qKR)5O~wDRoug8FmZ Y($U4nwH(riZ&#ixK@a_BC=Js5k{UwjHN8mlYJS+ zT8t&b$WoROF~(S9kYyNtx953}_xQcpIWt+&<@dPTb8K zrYBFFJ#pm7k(1Z18e1PZa&#KFq_{YMCo)I-g@BKvXlqmWky0{b9=Ksg8d?}0IZ~0t zvv=<}aL?^~)ee2+2w&Ubb+pr`$o0q(6|rl^hQ9|nE@#EnI#Ms!FV|Emoqm-o2Ec)c6 zBFq~T-lmiYTE7B}FpsnTzyM!3;LtN*um~vH*4Wtf@+oI9IQgj6-vh?~&OrKQ<(9oQ z*{<%{=L)6r^i1@P(JEx$XW`^QD@@^wrjRWS@>GHvIp|j1TyA{#R9md&yE|v`D9X2# zuy1zASA{v@OxGpuqWDX@0+5AIXS^P;?Ut)CBlUC3E}+65Q4=+@$3rgln+)`Ry)ocD zURSY`Z$3b;BXCj1miTh{O-66n-%$HYS2!TSQ$D10?Y2Pm)>2hp_$`E6{~s7P$f@b` z=OBTBtyC>WH0VagO&%-Eko~c*95gI-`|X(#4JcNfU|Fs(4BnjGgWl0b$S!53r>9qC z`SHs8^k!UiV=;eOOn4ay-Q#?_C~O@uA%r3a-t*Zv(ipD8=zp5+8ty75s{^ZRH#)Hhi+Eql>W<67Sp<81(;kr>U z@s@rLdQ>>@OZE;K!?uv0aA&T5dY)&&Q*iW+g2jlL?mdW(r;fkKh9#tLu1^$ChleqfsG=HC7DQV0P+H3^*e^QOL}Z4C z@m!s-Y;^;J$p7q@Vh7V1HqOW_lO3bra{@C%ei9P(6y01p>jM_=EZ0rNh-R!*UOJS= zr9J;HzLO6Mxu>dq#h(n$>><5xR&wRJNgJ#Z=%MJWummXf2eOM$HZZL9}$Gf`L0yCPn*v_}~*RoTLZkkwS<>A(R z!>5X;LzcbTz+cHaou(7YoOU{v&D825oy<@`fQPsP2-M4W)?R1|YJ6o}ZmNdyz(J_K zO(WFZyEAX zqvAVLHE>e;5;#e$pN(+9-h7AM9 zt$+(ukuHO^-np*h>h>0733VVRt=QSOs=6YzgaSsj-f4VlwGu!~B6Pg79BA0wz!}4| zA12KDoYqahMeG_H;C<n zP!`cuJWnf4xK&-A+Nn|B>eJ08@MvNs^YHkU={7n)uG;9hHk&%?uZL$|3&c*%Jm3@y zNY1&Eqh>8zQf9%1D0uBsog8iu0n=eYbvE<1_l_M%uBgNbZzHDdIPv1YA_ zUw-*z3xPwOW#Vl89lG!Q?fGVSK6x&Rz8NodiE-cx7v}_uHE0WgSl;1d+Km>m1EYqtu0miFd+>_$zM=;1a{ZtVTv>~FI#bYX?=nlAHq z<_ylfpk&2uJ{+yHeb9jJVtxBoR@3{dqUY_;q@E&b=BI4k=K2n)z?p|Pc~pl`WtNX# z6gJ!PbY^Qg`D$ zY;>K?bzw(1jmK?r;xJwLstmR1BUX+r;Vci>+E0P$OJO$QfUAY32SKp~9;vUvgJ;>b zU>2Qg0XR^ZL&K)}r$ev&=V>)*lgr=%4OKpPiXFBz`gIse4SD!GJ5@Mu50;wgBb59O zWuiM-Fz?V1cu~w1aPk~&MNb?RPpw%NgQSa~jZ`v5`s6bl>>Kw)3+dlBt zInYs48w?c?+T{-J=-ICerwS-}Z`jKlL>DeKTTzsgmB-@`t0{I*;4d@s>NA_8W_CeVyU8 zORSl@q-N57@zO!74(Wp^`8;Roexd&l9SM0(AEs^avT4rwSZ|iIb|_u5d#Yskl{7!A zym6m6>%Wm+LQ52qQ0R0F#zR5^Y_!8?YF^en$MAL<^Qgk>0SD?X3SNG^;@dc(Vt}X* z|EBJob>&VBb^DVZZLSZW+O`$m0NI<6>;7Ov-oJL|z<{JcXI+=B3*$WQXdL4@jU*m@ zM%mmv$F9+rLJi4kR%lTa?8K&O2I^X2b+_C6=Q?dt*e37X-E?3g-TI;r3jW}=ak=>; zE#1W@g=(6l&hp>;@}XBkmpMPuEJhp27Rm?@q0$PGol};$a9!q}5pAxDf(zGU4(ZW; z$*p)XcYbwxE0q~kzMtQc<_|4{5Dq$KwN_0TtKTOdU`cOvN`H&>KDds~6z(hg?r(=4 zcijCXhckEy>5SA2t%EfFY@MwWuba)msrLDW|5{v8D0BSvbGB~fpu3@aAI7p!I1}(?tWV6yD_GW(A+Le*W>XURs&9p0*WBj*EeNv5T!w8=rtu_dyMGl7-LAGai zky$?oRe`L4(Jg`jEX3fKTw++rH)RsoARzp~ro?`TXvH||>A^VDMOaQUX18VU5$#8! zMEdD3ACO4-N3UFV5&Cv6Es|@Ud`sSB%NE7}6%Z)*?Hw`64XH--huu6MevbcXu`&R$Sh721m()lgna!hCYFwe>X_ z7v%B^?}b+PVz^Oi&S~#V+b)QkJp^>Xx~dU>upTk0m{@a?>359uWmkV|)8C1@L;GUc ztfGT%A~aU4e?Gli7SIfdVAa??jiBX!G|*63rDPO*0o0}llFhm`s2EJ%yl9rGh!Wo> z7k{iu38rZS8zZ}a7I^BDDsPw3_`~MzZt#P)F~oz^`?!KTsa}96$lW+zoVa5A2$r7Lv{#Pb}f)pldEzqhe8vu)RAQ`(kRKG>Z=UfcVu zgOgcvf1fF{PM3I)`UO|8F*@mtzWz)mFm+7(qMvVHoJ>&abdtD7>a1IUNccdi`{Js_S?ZS6*x z`gvM&oreQv6@5J=u59qgE_GAJIQWixn_`W>NY&34eEhud z)4Q10Q1ryByYY?@dwBPEa#B+9F1gXRCWGVXUBILTk%n>TlHC?&1Zl!@&oG$Qi10o= zNN6qMDF6^#@SFu^vEUv)?qxqD0_B6=$9#B3u*INlc9mHMZd+2`Y#I57FTTrijCw6I z%Y075YXz%uHisF0c!3ps$AX^+C%=OGzmH|X8T2i#Mzpa zGFa?N1D^0}cH49JMyjNg)XN(%I4{a>daVp8p$1&OH1FRL(nK-gmCI0*+V?*fYU49^J+`Bh#wY7mf{$3azmD7(BbTQ4)I8 zHoP*Go`1#IRbc)KW_{UARKx2&`=84MZ-kWCyi3FK#h6f^28r~wxFn>-?h(NEs$(H~ zw*yxqnsgi1Yt~lPc1hopexAqSo9#;{_pEoAy?w9xCRfev4oY%Td0LqF?a(_$v;znK z2<^qTtyi@UzBW6oh{n7v=S6e700Zjmj2 zPX6)pV7_^6Bc2miKcR)cFnO~&1K1|`ihB#RI8a zX-YC{;7`lZk9Y%aphyH2gdY6NqGjYX-fUiD5YB*YORu@sW0(|@q#hyc zx-)kLbM$SGS(gKhxsq#kbJ8^D032ok!+jQ349ON4_YPc`4oZnQ9+iYuQGsPJQ^6S>LTtg3L>0}rPT5*3{ISto9@s zHQ)^3eJht9{$(*}0)r&l|yLV{23bbh%kjs z>rQLlt?EeLr@HZ{Rw2bkB}zvJ7M6J1j=OjC2!PKLvFU1R_uZdkPCo16#ieUhU18_! zca_zj@rON&UA@N-$4E*gj3nHYdd(%b48OX&M{SijzOS;{`Ba~l#NNWTZ~G-@um1-> zXN2yY**>QD;bX8fyYN$EcJOBqjo@Ac9iu*1acd8r=@J$4>^1FSbKEd*5$28O<~LZo zPoQSUk}M3u-`EFGSu`~pA0?r1#>Co$<@Y}(cFOK{i_lTZEv%pW?&o6!r)m%O@oSt1 zK_S5u10UrGzbP@AkHKSP@RrP@S2bzhze953%Nono+M zO*<25ICLaz>Cy}%h>)8}sy2v!6R>((Tg|)<%n0GL;^s#AUtT4lFycQWf%el3r_(->|!7IWa z5xf>1=|<0Pqy1P)@Tu{e>#CSd>i#wwF=tR0f+Q@&_Fas4pe599u&zdKsoLw%UJfIC zSxp@^xD`?HbK>2KLFiX%uSAijPeg@R`0A^*Z8T?wfd!!G@QWx2zmRo0C^j1``rCn1^}&!?EZ59 zoKfHjPHqR_?9rYs&`IUr9Dyy>iXFh|l8FU=g02eQBXh;K4FnJKbpYGRm+`l*^74=vSKj)0O47lGaAHb~|KNfOc5ZUs$ z!*xE1n6Qi=7M|VS3!QH~i3vF40sQpZ8%-!S9n~27S|!KSUaFb_kL5jvLv}Nolp~sd z#bGcThJek+ND(nVLLHee<^={cpzp>tpx!vELl}PT(4b2_$K2!cODo4GuMZFX6M##9 zoFDx_FJt<^jX*Oiz8@b-NCL{tz|GNZ@2!B`kHV(TwU z{Kw!X3al#!^{cU*|1TVzs~U*~RecANxwC>8(k|O(=du~5vg+Pp`0!Y8TJbsi%HnBY zUWRi?qdBjaj@fB&vPYp1)P0tuJ*HUiB}gv@fbqL@$0%W~M|b-$;KMw*$Gch)KMEyH zDyaEz@YRI-VVc4J(tkzGS7Go>pFf_5JSSZB95=o3XYFD(l?gA;R5mas^0WeB++nN( z{#ci!<$YI*uOA|Vg;t7cdM}0EGqTZ!Zu^l=<*qDS1Lx+kAxng&@&rq$&eIi$0ohh= z7=V{o0A4YiV_v@^n+nqfrW@78NZ(O#h>wQ=K;J2VzIK(;`?2y>5C{5vhW6jwUxAzJ zQG1@LQBSCbd%s_XojI|S=70fTjRN^)_O;(xBGDtL5S1OIyL(2T))oXBB{U*Y3xjr3 zG0hmv)tn6}BA2Avoha3)*FUU*w2Zk<6PM~x6~#6eki>NN$ASP)!P;W~6e1pjr$Z}l zf7B4V`Blgk5GHfMCT>4{1IX5n|I}^VRe%O_9^i@Y*WaSjm>_5Npi;_fpsI0OA(ESs zs&w}Tx9^f5FslaOot?BVrM}}<{6kIJJ*3&P5#WiP5--5U4r)(Sx^~jz3%A4T&v>wr zjXGb}bd;C@`47uOpRS%=9DRQ_zQ*LKuhg5r595RojktJnG|A~!_yos7uapvSAlv{| zm~&DXtr$?17#%}DRV#S{8tO4c$O6E%N*{jJOmpgFHqd zCTk#y+ZDBWHnmoBk<9kLD?3(yJL?=0J1<$Co1TN z`Jf4NKxapoDkl#O$2`geV)+eVW^&FTC84faVx-{EDu^ThvkoYJ008whQ(>gXB?PKB zcs8RRyC+ZD!-p&LKSKT>GlbJGsQq5=$klM>H&y9NRHfu#9LA&!mMAv!F9;W@yDR`S? zw$y_8`Rtw-c>EKAaYTMn`R$DmPxPLk0x?)SU1j*gDfC%(O+<0@w|NzAVIzn@M7vLbXjWI)sETLp{o$}_OY)oA`h zbvCaCN#`?)=b5AZhBg_O^qln;dMys=KZLJA0?(K_4+{+62<>jl0PEA-+qwPpJ}h4! z{NECEI|WqLqnidY%>eR`PN{#L26QO>-0krMt%%rCb+Zcf-uUc!p~%DHpULqJ=eI8+ zoTIg1a)Q$Z2kE8iT;9#L@pk@7hr}EGL;QHgDfsg*`1)fDui%esj({erwg_Ni|imi(eMEj~-Jf_?fvAtOD`Z!)d;i=f$^gbp?z z+2fsUaSY$tbfiX@Z`(=D%mfFBsyN@@C6nGSD{1!rpN+AxKoKhIEPu%6k~RT3r6Nw& z+ZwMtEKI2uqT5f~nD+#6otk!gBEex0?zHMNR{ef78n^jN-0DhlYnsjQ zJabNe=2Jfw_sBYWgi&6b`Dh}T_IH&A=B_L0L_IN_e}ZA5`F#k2qjo029_ZfWqdxWT zFob3cRJL(%tAaY6;zg`^Ii{tR-7lqp{lG!?a%`!g0rn|s*Vw$qc8|u6{tDeVDXNkA zz^8}L!>31>dr2;v1y0)n$`xQ)2TUXl!Ve8fQ9DnC^7wtx5C_?fiK>|#+dUsCf*{hu zxWH0^ej6fHuHESmp?u}9G*rZ$X_D@>L{t0f#0yf&a7YL?c!@+k+)!(z+#HIMnb{>R8k=GAC>k$6koije0$h6l?-Pl^~;_TG}h;40X0|C0`M^>v6Z5 zb9cuP)__#)Rhz!=$wJC)CBt>yR8~b?dH)={=Sn|2KP}rz~ z?hH~xcom+6YWRg3VHWJf!UkoCDbYoAd_N2YPdG^!f6Il_MvCKO^av#=RHQQ09awH3 zowI9S2%@L|0s-uCbQjiDwEQ_zk@t0Kh%xH%P&*SMVF166)_VC2;EVE7d*ImRyOLf> z%|Fy?)jH`L+9!=07PqglQ{IK?N^%A34#CY4QC$6G&>$pd@TCEdH`fia&U+_O(k85F z!}71XTp6(1A>tK@tp<0!rjy5bZ zKvmSMmJM@}y3*&eY2(8DDOYmZ1r+8@l`QyAZW684mU)ufzv+tS_6F~0jd57tJJZl43!V9Aass^g zu!^&R{g+joV#&JK(UiM&7H;~*F+9Y&w3NHQ&xR~s8=n}I^Xu<-R))8Eiysx);@N7q z{ytGP;;%JWyv`KmKgbV2KwUrn16PAj~J@#YwuS3v)>nVpqJu+(>Fu@UD_&10$K zXQH4)=-7#kD?eY#M}|&~T$Ml-ER7dxe?RwvVsh6*ZIjmgF>nmD%S~MWOILppMEmNt z`h-?zi)xm2OC8Vw}LrnUqu+sq>$5Ea0 zrlREv>`%hDoKeW{`^%JCO;#t$0%MeqMChUvCWt8{Ii=-Z8ofFy2y~SNS?rRadaB;! zTTY67Fq4i7*W$!?L|V+H-H!micGlx-K1ft#1$9eiBh3e6wUzux-}iUkfM9C8Z4lV>X|$npD3!eEE4LpY+?{UjCAyy;>ur|+DH|9(M=spMAumD zRyVlJHcsaxM1z(`$c&BC!DH^kep9oZnfGIS3#GMrIrq`u!#BUGjbv&vRgHC-$?7Ra z`eTqJ4wM2+vd_9ULSLRp;^j<d$eAzFVbWb(G7_rVJvq%|hw+0&;kBLp=4d?9aW z$CmKrqm*DL!#yn#V~2S;?y=|})!JiwdX((lb2`)G6Qvt3Pi%URRu*``mP>b~euU(6 z*JIIlMF=o|JBMFNn)lc?@nVDKxI}G~wQ%Cc$WKKV=x*?yt>0&geQh zVRp!U4p5HY8xbzmka!6pk5Q~!z3{n@8hnZB+Tv@?*Ph|aPsKS@C#^hLzULEof?&C@ zec$J&5wJ^~arrCGzD|bBm(6jGqFt^9Dg@=D5UDSXwLC_uO}T=p-;JTR8Zq-ujtt<7 z$K74@_X$2@-CZnVU_wd#0rSaYN|bQ&r_}p`j9`yT1H|TSaTJ%|+t4U^Y*=+#wZpO3 z02N9(R{Pf6R%v3yio}wBtZfER;e9M|Q$NrJ{nNC&FYI*S%$g*RD#h%7sS#w{mWeU! zdlTno&Hfui&iHYZe5T^Avxu6`y0CA$JpwZda>cm>pDNd0qQbHF&2CsWsQu!K7s&NyRa+*9A4f22(@kT^B{^qz_vG+VZ{pBKsp z;Pz*5U6m6RfOQ^B{Ff&BKUd%tdHi6Lb&8uO9xOcH<>uZAcj%t!-WVC)(tCDclzLBF z+)h3(jse-~KI~8Q))Yc!fE~D>L<@5Ge zSF5k?RoM=s9C&6YTPsQU&Au>yMQ?NTx2l_`obApCf6*Fv9ULXdS15d-FWns2xTmt9 zo(D89$d@guKF^yNlO{z{4KJXmN1?pQ2`*wV=O0>bP^_FQO+BwDC-q7&<7$t&&7OZA zh*bIUO_Hh5^o(zy7jnCw;c_k;giUMQyF4meo)uw?8p3L(WkGwju=c5EIwM`@w(vHm z-L>2q7&Gi$F4bU{Z`aS_>343FMr);I#W7UU%3&8zR4DS)M3S({ecPR)WNjTscY_^M zK{uM6)dW8w^ggGivS)3|`dXq*YU)UU`ZgMl&Cx zgCS6Nh~z?8UyLC%)eRaBG1Tf<+N(tOU~2<&1ME;k!SXPew)Iafwexo33odhiPY2XM z8QiKf!Zp1ZPpcBrgFLcPa4>g9eMh<-Oz3cP3@TTvs(YsyMy7iy=$h$Sd}qMxQHF{5 zK9ud(>g{Yo_X}ENb51_vlR8Mf2lrEnQuMmdj9lLlL4#MZGZ! z+LB`EItDC~D#D&Lyh^KW=#fcXzo!Z}zvBr0{NYEcz!C;emZ>~ZVGi_6bX|!20S)Dn zRIJ&y7)yA~q_RHw_Ff!L@cIB(90UTF%AO&3f_yvA>UvM|R+-6JeX!a9xFvXfc2;fR zPuQG*^Mc{3v?UP4-guS1PW~E0 zSjx?8Jo_o!@e=I9H2|EFc%7d(Ns2#tEy;y3kJ$!+2Bc!3y6N?e8qm_gn?lcsNQ|(@ zS|FvT8Xvi4lz(1f|K`vOz9|wd8M_hCa7~b9=oFHvp-3#nYdq%#)VQ)oQaqDGtv%9*`s`$R;Di&OQHPB+{rQyWYM8%tx1iK z@(tU%>pg0guln61yI%aWzo+*l#nVB5DGCEFVe9Ko<(w;6`ee>U!cx8X<)JfDK@Klu zi5m2i&IpxBjmv2cGrd(*Wl`f8;DqhmK|IFNM9Cs57-I#89Bb8&y1=etEa&qwsj~9p zabSjQ$?o@CkGLIU)WJf5#&!;7gQMgf{L%WQZf9{C?V;4mCx4T41j8|H6=_TS^SlS# z{=r*rFjS0G1NDB0kbxLs{ z+?YmRO1^ga-|3V67gF~>D8T=Ds@o^i5aGev)C9E~(I^T&YrIN@PeR}RTNwh89nPG8 zsY*5a{&$~l!=shYuXl`CLJkkD21{S{Bx+7)0+$>P~Y)XZ8@(!-gPonpK73kw2SN>ZVV%h3i|u_B`l%X8MtP*!5o9(-^^a zL+TJZ8$+6>K=jGDpaD)h-)CKxkQ`|9=(`N7-P(?AUpgVhL8CEoceLi zGBmKhChuftBbrXWrKaBfz19b@E<)BVOk7Fn3)klk{w745sgL|BMeOgZPR4im<)Q*4 zW}2sUJVwYvtX2)bAwe4Idwoq@VcFKw=wNs)8xZMFG~=sf z3L=M4lXls`P>U+5LkD_qpVGJQXuS;_LwQ_SLZ+IC1&n!7>}jr*P4TpkH4=(t#XHdx z$9@_9L}FG5D1v1rh`wz89Y%=o>>~NA1$^sNr<@50;P#2Bzq%NGr}7lQ!kKFBS^pSG z9hb}pJ(#L0ni?bs+i6I(tx_w?+SEFBJ-oZeBaW2Yf7?bVV4{7sI|HF>p_rzV;@?=6 zgQkqgX0vLx|CC+h3}^uWfDnW~HS4?q)`PEM5Z#+2LhEvgiZZ%W_nCHwGu53FJ)U`v0t$}6aW7WSpLU)3GFaG lv+@75eqjdhbh!EHV{ylKy+Lar|rEsTb{{hUB3~&Gd literal 0 HcmV?d00001 diff --git a/docs/assets/images/ResNet_img_size_2048.png b/docs/assets/images/ResNet_img_size_2048.png new file mode 100644 index 0000000000000000000000000000000000000000..a7088b7bacfd70330a396943153a399c98a2aa0b GIT binary patch literal 15597 zcmb`u1yodFxHhVQAfZDDh`$n!nHG9wgX7ByI-}^n!^FD9btCxxo2%Zq!xpU`%vXb1JJ9qA` z0Dlbkae*UG?~YIc|J`+dqbPf)WQb-9_<&<6qb_sjPI)vT%Jd%a8Q)P!*ZIyJQWwnM z-32Jb7Xy~y3_Zm$vUMTkR?Yd37Y)I6BB;oql(q=obZe7RBe`E z<73@#vJ`GH^28b47txz45=3-nBk=hnA+tYP0r(Wm66l^!cip?AR!43 zW$3(cU${Jx@GaP1UJyGU%Jd!@^pSd3uqWbsb+WZLR@=HUIbYDIsQHOR_CCEFyAC~i z6>@9!2TSeCROSo5=Z2LZ##uFujO?La4cnZWT3XO(l9i22xMZh#=rcXsa_SdP4Ky{)YadVx zCYkqsPImC!*4L7oq3+uKQIJ6pJ@b%3*k*!VpJW{yZu>TrGyRj&)s_nb${l|0z9mj2 zXq!~OTQgg+fE^?Ima}oEVmOZq*3q!~S**y)NuBcvH8lc$GYL=L8Y@UOY;;Qw z4#wqkSP*CHQj7*a@J&bLL`!}sX*%kU^rm-v9g(v7BIMz;cpC)k=wdZ`qpAr6k+e1P z+Is3dB2O8_i4c}?H;ri}CnvXDv(XOz*iAS!*>YCxvT&R^)8JZ&Z%e%$Vp(t2O%PUQ z1w>2P1b+N||LFyUl}_ZOfpb)G-H#>}e5{~nqBgNE3}19oUz(7N&&=pKZ{;PX@R;}L z2qg(SWk-1%8X9hn5>c>maR!oiUzQdXRkd=vY`&UZk-n-o77V417tD5B@Z8X)H){5( zp00B;53Z)Yr#fEyVZNEzkR^OB5w$@jrW(iXd3bOmBD~&xR~6Mk$VFtR>G;{k~~fBzrW6rO?>OQ>Upv`!stSh&CS%+L2CCCSI+$MXbsvQ_uOp&8uT%; z@%Ss^Z{yjFja?<}#z*`Z#zl#Fk(5AB>$>A1vEobvEknW35KzLiY?Y^fg>KLOjt}U0 zzCM%-P5Vy%VXMx2m?*O_EZK2OqbuBNBi&ssQ_{Cc-XL^s|HZ{zZ-}L8LyrGveZj-I zX75_{##^&d(!kD$2(k@XM{DM2&)~h}g0CcD|Br8bk|xaNUyIf;3EgXNq1bEPdU z3q3L1e*D+H505_8eO!=9v##jrh`Pcn$$I+fXvOH0hoH`|c%jdPH>XEA!>wn6xMoV1 zCw-PJeRf?mjmRiiQ=tuiCzOP7$5XKX@uA~N5ag=9J)UJ{sClB)*g^945?RpM-rxH` zDvoQeqWP_8-TroItNmW{`GUWG%rDIb(mFiERp#ZOZ#A8^3zCn<5 z#h!pX!FxMH!n>xp3k5YOVrV0Eve!}K_a0@(dnXReHGy;{VpR-q?AAjOd6TwIRabg7>4*3b1tUEBhc> zIIrwFyU%DOHzgL#I$!amPLlF|_;y4-iY}MtTK!M=C*cO0=VEh^@)JB)dLP&%zI9I1 z)#0C~_nltQ`88e4@53xR-8?k0LbQ#2$H9#)j{U_p%XDg}!={@=ZV1X}m3pxZ z`8Pk+p4H2R-y&Vc2*@VY>eKZrG7G46q1HYs(1^enVs}kqqjTuXJWBFkwpx$ksk9i# z#b7_UzsseaCW1=xWV=}NqXnq{<2hLyePmMz_te4kq#U}wZ-1hnLC?wu38JJZU_4qa z0SA?*?v&wV*mBPxoLoM{=7EI*a{7!F%Q(}zU6%(OsDw7IVyqE++?7Xm55i%*fChycRbF5+=5?uzVKF&j(OP7|<*mvzncB{=K<#6~8n;Sje| z^ILt9P`HyGdS&ss>DeQkWbwMcKMMz=;VARtqXDiE%ZgP6t!wP&vX$8jREADUV5@a=eQLN_Ku|JtLjxtzw8 zy6;MK!Q05w_+ym8l?VOGy$%7HuhBy@ED}R1YhV`^;+Nho^pCKdqCc}Z-jqBX^+sJB zMkb0~`QPk&h4H0sC!W@tNlGw)T8`pNr=a_rs`(%eMHh$RqTP_FwOKIm>dAmq9e>|! znebk`Y`MCbPJ{s^xd3DmPl;@S+M$W0A>Rriq3<8=FyG*+6<8^7u3#)nGt#0}5~cM; zry*1rJ96eIdsvGY-W7X6y!h;-^hhmx9xs&HbBNE>{8e(Mq~%tuJB=lw7^f1U_0csv zX9;g$jI#Fo_ZV2~em!HBkKIDkwBFW8JFE+vgiBsr?&u3KVFb0~8uNn55+S$B6yCsq zfYkA}!jOL<&5yT7%U5*s-<~MapW|4DSzPx_hFlE8qlS;Bl?1M;fM1!8PKN(# z5GMEc&N}SGVrL4B^?1aG)j<4!t4f`+~?x5aDB=u1dud zRhg=%0d)rq%R=g3?e+Mx5zRXLC6|w-#3^w`H8iShuMfl>nolPp6CiW2g8;}>%+%`` z*SZND_KnR@7b!40;(J6$DwCXxI)Ar?Pyj&y)t78U)mzz;ePooDtNmN>Lsp+#QC56b zmk@{e*RM^Z6be4!u+Y6$t^VTe-w6Jw+c;8pNvDR6%jk;Fhbo@<2!6~%MyE1r)6DvN zC%lZzzPL=_Fm~1*e(3%{eHJ!XIZyd|@GQ;u7iC5Wn{jw`}r*ac*+Nb6Pcc`yDeAOkb?%p(n>H-4TcwMnzdl^L+JGD4tQV zQS;1Tsu1!@x5if2L+~Dq*Z{|;FZSs=fOnx62g^`2>hhj`CkE|5-+Tm2uk7PkM82`6W@0dzulM)w z-$_oZJ;@yUHG$fgi0Gp)mz^s$mZ}7Rs`L0e?I2D|O>J#Ug{UVMHx*sPJquLyghArA0)9`zBlz-7m;?*DlT zl>?JT26YNsOhyC6&3lNv)1rMC`bBW9=RXc`oO!JE$3p>P2L*5)bo}SLGuR2W;y#yL zthLWR<8eRrgZ22Fb_#S04XWj^z#Y+sZXSG!eVu|d{N0PLSY{5~OpB8XrMhWyT2;7y zP7?6*W4(Mk5{L=DE1v=&9*XD33$;@{kJppi1Mu`;F9OWzs)5pn)L{`y4C zQitWzUtXBDJR`Pd8}=<~Bxab7QwzOxvHJV_hc*5ClOqhtV1AjFq!}8t$y$}TL=jPB zNgu1B9Qe3GK>fJ6wCL?fyp={ft&b^$&qKP*aB3E9L<7Fv`(=IHI2o|36yO!JN(w`g+~qddkI1k&{HxqdQryR=XIIIzp89WW+MU?&2VZTr=+OOn7z~; zwtcW(JXVnrq(lm2EUXz#$J?E;=HfQN2)q6jiH)>}Z{`?84h1KiRg#Q5!XL_|oKOS+ zk24b;OUQF>np=4lBu`USyOiH0_-J8x^hQS(7+Mnb{ka{FkUp{iG5Y`#D5<9~QAu2! z^Imq)q?lIBE$J93wa9KzH{3uj)*eh8G@9Xmo%|6;NYnQ6=Au_LbBSW$@asNqN80o` z>MylrM`aNGwusO^{2)eT{vykBV-WA`+~r{VjBnn5v1cjCcY^{AtJv>Fuw19vTpiAy z&zVS}9?W0vQabHNazz*>$KEf$y&Rn=)<+0T8kn=p4R5GkPM;iWBS5f2yyLLwO!Fa$ z^R%T}#Sco>404@LQOFBuo2rzBeCT9}p;o92g%hl$Y3Vk|`>?|# zyGj~wwmxfE@t&9~XQR}1FQxb5qAu4xQ=Mhcq1TtSsY|#MNo0C3%>;V~Z>bTl&&^?{ zbL@lnPa)APlV16176f^f{EYVPrDPh$E>7ms2JU`|xwcq{mg>67G|gH$Q(%SAi>fBr zWaqc)yvK{Uu=HLi3$T$pGzeVE@2oauH*6jA`dQ+Rq?kw9B2!Z3aN@8=Daoy717Qi{v>AkF%9`##u(0dlIn2F<@|%uicA>8! z{PJ@tZ(7^S;Vsh~;?eq>8@9EBHlyeF=VMXpQcGCua|=|6{vC7Wi1)IpJ*pYpjK53A zDJ>;f?iv##DOBX1KK-tx!-vWp>OOt*i?Cmx%0K=2!n9m^!?#Dn@ZAWPnVF%{_j?gu zmibJwI~tbLh1#VjU0`(6Fhf0E$IGf>?$*rbqx(UD!RPr) zLat5E38A{3^SC89{DlV@&XwGm4+BNVyL{K@JBl~VtUxwZVovMLAWd0Xl1-@1^^UWF zCTq)4HzC5Gnsd8s|41wr2vRAX?Sth`=>9^pR-C@x$z9Yv+MX!tMAvF&AqHwxH!VsY z!v8gqIQ{FS20!#fqLLPW&PPUE{~5_}+&5;#liimGSn2PYI+>55=f5SjL?^s#F1y29 zHA3mR^scNkfKW-3FeBW~(wW^fM|%aswI&;=p*YA_9}i{m?eXxy9Xx zL){fx!K{$ITbti^j$2DxtHubNLRu*pszR4{rYjN2e3mG)bQL z{X3WAO_#^sT)jNJ;Ssr6NBvI4z~IT&7yLw(c7AEfx4jjfWR0)u<_Nv|(SZyJ_LO^MJ-+g6LE$1gOi7TsxfRXGF6 zSYA%410dy>;w}1ScjAc(uTEk0ML@L)ur1Z#&p3E}Cg2IEE=%i;r25LEGf;ocz=@Fp z)s=B=M&360Tpksv_gqRlQWbU6aS5I|z4%7Su44ixF}Q+lL7?Ukp`r!luguA=+Y=Ub z7)4Pzhl!&6y6D&K2?jT9?#1O=KT(~^w4BLO76it1aaUo$=#YWG9$poCE?}PQ@$(5iJ*yduO#I~F59nZa=ej3;$tC=2 zxxnRnFGC1<2*t{luoB90zQ{u`L+(i{E2t|4b~F4KOnXjZI`^Mown&Vj#bO}gfE9R;3?eRd#O6U} zF-^_2I(W@vf+c?$yOgQ`9Lw+Ludj%W}6PvlagcfLDoDJ2pc#0tcYm?mk& zRMcCAz)M^F!gz;(EhL9a6^7)DynC)ZR(0*NdAX5Ui~WPx0!J5ki9po;IevO&x791* zydJqbIqvmOIw#-=cm_mKmt3Esy7LQlzUROPeoa#PbulAJMz1A5JljhCOmOjk1?pff1xt}fAF2dSG?SpsGWXEOndFnoUHW{GQw z3lpo_!^Tl5|HS$1KRFm;c0WcLSd%;-W+3zkG=@hheaLSZyr&9T35nwV*e>xC$Y3aL zUT*+hN*V=fcLVb+U~oxe&iJ*bLKr;)IUArZt?`?RFFtP%!>-*B&_So1oE zoh5MS50;sBcI3njPfkpvh&Zn&rKCLJv!4^*R{R%OPf1q7!wJ~Fzg~9D$*=V7c?)3h zMB(@0Y6$}GQ`Cf}271GeG14Cw-H8vlyP@lquCBGBwI!JkCYoQGZ81uE9_6OHKe^j8 z$`SXSn6c=-m@0cspuPtt9Vn3kajR2RB(_6wu~=>%GTr1^xi#NdE73FBt)WUpBg{Q# z1Kk)(pJ>k89_a5c3c}PYn4)5RTI7xoD=Y>&avt5^1WEMf_Ij^hD7OBH4;Rg3vUZ$0$V%}#o`sRAJSRn3?m|03R#OFYU&$I()c7L0F z3ZuODePs!08Nxk$|7C)scFrtXyTrg6&u1+K2dBIs*{dg#>MWyBtGFUNsyoWW+1R)X zlR4_1_qlhS42-f5qDCK{Z7~j=E0g#Sm~c$yA|&napZMP++;QM6`>z02{4gdZ4wO9( ziZHiBBRa>RU@K%lXVtnTFFwD-x}l|XTLzpNA%7kwA32k}WCD0w{+*Qz$Gkp1 z{$p88;%4|7dqTpeaRP8m;AQ^Bd}CxDxS$+5f}u@A`AoiDp{1$1s=#a!C&zU$7&Unu z>PB2qj`HscSywjEv$4q^Cd#G>+Ua=(Jq?8aC;j+bnKv>-k375bHICiSu6py~+SD`K_;iw7jXzrRDc_t~qjR#fm_dmsH# zc0SpB-FU(fp(YI^JZmefvh7FzcAV_az$E6azkfe8f*ot3oG+dSef((D?=2irlD@ErbWyPjQXq zqkSKj1--h)t|ndwAcLOiXE0^Q|40DSp=A{BQ%j$yCdufYK(4WN=jKg{zrXr-b95uJ zMCc9Gy)h;nuhYNd(1j+?nrM-RpI`TpzAgO!Q6oe77n=evHTw~h(ZAwc3DH&R6fzN* z(9pgATP<%_ecl{8a%HLR>oMi|T?koX{GNh?pynkNoL-a#pg!Nwx|QI&AN5%tWnWoN zrpiZ;2PqbR+oL)@DQD#iJ6Uuoehy04%TL-NEsyrTykG<`2QRswW9|)@ zuQ~!uC~Fn=(Ro+>M_KVF+{yJv%QVZ0#@=6b`Jb0rvGt_Xryx{2u(Ftmv>m*!T@Ebz-G)jFJ4D69e0duy8xQEKkwz(+k;}hsc>lYM$2O&7f zP7j20bu|!hLgUkN>2pKxe4rEb;|VhfMW3uJdxKDKbRNgJmMPmgD<3n5PyD(U4u`tq z&S`A|sF0T&s64JNJEd@^DQDbCFcxF>kiU=0D4yva`@F1;i{-wvWhv`d$cH=t^NKQb z`o%XLo0yDv?VZ7TA3o9O3^>QN97#wuefNVBpR)T%1Fx5}lpqrB>~S|f5pM&Ovhm{S z%VzG#Lv~yGSieA>(3)4DzRb$J%_6V;Ws^NbXCs^`NKWo|t^0GHV201fc^*RHmi3iHiVTPovqw9A}gR*?cH{&WZ))HsLS>yD^Ce$E}Ow><;hL@+*etj<6Cw#dFuvcQ$jWtUfd z=sr0@8j}an4`n(iU*d2j8Mvu-!!P(8C=WrO8sCm%YHOSaju<`4GNo@VL>^+_dgFH7 z2|*9z5$4loAUzQ~FfU@a2(8&tyH}^)|pZF?+C&W$C&JA}1Soan!+*8H@ zjcW&6A-+9?TwigI`%5$U3Z#T)h!|IVuTuCzGWc3==N=!sZWDiTk~lBqwS+ZHh#!sp zzS(ixq|vlMDs~WBd8}#6q)A#TTrc0d)Ui1JnRhXn`r)_1^k1vvj7kSUqNMLSrG;8qS$GJFsHp}{EyzH92EoY;A=ZF{Y z5mHPo8o>T9(11}{uR{R3Mi%@J1mP98F#Q7o*&G-UkS#{{zvJncrPA`yYw@X1SIPzS z+9Bt8t@{CGNvdL$gW!aHpC7L!bJrzVZfxzZ#%pAS|JanocR3n2x)rA%Fe`r-&+YF7 zG-o@o?P80Biyd`|_#eIDU>{4f9HxYj>-|y3z5BO{uOB-o`V_#p{LhK}?Bg{#PYN9R^EyCBbjc z;X=kYf3<-DiC>M%G=(v~0(&UepNy*Re(((4-mP7<4?;$Qw;4vlqkTUEC%iWu&+3F!6$hu+`c6;iFF#z1A*f$;ahEOs>y%-LO%`x|Phd8Q86yYPT~?-+Yw1eUJY1ZPnd% zV$m?55!o2sI2cowj8#PWXKFzb_Y6EA39*8H)M+I1cK2)n46nkz%NJld>g|L{5)X}N zDiT7fZ6@;vIssCYy5MuH?md(@9<)J6Gjuu0j6eF?(9mw6b02=|y{%xNoT>^s+z|Fq zF5jT#v?Fd>nnrcBsumI=mp5YIc5kHN&6&ejV5_@rSh-w$#EjcU@7~38f^=-+ zLcYOj*8C(b`mwL;)&j(@6XBPXshsSqKIn3yLq#wi18qaX!^ucB&n3@$W*_FCgD~a+ z&qo}fp!^5VKqa7*gX7CyQFq_-Ja&*I>B>WYSvmUIN=uCftk1Hhrs?lduT96gev|Se zM~eL6DThbPfuA@^@*l4Slq1SJgCFoUy6-Hd#PHhl%Sc!i` zKyG}9@D&-!!~j?8+`0^>m7YA^V=H-2I=d2lA?iQV8n7d>Ssc}d7StNo#;FF)fuiR@ zYQO4(H9ujg^Eqb`Jlp!JLjyu(>Q~?k(BVp_>m>@l#%5fOUSS2%(c4?XE;=Fk91mHa zEF5g!+E`g}Ye<~Nxw>!M+@*LVm|(*mY_(__y9wllnP1qB*4ek9GB=Ft>HK_eKGn_L zlG>X%RdvF_q(buiFVwRp(eFQEp!AA!{_F49A2ELAXe+ByEd8x}jjTt}ICX%Dzt0A%dz^uob z7;>wtAa&}vR}vO1L#v$IxPk3(Mbt(Q%MByx9C~8OO~Ck5L@}>JZ<>F6ImES^>fZH; zB1E^frO{^S-Vv{M##55`kan$ChMp!}Wj3dEe_~Zfoupu>Iw}b-m#g|kmofDkai5Fj zkcra$y%cA^rd)%OaOsPJm+7o3MJkAKZDYIEXO765ZR6VpaszF!6|37`Zt~GVv4byL zx*(9`E;Cahw->!s>NnD!ZRw7%!w6%hnhY1K-!Aw|9Gqc{x~fq}^cGYjJE8zL*7tO? zd<#w)=#;uWma^1=&6D(FGdA6>JD`~-){oxgoc<#wNip*Oe1d`h=q3a7f3g&gD4V=} zUa5GQ$wQ9C=&TzPwk^iy^RtUZwRNohy%X}+#Ha0@T z8QhNb(wR6HkA>a(*jR6oaC1?l?C|%khk+gYT1Y3hUk9&S_i^pf#!%glg=4QSc|kn) zV#A2%6nE4ndotq)QdIs= zcC~)dBa_^aYhi?p4IQLkv^t<4?<5wqv@$JaFk0vEIxZso2MD2WKC*E;fpua+`Zzw= z*zA@^vth$vq__+S0&z6K%QzNP{MD?5RIxbWZ9VUJfxdiSG^4czFCO#3#b{#7dqIZ< z+Z$n&rRwx~r%B6)Tlk9BZWhheZQ&c~nM?2)U*=_`dO+(HXan^w-atHO>$Vq;Qfz!n zw@AsyNclDH4l;Q>IjLf#xa*Oj;j{0fW2bkPSNBS1PNE!qI(v3o9vY;FI!d1Y(3sxn za!Kb*Z~L7tF1ClLOqKL)@H`#YwlhkX^qt9v*j$wNf-4sv-JaF93P@eFgLqfG04JdZ zLNP;Vx%IGzx<>Mw0r!tyj3Eu?blpMO@+E0JdDxgCdpuF}zSh04l9P5;I(t`gm~ITS zhpKuh@rsF*7YEz-vKckH^|Q3r*ay9w21C8|SfFzjbnbv3HQRO_ztci-)oMY!jpKRg zy`e_bVQElGN{Wl#jkpRHtjTZ=>ve@aXVg&i0OJM(we9CSmwibUbX-Ez70(i=#!|8dPCvJ;#0fd_*L6rRm<+k$@6zcA({MYdvne;XQq-f z|4mv%N$wxx?V}js>&s9`$Fk^Xq}cUzhqI%)cR}mps;f76owIBdv<}v7v)m@_^~Och z3HJ#?(g{A-3N9qkGAe#n(}y+P942h#4|isF5QJe{x|-%S2;;zU!~`<+ErM&`FRoC)&x@86xbzU^=% zji}47m-<_y!H`kej4whQBnS;W3q-u1td`?nr6lMGjaRtIaI#V4OOQI^lPI`(mf z2ikf4c&3q01o~?zF8^F*%4Lby#ruf<6tCL|T~m=HhOocLXuiJH6vTb-H`|y0i@7_)OW{F8ifYrdC3{!W+V~33rv&PL{Iv`{{2KQh3Ow0*Z68%+e=dBl zQ~fJ1wMDKqc3nfc^4W&Uv>bjHcKiD`aJev* zxI~Xl;L2ia85`0fd1CE(sl`*Ak67j;gU}kEJbca@7pq~ax?liPN;t1{g*C;)IFnLo zMzWE00=Ie3>|Gg-=qBNXj8?l?MEqlXiGs+_w?5^kR*yOuZH1Z!bc%h;CG;5Z)aA3S0kx?a zW34$S>#Vi5cO|fli}!^-?f8Z*PjOd+EmyC+`tSrt_q67$A8-VXfaBlkFL(P>+_}2) zP!MM`k~`w?)49(P6tfQ}WjT>I>uXp8U1pkhn^vy{BocMaHtQVjAk##?{?#Y?Ml@XO zwH!pbP9A6Q=gPqq=cLul^o`_?!l_;k$R9GUa`s@Et2uav50fzgqSt`Q^xq7vr1V=c z%h^7Y=GW%&0=eV#1HOR#UVU-cWRb4>;3nFg95k-*8#~)Fb#U zT`1{gTaPrm_QNTUu>N3gG=Fg;rEqAl3=G?XuEAe3?SROdcg+Gv38WrgqAQug2{}jQ zScp8qMK%uKD$l*r^2C3dfuaPQ&!yGEyeawm{X$$Pe<(YgIv)rnoo-LgIzGdu+0f_j z2uswj4r{V-WhmiV`|z$LBm1d;LpbHv1iih@(X7HKn@}!9C;!OT=s(Kzmzzcey6ykq zt@@I(Ub=ElzIkAOcNEiF^l19GBaTxgBTO|0r} zy#o$Mp2ef1=bHKnTh5C%jfh+3MDBV1WLtY&=GUUJ24K+-{jxFC+Tf(f9O7pnAGTfS z2pa`7*;(^##UX8VqtIHrx+|5X=NIA#PQTQJuTr#d7wt>l?JrX5&`~IajP^y=nqPAI zTX2J9(a@j%PJoT>Ix9v#2ch34s_l*FwL1`!nvKwNcYmuCPZM}lt?*G-`E3fvWnWOL zQ@^^OQr`ADFLd!W14b};nipRs@(e{-7ba-4Kb@Rh;pwm+7dRnJH zAW+BUzI~Vifj~TM5Y5Hd0duNNvLy4jjR)~kPpPkqjMnt!yU~ZAuE}tnZ^B10H~x|1 z-LJ~7nLP3L-X(*T>2_4|8F`hl`eR5{&$dzM!|e(7v=J>pV0vHI&$Y6c+;6a<~k^KY#u#x)Y#H=o*i{#P&RxuQ*E=$%CG!sJZ>s*a|q~D*_0-3S{q0#&~`Ib?f^H=qvj0F4{f5ebV&e*p6c$c?OFu%3zwwaS%K6Il^0ou z;WY}589tN@PG#W3<<5{x64@#~K%7#ynJBXMx?D@#Jp^h`sTAxw!;kL!OSmnehhKoE zeNhV@t5FjiD*WOlPUtzDqOwVF_qL@!;(WkPS zw0tKmuhDjzcQ$vssGY5(t^<(ge(wcTW)x^-+5?5+_}Y3QVji;ludgzI_8md4=94!@ zHh@g!SJNjxQ?T8OEd^kiDlLga2*1^<36H@fo=!paYu{np3*-cmB&KFK1W-rr?TI%} z2%;#Rl0;p9J_tCW75UiJKp4aHZ`xpxL%L{1;z(f!enqC#R7t(*uBz@O=7?Sw>s=+9{_ z5GexdqpDmFkncmv%0Ia8OqrINI;(fpAK-m<;%e1xdW`A2n8i^O=Fc&=t~BlY3_vYG z9!k>8iClUDE|IxZZ&FGlXrK~<#3nlo*n;Y{;b_-tH#lv|Xb+>?4W7i`4}NK#!D~x_Z=2 zUWek~$M(xJ;vicC&$W1HcQ|>zaVsoK)rk&GNXf~M>4`HG6#^1LCDLMRG!Kf*knl8q zB^=ULz4nxuQal=vCKn|$C!{@R8r@5ETVc(fe_|BXqQMe_gj`L;8RXfgK(0>A1;XZJ zXL?MJd+13j;87+Q!SMjbbaPH%+e_=sbU>i5CR>N<1gQfz_S$a_LIl*iclG~r%n$H* z9GD!pVS|!XIqR8RwnPsMfXZKd3?Ap=uaNEt%2c2kEg2*4`gBK$cSPS30gZh{CaW0( zqG!e09Y^{iC`*Mpu2xhgy|(Q=@!r~VQ>-+~0n*~MaLTA6zbRZv6< zk&wHDJ?bOvg;RL!zJaBm%Qdoltv))__z&Ub-ciVjVkx~bAm%A})1vq4H^7biWC+W) zsOt+De#EsCXW7iy21?k1QC~;43KtWoo4j@TTbQP}V*OgZf!@<W2y_P(dBrKQKVb8)z8O&H~9olyh`_dq4RfBMnqydYOWrxiKxQoz!s z8-eT&+jcgbC3jRG&h?wSBKevuWc~Z^Xu~S9@!V)6jiy zj`y5zg~b1Qc)Q_5wC>wE_f^%;>EkU;s!BZr{DW*&Xo-lk|q*Tmkf> zn(Hl4HtF-ohs_=IHe;#I0_J7ia|1_sNxG0jh8>hLo8wz+MW}q^8&&{;ooBxR-j6mP ztHZDHOd*rDJ2o-27$4DdFsqxKuouYkMkK~6+PgZ~J=2?gj_two5eF68z=Mx3V00al z;-$6}Jjm#T@cBmf5YFxwkDWJ$1RMZjvKfnY2nTx5iiHCjDFgbB{O+g*%6l`?Q|uCG zefSZ&5CJtxA-6uG75ETGu9y-Bfuo$Z(SN>bq+q;stwZThXc2b`%Ov33xv`*@jwfVw zVHYehKhUnCA|J5p{Y-l%j{XdmNO9s4v`iK;NXqRs+{S6HK0&RVE*n|5F$m^|%(ECP zbpkHA;;F+o9hF&O;B!FNy{@6YlSbFjuE)P|eW|$c2n&gP(3G9WaAg3n|3snnq^1sP6|35%} xi4-jKMXwa#K@1+oIMRsozPax&tlwgX=6s-|d)8eEv?$+EmVYT%B5UIRe*g=qbxZ&N literal 0 HcmV?d00001 From df1cbdd3c100a0635f78430913b1a5b53d925c8d Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Sun, 9 Jul 2023 23:40:09 -0400 Subject: [PATCH 07/18] LP bug fixes Signed-off-by: Radha Guhane --- benchmarks/layer_parallelism/amoebanet_lp.py | 11 +++++++++-- benchmarks/layer_parallelism/resnet_lp.py | 12 ++++++++++-- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/benchmarks/layer_parallelism/amoebanet_lp.py b/benchmarks/layer_parallelism/amoebanet_lp.py index 990d48e1..f342610c 100644 --- a/benchmarks/layer_parallelism/amoebanet_lp.py +++ b/benchmarks/layer_parallelism/amoebanet_lp.py @@ -4,6 +4,7 @@ import numpy as np import sys import math +import logging from torchgems import parser import time from torchgems.mp_pipeline import model_generator, train_model @@ -14,6 +15,9 @@ parser_obj = parser.get_parser() args = parser_obj.parse_args() +if args.verbose: + logging.basicConfig(level=logging.DEBUG) + gems_comm.initialize_cuda() @@ -139,7 +143,7 @@ def __getattr__(self, attr): [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] ) torch.manual_seed(0) -if ENABLE_APP: +if ENABLE_APP == True: trainset = torchvision.datasets.ImageFolder( "/train", transform=transform, target_transform=None ) @@ -169,6 +173,7 @@ def __getattr__(self, attr): def run_epoch(): for i_e in range(epoch): + loss = 0 t = time.time() for i, data in enumerate(my_dataloader, 0): start_event = torch.cuda.Event(enable_timing=True, blocking=True) @@ -179,6 +184,8 @@ def run_epoch(): break inputs, labels = data + temp_loss = tm.run_step(inputs, labels) + loss += temp_loss tm.update() end_event.record() @@ -186,7 +193,7 @@ def run_epoch(): t = start_event.elapsed_time(end_event) / 1000 if local_rank == mp_size - 1: - None + logging.info(f"Step :{i}, LOSS: {temp_loss}, Global loss: {loss/(i+1)}") if local_rank == 0: print("Epoch: {} images per sec:{}".format(i_e, batch_size / t)) diff --git a/benchmarks/layer_parallelism/resnet_lp.py b/benchmarks/layer_parallelism/resnet_lp.py index f9c68de8..cdcc2b90 100644 --- a/benchmarks/layer_parallelism/resnet_lp.py +++ b/benchmarks/layer_parallelism/resnet_lp.py @@ -4,6 +4,7 @@ import numpy as np import sys import math +import logging from torchgems import parser import time from torchgems.mp_pipeline import model_generator, train_model @@ -14,6 +15,10 @@ parser_obj = parser.get_parser() args = parser_obj.parse_args() +if args.verbose: + logging.basicConfig(level=logging.DEBUG) + + gems_comm.initialize_cuda() @@ -149,7 +154,7 @@ def get_depth(version, n): [transforms.ToTensor(), transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))] ) torch.manual_seed(0) -if ENABLE_APP: +if ENABLE_APP == True: trainset = torchvision.datasets.ImageFolder( "/usr/workspace/jain8/project/cancer/1024_1024_5/train", transform=transform, @@ -181,6 +186,7 @@ def get_depth(version, n): def run_epoch(): for i_e in range(epoch): + loss = 0 t = time.time() for i, data in enumerate(my_dataloader, 0): start_event = torch.cuda.Event(enable_timing=True, blocking=True) @@ -192,6 +198,8 @@ def run_epoch(): inputs, labels = data + temp_loss = tm.run_step(inputs, labels) + loss += temp_loss tm.update() end_event.record() @@ -199,7 +207,7 @@ def run_epoch(): t = start_event.elapsed_time(end_event) / 1000 if local_rank == mp_size - 1: - None + logging.info(f"Step :{i}, LOSS: {temp_loss}, Global loss: {loss/(i+1)}") if local_rank == 0: print("Epoch: {} images per sec:{}".format(i_e, batch_size / t)) From 256d9ad45632b2afaa1e085dec2f02ddca3e81a0 Mon Sep 17 00:00:00 2001 From: Radha Guhane Date: Wed, 12 Jul 2023 00:37:22 -0400 Subject: [PATCH 08/18] Update Performance Images Signed-off-by: Radha Guhane --- .../assets/images/AmeobaNet_img_size_1024.png | Bin 12767 -> 14297 bytes .../assets/images/AmeobaNet_img_size_2048.png | Bin 14147 -> 14297 bytes docs/assets/images/ResNet_img_size_1024.png | Bin 12424 -> 13952 bytes docs/assets/images/ResNet_img_size_2048.png | Bin 15597 -> 327725 bytes 4 files changed, 0 insertions(+), 0 deletions(-) diff --git a/docs/assets/images/AmeobaNet_img_size_1024.png b/docs/assets/images/AmeobaNet_img_size_1024.png index 58864657d54ae12ff107fbdf01e1695958c4ec3e..1f587c622110f9cc157d4fb91071ef590401bf7a 100644 GIT binary patch literal 14297 zcmb_@cT|&4x2^>R1R)ebq<4^Jq=Xu(^eRYE0RgEJF!UlIO+pvxT?B&k4$@UZM_T9z zNT?wc=@8m|`F-DA=l*rpIp4ao*1M9o%*@_<_RN0fnVE21ZI!!s9^AQh?b=;6Ri&rb zu3g6f7aP%S;0fpTVOrqly6aOFg==L43|QcXz*b&M{@S(5Sdue~TfjZ>TU8U+YuCuy zudeG|&R<_&y9RMmQ<69EGTY1yZ({85URgC)x9RB@7Ea|@L}_EhBdb|stHrHUJ56rE zn|pqWz2b15{KHuBqwdk|7gcwOnk}Z@gNg}K7&%yJ)L2Y6n+(uxtw+zG_r`1+jxx4f zxBLPI^#`%~rltqao*f-|JafMMao;Km%vf#J6|+C8uOlJ;PpYB!e@+4phyr)sM+>f% zhfV??VTnj8ukKlCEP)(IMSfK0eJjl{V?;E#Gli@YM1NI|C94?lR1&4&2;_A`{3|2q zzt5OnSp}P^bGtZMEf9E7*WZy+XVT{H_q^8OnWV?Ym5l5Rd$%4&)e|Gv%b@_}$(~lLa{jp@`U4rSVyZ#cL=r0v$ z_Zhj%R)QNZEuIN-Ni>58f38Zy#~j=F_8_J2!jxx3b38a<`VBz`=ey}#YsryQEISpXL;rXs37MOP$uqblcUK;|76pSszj1uqyN9uWCCE{zZw>}v7A1GN z{GgWj>Mc7Wd`w^y#-Z)-`PeY^8{elvcQKG6VuSe-o@8SbexDTdHz zBp?P4giE{ra8N|>D#El+_7w#YV68mf7M}3bA<5mfWF0arA&j;bJR?%8p-DxT$7mAv zzKaUaWJGra>`U%7K_w6t&mHG!alvas5Ax8P8(-tJ6t zLc^_Z%OHurOAJfD5s_GpnLau_8lJYbGs5V|<=&*HlD!W^0(OTVACh3gtr#m9XS&3D z!|UEV8u$^0R^|GAew=HnU=2ykh3!dOwd5E$g@>B4J2}u^bouFuOflO#1W$SfAFA|_ zO@BDE4t8XVVTrSxIrHJ0={#4kw`8!daJ8{PUJOb)B5tU^>`#}3Z_-MoAyW%)eo0%s zrK5fzXLqXblN&CKSoRk=uIGO5*@{-aQ_JPdQy66GIUAL0Ky&!4E zwzEN44?3~CuN$^lpRA>2y#(W-SCb|6hO*G_;cuKIE2(yqjp-~cC-iRKI=LHf1~l#5 zR?r!5fsx5%);7djBKVLH=PH%Zh9Xi0oOqdC-TQ^g*Y@?X;=w~}`*i6tUUNq|CEsW- zYwlB`gI@I0K`z$uM!iXk=`IE7CvV>_Yny5Fb@H0}g9K5;oyTTmMp%PGmJ?BVraV@G zrGt@xDfxFL<`?EmJHeArgAV&=m!g%UE9 zNEzPgQu2SKin}JiwSOBP6fFKSaW=u6Ne6FC*v;r)Xf~Ww`(`3bUv^~qKme?osWFu0 zO~3Du-V9YZPfISxcR~jg9V7M7qk?Az8(hAHI!Y%ZwQ6Ps_k9kSmsdAU)~QAgs4UM5 zq{I2)aHW#VVQ6nGx_R<(5k0_3!W8eG;OiLSR-qkUGEB*XF>@ICL!1 zMI{<9IL%+tFbWwK?C5~fg_tcX8#BY+Kb`qlmJPCE{(0{-${HW$26ssNCcRIk;@EtQ zi5zSQ*L+11#C&wKW7uQ*gY*}WO1efYM;tSj5EwebLcQ*MM0Yc464(YS{`nJsGm22` z?e`V!${sAQZ0IN9XA)pjGp>_8B*&r9hEe0%Upe$i$jdIo-buUIRV zIG1}MH&Tp9EAQ^r_1wxG^#CWV^Hj7YN0a0vn|h~uzgXIwE-qC+=SVy3VHjWCeC|Mh z)v_$iAsS*;+9#~2$rlf-cbTQW{3+}=o;Sl3G$&?#b_w0km)U(xU19Q5x`Yu)e-&hf zuMs(^b-M(L*ckuoR*N0??#@(-#Z}Dhf1iL1-s633^`YZ`6LbH!!CEsEgtJ=SVV{dT zzP@_~M30JeuR3QusNo6`+|y@8A&M2qESp@Gd!N5P5#!0`Sk$+M(}8eYtPlVXG+JUp z@3TIUJ4g@yP8o8+K6Dk^1iC+7UcKZB45zVFqS~>pk%|JvmvNvTNL)!5|5J$sPCX?b zY77h>%I{N)>aGw*DhJnd1ckwD$g@T|5*OC-OFd1kw!`T+?l=F1T=ws*b-c4U4Znv` zMI*$uC8&(K$bLja-c-V<9f3k^C{xr~mF<#&NPc(!UA12zMd4@1%>CN|i0wIQHwc7gv6;b^xq{8mc(V}}`2{u&~ciTIaVPoPX`U+he3Q;yw z{g>Jb{IW0F-Tq8_B7-3%qH2|PJS$zKCl;(u4jKU0^cM#SrBUA0IKRjO6{^gvME$HC_ zqnpykAWItDb>%qD&0)f|THi6fHIZx{-mx)tNjslqW7v}8!R0G-T3fRnQ9lYNEW12i zFF8NQ{w*zD$9~JDcUJAH|DC2S?Pny}usFQFaUSZng0OYajC2D>oc|3djtQU7rtdK9 zQgr{#(6WkiKjE&INzvXcj0vVvSs{J-T(buBrl`T~+$dX~13)-%RKHKa?P1csR4YPs zVWv(2mCjiW&98QxY%1B@TfOPalltgoNkdYax5DNF8RC+BWl~IPUB!M!ljrl=D*pIo zx8tNfzCb_E>40CCr3J1>W`}JNr*jHF`;^LuktijSou`XKo>1fY<*vY=*V_ZzuLA}4QWz0ZL&RCg#23)QLeB`Zbof_p zV%*o4xkq-o0-k2aF}H47VgIBB6f*Q=$9AaIQta^sy+B}~YqIq!Wzh+c)7Eu0x^HNLlxb$eiEV5LkQdX6_ zUiKx)WorDPB1wmWwEW-SFR-uH?tz?;zJ~!PpJU)4{H}IELNNmy$bY=h$QRdZW!|G{wGbSLyO-UhlORZEnFcQgo`V%#!Z zDO*;YKkqJ?|MNOqZ)2d~jg*#@F@l?j?8C@);iB)N56aFrE=SDsg6EHur-HY;vNcq; z&us*rAGoI()jzXcAY9>3)6{1663u+K5NE=DSY1AvdsFH4yzSiciAsZ`oS%dN4GsnK zrRCPFDN%Ck_I0aU8U;riUrEJ9ZF|-%2n=+DTK6N>Cm{plae70U=0(a|M8^@m@C&Q4 zumBU{V`hVaeaJ)KOREMZD^g$CGxIkT*(a0q=vD`%PPKsxq2`vX4vDwtP3yxCxOovI z+zMga#|E=8Dghoa(Jy#qcZ`dxqL;&GO%YSxwG^^{o43AAtfO- z;do&Ag{dEnR5c#b<*RFc#%I5>*k&p*Q9l-QIW!%KtM5(^fx zD=CWvIf3+w~_dql3^k^VZZkhSOjU<%n=ifekj{?aOhY%hTK=zNBig5xfH6Xmphd*;DQso&tG|@?R8}r7lb#o_KT`3 zSL8OH@P24vKCkuR2>APf1V^8~9`}>Vb9{olvX5V0J3FK0`qogEkMi>9)+$bFzJV?d z*Ti6X)eussLs%ZNG6z(>lYPF*BMxlh(am{;PURDF~{!Kd15(Ora z%gi%okm6JrvADS5zDZj*8e}5d*8S^=benOs*P;0yWL3JY+Z~FOY(N_6SQzErcAAu^ z#Z7U#d)9+H<=!YrGlNF_fxs3GU16zQ&1Ztsy>m?g(2d&bFQMW=f0E|zMSt&%tGZal zrq}%g2J_rmkr@^!-G5^aPF9=rPavTFD=Zj29wJUVYN7I#$PSxJ-T)H5XG)1d7Qz6C zgVcCVVTmG#D^X9BnCM4x^r7yuN9DybyT2bnqaNbM{Z7};RNF>4Sa zC&UY6q2$Kdti^75@upV-$vW=^05GOoGXSHNN&u87$FazLH@)mcc^Af!=U6bva%0E) z!K)S~bA^O3y1Q3M6b1}HSHCL6xN{U;Ariyp_%%X=jugkuPVlWWc#jF{-Zzi zo5+`}*JMl*sarlvwRoSM6K3CUJe}ZXt1(3^$-Ey_X1_+I518Hv)QGXgSQb87bAN%N z)_bDqo8tmH0{~|Ike>I1bj#waMr?slKKaW%FP}bN-`J%2{c_KV-1Qe*-kh6hJ9blm z+a!)FqeudW_dl1nk9IeSun9(g^#%82%l|%Gi^p~z#OB4mJbi_a)YuY;fRj3xkG%As zRm7fpLJj)|e*nqi;>uyf%|TkqmBA5@a%c1LbV|y7^No8QcrqNMt>DP+#Bi$SL*Pus?e7}^o; zwgXK+wLd|zvKe!Ymin_!8Gp|f2)DEXE^{S=_{Wm6&XkkWA&d+_S@+eN3?m zPUFs`%LWQ7(H8|qtMdVJub0^pSE``muYYl|T0WcK>*yMBZfgRpeKgku_FHP?Ml&95 zs{4{>r1V+9-{SjI@msrH^^9M23|jZ9lSh&t!kbGd`<4^EE}n2A&uyq0Noo9r`3tnO z3s3j{R_eRY4=_mU9qkRYbd6{{%XXf3&pRX^Z1&=jX*hr7T> z-za1*UwS+l%RHUw|Ms%Xhbx*})QgH`Tk%h8qNwb2Ta>u00kGjT5+ zJ#Tn3*wcR_XFtXP^aFfS7yH|MdoSSZht|QxAew`Lq}eKPG3arzqA$k&2_i7g`CW*b z{-}+Mxc{zOBW`$Nq@eY{5nmj8$SiO<%MB?g|6SJtmE)#kxU0SVqc`xko&*WQ&51$p zX56KpgDW2Wv-k<441#=yWwwy>9nOR$cal3 zuPh)To6H7f|cdGj(I!FN&SL8ll&Hk7${Sb$aJ2*MBq)&W)@-k&hHG zSf!`Z0md|BY-u?|*`VEJspLm*!G@Z6zP~gT8f}NKH{FgH9h__W-8Tx%#8bBX-E{Cy z%lj1X#}e;1H&NbJ`ZB*a-hSKW?wmjNCN{hL9fzsQdHey1fmhw>K$y8!EVQl7pZPd^ zJ>3-T{@Yne43q7bJAyL`^i}6+xi!ta0t$#FiJUsb62+)kr$Qt(Y9J zScvzLLC?|pZ4K{}XU}~axIAt@P@`lfJjuP6zWxT^9NYXFR}<19xL%;0S*ViHd>%vh zp{wXT#3U@+@RU@T0%CMZUQLxcIu+ftUn#~m|ai}b6Z z^XuCegmt+Ib&12eMINDl)NrEPHL1`@O?qKJ&6&rU{WXta{UXBXUp1oWK}|*U3K1I` zT_eMNr1=6(sA=W;Sl#?l%P$d#-_IHddVc>ddZE8@SVc3w+8J%Mv8tJiR?#eWtx}v{ zkV^5$=6X(LiOf0Rp|BXLcS!9aSVg5u=Pu8uH8G*7FaDk%jZZue*#EwTeidVFVa@$6xRQ;ZAKY4E5N6KAm?z%M!Y*gx7rw zB|8lOrp=2#9S;~0*f&MnEWFJn$kk_{Mca#u$67{%SLW?uN({`Vz3nXMPA!G!l$Hdo z94PeRKU7v^_En6%4t`=lfzEh|5FHr^y?@=NX!;7w*nCR4!>VkYJE?p{$&Da@$iLOW z0Z<(O1Xq`zpDsu(GhDdVB&+Ji+_7esPFIVA9RM}PmS(OqZU@jJ1Q?9Z4}AXdKUJX> zf!J{EF`G52D9%AJ?TV0lns*N%0Vyn%j3N3ItLX(m0sQWO+k}_3sa!2Jo=E|Eu1N}F zNDw;}p5QCE771Xw@88DfL30GY{^tL3Gpt-#rk`7)v~AZrZ=>i5U|CgiGX zH4Qc`>!!`=F%vE&#^vl^44x(+%cIBmC-*r~!BGg#sy7p5FinlL5i8w1PBZ6mj172} z#|9`y^9~@v61$O>9=U?huoETbUAlb8JW#)V`gn*J>pm5R=daHTqkmIt;P}o_61dWr zemA#PEp#^!%%iZMfSJS&`q%Ry>yS9TlCniH@K?~y<%AHg6zLGz;I%(MdZe0iJjmjZ zSA?}9=7cyUDoNVvQg8|ZmIiDjGMol_^W6BShBpy7s3!p9iio_SEvHHG-0!wt`G5f; z_s#gfQFiJ;h!+~EWqZpQ2S}che}z(`3>Lc+;{@3)lVq%~nzV2v6{Ha*LkX~ zs6njnmtiB4-g*2n&>%^`B(t)NguR;*V+OGY03=u=6ac=E*Q`HV)%pN!QuFVz-bNX* zTM$hC>%G989R)cQ#Wr&F{bDsL@7|7t<@|lXa5j{wcC5N6WHLn1og{@!X3L8J zaMB*Y^7DkX=rQne8 zOH#I_APLeIr3Cjhml$^Ekyc)}q`mIf8Z6z;rghH8Vl0xz7p2wYB*C{>$+Uh%2G+ zNr1+6YXCu_Gp@RYU}_@;{LEaTe1xH6wC2B@En<*mAWS?{bBLQP9bpgp4a}Wx%<7=7 zveW_1b{kdw52+c2Bc?Qa)gl{gcQqCOg!q+`G{XP$jRM^$68JmqZ`gw*EqjbB>fd$+ zCXn}8H;?F0ND*%zRO5ti(Lb`L0RAFP%yvnxWe-{lZGAHhn3fKr&y70yTF0xUh7fhR zs-7#fkkS&FdP-b5vILAK!p~q=QVFYR&LYGw!jK}lDQSncAVvlt-6qz+3nPHiuJp@Y zdijrjW&h}>AIqhtdews{)a~E@-}bh_{IjpYr2jwJ`(PE49EqZh(&XFe;5lh(T_tgr z#vMi5^^T+ucivr2FuNmczHd`+<~>{ndq?3o>V9(zftwP50p^wE3y2sSPO;uA8+JbsnS&C97jb=CkTD*!JSG z#)q)kMt4D^A3w?}B-qP~bS# zf7stK8h)pU&|Bz5T}h19d=ig4FN$rP3ZP_qh#Jjh3S-ylPKOkl5m*SDO_wD%cLnK& z8^a`m7A{Vx`@rLY2TQ*5&ED}Cjg$A83>A{9l?4Vy7`1kfVT;17UL923P#x_3<|w$1 z&l-o4+svAw!coA&KAl%GrcJ*ZO;}f+I>!r5_xh`z@128H++$|yo|HpTJ(l`y&^&)} zmuLicepW*0`&Y7<>j^zQAt5AjyazeTKj3r*&db~ew#d+#tTZNM7JT}-yS}W7g|+X+ z>Vd)R?!M_ORwnih{*%BT+JRQzEkKhLU*geX{HwX35tbpClz5;iC09@e?b@Hvr;KW} zn=vy~U(-kUm?fHs#8B^4?piyz-NzBE)3ca)ypnS7Mys5jJi{Z!LMFF+Jq=Ke%hoD0 z8g*6ChkE&XKI2C?=O1F=dH81^HsrAO+CM#Ww36EpI|s%bY*vp4B` zw;@5q^1G18QZ0r4M)6MuP)=4Rqgl54d4J#8@giI55qTW%4CatO8DGJH8jWx6GSRUB z^w{|vcJ|rYc0_w%7pav#P{v)+FBT^qTbBc!Oq(eP&Ml1be)k(%7r%QQHY1i0DKkv- zF%t|OXV(!r%|�AZF@9*OQV3X3`y_cffv{H;Y`E=EaI^=eW=DAMuGHUZ2*(&l7~2 zTmyf4j9owFXob^Z4jpK9p{7v?b_CdiC5ZCr+Ot731nf?g87%@a>J}jr{}?etquVRZYD^Ug$I}?&Hj_^7RKZ;pV$H{P*y& zHysxqP%EVqkAlld)emCK|i?clY_MOY|BZ!q{|DJUc($R|1xb@8*<`t-~8wa{k3aM z$Rc-5I`z7arTt2BF%sQVSw1{MWAX765(Xo+%LF_cV%4MxS}h{aO1usdH=fB*3+==l z@&$|!bCtxx9%-w)=WZY`_@*8#VpZ1jo7K;?%wXFE`96p1inA&APC;CiSqY`Q#_Pex zdR&&2r;j4;!`BtVqjqz&9b-3pHT3HwZ#Uhi5Wd4ymz4&MIWU0!{{DnPp-&7PF!BQN zOXAOc5q!#RH&CCqfbthAGe`c=i{SjBFv|RnI|m6L1r7O8R6lDZNJ)0w07A{59~DXX z-)}b~mwWf}N&R3fhKtsTZNY|e7lR~mQnzE9d)dxTlw0*9#loV^>e5P<;e%S6yk^(f zBgn10E*oTh3e0pPxh6habprS)nTYhV?zVZ_T;;od(pz2W{HUAXV}4!l6=M0^E}28Q zchAyl`opb!H$_ z>dgWR6lo+=k-A?Lxf{Egdsb+rslkBVmA24j9rw={3o#R;h}Fsqz+(#rBP5o4$i{JZ zUwF4&x34tC_Q)bRe|yZpR`bIB99grjtv!+qHUC`g_BL{jdeVE6Se=j?H_23^-xZwt z8KMz5vUEZ<__(c#L{(*~m$r$R->SMdi#&hV;P&!$+%u-Wh`|}%rGtpg&VX8B+?aAM zR}woyU&b&@K z6zq3ThM+ZSXg+p!N5vPUyEJ8X1sln~s?@k?IMd$)-{yEerJ~(?{o+__3dqF(2OSHfhHl4eO(CPf-^t$ApV zlH^RuoNjcN$eqfUx-&endY~C|v1*2+H#h9n&5wU>O+p;VcVC$tPzBd?@F3j74? ziOPbWcP}|OYG>N);E$|WA!ChJO%uVg3dIw-Li8iCWpC7>Me63R$M-F}7p+>>>ln&b1+vo%i>q)!vgO#E0*&*j|SN8|7^28T-MoJfn-rP(|ye6f%R` zOEbTc&6YbZD>o)T49A!qM9kD*KNwb1Hmd^yD95M z+KZe~YSMV6L#;==^N;k8_uPlKQUsG|llOeY#sUE3L->ARog2j;|3N!)2juzEzc$9) zy@t%@#)Qvq6@pufH{g|weioSVD=7~QVs2no!`N-U`ElX}8|~(+hQe7)W1TL&@{8j^ z6CpU3O zRGWhGCQIxuI8f7ueJzd=-b=VqE-ig@l|Y^S%b0^G@8tt#Lckrn$&G&mFo5;%5C5!dYq8!}v|k`N<}@!9Ql#)qFPXR4Yz6o-y*Joqj- z^}du+(2<82EQVWQzwwh2hZNU6=X4w3E)L zr1;tda;g2-^U9X#Gs*bS$tY^F5V-hE*$=vX7huLKY|f9e$-4slHsb%YozMLCAN3|0L78dW=f+w;HvQSNHCx;(l56#$NraBNRwE(jS&9R6 zVt#q%%q|)0-mhiCpxQvaYUofdct#Jt#-! z>Dx!sG#Df_gx=}f!p7s}wGQ^hp9+piBH|h$agZQvzcE{+;uu}Ea)Eleh;Kgo|5J@c&udC62=ysOT`>rb{&f&OfR3=(iO^K=AsRJF#OS(%mT z3G+IicEcaBj#x^PN|O;51ku+FyLd&;m|*A~Fl#F$P2@boNe$5h5fVF8YIWbjZ6Nqg<>_1TU5c{l6ftWPKd*x^qil9m z9`IMmr-eSvOBsE_(@;q6$~s;tF);k-s%GVwU(HElx$hErATdkN6(*vU4ngbZ^9 zBg?}5O6S0N{~KviaJoBn|4K^}A4Tc1q@3{Z6OchRNbc-~>MsWhCi0V;rHz^F7~g}X5=p@ z?RQ6rpO9fettre{)$ECM5eM=ceZiv^P>;lQJIx7TK%o;RQfEk1dn6b1#r?ruN})h% zN=WF-cZ*d70!X6<@benx?M2O)plbvawa7Wy?KmHZ&y}Equix~5dH$ftlGv7*wB*Bi1 zpL+!BqH`Cy=%cVeZw6c4u9OquxXl);!s0_*Nf6>rluw}hg(`U}ruobF@Dxw87+vTI zp6f^CM`d`GqPM7)mWotZ=}nQTdmS%NI@~%nZ~W7QH{RQ^Fi9Xnk(z4k@Sad*HU&xK z8%`%s$0I>@DBQtFA9bZxow~*ar&Jn&(5Gbj&;;v_J|>K1ng#A!%SMDXB{P~%nD{^8 zR{-kg|Hq+}|G=dIQVci-lm4V#bmC?FfYf$#wMAc&0Km+2Ra$k8BpR8e)&Y}MT4C`7 zzlBg*q~Gb`uMSf3g~JQ(FA$ z+HkJxo2XP~pcB}A2lxHhjf_|7Pm=@rg#T0s@ygL%*)rH6-fCr-1CwStipu2HyK%SBF|{j@-w5K;QU{wKusOT~iRvJ!kq{NP(O z#uERjy~~?Q$A{vUik&a!IPq9H-NmOgo#w0O25qf3M;+}Zr3z|PcK(A$98Y|Vo>*U8 zb7r78aV3jTuhndt=$nDENU=noW$XTTR|?(KHd1(yDNLr%&K(qTcU6#3#|M*({>50z zu(vhaG#E3~?fDPOl;N#ee7Q8}<4+79EqG@+Wm!p7Ih`u`)aZc9y>Eu4?%t(Zy9MTqT|dr(8uO=_Pz?g#kj}&QbJ#`im|VvA+qUn6=!9JH>`;z ztSB<88EoACfT6V7-I*S|S|JBfBRuRn^e|kq4##wpE}7qDRgGhPDv3uHR4@HW~mz1M+qj>tN09O96wcU4URN6I28o1|4nkuqv{YL}0Y zX%VpE_*%bffehk(m5g_14eH4-PvV$T>!!kFH741dJ86-OH`Cs6)fkc#ZtEx5nm2sN zcBR#upo>@2sb^ya`4{|&iS0fc)kNuf7FTujQNS|{6BIiZ9zW%bk&1(NgR`(I8UC6o z8BXJDW>^h{4CjL81hMHSlH(mPqK3V}2`V4rUo37CY%vEP-;Ze5(8BjM9ADJZ&2+js zw8@90Z0^#Re*vv}fD;+(Sy7K!%ylYw(nJ4~Lt_7NfBb(la@^U;JcGaeC)o*0AD}PE ZaRSno!@UMyf#Z1B)ShT7l_|Um`aeGWF*pDK literal 12767 zcmb_@cT|(zx2Aq5f(A)IDH2LR1*I1uAP@*uDS{1(e;@mk=*9XdqPd!aGQbVh6=@~CAnGPlzY(KXSkElo(AiWU>|8up)#)b4K^*p@!2vTfQgnW^5B-#xJA=H?a?>6bsY+Z{KLTyd32)y`iV z^UZ~z+?AD;b4-7j2I(s+TX0=NK_JoRbulW+%GAm*CJ2NT{Lcl2|CbAi{Ey^zpSrbQ z*L(VU|98VYk8g#`Qr~JYn|*(ESE7&N@O9B*4vyZ8)Uy| z%qX`adrAFPSKcYJuu1)TwcFb0>hR|$y^Tet*N7J{%!Mlw8eZ=&(?KkgI|{UwXhb{S zOK7yLDq+6!Os0@mLLP%%5;XM=GPHW$!BuZ`=RQFm*br__Ccuq1KDJrp*{DUpejWaG z^A`)rV`H+)P6j;N?QWus>X~Q#Y6k^o;0fd6wV&<_LWJ-1J!%vx5F2aU$2gab7p#C}hhG;?yn5w$R?hf5H$Ezwuxi9#Iyi;Y%CPydc>U zYk7`LgqIWx9W02 ziZW|<;}CMX)(7mh=zKBSLr)%3ttmdnHo1j1ucAOas;(S+fvAla`dzCOe}A(%&Mt;l z(ygemW@#SGB6Bc?r!xcuOs&3)`m?B4~&;`rVC%=bGx(#hizv zO3Z~ji>VgqvBr73(>g&7L!H$re7GwYc=c@N+g@2V(deZG7K*-`@c-Bn?ijrGKv-{zM@6u_l--4gc(W57ElCS#9>9RCy5q}N-VIY=BqWKLC8@DqGjl2A4>sG<8 zBTYsBCJ-8}RPz{;bL)#L;{_fw3%QR z>?9K`3{J_&gp3pjr3t0{WqbwTv?_0BHR6J za->aeNNSJU*?l9%Io4}lc{sS0RCbG-@mE7_etTWU2e4eBsrs%)B9 zHVooeyPVr{WFGGsZ2j)?C|>zHemTThKov2+32~3x(k5_1g$cFAXqxI2@F!I!IWA2m zRI5(>14`-R87#$04{4A$-oJ|}dmd*a*bUoOgPDMdf!(US0gCVE%*b7zqC^Y|-fZR$ z*mj3*ke!1u@!_PK2#{+GZ&!RM$dvA>J#~f^h~K1hrz2qd3M}D+ z%@!h(EElvJMcy3ZsI|uj`P^9>)P;?bMZlpv+M>(y z(Tx;~bc~S>^cr>U6yd{hb3b|W3rlNoCyKiw3ml5kR2=K6GeR;YUIzlR#ly+CfbLrs z(FXrbA^U$te+yV3T>g0)f|&Z9lS>$}(|hR~iPvu|q!WM!!6;ZDixZvRyK(#J`dWxV zBV8Bo_k*Vf)6}>+jep&+gA9w}&LA1SYbL%MSIelj|2at`7IO^15c8MA$go%3jXmzx z5L9awty515Fl0w-K@dE2y%GtXeF#&1WANoUfI%!Y!(Lz@!!@*p8&(2*LGZmXl+DpX zys7;S_57g~*SZH+8}#qsIE%s7byhg9@2q%9KN|Z+gU}j-uV+IxBQJ|0UynS~yVvxl zxg$(ER1iN$x~1$DH}MsNQRfs01JsErQu{AK8;c!e%1p_~9`dRQv4!_4kEWjq9HF*u z4F;qPJCkzI_?!DZ0_GLrTh=#nx0Ef6h;~?HfJ8%_XN}xuJ3~imml~g>`mWoKtQ>nh z<@NaaBK3eAw>r){{erz=!~ak3eousaxpK*0$5_^7>v0S7ryvOGr8|^z=ngTPw8B*g zX&tjFr)pRGO=XYl@2!`l)%>|K(kMKmD_6B0Fd|Nvd2;fSK=`ucaJ;eFo;;(^elV0? zVp4aiW~ky+Jf$&4(XXO3Af7ZFoSLGyZc(h|t8%@M`skk?H@d#w#Yz1xYTR#@8<~rg z9ex}_s&fM-yKpbiW0JKjUC~HZC=Cv*Z2zatv$x6h z@XPV9T}r0RPJU_OiHRwD?XK3yK;((G8~GEkKR&;Ij(iuMHcG8q;?W|D)B9)29G|>S zE?n3amV0Il-*a{&O(km7zAqAHnOQRRERs2HcwIR3T7E3$67O$shl;1KciNgJdidxY zju8#2;IYAb;$uhkH=7nyinbBERrGrKoClgNCN6CwL2bgG9PT22!yXD4(}3~{hhSc9wpNPZQnWbUfM2-s+zsm6Sw7#^eJ7uZd?4EMcP02rNTvR;~MsU&KPkS!FPPz zNW*0su?&ffQ=NeYzx}gVUt4n=hAg(0*Cu}ENXH|pYHJT@-c2d3(b3Mh;!#?{e{54kHET7Kq<@Mh=;q#!{ks%`cjs7J4%sm)ue!F(-df zGW5&Zoi}k^Dl*D@+o+n86r0A|G!YguG(#pVq9T(MSj5#3M~1x@&0G{;L>6L6*wK5H7`hSvP>N(a#F(XCZB9+U zU<>c1lrvRRUYM>>yxrCr9ZIyNUu7Kf1dX8 zNbR;?H;<4f3SSdyPP!kk=4U6QuohYw@MqnlNbA~IU48GtXW(}|@Q6>d z@u3Log-jpMB*jzLZ|(U{zrZ)Lqo#X$Y^kPt`fPO_td^_3N>%Jka*022xX>us6W%#? zGDYJnXX}7#lpGHKHCEvPau*LMJ_55{*;h@Lij8yqqg8n z3xBA7y5gDz(P%>Do>)lsf?LZ*^*>>#hQWKo-)vRLL%VYmL)8VwSAgC#^*qQN( zv6>LbXk&bQhZ~3P-^3n%x2^R&QZ}~(SBt;?Vrxhy!_L_1FDVtwK@# z&-V}d{KqhtGzRdL-;=BJKMmOe-gl-Sx3pF@O0!9op_;S+t(ZT9#DXDk>nO0ajdr6Gar|$ z25R&-bvtcltlE5SDes;Dk<=*K1wq>SgKmhcS?TrMgj}@3^wJ@BdkP_kFGOlE=C>y? zzH;I!nhDa}N4+2IJoas8AE1CP>kS{^l!m68`??ZmVpZ z4pxR!p6$-7S&bpjI`Aw))Cb>kYgaY`ny{|>NBNS%KMQt72uWm5a(?Xun!>0d#qY)Y zH7sU25N`4n(toUdY=FEVOSFLX|Kc>;qVyd~6II9{Zd1ltmBsLu+QaxHW+M>|`NLfL z;b5!R+nhs(%$%h{X|B%{l`V8vv@m*qA?v2B!L*E=E;MCJxg1_>_0pOEaOniFP{1#2 z1OJV;z2Xbw!fhVy%m5YhaXKi$#|qM>pJer(RaDT15k;(|1tQZmNOajiGC2O7%o^zW zdGztJ(jA5KwQ3ByUm@Yxfv#gGSBQnCsQZ8~E099kS@bO%>)8F6qH zQ}c(k=2R)H$~bz6GDr!N4^dT?3aqwO?ePEkN_y)F3n|zS>3)txrG1dB>nn21TujE z73|i z&k81P4$7l5(Su9^YcvF`-`1c@Yt|ort7k2mI@2swLoX&Vykp(@~H-pPPC^BsMXy6iUU)ndF4TNi*_fDeu zNW+X*Zryexb7ADniJjZf9`6?q-<^Tg)nv{ur%T`dl=|(V_n#sBM&w4;rpe)}O_Ubj z5n=mj@jtdDH8mfDyyft=ZK{L(b3Bq-eE54}WsmYJ-PQ z>EzL8-Y@h%R;Ig|_tQ({-S@>u*iBCrtLAL`_0pU6+gE=(uot**czr(VXe)g>}CQ?F?VxO>){jsK6(Q?rk<^_j(@h?O&BLci88-l103gQ_B1v zKG|jVsL8KLTDau+hexu@{669!bHQSbh5o4(IdXs-Lizqc*_@}6!A9E!7uPmT!AGy zMcOcQjUTf%u&BI{Lmp>Nv(d)haw@hik(x3+nR@qOElU~%rlzF|YH59A_Cv&xMHa+G z4WWA9f_sC~5&gCOx?@pT&6z?I?YU97Cs6;&E{NrbQF7< zo4(?3lzM6ntqhM0U%!Sj;L`?DidO#oFKTHrFMLlj^}h7~JVx@A^!DrymTxB8duB8q zP5qSlHP$66ZgrSQ){kiPI%XWf3dM@(#{NceNl`d^#h$6x7~*;947qpJdbRM+Si=I_ z$RnelCLz1TYX`Tk^|ZV8VJxqu$hc&BH%fBAN_vbCejb7)@ux19w3vmobk67+p&R5< zU-Dpk^JzeS@~tM`bi_2dzgLWtnTkBqJIJA$9C|)(TV%=z>kIry)pq8ov5n zmSi172g8#5uH{O+sOpIULx<204uR|9S8X`sq6{A9HR!*vDjM`NTyyFk6yNy%SVCj4 zQdnbH;Mvf$fU)?ZjlUgOgRpZ+<7W!<4n4oEK+uq>K)B2AAO1ykWv<%(EaC*q>O=_hX7HodJtGW;YUoZ_arN_Ip_FJId$O?HgPwob3?OD;;wZd$J1O}9 znuCS8^Z0*Ubyy-0=+0ZHc7@%u`jY$V0|sNK3t@&cDJ6eozOLYQ?!r~yz>X#fH<5O_LmSBx>=@dg0#(8&PoiN>IS);1i{Jt(Ag z7AtQ8`o8X=RXT4a*y<*CNs~fSXjcYN9cxAxVf6!4n&VYLDPs zngny5z;?uRQF|}SDLVleSs!pCnJKj$47(DV3>!i)p*c++G7!Xk0i;$MybuV#;!|H- zsD=60>cWGViJE5_yk$yBVg>N{KS0402+Uu@MZ`kGDNf%Ko@3tnOa^+!N&JNfC^(~B zY^IhG0W9Jp`tnyZ=R1leqlFTmD?jM|C-`y$YsHs4(g+{Ia5T4%58{8tUxbDva95I8 zah`5gHL~9pYyLqiqT&hQKhq7z<>&CCKzs`a;v1;v>2^Oc@vu zm6>)V#;7v?yFm{4Q$rrM zqy#A})=5vT~rZT$NQdmn0u^Q3{@zvBb|Mt!05 z17=wtH*JLzpu>xup(KF;LRgIvszdAe;3Kf9-nrGR{}DVb-o+Vf49*%Qohdm&{GWOs zYXf-6I`-Q^G$Y- zx1Ee=gdG7U1Au!8faJi`Mb`gi4lkYYw|9f3UIH2VFW@XH_BTxRAD^X{ASR2FTag<% zLwx{uqAKg8SW9ng23{U%w0!{_^*(0{?jD(u~N6u1U zDh&QdNiU&FKMCO1&MK|R9q8qLF83fs$L-!1Js&mB*2J3h&zeZD9@E@GvZX5qVnv#r z{^i`PcFvG++9W9{v!LWADAOr~lMyw_n(Q+rSQlg($$%s8U!`$j!fIs%-TtIa1fS%=OMmN(wC zta~GluxNtNn3Pq>u#{C8JYue&wAqc%DDHJMXC}T(n}c15bnOu1Ckb+F*gTXMdS+!(QjoYbZSDX|KK2y!Xo?k_|hx#9={8Z;5x2ntxEjbB9UJCLI z0C$LoQc;Dqv^7&vBU$1>c=M>2u8s;dl5-Eo)y47bRM@u16QUP%B{SA59{?=BZWGto zC-F)8D)^9y~HzWy^0&MWIJP~zq;;rBmsE4Kw zz0loq^~cXwnRYYD>+#*c4K%mI;Hxi8*5@T3W29I>KGduMIT9=rHL5i=S)q!^dN|C* z2m79Fmi1+`gLE^-G>;$!h>{4-EZ}v^AS-xft=p+d+gxc~l>8t9X)NyTlDvxfG{pXa zZ(F~kEr-ImbP)v>H|^49N7`liD`C0}*wI6`a5+H*OLYT6Xrv~qbDa{zauKo3XB)#i zR#j~O;}*`2QL1&uxmz$5nPNSv6b9(0PsCF{ZK}!Nzp&-bNyWgj7<>YujeI2zH*ay* zzstOkyFUE3P9a8a^4*M4|9X1ck_dQL=mbw=Nm!mcuh==4c?~tmt&H&wPOzdt(+&(A zl^mU?%8(QKSbtJGvtdxpPCsh$7EY9d_%2v-m4TS4i&;8H*~ew%w}}dTC)njHf-wtgx8)tq>YgXecGV0s)SM*9tlx zP55iVIZWg@z${%s{)9n^6ZUuA?&PMFleZED=1jyCd4l;hBEg~Q#5s|VcOkq^Hy!+A zlm2aZouCv?1nCXJj9Uw^kLF(=Zr$^n!!^JBRgMtRM;*>j+LkuRs&li)ecb}V^7}m! z=jJ&=TWFo7rT)IJ6IZ}UP^g741v2?#ZJ05e!=aKGEUo_}rcUJr8qI`;I^mJOOPBY< zB{{%v3m6xWEPNi)94L3_>}O_lly%@iMGD*8+saoJ`lwsvoZOYnOxdzw%@d@>9a|21 z-KBaTs9OMVn4+xvq)%ArFO#UPi#p02>_Tp}DYE@wVq?d*A;LqzGvS&vPMANcim>0b zbbSoi0Az}jEFX?FNdK(;V}<>H+x}s|Rg)NmH}urBWEyRwL&<7%@!YMD_hyLP-+3_Q zTA7smgHNjTP1sVM+WRPnaFQUB{5Hq>#ReKxE3n0xHHbJL(T+HxK)jLi@tR68_5ZyWt4h=orv z)g(L^cpj~0s^bynH~iK)=$s=+QJ~vkl#&_3@%LmTJ_#H`(_iLEc<%H5PW~hP)#Ram zGba*v)Rx>R1ko|}Hy8UVr|?LpIDzbZwd;nkWTtV}>EQx&$IW20#64&I;1l&}y?K&k zj~OZ1icb^SLsk@Ph0Kbd({IhoeL4=q|0jLEC4@D^0g{O(A&-FVXbNL z<{;3>?JzvcCtdn328V`s1@xz3OGfF?rXqUkZ%`h^5{G#L?|kSi?~Lc%GduI;Z0(p& zdV1VNc1wE3s_a|a?a|sa8kN_Z!h*mX<9(-<*PVdc%z&!x;o|3V=)AGL+I`)n@>3|k z^;c>8N^^{7C6AeUi081h3EgCzi9$+M2}OVp=MsE%;RC-+T#G_T$obj$mM#ZS@S6<` z-rcAdlIGgA3F@L@6e7}}w+syh{bELRT{U}^rIl4ws|D|HALd<6Zi_c_ls4rO4)f8@ z7IyYs=7AEN+BqUb4%xdEC;D|g24CKv6#u24!^-@3ZSiqVaiq&(>%eWaW!TD?H(IEPnK_t@hm}4KY8Kc#W9$ z^5sjGt5CS(1RqsX87CFr%lq^Km`|S+hp+o->Ih%R@BOiOVzf7Pdjh<4>YPWlcUjba6hYzRGd8OhxjX0;w*mM&oCf!?h)e0v8Ud?o{6GV4rV+3C@db`E{p^ea9H3 zzyMN@| z??38)hm;iofH-$FWH2LMCd4g?Ns7NOxI>SqE7M zY`b*Isd^j;?mLDvG30{GatJy4i=|Gc*iFTM+&&~4Li-T+4;l!NMy-puTS@k@$ zELR}xgRMtyYWic>v|KO|B45Gybks)w?wBQJ#`1zg@L0)#oP(`NNg^?=c4!9Z93s~{rbC7E(8If#-~u78}64?%UD&zj!1W4VS(yyE11!HlJX z)ZHPm`6sWzfWLKaYfj^^qtwwC zK)+|*q5n@!pg^AVpZiHIT4omx52-Gf{Zu(r49cA40Uc(8xY_7}^ey_hJbi+N{eN0Y zm!$-a(1@BHD&I_hXQ;uaHtZ%H{S0H74H$e6>jnB*rE_M*?GAS0sIAC9m)auO@82GM z28#=iKnjRi1KC&%k?O!}v2$OK2h@aNo!xBOpMIS@L!(ijp@I2+sU;FVn;ryrSlZN$ zgGaQb&pkq%>?}CeLKW}eST9I*F*5TUpb6Lw=OpRh&Tjf8MmeAKc;J0%ODY|KXA^*_ zE>Mxq;VvjDScn~jya6+bf@Z(RtN_g{tiko*n|?|-nordPt2|3-b%b(KCGlNWpd*c` zcBEsV@S0Y_D|#qlXxQx5!rz72oj-u%>O!sfy`mfpqEUQrF3N^OTo9vix}x{NVfw0K z_bks$iu?3!j@gH#o`qj5UKgZ+7CfDFutDoWjpCh%iqZ48V?$?vE?Q&l>|zkN5^(N& zSaE+1D9DPzo*IbM1;RP`=PU=lbE~rWX*f?>%D*WnR^F`R(!`)6*l=YpM`aZBYB9;y zcwun6}&lVmA}G|#uKqDS_R6Jjru{x;K$ z61Fsd-5n=b$lwOi!C3y(x$MO9H`ZLz9cajwK)n$OH0?1WEqNYn%rrGYP&g4hnkQe) zFObKc&kCHjPIeWE_=xr4po0DKmPy-9W3Ap4|9CLBnf)Oq6?BxmBZ6EK^uF2nb(teeWEV% zVh#>SY{ca5r(!GPSqcpVBNd|yQ&^ah671H zwj1x9IQdFL!{QbnuH3$PIXKE{F03Uikx?j+ZjOUGZS%l7+wjJuNYACK&>C4Ko}WBw z5fV*iJuc(pyuK0Jq1_dY>by(rMa*YPix|Fl^u8_@9Yy%y+xiulDhN&PSSK6cXz#Z_;4r!2;!w!U!Tz?xFvy)enS+8fO|?;bqf@79g}2>2iCF8A*n zhDf8hmT3ODQR zsF#==qTwSw5D#e=4plWYF^+N4YvUg~-L%CyPnUcVa>u%qun`Vi>aUr$Dw8wpn6ad_ zcc(8W5_hs65soSZOBo`b$gAj&UJlY9O@UeCybMG^cd6qO@A^%>4^!+r6+obNT;OD) zl^~hN2PiF*xQl6$yH(&33-Fr3gLu5tr$DP4!6}d>@4z>M+Lg?HI~w+cSK`D)#{s_~ z8c|XgTWL8segSpoKE7o^HbXYzNsMLHe*z}|N5t~~f8k@$LHGLdzy9SgB)1n!p+Db2 X-IR1R)ebq<4^Jq=Xu(^eRYE0RgEJF!UlIO+pvxT?B&k4$@UZM_T9z zNT?wc=@8m|`F-DA=l*rpIp4ao*1M9o%*@_<_RN0fnVE21ZI!!s9^AQh?b=;6Ri&rb zu3g6f7aP%S;0fpTVOrqly6aOFg==L43|QcXz*b&M{@S(5Sdue~TfjZ>TU8U+YuCuy zudeG|&R<_&y9RMmQ<69EGTY1yZ({85URgC)x9RB@7Ea|@L}_EhBdb|stHrHUJ56rE zn|pqWz2b15{KHuBqwdk|7gcwOnk}Z@gNg}K7&%yJ)L2Y6n+(uxtw+zG_r`1+jxx4f zxBLPI^#`%~rltqao*f-|JafMMao;Km%vf#J6|+C8uOlJ;PpYB!e@+4phyr)sM+>f% zhfV??VTnj8ukKlCEP)(IMSfK0eJjl{V?;E#Gli@YM1NI|C94?lR1&4&2;_A`{3|2q zzt5OnSp}P^bGtZMEf9E7*WZy+XVT{H_q^8OnWV?Ym5l5Rd$%4&)e|Gv%b@_}$(~lLa{jp@`U4rSVyZ#cL=r0v$ z_Zhj%R)QNZEuIN-Ni>58f38Zy#~j=F_8_J2!jxx3b38a<`VBz`=ey}#YsryQEISpXL;rXs37MOP$uqblcUK;|76pSszj1uqyN9uWCCE{zZw>}v7A1GN z{GgWj>Mc7Wd`w^y#-Z)-`PeY^8{elvcQKG6VuSe-o@8SbexDTdHz zBp?P4giE{ra8N|>D#El+_7w#YV68mf7M}3bA<5mfWF0arA&j;bJR?%8p-DxT$7mAv zzKaUaWJGra>`U%7K_w6t&mHG!alvas5Ax8P8(-tJ6t zLc^_Z%OHurOAJfD5s_GpnLau_8lJYbGs5V|<=&*HlD!W^0(OTVACh3gtr#m9XS&3D z!|UEV8u$^0R^|GAew=HnU=2ykh3!dOwd5E$g@>B4J2}u^bouFuOflO#1W$SfAFA|_ zO@BDE4t8XVVTrSxIrHJ0={#4kw`8!daJ8{PUJOb)B5tU^>`#}3Z_-MoAyW%)eo0%s zrK5fzXLqXblN&CKSoRk=uIGO5*@{-aQ_JPdQy66GIUAL0Ky&!4E zwzEN44?3~CuN$^lpRA>2y#(W-SCb|6hO*G_;cuKIE2(yqjp-~cC-iRKI=LHf1~l#5 zR?r!5fsx5%);7djBKVLH=PH%Zh9Xi0oOqdC-TQ^g*Y@?X;=w~}`*i6tUUNq|CEsW- zYwlB`gI@I0K`z$uM!iXk=`IE7CvV>_Yny5Fb@H0}g9K5;oyTTmMp%PGmJ?BVraV@G zrGt@xDfxFL<`?EmJHeArgAV&=m!g%UE9 zNEzPgQu2SKin}JiwSOBP6fFKSaW=u6Ne6FC*v;r)Xf~Ww`(`3bUv^~qKme?osWFu0 zO~3Du-V9YZPfISxcR~jg9V7M7qk?Az8(hAHI!Y%ZwQ6Ps_k9kSmsdAU)~QAgs4UM5 zq{I2)aHW#VVQ6nGx_R<(5k0_3!W8eG;OiLSR-qkUGEB*XF>@ICL!1 zMI{<9IL%+tFbWwK?C5~fg_tcX8#BY+Kb`qlmJPCE{(0{-${HW$26ssNCcRIk;@EtQ zi5zSQ*L+11#C&wKW7uQ*gY*}WO1efYM;tSj5EwebLcQ*MM0Yc464(YS{`nJsGm22` z?e`V!${sAQZ0IN9XA)pjGp>_8B*&r9hEe0%Upe$i$jdIo-buUIRV zIG1}MH&Tp9EAQ^r_1wxG^#CWV^Hj7YN0a0vn|h~uzgXIwE-qC+=SVy3VHjWCeC|Mh z)v_$iAsS*;+9#~2$rlf-cbTQW{3+}=o;Sl3G$&?#b_w0km)U(xU19Q5x`Yu)e-&hf zuMs(^b-M(L*ckuoR*N0??#@(-#Z}Dhf1iL1-s633^`YZ`6LbH!!CEsEgtJ=SVV{dT zzP@_~M30JeuR3QusNo6`+|y@8A&M2qESp@Gd!N5P5#!0`Sk$+M(}8eYtPlVXG+JUp z@3TIUJ4g@yP8o8+K6Dk^1iC+7UcKZB45zVFqS~>pk%|JvmvNvTNL)!5|5J$sPCX?b zY77h>%I{N)>aGw*DhJnd1ckwD$g@T|5*OC-OFd1kw!`T+?l=F1T=ws*b-c4U4Znv` zMI*$uC8&(K$bLja-c-V<9f3k^C{xr~mF<#&NPc(!UA12zMd4@1%>CN|i0wIQHwc7gv6;b^xq{8mc(V}}`2{u&~ciTIaVPoPX`U+he3Q;yw z{g>Jb{IW0F-Tq8_B7-3%qH2|PJS$zKCl;(u4jKU0^cM#SrBUA0IKRjO6{^gvME$HC_ zqnpykAWItDb>%qD&0)f|THi6fHIZx{-mx)tNjslqW7v}8!R0G-T3fRnQ9lYNEW12i zFF8NQ{w*zD$9~JDcUJAH|DC2S?Pny}usFQFaUSZng0OYajC2D>oc|3djtQU7rtdK9 zQgr{#(6WkiKjE&INzvXcj0vVvSs{J-T(buBrl`T~+$dX~13)-%RKHKa?P1csR4YPs zVWv(2mCjiW&98QxY%1B@TfOPalltgoNkdYax5DNF8RC+BWl~IPUB!M!ljrl=D*pIo zx8tNfzCb_E>40CCr3J1>W`}JNr*jHF`;^LuktijSou`XKo>1fY<*vY=*V_ZzuLA}4QWz0ZL&RCg#23)QLeB`Zbof_p zV%*o4xkq-o0-k2aF}H47VgIBB6f*Q=$9AaIQta^sy+B}~YqIq!Wzh+c)7Eu0x^HNLlxb$eiEV5LkQdX6_ zUiKx)WorDPB1wmWwEW-SFR-uH?tz?;zJ~!PpJU)4{H}IELNNmy$bY=h$QRdZW!|G{wGbSLyO-UhlORZEnFcQgo`V%#!Z zDO*;YKkqJ?|MNOqZ)2d~jg*#@F@l?j?8C@);iB)N56aFrE=SDsg6EHur-HY;vNcq; z&us*rAGoI()jzXcAY9>3)6{1663u+K5NE=DSY1AvdsFH4yzSiciAsZ`oS%dN4GsnK zrRCPFDN%Ck_I0aU8U;riUrEJ9ZF|-%2n=+DTK6N>Cm{plae70U=0(a|M8^@m@C&Q4 zumBU{V`hVaeaJ)KOREMZD^g$CGxIkT*(a0q=vD`%PPKsxq2`vX4vDwtP3yxCxOovI z+zMga#|E=8Dghoa(Jy#qcZ`dxqL;&GO%YSxwG^^{o43AAtfO- z;do&Ag{dEnR5c#b<*RFc#%I5>*k&p*Q9l-QIW!%KtM5(^fx zD=CWvIf3+w~_dql3^k^VZZkhSOjU<%n=ifekj{?aOhY%hTK=zNBig5xfH6Xmphd*;DQso&tG|@?R8}r7lb#o_KT`3 zSL8OH@P24vKCkuR2>APf1V^8~9`}>Vb9{olvX5V0J3FK0`qogEkMi>9)+$bFzJV?d z*Ti6X)eussLs%ZNG6z(>lYPF*BMxlh(am{;PURDF~{!Kd15(Ora z%gi%okm6JrvADS5zDZj*8e}5d*8S^=benOs*P;0yWL3JY+Z~FOY(N_6SQzErcAAu^ z#Z7U#d)9+H<=!YrGlNF_fxs3GU16zQ&1Ztsy>m?g(2d&bFQMW=f0E|zMSt&%tGZal zrq}%g2J_rmkr@^!-G5^aPF9=rPavTFD=Zj29wJUVYN7I#$PSxJ-T)H5XG)1d7Qz6C zgVcCVVTmG#D^X9BnCM4x^r7yuN9DybyT2bnqaNbM{Z7};RNF>4Sa zC&UY6q2$Kdti^75@upV-$vW=^05GOoGXSHNN&u87$FazLH@)mcc^Af!=U6bva%0E) z!K)S~bA^O3y1Q3M6b1}HSHCL6xN{U;Ariyp_%%X=jugkuPVlWWc#jF{-Zzi zo5+`}*JMl*sarlvwRoSM6K3CUJe}ZXt1(3^$-Ey_X1_+I518Hv)QGXgSQb87bAN%N z)_bDqo8tmH0{~|Ike>I1bj#waMr?slKKaW%FP}bN-`J%2{c_KV-1Qe*-kh6hJ9blm z+a!)FqeudW_dl1nk9IeSun9(g^#%82%l|%Gi^p~z#OB4mJbi_a)YuY;fRj3xkG%As zRm7fpLJj)|e*nqi;>uyf%|TkqmBA5@a%c1LbV|y7^No8QcrqNMt>DP+#Bi$SL*Pus?e7}^o; zwgXK+wLd|zvKe!Ymin_!8Gp|f2)DEXE^{S=_{Wm6&XkkWA&d+_S@+eN3?m zPUFs`%LWQ7(H8|qtMdVJub0^pSE``muYYl|T0WcK>*yMBZfgRpeKgku_FHP?Ml&95 zs{4{>r1V+9-{SjI@msrH^^9M23|jZ9lSh&t!kbGd`<4^EE}n2A&uyq0Noo9r`3tnO z3s3j{R_eRY4=_mU9qkRYbd6{{%XXf3&pRX^Z1&=jX*hr7T> z-za1*UwS+l%RHUw|Ms%Xhbx*})QgH`Tk%h8qNwb2Ta>u00kGjT5+ zJ#Tn3*wcR_XFtXP^aFfS7yH|MdoSSZht|QxAew`Lq}eKPG3arzqA$k&2_i7g`CW*b z{-}+Mxc{zOBW`$Nq@eY{5nmj8$SiO<%MB?g|6SJtmE)#kxU0SVqc`xko&*WQ&51$p zX56KpgDW2Wv-k<441#=yWwwy>9nOR$cal3 zuPh)To6H7f|cdGj(I!FN&SL8ll&Hk7${Sb$aJ2*MBq)&W)@-k&hHG zSf!`Z0md|BY-u?|*`VEJspLm*!G@Z6zP~gT8f}NKH{FgH9h__W-8Tx%#8bBX-E{Cy z%lj1X#}e;1H&NbJ`ZB*a-hSKW?wmjNCN{hL9fzsQdHey1fmhw>K$y8!EVQl7pZPd^ zJ>3-T{@Yne43q7bJAyL`^i}6+xi!ta0t$#FiJUsb62+)kr$Qt(Y9J zScvzLLC?|pZ4K{}XU}~axIAt@P@`lfJjuP6zWxT^9NYXFR}<19xL%;0S*ViHd>%vh zp{wXT#3U@+@RU@T0%CMZUQLxcIu+ftUn#~m|ai}b6Z z^XuCegmt+Ib&12eMINDl)NrEPHL1`@O?qKJ&6&rU{WXta{UXBXUp1oWK}|*U3K1I` zT_eMNr1=6(sA=W;Sl#?l%P$d#-_IHddVc>ddZE8@SVc3w+8J%Mv8tJiR?#eWtx}v{ zkV^5$=6X(LiOf0Rp|BXLcS!9aSVg5u=Pu8uH8G*7FaDk%jZZue*#EwTeidVFVa@$6xRQ;ZAKY4E5N6KAm?z%M!Y*gx7rw zB|8lOrp=2#9S;~0*f&MnEWFJn$kk_{Mca#u$67{%SLW?uN({`Vz3nXMPA!G!l$Hdo z94PeRKU7v^_En6%4t`=lfzEh|5FHr^y?@=NX!;7w*nCR4!>VkYJE?p{$&Da@$iLOW z0Z<(O1Xq`zpDsu(GhDdVB&+Ji+_7esPFIVA9RM}PmS(OqZU@jJ1Q?9Z4}AXdKUJX> zf!J{EF`G52D9%AJ?TV0lns*N%0Vyn%j3N3ItLX(m0sQWO+k}_3sa!2Jo=E|Eu1N}F zNDw;}p5QCE771Xw@88DfL30GY{^tL3Gpt-#rk`7)v~AZrZ=>i5U|CgiGX zH4Qc`>!!`=F%vE&#^vl^44x(+%cIBmC-*r~!BGg#sy7p5FinlL5i8w1PBZ6mj172} z#|9`y^9~@v61$O>9=U?huoETbUAlb8JW#)V`gn*J>pm5R=daHTqkmIt;P}o_61dWr zemA#PEp#^!%%iZMfSJS&`q%Ry>yS9TlCniH@K?~y<%AHg6zLGz;I%(MdZe0iJjmjZ zSA?}9=7cyUDoNVvQg8|ZmIiDjGMol_^W6BShBpy7s3!p9iio_SEvHHG-0!wt`G5f; z_s#gfQFiJ;h!+~EWqZpQ2S}che}z(`3>Lc+;{@3)lVq%~nzV2v6{Ha*LkX~ zs6njnmtiB4-g*2n&>%^`B(t)NguR;*V+OGY03=u=6ac=E*Q`HV)%pN!QuFVz-bNX* zTM$hC>%G989R)cQ#Wr&F{bDsL@7|7t<@|lXa5j{wcC5N6WHLn1og{@!X3L8J zaMB*Y^7DkX=rQne8 zOH#I_APLeIr3Cjhml$^Ekyc)}q`mIf8Z6z;rghH8Vl0xz7p2wYB*C{>$+Uh%2G+ zNr1+6YXCu_Gp@RYU}_@;{LEaTe1xH6wC2B@En<*mAWS?{bBLQP9bpgp4a}Wx%<7=7 zveW_1b{kdw52+c2Bc?Qa)gl{gcQqCOg!q+`G{XP$jRM^$68JmqZ`gw*EqjbB>fd$+ zCXn}8H;?F0ND*%zRO5ti(Lb`L0RAFP%yvnxWe-{lZGAHhn3fKr&y70yTF0xUh7fhR zs-7#fkkS&FdP-b5vILAK!p~q=QVFYR&LYGw!jK}lDQSncAVvlt-6qz+3nPHiuJp@Y zdijrjW&h}>AIqhtdews{)a~E@-}bh_{IjpYr2jwJ`(PE49EqZh(&XFe;5lh(T_tgr z#vMi5^^T+ucivr2FuNmczHd`+<~>{ndq?3o>V9(zftwP50p^wE3y2sSPO;uA8+JbsnS&C97jb=CkTD*!JSG z#)q)kMt4D^A3w?}B-qP~bS# zf7stK8h)pU&|Bz5T}h19d=ig4FN$rP3ZP_qh#Jjh3S-ylPKOkl5m*SDO_wD%cLnK& z8^a`m7A{Vx`@rLY2TQ*5&ED}Cjg$A83>A{9l?4Vy7`1kfVT;17UL923P#x_3<|w$1 z&l-o4+svAw!coA&KAl%GrcJ*ZO;}f+I>!r5_xh`z@128H++$|yo|HpTJ(l`y&^&)} zmuLicepW*0`&Y7<>j^zQAt5AjyazeTKj3r*&db~ew#d+#tTZNM7JT}-yS}W7g|+X+ z>Vd)R?!M_ORwnih{*%BT+JRQzEkKhLU*geX{HwX35tbpClz5;iC09@e?b@Hvr;KW} zn=vy~U(-kUm?fHs#8B^4?piyz-NzBE)3ca)ypnS7Mys5jJi{Z!LMFF+Jq=Ke%hoD0 z8g*6ChkE&XKI2C?=O1F=dH81^HsrAO+CM#Ww36EpI|s%bY*vp4B` zw;@5q^1G18QZ0r4M)6MuP)=4Rqgl54d4J#8@giI55qTW%4CatO8DGJH8jWx6GSRUB z^w{|vcJ|rYc0_w%7pav#P{v)+FBT^qTbBc!Oq(eP&Ml1be)k(%7r%QQHY1i0DKkv- zF%t|OXV(!r%|�AZF@9*OQV3X3`y_cffv{H;Y`E=EaI^=eW=DAMuGHUZ2*(&l7~2 zTmyf4j9owFXob^Z4jpK9p{7v?b_CdiC5ZCr+Ot731nf?g87%@a>J}jr{}?etquVRZYD^Ug$I}?&Hj_^7RKZ;pV$H{P*y& zHysxqP%EVqkAlld)emCK|i?clY_MOY|BZ!q{|DJUc($R|1xb@8*<`t-~8wa{k3aM z$Rc-5I`z7arTt2BF%sQVSw1{MWAX765(Xo+%LF_cV%4MxS}h{aO1usdH=fB*3+==l z@&$|!bCtxx9%-w)=WZY`_@*8#VpZ1jo7K;?%wXFE`96p1inA&APC;CiSqY`Q#_Pex zdR&&2r;j4;!`BtVqjqz&9b-3pHT3HwZ#Uhi5Wd4ymz4&MIWU0!{{DnPp-&7PF!BQN zOXAOc5q!#RH&CCqfbthAGe`c=i{SjBFv|RnI|m6L1r7O8R6lDZNJ)0w07A{59~DXX z-)}b~mwWf}N&R3fhKtsTZNY|e7lR~mQnzE9d)dxTlw0*9#loV^>e5P<;e%S6yk^(f zBgn10E*oTh3e0pPxh6habprS)nTYhV?zVZ_T;;od(pz2W{HUAXV}4!l6=M0^E}28Q zchAyl`opb!H$_ z>dgWR6lo+=k-A?Lxf{Egdsb+rslkBVmA24j9rw={3o#R;h}Fsqz+(#rBP5o4$i{JZ zUwF4&x34tC_Q)bRe|yZpR`bIB99grjtv!+qHUC`g_BL{jdeVE6Se=j?H_23^-xZwt z8KMz5vUEZ<__(c#L{(*~m$r$R->SMdi#&hV;P&!$+%u-Wh`|}%rGtpg&VX8B+?aAM zR}woyU&b&@K z6zq3ThM+ZSXg+p!N5vPUyEJ8X1sln~s?@k?IMd$)-{yEerJ~(?{o+__3dqF(2OSHfhHl4eO(CPf-^t$ApV zlH^RuoNjcN$eqfUx-&endY~C|v1*2+H#h9n&5wU>O+p;VcVC$tPzBd?@F3j74? ziOPbWcP}|OYG>N);E$|WA!ChJO%uVg3dIw-Li8iCWpC7>Me63R$M-F}7p+>>>ln&b1+vo%i>q)!vgO#E0*&*j|SN8|7^28T-MoJfn-rP(|ye6f%R` zOEbTc&6YbZD>o)T49A!qM9kD*KNwb1Hmd^yD95M z+KZe~YSMV6L#;==^N;k8_uPlKQUsG|llOeY#sUE3L->ARog2j;|3N!)2juzEzc$9) zy@t%@#)Qvq6@pufH{g|weioSVD=7~QVs2no!`N-U`ElX}8|~(+hQe7)W1TL&@{8j^ z6CpU3O zRGWhGCQIxuI8f7ueJzd=-b=VqE-ig@l|Y^S%b0^G@8tt#Lckrn$&G&mFo5;%5C5!dYq8!}v|k`N<}@!9Ql#)qFPXR4Yz6o-y*Joqj- z^}du+(2<82EQVWQzwwh2hZNU6=X4w3E)L zr1;tda;g2-^U9X#Gs*bS$tY^F5V-hE*$=vX7huLKY|f9e$-4slHsb%YozMLCAN3|0L78dW=f+w;HvQSNHCx;(l56#$NraBNRwE(jS&9R6 zVt#q%%q|)0-mhiCpxQvaYUofdct#Jt#-! z>Dx!sG#Df_gx=}f!p7s}wGQ^hp9+piBH|h$agZQvzcE{+;uu}Ea)Eleh;Kgo|5J@c&udC62=ysOT`>rb{&f&OfR3=(iO^K=AsRJF#OS(%mT z3G+IicEcaBj#x^PN|O;51ku+FyLd&;m|*A~Fl#F$P2@boNe$5h5fVF8YIWbjZ6Nqg<>_1TU5c{l6ftWPKd*x^qil9m z9`IMmr-eSvOBsE_(@;q6$~s;tF);k-s%GVwU(HElx$hErATdkN6(*vU4ngbZ^9 zBg?}5O6S0N{~KviaJoBn|4K^}A4Tc1q@3{Z6OchRNbc-~>MsWhCi0V;rHz^F7~g}X5=p@ z?RQ6rpO9fettre{)$ECM5eM=ceZiv^P>;lQJIx7TK%o;RQfEk1dn6b1#r?ruN})h% zN=WF-cZ*d70!X6<@benx?M2O)plbvawa7Wy?KmHZ&y}Equix~5dH$ftlGv7*wB*Bi1 zpL+!BqH`Cy=%cVeZw6c4u9OquxXl);!s0_*Nf6>rluw}hg(`U}ruobF@Dxw87+vTI zp6f^CM`d`GqPM7)mWotZ=}nQTdmS%NI@~%nZ~W7QH{RQ^Fi9Xnk(z4k@Sad*HU&xK z8%`%s$0I>@DBQtFA9bZxow~*ar&Jn&(5Gbj&;;v_J|>K1ng#A!%SMDXB{P~%nD{^8 zR{-kg|Hq+}|G=dIQVci-lm4V#bmC?FfYf$#wMAc&0Km+2Ra$k8BpR8e)&Y}MT4C`7 zzlBg*q~Gb`uMSf3g~JQ(FA$ z+HkJxo2XP~pcB}A2lxHhjf_|7Pm=@rg#T0s@ygL%*)rH6-fCr-1CwStipu2HyK%SBF|{j@-w5K;QU{wKusOT~iRvJ!kq{NP(O z#uERjy~~?Q$A{vUik&a!IPq9H-NmOgo#w0O25qf3M;+}Zr3z|PcK(A$98Y|Vo>*U8 zb7r78aV3jTuhndt=$nDENU=noW$XTTR|?(KHd1(yDNLr%&K(qTcU6#3#|M*({>50z zu(vhaG#E3~?fDPOl;N#ee7Q8}<4+79EqG@+Wm!p7Ih`u`)aZc9y>Eu4?%t(Zy9MTqT|dr(8uO=_Pz?g#kj}&QbJ#`im|VvA+qUn6=!9JH>`;z ztSB<88EoACfT6V7-I*S|S|JBfBRuRn^e|kq4##wpE}7qDRgGhPDv3uHR4@HW~mz1M+qj>tN09O96wcU4URN6I28o1|4nkuqv{YL}0Y zX%VpE_*%bffehk(m5g_14eH4-PvV$T>!!kFH741dJ86-OH`Cs6)fkc#ZtEx5nm2sN zcBR#upo>@2sb^ya`4{|&iS0fc)kNuf7FTujQNS|{6BIiZ9zW%bk&1(NgR`(I8UC6o z8BXJDW>^h{4CjL81hMHSlH(mPqK3V}2`V4rUo37CY%vEP-;Ze5(8BjM9ADJZ&2+js zw8@90Z0^#Re*vv}fD;+(Sy7K!%ylYw(nJ4~Lt_7NfBb(la@^U;JcGaeC)o*0AD}PE ZaRSno!@UMyf#Z1B)ShT7l_|Um`aeGWF*pDK literal 14147 zcmb`u2UL?yv^J_ADkw@3r6;scQL2$1nsgLMM5H$f(z_&df=Ub0L_j(yB?0NZgFq-! zh0r^ph8{XdxxsVpUFWWK|9j6__n)=0^0t|GX3w74&))laLta8u?%ZO&b>+&HJI_@W zUtPI!6;JwcQrsY2d2qF#f%JFP^_7bJm4aSY0_o(Mm7Ip$l`F-Ol&5CbN#{46RrOu3 zT%l^Y{9SE#%6@z0is zg5XMF;3(-Jhk~*g)W)dAK@T$z1EJ;lu%z4SfqJNLQUkY&X;?^C@54AmNmqHW^7f== zU!(q`BF+DM#iY~{yO9ETuf%d@IEz@spLk@c&+(41-}$MqsHo_s`3+j6LaAlX3l>p( z#zPS9IwkRneDI5QuY)lwh7|jhgclq@AP};g;%KYv9w3O!d~A@knkL}IC86#Zr4@^lgZLjPUmg!NG|{lt@#EqkV}R%-?=qN zOqOt!(Il62-(;1w<4En*Ly^;fQ^!iJ7z&J+bw%p$;-9+2XVbXbW?TaWMO!O@bgDcY zP|?Am?5~XaCaGd2ZWXROJZLNVAhq0^V#u1iX%HLj+V8lSHj>+!Q5Iubdg3%>nM2p9 zz$iENu90=ek-_rix7*99-j14^S{O(Q(Xh(dj2QsnZ%87926|liKA$dRSnq|{kmR&) zY12vZ!>Ha{NmdnN;9Sj0pechr741Yn60m6gXEZ64M)jpmo*wbBq__YbbYQhCZz_P_ ze@Bt!^-|q=(olwC;X{65I<(71l7UQ(?tMCCE$FVW z%%*~|4)Ccdq&Sr66D1hdUaWu>8%v~I?*&y&g`v7S93GX@!F?b31vu9TcuooPs-;>n zC3mew@3|zKGsg*2`2TDLH!U`@{DdyMcyI8>>!O+59RNXT|Yj25GV|=0gV@eg1 z6f_hJZbN>MMJAKsX=P8Mpk>$JW^_*6sRnckjX#KS@8%vivJ4&wC8|nRe6UpC1G-R& zwwPV#yk79e>I=?5US3_nj>4-k$8ysGot@Ef`m=0Z!CYxNpB)`l z^iKl10`)4s8gt7r2E;puvFB{L$%pYho zemfO6Z%0!-aO=M7U&l4ClHxvl`o!ESM=DXGw{jJ`Q@6Vpc$yJz;n^Nhk!Qr(B@khL zTqFpE3Iy3%n)7z90^XYIPi6b~;9+Cmx~Yy^_ZH%orH4AE^=5t^PP)BJnN?O+elbBY zL*&l;DF~%*EUDwNKSZky7m-yC%djEd0n8^@_l~5+TfC-=E*`Dj?c-M4Zotd z_}F#u!>WvlmbG@|$J1TAXVadn&E<43-*(FoKBS9c-d6GL1vZRlvBeMN8wPKyqvO@Js?Q~X(X$YPm>##Q&^UX%XPElYF6oxa{Wox-enuPX$hV)sd`38%A4sA zce&9+-J%2OS__0(Yb{4Lp|=^U-vEQ=xp3Fo@A^z!7H%=dLN5~}%d1)5s^}Y%83&*G zaEsD%4}aL{EVSX?SCa9RUM7&ujo$>>DE5;zD{?%%z37PWi>-NFD<(<%E@@BttAYc6 zrOkhej(vrkK{AB`@ToU9quVba;Flz#nNH9x)xy(7%4xbz$mKp1z*M%A;r~^-L@=n! zK??)iu+|Yk{0NA&>5$!u4`JICD8(=Z*l>$l5TdMVJ$~deD(};D<-Kbyf!r_QC@f`N ze_SiNf!E{)sh5_-#7YJ~>A%r|Bfpg1j&c=VWS)_Qf&qgjb{R{$VB6FLsu$>}qi)*E zZSUHJvF)deXVM0Wj~gAlxJ#gI(2`U~kW}&tg4vUtE(8z*sFU=bnuLCcp z+N}v$v<@0@a_4u>A8&r!vBpy)1uamJGSw-R2>6BjRakFiLM@Tpw^q*Qd)aGtK!Dn@ zD}=y-HNur>KAb7$x|O=~+Mp~@2ahSc55)Kcu4_r`NG%lUH&5mR(9D-c9Cww%G2!## zKQ`)>m&P1TmKdJ5`}nPSP3xr<2fPvmBttn^aQy$f?f!oS3~$*3s_aXhqMUvLTWm5A zr#GXW7@aO5ghqa#9%n6_Fi_?^*Nm1AJ6hsY(>oojxIH?^A}2<|8mri6VJN0CH(sn~ zvW^W2r+~CU`~l(+4WXUDryQwbPScoc6PNJi%LfKLN-=@3A_XapD!oK94{$F~AO8Hb zke}3G#?ME}3h;FFUY3Gb1;K~6h?sC1p>d0ogzbQEYCDeKi4~X7W>uclA+4U8i(K!Y z6dUVfrJ>M<_T9TCvvnq>@-ekccP>YD_J&>g5*>BJ|BCRx2ZWd?I2_CGr2m#|cL{zF zv5y38(e0Z2MO@sr{MfrAIuq=-6m=oatPDp_E<4h8IqqK-cVSqabh}H~C8T=0j%Gf1 z_NkWu9ZW;RJ4B7t6~x=w!1!`>d4hrfmi9C$MTk`O2W&P_Ji=?S^IB!Bs#wbe`}i2C zAn_m59DagTakwvP`YqWuq&t2 zxIzrsMOhvsK_p@!05&y@GIeG&xh#Y`cpAzrGoAqPsgefxxod$NTpfrR;x+rna5yg7 z#{P`UW<8c1xAMaqw-V}(=IXn61n@j*gC7y?2If=>RD-|z3Ie=lB?+^>GwS4<=U(9L zc|Uco&~usa$(<}=pLx@)xi#65BeOG2=ERFyn+uD^gag0B#w=e~zkG2OuhBPQ3Hr&b zC+y+*)r~7c2gavsN@+Ax=Usji^7dZXoM^&eaX);uLlx{fn*o20zaC&B+Nhn~$0`6;s^q`$%!Fqj1eHqrc-~7sKp3wD~jg;`^huZ|B=# zpbd$GTwP}0k#ZZ4xwNiFraL8$hJA#Ct!QZ@H1ivy8xB)H}{2fp+BocPl6vsn@7V08e#JuNZs|9)piDq^g}|Imjkyuk+w zdcyRU4=aMsBWmLX&iBhMyf36T8c+Si&KXZlm}{1x1KZTHX+cgW^32s6U4)FR_89KZ2^kz#4IY*U=xswz)341!`hCSu0V3KM+0{n6k0oKMQ9YFM-47ci`|Y zA;RbZ!$2{>Zh%-RmN~!GXacX`KCDlIM65r@P*G=TsYD7RQ@ELB%fvqUwS^6;IoMa7 z>7GKsb*HOvynp_@i1m#Wy4o2&WpG(#i5(DNX6;_aKOV_+JEE~(2H?3GlrH8_y@@Xdm5mhSbHtZgb35;}H`5~)L&Q_J}5roI8 zn7B8jvHCf=f{Na3zjSj*pNH>yFzW`tNbBliAPPf|HkZ_iY(Eta^GI0aV6A)}RB+Uo zWuw%7o32!~`yPnpEt<*_7vbo|3Gi}u9KZ0t&@#BNnykARVB9}91lhH3+@gzJDQXv1k`sJDjtucUJqyuSlxi`>Lrhk%G;sorO@Xj1Y8-msjjp7K?yt*(ch2E%SrlKvcj9S&oZ@*NK*^;Bl_d}=(L7}QW|c8K89|u z)|*UIZ^DJcP?dBJ1D%y}VOw|Iqtg;(;BeXF;4~*Vd*~e@<*i{HD__>k?`39KmJq^@$(3=$X^nOQ)}YS2Zf4m zJ1W5mpZXb)M(SEg&;c=yREoPJA@=@{=B%Qqw_D86(SLq%Rr-m9=lf=_yzRzHoSzTl zO*Xg79k7QgeuiNNky~B}DNDcEsZ(>O0(;at9_Ce(k`h?E&`7^k$mY3O(Uul?K5obW zS)bBW=U}UvPu>CFiWn-OI>Yb4eEp+?55h5TI|#WzlmoH ze|n+q71FZ6t}=guk%{iS&0@((3@`R@%P zXw=VpvDCYWbB;XQrn^I|UxL0^gA~NOAvcHdzC^hXvh*3eahVF#CdolgM_mK&79-y6 zSOwT@;{iuk7qAWPpWX?@8tJsH#DB50P2?V}-&@BLl4!U-Pv$PB@~4mXP?T%yp_BJ% z^msXx-=EdQ*eIzSMi)xv#wWhb=})+Ictj=ZVyxg}82MS*@lO(b;dlbwbEb{9XgBfv zMw1$q)GNU6x~c4#8cQYE$OG^_t2PjP)+8)VxxG!JtuK*2Z^j4g-0|Jb$q*#L369Sw z{Feu6x1tvGxW?Sd?!s>Rz>gMP46^!fJNb*dP8GB>WKD_rR+n*onW*2UBy6ZSsyo?x z<_m`t7ToGit-ZFULvbp)WynWXefxe1)4toE%I33FUf>cSoB0Sqv~d?h*5MYo@sU{J zT+GmOxOagJZ@u5)b(@iBceI#T!DEd|w#^=d;C+X~7Q@EG1LMi@&B%reLxR6#L#mr} zv3Py8nYN9W8(GmEPJA#0XV6`|V?Z&k$VE|o^r(pboVfqk9Q=L$&#%K&UA|hels7!7 zJ@VtyX5E~3S}6$i3DSr@vgaydvc`4vVmld)ygGqAxrHkp!HKckdK(2J&}{WI0lXz$_*}R3L2V~oRR(t>LDYJEQ*&P$M*0f8nkh#2w4|G zBwMocVdG~+;9W;?-CLq!9z$(Bqtu5<#@L;(tA%nK7fnua!>(`z-E$=PsMHBlEJB@0=j|tw z`qK4?OfOGoxJlYxRR4|3eNp2lLjw#Pmj8CD#?ls`!$fLCEPoSL&K4#bjMk;J?K3sW zR*I-)-L(7rSUy_6f?89(LZygp=P491+xZHYbd-vaR-LI=~FHVVR}%b_nn&dvpkO-7?5YTza>;cH?vme6Z9t|GoQLzZeN`!aZbCGiNsc(I#)NejV;5n_#1?{LG?Ajg~9p`*Hr^G(Yz6 zIOu3|3IxP4YH{C?Mc?|2li?b}mu2S9^!(n>E^AKGk-p zZdkH76B6kbu?!2#;Vq|!0T*m5NsFTf*7^xAE_3e&MX}ECQ%jna1>`%S_O@$v8MZ_b z%c7M~n&p4S_)}y?V}na+lH@M8Yd+nM0_7CGgEwJ0+`cTdQ003QH;d8Vb_vVkB2%`u z6Y=2a8R*i~Z&<6q%&v%~!S701mj+8=sz!+NuapSX^HAyF3`R;tCM%MsG8objj^b9p z$lvZ!o;K*uWvRQ9oMP1g0n?g(caDc?kgcJw8aB%l z>`P|J`jw{z#JJe(?Xn?F`l8zF=lF!~~|I8ZM$82z4xCS%pSK+mq6*yccP;*4hTnTP%dZ z9AXEVGN~)%C3x2-fx4LB0kOS zh^L^OPE$jpT{pcmARduzxJVi~F199Q?YPn&5;x;(V6^2h`aznv za$U`EM%k!+V6;Da{wvh)M3gi2c1naLt-iy_=k~c|bbdI<68w_Es$@B=KDDqdL51R z5ZeBXV4SA*UL@{M#b%|8Yj5UJnnFx#;Myn4xw;ri=3`@Id#=qKbOa;GHXRT2vDNb0 zY2~+B%r(R9rFYUO74Xe^t(YW%PHB94F$7~y zB&@K+r6RxO#M3Jq%}DF1n;IIesgbsvt8W!QS=(kNhN4Lm?S4ST#-Dmnze}vdxzpFD zo}YXX$-<9!`(}X>7e!LYb?VYY8$^;L!tNa4nwNxqX<^qGIg|tT$L}JM9-WLr#$)wP zeD&Mo?KVX7+JQ=^iC0qmMh^>mCPxa4O!Mf=_X4R0mQ2GlGmyV+#!2Yx>x!2wjO2mO zpG7L;cGqNI$a=lCA{sy{p^q_BaJ$whR^Yv+PwPq!mtmF8DKMNa&BP4gFK;U&9=e z06uw0T8Ki{>!;h!UOTbUtz<7}s^-e~xEbnN5@bbnXJMHSg|zADrrO6cYT+gYi>vdMCC#Vg5TWumR7(2W-QC=QBqql3bR5I#(Dt zVrbl-aw=y^j@^h7MzD&xJwuh3d<9v0wX8|kEw!x4)E%^}$y$)+kmyBQ$V9B6Ed&rx zqCjWg1m$nv9SRA1DM{7H?;hBz|uANPfV|I%uibd2xtuKG_3((6}tHyV%4L&rr zHlS3Qr*Q!vT7@4x;Ki}|lJy4z4>-Hk;6PSS)_+~@#45zVO?s4 zBY&j>T4MFX$v2a1w$+f6^yzHb)#5y4Ey52$?A)W)Er2jg*Hu(ahq8?LjtM0V2FT0UEVAbgS>xj!*)Tyr$EH>FgF_LMb9 zS`uAdXjW#iSgYH0@_5jrfAr#{ZQpKSq|h+>d_Pg7wZbcAEBTmYiwgB(&o_6p30-md zJQb6F@P|6|h-2Md9du*-pLZvxN`ed%MBjstam;>iOTIBgbi z&ph7TGIm~Dg4LGxA-!!6_!iH(4jHQ>JL=^($=~%w{_&cuO`>{9)u~YZh;1>mE+%Ow zZb+>%!n<~|@obnsDRd2fsI_(PP^;RivyIRepP(`mpl7F{=JAV~Ls|7W!jB5MQ6Z-| zIDgEjY{-tU0o;Ld#|ume%NTpj2yJDt$y8-{s0Qmwc>KnwKdC|Z$+#m zjjIKW%dFLiDzlir2Hm>2uWEM>IeQaNg1q2Yb{xv$w7*F)mh338 z4CiOxXzfV|4h^}DMKOMGs2mn%|2cjG6p6E8Cbgq3dx>$M4($~iVQ{S}pR8}|KEyKf zUxwom-v#AaResQYr{h>+HH%RJlCzw~RQv_uVjN57_PiVpO5F15C&&H<$EYe}W(maT zjsCxhfb+|B@eij&|46Js$RlJJe79_|Fu45P_rIxtXQqKw6W-WlF7+@4FokQa1I>qB zFW;P@HRxr3v`Ip`l&g8zV~i6Q3>_q*aY>4Z$L%>NCPP8fE2sGrrE}H`5>%~0;DxKb*s2iuB?<) z*06W5qNr%IAOD#IP@f_-9>-7d#mRq^C#>Qb*3)2|^hy8|iuN)wA!PhNV11DMJ5O>C z&I$H5_YlvKc=w1)uLi71VpnvFqI@{J$9+STWa_emFQ)}W!-aS2(v=6Ir`#`RoT~fL ze}L`EpI@?GUN@04D1UvcPya2R%HL`DzX&dqWMw}XC_~hTWI7D0 z6nO()*7WhZ)p*PA-@lI%LKM8?g{&asMP@BkeHi^Z4;P1V|00^h zD@exiKk;pim!58wp(-Y+x7JZ!3b*zF@&V8|cvZl#!d@P)zzQCR%nxg$rhq}X2XRJU z+%_lmebUsupZx^iFQ~aiBxSlxjFLz+6OA9zpQ$!Jz`;%OQVJ+Q#BBR}wr~xue?@Mb zE@GmnY&xJ3i(s1~Y}Z)~B!ziYy^tK-{RsPUOYJM4gqBG-)#F0tF!A2`KF{M;*Oxk0 zRW+2wD;D0LXQLp-;|lj+n!`XE{mxuku87^tCKMD&WR(x-=!gWH$Ebr546qORF&_@T zC6I+X664Z3m6>;vDa^HC>(S1nV~Q z^e|RZRRJLr5Eb@`D-$_%V;vH)Q0%)`k%P2pG+oS^)k#B|Lsinr6#y3!9HFU5^WLmZ zqe(6~UGF~#KQ|aaA7#ZT z?8D3y{_oZk*zynO6BUfIywn-Hxlcq$?zZrgSs|x4*go8bSV1sOltZ3}+u|qkCvdE{ zcs)~qf@MwB>|R4&?+%zA8$!?tv@cNewnoSn>$2ZqK88tJd>&mnd4bx|1R!wgsOOZL zkro*yodJw2uo`WSq-rYxLklRMo&MOj1R7U$qd?O#YfUYV2(C4rasR-o_TL`ysgtEv zP@CZBKkbkV{b0JYeXf8KDvZFTPy*{mAxw`bJKTn@rxhpn%LYZ7JzMNOTcfDx**kd8 zjy?EH2|{qtrYGOUt+2Jw$=)is#@&|BJK9!uwmDU+7ASrg2^&{O4FUIpgTS^AvD)Sw zn^z~dG&7bz;)=t+HL{KVAK7s{T)vIU)$I zMk*W)htuADqX8+Sxh0ywo7txiig*lbX^Y3n&I2?PNqX^5xSE!&&uXw-@Q2XzXXgET zzb30)`-JaiwcOKT<-qQZ#p%m#*1*L%$0fAZt{T367;&|hmOg;qFCvHGrAMa{Alq>P z3eXhI($=y1VcwcgN%SJr^pzdFr^r?x4Se#(eHthX*13ETOUrStzD!yn^a%80o#x8K z!~*8^{?|f_(H$^5dukrnWEq{YXN2ECe1`fB(t>-wcS0LT%~jJx5J!1D}g zRUfOTcVQu0>P_;K+>>+3ZToLHBC4eIP+at|zqB}<`~QkaXZe2CjNZg)P9)Ybo{Abp z6nI>i4ga$mVzNcx_FA!iO8wnTOzqjK?;H}OU3jQ23Mg=|O^NZ6yI&}#j|$I`(Bk;l zp9gz7w{ieg>3uzDTB#mL0~I-zXYa`lB^}f}+(_-Hpn)pv38uz%mOCqK2Db=qJrQq< zWGkiT`I!^7{oH8a)i_qO>X+M_=<9#p8N&2wZiM)E-WpWOqsx+$Lf8lB>OYqpwz;iK z4jX&KYth!A$3Rj(I51e&aFLiGnH7g2`7nbi%~6%jIR$@|_;|5vog%!YQM-Kha{45q zi`T<2qq+)D=3)V7=I5Haof)Pt-N3%FS8M;hlB_`ED!IGkG+vgIEo)*Y&w4M_)xbeb z64bhPQ_a)-1jGlS{IIa5$J;PI@|GRYS*Rp}?+_K}{g7`k8sMdQJdbBJTALiU+(&I3 zMc+8j18D%-ijMYcPN&{*Y=VTpou&_LZKsQK zc?7X*O(=NRmP~u-=7y%jtHq!6`Civ~sL#um^->vS1*rjU)5?l`g*zME3u^7q1_R=Y zcK99-om&a~L35{z(dsJ;)D{CT;AZIRC+PS`MZ@;~v>Hyx7)lQz9Loz`bo=a%%=1d` z_B@t|i+p2lL}K$9?W-~lZ~!fa2fkSlZsOGu$F(F)Gq3t~IzJGY1#)g0`mp`nd{HZ0 z&N5lmoMpTRY5Caiyfa+SGt!dqpc<5x0x$T+oZ!%G`J+rXn{f?Gb%!8A-EQT)A#Z7g3 zyo_ASH}lb2_pJm7)6t;22z!%@z1}xA?(l5=Bd}aGX}gaQMMhS%gF^r=d`_j?i0j_E zg(k)^aqpH997hFD(#H@2YFxug>bd&JdS%f!PGchBF=LD^Z96epv}=#iu_LCk?4~!l zI)KOv&D(N-#%5tU{3Dv<>iD#A@|QE6fXFXGQ`fRD7uQ#ghAE!=3wDwML*Q?>NN{9p1ZY= z*k-OySU9cUgQYB-^El-x%+@v!(b`+gc^aFtT0~mBuZv)B?B(&+XC7oi`rzXm9`Gmc zY>F8%&=Yi-in8nts=M1f&djGSdmmz6o?mx-kf95SPQBZQK` zrC4*oBWRX%^Nm!HUZoGuSuxsKx7Z@ zyxXGXIP^R(S4WaWajdE{`JsG}bLSZ!ON8s-bwv8fKdsu*Z6)o(6bgFfS5Il5HP5~t zIa3yGTT{~4c8tyKR&LF*KL)mf?sT$FJgc2~UdSn1r_f3$c0cT7B2Z<7nTHiFJE290Co6 zYKc+_1+HgEmEVK4(tEI9@;K&l^7^%jh*QA4t*_OEO7?2N(=6>Bz zeDP+e|Hp2tEYQ_4{Wv~k?%)llE%n(w{3wvdWc?%OH5<#BAbU}iuoX~JG_+4ZPmyu5wyMkP7jV>@b!?~1QR3|K(UDd{wJfq9JR z`^UEWnV1J4uAWox@W9c%Ys~du1u~;EO6SF!BO}xtdkr2PjFGMsAoot0)nruzTP0<`>DK+?)N-i++)Nj?_Xd#tJu0?!Ax^86RRE zHxj!E%9!(Th^60fs#VVUYq9fjW~TDp}-o-n`)BC((CUW;<}Dv3aI!=$JPt&gpDR^Psz;=PphN;}6O zs4ap4^SE=AcsJq+tNXZFTZFF5QSj}BmIA4bfhYC;HHBKZVG32t~iFKzEL?N zLnkoK@s1;cy{bgTPkGz0g%`G?M^W?ATmUy z%19~$kjzC*yW7h_wHVcLSD@{FwYZz%4QmUAG}8A6;ArK9J8iE+{z_1fof)N~D1ZNY zMgmQ{t7qLKuqKwG#R{S>*+))O9ztOvM8Q=Y?h@D=;@EYiXPwiL6(CmH6wgC)m~rMb zl!^*4_IG~Q(n(RaXX3#xUtD)a}hg%!dTl9ht4VT@u9Gu7VzZzBoPJ z4Sjd#X;+-!1r{;)=k0eS_bWl8ND0>^^&bv2?!$TdO8QkY85tS#R$&Q@^e`cZZ{|AW zUEC@E!iH~O`yTs9T^Yqq>DikqEy-_O12>>3UQKw{lO$HS$ab&SWy2saA)5wDKYr~P z#?nrr)yKzVvyff3(h;Z!5NT%)PjT&)2xMfxy@Ttx@7LSq)R!t9NW&Eba%2KY?{KlY z47scsmDVMOK=|6feW5H~eK7oCHQei)%z{eY3dZi4-hf;Y(Md-o0JjobhEL1&=uvei zeORJo5*D9f%G;r9A!JFi5Itoaw}254aqe3zfNZcP*c!agsDcFf=m$fW!)51`?y-J9 zIwC$MhKnwifyExZ-1IPhy&R=yI;h4)k{Op<%qYW(5ml(%Joat#+3 zS1Sg*%cwK+JZ;wpkI#k@uyw6)TG)_~MP^t4=VJSRh5U%)Ny5q;=6|X3}&miAxr>fBOX3MZfm%Ef=YXh5fs0 zV!>7FV$oJejSPp*+=OlAj?g0H%P#fnq#0X`xoIhwf=U6UkYZrPrHfhsk3Eb(+B`hP zbK1t8^Bb1ChmpSV5wnG-2v?*e(?QQ!U)tR+liC=FNdB^(XasTDN1!1<(hT>Ny8 zMC!R^`Vi|Wm6cW0FVO5F(AKn=;)eu|ys!p;@1%9%3%(ZPmEN#DUxTgDPN5YG(mF$M zVzFIpb)V#uCt6>-h53hiK0^YnG)1HT2lV6rK`8IP0gzPh;g^qKR)5O~wDRoug8FmZ Y($U4nwH(riZ&#ixK@72*QvkBYKDyHDPoijNbbg z-4MOJEC2Pax7K^#`rcdLTkotjYv$fN=iGDl+2`!v{_TAtH6hBx_h|0n;o%WKS5bI@ zhljrie3%ID0C!mM2dRKBeD@d1a(HF^bXefx)@xY}SvkodX^Osk6c)}FV6=ZdM%(f7b8K&*Ns~Gbh;rQ45-Rd0>e#1;r*Tws)4s7YvmEvCV z4g|Cze9_Q4Ebu=wr9iZ#!)Kt5ksr|=1hOdJf{ZzthQ^mN(q2k-$QamQ5P0086kv8vI zdFVK`Rg4`WQN{INA5&Ud#qc3zxlkR{R_eh95wUz7X?H3>w7^|rOGaVfE)7b~0cch* z;@^s(|3gJ|__SLucc_3shOu*#5CVRAV&~%GGToULX%K;VEH5vwh;iu`!(^^dav3Bj zDJyr%dxrnn7^AYw_TT=Fo$;QdKawK$O?++oPF{Naord*Guvd}v7v^=27kb{kAGm2{ zBuG;dU6N9gzN&rTdLsQiX?M_>|CMq2jGX8192IaPhWHxX3)MjjWB(OI>nI_!{nm1Ig)M84@b z)?ZX*Lo+^r*_zw5QsW^luEV7C@Cge#pQB+I72~La+RmEn# zXFH}&MYKn!a6`6$dcl7-UG?EwgsU~8^cKXzvfg8sOa>RTm{a8*^d{vs3{pm)n1uNL zNoOP+7bT3Tx+m{$Wd>nQ2!AlE8lYC)`#egJ?9bfm`@NfXtWDqgm~Y$tUKLOQE<$S5l9vY?MUUh&PV?RN8njA zt{b))YaN(oV<^`9c-mu|^yXgACQGS7ft**-MbEs6p=qb!lwzivSLOh*oTZaG}2mMoJ=M(5|)<^mB6s8$E4=sJ|GI{s7zdBEgoo3j90iUw7!sF z@Lf(^Y)D63D`LfdG>?~Ug7?Qykm&xKYl7de3`g*jkm75N+I{GEUbv7nr-m=V1HIix zmAGK>En8?Fmy?ro`bCoLIHWaN?@#!WXpVRxS!PjPvdAd>vMFj>TKXg_HD$+FY_dCe zHzK1)-hcrawUu=CWTw=(mU8a4u;zARVxntyWI{YlW;}zatjCxYl^K0vN#M2O^;9kk zklwd{rC0cE=&13Vu5z;X?4Y@8v3OVG{;~vu(ziG%9?)H1xhy9M&<9aeY!`AG+?9$` zlm&74SApH8dU_@}hxe5IQ{aY8MGz0}Wfd+H>0>4lpSnju1BnkN%j75FV(v%K;J}q z$yYpST&yUmSHgBhah$pZAt}4J&tJu~)5ZLKh79TuX&Fdw{cbc02C|$8T%y)lt@Mjs zm<+1wnG|EIg>E5gnLSfdnVnnCX*3@#2~k|+FlXH=FL4U=D4x9E06O=f?~3>Mr;?*Q zByCsg3-hum>riD|bV#4szvG9}zu3g<6|almFW@IzZEx0vLUGp+iP3?3(SM7^xJ=HZ^(e?z3-g9SQg`)eU0h^R&NW`Rf zNFWENlO%RH#Rp7}&1h?cAeD*I#7DU@ngv>IHo$=OmE7Tr({B2>Erc}x`u||M{*N~A zEiY>GeGa`g?pGV*+_99CBc+JjfXE{ez)GG5HB(xKc%0x?+WDBGx>hZXR^q9b2JXxU zM)+?C04-lA&BaID&Iy?D9Mc%@YEX^X-__l_wzjNTf5y>aLG8W>JCYl81xTT{rmU!a z%s}?z@oSr$)R#uecOxp{Lu5q)y8u0ch!4}F)c@}jYQ{TCo%I;w(AMPG2noc;*#>WC zEm^O>rIgN(Pd8$&Rxaj~s?QWXDd+unwBS*>PrX_7*H-;wUU{g2_Al!rIKiDeQP6V_ z1}f#g%&`r%2sgG|i)&0ka%%J!4NS!S-x>b@U3w~WL=|4^g2yv5kTagsl&Y)BtHe9yrx$_;7c$T`5tuN2-31u^ zv!NdvzTF~95A%@~$Dwiranp`=9)Fs!Yp3DpieCeHOU7e)OFJ!Dkj~W3$P$a@RX5s=S?)hp{#2t6+^ZY;N|DAD-KiLgh<(|{5@bV~U z_2uD(&lQQ-NkQD=4E8rK4=s6fi^du6)s|agriIg0K3FZ7NIEO5>?BF>yv==Rm-{M# zWO^^ZLWa^D8qQG1L=W4X^jJ>MKzH>t46XG($ji;hQtzDg)%n!qXm>GDhUAkWGRA#5zm?8UEcHT#mjYn*)R z;JWYFmn~vrZR>v7$EVgN^dq~Wa>Givo*8=SvZ?&`O{45tYBQCkhK4|UOxkMlT|U>I z5Ci7dRzs0iR`hH3iWncC32$@|_)3lDydRu=nP&X`LjMvX?y^A@73c2pYXH5sRgTnF zj*s!2XSSk0nyn!5pE8in7^?8rE66REK`4z$B`fL8XvdVqOTtXW&wtmVKW76<#>1dO z;3UCIJhwY$w>AmQ{~6knaR}dNNnN#j{d%lftpE~twRV>JNPtP{S#OPOUuat48b4#% z$4TLD+2sDS8pyLO8e`|1 z%+&~Br_OfAt1VD@nd9QcEFn1Mp()N>t=IasNCrX~C50ChJ4Giq5kajl%yc+>uo-68 zX5}nxPPp=!B{gp$Ls}-^ZsoJRw2-je1#Ue4_|(IrjW_y9*Tj#)C1rHC{Mk_C+bMM9 z12bOae7vJsR$LkUpV+cXLmJ=hLyzT1a7^qr6;l3a07|e^KYwt5iOuFlN~Ev8^7~fO zZo;jnERkoO>m_O8Hj`In`%vf; zfeZLmdqKg+@5H;i@6g)6r%8M)2eE~APDGO4Q`jf=P7g|&j}CN%YwwQA1o%Z|Ox@1N z7L)q(%kd_|u9a$Lp)13|)JX^Si95qFZNSxmj`)L1`bXkY?fo|cVP>Cp*6bOTJXjia znJt^`d&I(IU6$I6B77chdX4|P1757mjvuS~y>7p0Hzhw-aJ1d{^sUbY2A#zzqcxje zf$x?|IqP#3(#fEp zAc341IKv&$^08Eqb$r-s>H-Lo9PIR5+1ykw1_Fh0!o{~5j`h_-t9!Ou&TFFv`Er*N zG3k0{=@5o-fviZtgHkgmsz{f}oNi&JpTarN815}u8hIRd4CG&=Ug#Bm)vG^g$qJ*q zuom|j(2pvV-pc#~#z-Z#BC{P_ZVJeoSRwtg1npGe?zRUqgy za`l92lrk7}%1OQQY1)(124Kjm>hy&im15!rgwZo!<M!B$@78dW1m&K) zlXuFkzka<_K1yxXXObt(IJ%+WS3i;Qv=8D#lg^eIt@sZ={%YaM(vBm|1G^Z$J8#G> z1sW`V;!*zd@@KF~;KV;ciUffFsSdcI$lDtn`#pp6N77HYmjd{h9#V?E4+ zT3$M9{T7(SHq;gEo&R|U?)MH4T(=aB9shvoxJd)Jyo!!;eQie5w4*$#bzY+|JxP;* zaW$80v9e|gFkUpL{YjR`(aY4&pEs^{6!4obz|qw%fPJola?8<~n;SLyxH2ZtK@U&K z76{9iDjn~im>o*DZVk6hbuv1Ft7X+x#eQm`A<{6 z;vzR*iTo2%4^t?sAuEC$AxEI`Kp4WV)WyWU86}(X-Q+J1XM*>x0689-{I26m{6h zog(?sgWo~!We&>@tZtjJ7<7AnJ?d#pkr+8Vb-<;fBwKj6_ zYswrtHhZL3|F$G9HE!gOfK^QkZdG&gbYVfzXLnqEdgyiMR9^SEPh;}xhj z>8rI|CR~JE?67~`V|Tk;9rRLI?lsy+m{R0hc5WvWxz!N&kY4bJ$Qiaa<1qmhE`T?9 zy*=$aAIV1)&KykdAg%&V4-~E5iLJKif z%4`4SBtL5wc)R#$c5C!?4908LN}bA5-m^L^2K zDDJFG#ksj2DVR0W=Rs>em6>Jw$H)kW3iH^1?`gLHHG`O6{QP(jdb@Zi8@?1eMDn{# z>ZrkL8_G*Z9_?JlwZ1=BZiwJHUzqebapcY4;fArycwE(cVA{%&`Bt7+y%ZTk`}SsP z8^R7ejs@xK!VZ&}W(H?Qew*K8vCZaADzz7uZ%;30${%{}R&R{u!N2?Io^D$6#>IyC z!%MY}GDK3RCLF`f)F~Fnm{4RhkgShN3Z6S6vo~2UfBbm1*L~z;)=G4ZmzQ@2cPGAf z?0?Rm%Y6Lfto%3|+qD!u)V*N3#Q00Sm3eYo6!|gT&1U_iHA6U`b7%TJ?CbK@K~iT8 z6>}?_`K0Oa-l`)3`pjQxD2TbU)A&afk@l~%2Xvea2bQOq;!VycXYE4%BQws(^7vT4 zD{F^?UiP#>7ek3^H@MqE&m%sBX*4MK7Wv+7g(29ZIkEm zmTqQ7oFVj8B~AizWnU`x$a z3tI~Z+u`BS`cLA1JsbS`n?|7aD`)^8Lb9c<%+SJfZGHzR=lQ&CvH93zg7WzAOa{qk zZbwNH_dauNoM*Vf&#P0gnQCKs`yH1LOUbFZt)SKK)hj|Cj>n1Wi-N|P1WTZA6y-$0XudKhtaY^{KXW$5Ss8@f`nSq9%yU6d6EH_qY zJKS`a}yJRw)`%hcB$jQH7 z)y}Uh1{HEoJ&5bfq|cP}=D`5(aU{~>C(Ax60EcI_Sf?s*W+vhDUWHtvWM ze(-y=%1-^$H4;HgDxr`Kbf#A-G?#;O8lZUqew#b!kKg5LY~E}~w2N1BjhnGHih7xU zU!dA~c=Q^85gy@y+k>nUdRJzY!(UdGe_Q@>S&5Lb(wwr0R3gc=*X%8PhWg7)!TQ6? zJG6Y6?nMeZqHA+u0&QhM29|=o>;V7lxbOOU&KdN z8{g@)0+11wqWW%yQq5#I^;kDqDq*RxN5tc=u6ZRpzNd)QFWB#)b* z$V3X6=LK=GG2vX{1aaflq1;mjxKec=*OKlYCaPkyDH#SdKwHpX+dX!<7(%Z~tbCrT zN^eM-M{h`-gThQX9sDTYV|Vr{ll_N2;3MPueqo0bN001M%crzUzV(>pTen7A{k(U` z^VVXidCveKE7p;ej7s-O6;YaUCM+e^sAIA3AOQ-1CL!Ea_eniDjw8nYzMz7F=KuPN zxaB|CJSgQE(%to4rOg1V3mbSpagBNTL70zM5>GV|*>$udsLP`yw3vPwPIMxyc(7iY*m_ zLJ4aRx{2Far!WEE{;WgIo4q{imlpJ+W9#OKS$*oCDH;0IFpQ@J9jtVxvov3+! zbZ{{QV9y~6@WTu~AY_#O9S~Vez;|U@#Z;?vNnpPaCSxZz6%(ec+etr)ZVCXjypIGx z1M{%LnKRpj~cE<97ReV4TC_T!JZr~x73J@mtcbTnPa(Z0_S`Xq{0lZB+$3_G zfzA&<2k<@~e~uzG3rZA90YvN4H-Onmh<6fFbOfIFSp!}aAk4pB6YwRd0yr_?Q8K!H zJl)H{6d>{^kP0a7(uGA6rZ?Ewn(1#P3ZPQy8FxR&6&_*hi`P{E6VL6(=??RwPD0akQh z@fmJMlLZw~S1T(B1cG<^L>c>)s|j5)Kk6OxGmfrZzY)bKXa&?7)Jkk z+_(_@cY=U7GmaRcPbLR*IMf*!Dx9j_ldj|6x88E2BG@3ybDufx-;kP_Nxhb^nZQzr zD0oCK^a;7;ziS+3p>&rI-IsK?qIDTqcmleP*F8i-`>q8+qH9slC-h5cU^p6aT}(Ou z`VB{=pibcuBjW4G zyXWttzcKedoBu{!evzSdk;yeEFzv%1Y#4CS=q*Ohn4WVW28umV&!5*j5 zp&YY~anh10>zhM%)hZ;H$r{ny!?L`+V*LF4+wQ`iDM<$d5<87K4(B`>!pQW?j?# zI-W{HM@M&*-c?#VYc%ByvJ`V}K3O&Dc}~k?RK@N%Q|~!l0OLYQg@9da#Y2$uOtEUA zJm$>-s)&9a@sj!wQFCce>`*@U)nlEYQpC9yV~ZJ)Yi*AbU0JZ#(kP2&PR5E!or{H< zYt*n71os2!bbg046S6q^1jT`=_uPouL36)uXq~h>Hhamq&1fyBu3QS=qR~?LxzGR` z7mZaA@*DQ0yKK5lOfY|F8RQsE#Vf6yC0hThS5y+9dwO`YnBNY40I!V1^Ny&%FB zUU4{nNu-MF&laQ>>xPyUWyZYUU}y7HV5GEm>;e;p#vfYQA0%musrb(}`i6qXY7`2e zGg4M$jvMGglzs!L3%D8Oo zW{@Qg%Tk7CipJkbp1@&Jt6%!~oJ_0U$cq~~d@Lb;0;d12gt~YE!O1=Q3f{xREf0i* z5G6jbmydn7#P?-d75b%5ooGx-Xy&=tQdM;i(krf2HJ}-@qBDWp)wmYA3D?Pk;OKX8 zK_oXjLqQT^?F*vdP|%&AvW8&taTbf#!u=Z;KA%fsLZZDok41ZsyQ<0v(_6Hb^)q3T zLgF#jH>!!}c_qFg_?H)QF9J9Mbm`?Byz{5HEhlh|xm(sXWXfY-TG=HrZ1&EwnQ2bE znqs7ATQ>F4u;g;phzb#~qJBye#B62Lh9G`RHA%AD_y)5vDSO;LDNl{wT|UgB$_s4r zUDxs+O5cM)<>-1nvD}0jW5@*MlDo-h7@#p?&l^%#9#SqckiicUu57=jc_g|wz8NVJ zz_`+pv%N3S5=ka7c0W5<`odwg4PNT>dm;X*&Qrd5oZJzA;?aeb7;d*xCd!~BoVu1r zr$*9w!pz$KAb!D$S~K1Ks$NTMUpO5Qvw1q`NL#ET``p@_M{~T4|7A8xgh4*0+z&q= zjfxO`M{PyG#LgJ1x8lHps&YKg{}1WzE#9Fqkui$Hj(a5TcaO$pVoYFWD8co>=+x+} zfFP?Lh0Am82