From db02af2579b994797f13a2187f5e92e30374e707 Mon Sep 17 00:00:00 2001 From: tanyapole Date: Sun, 22 Dec 2024 00:20:11 +0300 Subject: [PATCH 1/6] unify model creation --- tests/test_utils.py | 40 +++++++++++++++++++++------------------- tests/tests.py | 2 +- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 08f08e8..82b6069 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -21,33 +21,35 @@ def create_testing_data(): return N, dim, data -def create_testing_model(num_classes=10): - return nn.Sequential( +def create_testing_model(architecture='fcn', num_classes=10): + architecture = architecture.lower() + if architecture == 'fcn': + return nn.Sequential( + OrderedDict( + [ + ("first_layer", nn.Linear(256, 128)), + ("second_layer", nn.Linear(128, 64)), + ("third_layer", nn.Linear(64, num_classes)), + ], + ), + ) + elif architecture == 'rnn': + return nn.Sequential( OrderedDict( [ - ("first_layer", nn.Linear(256, 128)), - ("second_layer", nn.Linear(128, 64)), - ("third_layer", nn.Linear(64, num_classes)), + ('first_layer', nn.LSTM(256, 128, 1, batch_first=True)), + ('extract', ExtractTensor()), + ('second_layer', nn.Linear(128, 64)), + ('third_layer', nn.Linear(64, num_classes)), ], ), ) + else: + raise Exception(f'Unsupported architecture type: {architecture}') class ExtractTensor(nn.Module): def forward(self, x): tensor, _ = x x = x.to(torch.float32) - return tensor[:, :] - - -def create_testing_model_lstm(num_classes=10): - return nn.Sequential( - OrderedDict( - [ - ('first_layer', nn.LSTM(256, 128, 1, batch_first=True)), - ('extract', ExtractTensor()), - ('second_layer', nn.Linear(128, 64)), - ('third_layer', nn.Linear(64, num_classes)), - ], - ), - ) + return tensor[:, :] \ No newline at end of file diff --git a/tests/tests.py b/tests/tests.py index 43edb93..aed75a5 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -55,7 +55,7 @@ def test_visualization(): def test_embed_visualization(): data = torch.randn((20, 1, 256)) labels = torch.randn((20)) - model = utils.create_testing_model_lstm() + model = utils.create_testing_model('rnn') layers = ["second_layer", "third_layer"] res = viz_api.visualize_recurrent_layer_manifolds(model, "umap", data, layers=layers, labels=labels) From dc770a5f407106f6282052f1913f522bc7a10530 Mon Sep 17 00:00:00 2001 From: tanyapole Date: Sun, 22 Dec 2024 00:46:14 +0300 Subject: [PATCH 2/6] change data creation function --- tests/test_utils.py | 9 ++++----- tests/tests.py | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 82b6069..6dc2cff 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -14,11 +14,10 @@ def compare_values(expected, got, message_header=None): ), f"{_form_message_header(message_header)}: expected {expected}, got {got}" -def create_testing_data(): - N = 20 - dim = 256 - data = torch.randn((N, dim)) - return N, dim, data +def create_testing_data(architecture='fcn'): + architecture = architecture.lower() + if architecture == 'fcn': + return torch.randn((20, 256)) def create_testing_model(architecture='fcn', num_classes=10): diff --git a/tests/tests.py b/tests/tests.py index aed75a5..08bd958 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -17,10 +17,10 @@ def test_check_random_input(): def _check_reduce_dim(mode): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() reduced_data = viz_api.reduce_dim(data, mode) utils.compare_values(np.ndarray, type(reduced_data), "Wrong result type") - utils.compare_values((N, 2), reduced_data.shape, "Wrong result shape") + utils.compare_values((len(data), 2), reduced_data.shape, "Wrong result shape") def test_reduce_dim_umap(): @@ -32,7 +32,7 @@ def test_reduce_dim_pca(): def test_visualization(): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() model = utils.create_testing_model() layers = ["second_layer", "third_layer"] res = viz_api.visualize_layer_manifolds(model, "umap", data, layers=layers) @@ -81,7 +81,7 @@ def _test_bayes_prediction(mode: str): "gauss": dict(sigma=1e-2), } - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() num_classes = 17 model = utils.create_testing_model(num_classes=num_classes) n_iter = 7 @@ -94,8 +94,8 @@ def _test_bayes_prediction(mode: str): utils.compare_values(dict, type(res), "Wrong result type") utils.compare_values(2, len(res), "Wrong dictionary length") utils.compare_values(set(["mean", "std"]), set(res.keys()), "Wrong dictionary keys") - utils.compare_values(torch.Size([N, num_classes]), res["mean"].shape, "Wrong mean shape") - utils.compare_values(torch.Size([N, num_classes]), res["std"].shape, "Wrong mean std") + utils.compare_values(torch.Size([len(data), num_classes]), res["mean"].shape, "Wrong mean shape") + utils.compare_values(torch.Size([len(data), num_classes]), res["std"].shape, "Wrong mean std") def test_basic_bayes_wrapper(): @@ -111,13 +111,13 @@ def test_gauss_bayes_wrapper(): def test_data_barcode(): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() res = topology_api.get_data_barcode(data, "standard", "3") utils.compare_values(dict, type(res), "Wrong result type") def test_nn_barcodes(): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() model = utils.create_testing_model() layers = ["second_layer", "third_layer"] res = topology_api.get_nn_barcodes(model, data, layers, "standard", "3") @@ -133,14 +133,14 @@ def test_nn_barcodes(): def test_barcode_plot(): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() barcode = topology_api.get_data_barcode(data, "standard", "3") plot = topology_api.plot_barcode(barcode) utils.compare_values(matplotlib.figure.Figure, type(plot), "Wrong result type") def test_barcode_evaluate_all_metrics(): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() barcode = topology_api.get_data_barcode(data, "standard", "3") result = topology_api.evaluate_barcode(barcode) utils.compare_values(dict, type(result), "Wrong result type") @@ -166,7 +166,7 @@ def test_barcode_evaluate_all_metrics(): def test_barcode_evaluate_one_metric(): - N, dim, data = utils.create_testing_data() + data = utils.create_testing_data() barcode = topology_api.get_data_barcode(data, "standard", "3") result = topology_api.evaluate_barcode(barcode, metric_name="mean_length") utils.compare_values(float, type(result), "Wrong result type") From 18f560f428c626a2639fa09c0983d6471bec1264 Mon Sep 17 00:00:00 2001 From: tanyapole Date: Sun, 22 Dec 2024 01:28:11 +0300 Subject: [PATCH 3/6] add visualization tests for CNNs --- tests/test_utils.py | 16 ++++++++++++++++ tests/tests.py | 22 +++++++++++++++++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 6dc2cff..0877dc7 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -18,6 +18,10 @@ def create_testing_data(architecture='fcn'): architecture = architecture.lower() if architecture == 'fcn': return torch.randn((20, 256)) + elif architecture == 'cnn': + return torch.randn((20, 3, 32, 32)) + else: + raise Exception(f'Unsupported architecture type: {architecture}') def create_testing_model(architecture='fcn', num_classes=10): @@ -32,6 +36,18 @@ def create_testing_model(architecture='fcn', num_classes=10): ], ), ) + elif architecture == 'cnn': + return nn.Sequential( + OrderedDict( + [ + ("first_layer", nn.Conv2d(in_channels=3, out_channels=10, kernel_size=7)), + ("second_layer", nn.Conv2d(in_channels=10, out_channels=20, kernel_size=7)), + ("avgpool", nn.AdaptiveAvgPool2d(1)), + ("flatten", nn.Flatten()), + ("fc", nn.Linear(20, num_classes)), + ], + ), + ) elif architecture == 'rnn': return nn.Sequential( OrderedDict( diff --git a/tests/tests.py b/tests/tests.py index 08bd958..b5de1a1 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -31,7 +31,7 @@ def test_reduce_dim_pca(): _check_reduce_dim("pca") -def test_visualization(): +def test_visualization_fcn(): data = utils.create_testing_data() model = utils.create_testing_model() layers = ["second_layer", "third_layer"] @@ -51,6 +51,26 @@ def test_visualization(): f"Wrong value type for key {key}", ) +def test_visualization_cnn(): + data = utils.create_testing_data(architecture='cnn') + model = utils.create_testing_model(architecture='cnn') + layers = ["first_layer", "second_layer", "avgpool", "flatten", "fc"] + res = viz_api.visualize_layer_manifolds(model, "umap", data, layers=layers) + + utils.compare_values(dict, type(res), "Wrong result type") + utils.compare_values(6, len(res), "Wrong dictionary length") + utils.compare_values( + set(["input"] + layers), + set(res.keys()), + "Wrong dictionary keys", + ) + for key, plot in res.items(): + utils.compare_values( + matplotlib.figure.Figure, + type(plot), + f"Wrong value type for key {key}", + ) + def test_embed_visualization(): data = torch.randn((20, 1, 256)) From c5862ee11a21a54c1489218696bcafae5e2c1d03 Mon Sep 17 00:00:00 2001 From: tanyapole Date: Sun, 22 Dec 2024 01:28:29 +0300 Subject: [PATCH 4/6] add bayes tests for CNNs --- tests/tests.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index b5de1a1..12048cc 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -94,16 +94,16 @@ def test_embed_visualization(): ) -def _test_bayes_prediction(mode: str): +def _test_bayes_prediction(mode: str, architecture='fcn'): params = { "basic": dict(mode="basic", p=0.5), "beta": dict(mode="beta", a=0.9, b=0.2), "gauss": dict(sigma=1e-2), } - data = utils.create_testing_data() + data = utils.create_testing_data(architecture=architecture) num_classes = 17 - model = utils.create_testing_model(num_classes=num_classes) + model = utils.create_testing_model(architecture=architecture, num_classes=num_classes) n_iter = 7 if mode != 'gauss': res = bayes_api.DropoutBayesianWrapper(model, **(params[mode])).predict(data, n_iter=n_iter) @@ -130,6 +130,10 @@ def test_gauss_bayes_wrapper(): _test_bayes_prediction("gauss") +def test_bayes_wrapper_cnn(): + _test_bayes_prediction("basic", architecture='cnn') + + def test_data_barcode(): data = utils.create_testing_data() res = topology_api.get_data_barcode(data, "standard", "3") From 358bd1e82e71f49207d75d1600a3dca18acd8f58 Mon Sep 17 00:00:00 2001 From: tanyapole Date: Sun, 22 Dec 2024 01:40:16 +0300 Subject: [PATCH 5/6] add topology tests for CNNs --- tests/tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/tests.py b/tests/tests.py index 12048cc..b2ddf97 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -156,6 +156,22 @@ def test_nn_barcodes(): ) +def test_nn_barcodes_cnn(): + data = utils.create_testing_data(architecture='cnn') + model = utils.create_testing_model(architecture='cnn') + layers = ["second_layer", "flatten"] + res = topology_api.get_nn_barcodes(model, data, layers, "standard", "3") + utils.compare_values(dict, type(res), "Wrong result type") + utils.compare_values(2, len(res), "Wrong dictionary length") + utils.compare_values(set(layers), set(res.keys()), "Wrong dictionary keys") + for layer, barcode in res.items(): + utils.compare_values( + dict, + type(barcode), + f"Wrong result type for key {layer}", + ) + + def test_barcode_plot(): data = utils.create_testing_data() barcode = topology_api.get_data_barcode(data, "standard", "3") From 4438c51b72081a5ad1f4f4d9e035775d5e02b5fb Mon Sep 17 00:00:00 2001 From: tanyapole Date: Sun, 22 Dec 2024 02:02:54 +0300 Subject: [PATCH 6/6] fix style --- tests/test_utils.py | 20 ++++++++++---------- tests/tests.py | 6 ++++-- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/tests/test_utils.py b/tests/test_utils.py index 0877dc7..e6b29c5 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -50,15 +50,15 @@ def create_testing_model(architecture='fcn', num_classes=10): ) elif architecture == 'rnn': return nn.Sequential( - OrderedDict( - [ - ('first_layer', nn.LSTM(256, 128, 1, batch_first=True)), - ('extract', ExtractTensor()), - ('second_layer', nn.Linear(128, 64)), - ('third_layer', nn.Linear(64, num_classes)), - ], - ), - ) + OrderedDict( + [ + ('first_layer', nn.LSTM(256, 128, 1, batch_first=True)), + ('extract', ExtractTensor()), + ('second_layer', nn.Linear(128, 64)), + ('third_layer', nn.Linear(64, num_classes)), + ], + ), + ) else: raise Exception(f'Unsupported architecture type: {architecture}') @@ -67,4 +67,4 @@ class ExtractTensor(nn.Module): def forward(self, x): tensor, _ = x x = x.to(torch.float32) - return tensor[:, :] \ No newline at end of file + return tensor[:, :] diff --git a/tests/tests.py b/tests/tests.py index b2ddf97..746582e 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -51,6 +51,7 @@ def test_visualization_fcn(): f"Wrong value type for key {key}", ) + def test_visualization_cnn(): data = utils.create_testing_data(architecture='cnn') model = utils.create_testing_model(architecture='cnn') @@ -114,8 +115,9 @@ def _test_bayes_prediction(mode: str, architecture='fcn'): utils.compare_values(dict, type(res), "Wrong result type") utils.compare_values(2, len(res), "Wrong dictionary length") utils.compare_values(set(["mean", "std"]), set(res.keys()), "Wrong dictionary keys") - utils.compare_values(torch.Size([len(data), num_classes]), res["mean"].shape, "Wrong mean shape") - utils.compare_values(torch.Size([len(data), num_classes]), res["std"].shape, "Wrong mean std") + N = len(data) + utils.compare_values(torch.Size([N, num_classes]), res["mean"].shape, "Wrong mean shape") + utils.compare_values(torch.Size([N, num_classes]), res["std"].shape, "Wrong mean std") def test_basic_bayes_wrapper():