diff --git a/core/testcasecontroller/algorithm/paradigm/federated_learning/federated_class_incremental_learning.py b/core/testcasecontroller/algorithm/paradigm/federated_learning/federated_class_incremental_learning.py index 6fad1f0e..08408da1 100644 --- a/core/testcasecontroller/algorithm/paradigm/federated_learning/federated_class_incremental_learning.py +++ b/core/testcasecontroller/algorithm/paradigm/federated_learning/federated_class_incremental_learning.py @@ -269,33 +269,27 @@ def evaluation(self, testdataset_files, incremental_round): [testdataset_files[index]["y"][data_index]], res ) acc_list.append(acc) - if index == len(testdataset_files) - 1: - self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value][ - "accuracy" - ] = np.mean(acc_list) old_class_acc_list.extend(acc_list) current_forget_rate = 0.0 max_acc_sum = 0 self.accuracy_per_round.append(old_class_acc_list) + self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value]["accuracy"] = ( + np.mean(old_class_acc_list) + ) # caculate the forget rate for i in range(len(old_class_acc_list)): max_acc_diff = 0 for j in range(incremental_round): acc_per_round = self.accuracy_per_round[j] if i < len(acc_per_round): - LOGGER.info( - f"acc_per_round: {acc_per_round[i]}" - + f" and diff is {acc_per_round[i] - old_class_acc_list[i]}" - ) max_acc_diff = max( max_acc_diff, acc_per_round[i] - old_class_acc_list[i] ) max_acc_sum += max_acc_diff - LOGGER.info(f"max_acc_diff: {max_acc_diff}") current_forget_rate = ( max_acc_sum / len(old_class_acc_list) if incremental_round > 0 else 0.0 ) LOGGER.info( - f"for current round: {incremental_round} forget rate: {current_forget_rate}" + f"for current round: {incremental_round} forget rate: {current_forget_rate} task avg acc: {self.system_metric_info[SystemMetricType.TASK_AVG_ACC.value]['accuracy']}" ) return current_forget_rate diff --git a/core/testenvmanager/testenv/testenv.py b/core/testenvmanager/testenv/testenv.py index 27d665bd..30952a3d 100644 --- a/core/testenvmanager/testenv/testenv.py +++ b/core/testenvmanager/testenv/testenv.py @@ -39,12 +39,13 @@ def __init__(self, config): "url": "", }, "threshold": 0.9, - "operator": ">" + "operator": ">", } self.metrics = [] self.incremental_rounds = 2 self.task_size = 1 self.round = 1 + self.client_number = 1 self.dataset = None self._parse_config(config) @@ -53,8 +54,10 @@ def _check_fields(self): raise ValueError(f"not found testenv metrics({self.metrics}).") if not isinstance(self.incremental_rounds, int) or self.incremental_rounds < 2: - raise ValueError(f"testenv incremental_rounds(value={self.incremental_rounds})" - f" must be int type and not less than 2.") + raise ValueError( + f"testenv incremental_rounds(value={self.incremental_rounds})" + f" must be int type and not less than 2." + ) def _parse_config(self, config): config_dict = config[str.lower(TestEnv.__name__)] @@ -69,7 +72,7 @@ def _parse_config(self, config): self._check_fields() def prepare(self): - """ prepare env""" + """prepare env""" try: self.dataset.process_dataset() except Exception as err: diff --git a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/FedCiMatch.py b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/FedCiMatch.py index 4992b9f6..2b4815b4 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/FedCiMatch.py +++ b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/FedCiMatch.py @@ -140,6 +140,7 @@ def before_train(self, task_id, round, train_data, task_size): if self.task_size is None: self.task_size = task_size is_new_task = task_id != self.old_task_id + self.is_new_task = is_new_task if is_new_task: self.best_old_model = ( (self.feature_extractor, self.classifier) @@ -152,7 +153,7 @@ def before_train(self, task_id, round, train_data, task_size): logging.info(f"num_classes: {self.num_classes}") if self.current_classes is not None: self.last_classes = self.current_classes - self.build_classifier() + # self.build_classifier() self.current_classes = np.unique(train_data["label_y"]).tolist() logging.info(f"current_classes: {self.current_classes}") @@ -196,10 +197,10 @@ def get_train_loader(self): logging.info( f"train_x shape: {train_x.shape} and train_y shape: {train_y.shape}" ) - logging.info( - f"unlabel_x shape: {self.unlabeled_train_set[0].shape} and unlabel_y shape: {self.unlabeled_train_set[1].shape}" - ) + logging.info( + f"train_x shape: {train_x.shape} and train_y shape: {train_y.shape}" + ) label_data_loader = self.data_preprocessor.preprocess_labeled_dataset( train_x, train_y, self.batch_size ) @@ -210,6 +211,9 @@ def get_train_loader(self): self.unlabeled_train_set[1], self.batch_size, ) + logging.info( + f"unlabel_x shape: {self.unlabeled_train_set[0].shape} and unlabel_y shape: {self.unlabeled_train_set[1].shape}" + ) return label_data_loader, unlabel_data_loader def build_exemplar(self): @@ -243,15 +247,25 @@ def construct_exemplar_set(self, images, class_id, m): exemplar_data = [] exemplar_label = [] class_mean, fe_ouput = self.compute_exemplar_mean(images) - now_class_mean = np.zeros((1, fe_ouput.shape[1])) - for i in range(m): - x = class_mean - (now_class_mean + fe_ouput) / (i + 1) - x = np.linalg.norm(x) - index = np.argmin(x) - now_class_mean += fe_ouput[index] - exemplar_data.append(images[index]) - exemplar_label.append(class_id) + # cl_mean = tf.reduce_mean(class_mean, axis=0) + # now_class_mean = np.zeros((1, fe_ouput.shape[1])) + diff = tf.abs(fe_ouput - class_mean) + distance = [float(tf.reduce_sum(dis).numpy()) for dis in diff] + + sorted_index = np.argsort(distance).tolist() + if len(sorted_index) > m: + sorted_index = sorted_index[:m] + exemplar_data = [images[i] for i in sorted_index] + exemplar_label = [class_id] * len(exemplar_data) self.exemplar_set.append((exemplar_data, exemplar_label)) + # for i in range(m): + # x = class_mean - (now_class_mean + fe_ouput) / (i + 1) + # x = np.linalg.norm(x) + # index = np.argmin(x) + # now_class_mean += fe_ouput[index] + # exemplar_data.append(images[index]) + # exemplar_label.append(class_id) + # self.exemplar_set.append((exemplar_data, exemplar_label)) def compute_exemplar_mean(self, images): images_data = ( @@ -261,24 +275,28 @@ def compute_exemplar_mean(self, images): ) fe_output = self.feature_extractor.predict(images_data) print("fe_output shape:", fe_output.shape) - fe_output = tf.nn.l2_normalize(fe_output).numpy() - class_mean = np.mean(fe_output, axis=0) + class_mean = tf.reduce_mean(fe_output, axis=0) + # class_mean = np.mean(fe_output, axis=0) return class_mean, fe_output def train(self, round): - + # optimizer = keras.optimizers.SGD( + # learning_rate=self.learning_rate, momentum=0.9, weight_decay=0.0001 + # ) optimizer = keras.optimizers.Adam( - learning_rate=self.learning_rate, weight_decay=0.00001 + learning_rate=self.learning_rate, weight_decay=0.0001 ) - feature_extractor_params = self.feature_extractor.trainable_variables - classifier_params = self.classifier.trainable_variables + q = [] + logging.info(f"is new task: {self.is_new_task}") + # if self.classifier is not None: + # q = self.caculate_pre_update() + if self.is_new_task: + self.build_classifier() + # self.is_new_task = False all_params = [] - all_params.extend(feature_extractor_params) - all_params.extend(classifier_params) - # all_params = [] - # all_params.extend(self.feature_extractor.trainable_variables) - # all_params.extend(self.classifier.trainable_variables) - q = self.caculate_pre_update() + all_params.extend(self.feature_extractor.trainable_variables) + all_params.extend(self.classifier.trainable_variables) + for epoch in range(self.epochs): # for (labeled_data, unlabeled_data) in zip(self.labeled_train_loader, self.unlabeled_train_loader): for step, (labeled_x, labeled_y) in enumerate(self.labeled_train_loader): @@ -295,13 +313,16 @@ def train(self, round): correct = tf.reduce_sum( tf.cast(tf.equal(label_pred, labeled_y), dtype=tf.int32) ) - loss = self.supervised_loss(labeled_x, labeled_y, q, step) + CE_loss = self.supervised_loss(labeled_x, labeled_y) + KD_loss = self.distil_loss(labeled_x, labeled_y, q, step) # loss = tf.reduce_mean(keras.losses.sparse_categorical_crossentropy(labeled_y, y_pred, from_logits=True)) # if round > self.warm_up_round: # unsupervised_loss = self.unsupervised_loss(weak_unlabeled_x, strong_unlabeled_x, unlabeled_x) # loss = 0.5 * supervised_loss + 0.5 * unsupervised_loss + loss = CE_loss if KD_loss == 0 else CE_loss + 0.4 * KD_loss + loss = CE_loss logging.info( - f"epoch {epoch} step {step} loss: {loss} correct {correct} and total {labeled_x.shape[0]}" + f"epoch {epoch} step {step} loss: {loss} correct {correct} and total {labeled_x.shape[0]} class is {np.unique(labeled_y)}" ) grads = tape.gradient(loss, all_params) optimizer.apply_gradients(zip(grads, all_params)) @@ -316,7 +337,7 @@ def caculate_pre_update(self): logging.info(f"q shape: {len(q)}") return q - def supervised_loss(self, x, y, q, step): + def supervised_loss(self, x, y): input = x input = self.feature_extractor(input, training=True) y_pred = self.classifier(input, training=True) @@ -324,35 +345,43 @@ def supervised_loss(self, x, y, q, step): loss = keras.losses.categorical_crossentropy(target, y_pred, from_logits=True) logging.info(f"loss shape: {loss.shape}") loss = tf.reduce_mean(loss) - KD_loss = self.distil_loss(x, y, q, step) - return loss + KD_loss + logging.info(f"CE loss: {loss}") + + return loss def distil_loss(self, x, y, q, step): KD_loss = 0 + if len(self.learned_classes) > 0 and self.best_old_model is not None: g = self.feature_extractor(x, training=True) g = self.classifier(g, training=True) og = self.best_old_model[0](x, training=False) og = self.best_old_model[1](og, training=False) - logging.info(f"og shape: {og.shape} g shape: {g.shape}") - og = tf.nn.sigmoid(og) - g = tf.nn.sigmoid(g) - distil_target = tf.Variable(g) - distil_target[:, : og.shape[1]].assign(og) - q_i = q[step] - are_equal = tf.reduce_all(tf.equal(q_i, g)) - logging.info( - f"q_i shape: {q_i.shape} g shape: {g.shape} are equal: {are_equal.numpy()}" + # logging.info(f"og shape: {og.shape} g shape: {g.shape}") + sigmoid_og = tf.nn.sigmoid(og) + sigmoid_g = tf.nn.sigmoid(g) + softmax_og = tf.nn.softmax(og) + softmax_g = tf.nn.softmax(g) + kl_loss = keras.losses.kl_divergence( + softmax_og, softmax_g[:, : og.shape[1]] ) + # distil_target = tf.Variable(g) + # distil_target[:, : og.shape[1]].assign(og) + # q_i = q[step] + # are_equal = tf.reduce_all(tf.equal(q_i, g)) + # logging.info( + # f"q_i shape: {q_i.shape} g shape: {g.shape} are equal: {are_equal.numpy()}" + # ) BCELoss = keras.losses.BinaryCrossentropy() - test_distill_loss = BCELoss(distil_target, g) - logging.info(f"test loss {test_distill_loss}") + # test_distill_loss = BCELoss(og, g[:, : og.shape[1]]) + # logging.info(f"test loss {kl_loss.numpy()}") loss = [] for y in self.learned_classes: if y not in self.current_classes: - loss.append(BCELoss(q_i[:, y], g[:, y])) - # logging.info(f"test loss: {len(loss)}") + loss.append(BCELoss(sigmoid_og[:, y], sigmoid_g[:, y])) + # logging.info(f"test loss: {len(loss)} {loss}") KD_loss = tf.reduce_sum(loss) + # KD_loss = tf.reduce_mean(kl_loss) logging.info(f"KD_loss: {KD_loss}") return KD_loss @@ -377,8 +406,8 @@ def predict(self, x): prob = tf.nn.softmax(pred, axis=1) pred = tf.argmax(prob, axis=1) pred = tf.cast(pred, dtype=tf.int32) - return pred - + return pred + def icarl_predict(self, x): mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) diff --git a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/algorithm.yaml b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/algorithm.yaml index 80454a10..52133a13 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/algorithm.yaml +++ b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/algorithm.yaml @@ -9,18 +9,18 @@ algorithm: modules: - type: "basemodel" - name: "fci_ssl" + name: "FedCILMatch" url: "./examples/cifar100/fci_ssl/fed_ci_match/algorithm/basemodel.py" hyperparameters: - batch_size: values: - - 128 + - 64 - learning_rate: values: - 0.001 - epochs: values: - - 1 + - 24 - type: "aggregation" name: "FedAvg" url: "./examples/cifar100/fci_ssl/fed_ci_match/algorithm/aggregation.py" diff --git a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/basemodel.py b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/basemodel.py index 8539b766..c9a4805b 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/basemodel.py +++ b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/basemodel.py @@ -11,7 +11,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import sys +sys.path.append(".") +sys.path.append("..") import os import numpy as np import keras @@ -26,17 +29,17 @@ logging.getLogger().setLevel(logging.INFO) -@ClassFactory.register(ClassType.GENERAL, alias="fci_ssl") +@ClassFactory.register(ClassType.GENERAL, alias="FedCILMatch") class BaseModel: def __init__(self, **kwargs) -> None: self.kwargs = kwargs self.learning_rate = kwargs.get("learning_rate", 0.001) self.epochs = kwargs.get("epochs", 1) self.batch_size = kwargs.get("batch_size", 32) - self.task_size = kwargs.get("task_size", 10) + self.task_size = kwargs.get("task_size", 2) self.memory_size = kwargs.get("memory_size", 2000) # self.fe = self.build_feature_extractor() - self.num_classes = 10 # the number of class for the first task + self.num_classes = 50 # the number of class for the first task self.FedCiMatch = FedCiMatch( self.num_classes, self.batch_size, @@ -71,7 +74,7 @@ def predict(self, data_files, **kwargs): for data in data_files: x = np.load(data) logging.info(f"predicting {x.shape}") - res = self.FedCiMatch.icarl_predict(x) + res = self.FedCiMatch.predict(x) # pred = tf.cast(tf.argmax(logits, axis=1), tf.int32) result[data] = res.numpy() print("finish predict") diff --git a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/data_prepocessor.py b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/data_prepocessor.py index 37a8ee18..24f7a5dc 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/data_prepocessor.py +++ b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/data_prepocessor.py @@ -16,41 +16,53 @@ import numpy as np from agumentation import Base_Augment + class Dataset_Preprocessor: - def __init__(self, - dataset_name:str, - weak_augment_helper:Base_Augment, - strong_augment_helper:Base_Augment) -> None: + def __init__( + self, + dataset_name: str, + weak_augment_helper: Base_Augment, + strong_augment_helper: Base_Augment, + ) -> None: self.weak_augment_helper = weak_augment_helper self.strong_augment_helper = strong_augment_helper self.mean = 0.0 self.std = 1.0 # 数据集统计特征 - if dataset_name == 'cifar100': + if dataset_name == "cifar100": self.mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) self.std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) print(f"mean: {self.mean}, std: {self.std}") + def preprocess_labeled_dataset(self, x, y, batch_size): # wx = self.weak_augment_helper(x) - return tf.data.Dataset.from_tensor_slices((x, y)).shuffle(100000).map( - lambda x,y:( - (tf.cast(x, dtype=tf.float32) / 255. - self.mean) / self.std, - tf.cast(y, dtype=tf.int32) + return ( + tf.data.Dataset.from_tensor_slices((x, y)) + .shuffle(100000) + .map( + lambda x, y: ( + (tf.cast(x, dtype=tf.float32) / 255.0 - self.mean) / self.std, + tf.cast(y, dtype=tf.int32), + ) ) - ).batch(batch_size) - + .batch(batch_size) + ) def preprocess_unlabeled_dataset(self, ux, uy, batch_size): # unlabeled_train_db = tf.data.Dataset.from_tensor_slices((ux, ux, ux, uy)) - + wux = self.weak_augment_helper(ux) sux = self.strong_augment_helper(ux) - return tf.data.Dataset.from_tensor_slices((ux, wux, sux, uy)).shuffle(1000).map( - lambda ux,wux,sux,uy: ( - (tf.cast(ux, dtype=tf.float32) / 255. - self.mean) / self.std, - (tf.cast(wux, dtype=tf.float32) / 255. - self.mean) / self.std, - (tf.cast(sux, dtype=tf.float32) / 255. - self.mean) / self.std, - tf.cast(uy, dtype=tf.int32) + return ( + tf.data.Dataset.from_tensor_slices((ux, wux, sux, uy)) + .shuffle(1000) + .map( + lambda ux, wux, sux, uy: ( + (tf.cast(ux, dtype=tf.float32) / 255.0 - self.mean) / self.std, + (tf.cast(wux, dtype=tf.float32) / 255.0 - self.mean) / self.std, + (tf.cast(sux, dtype=tf.float32) / 255.0 - self.mean) / self.std, + tf.cast(uy, dtype=tf.int32), + ) ) - ).batch(batch_size) - + .batch(batch_size) + ) diff --git a/examples/cifar100/fci_ssl/fed_ci_match/algorithm/test_train.py b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/test_train.py new file mode 100644 index 00000000..b36c2f3b --- /dev/null +++ b/examples/cifar100/fci_ssl/fed_ci_match/algorithm/test_train.py @@ -0,0 +1,209 @@ +import sys + +sys.path.append(".") +sys.path.append("..") +sys.path.append("./.") +sys.path.append("./..") +from sedna.datasources import TxtDataParse +import numpy as np +import tensorflow as tf +from basemodel import BaseModel +import keras +import logging + +logging.getLogger().setLevel(logging.INFO) + + +def build_classifier(feature_extractor): + classifier = keras.Sequential( + [ + # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), + keras.layers.Dense(10, kernel_initializer="lecun_normal") + ] + ) + classifier.build(input_shape=(None, feature_extractor.layers[-2].output_shape[-1])) + logging.info(f"finish ! initialize classifier {classifier.summary()}") + return classifier + + +def read_data_from_file_to_npy(files, incremental_round=10): + """ + read data from file to numpy array + + Parameters + --------- + files: list + the address url of data file. + + Returns + ------- + list + data in numpy array. + + """ + + # print(files.x, files.y) + tasks = [] + for i in range(incremental_round): + x_train = [] + y_train = [] + start = i * incremental_round + end = (i + 1) * incremental_round + print(files.x[start:end]) + for i, file in enumerate(files.x[start:end]): + x = np.load(file) + # print(x.shape) + # print((files.y[][i])) + y = np.full((x.shape[0],), (files.y[start:end][i]).astype(np.int32)) + x_train.append(x) + y_train.append(y) + x_train = np.concatenate(x_train, axis=0) + y_train = np.concatenate(y_train, axis=0) + tasks.append((x_train, y_train)) + print(x_train.shape, y_train.shape, np.unique(y_train), len(tasks)) + return tasks + + +def train( + feature_extractor, + classifier, + train_data, + valid_data=None, + epochs=60, + batch_size=128, + learning_rate=0.01, + validation_split=0.2, +): + """Model train""" + mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) + std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) + all_parameter = [] + all_parameter.extend(feature_extractor.trainable_variables) + all_parameter.extend(classifier.trainable_variables) + optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate) + train_loader = ( + tf.data.Dataset.from_tensor_slices(train_data) + .shuffle(500000) + .map( + lambda x, y: ( + (tf.cast(x, dtype=tf.float32) / 255.0 - mean) / std, + tf.cast(y, dtype=tf.int32), + ) + ) + .batch(batch_size) + ) + for epoch in range(epochs): + epoch_loss = 0 + step = 0 + for _, (x, y) in enumerate(train_loader): + with tf.GradientTape() as tape: + feature = feature_extractor(x) + logits = tf.nn.softmax(classifier(feature)) + loss = tf.reduce_mean( + keras.losses.sparse_categorical_crossentropy(y, logits) + ) + step += 1 + logging.info(f"epoch {epoch} step {step} loss: {loss}") + epoch_loss += loss + grads = tape.gradient(loss, all_parameter) + optimizer.apply_gradients(zip(grads, all_parameter)) + logging.info(f"epoch {epoch} loss: {epoch_loss/step}") + + +def evaluate(feature_extractor, classifier, test_data_x, test_data_y): + mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) + std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) + test_loader = ( + tf.data.Dataset.from_tensor_slices((test_data_x, test_data_y)) + .map( + lambda x, y: ( + (tf.cast(x, dtype=tf.float32) / 255.0 - mean) / std, + tf.cast(y, dtype=tf.int32), + ) + ) + .batch(32) + ) + acc = 0 + total_num = 0 + total_correct = 0 + for _, (x, y) in enumerate(test_loader): + # feature = feature_extractor(x) + # logits = classifier(feature) + # pred = tf.cast(tf.argmax(logits, axis=1), tf.int64) + # y = tf.cast(y, tf.int64) + # acc += tf.reduce_sum(tf.cast(tf.equal(pred, y), tf.float32)) + print(x.shape) + logits = classifier(feature_extractor(x, training=False)) + prob = tf.nn.softmax(logits, axis=1) + pred = tf.argmax(prob, axis=1) + pred = tf.cast(pred, dtype=tf.int32) + pred = tf.reshape(pred, y.shape) + # print(f"pred: {pred} y: {y}") + correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) + # print(correct) + correct = tf.reduce_sum(correct, axis=0) + + # print(f"correct: {correct} total: {x.shape[0]}") + total_num += x.shape[0] + total_correct += int(correct) + + acc = total_correct / total_num + logging.info(f"test acc: {acc}") + + +def task_to_data(task): + train_data = {} + train_data["label_x"] = task[0] + train_data["label_y"] = task[1] + # print(np.unique(train_data["label_y"])) + train_data["unlabel_x"] = [] + train_data["unlabel_y"] = [] + return train_data + + +def main(): + train_file = "/home/wyd/ianvs/project/data/cifar100/cifar100_train.txt" + train_data = TxtDataParse(data_type="train") + train_data.parse(train_file) + test_file = "/home/wyd/ianvs/project/data/cifar100/cifar100_test.txt" + test_data = TxtDataParse(data_type="eval") + test_data.parse(test_file) + # print(train_data.x, train_data.y) + # print(test_data.x, test_data.y) + incremental_round = 10 + + tasks = read_data_from_file_to_npy(train_data, incremental_round) + test_tasks = read_data_from_file_to_npy(test_data, incremental_round) + config = { + "learning_rate": 0.01, + "epochs": 10, + "batch_size": 128, + "task_size": 10, + "memory_size": 2000, + } + estimator = BaseModel(**config) + + for i, task in enumerate(tasks): + + train_data = task_to_data(task) + + estimator.train(train_data, None, task_id=i, round=i) + + test_x = [] + test_y = [] + for test_task in test_tasks[:1]: + print(test_task[0].shape, test_task[1].shape) + test_x.append(test_task[0]) + test_y.append(test_task[1]) + test_x = np.concatenate(test_x, axis=0) + test_y = np.concatenate(test_y, axis=0) + evaluate( + estimator.FedCiMatch.feature_extractor, + estimator.FedCiMatch.classifier, + test_x, + test_y, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/cifar100/fci_ssl/fed_ci_match/benchmarkingjob.yaml b/examples/cifar100/fci_ssl/fed_ci_match/benchmarkingjob.yaml index 75917662..acb1e39e 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match/benchmarkingjob.yaml +++ b/examples/cifar100/fci_ssl/fed_ci_match/benchmarkingjob.yaml @@ -25,7 +25,7 @@ benchmarkingjob: rank: # rank leaderboard with metric of test case's evaluation and order ; list type; # the sorting priority is based on the sequence of metrics in the list from front to back; - sort_by: [ { "accuracy": "descend" } ] + sort_by: [ { "task_avg_acc": "descend" } ] # visualization configuration visualization: @@ -57,7 +57,7 @@ benchmarkingjob: # currently the options of value are as follows: # 1> "all": select all metrics in the leaderboard; # 2> metrics in the leaderboard, e.g., "F1_SCORE" - metrics: [ "accuracy", "forget_rate" ] + metrics: [ "task_avg_acc", "forget_rate" ] # network of save selected and all dataitems in workspace `./rank` ; string type; # currently the options of value are as follows: diff --git a/examples/cifar100/fci_ssl/fed_ci_match/test_train.py b/examples/cifar100/fci_ssl/fed_ci_match/test_train.py deleted file mode 100644 index 7317f28a..00000000 --- a/examples/cifar100/fci_ssl/fed_ci_match/test_train.py +++ /dev/null @@ -1,152 +0,0 @@ -from sedna.datasources import TxtDataParse -import numpy as np -import tensorflow as tf -from algorithm.model import resnet10 -import keras -import logging -logging.getLogger().setLevel(logging.INFO) -def build_classifier(feature_extractor): - classifier = keras.Sequential([ - # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), - keras.layers.Dense(10, kernel_initializer='lecun_normal') - ]) - classifier.build(input_shape=(None, feature_extractor.layers[-2].output_shape[-1])) - logging.info(f"finish ! initialize classifier {classifier.summary()}") - return classifier -def read_data_from_file_to_npy( files): - """ - read data from file to numpy array - - Parameters - --------- - files: list - the address url of data file. - - Returns - ------- - list - data in numpy array. - - """ - x_train = [] - y_train = [] - print(files.x, files.y) - for i, file in enumerate(files.x[:10]): - x = np.load(file) - # print(x.shape) - # print(type(files.y[i])) - y = np.full((x.shape[0], 1), (files.y[i]).astype(np.int32)) - x_train.append(x) - y_train.append(y) - x_train = np.concatenate(x_train, axis=0) - y_train = np.concatenate(y_train, axis=0) - print(x_train.shape, y_train.shape) - return x_train, y_train - -def train(feature_extractor, classifier, - train_data, valid_data=None, - epochs=60, - batch_size=128, - learning_rate=0.01, - validation_split=0.2): - """ Model train """ - mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) - std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) - all_parameter = [] - all_parameter.extend(feature_extractor.trainable_variables) - all_parameter.extend(classifier.trainable_variables) - optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate) - train_loader = tf.data.Dataset.from_tensor_slices(train_data).shuffle(500000).map( - lambda x,y:( - (tf.cast(x, dtype=tf.float32) / 255. - mean) / std, - tf.cast(y, dtype=tf.int32) - ) - ).batch(batch_size) - for epoch in range(epochs): - epoch_loss = 0 - step = 0 - for _, (x, y) in enumerate(train_loader): - with tf.GradientTape() as tape: - feature = feature_extractor(x) - logits = tf.nn.softmax(classifier(feature)) - loss = tf.reduce_mean(keras.losses.sparse_categorical_crossentropy(y , logits)) - step += 1 - logging.info(f"epoch {epoch} step {step} loss: {loss}") - epoch_loss += loss - grads = tape.gradient(loss, all_parameter) - optimizer.apply_gradients(zip(grads, all_parameter)) - logging.info(f"epoch {epoch} loss: {epoch_loss/step}") - -def evaluate(feature_extractor, classifier, test_data_x, test_data_y): - mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) - std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) - test_loader = tf.data.Dataset.from_tensor_slices((test_data_x, test_data_y)).map( - lambda x,y:( - (tf.cast(x, dtype=tf.float32) / 255. - mean) / std, - tf.cast(y, dtype=tf.int32) - ) - ).batch(32) - acc = 0 - total_num = 0 - total_correct = 0 - for _, (x, y) in enumerate(test_loader): - # feature = feature_extractor(x) - # logits = classifier(feature) - # pred = tf.cast(tf.argmax(logits, axis=1), tf.int64) - # y = tf.cast(y, tf.int64) - # acc += tf.reduce_sum(tf.cast(tf.equal(pred, y), tf.float32)) - logits = classifier(feature_extractor(x, training=False)) - prob = tf.nn.softmax(logits, axis=1) - pred = tf.argmax(prob, axis=1) - pred = tf.cast(pred, dtype=tf.int32) - pred = tf.reshape(pred, y.shape) - # print(f"pred: {pred} y: {y}") - correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) - # print(correct) - correct = tf.reduce_sum(correct, axis=0) - - # print(f"correct: {correct} total: {x.shape[0]}") - total_num += x.shape[0] - total_correct += int(correct) - - acc = total_correct / total_num - logging.info(f"test acc: {acc}") - -def main(): - train_file = '/home/wyd/ianvs/project/data/cifar100/cifar100_train.txt' - train_data = TxtDataParse(data_type='train') - train_data.parse(train_file) - train_data = read_data_from_file_to_npy(train_data) - train_data_x = train_data[0] - train_data_y = train_data[1] - print(len(train_data_x), len(train_data_y)) - test_file = '/home/wyd/ianvs/project/data/cifar100/cifar100_test.txt' - test_data = TxtDataParse(data_type='eval') - test_data.parse(test_file) - test_data = read_data_from_file_to_npy(test_data) - test_data_x = test_data[0] - test_data_y = test_data[1] - feature_extractor = resnet10() - feature_extractor.build((None, 32,32,3)) - feature_extractor.call(keras.Input(shape=(32,32,3))) - classifier = build_classifier(feature_extractor) - train(feature_extractor, classifier, train_data) - evaluate(feature_extractor, classifier, test_data_x, test_data_y) - -import time -import threading -def local_thread(t): - time.sleep(3) - return t - -def local_thread_v2(t): - val = local_thread(t) - print(val, t) -if __name__ == '__main__': - a = [] - for i in range(3): - t = threading.Thread(target=local_thread_v2, args=(i,)) - t.start() - a.append(t) - for t in a: - t.join() \ No newline at end of file diff --git a/examples/cifar100/fci_ssl/fed_ci_match/testenv/testenv.yaml b/examples/cifar100/fci_ssl/fed_ci_match/testenv/testenv.yaml index 08c9937a..9c567b11 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match/testenv/testenv.yaml +++ b/examples/cifar100/fci_ssl/fed_ci_match/testenv/testenv.yaml @@ -31,6 +31,7 @@ testenv: # the url address of python file url: "/home/wyd/ianvs/project/ianvs/examples/cifar100/fci_ssl/fed_ci_match/testenv/acc.py" - name: "forget_rate" + - name: "task_avg_acc" # incremental rounds setting of incremental learning; int type; default value is 2; - incremental_rounds: 10 + incremental_rounds: 2 round: 1 \ No newline at end of file diff --git a/examples/cifar100/fci_ssl/fed_ci_match_v2/testenv/testenv.yaml b/examples/cifar100/fci_ssl/fed_ci_match_v2/testenv/testenv.yaml index 2091bb43..bb906c28 100644 --- a/examples/cifar100/fci_ssl/fed_ci_match_v2/testenv/testenv.yaml +++ b/examples/cifar100/fci_ssl/fed_ci_match_v2/testenv/testenv.yaml @@ -33,4 +33,5 @@ testenv: - name: "forget_rate" # incremental rounds setting of incremental learning; int type; default value is 2; incremental_rounds: 50 - round: 1 \ No newline at end of file + round: 1 + client_number: 2 \ No newline at end of file diff --git a/examples/cifar100/fci_ssl/fedavg/algorithm/aggregation.py b/examples/cifar100/fci_ssl/fedavg/algorithm/aggregation.py index 99333d66..7f678962 100644 --- a/examples/cifar100/fci_ssl/fedavg/algorithm/aggregation.py +++ b/examples/cifar100/fci_ssl/fedavg/algorithm/aggregation.py @@ -7,15 +7,17 @@ from sedna.algorithms.aggregation.aggregation import BaseAggregation from sedna.common.class_factory import ClassType, ClassFactory + @ClassFactory.register(ClassType.FL_AGG, "FedAvg") class FedAvg(BaseAggregation, abc.ABC): def __init__(self): super(FedAvg, self).__init__() + """ Federated averaging algorithm """ - def aggregate(self, clients:List): + def aggregate(self, clients: List): """ Calculate the average weight according to the number of samples @@ -30,20 +32,17 @@ def aggregate(self, clients:List): final weights use to update model layer """ - print("aggregation....") if not len(clients): return self.weights self.total_size = sum([c.num_samples for c in clients]) # print(next(iter(clients)).weights) - old_weight = [np.zeros(np.array(c).shape) for c in - next(iter(clients)).weights] + old_weight = [np.zeros(np.array(c).shape) for c in next(iter(clients)).weights] updates = [] for inx, row in enumerate(old_weight): for c in clients: - row += (np.array(c.weights[inx]) * c.num_samples - / self.total_size) + row += np.array(c.weights[inx]) * c.num_samples / self.total_size updates.append(row.tolist()) self.weights = deepcopy(updates) print("finish aggregation....") - return updates + return [np.array(layer) for layer in updates] diff --git a/examples/cifar100/fci_ssl/fedavg/algorithm/algorithm.yaml b/examples/cifar100/fci_ssl/fedavg/algorithm/algorithm.yaml index 427a19f9..6051e0f0 100644 --- a/examples/cifar100/fci_ssl/fedavg/algorithm/algorithm.yaml +++ b/examples/cifar100/fci_ssl/fedavg/algorithm/algorithm.yaml @@ -27,7 +27,7 @@ algorithm: - type: "basemodel" # name of python module; string type; # example: basemodel.py has BaseModel module that the alias is "FPN" for this benchmarking; - name: "fcil" + name: "fedavg" # the url address of python module; string type; url: "./examples/cifar100/fci_ssl/fedavg/algorithm/basemodel.py" @@ -36,7 +36,7 @@ algorithm: # name of the hyperparameter; string type; - batch_size: values: - - 32 + - 64 - learning_rate: values: - 0.001 diff --git a/examples/cifar100/fci_ssl/fedavg/algorithm/basemodel.py b/examples/cifar100/fci_ssl/fedavg/algorithm/basemodel.py index 74896252..6d21173e 100644 --- a/examples/cifar100/fci_ssl/fedavg/algorithm/basemodel.py +++ b/examples/cifar100/fci_ssl/fedavg/algorithm/basemodel.py @@ -7,58 +7,62 @@ from keras import Sequential from keras.src.layers import Conv2D, MaxPooling2D, Flatten, Dropout, Dense from sedna.common.class_factory import ClassType, ClassFactory -from resnet import resnet10 -from network import NetWork, incremental_learning +from model import resnet10 + __all__ = ["BaseModel"] -os.environ['BACKEND_TYPE'] = 'KERAS' +os.environ["BACKEND_TYPE"] = "KERAS" logging.getLogger().setLevel(logging.INFO) -@ClassFactory.register(ClassType.GENERAL, alias='fcil') +@ClassFactory.register(ClassType.GENERAL, alias="fedavg") class BaseModel: def __init__(self, **kwargs): self.kwargs = kwargs print(f"kwargs: {kwargs}") - self.batch_size = kwargs.get('batch_size', 1) + self.batch_size = kwargs.get("batch_size", 1) print(f"batch_size: {self.batch_size}") - self.epochs = kwargs.get('epochs', 1) - self.lr = kwargs.get('lr', 0.001) - self.optimizer = keras.optimizers.SGD(learning_rate=self.lr) + self.epochs = kwargs.get("epochs", 1) + self.learning_rate = kwargs.get("learning_rate", 0.001) + self.num_classes = 50 + self.task_size = 50 self.old_task_id = -1 - self.fe = resnet10(10) + self.mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) + self.std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) + self.fe = resnet10() logging.info(type(self.fe)) - self.model = NetWork(100, self.fe) + self.classifier = None self._init_model() - - def _init_model(self): - self.model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) - x = np.random.rand(1, 32, 32, 3) - y = np.random.randint(0, 10, 1) - - self.model.fit(x, y, epochs=1) + self.fe.compile( + optimizer="sgd", + loss="sparse_categorical_crossentropy", + metrics=["accuracy"], + ) + self.fe.call(keras.Input(shape=(32, 32, 3))) + fe_weights = self.fe.get_weights() + self.fe_weights_length = len(fe_weights) def load(self, model_url=None): logging.info(f"load model from {model_url}") - extra_model_path = os.path.basename(model_url) + "/model" - with zipfile.ZipFile(model_url, 'r') as zip_ref: - zip_ref.extractall(extra_model_path) - self.model = tf.saved_model.load(extra_model_path) def _initialize(self): logging.info(f"initialize finished") def get_weights(self): - logging.info(f"get_weights") - weights = [layer.tolist() for layer in self.model.get_weights()] - logging.info(len(weights)) + weights = [] + fe_weights = self.fe.get_weights() + self.fe_weights_length = len(fe_weights) + clf_weights = self.classifier.get_weights() + weights.extend(fe_weights) + weights.extend(clf_weights) return weights def set_weights(self, weights): - weights = [np.array(layer) for layer in weights] - self.model.set_weights(weights) - logging.info("----------finish set weights-------------") + fe_weights = weights[: self.fe_weights_length] + clf_weights = weights[self.fe_weights_length :] + self.fe.set_weights(fe_weights) + self.classifier.set_weights(clf_weights) def save(self, model_path=""): logging.info("save model") @@ -67,45 +71,106 @@ def model_info(self, model_path, result, relpath): logging.info("model info") return {} - - - def train(self, train_data, valid_data, **kwargs): + def build_classifier(self): + if self.classifier != None: + new_classifier = keras.Sequential( + [ + # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), + keras.layers.Dense( + self.num_classes, kernel_initializer="lecun_normal" + ) + ] + ) + new_classifier.build( + input_shape=(None, self.fe.layers[-2].output_shape[-1]) + ) + new_weights = new_classifier.get_weights() + old_weights = self.classifier.get_weights() + # 复制旧参数 + # weight + new_weights[0][0 : old_weights[0].shape[0], 0 : old_weights[0].shape[1]] = ( + old_weights[0] + ) + # bias + new_weights[1][0 : old_weights[1].shape[0]] = old_weights[1] + new_classifier.set_weights(new_weights) + self.classifier = new_classifier + else: + logging.info(f"input shape is {self.fe.layers[-2].output_shape[-1]}") + self.classifier = keras.Sequential( + [ + # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), + keras.layers.Dense( + self.num_classes, kernel_initializer="lecun_normal" + ) + ] + ) + self.classifier.build( + input_shape=(None, self.fe.layers[-2].output_shape[-1]) + ) + + logging.info(f"finish ! initialize classifier {self.classifier.summary()}") + + def train(self, train_data, valid_data, **kwargs): + optimizer = keras.optimizers.SGD( + learning_rate=self.learning_rate, momentum=0.9, weight_decay=0.0001 + ) round = kwargs.get("round", -1) task_id = kwargs.get("task_id", -1) task_size = kwargs.get("task_size", 10) - self.model.compile(optimizer=self.optimizer, loss='sparse_categorical_crossentropy', metrics=['accuracy']) - logging.info(f"train data: {train_data[0].shape} {train_data[1].shape}") - train_db = self.data_process(train_data) + if self.old_task_id != task_id: + self.old_task_id = task_id + self.num_classes = self.task_size * (task_id + 1) + self.build_classifier() + # logging.info(f"train data: {train_data[0].shape} {train_data[1].shape}") + data = (train_data["label_x"], train_data["label_y"]) + train_db = self.data_process(data) logging.info(train_db) + all_params = [] + all_params.extend(self.fe.trainable_variables) + all_params.extend(self.classifier.trainable_variables) for epoch in range(self.epochs): total_loss = 0 total_num = 0 logging.info(f"Epoch {epoch + 1} / {self.epochs}") logging.info("-" * 50) - for x, y in train_db: + for x, y in train_db: # self.model.fit(x, y, batch_size=self.batch_size) with tf.GradientTape() as tape: - logits = self.model(x, training=True) - loss = tf.reduce_mean(keras.losses.sparse_categorical_crossentropy(y, logits, from_logits=True)) - grads = tape.gradient(loss, self.model.trainable_variables) - self.optimizer.apply(grads, self.model.trainable_variables) + logits = self.classifier(self.fe(x, training=True), training=True) + loss = tf.reduce_mean( + keras.losses.sparse_categorical_crossentropy( + y, logits, from_logits=True + ) + ) + grads = tape.gradient(loss, all_params) + optimizer.apply(grads, all_params) # self.optimizer.apply_gradients(zip(grads, self.model.trainable_variables)) total_loss += loss total_num += 1 - logging.info(f"train round {round}: Epoch {epoch + 1} avg loss: {total_loss / total_num}") + logging.info( + f"train round {round}: Epoch {epoch + 1} avg loss: {total_loss / total_num}" + ) logging.info(f"finish round {round} train") - self.eval(train_data, round) - return {"num_samples": train_data[0].shape[0]} + # self.eval(train_data, round) + return {"num_samples": data[0].shape[0]} - def predict(self, data, **kwargs): + def predict(self, data_files, **kwargs): result = {} - for data in data.x: + for data in data_files: x = np.load(data) - logits = self.model(x, training=False) - pred = tf.cast(tf.argmax(logits, axis=1), tf.int32) + logging.info(f"predicting {x.shape}") + mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) + std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) + x = (tf.cast(x, dtype=tf.float32) / 255.0 - mean) / std + pred = self.classifier(self.fe(x, training=False)) + prob = tf.nn.softmax(pred, axis=1) + pred = tf.argmax(prob, axis=1) + pred = tf.cast(pred, dtype=tf.int32) + # pred = tf.cast(tf.argmax(logits, axis=1), tf.int32) result[data] = pred.numpy() - logging.info("finish predict") + print("finish predict") return result def eval(self, data, round, **kwargs): @@ -118,7 +183,6 @@ def eval(self, data, round, **kwargs): # prob = tf.nn.softmax(logits, axis=1) pred = tf.argmax(logits, axis=1) pred = tf.cast(pred, dtype=tf.int32) - pred = tf.reshape(pred, y.shape) # print(pred.shape, y.shape) correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) correct = tf.reduce_sum(correct) @@ -132,6 +196,16 @@ def eval(self, data, round, **kwargs): def data_process(self, data, **kwargs): - assert data is not None, "data is None" - # data[0]'shape = (50000, 32,32,3) data[1]'shape = (50000,1) - return tf.data.Dataset.from_tensor_slices((data[0], data[1])).shuffle(100000).batch(self.batch_size) + assert data is not None, "data is None" + # data[0]'shape = (50000, 32,32,3) data[1]'shape = (50000,) + return ( + tf.data.Dataset.from_tensor_slices((data[0], data[1])) + .shuffle(100000) + .map( + lambda x, y: ( + (tf.cast(x, dtype=tf.float32) / 255.0 - self.mean) / self.std, + tf.cast(y, dtype=tf.int32), + ) + ) + .batch(self.batch_size) + ) diff --git a/examples/cifar100/fci_ssl/fedavg/algorithm/model.py b/examples/cifar100/fci_ssl/fedavg/algorithm/model.py new file mode 100644 index 00000000..db7e875c --- /dev/null +++ b/examples/cifar100/fci_ssl/fedavg/algorithm/model.py @@ -0,0 +1,157 @@ +# Copyright 2021 The KubeEdge Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tensorflow as tf +import keras +# import keras +# from keras import layers, Sequential + +# 卷积块 +# Input--conv2D--BN--ReLU--conv2D--BN--ReLU--Output +# \ / +# ------------------------------ +class BasicBlock(keras.layers.Layer): + def __init__(self, filter_num, stride=1): + super(BasicBlock, self).__init__() + + self.conv1 = keras.layers.Conv2D(filter_num, (3, 3), strides=stride, padding='same') + self.bn1 = keras.layers.BatchNormalization() + self.relu = keras.layers.Activation('relu') + + self.conv2 = keras.layers.Conv2D(filter_num, (3, 3), strides=1, padding='same') + self.bn2 = keras.layers.BatchNormalization() + + if stride != 1: + self.downsample = keras.models.Sequential() + self.downsample.add(keras.layers.Conv2D(filter_num, (1, 1), strides=stride)) + else: + self.downsample = lambda x:x + + def call(self, inputs, training=None): + # [b, h, w, c] + out = self.conv1(inputs) + out = self.bn1(out,training=training) + out = self.relu(out) + + out = self.conv2(out) + out = self.bn2(out,training=training) + + identity = self.downsample(inputs) + + output = keras.layers.add([out, identity]) + output = tf.nn.relu(output) + + return output + + +# 残差神经网络 +class ResNet(keras.Model): + def __init__(self, layer_dims): # [2, 2, 2, 2] + super(ResNet, self).__init__() + self.layer_dims = layer_dims + + self.stem = keras.models.Sequential([keras.layers.Conv2D(64, (3, 3), strides=(1, 1)), + keras.layers.BatchNormalization(), + keras.layers.Activation('relu'), + keras.layers.MaxPool2D(pool_size=(2, 2), strides=(1, 1), padding='same') + ]) + + self.layer1 = self.build_resblock(64, layer_dims[0]) + self.layer2 = self.build_resblock(128, layer_dims[1], stride=2) + self.layer3 = self.build_resblock(256, layer_dims[2], stride=2) + self.layer4 = self.build_resblock(512, layer_dims[3], stride=2) + + # output: [b, 512, h, w], + self.avgpool = keras.layers.GlobalAveragePooling2D() + def call(self, inputs, training=None): + x = self.stem(inputs,training=training) + + x = self.layer1(x,training=training) + x = self.layer2(x,training=training) + x = self.layer3(x,training=training) + x = self.layer4(x,training=training) + x = self.avgpool(x) + return x + + def build_resblock(self, filter_num, blocks, stride=1): + res_blocks = keras.models.Sequential() + # may down sample + res_blocks.add(BasicBlock(filter_num, stride)) + for _ in range(1, blocks): + res_blocks.add(BasicBlock(filter_num, stride=1)) + return res_blocks + + def get_config(self): + return { + 'layer_dims': self.layer_dims, + } + + @classmethod + def from_config(cls, config): + return cls(**config) + + + +class LeNet(keras.Model): + def __init__(self, input_shape, channels=3, num_classes=10): + super(LeNet, self).__init__() + self.input_shape = input_shape + self.channels = channels + self.num_classes = num_classes + + self.conv1 = keras.layers.Conv2D(6, kernel_size=5, strides=1, activation='relu', input_shape=(input_shape, input_shape, channels)) + self.pool1 = keras.layers.MaxPool2D(pool_size=2, strides=2) + self.conv2 = keras.layers.Conv2D(16, kernel_size=5, strides=1, activation='relu') + self.pool2 = keras.layers.MaxPool2D(pool_size=2, strides=2) + self.flatten = keras.layers.Flatten() + + self.fc1 = keras.layers.Dense(120, activation='relu') + self.fc2 = keras.layers.Dense(84, activation='relu') + self.fc3 = keras.layers.Dense(num_classes, activation='softmax') + + def call(self, inputs, training=None): + x = self.conv1(inputs) + x = self.pool1(x) + x = self.conv2(x) + x = self.pool2(x) + x = self.flatten(x) + x = self.fc1(x) + x = self.fc2(x) + x = self.fc3(x) + return x + + def get_config(self): + return { + 'input_shape': self.input_shape, + 'channels': self.channels, + 'num_classes': self.num_classes + } + + @classmethod + def from_config(cls, config): + return cls(**config) + + +def lenet5(input_shape, num_classes:int): + return LeNet(input_shape, 3, num_classes) + + +def resnet10(): + return ResNet([1, 1, 1, 1]) + +def resnet18(num_classes:int): + return ResNet([2, 2, 2, 2]) + +def resnet34(num_classes:int): + return ResNet([3, 4, 6, 3]) \ No newline at end of file diff --git a/examples/cifar100/fci_ssl/fedavg/algorithm/network.py b/examples/cifar100/fci_ssl/fedavg/algorithm/network.py deleted file mode 100644 index 87c6f779..00000000 --- a/examples/cifar100/fci_ssl/fedavg/algorithm/network.py +++ /dev/null @@ -1,68 +0,0 @@ -import keras -import tensorflow as tf -import numpy as np -from keras.src.layers import Dense -from resnet import resnet10 - - -class NetWork(keras.Model): - def __init__(self, num_classes, feature_extractor): - super(NetWork, self).__init__() - self.num_classes = num_classes - self.feature = feature_extractor - self.fc = Dense(num_classes, activation='softmax') - - def call(self, inputs): - # print(type(self.feature)) - x = self.feature(inputs) - x = self.fc(x) - return x - - def feature_extractor(self, inputs): - return self.feature.predict(inputs) - - def predict(self, fea_input): - return self.fc(fea_input) - - def get_config(self): - return { - 'num_classes': self.num_classes, - 'feature_extractor': self.feature, - } - - @classmethod - def from_config(cls, config): - return cls(**config) - - - - -def incremental_learning(old_model:NetWork, num_class): - new_model = NetWork(num_class, resnet10(num_class) ) - x = np.random.rand(1, 32, 32, 3) - y = np.random.randint(0, num_class, 1) - new_model.compile(optimizer='sgd', loss='sparse_categorical_crossentropy', metrics=['accuracy']) - new_model.fit(x, y, epochs=1) - print(old_model.fc.units, new_model.fc.units) - for layer in old_model.layers: - if hasattr(new_model.feature, layer.name): - new_model.feature.__setattr__(layer.name, layer) - if num_class > old_model.fc.units: - original_use_bias = hasattr(old_model.fc, 'bias') - print("original_use_bias", original_use_bias) - init_class = old_model.fc.units - k = new_model.fc.kernel - new_model.fc.kernel.assign(tf.pad(old_model.fc.kernel, - [[0, 0], [0, num_class - init_class]])) # 假设初始类别数为10 - if original_use_bias: - new_model.fc.bias.assign(tf.pad(old_model.fc.bias, - [[0, num_class - init_class]])) - new_model.build((None, 32, 32, 3)) - return new_model - -def copy_model(model: NetWork): - cfg = model.get_config() - - copy_model = model.from_config(cfg) - return copy_model - diff --git a/examples/cifar100/fci_ssl/fedavg/benchmarkingjob.yaml b/examples/cifar100/fci_ssl/fedavg/benchmarkingjob.yaml index f287fc0c..1af09870 100644 --- a/examples/cifar100/fci_ssl/fedavg/benchmarkingjob.yaml +++ b/examples/cifar100/fci_ssl/fedavg/benchmarkingjob.yaml @@ -19,13 +19,13 @@ benchmarkingjob: - name: "fcil_test" # the url address of test algorithm configuration file; string type; # the file format supports yaml/yml - url: "./examples/cifar100/fci_ssl/fedavg/algorithm/aggregation.py" + url: "./examples/cifar100/fci_ssl/fedavg/algorithm/algorithm.yaml" # the configuration of ranking leaderboard rank: # rank leaderboard with metric of test case's evaluation and order ; list type; # the sorting priority is based on the sequence of metrics in the list from front to back; - sort_by: [ { "accuracy": "descend" } ] + sort_by: [ { "task_avg_acc": "descend" } ] # visualization configuration visualization: @@ -57,7 +57,7 @@ benchmarkingjob: # currently the options of value are as follows: # 1> "all": select all metrics in the leaderboard; # 2> metrics in the leaderboard, e.g., "F1_SCORE" - metrics: [ "accuracy" ] + metrics: ["task_avg_acc","forget_rate"] # network of save selected and all dataitems in workspace `./rank` ; string type; # currently the options of value are as follows: diff --git a/examples/cifar100/fci_ssl/fedavg/testenv/testenv.yaml b/examples/cifar100/fci_ssl/fedavg/testenv/testenv.yaml index d7747e83..74eff7c0 100644 --- a/examples/cifar100/fci_ssl/fedavg/testenv/testenv.yaml +++ b/examples/cifar100/fci_ssl/fedavg/testenv/testenv.yaml @@ -30,7 +30,9 @@ testenv: - name: "accuracy" # the url address of python file url: "/home/wyd/ianvs/project/ianvs/examples/cifar100/fci_ssl/fedavg/testenv/acc.py" - + - name: "task_avg_acc" + - name: "forget_rate" # incremental rounds setting of incremental learning; int type; default value is 2; incremental_rounds: 2 - round: 2 \ No newline at end of file + round: 1 + client_number: 1 \ No newline at end of file diff --git a/examples/cifar100/fci_ssl/glfc/algorithm/GLFC.py b/examples/cifar100/fci_ssl/glfc/algorithm/GLFC.py index f4089584..556d568b 100644 --- a/examples/cifar100/fci_ssl/glfc/algorithm/GLFC.py +++ b/examples/cifar100/fci_ssl/glfc/algorithm/GLFC.py @@ -16,10 +16,11 @@ import numpy as np import tensorflow as tf import keras -import logging +import logging from network import NetWork, incremental_learning, copy_model from model import resnet10 + def get_one_hot(target, num_classes): # print(f'in get one hot, target shape is {target.shape}') y = tf.one_hot(target, depth=num_classes) @@ -29,136 +30,174 @@ def get_one_hot(target, num_classes): # print(f'in get one hot, after tf.squeeze y shape is {y.shape}') return y + class GLFC_Client: - def __init__(self, num_classes, batch_size, task_size, memory_size, epochs, learning_rate, encode_model): + def __init__( + self, + num_classes, + batch_size, + task_size, + memory_size, + epochs, + learning_rate, + encode_model, + ): self.epochs = epochs self.learning_rate = learning_rate - + # self.model = NetWork(num_classes, feature_extractor) self.encode_model = encode_model - + self.num_classes = num_classes - logging.info(f'num_classes is {num_classes}') + logging.info(f"num_classes is {num_classes}") self.batch_size = batch_size self.task_size = task_size - - self.old_model = None - self.train_set = None - - self.exemplar_set = [] # + + self.old_model = None + self.train_set = None + + self.exemplar_set = [] # self.class_mean_set = [] self.learned_classes = [] self.learned_classes_numebr = 0 self.memory_size = memory_size - + self.old_task_id = -1 self.current_classes = None - self.last_class = None - self.train_loader = None + self.last_class = None + self.train_loader = None self.build_feature_extractor() - self.classifier = None + self.classifier = None # self._initialize_classifier() # assert self.classifier is not None self.labeled_train_set = None - self.unlabeled_train_set= None - + self.unlabeled_train_set = None + def build_feature_extractor(self): self.feature_extractor = resnet10() - self.feature_extractor.build(input_shape=(None, 32, 32, 3)) + self.feature_extractor.build(input_shape=(None, 32, 32, 3)) self.feature_extractor.call(keras.Input(shape=(32, 32, 3))) - + def _initialize_classifier(self): if self.classifier != None: - new_classifier = tf.keras.Sequential([ - # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), - tf.keras.layers.Dense(self.num_classes, kernel_initializer='lecun_normal') - ]) - new_classifier.build(input_shape=(None, self.feature_extractor.layers[-2].output_shape[-1])) + new_classifier = tf.keras.Sequential( + [ + # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), + tf.keras.layers.Dense( + self.num_classes, kernel_initializer="lecun_normal" + ) + ] + ) + new_classifier.build( + input_shape=(None, self.feature_extractor.layers[-2].output_shape[-1]) + ) new_weights = new_classifier.get_weights() old_weights = self.classifier.get_weights() # 复制旧参数 # weight - new_weights[0][0:old_weights[0].shape[0], 0:old_weights[0].shape[1]] = old_weights[0] + new_weights[0][0 : old_weights[0].shape[0], 0 : old_weights[0].shape[1]] = ( + old_weights[0] + ) # bias - new_weights[1][0:old_weights[1].shape[0]] = old_weights[1] + new_weights[1][0 : old_weights[1].shape[0]] = old_weights[1] new_classifier.set_weights(new_weights) self.classifier = new_classifier else: - logging.info(f'input shape is {self.feature_extractor.layers[-2].output_shape[-1]}') - self.classifier = tf.keras.Sequential([ - # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), - tf.keras.layers.Dense(self.num_classes, kernel_initializer='lecun_normal') - ]) - self.classifier.build(input_shape=(None, self.feature_extractor.layers[-2].output_shape[-1])) - + logging.info( + f"input shape is {self.feature_extractor.layers[-2].output_shape[-1]}" + ) + self.classifier = tf.keras.Sequential( + [ + # tf.keras.Input(shape=(None, self.feature_extractor.layers[-2].output_shape[-1])), + tf.keras.layers.Dense( + self.num_classes, kernel_initializer="lecun_normal" + ) + ] + ) + self.classifier.build( + input_shape=(None, self.feature_extractor.layers[-2].output_shape[-1]) + ) + logging.info(f"finish ! initialize classifier {self.classifier.summary()}") - + def before_train(self, task_id, train_data, class_learned, old_model): logging.info(f"------before train task_id: {task_id}------") # print(f'train data len is :{len(train_data[1])}') - self.need_update = (task_id != self.old_task_id) + self.need_update = task_id != self.old_task_id if self.need_update: self.old_task_id = task_id self.num_classes = self.task_size * (task_id + 1) - if self.current_classes is not None: + if self.current_classes is not None: self.last_class = self.current_classes - logging.info(f'self.last_class is , {self.last_class}, {self.num_classes}') + logging.info(f"self.last_class is , {self.last_class}, {self.num_classes}") self._initialize_classifier() - self.current_classes = np.unique(train_data['label_y']).tolist() + self.current_classes = np.unique(train_data["label_y"]).tolist() self.update_new_set(self.need_update) - self.labeled_train_set = (train_data['label_x'], train_data['label_y']) - self.unlabeled_train_set= (train_data['unlabel_x'], train_data['unlabel_y']) + self.labeled_train_set = (train_data["label_x"], train_data["label_y"]) + self.unlabeled_train_set = ( + train_data["unlabel_x"], + train_data["unlabel_y"], + ) if len(old_model) != 0: self.old_model = old_model[0] - self.labeled_train_set = (train_data['label_x'], train_data['label_y']) - self.unlabeled_train_set= (train_data['unlabel_x'], train_data['unlabel_y']) + self.labeled_train_set = (train_data["label_x"], train_data["label_y"]) + self.unlabeled_train_set = (train_data["unlabel_x"], train_data["unlabel_y"]) self.train_loader = self._get_train_loader(True) - logging.info(f'------finish before train task_id: {task_id} {self.current_classes}------') - + logging.info( + f"------finish before train task_id: {task_id} {self.current_classes}------" + ) + def update_new_set(self, need_update): if need_update and self.last_class is not None: # update exemplar self.learned_classes += self.last_class self.learned_classes_numebr += len(self.last_class) - m = int(self.memory_size / self.learned_classes_numebr) + m = int(self.memory_size / self.learned_classes_numebr) self._reduce_exemplar_set(m) for i in self.last_class: images = self.get_train_set_data(i) # print(f'process class {i} with {len(images)} images') self._construct_exemplar_set(images, i, m) # print(f'-------------Learned classes: {self.learned_classes} current classes :{self.current_class} last classes : {self.last_class}--------------') - + def _get_train_loader(self, mix): self.mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) self.std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) # print(self.train_set[0].shape, self.train_set[1].shape) train_x = self.labeled_train_set[0] train_y = self.labeled_train_set[1] - if mix : + if mix: for exm_set in self.exemplar_set: - logging.info(f'mix the exemplar{len(exm_set[0])}, {len(exm_set[1])}') + logging.info(f"mix the exemplar{len(exm_set[0])}, {len(exm_set[1])}") label = np.array(exm_set[1]) - label = label.reshape(-1, 1) - train_x = np.concatenate((train_x,exm_set[0]), axis=0) - train_y = np.concatenate((train_y,label), axis=0) + # label = label.reshape(-1, 1) + train_x = np.concatenate((train_x, exm_set[0]), axis=0) + train_y = np.concatenate((train_y, label), axis=0) # logging.info(f'{ train_set[0].shape}, {self.train_set[1].shape}') - return tf.data.Dataset.from_tensor_slices((train_x, train_y)).shuffle(buffer_size=10000000).batch(self.batch_size).map( - lambda x,y:( - (tf.cast(x, dtype=tf.float32) / 255. - self.mean) / self.std, - tf.cast(y, dtype=tf.int32) + return ( + tf.data.Dataset.from_tensor_slices((train_x, train_y)) + .shuffle(buffer_size=10000000) + .batch(self.batch_size) + .map( + lambda x, y: ( + (tf.cast(x, dtype=tf.float32) / 255.0 - self.mean) / self.std, + tf.cast(y, dtype=tf.int32), + ) ) ) - + def train(self, round): # self._initialize_classifier() - opt = keras.optimizers.Adam(learning_rate=self.learning_rate, weight_decay=0.00001) + opt = keras.optimizers.Adam( + learning_rate=self.learning_rate, weight_decay=0.00001 + ) # print(self.train_loader is None) feature_extractor_params = self.feature_extractor.trainable_variables classifier_params = self.classifier.trainable_variables all_params = [] all_params.extend(feature_extractor_params) all_params.extend(classifier_params) - + for epoch in range(self.epochs): for step, (x, y) in enumerate(self.train_loader): # opt = keras.optimizers.SGD(learning_rate=self.learning_rate, weight_decay=0.00001) @@ -168,43 +207,54 @@ def train(self, round): # target = get_one_hot(y, self.num_classes) # loss = tf.reduce_mean(keras.losses.sparse_categorical_crossentropy(y, y_pred, from_logits=True)) loss = self._compute_loss(x, y) - logging.info(f'------round{round} epoch{epoch} step{step} loss: {loss} and loss dim is {loss.shape}------') + logging.info( + f"------round{round} epoch{epoch} step{step} loss: {loss} and loss dim is {loss.shape}------" + ) grads = tape.gradient(loss, all_params) # # print(f'grads shape is {len(grads)} and type is {type(grads)}') opt.apply_gradients(zip(grads, all_params)) - - logging.info(f'------finish round{round} traning------') - + + logging.info(f"------finish round{round} traning------") + def model_call(self, x, training=False): - input = self.feature_extractor(inputs=x,training=training) + input = self.feature_extractor(inputs=x, training=training) # logging.info(input.shape) return self.classifier(inputs=input, training=training) - + def _compute_loss(self, imgs, labels): - logging.info(f'self.old_model is available: {self.old_model is not None}') + logging.info(f"self.old_model is available: {self.old_model is not None}") y_pred = self.model_call(imgs, training=True) target = get_one_hot(labels, self.num_classes) logits = y_pred - # prob = tf.nn.softmax(logits, axis=1) + # prob = tf.nn.softmax(logits, axis=1) pred = tf.argmax(logits, axis=1) pred = tf.cast(pred, dtype=tf.int32) pred = tf.reshape(pred, labels.shape) - + y = tf.cast(labels, dtype=tf.int32) correct = tf.cast(tf.equal(pred, y), dtype=tf.int32) correct = tf.reduce_sum(correct) - logging.info(f'current class numbers is {self.num_classes} correct is {correct} and acc is {correct/imgs.shape[0]}') - # print(f"total_correct: {total_correct}, total_num: {total_num}") + logging.info( + f"current class numbers is {self.num_classes} correct is {correct} and acc is {correct/imgs.shape[0]}" + ) + # print(f"total_correct: {total_correct}, total_num: {total_num}") if self.old_model == None: - w = self.efficient_old_class_weight(target, labels ) + w = self.efficient_old_class_weight(target, labels) # print(f"old class weight shape: {w.shape}") - loss = tf.reduce_mean(keras.losses.categorical_crossentropy(target, y_pred, from_logits=True) * w) - # print(f'in _compute_loss, without old model loss is {loss} and shape is {loss.shape}') + loss = tf.reduce_mean( + keras.losses.categorical_crossentropy(target, y_pred, from_logits=True) + * w + ) + logging.info( + f"in _compute_loss, without old model loss is {loss} and shape is {loss.shape}" + ) return loss - else : + else: w = self.efficient_old_class_weight(target, labels) # print(f"old class weight shape: {w.shape}") - loss = tf.reduce_mean(keras.losses.binary_crossentropy(target, y_pred, from_logits=True) * w) + loss = tf.reduce_mean( + keras.losses.binary_crossentropy(target, y_pred, from_logits=True) * w + ) # logging.info(f'loss new is {keras.losses.binary_crossentropy(target, y_pred, from_logits=True) * w}') # print(f'in _compute_loss, loss is {loss} and shape is {loss.shape}') distill_target = tf.Variable(get_one_hot(labels, self.num_classes)) @@ -214,10 +264,14 @@ def _compute_loss(self, imgs, labels): old_task_size = old_target.shape[1] # print(f'old_target shape: {old_target.shape} and old_task_size: {old_task_size}') distill_target[:, :old_task_size].assign(old_target) - loss_old = tf.reduce_mean(keras.losses.binary_crossentropy(distill_target, y_pred, from_logits=True)) - logging.info(f'loss old is {loss_old}') + loss_old = tf.reduce_mean( + keras.losses.binary_crossentropy( + distill_target, y_pred, from_logits=True + ) + ) + logging.info(f"loss old is {loss_old}") return 0.5 * loss + 0.5 * loss_old - + def efficient_old_class_weight(self, output, labels): # print("---calculate efficient old class weight---") pred = tf.sigmoid(output) @@ -227,7 +281,7 @@ def efficient_old_class_weight(self, output, labels): class_mask = tf.zeros([N, C], dtype=tf.float32) class_mask = tf.Variable(class_mask) # print(f"class_mask shape: {class_mask.shape}") - ids = np.zeros([N,2], dtype=np.int32) + ids = np.zeros([N, 2], dtype=np.int32) for i in range(N): ids[i][0] = i ids[i][1] = labels[i] @@ -239,29 +293,33 @@ def efficient_old_class_weight(self, output, labels): target = get_one_hot(labels, self.num_classes) # print(f'target shape: {target.shape}') g = tf.abs(target - pred) - g = tf.reduce_sum(g*class_mask, axis=1) + g = tf.reduce_sum(g * class_mask, axis=1) # print(f"g shape: {g.shape}") - idx = tf.cast(tf.reshape( labels ,(-1,1)), tf.int32) + idx = tf.cast(tf.reshape(labels, (-1, 1)), tf.int32) if len(self.learned_classes) != 0: # learned_classes_tensor = tf.constant(self.learned_classes, dtype=tf.int32) for i in self.learned_classes: mask = tf.math.not_equal(idx, i) negative_value = tf.cast(tf.fill(tf.shape(idx), -1), tf.int32) idx = tf.where(mask, idx, negative_value) - # 计算 index1 和 index2 + # 计算 index1 和 index2 index1 = tf.cast(tf.equal(idx, -1), tf.float32) index2 = tf.cast(tf.not_equal(idx, -1), tf.float32) # 计算 w1 和 w2 w1 = tf.where( tf.not_equal(tf.reduce_sum(index1), 0), - tf.math.divide(g * index1, (tf.reduce_sum(g * index1) / tf.reduce_sum(index1))), - tf.zeros_like(g) + tf.math.divide( + g * index1, (tf.reduce_sum(g * index1) / tf.reduce_sum(index1)) + ), + tf.zeros_like(g), ) # print(f"w1 shape: {w1.shape}") w2 = tf.where( tf.not_equal(tf.reduce_sum(index2), 0), - tf.math.divide(g * index2, (tf.reduce_sum(g * index2) / tf.reduce_sum(index2))), - tf.zeros_like(g) + tf.math.divide( + g * index2, (tf.reduce_sum(g * index2) / tf.reduce_sum(index2)) + ), + tf.zeros_like(g), ) # print(f"w2 shape: {w2.shape}") # 计算最终的 w @@ -269,9 +327,9 @@ def efficient_old_class_weight(self, output, labels): return w else: return tf.ones(g.shape, dtype=tf.float32) - + def get_train_set_data(self, class_id): - + images = [] train_x = self.labeled_train_set[0] train_y = self.labeled_train_set[1] @@ -280,43 +338,47 @@ def get_train_set_data(self, class_id): # print(train_x[i].shape) images.append(train_x[i]) return images - + def get_data_size(self): - logging.info(f'self.labeled_train_set is None :{self.labeled_train_set is None}') - logging.info(f'self.unlabeled_train_set is None :{self.unlabeled_train_set is None}') - data_size = len(self.labeled_train_set[0]) + logging.info( + f"self.labeled_train_set is None :{self.labeled_train_set is None}" + ) + logging.info( + f"self.unlabeled_train_set is None :{self.unlabeled_train_set is None}" + ) + data_size = len(self.labeled_train_set[0]) logging.info(f"data size: {data_size}") - return data_size - + return data_size + def _reduce_exemplar_set(self, m): for i in range(len(self.exemplar_set)): old_exemplar_data = self.exemplar_set[i][0][:m] old_exemplar_label = self.exemplar_set[i][1][:m] self.exemplar_set[i] = (old_exemplar_data, old_exemplar_label) - - def _construct_exemplar_set(self, images,label, m): + + def _construct_exemplar_set(self, images, label, m): class_mean, fe_outpu = self.compute_class_mean(images) exemplar = [] labels = [] - now_class_mean = np.zeros((1,512)) + now_class_mean = np.zeros((1, 512)) for i in range(m): - x = class_mean - (now_class_mean + fe_outpu)/(i + 1) + x = class_mean - (now_class_mean + fe_outpu) / (i + 1) x = np.linalg.norm(x) index = np.argmin(x) now_class_mean += fe_outpu[index] exemplar.append(images[index]) labels.append(label) self.exemplar_set.append((exemplar, labels)) - + def compute_class_mean(self, images): images_data = tf.data.Dataset.from_tensor_slices(images).batch(self.batch_size) fe_output = self.feature_extractor.predict(images_data) - fe_output = tf.nn.l2_normalize( fe_output).numpy() + fe_output = tf.nn.l2_normalize(fe_output).numpy() # print(f"fe_output shape is {fe_output.shape}") class_mean = tf.reduce_mean(fe_output, axis=0) # print(f'class mean is {class_mean.shape}') return class_mean, fe_output - + def proto_grad(self): if self.need_update == False: return None @@ -325,7 +387,7 @@ def proto_grad(self): cri_loss = keras.losses.SparseCategoricalCrossentropy() proto = [] proto_grad = [] - logging.info(f'self. current class is {self.current_classes}') + logging.info(f"self. current class is {self.current_classes}") for i in self.current_classes: images = self.get_train_set_data(i) # print(f'image shape is {len(images)}') @@ -333,7 +395,7 @@ def proto_grad(self): dis = np.linalg.norm(class_mean - fe_output, axis=1) pro_index = np.argmin(dis) proto.append(images[pro_index]) - + for i in range(len(proto)): data = proto[i] data = tf.cast(tf.expand_dims(data, axis=0), tf.float32) @@ -342,7 +404,9 @@ def proto_grad(self): # print("in proto_grad, label shape is ", label.shape) label = tf.constant([label]) target = get_one_hot(label, self.num_classes) - logging.info(f'proto_grad target shape is {target.shape} and num_classes is {self.num_classes}') + logging.info( + f"proto_grad target shape is {target.shape} and num_classes is {self.num_classes}" + ) proto_fe = resnet10() proto_fe.build(input_shape=(None, 32, 32, 3)) proto_fe.call(keras.Input(shape=(32, 32, 3))) @@ -351,7 +415,7 @@ def proto_grad(self): proto_param = proto_fe.trainable_variables proto_param.extend(proto_clf.trainable_variables) # opt = keras.optimizers.SGD(learning_rate=self.learning_rate, weight_decay=0.00001) - + # for _ in range(iters): # with tf.GradientTape() as tape: # output = proto_clf(proto_fe(data)) @@ -369,10 +433,9 @@ def proto_grad(self): original_dy_dx = [tf.identity(grad) for grad in dy_dx] proto_grad.append(original_dy_dx) return proto_grad - - + def evaluate(self): - logging.info('evaluate') + logging.info("evaluate") total_num = 0 total_correct = 0 for x, y in self.train_loader: @@ -391,4 +454,4 @@ def evaluate(self): acc = total_correct / total_num del total_correct logging.info(f"finsih task {self.old_task_id} evaluate, acc: {acc}") - return acc \ No newline at end of file + return acc diff --git a/examples/cifar100/fci_ssl/glfc/algorithm/aggregation.py b/examples/cifar100/fci_ssl/glfc/algorithm/aggregation.py index c5c0b69d..8c726c8b 100644 --- a/examples/cifar100/fci_ssl/glfc/algorithm/aggregation.py +++ b/examples/cifar100/fci_ssl/glfc/algorithm/aggregation.py @@ -32,12 +32,10 @@ class FedAvg(BaseAggregation, abc.ABC): def __init__(self): super(FedAvg, self).__init__() self.proxy_server = ProxyServer( - learning_rate=0.01, - num_classes=10, - test_data=None + learning_rate=0.01, num_classes=10, test_data=None ) - self.task_id = -1 - self.num_classes =10 + self.task_id = -1 + self.num_classes = 50 def aggregate(self, clients): """ @@ -59,37 +57,35 @@ def aggregate(self, clients): return self.weights self.total_size = sum([c.num_samples for c in clients]) # print(next(iter(clients)).weights) - old_weight = [np.zeros(np.array(c).shape) for c in - next(iter(clients)).weights] + old_weight = [np.zeros(np.array(c).shape) for c in next(iter(clients)).weights] updates = [] for inx, row in enumerate(old_weight): for c in clients: - row += (np.array(c.weights[inx]) * c.num_samples - / self.total_size) + row += np.array(c.weights[inx]) * c.num_samples / self.total_size updates.append(row.tolist()) - - self.weights = [np.array(layer) for layer in updates] - + + self.weights = [np.array(layer) for layer in updates] + print("finish aggregation....") return self.weights - def helper_function(self,train_infos, **kwargs): + def helper_function(self, train_infos, **kwargs): proto_grad = [] task_id = -1 # print(train_infos) - for key, value in train_infos.items(): - if 'proto_grad' == key and value is not None: + for key, value in train_infos.items(): + if "proto_grad" == key and value is not None: # print(info) for grad_i in value: proto_grad.append(grad_i) # proto_grad.append(info['proto_grad']) - if 'task_id' == key: + if "task_id" == key: task_id = max(value, task_id) self.proxy_server.dataload(proto_grad) if task_id > self.task_id: self.task_id = task_id - print(f'incremental num classes is {self.num_classes * (task_id + 1)}') - self.proxy_server.increment_class( self.num_classes * (task_id + 1) ) + print(f"incremental num classes is {self.num_classes * (task_id + 1)}") + self.proxy_server.increment_class(self.num_classes * (task_id + 1)) self.proxy_server.set_weights(self.weights) - print(f'finish set weight for proxy server') - return {'best_old_model': self.proxy_server.model_back()} \ No newline at end of file + print(f"finish set weight for proxy server") + return {"best_old_model": self.proxy_server.model_back()} diff --git a/examples/cifar100/fci_ssl/glfc/algorithm/algorithm.yaml b/examples/cifar100/fci_ssl/glfc/algorithm/algorithm.yaml index 51711d41..a0e4ab8f 100644 --- a/examples/cifar100/fci_ssl/glfc/algorithm/algorithm.yaml +++ b/examples/cifar100/fci_ssl/glfc/algorithm/algorithm.yaml @@ -36,13 +36,13 @@ algorithm: # name of the hyperparameter; string type; - batch_size: values: - - 128 + - 64 - learning_rate: values: - 0.001 - epochs: values: - - 10 + - 24 - type: "aggregation" name: "FedAvg" url: "./examples/cifar100/fci_ssl/glfc/algorithm/aggregation.py" diff --git a/examples/cifar100/fci_ssl/glfc/algorithm/basemodel.py b/examples/cifar100/fci_ssl/glfc/algorithm/basemodel.py index 5d8590c1..c780ce94 100644 --- a/examples/cifar100/fci_ssl/glfc/algorithm/basemodel.py +++ b/examples/cifar100/fci_ssl/glfc/algorithm/basemodel.py @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os +import os import numpy as np import keras import tensorflow as tf @@ -20,30 +20,39 @@ from model import resnet10, lenet5 from network import NetWork, incremental_learning from GLFC import GLFC_Client -import logging +import logging -os.environ['BACKEND_TYPE'] = 'KERAS' +os.environ["BACKEND_TYPE"] = "KERAS" __all__ = ["BaseModel"] logging.getLogger().setLevel(logging.INFO) -@ClassFactory.register(ClassType.GENERAL, alias='glfc') + +@ClassFactory.register(ClassType.GENERAL, alias="glfc") class BaseModel: def __init__(self, **kwargs) -> None: self.kwargs = kwargs - self.learning_rate = kwargs.get('learning_rate', 0.001) - self.epochs = kwargs.get('epochs', 1) - self.batch_size = kwargs.get('batch_size', 32) - self.task_size = kwargs.get('task_size', 10) - self.memory_size = kwargs.get('memory_size', 2000) + self.learning_rate = kwargs.get("learning_rate", 0.001) + self.epochs = kwargs.get("epochs", 1) + self.batch_size = kwargs.get("batch_size", 32) + self.task_size = kwargs.get("task_size", 50) + self.memory_size = kwargs.get("memory_size", 2000) self.encode_model = lenet5(32, 100) self.encode_model.call(keras.Input(shape=(32, 32, 3))) # print(self.encode_model.get_weights()) print(self.encode_model.summary()) # keras.initializers.glorot_uniform() # self.fe = self.build_feature_extractor() - self.num_classes = 10 # the number of class for the first task - self.GLFC_Client = GLFC_Client( self.num_classes, self.batch_size, self.task_size, self.memory_size, self.epochs, self.learning_rate, self.encode_model) - self.best_old_model = [] + self.num_classes = 10 # the number of class for the first task + self.GLFC_Client = GLFC_Client( + self.num_classes, + self.batch_size, + self.task_size, + self.memory_size, + self.epochs, + self.learning_rate, + self.encode_model, + ) + self.best_old_model = [] self.class_learned = 0 self.fe_weights_length = len(self.GLFC_Client.feature_extractor.get_weights()) @@ -55,45 +64,51 @@ def get_weights(self): weights.extend(fe_weights) weights.extend(clf_weights) return weights - + def set_weights(self, weights): print("set weights") - fe_weights = weights[:self.fe_weights_length] - - clf_weights = weights[self.fe_weights_length:] + fe_weights = weights[: self.fe_weights_length] + + clf_weights = weights[self.fe_weights_length :] self.GLFC_Client.feature_extractor.set_weights(fe_weights) self.GLFC_Client.classifier.set_weights(clf_weights) - - def train(self, train_data,val_data, **kwargs): - task_id = kwargs.get('task_id', 0) - round = kwargs.get('round', 1) + + def train(self, train_data, val_data, **kwargs): + task_id = kwargs.get("task_id", 0) + round = kwargs.get("round", 1) logging.info(f"in train: {round} task_id: {task_id}") self.class_learned += self.task_size - self.GLFC_Client.before_train(task_id, train_data, self.class_learned, old_model=self.best_old_model) - + self.GLFC_Client.before_train( + task_id, train_data, self.class_learned, old_model=self.best_old_model + ) + self.GLFC_Client.train(round) proto_grad = self.GLFC_Client.proto_grad() print(type(proto_grad)) # self.GLFC_Client.evaluate() - return {'num_samples': self.GLFC_Client.get_data_size() , 'proto_grad' : proto_grad, 'task_id': task_id} - + return { + "num_samples": self.GLFC_Client.get_data_size(), + "proto_grad": proto_grad, + "task_id": task_id, + } + def helper_function(self, helper_info, **kwargs): - self.best_old_model = helper_info['best_old_model'] + self.best_old_model = helper_info["best_old_model"] print(self.best_old_model) if self.best_old_model[1] != None: self.GLFC_Client.old_model = self.best_old_model[1] else: self.GLFC_Client.old_model = self.best_old_model[0] - + def predict(self, datas, **kwargs): result = {} mean = np.array((0.5071, 0.4867, 0.4408), np.float32).reshape(1, 1, -1) std = np.array((0.2675, 0.2565, 0.2761), np.float32).reshape(1, 1, -1) for data in datas: x = np.load(data) - x = (tf.cast(x, dtype=tf.float32) / 255. - mean) /std - logits = self.GLFC_Client.model_call(x,training=False) + x = (tf.cast(x, dtype=tf.float32) / 255.0 - mean) / std + logits = self.GLFC_Client.model_call(x, training=False) pred = tf.cast(tf.argmax(logits, axis=1), tf.int32) result[data] = pred.numpy() print("finish predict") - return result \ No newline at end of file + return result diff --git a/examples/cifar100/fci_ssl/glfc/benchmarkingjob.yaml b/examples/cifar100/fci_ssl/glfc/benchmarkingjob.yaml index 17e861fa..92be51e4 100644 --- a/examples/cifar100/fci_ssl/glfc/benchmarkingjob.yaml +++ b/examples/cifar100/fci_ssl/glfc/benchmarkingjob.yaml @@ -25,7 +25,7 @@ benchmarkingjob: rank: # rank leaderboard with metric of test case's evaluation and order ; list type; # the sorting priority is based on the sequence of metrics in the list from front to back; - sort_by: [ { "accuracy": "descend" } ] + sort_by: [ { "task_avg_acc": "descend" } ] # visualization configuration visualization: @@ -57,7 +57,7 @@ benchmarkingjob: # currently the options of value are as follows: # 1> "all": select all metrics in the leaderboard; # 2> metrics in the leaderboard, e.g., "F1_SCORE" - metrics: [ "accuracy" ] + metrics: [ "task_avg_acc","forget_rate" ] # network of save selected and all dataitems in workspace `./rank` ; string type; # currently the options of value are as follows: diff --git a/examples/cifar100/fci_ssl/glfc/testenv/testenv.yaml b/examples/cifar100/fci_ssl/glfc/testenv/testenv.yaml index ead9dd4e..47c08807 100644 --- a/examples/cifar100/fci_ssl/glfc/testenv/testenv.yaml +++ b/examples/cifar100/fci_ssl/glfc/testenv/testenv.yaml @@ -24,6 +24,7 @@ testenv: # the url address of python file url: "/home/wyd/ianvs/project/ianvs/examples/cifar100/fci_ssl/glfc/testenv/acc.py" - name: "forget_rate" + - name: "task_avg_acc" # incremental rounds setting of incremental learning; int type; default value is 2; - incremental_rounds: 10 - round: 3 \ No newline at end of file + incremental_rounds: 2 + round: 1 \ No newline at end of file