diff --git a/examples/experiment_analyzer/experiment_analyzer.py b/examples/experiment_analyzer/experiment_analyzer.py index e9cf2170e..6a61b5739 100644 --- a/examples/experiment_analyzer/experiment_analyzer.py +++ b/examples/experiment_analyzer/experiment_analyzer.py @@ -12,7 +12,7 @@ """ The result of analysis can be seen without running the script in '~/GOLEM/examples/experiment_analyzer/result_analysis.tar.gz'. """ - path_to_root = os.path.join(project_root(), 'examples', 'experiment_analyzer') + path_to_root = os.path.join('Z://', 'Pinchuk') # extract data if there is an archive if 'data.tar.gz' in os.listdir(path_to_root): @@ -20,63 +20,63 @@ tar.extractall() tar.close() - path_to_experiment_data = os.path.join(path_to_root, 'data') + path_to_experiment_data = path_to_root path_to_save = os.path.join(path_to_root, 'result_analysis') analyzer = ExperimentAnalyzer(path_to_root=path_to_experiment_data, folders_to_ignore=['result_analysis', 'Thumbs.db']) - - # to get convergence table with mean values - path_to_save_convergence = os.path.join(path_to_save, 'convergence') - - convergence_mean = analyzer.analyze_convergence(history_folder='histories', is_raise=False, - path_to_save=path_to_save_convergence, - is_mean=True) - - # to get convergence boxplots - convergence = analyzer.analyze_convergence(history_folder='histories', is_raise=False) - path_to_save_convergence_boxplots = os.path.join(path_to_save_convergence, 'convergence_boxplots') - - metrics = list(convergence.keys()) - setups = list(convergence[metrics[0]].keys()) - datasets = list(convergence[metrics[0]][setups[0]].keys()) - for dataset in datasets: - for metric_name in convergence.keys(): - to_compare = dict() - for setup in convergence[metric_name].keys(): - to_compare[setup] = [i for i in convergence[metric_name][setup][dataset]] - plt.boxplot(list(to_compare.values()), labels=list(to_compare.keys())) - plt.title(f'Convergence on {dataset}') - os.makedirs(path_to_save_convergence_boxplots, exist_ok=True) - plt.savefig(os.path.join(path_to_save_convergence_boxplots, f'convergence_{dataset}.png')) - plt.close() + # + # # to get convergence table with mean values + # path_to_save_convergence = os.path.join(path_to_save, 'convergence') + # + # convergence_mean = analyzer.analyze_convergence(history_folder='histories', is_raise=False, + # path_to_save=path_to_save_convergence, + # is_mean=True) + # + # # to get convergence boxplots + # convergence = analyzer.analyze_convergence(history_folder='histories', is_raise=False) + # path_to_save_convergence_boxplots = os.path.join(path_to_save_convergence, 'convergence_boxplots') + # + # metrics = list(convergence.keys()) + # setups = list(convergence[metrics[0]].keys()) + # datasets = list(convergence[metrics[0]][setups[0]].keys()) + # for dataset in datasets: + # for metric_name in convergence.keys(): + # to_compare = dict() + # for setup in convergence[metric_name].keys(): + # to_compare[setup] = [i for i in convergence[metric_name][setup][dataset]] + # plt.boxplot(list(to_compare.values()), labels=list(to_compare.keys())) + # plt.title(f'Convergence on {dataset}') + # os.makedirs(path_to_save_convergence_boxplots, exist_ok=True) + # plt.savefig(os.path.join(path_to_save_convergence_boxplots, f'convergence_{dataset}.png')) + # plt.close() # to get metrics table with mean values - path_to_save_metrics = os.path.join(path_to_save, 'metrics') - metric_names = ['roc_auc', 'f1'] + path_to_save_metrics = os.path.join(path_to_save, 'metrics_with_UCB') + metric_names = ['roc_auc', 'f1', 'logloss'] metrics_dict_mean = analyzer.analyze_metrics(metric_names=metric_names, file_name='evaluation_results.csv', is_raise=False, path_to_save=path_to_save_metrics, is_mean=True) - # to get metrics boxplots - metrics_dict = analyzer.analyze_metrics(metric_names=metric_names, file_name='evaluation_results.csv', - is_raise=False) - path_to_save_metrics_boxplots = os.path.join(path_to_save_metrics, 'metrics_boxplot') - - for metric in metric_names: - for dataset in metrics_dict[metric][list(metrics_dict[metric].keys())[0]].keys(): - to_compare = dict() - for setup in metrics_dict[metric].keys(): - to_compare[setup] = [-1 * i for i in metrics_dict[metric][setup][dataset]] - plt.boxplot(list(to_compare.values()), labels=list(to_compare.keys())) - plt.title(f'{metric} on {dataset}') - cur_path_to_save = os.path.join(path_to_save_metrics_boxplots, metric) - os.makedirs(cur_path_to_save, exist_ok=True) - plt.savefig(os.path.join(cur_path_to_save, f'{metric}_{dataset}.png')) - plt.close() - - # to get stat test results table - path_to_save_stat = os.path.join(path_to_save, 'statistic') - stat_dict = analyzer.analyze_statistical_significance(data_to_analyze=metrics_dict['roc_auc'], - stat_tests=[mannwhitneyu, kruskal, ttest_ind], - path_to_save=path_to_save_stat) + # # to get metrics boxplots + # metrics_dict = analyzer.analyze_metrics(metric_names=metric_names, file_name='evaluation_results.csv', + # is_raise=False) + # path_to_save_metrics_boxplots = os.path.join(path_to_save_metrics, 'metrics_boxplot') + # + # for metric in metric_names: + # for dataset in metrics_dict[metric][list(metrics_dict[metric].keys())[0]].keys(): + # to_compare = dict() + # for setup in metrics_dict[metric].keys(): + # to_compare[setup] = [-1 * i for i in metrics_dict[metric][setup][dataset]] + # plt.boxplot(list(to_compare.values()), labels=list(to_compare.keys())) + # plt.title(f'{metric} on {dataset}') + # cur_path_to_save = os.path.join(path_to_save_metrics_boxplots, metric) + # os.makedirs(cur_path_to_save, exist_ok=True) + # plt.savefig(os.path.join(cur_path_to_save, f'{metric}_{dataset}.png')) + # plt.close() + # + # # to get stat test results table + # path_to_save_stat = os.path.join(path_to_save, 'statistic') + # stat_dict = analyzer.analyze_statistical_significance(data_to_analyze=metrics_dict['roc_auc'], + # stat_tests=[mannwhitneyu, kruskal, ttest_ind], + # path_to_save=path_to_save_stat) diff --git a/experiments/experiment_analyzer.py b/experiments/experiment_analyzer.py index 5d0a2310e..9d1451720 100644 --- a/experiments/experiment_analyzer.py +++ b/experiments/experiment_analyzer.py @@ -409,6 +409,8 @@ def _get_path_to_launch(self) -> Tuple[str, str, str]: for setup in os.listdir(self.path_to_root): if setup in self._folders_to_ignore: continue + if setup != 'FEDOT_MAB_UCB' and setup != 'FEDOT_MAB' and setup != 'FEDOT_Classic': + continue path_to_setup = os.path.join(self.path_to_root, setup) for dataset in os.listdir(path_to_setup): if dataset in self._folders_to_ignore: diff --git a/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py b/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py index ccd66480d..86122bb06 100644 --- a/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py +++ b/golem/core/optimisers/adaptive/mab_agents/contextual_mab_agent.py @@ -34,7 +34,7 @@ def __init__(self, actions: Sequence[ActType], super().__init__(actions=actions, n_jobs=n_jobs, enable_logging=enable_logging, decaying_factor=decaying_factor, is_initial_fit=False) self._agent = MAB(arms=self._indices, - learning_policy=LearningPolicy.UCB1(alpha=1.25), + learning_policy=LearningPolicy.UCB1(alpha=0.8), neighborhood_policy=NeighborhoodPolicy.Clusters(), n_jobs=n_jobs) self._context_agent = context_agent_type if isinstance(context_agent_type, Callable) else \ @@ -81,6 +81,8 @@ def partial_fit(self, experience: ExperienceBuffer): obs, arms, processed_rewards = self._get_experience(experience) contexts = self.get_context(obs=obs) self._agent.partial_fit(decisions=arms, rewards=processed_rewards, contexts=contexts) + if self._path_to_save: + self.save() def _get_experience(self, experience: ExperienceBuffer): """ Get experience from ExperienceBuffer, process rewards and log. """ diff --git a/golem/core/optimisers/adaptive/mab_agents/mab_agent.py b/golem/core/optimisers/adaptive/mab_agents/mab_agent.py index 538c2ed2f..91b0274d7 100644 --- a/golem/core/optimisers/adaptive/mab_agents/mab_agent.py +++ b/golem/core/optimisers/adaptive/mab_agents/mab_agent.py @@ -82,7 +82,7 @@ def save(self, path_to_save: Optional[str] = None): if not path_to_save.endswith('.pkl'): os.makedirs(path_to_save, exist_ok=True) mabs_num = [int(name.split('_')[0]) for name in os.listdir(path_to_save) - if re.fullmatch(r'\d_mab.pkl', name)] + if re.fullmatch(r'\d*_mab.pkl', name)] if not mabs_num: max_saved_mab = 0 else: