-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
v1docq
committed
Apr 9, 2024
1 parent
787135d
commit 27b42cd
Showing
8 changed files
with
176 additions
and
112 deletions.
There are no files selected for viewing
34 changes: 12 additions & 22 deletions
34
...les/automl_example/api_example/time_series/ts_classification/ts_classification_example.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,31 +1,21 @@ | ||
from fedot.core.pipelines.pipeline_builder import PipelineBuilder | ||
from fedot_ind.tools.example_utils import industrial_common_modelling_loop | ||
|
||
from fedot_ind.api.main import FedotIndustrial | ||
from fedot_ind.tools.loader import DataLoader | ||
|
||
if __name__ == "__main__": | ||
dataset_name = 'Handwriting' | ||
finetune = True | ||
initial_assumption = PipelineBuilder().add_node('channel_filtration').\ | ||
initial_assumption = PipelineBuilder().add_node('channel_filtration'). \ | ||
add_node('quantile_extractor').add_node('rf') | ||
metric_names = ('f1', 'accuracy', 'precision', 'roc_auc') | ||
api_config = dict(problem='classification', | ||
metric='f1', | ||
timeout=5, | ||
initial_assumption=initial_assumption, | ||
n_jobs=2, | ||
logging_level=20) | ||
|
||
industrial = FedotIndustrial(problem='classification', | ||
metric='f1', | ||
timeout=5, | ||
initial_assumption=initial_assumption, | ||
n_jobs=2, | ||
logging_level=20) | ||
|
||
train_data, test_data = DataLoader(dataset_name=dataset_name).load_data() | ||
if finetune: | ||
model = industrial.finetune(train_data) | ||
else: | ||
model = industrial.fit(train_data) | ||
|
||
labels = industrial.predict(test_data) | ||
probs = industrial.predict_proba(test_data) | ||
metrics = industrial.get_metrics(target=test_data[1], | ||
rounding_order=3, | ||
metric_names=['f1', 'accuracy', 'precision', 'roc_auc']) | ||
model, labels, metrics = industrial_common_modelling_loop(api_config=api_config, | ||
dataset_name=dataset_name, | ||
finetune=finetune) | ||
print(metrics) | ||
_ = 1 |
38 changes: 38 additions & 0 deletions
38
examples/automl_example/api_example/time_series/ts_forecasting/forecasting_analysis.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
import os | ||
import pandas as pd | ||
|
||
from fedot_ind.api.utils.path_lib import PROJECT_PATH | ||
|
||
forecast_result_path = PROJECT_PATH + '/examples/automl_example/api_example/time_series/ts_forecasting/forecasts/' | ||
|
||
|
||
def read_results(forecast_result_path): | ||
results = os.listdir(forecast_result_path) | ||
df_forecast = [] | ||
df_metrics = [] | ||
for file in results: | ||
df = pd.read_csv(f'{forecast_result_path}/{file}') | ||
name = file.split('_')[0] | ||
df['dataset_name'] = name | ||
if file.__contains__('forecast'): | ||
df_forecast.append(df) | ||
else: | ||
df_metrics.append(df) | ||
return df_forecast, df_metrics | ||
|
||
|
||
def create_comprasion_df(df, metric: str = 'rmse'): | ||
df_full = pd.concat(df) | ||
df_full = df_full[df_full['Unnamed: 0'] == metric] | ||
df_full = df_full .drop('Unnamed: 0', axis=1) | ||
df_full['Difference_industrial'] = (df_full.iloc[:, 1:2].min(axis=1) - df_full['industrial']) | ||
df_full['industrial_Wins'] = df_full.apply(lambda row: 'Win' if row.loc['Difference_industrial'] > 0 else 'Loose', | ||
axis=1) | ||
return df_full | ||
|
||
|
||
if __name__ == "__main__": | ||
for metric in ['rmse', 'smape']: | ||
df_forecast, df_metrics = read_results(forecast_result_path) | ||
df_comprasion = create_comprasion_df(df_metrics, metric) | ||
print(df_comprasion['industrial_Wins'].value_counts()) |
108 changes: 50 additions & 58 deletions
108
examples/automl_example/api_example/time_series/ts_forecasting/ts_forecasting_example.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,69 +1,61 @@ | ||
import os | ||
|
||
import pandas as pd | ||
from fedot.core.pipelines.pipeline_builder import PipelineBuilder | ||
|
||
from fedot_ind.api.main import FedotIndustrial | ||
from fedot_ind.api.utils.path_lib import PROJECT_PATH | ||
from fedot_ind.core.metrics.metrics_implementation import calculate_forecasting_metric | ||
from fedot_ind.core.repository.constanst_repository import M4_FORECASTING_BENCH | ||
from fedot_ind.tools.loader import DataLoader | ||
from fedot_ind.tools.example_utils import industrial_forecasting_modelling_loop, compare_forecast_with_sota | ||
|
||
if __name__ == "__main__": | ||
|
||
#dataset_name = 'D1317' | ||
forecast_result_path = os.listdir(PROJECT_PATH + | ||
'/examples/automl_example/api_example/time_series/ts_forecasting/forecasts/') | ||
forecast_result_path = set([x.split('_')[0] for x in forecast_result_path]) | ||
forecast_col = ['industrial', 'target', 'AG', 'NBEATS'] | ||
metric_col = ['industrial', 'AG', 'NBEATS'] | ||
benchmark = 'M4' | ||
horizon = 14 | ||
finetune = False | ||
for dataset_name in M4_FORECASTING_BENCH: | ||
try: | ||
autogluon = PROJECT_PATH + f'/benchmark/results/benchmark_results/autogluon/' \ | ||
f'{dataset_name}_{horizon}_forecast_vs_actual.csv' | ||
n_beats = PROJECT_PATH + f'/benchmark/results/benchmark_results/nbeats/' \ | ||
f'{dataset_name}_{horizon}_forecast_vs_actual.csv' | ||
n_beats = pd.read_csv(n_beats) | ||
autogluon = pd.read_csv(autogluon) | ||
|
||
n_beats_forecast = calculate_forecasting_metric(target=n_beats['value'].values, | ||
labels=n_beats['predict'].values) | ||
autogluon_forecast = calculate_forecasting_metric(target=autogluon['value'].values, | ||
labels=autogluon['predict'].values) | ||
|
||
initial_assumption = PipelineBuilder().add_node('eigen_basis', | ||
params={'low_rank_approximation': False, | ||
'rank_regularization': 'explained_dispersion'}).add_node( | ||
'ar') | ||
industrial = FedotIndustrial(problem='ts_forecasting', | ||
metric='rmse', | ||
task_params={'forecast_length': horizon}, | ||
timeout=5, | ||
with_tuning=False, | ||
initial_assumption=initial_assumption, | ||
n_jobs=2, | ||
logging_level=30) | ||
|
||
train_data, _ = DataLoader(dataset_name=dataset_name).load_forecast_data(folder=benchmark) | ||
|
||
if finetune: | ||
model = industrial.finetune(train_data) | ||
else: | ||
model = industrial.fit(train_data) | ||
|
||
labels = industrial.predict(train_data) | ||
metrics = industrial.get_metrics(target=train_data.values[-horizon:].flatten(), | ||
metric_names=('smape', 'rmse', 'median_absolute_error')) | ||
industrial.save_best_model() | ||
forecast = pd.DataFrame([labels, | ||
train_data.values[-horizon:].flatten(), | ||
autogluon['predict'].values, | ||
n_beats['predict'].values]).T | ||
forecast.columns = ['industrial', 'target', | ||
'AG', | ||
'NBEATS'] | ||
metrics_comprasion = pd.concat([metrics, autogluon_forecast, n_beats_forecast]).T | ||
metrics_comprasion.columns = ['industrial', | ||
'AG', | ||
'NBEATS'] | ||
forecast.to_csv(f'./{dataset_name}_forecast.csv') | ||
metrics_comprasion.to_csv(f'./{dataset_name}_metrics.csv') | ||
except Exception: | ||
print(f'Skip {dataset_name}') | ||
initial_assumption = PipelineBuilder().add_node('eigen_basis', | ||
params={'low_rank_approximation': False, | ||
'rank_regularization': 'explained_dispersion'}).add_node( | ||
'ar') | ||
api_config = dict(problem='ts_forecasting', | ||
metric='rmse', | ||
task_params={'forecast_length': horizon}, | ||
timeout=5, | ||
with_tuning=False, | ||
initial_assumption=initial_assumption, | ||
n_jobs=2, | ||
logging_level=30) | ||
|
||
for dataset_name in M4_FORECASTING_BENCH: | ||
if dataset_name in forecast_result_path: | ||
print('Already evaluated') | ||
else: | ||
try: | ||
n_beats_forecast, n_beats_metrics, \ | ||
autogluon_forecast, autogluon_metrics = compare_forecast_with_sota(dataset_name=dataset_name, | ||
horizon=horizon) | ||
model, labels, metrics, target = industrial_forecasting_modelling_loop(dataset_name=dataset_name, | ||
benchmark=benchmark, | ||
horizon=horizon, | ||
api_config=api_config, | ||
finetune=finetune) | ||
|
||
forecast = pd.DataFrame([labels, | ||
target, | ||
n_beats_forecast, | ||
autogluon_forecast]).T | ||
forecast.columns = forecast_col | ||
|
||
metrics_comprasion = pd.concat([metrics, | ||
autogluon_forecast, | ||
n_beats_forecast]).T | ||
metrics_comprasion.columns = metric_col | ||
|
||
forecast.to_csv(f'./{dataset_name}_forecast.csv') | ||
metrics_comprasion.to_csv(f'./{dataset_name}_metrics.csv') | ||
|
||
except Exception as ex: | ||
print(f'Skip {dataset_name}. Reason - {ex}') |
37 changes: 14 additions & 23 deletions
37
examples/automl_example/api_example/time_series/ts_regression/ts_regression_example.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,29 +1,20 @@ | ||
from fedot.core.pipelines.pipeline_builder import PipelineBuilder | ||
|
||
from fedot_ind.api.main import FedotIndustrial | ||
from fedot_ind.tools.loader import DataLoader | ||
from fedot_ind.tools.example_utils import industrial_common_modelling_loop | ||
|
||
if __name__ == "__main__": | ||
dataset_name = 'IEEEPPG' #BeijingPM10Quality | ||
dataset_name = 'IEEEPPG' # BeijingPM10Quality | ||
finetune = True | ||
initial_assumption = PipelineBuilder().add_node('channel_filtration').add_node('quantile_extractor').add_node('treg') | ||
|
||
industrial = FedotIndustrial(problem='regression', | ||
metric='rmse', | ||
timeout=5, | ||
initial_assumption=initial_assumption, | ||
n_jobs=2, | ||
logging_level=20) | ||
|
||
train_data, test_data = DataLoader(dataset_name=dataset_name).load_data() | ||
if finetune: | ||
model = industrial.finetune(train_data) | ||
else: | ||
model = industrial.fit(train_data) | ||
|
||
labels = industrial.predict(test_data) | ||
metrics = industrial.get_metrics(target=test_data[1], | ||
rounding_order=3, | ||
metric_names=('r2', 'rmse', 'mae')) | ||
initial_assumption = PipelineBuilder().add_node('channel_filtration').add_node('quantile_extractor').add_node( | ||
'treg') | ||
api_config = dict(problem='regression', | ||
metric='rmse', | ||
timeout=5, | ||
initial_assumption=initial_assumption, | ||
n_jobs=2, | ||
logging_level=20) | ||
metric_names = ('r2', 'rmse', 'mae') | ||
model, labels, metrics = industrial_common_modelling_loop(api_config=api_config, | ||
dataset_name=dataset_name, | ||
finetune=finetune) | ||
print(metrics) | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters