refactoring

autotm/clustering.py

@@ -1,21 +1,18 @@
-from matplotlib import offsetbox
 import matplotlib.pyplot as plt
-import matplotlib.patheffects as PathEffects
-import seaborn as sns
-import pandas as pd
+import copy
+import warnings
+
+import matplotlib.pyplot as plt
 import numpy as np
-import plotly.graph_objects as go
+import pandas as pd
 from sklearn.cluster import KMeans
-from sklearn.preprocessing import StandardScaler
 from sklearn.manifold import TSNE
-import warnings
-import copy
-
+from sklearn.preprocessing import StandardScaler
 
 warnings.filterwarnings('ignore')
 
 
-def cluster_phi(phi_df: pd.dataFrame, n_clusters=10, plot_img=True):
+def cluster_phi(phi_df: pd.DataFrame, n_clusters=10, plot_img=True):
     _phi_df = copy.deepcopy(phi_df)
     y = _phi_df.index.values
     x = _phi_df.values

autotm/content_splitter.py

@@ -1,6 +1,5 @@
-import re
-from abc import ABC, abstractmethod
-from autotm.preprocessing.text_preprocessing import process_dataset
+from abc import ABC
+
 
 class BaseTextSplitter(ABC):
 

autotm/main_fitness_worker.py

@@ -7,7 +7,7 @@ def main():
     os.environ['AUTOTM_COMPONENT'] = 'worker'
     os.environ['AUTOTM_EXEC_MODE'] = 'cluster'
 
-    from autotm.fitness.tasks import make_celery_app
+    from autotm.fitness.cluster_tasks import make_celery_app
     from autotm.fitness.tm import TopicModelFactory
 
     if "DATASETS_CONFIG" in os.environ:

autotm/params.py

@@ -1,5 +1,4 @@
 import copy
-import logging
 import random
 from typing import List
 
@@ -11,7 +10,6 @@
 from autotm.graph_ga import create_pipeline, crossover_pipelines, mutate_pipeline
 from autotm.pipeline import Pipeline, Stage, StageType, Param, create_stage, IntRangeDistribution, \
     FloatRangeDistribution
-from autotm.utils import do_suppress_stdout
 
 PARAM_NAMES = [
     "val_decor",

autotm/utils.py

@@ -102,6 +102,8 @@ def parallelize_dataframe(df: pd.DataFrame, func, n_cores, return_type="df", **k
         elif isinstance(map_res[0], tuple):
             zipped_elems = list(zip(*map_res))
             res = (merge_dicts(zipped_elems[0]), merge_dicts(zipped_elems[1]))
+    else:
+        raise ValueError(f"Unsupported return_type: {return_type}")
     pool.close()
     pool.join()
     return res

tests/unit/conftest.py

@@ -1,6 +1,6 @@
 import itertools
 import os.path
-from typing import List, Dict, Tuple
+from typing import Dict
 
 import pytest