Adding surrogate models and train loops

.gitignore

@@ -28,6 +28,15 @@ share/python-wheels/
 MANIFEST
 .idea/
 
+#Train data for surrogate models
+data_from_comsol/generated_data
+data_from_comsol/test_gen_data
+data_from_comsol/gen_data_extend
+#Weights 
+weights
+*.npz
+events.out*
+
 # PyInstaller
 #  Usually these files are written by a python script from a template
 #  before PyInstaller builds the exe, so as to inject date/other infos into it.

cases/microfluidic/config_generate.py

@@ -0,0 +1,118 @@
+from pathlib import Path
+
+from gefest.core.configs.optimization_params import OptimizationParams
+from gefest.core.configs.tuner_params import TunerParams
+from gefest.core.geometry.datastructs.structure import Structure
+from gefest.core.geometry.domain import Domain
+from gefest.core.opt.objective.objective import Objective
+from gefest.tools.estimators.simulators.comsol.comsol_interface import Comsol
+from gefest.tools.estimators.estimator import Estimator
+
+class sosi_hiu(Estimator):
+    def __init__(self, domain, estimator):
+        super().__init__(domain, estimator)
+    def _evaluate(self, ind: Structure):
+        return 'SOLID соси хуй (С)Олег'
+
+# # # Metrics # # #
+class ComsolMetric(Objective):
+    """Comsol metric."""
+    def __init__(self, domain, estimator):
+        super().__init__(domain, estimator)
+
+    def _evaluate(self, ind: Structure):
+        return self.estimator(ind)
+
+bottom_aa = [(-130 -49),
+(-130, 21),
+(21, 21),
+(-7, -69),
+(-31, -170),
+(-130, -170),
+(-130 ,-49)]
+aasd = [(-130,-49),(-130,21),(21,21),(-7,-69),(-31, -170),
+(-130, -170),
+(-130 ,-49)]
+standart_aa = [
+        (-125, 100),
+        (-75, 170),
+        (15, 170),
+        (30, 90),
+        (-20, -130),
+        (-20, -170),
+        (-125, -170),
+        (-125, 100),
+    ]
+# # # Precompute domain arguments # # #
+pass
+# # #
+
+domain_cfg = Domain(
+    allowed_area=aasd,
+    name='main',
+    min_poly_num=1,
+    max_poly_num=2,
+    min_points_num=3,
+    max_points_num=10,
+    polygon_side=0.0001,
+    min_dist_from_boundary=0.01,
+    geometry_is_convex=False,
+    geometry_is_closed=True,
+    geometry='2D',
+)
+
+tuner_cfg = TunerParams(
+    tuner_type='optuna',
+    n_steps_tune=10,
+    hyperopt_dist='uniform',
+    verbose=True,
+    timeout_minutes=60,
+)
+
+opt_params = OptimizationParams(
+    optimizer='gefest_ga',
+    domain=domain_cfg,
+    tuner_cfg=tuner_cfg,
+    n_steps=50,
+    pop_size=50,
+    postprocess_attempts=3,
+    mutation_prob=0.6,
+    crossover_prob=0.6,
+    mutations=[
+        'rotate_poly',
+        'resize_poly',
+        'add_point',
+        'drop_point',
+        'add_poly',
+        'drop_poly',
+        'pos_change_point',
+    ],
+    selector='tournament_selection',
+    mutation_each_prob=[0.125, 0.125, 0.15, 0.35, 0.00, 0.00, 0.25],
+    crossovers=[
+        'polygon_level',
+        'structure_level',
+    ],
+    crossover_each_prob=[0.0, 1.0],
+    postprocess_rules=[
+        'not_out_of_bounds',
+        'valid_polygon_geom',
+        'not_self_intersects',
+        'not_too_close_polygons',
+        # 'not_overlaps_prohibited',
+        'not_too_close_points',
+    ],
+    extra=5,
+    n_jobs=-1,
+    log_dir='logs',
+    run_name='run_name',
+    golem_keep_histoy=False,
+    golem_genetic_scheme_type='steady_state',
+    golem_surrogate_each_n_gen=5,
+    objectives=[
+        ComsolMetric(
+            domain_cfg,
+            sosi_hiu,
+        ),
+    ],
+)

cases/sound_waves/configuration/config_parallel.py

@@ -82,16 +82,16 @@ def _evaluate(self, ind: Structure):
     hyperopt_dist='uniform',
     verbose=True,
     variacne_generator=partial(percent_edge_variance, percent=0.5),
-    timeout_minutes=30,
+    timeout_minutes=130,
 )
 
 
 opt_params = OptimizationParams(
-    optimizer='gefest_ga',
+    optimizer='gefest_ga_h',
     domain=domain_cfg,
     tuner_cfg=tuner_cfg,
-    n_steps=100,
-    pop_size=100,
+    n_steps=10,
+    pop_size=10,
     postprocess_attempts=3,
     mutation_prob=0.9,
     crossover_prob=0.6,
@@ -120,8 +120,8 @@ def _evaluate(self, ind: Structure):
         'not_too_close_points',
     ],
     extra=5,
-    estimation_n_jobs=-1,
-    n_jobs=-1,
+    estimation_n_jobs=1,
+    n_jobs=1,
     log_dir='logs/tuners_exp',
     run_name='roulette_1_obj',
     golem_keep_histoy=True,

data_from_comsol/coords.txt

@@ -0,0 +1,7 @@
+-130 -49.02983247582813
+-130 20.52414239317181
+20.604039813602455 20.524142393171942
+-6.907702544715789 -69.02983247582813
+-31.56130231388761 -170
+-130 -170
+-130 -49.02983247582813

data_from_comsol/create_matrix.py

@@ -0,0 +1,58 @@
+import pandas as pd
+import numpy as np
+from matplotlib import pyplot as plt
+from skimage.morphology import reconstruction
+from scipy.ndimage.morphology import grey_dilation, generate_binary_structure, iterate_structure
+
+def create_mask(path,dilation:bool=True):
+    df=pd.read_csv(path,names=['x','y','color'],sep='\t')
+    array = np.zeros((400,400))
+    x = (np.array(df['x'])+200)
+    y = (np.array(df['y'])+200)
+    value = np.array(df['color'])
+    array[np.intc(x),np.intc(y)]=value
+    if dilation:
+        array = grey_dilation(array,size=(2,2))
+    return array
+
+def create_flow(path,dilation:bool=True):
+    df=pd.read_csv(path,names=['x','y','color'],sep='\t')
+    array = np.ones((400,400))*-1
+    x = (np.array(df['x'])+200)
+    y = (np.array(df['y'])+200)
+    value = np.array(df['color'])
+    array[np.intc(x),np.intc(y)]=value
+    if dilation:
+        array = grey_dilation(array,size=(2,2))
+    return array
+
+
+
+if __name__=="__main__":
+    df=pd.read_csv('data_from_comsol//velocity.txt',names=['x','y','color'],sep='\t')
+    #df=pd.read_csv('data_from_comsol\mask.txt',names=['x','y','color'],sep='\t')
+
+    x = (np.array(df['x'])+200)
+    y = (np.array(df['y'])+200)
+    value = np.array(df['color'])
+
+    array_2 = np.ones((400,400))*-256
+    array_2[np.intc(x),np.intc(y)]=value
+
+
+    array_111 = grey_dilation(array_2,size=(2,2))
+    plt.imshow(array_111,interpolation='nearest')
+    plt.show()
+
+
+    array = np.ones((400,400))*-256
+    array[np.intc(x),np.intc(y)]=value
+    # array[np.intc(x+0.25),np.intc(y+0.25)]=value
+    # array[np.intc(x-0.25),np.intc(y-0.25)]=value
+
+    # array[np.intc(x+0.25),np.intc(y-0.25)]=value
+    # array[np.intc(x-0.25),np.intc(y+0.25)]=value
+    plt.imshow(array,interpolation='nearest')
+    plt.show()
+
+    print()

data_from_comsol/read.py

@@ -0,0 +1,11 @@
+import pandas as pd
+import numpy as np
+from matplotlib import pyplot as plt
+from skimage.morphology import reconstruction
+from scipy.ndimage.morphology import grey_dilation, generate_binary_structure, iterate_structure
+
+data = np.load('data_from_comsol\generated_data\data_1.npz',allow_pickle=True)
+
+for i in data['arr_0'].item(0)['flow']:
+    plt.imshow(i,interpolation='nearest')
+    plt.show()

data_from_comsol/read_vtu.py

@@ -0,0 +1,12 @@
+import vtk
+
+# The source file
+file_name = "GEFEST\data_from_comsol\plot_data.vtu"
+
+# Read the source file.
+reader = vtk.vtkXMLUnstructuredGridReader()
+reader.SetFileName(file_name)
+reader.Update()  # Needed because of GetScalarRange
+output = reader.GetOutput()
+potential = output.GetPointData().GetArray("potential")
+print()

data_from_comsol/rename.py

@@ -0,0 +1,6 @@
+import os 
+path = 'data_from_comsol\generated_data'
+print(os.listdir(path))
+
+for i , file in enumerate(os.listdir(path)):
+    os.rename(path+f'/{file}',path+f'/data_gen_{i}.npz')

data_from_comsol/sampler_test.py

@@ -0,0 +1,41 @@
+import os
+import sys
+import numpy as np
+sys.path.append(os.getcwd())
+sys.path.append(str(os.getcwd())+'/GEFEST/')
+import matplotlib.pyplot as plt
+from gefest.core.geometry.datastructs.structure import Structure,Polygon,Point
+from gefest.core.geometry.domain import Domain
+from gefest.tools.estimators.simulators.comsol.comsol_interface import Comsol_generate
+from cases.microfluidic.config_generate import opt_params
+from gefest.tools.samplers.standard.standard import StandardSampler
+from data_from_comsol.create_matrix import create_mask,create_flow
+ITERATIONS = range(10)
+SAMPLES = 10
+
+sampler = StandardSampler(opt_params=opt_params)
+
+def plot_polygons(struct:Structure):
+
+    for points in struct:
+        x,y = [p.coords[0] for p in points],[p.coords[1] for p in points]
+        plt.plot(x,y)
+
+
+domain_points = opt_params.domain.allowed_area.points
+x,y = [p.coords[0] for p in domain_points],[p.coords[1] for p in domain_points]
+plt.plot(x,y)
+for i in ITERATIONS:
+    structs = sampler.sample(n_samples=SAMPLES)
+    for s in structs:
+        plot_polygons(s)
+        print()
+    plt.show()
+
+
+
+
+print()
+
+
+

data_from_comsol/test_to_generate.py

@@ -0,0 +1,34 @@
+import os
+import sys
+import numpy as np
+sys.path.append(os.getcwd())
+print(os.getcwd())
+sys.path.append(str(os.getcwd())+'/GEFEST/')
+from gefest.core.geometry.datastructs.structure import Structure,Polygon,Point
+from gefest.core.geometry.domain import Domain
+from gefest.tools.estimators.simulators.comsol.comsol_interface import Comsol_generate
+from cases.microfluidic.config_generate import opt_params
+from gefest.tools.samplers.standard.standard import StandardSampler
+from data_from_comsol.create_matrix import create_mask,create_flow
+ITERATIONS = range(100)
+SAMPLES = 10
+
+comsol = Comsol_generate(path_to_mph='data_from_comsol//gen_setup.mph')
+sampler = StandardSampler(opt_params=opt_params)
+
+for i in ITERATIONS:
+    data = {'mask':[],'flow':[],'struct':[]}
+    structs = sampler.sample(n_samples=SAMPLES)
+    for s in structs:
+        results = comsol.estimate(structure = s)
+        flow = create_flow(path='data_from_comsol//velocity.txt')
+        mask = create_mask(path='data_from_comsol\mask.txt')
+        data['flow'].append(flow)
+        data['mask'].append(mask)
+        data['struct'].append(s)
+    np.savez(f'data_from_comsol/gen_data_extend/data__{i}',data,allow_pickle=False)
+    print(f'Saved {(i+1)*SAMPLES} samples of simulation')
+print()
+
+
+

gefest/core/geometry/utils.py

@@ -166,7 +166,7 @@ def _create_area(domain: Domain, structure: Structure, geometry: Geometry2D) ->
             geom._poly_to_shapely_poly(poly).convex_hull.buffer(domain.dist_between_polygons, 1),
         ).intersection(area)
 
-    sigma_max = 0.95 * _get_sigma_max(area, (min(domain.max_x, domain.max_y) / 2) * 1.01)
+    sigma_max = 0.95 * _get_sigma_max(area, (min(domain.max_x, domain.max_y) / 2) * 6.01)
     sigma_min = max(domain.max_x - domain.min_x, domain.max_y - domain.min_y) * 0.05
 
     sigma = np.random.uniform(sigma_min, sigma_max)