From f9e88d2a3cef9480438c646e7a4bfc84822e7e39 Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 9 Aug 2023 18:46:52 +0300 Subject: [PATCH 01/22] make it similar to feature/ma_experiments --- sampo/scheduler/genetic/operators.py | 14 ++ sampo/scheduler/genetic/schedule_builder.py | 175 ++++++++++++-------- 2 files changed, 121 insertions(+), 68 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index f2dfb6f8..ed1ca8e4 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -6,6 +6,7 @@ import numpy as np from deap import creator, base, tools +from deap.tools import initRepeat from sampo.scheduler.genetic.converter import convert_chromosome_to_schedule from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome, ChromosomeType @@ -195,6 +196,19 @@ def copy_chromosome(chromosome: ChromosomeType) -> ChromosomeType: return chromosome[0].copy(), chromosome[1].copy(), chromosome[2].copy(), deepcopy(chromosome[3]) +def wrap(chromosome: ChromosomeType) -> Individual: + """ + Created an individual from chromosome. + """ + + def ind_getter(): + return chromosome + + ind = initRepeat(Individual, ind_getter, n=1) + ind.fitness.invalid_steps = 0 + return ind + + def generate_chromosome(wg: WorkGraph, contractors: list[Contractor], work_id2index: dict[str, int], diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index ea12efa0..04e25e35 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -1,18 +1,20 @@ import math import random import time -from typing import Callable +from typing import Callable, Tuple import numpy as np import seaborn as sns from deap import tools +from deap.base import Toolbox from deap.tools import initRepeat from matplotlib import pyplot as plt +from numpy import ndarray from pandas import DataFrame from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, Individual, copy_chromosome, \ - FitnessFunction, TimeFitness, is_chromosome_correct + FitnessFunction, TimeFitness, is_chromosome_correct, wrap from sampo.scheduler.native_wrapper import NativeWrapper from sampo.scheduler.timeline.base import Timeline from sampo.schemas.contractor import Contractor, WorkerContractorPool @@ -26,57 +28,28 @@ from sampo.utilities.collections_util import reverse_dictionary -def build_schedule(wg: WorkGraph, +def create_toolbox(wg: WorkGraph, contractors: list[Contractor], worker_pool: WorkerContractorPool, - population_size: int, - generation_number: int, selection_size: int, mutate_order: float, mutate_resources: float, init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], rand: random.Random, - spec: ScheduleSpec, - landscape: LandscapeConfiguration = LandscapeConfiguration(), - fitness_constructor: Callable[[Callable[[list[ChromosomeType]], list[int]]], - FitnessFunction] = TimeFitness, - work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), - show_fitness_graph: bool = False, - n_cpu: int = 1, - assigned_parent_time: Time = Time(0), - timeline: Timeline | None = None, - time_border: int = None) \ - -> tuple[ScheduleWorkDict, Time, Timeline, list[GraphNode]]: - """ - Genetic algorithm. - Structure of chromosome: - [[order of job], [numbers of workers types 1 for each job], [numbers of workers types 2], ... ] - - Different mate and mutation for order and for workers. - Generate order of job by prioritization from HEFTs and from Topological. - Generate resources from min to max. - Overall initial population is valid. - - :param spec: spec for current scheduling - :param n_cpu: number or parallel workers to use in computational process - :param assigned_parent_time: start time of the whole schedule(time shift) - :return: scheduler - """ - - if show_fitness_graph: - fitness_history = [] - - global_start = time.time() - + spec: ScheduleSpec = ScheduleSpec(), + work_estimator: WorkTimeEstimator = None, + landscape: LandscapeConfiguration = LandscapeConfiguration()) -> tuple[ + Toolbox, ndarray, ndarray, ndarray]: start = time.time() + # preparing access-optimized data structures nodes = [node for node in wg.nodes if not node.is_inseparable_son()] - index2node: dict[int, GraphNode] = dict(enumerate(nodes)) + index2node: dict[int, GraphNode] = {index: node for index, node in enumerate(nodes)} work_id2index: dict[str, int] = {node.id: index for index, node in index2node.items()} worker_name2index = {worker_name: index for index, worker_name in enumerate(worker_pool)} index2contractor = {ind: contractor.id for ind, contractor in enumerate(contractors)} - index2contractor_obj = dict(enumerate(contractors)) + index2contractor_obj = {ind: contractor for ind, contractor in enumerate(contractors)} contractor2index = reverse_dictionary(index2contractor) worker_pool_indices = {worker_name2index[worker_name]: { contractor2index[contractor_id]: worker for contractor_id, worker in workers_of_type.items() @@ -109,9 +82,9 @@ def build_schedule(wg: WorkGraph, inseparable_parents[child] = node # here we aggregate information about relationships from the whole inseparable chain - children = {work_id2index[node.id]: list({work_id2index[inseparable_parents[child].id] - for inseparable in node.get_inseparable_chain_with_self() - for child in inseparable.children}) + children = {work_id2index[node.id]: [work_id2index[inseparable_parents[child].id] + for inseparable in node.get_inseparable_chain_with_self() + for child in inseparable.children] for node in nodes} parents = {work_id2index[node.id]: [] for node in nodes} @@ -119,29 +92,106 @@ def build_schedule(wg: WorkGraph, for child in node_children: parents[child].append(node) - print(f'Genetic optimizing took {(time.time() - start) * 1000} ms') - - start = time.time() - # initial chromosomes construction init_chromosomes: dict[str, ChromosomeType] = \ {name: convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec, order) - if schedule is not None else None + if schedule is not None else None for name, (schedule, order, spec) in init_schedules.items()} - toolbox = init_toolbox(wg, contractors, worker_pool, landscape, index2node, - work_id2index, worker_name2index, index2contractor, - index2contractor_obj, init_chromosomes, mutate_order, - mutate_resources, selection_size, rand, spec, worker_pool_indices, - contractor2index, contractor_borders, node_indices, parents, - assigned_parent_time, work_estimator) - for name, chromosome in init_chromosomes.items(): if chromosome is not None: if not is_chromosome_correct(chromosome, node_indices, parents): raise NoSufficientContractorError('HEFTs are deploying wrong chromosomes') + print(f'Genetic optimizing took {(time.time() - start) * 1000} ms') + + return init_toolbox(wg, + contractors, + worker_pool, + landscape, + index2node, + work_id2index, + worker_name2index, + index2contractor, + index2contractor_obj, + init_chromosomes, + mutate_order, + mutate_resources, + selection_size, + rand, + spec, + worker_pool_indices, + contractor2index, + contractor_borders, + node_indices, + parents, + Time(0), + work_estimator), resources_border, contractors_capacity, resources_min_border + + +def build_schedule(wg: WorkGraph, + contractors: list[Contractor], + worker_pool: WorkerContractorPool, + population_size: int, + generation_number: int, + selection_size: int, + mutate_order: float, + mutate_resources: float, + init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], + rand: random.Random, + spec: ScheduleSpec, + landscape: LandscapeConfiguration = LandscapeConfiguration(), + fitness_constructor: Callable[[Callable[[list[ChromosomeType]], list[int]]], FitnessFunction] = TimeFitness, + work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), + show_fitness_graph: bool = False, + n_cpu: int = 1, + assigned_parent_time: Time = Time(0), + timeline: Timeline | None = None, + time_border: int = None) \ + -> tuple[ScheduleWorkDict, Time, Timeline, list[GraphNode]]: + if show_fitness_graph: + fitness_history = [] + + global_start = time.time() + + # preparing access-optimized data structures + nodes = [node for node in wg.nodes if not node.is_inseparable_son()] + + work_id2index: dict[str, int] = {node.id: index for index, node in enumerate(nodes)} + worker_name2index = {worker_name: index for index, worker_name in enumerate(worker_pool)} + contractor2index = {contractor.id: ind for ind, contractor in enumerate(contractors)} + worker_pool_indices = {worker_name2index[worker_name]: { + contractor2index[contractor_id]: worker for contractor_id, worker in workers_of_type.items() + } for worker_name, workers_of_type in worker_pool.items()} + + # construct inseparable_child -> inseparable_parent mapping + inseparable_parents = {} + for node in nodes: + for child in node.get_inseparable_chain_with_self(): + inseparable_parents[child] = node + + # here we aggregate information about relationships from the whole inseparable chain + children = {work_id2index[node.id]: list({work_id2index[inseparable_parents[child].id] + for inseparable in node.get_inseparable_chain_with_self() + for child in inseparable.children}) + for node in nodes} + + parents = {work_id2index[node.id]: [] for node in nodes} + for node, node_children in children.items(): + for child in node_children: + parents[child].append(node) + + start = time.time() + + toolbox, resources_border, contractors_capacity, resources_min_border = create_toolbox(wg, contractors, worker_pool, + selection_size, + mutate_order, + mutate_resources, + init_schedules, + rand, spec, work_estimator, + landscape) + native = NativeWrapper(toolbox, wg, contractors, worker_name2index, worker_pool_indices, parents, work_estimator) # create population of a given size @@ -199,7 +249,9 @@ def build_schedule(wg: WorkGraph, prev_best_fitness = best_fitness # select individuals of next generation - offspring = toolbox.select(pop, int(math.sqrt(len(pop)))) + # offspring = toolbox.select(pop, int(math.sqrt(len(pop)))) + offspring = toolbox.select(pop, selection_size) + # clone selected individuals # offspring = [toolbox.clone(ind) for ind in offspring] @@ -360,16 +412,3 @@ def build_schedule(wg: WorkGraph, def compare_individuals(first: tuple[ChromosomeType], second: tuple[ChromosomeType]): return (first[0][0] == second[0][0]).all() and (first[0][1] == second[0][1]).all() - - -def wrap(chromosome: ChromosomeType) -> Individual: - """ - Creates an individual from chromosome. - """ - - def ind_getter(): - return chromosome - - ind = initRepeat(Individual, ind_getter, n=1) - ind.fitness.invalid_steps = 0 - return ind From a1c5b3d84952b41c74c55c408be84d0cf95c416c Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 9 Aug 2023 19:04:31 +0300 Subject: [PATCH 02/22] make it similar to feature/ma_experiments --- sampo/scheduler/genetic/schedule_builder.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 04e25e35..b4055ea7 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -1,19 +1,18 @@ import math import random import time -from typing import Callable, Tuple +from typing import Callable import numpy as np import seaborn as sns from deap import tools from deap.base import Toolbox -from deap.tools import initRepeat from matplotlib import pyplot as plt from numpy import ndarray from pandas import DataFrame from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome -from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, Individual, copy_chromosome, \ +from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, copy_chromosome, \ FitnessFunction, TimeFitness, is_chromosome_correct, wrap from sampo.scheduler.native_wrapper import NativeWrapper from sampo.scheduler.timeline.base import Timeline @@ -38,8 +37,8 @@ def create_toolbox(wg: WorkGraph, rand: random.Random, spec: ScheduleSpec = ScheduleSpec(), work_estimator: WorkTimeEstimator = None, - landscape: LandscapeConfiguration = LandscapeConfiguration()) -> tuple[ - Toolbox, ndarray, ndarray, ndarray]: + landscape: LandscapeConfiguration = LandscapeConfiguration()) \ + -> tuple[Toolbox, ndarray, ndarray, ndarray]: start = time.time() # preparing access-optimized data structures @@ -96,7 +95,7 @@ def create_toolbox(wg: WorkGraph, init_chromosomes: dict[str, ChromosomeType] = \ {name: convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec, order) - if schedule is not None else None + if schedule is not None else None for name, (schedule, order, spec) in init_schedules.items()} for name, chromosome in init_chromosomes.items(): @@ -142,7 +141,8 @@ def build_schedule(wg: WorkGraph, rand: random.Random, spec: ScheduleSpec, landscape: LandscapeConfiguration = LandscapeConfiguration(), - fitness_constructor: Callable[[Callable[[list[ChromosomeType]], list[int]]], FitnessFunction] = TimeFitness, + fitness_constructor: Callable[ + [Callable[[list[ChromosomeType]], list[int]]], FitnessFunction] = TimeFitness, work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), show_fitness_graph: bool = False, n_cpu: int = 1, From d5697c279422b46b936464fee545b5320cef6a8d Mon Sep 17 00:00:00 2001 From: Egor Date: Tue, 15 Aug 2023 14:49:32 +0300 Subject: [PATCH 03/22] changing selection --- sampo/scheduler/genetic/schedule_builder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index b4055ea7..0f0fdcd6 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -250,7 +250,7 @@ def build_schedule(wg: WorkGraph, # select individuals of next generation # offspring = toolbox.select(pop, int(math.sqrt(len(pop)))) - offspring = toolbox.select(pop, selection_size) + offspring = toolbox.select(pop, population_size) # clone selected individuals # offspring = [toolbox.clone(ind) for ind in offspring] From baa643227366ed63b7aef4ebc914e709ec111c9d Mon Sep 17 00:00:00 2001 From: Egor Date: Tue, 15 Aug 2023 17:54:58 +0300 Subject: [PATCH 04/22] changing crossover --- sampo/scheduler/genetic/operators.py | 5 ++- sampo/scheduler/genetic/schedule_builder.py | 45 +++++++++++---------- tests/scheduler/genetic/full_scheduling.py | 10 ++--- 3 files changed, 32 insertions(+), 28 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index ed1ca8e4..6166a6cb 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -323,8 +323,9 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand order1 = ind1[0] order2 = ind2[0] + border = len(order1) // 4 # randomly select the point where the crossover will take place - crossover_point = rand.randint(1, len(ind1)) + crossover_point = rand.randint(border, len(order1) - border) ind1_new_tail = get_order_tail(order1[:crossover_point], order2) ind2_new_tail = get_order_tail(order2[:crossover_point], order1) @@ -419,7 +420,7 @@ def mate_for_resources(ind1: ChromosomeType, ind2: ChromosomeType, mate_position mate_positions = rand.sample(list(range(len(res1))), cxpoint) res1[mate_positions], res2[mate_positions] = res2[mate_positions], res1[mate_positions] - return ind1, ind2 + return ind1, ind2 # это не должно работать так как если mate_positions это массив, то advanced indexing вернет копию def mate_for_resource_borders(ind1: ChromosomeType, ind2: ChromosomeType, diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 0f0fdcd6..aaf60c44 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -229,7 +229,7 @@ def build_schedule(wg: WorkGraph, if show_fitness_graph: fitness_history.append(sum(fitness) / len(fitness)) - generation = 0 + generation = 1 # the best fitness, track to increase performance by stopping evaluation when not decreasing prev_best_fitness = Time.inf() @@ -239,7 +239,7 @@ def build_schedule(wg: WorkGraph, plateau_steps = 0 max_plateau_steps = generation_number # 8 - while generation < generation_number and plateau_steps < max_plateau_steps \ + while generation <= generation_number and plateau_steps < max_plateau_steps \ and (time_border is None or time.time() - global_start < time_border): print(f'-- Generation {generation}, population={len(pop)}, best time={best_fitness} --') if best_fitness == prev_best_fitness: @@ -250,7 +250,6 @@ def build_schedule(wg: WorkGraph, # select individuals of next generation # offspring = toolbox.select(pop, int(math.sqrt(len(pop)))) - offspring = toolbox.select(pop, population_size) # clone selected individuals # offspring = [toolbox.clone(ind) for ind in offspring] @@ -262,22 +261,23 @@ def build_schedule(wg: WorkGraph, cur_generation = [] - for child1, child2 in zip(offspring[::2], offspring[1::2]): - if rand.random() < cxpb: - ind1, ind2 = toolbox.mate(child1[0], child2[0]) - # add to population - cur_generation.append(wrap(ind1)) - cur_generation.append(wrap(ind2)) + for child1, child2 in zip(pop[::2], pop[1::2]): + # if rand.random() < cxpb: + ind1, ind2 = toolbox.mate(child1[0], child2[0]) + # add to population + cur_generation.append(wrap(ind1)) + cur_generation.append(wrap(ind2)) # mutation # take 1 individuals as input and return 1 individuals as output - for mutant in offspring: + for i, mutant in enumerate(cur_generation): if rand.random() < mutpb: ind_order = toolbox.mutate(mutant[0][0]) ind = copy_chromosome(mutant[0]) ind = (ind_order[0], ind[1], ind[2], ind[3]) # add to population - cur_generation.append(wrap(ind)) + cur_generation[i] = wrap(ind) + # cur_generation.append(wrap(ind)) if worker_name2index: # operations for RESOURCES @@ -292,24 +292,26 @@ def build_schedule(wg: WorkGraph, for worker in workers: low = resources_border[0, worker] if worker != len(worker_name2index) else 0 up = resources_border[1, worker] if worker != len(worker_name2index) else 0 - for mutant in offspring: + for i, mutant in enumerate(cur_generation): if rand.random() < mutpb_res: ind = toolbox.mutate_resources(mutant[0], low=low, up=up, type_of_worker=worker) # add to population - cur_generation.append(wrap(ind)) + cur_generation[i] = wrap(ind) + # cur_generation.append(wrap(ind)) # resource borders mutation for worker in workers: if worker == len(worker_name2index): continue - for mutant in offspring: + for i, mutant in enumerate(cur_generation): if rand.random() < mutpb_res: ind = toolbox.mutate_resource_borders(mutant[0], contractors_capacity=contractors_capacity, resources_min_border=resources_min_border, type_of_worker=worker) # add to population - cur_generation.append(wrap(ind)) + cur_generation[i] = wrap(ind) + # cur_generation.append(wrap(ind)) # for the crossover, we use those types that did not participate # in the mutation(+1 means contractor 'resource') @@ -319,7 +321,7 @@ def build_schedule(wg: WorkGraph, workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - for child1, child2 in zip(offspring[::2], offspring[1::2]): + for child1, child2 in zip(pop[:len(pop)//2], pop[len(pop)//2:]): for ind_worker in workers: # mate resources if rand.random() < cxpb_res: @@ -341,17 +343,17 @@ def build_schedule(wg: WorkGraph, evaluation_start = time.time() # Gather all the fitness in one list and print the stats - invalid_ind = [ind for ind in cur_generation if toolbox.validate(ind[0])] + offspring = [ind for ind in cur_generation if toolbox.validate(ind[0])] + print(f'Len offspring: {len(offspring)}') # for each individual - evaluation # print(pool.map(lambda x: x + 2, range(10))) - invalid_fit = fitness_f.evaluate([ind[0] for ind in invalid_ind]) - for fit, ind in zip(invalid_fit, invalid_ind): + invalid_fit = fitness_f.evaluate([ind[0] for ind in offspring]) + for fit, ind in zip(invalid_fit, offspring): ind.fitness.values = [fit] evaluation_time += time.time() - evaluation_start # add mutant part of generation to offspring - offspring.extend(invalid_ind) cur_generation.clear() if show_fitness_graph: @@ -364,7 +366,8 @@ def build_schedule(wg: WorkGraph, # print(f'----| Filtered out {pop_size - len(pop)} invalid individuals') # renewing population - pop[:] = offspring + pop += offspring + pop = toolbox.select(pop, population_size) hof.update(pop) best_fitness = hof[0].fitness.values[0] diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 41df205a..710ea853 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -4,10 +4,10 @@ def test_multiprocessing(setup_scheduler_parameters): setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters - genetic = GeneticScheduler(number_of_generation=500, - mutate_order=1.0, - mutate_resources=1.0, - size_of_population=100, - size_selection=500) + genetic = GeneticScheduler(number_of_generation=50, + mutate_order=0.01, + mutate_resources=0.01, + size_of_population=50, + size_selection=50) genetic.schedule(setup_wg, setup_contractors) From 022069adce4b514952d5a8cc604cbc8c78b7d316 Mon Sep 17 00:00:00 2001 From: Egor Date: Tue, 15 Aug 2023 20:18:32 +0300 Subject: [PATCH 05/22] changing crossover --- sampo/scheduler/genetic/operators.py | 39 ++--- sampo/scheduler/genetic/schedule_builder.py | 152 ++++++++------------ tests/scheduler/genetic/full_scheduling.py | 6 +- 3 files changed, 71 insertions(+), 126 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 6166a6cb..437a6ed3 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -156,10 +156,8 @@ def init_toolbox(wg: WorkGraph, contractor2index=contractor2index, contractor_borders=contractor_borders, spec=spec, init_chromosomes=init_chromosomes, rand=rand, work_estimator=work_estimator, landscape=landscape) - # create from generate_chromosome function one individual - toolbox.register('individual', tools.initRepeat, Individual, toolbox.generate_chromosome, n=1) # create population from individuals - toolbox.register('population', tools.initRepeat, list, toolbox.individual) + toolbox.register('population', tools.initRepeat, list, lambda: Individual(toolbox.generate_chromosome())) # crossover for order toolbox.register('mate', mate_scheduling_order, rand=rand) # mutation for order. Coefficient luke one or two mutation in individual @@ -196,19 +194,6 @@ def copy_chromosome(chromosome: ChromosomeType) -> ChromosomeType: return chromosome[0].copy(), chromosome[1].copy(), chromosome[2].copy(), deepcopy(chromosome[3]) -def wrap(chromosome: ChromosomeType) -> Individual: - """ - Created an individual from chromosome. - """ - - def ind_getter(): - return chromosome - - ind = initRepeat(Individual, ind_getter, n=1) - ind.fitness.invalid_steps = 0 - return ind - - def generate_chromosome(wg: WorkGraph, contractors: list[Contractor], work_id2index: dict[str, int], @@ -317,11 +302,11 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand :return: two cross individuals """ - ind1 = copy_chromosome(ind1) - ind2 = copy_chromosome(ind2) + child1 = Individual(copy_chromosome(ind1)) + child2 = Individual(copy_chromosome(ind2)) - order1 = ind1[0] - order2 = ind2[0] + order1 = child1[0] + order2 = child2[0] border = len(order1) // 4 # randomly select the point where the crossover will take place @@ -333,7 +318,7 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand order1[crossover_point:] = ind1_new_tail order2[crossover_point:] = ind2_new_tail - return ind1, ind2 + return child1, child2 def mut_uniform_int(ind: ChromosomeType, low: np.ndarray, up: np.ndarray, type_of_worker: int, @@ -346,7 +331,7 @@ def mut_uniform_int(ind: ChromosomeType, low: np.ndarray, up: np.ndarray, type_o :param up: upper bound specified by `WorkUnit` :return: mutate individual """ - ind = copy_chromosome(ind) + # ind = copy_chromosome(ind) # select random number from interval from min to max from uniform distribution size = len(ind[1]) @@ -377,7 +362,7 @@ def mutate_resource_borders(ind: ChromosomeType, contractors_capacity: np.ndarra """ Mutation for contractors' resource borders. """ - ind = copy_chromosome(ind) + # ind = copy_chromosome(ind) num_resources = len(resources_min_border) num_contractors = len(ind[2]) @@ -409,8 +394,8 @@ def mate_for_resources(ind1: ChromosomeType, ind2: ChromosomeType, mate_position :param rand: the rand object used for exchange point selection :return: first and second individual """ - ind1 = copy_chromosome(ind1) - ind2 = copy_chromosome(ind2) + ind1 = Individual(copy_chromosome(ind1)) + ind2 = Individual(copy_chromosome(ind2)) # exchange work resources res1 = ind1[1][:, mate_positions] @@ -428,8 +413,8 @@ def mate_for_resource_borders(ind1: ChromosomeType, ind2: ChromosomeType, """ Crossover for contractors' resource borders. """ - ind1 = copy_chromosome(ind1) - ind2 = copy_chromosome(ind2) + # ind1 = copy_chromosome(ind1) + # ind2 = copy_chromosome(ind2) num_contractors = len(ind1[2]) contractors_to_mate = rand.sample(list(range(num_contractors)), rand.randint(1, num_contractors)) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index aaf60c44..f30f6136 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -13,7 +13,7 @@ from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, copy_chromosome, \ - FitnessFunction, TimeFitness, is_chromosome_correct, wrap + FitnessFunction, TimeFitness, is_chromosome_correct from sampo.scheduler.native_wrapper import NativeWrapper from sampo.scheduler.timeline.base import Timeline from sampo.schemas.contractor import Contractor, WorkerContractorPool @@ -95,7 +95,7 @@ def create_toolbox(wg: WorkGraph, init_chromosomes: dict[str, ChromosomeType] = \ {name: convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec, order) - if schedule is not None else None + if schedule is not None else None for name, (schedule, order, spec) in init_schedules.items()} for name, chromosome in init_chromosomes.items(): @@ -199,7 +199,13 @@ def build_schedule(wg: WorkGraph, print(f'Toolbox initialization & first population took {(time.time() - start) * 1000} ms') - if not native.native: + if native.native: + native_start = time.time() + best_chromosome = native.run_genetic(list([ind[0] for ind in pop]), + mutate_order, mutate_order, mutate_resources, mutate_resources, + mutate_resources, mutate_resources, selection_size) + print(f'Native evaluated in {(time.time() - native_start) * 1000} ms') + else: # save best individuals hof = tools.HallOfFame(1, similar=compare_individuals) @@ -207,16 +213,13 @@ def build_schedule(wg: WorkGraph, cxpb, mutpb = mutate_order, mutate_order mutpb_res, cxpb_res = mutate_resources, mutate_resources - # def evaluate_chromosomes(chromosomes: list[ChromosomeType]): - # return native.evaluate([chromo for chromo in chromosomes if toolbox.validate(chromo)]) - fitness_f = fitness_constructor(native.evaluate) start = time.time() # map to each individual fitness function - pop = [ind for ind in pop if toolbox.validate(ind[0])] - fitness = fitness_f.evaluate([ind[0] for ind in pop]) + pop = [ind for ind in pop if toolbox.validate(ind)] + fitness = fitness_f.evaluate([ind for ind in pop]) evaluation_time = time.time() - start @@ -237,7 +240,7 @@ def build_schedule(wg: WorkGraph, start = time.time() plateau_steps = 0 - max_plateau_steps = generation_number # 8 + max_plateau_steps = generation_number while generation <= generation_number and plateau_steps < max_plateau_steps \ and (time_border is None or time.time() - global_start < time_border): @@ -248,36 +251,14 @@ def build_schedule(wg: WorkGraph, plateau_steps = 0 prev_best_fitness = best_fitness - # select individuals of next generation - # offspring = toolbox.select(pop, int(math.sqrt(len(pop)))) - - # clone selected individuals - # offspring = [toolbox.clone(ind) for ind in offspring] - - # operations for ORDER - # crossover - # take 2 individuals as input 1 modified individuals - # take after 1: (1,3,5) and (2,4,6) and get pairs 1,2; 3,4; 5,6 - cur_generation = [] - for child1, child2 in zip(pop[::2], pop[1::2]): + for ind1, ind2 in zip(pop[::2], pop[1::2]): # if rand.random() < cxpb: - ind1, ind2 = toolbox.mate(child1[0], child2[0]) + child1, child2 = toolbox.mate(ind1, ind2) # add to population - cur_generation.append(wrap(ind1)) - cur_generation.append(wrap(ind2)) - - # mutation - # take 1 individuals as input and return 1 individuals as output - for i, mutant in enumerate(cur_generation): - if rand.random() < mutpb: - ind_order = toolbox.mutate(mutant[0][0]) - ind = copy_chromosome(mutant[0]) - ind = (ind_order[0], ind[1], ind[2], ind[3]) - # add to population - cur_generation[i] = wrap(ind) - # cur_generation.append(wrap(ind)) + cur_generation.append(child1) + cur_generation.append(child2) if worker_name2index: # operations for RESOURCES @@ -285,33 +266,6 @@ def build_schedule(wg: WorkGraph, # select types for mutation # numbers of changing types number_of_type_for_changing = rand.randint(1, len(worker_name2index) - 1) - # workers type for changing(+1 means contractor 'resource') - workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - - # resources mutation - for worker in workers: - low = resources_border[0, worker] if worker != len(worker_name2index) else 0 - up = resources_border[1, worker] if worker != len(worker_name2index) else 0 - for i, mutant in enumerate(cur_generation): - if rand.random() < mutpb_res: - ind = toolbox.mutate_resources(mutant[0], low=low, up=up, type_of_worker=worker) - # add to population - cur_generation[i] = wrap(ind) - # cur_generation.append(wrap(ind)) - - # resource borders mutation - for worker in workers: - if worker == len(worker_name2index): - continue - for i, mutant in enumerate(cur_generation): - if rand.random() < mutpb_res: - ind = toolbox.mutate_resource_borders(mutant[0], - contractors_capacity=contractors_capacity, - resources_min_border=resources_min_border, - type_of_worker=worker) - # add to population - cur_generation[i] = wrap(ind) - # cur_generation.append(wrap(ind)) # for the crossover, we use those types that did not participate # in the mutation(+1 means contractor 'resource') @@ -321,50 +275,68 @@ def build_schedule(wg: WorkGraph, workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - for child1, child2 in zip(pop[:len(pop)//2], pop[len(pop)//2:]): + for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): for ind_worker in workers: # mate resources if rand.random() < cxpb_res: - ind1, ind2 = toolbox.mate_resources(child1[0], child2[0], ind_worker) + child1, child2 = toolbox.mate_resources(ind1, ind2, ind_worker) # add to population - cur_generation.append(wrap(ind1)) - cur_generation.append(wrap(ind2)) + cur_generation.append(child1) + cur_generation.append(child2) # mate resource borders if rand.random() < cxpb_res: if ind_worker == len(worker_name2index): continue - ind1, ind2 = toolbox.mate_resource_borders(child1[0], child2[0], ind_worker) - + child1, child2 = toolbox.mate_resource_borders(ind1, ind2, ind_worker) # add to population - cur_generation.append(wrap(ind1)) - cur_generation.append(wrap(ind2)) + cur_generation.append(child1) + cur_generation.append(child2) + + # workers type for changing(+1 means contractor 'resource') + workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) + + # resources mutation + for worker in workers: + low = resources_border[0, worker] if worker != len(worker_name2index) else 0 + up = resources_border[1, worker] if worker != len(worker_name2index) else 0 + for mutant in cur_generation: + if rand.random() < mutpb_res: + toolbox.mutate_resources(mutant, low=low, up=up, type_of_worker=worker) + + # resource borders mutation + for worker in workers: + if worker == len(worker_name2index): + continue + for mutant in cur_generation: + if rand.random() < mutpb_res: + toolbox.mutate_resource_borders(mutant, + contractors_capacity=contractors_capacity, + resources_min_border=resources_min_border, + type_of_worker=worker) + + # mutation + # take 1 individuals as input and return 1 individuals as output + for mutant in cur_generation: + if rand.random() < mutpb: + toolbox.mutate(mutant[0]) evaluation_start = time.time() # Gather all the fitness in one list and print the stats - offspring = [ind for ind in cur_generation if toolbox.validate(ind[0])] + offspring = [ind for ind in cur_generation if toolbox.validate(ind)] print(f'Len offspring: {len(offspring)}') - # for each individual - evaluation - # print(pool.map(lambda x: x + 2, range(10))) - invalid_fit = fitness_f.evaluate([ind[0] for ind in offspring]) + invalid_fit = fitness_f.evaluate([ind for ind in offspring]) for fit, ind in zip(invalid_fit, offspring): ind.fitness.values = [fit] evaluation_time += time.time() - evaluation_start - # add mutant part of generation to offspring - cur_generation.clear() - if show_fitness_graph: _ftn = [f for f in fitness if not math.isinf(f)] if len(_ftn) > 0: fitness_history.append(sum(_ftn) / len(_ftn)) - # pop_size = len(pop) - # pop = [ind for ind in pop if valid(ind)] - # print(f'----| Filtered out {pop_size - len(pop)} invalid individuals') - # renewing population pop += offspring pop = toolbox.select(pop, population_size) @@ -372,32 +344,20 @@ def build_schedule(wg: WorkGraph, best_fitness = hof[0].fitness.values[0] - # best = hof[0] - # fits = [ind.fitness.values[0] for ind in pop] - # evaluation = chromosome_evaluation(best, index2node, resources_border, work_id2index, worker_name2index, - # parent2inseparable_son, agents) - # print('fits: ', fits) - # print(evaluation) generation += 1 native.close() - chromosome = hof[0][0] + best_chromosome = hof[0] # assert that we have valid chromosome assert hof[0].fitness.values[0] != Time.inf() - print(f'Final time: {hof[0].fitness.values[0]}') + print(f'Final time: {best_fitness}') print(f'Generations processing took {(time.time() - start) * 1000} ms') print(f'Evaluation time: {evaluation_time * 1000}') - else: - native_start = time.time() - chromosome = native.run_genetic(list([ind[0] for ind in pop]), - mutate_order, mutate_order, mutate_resources, mutate_resources, - mutate_resources, mutate_resources, selection_size) - print(f'Native evaluated in {(time.time() - native_start) * 1000} ms') - scheduled_works, schedule_start_time, timeline, order_nodes = toolbox.chromosome_to_schedule(chromosome, + scheduled_works, schedule_start_time, timeline, order_nodes = toolbox.chromosome_to_schedule(best_chromosome, landscape=landscape, timeline=timeline) diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 710ea853..77636e40 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -5,9 +5,9 @@ def test_multiprocessing(setup_scheduler_parameters): setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters genetic = GeneticScheduler(number_of_generation=50, - mutate_order=0.01, - mutate_resources=0.01, + mutate_order=0.0, + mutate_resources=0.0, size_of_population=50, - size_selection=50) + size_selection=100) genetic.schedule(setup_wg, setup_contractors) From 35a0b2a88e141ea9e3bc677342f96ba3cb8123e3 Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 16 Aug 2023 17:31:48 +0300 Subject: [PATCH 06/22] changing mutation --- sampo/scheduler/genetic/operators.py | 12 +++++- sampo/scheduler/genetic/schedule_builder.py | 47 ++++++++++----------- tests/scheduler/genetic/full_scheduling.py | 4 +- 3 files changed, 36 insertions(+), 27 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 437a6ed3..8d2536a4 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -161,7 +161,8 @@ def init_toolbox(wg: WorkGraph, # crossover for order toolbox.register('mate', mate_scheduling_order, rand=rand) # mutation for order. Coefficient luke one or two mutation in individual - toolbox.register('mutate', tools.mutShuffleIndexes, indpb=mutate_order) + # toolbox.register('mutate', tools.mutShuffleIndexes, indpb=mutate_order) + toolbox.register('mutate', mutate_scheduling_order, indpb=mutate_order, rand=rand) # selection. Some random individuals and arranges a battle between them as a result in a continuing genus, # this is the best among these it toolbox.register('select', tools.selTournament, tournsize=selection_size) @@ -321,6 +322,15 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand return child1, child2 +def mutate_scheduling_order(ind: ChromosomeType, indpb: float, rand: random.Random) -> ChromosomeType: + order = ind[0] + for i in range(1, len(order) - 2): + if rand.random() < indpb: + order[i], order[i + 1] = order[i + 1], order[i] + + return ind + + def mut_uniform_int(ind: ChromosomeType, low: np.ndarray, up: np.ndarray, type_of_worker: int, probability_mutate_resources: float, contractor_count: int, rand: random.Random) -> ChromosomeType: """ diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index f30f6136..c97a4e92 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -273,25 +273,25 @@ def build_schedule(wg: WorkGraph, # crossover # take 2 individuals as input 1 modified individuals - workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - - for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): - for ind_worker in workers: - # mate resources - if rand.random() < cxpb_res: - child1, child2 = toolbox.mate_resources(ind1, ind2, ind_worker) - # add to population - cur_generation.append(child1) - cur_generation.append(child2) - - # mate resource borders - if rand.random() < cxpb_res: - if ind_worker == len(worker_name2index): - continue - child1, child2 = toolbox.mate_resource_borders(ind1, ind2, ind_worker) - # add to population - cur_generation.append(child1) - cur_generation.append(child2) + # workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) + # + # for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): + # for ind_worker in workers: + # # mate resources + # if rand.random() < cxpb_res: + # child1, child2 = toolbox.mate_resources(ind1, ind2, ind_worker) + # # add to population + # cur_generation.append(child1) + # cur_generation.append(child2) + # + # # mate resource borders + # if rand.random() < cxpb_res: + # if ind_worker == len(worker_name2index): + # continue + # child1, child2 = toolbox.mate_resource_borders(ind1, ind2, ind_worker) + # # add to population + # cur_generation.append(child1) + # cur_generation.append(child2) # workers type for changing(+1 means contractor 'resource') workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) @@ -318,8 +318,7 @@ def build_schedule(wg: WorkGraph, # mutation # take 1 individuals as input and return 1 individuals as output for mutant in cur_generation: - if rand.random() < mutpb: - toolbox.mutate(mutant[0]) + toolbox.mutate(mutant) evaluation_start = time.time() @@ -327,8 +326,8 @@ def build_schedule(wg: WorkGraph, offspring = [ind for ind in cur_generation if toolbox.validate(ind)] print(f'Len offspring: {len(offspring)}') - invalid_fit = fitness_f.evaluate([ind for ind in offspring]) - for fit, ind in zip(invalid_fit, offspring): + offspring_fitness = fitness_f.evaluate(offspring) + for fit, ind in zip(offspring_fitness, offspring): ind.fitness.values = [fit] evaluation_time += time.time() - evaluation_start @@ -374,4 +373,4 @@ def build_schedule(wg: WorkGraph, def compare_individuals(first: tuple[ChromosomeType], second: tuple[ChromosomeType]): - return (first[0][0] == second[0][0]).all() and (first[0][1] == second[0][1]).all() + return (first[0] == second[0]).all() and (first[1] == second[1]).all() diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 77636e40..5cf3084b 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -5,8 +5,8 @@ def test_multiprocessing(setup_scheduler_parameters): setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters genetic = GeneticScheduler(number_of_generation=50, - mutate_order=0.0, - mutate_resources=0.0, + mutate_order=0.05, + mutate_resources=0.01, size_of_population=50, size_selection=100) From 43f01dddc326f672dc592fa5ca44a25f23744d81 Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 16 Aug 2023 18:54:35 +0300 Subject: [PATCH 07/22] changing selection --- sampo/scheduler/genetic/operators.py | 16 +++++++++++++++- sampo/scheduler/genetic/schedule_builder.py | 18 +++++++++++++++--- 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 8d2536a4..d2e7c6f9 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -3,6 +3,7 @@ from copy import deepcopy from functools import partial from typing import Iterable, Callable +from operator import attrgetter import numpy as np from deap import creator, base, tools @@ -134,6 +135,7 @@ def init_toolbox(wg: WorkGraph, mutate_order: float, mutate_resources: float, selection_size: int, + population_size: int, rand: random.Random, spec: ScheduleSpec, worker_pool_indices: dict[int, dict[int, Worker]], @@ -165,7 +167,8 @@ def init_toolbox(wg: WorkGraph, toolbox.register('mutate', mutate_scheduling_order, indpb=mutate_order, rand=rand) # selection. Some random individuals and arranges a battle between them as a result in a continuing genus, # this is the best among these it - toolbox.register('select', tools.selTournament, tournsize=selection_size) + # toolbox.register('select', tools.selTournament, tournsize=selection_size) + toolbox.register('select', select_new_population, pop_size=population_size) # mutation for resources toolbox.register('mutate_resources', mut_uniform_int, probability_mutate_resources=mutate_resources, @@ -244,6 +247,11 @@ def randomized_init() -> ChromosomeType: return chromosome +def select_new_population(population: list[ChromosomeType], pop_size: int) -> list[ChromosomeType]: + population.sort(key=attrgetter('fitness'), reverse=True) + return population[:pop_size] + + def is_chromosome_correct(chromosome: ChromosomeType, node_indices: list[int], parents: dict[int, list[int]]) -> bool: @@ -305,6 +313,8 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand """ child1 = Individual(copy_chromosome(ind1)) child2 = Individual(copy_chromosome(ind2)) + child1.type = 'mate' + child2.type = 'mate' order1 = child1[0] order2 = child2[0] @@ -326,6 +336,7 @@ def mutate_scheduling_order(ind: ChromosomeType, indpb: float, rand: random.Rand order = ind[0] for i in range(1, len(order) - 2): if rand.random() < indpb: + ind.type = 'mutate' order[i], order[i + 1] = order[i + 1], order[i] return ind @@ -351,12 +362,14 @@ def mut_uniform_int(ind: ChromosomeType, low: np.ndarray, up: np.ndarray, type_o # print('Contractor mutation!') for i in range(size): if rand.random() < probability_mutate_resources: + ind.type = 'mutate_uniform_int' ind[1][i][type_of_worker] = rand.randint(0, contractor_count - 1) return ind # change in this interval in random number from interval for i, xl, xu in zip(range(size), low, up): if rand.random() < probability_mutate_resources: + ind.type = 'mutate_uniform_int' # borders contractor = ind[1][i][-1] border = ind[2][contractor][type_of_worker] @@ -378,6 +391,7 @@ def mutate_resource_borders(ind: ChromosomeType, contractors_capacity: np.ndarra num_contractors = len(ind[2]) for contractor in range(num_contractors): if rand.random() < probability_mutate_contractors: + ind.type = 'mutate_resource_borders' ind[2][contractor][type_of_worker] -= rand.randint(resources_min_border[type_of_worker] + 1, max(resources_min_border[type_of_worker] + 1, ind[2][contractor][type_of_worker] // 10)) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index c97a4e92..f7eb621a 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -31,6 +31,7 @@ def create_toolbox(wg: WorkGraph, contractors: list[Contractor], worker_pool: WorkerContractorPool, selection_size: int, + population_size: int, mutate_order: float, mutate_resources: float, init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], @@ -118,6 +119,7 @@ def create_toolbox(wg: WorkGraph, mutate_order, mutate_resources, selection_size, + population_size, rand, spec, worker_pool_indices, @@ -186,6 +188,7 @@ def build_schedule(wg: WorkGraph, toolbox, resources_border, contractors_capacity, resources_min_border = create_toolbox(wg, contractors, worker_pool, selection_size, + population_size, mutate_order, mutate_resources, init_schedules, @@ -260,6 +263,12 @@ def build_schedule(wg: WorkGraph, cur_generation.append(child1) cur_generation.append(child2) + # offspring_fitness = fitness_f.evaluate(cur_generation) + # for fit in offspring_fitness: + # if fit < best_fitness: + # print('crossover') + # break + if worker_name2index: # operations for RESOURCES # mutation @@ -338,10 +347,13 @@ def build_schedule(wg: WorkGraph, # renewing population pop += offspring - pop = toolbox.select(pop, population_size) - hof.update(pop) + # pop = toolbox.select(pop, population_size) + pop = toolbox.select(pop) + hof.update([pop[0]]) best_fitness = hof[0].fitness.values[0] + if best_fitness < prev_best_fitness: + print(hof[0].type) generation += 1 @@ -373,4 +385,4 @@ def build_schedule(wg: WorkGraph, def compare_individuals(first: tuple[ChromosomeType], second: tuple[ChromosomeType]): - return (first[0] == second[0]).all() and (first[1] == second[1]).all() + return (first[0] == second[0]).all() and (first[1] == second[1]).all() and (first[2] == second[2]).all() From fb441238689607b7fc29c3afef55003b71e816db Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 18 Aug 2023 17:44:41 +0300 Subject: [PATCH 08/22] changing population initialization --- experiments/algorithms_2_multi_agency.py | 2 +- experiments/genetic_2_multi_agency.py | 10 ++-- .../multi_agency_comparison.py | 20 +++---- sampo/scheduler/genetic/base.py | 14 +---- sampo/scheduler/genetic/operators.py | 52 +++++++++++++++++-- sampo/scheduler/genetic/schedule_builder.py | 8 ++- tests/scheduler/genetic/fixtures.py | 45 +++++++--------- tests/scheduler/genetic/full_scheduling.py | 3 +- tests/scheduler/genetic/operators_test.py | 14 ++--- 9 files changed, 94 insertions(+), 74 deletions(-) diff --git a/experiments/algorithms_2_multi_agency.py b/experiments/algorithms_2_multi_agency.py index 54e3603a..672bd24c 100644 --- a/experiments/algorithms_2_multi_agency.py +++ b/experiments/algorithms_2_multi_agency.py @@ -27,7 +27,7 @@ def log(message: str, logfile: IO): schedulers = [HEFTScheduler(), HEFTBetweenScheduler(), TopologicalScheduler()] - # GeneticScheduler(50, 50, 0.5, 0.5, 20)] + # GeneticScheduler(50, 0.5, 0.5, 20)] contractors = [p_rand.contractor(10) for _ in range(len(schedulers))] diff --git a/experiments/genetic_2_multi_agency.py b/experiments/genetic_2_multi_agency.py index cf2f0db7..0d944d63 100644 --- a/experiments/genetic_2_multi_agency.py +++ b/experiments/genetic_2_multi_agency.py @@ -18,11 +18,11 @@ def obstruction_getter(i: int): if __name__ == '__main__': - schedulers = [GeneticScheduler(50, 50, 0.5, 0.5, 100),] - # GeneticScheduler(50, 50, 0.5, 0.5, 100), - # GeneticScheduler(50, 50, 0.5, 0.5, 100), - # GeneticScheduler(50, 50, 0.5, 0.5, 100), - # GeneticScheduler(50, 50, 0.5, 0.5, 100)] + schedulers = [GeneticScheduler(50, 0.5, 0.5, 100),] + # GeneticScheduler(50, 0.5, 0.5, 100), + # GeneticScheduler(50, 0.5, 0.5, 100), + # GeneticScheduler(50, 0.5, 0.5, 100), + # GeneticScheduler(50, 0.5, 0.5, 100)] contractors = [p_rand.contractor(10) for _ in range(len(schedulers))] diff --git a/experiments/modular_examples/multi_agency_comparison.py b/experiments/modular_examples/multi_agency_comparison.py index 4f6f2cb3..94f933e4 100644 --- a/experiments/modular_examples/multi_agency_comparison.py +++ b/experiments/modular_examples/multi_agency_comparison.py @@ -74,16 +74,16 @@ def run_iteration(args): if mode == 0: schedulers = [HEFTScheduler(), HEFTBetweenScheduler(), TopologicalScheduler()] else: - schedulers = [GeneticScheduler(50, 50, 0.5, 0.5, 100), - GeneticScheduler(50, 100, 0.25, 0.5, 100), - GeneticScheduler(50, 100, 0.5, 0.75, 100), - GeneticScheduler(50, 100, 0.75, 0.75, 100), - GeneticScheduler(50, 50, 0.9, 0.9, 100), - GeneticScheduler(50, 100, 0.5, 0.5, 500), - GeneticScheduler(50, 200, 0.25, 0.5, 500), - GeneticScheduler(50, 50, 0.5, 0.75, 500), - GeneticScheduler(50, 100, 0.75, 0.75, 500), - GeneticScheduler(50, 50, 0.5, 0.9, 500)] + schedulers = [GeneticScheduler(50, 0.5, 0.5, 100), + GeneticScheduler(50, 0.25, 0.5, 100), + GeneticScheduler(50, 0.5, 0.75, 100), + GeneticScheduler(50, 0.75, 0.75, 100), + GeneticScheduler(50, 0.9, 0.9, 100), + GeneticScheduler(50, 0.5, 0.5, 500), + GeneticScheduler(50, 0.25, 0.5, 500), + GeneticScheduler(50, 0.5, 0.75, 500), + GeneticScheduler(50, 0.75, 0.75, 500), + GeneticScheduler(50, 0.5, 0.9, 500)] variate_block_size(logfile, schedulers) diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index 7d20c2b9..26fc8c6e 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -33,7 +33,6 @@ class GeneticScheduler(Scheduler): def __init__(self, number_of_generation: Optional[int] = 50, - size_selection: Optional[int or None] = None, mutate_order: Optional[float or None] = None, mutate_resources: Optional[float or None] = None, size_of_population: Optional[float or None] = None, @@ -48,7 +47,6 @@ def __init__(self, resource_optimizer=resource_optimizer, work_estimator=work_estimator) self.number_of_generation = number_of_generation - self.size_selection = size_selection self.mutate_order = mutate_order self.mutate_resources = mutate_resources self.size_of_population = size_of_population @@ -75,13 +73,6 @@ def get_params(self, works_count: int) -> tuple[int, float, float, int]: :param works_count: :return: """ - size_selection = self.size_selection - if size_selection is None: - if works_count < 300: - size_selection = 20 - else: - size_selection = works_count // 15 - mutate_order = self.mutate_order if mutate_order is None: if works_count < 300: @@ -104,7 +95,7 @@ def get_params(self, works_count: int) -> tuple[int, float, float, int]: size_of_population = 50 else: size_of_population = works_count // 50 - return size_selection, mutate_order, mutate_resources, size_of_population + return mutate_order, mutate_resources, size_of_population def set_use_multiprocessing(self, n_cpu: int): """ @@ -200,7 +191,7 @@ def schedule_with_cache(self, """ init_schedules = self.generate_first_population(wg, contractors, landscape) - size_selection, mutate_order, mutate_resources, size_of_population = self.get_params(wg.vertex_count) + mutate_order, mutate_resources, size_of_population = self.get_params(wg.vertex_count) worker_pool = get_worker_contractor_pool(contractors) scheduled_works, schedule_start_time, timeline, order_nodes = build_schedule(wg, @@ -208,7 +199,6 @@ def schedule_with_cache(self, worker_pool, size_of_population, self.number_of_generation, - size_selection, mutate_order, mutate_resources, init_schedules, diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index d2e7c6f9..5ecfd73e 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -7,7 +7,6 @@ import numpy as np from deap import creator, base, tools -from deap.tools import initRepeat from sampo.scheduler.genetic.converter import convert_chromosome_to_schedule from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome, ChromosomeType @@ -134,7 +133,6 @@ def init_toolbox(wg: WorkGraph, init_chromosomes: dict[str, ChromosomeType], mutate_order: float, mutate_resources: float, - selection_size: int, population_size: int, rand: random.Random, spec: ScheduleSpec, @@ -159,7 +157,11 @@ def init_toolbox(wg: WorkGraph, init_chromosomes=init_chromosomes, rand=rand, work_estimator=work_estimator, landscape=landscape) # create population from individuals - toolbox.register('population', tools.initRepeat, list, lambda: Individual(toolbox.generate_chromosome())) + # toolbox.register('population', tools.initRepeat, list, lambda: toolbox.generate_chromosome()) + toolbox.register('population', generate_population, wg=wg, contractors=contractors, + work_id2index=work_id2index, worker_name2index=worker_name2index, + contractor2index=contractor2index, contractor_borders=contractor_borders, spec=spec, + init_chromosomes=init_chromosomes, rand=rand, work_estimator=work_estimator, landscape=landscape) # crossover for order toolbox.register('mate', mate_scheduling_order, rand=rand) # mutation for order. Coefficient luke one or two mutation in individual @@ -198,6 +200,46 @@ def copy_chromosome(chromosome: ChromosomeType) -> ChromosomeType: return chromosome[0].copy(), chromosome[1].copy(), chromosome[2].copy(), deepcopy(chromosome[3]) +def generate_population(n: int, + wg: WorkGraph, + contractors: list[Contractor], + work_id2index: dict[str, int], + worker_name2index: dict[str, int], + contractor2index: dict[str, int], + contractor_borders: np.ndarray, + init_chromosomes: dict[str, ChromosomeType], + spec: ScheduleSpec, + rand: random.Random, + work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), + landscape: LandscapeConfiguration = LandscapeConfiguration()) -> list[Individual]: + """ + Generates population using chromosome weights. + Do not use `generate_chromosome` function. + """ + def randomized_init() -> ChromosomeType: + schedule = RandomizedTopologicalScheduler(work_estimator, + int(rand.random() * 1000000)) \ + .schedule(wg, contractors, landscape=landscape) + return convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, + contractor2index, contractor_borders, schedule, spec) + + # chromosome types' weights + # these numbers are the probability weights: prob = norm(weights), sum(prob) = 1 + count_for_specified_types = (n // 3) // len(init_chromosomes) + count_for_specified_types = count_for_specified_types if count_for_specified_types > 0 else 1 + count_for_topological = n - count_for_specified_types * len(init_chromosomes) + count_for_topological = count_for_topological if count_for_topological > 0 else 1 + counts = [count_for_specified_types for _ in range(len(init_chromosomes))] + [count_for_topological] + + chromosome_types = rand.sample(list(init_chromosomes.keys()) + ['topological'], k=n, counts=counts) + + chromosomes = [Individual(init_chromosomes[generated_type]) + if generated_type != 'topological' else Individual(randomized_init()) + for generated_type in chromosome_types] + + return chromosomes + + def generate_chromosome(wg: WorkGraph, contractors: list[Contractor], work_id2index: dict[str, int], @@ -208,7 +250,7 @@ def generate_chromosome(wg: WorkGraph, spec: ScheduleSpec, rand: random.Random, work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), - landscape: LandscapeConfiguration = LandscapeConfiguration()) -> ChromosomeType: + landscape: LandscapeConfiguration = LandscapeConfiguration()) -> Individual: """ It is necessary to generate valid scheduling, which are satisfied to current dependencies That's why will be used the approved order of works (HEFT order and Topological sorting) @@ -244,7 +286,7 @@ def randomized_init() -> ChromosomeType: if chromosome is None: chromosome = randomized_init() - return chromosome + return Individual(chromosome) def select_new_population(population: list[ChromosomeType], pop_size: int) -> list[ChromosomeType]: diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index f7eb621a..97471147 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -30,7 +30,6 @@ def create_toolbox(wg: WorkGraph, contractors: list[Contractor], worker_pool: WorkerContractorPool, - selection_size: int, population_size: int, mutate_order: float, mutate_resources: float, @@ -118,7 +117,6 @@ def create_toolbox(wg: WorkGraph, init_chromosomes, mutate_order, mutate_resources, - selection_size, population_size, rand, spec, @@ -136,7 +134,6 @@ def build_schedule(wg: WorkGraph, worker_pool: WorkerContractorPool, population_size: int, generation_number: int, - selection_size: int, mutate_order: float, mutate_resources: float, init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], @@ -187,7 +184,6 @@ def build_schedule(wg: WorkGraph, start = time.time() toolbox, resources_border, contractors_capacity, resources_min_border = create_toolbox(wg, contractors, worker_pool, - selection_size, population_size, mutate_order, mutate_resources, @@ -206,7 +202,7 @@ def build_schedule(wg: WorkGraph, native_start = time.time() best_chromosome = native.run_genetic(list([ind[0] for ind in pop]), mutate_order, mutate_order, mutate_resources, mutate_resources, - mutate_resources, mutate_resources, selection_size) + mutate_resources, mutate_resources, population_size) print(f'Native evaluated in {(time.time() - native_start) * 1000} ms') else: # save best individuals @@ -254,6 +250,8 @@ def build_schedule(wg: WorkGraph, plateau_steps = 0 prev_best_fitness = best_fitness + rand.shuffle(pop) + cur_generation = [] for ind1, ind2 in zip(pop[::2], pop[1::2]): diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index 535071cc..84e3cb84 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -17,12 +17,7 @@ from sampo.utilities.collections_util import reverse_dictionary -def get_params(works_count: int) -> Tuple[int, float, float, int]: - if works_count < 300: - size_selection = 20 - else: - size_selection = works_count // 15 - +def get_params(works_count: int) -> Tuple[float, float, int]: if works_count < 300: mutate_order = 0.006 else: @@ -39,47 +34,42 @@ def get_params(works_count: int) -> Tuple[int, float, float, int]: size_of_population = 50 else: size_of_population = works_count // 50 - return size_selection, mutate_order, mutate_resources, size_of_population + return mutate_order, mutate_resources, size_of_population def create_toolbox(wg: WorkGraph, - contractors: List[Contractor], + contractors: list[Contractor], worker_pool: WorkerContractorPool, - selection_size: int, + population_size: int, mutate_order: float, mutate_resources: float, init_schedules: Dict[str, Schedule], rand: Random, spec: ScheduleSpec = ScheduleSpec(), - work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), - landscape: LandscapeConfiguration = LandscapeConfiguration()) -> Tuple[Toolbox, np.ndarray]: + work_estimator: WorkTimeEstimator = None, + landscape: LandscapeConfiguration = LandscapeConfiguration()) \ + -> tuple[Toolbox, np.ndarray]: + + # preparing access-optimized data structures nodes = [node for node in wg.nodes if not node.is_inseparable_son()] - index2node: Dict[int, GraphNode] = {index: node for index, node in enumerate(nodes)} - work_id2index: Dict[str, int] = {node.id: index for index, node in index2node.items()} + index2node: dict[int, GraphNode] = {index: node for index, node in enumerate(nodes)} + work_id2index: dict[str, int] = {node.id: index for index, node in index2node.items()} worker_name2index = {worker_name: index for index, worker_name in enumerate(worker_pool)} index2contractor = {ind: contractor.id for ind, contractor in enumerate(contractors)} index2contractor_obj = {ind: contractor for ind, contractor in enumerate(contractors)} contractor2index = reverse_dictionary(index2contractor) - index2node_list = [(index, node) for index, node in enumerate(nodes)] worker_pool_indices = {worker_name2index[worker_name]: { contractor2index[contractor_id]: worker for contractor_id, worker in workers_of_type.items() } for worker_name, workers_of_type in worker_pool.items()} node_indices = list(range(len(nodes))) - contractors_capacity = np.zeros((len(contractors), len(worker_pool))) - for w_ind, cont2worker in worker_pool_indices.items(): - for c_ind, worker in cont2worker.items(): - contractors_capacity[c_ind][w_ind] = worker.count - resources_border = np.zeros((2, len(worker_pool), len(index2node))) - resources_min_border = np.zeros((len(worker_pool))) for work_index, node in index2node.items(): for req in node.work_unit.worker_reqs: worker_index = worker_name2index[req.kind] resources_border[0, worker_index, work_index] = req.min_count resources_border[1, worker_index, work_index] = req.max_count - resources_min_border[worker_index] = max(resources_min_border[worker_index], req.min_count) contractor_borders = np.zeros((len(contractor2index), len(worker_name2index)), dtype=int) for ind, contractor in enumerate(contractors): @@ -103,6 +93,7 @@ def create_toolbox(wg: WorkGraph, for child in node_children: parents[child].append(node) + # initial chromosomes construction init_chromosomes: Dict[str, ChromosomeType] = \ {name: convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec) @@ -120,7 +111,7 @@ def create_toolbox(wg: WorkGraph, init_chromosomes, mutate_order, mutate_resources, - selection_size, + population_size, rand, spec, worker_pool_indices, @@ -137,18 +128,18 @@ def setup_toolbox(setup_default_schedules) -> tuple: (setup_wg, setup_contractors, setup_landscape_many_holders), setup_default_schedules = setup_default_schedules setup_worker_pool = get_worker_contractor_pool(setup_contractors) - selection_size, mutate_order, mutate_resources, size_of_population = get_params(setup_wg.vertex_count) + mutate_order, mutate_resources, size_of_population = get_params(setup_wg.vertex_count) rand = Random(123) work_estimator: WorkTimeEstimator = DefaultWorkEstimator() - return create_toolbox(setup_wg, + return (create_toolbox(setup_wg, setup_contractors, setup_worker_pool, - selection_size, + size_of_population, mutate_order, mutate_resources, setup_default_schedules, rand, work_estimator=work_estimator, - landscape=setup_landscape_many_holders), setup_wg, setup_contractors, setup_default_schedules, \ - setup_landscape_many_holders + landscape=setup_landscape_many_holders), \ + setup_wg, setup_contractors, setup_default_schedules, setup_landscape_many_holders) diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 5cf3084b..ab7df489 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -7,7 +7,6 @@ def test_multiprocessing(setup_scheduler_parameters): genetic = GeneticScheduler(number_of_generation=50, mutate_order=0.05, mutate_resources=0.01, - size_of_population=50, - size_selection=100) + size_of_population=50) genetic.schedule(setup_wg, setup_contractors) diff --git a/tests/scheduler/genetic/operators_test.py b/tests/scheduler/genetic/operators_test.py index 3e87ff85..4e10506b 100644 --- a/tests/scheduler/genetic/operators_test.py +++ b/tests/scheduler/genetic/operators_test.py @@ -18,7 +18,7 @@ def test_mutate_order(setup_toolbox): for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() - mutant = tb.mutate(individual[0]) + mutant = tb.mutate(individual) order = mutant[0] # check there are no duplications @@ -43,14 +43,14 @@ def test_mutate_resources(setup_toolbox): def test_mate_order(setup_toolbox, setup_wg): (tb, resources_border), _, _, _, _ = setup_toolbox - _, _, _, population_size = get_params(setup_wg.vertex_count) + _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) for i in range(TEST_ITERATIONS): - individual1, individual2 = tb.select(population, 2) + individual1, individual2 = population[:2] - individual1, individual2 = tb.mate(individual1[0], individual2[0]) + individual1, individual2 = tb.mate(individual1, individual2) order1 = individual1[0] order2 = individual2[0] @@ -61,16 +61,16 @@ def test_mate_order(setup_toolbox, setup_wg): def test_mate_resources(setup_toolbox, setup_wg): (tb, resources_border), _, _, _, _ = setup_toolbox - _, _, _, population_size = get_params(setup_wg.vertex_count) + _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) rand = Random() for i in range(TEST_ITERATIONS): - individual1, individual2 = tb.select(population, 2) + individual1, individual2 = population[:2] worker = rand.sample(list(range(len(resources_border) + 1)), 1)[0] - individual1, individual2 = tb.mate_resources(individual1[0], individual2[0], worker) + individual1, individual2 = tb.mate_resources(individual1, individual2, worker) # check there are correct resources at mate positions assert (resources_border[0][worker] <= individual1[1][:, worker]).all() and \ From ba569e15c17a2e90a2ab3032febf9acfd02e1207 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 18 Aug 2023 17:46:35 +0300 Subject: [PATCH 09/22] changing population initialization --- sampo/scheduler/genetic/schedule_builder.py | 2 +- tests/scheduler/genetic/fixtures.py | 19 +++++++++---------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 97471147..a3b03922 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -12,7 +12,7 @@ from pandas import DataFrame from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome -from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, copy_chromosome, \ +from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, \ FitnessFunction, TimeFitness, is_chromosome_correct from sampo.scheduler.native_wrapper import NativeWrapper from sampo.scheduler.timeline.base import Timeline diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index 84e3cb84..ba994b1e 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -49,7 +49,6 @@ def create_toolbox(wg: WorkGraph, work_estimator: WorkTimeEstimator = None, landscape: LandscapeConfiguration = LandscapeConfiguration()) \ -> tuple[Toolbox, np.ndarray]: - # preparing access-optimized data structures nodes = [node for node in wg.nodes if not node.is_inseparable_son()] @@ -133,13 +132,13 @@ def setup_toolbox(setup_default_schedules) -> tuple: work_estimator: WorkTimeEstimator = DefaultWorkEstimator() return (create_toolbox(setup_wg, - setup_contractors, - setup_worker_pool, - size_of_population, - mutate_order, - mutate_resources, - setup_default_schedules, - rand, - work_estimator=work_estimator, - landscape=setup_landscape_many_holders), \ + setup_contractors, + setup_worker_pool, + size_of_population, + mutate_order, + mutate_resources, + setup_default_schedules, + rand, + work_estimator=work_estimator, + landscape=setup_landscape_many_holders), setup_wg, setup_contractors, setup_default_schedules, setup_landscape_many_holders) From 54aaf336cf21961da8b88e7685da2b99a002d07f Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 18 Aug 2023 17:55:11 +0300 Subject: [PATCH 10/22] changing population initialization --- sampo/scheduler/genetic/operators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 5ecfd73e..88addf45 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -213,7 +213,7 @@ def generate_population(n: int, work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), landscape: LandscapeConfiguration = LandscapeConfiguration()) -> list[Individual]: """ - Generates population using chromosome weights. + Generates population. Do not use `generate_chromosome` function. """ def randomized_init() -> ChromosomeType: From cfa59cbfb2a43f603a4f68a14a1ef893891865c3 Mon Sep 17 00:00:00 2001 From: Egor Date: Fri, 18 Aug 2023 20:40:46 +0300 Subject: [PATCH 11/22] changing genetic algorithm --- sampo/scheduler/genetic/operators.py | 40 +++++++-------- sampo/scheduler/genetic/schedule_builder.py | 56 ++++++++++----------- tests/scheduler/genetic/full_scheduling.py | 2 +- 3 files changed, 48 insertions(+), 50 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 88addf45..050d85d6 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -223,8 +223,6 @@ def randomized_init() -> ChromosomeType: return convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec) - # chromosome types' weights - # these numbers are the probability weights: prob = norm(weights), sum(prob) = 1 count_for_specified_types = (n // 3) // len(init_chromosomes) count_for_specified_types = count_for_specified_types if count_for_specified_types > 0 else 1 count_for_topological = n - count_for_specified_types * len(init_chromosomes) @@ -237,6 +235,8 @@ def randomized_init() -> ChromosomeType: if generated_type != 'topological' else Individual(randomized_init()) for generated_type in chromosome_types] + for ind in chromosomes: + ind.type = '' return chromosomes @@ -378,7 +378,7 @@ def mutate_scheduling_order(ind: ChromosomeType, indpb: float, rand: random.Rand order = ind[0] for i in range(1, len(order) - 2): if rand.random() < indpb: - ind.type = 'mutate' + # ind.type = 'mutate' order[i], order[i + 1] = order[i + 1], order[i] return ind @@ -394,8 +394,6 @@ def mut_uniform_int(ind: ChromosomeType, low: np.ndarray, up: np.ndarray, type_o :param up: upper bound specified by `WorkUnit` :return: mutate individual """ - # ind = copy_chromosome(ind) - # select random number from interval from min to max from uniform distribution size = len(ind[1]) workers_count = len(ind[1][0]) @@ -427,8 +425,6 @@ def mutate_resource_borders(ind: ChromosomeType, contractors_capacity: np.ndarra """ Mutation for contractors' resource borders. """ - # ind = copy_chromosome(ind) - num_resources = len(resources_min_border) num_contractors = len(ind[2]) for contractor in range(num_contractors): @@ -460,18 +456,20 @@ def mate_for_resources(ind1: ChromosomeType, ind2: ChromosomeType, mate_position :param rand: the rand object used for exchange point selection :return: first and second individual """ - ind1 = Individual(copy_chromosome(ind1)) - ind2 = Individual(copy_chromosome(ind2)) + child1 = Individual(copy_chromosome(ind1)) + child2 = Individual(copy_chromosome(ind2)) + child1.type = 'mate_for_resources' + child2.type = 'mate_for_resources' # exchange work resources - res1 = ind1[1][:, mate_positions] - res2 = ind2[1][:, mate_positions] + res1 = child1[1][:, mate_positions] + res2 = child2[1][:, mate_positions] cxpoint = rand.randint(1, len(res1)) - mate_positions = rand.sample(list(range(len(res1))), cxpoint) + mate_positions = rand.sample(range(len(res1)), cxpoint) res1[mate_positions], res2[mate_positions] = res2[mate_positions], res1[mate_positions] - return ind1, ind2 # это не должно работать так как если mate_positions это массив, то advanced indexing вернет копию + return child1, child2 # это не должно работать так как если mate_positions это массив, то advanced indexing вернет копию def mate_for_resource_borders(ind1: ChromosomeType, ind2: ChromosomeType, @@ -479,23 +477,23 @@ def mate_for_resource_borders(ind1: ChromosomeType, ind2: ChromosomeType, """ Crossover for contractors' resource borders. """ - # ind1 = copy_chromosome(ind1) - # ind2 = copy_chromosome(ind2) + child1 = Individual(copy_chromosome(ind1)) + child2 = Individual(copy_chromosome(ind2)) + child1.type = 'mate_for_resource_borders' + child2.type = 'mate_for_resource_borders' num_contractors = len(ind1[2]) - contractors_to_mate = rand.sample(list(range(num_contractors)), rand.randint(1, num_contractors)) + contractors_to_mate = rand.sample(range(num_contractors), rand.randint(1, num_contractors)) + border1 = child1[2][contractors_to_mate] + border2 = child2[2][contractors_to_mate] if rand.randint(0, 2) == 0: # trying to mate whole contractors - border1 = ind1[2][contractors_to_mate] - border2 = ind2[2][contractors_to_mate] border1[:], border2[:] = border2[:], border1[:] else: # trying to mate part of selected contractors - border1 = ind1[2][contractors_to_mate] - border2 = ind2[2][contractors_to_mate] for c_border1, c_border2 in zip(border1, border2): # mate_positions = rand.sample(list(range(len(c_border1))), rand.randint(1, len(c_border1))) c_border1[mate_positions], c_border2[mate_positions] = c_border2[mate_positions], c_border1[mate_positions] - return ind1, ind2 + return child1, child2 diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index a3b03922..4e31f07d 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -280,25 +280,25 @@ def build_schedule(wg: WorkGraph, # crossover # take 2 individuals as input 1 modified individuals - # workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - # - # for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): - # for ind_worker in workers: - # # mate resources - # if rand.random() < cxpb_res: - # child1, child2 = toolbox.mate_resources(ind1, ind2, ind_worker) - # # add to population - # cur_generation.append(child1) - # cur_generation.append(child2) - # - # # mate resource borders - # if rand.random() < cxpb_res: - # if ind_worker == len(worker_name2index): - # continue - # child1, child2 = toolbox.mate_resource_borders(ind1, ind2, ind_worker) - # # add to population - # cur_generation.append(child1) - # cur_generation.append(child2) + workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) + + for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): + for ind_worker in workers: + # mate resources + if rand.random() < cxpb_res: + child1, child2 = toolbox.mate_resources(ind1, ind2, ind_worker) + # add to population + cur_generation.append(child1) + cur_generation.append(child2) + + # mate resource borders + # if rand.random() < cxpb_res: + # if ind_worker == len(worker_name2index): + # continue + # child1, child2 = toolbox.mate_resource_borders(ind1, ind2, ind_worker) + # # add to population + # cur_generation.append(child1) + # cur_generation.append(child2) # workers type for changing(+1 means contractor 'resource') workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) @@ -312,15 +312,15 @@ def build_schedule(wg: WorkGraph, toolbox.mutate_resources(mutant, low=low, up=up, type_of_worker=worker) # resource borders mutation - for worker in workers: - if worker == len(worker_name2index): - continue - for mutant in cur_generation: - if rand.random() < mutpb_res: - toolbox.mutate_resource_borders(mutant, - contractors_capacity=contractors_capacity, - resources_min_border=resources_min_border, - type_of_worker=worker) + # for worker in workers: + # if worker == len(worker_name2index): + # continue + # for mutant in cur_generation: + # if rand.random() < mutpb_res: + # toolbox.mutate_resource_borders(mutant, + # contractors_capacity=contractors_capacity, + # resources_min_border=resources_min_border, + # type_of_worker=worker) # mutation # take 1 individuals as input and return 1 individuals as output diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index ab7df489..297bb679 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -6,7 +6,7 @@ def test_multiprocessing(setup_scheduler_parameters): genetic = GeneticScheduler(number_of_generation=50, mutate_order=0.05, - mutate_resources=0.01, + mutate_resources=0.05, size_of_population=50) genetic.schedule(setup_wg, setup_contractors) From ce5e599333504721506444a5edb2f7c48664fc58 Mon Sep 17 00:00:00 2001 From: Egor Date: Tue, 22 Aug 2023 20:25:35 +0300 Subject: [PATCH 12/22] changing genetic algorithm --- sampo/scheduler/genetic/operators.py | 224 ++++++++++---------- sampo/scheduler/genetic/schedule_builder.py | 151 +++---------- tests/scheduler/genetic/full_scheduling.py | 4 +- 3 files changed, 143 insertions(+), 236 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 050d85d6..e6ef8428 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -128,7 +128,6 @@ def init_toolbox(wg: WorkGraph, index2node: dict[int, GraphNode], work_id2index: dict[str, int], worker_name2index: dict[str, int], - index2contractor: dict[int, str], index2contractor_obj: dict[int, Contractor], init_chromosomes: dict[str, ChromosomeType], mutate_order: float, @@ -141,6 +140,8 @@ def init_toolbox(wg: WorkGraph, contractor_borders: np.ndarray, node_indices: list[int], parents: dict[int, list[int]], + resources_border: np.ndarray, + resources_min_border: np.ndarray, assigned_parent_time: Time = Time(0), work_estimator: WorkTimeEstimator = DefaultWorkEstimator()) -> base.Toolbox: """ @@ -150,38 +151,34 @@ def init_toolbox(wg: WorkGraph, :return: Object, included tools for genetic """ toolbox = base.Toolbox() - # generate initial population + # generate chromosome toolbox.register('generate_chromosome', generate_chromosome, wg=wg, contractors=contractors, work_id2index=work_id2index, worker_name2index=worker_name2index, contractor2index=contractor2index, contractor_borders=contractor_borders, spec=spec, init_chromosomes=init_chromosomes, rand=rand, work_estimator=work_estimator, landscape=landscape) - # create population from individuals + # create population # toolbox.register('population', tools.initRepeat, list, lambda: toolbox.generate_chromosome()) toolbox.register('population', generate_population, wg=wg, contractors=contractors, work_id2index=work_id2index, worker_name2index=worker_name2index, contractor2index=contractor2index, contractor_borders=contractor_borders, spec=spec, init_chromosomes=init_chromosomes, rand=rand, work_estimator=work_estimator, landscape=landscape) + # selection + toolbox.register('select', select_new_population, pop_size=population_size) # crossover for order toolbox.register('mate', mate_scheduling_order, rand=rand) - # mutation for order. Coefficient luke one or two mutation in individual - # toolbox.register('mutate', tools.mutShuffleIndexes, indpb=mutate_order) - toolbox.register('mutate', mutate_scheduling_order, indpb=mutate_order, rand=rand) - # selection. Some random individuals and arranges a battle between them as a result in a continuing genus, - # this is the best among these it - # toolbox.register('select', tools.selTournament, tournsize=selection_size) - toolbox.register('select', select_new_population, pop_size=population_size) - - # mutation for resources - toolbox.register('mutate_resources', mut_uniform_int, probability_mutate_resources=mutate_resources, - contractor_count=len(index2contractor), rand=rand) - # mutation for resource borders - toolbox.register('mutate_resource_borders', mutate_resource_borders, - probability_mutate_contractors=mutate_resources, rand=rand) + # mutation for order + toolbox.register('mutate', mutate_scheduling_order, mutpb=mutate_order, rand=rand) # crossover for resources toolbox.register('mate_resources', mate_for_resources, rand=rand) + # mutation for resources + toolbox.register('mutate_resources', mutate_for_resources, resources_border=resources_border, + mutpb=mutate_resources, rand=rand) # crossover for resource borders toolbox.register('mate_resource_borders', mate_for_resource_borders, rand=rand) + # mutation for resource borders + toolbox.register('mutate_resource_borders', mutate_resource_borders, resources_min_border=resources_min_border, + mutpb=mutate_resources, rand=rand) toolbox.register('validate', is_chromosome_correct, node_indices=node_indices, parents=parents) toolbox.register('schedule_to_chromosome', convert_schedule_to_chromosome, wg=wg, @@ -216,9 +213,9 @@ def generate_population(n: int, Generates population. Do not use `generate_chromosome` function. """ + def randomized_init() -> ChromosomeType: - schedule = RandomizedTopologicalScheduler(work_estimator, - int(rand.random() * 1000000)) \ + schedule = RandomizedTopologicalScheduler(work_estimator, int(rand.random() * 1000000)) \ .schedule(wg, contractors, landscape=landscape) return convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec) @@ -235,8 +232,6 @@ def randomized_init() -> ChromosomeType: if generated_type != 'topological' else Individual(randomized_init()) for generated_type in chromosome_types] - for ind in chromosomes: - ind.type = '' return chromosomes @@ -290,7 +285,7 @@ def randomized_init() -> ChromosomeType: def select_new_population(population: list[ChromosomeType], pop_size: int) -> list[ChromosomeType]: - population.sort(key=attrgetter('fitness'), reverse=True) + population = sorted(population, key=attrgetter('fitness'), reverse=True) return population[:pop_size] @@ -301,7 +296,7 @@ def is_chromosome_correct(chromosome: ChromosomeType, Check order of works and contractors. """ return is_chromosome_order_correct(chromosome, parents) and \ - is_chromosome_contractors_correct(chromosome, node_indices) + is_chromosome_contractors_correct(chromosome, node_indices) def is_chromosome_order_correct(chromosome: ChromosomeType, parents: dict[int, list[int]]) -> bool: @@ -355,8 +350,6 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand """ child1 = Individual(copy_chromosome(ind1)) child2 = Individual(copy_chromosome(ind2)) - child1.type = 'mate' - child2.type = 'mate' order1 = child1[0] order2 = child2[0] @@ -374,126 +367,129 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand return child1, child2 -def mutate_scheduling_order(ind: ChromosomeType, indpb: float, rand: random.Random) -> ChromosomeType: +def mutate_scheduling_order(ind: ChromosomeType, mutpb: float, rand: random.Random) -> ChromosomeType: order = ind[0] for i in range(1, len(order) - 2): - if rand.random() < indpb: - # ind.type = 'mutate' + if rand.random() < mutpb: order[i], order[i + 1] = order[i + 1], order[i] return ind -def mut_uniform_int(ind: ChromosomeType, low: np.ndarray, up: np.ndarray, type_of_worker: int, - probability_mutate_resources: float, contractor_count: int, rand: random.Random) -> ChromosomeType: - """ - Mutation function for resources. - It changes selected numbers of workers in random work in a certain interval for this work. - - :param low: lower bound specified by `WorkUnit` - :param up: upper bound specified by `WorkUnit` - :return: mutate individual - """ - # select random number from interval from min to max from uniform distribution - size = len(ind[1]) - workers_count = len(ind[1][0]) - - if type_of_worker == workers_count - 1: - # print('Contractor mutation!') - for i in range(size): - if rand.random() < probability_mutate_resources: - ind.type = 'mutate_uniform_int' - ind[1][i][type_of_worker] = rand.randint(0, contractor_count - 1) - return ind - - # change in this interval in random number from interval - for i, xl, xu in zip(range(size), low, up): - if rand.random() < probability_mutate_resources: - ind.type = 'mutate_uniform_int' - # borders - contractor = ind[1][i][-1] - border = ind[2][contractor][type_of_worker] - # TODO Debug why min(xu, border) can be lower than xl - ind[1][i][type_of_worker] = rand.randint(xl, max(xl, min(xu, border))) - - return ind - - -def mutate_resource_borders(ind: ChromosomeType, contractors_capacity: np.ndarray, resources_min_border: np.ndarray, - type_of_worker: int, probability_mutate_contractors: float, rand: random.Random) \ - -> ChromosomeType: - """ - Mutation for contractors' resource borders. - """ - num_resources = len(resources_min_border) - num_contractors = len(ind[2]) - for contractor in range(num_contractors): - if rand.random() < probability_mutate_contractors: - ind.type = 'mutate_resource_borders' - ind[2][contractor][type_of_worker] -= rand.randint(resources_min_border[type_of_worker] + 1, - max(resources_min_border[type_of_worker] + 1, - ind[2][contractor][type_of_worker] // 10)) - if ind[2][contractor][type_of_worker] <= 0: - ind[2][contractor][type_of_worker] = 1 - - # find and correct all invalidated resource assignments - for work in range(len(ind[0])): - if ind[1][work][num_resources] == contractor: - ind[1][work][type_of_worker] = min(ind[1][work][type_of_worker], - ind[2][contractor][type_of_worker]) - - return ind - - -def mate_for_resources(ind1: ChromosomeType, ind2: ChromosomeType, mate_positions: np.ndarray, +def mate_for_resources(ind1: ChromosomeType, ind2: ChromosomeType, rand: random.Random) -> (ChromosomeType, ChromosomeType): """ CxOnePoint for resources. :param ind1: first individual :param ind2: second individual - :param mate_positions: an array of positions that should be mate :param rand: the rand object used for exchange point selection :return: first and second individual """ child1 = Individual(copy_chromosome(ind1)) child2 = Individual(copy_chromosome(ind2)) - child1.type = 'mate_for_resources' - child2.type = 'mate_for_resources' - # exchange work resources - res1 = child1[1][:, mate_positions] - res2 = child2[1][:, mate_positions] - cxpoint = rand.randint(1, len(res1)) + res1 = child1[1] + res2 = child2[1] + num_works = len(res1) + border = num_works // 4 + cxpoint = rand.randint(border, num_works - border) - mate_positions = rand.sample(range(len(res1)), cxpoint) + mate_positions = rand.sample(range(num_works), cxpoint) res1[mate_positions], res2[mate_positions] = res2[mate_positions], res1[mate_positions] - return child1, child2 # это не должно работать так как если mate_positions это массив, то advanced indexing вернет копию + return child1, child2 + + +def mutate_for_resources(ind: ChromosomeType, resources_border: np.ndarray, + mutpb: float, rand: random.Random) -> ChromosomeType: + """ + Mutation function for resources. + It changes selected numbers of workers in random work in a certain interval for this work. + + :return: mutate individual + """ + # select random number from interval from min to max from uniform distribution + res = ind[1] + res_count = len(res[0]) + for i, work_res in enumerate(res): + for type_of_res in range(res_count - 1): + if rand.random() < mutpb: + xl = resources_border[0, type_of_res, i] + xu = resources_border[1, type_of_res, i] + contractor = work_res[-1] + border = ind[2][contractor, type_of_res] + # TODO Debug why min(xu, border) can be lower than xl + work_res[type_of_res] = rand.randint(xl, max(xl, min(xu, border))) + if rand.random() < mutpb: + work_res[-1] = rand.randint(0, len(ind[2]) - 1) + + # type_of_res = rand.randint(0, res_count - 1) + # num_works = len(res) + # if type_of_res == res_count - 1: + # for i in range(num_works): + # if rand.random() < mutpb: + # res[i][type_of_res] = rand.randint(0, len(ind[2]) - 1) + # return ind + # + # # change in this interval in random number from interval + # low = resources_border[0, type_of_res] + # up = resources_border[1, type_of_res] + # for i, xl, xu in zip(range(num_works), low, up): + # if rand.random() < mutpb: + # # borders + # contractor = res[i][-1] + # border = ind[2][contractor][type_of_res] + # res[i][type_of_res] = rand.randint(xl, max(xl, min(xu, border))) + + return ind def mate_for_resource_borders(ind1: ChromosomeType, ind2: ChromosomeType, - mate_positions: np.ndarray, rand: random.Random) -> (ChromosomeType, ChromosomeType): + rand: random.Random) -> (ChromosomeType, ChromosomeType): """ Crossover for contractors' resource borders. """ child1 = Individual(copy_chromosome(ind1)) child2 = Individual(copy_chromosome(ind2)) - child1.type = 'mate_for_resource_borders' - child2.type = 'mate_for_resource_borders' - num_contractors = len(ind1[2]) - contractors_to_mate = rand.sample(range(num_contractors), rand.randint(1, num_contractors)) - border1 = child1[2][contractors_to_mate] - border2 = child2[2][contractors_to_mate] + borders1 = child1[2] + borders2 = child2[2] + num_contractors = len(borders1) + contractors = rand.sample(range(num_contractors), rand.randint(1, num_contractors)) - if rand.randint(0, 2) == 0: - # trying to mate whole contractors - border1[:], border2[:] = border2[:], border1[:] - else: - # trying to mate part of selected contractors - for c_border1, c_border2 in zip(border1, border2): - # mate_positions = rand.sample(list(range(len(c_border1))), rand.randint(1, len(c_border1))) - c_border1[mate_positions], c_border2[mate_positions] = c_border2[mate_positions], c_border1[mate_positions] + num_res = len(borders1[0]) + res_indices = list(range(num_res)) + border = num_res // 4 + mate_positions = rand.sample(res_indices, rand.randint(border, num_res - border)) + + (borders1[contractors, mate_positions], + borders2[contractors, mate_positions]) = (borders2[contractors, mate_positions], + borders1[contractors, mate_positions]) return child1, child2 + + +def mutate_resource_borders(ind: ChromosomeType, resources_min_border: np.ndarray, + mutpb: float, rand: random.Random) -> ChromosomeType: + """ + Mutation for contractors' resource borders. + """ + num_resources = len(resources_min_border) + num_contractors = len(ind[2]) + type_of_res = rand.randint(0, len(ind[2][0]) - 1) + for contractor in range(num_contractors): + if rand.random() < mutpb: + ind[2][contractor][type_of_res] -= rand.randint(resources_min_border[type_of_res] + 1, + max(resources_min_border[type_of_res] + 1, + ind[2][contractor][type_of_res] // 10)) + if ind[2][contractor][type_of_res] <= 0: + ind[2][contractor][type_of_res] = 1 + + # find and correct all invalidated resource assignments + for work in range(len(ind[0])): + if ind[1][work][num_resources] == contractor: + ind[1][work][type_of_res] = min(ind[1][work][type_of_res], + ind[2][contractor][type_of_res]) + + return ind diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 4e31f07d..7d58e87a 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -1,15 +1,10 @@ -import math import random import time from typing import Callable import numpy as np -import seaborn as sns from deap import tools from deap.base import Toolbox -from matplotlib import pyplot as plt -from numpy import ndarray -from pandas import DataFrame from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, \ @@ -38,7 +33,7 @@ def create_toolbox(wg: WorkGraph, spec: ScheduleSpec = ScheduleSpec(), work_estimator: WorkTimeEstimator = None, landscape: LandscapeConfiguration = LandscapeConfiguration()) \ - -> tuple[Toolbox, ndarray, ndarray, ndarray]: + -> Toolbox: start = time.time() # preparing access-optimized data structures @@ -47,19 +42,13 @@ def create_toolbox(wg: WorkGraph, index2node: dict[int, GraphNode] = {index: node for index, node in enumerate(nodes)} work_id2index: dict[str, int] = {node.id: index for index, node in index2node.items()} worker_name2index = {worker_name: index for index, worker_name in enumerate(worker_pool)} - index2contractor = {ind: contractor.id for ind, contractor in enumerate(contractors)} index2contractor_obj = {ind: contractor for ind, contractor in enumerate(contractors)} - contractor2index = reverse_dictionary(index2contractor) + contractor2index = {contractor.id: ind for ind, contractor in enumerate(contractors)} worker_pool_indices = {worker_name2index[worker_name]: { contractor2index[contractor_id]: worker for contractor_id, worker in workers_of_type.items() } for worker_name, workers_of_type in worker_pool.items()} node_indices = list(range(len(nodes))) - contractors_capacity = np.zeros((len(contractors), len(worker_pool))) - for w_ind, cont2worker in worker_pool_indices.items(): - for c_ind, worker in cont2worker.items(): - contractors_capacity[c_ind][w_ind] = worker.count - resources_border = np.zeros((2, len(worker_pool), len(index2node))) resources_min_border = np.zeros((len(worker_pool))) for work_index, node in index2node.items(): @@ -95,7 +84,7 @@ def create_toolbox(wg: WorkGraph, init_chromosomes: dict[str, ChromosomeType] = \ {name: convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, spec, order) - if schedule is not None else None + if schedule is not None else None for name, (schedule, order, spec) in init_schedules.items()} for name, chromosome in init_chromosomes.items(): @@ -112,7 +101,6 @@ def create_toolbox(wg: WorkGraph, index2node, work_id2index, worker_name2index, - index2contractor, index2contractor_obj, init_chromosomes, mutate_order, @@ -125,8 +113,10 @@ def create_toolbox(wg: WorkGraph, contractor_borders, node_indices, parents, + resources_border, + resources_min_border, Time(0), - work_estimator), resources_border, contractors_capacity, resources_min_border + work_estimator) def build_schedule(wg: WorkGraph, @@ -134,8 +124,8 @@ def build_schedule(wg: WorkGraph, worker_pool: WorkerContractorPool, population_size: int, generation_number: int, - mutate_order: float, - mutate_resources: float, + mutpb_order: float, + mutpb_res: float, init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], rand: random.Random, spec: ScheduleSpec, @@ -143,14 +133,12 @@ def build_schedule(wg: WorkGraph, fitness_constructor: Callable[ [Callable[[list[ChromosomeType]], list[int]]], FitnessFunction] = TimeFitness, work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), - show_fitness_graph: bool = False, n_cpu: int = 1, assigned_parent_time: Time = Time(0), timeline: Timeline | None = None, - time_border: int = None) \ + time_border: int = None, + optimize_resources: bool = False) \ -> tuple[ScheduleWorkDict, Time, Timeline, list[GraphNode]]: - if show_fitness_graph: - fitness_history = [] global_start = time.time() @@ -183,13 +171,8 @@ def build_schedule(wg: WorkGraph, start = time.time() - toolbox, resources_border, contractors_capacity, resources_min_border = create_toolbox(wg, contractors, worker_pool, - population_size, - mutate_order, - mutate_resources, - init_schedules, - rand, spec, work_estimator, - landscape) + toolbox = create_toolbox(wg, contractors, worker_pool, population_size, mutpb_order, mutpb_res, init_schedules, + rand, spec, work_estimator, landscape) native = NativeWrapper(toolbox, wg, contractors, worker_name2index, worker_pool_indices, parents, work_estimator) @@ -200,25 +183,20 @@ def build_schedule(wg: WorkGraph, if native.native: native_start = time.time() - best_chromosome = native.run_genetic(list([ind[0] for ind in pop]), - mutate_order, mutate_order, mutate_resources, mutate_resources, - mutate_resources, mutate_resources, population_size) + best_chromosome = native.run_genetic(pop, mutpb_order, mutpb_order, mutpb_res, mutpb_res, mutpb_res, mutpb_res, + population_size) print(f'Native evaluated in {(time.time() - native_start) * 1000} ms') else: # save best individuals hof = tools.HallOfFame(1, similar=compare_individuals) - # probability to participate in mutation and crossover for each individual - cxpb, mutpb = mutate_order, mutate_order - mutpb_res, cxpb_res = mutate_resources, mutate_resources - fitness_f = fitness_constructor(native.evaluate) start = time.time() # map to each individual fitness function pop = [ind for ind in pop if toolbox.validate(ind)] - fitness = fitness_f.evaluate([ind for ind in pop]) + fitness = fitness_f.evaluate(pop) evaluation_time = time.time() - start @@ -228,9 +206,6 @@ def build_schedule(wg: WorkGraph, hof.update(pop) best_fitness = hof[0].fitness.values[0] - if show_fitness_graph: - fitness_history.append(sum(fitness) / len(fitness)) - generation = 1 # the best fitness, track to increase performance by stopping evaluation when not decreasing prev_best_fitness = Time.inf() @@ -255,76 +230,28 @@ def build_schedule(wg: WorkGraph, cur_generation = [] for ind1, ind2 in zip(pop[::2], pop[1::2]): - # if rand.random() < cxpb: - child1, child2 = toolbox.mate(ind1, ind2) - # add to population - cur_generation.append(child1) - cur_generation.append(child2) - - # offspring_fitness = fitness_f.evaluate(cur_generation) - # for fit in offspring_fitness: - # if fit < best_fitness: - # print('crossover') - # break + # mate order + cur_generation.extend(toolbox.mate(ind1, ind2)) if worker_name2index: # operations for RESOURCES - # mutation - # select types for mutation - # numbers of changing types - number_of_type_for_changing = rand.randint(1, len(worker_name2index) - 1) - - # for the crossover, we use those types that did not participate - # in the mutation(+1 means contractor 'resource') - # workers_for_mate = list(set(list(range(len(worker_name2index) + 1))) - set(workers)) - # crossover - # take 2 individuals as input 1 modified individuals - - workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): - for ind_worker in workers: - # mate resources - if rand.random() < cxpb_res: - child1, child2 = toolbox.mate_resources(ind1, ind2, ind_worker) - # add to population - cur_generation.append(child1) - cur_generation.append(child2) + # mate resources + cur_generation.extend(toolbox.mate_resources(ind1, ind2)) + if optimize_resources: # mate resource borders - # if rand.random() < cxpb_res: - # if ind_worker == len(worker_name2index): - # continue - # child1, child2 = toolbox.mate_resource_borders(ind1, ind2, ind_worker) - # # add to population - # cur_generation.append(child1) - # cur_generation.append(child2) - - # workers type for changing(+1 means contractor 'resource') - workers = rand.sample(range(len(worker_name2index) + 1), number_of_type_for_changing) - - # resources mutation - for worker in workers: - low = resources_border[0, worker] if worker != len(worker_name2index) else 0 - up = resources_border[1, worker] if worker != len(worker_name2index) else 0 - for mutant in cur_generation: - if rand.random() < mutpb_res: - toolbox.mutate_resources(mutant, low=low, up=up, type_of_worker=worker) - - # resource borders mutation - # for worker in workers: - # if worker == len(worker_name2index): - # continue - # for mutant in cur_generation: - # if rand.random() < mutpb_res: - # toolbox.mutate_resource_borders(mutant, - # contractors_capacity=contractors_capacity, - # resources_min_border=resources_min_border, - # type_of_worker=worker) - - # mutation - # take 1 individuals as input and return 1 individuals as output + cur_generation.extend(toolbox.mate_resource_borders(ind1, ind2)) + + for mutant in cur_generation: + # resources mutation + toolbox.mutate_resources(mutant) + if optimize_resources: + # resource borders mutation + toolbox.mutate_resource_borders(mutant) + for mutant in cur_generation: + # order mutation toolbox.mutate(mutant) evaluation_start = time.time() @@ -336,22 +263,15 @@ def build_schedule(wg: WorkGraph, offspring_fitness = fitness_f.evaluate(offspring) for fit, ind in zip(offspring_fitness, offspring): ind.fitness.values = [fit] - evaluation_time += time.time() - evaluation_start - if show_fitness_graph: - _ftn = [f for f in fitness if not math.isinf(f)] - if len(_ftn) > 0: - fitness_history.append(sum(_ftn) / len(_ftn)) + evaluation_time += time.time() - evaluation_start # renewing population pop += offspring - # pop = toolbox.select(pop, population_size) pop = toolbox.select(pop) hof.update([pop[0]]) best_fitness = hof[0].fitness.values[0] - if best_fitness < prev_best_fitness: - print(hof[0].type) generation += 1 @@ -370,15 +290,6 @@ def build_schedule(wg: WorkGraph, landscape=landscape, timeline=timeline) - if show_fitness_graph: - sns.lineplot( - data=DataFrame.from_records([(generation * 4, v) for generation, v in enumerate(fitness_history)], - columns=['Поколение', 'Функция качества']), - x='Поколение', - y='Функция качества', - palette='r') - plt.show() - return {node.id: work for node, work in scheduled_works.items()}, schedule_start_time, timeline, order_nodes diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 297bb679..c7d2db35 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -5,8 +5,8 @@ def test_multiprocessing(setup_scheduler_parameters): setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters genetic = GeneticScheduler(number_of_generation=50, - mutate_order=0.05, - mutate_resources=0.05, + mutate_order=3/len(setup_wg.nodes), + mutate_resources=1/len(setup_wg.nodes)/len(setup_contractors[0].workers), size_of_population=50) genetic.schedule(setup_wg, setup_contractors) From a26d9a3b7b43871612d25afe2418e44be48aa2e1 Mon Sep 17 00:00:00 2001 From: Egor Date: Tue, 22 Aug 2023 20:34:48 +0300 Subject: [PATCH 13/22] changing genetic algorithm --- sampo/scheduler/genetic/operators.py | 18 ------------------ sampo/scheduler/genetic/schedule_builder.py | 1 - 2 files changed, 19 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index e6ef8428..2ec64384 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -424,24 +424,6 @@ def mutate_for_resources(ind: ChromosomeType, resources_border: np.ndarray, if rand.random() < mutpb: work_res[-1] = rand.randint(0, len(ind[2]) - 1) - # type_of_res = rand.randint(0, res_count - 1) - # num_works = len(res) - # if type_of_res == res_count - 1: - # for i in range(num_works): - # if rand.random() < mutpb: - # res[i][type_of_res] = rand.randint(0, len(ind[2]) - 1) - # return ind - # - # # change in this interval in random number from interval - # low = resources_border[0, type_of_res] - # up = resources_border[1, type_of_res] - # for i, xl, xu in zip(range(num_works), low, up): - # if rand.random() < mutpb: - # # borders - # contractor = res[i][-1] - # border = ind[2][contractor][type_of_res] - # res[i][type_of_res] = rand.randint(xl, max(xl, min(xu, border))) - return ind diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 7d58e87a..3f4e3a4b 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -258,7 +258,6 @@ def build_schedule(wg: WorkGraph, # Gather all the fitness in one list and print the stats offspring = [ind for ind in cur_generation if toolbox.validate(ind)] - print(f'Len offspring: {len(offspring)}') offspring_fitness = fitness_f.evaluate(offspring) for fit, ind in zip(offspring_fitness, offspring): From 775db6f36ad7f84d1ab005d10db39ec6a4a2177f Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 23 Aug 2023 15:29:12 +0300 Subject: [PATCH 14/22] changing test arguments --- tests/scheduler/genetic/full_scheduling.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index c7d2db35..ca27a80d 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -5,8 +5,8 @@ def test_multiprocessing(setup_scheduler_parameters): setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters genetic = GeneticScheduler(number_of_generation=50, - mutate_order=3/len(setup_wg.nodes), - mutate_resources=1/len(setup_wg.nodes)/len(setup_contractors[0].workers), + mutate_order=0.05, + mutate_resources=0.005, size_of_population=50) genetic.schedule(setup_wg, setup_contractors) From a1b05b6ba168f6b090b041b765cdb68125a0a6a2 Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 23 Aug 2023 16:34:17 +0300 Subject: [PATCH 15/22] changing tests --- sampo/scheduler/genetic/base.py | 17 ++++++------- tests/scheduler/genetic/fixtures.py | 13 +++++----- tests/scheduler/genetic/operators_test.py | 25 ++++++------------- .../resources_in_time/basic_res_test.py | 6 ++++- 4 files changed, 28 insertions(+), 33 deletions(-) diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index 26fc8c6e..8efc576e 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -61,12 +61,12 @@ def __init__(self, def __str__(self) -> str: return f'GeneticScheduler[' \ f'generations={self.number_of_generation},' \ - f'size_selection={self.size_selection},' \ + f'population_size={self.size_of_population},' \ f'mutate_order={self.mutate_order},' \ f'mutate_resources={self.mutate_resources}' \ f']' - def get_params(self, works_count: int) -> tuple[int, float, float, int]: + def get_params(self, works_count: int) -> tuple[float, float, int]: """ Return base parameters for model to make new population @@ -83,18 +83,18 @@ def get_params(self, works_count: int) -> tuple[int, float, float, int]: mutate_resources = self.mutate_resources if mutate_resources is None: if works_count < 300: - mutate_resources = 0.1 + mutate_resources = 0.01 else: - mutate_resources = 6 / math.sqrt(works_count) + mutate_resources = 5 / math.sqrt(works_count) size_of_population = self.size_of_population if size_of_population is None: if works_count < 300: - size_of_population = 20 - elif 1500 > works_count >= 300: size_of_population = 50 + elif 1500 > works_count >= 300: + size_of_population = 100 else: - size_of_population = works_count // 50 + size_of_population = works_count // 25 return mutate_order, mutate_resources, size_of_population def set_use_multiprocessing(self, n_cpu: int): @@ -144,7 +144,7 @@ def init_schedule(scheduler_class): try: return scheduler_class(work_estimator=self.work_estimator).schedule(wg, contractors, landscape=landscape), \ - list(reversed(prioritization(wg, self.work_estimator))), spec + list(reversed(prioritization(wg, self.work_estimator))), spec except NoSufficientContractorError: return None, None, None else: @@ -166,7 +166,6 @@ def init_schedule(scheduler_class): "87.5%": init_k_schedule(HEFTScheduler, 8 / 7) } - def schedule_with_cache(self, wg: WorkGraph, contractors: list[Contractor], diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index ba994b1e..82343d9b 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -1,5 +1,5 @@ from random import Random -from typing import List, Dict, Tuple +from typing import Dict, Tuple import numpy as np from deap.base import Toolbox @@ -14,7 +14,6 @@ from sampo.schemas.schedule_spec import ScheduleSpec from sampo.schemas.time import Time from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator -from sampo.utilities.collections_util import reverse_dictionary def get_params(works_count: int) -> Tuple[float, float, int]: @@ -48,27 +47,28 @@ def create_toolbox(wg: WorkGraph, spec: ScheduleSpec = ScheduleSpec(), work_estimator: WorkTimeEstimator = None, landscape: LandscapeConfiguration = LandscapeConfiguration()) \ - -> tuple[Toolbox, np.ndarray]: + -> Tuple[Toolbox, np.ndarray]: # preparing access-optimized data structures nodes = [node for node in wg.nodes if not node.is_inseparable_son()] index2node: dict[int, GraphNode] = {index: node for index, node in enumerate(nodes)} work_id2index: dict[str, int] = {node.id: index for index, node in index2node.items()} worker_name2index = {worker_name: index for index, worker_name in enumerate(worker_pool)} - index2contractor = {ind: contractor.id for ind, contractor in enumerate(contractors)} index2contractor_obj = {ind: contractor for ind, contractor in enumerate(contractors)} - contractor2index = reverse_dictionary(index2contractor) + contractor2index = {contractor.id: ind for ind, contractor in enumerate(contractors)} worker_pool_indices = {worker_name2index[worker_name]: { contractor2index[contractor_id]: worker for contractor_id, worker in workers_of_type.items() } for worker_name, workers_of_type in worker_pool.items()} node_indices = list(range(len(nodes))) resources_border = np.zeros((2, len(worker_pool), len(index2node))) + resources_min_border = np.zeros((len(worker_pool))) for work_index, node in index2node.items(): for req in node.work_unit.worker_reqs: worker_index = worker_name2index[req.kind] resources_border[0, worker_index, work_index] = req.min_count resources_border[1, worker_index, work_index] = req.max_count + resources_min_border[worker_index] = max(resources_min_border[worker_index], req.min_count) contractor_borders = np.zeros((len(contractor2index), len(worker_name2index)), dtype=int) for ind, contractor in enumerate(contractors): @@ -105,7 +105,6 @@ def create_toolbox(wg: WorkGraph, index2node, work_id2index, worker_name2index, - index2contractor, index2contractor_obj, init_chromosomes, mutate_order, @@ -118,6 +117,8 @@ def create_toolbox(wg: WorkGraph, contractor_borders, node_indices, parents, + resources_border, + resources_min_border, Time(0), work_estimator), resources_border diff --git a/tests/scheduler/genetic/operators_test.py b/tests/scheduler/genetic/operators_test.py index 4e10506b..4bed4644 100644 --- a/tests/scheduler/genetic/operators_test.py +++ b/tests/scheduler/genetic/operators_test.py @@ -26,23 +26,17 @@ def test_mutate_order(setup_toolbox): def test_mutate_resources(setup_toolbox): - (tb, resources_border), _, _, _, _ = setup_toolbox - - rand = Random() + (tb, _), _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() - type_of_resource = rand.randint(0, len(resources_border[0]) - 1) - mutant = tb.mutate_resources(individual, - resources_border[0][type_of_resource], - resources_border[1][type_of_resource], - type_of_resource) + mutant = tb.mutate_resources(individual) assert tb.validate(mutant) def test_mate_order(setup_toolbox, setup_wg): - (tb, resources_border), _, _, _, _ = setup_toolbox + (tb, _), _, _, _, _ = setup_toolbox _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) @@ -64,19 +58,16 @@ def test_mate_resources(setup_toolbox, setup_wg): _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) - rand = Random() for i in range(TEST_ITERATIONS): individual1, individual2 = population[:2] - - worker = rand.sample(list(range(len(resources_border) + 1)), 1)[0] - individual1, individual2 = tb.mate_resources(individual1, individual2, worker) + individual1, individual2 = tb.mate_resources(individual1, individual2) # check there are correct resources at mate positions - assert (resources_border[0][worker] <= individual1[1][:, worker]).all() and \ - (individual1[1][:, worker] <= resources_border[1][worker]).all() - assert (resources_border[0][worker] <= individual1[1][:, worker]).all() and \ - (individual1[1][:, worker] <= resources_border[1][worker]).all() + assert (resources_border[0] <= individual1[1].T[:-1]).all() and \ + (individual1[1].T[:-1] <= resources_border[1]).all() + assert (resources_border[0] <= individual1[1].T[:-1]).all() and \ + (individual1[1].T[:-1] <= resources_border[1]).all() # check the whole chromosomes assert tb.validate(individual1) diff --git a/tests/scheduler/resources_in_time/basic_res_test.py b/tests/scheduler/resources_in_time/basic_res_test.py index ad85e57f..dda836a3 100644 --- a/tests/scheduler/resources_in_time/basic_res_test.py +++ b/tests/scheduler/resources_in_time/basic_res_test.py @@ -35,7 +35,11 @@ def test_genetic_deadline_planning(setup_scheduler_parameters): setup_wg, setup_contractors, landscape = setup_scheduler_parameters deadline = Time.inf() // 2 - scheduler = GeneticScheduler(fitness_constructor=DeadlineResourcesFitness.prepare(deadline)) + scheduler = GeneticScheduler(number_of_generation=50, + mutate_order=0.05, + mutate_resources=0.005, + size_of_population=50, + fitness_constructor=DeadlineResourcesFitness.prepare(deadline)) scheduler.set_deadline(deadline) try: From 1d58430d4f93300e47411a79cf7b2f40997d4a95 Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 23 Aug 2023 16:56:47 +0300 Subject: [PATCH 16/22] changing tests --- sampo/scheduler/genetic/base.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index 8efc576e..28f071e2 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -75,17 +75,11 @@ def get_params(self, works_count: int) -> tuple[float, float, int]: """ mutate_order = self.mutate_order if mutate_order is None: - if works_count < 300: - mutate_order = 0.05 - else: - mutate_order = 2 / math.sqrt(works_count) + mutate_order = 0.05 mutate_resources = self.mutate_resources if mutate_resources is None: - if works_count < 300: - mutate_resources = 0.01 - else: - mutate_resources = 5 / math.sqrt(works_count) + mutate_resources = 0.005 size_of_population = self.size_of_population if size_of_population is None: From c1713093541ae899e37c644d2b1743012816cbee Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 23 Aug 2023 18:12:11 +0300 Subject: [PATCH 17/22] changing tests --- sampo/scheduler/genetic/schedule_builder.py | 3 +-- tests/scheduler/genetic/converter_test.py | 6 ++--- tests/scheduler/genetic/fixtures.py | 26 +++++++++++---------- tests/scheduler/genetic/operators_test.py | 10 ++++---- 4 files changed, 23 insertions(+), 22 deletions(-) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 3f4e3a4b..635f4978 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -19,7 +19,6 @@ from sampo.schemas.schedule_spec import ScheduleSpec from sampo.schemas.time import Time from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator -from sampo.utilities.collections_util import reverse_dictionary def create_toolbox(wg: WorkGraph, @@ -50,7 +49,7 @@ def create_toolbox(wg: WorkGraph, node_indices = list(range(len(nodes))) resources_border = np.zeros((2, len(worker_pool), len(index2node))) - resources_min_border = np.zeros((len(worker_pool))) + resources_min_border = np.zeros(len(worker_pool)) for work_index, node in index2node.items(): for req in node.work_unit.worker_reqs: worker_index = worker_name2index[req.kind] diff --git a/tests/scheduler/genetic/converter_test.py b/tests/scheduler/genetic/converter_test.py index ae926fe9..6bc5ea78 100644 --- a/tests/scheduler/genetic/converter_test.py +++ b/tests/scheduler/genetic/converter_test.py @@ -7,7 +7,7 @@ def test_convert_schedule_to_chromosome(setup_toolbox): - (tb, _), setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox + tb, _, setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox schedule = HEFTScheduler().schedule(setup_wg, setup_contractors, validate=True, landscape=setup_landscape_many_holders) @@ -16,7 +16,7 @@ def test_convert_schedule_to_chromosome(setup_toolbox): def test_convert_chromosome_to_schedule(setup_toolbox): - (tb, _), setup_wg, setup_contractors, _, _ = setup_toolbox + tb, _, setup_wg, setup_contractors, _, _ = setup_toolbox chromosome = tb.generate_chromosome() schedule, _, _, _ = tb.chromosome_to_schedule(chromosome) @@ -26,7 +26,7 @@ def test_convert_chromosome_to_schedule(setup_toolbox): def test_converter_with_borders_contractor_accounting(setup_toolbox): - (tb, _), setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox + tb, _, setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox chromosome = tb.generate_chromosome(landscape=setup_landscape_many_holders) diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index a0c746e6..c2406a2c 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -1,19 +1,13 @@ from random import Random -from typing import List, Dict, Tuple +from typing import Tuple -import numpy as np -from deap.base import Toolbox from pytest import fixture -from sampo.scheduler.genetic.converter import ChromosomeType, convert_schedule_to_chromosome -from sampo.scheduler.genetic.operators import init_toolbox +from sampo.schemas.graph import GraphNode +import numpy as np + from sampo.scheduler.genetic.schedule_builder import create_toolbox -from sampo.schemas.contractor import Contractor, WorkerContractorPool, get_worker_contractor_pool -from sampo.schemas.graph import WorkGraph, GraphNode -from sampo.schemas.landscape import LandscapeConfiguration -from sampo.schemas.schedule import Schedule -from sampo.schemas.schedule_spec import ScheduleSpec -from sampo.schemas.time import Time +from sampo.schemas.contractor import get_worker_contractor_pool from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator @@ -46,6 +40,14 @@ def setup_toolbox(setup_default_schedules) -> tuple: rand = Random(123) work_estimator: WorkTimeEstimator = DefaultWorkEstimator() + nodes = [node for node in setup_wg.nodes if not node.is_inseparable_son()] + worker_name2index = {worker_name: index for index, worker_name in enumerate(setup_worker_pool)} + resources_min_border = np.zeros(len(setup_worker_pool)) + for work_index, node in enumerate(nodes): + for req in node.work_unit.worker_reqs: + worker_index = worker_name2index[req.kind] + resources_min_border[worker_index] = max(resources_min_border[worker_index], req.min_count) + return (create_toolbox(setup_wg, setup_contractors, setup_worker_pool, @@ -55,5 +57,5 @@ def setup_toolbox(setup_default_schedules) -> tuple: setup_default_schedules, rand, work_estimator=work_estimator, - landscape=setup_landscape_many_holders), + landscape=setup_landscape_many_holders), resources_min_border, setup_wg, setup_contractors, setup_default_schedules, setup_landscape_many_holders) diff --git a/tests/scheduler/genetic/operators_test.py b/tests/scheduler/genetic/operators_test.py index 4bed4644..d6789e98 100644 --- a/tests/scheduler/genetic/operators_test.py +++ b/tests/scheduler/genetic/operators_test.py @@ -6,7 +6,7 @@ def test_generate_individual(setup_toolbox): - (tb, _), _, _, _, _ = setup_toolbox + tb, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): chromosome: ChromosomeType = tb.generate_chromosome() @@ -14,7 +14,7 @@ def test_generate_individual(setup_toolbox): def test_mutate_order(setup_toolbox): - (tb, _), _, _, _, _ = setup_toolbox + tb, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() @@ -26,7 +26,7 @@ def test_mutate_order(setup_toolbox): def test_mutate_resources(setup_toolbox): - (tb, _), _, _, _, _ = setup_toolbox + tb, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() @@ -36,7 +36,7 @@ def test_mutate_resources(setup_toolbox): def test_mate_order(setup_toolbox, setup_wg): - (tb, _), _, _, _, _ = setup_toolbox + tb, _, _, _, _, _ = setup_toolbox _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) @@ -54,7 +54,7 @@ def test_mate_order(setup_toolbox, setup_wg): def test_mate_resources(setup_toolbox, setup_wg): - (tb, resources_border), _, _, _, _ = setup_toolbox + tb, resources_border, _, _, _, _ = setup_toolbox _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) From 10007f1eae348ef71182ec8b97b46c4f6ad2aa77 Mon Sep 17 00:00:00 2001 From: Egor Date: Wed, 23 Aug 2023 19:18:33 +0300 Subject: [PATCH 18/22] merge main into branch --- sampo/scheduler/genetic/base.py | 1 - sampo/scheduler/genetic/operators.py | 29 +++++++++------------ sampo/scheduler/genetic/schedule_builder.py | 12 ++++----- tests/scheduler/genetic/converter_test.py | 2 ++ tests/scheduler/genetic/fixtures.py | 8 +++--- tests/scheduler/genetic/operators_test.py | 3 ++- 6 files changed, 27 insertions(+), 28 deletions(-) diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index 291550a3..99a7aea1 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -1,4 +1,3 @@ -import math import random from typing import Optional, Callable diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 2ec64384..bd215e09 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -6,7 +6,7 @@ from operator import attrgetter import numpy as np -from deap import creator, base, tools +from deap import creator, base from sampo.scheduler.genetic.converter import convert_chromosome_to_schedule from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome, ChromosomeType @@ -129,7 +129,7 @@ def init_toolbox(wg: WorkGraph, work_id2index: dict[str, int], worker_name2index: dict[str, int], index2contractor_obj: dict[int, Contractor], - init_chromosomes: dict[str, ChromosomeType], + init_chromosomes: dict[str, tuple[ChromosomeType, float]], mutate_order: float, mutate_resources: float, population_size: int, @@ -200,14 +200,14 @@ def copy_chromosome(chromosome: ChromosomeType) -> ChromosomeType: def generate_population(n: int, wg: WorkGraph, contractors: list[Contractor], + spec: ScheduleSpec, work_id2index: dict[str, int], worker_name2index: dict[str, int], contractor2index: dict[str, int], contractor_borders: np.ndarray, - init_chromosomes: dict[str, ChromosomeType], - spec: ScheduleSpec, + init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]], rand: random.Random, - work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), + work_estimator: WorkTimeEstimator = None, landscape: LandscapeConfiguration = LandscapeConfiguration()) -> list[Individual]: """ Generates population. @@ -228,7 +228,7 @@ def randomized_init() -> ChromosomeType: chromosome_types = rand.sample(list(init_chromosomes.keys()) + ['topological'], k=n, counts=counts) - chromosomes = [Individual(init_chromosomes[generated_type]) + chromosomes = [Individual(init_chromosomes[generated_type][0]) if generated_type != 'topological' else Individual(randomized_init()) for generated_type in chromosome_types] @@ -241,7 +241,7 @@ def generate_chromosome(wg: WorkGraph, worker_name2index: dict[str, int], contractor2index: dict[str, int], contractor_borders: np.ndarray, - init_chromosomes: dict[str, ChromosomeType], + init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]], spec: ScheduleSpec, rand: random.Random, work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), @@ -264,23 +264,20 @@ def randomized_init() -> ChromosomeType: chance = rand.random() if chance < 0.2: - chromosome = init_chromosomes['heft_end'] + chromosome = init_chromosomes['heft_end'][0] elif chance < 0.4: - chromosome = init_chromosomes['heft_between'] + chromosome = init_chromosomes['heft_between'][0] elif chance < 0.5: - chromosome = init_chromosomes['12.5%'] + chromosome = init_chromosomes['12.5%'][0] elif chance < 0.6: - chromosome = init_chromosomes['25%'] + chromosome = init_chromosomes['25%'][0] elif chance < 0.7: - chromosome = init_chromosomes['75%'] + chromosome = init_chromosomes['75%'][0] elif chance < 0.8: - chromosome = init_chromosomes['87.5%'] + chromosome = init_chromosomes['87.5%'][0] else: chromosome = randomized_init() - if chromosome is None: - chromosome = randomized_init() - return Individual(chromosome) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 635f4978..a4c49dc8 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -79,16 +79,16 @@ def create_toolbox(wg: WorkGraph, for child in node_children: parents[child].append(node) - # initial chromosomes construction - init_chromosomes: dict[str, ChromosomeType] = \ - {name: convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, - contractor2index, contractor_borders, schedule, spec, order) + init_chromosomes: dict[str, tuple[ChromosomeType, int]] = \ + {name: (convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, + contractor2index, contractor_borders, schedule, chromosome_spec, order), + importance, chromosome_spec) if schedule is not None else None - for name, (schedule, order, spec) in init_schedules.items()} + for name, (schedule, order, chromosome_spec, importance) in init_schedules.items()} for name, chromosome in init_chromosomes.items(): if chromosome is not None: - if not is_chromosome_correct(chromosome, node_indices, parents): + if not is_chromosome_correct(chromosome[0], node_indices, parents): raise NoSufficientContractorError('HEFTs are deploying wrong chromosomes') print(f'Genetic optimizing took {(time.time() - start) * 1000} ms') diff --git a/tests/scheduler/genetic/converter_test.py b/tests/scheduler/genetic/converter_test.py index 6bc5ea78..ef3ffd6b 100644 --- a/tests/scheduler/genetic/converter_test.py +++ b/tests/scheduler/genetic/converter_test.py @@ -1,6 +1,8 @@ from uuid import uuid4 from tests.scheduler.genetic.fixtures import * +from sampo.schemas.schedule import Schedule +from sampo.schemas.contractor import Contractor from sampo.scheduler.heft.base import HEFTScheduler from sampo.schemas.resources import Worker from sampo.utilities.validation import validate_schedule diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index c2406a2c..14516e22 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -3,7 +3,6 @@ from pytest import fixture -from sampo.schemas.graph import GraphNode import numpy as np from sampo.scheduler.genetic.schedule_builder import create_toolbox @@ -42,11 +41,12 @@ def setup_toolbox(setup_default_schedules) -> tuple: nodes = [node for node in setup_wg.nodes if not node.is_inseparable_son()] worker_name2index = {worker_name: index for index, worker_name in enumerate(setup_worker_pool)} - resources_min_border = np.zeros(len(setup_worker_pool)) + resources_border = np.zeros((2, len(setup_worker_pool), len(nodes))) for work_index, node in enumerate(nodes): for req in node.work_unit.worker_reqs: worker_index = worker_name2index[req.kind] - resources_min_border[worker_index] = max(resources_min_border[worker_index], req.min_count) + resources_border[0, worker_index, work_index] = req.min_count + resources_border[1, worker_index, work_index] = req.max_count return (create_toolbox(setup_wg, setup_contractors, @@ -57,5 +57,5 @@ def setup_toolbox(setup_default_schedules) -> tuple: setup_default_schedules, rand, work_estimator=work_estimator, - landscape=setup_landscape_many_holders), resources_min_border, + landscape=setup_landscape_many_holders), resources_border, setup_wg, setup_contractors, setup_default_schedules, setup_landscape_many_holders) diff --git a/tests/scheduler/genetic/operators_test.py b/tests/scheduler/genetic/operators_test.py index d6789e98..d6f582db 100644 --- a/tests/scheduler/genetic/operators_test.py +++ b/tests/scheduler/genetic/operators_test.py @@ -1,5 +1,6 @@ from tests.scheduler.genetic.fixtures import * from sampo.scheduler.genetic.converter import ChromosomeType +import random TEST_ITERATIONS = 10 @@ -60,7 +61,7 @@ def test_mate_resources(setup_toolbox, setup_wg): population = tb.population(n=population_size) for i in range(TEST_ITERATIONS): - individual1, individual2 = population[:2] + individual1, individual2 = random.sample(population, 2) individual1, individual2 = tb.mate_resources(individual1, individual2) # check there are correct resources at mate positions From 9c9cefec65472261b3e5b3a166c8431ce06f5a6d Mon Sep 17 00:00:00 2001 From: Egor Date: Thu, 24 Aug 2023 14:20:46 +0300 Subject: [PATCH 19/22] add docs --- sampo/scheduler/genetic/operators.py | 8 ++++++++ sampo/scheduler/genetic/schedule_builder.py | 16 +++++++++++++++- 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index bd215e09..9700a546 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -282,6 +282,10 @@ def randomized_init() -> ChromosomeType: def select_new_population(population: list[ChromosomeType], pop_size: int) -> list[ChromosomeType]: + """ + Selection operator for genetic algorithm. + Selecting top n individuals in population. + """ population = sorted(population, key=attrgetter('fitness'), reverse=True) return population[:pop_size] @@ -365,6 +369,10 @@ def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: rand def mutate_scheduling_order(ind: ChromosomeType, mutpb: float, rand: random.Random) -> ChromosomeType: + """ + Mutation operator for order. + Swap neighbors. + """ order = ind[0] for i in range(1, len(order) - 2): if rand.random() < mutpb: diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index a4c49dc8..1087967f 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -138,7 +138,21 @@ def build_schedule(wg: WorkGraph, time_border: int = None, optimize_resources: bool = False) \ -> tuple[ScheduleWorkDict, Time, Timeline, list[GraphNode]]: - + """ + Genetic algorithm. + Structure of chromosome: + [[order of job], + [[numbers of workers types 1 for each job], [numbers of workers types 2], ... ], + [[border of workers types 1 for each contractor], [border of workers types 2], ...] + ] + + Different mate and mutation for order and for workers. + Generate order of job by prioritization from HEFTs and from Topological. + Generate resources from min to max. + Overall initial population is valid. + + :return: schedule + """ global_start = time.time() # preparing access-optimized data structures From cf4ef44bc9d143fbd05da849a54fbebf85533477 Mon Sep 17 00:00:00 2001 From: Egor Date: Thu, 24 Aug 2023 15:05:15 +0300 Subject: [PATCH 20/22] add weights --- sampo/scheduler/genetic/operators.py | 13 ++++++++++--- sampo/scheduler/genetic/schedule_builder.py | 4 ++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 9700a546..26aaf840 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -1,4 +1,5 @@ import random +import math from abc import ABC, abstractmethod from copy import deepcopy from functools import partial @@ -129,7 +130,7 @@ def init_toolbox(wg: WorkGraph, work_id2index: dict[str, int], worker_name2index: dict[str, int], index2contractor_obj: dict[int, Contractor], - init_chromosomes: dict[str, tuple[ChromosomeType, float]], + init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]], mutate_order: float, mutate_resources: float, population_size: int, @@ -222,9 +223,15 @@ def randomized_init() -> ChromosomeType: count_for_specified_types = (n // 3) // len(init_chromosomes) count_for_specified_types = count_for_specified_types if count_for_specified_types > 0 else 1 - count_for_topological = n - count_for_specified_types * len(init_chromosomes) + sum_counts_for_specified_types = count_for_specified_types * len(init_chromosomes) + counts = [count_for_specified_types * importance for _, importance, _ in init_chromosomes.values()] + + weights_multiplier = math.ceil(sum_counts_for_specified_types / sum(counts)) + counts = [count * weights_multiplier for count in counts] + + count_for_topological = n - sum_counts_for_specified_types count_for_topological = count_for_topological if count_for_topological > 0 else 1 - counts = [count_for_specified_types for _ in range(len(init_chromosomes))] + [count_for_topological] + counts += [count_for_topological] chromosome_types = rand.sample(list(init_chromosomes.keys()) + ['topological'], k=n, counts=counts) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 1087967f..5c315993 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -27,7 +27,7 @@ def create_toolbox(wg: WorkGraph, population_size: int, mutate_order: float, mutate_resources: float, - init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], + init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, int]], rand: random.Random, spec: ScheduleSpec = ScheduleSpec(), work_estimator: WorkTimeEstimator = None, @@ -125,7 +125,7 @@ def build_schedule(wg: WorkGraph, generation_number: int, mutpb_order: float, mutpb_res: float, - init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec]], + init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, int]], rand: random.Random, spec: ScheduleSpec, landscape: LandscapeConfiguration = LandscapeConfiguration(), From 334172b5d5ab51ac2236647f60414062fd89718f Mon Sep 17 00:00:00 2001 From: Egor Date: Thu, 24 Aug 2023 15:07:51 +0300 Subject: [PATCH 21/22] add weights --- sampo/scheduler/genetic/schedule_builder.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 5c315993..e4246222 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -27,7 +27,7 @@ def create_toolbox(wg: WorkGraph, population_size: int, mutate_order: float, mutate_resources: float, - init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, int]], + init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, float]], rand: random.Random, spec: ScheduleSpec = ScheduleSpec(), work_estimator: WorkTimeEstimator = None, @@ -79,7 +79,7 @@ def create_toolbox(wg: WorkGraph, for child in node_children: parents[child].append(node) - init_chromosomes: dict[str, tuple[ChromosomeType, int]] = \ + init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]] = \ {name: (convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, contractor2index, contractor_borders, schedule, chromosome_spec, order), importance, chromosome_spec) @@ -125,7 +125,7 @@ def build_schedule(wg: WorkGraph, generation_number: int, mutpb_order: float, mutpb_res: float, - init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, int]], + init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, float]], rand: random.Random, spec: ScheduleSpec, landscape: LandscapeConfiguration = LandscapeConfiguration(), From fa8c52df3abd2c158301a0a9367a37e29d3fa043 Mon Sep 17 00:00:00 2001 From: Egor Date: Thu, 24 Aug 2023 15:41:13 +0300 Subject: [PATCH 22/22] add weights --- sampo/scheduler/genetic/base.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index 99a7aea1..9a7343e7 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -54,7 +54,7 @@ def __init__(self, self.fitness_constructor = fitness_constructor self.work_estimator = work_estimator self._n_cpu = n_cpu - self._weights = None + self._weights = weights self._time_border = None self._deadline = None @@ -114,8 +114,8 @@ def set_deadline(self, deadline: Time): def set_weights(self, weights: list[int]): self._weights = weights - @classmethod - def generate_first_population(self, wg: WorkGraph, contractors: list[Contractor], + @staticmethod + def generate_first_population(wg: WorkGraph, contractors: list[Contractor], landscape: LandscapeConfiguration = LandscapeConfiguration(), spec: ScheduleSpec = ScheduleSpec(), work_estimator: WorkTimeEstimator = None,