diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index 9a7343e7..2d4bb778 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -1,8 +1,6 @@ import random from typing import Optional, Callable -from deap.base import Toolbox - from sampo.scheduler.base import Scheduler, SchedulerType from sampo.scheduler.genetic.operators import FitnessFunction, TimeFitness from sampo.scheduler.genetic.schedule_builder import build_schedule @@ -39,10 +37,12 @@ def __init__(self, seed: Optional[float or None] = None, n_cpu: int = 1, weights: list[int] = None, - fitness_constructor: Callable[[Toolbox], FitnessFunction] = TimeFitness, + fitness_constructor: Callable[[Time | None], FitnessFunction] = TimeFitness, scheduler_type: SchedulerType = SchedulerType.Genetic, resource_optimizer: ResourceOptimizer = IdentityResourceOptimizer(), - work_estimator: WorkTimeEstimator = DefaultWorkEstimator()): + work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), + optimize_resources: bool = False, + verbose: bool = True): super().__init__(scheduler_type=scheduler_type, resource_optimizer=resource_optimizer, work_estimator=work_estimator) @@ -53,8 +53,10 @@ def __init__(self, self.rand = rand or random.Random(seed) self.fitness_constructor = fitness_constructor self.work_estimator = work_estimator + self._optimize_resources = optimize_resources self._n_cpu = n_cpu self._weights = weights + self._verbose = verbose self._time_border = None self._deadline = None @@ -114,6 +116,12 @@ def set_deadline(self, deadline: Time): def set_weights(self, weights: list[int]): self._weights = weights + def set_optimize_resources(self, optimize_resources: bool): + self._optimize_resources = optimize_resources + + def set_verbose(self, verbose: bool): + self._verbose = verbose + @staticmethod def generate_first_population(wg: WorkGraph, contractors: list[Contractor], landscape: LandscapeConfiguration = LandscapeConfiguration(), @@ -199,6 +207,8 @@ def schedule_with_cache(self, mutate_order, mutate_resources, size_of_population = self.get_params(wg.vertex_count) worker_pool = get_worker_contractor_pool(contractors) + fitness_object = self.fitness_constructor(self._deadline) + deadline = None if self._optimize_resources else self._deadline scheduled_works, schedule_start_time, timeline, order_nodes = build_schedule(wg, contractors, @@ -211,12 +221,15 @@ def schedule_with_cache(self, self.rand, spec, landscape, - self.fitness_constructor, + fitness_object, self.work_estimator, - n_cpu=self._n_cpu, - assigned_parent_time=assigned_parent_time, - timeline=timeline, - time_border=self._time_border) + self._n_cpu, + assigned_parent_time, + timeline, + self._time_border, + self._optimize_resources, + deadline, + self._verbose) schedule = Schedule.from_scheduled_works(scheduled_works.values(), wg) if validate: diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 26aaf840..1acd022e 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -2,9 +2,9 @@ import math from abc import ABC, abstractmethod from copy import deepcopy -from functools import partial -from typing import Iterable, Callable +from typing import Iterable from operator import attrgetter +from enum import Enum import numpy as np from deap import creator, base @@ -22,6 +22,7 @@ from sampo.schemas.time import Time from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator from sampo.utilities.resource_cost import schedule_cost +from sortedcontainers import SortedList # logger = mp.log_to_stderr(logging.DEBUG) @@ -31,14 +32,14 @@ class FitnessFunction(ABC): """ Base class for description of different fitness functions. """ - - def __init__(self, evaluator: Callable[[list[ChromosomeType]], list[Schedule]]): - self._evaluator = evaluator + + def __init__(self, deadline: Time | None): + self._deadline = deadline @abstractmethod - def evaluate(self, chromosomes: list[ChromosomeType]) -> list[int]: + def evaluate(self, schedules: list[Schedule]) -> list[int]: """ - Calculate the value of fitness function of the all chromosomes. + Calculate the value of fitness function of the all schedules. It is better when value is less. """ ... @@ -48,12 +49,12 @@ class TimeFitness(FitnessFunction): """ Fitness function that relies on finish time. """ + + def __init__(self, deadline: Time | None = None): + super().__init__(deadline) - def __init__(self, evaluator: Callable[[list[ChromosomeType]], list[Schedule]]): - super().__init__(evaluator) - - def evaluate(self, chromosomes: list[ChromosomeType]) -> list[int]: - return [schedule.execution_time.value for schedule in self._evaluator(chromosomes)] + def evaluate(self, schedules: list[Schedule]) -> list[int]: + return [schedule.execution_time.value for schedule in schedules] class TimeAndResourcesFitness(FitnessFunction): @@ -61,12 +62,11 @@ class TimeAndResourcesFitness(FitnessFunction): Fitness function that relies on finish time and the set of resources. """ - def __init__(self, evaluator: Callable[[list[ChromosomeType]], list[Schedule]]): - super().__init__(evaluator) + def __init__(self, deadline: Time | None = None): + super().__init__(deadline) - def evaluate(self, chromosomes: list[ChromosomeType]) -> list[int]: - evaluated = self._evaluator(chromosomes) - return [schedule.execution_time.value + get_absolute_peak_resource_usage(schedule) for schedule in evaluated] + def evaluate(self, schedules: list[Schedule]) -> list[int]: + return [schedule.execution_time.value + get_absolute_peak_resource_usage(schedule) for schedule in schedules] class DeadlineResourcesFitness(FitnessFunction): @@ -74,22 +74,13 @@ class DeadlineResourcesFitness(FitnessFunction): The fitness function is dependent on the set of resources and requires the end time to meet the deadline. """ - def __init__(self, deadline: Time, evaluator: Callable[[list[ChromosomeType]], list[Schedule]]): - super().__init__(evaluator) - self._deadline = deadline - - @staticmethod - def prepare(deadline: Time): - """ - Returns the constructor of that fitness function prepared to use in Genetic - """ - return partial(DeadlineResourcesFitness, deadline) + def __init__(self, deadline: Time): + super().__init__(deadline) - def evaluate(self, chromosomes: list[ChromosomeType]) -> list[int]: - evaluated = self._evaluator(chromosomes) + def evaluate(self, schedules: list[Schedule]) -> list[int]: return [int(get_absolute_peak_resource_usage(schedule) * max(1.0, schedule.execution_time.value / self._deadline.value)) - for schedule in evaluated] + for schedule in schedules] class DeadlineCostFitness(FitnessFunction): @@ -97,22 +88,13 @@ class DeadlineCostFitness(FitnessFunction): The fitness function is dependent on the cost of resources and requires the end time to meet the deadline. """ - def __init__(self, deadline: Time, evaluator: Callable[[list[ChromosomeType]], list[Schedule]]): - super().__init__(evaluator) - self._deadline = deadline - - @staticmethod - def prepare(deadline: Time): - """ - Returns the constructor of that fitness function prepared to use in Genetic - """ - return partial(DeadlineCostFitness, deadline) + def __init__(self, deadline: Time): + super().__init__(deadline) - def evaluate(self, chromosomes: list[ChromosomeType]) -> list[int]: - evaluated = self._evaluator(chromosomes) + def evaluate(self, schedules: list[Schedule]) -> list[int]: # TODO Integrate cost calculation to native module return [int(schedule_cost(schedule) * max(1.0, schedule.execution_time.value / self._deadline.value)) - for schedule in evaluated] + for schedule in schedules] # create class FitnessMin, the weights = -1 means that fitness - is function for minimum @@ -122,6 +104,14 @@ def evaluate(self, chromosomes: list[ChromosomeType]) -> list[int]: Individual = creator.Individual +class IndividualType(Enum): + """ + Class to define a type of individual in genetic algorithm + """ + Population = 'population' + Offspring = 'offspring' + + def init_toolbox(wg: WorkGraph, contractors: list[Contractor], worker_pool: WorkerContractorPool, @@ -131,8 +121,8 @@ def init_toolbox(wg: WorkGraph, worker_name2index: dict[str, int], index2contractor_obj: dict[int, Contractor], init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]], - mutate_order: float, - mutate_resources: float, + mut_order_pb: float, + mut_res_pb: float, population_size: int, rand: random.Random, spec: ScheduleSpec, @@ -140,16 +130,15 @@ def init_toolbox(wg: WorkGraph, contractor2index: dict[str, int], contractor_borders: np.ndarray, node_indices: list[int], - parents: dict[int, list[int]], + parents: dict[int, set[int]], resources_border: np.ndarray, - resources_min_border: np.ndarray, assigned_parent_time: Time = Time(0), work_estimator: WorkTimeEstimator = DefaultWorkEstimator()) -> base.Toolbox: """ Object, that include set of functions (tools) for genetic model and other functions related to it. - list of parameters that received this function is sufficient and complete to manipulate with genetic + list of parameters that received this function is sufficient and complete to manipulate with genetic algorithm - :return: Object, included tools for genetic + :return: Object, included tools for genetic algorithm """ toolbox = base.Toolbox() # generate chromosome @@ -166,22 +155,26 @@ def init_toolbox(wg: WorkGraph, init_chromosomes=init_chromosomes, rand=rand, work_estimator=work_estimator, landscape=landscape) # selection toolbox.register('select', select_new_population, pop_size=population_size) + # combined crossover + toolbox.register('mate', mate, rand=rand) + # combined mutation + toolbox.register('mutate', mutate, order_mutpb=mut_order_pb, res_mutpb=mut_res_pb, rand=rand, + parents=parents, resources_border=resources_border) # crossover for order - toolbox.register('mate', mate_scheduling_order, rand=rand) + toolbox.register('mate_order', mate_scheduling_order, rand=rand) # mutation for order - toolbox.register('mutate', mutate_scheduling_order, mutpb=mutate_order, rand=rand) + toolbox.register('mutate_order', mutate_scheduling_order, mutpb=mut_order_pb, rand=rand, parents=parents) # crossover for resources - toolbox.register('mate_resources', mate_for_resources, rand=rand) + toolbox.register('mate_resources', mate_resources, rand=rand) # mutation for resources - toolbox.register('mutate_resources', mutate_for_resources, resources_border=resources_border, - mutpb=mutate_resources, rand=rand) - # crossover for resource borders - toolbox.register('mate_resource_borders', mate_for_resource_borders, rand=rand) + toolbox.register('mutate_resources', mutate_resources, resources_border=resources_border, + mutpb=mut_res_pb, rand=rand) # mutation for resource borders - toolbox.register('mutate_resource_borders', mutate_resource_borders, resources_min_border=resources_min_border, - mutpb=mutate_resources, rand=rand) + toolbox.register('mutate_resource_borders', mutate_resource_borders, contractor_borders=contractor_borders, + mutpb=mut_res_pb, rand=rand) - toolbox.register('validate', is_chromosome_correct, node_indices=node_indices, parents=parents) + toolbox.register('validate', is_chromosome_correct, node_indices=node_indices, parents=parents, + contractor_borders=contractor_borders) toolbox.register('schedule_to_chromosome', convert_schedule_to_chromosome, wg=wg, work_id2index=work_id2index, worker_name2index=worker_name2index, contractor2index=contractor2index, contractor_borders=contractor_borders, spec=spec) @@ -191,6 +184,9 @@ def init_toolbox(wg: WorkGraph, work_estimator=work_estimator, worker_name2index=worker_name2index, contractor2index=contractor2index, landscape=landscape) + toolbox.register('copy_individual', lambda ind: Individual(copy_chromosome(ind))) + toolbox.register('update_resource_borders_to_peak_values', update_resource_borders_to_peak_values, + worker_name2index=worker_name2index, contractor2index=contractor2index) return toolbox @@ -291,23 +287,22 @@ def randomized_init() -> ChromosomeType: def select_new_population(population: list[ChromosomeType], pop_size: int) -> list[ChromosomeType]: """ Selection operator for genetic algorithm. - Selecting top n individuals in population. + Select top n individuals in population. """ population = sorted(population, key=attrgetter('fitness'), reverse=True) return population[:pop_size] -def is_chromosome_correct(chromosome: ChromosomeType, - node_indices: list[int], - parents: dict[int, list[int]]) -> bool: +def is_chromosome_correct(chromosome: ChromosomeType, node_indices: list[int], parents: dict[int, set[int]], + contractor_borders: np.ndarray) -> bool: """ - Check order of works and contractors. + Check correctness of works order and contractors borders. """ return is_chromosome_order_correct(chromosome, parents) and \ - is_chromosome_contractors_correct(chromosome, node_indices) + is_chromosome_contractors_correct(chromosome, node_indices, contractor_borders) -def is_chromosome_order_correct(chromosome: ChromosomeType, parents: dict[int, list[int]]) -> bool: +def is_chromosome_order_correct(chromosome: ChromosomeType, parents: dict[int, set[int]]) -> bool: """ Checks that assigned order of works are topologically correct. """ @@ -315,175 +310,306 @@ def is_chromosome_order_correct(chromosome: ChromosomeType, parents: dict[int, l used = set() for work_index in work_order: used.add(work_index) - for parent in parents[work_index]: - if parent not in used: - # logger.error(f'Order validation failed: {work_order}') - return False + if not parents[work_index].issubset(used): + # logger.error(f'Order validation failed: {work_order}') + return False return True -def is_chromosome_contractors_correct(chromosome: ChromosomeType, - work_indices: Iterable[int]) -> bool: +def is_chromosome_contractors_correct(chromosome: ChromosomeType, work_indices: Iterable[int], + contractor_borders: np.ndarray) -> bool: """ Checks that assigned contractors can supply assigned workers. """ - for work_ind in work_indices: - resources_count = chromosome[1][work_ind, :-1] - contractor_ind = chromosome[1][work_ind, -1] - contractor_border = chromosome[2][contractor_ind] - for ind, count in enumerate(resources_count): - if contractor_border[ind] < count: - # logger.error(f'Contractor border validation failed: {contractor_border[ind]} < {count}') - return False - return True + if not work_indices: + return True + resources = chromosome[1][work_indices] + resources = resources[resources[:, -1].argsort()] + contractors, indexes = np.unique(resources[:, -1], return_index=True) + chromosome_borders = chromosome[2][contractors] + res_grouped_by_contractor = np.split(resources[:, :-1], indexes[1:]) + max_of_res_by_contractor = np.array([r.max(axis=0) for r in res_grouped_by_contractor]) + return (max_of_res_by_contractor <= chromosome_borders).all() and \ + (chromosome_borders <= contractor_borders[contractors]).all() -def get_order_tail(head_set: np.ndarray, other: np.ndarray) -> np.ndarray: +def get_order_part(order: np.ndarray, other_order: np.ndarray) -> np.ndarray: """ - Get a new tail in topologic order for chromosome. + Get a new part in topologic order for chromosome. This function is needed to make crossover for order. """ - head_set = set(head_set) - return np.array([node for node in other if node not in head_set]) + order = set(order) + return np.array([node for node in other_order if node not in order]) -def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: random.Random) \ - -> (ChromosomeType, ChromosomeType): +def mate_scheduling_order(ind1: ChromosomeType, ind2: ChromosomeType, rand: random.Random, copy: bool = True) \ + -> tuple[ChromosomeType, ChromosomeType]: """ - Crossover for order. - Basis crossover is cxOnePoint. - But we checked not repeated works in individual order. + Two-Point crossover for order. + + :param ind1: first individual + :param ind2: second individual + :param rand: the rand object used for randomized operations + :param copy: if True individuals will be copied before mating so as not to change them - :return: two cross individuals + :return: two mated individuals """ - child1 = Individual(copy_chromosome(ind1)) - child2 = Individual(copy_chromosome(ind2)) + child1, child2 = (Individual(copy_chromosome(ind1)), Individual(copy_chromosome(ind2))) if copy else (ind1, ind2) + + order1, order2 = child1[0], child2[0] + + min_mating_amount = len(order1) // 4 - order1 = child1[0] - order2 = child2[0] + # randomly select the points where the crossover will take place + mating_amount = rand.randint(min_mating_amount, 3 * min_mating_amount) + crossover_head_point = rand.randint(1, mating_amount - 1) + crossover_tail_point = mating_amount - crossover_head_point - border = len(order1) // 4 - # randomly select the point where the crossover will take place - crossover_point = rand.randint(border, len(order1) - border) + ind_new_part = get_order_part(np.concatenate((order1[:crossover_head_point], order1[-crossover_tail_point:])), + order2) + order1[crossover_head_point:-crossover_tail_point] = ind_new_part - ind1_new_tail = get_order_tail(order1[:crossover_point], order2) - ind2_new_tail = get_order_tail(order2[:crossover_point], order1) + # randomly select the points where the crossover will take place + mating_amount = rand.randint(min_mating_amount, 3 * min_mating_amount) + crossover_head_point = rand.randint(1, mating_amount - 1) + crossover_tail_point = mating_amount - crossover_head_point - order1[crossover_point:] = ind1_new_tail - order2[crossover_point:] = ind2_new_tail + ind_new_part = get_order_part(np.concatenate((order2[:crossover_head_point], order2[-crossover_tail_point:])), + order1) + order2[crossover_head_point:-crossover_tail_point] = ind_new_part return child1, child2 -def mutate_scheduling_order(ind: ChromosomeType, mutpb: float, rand: random.Random) -> ChromosomeType: +def mutate_scheduling_order(ind: ChromosomeType, mutpb: float, rand: random.Random, + parents: dict[int, set[int]]) -> ChromosomeType: """ Mutation operator for order. Swap neighbors. + + :param ind: the individual to be mutated + :param mutpb: probability of gene mutation + :param rand: the rand object used for randomized operations + :param parents: mapping object of works and their parent-works to create valid order + + :return: mutated individual """ order = ind[0] - for i in range(1, len(order) - 2): - if rand.random() < mutpb: - order[i], order[i + 1] = order[i + 1], order[i] + num_possible_muts = len(order) - 3 + mask = np.array([rand.random() < mutpb for _ in range(num_possible_muts)]) + if mask.any(): + indexes_to_mutate = [rand.randint(1, num_possible_muts + 1) for _ in range(mask.sum())] + for i in indexes_to_mutate: + if order[i] not in parents[order[i + 1]]: + order[i], order[i + 1] = order[i + 1], order[i] return ind -def mate_for_resources(ind1: ChromosomeType, ind2: ChromosomeType, - rand: random.Random) -> (ChromosomeType, ChromosomeType): +def mate_resources(ind1: ChromosomeType, ind2: ChromosomeType, optimize_resources: bool, + rand: random.Random, copy: bool = True) -> tuple[ChromosomeType, ChromosomeType]: """ - CxOnePoint for resources. + One-Point crossover for resources. :param ind1: first individual :param ind2: second individual - :param rand: the rand object used for exchange point selection - :return: first and second individual + :param optimize_resources: if True resource borders should be changed after mating + :param rand: the rand object used for randomized operations + :param copy: if True individuals will be copied before mating so as not to change them + + :return: two mated individuals """ - child1 = Individual(copy_chromosome(ind1)) - child2 = Individual(copy_chromosome(ind2)) + child1, child2 = (Individual(copy_chromosome(ind1)), Individual(copy_chromosome(ind2))) if copy else (ind1, ind2) - res1 = child1[1] - res2 = child2[1] + res1, res2 = child1[1], child2[1] num_works = len(res1) - border = num_works // 4 - cxpoint = rand.randint(border, num_works - border) - + min_mating_amount = num_works // 4 + cxpoint = rand.randint(min_mating_amount, num_works - min_mating_amount) mate_positions = rand.sample(range(num_works), cxpoint) res1[mate_positions], res2[mate_positions] = res2[mate_positions], res1[mate_positions] + + if optimize_resources: + for res, child in zip([res1, res2], [child1, child2]): + mated_resources = res[mate_positions] + contractors = np.unique(mated_resources[:, -1]) + child[2][contractors] = np.stack((child1[2][contractors], child2[2][contractors]), axis=0).max(axis=0) + return child1, child2 -def mutate_for_resources(ind: ChromosomeType, resources_border: np.ndarray, - mutpb: float, rand: random.Random) -> ChromosomeType: +def mutate_resources(ind: ChromosomeType, mutpb: float, rand: random.Random, + resources_border: np.ndarray) -> ChromosomeType: """ Mutation function for resources. It changes selected numbers of workers in random work in a certain interval for this work. - :return: mutate individual + :param ind: the individual to be mutated + :param resources_border: low and up borders of resources amounts + :param mutpb: probability of gene mutation + :param rand: the rand object used for randomized operations + + :return: mutated individual """ - # select random number from interval from min to max from uniform distribution res = ind[1] - res_count = len(res[0]) - for i, work_res in enumerate(res): - for type_of_res in range(res_count - 1): - if rand.random() < mutpb: - xl = resources_border[0, type_of_res, i] - xu = resources_border[1, type_of_res, i] - contractor = work_res[-1] - border = ind[2][contractor, type_of_res] - # TODO Debug why min(xu, border) can be lower than xl - work_res[type_of_res] = rand.randint(xl, max(xl, min(xu, border))) - if rand.random() < mutpb: - work_res[-1] = rand.randint(0, len(ind[2]) - 1) + num_works = len(res) + + num_contractors = len(ind[2]) + if num_contractors > 1: + mask = np.array([rand.random() < mutpb for _ in range(num_works)]) + if mask.any(): + new_contractors = np.array([rand.randint(0, num_contractors - 1) for _ in range(mask.sum())]) + contractor_mask = (res[mask, :-1] <= ind[2][new_contractors]).all(axis=1) + new_contractors = new_contractors[contractor_mask] + mask &= contractor_mask + res[mask, -1] = new_contractors + + num_res = len(res[0, :-1]) + res_indexes = np.arange(0, num_res) + works_indexes = np.arange(0, num_works) + masks = np.array([[rand.random() < mutpb for _ in range(num_res)] for _ in range(num_works)]) + mask = masks.any(axis=1) + + if not mask.any(): + return ind + + works_indexes, masks = works_indexes[mask], masks[mask] + res_up_borders = np.stack((resources_border[1].T[mask], ind[2][res[mask, -1]]), axis=0).min(axis=0) + res_low_borders = resources_border[0].T[mask] + masks &= res_up_borders != res_low_borders + mask = masks.any(axis=1) + + mutate_values(res, works_indexes[mask], res_indexes, res_low_borders[mask], res_up_borders[mask], masks[mask], -1, + rand) return ind -def mate_for_resource_borders(ind1: ChromosomeType, ind2: ChromosomeType, - rand: random.Random) -> (ChromosomeType, ChromosomeType): +def mate(ind1: ChromosomeType, ind2: ChromosomeType, optimize_resources: bool, rand: random.Random) \ + -> tuple[ChromosomeType, ChromosomeType]: """ - Crossover for contractors' resource borders. + Combined crossover function of Two-Point crossover for order and One-Point crossover for resources. + + :param ind1: first individual + :param ind2: second individual + :param optimize_resources: if True resource borders should be changed after mating + :param rand: the rand object used for randomized operations + + :return: two mated individuals """ - child1 = Individual(copy_chromosome(ind1)) - child2 = Individual(copy_chromosome(ind2)) + child1, child2 = mate_scheduling_order(ind1, ind2, rand, copy=True) + child1, child2 = mate_resources(child1, child2, optimize_resources, rand, copy=False) - borders1 = child1[2] - borders2 = child2[2] - num_contractors = len(borders1) - contractors = rand.sample(range(num_contractors), rand.randint(1, num_contractors)) + return child1, child2 - num_res = len(borders1[0]) - res_indices = list(range(num_res)) - border = num_res // 4 - mate_positions = rand.sample(res_indices, rand.randint(border, num_res - border)) - (borders1[contractors, mate_positions], - borders2[contractors, mate_positions]) = (borders2[contractors, mate_positions], - borders1[contractors, mate_positions]) +def mutate(ind: ChromosomeType, resources_border: np.ndarray, parents: dict[int, set[int]], + order_mutpb: float, res_mutpb: float, rand: random.Random) -> ChromosomeType: + """ + Combined mutation function of mutation for order and mutation for resources. - return child1, child2 + :param ind: the individual to be mutated + :param resources_border: low and up borders of resources amounts + :param parents: mapping object of works and their parent-works to create valid order + :param order_mutpb: probability of order's gene mutation + :param res_mutpb: probability of resources' gene mutation + :param rand: the rand object used for randomized operations + + :return: mutated individual + """ + mutant = mutate_scheduling_order(ind, order_mutpb, rand, parents) + mutant = mutate_resources(mutant, res_mutpb, rand, resources_border) + return mutant -def mutate_resource_borders(ind: ChromosomeType, resources_min_border: np.ndarray, - mutpb: float, rand: random.Random) -> ChromosomeType: + +def mutate_resource_borders(ind: ChromosomeType, mutpb: float, rand: random.Random, + contractor_borders: np.ndarray) -> ChromosomeType: """ Mutation for contractors' resource borders. + + :param ind: the individual to be mutated + :param contractor_borders: up borders of contractors capacity + :param mutpb: probability of gene mutation + :param rand: the rand object used for randomized operations + + :return: mutated individual """ - num_resources = len(resources_min_border) - num_contractors = len(ind[2]) - type_of_res = rand.randint(0, len(ind[2][0]) - 1) - for contractor in range(num_contractors): - if rand.random() < mutpb: - ind[2][contractor][type_of_res] -= rand.randint(resources_min_border[type_of_res] + 1, - max(resources_min_border[type_of_res] + 1, - ind[2][contractor][type_of_res] // 10)) - if ind[2][contractor][type_of_res] <= 0: - ind[2][contractor][type_of_res] = 1 - - # find and correct all invalidated resource assignments - for work in range(len(ind[0])): - if ind[1][work][num_resources] == contractor: - ind[1][work][type_of_res] = min(ind[1][work][type_of_res], - ind[2][contractor][type_of_res]) + borders = ind[2] + res = ind[1] + num_res = len(res[0, :-1]) + res_indexes = np.arange(0, num_res) + resources = res[res[:, -1].argsort()] + contractors, indexes = np.unique(resources[:, -1], return_index=True) + res_grouped_by_contractor = np.split(resources[:, :-1], indexes[1:]) + masks = np.array([[rand.random() < mutpb for _ in range(num_res)] for _ in contractors]) + mask = masks.any(axis=1) + + if not mask.any(): + return ind + + contractors, masks = contractors[mask], masks[mask] + contractor_up_borders = contractor_borders[contractors] + contractor_low_borders = np.array([r.max(axis=0) for r, is_mut in zip(res_grouped_by_contractor, mask) if is_mut]) + masks &= contractor_up_borders != contractor_low_borders + mask = masks.any(axis=1) + + mutate_values(borders, contractors[mask], res_indexes, contractor_low_borders[mask], contractor_up_borders[mask], + masks[mask], len(res_indexes), rand) + + return ind + +def mutate_values(chromosome_part: np.ndarray, row_indexes: np.ndarray, col_indexes: np.ndarray, + low_borders: np.ndarray, up_borders: np.ndarray, masks: np.ndarray, mut_part: int, + rand: random.Random) -> None: + """ + Changes numeric values in m x n part of chromosome. + This function is needed to make mutation for resources and resource borders. + """ + for row_index, l_borders, u_borders, row_mask in zip(row_indexes, low_borders, up_borders, masks): + cur_row = chromosome_part[row_index] + for col_index, current_amount, l_border, u_border in zip(col_indexes[row_mask], cur_row[:mut_part][row_mask], + l_borders[row_mask], u_borders[row_mask]): + choices = np.concatenate((np.arange(l_border, current_amount), + np.arange(current_amount + 1, u_border + 1))) + weights = 1 / abs(choices - current_amount) + cur_row[col_index] = rand.choices(choices, weights=weights)[0] + + +def update_resource_borders_to_peak_values(ind: ChromosomeType, schedule: Schedule, worker_name2index: dict[str, int], + contractor2index: dict[str, int]): + """ + Changes the resource borders to the peak values obtained in the schedule. + + :param ind: the individual to be updated + :param schedule: schedule obtained from the individual + :param worker_name2index: mapping object of resources and their index in chromosome + :param contractor2index: mapping object of contractors and their index in chromosome + + :return: individual with updated resource borders + """ + df = schedule.full_schedule_df + contractors = set(df.contractor) + actual_borders = np.zeros_like(ind[2]) + for contractor in contractors: + contractor_df = df[df.contractor == contractor] + points = contractor_df[['start', 'finish']].to_numpy().copy() + points[:, 1] += 1 + points = SortedList(set(points.flatten())) + contractor_res_schedule = np.zeros((len(points), len(worker_name2index))) + contractor_id = '' + for _, r in contractor_df.iterrows(): + start = points.bisect_left(r['start']) + finish = points.bisect_left(r['finish'] + 1) + swork = r['scheduled_work_object'] + workers = np.array([[worker_name2index[worker.name], worker.count] for worker in swork.workers]) + if len(workers): + contractor_res_schedule[start: finish, workers[:, 0]] += workers[:, 1] + if not contractor_id: + contractor_id = swork.workers[0].contractor_id + if contractor_id: + index = contractor2index[contractor_id] + actual_borders[index] = contractor_res_schedule.max(axis=0) + ind[2][:] = actual_borders return ind diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index e4246222..17cf3c6f 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -1,38 +1,39 @@ import random import time -from typing import Callable import numpy as np from deap import tools from deap.base import Toolbox from sampo.scheduler.genetic.converter import convert_schedule_to_chromosome -from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, \ - FitnessFunction, TimeFitness, is_chromosome_correct +from sampo.scheduler.genetic.operators import init_toolbox, ChromosomeType, IndividualType, FitnessFunction, TimeFitness from sampo.scheduler.native_wrapper import NativeWrapper from sampo.scheduler.timeline.base import Timeline from sampo.schemas.contractor import Contractor, WorkerContractorPool -from sampo.schemas.exceptions import NoSufficientContractorError from sampo.schemas.graph import GraphNode, WorkGraph from sampo.schemas.landscape import LandscapeConfiguration from sampo.schemas.schedule import ScheduleWorkDict, Schedule from sampo.schemas.schedule_spec import ScheduleSpec from sampo.schemas.time import Time from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator - - -def create_toolbox(wg: WorkGraph, - contractors: list[Contractor], - worker_pool: WorkerContractorPool, - population_size: int, - mutate_order: float, - mutate_resources: float, - init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, float]], - rand: random.Random, - spec: ScheduleSpec = ScheduleSpec(), - work_estimator: WorkTimeEstimator = None, - landscape: LandscapeConfiguration = LandscapeConfiguration()) \ - -> Toolbox: +from sampo.scheduler.utils.peaks import get_absolute_peak_resource_usage +from sampo.schemas.resources import Worker + + +def create_toolbox_and_mapping_objects(wg: WorkGraph, + contractors: list[Contractor], + worker_pool: WorkerContractorPool, + population_size: int, + mutate_order: float, + mutate_resources: float, + init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, float]], + rand: random.Random, + spec: ScheduleSpec = ScheduleSpec(), + work_estimator: WorkTimeEstimator = None, + assigned_parent_time: Time = Time(0), + landscape: LandscapeConfiguration = LandscapeConfiguration(), + verbose: bool = True) \ + -> tuple[Toolbox, dict[str, int], dict[int, dict[int, Worker]], dict[int, list[int]]]: start = time.time() # preparing access-optimized data structures @@ -48,20 +49,6 @@ def create_toolbox(wg: WorkGraph, } for worker_name, workers_of_type in worker_pool.items()} node_indices = list(range(len(nodes))) - resources_border = np.zeros((2, len(worker_pool), len(index2node))) - resources_min_border = np.zeros(len(worker_pool)) - for work_index, node in index2node.items(): - for req in node.work_unit.worker_reqs: - worker_index = worker_name2index[req.kind] - resources_border[0, worker_index, work_index] = req.min_count - resources_border[1, worker_index, work_index] = req.max_count - resources_min_border[worker_index] = max(resources_min_border[worker_index], req.min_count) - - contractor_borders = np.zeros((len(contractor2index), len(worker_name2index)), dtype=int) - for ind, contractor in enumerate(contractors): - for ind_worker, worker in enumerate(contractor.workers.values()): - contractor_borders[ind, ind_worker] = worker.count - # construct inseparable_child -> inseparable_parent mapping inseparable_parents = {} for node in nodes: @@ -74,10 +61,22 @@ def create_toolbox(wg: WorkGraph, for child in inseparable.children] for node in nodes} - parents = {work_id2index[node.id]: [] for node in nodes} + parents = {work_id2index[node.id]: set() for node in nodes} for node, node_children in children.items(): for child in node_children: - parents[child].append(node) + parents[child].add(node) + + resources_border = np.zeros((2, len(worker_pool), len(index2node))) + for work_index, node in index2node.items(): + for req in node.work_unit.worker_reqs: + worker_index = worker_name2index[req.kind] + resources_border[0, worker_index, work_index] = req.min_count + resources_border[1, worker_index, work_index] = req.max_count + + contractor_borders = np.zeros((len(contractor2index), len(worker_name2index)), dtype=int) + for ind, contractor in enumerate(contractors): + for ind_worker, worker in enumerate(contractor.workers.values()): + contractor_borders[ind, ind_worker] = worker.count init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]] = \ {name: (convert_schedule_to_chromosome(wg, work_id2index, worker_name2index, @@ -86,12 +85,8 @@ def create_toolbox(wg: WorkGraph, if schedule is not None else None for name, (schedule, order, chromosome_spec, importance) in init_schedules.items()} - for name, chromosome in init_chromosomes.items(): - if chromosome is not None: - if not is_chromosome_correct(chromosome[0], node_indices, parents): - raise NoSufficientContractorError('HEFTs are deploying wrong chromosomes') - - print(f'Genetic optimizing took {(time.time() - start) * 1000} ms') + if verbose: + print(f'Genetic optimizing took {(time.time() - start) * 1000} ms') return init_toolbox(wg, contractors, @@ -113,9 +108,8 @@ def create_toolbox(wg: WorkGraph, node_indices, parents, resources_border, - resources_min_border, - Time(0), - work_estimator) + assigned_parent_time, + work_estimator), worker_name2index, worker_pool_indices, parents def build_schedule(wg: WorkGraph, @@ -129,14 +123,15 @@ def build_schedule(wg: WorkGraph, rand: random.Random, spec: ScheduleSpec, landscape: LandscapeConfiguration = LandscapeConfiguration(), - fitness_constructor: Callable[ - [Callable[[list[ChromosomeType]], list[int]]], FitnessFunction] = TimeFitness, + fitness_object: FitnessFunction = TimeFitness(), work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), n_cpu: int = 1, assigned_parent_time: Time = Time(0), timeline: Timeline | None = None, time_border: int = None, - optimize_resources: bool = False) \ + optimize_resources: bool = False, + deadline: Time = None, + verbose: bool = True) \ -> tuple[ScheduleWorkDict, Time, Timeline, list[GraphNode]]: """ Genetic algorithm. @@ -153,150 +148,233 @@ def build_schedule(wg: WorkGraph, :return: schedule """ - global_start = time.time() + global_start = start = time.time() - # preparing access-optimized data structures - nodes = [node for node in wg.nodes if not node.is_inseparable_son()] + toolbox, *mapping_objects = create_toolbox_and_mapping_objects(wg, contractors, worker_pool, population_size, + mutpb_order, mutpb_res, init_schedules, rand, spec, + work_estimator, assigned_parent_time, landscape, + verbose) - work_id2index: dict[str, int] = {node.id: index for index, node in enumerate(nodes)} - worker_name2index = {worker_name: index for index, worker_name in enumerate(worker_pool)} - contractor2index = {contractor.id: ind for ind, contractor in enumerate(contractors)} - worker_pool_indices = {worker_name2index[worker_name]: { - contractor2index[contractor_id]: worker for contractor_id, worker in workers_of_type.items() - } for worker_name, workers_of_type in worker_pool.items()} - - # construct inseparable_child -> inseparable_parent mapping - inseparable_parents = {} - for node in nodes: - for child in node.get_inseparable_chain_with_self(): - inseparable_parents[child] = node - - # here we aggregate information about relationships from the whole inseparable chain - children = {work_id2index[node.id]: list({work_id2index[inseparable_parents[child].id] - for inseparable in node.get_inseparable_chain_with_self() - for child in inseparable.children}) - for node in nodes} - - parents = {work_id2index[node.id]: [] for node in nodes} - for node, node_children in children.items(): - for child in node_children: - parents[child].append(node) - - start = time.time() - - toolbox = create_toolbox(wg, contractors, worker_pool, population_size, mutpb_order, mutpb_res, init_schedules, - rand, spec, work_estimator, landscape) + worker_name2index, worker_pool_indices, parents = mapping_objects native = NativeWrapper(toolbox, wg, contractors, worker_name2index, worker_pool_indices, parents, work_estimator) # create population of a given size pop = toolbox.population(n=population_size) - print(f'Toolbox initialization & first population took {(time.time() - start) * 1000} ms') + if verbose: + print(f'Toolbox initialization & first population took {(time.time() - start) * 1000} ms') if native.native: native_start = time.time() best_chromosome = native.run_genetic(pop, mutpb_order, mutpb_order, mutpb_res, mutpb_res, mutpb_res, mutpb_res, population_size) - print(f'Native evaluated in {(time.time() - native_start) * 1000} ms') + if verbose: + print(f'Native evaluated in {(time.time() - native_start) * 1000} ms') else: + have_deadline = deadline is not None # save best individuals hof = tools.HallOfFame(1, similar=compare_individuals) - fitness_f = fitness_constructor(native.evaluate) - - start = time.time() + evaluation_start = time.time() # map to each individual fitness function pop = [ind for ind in pop if toolbox.validate(ind)] - fitness = fitness_f.evaluate(pop) + schedules = native.evaluate(pop) + fitness = fitness_object.evaluate(schedules) - evaluation_time = time.time() - start + evaluation_time = time.time() - evaluation_start - for ind, fit in zip(pop, fitness): + for ind, fit, schedule in zip(pop, fitness, schedules): ind.fitness.values = [fit] + if optimize_resources: + toolbox.update_resource_borders_to_peak_values(ind, schedule) + ind.type = IndividualType.Population hof.update(pop) best_fitness = hof[0].fitness.values[0] - generation = 1 - # the best fitness, track to increase performance by stopping evaluation when not decreasing - prev_best_fitness = Time.inf() + if verbose: + print(f'First population evaluation took {evaluation_time * 1000} ms') - print(f'First population evaluation took {(time.time() - start) * 1000} ms') start = time.time() + generation = 1 plateau_steps = 0 - max_plateau_steps = generation_number + new_generation_number = generation_number if not have_deadline else generation_number // 2 + max_plateau_steps = new_generation_number // 2 - while generation <= generation_number and plateau_steps < max_plateau_steps \ + while generation <= new_generation_number and plateau_steps < max_plateau_steps \ and (time_border is None or time.time() - global_start < time_border): - print(f'-- Generation {generation}, population={len(pop)}, best time={best_fitness} --') - if best_fitness == prev_best_fitness: - plateau_steps += 1 - else: - plateau_steps = 0 - prev_best_fitness = best_fitness + if verbose: + print(f'-- Generation {generation}, population={len(pop)}, best fitness={best_fitness} --') rand.shuffle(pop) - cur_generation = [] + offspring = [] for ind1, ind2 in zip(pop[::2], pop[1::2]): - # mate order - cur_generation.extend(toolbox.mate(ind1, ind2)) - - if worker_name2index: - # operations for RESOURCES - for ind1, ind2 in zip(pop[:len(pop) // 2], pop[len(pop) // 2:]): - # mate resources - cur_generation.extend(toolbox.mate_resources(ind1, ind2)) - - if optimize_resources: - # mate resource borders - cur_generation.extend(toolbox.mate_resource_borders(ind1, ind2)) - - for mutant in cur_generation: - # resources mutation - toolbox.mutate_resources(mutant) - if optimize_resources: - # resource borders mutation - toolbox.mutate_resource_borders(mutant) - - for mutant in cur_generation: - # order mutation + # mate + offspring.extend(toolbox.mate(ind1, ind2, optimize_resources)) + + for mutant in offspring: + # mutation + if optimize_resources: + # resource borders mutation + toolbox.mutate_resource_borders(mutant) toolbox.mutate(mutant) evaluation_start = time.time() - # Gather all the fitness in one list and print the stats - offspring = [ind for ind in cur_generation if toolbox.validate(ind)] + schedules = native.evaluate(offspring) + offspring_fitness = fitness_object.evaluate(schedules) - offspring_fitness = fitness_f.evaluate(offspring) - for fit, ind in zip(offspring_fitness, offspring): + for ind, fit, schedule in zip(offspring, offspring_fitness, schedules): ind.fitness.values = [fit] + if optimize_resources: + ind.schedule = schedule + ind.type = IndividualType.Offspring evaluation_time += time.time() - evaluation_start # renewing population pop += offspring pop = toolbox.select(pop) + if optimize_resources: + for ind in pop: + if ind.type is IndividualType.Offspring: + toolbox.update_resource_borders_to_peak_values(ind, ind.schedule) + del ind.schedule + ind.type = IndividualType.Population hof.update([pop[0]]) + prev_best_fitness = best_fitness best_fitness = hof[0].fitness.values[0] + if best_fitness == prev_best_fitness: + plateau_steps += 1 + else: + plateau_steps = 0 + + if have_deadline and best_fitness <= deadline: + if all([ind.fitness.values[0] <= deadline for ind in pop]): + break generation += 1 - native.close() + # Second stage to optimize resources if deadline is assigned - best_chromosome = hof[0] + if have_deadline: + if best_fitness > deadline: + print(f'Deadline not reached !!! Deadline {deadline} < best time {best_fitness}') + # save best individuals + hof = tools.HallOfFame(1, similar=compare_individuals) + pop = [ind for ind in pop if ind.fitness.values[0] == best_fitness] + + evaluation_start = time.time() - # assert that we have valid chromosome - assert hof[0].fitness.values[0] != Time.inf() + fitness = [get_absolute_peak_resource_usage(schedule) for schedule in native.evaluate(pop)] + for ind, fit, schedule in zip(pop, fitness, schedules): + ind.time = ind.fitness.values[0] + ind.fitness.values = [fit] - print(f'Final time: {best_fitness}') - print(f'Generations processing took {(time.time() - start) * 1000} ms') - print(f'Evaluation time: {evaluation_time * 1000}') + evaluation_time += time.time() - evaluation_start + + hof.update(pop) + else: + optimize_resources = True + # save best individuals + hof = tools.HallOfFame(1, similar=compare_individuals) + + pop = [ind for ind in pop if ind.fitness.values[0] <= deadline] + + evaluation_start = time.time() + + schedules = native.evaluate(pop) + fitness = [get_absolute_peak_resource_usage(schedule) for schedule in schedules] + for ind, fit, schedule in zip(pop, fitness, schedules): + ind.time = ind.fitness.values[0] + ind.fitness.values = [fit] + toolbox.update_resource_borders_to_peak_values(ind, schedule) + ind.type = IndividualType.Population + + evaluation_time += time.time() - evaluation_start + + hof.update(pop) + + plateau_steps = 0 + new_generation_number = generation_number - generation + 1 + max_plateau_steps = new_generation_number // 2 + best_fitness = hof[0].fitness.values[0] + + if len(pop) < population_size: + individuals_to_copy = rand.choices(pop, k=population_size - len(pop)) + copied_individuals = [toolbox.copy_individual(ind) for ind in individuals_to_copy] + for copied_ind, ind in zip(copied_individuals, individuals_to_copy): + copied_ind.fitness.values = [ind.fitness.values[0]] + copied_ind.time = ind.time + copied_ind.type = ind.type + pop += copied_individuals + + while generation <= generation_number and plateau_steps < max_plateau_steps \ + and (time_border is None or time.time() - global_start < time_border): + if verbose: + print(f'-- Generation {generation}, population={len(pop)}, best peak={best_fitness} --') + rand.shuffle(pop) + + offspring = [] + + for ind1, ind2 in zip(pop[::2], pop[1::2]): + # mate + offspring.extend(toolbox.mate(ind1, ind2, optimize_resources)) + + for mutant in offspring: + # resource borders mutation + toolbox.mutate_resource_borders(mutant) + # other mutation + toolbox.mutate(mutant) + + evaluation_start = time.time() + + schedules = [schedule for schedule in native.evaluate(offspring)] + + for ind, schedule in zip(offspring, schedules): + ind.time = schedule.execution_time.value + if ind.time <= deadline: + ind.fitness.values = [get_absolute_peak_resource_usage(schedule)] + ind.type = IndividualType.Offspring + ind.schedule = schedule + + offspring = [ind for ind in offspring if ind.time <= deadline] + + evaluation_time += time.time() - evaluation_start + + # renewing population + pop += offspring + pop = toolbox.select(pop) + for ind in pop: + if ind.type is IndividualType.Offspring: + toolbox.update_resource_borders_to_peak_values(ind, ind.schedule) + del ind.schedule + ind.type = IndividualType.Population + hof.update([pop[0]]) + + prev_best_fitness = best_fitness + best_fitness = hof[0].fitness.values[0] + if best_fitness == prev_best_fitness: + plateau_steps += 1 + else: + plateau_steps = 0 + + generation += 1 + + native.close() + + if verbose: + print(f'Final time: {best_fitness}') + print(f'Generations processing took {(time.time() - start) * 1000} ms') + print(f'Evaluation time: {evaluation_time * 1000}') + + best_chromosome = hof[0] scheduled_works, schedule_start_time, timeline, order_nodes = toolbox.chromosome_to_schedule(best_chromosome, landscape=landscape, diff --git a/sampo/scheduler/native_wrapper.py b/sampo/scheduler/native_wrapper.py index 7bbac60b..bbad9567 100644 --- a/sampo/scheduler/native_wrapper.py +++ b/sampo/scheduler/native_wrapper.py @@ -32,7 +32,7 @@ def __init__(self, contractors: list[Contractor], worker_name2index: dict[str, int], worker_pool_indices: dict[int, dict[int, Worker]], - parents: dict[int, list[int]], + parents: dict[int, set[int]], time_estimator: WorkTimeEstimator): self.native = native if not native: @@ -59,7 +59,7 @@ def fit(chromosome: ChromosomeType) -> Schedule | None: self.numeration = numeration # for each vertex index store list of parents' indices self.parents = [[rev_numeration[p] for p in numeration[index].parents] for index in range(wg.vertex_count)] - head_parents = [parents[i] for i in range(len(parents))] + head_parents = [list(parents[i]) for i in range(len(parents))] # for each vertex index store list of whole it's inseparable chain indices self.inseparables = [[rev_numeration[p] for p in numeration[index].get_inseparable_chain_with_self()] for index in range(wg.vertex_count)] diff --git a/sampo/scheduler/resources_in_time/average_binary_search.py b/sampo/scheduler/resources_in_time/average_binary_search.py index 0f30e062..0676457c 100644 --- a/sampo/scheduler/resources_in_time/average_binary_search.py +++ b/sampo/scheduler/resources_in_time/average_binary_search.py @@ -60,7 +60,7 @@ def fitness(k: float, inner_spec: ScheduleSpec): result_min_resources = fitness(k_max, copied_spec) if result_min_resources < deadline: - print('Can keep deadline at minimum resources') + # print('Can keep deadline at minimum resources') # we can keep the deadline if pass minimum resources, # so let's go preventing the works going in parallel for node in wg.nodes: diff --git a/sampo/scheduler/timeline/just_in_time_timeline.py b/sampo/scheduler/timeline/just_in_time_timeline.py index ccfee817..271be199 100644 --- a/sampo/scheduler/timeline/just_in_time_timeline.py +++ b/sampo/scheduler/timeline/just_in_time_timeline.py @@ -116,7 +116,7 @@ def update_timeline(self, worker_timeline = self._timeline[(worker.contractor_id, worker.name)] count_workers = sum([count for _, count in worker_timeline]) worker_timeline.clear() - worker_timeline.append((finish_time, count_workers)) + worker_timeline.append((finish_time + 1, count_workers)) else: # For each worker type consume the nearest available needed worker amount # and re-add it to the time when current work should be finished. diff --git a/sampo/structurator/base.py b/sampo/structurator/base.py index 8be4e59b..783b100b 100644 --- a/sampo/structurator/base.py +++ b/sampo/structurator/base.py @@ -44,7 +44,7 @@ def fill_parents_to_new_nodes(origin_node: GraphNode, id2new_nodes: GraphNodeDic parents_zero_stage: list[tuple[GraphNode, float, EdgeType]] = [] parents_last_stage: list[tuple[GraphNode, float, EdgeType]] = [] for edge in origin_node.edges_to: - indent = 1 if not edge.start.work_unit.is_service_unit and not edge.finish.work_unit.is_service_unit else 0 + indent = 1 if not (edge.start.work_unit.is_service_unit or edge.finish.work_unit.is_service_unit) else 0 if edge.type in [EdgeType.FinishStart, EdgeType.InseparableFinishStart]: lag = edge.lag if not edge.lag % 1 else ceil(edge.lag) lag = lag if lag > 0 else indent diff --git a/tests/scheduler/genetic/converter_test.py b/tests/scheduler/genetic/converter_test.py index ef3ffd6b..e340edd5 100644 --- a/tests/scheduler/genetic/converter_test.py +++ b/tests/scheduler/genetic/converter_test.py @@ -11,7 +11,8 @@ def test_convert_schedule_to_chromosome(setup_toolbox): tb, _, setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox - schedule = HEFTScheduler().schedule(setup_wg, setup_contractors, validate=True, landscape=setup_landscape_many_holders) + schedule = HEFTScheduler().schedule(setup_wg, setup_contractors, validate=True, + landscape=setup_landscape_many_holders) chromosome = tb.schedule_to_chromosome(schedule=schedule) assert tb.validate(chromosome) @@ -44,10 +45,22 @@ def test_converter_with_borders_contractor_accounting(setup_toolbox): for i in range(len(chromosome[2])): contractors.append(Contractor(id=setup_contractors[i].id, name=setup_contractors[i].name, - workers={name: Worker(str(uuid4()), name, count, contractor_id=setup_contractors[i].id) - for name, count in zip(workers, chromosome[2][i])}, + workers={ + name: Worker(str(uuid4()), name, count, contractor_id=setup_contractors[i].id) + for name, count in zip(workers, chromosome[2][i])}, equipments={})) schedule = Schedule.from_scheduled_works(schedule.values(), setup_wg) validate_schedule(schedule, setup_wg, contractors) + + +def test_converter_with_borders_update(setup_toolbox): + tb, _, setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox + chromosome = tb.generate_chromosome(landscape=setup_landscape_many_holders) + schedule, _, _, _ = tb.chromosome_to_schedule(chromosome, landscape=setup_landscape_many_holders) + schedule = Schedule.from_scheduled_works(schedule.values(), setup_wg) + updated_chromosome = tb.update_resource_borders_to_peak_values(chromosome, schedule) + updated_schedule, _, _, _ = tb.chromosome_to_schedule(updated_chromosome, landscape=setup_landscape_many_holders) + updated_schedule = Schedule.from_scheduled_works(updated_schedule.values(), setup_wg) + assert schedule.execution_time == updated_schedule.execution_time diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index 14516e22..92094556 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -5,7 +5,7 @@ import numpy as np -from sampo.scheduler.genetic.schedule_builder import create_toolbox +from sampo.scheduler.genetic.schedule_builder import create_toolbox_and_mapping_objects from sampo.schemas.contractor import get_worker_contractor_pool from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator @@ -48,14 +48,15 @@ def setup_toolbox(setup_default_schedules) -> tuple: resources_border[0, worker_index, work_index] = req.min_count resources_border[1, worker_index, work_index] = req.max_count - return (create_toolbox(setup_wg, - setup_contractors, - setup_worker_pool, - size_of_population, - mutate_order, - mutate_resources, - setup_default_schedules, - rand, - work_estimator=work_estimator, - landscape=setup_landscape_many_holders), resources_border, + return (create_toolbox_and_mapping_objects(setup_wg, + setup_contractors, + setup_worker_pool, + size_of_population, + mutate_order, + mutate_resources, + setup_default_schedules, + rand, + work_estimator=work_estimator, + landscape=setup_landscape_many_holders, + verbose=False)[0], resources_border, setup_wg, setup_contractors, setup_default_schedules, setup_landscape_many_holders) diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 4eddc9d7..96029e4c 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -4,7 +4,7 @@ def test_multiprocessing(setup_scheduler_parameters): setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters - genetic = GeneticScheduler(number_of_generation=50, + genetic = GeneticScheduler(number_of_generation=10, mutate_order=0.05, mutate_resources=0.005, size_of_population=50) diff --git a/tests/scheduler/genetic/operators_test.py b/tests/scheduler/genetic/operators_test.py index d6f582db..9cabd317 100644 --- a/tests/scheduler/genetic/operators_test.py +++ b/tests/scheduler/genetic/operators_test.py @@ -19,11 +19,12 @@ def test_mutate_order(setup_toolbox): for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() - mutant = tb.mutate(individual) + mutant = tb.mutate_order(individual) order = mutant[0] # check there are no duplications assert len(order) == len(set(order)) + assert tb.validate(mutant) def test_mutate_resources(setup_toolbox): @@ -36,6 +37,16 @@ def test_mutate_resources(setup_toolbox): assert tb.validate(mutant) +def test_mutate_resource_borders(setup_toolbox): + tb, _, _, _, _, _ = setup_toolbox + + for i in range(TEST_ITERATIONS): + individual = tb.generate_chromosome() + mutant = tb.mutate_resource_borders(individual) + + assert tb.validate(mutant) + + def test_mate_order(setup_toolbox, setup_wg): tb, _, _, _, _, _ = setup_toolbox _, _, population_size = get_params(setup_wg.vertex_count) @@ -45,13 +56,15 @@ def test_mate_order(setup_toolbox, setup_wg): for i in range(TEST_ITERATIONS): individual1, individual2 = population[:2] - individual1, individual2 = tb.mate(individual1, individual2) - order1 = individual1[0] - order2 = individual2[0] + child1, child2 = tb.mate_order(individual1, individual2) + order1 = child1[0] + order2 = child2[0] # check there are no duplications assert len(order1) == len(set(order1)) assert len(order2) == len(set(order2)) + assert tb.validate(child1) + assert tb.validate(child2) def test_mate_resources(setup_toolbox, setup_wg): @@ -62,7 +75,7 @@ def test_mate_resources(setup_toolbox, setup_wg): for i in range(TEST_ITERATIONS): individual1, individual2 = random.sample(population, 2) - individual1, individual2 = tb.mate_resources(individual1, individual2) + individual1, individual2 = tb.mate_resources(individual1, individual2, optimize_resources=False) # check there are correct resources at mate positions assert (resources_border[0] <= individual1[1].T[:-1]).all() and \ diff --git a/tests/scheduler/resources_in_time/basic_res_test.py b/tests/scheduler/resources_in_time/basic_res_test.py index dda836a3..687e2dd2 100644 --- a/tests/scheduler/resources_in_time/basic_res_test.py +++ b/tests/scheduler/resources_in_time/basic_res_test.py @@ -1,11 +1,11 @@ import pytest +import math from sampo.scheduler.genetic.base import GeneticScheduler from sampo.scheduler.genetic.operators import DeadlineResourcesFitness from sampo.scheduler.heft.base import HEFTScheduler from sampo.scheduler.resources_in_time.average_binary_search import AverageBinarySearchResourceOptimizingScheduler from sampo.scheduler.utils.peaks import get_absolute_peak_resource_usage -from sampo.schemas.exceptions import NoSufficientContractorError from sampo.schemas.time import Time from sampo.utilities.resource_cost import schedule_cost @@ -35,19 +35,20 @@ def test_genetic_deadline_planning(setup_scheduler_parameters): setup_wg, setup_contractors, landscape = setup_scheduler_parameters deadline = Time.inf() // 2 - scheduler = GeneticScheduler(number_of_generation=50, + scheduler = GeneticScheduler(number_of_generation=10, mutate_order=0.05, mutate_resources=0.005, size_of_population=50, - fitness_constructor=DeadlineResourcesFitness.prepare(deadline)) + fitness_constructor=DeadlineResourcesFitness, + optimize_resources=True, + verbose=False) + scheduler.set_deadline(deadline) - try: - schedule = scheduler.schedule(setup_wg, setup_contractors, landscape=landscape) + schedule = scheduler.schedule(setup_wg, setup_contractors, landscape=landscape) - print(f'Planning for deadline time: {schedule.execution_time}, cost: {schedule_cost(schedule)}') - except NoSufficientContractorError: - pytest.skip("Given contractors can't satisfy given work graph") + print(f'Planning for deadline time: {schedule.execution_time}, ' + + f'peaks: {get_absolute_peak_resource_usage(schedule)}, cost: {schedule_cost(schedule)}') def test_true_deadline_planning(setup_scheduler_parameters): @@ -72,3 +73,48 @@ def test_true_deadline_planning(setup_scheduler_parameters): print(f'Peak with deadline: {peak_deadlined}, time: {deadlined_schedule.execution_time}') print(f'Peak without deadline: {peak_not_deadlined}, time: {not_deadlined_schedule.execution_time}') + + +def test_lexicographic_genetic_deadline_planning(setup_scheduler_parameters): + setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + + scheduler = HEFTScheduler() + schedule, _, _, _ = scheduler.schedule_with_cache(setup_wg, setup_contractors, landscape=setup_landscape) + + # assigning deadline to the time-10^(order_of_magnitude(time) - 1) + # time - time of schedule from HEFT + deadline = schedule.execution_time + deadline -= 10 ** max(0, int(math.log10(deadline.value) - 1)) + + print(f'Deadline time: {deadline}') + + scheduler_combined = GeneticScheduler(number_of_generation=10, + mutate_order=0.05, + mutate_resources=0.005, + size_of_population=50, + fitness_constructor=DeadlineResourcesFitness, + optimize_resources=True, + verbose=False) + + scheduler_combined.set_deadline(deadline) + + scheduler_lexicographic = GeneticScheduler(number_of_generation=10, + mutate_order=0.05, + mutate_resources=0.005, + size_of_population=50, + verbose=False) + + scheduler_lexicographic.set_deadline(deadline) + + schedule = scheduler_combined.schedule(setup_wg, setup_contractors, landscape=setup_landscape) + time_combined = schedule.execution_time + + print(f'\tCombined genetic: time = {time_combined}, ' + + f'peak = {get_absolute_peak_resource_usage(schedule)}') + + schedule = scheduler_lexicographic.schedule(setup_wg, setup_contractors, landscape=setup_landscape) + time_lexicographic = schedule.execution_time + + print(f'\tLexicographic genetic: time = {time_lexicographic}, ' + + f'peak = {get_absolute_peak_resource_usage(schedule)}') + print() diff --git a/tests/structurator_test.py b/tests/structurator/structurator_test.py similarity index 100% rename from tests/structurator_test.py rename to tests/structurator/structurator_test.py