diff --git a/examples/contractor_spec.py b/examples/contractor_spec.py new file mode 100644 index 00000000..cfe115e8 --- /dev/null +++ b/examples/contractor_spec.py @@ -0,0 +1,39 @@ +from random import Random + +from sampo.scheduler import GeneticScheduler, TopologicalScheduler, RandomizedTopologicalScheduler, LFTScheduler, \ + RandomizedLFTScheduler +from sampo.pipeline.lag_optimization import LagOptimizationStrategy +from sampo.generator.base import SimpleSynthetic +from sampo.generator.environment.contractor_by_wg import get_contractor_by_wg, ContractorGenerationMethod +from sampo.generator import SyntheticGraphType +from sampo.pipeline import SchedulingPipeline +from sampo.scheduler.heft.base import HEFTScheduler +from sampo.schemas.schedule_spec import ScheduleSpec + +ss = SimpleSynthetic(rand=231) +rand = Random() +size = 100 + +wg = ss.work_graph(bottom_border=size - 5, top_border=size) + +contractors = [get_contractor_by_wg(wg) for _ in range(10)] +spec = ScheduleSpec() + +for node in wg.nodes: + if not node.is_inseparable_son(): + selected_contractor_indices = rand.choices(list(range(len(contractors))), + k=rand.randint(1, len(contractors))) + spec.assign_contractors(node.id, {contractors[i].id for i in selected_contractor_indices}) + + +scheduler = GeneticScheduler(number_of_generation=10) + +project = SchedulingPipeline.create() \ + .wg(wg) \ + .contractors(contractors) \ + .lag_optimize(LagOptimizationStrategy.TRUE) \ + .spec(spec) \ + .schedule(scheduler, validate=True) \ + .visualization('2022-01-01')[0] \ + .shape((14, 14)) \ + .show_gant_chart() diff --git a/sampo/backend/default.py b/sampo/backend/default.py index 96203e77..d2b5eb76 100644 --- a/sampo/backend/default.py +++ b/sampo/backend/default.py @@ -62,8 +62,8 @@ def _ensure_toolbox_created(self): from sampo.scheduler.genetic.utils import init_chromosomes_f, create_toolbox_using_cached_chromosomes if self._init_schedules: - init_chromosomes = init_chromosomes_f(self._wg, self._contractors, self._init_schedules, - self._landscape) + init_chromosomes = init_chromosomes_f(self._wg, self._contractors, self._spec, + self._init_schedules, self._landscape) else: init_chromosomes = [] diff --git a/sampo/backend/multiproc.py b/sampo/backend/multiproc.py index 944e0628..691ecd1e 100644 --- a/sampo/backend/multiproc.py +++ b/sampo/backend/multiproc.py @@ -140,8 +140,8 @@ def cache_genetic_info(self, weights, init_schedules, assigned_parent_time, fitness_weights, sgs_type, only_lft_initialization, is_multiobjective) if init_schedules: - self._init_chromosomes = init_chromosomes_f(self._wg, self._contractors, init_schedules, - self._landscape) + self._init_chromosomes = init_chromosomes_f(self._wg, self._contractors, self._spec, + init_schedules, self._landscape) else: self._init_chromosomes = [] self._pool = None diff --git a/sampo/generator/base.py b/sampo/generator/base.py index 9fa7c00c..c34f3ce4 100644 --- a/sampo/generator/base.py +++ b/sampo/generator/base.py @@ -20,6 +20,9 @@ def __init__(self, rand: int | Random | None = None) -> None: else: self._rand = Random(rand) + def get_rand(self): + return self._rand + def small_work_graph(self, cluster_name: str | None = 'C1') -> WorkGraph: """ Creates a small graph of works consisting of 30-50 vertices; diff --git a/sampo/scheduler/generic.py b/sampo/scheduler/generic.py index fec055c0..211ca5b4 100644 --- a/sampo/scheduler/generic.py +++ b/sampo/scheduler/generic.py @@ -99,7 +99,7 @@ def run_with_contractor(contractor: Contractor) -> tuple[Time, Time, list[Worker assigned_parent_time, work_estimator) return c_st, c_ft, workers - return run_contractor_search(contractors, run_with_contractor) + return run_contractor_search(contractors, spec, run_with_contractor) return optimize_resources_def @@ -126,7 +126,7 @@ def schedule_with_cache(self, ) if validate: - validate_schedule(schedule, wg, contractors) + validate_schedule(schedule, wg, contractors, spec) return [(schedule, schedule_start_time, timeline, ordered_nodes)] diff --git a/sampo/scheduler/genetic/base.py b/sampo/scheduler/genetic/base.py index b2316ffa..4712f733 100644 --- a/sampo/scheduler/genetic/base.py +++ b/sampo/scheduler/genetic/base.py @@ -171,14 +171,15 @@ def generate_first_population(wg: WorkGraph, schedule, _, _, node_order = LFTScheduler(work_estimator=work_estimator).schedule_with_cache(wg, contractors, spec, - landscape=landscape)[0] + landscape=landscape, + validate=True)[0] init_lft_schedule = (schedule, node_order, spec) def init_k_schedule(scheduler_class, k) -> tuple[Schedule | None, list[GraphNode] | None, ScheduleSpec | None]: try: schedule, _, _, node_order = (scheduler_class(work_estimator=work_estimator, resource_optimizer=AverageReqResourceOptimizer(k)) - .schedule_with_cache(wg, contractors, spec, landscape=landscape))[0] + .schedule_with_cache(wg, contractors, spec, landscape=landscape, validate=True))[0] return schedule, node_order, spec except NoSufficientContractorError: return None, None, None @@ -187,7 +188,7 @@ def init_k_schedule(scheduler_class, k) -> tuple[Schedule | None, list[GraphNode def init_schedule(scheduler_class) -> tuple[Schedule | None, list[GraphNode] | None, ScheduleSpec | None]: try: schedule, _, _, node_order = (scheduler_class(work_estimator=work_estimator) - .schedule_with_cache(wg, contractors, spec, landscape=landscape))[0] + .schedule_with_cache(wg, contractors, spec, landscape=landscape, validate=True))[0] return schedule, node_order, spec except NoSufficientContractorError: return None, None, None @@ -197,7 +198,7 @@ def init_schedule(scheduler_class) -> tuple[Schedule | None, list[GraphNode] | N try: (schedule, _, _, node_order), modified_spec = AverageBinarySearchResourceOptimizingScheduler( scheduler_class(work_estimator=work_estimator) - ).schedule_with_cache(wg, contractors, deadline, spec, landscape=landscape) + ).schedule_with_cache(wg, contractors, deadline, spec, landscape=landscape, validate=True) return schedule, node_order, modified_spec except NoSufficientContractorError: return None, None, None @@ -309,6 +310,6 @@ def schedule_with_cache(self, if validate: for schedule, *_ in schedules: - validate_schedule(schedule, wg, contractors) + validate_schedule(schedule, wg, contractors, spec) return schedules diff --git a/sampo/scheduler/genetic/converter.py b/sampo/scheduler/genetic/converter.py index 04cefb5c..4cea5297 100644 --- a/sampo/scheduler/genetic/converter.py +++ b/sampo/scheduler/genetic/converter.py @@ -19,6 +19,7 @@ from sampo.schemas.schedule_spec import ScheduleSpec from sampo.schemas.time import Time from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator +from sampo.utilities.collections_util import reverse_dictionary from sampo.utilities.linked_list import LinkedList @@ -63,16 +64,25 @@ def convert_schedule_to_chromosome(work_id2index: dict[str, int], for node in order: node_id = node.id index = work_id2index[node_id] - for resource in schedule[node_id].workers: - res_count = resource.count - res_index = worker_name2index[resource.name] - res_contractor = resource.contractor_id - resource_chromosome[index, res_index] = res_count - resource_chromosome[index, -1] = contractor2index[res_contractor] + + if schedule[node_id].workers: + for resource in schedule[node_id].workers: + res_count = resource.count + res_index = worker_name2index[resource.name] + res_contractor = resource.contractor_id + + resource_chromosome[index, res_index] = res_count + resource_chromosome[index, -1] = contractor2index[res_contractor] + else: + contractor_list = spec[node_id].contractors + if not contractor_list: + contractor_list = contractor2index.keys() + random_contractor = list(contractor_list)[0] + resource_chromosome[index, -1] = contractor2index[random_contractor] resource_border_chromosome = np.copy(contractor_borders) - return order_chromosome, resource_chromosome, resource_border_chromosome, spec, zone_changes_chromosome + return order_chromosome, resource_chromosome, resource_border_chromosome, copy.deepcopy(spec), zone_changes_chromosome def convert_chromosome_to_schedule(chromosome: ChromosomeType, diff --git a/sampo/scheduler/genetic/operators.py b/sampo/scheduler/genetic/operators.py index 8e874e80..8fd3114c 100644 --- a/sampo/scheduler/genetic/operators.py +++ b/sampo/scheduler/genetic/operators.py @@ -20,7 +20,7 @@ from sampo.schemas.landscape import LandscapeConfiguration from sampo.schemas.resources import Worker from sampo.schemas.schedule import Schedule -from sampo.schemas.schedule_spec import ScheduleSpec +from sampo.schemas.schedule_spec import ScheduleSpec, WorkSpec from sampo.schemas.time import Time from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator from sampo.utilities.resource_usage import resources_peaks_sum, resources_costs_sum, resources_sum @@ -165,6 +165,7 @@ def init_toolbox(wg: WorkGraph, parents: dict[int, set[int]], children: dict[int, set[int]], resources_border: np.ndarray, + contractors_available: np.ndarray, assigned_parent_time: Time = Time(0), fitness_weights: tuple[int | float, ...] = (-1,), work_estimator: WorkTimeEstimator = DefaultWorkEstimator(), @@ -200,7 +201,8 @@ def init_toolbox(wg: WorkGraph, # combined mutation toolbox.register('mutate', mutate, order_mutpb=mut_order_pb, res_mutpb=mut_res_pb, zone_mutpb=mut_zone_pb, rand=rand, parents=parents, children=children, resources_border=resources_border, - statuses_available=statuses_available, priorities=priorities) + contractors_available=contractors_available, statuses_available=statuses_available, + priorities=priorities) # crossover for order toolbox.register('mate_order', mate_scheduling_order, rand=rand, toolbox=toolbox, priorities=priorities) # mutation for order @@ -210,7 +212,7 @@ def init_toolbox(wg: WorkGraph, toolbox.register('mate_resources', mate_resources, rand=rand, toolbox=toolbox) # mutation for resources toolbox.register('mutate_resources', mutate_resources, resources_border=resources_border, - mutpb=mut_res_pb, rand=rand) + contractors_available=contractors_available, mutpb=mut_res_pb, rand=rand) # mutation for resource borders toolbox.register('mutate_resource_borders', mutate_resource_borders, contractor_borders=contractor_borders, mutpb=mut_res_pb, rand=rand) @@ -219,7 +221,7 @@ def init_toolbox(wg: WorkGraph, statuses_available=landscape.zone_config.statuses.statuses_available()) toolbox.register('validate', is_chromosome_correct, node_indices=node_indices, parents=parents, - contractor_borders=contractor_borders, index2node=index2node) + contractor_borders=contractor_borders, index2node=index2node, index2contractor=index2contractor_obj) toolbox.register('schedule_to_chromosome', convert_schedule_to_chromosome, work_id2index=work_id2index, worker_name2index=worker_name2index, contractor2index=contractor2index, contractor_borders=contractor_borders, spec=spec, @@ -324,9 +326,6 @@ def randomized_init(is_topological: bool = False) -> ChromosomeType: case _: ind = init_chromosomes[generated_type][0] - if not toolbox.validate(ind): - SAMPO.logger.warn('HELP') - ind = toolbox.Individual(ind) chromosomes.append(ind) @@ -390,12 +389,13 @@ def select_new_population(population: list[Individual], k: int) -> list[Individu def is_chromosome_correct(ind: Individual, node_indices: list[int], parents: dict[int, set[int]], - contractor_borders: np.ndarray, index2node: dict[int, GraphNode]) -> bool: + contractor_borders: np.ndarray, index2node: dict[int, GraphNode], + index2contractor: dict[int, Contractor]) -> bool: """ Check correctness of works order and contractors borders. """ return is_chromosome_order_correct(ind, parents, index2node) and \ - is_chromosome_contractors_correct(ind, node_indices, contractor_borders) + is_chromosome_contractors_correct(ind, node_indices, contractor_borders, index2node, index2contractor) def is_chromosome_order_correct(ind: Individual, parents: dict[int, set[int]], index2node: dict[int, GraphNode]) -> bool: @@ -419,13 +419,27 @@ def is_chromosome_order_correct(ind: Individual, parents: dict[int, set[int]], i def is_chromosome_contractors_correct(ind: Individual, work_indices: Iterable[int], - contractor_borders: np.ndarray) -> bool: + contractor_borders: np.ndarray, + index2node: dict[int, GraphNode], + index2contractor: dict[int, Contractor]) -> bool: """ Checks that assigned contractors can supply assigned workers. """ if not work_indices: return True + + order = ind[0] resources = ind[1][work_indices] + + # check contractor align with the spec + spec: ScheduleSpec = ind[3] + contractors = resources[:, -1] + for i in range(len(order)): + work_index = order[i] + work_spec = spec[index2node[work_index].id] + if not work_spec.is_contractor_enabled(index2contractor[contractors[work_index]].id): + return False + # sort resource part of chromosome by contractor ids resources = resources[resources[:, -1].argsort()] # get unique contractors and indexes where they start @@ -579,8 +593,8 @@ def mutate_scheduling_order(ind: Individual, mutpb: float, rand: random.Random, """ order = ind[0] - priority_groups_count = len(set(priorities)) - mutpb_for_priority_group = mutpb #/ priority_groups_count + # priority_groups_count = len(set(priorities)) + mutpb_for_priority_group = mutpb # / priority_groups_count # priorities of tasks with same order-index should be the same (if chromosome is valid) cur_priority = priorities[order[0]] @@ -634,15 +648,17 @@ def mate_resources(ind1: Individual, ind2: Individual, rand: random.Random, def mutate_resources(ind: Individual, mutpb: float, rand: random.Random, - resources_border: np.ndarray) -> Individual: + resources_border: np.ndarray, + contractors_available: np.ndarray) -> Individual: """ Mutation function for resources. It changes selected numbers of workers in random work in a certain interval for this work. :param ind: the individual to be mutated - :param resources_border: low and up borders of resources amounts :param mutpb: probability of gene mutation :param rand: the rand object used for randomized operations + :param resources_border: low and up borders of resources amounts + :param contractors_available: mask of contractors available to do tasks :return: mutated individual """ @@ -654,9 +670,22 @@ def mutate_resources(ind: Individual, mutpb: float, rand: random.Random, mask = np.array([rand.random() < mutpb for _ in range(num_works)]) if mask.any(): # generate new contractors in the number of received True values of mask - new_contractors = np.array([rand.randint(0, num_contractors - 1) for _ in range(mask.sum())]) + + # [rand.randint(0, num_contractors - 1) for _ in range(mask.sum())] + new_contractors_list = [] + + # TODO Rewrite to numpy functions if heavy + for task, task_selected in enumerate(mask): + if not task_selected: + continue + contractors_to_select = np.where(contractors_available[task] == 1) + new_contractors_list.append(rand.choices(contractors_to_select[0], k=1)[0]) + + new_contractors = np.array(new_contractors_list) + # obtain a new mask of correspondence # between the borders of the received contractors and the assigned resources + contractor_mask = (res[mask, :-1] <= ind[2][new_contractors]).all(axis=1) # update contractors by received mask new_contractors = new_contractors[contractor_mask] @@ -720,9 +749,9 @@ def mate(ind1: Individual, ind2: Individual, optimize_resources: bool, return toolbox.Individual(child1), toolbox.Individual(child2) -def mutate(ind: Individual, resources_border: np.ndarray, parents: dict[int, set[int]], - children: dict[int, set[int]], statuses_available: int, priorities: np.ndarray, - order_mutpb: float, res_mutpb: float, zone_mutpb: float, +def mutate(ind: Individual, resources_border: np.ndarray, contractors_available: np.ndarray, + parents: dict[int, set[int]], children: dict[int, set[int]], statuses_available: int, + priorities: np.ndarray, order_mutpb: float, res_mutpb: float, zone_mutpb: float, rand: random.Random) -> Individual: """ Combined mutation function of mutation for order, mutation for resources and mutation for zones. @@ -740,7 +769,7 @@ def mutate(ind: Individual, resources_border: np.ndarray, parents: dict[int, set :return: mutated individual """ mutant = mutate_scheduling_order(ind, order_mutpb, rand, priorities, parents, children) - mutant = mutate_resources(mutant, res_mutpb, rand, resources_border) + mutant = mutate_resources(mutant, res_mutpb, rand, resources_border, contractors_available) # TODO Make better mutation for zones and uncomment this # mutant = mutate_for_zones(mutant, statuses_available, zone_mutpb, rand) diff --git a/sampo/scheduler/genetic/schedule_builder.py b/sampo/scheduler/genetic/schedule_builder.py index 971cb582..e1eb839c 100644 --- a/sampo/scheduler/genetic/schedule_builder.py +++ b/sampo/scheduler/genetic/schedule_builder.py @@ -41,7 +41,7 @@ def create_toolbox(wg: WorkGraph, worker_pool, index2node, index2zone, work_id2index, worker_name2index, index2contractor_obj, \ worker_pool_indices, contractor2index, contractor_borders, node_indices, priorities, parents, children, \ - resources_border = prepare_optimized_data_structures(wg, contractors, landscape) + resources_border, contractors_available = prepare_optimized_data_structures(wg, contractors, landscape, spec) init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]] = \ {name: (convert_schedule_to_chromosome(work_id2index, worker_name2index, @@ -79,6 +79,7 @@ def create_toolbox(wg: WorkGraph, parents, children, resources_border, + contractors_available, assigned_parent_time, fitness_weights, work_estimator, diff --git a/sampo/scheduler/genetic/utils.py b/sampo/scheduler/genetic/utils.py index c1d67afe..abd6406b 100644 --- a/sampo/scheduler/genetic/utils.py +++ b/sampo/scheduler/genetic/utils.py @@ -1,4 +1,5 @@ import random +from operator import attrgetter import numpy as np from deap.base import Toolbox @@ -13,11 +14,12 @@ def init_chromosomes_f(wg: WorkGraph, contractors: list[Contractor], + spec: ScheduleSpec, init_schedules: dict[str, tuple[Schedule, list[GraphNode] | None, ScheduleSpec, float]], landscape: LandscapeConfiguration = LandscapeConfiguration()): worker_pool, index2node, index2zone, work_id2index, worker_name2index, index2contractor_obj, \ worker_pool_indices, contractor2index, contractor_borders, node_indices, priorities, parents, children, \ - resources_border = prepare_optimized_data_structures(wg, contractors, landscape) + resources_border, contractors_available = prepare_optimized_data_structures(wg, contractors, landscape, spec) init_chromosomes: dict[str, tuple[ChromosomeType, float, ScheduleSpec]] = \ {name: (convert_schedule_to_chromosome(work_id2index, worker_name2index, @@ -32,7 +34,8 @@ def init_chromosomes_f(wg: WorkGraph, def prepare_optimized_data_structures(wg: WorkGraph, contractors: list[Contractor], - landscape: LandscapeConfiguration): + landscape: LandscapeConfiguration, + spec: ScheduleSpec): # preparing access-optimized data structures index2zone = {ind: zone for ind, zone in enumerate(landscape.zone_config.start_statuses)} @@ -63,15 +66,20 @@ def prepare_optimized_data_structures(wg: WorkGraph, priorities = np.array([index2node[i].work_unit.priority for i in range(len(index2node))]) resources_border = np.zeros((2, len(worker_pool), len(index2node))) + contractors_available = np.zeros((len(index2node), len(contractor2index))) for work_index, node in index2node.items(): for req in node.work_unit.worker_reqs: worker_index = worker_name2index[req.kind] resources_border[0, worker_index, work_index] = req.min_count resources_border[1, worker_index, work_index] = req.max_count + contractors_spec = spec.get_work_spec(node.id).contractors or map(attrgetter('id'), contractors) + for contractor in contractors_spec: + contractors_available[work_index, contractor2index[contractor]] = 1 + return (worker_pool, index2node, index2zone, work_id2index, worker_name2index, index2contractor_obj, worker_pool_indices, contractor2index, contractor_borders, node_indices, priorities, parents, - children, resources_border) + children, resources_border, contractors_available) def create_toolbox_using_cached_chromosomes(wg: WorkGraph, @@ -92,7 +100,7 @@ def create_toolbox_using_cached_chromosomes(wg: WorkGraph, is_multiobjective: bool = False) -> Toolbox: worker_pool, index2node, index2zone, work_id2index, worker_name2index, index2contractor_obj, \ worker_pool_indices, contractor2index, contractor_borders, node_indices, priorities, parents, children, \ - resources_border = prepare_optimized_data_structures(wg, contractors, landscape) + resources_border, contractors_available = prepare_optimized_data_structures(wg, contractors, landscape, spec) return init_toolbox(wg, contractors, @@ -119,6 +127,7 @@ def create_toolbox_using_cached_chromosomes(wg: WorkGraph, parents, children, resources_border, + contractors_available, assigned_parent_time, fitness_weights, work_estimator, diff --git a/sampo/scheduler/lft/base.py b/sampo/scheduler/lft/base.py index 0cd4e55b..3732d9e2 100644 --- a/sampo/scheduler/lft/base.py +++ b/sampo/scheduler/lft/base.py @@ -22,7 +22,7 @@ def get_contractors_and_workers_amounts_for_work(work_unit: WorkUnit, contractors: list[Contractor], spec: ScheduleSpec, worker_pool: WorkerContractorPool) \ - -> tuple[list[Contractor], np.ndarray]: + -> tuple[list[Contractor], np.ndarray, np.ndarray]: """ This function selects contractors that can perform the work. For each selected contractor, the maximum possible amount of workers is assigned, @@ -57,6 +57,8 @@ def get_contractors_and_workers_amounts_for_work(work_unit: WorkUnit, contractor contractors_mask = (contractors_amounts >= min_req_amounts).all(axis=1) # update bool mask of contractors to satisfy amounts of workers assigned in schedule spec contractors_mask &= (contractors_amounts[:, in_spec_mask] >= work_spec_amounts[in_spec_mask]).all(axis=1) + # handle spec + contractors_mask &= np.array([work_spec.is_contractor_enabled(contractor.id) for contractor in contractors]) # check that there is at least one contractor that satisfies all the constraints if not contractors_mask.any(): raise NoSufficientContractorError(f'There is no contractor that can satisfy given search; contractors: ' @@ -81,7 +83,7 @@ def get_contractors_and_workers_amounts_for_work(work_unit: WorkUnit, contractor # assign to all accepted contractors assigned in schedule spec amounts of workers workers_amounts[:, in_spec_mask] = work_spec_amounts[in_spec_mask] - return accepted_contractors, workers_amounts + return accepted_contractors, workers_amounts, contractors_mask class LFTScheduler(Scheduler): @@ -127,6 +129,7 @@ def schedule_with_cache(self, schedule, schedule_start_time, timeline = self.build_scheduler(ordered_nodes, contractors, landscape, spec, self.work_estimator, assigned_parent_time, timeline) + del self._node_id2workers schedule = Schedule.from_scheduled_works( schedule, @@ -134,7 +137,7 @@ def schedule_with_cache(self, ) if validate: - validate_schedule(schedule, wg, contractors) + validate_schedule(schedule, wg, contractors, spec) return [(schedule, schedule_start_time, timeline, ordered_nodes)] @@ -156,10 +159,11 @@ def build_scheduler(self, for index, node in enumerate(ordered_nodes): work_unit = node.work_unit - work_spec = spec.get_work_spec(work_unit.id) + work_spec = spec[node.id] # get assigned contractor and workers contractor, best_worker_team = self._node_id2workers[node.id] + # find start time start_time, finish_time, _ = timeline.find_min_start_time_with_additional(node, best_worker_team, node2swork, work_spec, None, @@ -182,6 +186,12 @@ def build_scheduler(self, if index == len(ordered_nodes) - 1: # we are scheduling the work `end of the project` node2swork[node].zones_pre = finalizing_zones + # for swork in node2swork.values(): + # work_spec = spec[swork.id] + # assert work_spec.is_contractor_enabled(swork.contractor_id) + # for worker in swork.workers: + # assert work_spec.is_contractor_enabled(worker.contractor_id) + return node2swork.values(), assigned_parent_time, timeline def _contractor_workers_assignment(self, head_nodes: list[GraphNode], contractors: list[Contractor], @@ -196,10 +206,8 @@ def _contractor_workers_assignment(self, head_nodes: list[GraphNode], contractor for node in head_nodes: work_unit = node.work_unit # get contractors that can perform this work and workers amounts for them - accepted_contractors, workers_amounts = get_contractors_and_workers_amounts_for_work(work_unit, - contractors, - spec, - worker_pool) + accepted_contractors, workers_amounts, accepted_contractors_mask = \ + get_contractors_and_workers_amounts_for_work(work_unit, contractors, spec, worker_pool) # estimate chain durations for each accepted contractor durations = np.array([get_chain_duration(node, amounts, self.work_estimator) @@ -208,7 +216,7 @@ def _contractor_workers_assignment(self, head_nodes: list[GraphNode], contractor # assign a score for each contractor equal to the sum of the ratios of # the duration of this work for this contractor to all durations # and the number of assignments of this contractor to the total amount of contractors assignments - scores = durations / durations.sum() + contractors_assignments_count / contractors_assignments_count.sum() + scores = durations / durations.sum() + contractors_assignments_count[accepted_contractors_mask] / contractors_assignments_count.sum() # since the maximum possible score value is 2 subtract the resulting scores from 2, # so that the higher the score, the more suitable the contractor is for the assignment scores = 2 - scores @@ -223,6 +231,9 @@ def _contractor_workers_assignment(self, head_nodes: list[GraphNode], contractor # increase the counter for the assigned contractor contractors_assignments_count[contractor_index] += 1 + if not spec[work_unit.id].is_contractor_enabled(assigned_contractor.id): + raise + # get workers of the assigned contractor and assign them to the node in mapper workers = [worker_pool[req.kind][assigned_contractor.id].copy().with_count(amount) for req, amount in zip(work_unit.worker_reqs, assigned_amount)] diff --git a/sampo/scheduler/utils/multi_contractor.py b/sampo/scheduler/utils/multi_contractor.py index eec407f4..3ca2e91d 100644 --- a/sampo/scheduler/utils/multi_contractor.py +++ b/sampo/scheduler/utils/multi_contractor.py @@ -7,6 +7,7 @@ from sampo.schemas.exceptions import NoSufficientContractorError from sampo.schemas.requirements import WorkerReq from sampo.schemas.resources import Worker +from sampo.schemas.schedule_spec import WorkSpec from sampo.schemas.time import Time @@ -42,6 +43,7 @@ def get_worker_borders(agents: WorkerContractorPool, contractor: Contractor, wor def run_contractor_search(contractors: list[Contractor], + work_spec: WorkSpec, runner: Callable[[Contractor], tuple[Time, Time, list[Worker]]]) \ -> tuple[Time, Time, Contractor, list[Worker]]: """ @@ -60,6 +62,8 @@ def run_contractor_search(contractors: list[Contractor], # heuristic: if contractors' finish times are equal, we prefer smaller one best_contractor_size = float('inf') + contractors = work_spec.filter_contractors(contractors) + for contractor in contractors: start_time, finish_time, worker_team = runner(contractor) contractor_size = sum(w.count for w in contractor.workers.values()) diff --git a/sampo/schemas/schedule_spec.py b/sampo/schemas/schedule_spec.py index 8888edc4..a69173c2 100644 --- a/sampo/schemas/schedule_spec.py +++ b/sampo/schemas/schedule_spec.py @@ -2,6 +2,7 @@ from copy import copy from dataclasses import dataclass, field +from sampo.schemas import Contractor from sampo.schemas.resources import Worker from sampo.schemas.time import Time from sampo.schemas.types import WorkerName @@ -13,25 +14,30 @@ class WorkSpec: """ Here are the container for externally given terms, that the resulting `ScheduledWork` should satisfy. Must be used in schedulers. - :param chain: the chain of works, that should be scheduled one after another, e.g. inseparable, - that starts from this work. Now unsupported. + :param contractors: list of contractors' ids that can be assigned to this work. + Should match the global contractors list :param assigned_workers: predefined worker team (scheduler should assign this worker team to this work) :param assigned_time: predefined work time (scheduler should schedule this work with this execution time) :param is_independent: should this work be resource-independent, e.g. executing with no parallel users of its types of resources """ - chain: list[WorkUnit] | None = None # TODO Add support + contractors: set[str] = field(default_factory=set) assigned_workers: dict[WorkerName, int] = field(default_factory=dict) assigned_time: Time | None = None is_independent: bool = False + def is_contractor_enabled(self, contractor_id: str) -> bool: + return len(self.contractors) == 0 or contractor_id in self.contractors + + def filter_contractors(self, contractors: list[Contractor]) -> list[Contractor]: + return [contractor for contractor in contractors if self.is_contractor_enabled(contractor.id)] + + @dataclass class ScheduleSpec: """ Here is the container for externally given terms, that Schedule should satisfy. Must be used in schedulers. - - :param work2spec: work specs """ _work2spec: dict[str, WorkSpec] = field(default_factory=lambda: defaultdict(WorkSpec)) @@ -42,17 +48,26 @@ def set_exec_time(self, work: str | WorkUnit, time: Time) -> 'ScheduleSpec': self._work2spec[work].assigned_time = time return self - def assign_workers_dict(self, work: str, workers: dict[WorkerName, int]) -> 'ScheduleSpec': + def assign_workers_dict(self, work: str | WorkUnit, workers: dict[WorkerName, int]) -> 'ScheduleSpec': if isinstance(work, WorkUnit): work = work.id self._work2spec[work].assigned_workers = copy(workers) return self - def assign_workers(self, work: str, workers: list[Worker]) -> 'ScheduleSpec': + def assign_workers(self, work: str | WorkUnit, workers: list[Worker]) -> 'ScheduleSpec': if isinstance(work, WorkUnit): work = work.id self._work2spec[work].assigned_workers = {worker.name: worker.count for worker in workers} return self + def assign_contractors(self, work: str | WorkUnit, contractors: set[str]) -> 'ScheduleSpec': + if isinstance(work, WorkUnit): + work = work.id + self._work2spec[work].contractors = contractors + return self + def get_work_spec(self, work_id: str) -> WorkSpec: return self._work2spec[work_id] + + def __getitem__(self, item): + return self.get_work_spec(item) diff --git a/sampo/schemas/scheduled_work.py b/sampo/schemas/scheduled_work.py index 634a6fdd..5b53db21 100644 --- a/sampo/schemas/scheduled_work.py +++ b/sampo/schemas/scheduled_work.py @@ -31,7 +31,7 @@ def __init__(self, work_unit: WorkUnit, start_end_time: tuple[Time, Time], workers: list[Worker], - contractor: Contractor | str, + contractor: Contractor, equipments: list[Equipment] | None = None, zones_pre: list[ZoneTransition] | None = None, zones_post: list[ZoneTransition] | None = None, @@ -52,11 +52,10 @@ def __init__(self, self.materials = materials if materials is not None else MaterialDelivery('') self.object = c_object if c_object is not None else [] + self.contractor_id = '' if contractor is not None: - if isinstance(contractor, str): - self.contractor = contractor - else: - self.contractor = contractor.name if contractor.name else contractor.id + self.contractor = contractor.name if contractor.name else contractor.id + self.contractor_id = contractor.id else: self.contractor = "" diff --git a/sampo/utilities/validation.py b/sampo/utilities/validation.py index 47a524fd..9eba876b 100644 --- a/sampo/utilities/validation.py +++ b/sampo/utilities/validation.py @@ -4,11 +4,15 @@ from sampo.schemas.contractor import Contractor from sampo.schemas.graph import WorkGraph from sampo.schemas.schedule import ScheduledWork, Schedule +from sampo.schemas.schedule_spec import ScheduleSpec from sampo.schemas.time import Time from sampo.utilities.collections_util import build_index -def validate_schedule(schedule: Schedule, wg: WorkGraph, contractors: list[Contractor]) -> None: +def validate_schedule(schedule: Schedule, + wg: WorkGraph, + contractors: list[Contractor], + spec: ScheduleSpec = ScheduleSpec()) -> None: """ Checks if the schedule is correct and can be executed. If there is an error, this function raises AssertException with an appropriate message @@ -17,6 +21,7 @@ def validate_schedule(schedule: Schedule, wg: WorkGraph, contractors: list[Contr :param contractors: :param wg: :param schedule: to apply verification + :param spec: """ # checking preconditions # check_all_workers_have_same_qualification(schedule.workGraph, schedule.resourcePools) @@ -27,6 +32,7 @@ def validate_schedule(schedule: Schedule, wg: WorkGraph, contractors: list[Contr # checking the schedule itself _check_all_tasks_scheduled(schedule, wg) _check_parent_dependencies(schedule, wg) + _check_all_tasks_corresponds_to_spec(schedule, wg, spec) _check_all_tasks_have_valid_duration(schedule) _check_all_workers_correspond_to_worker_reqs(wg, schedule) _check_all_allocated_workers_do_not_exceed_capacity_of_contractors(schedule, contractors) @@ -52,6 +58,18 @@ def _check_parent_dependencies(schedule: Schedule, wg: WorkGraph) -> None: assert pstart <= pend <= start <= end +def _check_all_tasks_corresponds_to_spec(schedule: Schedule, wg: WorkGraph, spec: ScheduleSpec) -> None: + scheduled_works: dict[str, ScheduledWork] = {work.id: work for work in schedule.works} + + # contractors + for node in wg.nodes: + work_spec = spec[node.id] + if work_spec.contractors: + assert work_spec.is_contractor_enabled(scheduled_works[node.id].contractor_id) + + # TODO Check other spec entries + + def _check_all_tasks_have_valid_duration(schedule: Schedule) -> None: # 3. check if all tasks have duration appropriate for their working hours service_works_with_incorrect_duration = [ diff --git a/tests/conftest.py b/tests/conftest.py index a7e1c329..7e8afdaa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -17,22 +17,23 @@ from sampo.schemas.landscape import LandscapeConfiguration from sampo.schemas.requirements import MaterialReq from sampo.schemas.resources import Worker +from sampo.schemas.schedule_spec import ScheduleSpec from sampo.schemas.time_estimator import WorkTimeEstimator, DefaultWorkEstimator from sampo.structurator.base import graph_restructuring from sampo.utilities.sampler import Sampler -@fixture +@fixture(scope='module') def setup_sampler(request) -> Sampler: return Sampler(1e-1) -@fixture +@fixture(scope='module') def setup_rand() -> Random: return Random(231) -@fixture +@fixture(scope='module') def setup_simple_synthetic(setup_rand) -> SimpleSynthetic: return SimpleSynthetic(setup_rand) @@ -50,9 +51,35 @@ def setup_simple_synthetic(setup_rand) -> SimpleSynthetic: for generate_materials in [True, False] for graph_type in ['manual', 'small plain synthetic', 'big plain synthetic'] if generate_materials and graph_type == 'manual' - or not generate_materials]) + or not generate_materials], + scope='module') # 'small advanced synthetic', 'big advanced synthetic']]) def setup_wg(request, setup_sampler, setup_simple_synthetic) -> WorkGraph: + # TODO Rewrite tests with random propagation. Now here is backcompat + return generate_wg_core(request, setup_sampler, setup_simple_synthetic)[0] + + +@fixture(params=[(graph_type, lag, generate_materials) + for lag in [True, False] + for generate_materials in [True, False] + for graph_type in ['manual', 'small plain synthetic', 'big plain synthetic'] + if generate_materials and graph_type == 'manual' + or not generate_materials + ], + # 'small advanced synthetic', 'big advanced synthetic']], + ids=[f'Graph: {graph_type}, LAG_OPT={lag_opt}, generate_materials={generate_materials}' + for lag_opt in [True, False] + for generate_materials in [True, False] + for graph_type in ['manual', 'small plain synthetic', 'big plain synthetic'] + if generate_materials and graph_type == 'manual' + or not generate_materials], + scope='module') +# 'small advanced synthetic', 'big advanced synthetic']]) +def setup_wg_with_random(request, setup_sampler, setup_simple_synthetic) -> tuple[WorkGraph, Random]: + return generate_wg_core(request, setup_sampler, setup_simple_synthetic) + + +def generate_wg_core(request, setup_sampler, setup_simple_synthetic) -> tuple[WorkGraph, Random]: SMALL_GRAPH_SIZE = 100 BIG_GRAPH_SIZE = 300 BORDER_RADIUS = 20 @@ -61,6 +88,8 @@ def setup_wg(request, setup_sampler, setup_simple_synthetic) -> WorkGraph: graph_type, lag_optimization, generate_materials = request.param + rand = setup_simple_synthetic.get_rand() + match graph_type: case 'manual': sr = setup_sampler @@ -105,28 +134,54 @@ def setup_wg(request, setup_sampler, setup_simple_synthetic) -> WorkGraph: materials_name = ['stone', 'brick', 'sand', 'rubble', 'concrete', 'metal'] for node in wg.nodes: if not node.work_unit.is_service_unit: - work_materials = list(set(random.choices(materials_name, k=random.randint(2, 6)))) - node.work_unit.material_reqs = [MaterialReq(name, random.randint(52, 345), name) for name in + work_materials = list(set(rand.choices(materials_name, k=rand.randint(2, 6)))) + node.work_unit.material_reqs = [MaterialReq(name, rand.randint(52, 345), name) for name in work_materials] - return graph_restructuring(wg, use_lag_edge_optimization=lag_optimization) + wg = graph_restructuring(wg, use_lag_edge_optimization=lag_optimization) + + return wg, rand + + +def create_spec(wg: WorkGraph, + contractors: list[Contractor], + rand: Random, + generate_contractor_spec: bool) -> ScheduleSpec: + spec = ScheduleSpec() + + if generate_contractor_spec: + for node in wg.nodes: + if not node.is_inseparable_son(): + selected_contractor_indices = rand.choices(list(range(len(contractors))), + k=rand.randint(1, len(contractors))) + spec.assign_contractors(node.id, {contractors[i].id for i in selected_contractor_indices}) + + return spec # TODO Make parametrization with different(specialized) contractors -@fixture(params=[(i, 5 * j) for j in range(2) for i in range(1, 2)], - ids=[f'Contractors: count={i}, min_size={5 * j}' for j in range(2) for i in range(1, 2)]) -def setup_scheduler_parameters(request, setup_wg, setup_simple_synthetic) -> tuple[ - WorkGraph, list[Contractor], LandscapeConfiguration | Any]: +@fixture(params=[(i, 5 * j, generate_contractors_spec) + for j in range(2) + for i in range(2, 3) + for generate_contractors_spec in [True, False]], + ids=[f'Contractors: count={i}, min_size={5 * j}, generate_contractor_spec={generate_contractors_spec}' + for j in range(2) + for i in range(2, 3) + for generate_contractors_spec in [True, False]], + scope='module') +def setup_scheduler_parameters(request, setup_wg_with_random, setup_simple_synthetic) \ + -> tuple[WorkGraph, list[Contractor], LandscapeConfiguration | Any, ScheduleSpec, Random]: + num_contractors, contractor_min_resources, generate_contractors_spec = request.param + wg, rand = setup_wg_with_random + generate_landscape = False - materials = [material for node in setup_wg.nodes for material in node.work_unit.need_materials()] + materials = [material for node in wg.nodes for material in node.work_unit.need_materials()] if len(materials) > 0: generate_landscape = True resource_req: Dict[str, int] = {} resource_req_count: Dict[str, int] = {} - num_contractors, contractor_min_resources = request.param - - for node in setup_wg.nodes: + for node in wg.nodes: for req in node.work_unit.worker_reqs: resource_req[req.kind] = max(contractor_min_resources, resource_req.get(req.kind, 0) + (req.min_count + req.max_count) // 2) @@ -135,7 +190,7 @@ def setup_scheduler_parameters(request, setup_wg, setup_simple_synthetic) -> tup for req in resource_req.keys(): resource_req[req] = resource_req[req] // resource_req_count[req] + 1 - for node in setup_wg.nodes: + for node in wg.nodes: for req in node.work_unit.worker_reqs: assert resource_req[req.kind] >= req.min_count @@ -150,12 +205,15 @@ def setup_scheduler_parameters(request, setup_wg, setup_simple_synthetic) -> tup for name, count in resource_req.items()}, equipments={})) - landscape = setup_simple_synthetic.synthetic_landscape(setup_wg) \ + landscape = setup_simple_synthetic.synthetic_landscape(wg) \ if generate_landscape else LandscapeConfiguration() - return setup_wg, contractors, landscape + + spec = create_spec(wg, contractors, rand, generate_contractors_spec) + + return wg, contractors, landscape, spec, rand -@fixture +@fixture(scope='module') def setup_empty_contractors(setup_wg) -> list[Contractor]: resource_req: set[str] = set() @@ -176,30 +234,33 @@ def setup_empty_contractors(setup_wg) -> list[Contractor]: return contractors -@fixture +@fixture(scope='module') def setup_default_schedules(setup_scheduler_parameters): work_estimator: WorkTimeEstimator = DefaultWorkEstimator() - setup_wg, setup_contractors, landscape = setup_scheduler_parameters + setup_wg, setup_contractors, landscape, spec, rand = setup_scheduler_parameters return setup_scheduler_parameters, GeneticScheduler.generate_first_population(setup_wg, setup_contractors, + spec=spec, landscape=landscape, work_estimator=work_estimator) @fixture(params=[HEFTScheduler(), HEFTBetweenScheduler(), TopologicalScheduler(), GeneticScheduler(3)], - ids=['HEFTScheduler', 'HEFTBetweenScheduler', 'TopologicalScheduler', 'GeneticScheduler']) + ids=['HEFTScheduler', 'HEFTBetweenScheduler', 'TopologicalScheduler', 'GeneticScheduler'], + scope='module') def setup_scheduler(request) -> Scheduler: return request.param -@fixture +@fixture(scope='module') def setup_schedule(setup_scheduler, setup_scheduler_parameters): - setup_wg, setup_contractors, landscape = setup_scheduler_parameters + setup_wg, setup_contractors, landscape, spec, rand = setup_scheduler_parameters scheduler = setup_scheduler try: return scheduler.schedule(setup_wg, setup_contractors, + spec=spec, validate=False, landscape=landscape)[0], scheduler.scheduler_type, setup_scheduler_parameters except NoSufficientContractorError: diff --git a/tests/pipeline/basic_pipeline_test.py b/tests/pipeline/basic_pipeline_test.py index 2e9b8b9c..7608c686 100644 --- a/tests/pipeline/basic_pipeline_test.py +++ b/tests/pipeline/basic_pipeline_test.py @@ -10,12 +10,13 @@ def test_plain_scheduling(setup_scheduler_parameters): - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand = setup_scheduler_parameters project = SchedulingPipeline.create() \ .wg(setup_wg) \ .contractors(setup_contractors) \ .landscape(setup_landscape) \ + .spec(spec) \ .schedule(HEFTScheduler()) \ .finish()[0] @@ -23,12 +24,13 @@ def test_plain_scheduling(setup_scheduler_parameters): def test_local_optimize_scheduling(setup_scheduler_parameters): - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand = setup_scheduler_parameters project = SchedulingPipeline.create() \ .wg(setup_wg) \ .contractors(setup_contractors) \ .landscape(setup_landscape) \ + .spec(spec) \ .optimize_local(SwapOrderLocalOptimizer(), range(0, setup_wg.vertex_count // 2)) \ .schedule(HEFTScheduler()) \ .optimize_local(ParallelizeScheduleLocalOptimizer(JustInTimeTimeline), range(0, setup_wg.vertex_count // 2)) \ diff --git a/tests/scheduler/genetic/converter_test.py b/tests/scheduler/genetic/converter_test.py index 59147f8d..246b113c 100644 --- a/tests/scheduler/genetic/converter_test.py +++ b/tests/scheduler/genetic/converter_test.py @@ -12,9 +12,10 @@ def test_convert_schedule_to_chromosome(setup_toolbox): - tb, _, setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox + tb, _, setup_wg, setup_contractors, spec, rand, _, setup_landscape_many_holders = setup_toolbox - schedule, _, _, node_order = HEFTScheduler().schedule_with_cache(setup_wg, setup_contractors, validate=True, + schedule, _, _, node_order = HEFTScheduler().schedule_with_cache(setup_wg, setup_contractors, + spec=spec, landscape=setup_landscape_many_holders)[0] chromosome = tb.schedule_to_chromosome(schedule=schedule, order=node_order) @@ -22,7 +23,7 @@ def test_convert_schedule_to_chromosome(setup_toolbox): def test_convert_chromosome_to_schedule(setup_toolbox): - tb, _, setup_wg, setup_contractors, _, _ = setup_toolbox + tb, _, setup_wg, setup_contractors, spec, rand, _, _ = setup_toolbox chromosome = tb.generate_chromosome() schedule, _, _, _ = tb.chromosome_to_schedule(chromosome) @@ -30,11 +31,11 @@ def test_convert_chromosome_to_schedule(setup_toolbox): assert not schedule.execution_time.is_inf() - validate_schedule(schedule, setup_wg, setup_contractors) + validate_schedule(schedule, setup_wg, setup_contractors, spec) def test_converter_with_borders_contractor_accounting(setup_toolbox): - tb, _, setup_wg, setup_contractors, _, setup_landscape_many_holders = setup_toolbox + tb, _, setup_wg, setup_contractors, spec, rand, _, setup_landscape_many_holders = setup_toolbox chromosome = tb.generate_chromosome(landscape=setup_landscape_many_holders) @@ -58,4 +59,4 @@ def test_converter_with_borders_contractor_accounting(setup_toolbox): schedule = Schedule.from_scheduled_works(schedule.values(), setup_wg) - validate_schedule(schedule, setup_wg, contractors) + validate_schedule(schedule, setup_wg, contractors, spec) diff --git a/tests/scheduler/genetic/fixtures.py b/tests/scheduler/genetic/fixtures.py index 405be3d2..1f3bc853 100644 --- a/tests/scheduler/genetic/fixtures.py +++ b/tests/scheduler/genetic/fixtures.py @@ -30,11 +30,10 @@ def get_params(works_count: int) -> tuple[float, float, float, int]: @fixture def setup_toolbox(setup_default_schedules) -> tuple: - (wg, contractors, landscape), setup_default_schedules = setup_default_schedules + (wg, contractors, landscape, spec, rand), setup_default_schedules = setup_default_schedules setup_worker_pool = get_worker_contractor_pool(contractors) mutate_order, mutate_resources, mutate_zones, size_of_population = get_params(wg.vertex_count) - rand = Random(123) work_estimator: WorkTimeEstimator = DefaultWorkEstimator() nodes, *_ = get_head_nodes_with_connections_mappings(wg) @@ -54,7 +53,8 @@ def setup_toolbox(setup_default_schedules) -> tuple: mutate_zones, setup_default_schedules, rand, + spec=spec, work_estimator=work_estimator, landscape=landscape, - verbose=False), resources_border, - wg, contractors, setup_default_schedules, landscape) + verbose=False), + resources_border, wg, contractors, spec, rand, setup_default_schedules, landscape) diff --git a/tests/scheduler/genetic/full_scheduling.py b/tests/scheduler/genetic/full_scheduling.py index 73184b16..61110ba4 100644 --- a/tests/scheduler/genetic/full_scheduling.py +++ b/tests/scheduler/genetic/full_scheduling.py @@ -7,7 +7,7 @@ def test_multiprocessing(setup_scheduler_parameters): - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, _ = setup_scheduler_parameters SAMPO.backend = DefaultComputationalBackend() @@ -17,14 +17,14 @@ def test_multiprocessing(setup_scheduler_parameters): size_of_population=50) start_default = time.time() - genetic.schedule(setup_wg, setup_contractors, validate=True, landscape=setup_landscape) + genetic.schedule(setup_wg, setup_contractors, spec=spec, validate=True, landscape=setup_landscape) time_default = time.time() - start_default n_cpus = 10 SAMPO.backend = MultiprocessingComputationalBackend(n_cpus=n_cpus) start_multiproc = time.time() - genetic.schedule(setup_wg, setup_contractors, landscape=setup_landscape) + genetic.schedule(setup_wg, setup_contractors, spec=spec, landscape=setup_landscape) time_multiproc = time.time() - start_multiproc print('\n------------------\n') diff --git a/tests/scheduler/genetic/operators_test.py b/tests/scheduler/genetic/operators_test.py index 658d2869..edd0b051 100644 --- a/tests/scheduler/genetic/operators_test.py +++ b/tests/scheduler/genetic/operators_test.py @@ -7,7 +7,7 @@ def test_generate_individual(setup_toolbox): - tb, _, _, _, _, _ = setup_toolbox + tb, _, _, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): chromosome = tb.generate_chromosome() @@ -15,7 +15,7 @@ def test_generate_individual(setup_toolbox): def test_mutate_order(setup_toolbox): - tb, _, _, _, _, _ = setup_toolbox + tb, _, _, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() @@ -28,7 +28,7 @@ def test_mutate_order(setup_toolbox): def test_mutate_resources(setup_toolbox): - tb, _, _, _, _, _ = setup_toolbox + tb, _, _, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() @@ -38,7 +38,7 @@ def test_mutate_resources(setup_toolbox): def test_mutate_resource_borders(setup_toolbox): - tb, _, _, _, _, _ = setup_toolbox + tb, _, _, _, _, _, _, _ = setup_toolbox for i in range(TEST_ITERATIONS): individual = tb.generate_chromosome() @@ -48,7 +48,7 @@ def test_mutate_resource_borders(setup_toolbox): def test_mate_order(setup_toolbox, setup_wg): - tb, _, _, _, _, _ = setup_toolbox + tb, _, _, _, _, _, _, _ = setup_toolbox _, _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) @@ -68,7 +68,7 @@ def test_mate_order(setup_toolbox, setup_wg): def test_mate_resources(setup_toolbox, setup_wg): - tb, resources_border, _, _, _, _ = setup_toolbox + tb, resources_border, _, _, _, _, _, _ = setup_toolbox _, _, _, population_size = get_params(setup_wg.vertex_count) population = tb.population(n=population_size) @@ -86,5 +86,3 @@ def test_mate_resources(setup_toolbox, setup_wg): # check the whole chromosomes assert tb.validate(individual1) assert tb.validate(individual2) - - diff --git a/tests/scheduler/lft/fixtures.py b/tests/scheduler/lft/fixtures.py index 2ec94588..1e1f702b 100644 --- a/tests/scheduler/lft/fixtures.py +++ b/tests/scheduler/lft/fixtures.py @@ -13,6 +13,6 @@ def setup_schedulers_and_parameters(request, setup_scheduler_parameters) -> tupl else: scheduler = scheduler() - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand = setup_scheduler_parameters - return setup_wg, setup_contractors, setup_landscape, scheduler + return setup_wg, setup_contractors, setup_landscape, spec, rand, scheduler diff --git a/tests/scheduler/lft/prioritization_test.py b/tests/scheduler/lft/prioritization_test.py index 9b73a819..c93094ae 100644 --- a/tests/scheduler/lft/prioritization_test.py +++ b/tests/scheduler/lft/prioritization_test.py @@ -5,7 +5,7 @@ def test_correct_order(setup_schedulers_and_parameters): - setup_wg, setup_contractors, _, scheduler = setup_schedulers_and_parameters + setup_wg, setup_contractors, _, _, _, scheduler = setup_schedulers_and_parameters worker_pool = get_worker_contractor_pool(setup_contractors) nodes, node_id2parent_ids, node_id2child_ids = get_head_nodes_with_connections_mappings(setup_wg) node_id2duration = scheduler._contractor_workers_assignment(nodes, setup_contractors, worker_pool) diff --git a/tests/scheduler/lft/scheduling_test.py b/tests/scheduler/lft/scheduling_test.py index 370a75c2..52a57042 100644 --- a/tests/scheduler/lft/scheduling_test.py +++ b/tests/scheduler/lft/scheduling_test.py @@ -3,9 +3,10 @@ def test_lft_scheduling(setup_schedulers_and_parameters): - setup_wg, setup_contractors, setup_landscape, scheduler = setup_schedulers_and_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand, scheduler = setup_schedulers_and_parameters schedule = scheduler.schedule(setup_wg, setup_contractors, + spec=spec, validate=True, landscape=setup_landscape)[0] lft_time = schedule.execution_time @@ -13,6 +14,6 @@ def test_lft_scheduling(setup_schedulers_and_parameters): assert not lft_time.is_inf() try: - validate_schedule(schedule, setup_wg, setup_contractors) + validate_schedule(schedule, setup_wg, setup_contractors, spec) except AssertionError as e: raise AssertionError(f'Scheduler {scheduler} failed validation', e) diff --git a/tests/scheduler/material_scheduling_test.py b/tests/scheduler/material_scheduling_test.py index 33a6905d..ae1149dd 100644 --- a/tests/scheduler/material_scheduling_test.py +++ b/tests/scheduler/material_scheduling_test.py @@ -13,7 +13,7 @@ def test_empty_node_find_start_time(setup_default_schedules): - wg, _, landscape = setup_default_schedules[0] + wg, _, landscape, _, _ = setup_default_schedules[0] if wg.vertex_count > 14: pytest.skip('Non-material graph') @@ -83,15 +83,15 @@ def test_empty_node_find_start_time(setup_default_schedules): def test_momentum_scheduling_with_materials(setup_default_schedules): - setup_wg, setup_contractors, landscape = setup_default_schedules[0] + setup_wg, setup_contractors, landscape, spec, _ = setup_default_schedules[0] if setup_wg.vertex_count > 14: pytest.skip('Non-material graph') scheduler = HEFTBetweenScheduler() - schedule = scheduler.schedule(setup_wg, setup_contractors, validate=True, landscape=landscape)[0] + schedule = scheduler.schedule(setup_wg, setup_contractors, validate=False, spec=spec, landscape=landscape)[0] try: - validate_schedule(schedule, setup_wg, setup_contractors) + validate_schedule(schedule, setup_wg, setup_contractors, spec) except AssertionError as e: raise AssertionError(f'Scheduler {scheduler} failed validation', e) @@ -99,10 +99,10 @@ def test_momentum_scheduling_with_materials(setup_default_schedules): def test_scheduler_with_materials_validity_right(setup_schedule): schedule = setup_schedule[0] - setup_wg, setup_contractors, landscape = setup_schedule[2] + setup_wg, setup_contractors, landscape, spec, _ = setup_schedule[2] try: - validate_schedule(schedule, setup_wg, setup_contractors) + validate_schedule(schedule, setup_wg, setup_contractors, spec) except AssertionError as e: raise AssertionError(f'Scheduler {setup_schedule[1]} failed validation', e) diff --git a/tests/scheduler/resources_in_time/basic_res_test.py b/tests/scheduler/resources_in_time/basic_res_test.py index a9da7a07..c5765215 100644 --- a/tests/scheduler/resources_in_time/basic_res_test.py +++ b/tests/scheduler/resources_in_time/basic_res_test.py @@ -9,7 +9,7 @@ def test_deadline_planning(setup_scheduler_parameters): - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand = setup_scheduler_parameters scheduler = AverageBinarySearchResourceOptimizingScheduler(HEFTScheduler()) @@ -30,7 +30,7 @@ def test_deadline_planning(setup_scheduler_parameters): def test_genetic_deadline_planning(setup_scheduler_parameters): - setup_wg, setup_contractors, landscape = setup_scheduler_parameters + setup_wg, setup_contractors, landscape, spec, rand = setup_scheduler_parameters deadline = Time.inf() // 2 scheduler = GeneticScheduler(number_of_generation=5, @@ -49,7 +49,7 @@ def test_genetic_deadline_planning(setup_scheduler_parameters): def test_true_deadline_planning(setup_scheduler_parameters): - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand = setup_scheduler_parameters scheduler = AverageBinarySearchResourceOptimizingScheduler(HEFTScheduler()) @@ -73,7 +73,7 @@ def test_true_deadline_planning(setup_scheduler_parameters): def test_lexicographic_genetic_deadline_planning(setup_scheduler_parameters): - setup_wg, setup_contractors, setup_landscape = setup_scheduler_parameters + setup_wg, setup_contractors, setup_landscape, spec, rand = setup_scheduler_parameters scheduler = HEFTScheduler() schedule, _, _, _ = scheduler.schedule_with_cache(setup_wg, setup_contractors, landscape=setup_landscape)[0] diff --git a/tests/scheduler/timeline/just_in_time_timeline_test.py b/tests/scheduler/timeline/just_in_time_timeline_test.py index 601234d7..ada32d61 100644 --- a/tests/scheduler/timeline/just_in_time_timeline_test.py +++ b/tests/scheduler/timeline/just_in_time_timeline_test.py @@ -15,14 +15,14 @@ @fixture(scope='function') def setup_timeline(setup_scheduler_parameters): - setup_wg, setup_contractors, landscape = setup_scheduler_parameters + setup_wg, setup_contractors, landscape, spec, rand = setup_scheduler_parameters setup_worker_pool = get_worker_contractor_pool(setup_contractors) return JustInTimeTimeline(setup_worker_pool, landscape=landscape), \ - setup_wg, setup_contractors, setup_worker_pool + setup_wg, setup_contractors, setup_worker_pool, spec, rand def test_init_resource_structure(setup_timeline): - setup_timeline, _, _, _ = setup_timeline + setup_timeline, _, _, _, _, _ = setup_timeline assert len(setup_timeline._timeline) != 0 for setup_timeline in setup_timeline._timeline.values(): @@ -52,7 +52,7 @@ def test_init_resource_structure(setup_timeline): def test_schedule(setup_timeline): - setup_timeline, setup_wg, setup_contractors, setup_worker_pool = setup_timeline + setup_timeline, setup_wg, setup_contractors, setup_worker_pool, _, _ = setup_timeline nodes, node_id2parent_ids, node_id2child_ids = get_head_nodes_with_connections_mappings(setup_wg) ordered_nodes = prioritization(nodes, node_id2parent_ids, node_id2child_ids, DefaultWorkEstimator()) diff --git a/tests/scheduler/timeline/material_timeline_test.py b/tests/scheduler/timeline/material_timeline_test.py index d4c54277..5027d9c1 100644 --- a/tests/scheduler/timeline/material_timeline_test.py +++ b/tests/scheduler/timeline/material_timeline_test.py @@ -14,7 +14,7 @@ def setup_timeline(setup_scheduler_parameters): def test_supply_resources(setup_scheduler_parameters, setup_rand): - wg, contractors, landscape = setup_scheduler_parameters + wg, contractors, landscape, _, _ = setup_scheduler_parameters if not landscape.platforms: pytest.skip('Non landscape test') timeline = HybridSupplyTimeline(landscape) diff --git a/tests/scheduler/timeline/momentum_timeline_test.py b/tests/scheduler/timeline/momentum_timeline_test.py index e94ebc4a..ab7d8c8e 100644 --- a/tests/scheduler/timeline/momentum_timeline_test.py +++ b/tests/scheduler/timeline/momentum_timeline_test.py @@ -13,15 +13,15 @@ @fixture def setup_timeline_context(setup_scheduler_parameters): - setup_wg, setup_contractors, landscape = setup_scheduler_parameters + setup_wg, setup_contractors, landscape, spec, rand = setup_scheduler_parameters setup_worker_pool = get_worker_contractor_pool(setup_contractors) worker_kinds = set([w_kind for contractor in setup_contractors for w_kind in contractor.workers.keys()]) return MomentumTimeline(setup_worker_pool, landscape=landscape), \ - setup_wg, setup_contractors, setup_worker_pool, worker_kinds + setup_wg, setup_contractors, spec, rand, setup_worker_pool, worker_kinds def test_init_resource_structure(setup_timeline_context): - timeline, wg, contractors, worker_pool, worker_kinds = setup_timeline_context + timeline, wg, contractors, _, _, worker_pool, worker_kinds = setup_timeline_context assert len(timeline._timeline) != 0 for contractor_timeline in timeline._timeline.values(): @@ -37,7 +37,7 @@ def test_init_resource_structure(setup_timeline_context): def test_insert_works_with_one_worker_kind(setup_timeline_context): - timeline, wg, contractors, worker_pool, worker_kinds = setup_timeline_context + timeline, wg, contractors, _, _, worker_pool, worker_kinds = setup_timeline_context worker_kind = worker_kinds.pop() worker_kinds.add(worker_kind) # make worker_kinds stay unchanged diff --git a/tests/utils/validation_test.py b/tests/utils/validation_test.py index 55118f98..96d0d6db 100644 --- a/tests/utils/validation_test.py +++ b/tests/utils/validation_test.py @@ -26,7 +26,7 @@ def is_resources_break(self) -> bool: def test_check_order_validity_right(setup_default_schedules): - (setup_wg, _, _), setup_default_schedules = setup_default_schedules + (setup_wg, _, _, _, _), setup_default_schedules = setup_default_schedules for scheduler, (schedule, _, _, _) in setup_default_schedules.items(): try: @@ -37,7 +37,7 @@ def test_check_order_validity_right(setup_default_schedules): def test_check_order_validity_wrong(setup_default_schedules): - (setup_wg, _, _), setup_default_schedules = setup_default_schedules + (setup_wg, _, _, _, _), setup_default_schedules = setup_default_schedules for (schedule, _, _, _) in setup_default_schedules.values(): for break_type in BreakType: @@ -54,7 +54,7 @@ def test_check_order_validity_wrong(setup_default_schedules): def test_check_resources_validity_right(setup_default_schedules): - (setup_wg, setup_contractors, _), setup_default_schedules = setup_default_schedules + (setup_wg, setup_contractors, _, _, _), setup_default_schedules = setup_default_schedules for scheduler, (schedule, _, _, _) in setup_default_schedules.items(): try: @@ -65,7 +65,7 @@ def test_check_resources_validity_right(setup_default_schedules): def test_check_resources_validity_wrong(setup_default_schedules): - (setup_wg, setup_contractors, _), setup_default_schedules = setup_default_schedules + (setup_wg, setup_contractors, _, _, _), setup_default_schedules = setup_default_schedules setup_worker_pool = get_worker_contractor_pool(setup_contractors) for (schedule, _, _, _) in setup_default_schedules.values():