From 37c5338938e2514911815bbbbd8aa6c9272f3f3c Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Fri, 23 May 2025 11:12:20 +0200 Subject: [PATCH 01/34] Typo. --- kernel_tuner/runners/sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index aeebd5116..194eb0545 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -55,7 +55,7 @@ def run(self, parameter_space, tuning_options): :param tuning_options: A dictionary with all options regarding the tuning process. - :type tuning_options: kernel_tuner.iterface.Options + :type tuning_options: kernel_tuner.interface.Options :returns: A list of dictionaries for executed kernel configurations and their execution times. From 6c5b360cad115d840852f618f6ffda9cf89addbd Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Fri, 23 May 2025 11:12:43 +0200 Subject: [PATCH 02/34] Add missing parameter to the interface. --- kernel_tuner/runners/runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel_tuner/runners/runner.py b/kernel_tuner/runners/runner.py index 80ab32146..8c4de22d7 100644 --- a/kernel_tuner/runners/runner.py +++ b/kernel_tuner/runners/runner.py @@ -14,7 +14,7 @@ def __init__( pass @abstractmethod - def get_environment(self): + def get_environment(self, tuning_options): pass @abstractmethod From a21caf8398306cb9f7b136043ae97f4a084c2759 Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Fri, 23 May 2025 11:53:57 +0200 Subject: [PATCH 03/34] Formatting. --- kernel_tuner/runners/sequential.py | 53 +++++++++++++++++------------- kernel_tuner/runners/simulation.py | 40 +++++++++++----------- 2 files changed, 50 insertions(+), 43 deletions(-) diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index 194eb0545..eeeedbd29 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -20,15 +20,13 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob :param kernel_options: A dictionary with all options for the kernel. :type kernel_options: kernel_tuner.interface.Options - :param device_options: A dictionary with all options for the device - on which the kernel should be tuned. + :param device_options: A dictionary with all options for the device on which the kernel should be tuned. :type device_options: kernel_tuner.interface.Options - :param iterations: The number of iterations used for benchmarking - each kernel instance. + :param iterations: The number of iterations used for benchmarking each kernel instance. :type iterations: int """ - #detect language and create high-level device interface + # detect language and create high-level device interface self.dev = DeviceInterface(kernel_source, iterations=iterations, observers=observers, **device_options) self.units = self.dev.units @@ -41,7 +39,7 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob self.last_strategy_time = 0 self.kernel_options = kernel_options - #move data to the GPU + # move data to the GPU self.gpu_args = self.dev.ready_argument_list(kernel_options.arguments) def get_environment(self, tuning_options): @@ -53,16 +51,14 @@ def run(self, parameter_space, tuning_options): :param parameter_space: The parameter space as an iterable. :type parameter_space: iterable - :param tuning_options: A dictionary with all options regarding the tuning - process. + :param tuning_options: A dictionary with all options regarding the tuning process. :type tuning_options: kernel_tuner.interface.Options - :returns: A list of dictionaries for executed kernel configurations and their - execution times. - :rtype: dict()) + :returns: A list of dictionaries for executed kernel configurations and their execution times. + :rtype: dict() """ - logging.debug('sequential runner started for ' + self.kernel_options.kernel_name) + logging.debug("sequential runner started for " + self.kernel_options.kernel_name) results = [] @@ -77,33 +73,46 @@ def run(self, parameter_space, tuning_options): x_int = ",".join([str(i) for i in element]) if tuning_options.cache and x_int in tuning_options.cache: params.update(tuning_options.cache[x_int]) - params['compile_time'] = 0 - params['verification_time'] = 0 - params['benchmark_time'] = 0 + params["compile_time"] = 0 + params["verification_time"] = 0 + params["benchmark_time"] = 0 else: # attempt to warmup the GPU by running the first config in the parameter space and ignoring the result if not self.warmed_up: warmup_time = perf_counter() - self.dev.compile_and_benchmark(self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options) + self.dev.compile_and_benchmark( + self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options + ) self.warmed_up = True warmup_time = 1e3 * (perf_counter() - warmup_time) - result = self.dev.compile_and_benchmark(self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options) + result = self.dev.compile_and_benchmark( + self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options + ) params.update(result) if tuning_options.objective in result and isinstance(result[tuning_options.objective], ErrorConfig): - logging.debug('kernel configuration was skipped silently due to compile or runtime failure') + logging.debug("kernel configuration was skipped silently due to compile or runtime failure") # only compute metrics on configs that have not errored if tuning_options.metrics and not isinstance(params.get(tuning_options.objective), ErrorConfig): params = process_metrics(params, tuning_options.metrics) # get the framework time by estimating based on other times - total_time = 1000 * ((perf_counter() - self.start_time) - warmup_time) - params['strategy_time'] = self.last_strategy_time - params['framework_time'] = max(total_time - (params['compile_time'] + params['verification_time'] + params['benchmark_time'] + params['strategy_time']), 0) - params['timestamp'] = str(datetime.now(timezone.utc)) + total_time = 1000 * ((perf_counter() - self.start_time) - warmup_time) + params["strategy_time"] = self.last_strategy_time + params["framework_time"] = max( + total_time + - ( + params["compile_time"] + + params["verification_time"] + + params["benchmark_time"] + + params["strategy_time"] + ), + 0, + ) + params["timestamp"] = str(datetime.now(timezone.utc)) self.start_time = perf_counter() if result: diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index 22c7c667c..7a167bfcf 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -14,11 +14,11 @@ class SimulationDevice(_SimulationDevice): @property def name(self): - return self.env['device_name'] + return self.env["device_name"] @name.setter def name(self, value): - self.env['device_name'] = value + self.env["device_name"] = value if not self.quiet: print("Simulating: " + value) @@ -38,12 +38,10 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob :param kernel_options: A dictionary with all options for the kernel. :type kernel_options: kernel_tuner.interface.Options - :param device_options: A dictionary with all options for the device - on which the kernel should be tuned. + :param device_options: A dictionary with all options for the device on which the kernel should be tuned. :type device_options: kernel_tuner.interface.Options - :param iterations: The number of iterations used for benchmarking - each kernel instance. + :param iterations: The number of iterations used for benchmarking each kernel instance. :type iterations: int """ self.quiet = device_options.quiet @@ -70,21 +68,18 @@ def run(self, parameter_space, tuning_options): :param parameter_space: The parameter space as an iterable. :type parameter_space: iterable - :param tuning_options: A dictionary with all options regarding the tuning - process. + :param tuning_options: A dictionary with all options regarding the tuning process. :type tuning_options: kernel_tuner.iterface.Options - :returns: A list of dictionaries for executed kernel configurations and their - execution times. + :returns: A list of dictionaries for executed kernel configurations and their execution times. :rtype: dict() """ - logging.debug('simulation runner started for ' + self.kernel_options.kernel_name) + logging.debug("simulation runner started for " + self.kernel_options.kernel_name) results = [] # iterate over parameter space for element in parameter_space: - # check if element is in the cache x_int = ",".join([str(i) for i in element]) if tuning_options.cache and x_int in tuning_options.cache: @@ -98,21 +93,22 @@ def run(self, parameter_space, tuning_options): # configuration is already counted towards the unique_results. # It is the responsibility of cost_func to add configs to unique_results. if x_int in tuning_options.unique_results: - - result['compile_time'] = 0 - result['verification_time'] = 0 - result['benchmark_time'] = 0 + result["compile_time"] = 0 + result["verification_time"] = 0 + result["benchmark_time"] = 0 else: # configuration is evaluated for the first time, print to the console - util.print_config_output(tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units) + util.print_config_output( + tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units + ) # Everything but the strategy time and framework time are simulated, # self.last_strategy_time is set by cost_func - result['strategy_time'] = self.last_strategy_time + result["strategy_time"] = self.last_strategy_time try: - simulated_time = result['compile_time'] + result['verification_time'] + result['benchmark_time'] + simulated_time = result["compile_time"] + result["verification_time"] + result["benchmark_time"] tuning_options.simulated_time += simulated_time except KeyError: if "time_limit" in tuning_options: @@ -122,13 +118,15 @@ def run(self, parameter_space, tuning_options): total_time = 1000 * (perf_counter() - self.start_time) self.start_time = perf_counter() - result['framework_time'] = total_time - self.last_strategy_time + result["framework_time"] = total_time - self.last_strategy_time results.append(result) continue # if the element is not in the cache, raise an error - check = util.check_restrictions(tuning_options.restrictions, dict(zip(tuning_options['tune_params'].keys(), element)), True) + check = util.check_restrictions( + tuning_options.restrictions, dict(zip(tuning_options["tune_params"].keys(), element)), True + ) err_string = f"kernel configuration {element} not in cache, does {'' if check else 'not '}pass extra restriction check ({check})" logging.debug(err_string) raise ValueError(f"{err_string} - in simulation mode, all configurations must be present in the cache") From a2328c4c5eb5abdfc99219dcda53615f2c7ed42f Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Thu, 5 Jun 2025 11:12:02 +0200 Subject: [PATCH 04/34] First early draft of the parallel runner. --- examples/cuda/vector_add_parallel.py | 35 ++++++ kernel_tuner/interface.py | 14 ++- kernel_tuner/runners/parallel.py | 166 +++++++++++++++++++++++++++ 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 examples/cuda/vector_add_parallel.py create mode 100644 kernel_tuner/runners/parallel.py diff --git a/examples/cuda/vector_add_parallel.py b/examples/cuda/vector_add_parallel.py new file mode 100644 index 000000000..d1c112aa5 --- /dev/null +++ b/examples/cuda/vector_add_parallel.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python + +import numpy +from kernel_tuner import tune_kernel + + +def tune(): + kernel_string = """ + __global__ void vector_add(float *c, float *a, float *b, int n) { + int i = (blockIdx.x * block_size_x) + threadIdx.x; + if ( i < n ) { + c[i] = a[i] + b[i]; + } + } + """ + + size = 10000000 + + a = numpy.random.randn(size).astype(numpy.float32) + b = numpy.random.randn(size).astype(numpy.float32) + c = numpy.zeros_like(b) + n = numpy.int32(size) + + args = [c, a, b, n] + + tune_params = dict() + tune_params["block_size_x"] = [32 * i for i in range(1, 33)] + + results, env = tune_kernel("vector_add", kernel_string, size, args, tune_params, parallel_runner=4) + + return results + + +if __name__ == "__main__": + tune() diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index bd421aeab..ed7b56487 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -32,6 +32,7 @@ import kernel_tuner.core as core import kernel_tuner.util as util from kernel_tuner.integration import get_objective_defaults +from kernel_tuner.runners.parallel import ParallelRunner from kernel_tuner.runners.sequential import SequentialRunner from kernel_tuner.runners.simulation import SimulationRunner from kernel_tuner.searchspace import Searchspace @@ -463,6 +464,7 @@ def __deepcopy__(self, _): ), ("metrics", ("specifies user-defined metrics, please see :ref:`metrics`.", "dict")), ("simulation_mode", ("Simulate an auto-tuning search from an existing cachefile", "bool")), + ("parallel_runner", ("If the value is larger than 1 use that number as the number of parallel runners doing the tuning", "int")), ("observers", ("""A list of Observers to use during tuning, please see :ref:`observers`.""", "list")), ] ) @@ -574,6 +576,7 @@ def tune_kernel( cache=None, metrics=None, simulation_mode=False, + parallel_runner=1, observers=None, objective=None, objective_higher_is_better=None, @@ -600,6 +603,8 @@ def tune_kernel( if iterations < 1: raise ValueError("Iterations should be at least one!") + if parallel_runner < 1: + logging.warning("The number of parallel runners should be at least one!") # sort all the options into separate dicts opts = locals() @@ -650,7 +655,14 @@ def tune_kernel( strategy = brute_force # select the runner for this job based on input - selected_runner = SimulationRunner if simulation_mode else SequentialRunner + # TODO: we could use the "match case" syntax when removing support for 3.9 + if simulation_mode: + selected_runner = SimulationRunner + elif parallel_runner > 1: + selected_runner = ParallelRunner + tuning_options.parallel_runner = parallel_runner + else: + selected_runner = SequentialRunner tuning_options.simulated_time = 0 runner = selected_runner(kernelsource, kernel_options, device_options, iterations, observers) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py new file mode 100644 index 000000000..a4362b0eb --- /dev/null +++ b/kernel_tuner/runners/parallel.py @@ -0,0 +1,166 @@ +"""A specialized runner that tunes in parallel the parameter space.""" +import logging +from time import perf_counter +from datetime import datetime, timezone + +from ray import remote, get, put + +from kernel_tuner.runners.runner import Runner +from kernel_tuner.core import DeviceInterface +from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache + + +class ParallelRunnerState: + """This class represents the state of a parallel tuning run.""" + + def __init__(self, observers, iterations): + self.device_options = None + self.quiet = False + self.kernel_source = None + self.warmed_up = False + self.simulation_mode = False + self.start_time = None + self.last_strategy_start_time = None + self.last_strategy_time = 0 + self.kernel_options = None + self.observers = observers + self.iterations = iterations + + +@remote +def parallel_run(task_id: int, state: ParallelRunnerState, parameter_space, tuning_options): + dev = DeviceInterface( + state.kernel_source, iterations=state.iterations, observers=state.observers, **state.device_options + ) + # move data to the GPU + gpu_args = dev.ready_argument_list(state.kernel_options.arguments) + # iterate over parameter space + results = [] + elements_per_task = len(parameter_space) / tuning_options.parallel_runner + first_element = task_id * elements_per_task + last_element = ( + (task_id + 1) * elements_per_task if task_id + 1 < tuning_options.parallel_runner else len(parameter_space) + ) + for element in parameter_space[first_element:last_element]: + params = dict(zip(tuning_options.tune_params.keys(), element)) + + result = None + warmup_time = 0 + + # check if configuration is in the cache + x_int = ",".join([str(i) for i in element]) + if tuning_options.cache and x_int in tuning_options.cache: + params.update(tuning_options.cache[x_int]) + params["compile_time"] = 0 + params["verification_time"] = 0 + params["benchmark_time"] = 0 + else: + # attempt to warm up the GPU by running the first config in the parameter space and ignoring the result + if not state.warmed_up: + warmup_time = perf_counter() + dev.compile_and_benchmark(state.kernel_source, gpu_args, params, state.kernel_options, tuning_options) + state.warmed_up = True + warmup_time = 1e3 * (perf_counter() - warmup_time) + + result = dev.compile_and_benchmark( + state.kernel_source, gpu_args, params, state.kernel_options, tuning_options + ) + + params.update(result) + + if tuning_options.objective in result and isinstance(result[tuning_options.objective], ErrorConfig): + logging.debug("kernel configuration was skipped silently due to compile or runtime failure") + + # only compute metrics on configs that have not errored + if tuning_options.metrics and not isinstance(params.get(tuning_options.objective), ErrorConfig): + params = process_metrics(params, tuning_options.metrics) + + # get the framework time by estimating based on other times + total_time = 1000 * ((perf_counter() - state.start_time) - warmup_time) + params["strategy_time"] = state.last_strategy_time + params["framework_time"] = max( + total_time + - ( + params["compile_time"] + + params["verification_time"] + + params["benchmark_time"] + + params["strategy_time"] + ), + 0, + ) + params["timestamp"] = str(datetime.now(timezone.utc)) + state.start_time = perf_counter() + + if result: + # print configuration to the console + print_config_output(tuning_options.tune_params, params, state.quiet, tuning_options.metrics, dev.units) + + # add configuration to cache + store_cache(x_int, params, tuning_options) + + # all visited configurations are added to results to provide a trace for optimization strategies + results.append(params) + + return results + + +class ParallelRunner(Runner): + """ParallelRunner is used to distribute configurations across multiple nodes.""" + + def __init__(self, kernel_source, kernel_options, device_options, iterations, observers): + """Instantiate the ParallelRunner. + + :param kernel_source: The kernel source + :type kernel_source: kernel_tuner.core.KernelSource + + :param kernel_options: A dictionary with all options for the kernel. + :type kernel_options: kernel_tuner.interface.Options + + :param device_options: A dictionary with all options for the device + on which the kernel should be tuned. + :type device_options: kernel_tuner.interface.Options + + :param iterations: The number of iterations used for benchmarking + each kernel instance. + :type iterations: int + """ + self.state = ParallelRunnerState(observers, iterations) + self.state.quiet = device_options.quiet + self.state.kernel_source = kernel_source + self.state.warmed_up = False + self.state.simulation_mode = False + self.state.start_time = perf_counter() + self.state.last_strategy_start_time = self.state.start_time + self.state.last_strategy_time = 0 + self.state.kernel_options = kernel_options + + def get_environment(self, tuning_options): + # TODO: we are going to fix this one later + return None + + def run(self, parameter_space, tuning_options): + """Iterate through the entire parameter space using a single Python process. + + :param parameter_space: The parameter space as an iterable. + :type parameter_space: iterable + + :param tuning_options: A dictionary with all options regarding the tuning process. + :type tuning_options: kernel_tuner.interface.Options + + :returns: A list of dictionaries for executed kernel configurations and their execution times. + :rtype: dict() + """ + # given the parameter_space, distribute it over Ray tasks + logging.debug("parallel runner started for " + self.state.kernel_options.kernel_name) + + results = [] + tasks = [] + parameter_space_ref = put(parameter_space) + state_ref = put(self.state) + tuning_options_ref = put(tuning_options) + for task_id in range(0, tuning_options.parallel_runner): + tasks.append(parallel_run.remote(task_id, state_ref, parameter_space_ref, tuning_options_ref)) + for task in tasks: + results.append(get(task)) + + return results From 68a569ba8849673cde3735824e3fd0a6a5c4e3ef Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Thu, 5 Jun 2025 11:36:35 +0200 Subject: [PATCH 05/34] Need a dummy DeviceInterface even on the master. --- kernel_tuner/runners/parallel.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index a4362b0eb..e8d759ecd 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -133,10 +133,12 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob self.state.last_strategy_start_time = self.state.start_time self.state.last_strategy_time = 0 self.state.kernel_options = kernel_options + # define a dummy device interface + self.dev = DeviceInterface(kernel_source) def get_environment(self, tuning_options): - # TODO: we are going to fix this one later - return None + # dummy environment + return self.dev.get_environment() def run(self, parameter_space, tuning_options): """Iterate through the entire parameter space using a single Python process. From 9d0dee4a4870ef7d7694ae363855c0cd5ca237ef Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Thu, 5 Jun 2025 11:41:59 +0200 Subject: [PATCH 06/34] Missing device_options in state. --- kernel_tuner/runners/parallel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index e8d759ecd..5b501d9c5 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -125,6 +125,7 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob :type iterations: int """ self.state = ParallelRunnerState(observers, iterations) + self.state.device_options = device_options self.state.quiet = device_options.quiet self.state.kernel_source = kernel_source self.state.warmed_up = False From aff21f035dd68b19e6b0de8c64c75efee3e01a4d Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Thu, 5 Jun 2025 15:12:08 +0200 Subject: [PATCH 07/34] Flatten the results. --- kernel_tuner/runners/parallel.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index 5b501d9c5..2d20bd4bc 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -2,6 +2,7 @@ import logging from time import perf_counter from datetime import datetime, timezone +from itertools import chain from ray import remote, get, put @@ -166,4 +167,4 @@ def run(self, parameter_space, tuning_options): for task in tasks: results.append(get(task)) - return results + return [chain.from_iterable(results)] From d7e8cae2778b7aad7408b76bbfe313aa69f05841 Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Thu, 5 Jun 2025 15:18:59 +0200 Subject: [PATCH 08/34] Various bug fixes. --- kernel_tuner/runners/parallel.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index 2d20bd4bc..f454d0686 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -28,7 +28,7 @@ def __init__(self, observers, iterations): self.iterations = iterations -@remote +@remote(num_cpus=1, num_gpus=1) def parallel_run(task_id: int, state: ParallelRunnerState, parameter_space, tuning_options): dev = DeviceInterface( state.kernel_source, iterations=state.iterations, observers=state.observers, **state.device_options @@ -37,9 +37,9 @@ def parallel_run(task_id: int, state: ParallelRunnerState, parameter_space, tuni gpu_args = dev.ready_argument_list(state.kernel_options.arguments) # iterate over parameter space results = [] - elements_per_task = len(parameter_space) / tuning_options.parallel_runner - first_element = task_id * elements_per_task - last_element = ( + elements_per_task = int(len(parameter_space) / tuning_options.parallel_runner) + first_element = int(task_id * elements_per_task) + last_element = int( (task_id + 1) * elements_per_task if task_id + 1 < tuning_options.parallel_runner else len(parameter_space) ) for element in parameter_space[first_element:last_element]: @@ -167,4 +167,4 @@ def run(self, parameter_space, tuning_options): for task in tasks: results.append(get(task)) - return [chain.from_iterable(results)] + return list(chain.from_iterable(results)) From b4ff7fa49574c8e277a71f70c289102e73243388 Mon Sep 17 00:00:00 2001 From: Alessio Sclocco Date: Fri, 6 Jun 2025 11:21:07 +0200 Subject: [PATCH 09/34] Add another example for the parallel runner. --- examples/cuda/sepconv_parallel.py | 88 +++++++++++++++++++++++++++++++ kernel_tuner/runners/parallel.py | 5 +- 2 files changed, 92 insertions(+), 1 deletion(-) create mode 100644 examples/cuda/sepconv_parallel.py diff --git a/examples/cuda/sepconv_parallel.py b/examples/cuda/sepconv_parallel.py new file mode 100644 index 000000000..074200e1b --- /dev/null +++ b/examples/cuda/sepconv_parallel.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python +import numpy +from kernel_tuner import tune_kernel +from collections import OrderedDict + + +def tune(): + with open("convolution.cu", "r") as f: + kernel_string = f.read() + + # setup tunable parameters + tune_params = OrderedDict() + tune_params["filter_height"] = [i for i in range(3, 19, 2)] + tune_params["filter_width"] = [i for i in range(3, 19, 2)] + tune_params["block_size_x"] = [16 * i for i in range(1, 65)] + tune_params["block_size_y"] = [2**i for i in range(6)] + tune_params["tile_size_x"] = [i for i in range(1, 11)] + tune_params["tile_size_y"] = [i for i in range(1, 11)] + + tune_params["use_padding"] = [0, 1] # toggle the insertion of padding in shared memory + tune_params["read_only"] = [0, 1] # toggle using the read-only cache + + # limit the search to only use padding when its effective, and at least 32 threads in a block + restrict = ["use_padding==0 or (block_size_x % 32 != 0)", "block_size_x*block_size_y >= 32"] + + # setup input and output dimensions + problem_size = (4096, 4096) + size = numpy.prod(problem_size) + largest_fh = max(tune_params["filter_height"]) + largest_fw = max(tune_params["filter_width"]) + input_size = (problem_size[0] + largest_fw - 1) * (problem_size[1] + largest_fh - 1) + + # create input data + output_image = numpy.zeros(size).astype(numpy.float32) + input_image = numpy.random.randn(input_size).astype(numpy.float32) + filter_weights = numpy.random.randn(largest_fh * largest_fw).astype(numpy.float32) + + # setup kernel arguments + cmem_args = {"d_filter": filter_weights} + args = [output_image, input_image, filter_weights] + + # tell the Kernel Tuner how to compute grid dimensions + grid_div_x = ["block_size_x", "tile_size_x"] + grid_div_y = ["block_size_y", "tile_size_y"] + + # start tuning separable convolution (row) + tune_params["filter_height"] = [1] + tune_params["tile_size_y"] = [1] + results_row = tune_kernel( + "convolution_kernel", + kernel_string, + problem_size, + args, + tune_params, + grid_div_y=grid_div_y, + grid_div_x=grid_div_x, + cmem_args=cmem_args, + verbose=False, + restrictions=restrict, + parallel_runner=1024, + cache="convolution_kernel_row", + ) + + # start tuning separable convolution (col) + tune_params["filter_height"] = tune_params["filter_width"][:] + tune_params["file_size_y"] = tune_params["tile_size_x"][:] + tune_params["filter_width"] = [1] + tune_params["tile_size_x"] = [1] + results_col = tune_kernel( + "convolution_kernel", + kernel_string, + problem_size, + args, + tune_params, + grid_div_y=grid_div_y, + grid_div_x=grid_div_x, + cmem_args=cmem_args, + verbose=False, + restrictions=restrict, + parallel_runner=1024, + cache="convolution_kernel_col", + ) + + return results_row, results_col + + +if __name__ == "__main__": + results_row, results_col = tune() diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index f454d0686..e689096f9 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -135,7 +135,10 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob self.state.last_strategy_start_time = self.state.start_time self.state.last_strategy_time = 0 self.state.kernel_options = kernel_options - # define a dummy device interface + # fields used directly by strategies + self.last_strategy_time = perf_counter() + self.state.last_strategy_start_time = self.last_strategy_time + # define a dummy device interface on the master node self.dev = DeviceInterface(kernel_source) def get_environment(self, tuning_options): From 426dd2a188ab22c5b8b5c8e784fb6a8be7bcae8e Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 19 Jan 2026 16:43:49 +0100 Subject: [PATCH 10/34] Rewrite parallel runner to use stateful actors --- examples/cuda/vector_add_parallel.py | 4 +- kernel_tuner/interface.py | 37 +-- kernel_tuner/runners/parallel.py | 342 +++++++++++++++++---------- kernel_tuner/runners/runner.py | 7 + kernel_tuner/runners/sequential.py | 3 + kernel_tuner/runners/simulation.py | 3 + 6 files changed, 248 insertions(+), 148 deletions(-) diff --git a/examples/cuda/vector_add_parallel.py b/examples/cuda/vector_add_parallel.py index d1c112aa5..8d35ce7c7 100644 --- a/examples/cuda/vector_add_parallel.py +++ b/examples/cuda/vector_add_parallel.py @@ -26,8 +26,8 @@ def tune(): tune_params = dict() tune_params["block_size_x"] = [32 * i for i in range(1, 33)] - results, env = tune_kernel("vector_add", kernel_string, size, args, tune_params, parallel_runner=4) - + results, env = tune_kernel("vector_add", kernel_string, size, args, tune_params, parallel_workers=True) + print(env) return results diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index 2faa96213..46974cd28 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -37,9 +37,6 @@ import kernel_tuner.util as util from kernel_tuner.file_utils import get_input_file, get_t4_metadata, get_t4_results from kernel_tuner.integration import get_objective_defaults -from kernel_tuner.runners.parallel import ParallelRunner -from kernel_tuner.runners.sequential import SequentialRunner -from kernel_tuner.runners.simulation import SimulationRunner from kernel_tuner.searchspace import Searchspace try: @@ -469,7 +466,7 @@ def __deepcopy__(self, _): ), ("metrics", ("specifies user-defined metrics, please see :ref:`metrics`.", "dict")), ("simulation_mode", ("Simulate an auto-tuning search from an existing cachefile", "bool")), - ("parallel_runner", ("If the value is larger than 1 use that number as the number of parallel runners doing the tuning", "int")), + ("parallel_workers", ("Set to `True` or an integer to enable parallel tuning. If set to an integer, this will be the number of parallel workers.", "int|bool")), ("observers", ("""A list of Observers to use during tuning, please see :ref:`observers`.""", "list")), ] ) @@ -581,7 +578,7 @@ def tune_kernel( cache=None, metrics=None, simulation_mode=False, - parallel_runner=1, + parallel_workers=None, observers=None, objective=None, objective_higher_is_better=None, @@ -608,8 +605,6 @@ def tune_kernel( if iterations < 1: raise ValueError("Iterations should be at least one!") - if parallel_runner < 1: - logging.warning("The number of parallel runners should be at least one!") # sort all the options into separate dicts opts = locals() @@ -669,15 +664,21 @@ def tune_kernel( # select the runner for this job based on input # TODO: we could use the "match case" syntax when removing support for 3.9 - if simulation_mode: - selected_runner = SimulationRunner - elif parallel_runner > 1: - selected_runner = ParallelRunner - tuning_options.parallel_runner = parallel_runner - else: - selected_runner = SequentialRunner tuning_options.simulated_time = 0 - runner = selected_runner(kernelsource, kernel_options, device_options, iterations, observers) + + if parallel_workers and simulation_mode: + raise ValueError("Enabling `parallel_workers` and `simulation_mode` together is not supported") + elif simulation_mode: + from kernel_tuner.runners.simulation import SimulationRunner + runner = SimulationRunner(kernelsource, kernel_options, device_options, iterations, observers) + elif parallel_workers: + from kernel_tuner.runners.parallel import ParallelRunner + num_workers = None if parallel_workers is True else parallel_workers + runner = ParallelRunner(kernelsource, kernel_options, device_options, iterations, observers, num_workers=num_workers) + else: + from kernel_tuner.runners.sequential import SequentialRunner + runner = SequentialRunner(kernelsource, kernel_options, device_options, iterations, observers) + # the user-specified function may or may not have an optional atol argument; # we normalize it so that it always accepts atol. @@ -696,7 +697,8 @@ def tune_kernel( tuning_options.cachefile = None # create search space - searchspace = Searchspace(tune_params, restrictions, runner.dev.max_threads, **searchspace_construction_options) + device_info = runner.get_device_info() + searchspace = Searchspace(tune_params, restrictions, device_info.max_threads, **searchspace_construction_options) restrictions = searchspace._modified_restrictions tuning_options.restrictions = restrictions if verbose: @@ -716,6 +718,9 @@ def tune_kernel( results = strategy.tune(searchspace, runner, tuning_options) env = runner.get_environment(tuning_options) + # Shut down the runner + runner.shutdown() + # finished iterating over search space if results: # checks if results is not empty best_config = util.get_best_config(results, objective, objective_higher_is_better) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index e689096f9..18d1efbad 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -1,84 +1,92 @@ """A specialized runner that tunes in parallel the parameter space.""" import logging +import socket from time import perf_counter +from kernel_tuner.interface import Options +from kernel_tuner.util import ErrorConfig, print_config, print_config_output, process_metrics, store_cache +from kernel_tuner.core import DeviceInterface +from kernel_tuner.runners.runner import Runner from datetime import datetime, timezone -from itertools import chain -from ray import remote, get, put +try: + import ray +except ImportError as e: + raise Exception(f"Unable to initialize the parallel runner: {e}") -from kernel_tuner.runners.runner import Runner -from kernel_tuner.core import DeviceInterface -from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache +@ray.remote(num_gpus=1) +class DeviceActor: + def __init__(self, kernel_source, kernel_options, device_options, iterations, observers): + # detect language and create high-level device interface + self.dev = DeviceInterface(kernel_source, iterations=iterations, observers=observers, **device_options) + + self.units = self.dev.units + self.quiet = device_options.quiet + self.kernel_source = kernel_source + self.warmed_up = False if self.dev.requires_warmup else True + self.start_time = perf_counter() + self.last_strategy_start_time = self.start_time + self.last_strategy_time = 0 + self.kernel_options = kernel_options -class ParallelRunnerState: - """This class represents the state of a parallel tuning run.""" + # move data to the GPU + self.gpu_args = self.dev.ready_argument_list(kernel_options.arguments) - def __init__(self, observers, iterations): - self.device_options = None - self.quiet = False - self.kernel_source = None - self.warmed_up = False - self.simulation_mode = False - self.start_time = None - self.last_strategy_start_time = None - self.last_strategy_time = 0 - self.kernel_options = None - self.observers = observers - self.iterations = iterations - - -@remote(num_cpus=1, num_gpus=1) -def parallel_run(task_id: int, state: ParallelRunnerState, parameter_space, tuning_options): - dev = DeviceInterface( - state.kernel_source, iterations=state.iterations, observers=state.observers, **state.device_options - ) - # move data to the GPU - gpu_args = dev.ready_argument_list(state.kernel_options.arguments) - # iterate over parameter space - results = [] - elements_per_task = int(len(parameter_space) / tuning_options.parallel_runner) - first_element = int(task_id * elements_per_task) - last_element = int( - (task_id + 1) * elements_per_task if task_id + 1 < tuning_options.parallel_runner else len(parameter_space) - ) - for element in parameter_space[first_element:last_element]: - params = dict(zip(tuning_options.tune_params.keys(), element)) + def shutdown(self): + ray.actor.exit_actor() + + def get_environment(self): + # Get the device properties + env = dict(self.dev.get_environment()) + + # Get the host name + env["host_name"] = socket.gethostname() + # Get info about the ray instance + ray_info = ray.get_runtime_context() + env["ray"] = dict( + node_id=ray_info.get_node_id(), + worker_id=ray_info.get_worker_id(), + actor_id=ray_info.get_actor_id(), + ) + + return env + + def run(self, element, tuning_options): + # TODO: logging.debug("sequential runner started for " + self.kernel_options.kernel_name) + objective = tuning_options.objective + metrics = tuning_options.metrics + tune_params = tuning_options.tune_params + + params = dict(element) result = None warmup_time = 0 - # check if configuration is in the cache - x_int = ",".join([str(i) for i in element]) - if tuning_options.cache and x_int in tuning_options.cache: - params.update(tuning_options.cache[x_int]) - params["compile_time"] = 0 - params["verification_time"] = 0 - params["benchmark_time"] = 0 - else: - # attempt to warm up the GPU by running the first config in the parameter space and ignoring the result - if not state.warmed_up: - warmup_time = perf_counter() - dev.compile_and_benchmark(state.kernel_source, gpu_args, params, state.kernel_options, tuning_options) - state.warmed_up = True - warmup_time = 1e3 * (perf_counter() - warmup_time) - - result = dev.compile_and_benchmark( - state.kernel_source, gpu_args, params, state.kernel_options, tuning_options + # attempt to warmup the GPU by running the first config in the parameter space and ignoring the result + if not self.warmed_up: + warmup_time = perf_counter() + self.dev.compile_and_benchmark( + self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options ) + self.warmed_up = True + warmup_time = 1e3 * (perf_counter() - warmup_time) - params.update(result) + result = self.dev.compile_and_benchmark( + self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options + ) - if tuning_options.objective in result and isinstance(result[tuning_options.objective], ErrorConfig): - logging.debug("kernel configuration was skipped silently due to compile or runtime failure") + if objective in result and isinstance(result[objective], ErrorConfig): + logging.debug("kernel configuration was skipped silently due to compile or runtime failure") + + params.update(result) # only compute metrics on configs that have not errored - if tuning_options.metrics and not isinstance(params.get(tuning_options.objective), ErrorConfig): - params = process_metrics(params, tuning_options.metrics) + if metrics and not isinstance(params.get(objective), ErrorConfig): + params = process_metrics(params, metrics) # get the framework time by estimating based on other times - total_time = 1000 * ((perf_counter() - state.start_time) - warmup_time) - params["strategy_time"] = state.last_strategy_time + total_time = 1000 * ((perf_counter() - self.start_time) - warmup_time) + params["strategy_time"] = self.last_strategy_time params["framework_time"] = max( total_time - ( @@ -89,85 +97,159 @@ def parallel_run(task_id: int, state: ParallelRunnerState, parameter_space, tuni ), 0, ) - params["timestamp"] = str(datetime.now(timezone.utc)) - state.start_time = perf_counter() - if result: - # print configuration to the console - print_config_output(tuning_options.tune_params, params, state.quiet, tuning_options.metrics, dev.units) + params["timestamp"] = str(datetime.now(timezone.utc)) + params["ray_actor_id"] = ray.get_runtime_context().get_actor_id() + params["host_name"] = socket.gethostname() - # add configuration to cache - store_cache(x_int, params, tuning_options) + self.start_time = perf_counter() # all visited configurations are added to results to provide a trace for optimization strategies - results.append(params) + return params + - return results +class DeviceActorState: + def __init__(self, actor): + self.actor = actor + self.running_jobs = [] + self.maximum_running_jobs = 1 + self.is_running = True + self.env = ray.get(actor.get_environment.remote()) + + def __repr__(self): + actor_id = self.env["ray"]["actor_id"] + host_name = self.env["host_name"] + return f"{actor_id} ({host_name})" + + def shutdown(self): + if self.is_running: + self.is_running = False + self.actor.shutdown.remote() + + def submit(self, *args): + job = self.actor.run.remote(*args) + self.running_jobs.append(job) + return job + + def is_available(self): + if not self.is_running: + return False + + # Check for ready jobs, but do not block + ready_jobs, self.running_jobs = ray.wait(self.running_jobs, timeout=0) + ray.get(ready_jobs) + + # Available if this actor can run another job + return len(self.running_jobs) < self.maximum_running_jobs class ParallelRunner(Runner): - """ParallelRunner is used to distribute configurations across multiple nodes.""" + def __init__(self, kernel_source, kernel_options, device_options, iterations, observers, num_workers=None): + if not ray.is_initialized(): + ray.init() + + if num_workers is None: + num_workers = int(ray.cluster_resources().get("GPU", 0)) + + if num_workers == 0: + raise Exception("failed to initialize parallel runner: no GPUs found") + + if num_workers < 1: + raise Exception(f"failed to initialize parallel runner: invalid number of GPUs specified: {num_workers}") + + self.workers = [] + + try: + for index in range(num_workers): + actor = DeviceActor.remote(kernel_source, kernel_options, device_options, iterations, observers) + worker = DeviceActorState(actor) + self.workers.append(worker) + + logging.info(f"launched worker {index}: {worker}") + except: + # If an exception occurs, shut down the worker + self.shutdown() + raise + + # Check if all workers have the same device + device_names = {w.env.get("device_name") for w in self.workers} + if len(device_names) != 1: + self.shutdown() + raise Exception( + f"failed to initialize parallel runner: workers have different devices: {sorted(device_names)}" + ) - def __init__(self, kernel_source, kernel_options, device_options, iterations, observers): - """Instantiate the ParallelRunner. - - :param kernel_source: The kernel source - :type kernel_source: kernel_tuner.core.KernelSource - - :param kernel_options: A dictionary with all options for the kernel. - :type kernel_options: kernel_tuner.interface.Options - - :param device_options: A dictionary with all options for the device - on which the kernel should be tuned. - :type device_options: kernel_tuner.interface.Options - - :param iterations: The number of iterations used for benchmarking - each kernel instance. - :type iterations: int - """ - self.state = ParallelRunnerState(observers, iterations) - self.state.device_options = device_options - self.state.quiet = device_options.quiet - self.state.kernel_source = kernel_source - self.state.warmed_up = False - self.state.simulation_mode = False - self.state.start_time = perf_counter() - self.state.last_strategy_start_time = self.state.start_time - self.state.last_strategy_time = 0 - self.state.kernel_options = kernel_options - # fields used directly by strategies - self.last_strategy_time = perf_counter() - self.state.last_strategy_start_time = self.last_strategy_time - # define a dummy device interface on the master node - self.dev = DeviceInterface(kernel_source) + self.device_name = device_names.pop() + + # TODO + self.units = "sec" + self.quiet = device_options.quiet + + def get_device_info(self): + return Options(dict(max_threads=1024)) def get_environment(self, tuning_options): - # dummy environment - return self.dev.get_environment() + return dict( + device_name=self.device_name, + workers=[w.env for w in self.workers], + ) + + def shutdown(self): + for worker in self.workers: + try: + worker.shutdown() + except Exception as err: + logging.warning(f"error while shutting down worker {worker}: {err}") + + def submit_job(self, *args): + while True: + # Find an idle actor + for i, worker in enumerate(list(self.workers)): + if worker.is_available(): + # push the worker to the end + self.workers.pop(i) + self.workers.append(worker) + + # Submit the work + return worker.submit(*args) + + # Gather all running jobs + running_jobs = [job for w in self.workers for job in w.running_jobs] + + # If there are no running jobs, then something must be wrong. + # Maybe a worker has crashed or gotten into an invalid state. + if not running_jobs: + raise Exception("invalid state: no Ray workers are available to run job") + + # Wait until any running job completes + ray.wait(running_jobs, num_returns=1) def run(self, parameter_space, tuning_options): - """Iterate through the entire parameter space using a single Python process. - - :param parameter_space: The parameter space as an iterable. - :type parameter_space: iterable - - :param tuning_options: A dictionary with all options regarding the tuning process. - :type tuning_options: kernel_tuner.interface.Options - - :returns: A list of dictionaries for executed kernel configurations and their execution times. - :rtype: dict() - """ - # given the parameter_space, distribute it over Ray tasks - logging.debug("parallel runner started for " + self.state.kernel_options.kernel_name) - - results = [] - tasks = [] - parameter_space_ref = put(parameter_space) - state_ref = put(self.state) - tuning_options_ref = put(tuning_options) - for task_id in range(0, tuning_options.parallel_runner): - tasks.append(parallel_run.remote(task_id, state_ref, parameter_space_ref, tuning_options_ref)) - for task in tasks: - results.append(get(task)) - - return list(chain.from_iterable(results)) + running_jobs = dict() + completed_jobs = dict() + + # Submit jobs which are not in the cache + for config in parameter_space: + params = dict(zip(tuning_options.tune_params.keys(), config)) + key = ",".join([str(i) for i in config]) + + if key in tuning_options.cache: + completed_jobs[key] = tuning_options.cache[key] + else: + assert key not in running_jobs + running_jobs[key] = self.submit_job(params, tuning_options) + completed_jobs[key] = None + + # Wait for the running jobs to finish + for key, job in running_jobs.items(): + result = ray.get(job) + completed_jobs[key] = result + + if result: + # print configuration to the console + print_config_output(tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units) + + # add configuration to cache + store_cache(key, result, tuning_options) + + return list(completed_jobs.values()) diff --git a/kernel_tuner/runners/runner.py b/kernel_tuner/runners/runner.py index 8c4de22d7..3a886ad16 100644 --- a/kernel_tuner/runners/runner.py +++ b/kernel_tuner/runners/runner.py @@ -13,6 +13,13 @@ def __init__( ): pass + def shutdown(self): + pass + + @abstractmethod + def get_device_info(self): + pass + @abstractmethod def get_environment(self, tuning_options): pass diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index 8228402dc..c7b865d44 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -42,6 +42,9 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob # move data to the GPU self.gpu_args = self.dev.ready_argument_list(kernel_options.arguments) + def get_device_info(self): + return self.dev + def get_environment(self, tuning_options): return self.dev.get_environment() diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index 7a167bfcf..f1d392c98 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -56,6 +56,9 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob self.last_strategy_time = 0 self.units = {} + def get_device_info(self): + return self.dev + def get_environment(self, tuning_options): env = self.dev.get_environment() env["simulation"] = True From f585d42b6d11f2a83755977018ae5ba63554a5ef Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 20 Jan 2026 12:00:06 +0100 Subject: [PATCH 11/34] Move `tuning_options` to constructor of `ParallelRunner` --- kernel_tuner/interface.py | 5 +-- kernel_tuner/runners/parallel.py | 43 +++++++++++------------ kernel_tuner/runners/sequential.py | 2 +- kernel_tuner/util.py | 56 ++++++++++++++---------------- test/test_util_functions.py | 2 +- 5 files changed, 53 insertions(+), 55 deletions(-) diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index 41331ebea..73c028c21 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -682,10 +682,11 @@ def preprocess_cache(filepath): # process cache if cache: cache = preprocess_cache(cache) - util.process_cache(cache, kernel_options, tuning_options, runner) + tuning_options.cachefile = cache + tuning_options.cache = util.process_cache(cache, kernel_options, tuning_options, runner) else: - tuning_options.cache = {} tuning_options.cachefile = None + tuning_options.cache = {} # create search space tuning_options.restrictions_unmodified = deepcopy(restrictions) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index 18d1efbad..f033f3857 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -16,7 +16,7 @@ @ray.remote(num_gpus=1) class DeviceActor: - def __init__(self, kernel_source, kernel_options, device_options, iterations, observers): + def __init__(self, kernel_source, kernel_options, device_options, tuning_options, iterations, observers): # detect language and create high-level device interface self.dev = DeviceInterface(kernel_source, iterations=iterations, observers=observers, **device_options) @@ -28,6 +28,7 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob self.last_strategy_start_time = self.start_time self.last_strategy_time = 0 self.kernel_options = kernel_options + self.tuning_options = tuning_options # move data to the GPU self.gpu_args = self.dev.ready_argument_list(kernel_options.arguments) @@ -52,11 +53,10 @@ def get_environment(self): return env - def run(self, element, tuning_options): + def run(self, element): # TODO: logging.debug("sequential runner started for " + self.kernel_options.kernel_name) - objective = tuning_options.objective - metrics = tuning_options.metrics - tune_params = tuning_options.tune_params + objective = self.tuning_options.objective + metrics = self.tuning_options.metrics params = dict(element) result = None @@ -66,16 +66,16 @@ def run(self, element, tuning_options): if not self.warmed_up: warmup_time = perf_counter() self.dev.compile_and_benchmark( - self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options + self.kernel_source, self.gpu_args, params, self.kernel_options, self.tuning_options ) self.warmed_up = True warmup_time = 1e3 * (perf_counter() - warmup_time) result = self.dev.compile_and_benchmark( - self.kernel_source, self.gpu_args, params, self.kernel_options, tuning_options + self.kernel_source, self.gpu_args, params, self.kernel_options, self.tuning_options ) - if objective in result and isinstance(result[objective], ErrorConfig): + if isinstance(result.get(objective), ErrorConfig): logging.debug("kernel configuration was skipped silently due to compile or runtime failure") params.update(result) @@ -139,12 +139,12 @@ def is_available(self): ready_jobs, self.running_jobs = ray.wait(self.running_jobs, timeout=0) ray.get(ready_jobs) - # Available if this actor can run another job + # Available if this actor can now run another job return len(self.running_jobs) < self.maximum_running_jobs class ParallelRunner(Runner): - def __init__(self, kernel_source, kernel_options, device_options, iterations, observers, num_workers=None): + def __init__(self, kernel_source, kernel_options, device_options, tuning_options, iterations, observers, num_workers=None): if not ray.is_initialized(): ray.init() @@ -152,16 +152,16 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob num_workers = int(ray.cluster_resources().get("GPU", 0)) if num_workers == 0: - raise Exception("failed to initialize parallel runner: no GPUs found") + raise RuntimeError("failed to initialize parallel runner: no GPUs found") if num_workers < 1: - raise Exception(f"failed to initialize parallel runner: invalid number of GPUs specified: {num_workers}") + raise RuntimeError(f"failed to initialize parallel runner: invalid number of GPUs specified: {num_workers}") self.workers = [] try: for index in range(num_workers): - actor = DeviceActor.remote(kernel_source, kernel_options, device_options, iterations, observers) + actor = DeviceActor.remote(kernel_source, kernel_options, device_options, tuning_options, iterations, observers) worker = DeviceActorState(actor) self.workers.append(worker) @@ -175,14 +175,14 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob device_names = {w.env.get("device_name") for w in self.workers} if len(device_names) != 1: self.shutdown() - raise Exception( + raise RuntimeError( f"failed to initialize parallel runner: workers have different devices: {sorted(device_names)}" ) self.device_name = device_names.pop() - # TODO - self.units = "sec" + # TODO: Get this from the device + self.units = {"time": "ms"} self.quiet = device_options.quiet def get_device_info(self): @@ -237,7 +237,7 @@ def run(self, parameter_space, tuning_options): completed_jobs[key] = tuning_options.cache[key] else: assert key not in running_jobs - running_jobs[key] = self.submit_job(params, tuning_options) + running_jobs[key] = self.submit_job(params) completed_jobs[key] = None # Wait for the running jobs to finish @@ -245,11 +245,10 @@ def run(self, parameter_space, tuning_options): result = ray.get(job) completed_jobs[key] = result - if result: - # print configuration to the console - print_config_output(tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units) + # print configuration to the console + print_config_output(tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units) - # add configuration to cache - store_cache(key, result, tuning_options) + # add configuration to cache + store_cache(key, result, tuning_options.cachefile, tuning_options.cache) return list(completed_jobs.values()) diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index c7b865d44..2bd554bfc 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -123,7 +123,7 @@ def run(self, parameter_space, tuning_options): print_config_output(tuning_options.tune_params, params, self.quiet, tuning_options.metrics, self.units) # add configuration to cache - store_cache(x_int, params, tuning_options) + store_cache(x_int, params, tuning_options.cachefile, tuning_options.cache) # all visited configurations are added to results to provide a trace for optimization strategies results.append(params) diff --git a/kernel_tuner/util.py b/kernel_tuner/util.py index 2d9e3f1b3..635c6de78 100644 --- a/kernel_tuner/util.py +++ b/kernel_tuner/util.py @@ -1152,7 +1152,7 @@ def check_matching_problem_size(cached_problem_size, problem_size): if cached_problem_size_arr.size != problem_size_arr.size or not (cached_problem_size_arr == problem_size_arr).all(): raise ValueError(f"Cannot load cache which contains results for different problem_size, cache: {cached_problem_size}, requested: {problem_size}") -def process_cache(cache, kernel_options, tuning_options, runner): +def process_cache(cachefile, kernel_options, tuning_options, runner): """Cache file for storing tuned configurations. the cache file is stored using JSON and uses the following format: @@ -1181,9 +1181,9 @@ def process_cache(cache, kernel_options, tuning_options, runner): raise ValueError("Caching only works correctly when tunable parameters are stored in a dictionary") # if file does not exist, create new cache - if not os.path.isfile(cache): + if not os.path.isfile(cachefile): if tuning_options.simulation_mode: - raise ValueError(f"Simulation mode requires an existing cachefile: file {cache} does not exist") + raise ValueError(f"Simulation mode requires an existing cachefile: file {cachefile} does not exist") c = dict() c["device_name"] = runner.dev.name @@ -1197,15 +1197,14 @@ def process_cache(cache, kernel_options, tuning_options, runner): contents = json.dumps(c, cls=NpEncoder, indent="")[:-3] # except the last "}\n}" # write the header to the cachefile - with open(cache, "w") as cachefile: - cachefile.write(contents) + with open(cachefile, "w") as f: + f.write(contents) - tuning_options.cachefile = cache - tuning_options.cache = {} + return {} # if file exists else: - cached_data = read_cache(cache, open_cache=not tuning_options.simulation_mode) + cached_data = read_cache(cachefile, open_cache=not tuning_options.simulation_mode) # if in simulation mode, use the device name from the cache file as the runner device name if runner.simulation_mode: @@ -1231,17 +1230,16 @@ def process_cache(cache, kernel_options, tuning_options, runner): ) raise ValueError( f"Cannot load cache which contains results obtained with different tunable parameters. \ - Cache at '{cache}' has: {cached_data['tune_params_keys']}, tuning_options has: {list(tuning_options.tune_params.keys())}" + Cache at '{cachefile}' has: {cached_data['tune_params_keys']}, tuning_options has: {list(tuning_options.tune_params.keys())}" ) - tuning_options.cachefile = cache - tuning_options.cache = cached_data["cache"] + return cached_data["cache"] -def correct_open_cache(cache, open_cache=True): +def correct_open_cache(cachefile, open_cache=True): """If cache file was not properly closed, pretend it was properly closed.""" - with open(cache, "r") as cachefile: - filestr = cachefile.read().strip() + with open(cachefile, "r") as f: + filestr = f.read().strip() # if file was not properly closed, pretend it was properly closed if len(filestr) > 0 and filestr[-3:] not in ["}\n}", "}}}"]: @@ -1253,15 +1251,15 @@ def correct_open_cache(cache, open_cache=True): else: if open_cache: # if it was properly closed, open it for appending new entries - with open(cache, "w") as cachefile: - cachefile.write(filestr[:-3] + ",") + with open(cachefile, "w") as f: + f.write(filestr[:-3] + ",") return filestr -def read_cache(cache, open_cache=True): +def read_cache(cachefile, open_cache=True): """Read the cachefile into a dictionary, if open_cache=True prepare the cachefile for appending.""" - filestr = correct_open_cache(cache, open_cache) + filestr = correct_open_cache(cachefile, open_cache) error_configs = { "InvalidConfig": InvalidConfig(), @@ -1279,25 +1277,25 @@ def read_cache(cache, open_cache=True): return cache_data -def close_cache(cache): - if not os.path.isfile(cache): +def close_cache(cachefile): + if not os.path.isfile(cachefile): raise ValueError("close_cache expects cache file to exist") - with open(cache, "r") as fh: + with open(cachefile, "r") as fh: contents = fh.read() # close to file to make sure it can be read by JSON parsers if contents[-1] == ",": - with open(cache, "w") as fh: + with open(cachefile, "w") as fh: fh.write(contents[:-1] + "}\n}") -def store_cache(key, params, tuning_options): +def store_cache(key, params, cachefile, cache): """Stores a new entry (key, params) to the cachefile.""" # logging.debug('store_cache called, cache=%s, cachefile=%s' % (tuning_options.cache, tuning_options.cachefile)) - if isinstance(tuning_options.cache, dict): - if key not in tuning_options.cache: - tuning_options.cache[key] = params + if isinstance(cache, dict): + if key not in cache: + cache[key] = params # Convert ErrorConfig objects to string, wanted to do this inside the JSONconverter but couldn't get it to work output_params = params.copy() @@ -1305,9 +1303,9 @@ def store_cache(key, params, tuning_options): if isinstance(v, ErrorConfig): output_params[k] = str(v) - if tuning_options.cachefile: - with open(tuning_options.cachefile, "a") as cachefile: - cachefile.write("\n" + json.dumps({key: output_params}, cls=NpEncoder)[1:-1] + ",") + if cachefile: + with open(cachefile, "a") as f: + f.write("\n" + json.dumps({key: output_params}, cls=NpEncoder)[1:-1] + ",") def dump_cache(obj: str, tuning_options): diff --git a/test/test_util_functions.py b/test/test_util_functions.py index 4a1858f37..ae50dac18 100644 --- a/test/test_util_functions.py +++ b/test/test_util_functions.py @@ -632,7 +632,7 @@ def assert_open_cachefile_is_correctly_parsed(cache): # store one entry in the cache params = {"x": 4, "time": np.float32(0.1234)} - store_cache("4", params, tuning_options) + store_cache("4", params, tuning_options.cachefile, tuning_options.cache) assert len(tuning_options.cache) == 1 # close the cache From ad55ba47cc57a646282f29b3c7ecaa01fb8c6cd9 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 20 Jan 2026 14:28:42 +0100 Subject: [PATCH 12/34] Fix several errors related to parallel runner --- kernel_tuner/interface.py | 2 +- kernel_tuner/runners/parallel.py | 60 ++++++++++++++++++-------------- test/test_util_functions.py | 14 ++++---- 3 files changed, 41 insertions(+), 35 deletions(-) diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index 73c028c21..053f71f29 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -662,7 +662,7 @@ def tune_kernel( elif parallel_workers: from kernel_tuner.runners.parallel import ParallelRunner num_workers = None if parallel_workers is True else parallel_workers - runner = ParallelRunner(kernelsource, kernel_options, device_options, iterations, observers, num_workers=num_workers) + runner = ParallelRunner(kernelsource, kernel_options, device_options, tuning_options, iterations, observers, num_workers=num_workers) else: from kernel_tuner.runners.sequential import SequentialRunner runner = SequentialRunner(kernelsource, kernel_options, device_options, iterations, observers) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index f033f3857..a05fc2fa5 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -2,16 +2,18 @@ import logging import socket from time import perf_counter -from kernel_tuner.interface import Options -from kernel_tuner.util import ErrorConfig, print_config, print_config_output, process_metrics, store_cache from kernel_tuner.core import DeviceInterface +from kernel_tuner.interface import Options from kernel_tuner.runners.runner import Runner +from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache from datetime import datetime, timezone +logger = logging.getLogger(__name__) + try: import ray except ImportError as e: - raise Exception(f"Unable to initialize the parallel runner: {e}") + raise ImportError(f"unable to initialize the parallel runner: {e}") from e @ray.remote(num_gpus=1) @@ -44,12 +46,12 @@ def get_environment(self): env["host_name"] = socket.gethostname() # Get info about the ray instance - ray_info = ray.get_runtime_context() - env["ray"] = dict( - node_id=ray_info.get_node_id(), - worker_id=ray_info.get_worker_id(), - actor_id=ray_info.get_actor_id(), - ) + ctx = ray.get_runtime_context() + env["ray"] = { + "node_id": ctx.get_node_id(), + "worker_id": ctx.get_worker_id(), + "actor_id": ctx.get_actor_id(), + } return env @@ -98,7 +100,7 @@ def run(self, element): 0, ) - params["timestamp"] = str(datetime.now(timezone.utc)) + params["timestamp"] = datetime.now(timezone.utc).isoformat() params["ray_actor_id"] = ray.get_runtime_context().get_actor_id() params["host_name"] = socket.gethostname() @@ -122,15 +124,22 @@ def __repr__(self): return f"{actor_id} ({host_name})" def shutdown(self): - if self.is_running: - self.is_running = False + if not self.is_running: + return + + self.is_running = False + + try: self.actor.shutdown.remote() + except Exception: + logger.exception("Failed to request actor shutdown: %s", self) - def submit(self, *args): - job = self.actor.run.remote(*args) + def submit(self, config): + logger.info(f"jobs submitted to worker {self}: {config}") + job = self.actor.run.remote(config) self.running_jobs.append(job) return job - + def is_available(self): if not self.is_running: return False @@ -165,7 +174,7 @@ def __init__(self, kernel_source, kernel_options, device_options, tuning_options worker = DeviceActorState(actor) self.workers.append(worker) - logging.info(f"launched worker {index}: {worker}") + logger.info(f"launched worker {index}: {worker}") except: # If an exception occurs, shut down the worker self.shutdown() @@ -186,31 +195,28 @@ def __init__(self, kernel_source, kernel_options, device_options, tuning_options self.quiet = device_options.quiet def get_device_info(self): - return Options(dict(max_threads=1024)) + return Options({"max_threads": 1024}) def get_environment(self, tuning_options): - return dict( - device_name=self.device_name, - workers=[w.env for w in self.workers], - ) + return { + "device_name": self.device_name, + "workers": [w.env for w in self.workers] + } def shutdown(self): for worker in self.workers: try: worker.shutdown() except Exception as err: - logging.warning(f"error while shutting down worker {worker}: {err}") + logger.exception("error while shutting down worker {worker}") def submit_job(self, *args): while True: - # Find an idle actor + # Round-robin: first available worker gets the job and goes to the back of the list for i, worker in enumerate(list(self.workers)): if worker.is_available(): - # push the worker to the end self.workers.pop(i) self.workers.append(worker) - - # Submit the work return worker.submit(*args) # Gather all running jobs @@ -219,7 +225,7 @@ def submit_job(self, *args): # If there are no running jobs, then something must be wrong. # Maybe a worker has crashed or gotten into an invalid state. if not running_jobs: - raise Exception("invalid state: no Ray workers are available to run job") + raise RuntimeError("invalid state: no Ray workers are available to run job") # Wait until any running job completes ray.wait(running_jobs, num_returns=1) diff --git a/test/test_util_functions.py b/test/test_util_functions.py index ae50dac18..56a5a7617 100644 --- a/test/test_util_functions.py +++ b/test/test_util_functions.py @@ -621,25 +621,25 @@ def assert_open_cachefile_is_correctly_parsed(cache): try: # call process_cache without pre-existing cache - process_cache(cache, kernel_options, tuning_options, runner) + tuning_options.cachefile = cache + tuning_options.cache = process_cache(cache, kernel_options, tuning_options, runner) # check if file has been created assert os.path.isfile(cache) assert_open_cachefile_is_correctly_parsed(cache) - assert tuning_options.cachefile == cache assert isinstance(tuning_options.cache, dict) assert len(tuning_options.cache) == 0 # store one entry in the cache params = {"x": 4, "time": np.float32(0.1234)} - store_cache("4", params, tuning_options.cachefile, tuning_options.cache) + store_cache("4", params, cache, tuning_options.cache) assert len(tuning_options.cache) == 1 # close the cache close_cache(cache) # now test process cache with a pre-existing cache file - process_cache(cache, kernel_options, tuning_options, runner) + tuning_options.cache = process_cache(cache, kernel_options, tuning_options, runner) assert_open_cachefile_is_correctly_parsed(cache) assert tuning_options.cache["4"]["time"] == params["time"] @@ -648,7 +648,7 @@ def assert_open_cachefile_is_correctly_parsed(cache): # a different kernel, device, or parameter set with pytest.raises(ValueError) as excep: kernel_options.kernel_name = "wrong_kernel" - process_cache(cache, kernel_options, tuning_options, runner) + tuning_options.cache = process_cache(cache, kernel_options, tuning_options, runner) assert "kernel" in str(excep.value) # correct the kernel name from last test @@ -656,7 +656,7 @@ def assert_open_cachefile_is_correctly_parsed(cache): with pytest.raises(ValueError) as excep: runner.dev.name = "wrong_device" - process_cache(cache, kernel_options, tuning_options, runner) + tuning_options.cache = process_cache(cache, kernel_options, tuning_options, runner) assert "device" in str(excep.value) # correct the device from last test @@ -664,7 +664,7 @@ def assert_open_cachefile_is_correctly_parsed(cache): with pytest.raises(ValueError) as excep: tuning_options.tune_params["y"] = ["a", "b"] - process_cache(cache, kernel_options, tuning_options, runner) + tuning_options.cache = process_cache(cache, kernel_options, tuning_options, runner) assert "parameter" in str(excep.value) finally: From 4d8f4f53c50a3e5c76299e20badd252f8b427d74 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 20 Jan 2026 17:33:59 +0100 Subject: [PATCH 13/34] Extend several strategies with support for parallel tuning: DiffEvo, FF, GA, PSO, all hillclimbers, random --- kernel_tuner/strategies/bayes_opt.py | 27 ++++++++++++++++ kernel_tuner/strategies/common.py | 6 ++++ kernel_tuner/strategies/diff_evo.py | 4 +-- kernel_tuner/strategies/firefly_algorithm.py | 13 ++++---- kernel_tuner/strategies/genetic_algorithm.py | 20 ++++++------ kernel_tuner/strategies/hillclimbers.py | 28 ++++++++++------- kernel_tuner/strategies/pso.py | 33 +++++++++++--------- kernel_tuner/strategies/random_sample.py | 13 +++----- 8 files changed, 92 insertions(+), 52 deletions(-) diff --git a/kernel_tuner/strategies/bayes_opt.py b/kernel_tuner/strategies/bayes_opt.py index a814e7ce2..31a68ca89 100644 --- a/kernel_tuner/strategies/bayes_opt.py +++ b/kernel_tuner/strategies/bayes_opt.py @@ -455,6 +455,33 @@ def fit_observations_to_model(self): """Update the model based on the current list of observations.""" self.__model.fit(self.__valid_params, self.__valid_observations) + def evaluate_parallel_objective_function(self, param_configs: list[tuple]) -> list[float]: + """Evaluates the objective function for multiple configurations in parallel.""" + results = [] + valid_param_configs = [] + valid_indices = [] + + # Extract the valid configurations + for param_config in param_configs: + param_config = self.unprune_param_config(param_config) + denormalized_param_config = self.denormalize_param_config(param_config) + if not self.__searchspace_obj.is_param_config_valid(denormalized_param_config): + results.append(self.invalid_value) + else: + valid_indices.append(len(results)) + results.append(None) + valid_param_configs.append(param_config) + + # Run valid configurations in parallel + scores = self.cost_func.run_all(valid_param_configs) + + # Put the scores at the right location in the result + for idx, score in zip(valid_indices, scores): + results[idx] = score + + self.fevals += len(scores) + return results + def evaluate_objective_function(self, param_config: tuple) -> float: """Evaluates the objective function.""" param_config = self.unprune_param_config(param_config) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 9ffe999b7..e567d9981 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -103,6 +103,12 @@ def __init__( def __call__(self, x, check_restrictions=True): + return self.run_one(x, check_restrictions=check_restrictions) + + def run_all(self, xs, check_restrictions=True): + return [self.run_one(x, check_restrictions=check_restrictions) for x in xs] + + def run_one(self, x, check_restrictions=True): """Cost function used by almost all strategies.""" self.runner.last_strategy_time = 1000 * (perf_counter() - self.runner.last_strategy_start_time) diff --git a/kernel_tuner/strategies/diff_evo.py b/kernel_tuner/strategies/diff_evo.py index d80b6e8e0..6350b7d9f 100644 --- a/kernel_tuner/strategies/diff_evo.py +++ b/kernel_tuner/strategies/diff_evo.py @@ -140,7 +140,7 @@ def differential_evolution(searchspace, cost_func, bounds, popsize, maxiter, F, population[0] = cost_func.get_start_pos() # Calculate the initial cost for each individual in the population - population_cost = np.array([cost_func(ind) for ind in population]) + population_cost = np.array(cost_func.run_all(population)) # Keep track of the best solution found so far best_idx = np.argmin(population_cost) @@ -209,7 +209,7 @@ def differential_evolution(searchspace, cost_func, bounds, popsize, maxiter, F, # --- c. Selection --- # Calculate the cost of the new trial vectors - trial_population_cost = np.array([cost_func(ind) for ind in trial_population]) + trial_population_cost = np.array(cost_func.run_all(trial_population)) # Keep track of whether population changes over time no_change = True diff --git a/kernel_tuner/strategies/firefly_algorithm.py b/kernel_tuner/strategies/firefly_algorithm.py index a732d4041..861c5f861 100644 --- a/kernel_tuner/strategies/firefly_algorithm.py +++ b/kernel_tuner/strategies/firefly_algorithm.py @@ -44,13 +44,14 @@ def tune(searchspace: Searchspace, runner, tuning_options): swarm[0].position = x0 # compute initial intensities - for j in range(num_particles): - try: + try: + for j in range(num_particles): swarm[j].compute_intensity(cost_func) - except StopCriterionReached as e: - if tuning_options.verbose: - print(e) - return cost_func.results + except StopCriterionReached as e: + if tuning_options.verbose: + print(e) + return cost_func.results + for j in range(num_particles): if swarm[j].score <= best_score_global: best_position_global = swarm[j].position best_score_global = swarm[j].score diff --git a/kernel_tuner/strategies/genetic_algorithm.py b/kernel_tuner/strategies/genetic_algorithm.py index 804758eef..230cfd49a 100644 --- a/kernel_tuner/strategies/genetic_algorithm.py +++ b/kernel_tuner/strategies/genetic_algorithm.py @@ -43,19 +43,17 @@ def tune(searchspace: Searchspace, runner, tuning_options): # determine fitness of population members weighted_population = [] - for dna in population: - try: - # if we are not constraint-aware we should check restrictions upon evaluation - time = cost_func(dna, check_restrictions=not constraint_aware) - num_evaluated += 1 - except StopCriterionReached as e: - if tuning_options.verbose: - print(e) - return cost_func.results - - weighted_population.append((dna, time)) + try: + # if we are not constraint-aware we should check restrictions upon evaluation + times = cost_func.run_all(population, check_restrictions=not constraint_aware) + num_evaluated += len(population) + except StopCriterionReached as e: + if tuning_options.verbose: + print(e) + return cost_func.results # population is sorted such that better configs have higher chance of reproducing + weighted_population = list(zip(population, times)) weighted_population.sort(key=lambda x: x[1]) # 'best_score' is used only for printing diff --git a/kernel_tuner/strategies/hillclimbers.py b/kernel_tuner/strategies/hillclimbers.py index ccd4eebf0..cc53d7db4 100644 --- a/kernel_tuner/strategies/hillclimbers.py +++ b/kernel_tuner/strategies/hillclimbers.py @@ -72,33 +72,39 @@ def base_hillclimb(base_sol: tuple, neighbor_method: str, max_fevals: int, searc if randomize: random.shuffle(indices) + children = [] + # in each dimension see the possible values for index in indices: neighbors = searchspace.get_param_neighbors(tuple(child), index, neighbor_method, randomize) # for each value in this dimension for val in neighbors: - orig_val = child[index] + child = list(child) child[index] = val + children.append(child) + if restart: + for child in children: # get score for this position score = cost_func(child, check_restrictions=False) - # generalize this to other tuning objectives if score < best_score: best_score = score base_sol = child[:] found_improved = True - if restart: - break - else: - child[index] = orig_val + break + else: + # get score for all positions in parallel + scores = cost_func.run_all(children, check_restrictions=False) - fevals = len(tuning_options.unique_results) - if fevals >= max_fevals: - return base_sol + for child, score in zip(children, scores): + if score < best_score: + best_score = score + base_sol = child[:] + found_improved = True - if found_improved and restart: - break + if found_improved and restart: + break return base_sol diff --git a/kernel_tuner/strategies/pso.py b/kernel_tuner/strategies/pso.py index e8489d12a..4e38aa311 100644 --- a/kernel_tuner/strategies/pso.py +++ b/kernel_tuner/strategies/pso.py @@ -51,24 +51,26 @@ def tune(searchspace: Searchspace, runner, tuning_options): if tuning_options.verbose: print("start iteration ", i, "best time global", best_score_global) + try: + scores = cost_func.run_all([p.position for p in swarm]) + except StopCriterionReached as e: + if tuning_options.verbose: + print(e) + return cost_func.results + # evaluate particle positions - for j in range(num_particles): - try: - swarm[j].evaluate(cost_func) - except StopCriterionReached as e: - if tuning_options.verbose: - print(e) - return cost_func.results + for p, score in zip(swarm, scores): + p.set_score(score) # update global best if needed - if swarm[j].score <= best_score_global: - best_position_global = swarm[j].position - best_score_global = swarm[j].score + if score <= best_score_global: + best_position_global = p.position + best_score_global = score # update particle velocities and positions - for j in range(0, num_particles): - swarm[j].update_velocity(best_position_global, w, c1, c2) - swarm[j].update_position(bounds) + for p in swarm: + p.update_velocity(best_position_global, w, c1, c2) + p.update_position(bounds) if tuning_options.verbose: print("Final result:") @@ -92,7 +94,10 @@ def __init__(self, bounds): self.score = sys.float_info.max def evaluate(self, cost_func): - self.score = cost_func(self.position) + self.set_score(cost_func(self.position)) + + def set_score(self, score): + self.score = score # update best_pos if needed if self.score < self.best_score: self.best_pos = self.position diff --git a/kernel_tuner/strategies/random_sample.py b/kernel_tuner/strategies/random_sample.py index 33b5075d3..4efe86151 100644 --- a/kernel_tuner/strategies/random_sample.py +++ b/kernel_tuner/strategies/random_sample.py @@ -20,16 +20,13 @@ def tune(searchspace: Searchspace, runner, tuning_options): num_samples = min(tuning_options.max_fevals, searchspace.size) samples = searchspace.get_random_sample(num_samples) - cost_func = CostFunc(searchspace, tuning_options, runner) - for sample in samples: - try: - cost_func(sample, check_restrictions=False) - except StopCriterionReached as e: - if tuning_options.verbose: - print(e) - return cost_func.results + try: + cost_func.run_all(samples, check_restrictions=False) + except StopCriterionReached as e: + if tuning_options.verbose: + print(e) return cost_func.results From fd41333745477b0d8a0b0b3496894c27ccaf3b74 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 27 Jan 2026 10:11:01 +0100 Subject: [PATCH 14/34] Add `pcu_bus_id` to environment for Nvidia backends --- kernel_tuner/backends/cupy.py | 1 + kernel_tuner/backends/nvcuda.py | 1 + kernel_tuner/backends/pycuda.py | 1 + 3 files changed, 3 insertions(+) diff --git a/kernel_tuner/backends/cupy.py b/kernel_tuner/backends/cupy.py index 51613be7c..87ba1514c 100644 --- a/kernel_tuner/backends/cupy.py +++ b/kernel_tuner/backends/cupy.py @@ -73,6 +73,7 @@ def __init__(self, device=0, iterations=7, compiler_options=None, observers=None s.split(":")[0].strip(): s.split(":")[1].strip() for s in cupy_info } env["device_name"] = info_dict[f"Device {device} Name"] + env["pci_bus_id"] = info_dict[f"Device {device} PCI Bus ID"] env["cuda_version"] = cp.cuda.runtime.driverGetVersion() env["compute_capability"] = self.cc diff --git a/kernel_tuner/backends/nvcuda.py b/kernel_tuner/backends/nvcuda.py index 15259cb23..ef5e0bdf1 100644 --- a/kernel_tuner/backends/nvcuda.py +++ b/kernel_tuner/backends/nvcuda.py @@ -99,6 +99,7 @@ def __init__(self, device=0, iterations=7, compiler_options=None, observers=None cuda_error_check(err) env = dict() env["device_name"] = device_properties.name.decode() + env["pci_bus_id"] = device_properties.pciBusID env["cuda_version"] = cuda.CUDA_VERSION env["compute_capability"] = self.cc env["iterations"] = self.iterations diff --git a/kernel_tuner/backends/pycuda.py b/kernel_tuner/backends/pycuda.py index c8f3e689a..8f9326c2d 100644 --- a/kernel_tuner/backends/pycuda.py +++ b/kernel_tuner/backends/pycuda.py @@ -139,6 +139,7 @@ def _finish_up(): # collect environment information env = dict() env["device_name"] = self.context.get_device().name() + env["pci_bus_id"] = self.context.get_device().pci_bus_id() env["cuda_version"] = ".".join([str(i) for i in drv.get_version()]) env["compute_capability"] = self.cc env["iterations"] = self.iterations From 96e168d53c2a374584ef7bc4d02517969e84cba3 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 27 Jan 2026 10:25:06 +0100 Subject: [PATCH 15/34] Add support `eval_all` in `CostFunc` --- kernel_tuner/strategies/bayes_opt.py | 2 +- kernel_tuner/strategies/common.py | 154 +++++++++++-------- kernel_tuner/strategies/diff_evo.py | 4 +- kernel_tuner/strategies/genetic_algorithm.py | 2 +- kernel_tuner/strategies/hillclimbers.py | 2 +- kernel_tuner/strategies/pso.py | 2 +- kernel_tuner/strategies/random_sample.py | 2 +- 7 files changed, 98 insertions(+), 70 deletions(-) diff --git a/kernel_tuner/strategies/bayes_opt.py b/kernel_tuner/strategies/bayes_opt.py index 31a68ca89..64d4c6234 100644 --- a/kernel_tuner/strategies/bayes_opt.py +++ b/kernel_tuner/strategies/bayes_opt.py @@ -473,7 +473,7 @@ def evaluate_parallel_objective_function(self, param_configs: list[tuple]) -> li valid_param_configs.append(param_config) # Run valid configurations in parallel - scores = self.cost_func.run_all(valid_param_configs) + scores = self.cost_func.eval_all(valid_param_configs) # Put the scores at the right location in the result for idx, score in zip(valid_indices, scores): diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index e567d9981..3da151733 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -91,6 +91,7 @@ def __init__( self.tuning_options["max_fevals"] = min( tuning_options["max_fevals"] if "max_fevals" in tuning_options else np.inf, searchspace.size ) + self.constraint_aware = tuning_options.strategy_options.get("constraint_aware") self.runner = runner self.scaling = scaling self.snap = snap @@ -100,90 +101,117 @@ def __init__( self.return_raw = f"{tuning_options['objective']}s" self.results = [] self.budget_spent_fraction = 0.0 + + def _normalize_and_validate_config(self, x, check_restrictions=True): + # snap values in x to nearest actual value for each parameter, unscale x if needed + if not self.snap: + if self.scaling: + config = unscale_and_snap_to_nearest(x, self.searchspace.tune_params, self.tuning_options.eps) + else: + config = snap_to_nearest_config(x, self.searchspace.tune_params) + else: + config = x + is_legal = True - def __call__(self, x, check_restrictions=True): - return self.run_one(x, check_restrictions=check_restrictions) - - def run_all(self, xs, check_restrictions=True): - return [self.run_one(x, check_restrictions=check_restrictions) for x in xs] - - def run_one(self, x, check_restrictions=True): - """Cost function used by almost all strategies.""" + # else check if this is a legal (non-restricted) configuration + if check_restrictions: + is_legal = self.searchspace.is_param_config_valid(tuple(config)) + + # Attempt to repare the config + if not is_legal and self.constraint_aware: + # attempt to repair + new_config = unscale_and_snap_to_nearest_valid(x, config, self.searchspace, self.tuning_options.eps) + + if new_config: + config = new_config + is_legal = True + + return config, is_legal + + def _run_configs(self, xs, check_restrictions=True): + """ Takes a list of Euclidian coordinates and evaluates the configurations at those points. """ self.runner.last_strategy_time = 1000 * (perf_counter() - self.runner.last_strategy_start_time) # error value to return for numeric optimizers that need a numerical value logging.debug("_cost_func called") - logging.debug("x: %s", str(x)) # check if max_fevals is reached or time limit is exceeded self.budget_spent_fraction = util.check_stop_criterion(self.tuning_options) - # snap values in x to nearest actual value for each parameter, unscale x if needed - if self.snap: - if self.scaling: - params = unscale_and_snap_to_nearest(x, self.searchspace.tune_params, self.tuning_options.eps) + batch_configs = [] # The configs to run + batch_indices = [] # Where to store result in `final_results`` + final_results = [] # List returned to the user + + for x in xs: + config, is_legal = self._normalize_and_validate_config(x, check_restrictions=check_restrictions) + logging.debug("normalize config: %s -> %s (legal: %s)", str(x), str(config), is_legal) + + if is_legal: + batch_configs.append(config) + batch_indices.append(len(final_results)) + final_results.append(None) else: - params = snap_to_nearest_config(x, self.searchspace.tune_params) - else: - params = x - logging.debug("params %s", str(params)) + result = dict(zip(self.searchspace.tune_params.keys(), config)) + result[self.tuning_options.objective] = util.InvalidConfig() + final_results.append(result) - legal = True - result = {} - x_int = ",".join([str(i) for i in params]) + # compile and benchmark the batch + batch_results = self.runner.run(batch_configs, self.tuning_options) + self.results.extend(batch_results) - # else check if this is a legal (non-restricted) configuration - if check_restrictions and self.searchspace.restrictions: - legal = self.searchspace.is_param_config_valid(tuple(params)) - - - if not legal: - if "constraint_aware" in self.tuning_options.strategy_options and self.tuning_options.strategy_options["constraint_aware"]: - # attempt to repair - new_params = unscale_and_snap_to_nearest_valid(x, params, self.searchspace, self.tuning_options.eps) - if new_params: - params = new_params - legal = True - x_int = ",".join([str(i) for i in params]) - - if not legal: - params_dict = dict(zip(self.searchspace.tune_params.keys(), params)) - result = params_dict - result[self.tuning_options.objective] = util.InvalidConfig() - - if legal: - # compile and benchmark this instance - res = self.runner.run([params], self.tuning_options) - result = res[0] - - # append to tuning results + # set in the results array + for index, result in zip(batch_indices, batch_results): + final_results[index] = result + + # append to `unique_results` + for config, result in zip(batch_configs, batch_results): + x_int = ",".join([str(i) for i in config]) if x_int not in self.tuning_options.unique_results: self.tuning_options.unique_results[x_int] = result - self.results.append(result) + # upon returning from this function control will be given back to the strategy, so reset the start time + self.runner.last_strategy_start_time = perf_counter() + return final_results + + def eval_all(self, xs, check_restrictions=True): + """Cost function used by almost all strategies.""" + results = self._run_configs(xs, check_restrictions=check_restrictions) + return_values = [] + return_raws = [] + + for result in results: + # get numerical return value, taking optimization direction into account + return_value = result[self.tuning_options.objective] + + if not isinstance(return_value, util.ErrorConfig): + # this is a valid configuration, so invert value in case of maximization + if self.tuning_options.objective_higher_is_better: + return_value = -return_value + else: + # this is not a valid configuration, replace with float max if needed + if not self.return_invalid: + return_value = sys.float_info.max + + # include raw data in return if requested + if self.return_raw is not None: + try: + return_raws.append(result[self.return_raw]) + except KeyError: + return_raws.append([np.nan]) - # upon returning from this function control will be given back to the strategy, so reset the start time - self.runner.last_strategy_start_time = perf_counter() + return_values.append(return_value) - # get numerical return value, taking optimization direction into account - return_value = result[self.tuning_options.objective] - if not isinstance(return_value, util.ErrorConfig): - # this is a valid configuration, so invert value in case of maximization - return_value = -return_value if self.tuning_options.objective_higher_is_better else return_value + if self.return_raw is not None: + return return_values, return_raws else: - # this is not a valid configuration, replace with float max if needed - if not self.return_invalid: - return_value = sys.float_info.max + return return_values - # include raw data in return if requested - if self.return_raw is not None: - try: - return return_value, result[self.return_raw] - except KeyError: - return return_value, [np.nan] + def eval(self, x, check_restrictions=True): + return self.eval_all([x], check_restrictions=check_restrictions)[0] - return return_value + def __call__(self, x, check_restrictions=True): + return self.eval(x, check_restrictions=check_restrictions) def get_start_pos(self): """Get starting position for optimization.""" diff --git a/kernel_tuner/strategies/diff_evo.py b/kernel_tuner/strategies/diff_evo.py index 6350b7d9f..9f7986e60 100644 --- a/kernel_tuner/strategies/diff_evo.py +++ b/kernel_tuner/strategies/diff_evo.py @@ -140,7 +140,7 @@ def differential_evolution(searchspace, cost_func, bounds, popsize, maxiter, F, population[0] = cost_func.get_start_pos() # Calculate the initial cost for each individual in the population - population_cost = np.array(cost_func.run_all(population)) + population_cost = np.array(cost_func.eval_all(population)) # Keep track of the best solution found so far best_idx = np.argmin(population_cost) @@ -209,7 +209,7 @@ def differential_evolution(searchspace, cost_func, bounds, popsize, maxiter, F, # --- c. Selection --- # Calculate the cost of the new trial vectors - trial_population_cost = np.array(cost_func.run_all(trial_population)) + trial_population_cost = np.array(cost_func.eval_all(trial_population)) # Keep track of whether population changes over time no_change = True diff --git a/kernel_tuner/strategies/genetic_algorithm.py b/kernel_tuner/strategies/genetic_algorithm.py index 230cfd49a..fa1f6cc98 100644 --- a/kernel_tuner/strategies/genetic_algorithm.py +++ b/kernel_tuner/strategies/genetic_algorithm.py @@ -45,7 +45,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): weighted_population = [] try: # if we are not constraint-aware we should check restrictions upon evaluation - times = cost_func.run_all(population, check_restrictions=not constraint_aware) + times = cost_func.eval_all(population, check_restrictions=not constraint_aware) num_evaluated += len(population) except StopCriterionReached as e: if tuning_options.verbose: diff --git a/kernel_tuner/strategies/hillclimbers.py b/kernel_tuner/strategies/hillclimbers.py index cc53d7db4..1218ed03e 100644 --- a/kernel_tuner/strategies/hillclimbers.py +++ b/kernel_tuner/strategies/hillclimbers.py @@ -96,7 +96,7 @@ def base_hillclimb(base_sol: tuple, neighbor_method: str, max_fevals: int, searc break else: # get score for all positions in parallel - scores = cost_func.run_all(children, check_restrictions=False) + scores = cost_func.eval_all(children, check_restrictions=False) for child, score in zip(children, scores): if score < best_score: diff --git a/kernel_tuner/strategies/pso.py b/kernel_tuner/strategies/pso.py index 4e38aa311..eefbc8661 100644 --- a/kernel_tuner/strategies/pso.py +++ b/kernel_tuner/strategies/pso.py @@ -52,7 +52,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): print("start iteration ", i, "best time global", best_score_global) try: - scores = cost_func.run_all([p.position for p in swarm]) + scores = cost_func.eval_all([p.position for p in swarm]) except StopCriterionReached as e: if tuning_options.verbose: print(e) diff --git a/kernel_tuner/strategies/random_sample.py b/kernel_tuner/strategies/random_sample.py index 4efe86151..194401491 100644 --- a/kernel_tuner/strategies/random_sample.py +++ b/kernel_tuner/strategies/random_sample.py @@ -23,7 +23,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): cost_func = CostFunc(searchspace, tuning_options, runner) try: - cost_func.run_all(samples, check_restrictions=False) + cost_func.eval_all(samples, check_restrictions=False) except StopCriterionReached as e: if tuning_options.verbose: print(e) From d7129cdd9b593c2685a2841693be00eb7989b434 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 27 Jan 2026 10:28:55 +0100 Subject: [PATCH 16/34] Remove `return_raw` from `CostFunc` as it is unused --- kernel_tuner/strategies/common.py | 27 +++++++-------------------- 1 file changed, 7 insertions(+), 20 deletions(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 3da151733..413e74877 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -72,7 +72,6 @@ def __init__( scaling=False, snap=True, return_invalid=False, - return_raw=None, ): """An abstract method to handle evaluation of configurations. @@ -83,7 +82,6 @@ def __init__( scaling: whether to internally scale parameter values. Defaults to False. snap: whether to snap given configurations to their closests equivalent in the space. Defaults to True. return_invalid: whether to return the util.ErrorConfig of an invalid configuration. Defaults to False. - return_raw: returns (result, results[raw]). Key inferred from objective if set to True. Defaults to None. """ self.searchspace = searchspace self.tuning_options = tuning_options @@ -91,14 +89,13 @@ def __init__( self.tuning_options["max_fevals"] = min( tuning_options["max_fevals"] if "max_fevals" in tuning_options else np.inf, searchspace.size ) - self.constraint_aware = tuning_options.strategy_options.get("constraint_aware") + self.objective = tuning_options.objective + self.objective_higher_is_better = tuning_options.objective_higher_is_better + self.constraint_aware = bool(tuning_options.strategy_options.get("constraint_aware")) self.runner = runner self.scaling = scaling self.snap = snap self.return_invalid = return_invalid - self.return_raw = return_raw - if return_raw is True: - self.return_raw = f"{tuning_options['objective']}s" self.results = [] self.budget_spent_fraction = 0.0 @@ -153,7 +150,7 @@ def _run_configs(self, xs, check_restrictions=True): final_results.append(None) else: result = dict(zip(self.searchspace.tune_params.keys(), config)) - result[self.tuning_options.objective] = util.InvalidConfig() + result[self.objective] = util.InvalidConfig() final_results.append(result) # compile and benchmark the batch @@ -178,15 +175,14 @@ def eval_all(self, xs, check_restrictions=True): """Cost function used by almost all strategies.""" results = self._run_configs(xs, check_restrictions=check_restrictions) return_values = [] - return_raws = [] for result in results: # get numerical return value, taking optimization direction into account - return_value = result[self.tuning_options.objective] + return_value = result[self.objective] if not isinstance(return_value, util.ErrorConfig): # this is a valid configuration, so invert value in case of maximization - if self.tuning_options.objective_higher_is_better: + if self.objective_higher_is_better: return_value = -return_value else: # this is not a valid configuration, replace with float max if needed @@ -194,18 +190,9 @@ def eval_all(self, xs, check_restrictions=True): return_value = sys.float_info.max # include raw data in return if requested - if self.return_raw is not None: - try: - return_raws.append(result[self.return_raw]) - except KeyError: - return_raws.append([np.nan]) - return_values.append(return_value) - if self.return_raw is not None: - return return_values, return_raws - else: - return return_values + return return_values def eval(self, x, check_restrictions=True): return self.eval_all([x], check_restrictions=check_restrictions)[0] From 57fd617297aed461f3ac0782a30f068ca9756219 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 27 Jan 2026 10:31:08 +0100 Subject: [PATCH 17/34] Fix timings and handling of duplicate jobs in parallel runner --- kernel_tuner/runners/parallel.py | 246 ++++++++++++++++++++----------- kernel_tuner/runners/runner.py | 4 + 2 files changed, 160 insertions(+), 90 deletions(-) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index a05fc2fa5..a14a30823 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -1,4 +1,5 @@ """A specialized runner that tunes in parallel the parameter space.""" +from collections import deque import logging import socket from time import perf_counter @@ -18,17 +19,18 @@ @ray.remote(num_gpus=1) class DeviceActor: - def __init__(self, kernel_source, kernel_options, device_options, tuning_options, iterations, observers): + def __init__( + self, kernel_source, kernel_options, device_options, tuning_options, iterations, observers + ): # detect language and create high-level device interface - self.dev = DeviceInterface(kernel_source, iterations=iterations, observers=observers, **device_options) + self.dev = DeviceInterface( + kernel_source, iterations=iterations, observers=observers, **device_options + ) self.units = self.dev.units self.quiet = device_options.quiet self.kernel_source = kernel_source self.warmed_up = False if self.dev.requires_warmup else True - self.start_time = perf_counter() - self.last_strategy_start_time = self.start_time - self.last_strategy_time = 0 self.kernel_options = kernel_options self.tuning_options = tuning_options @@ -55,11 +57,8 @@ def get_environment(self): return env - def run(self, element): + def run(self, key, element): # TODO: logging.debug("sequential runner started for " + self.kernel_options.kernel_name) - objective = self.tuning_options.objective - metrics = self.tuning_options.metrics - params = dict(element) result = None warmup_time = 0 @@ -77,41 +76,19 @@ def run(self, element): self.kernel_source, self.gpu_args, params, self.kernel_options, self.tuning_options ) - if isinstance(result.get(objective), ErrorConfig): - logging.debug("kernel configuration was skipped silently due to compile or runtime failure") - params.update(result) - # only compute metrics on configs that have not errored - if metrics and not isinstance(params.get(objective), ErrorConfig): - params = process_metrics(params, metrics) - - # get the framework time by estimating based on other times - total_time = 1000 * ((perf_counter() - self.start_time) - warmup_time) - params["strategy_time"] = self.last_strategy_time - params["framework_time"] = max( - total_time - - ( - params["compile_time"] - + params["verification_time"] - + params["benchmark_time"] - + params["strategy_time"] - ), - 0, - ) - params["timestamp"] = datetime.now(timezone.utc).isoformat() params["ray_actor_id"] = ray.get_runtime_context().get_actor_id() params["host_name"] = socket.gethostname() - self.start_time = perf_counter() - # all visited configurations are added to results to provide a trace for optimization strategies - return params + return key, params class DeviceActorState: - def __init__(self, actor): + def __init__(self, index, actor): + self.index = index self.actor = actor self.running_jobs = [] self.maximum_running_jobs = 1 @@ -121,7 +98,7 @@ def __init__(self, actor): def __repr__(self): actor_id = self.env["ray"]["actor_id"] host_name = self.env["host_name"] - return f"{actor_id} ({host_name})" + return f"{self.index} ({host_name}, {actor_id})" def shutdown(self): if not self.is_running: @@ -134,26 +111,41 @@ def shutdown(self): except Exception: logger.exception("Failed to request actor shutdown: %s", self) - def submit(self, config): - logger.info(f"jobs submitted to worker {self}: {config}") - job = self.actor.run.remote(config) + def submit(self, key, config): + logger.info(f"job submitted to worker {self}: {key}") + job = self.actor.run.remote(key, config) self.running_jobs.append(job) return job - + def is_available(self): if not self.is_running: return False # Check for ready jobs, but do not block ready_jobs, self.running_jobs = ray.wait(self.running_jobs, timeout=0) - ray.get(ready_jobs) + + for job in ready_jobs: + try: + key, _result = ray.get(job) + logger.info(f"job finished on worker {self}: {key}") + except Exception: + logger.exception(f"job failed on worker {self}") # Available if this actor can now run another job return len(self.running_jobs) < self.maximum_running_jobs class ParallelRunner(Runner): - def __init__(self, kernel_source, kernel_options, device_options, tuning_options, iterations, observers, num_workers=None): + def __init__( + self, + kernel_source, + kernel_options, + device_options, + tuning_options, + iterations, + observers, + num_workers=None, + ): if not ray.is_initialized(): ray.init() @@ -164,97 +156,171 @@ def __init__(self, kernel_source, kernel_options, device_options, tuning_options raise RuntimeError("failed to initialize parallel runner: no GPUs found") if num_workers < 1: - raise RuntimeError(f"failed to initialize parallel runner: invalid number of GPUs specified: {num_workers}") + raise RuntimeError( + f"failed to initialize parallel runner: invalid number of GPUs specified: {num_workers}" + ) self.workers = [] try: + # Start workers for index in range(num_workers): - actor = DeviceActor.remote(kernel_source, kernel_options, device_options, tuning_options, iterations, observers) - worker = DeviceActorState(actor) + actor = DeviceActor.remote( + kernel_source, + kernel_options, + device_options, + tuning_options, + iterations, + observers, + ) + worker = DeviceActorState(index, actor) self.workers.append(worker) - logger.info(f"launched worker {index}: {worker}") + logger.info(f"connected to worker {worker}") + + # Check if all workers have the same device + device_names = {w.env.get("device_name") for w in self.workers} + if len(device_names) != 1: + raise RuntimeError( + f"failed to initialize parallel runner: workers have different devices: {sorted(device_names)}" + ) except: - # If an exception occurs, shut down the worker + # If an exception occurs, shut down the worker and reraise error self.shutdown() raise - # Check if all workers have the same device - device_names = {w.env.get("device_name") for w in self.workers} - if len(device_names) != 1: - self.shutdown() - raise RuntimeError( - f"failed to initialize parallel runner: workers have different devices: {sorted(device_names)}" - ) - self.device_name = device_names.pop() - # TODO: Get this from the device + # TODO: Get units from the device? + self.start_time = perf_counter() self.units = {"time": "ms"} self.quiet = device_options.quiet def get_device_info(self): + # TODO: Get this from the device? return Options({"max_threads": 1024}) def get_environment(self, tuning_options): - return { - "device_name": self.device_name, - "workers": [w.env for w in self.workers] - } + return {"device_name": self.device_name, "workers": [w.env for w in self.workers]} def shutdown(self): for worker in self.workers: try: worker.shutdown() except Exception as err: - logger.exception("error while shutting down worker {worker}") + logger.exception(f"error while shutting down worker {worker}") - def submit_job(self, *args): - while True: - # Round-robin: first available worker gets the job and goes to the back of the list - for i, worker in enumerate(list(self.workers)): - if worker.is_available(): - self.workers.pop(i) - self.workers.append(worker) - return worker.submit(*args) + def available_parallelism(self): + return len(self.workers) - # Gather all running jobs - running_jobs = [job for w in self.workers for job in w.running_jobs] + def submit_jobs(self, jobs): + pending_jobs = deque(jobs) + running_jobs = [] - # If there are no running jobs, then something must be wrong. - # Maybe a worker has crashed or gotten into an invalid state. - if not running_jobs: - raise RuntimeError("invalid state: no Ray workers are available to run job") + while pending_jobs or running_jobs: + should_wait = True - # Wait until any running job completes - ray.wait(running_jobs, num_returns=1) + # If there is still work left, submit it now + if pending_jobs: + for i, worker in enumerate(list(self.workers)): + if worker.is_available(): + # Push worker to back of list + self.workers.pop(i) + self.workers.append(worker) + + # Pop job and submit it + job = pending_jobs.popleft() + ref = worker.submit(*job) + running_jobs.append(ref) + + should_wait = False + break + + # If no work was submitted, wait until a worker is available + if should_wait: + if not running_jobs: + raise RuntimeError("invalid state: no ray workers available") + + ready_jobs, running_jobs = ray.wait(running_jobs, num_returns=1) + + for result in ready_jobs: + yield ray.get(result) def run(self, parameter_space, tuning_options): - running_jobs = dict() - completed_jobs = dict() + metrics = tuning_options.metrics + objective = tuning_options.objective - # Submit jobs which are not in the cache - for config in parameter_space: + jobs = [] # Jobs that need to be executed + results = [] # Results that will be returned at the end + key2index = dict() # Used to insert job result back into `results` + duplicate_entries = [] # Used for duplicate entries in `parameter_space` + + # Select jobs which are not in the cache + for index, config in enumerate(parameter_space): params = dict(zip(tuning_options.tune_params.keys(), config)) key = ",".join([str(i) for i in config]) if key in tuning_options.cache: - completed_jobs[key] = tuning_options.cache[key] + params.update(tuning_options.cache[key]) + params["compile_time"] = 0 + params["verification_time"] = 0 + params["benchmark_time"] = 0 + results.append(params) else: - assert key not in running_jobs - running_jobs[key] = self.submit_job(params) - completed_jobs[key] = None + if key not in key2index: + key2index[key] = index + else: + duplicate_entries.append((key2index[key], index)) + + jobs.append((key, params)) + results.append(None) + + total_worker_time = 0 + + # Submit jobs and wait for them to finish + for key, result in self.submit_jobs(jobs): + results[key2index[key]] = result + + # Collect total time spent by worker + total_worker_time += ( + params["compile_time"] + params["verification_time"] + params["benchmark_time"] + ) - # Wait for the running jobs to finish - for key, job in running_jobs.items(): - result = ray.get(job) - completed_jobs[key] = result + if isinstance(result.get(objective), ErrorConfig): + logging.error( + "kernel configuration {key} was skipped silently due to compile or runtime failure", + key, + ) # print configuration to the console - print_config_output(tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units) + print_config_output( + tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units + ) # add configuration to cache store_cache(key, result, tuning_options.cachefile, tuning_options.cache) - return list(completed_jobs.values()) + # Copy each `i` to `j` for every `i,j` in `duplicate_entries` + for i, j in duplicate_entries: + results[j] = dict(results[i]) + + total_time = 1000 * (perf_counter() - self.start_time) + self.start_time = perf_counter() + + strategy_time = self.last_strategy_time + self.last_strategy_time = 0 + + runner_time = total_time - strategy_time + framework_time = max(runner_time * len(self.workers) - total_worker_time, 0) + + # Post-process all the results + for params in results: + # Amortize the time over all the results + params["strategy_time"] = strategy_time / len(results) + params["framework_time"] = framework_time / len(results) + + # only compute metrics on configs that have not errored + if metrics and not isinstance(params.get(objective), ErrorConfig): + params = process_metrics(params, metrics) + + return results diff --git a/kernel_tuner/runners/runner.py b/kernel_tuner/runners/runner.py index 3a886ad16..e95b7811c 100644 --- a/kernel_tuner/runners/runner.py +++ b/kernel_tuner/runners/runner.py @@ -16,6 +16,10 @@ def __init__( def shutdown(self): pass + def available_parallelism(self): + """ Gives an indication of how many jobs this runner can execute in parallel. """ + return 1 + @abstractmethod def get_device_info(self): pass From e1259b1a2cd943953636bb56d156e70c8560d95a Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Thu, 29 Jan 2026 16:06:31 +0000 Subject: [PATCH 18/34] fix bug for continuous optimization --- kernel_tuner/strategies/common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 413e74877..898835f79 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -101,7 +101,7 @@ def __init__( def _normalize_and_validate_config(self, x, check_restrictions=True): # snap values in x to nearest actual value for each parameter, unscale x if needed - if not self.snap: + if self.snap: if self.scaling: config = unscale_and_snap_to_nearest(x, self.searchspace.tune_params, self.tuning_options.eps) else: From 72bfe945f3be2d1f6c52ee4a799903300270a848 Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Thu, 29 Jan 2026 16:07:29 +0000 Subject: [PATCH 19/34] fix test_time_keeping test ensuring at least two GA generations --- test/test_runners.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_runners.py b/test/test_runners.py index 3a0a26e22..de15352b4 100644 --- a/test/test_runners.py +++ b/test/test_runners.py @@ -163,8 +163,8 @@ def test_time_keeping(env): answer = [args[1] + args[2], None, None, None] options = dict(method="uniform", - popsize=10, - maxiter=1, + popsize=5, + maxiter=50, mutation_chance=1, max_fevals=10) start = time.perf_counter() From cc6bb9781eae5db28ce8942de034c39b6b30a6cc Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Thu, 29 Jan 2026 16:08:21 +0000 Subject: [PATCH 20/34] fix tests needing more context for tuning_options --- test/strategies/test_bayesian_optimization.py | 2 ++ test/test_common.py | 18 ++++++++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/test/strategies/test_bayesian_optimization.py b/test/strategies/test_bayesian_optimization.py index f8c889aab..5c7bab322 100644 --- a/test/strategies/test_bayesian_optimization.py +++ b/test/strategies/test_bayesian_optimization.py @@ -19,6 +19,8 @@ strategy_options = dict(popsize=0, max_fevals=10) tuning_options = Options(dict(restrictions=[], tune_params=tune_params, strategy_options=strategy_options)) tuning_options["scaling"] = True +tuning_options["objective"] = "time" +tuning_options["objective_higher_is_better"] = False tuning_options["snap"] = True max_threads = 1024 searchspace = Searchspace(tune_params, [], max_threads) diff --git a/test/test_common.py b/test/test_common.py index 7c1bd6838..40d25de81 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -1,20 +1,26 @@ import random import numpy as np +import pytest import kernel_tuner.strategies.common as common from kernel_tuner.interface import Options from kernel_tuner.searchspace import Searchspace +@pytest.fixture +def tuning_options(): + tuning_options = Options() + tuning_options["strategy_options"] = {} + tuning_options["objective"] = "time" + tuning_options["objective_higher_is_better"] = False + return tuning_options + -def test_get_bounds_x0_eps(): +def test_get_bounds_x0_eps(tuning_options): tune_params = dict() tune_params['x'] = [0, 1, 2, 3, 4] searchspace = Searchspace(tune_params, [], 1024) - tuning_options = Options() - tuning_options["strategy_options"] = {} - bounds, x0, eps = common.CostFunc(searchspace, tuning_options, None, scaling=True).get_bounds_x0_eps() assert bounds == [(0.0, 1.0)] @@ -27,7 +33,7 @@ def test_get_bounds_x0_eps(): assert eps == 1.0 -def test_get_bounds(): +def test_get_bounds(tuning_options): tune_params = dict() tune_params['x'] = [0, 1, 2, 3, 4] @@ -39,7 +45,7 @@ def test_get_bounds(): expected = [(0, 4), (0, 9900), (-11.2, 123.27)] searchspace = Searchspace(tune_params, None, 1024) - cost_func = common.CostFunc(searchspace, None, None) + cost_func = common.CostFunc(searchspace, tuning_options, None) answer = cost_func.get_bounds() assert answer == expected From ce8123a04129e6052103f1cdb0362c2aef106839 Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Thu, 29 Jan 2026 18:36:27 +0000 Subject: [PATCH 21/34] do not count invalid for unique_results and avoid overshooting budget --- kernel_tuner/strategies/common.py | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 898835f79..77838828d 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -139,6 +139,7 @@ def _run_configs(self, xs, check_restrictions=True): batch_configs = [] # The configs to run batch_indices = [] # Where to store result in `final_results`` final_results = [] # List returned to the user + benchmark_config = [] for x in xs: config, is_legal = self._normalize_and_validate_config(x, check_restrictions=check_restrictions) @@ -148,10 +149,24 @@ def _run_configs(self, xs, check_restrictions=True): batch_configs.append(config) batch_indices.append(len(final_results)) final_results.append(None) + x_int = ",".join([str(i) for i in config]) + benchmark_config.append(x_int not in self.tuning_options.unique_results) else: result = dict(zip(self.searchspace.tune_params.keys(), config)) result[self.objective] = util.InvalidConfig() final_results.append(result) + benchmark_config.append(False) + + # do not overshoot max_fevals if we can avoid it + if "max_fevals" in self.tuning_options: + budget = self.tuning_options.max_fevals - len(self.tuning_options.unique_results) + if sum(benchmark_config) > budget: + # find index 'budget'th True value (+1 for including this last index) + last_index = benchmark_config.index(True, budget)+1 + # mask configs we cannot benchmark + batch_configs = batch_configs[:last_index] + batch_indices = batch_indices[:last_index] + final_results = final_results[:last_index] # compile and benchmark the batch batch_results = self.runner.run(batch_configs, self.tuning_options) @@ -162,10 +177,14 @@ def _run_configs(self, xs, check_restrictions=True): final_results[index] = result # append to `unique_results` - for config, result in zip(batch_configs, batch_results): - x_int = ",".join([str(i) for i in config]) - if x_int not in self.tuning_options.unique_results: - self.tuning_options.unique_results[x_int] = result + for config, result, benchmarked in zip(batch_configs, batch_results, benchmark_config): + if benchmarked: + x_int = ",".join([str(i) for i in config]) + if x_int not in self.tuning_options.unique_results: + self.tuning_options.unique_results[x_int] = result + + # check again for stop condition + self.budget_spent_fraction = util.check_stop_criterion(self.tuning_options) # upon returning from this function control will be given back to the strategy, so reset the start time self.runner.last_strategy_start_time = perf_counter() From f6c63bff630b61d1f12dcb032b3b8045ea06de46 Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Thu, 29 Jan 2026 22:23:15 +0000 Subject: [PATCH 22/34] fix for not overshooting/undershooting budget --- kernel_tuner/strategies/common.py | 15 ++++++++++----- test/strategies/test_strategies.py | 2 +- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 77838828d..83793c256 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -123,9 +123,10 @@ def _normalize_and_validate_config(self, x, check_restrictions=True): if new_config: config = new_config is_legal = True - + return config, is_legal + def _run_configs(self, xs, check_restrictions=True): """ Takes a list of Euclidian coordinates and evaluates the configurations at those points. """ self.runner.last_strategy_time = 1000 * (perf_counter() - self.runner.last_strategy_start_time) @@ -137,7 +138,7 @@ def _run_configs(self, xs, check_restrictions=True): self.budget_spent_fraction = util.check_stop_criterion(self.tuning_options) batch_configs = [] # The configs to run - batch_indices = [] # Where to store result in `final_results`` + batch_indices = [] # Where to store result in `final_results`` final_results = [] # List returned to the user benchmark_config = [] @@ -155,14 +156,13 @@ def _run_configs(self, xs, check_restrictions=True): result = dict(zip(self.searchspace.tune_params.keys(), config)) result[self.objective] = util.InvalidConfig() final_results.append(result) - benchmark_config.append(False) # do not overshoot max_fevals if we can avoid it if "max_fevals" in self.tuning_options: budget = self.tuning_options.max_fevals - len(self.tuning_options.unique_results) if sum(benchmark_config) > budget: - # find index 'budget'th True value (+1 for including this last index) - last_index = benchmark_config.index(True, budget)+1 + # find index 'budget'th True value + last_index = _get_nth_true(benchmark_config, budget)+1 # mask configs we cannot benchmark batch_configs = batch_configs[:last_index] batch_indices = batch_indices[:last_index] @@ -272,6 +272,11 @@ def get_bounds(self): return bounds +def _get_nth_true(lst, n): + # Returns the index of the nth True value in a list + return [i for i, x in enumerate(lst) if x][n-1] + + def setup_method_arguments(method, bounds): """Prepare method specific arguments.""" kwargs = {} diff --git a/test/strategies/test_strategies.py b/test/strategies/test_strategies.py index ea5a2994d..8f77a9516 100644 --- a/test/strategies/test_strategies.py +++ b/test/strategies/test_strategies.py @@ -96,7 +96,7 @@ def test_strategies(vector_add, strategy): tune_params = vector_add[-1] unique_results = {} for result in results: - x_int = ",".join([str(v) for k, v in result.items() if k in tune_params]) + x_int = ",".join([str(v) for k, v in result.items() if k in tune_params.keys()]) if not isinstance(result["time"], InvalidConfig): unique_results[x_int] = result["time"] assert len(unique_results) <= filter_options["max_fevals"] From ce7e330d2926e850678d1d869f25599744f9a697 Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Thu, 29 Jan 2026 22:24:27 +0000 Subject: [PATCH 23/34] fix time accounting when using batched costfunc --- kernel_tuner/runners/simulation.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index b369b85a0..f09772635 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -117,6 +117,10 @@ def run(self, parameter_space, tuning_options): # self.last_strategy_time is set by cost_func result["strategy_time"] = self.last_strategy_time + # Reset last strategy time to avoid counting it more than once in the + # framework time when strategy requests multiple configs at once + self.last_strategy_time = 0 + try: simulated_time = result["compile_time"] + result["verification_time"] + result["benchmark_time"] tuning_options.simulated_time += simulated_time @@ -128,7 +132,7 @@ def run(self, parameter_space, tuning_options): total_time = 1000 * (perf_counter() - self.start_time) self.start_time = perf_counter() - result["framework_time"] = total_time - self.last_strategy_time + result["framework_time"] = total_time - result["strategy_time"] results.append(result) continue From faf6cdcc19dbfad6733d09dd63b078e0ec7c6a3c Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Fri, 30 Jan 2026 08:10:33 +0000 Subject: [PATCH 24/34] fix timing issues --- kernel_tuner/runners/sequential.py | 11 ++++++++--- kernel_tuner/runners/simulation.py | 21 +++++++++++---------- kernel_tuner/strategies/common.py | 3 ++- kernel_tuner/util.py | 12 +++++++++++- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index 2bd554bfc..7e9d76934 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -5,7 +5,7 @@ from kernel_tuner.core import DeviceInterface from kernel_tuner.runners.runner import Runner -from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache +from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache, stop_criterion_reached class SequentialRunner(Runner): @@ -65,10 +65,16 @@ def run(self, parameter_space, tuning_options): results = [] + # self.last_strategy_time is set by cost_func + strategy_time_per_config = self.last_strategy_time / len(parameter_space) if len(parameter_space) > 0 else 0 + # iterate over parameter space for element in parameter_space: params = dict(zip(tuning_options.tune_params.keys(), element)) + if stop_criterion_reached(tuning_options): + return results + result = None warmup_time = 0 @@ -104,14 +110,13 @@ def run(self, parameter_space, tuning_options): # get the framework time by estimating based on other times total_time = 1000 * ((perf_counter() - self.start_time) - warmup_time) - params["strategy_time"] = self.last_strategy_time + params["strategy_time"] = strategy_time_per_config params["framework_time"] = max( total_time - ( params["compile_time"] + params["verification_time"] + params["benchmark_time"] - + params["strategy_time"] ), 0, ) diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index f09772635..986bf9b25 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -83,8 +83,15 @@ def run(self, parameter_space, tuning_options): results = [] + # self.last_strategy_time is set by cost_func + strategy_time_per_config = self.last_strategy_time / len(parameter_space) if len(parameter_space) > 0 else 0 + # iterate over parameter space for element in parameter_space: + + if util.stop_criterion_reached(tuning_options): + return results + # check if element is in the cache x_int = ",".join([str(i) for i in element]) if tuning_options.cache and x_int in tuning_options.cache: @@ -94,7 +101,6 @@ def run(self, parameter_space, tuning_options): if tuning_options.metrics and not isinstance(result.get(tuning_options.objective), util.ErrorConfig): result = util.process_metrics(result, tuning_options.metrics) - # Simulate behavior of sequential runner that when a configuration is # served from the cache by the sequential runner, the compile_time, # verification_time, and benchmark_time are set to 0. @@ -114,12 +120,7 @@ def run(self, parameter_space, tuning_options): ) # Everything but the strategy time and framework time are simulated, - # self.last_strategy_time is set by cost_func - result["strategy_time"] = self.last_strategy_time - - # Reset last strategy time to avoid counting it more than once in the - # framework time when strategy requests multiple configs at once - self.last_strategy_time = 0 + result["strategy_time"] = strategy_time_per_config try: simulated_time = result["compile_time"] + result["verification_time"] + result["benchmark_time"] @@ -132,7 +133,7 @@ def run(self, parameter_space, tuning_options): total_time = 1000 * (perf_counter() - self.start_time) self.start_time = perf_counter() - result["framework_time"] = total_time - result["strategy_time"] + result["framework_time"] = total_time results.append(result) continue @@ -145,11 +146,11 @@ def run(self, parameter_space, tuning_options): result['compile_time'] = 0 result['verification_time'] = 0 result['benchmark_time'] = 0 - result['strategy_time'] = self.last_strategy_time + result['strategy_time'] = strategy_time_per_config total_time = 1000 * (perf_counter() - self.start_time) self.start_time = perf_counter() - result['framework_time'] = total_time - self.last_strategy_time + result['framework_time'] = total_time result[tuning_options.objective] = util.InvalidConfig() results.append(result) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 83793c256..ab3929bf9 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -130,6 +130,7 @@ def _normalize_and_validate_config(self, x, check_restrictions=True): def _run_configs(self, xs, check_restrictions=True): """ Takes a list of Euclidian coordinates and evaluates the configurations at those points. """ self.runner.last_strategy_time = 1000 * (perf_counter() - self.runner.last_strategy_start_time) + self.runner.start_time = perf_counter() # start framework time # error value to return for numeric optimizers that need a numerical value logging.debug("_cost_func called") @@ -166,7 +167,7 @@ def _run_configs(self, xs, check_restrictions=True): # mask configs we cannot benchmark batch_configs = batch_configs[:last_index] batch_indices = batch_indices[:last_index] - final_results = final_results[:last_index] + final_results = final_results[:batch_indices[-1]+1] # compile and benchmark the batch batch_results = self.runner.run(batch_configs, self.tuning_options) diff --git a/kernel_tuner/util.py b/kernel_tuner/util.py index 635c6de78..7abd7a60d 100644 --- a/kernel_tuner/util.py +++ b/kernel_tuner/util.py @@ -213,8 +213,18 @@ def check_stop_criterion(to: dict) -> float: if time_spent > to.time_limit: raise StopCriterionReached("time limit exceeded") return time_spent / to.time_limit - +def stop_criterion_reached(to: dict) -> bool: + """Returns True if stop criterion has been reached""" + res = False + if "max_fevals" in to: + if len(to.unique_results) >= to.max_fevals: + res = True + if "time_limit" in to: + time_spent = (time.perf_counter() - to.start_time) + (to.simulated_time * 1e-3) + to.startup_time + if time_spent > to.time_limit: + res |= True + return res def check_tune_params_list(tune_params, observers, simulation_mode=False): """Raise an exception if a tune parameter has a forbidden name.""" From ec30052a330f11e3466772b52704cfef5df73861 Mon Sep 17 00:00:00 2001 From: Ben van Werkhoven Date: Fri, 30 Jan 2026 09:10:38 +0000 Subject: [PATCH 25/34] fix budget overshoot issue for sequential tuning --- kernel_tuner/runners/sequential.py | 2 ++ kernel_tuner/runners/simulation.py | 2 ++ kernel_tuner/strategies/common.py | 2 ++ test/test_runners.py | 1 + 4 files changed, 7 insertions(+) diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index 7e9d76934..4b7805946 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -132,5 +132,7 @@ def run(self, parameter_space, tuning_options): # all visited configurations are added to results to provide a trace for optimization strategies results.append(params) + if x_int not in tuning_options.unique_results: + tuning_options.unique_results[x_int] = result return results diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index 986bf9b25..659de3fe7 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -136,6 +136,8 @@ def run(self, parameter_space, tuning_options): result["framework_time"] = total_time results.append(result) + if x_int not in tuning_options.unique_results: + tuning_options.unique_results[x_int] = result continue # if the configuration is not in the cache and not within restrictions, simulate an InvalidConfig with warning diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index baff0f14e..f6375cf78 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -188,6 +188,8 @@ def _run_configs(self, xs, check_restrictions=True): self.tuning_options.unique_results[x_int] = result # check again for stop condition + # this check is necessary because some strategies cannot handle partially completed requests + # for example when only half of the configs in a population have been evaluated self.budget_spent_fraction = util.check_stop_criterion(self.tuning_options) # upon returning from this function control will be given back to the strategy, so reset the start time diff --git a/test/test_runners.py b/test/test_runners.py index de15352b4..609ccad32 100644 --- a/test/test_runners.py +++ b/test/test_runners.py @@ -287,6 +287,7 @@ def test_runner(env): device_options = Options([(k, opts.get(k, None)) for k in _device_options.keys()]) tuning_options.cachefile = None + tuning_options.unique_results = {} # create runner runner = SequentialRunner(kernelsource, From d18bfdfbe97cdd56901e132ed74b82022a3c10ce Mon Sep 17 00:00:00 2001 From: stijn Date: Sun, 1 Feb 2026 20:29:30 +0100 Subject: [PATCH 26/34] Add `TuningBudget` class and modify runners to respect the budget --- kernel_tuner/interface.py | 30 +++++-- kernel_tuner/runners/parallel.py | 105 +++++++++++++++--------- kernel_tuner/runners/sequential.py | 9 ++- kernel_tuner/runners/simulation.py | 58 ++++++-------- kernel_tuner/strategies/common.py | 83 ++++++++++++------- kernel_tuner/util.py | 124 +++++++++++++++++++---------- 6 files changed, 258 insertions(+), 151 deletions(-) diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index 215f4cdb8..975078248 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -620,11 +620,14 @@ def tune_kernel( # copy some values from strategy_options searchspace_construction_options = {} + max_fevals = None + time_limit = None + if strategy_options: if "max_fevals" in strategy_options: - tuning_options["max_fevals"] = strategy_options["max_fevals"] + max_fevals = strategy_options["max_fevals"] if "time_limit" in strategy_options: - tuning_options["time_limit"] = strategy_options["time_limit"] + time_limit = strategy_options["time_limit"] if "searchspace_construction_options" in strategy_options: searchspace_construction_options = strategy_options["searchspace_construction_options"] @@ -703,14 +706,27 @@ def preprocess_cache(filepath): print(f"Searchspace has {searchspace.size} configurations after restrictions.") # register the times and raise an exception if the budget is exceeded - if "time_limit" in tuning_options: - tuning_options["startup_time"] = perf_counter() - start_overhead_time - if tuning_options["startup_time"] > tuning_options["time_limit"]: + startup_time = perf_counter() - start_overhead_time + + if time_limit is not None: + if startup_time > time_limit: raise RuntimeError( - f"The startup time of the tuning process ({tuning_options['startup_time']} seconds) has exceeded the time limit ({tuning_options['time_limit']} seconds). " + f"The startup time of the tuning process ({startup_time} seconds) has exceeded the time limit ({time_limit} seconds). " "Please increase the time limit or decrease the size of the search space." ) - tuning_options["start_time"] = perf_counter() + + time_limit -= startup_time + + if max_fevals is None or max_fevals > searchspace.size: + logging.info(f"evaluation limit has been adjusted from {max_fevals} to {searchspace.size} (search space size)") + max_fevals = searchspace.size + + # Create the budget. Add the time spent on startup to the budget + budget = util.TuningBudget(time_limit, max_fevals) + tuning_options["time_limit"] = time_limit # TODO: Is this used? + tuning_options["max_fevals"] = max_fevals # TODO: Is this used? + tuning_options["budget"] = budget + # call the strategy to execute the tuning process results = strategy.tune(searchspace, runner, tuning_options) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index a14a30823..2fc338d4c 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -3,10 +3,18 @@ import logging import socket from time import perf_counter +from typing import List, Optional from kernel_tuner.core import DeviceInterface from kernel_tuner.interface import Options from kernel_tuner.runners.runner import Runner -from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache +from kernel_tuner.util import ( + BudgetExceededConfig, + ErrorConfig, + TuningBudget, + print_config_output, + process_metrics, + store_cache, +) from datetime import datetime, timezone logger = logging.getLogger(__name__) @@ -213,31 +221,31 @@ def shutdown(self): def available_parallelism(self): return len(self.workers) - def submit_jobs(self, jobs): + def submit_jobs(self, jobs, budget: TuningBudget): pending_jobs = deque(jobs) running_jobs = [] - while pending_jobs or running_jobs: - should_wait = True + while pending_jobs and not budget.is_done(): + job_was_submitted = False # If there is still work left, submit it now - if pending_jobs: - for i, worker in enumerate(list(self.workers)): - if worker.is_available(): - # Push worker to back of list - self.workers.pop(i) - self.workers.append(worker) + for i, worker in enumerate(list(self.workers)): + if worker.is_available(): + # Push worker to back of list + self.workers.pop(i) + self.workers.append(worker) - # Pop job and submit it - job = pending_jobs.popleft() - ref = worker.submit(*job) - running_jobs.append(ref) + # Pop job and submit it + key, config = pending_jobs.popleft() + ref = worker.submit(key, config) + running_jobs.append(ref) - should_wait = False - break + job_was_submitted = True + budget.add_evaluations(1) + break # If no work was submitted, wait until a worker is available - if should_wait: + if not job_was_submitted: if not running_jobs: raise RuntimeError("invalid state: no ray workers available") @@ -246,14 +254,28 @@ def submit_jobs(self, jobs): for result in ready_jobs: yield ray.get(result) - def run(self, parameter_space, tuning_options): + # If there are still pending jobs, then the budget has been exceeded. + # We return `None` to indicate that no result is available for these jobs. + while pending_jobs: + key, _ = pending_jobs.popleft() + yield (key, None) + + # Wait until running jobs complete + while running_jobs: + ready_jobs, running_jobs = ray.wait(running_jobs, num_returns=1) + + for result in ready_jobs: + yield ray.get(result) + + def run(self, parameter_space, tuning_options) -> List[Optional[dict]]: metrics = tuning_options.metrics objective = tuning_options.objective jobs = [] # Jobs that need to be executed results = [] # Results that will be returned at the end key2index = dict() # Used to insert job result back into `results` - duplicate_entries = [] # Used for duplicate entries in `parameter_space` + + total_worker_time = 0 # Select jobs which are not in the cache for index, config in enumerate(parameter_space): @@ -262,28 +284,33 @@ def run(self, parameter_space, tuning_options): if key in tuning_options.cache: params.update(tuning_options.cache[key]) - params["compile_time"] = 0 - params["verification_time"] = 0 - params["benchmark_time"] = 0 + + # Simulate compile, verification, and benchmark time + tuning_options.budget.add_time_spent(params["compile_time"]) + tuning_options.budget.add_time_spent(params["verification_time"]) + tuning_options.budget.add_time_spent(params["benchmark_time"]) results.append(params) else: - if key not in key2index: - key2index[key] = index - else: - duplicate_entries.append((key2index[key], index)) + assert key not in key2index, "duplicate jobs submitted" + key2index[key] = index jobs.append((key, params)) results.append(None) - total_worker_time = 0 # Submit jobs and wait for them to finish - for key, result in self.submit_jobs(jobs): + for key, result in self.submit_jobs(jobs, tuning_options.budget): + # `None` indicate that no result is available since the budget is exceeded. + # We can skip it, meaning that `results` contains `None`s for these entries + if result is None: + continue + + # Store the result into the output array results[key2index[key]] = result # Collect total time spent by worker total_worker_time += ( - params["compile_time"] + params["verification_time"] + params["benchmark_time"] + result["compile_time"] + result["verification_time"] + result["benchmark_time"] ) if isinstance(result.get(objective), ErrorConfig): @@ -300,10 +327,6 @@ def run(self, parameter_space, tuning_options): # add configuration to cache store_cache(key, result, tuning_options.cachefile, tuning_options.cache) - # Copy each `i` to `j` for every `i,j` in `duplicate_entries` - for i, j in duplicate_entries: - results[j] = dict(results[i]) - total_time = 1000 * (perf_counter() - self.start_time) self.start_time = perf_counter() @@ -313,14 +336,20 @@ def run(self, parameter_space, tuning_options): runner_time = total_time - strategy_time framework_time = max(runner_time * len(self.workers) - total_worker_time, 0) + num_valid_results = sum(bool(r) for r in results) # Count the number of valid results + # Post-process all the results - for params in results: + for result in results: + # Skip missing results + if not result: + continue + # Amortize the time over all the results - params["strategy_time"] = strategy_time / len(results) - params["framework_time"] = framework_time / len(results) + result["strategy_time"] = strategy_time / num_valid_results + result["framework_time"] = framework_time / num_valid_results # only compute metrics on configs that have not errored - if metrics and not isinstance(params.get(objective), ErrorConfig): - params = process_metrics(params, metrics) + if not isinstance(result.get(objective), ErrorConfig): + result = process_metrics(result, metrics) return results diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index 4b7805946..b1acc5ccb 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -70,6 +70,7 @@ def run(self, parameter_space, tuning_options): # iterate over parameter space for element in parameter_space: + tuning_options.budget.add_evaluations(1) params = dict(zip(tuning_options.tune_params.keys(), element)) if stop_criterion_reached(tuning_options): @@ -82,9 +83,11 @@ def run(self, parameter_space, tuning_options): x_int = ",".join([str(i) for i in element]) if tuning_options.cache and x_int in tuning_options.cache: params.update(tuning_options.cache[x_int]) - params["compile_time"] = 0 - params["verification_time"] = 0 - params["benchmark_time"] = 0 + + # Simulate compile, verification, and benchmark time + tuning_options.budget.add_time_spent(params["compile_time"]) + tuning_options.budget.add_time_spent(params["verification_time"]) + tuning_options.budget.add_time_spent(params["benchmark_time"]) else: # attempt to warmup the GPU by running the first config in the parameter space and ignoring the result if not self.warmed_up: diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index 659de3fe7..9f9119b94 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -54,6 +54,7 @@ def __init__(self, kernel_source, kernel_options, device_options, iterations, ob self.kernel_options = kernel_options self.start_time = perf_counter() + self.total_simulated_time = 0 self.last_strategy_start_time = self.start_time self.last_strategy_time = 0 self.units = {} @@ -64,7 +65,7 @@ def get_device_info(self): def get_environment(self, tuning_options): env = self.dev.get_environment() env["simulation"] = True - env["simulated_time"] = tuning_options.simulated_time + env["simulated_time"] = self.total_simulated_time return env def run(self, parameter_space, tuning_options): @@ -89,55 +90,48 @@ def run(self, parameter_space, tuning_options): # iterate over parameter space for element in parameter_space: - if util.stop_criterion_reached(tuning_options): - return results - + # Append `None` to indicate that the tuning budget has been exceeded + if tuning_options.budget.is_done(): + results.append(None) + continue + # check if element is in the cache - x_int = ",".join([str(i) for i in element]) - if tuning_options.cache and x_int in tuning_options.cache: - result = tuning_options.cache[x_int].copy() + key = ",".join([str(i) for i in element]) + + if key in tuning_options.cache: + # Get from cache and create a copy + result = dict(tuning_options.cache[key]) # only compute metrics on configs that have not errored if tuning_options.metrics and not isinstance(result.get(tuning_options.objective), util.ErrorConfig): result = util.process_metrics(result, tuning_options.metrics) - # Simulate behavior of sequential runner that when a configuration is - # served from the cache by the sequential runner, the compile_time, - # verification_time, and benchmark_time are set to 0. - # This step is only performed in the simulation runner when a configuration - # is served from the cache beyond the first timel. That is, when the - # configuration is already counted towards the unique_results. - # It is the responsibility of cost_func to add configs to unique_results. - if x_int in tuning_options.unique_results: - result["compile_time"] = 0 - result["verification_time"] = 0 - result["benchmark_time"] = 0 - - else: - # configuration is evaluated for the first time, print to the console - util.print_config_output( - tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units - ) + # configuration is evaluated for the first time, print to the console + util.print_config_output( + tuning_options.tune_params, result, self.quiet, tuning_options.metrics, self.units + ) # Everything but the strategy time and framework time are simulated, result["strategy_time"] = strategy_time_per_config + # Simulate the evaluation of this configuration + tuning_options.budget.add_evaluations(1) + tuning_options.budget.add_time_spent(result["compile_time"]) + tuning_options.budget.add_time_spent(result["verification_time"]) + tuning_options.budget.add_time_spent(result["benchmark_time"]) + try: - simulated_time = result["compile_time"] + result["verification_time"] + result["benchmark_time"] - tuning_options.simulated_time += simulated_time + self.total_simulated_time += result["compile_time"] + result["verification_time"] + result["benchmark_time"] except KeyError: - if "time_limit" in tuning_options: - raise RuntimeError( - "Cannot use simulation mode with a time limit on a cache file that does not have full compile, verification, and benchmark timings on all configurations" - ) + raise RuntimeError( + "Cannot use simulation mode with a time limit on a cache file that does not have full compile, verification, and benchmark timings on all configurations" + ) total_time = 1000 * (perf_counter() - self.start_time) self.start_time = perf_counter() result["framework_time"] = total_time results.append(result) - if x_int not in tuning_options.unique_results: - tuning_options.unique_results[x_int] = result continue # if the configuration is not in the cache and not within restrictions, simulate an InvalidConfig with warning diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index f6375cf78..0846c76b7 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -139,53 +139,68 @@ def _run_configs(self, xs, check_restrictions=True): logging.debug("_cost_func called") # check if max_fevals is reached or time limit is exceeded - self.budget_spent_fraction = util.check_stop_criterion(self.tuning_options) + self.tuning_options.budget.raise_exception_if_done() batch_configs = [] # The configs to run - batch_indices = [] # Where to store result in `final_results`` + batch_keys = [] # The keys of the configs to run + pending_indices_by_key = dict() # Maps key => where to store result in `final_results` final_results = [] # List returned to the user benchmark_config = [] + # Loop over all configurations. For each configurations there are four cases: + # 1. The configuration is valid, we can skip it + # 2. The configuration is in `unique_results`, we can get it from there + # 3. The configuration is in `pending_indices_by_key`, it is duplicate in `xs` + # 4. The configuration must be evaluated by the runner. for x in xs: config, is_legal = self._normalize_and_validate_config(x, check_restrictions=check_restrictions) logging.debug("normalize config: %s -> %s (legal: %s)", str(x), str(config), is_legal) + key = ",".join([str(i) for i in config]) - if is_legal: - batch_configs.append(config) - batch_indices.append(len(final_results)) - final_results.append(None) - x_int = ",".join([str(i) for i in config]) - benchmark_config.append(x_int not in self.tuning_options.unique_results) - else: + # 1. Not legal, just return `InvalidConfig` + if not is_legal: result = dict(zip(self.searchspace.tune_params.keys(), config)) result[self.objective] = util.InvalidConfig() final_results.append(result) - # do not overshoot max_fevals if we can avoid it - if "max_fevals" in self.tuning_options: - budget = self.tuning_options.max_fevals - len(self.tuning_options.unique_results) - if sum(benchmark_config) > budget: - # find index 'budget'th True value - last_index = _get_nth_true(benchmark_config, budget)+1 - # mask configs we cannot benchmark - batch_configs = batch_configs[:last_index] - batch_indices = batch_indices[:last_index] - final_results = final_results[:batch_indices[-1]+1] + # 2. Attempt to retrieve from `unique_results` + elif key in self.tuning_options.unique_results: + result = dict(self.tuning_options.unique_results[key]) + final_results.append(result) + + # 3. We have already seen this config in the current batch + elif key in pending_indices_by_key: + pending_indices_by_key[key].append(len(final_results)) + final_results.append(None) + + # 4. A new config, we must evaluate this + else: + batch_keys.append(key) + batch_configs.append(config) + pending_indices_by_key[key] = [len(final_results)] + final_results.append(None) # compile and benchmark the batch batch_results = self.runner.run(batch_configs, self.tuning_options) - self.results.extend(batch_results) - # set in the results array - for index, result in zip(batch_indices, batch_results): - final_results[index] = result + for key, result in zip(batch_keys, batch_results): + # Skip. Result is missing because the runner has exhausted the budget + if result is None: + continue + + # set in the results array + for index in pending_indices_by_key[key]: + final_results[index] = dict(result) + + # Disable the timings. Only the first result must get these. + result["compile_time"] = 0 + result["verification_time"] = 0 + result["benchmark_time"] = 0 + + # Put result in `unique_results` + self.tuning_options.unique_results[key] = result + self.results.append(result) - # append to `unique_results` - for config, result, benchmarked in zip(batch_configs, batch_results, benchmark_config): - if benchmarked: - x_int = ",".join([str(i) for i in config]) - if x_int not in self.tuning_options.unique_results: - self.tuning_options.unique_results[x_int] = result # check again for stop condition # this check is necessary because some strategies cannot handle partially completed requests @@ -194,6 +209,16 @@ def _run_configs(self, xs, check_restrictions=True): # upon returning from this function control will be given back to the strategy, so reset the start time self.runner.last_strategy_start_time = perf_counter() + + # Check the tuning budget again + self.tuning_options.budget.raise_exception_if_done() + self.budget_spent_fraction = self.tuning_options.budget.get_fraction_consumed() + + # If some results are missing (`None`), then the runner did not return all results + # because the budget has been exceed or some other reason causing the runner to fail. + if not all(final_results): + raise util.StopCriterionReached("runner did not evaluate all given configurations") + return final_results def eval_all(self, xs, check_restrictions=True): diff --git a/kernel_tuner/util.py b/kernel_tuner/util.py index 02b672576..6c6384ab7 100644 --- a/kernel_tuner/util.py +++ b/kernel_tuner/util.py @@ -1,5 +1,6 @@ """Module for kernel tuner utility functions.""" import ast +from datetime import timedelta import errno import json import logging @@ -187,40 +188,78 @@ def check_argument_list(kernel_name, kernel_string, args): warnings.warn(errors[0], UserWarning) -def check_stop_criterion(to: dict) -> float: - """Check if the stop criterion is reached. - - Args: - to (dict): tuning options. +class TuningBudget: + def __init__(self, time_limit=None, max_fevals=None): + if max_fevals is None: + max_fevals = float("inf") + + if time_limit is None: + time_limit = timedelta.max + + if not isinstance(time_limit, timedelta): + time_limit = timedelta(seconds=time_limit) + + if max_fevals <= 0: + raise ValueError("max_fevals must be greater than zero") + + if time_limit <= timedelta(seconds=0): + raise ValueError("time_limit must be greater than zero") + + self.start_time_seconds = time.perf_counter() + self.time_spent_extra = timedelta() + self.time_limit = time_limit + self.num_fevals = 0 + self.max_fevals = max_fevals + + def add_evaluations(self, n=1): + self.num_fevals += n + + def add_time_spent(self, delta): + if not isinstance(delta, timedelta): + delta = timedelta(seconds=delta) + self.time_spent_extra += delta + + def get_time_spent(self) -> timedelta: + seconds_passed = time.perf_counter() - self.start_time_seconds + return timedelta(seconds=seconds_passed) + self.time_spent_extra + + def get_time_remaining(self) -> timedelta: + return max(self.time_limit - self.get_time_spent(), timedelta(seconds=0)) + + def get_evaluations_spent(self) -> int: + return max(self.max_fevals - self.num_fevals, 0) + + def get_evaluations_remaining(self) -> int: + return max(self.max_fevals - self.num_fevals, 0) + + def is_done(self) -> bool: + if self.num_fevals >= self.max_fevals: + return True - Raises: - StopCriterionReached: if the max_fevals is reached or time limit is exceeded. + if self.get_time_spent() > self.time_limit: + return True - Returns: - float: fraction of budget spent. If both max_fevals and time_limit are set, it returns the fraction of time. - """ - if "max_fevals" in to: - if len(to.unique_results) >= to.max_fevals: - raise StopCriterionReached(f"max_fevals ({to.max_fevals}) reached") - if not "time_limit" in to: - return len(to.unique_results) / to.max_fevals - if "time_limit" in to: - time_spent = (time.perf_counter() - to.start_time) + (to.simulated_time * 1e-3) + to.startup_time - if time_spent > to.time_limit: + return False + + def raise_exception_if_done(self): + if self.num_fevals >= self.max_fevals: + raise StopCriterionReached(f"max_fevals ({self.max_fevals}) reached") + + if self.get_time_spent() > self.time_limit: raise StopCriterionReached("time limit exceeded") - return time_spent / to.time_limit - -def stop_criterion_reached(to: dict) -> bool: - """Returns True if stop criterion has been reached""" - res = False - if "max_fevals" in to: - if len(to.unique_results) >= to.max_fevals: - res = True - if "time_limit" in to: - time_spent = (time.perf_counter() - to.start_time) + (to.simulated_time * 1e-3) + to.startup_time - if time_spent > to.time_limit: - res |= True - return res + + def get_fraction_consumed(self) -> float: + if self.num_fevals >= self.max_fevals: + return 1.0 + + time_spent = self.get_time_spent() + + if time_spent > self.time_limit: + return 1.0 + + return max(time_spent / self.time_limit, self.num_fevals / self.max_fevals) + + def check_tune_params_list(tune_params, observers, simulation_mode=False): """Raise an exception if a tune parameter has a forbidden name.""" @@ -694,17 +733,18 @@ def process_metrics(params, metrics): :rtype: dict """ - if not isinstance(metrics, dict): - raise ValueError("metrics should be a dictionary to preserve order and support composability") - for k, v in metrics.items(): - if isinstance(v, str): - value = eval(replace_param_occurrences(v, params)) - elif callable(v): - value = v(params) - else: - raise ValueError("metric dicts values should be strings or callable") - # We overwrite any existing values for the given key - params[k] = value + if metrics: + if not isinstance(metrics, dict): + raise ValueError("metrics should be a dictionary to preserve order and support composability") + for k, v in metrics.items(): + if isinstance(v, str): + value = eval(replace_param_occurrences(v, params)) + elif callable(v): + value = v(params) + else: + raise ValueError("metric dicts values should be strings or callable") + # We overwrite any existing values for the given key + params[k] = value return params From 05bc4a64326af9581c785e6bbd50e348293ceb01 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 11:02:00 +0100 Subject: [PATCH 27/34] Add tests for `TuningBudget` --- kernel_tuner/runners/parallel.py | 6 +-- kernel_tuner/runners/sequential.py | 17 +++--- kernel_tuner/runners/simulation.py | 6 +-- kernel_tuner/strategies/common.py | 10 +--- kernel_tuner/util.py | 55 ++++++++++--------- test/test_util_functions.py | 87 ++++++++++++++++++++++++++++++ 6 files changed, 132 insertions(+), 49 deletions(-) diff --git a/kernel_tuner/runners/parallel.py b/kernel_tuner/runners/parallel.py index 2fc338d4c..0a8edcf8a 100644 --- a/kernel_tuner/runners/parallel.py +++ b/kernel_tuner/runners/parallel.py @@ -286,9 +286,9 @@ def run(self, parameter_space, tuning_options) -> List[Optional[dict]]: params.update(tuning_options.cache[key]) # Simulate compile, verification, and benchmark time - tuning_options.budget.add_time_spent(params["compile_time"]) - tuning_options.budget.add_time_spent(params["verification_time"]) - tuning_options.budget.add_time_spent(params["benchmark_time"]) + tuning_options.budget.add_time(milliseconds=params["compile_time"]) + tuning_options.budget.add_time(milliseconds=params["verification_time"]) + tuning_options.budget.add_time(milliseconds=params["benchmark_time"]) results.append(params) else: assert key not in key2index, "duplicate jobs submitted" diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index b1acc5ccb..ff6aaef28 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -5,7 +5,7 @@ from kernel_tuner.core import DeviceInterface from kernel_tuner.runners.runner import Runner -from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache, stop_criterion_reached +from kernel_tuner.util import ErrorConfig, print_config_output, process_metrics, store_cache class SequentialRunner(Runner): @@ -70,12 +70,15 @@ def run(self, parameter_space, tuning_options): # iterate over parameter space for element in parameter_space: + # If the time limit is exceeded, just skip this element. Add `None` to + # indicate to CostFunc that no result is available for this config. + if tuning_options.budget.is_done(): + results.append(None) + continue + tuning_options.budget.add_evaluations(1) params = dict(zip(tuning_options.tune_params.keys(), element)) - if stop_criterion_reached(tuning_options): - return results - result = None warmup_time = 0 @@ -85,9 +88,9 @@ def run(self, parameter_space, tuning_options): params.update(tuning_options.cache[x_int]) # Simulate compile, verification, and benchmark time - tuning_options.budget.add_time_spent(params["compile_time"]) - tuning_options.budget.add_time_spent(params["verification_time"]) - tuning_options.budget.add_time_spent(params["benchmark_time"]) + tuning_options.budget.add_time(milliseconds=params["compile_time"]) + tuning_options.budget.add_time(milliseconds=params["verification_time"]) + tuning_options.budget.add_time(milliseconds=params["benchmark_time"]) else: # attempt to warmup the GPU by running the first config in the parameter space and ignoring the result if not self.warmed_up: diff --git a/kernel_tuner/runners/simulation.py b/kernel_tuner/runners/simulation.py index 9f9119b94..0c0affd76 100644 --- a/kernel_tuner/runners/simulation.py +++ b/kernel_tuner/runners/simulation.py @@ -116,9 +116,9 @@ def run(self, parameter_space, tuning_options): # Simulate the evaluation of this configuration tuning_options.budget.add_evaluations(1) - tuning_options.budget.add_time_spent(result["compile_time"]) - tuning_options.budget.add_time_spent(result["verification_time"]) - tuning_options.budget.add_time_spent(result["benchmark_time"]) + tuning_options.budget.add_time(milliseconds=result["compile_time"]) + tuning_options.budget.add_time(milliseconds=result["verification_time"]) + tuning_options.budget.add_time(milliseconds=result["benchmark_time"]) try: self.total_simulated_time += result["compile_time"] + result["verification_time"] + result["benchmark_time"] diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 0846c76b7..41d69134b 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -145,7 +145,6 @@ def _run_configs(self, xs, check_restrictions=True): batch_keys = [] # The keys of the configs to run pending_indices_by_key = dict() # Maps key => where to store result in `final_results` final_results = [] # List returned to the user - benchmark_config = [] # Loop over all configurations. For each configurations there are four cases: # 1. The configuration is valid, we can skip it @@ -201,16 +200,11 @@ def _run_configs(self, xs, check_restrictions=True): self.tuning_options.unique_results[key] = result self.results.append(result) - - # check again for stop condition - # this check is necessary because some strategies cannot handle partially completed requests - # for example when only half of the configs in a population have been evaluated - self.budget_spent_fraction = util.check_stop_criterion(self.tuning_options) - # upon returning from this function control will be given back to the strategy, so reset the start time self.runner.last_strategy_start_time = perf_counter() - # Check the tuning budget again + # this check is necessary because some strategies cannot handle partially completed requests + # for example when only half of the configs in a population have been evaluated self.tuning_options.budget.raise_exception_if_done() self.budget_spent_fraction = self.tuning_options.budget.get_fraction_consumed() diff --git a/kernel_tuner/util.py b/kernel_tuner/util.py index 6c6384ab7..0c456cec9 100644 --- a/kernel_tuner/util.py +++ b/kernel_tuner/util.py @@ -190,19 +190,13 @@ def check_argument_list(kernel_name, kernel_string, args): class TuningBudget: def __init__(self, time_limit=None, max_fevals=None): - if max_fevals is None: - max_fevals = float("inf") - - if time_limit is None: - time_limit = timedelta.max - - if not isinstance(time_limit, timedelta): + if time_limit is not None and not isinstance(time_limit, timedelta): time_limit = timedelta(seconds=time_limit) - if max_fevals <= 0: + if max_fevals is not None and max_fevals <= 0: raise ValueError("max_fevals must be greater than zero") - if time_limit <= timedelta(seconds=0): + if time_limit is not None and time_limit <= timedelta(seconds=0): raise ValueError("time_limit must be greater than zero") self.start_time_seconds = time.perf_counter() @@ -214,50 +208,55 @@ def __init__(self, time_limit=None, max_fevals=None): def add_evaluations(self, n=1): self.num_fevals += n - def add_time_spent(self, delta): - if not isinstance(delta, timedelta): - delta = timedelta(seconds=delta) - self.time_spent_extra += delta + def add_time(self, seconds=0, milliseconds=0): + self.time_spent_extra += timedelta(seconds=seconds, milliseconds=milliseconds) def get_time_spent(self) -> timedelta: seconds_passed = time.perf_counter() - self.start_time_seconds return timedelta(seconds=seconds_passed) + self.time_spent_extra def get_time_remaining(self) -> timedelta: - return max(self.time_limit - self.get_time_spent(), timedelta(seconds=0)) + if self.time_limit is not None: + return max(self.time_limit - self.get_time_spent(), timedelta(seconds=0)) + else: + return timedelta.max def get_evaluations_spent(self) -> int: - return max(self.max_fevals - self.num_fevals, 0) + return self.num_fevals def get_evaluations_remaining(self) -> int: - return max(self.max_fevals - self.num_fevals, 0) + if self.max_fevals is not None: + return max(self.max_fevals - self.num_fevals, 0) + else: + return float("inf") def is_done(self) -> bool: - if self.num_fevals >= self.max_fevals: + if self.max_fevals is not None and self.num_fevals >= self.max_fevals: return True - if self.get_time_spent() > self.time_limit: + if self.time_limit is not None and self.get_time_spent() > self.time_limit: return True return False def raise_exception_if_done(self): - if self.num_fevals >= self.max_fevals: + if self.max_fevals is not None and self.num_fevals >= self.max_fevals: raise StopCriterionReached(f"max_fevals ({self.max_fevals}) reached") - if self.get_time_spent() > self.time_limit: + if self.time_limit is not None and self.get_time_spent() > self.time_limit: raise StopCriterionReached("time limit exceeded") def get_fraction_consumed(self) -> float: - if self.num_fevals >= self.max_fevals: - return 1.0 - - time_spent = self.get_time_spent() + if self.max_fevals is not None and self.time_limit is not None: + time_spent = self.get_time_spent() + return min(1.0, time_spent / self.time_limit, self.num_fevals / self.max_fevals) + elif self.max_fevals is not None: + return min(1.0, self.num_fevals / self.max_fevals) + elif self.time_limit is not None: + return min(1.0, self.get_time_spent() / self.time_limit) + else: + return 0.0 - if time_spent > self.time_limit: - return 1.0 - - return max(time_spent / self.time_limit, self.num_fevals / self.max_fevals) diff --git a/test/test_util_functions.py b/test/test_util_functions.py index f2ab9ca5b..4b63e864d 100644 --- a/test/test_util_functions.py +++ b/test/test_util_functions.py @@ -3,6 +3,7 @@ import json import os import warnings +import datetime import numpy as np import pytest @@ -429,6 +430,92 @@ def test_check_argument_list7(): assert_user_warning(check_argument_list, [kernel_name, kernel_string, args]) +def test_tuning_budget1(): + budget = TuningBudget() + assert budget.get_evaluations_spent() == 0 + assert budget.get_evaluations_remaining() == float("inf") + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == 0.0 + + budget.add_evaluations(9000) + assert budget.get_evaluations_spent() == 9000 + assert budget.get_evaluations_remaining() == float("inf") + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == 0.0 + + budget.add_time(seconds=9000) + assert budget.get_evaluations_spent() == 9000 + assert budget.get_evaluations_remaining() == float("inf") + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == 0.0 + +def test_tuning_budget2(): + budget = TuningBudget(max_fevals=5) + assert budget.get_evaluations_spent() == 0 + assert budget.get_evaluations_remaining() == 5 + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == 0.0 + + budget.add_evaluations(4) + assert budget.get_evaluations_spent() == 4 + assert budget.get_evaluations_remaining() == 1 + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == 4/5 + + budget.add_evaluations(1) + assert budget.get_evaluations_spent() == 5 + assert budget.get_evaluations_remaining() == 0 + assert budget.is_done() + assert pytest.raises(StopCriterionReached, budget.raise_exception_if_done) + assert budget.get_fraction_consumed() == 1.0 + + +def test_tuning_budget3(): + # Two values are similar if they are within 0.01 + approx = lambda x: pytest.approx(x, abs=0.01) + + budget = TuningBudget(time_limit=5) + assert budget.get_time_spent().total_seconds() == approx(0) + assert budget.get_time_remaining().total_seconds() == approx(5) + assert budget.get_evaluations_spent() == 0 + assert budget.get_evaluations_remaining() == float("inf") + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == approx(0.0) + + budget.add_evaluations(1) + assert budget.get_time_spent().total_seconds() == approx(0) + assert budget.get_time_remaining().total_seconds() == approx(5) + assert budget.get_evaluations_spent() == 1 + assert budget.get_evaluations_remaining() == float("inf") + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == approx(0.0) + + budget.add_time(seconds=2) + assert budget.get_time_spent().total_seconds() == approx(2) + assert budget.get_time_remaining().total_seconds() == approx(3) + assert budget.get_evaluations_spent() == 1 + assert budget.get_evaluations_remaining() == float("inf") + assert not budget.is_done() + budget.raise_exception_if_done() # Should not raise + assert budget.get_fraction_consumed() == approx(2/5) + + budget.add_time(seconds=4) + assert budget.get_time_spent().total_seconds() == approx(6) + assert budget.get_time_remaining().total_seconds() == approx(0) + assert budget.get_evaluations_spent() == 1 + assert budget.get_evaluations_remaining() == float("inf") + assert budget.is_done() + assert pytest.raises(StopCriterionReached, budget.raise_exception_if_done) + assert budget.get_fraction_consumed() == 1.0 + + def test_check_tune_params_list(): tune_params = dict( zip( From 8e28f029263dfab097747ede143f823677c6b9bc Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 11:04:55 +0100 Subject: [PATCH 28/34] Fix mismatch between milliseconds and seconds --- kernel_tuner/runners/sequential.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index ff6aaef28..e193dde28 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -115,7 +115,7 @@ def run(self, parameter_space, tuning_options): params = process_metrics(params, tuning_options.metrics) # get the framework time by estimating based on other times - total_time = 1000 * ((perf_counter() - self.start_time) - warmup_time) + total_time = 1000 * (perf_counter() - self.start_time) - warmup_time params["strategy_time"] = strategy_time_per_config params["framework_time"] = max( total_time From 6d09ca685389ae02d1dff74917fa3efa49a03ad2 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 11:46:54 +0100 Subject: [PATCH 29/34] Move `unique_results` from `tuning_options` to `CostFunc` --- kernel_tuner/interface.py | 1 - kernel_tuner/runners/sequential.py | 2 -- kernel_tuner/strategies/common.py | 21 ++++++++++++++----- kernel_tuner/strategies/diff_evo.py | 4 ++-- kernel_tuner/strategies/greedy_ils.py | 3 +-- kernel_tuner/strategies/greedy_mls.py | 3 +-- kernel_tuner/strategies/pyatf_strategies.py | 2 +- .../strategies/simulated_annealing.py | 2 +- 8 files changed, 22 insertions(+), 16 deletions(-) diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index 975078248..f874e701b 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -616,7 +616,6 @@ def tune_kernel( kernel_options = Options([(k, opts[k]) for k in _kernel_options.keys()]) tuning_options = Options([(k, opts[k]) for k in _tuning_options.keys()]) device_options = Options([(k, opts[k]) for k in _device_options.keys()]) - tuning_options["unique_results"] = {} # copy some values from strategy_options searchspace_construction_options = {} diff --git a/kernel_tuner/runners/sequential.py b/kernel_tuner/runners/sequential.py index e193dde28..1814cefe1 100644 --- a/kernel_tuner/runners/sequential.py +++ b/kernel_tuner/runners/sequential.py @@ -138,7 +138,5 @@ def run(self, parameter_space, tuning_options): # all visited configurations are added to results to provide a trace for optimization strategies results.append(params) - if x_int not in tuning_options.unique_results: - tuning_options.unique_results[x_int] = result return results diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 41d69134b..3cb6e84e5 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -98,6 +98,7 @@ def __init__( self.scaling = scaling self.snap = snap self.return_invalid = return_invalid + self.unique_results = dict() self.results = [] self.budget_spent_fraction = 0.0 self.invalid_return_value = invalid_value @@ -147,7 +148,7 @@ def _run_configs(self, xs, check_restrictions=True): final_results = [] # List returned to the user # Loop over all configurations. For each configurations there are four cases: - # 1. The configuration is valid, we can skip it + # 1. The configuration is invalid, we can skip it # 2. The configuration is in `unique_results`, we can get it from there # 3. The configuration is in `pending_indices_by_key`, it is duplicate in `xs` # 4. The configuration must be evaluated by the runner. @@ -163,8 +164,8 @@ def _run_configs(self, xs, check_restrictions=True): final_results.append(result) # 2. Attempt to retrieve from `unique_results` - elif key in self.tuning_options.unique_results: - result = dict(self.tuning_options.unique_results[key]) + elif key in self.unique_results: + result = dict(self.unique_results[key]) final_results.append(result) # 3. We have already seen this config in the current batch @@ -197,8 +198,12 @@ def _run_configs(self, xs, check_restrictions=True): result["benchmark_time"] = 0 # Put result in `unique_results` - self.tuning_options.unique_results[key] = result - self.results.append(result) + self.unique_results[key] = result + + for result in final_results: + # Skip if None. Result is missing if runner exhausted the budget + if result is not None: + self.results.append(result) # upon returning from this function control will be given back to the strategy, so reset the start time self.runner.last_strategy_start_time = perf_counter() @@ -243,6 +248,12 @@ def eval(self, x, check_restrictions=True): def __call__(self, x, check_restrictions=True): return self.eval(x, check_restrictions=check_restrictions) + + def get_results(self): + return self.results + + def get_num_unique_results(self): + return len(self.unique_results) def get_start_pos(self): """Get starting position for optimization.""" diff --git a/kernel_tuner/strategies/diff_evo.py b/kernel_tuner/strategies/diff_evo.py index a9819b8fc..ad1cd57c4 100644 --- a/kernel_tuner/strategies/diff_evo.py +++ b/kernel_tuner/strategies/diff_evo.py @@ -115,7 +115,7 @@ def generate_population(tune_params, max_idx, popsize, searchspace, constraint_a return population -def differential_evolution(searchspace, cost_func, bounds, popsize, maxiter, F, CR, method, constraint_aware, verbose): +def differential_evolution(searchspace, cost_func: CostFunc, bounds, popsize, maxiter, F, CR, method, constraint_aware, verbose): """ A basic implementation of the Differential Evolution algorithm. @@ -244,7 +244,7 @@ def differential_evolution(searchspace, cost_func, bounds, popsize, maxiter, F, print(f"Generation {generation + 1}, Best Cost: {best_cost:.6f}") if verbose: - print(f"Differential Evolution completed fevals={len(cost_func.tuning_options.unique_results)}") + print(f"Differential Evolution completed fevals={cost_func.get_num_unique_results()}") return {"solution": best_solution, "cost": best_cost} diff --git a/kernel_tuner/strategies/greedy_ils.py b/kernel_tuner/strategies/greedy_ils.py index d9cf67ecc..8c8a8fd3e 100644 --- a/kernel_tuner/strategies/greedy_ils.py +++ b/kernel_tuner/strategies/greedy_ils.py @@ -37,7 +37,6 @@ def tune(searchspace: Searchspace, runner, tuning_options): last_improvement = 0 while fevals < max_fevals: - try: candidate = base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=True) new_score = cost_func(candidate, check_restrictions=False) @@ -46,7 +45,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): print(e) return cost_func.results - fevals = len(tuning_options.unique_results) + fevals = cost_func.get_num_unique_results() if new_score < best_score: last_improvement = 0 else: diff --git a/kernel_tuner/strategies/greedy_mls.py b/kernel_tuner/strategies/greedy_mls.py index 4edd2f0a4..41fb046f3 100644 --- a/kernel_tuner/strategies/greedy_mls.py +++ b/kernel_tuner/strategies/greedy_mls.py @@ -36,8 +36,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): return cost_func.results candidate = searchspace.get_random_sample(1)[0] - - fevals = len(tuning_options.unique_results) + fevals = cost_func.get_num_unique_results() return cost_func.results diff --git a/kernel_tuner/strategies/pyatf_strategies.py b/kernel_tuner/strategies/pyatf_strategies.py index d0d67778b..1b82391c4 100644 --- a/kernel_tuner/strategies/pyatf_strategies.py +++ b/kernel_tuner/strategies/pyatf_strategies.py @@ -85,7 +85,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): try: # optimization loop (KT-compatible re-implementation of `make_step` from TuningRun) - while len(tuning_options.unique_results) < searchspace.size: + while cost_func.get_num_unique_results() < searchspace.size: # get new coordinates if not coordinates_or_indices: diff --git a/kernel_tuner/strategies/simulated_annealing.py b/kernel_tuner/strategies/simulated_annealing.py index 962a1e34c..b73bf0d6d 100644 --- a/kernel_tuner/strategies/simulated_annealing.py +++ b/kernel_tuner/strategies/simulated_annealing.py @@ -68,7 +68,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): pos = new_pos old_cost = new_cost - c = len(tuning_options.unique_results) + c = cost_func.get_num_unique_results() T = T_start * alpha**(max_iter/max_fevals*c) # check if solver gets stuck and if so restart from random position From da4fa976534da5cf9b1865f3db4542818e4acfc2 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 11:49:55 +0100 Subject: [PATCH 30/34] Remove `tuning_options` from `base_hillclimb` --- kernel_tuner/strategies/greedy_ils.py | 2 +- kernel_tuner/strategies/greedy_mls.py | 2 +- kernel_tuner/strategies/hillclimbers.py | 6 +----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/kernel_tuner/strategies/greedy_ils.py b/kernel_tuner/strategies/greedy_ils.py index 8c8a8fd3e..fde8bb13a 100644 --- a/kernel_tuner/strategies/greedy_ils.py +++ b/kernel_tuner/strategies/greedy_ils.py @@ -38,7 +38,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): last_improvement = 0 while fevals < max_fevals: try: - candidate = base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=True) + candidate = base_hillclimb(candidate, neighbor, max_fevals, searchspace, cost_func, restart=restart, randomize=True) new_score = cost_func(candidate, check_restrictions=False) except StopCriterionReached as e: if tuning_options.verbose: diff --git a/kernel_tuner/strategies/greedy_mls.py b/kernel_tuner/strategies/greedy_mls.py index 41fb046f3..dd02ff446 100644 --- a/kernel_tuner/strategies/greedy_mls.py +++ b/kernel_tuner/strategies/greedy_mls.py @@ -29,7 +29,7 @@ def tune(searchspace: Searchspace, runner, tuning_options): #while searching while fevals < max_fevals: try: - base_hillclimb(candidate, neighbor, max_fevals, searchspace, tuning_options, cost_func, restart=restart, randomize=randomize, order=order) + base_hillclimb(candidate, neighbor, max_fevals, searchspace, cost_func, restart=restart, randomize=randomize, order=order) except StopCriterionReached as e: if tuning_options.verbose: print(e) diff --git a/kernel_tuner/strategies/hillclimbers.py b/kernel_tuner/strategies/hillclimbers.py index 7174360e7..a46328010 100644 --- a/kernel_tuner/strategies/hillclimbers.py +++ b/kernel_tuner/strategies/hillclimbers.py @@ -4,7 +4,7 @@ from kernel_tuner.strategies.common import CostFunc -def base_hillclimb(base_sol: tuple, neighbor_method: str, max_fevals: int, searchspace: Searchspace, tuning_options, +def base_hillclimb(base_sol: tuple, neighbor_method: str, max_fevals: int, searchspace: Searchspace, cost_func: CostFunc, restart=True, randomize=True, order=None): """Hillclimbing search until max_fevals is reached or no improvement is found. @@ -25,10 +25,6 @@ def base_hillclimb(base_sol: tuple, neighbor_method: str, max_fevals: int, searc :params searchspace: The searchspace object. :type searchspace: Seachspace - :param tuning_options: A dictionary with all options regarding the tuning - process. - :type tuning_options: dict - :param cost_func: An instance of `kernel_tuner.strategies.common.CostFunc` :type runner: kernel_tuner.strategies.common.CostFunc From 40a956ee533721688a07b84895a1a0bf7f8d90c4 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 17:38:11 +0100 Subject: [PATCH 31/34] Fix `CostFunc` returning results containing `InvalidConfig` --- kernel_tuner/strategies/common.py | 16 +++++++++------- kernel_tuner/util.py | 2 +- test/strategies/test_strategies.py | 1 + test/test_common.py | 2 ++ 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index 3cb6e84e5..f3bb06bd4 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -146,13 +146,14 @@ def _run_configs(self, xs, check_restrictions=True): batch_keys = [] # The keys of the configs to run pending_indices_by_key = dict() # Maps key => where to store result in `final_results` final_results = [] # List returned to the user + legal_indices = [] # Indices in `final_results` that are legal # Loop over all configurations. For each configurations there are four cases: # 1. The configuration is invalid, we can skip it # 2. The configuration is in `unique_results`, we can get it from there # 3. The configuration is in `pending_indices_by_key`, it is duplicate in `xs` # 4. The configuration must be evaluated by the runner. - for x in xs: + for index, x in enumerate(xs): config, is_legal = self._normalize_and_validate_config(x, check_restrictions=check_restrictions) logging.debug("normalize config: %s -> %s (legal: %s)", str(x), str(config), is_legal) key = ",".join([str(i) for i in config]) @@ -166,18 +167,19 @@ def _run_configs(self, xs, check_restrictions=True): # 2. Attempt to retrieve from `unique_results` elif key in self.unique_results: result = dict(self.unique_results[key]) + legal_indices.append(index) final_results.append(result) # 3. We have already seen this config in the current batch elif key in pending_indices_by_key: - pending_indices_by_key[key].append(len(final_results)) + pending_indices_by_key[key].append(index) final_results.append(None) # 4. A new config, we must evaluate this else: batch_keys.append(key) batch_configs.append(config) - pending_indices_by_key[key] = [len(final_results)] + pending_indices_by_key[key] = [index] final_results.append(None) # compile and benchmark the batch @@ -190,6 +192,7 @@ def _run_configs(self, xs, check_restrictions=True): # set in the results array for index in pending_indices_by_key[key]: + legal_indices.append(index) final_results[index] = dict(result) # Disable the timings. Only the first result must get these. @@ -200,10 +203,9 @@ def _run_configs(self, xs, check_restrictions=True): # Put result in `unique_results` self.unique_results[key] = result - for result in final_results: - # Skip if None. Result is missing if runner exhausted the budget - if result is not None: - self.results.append(result) + # Only things in `legal_indices` are valid results + for index in sorted(legal_indices): + self.results.append(final_results[index]) # upon returning from this function control will be given back to the strategy, so reset the start time self.runner.last_strategy_start_time = perf_counter() diff --git a/kernel_tuner/util.py b/kernel_tuner/util.py index 0c456cec9..99c2a2f69 100644 --- a/kernel_tuner/util.py +++ b/kernel_tuner/util.py @@ -732,7 +732,7 @@ def process_metrics(params, metrics): :rtype: dict """ - if metrics: + if metrics is not None: if not isinstance(metrics, dict): raise ValueError("metrics should be a dictionary to preserve order and support composability") for k, v in metrics.items(): diff --git a/test/strategies/test_strategies.py b/test/strategies/test_strategies.py index 8f77a9516..63d01dbd3 100644 --- a/test/strategies/test_strategies.py +++ b/test/strategies/test_strategies.py @@ -53,6 +53,7 @@ def vector_add(): strategies.append(pytest.param(s, marks=skip_if_no_pyatf)) else: strategies.append(s) + @pytest.mark.parametrize('strategy', strategies) def test_strategies(vector_add, strategy): options = dict(popsize=5, neighbor='adjacent') diff --git a/test/test_common.py b/test/test_common.py index 40d25de81..e23a55882 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -4,6 +4,7 @@ import pytest import kernel_tuner.strategies.common as common +import kernel_tuner.util from kernel_tuner.interface import Options from kernel_tuner.searchspace import Searchspace @@ -13,6 +14,7 @@ def tuning_options(): tuning_options["strategy_options"] = {} tuning_options["objective"] = "time" tuning_options["objective_higher_is_better"] = False + tuning_options["budget"] = kernel_tuner.util.TuningBudget() return tuning_options From 2b61ddb5a91aa2d3f00a670fa9a71071f9323821 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 17:51:07 +0100 Subject: [PATCH 32/34] Fix `random_sample` incorrectly sampling too many configurations --- kernel_tuner/interface.py | 10 +++++----- test/strategies/test_common.py | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/kernel_tuner/interface.py b/kernel_tuner/interface.py index f874e701b..b05fd5350 100644 --- a/kernel_tuner/interface.py +++ b/kernel_tuner/interface.py @@ -625,11 +625,14 @@ def tune_kernel( if strategy_options: if "max_fevals" in strategy_options: max_fevals = strategy_options["max_fevals"] + tuning_options["max_fevals"] = max_fevals # TODO: Is this used? if "time_limit" in strategy_options: time_limit = strategy_options["time_limit"] + tuning_options["time_limit"] = time_limit # TODO: Is this used? if "searchspace_construction_options" in strategy_options: searchspace_construction_options = strategy_options["searchspace_construction_options"] + # log the user inputs logging.debug("tune_kernel called") logging.debug("kernel_options: %s", util.get_config_string(kernel_options)) @@ -720,11 +723,8 @@ def preprocess_cache(filepath): logging.info(f"evaluation limit has been adjusted from {max_fevals} to {searchspace.size} (search space size)") max_fevals = searchspace.size - # Create the budget. Add the time spent on startup to the budget - budget = util.TuningBudget(time_limit, max_fevals) - tuning_options["time_limit"] = time_limit # TODO: Is this used? - tuning_options["max_fevals"] = max_fevals # TODO: Is this used? - tuning_options["budget"] = budget + # Create the budget + tuning_options["budget"] = util.TuningBudget(time_limit, max_fevals) # call the strategy to execute the tuning process diff --git a/test/strategies/test_common.py b/test/strategies/test_common.py index 90f6c63e7..75da20c56 100644 --- a/test/strategies/test_common.py +++ b/test/strategies/test_common.py @@ -7,7 +7,7 @@ from kernel_tuner.searchspace import Searchspace from kernel_tuner.strategies import common from kernel_tuner.strategies.common import CostFunc -from kernel_tuner.util import StopCriterionReached +from kernel_tuner.util import StopCriterionReached, TuningBudget try: from mock import Mock @@ -30,7 +30,7 @@ def fake_runner(): def test_cost_func(): x = [1, 4] - tuning_options = Options(scaling=False, snap=False, tune_params=tune_params, + tuning_options = Options(tune_params=tune_params, budget=TuningBudget(), restrictions=None, strategy_options={}, cache={}, unique_results={}, objective="time", objective_higher_is_better=False, metrics=None) runner = fake_runner() @@ -41,7 +41,7 @@ def test_cost_func(): # check if restrictions are properly handled def restrictions(x, y): return False - tuning_options = Options(scaling=False, snap=False, tune_params=tune_params, + tuning_options = Options(tune_params=tune_params, budget=TuningBudget(), restrictions=restrictions, strategy_options={}, verbose=True, cache={}, unique_results={}, objective="time", objective_higher_is_better=False, metrics=None) From 9f53715a35c8ac96b67b2d2777199cc96e28d327 Mon Sep 17 00:00:00 2001 From: stijn Date: Mon, 2 Feb 2026 18:48:41 +0100 Subject: [PATCH 33/34] Fix `test_cost_func` --- kernel_tuner/strategies/common.py | 4 ---- test/strategies/test_common.py | 5 ++--- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/kernel_tuner/strategies/common.py b/kernel_tuner/strategies/common.py index f3bb06bd4..0964597d3 100644 --- a/kernel_tuner/strategies/common.py +++ b/kernel_tuner/strategies/common.py @@ -87,10 +87,6 @@ def __init__( """ self.searchspace = searchspace self.tuning_options = tuning_options - if isinstance(self.tuning_options, dict): - self.tuning_options["max_fevals"] = min( - tuning_options["max_fevals"] if "max_fevals" in tuning_options else np.inf, searchspace.size - ) self.objective = tuning_options.objective self.objective_higher_is_better = tuning_options.objective_higher_is_better self.constraint_aware = bool(tuning_options.strategy_options.get("constraint_aware")) diff --git a/test/strategies/test_common.py b/test/strategies/test_common.py index 75da20c56..54ecde6fc 100644 --- a/test/strategies/test_common.py +++ b/test/strategies/test_common.py @@ -46,9 +46,8 @@ def restrictions(x, y): verbose=True, cache={}, unique_results={}, objective="time", objective_higher_is_better=False, metrics=None) - with raises(StopCriterionReached): - time = CostFunc(Searchspace(tune_params, restrictions, 1024), tuning_options, runner)(x) - assert time == sys.float_info.max + time = CostFunc(Searchspace(tune_params, restrictions, 1024), tuning_options, runner)(x) + assert time == sys.float_info.max def test_setup_method_arguments(): From d7eb725437e66663e1fa8daf4f3c45a6f11ea9d1 Mon Sep 17 00:00:00 2001 From: stijn Date: Tue, 3 Feb 2026 18:21:47 +0100 Subject: [PATCH 34/34] Add page on parallel tuning to documentation --- doc/source/contents.rst | 1 + doc/source/launch_ray.sh | 34 ++++++++ doc/source/parallel.rst | 143 +++++++++++++++++++++++++++++++++ doc/source/parallel_runner.png | Bin 0 -> 199587 bytes doc/source/submit_ray.sh | 26 ++++++ 5 files changed, 204 insertions(+) create mode 100644 doc/source/launch_ray.sh create mode 100644 doc/source/parallel.rst create mode 100644 doc/source/parallel_runner.png create mode 100644 doc/source/submit_ray.sh diff --git a/doc/source/contents.rst b/doc/source/contents.rst index cac6d59aa..bd909225c 100644 --- a/doc/source/contents.rst +++ b/doc/source/contents.rst @@ -36,6 +36,7 @@ The Kernel Tuner documentation optimization metrics observers + parallel .. toctree:: :maxdepth: 1 diff --git a/doc/source/launch_ray.sh b/doc/source/launch_ray.sh new file mode 100644 index 000000000..954d5f112 --- /dev/null +++ b/doc/source/launch_ray.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Get SLURM variables +NODELIST="${SLURM_STEP_NODELIST:-${SLURM_JOB_NODELIST:-}}" +NUM_NODES="${SLURM_STEP_NUM_NODES:-${SLURM_JOB_NUM_NODES:-}}" + +if [[ -z "$NODELIST" || -z "$NUM_NODES" ]]; then + echo "ERROR: Not running under Slurm (missing SLURM_* vars)." + exit 1 +fi + +# Get head node +NODES=$(scontrol show hostnames "$NODELIST") +NODES_ARRAY=($NODES) +RAY_IP="${NODES_ARRAY[0]}" +RAY_PORT="${RAY_PORT:-6379}" +RAY_ADDRESS="${RAY_IP}:${RAY_PORT}" + +# Ensure command exists (Ray >= 2.49 per docs) +if ! ray symmetric-run --help >/dev/null 2>&1; then + echo "ERROR: 'ray symmetric-run' not available. Check Ray installation (needs Ray 2.49+)." + exit 1 +fi + +# Launch cluster! +echo "Ray head node: $RAY_ADDRESS" + +exec ray symmetric-run \ + --address "$RAY_ADDRESS" \ + --min-nodes "$NUM_NODES" \ + -- \ + "$@" + diff --git a/doc/source/parallel.rst b/doc/source/parallel.rst new file mode 100644 index 000000000..f1e83f099 --- /dev/null +++ b/doc/source/parallel.rst @@ -0,0 +1,143 @@ +Parallel and Remote Tuning +========================== + +By default, Kernel Tuner benchmarks GPU kernel configurations sequentially on a single local GPU. +While this works well for small tuning problems, it can become a bottleneck for larger search spaces. + +.. image:: parallel_runner.png + :width: 700px + :alt: Example of sequential versus parallel tuning. + + +Kernel Tuner also supports **parallel tuning**, allowing multiple GPUs to evaluate kernel configurations in parallel. +The same mechanism can be used for **remote tuning**, where Kernel Tuner runs on a host system while one or more GPUs are located on remote machines. + +Parallel/remote tuning is implemented using `Ray `_ and works on both local multi-GPU systems and distributed clusters. + +How to use +---------- + +To enable parallel tuning, pass the ``parallel_workers`` argument to ``tune_kernel``: + +.. code-block:: python + + kernel_tuner.tune_kernel( + "vector_add", + kernel_string, + size, + args, + tune_params, + parallel_workers=True, + ) + +If ``parallel_workers`` is set to ``True``, Kernel Tuner will use all available Ray workers for tuning. +Alternatively, ``parallel_workers`` can be set to an integer ``n`` to use exactly ``n`` workers. + + +Parallel tuning and optimization strategies +------------------------------------------- + +The achievable speedup from using multiple GPUs depends in part on the **optimization strategy** used during tuning. + +Some optimization strategies support **maximum parallelism** and can evaluate all configurations independently. +Other strategies support **limited parallelism**, typically by repeatly evaluating a fixed-size population of configurations in parallel. +Finally, some strategies are **inherently sequential** and always evaluate configurations one by one, providing no parallelism. + +The current optimization strategies can be grouped as follows: + +* **Maximum parallelism**: + ``brute_force``, ``random_sample`` + +* **Limited parallelism**: + ``genetic_algorithm``, ``pso``, ``diff_evo``, ``firefly_algorithm`` + +* **No parallelism**: + ``minimize``, ``basinhopping``, ``greedy_mls``, ``ordered_greedy_mls``, + ``greedy_ils``, ``dual_annealing``, ``mls``, + ``simulated_annealing``, ``bayes_opt`` + + + +Setting up Ray +-------------- + +Kernel Tuner uses `Ray `_ to distribute kernel evaluations across multiple GPUs. +ay is an open-source framework for distributed computing in Python. + +To use parallel tuning, you must first install Ray itself: + +.. code-block:: bash + + $ pip install ray + +Next, you must set up a Ray cluster. +Kernel Tuner will internally attempt to connect to an existing cluster by calling: + +.. code-block:: python + + ray.init(address="auto") + +Refer to the Ray documentation for details on how ``ray.init()`` connects to a local or remote cluster +(`documentation `_). +For example, you can set the ``RAY_ADDRESS`` environment variable to point to the address of a remote Ray head node. +Alternatively, you may manually call ``ray.init(address="your_head_node_ip:6379")`` before calling ``tune_kernel``. + +Here are some common ways to set up your cluster: + + +Local multi-GPU machine +*********************** + +By default, on a machine with multiple GPUs, Ray will start a temporary local cluster and automatically detect all available GPUs. +Kernel Tuner can then use these GPUs in parallel for tuning. + + +Distributed cluster with SLURM (easy, Ray ≥2.49) +************************************************ + +The most straightforward way to use Ray on a SLURM cluster is to use the ``ray symmetric-run`` command, available from Ray **2.49** onwards. +This launches a Ray environment, runs your script, and then shuts it down again. + +Consider the following script ``launch_ray.sh``. + +.. literalinclude:: launch_ray.sh + :language: bash + +Next, run your Kernel Tuner script using ``srun``. +The exact command depends on your cluster. +In the example below, ``-N4`` indicates 4 nodes and ``--gres=gpu:1`` indicates 1 GPU per node. + +.. code-block:: bash + + $ srun -N4 --gres=gpu:1 launch_ray.sh python3 my_tuning_script.py + + +Distributed Cluster with SLURM (manual, Ray <2.49) +************************************************** + +An alternative way to use Ray on SLURM is to launch a Ray cluster, obtain the IP address of the head node, and the connect to it remotely. + +Consider the following sbatch script ``submit_ray.sh``. + +.. literalinclude:: submit_ray.sh + :language: bash + +Next, submit your job using ``sbatch``. + +.. code-block:: bash + + $ sbatch submit_ray.sh + Submitted batch job 1223577 + +After this, inspect the file `slurm-1223577.out` and search for the following line: + +.. code-block:: + + $ grep RAY_ADDRESS slurm-1223577.out + Launching head node: RAY_ADDRESS=145.184.221.164:6379 + +Finally, launch your application using: + +.. code-block:: + + RAY_ADDRESS=145.184.221.164:6379 python my_tuning_script.py diff --git a/doc/source/parallel_runner.png b/doc/source/parallel_runner.png new file mode 100644 index 0000000000000000000000000000000000000000..30785a631f43737e4913deb48041ba1d1de28484 GIT binary patch literal 199587 zcmeFaSFZC~x*%3)7>3b6zx9*f3$P#hf;l`gh@?b{L82J+(qN`2<|uLo9*1Y(*%UM`}`~FATk`@9|K?x+&;R-lfB3_{IcxIm4}bU{ z{*OQW;a~mV|I_~nE%pETf8GA@um0`-KA(cQ`Q)!U`@DRYE5?I+y^B+n1j|36* z{Xe9ys2HPW`iC}~p$V=>Mb>_IxOT^@uA@H^3jGflT>4)E9RDLJ!Z+j9E+6>(aLm7F zs9$a4`bSd03GN@T{}ryNS@Q1SP6ER*c;V2byv8i=;5Ox}*RSc;Y@A+Q`)fr6{}1Fp zFuyKLNmOUw&A(dWZcMZBR}+nz$kQJ&0R~1dQu^Ks{Ai=*`%W8n^Y378w6RzE>(zCvzJ8huXV#{#`ry^$*&odF#ZwmQt$i_MQ zp>Ndm*Aew6bg5a_Wb?QK%F5r74F3)2?N8+O8xeg^L;aB3Hxp4L|HJ*!H+>hs4UV|x zZ=nI`^G!m3eFI1&{v%2|eYl<&ZgGQ&a!6&`)45&mv8hE z2!RNVe?Q3m3-aNw@h)P{|2W=Fqm0Xw-#GF2JeqJ>od0&^pW#tG8{01xio~jvD81sDEHm;&9FZ|e-Qe^zkfdn{jv)P>|eO`&$;ynaeqY^6Xf4Lw|-lOe=BnT z|IV$%4@3X`vMWLTE!q{N9N3M&w~+pl5dL#3sO?%XZ~n}b`fiHqSAqPE1WZEYPKZd3 zzQ8mVzArSdG(~os_~%C3ukrsGIjte@>c5C|{$bC*!oq+5`j+^cG@8mU{|QO;-}y?Y-HvA9WW>{8yg{hp z$0PdTRn+q5y!_3gf9?bQb${WXb$EV^p)sqY`L%vKSKsF3@3{7--(NDC{+1>DX(4v^ zf=~@aVPNO5BjU$L|0W0eNB;lJGvta=Z_=-@h zrGWs~H&6U$b}Ie`G4_wt)<2=4|MoEeg8TnCG1$=<)%BNy`CpY3s2}q95n(^SzeU*J zAuar@@V^Ja@Hd&Pw+Myq{BC^zmkKQWfb4fl@W0DY;7^SIucQ7?mIji=f5}t%o%!;Q zLcJ-SaVhni0rR_X5l6%%!~AyT$F%tm2>1Uovi!$#`5hjA^!;BB5Ru4m_&baE{U;iv zV88yT*sPAb?ewPx@v8xBupdVfu=G0vHmRd&dZ8HL@9>^1-N&0Ouyeuu|E$j{Sd`}GqV@}A+jGY z=qK~=V}t(ZH+%o_l>daz|GPDN$-l|$g`8iH{F_(vm1g`cf?`B*R@>e5nt!Eq<8Ge2 z2B^437sM!e7hkmgBiGU_kCqz6c79mrCk8WjC?^(8{jcO_{`w$p5x*uBep(T?d@n&x zqd9{7^RJ(T(znGQ3F+mD)@{Rdwde%+XWk#>=~2Kp^cK#7B!uuOW0~nPs72IF>Fzma zNhr>_P`+PZy~NAyq&zZ(om;2f3_SH{oO8B5bmp=Yq?7d#`^%@pdaiGtw3K2K*y zfJL5RcM@#JDbb#OT|iAI_*s5k_?N!_0*8O02gES{mH6TfGUkzLS72Fgbv?tMbE)Gp zay-?g?3_T7K;ANhx0WuX!tn83e%=%B?X_?tZ8fD#9Ulq#My#z#lb8m zN-Vx*z~yz>WZDu$_h3PcC@1B(eoF#L)QfgvwwsW+aNo9fN1^=44^?~#>$;s(A=p~D zYPR=G%ME`#t`zRfRK}r1Fj6#_t{J~{&arKDjZP≥`vmHa4Avoh!naIYnM*ienIH9$AS?G z)k+1z7>Y1ela$-9@$AA#gr;adIw3q?^z<`NeUl62Egt9b?P1ql8E@ax^sG#x`H4^F zVrgOuz{sWe6hE&@G3s?qg}*`WaNJ}I2GUEB^iKITVbp2w9&bf-ek8Rd9s=}=M*v-} z-o`wwVm9jDf0EwsOCwmgKrV?j7g9LH-|z#8|G24=E`@*?Tsjj6Z2L71x=~2Kw1F|6 z7s0}63=7JTCG&_@Qc-C%>^ItY!wQ9XG)9$Dm2s=H?N|>a%(5dSraNoL0bc(BxoHj}M zM1&wrt#bLHM_n>EWpf3ywX^$LTzpKNZrE=+t_Ub8!6+e_DSeT8$zp+16kF~8r1?zo z6ke%IVI2tK=WRfMd&0tC+8#@D1Ihp zxLP_$G>!R5({uecwC=Yx(ws?Q;q1GE>3gu@g)l<&SCkT=$2<*-ulD?0mQ4MYbH)9A z$)zx|ckiD`z89(v6ZwhCGyTb1-j710*6@(zgX-dge=BdHjd(P2xqo&YB}QUqka9Bjg!tbGHbpHO1qUn#8=wI=C9j z*kVv%r3?Yfa*=JLwVc)Z^eVwkSC&xU6rGg=`ZqZu3zw@Pe5e6wl`nhk?%zA%vMJ}I zdXo7ZgkvFXsLjUY{?d!^juTPJ~Ly$Yx8ENS{T6dYCR#Y-U=vIn9;6tHhu zAZ(H>h*t~b*bmn$#UR{6;m%=6{)R>UnhD5Z6J+hx!A^cjq{~w;gjRS|mPUSlENvBy z1{-0%Hu6Os(E~=)H~e~(=JpFG&Q7E?f=rRZT~F#;LMsMkDb-YNzBbkRClS>CyzPZKg zsZwzBr`|UvJWa)qw3i(xlhY~V%JU;lWM7aT7I&hq(QUe9C)?<%Dcb?)zu?J>8s#bu zWn}k!eoea&m50!Z`^9SUvv=%3k>cvJ=j4~=adV`mxF>3Bn6)Ej0(OXI!=*uffu+RT z3xSe7lF30DLo5#GwkVv+pEI{}pMYMgzsx&JuZQu@DQ{cR@Y!k!moI$B4`(@XC$3-O z5bTl1HJJs8rAO!#Z0Cad^4rpz?GT$nN@D}eAlFEf`KRQp81Q8YjzUPjw#Ij=fcH1L zER0%uOSHa83~lq;IfQ=C@A7bVVG7%6tGW|NchT5rI#zl-dPApQ zJb}df{Y1$ZLSaTKSWf{yh9wd{JTgON^+yJ!qD7f;A}obg6Z(wy*p7P3pg;zkh9zHLvx2d%-VqRExEIG zqISYHgcPlCle>#6dkF3ido1l&&enLYEa#}fjNFaZ@j;tBzzIg+PCFNrPa>Y(w`hJ_ zzpCtsdKj)71&bU-7qqh2Qy$oSqDPBMIgho&oGOZZ#7r9DOa2{h>9R+C8H zeMuX!$hU$DsS;BuWC-vvH-1!{lUGs|gy)oMWYU=W%AK#rHM7vRDB<+@JV4sW!o6tU zDk7m}3rtV~iUn`??6WeOsbRAq;A@*;OH#g&9EMzoUpOkR=NlNl+NicGL&wnm-T)+u zxNr|8JB8FHp`YfqcFu+*n=Cv81L;D|H|>->0Hxq6!q-u+`&9bn#0($27g?R zADeo75_e=Uw|c;l6Ev4!TEdqTF{L-=cFI?1j3={BFC>PqI=eG$U|-0If`bX-7Z2tS zml1pIv4$xo^f1_bXZQE#E;QNop^~;wJrNKaPkwwEtk!lBf^@RRJ2&$4`5s{8v3oqp z#^wc9f$W7}s~{v!!q-Kfg55;k~_0AM8CriZ$%32R%?K85QCsX}ROod2(Ujk{2 zEP6QyiM4p-*?s6%{FG-lzX)$ax{zIrEGfW`!UhBhAMUkiofRTe+O<>!6nD}M4bf&O z*oE7f;5dCrj3K%+434eo%zbWx&5z{~jL#lQ%0o@t*4xvPsuIS77b3wu$PYDAN(jys zX$R)OX}hvFh_N*vW=E1B*be%xUihIUfI2v3;HgwupeI%bvO39Y zv?e2G+xvsC8cujinNv*Jdc34b#=K3iZWY{Cz0u&z7KPe_{H`+qz&P7n%CCd|TQ%dYCr`aliON@A1l=($C zr~EkdIj?6*BnGEDW=D-gBfSHsMU3@|}#ylDKN6xff)V zuxDhYcO`Y9Nm3ROPL9ShTW~~rWu0I{P)=9>;)2{b;em?Ir$Yn6O zj~W~8nb?T%lRWNRlAwqJW8?&}}~vY!Y83 zI|S{gOwBsC9-ln;(T~aYCXh%cR&5xSi7w^3>;QFKUUs2PR&o$Hn@(mUYSThuF3B82 zwUuMA^`C@73ds}@f;FSP?cEt}6br2|mL)lGY<#XJkk*8>9pMK1F_;`T8XNJslAdPw z%-y1Lr>0V_o}Rf9#?gD8hXtdAe##nUGEVZk+_BO{lgT- z8G>CF#JFHW@Gr!i7Rk2vCoPnFhZxyDZ)04N zbnAgj5wioyR_Et1@GReOmn^8xSMObMjeTK4`t_i8-E&Viow>e8Fo0C6kX3@dMz|Q@ zX~&-+6y#iFDXw_(pe}OYQ8P=Wo63Cg4Y$NVSYolz<^u-;2OLp81=jZvd^)22^!tD{ zuVox>oy*QRU{TS{#Gi<(SAdom;|CJ$(_DB6!Rdoa8Qh5Qoe(eB;SqjG&JewWyBZ?T zm3{yH4%wI~2Ow6Z(oVE1o&q)r`O}1bko*(vLPQ>pIt&DlB=>@I*NONYFM%uEN*!dOO8ee>^UEO3)q+PL6xg(rZMl^Z9ayK=R9D>EJRU4j@x|Zm7XR zj(@6y1Wq*u!H4H%E)Oc>F=`|zPu&~*qRuSt=t*3 zGc0#KBuI$MV|Q*X@ApJo!s3NIQK-^@o8s4-1xH@Q#@OVsybtIhy5b)eZ&$YB#@@QO z5uHWBadNFg!GK`J%Jp#GY4;N7SZt4&hrx*TrQfzYBc2X?SKU0PrW3aquaFf?OT^Zv z$#2<69tkky{6RCbZjP_vU}-6Lpyfal{0(if|2!S%P^O8uQ;LI{TRuSKW?v43193x_GzFJ0Fns8Pu?N|z9lljuK*S?yT%+$3WLnR z--Wg$gp}OO&-N$yldH#mz{PHl2=NBITx4S(I#!Rz7dyBkf=yu$$lL_A^Wj(v!DQ^} zIL(`lgXR)8{l>!LWKF>hz7KTfy_s^)j_I zf^;EC(1(UzHJa?wX?>Y+H5&-FYDkURSWPrK>21JBtY)&0nYfUT53#8rMDIk*j`=Xq zLCFBy2jyk9Nx_?Dob}2vACq4HCZkjZ0)lfkUt zfi-c77kPQfGZ^67eV$0t=RHJE#OVUtLG)O~ksZZef>R7mw^=;3df6JU&%Csw^-&** z<6?eRjF;&!FHOckSeHuf%xi`?PMo9@)5dIXsr#Muu`y-uY>WVE!n+>h{7nR7L#&56 z8S3dyUE}y;WK9gAvSiUzB0_8NkB73lpiRcD=Il$cfm|$uhqLjB04PvkD@net+6H}l zb8JHx93i4lXXL^lBgV6Lj5uuXLhjyGSVzW1EDL>tm_j7rl8|rURi?h$gVa)G-n<;I zkrNzs+gT+)K75D0m8ZR_@VH<)tzvl0MR{xZiO* zZSD3#q)~$OFf^+#mJZ<*a^hs4(g#`+jYjk7b!b*P-5X+q5;0KN;?sn=4m4NdjlZ9Z z!kKlLAaqy8)u-asB@@s4<%v(NN7DOTn=j7f02rbVSOs}#&~p@GCz~B>{)&0Iyz6&k z=X29vTK=pqRvWgdrC`=G+he#5T&|o$-b{PtlN8dIc-IO(YUmplv^}4&PX}ouIll(| zHa0T8+`t~)p7JAKss}~KLRKviw!uhu9?KvW-2DEo(?qi>9R8nxX_=%K#b?j@%b%4o-8P>)W4kLJo6v9c07xX zwut8pp<8mE9nJ-b+)qmG8XnLU(aLRa6G-zObtkg6F2PqQ!vYYA2Pb{_svq|p$W8o+ z2viD?7pM{-%J1&1wxcTJ<07Qpsp6g`SL7uG>(V)Ae6ggf_#8CPwE2O_z(m} zhK&fCKaItlv+(}S!4Ne^d-A&26&6vKSjvPnLcUcu&`>b<4_M&e4#hpJ^Z@(z02Youz2j6IH{g7~x z4e3iHq$Eus!wqi38GMlgigJ5l4Q(fT%;>`7*Tv-oWjW-cgU{ditl0(EZEbP82;|i^`Lw}p z+=ZL&xAy2^PLXH&btj0x68y4bReLaY*qf5DX26dA-g5Amhp(k6r<2p_&tMtS_CC@g zRvKGDCyyJqcsJ}0LrI<*TyX`yD$x-n+>vPec1?V8XcK|k71r#a@*g@L!{r#s1tYti zk`3VKvTtIH5F9Sg7yMl~c?Jo!V=WWFOmQQ>lKGVm8~(`}^_`oT!NmgN0K+nm+ z4{IxTM}9oG3qb#%=)0I?(F6*S9Wo$Zv9=`h2oBhT0+uRZ^3G*)9tXR%@|X-FKnZR$ zqU0f`m#%?q15q#47+vr3CCM7`0(_3&Gm;)d9eRt2%A9v>1Q`ev%s372uk}=x9CnZI zm3561P8{TZ)caT%I9!XnXa zoJcgwb{fLqqD3yR1qRcjB%IQObvh5~P~J0@34p|u2ZBrea*SJIic@fe?y|g|XvAi9yaQizMHZyO0Ea8#93lEZwKJXNk`ePry z8L#n^_z=w6W?Fi#WW096Y)EI^9&1Gy+y~!3?=I4HLmh85q&^|aFua1A1ho@8AOb1? zw<1@3-^;x9e#&cimmUT6A#Z*XkV`Bd^3kOA-mTH@DTBsIM96+FV*4D0v8mCxGD zMp6u%OXa#LiCiuC94ShM(bnsN4JS>wEE?!GMO|kNnb+{jla zj@P~bvOF-%TqaDQn#tF&9~AMUg!>=FsS&FOPfz09ta#Fln_LA z8>v);&0w%W3xxA`nA*;(=NdpBt5QBUyJWidc6n1!w^Y8Uv%3mK9+SwvRfL8d3m716t>E>I9iJQV)0W6Mp1KlVc1N3?1=(`)oWi{&JFD zF7C#v+E(Vg=JTYdOGiZ6$IQ1+|H9Q9)h4T$JLQ_3mZ}r2EpihXF(N~#X~tXf79&P& zpF*@r7i$mYMn!^mOa}wlVkGeFCFjJwVaKbPN~zjuo%e`>9d{X;@>c%XuGvmCAhN}$ zFJlO0x?`{GowwMvW4${)#CZzI=?ZF_K3DX_%2K^9wmhbH{cB@o9&NGaQ?|GnPq4 zycMq>`xy^Sh7^EB*#QvULoy1DTuv`y+{|!?(j;QjnBsys=3E2IhT2C0>J0UCe@p{@CdV~6X%nZ-3xgQ4*eTql&Di^AV z#}UF~EQx0xpR+5nW*Wz(5cHo!uumX_M175Ow8Qg^>Ciz$+HNpsT#Aj!ysCEK%2Dm) zb^V<7J~oQ_ec60{p`5#gS8?v^0)Zcf=Cdj$4Q>2n#ubODmGpYF8lr-5%x^-=9a1i+ z#|gP4`T?xZj#hIR*v?;d{$vNe%sBm-i{~4X6ifUZv6Z86yv2Bm5fB8?T7`W|9kpeb zt#!7Sx}aR9lE<;@kk}@z%HcyI7_lxie_mzF^Z|b}49(6?ZAlB3?r-9Sz2BOgjMZ0V zrm^>)w1p=;K#Q&(1Qb&?`@Flc)#Syek7 z>-m~Brd7Hs_K|KcABU(CT*WYQx0Q`q0bo&mKisXyAcY4Z@;(9(iNBie0)aPb%9~MB- zJ+N1A&Eef_0)|;0mwchfMGFdY>bbbbQ*$}_+qs@_f{bH?awUYpmQ;{x*5_I%3t*v< zUT^-DwUKg(7Yz#L7Lb5Vf%@}~KTBUlH*~oj%wmqZ2n<7a)1#{QJ0Y01A4RHSun*PZ zM`Znku4OOac<3#t!!i9jyx`yGU@g)Xj5_}Ots?E zq>7z`ij!Fda}v6CC(g*{J=onit$4X@mazBb`6l`7i20;_BR0V{%j3aNx2Ua0`^4-e z6u2P*Q?>KD(z-zc#X@fWniI`-VXe6Zy_Rk>-(U!F(J#J_##x7G?{`v! znzEw?!zZ(nwK2vWc=kY6>CEmis3-}8aYDpZ{@B`F<_{=2pj6wzEi4KVh*gp6O{e5@Y!BJmT%%;2<$zl>_iN(|dk?MyJmWV8`@6>{tpX6UMy2BJ2(P)JjjPt;}~<>)l%*IlVTnZ-hH&`>ftor1Vq)Q z=g+~tuI-tEryPy_VpsvG;Jv1ePc(zi^N?@>2RHvZ7y&KQ(MmgT0*ecWeOh{m2vP^( zwhp^F6>m{M%Y<(|?~sz6gMhT`W_($pfz`@EOWe7*w#dAN?icL8(dZO4Bl)~7hQC-3OS9*AB4dLZlX^otpz7I9y*+?89Ob+Ar zL%i{0?oEx7TLjS3$AUzFL>`k*=3l^UAc)~W#F|FHbdS^?E+-bt`*!V)opBj+zyB^Drf(W5qnNfrnpa zeT}43Zf7n%G~0}o!s%Md=c}WgAqtlPR75S4NzeYY*SBNwI;cUV682m6(jfQz;4R6@ zFN!_A$r<_<;&oJqH7@yMA&)t?pZtSg#ilTk|HCMr2PUq5BA~WQ-jwpQQ(#qzjhY=$ z%zV%KmZl#AyWGbr=ApWxU6{!u?n_Q9i}mhIPlk8^*l4ZHk_K%GayZQ~0$fEwU|B%P z196u&1*%yLM+}gx@gBs;_+8+E7fPzcG_%*qrJH$>Q4OSsDV-X-EUY=I>?P2$)eZ~9 z@;5u)T6l1uY)r--I=+|};xBm3kUhq8&S58Ufq0zx#f1ss7Rs|8;5x+k-uRwE)Ky8s zh&1|_m+_w^BKbut+7BLya5vse43@tkpkn?ph=gq|e=iaWg`$%2YgILlU z2UlaW5Z2ZekS#qi-8+Fb(a_B0v96BM)mEk@9TC6~ZHi!2-+;};vz=BBY*SHSe*tZ} zEqPs$;(?|y4x!>>mC@!-*D++f39|2>;cXSwoOOJK+4FG~7dw5qxp`sGsIjLnJl3aG z&BtVbjWl8lwac|g-?{zM!s9IY5o3F?k1`kk;83*RRAsj0mr#nhmr!q_4k`O|gMuFa z@N6PO=LWra%kCOyT#FC(k!X8$-F#K~3?O;RY9de9Hrk z3=qFU^@J#GXhJ>^!~@HMI}S3bZbM;t#SESY8IGoKj+(Ss46}2z`&bT&^7>V`s^6&h zXKXL#%LFWiL2A8hp3L$#TXTiIBifT7u5N;^vkCb%R zu^rY6VhqS(8i?wGkYLZ6K%>s5-CE2`TJo@kDA@_~%V7`g?iOOVDNy9XpzVMXp|%%Q;*OF{>`{PesRc26?25O0u=X$_ zwp!Xqtx)gc7WTQPB{ytC+eVL>5SI)=%NSh;j6el6nVCtak4_og({HmNGd;F~Z18wd zg^bvzxAXou4j4U9FeJTY`X+t)GM?@;Ds)ry6BWGofF(;W7+ZAIT=5`u<0R5~gW}m+ zAsX=&2uN8vB(8%sUWfIu61l@{JXkThdX;RBgA$3)0EvV`IM2V&n!yPfFu#R80yC?p+?WT zQjQzsa6J|W~jqCUlTZg22`plA-N_rzqsHplw{HD+iWc<=B|g3Qy`Tu0nqThTog zNP=&-54hAP2Ii38FKsdt(U@-ST_zwTNqYW3N+1K>rEVROUun$Dc44y|*y!b2u~ksC z#XUcjF}AkNoYV)E){nK!D7`0hpeAEIXkeLV8{*F%WQ3S|k1H0Dz-V0LQcdx)A1<}c z##YiduBgu`m?=%}*t;$iw|S@-lwwh~__adjUK1wKcbFo-iu`FE-+TEU*431>My7^D zc`#hWj=~m*LksbZ{C;jC9BI3iSktjm=^F;3NjZNy1d?Bs$1(c^oH}>O5R_1OyBE`k zYu)i&+UagiSUcoACV8SK@z+l)Upu7(mg68h2pzD{Ot+=a-$NR}KtGgIX&7 znbEq*rTO6bUkiIJGQPd@v~T6tiad^ae;(qxXAp^jX=CkzC5zE5=NUv1tWlh)4`anP z-X<=CY9Ll-hVTe?Vc*VkMw@V16D!(wmz&S|xfKwXkxg~TmJ54u=8!9X zp~U=~0C2%!r2>ayOE5eZ8_85KVt~-awkPeEB4vn*&SVpC46piq`0LObDh{9tqqrwcl0Q{ryB9=0-r0 zuu#q?IBg`8+O%~aJZolwa|PYILd>)VMCMgiW*LRJ8$3E?sF2#+G>2rvIC}xil*pl~Y&F-Fnp#90~^-&h0)rJe=Bj|*=#bxTu&bYzUB2b{h77>L0NJTt!wA@YDKk;ZYCGlE6@)c71 zP~1dwbUk<+l1&JlYG(M}bHhP%?m6~Q-G!z@zvoRVg&{;NXWXYUHcJY`)O~zi+ zt&Xl{@Oi{#LW}<#M#SjEZGT=~$!Z4c#bSlHx+Dv}@>h;@KNS?~4s`#i#|Ot?*_rjF zXSGL@Ja;X@FFPK)E`%L1I=!!k1T`&C#R$Xq;|8az_zplXB56%<5u7(%)^(n~GJXj+ z2agZQ9U8?lEJR%klCVyxn5G)6PN5N}lljZ0sMVtoqq7qRTy%()k(sp@ml^qNlqpp+ zauAZJ^ANea45b*lo0zwB^8`NkamB^#q>|pFhqj!?OZ;KI-G@SpcxCjIEJ#64Z{tds z>w>s-8L8>%pTHok$gwy+>8df0@dj6}7WhZ41K-l-b*m0|JZ%MsdxGGZtl8z3G@1%$ zF*1KsmG?~JFV)+5qJor7p4zK;T0K{d1J5Un>i}aYPnYo7n=9%{1t%&&nka}dx+sNm zD(>=dZ2@#d26mu=py!OKZ!UdSqF96Sy1b8eC6*=U%>5Ka;MR!j<+*r_aX(i)TjDIw zEEdRYrCdsJK9EavNKm*|-uBz0GH(GnZDAynTb~9GCAXcN09SKk<4I>#yF%yB=jAY_~Ht$2H89jb&k=2Dc;sb?_GsH&mMuK4(-}qF-Kfqh^9UMv?-GmsK zN3xQ3q}YKkP=#r>vTqgb88T@N22KL>QS6IB4X(DGX&^-On9~4G|}uXOo~&0+}8mcGWphS!0C7dMnp~!X^+XP*MSZxKeJ3 z)8WxiB+*Xod{&=4+5mooD9s8%v$!P|N~P?vCuZp4;X9fFXP~0wct(s0$Ka3(4$KE8 z&Dwn>pwi$G6_ELVB4)w~d4ea>1L3T!v26^(gGpRjln58^NTLiwW4TPzb5)Bkg3j2# z#Y*>9t+ELoM^VHa0+fBdss^g(aIuwKCfsJ!`)tDh9AVv-@>oAJ(~kV4LqdHoRLo(Z zOcqCipW)jCYvKIl!O5_Ak+--Qd2OH+#W~NovEZ{cn5I(J3=&J=1Z8!o`7K;!k{h!a z-J!b$5}+m=0tk3A=G9e)=Mxrn?mUB)yBxt8H$ac#qJ4u3>4)TO&6|fgL~v*AS@V!c zLA-aO`I0QLt6+{DCm(mu%Q5nitT+?(I#SY?M+kpyGBGnm!eE0J`mJTU4^;%G;mOem zPaB*Iz943(`SXExsBVn69Z#>?U+07(@ZcdwYJcodO_}#LbcEyB)d`9Jh*|n_8`}U* z3~q?gY`?y)=3MHoU_VMedMqn$5Oy^59g%h>=Mh!lVYlplRbJ1jd<%mRr)$p-&rxA} z;WWnarb8YzBckWJ26yF^jxHdJ6-_6~Wvp$S&gRGcL)uVz=4jYpE)0=*V>$U0 zL%V)%5^Vdnc?IeOV3qNghvQOHsf)NA$;6V6&W<(QBcgu&rO6neHw7bYmc zk>rl&7g>e5mpvgj=RpLuM$B0@W%q)u3VGGZ-e?`KVvy5gGR&WbF|xNe5jZzA4+@ zZ@5LP5IIyNpX}ZvNJLS2>~K_-l48UVHj^yf{O%yq4s0MQ?Cl7dd?e|tsxphI#EQKa z=tRC)ye@SGVj7OKJ=O4o$b>coJGk*DJ!#K(izrcaDe742Q%}4$0_7+xkWz+pvaa)B`pf%I#D%HvO6=CcD8d5$Y*dv01#Cu z@jta2C_SQY=5j$Ys~ef>I)5Fnf}69@$Ukk&rz#r^$=3l$0EePY$J-FtoPV~O5a~A@ zVv{q^UZ^Mdpm~qBI_kv7P^WTvhe~>MNmp~{<%1_Zl6-Rl)fB1!S)gi~-$_a{^P+sC zx=gs}Y)f_`9`f)NUv_DDC;;RjNR7u)Fz~s+pYX}>4Sd#uGFjUm_z1)IEuP0GgV4~8 z;IPcsm9nlbW0XWRBk2yIK#sm6;c0<|2Hk3Ht9!|~F8AJSS&I7btn(=vO*dSxY+tcr zrZ~GaW|vOLHz8U^=2(nEvGTdRRYu?q8e2ltD%2)8mDOe#KH*NTrbxaZ8U|$#k3vu! zv{v>$FSGG_@aIaaucnKmi?69<%j8*e;a3IWHK_~u%_vw>z(k=G%OZ_djWy6O7?FixlqY`G-WAkdLHijP6>uro{!)w-CsGDendY#k)5R2FQFHQew*vpDvLl=~Ra!zvW zN?+PofxSS<$%|Z<^%=J_5)NcML)|KTiEAfgAgq_i*ns1HW83W&NO)Ywn0l}&c})g< zJZ>W&dJnB7@p3(wgI04JEvSJLY=}UH{C#g}!@_Oib#FolX2ZphC@m0&Y0R=_JW5Ls zHTh_2B}dA3tbI6?cFmw3XnM~d;7GV7*+Aro(!%xGom4a%7xITv(>7*o@8>epCwZKY z`L0;aB6J>Z`H%^e0)5{Q9etIzR<(^$gO8D$HU?^7R10Gx^2O!=lmwI? z22uEHRzxGxwAB2=YQN-rds+CT=MO${+A5MOx=Dxpfe&Pps%siX977kUedwdSB2PY;v0}*PLa1R@faC^Q6zba{i1@WbksyRRTFc0s#z^6nIOg+~Y3a`KdOoEG?az@dy%%yp7#l1e*57bTCYx^#{C%-;@$Yc>zFB?u7 zKAjWn zeNn-h5kuzE$Ku2q0R&^=_fo;3H{CyBVkH)chu2t8%;&%Z;@A03yKqz`FMuu~>jy=d z@EKfXxqEZk@S<-Y@e(;9tR#Vle*yXx86y?$TxJ8G#&(JAb)qHgjk;~fDyzUM^ZmJP z*4pe-@l8@@!SV%?!EfRT=jX~Z-i>w$58F6Z;oGRALdR(`J|~Y0QfK(bEfy88-T}`6 z=#@Lv85eP@mPqzLQI9lL1%kRRxi_l$q2L{9Lf8R;%Imb(oVSun+XPeKNN}*9VgwdZ zj=pgz*Z(R!W04Ig`a5b43)rr5l<*jz8O_j<)dqVNRd^ff`+?d}Rcve$6louCY~bh3 zE;)D@M5PS0>+Nd}s>YE_{R#%jbBzgU(QL$z*N`D}+PGI?W@=;k2ZL!eW} zBAsN`08bW)HKL0w_<5oW5HO+#2eW6q5>N&h2^>X_s7R4-C|4#%s4-eM0zC;$+(~#y z2t&qxen)h$MnlZlC!bfpGT~F;RmaoM0}JYe&k|=S$(Hf4RU6I7=T$s?Tw7}xpM#py zExBH{SFjM^iW-@xQc0mfMHTf~svx(|&lBQ%!hMl6nL`MnQ{NSx`x~+vh}-Lh_H*3o zqe0XKD}41ffLAn9=9tk!p()J$Vj%E0>I$zO{LwLc6V_{>_|YB^b7+xzEZ%00pnQ}X zqwjPba4biwpAj6Ax=QBY+`;1liqFgC;m}E);c%9WeJ__+7>DTqrStMmt_?oWpSFV| zkx7F?i4zRwq$h-EG1H^efRyND#L$Y~exMh=A0RO|KBD*yK|%&IA%j}Cpzt56{-Po( zqc=of`3ez9T85(0f#om2q}uEY1(Y}V%ng^B{WEA5(GistU-e$079#$Uk_%U>iVbu9&27;k+zG5CSv1>p|wLYeK9{p=LR6k$3xbOK)`t-j!_s(3Rl=hu^S84Y=ka2dtOs*Q}VbsQ%*&dOwU9sYiLjdFuA zfiRtSFI-J#Y=tOz<8juo>P6a9$L%rvoY(ri2U$LI#F+E4%Xa$RgVqZZyWU``L?QR! zoM@cLjO=;?l7VPIPAqvGYe+w^f*eulH=G3=<)#d_0Z#v&wGjg|DX>_@_x4P4kuBxF zW_T9%fqKL~WDW;}sUuIss%yG?emkxSDP@#6H$L0X{1d#0a8T+r!{OwH2M|979@;#L zyD`Y#f`fDwFNde(Ih+nV4oyfdd43hA6H7$|Y*J1@p(1|O7vL}PUB%k)q(ll__;tO< z`h^W|koXexZhIvV4VYK+IM0e2)z?Q>Jit<0c=~?uv->gb`lk;GMXr7v8jSHs9|!Cc z(a@MZ_@W?hM`lv}=UkS3I!%X!ElrxKp~szHSh+szX)?X>m#+9#~^} z1`rMn3BR0m*BA%#S>U$ZmfK;SA9(NEfl*%fTMI9KUT-V3wslStiPR~4SY~`9ZHoB1 zbkqBJoN~X~LqDRqxLJk*OAW%lT}cd7HDne`BjkaIlLLpAXq*2U`UE`q*gH;n z%w;ELo{<4wog|1oE&2I|#^w%TO|n(34?-Gr*acNtd18I8qYqel0b+p!EaJ6~p9;4K zcrDBr8~{S?PM$w^;6C*XM!u>;%w;RwrYkUvy53gqb%8%z$=1+f zK6-mNfkN9=pEE*&xhStQg!D9zP=(Ecv1oaQn7pZxV@;6)Y01!>Eu7*Jj&D)ys^(TPQt0{{h`3h8$3K> zh!lrg4H&Q`oJFA%$2cBB{$A`>>h@2Xxjq<(_-DlYe0f1mtpPGb26T2%{p!V^ty*vy zkx^*K(j|oPS~*Oczq}O|Wd?Q0<TboNl>q%q!1OW1%) z)N@>blQ+rw-|PtWq!DU{Mlu{z)4R&w#Ql$)9pTQnQZ&n}*8TPuMZ)zBg zH7rt;Xh~f}Jke)A3eKPK5O=R0{40la7z+kSkI%4=+^pl78m3Pu@S|TLQ)L6E-Z?7l z9|W1lgJj*D<`WDi_8vjMlM3aT5^L-sDVSm_4?4Rb5+RChmW&$`-rC6O;U^gFe!CrH z=?W)UGp!!UH%bh#LShK~fjIaRqnM3h!#^T(jLQB6Nw#>%LuI4MW&4JwEqx^*4i*lD z?22$x0wt7Pf1?b7Wz#}zL=5)R;EfFI=;g_tMe8xVIahK(n)CL4#E_H2$9F^rcL)N# zy&#aVDvCiqNU#ZWT(A1y+Y>-Jb@U*$#R4fvaQoe*yAH>o*!wXFb5KHJyv2>kXi9m4 zqi1RcN!2Wfi?<X`QeJ4>%N(!+ zVf84-pW*3b<7nDFok&QU^1PZoZ_Yezb`qcP3zgPYwfK8t;7$A>|=< z#z7;?%UFKG-)A@zE-VHlW|4L2ldEalsa#;uSthlcVmL_6=W#RHn30iqc7m&3j;%44 z197MoWFg_MN!`QZD>mq0E=N4>GAuuYE8K=gpvB^7mfm1XY*GpU8ZFA6;Bat2@wisa z(g-LYnD6B@5|j@&7+C*~DQyHNbj@%W9cDd327JgD1GFb*V-!j@=O;6bL{>ag0Wmx- zD92%VIS)cyc-8^-z?bmkX%6RNGQ;@}m)i|x@teY%ogCK53-e1qi)zYDiIEo+pF5EArhVm{}$*gCoec$MqS zAK@8bGnK3JJ7p%OWBg=b)NzFmOPL~g#z&Ft7=*p5UGV`I7umbmbBDP3%}8H_ zSd)kfLu5jV>muNv2w#7Ug94B$%5S?6DvypCO#!SfPL6H}gvN|u|^9O!s>35b$ zc?lM(n9+B9d0!x7mFPiSFtX33fgCu_r{26M@) z%qj~Jcu&|(=N5k0cgB4pMsVd60PbG{oafFxv{#OUp)Moth7;CEX9(nHMFe@4pffkC z7mT@AT1(>{n7S{ z$ZCCsT!%k%cHqR~a(1rzS1K-3>j>)HoM0`c(oMO0 zsOusm$hkun;N{*O8f+k# z@uu{9VN>y$FD=gxT38=x;U{u3M;UO&>HhI*0hvaR@i4Y6X>Gb-^ zp^GdFXbI@cx3lw<_?xdCa)&`OI7c|g^4a*!Tk09&gmUmQ1Lcx`&q109%1tx|TX_G; z@BOYp@f2pi7vxVCRg^dW>_ODasJibCY;V|xYH#g$KwGq9!ouIE9#GoR3HKQ z35L#gnt_!d?D%Dia);7Rqr8yaPz`1V!5`Mv+HlUihwy@Ok{K2@pC~f_3_2d}AP~fD zgQ(12>#9qJkXGvf4*F~SuvGpwWiI=IQyFlS+M*>7l2QPv zCi9fOZ%o5+<}eheo`JKN=An8JDFmW#90gukj=EU(G18VSfwZ!3iddo7okNn#sblof z5)MFw;s#bhaY#GW^QPW1%;dtpIM5Hsm~glgG3Rj1&f{_Fe$JoT434DOg#&h0boLnT zDopu>8X$KYX;Mw+*h<7aAz7Vl=Vv&k0N}qb*oQCm1Lw&S21!TJhAHVn1aiW6kc%DM zHL(ImrrbT0S8kF*C2t#xmhCWH$XPG(g${HL%~k~?#Mv4eAy6meX)PFJ>)?KX}K+urV{VtE}Y_RK@K8yuRg)a9Slr9SBp15=FK+VAjv8?CtH8Bg3W!| z_v^gey)HTy#cAVtK6z(PNC<RO;N(AKZ-wRfy&;^2 z0pTt_;Mwd3yTmUn(N6(R$@97!N)Osp40^UVKt700T09SA`+Pe{S|d&1P?#DDBSROu z1A{Eq&1osG!h0@47!!4q9uI>unw36VWn;|87GFd9h#FdEj1B7raHa()IFmJ3}S2{|e4;y(_r%J{I@+09@oMn2Xm! zdEkW#LBi5fJVUzouE#~xFG6NC^0~>M9A3?okMj<(VEn5RAZExMfhIEmSYj9Ot>d4Q zE1^5k{KvFMQURPFDt%vO=jnHlyd*_L#W81<3CYTO$`y7;r?Ev}*WG#H!6T$MH>E&C z#)D`EN`#?Sd-CNHk!Wrsr9+C5g>xT~YfKd>g8$boa&v=>!qNaHX}Yi)q*!5C0|i;a zR+0<zJog5sjqe;(Jh(|{c%^@7k9wA`3 z3zQ=%PqmSw#38EZnC5jSm50ho=lgZB+!c2iIvcKB93dToPrQ-WJ)Wb|GqcPR>JI!J z6w^wi(`lP)@(kZ!{892fNCk7XHykbLec^Q~j}Rkz;<{&3PNy3!WsoBxsvpRS1UmY3 zDJ>D66>AQlJu*n>LB-t5(cApol*;7lY)w6DR&PShwxShjMfh+Lg;K?eaL92@Kc?y@ zBW@*SdmmB4EV*)CEr|Xv4QJm0WA6wVZhtMCOvqDEx}pkuW3xXKSF^2+TPlEfyCb%9iDe`N_q&4)Fs17}CfrCWR0D}}O33lIyx zm<-@uqlAeO8m+zSR0X*WNNhZx;RNN~`!y@2-pxofS5oAo{lpWwthw!GTE z#sjh`Dhn-S=p33#i`GR<{@4l2fzsL|tQDYxEw?`I4;bqoQ%sgF%j%s3MAIzZ!PD}NPIXg=$Uwr9M0;M^pR^# ze1M6CO(C+*)?3^*q zdf+RwjU()8p5B0V~3LVdNbloEHPxi}l@coWE#GiX{xv;J7DO zr+$!NiWq|sOUyX*44}2a1Mt!MPY7Kk^1cGr`G=}}0`lpkhuN4*YNHXaL%}X_(5%KR zY}Z|WN_qmO?2ly;a!7Mlm^z@VL1)39vygN}xqgd=Hy`d~x)o|)uM?V|{Q@t59atCT zTj_*e2;5AbOJYO}@~Yparv9l(5spIHV$lYvgC^;8fP_0&jUM4_e~aln3o5sSydE4@ zFK+s$=V(y>H{nbDnZKVG{v}I(nBm)o?>KyE5(M!-;|p-_96%GRGW>Kmoq4WUWsFb~ zSgKO}*Lm`4Ndojq*qnhl^|3@)m z7lB<#?q5IE5ZK@FekuN6DOhlz_x-+#r~j3pVKMR7e?NDB!B|O@$QqUY1IGHl70ewn zW&U4RfJ}<7m&XN%gBRvn{FmS1`cDVqTp*m_|GiWE2fO!g6Z~HRI#Uc#GMc~N`J@>B zU;oq1DA-=8iT~{x{bJUBF>8O(vOWgbKa=}#?QbiWFK+ISiS!+Z?>PM7i!WyF7qb@0 zgD+<77qj+@S^LGT{bJUBF>Ak=wO`Cyh+F?+)_yT-znHaO%-Szz?H98a`(oC9F>Ak= zwGfc`#jO2e)_yT-A++?1S^LGT{bJUBF>Ak=wO`EIFJ|o*v-XQw`^BvNV%EZ`*e_=7 z7qj+@S^LGT{bJUlBUE3^+An797qb@5oqRECznHaO%-Szz?H9B5i&^`{to>rvelcsm zn6+Qb+An797qj+@S^LGT1y@l7i{vk6?H9B5i&^`{tVRB^FJ|o*v-XQw3;7Pk^B1%B zi&^`{to>rvelcsmn6+RUeKBjln6*F#U(DJsX6+ZV_KR8j#jLHqn6+Qb+An797qj+@ zS^LGT1&7%evleZpeKBjln6+Qb+An4;h=MO>?H9B5i&^`{to>rvelcsmn6+Qb+7FcR zKSS32V%B~!YrmMaU(DJsX6+ZV_KR67eKBjln6+Qb+Ap^37hCp=E&Cr}%l?0bG5E!- z1r*T#LCji;1<=}G7}Gz%rf}&$9kA&y(r%U(zevBofV}?<>GVDI$!{rTx%Nlz+P*el7qSEkYH3BvAzz#95OHeg9tIYJ#w8 z9|>G6#~SW^z>AZqpR@EA!M1--P5Re2c=Y_((R(>~W*DP>{M&c9t3Ikf)%-Jz9FAd-;BUz=YND-{$Ipa{sX+*XH_&l=6{^szoGH|DfiFNlmADWKmd9#jDQ657H#n7 zpL*IKY-#A(|9CBb>Z;cdGBwnOM*G8b|A-Mx;=kWvBk6yleo}(+KMnmKZxkv0yXFvo zAdG)Go8W(&^QZH}hgE(%f#&vqww?d(l@#?KjKH5){*c-KgI9hArCzn{4Yb=skCcDG zgin*WmOg5M zAO9vLa^Q67UA++>{tH3oH2NQkTj%yBg}U+R)4tSzoH3>{7;CS{fATknL_zDMFFt#AR_-hiT`Yl|8b(4 zcLP#Q{QGSGUW%$m(ZJ*Sv#DAo#f2J4y-E^NmV($9xUr_ar z;06Vj{m#ipeg3x@HTejoJM2xFQoAEl~CKJ_!mNY z1fp|VFheWeux1kM^QBZSun{GoAK@TK!VP}QSZ4bSx+3bfbPl|OBov2SC|{3HDe-tx z)JvwaW8*c8iKhXLbKcV4ys<0=NpV)OJ+AdqT%JW|3Egcm9tA5(kFGYAH^*X3}ys^-!jiv76HU~j3D8j{Rh@iRZC zB$>`&(2FDd^p^}79Z&#%ixf?h_QbO=cWPKuxuZy;Abdf`|A3AQf^ztR(0}RSKNCH? zK*n5B-2#^7W!`Zq69+Ld%TqKNnRHr_Q%CIJJ^aJtWU3s;NlT( z3J(ph!%d{WWsC=O;_p zkwJ>2cgoKRqYc~p@=`_bcjlo#y8xx)9)>R0PJNy}V>WufU>wSy_q~p_;NSu|CC*q# zw=Vu2KNzRWNs|oe1`~rz2V#eAKl;uv3ki*(AejAO5*)0;u&}(bWFFCKDyp@P{Y4uu zXrU1IX0K5ya|EkT#JRiHELUl&AG^H#p-<48@<8!)^9Wu-e#q}KXxUFKQYqcC!YE!Q z%%>!+HAv&1nJ+QrUOt6E8ViH627!})2uAb@OGuPOQrCzOgrQN7f9TPW>_uIC!EW5_ zJQv3+CJra;_c>7RAtfakB_tzdOj0X3EO3hIYAy8jx23kI+$`!?I6E5gL)j;c3xCZ< zBoM+)l$4)w^<+*=cX=8~@%6mt`f7L-l=P%H^=j4VD7eS z^0ia8pYeLW4vnbVx_`RGFT`U{tfB4AGf3;sXkAkdT8|4SA=JH|IxLeXG;CDMH#)l= zhupd!Zk*%9QGfWuyM{m&$t)LmrZ*zKad$t%Ws$1xi@>O{P+_EC`{E!?`kQbc@ge3; zO9$`ej<4fTAns6wSqQK+Q`iGbh3`IghxQhWP)PI?PZ3Q587&#El^zmJeXi(6Zk)Qt z|8?=#2U3_g_v~TD7A|-p^x()5r9^14K*wUs9q-eWX}@x=IKPg$bc@`Ld^O}Dx_QRZ z7qyVjHPmiWz6et!rYEl&J|n-Cu6K0g%S&5Bp`(5GapN}SB!FW=U2)#9c{kVw_}yg&O(>BjHyUvMd+5E^5m` ztCU9xzGij08H;MLa!CIrM`Yn>3BpQsNJxAn-}!oe3&%x0>?Qw)(*zOd61r(+-LG`~ z)LIMDVEY#3C%-xgG^-^j%R$nO-%;>1sTGffV9EiA22sF%Wr46rvLIRx$gw|MuNIwf zz6oazOY#>q>gP>B4vQe`k2mz>M~0;`l}lR4b=u6VLl!CP)Bru+KdIa z_R?N{=ESpvPe+PF(*1Bh%%t&BYM4PgbSc^U)$Gub(9S&)mSgugE_0w6^=_LS#h+Z6 z!Z{oIl4;&+KT_Z^Q#nh~UJ`0CC`+lLa{IBcW|`}=gY1rKyHoFh&5ZoU81qwpi7;E? zi1$A5dN89k6|_W?W=Sj0oYQ#&W=-t7Pjr7{i5Hs&uyDaICtd(!saty@-yxm>1O$}@tW+!!B~9e3AYB|zp{(dM95cl=-_If2V92=!UxUuxSo3ES-mKjx_&5m&V%y)e8CWTYDc8YIck6f%|6eO1Jp-{LS z3hJ|7Nl&gvEGjAWH86u*Ax(Bwl7ni(A5*v&Lh|Vv|JDS&J<(-hR?xf=^oJ&7~99r>*&s zDN6`6AoA4%^dpoyXuWjbGK#Q;MOmJ21*E%ZE;JphEgrp~&<{@_@qSh)`3Nc2dMcci z5btj#5B4DpBkz>oOu{>> z5g4T^%e|1@tTIX^<(I8#h3yENRue;xe$99F8Okto4a&FV4vs=?gl`HdYT+WkkG>qB zalg2I={|C{!gF;xL^WpSel+(N+U8-LpaxFaxUjqv@#w!q``7x_WhQOF`YJ64NMT z2=Fo2uNuzDGpPx}eMogOscmEC4%TjE4(b*qD3|pD(nc1}Mf1`S2~A6AgA!03`0)={ zRz`2C*(?nC$|cy6l#e8bp(Vu692M5#1ctBPYt78iF7ul6#( zy#+}&S-1!$(uMlfc2jZ>xfTo_)Ip*s7c=fNPHGe<|BO3(&FuMy)5T?_`Cf85(Pgt$ zncGR-o+Yr5EO;t6ZfHn@BfnNK;N zc)X=a5O(J1h#7W&5ro0(hl*M9C;A5*- zW;y!a`>D9C$$V^qm`cYK8k_cvqiUXe7=XWOb5Ps7+?hHs=fB)SU2= zGOrl2d4EWQj0KC}oSyL5^MxvBmguHmNN%8|1ldF6*ulTX2*jyghJjqVhFtUQ*-B)2 zEgzr`OyJ7=;7Q`CFwCY9-iIPA&XN-dry(UameXAghD(fiSCsilIHY{vwK;EOY9xlr z8FQmrqLJWTlc(L2_&dIkasPCsb4p0YrWE?U^!Vo0L+Gy)WY(k*mrda$HSg?#y+D_oAoMa00nxJT8kZKJz$(Mrizf z$}lu^_X11?7?eIje1n9?9wSCKPeV4Ahb*mcBdO#Y8IvVp>7~6DWR$Q6WTn4L>ImET zvWOU;yvg_CWR9o>@R>3{CxvQK?o+PihN{RpqI)Kq>E)WK;gPfOjsX27sV8ySv2l{j zJ3lSTUUXrclZ~=Npa7vpWFpBy@5)uf7_B{#+*bD6XzmL}FaGXa4qzv^2R}LRE55n$ zJQ9MIXz(h4=v{;8I(e5)u6%lAog1Ru>#*bcTQZl8_JhlPU0(-~%V2Wv71p~Wu@E;U z+t~bM=W5nQ#RY`^a%*iGo_(jgdpWl_Gs@&k|61#f{CBdO*;^55?hjc1MQ~{^*lBK zpIrF<8k5aQAdyadHn*ru3@JBc52)k#a0`8KlAXZWbg*ktA0`s>NoMb!OWB7@yCxJ; zNVb3`ST~!~_CCOaVxbr2G$lKZ_4nBZ(i)JqBm8jNJDcNrb0JoX40N|;&JLA(6_s+; zaL?6S9KFV|n=ty;h~q4?+^mHM<^7J)(2-15-DRD!883Mp&scGAAB<7hh$i6nsW0$z zih&I%+KqJ_kzEyMDeR+vKS^xQCI(&#sk22RD^=gDV`DP+A|*7f`3 zb_}i4&Suzh{5))zG1INma-D$*t~P&t7+1=tppNIpE`h z7sC2N%xRHq+Uua-JV6NzN?`E~{$d^PM8!yeI6>s}@H?8Er#Q=* zX2@~ausRya$t)t6p$cg=C7gr1;<-hdUKu`lBH2DAS1Es~>m@CuT42HvHGysdSW?7n zN3zeuz3X_EueoCuJ`Yy#dUCaUWN!51LTyIiD=wY+*MMMPQk_E92-=EpF~rk1uYfDa zJIYf0+3hj$Z$R;;l8L$<}KhZ5jw8GK4j^L5xo^Z}uHFE-;1xAs0_xv9BCO_H9 zEFF3QU&a^fUUb!%bwMYurs|@n+4tYyNXJZ*9M}syuS~*v=HtM7-q{B-8%BYI^~;3> zZL{YZ{Fz!OCAO~pe#}#X4ro~9_(wgpB4Qp7#{)Q$9|22;#{qEwnYv?5bq-qirv?H^ z*BLZEJTG(cp@?$|5l6te4sq6*Xp)m-a)symXNy+)+o;_hK^=x{MRdqkew(z{O=sgq zkl>fc&fHv{&xtSyM^CvOH3u%izc*KG*SqKK>M-zb3erbqO1IZYE5spy0r)1p%!S@q zjo3`)!I24|9d%Zy%GiV*hD{@3*q|d1=UlBzo8c=00~Qe;)Ji?`;`q`XN>b4r`?wi;I1TNCmEtAo z)21x9fVK#(y7Xkv@25wv-X82LzS2M?Fi0W@KY57=_$!eKeuiNYGb`rcrBKNT{9Wi% zLP*KUUYCCxe{v4k6_(h|9wFXO64w;*vc2D?--@8zWnPs`+~H}s3}w^E9qB`w&0tvF5p`b8`aQ$SKR3$M(FxLr zO@j7j8qZpneLAg<1FmHq!9HtJrxsQh&A0S2p(a+h*~>^A$;(PC8VJ!d5wm^Xb#z!V z!1iHz99>cfh7o6jvd{aZHP+mCoW}mevW$YdRLa2is;{*^Ze+K*?Q1f*j76ri*Ed)b z$9R&bhdhD-uAj$&B(Hpc=!sB{&>cjN&p2|UI7o1c!RaQ8hsG!y^RbRo(>tqnNjx9B zW<`G-cKcXo47hcvO|p!oDWv-5yL$ zIa5{~uVz-q;3`WdT_YmYmg{~|XCJglzf_!iEEbT9Wq9!x9uWWq3T!6Hhovu2wzd1_ z29=|UXwwlbVUQ6MxEn_7Zg3>Gz!K)3@e#{HnJ}gh3HT)JH}D!$S#BpaRGHTgFR0}N zN5wW)&G(gmV=wjYE*d%WMT)(GK_mp`Vs}srlQTGO1|6xU5_x=RjE$s^bSlng+)W#| zITC4CL%hPQf;=?oF}h)j%k~v-VL>i$#@XEX z+`di?f6ykUxizVyV$MC=Vt5ReTsenUGmXV3DQsWj??$+y3vJDUw&xS}?jda?=T_J* zeJ$hD3GC73E?@HLd7D4j_YN($xY z)@pgEkr&G?=S*TNI~5bwNIRNZ-`N2woanVM5#t48e0T}4PZpL=dOaTEJiA_UeZLnA z?GX17T({&ndYlguxgFHpHv^z6qLG`{C6MObtGCFyh6I21u=%De;?7Ac|7`m$2XYfv zksR$8C9T$V$Ieq4^7R12qJoPutOmI`hLq)ipD)jDp3d>`K5*qunIP}5nkNl`t*_t& zMuv?Dnm>)joOAH@#X%JnM+fpexlb&jjW=Yl0%B}TVcMrUUnJLjmyu?NrAAOvLCrP4{;~0+*;Eg+B(hG zbg#jn$h)1cWtvKN6O94l{&{+sd0?L#N;+bhPm1ON$J3q9U-+oI1@~T>;&c=s1*XlX zHSXd*Jaj%adkgOrd8VB=f(RYqTE1D$?aU4OrY5W%vi)^yxbcnF&hSwbuQBf7)TPb2 zr$wwZmx4|%CvFN(*cqyl0xh&~75-GCJxsVg(YM7)u4LCF0=KEG{f3BFD7bgWy(cG( zd~YN_)J2ypFpfU^BKio8!}0!rejW!lK&M;02YmYf%3Rvi?Be>RJYQ%GS#HTw(Ew3mrkoAKm3-;^D z`nA2P@~w61DM$3$2My|JMA+0s!RjL*%tPP_q|1c3?UlVH(Sa?%cFG_!;&lGii4L?! zm?XN16NzTow~l6T(j%YOLz8LICY)N|s&wqMt~_TN8FoIh1Z)sSXw2+19L;cqzaFlS ztrxkODuS1hm3FjziY6X)P3sBlk@!3}lJ~qG(Hq`ijEnu=G*-OVq3cFs;^I4N@L8=nD}^2AOKF^FVuuyHkFKL@Rtkn`ke!? zrpKk~soNX;04yJ$78k4meyR43q2x(O6Ybbs*@)%ckcG!zNw?T}eZ>+U>ATxbveC*= zM76s6)sE-*POJnoH<_Luo-$tfx2#J?-0dq>===-c-p@YLbzK!N6>NQim!W%v6$xr5 z_CN$YVYpTK$+xY{JFmMuzt7U8pjPs9O+s>t-XoS z;!h2!x%UV&->z*0q&nh8H+SScTH8cI`{9#|+Hr+-axh4Z$FCaApGYc37REHgr~_Gv zzHv(0nLDF#)1FblJ3>0A3n#rwc4>}dO!HTkr*giePDA0^FSR9to8PUs@X424TCeSG zN5dgP2tnjFkw!(>2nHKm0eAieuXgk2eGXwC>#43Mw`AVU>G+}`ZmE1w2Y(idJYu_^ zW7EEehv=NeXZVfJ%uOE;;4CQj!=eo{MG-ptI;&~4b(j0-2Q%A)@p~fZrWh~IV5Z0G z${w>F8`Xxz>>2?bFT#w&)2wd&oI)uG!wQQz_Nn`a8Hp;fDJRO$9T* zFIKiuHHd8Sew8saWrpX@?3s7i`OOArx{LD^Hm9Eu+XPxiEC=x*e3lgKe(k$OjgE|b zU2jJQG_Lq)Er>PM=Mh|&p?f(vBty>=1$IPR?n^3zVA&dK$ACoJBk`i3a$g&%Qv%3R*?RFFl7RLjvpoo01TTmiry&g&h_P zOaNpy7QP+Trf?U!mjXgIj_Pd7x{GXRV9TpggjRB1jMTUvlJm?}b9}YVl#@+Ho>bz@ z+t2x`o$j{lu@T>Pq#T$aAYjehzf)7DZev_j7eaVlsnPImtklM{vefhf!Q_~IJ3Pj= z??Cja0m*CJO+!46COpQHc;xXhvye4Ydp?Cm|4xM4076JK<~T<^JdBtD1w^FnI(x*W z*c@z2b3?gKSyZp%ay2H(J3II@&aZ|*;Jc|`Sq+oACcZQMjKizd)Y=;z zQNgXxPeQ}(QZ8t_f|ex44y@1iPQ9Dh##;ujxM3?ZUVGr;@q{GB6yJMn<|rI*FrH!r zG=iwD!ZxIy*09skc*{eZ&|apR$Fc8`*dop9eua%-#JX^KeUuH;hWyDibvIL*k`^4} zx`;>ie5!LY)*eqgjf2;qUu^V9S{T!3r-M4f(}-&idhV@UN9a5UZy zufxmR++!sv_VqZ)Et9@$3`Hna2Mj>Z62uMNCIC67Fr&=qUCsV}Ri zDoN3JFsOSo`Fw74Y~*uR+fM0g*eabKR~)=ba2LbK%~cm>h5?JJ^X@M#23vUGA}<7` zQoO*MWOH^I!%J4SAK&)i!k!kUT%PS2$MI?GvU>mx36rg1Ln-xR-4BusBpY1JDX!(S zL$s}yP#x;Uzc?_8{*FCbXS|*5I%JsXa?B@+ob<3Dhn9<5Jhdm~x*VzjC&)NPQ?A~i zvMCj$ioJ3Tgaxp0k)BUii*=E5i6Qp9)WDpCqHl#W^KlC|KTe;#+|<*py~^WBy0SfXCEXLT z2(Df37lt}TP1U;!vy~9wh6qg45A#eLCLGs9d-GOKOv6p9liE9<@M#%l{6fd>XJ7cU zLkFHK6$z1wl-gL&+x3WP5edg5A*dyd9QV&-%0g;sQT}Hnx)_cy~ZDaI{(FQ&R z?xyN3$0TXOX2rTg`PVUCI36~e=ppyhwntugB!oCb=SiR=F21H3AB5bx-)a(=k^94WRG0w-=>k3Qmgk37imEA6zYb#3TQ4YCD z_dgw<*mG!sw-ATQ)M?BHRLO$-cB;Sw&6>gbhXo-omZ_fY9D!CmXbr2yG3Gb3qq*lh z#au4De`(S=z-hb=iD#1@)}4D;&4Kd4LDWvB6Ot<4>Uw`iZ@BV2Y`DMzH~$=rfGfk^ zNE>hhiwnD}Of7f>sh#i}k6pYc?@&O?#MQZ;x6vi#hrv>J}+QN@s6G5(* zPS0J_*REwe8;Kpb$)Vo1ixi$9+E~V>}4Bt$Zy(CDHLg<1Pjc;5sRo zG>*$DN&DK-`1d{WSR?U{_!19kB#=zHJ2Ga3@8jLi_X(m|7$-f!2H6p}S^w#7Ud$fa zb+1@)vv*rUI#$dR7kKz#S5_n`xtsZPS1%)03W`*YDT z0cucv3fn1r=&<*E;T_4zkE%Po$PvmGqSb5N9GCn)k^7jtcmBf9VqMs1{lln104A=X z5D?oXFKW4N6j)Vatz|m|GoQ1zq3KJ4nPv)R2Bh~-aiyfknyRW=9n3ZbIxN2aRPsw{o%rd z@C)@`b#Nbid@p=U!RrbbI}vHNk09gMDI)nvFPasj0n+PW`>%56VT%8 zXpf7Jh2FWDY5$V`-|W2!xD?g3Hf)SDF=~`(NDz!sV;tz}nlXUYL-pKUJy&-W?dq!P zd8)3VyAg4?PB9Up5y5#zjYN2XMY7B;%!zak)D7+@m!~2P3I8nz#CYpHQkS1jS@nO2q%eg~ZhHkWB z_r24qL<(hHX|QFRp!Y%!Ly$`eHSDL*U8PjOU~NsgTxBI(v}q(77M_Oc%>}wlw^Pq) z8mrmprlXlt5?BWV(0;94R1DRhapss3SG61g*s_-`0Q98&$vnn0MjLP{Ub~a62kM1v z6$@HA&FGb8#*pl;Jd9pPsYOdHK?XyTuP&?6S~pvP3`|GHOOmh|T1wX_*tzyJoRG!_ z=GpK%U=hk=IBfth1&%<>6h3EJ+!upwV}Kj=h_=*s+~8yyvsBTGi)g$8dcxtf&;wbK zbM$PA7{|aSRqGkfC`lBiiVB7v5$#p+2yL?oDtdd#B^HXIK5jJ-BPt!O?hlv3&`KFm zj+6uzSJb*eTR&L`;aZulv?R#^=KMA5(pA<1RqZN?TL&_P0Dz})aotMj|K+p zVFppJ6Ek+&=jBLrvAY{F!g*DK;~LBC9`EFO}iSUEEc?&fG6bQN;bgMI&B4iwbN#E;U*nTqy=BW zs3lZ;FA@h6&I%nim~I3i=LEfXrVL;*PJ_5k)7@5LezmROa+)!^Rn)#}G45Q+57~m-!aj^!jb4pc`n6II;GOBBw4YrALuD@TPJJnJep9Zp;js`GcI;p1uKNdx^TUtM3f=$M|J96 zUoh&#`w!zas!lL9v(~V;EjWU68A#lUyD<}k8JdSa&AZG>I%6~?LKq2&VqC--H{nd8 z(d6r1%1Rc?cm`0pd58?4;Y2$Gp`nJede$mHT*ejlv=qZn_n2@?<s2E=X1? zU;wrR$HNeiO$8?gI=Z9Y#Dx;?R8TMS;c`auHQhR7T_p;xMi*_;6nu=EDztlqizwRb z#YETT&f42rs^hble1L_I#L1@GE9cxfGgs=hQJqHA9LMz9^$}EdGw#NFVapDigazK` zhHN7j?rmF*4yM#}j!AHVR0v|GRcJ`HtP~Z6K%*O0bbPCfrcI!(#p#Tsl%US|a;U2( z!Wf#M3AYu&A?&S)Z5$vVayS9!krV8<$BTM2$9Xsdt_i1b53I0)Z~_^^a68c`$DpB1{Ic&(csw#vTEP10Y7iwTZ*NAi> ztMt*iO_m3b3tht-_PV#_N8<=CvLy=sb|c&f#^BKo=z%<4$U*22=N-KODz3$KJCY_) zMSIEYxI(bC58x(}A*-zvgKQJ3PLC2yr?M6XGiO(|B|Qm)Bs1wI8zyA9g~O)OkVX@% z^;nwl`Vu6UMMxe7>zOG9^-h&hl5L)eqbf6$Y+E6_wq`q-2XqW1&xu*cjFJ*USGL($ z8_|(f(dJK4Jx^Y1dIOOLS=M3A<1pH&@SR>TM+NM$Zq{GQn_gL)5i^+aqq-Aj;kesAj}kj z20Gog!6@oE%0VSuO-V%s&(`?1>{0L*m(Mj)sJka9f)54~`EX3OQXH$MF+O1v@?~d5 zOQBbmB}3}ascss87LrK?WW0cLE;$BbtzpwZqt(!eb~fz>hqR1qgnWc*u%~D1LY&KssiVnSpmP9=#0N0AK zZY)_2$3(nsWpXauYRX~~fFRi@R&|9+B^ML2sUCpfB+vbH*Dq>X(HPf;ySm=o= zizX5soKXpbEtEJZVps!w2M6qQbUbPJPpWS3wA5+<03GNf?S$8bNEC_)pf_|eNehkxR%JnF?z?cQ;`j)+*3w^^$GAWftPB6)rf_P2LHKDKbg*quowNgLds+e=x zdvGk(A!UWs`maa_I6cRTJSiz?7)cfmj-?c1We3)QvE_=xf|5mFLIasawvvoGr6~Ld zP?+JmD;+7+eXx^Ov%w+(uHl6)6)E>BX;$~>J{M+2Gd#x=-V#BoS^*pjM_jIo_Bw6v zk!bbCQ$#qF=){rPaA0^eTg#)nh%Qot*i|D6$eLiLV8)7Oi?Fr9pa7%-elbN?-R|zC z$SksG?z-;tWLfkASPY^xCIro-y4@f=grGI;I$V;%y66!Y1&ESfHb;3N4GvAgVmN7Y z>fX3%2c$tVSAd=WEE-Ih-PHyT73er79keV8wL{n*qR~dUBaSRet5u66s9KXL*D-?& zP{OCN(r(>TR@y*EQIHr^Q9>x>2_ig4!e-(|8xtr`Tn`r;ZPPM}J+q@zb&YKYU_-rI z@Ytdhz+^FG_$juYx78tkvd6ff@D4V@NNCjtlcGac6-FztiYd0+MVB{-+wzb>S?+n% zSjKeWjHXEF8-YXt8PG5!0f;Ovu^ux3L^Xw~8qJC(B8=lOY7?o=V&?N0oX5hDg=feu>!r+4! z0(ak1XrXigz>&bLb zHz9+ZAjVjCAYb>zBlaSSF6A0D(W$msuLeDbH(^hnfpw$9Xw+HU885^McX}8HiF$O^ z4p^)zys9qP3>%c(5d_#He9wkVAKhijJ~fdHOD@(H38BH)3qi0(t>O&;Ulk)IRAi^e zlw{EzD2p;x4(ncDwyoN_f)(fWpyV<5GQ>yWn^e&Oeuz(oG&7g2rWCM{nB#36WGBL^ zpLes7Ix>g8d;()~W(+_K0J6mqq=kpQPJ{-^ccVHXH-uJfJD$bjJYHG~2}qZ&)o9D+ref zC!?03xa~<-s~=xZkU$Cmdl0yn#ZWLXYrwBemSSu0x61MDh}9L-sD{LFl5A=;W|wa9hQ!E3N>#am`D}t zC5aEq7;V(T4Y`fhCiXhzVrUzIV&!@_$VqX9oG##SW2?KvbP!vCopuf)GpG+e74GD`Ut{MzHGPUJ8*S^*IZ4#+)5cT ziTQOoobJ(Swb6okK?85V5f=hJ-WQ_fbVF$inlqQzCCWs87TZ=0jr5?tRmrrf6x3fU z7Kw9-1ePb`Y}J-&CrJ{Lw^+Zdx+P~d#3TgBfGB%Is@sD8o2wIHWC1W7je8w@EBFIN z0;GlrNDWkM5v-fo(R7!$meKM#N1&O`QjpY2Ii0SfH(2DC)Vjdj{BBSK**(T;YN1OvT|*#v*z z?BW)l#f_uI*6Ue;57bdb*|<)oFW0@@g^D3>Zh&8)HBNQJA`avI=u z610uZ(^ZL0wx3Amsf2e(qET&co%@2lyK89vr=70#sQafz^%f6q-s>5 zAgrfKwibq^K_a~#f-bC?nnWaN84u-KnAVGFY$lcT)^V(8_NX4|Q3X_>8f37c4$@L9 z@j4ldV1m6AuVo;Z4JR}9Vg$^gEYWmW(ws&UF!|`wyox4Ut(u0Uw2%ULpms&?z=(ve zU`4tc4f7-9nQ~#F!`!(5`?`=CA+@u+nkV>#U zFi!oz6Gq2cdaf$tcG1+e7&4Y!XysbcW5VvEJ!y+a{j{fPEAvc~?3GCyO;Q=iZU}lgFajiP zX)$6lcm@Y>eWekBgN=ad#<3*O0)mZ?d&o{V$~%p!8Ziu?+D55Mz|-N(Ti{E2NxuiY z8ApclHzh}#4vG+rh0jJAlHLMox6M&@6wnwpC5{hgTP!rZsmI9#q^hU|=$EkT2cS&& zD~4RIMuqXVN@T3#@aLi#C`lg3zo7Yfmks9`Ur}kXvLqC3-D=*Or(;CSLS9)Jys~s! zuh-0KxG6g-I1w(;YysKeiaiVYb44H4(1Z+-ZMe&sO0b$M1TYepeEFmwO=nmqQ7}AM zxgxR*R6QRDoN+;_dy2^RXVDe1?BVQ0$WT+7NA2-g0}n&k0aaD1;VtQE-RU6(J5|7t z;h;MO4TcEcYt|A(BHag`(Z~mMq`T;FrU0+Y_liK{qoXG@@UjK(>hQq7pr?{V#}F!3 z3fl=>=v8bjR=0#A15`voG33?L2?HZ#A7m`Kpif7t9-iz`k($HA{E>)P0<^ANPSXvH zci}NQpNmD&++#;s#tt&CL|VWsB6;nNTm(M4$pRP{M+y?Nb*#*x9bjZ|_&|=x@va!! zD|6(~!JKK?5fhrDivvjrg-dBQj{1R_Ye|&gPP3l=PIy~Pm3!H=Dn~k_fK5r?k z6sZPl8mpG2b|)ctYgC#6%&C6nF!}IR8yP37B11Ezr=!7wq*~7FqOSpT0fkN}!9vnW zRly*ajIJ2|Qbx+iEr8Cuy0}@xT4bl*fRxC54FiZ1+$<`)QA0bFKu!+AmS{UD!BO-t zEd~ndw1~`{mdr^N2oh4ZFzldK>oDObJ%JLDBNWm?{Yy0=B1yUcG}@wBKMYc@o}j=Z z0K~mek)4%#y(iXW+7uFiL6U-D+Fp#xvXash&6?DVb>PZ?jqNb1tId#YC=MM7*^||v z+RL~!qrW^1n7Ek87R6vZtyS1|ybGBY27@U^w%7`!)eh0`6-2@=TOvb*dU21Nb_wW3 zTCFiu>D=40kV)+m$m}#;HkE{f2a4Z{6M6whNJ1B3*NQeM@Gy403 z;j)1F)MPadU@t%q*`Vw;XSiI~De#G*-FPkU2NZBUfs<$o#voso!0iblPSkTTKWan( z#dPc?KuouS734j**lId&?@$F^QP~iVrNE;M8Vy<#Lg;o}9{F#`M3!S|yI@6#4yx*y zso*XP7xG@11^YM<+_cfGE9EbCA(sV@!8JM#`w1li!gUTNhFUoj0FMdvz=)a!)I$ow zudzDjRRF(Wp#BR0f_kVw;HfH2l!KB?FrvTUgou+`a%-)y!zc!Oi4p+fY-PJiS1W-~ zs~Glz(T26B!(Q>liC)-cma9cUPl;qxOE#5Q0frIqGNR9Snlh_x$fFU5BfYK=*B!_d*%QnHytMQvzA7mXCinSzZb zSJ_4M*kqy#(M%yJwJSs|*o)GTyyUb&h>DFuqEuPdC|#jFcpz} zQ7$3cMm7*tNQ}n)dJX6YI$%e5vH`h(x~pDHb=#2r7x(BkaFbFY+MS5@6pw&>sYJzu z@4`NihK3KWLV_^iXYCR%1cjcO@ptmLQz0B?Jwak?!6ay#bhVNis8QdQFqBSsTBJ8V)K&_f+q+H1l1ax|)vr(Gr-1cdNw zwVvcn(qaaPFJazortPQ?1k*vkX}bAbEnQLyCGgZNmdyGRO4cv6v~mInMbZgL3{ZiR zT=Loah=zJlgIE;sb|^u@>}5e(9&QvDL{CtsrAm!WJHI5(EN=x64=Yv|y-E~3KQh?g66_IPCtb+^X4MO< z5Kb#n+6^$OiP4QfnvIsX~Kic=}ZSo+l<7V zIGRr3ho9q`C(c!zQTc>Mqc@5Z= zy0)UrVn}~Y2cL{@MQhET5lAH5tyEiVvsxdB-_~^07MW1M4NDDsIFrV0FjWJyn4Ciq z2r2Jl&=ReisI;{_MDT45NoH*VZBV9(9Pnz)3E0yjYi3a0WC7OX3ROELyR+?uT~MBO z8FYeakv>~737rK9SR83hD!Unr!)lyqg9IQnos}6hF&7FHmbKZb4pC$qb(#EY&KcMs zz65L_g2Y%Ik65N^25HA(=MTIkwhi$`BHt-^m|VQUgq4)wtGZ2lvn|;zV7V&SLe5&! zT@Pn+u)GkgI>qL!P=f&Q@^PKoUd^N(4V*Q7STdKG;s*kwhe|Q zbR{I%eu{RuMl8g!9lB`P;q>Pi4AQJ!me5i7X|$p=%&A5ajU|^&s7A}xm{bD7u8@O) zutE>=LYmOC4a?W*&p&oFEPSmdV656=$5kQ=EQD5601{5;vb-}ywVotBYBE_Av zWC3-aTz@|b@=sX6-SfUgnn80I4GxHgnXr#s@91%&ZS(~X9!U%0hPVf zc7`&MdLInt(4uy-k*^}2DM!s#a=LBqu)9PmNz_H`g|6ugWRUUp=(JbtgVC~?jL+pv zLk6qjsp)tF5kqu<7y>Jy5!}ZpD!N!_OSzz)D{2)W*rtS#uz-RD3pZMF&kJc#bk?62RhWbvQdg=w zD37}sgVZx#0aCRP;NrtTm;on^;=Wd=0HF-GiYR0ul8ZD_A|@7Nai(Foq(nv2RU9ZU zJWM6nw)yNK4RQPhqF%fn@p!VbfdgquIUUs4pvjWKmXpo179rQFRVx9EhLK2*Ysx$x zVEi^U7>HwlOmf*12w4M9ECw6>9ABUr5c~jfYP4!$wNkPvC}WW)E#w#C1*73&9ZEwe zu@0x6)l`6#i?%ohGqTIli+%XJ2|3}M6#!xum)F_nYU=g~m7`HE6W=l|p>yz6U?E}B`I-icFDXE#%%YA-Eybq##15Q>LZHQ>9xO)NVJDq0 zaz3xi;fit*1`-tgood~xbBGTt5miZKC`&@BO((K8cQXeWxWMzFa62hj6nnOq}d0@Yr) zA3bv#Y059KfV&iu4xDy(IoS#G5?{CAo(T+c~vz<|MxLddirGY1cX z9Y$0~aDRtKg6(!y%!z$wPmilZka=f5l+d{bsq-|+KrK1|T?5G;1kd(}oLd-XPzA(0 zY)$n`9cx=MSEVh;S*>w7^o$gSha9R~>yRz2ghvEUXkf>6rWfI6zIVr&=Ew!)RBG<*@%I79`oLbud{`+41J(!kMR zhEK3OVS8MrCKqdnj1ZX9JPy@VGg8Pe6daUo^8(#q6t;nE;AeY-O}RP} z%LT}@1`mN`2tqM5Bd*x7JC0LoI1i{a218W^tW-51GxCHzINqTu zqBqOBObGU~Fugh;F^Y&K`{M*Y!HkZMSc>JM`aQ*;^;`Xg2F-Z`opc6xRT>spMtG3N zq&uR|ne{``VxPw`0@V1rM98VAO~&iKP%>{bNGn`wyPJS&OagEhz6Cu(L1r;y6@kOi zQIMmtI8boPQOT@p6lJ$dvcXm5hA#&M2Su#R11MdP0IbvXni}Ji>U~;hWHD^@yu%GB z!5U=+R2K$bnplP?PP_vIMZ@*vDOB<`z{c5wrC>P~io(lzGN}FZQZ!@}Y=I)O3&Y%+ zNnemFW*b}`96P3oQ0Tlb0{$>h$xa=1^FQTOhK-?uDA>3Vu$XiSe?)}v5m|3nMatLF zP@m6~9YlsH2$22d2muS574-Qck-Pv&bfA=G2IMZTPzH*55O|U&Lak_uRZCcwCL2oL zsTRRQbtq&Ovl>ZYta3yF7mWOKXB%%bOoC{zLeUAxjJzuzf_+gq11=N%_Gk?9$7Gk; zlG`T3QNUDZ`K*4TVaS~-IBE$!=1-(-5b3vCWE#=FIw8v`PlGK+a+@uHyPi;KCFFt^ zbL8`-a;={0x`5R>4P1xFIQEL*fu%Xp&nbl}l<@|hZ>%PGx(1zb$kI!8C0b`R$7 z0yzl8@tRN|l_i*fX4%PDisf0v;o5*zt)GDJ!rrn-G+af-!#NEDf<=7A@6cQ-*~+T< zfECDAQ*w+gXkjM{6za`b*I`<9)Zh)86UbPP16I(!%?)xES8PFvvRcr^mKZ+Y4oG1E zuqp&_)){eGA=11f;PSyviB#z2!f1}Efl&sR5uTcY(I04LGcmIrG!kal%^`N+e27tL zv#M2Ohh;P)9BaTiKSs&Qm~-Uep1?+?4YXk0a>v8#Kfw zAj3K+f;|^RT{pdnmMzNwBd83sFJE*skR=m_WJ;;wH=xg>w(6w99XmsHBL&o10Y}hc z-N=53F`G$JH5&tKUr44%!>qy!n5R-|5Z-JdOh|FI7>XuRO;{g!#6EITMGQcEE|RV? z`2xZ!I4DLU!a)|jeFSI2z=lN@}7FEP88jk{MI9qTB2n?P&Y=5Yd z#6cUu7(m;|!$hixec1T?8O*;X0TnRf|Ekrwwcp+QuYCEU^_#@P9 zh9S>f19(Bva8WdDJ`uRuuXIf8fkAL|yMRhg(N3f5X9PAIvi!kv*cSGh31D3na1m&= zN)Vt=OQjIcW(-B3E0C1|snj9T83j@b=&E_skQ*7Q&M?7tyAabT$i)o$swKM&K=dv{ zKvb3?lA*9a=dqML(8@Lldk5aS6~S>U)}p+6i1R@wbacThDEQp`tmg&JbQ9#lgxo#3%ffkM6x29$u{skyQ}k4+TnnB}Os>QW6b1yN3D zwCD&9%RCokEg&-+5vxffqSnh?x><@xk@p5aHHL@y0l>uwJIUKf52PN2>%oxFMAp>F zCNKfAc$;7c(R9z3A;_I-Q>t0DqHSQ_Y)V}qS%u(aGtmvDxbRIl9X8;n|q5zEa!-$8nImn{MR6A(8 z@pYnhfbDa%NlAcW0-|^#X*JWF=mxO|-EG?Zfj9w~|G?e~%kga89;s7c+#MzOu9AUW z;)*TrjioTcWp;W2w+6GRBc;(=0PKS!A{WX&yjgBUkhMlLL4>gJBBxVi!5e`}cGT)I ztC;2*WI|wDx4Avp+pLp6TK9=kj<2}U{UafTN0VH1)H`mBoL9!0BC9W zb*dWFR9@(0Q+8+yM_og`NmT10)$48%ZqnDqewny4eFP7+a}wfEfzv&?glTSfVY6 ztz-Sn6;WuDY~Qs@n4~!2D`v0>}-QeDqFM% zC{`#sfIyb8l_bD#UC#p>2hoA8-)z2!1!`ut(zf+;-cn`GQkLmNuLJ*26^xEuHey|~hR5M^h4niVQJ{iJhr1YB(kHk|tmp@fXpCu<>qR4$ zA)N|vMC58E6hwkKI+iSk9PnL7jR9#71qeL|iK#@qO*LLGx(&fg*NAeB_SQvDIF#$4 zSrLAi9D%4}9gyVMk^8P{pAom}RC-#D;7q6BQmYo~|H(QN&Vr7Z9~f>&JR5~mWtemw zJnW5`3Dd6k8UnZnjz(T-e#(sxAqk8V^Om==O(@cN)FFEuOyNo+n6N&*$zr!2v! z_^@V;K<-G;YIQW&D}`&_O^{d^(PV(=8o?P9SNAltUZM)z2DnX+$036<8|}O9{d;kl zc%GB10H*}(sx62!YRwZ!Mn-&bw3^TzkqjteMVUU>BhKosOq3a(F8gy+#ocUzR!0Oa zdBQOv3A69;*-*!q2{rs&ADo!AS(WCe^#N>(Zj0m;l!wNXj!@E(-{&V-KEl)V!&(7e zu*q~v*%H+Cuq)7xd_)L zIU$=wFg2L50M^_dRjSU(*Svh z;U1QWFc8@EM9X%20#zwX;~g8k>#({&zT?gS6#*b>1#}X#vK@=q*MbFSB&u{nimgMX z;G|I+iC(~pFk1+yEjt&t0?1#iWh0Tur+v#QIc-3LBg6zRF{}hrkZb_4v;}+`1!`^Z z1^Dr3eF$C7!6w|`o%efH!VY{ovd;{dMWRchu?`ou@)6Xk5)IpR$>A%boXy=USr(@P z%~>Jn1$DK46afS~;5&%xw@_z;CBX;^Ra8FBYxg9S5#a^!18aq1!`{@Ow7CBUn}FK${kxZpo95<(OcycmADbDIh~+B37a#3Q}2rh zm+HV;{#^_lIq(3HkT3d!!6d!!KO>yQ-co}#B+h;UQ3HD3^798m4K3P+WJQJ~1qeJ_YO=-p&sZ^1IT)yLp{); z9_Ua5TzsHIJZ=ui)Is0TXK10Cvt4)s8XdZ0r+(4ijaP(!BsK!-p@xK}fe!URhkBqxJZ=ui)Is0TXK10Cvt4)s8XdZ0r+(4ija zP!Dvd2RhUcD$2nkd7wi*(4ijaP!DvdQM_!RLp{);9_Ua5hp8hz(4ijaP!Dvd2RhUP z9qNG&HMm9t9qNG&HFSf44)s8XdZ0r+(4ijaP*(>!)B_#rfe!URhkBqxJ6CO4 zeeXI%yO#dUh@1qMTgZFO)=Uhk=eBTE_(MmN%SG191UnGY@Fge~T8&)o418od9ay0MwFUYcmxI7Qo&Wz;-{QZl z4@40oGu4P6h_j#8!G9$m=nIDcHsZ|v2aBg_wbX2bH}7Z{b4}#c=c^3^RzD7_Skjev zGnYpPEiKoCW1m?ieUW>h!o5->3z(`>$mP3SJzs2SeGjrdasplCh5uu4zhN8sH-pq} zLyI+K#8vMv0spaA_}4D}zgWAlBi7(E%M=VBS#0iG5#S@$wXY}atG<2xpz!Ik=%4ex z|Ia(M=rz%^|8avt#lhn6vn};S4LRI;&PKI?7M%4)9rXaURIUEapm1G&My#HpQoHO)Bdw8Xc+F4WJLr&^1(b0;wINqBR=OXzr72Tz_Q;4c-6Qw~T_pR}3{| z1tJumxu$%27%75FBOMU0Ys~q3{f?FObx4!a4@9J_~H2$=&t#F-i)EC_HkOj;^@CNaVe?0oM z=ALDVYu@>rtM)&%dE%HWXI_6)Gj`GV7oJn+t^8bFp8eK=hb%a3!$Cj&g10z({NeD9 z#4PcEFR0AhKY~iTtnp?4MQ6WyKxdJyd4fIH$V9B(tm$q$r&_t-h}6$yY&x?qsO1|BWq{uEGYTY)WvUxE?%bONa%WLsUixJE>$m;4Q6` z|N5uqmL%Sk z{Lx(M$Vsc+!`=7da}NE^!I_~uHvMMwS*t2{%vLAAw0y?a(dRGNG4$Bu-xHpG|7-Gr z2L!XXj6dz1$fK7-hZZLsNh}+B_0^A_{$X}9w9r%6jxk9J6R%zf&`=qt~BWdR|N8uQ>gciwo;rhmKZ`MWNAi&-!lb-IhU z9dPZX4+d70r_Oq3!!x$6D_$8r<>ezp<16XiYaTuI0q*N-tVhTCeu_F^GXYgfuK4BV zr$3lDskCwGso!c4Q`UG6H&!lvXMG?W9v6CY{qmi2G80m%xtS@xX}?=O;iku*y!QQf z^H;B#v*TO%WB9_!<|pIu4OR8D|;yfG`M-@53- zDRV!JkXw&eS4Z|U&fa;%iU$|3y7#0rrp?;1=fn*!W#`WSgxG4Rt9uV)Tfc)JL2Ny1 z$va=4Gk^EYO7E$ax7K^nd*jJpU-$6^@o{9Wpgy!Eh;O-l=a^a3uS=|c;L#fvb^fr9 zQ z30JM8R&IG>?~OYidFWf+b2emPs?4FLeCE&t4qre#{mQS`8!Lal*~E`{=$AI{W2vD- z-dw2Gel?p+|8u7<88@^cJ14aB-tUf?!jFuJBM)3~_<|d6v~MC`yGC7o z1WR7&Cnwb|ykq7mz4)KWo2RzlePo_>%c>VIc!*0;>t3XmJ-_kBH|~Y{p;o>8(cKR% zx^(K#?w`42_wS~RtuA?M=*<5y<$Kg+FmzlQ=)Twg?l;dZe0i#W?c1K;95nVTQ>U#j zHs<{C{;PIeJlVYO*FOyp-9P8!_e$!<6PLet_}BN&-Z8{%c;2~u&7Sq=t=(C;X71wk zxjnnVgqDsNcX&!)dF$D0FZJ&|W#=?}%RJx2&8KwNe)G9$yC&@T_OjD%n3||n)WaiB z`X0My(xa*LuG=5L7R1h-bKm_JE_i40^O^IuJovt_X8D-oy1#yT;?_r|{;0Iyy!gwb z#?C)}?M>mCr_Fftg4crIKYs4-*1h-jt@z$k?pwU#ld1R3+fx3<(wj2B2~Yp^MfB!h z+;!W=(z*J|_xF_5y_E+8%Rk9lw_JD1j9J%*cBU5iF8sHV3Gjqb(Em1Xzw^T8y{j~Q z+L{~An=}8DpZ(?W3qN{b@v7}-tbO97bH{DEV&RFW%?>QNeAV&W_F#{#o%Ctf-A8{4f9 z;a%pm)Q=r!zJJLDkIsE$>FmkjdB_ zv2PHamA(l@{eshOxpu-O+giTqr$?t9wfX$NOk8qldd9g^$DQ@+)u-J89Z26fbk!Z< ztCznw>DqZ4EOKjl^i!*5Y`gQtos)kwYV6^6)yZ}Z-}3gg6EDAvUST`$zI8Lz)gSrB zeX{kLd(tb`ti3vT%3|>q&({`vqU15(U%!0)H@~@8TC)1>-=1~mt`p`={_X3JNoS3@ zZ}YP&Z&Fj0_Ya>pQy)5I!m@JVLg>pU*Iu04=q`GB?!EWkiKdCs`zH^~3~@JH`1Izz zLU8`7ZQ0nAZ@qTJBTxF~&*u{t+_mO}p|@`P_;)LP^9$RD-1DBF{G$tgvuoFv@hw+e zv$pM>vgn23b~p#GOsYPYV zja~V3-amiIn$~U50K~FC-4L2~{^k?A>+gR1z~W;iRP`4YR=@W1$SH@I?3eDZP8lHiu{3g2(4B{?kc|Zn$yEqE}m|?z(i%l*W?1?Pr%fa_p@iS6BIF zo;Y?(@RZ%JCbryl!gVuxuUzo5*;wr%##m)hE}BV}3rgVd~vat$B3SpB{hm zxls!i)Hd8Lv0KmJX5*od8@Y|Ap&3m-?4Gy9%~z;nVqNDqH|>3D+n8r=(!YDy;?>)F zKioP08`sT$YugxpzOpO+=(-nAMiwOVLivU5w_J0_18e>?F@(20cx67*SUK(B>2~M* z9~*z3)}DTY{`)_j0hRmvU!5&-vo}PiJyo1A{_tsErM4cbuKp6axCGN}1K+|NI5a2o zt5r)!uiJXXaYLm!AD`{Nbz5uVbUiZ9cFBVWuP}ElzWRahEu3nbE^qwf)wfd{3wwU= zf9T2Mi@#d2YQ=lo?|q(H_UGsC&E0r;Z|$hS6V)wqazpsuW7JviJo2;0cK`9?*B;)p zw!Pw}UB?o8#=kHwQGSVhfARI-FGWuKaK#Nnuix?LBNvW2>xIW||LQUGCw;Q(y02b1 z4)Azb2Q&dloCtay zXWg^rk)ex4C4TR^7l~!he+(5ghFEskmZ9+cw;#FT`tZ%?Cd<^rfFK z5wF_v#H_uyzy2+#p(Blp&p9CT+{r^%&N=9~^{*c=G;Q~|C6f=jZ*Mzj4Xr=@zWg^f z-K7qlJo%R6Z3{nH{r$b>wm*??E(zxj$gI8ocT@OuQd!NNZoK@-^XofzU3$~B>n0z+ zdB)bM$8obCEzQ99zP5bKPdDFvn_PP7q_IB>KCxojv9;VmnY&h)i_KT2pM0KPnosUs z_r3UmnTh)?p8UNBt~+!7mXjjAqaHB+bn;j_uFu9o3#Gw^B0{m>zeL8Ykt}b z-7)Sjn-@JYJ}{r$b@30=&sHD3r8MgvXGCtVcu@Ju82K_|`nFx@ z+l=6thnIhH#{9*Q7T-^2Co}8h^U8 zWNK$_^`VPzn>uIn-ILZX^30xW-!yJ-z!^Dr!<5ahkBSUk`1=DhZ#>%29zSQtt)5Zm zmP+rUMeXY+2(KNN`N3)Te7t1k4eLAk$KUNdG{)>sSo6x#BeuIIO{|@MprQ~>z;9E_kQCoM*i}`X(eLueBqn7Otvqa{MM}RFF9?EW$p^a z4ucB0{r3I2OHP0PwYzdZ{$%!1W7jO(d-B(2JiEx6HT7GY?tbfsOWgd@ShlJ*gbzR1 zxaE(Lmvwe5bIvK#@i$|5rE%i>$-A%n;Hh(;r55i#=?c%DtG_m5zdK*pxgk4m!?>c_ zIN$f-&X-mkJ2d47@9)0nx(j>r-o9wZDeq_AnQ`=OvjfwzsM6LqJn3V?07$=&y!!vdE;yH4(IpAI+IsC`Oy*c?-=*!yW4MFQwdCwU(HSW<&uSeh|F@J)uwsq0f#PF zbn?)n?AR~Aa1%7_)YCV8^Id)IZ7n4&R=}Hd&>Q7$ePy05zM5BJ{o zd7zO-{`!0=Gygge%3u-o>zW+_x$A_ z&Zx^x}!qd(spy!659W4DB6UB7}{xcY_J;{!{d`129i z9lxGE^(HV;W4>FS@c7p0wL2!v`t$lDA6|3WIUC;DH1p~e$U%=DbI2HZ^SqPS{>Jmf z_jjIh_))3kuHQ|$a^qoJ@v)}`mYxLD;^Ia-ZXA*=HL%j+xSauw=7!o zvc-9rB0U8`H8i1%IrH?TJK&0WWyH0Fn!=e3N}9{t(gP5CRcsW%G?t~gfS ztPOp9Oyhk2;Y+q1<$QDY?D!{JLWLVAZ2R#A6AwIe!L<|CE;?n-<%zXVgytoVJ$cX4 zKkDrCe~TYHX$}32-fc`z+5Xf?ugo4b_KbCIb+z#7O^c>)`^A1^ADp`3@=c-M2@i0$ zTs~*wjtO5`tZe(`%SXPpWyin&VEV%2yKgL-KjQ}%{CLD0eEg_Gz9w%za?~Dr^t+Gz zkyclqU|)Ok=rvOg=_c1Se^nly4nOWY)VrU4eebm=@1CVCn7e&Ua%pP&$&0U9y`|Ln z!_b-ZjC6H0tO>7t@a&hz2Tpr9_tGl$<15(fFD`oTSzOw6+iy2%6W6~c;OVaun?59; zW>zlW^NTO(nZp;tsUAG@|#=W;K@6J4HG3o6Mh#ka(BM4W!;`# zjUT?co&4RZ-WKt^^>5#}XDL_v)yuDZ6rb|WtB-x9xA^zht}s68EnfDgnU~(9_Aa=8 znzHJqjaTg9cJ7`zTV1u}RP*Sa%y#MIJLl@{X-Z-c49j`gpx%}pLv)L{8<==XD_a!radC0>bJ~-6A z+Xnt0gvFjYWWl1ZkD)f5c*)R|*G*X0{!(AJ>U$d>TmIBbb28&vKdY{cJXL3!JZYu4s{&eL}%yc47+pWATa3p?j7iGCPSS6}_T-QyNa{$Spud-R`A+k4lH zUtN$qmixh|1^V=dE}ebe(nn*F>EFD|`o$xUj5_-L3m@A4isw{g+<7-<&MQ6mc470+ z?%MwC6*uWW!nZ%V*|GGrn~p!?hB=x4$=-X%Q{Dgn;}Q{x5{ht8$c~UbN{Z}^W6QDk z&UUCI;mF>VmCdoTsc`J(%Qujlje ze9Ze})L7e9(p5@g{xH4j`=*KtdVrvs<>rtytyyoNnrDi#n5$=Bt8D8Mtv|c#n>(Wq z7?&5v(`EHYVP!o-Js5^zt`tW=WM%UVdZ~^3)-;^(&sLFleJ6MPW>?j|u$c9M-{U^u z!!edYOQB(i>`u%)Z(&fs%TSnHBU|sH(6U)xe+LMGMM{cvl;yO?$&PG7FJttf8a{~1 zpi0;D=o~NP{wqLgQ)fdiWm-o^HsZA5lg9HCppcV2N+l~lH zKXT&=^?&&3J8ldlN%;RONRk9*^tJvoK13g2zlbk?fdD6@c?SV-Ca!@=I{tl4K=rOR z>F@u9hJO%{z@$o-nFWsT>GVP-!oiKHw8!m<2heu~1pj=2xGlh7Ctg~fIoayz&LPC$E3V?k6o2QR<9*cul-!pJiHoC6 zI?*ZFuNlAhPXs>_VBOFzk}GH7_~*gVJ{B;u{Nv3eOyI?Liy56HPG5NZr>AV<2(qb$ zRP+D-V)1wYW>5CS_Ta(~@PC>bc*PGA{$pJq@wZqK5ViM2*rz2#RS4ie;ZV$#8u znK8va_vUJ{gETIkycu#GUgVqoI_^_PD~H%clTu4uxW@S_aYNxTRAto#nPd@<@NJK# z#KrZqu;bBp#ZNWfIt$_v!fEcEgkNr}#r(iE>`Nrf?CcQ+UtT`yNEGSbbAK%!!Tdqt z9X)FDpiv1feiQS3w!eL!9nlxW&u_nLO}O=DM}Y>($~TXd zMy);27cEq;CI>kuCW5YqkPmrlC`;n3(@#w7a&qP_ow5)Fhzhwh9`+gB>y@DlD1v9L z`GVMXmu!C?P(-CA(;VNZI1K0s`eDZ_BjjhSr!CC_M#BB12_2@I)u!lbcR6X6AF>}) z3eJpC6dA?LSrOPHDCi*Dd$QLM);ltMSMGrW-L`y^c?<>OYj6O@b3LmhypMVNRHx)V zv8>E1aa|k?G3z4|PVHR^ImkTz5Qn_;@#DA0fu0GurX~&Hz`mG>p6qh$Imv6i$_CsB ziwpK0rmqc(XJ-$Mvj?JPtvzTijB^)VpG=)`B6B1+ffzFvlN%K)`%nq6#@#HE1UZ`r zO)sA^g9E(`l8HDz(3uu+b~YQxYm*wx+d+;DZSd&)B~+k~-&{@QCnV`H)v@3`7zw7v z$GKwr^Fc6#b>jAy1^lm4wRIbM+OffIC&vjT1A<}Mtz@f(^C`42^`^bbMH=?jAXcp) zf2_HhXGILfCwBV74Iq)JeIK&b2y7Wlj0&HtG5cnJV;)qzfC~>_c9E z-~PQ8YsOo@IWd09NX-+CNKQxso5{EP;K;D^?H+N{OCqZ2x3V0e3%yx$iZ{LDoPQtF z%irz?@8ki6U`7O$+r5=6Za1H(sQY?=>;0`zLE0DX*oW@$+mrJ?xcFKU z5kbu|SF<$op@e*{=40{InLkM^0R3_TgYkuL`T>XkCvvNy-={44@vE?yIUa3k6=T~n z1B5`^-&Fd?aV>!O?`+mz*MlekWNd)V=aI|?ji5pqX6DnABJHLc9zA*#;J7jZ-5#>w zEIZm;G}>P-9si)%62(bDy<X=22ZtfsZijv9*MsC%Clm*Bupxh{HDy<2Ps0=?Lm`Nty1ea^AA(nhu?|4Nr;j&RkI@T zQydSnj9MaZMhdKbC7l5~k@Yx4F@&I^ZyQe6Q|LXuux zzbs>^%6=6Y8S#Ga{Jb<2wV~&1t)vyWq1U7NX8nXCIw?TP+kon&9YeZEro6yR_TVi=Fru~RT5r;eD-YU!2mPGc?!l}xZmOtb(N~k&KeBI>~ zD*c#KzsvdeggpGmZw4i*qAGjrY`57d-6G3b6|N!WuD@K^Am&9FBi+mX4>57d6#89TFz}VRw>uwak$mb ztzV0^`)ewM^lX)TSaq2qtw~n<8=rsBuu@5X$Ey0&!-qy;wBC#cF2$P4uN=;b>{aRLpLP(8W^P#M7sIFdV6UQ z33!CbO>6!t)M=q-@I#S;@Y?+k{t^5oedSJzm;Gue$jBZw22+=6=r*>N&!@$uKnsln zW-0?{po6sd7spVSU`F1b?iMlv4XY>X_YN^M z+u{C>LZ#U^v3V`R9iUHQxQ$+4xPDiWqiRzbTJ2E=Zs%gaT0xpz9HU(9ok2@#&$E+b z?yKy>HZ@hP@d9BG*5})E4>)k{2lCe?zZm4b_`g+z7u*=qrQA2#_({)}_Gd#?j@ClF zQBaeFh-ZZFxBJR}d`cL`t0+0Ce%iUSdA1Sb+Op2o6(Dxj^_mkK+YtAVb|U=*~DtX@`hx13L@LcR_ROif9-Q9wjg;z;t$ zv+`TqW{y9PaVr8tAKz?sUDumy4foPA|MDR|XA%?6MNJ&cHP8WPrz7ZPYUHsK<5T~e zq2lwcr;A=OJa~ghPjr)M2%$~FPZ4xkVFSu4VllTuo|4b^0@mi{HzdO0BLxQ=AkE|j&nt%?GfN`>x-f0jXD0 zy{xa80VEt-mRI+cy;3P)Y4V3tnjI}}9mBjeI+0RNjiwPNle-}+_ zaT-s;fEyZ7cFh7Z;fAPBesMqqU@}me7I^kFnud%6*4ZUD@h%|!%e8ZszD*_x5yH6h zgAxVg$7b6{+!QtTD&!Pl*2>CicxKU6*$aWFu^rapc8?6<8STFxqC^SG_{{$btgyB#lyn4;k8+#8T z2aF4@O@?77_KxHi|3VUd9v~i*p5mY$ufj7!hYJK4bq5+$r@Z0-c%`*X*%$LI6ShWq3tFTZnaxIaSnZ~yDMj?WuDO$J!1M!If<-Ue+Xq_~ zvKz~#q2xge(=(s!xKOWG;BTdxHV#ec=4#j|o7PnCq6NIL6g`hgR(l&>)b-wD6lKMw zJ|)(!!6??QZS4C>uq$6eC!@m3>J~>XopR2@X1)|v?Z#Ndg7xJA4GxTV<5+fHJSrzV zBCG#VIv>$AKl|d`6=2=dD{;r~&eizxpTT`74fsn?hV2wBBsmK&di&mE1jVjdqV@n? zq*#)ruBFLu!=_x7W*J`en!?Zc9YGn1MuB-yYeJ0%UwBTGr;8bzQ|WEE=|x673|%km z-UaSIH**|myZ3dRQwrIUJ!HMr?1bPoJd(L~_yv0&siKmhQu98%dxIsnQA=WhHwzXJ z-m1quD##z2Hf%-eb`7v6Q?a>UJUZL_B0<=Dz-|}nG_IdqXssLE7Y{l_`hCjIAys9+ z0jcw)A;2y;ngm}Fg#s$?GkF$xZW7wCq_}-QiQAU%n=1hKb#)sYHGF*8IEw)d+USd+1-KS7| zSwih&e4iDYItHIuj1}y|1;<{TO|?HhvX2V3*-h!P^3b?Do&njKS{ylA%e7O_<*A)t zWG%UfwMWVe^+aA2>LJP@xp)rxCY0=R=a*ehocnyCq2Q$dsG+P?hWmm-jn}&ddhOw! zE>^5mS^Xv9{!i=W^qXGyuD5Jnp6|>tGF1gJdr((1(F{Fm4n$rmjn#_r=%?dfpO74% z(N^-EzC>TqTQX(jv!Eig?pV8m_`5wz8Ph)kcwpnfL@BOcY_Xq!dvvGdA->rpg#a!# zF;OE23)#d$Bm4w>z@*W^hV1Iy`VLZWMn1OiiEhRgH^6WzNdfB~rb ztZz_Y2xJi_w2cd-W}h+5_n1K9OSlB&u&Vtnwm~P8jb{`gWDQ4`O$h^oHDF-M_No$h{~;SLJ%pBUM6_`{dH|7 zZz}hERGGbG>FoO_z+PT?V8Q=3kOm5v$!cYQ;?0KMaaQH`b_yt%A0kxE&$)SfZ(0tz z_a~%`EDm3CH)k;otc@*BkO6Igy{d(H#!5i<4GhH`MeA6vV(s~Mm3TS=b{TK{vW z3b+TCfD*R6R*~>wMS7QkIlnqvyZXAAjyx0Uwf^Ev4XTU)3Digp`(QHuoLDgSv7lqY zKLXt-RfT)Yy8&1;qsNhz*oH7%$fg_Y5)4-y)|(BYd?UHW*T?k3!afFM8N9(4a7!3b z1mX`7WLP1pX9_K(19!wl2W-usmdZmqUdR8jW?YVH}+TCA|1lD>ARn`yZVj5>6EjZ;5s#mv4Tcp&-H1rqrE0RPk z%C93`@XW?OM0l((7v)$lEpm#6@+b)3nWEMmUFDefoG-qXM^Q`$TpGm61Y&{Nd7tdyKmS97m&K%^R8tp>l?5yFNy4%R1c?LTiCD{IZPayT!sZ5 zE$|4e_9T}Km_S`)!lNP{0Tcp+at?vxdak2ZU5k@7=Qy&KsQ|>eO^(d+P$C4PV|O$> zq(?F*R?*HGe(vOD5C`}GE_Ai#FjKp8-|-eFXH<>0eUP?;1G-g}O`8G4NvdnKm2=LW zu4u6Xr9`ha5kiZYBzik=%`q6>J?C1KpNY{A%KR7RYS4o~fPOZ%WS;Vd8va)n)E?}s zyM3nsQ}fSz*-LtUd%9boF50J66(~-d#UM$H{;W-6&pJU& z@HK9b#DM~U7v&Rg3}MSdyz6rgoZ(t{!r>~jGcA!ahv$Y>%R_iA)MS^3h94H2o2{j( zwlm{GDQ~Fppvn3_hN_MFaugMpe6$?ie2CAo(zmAsQd1jxy^XMHU!agwOL1jSphZyb zbRUwFQol`AW@lIv=^!s4sg|dCiz&%2am00eN{c8|^x9f(<@#qHZo2~wC|AvW&0;eq zX_^ftfvoSQy1JCF@mWT1E)DYJES_&?IlX9n9I}UtpSx$N%9=r;<6J z>MeGYfBP`-j#;0iHPx^D>-N8HD_6NW#cVHj1SF~@i|#6dS}7)1hv@-1o2?Ds?2=_~ zLpKX6&cjt&B=pdN&K$})%{>Xpy z^811F>4`V{Hy1hOV)+y{f4D8zo6NLp9kE!b<-M(-_0W{RJJb4AumlgPiC(C$53p-G z@@Rh%@Y6iz{49qE`pT|(Pa|^t3?9>GVVmX?8eEZxBp9Hab~tKTy4!99L^A}!T9_P~ zdK#Mg;@{^(l}~4St&e+)+}e( zubB!W&NZ9p;a>ZRI{%h1w5v(r`q3X{9NA~(5>DJEnoI&SySAm&vWRxxG2Yres_3-u zQrf>s!*gIiIr4SYzL$Ldh>ir7%35#FqHl=NG8=|GQ5Z)Hgg7A&`%XWPp$)p*Ug-lB{KT+@0sO|%z9%n3tmstjJd4rBp~>E%kvT(};|#KkKkF#OUTte{DD0u;M5*I$ zoNJ&xocjDV@AtVb{nvB#W6srpYCiY~BRV!MSa3@mdMcQiQOBY~>WJ;INPye8qw4B? z;v8dk&E{d+pyA0NUrtf}`+GW`1|Oorc3$am4;1NnzbzQ|2>n$hN)SC@U9Z1Nu~T*H zcCGt5xGHsR_bgM+fa!UTjp&K)5V3qH!bv}hFCY;necj$5R=GkEN*rm(Nw#&R>O`Ve6O~+fQR;8(4T{RQfb+>Yr z=xcKsvl%mgX4VXzpK7jAv`3^?WqS>0I)C7Y3WoJtv}|L3keLZ#RksQaPUpA>WdAqc z4=@1mezwt;1LL;hKx1e_vrk`#A5wI~%s0n7BUM|Pb4 zu1Vj8jb8@tX1!^iPh&utulI^w{u|(7xNb)f8;u_l4}D3)IFd_qU`T@)HZejAu{|k`qle$ zP)gJJ%^t%YS#*sK`*2?l)7ceT7ChQ99rNDIgsaaEZTfZSX{&eAmU`Ao*Jn|yDIY&F zn@IG<_GV4zTPMKkD$DGj?T}7C;L-PN>07=$Wn8u2*J6K2R&_sWElPw|1i7;4U3jno zKME+lA-b`S7XV*#qU7JUC_C`&hONZM1g!d<`-|KcFyy93DZ`n!FdX5D(PkAJ*dWhi zosH>^`ICyP5@+*coB2s|dnDI@;o5f7vrcKE{ya{(18sNV`6tN_dd|<6&<}Yaic_Kt?x5eBdTlJi+90ret50}Xf2wUn;mOPKN?*!`i+8;GV zQVao-HPdLNDguaaw6i0Rh)AHaIL-aTI7vXg7;S;T?7LCYs#lKRT$ZJ#<-X{8D~-7= zqd$Gmj~)JSUB`{a+;srS^rt+*KMq`M z`+34SQ=r7>P}aJbn%!y(e(|5x|oPE#(npJ_tbDe?QYv zTXLl|=!&&pRN96CT3shSrIZqY<6pZGDj&T8i?&3!&Xa+t-uh@jH4#80vEc+KjNA?Y zS%LoPC((eQjaXoh5D=JSJT9`l;wnSui8*N3i!%?74m@?h5 zYfviyAm_Q*cmhyQ@arn5pu>6&0`va^P$VB9R#=i0cxLv;wE%z<{{4Kr{2}W!fVY}+ zqzA(=A;#lV*1E3)@cISzhr2xF0Ch3sPFDvJe)+DUx@)_4frT>5N%hNJCcqxrVJZe7 znx{Pr_UnTHLFvB(B)EJ`|69;U5dpFA96aZ`vj|1b&W{rUtbP{u_GY;%Xh5fYsx@^usI{~UmAn?=CM%Mi$Xe?*N|FysaZ09qf2Fk!9H{i8cK1P)mm2j!hVu{npHZyy^D-9fi$i z0z-Nf!M)Bgt7%M$nTd^iV;KzK(Ie$Om=^KhYQ$C=0GpAQ6=JHijX78%5{CUZw(q*c z2E`h)a=A@Dp6^M6JC4v*77YG{kPe^yR!TEhLqalm{{he(+>=iYjwJ%?j4TF{%xFT{ z%*iB_6Q!s+G&CbOuLj5!q_t`=TlD8di8-6ysF(V_*`R#gH+$eZeT3Q#3K3)_HAutn z6d0;3-$f7hP=@0k?@b)?O1#c5JHn!`<>&32UAb~RW?fwIb)|kj`PRa?4Tr*7eL1W= zbZUedHFD4BB9kT5go)o?DQ?C@T^$+sGV~*64%E5aoZp<>BjPLu%-2lcmF-7;d=!{s zmb{$0d~vh2Swcvt>d|sQzUd&#pi7i-7t%n}-{XPqMc@LVm=*}Yb%L5NVQAk^ZV$8w+VUo5*dF`Z%^gn@^_T zj`gRUJ0G>Q%=?%l=~cQWCl~Cj@D|7v-^D%&D^6WZb@Ys3gH}HvXV+oxeSP2AtT3U- z#yI#rVoXs3dK>6QzQmcVIR?}YzE@(-T#XTZ@8*Ge)JD=w<4gAXsRI*PS;dPdHwpK^ z*Pwv8MsPrpYR0Yn@UO4Ap%vBIeG!p!Ul_ReuR1z#&bv2$LxuDA z2*OHJS(a4~Dn}TJMStn9IM+sY1DI4Mj^!;OIX&w2jVCmOZM&Qy1FB1}H4&;TP=npJpfuw4gGdqoq_CUc4@?vag`OQ1;w7C9s0hYXOBa1F3oT6l{=57T zIrq~6c_SXZWabPKtJ@PQmyfAXs|4)JD?VX<`|Uf|!AFr4hs@{X7iEe>C32Xg-)HT( z4ZY}C%fID`!Q6^$$8a|7msA4olrO-?5Ix0BI#lxu^L#6NkSnH`9g0VP?ty8Y8pe50 zDVv%M`_7HoJ~P<pf^5d5 z?Wsq6!Z=`eMACm1nVVozNM`0NuTSiNPwg!5^Dc?dgPIq!yO>Pm&szam;IDE^gttx$ zA#8PlWA(uwBzFF9)@nduluGf9|97ili6EcaUBEO1ya5SYj6zZexhicd*M-v(h>1$@ z6usN;wgFol=ynnAO}huc!T-~7e|$_!OiHMe!S@fzp1xsozbK|ID~XsfZ&9&1CGH$} z!N0Heje(WC5krc$ADBxQy=kWkUfCO&K6a|j+6lo^|3k392=X5YCho`fXF}H`MA4F^ zV5HS^O7YqPCa9Ud=R*VCRcx;l1;u+lu``V4k;vzUyeGE=7koJ(|EJ0Y;t34RX3l!~ zG{~-FN?vd6KvfZ{%F6UXRQ!>XPY)6Px1z9r=ldgC&c3$#Q+VVf3pkvkn9oTH>(6MX z3oZC4dyq=2z&PfEm%f9}JkxOT6=t z+4^*tf~IE0GqOMktM2k6t7FdrE)2YZnM-jxbUxz3Ku54yH-Z|$mYSn2&fTN^RnpWE ztJ2YwM`4UzRqigko81bj(jheKyK_k(ZB}g59DZvZRE9D}S_~CsRDycWt_&5XKnseb z4+2+Vw;!l^T+}l^ZnS#=NW+pX<}M8sD0nAy#PmoPdi1o!W?a{KzyMDG)F{tPg^0;- z)~+jTHZF}C(2jtu@NIv3{I3yapj;p3JJHa}S&CPHoRiI=bz)ynQ5JonuXqfj$^hw0P z3creP@k(OBuVZSjuHJpqmOjiHIHqA>YEn(~rzGl_nNKV6V9Wq)q0@Lzo|makUR6NQ z%S{j1_QegsKBtK=r?4?41T5`r$}9mn8$@{!}m3pMcVF;D-W&FEcywK22ooL zQLBe-Ju<7UrZIY#p}+R`4iPedS$rEC5OnvZSG~~{Y*6HVuf2IDAfr$mi%UuLLjR4o zcQ5^PGoqdy9u?f!V&*Bmv!Tet&%H5dY`vm2>#)IFRj}UuU}QwlsMQN;$70D=u8yyKn7$8J?pSX0EJV^`K5x2BG?s@g3F)4_v{q=6zAw+`9Fa zAPe62HU+Ci3r4*0A~>i1H@2Zjr|*d<_E^3Q{bZiYrpv3)m#fFV*xWC(^?(hvYyvNe>b$&GU;0(cwKfKW6DjDmLs2(DErJdG8ugfX{r`O{BrSvTRP)hzJXcmUZ616#1a?u4*MFQgbuTl_T>eh%D?}I!I8ERQotd^iB3|%=AlzUFNmYtqHn9*8jGP$XZP1s#js7dmyVC zXJkFAiR_^!(G0(wbA(%QNsHMF>fhx|>C#uIr2jez{3>C}Z*jaWivN)k8y|->p9K|L zXqow#;~Iky@378miypdKg1(aWJQ1rpoLjsh$xxtwM6N_BP^nt7i8=!@DA?CFJ~*i)W4=*T$N0NEgM9iSLvv2lr!)iJk~V|#6Gf>Jpy6D8klQ_o7t zAa+lZm9x_PNvZWHE~X^?-bDe}KE`GbgU+61&?Tc*3fhr|`78(WRY6}3Rm|3cnyM

9k!`=TXRs-dfyJwTc?Y zcp?S_SMH!FdAD@T$~^G~HBv;hV@Oa{wI9eFV9VuX_Tu}=lEc>|)J(%IPvL912{o1P zS#6a$F*{rvE|0J%MV;2$q$*a*Ee6lvU03|_TrCK+N#*rX3p$#rukFVV#~>_vf2~b1 z0p*p>(^k2Vl~+BK|Fr&fQd@9itZPvW#cU`Oe;Xb-fGYN zx4Knsy<)r5VM2~e18Z0t6|*Iu<_!bFKx4(>Y@E%}Cg?ffh!%ETk4m#NWSLbh2@kUe zW~LGY>K&GwzRhOQxwW;x7ZAlcP^j_wS!HV^B$c zzGHs||1QV7vxTe4-&O7G)*1pit2J_~_ZF%^>xSp2-I$BbG=pYf2Ty544t6`GX%BZ3 zT*h`y0qcBuY{dAWMDhwKul);`9pto;jbp4py)-u?t7ExC5T>b*^1GJ;bAF_GACJ~9 z|N66!{eFTus*+IC7p~b(o$FA0TmPwhgEar_OSY8|A1C-!b56t9XWKbF$;-_(Nz^lT z6?7nuwss6PT+;vg>XHhL*RStpq>PV$9e^kHN;9TH?6R4hfX8xhJ|%)4(-Rzgt1H3p z98o!U_6N^j-7Q@O^`gp*GX_>~m#=8NbJ%Eg6VdZJoavft9&_c~8tPKz_UwcS*w3j* z4wn9QiC)-G=U@8cTG&1m$FmZqP@db%u03e=3$v8%Ucs4zn}yFObk*HzHI^&Z>lc;l@5 z5t3D8nR9S=he}|j)Wqb;*b_o98x+%T{U;CxI{MBwb~cAj`N8IhL&sqWaFtDBlBD)J z>mWqDK#m9bM;*Jru2%0`zZ-0S1sQ&-1Lj+*yn831R$*LGkwH_aX#)(KSO3#~x#FSf zwQc0H_tnp4yj2Rx>Yw0xjou6!OHJk zlU6comr&`zR}w8YEYF*kCSZzuOoHy%bO+rxY;l@?$P)~q;PN*G+mK<=nC^~0q=X2;r&`S$<842mC)M-Z>J^L0BW}0mvCiSO9{IFR zWmCCU7|I|m3LCTj^;WAmy#yGrAh2;h^X$S$wc{HLMGo>2TIUx0Z%HD^Rt=v+{;@U> z+_P=6vDm!Ve9R=tv#+AV-4ja-ikY9;nNE`K}-Db zh;zMx?@w!0ZF6lya4y?zW@hGT!{vRi*67ZLN0o!{R4aBqva{BV1iU)!#pY_v6Rr@7NBeu&NBujv#hFxet&cNN*-U{?R+ zYzZk$;2KEByqEPp_&qWO0|&$T5`O2ToEH16j2o1P@}I3uzQ|I~%dOV0^SwFSS!7N~!J%E~INxQT zVA7FTWw$=v1e#mfyGp;Cy^9deYe#Ah5iV|mR%W{`U_UD2;3FX@d!g(|N%1e)LMxx9 z`lV@Hr=7b?(0CXsBS69P3?b6ey+Zdrala!C@na^`djFSmhGUvnf1yD3<)l2?R{mW= z##*lN%E!8h^BJ$5N1giDb9E~EmO$+4H|jE86?5KyD0Y1LdfJ=w#pFkzx0eT4jG|f` zg_bt$JrEp{M}&vB0u4KwXRJ3K2h}U6VhcK?QA>Uh6h`}O0^SwKQU0+4T78dlG5;Ys zwr)qZP0isp8Z_}$xfc%LWC0WN2H-sSNM)Ka9O(Sv4CoX3TRy%0ykjnY-kn-a@2*Eo zIW<6DwN`-otN{(S%bO;{WHPX^O4kj!hwF==B~WB;CQ46Nz-2|7^IMm<2~$KCeD~;$ zRmoslA~anE^euL9J}J`~%A2=h0aW9wA?%mmtMCOyGfvrR=oq&VWd_x2qz8$%1JHms?z#~n#At03CWO!7W* zEk9pg&1KZ9DBHLoIF*@|o9PTx{)$QU*m$4d*8=S{yo3>tjkcufwFW9dM71bmGe!Y? zx(3UERz27rv!jg_y9aNkA$&a7DM>}cw5c8ereItCVy6I2N+*TTVAjJA{UyBlkxzD0 zQv4&f_ofm>+?}oavIl9~IvlZf9?Get%j*b(reeFi%Jt?-1+hcN228tR0RRis?-y`{ z`YI@CbQ|U#0MVnlYd?*On^dI5Kt6T2@k^ogaRh~8w!ypIx9xWr&}geqIM55*iHDtE zfh+tw*G*yvpVjp-qM_36&C{zP*C}`G{3+wz{v5P@^@C2>4r9y0;l~}>bN7assemm* z5!|JKRUJ((+E2|q-<1f_$cqB*beM$Nw=e!%$5bA|7KH3e0KGF*tdJzSh|#>;v&+7M zzg%-n)dRbRS-2c#AD2z|5>|Vo4|ldMeVM-9YRS$wgNg3XgY`w6(No;dYA594OGLIK zlbtVC4vrTOsPb8FCNl4j&sOj4$_~0PQL{{Vhm9jMb#11My_o8GU!RR)jUxh;@jyAA zGy~l@XCl_wI;ZW>#=AQ1)UV@DPC;Qo3{WzA4%-fq9Tfk#$Ho9~FrFE-yEI$`c>!~P zG+g!^=1>vDj*9op@Yp<(zD3an#-asXVYz<;C6iuUT7Cg^G@Ml2-z9{0%@9JS5aI*Alre7sI~0u3H-%*8&u}#j}%ECqPnD zQ?DQzAcwvw{5yA>uIgex2MjZJ&Q~K0y zJEJbZ@7|rM;ddJEF4$AiX~EeXYuId#H9zMi_6+EH9yV5*daw;)P-;;R2)CByY*AJL zoC3P3dvgjV@ysq4${D7%ty9S?$znM%vDqjO5-Th5O&48n_1qN)r_@HBii^wB>x-OW3?V|w2mA3{ zJMP!^zA3u?Sn*eRsW)X36t+;`+bx)Q9(iN!*KEGQmlP8P6HMbw>Q;Z93gN%`VUjBBL&4>At0kaQZ4L-Mkc2SUNBgZDyF_ysQ~_=wTt!R=xYQDBgXt zOv`d^b!VK8DKXbZeQfVOVg`m#?KP-dJr;B~Y`i#nS~Q4^BDQ~YPUX!mPX17ZU#_^J zsHRrp2;zViOnbIf{cEzq)_$qkaaB-*u2p5;%Mv5zeRO%Fa1uBUk=|hLgawmABHOGB zVUSV)=RlW_Y>nRhTWTk%-{mr{_{j2A?>CRpPHt+AGDZ@8G*;UqPf2z-y{_NfqUy@OYxgz>ROo_xvw)laTRobpF%HuK}rI=mUphFxmqRo4swUN7^QQv(*y` zkU)IQ_>M*iHKUNrUScMdSIEeWdSgcgS2$au*vE@!8|6ZULiIAj27{Y12ei_vL z{QjlJd#hJ{&U3Avf>4e#Zrt){SQJH>I~Xmc5U)R0U1A#CO~yL+z}|Atlun5L2%#{~R`=AZFjCjVFh z{g7IDFwdZ7XyjA>5bzh<3(_^f%&5=j*))qxL0fw=V-;wR*pLps4&}4x%eHA{2>2ow zP|c#9F~Eb3{|h&s7`g6rSKG6t*M6*i&bX>#eVquY)zZTmRA%urp=+v#=GqInqZh;j zGvd&4m|63BDrHt6_;)TKtcz38WPezymJ+`F_)%`vT%vmE@jZi`Xym)c&mR+7gd1bVOUG@Ak`zDO+j#qzY=MGI zGAbEi+srAHz zc$^$l-5)aj=p^IK??OEK!f3OOXV^5H!FGK}6H)A)eik;^WfWI3nrX%Ax|-!eIxe>V z)q#6r_Dr0YIsg$J-C(Koi9n@fsIlkVIj}>?AF@jQ7_D%jR%sE! zBcgOPx>~IBCx%>(U8ZT6nt5SE4zFxUchtwaN zH8^ZLYaJ1EQwWbT59Famb+HApvQo}ef_Sw<(}*MQTKf4 z*7BQzI4kwVC#V2O7GiJdxK*2g&Gx&`nX`hrE~8HGJ;Ze;%JbcE2spT0Rz|h}hm5O( zaSd1botFmi0+3f0{0Nb01Rtuh+7ZNaCGylLig==?f7OXpL=vPRhX%( zm=Qd)O+D14l~fb4t~frF954^O?H-;aZxhhY-(#$IEX<+e>A=6=#3)*tw8wbRCmgSR z#iwywsV=6+-kwWSNEF`P1t7VY70VOyT^Xe_IBaCLq9}qcju>H?x$%y^hISc_5HaS3 zel(-NR3%S|=JYg|#n)1X6lh>tTwmF0@ILIv7X|B*piJMiQ7xBH-O4SfuZ)6l-R3vz>QCZC_5 zcZtsAcw^_3L5YnEGVcW9Npc9M-Zu3|d9v*a?s=+JcHe5~yeCq`6vz~B?vN~^^}Pj- z!l#rY=Vz<=TUgOvyleZ@(j?vIksMXCZ{>!of8g;H5x!*i3Cam_cnQWh5y2 zgi#{Wd(e)jCTxU!LXd62@td#XnADbmm|qnpTOvd8UQeep#hS(7prV7YMNhFp9aIfFIl?_ z$x^Iqe&r&KGMY0NubBy#hdk-+jF7KFQmX_)EF2F1-UL>Yo;ei!8wUOTIy%BTA62vZ zbikCV(MC&k6ObK7`PCAO9prxpMw{BoO>&IatShB|BVX1T?n8RwN_Bq1xKi&cM`50> z86Kie(M1*|?qi{}6==I_!umhc4%0|~O@Bae;6i%?vuxZxUGtO?uz$q39?2x1%d z?CcnA0we($dD`9vpt{E-`1t#3QXu9VMT4m-LOg#hhalk+K~BGB&HeQqjAmM~u1+8T zqZe^VG5q{_2R%?heD~JtqL8VO!0GrK4?SaD9jyrZGBh)fg*I}e3d#JXtHOrqmnfc-Lif10q%NY&JId)x~0sM>fr zcD;^YWusk=e*!sbORR7MwN4WC;n;^W#a3le&2%h;b~!%J(C~Yw8oUP~VS)n#Yz;7$ z`pVv)OLvRoO6Vs_fRR1-0n`Xojl%t@nO#Ke({qHF@T7J`T=O;s@)ZjDZzkp0u=kWw zFw&M$*{8}D9=Ll%-l58p&PrULTM)~6dM;5wF_+TGNE5(6hqC2S?BN#_fDb~V_&oK+ zHBx0D9T*X*K40`vujZ%;peB=6FRDWw7=YLRhFZwE4akjK9hltHFa-kbcATfQu-!}9 z*}D!wjy8h~dc=b|=64q|JJ(y2JEnECv^|CRm+7e-_|pz=ytyhMDKEmh6?wkCji8W+ z0VL{|InM9l;y@)<=@$HdzqnQmn;O(6xeS4fytbNE{)!Hms~s)$Y>Jw{+3suz`<)ApjFIgq znd`WIcYN7G+v50Jo}VX$P1M3m8dM&mweA%jUx5&|2p%zwFsc>H;FgN>K^ZId3!Fr^ zg^VB&`cv}&Ah2h@Vgmg0M7RDQ_TDnCs%{G()h!?)f+EF%%# zFr=xl)4nbNW}kH_|^ml{#FFP+pde6J6)(qa@)?**BeSvq`A+qVWsn=Ye z;ggU{6l=oqh5h>-E}I$fA12A<`lmNHyOrSsn~q7-v$EzaV56 za>?Oy;AV?_#nZR_qhzx<7}qApTw-H6m`TU47X6pb?#kIo9>`7vn?pK?LwxWgFuLmE zC}%umFo?FSAwgpQ!4LP{=6-$#^mPF)oLLXn%3jA$jcc^JhibI6WSBYWmrobzWK?EF zVDL%GfTwC!?0NDtE_$6*smpsAMk{p7;}RC*Ol|XRqUDC88Z=|!MtglIe=YPeAVX={m5liCiPv7zfg%fSbeMg8ksP{Po2x zswb#g3=$5j5G=Q?SUInE`&Lj4oMqZHRH-aC9n!R~QC#MG^^K8_y{)oaWTCiLILEPb zx~%;oXRi{ST%2n18(J1wX4L}o@rDMdz3t0Ob^pHpsM&yg? zr3Hl>3m@NRHFIg9R(!i8kIzf-(j{Q+d%w|}C8?5cI}(yMQ(DV`^{)55 z*92w8_jjZd`nV1IMd@7E?HZt%bOafbai zk9xOkcd&ntbVXo{UCwQ-_KI2i5(g}9ow&#ws>{*>PLaZ(gN-dG39H{k>zr?cHDWqJzY5QL-O63sB) z{Ne$Uo~)paX#`^ms-d9m%%EC$56wI2)n1+_>s~c)JqzCI%aNb0>mkVLO4p#*c)Ebi zOt78q46N5GpSX54m+XlK@2{JD2I2b_o-hGr)`44vOjl3l?$N_7_|3>})0<~>%n8`q zI=f4)IDorbSsP&&XyKb+zAWPEER+oCC?KfA_HYjPR~v=v8Cay;+AZCfX7w*yjJw&) zz*^Tf=70I3ZBV*;!FXwgUKDnD*=e68utrWhwBKuVg&;?2bnG8%Q5CVrUN_{yifJU? z>`d5pn1@K6hbxg?b^ENwRr*-_U*>d%dwe6GuN^rU2e^4gcWPzyY7?XRhtBNXH^B?~ zZYq=aJN0~d`d!N}J#Db;8X_m>)^M9zTW)K=eO&3Oh*j97CqIq?@uV2GZoD~454<@9 z*`+>GPr*$84{9_|&@tDQ-nSd)yj=f%o?KEA2EKSzA-RsnwQtt&NGQuX{FbjB^Weq* zvGqxSRhnpy`4dsEc=mWJS6jIrNq0)@6d1*SbgPi0eRpEI+__;Q_~hZ0yqSTfC_2yfz^Cfz>ae6iv%YvcXw(YH~lRAyCu53>w+8qkr z7xj3>j|%ici&e*}Tli;~2Q|+vNO*J|zcmagy+D9#J=tU_>~H%rYh88)-TC$cJ`NyM zAx2*pcWlfrtb4G3K5p3=S}Slc%q)3A#=M7kFb`2U)9>$UgURUS1mhJFr96;4B>D|X zpSb~w!UA(`(rkM57MG}w^v*(;8oaNoy1iJ^pr^?4L4|dt+G3)Oah}0o+$s3j9>;Lr zzqIZ-boVkf%O7}?PziT18}u@ngbvpLjX42S9EB88mD2^}pu+hx07gK$Agmd!!s}`M zHe=VeZ=R2hxHNx)cT{(Ct1**<`TDsq`d+<>^cNqEotv9g8r?q~bGG%=wmL;%OfbXO+Rg3G2kWD z4?{Az7SL}!BUDi(!BDrg@opdM>ODFHsfqxuRi6Z_`L7ed+m7Q}0Fmr<0jK{`HM%NQ z%myW2Om|$rTGn1Jl(sYZJaFn|TouB73-TLu^cY@$Te^B7(+It&@2NT&G*(QxMEB$8 z$Wg1==-_!5Ux6Vx_YmXO6emh5rtQmlT7h4a`rg>g{QTh24h5g+hA|CbVEAX~0&G1} z#*a@XQ94d0xbzlUXbH{aCqUqHcNlnDLujS3&aRLNnK#gXobG$S-G~vDfdGxQ^T_6Z z@YU9&o5ls1yN5Uni%~e~_PBogBV#~<%lvIB`Q4kX4J1op@q&>2ZNFTvm#8)l1)C-_ z*oPvzj`M_vH&5;w`XG#0V5Z#iF zcivpCjVL@fxSHvEF>uoG{r{*%Hk-kQ+D>eosZ)*QKy(}B)=yCVx>gS$S1eH@)@ z!4>vNb=*A!9YI10p(A{%ZN$Je*rjjusOI?@#3+K{e}dz4ITxE3$lS zG+0&~?q0ohL%M~q(`|A1(=t$(Hq!VDQX9lN=xg^$eD$IDMw%)Uaf2!>ro3oTQ_KI{gFs*U33X<95JU6U!7M)H42>TMuo5)w#ZnK{&fz;48=DB2jMM`zT9Y z*9$({?aCm3aPMEX*hyZxfHE@FUPlu?w65{HNDs-{(Ek2@=e17mt8XCpR|gLCd)M8a zebLxv^i1XcZsraYGOa+rO9B1`HF05;Q0VvYv#r;zw%>1`jWLgH&3vY*HSH}LfSj6R zQW1J)n1&?iI*A`c0h=`8=Q4Xz{igk!4WVpdnUlY!zuOdkSRT*gQBbDs%*Vlebh3c5 zZ`W}@k7LJsFUV^O9uN5o;lkIw_AM2FI3zOByxCR&th%7Dilq;u_e{I~xaMVqkn$wu z7F|&Uhd~2awo$m{VL5t@4c%I9EG%kyi|>Wy2webKyrQpg8A_`NI9jSZLn9G{Z5f_z zgo+t!VZW6xggSQ|S0}GL_#hM5mMA?jz!)Xg_31bjGn8Hn);cTQ+~}MA=eYcZ*28yz zXnrVSV*Lb}j>~Cv{0naYj9uWm2t|@y+l|`$kc8@H=b10n>(k#f+`}lUTX-=X*Yr-s$!^OGy1M8L4uvtP zD0^wSMT9Uat;(c*dF7V&kanZrQ9i*cvvRJ4dLi_Z^X|mv{$2mUf%KK2{hjAx^rIS4 zDbX^_ZRObWm?LV140d3bC#*81VGzko0FvadG{jmJETQ(dG;_h@Kfc=6cA z_{J2cCz|PUI(4hwf8Wb4+y+B;J#ME36K=;LV{YdK{k{D`Os3Lx@+_e5IK|&+w6!_~ zNDxxsrT)BsH_Fd0QZH&5@FDdG$!(m=Iwo5Qaj$hp>jK+KpIVqGEB|`w7yuPg#*ZLH z+$FVZPI8f)l9~=<!(GcQGDKzMa}m&~~Z`0{#-S6gnF{A_aO z$WdUCw#hv?6>=N!CKSN%t@c-oEaEsCzvR-ca#sL^f;HKtOXOvY6dCvV#guH6Q`HA?~Ex$aC?`Z7@8);;pRtjxUGUu>-hw9H+v)Ax5KL>z1x8SB~# z!qIJ82sDEzbQ=J_eQM?(we(-e4h~uL70AZAZW)=3*vH@&m=Ewb{)~TiG6)i3{VAE0 z{He0XK6%!Hp{EDeRA-QCA4uT4m~$aZtzferF~886Bu{0sjsY)VStDbE>*t&PC)LDO zW1%W)Q7a^|l!j5&XZWd;i({QepkzU8?UXEl5`!_P4+VVLM(Jb$Q1l93}O#dOgg5(Tdq~7!9CSeIBpz?Op$i=$lb{;A0 zHxEb@^TfMxp{WeWww2j~dCkj~?p){irVG%Kw-$VWBq>OHSgc^<#v&xgKXn!1Hcua# z(Y#l}rU-?w^lz!d2?>69>?NCJ*?don*uWUwpz1vc6<#a}Fn(?R**f+iABU?bWJ__bO2pm()DzTTN;knKavKMDINQ71c46nhJW?<@Q z<5`Y@G)Y$!iLG7Ge0IKGBLx%`-%MH+_T(B&Xha!e04tKdZFJ6NG*BvX>l~MH2m%Dq z;chGBj(_S{M6O%@`x+{`=; zM?(SIbu1>wxY+1PGRJ3|kzI?5%{z8OE>CC5ZUrQ^T%b0#Puw6LN#I!dAQ!_3(akG< zRlhn{TrwjZobR;Lb4`p$<(w&u7@GCP@pCLBvvZ?D(+diYNYb|^M zhxh5m{nLrl%+$oUR#k|TGD@ZynVw4u%NvkSS&8K)l!yDx5h-cM98Ld=1-NTq9#P1x z_UVn6wj-o!&cC&YZWq0?kuG(~db}yyGaZ1rYPlgnL(Hm*(zcj5Jbr$r)MTf=PD>_W zmkN?j)APE`64i0u2;#O-z@@4+UL=!9K}(AZy7ayEQk~XZWS3UG9d;!J2x3RlD&=r6 zTTbQcaE%M4NyArc=pq13=`j5P(4>#mAYQ78Nc_+mafE3J!^du$NR59>7HuNdC9y(x zxBNn>q})e0P5O&m{k#Q9Dy4%tR+|lFAGECIXsyO^4RCyPWap`Jtmdev`gY>R{38^h ze}YUX3<)zRask6y7_<~mK@;2U_Z z5mpSlC1hjt7Eenb>&LatXlZ!&!zaIrQ};Il7^lwh+a#~y-6XE#yk?Dv&s31GhK`Z` z$$Ich(jp9KFW<(+l(b@7D%!M~WA7<|e!uy(43M*LcN4Z>HOGkzZEYh*^_hZ2O|dQg z*wV%ph2%h~Or?!o9Oi8)P_uUhBQec!8T&O?G@~Rf>C0E5U7l%d6lH9oOdZ_qGE6nk zBlYoaYZnt_7ns(EQu>iRNtC*N;=d!7#fREiJ?>>~%VUisTp~6b8$0HdL-FquUzE^e z@$Q$X1hWRV6L&r^yOh!RlQZBqAg$7@MSc$-GntB_tcS$x^5-u>;zAM|0g*fwvH8)= z+472(&E4PbR=bXSzbJ#5=TyO%~i$TH`dWaR-Ku=(nECTt?sDzR0f( zWmRIH{&wVA_Lm1jwlxo_>uxwn)&necaw&ME?DrSzx1T~=$~bl_^usjlHh=7F)d0|F zEr%cr%g+|WbV{`9BeH0%z29=mqpV?SN*8$31-h>c-_;&m(2}k;Zr0m~G`p;!wMgyB zta6UwZ#!2%Id5Z=r_mm`!Lvp_wi8+l{pni0iJ@n7X$d=;3v9)$h#%X;(G6$|Sqg%W z*d>z!bx@XtRz~HtYjhz2Q0P~85~n?maa7Qu2&ve;$MdwikkVMg#*LoXYWC6=g}}gF z&4omdsF-SIrCjc}-8|d0>%&$icd&z6#5VQsTS(itufBnaAayWa!+cjGN8`R&*Jyok zI%(5lTh*xN7p6Kq~dQbbwC`J*`r(EMbze@m$M%JA3jYsCmT5DbzS~-&c3MrXK z8ZKsrCo^!0h;Bb$9(b{Z2~3BwW%XHpi=Ov&lh8UE8|i3>Qws}l&C^LeI^lPH-em5lo`L)| z;)tf8d#QD!+`KT!)x8udSLp>a6{@%-1G9VJ!4~S@F=9`YK59Fz2tety{kR{@mC0yn z17^hI8H>97AmS>}Mg=FB3sQX%kE0?Jbv4pFk@b1jdo6`vdysu?hyEzBPU1&0)b&$! zFe$g9-OWXwF(i{Nw4wVY0n9cms9NSC(i^DhY)k6-=`a++kxmBlr92=0oiD{oFdA;v z!m}+fXZ-oxz_5!}SOf8`>vqqW)6+0}wEB{I6^U*D7tdWcbcDGkw|LNO_LsmR@4ARj zgomkbBx zE3g_D)PUoECPhxCdTSIdU45O{*yh!HFV}i+FI$aLW4uU_TTDO!c5#44nx|7MTjvG& zX$nD`g6^gWjzqXaU9k@xpF>tEPyE^UvzqolePK|6b|U5t|6tw(-^$anz;E-# zdx~LW!|($*o4ykdLz0pKCcQ{Y>p|azvgrGxLFT93po)EVvp_sOu4S)H&wlH}EsbAc zygr0Er#YH!o;jvFEhk(FV)f`K&kB0ROdP@~!*SG)i!6OO5FceblvQB1zlCAxR41}E zN@(eqTI;&$@?aF|JB&yykpHQS@)5j@lQoerCAJTN>6Y8o%^Jy+XT!Qv%#R8$ECBK@&2>XBZEXv_&N52f3s;vP`k3E zxkJ%>*1b_Bm(I9v6=}Y(@l1obtN!x~qA{5GlE40;jb=&trp5((C?2Rqy_rd9oVI!P z8ywHJt$lrlh-j$4Y);3iYKH$7A26ATwr-Iu^#e4oH?*7fm&o>@LMmLjDho1kfe|eU091?6u)iC+J5k}v@7CX zsut=Uq6@F+W9#~|G^;s?-F#DAaeC0xN6l+hJVOaZ`iZ7ks$PdOJ$!UW+NeEzdP*)kLcAZYKH$<$AF(eNAx| z^4ns=z}sAR7d<1^)hnAi7OX`lH85Q&Nsc|I-x2XJT1~riOE1T;v$ZY%zQ{+7?9pzb zq{rv0(Y``CZ4RQmf2OD`;P!^HsITugk#u#+++(lX?$et%37`Z31{adwBZeU$Fp$DR zN%dG9R6o=`q=ESh>3Pl3{)A#m``)7p|33$N8B&{_gzilWIb~kcr}>HPIJMjKY%cIm zDe%#Idtm@8L8_61<>gdtk5GFhJ~Y1ZL$*dmZA{4C(JGLhm>B zKOJ&O-^~QEi}{!+hS6!?2C%#O&)}B3?J({al9YMt^-`Bgj+S|8UXX%`A2pE-So~%i z5n`xczGii!rd9uv> zgcya-OIGI?kl4BBGG}qadWoK{$@otASQ=Gv$~lj*@lCa&Wr?Z&tU^w13nrUwU`|pG zS^g`J#2%63{inv8EykKI4tu?~)R>f$r9belmbtmX{O0P5OeHiP1;`acu6}cl=T!}z zq}&L&sa^v7WYPxgQ^KkjoN1{bTJ#~WDq8v=G@vET^SYbX0jh4PijFK8OMca%O|xEWP$;?&$kjmSv4sJxI(O^BBv zSmy}xRaz?W(u29+=Q51K6BDrJo^iaZt+pHv10>Qj@wvZjD5B@rl-L7&Zw4gJabFx^ zyCF?C5ygXF(2)Do&<;Qr2F!gvhG6bL)QA?<(fSCJ zQoa&Kc{)xqGY@oBz`U2P*&8PAk`>X-J~th-9JTSv2bS!A03bKfRWBON;bE{ zd_PF~Nv*xN3$4`TX$08A(U=<*tF2n2{%g|j%?8ZdveX{bU*BH=;KB0E&>Me|X`s;q?sIPs&rtW+~w++ruwTWxECP|;Now|G+Mc_f@wsU_&V7p=V{)a~tQB^C3 z=;3V9cu!`tnXP6M-Ig{sz0Iveg(PF~kC@`v%?Ik8tKmMss?k|4wcMCuyKHhDC8kQ> zyxzp%87Ccbo0kVR{?WM;9S50q^hs2=5H$59$_cG?%>4TH!!*wlDn@U$w)86jf?5Za z9Nls=goFrnK|g{vMJs~#Bfo~fP_F$g(#3mK$~Sepd{o$N&*^mulB{I0Zu)2%VV^he z?lNJrAAMA0vA|x4ls#sml){#1-4MPunD{=#*CJ*mRkpv@#iO~8yB?b!rQ6>15EE6Z zu*a#@+0Y#{p5H6=j8Rq)BXu``76&sg(hzdUY9(5N+K!Sf+eZ36B%+8`7Vu-uC8OgM z;%~kn%&{D~WAf}ba5TPhcpQ@-$afqRclc6~1AvEvo0`%dHw{_bfg(WJDSXp_P{Ry7 za^JmZw#rN`=8N2!p|a!q7TgKA$59uOLRgiQ8$wvw^rSN(+h48ouPQyg1zMirei)(iW32f$il`3}E)lDAQ$Gl3cv_jvDu7W== z{c8H$?O6|p38w;-ru^K~rLw#qhJL0DZ$w3ir#G*Rwqnx*<- zC18)@+&ZK&3q>I4Jx&AO+$XYdFbIM>i`8txD@`2qIGJ`G_#TV!<1Z>njKr{z56bC3 z+E01V^XXwvpvco)J~-Rc(YMokT~jtGAOE>TRa1kSz@yL4x%J0KKQDGfMpBcW1R%+i zRkHjOS)wPNr5)M8Q#yopqnA=MPEQM8Binon_N>7A<#ND>(x2j!zW4 zuyhqLEd`rkFtZqca6uL8IfB`HxO>~|7&P{iSIH$v9UjH42cjaSB1M1<5(h$tB)spY z)af_T{3ytKhK0wG%U!yz^M&ZU5jl$(dK_k*i2i8ob`pj!RE#J`UuOAI*flFQfRdQwN7}yTaE9eOO2kJ7w4j z6_gdEFEc^xBk7tbGCFaj*8eP-;E)wSSG8V2>r)ZJj~7cz`&AK<0VmzBtC*yog9TvD z8sC|uE_-JLi*e5__t(4 zH)>Nf%eTsp9X~nB^6u1(dSo!dZZf-ZlncKYxd;Ip=Q z7TW&a(n3!bTOH&GE7+xpwj+B~XWn-5t34BC$N2|3>b%?uzQUtV$=vy(XjN2+#8*JL zn))SacSiRfYD9O@ySOP&7Y28W>EoPpHXuo_%>CXhz8C+N6!gd8-q3FfHw)8roHBww zbO(L~zjxR1aqhZPl^yFiulOHcz9Sx-IF7Ln;}M0zFh)?Qy~yhUWCdtEwihb3AYA|5 z1tGTUONqjGRUA@wkVHW|$mnMrktS*k2tPe2L z1mVE%(_%_&rm)~`ud+X0@(pU4fR=`;;v++I2em;?tMl$~R00!`U$QQVj0dJ}xqs|Joik)$2Nn0y{J3koNl8<1lTHv}w z7h8g9`S&W1)00D$`^Cf;xdn%q&5XMf#Jxp3{Tk4uw-4M8o~L2Z)VY)az^DD7YXyPa zS4eMxgre68ZEvD~pqN=s*zMmFI9S>_qHtO-A%LM+M-k+gAcQrg{;cWI`q$p+DPjibtzL$2vOfnJMg!h!^Og6{u#7ldfNwaDKpIPqA54il zq!4N+O05NO-m)x0UfQ?ktjW9jOv=gb%%M=aFTH2xDBAgPJmD6Rr|rda|7U&pMbz3@ z{`?F8bkqTc*@j2tN%ze6civEQY6#6?!0{*|d`%^y=?WS_52jzvPJ-j-=a+hH?hZN% zQSKcR|3w*vZ_lia6<#yzNd8U>b4BOc3oVAE;Oc~EJRRs9DOC_eeLt-V0lDEzrEHzA zwgxe_T|f0Hxm%fI_YYOodbtKb8({Gisld6Qx3*fOeo^H)y%URFwiq z3v=k#G@ouuY&0{SZh7Y+x{kO8L&rP*fSe8vIgJ1`c}^=pa<=yY#;CCmseCL{Z+gGt zs0l#XQT?aC_PP;^0z}eBJ#iW>w21=tq z&@U95se3W5mgsx;wEGIoJe-%Rkk}hA4|x?$npVDV_@|S_LYY)zWk{&qMYBNEJUgj6 z@I2V@v+~$al;661=#M{9iSK~fuj`z-3jC2h>MYIO<{y4X3fRpiYrCeK?zz3$bfG1? z7s6G^lQgHhi-E)wGVk#b73W)MxNLg6HOwm^kBXQwlj$N5O!bsiWg~|opi1JbW zNZRLxu(MmZ*i-tY21O#M+hA#ybfH3D0s{7*BIb2+_S}M#*DdTzN3iPVMoJkEi^Cbe z%|3UJ=%GjG5c16W?65teyI080#OUkq+uq2HYcNb!%GH-0%EB^Zs9ObBUZ<%S${CQG z4*(a`ATKlkXK>g?&Z*ND0X&cwXql|O)ZG5BtAW;*h?k4ur;mfY91YTqt`C@|=t)8( z=GD=kV!6!wL|cIDT>3X`Bx$PbL`w`CO!jPkG%g!iiuk*coJPO4n1yp?l*e0QOw5M& zz_)wFYQf1S`1>O~$VTP7_=zWKEA35&Ad5I!4Ho--T3+q16Aj-XI*cR3ZaD~5?Tk{G zvpJ2IVWn7i)&ta+CWP?$_0JKakjN915g&77hzm7D6Nn$ znI$~xlHM&_YQm|kzpl&u-r)`3cp>7F}M;sowT8s z5I0pMnHcqmRNLs$ImcK0hXoN*b`JmwXxRZNm;X6I`zLBNiWJ`rIQuP({Ou!yw!7cL~5a~GSfrf5B+-#L(q^})9f-9u# z7Y&C@vFBo)pdhb8Bxd{ZEfI>tpd|zu%-Lj`I)nHZH-f3D02`yG&)vxib+*Bt=MUDJ zo6-sO@st^Md!dvfr$6#rWF25m7R@rLLpO>^p!)NlAN|Yp0;cI2f#MH8L^P}G2;03KISi_hBD*0)9D`2OnOWqlQ{$<3^<-&ID)|6AuZ(vm4Mf z*#PC+Vmr{o(iQF;IjoBajE5<+j2b5D#umFP^V7w&gHeYsDsYrgJjmD&*{th4}Rn4vzEzzFVE6+ zSyPG-c5$3XiZ(+Cc1#zQwou(iZTtqfcaWwXb^rTXhgR~y-TXy!uO3H@M%J1{U?wfs zZU$ns}mAUy!+>U zcUhjmOlzBu3W*I|S#!<~(LatN}14IF`F7y9k&jbN7I%xj(Ht)$9ld=A(36u`H&j9780tn7XJHCha z@~WU3b`8+|q0n^7Vm45+rmlbfrwd(Gq{mq}2vQuxUt2}l{ul4)5sy4vnX0xAFr`Tx zyn(JE95p}3jd_g}=oT{ytHz}+E-vnxWg^aERLR)Mjqq6Wx$g1}M^$75QQkO`5H{Z} z7t)C_*HLLx+xFcN8v_dKuh3~(>q4kMku zGY_8{(^G6cVX(D6w;fK9rOeA1;ESBFKd3`Nybty`)JRbNNiI?H@Z#LMtqn~?5=GTOV=^YsifAwhP9z zpIJCe81C7!X50W#ZYD|x$BkYZ%l!QbYj@tGHyC}s0p@JyTwp0#$TDT;)R4Ljc;Iue z5=wAQ@2bE-Yg}*nOUouo6z=!SBod=`9Ejwyy)iyN2Ag_~l>xA(M(5|}|Lrdppr6sg z6aAfy<6ZrhQYA1?jyO5VlbS1?VMgbV68?FbJ*uXV{bD?ovGG7J&O^UutMV<7)qL4_#d3(G z63G&jLSZ|@GzYyj6ENz`$r>V9{vNhJ-A0NKURT{)z$x8VNRx~ zqk&nv-waSV#LYLp5`r#QYKjZxNl<)~@KI+kBX3h+HzGA>lS`sH<<>X7jG+nuIdIYK z^geQ}{MwY2qNo307a%AfiMJ-`^3*gy9c_-!IOWPLs^?~QULsRYf62htj7aQ%MM_wT z5C3ISMC(zhO$?pV!hRfxK-+_V`L%T4NC+i|#Cyq>P*ylJVfV_AG!Fuu(H4j8#XFt! zFAh9-f)f6w!wH!s&>F;`H4Im2(*H+mJb^!dQM#Wj1TETth#e#2EEGfEgka|NJ%hG0 zf7B}eYf!k_C7wPYt_W1pvmYk9dT1lgO|Dk*-`Mjxq@I}icO~W^R;CAmi;FM*_nC+J zPHT#pGmAouQX zF8(q6yuSwnWI*3S2Q|5$>wEG)`*rXkq~@>{Pd2FX;U2`nhB|wW|95-|tGteQ8kC3< zfL&w0csCl-uA4ZjYIVo{NpBGwY*z&XE%;tIuqV_%K5`QLN7o&G?#o2jIOfurJ2d}i zXz=HAhLHx2?GH#+{1jl$e6*9=uKt&8B*l+`7fhDCCWIGo!=qh_{$V!5=-u)9O{4;D%JU3^~ zih_L_!f9EWlk2Zjf>o;UG5n(md>K+D{(;N+o4;aMTlf(o_CZuXpCXGD>UAOMQwA~R zmjeDN|Hg?^5-o}iJ=XS-KufwW)r`2iyP;n5?5Al|l8Iqw-UC2!6!8H!SwDEdqD5nm zP^~(gBVCktUmB0&dV14+iQ<<59N(81iOy4XL9krxa4`$WNC56p;Yov|G3qtS@JwGu z(oYPjcA4JMfpqoLy2Z4Te1A_=bQR}!JXN`H;_Xl5+GV(f7Ri`ZoO2de<{x;bkv^qV z`lIZQ;s{aPqj)+%3sK>QP4oG32s_%rTH74?+0( z6?XJ++>w!H5tI4nM@Cq_w8G~dtUZ7hwwd{pHIkx76QP{HwW9b7U#G(}1sd|zx#%kU zMpTJ<;ng}T=^(2=qI<++l3-Cx#L-2{*ORHrp2&RYUhX6T`{2#~Bsi#f!4z`KayMU6)2v$I&0?$H}Gg=yshq$8-@beRoM5{!O)>{% z$2JY-wtyS+A$_7K4mDd<+vmEc*NKr?#>?>Yt<`_!qnHelCXr)*)-H-g#2cp-!iOUs z%=oZTM3enD3gDR^c0$Chp0KqN1VY#??*h!YY7XD)aW|2I6g(%QQwa7wpWPRaxSmZp2+i@*h) z3&?F5g>#DY{wp8W^oTUM;sMVpj4BX@wzt2N3aIp3qusor(_U1^>sIC}n6h?Fcn+=!wYr%|>Jq2t( zM)c_n76)T4sc`b|k&+6*PDy0S8{K~bd;e+fqA`UKBKq5q%RW#hf9;$+D}? z0cYq(2ceVgtwr;1?JBAY`Z9OnUQl6Jn=_9A_cC1G%ay}dOcXXF5)Ol zFZNR)d+!MwmFwQZ@)$0&uUU8dA~S4%j{AHU3qekQ(0k}4`(v^kVW<2xkm(lMfm+$ARzKvC~0^hekzRAtgXA_S=g_IEa;|5BRMz<0?rP)wCeOtyLPC zX*r(yVe^7+#WY>aO5mnYZnk{@>0Quu*x0iMGy$(*sbu9xw4?*bhg%Yi@o^O4M#Wzi z&xgz6aRSGjy<1bKg7NG%NNOwLqSC%F#57shSg^owp#Sq7?wat9Z*LOR{%ItC)a~6= zTgyVXft^HFiBWjrDrknOC`(+F9Q=#i7|Qvwn6kTocgnwZQ(?lZ3$zOTW<3`hYvq0_ z2ZGMIU0`CQdIX+7IvXOBaw{P86=gRyDdL9K>CRus#sCipSs z*PjBXAZURfxu7YsPbTu={|2u3v12z15fzYg^JnCA)((p?a^& zr9eB>EkBcgPJ$s}$ZBR+Q;m;)WqZmjqEbd$G)9oWb+3|4_jWRe(SL~YJw;R8^7Xl~ zL`7#jM2{lA0?r>_VSN>Rh1g{UUR2;2@D;9>hTcII?ST5Y*9pZt!K?U1Y*qHwLm{V6 z-KB+h*X6=+@4z?j&*T4wD1D5EyYL`;arNG!R(gbAXyTjUf^1OQU*AR<>577!9i4h9 zlOlGwIR`#9{Ll9@*M1*F*uM&7EAO!;y@z60K|!jb>B-}vtL>B-t0Yo7NSZ; zH~EgJzYKy{W+tT@2;H!^(>|gmYEi1Ny-atPE50T?Q&sX64D4DSrMD5moP!?c}mrl$`J+nXW@W!x<_J ze6MaP)(iLE;rD7wAZa&5lz$;9oLbGKW;@(y=Wq0NIjM~f_@igR0k{PYz`;+9Y3)Dx zvpUjiFc!wQ_84Ft4qm==>&mbU99NTHD(MsIVcjxlcnuuCfb)w5yhQsAk#vbF4hTeA z4H5TQX1sg=`Ip%Da!_L^Lwda*E8;}I=J8BRUKO50yb1_Z z4ZyW|848k@RuWb7fnj*`nXs60WL@+d$X&xh;(`Nou!fx(s`=-lieNV0wgu-M^Kb{B z5;sjeSuP&uU|$Ip&niTaBEdliMv-e3C0BD1;=s}vyu%ChFP2}t1E=lc$!xLZsqhXi zw67~w`{!Z&gJpc068XcnE}8}bs?NYJX9dpTs_oNApU}beAFolu=RI}sMbUSyiBU@w zpexF@*&>-KQM!+mm5HRMK}heVd1do5C@2K1 zOOPI(GIEEK*g?m9{wCdVo$+vk6dmL25I zhbmTEzKKdPB=O!u+0|pR8gjdGErKgDsjNuC4xS`W>0Y1fON2JmWvE*FZ0$?Q=H_eQPeBVu0 zyFF_m*U1FzqDdXVuIReErqW|Xzab*GEH)HFvUQ&TzUwtly7~j&|WxM)zgO6K?8|i;eZ_( z@k_fwhl#X}$B;EdSb7g;`9SyJ{bb?CHLn8!5+S_MYD-rKRLLrnWzSCX-xc0j6z7{x zem4p@jrI2$I2a&6;5hgy`91LmG6{07S#P8QzS=)7d+`XiYI8ukpX5(?;&bLD^sgJL zE?c3Ab0E1Ix~2)Jh?OzGLua9Qcf}HiEujEV2XqMOiJm1pi*J2dgTB9$m=G|7jAWNp zJ*0$R;%NbNJC=Tq8VC+^$aF!;pLc!DQ*&`d_Bou<&c+u>2072ip3L-Z51)yaA*N?x zFKG?mo{lsTZ34Mj1z_Fj$$sq3Wg=<7rnh}}HaIc2di6Mx2_0858maM9sC0GdDKLu? z1hkhGR55&8hPPmR&x(>0mVSmi(yT1yEHjvEmsw(Je?$Y~H2! zjk`DM{?;dcfEXHgXPDt)kZy!a(8RPk3UHb82odZk*uDUOu-%1HPkO|?iD+xN!D8T~ z&eOM=5>_fXs{|kd!l{|p*N9_n7)I&>X?*8mMqO?V0w-V()R0$Npq{ufmQ{T+_a$)oqUk>}oK;5`4SZ9Nf}2S0>bR^AOGyr2N7Oh;zbf8-0~e zRZOQ8jV|W_h1_0IzvE2$axIk`0xt3)&H;i=ZCo?Q-PH4Uj4}*>xjUTb15u@6FNkT{GC< z+o&Ad-&@#E8`_v^mk~o z7rc2DEtmXts}ScUN>I7vk$;(z$^H_n^HE=SYVYxM{F@8Iw5>MnN6dgL3aY4; zqn(eS@3yPh@ytU;jKXE9FRPn4u32)s&$veXmd4kg=$2g{{Jl56o-Cyii#P%QYoXDE zVpu#r{!&SC>X@U6xv#xWoDb@rhTl~LP@6HX5Xfqp&BEZomVib|!XtmzDN05Dcar|u zk82x3RWunctOM9aI8IZCDg(4;BjCDpW_qM%gKYuXtIqNHbS;QF4N%vIl4lvkaLTlc zN*f4IF{gd`o*;fa)x3VL?Y!cJ0Hcd|#20j%Po%@xrmAqt9F|J0HV-oMubVE+eO8cwP8ejACdAwx_dDkRE)z z#?-<1hd6JUz$2MlX91K$D(cbJCp7~#Bi>-=$vuUV(VPCz#-(b3Cv#FAfbIx3KyTV0(Y z%Jeu^;tp{G7MPK7mk|&!aPCndtQu;)4ZSyQzYR+-yyN(hXXY59=&k70vt>_;R#n44#;o_E z|55?Q=6r6%*i=JnBWOwWUzN3B#CBrp>o<5*@bn8R60D(`Tjv8lkMfUV2GsYn`0dAc zgB-=hS=%X1?kLE~$w?-uyJbq_L~$BXZ+Xax(-B9l0$pgKGZ1ogoyY+E*ypWW=xrQ} zNtHeuyphc~uV25dJEb5va8Dx)cRHwS6r)yh6>+V*T$M_O6rotJ3k1AYP1kExk4E8X zzOpllPvl4-oG%Sct_4^P6Vh6e|J(`+MK5k%9cYvaJ3~iRPqk`G*Ncpp18&+tmh5*7*2@lS;%Q*?b2W0x^stkKImE6+iC68c4xtQ#Gg*71#LnbOH!l$ zGw`#$^sW|3Mdj|}>J6T!6$Jc<(>VX*!QZ${_ZsO3sx%-Bri6vO-URfXsgtT7y=V^t%UPFNla%VN6gmR3_SvwR$G4M zk^@XBSNO(49;B7Ov#F2qGA}7(vy2A#W0$b%EAE{ct^FMAnd$LqU`u>!8#yn6;D=^Z z2J$KZlJa8u?ax3jE5|nlO0Duc!)cGgTUOgJ+Vv8@7PxlW#^m`;WJm@~WW?!NXg0s0 zWfn4X@2*C^JsHFDA=hHiafW3I5IV1H?-F%-2ZNGj%TPNIfz89K!&6ZS=E&Q^)NSZ9 z64xwex)uFFxw|UgMoBM2)m41EGzB4qu0bmBz zN77aSp{kn6R{~ktm;uc}{$tWr`GToYRcK4#4|)?|Cb0tOoSHO_?H#30#3QzN;pY5t z_>HUSUcK1SC#*}yvMMl^P>N(oP2i|p?e`bns&n+!{>4xH!NyMI6OBVb2p;A_-hfpj z%QB#uFxyY2poKL-ETZLfJ4V;;&6l+FplR(SW|3CQWigRyF^ZA&NlN`PJ>eVYEk5!% zO>C?0P53ThV?b|)77Ng#NqLY=Kzgka8?;^CVgTazdJ-5t>g*)LFd|7GbG1a$34Da~ z!kx9r@72A>Qo%{V=CC@ni(ccaYZK`{e+&Gzto8U_;4Vnlb1Tv6g5etJ<>_sFr$Oe` zPv(~aP72dFpO-c&9!3sC%1tHyUPfC^1G791+>tSE*@G1>rCiUX+}Jy+WHxA}p5NV? zP_z3)v$Rq2d<>kz+`-jk>EpIN!*T+1SGU%04pcK)8iFj8#Dxv$VQD+BTz(+ZGIo5X zLqwHhaL)mC-eUgs9q@hp!1Z}eE0B2L*xTvyK|VMLf(Cm^;CvzTxQ2`2w@wECSfRia zV=@&@`6@z*zxr+%IUTp0d`*YD^hN{YIbB-p*=f_OF4n`qO`|sH=V>QEn=OyUyZKJS zzhKZOyv+1^@g-v4yo{3BMWmkrJ2$mfe{p~w%Q$VC15*tsQfh!c!QzDlsQ#SKbV5>u zfj*ADk(x=lb)qD8RId6XUQBydSSvWt{<@YWCvp zM^EPTXLt@J%+}D|(xr}G`=;l;MR%7t&t^*d6OR-fR?_&}2ahO9+80J|w1vv*4s=oV z|L7ti8m3d=7s{qZbUM2=?p%yRZ2W<&9wuhoi4dnyEo=%3gnV1u#L1QWAto*DXeaxa^HOS{x3hz`m|SLFSZQST7U{xd)x1&lVvaSUn~G_DSfuQ z@~_r4t{) zt;jK{wh0v>HeEwq;bqA5ag%c8AvIo|FZd>p9R_}DgxyX_?-F~)nx2WL4x>a1)|(xH zvLlU(do@~jBWQZ~GtHdwse@C7RM!a0k~8}tq5qj0^=Y*4f(4xHT2vH66QYT%O3z*+ z@t8!)d0Q?rM5lnJE5tRc>)Wjj<2Ljdz{7%pX=vbvY}ux~Tjut|DY~((nw!KfCq;Ii zn2cD*L?kwzv}G7NZx9{WeHOgl!iO+=r)$5PT~eGjNkS)%`TuI~OT(#rzkegjRLGQ& zZ7y>X$*fY5Od+!}RcMDy+e}D=GG!jhtjx2GC}hqUvdKJz%&`q;ZS?(p&;R^ioa;L0 z&AG1Ai!RfCp66cAeXrrOK5H$sfoNvWNOn4#sAwNasmPNb4|moe=12iXhR>ZIZI!M1 zt8wR36lv{_Me?(CbGVjMZUen4K8KIP=jN9kqYu~!@Tl}BTcV8T#o*JIv!u$E-|(Yv zKleQI^igTV_q)}J+PCbUKA!8b+g~o-f4f{PHg?~gvlmb~3w9SmNr?XC2=ZAeQkb)8 zR77q1=&zQGhVj#Mmz;Q=oJqn_00%<1zX{i>KR_V z36%Rcz{bAUd<5*!0}E}FWmUBocYl!g>I5zPgn3i$P!8^z7a?HCUxGnp6|Jgy^XtFN z*r)=QX!ePft_*KuyV5u6_48I)Sa>y9BRIGtO(2~Z z|204a+&|*B>ZOM+DvB!gUEM&PfvTDJTXFT@sh%#&Cct=Qt5M#1U6g^URLs((>!J{* zQuJgq(>{WdyUM?YOT9v3YZQV+AMroX;WVU~zv4DH2OKcuCR)#VzUWP8DAf{|=Ohis z0bqq{KQ+cDnR70o)p4cDfgXur@S8Zf4?*2SMWS$dPrx3nA;tW{L9n&?<~{_K9oSfQ zxgG$b(+*Nf8~XdW+cG+)lm$Ntm#m(y2swtJxDB5%S|=?c@jU>2K23{1rj!D-@y}M^ z89%K9v=Qm9Zu?L5On}U19JvDXNmzjGKy*T^Q=_K=4t?-rQ+UV$Z{KI%crxmHm#|~7V+lQWwxpVi1N#ZQ_@5fKy^YZV~+_s574wu%1J!} z3Kak|j230iBCImu;gpF3A?!J9f`yJP_U3V@&Y8ZFEy0|Xe_Zj z1S&vCz<2RYj!9?7|A}K_8YZY3?;QY+INwdZ5P()3p_|I`;F-R=D$G+5CBw;}aQ1=D z6L_$4qGzm1zi%I)j_u{%UTpk;IyxTjs+1w#_Z>WE<+u;B-ZC!m=0*oM3M)+x)}dfO zK}PTrmo3)&|6uX|5m;;@-l$V_zbsp?`nHHwYf+U5a*$e(B{P)ClS$2fYT3!~5>IdV z;|Yr`0~!eJycq%Ha&dnyXLGeahK}irsCXfJUzW+>_2*7VNnnhvh>0bawodC|cKub-uerzbM? zl5UJgR;+-GDiMHrRTA7u)x=}1ar^)(j-6gptGxo)ee$kJr_@0zL5_9AJfx`S`KcAn zme+(?ZejA5bZ(~D^q4vay>Lq$lSyRvY&EOLl%H^Kd}$Y8S`uybUi*UxhyI(TVL!3P z2bHEjcJlZkt!S`u5!1Oc`fD5iK29A=lmQU^pJK0EjuEYpBKxt&QRrvoclorgKGysw zz~EpR!6bfa?@@t<#vQquQ{O8&v~~pxHtmJlmHoCP-^WJvO!ZLqA_A5sd#EU@ZRn94 z8)9Q0OxW6TxhPh_yE4sT79+(Y*oExC=p)FBC=+ZG<+~Y{+%yx|y?q8q#WXy3MX^R! z>+kuf&j@(ZSkQeZ^SgD=u4&N)2*D)6t^|A zeA}7je5(eL!L`wA1iwhOtIU!tF@*-}kL*LrJ`V-=L%WUBNc)7-ZOzv_mewNy)ecR2 zVKGlaU{zU$W$OKiKLrzS-{NfGB<9@$5y8EhW`xE1oA#E$3GSO^%NoRa_}gUHk{(y@ zEO~dI(^b^zoB#FLww4+;rrtb?gh$?EiL=(x_{AR%iuWMj39TGkW`blkNH)b)z2_hz zP^ig5SoHW#^d8^QT-%6P)9pK+*?N}~g|FLEWiiYb&5xb854+2&+h26dmjgCOS?>h3 z$yLZEZiq4j6Q`B3A6!-~O%3>O`#BU7&Jm4}wTW8t_clJ!YUP0r$gt+@2|cDuo*Y?I ziv6)d`BNXUrM-o@1lob3edV!UmXNRgvun1Z`cs&Wzzdg%(wn~h@_^!T#IuF4k|OROjf!n&1 zlw1>rFAJfVi+Kug7-0+@5K<1)0>4=sx2)FM^B?S}WoV{U1@ryXBzm!3x~lj{J*}WS zr}HySkRa|6waTOA7$G0Rm}pImf0FQ7Ws%76uzl<32KY>VaSf(KpP{!dPko$n7v*oY zngE1iq?pDB6Zwa%af^a4>_rzrT*e5(8`Vq^UKyG zc?6ZjwQiJ17W~Qb@2TUKSM$@&#hn@(WPZxjZa9`ZJVdhx+iiha9z+?hR+Hqrh#ZnOPZ{ zT1SRyK&xEbNxO}izBdVCeg)&8Dx(AfZm3)~fI_hj+})4hS!2C-d#!ZG>zt5H3u_gP zpw}paJZgR*C!4H~GbU&%=F^y1U1F$mFB!KLvkX@6%tU5}dY|3D&m`_XF$ie=x%a8O zv~3Kln|O9!M$>mG0)tEOjvL=jp&-IlWF%JjHL*|_;FU;E&&AkDMN7*@_s*Xaw`bq& zqA_58%-@;%WU!DPcc+U#XG#L3BG_~iBJAS=${~40JJuTh-zB-1UpFu?Zii zm#Wc&w}(5?;SE7@2kxs3LLD#1E1T2qiiyspkK`UQpU5|d`QzeXTGk%(d$x~oCBB== zNh{@o7?@&3JhNjI_X_udx9Q0{!{O=;kX{UlhjG9A6vX{ejo@E#2)iyJ116xE?PT2A zLf|bmWFLw^n_faFu!QV4tK0c8C3fD62$&EneVQKGhD%ZYNpRKi@EC^uyQ&+Dgn#TcyXWNKr zez&Nr{BHA6VrX{!OG4V;mMCcc(FKeePz(E=eOGzwViwN zGyBL23|y2QZ2lVBcKe-*c=OSDNQISHeOv&2b@-&j5VMe0E zK0f1FQ$S}K?g#r450fXclC5C|Pt!fNj1)f){JNL=k-8@)y{ph}=Dsf^xhmSjb~=;o zILBHI+=Vc|#4cc7Iy|WCcMjT*r<{vp|MpQi;EFSLjBX|(bBrN)68LBW55JKf2j{l$ zU7)Bla7qZY*Av3J&G#|SCynZV8w{B6QUHP?erZ5`_2?NtNzXX@fm>z*o2eaN?IR2b z?u*^G`J%1!FsFS;0>HF*c$OjN>@ix#ixBGFD3s!pd`@jnWl~mqlLdMZc;UozZC^a; zgxZ!KbK4g-mjO|Lvh_-Qpuf+voJwQPrbh;rAn9Cr`?o(u<2G_&S2zIEKL#Ncb_TDI z28uu+FWnw5&PMSFhhM34DTS{nqY<4u6Lw?e^MgwD=sz#)7;xhjN86?y^><|00HCf#q4 zo5+ldLu8Hk6xnBRs8`32q9o^OPd_K$vfhLzG~3A7o|357T{45t!LE>kEkU@rvo$L0 ziH(ySYQNv$$TuJDrl8gCJe4=DlKbAaXtSS$(i_OA2%hIhYo@p(g?&#WkLqtaMJ*zw zq`%{#nE-@&;6DMA}gj8x{>730XOv*v!{f|;FD7Qc*o!Vtg@Xu zpSHgC#fP-%i(`WnLnfHTE?yHVFsr0lw_WzW^vT++{l4Buv&yoIVBSWB9z5j2UjQ+d zndAEKhp22B}=FKQ- zQMYv>Uuy+Ts-M8!sIWFg7gVa!MsHs{hPsRnK{RLRZS~R6finH&B!pRp(~{!$1ZNLP zYjHNqdCzK!_pe~UbTu0rMZQeHw(5y3c#L)NTlLmPfzuM#RrS#dZJ%#^V{}+$S%9Xn zzyW&Qej5)gDP;bVtykk%&~WxSD44ACu=}OWuy7dBXNhV-L+AHt{cGdn80;ft0q(!s zj!(o7F0|%SzrJNy$p-TSm9rSdJ!vlHce2vpVr(~vi4~P4h@|gP!SmKxqR=yY{h&Zw zyahpIopTr4pss^5f_C{pZ^5e{psK|8iML%Y6S7(s9xUO0a&;A?^)>#L`r?J2%ImiD zf`8b{W|~x;HW;3*XDrt%BAYU$HB+*$UWv)2GV#ytKEE~=J1+_Tm(|J@Kl+gC(rGU4 z1v~TUpQ+#oh`Qb+%2#0Ij)kC{5a=Y1?6`9S%-D)Lolf1?FUu7(Rc3s+3_}86m9C|F!(kvLrleLN^D^SJzIzkRFokl}&o=#Ow2mx4&v(zG#t$ z?lEk+U&qw7r4C?=`?YrOMY%`xj^}VH#-S>xb&Ri-BU-PxN(+Rp3go^?31nNC5f?VSPj7e$FM6Ps|Y|%(`_Pb|}6g5ZihC(-6 z!}Jz`Nmu>}o-k!`OK!(cCnAT=2}rFy(6K9~x|i4QjA|zbBt<|n2C`}x3D_s`%(z_? zSklcfb@AwOj=4XAXXnBf8lulYhv8XJJ~Dp;8d;ok@7Q>et&&$X*Bx=b0MkBg|7Biw zd4@beVS+pSCGRj8P~Dsgjd+(o8DVRRG&Phad4;*9e&r))wdqfI2=kj&?)ZEm@8ysd z294OC{{A9P-He+*M`XRHtou9t%R>NHwh*|vaAV=k(UandDlDXv0wIqU>GqD*g|=Vr zFaLNrm}#ZyEqXjg?qZc1-%;N4jpaM_7nc7V4W0`B&`h1?CV73`YLI8K9U9(M7S42} zsxZJLAg$mk?!|IdtygVWS8FWFl+$3lN7hl&$@-E>rD-n~x09|YQ;hL3eW;xh0uw-( zK;o6%QctaQSX2Vt$z3`ePDVZER38q)eqB^_7sxY0yA2l4FYnC{5r@!D#5F8QUnHwL zPg0kX$eBzz(K(G8y1*IpQ26DojBl6DR=m{F z`K)n)!tN6(^|4BUJCq?WY)I>VtxjXxuvJ}CUCItXQ*g*sqcM+nD^F-&*-oo)e^z}F z75r_Y_)cTDaPHSNyG+7n!w6qr-%Ph1uWXA{12*@niI177qf6~=P+sY1;*j&B;)s{Hv%jY$JhI1QUopLXN#8G8{G7J z-C>;Vyc^qkxSyGH_wKNko{Nvf({}HVESfVh;SHz;?~>-E8)wgOvy+}N-Z(pCy6e1h zm)O|qMhH!SKxciCEbW>+`nrw`|0@kCmjPr#?C@yY&9!yeYAlbgty`DH;;LEg<|~Ba zUS%g6elb*w^-I+SlEV<;(vH5P(&p=oVh){7udmqhuks+tX!#8h`V8gvvusVR-UZgK zeOsANTQ=i$8=p=ne+-*(kXFGMTE(O*o2<D?fxy z##~D;8L_M1ohe@u)uZio&!y<-&r zd2hIFuWEsf-odCLfo#J$nd9+WjR9*{*Er!E%XNv9)ml__PzfX9Gv)`C4DOPC^DJ3h z`ROIO7sFh{XZf7QI?tw;wDrRBKS+mjfm1sgRRCn)ecS)=RIXU#%d6vS<_mSJ^ddhp z$X-_2O^TY%UQ&0h$T?9Ebk@CadnsPJ5IN@38$tC_!|h?^%1)wj#;=uj^5OHxneCr$ zo+irwljY+^sy1bAqKfQs_vvV6OOLPHcI)i-MJT4Fx*LK@5T%+02NNA_u1xfuwI<7_ zxP^U8rUpv7csGbu%ku0EGGee>u^kun=&8k&mBjYSf34Y#QgZpPL{cak24Il1Qvwvd zZj#v50-N=jsjzBE=TE%!mmL$$tB1wsvCG?glEb?zP3_-pc4KgJR>lzlDAJuQ&OBev zANQ~)f>z)O+K?EUS}JY-oyYHu2`JKmW%)o1ShuYBded`d9o0YA=O+G81qTb?aLZZi zvA^5D{yL30!gYPH!Pan${EC5Qj?U2TN`1Y&)1yZXhFsZ0jZPe$P8wOOG?TX|a{3sR z9Ce;E%-vKJ)EwhYph&`aB@_odIlgk7p^q$Z>6LLY{)NF&xrYY+ufGr%TLe4644h&TvZw!(UX~RDBQI!@fqlA*wkw zpCP)Fdf%D7k%Wh)T3<8awTAO}x9)b?-T65&tk%IF_6?1 z_YvWra^Q0PvYdHz13hDXcg6>$uBtxhi^!^b<(~f2pSOtn0S}NKxo2kFj(sh5!2kkswe?`*jNp@FtUMeQ}1F{YIt+KXoa+_BEe3S@h?64y%PgGfo z7J(%#qA}meo}K}pp5(hr60_f>CLWI|eaDZt%RcXD$W;z^f#`H%p@d=@(-!6aj z9=NfivT?3~$^7Kq;!8i}h8IgSL>-uBUrB!)$tE+HSlgX{-D7TI?WQh9#ZmBzcnbKb zCX*2cf2)g=kBIgzA+r@F2?*tjoL`U|N%imo)iN=p1U~>rDP$-?DeUCB zG&jH3kG|m$wTF>t-_g4=q2~0b>c*8FmlC&%TZ{Ur=4cHI6L5;-+^UQ=n>#x?M*5Ma z_lg{QSb>EyCKQec(2Cn8NnfvzMo}eU!rUi{kvBtVq(FS>C5}C*ngCo{_w=9j90hNi z2H8e`0Y8K2JEKX;=JSw2LsgOicg5yA^yr4^WeMk^1k|^?*7z5Dav~$zBH^UJ@DY2<)Z&u$vp2SnP555V`kfzI{BO*>ItZ zVJageukK{#)}}2|ipzB%{zv9^rppQI$f}CQ0%XgFEvL7L=I1dpP2Q9BSANX~4D-)b z>)b?tNvJE8R6$x3)*+SAu4Z2j5Bg8KAZ5jUm0Q@+MSQ4oKhuE9-TJbo^<`pds0vz! z+0=gJx?5B>`zC7~BVH-}0-c{ng|CEd6)ww|cTdS_GMbilUuP&An-p)7&dSLNoeXbH zs^v^mov%(7XC7a0Nvv4G^|{E*ZRQJMZ5;LIt;SDB!{Lo@)n_pPW^t$GnoYTjjDOPHCsroM=RYHaaWnIa+|2=lkeyJ=A9i! z78HM~ni>f^ZhTtqi;NF5)ZeryTM2g1T%+W$v=OnivI>8FRbFFFb&D)vAcRL@Sao;c z0m5tY7#KdfwR)YEyC{3D3eAj?!g29MKa^s-!IU5RhS=j-zL%Q$R!Zr(Q#|z+Lz$}z zkx;$1>20FZP+*c`ermMxfv9c$?Xus$0-BE73^?2gopDeiC(DaHCY$s|_wB^C-RDlf z7ID{Zw?^kj4fo?4GkZw4I#7}W81&nxNy;JNtGo@)YF?-=QR(g&spiD^e3JawvC-}I z{$Eess8||FGVi+mtxL@SKX}HtTI5;gkt4L`x30nUl0 zg6rPzVg8U6cbh66FR-+eZ`b@C7W^6J*zK~gZOipn7FsW@rJBd7AW@8Ut8!>Qd(+Wv zTC=s%5>==0NQB~04Bw%yl6mLRDytc5be?R<%Zb!<`Fv@*P^Q%}ADie*Mr2L&Q#neV8y7;W|Q|4@5*_Tn@ij~A0 zZGwVObB^3TC{lx**kP|%+dAO#>CSHfOrxc4`AUWol4LJ0&jr$ZQiDqs9K78fs@zOo zwc_2#aYk)IuU#4M|9qK{p%m?)0fy<4Q;5qdABoiCu`vT}ke@qZl5@W#ysgKtSY5Q1KJy3 ze&+7h70%(+QAhslQ7X;jY|M1iIoHO_NG!dwrK5)Y$+v&Ney?0tgc_vVP{a@pQJ+82 zQ{y-y3)Z+NKXO%rm~3MKed5U7b5q(GkNTo~yx7)4(Eht@K2E1FQ<^CC+U(pxpXd8} zTO=9d)xUq2CWjD>Xz5u#=C#y`Yo{gd>Kp<|NAP#?ub1VSF%gg!78f@)vb@t{z)j-o z;SbD~`X%YeG4+mct=B2uv8cm&UJbc48;$Bx2Me*(%cA=?K;i_SP$@)g(G22%d1YrO zSKM?wN|PBLo7q+R{JA=gMkS0wNrctPVWs*#ba;&^uAHhQGmr zMuTlvoM;o_@Xq~BXB@^ypp8<9K&{xb14q77-INIp$8`>(fDMuqX@XxsHOY4mqG!R6 zBKl5kGGyPCFg}oAsB6!7;qkTP)Gl+3;lf^;^j1dY+SZ`|f`ETpWM#qoF7qeo%3mBa zy&qzpZ)W*~*XQt|KDfM~Q%2vba{sjP=CaAK)MiWkYGX#pbZ>V|HN9*^2{~A;sHbjh zjW4TC_T!2xafa&@Meg~=$Nk?}-}wdIlKF~mZb}Q|j9ayO*Pwb(ORt}AX}T)9A39csGBNBYC??J)x^>B>%?Ur@NG_c8f14W?R#+)77du2{iD z!c5^g%sbVO_26_q zxW4byiyYHdO{}bqZj!1k^v`ckEWVG=o}2%U+TQmE(wSwa_}+b7Xi>dJ<4 zSk>*z)sSe8x-f7+9PyBO0I`H^(qd6*GG>IU_$J#k%UT~P(OwMxeYs3QD^_0rnpD+3 zFFev_P^aS6^BC0O&7JB(@OL0L~oVtez#y$JzAN* z!#<-vC<+x`0~jnxCjIP(0)22wZFr~*oN=DrWqi@#3AevNq~qoID00ild#4+t_x!bhl$4u`+1>lSzb!4%BIGl~vLm>#>Hf%nM~= z-M?a`em-+bDI3%Htio_!*k;#j#JGJYXLY*2&TbAFclj1Ax-xycz}NSf*vj_BiCRVX zUlk>aX?mRG!y8Z2MDFvi8aiT~W_l{Lu*IrJFA(BSDBb=;XHo5)hJmxBwD{`M+T@OS z)JhD|T51W#kyUYX`T2>zhCEG1^+~o}G)+p*KuzYet0C)tM{g`~Hoi>0t{X~_E*kG4 z{77=Zjkh~S2MswB1NuO6yd8Ka>BC`Xp?Q%KC}h(SE)}Od=?C3JQBo^zP;K(Wi(NK? zxS-}jI*AH0p$Rt{0!V4SHq|YgN_HHy#t_n@(2vfyJm3|OhgLBO09@oz0}F~0kAIcw zeo<-{-Bi^%bGNbVWyeW@C;>wZws~StlIi_c<%P zMvDf7S583S%-?_3U_w@vKsF;k{wvHM7YqpV;AFyppqgE;wOn9}Oh&WG8ei61=6qC7)P>f9uCR z>6JDz`hmjgXBk-nx!L+x4VT+aiEXB-sAN{K>+S}+OT zDm7PdE5*57oR8tSOn(Lc7746RtO47t8{8O@`m^(cMKC*gLKWpX((K_v&E6;XSQ_SS zGoO8!%e>A>4hUMRY32-j@{u(_tw!G@Ig#P1<2i zs&=ikmzJD`-VgtN?-iscDS|SW!3T? zyDl?us^K+=Ogy8O^FWPl(>{pmxEAVhYfU~R<&oOVu|uOeO3Y4g;`;lmkoLChpt{ge z@-W}6ddK5Uy+n>gmb=fqEAt*#1=8Dxx#TUn>BKEl1BRuO7T@MIYxPhUJ+FYac6)dC z4fVN%VKSUvWjBwhqRdqXw|YKSOWsFj693jQ?Y0q&{)WOJkFTOhJD$sL=gCm3pLm6Z z#f>qG?V*y`c^~{{avDH@|ap1Yz zxADA_L)d!0_vSVC$Z`4Ym5o{TB%9kyDds9zMX!g8ojo3sQPZm|`t^-n&exAa*k+ci z4Mnas&fe=|Uv)#SqaPXKKl@X5WNosF`f{}@>n)0IlkCLuXJ;BUR3~Hhc~0FF!?W zQhZ;X;mSwtsP&f{7my*(aMWR6b?jcbpK{H>n-|>4x&GC6?=iLQ6jLR?H4C57HC~15 z=b?Z&k_kIlp5`4i^Ll>j=EX*vlj%i88kj#%lPa-4vpT;b&G4v9a$=3@DA4S2#MZ>r8Y-Sas&x-f(X0&*_t)|6%@?Y^H0S-LP-`X!aXT z3~s50b5dFFQZTDcfJDzd(nUIbgv{WeCfnEnAc#oA*Cy01(i7KpXo_HpF;N^m?zrM3 zGdSkd`JMKj-Qs}yY(+QM)>dLdF&_@+R@ww(;m=RehV);a%voXV1HHT(!EhsRpY`Ao zj-N`PtvM=`{)6=U8IuXO2dCjj-7(=^O^IYQDzLA4uX2Uf$<-L)8Sn`^+FTYUSmuTF z*Hr)5_fT4uvW^XL&RM=0`geO0@b;bJC4(f9_}p;L!b`L#9Vr`ORyGe-44A#SY90(0 zJZa&wk50j-xEed3D~tM2C1OZKC&(^e#R~_8+w`>d`(O0=8HC+=XDEK-ZovhP8%rXm z3_7DMj|QK6|MDrUts;ly*F?x7DT<$G(qT`uf#BZs8@|!G3|Mo9O+kSdX*v#+`$Tu5 z?uLvk98H>z?=T$A3*t|&R}WS7@WOGR-snd<`LoI%sDP=4UZQK1xK(GLt1Se+!woRQ zcKdM=g$YNkLRth-x*jwP&=%SxW3$7X<+HP~zh|`n@Z7(>?oS^)(OcY<{1Eb|m9`n>A0yu_0)!+LX!X1>sX3WK)%aR8xSe^8f?B|d{_g_i~thd~M zAR*^FVpj-mLxVLTYt74?1iXN1=Np}(mwSNEO$W8-{z}K0boph_+ESm$+Rq()4W3?l1C;)<42kK_@{%v{`9d>rbfvFWvq_~iQivAim3Fjzf;LwUCYuQxB5 zLIvJz_mA%+cr^PLR(oy6K=6_e!2K`r?()Il48#4ub>_%xg?gRb~S4@8-`@<8PP2 zc2C@oxGb+ra2IS^b40&6Bhh;Edd0xkj)XwR;ZK4X?kNq86MY_qa zVf6R}C+|~eU6M9eq8d`?&hy?|LWn~!x$)Xac0t=2kw=~HRil2z#<9Xk8yLG!|NY% z`tbPR)j;2oe>*<^`ZU4_X0}9#*?xNcPgwu2-~PcJT8Tq*y)WLs|JguC!tj7cqX5(~0>1;g25WNu`bF)j2Rm#6wRPF?pX=g| zC!!j6(&EH}pvC`xz4JL>-D@VL#(X-oo&Ub_CU`(ii$AQ#iA*?P-)Mg=_;>qXAk@Q{ zF>OM_BV^2Ygs=k%g+=~p?{;vT|JawmGW^f&)!-;A;6>^GYaRbA>ocw-26>a5h*}8; z@QI8C9?m2?e0)J1EkDG6M3AEKH9US0XTa41Uf18-Sc0JQABoVD{!en+zrIl(y!
HaMWqI+O4Oqa%mg#Q-_%GBT>ofxywy#Hn?wD3ZJ*-Z9N zGpGN&_*`g7)|}dJ!u#)6c_0FhFCbW2W&W#wvt0kW>Hix`8`;B?Hs!w6<95vW2>iQs M{m!+Iaz