From ac1040038719cc9e619d21b1d7c4e2293e56c52d Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 14:28:23 +0000 Subject: [PATCH 01/63] Merge demo --- clean_pufferl.py | 373 ++++++++++++++++++++++++++++++++++++++++++++++ demo.py | 381 ----------------------------------------------- 2 files changed, 373 insertions(+), 381 deletions(-) delete mode 100644 demo.py diff --git a/clean_pufferl.py b/clean_pufferl.py index bbc61bad74..9f54fa05ec 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -5,6 +5,13 @@ import random import psutil import time +import configparser +import argparse +import shutil +import glob +import uuid +import ast +import random from threading import Thread from collections import defaultdict, deque @@ -20,6 +27,17 @@ import pufferlib import pufferlib.utils import pufferlib.pytorch +import pufferlib.sweep +import pufferlib.vector + +from rich_argparse import RichHelpFormatter +from rich.console import Console +from rich.traceback import install +install(show_locals=False) # Rich tracebacks + +import signal # Aggressively exit on ctrl+c +signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) + def create(config, vecenv, policy, optimizer=None, wandb=None, neptune=None): random.seed(config.seed) @@ -1098,3 +1116,358 @@ def print_dashboard(data, clear=False, max_stats=[0]): console.print(dashboard) print('\033[0;0H' + capture.get()) + + +def init_wandb(args, name, id=None, resume=True, tag=None): + import wandb + wandb.init( + id=id or wandb.util.generate_id(), + project=args['wandb_project'], + group=args['wandb_group'], + allow_val_change=True, + save_code=False, + resume=resume, + config=args, + name=name, + tags=[tag] if tag is not None else [], + ) + return wandb + +def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): + import neptune + import neptune.exceptions + try: + workspace = args['workspace'] + run = neptune.init_run( + project=f"{workspace['name']}/{workspace['project']}", + capture_hardware_metrics=False, + capture_stdout=False, + capture_stderr=False, + capture_traceback=False, + tags=[tag] if tag is not None else [], + mode=mode, + ) + except neptune.exceptions.NeptuneConnectionLostException: + print("couldn't connect to neptune, logging in offline mode") + return init_neptune(args, name, id, resume, tag, mode="offline") + return run + +def make_policy(env, policy_cls, rnn_cls, args): + policy = policy_cls(env, **args['policy'], + #batch_size=args['train']['batch_size'], + use_p3o=args['train']['use_p3o'], + p3o_horizon=args['train']['p3o_horizon'], + use_diayn=args['train']['use_diayn'], + diayn_skills=args['train']['diayn_archive'], + ) + args['rnn']['input_size'] = policy.hidden_size + args['rnn']['hidden_size'] = policy.hidden_size + if rnn_cls is not None: + policy = rnn_cls(env, policy, **args['rnn']) + + return policy.to(args['train']['device']) + +def sweep(args, env_name, make_env, policy_cls, rnn_cls): + method = args['sweep']['method'] + if method == 'random': + sweep = pufferlib.sweep.Random(args['sweep']) + elif method == 'pareto_genetic': + sweep = pufferlib.sweep.ParetoGenetic(args['sweep']) + elif method == 'protein': + sweep = pufferlib.sweep.Protein( + args['sweep'], + resample_frequency=0, + num_random_samples=50, # Should be number of params + max_suggestion_cost=args['max_suggestion_cost'], + min_score = args['sweep']['metric']['min'], + max_score = args['sweep']['metric']['max'], + ) + elif method == 'carbs': + sweep = pufferlib.sweep.Carbs( + args['sweep'], + resample_frequency=5, + num_random_samples=10, # Should be number of params + max_suggestion_cost=args['max_suggestion_cost'], + ) + else: + raise ValueError(f'Invalid sweep method {method} (random/pareto_genetic/protein)') + + target_metric = args['sweep']['metric']['name'] + for i in range(args['max_runs']): + seed = time.time_ns() & 0xFFFFFFFF + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + + info = sweep.suggest(args) + if args['train']['minibatch_size'] >= args['train']['batch_size']: + sweep.observe(args, 0.0, 0.0) + continue + + scores, costs, timesteps, _, _ = train(args, make_env, policy_cls, rnn_cls, target_metric) + + # Hacky patch to prevent increasing total_timesteps when not swept + total_timesteps = args['train']['total_timesteps'] + for score, cost, timestep in zip(scores, costs, timesteps): + args['train']['total_timesteps'] = timestep + sweep.observe(args, score, cost) + + args['train']['total_timesteps'] = total_timesteps + + print('Score:', score, 'Cost:', cost, 'Timesteps:', timestep) + +def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_points=100, + elos={'model_random.pt': 1000}, vecenv=None, wandb=None, neptune=None): + if args['vec'] == 'serial': + vec = pufferlib.vector.Serial + elif args['vec'] == 'multiprocessing': + vec = pufferlib.vector.Multiprocessing + elif args['vec'] == 'ray': + vec = pufferlib.vector.Ray + elif args['vec'] == 'native': + vec = pufferlib.environment.PufferEnv + else: + raise ValueError(f'Invalid --vec (serial/multiprocessing/ray/native).') + + env_name = args['env_name'] + if vecenv is None: + vecenv = pufferlib.vector.make( + make_env, + env_kwargs=args['env'], + num_envs=args['train']['num_envs'], + num_workers=args['train']['num_workers'], + batch_size=args['train']['env_batch_size'], + zero_copy=args['train']['zero_copy'], + overwork=args['vec_overwork'], + seed=args['train']['seed'], + backend=vec, + ) + + policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) + + if args['ddp']: + from torch.nn.parallel import DistributedDataParallel as DDP + orig_policy = policy + policy = DDP(policy, device_ids=[args['rank']]) + # TODO: Test this? isinstance? + if hasattr(orig_policy, 'lstm'): + policy.lstm = orig_policy.lstm + + neptune = None + wandb = None + if args['neptune']: + neptune = init_neptune(args, env_name, id=args['exp_id'], tag=args['tag']) + for k, v in pufferlib.utils.unroll_nested_dict(args): + neptune[k].append(v) + elif args['wandb']: + wandb = init_wandb(args, env_name, id=args['exp_id'], tag=args['tag']) + + train_config = pufferlib.namespace(**args['train'], env=env_name, + exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) + data = create(train_config, vecenv, policy, wandb=wandb, neptune=neptune) + + timesteps = [] + scores = [] + costs = [] + target_key = f'environment/{target_metric}' + + vecenv.async_reset(train_config.seed) + while data.global_step < train_config.total_timesteps: + evaluate(data) + logs = train(data) + if logs is not None and target_key in logs: + timesteps.append(logs['agent_steps']) + scores.append(logs[target_key]) + #costs.append(data.profile.uptime) + + steps_evaluated = 0 + cost = time.time() - data.start_time + batch_size = args['train']['batch_size'] + while len(data.stats[target_metric]) < min_eval_points: + stats, _ = evaluate(data) + steps_evaluated += batch_size + + mean_and_log(data) + score = stats[target_metric] + print(f'Evaluated {steps_evaluated} steps. Score: {score}') + + scores.append(score) + costs.append(cost) + timesteps.append(data.global_step) + + def downsample_linear(arr, m): + n = len(arr) + x_old = np.linspace(0, 1, n) # Original indices normalized + x_new = np.linspace(0, 1, m) # New indices normalized + return np.interp(x_new, x_old, arr) + + scores = downsample_linear(scores, 10) + costs = downsample_linear(costs, 10) + timesteps = downsample_linear(timesteps, 10) + + if args['neptune']: + neptune['score'].append(score) + neptune['cost'].append(cost) + elif args['wandb']: + wandb.log({'score': score, 'cost': cost}) + + close(data) + return scores, costs, timesteps, elos, vecenv + +def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metric): + import torch.distributed as dist + args['rank'] = rank + args['train']['device'] = f'cuda:{rank}' + dist.init_process_group(backend='nccl', rank=rank, world_size=world_size) + train(args, make_env, policy_cls, rnn_cls, target_metric) + dist.destroy_process_group() + +if __name__ == '__main__': + parser = argparse.ArgumentParser( + description=f':blowfish: PufferLib [bright_cyan]{pufferlib.__version__}[/]' + ' demo options. Shows valid args for your env and policy', + formatter_class=RichHelpFormatter, add_help=False) + parser.add_argument('--env', '--environment', type=str, + default='puffer_squared', help='Name of specific environment to run') + parser.add_argument('--mode', type=str, default='train', + choices='train eval evaluate sweep autotune profile'.split()) + parser.add_argument('--vec-overwork', action='store_true', + help='Allow vectorization to use >1 worker/core. Not recommended.') + parser.add_argument('--eval-model-path', type=str, default=None, + help='Path to a pretrained checkpoint') + parser.add_argument('--baseline', action='store_true', + help='Load pretrained model from WandB if available') + parser.add_argument('--ddp', action='store_true', help='Distributed data parallel') + parser.add_argument('--render-mode', type=str, default='auto', + choices=['auto', 'human', 'ansi', 'rgb_array', 'raylib', 'None']) + parser.add_argument('--exp-id', '--exp-name', type=str, + default=None, help='Resume from experiment') + parser.add_argument('--data-path', type=str, default=None, + help='Used for testing hparam algorithms') + parser.add_argument('--track', action='store_true', help='Track on WandB') + parser.add_argument('--max-runs', type=int, default=200, help='Max number of sweep runs') + parser.add_argument('--wandb-project', type=str, default='pufferlib') + parser.add_argument('--wandb-group', type=str, default='debug') + parser.add_argument('--tag', type=str, default=None, help='Tag for experiment') + parser.add_argument('--wandb', action='store_true', help='Track on WandB') + parser.add_argument('--neptune', action='store_true', help='Track on Neptune') + #parser.add_argument('--wandb-project', type=str, default='pufferlib') + #parser.add_argument('--wandb-group', type=str, default='debug') + args = parser.parse_known_args()[0] + + file_paths = glob.glob('config/**/*.ini', recursive=True) + for path in file_paths: + p = configparser.ConfigParser() + p.read('config/default.ini') + + subconfig = os.path.join(*path.split('/')[:-1] + ['default.ini']) + if subconfig in file_paths: + p.read(subconfig) + + p.read(path) + if args.env in p['base']['env_name'].split(): + break + else: + raise Exception('No config for env_name {}'.format(args.env)) + + for section in p.sections(): + for key in p[section]: + if section == 'base': + argparse_key = f'--{key}'.replace('_', '-') + else: + argparse_key = f'--{section}.{key}'.replace('_', '-') + parser.add_argument(argparse_key, default=p[section][key]) + + # Late add help so you get a dynamic menu based on the env + parser.add_argument('-h', '--help', default=argparse.SUPPRESS, + action='help', help='Show this help message and exit') + + parsed = parser.parse_args().__dict__ + args = {'env': {}, 'policy': {}, 'rnn': {}} + env_name = parsed.pop('env') + for key, value in parsed.items(): + next = args + for subkey in key.split('.'): + if subkey not in next: + next[subkey] = {} + prev = next + next = next[subkey] + try: + prev[subkey] = ast.literal_eval(value) + except: + prev[subkey] = value + + package = args['package'] + module_name = f'pufferlib.environments.{package}' + if package == 'ocean': + module_name = 'pufferlib.ocean' + + import importlib + env_module = importlib.import_module(module_name) + + make_env = env_module.env_creator(env_name) + policy_cls = getattr(env_module.torch, args['policy_name']) + + rnn_name = args['rnn_name'] + rnn_cls = None + if rnn_name is not None: + rnn_cls = getattr(env_module.torch, args['rnn_name']) + + if args['baseline']: + assert args['mode'] in ('train', 'eval', 'evaluate') + args['track'] = True + version = '.'.join(pufferlib.__version__.split('.')[:2]) + args['exp_id'] = f'puf-{version}-{env_name}' + args['wandb_group'] = f'puf-{version}-baseline' + shutil.rmtree(f'experiments/{args["exp_id"]}', ignore_errors=True) + run = init_wandb(args, args['exp_id'], resume=False) + if args['mode'] in ('eval', 'evaluate'): + model_name = f'puf-{version}-{env_name}_model:latest' + artifact = run.use_artifact(model_name) + data_dir = artifact.download() + model_file = max(os.listdir(data_dir)) + args['eval_model_path'] = os.path.join(data_dir, model_file) + if args['mode'] == 'train' and args['ddp']: + import torch.multiprocessing as mp + world_size = 1 + os.environ["MASTER_ADDR"] = "localhost" + os.environ["MASTER_PORT"] = "29500" + target_metric = args['sweep']['metric']['name'] + mp.spawn(train_ddp, + args=(world_size, args, make_env, policy_cls, rnn_cls, target_metric), + nprocs=world_size, + join=True, + ) + elif args['mode'] == 'train': + target_metric = args['sweep']['metric']['name'] + train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) + elif args['mode'] in ('eval', 'evaluate'): + vec = pufferlib.vector.Serial + if args['vec'] == 'native': vec = pufferlib.environment.PufferEnv + rollout( + make_env, + args['env'], + policy_cls=policy_cls, + rnn_cls=rnn_cls, + agent_creator=make_policy, + agent_kwargs=args, + backend=vec, + model_path=args['eval_model_path'], + render_mode=args['render_mode'], + device=args['train']['device'], + ) + elif args['mode'] == 'sweep': + assert args['wandb'] or args['neptune'], 'Sweeps require either wandb or neptune' + sweep(args, env_name, make_env, policy_cls, rnn_cls) + elif args['mode'] == 'autotune': + pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) + elif args['mode'] == 'profile': + import cProfile + target_metric = args['sweep']['metric']['name'] + cProfile.run('train(args, make_env, policy_cls, rnn_cls, target_metric)', 'stats.profile') + import pstats + from pstats import SortKey + p = pstats.Stats('stats.profile') + p.sort_stats(SortKey.TIME).print_stats(10) + breakpoint() + pass diff --git a/demo.py b/demo.py deleted file mode 100644 index 8275d5e9bc..0000000000 --- a/demo.py +++ /dev/null @@ -1,381 +0,0 @@ -import configparser -import argparse -import shutil -import glob -import uuid -import ast -import os -import random -import time - -import numpy as np -import torch - -import pufferlib -import pufferlib.sweep -import pufferlib.utils -import pufferlib.vector - -from rich_argparse import RichHelpFormatter -from rich.console import Console -from rich.traceback import install -install(show_locals=False) # Rich tracebacks - -import signal # Aggressively exit on ctrl+c -signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) - -import clean_pufferl - -def init_wandb(args, name, id=None, resume=True, tag=None): - import wandb - wandb.init( - id=id or wandb.util.generate_id(), - project=args['wandb_project'], - group=args['wandb_group'], - allow_val_change=True, - save_code=False, - resume=resume, - config=args, - name=name, - tags=[tag] if tag is not None else [], - ) - return wandb - -def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): - import neptune - import neptune.exceptions - try: - workspace = args['workspace'] - run = neptune.init_run( - project=f"{workspace['name']}/{workspace['project']}", - capture_hardware_metrics=False, - capture_stdout=False, - capture_stderr=False, - capture_traceback=False, - tags=[tag] if tag is not None else [], - mode=mode, - ) - except neptune.exceptions.NeptuneConnectionLostException: - print("couldn't connect to neptune, logging in offline mode") - return init_neptune(args, name, id, resume, tag, mode="offline") - return run - -def make_policy(env, policy_cls, rnn_cls, args): - policy = policy_cls(env, **args['policy'], - #batch_size=args['train']['batch_size'], - use_p3o=args['train']['use_p3o'], - p3o_horizon=args['train']['p3o_horizon'], - use_diayn=args['train']['use_diayn'], - diayn_skills=args['train']['diayn_archive'], - ) - args['rnn']['input_size'] = policy.hidden_size - args['rnn']['hidden_size'] = policy.hidden_size - if rnn_cls is not None: - policy = rnn_cls(env, policy, **args['rnn']) - - return policy.to(args['train']['device']) - -def sweep(args, env_name, make_env, policy_cls, rnn_cls): - method = args['sweep']['method'] - if method == 'random': - sweep = pufferlib.sweep.Random(args['sweep']) - elif method == 'pareto_genetic': - sweep = pufferlib.sweep.ParetoGenetic(args['sweep']) - elif method == 'protein': - sweep = pufferlib.sweep.Protein( - args['sweep'], - resample_frequency=0, - num_random_samples=50, # Should be number of params - max_suggestion_cost=args['max_suggestion_cost'], - min_score = args['sweep']['metric']['min'], - max_score = args['sweep']['metric']['max'], - ) - elif method == 'carbs': - sweep = pufferlib.sweep.Carbs( - args['sweep'], - resample_frequency=5, - num_random_samples=10, # Should be number of params - max_suggestion_cost=args['max_suggestion_cost'], - ) - else: - raise ValueError(f'Invalid sweep method {method} (random/pareto_genetic/protein)') - - target_metric = args['sweep']['metric']['name'] - for i in range(args['max_runs']): - seed = time.time_ns() & 0xFFFFFFFF - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - - info = sweep.suggest(args) - if args['train']['minibatch_size'] >= args['train']['batch_size']: - sweep.observe(args, 0.0, 0.0) - continue - - scores, costs, timesteps, _, _ = train(args, make_env, policy_cls, rnn_cls, target_metric) - - # Hacky patch to prevent increasing total_timesteps when not swept - total_timesteps = args['train']['total_timesteps'] - for score, cost, timestep in zip(scores, costs, timesteps): - args['train']['total_timesteps'] = timestep - sweep.observe(args, score, cost) - - args['train']['total_timesteps'] = total_timesteps - - print('Score:', score, 'Cost:', cost, 'Timesteps:', timestep) - -def train(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_points=100, - elos={'model_random.pt': 1000}, vecenv=None, wandb=None, neptune=None): - if args['vec'] == 'serial': - vec = pufferlib.vector.Serial - elif args['vec'] == 'multiprocessing': - vec = pufferlib.vector.Multiprocessing - elif args['vec'] == 'ray': - vec = pufferlib.vector.Ray - elif args['vec'] == 'native': - vec = pufferlib.environment.PufferEnv - else: - raise ValueError(f'Invalid --vec (serial/multiprocessing/ray/native).') - - env_name = args['env_name'] - if vecenv is None: - vecenv = pufferlib.vector.make( - make_env, - env_kwargs=args['env'], - num_envs=args['train']['num_envs'], - num_workers=args['train']['num_workers'], - batch_size=args['train']['env_batch_size'], - zero_copy=args['train']['zero_copy'], - overwork=args['vec_overwork'], - seed=args['train']['seed'], - backend=vec, - ) - - policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) - - if args['ddp']: - from torch.nn.parallel import DistributedDataParallel as DDP - orig_policy = policy - policy = DDP(policy, device_ids=[args['rank']]) - # TODO: Test this? isinstance? - if hasattr(orig_policy, 'lstm'): - policy.lstm = orig_policy.lstm - - neptune = None - wandb = None - if args['neptune']: - neptune = init_neptune(args, env_name, id=args['exp_id'], tag=args['tag']) - for k, v in pufferlib.utils.unroll_nested_dict(args): - neptune[k].append(v) - elif args['wandb']: - wandb = init_wandb(args, env_name, id=args['exp_id'], tag=args['tag']) - - train_config = pufferlib.namespace(**args['train'], env=env_name, - exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) - data = clean_pufferl.create(train_config, vecenv, policy, wandb=wandb, neptune=neptune) - - timesteps = [] - scores = [] - costs = [] - target_key = f'environment/{target_metric}' - - vecenv.async_reset(train_config.seed) - while data.global_step < train_config.total_timesteps: - clean_pufferl.evaluate(data) - logs = clean_pufferl.train(data) - if logs is not None and target_key in logs: - timesteps.append(logs['agent_steps']) - scores.append(logs[target_key]) - #costs.append(data.profile.uptime) - - steps_evaluated = 0 - cost = time.time() - data.start_time - batch_size = args['train']['batch_size'] - while len(data.stats[target_metric]) < min_eval_points: - stats, _ = clean_pufferl.evaluate(data) - steps_evaluated += batch_size - - clean_pufferl.mean_and_log(data) - score = stats[target_metric] - print(f'Evaluated {steps_evaluated} steps. Score: {score}') - - scores.append(score) - costs.append(cost) - timesteps.append(data.global_step) - - def downsample_linear(arr, m): - n = len(arr) - x_old = np.linspace(0, 1, n) # Original indices normalized - x_new = np.linspace(0, 1, m) # New indices normalized - return np.interp(x_new, x_old, arr) - - scores = downsample_linear(scores, 10) - costs = downsample_linear(costs, 10) - timesteps = downsample_linear(timesteps, 10) - - if args['neptune']: - neptune['score'].append(score) - neptune['cost'].append(cost) - elif args['wandb']: - wandb.log({'score': score, 'cost': cost}) - - clean_pufferl.close(data) - return scores, costs, timesteps, elos, vecenv - -def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metric): - import torch.distributed as dist - args['rank'] = rank - args['train']['device'] = f'cuda:{rank}' - dist.init_process_group(backend='nccl', rank=rank, world_size=world_size) - train(args, make_env, policy_cls, rnn_cls, target_metric) - dist.destroy_process_group() - -if __name__ == '__main__': - parser = argparse.ArgumentParser( - description=f':blowfish: PufferLib [bright_cyan]{pufferlib.__version__}[/]' - ' demo options. Shows valid args for your env and policy', - formatter_class=RichHelpFormatter, add_help=False) - parser.add_argument('--env', '--environment', type=str, - default='puffer_squared', help='Name of specific environment to run') - parser.add_argument('--mode', type=str, default='train', - choices='train eval evaluate sweep autotune profile'.split()) - parser.add_argument('--vec-overwork', action='store_true', - help='Allow vectorization to use >1 worker/core. Not recommended.') - parser.add_argument('--eval-model-path', type=str, default=None, - help='Path to a pretrained checkpoint') - parser.add_argument('--baseline', action='store_true', - help='Load pretrained model from WandB if available') - parser.add_argument('--ddp', action='store_true', help='Distributed data parallel') - parser.add_argument('--render-mode', type=str, default='auto', - choices=['auto', 'human', 'ansi', 'rgb_array', 'raylib', 'None']) - parser.add_argument('--exp-id', '--exp-name', type=str, - default=None, help='Resume from experiment') - parser.add_argument('--data-path', type=str, default=None, - help='Used for testing hparam algorithms') - parser.add_argument('--track', action='store_true', help='Track on WandB') - parser.add_argument('--max-runs', type=int, default=200, help='Max number of sweep runs') - parser.add_argument('--wandb-project', type=str, default='pufferlib') - parser.add_argument('--wandb-group', type=str, default='debug') - parser.add_argument('--tag', type=str, default=None, help='Tag for experiment') - parser.add_argument('--wandb', action='store_true', help='Track on WandB') - parser.add_argument('--neptune', action='store_true', help='Track on Neptune') - #parser.add_argument('--wandb-project', type=str, default='pufferlib') - #parser.add_argument('--wandb-group', type=str, default='debug') - args = parser.parse_known_args()[0] - - file_paths = glob.glob('config/**/*.ini', recursive=True) - for path in file_paths: - p = configparser.ConfigParser() - p.read('config/default.ini') - - subconfig = os.path.join(*path.split('/')[:-1] + ['default.ini']) - if subconfig in file_paths: - p.read(subconfig) - - p.read(path) - if args.env in p['base']['env_name'].split(): - break - else: - raise Exception('No config for env_name {}'.format(args.env)) - - for section in p.sections(): - for key in p[section]: - if section == 'base': - argparse_key = f'--{key}'.replace('_', '-') - else: - argparse_key = f'--{section}.{key}'.replace('_', '-') - parser.add_argument(argparse_key, default=p[section][key]) - - # Late add help so you get a dynamic menu based on the env - parser.add_argument('-h', '--help', default=argparse.SUPPRESS, - action='help', help='Show this help message and exit') - - parsed = parser.parse_args().__dict__ - args = {'env': {}, 'policy': {}, 'rnn': {}} - env_name = parsed.pop('env') - for key, value in parsed.items(): - next = args - for subkey in key.split('.'): - if subkey not in next: - next[subkey] = {} - prev = next - next = next[subkey] - try: - prev[subkey] = ast.literal_eval(value) - except: - prev[subkey] = value - - package = args['package'] - module_name = f'pufferlib.environments.{package}' - if package == 'ocean': - module_name = 'pufferlib.ocean' - - import importlib - env_module = importlib.import_module(module_name) - - make_env = env_module.env_creator(env_name) - policy_cls = getattr(env_module.torch, args['policy_name']) - - rnn_name = args['rnn_name'] - rnn_cls = None - if rnn_name is not None: - rnn_cls = getattr(env_module.torch, args['rnn_name']) - - if args['baseline']: - assert args['mode'] in ('train', 'eval', 'evaluate') - args['track'] = True - version = '.'.join(pufferlib.__version__.split('.')[:2]) - args['exp_id'] = f'puf-{version}-{env_name}' - args['wandb_group'] = f'puf-{version}-baseline' - shutil.rmtree(f'experiments/{args["exp_id"]}', ignore_errors=True) - run = init_wandb(args, args['exp_id'], resume=False) - if args['mode'] in ('eval', 'evaluate'): - model_name = f'puf-{version}-{env_name}_model:latest' - artifact = run.use_artifact(model_name) - data_dir = artifact.download() - model_file = max(os.listdir(data_dir)) - args['eval_model_path'] = os.path.join(data_dir, model_file) - if args['mode'] == 'train' and args['ddp']: - import torch.multiprocessing as mp - world_size = 1 - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = "29500" - target_metric = args['sweep']['metric']['name'] - mp.spawn(train_ddp, - args=(world_size, args, make_env, policy_cls, rnn_cls, target_metric), - nprocs=world_size, - join=True, - ) - elif args['mode'] == 'train': - target_metric = args['sweep']['metric']['name'] - train(args, make_env, policy_cls, rnn_cls, target_metric) - elif args['mode'] in ('eval', 'evaluate'): - vec = pufferlib.vector.Serial - if args['vec'] == 'native': vec = pufferlib.environment.PufferEnv - clean_pufferl.rollout( - make_env, - args['env'], - policy_cls=policy_cls, - rnn_cls=rnn_cls, - agent_creator=make_policy, - agent_kwargs=args, - backend=vec, - model_path=args['eval_model_path'], - render_mode=args['render_mode'], - device=args['train']['device'], - ) - elif args['mode'] == 'sweep': - assert args['wandb'] or args['neptune'], 'Sweeps require either wandb or neptune' - sweep(args, env_name, make_env, policy_cls, rnn_cls) - elif args['mode'] == 'autotune': - pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) - elif args['mode'] == 'profile': - import cProfile - target_metric = args['sweep']['metric']['name'] - cProfile.run('train(args, make_env, policy_cls, rnn_cls, target_metric)', 'stats.profile') - import pstats - from pstats import SortKey - p = pstats.Stats('stats.profile') - p.sort_stats(SortKey.TIME).print_stats(10) - breakpoint() - pass From c7991feeb85c5a29f839346756c615f571ba1b68 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 14:49:32 +0000 Subject: [PATCH 02/63] temp --- clean_pufferl.py | 429 +++++++++++++++++++++++------------------------ 1 file changed, 208 insertions(+), 221 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 9f54fa05ec..02207e7e06 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -11,7 +11,6 @@ import glob import uuid import ast -import random from threading import Thread from collections import defaultdict, deque @@ -30,220 +29,224 @@ import pufferlib.sweep import pufferlib.vector +import signal # Aggressively exit on ctrl+c +signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) + from rich_argparse import RichHelpFormatter from rich.console import Console from rich.traceback import install install(show_locals=False) # Rich tracebacks -import signal # Aggressively exit on ctrl+c -signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) - - -def create(config, vecenv, policy, optimizer=None, wandb=None, neptune=None): - random.seed(config.seed) - np.random.seed(config.seed) - torch.backends.cudnn.deterministic = config.torch_deterministic - torch.backends.cudnn.benchmark = True - torch.set_float32_matmul_precision('high') - if config.seed is not None: - torch.manual_seed(config.seed) - - ext = 'cu' if 'cuda' in config.device else 'cpp' - puffer_cuda = load( - name='puffer_cuda', - sources=[f'pufferlib.{ext}'], - verbose=True - ) - compute_gae = puffer_cuda.compute_gae - compute_vtrace = puffer_cuda.compute_vtrace - compute_puff_advantage = puffer_cuda.compute_puff_advantage - - losses = pufferlib.namespace( - policy_loss=0, - value_loss=0, - entropy=0, - old_approx_kl=0, - approx_kl=0, - clipfrac=0, - explained_variance=0, - diayn_loss=0, - grad_var=0, - importance=0, - ) +ROUND_OPEN = rich.box.Box( + "╭──╮\n" + "│ │\n" + "│ │\n" + "│ │\n" + "│ │\n" + "│ │\n" + "│ │\n" + "╰──╯\n" +) - utilization = Utilization() - msg = f'Model Size: {abbreviate(count_params(policy))} parameters' - - vecenv.async_reset(config.seed) - total_agents = vecenv.num_agents - obs_shape = vecenv.single_observation_space.shape - atn_shape = vecenv.single_action_space.shape - obs_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_observation_space.dtype] - atn_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_action_space.dtype] - on_policy_rows = config.batch_size // config.bptt_horizon - off_policy_rows = int(config.replay_factor*config.batch_size // config.bptt_horizon) - experience_rows = on_policy_rows + off_policy_rows - pin = config.device == 'cuda' and config.cpu_offload - obs_device = config.device if not pin else 'cpu' - experience = pufferlib.namespace( - obs=torch.zeros(experience_rows, config.bptt_horizon, *obs_shape, - dtype=obs_dtype, pin_memory=pin, device='cpu' if pin else config.device), - actions=torch.zeros(experience_rows, config.bptt_horizon, *atn_shape, - dtype=atn_dtype, device=config.device), - logprobs=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - rewards=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - dones=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - truncateds=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - ratio = torch.ones(experience_rows, config.bptt_horizon, device=config.device), - ) - ep_uses = torch.zeros(experience_rows, device=config.device, dtype=torch.int32) - ep_lengths = torch.zeros(total_agents, device=config.device, dtype=torch.int32) - ep_indices = torch.arange(total_agents, device=config.device, dtype=torch.int32) - free_idx = total_agents - assert free_idx <= experience_rows, f'Total agents {total_agents} must be at least batch size {config.batch_size} / bptt_horizon {config.bptt_horizon} = {experience_rows}' - - diayn_skills = None - if config.use_diayn: - diayn_skills = torch.randint( - 0, config.diayn_archive, (total_agents,), dtype=torch.long, device=config.device) - experience.diayn_batch = torch.zeros(experience_rows, config.bptt_horizon, - dtype=torch.long, device=config.device) +c1 = '[cyan]' +c2 = '[white]' +b1 = '[bright_cyan]' +b2 = '[bright_white]' - if config.use_p3o: - batch_size = config.batch_size - p3o_horizon = config.p3o_horizon - device = config.device - experience.values_mean=torch.zeros(batch_size, p3o_horizon, device=device) - experience.values_std=torch.zeros(batch_size, p3o_horizon, device=device) - experience.reward_block = torch.zeros(batch_size, p3o_horizon, dtype=torch.float32, device=device) - experience.mask_block = torch.ones(batch_size, p3o_horizon, dtype=torch.float32, device=device) - experience.buf = torch.zeros(batch_size, p3o_horizon, dtype=torch.float32, device=device) - experience.advantages = torch.zeros(batch_size, dtype=torch.float32, device=device) - experience.bounds = torch.zeros(batch_size, dtype=torch.int32, device=device) - experience.vstd_max = 1.0 - else: - experience.values = torch.zeros(experience_rows, config.bptt_horizon, device=config.device) - - if config.use_vtrace or config.use_puff_advantage: - experience.importance = torch.ones(experience_rows, config.bptt_horizon, device=config.device) - - lstm_h = None - lstm_c = None - # TODO: This breaks compile - if isinstance(policy, torch.nn.LSTM): - assert total_agents > 0 - if config.env_batch_size > 1: - shape = (total_agents, policy.hidden_size) - lstm_h = torch.zeros(shape).to(config.device) - lstm_c = torch.zeros(shape).to(config.device) - else: - # TODO: Doesn't exist in native envs - n = vecenv.agents_per_batch - shape = (n, policy.hidden_size) - lstm_h = {slice(i*n, (i+1)*n):torch.zeros(shape).to(config.device) for i in range(total_agents//n)} - lstm_c = {slice(i*n, (i+1)*n):torch.zeros(shape).to(config.device) for i in range(total_agents//n)} - - minibatch_size = min(config.minibatch_size, config.max_minibatch_size) - uncompiled_policy = policy - if config.compile: - policy = torch.compile(policy, mode=config.compile_mode, fullgraph=config.compile_fullgraph) - - if config.optimizer == 'adam': - optimizer = torch.optim.Adam( - policy.parameters(), - lr=config.learning_rate, - betas=(config.adam_beta1, config.adam_beta2), - eps=config.adam_eps, +class CleanPuffeRL: + def __init__(self, config, vecenv, policy, optimizer=None, wandb=None, neptune=None): + self.config = config + self.vecenv = vecenv + self.wandb = wandb + self.neptune = neptune + + self.global_step = 0 + self.epoch = 0 + self.stats = defaultdict(list) + self.last_log_time = 0 + + self.device = config.device + + self.use_p3o = config.use_p3o + self.p3o_horizon = config.p3o_horizon + self.puf = config.puf + + self.use_diayn = config.use_diayn, + self.diayn_coef = config.diayn_coef, + + random.seed(config.seed) + np.random.seed(config.seed) + torch.backends.cudnn.deterministic = config.torch_deterministic + torch.backends.cudnn.benchmark = True + torch.set_float32_matmul_precision('high') + if config.seed is not None: + torch.manual_seed(config.seed) + + ext = 'cu' if 'cuda' in config.device else 'cpp' + puffer_cuda = load( + name='puffer_cuda', + sources=[f'pufferlib.{ext}'], + verbose=True ) - elif config.optimizer == 'muon': - from heavyball import ForeachMuon - import heavyball.utils - #heavyball.utils.compile_mode = "reduce-overhead" - optimizer = ForeachMuon( - policy.parameters(), - lr=config.learning_rate, - betas=(config.adam_beta1, config.adam_beta2), - eps=config.adam_eps, - + self.compute_gae = puffer_cuda.compute_gae + self.compute_vtrace = puffer_cuda.compute_vtrace + self.compute_puff_advantage = puffer_cuda.compute_puff_advantage + + self.losses = pufferlib.namespace( + policy_loss=0, + value_loss=0, + entropy=0, + old_approx_kl=0, + approx_kl=0, + clipfrac=0, + explained_variance=0, + diayn_loss=0, + grad_var=0, + importance=0, ) - elif config.optimizer == 'kron': - from heavyball import ForeachPSGDKron - import heavyball.utils - #heavyball.utils.compile_mode = "reduce-overhead" - optimizer = ForeachPSGDKron( - policy.parameters(), - lr=config.learning_rate, - precond_lr=config.precond_lr, - beta=config.adam_beta1, + + self.utilization = Utilization() + self.msg = f'Model Size: {abbreviate(count_params(policy))} parameters' + + vecenv.async_reset(config.seed) + total_agents = vecenv.num_agents + obs_shape = vecenv.single_observation_space.shape + atn_shape = vecenv.single_action_space.shape + obs_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_observation_space.dtype] + atn_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_action_space.dtype] + self.on_policy_rows = config.batch_size // config.bptt_horizon + self.off_policy_rows = int(config.replay_factor*config.batch_size // config.bptt_horizon) + experience_rows = self.on_policy_rows + self.off_policy_rows + pin = config.device == 'cuda' and config.cpu_offload + obs_device = config.device if not pin else 'cpu' + experience = pufferlib.namespace( + obs=torch.zeros(experience_rows, config.bptt_horizon, *obs_shape, + dtype=obs_dtype, pin_memory=pin, device='cpu' if pin else config.device), + actions=torch.zeros(experience_rows, config.bptt_horizon, *atn_shape, + dtype=atn_dtype, device=config.device), + logprobs=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), + rewards=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), + dones=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), + truncateds=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), + ratio = torch.ones(experience_rows, config.bptt_horizon, device=config.device), ) - else: - raise ValueError(f'Unknown optimizer: {config.optimizer}') - - epochs = config.total_timesteps // config.batch_size - assert config.scheduler in ('linear', 'cosine') - if config.scheduler == 'linear': - scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=0.0, total_iters=epochs) - elif config.scheduler == 'cosine': - scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) - - amp_context = nullcontext() - scaler = None - if config.precision != 'float32': - amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) - scaler = torch.amp.GradScaler() - - profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', - 'learn', 'train_copy', 'train_misc', 'custom'], frequency=5) - - data = pufferlib.namespace( - config=config, - vecenv=vecenv, - policy=policy, - uncompiled_policy=uncompiled_policy, - optimizer=optimizer, - scheduler=scheduler, - scaler=scaler, - experience=experience, - profile=profile, - losses=losses, - wandb=wandb, - neptune=neptune, - global_step=0, - epoch=0, - stats=defaultdict(list), - msg=msg, - last_log_time=0, - utilization=utilization, - use_p3o=config.use_p3o, - p3o_horizon=config.p3o_horizon, - puf=config.puf, - use_diayn=config.use_diayn, - diayn_coef=config.diayn_coef, - # Do we use these? - ptr=0, - step=0, - lstm_h=lstm_h, - lstm_c=lstm_c, - ep_uses=ep_uses, - ep_lengths=ep_lengths, - ep_indices=ep_indices, - free_idx=free_idx, - on_policy_rows=on_policy_rows, - off_policy_rows=off_policy_rows, - experience_rows=experience_rows, - device=config.device, - minibatch_size=minibatch_size, - compute_gae=compute_gae, - compute_vtrace=compute_vtrace, - compute_puff_advantage=compute_puff_advantage, - diayn_skills=diayn_skills, - total_agents=total_agents, - total_epochs=epochs, - start_time=time.time(), - uptime=0, - ) - print_dashboard(data, clear=True) - return data + self.ep_uses = torch.zeros(experience_rows, device=config.device, dtype=torch.int32) + self.ep_lengths = torch.zeros(total_agents, device=config.device, dtype=torch.int32) + self.ep_indices = torch.arange(total_agents, device=config.device, dtype=torch.int32) + self.free_idx = total_agents + assert self.free_idx <= experience_rows, f'Total agents {total_agents} must be at least batch size {config.batch_size} / bptt_horizon {config.bptt_horizon} = {experience_rows}' + self.total_agents = total_agents + + self.diayn_skills = None + if config.use_diayn: + self.diayn_skills = torch.randint( + 0, config.diayn_archive, (total_agents,), dtype=torch.long, device=config.device) + experience.diayn_batch = torch.zeros(experience_rows, config.bptt_horizon, + dtype=torch.long, device=config.device) + + if config.use_p3o: + batch_size = config.batch_size + p3o_horizon = config.p3o_horizon + device = config.device + experience.values_mean=torch.zeros(batch_size, p3o_horizon, device=device) + experience.values_std=torch.zeros(batch_size, p3o_horizon, device=device) + experience.reward_block = torch.zeros(batch_size, p3o_horizon, dtype=torch.float32, device=device) + experience.mask_block = torch.ones(batch_size, p3o_horizon, dtype=torch.float32, device=device) + experience.buf = torch.zeros(batch_size, p3o_horizon, dtype=torch.float32, device=device) + experience.advantages = torch.zeros(batch_size, dtype=torch.float32, device=device) + experience.bounds = torch.zeros(batch_size, dtype=torch.int32, device=device) + experience.vstd_max = 1.0 + else: + experience.values = torch.zeros(experience_rows, config.bptt_horizon, device=config.device) + + if config.use_vtrace or config.use_puff_advantage: + experience.importance = torch.ones(experience_rows, config.bptt_horizon, device=config.device) + + self.experience = experience + + lstm_h = None + lstm_c = None + # TODO: This breaks compile + if isinstance(policy, torch.nn.LSTM): + assert total_agents > 0 + if config.env_batch_size > 1: + shape = (total_agents, policy.hidden_size) + lstm_h = torch.zeros(shape).to(config.device) + lstm_c = torch.zeros(shape).to(config.device) + else: + # TODO: Doesn't exist in native envs + n = vecenv.agents_per_batch + shape = (n, policy.hidden_size) + lstm_h = {slice(i*n, (i+1)*n):torch.zeros(shape).to(config.device) for i in range(total_agents//n)} + lstm_c = {slice(i*n, (i+1)*n):torch.zeros(shape).to(config.device) for i in range(total_agents//n)} + + self.lstm_h = lstm_h + self.lstm_c = lstm_c + + self.minibatch_size = min(config.minibatch_size, config.max_minibatch_size) + self.uncompiled_policy = policy + if config.compile: + policy = torch.compile(policy, mode=config.compile_mode, fullgraph=config.compile_fullgraph) + + self.policy = policy + + if config.optimizer == 'adam': + optimizer = torch.optim.Adam( + policy.parameters(), + lr=config.learning_rate, + betas=(config.adam_beta1, config.adam_beta2), + eps=config.adam_eps, + ) + elif config.optimizer == 'muon': + from heavyball import ForeachMuon + import heavyball.utils + #heavyball.utils.compile_mode = "reduce-overhead" + optimizer = ForeachMuon( + policy.parameters(), + lr=config.learning_rate, + betas=(config.adam_beta1, config.adam_beta2), + eps=config.adam_eps, + + ) + elif config.optimizer == 'kron': + from heavyball import ForeachPSGDKron + import heavyball.utils + #heavyball.utils.compile_mode = "reduce-overhead" + optimizer = ForeachPSGDKron( + policy.parameters(), + lr=config.learning_rate, + precond_lr=config.precond_lr, + beta=config.adam_beta1, + ) + else: + raise ValueError(f'Unknown optimizer: {config.optimizer}') + + self.optimizer = optimizer + + epochs = config.total_timesteps // config.batch_size + self.total_epochs = epochs + assert config.scheduler in ('linear', 'cosine') + if config.scheduler == 'linear': + scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=0.0, total_iters=epochs) + elif config.scheduler == 'cosine': + scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) + + self.scheduler = scheduler + + amp_context = nullcontext() + scaler = None + if config.precision != 'float32': + amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) + scaler = torch.amp.GradScaler() + + self.scaler = scaler + + self.profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', + 'learn', 'train_copy', 'train_misc', 'custom'], frequency=5) + + self.start_time = time.time() + self.uptime=0 + self.print_dashboard(clear=True) def evaluate(data): profile = data.profile @@ -965,22 +968,6 @@ def run(self): def stop(self): self.stopped = True -ROUND_OPEN = rich.box.Box( - "╭──╮\n" - "│ │\n" - "│ │\n" - "│ │\n" - "│ │\n" - "│ │\n" - "│ │\n" - "╰──╯\n" -) - -c1 = '[cyan]' -c2 = '[white]' -b1 = '[bright_cyan]' -b2 = '[bright_white]' - def abbreviate(num): if num < 1e3: return f'{b2}{num:.0f}' From 7b7f048d5049c38ce1d92364088ad7f5eb949a11 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 16:26:42 +0000 Subject: [PATCH 03/63] temp --- clean_pufferl.py | 1351 ++++++++++++++++++++++---------------------- config/default.ini | 9 +- 2 files changed, 666 insertions(+), 694 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 02207e7e06..498ee3b78f 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -1,6 +1,3 @@ -from pdb import set_trace as T -import numpy as np - import os import random import psutil @@ -16,12 +13,11 @@ from collections import defaultdict, deque from contextlib import nullcontext -import rich -from rich.console import Console -from rich.table import Table +import numpy as np + import torch -import torch.distributed as dist -from torch.utils.cpp_extension import load +import torch.distributed +import torch.utils.cpp_extension import pufferlib import pufferlib.utils @@ -32,48 +28,24 @@ import signal # Aggressively exit on ctrl+c signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) -from rich_argparse import RichHelpFormatter + +import rich from rich.console import Console -from rich.traceback import install -install(show_locals=False) # Rich tracebacks - -ROUND_OPEN = rich.box.Box( - "╭──╮\n" - "│ │\n" - "│ │\n" - "│ │\n" - "│ │\n" - "│ │\n" - "│ │\n" - "╰──╯\n" -) - -c1 = '[cyan]' -c2 = '[white]' -b1 = '[bright_cyan]' -b2 = '[bright_white]' +from rich.table import Table +from rich_argparse import RichHelpFormatter +import rich.traceback +rich.traceback.install(show_locals=False) class CleanPuffeRL: - def __init__(self, config, vecenv, policy, optimizer=None, wandb=None, neptune=None): + def __init__(self, config, vecenv, policy): self.config = config self.vecenv = vecenv - self.wandb = wandb - self.neptune = neptune self.global_step = 0 self.epoch = 0 self.stats = defaultdict(list) self.last_log_time = 0 - self.device = config.device - - self.use_p3o = config.use_p3o - self.p3o_horizon = config.p3o_horizon - self.puf = config.puf - - self.use_diayn = config.use_diayn, - self.diayn_coef = config.diayn_coef, - random.seed(config.seed) np.random.seed(config.seed) torch.backends.cudnn.deterministic = config.torch_deterministic @@ -83,7 +55,7 @@ def __init__(self, config, vecenv, policy, optimizer=None, wandb=None, neptune=N torch.manual_seed(config.seed) ext = 'cu' if 'cuda' in config.device else 'cpp' - puffer_cuda = load( + puffer_cuda = torch.utils.cpp_extension.load( name='puffer_cuda', sources=[f'pufferlib.{ext}'], verbose=True @@ -106,7 +78,8 @@ def __init__(self, config, vecenv, policy, optimizer=None, wandb=None, neptune=N ) self.utilization = Utilization() - self.msg = f'Model Size: {abbreviate(count_params(policy))} parameters' + num_params = sum(p.numel() for p in policy.parameters() if p.requires_grad) + self.msg = f'Model Size: {abbreviate(num_params)} parameters' vecenv.async_reset(config.seed) total_agents = vecenv.num_agents @@ -244,579 +217,700 @@ def __init__(self, config, vecenv, policy, optimizer=None, wandb=None, neptune=N self.profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', 'learn', 'train_copy', 'train_misc', 'custom'], frequency=5) + if config.neptune: + self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.run_tag) + for k, v in pufferlib.utils.unroll_nested_dict(args): + neptune[k].append(v) + elif config.wandb: + self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.run_tag) + self.start_time = time.time() self.uptime=0 self.print_dashboard(clear=True) -def evaluate(data): - profile = data.profile - epoch = data.epoch - profile('eval', epoch) - profile('eval_misc', epoch, nest=True) - config = data.config - experience = data.experience - policy = data.policy - infos = defaultdict(list) - lstm_h = data.lstm_h - lstm_c = data.lstm_c - - data.full_rows = 0 - while data.full_rows < data.on_policy_rows: - profile('env', epoch) - o, r, d, t, info, env_id, mask = data.vecenv.recv() + def evaluate(self): + profile = self.profile + epoch = self.epoch + profile('eval', epoch) + profile('eval_misc', epoch, nest=True) + config = self.config + experience = self.experience + policy = self.policy + infos = defaultdict(list) + lstm_h = self.lstm_h + lstm_c = self.lstm_c + + self.full_rows = 0 + while self.full_rows < self.on_policy_rows: + profile('env', epoch) + o, r, d, t, info, env_id, mask = self.vecenv.recv() + + profile('eval_misc', epoch) + # Zero-copy indexing for contiguous env_id + if config.env_batch_size == 1: + gpu_env_id = cpu_env_id = slice(env_id[0], env_id[-1] + 1) + else: + cpu_env_id = env_id + gpu_env_id = torch.as_tensor(env_id).to(config.device, non_blocking=True) - profile('eval_misc', epoch) - # Zero-copy indexing for contiguous env_id - if config.env_batch_size == 1: - gpu_env_id = cpu_env_id = slice(env_id[0], env_id[-1] + 1) - else: - cpu_env_id = env_id - gpu_env_id = torch.as_tensor(env_id).to(config.device, non_blocking=True) + done_mask = d + t + self.global_step += mask.sum() - done_mask = d + t - data.global_step += mask.sum() + profile('eval_copy', epoch) + o = torch.as_tensor(o) + o_device = o.to(config.device, non_blocking=True) + r = torch.as_tensor(r).to(config.device, non_blocking=True) + d = torch.as_tensor(d).to(config.device, non_blocking=True) - profile('eval_copy', epoch) - o = torch.as_tensor(o) - o_device = o.to(config.device, non_blocking=True) - r = torch.as_tensor(r).to(config.device, non_blocking=True) - d = torch.as_tensor(d).to(config.device, non_blocking=True) + h = None + c = None + if lstm_h is not None: + h = lstm_h[gpu_env_id] + c = lstm_c[gpu_env_id] - h = None - c = None - if lstm_h is not None: - h = lstm_h[gpu_env_id] - c = lstm_c[gpu_env_id] + profile('eval_forward', epoch) + with torch.no_grad(): + state = pufferlib.namespace( + reward=r, + done=d, + env_id=gpu_env_id, + mask=mask, + lstm_h=h, + lstm_c=c, + ) + + if config.use_diayn: + state.diayn_z = self.diayn_skills[env_id] + + logits, value = policy(o_device, state) + action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=policy.is_continuous) + r = torch.clamp(r, -1, 1) + + profile('eval_copy', epoch) + with torch.no_grad(): + if lstm_h is not None: + lstm_h[gpu_env_id] = state.lstm_h + lstm_c[gpu_env_id] = state.lstm_c - profile('eval_forward', epoch) - with torch.no_grad(): - state = pufferlib.namespace( - reward=r, - done=d, - env_id=gpu_env_id, - mask=mask, - lstm_h=h, - lstm_c=c, - ) + o = o if config.cpu_offload else o_device + actions = self.store(state, o, value, action, logprob, r, d, gpu_env_id, mask) - if data.use_diayn: - state.diayn_z = data.diayn_skills[env_id] + profile('eval_misc', epoch) + for i in info: + for k, v in pufferlib.utils.unroll_nested_dict(i): + infos[k].append(v) - logits, value = policy(o_device, state) - action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=policy.is_continuous) - r = torch.clamp(r, -1, 1) + profile('env', epoch) + self.vecenv.send(actions) - profile('eval_copy', epoch) - with torch.no_grad(): - if lstm_h is not None: - lstm_h[gpu_env_id] = state.lstm_h - lstm_c[gpu_env_id] = state.lstm_c + profile('eval_misc', epoch) + for k, v in infos.items(): + if '_map' in k: + if self.wandb is not None: + self.stats[f'Media/{k}'] = self.wandb.Image(v[0]) + continue + elif self.neptune is not None: + # TODO: Add neptune image logging + pass + + if isinstance(v, np.ndarray): + v = v.tolist() + try: + iter(v) + except TypeError: + self.stats[k].append(v) + else: + self.stats[k] += v + + self.free_idx = self.total_agents + self.ep_indices = torch.arange(self.total_agents, device=config.device, dtype=torch.int32) + self.ep_lengths.zero_() + self.ep_uses.zero_() + profile.end() + return self.stats, infos + + def train(self): + profile = self.profile + epoch = self.epoch + profile('train', epoch) + config = self.config + experience = self.experience + losses = self.losses + + total_minibatches = int(config.update_epochs*config.batch_size/self.minibatch_size) + accumulate_minibatches = max(1, config.minibatch_size // config.max_minibatch_size) + n_samples = self.minibatch_size // config.bptt_horizon + for mb in range(total_minibatches): + profile('train_misc', epoch, nest=True) + loss = 0 + if config.use_p3o: + # Note: This function gets messed up by computing across + # episode bounds. Because we store experience in a flat buffer, + # bounds can be crossed even after handling dones. This prevent + # our method from scaling to longer horizons. TODO: Redo the way + # we store experience to avoid this issue + vstd_min = experience.values_std.min().item() + vstd_max = experience.values_std.max().item() + + self.mask_block.zero_() + self.buf.zero_() + self.reward_block.zero_() + self.bounds.zero_() + + r_mean = experience.rewards.mean().item() + r_std = experience.rewards.std().item() + + # TODO: Rename vstd to r_std + advantages = compute_advantages( + experience.reward_block, experience.mask_block, + experience.values_mean, experience.values_std, + experience.buf, experience.dones, experience.rewards, + experience.bounds, r_std, self.puf, config.p3o_horizon + ) + + horizon = torch.where(experience.values_std[0] > 0.95*r_std)[0] + horizon = horizon[0].item()+1 if len(horizon) else 1 + if horizon < 16: + horizon = 16 + + advantages = advantages.cpu().numpy() + torch.cuda.synchronize() + elif config.use_vtrace: + importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) + vs = torch.zeros(experience.values.shape, device=config.device) + self.compute_vtrace(experience.values, experience.rewards, experience.dones, + experience.ratio, vs, advantages, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) + elif config.use_puff_advantage: + importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) + vs = torch.zeros(experience.values.shape, device=config.device) + self.compute_puff_advantage(experience.values, experience.rewards, experience.dones, + experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) + else: + importance = advantages = self.compute_gae(experience.values, experience.rewards, + experience.dones, config.gamma, config.gae_lambda) - o = o if config.cpu_offload else o_device - actions = store(data, state, o, value, action, logprob, r, d, gpu_env_id, mask) + profile('train_copy', epoch) + batch = self.sample(importance, n_samples) - profile('eval_misc', epoch) - for i in info: - for k, v in pufferlib.utils.unroll_nested_dict(i): - infos[k].append(v) - - profile('env', epoch) - data.vecenv.send(actions) - - profile('eval_misc', epoch) - for k, v in infos.items(): - if '_map' in k: - if data.wandb is not None: - data.stats[f'Media/{k}'] = data.wandb.Image(v[0]) - continue - elif data.neptune is not None: - # TODO: Add neptune image logging - pass + profile('train_misc', epoch) + state = pufferlib.namespace( + action=batch.actions, + lstm_h=None, + lstm_c=None, + ) - if isinstance(v, np.ndarray): - v = v.tolist() - try: - iter(v) - except TypeError: - data.stats[k].append(v) - else: - data.stats[k] += v - - data.free_idx = data.total_agents - data.ep_indices = torch.arange(data.total_agents, device=config.device, dtype=torch.int32) - data.ep_lengths.zero_() - data.ep_uses.zero_() - profile.end() - return data.stats, infos - -def train(data): - profile = data.profile - epoch = data.epoch - profile('train', epoch) - config = data.config - experience = data.experience - losses = data.losses - - total_minibatches = int(config.update_epochs*config.batch_size/data.minibatch_size) - accumulate_minibatches = max(1, config.minibatch_size // config.max_minibatch_size) - n_samples = data.minibatch_size // config.bptt_horizon - for mb in range(total_minibatches): - profile('train_misc', epoch, nest=True) - loss = 0 - if config.use_p3o: - # Note: This function gets messed up by computing across - # episode bounds. Because we store experience in a flat buffer, - # bounds can be crossed even after handling dones. This prevent - # our method from scaling to longer horizons. TODO: Redo the way - # we store experience to avoid this issue - vstd_min = experience.values_std.min().item() - vstd_max = experience.values_std.max().item() - - data.mask_block.zero_() - data.buf.zero_() - data.reward_block.zero_() - data.bounds.zero_() - - r_mean = experience.rewards.mean().item() - r_std = experience.rewards.std().item() - - # TODO: Rename vstd to r_std - advantages = compute_advantages( - experience.reward_block, experience.mask_block, - experience.values_mean, experience.values_std, - experience.buf, experience.dones, experience.rewards, - experience.bounds, r_std, data.puf, config.p3o_horizon + if config.use_diayn: + state.diayn_z = batch.diayn_z.reshape(-1) + + profile('train_forward', epoch) + if not isinstance(self.policy, torch.nn.LSTM): + batch.obs = batch.obs.reshape(-1, *self.vecenv.single_observation_space.shape) + + # TODO: Currently only returning traj shaped value as a hack + logits, newvalue = self.policy.forward_train(batch.obs, state) + actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, + action=batch.actions, is_continuous=self.policy.is_continuous) + + profile('train_misc', epoch) + if config.use_diayn: + N = 1 + batch_logits = state.batch_logits[:, ::N] + batch_logits = torch.nn.functional.log_softmax(batch_logits, dim=-1) + mask = torch.nn.functional.one_hot(batch.actions[:, ::N], batch_logits.shape[-1]).bool() + #batch_logits = mask*batch_logits + batch_logits = batch_logits.view(batch_logits.shape[0], -1) + diayn_policy = self.policy.policy + q = diayn_policy.discrim_forward(batch_logits) + z_idxs = batch.diayn_z[:, 0] + q = q.view(-1, q.shape[-1]) + diayn_loss = torch.nn.functional.cross_entropy(q, z_idxs) + loss += config.diayn_loss_coef*diayn_loss + + newlogprob = newlogprob.reshape(batch.logprobs.shape) + logratio = newlogprob - batch.logprobs + ratio = logratio.exp() + experience.ratio[batch.idx] = ratio + + # TODO: Only do this if we are KL clipping? Saves 1-2% compute + with torch.no_grad(): + # calculate approx_kl http://joschu.net/blog/kl-approx.html + old_approx_kl = (-logratio).mean() + approx_kl = ((ratio - 1) - logratio).mean() + clipfrac = ((ratio - 1.0).abs() > config.clip_coef).float().mean() + + if config.use_vtrace or config.use_puff_advantage: + with torch.no_grad(): + adv = advantages[batch.idx] + vs = vs[batch.idx] + if config.use_vtrace: + self.compute_vtrace(batch.values, batch.rewards, batch.dones, + ratio, vs, adv, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) + elif config.use_puff_advantage: + self.compute_puff_advantage(batch.values, batch.rewards, batch.dones, + ratio, vs, adv, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) + + #advantages[batch.idx] = adv + #importance[batch.idx] = adv + + adv = batch.advantages + if config.norm_adv: + adv = (adv - adv.mean()) / (adv.std() + 1e-8) + + adv = adv * batch.prio + + # Policy loss + pg_loss1 = -adv * ratio + pg_loss2 = -adv * torch.clamp( + ratio, 1 - config.clip_coef, 1 + config.clip_coef ) + pg_loss = torch.max(pg_loss1, pg_loss2).mean() + + # Value loss + if config.use_p3o: + newvalue_mean = newvalue.mean.view(-1, config.p3o_horizon) + newvalue_std = newvalue.std.view(-1, config.p3o_horizon) + newvalue_var = torch.square(newvalue_std) + criterion = torch.nn.GaussianNLLLoss(reduction='none') + v_loss = criterion(newvalue_mean, batch.reward_block, newvalue_var) + v_loss = v_loss[:, :(horizon+3)] + mask_block = mask_block[:, :(horizon+3)] + v_loss = v_loss[mask_block.bool()].mean() + elif config.clip_vloss: + newvalue = newvalue#.flatten() + ret = batch.returns#.flatten() + v_loss_unclipped = (newvalue - ret) ** 2 + val = batch.values#.flatten() + v_clipped = val + torch.clamp( + newvalue - val, + -config.vf_clip_coef, + config.vf_clip_coef, + ) + v_loss_clipped = (v_clipped - ret) ** 2 + v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) + v_loss = 0.5 * v_loss_max.mean() + else: + newvalue = newvalue.flatten() + v_loss = 0.5 * ((newvalue - ret) ** 2).mean() - horizon = torch.where(experience.values_std[0] > 0.95*r_std)[0] - horizon = horizon[0].item()+1 if len(horizon) else 1 - if horizon < 16: - horizon = 16 + entropy_loss = entropy.mean() + loss += pg_loss - config.ent_coef*entropy_loss + v_loss*config.vf_coef - advantages = advantages.cpu().numpy() - torch.cuda.synchronize() - elif config.use_vtrace: - importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) - vs = torch.zeros(experience.values.shape, device=config.device) - data.compute_vtrace(experience.values, experience.rewards, experience.dones, - experience.ratio, vs, advantages, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) - elif config.use_puff_advantage: - importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) + # This breaks vloss clipping? + with torch.no_grad(): + experience.values[batch.idx] = newvalue + + profile('learn', epoch) + if self.scaler is not None: + loss = self.scaler.scale(loss) + + loss.backward() + + if self.scaler is not None: + self.scaler.unscale_(self.optimizer) + + # TODO: Delete? + with torch.no_grad(): + grads = torch.cat([p.grad.flatten() for p in self.policy.parameters()]) + grad_var = grads.var(0).mean() * config.minibatch_size + self.msg = f'Gradient variance: {grad_var.item():.3f}' + + if (mb + 1) % accumulate_minibatches == 0: + torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config.max_grad_norm) + + # TODO: Can remove scaler if only using bf16 + if self.scaler is None: + self.optimizer.step() + else: + self.scaler.step(self.optimizer) + self.scaler.update() + + self.optimizer.zero_grad() + + profile('train_misc', epoch) + losses.policy_loss += pg_loss.item() / total_minibatches + losses.value_loss += v_loss.item() / total_minibatches + losses.entropy += entropy_loss.item() / total_minibatches + losses.old_approx_kl += old_approx_kl.item() / total_minibatches + losses.approx_kl += approx_kl.item() / total_minibatches + losses.clipfrac += clipfrac.item() / total_minibatches + losses.grad_var += grad_var.item() / total_minibatches + losses.importance += ratio.mean().item() / total_minibatches + + if config.use_diayn: + losses.diayn_loss += diayn_loss.item() / total_minibatches + + if config.target_kl is not None: + if approx_kl > config.target_kl: + break + + # Reprioritize experience + profile('train_misc', epoch) + self.max_uses = self.ep_uses.max().item() + self.mean_uses = self.ep_uses.float().mean().item() + if config.replay_factor > 0: + advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) vs = torch.zeros(experience.values.shape, device=config.device) - data.compute_puff_advantage(experience.values, experience.rewards, experience.dones, + self.compute_puff_advantage(experience.values, experience.rewards, experience.dones, experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) + + exp = self.sample(advantages, self.off_policy_rows, method='random') + for k, v in experience.items(): + v[self.on_policy_rows:] = exp[k] + + experience.ratio[:self.on_policy_rows] = 1 + + if config.anneal_lr: + self.scheduler.step() + + if config.use_p3o: + y_pred = experience.values_mean + y_true = experience.reward_block else: - importance = advantages = data.compute_gae(experience.values, experience.rewards, - experience.dones, config.gamma, config.gae_lambda) + y_pred = experience.values.flatten() - profile('train_copy', epoch) - batch = sample(data, importance, n_samples) + # Probably not updated + y_true = advantages.flatten() + experience.values.flatten() - profile('train_misc', epoch) - state = pufferlib.namespace( - action=batch.actions, - lstm_h=None, - lstm_c=None, - ) + var_y = y_true.var() + explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y + #losses.explained_variance = explained_var.item() - if config.use_diayn: - state.diayn_z = batch.diayn_z.reshape(-1) + profile.end() + profile.clear() + logs = None + self.epoch += 1 + done_training = self.global_step >= config.total_timesteps + if done_training or self.global_step == 0 or time.time() - self.start_time - self.uptime > 1: + self.uptime = time.time() - self.start_time + logs = self.mean_and_log() + self.print_dashboard() + self.stats = defaultdict(list) - profile('train_forward', epoch) - if not isinstance(data.policy, torch.nn.LSTM): - batch.obs = batch.obs.reshape(-1, *data.vecenv.single_observation_space.shape) + for k in losses: + losses[k] = 0 - # TODO: Currently only returning traj shaped value as a hack - logits, newvalue = data.policy.forward_train(batch.obs, state) - actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, - action=batch.actions, is_continuous=data.policy.is_continuous) + if self.epoch % config.checkpoint_interval == 0 or done_training: + self.save_checkpoint() + self.msg = f'Checkpoint saved at update {self.epoch}' - profile('train_misc', epoch) - if config.use_diayn: - N = 1 - batch_logits = state.batch_logits[:, ::N] - batch_logits = torch.nn.functional.log_softmax(batch_logits, dim=-1) - mask = torch.nn.functional.one_hot(batch.actions[:, ::N], batch_logits.shape[-1]).bool() - #batch_logits = mask*batch_logits - batch_logits = batch_logits.view(batch_logits.shape[0], -1) - diayn_policy = data.policy.policy - q = diayn_policy.discrim_forward(batch_logits) - z_idxs = batch.diayn_z[:, 0] - q = q.view(-1, q.shape[-1]) - diayn_loss = torch.nn.functional.cross_entropy(q, z_idxs) - loss += config.diayn_loss_coef*diayn_loss - - newlogprob = newlogprob.reshape(batch.logprobs.shape) - logratio = newlogprob - batch.logprobs - ratio = logratio.exp() - experience.ratio[batch.idx] = ratio - - # TODO: Only do this if we are KL clipping? Saves 1-2% compute - with torch.no_grad(): - # calculate approx_kl http://joschu.net/blog/kl-approx.html - old_approx_kl = (-logratio).mean() - approx_kl = ((ratio - 1) - logratio).mean() - clipfrac = ((ratio - 1.0).abs() > config.clip_coef).float().mean() + return logs - if config.use_vtrace or config.use_puff_advantage: - with torch.no_grad(): - adv = advantages[batch.idx] - vs = vs[batch.idx] - if config.use_vtrace: - data.compute_vtrace(batch.values, batch.rewards, batch.dones, - ratio, vs, adv, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) - elif config.use_puff_advantage: - data.compute_puff_advantage(batch.values, batch.rewards, batch.dones, - ratio, vs, adv, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) - - #advantages[batch.idx] = adv - #importance[batch.idx] = adv - - adv = batch.advantages - if config.norm_adv: - adv = (adv - adv.mean()) / (adv.std() + 1e-8) - - adv = adv * batch.prio - - # Policy loss - pg_loss1 = -adv * ratio - pg_loss2 = -adv * torch.clamp( - ratio, 1 - config.clip_coef, 1 + config.clip_coef - ) - pg_loss = torch.max(pg_loss1, pg_loss2).mean() + def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): + config = self.config + exp = self.experience - # Value loss - if config.use_p3o: - newvalue_mean = newvalue.mean.view(-1, config.p3o_horizon) - newvalue_std = newvalue.std.view(-1, config.p3o_horizon) - newvalue_var = torch.square(newvalue_std) - criterion = torch.nn.GaussianNLLLoss(reduction='none') - v_loss = criterion(newvalue_mean, batch.reward_block, newvalue_var) - v_loss = v_loss[:, :(horizon+3)] - mask_block = mask_block[:, :(horizon+3)] - v_loss = v_loss[mask_block.bool()].mean() - elif config.clip_vloss: - newvalue = newvalue#.flatten() - ret = batch.returns#.flatten() - v_loss_unclipped = (newvalue - ret) ** 2 - val = batch.values#.flatten() - v_clipped = val + torch.clamp( - newvalue - val, - -config.vf_clip_coef, - config.vf_clip_coef, - ) - v_loss_clipped = (v_clipped - ret) ** 2 - v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) - v_loss = 0.5 * v_loss_max.mean() + # Fast path for fully vectorized envs + if self.config.env_batch_size == 1: + l = self.ep_lengths[env_id.start].item() + batch_rows = slice(self.ep_indices[env_id.start].item(), 1+self.ep_indices[env_id.stop - 1].item()) else: - newvalue = newvalue.flatten() - v_loss = 0.5 * ((newvalue - ret) ** 2).mean() + l = self.ep_lengths[env_id] + batch_rows = self.ep_indices[env_id] - entropy_loss = entropy.mean() - loss += pg_loss - config.ent_coef*entropy_loss + v_loss*config.vf_coef + exp.obs[batch_rows, l] = obs + exp.actions[batch_rows, l] = action + exp.logprobs[batch_rows, l] = logprob + exp.rewards[batch_rows, l] = reward + exp.dones[batch_rows, l] = done.float() - # This breaks vloss clipping? - with torch.no_grad(): - experience.values[batch.idx] = newvalue + if config.use_p3o: + exp.values_mean[batch_rows, l] = value.mean + exp.values_std[batch_rows, l] = value.std + else: + exp.values[batch_rows, l] = value.flatten() + #exp.values[l, batch_rows] = value.flatten() - profile('learn', epoch) - if data.scaler is not None: - loss = data.scaler.scale(loss) + if config.use_diayn: + exp.diayn_batch[batch_rows, l] = state.diayn_z - loss.backward() + # TODO: Handle masks!! + #indices = np.where(mask)[0] + #data.ep_lengths[env_id[mask]] += 1 + self.ep_lengths[env_id] += 1 + if config.env_batch_size == 1: + if l+1 >= config.bptt_horizon: + num_full = env_id.stop - env_id.start + self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config.device).int() + self.ep_lengths[env_id] = 0 + self.free_idx += num_full + self.full_rows += num_full + else: + full = self.ep_lengths[env_id] >= config.bptt_horizon + num_full = full.sum() + if num_full > 0: + full_ids = env_id[full] + self.ep_indices[full_ids] = self.free_idx + torch.arange(num_full, device=config.device).int() + self.ep_lengths[full_ids] = 0 + self.free_idx += num_full + self.full_rows += num_full + + return action.cpu().numpy() + + def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio'): + config = self.config + exp = self.experience + if method == 'topk': + _, idx = torch.topk(advantages.abs().sum(axis=1), n) + elif method == 'prio': + adv = advantages.abs().sum(axis=1) + probs = adv**config.prio_alpha + probs = (probs + 1e-6)/(probs.sum() + 1e-6) + idx = torch.multinomial(probs, n) + elif method == 'multinomial': + idx = torch.multinomial(advantages.abs().sum(axis=1) + 1e-6, n) + elif method == 'random': + idx = torch.randint(0, advantages.shape[0], (n,), device=self.device) + else: + raise ValueError(f'Unknown sampling method: {method}') - if data.scaler is not None: - data.scaler.unscale_(data.optimizer) - # TODO: Delete? - with torch.no_grad(): - grads = torch.cat([p.grad.flatten() for p in data.policy.parameters()]) - grad_var = grads.var(0).mean() * config.minibatch_size - data.msg = f'Gradient variance: {grad_var.item():.3f}' + self.ep_uses[idx] += 1 + output = {k: v[idx] for k, v in exp.items()} + output['idx'] = idx - if (mb + 1) % accumulate_minibatches == 0: - torch.nn.utils.clip_grad_norm_(data.policy.parameters(), config.max_grad_norm) - - # TODO: Can remove scaler if only using bf16 - if data.scaler is None: - data.optimizer.step() - else: - data.scaler.step(data.optimizer) - data.scaler.update() + if config.use_p3o: + output['reward_block'] = reward_block[idx] + output['mask_block'] = mask_block[idx] + output['values_mean'] = exp.values_mean[idx] + output['values_std'] = exp.values_std[idx] + else: + output['values'] = exp.values[idx] + output['advantages'] = advantages[idx] + output['returns'] = advantages[idx] + exp.values[idx] - data.optimizer.zero_grad() + if config.use_diayn: + output['diayn_z'] = exp.diayn_batch[idx] + + output['prio'] = 1 + if method == 'prio': + beta = config.prio_beta0 + (1 - config.prio_beta0)*config.prio_alpha*self.epoch/self.total_epochs + output['prio'] = (((1/len(probs)) * (1/probs[idx]))**beta).unsqueeze(1).expand_as(output['advantages']) + + return pufferlib.namespace(**output) + + def mean_and_log(self): + for k in list(self.stats.keys()): + v = self.stats[k] + try: + v = np.mean(v) + except: + del self.stats[k] + + self.stats[k] = v + + device = self.config.device + + agent_steps = int(dist_sum(self.global_step, device)) + logs = { + #'SPS': dist_sum(self.profile.SPS, device), + 'agent_steps': agent_steps, + 'epoch': int(dist_sum(self.epoch, device)), + 'learning_rate': self.optimizer.param_groups[0]["lr"], + 'max_uses': self.max_uses, + 'mean_uses': self.mean_uses, + **{f'environment/{k}': dist_mean(v, device) for k, v in self.stats.items()}, + **{f'losses/{k}': dist_mean(v, device) for k, v in self.losses.items()}, + #**{f'performance/{k}': dist_sum(v, device) for k, v in self.profile}, + } - profile('train_misc', epoch) - losses.policy_loss += pg_loss.item() / total_minibatches - losses.value_loss += v_loss.item() / total_minibatches - losses.entropy += entropy_loss.item() / total_minibatches - losses.old_approx_kl += old_approx_kl.item() / total_minibatches - losses.approx_kl += approx_kl.item() / total_minibatches - losses.clipfrac += clipfrac.item() / total_minibatches - losses.grad_var += grad_var.item() / total_minibatches - losses.importance += ratio.mean().item() / total_minibatches - - if data.use_diayn: - losses.diayn_loss += diayn_loss.item() / total_minibatches - - if config.target_kl is not None: - if approx_kl > config.target_kl: - break + if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: + return logs - # Reprioritize experience - profile('train_misc', epoch) - data.max_uses = data.ep_uses.max().item() - data.mean_uses = data.ep_uses.float().mean().item() - if config.replay_factor > 0: - advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) - vs = torch.zeros(experience.values.shape, device=config.device) - data.compute_puff_advantage(experience.values, experience.rewards, experience.dones, - experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) + if self.wandb is not None: + self.last_log_time = time.time() + self.wandb.log(logs) + elif self.neptune is not None: + self.last_log_time = time.time() + for k, v in logs.items(): + self.neptune[k].append(v, step=agent_steps) - exp = sample(data, advantages, data.off_policy_rows, method='random') - for k, v in experience.items(): - v[data.on_policy_rows:] = exp[k] + return logs - experience.ratio[:data.on_policy_rows] = 1 + def close(self): + self.vecenv.close() + self.utilization.stop() + config = self.config + if self.wandb is not None: + artifact_name = f"{config.exp_id}_model" + artifact = self.wandb.Artifact(artifact_name, type="model") + model_path = self.save_checkpoint(self) + artifact.add_file(model_path) + self.wandb.run.log_artifact(artifact) + self.wandb.finish() + elif self.neptune is not None: + self.neptune.stop() + + def save_checkpoint(self): + config = self.config + path = os.path.join(config.data_dir, config.exp_id) + if not os.path.exists(path): + os.makedirs(path) + + model_name = f'model_{self.epoch:06d}.pt' + model_path = os.path.join(path, model_name) + if os.path.exists(model_path): + return model_path + + torch.save(self.uncompiled_policy.state_dict(), model_path) + + state = { + 'optimizer_state_dict': self.optimizer.state_dict(), + 'global_step': self.global_step, + 'agent_step': self.global_step, + 'update': self.epoch, + 'model_name': model_name, + 'exp_id': config.exp_id, + } + state_path = os.path.join(path, 'trainer_state.pt') + torch.save(state, state_path + '.tmp') + os.rename(state_path + '.tmp', state_path) + return model_path - if config.anneal_lr: - data.scheduler.step() + def try_load_checkpoint(self): + config = self.config + path = os.path.join(config.data_dir, config.exp_id) + if not os.path.exists(path): + print('No checkpoints found. Assuming new experiment') + return - if config.use_p3o: - y_pred = experience.values_mean - y_true = experience.reward_block - else: - y_pred = experience.values.flatten() - - # Probably not updated - y_true = advantages.flatten() + experience.values.flatten() - - var_y = y_true.var() - explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y - #losses.explained_variance = explained_var.item() - - profile.end() - profile.clear() - logs = None - data.epoch += 1 - done_training = data.global_step >= config.total_timesteps - if done_training or data.global_step == 0 or time.time() - data.start_time - data.uptime > 1: - data.uptime = time.time() - data.start_time - logs = mean_and_log(data) - print_dashboard(data) - data.stats = defaultdict(list) - - for k in losses: - losses[k] = 0 - - if data.epoch % config.checkpoint_interval == 0 or done_training: - save_checkpoint(data) - data.msg = f'Checkpoint saved at update {data.epoch}' - - return logs - -def store(data, state, obs, value, action, logprob, reward, done, env_id, mask): - exp = data.experience - - # Fast path for fully vectorized envs - if data.config.env_batch_size == 1: - l = data.ep_lengths[env_id.start].item() - batch_rows = slice(data.ep_indices[env_id.start].item(), 1+data.ep_indices[env_id.stop - 1].item()) - else: - l = data.ep_lengths[env_id] - batch_rows = data.ep_indices[env_id] - - exp.obs[batch_rows, l] = obs - exp.actions[batch_rows, l] = action - exp.logprobs[batch_rows, l] = logprob - exp.rewards[batch_rows, l] = reward - exp.dones[batch_rows, l] = done.float() - - if data.use_p3o: - exp.values_mean[batch_rows, l] = value.mean - exp.values_std[batch_rows, l] = value.std - else: - exp.values[batch_rows, l] = value.flatten() - #exp.values[l, batch_rows] = value.flatten() - - if data.use_diayn: - exp.diayn_batch[batch_rows, l] = state.diayn_z - - # TODO: Handle masks!! - #indices = np.where(mask)[0] - #data.ep_lengths[env_id[mask]] += 1 - data.ep_lengths[env_id] += 1 - if data.config.env_batch_size == 1: - if l+1 >= data.config.bptt_horizon: - num_full = env_id.stop - env_id.start - data.ep_indices[env_id] = data.free_idx + torch.arange(num_full, device=data.device).int() - data.ep_lengths[env_id] = 0 - data.free_idx += num_full - data.full_rows += num_full - else: - full = data.ep_lengths[env_id] >= data.config.bptt_horizon - num_full = full.sum() - if num_full > 0: - full_ids = env_id[full] - data.ep_indices[full_ids] = data.free_idx + torch.arange(num_full, device=data.device).int() - data.ep_lengths[full_ids] = 0 - data.free_idx += num_full - data.full_rows += num_full - - data.step += 1 - - return action.cpu().numpy() - -def sample(data, advantages, n, reward_block=None, mask_block=None, method='prio'): - exp = data.experience - if method == 'topk': - _, idx = torch.topk(advantages.abs().sum(axis=1), n) - elif method == 'prio': - adv = advantages.abs().sum(axis=1) - probs = adv**data.config.prio_alpha - probs = (probs + 1e-6)/(probs.sum() + 1e-6) - idx = torch.multinomial(probs, n) - elif method == 'multinomial': - idx = torch.multinomial(advantages.abs().sum(axis=1) + 1e-6, n) - elif method == 'random': - idx = torch.randint(0, advantages.shape[0], (n,), device=data.device) - else: - raise ValueError(f'Unknown sampling method: {method}') + trainer_path = os.path.join(path, 'trainer_state.pt') + resume_state = torch.load(trainer_path, weights_only=False) + model_path = os.path.join(path, resume_state['model_name']) + self.policy.uncompiled.load_state_dict( + torch.load(model_path, weights_only=True), map_location=config.device) + self.optimizer.load_state_dict(resume_state['optimizer_state_dict']) + print(f'Loaded checkpoint {resume_state["model_name"]}') + + def print_dashboard(self, clear=False, max_stats=[0]): + utilization = self.utilization + profile = self.profile + config = self.config + console = Console() + if clear: + console.clear() + + c1 = '[cyan]' + c2 = '[white]' + b1 = '[bright_cyan]' + b2 = '[bright_white]' + + dashboard = Table(box=rich.box.ROUNDED, expand=True, + show_header=False, border_style='bright_cyan') + + table = Table(box=None, expand=True, show_header=False) + dashboard.add_row(table) + cpu_percent = np.mean(utilization.cpu_util) + dram_percent = np.mean(utilization.cpu_mem) + gpu_percent = np.mean(utilization.gpu_util) + vram_percent = np.mean(utilization.gpu_mem) + table.add_column(justify="left", width=30) + table.add_column(justify="center", width=12) + table.add_column(justify="center", width=12) + table.add_column(justify="center", width=13) + table.add_column(justify="right", width=13) + table.add_row( + f':blowfish: {b1}PufferLib {b2}2.0.0', + f'{c1}CPU: {b2}{cpu_percent:.1f}{c2}%', + f'{c1}GPU: {b2}{gpu_percent:.1f}{c2}%', + f'{c1}DRAM: {b2}{dram_percent:.1f}{c2}%', + f'{c1}VRAM: {b2}{vram_percent:.1f}{c2}%', + ) + + s = Table(box=None, expand=True) + SPS = 0 + delta = profile.eval.delta + profile.train.delta + remaining = 'A hair past a freckle' + if delta != 0: + SPS = config.batch_size/delta + remaining = duration((config.total_timesteps - self.global_step)/SPS) + + uptime = time.time() - self.start_time + s.add_column(f"{c1}Summary", justify='left', vertical='top', width=10) + s.add_column(f"{c1}Value", justify='right', vertical='top', width=14) + s.add_row(f'{c2}Env', f'{b2}{config.env}') + s.add_row(f'{c2}Steps', abbreviate(self.global_step)) + s.add_row(f'{c2}SPS', abbreviate(SPS)) + s.add_row(f'{c2}Epoch', abbreviate(self.epoch)) + s.add_row(f'{c2}Uptime', duration(uptime)) + s.add_row(f'{c2}Remaining', remaining) + + p = Table(box=None, expand=True, show_header=False) + p.add_column(f"{c1}Performance", justify="left", width=10) + p.add_column(f"{c1}Time", justify="right", width=8) + p.add_column(f"{c1}%", justify="right", width=4) + p.add_row(*fmt_perf('Evaluate', b1, delta, profile.eval)) + p.add_row(*fmt_perf(' Forward', c2, delta, profile.eval_forward)) + p.add_row(*fmt_perf(' Env', c2, delta, profile.env)) + p.add_row(*fmt_perf(' Copy', c2, delta, profile.eval_copy)) + p.add_row(*fmt_perf(' Misc', c2, delta, profile.eval_misc)) + p.add_row(*fmt_perf('Train', b1, delta, profile.train)) + p.add_row(*fmt_perf(' Forward', c2, delta, profile.train_forward)) + p.add_row(*fmt_perf(' Learn', c2, delta, profile.learn)) + p.add_row(*fmt_perf(' Copy', c2, delta, profile.train_copy)) + p.add_row(*fmt_perf(' Misc', c2, delta, profile.train_misc)) + if 'custom' in profile.profiles: + p.add_row(*fmt_perf(' Custom', c2, uptime, profile.custom)) + + l = Table(box=None, expand=True, ) + l.add_column(f'{c1}Losses', justify="left", width=16) + l.add_column(f'{c1}Value', justify="right", width=8) + for metric, value in self.losses.items(): + l.add_row(f'{c2}{metric}', f'{b2}{value:.3f}') + + monitor = Table(box=None, expand=True, pad_edge=False) + monitor.add_row(s, p, l) + dashboard.add_row(monitor) + + table = Table(box=None, expand=True, pad_edge=False) + dashboard.add_row(table) + left = Table(box=None, expand=True) + right = Table(box=None, expand=True) + table.add_row(left, right) + left.add_column(f"{c1}User Stats", justify="left", width=20) + left.add_column(f"{c1}Value", justify="right", width=10) + right.add_column(f"{c1}User Stats", justify="left", width=20) + right.add_column(f"{c1}Value", justify="right", width=10) + i = 0 + for metric, value in self.stats.items(): + try: # Discard non-numeric values + int(value) + except: + continue + u = left if i % 2 == 0 else right + u.add_row(f'{c2}{metric}', f'{b2}{value:.3f}') + i += 1 + if i == 30: + break - data.ep_uses[idx] += 1 - output = {k: v[idx] for k, v in exp.items()} - output['idx'] = idx + for i in range(max_stats[0] - i): + u = left if i % 2 == 0 else right + u.add_row('', '') - if data.use_p3o: - output['reward_block'] = reward_block[idx] - output['mask_block'] = mask_block[idx] - output['values_mean'] = exp.values_mean[idx] - output['values_std'] = exp.values_std[idx] - else: - output['values'] = exp.values[idx] - output['advantages'] = advantages[idx] - output['returns'] = advantages[idx] + exp.values[idx] + max_stats[0] = max(max_stats[0], i) - if data.use_diayn: - output['diayn_z'] = exp.diayn_batch[idx] + table = Table(box=None, expand=True, pad_edge=False) + dashboard.add_row(table) + table.add_row(f' {c1}Message: {c2}{self.msg}') - output['prio'] = 1 - if method == 'prio': - beta = data.config.prio_beta0 + (1 - data.config.prio_beta0)*data.config.prio_alpha*data.epoch/data.total_epochs - output['prio'] = (((1/len(probs)) * (1/probs[idx]))**beta).unsqueeze(1).expand_as(output['advantages']) + with console.capture() as capture: + console.print(dashboard) - return pufferlib.namespace(**output) + print('\033[0;0H' + capture.get()) def dist_sum(value, device): - if not dist.is_initialized(): + if not torch.distributed.is_initialized(): return value tensor = torch.tensor(value, device=device) - dist.all_reduce(tensor, op=dist.ReduceOp.SUM) + torch.distributed.all_reduce(tensor, op=torch.distributed.ReduceOp.SUM) return tensor.item() def dist_mean(value, device): - if not dist.is_initialized(): + if not torch.distributed.is_initialized(): return value - return dist_sum(value, device) / dist.get_world_size() - -def mean_and_log(data): - for k in list(data.stats.keys()): - v = data.stats[k] - try: - v = np.mean(v) - except: - del data.stats[k] - - data.stats[k] = v - - device = data.config.device - - agent_steps = int(dist_sum(data.global_step, device)) - logs = { - #'SPS': dist_sum(data.profile.SPS, device), - 'agent_steps': agent_steps, - 'epoch': int(dist_sum(data.epoch, device)), - 'learning_rate': data.optimizer.param_groups[0]["lr"], - 'max_uses': data.max_uses, - 'mean_uses': data.mean_uses, - **{f'environment/{k}': dist_mean(v, device) for k, v in data.stats.items()}, - **{f'losses/{k}': dist_mean(v, device) for k, v in data.losses.items()}, - #**{f'performance/{k}': dist_sum(v, device) for k, v in data.profile}, - } - - if dist.is_initialized() and dist.get_rank() != 0: - return logs - - if data.wandb is not None: - data.last_log_time = time.time() - data.wandb.log(logs) - elif data.neptune is not None: - data.last_log_time = time.time() - for k, v in logs.items(): - data.neptune[k].append(v, step=agent_steps) - - return logs - -def close(data): - data.vecenv.close() - data.utilization.stop() - config = data.config - if data.wandb is not None: - artifact_name = f"{config.exp_id}_model" - artifact = data.wandb.Artifact(artifact_name, type="model") - model_path = save_checkpoint(data) - artifact.add_file(model_path) - data.wandb.run.log_artifact(artifact) - data.wandb.finish() - elif data.neptune is not None: - data.neptune.stop() - -def save_checkpoint(data): - config = data.config - path = os.path.join(config.data_dir, config.exp_id) - if not os.path.exists(path): - os.makedirs(path) - - model_name = f'model_{data.epoch:06d}.pt' - model_path = os.path.join(path, model_name) - if os.path.exists(model_path): - return model_path - - torch.save(data.uncompiled_policy.state_dict(), model_path) - - state = { - 'optimizer_state_dict': data.optimizer.state_dict(), - 'global_step': data.global_step, - 'agent_step': data.global_step, - 'update': data.epoch, - 'model_name': model_name, - 'exp_id': config.exp_id, - } - state_path = os.path.join(path, 'trainer_state.pt') - torch.save(state, state_path + '.tmp') - os.rename(state_path + '.tmp', state_path) - return model_path - -def try_load_checkpoint(data): - config = data.config - path = os.path.join(config.data_dir, config.exp_id) - if not os.path.exists(path): - print('No checkpoints found. Assuming new experiment') - return - - trainer_path = os.path.join(path, 'trainer_state.pt') - resume_state = torch.load(trainer_path, weights_only=False) - model_path = os.path.join(path, resume_state['model_name']) - data.policy.uncompiled.load_state_dict( - torch.load(model_path, weights_only=True), map_location=config.device) - data.optimizer.load_state_dict(resume_state['optimizer_state_dict']) - print(f'Loaded checkpoint {resume_state["model_name"]}') - -def count_params(policy): - return sum(p.numel() for p in policy.parameters() if p.requires_grad) + return dist_sum(value, device) / torch.distributed.get_world_size() def rollout(env_creator, env_kwargs, policy_cls, rnn_cls, agent_creator, agent_kwargs, backend, render_mode='auto', model_path=None, device='cuda'): @@ -991,119 +1085,6 @@ def fmt_perf(name, color, delta_ref, prof): percent = 0 if delta_ref == 0 else int(100*prof.delta/delta_ref - 1e-5) return f'{color}{name}', duration(prof.elapsed), f'{b2}{percent:2d}{c2}%' -# TODO: Add env name to print_dashboard -def print_dashboard(data, clear=False, max_stats=[0]): - utilization = data.utilization - profile = data.profile - config = data.config - console = Console() - if clear: - console.clear() - - dashboard = Table(box=ROUND_OPEN, expand=True, - show_header=False, border_style='bright_cyan') - - table = Table(box=None, expand=True, show_header=False) - dashboard.add_row(table) - cpu_percent = np.mean(utilization.cpu_util) - dram_percent = np.mean(utilization.cpu_mem) - gpu_percent = np.mean(utilization.gpu_util) - vram_percent = np.mean(utilization.gpu_mem) - table.add_column(justify="left", width=30) - table.add_column(justify="center", width=12) - table.add_column(justify="center", width=12) - table.add_column(justify="center", width=13) - table.add_column(justify="right", width=13) - table.add_row( - f':blowfish: {b1}PufferLib {b2}2.0.0', - f'{c1}CPU: {b2}{cpu_percent:.1f}{c2}%', - f'{c1}GPU: {b2}{gpu_percent:.1f}{c2}%', - f'{c1}DRAM: {b2}{dram_percent:.1f}{c2}%', - f'{c1}VRAM: {b2}{vram_percent:.1f}{c2}%', - ) - - s = Table(box=None, expand=True) - SPS = 0 - delta = profile.eval.delta + profile.train.delta - remaining = 'A hair past a freckle' - if delta != 0: - SPS = config.batch_size/delta - remaining = duration((config.total_timesteps - data.global_step)/SPS) - - uptime = time.time() - data.start_time - s.add_column(f"{c1}Summary", justify='left', vertical='top', width=10) - s.add_column(f"{c1}Value", justify='right', vertical='top', width=14) - s.add_row(f'{c2}Env', f'{b2}{config.env}') - s.add_row(f'{c2}Steps', abbreviate(data.global_step)) - s.add_row(f'{c2}SPS', abbreviate(SPS)) - s.add_row(f'{c2}Epoch', abbreviate(data.epoch)) - s.add_row(f'{c2}Uptime', duration(uptime)) - s.add_row(f'{c2}Remaining', remaining) - - p = Table(box=None, expand=True, show_header=False) - p.add_column(f"{c1}Performance", justify="left", width=10) - p.add_column(f"{c1}Time", justify="right", width=8) - p.add_column(f"{c1}%", justify="right", width=4) - p.add_row(*fmt_perf('Evaluate', b1, delta, profile.eval)) - p.add_row(*fmt_perf(' Forward', c2, delta, profile.eval_forward)) - p.add_row(*fmt_perf(' Env', c2, delta, profile.env)) - p.add_row(*fmt_perf(' Copy', c2, delta, profile.eval_copy)) - p.add_row(*fmt_perf(' Misc', c2, delta, profile.eval_misc)) - p.add_row(*fmt_perf('Train', b1, delta, profile.train)) - p.add_row(*fmt_perf(' Forward', c2, delta, profile.train_forward)) - p.add_row(*fmt_perf(' Learn', c2, delta, profile.learn)) - p.add_row(*fmt_perf(' Copy', c2, delta, profile.train_copy)) - p.add_row(*fmt_perf(' Misc', c2, delta, profile.train_misc)) - if 'custom' in profile.profiles: - p.add_row(*fmt_perf(' Custom', c2, uptime, profile.custom)) - - l = Table(box=None, expand=True, ) - l.add_column(f'{c1}Losses', justify="left", width=16) - l.add_column(f'{c1}Value', justify="right", width=8) - for metric, value in data.losses.items(): - l.add_row(f'{c2}{metric}', f'{b2}{value:.3f}') - - monitor = Table(box=None, expand=True, pad_edge=False) - monitor.add_row(s, p, l) - dashboard.add_row(monitor) - - table = Table(box=None, expand=True, pad_edge=False) - dashboard.add_row(table) - left = Table(box=None, expand=True) - right = Table(box=None, expand=True) - table.add_row(left, right) - left.add_column(f"{c1}User Stats", justify="left", width=20) - left.add_column(f"{c1}Value", justify="right", width=10) - right.add_column(f"{c1}User Stats", justify="left", width=20) - right.add_column(f"{c1}Value", justify="right", width=10) - i = 0 - for metric, value in data.stats.items(): - try: # Discard non-numeric values - int(value) - except: - continue - - u = left if i % 2 == 0 else right - u.add_row(f'{c2}{metric}', f'{b2}{value:.3f}') - i += 1 - if i == 30: - break - - for i in range(max_stats[0] - i): - u = left if i % 2 == 0 else right - u.add_row('', '') - - max_stats[0] = max(max_stats[0], i) - - table = Table(box=None, expand=True, pad_edge=False) - dashboard.add_row(table) - table.add_row(f' {c1}Message: {c2}{data.msg}') - - with console.capture() as capture: - console.print(dashboard) - - print('\033[0;0H' + capture.get()) - def init_wandb(args, name, id=None, resume=True, tag=None): import wandb @@ -1240,18 +1221,9 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin if hasattr(orig_policy, 'lstm'): policy.lstm = orig_policy.lstm - neptune = None - wandb = None - if args['neptune']: - neptune = init_neptune(args, env_name, id=args['exp_id'], tag=args['tag']) - for k, v in pufferlib.utils.unroll_nested_dict(args): - neptune[k].append(v) - elif args['wandb']: - wandb = init_wandb(args, env_name, id=args['exp_id'], tag=args['tag']) - train_config = pufferlib.namespace(**args['train'], env=env_name, exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) - data = create(train_config, vecenv, policy, wandb=wandb, neptune=neptune) + pufferl = CleanPuffeRL(train_config, vecenv, policy) timesteps = [] scores = [] @@ -1259,28 +1231,28 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin target_key = f'environment/{target_metric}' vecenv.async_reset(train_config.seed) - while data.global_step < train_config.total_timesteps: - evaluate(data) - logs = train(data) + while pufferl.global_step < train_config.total_timesteps: + pufferl.evaluate() + logs = pufferl.train() if logs is not None and target_key in logs: timesteps.append(logs['agent_steps']) scores.append(logs[target_key]) #costs.append(data.profile.uptime) steps_evaluated = 0 - cost = time.time() - data.start_time + cost = time.time() - pufferl.start_time batch_size = args['train']['batch_size'] - while len(data.stats[target_metric]) < min_eval_points: - stats, _ = evaluate(data) + while len(pufferl.stats[target_metric]) < min_eval_points: + stats, _ = pufferl.evaluate() steps_evaluated += batch_size - mean_and_log(data) + pufferl.mean_and_log() score = stats[target_metric] print(f'Evaluated {steps_evaluated} steps. Score: {score}') scores.append(score) costs.append(cost) - timesteps.append(data.global_step) + timesteps.append(pufferl.global_step) def downsample_linear(arr, m): n = len(arr) @@ -1298,16 +1270,15 @@ def downsample_linear(arr, m): elif args['wandb']: wandb.log({'score': score, 'cost': cost}) - close(data) + pufferl.close() return scores, costs, timesteps, elos, vecenv def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metric): - import torch.distributed as dist args['rank'] = rank args['train']['device'] = f'cuda:{rank}' - dist.init_process_group(backend='nccl', rank=rank, world_size=world_size) - train(args, make_env, policy_cls, rnn_cls, target_metric) - dist.destroy_process_group() + torch.distributed.init_process_group(backend='nccl', rank=rank, world_size=world_size) + train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) + torch.distributed.destroy_process_group() if __name__ == '__main__': parser = argparse.ArgumentParser( diff --git a/config/default.ini b/config/default.ini index eb7893601a..00ce5dc4e4 100644 --- a/config/default.ini +++ b/config/default.ini @@ -6,15 +6,16 @@ policy_name = Policy rnn_name = None max_suggestion_cost = 3600 -[workspace] -name = pufferai -project = ablations - [env] [policy] [rnn] [train] +name = pufferai +project = ablations +run_id = None +run_tag = None + seed = 0 torch_deterministic = True cpu_offload = False From 530ef572154bad15753007068a8f32ea611edb63 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 17:17:12 +0000 Subject: [PATCH 04/63] clean up logging --- clean_pufferl.py | 31 +++++++++++++------------------ config/default.ini | 2 ++ 2 files changed, 15 insertions(+), 18 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 498ee3b78f..8f50e16d7f 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -28,7 +28,6 @@ import signal # Aggressively exit on ctrl+c signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) - import rich from rich.console import Console from rich.table import Table @@ -36,6 +35,12 @@ import rich.traceback rich.traceback.install(show_locals=False) +c1 = '[cyan]' +c2 = '[white]' +b1 = '[bright_cyan]' +b2 = '[bright_white]' + + class CleanPuffeRL: def __init__(self, config, vecenv, policy): self.config = config @@ -220,7 +225,7 @@ def __init__(self, config, vecenv, policy): if config.neptune: self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.run_tag) for k, v in pufferlib.utils.unroll_nested_dict(args): - neptune[k].append(v) + self.neptune[k].append(v) elif config.wandb: self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.run_tag) @@ -689,6 +694,7 @@ def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio return pufferlib.namespace(**output) def mean_and_log(self): + config = self.config for k in list(self.stats.keys()): v = self.stats[k] try: @@ -698,7 +704,7 @@ def mean_and_log(self): self.stats[k] = v - device = self.config.device + device = config.device agent_steps = int(dist_sum(self.global_step, device)) logs = { @@ -716,10 +722,10 @@ def mean_and_log(self): if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: return logs - if self.wandb is not None: + if config.wandb: self.last_log_time = time.time() self.wandb.log(logs) - elif self.neptune is not None: + elif config.neptune: self.last_log_time = time.time() for k, v in logs.items(): self.neptune[k].append(v, step=agent_steps) @@ -730,14 +736,14 @@ def close(self): self.vecenv.close() self.utilization.stop() config = self.config - if self.wandb is not None: + if config.wandb: artifact_name = f"{config.exp_id}_model" artifact = self.wandb.Artifact(artifact_name, type="model") model_path = self.save_checkpoint(self) artifact.add_file(model_path) self.wandb.run.log_artifact(artifact) self.wandb.finish() - elif self.neptune is not None: + elif config.neptune: self.neptune.stop() def save_checkpoint(self): @@ -789,11 +795,6 @@ def print_dashboard(self, clear=False, max_stats=[0]): if clear: console.clear() - c1 = '[cyan]' - c2 = '[white]' - b1 = '[bright_cyan]' - b2 = '[bright_white]' - dashboard = Table(box=rich.box.ROUNDED, expand=True, show_header=False, border_style='bright_cyan') @@ -1264,12 +1265,6 @@ def downsample_linear(arr, m): costs = downsample_linear(costs, 10) timesteps = downsample_linear(timesteps, 10) - if args['neptune']: - neptune['score'].append(score) - neptune['cost'].append(cost) - elif args['wandb']: - wandb.log({'score': score, 'cost': cost}) - pufferl.close() return scores, costs, timesteps, elos, vecenv diff --git a/config/default.ini b/config/default.ini index 00ce5dc4e4..f4bb3879d6 100644 --- a/config/default.ini +++ b/config/default.ini @@ -15,6 +15,8 @@ name = pufferai project = ablations run_id = None run_tag = None +neptune = False +wandb = False seed = 0 torch_deterministic = True From d35b9dea37be3a19bce4c49d96be1303d994acc9 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 20:16:25 +0000 Subject: [PATCH 05/63] Initial proper torch bind for cuda --- clean_pufferl.py | 18 ++-- pufferlib.cpp | 116 +++++++++++++++++++++---- pufferlib.cu => pufferlib/pufferlib.cu | 73 +++++++++++++--- shared.cpp => pufferlib/shared.cpp | 4 +- setup.py | 88 ++++++++++++++----- 5 files changed, 232 insertions(+), 67 deletions(-) rename pufferlib.cu => pufferlib/pufferlib.cu (76%) rename shared.cpp => pufferlib/shared.cpp (99%) diff --git a/clean_pufferl.py b/clean_pufferl.py index 8f50e16d7f..a100d68e77 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -16,6 +16,7 @@ import numpy as np import torch + import torch.distributed import torch.utils.cpp_extension @@ -24,6 +25,7 @@ import pufferlib.pytorch import pufferlib.sweep import pufferlib.vector +from pufferlib import _C import signal # Aggressively exit on ctrl+c signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) @@ -59,16 +61,6 @@ def __init__(self, config, vecenv, policy): if config.seed is not None: torch.manual_seed(config.seed) - ext = 'cu' if 'cuda' in config.device else 'cpp' - puffer_cuda = torch.utils.cpp_extension.load( - name='puffer_cuda', - sources=[f'pufferlib.{ext}'], - verbose=True - ) - self.compute_gae = puffer_cuda.compute_gae - self.compute_vtrace = puffer_cuda.compute_vtrace - self.compute_puff_advantage = puffer_cuda.compute_puff_advantage - self.losses = pufferlib.namespace( policy_loss=0, value_loss=0, @@ -388,7 +380,7 @@ def train(self): elif config.use_puff_advantage: importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) vs = torch.zeros(experience.values.shape, device=config.device) - self.compute_puff_advantage(experience.values, experience.rewards, experience.dones, + torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, experience.dones, experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) else: importance = advantages = self.compute_gae(experience.values, experience.rewards, @@ -451,7 +443,7 @@ def train(self): self.compute_vtrace(batch.values, batch.rewards, batch.dones, ratio, vs, adv, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) elif config.use_puff_advantage: - self.compute_puff_advantage(batch.values, batch.rewards, batch.dones, + torch.ops.pufferlib.compute_puff_advantage(batch.values, batch.rewards, batch.dones, ratio, vs, adv, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) #advantages[batch.idx] = adv @@ -555,7 +547,7 @@ def train(self): if config.replay_factor > 0: advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) vs = torch.zeros(experience.values.shape, device=config.device) - self.compute_puff_advantage(experience.values, experience.rewards, experience.dones, + torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, experience.dones, experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) exp = self.sample(advantages, self.off_policy_rows, method='random') diff --git a/pufferlib.cpp b/pufferlib.cpp index 6c8aab7fe5..a9ec5745d0 100644 --- a/pufferlib.cpp +++ b/pufferlib.cpp @@ -1,5 +1,77 @@ -#include "shared.cpp" +#include +#include +#include +#include +#include +extern "C" { + /* Creates a dummy empty _C module that can be imported from Python. + The import from Python will load the .so consisting of this file + in this extension, so that the TORCH_LIBRARY static initializers + below are run. */ + PyObject* PyInit__C(void) + { + static struct PyModuleDef module_def = { + PyModuleDef_HEAD_INIT, + "_C", /* name of module */ + NULL, /* module documentation, may be NULL */ + -1, /* size of per-interpreter state of the module, + or -1 if the module keeps state in global variables. */ + NULL, /* methods */ + }; + return PyModule_Create(&module_def); + } +} + +namespace pufferlib { + +static const int max_horizon = 256; +void puff_advantage_row(float* values, float* rewards, float* dones, + float* importance, float* vs, float* advantages, float gamma, float lambda, + float rho_clip, float c_clip, int horizon) { + vs[horizon-1] = values[horizon-1]; + float lastpufferlam = 0; + for (int t = horizon-2; t >= 0; t--) { + int t_next = t + 1; + float nextnonterminal = 1.0 - dones[t_next]; + float rho_t = fminf(importance[t], rho_clip); + float c_t = fminf(importance[t], c_clip); + // TODO: t_next works and t doesn't. Check original formula + float delta = rho_t*(rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]); + lastpufferlam = delta + gamma*lambda*c_t*lastpufferlam*nextnonterminal; + + //float delta = rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]; + //lastpufferlam = delta + gamma*lambda*lastpufferlam*nextnonterminal; + + + advantages[t] = lastpufferlam; + vs[t] = advantages[t] + values[t]; + //advantages[t] = rho_t*(rewards[t] + gamma*vs[t_next]*nextnonterminal - values[t]); + //vs[t] = lastpufferlam + values[t]; + } +} + +void vtrace_check(torch::Tensor values, torch::Tensor rewards, + torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + int num_steps, int horizon) { + + // Validate input tensors + torch::Device device = values.device(); + for (const torch::Tensor& t : {values, rewards, dones, importance, vs, advantages}) { + TORCH_CHECK(t.dim() == 2, "Tensor must be 2D"); + TORCH_CHECK(t.device() == device, "All tensors must be on same device"); + TORCH_CHECK(t.size(0) == num_steps, "First dimension must match num_steps"); + TORCH_CHECK(t.size(1) == horizon, "Second dimension must match horizon"); + TORCH_CHECK(t.dtype() == torch::kFloat32, "All tensors must be float32"); + assert(horizon <= max_horizon); + if (!t.is_contiguous()) { + t.contiguous(); + } + } +} + + +/* // [num_steps, horizon] void gae(float* values, float* rewards, float* dones, float* advantages, float gamma, float gae_lambda, int num_steps, int horizon){ @@ -34,6 +106,20 @@ void vtrace(float* values, float* rewards, float* dones, float* importance, } } +void compute_vtrace(torch::Tensor values, torch::Tensor rewards, + torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + float gamma, float rho_clip, float c_clip) { + int num_steps = values.size(0); + int horizon = values.size(1); + vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); + vtrace(values.data_ptr(), rewards.data_ptr(), + dones.data_ptr(), importance.data_ptr(), + vs.data_ptr(), advantages.data_ptr(), + gamma, rho_clip, c_clip, num_steps, horizon + ); +} +*/ + // [num_steps, horizon] void puff_advantage(float* values, float* rewards, float* dones, float* importance, float* vs, float* advantages, float gamma, float lambda, float rho_clip, float c_clip, @@ -47,22 +133,10 @@ void puff_advantage(float* values, float* rewards, float* dones, float* importan } } -void compute_vtrace(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, - float gamma, float rho_clip, float c_clip) { - int num_steps = values.size(0); - int horizon = values.size(1); - vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); - vtrace(values.data_ptr(), rewards.data_ptr(), - dones.data_ptr(), importance.data_ptr(), - vs.data_ptr(), advantages.data_ptr(), - gamma, rho_clip, c_clip, num_steps, horizon - ); -} -void compute_puff_advantage(torch::Tensor values, torch::Tensor rewards, +void compute_puff_advantage_cpu(torch::Tensor values, torch::Tensor rewards, torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, - float gamma, float lambda, float rho_clip, float c_clip) { + double gamma, double lambda, double rho_clip, double c_clip) { int num_steps = values.size(0); int horizon = values.size(1); vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); @@ -73,8 +147,12 @@ void compute_puff_advantage(torch::Tensor values, torch::Tensor rewards, ); } -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("compute_gae", &compute_gae, "Compute GAE with C"); - m.def("compute_vtrace", &compute_vtrace, "Compute VTrace with C"); - m.def("compute_puff_advantage", &compute_puff_advantage, "Compute PuffAdvantage with C"); +TORCH_LIBRARY(pufferlib, m) { + m.def("compute_puff_advantage(Tensor(a!) values, Tensor(b!) rewards, Tensor(c!) dones, Tensor(d!) importance, Tensor(e!) vs, Tensor(f!) advantages, float gamma, float lambda, float rho_clip, float c_clip) -> ()"); + } + +TORCH_LIBRARY_IMPL(pufferlib, CPU, m) { + m.impl("compute_puff_advantage", &compute_puff_advantage_cpu); +} + } diff --git a/pufferlib.cu b/pufferlib/pufferlib.cu similarity index 76% rename from pufferlib.cu rename to pufferlib/pufferlib.cu index 6cf490496c..f2da160d55 100644 --- a/pufferlib.cu +++ b/pufferlib/pufferlib.cu @@ -1,5 +1,10 @@ -#include "shared.cpp" +#include +#include +#include +namespace pufferlib { + +/* __global__ void p3o_kernel( float* reward_block, // [num_steps, horizon] float* reward_mask, // [num_steps, horizon] @@ -96,7 +101,6 @@ void compute_p3o(torch::Tensor reward_block, torch::Tensor reward_mask, int horizon) { // TODO: Port from python - /* assert all(t.is_cuda for t in [reward_block, reward_mask, values_mean, values_std, buf, dones, rewards, advantages, bounds]), "All tensors must be on GPU" @@ -116,7 +120,6 @@ void compute_p3o(torch::Tensor reward_block, torch::Tensor reward_mask, threads_per_block = 256 assert num_steps % threads_per_block == 0 blocks = (num_steps + threads_per_block - 1) // threads_per_block - */ // Launch the kernel int threads_per_block = 256; @@ -226,6 +229,53 @@ void compute_vtrace(torch::Tensor values, torch::Tensor rewards, throw std::runtime_error(cudaGetErrorString(err)); } } +*/ + +static const int max_horizon = 256; +__host__ __device__ void puff_advantage_row_cuda(float* values, float* rewards, float* dones, + float* importance, float* vs, float* advantages, float gamma, float lambda, + float rho_clip, float c_clip, int horizon) { + vs[horizon-1] = values[horizon-1]; + float lastpufferlam = 0; + for (int t = horizon-2; t >= 0; t--) { + int t_next = t + 1; + float nextnonterminal = 1.0 - dones[t_next]; + float rho_t = fminf(importance[t], rho_clip); + float c_t = fminf(importance[t], c_clip); + // TODO: t_next works and t doesn't. Check original formula + float delta = rho_t*(rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]); + lastpufferlam = delta + gamma*lambda*c_t*lastpufferlam*nextnonterminal; + + //float delta = rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]; + //lastpufferlam = delta + gamma*lambda*lastpufferlam*nextnonterminal; + + + advantages[t] = lastpufferlam; + vs[t] = advantages[t] + values[t]; + //advantages[t] = rho_t*(rewards[t] + gamma*vs[t_next]*nextnonterminal - values[t]); + //vs[t] = lastpufferlam + values[t]; + } +} + +void vtrace_check_cuda(torch::Tensor values, torch::Tensor rewards, + torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + int num_steps, int horizon) { + + // Validate input tensors + torch::Device device = values.device(); + for (const torch::Tensor& t : {values, rewards, dones, importance, vs, advantages}) { + TORCH_CHECK(t.dim() == 2, "Tensor must be 2D"); + TORCH_CHECK(t.device() == device, "All tensors must be on same device"); + TORCH_CHECK(t.size(0) == num_steps, "First dimension must match num_steps"); + TORCH_CHECK(t.size(1) == horizon, "Second dimension must match horizon"); + TORCH_CHECK(t.dtype() == torch::kFloat32, "All tensors must be float32"); + assert(horizon <= max_horizon); + if (!t.is_contiguous()) { + t.contiguous(); + } + } +} + // [num_steps, horizon] __global__ void puff_advantage_kernel(float* values, float* rewards, float* dones, float* importance, @@ -233,16 +283,16 @@ __global__ void puff_advantage_kernel(float* values, float* rewards, float* done float rho_clip, float c_clip, int num_steps, int horizon) { int row = blockIdx.x*blockDim.x + threadIdx.x; int offset = row*horizon; - puff_advantage_row(values + offset, rewards + offset, dones + offset, + puff_advantage_row_cuda(values + offset, rewards + offset, dones + offset, importance + offset, vs + offset, advantages + offset, gamma, lambda, rho_clip, c_clip, horizon); } -void compute_puff_advantage(torch::Tensor values, torch::Tensor rewards, +void compute_puff_advantage_cuda(torch::Tensor values, torch::Tensor rewards, torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, - float gamma, float lambda, float rho_clip, float c_clip) { + double gamma, double lambda, double rho_clip, double c_clip) { int num_steps = values.size(0); int horizon = values.size(1); - vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); + vtrace_check_cuda(values, rewards, dones, importance, vs, advantages, num_steps, horizon); TORCH_CHECK(values.is_cuda(), "All tensors must be on GPU"); assert(horizon <= max_horizon); @@ -274,11 +324,8 @@ void compute_puff_advantage(torch::Tensor values, torch::Tensor rewards, } } +TORCH_LIBRARY_IMPL(pufferlib, CUDA, m) { + m.impl("compute_puff_advantage", &compute_puff_advantage_cuda); +} -// Pybind11 module definition -PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { - m.def("compute_p3o", &compute_p3o, "Compute p3o advantages with CUDA"); - m.def("compute_gae", &compute_gae, "Compute GAE with CUDA"); - m.def("compute_vtrace", &compute_vtrace, "Compute VTrace with CUDA"); - m.def("compute_puff_advantage", &compute_puff_advantage, "Compute PuffAdvantage with CUDA"); } diff --git a/shared.cpp b/pufferlib/shared.cpp similarity index 99% rename from shared.cpp rename to pufferlib/shared.cpp index 1216564779..791215b0af 100644 --- a/shared.cpp +++ b/pufferlib/shared.cpp @@ -6,7 +6,9 @@ #define __device__ #endif +const int max_horizon = 256; // [horizon] +/* __host__ __device__ void gae_row(float* values, float* rewards, float* dones, float* advantages, float gamma, float gae_lambda, int horizon) { float lastgaelam = 0; @@ -46,7 +48,6 @@ torch::Tensor gae_check(torch::Tensor values, torch::Tensor rewards, } // [horizon] -const int max_horizon = 256; __host__ __device__ void vtrace_row(float* values, float* rewards, float* dones, float* importance, float* vs, float* advantages, float gamma, float rho_clip, float c_clip, int horizon) { float accum = 0.0;//values[horizon-1]; // Is this correct? @@ -62,6 +63,7 @@ __host__ __device__ void vtrace_row(float* values, float* rewards, float* dones, vs[t] = accum + values[t]; } } +*/ __host__ __device__ void puff_advantage_row(float* values, float* rewards, float* dones, float* importance, float* vs, float* advantages, float gamma, float lambda, diff --git a/setup.py b/setup.py index d7db6d52a6..d69ba88b49 100644 --- a/setup.py +++ b/setup.py @@ -6,9 +6,54 @@ import zipfile import tarfile import platform + +from setuptools.command.build_ext import build_ext as DefaultBuildExt # python3 setup.py built_ext --inplace +class CustomBuildExt(DefaultBuildExt): + def run(self): + # Split extensions into PyTorch and non-PyTorch + pytorch_extensions = [ext for ext in self.extensions if ext.name == 'pufferlib._C'] + other_extensions = [ext for ext in self.extensions if ext.name != 'pufferlib._C'] + + # Build PyTorch extensions with cpp_extension.BuildExtension + if pytorch_extensions: + pytorch_build_ext = cpp_extension.BuildExtension(self.distribution) + # Temporarily set extensions to only PyTorch ones + original_extensions = self.extensions + self.extensions = pytorch_extensions + pytorch_build_ext.run() + self.extensions = original_extensions # Restore original extensions + + # Build other extensions with default setuptools build_ext + if other_extensions: + self.extensions = other_extensions + super().run() + +''' + + cythonize( + [ + "pufferlib/extensions.pyx", + "c_advantage.pyx", + "pufferlib/puffernet.pyx", + *extensions, + ], + compiler_directives={ + 'language_level': 3, + 'boundscheck': False, + 'initializedcheck': False, + 'wraparound': False, + 'cdivision': True, + 'nonecheck': False, + 'profile': False, + }, + #nthreads=6, + #annotate=True, + #compiler_directives={'profile': True},# annotate=True + ), +''' + VERSION = '2.0.6' RAYLIB_BASE = 'https://github.com/raysan5/raylib/releases/download/5.5/' @@ -317,7 +362,7 @@ pure_c_extensions = ['squared', 'pong', 'breakout', 'enduro', 'blastar', 'grid', 'nmmo3', 'tactical', 'go', 'cartpole'] -extensions += [ +c_extensions = [ Extension( f'pufferlib.ocean.{name}.binding', sources=[f'pufferlib/ocean/{name}/binding.c'], @@ -329,6 +374,23 @@ for name in pure_c_extensions ] +from torch.utils import cpp_extension +torch_extensions = [ + cpp_extension.CUDAExtension( + "pufferlib._C", + ["pufferlib.cpp", "pufferlib/pufferlib.cu"], + extra_compile_args = { + "cxx": [ + "-fdiagnostics-color=always", + "-DPy_LIMITED_API=0x03090000", # min CPython version 3.9 + ], + "nvcc": [ + ], + }, + py_limited_api=True, + ), +] + # Prevent Conda from injecting garbage compile flags from distutils.sysconfig import get_config_vars cfg_vars = get_config_vars() @@ -341,7 +403,7 @@ for key, value in cfg_vars.items(): if value and '-fno-strict-overflow' in str(value): cfg_vars[key] = value.replace('-fno-strict-overflow', '') - + setup( name="pufferlib", description="PufferAI Library" @@ -375,25 +437,8 @@ 'common': common, **environments, }, - ext_modules = cythonize([ - "pufferlib/extensions.pyx", - "c_advantage.pyx", - "pufferlib/puffernet.pyx", - *extensions, - ], - compiler_directives={ - 'language_level': 3, - 'boundscheck': False, - 'initializedcheck': False, - 'wraparound': False, - 'cdivision': True, - 'nonecheck': False, - 'profile': False, - }, - #nthreads=6, - #annotate=True, - #compiler_directives={'profile': True},# annotate=True - ), + ext_modules = torch_extensions, + cmdclass={"build_ext": cpp_extension.BuildExtension}, include_dirs=[numpy.get_include(), RAYLIB_NAME + '/include'], python_requires=">=3.9", license="MIT", @@ -410,6 +455,7 @@ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", ], + options={"bdist_wheel": {"py_limited_api": "cp39"}}, ) #stable_baselines3 #supersuit==3.3.5 From 94b98a6f887512861e3210cadf0421419ebc1526 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 21:29:08 +0000 Subject: [PATCH 06/63] Cleaned create function --- clean_pufferl.py | 206 +++++++++++++++++++-------------------- config/default.ini | 2 +- pufferlib/environment.py | 2 + 3 files changed, 103 insertions(+), 107 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index a100d68e77..5bb2537375 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -45,74 +45,57 @@ class CleanPuffeRL: def __init__(self, config, vecenv, policy): - self.config = config - self.vecenv = vecenv - - self.global_step = 0 - self.epoch = 0 - self.stats = defaultdict(list) - self.last_log_time = 0 - - random.seed(config.seed) - np.random.seed(config.seed) + # Backend perf optimization + torch.set_float32_matmul_precision('high') # TODO: Check if this is what was messing up AMP torch.backends.cudnn.deterministic = config.torch_deterministic torch.backends.cudnn.benchmark = True - torch.set_float32_matmul_precision('high') - if config.seed is not None: - torch.manual_seed(config.seed) - - self.losses = pufferlib.namespace( - policy_loss=0, - value_loss=0, - entropy=0, - old_approx_kl=0, - approx_kl=0, - clipfrac=0, - explained_variance=0, - diayn_loss=0, - grad_var=0, - importance=0, - ) - self.utilization = Utilization() - num_params = sum(p.numel() for p in policy.parameters() if p.requires_grad) - self.msg = f'Model Size: {abbreviate(num_params)} parameters' + # Reproducibility + random.seed(config.seed) + np.random.seed(config.seed) + torch.manual_seed(config.seed) + # Vecenv info vecenv.async_reset(config.seed) + obs_space = vecenv.single_observation_space + atn_space = vecenv.single_action_space total_agents = vecenv.num_agents - obs_shape = vecenv.single_observation_space.shape - atn_shape = vecenv.single_action_space.shape - obs_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_observation_space.dtype] - atn_dtype = pufferlib.pytorch.numpy_to_torch_dtype_dict[vecenv.single_action_space.dtype] - self.on_policy_rows = config.batch_size // config.bptt_horizon - self.off_policy_rows = int(config.replay_factor*config.batch_size // config.bptt_horizon) - experience_rows = self.on_policy_rows + self.off_policy_rows - pin = config.device == 'cuda' and config.cpu_offload - obs_device = config.device if not pin else 'cpu' - experience = pufferlib.namespace( - obs=torch.zeros(experience_rows, config.bptt_horizon, *obs_shape, - dtype=obs_dtype, pin_memory=pin, device='cpu' if pin else config.device), - actions=torch.zeros(experience_rows, config.bptt_horizon, *atn_shape, - dtype=atn_dtype, device=config.device), - logprobs=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - rewards=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - dones=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - truncateds=torch.zeros(experience_rows, config.bptt_horizon, device=config.device), - ratio = torch.ones(experience_rows, config.bptt_horizon, device=config.device), - ) - self.ep_uses = torch.zeros(experience_rows, device=config.device, dtype=torch.int32) - self.ep_lengths = torch.zeros(total_agents, device=config.device, dtype=torch.int32) - self.ep_indices = torch.arange(total_agents, device=config.device, dtype=torch.int32) - self.free_idx = total_agents - assert self.free_idx <= experience_rows, f'Total agents {total_agents} must be at least batch size {config.batch_size} / bptt_horizon {config.bptt_horizon} = {experience_rows}' self.total_agents = total_agents - self.diayn_skills = None + # Experience buffer + device = config.device + batch_size = config.batch_size + horizon = config.bptt_horizon + self.on_policy_segments = batch_size // horizon + self.off_policy_segments = int(config.replay_factor*batch_size // horizon) + segments = self.on_policy_segments + self.off_policy_segments + if total_agents < segments: + raise pufferlib.exceptions.APIUsageError( + f'Total agents {total_agents} must be greater than or equal to segments {segments}' + ) + + self.ep_uses = torch.zeros(segments, device=device, dtype=torch.int32) + self.ep_lengths = torch.zeros(total_agents, device=device, dtype=torch.int32) + self.ep_indices = torch.arange(total_agents, device=device, dtype=torch.int32) + self.free_idx = total_agents + experience = pufferlib.namespace( + obs=torch.zeros(segments, horizon, *obs_space.shape, + dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[obs_space.dtype], + pin_memory=device == 'cuda' and config.cpu_offload, + device='cpu' if config.cpu_offload else device), + actions=torch.zeros(segments, horizon, *atn_space.shape, + dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[atn_space.dtype], device=device), + logprobs=torch.zeros(segments, horizon, device=device), + rewards=torch.zeros(segments, horizon, device=device), + dones=torch.zeros(segments, horizon, device=device), + truncateds=torch.zeros(segments, horizon, device=device), + ratio = torch.ones(segments, horizon, device=device), + ) + self.experience = experience if config.use_diayn: self.diayn_skills = torch.randint( - 0, config.diayn_archive, (total_agents,), dtype=torch.long, device=config.device) - experience.diayn_batch = torch.zeros(experience_rows, config.bptt_horizon, - dtype=torch.long, device=config.device) + 0, config.diayn_archive, (total_agents,), dtype=torch.long, device=device) + experience.diayn_batch = torch.zeros(segments, horizon, dtype=torch.long, device=config.device) if config.use_p3o: batch_size = config.batch_size @@ -127,39 +110,37 @@ def __init__(self, config, vecenv, policy): experience.bounds = torch.zeros(batch_size, dtype=torch.int32, device=device) experience.vstd_max = 1.0 else: - experience.values = torch.zeros(experience_rows, config.bptt_horizon, device=config.device) + experience.values = torch.zeros(segments, horizon, device=device) if config.use_vtrace or config.use_puff_advantage: - experience.importance = torch.ones(experience_rows, config.bptt_horizon, device=config.device) - - self.experience = experience + experience.importance = torch.ones(segments, horizon, device=device) - lstm_h = None - lstm_c = None + # LSTM # TODO: This breaks compile if isinstance(policy, torch.nn.LSTM): - assert total_agents > 0 - if config.env_batch_size > 1: - shape = (total_agents, policy.hidden_size) - lstm_h = torch.zeros(shape).to(config.device) - lstm_c = torch.zeros(shape).to(config.device) - else: - # TODO: Doesn't exist in native envs - n = vecenv.agents_per_batch - shape = (n, policy.hidden_size) - lstm_h = {slice(i*n, (i+1)*n):torch.zeros(shape).to(config.device) for i in range(total_agents//n)} - lstm_c = {slice(i*n, (i+1)*n):torch.zeros(shape).to(config.device) for i in range(total_agents//n)} - - self.lstm_h = lstm_h - self.lstm_c = lstm_c + # TODO: Doesn't exist in native envs + # TODO: Replace slice with env idx or similar + n = vecenv.agents_per_batch + self.lstm_h = {slice(i*n, (i+1)*n): torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} + self.lstm_c = {slice(i*n, (i+1)*n): torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} + + + # Gradient accumulation + minibatch_size = config.minibatch_size + max_minibatch_size = config.max_minibatch_size + self.minibatch_size = min(minibatch_size, max_minibatch_size) + if minibatch_size % max_minibatch_size != 0: + raise pufferlib.exceptions.APIUsageError( + f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' + ) - self.minibatch_size = min(config.minibatch_size, config.max_minibatch_size) self.uncompiled_policy = policy if config.compile: policy = torch.compile(policy, mode=config.compile_mode, fullgraph=config.compile_fullgraph) self.policy = policy + # Optimizer if config.optimizer == 'adam': optimizer = torch.optim.Adam( policy.parameters(), @@ -176,7 +157,6 @@ def __init__(self, config, vecenv, policy): lr=config.learning_rate, betas=(config.adam_beta1, config.adam_beta2), eps=config.adam_eps, - ) elif config.optimizer == 'kron': from heavyball import ForeachPSGDKron @@ -193,6 +173,7 @@ def __init__(self, config, vecenv, policy): self.optimizer = optimizer + # Learning rate scheduler epochs = config.total_timesteps // config.batch_size self.total_epochs = epochs assert config.scheduler in ('linear', 'cosine') @@ -203,17 +184,12 @@ def __init__(self, config, vecenv, policy): self.scheduler = scheduler - amp_context = nullcontext() - scaler = None + # Automatic mixed precision if config.precision != 'float32': - amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) - scaler = torch.amp.GradScaler() - - self.scaler = scaler - - self.profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', - 'learn', 'train_copy', 'train_misc', 'custom'], frequency=5) + self.amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) + self.scaler = torch.amp.GradScaler() + # Logging if config.neptune: self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.run_tag) for k, v in pufferlib.utils.unroll_nested_dict(args): @@ -221,10 +197,28 @@ def __init__(self, config, vecenv, policy): elif config.wandb: self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.run_tag) + # Profiling + self.uptime = 0 + self.last_log_time = 0 self.start_time = time.time() - self.uptime=0 + self.utilization = Utilization() + self.profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', + 'learn', 'train_copy', 'train_misc', 'custom'], frequency=5) + + # Initializations + self.config = config + self.vecenv = vecenv + self.global_step = 0 + self.epoch = 0 + self.stats = defaultdict(list) + self.losses = defaultdict(float) + + # Dashboard + num_params = sum(p.numel() for p in policy.parameters() if p.requires_grad) + self.msg = f'Model Size: {abbreviate(num_params)} parameters' self.print_dashboard(clear=True) + def evaluate(self): profile = self.profile epoch = self.epoch @@ -238,7 +232,7 @@ def evaluate(self): lstm_c = self.lstm_c self.full_rows = 0 - while self.full_rows < self.on_policy_rows: + while self.full_rows < self.on_policy_segments: profile('env', epoch) o, r, d, t, info, env_id, mask = self.vecenv.recv() @@ -497,12 +491,12 @@ def train(self): experience.values[batch.idx] = newvalue profile('learn', epoch) - if self.scaler is not None: + if config.precision != 'float32': loss = self.scaler.scale(loss) loss.backward() - if self.scaler is not None: + if config.precision != 'float32': self.scaler.unscale_(self.optimizer) # TODO: Delete? @@ -515,7 +509,7 @@ def train(self): torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config.max_grad_norm) # TODO: Can remove scaler if only using bf16 - if self.scaler is None: + if config.precision == 'float32': self.optimizer.step() else: self.scaler.step(self.optimizer) @@ -524,17 +518,17 @@ def train(self): self.optimizer.zero_grad() profile('train_misc', epoch) - losses.policy_loss += pg_loss.item() / total_minibatches - losses.value_loss += v_loss.item() / total_minibatches - losses.entropy += entropy_loss.item() / total_minibatches - losses.old_approx_kl += old_approx_kl.item() / total_minibatches - losses.approx_kl += approx_kl.item() / total_minibatches - losses.clipfrac += clipfrac.item() / total_minibatches - losses.grad_var += grad_var.item() / total_minibatches - losses.importance += ratio.mean().item() / total_minibatches + losses['policy_loss'] += pg_loss.item() / total_minibatches + losses['value_loss'] += v_loss.item() / total_minibatches + losses['entropy'] += entropy_loss.item() / total_minibatches + losses['old_approx_kl'] += old_approx_kl.item() / total_minibatches + losses['approx_kl'] += approx_kl.item() / total_minibatches + losses['clipfrac'] += clipfrac.item() / total_minibatches + losses['grad_var'] += grad_var.item() / total_minibatches + losses['importance'] += ratio.mean().item() / total_minibatches if config.use_diayn: - losses.diayn_loss += diayn_loss.item() / total_minibatches + losses['diayn_loss'] += diayn_loss.item() / total_minibatches if config.target_kl is not None: if approx_kl > config.target_kl: @@ -550,11 +544,11 @@ def train(self): torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, experience.dones, experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) - exp = self.sample(advantages, self.off_policy_rows, method='random') + exp = self.sample(advantages, self.off_policy_segments, method='random') for k, v in experience.items(): - v[self.on_policy_rows:] = exp[k] + v[self.on_policy_segments:] = exp[k] - experience.ratio[:self.on_policy_rows] = 1 + experience.ratio[:self.on_policy_segments] = 1 if config.anneal_lr: self.scheduler.step() diff --git a/config/default.ini b/config/default.ini index f4bb3879d6..2f0178ec5a 100644 --- a/config/default.ini +++ b/config/default.ini @@ -18,7 +18,7 @@ run_tag = None neptune = False wandb = False -seed = 0 +seed = 42 torch_deterministic = True cpu_offload = False device = cuda diff --git a/pufferlib/environment.py b/pufferlib/environment.py index bce092fbc6..7eb7715bb5 100644 --- a/pufferlib/environment.py +++ b/pufferlib/environment.py @@ -39,6 +39,8 @@ def __init__(self, buf=None): raise APIUsageError(ERROR.format('single_action_space')) if not hasattr(self, 'num_agents'): raise APIUsageError(ERROR.format('num_agents')) + if self.num_agents < 1: + raise APIUsageError('num_agents must be >= 1') if hasattr(self, 'observation_space'): raise APIUsageError('PufferEnvs must define single_observation_space, not observation_space') From a19fd37f8524ecaef285a1912423b2b8675a783a Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 2 May 2025 23:54:56 +0000 Subject: [PATCH 07/63] more refactor --- clean_pufferl.py | 192 +++++++++++++++++++---------------------------- 1 file changed, 76 insertions(+), 116 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 5bb2537375..7eee059c66 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -125,7 +125,7 @@ def __init__(self, config, vecenv, policy): self.lstm_c = {slice(i*n, (i+1)*n): torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} - # Gradient accumulation + # Minibatching & gradient accumulation minibatch_size = config.minibatch_size max_minibatch_size = config.max_minibatch_size self.minibatch_size = min(minibatch_size, max_minibatch_size) @@ -134,6 +134,15 @@ def __init__(self, config, vecenv, policy): f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' ) + self.accumulate_minibatches = max(1, config.minibatch_size // config.max_minibatch_size) + self.total_minibatches = int(config.update_epochs * batch_size / self.minibatch_size) + self.minibatch_segments = self.minibatch_size // horizon + if self.minibatch_segments * horizon != self.minibatch_size: + raise pufferlib.exceptions.APIUsageError( + f'minibatch_size {self.minibatch_size} must be divisible by horizon {horizon}' + ) + + # Torch compile self.uncompiled_policy = policy if config.compile: policy = torch.compile(policy, mode=config.compile_mode, fullgraph=config.compile_fullgraph) @@ -199,7 +208,6 @@ def __init__(self, config, vecenv, policy): # Profiling self.uptime = 0 - self.last_log_time = 0 self.start_time = time.time() self.utilization = Utilization() self.profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', @@ -210,10 +218,10 @@ def __init__(self, config, vecenv, policy): self.vecenv = vecenv self.global_step = 0 self.epoch = 0 - self.stats = defaultdict(list) - self.losses = defaultdict(float) + self.stats = defaultdict(list) # TODO: can this be set in eval and handle accum differently? # Dashboard + self.losses = {} num_params = sum(p.numel() for p in policy.parameters() if p.requires_grad) self.msg = f'Model Size: {abbreviate(num_params)} parameters' self.print_dashboard(clear=True) @@ -224,12 +232,10 @@ def evaluate(self): epoch = self.epoch profile('eval', epoch) profile('eval_misc', epoch, nest=True) + config = self.config experience = self.experience policy = self.policy - infos = defaultdict(list) - lstm_h = self.lstm_h - lstm_c = self.lstm_c self.full_rows = 0 while self.full_rows < self.on_policy_segments: @@ -237,13 +243,10 @@ def evaluate(self): o, r, d, t, info, env_id, mask = self.vecenv.recv() profile('eval_misc', epoch) - # Zero-copy indexing for contiguous env_id - if config.env_batch_size == 1: - gpu_env_id = cpu_env_id = slice(env_id[0], env_id[-1] + 1) - else: - cpu_env_id = env_id - gpu_env_id = torch.as_tensor(env_id).to(config.device, non_blocking=True) + # TODO: Port to vecenv + env_id = slice(env_id[0], env_id[-1] + 1) + # TODO: Handle truncations done_mask = d + t self.global_step += mask.sum() @@ -253,23 +256,19 @@ def evaluate(self): r = torch.as_tensor(r).to(config.device, non_blocking=True) d = torch.as_tensor(d).to(config.device, non_blocking=True) - h = None - c = None - if lstm_h is not None: - h = lstm_h[gpu_env_id] - c = lstm_c[gpu_env_id] - profile('eval_forward', epoch) with torch.no_grad(): state = pufferlib.namespace( reward=r, done=d, - env_id=gpu_env_id, + env_id=env_id, mask=mask, - lstm_h=h, - lstm_c=c, ) + if isinstance(policy, torch.nn.LSTM): + state.lstm_h = self.lstm_h[env_id] + state.lstm_c = self.lstm_c[env_id] + if config.use_diayn: state.diayn_z = self.diayn_skills[env_id] @@ -279,59 +278,44 @@ def evaluate(self): profile('eval_copy', epoch) with torch.no_grad(): - if lstm_h is not None: - lstm_h[gpu_env_id] = state.lstm_h - lstm_c[gpu_env_id] = state.lstm_c + if isinstance(policy, torch.nn.LSTM): + self.lstm_h[env_id] = state.lstm_h + self.lstm_c[env_id] = state.lstm_c o = o if config.cpu_offload else o_device - actions = self.store(state, o, value, action, logprob, r, d, gpu_env_id, mask) + actions = self.store(state, o, value, action, logprob, r, d, env_id, mask) profile('eval_misc', epoch) for i in info: for k, v in pufferlib.utils.unroll_nested_dict(i): - infos[k].append(v) + if isinstance(v, np.ndarray): + v = v.tolist() + elif isinstance(v, (list, tuple)): + self.stats[k].extend(v) + else: + self.stats[k].append(v) profile('env', epoch) self.vecenv.send(actions) profile('eval_misc', epoch) - for k, v in infos.items(): - if '_map' in k: - if self.wandb is not None: - self.stats[f'Media/{k}'] = self.wandb.Image(v[0]) - continue - elif self.neptune is not None: - # TODO: Add neptune image logging - pass - - if isinstance(v, np.ndarray): - v = v.tolist() - try: - iter(v) - except TypeError: - self.stats[k].append(v) - else: - self.stats[k] += v - self.free_idx = self.total_agents self.ep_indices = torch.arange(self.total_agents, device=config.device, dtype=torch.int32) self.ep_lengths.zero_() self.ep_uses.zero_() profile.end() - return self.stats, infos + return self.stats def train(self): profile = self.profile epoch = self.epoch profile('train', epoch) + config = self.config experience = self.experience - losses = self.losses + losses = defaultdict(float) - total_minibatches = int(config.update_epochs*config.batch_size/self.minibatch_size) - accumulate_minibatches = max(1, config.minibatch_size // config.max_minibatch_size) - n_samples = self.minibatch_size // config.bptt_horizon - for mb in range(total_minibatches): + for mb in range(self.total_minibatches): profile('train_misc', epoch, nest=True) loss = 0 if config.use_p3o: @@ -374,16 +358,17 @@ def train(self): elif config.use_puff_advantage: importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) vs = torch.zeros(experience.values.shape, device=config.device) - torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, experience.dones, - experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) + torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, + experience.dones, experience.ratio, vs, advantages, config.gamma, + config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) else: importance = advantages = self.compute_gae(experience.values, experience.rewards, experience.dones, config.gamma, config.gae_lambda) profile('train_copy', epoch) - batch = self.sample(importance, n_samples) + batch = self.sample(importance, self.minibatch_segments) - profile('train_misc', epoch) + profile('train_forward', epoch) state = pufferlib.namespace( action=batch.actions, lstm_h=None, @@ -393,7 +378,6 @@ def train(self): if config.use_diayn: state.diayn_z = batch.diayn_z.reshape(-1) - profile('train_forward', epoch) if not isinstance(self.policy, torch.nn.LSTM): batch.obs = batch.obs.reshape(-1, *self.vecenv.single_observation_space.shape) @@ -420,7 +404,7 @@ def train(self): newlogprob = newlogprob.reshape(batch.logprobs.shape) logratio = newlogprob - batch.logprobs ratio = logratio.exp() - experience.ratio[batch.idx] = ratio + experience.ratio[batch.idx] = ratio # TODO: Experiment with this # TODO: Only do this if we are KL clipping? Saves 1-2% compute with torch.no_grad(): @@ -429,6 +413,7 @@ def train(self): approx_kl = ((ratio - 1) - logratio).mean() clipfrac = ((ratio - 1.0).abs() > config.clip_coef).float().mean() + # TODO: Do you need to do this? Policy hasn't changed if config.use_vtrace or config.use_puff_advantage: with torch.no_grad(): adv = advantages[batch.idx] @@ -440,13 +425,11 @@ def train(self): torch.ops.pufferlib.compute_puff_advantage(batch.values, batch.rewards, batch.dones, ratio, vs, adv, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) - #advantages[batch.idx] = adv - #importance[batch.idx] = adv - adv = batch.advantages if config.norm_adv: adv = (adv - adv.mean()) / (adv.std() + 1e-8) + # Prioritized replay adv = adv * batch.prio # Policy loss @@ -467,10 +450,10 @@ def train(self): mask_block = mask_block[:, :(horizon+3)] v_loss = v_loss[mask_block.bool()].mean() elif config.clip_vloss: - newvalue = newvalue#.flatten() - ret = batch.returns#.flatten() + newvalue = newvalue + ret = batch.returns v_loss_unclipped = (newvalue - ret) ** 2 - val = batch.values#.flatten() + val = batch.values v_clipped = val + torch.clamp( newvalue - val, -config.vf_clip_coef, @@ -499,13 +482,7 @@ def train(self): if config.precision != 'float32': self.scaler.unscale_(self.optimizer) - # TODO: Delete? - with torch.no_grad(): - grads = torch.cat([p.grad.flatten() for p in self.policy.parameters()]) - grad_var = grads.var(0).mean() * config.minibatch_size - self.msg = f'Gradient variance: {grad_var.item():.3f}' - - if (mb + 1) % accumulate_minibatches == 0: + if (mb + 1) % self.accumulate_minibatches == 0: torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config.max_grad_norm) # TODO: Can remove scaler if only using bf16 @@ -517,22 +494,21 @@ def train(self): self.optimizer.zero_grad() - profile('train_misc', epoch) - losses['policy_loss'] += pg_loss.item() / total_minibatches - losses['value_loss'] += v_loss.item() / total_minibatches - losses['entropy'] += entropy_loss.item() / total_minibatches - losses['old_approx_kl'] += old_approx_kl.item() / total_minibatches - losses['approx_kl'] += approx_kl.item() / total_minibatches - losses['clipfrac'] += clipfrac.item() / total_minibatches - losses['grad_var'] += grad_var.item() / total_minibatches - losses['importance'] += ratio.mean().item() / total_minibatches + profile('train_misc', epoch) + losses['policy_loss'] += pg_loss.item() / self.total_minibatches + losses['value_loss'] += v_loss.item() / self.total_minibatches + losses['entropy'] += entropy_loss.item() / self.total_minibatches + losses['old_approx_kl'] += old_approx_kl.item() / self.total_minibatches + losses['approx_kl'] += approx_kl.item() / self.total_minibatches + losses['clipfrac'] += clipfrac.item() / self.total_minibatches + losses['importance'] += ratio.mean().item() / self.total_minibatches - if config.use_diayn: - losses['diayn_loss'] += diayn_loss.item() / total_minibatches + if config.use_diayn: + losses['diayn_loss'] += diayn_loss.item() / self.total_minibatches - if config.target_kl is not None: - if approx_kl > config.target_kl: - break + if config.target_kl is not None: + if approx_kl > config.target_kl: + break # Reprioritize experience profile('train_misc', epoch) @@ -564,7 +540,7 @@ def train(self): var_y = y_true.var() explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y - #losses.explained_variance = explained_var.item() + losses['explained_variance'] = explained_var.item() profile.end() profile.clear() @@ -574,12 +550,10 @@ def train(self): if done_training or self.global_step == 0 or time.time() - self.start_time - self.uptime > 1: self.uptime = time.time() - self.start_time logs = self.mean_and_log() + self.losses = losses self.print_dashboard() self.stats = defaultdict(list) - for k in losses: - losses[k] = 0 - if self.epoch % config.checkpoint_interval == 0 or done_training: self.save_checkpoint() self.msg = f'Checkpoint saved at update {self.epoch}' @@ -591,12 +565,8 @@ def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): exp = self.experience # Fast path for fully vectorized envs - if self.config.env_batch_size == 1: - l = self.ep_lengths[env_id.start].item() - batch_rows = slice(self.ep_indices[env_id.start].item(), 1+self.ep_indices[env_id.stop - 1].item()) - else: - l = self.ep_lengths[env_id] - batch_rows = self.ep_indices[env_id] + l = self.ep_lengths[env_id.start].item() + batch_rows = slice(self.ep_indices[env_id.start].item(), 1+self.ep_indices[env_id.stop - 1].item()) exp.obs[batch_rows, l] = obs exp.actions[batch_rows, l] = action @@ -609,7 +579,6 @@ def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): exp.values_std[batch_rows, l] = value.std else: exp.values[batch_rows, l] = value.flatten() - #exp.values[l, batch_rows] = value.flatten() if config.use_diayn: exp.diayn_batch[batch_rows, l] = state.diayn_z @@ -618,22 +587,12 @@ def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): #indices = np.where(mask)[0] #data.ep_lengths[env_id[mask]] += 1 self.ep_lengths[env_id] += 1 - if config.env_batch_size == 1: - if l+1 >= config.bptt_horizon: - num_full = env_id.stop - env_id.start - self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config.device).int() - self.ep_lengths[env_id] = 0 - self.free_idx += num_full - self.full_rows += num_full - else: - full = self.ep_lengths[env_id] >= config.bptt_horizon - num_full = full.sum() - if num_full > 0: - full_ids = env_id[full] - self.ep_indices[full_ids] = self.free_idx + torch.arange(num_full, device=config.device).int() - self.ep_lengths[full_ids] = 0 - self.free_idx += num_full - self.full_rows += num_full + if l+1 >= config.bptt_horizon: + num_full = env_id.stop - env_id.start + self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config.device).int() + self.ep_lengths[env_id] = 0 + self.free_idx += num_full + self.full_rows += num_full return action.cpu().numpy() @@ -654,7 +613,6 @@ def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio else: raise ValueError(f'Unknown sampling method: {method}') - self.ep_uses[idx] += 1 output = {k: v[idx] for k, v in exp.items()} output['idx'] = idx @@ -702,17 +660,15 @@ def mean_and_log(self): 'mean_uses': self.mean_uses, **{f'environment/{k}': dist_mean(v, device) for k, v in self.stats.items()}, **{f'losses/{k}': dist_mean(v, device) for k, v in self.losses.items()}, - #**{f'performance/{k}': dist_sum(v, device) for k, v in self.profile}, + **{f'performance/{k}': dist_sum(v.elapsed, device) for k, v in self.profile}, } if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: return logs if config.wandb: - self.last_log_time = time.time() self.wandb.log(logs) elif config.neptune: - self.last_log_time = time.time() for k, v in logs.items(): self.neptune[k].append(v, step=agent_steps) @@ -730,6 +686,7 @@ def close(self): self.wandb.run.log_artifact(artifact) self.wandb.finish() elif config.neptune: + # TODO: Add artifact self.neptune.stop() def save_checkpoint(self): @@ -983,6 +940,9 @@ def __init__(self, keys, frequency=1): ) for k in keys } + def __iter__(self): + return iter(self.profiles.items()) + def __getattr__(self, name): return self.profiles[name] @@ -1230,7 +1190,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin cost = time.time() - pufferl.start_time batch_size = args['train']['batch_size'] while len(pufferl.stats[target_metric]) < min_eval_points: - stats, _ = pufferl.evaluate() + stats = pufferl.evaluate() steps_evaluated += batch_size pufferl.mean_and_log() From e5449b9a526180c136c6fdc038f81c3feb977f79 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 3 May 2025 00:02:54 +0000 Subject: [PATCH 08/63] few small cleanups --- clean_pufferl.py | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 7eee059c66..fcbc25b12f 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -881,24 +881,19 @@ def rollout(env_creator, env_kwargs, policy_cls, rnn_cls, agent_creator, agent_k ) num_agents = env.observation_space.shape[0] - if hasattr(agent, 'recurrent'): + if isinstance(agent, torch.nn.LSTM): shape = (num_agents, agent.hidden_size) state.lstm_h = torch.zeros(shape).to(device) state.lstm_c = torch.zeros(shape).to(device) frames = [] tick = 0 - value = [0] - intrinsic = [0] - intrinsic_mean = None - intrinsic_std = None while tick <= 200000: if tick > 1000 and tick % 1 == 0: - #render = driver.render(overlay=float(intrinsic[0])) render = driver.render() if driver.render_mode == 'ansi': print('\033[0;0H' + render + '\n') - time.sleep(0.05) + time.sleep(1/20) elif driver.render_mode == 'rgb_array': frames.append(render) import cv2 @@ -921,6 +916,7 @@ def rollout(env_creator, env_kwargs, policy_cls, rnn_cls, agent_creator, agent_k print(f'Reward: {reward:.4f}, Tick: {tick}') tick += 1 + # TODO: Frames from raylib # Save frames as gif if frames: import imageio @@ -1054,14 +1050,14 @@ def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): try: workspace = args['workspace'] run = neptune.init_run( - project=f"{workspace['name']}/{workspace['project']}", - capture_hardware_metrics=False, - capture_stdout=False, - capture_stderr=False, - capture_traceback=False, - tags=[tag] if tag is not None else [], - mode=mode, - ) + project=f"{workspace['name']}/{workspace['project']}", + capture_hardware_metrics=False, + capture_stdout=False, + capture_stderr=False, + capture_traceback=False, + tags=[tag] if tag is not None else [], + mode=mode, + ) except neptune.exceptions.NeptuneConnectionLostException: print("couldn't connect to neptune, logging in offline mode") return init_neptune(args, name, id, resume, tag, mode="offline") From e666bca71bcdc3cdb478677f742102bb3059b9c9 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 3 May 2025 12:59:56 +0000 Subject: [PATCH 09/63] cleanup --- clean_pufferl.py | 155 +++++++++------------------------------------ config/default.ini | 6 -- 2 files changed, 29 insertions(+), 132 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index fcbc25b12f..2b1c3896c6 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -66,9 +66,8 @@ def __init__(self, config, vecenv, policy): device = config.device batch_size = config.batch_size horizon = config.bptt_horizon - self.on_policy_segments = batch_size // horizon - self.off_policy_segments = int(config.replay_factor*batch_size // horizon) - segments = self.on_policy_segments + self.off_policy_segments + segments = batch_size // horizon + self.segments = segments if total_agents < segments: raise pufferlib.exceptions.APIUsageError( f'Total agents {total_agents} must be greater than or equal to segments {segments}' @@ -85,6 +84,7 @@ def __init__(self, config, vecenv, policy): device='cpu' if config.cpu_offload else device), actions=torch.zeros(segments, horizon, *atn_space.shape, dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[atn_space.dtype], device=device), + values = torch.zeros(segments, horizon, device=device), logprobs=torch.zeros(segments, horizon, device=device), rewards=torch.zeros(segments, horizon, device=device), dones=torch.zeros(segments, horizon, device=device), @@ -97,21 +97,6 @@ def __init__(self, config, vecenv, policy): 0, config.diayn_archive, (total_agents,), dtype=torch.long, device=device) experience.diayn_batch = torch.zeros(segments, horizon, dtype=torch.long, device=config.device) - if config.use_p3o: - batch_size = config.batch_size - p3o_horizon = config.p3o_horizon - device = config.device - experience.values_mean=torch.zeros(batch_size, p3o_horizon, device=device) - experience.values_std=torch.zeros(batch_size, p3o_horizon, device=device) - experience.reward_block = torch.zeros(batch_size, p3o_horizon, dtype=torch.float32, device=device) - experience.mask_block = torch.ones(batch_size, p3o_horizon, dtype=torch.float32, device=device) - experience.buf = torch.zeros(batch_size, p3o_horizon, dtype=torch.float32, device=device) - experience.advantages = torch.zeros(batch_size, dtype=torch.float32, device=device) - experience.bounds = torch.zeros(batch_size, dtype=torch.int32, device=device) - experience.vstd_max = 1.0 - else: - experience.values = torch.zeros(segments, horizon, device=device) - if config.use_vtrace or config.use_puff_advantage: experience.importance = torch.ones(segments, horizon, device=device) @@ -167,16 +152,6 @@ def __init__(self, config, vecenv, policy): betas=(config.adam_beta1, config.adam_beta2), eps=config.adam_eps, ) - elif config.optimizer == 'kron': - from heavyball import ForeachPSGDKron - import heavyball.utils - #heavyball.utils.compile_mode = "reduce-overhead" - optimizer = ForeachPSGDKron( - policy.parameters(), - lr=config.learning_rate, - precond_lr=config.precond_lr, - beta=config.adam_beta1, - ) else: raise ValueError(f'Unknown optimizer: {config.optimizer}') @@ -238,7 +213,7 @@ def evaluate(self): policy = self.policy self.full_rows = 0 - while self.full_rows < self.on_policy_segments: + while self.full_rows < self.segments: profile('env', epoch) o, r, d, t, info, env_id, mask = self.vecenv.recv() @@ -318,39 +293,7 @@ def train(self): for mb in range(self.total_minibatches): profile('train_misc', epoch, nest=True) loss = 0 - if config.use_p3o: - # Note: This function gets messed up by computing across - # episode bounds. Because we store experience in a flat buffer, - # bounds can be crossed even after handling dones. This prevent - # our method from scaling to longer horizons. TODO: Redo the way - # we store experience to avoid this issue - vstd_min = experience.values_std.min().item() - vstd_max = experience.values_std.max().item() - - self.mask_block.zero_() - self.buf.zero_() - self.reward_block.zero_() - self.bounds.zero_() - - r_mean = experience.rewards.mean().item() - r_std = experience.rewards.std().item() - - # TODO: Rename vstd to r_std - advantages = compute_advantages( - experience.reward_block, experience.mask_block, - experience.values_mean, experience.values_std, - experience.buf, experience.dones, experience.rewards, - experience.bounds, r_std, self.puf, config.p3o_horizon - ) - - horizon = torch.where(experience.values_std[0] > 0.95*r_std)[0] - horizon = horizon[0].item()+1 if len(horizon) else 1 - if horizon < 16: - horizon = 16 - - advantages = advantages.cpu().numpy() - torch.cuda.synchronize() - elif config.use_vtrace: + if config.use_vtrace: importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) vs = torch.zeros(experience.values.shape, device=config.device) self.compute_vtrace(experience.values, experience.rewards, experience.dones, @@ -426,8 +369,7 @@ def train(self): ratio, vs, adv, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) adv = batch.advantages - if config.norm_adv: - adv = (adv - adv.mean()) / (adv.std() + 1e-8) + adv = (adv - adv.mean()) / (adv.std() + 1e-8) # Prioritized replay adv = adv * batch.prio @@ -440,33 +382,23 @@ def train(self): pg_loss = torch.max(pg_loss1, pg_loss2).mean() # Value loss - if config.use_p3o: - newvalue_mean = newvalue.mean.view(-1, config.p3o_horizon) - newvalue_std = newvalue.std.view(-1, config.p3o_horizon) - newvalue_var = torch.square(newvalue_std) - criterion = torch.nn.GaussianNLLLoss(reduction='none') - v_loss = criterion(newvalue_mean, batch.reward_block, newvalue_var) - v_loss = v_loss[:, :(horizon+3)] - mask_block = mask_block[:, :(horizon+3)] - v_loss = v_loss[mask_block.bool()].mean() - elif config.clip_vloss: - newvalue = newvalue - ret = batch.returns - v_loss_unclipped = (newvalue - ret) ** 2 - val = batch.values - v_clipped = val + torch.clamp( - newvalue - val, - -config.vf_clip_coef, - config.vf_clip_coef, - ) - v_loss_clipped = (v_clipped - ret) ** 2 - v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) - v_loss = 0.5 * v_loss_max.mean() - else: - newvalue = newvalue.flatten() - v_loss = 0.5 * ((newvalue - ret) ** 2).mean() + newvalue = newvalue + ret = batch.returns + v_loss_unclipped = (newvalue - ret) ** 2 + val = batch.values + v_clipped = val + torch.clamp( + newvalue - val, + -config.vf_clip_coef, + config.vf_clip_coef, + ) + v_loss_clipped = (v_clipped - ret) ** 2 + v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) + v_loss = 0.5 * v_loss_max.mean() + # Entropy loss entropy_loss = entropy.mean() + + # Total loss loss += pg_loss - config.ent_coef*entropy_loss + v_loss*config.vf_coef # This breaks vloss clipping? @@ -514,29 +446,14 @@ def train(self): profile('train_misc', epoch) self.max_uses = self.ep_uses.max().item() self.mean_uses = self.ep_uses.float().mean().item() - if config.replay_factor > 0: - advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) - vs = torch.zeros(experience.values.shape, device=config.device) - torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, experience.dones, - experience.ratio, vs, advantages, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) - - exp = self.sample(advantages, self.off_policy_segments, method='random') - for k, v in experience.items(): - v[self.on_policy_segments:] = exp[k] - - experience.ratio[:self.on_policy_segments] = 1 + experience.ratio[:] = 1 if config.anneal_lr: self.scheduler.step() - if config.use_p3o: - y_pred = experience.values_mean - y_true = experience.reward_block - else: - y_pred = experience.values.flatten() - - # Probably not updated - y_true = advantages.flatten() + experience.values.flatten() + y_pred = experience.values.flatten() + # TODO: Probably not updated + y_true = advantages.flatten() + experience.values.flatten() var_y = y_true.var() explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y @@ -573,12 +490,7 @@ def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): exp.logprobs[batch_rows, l] = logprob exp.rewards[batch_rows, l] = reward exp.dones[batch_rows, l] = done.float() - - if config.use_p3o: - exp.values_mean[batch_rows, l] = value.mean - exp.values_std[batch_rows, l] = value.std - else: - exp.values[batch_rows, l] = value.flatten() + exp.values[batch_rows, l] = value.flatten() if config.use_diayn: exp.diayn_batch[batch_rows, l] = state.diayn_z @@ -616,16 +528,9 @@ def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio self.ep_uses[idx] += 1 output = {k: v[idx] for k, v in exp.items()} output['idx'] = idx - - if config.use_p3o: - output['reward_block'] = reward_block[idx] - output['mask_block'] = mask_block[idx] - output['values_mean'] = exp.values_mean[idx] - output['values_std'] = exp.values_std[idx] - else: - output['values'] = exp.values[idx] - output['advantages'] = advantages[idx] - output['returns'] = advantages[idx] + exp.values[idx] + output['values'] = exp.values[idx] + output['advantages'] = advantages[idx] + output['returns'] = advantages[idx] + exp.values[idx] if config.use_diayn: output['diayn_z'] = exp.diayn_batch[idx] @@ -1066,8 +971,6 @@ def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): def make_policy(env, policy_cls, rnn_cls, args): policy = policy_cls(env, **args['policy'], #batch_size=args['train']['batch_size'], - use_p3o=args['train']['use_p3o'], - p3o_horizon=args['train']['p3o_horizon'], use_diayn=args['train']['use_diayn'], diayn_skills=args['train']['diayn_archive'], ) diff --git a/config/default.ini b/config/default.ini index 2f0178ec5a..a11c185edc 100644 --- a/config/default.ini +++ b/config/default.ini @@ -31,10 +31,8 @@ learning_rate = 0.025 gamma = 0.995 gae_lambda = 0.85 update_epochs = 1 -norm_adv = True # Consider raising clip coef to 0.2 clip_coef = 0.1 -clip_vloss = True vf_coef = 2.0 vf_clip_coef = 0.1 max_grad_norm = 0.5 @@ -70,10 +68,6 @@ diayn_archive = 256 diayn_loss_coef = 0.000 diayn_coef = 0.0 -use_p3o = False -p3o_horizon = 128 -puf = 0.0 - use_vtrace = False vtrace_rho_clip = 1.0 vtrace_c_clip = 1.0 From 31638d61bc8cad62f303b1ae6332915573437dad Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 3 May 2025 17:59:28 +0000 Subject: [PATCH 10/63] compile, ddp, amp --- clean_pufferl.py | 136 +++++++++------------------------ config/default.ini | 13 +--- config/ocean/moba.ini | 1 + config/ocean/nmmo3.ini | 4 - pufferlib/models.py | 16 ++-- pufferlib/ocean/environment.py | 72 +++++++---------- pufferlib/ocean/torch.py | 16 +--- pufferlib/pytorch.py | 15 +--- 8 files changed, 84 insertions(+), 189 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 2b1c3896c6..1ee06e08c3 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -46,7 +46,7 @@ class CleanPuffeRL: def __init__(self, config, vecenv, policy): # Backend perf optimization - torch.set_float32_matmul_precision('high') # TODO: Check if this is what was messing up AMP + torch.set_float32_matmul_precision('high') torch.backends.cudnn.deterministic = config.torch_deterministic torch.backends.cudnn.benchmark = True @@ -68,9 +68,9 @@ def __init__(self, config, vecenv, policy): horizon = config.bptt_horizon segments = batch_size // horizon self.segments = segments - if total_agents < segments: + if total_agents > segments: raise pufferlib.exceptions.APIUsageError( - f'Total agents {total_agents} must be greater than or equal to segments {segments}' + f'Total agents {total_agents} <= segments {segments}' ) self.ep_uses = torch.zeros(segments, device=device, dtype=torch.int32) @@ -92,17 +92,13 @@ def __init__(self, config, vecenv, policy): ratio = torch.ones(segments, horizon, device=device), ) self.experience = experience - if config.use_diayn: - self.diayn_skills = torch.randint( - 0, config.diayn_archive, (total_agents,), dtype=torch.long, device=device) - experience.diayn_batch = torch.zeros(segments, horizon, dtype=torch.long, device=config.device) if config.use_vtrace or config.use_puff_advantage: experience.importance = torch.ones(segments, horizon, device=device) # LSTM # TODO: This breaks compile - if isinstance(policy, torch.nn.LSTM): + if config.use_rnn: # TODO: Doesn't exist in native envs # TODO: Replace slice with env idx or similar n = vecenv.agents_per_batch @@ -114,7 +110,7 @@ def __init__(self, config, vecenv, policy): minibatch_size = config.minibatch_size max_minibatch_size = config.max_minibatch_size self.minibatch_size = min(minibatch_size, max_minibatch_size) - if minibatch_size % max_minibatch_size != 0: + if max_minibatch_size % minibatch_size != 0: raise pufferlib.exceptions.APIUsageError( f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' ) @@ -145,7 +141,7 @@ def __init__(self, config, vecenv, policy): elif config.optimizer == 'muon': from heavyball import ForeachMuon import heavyball.utils - #heavyball.utils.compile_mode = "reduce-overhead" + heavyball.utils.compile_mode = config.compile_mode if config.compile else None optimizer = ForeachMuon( policy.parameters(), lr=config.learning_rate, @@ -169,9 +165,9 @@ def __init__(self, config, vecenv, policy): self.scheduler = scheduler # Automatic mixed precision - if config.precision != 'float32': - self.amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) - self.scaler = torch.amp.GradScaler() + self.amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) + if config.precision not in ('float32', 'bfloat16'): + raise pufferlib.exceptions.APIUsageError(f'Use float32 or bfloat16, not {config.precision}') # Logging if config.neptune: @@ -232,7 +228,7 @@ def evaluate(self): d = torch.as_tensor(d).to(config.device, non_blocking=True) profile('eval_forward', epoch) - with torch.no_grad(): + with torch.no_grad(), self.amp_context: state = pufferlib.namespace( reward=r, done=d, @@ -240,20 +236,17 @@ def evaluate(self): mask=mask, ) - if isinstance(policy, torch.nn.LSTM): + if config.use_rnn: state.lstm_h = self.lstm_h[env_id] state.lstm_c = self.lstm_c[env_id] - if config.use_diayn: - state.diayn_z = self.diayn_skills[env_id] - logits, value = policy(o_device, state) action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=policy.is_continuous) r = torch.clamp(r, -1, 1) profile('eval_copy', epoch) with torch.no_grad(): - if isinstance(policy, torch.nn.LSTM): + if config.use_rnn: self.lstm_h[env_id] = state.lstm_h self.lstm_c[env_id] = state.lstm_c @@ -292,6 +285,8 @@ def train(self): for mb in range(self.total_minibatches): profile('train_misc', epoch, nest=True) + self.amp_context.__enter__() + loss = 0 if config.use_vtrace: importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) @@ -318,10 +313,7 @@ def train(self): lstm_c=None, ) - if config.use_diayn: - state.diayn_z = batch.diayn_z.reshape(-1) - - if not isinstance(self.policy, torch.nn.LSTM): + if not config.use_rnn: batch.obs = batch.obs.reshape(-1, *self.vecenv.single_observation_space.shape) # TODO: Currently only returning traj shaped value as a hack @@ -330,20 +322,6 @@ def train(self): action=batch.actions, is_continuous=self.policy.is_continuous) profile('train_misc', epoch) - if config.use_diayn: - N = 1 - batch_logits = state.batch_logits[:, ::N] - batch_logits = torch.nn.functional.log_softmax(batch_logits, dim=-1) - mask = torch.nn.functional.one_hot(batch.actions[:, ::N], batch_logits.shape[-1]).bool() - #batch_logits = mask*batch_logits - batch_logits = batch_logits.view(batch_logits.shape[0], -1) - diayn_policy = self.policy.policy - q = diayn_policy.discrim_forward(batch_logits) - z_idxs = batch.diayn_z[:, 0] - q = q.view(-1, q.shape[-1]) - diayn_loss = torch.nn.functional.cross_entropy(q, z_idxs) - loss += config.diayn_loss_coef*diayn_loss - newlogprob = newlogprob.reshape(batch.logprobs.shape) logratio = newlogprob - batch.logprobs ratio = logratio.exp() @@ -382,8 +360,8 @@ def train(self): pg_loss = torch.max(pg_loss1, pg_loss2).mean() # Value loss - newvalue = newvalue ret = batch.returns + newvalue = newvalue.view(ret.shape) v_loss_unclipped = (newvalue - ret) ** 2 val = batch.values v_clipped = val + torch.clamp( @@ -400,30 +378,18 @@ def train(self): # Total loss loss += pg_loss - config.ent_coef*entropy_loss + v_loss*config.vf_coef + self.amp_context.__enter__() # This breaks vloss clipping? with torch.no_grad(): - experience.values[batch.idx] = newvalue + experience.values[batch.idx] = newvalue.float() profile('learn', epoch) - if config.precision != 'float32': - loss = self.scaler.scale(loss) - loss.backward() - if config.precision != 'float32': - self.scaler.unscale_(self.optimizer) - if (mb + 1) % self.accumulate_minibatches == 0: torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config.max_grad_norm) - - # TODO: Can remove scaler if only using bf16 - if config.precision == 'float32': - self.optimizer.step() - else: - self.scaler.step(self.optimizer) - self.scaler.update() - + self.optimizer.step() self.optimizer.zero_grad() profile('train_misc', epoch) @@ -435,13 +401,6 @@ def train(self): losses['clipfrac'] += clipfrac.item() / self.total_minibatches losses['importance'] += ratio.mean().item() / self.total_minibatches - if config.use_diayn: - losses['diayn_loss'] += diayn_loss.item() / self.total_minibatches - - if config.target_kl is not None: - if approx_kl > config.target_kl: - break - # Reprioritize experience profile('train_misc', epoch) self.max_uses = self.ep_uses.max().item() @@ -492,9 +451,6 @@ def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): exp.dones[batch_rows, l] = done.float() exp.values[batch_rows, l] = value.flatten() - if config.use_diayn: - exp.diayn_batch[batch_rows, l] = state.diayn_z - # TODO: Handle masks!! #indices = np.where(mask)[0] #data.ep_lengths[env_id[mask]] += 1 @@ -532,9 +488,6 @@ def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio output['advantages'] = advantages[idx] output['returns'] = advantages[idx] + exp.values[idx] - if config.use_diayn: - output['diayn_z'] = exp.diayn_batch[idx] - output['prio'] = 1 if method == 'prio': beta = config.prio_beta0 + (1 - config.prio_beta0)*config.prio_alpha*self.epoch/self.total_epochs @@ -782,7 +735,6 @@ def rollout(env_creator, env_kwargs, policy_cls, rnn_cls, agent_creator, agent_k state = pufferlib.namespace( lstm_h=None, lstm_c=None, - diayn_z=torch.arange(env.num_agents, dtype=torch.long, device=device) % 4 ) num_agents = env.observation_space.shape[0] @@ -969,11 +921,7 @@ def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): return run def make_policy(env, policy_cls, rnn_cls, args): - policy = policy_cls(env, **args['policy'], - #batch_size=args['train']['batch_size'], - use_diayn=args['train']['use_diayn'], - diayn_skills=args['train']['diayn_archive'], - ) + policy = policy_cls(env, **args['policy']) args['rnn']['input_size'] = policy.hidden_size args['rnn']['hidden_size'] = policy.hidden_size if rnn_cls is not None: @@ -1058,14 +1006,17 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin ) policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) + args['train']['use_rnn'] = rnn_cls is not None - if args['ddp']: + if 'LOCAL_RANK' in os.environ: from torch.nn.parallel import DistributedDataParallel as DDP orig_policy = policy - policy = DDP(policy, device_ids=[args['rank']]) - # TODO: Test this? isinstance? - if hasattr(orig_policy, 'lstm'): - policy.lstm = orig_policy.lstm + policy = DDP(policy, device_ids=[args['local_rank']]) + policy.hidden_size = orig_policy.hidden_size + policy.is_continuous = orig_policy.is_continuous + policy.forward_train = orig_policy.forward_train + if args['train']['use_rnn']: + policy.cell = orig_policy.cell train_config = pufferlib.namespace(**args['train'], env=env_name, exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) @@ -1113,12 +1064,9 @@ def downsample_linear(arr, m): pufferl.close() return scores, costs, timesteps, elos, vecenv -def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metric): - args['rank'] = rank - args['train']['device'] = f'cuda:{rank}' - torch.distributed.init_process_group(backend='nccl', rank=rank, world_size=world_size) - train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) - torch.distributed.destroy_process_group() +#python -m torch.distributed.run --standalone --nnodes=1 --nproc-per-node=1 clean_pufferl.py --env puffer_nmmo3 --mode train +#from torch.distributed.elastic.multiprocessing.errors import record +#@record if __name__ == '__main__': parser = argparse.ArgumentParser( @@ -1135,7 +1083,6 @@ def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metr help='Path to a pretrained checkpoint') parser.add_argument('--baseline', action='store_true', help='Load pretrained model from WandB if available') - parser.add_argument('--ddp', action='store_true', help='Distributed data parallel') parser.add_argument('--render-mode', type=str, default='auto', choices=['auto', 'human', 'ansi', 'rgb_array', 'raylib', 'None']) parser.add_argument('--exp-id', '--exp-name', type=str, @@ -1146,11 +1093,9 @@ def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metr parser.add_argument('--max-runs', type=int, default=200, help='Max number of sweep runs') parser.add_argument('--wandb-project', type=str, default='pufferlib') parser.add_argument('--wandb-group', type=str, default='debug') + parser.add_argument('--local-rank', type=int, default=0, help='Used by torchrun for DDP') + parser.add_argument('--tag', type=str, default=None, help='Tag for experiment') - parser.add_argument('--wandb', action='store_true', help='Track on WandB') - parser.add_argument('--neptune', action='store_true', help='Track on Neptune') - #parser.add_argument('--wandb-project', type=str, default='pufferlib') - #parser.add_argument('--wandb-group', type=str, default='debug') args = parser.parse_known_args()[0] file_paths = glob.glob('config/**/*.ini', recursive=True) @@ -1211,6 +1156,10 @@ def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metr if rnn_name is not None: rnn_cls = getattr(env_module.torch, args['rnn_name']) + # Assume TorchRun DDP is used if LOCAL_RANK is set + if 'LOCAL_RANK' in os.environ: + torch.distributed.init_process_group(backend='nccl', rank=0, world_size=1) + if args['baseline']: assert args['mode'] in ('train', 'eval', 'evaluate') args['track'] = True @@ -1225,17 +1174,6 @@ def train_ddp(rank, world_size, args, make_env, policy_cls, rnn_cls, target_metr data_dir = artifact.download() model_file = max(os.listdir(data_dir)) args['eval_model_path'] = os.path.join(data_dir, model_file) - if args['mode'] == 'train' and args['ddp']: - import torch.multiprocessing as mp - world_size = 1 - os.environ["MASTER_ADDR"] = "localhost" - os.environ["MASTER_PORT"] = "29500" - target_metric = args['sweep']['metric']['name'] - mp.spawn(train_ddp, - args=(world_size, args, make_env, policy_cls, rnn_cls, target_metric), - nprocs=world_size, - join=True, - ) elif args['mode'] == 'train': target_metric = args['sweep']['metric']['name'] train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) diff --git a/config/default.ini b/config/default.ini index a11c185edc..d8ac20cd0d 100644 --- a/config/default.ini +++ b/config/default.ini @@ -37,7 +37,6 @@ vf_coef = 2.0 vf_clip_coef = 0.1 max_grad_norm = 0.5 ent_coef = 0.01 -target_kl = None adam_beta1 = 0.9 adam_beta2 = 0.999 adam_eps = 1e-12 @@ -55,19 +54,9 @@ replay_factor = 0.0 max_minibatch_size = 32768 bptt_horizon = 64 compile = False -compile_mode = reduce-overhead +compile_mode = max-autotune-no-cudagraphs compile_fullgraph = True -use_e3b = False -e3b_coef = 0.01 -e3b_norm = 0.001 -e3b_lambda = 10.0 - -use_diayn = False -diayn_archive = 256 -diayn_loss_coef = 0.000 -diayn_coef = 0.0 - use_vtrace = False vtrace_rho_clip = 1.0 vtrace_c_clip = 1.0 diff --git a/config/ocean/moba.ini b/config/ocean/moba.ini index bb345d6282..98012328a1 100644 --- a/config/ocean/moba.ini +++ b/config/ocean/moba.ini @@ -18,6 +18,7 @@ num_envs = 8 num_workers = 4 env_batch_size = 4 minibatch_size = 20_480 +max_minibatch_size = 20_480 batch_size = 409_600 bptt_horizon = 80 learning_rate = 0.05 diff --git a/config/ocean/nmmo3.ini b/config/ocean/nmmo3.ini index 4ebbb77fcc..1de6ee1f63 100644 --- a/config/ocean/nmmo3.ini +++ b/config/ocean/nmmo3.ini @@ -17,10 +17,6 @@ num_envs = 4 total_timesteps = 107000000000 checkpoint_interval = 1000 learning_rate = 0.0004573146765703167 -num_envs = 2 -num_workers = 2 -env_batch_size = 1 -update_epochs = 1 gamma = 0.7647543366891623 gae_lambda = 0.996005622445478 ent_coef = 0.01210084358004069 diff --git a/pufferlib/models.py b/pufferlib/models.py index 067c37c6e5..7e71079d52 100644 --- a/pufferlib/models.py +++ b/pufferlib/models.py @@ -125,13 +125,13 @@ def decode_actions(self, hidden): return logits, values -class LSTMWrapper(nn.LSTM): +class LSTMWrapper(nn.Module): def __init__(self, env, policy, input_size=128, hidden_size=128): '''Wraps your policy with an LSTM without letting you shoot yourself in the foot with bad transpose and shape operations. This saves much pain. Requires that your policy define encode_observations and decode_actions. See the Default policy for an example.''' - super().__init__(input_size, hidden_size) + super().__init__() self.obs_shape = env.single_observation_space.shape self.policy = policy @@ -147,11 +147,13 @@ def __init__(self, env, policy, input_size=128, hidden_size=128): elif "weight" in name: nn.init.orthogonal_(param, 1.0) + self.lstm = nn.LSTM(input_size, hidden_size) + self.cell = torch.nn.LSTMCell(input_size, hidden_size) - self.cell.weight_ih = self.weight_ih_l0 - self.cell.weight_hh = self.weight_hh_l0 - self.cell.bias_ih = self.bias_ih_l0 - self.cell.bias_hh = self.bias_hh_l0 + self.cell.weight_ih = self.lstm.weight_ih_l0 + self.cell.weight_hh = self.lstm.weight_hh_l0 + self.cell.bias_ih = self.lstm.bias_ih_l0 + self.cell.bias_hh = self.lstm.bias_hh_l0 #self.pre_layernorm = nn.LayerNorm(hidden_size) #self.post_layernorm = nn.LayerNorm(hidden_size) @@ -210,7 +212,7 @@ def forward_train(self, observations, state): hidden = hidden.transpose(0, 1) #hidden = self.pre_layernorm(hidden) - hidden, (lstm_h, lstm_c) = super().forward(hidden, lstm_state) + hidden, (lstm_h, lstm_c) = self.lstm.forward(hidden, lstm_state) #hidden = self.post_layernorm(hidden) hidden = hidden.transpose(0, 1) diff --git a/pufferlib/ocean/environment.py b/pufferlib/ocean/environment.py index 34728c3087..d87dd4172b 100644 --- a/pufferlib/ocean/environment.py +++ b/pufferlib/ocean/environment.py @@ -1,3 +1,4 @@ +import importlib import pufferlib.emulation import pufferlib.postprocess @@ -116,51 +117,34 @@ def make_multiagent(buf=None, **kwargs): env = pufferlib.postprocess.MultiagentEpisodeStats(env) return pufferlib.emulation.PettingZooPufferEnv(env=env, buf=buf) -MAKE_FNS = { - 'breakout': lambda: lazy_import('pufferlib.ocean.breakout.breakout', 'Breakout'), - 'blastar': lambda: lazy_import('pufferlib.ocean.blastar.blastar', 'Blastar'), - 'pong': lambda: lazy_import('pufferlib.ocean.pong.pong', 'Pong'), - 'enduro': lambda: lazy_import('pufferlib.ocean.enduro.enduro', 'Enduro'), - 'cartpole': lambda: lazy_import('pufferlib.ocean.cartpole.cartpole', 'Cartpole'), - 'moba': lambda: lazy_import('pufferlib.ocean.moba.moba', 'Moba'), - 'nmmo3': lambda: lazy_import('pufferlib.ocean.nmmo3.nmmo3', 'NMMO3'), - 'snake': lambda: lazy_import('pufferlib.ocean.snake.snake', 'Snake'), - 'squared': lambda: lazy_import('pufferlib.ocean.squared.squared', 'Squared'), - 'pysquared': lambda: lazy_import('pufferlib.ocean.squared.pysquared', 'PySquared'), - 'connect4': lambda: lazy_import('pufferlib.ocean.connect4.connect4', 'Connect4'), - 'tripletriad': lambda: lazy_import('pufferlib.ocean.tripletriad.tripletriad', 'TripleTriad'), - 'tactical': lambda: lazy_import('pufferlib.ocean.tactical.tactical', 'Tactical'), - 'go': lambda: lazy_import('pufferlib.ocean.go.go', 'Go'), - 'rware': lambda: lazy_import('pufferlib.ocean.rware.rware', 'Rware'), - 'trash_pickup': lambda: lazy_import('pufferlib.ocean.trash_pickup.trash_pickup', 'TrashPickupEnv'), - 'tower_climb': lambda: lazy_import('pufferlib.ocean.tower_climb.tower_climb', 'TowerClimb'), - 'grid': lambda: lazy_import('pufferlib.ocean.grid.grid', 'Grid'), - 'cpr': lambda: lazy_import('pufferlib.ocean.cpr.cpr', 'PyCPR'), - 'impulse_wars': lambda: lazy_import('pufferlib.ocean.impulse_wars.impulse_wars', 'ImpulseWars'), - 'gpudrive': lambda: lazy_import('pufferlib.ocean.gpudrive.gpudrive', 'GPUDrive'), - #'rocket_lander': rocket_lander.RocketLander, - 'foraging': make_foraging, - 'predator_prey': make_predator_prey, - 'group': make_group, - 'puffer': make_puffer, - 'continuous': make_continuous, - 'bandit': make_bandit, - 'memory': make_memory, - 'password': make_password, - 'stochastic': make_stochastic, - 'multiagent': make_multiagent, - 'spaces': make_spaces, - 'performance': make_performance, - 'performance_empiric': make_performance_empiric, +MAKE_FUNCTIONS = { + 'breakout': 'Breakout', + 'blastar': 'Blastar', + 'pong': 'Pong', + 'enduro': 'Enduro', + 'cartpole': 'Cartpole', + 'moba': 'Moba', + 'nmmo3': 'NMMO3', + 'snake': 'Snake', + 'squared': 'Squared', + 'pysquared': 'PySquared', + 'connect4': 'Connect4', + 'tripletriad': 'TripleTriad', + 'tactical': 'Tactical', + 'go': 'Go', + 'rware': 'Rware', + 'trash_pickup': 'TrashPickupEnv', + 'tower_climb': 'TowerClimb', + 'grid': 'Grid', + 'cpr': 'PyCPR', + 'impulse_wars': 'ImpulseWars', + 'gpudrive': 'GPUDrive', } -# Alias puffer_ to all names -MAKE_FNS = {**MAKE_FNS, **{'puffer_' + k: v for k, v in MAKE_FNS.items()}} - def env_creator(name='squared', *args, **kwargs): - if name in MAKE_FNS: - return MAKE_FNS[name](*args, **kwargs) - else: - raise ValueError(f'Invalid environment name: {name}') - + if 'puffer_' not in name: + raise pufferlib.exceptions.APIUsageError(f'Invalid environment name: {name}') + name = name.replace('puffer_', '') + module = importlib.import_module(f'pufferlib.ocean.{name}.{name}') + return getattr(module, MAKE_FUNCTIONS[name]) diff --git a/pufferlib/ocean/torch.py b/pufferlib/ocean/torch.py index 7fe567af62..dc13ad9314 100644 --- a/pufferlib/ocean/torch.py +++ b/pufferlib/ocean/torch.py @@ -29,7 +29,8 @@ def __init__(self, env, hidden_size=512, output_size=512, **kwargs): #self.dtype = pufferlib.pytorch.nativize_dtype(env.emulated) self.num_actions = env.single_action_space.n self.factors = np.array([4, 4, 17, 5, 3, 5, 5, 5, 7, 4]) - self.offsets = torch.tensor([0] + list(np.cumsum(self.factors)[:-1])).cuda().view(1, -1, 1, 1) + offsets = torch.tensor([0] + list(np.cumsum(self.factors)[:-1])).view(1, -1, 1, 1) + self.register_buffer('offsets', offsets) self.cum_facs = np.cumsum(self.factors) self.multihot_dim = self.factors.sum() @@ -57,10 +58,6 @@ def __init__(self, env, hidden_size=512, output_size=512, **kwargs): nn.Linear(output_size, self.num_actions), std=0.01) self.value_fn = pufferlib.pytorch.layer_init(nn.Linear(output_size, 1), std=1) - # Pre-allocate allows compilation - map_buf = torch.zeros(32768, self.multihot_dim, 11, 15, dtype=torch.float32) - self.register_buffer('map_buf', map_buf) - def forward(self, x, state=None): hidden = self.encode_observations(x) actions, value = self.decode_actions(hidden) @@ -76,15 +73,14 @@ def encode_observations(self, observations, state=None): ob_reward = observations[:, -10:] batch = ob_map.shape[0] - map_buf = self.map_buf[:batch] - map_buf.zero_() + map_buf = torch.zeros(batch, 59, 11, 15, dtype=torch.float32, device=observations.device) codes = ob_map.permute(0, 3, 1, 2) + self.offsets map_buf.scatter_(1, codes, 1) ob_map = self.map_2d(map_buf) player_discrete = self.player_discrete_encoder(ob_player.int()) - obs = torch.cat([ob_map, player_discrete, ob_player.float(), ob_reward], dim=1) + obs = torch.cat([ob_map, player_discrete, ob_player.to(ob_map.dtype), ob_reward], dim=1) obs = self.proj(obs) return obs @@ -333,10 +329,6 @@ def forward(self, observations, state=None): def encode_observations(self, observations, state=None): cnn_features = observations[:, :-26].view(-1, 11, 11, 4).long() - if cnn_features[:, :, :, 0].max() > 15: - print('Invalid map value:', cnn_features[:, :, :, 0].max()) - breakpoint() - exit(1) map_features = F.one_hot(cnn_features[:, :, :, 0], 16).permute(0, 3, 1, 2).float() extra_map_features = (cnn_features[:, :, :, -3:].float() / 255).permute(0, 3, 1, 2) cnn_features = torch.cat([map_features, extra_map_features], dim=1) diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index 4d4581cd70..17a7c014c3 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -100,10 +100,7 @@ def _nativize_dtype(sample_dtype: np.dtype, return subviews, dtype, shape, start_offset, all_delta -def nativize_tensor( - observation: torch.Tensor, - native_dtype: NativeDType, -) -> torch.Tensor | dict[str, torch.Tensor]: +def nativize_tensor(observation: torch.Tensor, native_dtype: NativeDType): return _nativize_tensor(observation, native_dtype) @@ -124,9 +121,7 @@ def compilable_cast(u8, dtype): return u8.view(dtype) # breaking cast -def _nativize_tensor( - observation: torch.Tensor, native_dtype: NativeDType -) -> torch.Tensor | dict[str, torch.Tensor]: +def _nativize_tensor(observation: torch.Tensor, native_dtype: NativeDType): if isinstance(native_dtype, tuple): dtype, shape, offset, delta = native_dtype torch._check_is_size(offset) @@ -157,13 +152,11 @@ def nativize_observation(observation, emulated): ) -def flattened_tensor_size(native_dtype: tuple[torch.dtype, tuple[int], int, int]): +def flattened_tensor_size(native_dtype): return _flattened_tensor_size(native_dtype) -def _flattened_tensor_size( - native_dtype: tuple[torch.dtype, tuple[int], int, int], -) -> int: +def _flattened_tensor_size(native_dtype): if isinstance(native_dtype, tuple): return np.prod(native_dtype[1]) # shape else: From 95e4fb11ec709a9709dbd1a0a1601454e0e838e7 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 3 May 2025 20:52:21 +0000 Subject: [PATCH 11/63] More refactor --- clean_pufferl.py | 194 +++++++++++++++----------------------- config/default.ini | 24 ++--- config/ocean/breakout.ini | 11 --- pufferlib/sweep.py | 16 ++-- pufferlib/vector.py | 6 ++ 5 files changed, 102 insertions(+), 149 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 1ee06e08c3..98d745cee9 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -1,3 +1,7 @@ +# TODO: Add information +# - Help menu +# - Docs link + import os import random import psutil @@ -44,7 +48,7 @@ class CleanPuffeRL: - def __init__(self, config, vecenv, policy): + def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Backend perf optimization torch.set_float32_matmul_precision('high') torch.backends.cudnn.deterministic = config.torch_deterministic @@ -170,11 +174,13 @@ def __init__(self, config, vecenv, policy): raise pufferlib.exceptions.APIUsageError(f'Use float32 or bfloat16, not {config.precision}') # Logging - if config.neptune: + self.neptune = neptune + self.wandb = wandb + if neptune: self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.run_tag) for k, v in pufferlib.utils.unroll_nested_dict(args): self.neptune[k].append(v) - elif config.wandb: + elif wandb: self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.run_tag) # Profiling @@ -524,9 +530,9 @@ def mean_and_log(self): if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: return logs - if config.wandb: + if self.wandb: self.wandb.log(logs) - elif config.neptune: + elif self.neptune: for k, v in logs.items(): self.neptune[k].append(v, step=agent_steps) @@ -536,14 +542,14 @@ def close(self): self.vecenv.close() self.utilization.stop() config = self.config - if config.wandb: + if self.wandb: artifact_name = f"{config.exp_id}_model" artifact = self.wandb.Artifact(artifact_name, type="model") model_path = self.save_checkpoint(self) artifact.add_file(model_path) self.wandb.run.log_artifact(artifact) self.wandb.finish() - elif config.neptune: + elif self.neptune: # TODO: Add artifact self.neptune.stop() @@ -905,9 +911,10 @@ def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): import neptune import neptune.exceptions try: - workspace = args['workspace'] + neptune_name = args['neptune_name'] + neptune_project = args['neptune_project'] run = neptune.init_run( - project=f"{workspace['name']}/{workspace['project']}", + project=f"{neptune_name}/{neptune_project}", capture_hardware_metrics=False, capture_stdout=False, capture_stderr=False, @@ -929,32 +936,24 @@ def make_policy(env, policy_cls, rnn_cls, args): return policy.to(args['train']['device']) +def downsample_linear(arr, m): + n = len(arr) + x_old = np.linspace(0, 1, n) # Original indices normalized + x_new = np.linspace(0, 1, m) # New indices normalized + return np.interp(x_new, x_old, arr) + def sweep(args, env_name, make_env, policy_cls, rnn_cls): - method = args['sweep']['method'] - if method == 'random': - sweep = pufferlib.sweep.Random(args['sweep']) - elif method == 'pareto_genetic': - sweep = pufferlib.sweep.ParetoGenetic(args['sweep']) - elif method == 'protein': - sweep = pufferlib.sweep.Protein( - args['sweep'], - resample_frequency=0, - num_random_samples=50, # Should be number of params - max_suggestion_cost=args['max_suggestion_cost'], - min_score = args['sweep']['metric']['min'], - max_score = args['sweep']['metric']['max'], - ) - elif method == 'carbs': - sweep = pufferlib.sweep.Carbs( - args['sweep'], - resample_frequency=5, - num_random_samples=10, # Should be number of params - max_suggestion_cost=args['max_suggestion_cost'], - ) - else: - raise ValueError(f'Invalid sweep method {method} (random/pareto_genetic/protein)') + if not args['wandb'] and not args['neptune']: + raise pufferlib.exceptions.APIUsageError('Sweeps require either wandb or neptune') - target_metric = args['sweep']['metric']['name'] + method = args['sweep'].pop('method') + try: + sweep_cls = getattr(pufferlib.sweep, method) + except: + raise pufferlib.exceptions.APIUsageError(f'Invalid sweep method {method}. See pufferlib.sweep') + + sweep = sweep_cls(args['sweep']) + target_metric = args['sweep']['metric'] for i in range(args['max_runs']): seed = time.time_ns() & 0xFFFFFFFF random.seed(seed) @@ -966,7 +965,10 @@ def sweep(args, env_name, make_env, policy_cls, rnn_cls): sweep.observe(args, 0.0, 0.0) continue - scores, costs, timesteps, _, _ = train(args, make_env, policy_cls, rnn_cls, target_metric) + scores, costs, timesteps = train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) + scores = downsample_linear(scores, 10) + costs = downsample_linear(costs, 10) + timesteps = downsample_linear(timesteps, 10) # Hacky patch to prevent increasing total_timesteps when not swept total_timesteps = args['train']['total_timesteps'] @@ -978,33 +980,8 @@ def sweep(args, env_name, make_env, policy_cls, rnn_cls): print('Score:', score, 'Cost:', cost, 'Timesteps:', timestep) -def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_points=100, - elos={'model_random.pt': 1000}, vecenv=None, wandb=None, neptune=None): - if args['vec'] == 'serial': - vec = pufferlib.vector.Serial - elif args['vec'] == 'multiprocessing': - vec = pufferlib.vector.Multiprocessing - elif args['vec'] == 'ray': - vec = pufferlib.vector.Ray - elif args['vec'] == 'native': - vec = pufferlib.environment.PufferEnv - else: - raise ValueError(f'Invalid --vec (serial/multiprocessing/ray/native).') - - env_name = args['env_name'] - if vecenv is None: - vecenv = pufferlib.vector.make( - make_env, - env_kwargs=args['env'], - num_envs=args['train']['num_envs'], - num_workers=args['train']['num_workers'], - batch_size=args['train']['env_batch_size'], - zero_copy=args['train']['zero_copy'], - overwork=args['vec_overwork'], - seed=args['train']['seed'], - backend=vec, - ) - +def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_points=100, wandb=None, neptune=None): + vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) args['train']['use_rnn'] = rnn_cls is not None @@ -1018,9 +995,10 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin if args['train']['use_rnn']: policy.cell = orig_policy.cell + env_name = args['env_name'] train_config = pufferlib.namespace(**args['train'], env=env_name, exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) - pufferl = CleanPuffeRL(train_config, vecenv, policy) + pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) timesteps = [] scores = [] @@ -1051,18 +1029,8 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin costs.append(cost) timesteps.append(pufferl.global_step) - def downsample_linear(arr, m): - n = len(arr) - x_old = np.linspace(0, 1, n) # Original indices normalized - x_new = np.linspace(0, 1, m) # New indices normalized - return np.interp(x_new, x_old, arr) - - scores = downsample_linear(scores, 10) - costs = downsample_linear(costs, 10) - timesteps = downsample_linear(timesteps, 10) - pufferl.close() - return scores, costs, timesteps, elos, vecenv + return scores, costs, timesteps #python -m torch.distributed.run --standalone --nnodes=1 --nproc-per-node=1 clean_pufferl.py --env puffer_nmmo3 --mode train #from torch.distributed.elastic.multiprocessing.errors import record @@ -1077,8 +1045,6 @@ def downsample_linear(arr, m): default='puffer_squared', help='Name of specific environment to run') parser.add_argument('--mode', type=str, default='train', choices='train eval evaluate sweep autotune profile'.split()) - parser.add_argument('--vec-overwork', action='store_true', - help='Allow vectorization to use >1 worker/core. Not recommended.') parser.add_argument('--eval-model-path', type=str, default=None, help='Path to a pretrained checkpoint') parser.add_argument('--baseline', action='store_true', @@ -1087,58 +1053,52 @@ def downsample_linear(arr, m): choices=['auto', 'human', 'ansi', 'rgb_array', 'raylib', 'None']) parser.add_argument('--exp-id', '--exp-name', type=str, default=None, help='Resume from experiment') - parser.add_argument('--data-path', type=str, default=None, - help='Used for testing hparam algorithms') - parser.add_argument('--track', action='store_true', help='Track on WandB') parser.add_argument('--max-runs', type=int, default=200, help='Max number of sweep runs') + parser.add_argument('--wandb', action='store_true', help='Use wandb for logging') parser.add_argument('--wandb-project', type=str, default='pufferlib') parser.add_argument('--wandb-group', type=str, default='debug') + parser.add_argument('--neptune', action='store_true', help='Use neptune for logging') + parser.add_argument('--neptune-name', type=str, default='pufferai') + parser.add_argument('--neptune-project', type=str, default='ablations') parser.add_argument('--local-rank', type=int, default=0, help='Used by torchrun for DDP') - parser.add_argument('--tag', type=str, default=None, help='Tag for experiment') args = parser.parse_known_args()[0] - file_paths = glob.glob('config/**/*.ini', recursive=True) - for path in file_paths: + # Load defaults and config + for path in glob.glob('config/**/*.ini', recursive=True): p = configparser.ConfigParser() - p.read('config/default.ini') - - subconfig = os.path.join(*path.split('/')[:-1] + ['default.ini']) - if subconfig in file_paths: - p.read(subconfig) - - p.read(path) + p.read(['config/default.ini', path]) if args.env in p['base']['env_name'].split(): break else: - raise Exception('No config for env_name {}'.format(args.env)) + raise pufferlib.exceptions.APIUsageError('No config for env_name {}'.format(args.env)) + # Dynamic help menu from config for section in p.sections(): for key in p[section]: - if section == 'base': - argparse_key = f'--{key}'.replace('_', '-') - else: - argparse_key = f'--{section}.{key}'.replace('_', '-') - parser.add_argument(argparse_key, default=p[section][key]) + try: + value = ast.literal_eval(p[section][key]) + except: + value = p[section][key] + + fmt = f'--{key}' if section == 'base' else f'--{section}.{key}' + parser.add_argument(fmt.replace('_', '-'), default=value) - # Late add help so you get a dynamic menu based on the env parser.add_argument('-h', '--help', default=argparse.SUPPRESS, action='help', help='Show this help message and exit') - parsed = parser.parse_args().__dict__ - args = {'env': {}, 'policy': {}, 'rnn': {}} + # Unpack to nested dict + parsed = vars(parser.parse_args()) + nested = lambda: defaultdict(nested) # TODO: Replace with pufferlib namespace + args = nested() env_name = parsed.pop('env') for key, value in parsed.items(): next = args - for subkey in key.split('.'): - if subkey not in next: - next[subkey] = {} - prev = next + split = key.split('.') + for subkey in split[:-1]: next = next[subkey] - try: - prev[subkey] = ast.literal_eval(value) - except: - prev[subkey] = value + + next[split[-1]] = value package = args['package'] module_name = f'pufferlib.environments.{package}' @@ -1147,7 +1107,6 @@ def downsample_linear(arr, m): import importlib env_module = importlib.import_module(module_name) - make_env = env_module.env_creator(env_name) policy_cls = getattr(env_module.torch, args['policy_name']) @@ -1179,7 +1138,8 @@ def downsample_linear(arr, m): train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) elif args['mode'] in ('eval', 'evaluate'): vec = pufferlib.vector.Serial - if args['vec'] == 'native': vec = pufferlib.environment.PufferEnv + if args['vec'] == 'native': + vec = pufferlib.environment.PufferEnv rollout( make_env, args['env'], @@ -1193,17 +1153,17 @@ def downsample_linear(arr, m): device=args['train']['device'], ) elif args['mode'] == 'sweep': - assert args['wandb'] or args['neptune'], 'Sweeps require either wandb or neptune' sweep(args, env_name, make_env, policy_cls, rnn_cls) elif args['mode'] == 'autotune': pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) elif args['mode'] == 'profile': - import cProfile - target_metric = args['sweep']['metric']['name'] - cProfile.run('train(args, make_env, policy_cls, rnn_cls, target_metric)', 'stats.profile') - import pstats - from pstats import SortKey - p = pstats.Stats('stats.profile') - p.sort_stats(SortKey.TIME).print_stats(10) - breakpoint() - pass + import torch + import torchvision.models as models + from torch.profiler import profile, record_function, ProfilerActivity + with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True) as prof: + with record_function("model_inference"): + target_metric = args['sweep']['metric']['name'] + train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) + + print(prof.key_averages().table(sort_by='cuda_time_total', row_limit=10)) + prof.export_chrome_trace("trace.json") diff --git a/config/default.ini b/config/default.ini index d8ac20cd0d..5d7a662461 100644 --- a/config/default.ini +++ b/config/default.ini @@ -1,11 +1,18 @@ [base] package = None env_name = None -vec = native policy_name = Policy rnn_name = None max_suggestion_cost = 3600 +[vec] +backend = Multiprocessing +num_envs = 2 +num_workers = 2 +batch_size = 1 +zero_copy = True +seed = 42 + [env] [policy] [rnn] @@ -15,8 +22,6 @@ name = pufferai project = ablations run_id = None run_tag = None -neptune = False -wandb = False seed = 42 torch_deterministic = True @@ -41,10 +46,6 @@ adam_beta1 = 0.9 adam_beta2 = 0.999 adam_eps = 1e-12 -num_envs = 2 -num_workers = 2 -env_batch_size = 1 -zero_copy = True data_dir = experiments checkpoint_interval = 200 batch_size = 524288 @@ -67,14 +68,9 @@ prio_alpha = 0.6 prio_beta0 = 0.4 [sweep] -method = protein -name = sweep - -[sweep.metric] +method = Protein +metric = score goal = maximize -name = score -min = 0 -max = 1 [sweep.env.num_envs] distribution = uniform_pow2 diff --git a/config/ocean/breakout.ini b/config/ocean/breakout.ini index ebc0ac1660..6e96e52669 100644 --- a/config/ocean/breakout.ini +++ b/config/ocean/breakout.ini @@ -3,7 +3,6 @@ package = ocean env_name = puffer_breakout policy_name = Policy rnn_name = Recurrent -vec = multiprocessing [env] num_envs = 4096 @@ -20,16 +19,6 @@ total_timesteps = 80_000_000 learning_rate = 0.05 minibatch_size = 32768 -[sweep] -method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = score -min = 0 -max = 864 - #[sweep.train.total_timesteps] #distribution = log_normal #min = 2e7 diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index cac9b0ef61..c213eb1577 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -115,7 +115,7 @@ def unnormalize(self, value): def _params_from_puffer_sweep(sweep_config): param_spaces = {} for name, param in sweep_config.items(): - if name in ('method', 'name', 'metric', 'max_score'): + if name in ('method', 'metric', 'goal'): continue assert isinstance(param, dict) @@ -156,8 +156,9 @@ def __init__(self, config, verbose=True): self.num = len(self.flat_spaces) self.metric = config['metric'] - assert self.metric['goal'] in ['maximize', 'minimize'] - self.optimize_direction = 1 if self.metric['goal'] == 'maximize' else -1 + goal = config['goal'] + assert goal in ('maximize', 'minimize') + self.optimize_direction = 1 if 'goal' == 'maximize' else -1 self.search_centers = np.array([ e.norm_mean for e in self.flat_spaces.values()]) @@ -325,12 +326,13 @@ def create_gp(x_dim, scale_length=1.0): optimizer = torch.optim.Adam(model.parameters(), lr=0.0001) return model, optimizer +# TODO: Eval defaults class Protein: def __init__(self, sweep_config, - max_suggestion_cost = None, - resample_frequency = 5, - num_random_samples = 10, + max_suggestion_cost = 3600, + resample_frequency = 0, + num_random_samples = 50, global_search_scale = 1, random_suggestions = 1024, suggestions_per_pareto = 256, @@ -699,7 +701,7 @@ def _carbs_params_from_puffer_sweep(sweep_config): class Carbs: def __init__(self, sweep_config: dict, - max_suggestion_cost: float = None, + max_suggestion_cost: float = 3600, resample_frequency: int = 5, num_random_samples: int = 10, ): diff --git a/pufferlib/vector.py b/pufferlib/vector.py index 7b5d008f88..f4ae689415 100644 --- a/pufferlib/vector.py +++ b/pufferlib/vector.py @@ -635,6 +635,12 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer if num_envs != int(num_envs): raise APIUsageError('num_envs must be an integer') + if isinstance(backend, str): + try: + backend = getattr(pufferlib.vector, backend) + except: + raise APIUsageError(f'Invalid backend: {backend}') + if backend == PufferEnv: env_args = env_args or [] env_kwargs = env_kwargs or {} From 16de80c7c083bee635672422b7c891e53f3a60cf Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 12:59:42 +0000 Subject: [PATCH 12/63] Cleanup old sweep comments --- pufferlib/sweep.py | 71 ++++------------------------------------------ 1 file changed, 5 insertions(+), 66 deletions(-) diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index c213eb1577..3e64fedef1 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -336,6 +336,7 @@ def __init__(self, global_search_scale = 1, random_suggestions = 1024, suggestions_per_pareto = 256, + seed_with_search_center = True, min_score = None, max_score = None, ): @@ -352,6 +353,7 @@ def __init__(self, self.global_search_scale = global_search_scale self.random_suggestions = random_suggestions self.suggestions_per_pareto = suggestions_per_pareto + self.seed_with_search_center = seed_with_search_center self.resample_frequency = resample_frequency self.max_suggestion_cost = max_suggestion_cost @@ -365,12 +367,8 @@ def __init__(self, def suggest(self, fill): # TODO: Clip random samples to bounds so we don't get bad high cost samples info = {} - #if self.suggestion_idx <= self.num_random_samples: - # suggestions = self.hyperparameters.sample(self.random_suggestions) - # best_idx = np.random.randint(0, self.random_suggestions) - # best = suggestions[best_idx] self.suggestion_idx += 1 - if len(self.success_observations) == 0: + if len(self.success_observations) == 0 and self.seed_with_search_center: best = self.hyperparameters.search_centers return self.hyperparameters.to_dict(best, fill), info elif len(self.success_observations) < self.num_random_samples: @@ -406,14 +404,9 @@ def suggest(self, fill): if np.max(y) > max_score + 1e-6: raise ValueError(f'Max score {max_score} is greater than max score in data {np.max(y)}') - # Linearize, exp transform, linearize + # Linearize y_norm = (y - min_score) / (max_score - min_score) - #yt = -np.log(1 - y_norm + eps) - #yt_min = np.min(yt) - #yt_max = np.max(yt) - #yt_norm = (yt - yt_min) / (yt_max - yt_min) - #self.gp_score.set_data(params, torch.from_numpy(yt_norm)) self.gp_score.set_data(params, torch.from_numpy(y_norm)) self.gp_score.train() gp.util.train(self.gp_score, self.score_opt) @@ -438,12 +431,6 @@ def suggest(self, fill): candidates, pareto_idxs = pareto_points(self.success_observations) pareto_costs = np.array([e['cost'] for e in candidates]) - #cost_dists = np.abs(np.log(pareto_costs[:, None]) - np.log(pareto_costs[None, :])) - ###cost_dists = np.abs(pareto_costs[:, None] - pareto_costs[None, :]) - #cost_dists += (np.max(pareto_costs) + 1)*np.eye(len(pareto_costs)) # mask self-distance - #idx = np.argmax(np.min(cost_dists, axis=1)) - #search_centers = candidates[idx]['input'] - ### Sample suggestions search_centers = np.stack([e['input'] for e in candidates]) suggestions = self.hyperparameters.sample( @@ -458,10 +445,7 @@ def suggest(self, fill): gp_y_norm = gp_y_norm.numpy() gp_log_c_norm = gp_log_c_norm.numpy() - # Unlinearize, inverse exp transform, unlinearize - #gp_yt = gp_yt_norm*(yt_max - yt_min) + yt_min - #gp_y_norm = -(np.exp(-gp_yt) - 1 - eps) - #gp_y = gp_y_norm*(max_score - min_score) + min_score + # Unlinearize gp_y = gp_y_norm*(max_score - min_score) + min_score gp_log_c = gp_log_c_norm*(log_c_max - log_c_min) + log_c_min @@ -472,8 +456,6 @@ def suggest(self, fill): gp_c_norm = (gp_c - gp_c_min) / (gp_c_max - gp_c_min) pareto_y = y[pareto_idxs] - #pareto_yt = yt[pareto_idxs] - #pareto_yt_norm = yt_norm[pareto_idxs] pareto_c = c[pareto_idxs] pareto_log_c_norm = log_c_norm[pareto_idxs] @@ -482,47 +464,19 @@ def suggest(self, fill): c_right = abs(pareto_log_c_norm[None, :] - gp_log_c_norm[:, None]) - #pareto_c_norm = (pareto_c - min_c) / (max_c - min_c) - #gp_c_norm = (gp_c - min_c) / (max_c - min_c) - #c_right = np.abs(pareto_c_norm[None, :] - gp_c_norm[:, None]) - - #pareto_log_c_norm = (np.log(pareto_c) - log_c_min) / (log_c_max - log_c_min) - #c_right = np.abs(pareto_log_c_norm[None, :] - gp_log_c_norm[:, None]) - sorted_dist = np.sort(c_right, axis=1) - #top_k = sorted_dist[:, :5] - #pareto_dist_weight = np.sum(top_k, axis=1) / top_k.shape[1] - nearest_idx = np.argmin(c_right, axis=1) nearest_pareto_dist = np.min(c_right, axis=1) nearest_pareto_y = pareto_y[nearest_idx] - #c_left = np.abs(gp_c[:, None] - pareto_c[None, :]) - #c_left[c_left < 0] = np.inf - #nearest_idx = np.argmin(c_left, axis=1) - #nearest_pareto_yt_norm = pareto_yt_norm[nearest_idx] - max_c_mask = gp_c < self.max_suggestion_cost - #suggestion_scores = self.hyperparameters.optimize_direction * max_c_mask * ( - # gp_yt_norm - nearest_pareto_yt_norm) * nearest_pareto_dist - #suggestion_scores = self.hyperparameters.optimize_direction * max_c_mask * ( - # gp_yt_norm - nearest_pareto_yt_norm)# / gp_c - - #np.argwhere(gp_c > c) cumsum_mask = c[None, :] <= np.clip(gp_c[:, None], min_c, max_c) cumsum_mask = cumsum_mask * c[None, :] cumsum = np.sum(cumsum_mask, axis=1) / np.sum(c) target = gp_c_norm weight = target - cumsum - #if np.random.rand() < 0.5: - # score = gp_y_norm - #else: - # score = gp_y_norm * weight - #suggestion_scores = self.hyperparameters.optimize_direction * max_c_mask * ( - # score)# / gp_c - target = 1.25*np.random.rand() weight = 1 - abs(target - gp_log_c_norm) @@ -530,22 +484,7 @@ def suggest(self, fill): suggestion_scores = self.hyperparameters.optimize_direction * max_c_mask * ( gp_y_norm*weight)# / gp_c - #suggestion_scores = self.hyperparameters.optimize_direction * max_c_mask * ( - # gp_y_norm*nearest_pareto_dist)# / gp_c - - #exp_scores = np.exp(suggestion_scores) - #sum_exp_scores = np.sum(exp_scores) - #softmax_scores = exp_scores / sum_exp_scores - #idxs = np.arange(len(softmax_scores)) - #best_idx = np.random.choice(idxs, p=softmax_scores) - - # This works and uncovers approximate binary search when the GP is perfect - # Can't include cost in denom because it biases this case - # Instead, use conservative score and/or cost estimates - # Just need to figure out why the GP is overconfident - best_idx = np.argmax(suggestion_scores) - #best_idx = np.argmax(gp_y_norm) info = dict( cost = gp_c[best_idx].item(), score = gp_y[best_idx].item(), From 6debb2d03e867d94bca15e36587aa4561cdaad13 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 13:08:41 +0000 Subject: [PATCH 13/63] Ocean ini files for new sweep --- config/ocean/blastar.ini | 4 ++-- config/ocean/cartpole.ini | 8 +------- config/ocean/enduro.ini | 10 +--------- config/ocean/grid.ini | 10 ---------- config/ocean/impulse_wars.ini | 10 ---------- config/ocean/moba.ini | 3 +-- config/ocean/nmmo3.ini | 5 ++--- config/ocean/pong.ini | 10 ---------- config/ocean/snake.ini | 11 ----------- config/ocean/tower_climb.ini | 3 +-- config/ocean/tripletriad.ini | 6 ------ 11 files changed, 8 insertions(+), 72 deletions(-) diff --git a/config/ocean/blastar.ini b/config/ocean/blastar.ini index 227096a0c9..e545f019b8 100644 --- a/config/ocean/blastar.ini +++ b/config/ocean/blastar.ini @@ -14,8 +14,8 @@ gamma = 0.95 learning_rate = 0.05 minibatch_size = 32768 -[sweep.metric] -name = environment/enemy_crossed_screen +[sweep] +metric = environment/enemy_crossed_screen goal = minimize [sweep.parameters.train.parameters.batch_size] diff --git a/config/ocean/cartpole.ini b/config/ocean/cartpole.ini index 9a5674c6f0..b1ea10cef3 100644 --- a/config/ocean/cartpole.ini +++ b/config/ocean/cartpole.ini @@ -16,13 +16,7 @@ minibatch_size = 32768 [sweep] method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = episode_length -min = 0 -max = 205 +metric = episode_length [sweep.train.total_timesteps] distribution = log_normal diff --git a/config/ocean/enduro.ini b/config/ocean/enduro.ini index 4f6455b57a..f67089205f 100644 --- a/config/ocean/enduro.ini +++ b/config/ocean/enduro.ini @@ -15,15 +15,7 @@ minibatch_size = 32768 [sweep] -method = protein -name = sweep -max_score = None - -[sweep.metric] -goal = maximize -name = days_completed -min = 0 -max = None +metric = days_completed [sweep.train.total_timesteps] distribution = log_normal diff --git a/config/ocean/grid.ini b/config/ocean/grid.ini index 511ed0428c..4983373b64 100644 --- a/config/ocean/grid.ini +++ b/config/ocean/grid.ini @@ -26,16 +26,6 @@ ent_coef = 0.00001 learning_rate = 0.005 minibatch_size = 32768 -[sweep] -method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = score -min = 0 -max = 1 - [sweep.train.total_timesteps] distribution = log_normal min = 5e7 diff --git a/config/ocean/impulse_wars.ini b/config/ocean/impulse_wars.ini index bacbf0b228..9fe5eff614 100644 --- a/config/ocean/impulse_wars.ini +++ b/config/ocean/impulse_wars.ini @@ -41,16 +41,6 @@ compile_mode = reduce-overhead compile_fullgraph = False device = cuda -[sweep] -method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = score -min = 0.0 -max = 1.0 - [sweep.env.num_envs] distribution = uniform_pow2 min = 16 diff --git a/config/ocean/moba.ini b/config/ocean/moba.ini index 98012328a1..cd9e621771 100644 --- a/config/ocean/moba.ini +++ b/config/ocean/moba.ini @@ -24,8 +24,7 @@ bptt_horizon = 80 learning_rate = 0.05 [sweep.metric] -goal = maximize -name = radiant_towers_alive +metric = radiant_towers_alive [sweep.train.total_timesteps] distribution = log_normal diff --git a/config/ocean/nmmo3.ini b/config/ocean/nmmo3.ini index 1de6ee1f63..4780006bc4 100644 --- a/config/ocean/nmmo3.ini +++ b/config/ocean/nmmo3.ini @@ -28,9 +28,8 @@ batch_size = 262144 minibatch_size = 32768 compile = False -[sweep.metric] -goal = maximize -name = min_comb_prof +[sweep] +metric = min_comb_prof [sweep.env.num_envs] distribution = uniform_pow2 diff --git a/config/ocean/pong.ini b/config/ocean/pong.ini index 7ddf13c044..b5599f96f5 100644 --- a/config/ocean/pong.ini +++ b/config/ocean/pong.ini @@ -13,16 +13,6 @@ total_timesteps = 80_000_000 learning_rate = 0.05 minibatch_size = 32768 -[sweep] -method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = score -min = -21 -max = 21 - [sweep.train.total_timesteps] distribution = log_normal min = 5e6 diff --git a/config/ocean/snake.ini b/config/ocean/snake.ini index 2f6e6194ba..001d0df092 100644 --- a/config/ocean/snake.ini +++ b/config/ocean/snake.ini @@ -22,17 +22,6 @@ total_timesteps = 300_000_000 learning_rate = 0.05 minibatch_size = 32768 -[sweep] -method = protein -name = sweep -max_score = None - -[sweep.metric] -goal = maximize -name = score -min = 0 -max = None - [sweep.train.diayn_archive] distribution = uniform_pow2 min = 2 diff --git a/config/ocean/tower_climb.ini b/config/ocean/tower_climb.ini index fe7f210ce2..01b2059423 100644 --- a/config/ocean/tower_climb.ini +++ b/config/ocean/tower_climb.ini @@ -19,8 +19,7 @@ learning_rate = 0.05 minibatch_size = 32768 [sweep.metric] -goal = maximize -name = environment/levels_completed +metric = environment/levels_completed [sweep.parameters.train.parameters.total_timesteps] distribution = uniform diff --git a/config/ocean/tripletriad.ini b/config/ocean/tripletriad.ini index b0c2392e32..203555ae2a 100644 --- a/config/ocean/tripletriad.ini +++ b/config/ocean/tripletriad.ini @@ -14,12 +14,6 @@ gamma = 0.95 learning_rate = 0.05 minibatch_size = 32768 -[sweep.metric] -goal = maximize -name = score -min = 0 -max = 9.0 - [sweep.train.total_timesteps] distribution = log_normal min = 5e7 From 8a49f992dfc7413d7884f8be53c32b8fd35f3235 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 13:16:45 +0000 Subject: [PATCH 14/63] sota nmmo3 --- pufferlib/ocean/torch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pufferlib/ocean/torch.py b/pufferlib/ocean/torch.py index dc13ad9314..a3baa8e7b2 100644 --- a/pufferlib/ocean/torch.py +++ b/pufferlib/ocean/torch.py @@ -37,9 +37,9 @@ def __init__(self, env, hidden_size=512, output_size=512, **kwargs): self.is_continuous = False self.map_2d = nn.Sequential( - pufferlib.pytorch.layer_init(nn.Conv2d(self.multihot_dim, 64, 5, stride=3)), + pufferlib.pytorch.layer_init(nn.Conv2d(self.multihot_dim, 256, 5, stride=3)), nn.ReLU(), - pufferlib.pytorch.layer_init(nn.Conv2d(64, 64, 3, stride=1)), + pufferlib.pytorch.layer_init(nn.Conv2d(256, 256, 3, stride=1)), nn.Flatten(), ) @@ -49,7 +49,7 @@ def __init__(self, env, hidden_size=512, output_size=512, **kwargs): ) self.proj = nn.Sequential( - pufferlib.pytorch.layer_init(nn.Linear(1689, hidden_size)), + pufferlib.pytorch.layer_init(nn.Linear(2073, hidden_size)), nn.ReLU(), ) From d3c1f8bfab36739717eab3f153c877da7089c6b9 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 13:34:45 +0000 Subject: [PATCH 15/63] sweep defaults --- clean_pufferl.py | 8 +++-- config/default.ini | 71 ++++----------------------------------- config/ocean/breakout.ini | 12 +++---- config/ocean/grid.ini | 24 +------------ 4 files changed, 19 insertions(+), 96 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 98d745cee9..4d7264c36a 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -69,6 +69,10 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Experience buffer device = config.device batch_size = config.batch_size + + if config.bptt_horizon == 'auto': + config.bptt_horizon = batch_size // total_agents + horizon = config.bptt_horizon segments = batch_size // horizon self.segments = segments @@ -1134,7 +1138,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin model_file = max(os.listdir(data_dir)) args['eval_model_path'] = os.path.join(data_dir, model_file) elif args['mode'] == 'train': - target_metric = args['sweep']['metric']['name'] + target_metric = args['sweep']['metric'] train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) elif args['mode'] in ('eval', 'evaluate'): vec = pufferlib.vector.Serial @@ -1162,7 +1166,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin from torch.profiler import profile, record_function, ProfilerActivity with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True) as prof: with record_function("model_inference"): - target_metric = args['sweep']['metric']['name'] + target_metric = args['sweep']['metric'] train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) print(prof.key_averages().table(sort_by='cuda_time_total', row_limit=10)) diff --git a/config/default.ini b/config/default.ini index 5d7a662461..1aace6561a 100644 --- a/config/default.ini +++ b/config/default.ini @@ -53,7 +53,7 @@ minibatch_size = 8192 replay_factor = 0.0 # Accumulate gradients above this size max_minibatch_size = 32768 -bptt_horizon = 64 +bptt_horizon = auto compile = False compile_mode = max-autotune-no-cudagraphs compile_fullgraph = True @@ -72,21 +72,6 @@ method = Protein metric = score goal = maximize -[sweep.env.num_envs] -distribution = uniform_pow2 -min = 64 -max = 4096 -mean = 1024 -scale = auto -#scale = 0.5 - -#[sweep.policy.hidden_size] -#distribution = uniform_pow2 -#min = 32 -#max = 1024 -#mean = 128 -#scale = auto - [sweep.train.total_timesteps] distribution = log_normal min = 5e7 @@ -96,15 +81,15 @@ scale = time [sweep.train.batch_size] distribution = uniform_pow2 -min = 32768 -max = 1048576 -mean = 262144 +min = 131072 +max = 2097152 +mean = 524288 scale = auto [sweep.train.minibatch_size] distribution = uniform_pow2 -min = 1024 -max = 32768 +min = 4096 +max = 131072 mean = 8192 scale = auto @@ -128,7 +113,6 @@ min = 0.8 mean = 0.98 max = 0.9999 scale = auto -#scale = 0.5 [sweep.train.gae_lambda] distribution = logit_normal @@ -136,7 +120,6 @@ min = 0.6 mean = 0.95 max = 0.995 scale = auto -#scale = 0.5 [sweep.train.update_epochs] distribution = int_uniform @@ -159,20 +142,6 @@ mean = 1.0 max = 5.0 scale = auto -[sweep.train.bptt_horizon] -distribution = uniform_pow2 -min = 4 -max = 128 -mean = 16 -scale = auto - -#[sweep.train.puf] -#distribution = logit_normal -#min = 0.01 -#mean = 0.5 -#max = 0.99 -#scale = auto - [sweep.train.adam_beta1] distribution = logit_normal min = 0.5 @@ -193,31 +162,3 @@ min = 0.00000000000001 mean = 0.00000001 max = 0.001 scale = auto - -#[sweep.train.horizon] -#distribution = uniform_pow2 -#min = 4 -#max = 128 -#mean = 32 -#scale = 0.25 - -#[sweep.train.diayn_archive] -#distribution = uniform_pow2 -#min = 2 -#max = 64 -#mean = 8 -#scale = auto - -#[sweep.train.diayn_loss_coef] -#distribution = uniform -#min = 0.0 -#max = 2.0 -#mean = 1.0 -#scale = auto - -#[sweep.train.diayn_coef] -#distribution = log_normal -#min = 0.0001 -#mean = 0.1 -#max = 0.99 -#scale = auto diff --git a/config/ocean/breakout.ini b/config/ocean/breakout.ini index 6e96e52669..502dc57c8b 100644 --- a/config/ocean/breakout.ini +++ b/config/ocean/breakout.ini @@ -19,9 +19,9 @@ total_timesteps = 80_000_000 learning_rate = 0.05 minibatch_size = 32768 -#[sweep.train.total_timesteps] -#distribution = log_normal -#min = 2e7 -#max = 1e8 -#mean = 5e7 -#scale = auto +[sweep.train.total_timesteps] +distribution = log_normal +min = 2e7 +max = 1e8 +mean = 8e7 +scale = auto diff --git a/config/ocean/grid.ini b/config/ocean/grid.ini index 4983373b64..a8f05d0907 100644 --- a/config/ocean/grid.ini +++ b/config/ocean/grid.ini @@ -29,28 +29,6 @@ minibatch_size = 32768 [sweep.train.total_timesteps] distribution = log_normal min = 5e7 -max = 2e8 +max = 3e8 mean = 1e8 scale = auto - -[sweep.train.e3b_coef] -distribution = logit_normal -min = 0.0001 -max = 0.99 -mean = 0.001 -scale = auto - -[sweep.train.e3b_lambda] -distribution = log_normal -min = 0.01 -max = 10.0 -mean = 0.1 -scale = auto - -[sweep.train.e3b_norm] -distribution = log_normal -min = 0.0001 -max = 0.1 -mean = 0.001 -scale = auto - From b6ffb1d3f6c67f641374bf7220e4d6a5bcbcc0eb Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 16:51:06 +0000 Subject: [PATCH 16/63] Fix logit norm bug --- clean_pufferl.py | 9 +++++---- pufferlib/pytorch.py | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 4d7264c36a..5ea4349177 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -118,7 +118,8 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): minibatch_size = config.minibatch_size max_minibatch_size = config.max_minibatch_size self.minibatch_size = min(minibatch_size, max_minibatch_size) - if max_minibatch_size % minibatch_size != 0: + if minibatch_size % max_minibatch_size != 0 and max_minibatch_size % minibatch_size != 0: + # todo: better error raise pufferlib.exceptions.APIUsageError( f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' ) @@ -181,11 +182,11 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.neptune = neptune self.wandb = wandb if neptune: - self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.run_tag) + self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.tag) for k, v in pufferlib.utils.unroll_nested_dict(args): self.neptune[k].append(v) elif wandb: - self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.run_tag) + self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.tag) # Profiling self.uptime = 0 @@ -1000,7 +1001,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin policy.cell = orig_policy.cell env_name = args['env_name'] - train_config = pufferlib.namespace(**args['train'], env=env_name, + train_config = pufferlib.namespace(**args['train'], env=env_name, tag=args['tag'], exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index 17a7c014c3..8d88ed5ebc 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -284,6 +284,7 @@ def sample_logits(logits: Union[torch.Tensor, List[torch.Tensor]], return action, log_probs, logits_entropy elif is_discrete: logits = logits.unsqueeze(0) + # TODO: Double check this else: #multi-discrete logits = torch.nn.utils.rnn.pad_sequence( [l.transpose(0,1) for l in logits], @@ -292,7 +293,7 @@ def sample_logits(logits: Union[torch.Tensor, List[torch.Tensor]], ).permute(1,2,0) normalized_logits = logits - logits.logsumexp(dim=-1, keepdim=True) - probs = logits_to_probs(normalized_logits) + probs = logits_to_probs(logits) if action is None: action = torch.multinomial(probs.reshape(-1, probs.shape[-1]), 1, replacement=True) From 9396f2919193ecb694318f1ec81aa6eab45fe5a0 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 17:13:45 +0000 Subject: [PATCH 17/63] Fix build for 5090 --- pyproject.toml | 2 +- setup.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b4d35b4021..d1bc56ce11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [build-system] -requires = ["setuptools", "wheel", "Cython", "numpy"] +requires = ["setuptools", "wheel", "Cython", "numpy", "torch"] build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index d69ba88b49..b0068d9108 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,7 @@ +#TODO: +# --no-build-isolation for 5090 +# Make c and torch compile at the same time + from setuptools import find_packages, find_namespace_packages, setup, Extension from Cython.Build import cythonize import numpy @@ -382,12 +386,12 @@ def run(self): extra_compile_args = { "cxx": [ "-fdiagnostics-color=always", - "-DPy_LIMITED_API=0x03090000", # min CPython version 3.9 + #"-DPy_LIMITED_API=0x03090000", # min CPython version 3.9 ], "nvcc": [ ], }, - py_limited_api=True, + #py_limited_api=True, ), ] @@ -424,6 +428,7 @@ def run(self): f'gym<={GYM_VERSION}', f'gymnasium<={GYMNASIUM_VERSION}', f'pettingzoo<={PETTINGZOO_VERSION}', + 'torch', 'shimmy[gym-v21]', 'psutil==5.9.5', 'pynvml', @@ -454,8 +459,9 @@ def run(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ], - options={"bdist_wheel": {"py_limited_api": "cp39"}}, + #options={"bdist_wheel": {"py_limited_api": "cp39"}}, ) #stable_baselines3 #supersuit==3.3.5 From 5c4f068fe8f2d3656595f04a34abc6746e85f43b Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 17:43:14 +0000 Subject: [PATCH 18/63] Sweep setup for maze, sota maze at 32 size --- clean_pufferl.py | 5 ++++- config/default.ini | 14 ++++++++++++++ config/ocean/grid.ini | 3 +-- pufferlib/ocean/grid/grid.h | 2 +- setup.py | 4 ++-- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 5ea4349177..1219fda3f4 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -1103,7 +1103,10 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin for subkey in split[:-1]: next = next[subkey] - next[split[-1]] = value + try: + next[split[-1]] = value + except: + breakpoint() package = args['package'] module_name = f'pufferlib.environments.{package}' diff --git a/config/default.ini b/config/default.ini index 1aace6561a..d42c878056 100644 --- a/config/default.ini +++ b/config/default.ini @@ -162,3 +162,17 @@ min = 0.00000000000001 mean = 0.00000001 max = 0.001 scale = auto + +[sweep.train.prio_alpha] +distribution = logit_normal +min = 0.1 +mean = 0.6 +max = 0.99 +scale = auto + +[sweep.train.prio_beta0] +distribution = logit_normal +min = 0.1 +mean = 0.4 +max = 0.99 +scale = auto diff --git a/config/ocean/grid.ini b/config/ocean/grid.ini index a8f05d0907..6504f4a19c 100644 --- a/config/ocean/grid.ini +++ b/config/ocean/grid.ini @@ -1,7 +1,6 @@ [base] package = ocean env_name = puffer_grid -vec = multiprocessing policy_name = Policy rnn_name = Recurrent @@ -13,7 +12,7 @@ input_size = 512 hidden_size = 512 [env] -max_size = 31 +max_size = 47 num_envs = 4096 num_maps = 8192 diff --git a/pufferlib/ocean/grid/grid.h b/pufferlib/ocean/grid/grid.h index 161e725d0b..6c22b69406 100644 --- a/pufferlib/ocean/grid/grid.h +++ b/pufferlib/ocean/grid/grid.h @@ -496,7 +496,7 @@ void c_render(Grid* env) { float frac = 0.0; float overlay = 0.0; if (env->renderer == NULL) { - env->renderer = init_renderer(16, env->width, env->height); + env->renderer = init_renderer(16, env->max_size, env->max_size); } Renderer* renderer = env->renderer; diff --git a/setup.py b/setup.py index b0068d9108..042def711f 100644 --- a/setup.py +++ b/setup.py @@ -442,8 +442,8 @@ def run(self): 'common': common, **environments, }, - ext_modules = torch_extensions, - cmdclass={"build_ext": cpp_extension.BuildExtension}, + ext_modules = c_extensions, + #cmdclass={"build_ext": cpp_extension.BuildExtension}, include_dirs=[numpy.get_include(), RAYLIB_NAME + '/include'], python_requires=">=3.9", license="MIT", From f2c9770161d09223c833e93da7ca921082036d97 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 18:10:20 +0000 Subject: [PATCH 19/63] Fix action sampling bug --- pufferlib/pytorch.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index 8d88ed5ebc..927e7cb4bc 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -301,7 +301,6 @@ def sample_logits(logits: Union[torch.Tensor, List[torch.Tensor]], else: batch = logits[0].shape[0] action = action.view(batch, -1).T - probs = logits_to_probs(normalized_logits) assert len(logits) == len(action) logprob = log_prob(normalized_logits, action) From 9ad5b552622a1b0684d47be85187803b250641ed Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 19:33:00 +0000 Subject: [PATCH 20/63] Move a bunch of stuff to a pufferlib.py --- pufferlib/__init__.py | 3 +- pufferlib/cleanrl.py | 51 -------- pufferlib/emulation.py | 35 +++--- pufferlib/environment.py | 96 --------------- pufferlib/exceptions.py | 20 --- pufferlib/namespace.py | 60 --------- pufferlib/ocean/environment.py | 23 ++-- pufferlib/policy_ranker.py | 104 ---------------- pufferlib/policy_store.py | 26 ---- pufferlib/postprocess.py | 219 --------------------------------- pufferlib/vector.py | 65 +++++----- pufferlib/wrappers.py | 57 --------- 12 files changed, 59 insertions(+), 700 deletions(-) delete mode 100644 pufferlib/cleanrl.py delete mode 100644 pufferlib/environment.py delete mode 100644 pufferlib/exceptions.py delete mode 100644 pufferlib/namespace.py delete mode 100644 pufferlib/policy_ranker.py delete mode 100644 pufferlib/policy_store.py delete mode 100644 pufferlib/postprocess.py delete mode 100644 pufferlib/wrappers.py diff --git a/pufferlib/__init__.py b/pufferlib/__init__.py index c501c01945..9bc5c3e56e 100644 --- a/pufferlib/__init__.py +++ b/pufferlib/__init__.py @@ -23,6 +23,5 @@ sys.stdout = original_stdout sys.stderr = original_stderr -from pufferlib.namespace import namespace, dataclass, Namespace +from pufferlib.pufferlib import * from pufferlib import environments -from pufferlib.environment import PufferEnv diff --git a/pufferlib/cleanrl.py b/pufferlib/cleanrl.py deleted file mode 100644 index 9292a1d7c4..0000000000 --- a/pufferlib/cleanrl.py +++ /dev/null @@ -1,51 +0,0 @@ -from pdb import set_trace as T - -class Policy(torch.nn.Module): - '''Wrap a non-recurrent PyTorch model for use with CleanRL''' - def __init__(self, policy): - super().__init__() - self.policy = policy - self.is_continuous = hasattr(policy, 'is_continuous') and policy.is_continuous - self.hidden_size = policy.hidden_size - - def get_value(self, x, state=None): - _, value = self.policy(x) - return value - - def get_action_and_value(self, x, action=None): - logits, value, e3b, intrinsic_reward = self.policy(x, e3b=e3b) - action, logprob, entropy = sample_logits(logits, action, self.is_continuous) - return action, logprob, entropy, value, e3b, intrinsic_reward - - def forward(self, x, action=None, e3b=None): - return self.get_action_and_value(x, action, e3b) - - -class RecurrentPolicy(torch.nn.Module): - '''Wrap a recurrent PyTorch model for use with CleanRL''' - def __init__(self, policy): - super().__init__() - self.policy = policy - self.is_continuous = hasattr(policy.policy, 'is_continuous') and policy.policy.is_continuous - self.hidden_size = policy.hidden_size - - @property - def lstm(self): - if hasattr(self.policy, 'recurrent'): - return self.policy.recurrent - elif hasattr(self.policy, 'lstm'): - return self.policy.lstm - else: - raise ValueError('Policy must have a subnetwork named lstm or recurrent') - - def get_value(self, x, state=None): - _, value, _ = self.policy(x, state) - - def get_action_and_value(self, x, state=None, action=None, e3b=None): - #logits, value, state, e3b, intrinsic_reward = self.policy(x, state, e3b=e3b) - logits, value_mean, value_logstd, state = self.policy(x, state, e3b=e3b) - action, logprob, entropy = sample_logits(logits, action, self.is_continuous) - return action, logprob, entropy, value_mean, value_logstd, state#, e3b, intrinsic_reward - - def forward(self, x, state=None, action=None, e3b=None): - return self.get_action_and_value(x, state, action, e3b) diff --git a/pufferlib/emulation.py b/pufferlib/emulation.py index a1aba05e98..9c35805448 100644 --- a/pufferlib/emulation.py +++ b/pufferlib/emulation.py @@ -8,11 +8,8 @@ import pufferlib import pufferlib.spaces -from pufferlib import utils, exceptions -from pufferlib.environment import set_buffers +from pufferlib import utils from pufferlib.spaces import Discrete, Tuple, Dict -import pufferlib.environment - def emulate(struct, sample): if isinstance(sample, dict): @@ -154,7 +151,7 @@ def __init__(self, env=None, env_creator=None, env_args=[], env_kwargs={}, buf=N self.render_modes = 'human rgb_array'.split() - set_buffers(self, buf) + pufferlib.set_buffers(self, buf) if isinstance(self.env.observation_space, pufferlib.spaces.Box): self.obs_struct = self.observations else: @@ -191,9 +188,9 @@ def reset(self, seed=None): def step(self, action): '''Execute an action and return (observation, reward, done, info)''' if not self.initialized: - raise exceptions.APIUsageError('step() called before reset()') + raise pufferlib.APIUsageError('step() called before reset()') if self.done: - raise exceptions.APIUsageError('step() called after environment is done') + raise pufferlib.APIUsageError('step() called after environment is done') # Unpack actions from multidiscrete into the original action space if self.is_atn_emulated: @@ -256,7 +253,7 @@ def __init__(self, env=None, env_creator=None, env_args=[], buf=None, env_kwargs self.num_agents = len(self.possible_agents) - set_buffers(self, buf) + pufferlib.set_buffers(self, buf) if isinstance(self.env_single_observation_space, pufferlib.spaces.Box): self.obs_struct = self.observations else: @@ -281,14 +278,14 @@ def done(self): def observation_space(self, agent): '''Returns the observation space for a single agent''' if agent not in self.possible_agents: - raise pufferlib.exceptions.InvalidAgentError(agent, self.possible_agents) + raise pufferlib.InvalidAgentError(agent, self.possible_agents) return self.single_observation_space def action_space(self, agent): '''Returns the action space for a single agent''' if agent not in self.possible_agents: - raise pufferlib.exceptions.InvalidAgentError(agent, self.possible_agents) + raise pufferlib.InvalidAgentError(agent, self.possible_agents) return self.single_action_space @@ -329,13 +326,13 @@ def reset(self, seed=None): def step(self, actions): '''Step the environment and return (observations, rewards, dones, infos)''' if not self.initialized: - raise exceptions.APIUsageError('step() called before reset()') + raise pufferlib.APIUsageError('step() called before reset()') if self.done: - raise exceptions.APIUsageError('step() called after environment is done') + raise pufferlib.APIUsageError('step() called after environment is done') if isinstance(actions, np.ndarray): if not self.is_action_checked and len(actions) != self.num_agents: - raise exceptions.APIUsageError( + raise pufferlib.APIUsageError( f'Actions specified as len {len(actions)} but environment has {self.num_agents} agents') actions = {agent: actions[i] for i, agent in enumerate(self.possible_agents)} @@ -344,7 +341,7 @@ def step(self, actions): if not self.is_action_checked: for agent in actions: if agent not in self.possible_agents: - raise exceptions.InvalidAgentError(agent, self.possible_agents) + raise pufferlib.InvalidAgentError(agent, self.possible_agents) self.is_action_checked = check_space( next(iter(actions.values())), @@ -355,7 +352,7 @@ def step(self, actions): unpacked_actions = {} for agent, atn in actions.items(): if agent not in self.possible_agents: - raise exceptions.InvalidAgentError(agent, self.agents) + raise pufferlib.InvalidAgentError(agent, self.agents) if agent not in self.agents: continue @@ -435,11 +432,11 @@ def check_space(data, space): try: contains = space.contains(data) except: - raise exceptions.APIUsageError( + raise pufferlib.APIUsageError( f'Error checking space {space} with sample :\n{data}') if not contains: - raise exceptions.APIUsageError( + raise pufferlib.APIUsageError( f'Data:\n{data}\n not in space:\n{space}') return True @@ -462,9 +459,9 @@ def _seed_and_reset(env, seed): return obs, info -class GymnaxPufferEnv(pufferlib.environment.PufferEnv): +class GymnaxPufferEnv(pufferlib.PufferEnv): def __init__(self, env, env_params, num_envs=1, buf=None): - from gymnax.environments.spaces import gymnax_space_to_gym_space + from gymnax.spaces import gymnax_space_to_gym_space gymnax_obs_space = env.observation_space(env_params) self.single_observation_space = gymnax_space_to_gym_space(gymnax_obs_space) diff --git a/pufferlib/environment.py b/pufferlib/environment.py deleted file mode 100644 index 7eb7715bb5..0000000000 --- a/pufferlib/environment.py +++ /dev/null @@ -1,96 +0,0 @@ -import numpy as np - -from pufferlib.exceptions import APIUsageError -import pufferlib.spaces - -ERROR = ''' -Environment missing required attribute {}. The most common cause is -calling super() before you have assigned the attribute. -''' - -def set_buffers(env, buf=None): - if buf is None: - obs_space = env.single_observation_space - env.observations = np.zeros((env.num_agents, *obs_space.shape), dtype=obs_space.dtype) - env.rewards = np.zeros(env.num_agents, dtype=np.float32) - env.terminals = np.zeros(env.num_agents, dtype=bool) - env.truncations = np.zeros(env.num_agents, dtype=bool) - env.masks = np.ones(env.num_agents, dtype=bool) - - # TODO: Major kerfuffle on inferring action space dtype. This needs some asserts? - atn_space = env.single_action_space - if isinstance(env.single_action_space, pufferlib.spaces.Box): - env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=atn_space.dtype) - else: - env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=np.int32) - else: - env.observations = buf.observations - env.rewards = buf.rewards - env.terminals = buf.terminals - env.truncations = buf.truncations - env.masks = buf.masks - env.actions = buf.actions - -class PufferEnv: - def __init__(self, buf=None): - if not hasattr(self, 'single_observation_space'): - raise APIUsageError(ERROR.format('single_observation_space')) - if not hasattr(self, 'single_action_space'): - raise APIUsageError(ERROR.format('single_action_space')) - if not hasattr(self, 'num_agents'): - raise APIUsageError(ERROR.format('num_agents')) - if self.num_agents < 1: - raise APIUsageError('num_agents must be >= 1') - - if hasattr(self, 'observation_space'): - raise APIUsageError('PufferEnvs must define single_observation_space, not observation_space') - if hasattr(self, 'action_space'): - raise APIUsageError('PufferEnvs must define single_action_space, not action_space') - if not isinstance(self.single_observation_space, pufferlib.spaces.Box): - raise APIUsageError('Native observation_space must be a Box') - if (not isinstance(self.single_action_space, pufferlib.spaces.Discrete) - and not isinstance(self.single_action_space, pufferlib.spaces.MultiDiscrete) - and not isinstance(self.single_action_space, pufferlib.spaces.Box)): - raise APIUsageError('Native action_space must be a Discrete, MultiDiscrete, or Box') - - set_buffers(self, buf) - - self.action_space = pufferlib.spaces.joint_space(self.single_action_space, self.num_agents) - self.observation_space = pufferlib.spaces.joint_space(self.single_observation_space, self.num_agents) - self.agent_ids = np.arange(self.num_agents) - - @property - def emulated(self): - '''Native envs do not use emulation''' - return False - - @property - def done(self): - '''Native envs handle resets internally''' - return False - - @property - def driver_env(self): - '''For compatibility with Multiprocessing''' - return self - - def reset(self, seed=None): - raise NotImplementedError - - def step(self, actions): - raise NotImplementedError - - def close(self): - raise NotImplementedError - - def async_reset(self, seed=None): - _, self.infos = self.reset(seed) - assert isinstance(self.infos, list), 'PufferEnvs must return info as a list of dicts' - - def send(self, actions): - _, _, _, _, self.infos = self.step(actions) - assert isinstance(self.infos, list), 'PufferEnvs must return info as a list of dicts' - - def recv(self): - return (self.observations, self.rewards, self.terminals, - self.truncations, self.infos, self.agent_ids, self.masks) diff --git a/pufferlib/exceptions.py b/pufferlib/exceptions.py deleted file mode 100644 index ec8d2cc844..0000000000 --- a/pufferlib/exceptions.py +++ /dev/null @@ -1,20 +0,0 @@ -class EnvironmentSetupError(RuntimeError): - def __init__(self, e, package): - super().__init__(self.message) - -class APIUsageError(RuntimeError): - """Exception raised when the API is used incorrectly.""" - - def __init__(self, message="API usage error."): - self.message = message - super().__init__(self.message) - -class InvalidAgentError(ValueError): - """Exception raised when an invalid agent key is used.""" - - def __init__(self, agent_id, agents): - message = ( - f'Invalid agent/team ({agent_id}) specified. ' - f'Valid values:\n{agents}' - ) - super().__init__(message) diff --git a/pufferlib/namespace.py b/pufferlib/namespace.py deleted file mode 100644 index a6ecfe3529..0000000000 --- a/pufferlib/namespace.py +++ /dev/null @@ -1,60 +0,0 @@ -from pdb import set_trace as T -from types import SimpleNamespace -from collections.abc import Mapping - -def __getitem__(self, key): - return self.__dict__[key] - -def __setitem__(self, key, value): - self.__dict__[key] = value - -def keys(self): - return self.__dict__.keys() - -def values(self): - return self.__dict__.values() - -def items(self): - return self.__dict__.items() - -def __iter__(self): - return iter(self.__dict__) - -def __len__(self): - return len(self.__dict__) - -class Namespace(SimpleNamespace, Mapping): - __getitem__ = __getitem__ - __setitem__ = __setitem__ - __iter__ = __iter__ - __len__ = __len__ - keys = keys - values = values - items = items - -def dataclass(cls): - # Safely get annotations - annotations = getattr(cls, '__annotations__', {}) - - # Combine both annotated and non-annotated fields - all_fields = {**{k: None for k in annotations.keys()}, **cls.__dict__} - all_fields = {k: v for k, v in all_fields.items() if not callable(v) and not k.startswith('__')} - - def __init__(self, **kwargs): - for field, default_value in all_fields.items(): - setattr(self, field, kwargs.get(field, default_value)) - - cls.__init__ = __init__ - setattr(cls, "__getitem__", __getitem__) - setattr(cls, "__setitem__", __setitem__) - setattr(cls, "__iter__", __iter__) - setattr(cls, "__len__", __len__) - setattr(cls, "keys", keys) - setattr(cls, "values", values) - setattr(cls, "items", items) - return cls - -def namespace(self=None, **kwargs): - if self is None: - return Namespace(**kwargs) - self.__dict__.update(kwargs) diff --git a/pufferlib/ocean/environment.py b/pufferlib/ocean/environment.py index d87dd4172b..b3770cd998 100644 --- a/pufferlib/ocean/environment.py +++ b/pufferlib/ocean/environment.py @@ -1,6 +1,5 @@ import importlib import pufferlib.emulation -import pufferlib.postprocess def lazy_import(module_path, attr): """ @@ -58,63 +57,63 @@ def make_continuous(discretize=False, buf=None, **kwargs): from . import sanity env = sanity.Continuous(discretize=discretize) if not discretize: - env = pufferlib.postprocess.ClipAction(env) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.ClipAction(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_squared(distance_to_target=3, num_targets=1, buf=None, **kwargs): from . import sanity env = sanity.Squared(distance_to_target=distance_to_target, num_targets=num_targets, **kwargs) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf, **kwargs) def make_bandit(num_actions=10, reward_scale=1, reward_noise=1, buf=None): from . import sanity env = sanity.Bandit(num_actions=num_actions, reward_scale=reward_scale, reward_noise=reward_noise) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_memory(mem_length=2, mem_delay=2, buf=None, **kwargs): from . import sanity env = sanity.Memory(mem_length=mem_length, mem_delay=mem_delay) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_password(password_length=5, buf=None, **kwargs): from . import sanity env = sanity.Password(password_length=password_length) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_performance(delay_mean=0, delay_std=0, bandwidth=1, buf=None, **kwargs): from . import sanity env = sanity.Performance(delay_mean=delay_mean, delay_std=delay_std, bandwidth=bandwidth) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_performance_empiric(count_n=0, count_std=0, bandwidth=1, buf=None, **kwargs): from . import sanity env = sanity.PerformanceEmpiric(count_n=count_n, count_std=count_std, bandwidth=bandwidth) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_stochastic(p=0.7, horizon=100, buf=None, **kwargs): from . import sanity env = sanity.Stochastic(p=p, horizon=100) - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf) def make_spaces(buf=None, **kwargs): from . import sanity env = sanity.Spaces() - env = pufferlib.postprocess.EpisodeStats(env) + env = pufferlib.EpisodeStats(env) return pufferlib.emulation.GymnasiumPufferEnv(env=env, buf=buf, **kwargs) def make_multiagent(buf=None, **kwargs): from . import sanity env = sanity.Multiagent() - env = pufferlib.postprocess.MultiagentEpisodeStats(env) + env = pufferlib.MultiagentEpisodeStats(env) return pufferlib.emulation.PettingZooPufferEnv(env=env, buf=buf) MAKE_FUNCTIONS = { diff --git a/pufferlib/policy_ranker.py b/pufferlib/policy_ranker.py deleted file mode 100644 index 8282f9fc9e..0000000000 --- a/pufferlib/policy_ranker.py +++ /dev/null @@ -1,104 +0,0 @@ -from pdb import set_trace as T -import numpy as np - -import sqlite3 - -ANCHOR_ELO = 1000.0 - - -def win_prob(elo1, elo2): - '''Calculate win probability such that a difference of - 50/100/150 elo corresponds to win probabilitit 68/95/99.7%''' - return 1 / (1 + 10 ** ((elo2 - elo1) / 400)) - -def update_elos(elos: np.ndarray, scores: np.ndarray, k: float = 4.0): - '''Update elos based on the result of a game - - The parameter k controls the magnitude of the update. - A higher k means that the elo will change more after a game. - This means that elos will converge faster but less precisely. - In particular, low k cannot distinguish between players of - similar skill, while a high k will just take longer to converge. - - The default is tuned for normally distributed player skill - You should lower it if you have very similar players. - Raise it if you are evaluating a diverse skill pool. - ''' - num_players = len(elos) - assert num_players == len(scores) - - elo_update = [[] for _ in range(num_players)] - for i in range(num_players): - for j in range(i+1, num_players): - delta = scores[i] - scores[j] - - # Convert to elo scoring format - if delta > 0: - score_i = 1 - elif delta == 0: - score_i = 0.5 - else: - score_i = 0 - - # Calculate elo update for pairs - expected_i = win_prob(elos[i], elos[j]) - expected_j = 1 - expected_i - score_j = 1 - score_i - - elo_update[i].append(k * (score_i - expected_i)) - elo_update[j].append(k * (score_j - expected_j)) - - elo_update = [np.mean(e) for e in elo_update] - return [elo + update for elo, update in zip(elos, elo_update)] - -class Ranker: - def __init__(self, db_path): - self.conn = sqlite3.connect(db_path) - with self.conn: - self.conn.execute(""" - CREATE TABLE IF NOT EXISTS ratings ( - policy TEXT PRIMARY KEY, - elo REAL - ); - """) - - def __repr__(self): - if len(self.ratings) == 0: - return '' - - sorted_dict = sorted(self.ratings.items(), key=lambda x: x[1], reverse=True) - return '\n'.join([ - f' - Policy: {name}, Elo: {elo:.3f}' - for name, elo in sorted_dict - ]) - - @property - def ratings(self): - with self.conn: - cursor = self.conn.execute("SELECT * FROM ratings;") - - return {row[0]: row[1] for row in cursor.fetchall()} - - def update(self, scores: dict): - if len(scores) < 2: - return - - # Load all elos from DB - elos = self.ratings - - flat_scores = [] - flat_elos = [] - for policy in scores.keys(): - flat_scores.append(scores[policy]) - if policy in elos: - flat_elos.append(elos[policy]) - else: - flat_elos.append(ANCHOR_ELO) - - flat_elos = update_elos(flat_elos, flat_scores) - elos = zip(scores.keys(), flat_elos) - with self.conn: - self.conn.executemany(""" - INSERT OR REPLACE INTO ratings (policy, elo) - VALUES (?, ?); - """, elos) diff --git a/pufferlib/policy_store.py b/pufferlib/policy_store.py deleted file mode 100644 index 7bbd96ad1a..0000000000 --- a/pufferlib/policy_store.py +++ /dev/null @@ -1,26 +0,0 @@ -from pdb import set_trace as T -import os -import torch - - -def get_policy_names(path: str) -> list: - # Assumeing that all pt files other than trainer_state.pt in the path are policy files - names = [] - for file in os.listdir(path): - if file.endswith(".pt") and file != 'trainer_state.pt': - names.append(file[:-3]) - return sorted(names) - -class PolicyStore: - def __init__(self, path: str): - self.path = path - - def policy_names(self) -> list: - return get_policy_names(self.path) - - def get_policy(self, name: str) -> torch.nn.Module: - path = os.path.join(self.path, name + '.pt') - try: - return torch.load(path) - except: - return torch.load(path, map_location=torch.device('cpu')) diff --git a/pufferlib/postprocess.py b/pufferlib/postprocess.py deleted file mode 100644 index cb311cc145..0000000000 --- a/pufferlib/postprocess.py +++ /dev/null @@ -1,219 +0,0 @@ -from pdb import set_trace as T -import numpy as np -import gymnasium - -import pufferlib.utils - -class ResizeObservation(gymnasium.Wrapper): - '''Fixed downscaling wrapper. Do NOT use gym.wrappers.ResizeObservation - It uses a laughably slow OpenCV resize. -50% on Atari just from that.''' - def __init__(self, env, downscale=2): - super().__init__(env) - self.downscale = downscale - y_size, x_size = env.observation_space.shape - assert y_size % downscale == 0 and x_size % downscale == 0 - y_size = env.observation_space.shape[0] // downscale - x_size = env.observation_space.shape[1] // downscale - self.observation_space = gymnasium.spaces.Box( - low=0, high=255, shape=(y_size, x_size), dtype=np.uint8) - - def reset(self, seed=None, options=None): - obs, info = self.env.reset(seed=seed, options=options) - return obs[::self.downscale, ::self.downscale], info - - def step(self, action): - obs, reward, terminal, truncated, info = self.env.step(action) - return obs[::self.downscale, ::self.downscale], reward, terminal, truncated, info - -class ClipAction(gymnasium.Wrapper): - '''Wrapper for Gymnasium environments that clips actions''' - def __init__(self, env): - self.env = env - assert isinstance(env.action_space, gymnasium.spaces.Box) - dtype_info = np.finfo(env.action_space.dtype) - self.action_space = gymnasium.spaces.Box( - low=dtype_info.min, - high=dtype_info.max, - shape=env.action_space.shape, - dtype=env.action_space.dtype, - ) - - def step(self, action): - action = np.clip(action, self.env.action_space.low, self.env.action_space.high) - return self.env.step(action) - - -class EpisodeStats(gymnasium.Wrapper): - '''Wrapper for Gymnasium environments that stores - episodic returns and lengths in infos''' - def __init__(self, env): - self.env = env - self.observation_space = env.observation_space - self.action_space = env.action_space - self.reset() - - def reset(self, seed=None, options=None): - self.info = dict(episode_return=[], episode_length=0) - # TODO: options - return self.env.reset(seed=seed)#, options=options) - - def step(self, action): - observation, reward, terminated, truncated, info = super().step(action) - - for k, v in pufferlib.utils.unroll_nested_dict(info): - if k not in self.info: - self.info[k] = [] - - self.info[k].append(v) - - self.info['episode_return'].append(reward) - self.info['episode_length'] += 1 - - info = {} - if terminated or truncated: - for k, v in self.info.items(): - try: - info[k] = sum(v) - continue - except TypeError: - pass - - if isinstance(v, str): - info[k] = v - continue - - try: - x = int(v) # probably a value - info[k] = v - continue - except TypeError: - pass - - return observation, reward, terminated, truncated, info - -class PettingZooWrapper: - '''PettingZoo does not provide a ParallelEnv wrapper. This code is adapted from - their AEC wrapper, to prevent unneeded conversions to/from AEC''' - def __init__(self, env): - self.env = env - - def __getattr__(self, name): - '''Returns an attribute with ``name``, unless ``name`` starts with an underscore.''' - if name.startswith('_') and name != '_cumulative_rewards': - raise AttributeError(f'accessing private attribute "{name}" is prohibited') - return getattr(self.env, name) - - @property - def unwrapped(self): - return self.env.unwrapped - - def close(self): - self.env.close() - - def render(self): - return self.env.render() - - def reset(self, seed=None, options=None): - try: - return self.env.reset(seed=seed, options=options) - except TypeError: - return self.env.reset(seed=seed) - - def observe(self, agent): - return self.env.observe(agent) - - def state(self): - return self.env.state() - - def step(self, action): - return self.env.step(action) - - def observation_space(self, agent): - return self.env.observation_space(agent) - - def action_space(self, agent): - return self.env.action_space(agent) - - def __str__(self) -> str: - '''Returns a name which looks like: "max_observation".''' - return f'{type(self).__name__}<{str(self.env)}>' - -class MeanOverAgents(PettingZooWrapper): - '''Averages over agent infos''' - def _mean(self, infos): - list_infos = {} - for agent, info in infos.items(): - for k, v in info.items(): - if k not in list_infos: - list_infos[k] = [] - - list_infos[k].append(v) - - mean_infos = {} - for k, v in list_infos.items(): - try: - mean_infos[k] = np.mean(v) - except: - pass - - return mean_infos - - def reset(self, seed=None, options=None): - observations, infos = super().reset(seed, options) - infos = self._mean(infos) - return observations, infos - - def step(self, actions): - observations, rewards, terminations, truncations, infos = super().step(actions) - infos = self._mean(infos) - return observations, rewards, terminations, truncations, infos - -class MultiagentEpisodeStats(PettingZooWrapper): - '''Wrapper for PettingZoo environments that stores - episodic returns and lengths in infos''' - def reset(self, seed=None, options=None): - observations, infos = super().reset(seed=seed, options=options) - self.infos = { - agent: dict(episode_return=[], episode_length=0) - for agent in self.possible_agents - } - return observations, infos - - def step(self, actions): - observations, rewards, terminations, truncations, infos = super().step(actions) - - all_infos = {} - for agent in infos: - agent_info = self.infos[agent] - for k, v in pufferlib.utils.unroll_nested_dict(infos[agent]): - if k not in agent_info: - agent_info[k] = [] - - agent_info[k].append(v) - - # Saved to self. TODO: Clean up - agent_info['episode_return'].append(rewards[agent]) - agent_info['episode_length'] += 1 - - agent_info = {} - all_infos[agent] = agent_info - if terminations[agent] or truncations[agent]: - for k, v in self.infos[agent].items(): - try: - agent_info[k] = sum(v) - continue - except TypeError: - pass - - if isinstance(v, str): - agent_info[k] = v - continue - - try: - x = int(v) # probably a value - agent_info[k] = v - continue - except TypeError: - pass - - return observations, rewards, terminations, truncations, all_infos diff --git a/pufferlib/vector.py b/pufferlib/vector.py index f4ae689415..b6a9002f1a 100644 --- a/pufferlib/vector.py +++ b/pufferlib/vector.py @@ -6,11 +6,8 @@ import time import psutil -from pufferlib import namespace from pufferlib.emulation import GymnasiumPufferEnv, PettingZooPufferEnv -from pufferlib.environment import PufferEnv, set_buffers -from pufferlib.exceptions import APIUsageError -from pufferlib.namespace import Namespace +from pufferlib import PufferEnv, set_buffers import pufferlib.spaces import gymnasium @@ -24,19 +21,19 @@ def recv_precheck(vecenv): if vecenv.flag != RECV: - raise APIUsageError('Call reset before stepping') + raise pufferlib.APIUsageError('Call reset before stepping') vecenv.flag = SEND def send_precheck(vecenv, actions): if vecenv.flag != SEND: - raise APIUsageError('Call (async) reset + recv before sending') + raise pufferlib.APIUsageError('Call (async) reset + recv before sending') actions = np.asarray(actions) if not vecenv.initialized: vecenv.initialized = True if not vecenv.action_space.contains(actions): - raise APIUsageError('Actions do not match action space') + raise pufferlib.APIUsageError('Actions do not match action space') vecenv.flag = RECV return actions @@ -77,7 +74,7 @@ def __init__(self, env_creators, env_args, env_kwargs, num_envs, buf=None, seed= ptr = 0 for i in range(num_envs): end = ptr + self.driver_env.num_agents - buf_i = namespace( + buf_i = pufferlib.namespace( observations=self.observations[ptr:end], rewards=self.rewards[ptr:end], terminals=self.terminals[ptr:end], @@ -179,7 +176,7 @@ def _worker_process(env_creators, env_args, env_kwargs, obs_shape, obs_dtype, at shape = (num_workers, num_envs*num_agents) atn_arr = np.ndarray((*shape, *atn_shape), dtype=atn_dtype, buffer=shm.actions)[worker_idx] - buf = namespace( + buf = pufferlib.namespace( observations=np.ndarray((*shape, *obs_shape), dtype=obs_dtype, buffer=shm.observations)[worker_idx], rewards=np.ndarray(shape, dtype=np.float32, buffer=shm.rewards)[worker_idx], @@ -249,7 +246,7 @@ def __init__(self, env_creators, env_args, env_kwargs, import psutil cpu_cores = psutil.cpu_count(logical=False) if num_workers > cpu_cores and not overwork: - raise APIUsageError(' '.join([ + raise pufferlib.APIUsageError(' '.join([ f'num_workers ({num_workers}) > hardware cores ({cpu_cores}) is disallowed by default.', 'PufferLib multiprocessing is heavily optimized for 1 process per hardware core.', 'If you really want to do this, set overwork=True (--vec-overwork in our demo.py).', @@ -258,7 +255,7 @@ def __init__(self, env_creators, env_args, env_kwargs, num_batches = num_envs / batch_size if zero_copy and num_batches != int(num_batches): # This is so you can have n equal buffers - raise APIUsageError( + raise pufferlib.APIUsageError( 'zero_copy: num_envs must be divisible by batch_size') self.num_environments = num_envs @@ -300,7 +297,7 @@ def __init__(self, env_creators, env_args, env_kwargs, from multiprocessing import RawArray, set_start_method # Mac breaks without setting fork... but setting it breaks sweeps on 2nd run #set_start_method('fork') - self.shm = namespace( + self.shm = pufferlib.namespace( observations=RawArray(obs_ctype, num_agents * int(np.prod(obs_shape))), actions=RawArray(atn_ctype, num_agents * int(np.prod(atn_shape))), rewards=RawArray('f', num_agents), @@ -315,7 +312,7 @@ def __init__(self, env_creators, env_args, env_kwargs, self.atn_batch_shape = (self.workers_per_batch, agents_per_worker, *atn_shape) self.actions = np.ndarray((*shape, *atn_shape), dtype=atn_dtype, buffer=self.shm.actions) - self.buf = namespace( + self.buf = pufferlib.namespace( observations=np.ndarray((*shape, *obs_shape), dtype=obs_dtype, buffer=self.shm.observations), rewards=np.ndarray(shape, dtype=np.float32, buffer=self.shm.rewards), @@ -631,24 +628,24 @@ def close(self): def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=PufferEnv, num_envs=1, seed=0, **kwargs): if num_envs < 1: - raise APIUsageError('num_envs must be at least 1') + raise pufferlib.APIUsageError('num_envs must be at least 1') if num_envs != int(num_envs): - raise APIUsageError('num_envs must be an integer') + raise pufferlib.APIUsageError('num_envs must be an integer') if isinstance(backend, str): try: backend = getattr(pufferlib.vector, backend) except: - raise APIUsageError(f'Invalid backend: {backend}') + raise pufferlib.APIUsageError(f'Invalid backend: {backend}') if backend == PufferEnv: env_args = env_args or [] env_kwargs = env_kwargs or {} vecenv = env_creator_or_creators(*env_args, **env_kwargs) if not isinstance(vecenv, PufferEnv): - raise APIUsageError('Native vectorization requires a native PufferEnv. Use Serial or Multiprocessing instead.') + raise pufferlib.APIUsageError('Native vectorization requires a native PufferEnv. Use Serial or Multiprocessing instead.') if num_envs != 1: - raise APIUsageError('Native vectorization is for PufferEnvs that handle all per-process vectorization internally. If you want to run multiple separate Python instances on a single process, use Serial or Multiprocessing instead') + raise pufferlib.APIUsageError('Native vectorization is for PufferEnvs that handle all per-process vectorization internally. If you want to run multiple separate Python instances on a single process, use Serial or Multiprocessing instead') return vecenv @@ -657,7 +654,7 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer # TODO: None? envs_per_worker = num_envs / num_workers if envs_per_worker != int(envs_per_worker): - raise APIUsageError('num_envs must be divisible by num_workers') + raise pufferlib.APIUsageError('num_envs must be divisible by num_workers') if 'batch_size' in kwargs: batch_size = kwargs['batch_size'] @@ -665,7 +662,7 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer batch_size = num_envs if batch_size % envs_per_worker != 0: - raise APIUsageError( + raise pufferlib.APIUsageError( 'batch_size must be divisible by (num_envs / num_workers)') @@ -683,19 +680,19 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer env_creators = env_creator_or_creators if len(env_creators) != num_envs: - raise APIUsageError('env_creators must be a list of length num_envs') + raise pufferlib.APIUsageError('env_creators must be a list of length num_envs') if len(env_args) != num_envs: - raise APIUsageError('env_args must be a list of length num_envs') + raise pufferlib.APIUsageError('env_args must be a list of length num_envs') if len(env_kwargs) != num_envs: - raise APIUsageError('env_kwargs must be a list of length num_envs') + raise pufferlib.APIUsageError('env_kwargs must be a list of length num_envs') for i in range(num_envs): if not callable(env_creators[i]): - raise APIUsageError('env_creators must be a list of callables') + raise pufferlib.APIUsageError('env_creators must be a list of callables') if not isinstance(env_args[i], (list, tuple)): - raise APIUsageError('env_args must be a list of lists or tuples') - if not isinstance(env_kwargs[i], (dict, Namespace)): - raise APIUsageError('env_kwargs must be a list of dictionaries') + raise pufferlib.APIUsageError('env_args must be a list of lists or tuples') + if not isinstance(env_kwargs[i], (dict, pufferlib.Namespace)): + raise pufferlib.APIUsageError('env_kwargs must be a list of dictionaries') # Keeps batch size consistent when debugging with Serial backend if backend is Serial and 'batch_size' in kwargs: @@ -707,7 +704,7 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer # Sanity check args for k in kwargs: if k not in ['num_workers', 'batch_size', 'zero_copy', 'overwork', 'backend']: - raise APIUsageError(f'Invalid argument: {k}') + raise pufferlib.APIUsageError(f'Invalid argument: {k}') # TODO: First step action space check @@ -720,28 +717,28 @@ def make_seeds(seed, num_envs): err = f'seed {seed} must be an integer or a list of integers' if isinstance(seed, (list, tuple)): if len(seed) != num_envs: - raise APIUsageError(err) + raise pufferlib.APIUsageError(err) return seed - raise APIUsageError(err) + raise pufferlib.APIUsageError(err) def check_envs(envs, driver): valid = (PufferEnv, GymnasiumPufferEnv, PettingZooPufferEnv) if not isinstance(driver, valid): - raise APIUsageError(f'env_creator must be {valid}') + raise pufferlib.APIUsageError(f'env_creator must be {valid}') driver_obs = driver.single_observation_space driver_atn = driver.single_action_space for env in envs: if not isinstance(env, valid): - raise APIUsageError(f'env_creators must be {valid}') + raise pufferlib.APIUsageError(f'env_creators must be {valid}') obs_space = env.single_observation_space if obs_space != driver_obs: - raise APIUsageError(f'\n{obs_space}\n{driver_obs} obs space mismatch') + raise pufferlib.APIUsageError(f'\n{obs_space}\n{driver_obs} obs space mismatch') atn_space = env.single_action_space if atn_space != driver_atn: - raise APIUsageError(f'\n{atn_space}\n{driver_atn} atn space mismatch') + raise pufferlib.APIUsageError(f'\n{atn_space}\n{driver_atn} atn space mismatch') def autotune(env_creator, batch_size, max_envs=194, model_forward_s=0.0, max_env_ram_gb=32, max_batch_vram_gb=0.05, time_per_test=5): diff --git a/pufferlib/wrappers.py b/pufferlib/wrappers.py deleted file mode 100644 index 3bda419715..0000000000 --- a/pufferlib/wrappers.py +++ /dev/null @@ -1,57 +0,0 @@ -from pdb import set_trace as T - -class GymToGymnasium: - def __init__(self, env): - self.env = env - self.observation_space = env.observation_space - self.action_space = env.action_space - self.render = env.render - self.metadata = env.metadata - - def reset(self, seed=None, options=None): - if seed is not None: - ob = self.env.reset(seed=seed) - else: - ob = self.env.reset() - return ob, {} - - def step(self, action): - observation, reward, done, info = self.env.step(action) - return observation, reward, done, False, info - - def close(self): - self.env.close() - -class PettingZooTruncatedWrapper: - def __init__(self, env): - self.env = env - self.observation_space = env.observation_space - self.action_space = env.action_space - self.render = env.render - - @property - def render_mode(self): - return self.env.render_mode - - @property - def possible_agents(self): - return self.env.possible_agents - - @property - def agents(self): - return self.env.agents - - def reset(self, seed=None): - if seed is not None: - ob, info = self.env.reset(seed=seed) - else: - ob, info = self.env.reset() - info = {k: {} for k in ob} - return ob, info - - def step(self, actions): - observations, rewards, terminals, truncations, infos = self.env.step(actions) - return observations, rewards, terminals, truncations, infos - - def close(self): - self.env.close() From 65014b58de787e5c039a3a9ad6bb8a90b4ebc184 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 5 May 2025 19:45:54 +0000 Subject: [PATCH 21/63] Delete unused utils, move rest to pufferlib --- clean_pufferl.py | 5 +- pufferlib/emulation.py | 18 +- pufferlib/utils.py | 410 ----------------------------------------- tests/test.py | 94 ++++++++++ 4 files changed, 112 insertions(+), 415 deletions(-) delete mode 100644 pufferlib/utils.py diff --git a/clean_pufferl.py b/clean_pufferl.py index 1219fda3f4..a1a0b473e8 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -25,7 +25,6 @@ import torch.utils.cpp_extension import pufferlib -import pufferlib.utils import pufferlib.pytorch import pufferlib.sweep import pufferlib.vector @@ -183,7 +182,7 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.wandb = wandb if neptune: self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.tag) - for k, v in pufferlib.utils.unroll_nested_dict(args): + for k, v in pufferlib.unroll_nested_dict(args): self.neptune[k].append(v) elif wandb: self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.tag) @@ -266,7 +265,7 @@ def evaluate(self): profile('eval_misc', epoch) for i in info: - for k, v in pufferlib.utils.unroll_nested_dict(i): + for k, v in pufferlib.unroll_nested_dict(i): if isinstance(v, np.ndarray): v = v.tolist() elif isinstance(v, (list, tuple)): diff --git a/pufferlib/emulation.py b/pufferlib/emulation.py index 9c35805448..686a24d573 100644 --- a/pufferlib/emulation.py +++ b/pufferlib/emulation.py @@ -8,7 +8,6 @@ import pufferlib import pufferlib.spaces -from pufferlib import utils from pufferlib.spaces import Discrete, Tuple, Dict def emulate(struct, sample): @@ -55,6 +54,7 @@ def nativize(arr, space, struct_dtype): struct = np.asarray(arr).view(struct_dtype)[0] return _nativize(struct, space) +# TODO: Uncomment? ''' try: from pufferlib.extensions import emulate, nativize @@ -62,6 +62,20 @@ def nativize(arr, space, struct_dtype): warnings.warn('PufferLib Cython extensions not installed. Using slow Python versions') ''' +def get_dtype_bounds(dtype): + if dtype == bool: + return 0, 1 + elif np.issubdtype(dtype, np.integer): + return np.iinfo(dtype).min, np.iinfo(dtype).max + elif np.issubdtype(dtype, np.unsignedinteger): + return np.iinfo(dtype).min, np.iinfo(dtype).max + elif np.issubdtype(dtype, np.floating): + # Gym fails on float64 + return np.finfo(np.float32).min, np.finfo(np.float32).max + else: + raise ValueError(f"Unsupported dtype: {dtype}") + + def dtype_from_space(space): if isinstance(space, pufferlib.spaces.Tuple): dtype = [] @@ -107,7 +121,7 @@ def emulate_observation_space(space): else: dtype = np.dtype(np.uint8) - mmin, mmax = utils._get_dtype_bounds(dtype) + mmin, mmax = get_dtype_bounds(dtype) numel = emulated_dtype.itemsize // dtype.itemsize emulated_space = gymnasium.spaces.Box(low=mmin, high=mmax, shape=(numel,), dtype=dtype) return emulated_space, emulated_dtype diff --git a/pufferlib/utils.py b/pufferlib/utils.py deleted file mode 100644 index 1150c76311..0000000000 --- a/pufferlib/utils.py +++ /dev/null @@ -1,410 +0,0 @@ -from pdb import set_trace as T - -from collections import OrderedDict -from contextlib import nullcontext - -import numpy as np - -import time -import os -import sys -import pickle -import subprocess -from contextlib import redirect_stdout, redirect_stderr, contextmanager -from io import StringIO -import psutil - -import warnings -from functools import wraps - -import functools -import inspect -import importlib - -def validate_args(fn, kwargs): - fn_kwargs = get_init_args(fn) - for param, val in kwargs.items(): - if param not in fn_kwargs: - raise ValueError( - f'Invalid argument\n{param}\nto\n{fn}\n' - f'which takes \n{fn_kwargs}\n' - f'Double check your config' - ) - -def get_init_args(fn): - if fn is None: - return {} - - if isinstance(fn, functools.partial): - return fn.keywords - - sig = inspect.signature(fn) - kwargs = {} - for name, param in sig.parameters.items(): - if name in ['env', 'policy']: - # Hack to avoid duplicate kwargs - continue - if param.kind == inspect.Parameter.VAR_POSITIONAL: - continue - elif param.kind == inspect.Parameter.VAR_KEYWORD: - continue - else: - kwargs[name] = param.default if param.default is not inspect.Parameter.empty else None - return kwargs - - -def unroll_nested_dict(d): - if not isinstance(d, dict): - return d - - for k, v in d.items(): - if isinstance(v, dict): - for k2, v2 in unroll_nested_dict(v): - yield f"{k}/{k2}", v2 - else: - yield k, v - -def install_requirements(env): - '''Pip install dependencies for specified environment''' - pip_install_cmd = [sys.executable, "-m", "pip", "install", "-e" f".[{env}]"] - proc = subprocess.run(pip_install_cmd, capture_output=True, text=True) - if proc.returncode != 0: - raise RuntimeError(f"Error installing requirements: {proc.stderr}") - -def install_and_import(package): - '''Install and import a package''' - try: - module = importlib.import_module(package) - except ImportError: - install_requirements(package) - module = importlib.import_module(package) - - return module - -def silence_warnings(original_func, category=DeprecationWarning): - @wraps(original_func) - def wrapper(*args, **kwargs): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", category=category) - return original_func(*args, **kwargs) - return wrapper - -def check_env(env): - #assert issubclass(env_cls, gym.Env), "Not a gymnasium env (are you on old gym?)" - assert hasattr(env, 'possible_agents') - assert len(env.possible_agents) - obs_space = env.observation_space(env.possible_agents[0]) - atn_space = env.action_space(env.possible_agents[0]) - for e in env.possible_agents: - assert env.observation_space(e) == obs_space, 'All agents must have same obs space' - assert env.action_space(e) == atn_space, 'All agents must have same atn space' - -def make_zeros_like(data): - if isinstance(data, dict): - return {k: make_zeros_like(v) for k, v in data.items()} - elif isinstance(data, (list, tuple)): - return [make_zeros_like(v) for v in data] - elif isinstance(data, np.ndarray): - return np.zeros_like(data) - elif isinstance(data, (int, float)): - return 0 - else: - raise ValueError(f'Unsupported type: {type(data)}') - -def compare_arrays(array_1, array_2): - assert isinstance(array_1, np.ndarray) - assert isinstance(array_2, np.ndarray) - assert array_1.shape == array_2.shape - return np.allclose(array_1, array_2) - -def compare_dicts(dict_1, dict_2, idx): - assert isinstance(dict_1, (dict, OrderedDict)) - assert isinstance(dict_2, (dict, OrderedDict)) - - if not all(k in dict_2 for k in dict_1): - raise ValueError("Keys do not match between dictionaries.") - - for k, v in dict_1.items(): - if not compare_space_samples(v, dict_2[k], idx): - return False - - return True - -def compare_lists(list_1, list_2, idx): - assert isinstance(list_1, (list, tuple)) - assert isinstance(list_2, (list, tuple)) - - if len(list_1) != len(list_2): - raise ValueError("Lengths do not match between lists/tuples.") - - for v1, v2 in zip(list_1, list_2): - if not compare_space_samples(v1, v2, idx): - return False - - return True - -def compare_space_samples(sample_1, sample_2, sample_2_batch_idx=None): - '''Compare two samples from the same space - - Optionally, sample_2 may be a batch of samples from the same space - concatenated along the first dimension of the leaves. In this case, - sample_2_batch_idx specifies which sample to compare. - ''' - if isinstance(sample_1, (dict, OrderedDict)): - return compare_dicts(sample_1, sample_2, sample_2_batch_idx) - elif isinstance(sample_1, (list, tuple)): - return compare_lists(sample_1, sample_2, sample_2_batch_idx) - elif isinstance(sample_1, np.ndarray): - assert isinstance(sample_2, np.ndarray) - if sample_2_batch_idx is not None: - sample_2 = sample_2[sample_2_batch_idx] - return compare_arrays(sample_1, sample_2) - elif isinstance(sample_1, (int, float)): - if sample_2_batch_idx is not None: - sample_2 = sample_2[sample_2_batch_idx] - if isinstance(sample_2, np.ndarray): - assert sample_2.size == 1, "Cannot compare scalar to non-scalar." - sample_2 = sample_2[0] - return sample_1 == sample_2 - else: - raise ValueError(f"Unsupported type: {type(sample_1)}") - -def _get_dtype_bounds(dtype): - if dtype == bool: - return 0, 1 - elif np.issubdtype(dtype, np.integer): - return np.iinfo(dtype).min, np.iinfo(dtype).max - elif np.issubdtype(dtype, np.unsignedinteger): - return np.iinfo(dtype).min, np.iinfo(dtype).max - elif np.issubdtype(dtype, np.floating): - # Gym fails on float64 - return np.finfo(np.float32).min, np.finfo(np.float32).max - else: - raise ValueError(f"Unsupported dtype: {dtype}") - -def is_dict_space(space): - # Compatible with gym/gymnasium - return type(space).__name__ == 'Dict' - -def is_multiagent(env): - import pettingzoo - import gym - if inspect.isclass(env): - env_cls = env - else: - env_cls = type(env) - - if not issubclass(env_cls, pettingzoo.AECEnv) and not issubclass(env_cls, pettingzoo.ParallelEnv): - assert issubclass(env_cls, gym.Env), 'Environment must subclass pettingzoo.AECEnv/ParallelEnv or gym.Env' - return False - return True - -def current_datetime(): - return time.strftime('%Y-%m-%d_%H-%M-%S', time.localtime()) - -def myprint(d): - stack = d.items() - while stack: - k, v = stack.pop() - if isinstance(v, dict): - stack.extend(v.iteritems()) - else: - print("%s: %s" % (k, v)) - -class RandomState: - def __init__(self, seed): - self.rng = np.random.RandomState(seed) - - def random(self): - return self.rng.random() - - def probabilistic_round(self, n): - frac, integer = np.modf(n) - if self.random() < frac: - return int(integer) + 1 - else: - return int(integer) - - def sample(self, ary, n): - n_rounded = self.probabilistic_round(n) - return self.rng.choice(ary, n_rounded, replace=False).tolist() - - def choice(self, ary): - return self.sample(ary, 1)[0] - -def format_bytes(size): - if size >= 1024 ** 4: - return f'{size / (1024 ** 4):.2f} TB' - elif size >= 1024 ** 3: - return f'{size / (1024 ** 3):.2f} GB' - elif size >= 1024 ** 2: - return f'{size / (1024 ** 2):.2f} MB' - elif size >= 1024: - return f'{size / 1024:.2f} KB' - else: - return f'{size} B' - -# TODO: 5% perf gain by doing cuda sync less frequently -class Profiler: - def __init__(self, elapsed=True, calls=True, memory=False, - pytorch_memory=False, sync_cuda=True, frequency=10, amp_context=nullcontext()): - self.elapsed = 0 if elapsed else None - self.calls = 0 if calls else None - self.memory = None - self.pytorch_memory = None - self.prev = 0 - self.delta = 0 - - self.track_elapsed = elapsed - self.track_calls = calls - self.track_memory = memory - self.track_pytorch_memory = pytorch_memory - self.sync_cuda = sync_cuda - self.frequency = frequency - self.epoch = 0 - - if memory: - self.process = psutil.Process() - - if pytorch_memory or sync_cuda: - import torch - self.torch = torch - - self.amp_context = amp_context - - ''' - @property - def serial(self): - return { - 'elapsed': self.elapsed, - 'calls': self.calls, - 'memory': self.memory, - 'pytorch_memory': self.pytorch_memory, - 'delta': self.delta - } - - @property - def delta(self): - ret = self.elapsed - self.prev if self.elapsed is not None else None - self.prev = self.elapsed - return ret - ''' - - def __call__(self, epoch): - self.epoch = epoch - return self - - def __enter__(self): - if self.epoch % self.frequency != 0: - return self - - if self.sync_cuda: - self.torch.cuda.synchronize() - self.amp_context.__enter__() - if self.track_elapsed: - self.start_time = time.perf_counter() - if self.track_memory: - self.start_mem = self.process.memory_info().rss - if self.track_pytorch_memory: - self.start_torch_mem = self.torch.cuda.memory_allocated() - return self - - def __exit__(self, *args): - if self.epoch % self.frequency != 0: - return self - - self.amp_context.__exit__(None, None, None) - if self.sync_cuda: - self.torch.cuda.synchronize() - if self.track_elapsed: - self.end_time = time.perf_counter() - self.delta += self.end_time - self.start_time - self.elapsed += self.delta - if self.track_calls: - self.calls += 1 - if self.track_memory: - self.end_mem = self.process.memory_info().rss - self.memory = self.end_mem - self.start_mem - if self.track_pytorch_memory: - self.end_torch_mem = self.torch.cuda.memory_allocated() - self.pytorch_memory = self.end_torch_mem - self.start_torch_mem - - def __repr__(self): - parts = [] - if self.track_elapsed: - parts.append(f'Elapsed: {self.elapsed:.4f} s') - if self.track_calls: - parts.append(f'Calls: {self.calls}') - if self.track_memory: - parts.append(f'Memory: {format_bytes(self.memory)}') - if self.track_pytorch_memory: - parts.append(f'PyTorch Memory: {format_bytes(self.pytorch_memory)}') - return ", ".join(parts) - - # Aliases for use without context manager - start = __enter__ - stop = __exit__ - -def profile(func): - name = func.__name__ - - def wrapper(*args, **kwargs): - self = args[0] - - if not hasattr(self, '_timers'): - self._timers = {} - - if name not in self._timers: - self._timers[name] = Profiler() - - timer = self._timers[name] - - with timer: - result = func(*args, **kwargs) - - return result - - return wrapper - -def aggregate_profilers(profiler_dicts): - merged = {} - - for key in list(profiler_dicts[0].keys()): - merged[key] = Profiler() - for prof_dict in profiler_dicts: - merged[key].elapsed += prof_dict[key].elapsed - merged[key].calls += prof_dict[key].calls - - return merged - -class Suppress(): - def __init__(self): - self.f = StringIO() - self.null_1 = os.open(os.devnull, os.O_WRONLY | os.O_TRUNC | os.O_CREAT) - self.null_2 = os.open(os.devnull, os.O_WRONLY | os.O_TRUNC | os.O_CREAT) - - def __enter__(self): - # Suppress C library outputs - self.orig_stdout = os.dup(1) - self.orig_stderr = os.dup(2) - os.dup2(self.null_1, 1) - os.dup2(self.null_2, 2) - - # Suppress Python outputs - self._stdout_redirector = redirect_stdout(self.f) - self._stderr_redirector = redirect_stderr(self.f) - self._stdout_redirector.__enter__() - self._stderr_redirector.__enter__() - - def __exit__(self, exc_type, exc_val, exc_tb): - # Enable C library outputs - os.dup2(self.orig_stdout, 1) - os.dup2(self.orig_stderr, 2) - os.close(self.orig_stdout) - os.close(self.orig_stderr) - os.close(self.null_1) - os.close(self.null_2) - - # Enable Python outputs - self._stdout_redirector.__exit__(exc_type, exc_val, exc_tb) - self._stderr_redirector.__exit__(exc_type, exc_val, exc_tb) diff --git a/tests/test.py b/tests/test.py index 813bd8dbe1..1890c3c2ef 100644 --- a/tests/test.py +++ b/tests/test.py @@ -13,6 +13,100 @@ import warnings warnings.filterwarnings("ignore") +class RandomState: + def __init__(self, seed): + self.rng = np.random.RandomState(seed) + + def random(self): + return self.rng.random() + + def probabilistic_round(self, n): + frac, integer = np.modf(n) + if self.random() < frac: + return int(integer) + 1 + else: + return int(integer) + + def sample(self, ary, n): + n_rounded = self.probabilistic_round(n) + return self.rng.choice(ary, n_rounded, replace=False).tolist() + + def choice(self, ary): + return self.sample(ary, 1)[0] + + +# TODO: Fix this. Was in utils.py. Only used for tests +def make_zeros_like(data): + if isinstance(data, dict): + return {k: make_zeros_like(v) for k, v in data.items()} + elif isinstance(data, (list, tuple)): + return [make_zeros_like(v) for v in data] + elif isinstance(data, np.ndarray): + return np.zeros_like(data) + elif isinstance(data, (int, float)): + return 0 + else: + raise ValueError(f'Unsupported type: {type(data)}') + +def compare_arrays(array_1, array_2): + assert isinstance(array_1, np.ndarray) + assert isinstance(array_2, np.ndarray) + assert array_1.shape == array_2.shape + return np.allclose(array_1, array_2) + +def compare_dicts(dict_1, dict_2, idx): + assert isinstance(dict_1, (dict, OrderedDict)) + assert isinstance(dict_2, (dict, OrderedDict)) + + if not all(k in dict_2 for k in dict_1): + raise ValueError("Keys do not match between dictionaries.") + + for k, v in dict_1.items(): + if not compare_space_samples(v, dict_2[k], idx): + return False + + return True + +def compare_lists(list_1, list_2, idx): + assert isinstance(list_1, (list, tuple)) + assert isinstance(list_2, (list, tuple)) + + if len(list_1) != len(list_2): + raise ValueError("Lengths do not match between lists/tuples.") + + for v1, v2 in zip(list_1, list_2): + if not compare_space_samples(v1, v2, idx): + return False + + return True + +def compare_space_samples(sample_1, sample_2, sample_2_batch_idx=None): + '''Compare two samples from the same space + + Optionally, sample_2 may be a batch of samples from the same space + concatenated along the first dimension of the leaves. In this case, + sample_2_batch_idx specifies which sample to compare. + ''' + if isinstance(sample_1, (dict, OrderedDict)): + return compare_dicts(sample_1, sample_2, sample_2_batch_idx) + elif isinstance(sample_1, (list, tuple)): + return compare_lists(sample_1, sample_2, sample_2_batch_idx) + elif isinstance(sample_1, np.ndarray): + assert isinstance(sample_2, np.ndarray) + if sample_2_batch_idx is not None: + sample_2 = sample_2[sample_2_batch_idx] + return compare_arrays(sample_1, sample_2) + elif isinstance(sample_1, (int, float)): + if sample_2_batch_idx is not None: + sample_2 = sample_2[sample_2_batch_idx] + if isinstance(sample_2, np.ndarray): + assert sample_2.size == 1, "Cannot compare scalar to non-scalar." + sample_2 = sample_2[0] + return sample_1 == sample_2 + else: + raise ValueError(f"Unsupported type: {type(sample_1)}") + + def test_gymnasium_emulation(env_cls, steps=100): raw_env = env_cls() From 41cbb4c898278d8a08a518660f90f46f2fba558f Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 15:46:27 +0000 Subject: [PATCH 22/63] Robustify nans --- clean_pufferl.py | 1 + pufferlib/pytorch.py | 1 + pufferlib/sweep.py | 6 +++--- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index a1a0b473e8..8966a8442f 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -482,6 +482,7 @@ def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio elif method == 'prio': adv = advantages.abs().sum(axis=1) probs = adv**config.prio_alpha + probs = torch.nan_to_num(probs, 0, 0, 0) probs = (probs + 1e-6)/(probs.sum() + 1e-6) idx = torch.multinomial(probs, n) elif method == 'multinomial': diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index 927e7cb4bc..d04987a428 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -296,6 +296,7 @@ def sample_logits(logits: Union[torch.Tensor, List[torch.Tensor]], probs = logits_to_probs(logits) if action is None: + probs = torch.nan_to_num(probs, 1e-8, 1e-8, 1e-8) action = torch.multinomial(probs.reshape(-1, probs.shape[-1]), 1, replacement=True) action = action.reshape(probs.shape[:-1]) else: diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 3e64fedef1..7a5703ba91 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -405,7 +405,7 @@ def suggest(self, fill): raise ValueError(f'Max score {max_score} is greater than max score in data {np.max(y)}') # Linearize - y_norm = (y - min_score) / (max_score - min_score) + y_norm = (y - min_score) / (np.abs((max_score - min_score)) + 1e-6) self.gp_score.set_data(params, torch.from_numpy(y_norm)) self.gp_score.train() @@ -420,7 +420,7 @@ def suggest(self, fill): # Linear input norm creates clean 1 mean fn log_c_min = np.min(log_c) log_c_max = np.max(log_c) - log_c_norm = (log_c - log_c_min) / (log_c_max - log_c_min) + log_c_norm = (log_c - log_c_min) / ((np.abs(log_c_max - log_c_min)) + 1e-6) self.gp_cost.mean_function = lambda x: 1 self.gp_cost.set_data(params, torch.from_numpy(log_c_norm)) @@ -453,7 +453,7 @@ def suggest(self, fill): gp_c_min = np.min(gp_c) gp_c_max = np.max(gp_c) - gp_c_norm = (gp_c - gp_c_min) / (gp_c_max - gp_c_min) + gp_c_norm = (gp_c - gp_c_min) / ((np.abs(gp_c_max - gp_c_min)) + 1e-6) pareto_y = y[pareto_idxs] pareto_c = c[pareto_idxs] From 1953f39b8fd6b70dcabadf20d9eea4d7654b94fa Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 15:47:35 +0000 Subject: [PATCH 23/63] pufferlib.py --- pufferlib/pufferlib.py | 509 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 509 insertions(+) create mode 100644 pufferlib/pufferlib.py diff --git a/pufferlib/pufferlib.py b/pufferlib/pufferlib.py new file mode 100644 index 0000000000..cdfed44f90 --- /dev/null +++ b/pufferlib/pufferlib.py @@ -0,0 +1,509 @@ +import os +import sys +import warnings + +from contextlib import redirect_stdout, redirect_stderr, contextmanager +from types import SimpleNamespace +from collections.abc import Mapping +from io import StringIO +from functools import wraps + +import numpy as np +import gymnasium + +import pufferlib.spaces + +ENV_ERROR = ''' +Environment missing required attribute {}. The most common cause is +calling super() before you have assigned the attribute. +''' + + +def set_buffers(env, buf=None): + if buf is None: + obs_space = env.single_observation_space + env.observations = np.zeros((env.num_agents, *obs_space.shape), dtype=obs_space.dtype) + env.rewards = np.zeros(env.num_agents, dtype=np.float32) + env.terminals = np.zeros(env.num_agents, dtype=bool) + env.truncations = np.zeros(env.num_agents, dtype=bool) + env.masks = np.ones(env.num_agents, dtype=bool) + + # TODO: Major kerfuffle on inferring action space dtype. This needs some asserts? + atn_space = env.single_action_space + if isinstance(env.single_action_space, pufferlib.spaces.Box): + env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=atn_space.dtype) + else: + env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=np.int32) + else: + env.observations = buf.observations + env.rewards = buf.rewards + env.terminals = buf.terminals + env.truncations = buf.truncations + env.masks = buf.masks + env.actions = buf.actions + +class PufferEnv: + def __init__(self, buf=None): + if not hasattr(self, 'single_observation_space'): + raise APIUsageError(ENV_ERROR.format('single_observation_space')) + if not hasattr(self, 'single_action_space'): + raise APIUsageError(ENV_ERROR.format('single_action_space')) + if not hasattr(self, 'num_agents'): + raise APIUsageError(ENV_ERROR.format('num_agents')) + if self.num_agents < 1: + raise APIUsageError('num_agents must be >= 1') + + if hasattr(self, 'observation_space'): + raise APIUsageError('PufferEnvs must define single_observation_space, not observation_space') + if hasattr(self, 'action_space'): + raise APIUsageError('PufferEnvs must define single_action_space, not action_space') + if not isinstance(self.single_observation_space, pufferlib.spaces.Box): + raise APIUsageError('Native observation_space must be a Box') + if (not isinstance(self.single_action_space, pufferlib.spaces.Discrete) + and not isinstance(self.single_action_space, pufferlib.spaces.MultiDiscrete) + and not isinstance(self.single_action_space, pufferlib.spaces.Box)): + raise APIUsageError('Native action_space must be a Discrete, MultiDiscrete, or Box') + + set_buffers(self, buf) + + self.action_space = pufferlib.spaces.joint_space(self.single_action_space, self.num_agents) + self.observation_space = pufferlib.spaces.joint_space(self.single_observation_space, self.num_agents) + self.agent_ids = np.arange(self.num_agents) + + @property + def emulated(self): + '''Native envs do not use emulation''' + return False + + @property + def done(self): + '''Native envs handle resets internally''' + return False + + @property + def driver_env(self): + '''For compatibility with Multiprocessing''' + return self + + def reset(self, seed=None): + raise NotImplementedError + + def step(self, actions): + raise NotImplementedError + + def close(self): + raise NotImplementedError + + def async_reset(self, seed=None): + _, self.infos = self.reset(seed) + assert isinstance(self.infos, list), 'PufferEnvs must return info as a list of dicts' + + def send(self, actions): + _, _, _, _, self.infos = self.step(actions) + assert isinstance(self.infos, list), 'PufferEnvs must return info as a list of dicts' + + def recv(self): + return (self.observations, self.rewards, self.terminals, + self.truncations, self.infos, self.agent_ids, self.masks) +### Postprocessing +class ResizeObservation(gymnasium.Wrapper): + '''Fixed downscaling wrapper. Do NOT use gym.wrappers.ResizeObservation + It uses a laughably slow OpenCV resize. -50% on Atari just from that.''' + def __init__(self, env, downscale=2): + super().__init__(env) + self.downscale = downscale + y_size, x_size = env.observation_space.shape + assert y_size % downscale == 0 and x_size % downscale == 0 + y_size = env.observation_space.shape[0] // downscale + x_size = env.observation_space.shape[1] // downscale + self.observation_space = gymnasium.spaces.Box( + low=0, high=255, shape=(y_size, x_size), dtype=np.uint8) + + def reset(self, seed=None, options=None): + obs, info = self.env.reset(seed=seed, options=options) + return obs[::self.downscale, ::self.downscale], info + + def step(self, action): + obs, reward, terminal, truncated, info = self.env.step(action) + return obs[::self.downscale, ::self.downscale], reward, terminal, truncated, info + +class ClipAction(gymnasium.Wrapper): + '''Wrapper for Gymnasium environments that clips actions''' + def __init__(self, env): + self.env = env + assert isinstance(env.action_space, gymnasium.spaces.Box) + dtype_info = np.finfo(env.action_space.dtype) + self.action_space = gymnasium.spaces.Box( + low=dtype_info.min, + high=dtype_info.max, + shape=env.action_space.shape, + dtype=env.action_space.dtype, + ) + + def step(self, action): + action = np.clip(action, self.env.action_space.low, self.env.action_space.high) + return self.env.step(action) + + +class EpisodeStats(gymnasium.Wrapper): + '''Wrapper for Gymnasium environments that stores + episodic returns and lengths in infos''' + def __init__(self, env): + self.env = env + self.observation_space = env.observation_space + self.action_space = env.action_space + self.reset() + + def reset(self, seed=None, options=None): + self.info = dict(episode_return=[], episode_length=0) + # TODO: options + return self.env.reset(seed=seed)#, options=options) + + def step(self, action): + observation, reward, terminated, truncated, info = super().step(action) + + for k, v in unroll_nested_dict(info): + if k not in self.info: + self.info[k] = [] + + self.info[k].append(v) + + self.info['episode_return'].append(reward) + self.info['episode_length'] += 1 + + info = {} + if terminated or truncated: + for k, v in self.info.items(): + try: + info[k] = sum(v) + continue + except TypeError: + pass + + if isinstance(v, str): + info[k] = v + continue + + try: + x = int(v) # probably a value + info[k] = v + continue + except TypeError: + pass + + return observation, reward, terminated, truncated, info + +class PettingZooWrapper: + '''PettingZoo does not provide a ParallelEnv wrapper. This code is adapted from + their AEC wrapper, to prevent unneeded conversions to/from AEC''' + def __init__(self, env): + self.env = env + + def __getattr__(self, name): + '''Returns an attribute with ``name``, unless ``name`` starts with an underscore.''' + if name.startswith('_') and name != '_cumulative_rewards': + raise AttributeError(f'accessing private attribute "{name}" is prohibited') + return getattr(self.env, name) + + @property + def unwrapped(self): + return self.env.unwrapped + + def close(self): + self.env.close() + + def render(self): + return self.env.render() + + def reset(self, seed=None, options=None): + try: + return self.env.reset(seed=seed, options=options) + except TypeError: + return self.env.reset(seed=seed) + + def observe(self, agent): + return self.env.observe(agent) + + def state(self): + return self.env.state() + + def step(self, action): + return self.env.step(action) + + def observation_space(self, agent): + return self.env.observation_space(agent) + + def action_space(self, agent): + return self.env.action_space(agent) + + def __str__(self) -> str: + '''Returns a name which looks like: "max_observation".''' + return f'{type(self).__name__}<{str(self.env)}>' + +class MeanOverAgents(PettingZooWrapper): + '''Averages over agent infos''' + def _mean(self, infos): + list_infos = {} + for agent, info in infos.items(): + for k, v in info.items(): + if k not in list_infos: + list_infos[k] = [] + + list_infos[k].append(v) + + mean_infos = {} + for k, v in list_infos.items(): + try: + mean_infos[k] = np.mean(v) + except: + pass + + return mean_infos + + def reset(self, seed=None, options=None): + observations, infos = super().reset(seed, options) + infos = self._mean(infos) + return observations, infos + + def step(self, actions): + observations, rewards, terminations, truncations, infos = super().step(actions) + infos = self._mean(infos) + return observations, rewards, terminations, truncations, infos + +class MultiagentEpisodeStats(PettingZooWrapper): + '''Wrapper for PettingZoo environments that stores + episodic returns and lengths in infos''' + def reset(self, seed=None, options=None): + observations, infos = super().reset(seed=seed, options=options) + self.infos = { + agent: dict(episode_return=[], episode_length=0) + for agent in self.possible_agents + } + return observations, infos + + def step(self, actions): + observations, rewards, terminations, truncations, infos = super().step(actions) + + all_infos = {} + for agent in infos: + agent_info = self.infos[agent] + for k, v in unroll_nested_dict(infos[agent]): + if k not in agent_info: + agent_info[k] = [] + + agent_info[k].append(v) + + # Saved to self. TODO: Clean up + agent_info['episode_return'].append(rewards[agent]) + agent_info['episode_length'] += 1 + + agent_info = {} + all_infos[agent] = agent_info + if terminations[agent] or truncations[agent]: + for k, v in self.infos[agent].items(): + try: + agent_info[k] = sum(v) + continue + except TypeError: + pass + + if isinstance(v, str): + agent_info[k] = v + continue + + try: + x = int(v) # probably a value + agent_info[k] = v + continue + except TypeError: + pass + + return observations, rewards, terminations, truncations, all_infos +### Exceptions +class EnvironmentSetupError(RuntimeError): + def __init__(self, e, package): + super().__init__(self.message) + +class APIUsageError(RuntimeError): + """Exception raised when the API is used incorrectly.""" + + def __init__(self, message="API usage error."): + self.message = message + super().__init__(self.message) + +class InvalidAgentError(ValueError): + """Exception raised when an invalid agent key is used.""" + + def __init__(self, agent_id, agents): + message = ( + f'Invalid agent/team ({agent_id}) specified. ' + f'Valid values:\n{agents}' + ) + super().__init__(message) + +class GymToGymnasium: + def __init__(self, env): + self.env = env + self.observation_space = env.observation_space + self.action_space = env.action_space + self.render = env.render + self.metadata = env.metadata + + def reset(self, seed=None, options=None): + if seed is not None: + ob = self.env.reset(seed=seed) + else: + ob = self.env.reset() + return ob, {} + + def step(self, action): + observation, reward, done, info = self.env.step(action) + return observation, reward, done, False, info + + def close(self): + self.env.close() + +### Namespace +def __getitem__(self, key): + return self.__dict__[key] + +def __setitem__(self, key, value): + self.__dict__[key] = value + +def keys(self): + return self.__dict__.keys() + +def values(self): + return self.__dict__.values() + +def items(self): + return self.__dict__.items() + +def __iter__(self): + return iter(self.__dict__) + +def __len__(self): + return len(self.__dict__) + +class Namespace(SimpleNamespace, Mapping): + __getitem__ = __getitem__ + __setitem__ = __setitem__ + __iter__ = __iter__ + __len__ = __len__ + keys = keys + values = values + items = items + +def dataclass(cls): + # Safely get annotations + annotations = getattr(cls, '__annotations__', {}) + + # Combine both annotated and non-annotated fields + all_fields = {**{k: None for k in annotations.keys()}, **cls.__dict__} + all_fields = {k: v for k, v in all_fields.items() if not callable(v) and not k.startswith('__')} + + def __init__(self, **kwargs): + for field, default_value in all_fields.items(): + setattr(self, field, kwargs.get(field, default_value)) + + cls.__init__ = __init__ + setattr(cls, "__getitem__", __getitem__) + setattr(cls, "__setitem__", __setitem__) + setattr(cls, "__iter__", __iter__) + setattr(cls, "__len__", __len__) + setattr(cls, "keys", keys) + setattr(cls, "values", values) + setattr(cls, "items", items) + return cls + +def namespace(self=None, **kwargs): + if self is None: + return Namespace(**kwargs) + self.__dict__.update(kwargs) + +### Wrappers +class PettingZooTruncatedWrapper: + def __init__(self, env): + self.env = env + self.observation_space = env.observation_space + self.action_space = env.action_space + self.render = env.render + + @property + def render_mode(self): + return self.env.render_mode + + @property + def possible_agents(self): + return self.env.possible_agents + + @property + def agents(self): + return self.env.agents + + def reset(self, seed=None): + if seed is not None: + ob, info = self.env.reset(seed=seed) + else: + ob, info = self.env.reset() + info = {k: {} for k in ob} + return ob, info + + def step(self, actions): + observations, rewards, terminals, truncations, infos = self.env.step(actions) + return observations, rewards, terminals, truncations, infos + + def close(self): + self.env.close() + +### Misc +def unroll_nested_dict(d): + if not isinstance(d, dict): + return d + + for k, v in d.items(): + if isinstance(v, dict): + for k2, v2 in unroll_nested_dict(v): + yield f"{k}/{k2}", v2 + else: + yield k, v + +def silence_warnings(original_func, category=DeprecationWarning): + @wraps(original_func) + def wrapper(*args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", category=category) + return original_func(*args, **kwargs) + return wrapper + +class Suppress(): + def __init__(self): + self.f = StringIO() + self.null_1 = os.open(os.devnull, os.O_WRONLY | os.O_TRUNC | os.O_CREAT) + self.null_2 = os.open(os.devnull, os.O_WRONLY | os.O_TRUNC | os.O_CREAT) + + def __enter__(self): + # Suppress C library outputs + self.orig_stdout = os.dup(1) + self.orig_stderr = os.dup(2) + os.dup2(self.null_1, 1) + os.dup2(self.null_2, 2) + + # Suppress Python outputs + self._stdout_redirector = redirect_stdout(self.f) + self._stderr_redirector = redirect_stderr(self.f) + self._stdout_redirector.__enter__() + self._stderr_redirector.__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): + # Enable C library outputs + os.dup2(self.orig_stdout, 1) + os.dup2(self.orig_stderr, 2) + os.close(self.orig_stdout) + os.close(self.orig_stderr) + os.close(self.null_1) + os.close(self.null_2) + + # Enable Python outputs + self._stdout_redirector.__exit__(exc_type, exc_val, exc_tb) + self._stderr_redirector.__exit__(exc_type, exc_val, exc_tb) From 23f2f7ef831627223cb9b7515007d9be8dff790d Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 15:48:16 +0000 Subject: [PATCH 24/63] Fix new api --- pufferlib/sweep.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 7a5703ba91..5f4542e67f 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -152,7 +152,7 @@ def _params_from_puffer_sweep(sweep_config): class Hyperparameters: def __init__(self, config, verbose=True): self.spaces = _params_from_puffer_sweep(config) - self.flat_spaces = dict(pufferlib.utils.unroll_nested_dict(self.spaces)) + self.flat_spaces = dict(pufferlib.unroll_nested_dict(self.spaces)) self.num = len(self.flat_spaces) self.metric = config['metric'] @@ -192,7 +192,7 @@ def sample(self, n, mu=None, scale=1): return np.clip(samples, self.min_bounds, self.max_bounds) def from_dict(self, params): - flat_params = dict(pufferlib.utils.unroll_nested_dict(params)) + flat_params = dict(pufferlib.unroll_nested_dict(params)) values = [] for key, space in self.flat_spaces.items(): assert key in flat_params, f'Missing hyperparameter {key}' @@ -646,7 +646,7 @@ def __init__(self, ): param_spaces = _carbs_params_from_puffer_sweep(sweep_config) - flat_spaces = [e[1] for e in pufferlib.utils.unroll_nested_dict(param_spaces)] + flat_spaces = [e[1] for e in pufferlib.unroll_nested_dict(param_spaces)] for e in flat_spaces: print(e.name, e.space) From 2cba8e40fe53a36cf1df923d419261ed0cc8108f Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 15:58:18 +0000 Subject: [PATCH 25/63] Fix cost fn passed to sweep: --- clean_pufferl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 8966a8442f..56fa56cc98 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -1017,7 +1017,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin if logs is not None and target_key in logs: timesteps.append(logs['agent_steps']) scores.append(logs[target_key]) - #costs.append(data.profile.uptime) + costs.append(pufferl.uptime) steps_evaluated = 0 cost = time.time() - pufferl.start_time From 36912017e1deb334a005244ae62c15923a5054f8 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 16:34:28 +0000 Subject: [PATCH 26/63] breakpoint --- pufferlib/sweep.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 5f4542e67f..888d9a20bd 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -425,7 +425,10 @@ def suggest(self, fill): self.gp_cost.mean_function = lambda x: 1 self.gp_cost.set_data(params, torch.from_numpy(log_c_norm)) self.gp_cost.train() - gp.util.train(self.gp_cost, self.cost_opt) + try: + gp.util.train(self.gp_cost, self.cost_opt) + except: + breakpoint() self.gp_cost.eval() candidates, pareto_idxs = pareto_points(self.success_observations) From 15c6e543165bb0160aa73a69803def67e8601b50 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 18:18:14 +0000 Subject: [PATCH 27/63] Several small fixes --- clean_pufferl.py | 14 +++++----- config/metta.ini | 23 +++++----------- pufferlib/emulation.py | 2 +- pufferlib/environments/metta/environment.py | 12 ++++----- pufferlib/ocean/environment.py | 12 +++++++-- pufferlib/pytorch.py | 3 --- pufferlib/sweep.py | 30 ++++++--------------- pufferlib/vector.py | 4 +-- 8 files changed, 39 insertions(+), 61 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 56fa56cc98..577c6c0c6d 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -109,8 +109,8 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # TODO: Doesn't exist in native envs # TODO: Replace slice with env idx or similar n = vecenv.agents_per_batch - self.lstm_h = {slice(i*n, (i+1)*n): torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} - self.lstm_c = {slice(i*n, (i+1)*n): torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} + self.lstm_h = {i*n: torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} + self.lstm_c = {i*n: torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} # Minibatching & gradient accumulation @@ -247,8 +247,8 @@ def evaluate(self): ) if config.use_rnn: - state.lstm_h = self.lstm_h[env_id] - state.lstm_c = self.lstm_c[env_id] + state.lstm_h = self.lstm_h[env_id.start] + state.lstm_c = self.lstm_c[env_id.start] logits, value = policy(o_device, state) action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=policy.is_continuous) @@ -257,8 +257,8 @@ def evaluate(self): profile('eval_copy', epoch) with torch.no_grad(): if config.use_rnn: - self.lstm_h[env_id] = state.lstm_h - self.lstm_c[env_id] = state.lstm_c + self.lstm_h[env_id.start] = state.lstm_h + self.lstm_c[env_id.start] = state.lstm_c o = o if config.cpu_offload else o_device actions = self.store(state, o, value, action, logprob, r, d, env_id, mask) @@ -1022,6 +1022,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin steps_evaluated = 0 cost = time.time() - pufferl.start_time batch_size = args['train']['batch_size'] + timesteps.append(pufferl.global_step) while len(pufferl.stats[target_metric]) < min_eval_points: stats = pufferl.evaluate() steps_evaluated += batch_size @@ -1032,7 +1033,6 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin scores.append(score) costs.append(cost) - timesteps.append(pufferl.global_step) pufferl.close() return scores, costs, timesteps diff --git a/config/metta.ini b/config/metta.ini index df284bf76f..9bfcbd20ce 100644 --- a/config/metta.ini +++ b/config/metta.ini @@ -3,17 +3,17 @@ package = metta env_name = metta policy_name = Policy rnn_name = Recurrent -vec = multiprocessing + +[vec] +num_envs = 128 +num_workers = 16 +batch_size = 64 [env] render_mode = auto -#num_envs = 128 [train] -total_timesteps = 5_000_000_000 -num_envs = 128 -num_workers = 16 -env_batch_size = 64 +total_timesteps = 100_000_000 learning_rate = 0.0013848535655657842 gamma = 0.9959746852829785 gae_lambda = 0.9283720217357007 @@ -45,17 +45,6 @@ adam_eps = 0.000249501214984291 #minibatch_size = 32768 #compile = False -[sweep] -method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = score -min = 0 -max = 10 -scale = auto - #[sweep.train.total_timesteps] #distribution = log_normal #min = 2e7 diff --git a/pufferlib/emulation.py b/pufferlib/emulation.py index 686a24d573..f52261771b 100644 --- a/pufferlib/emulation.py +++ b/pufferlib/emulation.py @@ -139,7 +139,7 @@ def emulate_action_space(space): class GymnasiumPufferEnv(gymnasium.Env): - def __init__(self, env=None, env_creator=None, env_args=[], env_kwargs={}, buf=None): + def __init__(self, env=None, env_creator=None, env_args=[], env_kwargs={}, buf=None, seed=0): self.env = make_object(env, env_creator, env_args, env_kwargs) self.initialized = False diff --git a/pufferlib/environments/metta/environment.py b/pufferlib/environments/metta/environment.py index af57437e72..c4ef967c26 100644 --- a/pufferlib/environments/metta/environment.py +++ b/pufferlib/environments/metta/environment.py @@ -15,7 +15,11 @@ class MettaPuff(pufferlib.PufferEnv): def __init__(self, config, render_mode='human', buf=None, seed=0): self.render_mode = render_mode import mettagrid.mettagrid_env - self.env = mettagrid.mettagrid_env.make_env_from_cfg(config, render_mode, buf=buf) + from omegaconf import OmegaConf + cfg = OmegaConf.load(config) + + from mettagrid.mettagrid_env import MettaGridEnv + self.env = MettaGridEnv(cfg, render_mode=render_mode, buf=buf) if render_mode == 'human': from mettagrid.gym_wrapper import RaylibRendererWrapper @@ -26,12 +30,6 @@ def __init__(self, config, render_mode='human', buf=None, seed=0): self.num_agents = self.env.num_agents super().__init__(buf) - #cfg = self.env._env_cfg - #cfg.eval.env = config_from_path(cfg.eval.env, cfg.eval.env_overrides) - #from mettagrid.renderer.raylib.raylib_renderer import MettaGridRaylibRenderer - #self.env._renderer = MettaGridRaylibRenderer(self.env._c_env, self.env._env_cfg['game']) - - def step(self, actions): obs, rew, term, trunc, info = self.env.step(actions) diff --git a/pufferlib/ocean/environment.py b/pufferlib/ocean/environment.py index b3770cd998..3e2404aafe 100644 --- a/pufferlib/ocean/environment.py +++ b/pufferlib/ocean/environment.py @@ -138,12 +138,20 @@ def make_multiagent(buf=None, **kwargs): 'cpr': 'PyCPR', 'impulse_wars': 'ImpulseWars', 'gpudrive': 'GPUDrive', + 'spaces': make_spaces, + 'multiagent': make_multiagent, } def env_creator(name='squared', *args, **kwargs): if 'puffer_' not in name: raise pufferlib.exceptions.APIUsageError(f'Invalid environment name: {name}') + # TODO: Robust sanity / ocean imports name = name.replace('puffer_', '') - module = importlib.import_module(f'pufferlib.ocean.{name}.{name}') - return getattr(module, MAKE_FUNCTIONS[name]) + try: + module = importlib.import_module(f'pufferlib.ocean.{name}.{name}') + return getattr(module, MAKE_FUNCTIONS[name]) + except ModuleNotFoundError: + return MAKE_FUNCTIONS[name] + + diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index d04987a428..67e70f186c 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -311,6 +311,3 @@ def sample_logits(logits: Union[torch.Tensor, List[torch.Tensor]], return action.squeeze(0), logprob.squeeze(0), logits_entropy.squeeze(0) return action.T, logprob.sum(0), logits_entropy - - - diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 888d9a20bd..21c589f0b9 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -371,7 +371,9 @@ def suggest(self, fill): if len(self.success_observations) == 0 and self.seed_with_search_center: best = self.hyperparameters.search_centers return self.hyperparameters.to_dict(best, fill), info - elif len(self.success_observations) < self.num_random_samples: + elif False: + # TODO: UNDO + #len(self.success_observations) < self.num_random_samples: suggestions = self.hyperparameters.sample(self.random_suggestions) self.suggestion = random.choice(suggestions) return self.hyperparameters.to_dict(self.suggestion, fill), info @@ -392,20 +394,20 @@ def suggest(self, fill): # Transformed scores min_score = self.min_score if min_score is None: - min_score = np.min(y) - np.min(np.abs(y)) + min_score = np.min(y) if np.min(y) < min_score - 1e-6: raise ValueError(f'Min score {min_score} is less than min score in data {np.min(y)}') max_score = self.max_score if max_score is None: - max_score = np.max(y) + np.max(np.abs(y)) + max_score = np.max(y) if np.max(y) > max_score + 1e-6: raise ValueError(f'Max score {max_score} is greater than max score in data {np.max(y)}') # Linearize - y_norm = (y - min_score) / (np.abs((max_score - min_score)) + 1e-6) + y_norm = (y - min_score) / (np.abs(max_score - min_score) + 1e-6) self.gp_score.set_data(params, torch.from_numpy(y_norm)) self.gp_score.train() @@ -420,7 +422,7 @@ def suggest(self, fill): # Linear input norm creates clean 1 mean fn log_c_min = np.min(log_c) log_c_max = np.max(log_c) - log_c_norm = (log_c - log_c_min) / ((np.abs(log_c_max - log_c_min)) + 1e-6) + log_c_norm = (log_c - log_c_min) / (log_c_max - log_c_min + 1e-6) self.gp_cost.mean_function = lambda x: 1 self.gp_cost.set_data(params, torch.from_numpy(log_c_norm)) @@ -456,7 +458,7 @@ def suggest(self, fill): gp_c_min = np.min(gp_c) gp_c_max = np.max(gp_c) - gp_c_norm = (gp_c - gp_c_min) / ((np.abs(gp_c_max - gp_c_min)) + 1e-6) + gp_c_norm = (gp_c - gp_c_min) / (gp_c_max - gp_c_min + 1e-6) pareto_y = y[pareto_idxs] pareto_c = c[pareto_idxs] @@ -465,22 +467,8 @@ def suggest(self, fill): max_c = np.max(c) min_c = np.min(c) - c_right = abs(pareto_log_c_norm[None, :] - gp_log_c_norm[:, None]) - - sorted_dist = np.sort(c_right, axis=1) - nearest_idx = np.argmin(c_right, axis=1) - nearest_pareto_dist = np.min(c_right, axis=1) - nearest_pareto_y = pareto_y[nearest_idx] - max_c_mask = gp_c < self.max_suggestion_cost - cumsum_mask = c[None, :] <= np.clip(gp_c[:, None], min_c, max_c) - cumsum_mask = cumsum_mask * c[None, :] - cumsum = np.sum(cumsum_mask, axis=1) / np.sum(c) - target = gp_c_norm - weight = target - cumsum - - target = 1.25*np.random.rand() weight = 1 - abs(target - gp_log_c_norm) @@ -491,8 +479,6 @@ def suggest(self, fill): info = dict( cost = gp_c[best_idx].item(), score = gp_y[best_idx].item(), - nearby = nearest_pareto_y[best_idx].item(), - dist = nearest_pareto_dist[best_idx].item(), rating = suggestion_scores[best_idx].item(), ) print('Predicted -- ', diff --git a/pufferlib/vector.py b/pufferlib/vector.py index b6a9002f1a..d7d767ced6 100644 --- a/pufferlib/vector.py +++ b/pufferlib/vector.py @@ -99,7 +99,7 @@ def __init__(self, env_creators, env_args, env_kwargs, num_envs, buf=None, seed= def _avg_infos(self): infos = {} for e in self.infos: - for k, v in pufferlib.utils.unroll_nested_dict(e): + for k, v in pufferlib.unroll_nested_dict(e): if k not in infos: infos[k] = [] @@ -894,7 +894,7 @@ def autotune(env_creator, batch_size, max_envs=194, model_forward_s=0.0, )) for config in configs: - with pufferlib.utils.Suppress(): + with pufferlib.Suppress(): envs = make(env_creator, **config) envs.reset() actions = [envs.action_space.sample() for _ in range(1000)] From 75d1dd16bb04b3f5e374c3f6c66eb5a7e8edd6c9 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 18:27:14 +0000 Subject: [PATCH 28/63] Sweep smoke fix --- pufferlib/sweep.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 21c589f0b9..c7233b7230 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -483,8 +483,6 @@ def suggest(self, fill): ) print('Predicted -- ', f'Score: {info["score"]:.3f}', - f'Nearby: {info["nearby"]:.3f}', - f'Dist: {info["dist"]:.3f}', f'Cost: {info["cost"]:.3f}', f'Rating: {info["rating"]:.3f}', ) From d3588d24346361b97492300ee3f362b6d59c1da7 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 19:00:10 +0000 Subject: [PATCH 29/63] sweep --- clean_pufferl.py | 3 ++- pufferlib/sweep.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 577c6c0c6d..7ad4136bc7 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -1014,7 +1014,8 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin while pufferl.global_step < train_config.total_timesteps: pufferl.evaluate() logs = pufferl.train() - if logs is not None and target_key in logs: + min_sweep_steps = args['sweep']['train']['total_timesteps']['min'] + if logs is not None and target_key in logs and pufferl.global_step >= min_sweep_steps: timesteps.append(logs['agent_steps']) scores.append(logs[target_key]) costs.append(pufferl.uptime) diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index c7233b7230..6738b87a08 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -473,7 +473,7 @@ def suggest(self, fill): weight = 1 - abs(target - gp_log_c_norm) suggestion_scores = self.hyperparameters.optimize_direction * max_c_mask * ( - gp_y_norm*weight)# / gp_c + gp_y_norm*weight) best_idx = np.argmax(suggestion_scores) info = dict( From 82eb92f53147a3abc292d0bbac0219956cf8ce95 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Tue, 6 May 2025 21:18:07 +0000 Subject: [PATCH 30/63] Actually maximize in sweep... --- clean_pufferl.py | 14 +++++++------- pufferlib/sweep.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 7ad4136bc7..ed85966cab 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -76,7 +76,7 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): segments = batch_size // horizon self.segments = segments if total_agents > segments: - raise pufferlib.exceptions.APIUsageError( + raise pufferlib.APIUsageError( f'Total agents {total_agents} <= segments {segments}' ) @@ -119,7 +119,7 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.minibatch_size = min(minibatch_size, max_minibatch_size) if minibatch_size % max_minibatch_size != 0 and max_minibatch_size % minibatch_size != 0: # todo: better error - raise pufferlib.exceptions.APIUsageError( + raise pufferlib.APIUsageError( f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' ) @@ -127,7 +127,7 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.total_minibatches = int(config.update_epochs * batch_size / self.minibatch_size) self.minibatch_segments = self.minibatch_size // horizon if self.minibatch_segments * horizon != self.minibatch_size: - raise pufferlib.exceptions.APIUsageError( + raise pufferlib.APIUsageError( f'minibatch_size {self.minibatch_size} must be divisible by horizon {horizon}' ) @@ -175,7 +175,7 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Automatic mixed precision self.amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) if config.precision not in ('float32', 'bfloat16'): - raise pufferlib.exceptions.APIUsageError(f'Use float32 or bfloat16, not {config.precision}') + raise pufferlib.APIUsageError(f'Use float32 or bfloat16, not {config.precision}') # Logging self.neptune = neptune @@ -949,13 +949,13 @@ def downsample_linear(arr, m): def sweep(args, env_name, make_env, policy_cls, rnn_cls): if not args['wandb'] and not args['neptune']: - raise pufferlib.exceptions.APIUsageError('Sweeps require either wandb or neptune') + raise pufferlib.APIUsageError('Sweeps require either wandb or neptune') method = args['sweep'].pop('method') try: sweep_cls = getattr(pufferlib.sweep, method) except: - raise pufferlib.exceptions.APIUsageError(f'Invalid sweep method {method}. See pufferlib.sweep') + raise pufferlib.APIUsageError(f'Invalid sweep method {method}. See pufferlib.sweep') sweep = sweep_cls(args['sweep']) target_metric = args['sweep']['metric'] @@ -1077,7 +1077,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin if args.env in p['base']['env_name'].split(): break else: - raise pufferlib.exceptions.APIUsageError('No config for env_name {}'.format(args.env)) + raise pufferlib.APIUsageError('No config for env_name {}'.format(args.env)) # Dynamic help menu from config for section in p.sections(): diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 6738b87a08..06a819da99 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -158,7 +158,7 @@ def __init__(self, config, verbose=True): self.metric = config['metric'] goal = config['goal'] assert goal in ('maximize', 'minimize') - self.optimize_direction = 1 if 'goal' == 'maximize' else -1 + self.optimize_direction = 1 if goal == 'maximize' else -1 self.search_centers = np.array([ e.norm_mean for e in self.flat_spaces.values()]) From e0d9b9865a6ddc9c943c57e6452cd854142925a3 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Wed, 7 May 2025 16:42:29 +0000 Subject: [PATCH 31/63] defaults for sweep --- config/default.ini | 37 ++++++++++++++++++++++++++----------- config/ocean/breakout.ini | 29 ++++++++++++++++++++++++++--- config/ocean/grid.ini | 24 +++++++++++++++++------- pufferlib/sweep.py | 4 +--- 4 files changed, 70 insertions(+), 24 deletions(-) diff --git a/config/default.ini b/config/default.ini index d42c878056..abb15cef84 100644 --- a/config/default.ini +++ b/config/default.ini @@ -86,9 +86,10 @@ max = 2097152 mean = 524288 scale = auto +# TODO: Mini minibatch optim is lower [sweep.train.minibatch_size] distribution = uniform_pow2 -min = 4096 +min = 16384 max = 131072 mean = 8192 scale = auto @@ -121,12 +122,26 @@ mean = 0.95 max = 0.995 scale = auto -[sweep.train.update_epochs] -distribution = int_uniform -min = 1 -max = 4 -mean = 1 -scale = 1.0 +#[sweep.train.update_epochs] +#distribution = int_uniform +#min = 1 +#max = 4 +#mean = 1 +#scale = 1.0 + +[sweep.train.clip_coef] +distribution = uniform +min = 0.01 +max = 1.0 +mean = 0.1 +scale = auto + +[sweep.train.vf_clip_coef] +distribution = uniform +min = 0.01 +max = 5.0 +mean = 0.1 +scale = auto [sweep.train.vf_coef] distribution = uniform @@ -157,10 +172,10 @@ max = 0.99999 scale = auto [sweep.train.adam_eps] -distribution = uniform -min = 0.00000000000001 -mean = 0.00000001 -max = 0.001 +distribution = log_normal +min = 1e-14 +mean = 1e-8 +max = 1e-4 scale = auto [sweep.train.prio_alpha] diff --git a/config/ocean/breakout.ini b/config/ocean/breakout.ini index 502dc57c8b..7ee6ef1c97 100644 --- a/config/ocean/breakout.ini +++ b/config/ocean/breakout.ini @@ -15,9 +15,32 @@ input_size = 128 hidden_size = 128 [train] -total_timesteps = 80_000_000 -learning_rate = 0.05 -minibatch_size = 32768 +total_timesteps = 75_000_000 + +# Highly sensitive +adam_beta1 = 0.99 + +adam_beta2 = 0.9999 +adam_eps = 1e-14 +batch_size = 524288 +ent_coef = 0.025 +gae_lambda = 0.85 + +# Highly sensitive +gamma = 0.975 + +learning_rate = 0.01 +max_grad_norm = 1.5 +minibatch_size = 16384 + +prio_alpha = 0.0 +# Doesn't matter +prio_beta0 = 1.0 + +# Just can't be low +vf_coef = 1.3 + +# TODO: Try tuning clip coefs [sweep.train.total_timesteps] distribution = log_normal diff --git a/config/ocean/grid.ini b/config/ocean/grid.ini index 6504f4a19c..1c0bb9544b 100644 --- a/config/ocean/grid.ini +++ b/config/ocean/grid.ini @@ -17,17 +17,27 @@ num_envs = 4096 num_maps = 8192 [train] -total_timesteps = 180_000_000 +total_timesteps = 250_000_000 +adam_beta1 = 0.9225899639773112 +adam_beta2 = 0.9 +adam_eps = 0.0004030478187254784 +anneal_lr = true +batch_size = 524288 +ent_coef = 0.0020159472963835016 +gae_lambda = 0.8829440612065992 +gamma = 0.9872971455373439 +learning_rate = 0.0003947934701844728 +max_grad_norm = 0.5296288081133984 +minibatch_size = 4096 +prio_alpha = 0.99 +prio_beta0 = 0.48469847315324566 +#update_epochs = 2 +vf_coef = 3.6777541336880786 checkpoint_interval = 1000 -gamma = 0.9944336976183826 -gae_lambda = 0.9474288929489364 -ent_coef = 0.00001 -learning_rate = 0.005 -minibatch_size = 32768 [sweep.train.total_timesteps] distribution = log_normal min = 5e7 -max = 3e8 +max = 6e8 mean = 1e8 scale = auto diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 06a819da99..0d7cdb85c9 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -371,9 +371,7 @@ def suggest(self, fill): if len(self.success_observations) == 0 and self.seed_with_search_center: best = self.hyperparameters.search_centers return self.hyperparameters.to_dict(best, fill), info - elif False: - # TODO: UNDO - #len(self.success_observations) < self.num_random_samples: + elif not self.seed_with_search_center and len(self.success_observations) < self.num_random_samples: suggestions = self.hyperparameters.sample(self.random_suggestions) self.suggestion = random.choice(suggestions) return self.hyperparameters.to_dict(self.suggestion, fill), info From f479f3ba9f3b21507869c0f3649936bfd2047fee Mon Sep 17 00:00:00 2001 From: Spencer Cheng Date: Wed, 7 May 2025 16:57:41 +0000 Subject: [PATCH 32/63] clean pufferdrive --- pufferlib/ocean/gpudrive/cy_gpudrive.pyx | 45 ++- pufferlib/ocean/gpudrive/gpudrive.c | 4 +- pufferlib/ocean/gpudrive/gpudrive.h | 378 ++++++++++++++++++----- pufferlib/ocean/gpudrive/gpudrive.py | 18 +- 4 files changed, 335 insertions(+), 110 deletions(-) diff --git a/pufferlib/ocean/gpudrive/cy_gpudrive.pyx b/pufferlib/ocean/gpudrive/cy_gpudrive.pyx index 2c7d3aaf8e..045e953eb9 100644 --- a/pufferlib/ocean/gpudrive/cy_gpudrive.pyx +++ b/pufferlib/ocean/gpudrive/cy_gpudrive.pyx @@ -1,4 +1,5 @@ -from libc.stdlib cimport calloc, free +from libc.stdlib cimport calloc, malloc, free +from libc.string cimport strcpy import numpy as np cdef extern from "gpudrive.h": int LOG_BUFFER_SIZE @@ -82,6 +83,8 @@ cdef extern from "gpudrive.h": float reward_offroad_collision; char* map_name; char* reached_goal_this_turn; + float world_mean_x; + float world_mean_y; ctypedef struct Client @@ -173,37 +176,47 @@ cdef class CyGPUDrive: free(agent_offsets) return total_count, py_offsets def __init__(self, float[:, :] observations, int[:,:] actions, - float[:] rewards, unsigned char[:] masks, unsigned char[:] terminals, int num_envs, + float[:] rewards, unsigned char[:] terminals, int num_envs, int human_agent_idx, reward_vehicle_collision, reward_offroad_collision, offsets): self.client = NULL self.num_envs = num_envs - self.envs = calloc(num_envs, sizeof(GPUDrive)) + cdef int num_clones + num_clones = 1 + self.envs = calloc(num_envs*num_clones, sizeof(GPUDrive)) self.agent_offsets = calloc(num_envs + 1, sizeof(int)) self.logs = allocate_logbuffer(LOG_BUFFER_SIZE) cdef int i for i in range(num_envs + 1): self.agent_offsets[i] = offsets[i] cdef int inc - for i in range(num_envs): - inc = self.agent_offsets[i] - print(inc) - map_file = f"resources/gpudrive/binaries/map_{i:03d}.bin".encode('utf-8') - print("cython map_name", map_file) + cdef int index + cdef int total_envs + total_envs = num_envs * num_clones + cdef int total_agents + total_agents = self.agent_offsets[num_envs] + cdef char* c_map_file + for i in range(total_envs): + env_index = i % num_envs + clone_index = i // num_envs + inc = self.agent_offsets[env_index] + count = self.agent_offsets[env_index+1] - self.agent_offsets[env_index] + clone_agent_offset = clone_index * total_agents + inc + map_file = f"resources/gpudrive/binaries/map_{env_index:03d}.bin".encode('utf-8') + c_map_file = malloc(len(map_file) + 1) + strcpy(c_map_file, map_file) self.envs[i] = GPUDrive( - observations=&observations[inc, 0], - actions=&actions[inc,0], - rewards=&rewards[inc], - masks=&masks[inc], - dones=&terminals[inc], + observations=&observations[clone_agent_offset, 0], + actions=&actions[clone_agent_offset,0], + rewards=&rewards[clone_agent_offset], + dones=&terminals[clone_agent_offset], log_buffer=self.logs, human_agent_idx=human_agent_idx, reward_vehicle_collision=reward_vehicle_collision, reward_offroad_collision=reward_offroad_collision, - map_name = map_file + map_name = c_map_file ) - print("init") init(&self.envs[i]) self.client = NULL @@ -219,7 +232,7 @@ cdef class CyGPUDrive: c_step(&self.envs[i]) def render(self): - cdef GPUDrive* env = &self.envs[211] + cdef GPUDrive* env = &self.envs[24] if self.client == NULL: import os cwd = os.getcwd() diff --git a/pufferlib/ocean/gpudrive/gpudrive.c b/pufferlib/ocean/gpudrive/gpudrive.c index 1b9a251bc2..0f8376c6c7 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.c +++ b/pufferlib/ocean/gpudrive/gpudrive.c @@ -205,7 +205,7 @@ void performance_test() { } int main() { - demo(); - //performance_test(); + //demo(); + performance_test(); return 0; } diff --git a/pufferlib/ocean/gpudrive/gpudrive.h b/pufferlib/ocean/gpudrive/gpudrive.h index ffea1a3c1b..fd4f4c59a4 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.h +++ b/pufferlib/ocean/gpudrive/gpudrive.h @@ -45,7 +45,7 @@ #define SLOTS_PER_CELL (MAX_ENTITIES_PER_CELL*2 + 1) // Max road segment observation entities -#define MAX_ROAD_SEGMENT_OBSERVATIONS 64 +#define MAX_ROAD_SEGMENT_OBSERVATIONS 200 #define MAX_CARS 64 // Observation Space Constants #define MAX_SPEED 100.0f @@ -201,7 +201,6 @@ struct GPUDrive { float* observations; int* actions; float* rewards; - unsigned char* masks; unsigned char* dones; LogBuffer* log_buffer; Log* logs; @@ -233,12 +232,14 @@ struct GPUDrive { float reward_vehicle_collision; float reward_offroad_collision; char* map_name; - char* reached_goal_this_turn; + char* reached_goal_this_episode; + float world_mean_x; + float world_mean_y; }; Entity* load_map_binary(const char* filename, GPUDrive* env) { FILE* file = fopen(filename, "rb"); - printf("fileanme: %s\n", filename); + //printf("fileanme: %s\n", filename); if (!file) return NULL; fread(&env->num_objects, sizeof(int), 1, file); fread(&env->num_roads, sizeof(int), 1, file); @@ -333,22 +334,16 @@ void set_start_position(GPUDrive* env){ void set_active_agents(GPUDrive* env){ env->static_car_count = 0; - env->num_cars = 0; + env->num_cars = 1; env->expert_static_car_count = 0; int active_agent_indices[MAX_CARS]; int static_car_indices[MAX_CARS]; int expert_static_car_indices[MAX_CARS]; - env->active_agent_count = 0; - for(int i = env->num_objects-1; i >= 0 && env->num_cars < MAX_CARS; i--){ + env->active_agent_count = 1; + active_agent_indices[0] = env->num_objects-1; + for(int i = 0; i < env->num_objects && env->num_cars < MAX_CARS; i++){ if(env->entities[i].type != 1) continue; if(env->entities[i].traj_valid[0] != 1) continue; - /*for(int j = 1; j < env->entities[i].array_size; j++){ - if(env->entities[i].traj_valid[j] != 1) { - env->entities[i].goal_position_x = env->entities[i].traj_x[j-1]; - env->entities[i].goal_position_y = env->entities[i].traj_y[j-1]; - break; - } - }*/ env->num_cars++; float cos_heading = cosf(env->entities[i].traj_heading[0]); float sin_heading = sinf(env->entities[i].traj_heading[0]); @@ -360,6 +355,7 @@ void set_active_agents(GPUDrive* env){ float distance_to_goal = relative_distance_2d(0, 0, rel_goal_x, rel_goal_y); env->entities[i].width *= 0.7f; env->entities[i].length *= 0.7f; + if(distance_to_goal >= 2.0f && env->entities[i].mark_as_expert == 0){ active_agent_indices[env->active_agent_count] = i; env->active_agent_count++; @@ -371,7 +367,6 @@ void set_active_agents(GPUDrive* env){ env->expert_static_car_count++; } } - } env->active_agent_indices = (int*)malloc(env->active_agent_count * sizeof(int)); env->static_car_indices = (int*)malloc(env->static_car_count * sizeof(int)); @@ -448,8 +443,6 @@ void init_grid_map(GPUDrive* env){ } } } - printf("top left: %f, %f\n", top_left_x, top_left_y); - printf("bottom right: %f, %f\n", bottom_right_x, bottom_right_y); env->map_corners = (float*)calloc(4, sizeof(float)); env->map_corners[0] = top_left_x; @@ -466,7 +459,7 @@ void init_grid_map(GPUDrive* env){ env->grid_cells = (int*)calloc(grid_cell_count*SLOTS_PER_CELL, sizeof(int)); // Populate grid cells for(int i = 0; i < env->num_entities; i++){ - if(env->entities[i].type == ROAD_EDGE){ + if(env->entities[i].type > 3 && env->entities[i].type < 7){ for(int j = 0; j < env->entities[i].array_size - 1; j++){ float x_center = (env->entities[i].traj_x[j] + env->entities[i].traj_x[j+1]) / 2; float y_center = (env->entities[i].traj_y[j] + env->entities[i].traj_y[j+1]) / 2; @@ -579,6 +572,46 @@ int get_neighbor_cache_entities(GPUDrive* env, int cell_idx, int* entities, int return pairs; } +void set_means(GPUDrive* env) { + float mean_x = 0.0f; + float mean_y = 0.0f; + int64_t point_count = 0; + + // Compute single mean for all entities (vehicles and roads) + for (int i = 0; i < env->num_entities; i++) { + if (env->entities[i].type == VEHICLE) { + for (int j = 0; j < env->entities[i].array_size; j++) { + // Assume a validity flag exists (e.g., valid[j]); adjust if not available + if (env->entities[i].traj_valid[j]) { // Add validity check if applicable + point_count++; + mean_x += (env->entities[i].traj_x[j] - mean_x) / point_count; + mean_y += (env->entities[i].traj_y[j] - mean_y) / point_count; + } + } + } else if (env->entities[i].type >= 4) { + for (int j = 0; j < env->entities[i].array_size; j++) { + point_count++; + mean_x += (env->entities[i].traj_x[j] - mean_x) / point_count; + mean_y += (env->entities[i].traj_y[j] - mean_y) / point_count; + } + } + } + env->world_mean_x = mean_x; + env->world_mean_y = mean_y; + for (int i = 0; i < env->num_entities; i++) { + if (env->entities[i].type == VEHICLE || env->entities[i].type >= 4) { + for (int j = 0; j < env->entities[i].array_size; j++) { + if(env->entities[i].traj_x[j] == -10000) continue; + env->entities[i].traj_x[j] -= mean_x; + env->entities[i].traj_y[j] -= mean_y; + } + env->entities[i].goal_position_x -= mean_x; + env->entities[i].goal_position_y -= mean_y; + } + } + +} + void init(GPUDrive* env){ env->human_agent_idx = 0; env->timestep = 0; @@ -586,12 +619,13 @@ void init(GPUDrive* env){ // printf("entities loaded\n"); // printf("num entities: %d\n", env->num_entities); env->dynamics_model = CLASSIC; + set_means(env); set_active_agents(env); set_start_position(env); // printf("Active agents: %d\n", env->active_agent_count); env->logs = (Log*)calloc(env->active_agent_count, sizeof(Log)); env->goal_reached = (char*)calloc(env->active_agent_count, sizeof(char)); - env->reached_goal_this_turn = (char*)calloc(env->active_agent_count, sizeof(char)); + env->reached_goal_this_episode = (char*)calloc(env->active_agent_count, sizeof(char)); init_grid_map(env); env->vision_range = 21; init_neighbor_offsets(env); @@ -608,7 +642,7 @@ void free_initialized(GPUDrive* env){ free(env->logs); free(env->fake_data); free(env->goal_reached); - free(env->reached_goal_this_turn); + free(env->reached_goal_this_episode); free(env->map_corners); free(env->grid_cells); free(env->neighbor_offsets); @@ -629,7 +663,6 @@ void allocate(GPUDrive* env){ env->observations = (float*)calloc(env->active_agent_count*max_obs, sizeof(float)); env->actions = (int*)calloc(env->active_agent_count*2, sizeof(int)); env->rewards = (float*)calloc(env->active_agent_count, sizeof(float)); - env->masks = (unsigned char*)calloc(env->active_agent_count, sizeof(unsigned char)); env->dones = (unsigned char*)calloc(env->active_agent_count, sizeof(unsigned char)); env->log_buffer = allocate_logbuffer(LOG_BUFFER_SIZE); // printf("allocated\n"); @@ -639,7 +672,6 @@ void free_allocated(GPUDrive* env){ free(env->observations); free(env->actions); free(env->rewards); - free(env->masks); free(env->dones); free_logbuffer(env->log_buffer); free_initialized(env); @@ -814,6 +846,7 @@ void collision_check(GPUDrive* env, int agent_idx) { if(entity_list[i] == agent_idx) continue; Entity* entity; entity = &env->entities[entity_list[i]]; + if(entity->type != ROAD_EDGE) continue; int geometry_idx = entity_list[i + 1]; float start[2] = {entity->traj_x[geometry_idx], entity->traj_y[geometry_idx]}; float end[2] = {entity->traj_x[geometry_idx + 1], entity->traj_y[geometry_idx + 1]}; @@ -882,9 +915,6 @@ void compute_observations(GPUDrive* env) { memset(env->observations, 0, max_obs*env->active_agent_count*sizeof(float)); float (*observations)[max_obs] = (float(*)[max_obs])env->observations; for(int i = 0; i < env->active_agent_count; i++) { - if(env->goal_reached[i] && !env->reached_goal_this_turn[i]){ - continue; - } float* obs = &observations[i][0]; Entity* ego_entity = &env->entities[env->active_agent_indices[i]]; if(ego_entity->type > 3) break; @@ -942,17 +972,9 @@ void compute_observations(GPUDrive* env) { cars_seen++; obs_idx += 7; // Move to next observation slot } - for(int j = cars_seen; j < MAX_CARS - 1; j++){ - obs[obs_idx] = 0; - obs[obs_idx + 1] = 0; - obs[obs_idx + 2] = 0; - obs[obs_idx + 3] = 0; - obs[obs_idx + 4] = 0; - obs[obs_idx + 5] = 0; - obs[obs_idx + 6] = 0; - obs_idx += 7; - } - + int remaining_partner_obs = (MAX_CARS - 1 - cars_seen) * 7; + memset(&obs[obs_idx], 0, remaining_partner_obs * sizeof(float)); + obs_idx += remaining_partner_obs; // map observations int entity_list[MAX_ROAD_SEGMENT_OBSERVATIONS*2]; // Array big enough for all neighboring cells int grid_idx = getGridIndex(env, ego_entity->x, ego_entity->y); @@ -1010,31 +1032,33 @@ void c_reset(GPUDrive* env){ collision_check(env, agent_idx); } memset(env->goal_reached, 0, env->active_agent_count*sizeof(char)); - memset(env->masks, 1, env->active_agent_count*sizeof(char)); - memset(env->dones, 0, env->active_agent_count*sizeof(char)); + memset(env->reached_goal_this_episode, 0, env->active_agent_count*sizeof(char)); compute_observations(env); } +void respawn_agent(GPUDrive* env, int agent_idx){ + env->entities[agent_idx].x = env->entities[agent_idx].traj_x[0]; + env->entities[agent_idx].y = env->entities[agent_idx].traj_y[0]; + env->entities[agent_idx].heading = env->entities[agent_idx].traj_heading[0]; + env->entities[agent_idx].vx = env->entities[agent_idx].traj_vx[0]; + env->entities[agent_idx].vy = env->entities[agent_idx].traj_vy[0]; +} + void c_step(GPUDrive* env){ memset(env->rewards, 0, env->active_agent_count * sizeof(float)); - memset(env->reached_goal_this_turn, 0, env->active_agent_count * sizeof(char)); env->timestep++; if(env->timestep == 91){ for(int i = 0; i < env->active_agent_count; i++){ - if(env->goal_reached[i] == 0){ + if(!env->reached_goal_this_episode[i]) { env->logs[i].score = 0.0f; - } - else { + } else { env->logs[i].score = 1.0f; - env->logs[i].dnf_rate = 0.0f; } int offroad = env->logs[i].offroad_rate; int collided = env->logs[i].collision_rate; - int goal_reached = env->goal_reached[i]; - if(!offroad && !collided && !goal_reached){ + if(!offroad && !collided && !env->reached_goal_this_episode[i]){ env->logs[i].dnf_rate = 1.0f; } - add_log(env->log_buffer, &env->logs[i]); } c_reset(env); @@ -1050,15 +1074,13 @@ void c_step(GPUDrive* env){ env->logs[i].score = 0.0f; env->logs[i].episode_length += 1; int agent_idx = env->active_agent_indices[i]; + if(env->goal_reached[i] || env->entities[agent_idx].collision_state > 0){ + respawn_agent(env, agent_idx); + env->goal_reached[i] = 0; + } env->entities[agent_idx].collision_state = 0; - if(env->goal_reached[i]){ - // env->masks[i] = 0; - env->entities[agent_idx].x = -10000; - env->entities[agent_idx].y = -10000; - continue; - } move_dynamics(env, i, agent_idx); - //move_expert(env, env->actions, agent_idx); + // move_expert(env, env->actions, agent_idx); collision_check(env, agent_idx); if(env->entities[agent_idx].collision_state > 0 && env->goal_reached[i] == 0){ if(env->entities[agent_idx].collision_state == VEHICLE_COLLISION){ @@ -1082,10 +1104,8 @@ void c_step(GPUDrive* env){ if(reached_goal){ env->rewards[i] += 1.0f; env->goal_reached[i] = 1; - env->reached_goal_this_turn[i] = 1; env->logs[i].episode_return += 1.0f; - // env->dones[i] = 1; - continue; + env->reached_goal_this_episode[i] = 1; } } compute_observations(env); @@ -1106,6 +1126,10 @@ struct Client { Vector3 camera_target; float camera_zoom; Camera3D camera; + Model cars[6]; + int car_assignments[MAX_CARS]; // To keep car model assignments consistent per vehicle + Vector3 default_camera_position; + Vector3 default_camera_target; }; Client* make_client(GPUDrive* env){ @@ -1116,22 +1140,33 @@ Client* make_client(GPUDrive* env){ InitWindow(client->width, client->height, "PufferLib Ray GPU Drive"); SetTargetFPS(60); client->puffers = LoadTexture("resources/puffers_128.png"); - + client->cars[0] = LoadModel("resources/gpudrive/RedCar.glb"); + client->cars[1] = LoadModel("resources/gpudrive/WhiteCar.glb"); + client->cars[2] = LoadModel("resources/gpudrive/BlueCar.glb"); + client->cars[3] = LoadModel("resources/gpudrive/YellowCar.glb"); + client->cars[4] = LoadModel("resources/gpudrive/GreenCar.glb"); + client->cars[5] = LoadModel("resources/gpudrive/GreyCar.glb"); + for (int i = 0; i < MAX_CARS; i++) { + client->car_assignments[i] = (rand() % 4) + 1; + } // Get initial target position from first active agent + float map_center_x = (env->map_corners[0] + env->map_corners[2]) / 2.0f; + float map_center_y = (env->map_corners[1] + env->map_corners[3]) / 2.0f; Vector3 target_pos = { - env->entities[env->active_agent_indices[0]].x, - env->entities[env->active_agent_indices[0]].y, // Y is up - env->entities[env->active_agent_indices[0]].z // Z is depth + 0, + 0, // Y is up + 1 // Z is depth }; - printf("target_pos: %f, %f, %f\n", target_pos.x, target_pos.y, target_pos.z); // Set up camera to look at target from above and behind - client->camera.position = (Vector3){ - target_pos.x, // Same X as target - target_pos.y + 120.0f, // 20 units above target - target_pos.z + 175.0f // 20 units behind target + client->default_camera_position = (Vector3){ + 0, // Same X as target + 120.0f, // 20 units above target + 175.0f // 20 units behind target }; - client->camera.target = target_pos; + client->default_camera_target = target_pos; + client->camera.position = client->default_camera_position; + client->camera.target = client->default_camera_target; client->camera.up = (Vector3){ 0.0f, -1.0f, 0.0f }; // Y is up client->camera.fovy = 45.0f; client->camera.projection = CAMERA_PERSPECTIVE; @@ -1140,6 +1175,34 @@ Client* make_client(GPUDrive* env){ } void draw_agent_obs(GPUDrive* env, int agent_index){ + // Diamond dimensions + float diamond_height = 3.0f; // Total height of diamond + float diamond_width = 1.5f; // Width of diamond + float diamond_z = 8.0f; // Base Z position + + // Define diamond points + Vector3 top_point = (Vector3){0.0f, 0.0f, diamond_z + diamond_height/2}; // Top point + Vector3 bottom_point = (Vector3){0.0f, 0.0f, diamond_z - diamond_height/2}; // Bottom point + Vector3 front_point = (Vector3){0.0f, diamond_width/2, diamond_z}; // Front point + Vector3 back_point = (Vector3){0.0f, -diamond_width/2, diamond_z}; // Back point + Vector3 left_point = (Vector3){-diamond_width/2, 0.0f, diamond_z}; // Left point + Vector3 right_point = (Vector3){diamond_width/2, 0.0f, diamond_z}; // Right point + + // Draw the diamond faces + // Top pyramid + DrawTriangle3D(top_point, front_point, right_point, PUFF_CYAN); // Front-right face + DrawTriangle3D(top_point, right_point, back_point, PUFF_CYAN); // Back-right face + DrawTriangle3D(top_point, back_point, left_point, PUFF_CYAN); // Back-left face + DrawTriangle3D(top_point, left_point, front_point, PUFF_CYAN); // Front-left face + + // Bottom pyramid + DrawTriangle3D(bottom_point, right_point, front_point, PUFF_CYAN); // Front-right face + DrawTriangle3D(bottom_point, back_point, right_point, PUFF_CYAN); // Back-right face + DrawTriangle3D(bottom_point, left_point, back_point, PUFF_CYAN); // Back-left face + DrawTriangle3D(bottom_point, front_point, left_point, PUFF_CYAN); // Front-left face + if(!IsKeyDown(KEY_LEFT_SHIFT)){ + return; + } int max_obs = 6 + 7*(MAX_CARS - 1) + 7*MAX_ROAD_SEGMENT_OBSERVATIONS; float (*observations)[max_obs] = (float(*)[max_obs])env->observations; float* agent_obs = &observations[agent_index][0]; @@ -1166,10 +1229,36 @@ void draw_agent_obs(GPUDrive* env, int agent_index){ float theta_y = agent_obs[obs_idx + 5]; float partner_angle = atan2f(theta_y, theta_x); // draw an arrow above the car pointing in the direction that the partner is going - float arrow_length = 10.0f; + float arrow_length = 7.5f; float arrow_x = x + arrow_length*cosf(partner_angle); float arrow_y = y + arrow_length*sinf(partner_angle); - DrawLine3D((Vector3){x, y, 1}, (Vector3){arrow_x, arrow_y, 1}, PUFF_CYAN); + DrawLine3D((Vector3){x, y, 1}, (Vector3){arrow_x, arrow_y, 1}, PUFF_WHITE); + // Calculate perpendicular offsets for arrow head + float arrow_size = 2.0f; // Size of the arrow head + float dx = arrow_x - x; + float dy = arrow_y - y; + float length = sqrtf(dx*dx + dy*dy); + if (length > 0) { + // Normalize direction vector + dx /= length; + dy /= length; + + // Calculate perpendicular vector + float px = -dy * arrow_size; + float py = dx * arrow_size; + + // Draw the two lines forming the arrow head + DrawLine3D( + (Vector3){arrow_x, arrow_y, 1}, + (Vector3){arrow_x - dx*arrow_size + px, arrow_y - dy*arrow_size + py, 1}, + PUFF_WHITE + ); + DrawLine3D( + (Vector3){arrow_x, arrow_y, 1}, + (Vector3){arrow_x - dx*arrow_size - px, arrow_y - dy*arrow_size - py, 1}, + PUFF_WHITE + ); + } obs_idx += 7; // Move to next agent observation (7 values per agent) } // Then draw map observations @@ -1200,15 +1289,95 @@ void draw_agent_obs(GPUDrive* env, int agent_index){ float x_end = x_middle + segment_length*cosf(rel_angle); float y_end = y_middle + segment_length*sinf(rel_angle); DrawLine3D((Vector3){0,0,0}, (Vector3){x_middle, y_middle, 1}, lineColor); + DrawCube((Vector3){x_middle, y_middle, 1}, 0.5f, 0.5f, 0.5f, lineColor); DrawLine3D((Vector3){x_start, y_start, 1}, (Vector3){x_end, y_end, 1}, BLUE); } } +void draw_road_edge(GPUDrive* env, float start_x, float start_y, float end_x, float end_y){ + Color CURB_TOP = (Color){220, 220, 220, 255}; // Top surface - lightest + Color CURB_SIDE = (Color){180, 180, 180, 255}; // Side faces - medium + Color CURB_BOTTOM = (Color){160, 160, 160, 255}; + // Calculate curb dimensions + float curb_height = 0.5f; // Height of the curb + float curb_width = 0.3f; // Width/thickness of the curb + + // Calculate direction vector between start and end + Vector3 direction = { + end_x - start_x, + end_y - start_y, + 0.0f + }; + + // Calculate length of the segment + float length = sqrtf(direction.x * direction.x + direction.y * direction.y); + + // Normalize direction vector + Vector3 normalized_dir = { + direction.x / length, + direction.y / length, + 0.0f + }; + + // Calculate perpendicular vector for width + Vector3 perpendicular = { + -normalized_dir.y, + normalized_dir.x, + 0.0f + }; + + // Calculate the four bottom corners of the curb + Vector3 b1 = { + start_x - perpendicular.x * curb_width/2, + start_y - perpendicular.y * curb_width/2, + 1.0f + }; + Vector3 b2 = { + start_x + perpendicular.x * curb_width/2, + start_y + perpendicular.y * curb_width/2, + 1.0f + }; + Vector3 b3 = { + end_x + perpendicular.x * curb_width/2, + end_y + perpendicular.y * curb_width/2, + 1.0f + }; + Vector3 b4 = { + end_x - perpendicular.x * curb_width/2, + end_y - perpendicular.y * curb_width/2, + 1.0f + }; + + // Draw the curb faces + // Bottom face + DrawTriangle3D(b1, b2, b3, CURB_BOTTOM); + DrawTriangle3D(b1, b3, b4, CURB_BOTTOM); + + // Top face (raised by curb_height) + Vector3 t1 = {b1.x, b1.y, b1.z + curb_height}; + Vector3 t2 = {b2.x, b2.y, b2.z + curb_height}; + Vector3 t3 = {b3.x, b3.y, b3.z + curb_height}; + Vector3 t4 = {b4.x, b4.y, b4.z + curb_height}; + DrawTriangle3D(t1, t3, t2, CURB_TOP); + DrawTriangle3D(t1, t4, t3, CURB_TOP); + + // Side faces + DrawTriangle3D(b1, t1, b2, CURB_SIDE); + DrawTriangle3D(t1, t2, b2, CURB_SIDE); + DrawTriangle3D(b2, t2, b3, CURB_SIDE); + DrawTriangle3D(t2, t3, b3, CURB_SIDE); + DrawTriangle3D(b3, t3, b4, CURB_SIDE); + DrawTriangle3D(t3, t4, b4, CURB_SIDE); + DrawTriangle3D(b4, t4, b1, CURB_SIDE); + DrawTriangle3D(t4, t1, b1, CURB_SIDE); +} void c_render(Client* client, GPUDrive* env) { BeginDrawing(); - ClearBackground(PUFF_BACKGROUND); + Color road = (Color){35, 35, 37, 255}; + ClearBackground(road); BeginMode3D(client->camera); + // Draw a grid to help with orientation // DrawGrid(20, 1.0f); DrawLine3D((Vector3){env->map_corners[0], env->map_corners[1], 0}, (Vector3){env->map_corners[2], env->map_corners[1], 0}, PUFF_CYAN); @@ -1260,33 +1429,74 @@ void c_render(Client* client, GPUDrive* env) { // Determine color based on active status and other conditions Color object_color = PUFF_BACKGROUND2; // Default color for non-active vehicles Color outline_color = PUFF_CYAN; + Model car_model = client->cars[5]; if(is_active_agent){ - object_color = PUFF_CYAN; // Active agents are blue + car_model = client->cars[client->car_assignments[i %64]]; } if(agent_index == env->human_agent_idx){ object_color = PUFF_CYAN; outline_color = PUFF_WHITE; } if(is_active_agent && env->entities[i].collision_state > 0) { - object_color = RED; // Collided agent + car_model = client->cars[0]; // Collided agent } // Draw obs for human selected agent if(agent_index == env->human_agent_idx && env->goal_reached[agent_index] == 0) { draw_agent_obs(env, agent_index); } // Draw cube for cars static and active - DrawCube((Vector3){0, 0, 0}, size.x, size.y, size.z, object_color); - DrawCubeWires((Vector3){0, 0, 0}, size.x, size.y, size.z, outline_color); + // Calculate scale factors based on desired size and model dimensions + + BoundingBox bounds = GetModelBoundingBox(car_model); + Vector3 model_size = { + bounds.max.x - bounds.min.x, + bounds.max.y - bounds.min.y, + bounds.max.z - bounds.min.z + }; + Vector3 scale = { + size.x / model_size.x, + size.y / model_size.y, + size.z / model_size.z + }; + DrawModelEx(car_model, (Vector3){0, 0, 0}, (Vector3){1, 0, 0}, 90.0f, scale, WHITE); rlPopMatrix(); + // FPV Camera Control + if(IsKeyDown(KEY_LEFT_CONTROL) && env->human_agent_idx== agent_index){ + if(env->goal_reached[agent_index] == 1){ + env->human_agent_idx = rand() % env->active_agent_count; + } + Vector3 camera_position = (Vector3){ + position.x - (25.0f * cosf(heading)), + position.y - (25.0f * sinf(heading)), + position.z + 15 + }; + + Vector3 camera_target = (Vector3){ + position.x + 40.0f * cosf(heading), + position.y + 40.0f * sinf(heading), + position.z - 5.0f + }; + client->camera.position = camera_position; + client->camera.target = camera_target; + client->camera.up = (Vector3){0, 0, 1}; + } + if(IsKeyReleased(KEY_LEFT_CONTROL)){ + client->camera.position = client->default_camera_position; + client->camera.target = client->default_camera_target; + client->camera.up = (Vector3){0, 0, 1}; + } // Draw goal position for active agents + if(!is_active_agent || env->entities[i].valid == 0) { continue; } - DrawSphere((Vector3){ - env->entities[i].goal_position_x, - env->entities[i].goal_position_y, - 1 - }, 0.5f, DARKGREEN); + if(!IsKeyDown(KEY_LEFT_SHIFT)){ + DrawSphere((Vector3){ + env->entities[i].goal_position_x, + env->entities[i].goal_position_y, + 1 + }, 0.5f, DARKGREEN); + } } // Draw road elements if(env->entities[i].type <=3 && env->entities[i].type >= 7){ @@ -1306,15 +1516,16 @@ void c_render(Client* client, GPUDrive* env) { Color lineColor = GRAY; if (env->entities[i].type == ROAD_LANE) lineColor = GRAY; else if (env->entities[i].type == ROAD_LINE) lineColor = BLUE; - else if (env->entities[i].type == ROAD_EDGE) lineColor = PUFF_CYAN; + else if (env->entities[i].type == ROAD_EDGE) lineColor = WHITE; else if (env->entities[i].type == DRIVEWAY) lineColor = RED; if(env->entities[i].type != ROAD_EDGE){ continue; } if(!IsKeyDown(KEY_LEFT_SHIFT)){ - DrawLine3D(start, end, lineColor); - DrawCube(start, 0.5f, 0.5f, 0.5f, lineColor); - DrawCube(end, 0.5f, 0.5f, 0.5f, lineColor); + draw_road_edge(env, start.x, start.y, end.x, end.y); + // DrawLine3D(start, end, lineColor); + // DrawCube(start, 0.5f, 0.5f, 0.5f, lineColor); + // DrawCube(end, 0.5f, 0.5f, 0.5f, lineColor); } } } @@ -1358,6 +1569,9 @@ void c_render(Client* client, GPUDrive* env) { } void close_client(Client* client){ + for (int i = 0; i < 5; i++) { + UnloadModel(client->cars[i]); + } CloseWindow(); free(client); } diff --git a/pufferlib/ocean/gpudrive/gpudrive.py b/pufferlib/ocean/gpudrive/gpudrive.py index 95d99ab88a..8877afbb1c 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.py +++ b/pufferlib/ocean/gpudrive/gpudrive.py @@ -12,26 +12,24 @@ def __init__(self, num_envs=1, render_mode=None, report_interval=1, human_agent_idx=0, reward_vehicle_collision=-0.1, reward_offroad_collision=-0.1, - buf = None, seed=1): + buf = None, + seed=1): # env self.num_agents = num_envs self.render_mode = render_mode self.report_interval = report_interval - print("Num envs: ", num_envs) - self.num_obs = 6 + 63*7 + 64*7 + self.num_obs = 6 + 63*7 + 200*7 self.single_observation_space = gymnasium.spaces.Box(low=-1, high=1, shape=(self.num_obs,), dtype=np.float32) self.single_action_space = gymnasium.spaces.MultiDiscrete([7, 13]) total_agents, agent_offsets =CyGPUDrive.get_total_agent_count( num_envs, human_agent_idx, reward_vehicle_collision, reward_offroad_collision) - self.num_agents = total_agents - print("Num agents: ", self.num_agents) super().__init__(buf=buf) - self.c_envs = CyGPUDrive(self.observations, self.actions, self.rewards, self.masks, + self.c_envs = CyGPUDrive(self.observations, self.actions, self.rewards, self.terminals, num_envs, human_agent_idx, reward_vehicle_collision, reward_offroad_collision, offsets = agent_offsets) @@ -167,7 +165,7 @@ def save_map_binary(map_data, output_file): geometry = road.get('geometry', []) road_type = road.get('map_element_id', 0) # breakpoint() - if(len(geometry) > 10 and road_type >= 14 and road_type <=16): + if(len(geometry) > 10 and road_type <=16): geometry = simplify_polyline(geometry, .1) size = len(geometry) # breakpoint() @@ -242,13 +240,13 @@ def process_all_maps(): except Exception as e: print(f"Error processing {map_path.name}: {e}") -def test_performance(timeout=10, atn_cache=1024, num_envs=256): +def test_performance(timeout=10, atn_cache=1024, num_envs=75): import time env = GPUDrive(num_envs=num_envs) env.reset() tick = 0 - num_agents = 1670 + num_agents = 3968 actions = np.stack([ np.random.randint(0, space.n + 1, (atn_cache, num_agents)) for space in env.single_action_space @@ -264,5 +262,5 @@ def test_performance(timeout=10, atn_cache=1024, num_envs=256): if __name__ == '__main__': - #test_performance() + # test_performance() process_all_maps() From 993e22e7e439ba4a6432973a6cbe97ad12225fb4 Mon Sep 17 00:00:00 2001 From: l1onh3art88 Date: Wed, 7 May 2025 12:01:13 -0500 Subject: [PATCH 33/63] car models --- pufferlib/resources/gpudrive/BlueCar.glb | Bin 0 -> 239892 bytes pufferlib/resources/gpudrive/GreenCar.glb | Bin 0 -> 239896 bytes pufferlib/resources/gpudrive/GreyCar.glb | Bin 0 -> 239892 bytes pufferlib/resources/gpudrive/RedCar.glb | Bin 0 -> 239876 bytes pufferlib/resources/gpudrive/WhiteCar.glb | Bin 0 -> 239892 bytes pufferlib/resources/gpudrive/YellowCar.glb | Bin 0 -> 239876 bytes 6 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 pufferlib/resources/gpudrive/BlueCar.glb create mode 100644 pufferlib/resources/gpudrive/GreenCar.glb create mode 100644 pufferlib/resources/gpudrive/GreyCar.glb create mode 100644 pufferlib/resources/gpudrive/RedCar.glb create mode 100644 pufferlib/resources/gpudrive/WhiteCar.glb create mode 100644 pufferlib/resources/gpudrive/YellowCar.glb diff --git a/pufferlib/resources/gpudrive/BlueCar.glb b/pufferlib/resources/gpudrive/BlueCar.glb new file mode 100644 index 0000000000000000000000000000000000000000..1f9ab06513e2e2d5b7cb7672bd140fc96d07b6d6 GIT binary patch literal 239892 zcmeEu1$0z7)PIsUxa(qtQe=@*+L@`+c`5GB;tmDMV%y@SP~2PG-QBe?FGUtu+-+fT zmqmAR{x@&t!5v_bbH4AK@BGg)CpW+3C3$(t&CN}IK-&&=$}vJn=A{a`otKbGwX0W) z_G%Fo72ee=B-*QOc&G5l7F|0>dWCpZZWr0PQ|Bnpwot{hREO|Rt-~WdD;BTr+1=`6 z_3;nz^7iT;9vKzUxf2vN`}lgrczcES>>A!lW<~uH72et_q>)$U3bmSbY|%A5GNMI? zsAl0EBch@rx`#K5>KYl|scqMGUQN8cqFTXQy+VAQC<>}Ydv$8j5%Roh>(9KsI(2Rx zE+6^YAhW+;fHf#4#yi@}{Wp}aQnGgKAN>lmHNaw)gXZV!A7l-%%CGpYyzl=E|8MlW zbgk-Df3Vxu-~bB@u*KIJ;1?8R@t58HzP$f|ZX0TDOjIL3Z}`Eieq0|?X}+~Kpoq~@ zZDYNJ`1#1uGPJTm*(z14*Ym7Wv3!L(KQ=0ke}5IHtc({q-}JVt9lhG)+zhHpch~0s>N6Tj#;E!GY$$zG*SBM#GMntF9 z5v^nu3%t#_Cqo^%XSi$7>g{DT=U%SsO77T2B1da^IRGYa%mL4<71=nKKDH}!!B+NCoE}~ zM$IlWWOkV%|F1TR=E(fXB$-`i$?P&s@aQW&Me}6-WTGrC6J>FksKBo^YC#ia`DCIj zE)!*OnW&%`-BWAP3Y0ghbEIPlwCvWVO?YJ8i0~e|+PCW5u}kMp;hnnH>D2`sM}I#v zOkk_d-8z9^>*pJ!e|b-E@ytH{;4Fa=vIO}`mnPWi4Zf?z92{r?(`5|`4zvb%`}z2R zpXg_`n*9U(!2$#aHi0>g07th`F|$vQzts{94wtWgU~rH{+BHj%e-LHPLFj-Ywp{D!a9*FX0wx+ursWBt#k0W|gV_oHcm?bWF7W)@)Se%`{zS%A^fEtX?7 zfWpFR2nhDI_``r%r0xd?m@PqYbP2G)*}@Or2g)2MmxCn`R)Rkq2ZBFNgV_h1Cb=AZ z<<9TtC;cW^FtCh*{Ve{$u)+d@E&6)+yory)@cDXxE`IwXzSe*rjPsAQ@cB&0E$H(W zK2Aj-9EZL&7jkd2%wULo!C;wTkY$zy z3{Rlh&+6|B#wH*DPX??V@VWdUk}E?t5NPp(QyVNQGfbw{&+1+5i!FS0D8FbLKKY2R zHt;ROAV=Y217DbmFRsduQ(=+g>}*e2K_0s;cy>=JtH3;0*`B>VOH6 z^YEn>J`Tg@i^A`V#sN*lM}7SACi4x`0NP=UBGmuFG<(=9`d#I0GqHe z4RQ|+2=e#0z`ac{T#Ez-fgysChh-;kaspsimzxGG4LEDSPzPHr0Rh&Jlc4X^7T6;E zVM__{4>aqu0Nc44ZZlx3mVQ0#-td|)H1V}rke=H&zu~J5e8VW{gYdC|&*wpUhoAT0 z<2?9*L-Wm<_{N$H_%IQm&0u=`0|Tvr0sg*zRxm3tHK5&~2jSyl zkRR~X`rj}JuJyk#313`{XcGLty&Ar;9)EZag8c*KMWe+U1Y3mF_oLhTy>sw|2EICm zUo;5cKL?g?n1gT4fjoSCt^+?b2Xdu+-o(dG@PpeR!z6)k1P27^|2tex-P;L*{n%m! zXA~}&VJnbU3T{GR3HV!m&HiR-!tf4lH3z_ji9ehy0{qQEy5fTYGy7Wno+aL1P7QSH;97FN3h0oW7Jhy$`hmZ4N4v^chvjtIRUAsvSnk!>Y?VSz1JzbJ6D_C2+o4SsJ6K>^bH z#;-RA_`<#ZcXuHWF8y5_0C!FIBGEVK_Y&^D0j|wl8vsjdmfJRJ;H!i1c>@;k;D4Y2 z(A7_R;O}sRzq1E{!LZ`+7=lOiX$QVG1QzhiV8~rsfYXN5tMBZ=7g`AN15e@mTY&8l z&YLcMfDIAW$M-h@#iez_p8-4J4~>L88Gq6On3TXD=){*=@OL;5-#M0WN%C1AfaCgs zJ_N{tcNqqM=}^0l|GC!g@O9>_WS zr2Y?Sf_`%UtcAcI7=tgg5Fl^xzkd+mu=MFf1_Z+4=m%PWv(zW!5D44qCw=&uDg?>x z#bqGD$A?Re?=}rzXu$GA%No4mPkUeiXZQPipdV#jKfwyOWUdYT;22oJ*>-KAippMfxt1je!|zrzz;t7Q?nEZHzA)44z$$RK;o}XTJGj z(6HZC_-i1sI}@*R1LuJJacwj3F5Sm)iA(2hr(;t$Nc&$GczV51Y{1kuTev(1(6#-@rzH%IE0MF_z^|hh10awT;vJV_;ck$F4(a z<9);Q@&>M2dWy)pc@^hPqdCSWx~>y*#5^wB$5^)WC}x7*AB^W^jS)3H*7HA#jp7(D zKRR1%D7lo+86LwirqjoYbs_6`j)sFc#{Z6;F7n=9%I_~5!ZGf*bESB3Y!2@{XBc;f z59MDdCOw?b|IIawe-D0PT_V`x>3ox49LIQp*Gn}^z8YX%Ug#MR_bBlXa zpCE2uTFTj9BR`CloKseBskpFnfj*}g8~P*1_ET)kugu5Tm@k=+u`$0g|6926{WCme zv5oxDu$~;_JWmt(A6d5WKa-B)7>~~Ngx5>COxFR7*DO5FGZx>%%OoAd-$q~N=a#MJ z6;||ii>-J5JFdTb-6yy7`G(F6ODB$*P0wpS-!&= zbD|u>PjTAC{)uu7Wj@A6T{&Myn+7)K=rGhRRQb9p^FPJ1%--LJ@nuV_i6bW8P*+E~ z;#SblL-s?eER*lFS;w-S$t%o>4>z6C%gcP(PVR^}y^MinUHM*_kFl&P-*vkED7~F8 z;pBM=CpP%CzrL2oFDjULa?%)mt~*!vNvx48R>yJ43nh*`Ka6KB9>)udFAMP5e_j7+~??h$iP`)1OF0AR3*cbXQ=kbpv1ryoiF*?q@ z(>w9^8$)#*{;Y7~`3Ivv!&wSt3wsefSI>WaDoenc_UFgy{b_zAcUZ?QQ}ubN2)IP5ar%3$M?J!-G?~h!{wIZT9#?d>qd#wRhGnQQ zTQBc@GH2Mt@-y`^vmn3XKVx;B{|DNCJ7lVk^Fo=?)uwW(+jDju)IxKyuoAQ~e_Az} z$T#5~KmVq&3+`7esaSmc1^1czE5~@#sCU}Ihi`d&%Wz)C>#???1`V6Dw+dJ3S?yvH z8djxzBNv?i=5y`$VI)i}Zp1N8JeNZ>ZT3>1YmAM)Ngb$lA(gmy;GJG(U-ieDwIvOc z`HgPGa@l{0%g>m`b6kF*m%(`4Kg0Q|29EMsn~&!eM>*Ot>dJ2~vvLxTz4nsZ{%Pid zGkA>U+uFY4`>QnJ7~hQ_$bU&j!@?Ri;uv4QGn|(%Ps7GMufhu@AIYQoldx_l8oA(0 zSx58j=ihR&G+eibhHqo|Gg2Rk@~BAS;I3_T zUGltJJh6*UTfKbu3&j$1R%@%biE;UMIl{KQYNMAo{3zK6jQ5nx9k%jy8@(>ZVVUxV zg@fNE`(yZBvM$DlDtUyJ0DnxbHH<@_@L`Lb>Wue@Px0n6{jICJ6Usr#Rm$ zarUVNRl}FaTx9tNEZ=M7x1L;3HT(*UA3}K}A7fe9$j3N`_j1*+SE(l!{D5T{Bj0P> zTD9u$8~N*OqxkpW72X#0P{)mYY0gpndvNH4!RqL#8+ol9qxhTnf$Ez=EBMloIDUU< zLG{;allA$bzLIe z;)>4xNSm|7*=LE3zDfMI*95WY_)_lEESj&e4igdmmT>ZPDBqSkMm(Rjmd|@OnE$kD zky!F*mTs?x=03@{wqL{V4DZQ{zDVROZWDihWH7H7dy`uxF4W8PJCZ>}oq48X&#M#p ztat~0RH3-K_QW#o`)DY~IArE(b$5~F{O-e{9OEQ)ENaf<>-p(BgE_{QMQhb@KX2e` zZVl!br*5%S%(}RM@156xyTb#gE)cCUEaoj1^y2RDs_aX|<8BLimDmm(<5P$Fi+`ui z;%iUEbBvE?pDjA?nadMfkLK?1(dom*=x4KehB@)v9UiiBgh=&tJ};9dp1Z>dC;#C6 z4zJ?teB-z~d}c)=&%AU#pLR5!yTdKSRerVGG~NPi9meO(m-yUci}|dEqq#f0u>3*p zU4JPbPU5*cy!F98E=n)tzfXzh?r`y}cX+D_;Lo?N%P~It_btBMYXu)$ErMhG#~;`E z`nfB4%}L$4J8bOBvF0Kil zJ}0aJcZUuCIfw&o3gxgdgLynjntwPDhyeCFvW z?hYr8D5V~(cArNd@4?;S5}gy(0r{`-*}ueaj2FyK!QaO|=h~o1?hXfR+oQ&>eZyC* zh~OCCN;ssZt973by4{0gyfo!~H9qB2p5kc~$2jknMWTJG*}TA_2E6F38KUi``Mkua z7+$;SLSb7uo5v08zz;5;E9SSH%NsB5<$`7Mdg5h^$Q4Qa2= zjhx4SUU!ps_HMvCPngUzZ+OE0NzVD(fEoPioCo}Jk{bN+t%=<0+znpi&rZCQZwLN< z%~QUyMH`+c*=WAw_xs#uetTY_Wq*G2^kbfScw2tnf4&yG^eJ!hG@Revlu&RQ~KB`^2@|bUU(uQXbBqB+&2R!Z82;RKuu+XDBXY-RW!};5L+igi2&gGF6hVy1E zvZ)W+EaW#g4RgUA)4ft#uA0O9LitMxTU7JWg}f$|!FcJb6Y7GObGa4T8MHJ7FSOz* ze-qY*-;POCPgTFdD`xzSCwSaf?f#E>-PLWm$H7Bt)tI~d#e!dXd~i*5N}VhG+&^9T zu%<(8^;bRRSu(fbc~_QF3lF%9#XIF z8K&nOSiV=Dv*fwPz|xPEZ5n4NiH)<8@m^WRxZl3rH1CJ|TZxT3Mv0AkKZ%pV+4jzk zL+Yov5}a*Ql)0|wV{F_X%JMnjKI$3Vvq?PsVd2EV7e{}FM{V^^EdS?FJ^!B?K8ao6 zE=!g-?t&zK5BD=hA22QrcRNNOFgEUvWL@LlM&bqUa)!+>H(PHX<1=utwR`JSJs)G^ z{zuj|?l&ZUYkRCUY(&E(Htr=Pp7nS*?_0}}FZlvFM=~E{;|^TrV{F_f$ow-)eG|9A z{eZ;Ax&BjZoY`eQ#%AaFSLV-x^P16zGtTp}%rEXd6G?2GgCySQJReECzTr}_;ot(j z4;b%TyGZnVHA~NrO?QX)j$Qs4E<5NZPd|5|o^SY{vQ3N)J0|lDpXyU=*fE*k;bKzM=2u*jNj)yrJ_FBi>3N-b%>vcdii~JJ*h0#<@0KurZIY?gYk&htkVC z?*#NR&btK{jJPL(v9UH~`xwi2Ie93(9RnloNnnh)CxJ2Io&?5-dlDEU?n$`Ah zgAw;6Fh)F-Ks=ONjCd%0?wt1UGi=PGoNJ7Yd6D@T8*?Y~zl9O6Brrz2lEB!ov2uPf zMjVqs9Ftp&I3|HOCV@C6fjB0CI3|HOCKqhjZP_=&{u|gBH^lXbPq8cm_w54lJ}wyX zJ_7MR0`We2zHA5h9T$wa9)U69ee`ym_x3Ir@je3aJ_7MR0`Wcq@jfmXaXkWK#PtY_ z5!WLyMqH0@hY{~15bq-p?<0);QO=RE-${(P5`nl9fw&TZxDtW55*<6a5`nl9y{?lh z(c3iU2yrFC*ehipj6GLk#FYrdl?cR@2*i~L#FdCWAGs2NxDtW55`nl9fw&T3?AO2- z2*j1RV8o9Ij1gBN5LY4)SK@*ZKO!(Td;{4B#96pt#8U{25lP1tUH}AU;CKyLw~X5Fa5BAEB2q{0GEA2#gUQArK!S5Fa5BAECEt>_>=$ z5Evs4LST$I2)!L+|3Z9(KzxK=#<1;(gAm_?5eFf@2O~a0AU;AM4#EW^4nnxYh>sA6 zgK)u!j}V5hE7v39AY3rwAOyyUgAf=aK0+Wq!YxKzgFsw^KwN`B{DMIIfob0%OD%2#gV5AlzZZ7YKJ4@dW~7#1{yR5nmwOVZ;{*cNp;n!W~9@fpCWrUm)CJ z#1{yR5nmwOVZ;{*cNp;n!W~9@fxsB?1p;Hl7YKLQIHSS+lE4`81;QOhe1UL>5nmuM zMtp(581V%HW5gE-j1gZT++oBQ2#gV5ATUOJfxsB?1;QOR&bDwDC){Dg7YK|IUm!3> ze1X6i@dW~7#1{y681V%HW5gE-j1gZTFh+cVz!>ob0%OD%h#$p>FAx|bzCd7%_yXY$ zBfda<4@P`}aEB3JATUOJfpCWrUm!3>e1UL>5nmwOVZ;{*j1gZT++oBQ2zMCq1p;Hl z7YKJ4@dW~7#1{yR5nmuMM%;lw+<`#cfk51W3r5_5K-_@~Mtp(57;y&zaR&l%2QC=# z1p;Hl9SFo72*e$@V8k5=#2pCZ&Q02O#2pC49SGyjP1-BO9SFo72*e!-#2pC49SFo7 z2*e!-#2pC49SFo72*e!-#2pC49SFo7xM0K`2*ek-V8k5=#2pC49SFo72*e!-#2pC4 z9SFo7xM0K=2#gVTAP{#T5O?5$jr&Y__Cef%KzzYx*vS{@`35%bNR4)kGY;YlKHNRR z-Kh&U?yO|KaYrOE;s{(Y;spf8#$A)Fi#P&-I06@pI0At>>-b2wj=)AXg!N&Pr zmO&iBhjX;dH_p@&8)xKCG2#dW;s^xd2n6B?KAf9mJBT9?h!=3dh$9e)BXGfpBlzHF z%JvPPQ)0tjNo@E|pJKya$$Z2S2*eS%U_<|9d1FmU4BVKc$DHw}SWnBp%jNu||9$H( z;(@iIM6l;<`S zfV5r*`H<^zzS4{sMxIJFagcwi4ok2V8z^Kc4#!d-Z?%h-Njji~T#x@K=@F zs5Os;X(2m$@Y^|yy5M;g?`}^O^2j#DvoFVZLjC|g)>>Rt%#%+Isa8hI_WYuHKB5J` zp8dQnrKPGZKB%vF)%c#4-r9^;JLj$5^-U!P_7CJk`(ARv#TxvpRi06rkGyhT#dyh( zvRd&iRrpWw8Mr$7CQd&y(?apqUkv{O$~^Wufe^MVhHY73k6(Z_h~nRfij zfs^|D%z4mM%YC7jo-gNhCd^&RIgYu*81gho>*X;1(%i{*oBBNac@v z@bj~ZyWqlO((^al{CTn1GwQNqf2p}Iw&bObdb!}J4aqfoss*+ym-`5o_FuJo+A{q9 zjB{$ma!XaO=_z=hN#*!~dehZ;TT}Can&|`(Iqu}L~Gt4Q-HeSS|xtFP8Pm< z{9-jl*;AsAG55QDqwv%XVa0XB`#lYqA|^k7F$BI_q$Zo$rzLxjVc(6W3Z- z4pd83?!kW!>!lqE>CTH&q6Y8iC|0-GsxC=7-^TIb##5EUaAsl^|!QBVCYyPCv{2MVySZyOB-$I z7{sd7xlhYToip^U!@g*K`aaQ=V4M-ugxpGiwx7R_P)RYPZ_-Wa-s^i1t{tudck z>51JoLoX}q=(>vZLzQ(6UAp+>wYn(Zb9)J`F>jsehFT%%4Q*CfIbJ8WC!f~1x*FTo zq@MFGt!^IQlMmaKN%c0H)B&?os80q)^ZB8T`RG<})!%x)vT5TQaFej9-N(LB6Q-5r zhr(~Fr~kg8UQS(1O<%7b-xjOzv{nCA|9sYj7g+Vm_Wt2zwdl{~`ATwAExh`f+O$|+ zHB)F4{mLRqVZ?wofR>r#6bW&t4Y8XKlG>`z`vCHmA%lJlD&W zD#km@4A=banfR;TWjV$jyZ@%`%*^@R>UsFu)PuAnzjWvMTCKAsFW16;xOxP)pFXN$ z+^|V~t>N+ZeC*j3D#nF+W)VG3RoC@lR;a%SJY?19F?-YkdkgP=dj5sSFEso5W;$lu zMr!^Qy6d=kx|OzRSsZ2B)$rgc|EZ|wdye$E$tvb<^_~)K>Ut(MZPy zc1^Zjm=nYEX1rrN{Vp?Kcp->C?;=!;b9&xYvmCC)54>2eVjOAhZ0oo_f+wCmtYW-n z>`1j#t!{k%z16nE@50qWQSJHR$Ymt!lyo3$sJr)b|yjC(y&t(WtNI(7R(E!4Kc_NL@-JT5~yF+W3hwP%K` z{PdY5>c!?$LhJ2};n^ET^WPt)QGcFkQZc?zaF9B0O=jDYf<3tRBMYzSbI^9bvX!5G z*@zdO{laEz_fG8`7{g8V;%o;tnOty(9Vykg=mMcWFxD8;%T0K2WG~y=n73-{`h(PV zN5+PdQa$*<4OTv6V@=zsKdt&ax%V5fKPhM0z27LGI{tjU4E6)R2fOntaK3ZD)W5~> zC1hWIG#~q}U&5QZ^y63O_qTUiG(8mKDbvbqD|5ZSx^Q+6j&b;mGTWx4iB)f2=)p1G z|64yXu0(&^LS@=^jFZeNvu*OLShbkegEv2$Z(HS}A?nm^(Y)_~z=Q%*%(f*v`u#Be zJH3+7y>K#H)eDX_Y&sXii}yF_djQ76PsQ*>_Z<0k&&2TUVAscNskybpfHwT<*1ueE z<;^v>{#&aJFBo%4#d!Uunp=l-Xv3Fex};*fF_V(;yrwtbRMWEZR|vXE~Nu%sp}1_M&J2FIA|xSlKC`iv8P;a~|2Mc06o< zb*GQm=3^62SRujA=TxyjhHWFZZ(B3>5CL-ss>6Gfw`1&Gu@7%~qNsZJ`qAwe<2l6` z>kP&fPPOHEr=L;>_n2Y7lr>6wwxB0p)@4K}#?@9%QDavXP`!?I5q=BH@WbbKsNBaS z{L@d-He`$9=OPZ-ri3=&KHZL}Ym2SYaQ?a}b62L*sGREfd+XKe{W@@r{eG__M!!iVO8$A+wrztc^txDLd!M@>$9UD- zllC09;NJ~+XVxBdIvkqq*ITw)<@<7s)l~J=_iyf~2TM2M?r?{VgKUv|Le$~lKjjH5 zC$ePsU&|leSu|=x;jaB{!jh=wy3fV#UH;MS4-AesNwvf9p`Epj#X8Ab?PV! z7CNNb>b=%ZuK!8&^{mD-?MVntea#}~MOj3#g)i;Xi$rK!e{Qb5zaA-$Ow2E8?FiwK zw6luwn=EO?oRv1-Y|0MR9p03LiJeUvajoM98^$D!muNOHueiPXhPMBVmk4@Mn)l^< zZ2O;G(xRIFr6o?NC~$tez{1?>5iDqps_MTN{IqfXuB*Y@oAWMzEwH7&)K!ch6t9+i z9J4*|b;2VGHsKf_4C-n3*;Ih%8d!>BTqgN@^{)3XV%78>>ZqGD?X6~9RrhUd&C6tp zQ@5q=AX0mcaLErl&|ZE2wv(3W&Tr!G9Us26wS|w&RZGQq(2_j--=r1zPm{KVV!W|t zQ69IlgzyP|VZWa?1uu}U9&ht@oekqrwa%)4O}(pa^Q$0IRW8aO%ns%8UH8}`Hss)y ziWU}gZ(P%GzVEn>p>!GJ<8w9Ufmv7C1G;n-i+>-hMr^UDy_R2B+fHrHF+Ln@7Sz*E z?6_81P4zTRy}2Y4pEtA&$2i&9{WDn8dXd?xs)%Uki|`zC3}$F&z} zo{vy*zS%pO*t@QVFwcrodqhXuR{zn5=YEq}TkAb0VQH@kwgI*K@wn1`c(!`p>Y5uz zw_|*LTh?up)IzFta3AgtkD9SN!N1r^TbbE?ImUAdceh%b-1z<&Z7! zroOz|==bWZ0`PB_=cdrgRO-Nsl-#KnEc*pRqWl_A08C=vwBMy7%frJ1$c#(+aI|^G-bakM!Eri~|#L)_<)gBx}Yo zes(39)^=!=*wH6W?Q%Nbw&|s2s44%5;ux2l)lU>yRo_0jc$3f`TWTg8OzOqUS;}#Y zAGKVTQ1hpqYVYeE_>f&gw8ZosMXT5mYV_+8;`p6{qFv{bDlUWbclV7J=L3VZRkNzv zx{vP7M;@u5HEP?#774yY(l;~IlFOs`fZa@8R5D8R7&um42;Y*W{X4Z^p{5+;UC_7R zx_j|MXUcMn1B=$>;Xz47%^6hu=je)pr}h%w!}E%5b8l$Hz%IR9Q9_*V@WOsvHfWg$s-PkFIIhF10b6#fAcYBKh0WYMDx_>?=EW6~BPZx$m1yJYQWy z+@2Pt9y~coOFOo`pwC9A4VEpijRyPt7ue?#YGM8)Yp|#bHXPg7h`jIBo&|mp3#au^ z@j6<2dAZgpq7zpRq}SZ77i?$Tt@l(Z7O79$bmqD4W`OT!>gCc=;!5mT^~TuV+}f{# z*7i+zTZ@~|)r2(l#GIo2)p?w~o4gEb^a?Nvalql6N z&ShO-{dc!^-e(-Ry=sfh>iNw0b~+v=fx zV5z?PnZ|ELF)glAAAY*x%&i!&4#=U#XO7_>kng;U6|yDgJ*$qLefbQ-ove`MWO--5 zbX@Rke{IOz9(+YbkC3SNj3WDuntbK)zg%$fU+amG8Yy|7`}aaN7VjihFWRQQjtvZv z_r>zA*x3hFzC&W@!-qR1jOD$s(TBr-$ME-><38$g{&H%o!j3zxlpcN5rMbU;uay|m zgHQX*F~-{#^yZ|6<4&w#gC%y0S3mv?zNOouQQCxp-TCal?}hNfYi)H>_T%A|Jwo;m zouJNn+m&Zc@iIjA)pcI*`S)`?a$1bYyf2j+y!BX6^SV)DMDp3{pZv@g3m-4XE>3ISFBfwkAyP#tc0<5hZoxBv+(b*=cFhRm{Q@z?=|A7ywkg2 z{Owrh(Rb=Pgv+_Z_&yzDe7BCV^M1h4QOQH;*m<|$f)NkJ5f7yy9?AtH?nz*bcqk3= zPy+E#9Pv;Z;-OqH;+`~&5%(nAVZ=RY--8kNBrrxilt4U`9dSLEPKcZobI14-CDKv}`PoZIqcnS?;#97!8PoZIqI17O|3xPNbJK`(^;w;3K zkDP^uG2$%jh_eugv(OM{ArNOF5N9C}PoZIqI17O|3xPNb4RIDO81WPWW5ii#h_eug zv(WbrcRs+6=3{^0d$7Ab21#)yLu7$XisV2tjlLor5tgg|_R zKzzgxW5h@3^XTLtbRPg?Cm*5bJNXC=@ewWZ_<~T3@%Uqm=NDtd9oP_GupMJ`$HEbJz!67aN8Evm^U*zwins%T zxC6Hs-Os3q7qB7jK%l!FySp-oFAx|b?!b=t0)a8Q3sMnZATUOJfpCWrFCfr;kqzA! z*>M?k*QDa|h&$MhxC0yF2<(VEP;oxGcjAaUP!V^q9o;qA5O=^4U!Y-(_yWxxMtp%C zW5gY(h$FBg?ttTb#2sjeI}nIF&=6lBFh+cVz!-4{8sZKFy4P~aN8ABNe1V2B;tRH8 zjJN~L7hoR2j_&9<;t1@BJ5X^x;tmAj4m89U2#gVTpdr2h{?EA&Jc2-cfeS_)fk51W zhVv13prJcG6>$dw@dcq6Bfdbm!-y{s7$fdL?LKXH!n^keZTmL$73eMz?m0hj2R2*= z@dW~7#245wMtp(57~OfQh%XQrBVItDyHFKz2R2*=aR(~m3k1f9JJ1kcATUPU!FI$K z2#gUgAkh7%hByKZm%;hy?$nOB13Tgl1mX@f#1{yR5nmuMM%;mhxC4Q>1DAZn9oP_e zz!7(#y4%{^$ zK=*r{+<}h2bWbJEF;4D4uZ!>KzjVeyyg)F9YjvxVX1PO>ENchs&JMyIbDo3ydaRgg2FMu)T1u*`B_j1&q&oJTwf)Ga# zg7Xna5Q2DtV2lw*kbrpsj1fnWfO!Fo5l65MaRe?HfBPpKfeO#y8x!?-*|NJbCd-zk z)W6#i$EuEe8O!`I?nuj6FXv1XzBI=LM}21uO{r;TjFx$7dOge!$b5q zxgIy?$y>%j`eU40uh*OhK|DKOcD-L&^UF9W;QjkNoGh#r`Y~?Cmz|1J-@kXngm~)Z zWqXeIxu)%lq5hJ6+rC(?LiugOlR^0!6BA^6-ff5K{kk4LR6SHb7bjCfw^jZPV)#Sh zJ+bc&mXPhl1_g(f?&ZP5nnZ^hX}$dpGvT@H2Z8WuoT!fvg?W(ULthVW1KlYN>3ZHV?I227`#vR|7ceEeV)Ai6Gy!l z^W$C8ok}^L*SOtF&nHpQYPTD?IMe6RlfQZ_`{&6A^cb5IK1%drI(ZA;}{nx{XxGH_2+}X+czY|fPWS8%Uaf|B1-@kuv zt98_&XO8bC>LYk==&N(Qo&AW_>*=k3&{NqD`A5dF+_X2dEDL{{bJ?D6-OOC}?^x{| zkRNMldE4>a5y$dM&GdX;S&wBsPrbZRU)K+r{h6M6a%(5~J9H|SMm%IhUYGpWP|oOQ zwjD$D{%Sj&>#cNOeSA8sbfnjeiBgZ}l>*I=H-SSL(Yb-~OfFU0D0{>_v=d^tRqv9$kE@16UAzHb=$@IB6bMAno0gk%5E*Ds`X z{XqG~b7|+~dmZKOI`l~2mts}PApN1Uj3 z8UNU}1)e+Jov2oT_vMFjMjHC##18%LBl{!Q3)FMkHF)mHM^#9!&!db#O-ub;a^(^H z`%zCl{xmJ~*EE*z@#LQEi8{4kc7A)zV_T(I*f-%jCymI?$G~?_hIkajCRJ5m0d-sZf>9JxO_((hnh@7Wlq zej3a7PYRx_=ljl!Q^R_D@a8eoLyb6NYOpVbvX*En(64!cy->|fG=%;b1 zBzbsj!XP zfqC1OCl^m1(MRp!FYR5dy22U1f_6g_yQ;M}d+3@p z8+L29K8{BGu)}0MEn_(kPkN4p@yNrMosi?@h~+pwIGfeAi5yoiMe0*?y}vRYw8&9DmopysYO4NR#VBCKH?Nh| zCi_D!_S5raTJ|?=cMUyF4m#r4Y&{(Q9K`Z_JtyxDWl%0QSsy(=^peA_hB#v*?N#0J z-aU!(J3YCK<^EM^to%Okw;-1Jh0;698F9+H@;f}acd#QrbFM~uTE?>8!5P2l@s@~w zAJVe?lynaNtWX9=eAL^aAJ$?~dbwew-G^BAKeSReJzb;xZ))@m4_;w><`1!K53iG= zQ{mqfXLQ&joS$Dy`WxS#iQEw&Y<}V&RwtX5u-sQoQCwpQWFy4cA>2pJ$ z*C|`J>G7OTVp8b3o_{v%u1nnRFGsyC&N!R-nEw3JIQ{!eHmMguPj7#S<@jYAEC2of z$GG)LFFk$qeQG^!=S*8GrP0$Bobd|P@!W_n%>7AUPZxHl*7fJN;@Lg_XOwqmhq={K$er| z1o^#Xoa;VhO1z#n;!)v_{#(SG{f2g51fvY`uY`Yt_>$`IEusx4H z#D=~HoXE}R{E}QXVy{i$7v@W=w{OhPqj<+W#`gJ1&-YF5n6JyF9QtDDRbuEZT|bOC zY@0)0WZZXQR=FRMvn6uM*eb_Srm;Pf{)IfpIPw#nvD_)1!haM@;ao>q@NIn!gP=ZuZC5j*k|)!1z{^>e+9WqrqUuzM(7DlCVd zmgf*hexfSNt+@#ONAaPw;MYUih~@dhXit_iAIqb+Z^WtLJR+YPu{^iPwDg}73!PMV z_H3%h(r=dO?IYp;7>8Jni|cne`XSFJvcB{KWk2OO8L=!c)5dd0`mUpY;J?XyIR|KiBnibMBezls-SMF`Qq5Wxi`H z*Zru0@;g1b?-)lMs>j6HPse-ZcVc~T=!Nvd9saU${&o7Dvgm*JHE@5UuS@;DCQ)Bs z|G(YK$oc;7KE?fg%76D5U%toq@803d_YTsY{&&CdfA@Yt`YjG#;nRDB&+iZL`9tml z_?#p6Ykcn_pZ|AH@ZUXw?l1gzPoUo;$o2ByJpr8e|GOv9&nN%g6a3$~C;0R{sNWCh z=hFY~1&njQaUWpB|3BUfcrZ!^6Aw0+q+%*HkwZkKhsYFCg1w==LG~s|SrTF*mq`)` zf06f;lgS__lME~`%gQE^ybunPr}P!w334Z?M?6_d)(d1Wl9Y_2uZR~NN4=;AP0#YP zlO#Qakt7Mr!;*naMrx33qzK7Evq8v8lMo9T1@9U~PLn*$lO+Y2lq7>5z?K62BlI%a zPLk1ebURr`V<0C9IR#HoLC^CsFLngv5psguAb!Ljq(8Y$UeVX|50HP5o#ZurNZW#J zOa6dMw=YH}eJQOMHkY`I-1qPYC%* zaq3Nqk?f?fE{hRwK)Fb6khvizH_1o*Xl@9_NGK^u%7QFQN&zZI%7ZKqDH{o<3dRvR3w!^RwAWnWvE$}6lc{)4qBY$puVgu)G9@xMOumcLK=|9 zARCj0q&{g%nt^Oa8j-rB3B+|FGzHX>v)Jji$w2WTuA2XY*wMv^fQkAyH5&;&9OVM)PODkxd2l0 z;n{K$%(~EyG?;ayR@RwnB$0HciDVnuOzeO+L(o8bW9SZ$JBT2?X%A`xX@l91rdyz7 zG~EVI65!bm(7w8~HuWI2snp1VppQja0lJwMWt(X+Hia@u=@bYfpk_(Xtt}AR(=gDM ziX=6;PHus`MbeYB>l4Q66NQ;r1JE`T`vunUAD}6}!g~6Z93Tf__Pc}ZPWHpE zJLtrI2>ZZ_^d>PNV@PtAjP)VWBpHh)DOd`Sy&&#IQnH6MD=9=C(n92K`ZvhD5a%V2 z=ws>y>+mtnOrFrEv>s^eQ`(9=qt9te(B|i~3HgWq3$iZxm)0dOXezK~!@!;mBdJ*$ zHk=G0Y1j~wmZf7u$sm%B4FVgKflUTI$-rikj4Tt#sU#DdN;0!7ASaP5VAHa))1ZSB zzz$6yXURE`V?d4}XW%ymwDt^yldxu|fL^I|4y@7&fM$T40pnj00ISS+`uz5eR)nqNmwImzM267F=Ye*ogPwRn>WoMf~bFwpy_iLlJJ}0zFW8BOAa{Y>1+%{s z=KKK215oOBnDhN0_me&F+Yd9o2f{8`cZbMPkVj$F%_hglagfJh6;12f`hi0@h|LkgY(c3$dTr zJW>ck68aO^=buS2I*DGO#UN~__emhw`uh+b!JgnxfcFdIS2lfTjVNhNgmF8rZo~K}ZIxEG111GBr&KztqrvQV0qi3v2ofnM~gj zMqiLIfM1X|kYUjNF!~(6;t_cQ@(FnezbDX}hY;?<7d;?cq#!E@@*2d~$j_`0 zO9H(t#FEm&tT0Fu#3ovVy@fWe!gyVUJ~Mh1T41nG6M7MP^#tDW1lqm?@)5{K@P;%r zJ&eIiGN0ZCd7I{?IoZ$fhMW*m!+zQfEOB0-3^dGX4rURJ{M!|%wPKVN9S_<~!9Iy^D(;N`C(1q~6G_(L%-y+nT7Nl9g7m&zXM?b+^tTd1o zCB;GCBnqSwh0!2dnwF!Xu#1Q3sGN?%NJUziR;8s$WfDdtstUV$7^zQd&_=WgtwZb5 z`Z{W&qeU2%0&MmoB2haXZ6&|bNZOTlpk3%zB2iZz^?`5eOJiwI+MD*#QLK(A z_!6UNJRJgGN_7;kqsgFaQ|Syk0mf^xj%Mg+A)QSZ(WP`QO$#F+(NZ0yrz`0Ox{0oa zu}rU{O*$$Dt4qKt6VyyCG?8i`H5x$ushws4nS~Z1M`SB(yj0-N(+N5Ot;Yl zS_tMo0m2rrg|%p7kd5hK_%)^tK{ljE;Mb7W1X+_F0^7QZZUniJ-h|&qx*p_udJBH* zVg28Na0B$W7wr$SKRpe<{{Mv*OWm@%GYovNC|dQ%2%*+zokCp z4ahgN1lt6@k`<(tttT76js~y|B!K0DF*P#_s|nuNA=&^|svm1W{a6k7qT^}wI3v7&9Vx=Hvn8-^2Z9WQe$RvbJR6$_IZH?ex*Ad$=uconlaqenT_lH) zmkk78BP+ zmX?hM&t@!m7alAXGlSPN0=$h>EH%Wb;n_a$zEZM%5Ik8Ak^)Xq(h})gZON2UJ)s@B(X->aZVE zRvo;Or{0w8p~w5#(p*>H`RdC zWpYz8;$^DF@+i5LYAm(I_SA|Bp|o;<4Nx|N+^l4R zoD1wa^y@m*DQLRN{sQ?ID`0xep0Rf9F@)C8qhstW$g}Ld@*d<^h{v+0N?Y~@zN{?- z&hD^KW&>$s4_HN3oBaav7gm#1VvSgRMom7%7mmTcq`0XrYsmbSx)7Q$tI~?KWYrI4<*)SFhVK~U)Y$*JOvpA4(Y!vIyMzeU3@oWSe$;Lpvkq`z!%{VAEmQ7?+Ku%#3 z;5UWM1UZvUXXDrmHXGz@HjPbXv)DY4^VnQA9>P43^Vl5t&0~{6PG*xJ%!5+%*~pK^q%0ACs?XyW#_(h%!@&SJHq?qoh^#vd&6psMi^^ zDhhNZhLvLF*i(>C*&|klm1J*0zGeTgrOJGzt+EtCgwlz5DhELxRC=(X$_-_YG8957 zC4-Vx*~KzI*v1r-$utb4)TAV)q^1EN2Y^l`gB9ElWIxciv+O+R%UK9x*`Mqj%>ADb zCILDFI&lud9G1eA$`nIVKF9eWkoshJqXl`us+DtHgqg1^rHIN@p5I(n1(a z(wj1vhJYMGhLen@Or}8~2a%y9vnh*d6Uz)?4a;i!$+QmSI?$kOrtDzPvO!n}mgp8} z;tdE3*==?cwDUHEm4L2+*4~7$6s%=5=vQe)>RB|5{6>%)A*T=6$2B0=Kq{O?fcAz% zm=CBUXmbRF#q3wGaSK5%gj5bwPSaYF1Hx*O(-dM_19Az49@-Q7@xghLe z9;Oneu8IeQL#&f>TJcmm$pBCHu=G$ey;987A9N{JiB`@lcR=1z)`NEK1>J~Mx`Msg z2YS|35umT_lxAS38iH)7XsiPBP>v}T*fFIZSo0C=1dQ3NXnVeU_|Q!vvfA^ZvGIL!Ph2wDASb}VC~zo7ZBRR-tdxj0NDZdhjOee`xoTDuvQAe zzR?<=5Tztj6lF2U z#Yzrkt#Vf>46?97pl?4Z*+FK9p8f!ZSdR;K>sR&*AGHvU?!!!M8kw9riBByO1hmDr|!PT^_QXe+IfTDcGMKD2cQ+Ik1_9rW%VwEPC- z8%Sk`5y+(^Q3@&#Kt52GD7j6g*xxKSyrqb#I_#es?57%Qt^CIFD(66+Q~D|lzCI_& zoXR@I&%{iT%nw2}^mHBuTV zEg>bbM87JnVU-FNp|nvtLP}zZdMI6#eXxqVDZeSbASJOx1K}I{!>^wbqYQ?W#1f4K zUm^|s;*^of1V~9N(Jaul8SqP~OjYJWN@9tYD+`om@XMhrQdU7qVu`lGxaWf3CS`-N z4N?+IWL5%|-5__vY70_ID7!%JQhrxLz@N(vGPkl*Ii;LYdV=f;E9H!GR_PA1JEYEn z1?vQ|6QnLGmy`}5JHWcWq+C|ogKQ6}%di)>0@(^u50ppB8jx#1XZ}{6C@Vp(R8}jG z!6qyOxlmaFyVw}zHpttsr;Sm@DK|mhgw%LtxRM`ae&v?3UC|U@kiM{!YD%Kw1JVal zLb1c#3kb!)7M@UAfNTNw@Pu+wX$GpcQIJQKLxAciM?fBd zRBhPf4}v_X99Hs}Y^Jv$-zue9c~+XeRmwwnqm+fzYml#%S4tT83ok*wRQ?}(?*SG? z(#8Fvps0%hFzdPkW=seefT=Tz5px!E78FGZf|6AP6A0$4ASmV>fT=UaHRqgj&RJRY zs~TzEGrZfoeBt??``mk-hvsK-x^hodch~vX7t^I;#`jqqKbw9sohxR6AI0&b7^@S+ ztWX|5n7%jFh&i=9z8BY@o0x0=G<6dpn*3TZR{s$3{1E>)F=J+bwW~O~ ziqUwR$xah_{N1dgI93$jh1*Q(m|5l9C+31KX8ZEHi19o_%njwSf?0WyD|^GSyxC@P zCtO9$G76cLGm9{pEbd9mnN1cyWzC{YCYh8G$1-N6&B9G0O-hMlDRI9w#l+gIx;R!B z`IfV0YqNFY$I2{K%%3gA(bBAh*)TD;C?<}@%qp8Di}`hNaV#$G?JkSb6&1&#W^#@` zNBmwTek{xuiT}x?g;^1^&!&lDe)ie4uxOt|F|+?o9Dft{e-Zf$nc0e?t*E1J;(cKu zvu^T^&8&+_SCfL`SkSD1Sx1x3CT8MjCdTKl`8t}&qp9d$?ZlmQNpUPGzNg!ZyHa_~ zZ{}duR@_U~6UTaHwZ(kyvnez45cirUX0L7M;;wUPPhlAbT1E+0z)5`V@N_1KHCU$ezYP_A~~vr!kN{je+cG3}jDZAbT1E+0z)v zp2k4-GzPM#F_1ltf$V7vWFKQ7`xpb+yBNsc#X$Bh2C{cCkiCn6>|G3G?_wZ(7X#U& z7+O`Fi~X;pf!H4vEK6SiFTX7YCi}Yr*~c}fEu_6t=aNs(Sc+k($%b=7#@v7c7V8X5 zt~(^x9g^z~$u))KnnH3d7%{mPkTSUzkTSUzkTSV0kX#q4dxn$i0?BppKfNC~xh{}g z7pnVylk4LD6MKD=>*D|Fp5ElTKyqCmxh{}g7f7xPqkW9Yb%Er%KyqE+wzXC8idl1T zZ}WS)TkGF5HZ7{gR?QEE$VJUFZbtgD*AIii!sZh1GR;el9VS~H^JDTzgQkEU43PC*R}TwNRX?L50DFT6t3)r^=#7U~fQdEvog-@LNsvzdEZAmoMB>Wzle z`cdP&mq4W~@53m0-NxT-O&|GY4{78>V; zNngUeFzHK}7gnn`8cyp+jq{Q>YLl0|Q5$kyG$GeT6LMX+kn6&QwuQ=d;X=9gXNZ2Moor5|hV@c$kE8yVj}`NikK-TzPcxl26<>E|wg|F8JDpUKuo$nEDInyu&XzvI`g=p-2X zvkwGz_PSwxe=%SATj+I&b>#o;&-}}8S?Z&(7Qg(SzdZb9|9`_D>;LZ$#<~apPwfw_ zP0UP8#TrUeu{PCQtidiJjwQrOSa>c?0i8^oMQB42CPSPA zxr?{`mBq2LSijj)oG1TP5$kWOiIv|~#iykqq?#ZHvAV@le3*$=e42`{KM?Qd8;Fmi z_?v_Hiwu`5K(*q>!LXvdxL9Y?Sgbs3V$w*g6_=~U{}7+zhLDPaT8LCDi;t~XJ#Ht~ zz_t)ZuiM{yLG=$U;R7g-8acm=EF%e~KEsm{4Oa%nD6vvigb>noga@JHFO-0Rh5bMIb zh+`MADkNIG!?X}b3vqr2vFbKVd?Lhiws7$t)mMDRiB;lb#cJ^m;uB^F_Z9yS7e5`u zT9)Y|wkhJLfLI0BNvsb4wPt+0Sb5%0eEN#ewjyl2`1BJLEmqkM7RSM26^Vy9FIId; zh&3(VVx`J(@$oQ(cnj((p6>aJPoP+H9w62%MTyU3@g#GSc!JqgeEbdJ^4Vs9_~|O1 z7Y-0>S_g^st^LJEzV{6g?}R&x&mcoch@inD^%(IPBG$nT>w_7t@@U2sovj25{cB|f9Y>S!OahH9GlL>oeU1Qilz z&k)BMBFsa?FkKv{i?S9FZ4fJtv4SH^9rcCRb_9!q!Jx*AjyUSAQkz3b{diF07lBUt z@1I@3=iOLPVd7;m-gTI-1N>lHn~}ee@jx!oV0SY zlFwMK$;oqWR?2(pO{1K$=Vqnep1g3*sl(hnoHfv!tm_T(Ts5{%aL||TH56nX)Od6W z2mPR!Q6S5r#@~+D*B9;W3$jlBjb%M6Hn~}eU$&PTEBTb|uEt7vW&crQ*IEtr9Xho# z^f5KI8Ro2S)uoG}uc~p}<%W9KB#ogTsO}S*HCF0G_B}N&w9-kh>DJHC57l_*J4b!D{38tgP>qZG zIqFw9j4|{>HC|fBQQyYQ&(IJ5jb-0cY;v;_ziclxmig==@+|wI8Y|_M{ZNfRzjxL< z^zwkn@?&!9&}+6Yw7=w}@6)cgA$)r;Cw`oVGJui?8J;k%q29ZjyTSh0sfPM# ze=v~o3^Qq@@7miHWc&)tbQK$!ALU$`&-U&MvGndu0R2NCsdN~2{(UbRuZN?h_Q3UmGvRy4 z9k{w~Gqm0?5vKh16qW_7H`wd1mdE}Bz8bjI(#25z&KCpQCT?Zz5AB4kvNM75hT4wo z42pIC&>w^xjS_++t7~F2A`N{!H?O$%|Iw5X&_% zX!Qgm-c)Waz70%)c9A}C@njT!UbGyxUW|t`OQzwh4=ccOr7=(4I9=?Ik_^p@#slT_ z3r8X1)F#+gKMpQ$&_l_zWLSJA9#UGJfZDy%V48_B?|pd$&^-kXEsO`s^-4d5A#+9l z9NGh}ynYPpZ8tzd2X8of^Z{&NwGlecA8*7;AC~X@N=7*2^O7&1dc$6jJgO2-UGN#M zk7O{ib`7kO{thNJ>j=ZPSfRQ76X<(k0<>w9%$tn=2rpgzU|VAoe0};AOf(%2!ROEL z1smVP0at(6>AVKEeR>8fwVt3{Bl6whk0&c(`17~$t#~IG(QYYx&i@%U{01<%nh#ux z{s=)SeqeEJ9t=o*4D)P5pi18;*t70Ev^wbzJHF3^F?p(YP z_n5hv&FHoP3b@1p<;gG3v-DRKMht0g63IPS!H1C98h+O ziE)cxzY><7kB3t3Wm)a9so-gD%ma30Y9_W#68e(yaH>R%#xmq1r1bTJ%@sPc8aDUg zD+_~Tdy28%1>eK6Oa4ZzpWcj}=FJQ`(AV@3ng2fkk4~5_R19|sm#N<7k>bq*S_tx|sHf8eOPW9b3d7r2H&YQmf zrcB?3Q>O35Dbsi4YE0e{8oftX3n%XpRo}I%y?3YY;3<>$j;inG)!x%9-^01z*{i+3 zS9_PQd~fG|udnuwU+sOr+Pi0@T(AsI3)HTQ{J#hCppSf!f*vwRHw+YYyo8 z1Ipx`tI@gy8cx?Js4-o;pvH7f17*6tfihj|K$)(4piI|5P^Rl4DATnOYE0e@8?B$9 z;bV6%#K}E2!Q?;v;pd1I_{}d3oSz0lMA2yMJZKHrt&f90R|Mg-&@C|P*i1P3{71qC*kry1ril0{AK6fn)hw|Cx?YZkyzuNKOE%)ndg!T-fxK`RJ!g1& zJ>0z*19{=YU-fM0)9ovux6!ZU&L zj)!_yDP|{Zv7ZT)-_M_mtwq}nJ>n0P?_WtlZ&CkpKKZ3yGCEIN`Cpt>kwb&oJE8v4 znSeGNju+Yi^M_3};x7%BfoYLnc{>}t5MuVeHN->N)GG=1)>#YXxIa*ixV#$MnyrCJ z_r2lBqZHf~lnQ4pJYimB6n+_!1Z6zp!NoKN5^HoiSB!_P$%o|OdYr(_v8SMSh8M?N&9x)2)Uw;dZ7Xu*0B@#YWeFHD<1w!(t=`bYlISkJXhD~+mfYXl$aBOiXlz9J* z6&jubo804#c}GT-JS{Hf&xAL+9%R0~(}1st1Io6WIeY(g13al52b2p;vSg{> zHbDR1;(&6iLiyR8qG_;cVH{ANQf#f}%B>Xm(=r}5M-F!L4pExyu1g&o?)O5I?etZat}tG2!rKw zud&S?opd9{a5ro%z1Gvl;DF5(Z2=Dk?m?_$ka*s>Pz~|O;@Gr zo?H}buDB15R=z;_98_lut~`O0;Xa_oWoI>Dohsjj7|U@$IqCX$%~`8g@MVZMsPQnb zaE+_^8|c``3n(8k$zo+EtcFRiVj(Yl@#7WtdUgs}XU0Na_>#*(wxa7=c+(*k)Yz|L z5=OeN0(0>Ugz`;IGEPca2~H(Ef%1+iE6{#o613PP?x$4vfa6l!{3IDXUIYSVjIhPJ zF)!eULmQwxs!?6MxbqRj-0uXG2eR_mFZm5z63@*kTRWG*!PVZuN%0*;xoP-&_B!h& zc&_jU$~9LP0keW{V3)5KP+pOGjWwS51Wa%G0Oc)HPqPv6ci>U%IH3IFr})lU{SsX+iq>Y4nU0%Vu<)Uv8z8*UY`ilJcCU^m5&$Po(dd+%Z&0~Rb zv#=vD`1V=|Toem=;pJZr!;iUZAn|Z4sPS(edI*@D3Qb4?R zx8dy{X;6cY1vM^I;RaauT?Z#h_ygt87mM&j+7|fW83L4NmS2V6gVsZ<>0v-QKa0hJ zAJf3%*leJDNvu1syL&sFUKaap*663vJX&xAFvLs>aT&Am7YM^>2wt8J}-w3{S$%m<)zc`MW>Z; zuSO!MvETOTc%}VnXf;0p)cE?vqfo#3W*EM6Hc;;9aRO3zrNM905+E;J?ZpvrPEG+k zn?z9KnDu%%RA@E4>6HkS2iQG>$4@sxnSMQha;IWXq1@O_@THA6P#*Z~A$)(i8ES4E z4{F>ks1gpVmj&VCnHA*=g=`T=dgpMuHS`1gUM(0XuWO&oJJkIIcd7;fw`g!-9wtO+vYFfN=7KCagb$aw)fh7C|EKID9`cMv(#Jnq3$11 zK>7H#lkB9$OL%Z56evG4eZdMHeh=TS1q0;?%L>C{H!+|02?EOXi+yEF7B7d#rm>*L zHC~@%UoNeHpATX{jXQa*!^x$8-Ct8?R?i^c$&H44H_E=^d!U#29+3Bmlsk*-;<>mk zWH{v;PxRtmY!6JH6bD9J-;w!R?*cwggy(@9MQ>%*M(qLuCIPZVk++p8%AL&JV&?rME(`xbL9+x#3-iYPb!m+4=+J z?S-yEc;+rB`*AE#?iH4dff4J#LtLlwHkY@ci$|@P1zcsBxV;Q*lkc)zCF10VvN~eHfSI40)3l!rd7hEuaXLA|doKzT`(didb> zJ6PPf4Nx9)%>{MEUP8f-oq+O>#G*LX`Vr`>MFQo0C%^E%;vK>2W1&Dk_2O@!T`=~57|7hxG0;nB%NWBj&ZK8m3Ilt5F`S15(u9&A#o;>p!8(sK0R4g10l+%a4XU%86fzQV#0_B+t zi$Iv@KV6qh0LpjXXR*?6QXu5YRG@5f`zjmpE*07?mt;hZtI3+vW8>8CGv$Ts^K^AJ&% zle&{k+SpxcU|^Zve7DaOIbNr=3kAKbQK$!2iZR>)U?scdXV9LM1GVy ziT_zytJK3`ksoO<>q@dx=Sf+%TBUB~xk{bLIF-7R@hJ5m+gG_RC@cM6=7F;8OVVEU z$-J=aSJGbgb2V13DVY{!S#L_-D9id+;#6bV7D_(V_`;+LhQ1{0obn|Rp4svN3-uDu zPx8pU^(Waq@ytrv{~ccx;mY@e8p}K=Qtq+h;nx52)61t)#{{e5hOuxS)mPv2+P26oqZ zk}78nsi%(>=T;Yakenvcef@Hd5z9DFh;%m%NB}jSH?p3-(AuA@)V}$!U*xB}D3{5r zIY#_`*YDb;BCUENEr+|mYln-xy%l*=<1-nyTBTg(V{ElbxoYdGX{Th@PqFb;wK8wA zy!)%w&|c22!$!iUOm^L-i+Ci<{K$Bevc&6t*IpI&B}JZP-lmE;pNX>f65rvEPi|o5 z`zNXLAYt=t!EWZCH3M>UNWf*b@ZB6kxWaveecz;??DDDkK)Kn4dU~0b{Jx_6d90KE zsfb^OPxEQ4j}UR@=5^Pa=w+NT{JiLIy9Hkp@l$rIUP-IOPx^RFGwAC_ky?cwYP;SV7*WMN3vMiKGiFl;_K~X1^Wt{g!c-HGEpzJ!To?eE_ zcqmsM=cLz)aG7pi*iY0?v+Q&!%lf%5!oP`hDJ$3Gr5p1N-xcyY&CSZZLWZkx+TA9G zYh2bpgpF)j5Lgcl$Gy)nYX`VsnMW>=jOH#9rcxZH8jLoxuSzUt#uy*A208qA1heeQ(pPN zQJ+uP%W%r6Q=If!!wmj@k)at5dUyLthIlB;`jl-~V1Nz$8!PoM?Il-h-$>sr&;@M8 zxs=a}@Npts+H{`mq~9;{&`;P<-h0zgzxV41Smj+7)YyGM1HJqH5ZEeg)Yy7Z1N}l_ zQ=@NjjT+y+<*2_a(vmi6{JMJ+{p7Ocp^ZpOjh|j=q~Cea0X&6`8c)C4NPk7x$h6e> zkw|NzNK4wNarByo`Zk5#VYEm~jf>ZG)}K4u17-;uHEvbQSzr8I4?|jNTykwgeJ_!g z%#Rw+t>d8Izjr8n=vG`q`N-ya`U#VQ4dIlRENQ4u?p+49^)9ZV41YN5^O@H(gi}8H zNvst=3UIM!aSi3af*a^xCv-K0Q~t8Cp1z7mmx;1d9#h*vzgXn6t_Y{Rypps2R>6AE zOoUTjE%LcR>~16Q{=OT2&Y`8b6vf^_#UvhQyM5&wyvk& zU382ZjZKXO|nM767=%EKT0p`B=*2+K;f0?K!jZM5HKj5UN)zHMupV^6vFJ-Zxx z%Hw<4Y9EMu4jKP}4R+d^w}T8k;*ZMOunlp*2ZsS=B@c(@3V)gGyh(nPud?>90pi|t zLZ~Wtc=3n!*zyF}pEUv0SSgE)lky!e8|^30AVYpAe_dju9S}Iq5GUnA4`t6ab~*K?#!7#Xb*{#~`|PyGehY-S4#7bA>Os4ldRAk&k z)6X`iAFA;&cU$dTv8G$*?ax}aIekox2WV|``l=c$X&KrmI}f>8$+K)PHCFPr;YKN- ztfVX3RE?GV%QjVGrM+Yx)L3bEnSV7_+EnIKX(O2~Wu<*(p4C|S4uNkzFk`TtcKWLb z`2KsK5f2c3>0{AAsMszTD4$KT(=K@*43#X#!I~I5?X^cCaIRngY)Z1#It`v~h)4Op zlJQf%Ci*~x@F$RQCWhH+Er!oDoa;5%R=eT$EJK`%eO7&&oH#W_%Nel}Po>+XfU*)# z#Xo+}iQlVA`J8gec$D%=R_Z~fMOi7Yj9)3Ej7KTEjQ{g98*PEx!wqvb$~Jq%Gm}7r zCQ&5iNz?4K7tV|@*i+unqpEhuh+lJ8%6Q05JEQL)gFR*6@>R9#uN!ocA}OzaWvjKi zJ z=E6U*OiQu(J67VCHfpToQ`)HU>F9l+l1_JZfzBt`8{Z zs<9G(Kb{Ex#HM>o!^eoZ1{*c5)4O7hjT*~5D{1{5D`k;3YP?vKU5Q_f?+F_vpK7es ziA+n4(@NNCi(DKGLrWiKl$GzO_!l}xIo~H6ZJjN{pxw3uOpTTAF?lX!-$XlYd_)sN zT9nIdw9}4R-Ov!H8jpK!r=8uUAKWgomr+)}>x$gn!qm9TdOPjLMnhrA8O|slJ!7Zc zUS}X2O5MfOSozMB@l)1)sG?1Ms5RtAjmtP!)lM^U`IRo=Va2OzCl_Icbk$h-&X;wl z#!j2%jCHpjjL!2&(av|DJ%L3LW2U;SkYk+zT2qAik^e;0Y^1fbRmQTbIP8Zm3kAt|2cJ-n-%Q^d9E5O z8V)iKYOH8M$g-%hqA4Nk5OYOH8M$a1N%qA4NkDZQ zkr0{{sOU?`cof|U8Nb3ZUByOdP~@B|G$@d0P$1EuK%zl`M1ul}?gSFu2_(7`NOUBO znCM6#Wuha2l!>Nl8zsnx=X<^vMV2O}n$4@fj0kZ3*_G0}NI%0%-4iRJ?m%?BeUIuA&hXg(m(d_bc4 zV8lf80g2`V63quBnh!`cACPE1Akln4qWOSC^8tzG0}{;#B$^Mx2PjAL0g2`V63quB znh!`cAB>pjJRoJF^MI6z&I3{=IuA&hXg(m(d@y37`G7?80g2`V63quACOQvDnP@&B z(R@In`G7?80g2{=5fjY^RULMs`G7>*L6wQ-15zeB4@Mg9G@NKV7-_rHaH8{ol!?v* zQYJbNsK!Lw0rSE{^FdWdo@hQG(RNT}qWOT7iOz$n20hVyK%(fiN*pFjRhnc3r0+I6;O?dt^!gfx(Y~{=qey(qN{+E ziLL@tCb|kpndmAYWumKql!>kaQYN|zsK!KB0Vxw*1yp0AtALb=t^($TiLL_Xg^8{L zsxi@3K*~f{0Vxw*1yp0AtALb=t^!gfx(Y~{=qey(qN{+EiLL@tCb|kpndmAYWumKq zl!>kaQYN|zNIB_0x(Y~{=qey(qN{+EiLL_Xg^8{Lsxi@3K*~f{0o9o3Dj;Q|tALb= zt^!gfx(Y~{=qjKZ6I}(QOmr2HGSO8)%0yQIDHB}kaQYN|zNSWv=AZ4PffRu@@0#YWr3P_pgDj;Q|tALb= zt^!gfx(Y~{=qey(qN{*;VWO*md10cffND&16)-PMbQLf!Omr1cjft)TQYN|zNSWv= zAZ4PffND&16;O?dt^!gfx(Y~{=qey(qN{+EiLL@tCb|l!#za>E)tKlWpb`H^_W+3o zf+`c;1I!B(-2PI& zmmi5QKN4SlB)&M1H`|vVi7&q)9wpDjf8Wp+*}mkc_-4xXqD=hvF)vK~ z_fd_B|2|SC{`-ddQSwIo_fd_B|30d*^lw$_oHFs>$Gmd3|GvTg@0j@SqZ%vz!tz>H zW8%w?#Frn5FTW8J|9zxPeEE_1@+0x(S7qYMkHnYXh>8EcA^vP%cq1mh{78KHjhOiI zBk|u?W#Y?^#Frn5FFz7rek8v9NPPK``0^X#QSwiG`H}eYBk|=|W#Ye&xjEZ^-w>`a z@!v5Wc!O z^^lvD{LA<$E9I4WpseIy#-qkc*=2qxD}Kka-qcv>53EO8=DUsXrur5q35J4!<9VC z_EKXdZ^FMDDJ$v9HdSLK|FTWhSZObr2Q^mOUFKPhl{S_6RN6?UOIc}OnP)XtzC(nc zIL`QwAG#4MzSzQN94RZl*uoDT6(4QkhmOP#9hL7Z89!ykXI#cf{LoSHAD3||_QE$j zCr;rrZp2DF!cQD2EAa?F@tpXD|9DQhWIRfFB`ftH)1s`DSH`cDQO2W`UB*v*y8n^0 z{f-Sbf5(b{w7i#8W5v%}+Nd${>CQ<@_!s{dXZtDV&XT@F#SK^o9YOLf_+NiPgq0aWp z#(&~$zifle-?8HREX%IOiqEpNQDd108Ncvx&PnU1dlI+1CqG4bibO0e7dA(at1_;edF@$cruznc@EZX=%bC>^SrJ3{w%iST0d zA^6<4p@H8G-wI7VYZ@4CropuqQ;leUjm%?*k^!mEkqIq#KwC`-ffU z8d%0tueXnZCyTUJCbclIOsiMh#s=>G@pMl3qkyYI&*7K-!XI~X;+cL?XxC@Q9}xLC zCtVqyo}C|QujEt4qvT)4uat4>GgE!F?DDQ|{~f<%m$$>ZLVEY?@>ch?&|l9kZ_>45 zdL{l+TZ-#XXO~xo*UHXM4PpNzySy?Ur|kU8v{JLnE7R(qU0&D2?@$R}*YYVkX4`+t zyo*XaW{+-Q&FuIqxL!vkT^XK{U0!K_BRiim9wq-xMEpt_W&29*)-Dlcoy+n{mhI9^ zlvlFSzLJ&pm8`U{5-!_UvTWZ+qP&u2JoQEXCCm0r66KXF)9NkCD_PdR5-!_C_6Hd* z+e?W@XjZ<3Yvl`Px$jA&ma z+&;U!O8d%i8BfFP{L8e~W|vo{H88upvVE0s*)Fm_$a9tUm8`U{WZAw-x=Q;h_DVjL z_LbpE`$~SX6*O55?b*VLq3|$vJREpEjSHe+0bxN7AEQqM7OdhLbimiid`izG0{*rM+ml3={c~8fdxeP^};4mHSef9P%9j^ z9qoz{&PCzpRr0#I@ zxo(u(uX(HSd2~c`>vJe*URMS8H_foAdoS0<(~7w*0tc?C9Ez#?x?riZ#bDL>OS;ve z?=pIg9m_vn48||^4Y65`da!Yf9ruj7$O>-qGF-n2m%Z@r@e780j$Zjj*M9UV&9|qa zFyxgzT5r%`O62bJ1`8tGEG`CfpSHo6QC9=+^4Ea;*DCN2k4J(_^l`(vXWdWojHx3H zamuoaG8=fgn=>>z`K#=e=hViorGJ%O)B(!6ko|xEg=E*`T{XP*ybz?kw`34BTs4_b z3u>jI{QcP|msxdO`RUCeNV#^KP#k>To8vSF8y_%Dcu!;JxH1&6w0BnUuYfM_}yyOZ@2YMVeYOmhfxtHF=8T zSSX&=0vCQq-tR_!_?GF35AvVpa~w9X)pb_sQWu6|;boR=|JeF`(vk@{A*eVT(lZD< zKbypZ8y42=n>P|ql|9DdMo-hx@SFX+!-8)`;6}zd-Gw_>G=-LhVZfx#noeC?LgkW` z(W#?e6IE;^tlfHoeVrPud$+7P(@qPj8f@fI63r40-jRuf|zHG!;AZTPhkDY~Wi0$_<(JWn0%)BwI-;HrFqnAB)T;mG!n|#p>dU=I!?*5FOuFx7RPKZ5Z zE7XL&7w)r}tv6}T`3Jz}v;-bEumPktae&c|TK?tje70uZNO0IWoj39{hx~gw!Lftm zc*D~3+(zGvz9)8-D9-9=&$sP7YsC3#6~ecNd*SS}w_GPI z2*Wy>BJ5hk?#x!F%yD>DFYNl*)9tT#AM|~u=Pf2q)|6-v1jP9PnGUADc(lzEy zLovkPnyqm8$zC+;gX=36W{J0k!1{R)`Tm`$nu_l>utH77p!?<$?89AiHX_C!_Jzjr z-OD0i%f?=sou(r)7wn(PDij|Lu`{Q0{TEjl{x8O!?} z)=-W=?G1Yhonn_wB6Y0S)Qoxl*4(dK7*bxf@TTTma+JV?j_U)wajJb=*O<|(nC02_e8QN~u)0V~m{H>-*YwO{lj1J(4VGRQ zzwl?~xq7|fUe|Mc?~xQ{Yju+^cr+Z_W?je_@9xb;RU63W-S>mOo=#9@lQT}OaXiD~ zz9S_0J3<|C&8|J{1orN~zLUN4-C@?FZn6Ed+CcFTC+HOCjO})xO$Q5Gs6EXFGFClg z8#gjqc2A-O#sb9XxlY zf5wBQk!};N&g3R##Jkr94PZxA2PpL2o#)$g*5yg7MBcbtAU5~dq}k{j0NoEI@ccdY zxmM^gn;%^oh%f4!!^h2?;K^ctzSx4feqEcu*NzIr6Bp*QVhJN5--qcu^ujHd!RO<- z#nwRdT(?>`>vjO#ni9|39{iY*@hO&XN)sArtR}uI-2{4cYR@f_3cEgckLT|WiFB>w zc!q_KsG8~gW}VqC)+1-}l|{y*x6Mx;T(vu-EDqua&A+-dwJ*u*cL>9e)Awo`i0_oa zqpg_lfMKpP@0R7IZNhNto%L*t_ZT!iUV?SWEbZoArW{{eF$|lS{$P5cBeir*5%$f0 z2m~~L%>TTXqIp)Yglln=QoL8oFf1@H!mV+uFg#wOB=hwShoDw>b+tUFy0v>eGozko z8NRGu7ZHL|(3Wg^}f)Q>pEb(NrF|3+uhrJI>x%@wY)~L3HSU40s}zKulWciKaXnW4F`Ji* z4#djhJ8l1CPfd|?{WE=bpLGem9nU>B2jZc!g;?cie`ppF$D1!{?iw(EHg}pEh#^_7 zaPDa}@bYcS4~B=pwMLEErseH*yDrVc_DUt%;2iJZ=YZDoYb#^+#=wzSg4KFI@PLxyM2cdt{iR@Yc(@}P{ zn(Wdt+?5yI5Q3CnmI#A&_sg=rTXtnqc8p)?ikqXjS!xhcw(*RBWb2u_sNfVg%DLu# zDCT~KIi8#kioY2+Ejj`h7aFT+{A6Y(mxkq8(dChDm8(wRZCeK8%M)(Md}?5cniZLt zhk~4s$~F>nR1kAi1JgNb$Mv(=ALA;pInVqs-MtOGSi7H%8~2rWt=I?lSf63HGn4u2 z9U~xZ)g@M9S%U83y)$k{+l9eLQ!DP+AXd}vVkjhdTk`zHLZNGNectBfbGP;z{Gs^K z#oQ^Plg9K#8*I6?JVu&4*7%*Sj#HfhUiE&UL%Uws$HyGcbl9xf^Q8(u`DOylJ}AZ; zVl!#+c3sw$3Gk^^JHEe^H&!ZlmbI=Ft^1JF7kTP=<~nLVpX0ckTPACm>6tNj!?zv& z3_Qu=H$LPYe>mc?C`UYK^-lL@xGk1HV1wuWy3ddFy3Mr>Zt`(f9_ZX{1fLPnk6YI8 z#{x5Jf@$d{_@Pr9R+Rr_11ojMX#YTF|8f?)S-}Ul-<`%rSgh9AJ`BK8Bjefr!`^I@ zjSt^n)eoziUSR_}KjkKCT4U4V6`-}4Bi_{Bb7v;y>m`G6rE@UrZozbv?_B)CD;4htZN#&gkO#xideKdmaD0_6x!PLp zWa$eR@>z20fStOCPN8rkz?%PDV#43({n0f!j-A~W!J}V|#$vzEVCf%SaoV$Lc$YUd z;{3Lu*!yHH9y*?BoOeZtHTrJ4o{O8i?Yl4$KF)LJBdq;2{7w%j_z$o^;b(Y3zv_JlgN?>%Z<=FJJ+s6)JaO8|7Z?OEJOH8Q}{H1>w+RTa3WStF# z^kJQilsbU%x_O*m4~H%HL-bzdz_B% z+F|xkdY>zPF8WqS=LtijF0)yip7K98wMIJb_w3`vzWDp_MGgIs&L8Ob+v(3)>_8bG zTwi56qvQCNmC2eLcLMNnTs))W^0u&_Y};?$@oi)f8|z&Y+}}6G^<~>LI@Z(iCGndp znmw(Cn`4`@O?IK!Vn;1*o~qH%G3&?92z+zaO;^g>&5e#bQ#M6#%eSL(S&Hoc0kAN@Yd-cVHxw=>B)7WnRGHh|OKiWrLWd$-vVCJ7LnWi^dfv)O7 z*7@*X;=XO0=E7(@&0lB3F-)5e7JTiC#dn=;Lg$wI3f$&1LT~Z{r97~kn4?w@bJT9W zr!taG59iai_2WhA_#>TjUiGv^moqkaReTrGx$D8tj@T#G5s!%PD>@IEb?80oxwI#` zhea_upP6c%rLpw!$EdgjM&~CP4|>4xY6Y;_5F1A4Il4;|!6VF_FLWQLq4Uh(;pX^s zb|(z=@@I6u%7!(-W}oWg&BN}D&Oh%4&*u-9jl>ToGZ>x6?ku`dH_kr**B?k=J%_GK z-*~MF55W+;vaL0C?pqPPy4=^$`v}TEQ>vq*7QklxQ{A9uJ#qNULg;oVP19T4zw~u1 z#YZ;}!xsTnc+*c4;MoE3okQ&DJ(0`h9lF^bq42I{NlxdYg@=9SF5dmX!u>a%+-DdD zp1jQ}omi#oIB=Nj^FPb-qPAhU!hQ{J;pPiH%Uc@FdxM*ZIQxadwlP+m-q%c9=8exs zpJEwet`JuHCa-sAI9gA;o-qjf;LZEzSo?&feB`FNJp2}8DJ{q1sUerSZ3Qp0={Z?wmG9&yDt>nt5m~M z2bzI-^IjR$dzh&M?b6^`QghG!A-LOqb3pjG@ zrf&R!kFIUUj%80S21C)bMncoG4&=M`2j4ieAgg^QL1$h$3<5nrGY^}AFl%;Yx1&d@ zhhDVfTpTe3q^@uRYfn>O8;98lj8 zIw>C)UI(jA)@if2w+vRHA?wsdd_R%cuooXx1ue=MDV;cttvi|v9iY0zXo z`S1_7hyd%%D(1!5gUMmync_&Cf8YpnN{rW0o26%Oxx8)InC(6n0%pJUfPvQw!^6gx zbnUDsyY@b4&8Bq@1Ijl?wS*&9m2qE#i)(w zpBbWmGSNT5Zr}~gRiAJyzWS0|;UEPGqC^z@;dJ(I&Y%3eFsTHqqeI)FubeFBUI}F&F1hXf6otqp# zaEJX-^cI^ru8o)%IN{+KXE=QBmW$t#k?i8N0nGKhAA}5d!lJ93VTkW3S23oziu*?< z?jK=yVoj{pxd}MLwBYUAEK9cubYlG49vyKblwY7{k6C9|!ke zxPcYchO2M4xCYF;%B@R3W6ZQQ=n89K_sZIE+}hN&)!p*=rFcy^n|YsEm(akcMYUnI zwW&+ZzV=vlwJY>+{-UY3eUqls_yG7YJ3-!Sr;B@SChoPNK(z(TN<2dwDekA$8uhoI{M75zv<$PhJ}i;6VYK{zN!FUVH=DmBDyoY;p`S4Q-Jk8VH{pz zT~Q47>xzr3-O$lzaelbM?Yl!ER(E3q#k1c+6Iy8~hsLkWsAHpHTNi`?Wlu*FNcq|uUu`gFUD^$W zzL)OvW>eQ_e9u^BHXT}yy(}m4_F^Lou^5Bb=apdl&i2okeYY&DXcGpXO*1vg4MOqt z{F2P)#q)G``;yE_%+GdanX|c5{h@_;1}Nqd>Ea%liF;&7+c%X37aa|o7f$C--n+u{ zC)Hrg+ooL1NixK}FcbH}K*L{541wlj8na&FeTJCxWQcoeChn=>)Y1qzT*_19dwFc; z-a=o~OTL@U9!w2{l)5D{T)Y#QYoKWBITNvFyVmUD=^))V`?eWv?j*1gLjpn5qzB%z zF9>hS*>W-dW{BqiOgsmG)1N-_kfDRG@hZo8bpkHyuX z^QSWW$ij%sftkVVlh_Ap*xt@AorgrRlUss}cu_zE3|$(jtJNpojdGX7l^JiRN3o0R zf`GEi_z2uNJxt@;W?3fXmRIuelpB|IpTa}YwM#X;v!oe_O}sNI`sy^zURBrZ+#HVY zkDFkErXRRjPjz9~6NYbuWmej;g+(&xA(aQVS)k>durpXoVRMHFlvbwp=UDO8-A3Dvpg|6ZSO>Xec zHAleZi*~NpB1VAYn#;^(dYq1qYjyUVbz9{jo{73xaXPLoUlgmUdo>h3Jt)Di`h-Hg zx7GQheJ|X?mK{uQQm+Iv^$&wJWddN~#|6B~^`4r6m+Uf+7c9nFigC>)a<_Pr(}Zm~ z90DPaJTSadIc)Lhndb3=2-m{5tl7#=VL*BC$Lct=4#0~7w{_NLo@o7`07htc%6U}I zI5j1-D)0V$0yrPC=G2~!Sw4rh>ZUFo57s7=xTDKCc5sw892>eWqv&)md~otSTWJ-d ztLk@~*>xWdO~U;$KF0M!lSUVqmDOy1qw8JP>f$h{wr7^x;v1>_+Y~o;YshH4+Px!o z%{tEZjl5xWZ%+Gd*+5%dsI$S65AJh1ewF99xsUlxZj;{w>G-87?8QG1?#G{1@<%#; zU9tG;8v87kbzMImI`6Fs3+pw(Dn2dPZzWCOY1QsnrGr1CW7#+BIF@MRgYk*e7#+)W zMOJH^o(ABk4)O9HEoUs-n{O0rvUe0OQO*zPSf-zMh4m=#l+T;j8tGUzzDfnS;aL+O z*SN>&Sk`E)J=ELdiqYcv03FNdm^G)_M%~hh0k||ifzdH*O~!maICUhNM@(nef8!a^ zEoZYzD*|CbhB-c2)(Kbc@nwO&QJL+KZ! zF8|_;7dFM%qbXr9c6hDK8BfN*!JE7J#T}j9Zagt#4bOxDwZGTpIBWm8H-vkx<7Fx= zcUv{A5Zk#v3~0Dzf-fArwUu>y8=$*XBMkl+U7ET7wlkBmmB+I5u1{Q9nKdCmxoYcW z88>y}R$+DsQ2w0-qtn_zR^CmcqkPuiIyw8rlP<}}v9yMw>ejR+B z)%>waN5`gX71Q{whht#q&k~%DO|LfY(s?xth1GLQ8jVf#`34=2=(7$nHlY}^P>flO zjz=`Sn|Oyk`cW}-(FS2 zHo;|C;XR#Q=KDsm2AhL`a^si?bZX$MiLM!wNjY|41YBMit2^kC>PESB!pe+B3!<1y zS`bi{W8trVk(mA4Fg}W{pR@m;^LHgY`}co^8`)p-2!h?ir|_~>W=PyJ(nVV)q;1DUQ2 zr#9K?80;x$=jB(MriH-t9cE1Ya6j?RIAhEJ-tYVb`~(iHWW!((zqx}*Yph7Clz8@% zZOdMzg`)ivE7RR?KvhQ2Ca z^*Pqx*cim>G>m@DhlG0LZx1RSzfTaRIy6jm3iC0_B-pRno|`j-%%f7%RHd9FpI`}2#m(&|q|U7V1^&X4(Za^2?}}(eS(2 z6P|tFyeeD~XHCwc%=kJ(xv_t(+s|K>bVuH>#J35`4CyjaSH>QQ@_kaiuggc7zAlsA zcTemv{Yul;rGEXs>X-T#TgTI9<_htF%TuepuKE>ASNW8l(pUL(x{^PS?C+mXS)TfG z!(7olSC{kl?p)!g1}{&oKdisnU$WePP*y~@frfSY@p_{=S=1ZJs$N~cD&xL^_?;W) zQx;twn9$~}C!Vj~0_iKF>6vq_>s2y*d0^xz=)}?##>NNfer9^+y|1q;&E9S2uS&+6 zS%Eb}!&GM?{tk2Y^_G!NSa0;craCk6{0`-PIa1}m(V=fnObAsh^is4RDJR!PH zR2r^D!|`#Bs5WH&OYyVQq3N_tHSV$eDZWm>vUKMPny&MfS+#QobRuX^bpB~OhZlY+ za_@myI&BYa=QH??yx!bC(Y!SMa9!Ano~bm)iGd@H?xPR96C9lbWNepESBCu3gd_++O;)9D;#*T+D5#u(Ca0)3}# z%jhHYGu8~PH;^|&ht!)ecY*Ybc~|AfuBE7ZD5KtB`C4awk@2BUZMyn{7uJrBpUQJ( zX`_`+=MKr=-sXqO7J7x;vyE2fUG_cB3a5th59#dNONWPQzS!3P=7-O+%{waJOOeC& z#E`^X(Qu&N=>3j*t7Xn_qiS?L{6oF*yqRl3_03<@KhkoT8{NlrU$aS%-Y;p75AYYy zJ(e8PRT$gHX?k{HrM>A$-q5Jd{rzCQukY*ce|xtb3Z{1-JS0@IcI?|D{?2(-UY)MW zSGp=su$-xD`$y_l5Lcy-4F8ed9JT;K{oCrbj2!96u1ZJQVbY%H_fIfyyf5Kkdc5zE zV0z}ebVmEH${Qbt=>4ztgXP5ga#i{9zGqdY>WeN@_2sI(@o~5+9eoL_c^o{azdxp| zhiAc8HE3AP1;!zw=JbtL1N^D>(c~pK7fi!ta}V(E`Q&19iyWfL ztyJv+{txAUO|DX>Wc5$R^!I-&nk)3tTW$>+E`GAVKNjD4&ptc9It`b@Z;cI3Iic_N zykFxn*acMFmNRsDP?e;`DCZ=~X<9LFDk*;z4U=+Kk-TZ7Oq#bkDYFI*lX9|cTP?Kh zMzrmja@CSZzfx(KwvC2K`PsH}4$c(jZXElzfQ+cPnt#ZUimNb9SGww2Tqh%MK%eT+ zax&8YopMxLl%A?9^Pm1c(C_+MW@_kX50QZ~lYA>@q&^P;XK z)D;fag));-W;kAE2xUfsW!}P$CN0H@hFk?-iO@Zb)D_*s^qxs|vI~=XBj}5+H`{W^ zm}IL%#xtRv|Ih|Ko~jLcJhLsIv>}vTnWPPBPP2R#(Ddx)G%#No<&ZJYwhgp=8YW|| z){h?ZgmOsvVLI<5Z^C?${zl07gh{^=#+KM+D;slP+1snN9k5}~G97l@nkyh7FHNU5 z__TbQPVD`a4SPmiG(Errkd=<&Qi$5XtM=1!NL|W?Jxjg|(q?7DPPq#*>LO#JY}j?* z6(54O>-TBBRG;b~ueSY9(5LBWgXW+}(4pzX#y@J`ujh`YBQ1I@I3`3#@h4=dL+Voe z37RWFjd8}9kTxs+1l{I@^+5U_V?)vt`YXu29#buB)IS?L7VUWN{-E4cPDbV!?nYcE5G^fzJc(KeIu$(Eyo>X5lp zYb>KKnoj04L5HT3wV~EgMmaP+u*R~|39iEo9a2|t9cJi|HV4;Xh7Rd_a2;mo(Dwf) zg$V~mgm6%V2nR)ka8M)@-iZj|od^-$i3s75NTOlFBaueKgi|6y?Ceqrr-V;9C6WoJ zgikmnLWDaaLbxNU-<8Q7;S=tN5aD_7X_)Xlq|-3rdGKkN@I0i^@Rm%@hcp@{JP$q% z6P^d3h6&F@5)Bj1hxB}tr-w(}S0TK)Qr8IKe25UvhcMxM2ouhSFyVGcC!7z-G@Woh zBooeubi(J5PBJZL{bi(-%CY%ou!ub#;oDb=Q^C3bwAHsz5AwoDGLWJ`n zL^vNpg!3UpI3FT}^C6vZK12xTLzr+rgb3$Dh;TlH2;`9SSJeZu*m&NFMy2cK|0P}@?Ua6W_x=Yvl;AHsz5!6%##)XvT) zoDX5b`H+5p#k@7@UMd*w_fUVIa6W_y=R+E?<@5>XLzr+rq!atl1RcWpkW6eOeZu)b zZ6STa`H)O(Eq%iIKy500!ub#)_Om|Wd`Kd;u|DB^Q0IX)=Yvl;972TiA(`0T`h@c# zMD+>hgHJdge8TyVPB zp8(>G4333#!m$vdI)q~(L^u|b3CBW+a4h(QV4^EQAQhLNei42osKlFyUAT6OM&2;aCU}j)f56 zSO^i0g%IIb2oa8jWWuYEM8kwvA(@5=$3lp3EF=?-g%IIb2oW2EFtIxb6OM(1FyUB8 zCmaiDgkvF{a4dug$3mFc4k$Y&&1dBkj)f56SnvtQLWpoIq!EqLO2%E2*<)I!m$t`919V`v5-bM79xaWA(e0}go!O`b;7Zb zPHa&lgkvF0I2Iy=VI2OXhjx|C!7N{L-gm5f`2**N%a4dufmqLixv_@z;;aEr}HmwoDv5-XU zSR=$YWWs?EA{+=2 z!hw(wCcFn}G)#C8A~Z~R50Yq@a3J`E1EB`tK=26%!YaankWM%dB7_4WLO2k@gaaXs za3J`E1HmU82&)JOf=@UQLWBb$jc_1@2nRwc;Xp{H&mF>np!nG|w@-v{AcP19LK@*f zh!75hbi#oUAsh%1!hw)Zcn=~pOgIga2?v6D&(RzR5yF8GqWbiiLO2jYgae@l;Xp_x z90)ZC2SSK&Af!#p4pD%<1fpYk)Bpe8o!)F!YK%g8x zse}U|LO2k@gad){`9uf@0_F3G5Doe90(D@fe<1b2+8!>K{ya7KTm{kAS66H=-(gc z^Md{@f^Z!uo?Feyqn;_68zD?M5Yp+hgK!|E5e|fO`b;4l2x0n6Ash&lws*l?4xCXK1kMbtOZTS@!g0|&PMjNy(e**UbvE^5s2ilfDnb`72 zZToebi7mfRZ29$CiApv9aeMiMbYjaNQu7?Qi}#6bKF+IVZA+`yrn*k`)mn+$-AB(+ z>9e@RmOmk#*z#wzEncR!&G%8Jw&f4u8tmp`_XM%!S8dMFS8XF}K3;CD#c^ByXqvWF z(dFqr#bgXKbuw)E(SChT#&qJg{0ZwnsFPT>&aY*HEq|EU@`s5ne?pkp?<-q$ZPyQ^Xk%df^n+wv#y3lLlWD33tgmOsjs5Vz%5?bo*a5n{_9A-4PxV#}Y`=9OJ~+=e|~e{2mAJNk@tbsrHs`iykdZem9t zrfZYf(W`o4D;g`GhKcQbgobrF@p`o#eXw3_N1sryu1A$Y?C2Bn5<7aNXO1(?OT)w# zK0?D(huFnOh+Vu-b%x42hZPMd*z_Gl<9(G00o;Q|1#n04fv{Kcj)2Xd-KqsRwR43DhI$BPK4y`v+ zN821n+nHb+PWz~?#{Na^$1`lGv(ZVgccwPxx)e1>|3y1=GCt94I`HdubVk3O*nSMw0Rry5lwcIf0@YgCQcgGY!RdbW9q?fD8C z4%8dBC2zDcbAB7G%)DQAuK0&~<9YSH5ZhOSw#;hYRG+g*$DU*u)W%%JwLVSHF08b( zv&DwptFlAS%2rz0p-0=K?Pr5+)9I>wZC4xA$<7u#JKOE-?62eY(YifZ+HS}D5(>5< z-giHkPT%q3?Y}B-RyNMTa^iits{DB0vnumn+SF%fkN>Z1kkwpZ9F!e;wzkaFPML;P z-mJ!%+9oTV?ChHVl^wUr`>*W5gZAB7t+Al(c6^TX+Q`xdJYH|M_V21~@%CqHM^D<8 zO2aB|R_i1LzlQ%~l!hSJZDHy}RO(ftK&Q}>il z+MGmfuBlx%4O3fe8m2bm)RvsuiIeiH(=aKgI?0sVx{?{!qH8%bo~2H7-t=>TYGmfq@5)+IEuEGj-e$K|4~Ql z#$`f#^fg^hRyz8cmj9#^zn0}(==>QnI~(WBwUuZG9k+1~)RARfQJqZvf7DTS+|WNQ~vy~#^CqOQhrL+YT(0bqxuS8%D4N( z(hH955xxH0nOIz1SNXJ_mZR#(kcoUss`BT4cM9B@rQ$MHuPR^FpIKk5-puyI+LhUk ztJ@Q7mu{bK2W?+g?bLcjww;c)uT)$HHbWDa+?R~S_eG_`j4s_Xq-y8zqi;v&=Va%}sP_hc{w`I2Osa7| zwS5-)+s5yEY;07%@z+$0LmQtJ8W$ah)lYZFIw|G1X?$ID{8Eba!aBOaFEx2xG(8Fb z$CtXkVYMo!lz(&HSU&HUsnK}B5-pB*l2oCj(T=-G+jy6|L^1LOG~}(wCs3V zt(!K!mTwfE7LCVb=K9j>tnA^~y3^9|lzu5+%gQHW>o7yk>=wOVIWE<@RQgIX$aNFC zCdia|9ink1m7ZSzS{kil_l1_FKG_((A2Q@UgJN+=y)Lz^@=-}YrQ=V@TK@Ri^?uC_ zr;>v*cz@{ly}PdW-)SmCL3!h{lD<(UcPJ=T`#Z+hua>9A#@4l#Dg6pZ`>Exfc@_VD zDPPOO_r>lPEvLlqBP~}a9gW^cL0L03c7G*G{an)bN#b)s)vMQ?+W*zMi#@;e^Flwr zGNs<gT1F zSkLk2cTfh$Ma8pw9;y3KJ+JhBl-+Yl^HAAd{oK(~KXqCqVnr~Q%gNwT4wB{@qJbAyIN-Kd+~ix&xe+! z$bJ~DFR*_G_YED_`;C@~`$#EY$Mrs>q@KU*WN?2B#)JE+j??G3dal#wzIqQ(*XeUz z$MySwmMUF8=d{$%MJ*Gbf0^%TvF9cBy}*90_jN7xzOAL+=amdT@6>$g=c1D8dhq#2 zpO;FXzF+A3Uhe~1>h$d1KlJ;Gmci$rj_c>7miqqDQtd1HeMC$8{-Lf1-%nIr^+P{r zVls1ojeR%8o==(cmifF<-%qOC;BzUlyv*+^_52C$w>qxMSKk3zs`a7vUoG|htYvUN z)^Yuv5S0Hn^F_ZS^nR}QaV_b7PWN?{p4In8R^J`^{ZH-ps=svK&-@P2`@fc%&jI>8 zQ2nIO1=SAyouc<4E%kFkOZ~jiQh&c_srNxG>HVOdANqNsrTV^6>00XVpRArU!F@_U zZ?cp6`4g0?|H1c>V4Oamls=6IpI17rzn`?!_nnri9Q_>Al0MI(_24^8%?JIy(s6ZN zKi{-05`WHVsh@XRs_!qAPfPuMrettDbv(Gw>$qBXiSJW*9>)JJqNKVnsZ{mS>ncv) zw^YCAdu1@L^b_Cjg7N>XeMt59zxuqmw*SQbe)9k2{&Q`=!G0Kczy813Z}h&R<^SLI zm-+L5!QPn?%39X+W9eEN@wnle#_CY+8Kd!BOO*49?T?nHrMk|y4UOti*(}$#s9sQF zFGaec_SC41l~Xe#uG2H+YUrVTsP}xOK2bf@E_IzsbscHyUp#$vRDS`|_fGB?&3C5g zFzBof$sYZ%&tFMyyklszoV@Y!PT_j2J%$nQU%Xu@(R{JuRl8|iwJ*hw=YzhAtNz7g ztpA3uWrJF4qSv)lda?QwrRtALkM%27pX%?GE0IvF-K#^Yy$hk&5dFxNe_K?)$oy^5 zbQM>BC)W8@dgeMXjM%)#q#9Qh*HZN}zFzct$cQh1-lZx1{Ge3&(e}orx~}S@QevH` zIF<2wu3DdJ-c&oZjE|>Eik6!xqvLE`#(X}peJI|$6Q$DA^Gju{Ufs`3sn?4dx5V`p zTTiMyx{hf1YCcq)u1g)K>n(Bp=>Ej|5$k74biE^Qd>v|;f_0lY|FQlgt{aseO5Z>FzS8$!a6QrWr`Ka}{n2%s-MWaE`!DYYrKj&BE!8|`UG9H}U zV*KA<{~pFt?fY7)`&Gp=WlCzJXq?K?Kb(*1RUKnR^DC*YYpJfQxRzQkv!2ZQXt}Ch z9jEQk-w}jRoU@-L>l@*dD?QL=|OoM={Yae^n>yo%2DY-`S^#whPtEtpgeT= z&uDs3?nQboloOP#QI1Lv%D?Zt6jJRE%9NvKwEaQZ7wM{;pv+n6QndV_4E873?_hg^ z?GBa~tT#Bm!FdRdPjDQA{R@tRT0b=>HjJ)UB~wymblqv$xUc6ge5;Wkl!F3#Iv4992i~avG|DkuAlZxte)6<^rH2=Q#2+YE?UeVp8wO3mZ}^T*HWizsnfN*t!9y^ z{)Sg`MrEtKMSQhiE2;LOx+vcs^F~y1$lV2t`crm~zDjad%$NAxHYim+D!-Bp^36|j zM$6aoNQ)2Jl(8rM1Lw<#FReDgWd>rGZWy-Fiz70J+-b&K( z*Yl&&RXcQCji0J7IzA{*OErE}s~U`t z$}~u&H)6t@p^cC=J53Dz{Ba?_E%em=q2&scQwIH1`9_|ZNV=&5!oebo+Cj*==}wevCL@A!GHXnfqh zoc=sYxgZTTIes9Z=Xh+(; z0__Y)==W3US;*-3K(M@Mf3mJO%W{JCk$ANHV0pp%be!Z%=tmaq*6sM8Z+{l=ZPE53 zJt)!s;cuKyR^y)_p_g|8(zB3QFISb*ZTn%hE{vf3;m@C<=|LG^e_6g?CbT2oE|vdd zv@VE5S89VV*NL|-btxeo_YA?GcuZ_dCb?_g5AH;s2Qon0ueXcn3 zcW?c>nEqXy{_b`4?_!C+d#T@zV&{}&GPAx|z3MtW-=*TZJ}v3*o9cQn9{hcj#)It( z##MQ=KUc@wTs!xB?c6Wk(Xu`lq|b9`89bMz?@emzOB@bK}47+%Mh#gZpzZuG$;i z|ATS5|114q{Gaywtm;eDzxw&6pHo!ob75Kr&xPszLZAE6QlI-$?Y?#{H_@Jl&a*z( zq-F42lN!IQ&M~R^rRSHFUhuq9@VvmabGb2ln``HCWA;OOU)7TC$Jfr~#?I-g`-;AQ zsP~AN9bw}0L+y)+_KewoCnT1a_}!rQ*`U<>u9m7D*UsgR!|(svK2V>>)$-c8Txy@F z&*!T39DI)m#_97(>C?D=P6efYZfU8{<7%nO(a$k0|973sP5hn@zF!67s=tZvTfz8$ z);^;8OTYJm-*NhTPN(a0{6VSD^J}TbRiEeAQk~~l>DSKn|JTp;UprU(KXtD5zjuF# z{cd*k@0|2}n>t6J&mmqrhZ-9f^;1Cb{GE=g`O))4 zWp=;c>UqC<-CR3|O7Ba3-)X7uH!Xwrtv*kxWpE$(-*#S9tt)j7B7QzrEy0AiUVmEt z(>m1WFR9e$E48HOFsYszM;Z_6>HJy-&j;#w@EoCz2hS_&c<@}Kjt9>d>Ui)RqK*g8 zJL-7wT%?W%&nN15@EoI#2hU6Dc<@}Mj_ZDCsr##?Zikk-y;|yWwAA%!smE1IJ^oti zanVwbpO(5`TI&95sm?E4JFl5}ZZmjJQ^(afT|2LN?YyQwr>W(&^P2y!=QY*2%&Xt; z^?U4963?%MbB8*P_jY~WFev}$&maD8Z~wLPlWLu1b^cSI1JqJI2d%cTDgTuK-PPjZqez-zL5x5dYDY!C5dAJHjMYubRyWs9IOgJ9Z z&mYw+71gO2y=q3&iy$Q@{9T4^FrzxKx^b^j$+*pU2KbE8!6<1IH5vdL7>x~KI7Ss< z6(h;0Y}{$I2evn!Gp;v^8;yXCjK>VmkVY~vIch3;x6ui!pp#L}C~e$eGzB&_9yf9s zIgFaXn#O%bS>s0I3E&e(b0eRT+qfTizwwZ9lTprS0c>HkG71`bjR%1b8g-3Zjhl_u zz}CjoMq#6XQ3qJZsBheE++wr=wlUfn#fn zF9BadteMfocp3OIVoi*OMlWD5#2Olp7`=hL5qrdV*ysc7gV@7HZKE%+FJiTg2aH#M zuORk-QOoED?1xw_{8^wsus>oU^rJJdGh*Kw?;3M~bB$5PK;sQ#9B`a5$Os$KoC=(3 z3^mpo`OF!>8OAVUgK>-b2Jj8&4K^kk$AQO7^Tc1@Y{{ojSr34z}d!E#tP$aTv-7> z#7HrIMQthY15o}_;{xyk>iN`IZkz+2GyXD`q0i%h+|sA%49ZUSyHHXFB_x0oA%8;nh8-y`Nvz@LoojQVCn z^9SG$#*fCM=ELaiqwwEDYlG38ZGvyf_85DO>gFE!Wb+$ixAB4T4SWsrxbd5@1f#qJ zQIk#3mgDtKn5VZJXJY4C=*&p2q@1H8w4&p2QlGVU@Dz*ja88T&D}hv4rr z!$!o&19@!e9UZOHU~C0A2-LDt<09d zmSzby(d=$^Gbh5IH>cqGq(z0D=&baNYUo4MZ{YA!a5v7zwk=0Njpb3Jgq znPRRmUpBh{yO;*+VJOnQKfrv+Twp#AeBOM~9DyqXfCEgAHAX$X z&BpM*q5O0+0*si$%;Dy0b1iVK`8wQM^!s)ACaCXi^DW?8=4WtknKOYi&5dv~(UXnv zo6*+k<}~0mb1U35b1HDExdU!0MrQ~7PPA{ixeU0>{L0*IerhfSE;YY3zcD{SZ@+=x zkGpfd(Ug4#|2UgtesBI{&Viq6&NGjg&jFt^=VN?dGlu|&m_Ne}F<%9~YMwMtn1j)) z6Y#%b94DJ^0^c;xz`bcs0!}i|!cD@=orOP-aZfc@0auxS!L2e^0#}-s;Z|Z~FT)$m zX78KtVKm<}8UDRzz6<%T$r%T}1Njc-ZUn|OH!wFFWsWp+v3zW#nU9SzN1J(B0X7=* zKGyuu>|kEUK7{|sTx1qvMc5*<2>aOl7+4Il7+Z|hRf1j5#+lc%@#c77X~@!Sg87Dd z1G|yEfwj~W;ihI7;V?VIrW^UqbLMp8oY~bpWt=pQ0gsu#nqT6+sKdU5@5a{SK5PPP z!kV(Ljga{s@IBU>DSub1C-bs%$BIb+!QaO0u;8K8N)Y_PfgLBlvqU>dnjuFv7yDC-#QC zR!{gO%Qs7#g8A?l%?DB51>gnqZ*vKD%e&YT_{!`CbF$Ijyus{mTC557iZoyvTZ#L< zvAG<$oPEk_A>ZF-E%+4B8)0DBOv7qkW!_}1gfC~kW~??Eu-D)#vgOz_ zYFNwRZ?wvpWw51|gLl|@<4wS!RRbW-iu>Lg0 z8c&*k!mly!HY=Hh+1>C3*=@-8xp^D>cC(uKzR?lb(L8FtW87^PWbYUS*|X5T!z{`w z!53x~pu63y0RJWSwNK0v))M$)Rz1w;9k_XDz}Gq3}AZfR)?KX_jKS;Y+dxC}Wq|0Ddp_mXFMW)`#%~;80*sFL}Rs>dLcd&2q4)-zeW0u0EqnE|3>F~d?JFxE3FcxVTiTCh4GOYLD zFSCJo=cxd!z;0zp7{8-t68sO?Q{OW$V?VvjF0qToz2*V)qH(~?&rtX>6L|n#F;|SqqHpyXGbA&X?F<>@7UM&H>M{ zKUq)qGJ6;JE_<6j!=7emfM?hr>{&>Kf3UNx9D9Nt10G|`SO=t?1)gQ+S!ebfy8yhv zF0v=tEi4t7%6?`qBJCpZBKw3v4XvENYE~FK#VFTduH$@KvlgaHTk~xOKfXk-fnbUT>AQCbLPbEU>Iq z-ujSDWTkz@HXW9!*HkP6qcjjS50!Zrgpv#rQkEs9%_yB71=cHnlllhtPT zu`hvNvaeWONQGaqz3hFIu@ShD-D%Zl53_Gsefa&TwSE-$qvk4X1S@P+f#1yDV(+lq ztheART92{D?0fbY{Ew^|q{1KB5zKU5l-vWJSx+GC2-edPtfm%NFFyl+W+zb72dt9y zF#NZy71B-sPq33%F|DF_604>$o6GLD8pD6j!uT$l4V=yDvCVjkZwYM4TCaJ_n{7IlY0WlGYZkO8n1}JMb(kH%Ti8%@6L1qN zVjVR08H3nC_}AD(yf^*;{DFPP4&e)92yh4+#txd_nX}D<@Ux)truh?_1pfox_&zW# zYb^XXY(Kt7h5?7M*V#6F3(Ww|V4c{Tct<@7Jj#Az-{Y%vByc1f&4%Dxs0*+QTfnB^ z{rNcXIQxbDX#8M|0ghqgSS!3O?gQ>)cUe>MzWpnk3V)n^XZ&Q02aabG*fhM8{|5Yx z{mPEvTlP)hn`{c}j1_x)~ana zx0+fH0w1)7TD7e4Rv%y=s~%FGww?h#W3{oKvf5hhf$gnlt(NfZf$gn!aP6(Dbt41nwn-yd$EH5fS9 z8iW=Mg6spI0yh{rQs7k?gQ3;LYKB%ciMHl(>j|{%arkGgC()PYz~*T46l)q_5&FdX6!8sI_TLA309v|>GQy>%FFJ=%8|{ztfVxcVdf5ovc_580l%{jSuL#hEYofQ|B*Gq z8fpCs{MGu!df$4-x&pjnG5Z~BhIJ9~i-?W1UbB7!{syf#P{twPA?pWgrZv^t3*2j6 zLTI*i26zUs4^aP6;8E)|TKNVv$H4z&-H$fz1n#uXTD`2U);{1qD~y)iXZ->E1FfrT zy=0{W)2%J2^#SV~@EmG>)Oy(Z9QZkUpUZA(KW*oNFJZejw{rk<*m>b{*g1hY?cDb5 zb|t$mu&&+2PO^RbVc^4deY=WX&29#4WWv8^m)$ zwzBh}wi3V+c2QeGDlBRjw1w^2g@A?Z!geXUq}>*-1IlR&Ukc^svnv2ApzM6OS_)VS zvFq$&_8q`G?1~5#x9j{^dPQJG#EKzr4PXtsDnj|}5HN&TKIqm0 z)(sgI(Hg0Bm44va8$4_9MVY?8oi0_D%ME!29e6?3?ZD z?S{aH_WkxP_HA}!U}O6+`%e3A`yt>%b{+dhyPW+X@Im`gyRv<&-4xi=u5CYKKZhPa zXFm&UZnv{L0Xx}G*{$u4n4i}0Esiz&?Jo9kNQGVO6?SL)MLPvE$u>CUJEA{}m-niBoS5*4T zh!2DBY`vE?Z6vO!^v;M+f**+4|F10@dF0&(Od&FJ~Tx+iauD3r&nV;L+fZObiz=4>9*}&QMt8laJS-@HLQ2RCe zeS11^x;+Tv-plR}>~DW;e`NQw`vUvgpV&+6KK4t%m+S?YkrnnD;2L|hJ<48fhk;>x zoITb~NB_pckHA<=w5I~6+FRhJ+LM8k?Jw-D_L~@mt?(OA)*#gR68r-6bT?*fxBV6H zD|-iUkNqvi@>_c!aG(7(aKHT>R?v6$_rUM%gTNo`V;IY0_Rqkd?IXbYPE)6kQy;#E z)5JMppR$`cr|egpEv+=I7OZ7fY&*Bof6IsPB~yX=SHWPQ_v|1 zEa{YS3OV_m(!kQr^-eyguu~pb-YMl2af&--fn}YWoJ!8E&dtD^o!gy?&Rx#Kz=xfC zoO;gP&Yf`gIS<0sLHth0dd@@6{Z4gYb*GARhf~1`0YgqLr-t)@QxjOzNp>nbb)DM4 z+D=u+cW!ZRgS*$M2A72RZIC|3?iq~GGfpdDJLh?f=<`l{U>E0QjOfcwcVJ&U4D)IE{dfoZe0^ry*8DFZdpq%Z^TGU}xubXN2-v0O?1Y?jf(0ok7_Q6>dcH@9Uo0!jFfcv$O>dbQvIs1Y8o!QPLXN$85 zxXGE|jC9sHe*ph*!p=hHd*?9luruG8;(XzJ4*cAC)A`8x$@u~JgY%&?!`bO<18#Gs zI!l~m&Jo}d=VRw>XScHhxWk$5EOSmcKLdYuK5^c2zIJv2cR6o4E1ln*Gv}Oh9(dl_0C(QG0KDLwg}dPV1^mnT9qupZBJiSf3httF33$o*74DMr zH}G%g7r4Kj%fQRdQMk*_72p-;N4P7F!GDDR&iM{#LYn-ba}dZN8QcF9Pl|l(a6Q~@Yi@Q z_+q>fugYHmzQULBx=1SqEXJqsM|gW40Y><{+~xQ2F~BjrAb*NK#ODC#@Xh=#-hzJw z{D}A9!;Odeo4_~uOygOekKe|hg5^%MjadfbIBJ{yBe!w*|K4 z+xa&BGWCc=-yxVPsWfgSm0a2zSL1_t3NVEy@fv&} z=DY@cRoA6E;t5d&+(^3Td@TBSFY%Hn1T2K{d`a{X1%U+->w~!~04#vm-~0+6 z1RNy#i_6>;Lx4j>iZC$WLxDrZtD=;+QM?X(UHrgHi!x%QC=EXxp|Y6YGVmjXC2TPs zI38=w7F{6N8O;;&$BGsbU5Cu|m|q9lug66G?oTsK&?h=R`;JyCd>f=gY+>qB{RXRN=+> zKAtR!!%xL7yI3p|_wq%eGM~(=i9zOMJ_x^OuH~J@i()N*QG~H8e<&7+yV3HyaF@I+ z-;vjgcjQUzfb+y0)IUd5;=6eTc?-t-7Wox+g}3D^@>l+fyaBmi6n%kx#XEc*?n`bmx^sFJ4iMlW)k1$aR~@!`~KbfNO-$ZxN&ATk;n1mOQ~{@!q1R zn1#~bgP3IONaCXNZr1AB%f<1u;^NmK9LKE$Dq;(Hqzs`KR+L zqMDeF(VmQzbqXu15d8CeD1TGz1Mb5NmB4896D8pL^ZvjAkORc4d?v3Us)(5=VLHE# z@5Rcx4!#Jlzz^`lz{C7{K7I3%5ub}Y;y&CH z_aRR?@tPba%ZXvKyx77Y5_N%f#Rh&mX7E|e;j^-U7|zFtalmn6G%q6t$ya3=@v1C~ zU1ES7BugU&djzgd08S7i`4_yNcu0JK(zftYqMsZnONoK^TXgm_7o#1rZk*%IrzrObn!wwvrBi-{hxxah=NiN0nh z-q)O9L#e!(Xf9GQk}G&Y(Lr{S1)*68v-Fy12y7@m#e2q7 zF&T3>S-5!meH-7=UTr2EztI4bhXg7T;+D1R1v#8a}B+#_1a-D0nJT0SNBB4v-5B1+39a*Ajo zKNsJMXXMlJTU^~M4vF@%ojfGk$pd1mcuba;TSa*}Q+y{n%J%X*Tsf!&k3!u@_pd@vZP!hs-uP_VjAk0Cd$atz|t7InHZUJ7@Kl(y7*J{ku&9=Vx}yL zUfwEi2Hq^+5hdh%vbQV&e@e^}x63ke7D|{VrXyE167N8>E_Cb4wPLooQ{FCT zLvxmxEAEna%DG6HEf$I@va(z#D$DueBjl=rd{q!%Dw1S%xfCs2B36l-GFh$?$?{W? zChnItWg1deiTCmQpS)<&JR5ucBe2czu480&m;vYaCG z!55Gl#iQ~O;3IOq*en{!N9AUuY!pW@%lBiR??)YRI%WU z2XFGV@l~)(>=X|nr4HV?*NHi-7N5h08TI(r;ww>)eX+xLOIUs{Giw`9_9PRBl!0CLcE3ViMMcXHx_Toyt1*#E0^<&{9ajAUc?*4CA`@m76(N` z6&-mlnL~C&t`0meQraM`4faqEtu2N2 zmcqMBd-Ng~`jHEI&!XfTQ1%TnAOBhYDZi3G!!MIB;`htLz{7I0Tq3`g4_Zs$n_9Ev ze7Qx=g8v@rD`YA#Rj!oFWx5Ol!*Z2eD$^jB!lxs&Mt%nT46$W0BG&=e$+dDd@}vS& z<$B}@k7h1`nJcDVz%1F_9=8{}sA?FfA-cL8@HwngrQ+yeh4LR+D^6Mh%` z4qy{&DQa(GrORgc1v4Ejd)!(spTIAeE70O6(VM2$&uGh9^mna1E>EDHX}~o3n>+51PkKbPIC?PyDP{D!$5BeWSKu?21T9{#922HXbRCV!Ma z0XG9T%foUGO4(+2@c8dUuxTRgB$iL+U_>1x{ zc}ZRYUXhn2b1y?O_lm@!C?E%N*GB1%D}ciF+#IfS|CTx6&m$*?RxUTEo6pS!pBFBl zTL4(V&F>b3F90my7J@6_76uk}uR{y2gDe7H6s|CG6opr16o%F={BC(!?t*9TZv1}v zm)s41S?+#wO2A5(fgJ96Ss7T_y$hiW@?PM*h%J_--0OkYyNl#Q`2F%$_aXR0a)G?j zE#)r2)x~nRyv@DC-7W8Mzm<>S_e;}#4E|GjM3!@Ja*xQH-0$Qk(s1v0Kauylzsr5{ zPWNtipS;`MFaMAaxV79rWG(l!JPk~SOm=^j`*HOiT))TNBY%(;+;Z*@DD8+mDMM}< z_oOW2{vv+?R)wtUo|9)~756sxELyT#9+J1Z70|{C$TM3exi`AA(b@%alg#7pvo^_n z_&xJGw5mGVRo(qS&XvX8-0obN+ub1dpvH679(fMGXD&joOQ7E++x?jwjj1Yge;`2Dg0uz_30 zo#Hleo54MaQk%g~L5VHhr+`nn`S5GwQ%KEcH-pxrZd>aQl{0iC8 zeID|8w+Md0d>&dw?6%NroG2 zxSidm?l5-z1>!BYqyU(6gbp<%x&Y2aEAkjyU)1o z-2rYtU_ZB$`-Jn48JK z$?jW7ndauV$HCk7r|xcdj{7P6`)&cRkayZW?f&XshP&dH^h$cgyaI5iqbVg3de{BH zJ?Op*|DhXpPa+nEf7e~+&T`*_TjVZ``g@44gWrap&2cXPFSuLW+^FYA)ORRa@;t>%iW#8o$fan z_ap8nz)vv#Ke<1<9|J#jm$=8=@7x8z1@1?fk+ikdz3<^SO!yOa zKkrxkW_jH1<@JSo$r}JS$m<7p+*T#O>J9T|!oBIK z-!Xr;ha!9uSjH>ml?RshDtHyWyS#gV_jq@Bw|IAZw*hbS%6jFz+r3J_O5TlLY41+& zUf{jnP2SC36|XX|vUjUj!>j6rfFZAz_kj1X_XzM2ueNueSI>J8_@I~MC3_Efb%Axg z>RvUkj@JO#0RP2QP47{!KCr%bzxSlq%xeg2=r!>k^ICb`f!)34-VM9pKt| zPs6oFyaQxYjMp%4B-}9XHH_$ZZvxzSZ;Uq?*PG(KV4vU3b)K#4!6Qv3Af5ig-i3&;lf@7Znd`tZmstj z+&XVP+y>-aihL^9%BX&NRC7&KXMObQ@@RSlDXH*Fy-nUmZ!_E$?{m1V-WPD&yzOv1 zyq$1gdb|Eld*2;qMe(fNJ<|iYWRN6LKm|dRoOaKwfQpD26QE)iML~ZcsGutt5m8Z5 zKoJ2EBM6EL?3qY;Am>9It8zZV@o~;4I6lRdui`qo>Mbk(u9f`I%6wwKeZ%tK2j?B6S93nkS)KD4 zjvsS=!0~;~cR0Sy*@WYpoUd_g%-MkBtDG-!e37#r$GV)gIM(3G&v6}H^_`Xf+Dd+D zW!Bnnf3W-w;A}$rJm;sJ%{g0e1eG-oRe49wj+}pTw&VCaXB&>ca{k2eN6uCpzvujh zyGQZewb!F1?Cpg=Ye#+VUOXs5YC-;jB_PRKhc<*yM z7aexoaq;=L4UZ*0a#g3ILH!3>d5IUF+^J}vW6p>-Y#16#eEGJHMd5Yl#v?|a6-&J4 zwT?yI`VEci+%hOG3$JZ)e1=b$uN|LzLXCTAFFP0A zvBz0fUg96F>s)k6>U>*|5+669bJ4u;LR*hzVe$j>3G=n&H;E~)Dae=VQDU}@>P2EM zBh{V6>=Tlgn9EN3B~G2vt!V4Vv8_jmKbhFA==s`twjRsERCmlL%-2#qO3XG;`(it( zO|cDBzg#Yo@%fy6NO>_X?N6WIIA8J=+e7|g8^|YYGt~>1i}W)tT}EyrusM;K+ZW21 zNX%`JPupd4Z2oh z5vz&BB5o6jxs8*Z5_1ewS<1rX2j&yzYsqgCibtf_Vgk&V< zvQs@uEZR#VF}D}8xhzcOWjzw!6<%EhO~;OBv#BEFQ7 ze8{IDpCWmQ7YsZ;RrS-{_>sviV~Jmzdrj)shtG>gHCm98_{GgPr8eAnnSIxr;HM>D z=t-oE$TyKP!p=m>NWLgnA~EFI4|Ht?Lw_3ON>DD!D}!NA8s$n*uC&CkKaFxFc(jNT z{FLMiJ+YJ#`H~MguM%}3Wkk8WGNtrn>KA&->k)M-{Qv)~9;yB*jwuHICt+WE5ph_e zO-a6pw-Rj=@yY#0iMEV-;eLbaC4*)EAo$e2GX06rN$rcTO$i&QeR00jcFJrQk`J8e zGbNVoQDWKdB!)fQZ%}`j!Lol4eCiJ~{fW?1B5p;#CHg2~XNf*dl&eI2{fi#vN zJoh0b>Q4BU>}3C!*P|O3eD(N0sjrY_p?*-6zt^_>y|(3XP~Hy8`+@(4{XmGP87icf zc$#8UJj<{#Qg1x*uqK|TSPiKYo`k6J)WhaTt?)F(z40`_CP;no^uyYCf?^G%E_h0# z!IKUHqmc>SlN&4H35gYv z4!~0r>*Gm+^^o%LOvxNP&2bN;1M#%Sz3^Pd21qfU(pXh>1nr1tK{mv5CVPSQ!m|kJ z`IDVNJL9>L{ZwB(mGN*qYq39$Bk`Qbqwv(lV{jacCq^EJCm;^MaRQ!-cp{$scruPt za7{m4LGnjg$>XfdiT2xmmVXR5{gLRYkXPYpHv^GQ#nTh-!xM}~BVCTCHBQEpjm}59 z5Kl%Nk0)o0MH+;sD$c}{5yv21i6>LuhNtgbfOHX_)>wci-wZ}N4Nq8{g{NFxfiw=! zQk;UPAr3>j7*BV+4o?R;9cc)j1bLU53OW_ffjk4x1Gy3OM*D2YyYXzoOOP(b^D%G2 z(;Ch~IvY=doTjFOPRDZ~&rvtynT6*dorfnu&cOdJjY7H%&&r&Hry`w;G*rz|v(;Q2 zi}8Fcdd}5C91GMFI38Dz;doR%g5zO4-*G;knm7-~gSci6t|0k^R`M|`Gv9tY$MT;5 zXA#moHTTEn`L)&#p>|L(pm)ps_NV1q%>1173sWcV*&@H*hlAs_InSk@__KL_w}q$2 zFU?z$n%%f%e%tf&EuXI?`F`D6=I?P!o|O@L(n3#K=t&DbX`v@A^rWjDH!gl?MQwW@ z_um;GxxOO#FFreO3)vd?$LDG%tV?~-yh6Nj&6?Dbrz^(8!nLXHOMgkxXOg4uZ@qI% ziunIk`3mR@=#SdF!LwiSuc>Ic^aNOCIn>7SxJI5VT;`fzTaIT;xr{{H1>;$yZgw0a~yVaKBQM`xSF+nYZVH#&W>m6y2LSx?7*Hz_WQ z#G5zHi}j1ee2K{?V_d!xx12CHZqT$?-iHr+adv#Lv%!bSX7-!JRK_>{n4T#wW3su$ zhsAXyF_lr>bx)?egeN{ZJyWNHx80kpzk06C>p!bzBx^<#*Y&!`?oIAJdXD9j4Ao=n z%NN^gxvoiuaIKk7#S)Vq;$J=T=~!aYbLoQ*#TBl+C-z~ojjpA#OHBIn&YvGQKJjjw zS6P_yTJ`(W(q#*=yX>al0oL+Ir!(Mm94ho#E^yaq_`JTStsZ=ZG=S#M`zkwDrQ6bgt|8RQy8C z;<(~=M|uk0cq%@5Z1eSG}^x z+QXRi-2d50>R!;DFf`fX7rZW)uF7f`Nv#k5ShxczdwCY?i2##wErWy&(E@SYNw^sI-UPy3?S z9_}B=HpZlf{9Ls9ZhI~H_L_zzG3n9hD?Zv$90T0%(Y0;SS3I=6xPFV!hgAE!B&KUC zb$Tdn=i-q2Qo8oast?8WT^uqdpY;A|UVOcaL&l_M{K~npKB%~^x&I_RU%fUbegJh% z*UsGo*wrs%y7u%%55}Llc;o(={J@y>ys+1U@f9xi7?aMI9-SL6b+N}-jMs%?ye<^u zb)guq3&nU{D8}nTF&L8&54*m?%1G?`CCiuCwPnkf*!2~bFR_a&%a_>2s^v@U>fQ2v*u|KY@nP4-t&GG} zM)V<;FR^QPmM^hu*Ot$i;*)HqdZ8G0<8?wY>Bj4X;?|AVR)*>q<8?yUa$S=QVT{*S zo-yemjPcs)WK4Ov@!IMv3$uQ*O=8N+jnOu*vM}ZA#%P=G?l9@+*pry-bmO(PlQHRc zBj3!SukFgz@*2G*Y;ZOAE@palO8u- z+iTr;T>_IHH(pyk-0#t~ZoIbgZoDpm=~_2l+iSTmrEA?7ZGFO+eB#DvtA{b^apSes z!~G}eapSd>cVl!3OxL>c+Fr~3Gx>or>2YJUwTCg;G>kNzE-u_a0H!=xieER=Y&o%Fii-*sNcDa^#^j+zEFTx&}=dj*Vnom6X zt~5VqR+PT43_Z}Dl$GzLYuMpmGqVGEzARQ(4Fdgvk$5X8ze%C!4D`%4`4h1%*DmW^9%8 z_K%0gU$j_}e(LHfR!@gJ$B`d7uVkO7b24+wzVTh#ZcdL}eR*=}_TTe zo6@7}9hdZKyRd|Or?%~rh8OIgwA(N_HDK~($%mnS}#S0%S$ zq13r;@0|34?}x^w{R&%dwoR7z@gFmE?M>6UAZG#R7;(E ze3V{rjY$ojcV#^1jn2vA-InLwyQo7v`hwBPK3(?6TXo9#)RsH1OP^gqr~WjfO308N z(m}dN59uIX5|d8SM|w#o=_9?Qlk|~ZDRWi#aY?gdr{*0O2dR(SR!lE_VSB3Xi-Y3* z_wJU=S^Z>cMuolN_8*Q;ZaNC@LhpQ9s`_U)riU*0J$3i&6H3UC9@0U&NDt{CU8IL} zkS@|gI!G7kAswVk%7`*97v)L|n+xj<8=c&BSD`3ZTH?A(2PQY2yD@c_xh^frl@{en3qRXyou3oo^Mv$3M+&;E9_Qyo)J5|2 zWVFLaDyI(^@<48%S1P72zVFK9lA2-axp{+=aHdM?)t!|0)A}3Iv!;(sR&Ebdz3=Fn zJX))L^7Z%ydCn)xMI71~ay}^(aabtg(8i6FaXv|lx=w6)iv5tLGB`gZqJC{WIX@Iq ztgLHYF}+{m_VmgXgJOwYdD9~H(jxY3+(;QQwmv7Z99tz8W9zdL%WEZ;*Gep}m6-Yr zH(sL;@r}!6@?*Nzjo0YY{#Ay?YoGpq)#Dq#+<1*XzGS{-jbCJ^Z@hEk^)7m73@aYH zO6sR^g2yfzV~WR_k}^B_nelp;vOD`x_LA4S^6uhK-+ZI2`H2{>(_*|{F2?J$7_SSd ztmGpqZ*iIJcwJ%~UMC661ABj9K0}gS?i?RQy*E zi}4zD!G8s@7_a5{7>n_`guEE9(_*|Xajh7y^EfA}XE9!*e)+GUM0v${UE*3m+f4uHDd#=*{Z+pJtYaJ8+C&>LLIVRqM>*%}|a+l(K60W0j0py6k{h+^7bf)(e zQ%}I>=5=btLvH^mkIyZ?u4%t7eP;R6=hAZg`LRnH#FhSeGmp=g*QyuSKt6n~aB72i z#9?dm_Em;2#{RZIA<^_P|H=eKIs zil42vF5}11wQI$dR(_MmXZA0jpBY#k*A;X;_>>xPLE|d96ZWea7j>_a%l_l@{(GW6 zx>e~}`rP%?;(GF(*ksIsB z=l}6W=6osm3;%vIkIx$y*Na>I^m!(p>IL=VqFXoS@%gXA>c?uq<~%+>+#32;ugb(@ zy+V9{)apDwzkD*{C4X%mpI@tAKQ1_TdM180%t!p5F*TRZ0}x;8*eSVu?(}foxM*O- zT#mneo%9MbuKQLw9ihulLdgFn7j=6u$0 z$nR*BYvZ@Z&Nx15NxisW&xJjg|BCPb7}0~z18P-`@44`;OnVyqLFM?Kj&J47eKaTT zcI?jcyMI-W`;K0hX@`fesvggeH|6npIofHB$v@@s`J2Zmo}JOz8UQ#d+V&J>^l*lS%I$KluqeI#~4AA#dI zyYu-JyTiG*N~Rr@Il3T9)4qpASN6q+;Ya z7{^fTOCFBB$fIyvgT2W&V6StGW3c6q!sp>gF)Yi&{_9Ri{js0BCU$%0Bb|WqG{kP| zYDnF%6a505n<6#DZt)t}-Q5OhEOwA~#4hbBNPEJLs@OqZ3#mT#KpzN+8e z0q|v@mtik;1MH<90XhP5onZH|pvPk0dl&2!KMM3H>^koZ+7GlJY&;%i>;u{dd&N(} ze)_(keX$RGAa>X1gXW{W*T7F>K*wPB_qEtReg)_i*yVj4_K{x(dKqlE0pDK>dM$Qy zPXxUl^m^>qz8T+70G)txU5W1}f=e5{63@yATb?#!)GDUj_?`q*=(fyA#n@#iQj`X9S6O| zcRJF&khl&0o`EzC5ilIP(9c9V7rV_bu{+O4Vo&^e*oQs>bOd}k9MLiibQq*A!Cv`` zK`(~nf3PS10?-Q}H58F^I_T+;x)A&0&j39GQWs$#{HdU)LTVK9IT!R?cyTmxJ0J9X zc=2-Na~9}X@Zw;^XAjUG&~zI9TOk)T7n+9Pe-FBWc7vv_@XcMIcR{K3FA-;y^s#We?riI z0vw7&|EJIbl2wp8Vn6*{?4W-X=@IO(pNBp5k0U*XJ@@mm%YFgU6L_oT%h-><7;k5O z6344}Gw0jj5$8#umsD5%8R%!&VV{e46o0HfR$t)FmaFlm-Gg!DTFz>`JNg;CU;7Un z+tptB>#~+rz8K3_`dItF|Rz*1IJ>f5?w0aN6D)j-5mH2K6z9Ig5 z`1}IW5^VckjW<|+shZ+l%5Ct@=mt0{=-N1{YIJn^OGW$!_}m6b!LBW8E3Dm$cT|3* z{#46Am*Jh>i%=G=7a=8hgK2{Ei%8Yq@)z2fh zYM|Bd-txWlzWPJFYy3mJzqC+Q#OFeMBK*AC6L?Qu3HE=AH)(&2v<2uB^)u+t>Q|tT zVAm%|Kcj4a<4v1ek=El))jROU>}^P2;|X z+oktKzO9iO!(;8hZG*HA&_R%DhtvvhVLkw5I}oWg-W1*(+@?rv^xnE9r1nN?hc|yW z(``Z9Laqzm%X|pvA&87_x+mUN-3_Ua?t$|60_~*_#W#oO6le;u(+A(?gXZJYq3~B8 zXdd3Lo{P^NLFpaZDSX=*v@<@%__hOR2i*yEpM$r!S3s(*_rSZY>x0(Ub?|tsO7L)H zquS3sjq8+*3#AS&7P?7dPp_#O${A_hKQN!_!e&mM+)#wHLXFlHt<{{ z`?le|il7ydpMr)ph*Em*YXA-FK-b}2uLc@E1Eu%4u2ySQcf@gb z_~>)Iv$ZE^PxxppVzMh}SNNzs%K52kkMyzXh*Ez7`U&d36H5IN=trpgFHz&o5%JAY z9~;$oswHSk)W^4o-e#c9P#+5sCmldL=vUNbCpXXkYUt$nzB-4+FFw{HzxCSD77yLr zBLByii}?~S>Ag>W?zAyhMq<{p?V!EmCuTR#?{wzGIIn7h`1~`Q=Rdagmbg*V#wGBS zan19qoI5E#zvW)>&1bgApZMd|R-SDjJrCW`BLCBkS6Uf~-<{nef5tCktj-6zx6Dr^ z#d%50Hj^HSuiLX_e%%*~^ObnepDps25*8}MIy1Ni^PYyyrffNQ5T8CqAn7NMO`F5{N(&^3YVXBm0i;U z&I3+py|n4su1!CAP%FLv=e%kLr;qINIrBHLT5}m zCI8ppyuwWr{!Z2Jd0Gj1pUo&^9?FLn%WB^LD(i+V{3`$-SiODyUomSqw3Qi3n)B_-uWy_Asm z*(~a%gdR~ZCG?1TDIxDGm$dDG{QB{YwVqDBJfVs`^JhLYpU=b}c<;*8riFFlh1ai2 zwZrG@Z+|~?p7cju%g?D@J$?e;({(39{`@zp+RuC@{}?Be`5|IGP;;{0;lne3wmI8Dq}68Ce|Hl;(ogV z`q2+*cm6eS*S#lJ6KEwOz;}_o|8HdP#c5q2vj6$n0JH(tFKR+(4rmVU z(CdI!0F^|V|t;9Tj zlOBnAJSLq!%wsv_B{9!4NTAeAzbABeAH9%$$krk$h1X znfVsw%YGo85{tUX%pJ)FQ5Rk;>LN3bBzv+M`?Haeflb&ZGe>3J^^qJ;)lOfM%*28Dk&u7x7z9`l! z&ouv{eEH1z@R{a;lwanIc^=8ryicBKJ{UkhpG%jUKlAy2n?LARB!iih!R(7>YXOc1 zm^(GbT&f|CnwYKC#axSi{|PL=AwD-os*PD*70m7wQivH?Bm5dt8L29cy)mz=ic}3V zu%`IFI#Lz<3R4lkQq@4(v&4L^9%gkFFt4is8)zO`1GEP8wZiOcU(kIq=W2=B82v&* z&jIR&c~>5eUYI|{m_^aATb(UG51+dswX{zg$_35ET#M#my+M0pCU+p_VELf=_(b!t zT+m$1&^lnA*9Nprsrg)Y*w6~|yH=Rp9f0dQfOde)_ZPhs&rhEn@42I2?8VC##|vAW zUM%CmgI}5x_gQ=;&mGNP+q3(>HsanwE!&H|1Yah!a86W;0J}H>+a4yMsG5o9+-?!Atc<{GPKg#Iw z;jwkU&e-F_)1Ut(Q!XD~`Q^5Ze|)&#lmBG=?89^|*Ow2I4P5U&Ol9Ht@nP}_$Dt38 z-?%j1r((x=`J#JbAEr3pI=G9K@!_A(eKJ1yl-5?phvUy5jz2wQFDv82cqe-N?4cE{ zj1QkRXioel;&~Zt_F=lV)p1L$j1Q+^&sWFaX=Qwv%F=G&Fe~H36X3&Hvk$Q{K1_Z) z`G{^-#)nV%?VXGsAKp0flen$3$A|CvXI-XTK0NWm4e`FtKR!IN;OmT^eVDF&&efL> zlMP(&K1^ld`0-)#3CF4ra}1x*e@Z5v33J?*#vH4}mzd*|_!3hrQF)0kF~uSEvBZ~{ zV~^!2CJ9r#Z9wdm#$>~Rs9)ks%yC70i8+>tFEPgt@g*kT9^mv#O!dO{NX)T8dL*WJ z;POgLGE1F*B<6Z2JrZ+W6JKJkU$&p>oiL3T)TbS>?e?tj7au+y5A1ZD?N5Ao+$m4R z@0~IzZn|u0oDC*FZ)|yz)%kCD9_9(wPRM72*S#>r+CVyYhk4FG`m@0^S3DG7v8K1R z*@tWVG&hdt9bs+A2Gbm%BIYM#LpFH!^n2s+6A!R9_;9l1o_J>aj@E{3Fwc3&hHUVn z;djM$Y_;KRRvc3b@7kUgyp*^pXDY79O%yUAH2bwGX8>Sd&eb*psgAY>-r2pt+ZO8`Owt)F( zKW-Ph!PFM|9}-v_e3;t8i2g6(D+puKV*aL z+}OoHHn>N%=e7shiovl1)e|nFtvqzH-E?m^F1lqkPW6j zf#-5QOnm~+<+8zZkKCR-;?{F~_`Rzu7ALr%IZajy7e3%et2b_qCMPtjt{R` z-=JutThH;~@qHT>t#s=-K1|ns;MQ||m~3d|)^mKA%F^Gh=lC%BWT9Kn@nQ06 zV)m_;a~rJJK=KqaPra13H!%~`{b-h*v~#p*K&RNFxkNM?!#0T zjvpT;pKu)d@b%w@Mfp4EW-|#9$tKE8z56}LzS`pn3 z()yAQ*I8J-=pDD7y7dzuCcn*d>p4D5eml*r=lF2X zmw!%Jj}Jd`=%%EDv&V;jU%o!!a`~{?_*ug1T|V5f=jw$0?89{JGFM+dOg4Px>fMK_ zEF3>R+#UYmSoL9!VY>I1nB$iC5>uV_M%)r#VvbMZOUyAze2F;@i7zq79?Mfq66Sa- zjmd^iZoNffjw_Ovm}80f5_9|zUt;nt#WV3G<~Sj~#8juPoy`(+JdnJ^90SCcnCqSO zQ#=qR8`%F6bN!Nx#5|tT93UIK>e32DgWP(K4_9hmvFLrbo|6qGKY!!abN&rK^e~|IPJ_UUKU>*l0b2jz^%~k#lQ`sMJ_iH{(Wv}bjbF#raC!~C{!4v}ry7e3%rWkm^-M40g zsVyAi?%%V))E16$>p4D5ZDFjtf6oR}zf|PbbF#tIFWu|bb9|WkrIl_yCmT$Ch+0^@ zo|7#e;ns6V-iK9g@p_KcxjT&cC;NfspS#1;pZ+0XKlpIx?%yZuhiov-jX55&!6(dJ zn{Yh%@GC1nOGdhQ$OiL01+|N8@F`b(kZ`;3;n7o9CPl7YWP|yhg>1+MUwHIe3HJ{^ z{KWEC6Yd|f!E{f={bV+{#YeAa=AS;iU!|9=4E2-QV7}*KKhV9{zhNr-HaA}QFqNIh zi)=8(0FRs5V2S}AH+`65fXB^jFyAv$*|Wja7I^;P!_*dfx%HfEFyE7s4cTDo6L>D? z!_+76TrL|->(Xw2PU6Ko_v9q}dm{Gpl)(I(8u5LYe(!MmG!h@C-(@x+CP|ME({Dy^ zW4=)XGCoY#zK1yg$@nnYP#^0#B;&*U`v}SSF#l#jGG0u-DLi@KQY+)Zyyj0bKFn+K zB;&)hM#|;#VOq;&|M)PiIkTUAn6Bme@?o-p>)nTWO_l21hk4DDWPF(SGhqEB@nMQ{ zw?D_md|8Nc(e0#)o+y1E9iuvGtI(M_K*ZV6i7B@nPC~Kjv2J@aAvLPExdva)R8}(y8 zOnY)jhHS_N^PCO)F*5VFf5TMv`B=}P{@;grPfio(2imLiZp6D*xjRg4p&!p8ZM+8yRSIb?$mQ@>P% z^&GMx8%%wOThFomNj8}F+g<8%%R! zj)812?LFXl@L}3}FwDh6Hkj`zs9j`(c~1`c!H0QI4%J0AnD1H0hHNnHMWOwBWP=a$ zo*c3v8_auhsGrOR)1DmeCw-XqT2-jhRpbvBsq>Bxp` zu-KDh%U4%v_mrnbQIhitIelau%`@5vz>vcc3R@cb%U z4%v_mrgdqzKPT~F{@sx7qkWivQzIE4rr$W+K8?hO`8O%j4)?1*OnY)jhHS_N^PG+D*RsL9Cx`lfALczdbibAj<~bqR zkPW67XovkdWP=Y=46MLD1hOF;Ol`sK$FS{UcbNC&kPSY}dveHzY%ujpE8TidHkkM1 zkPSY}dveHzY%uj9w7;F|B3n#*a>xdf_u*#PlS4L;%x*E}=j;cXf9?+Ro*c5lhj~v9 z*^mvUxiQB;HkkM1kPSXedk@CA^_*-l-&0UsWP@oB2Db|zrac(6Z;kwr4d#0mvLPGH zdveGRKFoV^$cAh%-xEBjroUke{uLl=sPyZ#Z8mBl)l=^Nvj^Bl*w~lTACnCuIGc;lr+&Y$`3|_$lK1 zWSo8}WARR11@z8QveU+cl$SEDTvkTPOBvXhiB%~tWT+e&KTBWvWIT02zRCDs`cvwW ze2<@F$@lm$&ce^E=imHneY_h#bGiP_&+J1hvl~BK8RtWbrGF$JddliIYz&f*=sMVL zV`ZlfLIqT=3?H%y*;HCa^5M_2`VI1PhVO9NqsQ`nGR~fau66nDqQ{lj%1HU0`4y0# zslT-{QXc#=V_5P_U0Y_XB46t}XQ%abX^i}>9mIFo`d0F(e%R+y#_6|pCuK;MK9f!< zxw4w3fpI%JKGIm?GD*gHLZrTmKj3gTx^8dW&&hUZgn zjvp3Z@Ok_6l&Mdo&YUo;1YhcxGEzTlr1k}!d8||Fhn|$MGt>7<{nCD^M_yY>XQu4Z z9$DT~^Y%-S;^#KMn3DA?ZIq$b3-` zY2GDKXQT}-{;eNmzCOOpOX_!RJQ01AwL!|uK26%-+Iw2` zQPv*Gm-?k`Qorj%*oMrQD6f@$Y$+S4PqQ}2e5rjfU+R}KQooczUKtx;XNu~~)lp{r z_1Qpu1I2{vtIM=O%1HeZ!{?d2P+u|CnTyrTSTAjmIuQ>kiV4?GX6%uCnU~ZrF=8*1 z7km)o8g8?hIfk@B>W2?fRNv$WA7ADr^-GL4n8^$6H%33n{X=FxBW;lRqW)7<->#p` z*dzHeFR9;Q+b2uDoEJH~OCRO*+x}VR<>UMEa<=ien`1~lKjBq1^HnY zzRXMNcXh;LG{r+q`AB(JN4tz!uHNl*LG3q<$BxJYS-AVQrA|E>mPV7mzg6;dDmulvBC94JeQ;PXzh`FsbAVA^^0=Zxh>T_@#NgBluVjs zCQfUA~^P^ZqW=ZoB%&cYW+G{k-Gbw&0U-`lXD;qE6YKSlF|RjO<%n zxvad@`BNDXMYBh&nQ3Q zmw~ywByX{kvDoRizIFO7-`T@HCwnY*_FF7v-2DR20WxDN#_00(9MlWuOXcf1@B`-0 z?v|eZLL!Fg+@V)=Sb@tSk_dX88-a`$334&D8l^^aUHk}~do&8`V@W=+Y>KjWSK z0in$A$d>QsrMt*TzMNO@)b+pbzD$C8Of*mr&uX1Bl!-eOUpPtmhY2s_9S$z zl;5cTv%Jc~m!%m7HTP}%_4~+p_UcxEL zSMr^_+T{gy9f#ClFzmQk$dq-7Z?zv91wgzi&mzJ=cw6S_}z zzv-0VOZ`$t>SvAA2F33|rSe3_TjFR{$a-M`aspcHqRSdunK{ip|8H=ucP;^WJ_q<)EIUSd6m z+C(O9r42G)_?Uc6zqcknzRXMNF9Xy1Mu~i7TaY%$d>y8>l*GsP<>hRnH5T@{ZFf#y z+GF|B23KCXU*>$R4L-iiOX_!ZMEA%Xd$#>bdD%Zm8(h89{WAOC+T-KPyrg~?t8_2V z{eZPW%FBL6+Thy7F8TWSGB2s$wNu&~!+n&sLCVWMP1@l4C|a-J_Gs;qe5qgBCiS~M zgl(Yy$J!vTm3?d}8+Z>4_i5H1*GE|__4{m-GOiC~Kk)t>s-w*K>$8FS28u9?6$^N&OPbyb!m%KgW*G6cao1CsaWF zVJ2VrfcL|A`7$r5Ut+Yu%zOszm-pvTf0dcf?6jkR+G8eP^h>lah5Lsx`7$r5-{BPX zjgs&3O;Mgc8(hA&f0j1*_`bZHZ9F!SA8fu-UfNTJA5yNLBtKYtoPT!V%eGC8Rm+z)xcIl_mHGPkGB2s$ z^$&KQBj-p`-nH3XY;gSm&*gZ34)+h1FZD~?q<&E@JAbFTC!U;}m6EaTgm~0msh)Oz zPpH7QE#|wiW)~UBhiqAMB-+oA;lr-7<}m-?{NKb*AK5jr%y{G(gSajdy3WOb#Zr&t z%YIeLNIumG)!EMP2^IWL-8W+6h2n-vDSbtsi~lnJV`ybObzx(WVv(+q{&Y%k`d_^d ziv1juKRo>g$?UG*AU|jN6WKqM@^j{%UG@)BM)E!V2CbV}yXiWIOScPJH?ubVn~by5 z#)g!a_PBC!{E+dIhnRG^@)EY=ujD&< z$+y_)w=p1PoXwUmWt{z%FJ&a3Y@xi#9*LcQ_}VgK26>ekC$P=7XIIA3F~H>_ewt*Q zyu~t3oc^S=jI+)1rHr%R@}&&MX#SrgzE`s4!Whl_bND{a@+rUKzw-4Q_5Z&!)+6L%lC%bJqAU$e35?$@lJC3fSctqXTw$+23# zo>RR3oSDzam@HqFNx18|ZE7R&}g8N^Jz) zsD4sI)wwFAhaw%W2Y~mJ8i4e-8isEU*Tawo=##+vTb+bdQTI~0YOd;qv{Vg;)Brsk z>2y61-&X{!sB7t1jZ&kC>POY3kUCvoigbZKRSi_NKx^s7y01D!y`=ggt%UTY>HsI;*e7{m1fwWqkrG}_h`Yfc5`Y7;LqYSH2fT{7& zI#5PEpaP^PRirAaXF#7(b5tdjqh0`gK|QA`tE%cP(6`hIRZZ1UpMZX%-cdExo@xW= z2KA|`t?H>Qpj*_Js;=5gZ3Eq=epC%qV^u*nK>AZPQq5FN-3ZC(rmDHx3v@4CUAIvC zsC|g)MtWb>N*x4xkls(XR&7*I(4M-3ZmSMdeL?%`Jl#%pP{)HFuMgAhRcCb?=xO>` z-AQ#*LqUh?!MdyJsYZg1)aU6QN~;B`2hydwpE^=aQ2mguQJ1NM)s>)Es+sCcb+(!U zIz`>C`m1Bqb)eU&@#-9Pp1KqCPIZSmS{<)$0KGw7tIk&!sHvb+)m`d1b)vcn^d@z^ z%2#=64Cokjg}P8(tfqlZQ+KNq)G6v_(3{nbDy4d>v7lqsZ1--SAkxo#;Fl%q`D9EJ~dqpR;Q`Spp(@t>M(VP65O)tl-w^`80w^aHg7d zuhy%NK|fYssbAFBY7^)t^_$wNzEfX-exWw2f7H+FSI}RT(t-Y6{h$K*@NsvJOE*T|+m~wRIiPI=Y$OTi4aqK&$Bny0vbq zn}asjZS(=Ug>DGiP`A{b^!~anXj|Pychl|kzM%W+19hxB>#m?(^})Kg?ylQ|w%0xM z5jtP@0_~-b)JN++Iu|roAF5B({d9lO{`zD+P#>ia0X;+?r_azQ=#xNC(r4*&^eOrn z&|~zedYB%f&jdYFU#v&yv-Kd*LHc|>T3@Iy0=-CIp|8}#^|_$u>i_7+)Ma`M=or07 zEmoK7OY~xOiGBh-OROFReH6XRczv}Vi{oj%2*(2b7>r-L&OX}n&oKi5@))ku|txAg1!Owcof zi-HaM3%wL{Y4B38Qop4y0=+006MUmL=$Amj4c^x)^%&4G!Gs{KpViGkn+08h@ANl% zCFsgvb+AgmuP1;`2qp*1^iTRHqQQQ_kNP{k8gzBAG5AEU(vv|a2h)NT`gwgY=)pn% z;3xf~-Ux~me5OCq(?F*MbAwm(3f&*He{fQ;R)41Fg3b*V1h4B?^huy61!o4&>$L6y z+9fzR_(HGM3qThHOM|O)fo=@iI5;4&IzAW`+@tT+ zw}9Rf6a+K$Og$BJYH)LKzkWbJKs2}~n62mP$3Pzo<^&Jwhx9X`&je2d^Yx>81?Y<4 zx!@7~gnkF~o#4gbas8zJ6!g>Jtze;kN^bz&5PTdg*3alIpj(13gC%;LZUfpT_%V1{ zzovKSmyx#XpLL<$3c6MA6|B+gbT!awLDk?@{ifESI`~Jg*I()y!Fr_X!5jJ=9fF2| z3ck`CbuG|ZLCxT8{hp3MqaX;D>*w^ZdO6bX`fL5It`mHXR6BTAf2b>hRt!wAR44im zy%gy;y-EL|>jj&T_6$DIAL&Y&y291KQLGxfA(0zjLLC;{{ zpb2P`phM6r=oa(7cW?;kA;HnXvB6pW0gOh`!f@46B z2?hp(gX4mJp#6g5gR_HEfJef4hn-Lcsh6-^zmRx@Jg^eNQ0(>*Mc{KqTpH3 zXM-1l_kx#$S3zG5J_uF?uLsYAJ|Datd>*_Tybt<*ur^p9d>Fh5`eyJ+@J;Ypum*HZ z@NMvYurBxr^rPUb;FsX*U=!%3;P+r_@Lli)=oi7}pjGg5@GI!ALA&6<;J4rh&>w>R zgJ<VKQV@pW^6(|lm%=El5WW(=3;J$Y zDcmD`FZ>+z^RP--Ib0on1Nu!^J**aP41WRrC9D=iZ)_cV=7!?3Yw95xA?npUPs*vd2u_YU_r9nIchN7Eu~8Fn`Lre&CK z_6zq7V{?SrH$1}ZAGQt;Hz%6b;fdyeux&WNoMGCAXPAS+1H;qJFmqry%ybMpgcq2H zKpzVK40?r^hgXKZkPZni4yT6y0sT+dJscR`6ix)47!D5mgja;)K*xoB!&AeX!%3i% z!qdaU!m;7ipjU@SghRqx!pWeM!!yHv;Z@-^px1;)h26qS!x5k(!p`AY;jQ8AptpzT zhDU}4;RMhL;jv+t@RD#O=*X~Vcusg*cn9bm;n47yaC~?z=(XYTVQx4oybScRFcqF3 zP6_V)6fOZ>5XP?AzTl-K3p4a4!;gR1^qPqGWg*A{uQ`gitRY9wo%BF#-X)1tLFxAZ7riIxHbT3oS z>|+|4+Mu;fL(|r@Gc7?|n&##pv!7`K+QjT@x|;6hK+prt0j8(vU|NH=HtkJsbBO5y z+QW1+hnbY=1lq~unxo7yrVnTz)5{!Z`k6e?Jaecy#SAh>gC1>;G^d&qOkdEx=2&yK zIoAvX9cWHA=bItsc+lg`U^CoYYR&;Y$DC#UV=gqOfu3g0Gjqd7!V#b&%*E!>a9%hR zbf~$+bV9E(2mQ(%^dKY6XmbUQai##r1am!(o6IB}x0)$9?lSk_m|!Wue`Lb4qlhxxt(iogLj^&W>(26U~{? z@MxkL9!)m4n2Vw-qFc-r(e36oGbXw&y3Je{-D&PH6Qaq{9cFSg)!c1vi>5_)n`zNB zGu_OH=0?-a+~{6&pP3gei0(5BqWjG(^JKI zGiJT15>=1Zo9fX9^QQU7sOU|kcg(ZqC-XDt&t|Fl%2bPLMqinl(KlwL!Spm*X@clI zv&{Treg*y26q>J1ji`3?wW%F_XBtEeqgGJ^q`jj?(VkID(3Vl_=)kCb)EKmJv{%$I z+CQobS~qGE^@wt#4xk;PgQ8T_Icf{qHtG}|5*-%hf#yX$qr;>8s4HmKD2|Saj*I$& z_KgmW21G|hy+C_K{i4CqY0>eZ$4AFTL!y(T{-FJ%6Qc8?q0ym@o*$hXT^3y&odJ4AbU`#ODv16A`k&~M=$h#AXc*|QXjF7#baQkS=vC3! zXi{`-G#YetbagZ(x-+^7^rq;B=&tCNXguinXkv6J9;^K4)nR`h3LcRqv#FLH=@^~Poj6D6`(7kx1x2?7tt!v zRnZ60SJ7wDJD~4GA4lIsKSt|8*GFrk&C%DX{|4rP#YzVDpx{LeC*n_u#hyu9S*<|aSbwjJt}V}y`wOBM1UFCmp`SFaZ3 z(IPT3tgA&9=>CrtbA~L*lCn#+8^7e@K^a$(OHLR1&iu^4ythGmQBag}zYBlTFqH9=0c#95^ z&B8i{M@EKs4{H|LH6pB2+pg_Ans|Cdwt}~M1baJBBvgy?=+vSkfAa^ zKJac54K<8Zlt;&~$aZ>FNBPn%BA{Opox8T^s&}lBw`VaQPj63)mv4ZNk8fb0InY1A zHz3G2teAJ8-qxqi>91|wxm(K)VYS0s%ied5=oSVqv-hM{*)q^_mzEJ#!@9QU&>_54 ztP%Xr&KdBX2KnZ+~BlPmsS~pns4z{1+7D>Fwp~ z9c1yg`1|3mIKcjq=lC^99?1z{w{=Q~8Za&_Affj$g(?68=ipaY!MgB}3vwKmPu{{+1v= zzaVcjjQo#e{SS=0W3`s5QnGaApX{`^uQ|XH?#K!AUsw0WO93?}w}p}UUJs8g|4*`GFi-WIFT_eI< zcIzsWQ64p_*REKnV)bhJURbSqt*Rxfcm(@+dekXfzjXELwaPT}_6RnE%?R(*I=q#v z;tOwcY|Btb?isEcw0L?L4f;7Z=#Lu=zzzC*tHHp}4F)+k=mQ=3+(jSg|K~nJ?_KxN z$NafXANVx<**+GhM&TPiH|qP%M*UHv@Oe&+20Aqg-|@Loc%SQ@nqe0=h7*>wQ=?|5 z88SOfk>58PMRR2SVv@{Gvt)LfCV2FXo}zg&e=$+MP7~$pG*JQHYSb4^lNEn0!{Ms|*{FM*cb+O!Fas2d*ELs$D&ojZ2v+$pS6*E+qrfbZz%V}=QA z)wx?IaBY3O1NATO30|Jr%MY9-Fhah8-qNKBvUq~~>T3=P@CDOl2@DFb_(SD>H8HwYXqZ@++`KwoLsd;|Rgy#supu)jIb+s_Pc z=hs^3s&~z!ZrRel;4g#Uh&};E@%x|ftp;Fhf2IL*q8`A6ByK>@)CbWQte~fdqX)H1 zm#k8yRwcOYyb;LrJfo(CWBrlGLK#~<8!Xb48#$II*s&aihtU;u33zA#C0BEHhZ zx5g1P@4Mgdt@`@senS`K7<{gucp5-c6Tcr#18lEGeHXI;OE+-~pJxF^OSf3|)c^_$ zs{sxmzJ4%ZzEbyt{LQ|BzS4t(vjrSsAj>B}E(hNLSP6b`90>Y64Q4NJn&fiumOHK%_*MfxFwQ^ILgJZ_TTtQ_K2Jpe z9EZL)7jkb(+{Wk0@Cp31B&22W4)O`|3krg%0FC#S{zed#fRm!x8w{2i23cnLg5e1; z`&j(E!Pxlw-v13s4@L~>=w1_FG2;M4|-$_$ff@v(Ro`)Ui{9LlenhA%$in+<%= zFvwB(+`w0+;;XCj^HliCadtG|VpaI){r%R$G;9-W5&r)EaCQtd`H-vKnpMjKCn~*U_LD1LFy9$S>O%(!|H$uk@N7i z7CsL{;zi-}RpWpr;?`NGvrkT2Xn1cD)gk%wg`FLV51SC^XxEDbnoz)%NSeEt0`pC>`zseNIK@PjSI z-!H(d&jM`cX1L9Oty=o^uzSO6zS6|EWwl3j5rLr zezdc@Oi*9W44n1?;3Vs-drU#FYlE`|E6NWJim=tgYWPY6p9dlFVvxRW;z9Vl7~}_h zv;KDsf^+?^Ou|AxG?d9lZC&ZIZ#)8FkohHUmtm0Chw{JEdhSI z$CS90Z_R}~QGNeIzS+WeOoDwVKDUr~O~`Xw;y!$y53|4Ah8-;!8#Rp1=aUdV-026q zmc6=$Rk70r!9H-E3w@MX)!VeuX$$E%w1{Xc84O?8g7u36duu;Z+tJ{Uwg4CP(0%-R zv%hzsv@<{0g#e3m|8WE0uIXMRYCyl2aP61-}f2+^GdPZAiWP!5)02g+L$h6n?w~*bd>m z>C^|<5Mh1%coR@uS~vU|uoM2&NXV1%7cGEE3HXUle60mPyYujaV+ofeiTV%#_V33B z!e0)&(=Y&Mru#3b3*SWEd2ITR>`4iw6_Y-3v=j@C6Kcxx! z$vsgE0Y5PYUunT#-r)cEAi!bi%Zc<4fWy&Gv;b$RFUBDNw$(5C@GVscl-rBbK!T4C zml{888otti?@ujj@QT0efiF0_Ki&iVDC_(Q7PuvIZr~@!zyi*;a|7SnBEB*Pe*XHk z6k0a@6Tb5AMc6$ChyR3}Cb$!Y8z8tscTYc5A*Cu-BRTWU7lVfV zsls0ciP@QOjT<-z9(HlFE)x}y!_Z~ zv7zKrK4*9|$CyqZE7k?C<2f1*;uyaiJ6+_xvy?wrHiTo`Z|6$!>i8VqdCoBI3LnnD zP)vF>pTEpCjQhMrbAdjm7#sQ{$M#EX%&*MH*qASwkFhboGXHzH@q@EGWwDL? z@UWg7<2=t2`2H+g_+Lp!ag0ajddlmiT&C*)#%mUy;2Dc=;boEz;_sub@bk-7^9n0^ zyTq2efAK6ISM#{2A>6a{0e;N7l?R0l=b6f%=I1A_=IO;qo^IxCUL(ys{s`tGUGay! z$cDvy-|S($`NJEW)mz4&Cv(O|f8;n1N_Cgd?7CbZXN+5t;t9q&m1|9or!3#$tT{oB z;g>k=V!s4AhB6;xqpqAUqfG-Fb94mi7OH$hmHA&{S!VB_!}zi#mV^p;!oN+7Y z=V9AnRhG$j#;jx6&g2#5gh!iB>*Zy>Y$tbktX{^zvaWou%*R;PmG3&!ew5zM*KqPY zg%cY5-d|tK;};c7I5laEKG&VAdnMG!6{F+WuwPJb01;aST%SskWSe(-U7h#VBh=7IS=Z1d*`}dmvK>Qzj4_e#GR8{c+rMTD zZGT~`-k;`2bBA`^GF6|KihxU`8mGV4a?CBXN|UKv>VHxg=W#`6+xzoAXK02Bv-R?x zr*ei)EI(5(GYj%7{ySFJ`G2AP_d}-YI4_hLU2Q6tx;(`K*rxyIP&o790?7gLG*2S4a#_EmqPSz6LincwJU z443_vxcrQ1Jjaz+dKrwz{X3kmYG5y)wfT5nag@Csqptk+GAk$XnCq{(_1|VrID^|* zzOC&CexOPdj`6*?f&8~*G&HnfBaZQnyTf_;@-%eJiz>WO@{v5UKMC!2vXK+MoOLwc ze&IbQOT%<~X!tgUKO^-)+E|${{Sb_$U6uLL55c(ALhl6X(N@=iEsu*N4DQ-i*CqFR z#S^-CwbjdazgR3GXSKF^n;4gGmm_q`n>KoR!;g}Ez<5u|+@UMqw$bZi9GWR_Xc+ii zvOk93CF^2*xRP6F3Gm0{TEjTxX}-{*kT2_2@W>n57O?zYj58Nrt{Q7U`}BgJ@SN^_ zKjAB53#!H#t|_+s6P9I+{A0zJtA;;0XLP~O7|Iy=WvAKOm;J}MyQ2>>e~RP15@(-U zP&Iss%te-e!t%XFe(T8vRl~2q_z{#h@-dcmjeLxAcrI5BdzE@(!B1G0G4egetyQc3 zxskukHj4iUUg7Dh9`3l2FU>iM{|FA5FjyTubtA8pV-$ZEH&A_7Xa!#y9LpaJEvWup zZL&VU7|$~=R|hY!=expkJbU)Cm)TcuuWHyLjFUfa`YHcbyXl{BMB2UT>ZCOEX3a(% zN0n!N7|zOGAzOI*>> zA8B)zIQlHH(Km_z@t7bsomk4fnnm$7mSG~i-x5xq4dvTXM~fG;*7AAJ2lHQ6EfPx} z&(iJH(A=l^*7j@o-QhiX(N_ta#ctvsj}GP)V{UQZi3{~I{f=f3k!PRl*!|i>J}b_S zA6F=@u06Sodp{n^F%F)&THRe_IluR4D91QS9bYx)iS_)<-N77V-$iTHaldZhYis{Vz0{HW->vD|G{d1cy_gKNlRtx7C@85reub;b; z*PPUyyTZo49AhrRG0xo2Pu$P^fdA@WpS!}I$1;lF)i-=pr)Jz0o?uHQaxeS9Pn@X6 zF`oZ>ZZTx?Oa51Tm1ErAvyjLi|CA3XRh?rTwkD%UR`d>Ue5x16xMIcmT2S{F{9%a* z?h40UxS@Hkc*k=M4d)oI3nb#7qmOuRr8~#C=gDK*6M3^mxA{bmDDDa$Z!w0OSG?jmJUVcUf15FcKi~L{H$M>0G0xw( zKi{(K883f0l4HE#{V0BL(j&h1PIr#+X8K00HGM8$b|Q{rTzAxJb!43tJPGVCKa1Cu z?yvUun9K8@jN=#&d~i~Ye>9ILn-Rw`9KlL@fUi zY#BNmV+!glUwnvRy`;MEbf*W$cxlQ9YFx@^JjJs}j&a^Ci$wcWvw49<4S3O6Gep}>^LdF;(Y$uk zg~GaWHjf?Hfgf5vSIloYmp5MA%L(^7&|d_uoyqU^8_wS@9U>#}!c}hg8`55#A32Zz zy6zV5?Ad^Ko-mnb-td(Fo1F9a{xkTsIS=`jBsKVx+Y`CR`J24PU!8aSWK$ou zS;%i~8s>yMrhB8dTs4RHh4Po#usovt(|=^R6tV79MbqdyM$q33vNDt-2)W3C|hZmSgPaMiS0l8m{}&p`~bouXQZ< z2cP6tBukh(dlFwUZaTf3gyuI8FwNJZP{v~dha9F*zXPBOE zVEJBo&XVUE14}2)4ZSVZzVSF7$r9D{UlBbXWP3w4y#|{ zN^rJKQRar8kFjxoD9h)7`>5w|&nEHkM}-pxUmBeVkJ{>)Q2wu>dj7vPyb`*=U6w3w z+yzPe5$H+9kVKzM*eoY^()Y-q3l85pN|BZzbgTJJyJf9cxD~<5-(c*qBFHcLHO?L+RxmcLI7D z$K8SxM%D6L?Et2Ag)9pu0$ZNM8^)UL?Et2uj}AS^frw- zLR^V3_Db0YW6zZsaU}wAB?56J0&yh*aV28UXRbsbu0$ZNL?Et2Ag)9h`!(OaTZP(@e~4M#8U{25oaL~XW@hqXQ8iUV_!3T z3^^}|r*OiErw|w;&O#v0LLkmUAfCbrBhEq?{*3Gc#)zj77$eR?7=Dhdi!tIU1jdN7 z5QwL6!ibL$h>sBRuHG0o#779kN9bh?{{e9j0%OES2*gJS#779kN9b)D`w`+G1jdMi z5Evs4LT|^|zYrfG5FeqJF>E{HAjFSg#6gH3!HAC#h>sA6gK)x#gAlGT;v)p&Ae=Db zBZT4W%Jqmi2q%m<2!S!;AOyyUj}VBDaETGuAQ0Ce5Z53OzaS95AP`^Rgb`mLFh+cV zz!>ob0%OD%2v->K1;Q0Ze1X6i@dW~7#1{xx81V(d6-Io4aD@?HAY5U@7YJ7v@dW~7 z#1{xx81V(d6-Io4aD@?HATUOJfxsB?1;Q0J&S-GIBrryNfpCQpUm#pz#1{yR5nmuM zMtp(581V%HW5gE-R~YdH0%OD%2#gV5ATUOJfpCS5vn|}k30D~L1p;Hl7YK|IUm!3> ze1X6i@dd&aMtp(581V%HW5gE-j1gZTFh+cVz!>ob;%7193k1f9FAx|bzCgIbh%XR7 zf)QUJTw%l)2#gV5AY5U@7YK|IUm#pz#1{xx81V%HW5gE-R~YdH!WBk*fxsB?1;Q0Z ze1X6i@dW~7#1{yR5qBUEcOVdVAP{%pgb{Zj5O?5&5nmuMM%;lw+<`#cffGi2fxsAX z2Lf>i0&xdU7;y&zaRllBU62Lf>i0&xcdaR&l%2Lf>i0&xcd zaR&l%2Lf>i0&xcdaR&l%2Lf>iP8e|q0`Ub-7;y&zaR&l%2Lf>i0&xcdaR&l%2Lf>i zP8jh80%OD-2*e!-#2q+c<33ZKeGqpb5MPi8JNN=U-@wKlsnL#c#zB0+r@KeEJ9Waw zot4Zt?uaBt9Dx%?ynw*ixNDMi5l0{pN8p4JM<5VK@aYajwu5*9CyaOjfw6ICBI_FW z6%r#}zzHK>Kwxa#d&s)RorA=PBM`%Yafg6+bm?QoY>uLFSxg3A=zi<6jJg`=z z2y&k-|6V;AJv~|+U6xA6Bcr1E`p|SbKHsDtZ}TS3>S%y;o}WbPe>7TeA7jW9kk;#9 z95G;!=2pIkUjDhsPe9Od$MsFbRzjTcfGbaICvxw%RsXk-XjU`3IIyz}e^a@QTJw0Q z7QCYezmv156P{P`-u6_%kF8VO`*MsY9POQ#7mpoB_1h-o9+7@fOsg7LGl4Be{XQ6g}R!Y8fQ8|9LXl-7q zL|V~d+G*>IE_t;2JRZO5-3Jf+Xi zoQF-d+!uT4`Ep)o!rY~tW1l;WAy0#}UJm20&7ExLX~ZDST+u$)7~^?2@bX8`tq1-& zVT(P`N3^)+#xLe9&E4l$REJJ4qwdIjNwv*r!7JyqipFk*#Hf)u)eqYORsN(0zc8z~ z6D~X^J%6{&j~9zMt1dhKx0?G>OJ3@jhZBz6kX*B+T423;rH^1~U#i{Hmf;U(oL4KB zTdI0YPr>_4D#s7jo375=nwmG|<+)#tYu4$DE~{Z9TJr{({M8lLEAcyZvhdyG7po~6 zSLc25y;d=fDUwZXRmeV0;bYy^DyQvZ{`_`(5j^;yKIV?KBIGzZ)`upsV}0oPj_~G@(O9%w_f|WT{IwP1dAk~mUP)iu?#z3uUN2Qn z+)jQ|gW!arj?Ga=m2F5Y+mU4*byTRY$#$SUjp;Dze%j#M_Vuy&PCZd#R0@4G>Vz_vU$f*U;OkT&<5f zB4E+)&islE;QC)wejh6Z+bxGP{sdExb8*S(q#H!S} zFUv@sGxV*)z9@d?0nwD>CiSFS67{x^NlRE3#bN$cLuaJk7`lY?Ozn5QF`ruLsm(e= zFDvWlx{CBem30kWy7cs|x+vcZTM4Z(Z=LC;S|RC8ZB}SGUMHp}pVqj#8q?OKp7$)R zZXVy058IVV^)#E*0kc!6PX|Wv`5}$@=vMF5KYG5gYU3Jkld!7Y$G%hJr<*Y#mmh@S{JY|-a2d*lLJ3(tOf{>3M+G~4=SI%eBOYJL^E z>$rKkmDXul>}A^3aN{Zet*Ga_kN4!Yvs#7Jhs&eKt9|ly)AJY9R`_VqNXG?sO}1W~ z6V3BxylXx4Av0fiF_6FLB2I3j`jLupj|LIqbkdnB^*CU? zKGCf8GL^N?+7rc7v~MQHy&b96%Xw6tx_zM*VqIZ=4=x6 zQu8Sx^>#+{>FR$oz$oinNg`a!fh!>vy z%4%)*LG2t6%}w=Utp_)moN$L7Db?7h0wG>7))><(O?Xg5FYDUq_iF3fsbEChyaCQ%laoCJ9+oq(6QEy%B!7)DYM?W#H zM1Si-W!iR(lgui!ZStEKwV2j}H$RteTjip`>eOvfyzhX3_ySYR)+Iao{WSh3y%OKO za58Jvi}p2aIv>r8_c!T#0LH^lNApGZ?fG@jM)U1p*T-zBxwXWAHvHPwznyU9%{8~a ztks4WjJ~X5yna*7twTDr;Y%`IRx#d~Nr`_^)01ziY2`-;Y>t0yJ*Ia5JKQDy5k0E9 z_=&B|F+|t$JVMHkJ#Sa?IX5% zS;bRUNU#ezRqT&p+lcMk*33PG|J;G<@E+xD7<*Ri!&{y#s@}VCY&*tyPBF$hgK>q^ zZF%15r`5qdX4o!gjntkm=*gFL84-eUwUtxUm{kQ-kK&Iz*_`l&9v{O&Lx3?I-)astIAK!Vag;pW| z1GS~n*d<=*?yYv5+Kb0!O0R8d*@N#6AE2I0lgZYx-B?@7fuU-$y}kLB2JQLj4hOYi zeFj=xmFYApr#k-rdbN7L4jf~jKkJCm?^219e;u)I+h7X0A(mJ_=I+NaUiJQzEyr#6 zcLSc8wa1-~gk<~uwzXFIz8qsURXz3NySwV4(oMK4++pJ&Ys8*lbvXD>c>>CbEZN<8 zj)*OY$SySrTK#Wq$K5qfHszlUGWeNyFtc4GmNm?Tz9?cXbOXJ0e_^>2yT( zzBt~tz;}?kI!Sl_Z~X(-$g&N^{@+fhCF?BIaQ@JabF~b|tE#;^brcIymJ=fj9agRN z-fE}T|04RjSL2!X#D}E5?kna+`if!;U)!b^3D>s%+Fbj1BSIXVm|xV|5zHfKXBFdj zS<;F*E3LfQlpU%oyeSD2JDW7(TE`7mj7b^~(QIH|acA{S?Z8}m7bRDkCiSc+p@CizG8p6733)$|_fs9Q5_t!7+P_ib#=%VdgGx25hN zQhSVW$`3u*Uj6vKla}f3AL8C!FTS<4FCUq!mWuJ9C3*PEq!sutleUFmys>6c9=o!H z@CtfmdyqB-FOaStZ}Wbg72{F0&Z&P-y{B#SsUT8SF3KOy4&iZK_gKR>RGIMYe^EhV-rtAkewSHW>p3QVX|D;^0k!+_*wTG?wtAlGnw!VAV|-&< z)@_s2LaJqOAMOf|nz1|Huh=PTnc00g#&at#QX4hu%#+{GpuL#7JN}06VQbz^eR;Lf zAJtg};NLFKO`(;k)PWZ%xl=7%exMDPq43_EOY*BJmiOTpH%rw+?6@&d?RdSs4dY$K z#>A&yI^Np$w|*Ss7bCXX%+7{0Ljd+@_& z%W{kZiq_>}fk{Qp86N7b*306zE#0Yh+0%h9yZ=Hxl&YR+>($>We;e#;BVk`_Q+kH& z$dpUgjuJ$IpUlgoTNAy^pT7;HAW_4xH zv2DkD!Q3gAY$xXTws~rs`@LXu zZQ}I^f$e!Xy`p9M{cmmO=!$};_7I-K^NMYAZ)(NBF1=n+LY(RF%65Exdv(UGPTG_E ze~88Fa_}aF3yb`ZuWQ&YwK1E;h5|k!`TNppnM$i{D?4`;zk$tp;GImoSY1QhnHH%Y zIyFg4JGQ-`&qt^YmMyT32K)Rs*yj>zVg590kf;kb9NXCNydTw`1%49?r}a?rI$C>W zxz;JX6ITwV*Ica^Y-e1p_f#nssn6PU=DF@=fbVDOmC}*oYRp*m=Gflc(yxNn_FZ>t zi(4<$_%!vzoTB~Jd4YBLKlzi2dILOE_nG~~&f5)aDNi&F`7>K`&3$mBDAh04Xw;80Jsg(F08=pboxqxzdU8pCZzK)>+-upJ~x$Z^~g4`RA2o} zCUs2I5I5IAy$bPdXUwPthCtUpZdLp<+O5W$e{oswoJBihcwyAGp0)pjzvAipG z^g)&HkQn;#=}rk_c`t1A;mAMH{G(>SkGhh-oZ70e{f;Z8TOW04?r+~~B?R~2)Bd)P z@wNrMIcZ_P6D!zYiOtueAAb(t(rwWwZ9>8BeD**0gL&b#);cNs@vzEn!3Ty;Q0Khw z%Cn|;9W49mJTLhCn;4Ir7A-RGOQi;FJs#M+ZloBIe71U}(##;tm0*l{6O4c2z1&__ z?`I-xoOi;v9^XP#cJ?6 z5AO%#GZDraPUhkj>s8_-p$tANVeHxAmA3gD{5$M9DM|#SRCw|Gjd&{0^iCLmJJxyh zow^R;a;`AGPsbSFtz+!CA24)O@=!W<+$}g^#6xk!LurVIa>9sv5*Q;MN<%!9Ks*#j zJd}obC?|}#Ck^i<@lg8MI(Q}B?qG~<8OC3m z7pV)_-eLST+lM$NjyNU_aZCbnOd8^toG{{;G{iAEVZypQC54Bkh_M%r=LF7MI}4oAm`_c1se$@@5AT*qlheu zJqFk0^iIhqA>KzI-bX{cj}7rY0`Wcq@je3aK28{MJvNLH*P~&KxE{?FM!b)Pcpn?$ zedN3tT#1Ib5)E-BlA|-Y5*v)6!Ie01a)>LDT%5s`*brBub0@}_Bd$asu0$ZNL_=JO zhPV=exDp%UN;JfkXoxEjh%2!nu0%szi9lS5KwODHT!{_wBO1nt9}yTMu0$ZN#D@40 z4P(TQXc!~T!iIPX4P(SpXc!}&Lc2>}n6f`L6tmpUuZU%-1lwL*9-t;vfXZh=ULqqdV#lj1eCp5Fa5BAMw)| z@e%qwIyea32f*0DN9g$uK0-r$gcC*_goZKVAT*2-A0ZGQ;SwXRfg`TLhPVa|@e3UB z3mW1JtQaG{AOvGP{utx=#Tao1R>T)<#~9tQaKs&O#1YsKcc9{YbPuBAhf9D#<*;Cys+MDoxfw%*kD|?P@JKhTrcfj3%J6O@`h#Cugl`9*LxC0e&2OQgT#2s+N z9jJ&qup;h&Bkn-Ob_sC@z{h>!4m89aXox$|5O<&ua82`k3IqFX$jJSY6#1RDJe8dq1 zBVHf~W5f}}V_pDb#1X_}UI1gn5o|*offL5x{sl*%!ZY~B1U+81?4FFtvSlgt@3zM= zsy$!EGC!2t(=yh}InsnL&2iCQ-w{JoYT6m2WuBT|kM;M+S3q8jo}cu{5Is(=$IW^2 zma&lj9H-XnHRpj4&(4=!?^o9RG7j|r_%RPB3u}dZj+^mir(@NRAMG(A?s|FIp8b8! zY1?9`zhvLGua>J&e%tV5P`<{*c-fw3+o5{DZiEe057*Dd$&`?7mH&Vk{!n;N%!flI zWIHi|K_R7kx$)2@Q6WZJZ@q<+MSzv$#MJ~XO4^1(?)Ec4|g64@00yMmQ{YAJ8%EgUhmcXIHz={ zQugOH?)1|0No17T?Pe~{^m%mWZ=T5hx$^-%hwAa1UiR|#?}c&HzuSoA{2TfbDxc}; z#IgKN&@&i^S3bS;a&mm^;{v5W=~sgOd^wamH(CF8BI?uZ^8GMwk$w2bj~}hIj@k9h z{@p}<49^XHb&R*8A2E78z4cFeD*GY-$T)_Z_GXr4;ZJie+w-oQnalniubl((V|-iQ zu|K!RF}zYUJ)c+BV_DB#FK^V>^+RSS(o=VC=_J3WIiHemsCr>qX8wLyA3oG8hhE=S zAO|l3&%NONa-8h-2@k3u-zU>@yg%nV-oq1iW&j1T1L-_Py( zMn1*WO74*3!i`v#lj9-h$B6Cg&aT%VKMK`84+;ACuYhtjU|!N>JffG+w(N+WFVj-5 z4E>k+vYm?1Z`rQYGiettq}Rt)&6mPW&(EDT2bb-+-^#`9^Y5;w<#)?^GM4k+GPAv2 z^Q7{7V^ps@c0D#(8}kPz#(@3M$Hi$rVmjZpukZPec=YZP>3THf=`YyVN$C1AA-)eA z>+Q++#JJBcA$By?)6#y(aWZ209-O|=OO_-2-OhY^Y|N7p4;hiyDgP~$Gy0ir$56e$ z+D^xME8SNgpAIYS=`~{_)e|{oyD{pV(Rum31bcrUTD3>4$oyQ?Z4D}$3CF%8%92Sk7FN^_2fQb-#_&A3u#?HP`>e8 z+Bx}Nd%1gdJ<|837*)PkmXonj&R)-+|6$rdcushd13mRvK9gzL-mq~`tTNv2pUn79 zeLu?!KTI(?v?mhH)YUxa-5{)~_>W0`M{6Vxu_pIEoR zbNjm!)C%yv{7}wFLw_9DuHSuRf8=_BdJek=&)xZ`3d!|(l<}8osh>-(K8Alk>aNFM zre*$`#_~Px+`TPrl z8=~u1kR#svM`=ARZHd%7*>2-mY1DR8LioFn{q%JDs2kRqT}pq_rz?Mbv<@&y+au=y z^edOWKRxT( zGsK98K)Hl7)j~#ZbmLVD+V}PCq227`4DprqZt(w8@1VzpP(0?AoZlGr9>iT>-nQk* z#gm8kQG57Fdl#dwaKvw*-H?Q?YVFN#{PCGy>d3Wj`aa(T@~3o;P-Q#=#d7?H-JY$F zqY*#qFj-H_SkA-Ko?~G=^6+IR<#^d+IgXEyF3`u*htC60Tv8;D!#vgjTCA{CKv@Aa* zo!vhxl))Y!^R(-SrC6k1ZWw9zDVF^YsnktR*C_vo8a2a>R~Vo9Q!LxV>!j#Z_&3EF z?e+-g=bQ}sSedA~9PyrMZv34`TGfcHXL9p%BAIH$anR1FxD@*Le7P=hdTC|Y*E2n} z9v|z*y_cT*6wCKbsYp~Auc$=S+X=b(tA|glU&c>%x$)%5o>~WtcjI09+|=iF%9d?< zJm-s;6uP13p9{U`6u0}^UT=#d&SpNYKmRgL|M9X_>P6tQJD*}Xe%Z##fB*kEZhgu_ zPapf3T94Z~(w0hT^mGMByh61>|#oeiOeg1cXTyMrXr#WBXE!SCdZs}=X z4>C6F&D#gLdCaK1Ax4}q5$w#)79mD_8SI6$6SAMuj>z>r!Lhz&{G}a`<>Wa*es3Aa zx(}Wbr>Bj0RG7U#*BtBFSl{iQNPTY3$Ne%*f1j}~@?Wv91Cr97Z+T$Xd&|r}^m0<~ zrM~VS-$jpOmbTUNjrB^J*w?lD`n~#kboc8YV(3l4YBwI!?xyv|uG~Cy&*M+Aq3`}D zbMraBC0C8uV-xs=`O@m`8}sux&OVPZeSXpNz0=$0>q;rRz8HFy5OQ1B4kmnOwU;2TvpK_dxSeBP*QzB~`=_XGO5^uK!n z@vPmQ_gd^k`eM5JG+)3&Ycb1a%0@;fsCFAHD;z7qz59&tKv;6E7Ne^Kp zNy755WFV7~8YCMjLbA|o5VF!F#FvbMca0)vNFL_Sl7dW1l0gq(OM(6odWCE!$>=(| zovfqLkduU*hNq{Y=lPfiI|}kBIZ193AL0knkK7?|=v%rUL_X$4Y} za$1581UZm?Bo#JILrdqlI6LgW!GME;@wfXoYVUhz?uyMdp3-uW@*@PGK8dI zLr7YdjtwP)NIEtMY)}R^8T2Frn@KXVOdzL{Ol&I2%(8%-M6!TQ%gWAx4o(0&G=ZEW z=RuAEIfk5t-x$!^vk*?fnwEy$l_FUY-MCmMp>1#%b6{!Wp z4r6>4=cWDY(o2@{$0-Y|zeqr-SAqYw6 zFJPa4CB^6@dXW}`u$?|20buJNKzIy$f*%P28AKk!`nXN*gS=1Dkpe6?^8x7tJI7V> zo@51?750S9M4`!HT$589tfEV>M?8U$iukck^ceMH$EZJhOVZML^evf3Uy>E{Imu5~ z(ERi!^lbr612PRw1-~?~bESfi3|3i6ni^zkniPJiq5Y%~6gn2x^gA+{z9)>nB4Yr* zBJaps_`M^;p#5R=1$@P0@)YD#@(6xUp*N2p+=DNANd5u&54j7!f8d+$LbwfIa|h5( z2)E#O7jkYvxB*{xiChMGnOuY4W!MR?LAU}Va1%-tV7Ev?RuJTMh_91hSs|7LdRd4i zrG;5xkS2&tv909?4X&2fxPKOPf?YeqS@)cWF<`pG98S97pn-Kh!^AF zi##KPsXIt_nvZ@U6R8(SFR&%SP%k?ThMGIUQ&>yafLuen(EzrFtc10-lK8N9WC|S) zV?3M=r9re5?8P}?9b~3CAZ(!v;eBan0kFPBs3$E*vw$xkk*AJ+fwx#_04++2gT6@= zKqU&LfwVL&M?+v257kjQ9fgvLv@)$qOOwhZlt@$+cJ)wFpVpv_XcJn8)}{4z)I>*% zU>w@ecC-c9>_tSPb~@They0(%EA2qL(5*zGt~%-i-`1DL(4Mq6?W3a@9Z~QlM$tGr z1iqB&C{9O{LD#0z8FT`S*JK^d(9uFVn=YbD>0Fu?MnIyaI!aGh(hYPIT@7QIUPqgB zR18*^fK?`_nflTMs)5v~KlP(FngwJQT7(>1Y~DuhBdp^Uy)Cix!|ongetkwf z>3Eu(UIuxYPN&o8-}Dm5OLQjuE`f!c31JegMEB8yAP>?i@H{N70m$YG+_T;(0^$<*iAE#l(5G>rzv5VO$)x>6Z(`!!0y`) z{J?+cW7-~e-VXB&t=%LQXiY16Zj(tO505HQ5K3}`%*_fgLjHn!1VT=bxmYmBVD_4BgxxogZ6txv?{%;n=V0p~ zXCvT@@cwnA2+Pivz`h*-G64EL5BBqHY#!t+AsOguNM)eEfDKJf`hjy(HX1ye zvEW^}u~f_qUe5^dHd3+F5T}M``@s83$@W2TXFW&?IGObT91ebaGVr}RLrxEv*Agr# zdrFhCcHrA5Wyv8<&bq)3zY9)Zzd=aC>VU5&WfE3{)B+z+VYR>utWB!JeoR?)@D6K| z*Pu@X#)&|>3fScgtO|TZUs4t9^ha71a;lIl$PAk*Q2Ir7-m-Duj2GvwQ3T;}BHFOqoqJ}E9E%*pyV?p1>Z8CsTWCOnyh#zzbcaQ?E7p=#htLXSE7lBt ztymq9byzJ3t(d=32SNkZnRR15LH1-_;n$P(2HBhSVqGBg2HBgb@MYzcC{{*Er3_Qb zKnR6Yc_o@fv1rzZ^@Up9nAzlO%0bN#s*#?MUtZ}6p*QQt2C*R^hp>Tc02|DPgB;F= zu^0%$K@Mj_;WwPcf{bOOSbsK}#es}tBiKkb2I`H3FbHbKLaDKABAWto3Y!4GDQqUl znQS^6$7ZnEAZN2_Y$}_@=7F5Y=Cbh+=7F5Y=D=?rn+$R?n*?DVl$y^Lvn3#xuto4& z!j^+v&X%zS5SD{n&X&S&Ia>vC6*hT z%K%{;Q%ok)FpyG{l9-a327nv@I+YAoa6gd!K;O=>3!pFOAdF>yvGXwZe?gc8=q%{O zc?fe@3R5amG)V!WH%VzqZHfXJMf#A`rZiwDQbUL$&y?p%Q&^WxSrztNd7;z;S&ubk z|0*w)N+2u2N_?ffRz`yy4Z8bAd8-TsITZByo$_9Z0T~1Ooz|4jG>oK$Fqot_WiSl^ zIfM)+8BLi?gFp@+7Skq{8NwQt)%1&L9msW{LD@{%!JcJ=tO}9SAD{T?ehb1z{;z%P7#V(u&lxC>Z&TAU8rzAFz*WK(2vQ7z+pO4TCTrP)E?_ za0rXp?_lE=f?Nox9HyM6wIm0G)g-4W*t7=Z8aSCYQEaTSlFQ^~I?QrG*u&gRB}`ov zHwcGWC*_Rdu5^+Cp6+4kp=5fcn5jSLQj8L%Tu|FRn zATPk&pJJzBrcXim3(yIe`O^^2!>DXwmqA_zJG})~-BplR*%d&m*(H#dAaxO*-UfLa ztoRj3-vW6H#`-E)^J^flLCR`UOfev1*n4H6@<_P^@{$s!WC2}n2eO^AlNB{(P_}{G z2BTS(H3184R(2>?L0(m+E3d%Xw`H#&w1>UnHR}Mf1MCmwSXuTG8DwTiO;VD>s-6L>I)!3_ z_L754u1o|(mC+zaL+TZK!#=>be1LCy1FQNy$oG(X&R(#$Am76Gy@2m|3GyYRp0KBY zol3g)vuQA1$xcU^c}Kq#LY+WiU@UmEvH18^NzKE2{WFN@9tE;4M{IRqz5@ zun)>&VdE%1hNyP zE-9Cl4j?sGfi8wYfYi!m-)NMm?Y-r|e)?Zu~PU7e!#>0-HJdWb%Xx6~2o=KT}^~JHiSzR;h zd{+7Dh+`e`&15O=O69S(nZ4O{aW7d*9BYYhva2T7P2{nrSq-zoCe_TUiepu?LMCU$ z9fqCR+5C29m&6@t4Ko{IV`G+2EWxcLzVl4Pxz_}L$)^#lG5f=8hsd?7IJ%0_c&o_{ z6M6jItfDwp6yJqgP3o9g<=ZRff-Yuz^Sg-gJX6dK<*|ZUd66r7!?C>CCUGZRMa(h^ znUpgNH<=>tNz0i{5kFtS<5`XU*1T z>%@*C^AT-@7T5~V9Djz!Jn9DT0%y;}TO zm@O9nlSd1)B4(dV6U6-NlWAemJ_%xG|C>1eChq^j^A|F+6-Qf9N8QBx!a`=<p#L-NQ&tLL&G?7PB(ZAY>JL!_*SWDJ|Hx~PYJphjmDKN5&H;_GzL8&*f zr!kN{je+cG3`%{9J&l3vX$)jfV<3AP1KHCU$ezYP_A~~vr!kN{je+cG3}jDZAbT1E z+0z)vp2k4-GzPMdF_3+Xf$UujWba}idlv)QyBNsc#X$Bh2C{cCkiCn6>`@G@D$c|H zm(xJ(j|!Hh&x4oWW&@M`U4iW5n$s52Ua51*CuT0gu#_ajxgle&Ljj9*1}4`XlIsr1 zb%*4dLUK(ZxfYC=Tnk8VNUjUjeZR?d z@&AdvzR7j*e|1l9a$O*~E|6RoNUjSc*M-qO#^ky{a$O*~E^zDGDtOtfIk>m^J;SZ_ z@0sa~tFhG!LLp*t^UNC&zUc$h%DUl4FO-7RUUUG}53cU{c~jHLO>=nIXK@ zky_xkoxwb@|5=AxZ8Ix<4rH&z{*HD%>cFZY248Cv$FTfR9e$lVZo&!ned0*Nx&8M$ z;kcH+_7%QVSOX_l)i#8eeO*5@eTkNp7!v}NzrCrSIry2DIgAVe%DqIM%kPh1Lk|W4 zFn>hXJ~8q>PU z3(qa6fq`pk!`>ZdcISmxin^K^oxnmp0wFIvSnQiu)_e|gPYZ;+uv)#*a9TfVoR_>& zo4n+W+IU7(!;1mUz)I}bQTknNn8V%R;*K+Wjy$`-qvw0UZja>|Z^V9JF|$TOL9xdY zZ426myB^4gu%RzFzZ{h@?ngc_o6#4FiT#ksb>~8^J0#Z~l55I^TvJG{1tTWc0#YW| z0#YW|0#YV@-j!SzE;O7nX**P7(vC>33#8$c$#vmEt_4>$ruCl}Cf7payfEoYm=`8} z3G>2g^+v;K{itzX@$mh{%^m)~<9{RL`zOEn-2U*2|GU)7p#JYxW|g4t4nNlJn$Rsc z_DfeoS;yWim%sb}2|stK=OF#uzx|ni`7KL*6xQOG-}C1OzwG~S_+$P5{lQrG;Qy)pp|y#biK$pa zX)4yHnu|5qCB(6WSPAPco>p0iqlH-W>?&4+nu||uu^PLsiC70BJ}zSItD9Iy?J7Rz zhH$y&x~}+f73-$u3if8=r?pslSxl@j$R}2LRS=(YVl|+XiL(f8D8gijlOT8Tw!gAC zRu=0wTZ;4Kzbaz=Z8fp-yQ=uKG=x+W*hKr-)cRUs{x_wWzOB;!{TazqR;XR;-;XD9Ye2>Z^wM)H1Ob zpPGh{8iEQ5Y9o$qL@XwvjIG77wTP*J;FjXpQmk&AAy&?sileEhxej7oco%W(B36Y& ziFcS5;%Fhx?;uv)hKWzOc+NISyhrsFpK)T9_*k)8yo30J8Nz+V|0ju`4q`3K3=!K@ z@l!yog6kw!hyPkLK3=Rm??%I~hH&|8GeG=w70(L?h&8Q) z#QN6$;v?VthKP5Y%9_%EZQJi z9HWKp46#01u95W=D`$I(+M6M`r#ME5+>a8U(PDM9k61%BU3{VpAwGf%iL+;l<4h6e zA!3*zjx$7A3y3y|5yu$8;iiuI!fQK%#erZ@o)w$iti&(dOO2I$%63;{rM$BLsIhCUhWZYjS{eG78ruwW z*0<`?#n4yPIQCLQy=$Vz&=1wvsaGTYOFyv(xhR(!r!H%xZ?>v|p&$Mm%X(I9al){YQ-tpE^-!1um_%*2_s>wbLMY@W2};d~?*BMh%Ct0Yl*C5l4Mv4{s7&99&#J5tlI>w zH%x@7zdeEF0qYI+`YYwJ|9~$BZnbO)l)wGiz_tlnSo?!J;9J>QKzT!LM|NWL4oIso z3n-Ik_duCE$5(xqMxO5*y;q~}*vNB#piJHY7`H)ep8dHo;e?UcyAv@eq9OG+(&!9qf1Y zhaJwTu=V3pSf%v@8v<4OM#AoO@1WHQf7t$g793mf61F@Z54-YxWnGr8hIMD-jJU_FO>Abj z4N$-(7AQ}7evW0lOa${~aX@*_`rT~SwG;>_7zdP_SFguLe_s#nU1Nc=Q*^Xj-1=3p z>|7j_axcqjk4*tjb7LN`JxepOZKBYZjDwRUqBWKwA0WA}A8e}7nbokl2VYni9Nk@v z^)C1hmS6NYV*QL}?0ARg5OOvMD7*UV+2ktsU}g6(ShKP){EYYje5xNj$~eJ_cX@O~$U7+f&L7CTHzOwR;Z)yMtG&0T@31M8_jao9w#oZE)py?X{WoR$E}Sxb zFHV`hBUfYcj?m~mx>`7SkEr^tUG2R)eFsmOymwT6H?Q`dUilu*{mx$P{k__|eC2yP z_j`S{cl>Ja`_j~7>7O1T=P+N0A*B?+O?_7=6 zCD3rXMnR3~+66VHYZ@ri^$nEiS_jH>-2-L127)qO4?&r(jZkCqZrEu31Pvd%YY|TA zkq%S-^oO4#R^nH`G;n?r2;oJeu=Ai)uv;Gsf36I|>7kop)X`aR06X(EHm4 zL->#Q^N?=78K#Q(DIeZYlGQA+9lBnMg1qpYL`yc}^m^#2kA}Q(7d>ZqYCYV!5Dj_Z zLtpf4=#y}wR{g@;2KtZnnwP)5qBZ7O4#w2y}HF;OMi)%81|TH#qhdHVxBs}#Kh zHrvkv%I_A;!`7nhh933@%J(iOqqnGkIiLJoFA1Heulg_kR*^%4IXj^KvRQyO9F7&* z4hx1&Gvdz;mV;@LUwJzdya=NAyfMT>+0-i$_taSn<+wjk4!^Vp+nS}qeSINq!#IJUOK_Vyp((PtMJ9%GMni@$+^4cov4 z)(E-g8Du^01oxH|#Tr8&K!Uh$+I93Z_gwxAp4rEvVah63s8)A5j4A#RmYuH&&BI4Q z{cCT)@j?J3yF|dps;}Ys-9SkCI0J?RK7-*|!H`~OE;#+T4@Z}TLWy@@S)t*{knSF5 z#M7Q!W;dcz;7M^YenE5NO1AD^zETlb^!NL%~4V%C`Y)@Zv56dxn8N=oI^N$Xyt9JPcOMyUI3ocnVpS zCqVG1ckI~dS5WX?AnaUH1nypa1A*HDz$GjaT|!nv$M-Qt{HIAYnqFKD2k*sz8t=F; z4SQ}+gaU_Sfbyh=i(yi&*MQ4if$|UkMevrtfmxy*DfhUz9DHs*10ORFplmlf2`(Rh z0Dez91Lca5$9Ztimr$*MH&8zCWi@Z`{T0kC?*)|4ANtIr&p&}N^F{&X>Pbbh=E{5E zXypr(&q8&!@bY6gG06wixa{l(tW)JX5N$aQC?{U~t~q1%5uVJUJ7f@cAa+Nin_!vxY_yFb2(@wDwakt@N%s8O@RK1COSuFyv}?pj`3g9kE6=74}~k4wQfEcMIP9kp?yR zSWx3a6|RGI-*s@jgg;OYeZClvr)`G!o*_VaR{7QVJ!n0&nh^$+^RpN%_#q7}j?Mwf z7sa~sy1TZ)sdceHIb_cruzkD*&hMTGln*3afeN;}VC0NopnN7h8EuE8!J_SAZz>fY z;FyLR4yMAcDLz2?UBEiDs-Ftct2}|S)2T?*eOdt>`X>P8OUtI?^G>VaZjA&`W4~=P z@N)Y#&}uks1gpV z_YEeAXI7NY7qUeh@dfNgF`&HiusvR#@g6dpcLd5C$~VEM4_<&T>jjjfQmnA^)yL3g z|0JN?*u@0jO??HLwPtAw? zENgK#1=_{MgBmxUlcnjhF%eExNdU^tU83Da6j=>ceG`DP?&ma3L&uL0*D?qwFI-=Y zZQ<|0TOSN+>>kpL-72p@-&^!2gH5e#QSY8;GxQY3^PY_V9U+fE8x?}}BGK~Q>uJP(D`+RXF z{Jb9xYTU_d9Zo6z>;9TDvw8~oPHZ&XyHWNP-vhnG_kg@lq}*9t7th3XA;T$Of2neP` zc0HtuXBU*Mt)uYs*;JTWKOQI-T@Zw=N^gN+ao<7tQ^PwD*>EdVv-Jnc+X`KUNm)Cg z?1!;HxmQ>c28OQ#kCXnO#+BbBW3g^&aO|NsP~Px01?`Wng@huWK>2-SByw>ty!~uE zPPs7xFYoKd#JW!s$<`B%V-w07<X&uA|#zS7X z`@2KX{lQw8loStY+`fe#N}W!H%%kx@IU?~P!1ztj?NoQ5yzkEkP|a*J>^(UgDBG>S z2hVD4fjS?@isw*&&yTO$VYf-2;i#DRQXcxC8czH65$b($0m@6O)WiF?-olc`ZGiHa zt1hT3_5uoi=meC1BoxK5)(=5nEdnU-J@J|M74Hbv91R7^`uWFs{MFYG+jt^Swy;d( zrjy@7(e@L7@{zA;F#Omv7*VVXQ2sP(4K%p-29is80Oe7WS3zs>y;m~H6)0D=S_$or zd;;eSHGy*Fuu z+8+Uw+edz53(lm#u_2tqPIJPjzD+`7U>yiI|&3#S2Pho>o6Vw3P+72^oy2D2Ul@4CqlPPxY; zJG7keYYhP9{AE_c`Td^^_LTQoacER;C)_C$2bAYcFUjU~-U$t-#6n*9)MY(0t+Nxh zCdZ0C@bBFJ&`(GhVF_c$&jB?apWaX}ZDiRgkIh$AtAtaQ@l1_5$Xb<;G{mXK^Fs6G zq)U14{u8XPcpfF=QR9o+`z&!ptik5*c(6%5y-Zi;nX=5kOiPCQTxy_~@#p4Zsm^*C zrwpg8oGZ(v#QATmlDF|cl}lYYj+2W*p{F%J=CIiWkjq>bIh z1_qYtF5J*S|8nOAgS`xw@l^7THP|RD&z0%QaLV#rnbyUTvkdY49m_VA@yL4U+x2&? zQlFFyeYMjnbwzo2EgP*;x0IE0W&YJzNmsFvd650HLQNa3tOpt1N90GTlenLiwMso4 z68Vw#vaTd6b)NXmR;$#lJXfg`8K+WLG9INKWcw=D1!blG%REq)eM#EOKA9Jm{Yu)) zey+yKH6_!cEbC3l8)aGlN}Ose+d|2w8lRtB!O)jvom0Lj!n0c5XQ5u=`AHtRxBdjX zE1p?N`@iE0B3$`?P-B?~rMzkkBOB{0UjNC=p3Q{A=bGrRrT%2;FC)Q8gh%hV#DeY^ zb7{Lq`V)3n*@})cf$}9UC;f_byA1ZlD>&)n?(JnOg-xT_dip*)H?TX-lU4cKkb3$U zac*^y2gzw7-B&N>8nKM?xJWmBKs>1N{E_wah1ULLrS>j>eIh^QMY&8~&Nbq9JAcm-r5UbYcTD-#1y62ML>} z3wJU9Z!;k`hXh<=i{8#Pge%-f*!NBR$u6B-0F;}Zucw!3$?q%5pT;`rpNRNn_;jDf z`fw3vZeDk_iC)Gj!_SHSwoC9;5kFXo!g{FLA9x78}~Pr+*1(Cqlv)TyEM%#O4C z0b4B-*SohU3+0CVckLY!F3UoBl!!;#9}smyS;l!+gnxS#36xz&)ziyx84u;kMEF;cE@kCZZdl^nS zWvY|@+c1N_Uqoo8gWlbKvLPPIvOZZnaW3UEB7B?( zmo}ZJIO+F^JoFPbl=s|l)bIH+0#;gk;qHPCMr>9!W(lwCwVYl?i<5aE=|bgrxS7vBRGw@U-%%GUMtyK0X&T$_|{ zPOqzvY!wD?+g1WKP6=tCFYGZ9{KU1R#>@9O=qrlrdWA?=jgK{Jpdaii)(gBV2SeXF z=tpjifXH^0RC)OQKeQ9A6JU9%RzUeql8yHJ%&~@W%C~H7bL=VizH66bPkDSVTkU;u z&mrUAzrjvh^Hz|7NBmJ)8@3@9_~0<0tmNU~JmD{soj1u3^HtXVH9*{(P6$=y4$uG4 z9$gU+`@T&8HCD#|G))h za>^oEDR1cE5=N}#Ik;ZQoc!lzr5>(tvCOH5+^pnZ#!p!(ugn8wCI2!WHCD1#+aVYzUpZixQ_pJbchxSZjVLSqQ>LrNxBJ=V^g}g1 z>Tau@C)RY!y!~0rHm8rN@c^xDPG41HB`rf6W#=I`D|wdfrN&C$He4?Sl$CU4o2s#r zf7zyLthATRgBmOCF7vO(N}I}jDs3dwrL45C%(EIR-y!hT2WAeo)6RGq4&Q$dG~xlG zFMTK)2o>7}1LZS`cG{)yf}xVdI7p4Q(_VcT0%r>bKzgFB)@kqzLp;j&m5iVARnZ5+ zg+GCeGa<}YYcYJ5;aso5w%QH1W*g#E?7!8w$%#`_w44zu@l?813Meb_RQ%)jocO(} zl+P)bj7KT2WThTtT9lRY%J`Ks%6OEr%lJP%wb2&1HQX>~qinNVJTnP2Xc9$Go;=-7 zd;atYgFWR9J*sMljQBNorHlvdv@`n-GT2l0Enii;{+dA-DT4Bvm$q7~TLTUD-Q25a z9s2*W@znp$9=3}!JoEV{9?>&8$L8<2)TmgU5|0`e^@-If=c=*HgS44EcOLu`%d`}m zzhfnSX`{wUKBbKspNiTGO1b`tm2yd&zhmo1YjWaIV^d+HoU6u4{4&q~#7dngc~70vy#@|u~HUkqsB`_*_HU!_^z-~@~Os3oyfG*IIV=O zw#bFSFtqd`Mp^leihHhOl=FSG(bm~K4BBno&(v7?9+T%%_D!(U#)UUAq(!;RMmz18 zH4P1Ms`0pIcG@{z`oXOtdl+TqyROKc%}kBUthdu{Y%~;>p5~15k<)hCZFL61!IYg$ zjg{|A89!y+`zqR$2UWmtW};9#*`nc1jUuNLP)O?|fN@YV5ShPJ89h z5JP>caji9W+N~dk!rD%HMtRmeF~?gy+)(Fg{Iz~nZU1rN9k}Qll$CKu=1q;w+^TA$ znl&+`OF6M^RjuoSMuv3NSQ(pSo%bkdr!8}#uc4mPzu0OUZysQ%H_C~>SJBq?`qdW7 zSSHW)Xk0~Gc(%r1Pg&7V5E>My#)=Mu@ZCl=R`eW%4>+o^q6;B>$x)4o?gaiD3xEF{ zo7}9#FMR)V(#p+BK85dpPM&kKQeNTvpHueStkj$E{m-ex+^lFX$aB?L(QuG?P-8_4 zLY7616-^0QC;!I6-#^DDH!JbW_EKXdpR(Q6SShdUKWeP#Do7hOR&*6)UsYp8!$Iap zjTJ2jSuQnJG$mx6{2R-9R%~*!62ELOHCFN|+g**7I+6WHjg>l)eNT-Q?FAW+8WY_K zRAWU8LY7O76-^0QC;!H>o)w$iti&(dOO1){gdxw_dK5;ilvnmKHCFT=ga$=U9SRK! zBpMV*G$@d0P@tkaA+#WJSZGA#*a+L1riMkBpMV*bSIGLP9V{pK%ygI#6(8|DH9zD zq)ap=RCS_>ri77ZG|`SQ(vK!O5=Q#bMCU;@oM<~3X-uny6Kw}0?P;}eqU~U$Nv#%6 zG#`*?J{U34d_bc4fJF1bh>6YvQYM-YNHiaiXg(M*(Ro11MDqcO<^vMV2O}n$4@fj0 zkZ3+2(R@In`G7?80g2`V63quBnh!`cACPE1AklmfK0rB|4@fj0kZ3+2(R@In`C!CE z=K(1bod={$bRLj0(Ro11MDqcO=7SLv%?Bi!4@fj0kZ3*_G0}NI%0%-4iRJ?m%?Bi! z4@fj0jF@OXsOqp2%?Bjf4ysHvACNN9c`(v&r{P4~!ARSkh7+9!q)c=kkTTJEKs6@X z4wx4vnh&Zv@+jsxi@a zKs6>h4@O%2G@R%>pc)gM2c%4N9*{E8c|giU=K(1bod={$bRLj0(Rn~MCfW|D#zgZ0 ziRJ?m%?Bi!4@fj0kZ3+2(R?ssqVs^1iG~AGCK?Wy7bY4Gm=`7*4wx4{^dCJ3%nK8J z2Bb{%8IUs3XF$qChXIKWgAo&51*A+g7LaHxAkkPrqOoAaL}LMo#sU(J1tTW93P^eR ze>4`5Xe=PnSU{q&V8ldY0g1)}5{(5U8Vg7?7LaHxAkkPrqOpKPV*!cA0uqe{BpM4y zG!~F(EFjTXK%%jLL}S5-iLL@tCb|kpndmAYWumKql!>kaQYIP;NHi9ZXe=PnSTJIu ztALb=#sU(J1tb~^NHi9ZXe=PnSTJIutAJ`ubQO>?(N#doL{|YR6I}(QOmr2HGSO8) z%0yQIDHB}qSM0#YWr z3P_pgDxew@T?M2}bQO>?(N#doL{|YR6I}(QOmr2HGSO8)%0yQIDHB}qSM0#YWr3aG|JR{<##T?M2}bQO>?(N#do zL{|aTnCL1XWumKql!>kaQYN|zNSWv=AZ4PffRu@@0#YWr3aG|JR{<##T?M2}bQO>? z(N#b-Cb|l!#za>EDHB}<%nK7;1yp0AtALb=t^!gfx(Y~{=qjKZ6I}(QOmr2HGSO8) z%0yQIDHB}?(N#do zL{|YR6I}(&3lm)h%nK7;1yp0AtAKf7qN{*;VWO*mYD{z$kTTI#K*~f{0Vxw*1yp0A ztAJ`ubQO>?(N#doL{|YR6I}(QOmr2HGSO8)H72?WsK!M10FC%Rx(7%!5LB7y9$;RW z=pGqT10a7Nq2S}Od9w23+dw^<8bPrICiS7YX zCb|bmndlxMWukk4l!@*EQYN|wNSWv!AZ4OUfRu?Y0p^8?E`eyH9E}21R(uA9KQjIk zXZsNwY}7d0m)u}OIa_DI5U$4AIs*oqzhmj^C*u_U+DP;P{*jeD5KREg&C*8rvm?<5 zFk+$)fJ7evi9UcKT*f1G0F0Pu0w86g4`7HV+b`aTiT^&TG4bU$*eK;CzWhjh`H}eY zBk|=oaJDZ$5?_A9x!Jz_NPPJX@hEvF{`-cu$o3^i#Wz#77iHqVk9lF@zmIB6{P&SE z@!vPpkCHdyzmIB6{P$6frGKkZ=ah;6KIWCP{r3&_f5*gsAJtg#7nawu8WUfBB)(Z3mmi5QzbX@7ek8v9Moj$o4e@9D!W%L1tggt_9&EpL1R0W@YS?;c85L z;W4kA?F(VY5@jch6G4VausWI_AM>QtC=cvZS_gtsO#P?jM#>Dp=)tLC6qZ$+6bDbI! z-*cTB6W?BOnlFgGVwh}%Eb2^ zDHGpwq)dFzkuvc;N6N(a9Mzcko}(HQ-*Z%B;)jmJ4_%cNUuoedj+7NYWf@Lc@rRY+ zlocOU8BSU8&6MGazocZv7hCwD8?llH;X9m@AIXaUweUk%WyOzM_@Sd3D`k;!Qda!6 zWqv3tKG!l%%8I|X%#Ru?Wf4BSIc1TolwJ7k8nKdR;h&q6|JX}^QQP~%e2&3@%xr}R%0c8LmU0a4?QPc8Ls46wwD?!c@zHK zNLfi&wy7E``Il{~#!7q1JgBkK?lRA6thA}jr_x3;UCK)P$~>#F@*N`l#Bt_-{Lqb9 z@x>NC<49TY#TI_(sQ73LKXfF1=%{>O$@nQNKI1Y@;)jlk|G11(u@}DKIdKY~aU)jZ z5q{!GS&2vZiRZ*G{Ks?3CF4=bD_N-rnHFWGyfS{Jj4~dj>@t4h)BTT}?RRXj`8!tp zqvgG%8Y_O*(ngJmPj^mQ!oT>xINMJ-C(nwF60XEA!_`>Hr?gRH#Xnl+`JY&+6UFCR zu~)(s|7aPm#)_Y{v{7RvewpWgVx=CGJS+A}xDvk%S7Rlg(ngJ?4|TR*HvSW5`(+z! z{*D#jXIXYNR(zJFjT*~5$oPeib52@+$JxHp1{*cb_PI9LsByMWx54J`SSgFlgBlZ` zZd7C9(~WAZ)QL<>jfqb;r%ZghIc4J0&DEIrbaTqYr<+qIKHXf6iBC7DOnkbz8WW#x zPMP?0b2TPD-JCM<>E>!o{JS}2;?vF5nD}&aH6}jYT#boOH>XT|y15z?pKeZ>_;hnM zCO+MqGV$r=YE1mQIq~o2#HZVciGMdI{@tATbQ|&HhZ#`S+!4CBOMvI255lLu4GsKu z_!emDS<}F9BMq*$C~x5Q1(RV5~K}eD5=Pt_-iyCBwi>*gxnx&%iRCdcA!N zJVm6nDzSxuWm>)3Ha2kg52td%9|l|zdJezr7yY=M6VHqbLc2aY{(y)_IqAyqjO_eK zdnKPT9wq-Wex;1lo|@{bWtVqN`|tQUySyFN71FzBm$$mFh5lN0c@wV|(<||p+FV?J zD!aTgyjFI8Y6$zs+2xh-IA!Nwrj?RiUYS<^?DD!EdW%ZB{iT?D9(c>)H8~@hJIkBH~xdDBD+Zw{{6A>s*#svTT=TqP&uo_LZ!( zuVkftm2la4(s)49;PG!fxC5zhZ9{LHJu{CmttetKI!qjPEarruq7(W`E_ zF1Cu14GllCCX&BT8-alpH!+$A8ZPrC{)RGdDmGlrhNOigP8nZLc}bp;+EW`+CrIr{ z9io~&wGrnsQEyDtn~^;Y7kOZ!-ayowkqr$OWnrS;K-8O2S!lSZ6GNT;O}#1SGBq2L z7LYg%@yWWDagsa(wa--#Iem=U=d#J^Z^DN5E0p<@X^}bv(iXB`<=D`3MSf`C%dw&1 zq(8`V<=D`0SyuTsnFke{+zL#S|@|?Tx<=Bw&8udMzA5uT^x>o8__8%IL z(iSov+7EMVXgFz8qj+dI=^KW6Qre4#%P^4-sezWe4%PZ$PJT#Qx%**`4asxvewbrJ z>IbMjt)Kt5vPJv{&50$)HIJ`{Lg3Se*m6=`_!?@%VaW|$spcJZ`)f^ttw*|IxN}kX z`RIynbg~y7OSxdU#)EEr(eVwZG#2|p;rlHOobOc~zMd$@XV)*piWOU?i~SM`ZO4uT zw(f+%{sB0{%Og&Pw%)#=HkPo~82E{QIL3Sr*OVQgNd9icnae4ZQS_Ni|) zJ`WFTZhi^{&8w>5{<;}9b?@ccczQ9n#o)j-l|wOQZx<|erWmX~cTu+{^lfIZv19p% z3&Hr=z9BZNQ4cnbvE!bR7g)h`FT?d4f5{8)96N8w=jc_hb?rx=)O>vs3PWDnqxA+2 zCP(bbXs|Hc&Ei5Z_h}o9nRPYrHh&Gsf3*UC|7aw*L>)7nd&d0)&zv^W5T`7wD6@fA zxH&_k6Tiw{d2Vg&TKZSnMIE553)%noolkN-)>Xq>&ksS$drAgD!_`yx^q^K6%HN-k za+zJnm7m%af|P5w3B|$Z?D^CCr`;$Y9TyG>u93QA(NM^8i6qjF7hM87i((GT*|My*W}5LW1;xB7P#m; z@_yIz!`CcNyr2IRpX;!Jt*NtGm$E1n3oo~1`^MJilb24w2|>l#ke)%<`RQaH+_11_ z@BEQ?vg}b7J9@f~hTrJl9Tt8q0@pLo>dxQ3tSPiS3|;oiHWync)KY~d3}C^^~@?32G}uHE)#W#0~9b~pUMp@JlgIl(euZ!e%Hsr%wG1`G1V0oH2JI<^x`t#)cq+tRiQOl92a}YR;USk&fjCR zTBmEy`Uk+Kw0IsnumPkrae&c|TK@UX0+u>|BslDt!5ev+L;l^J;OK#IykY72Zlmvp zW4}N5yIq_e2EGG+<4QN%o30ZUhG88|5q33v zS5~W&<~aOYFYNlr)9tS~AM|~y=Pf2q(UfQq1jP2XutH77p!=p0?EM{cHX_;|_J+ptUCYB^^TuA9 z9i}6)7VewIDij|LF|%fH{byGg{-_#gGMgIl$IuYKPL0{qF&%W*<0j&fg{@in<3YMf zYxDCf&HbTwKs-P0&;xHc6okhkY99ElfvjjpvVEkN7qv~KXT zXBp>@H{!D|h2U$C-(9ztE5!%22*d7<&Do;a!RTRO&HiZA5=xb;jOG0fX(-2?@`l}o zPO?iT5jxguTIPIzYwp)A3@NW(bVGACDN<+tEexLDG{N09`l0szlq}ns-eB%~lHc~9 ztFbI~mg@t&ahiQw*XYr!ndO=Ge8QN~u%<{$m|5cl*Yy0xCdXdl8!WvrZqd)Iv-Ntz z-L7Z(p2Nw^*6Idd_;5J3{dPWcyt_9WRc#=ff6ov4dOAUsbZ4Aa<5;G}Jx56NcZ53P znq7Oy3GCf}eJ6Y8yUnae-DLY_w}IjzPS7dV8QbkTlK~dCP^al;%(jty*~2mH&9iY(qk|jTc-28|!P;4= zTg7!+vnEt|eUDud_c{&F*T!-l%lFLca+%*;{FG%nw1(>A-O#sb9Xxxwf9CyV5pEN% z%;F|x#Jkr94Pbj!2PpL2o#)$q#^rIV1m3t?AU5|%*KG6+fbIw5dHx=IT`TmM!;dTr z#OL+R;lrj*@OX(oUt+;rzpRbtYexm*@$(B_T^Diz?Z_@8%^ zHBal6a4l|9iuY<6h6Uz_yESeVhQ~^jWWL^$AgI+HT`kXPZtWh;%B<&EhA*!dh8<@A zZ8zVb?Z+@qE00AU=qnh)&_H*@(Z!>y{3!4l+{#ib|&bS2LisK%e0`XwkLacI>KQs%E<;|BicMVuDhda#+#E@^UaP~Q z{KxtrqHIEuNPWOr$vS1l0suOjUUg-qWryYINaCF&}FnsaidJ#4>egI5scX7F6vdnz#GKqv zx4GuDAm+4&@R~C;SYUf~?9#A2E4m`Wt#Z`~ylu;1d~w_jnNJNYQL`cw^H7lUQQ1af zjtXLqYG67??YMq6`(s=MHutF?X1KS3=WF+|apS)5t`+;hZtK(RR#p;!wS5Gnt-i=g zERWY+xO>{|NV_ojU~0u38^mbZT?mDEZ%dxPSSWN&s?XcJc;?oAgFh5MvV=Q@chZ;WF)sF8g<&BlfonfsjMd{ur_C=m@j=7Fnz~?%y;Fd`mW_o%IUiWQ>KLby&xQ!2Z z#~+S(G|~|dSiRM~9&U@}_uJsvzwYtFy>4-BgByIDl?OU^8^LFW_v4l|{IS5SnqXSG z3BK>th85*M*}zKOG0H!X*}s_0ZdCBWZFi=#5f*DSwhscZ)W|rt?~pf3xAEcos`_Dd z)5~mN=O^4GwKX;^UIAK*IpPiNT|U^`9(=hg+FX3Cb1kt^moF#)=dF)tQxX>N*rbto zpU+^AXPV<5OFN~ZXCTm$S_)B)F-Xj#41rfj9@Xn0j%C%>N5BWuZlAUL=pg1>BK z%36r;gx?PA;7$kbvg615!6jQC9))*ULPkH>R%{$kZR!iFBDb+h_TzP*D}+JuePvkn z8#}TnUn?1mtDJ*bcMGPYeEY&@Ua5FLXd|A@gxnv7){Aej_+zVeN!8YJCre*ApU;w8 z2kg*=cM65;0oMHIQWO4K?~krYvFyy&a31w?G#2}PCd>HXiqoG~!#ljG5$CrJ#oi}s z@zC*1}q;21c^) zt-ookM*HK>$aqHQT8>Rmxqavm28T~u@&+poxI~vK!C&}?q0QWA&9^h5knz-#N3Rco zI+`YYYn!KTPb*b-TPWsZ7o);3)yp3itX#$?iTUmEZ{=Z`UrlV@;V!4+yLOm8l-}!# zpNhWG(Rsqq$V+T?`V;HL9?zn%V^&Gwh^!Sz*UFglKJ zUX`S|emejk#l|r@E^iI{$+rI19p6L*v9aDY!TnuhTwk_5qhmcCUlP8$qS=#bxGAP7 zOScQf7Tara^AwGSj#)o;gyZWoZn{$5Zf>EApHvU`O?VAqe5{888>s5$GJF2ibyn;7e}#8?kvtmo9mHu3{I^{yWry;_9R zF}=>0iKu(ul4Y3}(NUWRZ^nRgl^rbo+jt$FC;Sle_CsRc9c?1ZMr|l4W|m73iuSV4V;BCGOj{ zYR-?g)BJU25{7B>!NMA&MTg_=yKWyuZZs=I(I$r$r1a+IO1XPeMRRXvk$&wJ(u-F_pnGt=QGo+ zziBLe{4p{%p3(V9=KUTpyjlTlHpGU}d5-SlMDPf6=ZoCOY3MvNc(^$}nbQeFz5E%S zud-ncu-V7@c;k>eqw~)@!3+3<cquP*mA^ge>}&*bXps0FZD|3o*aSx+4Pq7b?rOw;rh_b+`NOYzao!|-`P z72fpY1bDh%eCH5*dQaqXX}fNYM<~2)S(4MaXyIX>xQll`uyFs4C-oVIfhTUUO2=30 zIu0D>`s~lLyr^v$uC!0(E!=#eXL(Dbd2etN5ofQa%=goU+POYAV?aJlDCk6YVvvzV;qku1(9c@l)*6s}j_bSz})c$5*-n>_4 z_oBCTX6rlZF566k+DRtZs(U}M5%c#j_usOvr#Y~V6+)rMgl^cTqy-#4dP6sU{|DE$ zW5=?`7lNT^S|g$9SqJi6{ey3uRgl#_9j`O590q}&pO}ZuK$tzJvfGivRm8m19hSFV z?&3WEr*7t#V;YxTp|ESj3O-v`o7bM_3w54d;tk7rK@pSgZcjS5!ld$?51pJ3jugMn zXI32n@xyAl)^_iWhZdaV0}HKV?lUj(aAz+VdHQC?+A$+=$g)ekVz&hv@g1Hi?w6Uk zUxp3yR_l(=55?1`ELn`WXInP71gkSH49;ZM=RcNBz)5e4u?y{jFmccnKIPC4x9|Y# ztSaWk*!?MC;+f(|T(JKzb4rNQQJZCFZo0f_*qH4)8vy)z8h>3Fvnv2CHz^27Ye zr|0bKb4TFbj*yaOxc^0Q{|j=C=VxyV@BM9{ebfV{+3AGSx;n$OU7ed8+kcz=QS>I8 zHLi`A7dYXeXlFQd_NI&9(vj@K)d9@)oF9Y?cfz8poneUYNmntZxQhEnChi|$S3*s! z*0~8dM7QAW+bqwp8abP-C^8;)U--!jYP!Seq5kr|GE>}FGI3uCeSKs3b!#7RPM9w5 zPcy{*DHHdn(AYXz_s7Ek@b47IZ+>xuw(D!dDKYNX?mwDW@EF5B9~%evp1Xk+)`lx@ zHoFGQy27nXKV{6cHRuX!VArbJaLn4&wbh;S__=sZIFogcS(nhj$Hlc_jkT#u&A#?n zc8x3aaQ>{Rw=G@MX?y^@pA#?dwKK%MHWT;SP@vjEW+k4XjTHA&uO~Ii{I)ipZ5R~@ zt7n?S_{>gF^^`B4R)S}i@R-9EuMl~j-9YS<;{g7X-T1NP5m|js%wm~k$HSi$Qr(7M zpM>SSvUK#BTYu9fnGFjSW5=Vyzc-`48F1i5gecU*_!n&du?AH~S zRJ*RD&*J=WrQ3IhLagq_FrYTk7kzNaEaahErf7cFos+Sstt%Uz9s-o}^$5k)i)!)n zp6A>s|K2MI%*C_cLK9kPD2K+a%B*9fVOti40A)`{6G;Bj8((fPXI@n+Lf zHNK}UvziVq$6k~Zd3(N*g;TRkR6%Po`O#qz0k*YC%cn^ZZ$c zyM0OKB<5#3zL~Rm)BK@@cm^ou5*gwinTdO3NZUJ&1s5F+n-$wxLX1mtx!l@wLSNpb^ZEnZ25kmq&)1(L9v@Zy+%h_@< z{$`5j08BgwfKwko@T3ww;mzj=zDD%tb_)|&-`0WPmzJNeZ0Qdjz2bSZk(1zn^&Q>a z=qR^7mZljkcx}F~TPV^#pcX#({6w6d(T260WT6xHdR)A}FkH*;#(81bw2NF`*W$jL ziFXi&aNTF|U2^!8A)LN{k@ukT|I%K*hmrSwl0Pl3jitnOBDw7f20xZmgU%nz@WYG3 zvj%1bvyWmQs9}3LyL28B$xdtzGUCMn;V^VrgsxVfI5)~&5>{otnGwk@tP29lF5|;- z$BZzIYn$a+lv`fT$CIyL(tVs1imqL%;q9f(Ky2cjS4dXE)l!Llbj}O^Pvz3dFX+Y zI+eo~51(otEev-peAAk(>J$c)2Y;xJL+b!MFK|m|ZRUyA_X}XSc88ot<&0BPL#y)c z&nAHLL2FL!>6qnnaEor*vhiSTGMPKNoMi__dBf46>oSYZ@WT5i&aqWiA-bx5x0qe` z;m~A~U*?C{erVF@JhQTz!>@O}!&+S!2Gw@Yc3W~ig@2vu#%>N7jaRyN#IE0tvArX& z8{M1Jep@!s78mJku;l%FoQ_}R`7Q2aeuLZO_dq&+X$pJsPlNmMrd)Lpqk}=U-+$3OwQS=eI^WmW{7c0j_)2#78ymayphZ z8fy>rcDrJfcs@YKGCF3>ZMIRjY+?W|i;HJ;%u3B%zz3&{MDy?&?AmWUGpgkrR%vA* zEX*{=$ICn6s@=XU&^I!xy%@7ri!sY;X9Mg|%>k1-y0IeN-)npybOqMm0?+1OlKI@G z7<)K148{(xl{NG67&vfa7r(H*v)lE@W~|}qFrfB#yBuTfKlO%5p6hs-3M<@J4=coW ztPcYkZW-?j2X1a*-QEP~F4hQxKSq~k?!WEGqHN`{JfrJlS5_u91SnT+y*%@VPTVTY z2?5H#vtV>u8_3GLX>^p&_*-W+7UR)PF&_PF@s;=4-WSR*EyN>74aYBoud$jxR_o~4 zbhTm{zxiMc4EfUb24;N`EC86C4~ zoqESktn7)UyG1fOX3^&)bj;Gum;{YWT-Lo^5N}X*q1Q5fE=r$6(y_^DVR%*ppJ4V^=U`~wf#KJe)v!%)Syp&= zXO{)Ok*vX{AfVhhIvkxE_-dkRMrTou85jt6(B|2B+|V(aJZ|L6Q&3D5rhU*Sgf7d?Vt*YK&lYvA0>h#mv@f(sMG-;LvJIjsN- zIvw_3;mWzo#CxgtX3ijf{>w(h&4eAQ*<}A6WG=g15zkZq7+{zui1a|FE5oTxb~*-o z%Gr7O)uw47Fk`zJ6F=Nfyfe-mGl2IyHvvC_11s4u7{qVxAkrEu(kdmMy=2?6muVqb z8~cLS)^F@$p)WehiTnG)go0l*j}yP~d(B$Hqe2gu_%UK}jwbsYF5{3aFdul`bgRC!7 zZ>ZFptS_aWWnD{_VWJ#dDZ4DcQYWOoxGcX?pW^TG97COxa&mdDQYW(Bm3kX${eZVF z_<*g@wnSM5(O(Val6COgjx)P|_0|7`%f9qK*n1B!DT;P&xVyT$s=CQZvVsIfQAAPF z?lhPHK_mz&NkpQeBA^5b3nCynXN*V|1(CG7Gk|1~Ac!bIOn{;Ys02ay?wXm}eWuy< zeb4uu^I!jYFQ}{Tr>gp?TwUG0Q_qb`$Mvx4_d_chuU2Dl2IDbf z9{om3pN8XghEb=VU3mDW9AWp@Tw$-oD?a)YR{cW%62c2lo1v=Dw2X9r_e!Wmn^4H$ zy&}2SC5Pz!g?47#Q|MQS^ed$Mh4JzA_>eZKG4%Bqs=D>qs`1tBQDdk2pz=+gkRy~T zFy?4;H=l+4pD%}RpWV_A4Om%YV(ynCts9O`XdC7?evcVOSNwgAjc66=cG<-JmYKc> zXTWOBHzVP+Rtf3N@l5<3=R?YO@1KxP_xVl@M~CO2UbSv!TreX)k7^YSzl}Yi!@K5H z;fgqGau#LA*BQ!<{cF>4;hLmMdBYOlCMYwc%S2rndmzg9N%_7mA7%QwOnTovzRUD0 z&)AUq<-4k1>R)Uf&zzkr#0RcSt^T^|S1euSQ+i5Y<dTFCMfY4? z&YOF4g`XU}GPS|5{%U{8a{obD5#0tF*5$|Rjp}4kZzQXFb^WT0dk5loZk$h9a(Q4v zo45b+Y>k#kUlmQyoNHaLlHtn(BTqsnmZmT^K1la7(=+dVeO+nxZntnvGS+~zjcdw%9I&YaZyH`Ocg7!q`pSE*&;g=%!9EzpW z_Rw}djo--Y&+ik>OVf|kgDrZujP%U*q`w?F)MI$)$5p$lja~P0q}=1fb>0k}Rs~*) z96CQSGzB`UU8)bNepP;~4=E9|ct~2}38^o@{h{szb+3m!jHC>!LceooXC(y_tRA zU5%zgKdk0k=>>E$#-HlwwSo1c%AqEycs&A-h{aeq-V^#DnE8DMcqRg^#;q=Itz=84|QqV%^$qDPIUZKo-4~6uXeh2 zO8)vbKUB8RE99PSygKiS?{HQ)HI#oySKnSfJXGuXcK%o2f0Aw9QTbkq9I+>bB<6~S z1NBDlchp-wbAB6Fr|aP#>W$~kTnnmi{*wNYR>R!rKBoJcO?vcxNqcO7zjXfb1qkZjR=-u`XlHg+I?4`{_C&vbf_dY82?x{TeUAjwGvB2% z+J9Bv_&7xGf2|)ZC*GH<%8&Owt1?wzbeXCzSLKb5!&T|%OIXe0;CcQ1F=ahG3%;mH z!)h)t4iPn{6Iu`Or`yMpm*HG64VTS5z`y&Wi^(lhR@S6pl{c$#UjJ8r|A~=*C(m)- zO;S32%RU(?dgiZC&n5%>dWVlE&-%!ZrlY^Unrl*5Dh;c=7^`G8@1)JCG~7Jb0RPLQ z7n3*RolNOxwZ=%jX-X$PN74I4>ytJ|b>j6#NL^QrTfF^Y(&mJ5BWA8VGPNfM#nHdU$ibgZda{C(&n^Mbq4s~ zm;W`nYTc4GJ{r^C|E*}Q&ba_zKq@^h5B+6-4F>fjRMbUBX2;T>dmO1yVi}%gu3*#NGw&4ca}N<-2@$- z|KHKkx^elB<>0%zadoPZ#rjFm$=5n>WK5T)fp%oyuT1@a)bV3wCh8z>JYBU=&%wW? zlhEc^|6=(w%T#}}ysGcfevvjO=wzlx+oQ)hL+26eSX%G59|+aQ`!n5tD&hSif6v~2 z=dE?3`-<8lR0%&6ZkPW4_v=DsUhV2%7%0Py-Dmv4r$0&1LEaGZMuK@!R}$(92kSza z$tW`%FEfNPBf&CnVaJk|<3vNQg0DpAo<{15?qPb*q&nG!Nxc#DMc12cIb=+-)gj}V z(9VBogC0-S20fnHmQUIc%C1b(1~sQyJ_~4ic5@n-uZ(iYm}lDtT0RYvF<0wHk9k5l zr2H_Qcak?@zDR!~WPHM;UkPJN?6Q@Oxv%W))!Gi&uxFVLJ8sPtkdT+AQyY9*K20a~ z{>p|uqb`~r-~q@=M{y}cZQxb=X*r}WWy78&-vw#2vSFv(1sQdbF;O<`y6=h)LEH8F zv|g%Db&yxv{wL_ubhJTpP$cNkbYkNlweQz+N7IoOy%ro3qNDf|veY4UDgFe_6`;mA zV@yb!6@P+mbHaKceGl>;WRye3O6lmaRpmrf`&9dBIaHtO(7b4mp2vi8Xgd0(*MsU; zMqM;rjjOsAAH%G4f@?2Bhtw5Zdl@>U&B3*op+owcu=Z%1$@pZ;(Lr^{+^IE|Q5Q`o zbDE$-)5+RU>nNifnjTnVS?L7VVTKN=E4U6bbV!?n>o7xy^gXx^GjwSC|C7RmgCasW zC_;pTB0@MQk_qobgz!#;2=7FM@JJ-lFyWC%qhZ1+5g~SVsf1I)C!7+=gj2#NoDw0z z9T6ei5jF12FCg(#M4HKRRpN0v~gHOYR z=OKxP3Fkw4zNs_ABkrvb-ded^gm6AY26V8Va;e1FUoDU(w`H&DMoDa!_^C6jVK7w zCbpJ7;e4Prl|JEo2od{PpKv}T5!+awa6YK>z?$>HCmaqT!ugO)Y;S$S`4FP|g!92C zoDV+Xd`Ksp58>IFoDV+XeDH6}FyV0UX_#<0B+)S8aPVoEa5$vVFyUuNqG7_%kVeDw9g=W0_=JZcOn4Z4 z!ov_I91A|-SO{HzeM`SkVZHb(h0{xI^j!-QiYL^u|b3CBW+a4dv~4MLdM9fS$TLPD5uETj{Tg*3vk zkWM%j!h~ZXOl${~9h2s>@(IU6h;S_UgkvE@I2O_f$AV8d7SajF0_9MuPB<1Q7t<=j zu|T<%Y7&kG%F&cWpLc{~LD`~et|XsuEKu$vpKvUM3C9BEQ}XG4PPh@mgkvF{J|hUn zLK@xY3CBX1mP4N-gkvF0pP__5Aw)P9(g?>wi0Tmjgb;m(5{`vrszW#ylIe4qa4gg$ z9198W+k|5wnQ$!lgkvFza4duf$AV8d7FN@DIl{5v6OM(Ngk!;{?|6h`Aw=I53CBXh zbDwZ5gz5VO;aH#?P(I;Upj=Wu;aEtg?{$P@Axz&73C9BE`0@$I0_6(x3CBX1a4h(Q zV}bIvCA`BE&V*#bv5-zU7JS075Tg3@8-#EyP`)ppa4duf$3imUSnvtQLc)6{;aCXK z?+C)NkW4rhY7&lx5aC$x3CBV*;aCU}js>4^EF=+*g$UtTs7W{$B7|dM4dGab5RQdp z!m$t`919`Bu@E5~3u%O7VGZF}h!Bp22;o>rBOD76!m*G_I2OXh7PSW9SV$+fs1d@k z5GEW85yG($Cj1F$gkvE>I2Mwr4&hjc5RQdp!mALWVZxt~PB<2l3CBX1>JZ!22;o>r zBlfBh!m&`3a4e(~j)j_pV zV}aVJt|1%?)J8Rxa4bX!#{#uWjS!9nYL^-z91GMgH9|NRLc}gLjc_bbyVO*|u@E74 zsi}lxAx!L2*AR|{bi%QqzHK$fLWFQEP=HP z5Fs23VPeM`Ash?Tjx|C!7D9w$AwoD7LWD~pL~L3kG@WoPBomv~2;o>rB6h41Vy_w@ z91F?BUNu5E7O1^ygm5f`h`nlra4duf$3hyhSEW1~;q1~0$3imUKnM{Igb3k4NC*?& zgESf@yay2)CcFnpG)y=Ue8PcHlW-vTgacs>;Xp_y90(D@fe;}a2w}p3kVZHVe8Pd? z6Apwmgag4R90(!8fsjTx5JH3lA(e0-B-7^(;XqLQY?|9ALO2jYgaaXsa3Dko2SPgG zK!^|ygb3k4NGH4p5gI0(2FZj2LA~c_4ulBdKnPKN`b;4l2qD6OP?K;VBohvVnuG%( zL^u%AW@K_8_=E!?^p;sAGH$?Kk!NzBNv6*i!ht|Jd{z?<1j^yFhHxNI4xd!Qfe;}a z2w}p3K>2(kgad){`9uf@g8GKi90(D@f#B0;4dFnbd_Jj!10h78HG~5pnLcaiGlg&< zP~M&h;Xnuz4ulBdKnN2Kgb3k42op{NpKu^V2nRxla3DmeKH)kD5e|e1O(z@($%OYH ziG~U9L4<}0??Hry3GYEN4HFK82;o2o5e|f8`s^Sa2$Y{ELO2i-o*ne>5A=CK{}w^G z4iwL==HyY&6wQqgCL9Rq^w~i;5Yh+-LOOk>5DtVeeWnl&1j^eJp?}Aq&lJLeknkL# z&yK$a_l+D||469h<-Ey+qX4{+85{*+!b=dLVZuw0M8kxaAdQ9zFF}Nc)%lOuzl^ht zfpe6xtIBp-*ebX3_&FFPIBsB3-zYA>vK0W?2Ah}aI- zB>Vt=+?HM0s%u;SXqfN=Xj^~950DV1dE$2MiXR}E>Jd%=pN5ql&7;GF6F_qkB-E*R z2k1BxTYl9?ZFgLg*z!kt6XLe~iVH#8@+YGW+Lk|o`+(T;E6xLL%b!ea`J=Y|y3NFv z-zT>GdaXpIn*X@Hd_p?0~*=cx2qTw=?g zkWOs*Gujp}Q`_eIC{x?=hj0ycbFq7Z*z&73XXvZ8ku@JLH`d~~Eq^pk+p6gDbf01} zhM77Uw)|+nz9(Zkaa;a`^&iwpEL-Q-GQpNVOlABinLxC1h5`NPD1KSIOAem|p5TE7YtTmA^K=W{0ZrF%@7;>2wn5aEh#h@Kx@tGEqYu-yN$luV zy|5LHl~2ROc0NMGx}12u+KxV0uePI4s8`pc${=?133-VfJ<>DBndYToVhbOkVX8yy z;v>W^-lsamENH-h>eA`d);OS((HE+dX+s??CqswUo2jF1j-%~N zunnhuR99pFqW0q%Hq_bZB-lGs8*^QXnxp@s9Xc7G=-8~%?NM`>g&lgr_*0!sJ7qn_ zs-&pRb<`GG#pw9Z`K4n>)9LuD{IP$DeblM(zGsXfEhoG3Y5Sp{<=P13&CntBs`<^R zH;|r{9eVWs&|@1cKi)^5*n_Kih~HChc&Hu`dTjl*%_TWML?yS~W&~`gMM|y2!X#*awH(UF6)wX#1v$dlqZA+zLl{c$( z5*)Xz)*IHPenW;2P%2`9hq?}Y@hn`0Im0)X6 z>iRFYQ;mP@pZ?|xN_}0)jBC-goEgtjCpvHXIY2cs^XYeGEvc4H%aHN%veeOa{gX~4 zmQ$DakLAR56LbP~=&{IBN7oh4lckO>FRq)VjxHxFoxrs$+oN^-Sn2<$qjlplp*{MV zE+;D;eND@M(urToaxQfK44IvcbLQGgw1bY@I0x#;vaYC3rv5+bC_8TGC+Z+?JY9{K z9^ZdUC&T7CbEISWGRsWtS4=NMClYT@LOGdziMB_NL52>sjfEXgR5!DgA%m0%-`yczb$*@a2H+1eV@yktzWt&5CjLOW?W zG)%@*wL#l=XInmLLztFB!=w#rod2D1$kN6l8PC@E+PxN!&Fi&sxHb;a zJ?=l~zG&;qc4IX;TL1@`(`OWrCN3H;k8wLg)imXePZba$M%R` zf96aquCA+mT2ISSb!EszJ|$K8^S?a>?#xng8LL;7ujr~WxgTHW(sy`;xxS!fN2mNj9 z_dPx~DkuCk9pli}XNAT^$6@VL-LXze`E8qA7ahNpBE7JVZtzP@-4IPr!vFE5u5Vha z$|>dFoHv%w`+0gaUa&;V=(uj5^GH#$dUqdG5!UYS+WuRF0|hE)0KxNn+K(!X?L z?vR$r`>yx*M?^?VrBCN^_xO_j9Sim+A6t99fBx6g8FKK%l76>-KStaC)~eK~++XYG zs4Q~l?hL8=RqglV$y!FsAD<)IkM?_SjLOBKoKbo3?ZOz>C;hXN)c9sfb$_XN>^^Jj z&$(kN^fP6Z-m!6}QqRAZ7k|0lPZ?D@DJa)Y#`;+QR8mlW6JHNns&=b&qNQ3Flj7@w z%HaCZ>qw`o{CXcyQjN>~@%5x-jcH?}=|MTF!_;WHlB)mT#n+dXdfjQ+`LtR$ZG9~# z6rK@{$7JUE((A13k=VM^((sgiDPPMfzr@yIhMfIk^m>)JRO?ddE6E@?Ozf5*Qx24l_>RdN#7@l&jnSlUUzE$SL-hJ{L;@0{rt+5dY=zU z{ru47(C3sYm&R4SbU(=|PWLBuJsAI|=YZN5vU*-+)sDn+^m8;*#`>wBms(;y$DiLp z85|cC&+d7o?nCvw()&?%&n4A=wa@7NLhm2J=a7D0seb;u&!fcWkbXXu^7Zp5_#D#r zrG6d-pF?`z(9a|K98&Kw>iMMaFD>svPxv((|Qd@HwI5!RLy~ulG$Y^>}HSv5&_0RlV6a? z-4uI1WzJjX^G1C?sd9tQrNr_wzpK>qC%E70xGG;#0 z{@=_O{f^N4x!%XMr29GD*HwB}-y2zdcj)&&wco4$(tSVkJ4EmQT4p{6=<`7JlRg(z zJM?#o-iNf*&j~H{^FmAg{i3Db2eqX4gL;1G=ZTi;`$nZ}slR`+dd>v*DgC_3PU`1R zP^$h1-$#OR`g~IQG#-3j>A3!W(o)}dTB>sNb4*M6Jd4(Y?<_SR^!rN3)ph-R)3QkX zIj5z5-f5}6zf?Xg_4k>Q!SU4b;6AV8YTYHiPvLnO|GS8i>b|5>)l09dIDOwz{i5%c z!MM^-e7_6E|FiZX)!+Z>^Wxh66Z`wg|CjsEwfzSBVc`Ax|7O3@`-+zTf7@RcF8mpL zXG$n*S<8>5YiY#ehHn~cL%C;-#&a!EE+}>|TAr5bI^Qugsz+t>TsxwAL5aN->4w@< zqcT=bt&F%%&y;JShxVb~^OgHV^;Emmbt=_$q^W=L^tDm_MM&R2wO=&fnWDp>vo<7q z^us=XCArD2q0w^k#>+c}>#_D2M!bLVcBMq~#fn$$rg7E26hEF1`YNva7n8C68@`qe zYp;u5*HY=l>Q9uaKPo-euULJmzgMnALa}zQ4XO4nhF&A|BUk<%QT-wdcSO@wT>YI` z=U3^O>%cH#^B$9GTvc34)zA2P(d!{2z6g4kruFlKQt3zA8<*<3s*g&Eb)w=_#_PFi zeX4m=?a(qlo+>F?Zl;WmvvC>o`S{MEc<)Y>N>9%(m9cttKQpCXFKXNp*IR5osq*ML zqUEdkP;t60b)2rZ#Py^56YEE;pDEGxj=b@8sAUS)ZRY&P`jfbBRQ_1~vGoyL7kXXk zb#vAFh>m+obUmr{6TJUv{Gaak`27>VpHiaZg6_ocr>vyvf35g^RLj@%u4R1wGwv&W z|LFTl-+#gNMAx5QkHPgv*KKy|B3|ylydRXFzK^t2^O$wH|Mh(tMa9>@UN4F9e}DaZ z7)!P9YpL#670;9@sg0v?Do1~RKB`x3j1|qVq`I!9x~}3{YQ4;QGV7z|s(N*twnLXg zrMj-`)ACdm$L|qo@BbR=j`D-@nk}9L?PRk~JJ%8~Vjs2h;6wuRgRbFkB^DgAT z=sJ(ZM>KH#w4Y=3#MYx1t>>+xG5Ju@V*c>_ABVJ5<*2xpI$cYhuH|jDibVA{y_z#B zTjwp}tNmI@wGY)p`SzFzQOP0q7A)#d+dKLy$yqU9;&g zxQ={Umiiz^wEUn{?aw{+%V#ss{8Qsn zwa-JjN~-cz{k@SdD5LE}JzA=A)Hvz>=(z5$l4?AaRO4{}XF2>c4aP_1ACRhil|LvK zL7!t>UOE-aujWCGe{eqZJZV`Mdihb_2WN};s(hvIp`4)HgK<&yYN@U-d|*s;Jk_|U z@zqj|vl@RjE_5DLx~iwd{{5j3A+Lx00_Cdopj7!d#!btVJw<&RdV0K-q~ovWN2RNF z=(rj`RbO;`P@a}*{FI)SDqqKM4usUYUNL-nRJM6zV#q6y$8QU{3+1e7I6f-VAeG*T ziBm$GA!~J+6#D7ILVi2wsry6ARVb$n`l<4bJbzs@{psU{Gp@IowJVzM!=d}4`IJ=p z&G+pOm5s~WQQpw;dqPTXPyHg=o+Z#z?Na)x9jY89Rk~{DqsZU+(_GQ`xC1%;#Ym5i z3*KKcrFu@PxR&ab_I?sZ`gi zt@%BqWc9eryXn`EDlcDLD*b*9zKP~jdMd7Du>UHq#=(z|lZvZ-T923Ug6mHhe<-Bn zBuI6A+S%Cttn{?L9v>~$e5>)JAu zh2PcsCZwhAw;q?VlTL-ybuHD}#C@msvTO3%I9yu?|MB-h?Dr}4yH?iciZg%r*1wDC z-^JxvKW+JcpLSb6GmB&us;zJ~yMKT7UF+*6hyR1kdN_b2>U*m80)h zCDppu=W4Xf>O76oqvvi^zx8=0E%kX1E%o^)Efdc@T|4(n&&TU?XhEreXADX;kNP~A zmh>E$x*j}7rq7FMsn6YMNzd7-{Q8_;Q0jAeiBdl|{`=1T()~ZUKL_Kgy}|uI7^nNc z(htV}X}`~^zC``2pKtm(MWsF$re*M4nBFh+xi2mCxi8i3Yv*zk?Rn@t>vK(72G2FA z@yqHQlbT<8eo5&C&npGb3tT&w8?(2$b}lz&Kcx3nE$M!I?Obl`oUXdB==+CykBHe3 zCO$vZzL;pwnEiJ`VtI+*4SJsqO1^2t zNzb>ba|AjbJkO@m>G?Mu51x0`arHY|;(5ZvbAtN!MEp59A0F7Jx^3-_xr7$ z_p8^Zx(0@t~g0uVwIjppFO65$bsGyrPZ=&o%0J@O+_;2hSnuc<{WVjt9?0 z>Ui*cqK*g8G3t2myrhl?&sFNU?uVATzgp^cXsO$)r7lNHU7wbET(#8WucaOrE%o?m zsr#j+?!T7m{KB>KnwjS|gXc7LT#eJU^P1PrYwB~FT3$P^`Tu%eQ=QAa`u$$N$6h7z z{7N`?sN;BV*XIp`@_+vP;s5sbUpqgk)>&5PKlM34E!A`2+WE=M^P_4!uAQH}cFt0L z&|Euj`ro$>Ov5rboNMHS%VXq+D`XUbD`Av^D`S+0t6)@wt7O~>cei1}@u+_OsAj3C zPQ~a|Gn!rmDLLWqG;D(zHGnmYdyLA)ZN}5Ur;ScVNu#LI5ZKUYVhF=AssgJTNk$dp z4x=Nmqw$P!y;0m~3~X#XYIue;l7Y!lQ_;JOE?5O!jOs>d;|8M{u$l3gk;}+o)B@Hr z?lsC9HyV!vA2(VU`HbAgeZc#S2aTJIaz;yFOQW?>(8y~%0DQoxXWVMsY_tKkF`hCC z8wHHIz`8~Q<96c~qb;zl(cUO#TxZk=);At8su?wn=YY={T@iZLcpmsXV$T|#jTe9~ zAlBLFU~~g^L#%_*&Ug{{B4X`~CynmF?ub2Uv@&`Cdmz@zc*5uj?1|VDMswpO;7f=# zH<}tR17Ak0snN*j1?+`bBjaJCH?TKi4;v2|eSm!sd&sC`^ab`std4QN@e1%2#O^n0 z8~uR&5UY(p3-kx}M=XSXbOm-r>}%s~V?J=cG0GTdOfbd)#~Fi+up!Oq!0E)J z!?@9`W$rL)na7L)#%LptIRO3>A8v3IDAz#F%E3GKawLG+sB}H|7H88ebTzjK6VZ75oq*#rPGq zrN9qB`OA$9zzeA7V`HUp4tUP^%UFRvj{}Y~&Z55J=4Zgqj8Bc?W=V59aJ%uDQPM1G zZUb&Jwxg{T&8@(##%80UdAqp}3_p|_8~e+R8iMhmtDz7^YN>^Ewd`{0w!uZ+FMd&XDrHO&*oZ^kl=@-mF`Z^jv8 zDR3!br_rwzV2be}#<7685xCL##3*1EGB*G>7#od3W0M>u4>!XD&8-nok3tHanV+noZ3Xz!v6X<~Xyp*$UXoEWsw3 z-OU%xN$}^*X}JCv@G*0@*~fg%JOMmmzG5~vo0&JV=J0LI_sp?o0rnpJ5px!ur8QfeQ16NECyMOEye08!LDcH%&zY&%A!@Ny`1{!uqnNqPoMLPrurTY1y&LoMS99x&phJUCa;h zgeuQIgujDbF)kYUf%#b;b}QE1XXdT&pJLzn*u2474qwuG7o%PcSj{r5KaH`*6Xu`r z>&&~%%4T797koi>8}fZ>-Uh$ZtZu$*bOv@dkC|^7cbNs*TSh_F0os+!qO3A}VO9aU zJIxC4pJQM9$Sh$kgD+;)$9(QH>%;HCEWTy_#ZuubSgx7dEXiE>;@AzAnnf(^4Aynl z66_EPud@nRxy_trDV7_)Bx{H=_LvRf_hWDQz$|FJ51-F^6VHdcfp=TySWb+08DJS! ziq(Vee%yQeapx_-9_m>O;9aXZcJI@8PMu~E?6>nxY0ZP@*3ZU7qp$h1^5*M!zZuA0 zhyRGZif3g-U`1AmeT{dx4}l-D6gCsREN0Dw|BY3`x=X`Yq+ul9!Sl$l-hsc&2I8Hk z0Pj+sgD-(ydG$GnXF^fJ4|E*kfkhs=w{Au~Tq!TVELU|Ck4tulrfi-3z* zEq2oQ+4$K6n#XZRO*4OHkHV+18HQ^X2Nq{7F|u!)m#{luVt=tW@ccRlJjecIJ=x3b zZQ$GNP4+Z=||ec!6DHPq159 zDlnD(#GXgmMc_sDH|qwe@Nag7jbcx-Q@~TKsPz)-!5ENPw$+Ed#1z_=u%2SKv$bf) zTC}1c>%)ZA4?d^00Ck>c3*hguhO$>#9$+3Tzcri5I8#{2Dr`N&X0ls= zw^$e0Skzh68Vg_CT8y?_VT<9bS`%=kIIy^Ny)}tVU<$9dN?TLeWL6ef)+%qk&nB_b zz|vNAYdV|C$^*+=H(N8=bf)lT>lSM=TgGYuYguJ2A7xhsR<(Arn=oc8fGgN4jLyw$ z4R8%cXfRvCYFLBeb6dBw+gLjKoQ@tZW&K$$Ybm^MRYty5tTOy3$eF};0CynwZR`t?!xs?fS<69>~2Vf8`)-7omFL9fm_*j^HuzT6(z|Ywi ztRAGoFW7$eF3Q*p+|2H<8nB1h*Q^2jLDbqHiU(11RW^bZwyMH!WpA*z*lpGu@D;5` zSrhghdldc$)*Mpd59}yrx*kgI0ne<*k#-d8=_poHORSflfIqQcP}6&?vh@)B*Q_5P&;5d)}FnJcccx#4QxI84Bry%f$dpGHURJLj{zTJW!VtCQ*8onVjI|} z`1a@s?8rK^kokjoyA^_e2XCRn%x%DJYzy0kud6P=F6>#h3EyV70&it4*f(Z1b2e}` zyVV+ww~5c#aQIKzZoJ1o3w)M6hc~5B=3d}lwu|k=yZ&>)=hzGE2)=FF0^72Yc!L^a zehK`N?ZF$=So163S8N}94e!^Ltk>W-vHiwAqZhCj>&v#`d#@6(5^IMysPX0@;30N^ z?Zr3EE5KLS05%-oSDk^K*+MqgbS%@FYns*^Xiqec;9ctoJA}8eq2?Cg7FNVMY#cBK zvBU7Mu}OGu{2ur{`<8u!FN`6;A#4~sY<_FbH4nqjfyNZ`M>ZM$d%W?zXIj=+_^;SO ze2)wR4r8yg9rzZS1)RmYuqk*)JqA3+eq`U_t8^rABpc0!;9KYg;0tUKn}+x26TlPf zXZC~fy)gzjhK*yb@wRvXc!1q$O~?E8uWUN}3HGh=qcI*ho=s#k@J{|4@Hh4=JC1MJ zDZnXg8taa?-*186vMSa{d@D|5BjG<_v+zFuJMedQn*D@tvuVI-YzCW+cm6+tf3n}% zRD3&p4E&fKW5@63>&0~bKQl7G&20m@IwVt%vSsj5LtqxWz_>RDiR(rUP*0aE8tu8428LKO> ztJTTsY(0m3o#CHC&S#w5%{9jGup1_fzMmrF(OmYmhsUxO|>SXeN*8F zS(7k&y@9HH`1cX|$od%gF=DH%)z%8&3TwHw26+|(7h5ZlV+pROSrKb3 zaIKY!&^l{9a6Mw5SQ{WefnSHvMr#vr6JndKEs&exHzJe@%`Nbo;MW89p#JxP?^}D} z-bc&!!hd1yL!0LV=c6xQp)JdS%dN}S-)QGD;4%w;IJB1`w_z`}EIZBm+B%3<9Ykv) zz(c@8XvsR@Vc=o3>^rn#BXFa21a2eRcLe?ixDB}a1N>2IE6PX(rdr$KQqk7!@H^nP zAomXVU1-g5>jdxw#%Yc9i**us65})vcnWw5t!`pHVtoeu%-U%^XpOOc1pa7Uuo@t3 zCvc~=+j`Y{-TDdmlXc8`)M{;g2>j6c+?s5Sv%Up>Ykgz2wBE5yyCwVw)(C5)^(*jK z>u2j->n-aF@QTIkx2###MZ_;6Hqv^{`VIISv?id8Z-C!e-&?b->DGSWe(MrKbFDML zGl;#1`i}vRS*Ov;3D6t^|D$yu+PE9I+d6CYvbtFZfCsEFT6VAX2k;NHuAcRhl@3g| zwxQPht#iP0sQD4=A?s7%r|5kyyOsTvoeRE%?b_VV0nB0Nh09^*1m?7J+qc`5?Rvm^ zc2hga_U(s&57`aus&;j|Ik36?gk8q&WOsxsV3!A$N4zz1=7e+XmT+wm&k5Pu&V$-Y z087|KZ3(Hcs9n$&wr3Xt7P1T5rR?iFuc4y2_8~9d8vF-f8{J1{Qe$IXo_@e!Wooat$AA>l<(xFdEN+-k%RLUyx9+7s<*a5L=5aL-4} zf88DfHy+nU;)+V|iuh#sk?7$h`xD?N=;acWy&tXz%IRh=Kzu*s5_^I@!u}ZevAqdB zKW6_3*VFC}w*m1VA-BT!w0FXNZodup7UDZ0yW6wv_aNW1=K<&0Zvq$EOHt-hdpU5q z{XTGo9YNb8_IluYdkt`-{VB@))ZPKyVQ&Tw#2m~8&b42Kn`_Sj&asEuui5X~Gl4Vh zK^XU5c7I@h`$PK!yPw?`*w_BZUS{{PUjn{lFT#whveyCE*`w`I_F6j(4BO-Ev35H8 zHx_;b#$u8^9XQ?I1~=WF3Y=YTA(b~F zr-SpSea`OSoU?~Jr|m!N_D*}}8D{`oKg8QZp0)=%U7SvEEuFS-J)IZf`Z@#QIz>}n zMCc{wNv93m@AgT!=Mit?oU~tZF50GZ*}iCBw$IzPBb;204evS3xnw&)$I0RRZU1Gv zK-b~U1^bGf8<^XXj^S8NPGC+ak5j-Y>RboB&dKYPaBgtQ0n0fzI>nrVPDx-%r;Jm` z$?uc~mUgap@;QZ_^1$*=DW`~2+$jqz>)hm2c5ZcU2Hx!4?o@Q{bRGge#s1t$ayIklad&izg;U@a%vsp8ag>HzCF)g0fs z#kmdc9;Z5765_W(`WU;XF+xu}t%2>GXECDBIvs&8I4@&FUv|0!`#OU$-h-X~z^2YC z&LH3*=W*vTCk1o<7<@CQo72!~3~cQ5c6vFDuo`;7_rP3scDe$)I2&TGz7nDy7-hd4jm4N=O^b~ooYd%QEo83ni2NrzkGtb|+cEQMR*EP|Wo zybJf1GaGJ(GZk)=RI^N#bSvj@1xdBa)l{N|hho^U>P-gCZo_5t@f zZ#$nj=bZDv^Ufx?^Uej}1?Mc>1?Ml|U(WAve>oR{7oAgZ7oAJMOU|!wmz=+We>*?J z{q0-^UUrVbU3RVjuQ)%zU2zQl1N^tnw?Grp*r+<#+Oy`~%>Tl z&w+Xh0t@o4D7gpk4eZVD;#GJb{xa}oehej3{PAg>Fo%NN1b<#m8{`1^2mFgowUFF^a+^7g>?d^`V?Kh4_#+wq-z2Y(8^ z-2uN9+B?N(q7MH|JiwdrG`Kz<1p?Wd1qi}{s~-X-U--=Z-ncFk=+Qt z4&&I6Hv%@|@4_|Wj{qOx@4-ESnR^fZZH#I^J{UNd*WlIpAf5tD;YqwEABZ`x315x3 z;HCL;;BwxYx8fi1rNE`U4ct=9Z5#NOSYyNZDBvg_f*Zw007vjz+~==j9r^Hi_zmJa z{tfUO{wv?Z_wlcQU-8emCB}+YrUl>H6yjO&BJf50o|!}3FZS^q@OyY4{C?R+^npLm zzu;f<$>Iz6y}XEMDu#+8@J!?pN#bjs2mT9QK)fPe5C!0G7rXdr-U--Ac%qzmMAQb> z7P-X*euo$h94*RY{BQghewaTbeu2M5oZ)})Ucg@BI&qT! z%#ZPt@YjpKd4KT-{~K4&@ZG$fI1N0_ckv7SFMcP=zf;^HF7iwKZs6VGE^!Hc{tEH0 z5UVEc5&3}mQ13mWhR6%di&zcxB@ZwUVzq@YashLp&Azx#^_W74qy(%9ujp$ z1z-iVvyOOJ+yuNy+$*Ue?qho zt;9Lr2L3#6C7u#zfoJ(S{-kIpb^~|wKQWFSgeyA1dkA$A5-1Vtg1Hkwf!GV;IdL2C zHjMvs;zdyrSP`)oF}Ak?Z$+$^cu5oj7Q%SGB>ISgz=DYN!Q2%97C`K8euWPL4if#v zWp0Wgz#$?<7?|&&z@g$*QA*q>UI)G|zUQSy88K3nh98blSsaAwipi_kF{nC zE+zpdA?9EWO$JUD6MzrN3epi3`9-Q1W{0 znDfMY$oHPOgTKLF7QX|3=Ou7Qm6bPPTyK(laOahhH_BZY$=!S{?=HHDwJ2c&?#`LA zraZ%I%D?$0UQZkZ9^^OUE`M8&k-zaV@>Zh+xUZ`9ybuu@94!c}y>~pQr%00N7n&EzGCUb}xMoV)Ha16$=07kVF#tqm?$WJ68prHvbEeNTFbp+zj#VMDfc5~pO_{}%cgRgXevJyUyG;Z zQ}SzE-7mfo9c6p@jc6|qiS6Q1Szc}z<>hSgt>`Q}%5QP?8}Yq(Ms}9pBjsChMGTQG zedPtwSDqCYMT+b%FCyiF_*)E?De`ZmToi}Us>x{AWSLtsIaCe-4nf_8 zaOCT_;=nU0rV z?v!`P`AC^77K^H~id-zJ$c5qqo$_6FC(4S5I`RP-5f8|8u^u(nL7jCFUx<wI#moF?YK&_;#AEVNxdSU_yGTbZm92D9 z*?N!d5>Lp-?R@3B)&|L~IecAwCB765aJ4?Z zQl^S+xVN_9ed`>qR>t+p@;tsq4v4SBL(pu%)A6=29q$X%SI2}e-%HA$N0~p8NQLe5bvO`?_h7T#DlV~v_xHLi>CavI0-x{9> z8~&F#2RtWQ@xr30coZc)DjMUBvY~8*>y4y~H^2Mk^Jwq$vIyRWE{VT@e<9Dq7_mI^ zVV*~JN+z7x4IG7a%%(M&eS-qu{^6i;(Yn(}FG%67c9 zd8hTDY;8V>e&)p0$8r5}nM<_gT-vfNw`F^t3n?v-)&l!-C!RyPvJ-b@2j1PRVl|N6 z%?9XCeqKy+nIE~@;;W>UY=h@O8<|&h=DB1J*%`Sy@w`ZBi?p`bLp`*%6xv$~?=BtD zi(KeOF6ebY$v2?v8)QEIll)VDA%B8jA)m+Zmq&m{)8udR6mU6kxjcjR zynx>@H_IN@r)W=4{D%3de9_v8wsgmDm^(2-TQL&b(1!2ekICb}9l#y(2l*p#D{!kk zBIlu`?ZEAFv3wu68@OA3C_eyh18&2(uE5Ce2kyr>t(9NNL%>5Ar`7T>W?(h^GFjGj z-P_%=@YUVJRzbHSu%i2ybwPfK-!L!8^VU6XP4{NE2(XA-+C_@|TV8;_DF2d|Q;Nt9Rr2-R?g5y{zDtbH7JvN99Qwa?7|UWf}Kp z`7^K@WHtAkJS(fZx4CE0lD+a9d8=CiZLEMib7hiyqdOO^T_m^2JnjK&i#&keGrvWv zYM@;;-1p>sS=`O-&X>8}O>!StKWxgCHV+{SPn+|IzxZgKny+1Y&-@>#bC ze!+YeT1D)3&}-~I@4g6p5qX;edjfk#%k1g)g6tK|+Y4I7?dPHQvOCBf103Uab(^`v z+#$dr?qhB@ceMK|@Kv{m+rsVZ_6GKLTf1%CKJHN9Q1?-{tvkXU4jk@2?RInrxcz|r z+%E3p?rZKVz*pRs?lW$GcO-D6`;Z`&Wcd);~N$MEmE1-wGuY4^1It9u#lid)hv=@s(|z@3h!ltk!l_dWNp`!@Xh zZrDAESQ!3ocZEC0eFtudyBuz*yBKbjn+~@I*H++)N?(lla`+YavybcTclW!WyWhe6 z;O6o2csV^hzL2U?@*tGoTj0)x`^`NAcOBw$A@h4X+^z09xXtbcxJb0#Iqs)$yWAac z>!RuJAie>92YNQoy#T!6ZgX>^o*z)(H_?(8ApQeFxxFp!CHI881%9nt9R0q6{{Q99 zhg*pF70BYA<1LJ)>~p__lZelUbi6&l18`ruhv2?+zi^MBhGTF)xIe@F;C}D^f|maY zcg8&rcg8*CUPKKr%s|=r@g4X#W?{8^5O@%Cvf52^zXpDdSgO0y-3{FBeuZ&A>V5?L z2;={w`;+@2@I!Z*d))ojT?Aa@et;SI#61T*hxu9WZgT$s{^6c=H==JRfG6DFF&2im z6Sxy&VtAJK8Spd2n0E=Iz`QN)c9iuk>RbfB61{Xi_0RQkz_}jBY~}It!sYSu!R7IC zVSWmEh2aW$Mc@i~1-+se$?M@tdN;t8^h$U=y_w$Iz_-2r-Yed_9)81w|Hba-{fgf# zPuRV@zHl#j1K86i@w*`MW(7 z;gi5JUMa6Uu)J5ntLWY7-3`3ktK{9{-R0c|yv-}?mGf@*Dg!HfH+rSLJG^^<_josX zH+xmRD!?krIr!j1RFcvCT+Z@|s+W?+2hdJEy^dhcO=Q@jPh1(@f7nD4iN zZ+r8-Y2I62A7CHvW$#U_hn~QmUT@6hQg0G)66SOn=5#c0v^UvX_y<#dh6iUd!N8<@HWD2 zLeAyLr*f^1>ZeCF*F|+UMz5}nrbm#H3cuXj;%)Y}!fo?Dh1>3Z2Dii83AfAJ4fnaX z2X3$T1>8RGOSu2j-gk#tQ9NsR&-4H;86-&*P(ctSr`v62hS&L&0uKXO=(N*7B`LC_ymsV!2{q_gT z-vG`gq|bAH%GsQ=1xHX><4~1%~dJ%56;9qFf>t-o|GYJYOSxL~h~V~O`Zr*qL^#~l}+ zf7|d_;v-jeDjL*(pp}<+@yVTv_BrN^c*BOFvBZ~e>sSxkr@1lvhjSTm6kbHebeU+oe@|uEtsU9U}+o)b7<}y;;ejM9+l=zd0-HM*CooDN@EKGIBe8PM!)uY601GO)U$H&pAGU#f!ZuUAaJfi7!*(ou{FqNe&OnzWKVZN69Cb5Wlo3HCbGPY5@u$_dtj8u0LvrkAyVlF$?qr{@U zBocFbA)Cv>R9@y2=4;7660;3dUu-AU5!*oZ!sQ|vpU>Hc($A&+DSgN~OZ$rLAwJtc zK4F`wUbtMc?4`^2{gZLwwk~bb_cR(Czq@&l^!6KnPp@2jY7BlJ_$lH`8Oeuy3i2tE zmw3Uz<5N{X&5a+K+%lHSW?awQT&p8Y`AW-#=pQLY5#qP#K~_M}m+1m#Lg4Exh4SAs{2D8WxjzR(j( z8Idpfkn<{07g9!)%PUh#Po{pMr@S6fr^5gL&+3utpW>Ke;C~YKwHFbGCEAqai+C&1 zHW8oPZt1Jtg8+*aNf&o)L(D-@#KLBcwKXMr2()t+5VLPdvG?5}uG)5$OOtHL*UP zBv=nA56_g$!P6Y~Kspdld)y1pWo&>H<0*|*RY%Z{cot+sJZG{OXfHgAke)x;8MHH= z8`)3w#Zws%$Fmmu<2Vw}i98BVT|5TIv3O$Sad-ma030XasfZ`y$&V-FI0e`A!xbcd zl$AWr%A9Dw?PvMNfYTp|o(g#to^~@3=~O&D@jg7kXf)F0cv|CRJlW`cqzmz6#PN7? z##p35c&g$|JQ;Be(v^5JJ%%$LsKPkkgTd;7O2osi~k-@f^rA@H~(kL2tCrhP)fkCcFgcQam5?COoa-ETpsX zB*=HLpFUuY#CvoiDTw{tB232+u6%~NxK zY@T0h?GS1Q1p|7w%x`~MuEosHS-&uK;+`$?>wP#lUYqk=>WM#_=XYCpdi>J7C8^ns zTjsYtKi~5CT9WVAt!4fmx8zwFp(ic$q=lZe(32K=(n3$V+HvFJhgQ_K_i_K7@saB* zlKb~@s6n!Q+`u^5Cx1@;wUzM+b zzJUIyy&F9H75|!wmP=27RhC0-9FJ>6E39Ilr&`&zVtV>2{Z&D$t&CQ!)SU;kFM8p* zv*TX}RL^^L!GT4Wj~`?4j2T^u<_xSCFB(`qFK*hoNLM`2;@5u1?^UnG@rNf>&-;5! zhoa55A8+v=j~`gn64$1N){if4+rDTzbhZPY*`#yPnhFQTy@2OJ=VQ>>0eHpe4n>P_ zZC~IX(AgR~TLJHT$6d+EO=riQ+E3jDS8DTcyzPajEx$BodE%FaCw03w9@6ASd#w-W z_L-BB@!{`ZekML<+d`{H;uCf(ihp#rNxZ%JGjXHS7h8FWo1OJ^{CAV$vPisnlm~3XhNlayYDX&$( z&yN>gJ=NyBJ52gH_9Q0T8cv^SV^3nz-*sYf+)7Ng4H$Z#mG@!Nv*XoS8J&zNUyf~w zNzeTiW@mJkg~^8In~P<9nDTAEXi?nA)g8A5(*NSei{i-D9b?k7y72NzeVC zEs4i=S#0$%<}&_p$c#)GseN&|1~;8i0&}^Vg)=hcB20QZ!sis{3IR=idRg_l#AOhw`zeyd((~18bK(b3*L3aNJ%C;P zGNx-!U-V%7nTt2>pUDr5NzV&=Js4l%VvjNDeCg4-@lqFijKz3eD8}nTFHm&WLCGyJl!>nE*@54(QM%J{JB zE3Ayfu3xfziCtT^e2HCOVfhlfxUzhSU94KZ#ID{g--lg{Ss5R8ZQRO8Ol3qLV)+ug zc4zq#yLN5)j43|JW~vv8VK-hU6q9bePAG2Ocx`294$q>ePZRHu09>N%} ztxm?2mm9CG&ayD;C)*^ZyxbUV^C}BdzHW@R`R)#revUnf$xb(3TRRz(em7oQ{fx;r zH(p!Y%EF}6jnP&oW6GCfTVm4b#%QavEKK&e@!Hx`7N&gNcy0a3ZGrT=G1}^9Ogh~d zZFMpxJ#M_Vdbq8T&5TK>8>6jG#-!7Y(N-s8(&@%)tCQOu>2YJU)x((dxG~!5VN815 zcy0A?yCywujJA3hlO8ulTRn`qj2N#oWu&&o<-&Mf0&}@AMrX=JnDn^u+Unu2c$=)x-TBUF*hcEAPhZ5}2-aRZGyGbmsl~`UYvAkAdy8A!y&nuEG-ye>=3*0&=aruvUW%3h} zxVO%*`0ed4rFSEfK*p#4U-ew`&bWB^oM@M8iAUd+&i5khfq4$=Ev5Oyqwh-db7n>9 z`^wM*-AP&bUON8TbmPNDLyuCwn$8K*E;@Jem4a-7Ykc-6rR{O$-Nm2I{$1oze{(TzATT* zO68&QA|Y?3$POw8VJah)gD{nad_b7|AZ6yCeRZC_^%-89{y1{1C%}pC|cW-?k||y54a~ zueJ+I$aiYnK52Nt{z}Cm3(>Pb9q&A8x~5P+xE^$ zFZh0FY}&7|fpbdo;OOFBs(>6J29bsv{BJ9cW`adD9PxNXJs(igU;>b^KA-hc0I$(+?s zre;*wD{lYc=;WrO@GkVur=_ZYc4KJ>B4$?(>NC)X6J*0zl zksi`Px}=OK<8o21w6M9b&alzRT}O^jRqnb+p(s}g8R7G^@OdJ9o(P|(h0jaKh;kK* za-}7%yL4c3)43Z{cbV(bqFiZFuC(y8z1I0T5k5~y4|JrU%j$7{PDEWKPftcWe57*v zfFTd$_IagZ`r`YpOfIPzrkd+jTrnt?*p)XeVlORX&&G|E5o7Cf63ekwVllQpE3v#*VtK8^@>+?h&v4^4`Vil^ zTqZxJYu$K_KJ8y+XuS66|5rV}@ym_Z=;KS~Th{nRcKXIUH(u|ehsLntv8$wh8Yg({ zqA{j;oGB@@lb;!{cPYEGA7w9jtt;;?{`Ac^%9@{u@j5NW>*Zp+PK)unkjhFvqVg7( z*^bvG#z8JKmESj3iZMFRhsAiE6619&#_L#&M=3F0r^J}$oioU5sZ7Ox1+f^fQ5XDI z5R36zj*qbzuS>{_@j5NW>k`+B@j8!lqIwqNHR_lD3QCk$jMpWum1UP@{C_oe_{K8d zxF*JH@3`h0N8}hG#%u35;`(qJuS<+;q{ofdX)#`V$2C#L5_1wUUgwGN+B**sKL5X( zBZ+dAn1_h*Iwi(y9;bGhhX_BHm=}rhIu_%#)l=5INQ~FZ#dw`Q5##l1_o?L2ev=Bt zc%2sGb)oYKjn_Pfq4ApMCp2F39EQegp2N_1T|&nBgvM*GYZ|Z9&JQ$RbN$kI&GQx- zuX)}=<8@k$(TT)jEKXeP(Rgijl0MAycn(8jda|`s6j%8>KmHC6=%DjykN=RWiu1?o zezWH)?fZ4ngo~6%S zKP|2&-@^O8ozcJq1G%f)fP=S8j@rO(e=JC))y+r{TCvufD#mp%ut7u#3*+~B*q z@yZ9+W#ZuFf9l3#PS}*k=Q9u|vu^n*kI&~FUq3!(>+6{~qIjx-xEi^!etiBPZ)DDw zg1_+ZH}m+sadEx4)lZ*i;;CLxKQ6j;V;-OXI;?)I7HrPr^TVy7Z}qB7Jk~43_eZVH zw@_Cev&riGX`R1UYRJq%!#9Yh})74Tt=WM!7bA zYwV2Uqn6Z*3-(;tbNR3M{*Mto_&lIi)%czZ-^#S7!5>tP@9Fqf-rPrX;%>+8Jiq%_ z^|SZ)|mWL9-qHCy=r{htE)2Yug^hM;y(^so%j0pIq`^t z*JjSYoLnvL`}*g${T7IJTzOg5xM0M_JU)*&uxh*z^~C4${;{~9lH+ITevvYj*LJfWw4vMIre4mhdtFTaa6@F>RR9trzOzd z*qz-Cv>Wyjcfy|DdvM%`UBgqbXFG+XljTeS8i2jlwQ(Gdeb7f@7xWQ0j91bez0;izT#?eO^^q(&-6j)QRw#lGa> z*o!<0$2HiSd;|76$2bOC{wRDNjugYPJnX;jgw!AVxocv#cRtbyC{IJ|rmlw64Li{< zz_}?>L+lo>f!*D0kj7#MX-DkRu7b2D?5K(zJgwLAlC_Y9}9Xc_Puw(PVu8akHW6=&Y=B3`@zQJQN})?eXv*jBBxH}^!)>p`!_ ze(js_{RGenDA$$vej?~Z>`5O7Itg?V_LX0a?{5UX5k8rM*mwZ+0qh^Y6Z_2PfX-2O zV;}i!_+Tp19gw&O`^4`JsdgzZmplNd5$n=8)!FZ>I&c71$q~xy2CejgWhfbJ7Fq3(-Wy1 zO5O=IeGAg9IJ%&wZ$-KtCGU(HzYS?JqWenJLX6Z0|C2Bdwa^RcF#IP3{U^YoNc4XS zEg)G1sU!B&&&3Y1v&<|lEyiZ^q< z4IXix1bRtz)t`ZWh8^~~ct`Qa>SOf<-fX!VZ`wT=N3P|p#=E1R!TYuUz_DHZh2wX% z8OJZ`CmcVhXK?&s`J3_iFQjLX-(ppSgWePVf=a9RaI8`v;8=<8mf#!WzlYB+AT7bR z@6~vN<(H}{-lg0I?~HDMqk^uDqpC(nr@vIhZ-CEjkQD6NqPD`?t$0V}SL#o-40IXZ z>AeVL(RvY5f;X5ZIKPNgO~0d>=?!>y<_7f?{?p)Dyb*mN(nrX7Io5HPAU&zt>SmB@ zi}XI;a{7wupx;8u!<(TGz+1}qL;4VJPJL75>DQ5Z>zD9;)n1^zbXWa6a;pYf4eu@A zOYf^c#Jk2n#QRGNRYiO*#3#bft384D)RkcWr+Ab0$4Fa%K2bk|{;Yll`UrM?g7h=W z_BYOL%dyjU*y{w zsWCj(4%{|K`v4sTsdh-M@D}C+P__e+TH{UO&B1Mo)JE^ETS97Yq;`1ocQf4nm{pz{+ z+!2)Cp`F6Fok2U}Q;csrfOgQGQ1>}_i+cs6+IkPX+qynzeO(8S$EpMmS4OIbZ|dUB z^Rdmf_8AAx>^y8jY2-W(C%9QCnLeWzN2 zwnTk=i|B0z+6?uv5OLB0w1a*{U3PNw{I7;ij_<2;Sp4E+E%IBh9c}T@%`NhOe7Tq} z@si&AwR8pLm#B4L^ zk@&hjTjtk&u{d9e2mRS1e@X6StJ8AodvBxjBIwhXo zWS{)+W{tJw@?rKT>Lp>H5azO!#%vGqC1yRaIT7`eh=8Nr z;dOL>G!`Jms~b|W8Q78#wOjjMCpQ0 zx8!~>XF_@c_Od`v9`q!n=gr2i%Mh*_{{qGO!{i{UX|kWx6SJpUsvi( z`p7QMkMqx*A1KZ*$DPTJD$rZ{T)N!+na`xZw7>o*`=c_}B5GpYp(gIPE1(|@K|`z= zMDEUiZ+quo19#nfVl{zQA_9CD+57)S_FkOU1tR;OpAA48VEv*dbmoBO;10bGXeH1} z_CLJ$fcE-GwUJ{(+;vw*s){@RX1KeqhEyF#W9X=kR0Vh8dAL(L0I4Uu7#xVxdrb1dxG}FdPYaw>$L@K3!PnY zueU$w{`k}b`dfjv!b-vcST#5p^kC$F5LN_Y&=~h~?Qwli(4P3j<2uIs|EqZ$&n@D= zhGphq+Z&IHc`R>I<&w-i?7a#jOJJT)kUZN%dUy^&@;uKVJreVHOxH@x<2UJ%n8#z% z>BBshQ(hADJcD#f%ySg7L1La~kRBiAISSb$G5dk$VeH$|b2YZ1^jwY0E3bvknRyuO zOhmn8=3cO&1Qzv@usx(l)Jta0hccGHqAoIXCd!v>BRvv}y2#9#$R5cTb&;8GQNHX4 z(kZd1i_F}SY!G$f#iA}U^GLEMo3TF|89A=Y@m`MQzHw2`C*(Xs&LMnbwr_0qjpaTY zLo4erNrD<_@u=ASky}*>ZQb-Ta>HByj;{v zBFiZ1<-a_a74=f$TB%>uO9>fKFD2~p*)Mg9dPz#pNBA?JOV3m2^O75#1z4A=+FUptC zoDZLA9!U9R&Y0(sJk9&$ndXB5^z*rNx%o4n|F`*renm2vNg2$(XtoyMXn?s>W6Y%* z;;4z)T3yVw==Yz%@*CoFW2D-c#p%T|9z6J^ zIdPx0M_Cyk&fV+oxYA9Xt&9&>{9|(b)r>lEZP??(llGq!zcu`;6v_B7UHk4s_gNVq zCL8L`=$WboJw8lj$-nloJd*L@@8Oez2@mIzj2FYtYVmzbt&9hM+w`N19v>cC_v?&3 zK0N*TUoz$L;gw%*%lOBK`#t$j#?L-X*K&RNFxkNM?!#0TjvpT;pKu)d@c4~O<9#Z2 zjF&IEC-z~A^R0usSQ#Jw`P?VtgHLH~WqdgP{Nec1L-w*VK8$yw$Il*G(aQMnS%c=p zZz7(T!Db((Yg-++)XMm93if<;{GC?Dhp8;>1`e|_K0E-^)xBMZLH_}Pc)+UHz-`7qhQ_3pz| z7LFeuCZBMu`Y^}v3H_&J;+ZhVZE4K0N_>eqK8Y_e#S)d5_!3hbQXflvi8=OIo??

&ho8>B~KiU%&Q#3ZxS z`A1@|chVy<*ER7a=K5v(son|GctL&I5!-Ih3V-q8)A7Jg$JzeGhsT}rRQ%p4gW{&k zrpDP|^7F=)Ct02UhUa0PVC{r_HhA3&L#z#?b9b2M45U9BJaffE@fB-&Tbq5j#!qwO zc-|4#hHNm+0V-mCLN;WBXHUO39zXE_Yl9CbOYVthw(n?d$OiMAhiu3OFB*PV{O5T! ztqnf>`)9YsFAmw$+K>(AIThKE4Zd*eluWLN>U|!9Ql&g%9tw&(HC4*DkWbbg#jELN<8cVZX)m zT>s$1U;OiDrhmu=^F0yyAsamGtF4)S(udDo{CB3G%m(v47uk>vrn2*R;losR9xt-N z6azeNW`ijPc--`1iUA%sv%!4NNPfr$Q(NHqgAY?%$anLHY%t%Gk`38l>JxY_=fl(| z@LVn%Jom`$$s=w($A{m$x?<5Hx1Qs}4X&(Sbf{a;@!^M8)+yS^ggiuDbOHoEm3 zA0FSgVbMyrp5w!G?FVi>$A`&=Ms7XFhp8<6-Fl7>lTQ}9^&B52->L=0>p5OLp4D5*Y5AuD}1;W?D@v6=lC#{rK4Lv@nQ1YJhz_X!{oQq+Y_W@g?T?B)-HPlf;*plx6+tw=;YR0 zB<8pxd5Jleh%Yh65Ah`?-%>miUt*3E;!8|*+S=JHF~5i-&A5-&0V#$OfNs#Rmzu3m+amb!AfI+C?^)?^(!(Z19Cgzm;(R;KNTWe>LI$ zAsbBhMBGnigIj#`dS?FV!~0cw*~(BqnGNQ9F7^Z6i~Sp>vTt+ag%4BNdA!I5Qw;F9 znGL2G;BnK3DF%4l%m(v4Bb7ZHOl^VZ4?aw7p_g0F$p-U1DcO(>rapn^az0Fb0?*~L z!L%;z_U9yCymL=Z!oMeCKTiqFzo`-5hw1kYw@)MSVftNW17ec&_%Quu^fu-jH6Y`| zbnSbX1CWdllMVH;oq{Kl?E6!5|qQ=KUBXMT5np18tJzpHB)-H{mVt;X;!8~aVR?zkx3vF<|4ue!gQ<_2RlJ^K`>5Sv-jhQ%_%QWLMOe=v8?wRF zhq(0|+n;2EX-^K>K-c1+M(_R$XzehIsFz?AB8?wQ? zCx`mUY%uM~;eOJGX-^LKli6Ut=OP=j!Blp){%`$I7Un%U)K_PN`JRq!$Oel&IW`{L zUY!z{_vDZb*VP5kj86W073^cd&Ve-G*pJQXbEX;cwNRJQGJ_ol?Bk^J0 z^FVrhnD#%=J}8p$VY;@BTf^~T-XlR{mJd@|I=l4~ALjiMq{oNJhqOP3WPF(SQIL!e z^PUQl@nPOyK{7tfdo4)DhiTsh``L$i4+hnR5A)s(lJQ~Qn?W)@%=zeoybNv!uVjfRv zE|d)xdvX#Vrr*0i!2TSX17w5A&u)K?jU^wZ-+jnVlFtUOck4MMzdOux2GXAm7JG6M zAEvzr2fOv0Y%t9M=ywOQAsa0Acg}rhh)fxY%tH+=zc95%zJXE|My|ulSB7w*_Z?Mvcc3A+jv2J<};^^@6P+LOcmqz}`c9PTHx!F zEcWCiK1?yd<7PIP?-|Jt*W%SEmH#JvrouY%t%Gk`38lu_wp&4{onc2~2$g z&*iegcx;gSD@bbX*DN*Rld<+$yufM3ETQoJW66K@(r?m_)IZzGOBu;`@5L64= z<;w6On~+VVWh5W|EUVujKWF$3r#*Ts-zVejN$6Ua?=E^=d993;- zFEfTEztpv5#wzl)zH@e3Uzf(n-`YWZhplfVpX!HwE@hm4TX#~1Wa%^MlrqkK%a<~e z@5*ImBzFGcYcsLre9L*287HoexLl=UK=N%pI(dsxUXr)i<&{vmq>Qu8@}-Qk-}0r* zms@M6mrffR5B}-Jlq_SnWvkMkFF!8+X3L8ySzgMo_^%*-_M}mT^KN)P_2&3t@dcl^ zPfwZpMC!~5!%Fa_ekmjMvqoxP(3!_NrGDs12|F`=uhcK?mwM#2rF3S>F71)!JvDE? z1Sx)Q^NT52ztUz|t|2#1Oe3YPfggN)nU~ZrvCIqgkl>mW)rE{7X@krc^^iv1wk~{p znU~ZrvCM1Pw4upYsDo1xCoT?ctV$bXzNo`A)d9tZk1z9*`pdxS()r4^AZ?KOI$TIL z`1roOoNa00bKCBmytK#ir46pUX=)471|MJMCH1>HN<{40_ABLO+m|-DdQVdv%6xr% znU~b>Vl@%{fVDx&%YH`M;NsuEcXFEMD{tAVu{}e(>>SUQ)lrXoH!&(0*g|liWXK<}=a;nJ?-;MfL6a$&5XcFY}W6 z9kzY44mnUt6Il;MYz>nF(%JYJ9=cHzssq<&XN zJVsMI#FUSecXhPOnC0r-j@PbUY`#9e%uDKbvC8u$Y8Tc9Deq#{@}(bKyVxaPA7ADr z^}GIo=W>}jl9YFCb{895U&M1cYLC_)$(Q=2ZBoA|mz~>E-4jpF%}U9nSw_ygq>SXd zv8J^BG}oqN`EE=sUDxI7IXmy~GW~WbKWF$ZR+F9I6UwZgSibD{cFL8n=OCV~e|+-+ zIc|FVZ2K^ZY4WSb&%6#}`*zpI?$XaYzHJLW8K+;$SS;$4?TLjwyU57C#g)s-OFeGh zV);_uC*$ci?6^whq3dM-AoWN-`I2nf`8}cX^&ILe$rG+#7*n5Yv8yACU7hZNWqe93 zPLiDdUSP$(OMtv5XUmp+6Hd62qQ^>~QvHF!_w~BYqi}%S-YW zI~j|ee(PJO-}0S3>~pfmVrRd_QpVjc@EjmBwqlGfU(Z3kV7^qoo&!H%{#?GEv(pcJ zzeerXwmW>~{t8O0pWyz8*0uP4jn>)HrPq0IUqI`1_CD14+3I(($M?vYwk+jk+h+@C zoWlJGtt*zV=M=9wm#^oDwIg>gX5-M^uUY@d^&%S3Vg}UD*3S*lP7i ze@Z=)Fa0THB;Vu1Bnv;&Iw#fRzxkQwo)i!N=4UR~zxkPcXk~WeXDj1;Xt6B2!I z%CPf$LIq?`h7a3q8A{7YK6KbJl$Mcvs(*@=(lV0oaJsaN(_{HQ8D~#I*Gl=Fx-#Xm zGE!d3$k=m!u<~R(JqI@Oa$xFV) zPJc}DQpVYA`BKJ}%krg+ddk3Q;yZbZ zos7j!e^Oe;*<<-q#@TQAQbw%jq-7b!dQMuFk@744D@f=*mF8RcZ84$yRQH=s3BJ@X zWu$)ANNrI39#mQn@6RDSGxG$gU)nGAxcd}Zv$FTrY_q&p+9S&=)^pObPNmJVT<$)a ze(x*c6Umo(N&OPbyxjde{RT>Lmx(25gVc|DpmhV9Cnr9>%uDK*Smq_xbEr*Z;#S%q z^M#Me*Yta9;^WJ_r2aB6t#6dbSGEOdgUr`qT1!cMd|zJ9HdGDK zXQU0TUF?#tk1z9*`dvGvy)oQJSsSFh?9-$Tu8*Si3T}_q9?6&brEOBb>qFQE>VK>a z@>8Oz;K^t?QwmS#Zte|HYwx!K=uRg&!IZXjK4k`sBfTHr1l8TX|PQYYe&_vcW(WbBcAnU~ZrvCIo`%lmWe_)IaeGk-z_)E{Q@g%5Z? zjF&I-lKLe^8_djS(0+M;4)s@=`OHo`3aCA1@9fJ*Yx`$ugOBga%h|?b6ZygBE9IpUVWS`*S$*LG3q<$Bxv_FU2ptV8DyI8e+X@iS@TV9#3k1z9*`d$BE=Q(nY zB;{S3-Ngpi5Aa-$_vdi`VEIzNv`y+4<+Ag4s(a$exmhV0+fIl_?Um|j=l6sPY};bK z8*6rvk$lLOHAkZT3>iM`Dr*k&|IPnR?DUac6U&T8o-v5)BBASC3|K7nNWSb>rHtfL zolu?a{GL$3|I~dWHeM)hsFc!I^tt#i^FM}G##0wI1}PTl8tG4`1gHPi`=Hp*G5N#O zZ;;IH`VI1PrazJWLn%LJ?%8GkAY~-q({IqanYEj)bGUT7pmj5A)4$0$J8f)8d1;R; z7sn6TZ?Tk-u_xuh&-4{Ch9$q$wVC;(j8%zYXQm&J7=DOJmn$z}JN`<(lb3vpoqihw zQpVYA`BKK&Z~0P2^2rv;o9vO;`G>D9GiH!inQ;Q!YrbOF}V2tMfIpTXITP}>zyg!HU<1C-@EB-5A&tX4cZj)Kd!hDSOMU=1S zl&|O5`(@l8*)>G5uC)sv_XWH^hwjU0ACz4$!d#v9hLx}96!%fQmO_29?Z;dnwaZ!) z_fRx1qI+ekgYxy9%>Q(80_ zjEu?h^&Ij>W?jp!W5{(RxvnDla{Z)yJ;(0PN&oj;&na+o#qx7;yD#@cxgTtwSKcmc zAe6U<^7c^P9%${mJPyj^pga!B+d+9hP~H!e_XFkqKzTn<-Vc=b1Lgfdc|TCz50v)< z<^4c;KTzHel=lPW{Xls?P~H!e_XFkqKzTn<-Vc=b1Lgfdc|TCz50v)<<^4c;KTzHe zl=lPW{Xls?P~H!e_XFkqKm`@ruAsZ>f}^842*&|xe;oU%mN@oSO>i_+^>NfuwQy8d zRdDQ~D&SD|s)MY2ODkF5%2cu6cCq{>;Ovi7L7lG7P`82JrfyZos-x6K(2eRRHB_Ce zQhF%T;d%ggKdAvof2(2m=5Rd>X@EWnyuZ~+NELN2m8<5eUPw#Ta7Yc%!;wzc1Mz)D z(2Ba2j@2kNil}~6T?(nw^`%G`=u_1|RSUG1Zmj#NL)1&EFVad#U#c$9qbTVi>QvPj zw6SibkHGgU)e%Um)mdtYYNgLY>Zp$bZ#BxW8YMUcwk!o*3Qgxis-r#^DWwBt)B`F& zdQwHIqIw4O88t^$QaS1c&==HmsITpo)V1n-b%B}+I#u1Jj#DS9n?P?;*QSQ%YO#+>yCaON_ zP<0jPRcf3Xp+>6vK<`u2)nIj+nhZKw-J%XthpPh60(G_ej~cD+2fbh2s~%L3tEWJp zQV*&5>S?tQbfKE1mZ^E_VbF(Fp-R*vYBuO>wM4zFma8;qTD_`XS4HYs&}Y^2>Rt7U zdJXh7^}hO0y`f$NeNnxsK2z_h4?sUqYt%ZmO1%yGw)#kYqdr$_LD#Bp)%R+>`WWAiJbT@AFFZlGK1rn)(3bKOQC zpj+sMpbd3P-AV7S+k&>$U353yPVWo4uRc)6y0h*I+EpK{d+YAHJ!pH~Lm#2@buZ9f z`bd4W?xS--bM>M6MBPvK2koy<)&uoX`Vi1V^l|zOeS$s-^dx#;bV){Af~(2wDGSkJ>TN6*4>pPr86Zha??+x27|H|ra5 zT&Kt57;CS3%*xNQl6P8}8|}A?EPpyU^N_~t)%tT?C0LDAIe1IIuFnKLGq@<&puf;d zL6-(E1uOMi`XbPaf-%82dV_um6x`r_y;6?>9TQ9l()wB5476F$CHPK%qgR5i3|0rL z^!s`O=!9T$uuT7?eUVRJbEkQvr zL(kMxL8k^c2lwj-^aDhLdxF_|u6_*kv0zT{pnga{1NuzxL@-}Js#k!n2%ZZb(NE}i zK;H>o3?A1{>Q6yG4c-bC>ZkMu&<(-I!D9W4-U7NM_%c|c$LThpZGsaCz#^-y_uxKGvV=9MGI#kKjf9lKxx2 zh_p>_*1zb6!DgiT!QMgZ;2_Y0f)+u`pncF5v~AER=o&N+_5s}|=pOV8_6?eVHVHZe zy@GB*56~V#pWx6SH|PY~DaZ?s40;ELfF2SY9UL1R7Q~=&&^I_aI4U>>^q62^FgQ3a z=m**_I6gQ#I3*YaIw&|dI6pWw7yvpT7!nK*&I--}Jtw#%_)l$3Y(tmISW^%Y!s%I(RL3BPa@<1${PnA$Tu% zId~QH)!>6*Rq%T7Jm~Ym+rj6-yTSXQ?+0sx^}&b1o1kw7p9J3op9O0`*96}N-v{f0 zk3c^Pz6yQ`z795lZVG-6wg%q?Ux0oQYz|rlKL@{p{u;Ck4h((^egOR;*gtqyFVp*h z?iVx*{s^`O|KQM}!BH{H!BI7=funX<7e|Az5ss!|a~%7Gt#Gsn+u>**{()=%3FxXE zD__@2Hn%eE?6=198-Y^;>5t%~@P_cl@Fb*D!o$N&;Tq62;i2If;lZYhIV0?1P7C{o zzv7$zNQZ}KhyBgL=4^a>MtF3%1K%8t)IaPIE(-4jy*KO>o)?~E`kV70d3JbQm}7Q; z?tom^a87uU>56nwctLokImuiA$@9V!!a63$oB+w=!u;^nun@E`%nL7qyE!LAWDO;gjLgP$30j7%mTA0(~ir!V2Ll;k%&khLyrS!uP_@ zK|c?xgq6e9;Wwb)gw?}p;l}V6&|kt@Va@QT@E_2B!aCue;ospNW>2L0VZE@Tsb%Vg zwai{&!*Ea2*fb0qo5o?2u&HTfnuM)PvvBWlf78+I9d<|%>H5P@NjdYX&s(u4hY+Z1I!twZFq(`C_FGc-3&7ahQmz9utRu(c?k5O@Xw%E zczJkb*bC{9@ZxZ4_#e>!gx$k|;Z5O0(23#TuuphJI1Y4N*f%^iyg8f%Iw?FoJS-d= zUJZJ6ctkiPyd|6rIypQu>=#}YUIThfcvRReyfhpEIwI^Go)z92-VS%$YnI2;|00UZ+_99|gS8Qu+gcQ`z}FT6i|4D_+^!Ejc1PdFcRe)xE} zID9&M0Q7-yX826_L^vIEdN?~whehEM&?Vth;S1q&;X=@b;j`gu;Tz%epwEX%_*S?g zTn4%_{6!j+&a!%xER!XLu* zpzFi6;pXt`@KexF!!N_%!#~3BLB9{b4gU&%4mW^q2!9N*s;O)mn3|>nXa!Tv>}^_@y+HRe^~^q|k*N(@ z+cY$7O*_*Pw54fo4l?_hCZJ8szNV|`ZVm)J&>UcTnhvHlXlv8n^frf>9-uu;H*=Ut znNFacOs+Y~9Ao-`_A$N8ai*Wi1I;ssnp4amb2RAD=16m@Il=S=?Q4!TXPa})K+u8a zWOKe5VvYwr-V8Rw&86lX&~waL=0D~_a~kMr<~%bud?XwJI>KCR9u4P(LqUg{OH3#9 zDs#}U%s~$_(u_7&;238La7-}Q*&MMw<5xDk6H{nCL_Ek~t?D7` zxz4<8u8*#a-iG8GX0d4+ZHg9~P0=Q^(X5Hyk2XT`OY^R|J-R-67m{zA<)%;6Jz8$M zN6(o}kX!@VHIVzj+#B5!RDtbdHTRi$(Sqncvmmn{Q36Xfx<$^R!uI zqNqZ&%2bFxHbv%l^C#$^W`+60RE#P`pO{L~YV)G`!)ycHW?nI$nLVPM=rfZOtuZf~ zzsz>f?dEm!l=;zo5Bj}%(yTR=qpHzbQ#Ja+yk`D3J3x1sx6C5*gV_SQ#XMuyn<`QD zXuYW(Z7^?|e~gOWM0&?OYko36gZ^xmny*Z?sAlw)sTqA^RvJuCqm?Fz-ZRV0FXmU! zUrnL;+SG_@M_-%T(RZdn)G%rlH9*=sY834mwFGS$wT=#q+DDB+8%KLZ9i#oDx}bHV zCQ*+lH|hY|Av!2ZMV+Izplzd0(IL@cQ66Ys)H6Cf%8$B&c8%ibnCQ5uFKFNB&}cw( zMAQqkSJW>W9Gw;&4|;rbY&0Y~IqDDEKRO{gFB%%13VLcZD7r8@D>@PM#OU)1%qZ;^^tiOqqn1%qvt@Mi(ZI6j6RCq0DU8RE&3#SH(CL@B6=%Y7kv?}0$mk-5PcPW z7QF-dPV{l~UG!tL9&~-QHrgD09eoP=Y4l~(EZR5v9`yU@+i1V2d9(p^L-YfBl=@Lq J(5C1+{tq}!c@F>p literal 0 HcmV?d00001 diff --git a/pufferlib/resources/gpudrive/GreyCar.glb b/pufferlib/resources/gpudrive/GreyCar.glb new file mode 100644 index 0000000000000000000000000000000000000000..d5f8e768f1a4a61d0d5c3db5f2f757167b93dd43 GIT binary patch literal 239892 zcmeEu1$0z7)PIsUxa(qtQe=@*+UZp3ycBn5afbqBv2F2EDDExp?(SNcmm-V1!@}Y& zi!KiTn>X{|4rP#YzVDpx{LeBcH^1a1d3njr%}stl+YWWgF+xb@r3$%|myk-et5=Ki zXb~A1*3~08%A;*qr?7|?T{}m31bb9&7ty&>=ScUqP{qAehpMgA5U*4iVukw@hUwVHKo(KRe0yhVq| zW?>z}BO}ARhc%1r8WGm1ZP#`lO*}m!TftjBg1sFm5~@XcbZXHN@;qwm&pbUkb#5Id zA9y#3h8jjH%A;dgWIMg8qkQQW5zw!Q&Rtt{)jQV6+q0ODr?;oY%QwKs$2Tz09Oxh5 z8xZ6hR?It4Z|hU%^w+lT+^uDYu-f6RW$(L2bPI!**?Ur}Y#C^|OUsC=VO?8v=n&qj zR_AVQ+jRR=RVC&JpEWw9<#Kk++wx#p3U6G5h%j_y-1=y@Na<%g5W_ z-`CI2$2-XEV+jb9S$^I=e*S^bH@~1D-ynZaANatIj&~TX!PrC?ultne1+{#X0h)y_Eq)jFxH} zt0maSOOBSIlMTvNsZzb3dzFgiE7bY95rQS*4So8~$k(V+vQF8bYuL*i2!BAy{VhR$ zenH-582KN``X3l~$3iVtrDW;KKiO$-Uvq#Z$lDU+>+k1h_O{4{`D1zi1N}B;`IEkQ zd%@tzKLP$eet|v~Ggz=6$&!}QZmjB6s92}$|AJn8rK`T)`X3)3Y2wU*V8CGLynVo6 zm@NSovl)61CReu`-ae2S5ENhz0E-BP&7V6A*7bj(yN=PQQ?XXrpEi8n7QX=hARp+g z+1nrNY>=P+VLzJtQ+fg8AU867hx@ekmG0Iu%*)%~B2_seykmG**f;HaV~eg`Bf?vD z>nf8`9yO}hu2`pH^=kSySgm@kswJy<1p9b;)G1rPboJ`B$~5!#2sVSw2=CN7yp^ot z3vYAm$WTY_8Lk?%czPHO`Z+h~j~fiY4f=em!NAWA201tA10DI?MIY$@=RQL3UH8$) z{JBjZ_%!_4J{G4&;Tt|T>if+`{ZXUvc}|T6IyDO4@wriWpX;8QVHY=s6PC17qh_ZW zGCNI?-!~gYb7cNvlFUxCWOkY+c=U~)qIoiZF;Tuw6XokPQ32m-)E7;Z?-vv0>oifm zP7@Uvt@~*$T7mLLc8;(wftKCcvZDXdf1I=#Ap+vw+Gh6!xd zxmzdjYJI!|^)K%U4xZV|51b`1LcW3C(xnNqc!KBZYYqzV1=D2-3<|J-1LoxeUZRf$ zTsMC|umC{;O<<10!Od+{%$KZ4Q#M1zpn)v-_8en@h>bsZ)Sh|T@_&f_RTDrxuuLe+9 zSPlL`-oAb?V7^lKgZ$0Dfxgm%gR_N?Jdb$!1jyy!8vrZ84~_#tpQpj>1x}M(4&HL- z_wkW_6D$~5MnOKlenGIp{DXY;^^mxU&%=;-JwO+~{}JD6zz4?pXIe-+6LJem+`{Ln z2!P|z_vS+GO^Mt1JQ+TLpO%EQEZ#vrL2w9!sQ``lmi|T%lrV!K@&F) z;PmL{4aUabA5R9X9q_sQAd)LXHW1+J1E)4vRA!h=i;u;#*jHQl=1_jsG<@+9-)!J} zhCzaAPc-6!by(+(6=akltb99(_vSzzJ`(G|PtAc`DT$l-{0TmA8)TRy@QvVrApL*4%c*-ifv_L@TEH2F z>t)yqq?Ll35Lg0!7H_kkS(-4sLtD)LaAD#HCkuZ+bD*yHV8G1YzCQB0OkPp@TLS!a zk1260-#xYG}I zEqiqht72ygf_>mR7y2l(s<&yQGZxZuXc5s?k{7p+K_tngFX043&6ho%qYTk2*L2e0mY?t!=C{=;ZKc(JQ;t{0+^J5pXkKbTJW5AZR=?=Ow^SieZZA#) z2|hktYW%Qi_(}u5KeepEEB>+vK6D4CA*(;cHERgZ5J-a03Ka=D!YalT@6RvRs=YafiZ8PvL-N$f=OXqH_JTBYEShn*xdV<~`jOS&I7B$`0^Zms}ag3KA zn=Lk!T*~JRkLDQD>0`yZ;B`Dl!$BP5mt&`kymyxJ2g`x{rmZvPX zkslt`lVhCcSpwgmWefi+=_roz=v+^Ey_Cyz9l&_a!V^4W@h!Ye(n0)v)D?by*=k;4 zMQ@kba`!Ku<>P7|7d3=?mOj9bS-0|_u;DyY`P2OT#ML~#7|GMkyv=K*na3Z&T%;@h zkQdponD3iCj5mLHgR^?e`154W*yxWO=Rv9N@|j(i>*I`ZOHw?+Sf_HW$?=rsJDfEq z$T9p9r(NuqAjeSVV{Fuw^JTPYU}KJsK;1%>Z>Tc=ODxOm{c{*!w#1S!V)9LOb%Zl+ z1^qm1JFLnw`OcViEZdp9!kqAE(`miD%$M!t4v*E#7+BVo@0Ix&%ewMiXWEa_+xZ$! zo~Lj^gWvn>YkB;lf(fT4jnU`2b9JwT8o6S09GkpQ!pIB5c-G>vywLih2}xeX@FZX0 zyvZRy_b_hmJ&I!-bG2}S=ho4@9DEDL8?*kJu)J0*Z@6F>$9P%EA_=QAjp7H|!uYtr ze-0{^VCoygIn?d7$TQ*5uAzEA50v&yP<9UG>!I$#nqCQgq5pCo_b(}!z$TB;aqgX- z34h)ks^hTdg%d739GwVfDU>brRnS~L|Ly5Kp}jMW({b{TIYU2`ot+3zJCZxJY5Y_@ zALA~xK&YG}*@wop-9imLO#UWcsG(yRw<(f0)X+(cjk;1Fj5Z~f`Lcbv@8m3;H&p76 z%x?twhAzc@bPqN3XAitrmX|svv7xK;-g<-@Iw|WKx+L2)bWFBm=#Mcb@?FMQNqqa) zY@zKhjMe+o{Aljbj$5Yc^HLFTiB#kC_gap*g;r@Yl}r6k3gbMk=xlp`-scR>P+_)S z-t$z>(23<|>Sbm@e#L*s>N@`~wEup{R2}DqGNY?a^h`{aZAA?lI%jVcuF`Ybr6e@8 zO8G`kIRC8|+MmNns94;HW1Mh4hiKaDwLaGv8-0^HQ0rnUasS{4z0AJqPc%zQ8Y=S} z-HhR~{}PvoNA@S7-A*=g!k4p- z=G!m4=VWP^ZVwIL#_(sPK1drY^Q9kxv9zl)U-}^!w_50(Kt0;(IXH zey?~!7q7N@`R*5sCFHEuR&NvI^6hejZh6y2FK_r!vJV*VDVaNT<=ZxTU5rCB z)jv1#x7kMVAHgd;ebvJqH}a)9NAVxQArl6xqo;1Xg5=NIF7=H=?(1@?SbSdM4UUiLEk>g`nxTZD1)=S@H5-)cAg6OKr`S6!WyhTg2% zh-3V`SyOc&V%Azd@A+W<%c@0U z$>UkNy&9VP6yMr@4Zl0QColRcfwS06{NvHVykg8P?mKazUZ&sC3?lOEa~->1o5*Ly z+418F#nrVZmvQgMLpjF5Ggqs-i!A5&9u4IfC#mDB<~*^UpSe4jW9++Vtvc@44Sdb* z!5rh%EtZN|mlp87^BQnhc;M6pqE&{)yv2fE+!bDxeTjI|Z6U7`(}81r`fz{oa_TI; z_H-P__(b;EqVt})JfZby?g}59K3t4`KAUHl6USZQAuC6SRL|z~GHK$tD;$4nKks*B z6<_Ba%U$8KD-w9-rSti;V{zOSZXvGmYu%>t7GUczK5xFv=N@0oXEhwnUEzi04{^`> zOZjjT$6evA5BG6VdLjRFN*s5Ei)X#dTTK9ezI9!W@wtC)^W`2Z_}FUU9OM1_Z}9bV zSMr*Zx^q|9*q39>ML5Qp`}v9cnIG_9{p)jA*z;IM@w@tlkLuKnyTTJ}sYLE&ANYwA z)i}oUf6pz3On%A#YOivP+j|xg`QxAR0i~*QjKkJs6v>L-;f+uA;uu$~IA06u{(?U& z5y4&IxC=Kl?-lQOuA$)^<8^^V{B!gX@2zy_823DROq;y&8Q)bRl4HElZ!ur@?>#;z zv;lX84gciL&LZ3u7S+ac_wWzAm0xv^@rOHec$@oA_?vV!IL2+O%-}ECOTHo%=NPvv z*nyuJ{DSvb7{M`4ePJSRw&*sW=n=(T;o~jFaPx{+JcmaIj`43ZhVbVb-|^-L!a2tI z8~5j1mObO;4@Yv0H@qLk4^Dc-*WT&QG2TqysI{ig<;zaQag6JZTCI+(vw|mq{pDxz zy3+mC{vLCA{*!SWwNZa(H!Fivs3VoF)z3_D1y7f{@eDbackf46)VCy z#<$}StLbVz;Dhe;;21AW`9O_J`HZJ{7RfQryJeARpK3NQu&4nqI%|e#yJw_c(u(*Z8XwFXi2Ve_Zp7 zZ*0+q=Seo2@A&fp_nP0HS7_Ov-#YVzrykyxU+|l+#Vmcsn>-8S_cmn|55sQpEW*RMX|8=kh|*#n43lI$T*yEU9QZ#pdG*v{GfRP=EE{{D7rl7@46M1|qJS&MAy z!!`@~txdz6aL06S)RwE}@V-#~a{Lz6d~6}F31u)|`sSp%;PqT?fp!KhO~DJTc*fs_ zw&8c86V%hyukwl+|KRa%4^*4q6JB?9Tkdw~uv#_x9)GpqcODm1Q=L-hDnI{k7e1`% zP;32F&v=&1ZFt_5rPRU$?s1P1zdPY>f2UQK1U=z7W7~3!{oF{xxl6-!UpllDP4Kmj z<^JH4+=^rgb7xQDE9MPz!jbKigljve=;uBQ)HTim{(!fayP@`J7t6oI4HFKl*Y*t4 z^9?NDE6-W-Tw`GA$I3R1GnB-}S;=^>EMwen-)Wln)BUZ)#vP-?#=W1!N#SgJcgJD% zOI!)gwkgWo(DN}i?hj@89B?1?9PZg99{#9s!r)7z6X8)?Jrm0RHB`_4w}w|j7r4uk z<&C=_i9f>qjL`>-OT*oc(Fcr;yCYfGxVMpb!H1lo^UKZF+sF7U+-vRLI#tid*tq|Z zb&dNCiQikFXbl_DP>GFu35jPt8P5CGvgb>_K+ciO$Jn?7m-!eQ_X#rp>{9Q9ZE!yz zv2m{d5*ufBnUAsAasHM0v*5gD^x>@Iye#vJJI+KB8|NU2H#*Ko60dK#RBSl3K<@*_ z`_?WJ{oc&d^JCK8<-KE;C&Fb1-QwxzF4Xf4|5LV!v0=w#zTs1Si48j@^E(_%CC+Fc z^fp^H7|%EKO^l7TAj=y%FEQe+1mdlP9Dm0e(XnIg=w%#h(+L~%2e3yfV(%UgG;+_P?h< zPXc4aLkYw~xx|Qv(&x@$4-;Wy9_3tPY|M+y$Jm%Vng2bEcqM@`;*|u(hK-f;i!tJu z1mc)nV#F~C#4!oPF$u&m3B)l8#4$Nx!*0vI8TQ}6#<(G_M|_E88Mtp3i1%^Ai1!hQ z_YsKq(eq_H!0$L=#PtY_5$~h7 z1jdLj5EvuAKwymc0^tfHzCd7%_yU13;tK@Eh%XSXuyMA9yEx$rBfdajjQ9e9G2#mZ z#)vNv7$d$wxWb4p5EvuAKwymc0)a8&3k1f9FAx|bzCipeMtp(581V%HW5gE-R~YdH z;zuyz3xq3-_yU13;tPZ;jQ9e9G2#n^D~$L8;R++ZKwymc0^tfHzCgIbh%XQrBfdbm z!iX;r7$d$wV2tcwj=I9AnrgIcW%;NA?`pR?m!^!Kp^fwAnrgQ?m!^! zKp^fwAnrgQ?m!^!Kp^fwAnrgQ?!XBn?m!^EzzHMnKp^fwAnrgQ?m!^!Kp^fwAnrgQ z?!XBnzCd7%xC4Q>1A({$Cv4nj%CisR4g}&05@82lpywOdxFa>%G0r%MFZgu#2zRGW z*toNj`NkcQ#E2ts!iX0T7#nv@vM%BX1mXysFyaUV;s`$7fyj0cFW`g`FCZ{B?o4D| z3C#RG+!T@PRHk)^y6*b#918;kk0dyX#I~y>+NF>aU z9gHIe4AR`n_t48fH~9$&I_|i>iP%br6CQBoiS0!0{kH1=_7Tl$W)}x`mf>$Iw^3^z z57mNq^x$`L7Inh&D&E_kD)_N=ihEy<@r3;Te5|FosF)|88eFZ6mhHtQ^+I?Hek1z@ zYf9g$*0{jF;!WfGT6#+}UhTZ6de1wR7}!665AA!|2^VYdQmZ_pG#`2Of{O8yA!W7V zTdMG1;xceocw&LI+Qf;~x#yB6>W|=7OJ3VzZ8z1C3tDoF7$SF*t6~U)q|(> z`I+;ush0ar9xtlymHJhcV=7kk-p#{I$80?L3Vbq?s$)=Ne-??*?A}=(+X4 zKPPOl2l|K>*WCEUoTa(@{EF((>1EU%nJ=lf87+9_oL14;t&kWsGN<}sTcFCH^xzj} z6?ejg$E4@)w)yd5F=y3f$NyGyUuwxq9rJL)ksFe0wp0tOSFiLDEbU9Rd)hMm!Hn~2 z#d1qkkLf9RpGoET!Ftoxd0SKSro24&t8vXbebHq#Y(#6`Ad|nk;(8^1r%o2Wd;DTG zMdRwcPrlbG#xX^*sjUjx$0>ZQyISS6eaxTVZZCodAJoU(u~vi}C&&8GBzCM1J>M}l zi7>1mA5gSYz1Xae4w2mEV?kO6JcI)11XOh3RVmxnGW6>+=YulZ9Z`JFi z%8A>_Z)y;nFx0U*>Zq~}iDf&otfP(!^)=ZJw8t@)bscp$#*X*M`rH*>o{4L%D+j2h zD)->OhW6492Y2ViDN%!VbQG)GY*m*eoo{9Ng0wvUCgmqa*WvS~Y}Gur*HYQ~c6`m_ zYNGMZv1;bSuV>m(8SN{mXOudT7GGl=sswG!c6YB{w)p(s7S!~EVnZ|@pT zQAgzLspr?&@5>`q|3o-fP!E3U-XOjFyGPOd;5n-6kF?QJ|D-NSTP$@>Vriod9fMev zI`?H6sdI+Db=Vih&paTSa@?ezbW5V%_AzM*%c3~UziQ}=)Eh&Wke;dit~cgWD?PPY zXXs^R9bH$EeyFmpp-Y#ZzEv0Hdtob~HRi1|-Bc?iy{XL#EywG`^yJeTS65@&n$+{2 zrPa;jd-7qsGO3@ zncFT;Z%nI=n0f#O6JZ6tvU~A#oPtU*j#z^=*G zi*urR-i&vxXFg=+3oi!p7hQykaZdMpYL+9l_`z4pRg5Dnovj_$hx3GUM^ucrj2)?# zs@09JzrWgg_9Uyec)hyxz7iS16TH1i&%>Bt23L6Yq{=QUq(MtG49bILYz)IQ>7jU ztk)-+wO*#O)>(U^c#8JT#JIO3)p|LPs#CWw)IzK)tnW(x!DBO&6Z124S9@m2%Fmol zqF!n~C8XZYXr8@c6#w&48uiziCKcn01qZ3~)?~IWDcFO1KKA7my$)F)RJQPQuN(2g zvtL=Q?LMfT1ERU9Uaa-tCX*BHup^}!8&x303&t8_dZh^uis)rs8~t8wU4M|;?&#PM zQmO|ZxWU4QY^-TL{g*|bC)a)>_9x{`yY?I9Q~RH9m%)DEk6>4R18NQJY`y$ZDnrsR~OFi!7&b-QD)neG%@O}i#<5T2ma_M z#+B%AU8qdkj&YJ%WwuRz6QdT>dhq7w@@=bJG+3RwEsFOY5D;HrirKnkN57xO|D;#q zyBAJot$NYEhE3#^V$qITJj?O4V(!T^)>lRSd8tCh#mY|kRP5h&ob%XPwc`=no4b9) zHZQAq$_fc~A*YJ{F>D*LecPJ3hwz^}P#xZ*ybWW|ihX#?lSS2gH;!$`7|$ujSZ6S< zaJnteJN>jexW^3J<*bp~^94QmvMwV+Fs`<8iW;-3fa-C)i||=kh99}GL*-s3;g^1j zwjo<2KOcVBIwhn5_v&_3U0ZCGhVwVRnPdGptq=bD_O)Y!y-Qfe&lW8*9I<^~YOF1x9O}4i;ztW&RKi%P= zHmuJ;tE)1dM&(q;-(Rm*@7IB2?DJV1AveSl>&M*vIL52qpR(n+ z4gYSyGqd)%(~*#Dzu&gjD&Lo5tfs1`etdUVJyg00cZEA_9Au5y6RZvg|0z#EIgus1 zJI@iZ#};=qTx>~xL_14jthmf?zGbv2qkHm7sWWM~e5Rp+>Y}|-eCV!jA!SE|i!z;# zsNNUH+ZOl^QdcMG&i}1{z#3V$q1gZ13AJRMg&NKu+HtOy;doWGSEr6*LCSJsWTC^V zwccCp)cRjUU-xP})1LT{)YpB*yhvYBY~gF$^djNf)?b@zA8$m6qZ9LsT04Sy1nsP1 z{4PscF=wTfH=D9Ub%i%2VPa>KMqKN-!HO|S;~|<2%q#A!zNsBJ>mdSPmF9i<9_xYU zm$k^Ie`^U7Dhiz6E}$^CxCIGXqpJGyqK`JN-wicrdvo69?*-P>m%EDbgW}YZPolTy zy+L?*!6qEzLxDYQUYiQ=TmwsSjLRhdsNVDZO{|*ULmhQ%rmfYCYwEs@t$CSDvFf(e z9Yku65l;D`2ivP3-*?h7-Tgz{yX(cbw)W*CbJbEY9<(G6f0?ub|7Fs)5R5n0EXre7 zmJnV+uWS#}rr-tA)#Gj6ud`x2s@6I6@2U5+Z9Wx5s>((A!`UG`uInCa_=X(3QqjU< z?#=5O&i5YIF@!E-e0;9PJRs{Tn}3(CV)372)$lF8YOm!t)V5QbbBvD!nFV$C5j(Dz zR#QETRc|fH#ODny!!b^FuD^A0vtN1Zq$N4Vjf&5;4xb4=>hf0n#J)+|iE-^kninHf zoNx9_CibqYA zY|FZBl3GZ$4DQ2S;ZZYo$NLpKWi2ziFUNRpHF@MZU3sE1P36K%cvJLPYKeQhM{Yi&x; zupOCl$(p=+O>54=eTBJ5170rts2V$YmF8++!+Vyi{l(SZg!hYrRqBWy%TtTc^2e;M z>^ZjWcrTbc<&y2h{QlNtZ==P?(cvO(>mzELZsTpPZ1{=d4aK$EC)BO`7HZu~hiktV zY_3hb9wD$j52shOOuzrF?HpZE@YEi{b9i2{ZSGC27}%xPD@uqn9bVavuWzr;xYbE} za{mvpcwG+Oq;O%8|M7JV+od*Uv)E9;M}{|5V9LM_anW(^W`!G>cS8=m*0+OxoKV&SwNDqcrx zuPoO(g?Hl0!StG|^@8n;tM#5L#Uk}to6bDfy$taEOubS%Qe2H0tKJ;jn_K!-(AvK1 zZf$Yvg&Loxo|sd#zdA3lF8?QgQc-V!hw47FpV)c3fi2~UrXhc3ORl*OjufT(#X7AE ztpBdo&c}=cw^wbES-ns?nkUKmyKQWL#-C`7c#&PWV{VJ=^TAQG8UP zY`V`}Zb%ZX$1(f8-sU!gv@?|w-(%x52s{^1POl5~1k~5D^O;L}yw2yQvaKH32A1lp zpJ{wn6w_iW_2FkK&fJReYX2N+T;^!*2KkP=SRq?-+_UP~(U(LR?qr25C(AqfrQ?F< z`fEex_TVcjx&=qZWfa+O*5oTs{OyE`|6Wf7*GS3xJh&gcv3Mu3deJuZZA?J0yf2n_ z#g0Cx@*NUGA3ohFVJz>3jXoUtCz^lM?DtVu@|ROv6}I1TrF83~F3tVzd#!}v9(>y0 z_A%bJpf@Ki>~~@X8!WN;di3MZ;aj>b8l_Dr*qzV*=YB9Ryw+MLWj`KP*)90M&iwLPrR4g z%j*41gpKn~_}1fF%7(Sn{FcGLOD^vgkMp%1`PG8UK$rRk#2aTHUpUWUJo0^JQLR`F zUgzQcV0q5%;8FjJPM^3M1}G`w@(|CxJ2Ip#0aj7$Y7^A6o~nq}v^gu`R>+ zYx5#?0oyx_zh?Uo$HWoGq#=$;AdX2x9Fr499FvAPCMS$ICXP5J4RK5YaZCbnOagIC z0`W>3#)$Wk+>gQg=-5a*?%L&Dy20V-81X&^ha-6(CyeVj?%gHtW5eYT*CQ~N zE71^FA`n+%LtKf5xDpLvi>h=b5DMjV8OG2$Zx;v-yQ#5HilHP{f>pdo&N zBYr_ce1R2X#218MjK?2iJiizt?!b!pg6$ZiI~I<(1CBTX8{!UBoR98dRKy(!#2vWA z=zc~;ynq#P2Lj#g*j$xCe1X6iaR)ZU7YK~eU66|S0)a8&3xq3-cmaX#i>&Cr$cD?H zyCxNvN8G`7#2r`>M_@zTfr|6dy%R^=fr_|;?dYz_ins%g_yP@M#209;Fyaer7$fdL zMI3<*aR(gdBkn*$+<`#cfrj`3fidC>1jdLv&=7YZ(7l#ZKH?5I;tMp45nr$!W5gX` zz5w$GHgreF5l3J{+<}Vo5qBUEcc3A@Kwylx0}b&7@PE#I;t>Sm3!E_G2n6B|G@OsP z0}b8jsfareh%X4i81V(d6-Io4z!-4{YWHco<3D^nWZk!^uRwQ+aL@UPJFwz1h%XQr zBfh|jG2#mZ#^}ydMSOw481VuE-G!=%JFwz1h&xabUm!3>+<}Jp0)a8&4z?q{Kwyk` z0fFv6HN+8UxD3umcc(VQ9oP_eAP{$;A-+IhjQ9e9G2#w1#2pC49XRD9?!bz;1CF=@ z)z!X+_bgZYi>ti}?-z(Wu(`75*tX-n0C5N04Y-39t&XU%z*o7l;fOm>5qH3`JxAOD zN8EvmxC1NV4mjctRBV?JcL03cC+1A({$CyclQ-H&x}2fE+u z;0|>BwRqh$Dzc96|j5 zVU8dkaRl**BZx;FLHyUo-kvArH#veWh$GmFc>#GfEDk9-B>#pwA-j||b{CbU$y$;}+S6fBg8-TI-ly z&+Ok#)W`7L&{xNJJNgl$*V9}7q^GhU@{f#TxM^=@Sr+~@=dwNTx|zA`-|^ZxAV0>p z=ax?Ldz$kp`G%?&re)^uhxOq@y>jUF zZ3S}hBJkV`-Y>_=UZ3!w3i5q2Eyw$FzT-W-Qg->A@FLqm|54m6rJJ5MV#oMEp8oyZ zu5aX1T&?5|IWF9YWjQ$>a(;~1zV7UL{qdtv?embJkN*lNR|DoHO~xa7`E1LM==m}& z^~%tHnJ?R^2>q7rNord*zc)tpx?|U4leIB_aAFMD4}Dym<|C%_ZTtG3?}$h5E|IQBQ=a~UeVv4^FB9VX zu(95rd{2z~>=I%}Q#~#1ha4v(mhZvo3%z7H!r$%8r^m)T8S#)2d7bj#LOG+K*>()o z`>XAAthdsA_3`Pj(w<&3CQ?0-Q??tU&KaGT-%GIf_rcZP>frLSU8(QxeEXwb4m&UH z|2=&@eclh`z7V6^`FB(L@a6Da#?t;vy?5*b`o3Z0!}mD$5m`^}6ZZW>U%!yn^#kP_ z&!wG{@3oh^XV)WrUy4!Xdu2Hp8|Ccv?D-$24TR@}CppklkL5F&mhBB2_rxmW?f%J( z@6`9RYymQzlAjwZpX>XX`h0&+{k{J>;-u3D>S@`Y?Ds{;m+#LA`7)OI_BcW9GX9Bm z3p}^KJ3*}g@5>M6j5PGef$jR;NA^dq7pUj3Yw+BikE)PdpGO&gnU?ywl zfw3XFeg!$=y?>O})6$kmy_4-Wo|Q&zHzkC>``AxUr;oZ}o!O=ICw;o|*GKCBle9f@ z4nV(h+56M8u05__-yR1xBIS^ z^0~b|cb@IY6Mekg9kJzMGJSky{I&6O9{=Xtd!qa|z0G-%IdXrrr$4~D-m@`Q{W6yC zpArN%HWR zhB-rwcnFkBI8!ZT^hP&crJ#LZ-yYh{KF$zdS?>n_pLz#9E`;JSx8(fBsP`c50`s;l zPcEK3ypP($PujZ}b%i5-1MP++bX99_cH@uF^ioHzb<_9xCXhd+bA&4687P+HH|+Lo zeH@MWQHRNTTE=o7p7tCI%~buljAeb6McVUbyw7vH-X^q&(ETzaZeA;^RrZHm z>Zj++wCr!_?izZU9J0qT*?QRhIf&)=x=-F6!k}DCvOao#$Yr}-4R*vv+M~Mty?YYm zce-;K%l)g;SowY6Z$T{c3#GT0Gvbu@V{VyHBy~e@LZndb&pWKh&rhZoIc z)8jc`#H7#-J^x(jJ*T+c-}ZW29C0@DasBz1ar%#!tx_)npWXQs%kj%LR{s0{&vENh z9(wxN$JBb<&XKlMN~5PMIN}wm{kaieocoKuo-XcAt?To@6Xbd`&N%z!q@9rcly*d}?+K3eE#oikfGj7^3G#c( zIM#jelsG+Y#G}IO{ki5?&&K+0_eAP*b3X2uY5Mz&b&>yyeI1aL_I%3&yWU%7{-KwX zdN1{L_xLV)9J92oo^PyI(!{>5-PiBc*Q2{%{}4lO0#>{6n07a#2%Z#FU*%#Z{L`o$8q*~jOp`>p6{LBK3`W#+4aTHtAvo-x_%gO=r+5) z$hhyqta3jh=St+1u|gReH%lh``VE0hERA>%8EzcqL z`~+2&TXPBekK)5=!LNt35zF(1(Vi@4KAuN!--uJgc|<-pVtHrJA$$aNnuKQ5~ z<#)Ps?=ki`M30H1pZ534@5K6G*9+-~+x=zZ{Oj;LWzqlcYvBGyUzhrQO@h9@{(rld zk@NlEeTwV*l>hEAzJ8DK-@U`v?;WH){qKI^|L*;Q^jqw_!k705iSG~a`9tml_?#p6 zYkcn_pZ|AH@ZUXw?l1gzPoUo;$o2ByJpr8e|GOv9&nN%g6a3$~C;0L_sNWCh=hFY~ z1&njQaUWpB|3BUfxG_ow6E`-Pq+%*Hk;6o#hshLDg1w`?LG~s|SrTF*S4a{Ff0K`t zlgS__lME~`%gQE^ybz9%XY>u-334Z?N8DLT)(d1Wl9Y_2Z-@sSM?I(;P0#YPQzSiv zkt7Mr!;*naMrx33qzK7Evq8v8lMr7r3f?t}oFRFbJ4*^ODMMng^#avGkVhMwnR9_%Q{qvRyHNqmSONI!CiyrFODevtdgPV$yMqHRI8CHvvh zEeU*)1;S%;iCiTLNQE$xiR2`CK;|KCBs1|K1;_{L$-F^&6EEUUekI=29YTImoO+UC zBs(dr%VNY6P%e@iWNygGP4W>Rnj1nf5<*InvLMTnQh>^l@*vAY%1VN0d64BHPo)(| zNy=#nIuPVQ`jJ#16-gzKl}IUC8ETd##aT6ygBE8ws5dJMwMtQFkyawVkp`qO$i}20 zsZW}cW+0o9Mx-uj0&!gkO#!tetwFYiR6Ww1gnr9(gtcYgXe8Yd(sJH zC-OU>KS(6VNJzCKT}U^O-ADxKOnQ)>AbXOoB%E}II2=L`KpjX&kR2h_5uWuVBgkkH z2QrSt0vb!kfgA^^kz@?SBO#0hG=WS6IT2E$$asiHL6`t27D|qXFcHEykdw$%G9Bb} zG7Zp7G7ID^NKGa)Af60iCZIWFF37o%nnGqnJO#oWK+~Y)YzT89%mQgz0NqTBvdy#@n?f0-bP9wKP_rcH))oluX((t*MUtA_ zAh$u@Ch199a*sR#`Glk*3utTFk}iNSiIV|_>zNyef`3YG$7FNk}QlXLjI*MLDnTNXq zj+_TM2ILrW7Jg$uYtKSB1#5N+=#@(6z$&c(Xa>j`F#Z)mP6jy{uH4O_`>`Nni9h^e zL7V*{1d~9Jqd<-VoA(P_P1b^3OR}+SAlE>=h6J$sv>xbKcD5NbCp*(f4weICXPN`_ zE0Bf5>}8|jFn`Bsc2*5F#9`U&JTh- z2&MjnIX?jM0NDe-12E%zAnby5cbFUlc??$FY;v5O0C@sd;WUs(Kpuh7J`C3T9LRG} z>Nt$?S&(Pp`3aCGL7pT>Nicf_a~llWNZ1_OoSvg|Al#)XU~RSn*$Q;J5c`GABZVL& zp}&BA{*@G?ljucS48nH$fCPZ8e*obz>2+{_2059}OQ z$$OF&WLDS{HWP&=hjC3#ZLo?i!5;AhLMr0NI?-d)j~%1_>@7)4=h3%h9(_qx(B~vS zT|x8Ho6xrfG!4i!G!^{Pz|NHlLNZuoDQRkuscBO9rH1yCLQv>fSkv#wWcr>k`ihJJ z{EECIZ{hcj41@NE(HHO)kI7SzPstpvK-6e7v9EIVGLfA z`ScFRJ2WrN$$o`5)o45scN-Dy7hflQ=cAicnr1Vg>-G#F~`1W#ctT?29r?M4IG8nP1B)=J{T-jOME zIE?XdI+O;{Qm_~2fOU|W=76w;E`;}`p#{MD7NMTBAk6~4fJB};`UT!%p#iigDGvH3 zQ2>=Flm^n$v>XkAT|87r<#ZHED$>fdDlJValTad2RoK--Nqt&_Hlj^v9a@*x*HIH4 zErM}qL)+07V6ztyiQ4ICEBT#9(5|!t?LxN_iMr~j4}4o+8bf>1-n5U7Vsu2oml#Fk z=n(i)s-rj^O$J??N@vgsFkX{&G($%V>1?`)E~Rs6S{MO|mg*=yT}e04O>{MkWqKWL z(or#3T>@5_pl0ez6Q~AKqyE&7+GrM#S!fY*jCP`tuvQ}J3HU|QKS2ILPr~mHuwN%3 z9EbJXoVEqomR^8gTN(y3jQ$0`FxZLzf^Z)6Wjp>gGG1% z;V$fASLk(+*Xc<3U8h$;UZtaHEWJkafXqV&!7f^W7L|J`-3q^=v@poRbQ_JQg<$UE zA#4F#Sc^6W*_a-IUt`)3WJ7usehq0&kTvOHu&t};MvxonE%p`xkx8b)Q*8gn? zH$i`U(f%O&(=+hvPy2!FN6*5qA6VzJ5Ke=Z?xcG_?xAIA8Tu#P4RSZFK+DrzuqT&? zV5Q?}Zh9HyWjdWsqkq#&ATQCG@Vf*SZYG3Dv=ZG%4}v^MtHAFdJpl3mtxl`a{d6zL zy|fI;#nO;hG#6OZV3q;)%vUr6q|<=?dqMxD>0md_KvKdU`<$kPT{bQFdQa$68Ued+ zJMaVlp^s^M*p>f)UH=j6(`_NAJ=n;skWwJLr3_M1zJ)VEO0bhszJZ111Qw_W%LFzkBh3JosW8g`Ihp81uvovci(s? z0zy8v3w9`XknV66xj-n%4KgJbPzLFQt?AcNU!x)FBYK(>(tLciC+Zk&Uy zgPe_kGs64Vks>TRTLSxb0LTF7_dM9ov$1)QvxH=zt09$v{sJ~MIq3)9MREvv*+B3$ zvVzRY`jP(NgJfp?!3!Bc`hstgk@W@dBnEtz6eK@*E%{k`HWvIBDbur&;6I^wOU|vhG zr0gk8%G!Z%pOhtsI63PAJNzy)xkTg zNnV3K5f~=|=_+8CGq5V~6@5unu+txDRmiDAvXEclER}_LvJc?Ty@xkvhEwKK>TXJF z8cf_FP*Pf1%Pz9g%0;F!)s({2o2U@pQO@qM2aH2d88c-ztpT})MX>y)OKdgB)hxT| zmC~ObXRnmw?1PfeloWi+e5PI`jcKytq5P^$hLBXrWLm(kvrMMz>{nBAQw=y>CO0J` z9;Rw6kCI!d#&Rpo;Vn1WLXZnt8|DE|lfv_)(3T2sPpzmBN-GE10A(}C%}OT7xyWun zzivRCf~ITiZ;*ep0;VVIIcvwBKxhp;I?m34JjXsNA3=_Vcr1ISv}Nz$%i2QV>@Evo zR*+WqkX2;0*>50!V>MYN)`-<-)Z|6H;TY^qiks@PhRjc?3!w?KD6LpaRvkhskgZrV z__bnnK-OWkAhcrsN*xFdSZCIa^#s|Ib%kF~)*EDR){Aw4&>LiLroxw%Q=(WIC6zKv zDFY!CQstFs7R91jAJ!LYbz^3euPFyLL#Rf2LVkIrCxqUt9~;DmfE>aGvH@%`8xC?f z8^&TF37D;7eZJ8zg19U1q9i~Dk!y@t!EoRZeZ)+ zw}EX2xtVQZYana}xtVQ*-)0sMGM;T^YazsgjAvWm7Y{YIK#*<3Ln%L#zbPm6gRqJ; zRJOAO)=)`c8f&Het>jT!L8!*sLC$WFyV*{*jqPH8g8Y;1fZw0Y2GYiwLe1S!H^dZR zicmrzbW^%22iQ^8RXNHIvpo=wf;`F&v4iXgI|lL?XmLM9YCu1w3+toAD96|V76Y~R zvi#TH!dYwV5B0*Q8 zSt(YIJp=iSJ!WNCN%kJ(d-g9|s?1l~DoY`RE1j6TatP!hr3V|T+*I}`Lm`w>GALP< zT`U8HZA>wlOv6A*O-f=)Y8n7?0O(XQSi${3_5*!8$1Z@roP#iy{l(71-2Vk(5}>o7 z6Xzk!VJS?hOwl9-gx(~jDYYpIWEAN`Qk&9%ok$HKiab-ED@|ctHf2@VbLE9n4`e;o zl>Mu`R4Re21S|2C@>&@Uay01f8|AGs6y#9Q=Xc6`B?e>+=yzIEI@2(c7Q$eX-ju;K z1mqAhoMbd*G7SPbhzuo}O<7EvSY`-oSXR?7rgb3Kfd*wWWe0nf4Z=FGM7Kc`Z$ene z?yy^+op&It1auv=_7;StU@fCSze+1o&!S-DH-g*=dAW>;%XYkU9=e zFMzxNbAO7RhM7JE;V(cZVCGLlI1i(;g;+X|H>ko=D0^VtW(JvANvFJr)f572DFk+lKvR(Ev=Rs*Qc0#L%3_dg><#+>-|_*z=?$#v_aNUx>N$JC-hzA!-}eH(=OxIOkb1(N z0(t`BIiSbz_Ho^Zc58(+bY${^<2jo9cr>M!( zlu;=PA%o&+Dh9gd2_Yl&G_CRo#_18Xl~zfuJOKFs+PVvEeE|6ZdUqdMeh2a$q_V>Z z5Qp8jp_D>D=Q;oG&{$P2P^B~VFeH8{@pA%$GWu4+< zVx|b@1EHGokrh%3fGnVlP*N&Ym@i0QWtzg3y-Hq?d7=IQrMI$C8319E5^S=VR+3-{ zYe;fsloAUg8Vh5t!aiDFsljZD8%Q@;3Cm!faw^5a`Zj`JWmZ)2ft17&1;JaYvZ~+( zv|u5Sl31b&id88KzcNZmr81->mZ*kOO^F2i+?RD{wIC(2M9r1olqT?Nq%=@kLP}zZ zepgz${!n^BN@9ry!Z-GZUq2;U84M|jB^nF9L>l~fOUOYxuUcO*&b3?U@vY3vK6ErDvy;lAlHD-{G&WoR)SoqtX7_YO;`wW zp|S#Yu`$XWkau8D8>5U2<77DIiOU5{df<^ zJ<2|1iZVyJ0rG~DPnoUEQ?7x$rd$U!9W2~D2-lSAN=@Y$$YaW3Ky{R(Adf<-Htg|- zKps+#D0xg)(|eHbmC~#{E6v_3@}QIsGkSw%2`V9p8xV!)h$sWZkk zE9RVY&T-YRYNUD3@NVz&h39+jbMJK?nxDn#$~{%xUFTn77WhFNKZvn9PRt7B@xAFg zQ;nEY%i}w7{ke*{_D@q+5u(Yj6=U@e5zi0te-kri_E$a=@l(%am&taM#%8<3kE_`Z zlZyG8iepo=CT2ZF-8K@(MrKZC1I2aWD2|R|JnSUO;~W)02io0Q4dKpY#G)ibls zXO*w6IMx;4OqSxVR37V?)i%2>?j>u9V=eJbcGcv%i9FUct6^5yq?%b(aja@q$mFcJ z!>}_uo8QjtlDGq{VP+$2Y|QeBCAgKucb~Z!_6x zB9FhDRTRgH;=6F0NnJCmeEY;)(AjKXerGYBXNb9>JXSC(FLG7ea4c`OS=z#;#f-DZ%s3?Hmfd<)kVJLtl8RZ zz4);*OBVBIOL4R`D`7T5%q@zEV==SJW{F~cU0fWCi+j6EqI5;Yv8b7xqt6q+*N7hr zvnAqx@@QdJ#O#x4yqKSTGA%6HCtl3#e-p>w#Qk4b{z7K9;%F=CsJnPySjeoqykj%# zYSPW5pg0yZD`3{iq>G7}IGTy^`Afb|Ch}-1`d52#CtXq;ON#I54&ts<9`l>oo3#`7 zQuW2LzF8eHpZjFW%-qGjris}r+xfWbY%0_n6a-rwqtP>Lo$$+wf((lg{IGH>B)pG= zGHJ=U?)6H!UO&9s)a6)pRic4wmz$6IucaC|s?vGbJ9-N|nGp!JKkFfX%r5wOIub0b z&O`BuTMRbo!>&V`>3RdV4Sxs@3z7`1>~$&QSN61&{JKqjw)Nb4Xz)4;C_As%%nsJt z08d{=0p$YS9ax34+u`DqNTBSHyp_dQ-vL`5MFM5AH!_gDk%8=u3}kO)Ap0Q$*$)}W zzQ@pViyo?ECcwC$89??u2D0xlko}E;>~9QYe`6r~8;kwH9)NqtWEkDm6Ud&%pwye# z(-_E}#z6Ko2Bkj5p2k4-GzPM#F_1ltf$V7vWKUxtdm01T(-_E}#z6Ko2C}CykUfoo z>}d>SPh%i^8UxwK7|1@xK=v*MvUf3%y^DeDT?}OJVjz1L1KGP6$lk?3_9%wd73br? z%c&stM+M8$=fT5gi-F1hu0Zy2&1nm1uhhBZle3m%NOGd#+@NvSp@7AD1C#3x$#sY1 zx%wRsV{%;}xh{}g7r1R*6})WL0^HjEp6=S_ z_l&e9)!3Sa!4STrMaGSAZ}#dzAXwO3lnhzzHX3_wp7bXj0*zF-`+IH82U`h>_-Oy<=!ICtX6L{oYs#T=Ou5{ zCNFuTHhsdY;YGjZU?ukJDE+Pu%;m0dac9=vqt7nzs0ALd$9+Zm8?hf)^z6}4Q0%cp z+k*DtZU^%rZ0rwCFUO=$_>m9HX7-0-Vm~Bu-8qx%4#{GYzLq+78v2v?G%10%p>!K;SE}D|-!kJtb&a^F5t_x>!U5I^M)cj`#%P~Rx zd*sw?Dk%|1R}1sQWu`TKvx&;3lcK0brtKVUBt)S z5H8nT*AqW3V%@Y{!QNc_v=J*Wi-{Em`NS%(3gS~vtOj&6aT1}8M3@Y56yzq}_E#3i z%3}RyD{-FuS4FJ9ttM7}R~4UDhLCE4?8WL9OYvbQTJdQnzWzYGpKmBW4&rb2;x95> zt^n1FAA7@!^5SBhO%t*5u&GI7u~uBJ7XL$hiW@>I3Ti1*tt>vaV)eM4SOeQqu$?$s ziL~;GV?L9jVm(M(5mH$E6cMZEON(-~5%pC{e9DObw-LX~inVhEMH$>gebo@3S|+u{ zr=}sKhM+=%+KOXa5sQf^V;ga7BVsBbxRp4z5~~|$ij}jb;%F*ruA^8N-c=mCid7+z z;vJ@iI9iDFJBn4eA>tDzp0kCD_o&|DGeN8pA1_vmcNCuxL%6s2f2jEBDAuyf6tPVc zKLx}pxXxmA_^&nN6UEB&0pinNgtil56UAqMph&UGcBnWG6{|?x#d*=2$#<`{lrf<@w{-5SkpR0 ztZyACKJvY9ka#EDMSO-BLV^Sh6{$yw&oHr0euP*TKUDAtaqKSEy|fd@b|THfq79=S8NA+( z2Q`lW=&0Xv!~vQY^8ht2;Nqk&@UcGFihXJSjnB^Pl4FybmH4+Lw8}{h1A!r<^*>%_CVuy~+B%AkS4}>o|LT>E6RZ=0T0emax|ki5dg4 zENcAqcmsXWzTP0~4x(@obWaNy`jTJM}5EceGTE;dpqi@b{+x`9(cl}Zw`9X$dNGKZy4M>>Y#7p?g^#DUd0OE zvg?*(GsU$X$aAZRbBiBukrSuSLTyf3vnvBAd64086B_9~ySo|ekDY3ypYaC+8P5ol z#`1mHF)8wirwAP6HSi1PP@>u*>h;;b`62u;I)q+|eu*R?i3l zk0#L=I8|_q7|7o%6|e7KEAO4c`#-+~Me6Q^lhJ|j$2&b7Ewu-(6`T#^_!v1 z#>p`4wJf2BMQ9Q4J&t(Px_^0z-5*fxGE>u_i%d@DN}C~vIez)p_c38@uk z17-5;9w?LN_^R*H$n$-p_iFSV8+q;zl*u~)qjzldT^eQO{gA9H%F1(C8BUqJS1@|t zrWQ`#F&MpvQwt~W9*o}4sfCkw5=QUs)WR)>q~gjOYvIp?J}`C3S{!DX0z=kLGUBvy z>+p?#60{HZf(s`j@Y9l&u=PSLoLM#l=e%D9maC0<`lgv;f0RUMQ8X4Pr=LFxVW-ky zTZ0(5v{4TwQxjq7nOI0_eFEzAPK6mJ#=Q5%5kR*jIJ7txDAzCj1cuEQ{d0Ivxcur7 zY_Q!3aUDJ3=+XPIea$B5vT&jiD}7kL^D7zVgwIMohw6=bL*kf9IDOG4xHg)>=sGp9 zO8Q%v(!3Ll*kXm|wI4(O^OK-$+eF@U;sIPl;Zy!ku<nJRIks*e%}BcTw;K-V^oxD?1t5_ z{9G)Qax2T~j8BF>=Egi|N2X?Sy9A*x84IUML}@I8K0s1`AJ|->3#(ys55BMvIJT!4 z>s#<0thnfF#QK@d*@=$NA?R!XPBF|ZDk#|t|oiC7gZ$?bs!>PWjR(o$v-(gcG@9k9IZIkzTs_(q%`)|thT{va> zUYs(0N3O=?9ih>CbhU8u9#QpOyV`qq`VO8ldGDzDZeHy@z4ASr`<=bo`+K!_`O5cp z?)Um?@A%c;_p818S6c_5wkAMreSq3p0kw4lYHJA8))T0$El^u$ptk0Ku0Nnm-nkmB zOQ7L&je;7}wF_!Y*ECS3>l-N3wGNc&x(CX14FqMn9)dDm8==PJ-LTR62^v0r_hOvd zGYzKx=?gzct-`N9so?a)AHs@8VwWK)V7DO#{#+G+GlI9km}9fy=;vFo+stA>xUfZo~tAi$t@2$xx^u#U}JLm4Tjw5^O~Qa>8P$3>Q8S2yg0YK3P5?KDH5UH~fe%P`-CL2|Y#q%lYKz`ibZ?WA%UWw~8DZ&fN(Omd^&X;c&dr z4p=y1x)FbFxB^Uz{K{Ka;9`i{`^FFtWmAs?+*@}Yl;gfYIqcF}Y-g4NQ|@`fk%vjR zDo+#zt~N*FllJxpEg0@doRfN{k?!t(Pqp+(pj zXmITfI9%|9BDn>l;;Wg|SU90QaKOtECiUpKq7b2yrQX)wm1eTPb;=gbNOZx{An2ro5P2?dit$}Vy$E0?YR)mi*s+_ewZHw#(mcW z-g*V$8~u&=ZiVV>Nzzkzc{mU#TX{ER4PV@az&;_M4>-;K9CjDRoCtxH^RKea9iKvG z{Nc|$>iiQJH-8LJuAW#FYp%Kn z4p!bk`7Bgti!MKglc8Rq#%1R;WSuMDfhfxfKsn*scTJYnOZYs@6V!NwN2tcd{55oH z>;aUIn0#YpC#{7kFQXwZeBr}o_G)etSZ78HnSPp$ioFdFqbC66A3w!+&e|t%W1<&OUh6Us z9yhoL@!iG)x0BaEql$(bffuXn7 zf&Y?d$P2Ihd>DSrPl5Qu(V)h^x$D7iYBDr^5e<|hn&oF_%-6wz*3qEG)!MDoguhq| zV|39#xl8@wuIpzdLaR5?K-s3qJ#cT90>iHM0Lm3#-Vtk5Q{cdbkwE#k0k`1IAE{7- zj|Vj_RN*>U_g@buO85fh;O9&5MCulJ-zNwt&n~|PzXxo9)-ywZa()(#1wW*M#j&|S z`Jz~NUT^nyIK4gwCbc03({x=$;i(S9^X0oD!43&L$qzIBJ6) z4i#DpuY1P>V-K7j8pHbc!#6G4r;2UNll z^}j)=cxFZUd?8!JQD4Ar3`@x%wE| z9tZ`>O`J{e-LzNW^lCCtE_LM$|2F(R{9Y{(D6j93$UD~i2)C;S0Oc{|QsLB==g_`$ zAE3PYL;@ryy@QHnIs@gt4OYUP%+J7z0Z?ugz8pR;`3mz6)B?(^iWd}X_7UdJ2>{A% z`bL2Li1%=2XCP31@MSiHbb1ADXM=!p#_0vHpzmYoU>**XQ@ed(yAG{}ov!ha7p~sq z9CIuv=FC6iATJ!eYY$uDnga7<;y{hJ&)CfRp4bRI`CP~g=Pz2H1x(xsLr{D_#has3L2?s((CC*8QBWY2@$`Vp|0Omb;BrPVsBwU07q<86Jt$Z*0w~Y()U)K9_n_V%5kUF)wv+6n z#S6H9CKxC`HGR$s9exL2uLc6;Nh=D&Qdcpb_X+^Y4T^nX%a*Q$N2bxB#x-7@WuGsu zf}i)JK#e!xj-^opedpFA7;(MUC_#Tk=iIlsD>*ATXE@U|6>yP!~ zUThCcoe~2^+`xf(TkitCK!oRk8%J(s)yC`s6LC+M7k)Q*DZXj51qL1v_ql|}Zd;A7 z*KL3l@$7=KwRI$ZKAQrw8pHwRq6-7Cb?L1TDDFEbe`<6GA{uRjYPP;Wd3&L&5SqCQ z%6=FRlzWFHqJP+Wa6jb>YFznE5*F*83dbLM0_BZgld<-(br4^q4^Vy|5rJIX3-34^ z2b5jRqVU}BiSTY;9H?>K+tV>6-&*LF6bFF0{c#l1j=?B z?!mKKTcPg9@!~nu-}B?^cGx}iGaM81UdqECRKw}tK0^I3&OmutmHK%9)>~NGq%BY$ zchwnn#a=+c51oPXkNBcE-ufZvtAzvQeJ4Nj{^A|M+GD{$S-;=}kGuLBVwy|_$`+Oh z+;qxYDB58XP(J!K6-FL^2BV5~1R;!@> z(NExXp(aqS95Mzx7QTh{x=BF!Xp?a8zWo~fk52~5=iknRlJ%a$=cEvz+}Ca%cx-zB z9tXmKa)*d-Y++V193LjyT7}R0U18=AQ($W2I8ftNtq!u#j8wQ)Jr2~EZRD)9&nBpu zG#B#39+vEz?Yr!VJY>%CK9P31UHT#WfwG>8UuOZ)0g$kwC*n0 zmJ}oUz`ygr!#^Q@lqHOxI2Y7-Vp=1;w2@_}JU(AltrAXI#xpJY5NlmN!Vsq#&kxR* zlP=|b2Trp7;(3&eM~yFP@3VwaF$SBzxCvL0l3Kan4$PGWyn)++UI zSma0A%es=R)Oo@;Tdh*J@?51(WSmM}$#|4{knO8n7nGI$FY`cI_9baA`($2N_A6;G z`?(q`*OW|)vaB~HZO0cdo~M>oNKDTmhzLOy^H`y5gxVk5(~Iv z%%$xb>rdKUWh*<)0?L;>9Q7;L?>5*Mui&VUy|<685;l!v>g)UM+Q{xWO;P1KhhYJ4WcR;!fDe4MRTDOVj`HSM(Q`YATCs#fMrmUn-(8rneE$?x9wKa> zF51m}zs-W&9OQS2Eq*)C5Uy}PVc$RDC%bfNAy969zP?_jCBLsIe;V(oe4c`^=I?c_>yh4Vnaq69>hHG5bKjlQn z8iwyH8BY08FUSXO4J)+NC+c(9K|G(ppUF+$WRtz_c zgOru;f0?(xW2w=ggy-gV4;=KBdN(q}S-GOUKDA9h10OGMuOBa1+EZSA-$9>G*voLr z$ zxU}gq)lt7+7fkcRrj!lp+5;u7+lK)f46j8#QiS%Sm7SY)?a4YFu(%BYkg?mduYD&#!B* z-@kV_yzgFIL;1+&`ua&z0uAAmmo00gPwZO;w)HKpp$vaG>GPS_H-u9@`cbSEKMHW6 zS8)yHzXBWTU&VDZgj4>!slL97NSBGSQyy2xUcXf2vz`d2yt0y${$|1Y&|HL5UMups zQRK6s2&a57prL-7NVkm$r|c~9SySY*h6txzrb|7&ulOFYxLq13SGKON-(6>-;o78p zb4ERVMC%ZE+pZF*adJ>YePQ>>;3KXbHD0mTUSCmM*DFQ3YJ9wTL;X+}v0mU^IT-%d zUO#$UI7GCsq{<`j|Dm019Scf68>#`1SzXoPMar z$J}hS^TnEOnYTY{+2-^yH6En3&FQOZtfXaVqwGB7W+l(Ez0_FA+s5mqfU=UVY*RH> z@-N#|jg|J2c~E1e-DUpOSZPz4Po<4yx|Egnm3dZUUln~I zO!yPXIO9WXwH7018_x9@YOCFNYmOmK#r|6Zo18c`Mavnn5>KUDrGT;$PsKle&xzlo zO8J~}$#|6VN>=JYrbSsPuZ&+Qql`x>yNv(SQyXo8TO$o~Hp(`8#4{6rgC z)_&kGn?CyA*~9j+hG#zi#G`se<=Ffkml_kJQ{qwMqFymNHr?gSyj)xPmOqh+KZ~PONoB!*%x3u{??(?P}DDkMV-P?YkoU6u4`~!GA z{1cn*Ee#*S<{NC(xNhHyIW}r6^Q@%xcdV2}+NkkTQFbMMHNGoslzghOQYSJkHBK#I zt1WV2C=4%sm{C@~qhg=y80CB)ZM1c_jDYsr4lp%VzQ^Rbl)dBaw6S4L4QWv>v&l|7 zZfzq&oN7GbnVoiS*8y;=$X-TS`K~K+XA4u~G8^o)n;H*?WoI~}eDsW+c6;5ya42~f zQ)A^jQ^rqO_r8iY`GMAuA2lxHR8>2}#Q9gcghv#ws-0Sd8PZi_87XU(E5=j5O4_8h>q2RXcEkcn2=}24!X3k$F>NGuNux z$mUHA=~7N;S5@n>u(2UsHCDzZS?4`V+G)#N=x?a!v@f>WCR+v>>Wy;3?^U#QJbtx> zGM346-J4X=7M`Oq*i%;Y6NCl@so)w$iti&(dOO2I$%63;{rA}o3QDdb}WZzR`MSDTUqsByc z0@Ya2f{^7>V?|R!*2%xItY^h0H!JbW_EKY_J7LIkwjPBME9I4aOpO)&2cbccQ-?x> z0*M9%5)BF@8WgDLP6#cC92Oc8IW|H&!foM<~3X;Q0& z6U_%Cnh!=yG#`*?J|NM2Fk+(ffRu^m0}{;#B$^LKOmrTQGSPfMqWOSC^TCLT<^vMV z2PB#gNHiaiXg(m(d_bc4fJE~FiRJ?m%?Bi!4@fj0gbz@T<^vMV2PB#gNHiaiXg(M* z(Ro11MCSo16P*X7OmrTQGSPfMqWNINMDqcO<^vMV2PB#gMoe@bkTTJHK%)78MDqcO z<^vMV2O}n$52`xsMDqcOwu34Y%?G4RbRLW}+-W$`b}-U*r{P5B0Vxxm2c%4N9#D;m zwgcvciROc{!H9{@15zeB4@jBlJRoJF^MI6z z&I3{=IuA&h=sX~0qVs^1iOvI3COQvDndm$qWuo(dl!?v*QYJbNNSWw7AZ4QMfND&% z9Z-#l&V!K_KMf~352(gO=K(1bod={$bRLj0(Ro11MCSo16P*X7OmrSljfu7csxi@g zK%)78MDqcO<^vMV2PB#gNHiaenCLtpWuoDLl!=A|=7oud1LlQ^h6Cn>5C2Ec0rSE{ zp8+WoeFmgV^cj#c(P2QM!(hZjR{<##jRhnc3rI8;kZ3FzG0|8+qOpKPW5I}tt^!hC z@gI!^BpM4yG!~F(EEqA-SU{q&fJ9>fiN*pFjRhnc3rI8;kZ3F*(O5vDv4BKl0g1)} z5{(5U8Vg7?7LaHxAkkPbVxp^nl!>kaQYN|zNSWv=AZ4PffRu^G0uqe{BpM4yG!~4Q z=qey(qOpKPV*!cA0uqe{BpM4yG!~4Q=qjKZ6I}(QOmr2HGSO8)%0yQIDHB}a)GSO8)%0yQIDHB}<%nK7;1yp0AtALb=t^%qt(N#doL{|YR6I}(QOmr2H zGSO8)H72?WNSWv=AZ4PffRu@@0#YWr3P_pgDj;Q|tALb=t^%qt(N#doL{|YR6I}(Q zOmr1cjft)Tsxi@3K*~f{0rSE{R{_EDHB}?(N#doL{|YR6I}(QOmr2H zGSO8)%0yQI^TI?|0rSE{R{_qSM0;)05JwPM=kM02y4Fpvtx(ApS zCb|bmndlxMWukk4l!@*EQYN|wNSWv!AZ4O^fND&150Em^JwVDt_W&sq-2+r(qI-ap ziS7YXCb|ch7bdy~sK!M104WpQ1EfrJ50Em^JwP=kx(7&^=pGbg`pGzjzcvzmfPZ8q4@46HbF;J&{_IHf z0gRaF10c}{K%x&|2$%5)9RMRHngB?d=mQwy$@YskV&cD#YD|3j4K_-7i7!7AUw$OM z{78KH4V>-EkHnYXaBj9QKN4SlLp(~JiT}Q#EwX*dQSr@`?M0dR?_*w=`0t|{6aRgr zO#Jr^^`qpC`0t|{6aRfwW9i?j)H!A1zmIw4Z2x_O{ogV1-$yl8{DtMUtj5HbABitN z5?_8JCjR?KnfUS}@#RP2%dg7Bmmi5QzY!DveM9`&zVJp&eEE_1@*6Sn=l;z$@Yaelj+BXiI;t`8Lr2QQ_Z%q` z-*Z%B;(Ly2OnlFEYD|33b!tp}&rywu?>VY5@jch6G4VausWI_AM>QtC=cvZS_gtsO z#P?jM#>DqrN16DZ>nIc7a~);kd#N?=tivMLHG{msgHzUuM}xl$HJ| z(^X@|AKcLH|M5vj8m{ zR(!FA&p1+6e6fWeIx0Te!VevZA37@ES2BLeiqE)=llY;d;y*6qRP2Rscut(cXWWRD zc!ZxgQdZ&-e&RXt3;*$)a>;m<@=8|fL8e7nDX)xQDWi->DZ7lH_;mjxXZsx+Z2pcF z|7dwHsm6+*wX{)V;?tdzmhdnBFV6N;&dIZ4ql7E*%WyST@+oc9Sn-dRdHyF>>O}Fm zR_v8<#XnkxtFhu|Ep60TiC^aVpIE5}CC`ez60XEA!_`>Hr?gRH=|i3EmyQ3#*?!pu zo4;el_gR)*jTN6|X`{w64>Eq?bH6}jYT#boOH&V&dP;iGMdIKHWw<?c?G3*hBECekH}KvddfD+d_XWySxcki|Lj4OKmBx zKb>7(8D1+pKQ)B?8@t8fl zjy1F6ui$bGm2_ozMs|6n{q^j8%6OFgHx=~r?JHSnU&*q4WxYvO+E=n{-!r0pmGIix^!u)&mM}B7e0Hbqh_~yRdc+sn_ zxIU(ekqr$$x;BEpPaTE+6*n`Q2O2K(CH{soZz?ui&4#3fBu*J$PI*b5k=j!mQYT35 zNgbk^J+%?%GEr|#)SHn#4HtP}qTWE%n~@C-7iD3h-ayowQCVoXs1rk-{!P6p=Q1@L zk`|CS4e`mkmT{6i1GUdp4>^5|+UK&#>2Jb@_A8Y6lWCDU1kx6=U**`)b47k=-^;O~ z;iNyva^=|2a9LLQH<^14>)Q}!Pk zkJ1)09@-CcY-l)XQ=@ojIO!XPdQ#ephRZOK52=BcyAIX*VNQNXTDkjSjt$9k?tYkK zL+S^pJ*}Vrx3Wdt2hGW)Cp3?*2ZR69M%XH}9()b9;jr|Eu2hRox&yUBVcXGe80J(I zem=UQ8=K^T$CED@uJM2yUvzxqX^qAHVEBGZ1Lu2Jhp#8g@i`3&v0}xR>tenHL%Z># zfvrDjuzvtf@bajmp{;i;tb--2H3oj-8-~%}!!%_lF3+SK>>CEn&xGp^x0vrrxkJhu zjn~5?nwy`3LG!9AxV>(U&D?ssG?`J%bqUyWP32%r-q#gNWfg-p=Pv5j2EWbdJ$^j@ za3K&s*KUN(Yt)BLnup=~jl1N5caEPo z4JL)}PH(s<%+=yTAopq)h#B=X@HT%9$bYp0fB$GSI7c2goSWr#l4ndGZHQBrRg~Gl zD_xzS@yTChuRN~~b}Rj>?4k}()`jf<`_Cu39Pg&#Z59L}<-H{XpwXJCd`3WP4dw4o z$2iZa>%vcO4noRx+6Lp$bG7-?`)6D!ADa*c@h%a%B-5p?l+$)~Nxv2p!541|K+0_H zYL}qI2)-&M04eVpABJ}mBQ)cZmt|7+NF0UH3or7cBbR7u&05BX+F<>BU@W{jV^g{Fcw~6$@Y(Lz^5#mgp&e_vthjgu*=gaJg`w= z&AtVr@l@GkEN1Ks9Sy%Rum>#qS_H0VoYkGbeOXgzMF{#$*{tc@trb)*Ss5KW=`|6> zM#H+TC)k(ik-E1lS}^U5KpgVUicuTKe4a3-Y8GFxb+%@XeLvW?^(_DLa1~2ClFBkt z+<8f{f8dCdZQ$O!W4uAj_iWJ<2Piq#0ct0G(OkRj$;!SR#O!YPfPDp9co<>>&SFoI z;(c$kc!!&8%(S*JY(!0rZrl{UjcLoTmPpbqzv~CfJYsqBn4jFwsRvY^>CY!FfLXn3W2Y1sSlINlX2^@nd~=Vd>~w`TU~xk1AzPs)>^*;v z&2E#XIqT~On^WU>%;1KQ+|(Y%I%xUlHw#(Hg3(~Vb0%-x#~kwS=?upXPT-A7FK``u zHw*{-dBFAJoDlFH^c#P2bEoUlj=?ymQ*FL&XOnHHAUFf zu-%!hPnqM$Z@sbGqdu;G#d@LlQ$24vd8(#F!vH9Da0)LzxwR2bK9sI8Zx)O}zSeA& z^H27?c|Y7xu`r9jISe)|c)<7XOx9F_MBMSDSrWdzsGy~MgWdSI5t7w3!%qj1); z3w)&8a*gJJ3EZkT0B`Q<()85g8@fWBy67s@427SL*V&PZqws{KX-30ZeXy11X&$pE zjeVJUg~jb1g}rT?W_H}Pn1`oT<+fSl!9J-4p1Qh^Lp?JXu)45|`3x&Qa6ky|{JL8A zb7?Ss^t5D?%awBdvbQLoK3%*E>oXcJcpYca!Bcg+h6dn4&nf&|QESbSpJ zXV22lpJ>eITnfV1?!UY2C|8OPY8iq(99po&a{|%b!kYcjxD}KtR~gIu9M({dJ?#m5 z3Y}t?Ou}`n_wMYm$dE)fi?OdY9u3?s09r&biV_|KPRxqo^Nv`SjjZKNU#5Y=cVC>?bnP=cxrVW9JDv9N!Iq!eEGw1?9H9WA zuFU2pWyHJJh7DmyReLD(-HqqlljZ!lbv$p<-5*=Hr)f5M`$3OGaXf#|eJ&Mx&gDm! z`{VNl=I~*2XL!8SmoK$oE??Hg@pWVT@x=LstXSM=$oGCG4?cg>dFZ)VZn4!L`>bE9 zn{&$#ZcdBk?GAp($oLq|(^7>78mozKOE-m{ojY)ggu*V*++z9LLn2-47@lF_C8}m7 zzfpItv-RjXe07nD=xOtl2UhI?NlOFxLGv%p&1#qA4LXM4hnagd4aIlL(6Ls`d(a4% z*>}qF(l#Nu_4Wof&T|}^9xuVVW|nq!D^rfIs~Cb!O@A=G(2-i6QiOf=9R_|a9`Qf# zCTX75FX2+$q!jPnDg+BG2y<=HIs}iGD9OA%Lm{B`9bK(H(_Py?nw?RT;S}JYT)7B zj2{dQf~$?2u(XvObh|Fuxl}cu%loAIcpP_i+erNFPZi#1{Z)v`TH-gy@>!JGt8@3|(vF>`;NX@0=Ay}w?e%8bJ zqONUtFxK7a1Y=j!$72`mSk*%xG8XB_^Ws^7Sf%A}t}DOV^Gaodv1|U$aB!;?o_~E+ zL+3(bj)Y>4#N<3@htFu#t}^U9W^#PTH_gPk=X4i01>^RgwQxS308*YbEeyq6PO~sEK9lmD%ZV;?dTV%5F?XVz{4M}}n@wg{{g{rji`7);R-rDu z=*A$V{Gvn%tiM;5_2059ld?nXY8Tub!OfBbkg`pmFi5nXt&0dua;2PW?uTOTXPD#3 z`Jnilfm0*HaA~3OnkJ8DXHx#&KMd|`X6n-0C%aOnb6xXhUv+CQp3*ft5R76@4q{Gj zsM}m~S`c$uLwL=Z8Z5A*I(BVTo)ujg?pnF(B;Kx7Aig-^ip;A9mZ({giFqi<`KW9o zF-HY4M>Q~=qjuUbhy5|30-N{L2h-i!!t-_e*@OvSc(;oEV2||~b}KWHzuGYhQrBE$ zC04}gF5EridbE8Ad@!}*4h^F)=>FPM$3E*Yl`#QAijs3jL@l40fnmwPZ@RP46!Q6vlydgGImTuR5 zyF3X#wr!xE4gK&hMArjhu6K^<4^ySEOyfa z-sy(}9*c0mgH~^KuSeQq`2#k1_OE;VNbg%*+wcaTVC9ZZ-AD0RVFS2j4PPuUyC#^H zZi??aw`E27Pd2zx4~+EnXSH9Coh*fVqK!2me4GMv9` zZOU4T?}XnD?&OXK@3Ipo2EZj-FCK|^SbX{b*j{V`Pif{2t0T6vO0_5IK352V;`__6 z>Nj>~QodF)5LY_|vK|&pNBQ=J&%9Fc0nk=Fn+du<0M2InGxr=4aUAF zYw_TTOyjgGOsvs&)%9B1!gb&I$?#!;8y{uuqv5xELLvJCc%ilpTX^yv^H|vn`wfm@ z-`jlCSdI0?pAm74&b1txop$}uF$9jBvE&U`9(0Z>Rf50p4MCfEQJQaA!I1vcl1FU_ zg1VZfd|TV6u1_mfcU>grV;3VsFvY_c7Oq;(L&f~|#JBRW+@~hC=y;dY@m)KjHk97y zf}e`M(b0Lr@Q6!nPTCXxXIdMi<9@Gx9_+KP7hlrI2kHEQj=!D%oWl;3@xl#NW->aC zZ&{tFxqjOZAH~EnIxcSu`N_8Z)&t*!2e9#;HNov&6Wma?1EXU-9be+Vx}e#UYPdPN z8B4PZ#+Ey3ar0!2hK^Z3c81~WELUAAPghqu?o3My$RHYp?g1@LA}K z+T<8U$MkP~3bK2JgJAcMlKfFfDAb&NO_y#p)m4o3nPRL5G1hZxV;k{-oqjg}j$JLn z>6l*k%VgBOZ^bgri|DA$gE!;Asme~4_HCk$&J%u!dHZ28Zx32kIAg%#zA$ab8D>)^ z*=YVB|7a9^7T>GSC(YNrOrODa`<7u#6MeCE_!U+la};L&>6~eLy*22n9%Nk(|0V9* zwrS3fwbT5S6^bF+e6Z+Ce=NQ$t0|pZ?kjMM&kDZ53zTxl?qZHwLCjIRd!NckI6acj z*fxL{sq2e$&UvMeEjpjE!7Jjsh|XOPesaKm(GGY-d|%Od$ecs(Sg+;1&@Ci_(fQ1D z>u(xMFJFv^iDPtrl5xK$jI34wn-8;Lbe^NTI2qhS-1uU*2^u=j3>;~WPv&;UU=LqL z=c{Z)Lu~%B0p2+5#_0U>PT)fRV8v*BZ!(L~dF;-jn{*R={cyv9IM!?U`t(g#oAMwG z!pqy*V3+E}5Z^h(p57BVU)rIY>mCemTb1N=E?RiRC+_Sy04&^ofxIFu_EH7#sf~#t$@RqLL(5t+q(Y!aXsfcqxFl-xV#p!*`j1`{vZ0sqPA?6Ap zb#Czbw@0G&jB6P~upi#Icb0XCTh2$P&F7&v8B1z49#0Ls#BD2h;MZj)I)}k+@$92h z%su}rUaR;n-Ffp~n$v4Sv3+mar-Uvda6C!R3R98PU?fgY4}5TG zH-0>Od?65urZyIuo^>JL)j#;A*#%jhGjTfe$|2z2=M!_c84PpgR(3slq>7lAy1|Mz zE1aDc{M60*a$MuQI~aD4TFK|=>hL=Ay`k>2OT1Az4=7^N!}Up*)|gnH^WjtS!O`N^ z`K+p=AZ|o0mpX2J@$kY^d~l)l%x%^s9_Hi$qtDz-UpHeoD z1iQi4HCMbsvH04Hu7#Ji$gu2d%@TWr!0qm5*`5*N-gx7tjBSA)@ZhsXy^v&#{ucXNX2ySp?!e&9Cyqv%aG zdqP_=FL1=eQBH99>`iB%Wuw`JtAm)!IUfib>4-(wIKeRQQ!ZjmaS`{AOx!=h?)aKm ztxHp|k7~&~v|W*IHF^$PS!5#Yx$u)0)bxNe!+qs_Wrnz~Wa7RO`g_Ol>(*Z26hA}W zpQelZQzq_Dp^0^p?vICl;M+Nt-~8eV?Kaeb(_-AOGjJ@g;69FhK0X2NJ$D5wtOHlx zY;p0MeT7?>e#)3>8_*Tj!0y#`;JCG^OY1x3@pJK-kd=9lS(nhj$0c=Ot+lCh&HlBq z>{=J->GWAse|wsy^F%**KQ~U^Yp08QZ6@xup+L1o%t|~%8!hgqUWYc$__i*NZ5-nd zYi60l#Ei~R^|UviUV>+oaG%STtQ2{k(@^Y_V-LPlT>0@8;hFtT&Sn{9C&HfB6YGFy{_54}7 zTkVp}QOwVFelutDr~5)n@eEMRCDO$`G86a6kh*U=3oJSoHZPvZAHQ>fXOFAFxHrwX zn3H6PdtoN-g@J}YpBw}&#x-HR#rq5~=gAQF)J)t{!>Q$AaJW<-jrXPTnR^R;NiX?! zF1tV7ACl^o$Z+;a~n(f=L3#S8gUu(C^XnQ-3jT+_;nx;MRX6=ITx|}T+ z<8Ov|4#327066{e15Yf`3*LMV=W9iOZoep=^>5=3KB@Wns#d+89~}w@t?%gW zMn$^zvouX_$?Nd_-Gh<#0k!a<=O^Qg^tP;3sD)14>v8e^!f-9Wo8W;V(=T#)U5oo} zCf-39!gZg;cgc~{hH(1+Mc#wT|4V!M9!B2#N&d8?4we$viR5-G8T?pU4Z3_R!;dTu z%N(2;$UcgFphoQN;@o9e1UtDUz=)Ulg~9OU;ksJ=VqGbBjbEMdW@ZGtus#4NJ5LP5 zoijr;E^Sw2Qf_rQA5Xe|N%t`{7+tzn!`sW6gV@A7qoTJ?)BI(1&CboC`0ls~#%TtC ztMydpo7201d503X{pCGPg%-bMde60I$HlsZFO%Q86sLt>(t_r0#xPi%(-Of)t#IYhS zf3Sq6KH@&o9gbGs$F5ZR$TLkYbH|b%;Pb7!>-;7C;P9c-Y+LXeUeM$^?^1IVT)JTA zay4ueIHX)+&NE|lbX=>uC(Ct>yLcw*Y{lugwsJ|drrwoc_;|krzv2}P_1{$I5BEKH z4Oww8y=nat%+xmoQp)(j;tz{>m215;gD=`;9xqsowG!i+bNFuYB&R9caySTr9=c;_ z=W^Kc;Zx0{MPV+5Z(6g}okM`~&=1vdcwK8!wEFpTX!@rYo{mm|+1`9(Skt{@c!_d8NXIh$g3GLDfhT;yf;LFUvWZnH!1X>g@llPtoQ`FU z$Jd7Xdt5M5JRhKA86C6cHQ%ILKG_eK$Hp-_W~F2-@;}xB8^&W5L?;VlZL5x{z#F%BZt08u*W{-)TTv?GG?={{Jx&a$#foJnC&3JB8 zj6Iwd0^>*4%AECh92~r{n_t+`#r67QGuG%#2vGaGU5~R4pZY>*pY^;2gG3oDZn1eB|`S&?x=CvFwy z1_9;YSs*&D^JnE!zSfyd#CUX5j7L9Pe&zjk^oR1x3h{_BBk{}7YpmvvH99&r zU9FhPZ$20Y!+)0GbZmOLX_wBUc`&SB(M#rpL zr{A%Yt9oJS?h%ZRS@byx9kaAEL!n8D%euDSnI?X1Y!&_ zJa4396CIDTzIbBo1E&o(^jfCRMd@=$IyN~j3d?Nh70CYT5(q6iGW`0o8nz8A%L?!5 z;=Is1f;HS60F;|Vg`s0ZZ%t&)s7%VygTvs`;%MDL_heVfZQ@pEG+q?JoKpjUvK$M4 z{R_wJ--hu~Z2g@5|D3-o;n~0cE8NKbqI&@B9yyJ7^PiUy-g6LNcwv(GyK$VYrWarV zXF~ofTse2ScrW$d%n8KLf7yt*nXp4Oo9w@X%w_j0;(6*HgADToksio&WjM9TPRC$R zIXf@E+B7u?X6`U!;)nZ)cg7jx2Jr#sCgDf0XC)g2g80qtMOx!UTBXFZmuy@1GBpV6 zV1Mw~_KjUE^hHNG;Xr?wRPc-Dal$u#uX!tYROkT{KSnIh(PW>)WgL=aJkmD%94^jh zG9D2hQkLgtpM#XM(~;++8q2mJb;V^}WY;Gu^(os%)|U*I?IG(_viO~2Pi<&_ko86C z4V8M6^`+FatZT_KOq7EwWtZhw>V(u6m*rRLQ~X_?W2kddPA<O|JNQg6epAMiE> zAF!3$Rw&CL`m3Q_vJQURnYHIvU;R(G>`O9S{QZBh_a0zU6z$q@cXfAFb(5211qq5` zz(m@e1`{BN1VJTegRrOQ3y1IL(o|~}h_an<2u2N%g2IDbx#jdK5k-}KRx*Qsg z7LW3gR)y-TF;?ZM@zXj=s=9SuG+oslY+Jm{h;D;wlP*(T)9q1pkTz%NtMXN!WBrYd zL99;0=-X^?s2BeBpyKiS1YxQ}!&JvGkk3bc^ea(Er_*~Xs1vJS?;kq9mWJ`{q~?*N z9d>x~yvw1DH@q5|t(ha#_?_mFv|lcV^PMph!VMZd6gfNlXc#_3!y7)n5+3<{ z^T@li%+Mb5Sh#e_ha!+6+BSNR_gx3Go_&sJA9r5=yHoQfo^JNqFTW0!h zoB^vb*NlYIS|p@5#WV4DoDV7ArC&li-RIla9~FKd^{RC<^@17sd1Q-dcqaCQ=iW1~ z3RlEgld~u@zRpl?>|d*P^H(R`kvA;yZGtjGx=hrSu?M1jpOo+G@=>O*%cS?+lRHhn z($w{-U%jXLrT)d%@w6GaLVUoA)T(c&e#O#NKBcGhRX&}rR}DjB{!Ao4VHVrdFvZ}h&VIy3P64(06;seJ#a(AU3=4^=GGJ=%{|kX3&gA6+LZ4cDUK z_&7&Y8#4c;_}S^ubXukw_gMZEU#DMLwreF#*LlmV-n9}s5ws^d|FoUM3U`m(cQBSt z+e6#=EPf-eJGXZ|ct~2}38^o@{h{szb+3m!jHC>!LceooXC(y_tRA zRfVQQKdk0k=>>E$#-HlwwSo1c%AqEycs&A-h{aeq-V^#DnE8DMcqRg^#;q=I`fN+3q9Yuvp;BIt?2lvJXe-AT;+6Z zpZv{jeyD7rSIIrwa8=&r-{GuqYAFBUj=sHYSg6L!ZTzpl|1{gYBlC5S9I+>aB<6~S z1NBDlchp-obAB6ErR(7z>W$~kTnnmi{zv^HErz<$eN6W?oAl`YlJ-P@fAQSo$uV7p zv3;DTXBSr5n~&xVjqKRZ57zs}{(k;9ciW+0dY3_iLnUj)zCGgaoLA-5>8gCCtMUZP znX;~5q)r8KRr-kVAL-3t3lP-5t!|6R(GKjYbd((??TLQ>1oOuG5)P)v`yL6VXTD2k zwEwES@o|XW|5`s-PP{Kyl^^eWR%NQb=rUDbuF4x9hpW=jm#~`0L38@~qsw}D7JONq zhSgkP93pB?-)!06pJE?NUV?MMG+Z`!fB)X47n7T(tf)@IDsNWfyzZ}l{!=6VPJZ8c zFG=b2Df@J!=$XGl-5T}x>l`|sJbkGjO-FxyHP@uBR2o)!F;>ZH-btHNX}D>w{{B}- zFD7roJDJkYYK@V4)09qpj-vO8)+cR_>cs1fkh-oKw|M)*q|FKAM%q@LhDjUroJRkC zLfQ}=AH9B*e*At3lfFcC;&VsaaMc*b*8uHH!Wfb{jgDb_zGz)~+^$-Oq|IriYW4TO zFaK+D<=Q2yEgjv@|E*}Q(8q7P)oHl+$$tJAeCIv)-27@ZTn@iA)<5NhzTNv?^~Yfs zP;qNHHs$+m4Z(Y71Wwr9&# zO(OkDrD57O8Ybmu+s-*SQ<%F^?Arn|qT*`)Aww#z!Zcm!s%vqbjJyGTszb}kNdI@r zQEgFrs;uZ^*p`SfO2Fg&nSCtd(D=mfS#INZ*R7=;dujzP}I`MY=qt3rm zmo6u+n^2d|uXW-wOPzpjmO8qexNep@x}23?^-u56YA2}BC%9G-dXAdbQ5%R z{(nbD>&E3jmV@u=hE=IX7V9TLCtu6FkH-B;8ep-T9naGUfGzh4_F^IAv$!T=d==)T|&I{j&a4)TVOHxkT?x{^>=I9M0T zOh%dEc$p!T83~qo3p9_cT&hbPv;eCe_IxLBvzybvd}WkF#ys0L(DG@RjJaArddw5b zA?1hZypy~M^F{g_A>$J!{Yn^HVwbIK%zb5Vuhw?JhCR!4*l}yFfP}m>o!a2j@@YD; z_g6OT8FkV001rS`I*LmnY6Gv@Ps<^7DI4}I`7TJCl?^-PF36~hjES;g*L_!f2->dS zr}a{Os)M}R_CG3fj(Afp^IR!T>Yttuy?+Nau2%c1&Ihvr3l^gJe%L(|bOy&hD* zGU}q~YFyQ|_!wrT6I^>4I;5`P+RM-(Z4R!z3?0(ngtbT8OvWc$jt;6r=1#4#jJjw# znbQOvnoib+T1Oe>(Dc9>%StD>4l{H}UBPvjp+njnT!$Gtr0>CXn4v@4|DO~l92614 zK@lPx6cNHfkxY0eB7}D$M0h76ghwKYh6#^E8VwUpi3qW?OC_8VKH-!|CY%yJ;gkpw z?uZECj;MB5CU=BSxFbS@=fS68!t;<$!-VI-r(weLkVeB>GdUm9XqfOk_%uv-9()=m zJP%1UOgJCX^G%u-9)5p?@RmxQBZTuILO377g!3UxI3L1<+aaBBJ|xq0!ugO)I3Lmp zpF=v~d`Kpo4`Hf9I3Lmp=R=rqK12xTLzr+rq!Z4E2;qDP6V8VS;d}@Y&W8};dh!D<)bi(-%A)F6k!ub#)oDU(w`4A$U4N z=L5AB^$F(#wFC7D=Yu-WtT`Wi!uddLOMSxm5F(rpKH+=_6V3;pa6V8wJD+eqgbC+E z`U4g7R_ZcD2z;aKnq$3p1(>zn%>3+)UQU;JK(a4aMfj)f56SO^i0g*3vkkWM%j(g}Y8h&M7g z7SajFLWt@Rj)f56SV$%u3n9X>;1iC85aC!zCL9YP!m;2NdZ~T*f&MMS^U8Pi3CBX1 za4dug$3mEJEQATif=@UWe8RC1A{-0JgkvF0I2OW$VL^u{g zgkvE@I2Mu#uR;R zg>=HP5GEW8VPZR=?3gs4l}|VpLWE<%CmahQ!m*G>I2L@uv5-zU7AS{ORl>1AxtLZG zjs?oCRGn}vP>!Y~`n)3?3(6K%b0ztNV}WuX`GjL3OgI)OpOR1abHa@fCL9at^cg`o z7SiZGPdFCBv>f^zAsh=~`V1xf2_eFvEUPqg%H)J-ynozf%1L%gkvE@I2Mu#$AV8d782ew3CBW+ zen$|Fg=E69P@Qlrgb2riPdFBm3CBW+a4h(QVMU4=S zg)re*h!Bp2FyT)~BOD76!m*G{bqL2ogm5e*6JCV~4HN!^bi%QaOgI+8REOBEMhM43 z8nIW65RQfFgkvF{a4b|O91Gzka~FqZF02)HzJ4J>I2O_f$3mEJETmJrHo~z$ZB$ng zjs&g)p&8T}?O^(h0|c`nJ^^3lYMxKW10h5>5F&&FAt6k7 z57KCu@E$~HnD8DX(Jbo0`di;1dpn(7Wb6k+J>nj=YfjOfr4G5Do;&;j@ZxAW#mU)r13qa`>bY4ulBd zKnN2K1j^?VAsh&l&nH4S5Y#u6=0J!L4g{Y*YX}DdWE*=`)3JAW+_(2>m+-eWnl&goNh^ zeRljcs88hBy2nB#FXv4r90lNo%-|>p6JCM{4HI61BpN2X1Zgx(cnKmjtj>SL{$-qH z44k8kT~)Tz$_|^@z(Oza++*CX`KolCm+%Q(rK8GLdfDl~MqTp*PB&et?89%@en4SNs6URF7~1_%y8aXdWFVoB*1WAfZmh zJ3z;o*z&7BYP;j=#FjtGn-I6nc6nrN157|KZI+rn~U8O#Fk&RIYVEyjjZ{2xv>_ zG0fD-u;oYl^*tHWiQDogtpA`+V%a*smI=1}VPeZ4Cbs+uVPe0pY|*t{f0)?vhlwpe zV>Untb7_Kw(}7h*5$Wo7RhLeuw#EUSjJ{BvOdINGIT<>%-b@{Ba~y4F zf^9hMqq-XV7quVHu%XUIC&AvC+L-H7)Exa6?a;~iM8{^OZjYM7EbPz|#-HkB+9~TX zRwYGkuA{clDn`eL&MzH9noh@G<&XVK?4wSN_dR0_X*t=IPumauEZ0ULZ-x%3SIuum zy@B+s?9ik4haTHt`SCvb#2#GDL;Rj&LSOql3`FAa~0S6G(Eep($3Bn8+NbC z4m~SdX=R5VZIia24Yp0EtMav7ZBQpWTkPy?x3jaqj@w7;_GD?h9q&sh*oJuD{a`wM z$BVcBs=Qg*I0ws#_vNbc<9*Mn%ztT9pPfDazp_DAbAfSCcIesKGE+Nc8diC;8fR*o ztaP%oYyMYu+$!(CvIh^^cW1T6g0|c7Inrw*OB?Wbz1iBotG314pRFA|X3nbYb@KRe!l_*_T#7qm^?Q$A^P61BOe zcG)ycZLw*X+Kf|Ma%v||%CAPlq?~FbZxSi9I?-QE%1otUQqF1`Cgr3OJM=WtuLN6r zQrCaEoof7J|MWLsQ0nVSW?YM|<;-}NI?;L4&jG5DnNPnfYe}_qT8505m!*!b>z{NY zv7EZRe=H}io1hb@Lytw4I=ZfSo-B2Ad2!t=b#ysd=>)E2*&eOq$4dW49jzOe3GLC> zbU9h+=xbX3lTQ3vmUE%=XUObqoHN%}q8)VH#yL<&mUTsSGWGvaN7->hKT!vH}+PK z4zBL@G**<5~m7U!#)ggAi2{y~5t^^xl90@5CzRdTlKacQ7LM)<|Ec2;-|y(&cP&iT{Qu(d$#}NL*Y34=Y+kR8!?kgU z?s5M?_eE=8wiz9|a^(i!_@GhJl`BvC^S>O8-#1J7DOIY153jZAD|{*6?j1`nIHqg# z`U_`badlng(|THttSv((@+qmxpZo18aA%f^%UHdtd{uvDeX)8o+Y@V7W;?EKPq1CO zeYzdAeOa|r>lN94I@-QcaT$!?^Q(yVM@iLQ8V~j_7+3mqJg$zn_V1kdPUtemapNbW zGo)(oTU958YV;}Tx9&GB>HA(K{TlPyX2|=U8~l`J#gnvD<)+o>9y+qLq<{17j-kdY zO8Tev_706$T+(MoN~q;0CH?80x@1Vz&SA&iiO$c-j+0RD4gUPys{WW%<9=$#`{-|L zzt8b8QTgUyQ!oy#eO72}bR5<^(*^6Kl;671bW#=>iWht zs+>~(&3R+_yq~8;;{{7JkB;lM>90iPFr!0MHmLPV=+)^Z{o3OTW=NHvj{C-`CH+e` z<_>9@y!U#4UqpnoRQhxtca1CQ-#Kqz^0777`{#c>ogoK}FX?yg`(w2I@2*UZ%6&C{ zj>;l;?aGj=_|<~*N^X%T6{q`?x*m-G({n)W3t2s{vT8?SIr=%8DP#TA&r2<_p5xE& zpbUHUg`ZPyXTVXzuIT?exdh|;B!bnuT($(-RDu_b4Wj*O8NSE6nqZp z`%*uTg3lqnZ|LU{eGaMj81;P8_m`IXxud0i?r53S{L}T5-E&Cq11i6M9{sE5P{#8r z{yd`3AytlgKI!?=GWeX(@!)es<=6YBmU_Ij%-Bcc`>Nh|wanP};`^YU4=qcP{V-Zz zVE+v68#=D{8!Z#}ky5^n>wQQ`J%8EB;Qknl2lrJSr_XWqT&K@{^&X(E)91R5>-PaI zRl0u8X{n!!S|&dKGT+l;&r9rkf&E(V>sso4TT8vqD;a#=srk^)MJ3hs;Pa0@FO@!h zztH!+-UqbQ>Dj%1==T*ZgU>%5*Uw2U_5Guz+E?`Zh?eyILtPKPpQyO%hknk)Waj=F z`)-OopEBny^LeAbpH#WQ=Tc&Mncr3F`4il4bzGINz5}#W>qG6oTI%~*%iw;jI`bnP) zsvY_}Mejpe>gR-(`gx(H{(jL??}J*>`$0WF^z%eZ^?jq#wbb7~Sv_Zh`;>m(WGD6W zCn#0_gYP53IDI}TeHsrwuXJ30KWVA&J1tc?`Z=a0eV#?@!FQIL5Bhzj4bEcyOQBakcIe->2|AjQ?FkNp)XRsp_TIRh+(WseaM- z%3xgSC%)eWN=I`I?~j?c>0>C{sN@$o76X&?@ZC5&{-3b zUHf96zmnW&=a6VQdE@1s!u42t3?tsZc)L=f`C`SZcGI|OUy2{k2YnS+{fo(1{|#Tu z`Zd=^uWPCFV)Z9V)gP4}>sPEk)!!>uBB5Bj*MwAi7ecQA`jIRD_Nac5`P-xEDz5%c ztn;h%%ynQGv3ZY4HLfbIrRryVz3BCj5nlkkOOyNhL8X~s z9ntdDe5g2GmpV?@TjKiB{fYG>*3XpadPm;)I@B@+>o#-#WBo~7H!6Ru{@D5mt_!`c z^t!ofeMHASCAyx}`U&3uH2zQbd;I>1-%lygaY1+D_fu9<^}j~^KC0pCdDk*N{~7m{ zzJK(6rSHGsdZO!3ugBo}qw6-ibrCQ3U)~Q&Pv1vcs(H-1-2eK%jH2S}U$2+M_`kpY zJ&dK=_q9~_tBPmJl+=dNIF+NmKOfbrGTMsfS5jTqQe9VZEwx@|J(=~?z*Y2XLtY0HhK2jvN*=e$(I56W{WN2Lel6CeH>>Voov^6-&Aqv=7p z59zs3PEfW&IVwFU|Gx85NVPvGQ;wO@_6KDjq^oj*GH0br(ei^b*q>m(gY5~nJ6K+@ z-r)EK=OH*g!Ep@sFE|cr{nVJyAi7?aOi7i|b*E*cKAyku?S_6(4h-n&xGJwE%6SiR zKy;nQ;=}8?e%jBmdSdI*i`MgQ(U^RsXfc0S{!c<$s&Z6ZOP#KzPS^6b8bzY|8(+&A zl`ZoY@zs8 zwYZLaT9*1aN3{H)RPE0_>8ofzb3(R4``8-?GNe%T=lv!Wl&Zab8XSnWmr7N?f&7!= zQnk-Rxk{??RsFq?FDRq!L_J!na@08K{^+>wuaas!l~m*K;O9B~Gxf$rb8dY-hb4ZZv*@8h#Yd{w^E_fSqy?#8&NdbL#7=RY($I-Y7= z)c9(t##xQO8W%bbDqYo6V&A^dBFO6@zeKqzJt$Q^j&aj6Wp`2EhMpd8CF%I<`BCYr z9XhVYPt_M4AC#x18b76{rOMat@cxim*UN`ZiON=Qj}LhT^7ySGccPrt^~XhJ8l=)2 zKK`xHCdeAkPYC_AsF2?Vdg}hrawWEG@p`6 zzvI+w&3hRJ)YEYKJOENtLeJ`8e`-_$*g6K6Zaje<9MN zOsSreDz2q^u4RAy(Qifcx|VH|Pet=<8BC9ro5lM<{QXS7w`EAQBkewcb_OK$ z`>OOTWb}I=SYEV0S=XCoIl=l!JX(IRykLDgPVy!6Ba3$HcKpw`Ka2OaXnTZ)B^OzPj&Jek#@V zDyx4FDOoix^KSe#q{_<|mrB2Hy~ELbN>9a=4EA5e)j0U^aZ+)$PwVkAI=TLv;|_+D zoB*k=Pd*#lpOv20*W;t5nr}6JbR5fbDkhd&45tVwJ^f)g6 zMED)8!yzqozxB9`nQ$tku4}2*Chj}6mtB+B#^KsJ_>aF2V!uzR-?g$nSDg8~xBgvB z|1M5{_qzIbvBclK)bB>IbILK9SzoMPb)BB?QgL0Mmh|^cbv+ml{ys|M!S)5?syy1C ztK+S&o%_9Z?icT9S)U8i=Q*?tp3Bm4eQql#^|=`>)%v5qvu1bhCU`zapVQIlsvLd4 zDyi1JK3AhDswpdOlvCLkmj%J7Z9)dDQ2@ zw4~?2)b-#wGJRf5OMUK6OM1>u<=5x*f>NK$OO*P#@!xmum+t?;{W%y{?G5h#!8qOj zm3}b(Py2mV^(E?G{e08UDJu23FfD`U!t{Qj&wXjB&wZ(OUptqZXwO6ES)XgtGI*{@ zjbB#hnAH5z^GixEcwQ-ZUf|lf+?c)1wR5>K`ysurYDxFwYv*!f=XBM5Mc+Tvdqm8R zF!A}J_Qgbd#_Ycn63a{cZqWN|Q0jeGOVy5R=W@s5_kV34sL$hSdF@;-wNKRNbJcnd zzDES(^!cRpXJ-XwAANuwN&Nk=a`oNyUyh%e$NNruYz&a-^BN=VEjL8A5s0K z-+RIDIQ>1R)Ac$2pw#F2wN&G(&+}`k&hxAEYv=m^>*xBfovZzyI#>JOyFbK!H@o_G zPI|sgog>ik;CVKcPS3yTc<{Wdj;r6%63-JRo)gr+C*se^8Rz$uzCMqr(&@QFdY(|7 zD^S<}U(Tjg5=?DIj?MPRG^!=y{?tyWem1 zykEU;uAM`r_ocq?wAA;Tmcjd0pC{EaxDWhqJ1?r%l{yCzKOd`>U_xB4KP~@h9qRL! zRO<7UTGDfvR8Nf~jR*B~el3IN19d!jj!?&g=M{B4c&<^$gXar%Ja`UK$Ajk`bv$@3 zQpbbm6Lmazj#0;h=OuMKc&<{%bw9M!{nb*pLrdLWEp<6s>iV?Q*UUV(89b+{<7%9)o!7i}UQ?gb)biSS&HvZ)n(AEU)$jNEJ@zVz z=U2kHLmkI^yFPCil>hVR5C6Bf|JwOUwa&6S|EbRbYN?(B*UnF7o*z}?aqax%wR4v0 zgXY?K)BnD8U>cUe;anppTplAoTp^d+#SYUaQ7M}9FOYfk7|~R z>QszgHKXZ8kdhPrF2gpMQ4LtlxX-9$+-5ureAZ}hlr)MO^?~(`MuspPqcX6vk!0Ls z+-bA}wliKZt~ZJs4S@}f#|_VrMlvutYASlS@jO<+^F~#pv~h#c1lYuQ!pLRhFlqp6 z821}xjT?<8flnIEjC@9J;{o6U#>2)C%mHd+{6fn5=6VLWAY19n5~DWj>;9oQYQ zrbc6<2e1cXjg1CIPhd~P8W@iny@0(Cd(?Qu=nd?R*dsf}H7&n`njHSjO#+~L;_yfi;V}?=K90vcH zan!iq{0R7wG29qo%rR=4BjCR^1{;%&Qs!Xz9mX5RhsJE+Y~xE~rSUhetb`wIq!_=V zwiNjOD1Vu80eAuRd}6FH&H>LEe;Lcs=dr-C##z)?-25E)x$&7%+$?Es18y@uH%gjC z&8@(##x}IIqPYdQ#n@z2G;cRI12-F6jN8py%#FZ}#%8qdQS(ROkH)t~J+p!NJ@9+u z2jemG5%l&k`0t>#(P+jt!?$32jeSNnb1!_d`L(gf_`vuYzPfqB_{~^?QC@;^{>?aJ zECwz{>@@n70!%R$VH^vX8-N>(PmKa*A#*)&y|KY4Wacy10oNJpG42nVKLLL-ju{V{ z51Yq<$Bmziht2!Vqrjuaag1YGa~E)zvBM~9mN$0-cN)8l^5%`0x$^KkF{-JCWv+z( z+gNR+8>YD$o|$Qw?@LA+ykYJ)4jK0X?=@!`2aUtVUFJdfd(6Ye0nF`T_`A%o5ixQB zbD7eN7;B9jz#L{SbB*yj*3lYx&s=DBGoJ-MYqm2VHyfMHfX&P&%&}%mvjwn)S%OV4 zyO^(-6X4IAlX3kC;1lL9v$y%Wc>;LCeAR4fHZgBxP2pRaADCmz0_+3$Bj$8mX$5R$ zZZY3AQ_V}jOQvKE%}M4TW<&U1<`Q$7xgEIOJYWtn7n{Y{5cqU+fccKO0l2|TF;|*B z%uc{gropv&e zpKeBg5p$?H%v@ux1FkdQfLn)tzX9JE^}S=h4Sd`D6z*+v25^SC32p{@vI%|*+B(gg z3Y=G?1bNi_N_3N1DBg$ntRMo%w@o3<~Qco=2G6Wn0)HQ;OJN%I$T5PJ0s{BIb? zN#2-Q_a=D)#hJttIbuwRpw>5RT$aJ@CLKld*&>R z<}8!p-z;+`@Talq1$rP+A%P4fnJBYP8T zsR_bO%rL@Xc9=~w@|)+(X~sFTvw6xmX&eV0H-9z1z z<}BbW){EseM;krOyk<}Hck>tHl<_O@SMwBRys^28y#fC@TaP`sA+RB9%nCyHce5b; zX^ih6b1hcUTDF3XGb)&0nB$Bu%qOvn9x&cuOW~`qwYb+F1wP6euqx(mqYJPLo55}} z>zMn^o8Uh;uS0pKfv3$saVLh2x-0_!C>w-*=RV+l?0vQh_xD4-~;So zmLK{4H1orsGgGlc)L^Ob53;w6V&+!!En}%?su-?9El!GWcq20q&J#YXN)?>tpP9_pp!Q@587!H6y?X3$t$68}eG+;FB!h zENu$r!(TKXLU|W}7tFuSCD<+RVoTufVKmx0n^wj-0zLd6~Gnj z6IK)X{x)mEU%_2E+}w%#cPFc2)iUQ9t<73yYjZE|_LXKCs|S3@T8(vE0a(Gh$tr`g zEmj8JWDAX&C}E*-#eB*9-8gTAfnhTZt9i9~leG%Iob|e~#;DI;hp)(1V9%&-t$@GL zDrc6#mRb(pVdsrMjkV@^_%Lc+WtOv6!k4iI;(3+K2Etch%W*H?Xf1~?WtBF|nGP!r zFIWkT@kX-*{Ccb~tSsOutTKT)c^de%S%ytAE*mb;W&O;{hHd6xmyI0EW!EF$dh>et z&E^^N%JXlzVVLH7TDH&#&papMjqyvd00+X47!`mV(?qd_l*TcM_@5$(->nsW&R1j z*1X%SWEN(3!xvU`_ZSXtHs^)t}2Ve*DnE9@8w^@+AYZPS9LHiD~D60ftm{ox8 z4zmLM7ueU9nkB3y@WrgUn9sdtUHILY#dpoWSSox4%QbVGC7BCf9J|3{vxtSA!Me`+ z2s?zr>#PD+ZZoG@isgnc$?Btw-DZ9Geb`$*HVaxG!soNz!Smr>;Jwy4mJ{P$23Urb zVs)Uq5BJ_a+R%3`|Vs)T65sJ^|LYF=wtq@y!i&+Zw9b8 z;Fq%3@T{x|tjO+Q-{2i?5pWSpVbjpdV%9YH-`E{kcWD@lG>pV7JdX@(7W`#40Pj2% zfEC!SED7Ux%uIs+9((F6^D_3+%j^=nXxwKWG%p$l&HO9{?@wicWm$Q)(im(k04`uP z*h%AO<7X3S9>*Ou+5DM34xh%R8m?I!Se!M-$j&q`VRyd7{$g+A`E?F>j{V8Hu^wzD za3*_)Joi9S0s~%UOG*odup{=UGSg0=odbz%H_<*exs- zn96=)FC*dv|{24t3P^=93fLfaD7GwgP@2JKjb zR`g}PnXvl8=d|Xb&hu;@{C(CC_8Q9r%wy%ZhOr?`VScNSHJlA&3JY0jE2tI*VFk;EP)e(UvQ0A$(=)O_h1Xl9tx0SmD+?@ZmA5`*6If|r zX{)L=g-v4Rf#t26t!ZouQ+Ts=i#3rgVKsm?tg@DmvMU2CTRYfI7_;TT@ju- zcnCS}#`RBupRx_?UPy%-*d|t$Rc2d&Ti7<_tQy5_$X%29YzJ@$+r?_J``H)3FW8r? z4y3{_**^9j%Gd?mfs4odC{&#Whrb`(+8}Q^$7eotR>QZ z0sg{HV#TzK;z_KUMr=9Sh^TXf|**tIM|FExrY?1#87N;Tu3)B`c z_Gkxe$2zc(`Ga}86@s6Ix6q;HR^V2)neD{a)$_pT*^6u=zRhk0-pZP>!)6t825<(u z)f$GkiO<Hv09^f9flkLE}{!74@SSNM_-!`p*t=R~?L5()Q z0)EAI;|*$z`8DuswwJw*_v<^X*Wov^ea2p+C$J~$!?xmk?+)M{tPS3v#+e6!2ibnM z2j4WW0$*kQ*)V)xbpUo?^Vw|Eu}o{WXwIJOuwfG~P0QWE0`P#~a@Vre%$R|C$}Z z_sCG-Q1%Afj&Gsq!0GIH_7>hzj{%RdAK7>KDjfkF!A7yc_!jB}?8FwZ$#{Q00X)He zW4}v*;v*RZ;ShZ``KOA6ufW$%BH}dVBZ=)8smWD*myP-@8rJ$e`CM0>g_bz7@x_5%3?g>3E<29r!yt&3?kS*<|2kHkHl5JO7`+KiThW z622Wi0e-@cv19m_n+lxDrn95QaAO&88AAfb%1{cIbVcU59?K{FR-uG2d=L*064(vk9PEj>;>NsZh$oiILI1^77T>! z4W9xx2su*VRT+bz)!1r^Ry2;b<_YUbwCoA^=d7pDmuA3bX!B%iDq1udzLho2dK;~r z2LHVEjx`gle+RxT+SAAC3hZiiLa4v>3h))HTeMv-17EhfU_{;f7NP$aqJE3jaVy;%TVA@%s>Z>%ShlzYZyY! zF!Im9kFvhDez%STk6SCPL)Kr`QQ%SQQ){ku$~q4`Z+&I`Y3;N&0XJFSA!WX`5V+8q zW6iTZv_1lUWG%23TT37p!+(g-QtK1oCy1@IR$0q|%dKVBYUEi6TxhL8j*oCX&5BrS zfNQK&gw|T?fa?(Z)LIYuDg0W5Hdq^h8xh-NZHC+gzX73CXl{nz2)_=v8})w({LtD1 z_aR!g2mVWIFWNj8I2V2S8f{qyTxMOi{zf~O0GC+!!=b$dxea@Hu03 z0UiV%L`&8J4*?IMW#6F{8-N?EBXAqgz9aBIz^%vCAK;H#TTn(SFxA=ymx{J-gWnFf z8M(K^??h{kTPJ`gFixwjU#yeBlNhHtz*E3eXmum&G3#^S=hhDEVQaMYBk)J-f>jS` zJAgZ^UDj*X8`e+2pR8ln<5o*+5pa?9g*DL{Ykdp+);er8w`N(U-5ma7Yq&MS`W5)A z^|SS!^{#aVc*SD&yVi8;BH|Yj8)3a}{RaFET5qC^!@$GV_tp$+inR~8&$@)rZ0ij0 z3}PRk{$s#n)@iizO=ymW|IvB?ZQKRiWu3KpTAi)^!2MPjExX_P1NaA8SI6pZr32Hg zt*G@u>m2YLYJSXm#QF^Q8G4_~Zec%T=YlU`yEeCT0CU)R;d0nHfjRBm_U(2hyAH69 z-PlgDeftsMBX&KzvR&0~3T$dWWtXwr+wI^A*yVxc5pRi{IpG|;Ib3VRb3(SX^Psj8 zz!G**TS6)s?104t#Ee7IT)SPHT0>|*vE zz&q@U2o<;Q0^WsKal43pFYsQ(ir{)hU`51=A#ZhHb-M~e`Roudgjhc4)&$l>tT?V$ z0aih*hJBy?B=AYQu6=`D+O7|*Z#T57*~#{!z(?&T?6USv_Wi*7?Fa3f?d$CZzy|gM z_AT~pb|YXT`*HhD`)>PT;KO!p`$oH*{SfdW`!V|-`&PRNu!&vEe%5{gJ$}J{4%p0Y zYd;Ts-hSF{Wp}{*w1RJe6x+@Z%#Z5>?3e6UfUnqZ+o|?p`ykv8_A$6E_GY*hNKHkW zO4*Fii*`?YE^w~h$sPu&u#>&g?r6Vkr@#%ihr$iAd%(SF_lN6+YaMY#rT0L5D11kI zmc85F1-IA!8tx-|A>4GN&O(|>S%^?)yQ_Tw?ueZRw+itCke%%j_IP_T+*Era+{@AO z->^r+jl;DOxT4ZKB0dp*1bR5Z{uKBrdifE`-Urte<#e{^A-)gtBl}Hzxcv$66MG|i ze$4(6uAAKjZav~ZLT-WYX77Oe!k!8DF5){NyVx`A4_zs+c3-;>u#dgeUSjvQy92x13os)q?X|$Q_9%O#y~Yj$!}eHvjGd1D zje#GIv6x^_0Zy^E!cDO!0Vmm?+uQ87Fbdn?H=?Y8sIxo#0`znbW^9lBCGbmoCvdO* z4aV{tdp~f${T1+l{Vi6|xAu3y@9aas@9pCl%j5P>z@O}+zItN(ltu`jv0FH=IcMzGoU?W_r-L)pX$C*o zdCvLMK4(AYoU?~Hr|m!NwoY5;1*bn;U&Px&p0)=#&pYkmnmeuGx;d}F^>GHkwU4H} zf>3woX{Qz3@AgT!ml1E}oV2?;7j4tIY+tl5+vjcD5l$}0hW8xiT(TXYj$M^Sbj4X8m>e!OqWieU$RE-P!rg9_Ng9M#8Oe(&1J+E8vzni{U$gur=3)1o^#ka06gH#b|yMooz1|_&Uj~p zv)=gw_=gjA7CPTKM}SA1`OakLbLTVQXU<#B$Ig$=_rUL+51r}GE@wM%yEDaE;v9F5 z0*^Y2oOhf(&Q9P?XPUFz`NjDO_>;5LndN-t>;~?3-gZ_wzd0v>C!9~551enDy}-TB zOy^VQoO2#{-q{Fu-njt0;GBiK;QR&r%lRGdFXtlgqH_xFqH_s&$@vxTlJht4Z|7&Y zzn#m#%g!;l%gz;u}6HviK3 z66ioW{0rv`Acy39hqD7HAO+v%Yy(P2$+tLLfaiec(4VJx0e%yI3ceKgcurmzSeO^( z$$SWY!%XH=jG??HABx{FYx3#FSpE>M7X=pO^Y|d+cK!nJ1wO&Z#q;pjc`o>3ydkf` zUj@F(m-9MED+VmaC-X;nJ01Z>_)PBd`}t_#XkL&%%^&7-fOGg3eiv`fKL&oxyYgYi zBm6DkTYQG`9M8vZ<&X1nz;e7IuZ*;kz>>T# z&w+Xh0t@nvD7h=|1?o<&1-BS;+X}up*4R)!5;&5F;70P{z~Q_G_xT%GM?QQWeuMaq z9|j)gzw+IDFaH|&HUENJVvJ~ETJSARAzl=(0AIoHnK{IRVlU4Dznk~Q@0YDaZ}{{4 zOa2X?D87W>!;6TF9QEe9+o%8UE?R7Vr$rmF3%HB_iE(^RxS~D0htTsv0wrS4W9|e{Al6B|ByI!V zhVg$%ydo+BDJufZFh!LVR{4j*dVt&iOj}Vry#W>(NtTkJ3 zF#$LMF$ZgCB5$%k^z$o;bD1MDN- zd4^Y)fAfvJjyM23z;DJ~K2wgCzwy!XR;_Q#vv@br1K2~n&2QyDp`=^k zQ6j%XR1#fyCDD^t6l3L^vLbTbCi3ui#9H85;qzO>DEYR$MZ7J4;qUWaqMLXhrOo0! zk?S7hyGL}x4l`PgMV(`jKPR6q76BKDdwB&hLXMIZP{J+feIL;a*bDim@yepAn1<1w zgq3v)E2|Lvi+l)wOY8^k#|)LgX!I2&;QR4@!2Xc^#cO;9uP!Q!87N^IzmD(2%DN7| z2(Q2o@*}__{CYl^4-kXIU_MCn=Tk7t)iKZ2k*5s*ksk*h=eOV~^`4j_?%-2I4&EP{ zuS55B(TC6HAyGrj#|oUyZxX}gaCws$E^o%2{i2*EuM^WS?%mMqeZYOFdkG&ZhKnV9 zxLCv&^7}rr1*%>#T{`!?uq-6r<{0Q z4wdD^P+4AVDUROLUK1XR=c`4CX4v?k90C^*J=r;0RE5Eqc8eooq=3CHxOAO(g zamUrgJy#cbN{DW_=PgAa^LgIK9B<_1 z*UK{^FFzw5$J^B`@s7xaRg?#Baleb=@+n?i7T~!>Gx@sAjh>c6??;L|fOla16%x~k&A%H6n|n&5tFB6EnTMsssCa5ToT07kVv#tqm?$WJ5_`qdvZdTBTFO0QpLj+-E%za1ub3=K%f@oDXe>Vy--u`B zGx8f;-6sx}$_@>^UzEWQ^n$PV&*qIR-dJN?Als$CV=RLgtj?aV00bhf*gZmK#1N zw8qHyfbYqYa*3#h8kUHusAH-qBTEBIW9(*NWXfS|%E@WsPtjY>kbjC9vM73atGpR_ zvwT;Skh5ejSpxo)cwgKu%gFap!c;L0xvC;xRr#@a7n*gTTSu-Fv&EhAb~ziG?~A$O zE_tV%iBw0-^LkpLP)uM(>ma9dw{6wUQ2V@PI zhLqLfJ^X%omwb=ig|Z@|mV8J?#6vP&tV4~pP-iW~=cD97D0>jbI-i^*C(C^B1>`32 zn0yrYsN5j7h=%epxdkbk#8J%h1DNLrP{(IjMGdiz8sge^@q~O_ZpX^mCel$$B`aN2 zvOZut#Z&SLxf55ni!ZQRp2B*03h@mn`EF~2xZ9e`lJR7m4V=w}7`6E>u|w3xn|v*N z73>zf#KTCbjd$+#Vh*dx=dhthUH+B$Qq<*NiaPu(zfImD&+qR>Jj4@;tsq_KUB@BhakJ)A6=21@8+}ZH` zb?M{z=F2O1`~OZH5%qDk0lrTzi{0XB^yp!JnTO>4@-n|)ns|e%DIb)Ecu+FDSyz&G zSm*g2R&VnO{;T*|Ji&h!P4JEMrI>}j&cfbgiHBuvX^Gm>7LEC7aT0h^Jc%#6`E0)! zX6#4bI?JM>u;?rci;mE$E$ag7N{+X&*W}&Uckf0&Tk`Yb57CnUA)0X^>dE?2i272B zR{Sq<4tP$q;Dtp|@i5|GBZteJ65bWa{I~qKRyZy{)OtDW2t)H0870lx=uR z^DgUQ+0uL%{mhB0PvZKMGM8x0xwK_#Zp*ej7gCxbtr_;^_B@AlWqa<*=Xe+M9;=@0 zV%9@{^7CSn%lydI8eb(XWGg%eTFJbk1J5OM$PUQWp65kMYoxWt9_pdBrO@6|cz0=s zUgSbQazXDolzaoqzCq^WKgmDkm+~k0wP(2Zk6xD ze~0vyG8LFASIHGJU50^SxmqrhX^_j{(-B%LKLvh@*m4<>>w)X#I=KdUQh}**19F6M zeUto5eh&OxZbN8?+zH%?*cQ1Ratr(pgualwfx8jgDtAF{h5rJfZP45WzZ-rhu(7oa zwKulXWmEiunU0n{VXcr);up-7Xz^3%O%v-Uv}GOoyH1{vzo4CIz%==rJOx|^Tqe(; zJ)Q6y<|f(I`V8&qhTkwhldo7i(3URv4RZ%ZXbVPSE86fK{4sePxE;7%{vdw@ZUJtQ zN8}uov<vD|zKHxr#(;E4;JP169aatu0VFp&gFOg+k z*S*~>3t!beWEFHP0xPs&7cQS$09e4y z?-qnF04(4Zf-B$_1{QX&Lkq5hECOE?t}t>Gg;!-1hSqNUZh2YmhG*^`{C@eD+yj4E z?#1ty7l0Sg=CW=Lw5Tk+>-z5fXr&KtBlM8VfDEx*XvG057yLI?UbN&O@F0HARP8zh zJcQpfZ+6S0EjLHoRNlP>?JEy|9YPhMbqjnUx0qWJZ7T+!3!xj(){^i!5h{&7+yGw` zt?=ARz)F~b9PW8}5AYuME`%<~`+)Z$wpf;OuLoZ5ek32p@0Yi_55pgp3*?P%DR%*` zE|z=bZSEcJ9(jlRjeH!xUz+aY@Sn(|vYdO9dsN=!ek+$s!+pSADj#rvm;2?N?%nQw zdAEB&{vjW9Yr22Pn(k?N8kh{3?EWeb;Of1&ey_V%elIJy<=pR4+EIB@hTJmlNm<7I zS^f;H0$Ig9C(p{t?rrW_v}BJwEN^uypp6xfXSPgoZ**s)wF~5CnaAC4ZI=7-d*-)j zRW-D$n)`vAD~r3i-MKQiyHV~%jpwYr@*IB8{0O}+fqs{8=gB&5BX^Qp2mVdBmRrxw zVb_A^c0KnIHzzPBe$Q;{zTpmV+rqbZySRhg6kv+m-F?X&i(oKh3jcdzsMWru9d>Q<5{MpC#_PP7qFWm3o zesJ@6dAyt+9$!dRDR~ge@6B^(!~N!-fx8az*^v3Y?d}$LE!-w|JzOMO@B8j&a68@Y zaBHLKvk+eoza2fB<6Zz>aJRa-QO^&k?{Ku_d5HgjP;PItd&xcFZiZju7DvCYp#Oil zbK&MAeg(3)=Xmp@DSO?o;3VR6Asufwa6jBv?m@V(+%MfDsNopg5AM%!Ke*q!zo6xR z!kux?!<}(Yxff9b3^PzRetZZ1jagXb9snM|oUC%w+;4#2AeQQ`aCZTBxnE=4kGe~N zOELaGx<9##fQ#HE?s4~9cL8vL`!QzZQ}-P39Oh@8yV3mv_=kJe-GIKG0G@Dv$5D>TV(ktP0^QL(-fit~+-mBhw9)81w|Hba>{fgf# zPuM-ZK5*T={%`}mzHldOmC_%fGxk|~0dRpg*c;?6^gafD?2YwCd5gSr_9*y2?UCLe z_G!5Hz1eVYdLzBl_H5u7Z=5#=Zk9I%Zn`%N?lq*2L7GY#hR_gis5b-dEl>T9`MW&? z;gi5JUMa6Uu)J5ntLWY3-3z?eyTiN1yW6`Bc$-(&E9c$rRRUJ>ZuClfcY5~$@AGc* zZuTmB_WZ^d9jZ1wQK4^6vNQdJh2~@{+t{?_sYFu#Q*FtLoME z>I3WJznH4wJ?7N|*7F|lp7NS{4S)^2#@^#zORo#Ci`UG1(rfN@h3o9S1or~sT_KlFM3aS&wK5F?YxHGv)*%F2Ve)Uqu0V~?R5fn@?PYp6E@Zm9Pd}MHx=VM+nWzJ+xr0Xo8rv_&ci$pz z1RUhO;r(p?W={rAMmfv8rQRoS%e@tFE4@{4tG!gXG%p=4>_y<#cx&O-d7r|q_cp+7 zM9yW%r*f@|>ZeCF*G6?VM6a%hrbm#H3ct+T>}~S4z-{$DgWKkP4!7Og0k_lJ1^0!w z8*Y#HCEQ-`E4Y2$|7q{L!>lNtwYz6}0GAAsBnqe?h?3LpnH5kGF=GN$%%Ujh4+Ir- z1tTIVDhenfAYueTQGq?PD&~X<14>XqF@X^kk?*bEu0!pzE$2Sp_s9M2-JPdv-czTh zy1KeLOrPGFPjGySD__NRbk$o{{#`5ip_Tc>e*1>yzYoqkNU!F6p0hgVGaNtW{D9;8 zobPaao3jbWH#uM9*qE~c$5%OD;`kzGJ&tubYjLc>m7n7}y6QVC|FxC;(#ov0-~M3v z8^GCw^m)!tIh%8~;0P*f9IEn;oE8 zSN??S=&Bu7ew&rtYGr=0-|EVw=TC69BmI=K^_R{??N9C(7wmO$Eb-pwbS^sVxZ~pU zZyO#IQ5}%{G79BsZXG}Jig{i#EC(PH9e?$!`)`S)uY608`X=%Tt=!piPuDBe=;uI)}>AQo6MF5jls_YKSg{gBl(a|K|V$D z5-%8de5&fFx$z^DTgDQr@iP0aH^EO!zR;6M8If-y zWrUrHl#zTH*VSgIsO7LhACHN`H7kXkT zBl0C5a$Y6sLduA8d1XrJ$<#0Ol-DEbRQUh@Sv^wyQyfza{7=HZ_9Ei2M4OU)5pN~h zCgPL(jS_7c^}_uI)k_A;{z34meP#L+p_AGdUz-v(Q2XM1sqK{6E+ii~(`QO7+oQy? z-AN34xZj}uFoR|PAo$cDX8IGMr$pR}d`t9E!p;(XnkZL^`uZ0=?4J^K2YK#8O4Ob3 zE!oNbFRw>8F8J#4e^Or|%R>F2Du1tS`Fm~4Im8q&w^}-=S=nj?S*F%((@-fgLcMq zBm1eocq-%Jc-CTn97p0gkw@XFi^t$N7Eg>k4o^TFfa3%_74bwo`SD~Nr{J1?xPs)5 zvXaMHnG@}|{Ve|&aQY+BQz5Uy({2VLorV z7>hIrPgR_WCnJtQx)M*OybVv^xd7=RJgu<+Prez9bQ+$pI15j?xB_V$o~1YiPeUAr zbTOXpcpaV&ayrrwJPGnHH5GI!o&$LXo(FOx=#BQ-kay$RgqI*)isxhAgr_x}g>*Ka z1UXGj2c3@RK%S#+#xo1gK{^jlf}DZ>T^faS8J?9n2~R~j7ip-Pqh_nQI2PmiSoEB$ zg*XejddIZPAc)sI&JT-A1jt6ng99%*23$5g1R%X8ac8=vg0nQ?%d1~&D z>!m9YXD(U_kGd`Rz~3wV3%i>lda@+_Ob~y$=V+Yjd7UJ@IGr{B8?Rk6)U%BsIHn z%lx+I=UYBsOY;4?wank+mOLvX^rVHJw9u0ldeTBqTIfkvJ8oS3(2CmjKJLFWK5~6U z@?U&*-WIYo?vKyaPFR=vqIrdQ-tMV1l z7tkNIcY|lY;$KtIa_I@M%5tcU<8h5>g;nhHR4dz7Oiy2>zba_8mC>q|y7PecMK3&e zcKqvr>Upm&II!sQ@nbBWF{4Y-oPqV?MFXqn#Z5aG>52zh{MzsMz3R0%{_v#gd4G@T zP_+5><1PN<@dJxm;@Z^E`tikW+ZRoT&UU~vn{+N(Q{lk47w}x@d<;4}0IwL`p=c4V z?F-xkI$J|$E8uFl^u`>DI&N^KsFx4rPR<(I}RPyDj*q;B`dLz>)Zul3>F zK65fMKK%X5&&0=UTWIx2e8P@J@sG|niMKa@CT?{4Vk<9kv$LL#|87!T7Kt}+oEPgC zi}@0hPsX@>C2l!kZrq@0vAho-_TudLU}u95lg;cmiK&cl{4qUKUdCi|iw}$INMb6Z zy6c`yc?nN^a(bpt2XDJKS%39ho7aC<%}CaaD6Z>ukKLQxd-NR3CmE{8)|W4~*K%Ex z4B=WcpNb_WJ;c9y`6@eyG|^QTZzfG0YmSz@;*#@cDy<(qmwb^%dssn z>AAnc?2OK`Fxk+2bFqvMQ@-sNEs7hty5qJ$`d|EbQ5?CtV@!H>{JJQ<)YTnh(zDmN zr{i`{EVT8)ZH;VZOgh8aOXB2%g|?0ulg<%io{6_@S!nBpG3i{_@u~QQn#FO&?T+*m zyzx|gdi~AC;2 zCGprUi>)5UT*e;`nUN_YwJ$E$;HEQ5U@lj)a7LzFgh@|F_?+UL{KM^+^sIo-IffaN z9`)OzOx!XiJujeK@rr4ga&i0Rc1=2mpp3KHPRo>KR^dG*FzHzjKcDtRu|3>BkZp`f z5Ba%h_1*Sb^6fPZOJdTa(N}!5qc{e*-=k~WqOW*pdvX01p%1C{cS%gwR_gRn+|I=z z_oa00l~o^#>$^B)Og`!T)4ce47l({V&-j&dV|`F@U333QdcJyXPW%Asny#I@2e7MO z#&qrJiyn+WbMeOgGx>or>3LzV2jeST>@g;tFFiUpUg~0xu^6ul#duvP#_K{cUKfh- zx=@VQg<`x;#CV;E@j4OXbt1;=M2y#o7_WE1uHUx&(ir`1hF=zT{iK!gVb_mY86S3i zg_V)m^-Gp7v1`kgFR|+@EMH<5SC%iai&e{)*wwq``>=~KE91kijawOssf_4DEMH>R z?krzo*RCy}F~uj@O!Y!B?8fVaV$zM*3B|1&udNK#FUISHuI0KW8NwK^tvqAWLm1<= z)ybIha^tntSr%sfWShj4mm8yPUS(m**NxFO-`!!-&#@;l+3Ch>YbRsU@5XDZpE23y z#%pU^S(tRXG1}^6O!;zbOH4Z57;SZyg~=W_UR!(0!j!KYudP41Es%aUMqB-iNv9j5 ztxm?I$Boxk54Sb4nK9{fW3<)Dm~^@^+UjIXI^B3}b#l8SJ#LJ)dKi-)H%41Mj7g6h zudN<#*QCdd(N+&*(&NTxtA{a{5#x2HjMUb+To|uQU@jNN=uEi?lO8u-TRq%WoUdZ@MG3iQ7W{lke|HwOIOYH;Lu763c5Pme)#5cmD_ec}242`@@lUfm;V9F8>j)OnyQV z_tqH}zrFpX^loGl$oTaCtDbA#85a+q6YX*>@#wqK`Cf!QFwbGVr8J*-^j&Fw&a5bX zUm1F!J1HyQOUGZEZhY8i=uzrd(>X!fMdwbwQjkq>jnDq1v^}o8yZF=Dzl*#pZwda) z7CGsHf-%WkQ;tZ@KKrWV(sAnxr*xW~`+SoqIjzZw$@Ed}Q;iy)nY?-Vcj->km*r7e zsXSC(B;>6W*+JzXOl72U5T>$_4+xVVq|E%YuTC~sZm=?e;dc+J=<>Fpm6jlXEI zBK_3WRji&4ca9@Ja$d{}m+N!#@gVoOD5Q_2d;P=EC>ME&awOAMe*Jz3i0h^CbW4+cu>~*E=rh z)plVC`A%)yCk-#yKWVpNa%#Zj%aRqXu1a6=&j%&0edooik}pquF0V>%!$PTZ+uk|p z1>X;iP5TwL+-#dH@8yS%PJ-bhQ+NuMwO5uJ*0zlksi`P zx+Er@q>uEHPSQtuNhj$ey;A0??&FeX$4<>VE)G&3x2>37`oi{9-4_SN`|sT?nX~%I z)Qk#y#qB>Fo!oR3-i6-zv{d!aZcGnd@O$d+*(a2cAw8snbdes?LApo}=^$OChjfrG z(nC5(my{7@TrSF$7B&~w88$k(>&Wq`%3b#;6y+)*BYd6~K2LB(q^k5o<{ zFyw*UKCe_vUwq${$t5+z)N}I&C*e$$)T=uw@2B-Qq-RYZnXKF%rh4DeHF>mF`{e8K z3-X*#mWw#FG30zwDB`eC#G#EFDdT*S7ImH2@)Y|aO=WO?NJRbGcyfLyq*z(kx?+02 z!tLpmD+a|9yYi+*?4?EQ*|?E1Vr+d*VmY=-EXLMnC6?DpEU%SVUMn&68E(8rAL1LA z%jCy&tsAe=r~Ru8jn_W?|Ek9~e!1})eSFD$%NoDPPTzRv#_L`5&=^)cc9qmm;{=ag zG{zK-c~pT7AVp3Y zVliIJ@i7+TbqRSfUZ=%)UE*3XUgvR6RL^3(M*Z?%L5cE;@w&vdvh1>q|F6al-&p1w z*Ti`39oKy0h#VutcpU@Dd*>m-=l@r8 zBvGys^AIs!r^I;89wx^JX5OFRxWEu7P~`T;bFP@rc9L=JENa zzV+jEKW@(B^DT?2#|4n*v(0zMn|XY`qFvSa#?71aN}q$Ta^fL<*XETzPkgDaEf=3B zwQmp?95E%A&vk#R7ppyw?ZM}>t2c=AkEoT)XD;`{4Xc*$U+XU`+s|*+tQ9|7ZC%EX zqiffSE3Nz{kI(F1K0h&M4zeLWLL6i+n}S0gvpkI(<(jm-H{ z@E895W*(n6F0L21`swpbJk<;8$3?ep%;WQ4ht-eOg3Wn+ez-OCtzMOh$9jeM{;1V? ze17?4#7q9#JU+ixzkXbB?(|IjY?zPuJ!5Jvp9dhm)Ui`?`P}K@x^dCKin$zr`#ghq zd~QNdK992T`DqtE-~2V=@$nOK&q5p)xHzQq0i$47{SLW@R0e;1t<3qX;gH|aDA&es zjh%6P)RKB}!JZ3yF8>wZ|1qKmp9j>c8sBr_TbcGW_=C#vJssc5oBL=^-0j$%=Xd|A z9`_x+F4GPVUsXMxA8*Rz^K!J)8k2v@neKNk0s+%NK(`$ayN?icxU z>3)$vb6oP7`$ax;zsP6q7x~QnOnLuUa{MgaFY@Qo{UU!Z?;nfDY3>*K%>5#t%lpUT zevQkr8O540ba&$G+_Su&25uj;h#2T?;(ov;^84 zyR*B2cEdj6PT13X503k=Yj_IwY^QK^vYaVE1F+Y+Hjcxw5Bf;#f<6MradzkPDRzf* zZOcCboFkD2U`KUzn^#-xygmqfsoUY$3wyqsU{7}=9JMUJ9X=m~)JVn1aWIad*q1yU zdyz-sxCVQZZ@^yX7{_4CABE4ukz!buhyB-`kosdkcTMc}&PO@{VJG?p zI5$OVh~45fu)Dhr(pc;u?TB64Rgm_C9aXV|ycSY@?14TI5;r1^#lG`t*oEC1sRi~@ zx5qblcOdpe?~5JU(?F+TAN$_$+Dy=y*wx+`d#Nt~y#V=LhCS%*LEB^Rb_3wcKrh2y z>IT?LJpyzDSCa09--7W7)|=AH<8J?Qn= zuYEJVp8z@m<+>8zPXwKaJ?Y~>CxK4FzVfT_{f(eE!Y5M@8xMd!fc@ilVxRdO&^hXE z>?5BIA52BM0}}UOpZI-94?to%_J+?wq8;Hg;Ir9C_e0_q>=VBSX*v#ii|=%#dm(Wf z{5=C{8X{mgcA=k%bS`$AUt)KjkHnt%^RN$n1n3C(ayX)880auaU4p&x7lU35$^T$a z`~{#FKx!x==XB81A$1}4$DaXu2Ba>+KKN5XPlePdDh>`l>e-g%_7J4BahW~`1{{%P`iT+Qa z1thB=b;N%9x!6JfDAFU?VLuOh>K{jX414b9W0(B`q$luJ%a^eqe=*+9{3MQ7@n+7q z!6VL-KrgAT`ZLhau){tV?7WgyRSG430l6e=|P+h4c*aTdaz3(0jsPP-*oZj#cUd94qnN5`07a_we}zq$Sw) zy&7+@{8BZ=yOi7DozV?&RM53?RMqI{^p}eG4e+@Ql7d}Z)K*x#74NA0O8u#pfiA;4 zy%(V@S}#IM@CMTa=NFNx>338!y#epe+@PMqe;PcCH=-{@`Up8M$2#s3q$gEd-3)SV zk>1B!PG33#Kwc-Q!cczGa&w<4{_o2qxFUwuM|5 zyqEb9&_fUz-E>d9t-2dhAKe4x?*-aRABt}d(J9arVy6$j%?HiLr$gbdJkUJ6Up*I} zJA%?Xv{U%DGiYagit%j+&OKc=aj$?>TknB)Th|Azuj}CPSe4-6%1HI_O0{tc=90XWK!0$4|^|p5N)riE&=l2J!i4HqU=-?JaSmrj1MBDdU>wS2=f5 ze16Nl;+xNGkw5XrtF1iSKzbg!p+)|u8?Ur762CjUMgELm##o&XbZ?oTN{aK6m~AFK z5?{Ax%lx`87UwJRpg&vWFUg&3b^0)umvl;e)t@c%j~Y;5?U(qa?)&6F_W0#ir^M5n z?34f9tg*ISKFt0^y(H`t!d#ZpnC&6H#HeE7-v-xMxC=_)M=}bn7rYx8Rf@{r;EZlFO!V%)71C*rfZGC|&UB zmfR2KOh`|_UKZ%dgPw%+yxI7*ykmYFl+>B15@{#&EQg*H>5+WsEQHRObV~lO!Fh$7 zCj6bM-}AH*@;;kU#ypfUp)yK6C~q#xo2K$g{^`qJ%lmf9;3WF-pAz!Ea>;A6@p(2f zqFz#>USd(kSky~O)JrVtB^LFP684iGu9sNUODxMG>ZJr<)Jsari+U*`@3UFdO9?%q zUP|Z@^-@CKS1xJW0r~ag8*4qCdU-+>d*;u4W92*Wdnr z<~-?-x|W|)yL$WtzNhO>hWz<&RJEV^O#Ewc-M3B;pIJYjNnefLt5SUawt4;H>q?zT zAKAtEasHX}1I78}xHH*N1$s-LOP8BJ^O^LQ_SgSpe^ka=L`|$a)WrRE1@xmKXoxj~ z$ldwxZSVYR;I4a5tR~P(M1b!id;j0a-iy<^KxF^(vjJ!WtY6fG&K%Gj+@aS2tpr-h z{)hJ-&|V*@Hgar;yY9+JRdMIv40qSnkgDTo3?0>xs^Bg>4|hrjAhpJwduRK9==A^S z`M5tTUJdDlJF{-MBkO|I3R?TdSV=els|E*y9*q1C!iqo)8smPhJ+AKw+7q96T*r9-e>HF8xkdcf zu*^Jcd*e|tkL69OT#}iGy;osm3C!~el4pBJ56>Y;p63~)M`9k2=~{_-{3bmT^LR`; zeVE5`%1dINXOK>bd5%IhNX+vL(&NKCM=7nwN|*(3R)E;92i%9s5> zIwcl$k(oP^4Wcf*Sky&k9!d6OGxldABgb_)-pjGvH!jNggq&x{IfQS__KnTHvD{~a z9GiXPx|H$FIec>zpZ!v&oUdf#^K4{9y`)9Gl$iSppOly%i+V{!y_A@9i*l8imy3Ey zWEn-h{FmplqFzc|EA@+dDIp{3rGz~``=w4%FG=b72!G~t>3Ir$UUK8K6wQZ*%)G0PAJ$zCy;o)48@nZN{ExvE5mGR(jn|_qh?vt}P+Wqg?YcJdM3tc(wz z@Y_2XJwCi~SKv7F~=UuQ%n-3 zc-w&3D~-v915v-kmzd*<_!4t05np1CAL2_)zCFO{mze5>?U9&cgY-yD@xbMkm}HhZ z|47XBPI@Hfx+cEFT)%8T)jMGtFQ`vDV%zOm;V(XXIv&{RINP82@VHZ+ir+hBP~3Fc z)HoYVe%{#fB&+k^@I1^Dteue02CsWzh_!)q?hf;uf%IpCXRdfCzG6*pYqJm6_-Sq& z&pX1}kPW6eKt;??$cAk2?CJN$<0l?qZSdh_$vyGR_8qMa*C{5MQ7(E6@H)&?J@7)bxo$J&q$wrv6P&wku4c7v%c z^gkr9Hux~Lg%SN<&Lcl$gQ<_Y4f9*FAsbBn(w(bs&!hQ_4^zMN?CCYF4cTDoL)4=C zmRcLK#3w&7z}i69`tYot7v_>5NM?7K=A7(@Z1C^VhZ#Tk@SyRlGJePg+qtoefoyP( zdS7PZ!G~Y|c0+u=i-&A5-&0Uq$Od;g_{U7U@Zr7o`8i(h+C?^)?lrhi$Oi8_?6-KH z>mPjhi+}#i^bgr!z9%9-WP^u&wKdaE`tZ4n|IYN2*)nT`EF3>ROg`Z_ z^x^Bj4U6`5>p4D5{{O?R=lJkZk5nq^<<@h2c)#IQidMVz93P(jX|*D{AEfmqAFi{o zdeJ*>J;#UX+Wp;ng%7uaJ>R(X93Q5#bad+{K1_a_=hkz4nEZB{ThH;~o-hBLupS?N zE!sYT|v+=Wp*SmbUVb9eG``L%-+GVc3e3)$b%+q*jy;yAm?X^cRvMEHo!okh z#2i;7FEPgw@g?T?A-=@qTZ(7mOU!XXe2J+}TRWR2=6E1^i8%&{FEQ6U>!)}iOg6Cp zCFc4i8Hss3r8z)0c-5s9iUzs$93QUKzGBh)ZapU(On&~xt>^q3e#ouokW4n1*KkOF zcbMl4BOmhJG-GOY#247aAcF_d4p5w#k1$BzX^9;_lacn95$)t>^eK^-C+=dQLW&`Vh6Scs(auJi@K# zkh~A8+~V~dt8;f4^H25z%|CaCr$7Be!hZ1K&fULH*bmuYnj3RGWP?wbyEfr?@Znch zewK`M@sJJXdkSh7+2B*I_#okS;lrb+u1tztyT}IfJqy{84ZiT`w-W9jeE5mwuO{3- zWP|CRi2KQGaEp&#&&)r4c)v<7TN&ynv%!4N#eSfBv46u<_HAyw@L?)Dj~CfsiUA%s zv%wStJZ}0h#Q=|+*F#X=)_Gu(OOux%)KunSzAEw`o-o|{R24sAg zu6+-40Fv=xvY|fKb4bRA`S%f$@nQbWf@Hjyep7hzzNJ>igL%!LWPF&{&ho8ze6= z#{=;t<``gkTAL-z^_h6Wdhiow4Q&79e2J@aA@`Dfao*b%+Y%t%mkPX>j+KWQ__s9kx<~=!NLpGTA zOg+)w&2?aASOG8@eITx3Hwn9Aj-_wx|*G5IO=iv5fBtFc0 z9!QT5)BXqA2SqYIOxLz?YdAj4dn9Pg@?k1VXSaUh!@OUD^!PCOkoM=0j1TiZ3X<_* z-cvy`KFs?oNXCbGuLa5YFzvfwKl?E6!JxYEVcwfTGCs_EGf2jVd7lQ!_%O#Xts6+p zF-&}kIaY};F~=wICFYnUzQi1d#Fv<3kL4*Q33I%a#$-bmx85Q#m1VwL&ykqQGQ_Rt zNKEm=@)DD8Y5x!9D>26j@g?TiAil&L55$+4V}Rvp?-uK%wqF`^T@znou3zFy%;PD| zg|fk7Pfp^)^n3RQ*q=jlfNU`N+3nAD-6Voy%u z!?gF{V7H!=4W>B&{q8_EWP`jv z2J@VXY{&-Fo*eF1eVF#-kPO+74dyu;-LGYXc~1`Y|31una_D|78_aV;vLPExG0+bC zbI1lCrWja(eF$VjHkjIi+mB(}#qKcg$srqjnD^w64cTDomsYyTk<~=!NgAenb9I_!BOmkz7 zfow4E$srqjnD!owaqBtRV7{lIy2u989t>_5K1_QsXx|$7Asfv1EM!AAnD^w6AAFei zXg8|Cx`ry4d#1NvLPER_TVMmQbzJ&Pv#w)QbzKjBPN@6eox5yIm3rtG1*jF#_?0c_sKZ@QpVz) zx(eu>qhzOz2PrRQT)C`_l$SEFF%zp&UdT{6GJcl6^2vDWf_#(lzx1cnBl#Xb$CB^y zVVs4ZS>*-9?Wpua%MVJM$|bKU05eWu!d# zWyY}Nm%6shSVg|pcg{}h>(UtcTRVvFu=TCvQ~j{drHs>W>rTp$EPW=OQpVYD`BFym zUAe4`#Lho_Z6=nSZ#l0ryxlySCMzLatHTfUU} za%=7M(rH8E!9Ts2l4b0+Y*qU6<;TU}YAXI-uC_@nv38e;GJkI$zlqqzy7(hYQID zAK#akvn?%rZrh!cm-bk`w851(O>IHi;N#1@q<&XNiHJShexM)<$b5Z#nU~b>+IS-RC~Je1mwlSF!L|3a=%cJXk}vg3 z+oXQihp-KqF;QMC``A)8P@iUPkoi*kV7}BZWu$&7gS;{}z|IuanX9AB`0KNQ`UZ*# z*H@QmgOri_C5F#4d7-{ysxud>nXz8lAax=hQWO)epUl`J`7$r5Ut+{wCNKCP#x>k# zGjj}SgVYZnq^Q2h4?e!kOX`;xZ7`D;+HZ`0lKY3ud`8+J^F{rqsJ>l4nXyOmWnNOh z!?sVBd^s<2c$YrP>9_r}%*)63<>hSSu_-e*lk(D@GW?Ko{UrH;#|!eqE_|7n)bHwu z$7qU&nDUYGu8wvYvs}H~@!Hjk&DY15c}e{)R(ZZe?ZVn1$-eBXXpK0rr$2*=M3M)YO?ctLYegw%a{G$PPy{+9K^Hrk8eI8 z$4!r)Z68K4O@8(Gnb%=#-|qU@UHW;)w{5{Ep7?w%$Lg7bKnQepUc;CcKU(u z*Qov4c89OrUqOlW6Wkxsx)$HB(K=hY^g0jj3uwL0-iJCrTm3Hf_#QdamZiLG`)mP? zQ@9_Yb;a`aoZ>a-^7S0CcI58GY#h4#HR~U_ULuFRT}n}5bT`vXFm-;pif z%}aNYk$gF?-l^+<-+h@ZyYsW{hn=4-_Q}Y8Tgpg2{9I-qF8S#DY5jJW`H$qg`?y_X z96v?*`(&IRDPys;+1X=bM#@VWS1v0f<)w^_RVgoI?7bS@gAq^q$|vKg3w!?uTdf}H zPpL=pr9Y*NOWWZ`F8=cIc4H$T(dlj7mu{LJP0H$Srvt;}xxY-OAeEtX}MeCSD2 z8Fqe8sDSLr@L{_xLunbwhYnkY(lU}y^-r-5oZX$~c=XU&^?0S-zB!e6oe|CY=(K?UV=cC5HY?%t#E`Oq@teex>|KPZ>B(d?#QNQhvpM1qt1!(tHcQEhcoI>VDHH!I%1_ zjMUE>sSS$XgG%e+{W)Z3W}YDROZ%lBcb`IQR`%YSZI;(cdt`aVdQMu_skB*^%iTxQ z?|mhFBKa~esb6B5m%D$b-#{tuGO;9Wkor*%v~EE2mntpFhe0-Ug)L#as^^Fqw%C;bFkoh`HYblA3@5{^CMr$nWbKCBmytK#ir46pU zbid5`S{r>WJ=`JYp8El7gOr#3 zjI_bEi(T^d@nv38ziX$oH-`HtYlD=ReVVkv^-;85!R^u7Bl%Llv`y-FeF)n?{g1Ul zUMu_9Qa11&81B=oJ+6(>O>s!{v4{8j6ISs^OE`{mU$s=d4G-_pD8AG=1-`A`om1V@B#0K z@$zL}QoqD#gPHjZ+Ar_Vq5di}pV?_g0ky|WzUY@|UkdjRW%6ZSQoq9~>Ki5B<(r~B zeKxp!ZT~E7@bP_lIoo(_B0t!CrM$GK3_qk?KS_SD_Bj9S!k2kT{jQE^e-6i9O!-K8 zS4Y-1X@iRgJ6_9teSDdh)bC=I_UCXLv^Ge27ps;pZE*2#%PaHs@nv38zw00DJV(xv zq`YghyV&6R0iMh8{v7TfEMMxEwn_b>Tz39Wbx%AwH!CG$+X?Zgy;42x{GL#OZClKD zW6dryk`LLk=18=kA;X7VWzAv!zxlt3oj$T_Vwv&CGX`;8By^pN0gI&`$(Q}Al#zU@ z6RNYF-xDhMpSo|v#tX#_l~VeOJ{SLG{>RYDcSWy`VCq)vv$*U4wr5hv~Ff?`ZpP8r;QCMFYR&V z;`kx^EtWDe_M|-cnZ82Cu;iDzHZz};u_`g_%=7~i!w)g(a^)p#$6v{J@{(_{({E!y z$~c=XU&=W9Enmt=KG{NflRXkU|M0bC#tiZ*GfrTeZO^WZrDK50Mf^0$IC+a@oH+eS zX&GmmnH9$ZkIJBcfV$1)!naIKTGV!OHnVVIR$R6Sbi>U_vL;l_k->8%G-qv zg!1-K-X6-^1Fe0R$3b}__@_wMaA1Ln!%KL%xexSS`DDMZ#`+@R)pu8U_?+42Jf%1N!ydNm<2g>__ z@_wMaA1Ln!%KL%xexSS`sGvgI6?9i!aCB4$;W$9;k7Ga8635=E366%UK8`x77LMww z3XVNg1suv=b&!>BX(j7hnJV_%E|%W}oc)n1sMFOM>Ne2Z)UE1Rb(Go&x>5b4hN^Q_ zN)JUkTn_;6Cp7@+Z#4|x9Il5U4bUfn_qRF;siN+sa@Abb3u&nu4ygfpIMV5QAil2% zT2a^1u^OdD5!H{XOCfc-z7**KeX1I$YJt|$jdfpjh6eT@GovIpx zHrB255%_+kIs$36I!g^vt@K$)9raP*twtGEqXdV*mZhLeq3K*mb=2n~rF5W-dO!t8 zPpU{&RL_7uqvoheDo4El`ht2+RaRBiTcB^L6{?!5p*{iqM7^VGsy)>P&<*NSRa@0l zTR^v{FI8Q&m)ZupP5r1EsK%;-Zh-WsYNVQ}nz|8^(M?rzwHN4Ky1H(m_EGx~)s6JN zs+Bqj^dP;TZmrs=o}fK-2i;a3sQQBT)p@#|>Y$DXJzgKC+pEs%G|O6HP=$+~gb+kHO-2i%nx>lX9E>KfJr>eWuaq2{M6X;FqdX=y8)ELk) z>I!wCx>!vEou=+qC#X}@&7e1{8&yj6R%1cOs>{_y>Jl{rbcVV|ova3_NuZO|MAb(f zs;&aPN{v$^)JSz7=zVIs8mvxJlR+n|Thw9da8&?WpsrT`QKQxUp!chL)r0DB^%Uq+ z>LE2>J*^gkE>yGBGBr;<4EnGtREc^-%?6#VmZ+E2a+L;6t5?3u=>)d%WWch+4&yXu2=Z{1zD2W_u==p%H#?giRQ zAE}SleRM8pu0B+ssQc;up#Am9dZ0c^9|C%aK2D#ZPtYfUo}|yx=jc=PF`&ojQ}r-C zM4t(IroLE@&}ZvGpo8@JdbGY!Uj%xQzCvHAhwF1e&(;6YkEzS_7|=0#ky@-S*O%zU z>Jt3~dX`u{3i>E|m+|^)Jr>8)dJ&EV`Y{|2>v=fl=vg@K)6;R>t?$HfyPk~WW_=@$ z>-2aWW9?OsS@~I3@=hyrqy2V~pvtv~h4iaE+dzJA!r$ng-YE>vak=6?6`6&^PJBK@Sh|gNb^QJ^}QE z;E3QBeVaZ5^o-!d;8uNy9tJurI6au6r|Qw5qk{{AyYw_Y9&~&#D!51At8W3lB`64H z=$U#d=+xlm;C}spet>9jPcU20)sKNb7R(7A)DP)rK%WVo2)q^+mJ30gn0~LIwH|koT zwSt<#+xk5nfkr_PEZ5KJU-fdN-}Tq}TU{sk8mV^huKrM01g#jDV5v^@A9^X$Z+esd zLDvg5A?+D_pg+=;Kr00mf+GEb{!14j{i(m#TXciqd!)L-$NE#91DX@;5xl5h(tqn0 zk+$j0`WM|W*o;&^*gI$)90Yn$&?0CVv=7>XwhbBuU4!PqKA`&q-GiRNzCjbvCP9aw zSI{l!0oo(z6C4`k2Ax1V1$n`dLGR!Y&_jZwgJXlkf*3Ro`UWQlM+L`#9uo`<1_#Fl z{XqK##|LKzrv!sQ2LuB!Ck?F!Hi%g=*-}u;Njr@;BL^ngIU4CU|ujEbbhcX zcq(`#m<>8Rcr;iRJQ*woT^tk!N$_;=IOyZSlHiqKd5{K82d@Qh1VzEKpw9*`1n&he z2d{#@8hj9}3SJMM2Yo(xJNP_!H+UcP{a|gdKKL+r6ZFmCli-`+vtSMAn&8{u`(R!0 z5$H$3SHUmA*TE*xO~LQM*5JF~3(zlu%|WZ+=ipb+UxRkRfx&OV51>B;`v=eJWqLo* z{eot}AHlZZ9~?R~I4XuYII4yy!qGHrj$@y&6^=GxI~?u9KXC0o0bP}2 z=2oVi{nl81BXDXU{SllL-Vojxo`iHtczC!eTm!l$JTyEbJlJ$GXM|nMX<`5H zSA5eS>G1IEu)jIjoQ-eK2#*eT;G3h7`iDKjMd7`m_lBLq^TLx%e{&ur&km0ZbIcCV z9gyo9&IvCvU6C#dF9^>xCz%T%d0u!zSjXg;6Cin9m><3x7J?RrdErHnJQK2KLhj_S znW8cHuu`~3_+I!q z=;vXTuyVLM{08)!uzFZ6+!+1>`b$_VtQr0k{sa0?SSQ>w{5#yk?1@xAtQS@^wM@OR zmf0(88188rn}%Uy(>QDrHZ`qGldzR(7VaJHZ#tU2!;Yp!*fQ*F@=ePy-|QFe8^-1c zvu}8W**|O@9&S!Ft-}+|0b$#4fH}jo4bLzKg$IVGn_=d_aG2>Bb_g#p4}m@u{u%TN zFAuK_dm$YXUK~yh{{#A;uzNT#yeXUrIx!p^_6e^D$AOLu`-Z26H;0ozCxxenhlOLq zt3j_0j|hi^w}g{HCx>T-{lcrlYe26Fj|#hmmxd!iM}(cjv%*`$+d*#+&kc_Z3&IJY z6T)M|F5xBNNYIgC&+we^w(t(nJHnyiG2!^|TF`65xDs?__(}L(_(QlJ zbbYus+#G%#ehT_&_+|Ke_(%9X==b5b;a}m;;RetR;g6v<#%u-M8vYhWW_!2=bW8YW zSjAK~fvJL|Obt`XYzu23g{H2lZ>oY;HI+>RQ`1xctzfE|y-f?V7wBH5p4rDVGPOZ# zn}(*XX=hr3wlvMnL1sVG1hk3S*K{@A&4Hi?ngdKv)4{X`ZEf0{-sTX~1GI^4nEAxo`w!rf50cR4@Nb|lyMMUo#6Mbl2GUr4CqnFIU=mqn+DU23HpPNO| zr{+~NB048}72m#O)|pqM!e|}7{oK4^u8c-RZ{XWk%@Wftsvj*e^`nL6OY?s8YV;)} z*O|A?_0g5l+mL+2EH+J}P0?brDcWQ@`lMstHXDLOm4!JHl4Y$lpBqv6p+Gd!AX zZZQ`{S46j%E27)YZDve#U38ndF1pj)VJ1YAqdUyxXsWr}+!jrX?l#k+X=b{a5zURJ zo4L`w<~}noS`gi57DV@(S?0-Tc{Ix`kLH-!W?A$~G~2uq%`*?0m!kKg2hDrY!)Cr& z8GRnjH=jq3nMcj)=$q(K^G&qCJYhCQEutq(i)fTdo8Qb<(5>cq^R1~BZ3f+Jo;IsY z6jg{;nF`UzrpWwm{sjHgtT3OLiczKL6H_T#ZC*5gm~Eij%q!+IvqzK@eP(i^HRfgW zm)Q=w-Mns|GC!K{LBBUonzg2KR5e;_szzU!*UaB$2j~v-mRV$eFk3*km}ks-Qzfb% ztvA)94dzYrk5SQ^Nbi_u%}?fM(4Wmx^OdO<)r`I}HKT9LN`vWXw9*98duEyW#rz8T zt0^>Jn;KE==xb9u`pz_n8b+<621t8HjiNoHmY^-8*3p4c`=~K!<7ls_W3+!%7qo8F zBRbPBa~KdNey)96cR93i@dDaP&;{MD!r&gVEzr5wi9U|Li++sOgRYO(Mw_FrqfbFUjlPVUMf*nIgMJ@<8|@c0k2ZjAh<-qiQa@@6 J+7x}q{{aqCcq0G+ literal 0 HcmV?d00001 diff --git a/pufferlib/resources/gpudrive/RedCar.glb b/pufferlib/resources/gpudrive/RedCar.glb new file mode 100644 index 0000000000000000000000000000000000000000..b0e515f3d6547d5ace3cb25c9ebd1368721959dc GIT binary patch literal 239876 zcmeEu1$0z7)PIsUxa(qtQe=@*+UZp3ycBn5afbqBv2F2EDDExp?(SNcmm-V1!@}Y& zi!KiTn>X{|4rP#YzVDpx{LeBcH^1a1d3njr%}stl+YWWgF+zxGsY1@>C8SdA>eZq= zT0};Mb@d32@@N~@DJ-Hz*Uk|h!5)>{MRe}eInuo?RBZSVa+1DMuc^0+qIoX6HkxGR`6DjU~dPCglbV9omzB+JdfJ?Gf$6Bom+>= z2i{Ghp@xx)^5_^A*-o$OC||lo1oSJSbJrGK^^P_2_AKV(>FsIp@(u9u@eK?#2l@y2 z1_b$r74r_%+xpZw{k5$-cWc=ptaf;7+54^$-NN8y_MX%#TLxP0(lVlISl1REI)t~X z)wx^ScAdf^p<7WNEn7r}mG0c3b40lot@Pn*>c=7U-G}d{9S6n?`LIiwL9nPkCPcfj&OoK0yHifo4A+e@kF=v}cru z>mHY{QnGgKpX{@@+2ZeOmP6#@?H6eAhYouESl*9+hTnJkUAk8Fsz2FnOOU^>#XHE? z+v4v7!|f-#{bPCm1Kl>1+L)+DKA!M{S^c~kq>6lRB|s6QrP{{o2=?)kqh;t|gR)hs zRIle=rDFLCb$)JyK+mM6eP`rrR4G}f?9VmqWe$Wt{{FuHmLNaBAa65_{EuY)4~)BG zVV0^=vUKI2?6kM9IlvO+Z3*)A_wzG*TLSg9`6GG%1N}B;`IEkQd%@tzKLP$eet|v~ zGgz!2$&!}PZk*~>s92}$|AJn8rK`T)`X3*kAhUOnIWWjy4xP6T7!0!|z+yH-@4>|C zcEj5TG6RAF%mD%Zfl%1|xx-*h{};OJ7>zm=YnAGt}E zbAL)NU>xMerSE8;wz|^YT84Rf`&*zA%xy;hlK-X6hbuo>ZwmJKikLR z)F^zz=SF?M*{DBi6h6Q#0)1#&E)tc52k@G(%>mDf0VfqiBxI zUrdtOX_m}R(*%#c(Ni=}<}W77*J+}BohB;aTaEgniSqqoqI{hu%GYV40;6@mtVJtO z-pI}o_9f7=Tbnjv5p~1EdgyB3s&mIKojZkf>RP8)7jP5(e9SO`tvYw>1YWFnff65f)(`iaP**d>5^4S)=Ja_Xky|f z&^%b6g^n%cH28Q0di(hJ_`y5{S^|Q+!SD6)3W9{$5(Ms=zn@P4Xo!!OH*6q2{@&h! zu!Vp>=lgjce7t6c$#4e~`DY9}JkU z)cqiTvo9P-z=MOcg%7+BlsP~y2j2i#34U-K2>Lt?W-oA>m;rFWwD`lKGQ(tAd@P>DzS_b!hw`hY;fs&>W&__d4004cH}I9I`0A?s zJQcojoE;6gSQS2cf4{Xb4ci1;gulN(oE_m{ZVrNzx}1l=fIu)muyq7lEMWg-mM<&` za0}ts4F_a6IeeZ7XaVNH2bM|z%!dU$NGKlQ3t8X|`@`yh36b;gwH7`PL*hl@^Ht-3 zCgQU`etnbqj%fhxFh&vTe`OlJx+>8$Kv%v!3vv(n)_Qnhx!Nl`T4>vO^`3# zKLmmyf{}-1C+}_iVON)%1}qIYYrs$kS$zHdEuSYr->H3Ji|~Uj#osT$tj_{$=VrLg zfUR2k^{{)xYrfLNw`M_lZr}ZeZ#M89qo5DM=LQnbgY*s)_u%t9_<%$6-I@5#n)LrP z5unXrdi(+cECK$0-aZz%w}Pnw?FJq9gHa8FokHq2oasQvEimFRGdJcm80xWR(;%f8P_O*aB3fIZ76-X-uHzBYD z{4CyPKe*fV^TRu|#q19kCVp_T@b@zZ>WU8r%nbLy^14i3N&8y@{B(~gaVy`N3wfgY z{)c?Ch3}XI`%rvtA@Q1!=eERs_>Gf4L1iS}-ZbzOV)B7X|j#ex$ae!5?iQ&|iAr`1NLgxXl0YE(BPl z`;Qv{cTM*q(KqP#60W`huFaeq0849@+cs+8n}d+J0blUof1&}<)h~MBXLp2uum=G_ zu;TC#fJgLY2fj4~zTlU^kUO;irwyrBKiGq>v=Hb6p2Cl}0NWv)H=X(b8zQWaA8!JR zOY4R|19rln8VPwa{-OmiDFHvxiLbTbXLlZca4g}HBvBs%!2bRCK={jncNzvi=}&`<7(S_t@w zG5AUg{_+O@#|HrpOJ7c;e*hegexe09OMNj80kExp(T8uTLZIAUoCXqne7My3Vbk!H z27G^NS%X*nWe~imfbottH*-rOW{s}T);+8o*LRYkK z-MotPrcoT@lU>(|Ibt4{?PDz4c^o}K?+?cFvPO%VZtMB}Vxu_5%a6?#8%i$abB0HA zjOp~TVqNe$o}=L)j`7Q}(?#ApOZkIkLpa9$cCHk!j?dwp=M3Yn@ZtOm#iU2``O93x z_>bUMmL-BMp3XP<#Bz*xsl&viQ|tNEIm3Coz+s~2K&hspR z@6WP@|CMwU$9QzEr@UUuWx5Vvyk_AEp0W5AUMA@v{yypoKfi1>udt%GOKiFO7tiu> zHIIuL!aYkL;K!_6c~IDJo~it4etzO=o?eXP>1N*MHPX!Ek6^ zKfJ+Ny=DA)GG}b`M~?HLRCoEzuFLgt#<(RZo?xs~xz^-(%JLn~niJ$0eu>jA_Dhgs zDDyEk>dN^t+BC2+M@OJ;p~^Q@ng1o0W%m9#j4xYaNfSYWp>&o}ae2isX`K~kVN9pZ+4JXf2IHAGs{q?mx zeo?`MQ=obe@$3kE0#B0FpOiotYndd)tN@|18re^T;M+k6-zMnjo}>X z_FCkb@Mzaiy`Kk4dnPD5hw}AMcVSJhguc*!Igk676ii@~$LKisPS1otZw}RQ*z>{( z7aopIgtHXN7Wyh^uAcw)be_=Ona1fj`Ny20AIi>7gr^6{LJggibq!sTZ5lcz+cEUV7!&y}W2_{;{cE<+_7}$L{b_zQ zcWB2gQ}ubN2)IP5ar%2L$J|1zG?~h!{wIZT9#?d>y+7}BhGwWRTQBc= zVI)*6Zp1N8IG;l_ZT4E9YmAM)Ngb$lF_pM~@Pl4vU-c)Nr6moO`HgPIaM^!}%g>m` zb6k0)m%(`4zr*>e2KMqsk>dJ2~vvLxTx&E44|83@kGq{cA+uDBM2dXsT z7~hK<$bU;lLqi)j;uzn!JDisk^0-LC;I3_TU2?xyJfVwMTfKbu zi^URhR%@%biE;UMIYPI*X``1n{3zK6jQ5nx9lG*u8@(>Zp_%fAhJoKD`(yZBvM$Dl zE4hW10DnxbHH<@^<_jGP`Lb>WkG!F60n6{jICJ6Usm(ww9CkKm9AgVoVfH}YCJM)7xX1J!qhR`8|4vHZc%g6i+pChPNy@jUZ# zb?^dvzAG%pvu7`RnSJ&4s)jAXIQjFYpYm_DoBjz$q}{8oPD(>>)@;Nve%`F9IuP;~ zS8v2I_Rlk2oeKGTYBb^)cR4ssZL{tj&%HRDYyGFG$@{+G>$*g^#1$R=kv3 zeUtbfj|pPaiKX1DSrlJm879K}E#c(ZP`)j7w0JRVEuZ&%F#l!MBC+K0EZtrW&3%e* zZNG-!9o~}{eU-pj>?Z#4=wMzk<`(yzxKJH^U!!(!fIK`-tKugbneJn6QOSBdGsF+P2`zj!%y7GHZhj$?cx`)tv9 z&s?6+dNg;1k4+ygMn9j;Gt7zOuJDkRBSfla^Ld#xaoiP-KeeCtJF<$e^N!`N@Yxj! zJoD1|eA=-%?h3aM*Z8$=(|8N8br_#FU*>aXZ@voIEmw~@YaX> zxG24l|2ZX&yTZk@-sP<(fIr{5F30%XKezdEj}?4uwQ!E{{{1)j`nfB4%}L$4D{Sn` zG3Fv1ETi~ceZxm}YQ|mR3AR)s_p%TC#EEJgdx^s+so;;>aUipmgDiO&s-srcOulx5NpA*`EyTXQl@@8id z?h1=)-3KojJVC{U`iQx*8ngwpC{Em+U29k&1JSTNdoV&kTOSdn}CL z7^l84kvCg(n@{wJ;;!)V7Gt=1#Vek}qXWnIw;4nD^NsI#^8?`=J>qNcbmtgvrf<|*)93PKC*nB9bw{mMN7h-vlfeG+vv^(U{%U`Zxjg^L zIF9ka2Pf6|NAq~H8F3urG0V58qyAdNyC#j}u5g)Oi>n>qFXvTD#PT1(mZ3vK;+D_h zGoTNy@Ff5KYBxteG0svto9fwR5pR$zj=RFmHf*={t~`$)f^Wh2Ncy4H3{fxm!p0HY z6<(dQrdlZPb^cn7=B{wg%}vxl*1Y4d7ld<+4{S-RHca}A&pZ>!UEzcgrPM>!9`L9W zJ-91eqH}^eApdnf`?qM0@q*bY_{W$RTpJX@U19%id(^nK@A!%p;T+@J@rTuPwI1+6 zcY1J)m!^E6#-)75Q#_0080X!xNVHEin-^HrfES%LL$uvApO+XF&1*MZD6A`I^VoqM z_@U)<#r&3YdE>>soN%uL{YBu~nfz|Q;r#8=5n|rEd3?;)VNSU1l;I*@^4a`crQsao zvTG7}{_XR4r>NmP_S}A6e%DHFy)u;hKiS6u&MRo(*{C z36pu|4Nv*M$vJ=TKZ9SJ^N?RjQiDIaJ&}8yzsYO-)rpt#?!Z5;dB!)kXv6a)8_jq8 z`G9-PZ_g{V?9XqVdBRfD1SMAi)uc$kk^DV7%zQuQeE(RF1J8CgO;Y?g;qS{??T(~JJAX1>FQT` z#f*ROc((_t&F=}XySgoRJ9JpB8hwwyTJSrM3#zG3sdJT||F;Vt)^w<~{;Fp@OXfB_ z@5)kY;Q{x!$B5sZaJRqHs!M{N@SL%2ImUi&B;nkp;kqv!T8bw4TE}vK@JVh(vV^&_ zC-D{YhB@KLc1psvom2F4p9Sg~X90h}+soZh`?QPYU*d)dht+F)hUxhRmhYA4EP1Xm zu=HbPo5mSRV&kl2yjPYn?zitW&HL&8R$}9hQDWoXPvWF-w!ORKu=*vg1ZUe6Wp3#C z7#sJ8vV0D>k9rRGY!VNDR5)SqrO}D-sI8s}<^LM0=l@&7E1?VAWy$i!U68~d;eN*G z1IDG{ZpY{Y#>U-|tZUrcNW9=f&d~YgX6x-^d=~Dtc5j`k=VNT#|H!(={f5NvtxvRu zjcBOE#=V5Zvz`p+eQVkCC0`)tNakZ~+=0t{jE(yQnSXYvcfvNfACTBM*MEtPGrP>k z*z7p}%KTYyUNib|)^T2z`NbV)B8iQ2ki;7u=Oc;NH(V+<99p3F0popZ7m0pvX6gAc z>F)C0G0PL-vV(5%^m7;L`G)@~+r-$gV=~|HslLR99h3PTj-?W3v=4flts0Ey8~P^3 z##)f&4V{-5@m2!yRzi-yV~yz8v3B$_j`hbjJPL(G2)>F;-Oq( z#6#(G=dg!~urZHvt}!;|Mdo8{%$>~t9!9*9z!>pL0%OC*%K60@aZCbnOfE6vm;~aO z1mc(k;+O>Dm;~aOoUmcHW#0_@Z(w8G5Z5EV#Ig+Bw+qDkIAO&52*mpc#QW&^vK`=e zoG{{g1jdN>(c5v{+dE;z`v}DQ2*mpc#QO-u`#535^$3g+*CQ}ST#vvQaXrEnM!b(e zypKSHZ_}6~#FYqRuatc- z_FRb(S0WHsA`n+15LY4)S0eU&=1K(ON(ACc1ma2r;!1?EUjttt5Le=a5kDd@MqG(N zT!}zji4#Wrh``wJ4P+k>XW@hqPa!ZyJcYm*aTWq`7ETy(7W!H?_BF%Dkn@6g3MY(s z3V|`=ECk{#1mY|N;whXk;w*&W&&WPtjCcxxG2$$Q;pfP@7$crSV2n5mfp`ihjQ9wF z_y{5I>Wy(je1t%JgkHw*9}ov2Fh+cYKzxKie1t%Jgx;pHA0ZAxV2n5jfidDB^mdH> z3-J*G@ez6%!?q(1Li`9u9EA80jQ9wF_y~bG2q%m<2;mANK0+W4!U-cjLKwcTT#tx@ zaKea#5Evs4LST&e2!Z$rml$yk0&xujaSa0T3j*;A0`Ub-81V%HW5gE-j1gZTFh+cV zaD@?HAY5U@7YK|IUm!3>e1UL<5nmu&VZ;{*R~YdH!WBk*fpCQpUm!3>e1UL<5nmu& zVZ;{*R~YdH0%OD%2#gV5AY5VNj0X2h0%OD%2v->K1;Q0Ze1X6i@dW~7#1{yR5nmuM zMtp&Ag%MvMFh+cVz!>ob0%OD%2v^uR+rnL(aD@?HATUOJfxsB?1p;Hl7YK|IUm#pz z#1{yR5nmuMMtp(581V%HW5gE-j1gZTeikFXKwymc0)a8&3xq3-_yX}G81V(d6-Io4 zz!>ob!WBk*fxsB?1;Q0Ze1UL<5nmuMMtp&Ag%MvMTw%l)2#gV5AY5U@7YK|IUm!3> ze1X6iaR&l%2Lf>i0&xdU7;y&zaR*Kq@dW~7#2pC49SFo7IAO#W2#gVTAP{#T5O?5& z5qBUEcOZ;AH)-1ucOVdVAdEXVX|E7>AP{#T5O*LDcOVdVAP{#T5O*LDcOVdVAP{#T z5O*LDcOVdVAP{%pgb{Zj5MSVg5qBUEcOVdVAP{#T5O*LDcOVdVAP{%pgb`mLFh<;g zK-_^q+<_A|?la}t2XO}i@db&ngD=qY4Q$+z8toWo9K;uVx_gAXQzvZPS;>6kj!0s} z5jbJQ3kZyjyCzu|aRdT!1Wp)n1OjmcpYA|pJBSx>!iX0T7#nvcvaWGoAu-|woG{`A z1jfd_hpcPdIY^8+0%6<($b7^Re7ZxH`Nn;+#PaUZac}R0jq|%KgE)du=V+O4oT()? z&d6V4#1RO@5eUQ)2*eS5IycF75Jw;oFW`g`M<5VK;DixJ@X61V?HfL)#D=|+*zldc z#D=|+`G_MBh$C>qhW^X)#+s5CxG_nOIpQy|o|b=?%kfA5`_^B@18YT!Aotnw@70sh z)1$@FWvO&LGAf#{4^5}z^G*8kHgDprjs{5Q`AM|?N2B%jF@`(=X}u1{5d#KkZsmLE z<)54U1Oy#-T;D`&CBz92xbnnyBKLk<^?&<_W;L^m13Sy`H~JJ2{Iw z;dvGBZBG^a*gD0%FUNR7et$mJQe0HblTQt7{Qym~>!c*&5mTJbGa_%Cr8xGOxd zz*=qM#OmC0$rJTQaH}P+ZLzkS>c|BxImYpG7Ha2brQ}N&mE&iN*5;*3q!k^eowm;C zl1HmweuG-SOb3o}<~P%{(@#_L;)Toef{%)73!C)O$9U}7cKqtWQ~LbOdDv9TeX*CG zFXwe8%w5Vk_PN6t@-#^6W<8pRNIUeymC&fXzW%3$uzl;lg9m^LN|) zc(ItX>ayd1tGO?=zIN`_*$u(Q51=g!q`UsZxrP@7h8UA3#d9`A>rK-pD6ui%* za{OSu>FT_#sd-agp8M6fW}Uw1vKls`HE)o~UtMv%62DU?3*S9{v6`ZBb>1i6YZc>| zBH7edh3w-LKGt2Wa@s!T&u_OE!GjO#WA0chLXMMTeP|Lp)`y<&7@I^G){l_oWcf!+ zqr`{}CJllrV@F!Yjx_fajYYe4Z?!YYUt2Mrx2v(}mGrgk&b+tk^-|@;?c_H#2u>L4 z*c^3K*@ncj9a+{IN-D#fU{}@h5B4(|^|!0q2wR9Ut3RHwD%fDXPCyrwtsfvgl6Y_gwM14%Cb4rwzVs zUmuI_)DtB}rO?;b*wq=t`IuUXa4xl+TA)yrp5I}9Z=Sbz4ZWSp)%vI-a`x2oYwY*s zk*a?poGYjYKXq@AUjE&qXnyb<)%8c(XsLfvm!vJ0Iw!HT(T0vetV*5xvW(O@L*F{= zi{fV<5KTF5Qct=iQE&U0w1j0*9Ohp&bVllpp-V{5)PC0+^Qo1d+N?A5va*h@t4Kdo zS=Z2|OHbdbi}Jm&mCzdV)|qaq6_Vc6W`&mHbz*w*X^pF^F>OuidC$`7=J7rGuw9u{ zPqRrKFgu0%bYK*pAJUkQZuMUMqvso|Hm(6T39H(D>^n7nT3LQL?3Q}wpPTBH)Wy{F z_3H6$F$zyx^`-jP^CrB&syEh;kFKaie=W~fl3Qxw)z8(Y#qz3|LYnY@e6y$1+I&1QBEo?`shjZJRV=BfCo7C4Do@md`jX`PnE zUZ!0QH=gp}ih92Lcu!tCt5rySxIB8i+9zK(J%2%Mg^w1EbX;K9Wb4H_(L8U)yVf%w zGV_HO1Nnd#yz*#rV~k&Be7`_pL9ZAE_AkXb>SzC!MKMj|0~06U|yLQ(5b* zJyASG`(|R?+mUL$oJZBE+ZSpf))m%wCI8^D8On+I8M>=IGi2pw&L&YWHJ=hvZ)Y^m z-Y|;)`6!M0>r9i1@x_9J)Ol+%TbC5?`q+X+|RTyyKoT5Wj2=*ud` z>o?WhI;2Azz9iFS72}PWl=v4lJ^7}ZR(^EA=J?0fV`}%m!(HMZ(W9!1pV&Iw+GqRw zWgoF;{b)DswquOv6l1J27*{ymmgk**S{>YD zhV63JNbUK8o_txC5g`~?TRBCISye#wINnA0EG)y1T-c#~KSkS+Es~!PKWv>6 z(tvw)JF2cNwo1eK8{f>aew@~a{~Mk`JN49idyDZ)t?oJd@twC?Xch85P+KaEUE+oA z-fG9Gy?9)v^xCGDJ^1eM0qV&#nQR@~jkTp57^)`Q+nZl$(4L>}a8Mi8XQ0(pnNFi} zs^jmkSF88yz%lmuvyK@3E|nZ0H{h9Bd)(~5-GsZs9X1ZKM(hbzhlBr=C!n0jlHHx>h}dI`I~p#w zBtN2^B{5c9<~QFm+LX~fd8O2uG+aK@&_H$3-Y7nFSGSO|Bf>?QPDfPli{otzd&`9-ZA!90R?Rxy5;C9RmV(#o4n z*`d0^o02fGvq>Yab=+XZn56L#%?9QbcUIrj4xIH6fv-yQzI>1M!1K#mWYfR3gb5V| z&Tkh`m|NU}1g%k3eR9wNazaJo)_$+KZ{X<8SyLw&vZ`mscD8QJqx){_XPI z6k3@|9e9zFJJrJF2ikBM3h&LiB)^(sc^{5(vs69AjvE8jj@QfEFy2*cOnmC4&G#EF=DIDTn+4U$Bf##>AT}E3_D@nvY;==xF_TnpK{2WbxU6!qEk+$^_wN1D2Hdi+MMDd2=TJ00+)_n`L?xn-E-wQU^CSH#a*q(>e zD_W-C|JHVnt|)kF58*jHuh=&CrdAB>((4r^#F-ASY{%EPS7+Snq&>O+hgiHW2X9ii zu*m=Tx`ypi8?#w#DBvTKzb~zpskF+rvU6AQ8`zu&-pRy^)iuPOX_4xoQ_m{wm;)fv_?EhmVLG-2OIDqJ|-u8Eb5+ZcGoCAs!%rFXD&A+iPqzo z{a$Z#n?c%{N{R2W@fif33n-`8g?a+&>)83sB|To}b5q$?k8A@=_0`WbJ}Zi8v6cGp zGZklU#dx)U4mB=wG^E!jl_&mo!o`2DCxUCFOB^--7R{`S3ALU0d0?Qi=SZ(GotlNR0~CKT+>Xa93Qm=|7at&_4J53B4Jd|>DVb)JZp;A!LqN; z^McR6iSfv3(IWG{RBF)HwIs0YgV652a(r-GUQFJQPPfl!kaHCycl!fidEtG{i#*#6xk!LurVIa>9sv z(lAEclW>I*_oV#@M%(N|c#QSK7_pu?~N6w4Em1u}7 z(GXW6IXZ(YvB4M`T!|AWhqw~S#Ti_Q4RIwpcVdh=;z|VKN(ACcG{lu?h$|6@E3qN2 zL_=JOhPV=exDp%UN;Jfk2*i~L#FYrdmDmtJqG62q5rHw{N(ACcY=|GxFh=}{hB4wS zY>21OFh)FuhB4wPG>j2vVM9EHhB4wS1mY|N;w)^4vk-{05LZ8Q78=Hgv#=q~LLkmU zL!5;`oP|J~g+M%ohB4wS1mY|N;w&`8SvXBkn*&+`)Eq*JMT90Y`j+hB4v`G*=k$1vZQkcc3DUz=pU3j`I{KOqtaT&xH2#gV5V8s~m1p;Gq z=cyvTKwyk`0fFv9Rm2@waT&xNsE98R7$fdLLwte27;y*N5nmuMM!bMP_n#W#2sB&< z=cBt*8{!UZh&vF7JJ1kcATUOJfxsAX2O8oI1mX^y@)37nMce^L+=1$9U&DKrtNq2* z-h}rH#2wgN*>h~$@m_$q1MUXg!HQN#)L7uFT-k8M9jJ&q;Mkrc?tmliKt$d~ zaR(~4ONcuFKJF8Dpds!+L)?LexC0Gw2O74qh&#~mIzrrm>T11UJL77-Bkn*&+<}I; z0~_KFG{hZfh&$MhxC5tkf%V_j+CkibK-_^q+<_BD+=1@LI=BPf?{#noI{w-{l|08d zxC6Z|zN7!z83*wKK^P;BAPC>3eea%3zT*ctf_TIcIAO#Q#3PO%{{JvX5RW*5c*GIJ zBaR^cYh!QElk%G!!4||3Y{k3)#+Vnt_$S`WQGXI)#03N*jvyH4BaR>#@d7~@BaR>* z^8y$njvyZM0vIEXU>o8HoG||OFE|1fp20UJ=<%{;_hd|#Ela6?w>^$g?fEj6`JvpN zma$&WktTd;j*IsCju@I!)6N(z^VIZutiMOT0`g+?{G>;Q=y7sAZqAdpjD_^)IJI7{ zIS+(*cE0R-zq00+aiIUlk9jy*SS#dn+>9?f9jku)XpaeT*UQWH?C*0<+ZIFpCHuC0 zwOobr+lD8D@--&L%l16m4%Pd0BW$R8xPC59ri5&({0GGFhr)YeJ{&3`+ldJb3Mt*o zjfXag3Ng}p`yFP&bJ-69;mbHdA0G;z# z?emAz?%dQ%j^pPzb6ljJHe&mHxbskWpX~p!tn&NZdHbjKdavfkIi)+5vOlkJr~K%es0${@+q!Xa)%rj zZp5;j91l4^Mr>bqcD?@iQKXUnU;EG=)cUD?No$* z%XX!nNxN_%y*{pLz7%eHe(tO}xNOh;RxWOze|J4CzgyOmv7GmoneFwOCzanDqk7%3 z>#@n&m_Il%2JDAEE>80i)A_c2eb0Bqqj#4`*P|&Z%@7_#(j1P zv7@P;mi9xAlM&1J;Pi!FvK-;>cIMM#W1ft7$cVg7`EQ||(a&rAw2- zbXaLmuNf1mp2#WNjZx=}&dcv5*!%n7YHxLLdD*VicXz)1Q7?y`m-hdjzMeks2XbGC z(e3=ZDSh~IcrIgU|E1nL_5pq0F!JGh9Q%l@C-({a{-Lj5NbCB6@{Q-x&dK-M%iXi< zk-jg*sPetCoQ#cf_ImdG57P$1bHbAx=&8r@nM}*}hK+k-mGO4}WX5;u`&qUCnNG>i zjg`;!{Y-tnzo-7*e;sks=>zq&Y)|(4BIL{WXM}ti%Y1vBpmrJm#JUBZ+uxm_R)F{A zhjK<5`s2WM{q7_CBi9SmbJ#U_?#@S5NUqPLjK54v{akYOG5q^ccRl_xE%VnjmhW-r z?(K;>wO@9AXUr38r5M;Z;X5ad$j-;WcTa|R6vX95X6G{?&Nf)zU)8|a5M9559P!>i zO6zH9OQhb(b{o%1qqdt8!ry)Dr>E0L-LTH=Qu>oVUHR*yb%06Q9yte~U%Bl4=~>qv z*ROAn0~-=`H?%M3sRX=F#&(+}W5W-y@2kc>Z`e86uCdP<`o5O;xj+mIg)C@OXVSMIKv1|{olcH1M-xO!G+asKxb28{- zWuoSC#CxW>@pm3+RU@{Z$<5D+WU3LzK|7=3Qt03F<+{Y_rIle{&-B!Ke5@PyUV83R zEZ;Y!B2i_$q7qSWC*rKf#8$k?zqZy)65 zF{AQ^7;(ZxuroVbgc$Kl>$NHA>mv%swljj8ay=5HhK6px;o;Ko9 zVfOx9bF61$eYble^|?78_scZ>ea5=Tf5pBINJ@LY<$+!AEi?bn%SpYL`nr337d?(y z+E&ju)+=dZU)S#I_v-7>-LHR$p*I1m-FQs9o7Nk0JjdAc6CAPJD`UWJezevjlAmg(&y;r|$i zSdNSHciHb9?%py?@}p$$U8ndJO)hjQ4=wVt-$#9tVT} zVow|6^f~US*LUZ2a?5t?v0Ohl^mKFXp6j$eKh80nUxH-5b1c{WsDbi3-MRM|dmN(2 z#L-Xtd*ydxeX#3=^uz7`vT^=(_?@!ofA=+Tf1|HU{k|qaUtj;f-OI@N{_j4;^?k~J z_ZVNl$N2Bw;p_Jf(w_czzwm$eenI*zc3$DjdxXUI2l)IU_W^v)k^42icahKkyC?YX zoPeHgymt$KqezK zNH$W0WTDw0WTi=nFBt{z8b!{KJj|UX1(}p2gC4+^0{tWO3fWGQ(RFk?Sx2KGCkZ(X zPftV7^Dz&06y#BIlH4Rd#1Et&xkKL2w{$u5I2&Uc#s0*1NCIyAiaqfaVNhLZ|V*qKPgT5xI(imi8(vZ|AO-VD5%}66s zmo$O6E`+9lT9VcvTSKZIX->jGhLKjJ1=MK_vN34`HJZWmwxm7j1hNzP9nc>n5@aN# z+L11#8^~@Xf^;T5NKcSGNmmk1xHQlM`8huCF4Mj zgVabe2I7$r#sZo^CW4#@sZnG+#G@cg02B))$3vJ1VI0UwWGa~saypp?XeOBjau%c} zlNk_ChAgv(l!Zk>_bEx|A#@t>|)iI*%+ND?qLw%K$Z?3qUS_)O>ihoCL8hv?C2-9jS$N zrW#2gooNEuMm7^0;LQ*;(B5de1LO`ONN?JMT0vT2_M_+)C>ce!!IOA+wga@UF0D=7 zNNp-LvLNVVQC5I%rbXFiT8vGhj8ZxU!U(8Y5_D?|g!VKPw51|RO>U6eAa9fOBrUl| zo`8HpQjrC;HEl^3K$t~KkTPU1$h|Zh9YC#g2gn`t3Qa=i!Ph0B%ScK}SY6N@!kQAw z7|8mBvHC<|Ce{G7&BT6#HM}1*<#$+5zmtRH5X^peklo1v_;m-JH~?WESdrc&8e}v{ z&XTb{B#Ia{6qf%nHS=`~UUZ3)`^f;J)l(w89Xl9#kDc|}u!H5&%@Y#2$+(y-xV2uZ_+khClv8%hR| zbZijVpbTs>=t%}PlVoI>Ku#r@*i@34WdS*fWC5F&m7M_{oB(!c0y#&{gB$~L3^@zG zF`%_)A)JCWI|cMgrE_4FRsb{u!cezBm<{t$vmAjnZ5 zM}f`zg{>xQL9Qj)ST>MrAYMZPSbbU#bSyjD44RXjX(R{B0kSjA0s0ll!eREZ(Quf* z<1{-f!b~7dbTjEgqd>>H(^d2a?GE8Iq|1ShMA4F<+% zNiBMaW+aDbW*Ft2WH;D{-C!qLkUz;@kbA*SGz7T|wOO7IVg1;#`rAAv+(=` z$de#XlA|P;y@I(725ls44sA})(K!(A(iE^ZTY+o^I$enU!sd}e5R%Yez&`&s1z2w81JVa}j;rK7$qF(n>uoGT`a0N!-CX^_^ZjpklAjsjdDDxYqAEE>v(tacN}3L2Iv52nRuMcAFUG+ac}50PcaZKh zAN@ckQZJBRU`v9bUUnJ`HFtuiu$Ha?xrTP50c;If32SR5@nP@C6gnKncsLzOgJ>z( zi*vv_$V_uU*g_Y=`_j+?V10{FPg;;>0bf8OPaXXNZ?Vt-T9gzAeUm7FN)$>1X=z%H zhQKZ!s-to`3MCb3Wm=V%CY4Djk*F%{>Y=1Qtw9^nCbSN%OY7^XiH;V*IJBYdXbZ5} zi-<(+bhMTHP9tbn+JSbVTZu$nb<_vGtuKwCJ!x;+M@KO_qTox6qH%Nxd@0pYoQ@`g zu1%#g=mZ$A$vT>$qlI)fT|}4Cxil?|fJ94ml%B4n8|Wsw8pblcjyCD27_2SPKxf3&|DY$~_XpUolMs%>dTvhJf^17K zz^^S00~tpDf?pWy#D76J5Bf5nE(f`s-lzBIGP(rh68exnpo_sGJb-W)cCjn;I>_sE zB>b+^t01q^(KMD`qj^B)p@U!-EkKLPy_9Z+Ur|~ZWMR6E#?wMD_wf+6fGw;=8-r|2 zkHD`nZ3waz0DI;ungP;j!2Z3U|I&1@n`R&>VUK-IQ^GEr7JR)Y^eK&i-M1b1f&b9Qv_0&~ zf55K)2=?i=kkcM)WL8Ki5Z+P-DJkE=nII+DNh#mJ&i$Txk#`{9(GqME_(~R#7Pg*j z06Xf>HV}W73&zyUd|6HK#tzd4uu^?k1M0(Sz!w!~-mDUMz58f!@Pmr6;-nb+g=PW^ zRD@*$8lS$~lWV95%y3y^b>q#((O2TK7VAKL{xlsiaw zIE!2$l;j4Pn-yS$`~~$0gq$FAv0#wF>^0p8yKf-dNCKhX>tHv|!PY^}M!*^2{p&~( zmYpqueLDbT0Q7qv?C06oJjhu>GSJnK%0Pbs8=9Q-1Mea^guHAZ_!?P3W@Y_IfAB#v zv;N?P3?O~MH_6EQf_D-FK1&LcAH0_QEIk_w{)?39*+}rB)3CH`GZzoGz1_k`WJ6HI_%otyE*V zmFDo4n`|M-g{%$pfTu~}c~WRgg}0|xR0yS&gKU7Z8RTXq6XaZEH=tiPpiV*4HTE~i zzgYp(6ZV|7V^1Koh8`Vf=RlrgAC-?F$3i@oJyY7UckpFxA#iq=g)l2fD|^T)vfAu7 zkiW5-tP*R)>N9HcBHnNe_9n$mby-8^r__begjtkUtR<@sp%uthtQq`Tu{t2@uv!pW zF@L2Fga)iL>&AM5?8&;quP5sbvN!9+xa3$?m2v&q+#gPI{!BRwI%ywVdwZ`O|uVnaX9L|Qq zZ#atu8Ouho{%kah0~yCgu#s#G)Efz55Y&u?Qe)XfHU;DqHUWN9*i4W!*>pCJ&0w=Z z&SultR5pvv138b)W#b{t138b)f!{nf8RTR(3Bo)mHJ>eJOF%ARi{Q6}EeE-rEn^EH zEC;!qErs85whH7bwi4!kCBzFMtbpGtsIdZqY-1IaTFut84InqLb@1E3HiO*EHnBAj zHiO*EHo|W+iw7Ccwz9Pl;z7ot$u|Gln$#%f+Pi6yYV@;vvZm1h#3NS?|ArQJLU6lju zDC?>mWrx`w2uDF4Wrx^7c7z=Rc?`6;pCUD&pVEc(QDT&1>;Q{_T6@`kczOi1up+Ck z)P~SVsR`QHn0cAJP1_YO2t|~cN}Q4gWEv%{vX^yMIzzqApjDBeE77bJE61LJe8wKL zGOQ$f5Ar?xmn~K1D{Ymf5Wm`tW& zAf+ZHF(ow(06743DjBTcejxjSzMW$iKwr*57|Z@*=V9*uf-nitSX8)2pLW?nlhON zfgD7JlFX(orcEp}gf%Ry=@-*Fkn2E$vYE1jJn>LNV74e~Zv@hgzN1@ab* z^;NLu*Fau_l+~n|VnD{Q_sT-$k#Y&-B_&MB0=nD|WIJUiD{9K1Yy-IsMzbnw0v6h= z>`<Mh%&eqS-ot7NfwdF@yG5WW$aGo>gb=ADQxs(}$i+$yWvy~gDGai(LZENI zDA_?~ho1fdJJx29n<14!$*5!lnGIH3Mrb!P$jp$Mq$Gz`Jp)#C3dIENB?p;YnFxp~ zqd|^_)GPLeeSmNI0N?ZmR`q+3?;-V^y350pIfy&_%%`*C@mo+u|&Tstznf47Ou2W zIzmcfiFznqlzp&@yD5Jty&xsAL<8X)`@^rF60Hn|l*AH^1z#c!{9=`n$^=MBEYU2` zwHfeBsZ3SoLP}zZmMaUCW$??PEK*iMN@9t&!no&x-zH^)vJFxaOJr68l-(eA!)gmu zN+`QP?o$3#g2A854KlZ~Q#q}iReFN#2`lBSa!%Cfd%UXvJ<2(DVLQFAUnXi zzN}nP+JkHlsVlG-w*uJ;QV*5K${LVsKxh6@o+>Lru2fblPrxQD1i4UI0lU~3Z;;-wlWIzW;sw$RQbMu8+zSZBz!siVT7YZ; z_VA=~N@)hN8Kh1rCzQG%>q6?Ta$kuD84ou0zVblX0&)wa9)JZ{3vw-_&MSW@LqHAz z9sWzXs0;!*2vQfpK1YF!f)rOODF;Cw1f8v@R8{ta+^-x^s=$7{2jm`QpE5<6quc;_ zL&>MiR^}1DXyNZXSedN_C~Cat!1#w{=#dJua#HI4Dk3~f_$m`t4shd@WuaQ?>)ex zNV>RR6clwa0A^iRz?=~<08?iaBj%iQ0!2}Rpkx)n1cEs$2#5i50;bLw*Q}Ux&N;_b zzp9buJ;S@b%NL&SxzD}Vd1!tXrz`hVb$6YAiQ@+`R>z51p*+4feP^l>b82~fC$2wN zG1vZS>MBAs`L$xK{vqP|A^vY-#?1c8XCi*;nd~yzZqnFnm-ult+hI~MUsG{xYSzT8 zr>NUT;@HT{$!wswE*!aR)fLCO;+x4* z+?C2>9kbeI*TubLEpeE6?YhRW@q!;nOzch zpf${FgpG|^KCuM1lK9Ru5$9eL{3V}8u*U2Uvz;Q>F5>7SM&oTJJ5A*Ace9G(SW$cz zZZoNCW|eQBmx$rR@p33%&&`!V{vhBcS)44s5llilXLWW;`bWyV_~*L{7)V&%!-(O zGL0AWvrndlMf=2ynf-6#_?x)@3(H@~%vKz2MICh)?+Xi=b(eQ+W?fCXnG_Vqf@THG zI+=7aF%w5KF+P9E*U3a4O-27|FYcsEiepLfJ>5avmC9p&Gkddk;$Eu0IMz3-Bj$6T zOqrRxxYsl>du2Nxcb!dzdV_*si(@o;hOHBRSy7N-5rQ99ZiR&Rkx(Wr8P~mDDc9?V zcbmE#tFB5kaP4yQG5@tx14mUl4|_*%fhRKpq4sAzh|CVkj- zNHbk;;I`op!C^s?ft9^3W&FyXmXcq$sn52aTMrFhM*(H$6`R??S{vZ$%P62+pt}RB zaCSRfd=d$i9g?@Q`06`g%cDr3O!h_wvNtl2y^(?JjSOTzWFY$?1KIZ&T5i!pmCOVf z7c>LNzQ;iJJqEJBF_8U@f$VP#WPf9^KiC6s@0bjuyLtlI(-@R`6MGs1+0z)vp2ncm zr`XdN$ezYP_A~~vr!kN{je+cG3}jDZAbT1E+0z)vp2k4-GzPM#F_1ltf$V7vWKUxt z`xpb+#~8@o#X$Bh2C{cCkiCn6>|G3G?_wZ(7X#V57|0&Q(7NJ$9C$eu#QvyYS^7M9 z_-rvS+20k&KCU@!A?=kqmwa;8atujMG@KhW?m85(SZ`o*-66T|kX(02t|=tf6q0Mf zh{?5pl*zS#l*zS#l*x60HWaTb%Er%P~G>NTo?bJ*z22I7ynoH z^d{E@lIsG=b%Er%KyqCe?PE-?3nbSClIsGut*e5U&02t4+uze&+x(u9wxk+cvoILK zm$b;Z5$?@iJqQE~n~S{b3=cVWm~3^(kBOrVngS*zE>y$%b($N(TOX|jt~(ga7yF-e ztko`~(r176TI}y=*Rw9H9%k^hHgO2a57pt=~fFn+5^=n_@ONBLX zYIPk$c-hwtGSZf6S&4B$K>6F71{p)2X_@`#AfVh^|kpQahT!4wxC?+1=T&yI#ue!8k~#z6;5mgE@%YMdc#;w?+Ea-Blp zu&B>YqieGg!-9c^Q}+L43XR2HPj9nsraL@7!DcTV4KD7#XQ(l)tGw{Mf*KgSwhrvu znYAY`yh_y7tf+Vv?CuYF;h|#Ryt3wVnOmwqt1B;$L8VZU%mS|hhKHTkKK7@_^ z!Rh6g^a(%mf!WOdP)zKHM6Nq$a@`@h?vPwl&g7axaxEA!xfYNzxfYNzxfYNz>GLk+ zx^Sl9lu6s68k2TJa$O(|r%bL3XL2pLs4=boyfC>In&gE^U&6dF=}VXwR;xD}PU}aF z^O84elb5_v8**JVCD%n$a$Pu+>%y6~h01l|Os)&DuZx=h%wRbth@afQ|J$$TmtEy- z`(MAMA8YRL{~iAu8Q(wo#pm{iSNz|lUIz7lw=$~){dfAX_Sb}N!SP?Z8p=BM=DGac z|4;b2OFaka=PrN$ulTv2$<{~6?dKkxt>^H+a_7n(KPv$3?80 zmMhqsi=Q@P$5yN!w-ak%TMD)lM=Oz5K5@)vQdF!5 zX)8hsi=QH5^?Yejt~R2+N{LSy@&7jBcUiG^uAnG`o2aiE;#13{w)oUEgwzmJNKjjG zY%5|h5oK&6j%`Ft1q8Pe$5vu><4mz~)>Ir#Ma^{->%zN=V^^^%BvQP?v=B!Naeha! z>NZ4t!o+j7Q1KqsTYM&nRpR5tYVnTZ6JiMW7XJ?wKOMzdmYE{9Y2v4VSOwQvtPcOR zW_+Spc|JgV`iszZB5b1g3=kA4R@n{}$Dv{siMu#2T6{){H7%ZErOHV0aW{l`3hE}F z?)i$3zgTnbC)O-Qh|g5*9wB9wCn1#k!Yv;@D24Sy;3|lsHBS+nHj0v|JM{T=n+KRJWf+wq{r@gE)aTaGwD z^I{&L#syrQ^aVcF2V1c(?Z5HanO$;haGRpXdT zjr1-F8bd!+W5?c&^)G$I9^|52YMipXvA+50hK7FlZ!GIsvB}L!{Ib2&SjnescQsb( zMD`ywR_a9dJvA=0+EK6RKETiq)p+My2YvVaqYV8}jf?v@=vUc~GxS3>US8Kh-`32> z&=3EOW#3b5aAvE9I5_P>nymbJE-Qc8Bot<8tcIW3D%Jxag?w z*S@bIe0y(4ebvrG;K2h=nDos-ZyGrg#`_I}n@1h=P24@9l-R3S;ahgya%`r!wgY)? z6>)Cy<1KRH^jWCQNo#gx03{DHJZ?fGy=QkfgZ;5njr24AU?AfeVbWONt*;Bn_!XAv zDmF4d%DFP19o!aU>D_4n1A`!;bO?6&eLEbjI~z8fS%o{ArNZhNA>h#@8Uv>aZV?0d zd!^#_{cGjDQ+WU9m!L@9op3Tb5dL_lhohzTz_o(2;d{_+xUzmTwAnZrrv3H=R`_i& z*z2#9$AN>s7`XNFrBMF%X9L^DZ)F`0?SyY-X9MMpbsX5qu{$BP!fc>Sp4|gw@*H3F zT^f15Z}eV`zGEZL{ed!h2VnG$jlN5xth^tRbwycu4lBbcllKZn@7vVE$vXz4_i$?A z@lS&G;a+gz zWCVU%vJ$plh=nuDX5gIntH5%#F;Cw#Q|ynD2rY`n0_F7cM%XfYy!<_J0$>&hLQEx~bQwgUp`UKZTGZD9^4;N&$E#uFvp4XycxM>ZemQ)~{|Pq!1~9am7hH_|00BupU~zQ;3`%|k3v7d+ zO8*Ghv;G~lKIscPzR!l^3tz(4#}i?9zOSt7vNf>&Y^)LYoV}UN>b?;QIL83xsn5@` z^p^=>zC0Ev&)u+x&Ayfleg$KJa*OKq+1T$Jpo2>cPhA-aD|BHsZ0^Aq76Qli6k~l0zJnDPeT`T@ zvpGA_@i_#Y4FJk6-g-8r$~{=sBLvp2DhxlvKLDTR1CP>Avf^Ez!?ec%kXqz9i!JgF z3cvFO^6t%u$$L1}chzd|t?4^#%H+MB>bq_7K2P7AU+o>g+WUUB zcmHbZ0Myn5sI3oBTPvWpZa{4ff!cZkwY3Fm>kQP^9MJU#l*v0+qjd>1oUTz&W4d-h zjp>>O%5;4LWxCdZGF|sTnXZALOxHtDrfVbAn7kV{T0cR<$M0T@Q+uYt)IWXU=crZq z)h894p7=vp(MaqvBn9j?#K51c0&qs~78rAEHXQwY3zmG@47E%9!s8*ALI39t82D~H zq%H`+-akYz;BUg)-M^#^o<6}ny+`^b%hke9FKBfK^E=59Kcy59v8+B#_^wLK`Ubw5CGd#Tk?p%n1yzt>KdN%yYb{O$B z67s?$p$yisd21*m<&?ISu}tbmL-@GJlI-e+olvdtY@od3fu2>0+6h}~&j!lx7S6{u zqV0ws@de8FE+?U}M@%>3&ka|A zX^~%f%L-f!QG4GQ;-PHnk$`*au7h&i7bu5aT8r(>QeetGPdM^033mk~LzYD!SP&k8 zpNAzt8TVLlHjToVeF;#hsWHb~nT|Kb8l5f`V_|FJVR*A+BYc||1Fr**z=(^fQ2d24 z-+g}=E=R5dn+mZ&c||Kd3?H8YBbvv;pal8|xK+1A`m2g$t}Pa?LZyeBK%E zEiZ~ShChIKao@E2*k|5n#WQ$vGXU)7p5S%uUV~#tf7oD=z+WDH1LcSLfpwX6;BNU8 z_WtMsJ)#rfZH?D3rI{evsrG z4j-$&hUa(vA@So(80P;BMrH;=THSfz_~SktTN(@{-hE|-MkYa;TdWaJe{z}Kh)jkj z#l`%Y@Ftgo%)4(Y@KrHD*>*E$@4jq=$CYD%a)Bw9Ecxq382DQZP;Ol)Kbu!H719>R z0Oe`L)@d%^OoBfxV_|dnP*+dCwNR{eEWAAzqIq%d4crg&gTT1&n!sDHAbg|05#Oy) zoh?av3NH@_0%a@jhOFU>yAaqX1oQ!?*`LGi!k7~wuyX!Ywz=a|$gDgG0>`{#$IrZi zg7^Gk*U}AHIv>Eq?>EMLSaNd2t1J-FyaKX6``QZfqi4KJfs2o^}Ds6(dgY zz+Nw*S^-a>eDKQ}-thY?m{r~bD4#$4nMa*}0^{b70m{`Ai(<`H_rSr*8z`TJ>TJ>F z$8a*#3)HymoQAA(GxcvU&-hhk1e;kMIc9xR}3&PK`Z)@)477tn8$< zFy&=5~i~M-Uc>v`;sfVF-$_8L9qJeVrkRvek);jQC5)FCbm7fp8 zkNGJOe>fV{_&0Yw_)SfQrZ1v_azwNI?2P$3IM6y8)VNx^b(-)OYhjEo8Yp+EKiqZw ztVC$_CK@Q)6uAfP%~D|4)gC~(;>$Z?jcN)UxG)kZ|2E(jy!j&)YVh%(#)T?e2kZXp z;Y0~vpd9>s37$yZ0`L0-0p;1{*WmYn4bXaK2vE+?qOstIRIoTU7bss8>(1-#-VUeN z#{lJ^y?4O&@m4s$XEIPe7=HyS*zShWGXsHgR$3C;4oii_JH*~pDm=&`6*nGAf!$NR zfbu)P^=Q=~1)^5>0m_c2BT)BgC3GAZ50o!0pMlRiuZFud;z5mlw$H@N9o9nYg>j(9 z*Df4|1}!$j$enY6awqo_ki07uewz^odEsi$kAPER64=?qgBnL|(8HlZYvFb8c%VGU z?kPNavI)uz=n0fN7kdKb#;3vOww^$F@Y4tI{l#XexoIM(arb~qIHLYH2o=w)D4#E6 zi#X~F*o|R8dDW5Hcy;D`$Y{|CC~qv^6rVnL0p6@PP>xKt!Y)@IL)!zPK)H#t3BH^5 z3Y=a|2Fj(boZ;VwzlYze1p?*u9TIuRdLQ9-)c~M8rd%qV+VULQckTm}SD#3LxRd!ASTO*~t-_bX=OtfZ-ho;`nN{(EV$D9n+&KY2xlP{)upjXr&g={X z$`8KGhLBFL!0l`hP|i5L02cIp3?0nFfpTiMFKpML)v(hw9`eG~o19~g1;w2CXB^~( zgLmy=D_m1xeoP#w@%9;;S>F>Ip(mdUdExv;>$8A~8(|2F?}t3_R_n5?<=JFt9~%d1 z++=R1rt78zI9(+kD7SEqavfD<4OsP$2g0A8K0<7(0HC~RLov3MzXMNwAgHlh zP;++6`~_Uj2nIC{u(Ddao3$#CyR*;{-M^cLR(@;;Gr7ja!Y6W4_dr+odfUfhfAfvHnsz=#_- zFmLN!z!!+{JaFU4t*qLZU0@>a>GHzw1~0`oZMML`1L8iH@Yrpu@%6e5kRqO4P`0*? z#Ls6_U{-@Tpj>oe0Jbi@6#~V52jx$V?m$GNZBWhD7btHpbQMA~cR|??%Hys&qpsKsDEOf> zQ2r5L6vtaX1bww|puF$oXWn1DBUpPZ7%1x(oZxX+Uqeij$w1k{GJ%^;c?(55OajVB zzox>-7UJnNYIcbNHMT0+jpO%>$2Z55VILHIwE-p4h{Zee>M} z!()Yg9{BX-d~CgUDy*p-2WmY0X^LjEHW`jBi37^V9}aPCc5W?9I~^yk^Z)Rrtsxq# z*>B;L=Omz9HsHJF^rF|$e&J+LF+`>i-MuD=WJl!*n(^JkP~bGz(2HhL?8Hf9(edC z#E-Is@e}8Q8c$4Xq?b0b?3BmntEyGPDa&}KMIU0V%SRaERO9)<`Et^wyzjtC)?Yl2 zlJThVMeTi-Fe=7i^LISdq`qFJEAvcQ=3k~I!@Vvw)XVsDbFmaBy^K?aQ&!HEC-F-uym=zs^xVbI}8~*~gfNiL#v3on+F+?qWj&%XAlQY^Z;^ z>w>{vhRb*=dBzxQ6qe`8bY(bYd9Fa!Mo62}(J@oJPyH=@B%7wn#X_dO7 zJhGOJR;gRc%DFQCYOJKI*vLG{{#l`>jaJr!4DToMqtr?4&&pb*9uABANPAgVl9f77 z_-3nB>Q1D9gSi?PZ_L3(I~b?PWh#W96EXX;GH- zrsR#XtbZj=HI{9mUlie)t?sj65ApmYkK9*(lHC>0tfc+l@dXjC zd_SnM%!5*1HHOhm^cAoFWMHv8)Of+@`ualaezH>g7Q%j!pYozyCNJk1@w;8WYnO|(>Wj4O@BFSE zDf0G4eI>&wUlMJU zIr$O`dp8gA${pwLW_Q2LGT19D^S?**nQQx|81nyj{IP32{nCozhH;Rx^8GLK_IE5b z8kF$d-0p#czEbZ-hBzx%wAZJ$>1W{MUQC z>d{m`wQPB4E7DTqCzl)RcOJBdKEg(gXI^QnzbtHIT59}Iq%~QjC2iC=GNqBeZDBVU zE7DTq;x(Q0XR~_39ATrzt!p{yi=XXjNK1`Nu4|<4Ez*+tQRDe_?e+Wj4u|*Mi)$z! z*<4>gX-c3Wobs||jr56q%fPn2#Wj@S4<~&-^ZJHx%11wnwce%a-ihR}+;gnZaa?;-{SRa~;aLQ{%J~xVdHWcBM4+b>U zZxiXZ5#f}bMLuhaeAW=*l*@Ffr}q`#0~WVS1Lexr_4T{!Of+1ZlyA_S)+!itBo%NLP)IH*csP>LS()yekL8-`eX(ZwrTr_LWq5 zWeA~lD`>{`eAwQJAEVI!L@}FRclk(u{HrlUmeGU0hV{gOHP&&^6bT;FP$QxCaW$-j)BvQl1|2g*wRWjt!ElwIbBa-S5toO)AZr9a3z zS7YyecG_dV`9n;{K%ji(pj}QqtFg~jyPP(ntn^Qrt{UGSV4Kqq)%cj3t#-ax(=GG% zXD!>DKBmTlw6-~YRgIOj3~iL1huo~>S+^y%bPZ(v@wh#!CKWo2s$WUNR4A zthBq#zZxrTD)XtdkxZAe(!Mg!YOH*Rz*jGrHPlW!^JN%(|J~n+2Z_G)p{PGpY##`e zvl8sI%iaY-C5s7=5@n~o`Y;I27W9L(1Y51+(3ysKl1B*BUa+6bgL9lR^qAn$L~4udsHc(Q!W{gQeMeQ zJ;<~uE9I5(D`k}NC}o%Ne|l=8EpTh3Va`U`W{-Gg;&0F-3a30}hMo5OnNbFN${TxD z)eal=Ywk)J57}vF^&euer|ez6s&>OQgDz4y<+U$uwN|$V8|=HgRngiH{AJTe|2uow zKGyKe=bv~~uc#cGzvEJ4VsuJ8YFyMSMyH&s#xf7mX3D(z@J}q$Qf&T?mH4HN8Y}sf zHfnr2avvz=`X^S(C2jtWtskw;iARl1g^hBq8Y}V3JpU6bb)w{1u~)*C_+_{nEBTZ* zYTWT~B9;lWG4zdp;&StUJ@=M2f5&~^^aCXxHMV=(50rD&Sc!iCkB5I^)4iqPL)d(S zjT+bOTQSE*jb)ydwEm8jvPc^>UMkA2#IMG8g^iL=HCF0GrlrQIC2X}tE)0d?r4KX8 z%6C-ka~-3c@1u>j?v@eIe%k@2#>)4YJeRU}yqz{Stf?U_%4Ig$X~(T?WQbFZCp@#$ z&h0t?ZWY;jrusXOxegvD0p^I~Wcn?_z4Kd}qq|DeK-> z(I!988uFvYWt^&NXP7wuN|*46;#IX%i!ejFYOH+c%Q{qJ$IW)yD~E>}>Qjwtt+mr` z`!F2Vb=EV=v*(LB-kOnyI#=Vb4XSDfP7v?FMc<&Tj5{)KYHa3ORU6s7sUcm;3GJ$C zT^2Srq^rit*d*(`XGuG4nG5|5^_=#_R@-FDAVa-TPWZiwwvNZIwot}0d9HhtD%!$x zGzNRhihhF7pg=WNbQpy1Hmb3r=OBE*QH>Q{2;obPYD{z|@ZVVY`{&r?W+i^%`=66m zZdURseE)OuoST*M3g7>nvgc-{-h}UeP95fEMSDS>tHz3kgUo{(D_Rh;ENZN1O2|6- zHx~Z>IX1ajiC?yt8Y}sf?XJd3d1e1mV?|d%+NiOjt04QT8Y>zOGCyjpXhFzwsj;Fd zA?xJdSk|**lbe_2L()QRkSYOH84$avJ4=uV&-D_Rh;TxzUn zO2|6-H=vX7~;qW>T?D01phXiy;0pg^KQfkcA> z72OG;1(Cx-BO=E}Xh#^%&DNmEiBo7z+Ip+!u3M3j7NHi#rXiy;0pg^KKfkbx#iS7gv9SI{QIub~k=tv-CqA8)O6HPQF zj5MQ(c7&0BG|`bT(vK!O531or+rdaOfJF1bh>7L{ z63quBnh!=ybRLj0(R@In`G7?8!H9{@15zfM4@fj0kZ3*_G0}WLqWOSC^8tzG0}{;# zB$^LMG#`*?J|NM2K%)78MDqcO=7aD7%F%p4qWOSC^8tzG0}{;#BPKczNSWw7AZ4QS zfRu^O15zfM4@fj0jF@OXAkln4qWOSC^TCLT&I3{=nh!`cACPE1Akln4qWNINMDsya zhn;9XAklVEWup0jl!?xRk%l`BC)y51+U_)*=sX~0qVs^1iOvJ6G0}FwyfD#xP}Pwq znh!{{9aNcUJ|JbH^Ps9hPc$Ep=sXxP(Ro11MCSo16P*X7OmrTQGSPWJ%0%Y@DHELs zq)c=kkTTJEK*~ht0Vxxm2c%4N9*{E8c|giU=K(1bod={$v>i~5iM9i(G0}N2(&DG# zMCSq3nCLtpWuo(dl!?v*QYJbNNSWw7AZ4QSfRu^O1FA96c0e^Inh!`cACPE1Akln4 zqWOSC^8tzGgAo&*2c%3i9FQ{6aKOAU(Qv@LFwtfiN*pF zjRhnc3r0+I6_7H~RY1x_R{<##T?M2}bQO>?(O5vDv4BKl0g1+f5ffbnq)apxkZ3F* z(O5vDv4BKl0g1+f5ffbnRAZv6fRu@@0#YWr3P_pgDj;Q|tALb=t^!gfx(Y~{=qey( zqN{+EiLL^wG0{~(%0yQI)tKliAZ4PffO%n}tAKf7qN{*vOmr2HGSO8)%0yQI)tKli zAZ4PffRu@@0#YWr3P_pgDj;Q|tALb=t^!gfx(Y~{=qey(qN{+EiLL@tPWX?m0#YWr z3P_pgDj;Q|tAKf7qN{*vOmr2HGSO8)H72?WNSWv=AZ4PffRu@@0#YWr3aG|JR{<## zT?M2}bQO>?(N#doL{|YR6I}(QOmr2HGSO8)H72?WNSWv=AZ4PffRu@@0;)05RX{Z+ zx(Y~{=qg}dnCL2?8WUXwq)c=bkTTI#K*~f{0o9o3Dj;Q|tALb=t^!gfx(Y~{=qey( zqN{+EiLL@tCb|kpndmAYWumKql!>kaQYN|zNSWv=AZ4PffRu@@0#YWr3P_pgDqvoi z=qg}dnCL2?8WUXw%nK7;1uL36sWS|GbsF#@t-)`kJw zRAb`5k7_LaTa`MeO#JsTubl0_Z?OM6CjR@V#)`kNyq49N`0^w10&h5?_8JCcgYg{P$Iv`0^w1J>moNRW1kFHW8w>sdF5S9V_3N!Y4iF`!hEaA9kcneAtmP@lQuJCVuEhnfRU~W#W5|YD|33QH_c3 zxlWCV@3~HmiSIe8G4VY|H7362IyEM~=Q=eezUQdM#P=N4nE0OS)R_35>(rR|p6e(R z-*X*h;(M;6OnlFEl!@=TjxzB**HI?E=SZ3Oo+D-AdybTe?>SN?zUN4p_?{zW;(Lyi ziSId5CcfvW#>Dp=)tLC6qZ$)GbR>T0s;u}*3qNtBtoSL*aLS55tPH2D_^8To%8GBM z3|IUmB`dzz!Vleul{^UF;hg+PR{XDpAG#_le%!(j9o1MVi;R=9;;$|9Ls{{;mT^*6 z{Iz9%)L1Et@ZrrVi)5wj!f)4zl{^do+?@R9W~Cm4uWn8~+Dqm^jg@wnc~)bkO=UinHj?R5R@ztQS&fzN5aB0|v;O0UZp4Z&w(uE8%8Dr zCw}2So>MLvk5XRANwZTS>vwgY^Hh;%TS!5p6nD}&~8WW#xRAZ%1 zWLj!We7ZSh;?vD36Q6Fb#>A(aQzky$oHFt0=4woQx;bUy)6LbG_;hp1#HX99G4bi< zl!;F_S7YMe%_$R~Zm!0}r<E@J)Pd8U% z;@{1Qe>W#S-9}9OyE*ai=ESGlh^IVEhpOfd(4&1kJRf@qKJ{;8;I|{ULbE$Btc3xAZ8t_)Al&X2TL@+sp{@-O38 z$~gV0slHlvdDnLMj-Rv3+i`s%y<2v9t9x7MuVt4v;c79x5`U>J#r3DN%PYfcW#^}c zuz#FgUKx*LcK&5r$=T(VX${OSugl@LsD!U?^#mQV?LTJTK_wouhu5)YcKj7wuA!2y z4A00eue86OolhB$lK-Y6ex;1EeI<8qACI!mWqBpbc4;okD_Ln@$x8c5R@zqym+dQA zw(mnxUdb|^1|t8GW&0+G@=BIz^%donEbCthm+d0^gAAALrNpDOuMAh(SF+N+ioI-K z$ub@#|1y51j7s}T8>M|EEA1;;wy&%=$x8c5mhF2+w679gJG;C}`^s<`PowPo%e2;I zmsh4WIJ>;EeU)(8F0wz!bCvd$thBFW*}h7;O8YAIN8$um-WYD4M-sXeJfRI{fx;#?-` zjfr|QvZvu94@}e>hPKGJN`1=yL*r4}LdHY;VU7(A zCv9pJ4-F@M!%$C3d(m(iCh{RQ&~n$IT0hLm4@oO`Kg_WqdCuJrb8JZc0JW#}^Z!=1 zi2I;9x%7nQ@%3Qvf7%FJh1P?w!8RP0-q4k5(MflpRw!&c+6}{;io(xFS9D{OJn(q( z1;aHSaN~=PZ#=EB*dGkvZ)xCs@9OaNWH~;kK_OPG*m7OWmtbf&el)Q4Ck^%wzzJR+ zbu_f~j)irwgtf-NPkh5L`g@qB?8N1nl!JZ4p!u0_-QgDVT`6}+d86@qctmsaQ!r>= zRRy=#&9RwVZ6-@ImT)%ObJn+u(^M-tmUHw|uVeBc**C)X+>}74V-l)N(@ZISR7lpZ6 zTnOY|?E*2Qo(A6LuL1e5R^aa+jRxn)7xyC%Cd?w8+fIw6Er^gtL&BM z)xmD1f0bR-0m{0N{eS=YM3>{;G`!7%Af&vvWB@c;GnLN>Xsx0A{plFzIdxt5>CHh% zxlY?)9D1%ce|rCnE9GMo!XVxyLYHK^)Rl7Dt}f}ZeoOHT=KF^${vZMFnZxdests#O|4nW_*J)>Jjr1^6#v!|7k@`S;Cg=en%M{M z=ReKo*>7ZP>#oryFAm1SD=gXm@eTNtWs`7HKyfy#R{(Z-I)w)|Dy-SJU^Jd8dyK`5 zouQ-QHwN~AMPG}+^^CK+^S3W+3ato1zbTtFox8Py$|WnKV<){PqS$CyxAg@3GCflF zc0~)Ooe_vb-dQnfak35E zdv}aCX!)Kkdg1^j$2vgmq%WFlw>??ew}Y774Ii+tU<(gJY`|IUDN?-eZ5HovlZ~0y z7KV+eiP4Rl!nZMP`PC9hy5)EMV3|iOPagA=`#JT1$}|1>1WgPNwDf{H7iaJW1-*FG zf^lrX^$9SmcWvyH;sOhse%1_mahY%K@sype&;~3{h&^O0)P%j~@3Gl!(llp%{a|xy z9FG~?5R#kP!&nC`|NLelOIa`)?03%Ojr*8G{ym-H*ue?BQRxM)WABFHfIknoUYrvG z-h+PQPj2pXUD`1i=X9#gx9!X_;(WCV;hV#~acm~7HRjENG04}Nt#ba!o;UA@8!8rN z@i<h6NA!{+-F1if=cvLQTh^+vXDN{T*{QD#{o31;_B+E5cyQrrw&JrlT?!?Vrvn z6dw!GvuASsXBQaxs2XT8ni=uO;2^-xP1w_M9d*}ZC*#pYZCLpe0lLt2`T3O=zR=e% zj-Rmai8t&E!sAi4JmK93e!XZf2(gUd+PRll7e^1wviRbhabXnBT6TetbX%^`JTQS< z^#wYc`#*dzsY;w6$ zu3z>R<~k6khe zW35XtYIDuf)MY?;DW23U1k=8s*OY%0gcfy5vTE(SL%;?L(6&CK8~W^7`uP)$`J78Z z_}cwh5CT~CJaVU=G@43&VLeIJU=yHF2-oP9_Z0-z? zm-_Oh7R=?#x;VaWj6a?@zmOG+8x8s1&*Z`9Z#oY>7t1ZS`eUEw7#RO)n~eE`$w}g>h~$bSJV%|j&pwUl~w!U(=~-z z3wJN9)oeen*?OvGp81z_m;M!bzn{TIHdnVy#_L1cvmepLbfxxYId@zX&j+>f$Nh8i z^V@E|a3?&DKUh`RrLBJ)pXlq458@`HV^|wD>aU5qWy5Q^#03fa_Wn5J;Rl{BzC%{m zj^LrS+PQRl9>-S<@yCQFJ@HkQg3xP>SXq3h?SIrqQ{?QxOt0Np z&i=Pzx%*~+JXE$2s~qVI&BJ1Ni)Afb{1(pTj`RI7=$i|ieNqiPyqocZp+Ru9aTAud zvV(5dB|Dd@=5u+URDYa)U^>&67z@L~XL4%a`L7?kx&6=S2CNCjH?4MQ8eg{4H2X6Y zFWm18p4~0+jPotcx9~RzWlnDmFDm9vl#|~Dpl`Fu?5ZErQFgJK>f9>Sg%{lzgp^;D2!ZwY z%Ci1jc4bm_h+XZ1n4{=JUJf}e=~4uWEd_j zG+xu>@$5{>-}{Hbea%c=di!Ko%5<)4-t4Px?Zs2NW(R^%%*jE_$qjXzYfcMdPHPCS zIa7lLc2vi%jmop4E5ltYSDnP$wF<-+CtQ(v)xZ)pD>5+;1vwv;Z6xNXAm*qBrgPLz z8|JV-CRAYap88$?0a8_cD=EmmpPv4xLLF3a}|E_^(2^kP>eUkX3Em-x^I^!!N=C^`TkO# zSgBkVYf~vw_dcON^5k>OWz0f8&tWCEOw=&bGvn~OcYFNlf0D&+dcZsVaKK{`4tUV& zt?uYJN(SO; zr$E-jg6SyVzVMk>Dn0<(if1!H_eY@hk{c}U_!?bewRPOl(i_g_v*gx(J9S~5gWku#RO;mU)~QKd@o7rr5AGcQW>Eh`w(pIY*$4M9*>)0A&(`_%Pm zrRuJW#C+^xWC*5s_`Z-oA3ZO-m@mSy=#IS%64FMtf%8k{8tw=dr}QIM>k_>cEQ+kM=fri ztkKXh>&MP8e4XX0E9L3xO2?gPX<^**%~)JfbQYuW&~fec9uqzbeNmen!|0g)txrLA zuW%6T{!x-Y3JHapv#;sWt){w)u|89b^&rN2PHk)>KCsj82EehaMK~SP>wcMxy7#SE zrg;$^wR!Mn95_|k$4~)EavS&s|sffSlkz;4LQSX$|M`j|KlHxg3sc6 z_4%avx|iuQ*lyo4Y-yq|)(*eI3S^GL%s-toO|Q2GUDboE%i+JoecLw8`LTAIzp_Fx zM4Jy5ed&+IcV#uDbIW}NZt+>cH+X?k?$}+-Q7ec!YIpBb840IH@)_F(@FI16k7HV>%mVB*e}`vkBILpIuDt1=pE~|ycfEKL@+v^nQr|}W9j9K5ixO$ z&QCJ#_k@wv3SjeLHjK`5bQdRsdx#rf>^4C|=b3>c&GE_H&KT_B%jkTSjcAC?KQ_P{ zhus*Rf8GgP$RDg2jqgonF*=XkS#*1tCRgh6RYwOcfX#*{x*^Sb;m8+-(DhKNrmwhv>F-dAk8Kfx&;6?KW*;ZP(*xo= zhuG75BIipxbaUN<;ccsuoX$lHkNCu$JqLh=+iyIv-w5f5mGR-=#Zm z-b-_OO(?c6mk*y59Dq*RDVdG^zG!x~J)>E-FBsgaRKrpSnuB?Z-WffL-qx9I=%l-B z6AE<_O|W&30bnEM?;&o#WnNFUXPYVnL(fUwv295UICAWUZsLItF73vTXOAxgLebR5 zLesM@Cze#%X2<_Nu-5Z9b1yba_cDJ=)_+(Wmd zGi?L9!W!7Ux(*z-Hg#!zr#yZxUK6r1?=kBV8u+-R4y?5{b*|aJHkMuM0zI8RYwB-L z({!Hb2k+;`$$RZ|aj(t9y*3o6wuo7YXK16v{nYEw#u?w%#j%ZJ{9(;3bC{UX8LFQ4 z=F>~?j1um1*^-qa&vP1zeRAx0fK+f%Cd?!A@IpGQiO=p2c$HL~tGx_6pF7WJeH5m7%85eVs3~?{a#Jw=k z@aL0*pvAZ*thacdA?7?8;+~p`dulkfJPZz(>Z9?#G(K~0p)ctr-_B+Cr~5-vy%HJD zo^i~@U$phS$yl>}8+PGzfbMJUb{TDN$FWhv{6W*SC*G`G5MGzFDX00A=TiVYqWU?v07clQo z0=K`sr>W55w@mN3_UyP=xA0~1TbH7h$Fq}H1EIHjW9V789$Hna%sg*>VsU>Bf~98V zT`!M&ts9*cuCd({f_tM2u;TU3>r%Z$y4F>p)CM;&nX=pYX@@vg#N`i`(9}oVN4mq& z%KO-rN*{Tq$z|?X(gS?Hb$6Y=q#qnUbee4oUc(ETT<2YCj)F@U>|CyfjRJ?1OU!v@ zjE;_Lb@ybsu5lO7M4hcT9oJSaiPqG+5)2>jm*7{tf}#GK>iprp=dK|u4yHG)UxJzX zhCoUgKUn->5wCKsmuB!qyUgPSi?LQ>TyqZJEuQ2wWm^shLC`~Y4DDPFTRwcMd9*0Z zrSMH_wz_i&P#*fBIu5T3@Vvk+owZpXw7y>e!?Zi)JSu0LnigD@_joo5oDNxYYEQ>3 zuR~jP)0a;KYm+J5!TBsZIK~r>4PT#8bfyR1KY5O=whGc!^|{6DdW?jop*|TOVg{f| zo>(JUM<;gB~9Q-)gD--qc5Xl*;nfr7H{K)vGFq)9m{k@)@mG|_~Dq2vGN`*XDr*B zZwza?cMLC4&IjpOreAQG^(^p&FIdn9=~yt?@@O;Wy?km!}J$7MbQi6bT)ix_KZs^3V!rUOB{5uOo$94X! zysJh>Im_2NvxyjwZi?~fXUng=-;VxJepw+NF=ixw8G4P?{INzy$EK?lQ~AvY<6!vD z5}b}rFE{Pdc{C4(wew3FjZO6V1|5&+vkoyfp%}AJj9HA1M>M>+# zlrHYU3yEhS0pmKlI$fF!PT^tmW~4oSx*$3Ws#VBA9b(08o}=;je$;nEl%@K8mfM zv;Uv-cO^Xg_kV>O*caXX4enmV_{bP_}o*>c#nXU|{HreSI>?vpGv-0X9Z za&|iMd{krEHl(h&tc&dWM5R7u+sOKo;j%qsok|wJbL^=N?GLiPNWGy_Z?e9WdX{x9 zS%!&naHZ_B{7RjW`r@+uN_~pI%X18MPRhyUxk{bLdROXgxb*|xrr-m%Qrik;8AN|I zluOpZZ#%R0{OYU!3736IhKs**S?>SA-g|&aQM7Br-PPSy)lE*46(lH%0ToHR(_jJw zkszof5fM;O5Kw}I1rd;(Ge#tff=Jrk89=f~5JZ$97yv~PPzi$Y-8D0{`%JUz`=0MR z=fD2*UQk!vPgV6(xw^W0r=FXKmp59a#^4ObWBQ6+)gU8s)mLMz z%2DH|b(B$+&Vsyo=Wc$pF12Gu5Arn;uvqv{}S&d^uot3Jp28ykaIorcl3`QT75 z{Ov)-!@YMO2LmO{+ zH8R)99X?tsN2tkLEh1^ZTn^_uV2{Um?=3knR`8$JgUS+N8$N*JG&a)?=&2SGPxvo$7|- z9KL;43qRC-Gy)C*6@ZEb(oEGDEsd)RnOZqI{o}@9XkWrmxGS_uUgaO~2Cg^{HRIr~0M- z#n$nRS-C=dz>3srZ>WC7(p5gCr}R}mov!3hBl`L0Q&yzD(lA$a&(-C;wI^5jsX;4J z>ksXx_LnU8ACwi*ZJ=RYe!SkOP8Rh>vZ`0tugbV@0DkAj`IN<%2PCw4+b^A}w?O*J zXnN*c>w1+8Umg&73Ocbgg|YELx}TYzdGG7%O0!qn1*?;>W>#X&&@k1Rh`+;}eWPXM zIjlE&UsIh~cz%cSz7nZ&->A^nzf1^KEc9};AFCj%{WKxEPE;DMMZ@uNj;J(ec(gx{o7qZjI{DdNXzOxJJu~bbX_D z=&?GzqHUw&psvRLr6`>+bfWFaM#rZ*blh|)s?D@6szckU#zEJc+4o&lX*%@7YQB|T zKqq7Tsg7P7SU;*9s*^D0YJ!1@MIf1^@wq^7Y`Wb76)*Hy1p+o9T zn7crF#=NWYW7ksDJ(N*zuzanvpvbt;b8Wi%gBI0}j-SeNWm%(DPUjBE-`wVh$`*Q+ z+_Q~VIOA=YMmz9SWv*8#FjnvUcp-BmT~LRbHL0%2&E7Pq3V6>-t6NRuEUE zj|l&f-W;|7LH*n6wTv9?$gWC9*Q}yMlyzz0kDjj_Zt9cwWx1T?{tcPd8mo;cu%>~9GqULmb ztN#8p`&jZ)oC~Jmvbp>Fl|R0i+#+Q~4H{N?vl{1hfA#a99PxMZZ0EfsrPHVEQ<0)) z{tESI+~2Qz=y>wXkNs#m`s=H?CUvFKu*!?EN>=ku+MG(m&2shkzdCv`c@y5rlzvug zjMST^bmDUqy-&10X>(L3UT=idb=A1V+aD%vP8c`Rwi+}{+Mwq&`u7vkhUoa{^`rFT z_e+@cC8`sjJKBb;#xTAHXkQY>kj!ax4CC`f>(b+P)jA|?PAgTrzyE#tUz4lUDOvsF z(f$12islM^_@-NfhKrx<=a0d6-m}jxs7}M>@LOYpQ%>l}-uG%e2D^ZY+jE9453G{3 z1m&DWIZZ3(O(o^8reRXfYLYjNlu7eeCuP>4VNy=EZL5X0-H5h5Q?6PP=~pTZ)3(tt zDL>nG&c&I++>K-37LXAYSMv`UQgIcg=}K2!i|b_M4d_!HT24m#zf+ECi_%kdW&YFO z2l`!K%S;XZ>>)BxhSI&NoM>NZDMTlJP3NInx_*64$FtOlx8omm{++sXIdR>Dx^#Z6 z6PH=)1a!01(dEQ-v((Y$WTg|x|Brpwx^bCMm%bK>rRwp{QYWCBpriBuJ33l7F8{F{ zd{;NBMm4fnKM6YdTIG$5ey&NN9ohFQQ~w`z{8*WZI>;MOS1r_Y@NelPv^mzlSpLj1 z)!!_y>U*?bq|FIBnd#B?=yA@_dDJ?V*6W=IL-p|fO!uEV@P3iMM=!tQmfF#MMePx) zgdYmGP5&>wU&cg2UG?fQLMFV&|y$g6Gt z6ZC01+Mqco5_D)fvGI@E_v^W%=}3!S3yulVQTz#6>X5n=e}d);P-C1iCZx@ZKS8%S zVLgz(2YC-N${}N=boAJ&aw4jIs{OPas!w%jUbIKgV?sGJ9sSblLG>%6E}E{!Rb7jZ zVOBc9wU?no>I$yC3?0(u;M&X3A^lBQd$i4De6r=}pgLsk)EdjEi>8w~P0*p~WNoN* zlu-^%53I4Qbb{+JLxUU*w zNBD$0B1Cu|d>SS^59u^acpiKjCOi*mG`uyF^C69f3D1L1!-VI-r(weLkVM0T^C3Oo zlo{dS_f-gQsnj(>I3FT}^C3()AHsz5AxyX((h287GEFC(56Oh{A)W9!q!Z4EWWxCn zraFZ4A)RnOgbC+Egm6BD3FkvP;e3b?&WAAJe25UvhY;a>2ocVQ5aE0X5zdDQ;e1FZ zoDUJg`4A?Y4NC*?ohh)O}kW4rqLWJ`nL^vNpg!5rF z;e1FZoDZuA=R<^WKGYzb4`IUjkVH5iB82lHLhKNI!ub#;oDV+Xd`KpI4(WvR!6%## zVX8yyn|#9g5GI@t$%OO4C!7yH;d}@ao`-ZACY%q{KFcTeSw7)>h!D<)B*OWSPHdx6 z3FkwEa6W_y=R*?VdWV4al}awt_J91E0-X(i!UpxjC|2*(2D zXiB2bJHoM`Y*95=l214mDEE<1I2OW$V}bH1`E)-g+z4U9v5-!m5rktQjqdY=VNCKrtgP@V}Wvf`GjMEa)tSXVARG%3!m+TLa4bX!$3imUScnjgg%IIb zh!Bp2G{Uj4ns6*c2**N%a4e(|j)e%}SV$!t3t?i5TAgq#q!U}z2;o==6OM%l;aCV0 z{)9Bbu@E5~3&~W6a4bX!$3imURfy0q;ZH~>91F>WVMFvqKy6f46OILH zqnb)M79xaWf!d`;2*(1oON|hY1!|WXAsh=KVwajmI2Nc~YAWGah!DHfRKl?kCU&W- z3CBV@;aE`LwwhxhLO2$vy=sJTEQAQhLWFQEgb2q%gm5e*)4y>Mjsga`*hgm54vgbD9K8VwWPg9r^1 z-h(6>CL9Pp;XtTCI1qfofv}ozAfyuxgb3k4h!75hFyTN*BOC}m;Xv>S2f}K?f#4Gk zgb?9CNFy8wA;N)>N;nXb>2rs0ASiw|&FvE*90(!8fsjTx5F&&FA)Rm_LWEH-ku2IKnN2K zgb3k42ony32;o2o6HWu4a3Dko2SSK&AVjD>;W`Kr4ul9zCmaaLg!dqch6(RMgoX+4 zL4<}0??EyR6Apw3;Xnux4uoX->>wNnl%FR;I1mz^9rW)H^m#%57D2cU6wj^ZTU z+vfWyQ`_=~a1C~Iv3r8p@~bvy=&QDoH6JfG*5bG=e>6?os_61`pJFnGnK~J^{Aj6jszdDJBg8J=r#i$g zKHg^4%9Mz_(a=_u(s`uC|mHD&1`}_azqyM5EIvJnn*sRp;QFEAu9eTp}Q=LpZWj)5Kq^Qkx)D~LB z==jk2rDI6b>G-Ssv44qu)T!~lXN(~&C%f`#`=OuZ+6d&$&>{7z`OT;|ke-ztdi4I# zV;d|#-bbIrB(q+?Gq3~FPp;#!}kXBSr5+1X;l?p4{LXJsp`?9iia z()P2#w&`?LzP76k>SSk&ot^D=cJ|kC`)J*sEN!>reF+8I5bwJmOsDU7@%CSpH!BKNJzHC5YNt%YDsNWfOl^~uPIh+9|H_VA z<^5Oo;6eNDtkzi2b~`>tdTnHB10Jt8Tl;s_ws`xqwWB9(OQm6zH>-6L9Jj328|h2b zHatFev<(Ti@T#u(8lZhiu&F0=T0`k)#~Toz>*)T1wyAr{Cv8rmHrLcHn}(?^HVspo zacWCW?Zip>)oGZNQ=Q~ZB4ySf`m0HqsWeQ=Sxv*FoK#|mo<{nWU~5n6`Y*RrjeqQ) z{^koxeO<|nYtglw8P8HDI&b33xr`rH(EyuA8NfE+;FUz_l#fqjmgP>Hnysb>lLjJ^Gq1Co3I&P0N4M ziC@cdE_D73nVpSu=GscMgO1xc2kOYOuBc9?{y*v{J8tMF>L71CU5%F>-+xOd!{$13 zq+|Iq%S`N7OfN$x5^qmJIhlQlwnvXah7Prjg&l50+i_DnS(=`m%`DZS_PI2j*b6J$ z=Zw0tv)iRQ#LhRtW|`ELU?YsY8TQE8g-N~H+8WcmWK6QHi;QPNJ83yIOvY2SLECp{ zTRv$+n3hAsqz!7E|DAEj(#AY6Um4|)G0(OQ+0{$NJi$hr>XY(Ablyqcg!v-<^~v~z zvKw1+fBDzK(S6}RbsXaR9o_q`g~^)#Upzh;&(`?by%vwn>$P#XHV)A}?my_hXyePa zqeEA&+~6DUH%_{8cuB&`n zPs@>YWXME5B~|(Jj+_E_W~sQ0)vL-^^=H->t2eVfv36y)>UbOf&Pi{DE@K=welj{ks`gH(+2!ylLQZ{FQG)MQ0T|J2^zp%F_; z`pifPwfdx_KeJ1>45`{V?ATk;`8nBn3hKSVU$9%%ACqd_PwkkE{)gW_@$<-kESQ#|M*hZH?C3Tl=5%R8_VbY zJS`e8SfWLAT(`}9DJq8<9iy^g?UzEY&MfKInNTo8s{C}^H%>3Pnev`q zv2mtS&%c%zf4SaI8Cg0hDA!EF`dIgLQc!*yUk_TUcB^%wrCJvgsbiw)K{@i-DbaK#RsX+>uP-h2y3?}bX|--WT^T9-;+Nd~!oLe~VDGQUGKuB6h_>t9Qw zRqVddved_$qW43FtUNFlht%s*%PJq1^iw+in5^XwpI-0R+;}QED1-Nhj^DfcdjIXF zG8B|IE-&dDWpamtQnkNheEn*9YD{chYnjrwaI~LV-kDeN@0aqmJbYj5e$jGj{65lh zP13RGeH4^6Q)Bm6qSVhNeV-&g7gW7^-KqUwt-ILsOFu94^D9&8eLg7l^FxUot_I}*#$&(Ta7>!*HRYKiq6e|`sLa9mV8yXTR* z57qNZ??>4^msJ1NKBM;wy?+FsL;888`uXoZj}o6l`uSAK*UzKib4cHp`gs(54(WYE zKac2hNWI6X=aas_wA9ZXE%kFp%dF;~uAl6lLwX-j`StVYUp%eb5q%D+a@6xl z&zF|L=Y)<2pDQZA-Z!<>%palPMY znYfRX@^xJALrUuT%T5OO$6!3Tuj)8`j;rT7eeSFG0Ck-{*L7UK4``{<^>a>3{an;C z@%fkeo)&vvV&4nw*Lq*qQt#VZ>V00x;PXz+hkh<9sjdf~fAo2&^y&MBzVG!upruaF z?)^i*uV@*3{^_`WPHL&|A1&3sqTfffr0*Z3#Z^D_b0#J;_t)5WQ|$SaId7TI z8}iMCcCt9lS8YVsrny$9|^|k z^GWH`c<_0p<>OU!`ZR1H*{T zdrYcvRdFpFiHcJhuji`uspd_!L(BMh zs-$SSnKC-g#%0Xs6FY|By*p7VJw3lv#_HAm%#?b)sBueNZ?W~H%A@OumapbR#p$}# zak}0T*N^T`tRJy{rbO2}^2XPpmMK`bne!j(PvW{!`D69R)<bjQdx{7P5^)leX@D4qXnF>bkB^%TxC_ ze)mX2KPXRIZZth8Par+#rJ8P)`-3v&m>F$s85=R2f}&S~l+E`HS9c00V^EpMw?B&xshwVYAeDsK^A?bk}G zeW)(Vw?~hUN)EZFU{QbSo>5mx&WiaGzuN|-sz>Ell0i=XIA^qcEl;k+b>!2s)Q35u zK(xJ7s`?G&pAwg|RO6|n8ixly&*7h`KQ1c&fK=tH{6V=8`W)l(@~K#U zH4ke1gY%*1Ny|FW%a8ItJX^$9k*!S@XGx zp`Sh~9KONct420pXv9u42gE6-7C<}fP{Wum7ax+eh&o8 zi}ok$db2DiSRaW;%MX?ptWU>DzJz{c(Qe(2|M~W3@!l3~FVce&?H@M&bg~-%1PQ&o z6Of*T#Co}^oR@YSR_nqD%J2XDF`6Ef@%5MG`(;8q;_XuTKSDc$Qq^}D=|P!QyH$T` z{vxCMj+W}axcSP#=zVevWT63gE`D&}DmfqRi0Q>-|C>&R)IO3DmwU|G!SwZ8V?KQ*kAO{a0}{4t{)`R9x-Tdc2G-u0MXat;UazqZ+?UrGE{n^8Sv?=(>oOgZ&BemXA(Er5-0ej>|t0erN03ke0gN zdR)d#JQY&cwNz^p_nq3yuE}fTaBUs@$KMCB->1~?T3Mef&ivh5|1PF~7pK2_UH!XQ z;_qJScca)j<(SN@FIKO*PS1C#xUNr2`unE39*hTnAEohN`+{**9_`Q7@iy1a{a!ow zi+8lF&jso899jm?W$Cy+w-uE7+>DlL{n6i9vpaVaJfEY_>F9J-j=o=&RO?=!tI;y6 z^E672p1V=~*5{eD)aO04)aReHOg#5=?c6UtAFt1$1*QI-F(}nM>hoY)(sN+ydhi^X zJ};)FK6j@jJ!hx#>vMWRsn6vlO8wmU?>qNP_y6Gj9E_{>2KWD9obLZhKN$a~{XVPu z67{cszUk)_mHJ$mmcesjdcV-;zO>ZmzEr!foy$$M=b`hg&oyZoJlCYgFROD*YJTbY zC8ZZUuM|8laP3@f%--hOx!jojklt6dr2Fx;bGfl|y6V27?;q+tB4$UJ`20}&Vxm1` z_TLGKhri-s&e#mOw0dW=W-Lj=Y#K8!MN&g;`>%G{-3pvsQ%LLz2JA8{+`q6 z`W$~y>ht_ss&Uol`L$H%`BnP0bN&DIbN$!O)&5VNtNriYA7a0oUHv;JJ>RCz5$JgE zJex|V=ihWZc-~dV)$eGD=Lr+f3F_Yy@#o}>^Lt8PpGQ>b^xPpmPpHlnsO$eP=U3JF zx>z}hGWz=;%8#`#1@*+v7t(Wx*Uq8F#zp-U5IldU<7$5NJW-k5@3(s1uUibR0;C-vllWG~<2mZI67gg&@or8#`!Sjwf9y}MR*Y)Z?e6?w6Lj|5~c^ z3)jwTW}e#&p3~HEHBQ&gYhF9Asn2O@dF{OB|Lb{8buRPj_j~;wdzHlVE8*Osj^n*u zpEnH3|M~NW|J&Pt?fj%#XIY*9)aL-TRL_BH=O;7IkE-#wc7F2OIZO3HbM3t8f8RPV z4a?wgu8|WikC7j)kWmD#gi#8vj8Pu0f>9Cf4&yGk%7zKYqx$)ynx&#T6{AaU<0GEAq>Z;0<2;r8TS}>8tsAYjpvQ) zjp9ZlU?bx(!!x9j3`~xiir#HJhgI;LQOziA++Z{XHZ>kMav3>{n!uXIeMVX1M&k+K z6Gn3*pOM?RA9%mx_E9dd8zhRilRS0`LW+GeVt=7lAJ#*2(B-bOCljtfTR)(G}Pg zv1g68#!JAL5Nm5ZWpo2}L+mM|rO_SO9kG_ilSU6<55%4{ni($xUq-B%(ZqNK_zGf8 zjD|)}U{Ayv8jl#gfV~iV#CX`~4eX8B!$xhR53mnnwT%aiSAnl0_JC2#=nL$NSS|cn zpdYXwVj=XSGq5va-x%*0^MLb=k;VXHyfGFy));7n4QWmTPBVrW>x_KnOyEposIk$w z#T*YD54}OgMB@bTgz=&AnlZ`X=4m;;<+d}*vS{>GJ+@PmyM<5$#{0^c9yFEcIx zFQA@Jj1|T?;5p+jV>$Xf7C6>8i~5S2p94QPJ~N7&CCzQXZN}$DNwcWA6}Z*dhPGBT zw*a>on~aL)?dE3SW@C$SyLpSb5xCLVjP^ZZ{s{chIAYW{8=Bt(zc+p`9yK3EZy$yK z4q6+H=4>;3OSae8XH+-$!Y7+w8+(lRjj!Qrm?weC9gfI%7S?{Q>hQ;7`Ue;{o#_^EmLh@ssh8 zd7pU{c+@zKaV%@@0`4+)7-h}!=1$;FW0z6hyb&{39)2fAHPx`pmGFNXtBrKSG*`nj zGY#{7$w-4Y%>Bk8qcX6v`L1!$_|~|~JP3b}`K@sPbNem)U1r#b7`cGCOld}pwMGtL z4l|dz#&{j;Xbrq)E;4(V&j6n>+nbM>P0Z%N=H}z(ShJPc64=r#!6usB%$LlG@aN5` zxc)fsadVg1+kD+T0X$*8YBn>Qnm4j$@U6}F%`s*H_CEY!b0)5|2DUc0m~We@<|W`I zQ?f?p6!Qk4%}`YFo&2+%wlW^e7ZTne9PPb++e1dE6rETF2F9P!Md9( z%&*Mu@NLb$=6j~i`oi}&Up5z-oq(Op7tP_g(jVC0^jKrm)5~lO{~O9rHzUA^In*3x zt})jE*O_m?twX=xfNz5O-ZI|=zG;36_og`uILq7wHw!)41iuAsoncM~PB*u~O*f|j zr5nnuF3H4UGp8t zcTCPW@NLMqF?Yi;rn!N+*+_GQnTzFPBg}kkv^mPm%L=ejnD;T}2WAKJI`#qlhvs6l z5G%qKn?=}1=10I{kj2;%tgaI5dN$U)o{cld0ZT)cW)sZu<_+vdHXduKDZ)+7Fv4N> zEt_HFH_w?fjB{pJ^OSMYI1W5+{%U@K`=So}0{$ho0rz1OU=!AqePx8qcY*J+UM#OU z+URNKHG7)Bo4**Rj9-Djnx`=1P0Us74fxO5dhEfCfQ?uaRuH`^9A#FDzYS#@SHvVflDY83u^TKgi&)qhtm~}B*dY{NXBDt= zn>o!=EH`{f)&OPfHXFe2!`||tSOyxP?!A4u z^X6j@^{n~uuGI{?_h~$*PO}L1+j*w6=E8I9XJdlV$NX7&^9{V;3}A1-f6QLPv$7(v zBD;fqgLk-(fFH3GHUqsZX3c>AjopEDmxi%O!$`b~=aFH(3xAmnz&lR`UM-YW~a~gHL1A4c9CVEY4bBWZy9_VRyd7{$g+9`E?F>j{V7cuvgeSz<1bN>>2hn zI|Dq!{$S5SD*S_;W#!lt>^Sf^Th2Nl?JV#tJI^|^=h+3|1$L1=$!=k(z*P1Vdl6|D zffw1|tSh9#zu6Twl0C&v0Z*}_*2}CrV?buvR&Vw)Q)pYldYawN)}S40(2BmSHxpK0 z_?*^!)OnuGhribv!d_!}fO)L^)-X1NDa>ybvWBx^Okp9bu=PBf!EOQGVqIWkP-jtV z4195G5!!NvErPFNjmMSZz~a{R)1BB z4=iupY|UWPn8KT_TdYZJDXR&rX_d8nlwAc_#oEDc!k8@wE@vw-IybY`z||O`L2NOr zZViIZZQagpW9jH~I(oc>^<%lLCGfsg3HesCO7Nc|XA;{E+>YG0u}9e<;34F=8`nPt ze#$nm%8&{-uuZHQtHQPbx3F!Q1*$&_iwu{wf_pvX4U$8G(T}XvrvVH75 zl(7l8iQQ?{XAiS)Sbg{dsI`6+51{5MY&a`yRe|5a-ehmH+pIU?D_W1S#_T)x82k^c z8KlA=*ip=MU6kA%o>@;I?I_mMQLLsGST8>Te`3F&ruSJT>tXnBSSzIc0{n%Y#ENMZ z#gkYyjoCbQx78T_I~K-w(H!6$R*!ALTYO7kOV*lg!Z(1z)~pR%hwr7iz`3ju+lFtT zw!pTm9eWM$Nb7;?**f+)z9rfL+p+enKi=IR2R_cqvcY(#+6dgp*0ay>?a?0Co^@m) z^9S>GD+K>8-a?0(TY+2IX0{VwSI+^TW1ZMWe4E`0yp=U)-H>Hv09^f9flkLE}{tLhtSQmB}-!^T4ZP*CBL5()Q0)EAI;|*$z z`8DuswwJw*_v<^X*Wov^ea2p+C$J~$!?xmk?+)M{tS#Q4#+e6!2ibnM2j4WW0$*kQ z*)V)xbp&=~3)mdfu}o`@XTXa7=wX>*-&=KJYvo<55dod#$@wHHVOWFyz#wnTGkl&uh{{7j|>G4WpA+U z_!gQ8oXMVJlktvv40w$F$iBl@=?LHmHi`|#w@?>g7q*a1#ryLK;0g9K`@#6$7!4fF z# z7{2AE1E;f@?5Hu^SO#3iP=nRl8f;C4pKZNp^|HnQ$5^$k=2lbdLEwYd5UZ9o&gu>9 zZPi1{)7CSuc+G z>p1YZwbD9d{bd~m9<@HT=2@q#^T6}gSJt1_PHPizll2`^7Fdgbi>$fUeCq>iF>tZ9 z&{|?GgJp8!8WY^AlzS`J)pEwff5&m!O=YXx#F#`QESVyywLu~HFQYpnyW zL+n#)J>;kGYZ2ODZ3J#aY?HMaaufUpgi@io8Ga-DI^b^9{{ip=YY*H9XxSe4FRi_3 z^E}`@^yO=`Wf^dpb=mqG?OY06YT*xu_EO|F>?M|Er&-@v2hgemXiWro5O@$RSqnS_ zJcO2ghgNI=Zmg0Z*aTjjczm&w-y?JFJJS(bkW^AFT^keWdLG?yz=QuUT(c zKLLNTj#-act*noLA6Z{mldQ4U5#SN)TdRfju4URS;6Jp6TO+JrfxlWmTklzKTUUTr zEM~uL&9p8eei5+|*6Y@9z~7)X9%XzB{MP#3nq^J1_5t@a2Jf1q`Bt(UEIV7j#xwLV~-1D-?8 zk6I5~p8-EZ?{nEL?WgTr@Fi^5=5`KX4m&Sg4m&3>r=8or-L7QU1=h8j*h#i;KMZ`> zu5VYdtJ%$f&Fm-bGIj^MJzN31Jg_|Ct&lS(oMX3uYlC=B$X0e9)K&sm!Y*n{NQFi1 zg0`?dyAZIDUDz&Vm$ci$bwD|7;Y*?1e0Bw31(cl+S4#m)A$FZz%)SG7hg}h&;`Uv@ zyAUgG7qKe?D0{17HKYkzL(RwjTjLVn1$|wQsWT1KwvpVBc(CZ#M)swC}fXv2U{*0~_0q z*>~D^+YbRBvg_D4+U4vAfe+e`+V|ME+D(B??b`M;_VeiR^Y*jA=5{;#IpA~lQ+8{+ zBj%?yd`qO*c79-fTpwV+V7~->$$ryLwZF9w!u?<$gWFNRZjk*7+^cqfxL&x{8CO*LD~Jz;?`*$o@3wcr?X|y# zTWl|an~BtSk)~1>A=K6GZXbX?9p)JaBT#x zsPxW=Pl6wT9!|7B1%8TNE=Jk=;JTxnuJ(My_dzbU$J@j0Pk^7;8`1M)_K$Eq>~3)D z5&sc#3w#fI2izC-J8*9!z5}wGJ8q0GZ_H5v6dx-tI{hmDoIKv)@aqnsO1NO5& zvOl!@+I@h1?2qlGc5nM-;LG+x%*aZ6EpV+p${uO2vBSWyJ=Pv$r=x#k;D=)@Cfd`0 z)9kHq)9fk0DfZ{~HhVHgVH^BLlr<1_z6`$*J>7#D+hczT{LvHZr~58Q8m z1w3FM!3sKJe+T@|J_P*UK8~?GZvO=Q$vz6K?=*FKJN4naJ58Kl>{E6V=al`bbHe`B ze$08ydD3YH_lVON?op(kK&nbh!_Os48 zdzf?D{=;tPv~!+!`or}_ydC6edw}zt(*dr9(*~}G^AcPiX8>G>Xv#|nz3e>Yw1)fL zJ_+|C;;o&N_RG#i+jK757wyaTdE0h`lgqK;J%>4$YzOE#Ih?=kzib!iI^4NnU$Juo zb34*89LvcG%<1HD3OGfb>wwoed7To@4Nf^=Ip;>Fm{ZUx2`uT9aSA#4ozlS4&h<_{ zr?68VSl%h+6mg0>Wr1a#o199{tL7k6 zWIg90=YFR;u)0&lxx=a8gn%KZmQ%xdz^MtW=_EV%ICY)cz}ik#$9HaVZiBnmsRoyX z_-&9r#_k!6&@)aeU^}N1MzoXD9@xcs1ta>3(+$|i8HDj30tY%zIFCCi znDfWsn>t;c22LYjBd3?s(`ks+&=bBp=CY&H8Q9r*!x`?pfO#DbKMZr;(rF89>kM&T zcb>+qzYagx`PpuOQhv6(I=|WDoYBrmxHV2X+-he9+%jhg++t@T++629xVN2IaMPVB za1)(za3iDoE25eUqdK#qSI0%ummnn_eu^{3S?g>7Zg569Y0eqvB=DrO(wXNRbiM|D z?aX$@JDZ$!z;(`8C*u6+oCcnDQl0tEx6T3J0cVag$=T{`25xpHI3t|(&L6-(oUpUV z`OY~EJnSrRraGTHp8-E}COaQGKRVw7zjr=xW;(l^?ZEBMG-s)E+&Kz7>U`w9E4&&jX+56OCLv4}YEKf-lA!@v8h);H!K&uZy%| zz+!wVe}uQ^5nzPB!(DzK9}OJM3-YJW^t&>APSp7@uMe!xr^D6f^?>#GOt^aJ$xQgEXzN3~ z4zLbi2v>*K2G-^uz}3d+d;mWm?Q6r^0o(Cy{4@RxZwqY8cku1}Y4mnG{1#~M5TA?M z{B!XjZ_3m7D&Q*qIDdk#=4F9pc@vEB?Yt7O5`Ph{62B98Cx3}|<#(W0UEw=p9Pi@~ z03YDv;U3`k1MlY(;qJ%$O@tqdac|E%0z2|g;X3jTzz%!^TnCKo2KcoY#|FG1upxgB zt|5OE_$YrL?orI#`|$5zRQvKlz(Kq^uf_-R6krNZ;x+gH%y|v?s=PTb&6fd}@m9Pg z|ClcUF5#`=mSApM!?(a18_GulNAeKdNIo1moY&+&e*^2thtI=r5a01{f#339`EI_K ze+~Saf59y=Mzl07_*SM6oy1GPm+*UL4)K84%X7f*=DqRzWoywJ{yhJZf5Rtkw+wnZ+IT~FL?p+s^}sLz~3%*^3%Knu!HbKIq|5d1*|1$No<0s*-7k~49 z;t&2euAbq$cw2EAc$)9z7x-WNE|h#wQ0b2VxJ4I-&xw0@_(yJR)ua z-Xv}o_0ivQz;dFzxQ}lYn}M6f1H70hE}js@;G2pPqNF&=OTa%VT8ozA9B&POp0^ZF zi?hJ9{2YHuv=zI6yZE0N$7h8rI>37fJtrhkBK92SP5=dBUBnCGHsEa-{};qdq9U*& zVlQEAZw20pSWoe?C~agmC2<$OBwFCE`c$kDHTfD5;wyMw z`J-6De-w@R99~m25Oa6~v6)Ze9Ys4a2_;X!ow!DXp&J&-*x_fh`C^bUU)+v6J5{Vi zKURtwxZ_ueU@RxSXAd9iz>W0-_MgparkN2WtWJ>;$FU3 z+{35vYGR-{g%8BD3F5D%r$hYP7;%#{nJK%gV7xm8- zmG~ZBLEeJ#zD0hCUEwYHs{EC|DsMpU7eyanAMrL{&%27wVm(S;haGdScpv%R7kBbE z`77dg;P1Qy?x?czCXDM%ayRb0Qu0Q*6C=5cui@Q9SFr{qtjFCsL)MUIcn$eC-^lBV z1Hc3PX58iP$kFmQK3d+26*^u{l(&hA@^4RIiEg}- z=*cUJv2whuh+MacJp3)O7PwaU{1!1vzA0}JZ^~czY~D-s5VKL*ySyiI-GhAhh#uHs zM$56Nb1d@bh>_UO|kIqhtk?a0`0hNAv>rLjD=Nil`=LV6>-TWu3yxDg@t& z58;!=e&Bx0Pzj7iUr_?SAMXe357}S5#%J*wqKcS>5@zu0_&%(x>)?y<3j82H3_Q%Q z=Y#nGF-Q#NgG7Hm4YOPW^IQXY%J3igao}-&3!YN%iD}{vK27A{{h|3fbYB;J_yQgh zHN^s~z&ZRTF-#7ZH;Li$X587GlL#e5#_i2HC)+=o2n#OrdXEGLG_@?tB0 zNYn+^6&v~On89Z;htJ9aVi+GS#sbHRQM`;8C|{Fh#A~uFc8UISpe&6P>=C#+0XRX7 z;Ggq);vw-lO54gyiN11xEF}iW8?i&Tm6fghqOvu>90ASA(48!X@Xfg6>fxTNhdd=j z5BahzAzqdx@r1fXw#2$_Df3{beMxqg#YA^mTs+5Hi9Y6YypK7-$jh&nXGC6pMm&bM zt9Qj)A{SOs9=yfoPZbS`NJ*DeeH?f%R8NJSRKJLKxxegvZ|$ zGl4U)hEjPm(OjfrBvn>wL&X*<8Eq-`>CnSA*LHG%+bKn7{>w_)eacf4k$ODXd~Ood>EPh;+V)U z3(8}np!`Yf6;H`la<6D5_lSMsY5A1ghm^fyswgd+$f=@<{7if!o{>+>Z*X;=_*S%+ z?c}$jojfSEiN|DlxlNRpv&0e6QMQ*yaP?d9y?9=Bl;0!eh`1sK%NFvAXd%B4$3++U zf;=u>kVnNYqMPg@e?iJ|u?Mx^iu!Mr?}*<-PuWfWhO57b`S|_v89ARlgSsz>ezK3e zAo|F&;-W~A{p3ZYTo8YYK{7@Djg*VxAX+sE?V2QWOD2cN!N9?&yRe)tpO=NPWE zu5K4!V6{An_3|X*8&LAy)&_C6HIF6Z$v6i%hYd06@LghusDn58+W0EiEp~~AkWvTl z-0Q_$R*TPNLyda;EAgeM$G;SH`B{FOyhEPlcgS=2y7*drCFZ6 zIb5xT>y_kre2wfEUyFyKS)ZrlZDAVT7pBR_@s@bEtSm3^%JMI~ogWeh#H0Lxcm&@b zpNlu~J@F>)?Z#rV%qts*|euvfDe4PI( zeio1OpG8xABYi2}MPJ{=-eie~WF2XVI?@(R_-Szxcv3upFS`Y7zZhohN8h^2qN1?q zDhrFw(5fTr0qaSQx3Smc-Pm{UMn7Be^WqQDivJ;+b0O->22zLyQi|66FL4fdPPF8O zMN#n>N_tE*!W(4+*$~$oN*8Z_56BnM-WO#NybWCve*ynOo<}fZdE_HJk8ICfXf}p! zW9f;fxgjqDFQfmtvCDlYa${r~;LD<^Y=*t9nanAk;g&SzGu)JIc`NfS>mk|7d&dhje8J?#gF*H}f8=zU*e!M}PA3Vv@`J z$khg4B`sxZJO^6KyrLt|C3DD*$kl=8MM@i_wZR_hp|z#Z-copXX^&pyLO*gr?^%?5 z1IoTZ=HoxfKjoM5C-~*^Mf`qw7XeWgqVrpi@v zg-n-WU|6n}%VZklGWc|a*2+(TpCYzgM&x?ndbv)nL7r4#s@#AaVO-xNKa-yWKbPAO z+97uWcOte$Zin0gzXPE!b0u2*Bzn`-`U!1WhyJdUC*&_^XBsd~{w7ZWmjRc_GiXm2{D!$ncDFu5 zdwSqE%+KUY)(*6#8-Bywff3q*k=Tkhdy>Mt&b~AI52o{8}Ca9>h4Ul7}z@tKgT)vaaji?v{nG<{q*N zx)p&H-M_30@+$qyVs!w*FhG6FA7%}If}xoG73X$H-5LgEO)~*cMpEQ{7deEzbyCS_sa{w3utp$ zwwJ zyu1f^k9!wF7v#Obdl6eAOS#truXh*Ahw%I5t?onc-^zvZMz@r^5LcJTJ@Pj94tI~d z!~I4+hTktu_c8cS?|v-rcYl}r<(=-`?tXc2A zf5=+yX?Yr$44LfyDi7dlWn8c9?v>xm3T`>~dz5xmo|GZCjC)d+aetOS1FJ$-b7dbI{s_a{RcJ3SQ0Jj}{2e+F$$V~yJxG%dexMSRrz>#hjccwebeG;yvTflx2ekO80>`s82 z?2dzT@GIs7$eeaPw;}4R2Vd8H$bAH{hv4hE0>58205)*zxKrIGZZo(iQED^zsVK3f z`xNjgHy?g&d}Jq<)NSiN3w+jX1oy1l5!lf!j$a`=x}6|9xkc~`W+!MBvD-qg zk^7?i67VJDZ3658>=7-qhuagfXEbk5Xcf0#gx)LeKzB57wAJD`W0|&d0yItK; z?rXr;-0p63w~yNk*voC@wsw2FLx4lv$J{pVaCaDRnEQ;|-tF)91@?8HbDwZucV7j* z>b7v7cl)^`fFsn^{>0tm z&UHV5f6p!874lBIr`=!O%WzlRl3q!#m{$PqbTp+TLhrcmyNBF&;6HG~?n%VL@b9?G z-P!KDaEsk#a7)}pa4X$(xYf9}99LBOBE*-$FUOyKTyLMd&;7#v4(EZE( zRF#qkq5R%_cMjZd?isl25T65?-`nnPao56aa@WH}qV>*pKZD!pZiibNO@9~h_3+!# zv$^gC;01T9n;Z4~fcn0TmOLNv9}vpzZFVoYC)~~OYuw`K_Z9U2FLxf?0>rOC7WW)) zK{REr`xTr-d>*9Z?FR0L`^r5C_m%sldl)qwgZsh#8SV%7d-oT#{7<+u?s>Q~?kV>o zYJgz|%EpiHz`rpItK0*?1DKOlZkqcI@EgQZ-4*UG;4b%TjQdgdW8lXa{~z6-+>d}C zxl7&S?h$t(aH0DlX5>@%9Pk|GXPvvz{R8-id)D26zMTM`aDT^G7~T%x4vdN6S>ETs z&k3Zs)>*auRJ&xJR0Rh@OpSNymx@_c>TOrz4tu)h6(?R-PijSzgeEJdwPA~UiSLK z4fOiLov>9(e}vB1XYGZ+h2CIqkhjSD5cr`t)*I!0W7t zdZ+CUU^`7 zuYy<6yUVK#tnA(4-QwNt-3GkPE9;f>ZucqyD|t71rM)}7dx7_QH+eUERlIwE_jtE@ zHN2``2pIBec@KCGdyfDg@oIbbdG)*pfe(5~Ub6R)R~J~Ua%+4e(z~)$|_q z>I3V0_j^xz&Af)dhF%lzF|U=^4cN_V?mgkP@Vdiw^u?*QNN=6O@Sx4quL z-rg(TTUZZ0fIYlkn9C*JMBqfs=~B$;DBvh>lDE)X>mtI$YR`z^(Dt!maZ@gTHN!T@g)>ASD%knYY>7U{>c&HEf~ySD>wr?(643vV~v9`8%Iz1~-F z`@FB={!e?~9cD%Gtld4+1Gr?6BvC*GL6n?!&#Zuoh#3>0VirX~e;}x!D;N<`QBgn< z0TCkziVEzRRWT<_7*K)=iV2LUh^Ff^9<@@*Z9!t2hBM~plxmUzu;9gDj48yeTSWl&reUfbgM z44*JxJ3jY_5}5L8iF~WS(Z}Y?m~FdsYJMy+mt|73I8!ddTy~O|`1VV>6b&8F+v=D2 z^|!ke-PAM{lRXljqq-IyKd@&^HkXB|yv!%e*OGrE=Df(C5?^>#*P{Dt_P73*_^!=e zik_}Kz}A<<(cfK*KB{_%trv-(AJV1hwP>)_Bk_J;b}qVOkF%`2#6Miux#*JA`L-S< zK5jtgqIuzkwjRsEsANkT2Dv#B3YYi^N<;sym61SNJjNC?Gb0RUfFO)HnnA;%9ODy_|L}G65@Mj{i z=qr-4Fl^573G=lmR|zco4V$m)H!`-tKP51i1wPD_i!hg+z({BF~mvAeMrVOsu#ACFqe_)PGa^6$w1OP0NK z8NYurF5K3oP5PcjW8-%>?~&eqZiH! zBa>Uk62CV0n$)ikpBIm6v>+w%i<@ssZMg9=`>r>^PfNbglSmnnZz5%cor#o@d{M4M zV#u=}=-Ld1{xr&!pj?z!2E(2-%9Ws8X^CNf8s$pxXb~m&DajXlVksl?B_DELCF(-T zh;n&lO6keeFZ7hxBkEN6|NmJ%QvFjLQw;o1!oKz*;;=-Ul6(#IoH<412iWp#Cs} zW&a@f)E{R06QQR>+=_fl^ijgj5`CH|SBd)i7d`Bs5_Jc8?n6q{o$xK$$^I{|M>j6` z>hXV4Um?pv{h%s;uWk8zZOh}Jyd9MH1OE;Cfe=qKR7frHG{vTPmSJP0-gx3+O*~Jr z8d4`b2~p#zhs}{%;c1F{<7t3Rkow^1hqdtp#TrOm@RUY_CmjYz``V{IwgzpD=Oxy{ z^AUT1_P{d&@$Wl$3S@-T2G5ABi>Ec#LF$PoH&((E5-TDdfTt$b$CCu>A?4wjk~w&q z;~q!{;%Se2;kk?rkYYTgv8w6_+7Zu!Y>4Mf_5$sNXA#o#Cp&|7#&aY4slIqBsoQWqR zjzPK-Po}&LPv5x!=^{L>u>eoL8H{uqp0GFzPr0}PX&j!VI0a8b9ENl;p6+-Zo(^(4 z(hxie@-8(MbSj<$c?O;bawF)C_Sukk*W8Q?PHJpWXHl74IO-%=#j^{w0 zqi)7C3(rA14^M)ef&X0^g>)I7l{pDdMLHL0sG6f@tGPH94S3r~+w&&+tK3_}n z{kpZx-{Y1%DQ|A+V(!~zcW5^eMRzLe0JUzvNi6H z&(%&?m-?c4g?QtdHK`>}SB!^+Yg65q{*t24BuC%hdgqoD@&Bvx70?&ZAGLRbXTRcK zQ_*ti39!m?sEy-sjcA2c?DJGB+g40ZU!}h)XtkBms+GF)fc8Z%Ja=~d>wxNcuP!*S z=<@MnES@o=OVONx_2NYXtLMc{I~VDS2U`5v@A$pywK)Fpr0RKpkLggf`S#;2{^Rij zi(2B^)X@6z#ckUcO^42Qz%!e4E?QIJz_=IiTboyc|FLAT8o{s-+Qd|~^H*cI5>lcgp5|dBHxO^pU zIbm+xplPwZ4^F(2jBorgJyTxBWOIuTi|a^YDxX^<10Re^$*%){H2w>vfOao7{W!9Lpyes>jxsFSgfmU6TyqS~H)DB_=(@ zzk1};vBae3(gz=kD_nU`?89UmT}x$`nDpnJKR<4K;@vi{vM}Yf>i7Ba!mFp+e0PUQ zKgXWLWLv}OGi~fiO!~V{ERI`=$+iJQ@3ZngOnP>_IxC}-G3Cp#Eivi2zryT{&ayDs z(0p^Tj1N=3?H4VI8@amUwm|w{{CH6uxw>OadUpJ}D8AIy9b?k7*SM$Sc26v{^}=n9 zY-UV4!`VyXxD7tT-Wib_=TFqamDS9^c1}DRD62<;<#c= zdJ1|x9nbvuDXWJu>3P1+)A70k7g^i5U6Y>E8!d^idS#KdhcW57|Fb3W*e;8$9>!e8 z9}bz3DI>KnF4y3uGfH4CSF>Kh+&_?Qj7bmqxoGv>_FD4o zH4RH*(xcH=e6*uD2Dsm&Yulo)cxZcZ{T87QsrGkCOxITG^ibT+#Ub~lbnTT@AByX{ zIAlyd>HX8Z_<9$Ij7iV)mIyYYGVvn&HuM5R^T`0!uLNQ(!it)NojMs%?yiUY; zorv)|5#x0t#_L3k*NGUfcfqdTw*1l<{cVO{7IyulmGNQMk69TXc727Fk=XT1mM^hu z%a$*(>nki@Vi#AIFR_bN%a_>IyXE_^i!m$X!>)~68HuTk=tC@DV%P30Ut-s;EuS&P zC)rH(LNV;d>x5#`jn@gqtsAec4An2j>x8c5x+WRI7_Y57W70zySRp$a%@XXI^7s;b(V$69yeZFd&6jG#-zuM*H#a= zHL{s8>2zbX)ybH2x-r`7WK25Ucx`oZyCXerjJA3hlO8ulTRn_Pj~lP89&Xp9$Bofe z4`b5f#%QaDF_#hJb*7Be*0@|4uS;Mq7slvJxd@XUH(pyk+6ir#-zuM(N+&* z(&NVKOt~;#m%yacjn|p7V7xAYNsk+^?X}!LP~9;mJ#M_V*Shh#1SUOhytaC{-=k~Y zcx~m~cwGY1wQjt&*K%J<*Say<`h+q0#EsEb4`b5f#%rsG`%lv2#%n9@#^@55u65(J zy_WlD@&jYisLK64Z85Y02{iXD7 zWD>~u^#7}#Yu*_b51$k5axL-byVChyggr3NVZEg^pLq0LX@1VED1BcUdZ0TgE8k1U zUz=`x*l6fc>Q~b_LE1&m9R{-m@$uDrYW)7ig^yen@B{>&CR>4JhW$y-y7 zNXkFrJnw|T6lPEc@$%)DIQSDQW8lIWFdHHwgPScm=QCX=xR9+jL2~uv6)EPz_sK2&#wH){*DSs4lz~sx46|JsHU-8ceC9Zwv#jBDpPkb(~N^ZkKsdL-jIq3!84~Xh-REq7j*KD&ZW{b@#(kRd&!gLIJ|(m}c;CY_{@^pZ}} zM|w#o=_9>T=Bn=Fl4i$F%{wj*QXjXim|ptA_Eg;$2gUpE-7T53`pMLc3VX%vKOCLh zbQIo&-ubjt_0Mih4_)wk>h9Sml#n4kq=R&k9@0U&NDt{CU8IL}kS@|gI!Kq45oKI1 z%9R#27uFdzI=Sn}@u|vP_b3$QDj_3$o)$h&gwGS<^R)1J2^mqYLQ$@?#C4YrOl~@N zW9lw*U0ReYEy|S^ezw;-KPSTH3F(236m(fV&d-Uci{$CaXortfP9HGjf!sc?R7_ud z-<8QFHN(_%^9Cp3OqJBDJ1Os{^*5wvO&^)8+#aTS-_bRBv{w7%>+uWnoKKdEIJ7b3 zd{QXluu#OIjT6I%6#S**n zrbXUK#i}5HW#_N^QvW)+)#tz?D<{Q_{(%Vb=vuX#%r!$8n1cYLgO{hTWGvai!nNp zSd7Jqi#-~ztxnR1c^=PUXiQJGc8cOEf9J>F;Q<|VKJD=zQdM#Oc-?RIT&4Zr_I$(q*e-2PJ@pId%i z(|%w2%<`qrrRDhZW0y3DEB*6k9-l9-RWGiAeE3}9)CTd0!`9~U`KG@0<8?o7&g1hf zi>k*3kms|_cgLG~e7>Sx)%eEEoAXMagRgSpA$`~8l|D~=sje*-pC`3%5EmRVC6~{2 zf2$X(J&*0d=d-Ifi1UxAmCI)?_rndVmhfNeFDu*6Z`G_7KU-~G#*d?G*NQ8x{3egj z>|Z`VGq5_YE9iLeDK+AP##M4B>{l}`>Ru(6{m1A1_e6bktJ1Udx$CFJ_2fIVy?kEM zsMu~k&w9By4*0ytm810eS!<_Kd}h1&yk%AmTmI7L;PqnrN}n5iS2tey;JQp4y!=ny zc+3f#^7woP;$+q>KjrcHyyNS~$83E)6Gs$JH4s-LH`b5O|Kp9!`BLx~{{3bipEoY9 z7q|N9^GrO|3+l&3w{Fbi^IwP6kJWk=`DDaP{@Oe~ zzgE9~TyXC6O#Ez^kN7=fYA&A#AimVGQ*!y->EXI@(ZGti9Dn;fgLr&yLQg)Avhn$8 z7e3$oHRAE{6LQZ&92U4Zr1JrzU|0PPxrbB+e|)XX`K;lP-_a=7#&3X#PZ+mrBrv3Ffs7n0DVXO0A|2`)kaq!yA`InQc#eHA@ z+_v8W(T*!Gs~Q)K*qF!X5eHU{7owi{T;4wx_mkW&@|pWZK9}wn`E%)hkw0@>@|pWZ zK6AgwXYLpI%>7Jx|5$STEZr~i=hFQme=hGIi^pm17x~QnBA?6q$KrmH`$ay}o=J_J zleC+15A2Q%K|}1Sq}`DbXrv5wGdIV+?ESE(x+RXP*hO6nJmRzj+8evGyMcDYKH^T; z(|Zq&`><2!?6$gNbG_>0>^Q7=kqCchjVSqKLVU1kp^H# zb#JREzGN8z{zdy{X# zUgsFcV9Otc&%=>oSeA$V*PW31V?TFI?Doz_IsxTrh~3oHkh)>%xkUD{QU_Jkc(v4gx8Qhn@!J`fT&B8|nq^J&11YA80?=cs$D32ec3Nil2o2^nF45VjuWG?5@uT%}05! zfuF{Jj=}ElYq5X)3eYRC%lkU)BfkvvGT3kfzP}dqTI}YY2zou}_1Ld{GrpeyIsxUn z65me*orpc@<3J~YPQt$OtMUDfpf|!NQxF>ufIfiz<9A}8`5e$W>Tc{KpA8>OMY;nL z_h6s+eMk>LVmkJQ&qAUd;WOZ~*+}<8;uh=^zXxeL4tk64bfkMBaU1+S18Eu}U^sT6 zpNVuXcAH;fcb<>Lp7`^y4}Apa2>5b1qGcH9Fi2g3z48}>UJS|qU{Cx7pcg=DC?e-{ z(9d|-=5Elt?SCgsg=cyqbwkNJp{8#^x)nzk z)by=Lx1;2pQRBBEO-6KIiCTz}`rv;O#-SE^AsvSQgrNTfI24KgPoV`Qs~~m6e)_rC zLH{VyBiLa-4}0n#M|uo3aj#u$!&bPrM&XYhdsjm7n z(9f{LJ{RvO{#bpizQCI;SL02)2jj@KoYi=D^fP$B_8&O5tG{skt~TTNMg4^12lWh& zKP-PUKL3UE4Dws7ig3_-!e3Bn^&XB@>H{1r@!b-9L;Uyf`30mU*!H~|Z?ODQHO0G> z+u)ti4RBP@wQ*F{=;-v9iueuixebznU0c*vSi2SPsQgO(sg{8*!#lkfp)6W2LQ3!k z(*)-ik*eu;R5QH+@6Ozyp2B|`Jc~D?FGTtXIWNaL?h>RYRa@N*a&3{`$6HQcQ62PK zNO^cO^Z|HF`F=M#1t;d_H zci@fL+mOD-8?ygIY5zgmr1sIR@Mi3NkQxB(kGEd8LTZS&OYe((TO&1w$J&A025BFl zgCNxosTJPBd;rRJAX00*DZDwjO_AE@y>&}S?Tyq9Z~ktk+k&=*To=5T`4G@U5E zXMBqBZ3oZ}x)bU?2XAq&fK*%Wfp=Ti2d%H`;PF_M;Ni+h_3%wyym`JhQdN9YU)KVy zrK{tcJyGNJkZR(a8af0G5i`~CE#3}}6yTd`T7znB;JHS~tpQR6c&-U@YlwvRZNqyN zK`SCZ1r2KurS#s{020{LqrTzr;6V!btl=>skk5KntqQ;ve;+vyBHmdJbOVF06k8csZ%|M%> zJ{BTQI)HZ2uc*sTZl3?u(8=+Abq$RgT9=f?j{*Nyg^Ce!=d!PK=X=AL6 z#H?rAL3_tf%x<3F>CB07UeyNi`DZrIe{Ah7aigY2U z{zSbb>=VLVmeQE*A-=?{2R0|7UJ_9+C9tTMMAS>h2GS$yq68N8l8Cw}f!Ti2BeAH9 zL}F1Fi4SvmNvFi3E)t1FT_h5Vx=4KZ$@$+DE`r$4-|@YTmh#UB=2 zn||`h3kuJuqvGq@oSJm&Fg&;5lpy{7m*bMlrf@Sky}_%OdKf1YgukO3I6RDIxE(S=37jJ)&Mp=n?f&Lf%&{ zY1;w$_2V0BJ)L@aLKS=F&wOS+pNT*4-j%6M3+u!SuV0mFhtJpF{(j~>>5saWpHsVf z`~<$I>rRIJ`EOLUpZQGuYjNGTP7j}1Kc7iojozzLeEzn1{o?CNok<_r#rbjmnezk1 z`Q^AX*--_0OP@=Zn?LiJ^q2P6|73qu##%&8tUJ`i{dNWPqakRBHG|0A`R{G-{A=K@ zdrzz;&`Lyr?;?Bu-^kvJ)4D)p|MRl}XalTY)P&9)&>Y;M*8!~rTFL&0_a4w*AE`ES zY>2z=%1Bjl=idx>*VT}!<7f;W)sd>;E<6u+N(Ugd#+`d-`+w;4|LFO+KPz4h>4ZD8 zZnz`sg4GII`{<4D^6?$jMjZ{xW|{MWF|JZyX8Q8AC@ zO{!dynTNesVPpx+^9hn?dq@w@AxNI*8Kg&I9*^l-iFy1cJreVHOgep-$8yR`VxDJ^ zPKkMrLN-Xu^9<7C!#qbJdn9H*&^(NNTY9d>Hk6*Lae3vnusJgigPn<}m(1J?Hk81k zUJ|y4^oV-N%=u8p5?ItlX3j+UvTdYCVo?{FITP6<`Jyf|^DWAk{XjY;7Il%CJCY5e zF1%ROMP?pJ_GB~mXCouWbvfS4vD`N<%K3zxXUI8(Z_M_M&AzeRXM-G@edD^6@y$7W za}=NbQm34+WaIN}WJJBBMZJ`m`wE|wm>-LJNkqMrm~)GAm6(@{dP!s%MZNr&=dz++ zN?a@Ti+U*`BkHAuJwE%TPEjvO>G=qM=5y(J3VmL3CorXqf&s)4j; ziTPYT%<3v&URMD&&^)pRXbtFVh1u7>p!;IZ)e^HY`h|p^1Jn)kt~?yQFn@|Mi=tn* zI$M4oK6gWEX`eQf3!00$7R|$YgZ9Qu?m*1J@w;fjAuj=!2wC$0^9e0b9Slj66Af0ZH`AEs;Hedsg2>;BT9Ll+okEW9xpMvB!s}KmSXnTt2+= z%WWC|_;9}`|H=5-hv{0bFCQiwxZZu3%EIyE!{if=LmwW$acR6y#g6gvMfb!$OmV(- za2G4%!#|(p_%PW1TMLn~StA3kf)ocK+|^D@}%!*p${ zp@Xz=yMDA7W*EnEZC~5#6ke51;VcI~hGbym90w zaa(7P58w08x=guzc;bf};(eWee0XHR*BL+iFkSnct1llW8@S$mn99QOXI$%X?_ zzr>fAV@r*m}7(VNKEm-<&~IZmOB4P%=J!sB<8v% zzQkO=Y(LdIVHz)}Pdj4U?OEY3K72YJ*y%XipZM^&Q=W?7J7rMZblKE68%%!Q*zzQ+ z^WX41%oD7gkk1CMdtr#RfpqQ;^PGY7XM<<1cqqPNO>b+n57+o|pSABSRtI1Y|Vlo>{WnYQ;8~b5*n9BZ8^?7zY@?k1_?IyEQ zWJ5NX=Y$*&G*|pLOfk^c|Ix?VkPWtN0rSs(+%9&5sV($BB(OI4Ftvpd z{a?-_KV*ZckGc)>Te2Y=O#RZGt8dSv`HT-!zx3?sHLVTVVCqBEqWhLw8?wYFKQX}C zK-c>4tezLp?bc%JJYeE5rh{>=0b z*^ggdskO1TIAMqe7M1t)r$^w>p4FB@X9(xd${!+A6~J(LD5FHp5w#g`!+0E>DF_6 zn6CZ6t>^eK+0e+X=lC#{rN3Lx@nQ1GLbsmd!{l4Fpm;sUi)Vb7u%5XdT%R?)_I2wy zK1}}q!>#A|@KKLcD(dCdb9{Ke;Z=%OyY(C&p8aXHBDx=>^(7y!v#@&6J8nJ4hw0k= z-Fk%&w}L(2xb++#rm}Q&>nA=;ew*jkb9|WmcA8ty@!_5?|D3QMAAaP}O-To5j}QO8 ze0{>@@?o>_vxL{Xe7IrH)d~CAhw0j7uD*PjZ1~L8yAM-YIDUM%JN(13>cbqvbnh=Q z$1U+CraJA7xFx>C9G}FOm}8Ro5_239Ut*3umZz8`%<)zllMS8RdW*yyS0pbn#}e@+ z=J+AL#N=CwXW~oDaYB5FsZLuvnU3I`Xw2Oc|4^# zKsI>Qr4@<>x%C_$uGGF_(fe*aCmT$D{>H86{2PAAt>=(THkj9NNPc&i=M3bBZ19*X zs}}Km%!gAOsul5kEE`O70Q%j5Y{&**R-<;&1h<~!!{-HcitcyoIoV*I^Np4D5G4O)BZ_Nf%TR6tuzh{G~Ega+4b9|WE z!dQ3zo(-mcsmQJ8WP_<+y4S7e_%QWLE8TidHkkSlwXk?SCtEzit>=)u53Ahb^&G2n zcNp_e_5;m7cZa7x{X@cj@Zrwgzfaf?*7I!D$!u_o zk6zErKYe(=N-tX(>L;_oe9y&xpnI`@!<dZoKeeDm#xC*p9tAz9%Icvcc3R@LbM^sZZd!TsD~2rQQCV#EW_49yqJDdc=EobR>p&Q&7Wj^nAhY<#)oN*l*{GAw3f^M@nKqX zW3yASi4D%HCW^O`5g_%QEh!1_tz!xZOke~yj$vM}#$AU!_J`y5Ed zhk4Hf$@nnue;^qj=6w-MogP}dD2;iK1o3^C%5o6ye@VuNdA|h7_%Qj9_UDj{5A!|> zlJQ~M!$A9mNXCbGe+9|-Fz>Y>86W0-7vyIj<~mh58vih^ZVoy%u!?gE+)^o^)Y%tGBsEpZQu_q_-VcwHN zHe`c&&Oc@PT_T-QZ*^mw9IUDw4Wae-G zhNi>v{&cfFwY4o-)u1N$srqjnD^vRzS&@E3w?{%bL{+ccbM8j zKdk4F4L(e5f$rbQhHNnPQL~EIb8H{AJIs4>$Oa##eyIrSIb=gNnEDX6o@4uyY%uM~ zAsgsgA1=cF9I}CAc86)s$z{(5^PU{C!H4NLclJXznC8YD1KD8Od%*GF!?gEcn2U#O zFyB*9yT}Ifo*eRn5A&WJs*7we-?NYn*Li_i~1|Q}(Xw2PU6G-yCL02 z`!N5eMlwE3zj3&I8i^0{Z&IYkhj~v9t!el$|Gq;qK1?<=aqBrgOl6_{Fr>$a`8Nxa z@nO6E|3vY6ju*Q%f6~M2|D`dlJ+eJMOlzcEE+6JKZPM?2Fz>;jy6|D%n?W)@%zHCP#)o;I2Fds^$1trMNX#)ze2F<$i7zq7 zC-Eibm?XZ$9EZf0m}8ISDJBVXyp_geLl?K+A~BU^zFW_cn94H5t>;Kg@x$^GlW%GN z59KQ{#|iNz=GY*<#2gRAmzZOK!h|{8gpF}Ut+Fb;!DiqDb0nl!D3HN;=}ZN z_XpUYLvw&^F!|Z-&#|%O!}Plk*-7%*;Pq}jhvavMdCoxkv%z9dPU6F~_uyc+o|6rx zIRO3cKsIE9#h#qRhiMPSEbPyry2u9eoQG`42Gd>?_j_yN!@MVlY{&-loQiD72GgD# z?pJ-7_T-QZ*^mw9IUC)tWrKN74)y;&%zJX^ek~i!b3(Eq8%#0K4*PS+1|OyvSb=>A zWJ5NX+Jf7UVcW&-Fz?AB8+@4e>Gh zwwU(hkPRg7!_BZKhio93-D1qo*$*`T+#TjUIb?$m^PU{CAsb9{V~&ArFz?AB8+@4d z9*lA8IoV*or=YsX2Gbr4ZWlgGdoXC<8u=j`%=av0LpGTAZ`NC6azeNW`o6^oWzGI26)`e z2J<~5`5_xD_T<=h;r8m3z`Q4i{E!Xids4C?8!YzZ*#5!o)hU6gPvE&+HW-f$a(@L$ zt^Jy%CVVp1K8qJP&6p(=-hV9luU`61`jPr)TX`uX`R-kryU0jBbR@V&yfcxs6(o`m zTdaLM^C^~*e23GeWt<+%_sPh2dP@17x(difDzAM$v6PoG&OcU0%9oNMKU*0o?~@_l zaK2JT@?lTr9h*`{@}VOpn|6Lr$oe_Mhg~t*R9eRIQ^fblIQ>$_;+?t*=$)fvr;P_G zFJ)Z0tc;YGGO#fdt5RObP&qPwmcH`Ic|)9zVyD@9|-rg`Zi^zxmnv zcsG9La{Zg1*@sqUH-5G<&W9FD|42Ucl+|z87$hIjb+FyW%1#}G3aDHeK4cTJskDsb z!=GjK8|3E<-{G`JkLCMhoIMF$>+;=2k1Ma0k@7q9Djbvu+OE8({Jle%8)F5CY@5o*>Cw$M)F;`tc=9YKYVQ_ zmYi=nuQKDr)e)DgbPPzotw$$sG0IEw7Q4I>DwmXTwpqTEarRrjl=*UN?ex-VL*v0e zy_k|^?6z!G`t#+-#ougsF(u1O`4#^a#Lu2Is&L*7&!^rTKP&?F%~dSf|tvJt<*lrtg*drTtQmytb6iOxdM9vb?9}?Ux|M&uxA&CF@t( zEXy_I=80*f)HU#fk1z9*`X!cmp&k-klcKti@gr@J`Jx`u$lKP1k1z9*`X!cmEt@tp z`3iM#D&oY&p^a5(gUlCon5H_Q*zoaXUQ&MkaGPb`GLm^^208CnU~b>>WIf^iieo;k@BvN zb{Vr=z1#8H)r-y7$Cr6Y{VrB{zC`W9+92g!tXjVGgKHPN^%KjN{oYQw^7S0Vv-OW}J|M?UkDqNHMlnr( z_4t|BVQk;-`q*9idB?YH!6)PNOBstrow7Z#uxA$;*|)fIS$V0)&08#A%KKzI{e~S^ zsXTO@>>s2a$tPcuO*_9QRKA`=eI-1Z` zvxj|7_E_xfw^+)!`vsl@WX4vE(dFwos29wa%GYz?2h5+#*K>CIf$!I-{n~biuiRfj ziS-lQAJMuN->=a+Te|c*5AF+Sz0Tf;IzLU8(Zp6DaK&~Unbrs2%>nG*wImN%V{Wo6Ep?j{(nv$D;#yk51LYd!@E#J*ccaf2NIj`QS z>wn*UnJl~Wv+akSpDp&u$bMVONIv{rW*;v3==*8?c9;2&mT`J4-zVejN$6TBzf)JHTvkTPOBoq^&JR|e zY^S_Q*7<)IeyM9Sb3Ejg=@TS|ofJQ|ToNN68Uwhzgj1BSir<4u z>*4)5WM^icAoWZ8r5<;mLTgs`-kNQe*GhY2dBu88TGpwwS(eM)N7L_pC43_JGB2rL zVwsn_f2ZF-Def|{ByEuTQ4h3kK=b6p$Cr6Y{SwQ(#Ci_3iA>x|8)UxlG5MN)Z%urB znU~aG2B!6m68XxuAZ?KOI!tRRiI4Bg%h^V2EbMdJ?wq`|$MU5OuDo==%=uaye0-Ug z)bHwu?vXk6Z2Oh+vVV{^xO%7iW%j?d$H$j>N&PNX>0X}u0c(Slm;H>i!L^HB^7Zj$ zUQ)knr?fYQ`zULJl$U*)w88aJv|hpO(b^;VQopoK>UVtz+d%z~wLxAh``A)8@E#cM z)2uzNkFr?m_t_?8Tp!4O;QcvNN15^0X9M*O6pIv}J{zPA{E*QvvCIqg#rtz8Rx{(i zv_a}b9P<7gs+WvCk}va;`X!cmA#Qnpjvb#VCU)jesDS#zOup~|?}zd7WnNOh#At(= z`3%}G@6VzBDl?ziX-5II$4tKHmuO!K_YYVwLvia2vEXNO>2lmM?8^@o&p3^Y!s%UQ)m7AM89w&XJ_NYqPu9;Q9fc z%klmk?jI~)>X)`j{i0lU{!VpIJUKTjC1cwO@u9ae5w>6GB~zj_}O`#C0mc=`>J*>s3zqVHW)84T1^_=29iq}%8PqzJ->!WsAYvLY?=0$X`Om$Gc zo|F0Ct`h4f?mljpH6?eyW@FXeuUS7!?8Z%77w*22W3_xer+EE2GoO($S-ze_-pH(L z*>w!LjwIJrBwwzdl&|O5{W__@_wMaA1Ln!%KL%xexSS`DDMZ#`+@R) zpu8U_?+42Jf%1N!ydNm<2g>__@_wMaA1Ln!%KL%xexSS`DDMZ#`+@R)pu8U_?+42J zf%1N!ydS8bLfaK|S6y&)R0rWWK<$rXKh+Y)-l_?XhN?b}I;s|q>Z%HkJyZo8%3gJl zm2YV!>sy&B_S-I&-vpffkt(Ru)fwtG(A(6l>R5G@+6cN){iKGfb5%+YMLJv$0PiO? z0O@Zv4Bs5ChanBnCxQ33Iti(w?xk|oT-6I{sTvNc0eU#n>3Sf(uLxRE*V3^XrA86e zkE%-{b-KP3=>mPK8mMZ4*3yl2Uv-FjN%ciq3F%AK1$q=EJw%bfknyR5b0sTb1qiU)>)dtWF>Qhx))l*wQx2P{wUA33m z2D(lCs2Zros)BBS^rvd1nyH$)5t7kORdcl$=w7jRYO3 z&(l4WRtr=Qq)T-_b)=f0`XODTE>j1qD?zVRGu4^uY&8XRin?9(SI4O9K(ABd)j8@s zbtmYZ>JD|ZI$qrXdV{)Fov$uXQ$eSyyVP;&M0FGBP3n4;ukzFw&@t)?b)mXgO#_{# z?p7zLQ`F6%H>(>}O7&J_LC31g)kW$OH3M{px<{R?2B}G)lhj1jM;)rJ0=-I&QzO(! zbsy+`YPuS%PE(UXC#zf3Vd`*I09v4~R{v3>)%~FNt9#Xh>T&fH=u_$;HD5ig7J@ER zv(z#*PdyC!uqsrEdPL0zovoIrm(_BW22HD1)$6KAJq!A*dS1P&UQw@szNX$+AF4Oh zi=Z#6H`QnAJ@ouLr@jFFLTy(6sGrrZpuZ}m1O2=DK?V8;^_SX1Z&y1&cc?15s@D1sRTU}Hd+JI$ z2Q)|5)Ae;_9fF3shHj#3>pGxybThrTuB)qoR?`i1Yu!{g2W_s~=mT^M-4L{)ZmB!z z{dHT=wz`Y%rrYU#LHE@M>R5NyT|v9*gLQA+UAG5quY2etbiVEd+Djj)kJf#3E@-Yk zRG+B(>HeVo^~rjmK1v?~dWb$wpP^6CCxM=%&(i1UQ}i*Q$LLe_Fg-+{33{f!SdY+W z>p`G{^!a+UzEEETdXc_DU#W-db3xD5|Iv@B%k&t~F?x|&tS;A==*8+1{RDcJSUn2* zD0-Li`f5EE$J2Tdjs^NL91rVxIOgbCIPTNaaonx%#BsZxjN@i~BaZ9zcpPKxRgYQu zSyu8+D|4g$c9G>z2WKAAc)ePGuB!yAktzpo>DTp{pl1db1sn7idMW7A;H6-teoJ2j zdQmVY_(pHgFM)y^ysua4F`#3D2|-#vtDAu~3%Uf~>2LH((3QdJV3mGfPXL_|Ob(Xm zpY%^egZ+XZ^>=zT=;~l&@QGffCxcE7rUfhX^ZH=WgMBB(}5AuVFdXhc?^n~Du;1+$GJ_GcO z;KbloeTN1RNn37!b%>qqqp&=tXR!6W(!{SN3m!HdD;`bqsM z=%>M3!9x9%-T=BG_&8XspV3=Dw*+4XOY}J12DDA^WAL(mP4CbzBW>3|>q5O1bgSMg zSfkhJYM|AEs==%JO|3z7@Q+@vztlB?^+?r&H}pF?1Pucfe5E()TA;Opn!(%pJsp8Y zK@cq0&*@+Fa-`q&*ZNysC-@qvcJQwLP*()47?@zGPV^soDbjCxlm0>13pOF`8GN8W z(v?6f1r>rK{eu2W7a{$rzt>xIgW!9ly1~c#Q=J2v6YLSZs9(~5>lcx>>CO5V-7wgU zR6p1|XdN5`dQi|JXc@E*+Jd$X8UhoD!`E$9K-Bj^(x z8sr9@KsyC_!I44l;1JM5f}?|DgTsOtG!FU(CkICb$ABIa3=9Sb#|8aB`vu1bX9uSQ zgFpub=LY8orv?K+2LwZc;lWwKIiTkRmjwR_&I?WhJuSE}xFWbX7y&vW7#my_TpA1o z9U5E~To+s(TnTz*aAR;&FfJGgIx@H>xGlIgxE}QS;Ev$V;D+F8(5r(~VbTMayUCe1=|L|9Q(;w;Z@a(X^ zIoOl)4pFEU+` zE($LQ&on2Q3m|!3ctTjm zeMt32$`1#HZA>#W2;ZIztJmP0Ly>yp>OxqQhtwlH4W(# z^qa7HSS{Qb{sQ_-SSzd<{uKTL`cGIV+%x<;+{5gNR6nd2Ry4Ity|9+qD{L6}>K)%P`;U7w#Lz<_NQIc!b$MY#knM zPBg8<6U_l(+i-w6!?X?0Fb9PPhNqih=D={6=@@nhFE9^*J{0~L^a?KzuMB%39THv~ zP7VJ9`k%0SI54~^oCrEG931uuuL#G1jtl#Sr-nC&lRzhhr-z4yW5cUKuMUq0hlIC; zlR+nkXNLX4tHNtQuL+L|yM>pABS1%lox`)jTf^HyZx7E6j|>aK37`|gW5X`tCE-ZW zkzvp9oba~r4$wQoq2V#%`0!fLYs2Hi+;CKQ8R%tUDm*`&65a)RS2!#j5MCEv4|;ug zVi<>`!!e*^!h^#L!#l&fLGKQShxdi|hmV0i7Csox3hxQ$gU$~h4;P0|hYx^05Y7yr z37-h3gH8`;hv~2=Tmrfzd@6h)d@fuFx-fh;d@Xz-`qFbUrZSA@$zmxV8eAB3yI zH$mSFUk^VH-w9s^eK~wPTo-;3egyha_+j`}_*u9TbY=KS_+9uzxE^$UxHjAzejR=a z`f2!O_G9{7Di@!xCL}e_-9zfR5yXCf}~6h zQ^{-#YaoTDuBmUTf>t$^O#@TYQ~<4Ds+qk_3$qvKUZ$Se$22mvL2H|armbmbT7tGT z&CNk(Khp%XiP_h5HQmjDpa+@*Oi$Cnv<7W$+MC|y5Yq#+hv{YxGbz&vw3Ep-N10;PBkZ(zMy^0vF2=Zt{DhA(41_}H$%+vpvRlR zX1KZ3oCA7}Im`UVTxd=MJ64FTXzClGq?;8_+XkIesL<6Ij%)sac^SLRE7Db<%MbW3`RWl+wCwdj% zzGT*!SEIsc9lrhCykV}4MnrGm+gHsJ(=MtXEiv_@h2~51e)MYeB_!9Gx6Sp@mC@Uf ze8Vg@O`}cGVzVjQWHy>L(fiRxNPcPFHMd9CNAE)NZL{3;iMmJ2P50&#KnDbaQ2l;}otgE=WWJG#M~9o=junlq!}(L^&mnrv<{7e!Y@x0oxU z+s$oeOmtmzo4GE!)7)VuM3bXC%;adQx!c?pO^fa})1ql+x|tEpji#Hq(Y@wAGcQ^Y z-Deg=_nTSf$!K{r%Pf!PnAv7o^hz|_yb{eb51N;v_o4^Qd(p#YzF8T49?dtOM~|6D z&Fbiz=uz`cw7@)JHbyO?CrpcIlu4W4%vR8?=6Un2sTFMo-E5vVt4tJCh*p^j(Z{C9 z{BHgP{nM;4pO}hKrRWn=DOzn_G=G?FpxewV<}1njm1krnDnfb;13i_)lG+&z95Tt)d1RYF=*pxuc%|Re^eK=Zqy{|5#>f5Ks!VS zMX9KB)E2aD)G0b7IxNZq&5L?Qhe!EQSJ19e932xK7xe}08yy-Ah>nPQf%b~}MT4W$ zqT@l2kB*IoL?=i6LHkE1MCV0Aqf>dr0CjcH0bE)>S#)IXLJ+j zP0-9dvp$ zJ6aq)9X$&AX!LOOO!P$bAn1e9<53blA1wi05uqIC3j z^m6nZ=yTBv(TCAT(Ho#|M6X4kMDIo`KvzU>MeCw3qE(=)q7S04qR*mtK;MZzj=qb2 zjMjs$kJd(;qpzb+K|hVYjG9IJM&E;eAAK9`7d4MIfNqF>K#x*CY6{vEeaHU+01b4~ literal 0 HcmV?d00001 diff --git a/pufferlib/resources/gpudrive/WhiteCar.glb b/pufferlib/resources/gpudrive/WhiteCar.glb new file mode 100644 index 0000000000000000000000000000000000000000..4475db8575849118f4b5f2d8d4c970e9c66053d2 GIT binary patch literal 239892 zcmeEu1$0z7)PIsUxa(qtQe=@*+UZp3ycBn5afbqBv2F2EDDExp?(SNcmm-V1!@}Y& zi!KiTn>X{|4rP#YzVDpx{LeBcH^1a1d3njr%}stl+YWWgF+xb@r3$%|myk-et5=Ki zXb~A1*3~08%A;*qr?7|?T{}m31bb9&7ty&>=ScUqP{qAehpMgA5U*4iVukw@hUwVHKo(KRe0yhVq| zW?>z}BO}ARhc%1r8WGm1ZP#`lO*}m!TftjBg1sFm5~@XcbZXHN@;qwm&pbUkb#5Id zA9y#3h8jjH%A;dgWIMg8qkQQW5zw!Q&Rtt{)jQV6+q0ODr?;oY%QwKs$2Tz09Oxh5 z8xZ6hR?It4Z|hU%^w+lT+^uDYu-f6RW$(L2bPI!**?Ur}Y#C^|OUsC=VO?8v=n&qj zR_AVQ+jRR=RVC&JpEWw9<#Kk+&DT$>Q(t9c1?L4)PE5HhV%=V1T#3 zui4)cXn`D`z^`Tbzz248yu)Y>#wNme-KRV+|3DufZ=axmfIzdK540H_?HT3a`U}ce zDOtPrPkw{9*&_Q3L+0b{2c5P+kAE!h$3MgGJN+(Qt9sR+?6xJy-`CKDRZ7+=`*RI@nFHaEzrU})CCJY&NZ0xw$@(7{cgI34Ri$L< z%0Jm@Z(nnOCCJ+nO}XZE(ph52K7{{#ItX8Du8czeO%$v*-9K7N5d7Bg6|AIXxI z(Qd5jRj62}?EivZe5I?t-ufROpCB`sy1*cRIdtAWU@**<0E^iSy_bg2o(q`)K>_A~ z0I+^RX7lF`gLVC1=&oZl>Qt;%_NNV>x5Y2OKgb6G4I9(BsrFI~NQtuoELJ%Y_(Gr~Ky4sRu^_`=&9J2KRfdxon9EuJ1mgMQ8p z`r`%zaDzVIYB2C~gF(&>`anlMchLv>|GAIQd)IyRF@J8;2R;pdwvWZBQTT?>jrxAG zQGe7Ze4bOIfliIWcYJOX-siffX4u7z;e;ja)Tr5MhRjY=YM(ciBi&mh#k)0##OQ2=9 zHf_Qp>V}8)(ABf>mf`e!F@&jiH zjF4}jw{&TOES})G`kI3Re8F^C0)xO~^YroZ0WZpBX&PueH!s@0v&5vZZ~&RR)(4eFBW) z_dnrV4Zzs`OatabJ%9;G+<=~`527zvK~E1y4{Db#S*2vHL`{GuCT;@Fg9TdX*g{T& zj~6(4S<#42giY+&(mP`0;fqX2XDFa`{)N6STL}Rf_!}af?$RD2l?vj zA#oF*havHLfG&RjBfiyu4~+BAw2*iv?w)-rsL6Ov5(87U3^X?J(f}<{%hIIgEh;fna`M>j<=1!2ZiDUsw|07Q(R`4#;qF z_&gEN0?dIAER_J54-0sZ`b0n$c*Fj%I$%QNJbbN%&%=;-QTTk-IG~C6tdC#cWWHk> zKs$_4g!*5ZhOe$lG!4*|Z_k3?5e5HZUgOGSJNMAScAbeg7@&mqE|2qc3x&Bus;j4=gO@iO|SHpMK z<4?~)kY9iWE?;~tfv`nbyg$3GKRO3rY2ce<_*H}O<8$Eq9dq!#Igp2s#5(X(b0Akr z;wCKazX&J+auz;!P4QD#+d(?(}3 zq~p*cqOBw^d|?aLFAD6f{YY&`gFo6rpuhCK@$1e0-htB2{9qRXEYkhQ4S>6*dy%LC z{a(V=H^8--a|2*$&2rmD4SaJD5;x!r9{f)<0J{1`5B%(o@DKJNAP80*9s=-)zU;ua zhQJs6G8l5F7T~lY_38(E@Rb$--58xFvIL;3vnx0?xK`1K(1{uZ)48zkY4?gErwSW8iQ3nFaZkF$e&T z$@vq$H3mNL!C#uC0JsVHVi>!n<#>j1`U7M|c4i*Mm&k`ChUqptAt%U1IWD|)-cmb-uPEFV|%xTqoAv-AOe z%(|5ag$?JK%Ae-vC$8q{#YmoR=51ah%{=}H<|1A3hrGy!#eCoFVZ8an8=Tc!#-As1 z#zueSI1fs7m(T3FTpwqQTaw}l#yXX2O^&B5-{Gt|L5|^A7i7goG+tI z0~>R61nL&5d_$G_Ut(Ei@1Mi?vL%*;5tDDKt0SCoE9mE8+hJ9f$#=%AW7*E+73PFT zn@;QHWxi}DcX+H`#=x?!e6P&MSk{&AI@5lX-p@;rqT8vNd0U(4ec6-+oaX^cMC zovV8#)W{X1dzxrcFc?@=7%n5%^oJhzVK z<=|T|-k9~*gypqjdBX+6IL6CL7D-s0X%s)u7RJW~{&P^V1XJG_&Y^CvMV<+db`910 zd7!jsg0gccUk`N`*7Qo~3;mb#xPM8(1U7k$j&twyO!)KWP#uRoFPw1U;pjv-OQCF` zuY%_4`EO6>3GJO}oQ{)!%o+Nj?CeB%+L7F$P2;EP`51Sh1w!Q<$v!l$?G|e2Ve&Wm zLJb|mxJ{9~p@vRkY}A$dV6-W*%$M!UeJ5w(yrEKmWPT&aH*_iPqkE{KKYQT4vb@wW zi49$y_tqoS&`DX>&?VWXp<}WgLw}4hk?%6bO5)qUW(#e9VXWSt=0|gfcHA;mpO=b& zOQag7zt?iiEwoCLsa)!RQW)oPMQ7Xl^FC*2h6=Ow@}8%1hE6O$Q!g_M@+jA=Z_l~;NhjK}>u zoUdwNFQ2vfcwTXoy&a>j{Pr>{C-Io;uetT#W==SR+gQG>?FW9KN)wLpy|{t=w`4Rl zv|%HT@r}E~dHM1*bj*t?yioFyJhDFt?RK(}6TX~vG~a&VJts@UbbDy{HikbV^+DQL znJ@hijHO+b`O*);xYa`M1nSXN*MTjMizE#0+E&*k_j|tY<5DQ{>P_+7F;hTkRYVtlxgTWAUJ$K+bW zIOJ)*(4mko>sIi{8`>7I{9cSR7hbL!Yd`z+f}il5?tMSuD`N|)#u%$GE$r4>Es>zs)v^{|H{;>8l>@xREc- zIg0-X4w*1m9X)j;ua#pIe-}4UeOG7&Um6_C9}F$1{$6deKED{xGcQ*MFR)!F)i^I9rf0~-S?+d=JON2{Y(a|4ibCx*zEV0oyiU0AKAU2&? z%DtLJ@imrVBD~)cPM!_r+fql17qiy#dCv#)Usf#=OCHbC?bXoSr})px^)mgAW)P8QpX=ED+C)Ap&W;~fD6XzOxr}>19?CHep1E4x zU1T}G_h=}`I7uB}HRp--{LJ0K9An=_Yt?bTZs2Qf59S!BZn0F%y0n1ro!5Z7!ULx+ z5UnyS<}DWV;;!(j>`TOxZVP#pm<}A{(}(+ums4l)wWs4a#wW7R7M=IZeYlT{(hK>YQ{uQQ zTs-St-f9B)^R4T0jL-dZn=kiR!N*n$=NRwbe}k`|yOP(O)SbJ+#=ab5F2XU++|N(k z&-{S@>R+F`!k))6ir>{Yd{n1q+!dZ+OC@qI`@m0}sKzm#|9fsRWb#Y?S9_IX+}^X0 z$RGcd4=7cgV;r_7qexct4sU#_7st3_#raxL_ZR$Oi3sir$6dIgd9QfKa}5pW7_SQ? z;-90BcyFaU$GGRoW7_1E&-ktqksRZVevA3KfA8@*p$)hzZ1^W{b{65Tu&6ekyN7?^ zt^BHUj6d9&!`s|{!r!E;!7*-IWd?uAUh);GILEkU!4CY);1|5d!U&FW>I)NjvqiW0 zM2{%$3LkGVhMQNs;yFAzaEyPOF@!(g_>MO}5Y92q-?%^Dvg{cze>jq3yy5*QesIzw zzV=Rcj`3#tMy)k{E?;&cj$>SR)M|BPofSL@>@Poy*Ol(C_V<{}^Pi047!Q1KQjLE! zk0+ZE$1xtWe2Y5juSL9T(m3u4m-)50+VTBzUbRFl{}F5%Iy5A1`5Zn2`rryr^6#&9 zbMzDAETyxlo?RC42Fc>ME8J|uc5Cm-^Y|h77L1RiA8O4I^@1;K9Kl`T)j4abh4NnK zuhnSo3g_J1MEzsUJN|k>ILG+Fmb7Zaq|f-wGm+dCP8d;2Jyh)hk2=wVyTT)_TRQgja&PUuUHYzF}@vtSWQ>!0Uvay2gi76$_Hv( z%4a;qvq+9{-Yttn`&6@efkh2?(OEM@+fDO%iBZwKcGHEzx^gy;9oT^%T0U3IZ#kDY zUfjzG_d3vD1g@RQ@Aezc-!2^?=DnN8$7~(ugxgLTF7hRx&CgXD&M_{#CV}VQK96^b z8qQo4#L%IKxeLSJo0$!rba3@@$@*)09<0X9SO7N2rZZr5IKkT-QH(A}A zKOb?6zuvHrziL~TUm13l_q@NDpPAa-3IF4JlUJ&?h_|m5&L{Nn6lq7_=C?iTbHlHY zwtZy2JR;Qmh+nf-=Q~*v7VqrYfOnoSnP=Ycl>eKY z^Y{KU__a9?`IRI!_>Y9yMFZv-|(~z&mKTTl4K8g z+O6TddDCGb$9B%=7d!X9< zp76S>+j6%X}gfuc3PWzcsuPy1-qQEN|QeN&FG+XN*2zTpI3nj6Ps& z+#Sif#=VWi3qIrwonLOY-af`>;a+R^)~R|v#>V}RtZUqFNc`UVL~GcHhDvPQOGrHH z$#CAcmOWqc1#*sLKE}o!xXj1cxKEJzXP0^>Y=ip&iH&ppm)JP7%Y2N@j`Od~p9SYN zqYq~t=Vh5++;Jw7*fjg=M=#@8n@-r6M_6|PW5h%0p9&%fNlRK)jC=M!b(eypKST)u@C5>KB~BReBLZW@l?cR@2*i~*VZ@IJ zj1AvF_5pDgP8jhN0%OEe2#gVDArNQbgb`<;uM#rw|w;&O#V|j;xC@;wc2ih_eugr*OiEj}VBD5c00x7&pX6 z2*gL|WeooTaS#Gy#779kM+n462*gL|Z5sO#;vfXZh=ULqBMw4u$JoCRA0ZGQp_egi zJK`Y3k6^?>h#$d-j}VBD5Qu|t!ia+qt}x;w1mYl^FybSG;p@uvh&Tu*j5r8^G2$Qu z#)yv)h>viI5!WCP*B}trAP~PG5WgT0U*LohUm!3>e1X6i@dW~7#1{xx81V(d6-Io4 zz!>ob0%OD%2v->K1;Q0Ze1UL<5nmu&VZ;{*R~YdH0%OD%2v->K1;Q0Ze1UL<5nmuM zMtp(581V(d6*kUjaK9ulMtp&Ag%MvMTw%l)2#gV5ATUOJfxsB?1p;Hl7YJ7v@dW~7 z#1{yR5nmuMMtp&Ag^jZ<+{Fo381V%HW5gE-j1gZTFh+cVz!>ob!WBk*fxsB?1p;Hl z7YK|IUm!3>e1X6i@de^%G2#mZ#)vNv7$d$wxWb4p5I=$uUm#pz#1{yR5nmu&VZ;{* zj1gZTTw%l)2v->K1p;Hl7YJ7v@dd&aMtp(581V(d6-Io4z!>ob0%OD%2#gVTAP{#T z5O*LDci@B(cOVdV;DixhATUPUfk51WK-_^7Mtp(57;y&zaR&l%2TmAq2Lf>i!nkvj zwjFT?0&xezxO0>C3ULPlaR&l%2Lf>i0&xcdaR&l%2Lf>i0&xcdaR&l%2Lf>i0&xcd zaR*KqaR&nN1x^@o2Lf>i0&xcdaR&l%2Lf>i0&xcdaR*Kq@dW~7#2pC49SFo7IAP;H zQ=WYgcOVd7kO({Y0zKcr#vQ5Aj&a68e8H!?N4Pt6!p5DI%s1|cBt{&86Gps%z}UEJ zl64VBAP`63gb_y|5J&Lo4n($tcmXGjcmaX2ac3gy8ut|vBVND>BVIsYY}|Xuy2hP@ z#E2sh#yx<{M;yVYJ7k$}+$T#c?;aiZ_DtGx)V36ikzK34^xyesJ&~eB0 zO~h70obZ4vPi!Z0@3&R|w~uI6GrKsjvkZSzxs6)$c&HYl2f*V0>>@oMKi z)qCEl#K8Umd}!axPPkZums;f+rTNIK7gUUw3@NJ>-%^GD5|@Fy!V?Ru)h14?&OMhr zQGW!tTJqW!YrCnAT+otZ96x8Fc79e$zI0JJezs_BUaCY|(P7$Y>x?dWwEE>YsP)Tq z;239qGfg}FG&L_?xI8cTsHnEENgsWT$DVD+uO2+5&(EBPO|{$?d+GUdUT4DGrJQ4* zJB%SugS1`_X?TUj@*!3v!z;Ky?UjOU};~f-P4xg4`!TKE0$ZTdQ4Bj`%Egw57wKm&fA)r zH|6EIUyW5DF_VIx}e2ATZT71t~AJ9V<~-QyRlDH>Pjee%6lF^(ycO>I@kK2G6d z-PJ0m?PLD@c6$*#_@F-KjI62mbCb46E==qMZNrYkj2w6^+f3!48jM!k(AgD5S zq;>2_b5GG&v|IO9JCppi72|ok8jD^@U)%1?d#heARZiSaep7?sgrSbjQAd?+NG#it zWgT@?sISR(pgoSUtm~-5F?PI1*5|J9@=RQ7T{%E4Rk;WMHMEy@IJi46PKg?{qoY{e zW~;g+>3l287o_F+Hz_|cx(=T=Wvk|~y_U+>x8rLbR}+nQj#ZQ2Rd|ye>BQ(dl+UbL z$O%uR>BRlcwfU64rdu(7yriCX=}sfQI$!J&_Yy3Y3PtJp9p?Aud3)E;+o@cwk2)e}Pd&fJeqSD``X|D*%_Q^h1?(4PCnQ^sTxm z-wRs_tub$%>84sC=}m1`XgOXdrYE1)xVjqC)})^IEUj)H-;)p9l}Yt9o74fbQ>ae| zM)CO}jrr(S@6|tgzOicK8gP@as@=!FQ{$(V<%h#=sb~JVsa{E4Oif>}9^V$D@U&H5 zs((Fi!V9c=WBvH(idyv7@_Z$^r50ZOTy0t`ubL^O3IE48i@IU?Gj(-}raXAbHC61r zskV&1Y@7Z~Y_cvNos8Z#>uQl`6(N%M922Y?=6*-eoz)9lQUb?aa*i z-0FGw+SG%zB)@g%`C6^BCNI~*cBFbZx1BkrV%)GveXZe%_I&KQ6)MJsdS($lPFL6U zVOEHr2smuf=P`Ta0$U5uetQ1JC$BWy`er(2+eT`B6}s!VdAgO>X<6)L+SPF5DgUjg z=ev*h1^$|KAb0iMFa-%XX9vnN@##al;*n2SX7Y?q@%yGqH`HSy1_r9DPzR~Fp~Nt?Sn&wV~v z9JsRATEtR}U!B=pT+4Oe`ZD^FigAwy5#n^xnJV=-V7)%kto1ULwa(fT#Z$CzCdR!T zsn*MRRGqqgp%!9YVSQKf4<4JLoS2`XyV^5DR(|Ge67^E^DIxWCM)T|qqxheX(x|`A zG^rS0EI3G=wf6^RX|l=yk~Ypt6OZd)-vM#c1OpCkWxMPzzr5YWMfV1>Ax)cJh}E8 zu|Fwi+O^*(pW6R?yA1XNe+0YoD{#JJztq3Q@Fiqlel{Qbu3y8Oy7c4M=J&UCS~NWb z<0;e1Y%6o4zq)XC4~}uzj56D%q=`{)UF^XzKJZ6BF|I^^>q2GPc8rtEDzk0!n;5m2 z)`K@cmv39;qQUCaZBe}MfPnY{Q_R*SJNo@J{wKW>-@R}$Yt@VPHEcQ`&5QRp>3aai z!%s)^MfdIbb=e^JwuZ>nkKM+a<?ERr(*xMh0|?$-sz{+!98Z!E@zF@o-gRh zmvtEtf^oH#Q`DGM1yqmYU4+lVGW^Je9V+)S3BUAHv<=xJ`T6j})+r$kxL3EM>e^zf zG@QTj%^d5;X?^&=;Tg13PrbLd7{Ao&p0gj{d8>t1A^!ulrPA0XUg++vcAVOa$7M>d zZED$r?+zcJo=lU;*0J4KTgriMMh_2av{>Y>t2xGUUY;~;Cqo?vx2_)mEP%84x5-Fc3PJ+`=`;bKejBidOKW5s2D z^DU!I8QqgtN}Wl=`np%+nfAnoq`vMe z=0*C7Vhdl}rWXmFyum z-d!)gwY4uFnX8tH@t`Gn_{*df_%D;TgFryvV`yodS!c%HU%$`t{!jmew`KL zQMJyge^0%qZS$!hQdKU>AI=Wpab5RV!#CvMm5LS?b8lYPaK87rjv;gz$SkP4kJxd&w3_N!ta@unCO&Uy8IEzXbN#K0 zoBhgLCoRb_Zd81(b@)v1QJ1&kC-zO!PK;|W(!3a<;(W7bGO>4E4Pl-YsrHD9vaa6W zhv$BmSzGHlCVpwJ3DyC%`|;S)eR#Hdp6Z&L$F^g9V_Vj3lhi`0WpE$v3XhtxJKnF@ zDQlV8eL2Q+D=$(THR{Zh-_M}En7TXuhVNl(-c5aZwb38dSq0$VF3(M&m8sN$7b&?@ zEnI$}4VR(t-keMFt0|WE;TShd)kEyKF;MMzy}S+MUB$-4r(QbV+V;189OD-ww%W|q zz%F;psI8m6JO0A36V@#Y`f`kWLVocnhpbt*^yNXy2Q6WABR*qsO6^T#E77&gZFTRp zg*IHKTBa3Rx=>aEtx z;}w-oUu#o(hV96dOV;GoYg%&_?kmhi8t`)A zN7dNLt29^p8s4*9?JutOCcIx1tWrnxSe{yhmOo~7WzVr~$9uutDVJ;~=J&TIdmAlA zjt&=TTOU!|bQ^DTWy4PtZz!(SKA~>iw@~X|I$Zm`U~_Ha^$3COc{shIW%~VZZRhBU zf~WQnp2PEsZF6sG#lSATUQt4v>F~;Se0_U$#;s1;lly;&#p`nLCWQ-&{Ex3|*eZ&3WLROuSfKL)@7bsUA8tNlQDny`axWs124au#N`% z{5RO=5^7=oG;5Hk3pO0v*zmj`)t&`@6AP#HQ1Loidu6%SDZCR`4yM;!tru)(T&?$1 zDHf^E+H~f*?qz`QXX=&Gk>YC1SoP-E-rUl!g4Xt3cWaAVFVy%n^~9W_{ndGab@@N} zlZtu+JXH6Y{lw1O4Qwe-G!6MPTXM~PaHJ^JFV<;YVEuQsc0OhtxV>tN%<6^G(L71c z-)&?2GyX(t#FJ#%XM1w60UzRHa>B==?%8H{jpCyUWz&7;azm16J&xJ$^)|N|q@Agh z_#PXdLEyQ7a(Z2;C!oHLozGm-<8?kam2LINHn3D*{Y>MtqL>z2sSiI>apqQxSNrEs z<1$BcH^_I~#R}PyNZ({<2<$bZdD|YljmG6)k`ta#a31fLLZ1myC zKhgZ7X1|ZRlE0kVs<8czE2Uc>b!qNz-)kiV_u$k1wvX|)1-&_GVZReA*kFmx*P|bQ z4&Tyk(I{;~!R~zaKlg)q;kDK}Df{uT%5K32hE7oDyzk1hrg$AJ`|3O|`23p~kDL}Q zGVe>J25mha*t~9}7?FIodZp6LAk39ujCm7`f8xE|URLjCB5a&@!nYpZQZ}rm=C=&~ zU2=K1c$}~8$gdV$2D;QYAl^9p_`-P(^Uh)1f*1W@%xQpR`q(;nCEe~|jBOdlUz-=H3)tRa{59K$I3|ubCJk{+ z0&z?l;+UK;;+QnVF*#wxF>%B(X^3MIh+`6nV-kpC5{OsQFh;zOD-nn* zu_3NRLtKeKT!}zji9lS54e=uy#)uyg7$dGkAg;uQ_z?|b#E)ngBhJExcnS?;#8YS( zBc4LT7;zRh#8YS(BhEq~&O#v0!iG2tfjA3s^)qLoVT?En8{#Yk;w&`8SqQ{g2*gJW?(A0ZGQArK$&(-`p)`aC)~2;B$3*uh8W`3^op zLwtl2MjV8OG2$RJj1eCp5FgU1}#2087Bfda$g%Mw1!x(V~D&h!ih&$joA8`j7;tmAj4m89U z2#gV5ATUPUfrhvPf$p`O@)38y5nrHTjQE1>7$fcg^97hku%SCTjyM7v;to`tkGKPY zxC0IG1p;Hl9cYLzfd6yu6OSMeU*LohM<5V)py7PP9cbuIPet5;Kzu<6#)vNvt}x;Y z1jdLvP`gjt9sl9uA?vcfb*Mpklj(xC7wh zK5+*c;tn*#9cYL<&=7Z^VH=CM0}Zbu#2u)v)(f^XuGTx^4phV)Xox$oA?`p!+<}I; zgYAera9S5w|6Q#e#2pC49SFo7IAO#c=zgq&JJ9`J2X~<3uiaD0bBu#K(Cgwm`mddF z5HApfG2#e<@Lk&X?#bjket;v0M;w6@MjSyr;t1mZ4|4?Zh$Dzc96>zd2;#ps_Vzp} zzsV77K^(zW%nM+Qc>#=n;=LU8ClN+mKp^4>f^k0L2!atW5QH(}2;wm>fHC3-;xR9P zG2#feA&$Ta<8S|hBT(TPd}D$hFI#p`#$?&Dl=^qu;~3SRFJqY>%I#?x>*XA2!k6Z_ zXs_>xp(!=(jL|YrO|QrLd*mx1FGkN#dSr+mC)eZVJbBAlNPmt~>-C!RK!|7O%dYn; zYknCA`hWbGhm(c1LO#dM__EWn>c@}vm=Jfpyll_@KIgP;G1OnOZ`)VPRVcr0crqwo zV`98)&$I1Ny2j3NA%b}e>m;VO}*qe zevUK8Me1oIw$Fz<4~6&1{vXRKzt5ewe`>GyYJQwkx>G6p^BQ-0>G>owO6_(t7iaoB zy7MG%^mO7_ekbS|jKeFRUV1q>KK5~e z(x3DzL4Up+%AK36|2q-&X?FR37`Mni{Nu-u)>_BxdS?G_qCSS_hQ2z++tH60y`J9s zCq0$@kbh(x!%cfL%d+sNIhXBu*Uijj|Blzr0r@e$E$`T$+v6BsshOV7E9g)O;vlHp5JGXR_-_x8=$v0HJFfB8GKdcWQ>Xk#UZ!3_47lG$q@P0W?_WFbeRgmwK zX*u4X^BwQum9op{gcsQk`j6snDc$t65j(~Q^7QZLc6}qC;%X&#$Z_FDEX&F9kn>~2 z_H}31>yICWYM+M$ef(EIxf(DpX)+$s%V%45M9-ILsaJ;n%Y4~RMd-I|SL&Ix3m4Ms z*VFR5Wjz_od2gB7Uaxsl`Moi!*B!eao2-rbgA-%G ze(2-kG#@dYZ`;@Rd`CQbcZqa8n)37)?CT_SeVGv7hmH02GOUd_k|eU&cB<|hcAcc zGM4sV>b+wh(Dw}^AHK)2kH~s*pRn&A`uc^mt{*7hcrNXne6PLSJ-Z(1`%;W5-z&?> z*eGYOXV3pIZ6G`+JjsEcdMuyGv}|wKxF=Q_Z}(4Te5byjWebq$l>FRS`CQ-6)aUzq z>hJy65htBKP*2PDWWO&$zI=a1$d|Fqx5o);m+?=mTj06<-3e+1cwc@fXQZJ&4s6%& zKC(Y@y+A#OU4!TDd{l+x`aH_`%e2(bC08HAzaMqi<1f=Pe@$cg9(V5Eo~Tp%W#@Os zJh4`afqfIcbJB?Hd<=Z|WQa#WTyA7`J_F)xgZ2GY4U7%Z^()8`@BO2+o|d*m>YZ%2 z@vJmzyD1_3-N$}&z~tKk3tzzdl+An56BIa{&64%if=!b?tHe`t~@mAyIcj z`*NO2!24uuw^=ea`~dsDYV7ldos;bv`<$__$@<1Rlh5t#x$|sCp6KJ{?uac9lj-9t z21!7%#r(}J^caJ^`4Ef>X)&6|D>SFdcOC(ST(e_8*d&xJ;aDJ zrUv^`C~JwPGJcjtw<|;oE^)ml_zg>2hkO~AN|J}iG|U-d#6zH5!kKCzqc^(oDh2KP z`u5On_Hl;z%6d2W|I|C^aUm3sxh3Z}M!g4d7nrwgd2;dO;eFH|e$w8>s4E=t8)!Eq zp{rVZvm1YWrk6T$t((5jH-Y>qog-8k&p@#pzhSp$>*Hv|k2*}&(=wLx@U-Vx7>_)B z*-1HG_E?VNkAzoH!lwKd^k;rlNP^3OJ*ZV8efs5?rb2;L1 zZ>H+cWi0EnEYhAg<9(jn^){hJgzlFaar0VPt+GGlQa?Rkre%Lach}I<(#le+y!nUnsr3 zoDrwIC%?m;dj{F_Gv{igr)4bb9h&in9&ZWn_bDyQPf2I@&kAL*$HzSF`e7*+sh1l@ z+I@;;|3fNu)6+G||Di_BaN`xmXZ{q+_V79>Iu-s+aYnm6!udHTgFaR!YA#2-XPO&- z=aE)5V(XdQ{G3Rp8gU%7Gb%2H{ykr=OPpR>8TR!|Pp!wtx^eHN=RU>qeN!qDRmLkS z5%qRLZvN`w6YH1plU;5+d9tV00ps0xmp(W3d7ZLln;y^kA|{1y==tYD?>WWo{lG}YF(fIogmknan5PZ7kJBc)|^{<+Sh}O4SVzUL2e#1DsPApCrku8 zv$I8r5nl#-A?<|hr?ew-eNS+#ZyA4S2V^;UPLSVQ#IdN$T~ zyC+hgoAYtMOw-?Itc(0t?CXG}wC7tM*!A8r^AEk8)O)F~yT^CYBE`iB^L6R_Hi$F#d?y|F7d58dW|jL9Iaea5j4g5;Wg6Qv z>0iimj6FZW5zD(Q1(-flM&1EGHpD!r|;SO2mYJPmvf-U;9tsk5BM$i_l4?lF!(R_v@uSfaf{H|OrTPV4jI9K-n~Naj1oa@~&_D8JL4dylclA$m+4{j|SVekaxk zyIx2?-0m+M=U<24DU1GhUjz3y`nuHbYZCPJ_5a(wjGXWP?o(Xfr~G%1@%4L*|Lz^W ze(xad>3{bN|99^fq~Bua6~4SjNPK^Q&mVFhz~>yfU*mfh`TW0og8%LbbbsN$djkC) zL9Uno?g`+$|KB}QHdn7FaQ zBo$Msi5w;>Jxr#M66_uA4YD^$%90Qhxk8da_?vvBoJt$%vR)v2k)&iCeM3CxIO;*&XnK~Pog(QWj3h}|9+nJbGE#$NBSlCSnhio$ znuPe0QSh!&Cq8WZs(19Qa(vPG9sYoh;tVBxD z%22Z`DbA{q9JDygLA_a7s8xzWi?kB?jWi&QK{h50Nqy3kGy~a;G$M6L6Nu|VXbPw$ zX$`V9r0S98Bn)I2X+>H_#F; zXVQc81lf~xCE=ty#NiNn0O~+Gg6s&Xj_|A}89_#qIFNB97SLES4&*pUjU;0r9tmMA zpb2Cm$cc~|MaDxs3c>_Hu~2e6gozNwft*C9lIb9)lWBlvl35^UL25FY0r6xAGXc#Z zb3x99)D$us;wcd30Gb9RXG53^VHQX;TSanGGs{VR*>7|qSxkPTi^+V_6!bAGZ3-HB zp0=V($#T+)E{CV{$P%&w|hiA)45bHua(jeB6T3BbQkp$A2CXj7p zGqC~Q3_%0!jix(5?jVBnrahO;_+SHBIrcxsdf<6{y z1?Xm4lx?QP*c8errBfh`fSM&ix3)lNPeValDw5RX2DuILHc3y?l6&L{$R{KfSwLIU zmUIDxS+oQxL-vB)OS91d)Jk`N+(ECDWQyktWOxLPZVZi z4M5vW>^E4$`$1EFhxPP3IY~{y*og9E)chHFg5cYu;=}n?RMw8?$8S6u$NHP{h zQm_;tdqLcbq-2k1R#J#OqJ_vm^dFFUAXg$!_XS5Z0PG8WL zpv^C66Y?*839>GEN$ZkVG!*gMHWycA^FOlk5e#7wkkskh?(cg4y2*bAAxyK`8Yn%=rP32gn}y9e^3%17R1e zyTjxd$YZeTW|QOO1jrMx3a5cQ0`drq_F=Hz=Rlr=QpaJ8&w@M)&rg6n3GyU4N`l!d znA>2`M#ARM=JXt$1K}=B0c*1r$X1}!h1f4_9w`JN3H=4^^RJ{BokTCvVi2~|2P6P& z{R0S(VNdWQK_G+3Ls%cT$$gObNjg%1k-h{p_plLv+p{d}P26nDg z5R$!kT_ZCe!zX(N|;);8)}wc?-XHWEiwRjJ|-ccubyx zd`cd{?n@SYATN_^ z@Vg8<;WY?XU<7VLi302vDaZyWH5CH=}z;}4`d?s0_g>| zBpB*tr@>HjCwK~L=^Bu0Xg3IFs=}@wO6t=Zv=MDW>(IKizK)vcXc3G<8`_Sx0GqvtNYqY8TgmS< zf_9}HXcxMbNYqtFec;>r(iqy4_NIMw6r&>wzQiaRM~A?dQXR$VXfo*9R62uBfbp8F zqZv9{NN3YUbSa%n)4~Wyv{Xmw=}NkRZlbGUEYs^~la7kP>JqTZ1T|A%nm{#>8uh1s z)JC&_%tDKhW3&^EgtZb$Prxsd{sHn2dJ=wrfc-iN;W(`4=Cm!yw)6u0+R`wPVe~Kf zg~3kz7liYmFZ1bgkjv?PdXFxnOF%B659tHC7%ajA2zOx@yF#ynyiP~L?>fB-@+uuo zW9c=T2V@>P2zJo|w5Z%m=~nm^rG-HjrrT&dEd+BP4`B<~!dkR3$j0;t{2J4SARE%7 z@M}nGf~-jogKb?!H-g+qZ^3UPT@P|Sy$!$hu>NmDxC#2(i}nZEpPqqVf7%aZKYA8^ z{lGe(g>V|QbSK>dat|#_%g{gRZjifa1zMi&f<3uB1S=g+bJNQpFVpFC8vUDI0(ptf zgx@8wa5EuHqLt`AdJyD6S_OUw=>d=jXmwhR?x%Y}?xkf&E|!M8qPf7L2D1#XXTG8t zAe{#6-wXOLO$WPa29gr?*yl7Q?6PUW*Ly;r(g@gn+kqeW4}DDA!>;@X?D~&jpKc2| z?ZHN7g_Hu}EoG3B@-3VRQi7e7@(t|V@2MAg2l5>)!8U=fWC3Ym>&XVNqyB6I@n^YU zOwG)f)dX+sFl_)U)rU2pKCA|OQE}$YDuLI#j}`|%s2D3wim_j4Ca^$7SSGMR8EFQv zOodqn$jL-6g2nokT?Cu;7r6kItRTApITuL^lAL(36cF;UU9dyBgLH?p$OS@4ZjiZI z0Y=DQP>(>!2{IQ81{us=(~YqE2C|JL5c<6icH2=1&0NdYIb9)QEaZ%+okS7*rS0rOgdC1p=(Qq~T9`=l&6#K~C~*x`4< z>FYNLNmw25^`uO~YLHss11hW*c!9M^b=Z$7s}9~_P4XJ_iNH7!NLK;7oPkw=ujosv zf}Q?It3pl{l7;*VXQ?d2lYIbx?mfIYGn_J?Qg>5Y(_rEbfs)e7T6U3@RxUD?siqXB z-b97)j&gR7JzyMy%9ts$X${CVEP~}XU1F<2u4dUyuay4mID4fWXCIV&rljCo<}>vo zX-tz959L>7GK8c`Ces3Ton_H`3Q0>#ADerr7e30U)B}^XLnf$vx2m;hpZy2&3*&<8>`7Gu|}*u zqb4un4aZ<_QruLRHDrEDT?kE>MQO!avg#08fo#Q^!LJpo1F{aQ1)&x5SL#4$z&f*T ztS88xtSkI_vfdzjvtFzVgx(-~GZnt9oD#*#D5;cTN*M^DkSec4vnUqL`mnxGs~a<$ zd`&s18A3JE6Y|R|Jt6dF{n#Kj1mqAlkPTpi*>I4<*)SFZVK~U)Y$*JOvsjR^Y!vIy zMzc7Oacl$|$;Lpvkq`z!%~&WkmQ7?+Ku%#3;5UWM1UZvUXXDrmHXGz@HjPbXv)DY4 z^VnQA9>P43^Vl5t&0~{6PG*xJ%!5+%*>0>s>@h3DO0xGL-?M+& zQf0o|z-pY-5VaWEuuiYElwYQqusC z13;&e!3ypNvLEQ%Id%c`@Rj6=Ke1TlK`Csoj4C+4ohK5Wr`*#AoM0FO{q;$ zAfrehlG>C8>_ln^QRJEOTxkmHvMH;=o+~eudLZkurtDwkrBVrGC0L2Cl-J5=kfT9& z-zaaDp&*BXKEG4mD={ErK)=(P(wT;lv=9c9^rj4^As~m4;UuFelW7phL1ZY&Y|3KV z#4p?YskFC7|n|wYMNF1#1}v z`c+zydKLvEzY*j{$ms+2aSg~dkP2hrpuJ%b<^$>o+8hpHG5Z~C+(M8GA(g|F)3lc4 zfUuh6GzFX1fLsG7(3ui!{VW$A~V<$kKfYfn#dI97GnEO-gG|coV2!8=O0W*IZ z!g&~#E$lML%V4Lsz^c0n@+!LmXf?Y8@)D#j!qeLzZ-W)T0_j^IZ^2k!1#5l{!w zSo^l@6@>P%H@s#YKz4xrp&TpAUV?lHYo!qE8?8aMW_e8yVK1lxyFnF}LfHfBHZ#b~ zN;>5|tfmlHOChjZ1e$_Or*baTzX$mqQqS27_7>z@_`VnLJugANgwzxE6wnh0&jCG#Cr=@~fUkSR?t{D! z-|`4{*n1%FL8_3cunGQmc?eHfVN(&)KOp~sIz>&Mri@Ba2pJSlQ!&srPY4;Ir)iZ( zFiwx4t+YyNjTIS(7XH4@;i|4Ae9|PAeWLvDX2UI`A}J+pK7eN@(0VSoCkSc>8mjK`kWwhD(e&<6Ej6H9|+ZykF1bV0AvAWgpyLJ z!hAvcD$^9M>{arD%nS7gD7}@9$^ZzPlwgy^w2}lvSVNL4qm)<}(O4LB7535cN)2XH z+(5d)N>~Q-lv61V*0&M-Dzl=B52PfPC_?1ygDwQE6 zu|zeLYDy&7=f12vs|6{EC2FqxrZjgUN^t;j;R;gg&N*kpkq$HN8htfsa z2dlW7@`ut3QW8rv5WcZL{Q4=;%3w%IEYVo-CDOnzRvD>GfRw}%%>rGU0l$>WRAnxt zB$jBovOrk|zZ}XUWfi0(mS`)CdoK8GQZ^{tASJOxW+gz`4RSZEwm_wXvJ2!c1FY-I$`z$O$o7!B z0()^QkgXu~PXdRqsSC0$r0y#Bm3WZxU}Ntq50ot+w?OIvSb()4*Fx&N@|Q9M>>0sgJLAa(=S86K9Kps;L1FEAO1$h)wwPBAx1oDt_M9E{an%;wauasuxS!woO zDG%YDQWjEgLB3VqD52mlyaxGNd8Nz%kMAYOm&(7&1n>f1fP4Xa^;qx@@q zOZorUdk?TEk}mES1w~y9fLYfSFk?c%08E`xjF_{Sv!EzS5R|MUm_W=qfS?#KCt&J~ zam|W3=bUp~^{X0b-ZQ-0yL{pKp8MQ;ormUUak_F(Rd?6<*E@0jxrw>?q3PD2|S14b1A9l*w0L9P69aHM7oVm9LIC))C)Kmg25d9&4M~n_U<8 zlC{LKmiQ*SYI5B~9&4J_Fe_|Q&8(_8Ry8YRa#q}7*qNQpZ)bK%+=13Gvk^8nX8FVt z+)Cm*&qSPiP4Jg|8o?T~Kg@QDT)T>+s~C;9nd~%?$KTB=iep9bUAWDpj+s@yePS-? zVzw{8ix|%{#N1FGE0~oRxw1DL%bRT$cfwV~ETfP~IkO0p$>N^0oY`dYQ`RiXWRgi4 zaV%q2+AQ28(xj9)mJ;_{Q%tPQs*7WFk#9L`wl-TYeyq$=#QfP(94*aCm<W^lvG|`nT9_3v`(!#t%+EfV78dO@ zN6hSh6UX1g{a-}>LT0w&Xe;Wdn|NPX$gG>ZV>9bw($%D(I2JT3VAj#3vx%8Fnu+oG zOTLaK@@OjhS37YhT~ZuNitp+6;;vL4^P4%CwH5bL^~ABBS#2?&`((<@JjA`GiPyT!;-oR}lAA;lj zWCJUEUCQ{CJuM}_Ze5RUJ+~g}zm5UQF3UHwgS9rm)0Z(oxj;8ZR^jY+xcDR*C_AQX zWpk?UfGv-rfil?}8OYwqK=wukvNtl2{g8p|hYV!jV`#BO4^=V~VNA$0Ap0Hz+4mU8 z{>DJ|HwLo5F_8U@#r|Lqz@tM7jO^kKWKUyI>P_rv3}jDZAbT2vQlDZ^V<3AP1KHCU z$ezYP_A~~vr!kN{je+cG3}jDZAbT1E+0z)vp2k4-GzPM#F_1ltf$U=pWFKQ7dlv)Q zyBNsc#X$Bh2C{cCkiCn6>|G3G?_wZ(6ho_u^RWNrR1o{4f@SIR;N`c)z+`_{Ap5xH zw1u=+>Rj^48Oty%CCPAZ$e8O;z+%0D$#sY1xq2$kZ*pDye`2q1a$Wpi-P4;~7f7xPB-aI! z>jKGjVYH7ixh{}g7f7xP+_tU?UN&nE?rnZgcWeE7M%v(y>%2ybz^PTW4dG>9*Uv~> zqGct1F z8csRzlPNS3dp*6)x|#0y_yn7|WF)wH{GOr4w65~Pa|>!<;F{X7Z)et?yzojClT*!5Y!lW-@URbT(XgIAOHO@=ks7+q-Ms3J- z(S%$VO~`fOLaqxJ+7>F;g$ubZ#J(_;~|Nd{knqPL6v+aNVmVT_c!~b{u zZ)AM`1rtJ*qi0@cmF@(=Pvaeq@TO| z{lDVpekNNVA-A7&XAxpZS;HveZXm zEq?hue}3@G{{MzQ*8krhjCBwGpV}WJ zNHswYVs(q9_%IW#_%sz?e<0q^HxM63@izzY7a1;BfNI5$gJDH^ak0*(u~>Q7#H5i} zD=t@y{~c;6}<*cbVnu?n1Al8L<5yviKRY7UKL4V%2S!_(X{3Y~kWP zs;~Hr6RX6>iq+yB#3#%U?koNuE`BM`OoM68n^Cf3Cd7CcNGyNPu#ZN;&zNVBkLgBWp)5w_FC`e?aE)>EvU?I~(+y5OGT z7%g%?N_<9()zLm;4b?R9i8h4z2r4Abo*|AiM3{$&VY)a@7iBFV+8|aOV+BW;I_e9r z>j)MHgF%fKA92)Mr8I|<`thK~&jX$G-#@v4&)cz}#&bS8>9-tlgl5IOK#dEyI_nF3 ztOvGYU)q1;v(r20*yLs<{w;|ubJEJqN=vPSx5s~Q;k;lHu0XT>HrEAh+rQe!2bvfb5KsT0|M)L5w#+4t1A z&?+arrdvNlKUCwLZyojB@{cg|Lp3h$=cr%lFvidi)p%JQM|~SJKSMwKHwAI$nb=54fWpL+zs}}PBqj|`-6duXP8MNeb?TuAmdk9rmNV<{3z$je71LA zgr#?<0rU@n#L{8d`S;UePLAFQsTF1dW%BGED3j;-s_)Xs^L?ZDYV;i& zdF~IC$vXg}cWm@s8fE4EkgO}p%5zv5PMN${FnZsn7EazV7`=y63n%X$jNZ?wg_Cy@ zM(^#^!Yu}+;))w<;Lk))n7num4zXMdgVszi;@J(Pcw2SnC3n!!S)8ZAd^+G(H zSvn18yGu-ym=9lYV_(fhD{^(N@NV7w74 zeOSKpD;eR8&q_Xr>J58A(x^%}b>SztHj=@}+BL9B`dgUPtRoEDVuj}RkD>4R3DBlZ z5^pm81H5qcgKdpX@YU&;Fwt~81fM&@7jAk72VDJOr}J9a_VFpK(t3h&jmUS0KOV1w z;m_W{*W#UEM7w41DgP(f_#43BYCdo=`U3#P`7j{m5zMy@fhv8YV9)w@(CVZ= z?D#$tjxTr#TOW^y-TA(*JGoJN68q1Imklfb~HdpA(YS`R^FDwj>?J3537kmfHFZvs?etI)@qQi3tIU59& zU48XzQk8qKvU?b;Sy>o;X!IUkEu6eZRDIX3 z_THVogQra1JF32$S9?#dd=KY-XRr4DUhQ4J^1Yq=y}sHzezo`gYVZEl)&Z!k2~b-f zpte>(ZQX#{8UnTT1Zryw)YciOtvR6U4=9s&u14z;XgFP?pvH9Vf*R8`4V3Bn2Fi4; z17*7IfihhKL7A?HpiI|Bs4;mrY_xuYhL7F72q*VQgUNsT!_N^b@vC1dI6n!5h@#Qh zdC*$0+YkqTt_;Fyp<7_ov6*o6^DS8XWi!~9_J_xVE`$Eh9nk;XSV)~8guQ- zalhZ9(X3Ajl<6G{lr>-P!s`la4Rbuo!O;((_qUCP@E-~1ABy^s(>c99~MGg&S?}YlxW&+x9I9_N6EEqP`h(9-24yHwZzJhsn4rC|Od>%hbEDeV2x8M?OE==>{+#9$b5dgsn-!;LvUP0u>KqJ0ep*mZf{1jdu4hG6r zz71G|7k44pGYs@Wr`exF?!u@OVX$J}RkpdqQ^>450fI-pW5>_Df`az~Vb_u(aQEsP z2;3e3E@4sV60#aPzK=EHKTTrL^x|qbbT1avc;|(w*mFlB6gUzKl*1n`hVWXi0hhZ1 z=p;`fNpnUMl zYTn@cE0|H<3n-sI{F%p`e*$CXjRMNmlZs-^mG{8W$`>e~h3ahK<;QR`+y~US?5qZ? zQ{_7lV>u2eCtmxm$+CJ0pNDvZ8V~ad*SMO$hK`NAfbtQOZ>;QuH8AOAEaZhRe7MYB z%}xgE%vi_^UvxRhR(4$nuRFwo8v9jD#7NiGU@o44P`;r_!b!=iz^Q~MP~I_RCE8C+ zgcfPyeoBQ8I4;A@kCVXTc_2{62wSWh^BjISv;oSa8r8)MJ0C*Ky-q-RAS;jkl3v3_ z@!XuUwR0I9T%o?DF*j$}3Z@vc?l1gXs+) zpuA=3X*MGMHav_S2b6#O6yG^(p1_UqK0tYm>s)wT{~pZgIujnz9K)q30^?iGxac(Ub_KU^H`wVEbIsjzO@bl7sobW-#2VGLaNxplp!{3ETkz(ORH(tnf*Kd9a2>4su7?vP{DE@l^Tl`~ zbql=r3<1hB%df`oK^vge^e~{DpT%Oq52;{rY&KB7DAt|V-Mt-7ua5)DA$#wD?c=R* ze$Papd~nVcs9?JrMotd~%2{d2Xgeeo7VQvwQ>pL($5h;SXf5oX>;shF1*}J_`fDL( zl_yYkIvs_&Pb;89|2aVU(z0pzywfVUTVoEWvETOTc)9%=Xtf{#)cD$kqfo#3W*EM6 zHc;;9aRO3yrNVF15+E;J?fDUKPD%zln>nDyF&p%7sL&dC-D?g|9$@zr9zEFvW%~61 z%AJZmfpTNh;By;qpgi#D1Ni=8Gt}HP9@My7P$e8z?;C`RXI7NY7qUeh@dfNgF`&Hi zh&^7N{vI-#cLd5C%QwNN4_<&T>jjjfQ>?J_)yL50KsZot>|%oNrn~~@R}+D9sVisr zx1sOh_iDjFd42mN-l6VCxLq{}D32b;2q?Gi9R&`<-ou%l!9e-JmzfaO@fEnA z4FSp-r{}}`-jAWZc_dIy?fQl7IM&xX8k{-X6*(D;oo2*vk99(b#DS=Qog3bcz)05xtrJ5$qTQzD$M zG6yI(cZqQuQDik(^_>Hhbw8(S8ajT2_?AIHdEtg)Y%6~U-uhrrWA~6|?3VcpxSSCR zY8+(QneDxL4+@ry0?Ko}^(^J)J*fLf6i`0C?Ib&C@dECj2?feeO`o$uhu^{1tHD5d z!t%nf#7)fSeS&~;{bFC((j_b4k!dWbagA4J+2@NZ;phDrP~%Qs>v3}FU-#FPnblLs zcXE^A-i@-a_#Wsbz6a!eBIVBFx_Bn83mH!N`eVJg7uy4qC&hsg*LP&T*1Leu7vXu} zM$ubYwNbmkMBLNmh2ITaf^S-Hf&K@?eJYA+BzCPpIr+x>L&o@ zq6>nsRq3q|Ebcofe`&vAP>xJ|2rzy#bUWQ0DDVIC0aP>F0{c!42g-IE?!mKKTcOU!vEn(@-}B?^cGxZa zGaM81UdlrsRKuy?K0>`OE~N8xD8MqbJYcP#a=+c51oMWk2ytgto1|C zSBnJ7`%Zr5eZ@P1HOE4MvVQ&vo^bUw#5JA>lr1b1x#^_0P_+F7pnUXeDhxmV3`P{| z0+c_ES_2L4y@BLX9zc22q*c&beD9S^as|p&tyV(2qo2U}LQSAtIcyYoEqDv@HMnsFcH*vxJ7lgssA(Z*b)Y6d~0X}b|B>*w6uu=%H{k{vuD5GgLz_}LV5Dc zt88@PXHc`i?c9`5HbQn+TL=E-C_HqW^SVIsqu(e)o-)ew_>-NOKcYYt706X++gNI;9WNx!YTK7WQUd$eystZoWIOUIDg=i!JhJd zD-Mn7?SeaH;(_wKX(ie0&by%DQsVqKR?2ls#95>5 zeM6n(W@+>1p;%bI&PhLg;RCkW&zOgZvYgbNWYWg&Vgm!qbQf-HpntjRg27&f%Xlhz z#~ExCmgmZJWjJMdu1xFV$eD(C{*Gmv%6Mcw^zHh)R;f?Qg}&NpmAaxlyq1ktsawj* zxibH1tfZ^h$UMmYS)rzlR@Q?I?<4Y~)Jgo$%37r!4vYLqds$bKl{!!SW~)`|R-UWW ziHuXJD;bYc53+rg>w>b<|79L1%f2M-WuMFo%YG&8Wj|MA<(iUdQI_?lSJ{e=Gl23XFDLzq z^}7xB#Va`J_WTo~kfc+vr{K^qm(6H z_q+CrurDd{Eb}&1#Q9W|y_fh7e{^yqGv7Z+l?Mr%rwey8|8FxOH-`jVVvF9+HH0hN zN7(mG{K+nzS^$)rov){tY02*^%AdwM>7R)BW%x9o#`*{mXKr49wTWKFDZ|f+{6i6$O-CN7d8Ia2XHf%Hy2$S`jYO%?ta9`e~M(E@fFi_eA(t zkuGKBdc1gjf#JJCUZ=TPnODegHBP$3cfZUq*efjazen_$Yx^b{^8a`Iu}fY3l8TXrageg|{V((OcPup;lCq4?2LSuuu(mU2hBt{cIrN3{xrw{0td8mELb&=>ZY z2!7()QRC%%9rP8&b-hBQtH#HhHP8=s73&4wm4l&g9rPo&MM6}&N~%2k{vXanB*+Kd{kGTk}?sfk*sNSsS)7 z4*1|OpseKK&^+NUlbtuo5A#*l{xv|{n@$K-mZn0Q zW~CmkZ?(*+huo~>U&c>aDX+`}WhMVI9yM0VF7rd#bFE!Yy{WO%A7q`YvF|=R?Xlki zA+AF(P`+}|E~lQ=*zc-cP8(5H`ln1+jc@m}&FP0~e9YZeJ5Q|XmU;WLmTgWSQ{w?z z+nm0t#!6a-HpZ) z`Bd6Srb}6AUzuk$R=z{vs}IZ=Y^R<6G6KH;9%#e^L|^()G!QDb3kJ$riFVqh?}DL{ z#W+|SW2e3PFa*vP41lymTdmXJ>4tce?<*NU<*T9(Lk z-kN2IQ?dV6-zFzcP0?~jti)64RwiqG<5HvIbV@vGT+}B{r<|+CG7r*b(%gCQPb||?Z2pdw_@#{+EBTZ*YJ56+A1LMe zCsxWOZT^m}AFauWM~zK|jdHFUEAh)b{}U^9qU2ezSHhL}Ww;tE`II(l+~IH%mWi-2 z^o@Vwa`S#Y_m(z)$DVKcfD(@y+r8}r%DHN+#NUt4fq!Dty`|wp#5{wI8rSJvF~>%Y zWuBF^{*INhNEQWjcU1gy9iyD@ zqm8!CmSNCt+X1G=%J-N&m$L61J8gVK6GK{*%WSgKj#<;t5T_cCduFGd-K8JgDzcYR zR=(?s+}Xm^xXcDS?WRUUVd)vpC?7pzr`=v>ARJ2B#nf2&&Xn;}*1fNyO?jX-;?z)L7A!kahBJEd2d*Y;v;_ zziclxR`My^U5%CU%KoFqimrmRQDa3{LH1QORx})Be$-gef{^7>V?|R!*2%xItY^h0 zH!JbW_EKXdpR(Q6Sg8})f7Dp16WRCFSkYdP@u)G;oj^5Kv>;@;)L7A!kahBJEbCdZ z$<0dqvc1%p=uQ~&oUKP;#7cQ(A5&vR|3PR_i3SBKx)VYRB8P=W zM2?Nnjxd~?twE6!r_h?nNlR!@pppj}uINYz%?ecXC1gB`?u3kAVVSOCBQz*-&J`LI zNHi#rXiy;0pg^KQfkbx#iS7gv-3cT*5=KmPB#<)EkwD5sQ$kfInrKQGX+{(62qXPy zq9b9XA5C-~RKtn3gOSFxS~$^mFw&k@3n$tRMw--W;Y9NRiROb56U_%Cnh!`cAB>pj zJRoJF`G7?80g2{=5fhyUq)ap)kZ3+2(R?ssqWOSC^8tzG0}{;#B$^LMG#`*?J|NM2 zK%)78MDqcO<^vMV2jK&hqxpbD^8tzG0}{;#B$^LKOmrTQGSPWJ%0%Y@DHELsq)ap) zkZ3*_G0}WLqWOSC^8tzGgAo&*2c%3iACPE1Akln4qWOSC^TCLT=7XvZJJEbVqV1r{ zMDqbD6P*Vm4R;z&v>lAJ-Dx<{c|giU=K(1bod;B7qV0frVWRnfiN*pFjRhnc3rI8;jF{*u zAZ4PffRu@@0#YWr3P_pgDj;Q|v4BKl0g1)}5{(5TCb|kpnP@B^(O5vDv4BKl0g1)} z5{(5TCb|l!#za>EDHB}qSM0#YWr3P_pgDj;Q| ztALb=t^!gfx(Y~{=qey(qN{*vOmr2HGSO8)%0yQIDHB}kasxi@3K*~f{0Vxw*1*A-L6_7H~RY1x_R{<##T?M2} zbQO>?(N#doL{|YR6I}(QOmr2HGSO8)%0yQIDHB}?(N#b-Cb|l!#za>EDHB}=n`OFnCKFSHp9UgDHH#FL;WauBmVoS#>9Ug)mZws zDs@ho`0rz0Iop5VVE=ba{P$6f6@OuQEvqr{8C`QYOCqNPPK``0}eV z@#RP2%WuTQf8P*)wlBO96JLHLzWhc^eEE_1@2fKLtf_zVnzDCcg7XnfT5lW#w8BKJq!&MQ&EcJ{hjY#1|g(%Gti~2785NezJYx4f**y zR=zWZPkPSxXKp4w>`0mTup?#SpN?uw{Lqmy@jXY%#P=N4nE0Nf8WZ1hof;G0bDbI! z-*Z%B;(Ly2OnlFEYD|33b!tp}&rywu?>VY5@jch6G4VausWI_A*HI?E=Q_&7_gqJr z_@3)16W?1&stou~HV{ z!<$nU$x7LU->wlWc^3Y;Ir-1cN<9c)-JE*J%}V}d{FIgQ$~;h3@-O32W2NjeKa>@} zV_9!%tn>$2=W49@DGOh3q^$TO%X(I0#g|#O5oM)+%5>FO@dr1w`+t1Wk%lY&-7;^A zzqU+EjTOIdnP)Xt;y1L>fBevM(v{&#o@INfv6468-;I=&bY+{Wv66q;rfRITm&}72 zEA1}xtj0>4%6uwqB-5p=w6Dyw8Y|x+!cQD${KpU7h!tOK;WLhu6<=)OhmMMmw(vto z;)jmP_mzyFvf?u?<0O9QsQ8b|I2C*08=e!V@EJE^B_81?j+B*ngr9g${K9`cr(7}~ zrM!}rdXQ;RR>~{mSIQ{kQOYjkCqCW($k~3!2AjWQ#Xnl!ORBNrXDw~inD}((q$T`| z|BJKzlymZ|*eKyj{4!jPm3&GYHCFtiWuE_ul{!&;t`&PFT=9>V;cBe-SxXx=R^pd= z{wG%ILCLdXuY@b{%WyST@+oc9So%>`@D{Zh*<7}U6gN+(z`*a&@{*INh$ULYq@##i2CO+M$#!8*YwA7gRbaTqY zr<+qIKHXf6iBC7DOnkaIW#ZG#)tLBnbIQc0o2xPL>E@J)Pd8U%;?vD36Q6Fb#>Bsy zQzky$T#boOH&94iBGo?PkNXRRm~ltd%HRCeDops)VHC5-wxjjO+9NG7;dD()fVLq+`eEk z>}_do;30jI;H2+;CeM}OHM*o5m!}>yc_w4dk_qEVp%Pw!?)na-j{!&|t>rZEwSBBTh&QA?t|2Vt6G9IVw z{L8dbvdb&e>YrU+*TZj7318px2|8xmf6TmtN<3x{uVc;Z_$#LI6( zQTtprIsHx8(0+w7e=;pnhd|mw_NyEldalS1?Rz;kG@SGYS*{!#8ZOH!|0eUGVw1b? z<-|kM%H8*JY)GDS_q`k&QeLCJNApAKM_$)Teaikr<5Ai|#zXsIjtvbbZE6$`4JUoW zP)|yG(Qp|i@*y?Qa@V0+Kg`JwNh^0h%&{SP&fO1lY)Jh8wWsy-|5mn0_@Fttw& z^-u_W+7MfY*M+a4HXN4R(3NW5QFowLIBYxG6(gLB!p}!nbfc5K@Oa7v!!;gshSesIX^B zu8pS^b6X4!TvIs|Q}%ViQdz}d^|_0>HKA`adW{{+KU@gL&-M+mS&e$IX^b8BjJm)I zrg<5z--JtEc<1n*0`1?mA!6o{*;oL0ulRRVUNJE^mtfI^YUg72pjZXe5d*!*cv1{pHWfygT zvMyx*-+w;I^>|keZ#_Q*Deo;A1PxbD=F@^&X()exI?82M9anyOa|lwd-6j+VpR?yr z@1Jp_d~93<%yEs^K&Re`|q@z9a81gJkt}UT*$;#-|QLl+AHWJotJ;A>N1 zR*c#><@1J7RkQf~tur;V9Qwevt!Md{hbvjykyMtk)`OQ6`v(p?*&6P>JI3p`c+VC- zafFhi9l<{Ni{{#GZ&vp00A_c?4;(7k!ox5da1nco6z_eT&2hZRMono0Lx$DF*hWp@ z+o(4DYKdgsvbzDW)GMB+jQYs~oV!Ei>4AKlCXNSN`atcA(|G-YK0Idr7}oFlIGEAP z9y_jeg#}GMYX-fz%r|#`%1&2k4HhTF9#|>-%DNP(; zw4;`PezSnBoj(#Bc24JwJk25ho=$M=;5go}^nAC`cO$Uhp9kD7&I$wH0l)DlH+Q-% z=@5#uI@9U#&v;=5Q~ZopsZ7!oo1Dqbb6!M(oaPb;=xvf9r)^A9=d{74L(- zPxZXT#L1cx4T7N9!AZRM#8yT;@ld+PylE(g_*=7;E$ zXNsoc+l{PHlQHPNxdeNE$DECb@rQk(aeVjk2-vczmu9Ewh|Goir?LvgM?>t)>0JNW z6^1{m2AYhfM*J}}1h7+M_H;}K-Szm1cywWFR{lhgE__{nexSl{=F_Zr|9)Y(^Xn?z&n2Pw(c6+uELY0y%if}V>QwPA z%yT4O@Hx(6LnrHY4GzMC-jn#bqSl%t%MQ9cYh8ku^9#e|%Juo^#S<{zx&)&(*DOt4 z`$d-G$xXvB?fZF6`9~pWQM)9o)~*`_ZLk1st24U6&z_~9KhcQKx)g%1J$`rHQLYpp z&>{@GJ2q#FW(A{%g*E%5QA;RQt}>SQJFKA`f7%=N6gtH&nMCSXuc;aH{jIrQw=kr< zdeIHd*`z3)`L{55e$xc^)aZxW`;#+mXLy6T?vmN28nNv`SnjZKQX#5Y=cVf><>nP=l-z&34m0;}H6IAwR%o;scF@ykY6xajj)bE&Wf zPpztr1NLS$&e}ebFMBwKy?HhcYIJZz8?QR3Em%8q?KW|p)~pFtUf*Mv#Jx_#^R=;@ z$MU^1yIkfs7e8eg4y~d3csKNIS_jXb>7Q|bS)|*9D>J!C8S(D5K?B%P)d329cjx)` zWVt+UHHSCu7KqI~(lncV1EBk%1fIXgKGzC8X7i)V0`Yl$bNH~i6Fgqx&zD#**Dvc5 z__|Snc;fs5RxDv8X59*an^WR>+k+o6GCs!gv{a#i z#%ki*(oLX8r}o?;v9Rki_jvyHkVw}$j%Qf-h^m>+Z`7IXVm)#eUsYs0dfWWu!Bx9M z@{%Ba(EN)_Q~Q#ps$089Gc)RWmf_3mg<*$TKlzHPeemh(!mPQ657uhBpVw?P zSu@xCOS)^{ioDOyP$QeGTPEW5LG9R&*kZa;d$U|RES$p!v<}4mv-0!X?*4EmGJ!u> zS=hBrU;-cSABYbUCZbbBYc}Gq@w%l$Yq=(b2>W({IOyRAo-V#aR@q1K@LFwMyFO3g z%LfHwV&fk8s!BoVImwn+6xUJ2zSH(U^3)VL+dtE1ca}@wt$6OSIS>z( zEyOBE`$MycINp3|bJu_cv$@l}Kn(fj3TK~G125mE{9t$pTy4~trLAbM+jYs#wW|4S z?wJ~hQx8mK`VymINaS=*?K}PTLpQtcIbFZiq4=idE={A$cABPthU10%oxr=BC7yA) zrTG@w5N1BCiyjYb*z%Oey6at{HJ6TuVWGbHS$CIdKR+C*?hP(2j8$*!tixOe5{$5$ucgwC!%8v1? zTyb+0H%kdZ$~K-6kYqhm7ZsfBMmg8q55?TiFvpYgLGd>Or$$HMl0suOjUUg#4>egI5rE7X16vdnz#GKqvx4GuDAm+4&@S4*#SYStW?9#A2 zE4m`mt#Z`~ylu;1d~w1JnNJNYQL`cw^H7lUQQ1afjtXLqYG67??YLnU`(s=MHutF? zrn|R+=j-;fapS)5t`+;h9_usgR%Q}^wPOUNuD-}hEKkr~xO>L!XuB}@U~0u38^mhb zT?mB)Z%dxPSSWN&s?XcJc;?oAqdycsx`aDLbkdkUZ-Xtjmd8kwM;gD=)p4pbz{}qE zb!gWM`}mmSnGTyZdp=j;Ctpv1*$2gVLu@83*{=I`c>;WF)sF8k<&BlfWwF+kqIK^R z`yx*{$6QA(;By^UaLXhOGd(i~ulu&cpMfV?{H6!I;}1tX7UhTst={Th54Xkg2W;@{ zU-$TtUbncm!3{pn$^)Idjo>pP`f|e}c zH!Aqx_B+$q2#Ylu+Xn$yYGgdyf7qL)+4%7NRsFEK>18&s^Am2ewly{_UIAK*IpPiN zT|U^`9(=hg+FX3Cb1kt+moF#)=WR$}ljkhpaY-ZbKA+AW&oIY7mUcq-S>xE(_y*Xq zr~|Gy(Xx82P1#D*(eSFkE`B|!N9K})L2zhAB!Ah;l(i7w3BMiO$(;_~WhYMbgG;tP zJR0w?IqCgid$DnRZBt)Z6}6pJvLCPeTp-*wl+`Qo>r>vwouH+E=Gsp zS}%WCuyPp>7xUW_-^#->zna**!(C3tckM8HD80`WKNWqWqw|EJQJ2`Pv?u(}wAM(+ z{hoci*k^wqzPO?p-iybKAgBz+$XLKCjvMNb){dNF8ii>A-T;3M;lWqI0 zJHCkwVq?8)g8RG1xS?!&M#p+OzRdaRie^u$;pW(;EX^(yTkNRC%~LcQI%fUY8G)~} z+;pY9-Q4K7GbJs8TfP~M%ZtunG#)yxz20NOXP`f7Q{osM)4zEZWcLb(!0sO<`J=FK zs5$eRF5POfn;7de#aItwtmo9mHtGXA{jMJzyIO?PF}=>0iKu(ul4Y6~(NUWRZ^nRg zm7Of@+jt$FC;Sle_QPV{9~~e6K#AFi-a~eHz>C zUxqD7@<;o~E381~2+aJ`CDZhJE6`Ov$T}bXOWe0@)0`h|r}--@9K*EvVBwd(SbSGj z6FRrtSKt<(5qg6cDCL3O#2mGPn4@;{J(ZDodN`l9tsgH^#~PEdj(9|TU(tEUtV8cu&t*N)JuHgR`OH-7ZyHMUZ1%A}-Z<>e==}3e@B;o| z`AB?kGK0~1?9QT_bmRO3aKnKF)^q6k^i5Zr@DL2a%iCIG=e`xutIItNy^o;$Gr2lC zY5{CEJkbqm))R-nD1>f@QZ>EB{YziRQhaptFnk_Rg*W{;0iGTZ-#Nsd-V?c8+M%26 z5ejcxmgICUT6owe?&94KEZl$NNqvT4;K^I8(uvi&jsu6eKKrvQFKQcxEA7|v7H+=K zv%ICzyf?Ush_hcPY#U?6>3z+#<=*&g^eL7h<_cl8Z}57zhokkhYZ-&E58k+UmbFh< z#z&^jKlrAZ z1zGJg2|Dx2VG!u~iFw!zgjusIyB$4JMa)axVR`H2F3$6R>Sla7u5sBN3cE+F;InkK zdF^?=Q0Lht-msh(6fx=U_M~$wOe)X$&`J5=Xz}ZOM%59JFsznqZTH@Ic)=+?u+Vzu zKI0OPaQ1?cXKtpi8#4ljEW5-jc3Yqk-{BeJewm5;W!N}xweHybP&{+mlEsR9wq=7$ zusY+yAS26#6gq!S3 z+APbu>GGywW48Nj2$=oW0|s6z3=bM#)U~so?ArUFHJjEs3@G0i)e??aRmOca(luq; zjD$7b$C*d58M>GGTd=-Uf-&s16{9wye`bjO$wdDIyMfm=SA4>;_?nAug_kzZuBX$p z@~vz{S}R`R+DO<_=?+_aXBeeSnDPyLsL04DgY4K{-#SZ8Wt+XPDF=+ z`RW3ErEM^ti0IDny0cq+Oaa#WgmHL<^+hq*uPZL8c3nrG#rfe%x9<*xSlvxwKy6|! z`rwk8$V0bI*8HqHJAF}GS2jE?1Sse05sIr9)#B$p&$&_ly;l&Ji)X)uCbZH}4vk-x zQO8EZwk`|-%ASrUko=`LzT9Zey0jY#eJ|eQ&8Du^_@1%MY&x_Ydr?m0?fE7aVlf7< z%`d_BW%bXPeWxs|XcGpXOfxk}4MOqNf|AVV`LlF)`;yE_%+GdyGiURr`a=ux3{cD^ z(#1V86ZgoFx^F5AE;<@EFPhFDzjK9WkE_9$H%+;ilVpf{VJ7Z{frdYy7y`}5G-kcT z`wTJX$q@I{Ox#n$sbvvxxRj^H_tMzRy@kG{mwY>$-Jco=$#qL)xOgWp*Fe$Mb0=cW zcCFck(?Png_H8rT+)iL4h6IA9Ne{efUl3lGv*lv^%@EH4n0O8Vr$2t+NhNy1o6nJa zjp)zq7S3UPTL*$)YJR@5r9X7^O5n{#hQmSYJG#3u(QbV#P19TO+I)YvP^5i8Eqw6# zi8w924Qm;0p%eFdT)e+9T+8pqd12Vpi(FpU;=Y@ScMyhf-DmM#a^$ojoW6gN_n`9s z(q6uYk@tR*KP|3}rNnh2x$O!DKbBO3&L7M0Ba0$32WAGdk76IFVS78fbRH7LPHqV@ z;>7_GFmzd@u2!FTH_Bb+tjc&ZJ&Ije9|V+L#z)}J>0uhzHp??9x4fK>Ctts$`xqXI zu3f6(?WN5?Y~r0!(O0Kw_OiNW=jL#HciaRMH2uKMda}#Sshz>ReF@zD@}8zb^WQRk zXFIUtV%@@*iEmwtRvybvUJZs`9*v+!ow{gMtuphz^@%0?H2{{Fm3O;5=Cy8QR;0#u zPZ;ivEx?M`J+Djk5$Rf2g;E>b!DP~Im#6I$SP|DhSYi`DaUbacM=S4RS1NtvnI@OH zQ%NuI`_|2E-r_!R_|R##Ep#<6XmXu*t~mlOU9fY#8ZiPK*Ir^S)8lk>T&uGu%WbuX zcqZy%#p$@VVsWgd?v+sZc)tX{;u8w>-c;ug_dRzDTYfOTNxc%x)ISW?mI;7G9~SZ| z*LrFOUbM?RUa%NzDaJLI$lc;eP7}7}a0rAv^uX{=<*>!Wr3*O&XnNR#vn5^{#hV zs|&-R+MZc%ORlHzuT$LE%^{=lO81V~_1kf_Z{&5Odvn@v%LdxwBApGEynm0=@vA() z#eK|gaGU%dNXIWtVK4q^a6kUEl0VY%>$1fc*Vw1Atm}sH(0Ol7SX8eGR`F@Uek*AL zPpWpuDjob89m~F2$FVs!J{Uh|8lz*GuE-jV(~|%k)gfNqqveccd-IKAP4(+oW4I zF#wmvCono@t<6}#2d9ig^N8u}+HX7~y5($EX=NZR%rM8t%RAw!J-#f^H!8Ee7_(N3 zG0SRK1ME=E0h2nqu_E2yYkVJc1=imJ&*oo}@!X~udpIQw#tyHQIpgsdICx_>zp$gT z+x5q0tl^n3p!RpW9B1u6^@ecI^}I}l6>h7C6=FL#gaHk=Oz?$+H@C8GZvu1|YlOic zqf0aQ-*#qFw(?k>-u1C7E3-BPC|7O0JmZE=+$zit0m{F#V02m+$jZBEbd!eOY_gj?5znsc$R-rA)`>9)#293F-blwLIv!2pXrHaRVf$ZX&f%>L>e49z<*{Q9yQwh1oF3h(LcvcNZrHP{>klpDuHpi={1O?1te zOv{cMIJL=6$6!x6J1@W5G&KaK?=WNHhx>_l#u;M< z@P6ke;74#^B^w5V_{|+eT4P08rNpzBY+Lp+H3Vy8U+~)Yja@ACMMpXDKwp?p@Qdbg z;x~S;Sxb0S=m8TyMl8mgi=lgOsz=k>{ft%eEnP z#bsS&*C#6VDceTYmkgKfA?sAK_?=@tD9a%FtD#)74u0F2wdYq~ z{ZF{;OEO&ioy&52?CQK%*1=m*A4_&;H4(q@f3Wu+U{VzA+HiMucU5(hlVk)5ilT_3 zq}^#S0fI;nRFa4YD5yvV2@4`1AVEN~S+ap3X?JG;$s$1zL4ue71rbmQg7DonGqw9n zv+Mhw?>pze{_|c?SKUul^;5aJx_hUdo5EPbx*Qsg7LW3gR-NjrF;?ZM@zXj=s=9Su zG+oslY+Jm{h;D;wlP*(T)9q1pkTz%NtMXN!WBrYdL99;07|?Q9s4xEZpyKiS1YxQ} z!&JvGkk3bc^ea(Er_*~Xs1vJS?;kq9mWJ`vlva_X?RI$Tyvv~t*S{2*sgonr z?5$Rjv=f)Z`OcUL;l@qxkDQ%-Bn%&-;q@P136FWERpjk=&CqW1Xt+$t`y-Gc+BSNR z4_Frp_hEe^H@u%5I#P#)TXbv{89)0<=%b!2oOhOu<953mk^$R+6+~DwpFD2dsjj&+lE30?-R+rCOJg!FSIk`op4QH0%MLgck@}u|K)P{wpp$G(4gft zC+6-QY18PngtlRR{?{rWxCFZD0Bj%Uou72<=J zr`C8?^(&UH@+m!~ukz`1C4U}0&_AEDJhgY@T+uyOm-E){T;V5%E>CSZVxZbzvfO`A zRz$afhIRSzdZRj7)EmjFUR}Q`0k})&+V+4xXPFdILJDU8)bNepP;~4=E9< zL(|n*qHobL%+zU8BbHB|ct~2}38^o@{h{szb+3m!jHC>!LceooXC(y_tRAS)HasKdk0k=>>E$#-HlwwSo1c z%AqEycs&A-h{aeq-V^#DnE8D zMcqRg^#;q=I`fN62tCuTn?H16-RStKJXe-BS?P4`l>E)DeyCibm&iTaWM$rE-{Y)s zYAFA(uD-o=WT^J@?frc}e3otAG5LB$4%?GL5_3hvfqJ9&JL;{GIloP6(Dm>S^~UpN zt_9UM|D%DC)+5~LKBoJcO?vcxNqcmVzi95Ubi^;80me-ytJ|b>j6#NL^QrTfF^Y(&mJ5BWA8VGPNfM#nHdU$ibgZda{C(&n_%bqD!BRQNTyTD_7rmy8|g|5h|t=;Jrt zS~Ohz#Z%tBWEgB}}WZSknXxj~F+f(IhB$0ll(lBis4U_V-ZRZ@EDa_q8 z_H6+fQE@f@kRcUUVVbUV)wQ@zM&5uv)uH8Nr2jkRsJ19QRafRe{e7U{^|j2@(9a$s z17#@PtICP?m6k$u;@5N@s-^4K*K|Bfop?L`QRm;OOP3SZO{h!f*E(^TrA|OMOC4QK zTsKP{T~1ayf&Bm2cdZ+j33cgfkyxr8?<{o!x(PZu|G%T7b>s3M%fWYblNwYbi}jPB zldnzQ$k=C^1=^8)zcTgzQOA#!nW%%j@pRQfJqQ1mPC}bw{fp(#EK~i>@~XZ^`$gKE zpp%&%ZI2%344sFqqiKELxj)nZ@6UAqsf_oF{5|{noj2Ew?kj4KP$m3WxPAJEKdcFr zeYvZDVXzE0ai8^vp8hOB2YExt8wut`T}h}b9IOjvCZo)7yvz{Fj0DTPnH^19iW3dF z3ceJfdm5=Lx`*jKlj>v_CiOsdLOcJV4SGCP8}xW)TRv$+D7!LA z8`PX;`7EI6+0AKSzB0-oW1ejrX!$fu#$2r*J?07Jkn+QH-bvns`6B&|knstVekF`8 zvCCFA=DxDGS8F?9!=7b2?6@^oKtf)cPHpgM`81u_`zssvjJjxgfCnHe9mS;(wSiad zr{$2klnr~9d>5q6%7&eC7i82$#zfh$>%J>K1Z~&v(|V~s)j?iu`=6js)6oXaL6M+C z(}|6L)V^QO9Zg4C^jdICh>qe<$Wn*YrT7yxSAZJhj4>f?R{RON%?ay)^gYOXkWmg9 zE2X2yR+SS`?NjZi?LOuEPu+()Zvx%+R6j|4#}N4vGljpa>BTiU{GLNG7}!5yCqWBD@n3 z!XuGH!-Pj7jfM%QM1VZ!s^(=g$ANTcB`nVb)4G)#CNd>SS^4?YbOo`)nFCY%rH`KHVWkGiK~cypC* z5yJTpA)F6k!ub#;oDX5b?T}75AChT0;e1FYoDb=Q&mo;~J|q*)hcMM4oDb=Q^C3() zA0mYFAxtNF(RZ(+KB7LYQzqq>^*yNrdwujc`7s6V8W(FyVX%zm>`P5GI@t z>4ftkOgJAxg!3VZa6W_x=R-o6a6Tjx&WB{e`4A$U4qI^ld+NjM)O zg!7>m;d}@a&W9wz`4AzT4-sOA=o8L|FyVag3FkvH;d4kQoDV+XdYt_G0Fg)Oafj;4U2ouhS zG-Au?6V8V);e1FZ_Mr(ng!3Vp*hu<>^MTqz`h@c#nb=zTg!6&gRQiPTAw=wFeZu*W zL~LVy!ug=i18dF)pKv&Y2kVZHb(h0{xI^j!-QiY zL^u|b3CBW+a4dv~4MLdM9fS$TLPD5uETj{Tg*3vkkWM%j!h~ZXOl${~9h2s>@(IU6 zh;S_UgkvE@I2O_f$AV8d7SajF0_9MuK{ysD7t;#Du|T<%Y7veF%F&cWpLc{~LD`~e zt|XsuEKu$vpKvUM3C9BEQ}XG4PPh@mgkvF{J|hUnLK@xY3CBX1mP4N-gkvF0pP__5 zAw)P9(g?>wi0Tmjgb;m(5{`vrszW#ylIe4qa4gg!9198W+k|5wnQ$!lgkvFza4duf z$AV8d7FN=CIl{5v6OM&igk!;{?|6h`Aw=I53CBXhbDwZ5gz5VO;aH#?P(I;Upj=Wu z;aEtg?{$P@Axz&73C9BE`0@$I0_6(x3CBX1a4h(QV}bIvCA`BE&V*#bv5-zU7JS07 z5Tg3@8-#EyP`)ppa4duf$3imUSnvtQLc)6{;aCXK?+C)NkW4rhY7vfw5aC$x3CBV* z;aCU}js>4^EF=+*g$UtTs6{vyB7|dM72#Nj5RQdp!m$t`919`Bu@E5~3u%O7VHM$6 zh!Bp22;o>rBOD76!m*G_I2OXh7PThfSV$+fs1d@k5GEW85yG($Cj1F$gkvE>I2Mwr z4&hjc5RQdp!mALWVZxt~PB<2l3CBX1>JZ!22;o>rBlfBh!m&_`a4e(~j)hu;V-7yMmQFzjp|Cmu|RE9R}qc{YNMJ;I2Iy=V}aVG zMhM3OwM&f z5Fs23A;P5)A~vlNnoc+tl8H@ggm5e*5j)lhu~&@{j)i1muNom73)EgULO2#e#9lQ* zI2J;LV5(=#~`e8PbcdfTiT89(UG z$g{c6B-7^$;Xt4qJ}U_a0_E^oMK};BhfgZuK!^|ygfQVipnN_N!ht~fd?JJcL48AM z4ulBdK=A3ahHxNIKA%*=fe@n48p45)OrJIMnL;=aC~r@Ma3F*U2SS8!AcP4ALWFQ2 zgbAmCPdE@FgaaW&I1nOKpKu+72nRxhrV|c?WWsxpM8kylAVR~0_aH*Ug!dqsh6x8k zgm55)2nRwkeRdEI1j^47Ash$^&kp+c2l~9Ae~Tbo2a4xbbMmNXisnWL6ApxQ`s^Sa z2x)`^A)P)`2nRx#K2rz>0_E+A(7$8QXA0p!NO+FWXUAVd`$vwheK=I|a^7UZQ2<`Z z432^@;U$RBFySRgqG7^IkVeCVmmosJ>ikFSU&dL+z&Xm;Rb@M^?68RqEc6o3J;v>t zuS(Z>37^1KI;w1?mz@r5)HOcr0Gb~lL~Ms^5qMVw;cis#)98>b0q^Q+>5o;&%7Zb5!~)F0tiLNGG=Z8EuP~scrLpl&Nj`L%0UJ zx!65HZ2482GxSy4$eNFr8*6dgmOq-NZB=x6x=%3~!%UqFTYj`(-;*(&xGjIe`VZ

ckHnTA+yNQ3{9$6hAE9Al zzn@VjtzU(SEq{dA@+X9eEq|EU@`vgDMeiwM%dh$sirez5G10dC3H$=YmOsiP5Vz%z zawWuV`BnS1Eq{dA@<)g*e}vfbC$tUo8`0~8*zzMSGo9G-C!`Zw{)BY8W{3@bgsypI zmmarakJletL&T0gBVFA`#Ew2AUA3Fo(TC~UBzE+wUf7Dp%BNvsJ0GE8T~54SZATxh zSKHAi)T`@JWe_|1guKL#9_g9mO!Lw(v4xM&Fx4S;@eyJd?^7LO7awo4YGq19+oY>l ze0-wqMOfSRMwBgh%w{&h9yuc~?AhbGu`+2n)c!i5%uJi~xE*jTFR_PR5wz!x8|rLy66~F+ zjkzvG&C!3+4xNlobZl1W_NY0`!VW!Q{Hac+ow6QdRZ`UEI%*58Vsw1y{L(R`>2&;6 z{@B07KI+tX-!sOLmXlriwEfV}a%}|iX6TT5)%<4E8%WQ}4n2B*=&=o!AMc}2?7`JM z#P6vlHHaNLx!0Q1Aok!9Vuzk>USfN`f`$Y2#%;-)tjwI>CMz@Vmz^vAq2732eJ{lJ z)u1i2nm5(wEYh(j83wg6S8=UR)3XaJ?d)u^VfU)+(6h3YR(9yoHfj6WVB2)MDqq{x z26eKt#m>%lJ3IU9xP7#4PnNdZ@xFwDZHV{X52n+1ymy7j!Y8xJ(JKBZ>TX(YkS&&>nqFmy?x_zNY0r>BO&PITt#AhRn{! zIdg3#+Cj%{oC9@aSyxmiQ~w`zlpQzp6LpX`p037AkMF;wlVNk6InuFwnPn#SE2fvB z6N$Gcp`6UVMBAgsAVY`R#=;IaqV2e;oh(hy&SsYCQ2Sh(PV9x1?Q=$5+1c$<9b)I3 zV6#l>O0W?|-VA%>?82npY;BEcUNR=x)~-bSFZPs51J-j zx$=ZR|Esb1eY3QmQoRQF@Y<=q!k6~#ezEj|<9bA|KYJz?SJzcOt*7OfdNO1ppOUKl zx!;`vcV_9hjMb~kSM_Js7pph3J+XFWw&Uvd1ly(Cr`ti>msLBpUXg95qwOmlm%(_| zUq!S(N~-qKc(8xLxYDQNado_%fBWROLYFa)8$KPIAys?ds4*o}yMIZ)-M|@1KlCl> z*PhoQL*CCb$j zdxli)9C`Gu==_}QItBG!@6X?*>W@h^?x(iDkN&pv`yU$@m9PIb4dc+xXNAT`$6@u8 z-LXze`|X-u7ahNpB7LxquJ=n%Sr<)D!vFE5u5Vba$|>#NlsA^o`(;`*Ua&-~=(uj3 z`C?R#G&)CR=?vR$rdrJ9xBO;`w(x>ydb3#e~ z_IZ1gkFGA|pa1oAh8#Mvq~C48Pto?jy&^R#_tyR;DvR8)GefF=Rsa21vX;^EC*+9s zqr>hSqH2+4_aBSUaX?RM% zw6A5=6R~xeA!of9yeT2}kGq@U9Hr(`XE{H&B;d&8;ZpbXw0I)3-AQvTb`Whf|bSXRZGI5`zR=Dr^fED zM5&)k`aVf~E~t9-x>NhVT6eMMmwsO8=U1lG`+QL9=Z7wbKBrWDu%$4kqMeKfwW>U~$sjD0V@59;~QvNYKbqxA*$&)~kH<9ff*GI1X%?d!PShm_Rw zmz@mmkHL6wU)6E?99PeE`rKFV0qQz^uIspdAJ9^z>*t)7`njlO;`1-_JuUXU#J(5U zul2sJrQWx-)cd@W!RMWt5B*$JQe6)||LF5l>C^WMec$VSKuevT-TQ}rU(qu7{L^v$ zoYYd^KU%7NMZb?|N#8%z_2B!7imQI;=S)my?ys@$rr7f-bKWwaH|qOIl^c96C6<@@ zU8SBs!TnapRr%^WKufhg)c&ibzMr)W?#DW=zY~J;|7O1EcZA;0^**j8-OuU1uF|vm z-pJ~^L%;v2{a*E#?)#bFA$tGUGV?h=p9iX+^tqtgp}$k~KBT37PH3s07h3A?7cKQZ zs3pB0)bm3>Pqb9uH!59A{r!{Gb0)Y?>E}&$Qa^u!QuROhJ`#-6=abT>@!<1H$MyG< zmioTaQkA2hV_MSZS+pK}XQ}z1-&Z=WuIuNUmPO*vIW6_`PD}OurSfU1zt5Bmj;D?X z_jw&x>n`zq3eUs%-$j&E_a&98UV2@{>HC)I7k#e`#+82J`&}^ppS2IE{{B~=7uWWm z*xyh7zubSW?KjvD1Mk=WH~Wp=SG4^9+x{|t{x8@&Q$ktG+I}ouOCugPeA8GR%02Tn zJl7KCykZBU;cMBb&YI|TEtOuZ{zR$zqtavjiq)t3 zd*wF^)h{xCTQpt8)!&JAewCiN4h$nU?=h*yRmHVb{fw^{ zy&f{+3!ryt>Ht3|m439najCAW`lysxCn`>5yq>Gpr_t(FNu~hrMmg;_0@l2VL+9VpM^0gn% zNA;?YwW9fzRM)js*Hv6gt(RF(W_`3=Rj-cIcIa}bRM&NVTAr%v_&p+x{h&N;xzY5X zJdX67mumY#c@E{M^q_q7!(T()QGQT;l+{Gbf>C)n>`dxGr_mKUryIKIJo2#!y19E1G}j)Pi1wI?-> zu2&^fQe|}AY1y>D=P!J-i64|h0(v^G%BzEN-h&(*UFWg*sD`ee_Digu*n0G$^}Jm) zCLb(X%paNm(~y>`92M76r)#OxwY;@T4 zUVfDK@!2B2Dqrb)C?_a)VO&(bTB_^w?;jf-Pc<%Te6>{Ltj1rB3!Mj*uIeeVcW>wu z$WoABpWhvK%F|MfpVHG(J#40#3e`0XHfpqy2WCPZZ#q|zHT@r}?%$lA|L3jO>^A-_HJ)cv95 z3Y1e8{Z#ozpT91e{?xI;8P{9R+!4+9$?!eVd`c?)7JK%F%EjevC~x?LT_Gj6rhXM| z&qvTx?Na)x9jY89Rk~{DBgo(R^IXyR`29Kkg-DN%3*KKcrFu@PxR&aPb|>b_I?sZ`giulhZtWR1AYyW!W6DlcDLD*XWs zzm4WodMd7Du>UHq#=(z|lZvZ-T923Ug6qFN;b2I~Ns#LL)U&buS?OtgJw95h`Bvjc z$5D;nr82*URC#~LWprIc%fbExdGja7qf(EP9>-;$3csuMZAeSqZ#^#KCY=hY>sqR{ ziTh6NW!L1jak#b){^Re1*zZ&7cde|?6=(kLt$!EOzl+n~y{`UUEb(_Q^}A8*oN`QN z))%W+U8m=}R9x4mCH;LUg_r=YFr9`^7t2*5`usc@8au z=dyHMpW6ybeQriewf^Yutl6Er37*f<=X7+sDo5Y1N~(3Q&(&y|)p;7FN6+1;e(Up0 zTI%y2TI%ypS|*-*x_0iDo{!h((1KF`&KQ(x9`$)JE$KNhbv<~FOrICiQlGoilAg0u z`Sm%ypw#E`5~Y4_{P&&vrTc$ye-6e~dxQIbFi!V>r5}v{(|(^-eTn*4Ki~9oib{Ph zOv~W8Fuh;sb6;BOb6=|6*Usf8+VjwP*5{hE44!LJvN<7r1sV zH)d~h?Oblmen{`DTGIXa+PU1=IbC&M(f1Ga9uc!6OniQ*eKFCVG5ha?#PSlq8}vRK zlzQLQQnlmSx!m#i{a@P$>hri-UOSgd?GyF+T(zEq?-9W`eLg9D8rRRMpw!PTE%kX^ zEmb-CIi}_Ru5-DG-}Axut6*I9H}QQd82``OM^u05_g?TjPJhqobbXFLDD`=ME!DW{ z^ZZ(>^ZY9P+PVJ!`nmpV=W73_&ei_+?hmow&945Plb&x==LmE>c%DtA)AMgS9z5@= zc}459P<& zmx6j?=L_jM#B1kJW8DZ)l=h0<3T;0U(4Y6 zKphXBBh>NWc|{!$o@>&*FW7P5Bc}X1)o~zVx z-488wf3?)@&{DToOI?nZx;`!SxN51#UrRkMTI%uBQuj+s-G43B`GsreH8am`2G42g zxEiNx=QXdL*VN}UwY+v-^Z)g{raG5-_4~bkkG)Fb`IT_)P{;A!uFo3=<^TNo!~gB= zzjl67t+TAof9i99TB_&3weypi=SS6eTsuE`?VP3hpt*M5^uKQ%n1*F=IM>Jtm&eEt zSI8&=SHdU_SJtQiSJ9{hSJ}7&?k>ZG<5B(mQO(j(ol4QGW;DGBQgXuIVb}&UY65E- zcNxW_1G++aKge9UNR?fxQuHW;8bX0Q(@;*m%h3 z3+#*7L&k$fKVUz^9yID2{ek@vt83h6yaap+vHOfV#sJ^|#OmPB0t0~q5euOoU4dN@ z`^I?3m_Zhp54~%{AwanwjZ^mMb@?wnhZ^jv85pWS=r_rwzV2bex#<7689=P84 z%qUmu8S9NgWu5E+ zXD&2*noj|rGCP`&n9a*&5i|EWsw3-OU%xN$}^*skr_q@KJN8+0T5% zJPtf=zGSvAo0~VV7VvG&56p390rmm>VRI&~v<0>`H=A#pspci%B~!8{<`nY}vk81( zbFn$Y+y>lc9x#WSi_BtdIDEP}*nG=e4_t4im@CZQ<_o|VOoR0>mz!UkJ>c7$1I+hK zn+<>;WcD%_m|cKf%;(KfxH1Sh$n;oK)YI2&3jZ6*Pd6jLh&jR>X|6Wc0@s?a!mUNW zUxjam`rb0%1iop02KS~p3pmT%2saBo*$BTGZJl9G2TnJ)!c8})0jHTe;HF`8cEImM z`<9!_fXmFU%-!av=2GBN^BZ%Yxdgr42Y&!}=X#?#`wae3Hpl$l{K=dHKi8aR9xNTH<-=dGvCE%zH2i4d)Isi@*R^i4tyK(ZOq*$ zjA?FQZZ^goZRTS6*l05!8*9F1=4A!gYnb5OL&M?lI z-ON+QN#hvsnE9*uCGLxQ>`V9;*?QcE&4A5VbN00nGT#Nh%lfjs=2)YTnb+)N{%)Qy zP8q)fe>G2G#+#Wd*{kqhuyxpjn*f`zW~?A|e>V%lpT_tOHP>JjtzpaA1f!z)r8&X) z(tHfN=mFzZwgkR9TZ4P;A>c!-F{^IwGP(o1vsvs$v%b0Cyb=Bj^E#Aw8hG0L6L(_R zXuu-y53!-xckTw>&E97#aevA3sv1>VaZVEK{nPcuLKIWrYIL~WJ|e;<3p zC}wUk-!QhAPngTG*Vbao;X|y5`KB?%EMg8ZH<XCsWR<{i6Ph*_%xcMji8uL!GidmT5315)iihQ4&x595X zYnbmDoq?UrqvqSjon}Gywo#Bh4eiQiQC0=MFslgN?Pf*zFR`yJF-usB;fq-fFrRzO z2JpKui*K8Mu~hhqmTTrVOEMR}ICg_YW)TZJgLR$t5q1cL*I5Ou+-6R*G|LTNk~KmZ zyUa%Ld$G5CY!lb69(ck<<{)dq{2Vg zSyrAs#*P7xv1P0i(#`_Uvh%DfdzM`QUSJp5q&MSTa9+CMk@xeeoR;c;B#8@Q0I9z5B_dz zID47p0p_vtTO-+UrZB%%$Qs2)GKGb#!q&5F2D=$}vvq-uL!Cvfaqz{hg=otawh+FW z^*XK;2Nt(VS(Di7OkpXjj5UQ#X61n8tP0kLY!WL2EMwKMrm-ok0`nGIyVZIVzLNC_Ys$W7kHG)PT0koNksZNI*GI`c z;Fg9h4u0?@Mm@cHGROUSP#N~!`dM21n>kqi51f(iYKvZnzFgqS?UNtO47MxA@k;)~qeth;IOeZCN|E7T-&AfOA+AwiVw%?SbuC2lg`Fk=6m% zv9;_Ad`olyc3>UZAiTRj3Vf86W5e)HwE?(+tz)0#+oL0}BkRmU=8xuWRtWxGyoHW1 zw*a@WO>76guATut!@95y_%^!*cnfREzBQ|xvw*YME!If9O?<&d!hgSR~cBDwZ|LO1oI&9AluJ&V4F>^Fw6L-4P#NqBGk0r&&^j(v+SjA6iG zYy>-GerL`$55d0=jW^7n*kt%0@W%InX<6go_pt-`9vJ}~!CqzC@GUeGIFmiY-oQKR zQQ%Sb6Z;-trK5qP*=uYVzJ*=@zQ7i+sd#@r4m{3&VLuu_7-NBB*?86lZ;ShZ``I1V zG`w&B%BH~|XWtn=854jL*hDrR@8rJ$e`CM0WB8VR1Na7;%DUt2_dDQstg1B{--;92 zX!wuWOuWzk4*Z>+WX*z0zYL(*-?DUO$SbAGuaVi zl(7`Jl%WQztu@S=3jeRDiRtLC_Ru^Cw>lu{(tko6R)#_w*ww^=2&hSqnXBTMowqCLZ z00&t8;RaZPfrG6qZ7W7Z1mkoA{! z1bD>y%$jSRvd#m~TVGp$T05+bz>U`TNSSXf1TM7ZSo5q8t&e~oSqrR1)?&y-@E;bjj?0d9gJ#f8s z7;ZhrdnI!Qqk6}@Y~=vA@?@;9caxl>p1W@#%Yyx!a4~& ziE)|(JOw<3RyVaCw!Q#*LF;qi=jeSd zyS4qKoeRE%?b_VV0nB0Nh09^*1m?7J+qc+0r#*aWl$+142&{;*^Wkb~U}?mzvy0i4ftBq_2o<;Q0N#OEal43p7w|5` zir{)BU?s$gA#W{UExS5G`Roudgjhc4)&bT*tT?V$2UbU{wtct#81ON>fqlJQ#%=^` zWH+&E+R64qz=!Nd?Q-^w_C3IR?ECDS>{51PU}O7U`)2!AyD6}#{fK?LeW(2Z@BzD? zeS=-zz8`qM{jgotzQt}1Y;M=JpR%7tkDs-l2DY?2*v|l;v7fNp+MO{!ZQ)xZ#kTVU z^W*wp`#JkX;EVQ~cB=iYeGu+P`zYLIdlTGpq^2TGrEEf|i`~bb3!H1eV2^}U_=3H{ z?rJ}8r@#%fN5BoWd&9kC4}$B9Yh7_grT0dB1bkQfU3-_k6K;>a5AGv-A>2%)zKb-K zvJjzeb`Sdi++jNnZYAOeAiLS4?TPkOxasy}xaXtgziN+#n}BPhaYdzfMSL>+X!LNB z{Tc8x^ztK=y%(+r%IRj$LwqmfNA~OXDEm|3r}hT){HXmCTu-|@+&aX6g4_(>)7}pE zrTq@v+lX(6>~7DpKY;weo&%g?zXhCcFG87%?4`h^_J_b_b_8vY*lU4n?Nz|__U9<` zb9)uUVs@{VXpzMv0t;t*sJX@Fl>*v$Jy!V-#GYD7>h~vG~hIQ3*0n&3UG@3 zg}v2&1Ea7Negn!Hf;xM_FF;RsW5#ydUje_ecL4X;-(W1ivG)V_+g}3@*xz9VeP@3U z{N6qU{J}nku{>t~4E)(X0&M6scltRE;d?mEoD=pbyP0#!e#tp*|7t(tJmNg=w19ia zX$to+Qja55r8GtOjNRIK**Rms?3}e*I-Q*nPD}V<&eP7H_Bs1$=bSy#Ic@)8cW^p5 z&pLzP1|Z%6^0YnJdB*7k*UD)J*VB0suD>%Fu2VGSMTB}ePdIJiez#A;J&$->=cL`s zxoDftW&5Ii**T&biU4;@slg1iZ<)&8g(v;XDX@(7DTL;N0ol4tI}pKU_V;Z-;E)JmB2x)CAUa zsyUUNicSa^a_Ts>oco;Gz}il-Q`M>O)CJacsyn`OvvVul-A)a-B*brp^f7i%VT7J? z+5kH^T`;0uoQ}X3oZcAG-cEO5e`hGhd#E!I*vxs!83G*QJmx&=q+re;g>UY3a~e5K zfK8mfP9LW+Rzn~79+=C{PFG-8=T&Ev^Bm@N6#Picd26RVu)QR?&kbvPjJRMW8hXh>2Rx@<#0=#MQ|TE3*hEB@4>z8%z~TlOo5x^On@5`)n6Xf zToBcn6}>tkn!X4r>F`sWan2fNJ#fAAnv>?7aZUnHIxC#H&Ov7%aG&$O^SZOqSqoh2 zjCUf=pU!FEX(!d0=X~oN03L8=JCmI)&L-d{XQDIOS?Bx#{KE-53!U$s!@$GNd}pfj zh4VS^bLS1`W9KL52jCCRht5oAr?U;X&6(ybc8)nmfJdB9oVT3a&JN%XXNI%PIpO>a z{MlLJyz6}J>;mp`-gH(vzd6T&$DL1|51enDJ-|KAJI-g$Ip;j^yt4uBymJ9~!8r?e z!TAgLm-9Q^U(Q9~MduXUMduRmlJhIvCFgJ8-_9>^e><0fmz|?+e)9ngd{`61^JkU=uO-`NkeAT7Sv*$cEGZT^+>70`in_?OO?Kn}_Ic4s?KKnlLq z*$R}9l5ci41J41^p+ArF0{lk)IDBdD@tnLcurM#mllgG`hMCN#86$WdJ_5gC*5Naa z@%(;VFA6Nm=kcM&ZTwl_vwV_~i|665@Lcf4coSZozXW`VFXQ!*Rt#8-PvsBsjywX4 z@OQY&@8M&CV|hXT1b=|f0nXu@`5n9!{}}i&@4-hJ5ArvFZ}3^h(>x!)l|K!?hUe$) z_(tGHUV-Q4jrd!@w|GgOmp{VG1IzPDyc*I<0!#7%JO}D22rS6EqU0XDFR(AalUL>a zcyC~DemB31_d>t#g3pOMALI>z4f%AqhP(l=0iOxi06m!rKNW3#fY$@o;|t*G@w&je z{6o097@ZH{=b?S=cn4qyzLkH@pW^L-?fG`TjX#OrZiC+p?d{?VQI~%q?&r;U8ea)q z$sgs9@m0JWupDoOF}{sg0aoG9!&Tw818?Uq@@~8`desfSE5`92ejo5Y{yN-!{9fR_ zd=lKfn7>Kz<1y|Xd1qi}{ux|n-U--=uZQb|kzEhJ2IJU>HwHH5@4+?Z4+9_OAHY3~ znfn0#9gOM#J`^~V*W@+$5S{`|;Yqv}AB;J#1z(-F=stg#V%3~&q&!HwagfTMVA?({{}%Wy|CR6Jd-y)!KK>=Q#5mE~ zwBXyALUa)?0$;@MnK{ILVh_&&zl-<7@0V>wKltp zY2azTgJ0l(@jFod9pZLzkze9>0q+ubic9G8KE(GSR$bgJ@&WUq-n&Ijkr$X3v6|>h z9$+5C>Ih%t0_H-SeQ~eI3CxMuy%?Vyz#ND@DC&udz=~*RUGb2(5qP7xNi;-%%LB`c z3gRBVMQj3Y68G_9qPTcW6oYRrN{Eu;2rmKuxM(X{i*vj!{CVD5JSols&+>Ep3DI8c z1n%U2VjQ0quIL2sA@q!pK#ABhm^%R!h`k`56So3y#rQubUKEvpl@NOoV|xqm7R369 zUZN1N5XQ5Y=qCyS3nJDJb5{UZ0I|RM6+Q$wL<|&{xhaMLhlvznV7`X~hl`g*X>o&i z75J+7ftL|w#b{9mek4NWFu!HtM+-~XVghgi)|xH2m;{`Jn1eMm88}(I4t!8nl#Zw< zck>xWH}hFB!+2Iq;4koYqLp}ow-V1`C;U;QiY7c&RL7mUNo>GaZV>nK7W}vaiMSJ2i!gMq89G>m13Dl z;>$!0K7l_gI-}p6k-sKiE|!Rze2J*Wi}U?FSrmt#hFx}%_(6kbCNF{kh$ z_&swi?<$@bYx(mcj9vLdu|V93mfwN9q_=!qmJ)Bvlh^_0i8-i$j;O+S^NR9jjQ7p* zE9?qy$(Q7>{3Urkaz8Kn1N)1&`8wWBbQSAR@>=YebHoS8_kp;bzsY-x-+{mL61bzv z$r~}QH_BbO^GeGb>WcX?0I8`xXC$#3C5qoiBlQ6jG_s)+8qis-{DiShDvSqZsr z6?ynuVhwPO@cGT+HTkBzS-dGv@b`IN(NnyS(%$8LkgF>4RTVw4!;F>VQRjH%&&g+s zPk^6@yLd%0TD~SLqJ*2#`~IRYurKn@;MGJ8F$1GL1uN?mR#qYSE_^tDL+l6c#|)Lg zXbcb~;0N-7z(J6M#LIjZuO+I9Stwx!zmD(4%DN7|2(QQw^25Nxyc8eC2aBO%7#}JI z@oAXlTA1fr$Wxa8#E$`w@tg6KdQVIfmH9N0gAan{E6{yK^yl+=NYobdu>xoF8^uUD zO5P|&$(wLzcabyXbz%m_y(fCT7q}O7FXkh}D6yE25})ve{2mb!3sJ&+zJj+DKLUT` zH}LEEchI^Xelh<9nq#0lMtsEQ;*Pin_ryKOQ(n9xN67MGgsdR8@CQVFV12QH--a1{ z8guxxEFebmv0^-Mym*b56+`69vaEPnmcuSFNDh%@kb*q|S0@4|iqZTF-atGczCdYP zcxf>}4wj|GV0i;}==Sn1E5EqQ8f=b+<{Qv`Lk#DeaK|;kJ=XwvN{F7amnOZke-jh>cA@5hMBz{*&Eg~T(mi!6i@zD{`jO)(QV6Kg1yw-7Bw zDn@bzFDN?6XJkQW7Q!sOA{ql5i%;>MF-=Ut98M7~p8iY3b+Q`2PUhy%VV7%zeXb2! zxeIqwbKFnOWezdjXl0HCj>R|@z^Hb@xOPIh`9wR}UgpEdV3u~j@G zE6A;)f}AD36P;y8`5mr)D}E5q%Fgl!qvM73ai@XVVlYCp0knhUAvIP7o@xHiCmX+_L zgy~`ia@9b-8uDZDHZPL%zrEKv@w{SKcop z;(nPf)}qF`sIxBO^HK6plsyzDfs;s|E> zUd;2osN-|2q9#~JO>k|ScvLefO};L^3U-N|;sK=8!#npnF^ARRbJz%@ z0smTjB^vOrM16ji-zqE1v%IoAhp&r$;%m_mR~z6fWs2B>dut2cx6a{e6FaX>uG4~U2G?eT?p6W=I9)M-TAJJS6Xtm-#)?#2Zu{d7m`IeUjnLx{9o9o#&OUe&(b6SMiH@l>Z`{;~VKK z@h~Jjo4t8F(4} z&y8K~dyyL>(+FP{&1DPhZ7pO@@f5eDDWBq|Y|q=6cUTX|Hs%B9XHHyw4A&o%xkNk8 zr7hcWTXx{NkkS%qEwL|m;yI)%J8@S&&AXdbt%kC@*%1B7&x=Ve^CMR~e3i78ZSfpv zEAxuZJeSNNJ0n*oo);;I=mh z&G6e1`cm!!?m}#f+zGh_{!4_mLUSklF8Ce5X4X>F-popuE$|CwI$HLqwOl@iUocmo z#gC&m&8?r&mbK{bT6tWaKs(cbY4SID3b+)wRGvY5Uchgd8)XmcbF`-?e#87+zG!Vn zTe{;n%U65bn zH_Qw2ymhx*%e~1h0xaT|agiebmKWeJ%D?0#c?EbyUY5+g49VOp5{III9LQZ8r8}+w z3fFUUxYGSw=72wsoE%!Y+?;McHy3$~@$ zl|H0ZX|b$p`TJ@&>oGy8u@g$=&i+x3ariR(8LUkKp%9(|rW~ zQ+Y&|cW-o$$Q#}7xcbPvcsX?)(0$O&3CxM#GdsAix`W*g z@SWW5?oc-cnBw+wpL55#V}N7a7u=cdEcbD^)@}j&arl|Y`Jg)y?hSVWoP%F6Cqm}5 z8@P>8X9M{9?gQ>ah&=$`z!muYvJtS6ThE>9Hgj9RJ&saaz)wYqt=%VpPq_K;YvU70 z&1biO*28Xl_i5nMZWFkt-Oj+yZgKny+1c#^*~Kk_Uog8stBBnmdQIHt-4}r`B5yNb zPhihznLXV;kbR{XFz~yF=Wuz_D&ux4ApQ9R?icKI(RJUvpmuzU=mJTe|(- zzQDe28@H|7&m9gN?mptSb4R%&fg|0g+>Y)bcK~pJ`;7aT`-=M#@FlmE`>Z?A9St1q zKIu+FPv3@{<<4-&yOZ6wfN#0e-AS06DZnZ2n@E}N=C{Yg+xDmKZg-CRDg1kG0k4pE z+CA<5>RyJs;+FJEdd0i~aHpdwB@ueZ{lGosz61ZE8+K137KVSvUFN>;z63F+<`{BNJ55j%ze&rrU4M*XAbbo>S(fz?aftLRXcg8&rcg8*CUPKKr%s|=r@g4X# zW?`j!0C)g%veHd+zX5)OSgO0+-3i?3?!&krahCv>VElh_e|A3se&Q~6kGbEu3xEsU zk1->kx#xiAFh6VE4elSnKisqKdi3o$@VNUs#=`Kn1Gi&L4A1hu0DgfO^Dbc&n77H@ zin6{#oeSWXqnECy{<&TbIM?Httvp^{xIA7yxIA7i%ugY&FkB(82wWkrpjQ+lSqiSC zcRgH5uY}jro8i3!e8(H;z2v>;;Wte96ZQb_SNvvq-0tJ`hwJ4Hf*axufIDuhltBoc zvCrBIfD61~-cWC$_c8EeZ@l-K_lbATehvOldyMyoeH!k4Z#LZP-Wcz+JsUXAo8ZlX zd)J!=H`5yl_cBt)Ax))>L}<7-!kY#6hNphV{M{ao@JV1x9`Nb|>w7i58eToG5wH>di>cb)!(KyRL+@Viaj%8f z7}(fr<~`!I@wx-Mdo8`kyjEThxNhEaaL*#%1M)HNdG86YJ+Qsk#e39y#_I^|=r!@4 z@}BlO13P1ssKW{}A&&6gbp-)%(T%&7KOJigK2EOT16v zmU+wJR(LDnR(Yv#X6-V{ zsj056t`5_ucjk4=e-E6ukzUF9EN4y5r#ODd`5wo2Ip5;=CTBB_uXDb_u_9d?4bGGDc#Sv82I8^0- za{kHrJ7))uzjC(Y_%r7Z9KYvm!|_|ruQ-0m`5DJgxbjC_M_2u0<+oeOZC2)I`>n1_ zdj0@s2hxu@+kWm+)ZwK5alu{}#S-s*c9)_%?-l-_O_MCX+s54`U*S^}RsC)lmah+QR$7SJlEso3Z3G=lRat|+o zDX*5uxBBaSZN7}zwo9hw#}ac{Zf+K5%0-yVPVy4pesR~LVFUYE{Sv?SR@b5%o2Fv2 zN8+nvM`mG`Gome@{h!v7x`1-3$E-|bbrkO*8dXUwWVv()0GF>`jR;M zt6R~BRS&lHBJuM>yB56~4Y7J8-tUVpMR)9Rrj?iY`)j)tU7R}4)}zG74(w7iFTB9k zV_BH|zQR0s$ zbuW6pcAl-rvM|*h^9l2{RF4v~4b;BaPHIzZ1Jy5=i)4I0XCG2tj7$5|=Qqxme8u*V zf7k}{3ENEd!sR0Uj7yi1+X!qClm8{=*r587m}7&kl~}}TBC&|uL}G5^WT(U&!&H{CF!_P`g!x+X zo5UjKZN9D#$=F8q!gdnoGE&`1%swF*iMi}lj}nXal1R+$g={VhQ+b(Bn6D-ONX#}+ zeX*TXM{EPt3zv&zd_HF%N`9|s3Cfk081|=8t^|)3QG%b6e4!_nG9q8{A?H=1E~Jbomsh5go=p8h zPkB9}PKE#fpVcGPKgBV{!2cxdYcC=WOSCD;7x7l2Z6ZFo-zd?RQ7_zYP`zZZ>>mW5 z+E=DO5jv@T@wF*o1GO*Cm)cI5?LzW_GkvDSvOP*H+nvO)hx-lc4>MTy4}wqqVWvM3 zdP>Bt$hSlvCG0HGr-^cvsIPy~!~Q8zcaY~kq(t2b-;$l||MGft@GD@iaq))DllqY>HH<0*|*RVUC+cot+sJZG{uXm31=ke)x;1+)vE8`)p=!&4a#!?P9#;5Y)$i98Zd zT|64cF?eF+v3LUFKpe;8sfZ`w$&V-DI2qUU#}y=hq?J6@%A8=o?Qi)J!$$7}I)kkgQc;z^Kqsi~k- z@f^t0@jQ?lKyR?mhP)fkCcGHw5M%-{ zH?3WpTJm(oczC!j)nn<;Df&!u^!=@OZcP#Yzban=eF6PZdpCIYEB-YVEtj4Et1O4w zI3CxCR#?S8Pqngb#q{)5`m2IgTN$ldsXGtoQ1rrcXT`q^te*GEg7!t1O&DwOj2T^v z<_xMAFB()mFK*hUNLOrc@vFb#_o`Rp_=6Lx=lwOdW6_q|kF)sq$J-aR#I>nm_2Y}$ zb|{(-o$Y{UHtABdwnF>3H}G8Od<;4}0Fl_3 zhpD^ZN^KsFx4-bT<(I}RPyDj*&E4;fhc>yvUhBiTedlCkeE7SUo{5j%zR>EC`1pSo z#XmUPB;L{dnYhtui>X6!Rq}pNw_+O5Aed+_*v0VtF4v zbj9rWAZLRQlg;cmiK&dQ|2{oaUdCi|iw}zHNMb6Zy6c`yc?nN?a(bpthitz$*>Kfd zo7aC<&q&seEUxSIkKLQxd(<4uCmE{8)|V}|*K%Ex4B=WcpNb_WJ;c9i)YGxVr00?c zABrnnaZl{SWE)*eWtW)r=bbk{ZhXSsHm|ZU<+b{^`SHT5rrLaWhe`6@eyG<&NTZzfGfy3^z@;*#@{&{6qMkiy+mt$LE(sO@>*%_T>VX~q5mSPznrhGdr zS`;^Ob;oUi^so46Q5?CtV@!Jf`DIajiK{!tq-U@3Psi<^SZM2o+Zx%-m~@7-m&D2Y z3vC@SCY>Y4J`-=>y3p1OW74_4(^K&aHH+hl+a2jCc>SsPwED$y#hCOI^n5y=`O#BW z4`b5ve4VG`_3al~+qhkmp3@pFiLZQlk+p|0>AC;YCGohfi>)5UT*mJYo{=ddwJ$E$ zkft+AU@lj)a7LzFgh@{)_?+UL{KM^+^sI!>IffaN9`);@Ox!XiJujeK@ycnLa&i0R zc1=2mqKvcJPRo>KR^dG*FzMLBkZp`f5Ba%h&E57|^6k|POJdTa(N}!< zPjL)zzem@$MPKpIj^g?)LLXA?uacOqtf(+2XYvDM((}Sz z55||f*kepOUwm|Kywt@WV=-PAit)NojMs%?ye<^ub)guq3&nVyi19iR<8>m&>qLy# zi5RaFF<$S2UB7Mlr7`;348JVw`bjI}!>%8*GCu733M(VA>z6EFV%L@}Ut-r+SiZzA zt}I_-7ps;pv8#8>_hA=fR>p^28@DnNQyI~RSiZ!r-C4fGu3cL`V~S6*nd*gN*p1f- z#iSdr6N+0mURxQeUyRoYUCVV%GK4W+TY1K$hcL!#tCKP1<;H8PvnWp zyvo9quN$LnzPrPupJPvAveS*%)=tKx-;LK+KV!1Zjn~$;vM}j%W3<)DnDXV=mY8(9 zG1}@Z3zI!=ytej~g(+V*UR!^1TOj>zjJEn2lTJ5ATb+zaj~lP89&T%7Gh@=}#%QaP zG3j(;wAIO&bh`1{>g0AudfXUo^)MzqZj82i7?U11URyofu1Sv@qpcpsq{ofXRu5w? zBgX4Y8L6#txiDUrz+5hj(V21)COvMvwtBezk{&lkTRn_Pj~k<{9>%1{jn|oSVZ1JZ zNv9jHGiAYeT>_IHH(uLoxqqO#V@!J7cx|tB<8=v4dfa$z^>DvO*ShiA%DeHp1g2}< zcx|uczLc(YW3=@NWAcd`qpcpsq{ofdRuA`|q{ofdR^E-#B`{s<#%p^m_s`@9#-zuM z(bgWuWRDx8txm?z)ft`i*nCIopQn!LA@RU|W0R3LUS`)dByJ1&kH5Pk)ne&8-6fXS zN-VFHSY9hJ-Tk-!c|onf!z#?yWOCerv~z>D|aAkn!pNS3Ot1 zJw6^WC)(v&;?Z}d^SuyzV4lNzOKCpw=)2PVoLN!&zB2SccT!fqmrS@O-T2Tk(4*8Z zrb~jfi_V>Vr68N&8lU}1X?t9Gck!pQe;0XI-V*$oEppNY1!I#pryQP|eb$x9CF3^~ zPU$>5_xUDKa%z(klIf#6q#89mBYESpZ_}NpFUzB{QhBJnNXT0$vV+P&n94}yAWUT; z9}p%#NSXO(U6pL9-YBob)8`lZ@Y-=z(mOsH7JuGiW%{YBs#rZ8?;KBlv%;%!TihTl$YnKH9HYdfCa>%!$PTZ``$U}1>X&eO^21X+-#dH?`4OKNrDlh zQhkFrOI$0NmMwgHwJ*0zlksi`Px+Er@q>uEHPSQtuNhj$ey;A1N z9^;c{$4t#THV#rBwXK+5`ofM>-4%o5{rB#k%vtkfYDR^<;tn5-Np3t6??Uf#YO4CD zH>8Ix_$_t!?Bh$wkRH-Ox=0V{AYG(~bdWC6Lpn$o=^-7YOUj5cE*IrW3!4k;3?Gx+ zb;N{JRYn`7H;q!#_Kt~F?tRCm*MASv{^klTdM=GZe82UhN-rP>bZGC zl5nO<>eanD@5c?-r)Nzcm8{wkruy8`EqSz7hvcgX3-X*#mWw#FG30zwDB`eC#G#EF zDdT*S7ImH2@)Y|aO=WO?NJRbGcyfLyq*z(sx?+02!X4>VD+k9CyYi+*?4?EQ*|?E1 zVr+d*VmY=-EXLMnC6?DpEU%SVUMn&68E(8rAL1LA%jCy&tsAe=r~Ru8jn_W?|Ek9~ ze!1})eSFD$%NoDPPTzRv#_L`5&=^)cc9qmm;{=agG{zK-c~pT7AVp3YVliIJ@i7+TbqRSfUZ=%)UE*3X zUgvR6RL^3(M*Z?%L5cE;@w&vdvh1>q|F6al-&p1w*Ti`39oKy0h#VutcpU@Dd*>m-=l@r8BvGys^AIs!r^I;8t5UZ=%)UFdv5<2BD=XuRh6360l0 zhoSMB=P)#0mymHjq4ApQn#Sw2^8=07T)#A4^Sp(|Yo52zc%2qwbRw}BixU@nG+tYs zq!05vp2N_Xo^0zJ#Z~^wkH5tOI_P}r=pu^NF~Q&IOPo{*D9xO3|6#S4=$tpPSdI6%W1r$2>l_{Hmt?zVw;pOP@>2 z@#n`bZV*@c`;9z4UskJLTm$*=xxy(8;*p20%j5Hn{p!c-f7p`8=UWz4j|(8rXPfUo zZ{+d$@^)3@8@6o8D}4^W%!!BgTbEb*Jn6-{wp@I^xkH1v;P5HAe6IUzy;$veOiwZ5y=UZv07J}Isz-@^O8ozcJq1GOT}@( z=S8j@rO(e=JC))y+r{UtvufD#mp%ut728+(+~C`~@u~;cXX4_+iRIPqIjx-xEi&oeth2VuV>DefWPpsH}d$rX>q-{)sLTL;;CLx zKQ6j;Qy!oHJhXnS7HrAm^TVy7Z_VmVJk~43_eZYD`8*KurH+}B%jeDy*Nuw?Rm|o1+vgd?<8u>x@p-h3&riDY`KGTBkB=Xp zdnV$rz{Me*4;T%*>UYdNxH9+?YGux6j)43>jdE@L*4P=xM=q%s7woyP*YaQR{qG}t z@_Asbs_{J+yqRfFL*B0(-_z;Myt$9&#NChCd4A{5>T$m@>oe`}u+`P$`SIpFJ}*Z* ztugt>JU)MYTGjZrS5{}*U*7|(#J?Z9ChxWHa^jH(t;?K$KB-#V@3qft`z;Xdxbm{9 zalyz^~C4${;{~9lH+ITevvYj*LJfWw4vMIre4mhdtFTaa6@F>RR9trzOzd*qz-Sv^(|@cgCLHdvM%`UBgqb zXFG+Xv*k8twQ(GVeb7f>7xdvcj91bez0;izT#?eO_Pq(&-6jze$^!@lGZ*o!f!*D0kj7yLX(#N` zu7b2D?5K(zy)SlXPXnEXee8R~ zYcoM-Vpn@(?4>>*^nB!ZDfXau0PTRi+YNv(1-%q|sT*J~^+?c>kn0S)j{!Xf``)`^ zr}&YeM`G7`7tsEo{bA#AC}UsHzSt{%BKFhw1MP=>;DfNcJ|8q6<-Hny8VfoWyT7l& z{_)E}FUKzLYq5{~QqW6b!}a+78qjO7n|l)Ib)eT_zxGY|ej?~Zl?^+t-`@aw1AHEusNatX;`NekU z`6%p(KNtJZM}m%oFGnC+hJy}=)Wz5HadbsZ-->iQO5O!E zejCzcME4b_g&3(X{wHBPYN0pMq4-Y-`cHsEkm&yuT0pW2QYY-EpNk#zk0L#S9rp9E zr~YxI$FS#qK6cqJKzagiwR{Qt@fYLm%unKY1#jkj3q0aH3G|}srauM!6g%v5@s8q; z)JN)byxDRM-n4rVj$F%GgLg+igZFFyj$?=V6UT3A3yz=Fk2t5X`I=0^1t{?p)Dyb*mN(uc@-Io5HPAU&zt>SmB@i}W7ea{99BsNY1&!<(TGz+1}q zL;3)3PJKh=>DQ3@=oj&R)!v}JbvOMya;pYf4eu@AOYf^cz`Mpj!23%JRYiO*#3#bf zt384D)RkcWCwPvb!nhIqU5zR0&VQe$|m9k^|f_5nH&Qtgmh;VsMu zplt1tTH{UO&B1Mo)JE^ETS97Yq;`1ocQf4nm{pz{++zFK4p`F6FT|m3wQ;csrf_BuM zQTI7`i+cs6+IkPX+qynzeO(8S$EpMmS4OIbZ|dUB^RR`oy}_kfQ+!#i7hf%bxr)*&Xlfp&wBI-s1Ns18UU zsZJ>M$Dkjh?mMH@AA){}y8i+--W(C%9QCnDeXCl6wnTk=gXnDr+6?uv5OLBGw4;7m zU3yaU{4a-1j_<2;XuRUF7Wu8$jj?#xmKOOxyj0AWcuAjq@^hz+wK5X3p6v(j9X~O< zd4A_JCdGMG8^q_G(LDdLb+^Qgnl>(hr;Kl&U*(*er`Kk@%h2E%ImlJl5)bphwI6R8pLm#B4L^k@(s@TjtkYQJk;Dga2reza)3E z)#<}rUeYP?m4CF%KXPD!wO`^Fd+d|{*yER3of1!PvQPfEv&PwS`7rwv^^&kp2yY@Z@`$><)qAn7NMO`F5%;hDW z5{tS>Bo=j%NG$3i@!==ue_goz#4GKZ7H}SLLhGeX&l(>;_)bmxJ4!Y(hfLd4xaxp$ zasIJUy6t&mmUn7fF|}>!_~gpba|@6Ds8jqZ_K2MJ@cP169v>ZlP;gE9$s^7$JiU&J zuWfTm(!Jw|+=7#X^m|{7PcEIlDetycP^G4%W z^N#*?a8hTIN~E38vmAO-q(}0hvk*FC(kc1B1m_lRp7>X)ey>wY$op(Y8S_xagvu!O zpuD*#Z<@+0`KK*=HSe1#Lz3u+ze~vb$|bMO#^>3{haTj z>ZOTQ?3q9FnfZJs{-AqTr8Y0D6ED1Ob*ddcUw8X^ne)xR*R}kd+STJH@I76366DW& zy{i4pXX0Oj>%MV%_{{qGO!{i{S)JnZH_hu8Usvi(`p7QMkMqx*A1KZ*$DPTJD$rZ{ zT)N!+na`xZw7>o*`=c_}B5GpYp(gIPE1(|@K|`z=MDEUiZ+quo19#nfVl{zQA_9CD z+57)S_FkOU1tR;OpAA48VEv*dbmoBO;10bGXeH1}_CLJ$fcE-GwUJ{(+;vw*s){@R zX1KeqhEyF#W9X=kR0Vh8dAL(L0I4ku7#xVxdrb1dx7@CdPXPQ>$L@K3!U9?ueU$w{`k}r`dfjv!b-vcST#5Z z^dRJaAXWro&=~h~9dLaw&|dh&<2uIs|EqZ$&n@CVhiB$tI~tFUc`R>I<>Jgd?A;2Z zN?@K(kUZN%dUy^&@;uKVJreVHOxH@x<2UJ%n8#z%>BBshQ(hADJcD#f%ySg7L1La~ zkRBiAISSb$G5dk$VeH$|b2YZ1^jwY0E3bvknRyuOOhmn8=3cO&1Qzv@usx(l)Jta0 zhccGHqAoIXCd!v>BRvv}y2#9#$R5cTb&;8GQNHX4(kZd1i_F}SY!G$f#iA}U^GLEM zo3TF|89A=Y@m`MQzHw2`C*(Xs&LMnbwr_0qjpaTYLo4erNrD<_@u=ASky}*>ZQb-Ta>HByj;{vBFiZ1<-a_a74=f$TB%>uO9>fK zFD2~p*)Mg9dPz#pNBA?JOV3m2^O75;rD#4hbmkp)o;2x>sdoNDpJ^UMXPVcYfivlO z7VW6Y%*;;4z)T3yVw==Yz%@*CoFW2D-c zB&BrI2hvkCiVuscc^Sm~oZA#7OdccNOnBTR+?Ct#c9Pd9z5j5IdR{0M_L&l&fV+oxYCVXtc(v= z{C#r#<%~LUZP??(H}8LQ{N{)+QzYZVbnQD2-DhQdm~5y!qgSdH^!PB9CI6bo@<_&q zzk^Q-CO(`?GF}WntHt*%wK5+3P16rEdVF|X-LEqC`0(`Sf6kQ4hgW^EJ>wrA?*HW9 z89)0lUCZ_5!(;>3yAM-YIDUMXe8O?)!xJ_wjrXb8DPF$lp4f*e&bJNeYGr)*r*oc+ z4?4NEmGR;DvxnnP4&KYk_%Pmy9zT0XMJwaOXAYhdzkzsO2Ah4Du5ESfQY+)bDcJMn zad%o7AEvUj8#LU?`0zydaMtXDt&9(o-%dKbyOr_b<9~fSqsNCgjrurl>+JF2d;VUZ zDVGmV`e0+cuk()&k1F^o<7XeHYoBxV<-=qH*Sil>SvY=tn0&&q>cbqv#}Am2iD$wb zx1}-1D)A-e_$0o>6iZZI;!8|%NPR5vCFa;;d5THG6mJ_5d!;ei&>rFHb{@e6c1cpiAiRu^N++_@1#d!u502;%=OFmQ@s*|eu{Qf~jUVU6@w~&W4cTCt160KPglxzL&z^p7 zJYmuS)&?I=mfRE1?9j>DkPYTJ5803nUNqva_>XgIS{r=$w@+`2R}9_L+K>(AIThKE z4ZdL8luW zLN>VTK|f^Lg%9tw&rk7k*DkWbbg#jELN<8c;lIZ7T>s$1pa1vrn2*R;losR9xt-N6azeNW`ijPc--`1iUA%sv%!4N zNPfr$Q(NHqgAY?%$anLHY%t%Gk`38l>JxY_=fl(|@LVn%Joktl$s=w($A{m&s$$V1 zx1Qs}4X&tObckEe@!^M8)hXJ;t>^gg$_)*QHo5g2AD+;!VbLnLp5w!G?fY&$$A`&= zMs7XFhp8+B+L=0>p5OLQnbdc=lJmKPpTEs{UEI``EZ?u)r;PC>p4D5*Y5AuD}1;W?D^WQ z=lC#{rITAf@nQ1YJhz_X!{oP9-Fl7>_j>84g!TCFBZq8GIy!rN__yU75-yhyn@yi4 zyx!%*4STIg*v~#p*DiDQ<-=sdr>@?8n99QOlx6+tw=8B<8pxd5Jleh%Yh65Ah`?-%>mi zUt*3E;!8|*+S=JHF~gu|zgy4A2J@VUY{&-x@L=7dKiztc4-eQ< zzvxA`o|6rxITiJjWJ5N%=I8Y?dVQLHG-2HntnEIt6 zx1N&?rhe&Ox1Qs})Gw`a>p9tA>O<7R;`N+t@kqCxL-IbXa*NkTk zp8oXr3H!l^yY%=jVLxPpX>QE%kPSY5?z)8I!G~X7^=UH7#X~lj?{QZR6 zg%6LJx+*Dh?IIh@_bg;XHu!?0-b}cE@Zl$xzmjnOkPW7LBJL-%!7V;~Ei?c0;r%MT zWM!zI%m(v47yE(k#r_Rb*|)p#!iTBsJYHmjDF%4l%mz~o@VM#26azeNW`p^jk;@|Ho+RVLyq^KO#Jr5+~!@U22 zWPF(SMJ#oCXzijj<~&1c|%DK?%N zlh0{QlK2u+ozglI@g?T?B)-HPlf;*p>Rf91p~om}7wDX>FD;*L!J9Hn9IC zCY!nLB&PA=N&HqaVEgTM?ARR^dvX#V<~=#hmQ78v!Q^MRKgasqhiOlawG;B$U|Pc= z8%Y1|FpaGTuRGG}&jyP;lSBDtgQ+d_D_+mB^UvL3Y770boe3<&BBCO|-4cTDoL)?0f?N747v?qsbplf}&2>Wx$ z29ntwra32a>xcBrr+Gz57}Uv8*>a~gK6&p$Ab^k-h<&T98_auh z$PYfudvd5QvcY`MLN;WBX)g-x-y<7*nD^w64cTDclSBPvHkkJ0a6jq8v?quA$!sv+ zbCC_%U@E&?|F?c93-g{F>Z`NCd{0L)nTWO_lWcFt2%% zj1Ti32AW&?F!|r@&#^II7UsPTq{oM8pM%?{k@zt0c_2MLO#2^b9~8;>FkRcmt>O4E z?~$M}%ZI5fUEKPK5A%Kr(&NMAL)xE1GCs`vC`iVKc~1q&_%QFUAQ>O#y%r?n!?f>$ z{p`cM2ZQRuhk0)X$@nnu%^(>c=6xC@{UJvoUF)9>ByV}B0K0kXm5 zXSYAc#*z=y?>=NF$!CK%xb+;8-yP;T1L@BOi#<7s57XX*gWP&fHkjrB^t%JukPQ}l zauOe=Js7jFKZoif8_aVavLPExdr{o)t%(owo*c3v8_aVmvLPExdvds6^kCVcwHNHe`dTUs~nXbF#s_Cx>kCVcwHNHe`dT525|-R2SJ|+LJ>zkh~8! z!=4#~kFwKoQ2C~7tCx>kCVcL5z)~)AcgZZ9< z>LMFVdoZ|N_%Q9kpnYrPhiow4vycthVBV8Me(+)5lS4LSgZZ9_`pIlC?aASO(uZkJ z4)>GUV7})f8?wP-Pmc8ij~$ZFdvd6+&IVHq@VJ=`7JG6MAEp@KaWfmt_l)F+Y_Qmq zW7~z>t5X8=o*eQ+Hkj{8$%bsO*pp-X2e((J1g1WL=W^L#JT}Pv6(qIxYnGb$@i_Y| zUf?uimQZ;AvE;vU$=B&e>YruhrHtgecV+G(Bl*yg;2QDHMABA}NIq<__U+84SVr<4 zPM4N(dMw{3Bj4#M<#*~TARDQ?_W8t8UdlNCSQ#l_N{0MwWu&}MhJ3^MN*T$AJ(+iG zN*T$Aj+kuP`8^@)=L{cq#bi@y8OKi%-zVerOBsuI>MEdjj*^`=9;CdKapkfyQeMiy z#!ReAc_BmP$oN_M$|vKg3-V3I|I(jQkK}v&9812(hjA8uWPU zTAAJW*~&N{S}gq|`Os5VzhPsLd_>p5b{i`@br33`a%K3CO~|IwGLjE}mep^NpEG=i z(;hvR?~`%%By_FIcNaabyjDia@64}&{7n6=m67t`ml?y7U+UU2V-@*Y-#I(2uS;X( zZ|xwy!`8QwPxZq-moiSjtve}0vhjOQ#Kshy1uACCk`-+3NIX z%a4t}-nt?s%S-tc{}sf~o;bR2-u2I?-k2~vKL4{0=_ym6NS!fpcnQALFJ+{D)=2FO zI`dek)DJx=VP~fAmHMUqQjfg0l+H}qr9HB|r{wLIAjQvZS&@?UD{YqL8hX>DG*aqn z_`%1Qc}e{e%e+tz39d;|UC8*6HpqNY4{79W>%zyEc}e{e%eDIyeP!;^NT8 zs}RA6F8-|_ zWWGMW%uDKbZ9EZul(j+1%RWuo;M#jy^ikFx$(Q=2ZBoDML)eDQm?*E6eQYTks86#t z$b6}NFkkAIGE%>kL0%afU}uWz%+*n5{Po#DeFMdW>#NJOLCQ$|62s@2yii{;)tQUc z%vdjNkU9|$DT)c#PiE|qe3_TjFEL^-lNWpt;~H+WnK_2ELF$JOQdHmM2OnSNCG|^; zHkio^?Keh0$^AoSJ|k_A`J(<)RNtUVX-V>HD>O!-K8S4X>yS+3sgcYwhyD2Cck?8%Km;%Q%r3`ZF;jG3-gm4rhM`lg}tW;+KKByd-b2ld;(8x4w1yE#KM0J|}xDcJ^B= zW!(J&&jB)HE5_*Z^&Hd-=1b-4Iq(DK&*kemJN>}-Yt(*iyTe!Rub{;G3GR<*U5oG6 zXq_!xdYuRN1+-pg??auRt$r7Ke2<)I%TivpeYSwcDcq0Hx?=fyPVt&^`Ff66J9778 zHV)nWn)Q!dFOo9ue$B2INlarJ#UI6;yRYO}jd|RWF)4Mr`{nZWobvS?yKW%Yk>t9H zj#@=`{|s+1Qp_Fj$d!H6e)<&*K$g}wiStyYipr_>|)(w|aB@;yFGvhXvlb5cG2 zo1ba!N%8P+e&%xho1fW-R%SPTwldC#7R$0rKJ=ui3_HIkR6zD*_^{oUp|p(TLx(Lx zX&K3<`lnbaEhG63r%TH?J(lm2arPv1t(4!XD^o5jBju%xj6LTED^Ipl-X!b%zYD+A zwV62{^2+oH62nf4A6qVokq?alTwcN{%2)E8yyRQ#^v5JGWt`2HFJ)Z0EMLksf`slIvHp^?JJ+i!FJtr;eRN5@dS~sA1a^mC5yrh1KWnN-EhuTCYZlw(}U-+1OO~1D$KEBLL>MsM+`bLR- zWm}Lo$b229wUori_vPhmqcs-xxovk&UfN^%(gs&vx?kpetqnfD%uDKbbwu~b9DBC? zN_p8oNE=+e)BQ60-`eBj%eR!Nua$jlDI0hX4EJf)9@j@%EcN?r zlQOOkWIyo!9IB(t`0KNQ`UZ+cicg;nQU-p==$BaLh5F+CITWjzabMaXbs`RVe-714 z#vaL+c}e{e%e)Y`yg$c|&lD3o^Cwh5{b43w_<;Arc=<9fsb6BW!OVOH?U(oGP=A$~ z&+N3LfZAgwU-V0~FNOPuGWjwuso&ug^^KD6@=Z~mJ{w%Vwttp3`1roOoNYWdksoZn zQeN6qh96R{pCmt6dz^oE;mf?Fepg4dKZj#4rhKHlt0QZhw86!L9j|4+KEBLL>UXh9 z`*XMrS{tOii&e{)Hn{k=<(2vR_%bi4-}Mi6o+IZ-Qr@-MU2Jgu0MF%ke-8H#mM`^7 z+oXO`E<1mxx+k8To0XEW?Sy#LUa6jTeov^twk_tnv1S(;$%kxNb0pf&km19wvgR=V z-~8XiP9NDdvCMen8H2bk61vXCfW=ae9;W;Wt`2HFJ+wlmM>)_pKPJL$sUQF zfB4!mV+MJZ87HvKwr5wy(lNm0B7T}=oV>*{PMrRvw2ZUO@}-Qk-}0pl#%TVZBfeL% z<-!=v`*Zj{&hjb0;=l6s9QFg|Hkq|7%*SY7MEQD7`Ff7MU&j5BT|*S>TD$OZU%>lw z=)R2hLD}^p%++acSowNRaUaENDby$1e$4ezyR0>F4@L7Lx>u$;C|}RX{BKu@^%Hj= zx67K6yI-@h>h9O9pCxwVrmYKiU&*mrzMfOO{+yZ5$e1i&&mnJQ*0t<9hFnLI>nf5j z*H6mVbL{?{^ncIwoB}sjEI${w`*J^&`@!~kQ;4(I#O){-K2h0!_+w{rH3IMrU!!eqZ)|xml}?54%5St z2I>>R`%9gOR8jX#=4(6SiPwFA+3V+CF*=Vnvx!>PEn0P8|zm3aD2Z?9geg{ovDVZR{Bh& zPWnjj)}Rb)P=bSD%Tmy#&~y%@I_YzeQaVsZJ)i=lCsm{>s%Jo-QFBx!m7`t&eL+2^ zDyypMP0%;hN>xqOP#=SStlm~N)t+i2=tlL4s;%m&t)N@g7pkt>OKk_;u6|GrRAW^^ zH$eJBHB!w~P2C8|=%%W<+6#0qU0t_O`>1`0>PC8B)k+-*dZ6AoY)f4Fw z-CrG{CaV5OSF20aLFx+7E7VMNhB`}40iB|5R|C}1>RQli)dY36I#=BZdZ)TW9i@&_ z*MnZKu2JWy^VL+)sp>9utU5v62zsNsPUWjSH5PQNx?EkLE>hD#r>VQu@#N0hqx>(HsouTegC#k{eX3(3}B-K|PqOJtJQjJ$5)hKlz=zVIs8lp~B zlR+n|ThyWIFjW9rpsrH?QDfBop!chL)r0DB^%Uq+>LE2>J*^gkE>yGBGBr;<4EnGt zREc^-%?6#VmZ+E1a+L;6t5?)(sz^Nx`mB0hy`x@MuY$g+-cui_*VPKp73vN3sd`tv z5Bk1ZtJbU4>MhW>)Q9S8^_f}+x=wwgzEd02N1z|6FV)ZLE43MPv-(wSQ{Sr3K|fbp z)ZgkS^$X}PO6frVroLB!{$Bm5_Ru@jKcN4pD!QuH`gc_oDbjoDN;(HLN7vK!b!8ob zhPsAsqHF6qpmlUJy|=EbtASS24RmYWR5u50uG{DXbPL@Ow4rXPJL~;*ThO+;tM0Db z>3u=>)$MhxyXbD9-Sk1akM5y6fOgP5_2D{S_Xh2)kI+ZyzB(5)S0AEJ(EarQ&;j}+ zJxCv^4+cG0AFEH-$LkY8Pt<4Xv-QdPXwakeDSEgbs?PvDLtmsv>a+A<(82mVJw{)k zF9f|%U#_pvBlJ0-=ji|F$JC{IEa+IhNG(>E>5KJZb+LW|Jxi<}1$`8~%LILu9*5&; zy$Htw{TPmi^*kJN^ei0r>FGG`)_3B#T~EewlfD7RwR!@MarUamto$r1d8d`R!G62Q z@~49{4{3s4qd(JCf;C8$gE#eS`V7!Bf(wI<`g6S$bZPKnuu8wFF9f|X7#n=8H|iHb z!42NitMpjVvBAV3t)JD+K$`_!gKzcMdKKuZU`?=Ezo#dHP7Edo%k+=>N20-g!4LXd zy#{nmuqpUhuhx@6CkNAlmHK&o5a>a{fZ#{{gWd#+6nv^b*3&?z1#^R!^-4VebU<)o zuugxf=Yq}+76h;9m-UIDCkAH(&+D}A3feU|DEM5j(+fZs1WSV}b%AaS+Bi5MxLQxt zoj^MUO@nLnbvgx_3c3W>>l^i9poazd!6bdNJ|6V=;PBuUeVaZV^z`6_;8uNy9u7J@ zI4zi>r|L1FV}kR8yYw_Y0dztzI=DyQt8W3lB`64H=$U#d=+xk*;C}spet>9jPcU20 z)sKNb7R(7A)DP)rK%WVo2c8{~r0sf({#iE+wjk9H_6}MH2ZA0L zvL1)m;L0)h~ z&?h(;^x)vA;F#dhAO?+te!)q>k-^cRM+bv~A;Gahf6)HHalu)^$-!XI!NEDfdBG{c zK+u7~&|pMxW^gv>*}=uZe}Z#^Q$bG+E(k6UE(%71jts^HR|b~^!$5}xmj>4cmjzdV zUJ=|7+!%}xMuCnBt`2Sst_iLKy)L*TxHGsuxC->D;O1a@aBFZo=%Ia3EmG@2d@RsgFYX;6?_)F z6TAodUa&6M5PT540s2PpaqxBUX|NV_ZSYO-U9dj*5cI>~%i!nWt6($e=HRzrTkvi0 zIq2uXmY`MeQ}7GuFG0JYeei4WJ?Qtr{=u_)ncfd{zo1$0d$2wD8;1@Jj*4Lpj;dh| z9JRx`I2weFa5N2@WzcrTM2%H*7zXvCV z*M~QRCnB939u{s6*MhDM4+&2X4>DcN>0wuMYB(VL1>X!nIxIXZ9AFMIXW`q^!=u7~ z@Xb+31Hzu+qVQhOd&AD*x#5XsfH@bEXNAXxIp!bGe<0T_oD*JXx*=T{o*$lJPBiC3 z^4#$Fu#U+w$3ybiFh6`HECekK^TG=uc?M+9fZR!8GgHT$1j*yWKH+VuRY4i4Lx zW@a$HJqcE?#W#l_^}*GJuqY3yXLu?~(FWyc1Gztghr?;0)55=k9l?xnLAWDW5dIUW z@X2s#sE~p%43~#5g1#6=VTJJJ@Ey>1!b;&D;k)5ypr3_R!ph;A@N3Yo!|GwRa8vj* z=+9xTux9vU_&4a^VV!W#@UL(WvnNvhuwGcv)H3zLT4t}XVYsJhY#N4*P2;dh*wnN# zO~O{DS-5w&zv*Q54m+6^Vau?K$u}*-e6wG;Zy1}y&A#E`X8*8tc$hiCv<^=&2ZU|I zf#!76Hay)N7`6{jGs8{$aJcCdb_~xq4}m@u{t@&JFAJ{-dm|kjUKCCZ{{#A;utzv3 zyfK^vIw>3y_6;u&$AgX!`-P{3H-$HY-W;A59vY4duL8X)JUkp4-V#m*ogAJK_7ATN zuLiw3JTmMaUJ{N39T|2B&kS!3ZwI|SJSRLNEC?rpP7IF;yM`Bsqd-T6y~4A@+rm3Q z?+Ay5M~4%_Ye26Fj|+3d(cz__mxihEyl_f*7wBE#@Ni&wZFn8%b>RtN9F7Ucf{qOj z3NHxn4DSZLI~)<-7v3K}2Krd|U^pwhC!7yDKYTn~96lXB0Qx{UGkhj|BAgC7J)9k; z!=i8r=#ucM@P+WXa3Scz@Y(Ry@b&O{(C5P>d^21bE(2W_t_a@`SBGzaz7f6_eiXhP zz6APG_*S?+{5<>+^uzFj@XPSia24pP@Z<2?@cVEB=!S4zxF!55`~>uq@Qd)b@b~aL z(C@-;!au{G!i}IC!yiIzjM)adE&Mf%%#Ls?=+^L$u!^Z}0#gM^nHr{&*&fzF3Qb*8 z-&6&yYATxsrlzR?TESE^dz%(!FVMYAJ+qH#WNL%fHVsW%)6TR6ZE2dD1I>P>31}0u zujyubnD(IU%>kyD>1bMmwl*D1A9Jwj3EIgV z{$nmMr-Ghp&NXwxN5YYyBh5wT(QsZk40M>e*mOp(G6((29P}Wg%ouYyj`5}d$3$}- zjvLL*IBqplaNK3?!7;)pDtFE*1Q>^4nEAxo`w!rf50q16< zQRY2^iiqAbCi=jPHLB7v{a_mFNpdt~YO)>!K^7w;=htS!|j{o1?{M zbF|rPGHavvqD_$e!n|W{kFJZ}f#h3ex#=7Ah?bik(Q{@qB-cWAE#%%e_eQrz??dt( zlbEBTzEJ|n<>o{4P;_tfA-;Xzya3COg6&7)yXT;x9dxvVO`o6?51||nLGDtsPgEzm z)YOT_n#)YRs7-X4X%k&x#+d`6uF*KtHM-JVWx7WPMOT@FqN~jW(>|AEQszmv&@sx z@@SS>9?dbc&9dm_XtsGdnr9v~FGlZ151MzQhs}JmD*7y%Z$672Gmn}z(bv(V=IdyI zdBSXpT0~En7SU*vHoux}pxeyz<{MKh+5)=8JZ)B+D5?;xHWi|eOp*D``~muhS!q5t z6{AYg$EH%W#;h>Eo9&?6&CBLfvqzK@eQI)|wdN)Br`Z9z!@OpmGC!E_K)*9jnsugf zR5e;>sz#riSIuAMAJBiyn`V*u-fRWkYMwD0OqHm5w82!5Hkvog-$q4mAiZs#H9wl4 zKz}kz&6lQHR5SY0)QrA1s|=>6(JB)}@0w-iXY&i_FQ(9ZWoksVqpwWu=v&huY8bVO z8X)Z*HH!9(T7tHWT1V}p4pC#!#?f9;r)dAEE@<7TNz^mSjXHvMj1G)aQJ1JKXxpfB zbZ~TNln0s@^@BpMhU9`y$89rcffM5ji_fgTqf6Ag_{ ziUxoVh>nlWjfO?1fSwW!jxLDKj7|VOAv!I(BpMZ+2YOy~PIPH>QFJ=!>CySo_^2TI z59oiQi=(Tf%c9|+!=usB4be@}m7rHf_ zCq$E?`=a}!si0G%JE8}od!k!GZ;kGb=0}f2vp{D>Go#0%Ini{`>CxUAsVGvxlc!w7XkqukN8;WLCs)5ut6}f||HhtyHH)=T_ZA!^2v2 zif9qqIV>U~tY>J8i0B(O+4L8c)5GJ`*?W!dwF>W1egQ- z{Js4Hy+ccQ2Iy^l>YV=Cwq1L)?i5-#tgY;Q_wXK}@G@IZ>Xa`BEq7}jUOlvXt4^K5 z+SKXVqkV@ip%KunD7V(FB0|e{?bJ2ALaR3V@HO%D2ng`=^YZob_4hLSdk1*>xO;l| z2l)60nEiYLJOliEef$HVXI}68tjn+TqdH4l*d3kyT`a=u8UeHE#w0o4B z^A1<6R=RH8pX{=y*~ib@EJw)8(>K7!Pxkr8@_zg?e81E0vUO@!|H*Fq1p0aVcm{fV z`uKST1bF+(ZvR-`|3J45oi-+_iI+S4U{*gb2B{(6TL@6ZXsNETID))9VLev0?nR*=72yyIdq<0U@*)+{yt_i^d2m% zZZ|x=q~#0rH~ahh1wdi*=MIB0{a@&=eKhJ-u2cS}4WFlvufJcQ7j)L_=?8W;&{zMk zAI<$Ky?}9$`*~E|XDiwQAO_ zT(5G?8v3qRqh_7zrK`CGdAYmQE8nne&6;(}weWNcGK0+s>(Vx?jjZAgZ?kW}P)F_= z&KmS_cQYFFb!^ZNH|UQW^!iqV0iPQTbZpQII`X-TUeN!~eT3dS@1vLbbDLi9Y523f zd>k5uZ}{A(_ct5$Lyf}cIW!vJ&?tPz=SJau&USkexSnjL1y>@Y>X-)t1k zk@<^BGCRzY*G7OS&|HKJMVCdYc3Ny}@+(1O)o~__=#|c!5*s<>O=a_45S_5a{0w z<~R(z+9oB;9s#~S-htq7dHVVX26#)m<{jW0;OXxLh5gI{p1x*qFu&G9cfD(F^~;y_ z2G1BgMDz(Tir@c)Z#4j8`!fxglk@;4BxwVBrap+?U^-Pkwsf`9b&@mznwYc+ zG!H(|Lg!X;8oWFLJiWZUd|{rz{|)p6SJ=xV5E5peKycUme7*cZL%cjZVFU5<^YjdW zEd=~I@6Yq#<>3u(XrPamA2{&P5RAH)huIsPVNd@6f7rmiVUpxTe5HwRjU#B@cfa9V z_4UvFhAzr6_*_5fG=QcieLtE8*j|nLPG$j?ZqgP$&jO5=Zn12u0TdQigI}Phw=WEs zx77VWKeKm$xAfrPY~cm(17-G?%fZ_pR)Q}a2LeA&gV_U|Cb=9u<<9TrCH*E?FtCgQ zy}W$`VTJhxdh6>UX%nA^A?bR6E`I+bzSV#ijPuX5kaQ;G7L>Gw&r{(K$D!}dh1{Ex zw()r~yaGNg329k81HA%$0|Q|yK;u27zYz!}%wUK-;k;;uL6%wGV0iq^UOv8_U~K&S z@MOT+0iVkkBDpeT1ODD#aB71^WroT0@$zvm@zoZ-Ih0>D4PSi3Hyik#VUVNnxq+`t z#aCD5=c(|P<7{uh$*Sj5@lV;bZh>KEYa>kXGQ zf!=Wc5CDb?ZXoGANbfLd4?fR>7dSNEor&+P zNxx4M0on|v$JgKA$KTJ_)5{0U3QP@XH|V%8jA|h46jHz8Oy>_10V57Wt{?4eE)&$3 zGXtkRe>ll{>mE}e?AqXL!HV*QgCcD8uo}M7z~@0ox)`Ldn{*I9F9!Jm->m-~gWy>I zE0gfm#fT=s_xr2iJL~bM=OED6-v=&VynO;-i}3OM?6&^s9DJpLZ;s(t4Z@Glf%kXJ z!T07s9zK%lz)#JATq#ML`1}c8a2sTpB=C*kfFS*Uo6D(tI{~mCd;5Sh3b)Cy6-X-u zHzBYDe0@C4zGi8{@DA-`_Ja!(UpQI#`I-ZC#Rmgs_Vo6W*JbiX+Rw+|SNE8bw(_mH zkSD6|f5M`i4aMgclCBAPZcEyS&+}pSliRSp1!JRz(fNE5!iPKkVAr~L_t0uK zVj##1u5+P}GOK30b~-^J9fwxo?Il;?4O_5&QDAHBM{3&}{LvNy{G|7dUvKvF43Ku_ z2fN_!Bi(=80Jv+q7l|6s?z}Myo z|6mXN17XGCApno)%MN^N2)w~BgCTcl0ZtoIuYRxxUuhx03p|A%ZvnPLIBz=i0X9Tf zA3xp%6qnWwe+KM?KQ$8aWc)=7U{d^lq7z?h!Pn+I{NPx^B}tM#_=ElX@qzG@1Me^l zzS5y~Y~Wi8k#rn=1HrsHHt?M_@nsJJ!HxNe2ITR>@e|+}_r)iCs|RwtP+yxeh!NdPl z;jewf>`c7I4V(+|$G6YKyY(E)B`%w{gN|hxjEyp9mwP5=DA1nEc6zSzOO*K%x6bVr zvg%zs9q)c!Fl1s|TfS^_Vqf~HeFGc)shDd3$5@s>6LLeH-#%9FkAY>G9lH*zP4^Ag z%Nw|Q*{LG?=2e_GkK!1g?7mLS74x}lA7k0hw?ztT#W~Fj9-qMA@bi@${#En$}#T0bESB7d@k=gcQ|*34;PFV zlOHYMFY^rNKZ0NRED>z+48F-LmSen29WExHTFUoyLI2CeI>nJSCyG0l zmvZ*^s83@h=ae;ADlYC^sLv_JhW^O0{Sq7VEAufn=1b;dY|O9B{~m7o;4DvFVk191 zycfqf-?K!%Kid}mSMt#u<1u-j@&>7w={kV%n)nkubIC2dT=K#Eebg0xe%WeXX+nFW1KzamC)z#sSxDE94 zu=TJi%M>_c*0F47$_jJhqfMvv@-knxlQ%3@FJoX?SH4%~V=U{+cb(}tT5soTIAy+~ ziH&|Aps(c#iwY;6nmksY>#j9D5^Lp&(Q$0bB8j6e4CmQP#_}TTizg;~6~mK#f%B(? z{Jg`txzA{ham>}CiSAp+@CxuP7;nt}YvS@cvAprZ;T+>-rHdu5&N7-GXb zM53u*4Chd{_agViN4tjU{X9_CJyF>?jIW2f@wGh?`$7NZJnmmoIFU^mtK+;o-4p-3 zIZVf)&xHx$g9A4dj8wf`9k_+8L#7%A9II%C_g6&o_-{6Nb`hgdOpV8 zXrT}}N3su1>$-#(dYJM}fe=H-Fm6{Ye~6)z7#nq^J{WCEEc0dia^J~aG=GTHADQ0- z@(o>z`{){C=+7Q_uPiThOkzV<=f8CeF?3SaHFQa~Y3P`2$Iu^ROys+av6A@quQ@_G zUKpqMr{&SSA)U8O)90l!;8JPE>+khB<`Pn^*)%ToKRJx^_~LVH{du1|BvYk1dU^L# zxkDyZoTZnU4f&P-9jEL3ztH~sq0@AnAIglWF^x;zp1bRi7MzELl%iGnv+5~Cfr%gZ zg?CLIaQ_m?#o`mMxW~NTImVktf6xv+de7rphw^f6PqY=aXvo~X)woK}X_u1GkZKj1 zIN*Y}UTA*~Cm~{S6OM7>`COuTi`V*GV{G(I>Oh@~X~g}5AM`T&YCh3?TGJ4j-{fWt zm;INx;>_th*Oga#8H~sOJA$ujWGkP&#w=x-xdxylgl{1 zt^EgnpjtDI@x8b~{I?V|B&2Z@j`5AVBY4G%G-T|HYP?9wQ9NP*3F&dNi37fzeGK1z z;XNlyLv?#-_%?<=BlSVrSeY;V5R9c=mHE;S!MIJlXCifLuj{~;$HfwdbZ@WglIy*a ziQPQf>*af1ERmSIMti+Yj4O7?6|&_`JH5Q&N69{5yr*>Dkd<%S>2)y<$&x=L6#OpP zAH(mGbum6%#U-Q^_+xUdVI2IlK*%u2mvt+-1aQ9XZcizaC<{r&|1P4zXqK=uik=MyJn!k%1q`oV%f-en<}#-BHEa>aDW5n0lz*$kj88Z`{a$r-avE~8b`y^A z^A^q3L6E<=W)qIFU%naYG|1mms|m-r+rjB-yLIn)-o;^D8!%l>+3y8k*Dc&Bu59m* zv^h)ceU{kho5cUPO%$6>Eae_8qWBu0;UcX65>B2C z*6r1>yr=lqj%)bc5xscvSBad(ZsH%04&jw!ZgKBP@p_s5M>C0tv(I(xdTkP)9cROj zE0t8&o?ON~9}nXg2hCco?k={R-+MHSW1Os>x0?IJdVc2a5RS3;qP6PyUpMeIw})_y z)3#bFW?x#!_s(y`o#8>#7K%2R7V}mMdvj-aRn8^iNsoA5Ev6I4`1Iic;^nm2eC_Et zj`4|{b41rY^LS$0G29tGHe-Ys^L!4^G&hbr!$Vh&6ltC<;N{ZAac4N;)PCOo$SS_h zGnPBUXICWhtVBr)@Gu%pC+u!zHudki5eW^ z1;6JNL#Mpte|1zj#vR>@h=K`E`M@$YImV%DGK&<&@9?IldUK2`S6-k6_I$w~mI~+2 zaNLC(n&*mlJkPK&j`6wxBK|r0i1$%?a*TVOJf=-q`Hb%>6~Qsy=)0J&`}ZE78`6k7 z!-jwIW@j<(42v2QxNF!4-p02k$N0mYxxC%|C;Ux@S{&o{)n@XS>?L23hI5Qt7w*K* z40*vL2onFL*MJV?5}=Nj2foe4b)v9LIR<@-6DX>)T<+JBYUlUMdG%7U{710Q zuwlV*%jfc$&%_H%bx5o#7T6wp;pCozD-!w_toE<1kC6 zs24oGX*hR=SLd#+7Ri5|zgDBUGn{*KGxd)(@A&J5VI1QFThgnIlRx9L&O~r$IB{ed z^-zrmJnBRwcZN%KO;iUKyw2zR7R@nUI42eV81sT_gTuKq?6+-?8n^ZxU$G*LV|+W| zu$rOH13vgpB*%DZ>IZ6E>SsLFvj~oH{w<3{$24!0o4w5Cu}s z;peK1;24)*lgJBhpU=BQjo`88_VbFnR&vXgVchS@KAu=-Aum;KgafWr^$`E1=@Pzm zCHTn*w;OViA9h*Bo2~A{pO3u7UvG%#uiDq=SB78Zz3wmOXQuUZ!2fvPgcRPQR*ZQjqFXP#Xe_Zp7Z*0|$ z=Swk$@A&fp_gK)8S86?g-#YVzrybFrU+`U^#Vmcsn>`EV_cmn~4?}P9Y{NVA*1L~s z*RMX|8=kh~IsJ)9mf|5#zcq}vY(6~r*v>ipRP+e`{{D7Lvc~gxc%>1%MXMa@!*=of z)~4YOxO0X#YU@>Vc|RzBIbn-xJ{HewLm7;hzB#Ebd_9l*Ks$q%rs73bJmc>|+VMNl ziR$T^S9#^kfA9pC2ddTg39rApJ$E^DSgjs?kH1>@JC6&jtxm0Xm7o8&8z0_$n5E&W zXFOZhc0B*eGHTI*_qf~0-yLv|ztgKr0-x~QvF$m=zAhy3+@%q^FC9{bCVE@OaX;`$ zZbh)fd2=T774wHX;D`=N;d#O)HTimet@@ExS{s#5X-;BjS~;6*Y*t8^9?ND zE6-W-Tw`GA$I3R1GnB-}S;=^>EMwen-)Wxz)BUZ)#vP-?#=W1!$>D5!cgJD%OI!uc zwyDb9(DN}i?hj@8TyP)t9PZg99`UGX;*d*YlHk!>-4iSRHB8U{x0Xj@H@M4^<&C=_ zi9f>qjL`>-%fj7`(Fcr;yCYfGxVMpb;fLHI3o6Xf+sF7U+-vRLI!({V*tq|Zb&dNC ziQikEXpNiD5Q&X@35jPv8NvJ2vE@s?K+ciO$Jn?7m-!eQ_X#rp>{8FfZE!yzv2m{d z5*ufBnUAsAe*TsDv*EmE^x>@iye#ue+RsE18|NU2H`>of60dK(RBSl3Q11iA`_?WJ z{ol;i^J6mH<$Yq7C&A?h-{KkP#q0To|0&zV*sxCkmU`Xml*L@0`XQtj=z14=-9q?^fLCf>41%Sgmot{Mm&^W-hL;bm$Bb1 zIAFv*35<=kDci?bzRS)->FpR8aZdtc#61a&5%(l8M%2qhdhe@z8k8-XtHs(dN!nypq5e@k#Dm;~aO1mc(k;+Pz;VYg-94Et|jW84teBfiA44BWR1#QQj4#QO-u`v}DQ z==ri8;CCD_;(7$ei1*RkvESP}V8r_f#QO-u`v}DQ2*mq1V8rzZj1ku(Fh*RDz!-5o z!Wl-qk3hVSK)jDI_D4BK#(pO;;z|VKN(ACc1ma2r;!1RE=Sl?PO7yyRu0(Ism?OlM z2xG65eK7W1i4j*K5LY4)S0WHsA`n+1_I&0_1ma2r;z|VKN(ACcgt1=(Umy@y;(!r9 zA}~f=i9lS5KwOCfM*N7t*zgTx9}s8ZfDun2Fh)Fuz!-5B0&x}&7;zT*S~m7I!^e>G zf_MrCjCcxxG2$!);w%K>ECk{y95CW6gyGM~K46S^3V|`=EQI0b$hsIKoPjQtDo5d!fMdKtsEBMw6R2u2))_z{fw2!Z$rfj9^Uj5rA43?n{5AP&L-BR)bH zzOG!4h=Xvzh=ULqBMw4fjQ9wF_z0&MaSZ}-4FYiu0`Ut1@e2a+1r8YT1p;Hl7YK|I zUm!3>e1UL=5nmvjVZ;{*j1gZTFh+cVaE1|IAe>>u7YJt<@dd&eMtp&Ah7n&NFh+cV zaE1|IAe>>u7YJt<@dW~7#1{yR5nmvjVdIPj_e%m}#1{x>81V(d8Ag18z!>ob0%OD% z2#gV5ATUOJfpCTqUm!3>e1X6i@dW~7#1{x>*f`t5U7T=+5nmuMMtp(581V%HW5gE- zj1gZToMFTl2#gV5ATUOJfxsB?1p;Hl7YK|IUm$)KBfdajjQ9e9G2#n^GmQ8G@go@V z1;QCde1X6i@dd&eMtp(581V(d8Ag18aE1|IATUOJfpCTqUm%=e#1{yR5nmvjVZ;{* zj1gZTFh+cVz!-4{0&xcdaR&l%2M!o<2Lf>i4jAzT0%OD-2*e!-#2q+b#1{yR5qBUE zcOVdV;D8Z#AP{#Tj5{}J+Yxsl5O*MqJ2z>s5O*LDcOVdVAP{#T5O*LDcOVdVAP{#T z5O*LDcOVdVAP{#T5O*LDci?~#cOVd7;D8Z#AP{#T5O*LDcOVdVAP{#T5O*LDci?~# zUm!3>+<`#cfk51W12*n6<=F>u2Lka0NwA$S(DMy!+>sjX7-t;B7ks*Vgu7D*Y}{GN zeB+KtV#E5x0&xTm7;yvwaRi_4Kx8|J7jVFc7Z4a5cP6r~abF=Z z;sqQq;spf8#=VEEYuq_Vj5q>e+ylsb#1VYDLzel*eX_*z?$Lg4?|_Z-yDWn^f=}mY znQxq_B{t5;Ut+`&2*eQx#1RO@5qvr~$#xJ&AP_I$fDuO^5J%vE5l8UJ&y?*OKBvTn zy^`4Qoxa3|y^{HeBM^uqaKMKC%ksvWk{Gx#Nsrm%FR`AMf0xVtNB{fQU&Vv!M2JAw zIr8t-lQA=*#nEMHbUZ35ny(MZpyTt+`tx>g;w<(CNY@3)v;jw>_4YA_JOOFF4#wdF z2Wu`BBlYsnO}+wxjytb!CbkmdfCpZAVm*;}zqRJSeMO7fImLmU<@lSb?bO=GL$shB zk^D~X;tqIz<$K%H1UR0meHpLuW{a8z2})m3>x6ihxNPcfJ-!bsa2g>mXEr6LB)8<(DGWzE!Frh zahbR?JgLxHZPKKg+DuY1X?e+_6?x%D#kKfmef2ROceVq+dhnD!KXV^8 z*YaNMt>?>modt83daiBmForx0(t0_JzczQWou}c0HFIU#Tw{#q-N4HqJ+~bA=Y%!( zKwr`7nhU>}yDWEIP+1)|qnx@U>m}7XvlXwJ+aj8}6cM9Gn;{BJeyrPjR6F*gSsu_2{qO|#H)^-5pC(!W%DrZ2}I%sj7FuCP>f zn~{q5om_z*Y%oKezcnpy&MR`?TGuQy7F||DN4DjSviPYhu2Sg1*CoEP|HLc0} z7I>{<98)ZZ+NOwYoWjPrs?|>0#{Bv1jv{EtL4C~aYemR$vab(KV*C2g^X+4k1jG6f zvYagcXlaxfxxu7CP-SdS>)4*=?xLyau^_|;AmPCq?UZLMm_y^1L1!@CExL}on=!%U6HEhJ9YY?F)EAhB7V=4pzA<` zsQ%iJ+qU(w_)Y^+YIG`nZH-%? zbX`UIp~||3E?s*1R$Wx!g|(E{l()@tQ>~QzrZzjI0cBav)Te`@_=4c3d`z47>L0z{ShVqtxJg*lp5xxB3De8-!=bm-GymLF zucR%ZW^B-aZ;MfQ`sy#$zn(Yag;u?>e0+38E&gjozLMNhi>`jIHZPH1%@W*<|Kpua z-LU+bx;j;J9<=0|D)!z~J0?`%)0)It=PZlnv$x#0{1J6on_KQTp6B&S72}=dMrgj) zEc{KM@*LyNJ^#>lX61Zd&3t@q+QC|~-+J-_ZPr;*R%m5CQZtNO&m2=RZrrS)*7!t6 zKJMHK72_hkvWdvkHFbTM9qcRo5Buozm@{IbwUv8+J^$j9SDJNw3mvm8TYPs;#|5n!XT_?Eny4fv4>cizR6V$#1dg%EJ>nePVXrkjnyQWw!&W+~z zGvBqG`H+>zUkuE0GwRBz|#uLvSQ8C^!Zj@T4P7l8R z{%XsS520$2h>rY7_%an^bKPk5d~we2rpe3MlkD2!tz&}C#iDtR%Tc03m6Yn5gy)vB zZlkp;i|z!c&)bvdJs%_vT-j?W=2L=Soz+rY%X8oIGWwBaVj*D#jNJ4_4=|$!b|rIFh?R_U4s64p|;l_2K7UH{nI+ys}t2 zd{DdkM{`qySj)jpCI{SUM`|@Ts!*^8j5WseN;4i9-rKS^`n}q=;b67H(Q(0~Oe7z) z!G{msSle>?FCTrLocoQ~pOiE0+;5amZGXO92K#|Of}QylIN!cs>fd7c60$Epn~#0h zui;JI`txfG23Wf+nh}ig)am86mAf%Ojh_?AF%F$sZrjv!G3u?0ksRX#fAkmQOAWBZ zE7P}QoNRWvZByRFs3o*W-tt_5ZB>g0snfPa@qPpS6ADc=TbAtT|I_%Nj7mb!qA4uZ zFWT0y>3lRVIl!dv0T_=s9nBZrx8>JA8_l3I=*`ZZ6mgCTeC(Azj=ey5s?+G7`s>Q%Uho;uHL(GY&*tyPBF$hgK?$P?Rowg zr_~{mGp(1iM`+I%_TtOBjSR-P#>%N`%&J1F+wpF~E4~~*a$$$cJxszk<5X=!jtG7} z?675Oa3k)~~5+l)KIoi+}(gzpJbM}Ys7&%c7mmeZBz3g2Um zI~pdoq&%XXB{3FU<~Q$h+SD<z#o(FM zKjdIX_2c_4T9&(ih@al1NjvIDa@Nn8$VBV+q@ki&rULRLr}1UBmgF z<2wh_WsFbA)0F#XUuE^{)?FA&+?kU9H zb+v?fc7z%k6=hkyzc0`GE~~cIeQd(g-V-eY>-OidW&83R4cyf=H;-+{_{O&E+a{|; zRG%SzxidU^=I#XF5~nQX=Jewj&#SsfZPKJGPkBF+_F~%Zgd5(6E%`U~<2A;7RA(20 zf4e*{l~%4wCtj@dPPJ&oK~`Lb!h3TsDX6Ae-j`$CB2A>&abu9$`Fcew#=A<4O-Q?R zf~Ea${W-=jMsBs5Yk*zuoLO5pV|T)Z;U_Fx7WU&9_k#SAQx93PZ|TPal@D6tm?nJY z;?&xkh&G~ox!daAYw=cGrbd<(TGN(Yc+~!k+O^Ds5^^_ut0tsq!7+Y*HHFrGScKTo zH&*R-rogrtWoD|W_eXGyOV92v3ax5rol>${aO9TS35Sxq@e1A*IL41#FH5NX%TBe= zjZS>%uAy3D#?GQm%t$rrZ7Ff$Zeh`(YiSji!TG!UMTrak0otnB)h#{8^x>n9R??cZ zkFG2R7z`=h5DKYX@4$JoDk zeI6Q+T-2WFrrv71EMeQyoocr|o%pi*FVsV68i@8D103?V!M-*M_O*6pXIhU;y<|yQ zv$iF7(SE{QtP!sec2td>vPyHdui-t*+5X~eZ^HXU;cE3nwuEzwcGaG)QWMgry?g@44zId%?*)Z+*!Y#E)*TV(2 z=VA1UmgV=qwVh)s3!c_ZxR1y$w#~b#l>oc+dPOO5rqe6y@%0_mnYX%VPwxLA7O%_2 zn-wi83O>HBVY}4MY!({|d5M(o%c|w7tg^1`+FkqxHs^t73h`ofEpcahgnH=IWG(%; zj)FcPsWw`+&@u+>^WR{fOQ}Wq)9itwKG<+hpgJCKn9`x~ZJgeS|k&-&zGBRqr&$)Wqq6^16$ zB9Gbb^)|N~tevTn^d1|ZLEyQ7a(Z2;C!oHL9nW0S<8?eYm2dONI;c!P{Y>MvqJ$P( zr7u5IdDd2pSNr8s=G0ams#YzS(~ps@wWpm`FjHqR4X;_``~`i#*$sc>P6esw=w=f^1fK!72Eru%6CW% zefV^zgt5FAHu`YnpJ@J3v)xBsDOf>mQ`C0HmD;7Rx-{>%@3j(xBKh>cZDYJ`VINLf z+3v&&H(Fx#cI(fd!?*NUG+LWjxF?_U&;1}?bgiXc>i#^ms!Py;VH4H4@4NHtsa^-k zzBqDF~Yyxzn6LHJCB zai)`bc;yCF_$VlY&q^4(cY395J_r8}dv2;y{;3sS@_rMZ#yz70#@~*09(|{-L%5tX zjPKJi#&_!&+wTVq9hE$kj_r2~4jA!J9Pv;Z;-MTc;+_P?h=-I3^8oOb!@vOdN4c8seA);+O>Dm;~aO1mcx6j1li6 zxgUe~(Xo-X-?huTbc4guG2(p;4oC7n4j9+5-@8lR$BN4#u18=j%i%f(MqH1 zh_kRFoI17F6aOMO2Y(Dk} zegr$)gK)kxzv5@}u@Cb#jP8)PV~jWmfidDB1jgu&Iv8WbM+n462*gMHG)8=cK96<| zLiYhMw(}8szMYTI5Fg=y5eK1Rj5r7lW5h=Y#78*Ah-=`8Yp^1&K|}lkNBn|@_yP;Y zh%X4n7>_^3cz!WP+<^u01=}%3cPt!n2OMz(R>U2sI3L}^sE9idh&ynK(fy2ycmWII z4g|W}u{tY*_yU13;ts5cFAx}`yC40}b&70%OD%2#gVTpds!+pnENce8e4a#2087Bfele#)vz>d;#VWtmux8 zBaXm|xC0gEBkn*T?m$C)fxsAX2O8oF;QyTa#3KmA7dT+V5eUQ`XgD8n2O7H5QxSI{ z5ML0CG2#n^GmQ8GfidC^)SlCKCw%yL$g*!!KY{KN;hysocVNL~5MLlLMtp$PuziqxC0Gw2O8oIG{hZf*v2C6K*Q?@aR;ii^@8n;v-OU+0~K)x z8sZMDh&#{_cc3BeU_0Ut9M%Qae`jk4aR&l%2Lf>i4j6FCZYxh+0 z9AoDW^t$+t{%dC(#0vysj5vZoe3$mUdouZsAK(ZQ5J%vE5l4`KID&-#!yG{Z;s_ED zN05Lxf`qS)y)945Z*l}%5J#{T^8y%SUI62tcrQo&NrDj<5P&#>Ae@glf*`~T1Y(Rh zf&|P9V2n6|1k4Lyj5vaAh$C>o_}jnW2vm3m-d%)$xpPzWet^&Q)DTO7ly zw9xZ;RXvvVT=nuseO*6fb`m{xS5AVx|dF0aTTMOml#o)OI zykCx!tv=y_mE`+mT8{VUeEWNNm7MZ9;l;Lt{-d}@Y8O3i#P;!lJpKE*P2b3;xJKz6 za$L9(%W`r&Xo7YGGDe+ z8Tu{Tm3k)a!i9|bxT*zGx#;XA=ueY-O z^zrGm(w1H`HbOm-Tecgc&K;AV-%GUh_rcXZ>X3@EU8(P`eEXx`b~`Wa|2=&@eclh` zz7V6^`FB(M^5yVc#?t;vy|?cJ`o3Z0!}r+t5m`^}6SnXX`h0&c{k{L%1o-X?Ds{;m+#LE`7)OIwm4DkHsOh73p}^IJ5j9! z?<)x9j5PGej&1tgSN2D)7pP~qYw+BakFJzbpGO&gnU?ywO|)6$kmy_4-Wot;kYFg2LJ``BMkXNIS^^0}=&SDxd@ z6Mei~?Xl0p6#Dqe_-o_mIQ}iU=Op=WdRy{hbLIYMOMifMy=P;r`eiKNKRIxUp6@w7 zRt@Rn!dphq2sYx(X~Dh}$zH0tjGtxG?F!L?N?q>-e#6qX!C%H@lI7zujdKSZ@lYt2 zc&0}1n2j#HT4CG1zCEOeZJZ&#vfc&$KlM&}Tm;2qZ^`+MQSU+A4d!iIzC1i-SYI{L zSK7N6b%i~C1MLPUc310ecHxiD^j1f$baXX^wCr!l z?pk`99J0kRIU;TT9K`Z_U8n30W>79BMPEHX__EEe2H9gH?N-zF-aU!(J6*Yq<^EM= zocuoUw;-1JMKapT8FA`+@;h9)d!Q{pYn~>0TE?>8p_zZ^@s_aupVG4Y)C@NNtVkwX ze9YaZA3h}_^m4;VhflHWe{hu^db(D{Kh&t1F1*r&te;}p9$qKKr@_A|&TO+sI6wCk z(8sDo&0~-EOn2e$+|sK?Y&nycpA#umBaVZ1M#rVnzvs(!iPKA~!oHs6spa@M7w)~RkBasBz1amJ6AEmAK6p56Hr%kj%GPX7D<&vDyRZhHFI$FzFf z!JhW1l1@)ovd1e_+jAqnIPVvIJzdJsa!0!xO2`E&2Fgrt9xB)-oldCCzN>+I9V2eLcGR4hS~%#(%X7kLhsJa${Fs9*t1^ZpmHqoYv>ZF^2O?pv-rS<+>j|NPed)_Z(}B zgY}r$`)PZx{7$S7HocI3xXoWS&cAlQQx^U2z6S1Z^mVD<*Cgue>;JcV89Cqo-KRLe zPxEcg_XPSqf?O~E-4noh|G#?z{e1G@J;DE-dx9^|gZlk|elGp*Ucfl_8}|W5 z{Qu*#2V@_ToFyYBa)l&=@HhELIhg`-3dzLs zv+Qg#$q(TOc}CyRogjCT2E>)6X1zi7CdtWo`i8jC@zjmF(2T4gJ4G@=7)6q?d@KdX z6r>i(L5h)VGzWz2G#T+Gqv2hn$r+N5xw7OSlamzC1K3iae}rBk+er$#j&3LGXf)&` zBd6i%Y3O+Y=EjbKJW5WIo5YLwg7hVK$Q$~W?gzP_>?CjLBibHhd$J!c-IBo<`9OF~ zE|IH50jUs1vXI;)AIN;fg=8geq!9T)-I*syPvSvb$*;tdxX=Hg5)Gcbyn<7Vx}1=}5YO>_UDA^aqIm z83Cydq#Nl0vIhw#T}dSA1+o|EPQpk}h{GU60_sFMgX|2c&hV@k8A-;FIFNB97SK2{ z9^`mPjUrpowG>$VreIO(sA*8p1?Cu~2dXgh>#_gPcsJkr^Ookm-PCk=Y<; zLuv|{3GoyNvjEK{^FYpn)KoGD;;9hk0-6pb=RlYTVKzuJTSanHGs{iA*>5zSEGECv z#bg0#4*Hm#HV2J7PutL?WI1U=m&4QfWC>XTas^ois1aQVav`J^z_aBfkaeS-X&~!N zeOOnjkwns!CX#JrGqD2R3_%0!jix(5?jVBnp^?-A(gL#|MYll7D7p=vB*3#Bpndge zUFt&WQmK)JK_82=LUc1N&NkB$Y$|1x(y0(eLe0{kTU#J>q#>X!l}TE1gWLvrn`9*E z$vyG}{h%qo!+QFi93+Qe_IrZtNe;lTC+Nfh2>ZZ_^dZq8 zqe)7Zg7qa)Bn68isaPtIy&>*RQnN=iJ1IgQ(IVs@`VYwb5a%b4=@aS(>+lK9N}keZ zv;k=BGunncr!Q!0(B>Dk8TpsK1X-WFr1i-wng*=daIj~?Nm`bUjUYowIyRJ~XBpTq zGMHpwgTV%6VpBj*GO<}CGs^;U8p*<@k*q8m$jKxd*tG2I4CvrQutO8cIdUH4Sde4M zS@?|wtvw6j6s*~)pjRrL3#+sepqU_N!uVGPIR)etxN|maooGe=Bzr;b1v}9gv5K8?CbAABi z0kQ{v2VlncK-dNA?l3t9@))eTIpjDw0rCW_!s#H7fII@DeHg6wIgsa|)NvT&vmnpH z^AjLXf;>r%k|6d9<~9hlk+8Y6B|S&yLbyv)!P;yCvJL2T5%vq4Pl`ZDMt=eO{3|Iz zC)10x1cdGM0r3Z0{{X^c*b{t7Ajm-S5Z1?Sav$Vzcn!i87=fEmq7b`93bVo>uS0yD{K|^3WYEhZEIBR8ih?vj zY@)^3duZbtjMp{jGo#m_1qS;xp_ianPvIR;q3z2cAA@`hZ%9Wo!Wg_J3+Nq?cW8c^ zoBaxJ$PFPa?58cj66XZXEDGdJZ+eQV`i#jfeN8qlLiw7NhR8FwF+OfJE*(`UT$NL;Y!SQWEq{B7Z7T2o0cR zX$2Y#yLgC>D(EPLRHjvFby}8GB_Twj>aeSakcPAtZ9<#TdbB=msH0{&S_I?Jj&`7} zz-BKZ5_QngR`NRyr`>5M+Kp}{5_Q*6U--6uG=}z~eP~}D#psBFFEN_N(V_6AR7Y_- zngY5ujn1SKVZ5g3Xr_+h=^VO{MkWkwxs(oqRmT>@5_ zpl0e#6R8GLqkhzvT4^?r*=R9xjCP?BuvQ}I3HU|OKS2ILPr~mHuwN%39EbJXlC}rg zo?d`odm0Kdl>P<3P}qt8f^Z)6WdU6dayh+E@6ly+3CJb%A$>p>gGG1%;V$fASLk(+ z*XbzuU8h$;UZrDbEWJkafy_q-!!BBg7MFV|-3q_rv?$1;bQ?{eMPTj|AZ!6!Scf(R z*_0lEUsKu`WMg_1evN5ukhSSyu&t};MvxonE%p`xkx8b)Q*8gn?H$i`U(*Ym{ z&@=EGK>LI2PtU@yKUn9p5Ke=Z?xcG_?xE#rIr=Bv4RSZFL@Ux=uqRi9V4)LeUV0hi zWjcdSr+?E+ATQBb@Vf*SZWe^evf)UH=j6)9oRrBiP97kWwJLr3_M1zJ)VEYOs@1zJZctvSFIEe_s3h}bRlw`rM@xbqRDzWxCD<=C z3s|6HEDPA6%rp~NrlKqprV!N z50aG)054=9=?A_^X4Vh9lNj(>Qjvn-wG?C-**NfDq|C@hfft>QrDtQnvl$28g$ql= z%;5Em1aBh^OAB#Yc(xC`uheWG1XmVGQo+eA5^xy!?J2IylLFt4Rpa`u!aXC1(| zPtH<8oRW2e9ex*_zJ7y{jMW2QPs(Jh7O4Y1pu*~a7g(3ng#DPZn&2JQCa*!C2#gbf zbTzQcnOHUWihiUz*y)e7I^+%bsB8wgS;+!97ugNy*A1vs*mRBk z4f1bR$n=CgXC2rR2yLN9$JsfM=h#Q(Bgk5NeQKkY7>h1)&e?&jz!hAcwL+Y#1-OC&E|ug&*rfS5axrN&*s8!KAQq^3Y!dJK9pL(7PBQFm#{_fTf&xuT+WuU zg%Fm5T+WulZ#i29aur(%bH5VecnB-tw+d>kfFRpg1*KNA^=t#k4Qw6!Hn7bgH?vJ_ z4TQ}gH?xiK+sqO`Ca|q+ErbM+32Y1e5}?Kw2(pa?DCKMNGv%hf5LS`K%668>8Y_uR zV{Mebm3&GY2sKy-$k`2YH`~d!v0dy>kbklr@cWZlL0VaJsJR>J2Alj%;Yu)s9!huR z06WUMD@WO3wgU6rm-uPbO(1n5dME5j;mY^ISAv}U+g^0{a+9!13C*jaUQ~4mdcdI z6ire==tEMQ(wd?`Mv=ZGttlPYiL?-+$TQ`+(j3-hb5@N#S6(O$KsI2_*}uw5r3%O@ zuo7P>uaz+%$AIp>QQj)UKn??aey6-wVnD`#ey2BOFbyZ^Aq*iIO_@wXK@KG&NM=(O z(_oN;$uN@Dl+Cn>WreVYWjFm|S_g6+XiyGQPOxV=AglvRbQ?7BCWLr)hus40yaQn+ zpzENuw;(J9YZ(RlRaTLD76l`}5#&b5=?nI84ahZ+3T0uSy`c~m0O}0d90p-A`yFgt zJji%R>aK#k%B}!f%`Snw1gVSg^ft)bV8yRM`WDDr zFxFSWnqLEX4N?}9Vu}G7!`>_L$|L0x$V*D7k_~jZ1IP}_PFCEMN!bQ+8;oXk)(kAP zS=pgn1$kAOp}YcX-=4jK&=K~A*Q^uBPOv{zVCC6MkS}4a6oGxCEy%Vkzv&_D1=V0T zsK!z$dtlvW1({XJpuC6G6bx%A7CNKQH4O? zeo=CQ%n3dH1$L~>AU8uQlag7<0Wt@yw#?9OR*+dCHCagst9mA^>Qsse+Di#Cr7{T+ zRmOlE1F2W+4f_D!@&Uf-4Xo<-Am2mkIeWq0f_w|#_X57>CCHbMdcvLpdII4&pvUm! zDTEjBb&uG6koVzR9>ETK59B>a6)_bx!T&A~;R!2hDrWiz5AYQIaWzm4_f7 zDod2SrZVgwmKWYq%v2NhPYw1{jkQ(&VEL8vAkQoP6b4_P8)R-}o#JI;rf}v3p@#C2 z6;TR-EToK7QY+P%H%M<~y26#cN`8>}q5eRnkFrr22w{^FWb!esBta0?kd(@3B^E|B z7RFqKeYB!di&+&HkS?$imccybR!V~PZ34flthnL@DTyTtgtt^@)xisB#eyLvu|$;= zi&7qb<&@G&RY*xJQ7xs05&`zPAM45LKuThXS}MOO&EVHWX{5Af|SG(ZG~~q1HVnm24x$hB$mjm_$#|X?uOMC zpp;T}f!wA1sRV&PmltGSWv6mlIji&n*$Y<6S>>G46J$?FodXNj1!NaUT~aP9oj`Vi zb$wa6qI3k=5mHxRFKz>}4Wu3_kCinb*MQFaqdZksf?TPrR-S-OhzA+3tbko?ta1nB z9oW;xD&v(~Aa6lxf-*uW2(qAZTiLE?iYG`<*hw`dQSkuj0V$zaVeSQl5?~8YDy=}a z0(*E;Ii<7!*#c6hloLvQko6&TSGlhwfJ^`zdtZ5=Yyr6iQV+latOdCiQs@9KqmZf#d;B4ghm<2qK9j}t z9^`waEUU=MviC|w2=A2gka`R9t@1_*0e|5&$k)m%WhQuhFG0Rk{#7P|7x?1;vG*Qe zQ6yd5FA9pf7yz@bD`3tD7=WoWiV<_pIf0@mK~S=aU;@FM6$He9IRR5=jB8fRIp>_? zs$bPe^Pb_|-sKC=_uS{+>pV0+i_?{Rs=B+*zr^u_7^~yNtWX}`o4zyEh&i=9z7yA< ztC(y5G<6jrn*3TZR{s$3{1E>)F=J+bG7NBXMkG=43WdTo;bw=qSd+PNF;x;^<)3(5${mnS2ezv4L4VGwXa-`Ra;eUGdFi zDeg+;v5r}7v+LqsvX(g365nK3O|F~BV@JF~O-?aVHT zJJ1?tHp0fnET33{TSbGt1+(%ZSG5ht@@AXGop2Q~%P3@0&MeGis<f%^kn`znQ&RJ8>^nUmWY3)e-Z# zPo~VwUEFJ$n7y){kGsyMLcKvju*ESNJ;T-szpN<8un55qE4M!u;o!CP$qjL1KArH$ll06_C^M>A2N{rkb&%b3@x|lp-N@~ zj0>6pWZz>T`yK}d>2y@@@If$V7vWKUyI z>Qn4#3}jDZAbT1E+0z)vp2k4-GzPM#F_1ltf$V7vWKUxtdm01T(-_E}#z6Ko2C}Cy zkbR7S>|+dM?_wZ(7X#V57|7nmK=v*MvUf3%y^DeDT?}N8VrX4)J`TK`3Sxg$uq=HZ zJbbnonC$NgWFOa@wvhHpol8DBYdMA_CmPNT8h0HESgbcNx$cl$cSx=~B-a#@YYNG= zV8rBFK+5D=K+5D=K+5F0KyqEE?io(53nbUY|MY&~0@oc3=8OH$ zI@W5JQR%ZkdoA{NwCh)Z*GPO|TlM;p!^c)$@SwEDHL@TI~U zIJLTtA-wGC1{rBfwXDRrAfWv1O@oY~&$P^bbP!PPE%IFcKsXzIC;%vrtKKf-Yg7c= zwIKi~2kpAR)7E$hzfaSQ-e8K0kM{#dp=U?KDL-9RIOCv$B}?)Q0X5DLHu07vS-DOj za9Gr5r_r@piDAJ&!zufJGKI!sucxjs_R^-!s&h)>U43UO^2EURwwD z?abPf7hWamYF1P{3wHO1yzo%5Z(dpRxy&uqAM(O#^+v;K{itzX@;jKk-~oHwSERoY`+-Hz9t{P>9!s<>Xdmu&FdxFk z{^0a-O!|Z$`M_*ue<&vQLn7CmGr8`NTz5#WDQ9v`A-NWem|P1;nOqAo4DBDpS*hEpcjg)_MpT-2D>e_ohe3r+IEq%UD!nDiyg3#-)|4X5>^ z#(BvbwaH7~s13O;nv(0HDY-73$#vmO+d}2Ka3>2x8V3MT@7U&d-GiW z?*AwJ+@+p_^mCWL|5yCn&t&T(@Zx zF4oyJ5i1Xynlu(`#pP=8Kg6fFA*7<9mLk>4;$tgTkK2hguq_4KiKCTBE1x*#Gbt+8 zgR~VPg~d-1v3kC=C|4U%U!}ySjQD>W@w=>8J6BMY!A;ax4e_aEQd@j#8bWFaDkP|_ zIJOnBn20jA5yv(nrUHUniDN6Vx^bphIcqA8rlRILign>##j&ec6%r}lVOofzg*d;X zSalmBK4IcHTc~)C>McGK#47ReVzqcj@d+`6dyD^vil2^REz3+1+cfc0K&*o6ELMmA zS~EUTtUMneKK(^#I}tWfdQ*bYFj1;*aBR*rr>S!;qhH8fRL>fZ81Qilz&l1O3BFtUHFjE|7in10E zZ4fPv(SpNF9rT6QbpnfnfuP1qjyUM8l3PGY{X|gX=l+iR@1LB(>+N_@w~S>m-gTI?947XHn~}ee@jBEoV0SYlFw+a$;oqWR?2(xb>p0} z=Vqne9zS==sl(hnk~P$utnUl*Ts5|iv)7mIJse~n)Oc(Od;O58F(AvL#$S&&&=>9N z4YE%Djb%M6Hn~}eU$&PTEBTb|uEt7vW&crQms*YV9Xq!+^f5KI8R4XF-L2&Vh?grE;UYB-dNv!bwfix{5O{Mtk~pcC4SjnYOLf_w!0cD zbt3zZ8Y^`o`<@yXTJ5OUbRS^ohibg@t%JUM{!xa0sK&*89Q3Q~#~J#e8ZWQwpl@sD zW9Wzf# zw~9Eo`0*Awar!LO=A<>dGJui?86G#Gk>0bro5B9rsYd!4e=v~oj4)}e@7C7^Wc&)t zbQK$!ALU$`&kk;jvGndVfPq1fP&x#={JtHI)}0L-&aA>6%~E0Yj1cf>5{-dV1-FQS z{Jm1~`u?@@-YLBQ^Gi^q?oK!v9SDEC)5FnHd*E8Z+3-E+He6Z18QN@|4AXvl0xSGB z80_^|%HzO6Uku!O`BEr<`?G;<42pIC&>w^xjS_++s*7uDr1p{!HitQ&{(G!jy zy${>hY=SNeCmONRhvhrJl3`BxtmJd3-l#Vuj;Vyx7kz?jqZy2@Qv<7{zlAByJHd!8 zR%l-PG4ww_3EH+zI9CYw%#z;kE#qD}AMfQv8ebV`A3AD_Z% zZ68ps5&7=$$K%y7^4S~sTD&uiYQG#l<^Kd5e*+j=%?mC@et>`^AF#N(00t#Lf(5og zP^Eta>{*I;AJKtBT>Ld(Pg>W_8~P1)O7m^3><& zSo+HZFkc=El;>{P!)9Mg2ET%_K)FTr`fTj?4bZ_Q1}Hm5MY+aqSPje1#X>2!vaHVd zWawjV%!77hY9_Z!5c-m_aH>R<#xm#wB=z@!%@w+^8aDUf3k!i`dy28X1>eDni@rvz zpV^$9==dCh&ISNw7jHeAQso}3>Jb8KR~3ez;U9oc^MOa{Ct2~X&tcl*07xzJoW&M- z2Zi7H0(tjl#N<7k>bq*S_tx|sHf8eOPW9b3d7r2H&YQmfrcB?3Q>O35Dbsi4YE0e{ z8oftX3n%XpRo}I%y?3YY;3<>$j;inG)!x%9-^01z*{i+3S9_PQd~fG|udnuwU+sOr z+Pi0@T(AsI3)HTQ{J#hCppSf!f*vwRHw+YYyo81Ipx`tI@gy8cx?Js4-o; zpvH7f17*6tfihj|K$)(4piI|5P^Rl4DATnOYE0e@8?B$9;p2BN#;HBiVCtW~@N?8E z{OXemPEY(HtY{>58Il5a8)D$kRRK67cngdr2Ml~S z9#R(sVDBFyp1A2oJm9xTH0zfPW%@<~WzE;S@VY{ZVU9;RF!BNP{kG8%{v+-@q?vDl zX(E2gM>dvZHB0P(ZkHk?Ib=N^T?hBN|F0I9OW+^b`o+li6n1s6mk|E2Y4=f0e zz|X@Hpp1JgIGaXc%)SJu)YO<`u1v?9VvSChim|XY@i4sEu@Sz_i-FgHM_|OoR4D$! znD4$n43{IDzzJcqd_Xh5V`9WaZcTM1}R}j9@--z#4 zsLqxoJ%yKt1A(%YcSF|j#a#&O69W2x)9lY-cVWzl5Lh|?D%;%gDP&fj1c77TvEyf6 zLBV_euxn`%xO?>t_;2?E=a2|=4q5}9-bWkppC(agdT|XLx)%*p@(-D!KY?-c#{lK(iAAyIs(avIIG_Cc1}aqx$+%|vYY^v6Rv&NWLdq0&%-=HjYoKdYFx};L#M_bK>3KtH&%Ai zTA1=O8uG#yK3rz6<|cu4W;Eo5FFGG&tGca&*BzrljeRO6V7SW~Fc;52DBsW|;*_M- z;8>y$P~I_Z71o}d04>wR{getHa9EC;A18wQbAON)(dZwr*iG_Hpic0Pot zd!2#uU{)RnB)*1=;<-6xYo{_ewAx!ZDZZm9Hw%5oUVVE3eO7q_<(g}YfLX!Uu*=&6 zD6dMs%9>1m45l}{fby2zL(>=0KslmWes;!u9UN#K4QgDi-8xP9i?uLD7Y&rV)F1A; zepVv1dJ_$lZHn9j_hu}n67T=C@{u|_oo4qO-slz$s=3*P*Z3N`q6P~$=su7h>| z^>CtuFHjDCz64LCZh`lGf`Ibu@@w#Wzy@eNGXyB-XVF;jLn>Gtn+udLigoApc5jE% z>tld&(B3;>`*N9I=$U~)IV&v*ZHJ}8;vHgdDit2&kct}*rNHi~ zUO@Ss-+HuakOEPw`v7If(-Elqv=TZFj0eh>me0WFoma!%8u6gUKHF#FFH=SRRPF$wH!;z5n0Ht69{p|$Y3cRWxY zWcL&vJ=p|h2J{5Vor^tza^utBb6ZcKJoxDY`2J!u)Z8=?)VO;0kD={>P@vqz*#zHB zdj(FfCIjVCSI+Qn!{5X2)dGR?`VNV_W4(`XyJ`SX9#bwAPHlM(?K}4Y%BxQ#KyuPM zs92^mQ107cCEUsU46GOcP^lu$AV(c{4);n z!oj=tuobQ;Fh3>^)Oh=h&8+W*4j&=5RRB<4w4oT=%HM&fJ`mK{ zEvPxWW&Q##X9R;92UvDtd#~Puf+Zt>@;pyHOTKvz>irP`l#g#a$xd3lfcs~Hf$~$+ z=d94-ckuOUAW)vPqA)CV74vzo0HEBU*cY~J=}LHH8Vzb(Jfu-~n-;OL*+I)%bed21pUlE+|`D zN8;zRDKM)+98fO0FaTSZ-U@-@zJv0oMt2~h(Ke`N>kE{(7rF|enY*Cuhw(tUcSs`o zhph+qQ@)_amER;`vF@pG{GlgM-uN{cYad$&@kROo<@XU0$i=Ff4KzYI1!!Wb2bAXq9f8}9>mklR4)Vf1-W`S>57t3w zVjQS(hn9LMbtVNej>Q4x@Pvl|6E{Qm(>;Ll{y!f;HM1?S@6<@3Y`5VaJgc=8>V6zA zoAA-JGI8feq@-y!*-Vv-l77Ud23r_I3tFIxZ$z-5xVVS^9r@V!t9VP+g zqhC{DT%<3IWP}?dE~Uwg=#GARH)ni1@}9 zW+lV%VWO>7_^jU*X8te*rZ$cPHD1-~APdb%gE*tP&b9&KhXuohWsPRaP>TJ`%XW+gi1l0J}@P_O_@;zu}69JUV`J86Y ze!mCv#XN=b)Y(_r*uu}CV&PDroId;=YccyZd^$E6D9>J81VTjr>9%YVP`>@{8!P=f z34$(92g)Y5uCP&WlcC+B=|I{3X)>1BEc{o+I6}GM?1#X+Z8n5c?)k_LEhqh213)={ znN@K9z$b$}<^5J18rRsqB)0KIqEb}kZlHp#L8tP^Ixw%-1lU~Lt!znB0%5o`j z{u?XhIwj()(eA#XPI9xf`SVaTtY7D-pSkD(+w5b^!$et5>P|9gV|THkfn~ajHa671 z+;zcVFT-U#l{{k%HVVsgWx6t)vOHI&b#e4;Lp*=SvQ1??vL5<(`(3NlC*?w4?X*f= zQ65>#Myu2GOpRcvG)WdE#C(?%=nL5BAe`BCa5_Ge|SQV)kkex$vuE6GZo zCw#NjDs?N*Rq8~>snnH>N2v$dzRGn$S?T{W50qtJlJ>Gs=7nXylJ>HntFdxT$+RfT zdQSKSzTd8m?2gkERsJ@tzCK!u$>>t$N<`-<|X@s9c@B7PY@!>frtOvIU+*I#X_ zmvPGQbE3cP7JOC2PuaD4C9M)a zawGn`_KpabWuZJq#3StwiaMbzt(o%hjQf!j(V*Km+9t(eMJ2< z&rX-Jte<-#{HsWpvT{9MyuQ%zT_La2+^oziWVjlq-f3#M#%29ePIRnc_`Z_ilrM=k z%A98oLc@sCU~R1Y3oT8e0!(s9!8>YV#K-#nJ7EuadqtVOGQ5GiEzp*D>>bHq>+lX+=&LW>RMLuhYaLQ%6)YJQl?*WV3rGau~>-zfLbtW3FP0BZC)YC__4uQAr zDuEg&2Q}0ecApGB;@VN;6?^UV6~%SEQlzWK$D23Q4|Ngi1>Ti|;cxBrqql`aMEgpr zJo5e@+R4`Ou%c9JpnNCMM*Dr%ctbelTeh}2_LTeHwac-mJh8W}_P)61kntbbXs4}t zE5N{`{-~@C*%$+SXb4bN@^EOr@R!NXo8*W2Dr^55B<@Wo1*>w$=YMFAt&D^H-zI?? zD`k;!QoikBqy5+?z>puxUzXWu2l-Dh#7TMZbQ|s0x4wq_sIiiN|Al38$|6}QZ}5>4 zMy%vHuzty${O4w+953tSYhiZJx%~m^Ktm&3{ z`?Hp9P9Ia_L0a3KzN*GbT81{t&O>fi@+{j+jg`D@yj}_@E9uHMRbwUpvQ5=kX)l=v zHCEbP=3kAKHkJ8Q+DN8LS!rLHXEj#7L*T0y%o=K^o%u2hzW?rT#Dhd%`cTv#Dz*;< z%2^3^+GXzop_0V}NQtu3UVRt@XAAm4T7s?Cap+7#Jj(Z#jGyvV(FekWKY@%hKEzgQ zF>&(YJ>`u( zt7?ah`Zaf@jEC&Bv-%G)*i-f{Usb!|nn4#SobuY2wpy!OgAMlG-KuEq2mZ3@qyL>f zY#(cQ=JQWHs#jEw&EIjUF)=zN9yKoN6{AzmRb!b4X)|TseE27pX(={;$4dOtMvawx zN*gsk9k~yba{Uu4<&rjk$JUS5=ES4Mrou)!SB;hUWuE_ul{!)Ktk^5zO8hchjg@># z8#V5DI1$T)*%vFYB@@F8rz z!A6bi_N|y>qsB7NN?L!%N?D|h8ZQ-PSK?RWyTV4vry47DBGXdi)DpJZA{U0j@Y07F zW#v06_PLHx&iBzqTX)L{Xus_MQ)A_OOrA^GJKjzk8`jj27UeRV?6l+7HZsJi#uJ{| zY3Ftw0JnpJTh<=OMa9B<7?L!GPf*9KL!11E@g;G%C(R>mEfH#IhMt*VV|-qesT<%D)s zwJr-A8`4!{Wo(jl-m|2gw#VykVkWssrXC@1`0MO(+?S6e7!nLO9MNfmA3 zIU0jKWko+hXi%UUD>@9qcN^7M(Q^Qt86Zmf|{QYxmap0o6u$pCdCtvBd4=zPPT6y_Qg6cdKc^0Jv!cBq&sAeZ!$IajjTJ2jSr#=`G$mx6 z{2L2@{~Vj#ti&(dOO2I$%63;{rM$BLsIj7}AZ^rG(N&OrRgD!52bmu=Rg(1OTep%IZ|BeWw7=VohAlAJr`5uVwu6x-wOTmQd_bc4V8lf8 z0g2`V63quACOQvDnP@&B(R@In`C!CE=K(1b%?Bi!4@fj0jF@OXAkln4qWOSC^8tzG z0}{;#B$^LMG#`*?J|NM2K%)78MDs!T0Oe>tAkln4qWOSC^8tzGgAo&*2c%4N9*{E8 zc|giU=K(1b%?Bi!4@OKhACPE1Akln4qWNINMCSo16U_%Cnh!`cACPE1Aklm4n+ACPD}s4~%fK*~ht!AQfMh7)ZEBW-sYPIMlSGSPWJ%0%Y@)tG2IU|yJLKB(%* z6U_%C+77BrG#`*M(RonSpeLFSNOT^InCLtpWuo(dl!?v*QYJbNNSWw7AZ4QSfRu^O z15zeB4@jBlJRoJF^MI6z&I3{=IuA&h=sX~0qVs^1iOvI3CfW|D#zflz)tKl!7-{j- zaH8{oYD{zf ziN*pFjRhknx(Y~{=qey(qN{+EiLL@tCb|kpnP@B^(O5vDv4BKl!H9{j0#YU#3rI8; zkZ3F*(O5vDv4BKl!H9{j0;)05RY1x_R{<##T?M2}bQO>?(N#doL{|YR6I}(QOmr2H zGSO8)%0yQI)tKliAZ4PffND&16_7H~RlvM3(N(~_Fws>&H72?WNSWv=AZ4PffND&1 z6_7H~RY1x_R{<##T?M2}bQO>?(N#doL{|YR6I}(QOmr2HGSO8)%0yQIDJT3#R{<## zT?M2}bQO>?(N(~_Fws>&H72?WNSWv=pc)fh1*A-L6_7H~RY1x_R{<##T?JHQqN{+E ziLL@tCb|kpndmAYWumKql!>kaQYN|zNSWv=pc)fh1*A-L6_7H~RY1x_R{_qSM0#YWr3P_pgDj;Q|tAJ`ubQO>?(N#doL{|YR6I}(QOmr2H zGSO8)%0yQIDHB}qSM0_KH@t^($TiLL^wG0{~(%0yQIDHB}qSM0;)05RY1x_ zR{<##T?M2}bQO>?(N#doL{|aTnCL2?8WY_EG~)m09w5;`P-UWffO%n}dw`UQ?g3IJ zx(7&^=pG{GzwH%@fj5U$oNm3?MG~|QR8f1 za)S-!Y@Go^xEg2c3>a+wj-{`kj8ph)Bhd%=M^^GcGyyO-OB>1P`5`6$9 z`T&M-8IRBbFk+$!fRu?ofFYi2zjz}i{`;uL#FyV-qm-BU@+0x(N8-zm#FyW|*}nWp zeEALMX8ZCZ@#QzfqvV_-%QzFl!^a7=7ov>KB_VC-$%;Cf8S6)O5TY7 zKB_VC-$yl;{;f)#QzriVm{-pB-#6I*9TWe3RAa?oSYFF&Onmu~`0^w1wxVOnmu~`0^w1QtC=cvZS z_gtsO#P?jM#>Dp=)tLC6qZ$+6bDbI!-*cTB6W?=GW8!;`YD|33b!tp}&vj}{e9v{1 ziSN0NGVwjvQ6|3UI?BZNTt}Jsp6e(R-*co)e9w_G@jXY%#P=L26W?>BOnlFgGVwh} z%Eb2^DHGpwRAb_Mj%rMN&rywuA3730bX8V-rG=k3QdazwWjJNUA6AA_R(w=tIAz5* zQ-&-4l9Cl)Y~hD)#7Z87?{H3jBrE>c!Vg`Q6+dp_hmLBjltso#S@GAF`Jt@%T+28q zEB@LtKWeO$MfmXMltr>qcHy^c#7drpe{N3xbF)$p!dEw^9&)pie;GezrMxl^l$HF; zc+^-ayUY(|#qU_wn;I+qLDsn%D}Ktt*BdD-{>ZYP)mZUmmTg2?>7O!PHCFt=4ekCP zpLC?*ihsAvo8qr6(^6x_?_1_sjg|NfZS)^M^qh2MxRPhtUTUo5P55^sWhGtNrfRI@ zU$&_lEA1uopvFqO%RH;G(xx(>N*l>^DJ$(O^Q^|ocZl#4$65dJLpNf@7hCv@BW1-G zTlk@);-fA6(2@9|qw;+v#8#R_b)Y*R7_)nbemu;~5J63$3W!cqO z@mZEOYAo|0;}<^8IcfbJXZuPUY}7d0=h|SS#@RmI2AjWQr7SWJYD|2(QH_aDH>$Bx zCo(NHCO+MqGV$r=l!;F_S7YMS%_$R~Zcdr_baOQ(KHZ!$@#*GjOnkaIW#ZG#)tLBn zbIQc0o2xPL@8*<=Pd8U%;?vF5nD}&aH6}jYoHFt0=4woQx;bUy)6LbG_;hp1#HX99 zG4b!_#J`&ppKc>2{@tATcXQ&?ZNyU^rbAV82k6m09-faq1fTjhGVt4xTcKH>ng)g& zsc^Mrc>{MSm;`%UnHzXmzeG6ceV@s5Wq6IQ=>}%P{z13-2A1*E@9SmYsUoe_2`vpQ z)9T%>iGh23IGq#z(C>=SbNFSy_{Z&>cxGM@+V$D-`-MNsNmquaXXi)SEBTc1DEXK1 zD`lMi)Kp(ByS!^Ve8klro3yw$xe^w+Y>n{c(5UWvcdmg4%;+2xhtwX*Y5 zL)bsgF0YKoF+2YooeI-BN3Yu??YO}=^ zgW*B+L^$wj1{YeFCL&xX!udaiXT2)Szej)MXSNS8I+uoT?%RzQz3PhVW2zY0(D0*c zBl!E&QRrWBGoyK+;WA(1Zz%JoV#C#JNLonZl=0=1m*g3#J+&cqg4CYWA*$I^8*wfZ z^~OZK8QIftkq0K~4Me>e+0bxN7AEQqM7Od zhLbimiid`izG0{*rM+ml3={c~8fdxeP^};4#Si)Lk;3vLe82vp=Q+DF=Ov=H&VbJ_cxbAR^`L2{Zq`c91Jv^eh`6(DQ zud0IE>*m@>oE3|=IfJS81}L@T5r@~QuyxlhKs^n zEiMFduXcf$QBMPJ^VfjHuY3$o{|oe4@+oZW`WZK@d{jTQUF|t(nSa1hm#r{{D1~^PIXa{PgA^ zq+F+MFb+Len?JpO#+CB131JZL5}`{nUFu3XZC97{Ye5lw@umQz%;v6k2}+FMt5O1x z@~-h=csDUZGcI{qCS{MrQ5e1OB0oBEiKf=9W&EmJO`ha19*Tc!iHpA@A8HHB7$px>0un$F!?LFJN_(Xo?W6H#n5tlN5meVHDq zd%L0q)6NLQA@8ghwQ|B;E46ez43VmM4$-$^D#qK;@bKe1ay12U>bTor^PggMwZ> zYQZ=*;Q9oZ)w?!!N^yaOO+RafytvFa_jt-qS7-wkC&V7I6>7rX^Y_^7Hffr(zJ9Pd zHIBy&ZV1Uu?P08gmVbV;kfkgb4fZ={^2UA4A^)DvaO~g&-l+5f*RglQaKN7jTrbWE z0q;S-@h3NTx-RV)jB`5G=G%5=8F9W^h49Vc-Z(ewrpu&7Ay`*agk25Wo!R=7Igb3+ z8@oN~PSa7Di}p`v z6^f6A=-D&5{<8~=d{hlI8O@CNV{i~)=O*mwxQ@E(v6J!WqBgAji2z;by8Qe~3t#B# z7spT7_rx3a1>x~1Tb}Ul1HWFh7lc?waP8bntc#-uW?6i3&bTlNXDz$HN4hQ7Xdalr zt$G9S=B_SHPc6QoE7Ylru0qXF`003^9jQ19Pgt5}G_2JJTX~-5F^kgJmzh^s+}=^x z+qP+D$4!fQcv@9%n>8NnlUm@ZtNS?AGlK!E3%i)lu;K#;gy7Dvt93t@2IEIhOE$S& zDc3K1i}LBy#k;URqw#{*aTXmsRkv$s03P(5!p{}8)*M-W(D_-L61<#G2qsl-z{f6` zgt68o7`3@(Y3ec{ycADr7J_Nt&uhv*3POuIC0Vuh-63Fu1!!BJ(G7j}EdBh6#(d7D zAbjoqyUUJprTCzhA=tyA1zS8P5Zx`T*&mHtL8)?;vAoY=4dvL=p0KCTDR#*uT*rD( z&sgAV&3(FuAmufSZ)naYM(E7Hg~0QhCb*}@0My=}nrS=B6U@C&@!P)hG?t~#a=o7? zPOsh0C2H&%W|`H2PZ~EC))r|6vud2=nqJ@7l$c9=qooJNF8-N$wtio@+wCmhdnAe3 zTHWA_9*)Fz-_B=Dbn|3mstslf?)gCfK8{c&%?YR1IG$l~&jAvA9iXnbX4f5dgxYSu zzLP!k-DcKfZnFJz+CuRlN9Y{mgza}{rGten)R|!e8EYP}O`9FTs;?7H+nu$iZkO8l zWupsR^!lv1R9J(jR@cEnd$XEkZ6D2-KOD#2JevSDI=Z5bM_tqwtdp6tOlPg#b28>l|f6}_9)#j|GyX53#M?mFqpY;ICUynAif z5O!3xheF@oc)mSZ&W~Hi^CsQ>v4wk@W|Oxc^f(m9^Y`56QlaNuessA%K5t+SA2xS} z$4hHm z;D?NikI_6WRcN5Gn)tSKQ|Q^b1Gh*h?DEVlmcKnD(zTA^85UlmYG(2qb>});kDkL< z7nz8jHa~e_)gF+vG=Lv8|Ki-Nc1hl#V+ekjxmVLre5VW@YsI_=jc}QLrz|gR6M|cB zZ(!p*$D!%*60B=xX;-&0<@maaA=uRP2h$53spTm}*jL|S;Md|2|MPB==4t&BF2zks z@!qXMu)u;a*Cwq)@OX)m%-b^*0$ShE)#@|dwf&>n8TI>=;VbHgV8=N>`O2#O@adYu ztcAN5)@rt&*K9piGtc}>x=a6xyx-4YBb%#RCgb%X?b(m$V!Bd$vz$9Fisysc_~ZUL z`T1=(U$_$<#~-XJ?9$dhj!*RU#|Lqf(J`zI8}-*j-Lm1eT;hU+eS3c#^6&#s7vCYP zYe(?VTJ2oAJ&)rnhWKMblb-mhNE`AH=a>x1p81&5r&OWII9^TFP!O$SM+PDcz zTiHRk>yn*IRr9&LPpUsoKQNu?ON@nK;WIh4@BG&f-Q51?bOY7|tFYX`1~R ziWly82G8!6c*gmb=396pnEkLGx<9aCE0Q1Uu6K>pTsj_th5F}bJ)AG<+J*;X-JMP_ zc13+WcF~SiJ@g@Ck$yZco)w5yTK?v`@~b_sR5lp9=I;y#w_4%(*H<-kE+pniDCS5^ z&U1G7j7IG$!@grC$9H_wOq_d8cX3lNZvR;uhb-5iUFh!gZacfASB{S0KQ;s)cWd|3_{8;N`%1r zdu3VwExR%)JH)Pb!OaoeEI9xv+w=*8MC;kQh`=OQ%DLu#DCT~KIi8#kioY2+H8KpB z78xSD`h&@HE;G+xAx*GU9$thDCXoK=H!OD%{8Y5F{d?z z*PN-r0z0Z>*GA=8(Usw@m8(wT?OFxmixaNMylP;HniZLthk~4s$~F>nR1kAi1JgNb zrww!19}_CDc~5;X-K{M=U$>u4nDB*ntJn|rSf63HG86f$9it$1%|%vXMV#)!-7~I7 z+lRmhQ!DP!Fj~|8LNLU6TJrqGf}vYt1K#$pXacWTPA9l>6vkO-Mc;h^gqdBH$C8;emLN<2nRf9 z^;Y+Kq%D>|V1sA>y2p?7zQwf-Z}16L?&#Eg6rU9~fLqq^#R9Wyf@$fd_`Y*nR+Rr_ zgDdsGNMCV`nDiYb675 zwNoJLVZn5iZ(sP#D-|CAZN;;hp!*}xddUqIcYKX5vD!NBXz2~-^I39hzn!|U&cSfq z&zk>SX2M_VebFT`hGlIF1NVzJ+6vGflvIOAzGyu+Iraemui?0d2n51z<0PP@Xy z8huw?uca+q_nn^%9~QXrQPw^he!C|WvM+!aYTK}dC*LuTmA$av;0X4;%{PtJSYP}Z z5y$9U%c0q6*AE>-;K&(E-f-nX=crO8_zT|GJbYo{s^vUX%x_P8D-X+kYGR9ycR3y3wIgan>3uHvspuOWohJ;B zxWwk9J>h?*wLv=W_v+`tKKpv{C5?QL&L8Ob+xgEq>_8bW+)!mEqvQCN)rp$xxBc)@ zOe~}0^0ttlY};==@J)CC8}C^Y+}<_84P`qpI@Z(iCH|`mnmwt8o1>etG`nDIxuX_0 zPu6JYnDt|47{1PO)s^yeb*1CZw6rj8`DQGxC_0PLc<8wHdXEX8g}$gwj$w36|JJ7< zyH_{}cK;~JABBWM&Dqy<=~h!+#aN#y#(EHAJ*PId5g*v;cLU(q)gqjZ>2<$MM&0{X zEYrM*j@mqUGY*`p>||-*ChF)s;fI*F9~SfWpjCx41}yFi(}tX3Hf55H=Kt}JM!{$C zz50C8eBI0R8Em(28MZXh7i))KVFfZrVdkICnWooUgRbg9*5&YD;=XO0=KNSY&0kre z7^2Mwi@x;7;=8h%(z)fn0=M|A;2XR^DR=BH=BO3K9JRamsf>ivBl(PN19*|TzDVbs zSNhnZ^BEhwBEF00-1Xom2kaN^fJeml6`hC7IrNV8THXuYLLwNQ&rG-erm^($#fX?V zM&~CP_j|&~Y6Y>bhZf6Yk@MUzq%0@KA<{um2 zjl*t?&Oh%2F60kZjK=pSvlyMn?ku`VH^J8rHyns#y@sz(-*mMp55ge4ysZs(>0c2& zy57^!`v}TEld7YG7Qkl16Wx&Jy>R4u7 zqP8Kps&)!*>FN!=%3B)Edjp$_I0pp7wsBUR-q*}n;fc@2o?;ndt`Jh^2Csj6BwEk7 zmN5kT;f;G|S%(#{jSq^Y;+9-!iYK+Otg+f}!W6?%1}Z1spkcLpSlj2bXr^$Fs*50-GgKwH$kkvU8r!%h{0{(qIF?XB6FlTOM*P}OQ-~8)XK(H&b5 zjAu?;vS@M7wtQ#_R(C=OWMwqqKbB9z&^N`{h4ulMFk~v9diaNHn4fiK74u^3{?rih zOmQ?WJaB|L#>eWY&GM|9&TkquVY|-;f!S|8VeqxW@Sw>>U3=@PE`1MLvl(4Nfbxwo zt>B1NW!zUIT~ns*XjtocoVgd9rF)sbCF?&e5JO&DF=`|FXNKsXO!QB%8+=`J#VZtx zuf6D6cv*`K%f8kuu}28p?tYf-86obCH*U(<7U%&#PM_l*l_E8v7jLoit|Otqz!4eT zzCV~1JI5EEUczdv*vdwwwdNJBjfOpy?y!_QBY>TWbA1uh4jQjG!k>J4&dxq}0Pg7k z$*G3>UljMhAm?~KwQb?OuMKpFe84oj9C3O#Cz!swOVi^AZnHm%-ej{Uv=#FLM?4(m z1c%SwboN;`nq9a$h`F5efuNC&SagjO4D&waBE}RKasSA~{Uhv-uZh*VGzI&pmb^pT z73o%^=dhJUCc>TzKY2k-4>&X2SKe1yF*UmMG=b%CBvpEdQj zr)fG*^n>?v*CnPG5)Y-mN`t!=nPd) zd-Lffct#2LxopWwk>@!L#XdRq;5)^YA72rk+3)0RmQi*h{8=Hzb>#I>Ea#D_qtDz1 znl8;~RHzs`5g7vJYYOmHwt;vetOvvEPOh<01z6t`#^Dv#7sWuIZn(7Cbsc>c=Yy+U zzuOmL^)`h7wTZgug-d5658gUe^RwRE^u_I5*vPaXpq#H~Fs@l#i=Xdv&Xw}-y#v5p zJo_y)skMf3aO~=gx;7fNbx{yd?&DwrNniTn%Z=u&Yy08Q|KdH~e0qw;`;28~v*G2~ zi*h1w&o{9ki*a~uK?$}mYhcFQJ7rl#n-KV9nyE=_7>utLmSkSfpQXFiF3B9l{A}kp zb2fjvFSHcT0L5G)UECuxagPkC`=+zNqGMt6;+g#MI~RENxEhRm(~OHbNrt!=X5wBL zX!!HVLC|7c6V_Y2&k%E-3~^7*#62~fS{??6OZCxsUmBmex6qgLl5gj-`_ug)sa}Z; zXU{n1;xF2I-ej!Vz74x@IzacecDszWx8vBTVg8_L+7oZqE(ov7*>W-dW{BqiOgsmG z(;q+Z#1g&W&F65wR`logi{e@THvZs~nxC&~WXpj_z($q-#G*)AW|S z4&UEB7-=6+3mVqu2*( z#NIB>U4})llUo9ec!^&a3|}6utJN>om2%hk)fsPQMz9O(1Awyg#4y}BGeqOkc10%T zR+sbfr0bV-A47xDrE4|3y{tKiO}sNIdh0aJUsl)b+#HJUj+?-oyTnzAj2gCOXkJBD^Hhb|1Sk*vP#uTY1$bWImd@I&4_es`}hwc0EQy(@>v`4>1GK zr15!XWi^*y?{dr!tiLTT z*4bdm`}a5W)ws*)Sk`!aZK%J;1tZ1t z0Xmk^F>7A)O}gci{cw3~9HV1aO2$GyG&^ytNdY6hB-c7(HU3o z@n-(s5t$vtn6*ZXSysCmV#jLsnApjc73uL_AGe_A+~cv2+(lLIBz(3 zb1Uoq#!q*#MhN^dwls75ZD%HBEB6)Y-5$HJGAThoxoVpg88>v|R$*=sQ2w0-qT@P$ zR^C;kqnzbyo!LZ;M>oZI^t0tx-fu^LD8H-_j~Fu&zYM*`YW`TGqhr(6imCkOgK;qY zX9-Tnrk9&`={%YT!`k^Jjm9SWe1nch^jU`(n^25dD8?*C$0HivUA)5{`>+;&(F!zl zY)TjR;Dy99kbrR=U7apXhVjGP_@Poh8aih6c+eBN)h&RRzgA{+%&K+z9Xq+I7nbfG z!RVMppOerrOFJ_Znv}S#d%G~qXq=*Blik$Gcy?VUHmQiUPK-ey#vsG`%+rYA{@SZNt3%w&)!_5Ie zxk*$QIyUsyMAnSTq#Qjs3@$B>)*W2K>Yldjfk5GJ5;mD{yWHAcE2K?r~Wa>Fi#NaflOD1Q=9B`4EB_>^YW`rQ-fgU z4l^cxxQ}>eoH1??A8>9Gegu0~vQZ$2-`rlLHD07uN<4eXwq-9kpsW(|)N;J2Mwdw%uR|AfoFB*Vquxh(hpVDCM^q$t|8;qL10s_G^u$qEt_#ej;W-DxlZ zf=Ccll86W>C`0kpS+I^hA3x^-PNUDX|ITfEGOZi8x*E>m68?NN1*HfQLo@>QQ>{f&)5tWLw|+k9}S z7ykC3;_>?gVX8yJRL3xo&qsdrD^W+M(|anY6RThEA3DF5hVjgl7LlYKc6jRi%b|@o zyc(J7dh8D4ae&Y zqfS4&@UV?J!tSrR!d{72ee@@+`i1@_gcqDPLsgz{5$X2cl~D86p^(9QMslxB4$=Dy z?aa8R(612bS4j5@pH5furxE@9^C>G*Uul>ty65V0-rAEZ{M4Wo zsr85UQ~OJn`wz;B=r+)>EsMvmHvqqL<9y2E%L5YHyzQ4x)mtEa zWi&l=u64aihA$6@JO!Otn!?!lAl=VQ&%F2bb*0&>?Sj?GSTifJW@wn|OvK+|&c4wy z@*LJ1y|1axEIhwMd0&ZCxo=eH>t7~>Di(S<+K*L`)qa`~T_-9H*P`M0I7d_)GXJIc z+3C=9TBaKJSpF1Wr(apNYb8zBdCRQcwGuiJv?n_Mw4K8WzZ|*uU@V=shqm(>{6=1H zUhimLntr%0Y|*=Bq-VA#{gueU?!!VquH031%-UBX<(?R(^JeI@EbwyV;Q0xm$tcUeUJEaZp!d|5B7r7&_7RWTWF#9Xf8h6xC*07uBKdRO6uQ&FuTGsx%$?VKv`M zFQAh#{!~Y=4Xhtk4%Nw+7dk%K>CkjKN7?l;ke)Gyw46ZSY1=aT2>pyTL+cIX&CntB zCd^$RJ!9Tg`LSy$>K@9dH(0*bSx{tL=(#ps{XvUrN5@a)xw5R$DyMUYwo?Ir`hHmneXMuVS8doVy+a4bh#&;_VNUHYbc5XURbv=m1GFy*V@T#SI)?H2qIKzUyJ{VhHm8-U-QWMd{IAJX>XfYh z@#ud3Z$)#3K77-yLBqvQ_VdT!JMY&W!6_$nWbb=59)n###qBvmmj_l! zT7q&;qMW7`^QMyWSJN;lXEn*2M#`jltCKQo&@d?{+qTt0+ipbLo+(!?iS#R#hH2Yq zn3SJwJLlp|VeZDUZwttXimUmD45_#Z({!b)uEljS@&@#&4lO4m{og4^wMFTvx-$Ri z?*sj=uVto&e)bRgaOfx>@Szam=P0-Q#{~aBz8<+oB z4!)}!RiheNte*s(e68|EMnBgi(2nf;m8t)aI)1FoL>=Ufr>hp~Irz79651T=Uo3xS znd)zrSM@#GFVf}&oy_!Td-OPG=sap2OY8N{gQ0qOf2RA-9eBUU-=mk`aZBy!zM}RB zRl*O2+opf;{n}8O*E;(b2FP$D_j!NN=}!}MkT-<9kziibm4v#&!MadpGRh3c%M78+ zNU+RX*s-K#IMI-+;Hwe3r;)m%dzjucsZMrbQf~x((e-9q4jGedb;x)owDTX@pvP0S zL62v)<&!pqvMZCcLCtBF&jOmB-JAyIE2A7T=GnG^mQTZE%+>nQW1dhBDL+i-o#aiJ zFVf!#8J{reSHjp5yKH4+?kjtHwYCE`>{+J6j$3mDB;=*()CQlHPt%FLzp`P^sEeit zcmT4}QCtd98+g@zS`Mj8*|2BHcR|{$Y}hGxK}KC2ocVQG;;ntjc`6BgbC+EDmiDKL^vPP2VZ!;4PBd&WAAJd`Kdk4-vxo5FvJmKH+=_6V3;pa6TjxK8JL|`QQ`I zhcMM4_Dw$FdYMbtqLprsaBb*NrVwYE)a6W{I{asbU`9N(s zR}#(#YO|R{I3K7@Xf?w5Ky5#35F0w5a6V95IiGMogorJkPdFc_&7Ds;AE>RUPdFc_ z9jH$@AJlnf&H3OH&If8+>J!d~5aE3A3FkwYa6b5i^MTsg`GoT!OgJCX@2{A*M%_yV z!+js>=M&C{FyVYiBetAA;d}@a&WChjADW;;I3JRUjigUFAE+&)PdFcviLIqiI3K7@ zrB65?Ld1U7C!7yS#5UF^oDb?eu;zU535P?7a6Tjx+gqP-K7^<~;e7B3=Yvl;AJPfu zLwHsu=Yvl;AN<=gIUmA=^T8*a4?f|1NGE&_KH+nyPBaha_ALKH*^q6CMVi@Gyi4$AV8d z7DCrw-@@-)XlJPSlJ`P{Vu@EL43t_^s5GEW8A;Pf`A{+}L!m$t{919`Bv5-u7 z6_RL}@G2zJFyUAT5srmq!m$t{919_0gAgWm2VugokPs#u3+aSoA&qb>q!W&XFyUAT z6Wal0$E5kJe8RC1A{+}o;aCU}j)gSBvEUPqg>=HPKsl7E5sn4Q#k7)eEKqKx8iZqk zax^8;=N;i#P`0R=E6FDu3zYlFCmai5!m&X4lzh6M6K;es;aEtg&j`Y?kVf};!m$vh z<Aw-{{gkvF@>JW~FWcpkt91ArF$3nvUHsM%E zCL9Ys;aEr_919`BvEUPqg;n%jj&Lmagkzxw;aKqLJ09U!2+?;%!m*I>+$S6hVfy|+ zI2I@elutMoD3_E^I2O|BdmZ6e2-EjN!m&U(zI?*5K)J$v!m$u091A|-SfIRZ3GeWP zGa;F9ETj{T1)p#%gs49K1|b{^l<&(Y919`Bv5-tS7JS07kno;KI2J z8iZpZL^u|F!m*G{I2J;LW5FjJ3rU1yAwoD7Y7mZv2;o>*O*j@JgkvF@a4bX!$3lp3 zEJO&$LK@*%SWP$MXgRa7Sf3=YJ_ksgbBw&gm5f` z34cNw;aG?ej)i2ZLpT;9gkvF@@G3-TnD8g06OM&s!m$vhI>dH0LO2%Eh`nlra4ggy z91H1$W1$A&SO_njw*phcV5BOD9VE;W^KEJTQ1YAWGa2ot;1 z)r4aqop3CuZ(Gf=5Fs23)Lu10I2J;LVL2*(1ouMZQBg@km% zpO8%d_C+`rB2S z2SSK&Afyovgb?9CNF^Ky$@IBHI1m&+o96b35DtV8;Xp_u90(D@fsjr(5F&&FAwoD1 z(h2WDgoX*HK{DY$Q13aK10g~<5JFU+K2rz>LWpo6)F2!P$%F%;2H`*m5e|g3>6sh| zKH)$Jy=~qT8QcHv$n&|+B-7^$;Xt4qKC1`^0_E^oO*jxJhfgZuK!^|ygfQVipnN_N z!ht~fd?JJcL48AM4ulBdK=A3ahHxNIKA%*=fe@n48p45)OrJIMnL;=aC~r@Ma3F*U z2SS8!AcP4ALWFQ2gbAmCPdE@FgaaW&I1nOKpKu+72nRxhrV|c?WWsxpM8kylAVR~0 z_aH*Ug!dqsh6x8kgm55)2nRwkeRdEI1j^47Ash$^&kp+c2l~9Ae~Tbo2a4xbbMmNX zisnWL6ApxQ`s^Sa2x)`^A)P)`2nRx#K2rz>0_E+A(7$8QXA0p!NO+FWXUAWI`b3Vc zdo)z?a^7UZQ2<`Z432^@;U$RBFySRgqG7^IkVeCVmmosJ>ikFSU&dL+z&Xm;Rb@M^ z?68RqEc6o3J;v>tuS(Z>37^1KI;w1?mz@r5)HOcr0Gb~lL~Ms^5PkqZZp*H0 z)wQjEG)(vbw5`A52S^ChJaM~r#Sf57^#~_`Ps2)&=Fwro37|O%66#dE19Y5;Ex+oc zwmYstZ26MVw;cis#)98>b0q^Q+>5o;&%7Zb5!~)F0tiLNGG=Z8EuP~ zscrLpl&Nj`L%0UJx!65HZ2482GxSy4$eNFr8*6dgmOq-NZB=x6x=%3~!%UqFTYj`( z-;*(&xGjIe`VZckHnTA z+yNQ3{9$6hAE9Alzn@VjtzU(SEq{dA@+X9eEq|EU@`vgDMeiwM%dh$sirez5G10dC z3H$=YmOsiP5Vz%zawWuV`BnS1Eq{dA@<)g*e}vfbC$tUo8`0~8*zzMSGo9G-C!`Zw z{)BY8W{3@bgsypImmarakJletL&T0gBVFA`#Ew2AUA3Fo(TC~UBzE+wUf7Dp%BNvs zJ0GE8T~54SZATxhSKHAi)T`@JWe_|1guKL#9_g9mO!Lw(v4xM&Fx4S;@eyJd?^7LO z7awo4YGq19+oY>le0-wqMOfSRMwBgh%w{&h9yuc~?AhbGu`+2n)c!i5%uJi~xE*jT zFR_PR8MNn(8|rLy66~F+jkzvG&C!3+4xNlobZl1Y_NY0`!VW!Q{Hac+ow6QdRZ`UEI%*58 zVsw1y{L(R`>2&;6{@B07KI+tX-!sOLmXlriwEfV}a%}|iX6TT5)%<4E8%WQ}4n2B* z=&=o!AMc}2?7`JM#P6v_)rcKBxz`$1Blh4CVuzk>USfN`l7<8I#%;+Pt;(F=MyoRK zmt8CWq2732eJ{lJ)u1i2nm5(wEYh(j83wg6S8=UR)3XaJ?d)u^VfU)+(6h3YR(9yo zHfj6WVB2)MDqq{x26eKt#m>%lJ3IU9xP7#4PnNdZ@xFwDZHV{X52n+1ymy7j! zY8xJ(JKBZ>TX(YkS&&>nqFmy?x_zNY0r z>BO&PITt#AhRn{!Idg3#+Cj%{oC9@aSyxmiQ~w`zlpQzp6LpX`p037AkMF;wlVNk6 zInuFwnPn#SE2fvB6N$Gcp`6UVMBAgsAVY`R#=;IaqV2e;oh(hy&SsYCQ2Sh(PV9x1 z?Q=$5+1c$<9b)I3V6#l>O0W?|-VA%>?82npY;BEcUNR=x)01;LFkYeY2FGQnecR@Y<-p!k6;x-m&z8W4cGLKYu0`SJzcO zt*7P4Ix=J;pOUKlc}GrxJF`?=#_Cn&tNJtRi`ARio>;pw+i`V!g6-1n)9s+`%c`AP zugLb((e{;!%V7MTUq!S(N~-qKc(8xLxYDQNado_nf9IsPLYFa)8$TJHAys=PSDO;5 z*{7u6rr(UD?|YT>YtC<%A@6f;@Kc%>PtsDAn^yDX(BY3u`Zw?H9BQ(nqW@h^?x%LlMt|G*eU6Wb%JF|q z!#K3@S)sAfaai+oH>{IVew)VEMaM6tNKdSz8~jpJ)<@Hm@PB-%>l@doa!UC(=Z)p_ zex4SM7c9{tIr5z^Ays}l?i;6<^e^3*JEUdu-s}B+ z5fRc->C<`KHLj$8=lp%i$JSi$pa1oAh8#4Zq~EpgkJ0wOy)rc__tpG4DvR8;D?_S& zRsH>VvX;^E$K{CjqurhxqjFIwXH*_|r!dC#DgW$5HNKfr-CrsmyU*JAv+tY&{Y-gJ zuh=+KspntIi@#j&r;IF}6qIWwVSTK7Iw>f>jjsnSRlC(X(Ne98iSczoWpMrIb)?f( ze!UMUsmA4j_GiLr z(JFRdXj$sxP0{-yLslLbi$m&lsb!T9OZq7te@xc$hflBfYi>N19F)QPL&xvkeZBv7 zQyB`%8<&^#jWW4IL8;o`F}{AaJT)e^uC+|*TR7TJE$_^$`1ecsS{}YHcE4yjHGUsy zxhCmY^gas8nyIn-D^cp_lD3)(`obFHRdNBS^&jGbBWc9qtsvU{t=;vsrjP+ALFSW#ajz7PHGB_?Op560E z-G}OVrT3%ko=d9#YM;^jh2B4c&msN1QvLjQpGS$$A^m(RhaPtV;_z0t9swnGGpJ1?}K_iv@AvT!)Seh{WG|4=(yf* zv`pMbO8Gjj_aPH0aRrG74I znfUz6d{2u#FR||h_G`VbYpM5bE%iRHWbk>X=0iUhl~mV*&p-OSRQmM&Lf`j#AJ9^# zXZQZ0-&eE@KL2!FKPR=+_m7rpU(xR)TGICqbv^igqT;F_`Z*Jmnfq((yD9d3%AB{% z=Z*S)QsoApONr%WepjjIPjJ80aaF$h4$xAq54HblsqbekgZr_L>+gi1{J)tm`W>P7 zbG?siN%wQQudDQ|zBjV^?$GalYQIhCipgX5{=!F^uG)w)Z3pThGn{&x{2)qP2&s+V3@ar(Zc`bFO>gK?#w_Zx|A>r|@iNK^mf>1(3;3z5EWO5bR{Gew6&XH7_U?~8r@N^;|! zL!#y6jhA-{*JJH5jClXz?MjK}ixschP2;M4DSkX3^i^E-FD7ICH+(G{)LI+8uBFn8 z)t@L;e^h#`U$OdBf3IALgktSp6H@J61igmnN3Q(aqxwY_Y>%d^xcWP>&acul*MVWg z<~=6WxT?68s-N-oqSr%4d?EBMP3`LkrP7bKH!jt6RUefS>qNz=jMsD3`c(6#+M#89 zJXKP(+)NoAXX7&F^NAfp@ZOy$m7bnoDr5EPer8I&UevfHuD95FQsvQgM9Ww6q2hF1 z>Ns6*iR(xAC)STxKU1RX9eLyHP|Fmo+syfo^(S%NsQj_|W9uWhF7&$6>*lKU5gqrG z=z3D?CwTwU_&?q6@%tx!Kcz&+1>K3?PgzOT|C;gpsHU&yUCa3VXWUo%{?Yf9zW;*j ziLO7r9)s(TuG{R^MZDa9c|RyUeIIG5<}vGX|LglQii)p)y-FqUfH*HYcD zDxN7*QX56%RF3-od{nRMXe*juNp)RIbzQ}^)OwlqWY$N^RrTsPZHF$0N_Ab=r{$@8 z9KU;{p&yi|EjOATlqZm$^HNPeD9@oBl^&FjfADLl8_EyLZx8<&O%KX_NY90Gg0eNr zQRzYX_nntQs{KKka?FgjKPdYkU6m7*IV)X?mLHVC{sj9SY)`P=!SaIj2FEuz55e&X zj$^QY!EsRQr{=_l(eU1r2x|X-qEE3h<_*%}WY?Zf&ul8#t z)jm`g<=dmjM z_4h))pp3Q?^=PTeQRAfhqvN{2N~-ZxQjNm{pXcz;)E^g>e?Y48RsNt{2z`!mdHGZ< zznTX%{=xas^Q2`R=;cRwAD%7ZtMZk;hjM~)H^xQPtEIZW;K9++@l@lY##c)<&T9PC zxX^h}>8hR*`}T!Cg1jE`OO&h9gHq+=7&k3bb{F++=;`rRl8(QgAC<1!q2p@&RDIF$ zL3vuL@l$$Qs(kIg-5*lxdik(vQQ7*<2_dgQ9={FbPL#8{!MLbQgH(FMCrl1)f~@)6 z#L!P474q9cPu(9{u0%Oy&`*_b#QE!@>CYT5oN>MR%$?DE9}U?X&8MW&Z?<<|sBB!` zj`D_#+Z|GJTk4n5_AG{;YM0Vi?NH??snS(DA4C3*pXG|i$L`PRFG6~BT=4#qDb;gQ z#kEw=wd}7yazsS0YuPUOR5ZVq!Sq5vS_z%$Nzl$vv_ZdwioF^iS`d0e>z!>e}aTw z-U&$0LSnsKRnAL04y$!x1m*XC{uoUU%J}-r^8GTQ9r1Rl{2!s6L88ZGq!Tzhb8V5f7uO#@?qEpCiID30)U&bu zS?OtgJw95h`Bvjc$5D;nrP9BKRC#~LWprIc%fbExdCNy9qEe5O9>?XM2*0!SZAeSq zZ#^z!CY}nZ>sqR{iTh6NW!L1jak#b){^Re1*zZ&7cde|?6=(kLt$!EOzl+n~y{`UU zEb(_Q^}A8*oN`QN))%W+U8m=}R9x4mCH;LUf)L=YFr9 z`^7t2*5`usc@8au=dyHMpW6ybeQriewf^Yutl6Er37*f<=X7+sDo5Y1N~(3Q&(&y| z)p;7FN6+1;e(Up0TI%y2TI%ypS|*-*x_0iDo{!h((1KF`&KQ(x9`$)JE$KNhbv<~F zOrICiQlGoilAg0u`Sm%ypw#E`5~Y4_{P&&vrTc$ye-6e~dxQIbFi!V>r5}v{(|(^- zeTn*4Ki~9oib{PhOv~W8Fuh;sb6;BOb6=|6*Usf8+VjwP*5{hE44!LJvN<7r1sVH)d~h?Oblmen{`DTGIXa+PU1=IbC&M(f1Ga9uc!6OniQ*eKFCV zG5ha?#PSlq8}vRKlzQLQQnlmSx!ke%{a@P$>hri-UOSgd?GyF+T(zEq?-9W`eLg9D z8rRRMpw!PTE%kX^Emb-CIi}_Ru5-DG-}Axut6*I9H}QQd82``OM^u05_g?TjPJhqo zbbXFLDD`=ME!DW{^ZZ(>^ZY9P+PVJ!`nmpV=W73_&ei_+?hmow&945Plb&x==LmE> zc%DtA)AMgS9z5@=c}459P<&mx6j?=L_jM#B1kJW8DZ z)l=h0<3T;0U(4Y6KphXBBh>NWc|{!$o@>&*F zW7P5Bc}X1)o~zVx-488wf3?)@&{DToOI?nZx;`!SxN51#UrRkMTI%uBQuj+s-G43B z`GsreH8am`2G42gxEiNx=QXdL*VN}UwY+v-^Z)g{raG5-_4~bkkG)Fb`IT_)P{;A! zuFo3=<^TNo!~gB=zjl67t+TAof9i99TB_&3weypi=SS6eTsuE`?VP3hpt*M5^uKQ% zn1*F=IM>Jtm&eEtSI8&=SHdU-SH>t0SHY+VcZYEoTxG+A<5B(mQO#0Oor=+`W;DGB zQgXuIW!MHYsspPV_ZpRq+l*&`&lnwyl15RZ0kDD5*bs(eQ~_2ol8k$dJB{|h_QvzZ z^+s`{5wMZ*nBf`HNCqZHO-1iEp2I45&ZuUTHf}JQ0-G9-8@Y@eMonN%<36LTaij4B z@Cl>2k51&zGMgTM!ky2h=>%|>frYvXC7uu;IM1FU1z zH*Pm>G1>sz810N=#&t$LU_IkeqpDHEcmeo=(HWsm#*4rg5$j}hG`ax0AlA`%*60fC zirBM8TjM3*ONg~Ko-(=tyCL?J(bDJ+?2cGV<4L0jum@sK8qJKCfiEN0%xGe~0(=Fr zCPqV}C$J}C4UI>PUcg?6Jz_j;^al1u>|vv}(FfQEvD(H1#;d?r5qrR>W%LF1MXVP7 zEYJ_w53vyX(HYnov2TobjCsI$#z5~rn%jy zX&y8B8>5UoW`Fokjl;&n=FR3|<7RV{@v-rTai{q)`~hQ_G0P}y4uk*9IBMKyE(R_( zh8rV{xkepx1pEit#IIOM&l?@|PJG zfEQ5DC&min9Ppg+m$4ju9t#|6oJD=b&Ch|K8=o1)&64Ie;5OrPqoi5X+zQ-kY(rZs znp=Qdj7>&G^LBGHaI>++xZS+P+z8xgY)1PYF@FU9XdE%>n+?tHf!`ZH7>}9{qqmR3 ze+R9NMsv0qz9rjh>@%vHd*PGKuZ=y%`^MMsHOv#nZ^lxL@=}cRZ^jv832+Hwr_rwz zV2be(#<7680l2~V)F@yUGS>sw8yk#5W}*?iYHXnbqjWgdjT$NbhffVuq^{w_0YM2uX(T&6T5##$o> zFo&7TTw}bBb+iWFGZ&dX%x8elnC;ES%qC`YU~}_vbFA6QYzb^>mS7XjZstqoMELXO zR9t@?__(>t>}|eoo&cUOUp1SVP0brwGx*l#`{o$40DB+)usIV~S_4~~Tg zk||jubBg(g*$BRuxzwCtZU=5R512#DC1x=;1U}syV7_H;0B$f-%$4RVW*1-=(_r1r z73NoFclfquU-LcFW_{uNn=hLS%}&5h=8NWVTgi=RhW`!ar<)OA#2ji4 zGuN2wfa}aR;MSqvZ@@P}eQ%j>0^c-0g?rPS1)ODWf}4e&Y=Yl{w$3o81E-tY;HI0? zfYZ#KaMLh4JK=YseJjl6z~$zb<{tACa~W`%`HlIt`7wI?HT(hGog0j%>{Iy1* zvN;Jj$vg`;2{U&V{yfG#)m#l+ZTSkc^&%z{zG%I zS%?*3i_IeJBl9C*G00+U307ALc0C(wUeCsvSW*Ffx z`G)QIn;@Kfop%#muedWMixOl(_%?su- z?9El#GWhCjA?}rAYax6N>qG2!_plG)@5QJ$Gb6wV3$q^B8}eE`;FB!hENu$r!(TKX zM0pp07tFuSrPwX+VoTxgVKlOHrwHoWT01yK&wK1H)z-R`Y7}CTkUZIqP*}jnRO;4quV2z@AaVS^C9k8AGwCR{vj6BRS^RS$(7<4zA#o)J^vyFvDXJBXZIrAetp~|z5;O}Ht zjEhEoV1AZ|-HLVhxp^!6XV`Z>F>kPz!I!k&!>CsUR<#W4Ph*Vnr1>ZOTJvtRl3AGD z4PTJmhJ2ryx54i)tC{Z^9f2LqW9Hk&-DW}dwo#Bh3++41qO1~pVO9aUJIo63UtnMR z*eqc!g)e5+!+h>F>%s5FEWU02#ZuubSgx7dEXiE>;@AzAm_;n?4AynlV(btKud@nR zxy_trDV7_)Bx`^&cAE|0_hE1O&@5R%3`|UhaT65vK^|LX-=wtq@y!i&+Zw9b8;6G-s;aOP` zSdrbqzQH@(N5GF*3Y&pm7PDr+|HkgXx=X`Yq+ul9#q-Fp-i5!+2H>5i0P zj+sgD-(ydG*Sw7V^fJ4|E*kfm2hEGdK{G!~!TVELU|Ck4tuzK33xNw+O?J}w+4$K6 zn#XZRO*MaJkHM$0>4s|-2Nq{7FtYEMm#{luVt=tW@%%amJjecIJ=iPk9pF3cE%pq1 znw4PmdbJit6werp&T!W8DW3R%P1Fs87ORoHr-&0x0xZ?P`0F{rbs zH3q)8wFqsw!WO|-vBu*{abR)ldTSyZ&lFy7mA0m^NvtfetX1CnfK6njfu*f#)-*PS zl?Rr$ZnkEyX-whG)-Bc~wv^Qb*0jo6KFY2FtYYn8H(|_{1DCUv7@eEhYT#;&&>*&$ zRksGg=eBNVx3P5eIUPM-!uqjX))IK%s)T$iSta;Sku!;H2X061+t{P*5bzLk+>Ps> z0zYLNSY=3s8`vgRja6Y=fLqu$a&O0H>^JV0n}PQiU&}06*im|wyMByVQ;dx*=^RF@D;7cSY!4bdkp>u z)(leN59}yrx-LrY4$rJ7kaiU7=_poH3#^x)fIqQcP}BRYlJzkBH>?%XegXc%PGZHh zisDJEn#OD%yW46E{~ZhCyJ!w@4y(tu;4Qu-uqA8FHsKpUVQbcgt;6@yT;N>Rh;73+ zP+MSI){eb~cck^e^=uvc9N!Y{fbCd&)*tWgj{_fPW!Yf7Q*8upWb4^y`1WWIY|lEf zkokjoyA^_e7jL0M&8@(#Y%|-5udC;P&#_KyBfiaU1>VY|!j4%QZLP~*&lz=Lc*+k z_&xA@c7%P4FO0#!!E7iyWF9f+n1|qJLu0b}Bbx;OJ>K}m=nf+jVZ;S?x zW@A|^ye;kr?q_#d)9}9iE1L#?f*mn_G{ynPu?cKC-pPLh{>FY~$MG#Y8913uW!>=h zdjxod-D8cwx8ejg0{%lb6YulC1Ak|y*-!X3n+lxDrn6ai=l>JXW2nJuZ4I`j!q2u|w0c=%fMcxMR&%SV^&s#;Ylv0L8fWze z_O|LFnW?P)gIX1de&+Q-yYcBY6sWe>ICd$J%`etw>kqmTOF*9)(gnj z5&mi9>;$b>tXHkRz`j-=xW3i^-~g*X+R-1f7kods0oEYkAZs96Fc7jgd zI@Tj-TOIgUtOiyiw6y{Jt5!pdV{c$@#9Cuqh60CT20CI~Mgm7#!w_nYk$)O~l=Zds zyLB9R+*)ZJvi`D;0*_jsTJx+^)_LH0>nrO|Yp1mdxXJnsDGRJcz(v+vYrge?wHUbA zT4*h?mO?Ip{{W$ntxtfTAhyz4Wi1CTx0YF}k!KNbk+lLj7UO!F6|vR;*I20tt+mzx z*CF<)wI1?Q__YXaur>lWBDTre47mw@1460L+zh`FejRW(>i+=vfwc$j1GH=p{Fl~V zw0RzI9{Tb%+OiC|%(`s-jdm^tF17H7LwhN58}<^*veT?@tOID(0kkFpJP169maGLH z0vZ%Hza4Hfa&L#>c-Zi*5|;_tsT}w)@bWT;E&b?t3J|p0C!lstklmitVz~b>j?0O^{v&yde<`T7VsZh!>tk4ufSiepRM<-x2-F{ zD;Bfgwq{xv5x#Wt&>T2x=?zh5d*?ra@z(3Hsy4K59IxyYZidrAA&H>M% z=0~lEtj3>jLZAP3$Dww;u*R zY}dD|*wyT2z-IQ7b{V^a-5#!hT^?8-@m9#06V9<)z_mdw33s?)W;<#QFSQW9F_PzEK zz$ff__6>Gvy8*C)-N>$PC)ueTcl8`}5Vx7fGYje(8r z$Lu@pyX}X757~9>8|`xTgTM#vN9}v;TkWR6rgm-n8T)zk_<8$TU~{{j{T%Q)`zgD% z-4XNC8onh`Y&$SzGS~?r`q4z2jPCOkHKxRH^Z$!YAVuH%4UQ***)!f zzx?TZ{T0NA!gsdcwRhXQ;P%>I z!!5QK!OcYKyGTva3D9o?uUfn{H2ndof!68}?|pakw@D zS5$gu#3#XzKo2L{p8`KcFBhZieQ@1TPFH(A;`<;M+vDxw_9wtk?2YL8G5bfj9(Fgl z^@#rnxdpz5y#wwG`yIHq5#IsX&7Nhy5Ba`57dY2`3%I~uf-;xb%Ye)54}i<<2-+U8 z*8$hrtAQKr&rs%P_IBWQdlPT~=3owRj{O?k9D6o!wmrmt-G0xW0i0nE#JKmg`vLpe zAK4$;eeFKLKK94%gL}kj4EHEfPasvLG)DN0-O_o@Ib*-(oVA-f9i5?0bNIo|v(BIPIr~}X zoIT7rZU14nbJ{u2JN@DMBHj-2v^~Ii&glTx!f6B7!+8m=k23(SLp0?jgkE-@a$3Xv zZl8pE5%Jc}N&98zqHQ{t?Thwh`@C&C!pY^>@Sek*OSS`aoE*;I_FuLObRF(ou&>y; zfw>*&7>?!S1m<+|I0c-d&UL`+oV-p6=LV-7u$*(FQ_LyolmwP^$~c9b{7z|LY3F(; zpHtW=4=nGLa*8;`owC5P&P`4w=T_%t;LXnMPDSS~=V9Q(PGzT_bGLIR+a6-V4Q_HF0JmAy>)^w7cdz`vXZD4Jus^dGiIJd#w>r{hF zLi{#JA7l3nM(7!*6|kMt2_xFcX%Fn;yn+#Z#pwp@;|#)h4|4hen>epJ1AzmbC!EKf z6wLYK@J*erP6MYAu#wZt>FG4YYUl~y9dp^y=?v`byx|OYUckH#haZMHZ|SrJwsnR$ zuRBj;)?bGo?EGvuKq)`lU7g?Tan5LGB-|P&9d5O=0&bbJ1a7gj5N@vX9^Bi`EV${; z6u61bIJl8f{S{Hog;AYZ(W~R4=}VB34nM^i?JoHXZ*a}s#cS?SDk4mw{0 zzjkIj+}AJB%h`IpX@KnK#{UpQX? zIV9&hoE<;`Dfl*L8&E<@zQx%BJO@07{yfPG@SFIP@TIuNbMnH#!n`O?=0orsW-^~< z4CS@>Q2d5ji_bL1@&|FfD6lA>&j%T|^XGxj^NB_-o`=8AbHNwmjd)f5D)3dloYzHK zF<>!1l|RDU^9V4)-{CI5kB=jUzsCg3Jsp6BKb_*=lYcuAg@KgP=e%khf53erjfOY*)v2kI#ZEXX^f zONd=+pNf1E$TSM##Kvb+h#_;y|iSc$&~SBc*VypzAgyYf5GtFG{!F^>1~ z2Y?Uo@o*3D`+@iKiE#I0{wBhY#kjZU9f2MBr*Iv42Ve)j0j>i^b_4ucjAH}d5ZI8v z2iK543Vf8m5BDf$?tS=oFsgm|AmAWgomb-nc?vLvC-EA50Oq^~d{y3@m*&fW%Xll^ zl7GyX0GIIAa7!?^t>If>jSb}^fg^bcZX_QL9L{TUpTB{1`tuY5P( z%fAMG&A;H57$aJm7JMsHh)&`q;7j;DGlzIU?BzM&ck|x({j#;_4S$}0$-m)~#Fy}U zcoES=3=u`(naCrO#5X(-{Fl6dcvW-}1>kQNJNaqe0oXx!qMUeC)B@HLxy1#3rx*ns zCCZ8c`2Dgfu&T%}ZWZ0d34Sa5Z~Pa2h(9cTfxktZ;eYU+z@FkdagzVckMWc6*NeY- zKk*0u8&}WpUA(P04Lr?v@(cVgeizEWOWY|g@=Lrju(G&YTtc6}M*M5Us)~C>K43o7 zd#|W2@&fZBRvmrG1I&Y1E#Zq?z+7mvFYXsPfjJSoALEk)m;W`CEAKzz+L=LjN`My6&>I`gq{-;C=q)Ob0>fTu`c2TaU1Y9jQOc!6HQ% znC~ILA>uVrO57;k0KOr<=cPp%F+!AvABIp_%x@X^5yBF-7zZ4OwPp)0CITlS=3os? z0!|X+fe*_H(h(Kp9zMh9YCbP!7|)AwybEt5T8J*Zg?Irw;SVBJG~%hED(=k9Vk5?K zqqv_p<0nKR*^C#G>3j@tFHQkZ@j|>4e@3(solrs-%y?Bq(UQ9r)F<3b_%8GIquPFcIMfg|zK~V&L9(K8f;*z+FUlJ{FSA8nhh?;zj2=Ns> zul!N0;6IASd=9TE8i+Z(f!NF^@s6UMn1qri;7(j4!q5$iWbE*>*?cj`m@jU}ot-LH zq8}?o4czgo#B!0umy2qA9DiPPM87*Ce|5e>d@QQ-k3|(;obTt!qB#6C?6OP5VsS5D zEbie`cr`K5oWcj<_sn&?vv^Ug<1dOZcI6MmLUA`*ei!bNSLECBdhxbAi5+mhn2Y-7 zib{MBuOM&1c;6zw#IEp`d{zF+UzIl?_lu$ru#b3~ujgGwXR#h7ufvWxSG}elza!cjRdK8y_ui#R?rSC(7HzM0vY-m-i5_0ACSr@>}^&DCt&sl*sQ8l|(mQ zN%Z6u#aKCBRz$AbL>~T@SPNV$e13}oR^S|dlNct4%bUb-c{A?pPI894PRzi#_du`r0r#QqrF^IuE|&7);v>F@-zP$1 z5lUFVSMuiK2jCCxzy1cFf?jn8Rmf0Wpk^7Gr^9#VB4z43w|QGU7E^7P~}$IZ&2H3ib$GodBF5 zM)1#hJ@JtE9HnjLr9@vjK$a2%5desN6X zmj&f9QBeLQ_KK%uE4f#+l6%BH@w9wO?nBC6F;$e7P2^P3M1Cf|5zok{@ZSqZ zzJV(aJd;9>L`=Xt(v#!l7~mKwWf3_OSBk(3nNv={m7MS%N}YsQZup$g8YABWz9&n{ zrJ_1&SSqHYj_IO|EDbD;v73dFDTlEsCufL1MQ=Gv{wZe3qUhzV@@C-8@@-K0$M5KxPWlfoel-1%r{C;_t ze2?9QvLd3kd{9QjgEC#LLyfgjXKln6pyWX)dl1GtpPVA6%6#wzoWq6~b@(o^L)5{Wd~JLc>=wJkLrAHE zckcCKE~~}ovY|#j{+0Mr)Z<@@y8JA^P2M5T@;l@?d|iAkz7q9uwI04wriiV$x3=Pa z>m06D!u3k>JibQui?7AQ(5%nX@wPAx?+er9<9JKFTUM4AcxCw)-p&t+1L9GBKs~0EMP9jrU*z}7s`4V+01Cj7KG2|OvDz?alIWl>RB zbd`lgXK2-t^?>yx$J^Lz@^0+AccY)J_<8Y%XvO~!&AAZuWdkWh11Uvo{+BohJSST6 z!lI~n3?)4#8sUwyfozED4W)}WzX#-tXzz=%2;PP+iNAnT*w!D>jm-UcrWj=&{ z=ET(}aQz9HOSIu!+OiF|Wjme=Db11A9Q$$yo!UyUc`?al ze&lL{uacIsHJ$^lWnR&d=aM;ON95|j^CG1U(%N7T_0ZZt# z(Xz*_74ixEg1HhceiFTDYW;+^tV4g-$rJJyv@;EuCV!KsfXjf(?z{fmQHJWm(sCZ+FYWS91?p z1>K6kitb<51^E?z!@MBRTlcy(+?(AZz#?vG7b)^@c>(^S{7YVvSAbXKWy##jkj%Xz zaVQGNf!wuGy5kCs1>p+-3%G^g z3b=)Vh287Wg6klQz!!xpj2uPbRT+h$wHv=%UY5JznY#zSU;ZWcz+aYo@%!Zk;03g~ ztXmT;DhuzrzIz{9>BHLyJ?JtZLo63salpz2|BaOwEjb80h~G0+yAA;l;rGm&-STM5 z&CxcMcW*)a%EMoWP(^6n0$<22=9Wa;ioxeX=mxa4Bz#VUN}~@qz!ya;Jhu|C5@sNW zdtTlHyvMx@p$qa};Jt_~k)_=0f!Di>cgD_;2Mxd81p(U5KkoOomK$f0YMtwKA?(cK6EfWd*mK`#nlKDo@IgTgE*p%eX(wpMg~&tGegp zSy{!s%{_~j?2+HfTipt1V+G`yBa_@4-8pFOLb+MyaraxB<$nC0c?7Mhj&@ad-KIA@v*hBF3T!G&&8vq-)b=;|L6So=MlPI+r{8W_K z(tQf}l$#H~Ha>;ae0DQvJ?ge~p9MbaHiCQB?Fj7X7RRrU9ohir8(T z*T{X*eF^vy@-_kX0QQKM*~9G#*)y8AC$x&&FGBAXcc42OINI&(Hg$)(gMowH$K9^( zDEBqsYi@V9x!cF>1?=Uva$CE--66ms?qhBncepzYILv*WGPot~BDj@qI^1epTaGI#eG%fz;FsgiKCZXV-RFMceh2r1o5#!J<@E6Q zLaIv1gHV2NzB>o*H}?$Ob%@V_%_BOkh+!O9*_%&{E^!p0>|Cc)tZUN#~Ad7pB zw;-Cb*Zm4kB0dk&@pc3E!+qr*g!{_<(mjkCj=}xl{tWkn`@Q=MTK*^88TUNg8TXWX z5jDUt17+jKci`Wcg;nkW-~r6ZDmTsj2KWtPsqPAQ7jT#RHOBp@`!VoijQ@}BPwq#+ zkKCp1arcP35V+9&5Hs?rdk%OG^Rv#~=>7rx!#(S6K;KRPPq@EhEDUc4a0kZ3@GS3h z;OB@j?-E9Vd7IsBDC-F7TnN7cy>vbG&-HS^xgN)C)}dzH^7zjN_aiI8Qwdw7y(->4zs9xvd3C%7zy|m)rfPbR zdi8jv!RHTRzIT6o>zx_U3bJ&$;I$S1rPy{Ejkz_wl| z?{V)ruRXB6*T{Rud)Dg+?C5p&T6%4~F2F9{OI}m2vDX2vo%b|cYs5Q1HpO@i^+v!A z^?_+*by!pWSnCAhQ?{|Rjc=Noe-rHVp zU~lgg?=7r{9>5-6FU;iVzs%e0ZSuCjZS_8b+va@^x82(Tx6|7N_l36`ZjbjR++Oc1 zxP9K&aQ~;h?+&w~c-HQo=>c3aNRlX^f*?vxyJuEFMZ}B=P%(?5pg#~)&=rh`sHiBQ zh=7O@1Vsh*%&M3ZCJZP+1;qqLR7AeFdbM(tJ zXFkQ1ui`qo>Mbk(u9f`I%6wwKeZ%tK2j?B6S93nkS)KD4jvsS=!0~;~cR0Sy*@WYp zoUd_g%-MkBtDG-!e37#r$GV)gIM(3G&v6}H^_`Xf+Dd+DW!Bnnf3W-w;A}$rJm;sJ z%{g0e1eG-oRe49wj+}pTw&VCaXB&>ca{k2eN6uCpzvujhyGQZewb!F1?Cpg=Ye#+VUOXs5YC-;jB_PRKhc<*yM7aexoaq;=L4UZ*0a#g3I zLH!3>d5IUF+^J}vW6p>-Y#16#eEGJHMd5Yl#v?|a6-&J4wT?yI`VEci+%hOG3$JZ) ze1=b$uN|LzLXCTAFFP0AvBz0fUg96F>s)k6>U>*| z5+669bJ4u;LR*hzVe$j>3G=n&H;E~)Dae=VQDU}@>P2EMBh{V6>=Tlgn9EN3B~G2v zt!V4Vv8_jmKbhFA==s`twjRsERCmlL%-2#qO3XG;`(it(O|cDBzg#Yo@%fy6NO>_X z?N6WIIA8J=+e7|g8^|YYGt~>1i}W)tT}EyrusM;K+ZW21NX%`JPupd4Z2oh5vz&BB5o6jxs8*Z5_1ew zS<1rX2j&yzYsqgCibtf_Vgk&Vzw!6<%EhO~;OBv#BEFQ7e8{IDpCWmQ7YsZ;RrS-{ z_>sviV~Jmzdrj)shtG>gHCm98_{GgPr8eAnnSIxr;HM>D=t-oE$TyKP!p=m>NWLgn zA~EFI4|Ht?Lw_3ON>DD!D}!NA8s$n*uC&CkKaFxFc(jNT{FLMiJ+YJ#`H~MguM%}3 zWkk8WGNtrn>KA&->k)M-{Qv)~9;yB*jwuHICt+WE5ph_eO-a6pw-Rj=@yY#0iMEV- z;eLbaC4*)EAo$e2GX06rN$rcTO$i&QeR00jcFJrQk`J8eGbNVoQDWKdB!)fQZ%}`j z!Lol4eCiJ~{fW?1B5p;#CHg2~XNf*dl&eI2{fi#vNJoh0b>Q4BU>}3C!*P|O3 zeD(N0sjrY_p?*-6zt^_>y|(3XP~Hy8`+@(4{XmGP87icfc$#8UJj<{#Qg1x*uqK|T zSPiKYo`k6J)WhaTt?)F(z40`_CP;no^uyYCf?^G%E_h0#!IKUHqmc>SlN&4H35gYv4!~0r>*Gm+^^o%LOvxNP z&2bN;1M#%Sz3^Pd21qfU(pXh>1nr1tK{mv5CVPSQ!m|kJ`IDVNJL9>L{ZwB(mGN*q zYq39$Bk`Qbqwv(lV{jacCq^EJCm;^MaRQ!-cp{$scruPta7{m4LGnjg$>XfdiT2xm zmVXR5{gLRYkXPYpHv^GQ#nTh-!xM}~BVCTCHBQEpjm}595Kl%Nk0)o0MH+;sD$c}{ z5yv21i6>LuhNtgbfOHX_)>wci-wZ}N4Nq8{g{NFxfiw=!Qk;UPAr3>j7*BV+4o?R; z9cc)j1bLU53OW_ffjk4x1Gy3OM*D2YyYXzoOOP(b^D%G2(;Ch~IvY=doTjFOPRDZ~ z&rvtynT6*dorfnu&cOdJjY7H%&&r&Hry`w;G*rz|v(;Q2i}8Fcdd}5C91GMFI38Dz z;doR%g5zO4-*G;knm7-~gSci6t|0k^R`M|`Gv9tY$MT;5XA#moHTTEn`L)&#p>|L( zpm)ps_NV1q%>1173sWcV*&@H*hlAs_InSk@__KL_w}q$2FU?z$n%%f%e%tf&EuXI? z`F`D6=I?P!o|O@L(n3#K=t&DbX`v@A^rWjDH!gl?MQwW@_um;GxxOO#FFreO3)vd? z$LDG%tV?~-yh6Nj&6?Dbrz^(8!nLXHOMgkxXOg4uZ@qI%iunIk`3mR@=#SdF!LwiS zuc>Ic^aNOCIn>7SxJI5VT;`fzTaIT;xr{{H1> z;$yZgw0a~yVaKBQM`xSF+nYZVH#&W>m6y2LSx?7*Hz_WQ#G5zHi}j1ee2K{?V_d!x zx12CHZqT$?-iHr+adv#Lv%!bSX7-!JRK_>{n4T#wW3su$hsAXyF_lr>bx)?egeN{Z zJyWNHx80kpzk06C>p!bzBx^<#*Y&!`?oIAJdXD9j4Ao=n%NN^gxvoiuaIKk7#S)Vq z;$J=T=~!aYbLoQ*#TBl+C-z~ojjpA#OHBIn&YvGQKJjjwS6P_yTJ`(W(q#*=yX>al0oL+Ir!( zMm94ho#E^yaq_`JTStsZ=ZG=S#M`zkwDrQ6bgt|8RQy8C;<(~=M|uk0cq%@5Z1eSG}^x+QXRi-2d50>R!;DFf`fX7rZW)uF z7f`Nv#k5ShxczdwCY?i2##wErWy&(E@SYNw^sI-UPy3?S9_}B=HpZlf{9Ls9ZhI~H z_L_zzG3n9hD?Zv$90T0%(Y0;SS3I=6xPFV!hgAE!B&KUCb$Tdn=i-q2Qo8oast?8W zT^uqdpY;A|UVOcaL&l_M{K~npKB%~^x&I_RU%fUbegJh%*UsGo*wrs%y7u%%55}Ll zc;o(={J@y>ys+1U@f9xi7?aMI9-SL6b+N}-jMs%?ye<^ub)guq3&nU{D8}nTF&L8&54*m?%1G?`CCiuC zwPnkf*!2~bFR_a&%a_>2s^v@U>fQ2v*u|KY@nP4-t&GG}M)V<;FR^QPmM^hu*Ot$i z;*)HqdZ8G0<8?wY>Bj4X;?|AVR)*>q<8?yUa$S=QVT{*So-yemjPcs)WK4Ov@!IMv z3$uQ*O=8N+jnOu*vM}ZA#%P=G?l9@+*pry-bmO(PlQHRcBj3!SukFgz@*2G*Y;ZOAE@palO8u-+iTr;T>_IHH(pyk-0#t~ zZoIbgZoDpm=~_2l+iSTmrEA?7ZGFO+eB#DvtA{b^apSes!~G}eapSd>cVl!3OxL>c z+Fr~3Gx>or>2YJUwTCg;G>kNzE-u_a0 zH!=xieER=Y&o%Fii-*sNcDa^#^j+zEFTx&}=dj*Vnom6Xt~5VqR+PT43_Z}Dl$GzL zYuM zpmGqVGEzARQ(4Fdgvk$5X8ze%C!4D`%4`4h1%*DmW^9%8_K%0gU$j_}e(LHfR!@gJ z$B`d7uVkO7b24+wzVTh#ZcdL}eR*=}_TTeo6@7}9hdZKyRd|Or?%~r zh8OIgwA(N_HDK~($%mnS}#S0%S$q13r;@0|34?}x^w{R&%d zwoR7z@gFmE?M>6UAZG#R7;(Ee3V{rjY$ojcV#^1jn2vA z-InLwyQo7v`hwBPK3(?6TXo9#)RsH1OP^gqr~WjfO308N(m}dN59uIX5|d8SM|w#o z=_9?Qlk|~ZDRWi#aY?gdr{*0O2dR(SR!lE_VSB3Xi-Y3*_wJU=S^Z>cMuolN_8*Q; zZaNC@LhpQ9s`_U)riU*0J$3i&6H3UC9@0U&NDt{CU8IL}kS@|gI!G7kAswVk%7`*9 z7v)L|n+xj<8=c&BSD`3ZTH?A(2PQY2 zyD@c_xh^frl@{en3qRXyou3oo^Mv$3M+&;E9_Qyo)J5|2WVFLaDyI(^@<48%S1P72 zzVFK9lA2-axp{+=aHdM?)t!|0)A}3Iv!;(sR&Ebdz3=FnJX))L^7Z%ydCn)xMI71~ zay}^(aabtg(8i6FaXv|lx=w6)iv5tLGB`gZqJC{WIX@IqtgLHYF}+{m_VmgXgJOwY zdD9~H(jxY3+(;QQwmv7Z99tz8W9zdL%WEZ;*Gep}m6-YrH(sL;@r}!6@?*Nzjo0YY z{#Ay?YoGpq)#Dq#+<1*XzGS{-jbCJ^Z@hEk^)7m73@aYHO6sR^g2yfzV~WR_k}^B_ znelp;vOD`x_LA4S^6uhK-+ZI2`H2{>(_*|{F2?J$7_SSdtmGpqZ*iIJcwJ%~UMC661ABj9K0}gS?i?RQy*Ei}4zD!G8s@7_a5{7>n_` zguEE9(_*|Xajh7y^EfA}XE9!*e)+GUM0v${UE*3m+f4uHDd#=*{Z+pJt zYaJ8+C&>LLIVRqM>*%}|a+l(K60W0j0py6k{h+^7bf)(eQ%}I>=5=btLvH^mkIyZ? zu4%t7eP;R6=hAZg`LRnH#FhSeGmp=g*QyuSKt6n~aB72i#9?dm_Em;2#{RZIA<^_P|H=eKIsil42vF5}11wQI$dR(_Mm zXZA0jpBY#k*A;X;_>>xPLE|d96ZWea7j>_a%l_l@{(GW6x>e~}`rP%?;(GF(*ksIsB=l}6W=6osm3;%vIkIx$y z*Na>I^m!(p>IL=VqFXoS@%gXA>c?uq<~%+>+#32;ugb(@y+V9{)apDwzkD*{C4X%m zpI@tAKQ1_TdM180%t!p5F*TRZ0}x;8*eSVu?(}foxM*O-T#mneo%9MbuKQLw9ihulLdgFn7j=6u$0$nR*BYvZ@Z&Nx15NxisW z&xJjg|BCPb7}0~z18P-`@44`;OnVyqLFM?Kj&J47eKaTTcI?jcyMI-W`;K0hX@`fe zsvggeH|6npIofHB$v@@s`J2Zmo}JOz8UQ#d+V&J>^l*lS%I$KluqeI#~4AA#dIyYu-JyTiG*N~Rr@Il3T9)4qpASN6q+;Ya7{^fTOCFBB$fIyvgT2W& zV6StGW3c6q!sp>gF)Yi&{_9Ri{js0BCU$%0Bb|WqG{kP|YDnF%6a505n<6#DZt)t} z-Q5OhEOwA~#4hbBNPEJLs@OqZ3#mT#KpzN+8e0q|v@mtik;1MH<90XhP5 zonZH|pvPk0dl&2!KMM3H>^koZ+7GlJY&;%i>;u{dd&N(}e)_(keX$RGAa>X1gXW{W z*T7F>K*wPB_qEtReg)_i*yVj4_K{x(dKqlE0pDK>dM$QyPXxUl^m^>qz8T+70G)tx zU5W1}f=e5{63@yATb?#!)GDUj_?`q*=(fyA#n@#iQj`X9S6O|cRJF&khl&0o`EzC5ilIP z(9c9V7rV_bu{+O4Vo&^e*oQs>bOd}k9MLiibQq*A!Cv``K`(~nf3PS10?-Q}H58F^ zI_T+;x)A&0&j39GQWs$#{HdU)LTVK9IT!R?cyTmxJ0J9Xc=2-Na~9}X@Zw;^XAjUG z&~zI9TOk)T7n+9Pe-FBWc7vv_@XcMIcR{K3FA-;y^s#We?riI0vw7&|EJIbl2wp8Vn6*{ z?4W-X=@IO(pNBp5k0U*XJ@@mm%YFgU6L_oT%h-><7;k5O6344}Gw0jj5$8#umsD5% z8R%!&VV{e46o0HfR$t)FmaFlm-Gg!DTFz>`JNg;CU;7Un+tptB> z#~+rz8K3_`dItF|Rz*1IJ>f5?w0aN6D)j-5mH2K6z9Ig5`1}IW5^VckjW<|+shZ+l z%5Ct@=mt0{=-N1{YIJn^OGW$!_}m6b!LBW8E3Dm$cT|3*{#46Am*Jh>i%=G=7a=8h zgK2{Ei%8Yq@)z2fhYM|Bd-txWlzWPJFYy3mJ zzqC+Q#OFeMBK*AC6L?Qu3HE=AH)(&2v<2uB^)u+t>Q|tTVAm%|Kcj4a<4v1ek=El) z)jROU>}^P2;|X+oktKzO9iO!(;8hZG*HA z&_R%DhtvvhVLkw5I}oWg-W1*(+@?rv^xnE9r1nN?hc|yW(``Z9Laqzm%X|pvA&87_ zx+mUN-3_Ua?t$|60_~*_#W#oO6le;u(+A(?gXZJYq3~B8Xdd3Lo{P^NLFpaZDSX=* zv@<@%__hOR2i*yEpM$r!S3s(*_rSZY>x0(Ub?|tsO7L)HquS3sjq8+ z*3#AS&7P?7dPp_#O${A_hKQN!_!e&mM+)#wHLXFlHt<{{`?le| zil7ydpMr)ph*Em*YXA-FK-b}2uLc@E1Eu%4u2ySQcf@gb_~>)Iv$ZE^PxxppVzMh} zSNNzs%K52kkMyzXh*Ez7`U&d36H5IN=trpgFHz&o5%JAY9~;$oswHSk)W^4o-e#c9 zP#+5sCmldL=vUNbCpXXkYUt$nzB-4+FFw{HzxCSD77yLrBLByii}?~S>Ag>W?zAyh zMq<{p?V!EmCuTR#?{wzGIIn7h`1~`Q=Rdagmbg*V#wGBSan19qoI5E#zvW)>&1bgA zpZMd|R-SDjJrCW`BLCBkS6Uf~-<{nef5tCktj-6zx6Dr^#d%50Hj^HSuiLX_e%%*~ z^ObnepDps25*8} zMIy1Ni^PYyyrffNQ5T8CqAn7NMO`F5{N(&^3YVXBm0i;U&I3+py|n4s zu1!CAP%FLv=e%kLr;qINIrBHLT5}mCI8ppyuwWr{!Z2Jd0Gj1 zpUo&^9?FLn%WB^LD(i+V{3`$-SiODyUomSqw3Qi3n)B_-uWy_Asm*(~a%gdR~ZCG?1TDIxDG zm$dDG{QB{YwVqDBJfVs`^JhLYpU=b}c<;*8riFFlh1ai2wZrG@Z+|~?p7cju%g?D@ zJ$?e;({(39{`@zp+RuC@{}?Be`5|IGP; z;{0;lne3wmI8Dq}68Ce|Hl;(ogV`q2+*cm6eS z*S#lJ6KEwOz;}_o|8HdP#c5q2vj6$n0JH(tFKR+(4rmVU(CdI!0F^|V|t;9TjlOBnAJSLq!%wsv_B{9!4 zNTAeAzbABeAH9%$$krk$h1XnfVsw%YGo85{tUX%pJ)F zQ5Rk;>LN3bBzv+M`?Haeflb&ZGe>3J^^qJ;)lOfM%*28Dk&u7x7z9`l!&ouv{eEH1z@R{a;lwanI zc^=8ryicBKJ{UkhpG%jUKlAy2n?LARB!iih!R(7>YXOc1m^(GbT&f|CnwYKC#axSi z{|PL=AwD-os*PD*70m7wQivH?Bm5dt8L29cy)mz=ic}3Vu%`IFI#Lz<3R4lkQq@4( zv&4L^9%gkFFt4is8)zO`1GEP8wZiOcU(kIq=W2=B82v&*&jIR&c~>5eUYI|{m_^aA zTb(UG51+dswX{zg$_35ET#M#my+M0pCU+p_VELf=_(b!tT+m$1&^lnA*9Nprsrg)Y z*w6~|yH=Rp9f0dQfOde)_ZPhs&rhEn@42I2?8VC##|vAWUM%CmgI}5x_gQ=;&mGNP+q3(>HsanwE z!&H|1Yah!a86W;0J}H>+a4yMsG5o9+-?!Atc<{GPKg#Iw;jwkU&e-F_)1Ut(Q!XD~ z`Q^5Ze|)&#lmBG=?89^|*Ow2I4P5U&Ol9Ht@nP}_$Dt38-?%j1r((x=`J#JbAEr3p zI=G9K@!_A(eKJ1yl-5?phvUy5jz2wQFDv82cqe-N?4cE{j1QkRXioel;&~Zt_F=lV z)p1L$j1Q+^&sWFaX=Qwv%F=G&Fe~H36X3&Hvk$Q{K1_Z)`G{^-#)nV%?VXGsAKp0f zlen$3$A|CvXI-XTK0NWm4e`FtKR!IN;OmT^eVDF&&efL>lMP(&K1^ld`0-)#3CF4r za}1x*e@Z5v33J?*#vH4}mzd*|_!3hrQF)0kF~uSEvBZ~{V~^!2CJ9r#Z9wdm#$>~R zs9)ks%yC70i8+>tFEPgt@g*kT9^mv#O!dO{NX)T8dL*WJ;POgLGE1F*B<6Z2JrZ+W z6JKJkU$&p>oiL3T)TbS>?e?tj7au+y5A1ZD?N5Ao+$m4R@0~IzZn|u0oDC*FZ)|yz z)%kCD9_9(wPRM72*S#>r+CVyYhk4FG`m@0^S3DG7v8K1R*@tWVG&hdt9bs+A2Gbm% zBIYM#LpFH!^n2s+6A!R9_;9l1o_J>aj@E{3Fwc3&hHUVn;djM$Y_;KRRvc3b@7 zkUgyp*^pX zDY79O%yUAH2bwGX8>Sd&eb*psgAY>-r2pt+ZO8`Owt)F(KW-Ph!PFM|9}-v_e3;t8 zi2g6(D+puKV*aL+}OoHHn>N%=e7shiovl1)e|nFtvqzH-E?m^F1lqkPW6jf#-5QOnm~+<+8zZkKCR- z;?{F~_`Rzu7ALr%IZajy7e3%et2b_qCMPtjt{R`-=JutThH;~@qHT>t#s=- zK1|ns;MQ||m~3d|)^mKA%F^Gh=lC%BWT9Kn@nQ06V)m_ z;a~rJJK=KqaPra13H!%~`{b-h*v~#p*K&RNFxkNM?!#0TjvpT;pKu)d@b%w@Mfp4EW-|#9$tKE8z56}LzS`pn3()yAQ*I8J-=pDD7y7dzuCcn*d>p4D5eml*r=lF2Xmw!%Jj}Jd`=%%EDv&V;j zU%o!!a`~{?_*ug1T|V5f=jw$0?89{JGFM+dOg4Px>fMK_EF3>R+#UYmSoL9!VY>I1 znB$iC5>uV_M%)r#VvbMZOUyAze2F;@i7zq79?Mfq66Sa-jmd^iZoNffjw_Ovm}80f z5_9|zUt;nt#WV3G<~Sj~#8juPoy`(+JdnJ^90SCcnCqSOQ#=qR8`%F6bN!Nx#5|tT z93UIK>e32DgWP(K4_9hmvFLrbo|6qGKY!!abN&rK^e~|IPJ_UUKU>*l0b2jz^%~k#l zQ`sMJ_iH{(Wv}bjbF#raC!~C{!4v}ry7e3%rWkm^-M40gsVyAi?%%V))E16$>p4D5 zZDFjtf6oR}zf|PbbF#tIFWu|bb9|WkrIl_yCmT$Ch+0^@o|7#e;ns6V-iK9g@p_Kc zxjT&cC;NfspS#1;pZ+0XKlpIx?%yZuhiov-jX55&!6(dJn{Yh%@GC1nOGdhQ$OiL0 z1+|N8@F`b(kZ`;3;n7o9CPl7YWP|yhg>1+MUwHIe3HJ{^{KWEC6Yd|f!E{f={bV+{ z#YeAa=AS;iU!|9=4E2-QV7}*KKhV9{zhNr-HaA}QFqNIhi)=8(0FRs5V2S}AH+`65 zfXB^jFyAv$*|Wja7I^;P!_*dfx%HfEFyE7s4cTDo6L>D?!_+76TrL|->(Xw2PU6Ko z_v9q}dm{Gpl)(I(8u5LYe(!MmG!h@C-(@x+CP|ME({Dy^W4=)XGCoY#zK1yg$@nnY zP#^0#B;&*U`v}SSF#l#jGG0u-DLi@KQY+)Zyyj0bKFn+KB;&)hM#|;#VOq;&|M)Pi zIkTUAn6Bme@?o-p>)nTWO_l21hk4DDWPF(SGhqEB@nMQ{w?D_md|8Nc(e0#)o+y z1E9iuvGtI(M_K*ZV6i7B@nPC~Kjv2J@aAvLPExdva)R8}(y8OnY)jhHS_N^PCO)F*5VF zf5TMv`B=}P{@;grPfio(2imLiZp6D*xjRg4 zp&!p8ZM+8yRSIb?$mQ@>P%^&GMx8%%wOThFomNj8}F z+g<8%%R!j)812?LFXl@L}3}FwDh6 zHkj`zs9j`(c~1`c!H0QI4%J0AnD1H0hHNnHMWOwBWP=a$o*c3v8_auhsGrOR)1Dme zCw-XqT2-jhRpbvBsq>Bxp`u-KDh%U4%v_m zrnbQIhitIelau%`@5vz>vcc3R@cb%U4%v_mrgdqzKPT~F{@sx7 zqkWivQzIE4rr$W+K8?hO`8O%j z z4)?1*OnY)jhHS_N^PG+D*RsL9Cx`lfALczdbibAj<~bqRkPW67XovkdWP=Y=46MLD z1hOF;Ol`sK$FS{UcbNC&kPSY}dveHzY%ujpE8TidHkkM1kPSY}dveHzY%uj9w7;F| zB3n#*a>xdf_u*#PlS4L;%x*E}=j;cXf9?+Ro*c5lhj~v9*^mvUxiQB;HkkM1kPSXe zdk@CA^_*-l-&0UsWP@oB2Db|zrac(6Z;kwr4d#0mvLPGHdveGRKFoV^$cAh%-xEBjroUke{uLl=sPy zZ#Z8mBl)l=^Nvj^Bl*w~lTACnCuIGc;lr+&Y$`3|_$lK1WSo8}WARR11@z8QveU+c zl$SEDTvkTPOBvXhiB%~tWT+e&KTBWvWIT02zRCDs`cvwWe2<@F$@lm$&ce^E=imHn zeY_h#bGiP_&+J1hvl~BK8RtWbrGF$JddliIYz&f*=sMVLV`ZlfLIqT=3?H%y*;HCa z^5M_2`VI1PhVO9NqsQ`nGR~fau66nDqQ{lj%1HU0`4y0#slT-{QXc#=V_5P_U0Y_X zB46t}XQ%abX^i}>9mIFo`d0F(e%R+y#_6|pCuK;MK9f!<xw4w3f zpI%JKGIm?GD*gHLZrTmKj3gTx^8dW&&hUZgnjvp3Z@Ok_6l&Mdo&YUo; z1YhcxGEzTlr1k}!d8||Fhn|$MGt>7<{nCD^M_yY>XQu4Z9$DT~^Y%-S;^#KMn3DA? zZIq$b3-`Y2GDKXQT}-{;eNmzCOOpOX_!RJQ01AwL!|uK26%-+Iw2`QPv*Gm-?k`Qorj%*oMrQ zD6f@$Y$+S4PqQ}2e5rjfU+R}KQooczUKtx;XNu~~)lp{r_1Qpu1I2{vtIM=O%1HeZ z!{?d2P+u|CnTyrTSTAjmIuQ>kiV4?GX6%uCnU~ZrF=8*17km)o8g8?hIfk@B>W2?f zRNv$WA7ADr^-GL4n8^$6H%33n{X=FxBW;lRqW)7<->#p`*dzHeFR9;Q+b2uDoEJH~ zOCRO*+x}VR<>UMEa<=ien`1~lKjBq1^HnYzRXMNcXh;LG{r+q`AB(J zN4tz!uHNl*LG3q<$BxJYS-AVQrA|E>mPV7mzg6; zdDmulvBC94JeQ;PXzh`FsbAVA^^0=Zxh>T_@#NgBluVjsCQfUA~^P^ZqW=ZoB%&cYW+G{k-Gbw&0U-`lXD;qE6YKSlF|RjO<%nxvad@`BNDXMYBh&nQ3Qmw~ywByX{kvDoRizIFO7 z-`T@HCwnY*_FF7v-2DR20WxDN#_00(9MlWuOXcf1@B`-0?v|eZLL!Fg+@V)=Sb z@tSk_dX88-a`$334&D8l^^aUHk}~do&8`V@W=+Y>KjWSK0in$A$d>QsrMt*TzMNO@ z)b+pbzD$C8Of*mr&uX1Bl!-eOUpPtmhY2s_9S$zl;5cTv%Jc~m!%m7HTP}%_4~+p_UcxELSMr^_+T{gy9f#ClFzmQk$dq-7Z?zv91wgzi&mzJ=cw6S_}zzv-0VOZ`$t>SvAA2F33| zrSe3_Tj zFR{$a-M`aspcHqRSdunK{ip|8H=ucP;^WJ_q<)EIUSd6m+C(O9r42G)_?Uc6zqckn zzRXMNF9Xy1Mu~i7TaY%$d>y8>l*GsP<>hRnH5T@{ZFf#y+GF|B23KCXU*>$R4L-ii zOX_!ZMEA%Xd$#>bdD%Zm8(h89{WAOC+T-KPyrg~?t8_2V{eZPW%FBL6+Thy7F8TWS zGB2s$wNu&~!+n&sLCVWMP1@l4C|a-J_Gs;qe5qgBCiS~Mgl(Yy$J!vTm3?d}8+Z>4 z_i5H1*GE|__4{m-GOiC~Kk)t>s-w*K>$8FS28u9?6$^N&OPbyb!m%KgW*G6cao1CsaWFVJ2VrfcL|A`7$r5Ut+Yu z%zOszm-pvTf0dcf?6jkR+G8eP^h>lah5Lsx`7$r5-{BPXjgs&3O;Mgc8(hA&f0j1* z_`bZHZ9F!SA8fu-UfNTJA5yNLBtKYtoPT!V%eGC8Rm+z)xcIl_mHGPkGB2s$^$&KQBj-p`-nH3XY;gSm z&*gZ34)+h1FZD~?q<&E@JAbFTC!U;}m6EaTgm~0msh)OzPpH7QE#|wiW)~UBhiqAM zB-+oA;lr-7<}m-?{NKb*AK5jr%y{G(gSajdy3WOb#Zr&t%YIeLNIumG)!EMP2^IWL z-8W+6h2n-vDSbtsi~lnJV`ybObzx(WVv(+q{&Y%k`d_^div1juKRo>g$?UG*AU|jN z6WKqM@^j{%UG@)BM)E!V2CbV}yXiWIOScPJH?ubVn~by5#)g!a_PBC!{E+dIhnRG^@)EY=ujD&<$+y_)w=p1PoXwUmWt{z% zFJ&a3Y@xi#9*LcQ_}VgK26>ekC$P=7XIIA3F~H>_ewt*Qyu~t3oc^S=jI+)1rHr%R z@}&&MX#SrgzE`s4!Whl_bND{a@+rUKzw-4Q_5Z&!)+6L%lC%bJqAU$e35?$@lJC3fSctqXTw$+23#o>RR3oSDzam@HqFNx18|ZE7R&}g8N^Jz)sD4sI)wwFAhaw%W2Y~mJ z8i4e-8isEU*Tawo=##+vTb+bdQTI~0YOd;qv{Vg;)Brsk>2y61-&X{!sB7t1jZ&kC z>POY3kUCvoigbZKRSi_NKx^s7y01D!y`=ggt%UTY>HsI;*e7{m1 zfwWqkrG}_h`Yfc5`Y7;LqYSH2fT{7&I#5PEpaP^PRirAaXF#7( zb5tdjqh0`gK|QA`tE%cP(6`hIRZZ1UpMZX%-cdExo@xW=2KA|`t?H>Qpj*_Js;=5g zZ3Eq=epC%qV^u*nK>AZPQq5FN-3ZC(rmDHx3v@4CUAIvCsC|g)MtWb>N*x4xkls(X zR&7*I(4M-3ZmSMdeL?%`Jl#%pP{)HFuMgAhRcCb?=xO>`-AQ#*LqUh?!MdyJsYZg1 z)aU6QN~;B`2hydwpE^=aQ2mguQJ1NM)s>)Es+sCcb+(!UIz`>C`m1Bqb)eU&@#-9P zp1KqCPIZSmS{<)$0KGw7tIk&!sHvb+)m`d1b)vcn^d@z^%2#=64Cokjg}P8(tfqlZ zQ+KNq)G6v_(3{nbDy4d>v7lqsZ1--SAkxo#;Fl% zq`D9EJ~dqpR;Q`Spp(@t>M(VP65O)tl-w^`80w^aHg7duhy%NK|fYssbAFBY7^)t z^_$wNzEfX-exWw2f7H+FSI}RT(t-Y6{h$K*@NsvJOE*T|+m~wRIiPI=Y$OTi4aqK&$Bny0vbqn}asjZS(=Ug>DGiP`A{b z^!~anXj|Pychl|kzM%W+19hxB>#m?(^})Kg?ylQ|w%0xM5jtP@0_~-b)JN++Iu|ro zAF5B({d9lO{`zD+P#>ia0X;+?r_azQ=#xNC(r4*&^eOrn&|~zedYB%f&jdYFU#v&y zv-Kd*LHc|>T3@Iy0=-CIp|8}#^|_$u>i_7+)Ma`M=or07EmoK7OY~xOiGBh-OROFR zeH6XRczv}Vi{oj%2*(2b7>r-L&OX}n&oKi5@))ku|txAg1!Owcofi-HaM3%wL{Y4B38Qop4y z0=+006MUmL=$Amj4c^x)^%&4G!Gs{KpViGkn+08h@ANl%CFsgvb+AgmuP1;`2qp*1 z^iTRHqQQQ_kNP{k8gzBAG5AEU(vv|a2h)NT`gwgY=)pn%;3xf~-Ux~me5OCq(?F*M zbAwm(3f&*He{fQ;R)41Fg3b*V1h4B?^huy61!o4&>$L6y+9fzR_(HGM3qThHOM|O) zfo=@iI5;4&IzAW`+@tT+w}9Rf6a+K$Og$BJYH)LK zzkWbJKs2}~n62mP$3Pzo<^&Jwhx9X`&je2d^Yx>81?Y<4x!@7~gnkF~o#4gbas8zJ z6!g>Jtze;kN^bz&5PTdg*3alIpj(13gC%;LZUfpT_%V1{zovKSmyx#XpLL<$3c6MA z6|B+gbT!awLDk?@{ifESI`~Jg*I()y!Fr_X!5jJ=9fF2|3ck`CbuG|ZLCxT8{hp3M zqaX;D>*w^ZdO6bX`fL5It`mHXR6BTAf2b>hRt!wAR44imy%gy;y-EL|>jj&T_6$DI zAL&Y&y291KQLGxfA(0zjLLC;{{pb2P`phM6r=oa(7cW?;kA;HnXvB6pW0gOh`!f@46B2?hp(gX4mJp#6g5gR_HE zfJef4hn-Lcsh6-^zmRx@Jg^eNQ0(>*Mc{KqTpH3XM-1l_kx#$S3zG5J_uF? zuLsYAJ|Datd>*_Tybt<*ur^p9d>Fh5`eyJ+@J;Ypum*HZ@NMvYurBxr^rPUb;FsX* zU=!%3;P+r_@Lli)=oi7}pjGg5@GI!ALA&6<;J4rh&>w>RgJ<VKQV@pW^6(|lm%=El5WW(=3;J$YDcmD`FZ>+z^RP--Ib0on z1Nu!^J**aP41WRrC9D=iZ)_cV=7 z!?3Yw95xA?npUPs*vd2u_YU_r9nIchN7Eu~8Fn`Lre&CK_6zq7V{?SrH$1}ZAGQt; zHz%6b;fdyeux&WNoMGCAXPAS+1H;qJFmqry%ybMpgcq2HKpzVK40?r^hgXKZkPZni z4yT6y0sT+dJscR`6ix)47!D5mgja;)K*xoB!&AeX!%3i%!qdaU!m;7ipjU@SghRqx z!pWeM!!yHv;Z@-^px1;)h26qS!x5k(!p`AY;jQ8AptpzThDU}4;RMhL;jv+t@RD#O z=*X~Vcusg*cn9bm;n47yaC~?z=(XYTVQx4oybScRFcqF3P6_V)6fOZ>5XP?AzTl-K3p4a4!;gR z1^qPqGWg*A{uQ`gitRY9wo%BF#-X)1tLFxAZ7riIxHbT3oS>|+|4+Mu;fL(|r@Gc7?| zn&##pv!7`K+QjT@x|;6hK+prt0j8(vU|NH=HtkJsbBO5y+QW1+hnbY=1lq~unxo7y zrVnTz)5{!Z`k6e?Jaecy#SAh>gC1>;G^d&qOkdEx=2&yKIoAvX9cWHA=bItsc+lg` zU^CoYYR&;Y$DC#UV=gqOfu3g0Gjqd7!V#b&%*E!>a9%hRbf~$+bV9E(2mQ(%^dKY6 zXmbUQai##r1am!(o6IB}x0)$9?lSk_m|!Wue`Lb4qlhxxt(iogLj^&W>(26U~{?@MxkL9!)m4n2Vw-qFc-r z(e36oGbXw&y3Je{-D&PH6Qaq{9cFSg)!c1vi>5_)n`zNBGu_OH=0?-a+~{6&pP3ge zi0(5BqWjG(^JKIGiJT15>=1Zo9fX9^QQU7 zsOU|kcg(ZqC-XDt&t|Fl%2bPLMqinl(KlwL!Spm*X@clIv&{Treg*y26q>J1ji`3? zwW%F_XBtEeqgGJ^q`jj?(VkID(3Vl_=)kCb)EKmJv{%$I+CQobS~qGE^@wt#4xk;P zgQ8T_Icf{qHtG}|5*-%hf#yX$qr;>8s4HmKD2|Saj*I$&_KgmW21G|hy+C_K{i4Cq zY0>eZ$4AFTL!y(T{-FJ%6Qc8?q0ym@o*$hXT^3y& zodJ4AbU`#ODv16A`k&~M=$h#AXc*|QXjF7#baQkS=vC3!Xi{`-G#YetbagZ(x-+^7 z^rq;B=&tCNXguinXkv6 zJ9;^K4)nR`h3LcRqv#FLH=@^~Poj6D6`(7kx1x2?7tt!vRnZ60SJ7wDJD~4GA4lIs zKSt|8*GFrk&C%D Date: Wed, 7 May 2025 17:12:43 +0000 Subject: [PATCH 34/63] driving training --- clean_pufferl.py | 7 +++++-- config/ocean/gpudrive.ini | 23 ++++++++++++++--------- pufferlib/ocean/gpudrive/gpudrive.c | 4 ++-- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index ed85966cab..6a31790545 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -306,8 +306,11 @@ def train(self): elif config.use_puff_advantage: importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) vs = torch.zeros(experience.values.shape, device=config.device) - torch.ops.pufferlib.compute_puff_advantage(experience.values, experience.rewards, - experience.dones, experience.ratio, vs, advantages, config.gamma, + + # TODO: Eliminate + n = (experience.values.shape[0]//256)*256 + torch.ops.pufferlib.compute_puff_advantage(experience.values[:n], experience.rewards[:n], + experience.dones[:n], experience.ratio[:n], vs[:n], advantages[:n], config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) else: importance = advantages = self.compute_gae(experience.values, experience.rewards, diff --git a/config/ocean/gpudrive.ini b/config/ocean/gpudrive.ini index 56a273ba0b..d79ab62b1e 100644 --- a/config/ocean/gpudrive.ini +++ b/config/ocean/gpudrive.ini @@ -3,7 +3,11 @@ package = ocean env_name = puffer_gpudrive policy_name = GPUDrive rnn_name = Recurrent -vec = multiprocessing + +[vec] +num_workers = 16 +num_envs = 16 +batch_size = 8 [policy] input_size = 256 @@ -14,17 +18,18 @@ input_size = 256 hidden_size = 256 [env] -num_envs = 256 -reward_vehicle_collision = -0.25 -reward_offroad_collision = -0.25 +num_envs = 64 +reward_vehicle_collision = -0.75 +reward_offroad_collision = -0.75 [train] total_timesteps = 250_000_000 -learning_rate = 0.05 -minibatch_size = 32768 -num_workers = 2 -num_envs = 2 -env_batch_size = 1 +learning_rate = 0.005 +anneal_lr = True +batch_size = 752752 +minibatch_size = 23296 +max_minibatch_size = 23296 +bptt_horizon = 91 [sweep.env.reward_vehicle_collision] distribution = uniform diff --git a/pufferlib/ocean/gpudrive/gpudrive.c b/pufferlib/ocean/gpudrive/gpudrive.c index 0f8376c6c7..1b9a251bc2 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.c +++ b/pufferlib/ocean/gpudrive/gpudrive.c @@ -205,7 +205,7 @@ void performance_test() { } int main() { - //demo(); - performance_test(); + demo(); + //performance_test(); return 0; } From 8fe5229af5a1fcacbedfb110909688cb25473634 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Wed, 7 May 2025 17:14:35 +0000 Subject: [PATCH 35/63] Fix model size --- config/ocean/gpudrive.ini | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/config/ocean/gpudrive.ini b/config/ocean/gpudrive.ini index d79ab62b1e..ed7a4d3d43 100644 --- a/config/ocean/gpudrive.ini +++ b/config/ocean/gpudrive.ini @@ -10,12 +10,12 @@ num_envs = 16 batch_size = 8 [policy] -input_size = 256 -hidden_size = 256 +input_size = 64 +hidden_size = 512 [rnn] -input_size = 256 -hidden_size = 256 +input_size = 512 +hidden_size = 512 [env] num_envs = 64 From d158d45ba3eca63807ba74059b284e46a17bc2a6 Mon Sep 17 00:00:00 2001 From: Spencer Cheng Date: Wed, 7 May 2025 20:09:33 +0000 Subject: [PATCH 36/63] new binding is in --- pufferlib/ocean/gpudrive/cy_gpudrive.pyx | 254 ----------------------- pufferlib/ocean/gpudrive/gpudrive.c | 8 +- pufferlib/ocean/gpudrive/gpudrive.h | 105 ++++------ pufferlib/ocean/gpudrive/gpudrive.py | 43 ++-- setup.py | 6 +- 5 files changed, 74 insertions(+), 342 deletions(-) delete mode 100644 pufferlib/ocean/gpudrive/cy_gpudrive.pyx diff --git a/pufferlib/ocean/gpudrive/cy_gpudrive.pyx b/pufferlib/ocean/gpudrive/cy_gpudrive.pyx deleted file mode 100644 index 045e953eb9..0000000000 --- a/pufferlib/ocean/gpudrive/cy_gpudrive.pyx +++ /dev/null @@ -1,254 +0,0 @@ -from libc.stdlib cimport calloc, malloc, free -from libc.string cimport strcpy -import numpy as np -cdef extern from "gpudrive.h": - int LOG_BUFFER_SIZE - - ctypedef struct Log: - float episode_return; - float episode_length; - float score; - float offroad_rate; - float collision_rate; - float dnf_rate; - ctypedef struct LogBuffer - LogBuffer* allocate_logbuffer(int) - void free_logbuffer(LogBuffer*) - Log aggregate_and_clear(LogBuffer*) - - ctypedef struct Entity: - int type; - int road_object_id; - int road_point_id; - int array_size; - float* traj_x; - float* traj_y; - float* traj_z; - float* traj_vx; - float* traj_vy; - float* traj_vz; - float* traj_heading; - int* traj_valid; - float width; - float length; - float height; - float goal_position_x; - float goal_position_y; - float goal_position_z; - int mark_as_expert; - int collision_state; - float x; - float y; - float z; - float vx; - float vy; - float vz; - float heading; - int valid; - - ctypedef struct GPUDrive: - float* observations; - int* actions; - float* rewards; - unsigned char* masks; - unsigned char* dones; - LogBuffer* log_buffer; - Log* logs; - int num_agents; - int active_agent_count; - int* active_agent_indices; - int human_agent_idx; - Entity* entities; - int num_entities; - int num_cars; - int num_objects; - int num_roads; - int static_car_count; - int* static_car_indices; - int expert_static_car_count; - int* expert_static_car_indices; - int timestep; - int dynamics_model; - float* fake_data; - char* goal_reached; - float* map_corners; - int* grid_cells; - int grid_cols; - int grid_rows; - int vision_range; - int* neighbor_offsets; - int* neighbor_cache_entities; - int* neighbor_cache_indices; - float reward_vehicle_collision; - float reward_offroad_collision; - char* map_name; - char* reached_goal_this_turn; - float world_mean_x; - float world_mean_y; - - ctypedef struct Client - - void init(GPUDrive* env) - void free_allocated(GPUDrive* env) - void free_entity(Entity* entity) - - Client* make_client(GPUDrive* env) - void close_client(Client* client) - void c_render(Client* client, GPUDrive* env) - void c_reset(GPUDrive* env) - void c_step(GPUDrive* env) - Entity* load_map_binary(char* name, GPUDrive* env) - void set_active_agents(GPUDrive *env) - -cpdef entity_dtype(): - '''Make a dummy entity to get the dtype''' - # Create a numpy structured dtype that matches the Entity struct - return np.dtype([ - ('type', np.int32), - ('road_object_id', np.int32), - ('road_point_id', np.int32), - ('array_size', np.int32), - # For pointer fields, we use intp (integer large enough to hold a pointer) - ('traj_x', np.intp), - ('traj_y', np.intp), - ('traj_z', np.intp), - ('traj_vx', np.intp), - ('traj_vy', np.intp), - ('traj_vz', np.intp), - ('traj_heading', np.intp), - ('traj_valid', np.intp), - ('width', np.float32), - ('length', np.float32), - ('height', np.float32), - ('goal_position_x', np.float32), - ('goal_position_y', np.float32), - ('goal_position_z', np.float32), - ('collision_state', np.int32), - ('x', np.float32), - ('y', np.float32), - ('z', np.float32), - ('vx', np.float32), - ('vy', np.float32), - ('vz', np.float32), - ('heading', np.float32), - ('valid', np.int32) - ]) - -cdef class CyGPUDrive: - cdef: - GPUDrive* envs - Client* client - LogBuffer* logs - int num_envs - int* agent_offsets - int agent_count - - @staticmethod - def get_total_agent_count(int num_envs, int human_agent_idx, float reward_vehicle_collision, float reward_offroad_collision): - """Static method to count total agents across all environments""" - cdef int* agent_offsets = calloc(num_envs + 1, sizeof(int)) - cdef int total_count = 0 - cdef GPUDrive* temp_envs = calloc(num_envs, sizeof(GPUDrive)) - cdef int i - for i in range(num_envs): - temp_envs[i].human_agent_idx = human_agent_idx - temp_envs[i].reward_vehicle_collision = reward_vehicle_collision - temp_envs[i].reward_offroad_collision = reward_offroad_collision - - map_file = f"resources/gpudrive/binaries/map_{i:03d}.bin".encode('utf-8') - temp_envs[i].entities = load_map_binary(map_file, &temp_envs[i]) - set_active_agents(&temp_envs[i]) - - agent_offsets[i] = total_count - total_count += temp_envs[i].active_agent_count - if (temp_envs[i].active_agent_count ==0 ): - print("No active agents: ", map_file) - - agent_offsets[num_envs] = total_count - py_offsets = [agent_offsets[i] for i in range(num_envs + 1)] - for i in range(num_envs): - for x in range(temp_envs[i].num_entities): - free_entity(&temp_envs[i].entities[x]) - free(temp_envs[i].entities) - free(temp_envs[i].active_agent_indices) - free(temp_envs[i].static_car_indices) - free(temp_envs) - free(agent_offsets) - return total_count, py_offsets - def __init__(self, float[:, :] observations, int[:,:] actions, - float[:] rewards, unsigned char[:] terminals, int num_envs, - int human_agent_idx, reward_vehicle_collision, - reward_offroad_collision, offsets): - - self.client = NULL - self.num_envs = num_envs - cdef int num_clones - num_clones = 1 - self.envs = calloc(num_envs*num_clones, sizeof(GPUDrive)) - self.agent_offsets = calloc(num_envs + 1, sizeof(int)) - self.logs = allocate_logbuffer(LOG_BUFFER_SIZE) - cdef int i - for i in range(num_envs + 1): - self.agent_offsets[i] = offsets[i] - cdef int inc - cdef int index - cdef int total_envs - total_envs = num_envs * num_clones - cdef int total_agents - total_agents = self.agent_offsets[num_envs] - cdef char* c_map_file - for i in range(total_envs): - env_index = i % num_envs - clone_index = i // num_envs - inc = self.agent_offsets[env_index] - count = self.agent_offsets[env_index+1] - self.agent_offsets[env_index] - clone_agent_offset = clone_index * total_agents + inc - map_file = f"resources/gpudrive/binaries/map_{env_index:03d}.bin".encode('utf-8') - c_map_file = malloc(len(map_file) + 1) - strcpy(c_map_file, map_file) - self.envs[i] = GPUDrive( - observations=&observations[clone_agent_offset, 0], - actions=&actions[clone_agent_offset,0], - rewards=&rewards[clone_agent_offset], - dones=&terminals[clone_agent_offset], - log_buffer=self.logs, - human_agent_idx=human_agent_idx, - reward_vehicle_collision=reward_vehicle_collision, - reward_offroad_collision=reward_offroad_collision, - map_name = c_map_file - ) - init(&self.envs[i]) - self.client = NULL - - - def reset(self): - cdef int i - for i in range(self.num_envs): - c_reset(&self.envs[i]) - - def step(self): - cdef int i - for i in range(self.num_envs): - c_step(&self.envs[i]) - - def render(self): - cdef GPUDrive* env = &self.envs[24] - if self.client == NULL: - import os - cwd = os.getcwd() - os.chdir(os.path.abspath(os.path.join(os.path.dirname(__file__), "..", ".."))) - self.client = make_client(env) - os.chdir(cwd) - - c_render(self.client, env) - - def close(self): - if self.client != NULL: - close_client(self.client) - self.client = NULL - - free(self.envs) - - def log(self): - cdef Log log = aggregate_and_clear(self.logs) - return log diff --git a/pufferlib/ocean/gpudrive/gpudrive.c b/pufferlib/ocean/gpudrive/gpudrive.c index 1b9a251bc2..bc8a3519ff 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.c +++ b/pufferlib/ocean/gpudrive/gpudrive.c @@ -160,10 +160,10 @@ void demo() { // Handle human input for the controlled agent // handle_human_input(&env); c_step(&env); - c_render(client, &env); + c_render(&env); } - close_client(client); + close_client(env.client); free_allocated(&env); } @@ -205,7 +205,7 @@ void performance_test() { } int main() { - demo(); - //performance_test(); + //demo(); + performance_test(); return 0; } diff --git a/pufferlib/ocean/gpudrive/gpudrive.h b/pufferlib/ocean/gpudrive/gpudrive.h index fd4f4c59a4..c3d07c98bc 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.h +++ b/pufferlib/ocean/gpudrive/gpudrive.h @@ -81,65 +81,22 @@ static const int collision_offsets[25][2] = { {-2, 1}, {-1, 1}, {0, 1}, {1, 1}, {2, 1}, // Fourth row {-2, 2}, {-1, 2}, {0, 2}, {1, 2}, {2, 2} // Bottom row }; -#define LOG_BUFFER_SIZE 1024 +typedef struct GPUDrive GPUDrive; +typedef struct Client Client; typedef struct Log Log; + struct Log { float episode_return; float episode_length; + float perf; float score; float offroad_rate; float collision_rate; float dnf_rate; + float n; }; - -typedef struct LogBuffer LogBuffer; -struct LogBuffer { - Log* logs; - int length; - int idx; -}; - -LogBuffer* allocate_logbuffer(int size) { - LogBuffer* logs = (LogBuffer*)calloc(1, sizeof(LogBuffer)); - logs->logs = (Log*)calloc(size, sizeof(Log)); - logs->length = size; - logs->idx = 0; - return logs; -} - -void free_logbuffer(LogBuffer* buffer) { - free(buffer->logs); - free(buffer); -} - -void add_log(LogBuffer* logs, Log* log) { - if (logs->idx == logs->length) { - return; - } - logs->logs[logs->idx] = *log; - logs->idx += 1; - //printf("Log: %f, %f,\n", log->episode_return, log->episode_length); -} - -Log aggregate_and_clear(LogBuffer* logs) { - Log log = {0}; - if (logs->idx == 0) { - return log; - } - for (int i = 0; i < logs->idx; i++) { - log.episode_return += logs->logs[i].episode_return / logs->idx; - log.episode_length += logs->logs[i].episode_length / logs->idx; - log.score += logs->logs[i].score / logs->idx; - log.offroad_rate += logs->logs[i].offroad_rate / logs->idx; - log.collision_rate += logs->logs[i].collision_rate / logs->idx; - log.dnf_rate += logs->logs[i].dnf_rate / logs->idx; - } - logs->idx = 0; - return log; -} - typedef struct Entity Entity; struct Entity { int type; @@ -196,13 +153,13 @@ float relative_distance_2d(float x1, float y1, float x2, float y2){ return distance; } -typedef struct GPUDrive GPUDrive; struct GPUDrive { + Client* client; float* observations; int* actions; float* rewards; - unsigned char* dones; - LogBuffer* log_buffer; + unsigned char* terminals; + Log log; Log* logs; int num_agents; int active_agent_count; @@ -237,6 +194,25 @@ struct GPUDrive { float world_mean_y; }; +void add_log(GPUDrive* env) { + for(int i = 0; i < env->active_agent_count; i++){ + if(env->reached_goal_this_episode[i]) { + env->log.score += 1.0f; + env->log.perf += 1.0f; + } + int offroad = env->logs[i].offroad_rate; + env->log.offroad_rate += offroad; + int collided = env->logs[i].collision_rate; + env->log.collision_rate += collided; + if(!offroad && !collided && !env->reached_goal_this_episode[i]){ + env->log.dnf_rate += 1.0f; + } + env->log.episode_length += env->logs[i].episode_length; + env->log.episode_return += env->logs[i].episode_return; + env->log.n += 1; + } +} + Entity* load_map_binary(const char* filename, GPUDrive* env) { FILE* file = fopen(filename, "rb"); //printf("fileanme: %s\n", filename); @@ -663,8 +639,7 @@ void allocate(GPUDrive* env){ env->observations = (float*)calloc(env->active_agent_count*max_obs, sizeof(float)); env->actions = (int*)calloc(env->active_agent_count*2, sizeof(int)); env->rewards = (float*)calloc(env->active_agent_count, sizeof(float)); - env->dones = (unsigned char*)calloc(env->active_agent_count, sizeof(unsigned char)); - env->log_buffer = allocate_logbuffer(LOG_BUFFER_SIZE); + env->terminals= (unsigned char*)calloc(env->active_agent_count, sizeof(unsigned char)); // printf("allocated\n"); } @@ -672,8 +647,7 @@ void free_allocated(GPUDrive* env){ free(env->observations); free(env->actions); free(env->rewards); - free(env->dones); - free_logbuffer(env->log_buffer); + free(env->terminals); free_initialized(env); } @@ -1048,19 +1022,7 @@ void c_step(GPUDrive* env){ memset(env->rewards, 0, env->active_agent_count * sizeof(float)); env->timestep++; if(env->timestep == 91){ - for(int i = 0; i < env->active_agent_count; i++){ - if(!env->reached_goal_this_episode[i]) { - env->logs[i].score = 0.0f; - } else { - env->logs[i].score = 1.0f; - } - int offroad = env->logs[i].offroad_rate; - int collided = env->logs[i].collision_rate; - if(!offroad && !collided && !env->reached_goal_this_episode[i]){ - env->logs[i].dnf_rate = 1.0f; - } - add_log(env->log_buffer, &env->logs[i]); - } + add_log(env); c_reset(env); } // Move statix experts @@ -1372,7 +1334,12 @@ void draw_road_edge(GPUDrive* env, float start_x, float start_y, float end_x, fl DrawTriangle3D(t4, t1, b1, CURB_SIDE); } -void c_render(Client* client, GPUDrive* env) { +void c_render(GPUDrive* env) { + if (env->client == NULL) { + env->client = make_client(env); + } + Client* client = env->client; + BeginDrawing(); Color road = (Color){35, 35, 37, 255}; ClearBackground(road); diff --git a/pufferlib/ocean/gpudrive/gpudrive.py b/pufferlib/ocean/gpudrive/gpudrive.py index 8877afbb1c..57a4c51a67 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.py +++ b/pufferlib/ocean/gpudrive/gpudrive.py @@ -5,6 +5,7 @@ import pufferlib from pufferlib.ocean.gpudrive.cy_gpudrive import CyGPUDrive, entity_dtype +from pufferlib.ocean.gpudrive import binding class GPUDrive(pufferlib.PufferEnv): def __init__(self, num_envs=1, render_mode=None, report_interval=1, @@ -25,37 +26,55 @@ def __init__(self, num_envs=1, render_mode=None, report_interval=1, shape=(self.num_obs,), dtype=np.float32) self.single_action_space = gymnasium.spaces.MultiDiscrete([7, 13]) - total_agents, agent_offsets =CyGPUDrive.get_total_agent_count( - num_envs, human_agent_idx, reward_vehicle_collision, reward_offroad_collision) + agent_offsets = binding.shared(num_envs=num_envs) + total_agents = agent_offsets[-1] self.num_agents = total_agents super().__init__(buf=buf) - self.c_envs = CyGPUDrive(self.observations, self.actions, self.rewards, - self.terminals, num_envs, human_agent_idx, reward_vehicle_collision, reward_offroad_collision, offsets = agent_offsets) + env_ids = [] + for i in range(num_envs): + cur = agent_offsets[i] + nxt = agent_offsets[i+1] + env_id = binding.env_init( + self.observations[cur:nxt], + self.actions[cur:nxt], + self.rewards[cur:nxt], + self.terminals[cur:nxt], + self.truncations[cur:nxt], + seed, + human_agent_idx=human_agent_idx, + reward_vehicle_collision=reward_vehicle_collision, + reward_offroad_collision=reward_offroad_collision, + env_id=i + ) + env_ids.append(env_id) + self.c_envs = binding.vectorize(*env_ids) + pass - def reset(self, seed=None): - self.c_envs.reset() + def reset(self, seed=0): + binding.vec_reset(self.c_envs, seed) self.tick = 0 return self.observations, [] def step(self, actions): self.actions[:] = actions - self.c_envs.step() + binding.vec_step(self.c_envs) self.tick+=1 info = [] if self.tick % self.report_interval == 0: - log = self.c_envs.log() - if log['episode_length'] > 0: + log = binding.vec_log(self.c_envs) + if log: info.append(log) - info.append({'total_agents': self.num_agents}) + return (self.observations, self.rewards, self.terminals, self.truncations, info) def render(self): - self.c_envs.render() + binding.vec_render(self.c_envs) def close(self): - self.c_envs.close() + binding.vec_close(self.c_envs) + def calculate_area(p1, p2, p3): # Calculate the area of the triangle using the determinant method return 0.5 * abs((p1['x'] - p3['x']) * (p2['y'] - p1['y']) - (p1['x'] - p2['x']) * (p3['y'] - p1['y'])) diff --git a/setup.py b/setup.py index 042def711f..4887b8ae56 100644 --- a/setup.py +++ b/setup.py @@ -319,7 +319,7 @@ def run(self): # 'pufferlib/ocean/tactical/c_tactical', #'pufferlib/ocean/squared/cy_squared', 'pufferlib/ocean/snake/cy_snake', - 'pufferlib/ocean/gpudrive/cy_gpudrive', + #'pufferlib/ocean/gpudrive/cy_gpudrive', #'pufferlib/ocean/pong/cy_pong', # 'pufferlib/ocean/breakout/cy_breakout', # 'pufferlib/ocean/cartpole/cy_cartpole', @@ -364,14 +364,14 @@ def run(self): #c_args += "-Wsign-compare -DNDEBUG -g -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC".split() -pure_c_extensions = ['squared', 'pong', 'breakout', 'enduro', 'blastar', 'grid', 'nmmo3', 'tactical', 'go', 'cartpole'] +pure_c_extensions = ['gpudrive', 'squared', 'pong', 'breakout', 'enduro', 'blastar', 'grid', 'nmmo3', 'tactical', 'go', 'cartpole'] c_extensions = [ Extension( f'pufferlib.ocean.{name}.binding', sources=[f'pufferlib/ocean/{name}/binding.c'], include_dirs=[numpy.get_include(), 'raylib/include'], - extra_compile_args=extra_compile_args,# + ['-fsanitize=address,undefined,bounds,pointer-overflow,leak'], + extra_compile_args=extra_compile_args, # + ['-fsanitize=address,undefined,bounds,pointer-overflow,leak', '-static-libasan'], extra_link_args=extra_link_args,# + ['-fsanitize=address,undefined,bounds,pointer-overflow,leak', '-g'], extra_objects=[f'{RAYLIB_NAME}/lib/libraylib.a'], ) From d80dee0fed3cf8459a5275cc1e5d0b490441f9ba Mon Sep 17 00:00:00 2001 From: Spencer Cheng Date: Wed, 7 May 2025 21:30:52 +0000 Subject: [PATCH 37/63] small adjustments --- config/ocean/gpudrive.ini | 39 ++++++++++++++++++++++++++--- pufferlib/ocean/gpudrive/gpudrive.h | 4 +-- 2 files changed, 37 insertions(+), 6 deletions(-) diff --git a/config/ocean/gpudrive.ini b/config/ocean/gpudrive.ini index ed7a4d3d43..bc82352f7c 100644 --- a/config/ocean/gpudrive.ini +++ b/config/ocean/gpudrive.ini @@ -8,6 +8,7 @@ rnn_name = Recurrent num_workers = 16 num_envs = 16 batch_size = 8 +#backend = Serial [policy] input_size = 64 @@ -18,19 +19,49 @@ input_size = 512 hidden_size = 512 [env] -num_envs = 64 +num_envs = 72 reward_vehicle_collision = -0.75 reward_offroad_collision = -0.75 [train] -total_timesteps = 250_000_000 -learning_rate = 0.005 +total_timesteps = 100_000_000 +#learning_rate = 0.005 anneal_lr = True -batch_size = 752752 +batch_size = 738192 minibatch_size = 23296 max_minibatch_size = 23296 bptt_horizon = 91 +#adam_beta1 = 0.9225899639773112 +#adam_beta2 = 0.9 +#adam_eps = 0.0004030478187254784 +#ent_coef = 0.0020159472963835016 +#gae_lambda = 0.8829440612065992 +#gamma = 0.9872971455373439 +#learning_rate = 0.0003947934701844728 +#max_grad_norm = 0.5296288081133984 +#prio_alpha = 0.99 +#prio_beta0 = 0.48469847315324566 +#update_epochs = 2 +#vf_coef = 3.6777541336880786 +#checkpoint_interval = 1000 + +adam_beta1 = 0.9852000972032763 +adam_beta2 = 0.9948751690861872 +adam_eps = 0.000002967099767264975 +clip_coef = 0.3153578071651496 +ent_coef = 0.000369784972524992 +gae_lambda = 0.9385892578563558 +gamma = 0.9864999317644947 +learning_rate = 0.0022659903674495338 +max_grad_norm = 1.942292174080673 +prio_alpha = 0.9414003089586056 +prio_beta = 0.9429842108374631 +vf_clip_coef = 1.9533056765171148 +vf_coef = 3.2028923035616774 + + + [sweep.env.reward_vehicle_collision] distribution = uniform min = -1.0 diff --git a/pufferlib/ocean/gpudrive/gpudrive.h b/pufferlib/ocean/gpudrive/gpudrive.h index c3d07c98bc..03c3929a2f 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.h +++ b/pufferlib/ocean/gpudrive/gpudrive.h @@ -317,7 +317,7 @@ void set_active_agents(GPUDrive* env){ int expert_static_car_indices[MAX_CARS]; env->active_agent_count = 1; active_agent_indices[0] = env->num_objects-1; - for(int i = 0; i < env->num_objects && env->num_cars < MAX_CARS; i++){ + for(int i = 0; i < env->num_objects-1 && env->num_cars < MAX_CARS; i++){ if(env->entities[i].type != 1) continue; if(env->entities[i].traj_valid[0] != 1) continue; env->num_cars++; @@ -1042,7 +1042,7 @@ void c_step(GPUDrive* env){ } env->entities[agent_idx].collision_state = 0; move_dynamics(env, i, agent_idx); - // move_expert(env, env->actions, agent_idx); + //move_expert(env, env->actions, agent_idx); collision_check(env, agent_idx); if(env->entities[agent_idx].collision_state > 0 && env->goal_reached[i] == 0){ if(env->entities[agent_idx].collision_state == VEHICLE_COLLISION){ From 3216ba84564a684ef71506eec10b34b6652ba5d7 Mon Sep 17 00:00:00 2001 From: Spencer Cheng Date: Wed, 7 May 2025 21:34:48 +0000 Subject: [PATCH 38/63] binding.c --- pufferlib/ocean/gpudrive/binding.c | 62 ++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 pufferlib/ocean/gpudrive/binding.c diff --git a/pufferlib/ocean/gpudrive/binding.c b/pufferlib/ocean/gpudrive/binding.c new file mode 100644 index 0000000000..a8999d90f3 --- /dev/null +++ b/pufferlib/ocean/gpudrive/binding.c @@ -0,0 +1,62 @@ +#include "gpudrive.h" +#define Env GPUDrive +#define MY_SHARED +#include "../env_binding.h" + +static PyObject* my_shared(PyObject* self, PyObject* args, PyObject* kwargs) { + int num_envs = unpack(kwargs, "num_envs"); + GPUDrive* temp_envs = calloc(num_envs, sizeof(GPUDrive)); + PyObject* agent_offsets = PyList_New(num_envs+1); + int total_count = 0; + // getting agent counts and offsets + for(int i = 0;i< num_envs;i++) { + char map_file[100]; + sprintf(map_file, "resources/gpudrive/binaries/map_%03d.bin", i); + temp_envs[i].entities = load_map_binary(map_file, &temp_envs[i]); + set_active_agents(&temp_envs[i]); + PyObject* num = PyLong_FromLong(total_count); + PyList_SetItem(agent_offsets, i, num); + //Py_DECREF(num); + total_count += temp_envs[i].active_agent_count; + } + PyObject* num = PyLong_FromLong(total_count); + PyList_SetItem(agent_offsets, num_envs, num); + //Py_DECREF(num); + /* + for(int i = 0;ihuman_agent_idx = unpack(kwargs, "human_agent_idx"); + env->reward_vehicle_collision = unpack(kwargs, "reward_vehicle_collision"); + env->reward_offroad_collision = unpack(kwargs, "reward_offroad_collision"); + int env_id = unpack(kwargs, "env_id"); + + char map_file[100]; + sprintf(map_file, "resources/gpudrive/binaries/map_%03d.bin", env_id); + env->map_name = map_file; + init(env); + return 0; +} + +static int my_log(PyObject* dict, Log* log) { + assign_to_dict(dict, "perf", log->perf); + assign_to_dict(dict, "score", log->score); + assign_to_dict(dict, "episode_return", log->episode_return); + assign_to_dict(dict, "episode_length", log->episode_length); + assign_to_dict(dict, "offroad_rate", log->offroad_rate); + assign_to_dict(dict, "collision_rate", log->collision_rate); + assign_to_dict(dict, "dnf_rate", log->dnf_rate); + assign_to_dict(dict, "n", log->n); + return 0; +} From 78285d7325af1eb6a479cadadd5a61d250c5418e Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Thu, 8 May 2025 13:26:40 +0000 Subject: [PATCH 39/63] gpudrive --- config/ocean/gpudrive.ini | 6 +++--- pufferlib/ocean/gpudrive/gpudrive.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/config/ocean/gpudrive.ini b/config/ocean/gpudrive.ini index bc82352f7c..cc4a8d42c0 100644 --- a/config/ocean/gpudrive.ini +++ b/config/ocean/gpudrive.ini @@ -20,11 +20,11 @@ hidden_size = 512 [env] num_envs = 72 -reward_vehicle_collision = -0.75 -reward_offroad_collision = -0.75 +reward_vehicle_collision = 0.0 +reward_offroad_collision = 0.0 [train] -total_timesteps = 100_000_000 +total_timesteps = 250_000_000 #learning_rate = 0.005 anneal_lr = True batch_size = 738192 diff --git a/pufferlib/ocean/gpudrive/gpudrive.py b/pufferlib/ocean/gpudrive/gpudrive.py index 57a4c51a67..0180ba3943 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.py +++ b/pufferlib/ocean/gpudrive/gpudrive.py @@ -70,7 +70,7 @@ def step(self, actions): self.terminals, self.truncations, info) def render(self): - binding.vec_render(self.c_envs) + binding.vec_render(self.c_envs, 0) def close(self): binding.vec_close(self.c_envs) From cbe9e2e4419fa60d6a455901a9636b529beb3bd1 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Thu, 8 May 2025 17:11:59 +0000 Subject: [PATCH 40/63] Initial obs norm fix --- config/ocean/gpudrive.ini | 2 +- pufferlib/ocean/gpudrive/gpudrive.c | 11 ++++++----- pufferlib/ocean/gpudrive/gpudrive.h | 27 ++++++++++++++++++++------- pufferlib/ocean/gpudrive/gpudrive.py | 3 +-- 4 files changed, 28 insertions(+), 15 deletions(-) diff --git a/config/ocean/gpudrive.ini b/config/ocean/gpudrive.ini index cc4a8d42c0..67ac442910 100644 --- a/config/ocean/gpudrive.ini +++ b/config/ocean/gpudrive.ini @@ -24,7 +24,7 @@ reward_vehicle_collision = 0.0 reward_offroad_collision = 0.0 [train] -total_timesteps = 250_000_000 +total_timesteps = 150_000_000 #learning_rate = 0.005 anneal_lr = True batch_size = 738192 diff --git a/pufferlib/ocean/gpudrive/gpudrive.c b/pufferlib/ocean/gpudrive/gpudrive.c index bc8a3519ff..bcb1b20410 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.c +++ b/pufferlib/ocean/gpudrive/gpudrive.c @@ -102,17 +102,18 @@ void demo() { .human_agent_idx = 0, .reward_vehicle_collision = -0.1f, .reward_offroad_collision = -0.1f, - .map_name = "resources/gpudrive/binaries/map_063.bin" + .map_name = "resources/gpudrive/binaries/map_000.bin" }; allocate(&env); c_reset(&env); - Client* client = make_client(&env); + c_render(&env); + //Client* client = make_client(&env); printf("Human controlling agent index: %d\n", env.active_agent_indices[env.human_agent_idx]); int accel_delta = 1; int steer_delta = 1; while (!WindowShouldClose()) { // Handle camera controls - handle_camera_controls(client); + handle_camera_controls(env.client); int (*actions)[2] = (int(*)[2])env.actions; // // Reset all agent actions at the beginning of each frame // for(int i = 0; i < env.active_agent_count; i++) { @@ -205,7 +206,7 @@ void performance_test() { } int main() { - //demo(); - performance_test(); + demo(); + //performance_test(); return 0; } diff --git a/pufferlib/ocean/gpudrive/gpudrive.h b/pufferlib/ocean/gpudrive/gpudrive.h index 03c3929a2f..47119f55ff 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.h +++ b/pufferlib/ocean/gpudrive/gpudrive.h @@ -279,6 +279,8 @@ Entity* load_map_binary(const char* filename, GPUDrive* env) { } void set_start_position(GPUDrive* env){ + //InitWindow(800, 600, "GPU Drive"); + //BeginDrawing(); for(int i = 0; i < env->num_entities; i++){ int is_active = 0; for(int j = 0; j < env->active_agent_count; j++){ @@ -291,6 +293,10 @@ void set_start_position(GPUDrive* env){ e->x = e->traj_x[0]; e->y = e->traj_y[0]; e->z = e->traj_z[0]; + //printf("Entity %d is at (%f, %f, %f)\n", i, e->x, e->y, e->z); + //if (e->type < 4) { + // DrawRectangle(200+2*e->x, 200+2*e->y, 2.0, 2.0, RED); + //} if(e->type >3 || e->type == 0){ continue; } @@ -306,6 +312,10 @@ void set_start_position(GPUDrive* env){ e->heading = e->traj_heading[0]; e->valid = e->traj_valid[0]; } + //EndDrawing(); + int x = 0; + + } void set_active_agents(GPUDrive* env){ @@ -902,9 +912,12 @@ void compute_observations(GPUDrive* env) { // Rotate to ego vehicle's frame float rel_goal_x = goal_x*cos_heading + goal_y*sin_heading; float rel_goal_y = -goal_x*sin_heading + goal_y*cos_heading; - obs[0] = normalize_value(rel_goal_x, MIN_REL_GOAL_COORD, MAX_REL_GOAL_COORD); - obs[1] = normalize_value(rel_goal_y, MIN_REL_GOAL_COORD, MAX_REL_GOAL_COORD); - obs[2] = ego_speed / MAX_SPEED; + //obs[0] = normalize_value(rel_goal_x, MIN_REL_GOAL_COORD, MAX_REL_GOAL_COORD); + //obs[1] = normalize_value(rel_goal_y, MIN_REL_GOAL_COORD, MAX_REL_GOAL_COORD); + obs[0] = rel_goal_x/20.0f; + obs[1] = rel_goal_y/20.0f; + //obs[2] = ego_speed / MAX_SPEED; + obs[2] = ego_speed / 5.0f; obs[3] = ego_entity->width / MAX_VEH_WIDTH; obs[4] = ego_entity->length / MAX_VEH_LEN; obs[5] = (ego_entity->collision_state > 0) ? 1 : 0; @@ -932,8 +945,8 @@ void compute_observations(GPUDrive* env) { float rel_x = dx*cos_heading + dy*sin_heading; float rel_y = -dx*sin_heading + dy*cos_heading; // Store observations with correct indexing - obs[obs_idx] = normalize_value(rel_x, MIN_REL_AGENT_POS, MAX_REL_AGENT_POS); - obs[obs_idx + 1] = normalize_value(rel_y, MIN_REL_AGENT_POS, MAX_REL_AGENT_POS); + obs[obs_idx] = rel_x / 20.0f; + obs[obs_idx + 1] = rel_y / 20.0f; obs[obs_idx + 2] = other_entity->width / MAX_VEH_WIDTH; obs[obs_idx + 3] = other_entity->length / MAX_VEH_LEN; // relative heading @@ -982,8 +995,8 @@ void compute_observations(GPUDrive* env) { // Compute sin and cos of relative angle directly without atan2f float cos_angle = dx_norm*cos_heading + dy_norm*sin_heading; float sin_angle = -dx_norm*sin_heading + dy_norm*cos_heading; - obs[obs_idx] = normalize_value(x_obs, MIN_RG_COORD, MAX_RG_COORD); - obs[obs_idx + 1] = normalize_value(y_obs, MIN_RG_COORD, MAX_RG_COORD); + obs[obs_idx] = x_obs / 20.0f; + obs[obs_idx + 1] = y_obs / 20.0f; obs[obs_idx + 2] = length / MAX_ROAD_SEGMENT_LENGTH; obs[obs_idx + 3] = width / MAX_ROAD_SCALE; obs[obs_idx + 4] = cos_angle / MAX_ORIENTATION_RAD; diff --git a/pufferlib/ocean/gpudrive/gpudrive.py b/pufferlib/ocean/gpudrive/gpudrive.py index 0180ba3943..9a9a82c93e 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.py +++ b/pufferlib/ocean/gpudrive/gpudrive.py @@ -49,7 +49,6 @@ def __init__(self, num_envs=1, render_mode=None, report_interval=1, env_ids.append(env_id) self.c_envs = binding.vectorize(*env_ids) - pass def reset(self, seed=0): binding.vec_reset(self.c_envs, seed) @@ -70,7 +69,7 @@ def step(self, actions): self.terminals, self.truncations, info) def render(self): - binding.vec_render(self.c_envs, 0) + binding.vec_render(self.c_envs, 63) def close(self): binding.vec_close(self.c_envs) From b1b8945376401ef216537e2714bf875c2b31e0c5 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Thu, 8 May 2025 17:16:45 +0000 Subject: [PATCH 41/63] gpudrive fixes --- config/default.ini | 24 ++++++++++++------------ config/ocean/gpudrive.ini | 13 ++++++++++--- pufferlib/ocean/gpudrive/gpudrive.py | 5 +---- 3 files changed, 23 insertions(+), 19 deletions(-) diff --git a/config/default.ini b/config/default.ini index abb15cef84..4847359bae 100644 --- a/config/default.ini +++ b/config/default.ini @@ -79,20 +79,20 @@ max = 1e10 mean = 1e8 scale = time -[sweep.train.batch_size] -distribution = uniform_pow2 -min = 131072 -max = 2097152 -mean = 524288 -scale = auto +#[sweep.train.batch_size] +#distribution = uniform_pow2 +#min = 131072 +#max = 2097152 +#mean = 524288 +#scale = auto # TODO: Mini minibatch optim is lower -[sweep.train.minibatch_size] -distribution = uniform_pow2 -min = 16384 -max = 131072 -mean = 8192 -scale = auto +#[sweep.train.minibatch_size] +#distribution = uniform_pow2 +#min = 16384 +#max = 131072 +#mean = 8192 +#scale = auto [sweep.train.learning_rate] distribution = log_normal diff --git a/config/ocean/gpudrive.ini b/config/ocean/gpudrive.ini index 67ac442910..4391e1c7f1 100644 --- a/config/ocean/gpudrive.ini +++ b/config/ocean/gpudrive.ini @@ -60,18 +60,25 @@ prio_beta = 0.9429842108374631 vf_clip_coef = 1.9533056765171148 vf_coef = 3.2028923035616774 - - +[sweep.train.total_timesteps] +distribution = log_normal +min = 5e7 +max = 2e8 +mean = 1e8 +scale = time + [sweep.env.reward_vehicle_collision] distribution = uniform min = -1.0 max = -0.25 +max = 0.0 mean = -0.5 scale = auto - + [sweep.env.reward_offroad_collision] distribution = uniform min = -1.0 max = -0.25 +max = 0.0 mean = -0.5 scale = auto diff --git a/pufferlib/ocean/gpudrive/gpudrive.py b/pufferlib/ocean/gpudrive/gpudrive.py index 9a9a82c93e..0c3a038559 100644 --- a/pufferlib/ocean/gpudrive/gpudrive.py +++ b/pufferlib/ocean/gpudrive/gpudrive.py @@ -4,7 +4,6 @@ import struct import pufferlib -from pufferlib.ocean.gpudrive.cy_gpudrive import CyGPUDrive, entity_dtype from pufferlib.ocean.gpudrive import binding class GPUDrive(pufferlib.PufferEnv): @@ -219,6 +218,7 @@ def save_map_binary(map_data, output_file): f.write(struct.pack('f', float(goal_pos.get('y', 0.0)))) # Get y value f.write(struct.pack('f', float(goal_pos.get('z', 0.0)))) # Get z value f.write(struct.pack('i', road.get('mark_as_expert', 0))) + def load_map(map_name, binary_output=None): """Loads a JSON map and optionally saves it as binary""" with open(map_name, 'r') as f: @@ -226,9 +226,6 @@ def load_map(map_name, binary_output=None): if binary_output: save_map_binary(map_data, binary_output) - - entities = np.zeros(1, dtype=entity_dtype()) - return entities def process_all_maps(): """Process all maps and save them as binaries""" From 96279975ca4f661135a173d2f924c5d7812fe4cf Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Thu, 8 May 2025 20:07:32 +0000 Subject: [PATCH 42/63] Initial main script refactor --- clean_pufferl.py | 348 +++++++++++++++++++++-------------------------- 1 file changed, 154 insertions(+), 194 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 6a31790545..b068477dd5 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -1,6 +1,9 @@ # TODO: Add information # - Help menu # - Docs link +#python -m torch.distributed.run --standalone --nnodes=1 --nproc-per-node=1 clean_pufferl.py --env puffer_nmmo3 --mode train +#from torch.distributed.elastic.multiprocessing.errors import record +#@record import os import random @@ -728,71 +731,6 @@ def dist_mean(value, device): return dist_sum(value, device) / torch.distributed.get_world_size() -def rollout(env_creator, env_kwargs, policy_cls, rnn_cls, agent_creator, agent_kwargs, - backend, render_mode='auto', model_path=None, device='cuda'): - - if render_mode != 'auto': - env_kwargs['render_mode'] = render_mode - - # We are just using Serial vecenv to give a consistent - # single-agent/multi-agent API for evaluation - env = pufferlib.vector.make(env_creator, env_kwargs=env_kwargs, backend=backend) - - agent = agent_creator(env, policy_cls, rnn_cls, agent_kwargs).to(device) - if model_path is not None: - agent.load_state_dict(torch.load(model_path, map_location=device, weights_only=False)) - - ob, info = env.reset() - driver = env.driver_env - os.system('clear') - - state = pufferlib.namespace( - lstm_h=None, - lstm_c=None, - ) - - num_agents = env.observation_space.shape[0] - if isinstance(agent, torch.nn.LSTM): - shape = (num_agents, agent.hidden_size) - state.lstm_h = torch.zeros(shape).to(device) - state.lstm_c = torch.zeros(shape).to(device) - - frames = [] - tick = 0 - while tick <= 200000: - if tick > 1000 and tick % 1 == 0: - render = driver.render() - if driver.render_mode == 'ansi': - print('\033[0;0H' + render + '\n') - time.sleep(1/20) - elif driver.render_mode == 'rgb_array': - frames.append(render) - import cv2 - render = cv2.cvtColor(render, cv2.COLOR_RGB2BGR) - cv2.imshow('frame', render) - cv2.waitKey(1) - time.sleep(1/24) - elif driver.render_mode in ('human', 'raylib') and render is not None: - frames.append(render) - - with torch.no_grad(): - ob = torch.as_tensor(ob).to(device) - logits, value = agent(ob, state) - action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=agent.is_continuous) - action = action.cpu().numpy().reshape(env.action_space.shape) - - ob, reward = env.step(action)[:2] - reward = reward.mean() - if tick % 128 == 0: - print(f'Reward: {reward:.4f}, Tick: {tick}') - tick += 1 - - # TODO: Frames from raylib - # Save frames as gif - if frames: - import imageio - os.makedirs('../docker', exist_ok=True) or imageio.mimsave('../docker/eval.gif', frames, fps=15, loop=0) - class Profile: def __init__(self, keys, frequency=1): self.stack = [] @@ -950,101 +888,6 @@ def downsample_linear(arr, m): x_new = np.linspace(0, 1, m) # New indices normalized return np.interp(x_new, x_old, arr) -def sweep(args, env_name, make_env, policy_cls, rnn_cls): - if not args['wandb'] and not args['neptune']: - raise pufferlib.APIUsageError('Sweeps require either wandb or neptune') - - method = args['sweep'].pop('method') - try: - sweep_cls = getattr(pufferlib.sweep, method) - except: - raise pufferlib.APIUsageError(f'Invalid sweep method {method}. See pufferlib.sweep') - - sweep = sweep_cls(args['sweep']) - target_metric = args['sweep']['metric'] - for i in range(args['max_runs']): - seed = time.time_ns() & 0xFFFFFFFF - random.seed(seed) - np.random.seed(seed) - torch.manual_seed(seed) - - info = sweep.suggest(args) - if args['train']['minibatch_size'] >= args['train']['batch_size']: - sweep.observe(args, 0.0, 0.0) - continue - - scores, costs, timesteps = train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) - scores = downsample_linear(scores, 10) - costs = downsample_linear(costs, 10) - timesteps = downsample_linear(timesteps, 10) - - # Hacky patch to prevent increasing total_timesteps when not swept - total_timesteps = args['train']['total_timesteps'] - for score, cost, timestep in zip(scores, costs, timesteps): - args['train']['total_timesteps'] = timestep - sweep.observe(args, score, cost) - - args['train']['total_timesteps'] = total_timesteps - - print('Score:', score, 'Cost:', cost, 'Timesteps:', timestep) - -def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_points=100, wandb=None, neptune=None): - vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) - policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) - args['train']['use_rnn'] = rnn_cls is not None - - if 'LOCAL_RANK' in os.environ: - from torch.nn.parallel import DistributedDataParallel as DDP - orig_policy = policy - policy = DDP(policy, device_ids=[args['local_rank']]) - policy.hidden_size = orig_policy.hidden_size - policy.is_continuous = orig_policy.is_continuous - policy.forward_train = orig_policy.forward_train - if args['train']['use_rnn']: - policy.cell = orig_policy.cell - - env_name = args['env_name'] - train_config = pufferlib.namespace(**args['train'], env=env_name, tag=args['tag'], - exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8]) - pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) - - timesteps = [] - scores = [] - costs = [] - target_key = f'environment/{target_metric}' - - vecenv.async_reset(train_config.seed) - while pufferl.global_step < train_config.total_timesteps: - pufferl.evaluate() - logs = pufferl.train() - min_sweep_steps = args['sweep']['train']['total_timesteps']['min'] - if logs is not None and target_key in logs and pufferl.global_step >= min_sweep_steps: - timesteps.append(logs['agent_steps']) - scores.append(logs[target_key]) - costs.append(pufferl.uptime) - - steps_evaluated = 0 - cost = time.time() - pufferl.start_time - batch_size = args['train']['batch_size'] - timesteps.append(pufferl.global_step) - while len(pufferl.stats[target_metric]) < min_eval_points: - stats = pufferl.evaluate() - steps_evaluated += batch_size - - pufferl.mean_and_log() - score = stats[target_metric] - print(f'Evaluated {steps_evaluated} steps. Score: {score}') - - scores.append(score) - costs.append(cost) - - pufferl.close() - return scores, costs, timesteps - -#python -m torch.distributed.run --standalone --nnodes=1 --nproc-per-node=1 clean_pufferl.py --env puffer_nmmo3 --mode train -#from torch.distributed.elastic.multiprocessing.errors import record -#@record - if __name__ == '__main__': parser = argparse.ArgumentParser( description=f':blowfish: PufferLib [bright_cyan]{pufferlib.__version__}[/]' @@ -1053,13 +896,16 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin parser.add_argument('--env', '--environment', type=str, default='puffer_squared', help='Name of specific environment to run') parser.add_argument('--mode', type=str, default='train', - choices='train eval evaluate sweep autotune profile'.split()) - parser.add_argument('--eval-model-path', type=str, default=None, + choices='train eval sweep autotune profile'.split()) + parser.add_argument('--load-model-path', type=str, default=None, help='Path to a pretrained checkpoint') parser.add_argument('--baseline', action='store_true', help='Load pretrained model from WandB if available') parser.add_argument('--render-mode', type=str, default='auto', choices=['auto', 'human', 'ansi', 'rgb_array', 'raylib', 'None']) + parser.add_argument('--save-frames', type=int, default=0) + parser.add_argument('--gif-path', type=str, default='eval.gif') + parser.add_argument('--fps', type=float, default=15) parser.add_argument('--exp-id', '--exp-name', type=str, default=None, help='Resume from experiment') parser.add_argument('--max-runs', type=int, default=200, help='Max number of sweep runs') @@ -1077,8 +923,7 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin for path in glob.glob('config/**/*.ini', recursive=True): p = configparser.ConfigParser() p.read(['config/default.ini', path]) - if args.env in p['base']['env_name'].split(): - break + if args.env in p['base']['env_name'].split(): break else: raise pufferlib.APIUsageError('No config for env_name {}'.format(args.env)) @@ -1112,16 +957,13 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin except: breakpoint() - package = args['package'] - module_name = f'pufferlib.environments.{package}' - if package == 'ocean': - module_name = 'pufferlib.ocean' - + # Dynamically import environment and policy import importlib + package = args['package'] + module_name = 'pufferlib.ocean' if package == 'ocean' else f'pufferlib.environments.{package}' env_module = importlib.import_module(module_name) make_env = env_module.env_creator(env_name) policy_cls = getattr(env_module.torch, args['policy_name']) - rnn_name = args['rnn_name'] rnn_cls = None if rnn_name is not None: @@ -1131,6 +973,82 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin if 'LOCAL_RANK' in os.environ: torch.distributed.init_process_group(backend='nccl', rank=0, world_size=1) + if args['mode'] == 'sweep': + if not args['wandb'] and not args['neptune']: + raise pufferlib.APIUsageError('Sweeps require either wandb or neptune') + + method = args['sweep'].pop('method') + try: + sweep_cls = getattr(pufferlib.sweep, method) + except: + raise pufferlib.APIUsageError(f'Invalid sweep method {method}. See pufferlib.sweep') + + args['train']['use_rnn'] = rnn_cls is not None + env_name = args['env_name'] + exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8] + sweep = sweep_cls(args['sweep']) + target_metric = args['sweep']['metric'] + target_key = f'environment/{target_metric}' + min_sweep_steps = args['sweep']['train']['total_timesteps']['min'] + min_eval_points = 100 + for i in range(args['max_runs']): + seed = time.time_ns() & 0xFFFFFFFF + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + sweep.suggest(args) + + vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) + policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) + train_config = pufferlib.namespace(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) + pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) + + scores = [] + costs = [] + timesteps = [] + vecenv.async_reset(train_config.seed) + while pufferl.global_step < train_config.total_timesteps: + pufferl.evaluate() + logs = pufferl.train() + if logs is not None and target_key in logs and pufferl.global_step >= min_sweep_steps: + timesteps.append(logs['agent_steps']) + scores.append(logs[target_key]) + costs.append(pufferl.uptime) + + steps_evaluated = 0 + costs.append(time.time() - pufferl.start_time) + batch_size = args['train']['batch_size'] + timesteps.append(pufferl.global_step) + while len(pufferl.stats[target_metric]) < min_eval_points: + stats = pufferl.evaluate() + steps_evaluated += batch_size + + pufferl.mean_and_log() + scores.append(stats[target_metric]) + pufferl.close() + + scores = downsample_linear(scores, 10) + costs = downsample_linear(costs, 10) + timesteps = downsample_linear(timesteps, 10) + + # Hacky patch to prevent increasing total_timesteps when not swept + total_timesteps = args['train']['total_timesteps'] + for score, cost, timestep in zip(scores, costs, timesteps): + args['train']['total_timesteps'] = timestep + sweep.observe(args, score, cost) + + args['train']['total_timesteps'] = total_timesteps + + exit(0) + + if args['mode'] == 'autotune': + pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) + exit(0) + + args['train']['use_rnn'] = rnn_cls is not None + exp_id = args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8] + env_name = args['env_name'] + if args['baseline']: assert args['mode'] in ('train', 'eval', 'evaluate') args['track'] = True @@ -1145,37 +1063,79 @@ def train_wrap(args, make_env, policy_cls, rnn_cls, target_metric, min_eval_poin data_dir = artifact.download() model_file = max(os.listdir(data_dir)) args['eval_model_path'] = os.path.join(data_dir, model_file) - elif args['mode'] == 'train': - target_metric = args['sweep']['metric'] - train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) - elif args['mode'] in ('eval', 'evaluate'): - vec = pufferlib.vector.Serial - if args['vec'] == 'native': - vec = pufferlib.environment.PufferEnv - rollout( - make_env, - args['env'], - policy_cls=policy_cls, - rnn_cls=rnn_cls, - agent_creator=make_policy, - agent_kwargs=args, - backend=vec, - model_path=args['eval_model_path'], - render_mode=args['render_mode'], - device=args['train']['device'], + + if args['mode'] == 'eval': + args['vec'] = dict(backend='Serial', num_envs=1) + + vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) + policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) + + if args['load_model_path'] is not None: + policy.load_state_dict(torch.load( + args['load_model_path'], map_location=args['train']['device'])) + + if args['mode'] == 'train': + train_config = pufferlib.namespace(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) + pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) + + while pufferl.global_step < train_config.total_timesteps: + pufferl.evaluate() + logs = pufferl.train() + + vecenv.async_reset(train_config.seed) + for _ in range(10): + stats = pufferl.evaluate() + + pufferl.mean_and_log() + pufferl.close() + elif args['mode'] == 'eval': + ob, info = vecenv.reset() + driver = vecenv.driver_env + num_agents = vecenv.observation_space.shape[0] + state = pufferlib.namespace( + lstm_h=torch.zeros(num_agents, policy.hidden_size, device=args['train']['device']), + lstm_c=torch.zeros(num_agents, policy.hidden_size, device=args['train']['device']), ) - elif args['mode'] == 'sweep': - sweep(args, env_name, make_env, policy_cls, rnn_cls) - elif args['mode'] == 'autotune': - pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) + + frames = [] + while True: + render = driver.render() + if len(frames) < args['save_frames']: + frames.append(render) + + # TODO: Frames from raylib + if driver.render_mode == 'ansi': + print('\033[0;0H' + render + '\n') + time.sleep(1/args['fps']) + elif driver.render_mode == 'rgb_array': + import cv2 + render = cv2.cvtColor(render, cv2.COLOR_RGB2BGR) + cv2.imshow('frame', render) + cv2.waitKey(1) + time.sleep(1/args['fps']) + + with torch.no_grad(): + ob = torch.as_tensor(ob).to(args['train']['device']) + logits, value = policy(ob, state) + action, logprob, _ = pufferlib.pytorch.sample_logits( + logits, is_continuous=policy.is_continuous) + action = action.cpu().numpy().reshape(vecenv.action_space.shape) + + ob = vecenv.step(action)[0] + + if len(frames) > 0 and len(frames) == args['save_frames']: + import imageio + imageio.mimsave(args['gif_path'], frames, fps=args['fps'], loop=0) + frames.append('Done') elif args['mode'] == 'profile': import torch import torchvision.models as models from torch.profiler import profile, record_function, ProfilerActivity with profile(activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], record_shapes=True) as prof: with record_function("model_inference"): - target_metric = args['sweep']['metric'] - train_wrap(args, make_env, policy_cls, rnn_cls, target_metric) + for _ in range(10): + stats = pufferl.evaluate() + pufferl.train() print(prof.key_averages().table(sort_by='cuda_time_total', row_limit=10)) prof.export_chrome_trace("trace.json") From adec1c0bea93f5c5f4617f1cba49e321a3791aeb Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Thu, 8 May 2025 20:07:49 +0000 Subject: [PATCH 43/63] trade sim config --- config/trade_sim.ini | 19 +++++++------------ .../environments/trade_sim/environment.py | 2 +- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/config/trade_sim.ini b/config/trade_sim.ini index f31f37fc5b..2a7f5b4054 100644 --- a/config/trade_sim.ini +++ b/config/trade_sim.ini @@ -3,29 +3,24 @@ package = trade_sim env_name = trade_sim policy_name = Policy rnn_name = Recurrent -vec = multiprocessing + +[vec] +backend = Multiprocessing +num_envs = 1024 +num_workers = 16 +batch_size = 512 #[env] #num_envs = 128 [train] total_timesteps = 100_000_000 -num_envs = 1024 -num_workers = 16 -env_batch_size = 512 gamma = 0.95 learning_rate = 0.05 minibatch_size = 32768 [sweep] -method = protein -name = sweep - -[sweep.metric] -goal = maximize -name = final_capital -min = 0 -max = 20000 +metric = final_capital [sweep.train.total_timesteps] distribution = log_normal diff --git a/pufferlib/environments/trade_sim/environment.py b/pufferlib/environments/trade_sim/environment.py index af47073e42..0c245c0b25 100644 --- a/pufferlib/environments/trade_sim/environment.py +++ b/pufferlib/environments/trade_sim/environment.py @@ -8,7 +8,7 @@ def env_creator(name='metta'): return functools.partial(make, name) -def make(name, config_path='../nof1-trading-sim/config/experiment_config_3.yaml', render_mode='human', buf=None, seed=1): +def make(name, config_path='../nof1-trading-sim/config/experiment_cv.yaml', render_mode='human', buf=None, seed=1): '''Crafter creation function''' from nof1.utils.config_manager import ConfigManager from nof1.data_ingestion.historical_data_reader import HistoricalDataReader From 24b70930f83016241662825e9b32c3c0683ff97d Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Thu, 8 May 2025 20:37:21 +0000 Subject: [PATCH 44/63] Remove puffer namespace from clean_pufferl --- clean_pufferl.py | 313 +++++++++++++++++++++++--------------------- pufferlib/models.py | 20 +-- 2 files changed, 172 insertions(+), 161 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index b068477dd5..268ff36051 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -53,29 +53,30 @@ class CleanPuffeRL: def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Backend perf optimization torch.set_float32_matmul_precision('high') - torch.backends.cudnn.deterministic = config.torch_deterministic + torch.backends.cudnn.deterministic = config['torch_deterministic'] torch.backends.cudnn.benchmark = True # Reproducibility - random.seed(config.seed) - np.random.seed(config.seed) - torch.manual_seed(config.seed) + seed = config['seed'] + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) # Vecenv info - vecenv.async_reset(config.seed) + vecenv.async_reset(seed) obs_space = vecenv.single_observation_space atn_space = vecenv.single_action_space total_agents = vecenv.num_agents self.total_agents = total_agents # Experience buffer - device = config.device - batch_size = config.batch_size + device = config['device'] + batch_size = config['batch_size'] - if config.bptt_horizon == 'auto': - config.bptt_horizon = batch_size // total_agents + if config['bptt_horizon'] == 'auto': + config['bptt_horizon'] = batch_size // total_agents - horizon = config.bptt_horizon + horizon = config['bptt_horizon'] segments = batch_size // horizon self.segments = segments if total_agents > segments: @@ -87,11 +88,11 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.ep_lengths = torch.zeros(total_agents, device=device, dtype=torch.int32) self.ep_indices = torch.arange(total_agents, device=device, dtype=torch.int32) self.free_idx = total_agents - experience = pufferlib.namespace( + experience = dict( obs=torch.zeros(segments, horizon, *obs_space.shape, dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[obs_space.dtype], - pin_memory=device == 'cuda' and config.cpu_offload, - device='cpu' if config.cpu_offload else device), + pin_memory=device == 'cuda' and config['cpu_offload'], + device='cpu' if config['cpu_offload'] else device), actions=torch.zeros(segments, horizon, *atn_space.shape, dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[atn_space.dtype], device=device), values = torch.zeros(segments, horizon, device=device), @@ -103,22 +104,22 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): ) self.experience = experience - if config.use_vtrace or config.use_puff_advantage: - experience.importance = torch.ones(segments, horizon, device=device) + if config['use_vtrace'] or config['use_puff_advantage']: + experience['importance'] = torch.ones(segments, horizon, device=device) # LSTM # TODO: This breaks compile - if config.use_rnn: + if config['use_rnn']: # TODO: Doesn't exist in native envs # TODO: Replace slice with env idx or similar n = vecenv.agents_per_batch - self.lstm_h = {i*n: torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} - self.lstm_c = {i*n: torch.zeros(n, policy.hidden_size, device=config.device) for i in range(total_agents//n)} + self.lstm_h = {i*n: torch.zeros(n, policy.hidden_size, device=device) for i in range(total_agents//n)} + self.lstm_c = {i*n: torch.zeros(n, policy.hidden_size, device=device) for i in range(total_agents//n)} # Minibatching & gradient accumulation - minibatch_size = config.minibatch_size - max_minibatch_size = config.max_minibatch_size + minibatch_size = config['minibatch_size'] + max_minibatch_size = config['max_minibatch_size'] self.minibatch_size = min(minibatch_size, max_minibatch_size) if minibatch_size % max_minibatch_size != 0 and max_minibatch_size % minibatch_size != 0: # todo: better error @@ -126,8 +127,8 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' ) - self.accumulate_minibatches = max(1, config.minibatch_size // config.max_minibatch_size) - self.total_minibatches = int(config.update_epochs * batch_size / self.minibatch_size) + self.accumulate_minibatches = max(1, config['minibatch_size'] // config['max_minibatch_size']) + self.total_minibatches = int(config['update_epochs'] * batch_size / self.minibatch_size) self.minibatch_segments = self.minibatch_size // horizon if self.minibatch_segments * horizon != self.minibatch_size: raise pufferlib.APIUsageError( @@ -136,59 +137,62 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Torch compile self.uncompiled_policy = policy - if config.compile: - policy = torch.compile(policy, mode=config.compile_mode, fullgraph=config.compile_fullgraph) + if config['compile']: + policy = torch.compile(policy, mode=config['compile_mode'], fullgraph=config['compile_fullgraph']) self.policy = policy # Optimizer - if config.optimizer == 'adam': + # TODO: **optim params + if config['optimizer'] == 'adam': optimizer = torch.optim.Adam( policy.parameters(), - lr=config.learning_rate, - betas=(config.adam_beta1, config.adam_beta2), - eps=config.adam_eps, + lr=config['learning_rate'], + betas=(config['adam_beta1'], config['adam_beta2']), + eps=config['adam_eps'], ) - elif config.optimizer == 'muon': + elif config['optimizer'] == 'muon': from heavyball import ForeachMuon import heavyball.utils - heavyball.utils.compile_mode = config.compile_mode if config.compile else None + heavyball.utils.compile_mode = config['compile_mode'] if config['compile'] else None optimizer = ForeachMuon( policy.parameters(), - lr=config.learning_rate, - betas=(config.adam_beta1, config.adam_beta2), - eps=config.adam_eps, + lr=config['learning_rate'], + betas=(config['adam_beta1'], config['adam_beta2']), + eps=config['adam_eps'], ) else: - raise ValueError(f'Unknown optimizer: {config.optimizer}') + raise ValueError(f'Unknown optimizer: {config["optimizer"]}') self.optimizer = optimizer # Learning rate scheduler - epochs = config.total_timesteps // config.batch_size + epochs = config['total_timesteps'] // config['batch_size'] self.total_epochs = epochs - assert config.scheduler in ('linear', 'cosine') - if config.scheduler == 'linear': + if config['scheduler'] == 'linear': scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=0.0, total_iters=epochs) - elif config.scheduler == 'cosine': + elif config['scheduler'] == 'cosine': scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) + else: + raise pufferlib.APIUsageError(f'Unknown scheduler: {config["scheduler"]}') self.scheduler = scheduler # Automatic mixed precision - self.amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, config.precision)) - if config.precision not in ('float32', 'bfloat16'): - raise pufferlib.APIUsageError(f'Use float32 or bfloat16, not {config.precision}') + precision = config['precision'] + self.amp_context = torch.amp.autocast(device_type='cuda', dtype=getattr(torch, precision)) + if precision not in ('float32', 'bfloat16'): + raise pufferlib.APIUsageError(f'Invalid precision: {precision}: use float32 or bfloat16') # Logging self.neptune = neptune self.wandb = wandb if neptune: - self.neptune = init_neptune(args, env_name, id=config.run_id, tag=config.tag) + self.neptune = init_neptune(args, env_name, id=config['run_id'], tag=config['tag']) for k, v in pufferlib.unroll_nested_dict(args): self.neptune[k].append(v) elif wandb: - self.wandb = init_wandb(args, env_name, id=config.run_id, tag=config.tag) + self.wandb = init_wandb(args, env_name, id=config['run_id'], tag=config['tag']) # Profiling self.uptime = 0 @@ -220,6 +224,7 @@ def evaluate(self): config = self.config experience = self.experience policy = self.policy + device = config['device'] self.full_rows = 0 while self.full_rows < self.segments: @@ -236,22 +241,22 @@ def evaluate(self): profile('eval_copy', epoch) o = torch.as_tensor(o) - o_device = o.to(config.device, non_blocking=True) - r = torch.as_tensor(r).to(config.device, non_blocking=True) - d = torch.as_tensor(d).to(config.device, non_blocking=True) + o_device = o.to(device, non_blocking=True) + r = torch.as_tensor(r).to(device, non_blocking=True) + d = torch.as_tensor(d).to(device, non_blocking=True) profile('eval_forward', epoch) with torch.no_grad(), self.amp_context: - state = pufferlib.namespace( + state = dict( reward=r, done=d, env_id=env_id, mask=mask, ) - if config.use_rnn: - state.lstm_h = self.lstm_h[env_id.start] - state.lstm_c = self.lstm_c[env_id.start] + if config['use_rnn']: + state['lstm_h'] = self.lstm_h[env_id.start] + state['lstm_c'] = self.lstm_c[env_id.start] logits, value = policy(o_device, state) action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=policy.is_continuous) @@ -259,11 +264,11 @@ def evaluate(self): profile('eval_copy', epoch) with torch.no_grad(): - if config.use_rnn: - self.lstm_h[env_id.start] = state.lstm_h - self.lstm_c[env_id.start] = state.lstm_c + if config['use_rnn']: + self.lstm_h[env_id.start] = state['lstm_h'] + self.lstm_c[env_id.start] = state['lstm_c'] - o = o if config.cpu_offload else o_device + o = o if config['cpu_offload'] else o_device actions = self.store(state, o, value, action, logprob, r, d, env_id, mask) profile('eval_misc', epoch) @@ -281,7 +286,7 @@ def evaluate(self): profile('eval_misc', epoch) self.free_idx = self.total_agents - self.ep_indices = torch.arange(self.total_agents, device=config.device, dtype=torch.int32) + self.ep_indices = torch.arange(self.total_agents, device=device, dtype=torch.int32) self.ep_lengths.zero_() self.ep_uses.zero_() profile.end() @@ -295,95 +300,98 @@ def train(self): config = self.config experience = self.experience losses = defaultdict(float) + device = config['device'] for mb in range(self.total_minibatches): profile('train_misc', epoch, nest=True) self.amp_context.__enter__() loss = 0 - if config.use_vtrace: - importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) - vs = torch.zeros(experience.values.shape, device=config.device) - self.compute_vtrace(experience.values, experience.rewards, experience.dones, - experience.ratio, vs, advantages, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) - elif config.use_puff_advantage: - importance = advantages = torch.zeros(experience.values.shape, device=config.device).to(config.device) - vs = torch.zeros(experience.values.shape, device=config.device) + shape = experience['values'].shape + if config['use_vtrace']: + importance = advantages = torch.zeros(shape, device=device) + vs = torch.zeros(shape, device=device) + self.compute_vtrace(experience['values'], experience['rewards'], + experience['dones'], experience['ratio'], vs, advantages, + config['gamma'], config['vtrace_rho_clip'], config['vtrace_c_clip']) + elif config['use_puff_advantage']: + importance = advantages = torch.zeros(shape, device=device) + vs = torch.zeros(shape, device=device) # TODO: Eliminate - n = (experience.values.shape[0]//256)*256 - torch.ops.pufferlib.compute_puff_advantage(experience.values[:n], experience.rewards[:n], - experience.dones[:n], experience.ratio[:n], vs[:n], advantages[:n], config.gamma, - config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) + n = (shape[0]//256)*256 + torch.ops.pufferlib.compute_puff_advantage(experience['values'][:n], experience['rewards'][:n], + experience['dones'][:n], experience['ratio'][:n], vs[:n], advantages[:n], config['gamma'], + config['gae_lambda'], config['vtrace_rho_clip'], config['vtrace_c_clip']) else: - importance = advantages = self.compute_gae(experience.values, experience.rewards, - experience.dones, config.gamma, config.gae_lambda) + importance = advantages = self.compute_gae(experience['values'], experience['rewards'], + experience['dones'], config['gamma'], config['gae_lambda']) profile('train_copy', epoch) batch = self.sample(importance, self.minibatch_segments) profile('train_forward', epoch) - state = pufferlib.namespace( - action=batch.actions, + state = dict( + action=batch['actions'], lstm_h=None, lstm_c=None, ) - if not config.use_rnn: - batch.obs = batch.obs.reshape(-1, *self.vecenv.single_observation_space.shape) + if not config['use_rnn']: + batch['obs'] = batch['obs'].reshape(-1, *self.vecenv.single_observation_space.shape) # TODO: Currently only returning traj shaped value as a hack - logits, newvalue = self.policy.forward_train(batch.obs, state) + logits, newvalue = self.policy.forward_train(batch['obs'], state) actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, - action=batch.actions, is_continuous=self.policy.is_continuous) + action=batch['actions'], is_continuous=self.policy.is_continuous) profile('train_misc', epoch) - newlogprob = newlogprob.reshape(batch.logprobs.shape) - logratio = newlogprob - batch.logprobs + newlogprob = newlogprob.reshape(batch['logprobs'].shape) + logratio = newlogprob - batch['logprobs'] ratio = logratio.exp() - experience.ratio[batch.idx] = ratio # TODO: Experiment with this + experience['ratio'][batch['idx']] = ratio # TODO: Experiment with this # TODO: Only do this if we are KL clipping? Saves 1-2% compute with torch.no_grad(): # calculate approx_kl http://joschu.net/blog/kl-approx.html old_approx_kl = (-logratio).mean() approx_kl = ((ratio - 1) - logratio).mean() - clipfrac = ((ratio - 1.0).abs() > config.clip_coef).float().mean() + clipfrac = ((ratio - 1.0).abs() > config['clip_coef']).float().mean() # TODO: Do you need to do this? Policy hasn't changed - if config.use_vtrace or config.use_puff_advantage: + if config['use_vtrace'] or config['use_puff_advantage']: with torch.no_grad(): - adv = advantages[batch.idx] - vs = vs[batch.idx] - if config.use_vtrace: - self.compute_vtrace(batch.values, batch.rewards, batch.dones, - ratio, vs, adv, config.gamma, config.vtrace_rho_clip, config.vtrace_c_clip) - elif config.use_puff_advantage: - torch.ops.pufferlib.compute_puff_advantage(batch.values, batch.rewards, batch.dones, - ratio, vs, adv, config.gamma, config.gae_lambda, config.vtrace_rho_clip, config.vtrace_c_clip) - - adv = batch.advantages + adv = advantages[batch['idx']] + vs = vs[batch['idx']] + if config['use_vtrace']: + self.compute_vtrace(batch['values'], batch['rewards'], batch['dones'], + ratio, vs, adv, config['gamma'], config['vtrace_rho_clip'], config['vtrace_c_clip']) + elif config['use_puff_advantage']: + torch.ops.pufferlib.compute_puff_advantage(batch['values'], batch['rewards'], batch['dones'], + ratio, vs, adv, config['gamma'], config['gae_lambda'], config['vtrace_rho_clip'], config['vtrace_c_clip']) + + adv = batch['advantages'] adv = (adv - adv.mean()) / (adv.std() + 1e-8) # Prioritized replay - adv = adv * batch.prio + adv = adv * batch['prio'] # Policy loss pg_loss1 = -adv * ratio pg_loss2 = -adv * torch.clamp( - ratio, 1 - config.clip_coef, 1 + config.clip_coef + ratio, 1 - config['clip_coef'], 1 + config['clip_coef'] ) pg_loss = torch.max(pg_loss1, pg_loss2).mean() # Value loss - ret = batch.returns + ret = batch['returns'] newvalue = newvalue.view(ret.shape) v_loss_unclipped = (newvalue - ret) ** 2 - val = batch.values + val = batch['values'] v_clipped = val + torch.clamp( newvalue - val, - -config.vf_clip_coef, - config.vf_clip_coef, + -config['vf_clip_coef'], + config['vf_clip_coef'], ) v_loss_clipped = (v_clipped - ret) ** 2 v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) @@ -393,18 +401,18 @@ def train(self): entropy_loss = entropy.mean() # Total loss - loss += pg_loss - config.ent_coef*entropy_loss + v_loss*config.vf_coef + loss += pg_loss - config['ent_coef']*entropy_loss + v_loss*config['vf_coef'] self.amp_context.__enter__() # This breaks vloss clipping? with torch.no_grad(): - experience.values[batch.idx] = newvalue.float() + experience['values'][batch['idx']] = newvalue.float() profile('learn', epoch) loss.backward() if (mb + 1) % self.accumulate_minibatches == 0: - torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config.max_grad_norm) + torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config['max_grad_norm']) self.optimizer.step() self.optimizer.zero_grad() @@ -421,14 +429,14 @@ def train(self): profile('train_misc', epoch) self.max_uses = self.ep_uses.max().item() self.mean_uses = self.ep_uses.float().mean().item() - experience.ratio[:] = 1 + experience['ratio'][:] = 1 - if config.anneal_lr: + if config['anneal_lr']: self.scheduler.step() - y_pred = experience.values.flatten() + y_pred = experience['values'].flatten() # TODO: Probably not updated - y_true = advantages.flatten() + experience.values.flatten() + y_true = advantages.flatten() + experience['values'].flatten() var_y = y_true.var() explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y @@ -438,7 +446,7 @@ def train(self): profile.clear() logs = None self.epoch += 1 - done_training = self.global_step >= config.total_timesteps + done_training = self.global_step >= config['total_timesteps'] if done_training or self.global_step == 0 or time.time() - self.start_time - self.uptime > 1: self.uptime = time.time() - self.start_time logs = self.mean_and_log() @@ -446,7 +454,7 @@ def train(self): self.print_dashboard() self.stats = defaultdict(list) - if self.epoch % config.checkpoint_interval == 0 or done_training: + if self.epoch % config['checkpoint_interval'] == 0 or done_training: self.save_checkpoint() self.msg = f'Checkpoint saved at update {self.epoch}' @@ -460,20 +468,20 @@ def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): l = self.ep_lengths[env_id.start].item() batch_rows = slice(self.ep_indices[env_id.start].item(), 1+self.ep_indices[env_id.stop - 1].item()) - exp.obs[batch_rows, l] = obs - exp.actions[batch_rows, l] = action - exp.logprobs[batch_rows, l] = logprob - exp.rewards[batch_rows, l] = reward - exp.dones[batch_rows, l] = done.float() - exp.values[batch_rows, l] = value.flatten() + exp['obs'][batch_rows, l] = obs + exp['actions'][batch_rows, l] = action + exp['logprobs'][batch_rows, l] = logprob + exp['rewards'][batch_rows, l] = reward + exp['dones'][batch_rows, l] = done.float() + exp['values'][batch_rows, l] = value.flatten() # TODO: Handle masks!! #indices = np.where(mask)[0] #data.ep_lengths[env_id[mask]] += 1 self.ep_lengths[env_id] += 1 - if l+1 >= config.bptt_horizon: + if l+1 >= config['bptt_horizon']: num_full = env_id.stop - env_id.start - self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config.device).int() + self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config['device']).int() self.ep_lengths[env_id] = 0 self.free_idx += num_full self.full_rows += num_full @@ -487,30 +495,32 @@ def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio _, idx = torch.topk(advantages.abs().sum(axis=1), n) elif method == 'prio': adv = advantages.abs().sum(axis=1) - probs = adv**config.prio_alpha + probs = adv**config['prio_alpha'] probs = torch.nan_to_num(probs, 0, 0, 0) probs = (probs + 1e-6)/(probs.sum() + 1e-6) idx = torch.multinomial(probs, n) elif method == 'multinomial': idx = torch.multinomial(advantages.abs().sum(axis=1) + 1e-6, n) elif method == 'random': - idx = torch.randint(0, advantages.shape[0], (n,), device=self.device) + idx = torch.randint(0, advantages.shape[0], (n,), device=config['device']) else: raise ValueError(f'Unknown sampling method: {method}') self.ep_uses[idx] += 1 output = {k: v[idx] for k, v in exp.items()} output['idx'] = idx - output['values'] = exp.values[idx] + output['values'] = exp['values'][idx] output['advantages'] = advantages[idx] - output['returns'] = advantages[idx] + exp.values[idx] + output['returns'] = advantages[idx] + exp['values'][idx] output['prio'] = 1 if method == 'prio': - beta = config.prio_beta0 + (1 - config.prio_beta0)*config.prio_alpha*self.epoch/self.total_epochs + b0 = config['prio_beta0'] + a = config['prio_alpha'] + beta = b0 + (1 - b0)*a*self.epoch/self.total_epochs output['prio'] = (((1/len(probs)) * (1/probs[idx]))**beta).unsqueeze(1).expand_as(output['advantages']) - return pufferlib.namespace(**output) + return dict(**output) def mean_and_log(self): config = self.config @@ -523,8 +533,7 @@ def mean_and_log(self): self.stats[k] = v - device = config.device - + device = config['device'] agent_steps = int(dist_sum(self.global_step, device)) logs = { #'SPS': dist_sum(self.profile.SPS, device), @@ -535,7 +544,7 @@ def mean_and_log(self): 'mean_uses': self.mean_uses, **{f'environment/{k}': dist_mean(v, device) for k, v in self.stats.items()}, **{f'losses/{k}': dist_mean(v, device) for k, v in self.losses.items()}, - **{f'performance/{k}': dist_sum(v.elapsed, device) for k, v in self.profile}, + **{f'performance/{k}': dist_sum(v['elapsed'], device) for k, v in self.profile}, } if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: @@ -554,8 +563,9 @@ def close(self): self.utilization.stop() config = self.config if self.wandb: - artifact_name = f"{config.exp_id}_model" - artifact = self.wandb.Artifact(artifact_name, type="model") + exp_id = config['exp_id'] + artifact_name = f'{exp_id}_model' + artifact = self.wandb.Artifact(artifact_name, type='model') model_path = self.save_checkpoint(self) artifact.add_file(model_path) self.wandb.run.log_artifact(artifact) @@ -566,7 +576,8 @@ def close(self): def save_checkpoint(self): config = self.config - path = os.path.join(config.data_dir, config.exp_id) + exp_id = config['exp_id'] + path = os.path.join(config['data_dir'], exp_id) if not os.path.exists(path): os.makedirs(path) @@ -583,7 +594,7 @@ def save_checkpoint(self): 'agent_step': self.global_step, 'update': self.epoch, 'model_name': model_name, - 'exp_id': config.exp_id, + 'exp_id': exp_id, } state_path = os.path.join(path, 'trainer_state.pt') torch.save(state, state_path + '.tmp') @@ -592,7 +603,7 @@ def save_checkpoint(self): def try_load_checkpoint(self): config = self.config - path = os.path.join(config.data_dir, config.exp_id) + path = os.path.join(config['data_dir'], config['exp_id']) if not os.path.exists(path): print('No checkpoints found. Assuming new experiment') return @@ -601,7 +612,7 @@ def try_load_checkpoint(self): resume_state = torch.load(trainer_path, weights_only=False) model_path = os.path.join(path, resume_state['model_name']) self.policy.uncompiled.load_state_dict( - torch.load(model_path, weights_only=True), map_location=config.device) + torch.load(model_path, weights_only=True), map_location=config['device']) self.optimizer.load_state_dict(resume_state['optimizer_state_dict']) print(f'Loaded checkpoint {resume_state["model_name"]}') @@ -637,16 +648,16 @@ def print_dashboard(self, clear=False, max_stats=[0]): s = Table(box=None, expand=True) SPS = 0 - delta = profile.eval.delta + profile.train.delta + delta = profile.eval['delta'] + profile.train['delta'] remaining = 'A hair past a freckle' if delta != 0: - SPS = config.batch_size/delta - remaining = duration((config.total_timesteps - self.global_step)/SPS) + SPS = config['batch_size'] / delta + remaining = duration((config['total_timesteps'] - self.global_step)/SPS) uptime = time.time() - self.start_time s.add_column(f"{c1}Summary", justify='left', vertical='top', width=10) s.add_column(f"{c1}Value", justify='right', vertical='top', width=14) - s.add_row(f'{c2}Env', f'{b2}{config.env}') + s.add_row(f'{c2}Env', f'{b2}{config["env"]}') s.add_row(f'{c2}Steps', abbreviate(self.global_step)) s.add_row(f'{c2}SPS', abbreviate(SPS)) s.add_row(f'{c2}Epoch', abbreviate(self.epoch)) @@ -736,7 +747,7 @@ def __init__(self, keys, frequency=1): self.stack = [] self.frequency = frequency self.profiles = {k: - pufferlib.namespace( + dict( start = 0, buffer = 0, delta = 0, @@ -762,14 +773,14 @@ def __call__(self, name, epoch, nest=False): self.pop(tick) self.stack.append(name) - self.profiles[name].start = tick + self.profiles[name]['start'] = tick def pop(self, end): profile = self.profiles[self.stack.pop()] - delta = end - profile.start - profile.buffer += delta - profile.elapsed += delta - profile.calls += 1 + delta = end - profile['start'] + profile['buffer'] += delta + profile['elapsed'] += delta + profile['calls'] += 1 def end(self): torch.cuda.synchronize() @@ -780,10 +791,10 @@ def end(self): def clear(self): for v in self.profiles.values(): - if v.buffer != 0: - v.delta = v.buffer + if v['buffer'] != 0: + v['delta'] = v['buffer'] - v.buffer = 0 + v['buffer'] = 0 class Utilization(Thread): def __init__(self, delay=1, maxlen=20): @@ -834,8 +845,8 @@ def duration(seconds): return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" def fmt_perf(name, color, delta_ref, prof): - percent = 0 if delta_ref == 0 else int(100*prof.delta/delta_ref - 1e-5) - return f'{color}{name}', duration(prof.elapsed), f'{b2}{percent:2d}{c2}%' + percent = 0 if delta_ref == 0 else int(100*prof['delta']/delta_ref - 1e-5) + return f'{color}{name}', duration(prof['elapsed']), f'{b2}{percent:2d}{c2}%' def init_wandb(args, name, id=None, resume=True, tag=None): @@ -943,7 +954,7 @@ def downsample_linear(arr, m): # Unpack to nested dict parsed = vars(parser.parse_args()) - nested = lambda: defaultdict(nested) # TODO: Replace with pufferlib namespace + nested = lambda: defaultdict(nested) # TODO: Replace with dict args = nested() env_name = parsed.pop('env') for key, value in parsed.items(): @@ -1000,14 +1011,14 @@ def downsample_linear(arr, m): vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) - train_config = pufferlib.namespace(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) + train_config = dict(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) scores = [] costs = [] timesteps = [] - vecenv.async_reset(train_config.seed) - while pufferl.global_step < train_config.total_timesteps: + vecenv.async_reset(train_config['seed']) + while pufferl.global_step < train_config['total_timesteps']: pufferl.evaluate() logs = pufferl.train() if logs is not None and target_key in logs and pufferl.global_step >= min_sweep_steps: @@ -1075,14 +1086,14 @@ def downsample_linear(arr, m): args['load_model_path'], map_location=args['train']['device'])) if args['mode'] == 'train': - train_config = pufferlib.namespace(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) + train_config = dict(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) - while pufferl.global_step < train_config.total_timesteps: + while pufferl.global_step < train_config['total_timesteps']: pufferl.evaluate() logs = pufferl.train() - vecenv.async_reset(train_config.seed) + vecenv.async_reset(train_config['seed']) for _ in range(10): stats = pufferl.evaluate() @@ -1092,7 +1103,7 @@ def downsample_linear(arr, m): ob, info = vecenv.reset() driver = vecenv.driver_env num_agents = vecenv.observation_space.shape[0] - state = pufferlib.namespace( + state = dict( lstm_h=torch.zeros(num_agents, policy.hidden_size, device=args['train']['device']), lstm_c=torch.zeros(num_agents, policy.hidden_size, device=args['train']['device']), ) diff --git a/pufferlib/models.py b/pufferlib/models.py index 7e71079d52..845cdd00fc 100644 --- a/pufferlib/models.py +++ b/pufferlib/models.py @@ -161,8 +161,8 @@ def __init__(self, env, policy, input_size=128, hidden_size=128): def forward(self, observations, state): '''Forward function for inference. 3x faster than using LSTM directly''' hidden = self.policy.encode_observations(observations, state=state) - h = state.lstm_h - c = state.lstm_c + h = state['lstm_h'] + c = state['lstm_c'] # TODO: Don't break compile if h is not None: @@ -174,17 +174,17 @@ def forward(self, observations, state): #hidden = self.pre_layernorm(hidden) hidden, c = self.cell(hidden, lstm_state) #hidden = self.post_layernorm(hidden) - state.hidden = hidden - state.lstm_h = hidden - state.lstm_c = c + state['hidden'] = hidden + state['lstm_h'] = hidden + state['lstm_c'] = c logits, values = self.policy.decode_actions(hidden) return logits, values def forward_train(self, observations, state): '''Forward function for training. Uses LSTM for fast time-batching''' x = observations - lstm_h = state.lstm_h - lstm_c = state.lstm_c + lstm_h = state['lstm_h'] + lstm_c = state['lstm_c'] x_shape, space_shape = x.shape, self.obs_shape x_n, space_n = len(x_shape), len(space_shape) @@ -220,9 +220,9 @@ def forward_train(self, observations, state): logits, values = self.policy.decode_actions(flat_hidden) values = values.reshape(B, TT) #state.batch_logits = logits.reshape(B, TT, -1) - state.hidden = hidden - state.lstm_h = lstm_h.detach() - state.lstm_c = lstm_c.detach() + state['hidden'] = hidden + state['lstm_h'] = lstm_h.detach() + state['lstm_c'] = lstm_c.detach() return logits, values class Convolutional(nn.Module): From d037525c2c6e8045a49fa52f1c54843c073f160c Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 9 May 2025 15:48:24 +0000 Subject: [PATCH 45/63] neptune/wandb model save/load --- clean_pufferl.py | 163 ++++++++++++++++++++++------------------------- 1 file changed, 77 insertions(+), 86 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 268ff36051..4e128fb235 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -538,6 +538,7 @@ def mean_and_log(self): logs = { #'SPS': dist_sum(self.profile.SPS, device), 'agent_steps': agent_steps, + 'uptime': time.time() - self.start_time, 'epoch': int(dist_sum(self.epoch, device)), 'learning_rate': self.optimizer.param_groups[0]["lr"], 'max_uses': self.max_uses, @@ -562,16 +563,18 @@ def close(self): self.vecenv.close() self.utilization.stop() config = self.config + model_path = self.save_checkpoint() if self.wandb: exp_id = config['exp_id'] artifact_name = f'{exp_id}_model' artifact = self.wandb.Artifact(artifact_name, type='model') - model_path = self.save_checkpoint(self) artifact.add_file(model_path) self.wandb.run.log_artifact(artifact) self.wandb.finish() elif self.neptune: - # TODO: Add artifact + model = torch.load(model_path) + torch.save(model, '/tmp/model.pt') + self.neptune['model'].track_files('/tmp/model.pt') self.neptune.stop() def save_checkpoint(self): @@ -898,7 +901,29 @@ def downsample_linear(arr, m): x_old = np.linspace(0, 1, n) # Original indices normalized x_new = np.linspace(0, 1, m) # New indices normalized return np.interp(x_new, x_old, arr) - + +def experiment(vecenv, policy, args): + train_config = dict(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) + pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) + + all_logs = [] + while pufferl.global_step < train_config['total_timesteps']: + pufferl.evaluate() + logs = pufferl.train() + if logs is not None: + all_logs.append(logs) + + vecenv.async_reset(train_config['seed']) + for _ in range(10): + stats = pufferl.evaluate() + + logs = pufferl.mean_and_log() + if logs is not None: + all_logs.append(logs) + + pufferl.close() + return all_logs + if __name__ == '__main__': parser = argparse.ArgumentParser( description=f':blowfish: PufferLib [bright_cyan]{pufferlib.__version__}[/]' @@ -954,19 +979,15 @@ def downsample_linear(arr, m): # Unpack to nested dict parsed = vars(parser.parse_args()) - nested = lambda: defaultdict(nested) # TODO: Replace with dict - args = nested() env_name = parsed.pop('env') + args = {} for key, value in parsed.items(): next = args - split = key.split('.') - for subkey in split[:-1]: - next = next[subkey] + for subkey in key.split('.'): + prev = next + next = next.setdefault(subkey, {}) - try: - next[split[-1]] = value - except: - breakpoint() + prev[subkey] = value # Dynamically import environment and policy import importlib @@ -984,6 +1005,14 @@ def downsample_linear(arr, m): if 'LOCAL_RANK' in os.environ: torch.distributed.init_process_group(backend='nccl', rank=0, world_size=1) + if args['mode'] == 'autotune': + pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) + exit(0) + + args['train']['use_rnn'] = rnn_cls is not None + exp_id = args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8] + env_name = args['env_name'] + if args['mode'] == 'sweep': if not args['wandb'] and not args['neptune']: raise pufferlib.APIUsageError('Sweeps require either wandb or neptune') @@ -994,14 +1023,9 @@ def downsample_linear(arr, m): except: raise pufferlib.APIUsageError(f'Invalid sweep method {method}. See pufferlib.sweep') - args['train']['use_rnn'] = rnn_cls is not None - env_name = args['env_name'] - exp_id=args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8] sweep = sweep_cls(args['sweep']) - target_metric = args['sweep']['metric'] - target_key = f'environment/{target_metric}' - min_sweep_steps = args['sweep']['train']['total_timesteps']['min'] - min_eval_points = 100 + target_key = f'environment/{args["sweep"]["metric"]}' + total_timesteps = args['train']['total_timesteps'] for i in range(args['max_runs']): seed = time.time_ns() & 0xFFFFFFFF random.seed(seed) @@ -1011,94 +1035,61 @@ def downsample_linear(arr, m): vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) - train_config = dict(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) - pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) - - scores = [] - costs = [] - timesteps = [] - vecenv.async_reset(train_config['seed']) - while pufferl.global_step < train_config['total_timesteps']: - pufferl.evaluate() - logs = pufferl.train() - if logs is not None and target_key in logs and pufferl.global_step >= min_sweep_steps: - timesteps.append(logs['agent_steps']) - scores.append(logs[target_key]) - costs.append(pufferl.uptime) - - steps_evaluated = 0 - costs.append(time.time() - pufferl.start_time) - batch_size = args['train']['batch_size'] - timesteps.append(pufferl.global_step) - while len(pufferl.stats[target_metric]) < min_eval_points: - stats = pufferl.evaluate() - steps_evaluated += batch_size - - pufferl.mean_and_log() - scores.append(stats[target_metric]) - pufferl.close() - - scores = downsample_linear(scores, 10) - costs = downsample_linear(costs, 10) - timesteps = downsample_linear(timesteps, 10) - - # Hacky patch to prevent increasing total_timesteps when not swept - total_timesteps = args['train']['total_timesteps'] + all_logs = experiment(vecenv, policy, args) + + scores = downsample_linear([log[target_key] for log in all_logs], 10) + costs = downsample_linear([log['uptime'] for log in all_logs], 10) + timesteps = downsample_linear([log['agent_steps'] for log in all_logs], 10) + for score, cost, timestep in zip(scores, costs, timesteps): args['train']['total_timesteps'] = timestep sweep.observe(args, score, cost) + # Prevent logging final eval steps as training steps args['train']['total_timesteps'] = total_timesteps exit(0) - if args['mode'] == 'autotune': - pufferlib.vector.autotune(make_env, batch_size=args['train']['env_batch_size']) - exit(0) - - args['train']['use_rnn'] = rnn_cls is not None - exp_id = args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8] - env_name = args['env_name'] + if args['mode'] == 'eval': + args['vec'] = dict(backend='Serial', num_envs=1) + + vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) + policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) - if args['baseline']: + run_id = args['exp_id'] + if run_id is not None: assert args['mode'] in ('train', 'eval', 'evaluate') - args['track'] = True - version = '.'.join(pufferlib.__version__.split('.')[:2]) - args['exp_id'] = f'puf-{version}-{env_name}' - args['wandb_group'] = f'puf-{version}-baseline' - shutil.rmtree(f'experiments/{args["exp_id"]}', ignore_errors=True) - run = init_wandb(args, args['exp_id'], resume=False) - if args['mode'] in ('eval', 'evaluate'): - model_name = f'puf-{version}-{env_name}_model:latest' + if args['neptune']: + import neptune + neptune_name = args['neptune_name'] + neptune_project = args['neptune_project'] + run = neptune.init_run( + project=f"{neptune_name}/{neptune_project}", + with_id=run_id, mode="read-only") + run["model"].download(destination="downloaded_artifact") + elif args['wandb']: + args['track'] = True + version = '.'.join(pufferlib.__version__.split('.')[:2]) + args['exp_id'] = f'puf-{version}-{env_name}' + args['wandb_group'] = f'puf-{version}-baseline' + shutil.rmtree(f'experiments/{args["exp_id"]}', ignore_errors=True) + run = init_wandb(args, args['exp_id'], resume=False) + model_name = f'{env_name}-{run_id}_model:latest' artifact = run.use_artifact(model_name) data_dir = artifact.download() model_file = max(os.listdir(data_dir)) args['eval_model_path'] = os.path.join(data_dir, model_file) + else: + raise pufferlib.APIUsageError('No run id provided for eval') - if args['mode'] == 'eval': - args['vec'] = dict(backend='Serial', num_envs=1) - - vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) - policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) + policy.load_state_dict(torch.load("downloaded_artifact/model.pt")) if args['load_model_path'] is not None: policy.load_state_dict(torch.load( args['load_model_path'], map_location=args['train']['device'])) if args['mode'] == 'train': - train_config = dict(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) - pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) - - while pufferl.global_step < train_config['total_timesteps']: - pufferl.evaluate() - logs = pufferl.train() - - vecenv.async_reset(train_config['seed']) - for _ in range(10): - stats = pufferl.evaluate() - - pufferl.mean_and_log() - pufferl.close() + experiment(vecenv, policy, args) elif args['mode'] == 'eval': ob, info = vecenv.reset() driver = vecenv.driver_env From 3f934abb9036b8b7bdbf85a0f4670d5a95fa393e Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 9 May 2025 15:48:33 +0000 Subject: [PATCH 46/63] Remove scipy --- pufferlib/sweep.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pufferlib/sweep.py b/pufferlib/sweep.py index 0d7cdb85c9..8abfba2932 100644 --- a/pufferlib/sweep.py +++ b/pufferlib/sweep.py @@ -6,7 +6,6 @@ from copy import deepcopy import pufferlib -import scipy.stats import torch import pyro From ebe5f3ca97d01374b50512195289da666a6e7242 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 9 May 2025 16:59:39 +0000 Subject: [PATCH 47/63] Clean up model save and load --- clean_pufferl.py | 74 +++++++++++++++++++++--------------------------- 1 file changed, 33 insertions(+), 41 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 4e128fb235..a223a2e7bb 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -188,11 +188,15 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.neptune = neptune self.wandb = wandb if neptune: - self.neptune = init_neptune(args, env_name, id=config['run_id'], tag=config['tag']) + self.neptune = init_neptune(args, tag=config['tag']) + self.run_id = self.neptune._sys_id for k, v in pufferlib.unroll_nested_dict(args): self.neptune[k].append(v) elif wandb: - self.wandb = init_wandb(args, env_name, id=config['run_id'], tag=config['tag']) + self.wandb = init_wandb(args, tag=config['tag']) + self.run_id = self.wandb.run.id + else: + self.run_id = int(random.random() * 1e8) # Profiling self.uptime = 0 @@ -562,25 +566,20 @@ def mean_and_log(self): def close(self): self.vecenv.close() self.utilization.stop() - config = self.config model_path = self.save_checkpoint() + path = os.path.join(self.config['data_dir'], f'{self.run_id}.pt') + shutil.copy(model_path, path) if self.wandb: - exp_id = config['exp_id'] - artifact_name = f'{exp_id}_model' - artifact = self.wandb.Artifact(artifact_name, type='model') - artifact.add_file(model_path) + artifact = self.wandb.Artifact(self.run_id, type='model') + artifact.add_file(path) self.wandb.run.log_artifact(artifact) self.wandb.finish() elif self.neptune: - model = torch.load(model_path) - torch.save(model, '/tmp/model.pt') - self.neptune['model'].track_files('/tmp/model.pt') + self.neptune['model'].track_files(path) self.neptune.stop() def save_checkpoint(self): - config = self.config - exp_id = config['exp_id'] - path = os.path.join(config['data_dir'], exp_id) + path = os.path.join(self.config['data_dir'], self.run_id) if not os.path.exists(path): os.makedirs(path) @@ -597,7 +596,7 @@ def save_checkpoint(self): 'agent_step': self.global_step, 'update': self.epoch, 'model_name': model_name, - 'exp_id': exp_id, + 'run_id': self.run_id, } state_path = os.path.join(path, 'trainer_state.pt') torch.save(state, state_path + '.tmp') @@ -606,7 +605,7 @@ def save_checkpoint(self): def try_load_checkpoint(self): config = self.config - path = os.path.join(config['data_dir'], config['exp_id']) + path = os.path.join(config['data_dir'], self.run_id) if not os.path.exists(path): print('No checkpoints found. Assuming new experiment') return @@ -852,7 +851,7 @@ def fmt_perf(name, color, delta_ref, prof): return f'{color}{name}', duration(prof['elapsed']), f'{b2}{percent:2d}{c2}%' -def init_wandb(args, name, id=None, resume=True, tag=None): +def init_wandb(args, id=None, resume=True, tag=None): import wandb wandb.init( id=id or wandb.util.generate_id(), @@ -862,12 +861,11 @@ def init_wandb(args, name, id=None, resume=True, tag=None): save_code=False, resume=resume, config=args, - name=name, tags=[tag] if tag is not None else [], ) return wandb -def init_neptune(args, name, id=None, resume=True, tag=None, mode="async"): +def init_neptune(args, id=None, resume=True, tag=None, mode="async"): import neptune import neptune.exceptions try: @@ -903,7 +901,7 @@ def downsample_linear(arr, m): return np.interp(x_new, x_old, arr) def experiment(vecenv, policy, args): - train_config = dict(**args['train'], env=env_name, tag=args['tag'], exp_id=exp_id) + train_config = dict(**args['train'], env=env_name, tag=args['tag']) pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) all_logs = [] @@ -935,15 +933,13 @@ def experiment(vecenv, policy, args): choices='train eval sweep autotune profile'.split()) parser.add_argument('--load-model-path', type=str, default=None, help='Path to a pretrained checkpoint') - parser.add_argument('--baseline', action='store_true', - help='Load pretrained model from WandB if available') + parser.add_argument('--load-id', type=str, + default=None, help='Kickstart/eval from from a finished Wandb/Neptune run') parser.add_argument('--render-mode', type=str, default='auto', choices=['auto', 'human', 'ansi', 'rgb_array', 'raylib', 'None']) parser.add_argument('--save-frames', type=int, default=0) parser.add_argument('--gif-path', type=str, default='eval.gif') parser.add_argument('--fps', type=float, default=15) - parser.add_argument('--exp-id', '--exp-name', type=str, - default=None, help='Resume from experiment') parser.add_argument('--max-runs', type=int, default=200, help='Max number of sweep runs') parser.add_argument('--wandb', action='store_true', help='Use wandb for logging') parser.add_argument('--wandb-project', type=str, default='pufferlib') @@ -1010,8 +1006,8 @@ def experiment(vecenv, policy, args): exit(0) args['train']['use_rnn'] = rnn_cls is not None - exp_id = args['exp_id'] or env_name + '-' + str(uuid.uuid4())[:8] env_name = args['env_name'] + device = args['train']['device'] if args['mode'] == 'sweep': if not args['wandb'] and not args['neptune']: @@ -1056,33 +1052,29 @@ def experiment(vecenv, policy, args): vecenv = pufferlib.vector.make(make_env, env_kwargs=args['env'], **args['vec']) policy = make_policy(vecenv.driver_env, policy_cls, rnn_cls, args) - run_id = args['exp_id'] - if run_id is not None: - assert args['mode'] in ('train', 'eval', 'evaluate') + load_id = args['load_id'] + if load_id is not None: + if args['mode'] not in ('train', 'eval'): + raise pufferlib.APIUsageError('load_id requires mode to be train or eval') + if args['neptune']: import neptune neptune_name = args['neptune_name'] neptune_project = args['neptune_project'] run = neptune.init_run( project=f"{neptune_name}/{neptune_project}", - with_id=run_id, mode="read-only") - run["model"].download(destination="downloaded_artifact") + with_id=load_id, mode="read-only") + data_dir = 'artifacts' + run["model"].download(destination=data_dir) elif args['wandb']: - args['track'] = True - version = '.'.join(pufferlib.__version__.split('.')[:2]) - args['exp_id'] = f'puf-{version}-{env_name}' - args['wandb_group'] = f'puf-{version}-baseline' - shutil.rmtree(f'experiments/{args["exp_id"]}', ignore_errors=True) - run = init_wandb(args, args['exp_id'], resume=False) - model_name = f'{env_name}-{run_id}_model:latest' - artifact = run.use_artifact(model_name) + run = init_wandb(args, load_id, resume='must') + artifact = run.use_artifact(f'{load_id}:latest') data_dir = artifact.download() model_file = max(os.listdir(data_dir)) - args['eval_model_path'] = os.path.join(data_dir, model_file) else: raise pufferlib.APIUsageError('No run id provided for eval') - policy.load_state_dict(torch.load("downloaded_artifact/model.pt")) + policy.load_state_dict(torch.load(f'{data_dir}/{load_id}.pt', map_location=device)) if args['load_model_path'] is not None: policy.load_state_dict(torch.load( @@ -1095,8 +1087,8 @@ def experiment(vecenv, policy, args): driver = vecenv.driver_env num_agents = vecenv.observation_space.shape[0] state = dict( - lstm_h=torch.zeros(num_agents, policy.hidden_size, device=args['train']['device']), - lstm_c=torch.zeros(num_agents, policy.hidden_size, device=args['train']['device']), + lstm_h=torch.zeros(num_agents, policy.hidden_size, device=device), + lstm_c=torch.zeros(num_agents, policy.hidden_size, device=device), ) frames = [] From b7d2a6cbe33e81e681fc8f332e22e839c5386243 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Fri, 9 May 2025 21:51:52 +0000 Subject: [PATCH 48/63] Clean up train file --- clean_pufferl.py | 453 +++++++++++++++++------------------------ config/default.ini | 1 - pufferlib.cpp | 79 +------ pufferlib/models.py | 37 +--- pufferlib/pufferlib.cu | 255 +---------------------- setup.py | 4 +- 6 files changed, 209 insertions(+), 620 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index a223a2e7bb..e270484352 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -6,49 +6,36 @@ #@record import os -import random -import psutil -import time -import configparser -import argparse -import shutil import glob -import uuid import ast - +import time +import random +import shutil +import argparse +import configparser from threading import Thread from collections import defaultdict, deque -from contextlib import nullcontext import numpy as np +import psutil import torch - import torch.distributed import torch.utils.cpp_extension import pufferlib -import pufferlib.pytorch import pufferlib.sweep import pufferlib.vector +import pufferlib.pytorch from pufferlib import _C -import signal # Aggressively exit on ctrl+c -signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) - import rich -from rich.console import Console +import rich.traceback from rich.table import Table +from rich.console import Console from rich_argparse import RichHelpFormatter -import rich.traceback rich.traceback.install(show_locals=False) -c1 = '[cyan]' -c2 = '[white]' -b1 = '[bright_cyan]' -b2 = '[bright_white]' - - class CleanPuffeRL: def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Backend perf optimization @@ -69,10 +56,8 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): total_agents = vecenv.num_agents self.total_agents = total_agents - # Experience buffer - device = config['device'] + # Experience batch_size = config['batch_size'] - if config['bptt_horizon'] == 'auto': config['bptt_horizon'] = batch_size // total_agents @@ -84,69 +69,60 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): f'Total agents {total_agents} <= segments {segments}' ) + device = config['device'] + self.observations = torch.zeros(segments, horizon, *obs_space.shape, + dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[obs_space.dtype], + pin_memory=device == 'cuda' and config['cpu_offload'], + device='cpu' if config['cpu_offload'] else device) + self.actions = torch.zeros(segments, horizon, *atn_space.shape, device=device, + dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[atn_space.dtype]) + self.values = torch.zeros(segments, horizon, device=device) + self.logprobs = torch.zeros(segments, horizon, device=device) + self.rewards = torch.zeros(segments, horizon, device=device) + self.terminals = torch.zeros(segments, horizon, device=device) + self.truncations = torch.zeros(segments, horizon, device=device) + self.ratio = torch.ones(segments, horizon, device=device) + self.importance = torch.ones(segments, horizon, device=device) self.ep_uses = torch.zeros(segments, device=device, dtype=torch.int32) self.ep_lengths = torch.zeros(total_agents, device=device, dtype=torch.int32) self.ep_indices = torch.arange(total_agents, device=device, dtype=torch.int32) self.free_idx = total_agents - experience = dict( - obs=torch.zeros(segments, horizon, *obs_space.shape, - dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[obs_space.dtype], - pin_memory=device == 'cuda' and config['cpu_offload'], - device='cpu' if config['cpu_offload'] else device), - actions=torch.zeros(segments, horizon, *atn_space.shape, - dtype=pufferlib.pytorch.numpy_to_torch_dtype_dict[atn_space.dtype], device=device), - values = torch.zeros(segments, horizon, device=device), - logprobs=torch.zeros(segments, horizon, device=device), - rewards=torch.zeros(segments, horizon, device=device), - dones=torch.zeros(segments, horizon, device=device), - truncateds=torch.zeros(segments, horizon, device=device), - ratio = torch.ones(segments, horizon, device=device), - ) - self.experience = experience - - if config['use_vtrace'] or config['use_puff_advantage']: - experience['importance'] = torch.ones(segments, horizon, device=device) # LSTM - # TODO: This breaks compile if config['use_rnn']: # TODO: Doesn't exist in native envs - # TODO: Replace slice with env idx or similar n = vecenv.agents_per_batch - self.lstm_h = {i*n: torch.zeros(n, policy.hidden_size, device=device) for i in range(total_agents//n)} - self.lstm_c = {i*n: torch.zeros(n, policy.hidden_size, device=device) for i in range(total_agents//n)} - + h = policy.hidden_size + self.lstm_h = {i*n: torch.zeros(n, h, device=device) for i in range(total_agents//n)} + self.lstm_c = {i*n: torch.zeros(n, h, device=device) for i in range(total_agents//n)} # Minibatching & gradient accumulation minibatch_size = config['minibatch_size'] max_minibatch_size = config['max_minibatch_size'] self.minibatch_size = min(minibatch_size, max_minibatch_size) - if minibatch_size % max_minibatch_size != 0 and max_minibatch_size % minibatch_size != 0: - # todo: better error + if minibatch_size > max_minibatch_size and minibatch_size % max_minibatch_size != 0: raise pufferlib.APIUsageError( - f'max_minibatch_size {max_minibatch_size} must be a multiple of minibatch_size {minibatch_size}' - ) + f'minibatch_size {minibatch_size} > max_minibatch_size {max_minibatch_size} must divide evenly') - self.accumulate_minibatches = max(1, config['minibatch_size'] // config['max_minibatch_size']) + self.accumulate_minibatches = max(1, minibatch_size // max_minibatch_size) self.total_minibatches = int(config['update_epochs'] * batch_size / self.minibatch_size) self.minibatch_segments = self.minibatch_size // horizon if self.minibatch_segments * horizon != self.minibatch_size: raise pufferlib.APIUsageError( - f'minibatch_size {self.minibatch_size} must be divisible by horizon {horizon}' + f'minibatch_size {self.minibatch_size} must be divisible by bptt_horizon {horizon}' ) # Torch compile self.uncompiled_policy = policy + self.policy = policy if config['compile']: - policy = torch.compile(policy, mode=config['compile_mode'], fullgraph=config['compile_fullgraph']) + self.policy = torch.compile(policy, mode=config['compile_mode'], fullgraph=config['compile_fullgraph']) - self.policy = policy # Optimizer - # TODO: **optim params if config['optimizer'] == 'adam': optimizer = torch.optim.Adam( - policy.parameters(), + self.policy.parameters(), lr=config['learning_rate'], betas=(config['adam_beta1'], config['adam_beta2']), eps=config['adam_eps'], @@ -156,7 +132,7 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): import heavyball.utils heavyball.utils.compile_mode = config['compile_mode'] if config['compile'] else None optimizer = ForeachMuon( - policy.parameters(), + self.policy.parameters(), lr=config['learning_rate'], betas=(config['adam_beta1'], config['adam_beta2']), eps=config['adam_eps'], @@ -168,15 +144,8 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # Learning rate scheduler epochs = config['total_timesteps'] // config['batch_size'] + self.scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) self.total_epochs = epochs - if config['scheduler'] == 'linear': - scheduler = torch.optim.lr_scheduler.LinearLR(optimizer, start_factor=1.0, end_factor=0.0, total_iters=epochs) - elif config['scheduler'] == 'cosine': - scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs) - else: - raise pufferlib.APIUsageError(f'Unknown scheduler: {config["scheduler"]}') - - self.scheduler = scheduler # Automatic mixed precision precision = config['precision'] @@ -196,29 +165,26 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.wandb = init_wandb(args, tag=config['tag']) self.run_id = self.wandb.run.id else: - self.run_id = int(random.random() * 1e8) - - # Profiling - self.uptime = 0 - self.start_time = time.time() - self.utilization = Utilization() - self.profile = Profile(['eval', 'env', 'eval_forward', 'eval_copy', 'eval_misc', 'train', 'train_forward', - 'learn', 'train_copy', 'train_misc', 'custom'], frequency=5) + self.run_id = str(int(random.random() * 1e8)) # Initializations - self.config = config - self.vecenv = vecenv self.global_step = 0 + self.uptime = 0 self.epoch = 0 - self.stats = defaultdict(list) # TODO: can this be set in eval and handle accum differently? + self.config = config + self.vecenv = vecenv + self.start_time = time.time() + self.utilization = Utilization() + self.profile = Profile() + self.stats = defaultdict(list) + self.losses = {} # Dashboard - self.losses = {} num_params = sum(p.numel() for p in policy.parameters() if p.requires_grad) - self.msg = f'Model Size: {abbreviate(num_params)} parameters' + params, unit = abbreviate(num_params) + self.msg = f'Model Size: {params} {unit} parameters' self.print_dashboard(clear=True) - def evaluate(self): profile = self.profile epoch = self.epoch @@ -226,8 +192,6 @@ def evaluate(self): profile('eval_misc', epoch, nest=True) config = self.config - experience = self.experience - policy = self.policy device = config['device'] self.full_rows = 0 @@ -262,8 +226,9 @@ def evaluate(self): state['lstm_h'] = self.lstm_h[env_id.start] state['lstm_c'] = self.lstm_c[env_id.start] - logits, value = policy(o_device, state) - action, logprob, _ = pufferlib.pytorch.sample_logits(logits, is_continuous=policy.is_continuous) + logits, value = self.policy(o_device, state) + action, logprob, _ = pufferlib.pytorch.sample_logits( + logits, is_continuous=self.policy.is_continuous) r = torch.clamp(r, -1, 1) profile('eval_copy', epoch) @@ -272,8 +237,33 @@ def evaluate(self): self.lstm_h[env_id.start] = state['lstm_h'] self.lstm_c[env_id.start] = state['lstm_c'] - o = o if config['cpu_offload'] else o_device - actions = self.store(state, o, value, action, logprob, r, d, env_id, mask) + # Fast path for fully vectorized envs + l = self.ep_lengths[env_id.start].item() + batch_rows = slice(self.ep_indices[env_id.start].item(), 1+self.ep_indices[env_id.stop - 1].item()) + + if config['cpu_offload']: + self.observations[batch_rows, l] = o + else: + self.observations[batch_rows, l] = o_device + + self.actions[batch_rows, l] = action + self.logprobs[batch_rows, l] = logprob + self.rewards[batch_rows, l] = r + self.terminals[batch_rows, l] = d.float() + self.values[batch_rows, l] = value.flatten() + + # TODO: Handle masks!! + #indices = np.where(mask)[0] + #data.ep_lengths[env_id[mask]] += 1 + self.ep_lengths[env_id] += 1 + if l+1 >= config['bptt_horizon']: + num_full = env_id.stop - env_id.start + self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config['device']).int() + self.ep_lengths[env_id] = 0 + self.free_idx += num_full + self.full_rows += num_full + + action = action.cpu().numpy() profile('eval_misc', epoch) for i in info: @@ -286,7 +276,7 @@ def evaluate(self): self.stats[k].append(v) profile('env', epoch) - self.vecenv.send(actions) + self.vecenv.send(action) profile('eval_misc', epoch) self.free_idx = self.total_agents @@ -300,60 +290,66 @@ def train(self): profile = self.profile epoch = self.epoch profile('train', epoch) - - config = self.config - experience = self.experience losses = defaultdict(float) + config = self.config device = config['device'] + b0 = config['prio_beta0'] + a = config['prio_alpha'] + clip_coef = config['clip_coef'] + vf_clip = config['vf_clip_coef'] + anneal_beta = b0 + (1 - b0)*a*self.epoch/self.total_epochs + for mb in range(self.total_minibatches): profile('train_misc', epoch, nest=True) self.amp_context.__enter__() - loss = 0 - shape = experience['values'].shape - if config['use_vtrace']: - importance = advantages = torch.zeros(shape, device=device) - vs = torch.zeros(shape, device=device) - self.compute_vtrace(experience['values'], experience['rewards'], - experience['dones'], experience['ratio'], vs, advantages, - config['gamma'], config['vtrace_rho_clip'], config['vtrace_c_clip']) - elif config['use_puff_advantage']: - importance = advantages = torch.zeros(shape, device=device) - vs = torch.zeros(shape, device=device) - - # TODO: Eliminate - n = (shape[0]//256)*256 - torch.ops.pufferlib.compute_puff_advantage(experience['values'][:n], experience['rewards'][:n], - experience['dones'][:n], experience['ratio'][:n], vs[:n], advantages[:n], config['gamma'], - config['gae_lambda'], config['vtrace_rho_clip'], config['vtrace_c_clip']) - else: - importance = advantages = self.compute_gae(experience['values'], experience['rewards'], - experience['dones'], config['gamma'], config['gae_lambda']) + # TODO: Eliminate + shape = self.values.shape + n = (shape[0]//256)*256 + advantages = torch.zeros(shape, device=device) + torch.ops.pufferlib.compute_puff_advantage(self.values[:n], self.rewards[:n], + self.terminals[:n], self.ratio[:n], advantages[:n], config['gamma'], + config['gae_lambda'], config['vtrace_rho_clip'], config['vtrace_c_clip']) profile('train_copy', epoch) - batch = self.sample(importance, self.minibatch_segments) + adv = advantages.abs().sum(axis=1) + prio_weights = torch.nan_to_num(adv**a, 0, 0, 0) + prio_probs = (prio_weights + 1e-6)/(prio_weights.sum() + 1e-6) + idx = torch.multinomial(prio_probs, self.minibatch_segments) + mb_prio = (self.segments*prio_probs[idx, None])**-anneal_beta + self.ep_uses[idx] += 1 + mb_obs = self.observations[idx] + mb_actions = self.actions[idx] + mb_logprobs = self.logprobs[idx] + mb_rewards = self.rewards[idx] + mb_terminals = self.terminals[idx] + mb_truncations = self.truncations[idx] + mb_ratio = self.ratio[idx] + mb_values = self.values[idx] + mb_returns = advantages[idx] + mb_values + mb_advantages = advantages[idx] profile('train_forward', epoch) state = dict( - action=batch['actions'], + action=mb_actions, lstm_h=None, lstm_c=None, ) if not config['use_rnn']: - batch['obs'] = batch['obs'].reshape(-1, *self.vecenv.single_observation_space.shape) + mb_obs = mb_obs.reshape(-1, *self.vecenv.single_observation_space.shape) # TODO: Currently only returning traj shaped value as a hack - logits, newvalue = self.policy.forward_train(batch['obs'], state) + logits, newvalue = self.policy.forward_train(mb_obs, state) actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, - action=batch['actions'], is_continuous=self.policy.is_continuous) + action=mb_actions, is_continuous=self.policy.is_continuous) profile('train_misc', epoch) - newlogprob = newlogprob.reshape(batch['logprobs'].shape) - logratio = newlogprob - batch['logprobs'] + newlogprob = newlogprob.reshape(mb_logprobs.shape) + logratio = newlogprob - mb_logprobs ratio = logratio.exp() - experience['ratio'][batch['idx']] = ratio # TODO: Experiment with this + self.ratio[idx] = ratio # TODO: Experiment with this # TODO: Only do this if we are KL clipping? Saves 1-2% compute with torch.no_grad(): @@ -363,84 +359,63 @@ def train(self): clipfrac = ((ratio - 1.0).abs() > config['clip_coef']).float().mean() # TODO: Do you need to do this? Policy hasn't changed - if config['use_vtrace'] or config['use_puff_advantage']: - with torch.no_grad(): - adv = advantages[batch['idx']] - vs = vs[batch['idx']] - if config['use_vtrace']: - self.compute_vtrace(batch['values'], batch['rewards'], batch['dones'], - ratio, vs, adv, config['gamma'], config['vtrace_rho_clip'], config['vtrace_c_clip']) - elif config['use_puff_advantage']: - torch.ops.pufferlib.compute_puff_advantage(batch['values'], batch['rewards'], batch['dones'], - ratio, vs, adv, config['gamma'], config['gae_lambda'], config['vtrace_rho_clip'], config['vtrace_c_clip']) - - adv = batch['advantages'] - adv = (adv - adv.mean()) / (adv.std() + 1e-8) - - # Prioritized replay - adv = adv * batch['prio'] - - # Policy loss + with torch.no_grad(): + adv = advantages[idx] + torch.ops.pufferlib.compute_puff_advantage(mb_values, mb_rewards, mb_terminals, + ratio, adv, config['gamma'], config['gae_lambda'], + config['vtrace_rho_clip'], config['vtrace_c_clip']) + + adv = mb_advantages + adv = mb_prio * (adv - adv.mean()) / (adv.std() + 1e-8) + + # Losses pg_loss1 = -adv * ratio - pg_loss2 = -adv * torch.clamp( - ratio, 1 - config['clip_coef'], 1 + config['clip_coef'] - ) + pg_loss2 = -adv * torch.clamp(ratio, 1 - clip_coef, 1 + clip_coef) pg_loss = torch.max(pg_loss1, pg_loss2).mean() - # Value loss - ret = batch['returns'] - newvalue = newvalue.view(ret.shape) - v_loss_unclipped = (newvalue - ret) ** 2 - val = batch['values'] - v_clipped = val + torch.clamp( - newvalue - val, - -config['vf_clip_coef'], - config['vf_clip_coef'], - ) - v_loss_clipped = (v_clipped - ret) ** 2 - v_loss_max = torch.max(v_loss_unclipped, v_loss_clipped) - v_loss = 0.5 * v_loss_max.mean() + newvalue = newvalue.view(mb_returns.shape) + v_clipped = mb_values + torch.clamp(newvalue - mb_values, -vf_clip, vf_clip) + v_loss_unclipped = (newvalue - mb_returns) ** 2 + v_loss_clipped = (v_clipped - mb_returns) ** 2 + v_loss = 0.5*torch.max(v_loss_unclipped, v_loss_clipped).mean() - # Entropy loss entropy_loss = entropy.mean() - # Total loss - loss += pg_loss - config['ent_coef']*entropy_loss + v_loss*config['vf_coef'] - self.amp_context.__enter__() + loss = pg_loss + config['vf_coef']*v_loss - config['ent_coef']*entropy_loss + self.amp_context.__enter__() # TODO: Debug # This breaks vloss clipping? with torch.no_grad(): - experience['values'][batch['idx']] = newvalue.float() + self.values[idx] = newvalue.float() + + profile('train_misc', epoch) + losses['policy_loss'] += pg_loss.item() / self.total_minibatches + losses['value_loss'] += v_loss.item() / self.total_minibatches + losses['entropy'] += entropy_loss.item() / self.total_minibatches + losses['old_approx_kl'] += old_approx_kl.item() / self.total_minibatches + losses['approx_kl'] += approx_kl.item() / self.total_minibatches + losses['clipfrac'] += clipfrac.item() / self.total_minibatches + losses['importance'] += ratio.mean().item() / self.total_minibatches profile('learn', epoch) loss.backward() - if (mb + 1) % self.accumulate_minibatches == 0: torch.nn.utils.clip_grad_norm_(self.policy.parameters(), config['max_grad_norm']) self.optimizer.step() self.optimizer.zero_grad() - profile('train_misc', epoch) - losses['policy_loss'] += pg_loss.item() / self.total_minibatches - losses['value_loss'] += v_loss.item() / self.total_minibatches - losses['entropy'] += entropy_loss.item() / self.total_minibatches - losses['old_approx_kl'] += old_approx_kl.item() / self.total_minibatches - losses['approx_kl'] += approx_kl.item() / self.total_minibatches - losses['clipfrac'] += clipfrac.item() / self.total_minibatches - losses['importance'] += ratio.mean().item() / self.total_minibatches - # Reprioritize experience profile('train_misc', epoch) self.max_uses = self.ep_uses.max().item() self.mean_uses = self.ep_uses.float().mean().item() - experience['ratio'][:] = 1 + self.ratio[:] = 1 if config['anneal_lr']: self.scheduler.step() - y_pred = experience['values'].flatten() + y_pred = self.values.flatten() # TODO: Probably not updated - y_true = advantages.flatten() + experience['values'].flatten() + y_true = advantages.flatten() + self.values.flatten() var_y = y_true.var() explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y @@ -464,68 +439,6 @@ def train(self): return logs - def store(self, state, obs, value, action, logprob, reward, done, env_id, mask): - config = self.config - exp = self.experience - - # Fast path for fully vectorized envs - l = self.ep_lengths[env_id.start].item() - batch_rows = slice(self.ep_indices[env_id.start].item(), 1+self.ep_indices[env_id.stop - 1].item()) - - exp['obs'][batch_rows, l] = obs - exp['actions'][batch_rows, l] = action - exp['logprobs'][batch_rows, l] = logprob - exp['rewards'][batch_rows, l] = reward - exp['dones'][batch_rows, l] = done.float() - exp['values'][batch_rows, l] = value.flatten() - - # TODO: Handle masks!! - #indices = np.where(mask)[0] - #data.ep_lengths[env_id[mask]] += 1 - self.ep_lengths[env_id] += 1 - if l+1 >= config['bptt_horizon']: - num_full = env_id.stop - env_id.start - self.ep_indices[env_id] = self.free_idx + torch.arange(num_full, device=config['device']).int() - self.ep_lengths[env_id] = 0 - self.free_idx += num_full - self.full_rows += num_full - - return action.cpu().numpy() - - def sample(self, advantages, n, reward_block=None, mask_block=None, method='prio'): - config = self.config - exp = self.experience - if method == 'topk': - _, idx = torch.topk(advantages.abs().sum(axis=1), n) - elif method == 'prio': - adv = advantages.abs().sum(axis=1) - probs = adv**config['prio_alpha'] - probs = torch.nan_to_num(probs, 0, 0, 0) - probs = (probs + 1e-6)/(probs.sum() + 1e-6) - idx = torch.multinomial(probs, n) - elif method == 'multinomial': - idx = torch.multinomial(advantages.abs().sum(axis=1) + 1e-6, n) - elif method == 'random': - idx = torch.randint(0, advantages.shape[0], (n,), device=config['device']) - else: - raise ValueError(f'Unknown sampling method: {method}') - - self.ep_uses[idx] += 1 - output = {k: v[idx] for k, v in exp.items()} - output['idx'] = idx - output['values'] = exp['values'][idx] - output['advantages'] = advantages[idx] - output['returns'] = advantages[idx] + exp['values'][idx] - - output['prio'] = 1 - if method == 'prio': - b0 = config['prio_beta0'] - a = config['prio_alpha'] - beta = b0 + (1 - b0)*a*self.epoch/self.total_epochs - output['prio'] = (((1/len(probs)) * (1/probs[idx]))**beta).unsqueeze(1).expand_as(output['advantages']) - - return dict(**output) - def mean_and_log(self): config = self.config for k in list(self.stats.keys()): @@ -629,6 +542,11 @@ def print_dashboard(self, clear=False, max_stats=[0]): dashboard = Table(box=rich.box.ROUNDED, expand=True, show_header=False, border_style='bright_cyan') + c1 = '[cyan]' + c2 = '[white]' + b1 = '[bright_cyan]' + b2 = '[bright_white]' + table = Table(box=None, expand=True, show_header=False) dashboard.add_row(table) cpu_percent = np.mean(utilization.cpu_util) @@ -654,34 +572,37 @@ def print_dashboard(self, clear=False, max_stats=[0]): remaining = 'A hair past a freckle' if delta != 0: SPS = config['batch_size'] / delta - remaining = duration((config['total_timesteps'] - self.global_step)/SPS) + remaining = duration((config['total_timesteps'] - self.global_step)/SPS, b2, c2) uptime = time.time() - self.start_time s.add_column(f"{c1}Summary", justify='left', vertical='top', width=10) s.add_column(f"{c1}Value", justify='right', vertical='top', width=14) s.add_row(f'{c2}Env', f'{b2}{config["env"]}') - s.add_row(f'{c2}Steps', abbreviate(self.global_step)) - s.add_row(f'{c2}SPS', abbreviate(SPS)) - s.add_row(f'{c2}Epoch', abbreviate(self.epoch)) - s.add_row(f'{c2}Uptime', duration(uptime)) + steps, unit = abbreviate(self.global_step) + s.add_row(f'{c2}Steps', f'{b2}{steps:.1f}{c2}{unit}') + sps, unit = abbreviate(SPS) + s.add_row(f'{c2}SPS', f'{b2}{sps:.1f}{c2}{unit}') + epoch, unit = abbreviate(self.epoch) + s.add_row(f'{c2}Epoch', f'{b2}{epoch}{c2}{unit}') + s.add_row(f'{c2}Uptime', duration(uptime, b2, c2)) s.add_row(f'{c2}Remaining', remaining) p = Table(box=None, expand=True, show_header=False) p.add_column(f"{c1}Performance", justify="left", width=10) p.add_column(f"{c1}Time", justify="right", width=8) p.add_column(f"{c1}%", justify="right", width=4) - p.add_row(*fmt_perf('Evaluate', b1, delta, profile.eval)) - p.add_row(*fmt_perf(' Forward', c2, delta, profile.eval_forward)) - p.add_row(*fmt_perf(' Env', c2, delta, profile.env)) - p.add_row(*fmt_perf(' Copy', c2, delta, profile.eval_copy)) - p.add_row(*fmt_perf(' Misc', c2, delta, profile.eval_misc)) - p.add_row(*fmt_perf('Train', b1, delta, profile.train)) - p.add_row(*fmt_perf(' Forward', c2, delta, profile.train_forward)) - p.add_row(*fmt_perf(' Learn', c2, delta, profile.learn)) - p.add_row(*fmt_perf(' Copy', c2, delta, profile.train_copy)) - p.add_row(*fmt_perf(' Misc', c2, delta, profile.train_misc)) + p.add_row(*fmt_perf('Evaluate', b1, delta, profile.eval, b2, c2)) + p.add_row(*fmt_perf(' Forward', c2, delta, profile.eval_forward, b2, c2)) + p.add_row(*fmt_perf(' Env', c2, delta, profile.env, b2, c2)) + p.add_row(*fmt_perf(' Copy', c2, delta, profile.eval_copy, b2, c2)) + p.add_row(*fmt_perf(' Misc', c2, delta, profile.eval_misc, b2, c2)) + p.add_row(*fmt_perf('Train', b1, delta, profile.train, b2, c2)) + p.add_row(*fmt_perf(' Forward', c2, delta, profile.train_forward, b2, c2)) + p.add_row(*fmt_perf(' Learn', c2, delta, profile.learn, b2, c2)) + p.add_row(*fmt_perf(' Copy', c2, delta, profile.train_copy, b2, c2)) + p.add_row(*fmt_perf(' Misc', c2, delta, profile.train_misc, b2, c2)) if 'custom' in profile.profiles: - p.add_row(*fmt_perf(' Custom', c2, uptime, profile.custom)) + p.add_row(*fmt_perf(' Custom', c2, uptime, profile.custom, b2, c2)) l = Table(box=None, expand=True, ) l.add_column(f'{c1}Losses', justify="left", width=16) @@ -745,18 +666,10 @@ def dist_mean(value, device): return dist_sum(value, device) / torch.distributed.get_world_size() class Profile: - def __init__(self, keys, frequency=1): - self.stack = [] + def __init__(self, frequency=5): + self.profiles = defaultdict(lambda: defaultdict(float)) self.frequency = frequency - self.profiles = {k: - dict( - start = 0, - buffer = 0, - delta = 0, - elapsed = 0, - calls = 0, - ) for k in keys - } + self.stack = [] def __iter__(self): return iter(self.profiles.items()) @@ -829,26 +742,26 @@ def stop(self): def abbreviate(num): if num < 1e3: - return f'{b2}{num:.0f}' + return num, '' elif num < 1e6: - return f'{b2}{num/1e3:.1f}{c2}k' + return num/1e3, 'k' elif num < 1e9: - return f'{b2}{num/1e6:.1f}{c2}m' + return num/1e6, 'm' elif num < 1e12: - return f'{b2}{num/1e9:.1f}{c2}b' + return num/1e9, 'b' else: - return f'{b2}{num/1e12:.1f}{c2}t' + return num/1e12, 't' -def duration(seconds): +def duration(seconds, b2, c2): seconds = int(seconds) h = seconds // 3600 m = (seconds % 3600) // 60 s = seconds % 60 return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" -def fmt_perf(name, color, delta_ref, prof): +def fmt_perf(name, color, delta_ref, prof, b2, c2): percent = 0 if delta_ref == 0 else int(100*prof['delta']/delta_ref - 1e-5) - return f'{color}{name}', duration(prof['elapsed']), f'{b2}{percent:2d}{c2}%' + return f'{color}{name}', duration(prof['elapsed'], b2, c2), f'{b2}{percent:2d}{c2}%' def init_wandb(args, id=None, resume=True, tag=None): @@ -997,6 +910,10 @@ def experiment(vecenv, policy, args): if rnn_name is not None: rnn_cls = getattr(env_module.torch, args['rnn_name']) + # Aggressively exit on ctrl+c + import signal + signal.signal(signal.SIGINT, lambda sig, frame: os._exit(0)) + # Assume TorchRun DDP is used if LOCAL_RANK is set if 'LOCAL_RANK' in os.environ: torch.distributed.init_process_group(backend='nccl', rank=0, world_size=1) diff --git a/config/default.ini b/config/default.ini index 4847359bae..24460e1c7c 100644 --- a/config/default.ini +++ b/config/default.ini @@ -28,7 +28,6 @@ torch_deterministic = True cpu_offload = False device = cuda optimizer = muon -scheduler = cosine anneal_lr = True precision = float32 total_timesteps = 10_000_000 diff --git a/pufferlib.cpp b/pufferlib.cpp index a9ec5745d0..4260234779 100644 --- a/pufferlib.cpp +++ b/pufferlib.cpp @@ -27,37 +27,27 @@ namespace pufferlib { static const int max_horizon = 256; void puff_advantage_row(float* values, float* rewards, float* dones, - float* importance, float* vs, float* advantages, float gamma, float lambda, + float* importance, float* advantages, float gamma, float lambda, float rho_clip, float c_clip, int horizon) { - vs[horizon-1] = values[horizon-1]; float lastpufferlam = 0; for (int t = horizon-2; t >= 0; t--) { int t_next = t + 1; float nextnonterminal = 1.0 - dones[t_next]; float rho_t = fminf(importance[t], rho_clip); float c_t = fminf(importance[t], c_clip); - // TODO: t_next works and t doesn't. Check original formula float delta = rho_t*(rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]); lastpufferlam = delta + gamma*lambda*c_t*lastpufferlam*nextnonterminal; - - //float delta = rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]; - //lastpufferlam = delta + gamma*lambda*lastpufferlam*nextnonterminal; - - advantages[t] = lastpufferlam; - vs[t] = advantages[t] + values[t]; - //advantages[t] = rho_t*(rewards[t] + gamma*vs[t_next]*nextnonterminal - values[t]); - //vs[t] = lastpufferlam + values[t]; } } void vtrace_check(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + torch::Tensor dones, torch::Tensor importance, torch::Tensor advantages, int num_steps, int horizon) { // Validate input tensors torch::Device device = values.device(); - for (const torch::Tensor& t : {values, rewards, dones, importance, vs, advantages}) { + for (const torch::Tensor& t : {values, rewards, dones, importance, advantages}) { TORCH_CHECK(t.dim() == 2, "Tensor must be 2D"); TORCH_CHECK(t.device() == device, "All tensors must be on same device"); TORCH_CHECK(t.size(0) == num_steps, "First dimension must match num_steps"); @@ -71,63 +61,13 @@ void vtrace_check(torch::Tensor values, torch::Tensor rewards, } -/* -// [num_steps, horizon] -void gae(float* values, float* rewards, float* dones, float* advantages, - float gamma, float gae_lambda, int num_steps, int horizon){ - for (int offset = 0; offset < num_steps*horizon; offset+=horizon) { - gae_row(values + offset, rewards + offset, dones + offset, - advantages + offset, gamma, gae_lambda, horizon); - } -} - -torch::Tensor compute_gae(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, float gamma, float gae_lambda) { - int num_steps = values.size(0); - int horizon = values.size(1); - torch::Tensor advantages = gae_check(values, rewards, dones, num_steps, horizon); - gae(values.data_ptr(), rewards.data_ptr(), - dones.data_ptr(), advantages.data_ptr(), - gamma, gae_lambda, num_steps, horizon - ); - return advantages; -} - -// [num_steps, horizon] -void vtrace(float* values, float* rewards, float* dones, float* importance, - float* vs, float* advantages, float gamma, float rho_clip, float c_clip, - int num_steps, const int horizon){ - for (int offset = 0; offset < num_steps*horizon; offset+=horizon) { - vtrace_row(values + offset, rewards + offset, - dones + offset, importance + offset, - vs + offset, advantages + offset, - gamma, rho_clip, c_clip, horizon - ); - } -} - -void compute_vtrace(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, - float gamma, float rho_clip, float c_clip) { - int num_steps = values.size(0); - int horizon = values.size(1); - vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); - vtrace(values.data_ptr(), rewards.data_ptr(), - dones.data_ptr(), importance.data_ptr(), - vs.data_ptr(), advantages.data_ptr(), - gamma, rho_clip, c_clip, num_steps, horizon - ); -} -*/ - // [num_steps, horizon] void puff_advantage(float* values, float* rewards, float* dones, float* importance, - float* vs, float* advantages, float gamma, float lambda, float rho_clip, float c_clip, + float* advantages, float gamma, float lambda, float rho_clip, float c_clip, int num_steps, const int horizon){ for (int offset = 0; offset < num_steps*horizon; offset+=horizon) { puff_advantage_row(values + offset, rewards + offset, - dones + offset, importance + offset, - vs + offset, advantages + offset, + dones + offset, importance + offset, advantages + offset, gamma, lambda, rho_clip, c_clip, horizon ); } @@ -135,20 +75,19 @@ void puff_advantage(float* values, float* rewards, float* dones, float* importan void compute_puff_advantage_cpu(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + torch::Tensor dones, torch::Tensor importance, torch::Tensor advantages, double gamma, double lambda, double rho_clip, double c_clip) { int num_steps = values.size(0); int horizon = values.size(1); - vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); + vtrace_check(values, rewards, dones, importance, advantages, num_steps, horizon); puff_advantage(values.data_ptr(), rewards.data_ptr(), - dones.data_ptr(), importance.data_ptr(), - vs.data_ptr(), advantages.data_ptr(), + dones.data_ptr(), importance.data_ptr(), advantages.data_ptr(), gamma, lambda, rho_clip, c_clip, num_steps, horizon ); } TORCH_LIBRARY(pufferlib, m) { - m.def("compute_puff_advantage(Tensor(a!) values, Tensor(b!) rewards, Tensor(c!) dones, Tensor(d!) importance, Tensor(e!) vs, Tensor(f!) advantages, float gamma, float lambda, float rho_clip, float c_clip) -> ()"); + m.def("compute_puff_advantage(Tensor(a!) values, Tensor(b!) rewards, Tensor(c!) dones, Tensor(d!) importance, Tensor(e!) advantages, float gamma, float lambda, float rho_clip, float c_clip) -> ()"); } TORCH_LIBRARY_IMPL(pufferlib, CPU, m) { diff --git a/pufferlib/models.py b/pufferlib/models.py index 845cdd00fc..73eb55e214 100644 --- a/pufferlib/models.py +++ b/pufferlib/models.py @@ -21,7 +21,7 @@ class Default(nn.Module): the recurrent cell into encode_observations and put everything after into decode_actions. ''' - def __init__(self, env, hidden_size=128, use_p3o=False, p3o_horizon=32, use_diayn=False, diayn_skills=128): + def __init__(self, env, hidden_size=128): super().__init__() self.hidden_size = hidden_size self.is_multidiscrete = isinstance(env.single_action_space, @@ -38,7 +38,6 @@ def __init__(self, env, hidden_size=128, use_p3o=False, p3o_horizon=32, use_diay input_size = int(sum(np.prod(v.shape) for v in env.env.observation_space.values())) self.encoder = nn.Linear(input_size, self.hidden_size) else: - #self.encoder = nn.Linear(np.prod(env.single_observation_space.shape), hidden_size) self.encoder = torch.nn.Sequential( nn.Linear(np.prod(env.single_observation_space.shape), hidden_size), nn.GELU(), @@ -58,28 +57,8 @@ def __init__(self, env, hidden_size=128, use_p3o=False, p3o_horizon=32, use_diay self.decoder_logstd = nn.Parameter(torch.zeros( 1, env.single_action_space.shape[0])) - if use_diayn: - self.diayn_discriminator = nn.Sequential( - pufferlib.pytorch.layer_init(nn.Linear(hidden_size, hidden_size)), - nn.ReLU(), - pufferlib.pytorch.layer_init(nn.Linear(hidden_size, diayn_skills)), - ) - - self.use_p3o = use_p3o - self.p3o_horizon = p3o_horizon - if use_p3o: - self.value_mean = pufferlib.pytorch.layer_init( - nn.Linear(hidden_size, p3o_horizon), std=1) - self.value_logstd = nn.Parameter(torch.zeros(1, p3o_horizon)) - - #param = np.log10(np.arange(1, N+1)) - #param = 1 - np.exp(-np.sqrt(np.arange(N))) - #self.value_logstd = nn.Parameter(torch.tensor(param).view(1, N)) - #self.value_logstd = pufferlib.pytorch.layer_init( - # nn.Linear(hidden_size, 32), std=0.01) - else: - self.value = pufferlib.pytorch.layer_init( - nn.Linear(hidden_size, 1), std=1) + self.value = pufferlib.pytorch.layer_init( + nn.Linear(hidden_size, 1), std=1) def forward(self, observations, state=None): hidden = self.encode_observations(observations, state=state) @@ -114,15 +93,7 @@ def decode_actions(self, hidden): else: logits = self.decoder(hidden) - if self.use_p3o: - mean=self.value_mean(hidden) - values = pufferlib.namespace( - mean=mean, - std=torch.exp(torch.clamp(self.value_logstd, -10, 10)).expand_as(mean), - ) - else: - values = self.value(hidden) - + values = self.value(hidden) return logits, values class LSTMWrapper(nn.Module): diff --git a/pufferlib/pufferlib.cu b/pufferlib/pufferlib.cu index f2da160d55..c979fcf790 100644 --- a/pufferlib/pufferlib.cu +++ b/pufferlib/pufferlib.cu @@ -4,238 +4,10 @@ namespace pufferlib { -/* -__global__ void p3o_kernel( - float* reward_block, // [num_steps, horizon] - float* reward_mask, // [num_steps, horizon] - float* values_mean, // [num_steps, horizon] - float* values_std, // [num_steps, horizon] - float* buf, // [num_steps, horizon] - float* dones, // [num_steps] - float* rewards, // [num_steps] - float* advantages, // [num_steps] - int* bounds, // [num_steps] - int num_steps, - float r_std, - float puf, - int horizon -) { - int i = blockIdx.x * blockDim.x + threadIdx.x; - if (i >= num_steps) return; - - int k = 0; - for (int j = 0; j < horizon-1; j++) { - int t = i + j; - if (t >= num_steps - 1) { - break; - } - if (dones[t+1]) { - k++; - break; - } - k++; - } - - float gamma_max = 0.0f; - float n = 0.0f; - for (int j = k-1; j >= 0; j--) { - int idx = i * horizon + j; - n++; - - float vstd = values_std[idx]; - if (vstd == 0.0f) { - buf[idx] = 0.0f; - continue; - } - - float gamma = 1.0f / (vstd*vstd); - if (r_std != 0.0f) { - gamma -= puf/(r_std*r_std); - } - - if (gamma < 0.0f) { - gamma = 0.0f; - } - - if (gamma > gamma_max) { - gamma_max = gamma; - } - buf[idx] = gamma; - reward_mask[idx] = 1.0f; - } - - //float bootstrap = 0.0f; - //if (k == horizon-1) { - // bootstrap = buf[i*horizon + horizon - 1]*values_mean[i*horizon + horizon - 1]; - //} - - float R = 0.0f; - for (int j = 0; j <= k-1; j++) { - int t = i + j; - int idx = i * horizon + j; - float r = rewards[t+1]; - - float gamma = buf[idx]; - if (gamma_max > 0) { - gamma /= gamma_max; - } - - if (j >= 16 && values_std[idx] > 0.95*r_std) { - break; - } - - R += gamma * (r - values_mean[idx]); - reward_block[idx] = r; - buf[idx] = gamma; - } - - advantages[i] = R; - bounds[i] = k; -} - - -void compute_p3o(torch::Tensor reward_block, torch::Tensor reward_mask, - torch::Tensor values_mean, torch::Tensor values_std, torch::Tensor buf, - torch::Tensor dones, torch::Tensor rewards, torch::Tensor advantages, - torch::Tensor bounds, int num_steps, float vstd_max, float puf, - int horizon) { - - // TODO: Port from python - assert all(t.is_cuda for t in [reward_block, reward_mask, values_mean, values_std, - buf, dones, rewards, advantages, bounds]), "All tensors must be on GPU" - - # Ensure contiguous memory - tensors = [reward_block, reward_mask, values_mean, values_std, buf, dones, rewards, advantages, bounds] - for t in tensors: - t.contiguous() - assert t.is_cuda - - num_steps = rewards.shape[0] - - # Precompute vstd_min and vstd_max - #vstd_max = values_std.max().item() - #vstd_min = values_std.min().item() - - # Launch kernel - threads_per_block = 256 - assert num_steps % threads_per_block == 0 - blocks = (num_steps + threads_per_block - 1) // threads_per_block - - // Launch the kernel - int threads_per_block = 256; - int blocks = (num_steps + threads_per_block - 1) / threads_per_block; - - p3o_kernel<<>>( - reward_block.data_ptr(), - reward_mask.data_ptr(), - values_mean.data_ptr(), - values_std.data_ptr(), - buf.data_ptr(), - dones.data_ptr(), - rewards.data_ptr(), - advantages.data_ptr(), - bounds.data_ptr(), - num_steps, - vstd_max, - puf, - horizon - ); - - // Check for CUDA errors - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) { - throw std::runtime_error(cudaGetErrorString(err)); - } - return; -} - -// [num_steps, horizon] -__global__ void gae_kernel(float* values, float* rewards, float* dones, - float* advantages, float gamma, float gae_lambda, int num_steps, int horizon) { - int row = blockIdx.x*blockDim.x + threadIdx.x; - int offset = row*horizon; - gae_row(values + offset, rewards + offset, dones + offset, - advantages + offset, gamma, gae_lambda, horizon); -} - -torch::Tensor compute_gae(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, float gamma, float gae_lambda) { - int num_steps = values.size(0); - int horizon = values.size(1); - torch::Tensor advantages = gae_check(values, rewards, dones, num_steps, horizon); - TORCH_CHECK(values.is_cuda(), "All tensors must be on GPU"); - - int threads_per_block = 256; - int blocks = (num_steps + threads_per_block - 1) / threads_per_block; - assert(num_steps % threads_per_block == 0); - - gae_kernel<<>>( - values.data_ptr(), - rewards.data_ptr(), - dones.data_ptr(), - advantages.data_ptr(), - gamma, - gae_lambda, - num_steps, - horizon - ); - - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) { - throw std::runtime_error(cudaGetErrorString(err)); - } - - return advantages; -} - - // [num_steps, horizon] -__global__ void vtrace_kernel(float* values, float* rewards, float* dones, float* importance, - float* vs, float* advantages, float gamma, float rho_clip, float c_clip, int num_steps, int horizon) { - int row = blockIdx.x*blockDim.x + threadIdx.x; - int offset = row*horizon; - vtrace_row(values + offset, rewards + offset, dones + offset, - importance + offset, vs + offset, advantages + offset, gamma, rho_clip, c_clip, horizon); -} - -void compute_vtrace(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, - float gamma, float rho_clip, float c_clip) { - int num_steps = values.size(0); - int horizon = values.size(1); - vtrace_check(values, rewards, dones, importance, vs, advantages, num_steps, horizon); - TORCH_CHECK(values.is_cuda(), "All tensors must be on GPU"); - assert(horizon <= max_horizon); - - int threads_per_block = 128; - int blocks = (num_steps + threads_per_block - 1) / threads_per_block; - assert(num_steps % threads_per_block == 0); - - vtrace_kernel<<>>( - values.data_ptr(), - rewards.data_ptr(), - dones.data_ptr(), - importance.data_ptr(), - vs.data_ptr(), - advantages.data_ptr(), - gamma, - rho_clip, - c_clip, - num_steps, - horizon - ); - - cudaError_t err = cudaGetLastError(); - if (err != cudaSuccess) { - throw std::runtime_error(cudaGetErrorString(err)); - } -} -*/ - static const int max_horizon = 256; __host__ __device__ void puff_advantage_row_cuda(float* values, float* rewards, float* dones, - float* importance, float* vs, float* advantages, float gamma, float lambda, + float* importance, float* advantages, float gamma, float lambda, float rho_clip, float c_clip, int horizon) { - vs[horizon-1] = values[horizon-1]; float lastpufferlam = 0; for (int t = horizon-2; t >= 0; t--) { int t_next = t + 1; @@ -245,25 +17,17 @@ __host__ __device__ void puff_advantage_row_cuda(float* values, float* rewards, // TODO: t_next works and t doesn't. Check original formula float delta = rho_t*(rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]); lastpufferlam = delta + gamma*lambda*c_t*lastpufferlam*nextnonterminal; - - //float delta = rewards[t_next] + gamma*values[t_next]*nextnonterminal - values[t]; - //lastpufferlam = delta + gamma*lambda*lastpufferlam*nextnonterminal; - - advantages[t] = lastpufferlam; - vs[t] = advantages[t] + values[t]; - //advantages[t] = rho_t*(rewards[t] + gamma*vs[t_next]*nextnonterminal - values[t]); - //vs[t] = lastpufferlam + values[t]; } } void vtrace_check_cuda(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + torch::Tensor dones, torch::Tensor importance, torch::Tensor advantages, int num_steps, int horizon) { // Validate input tensors torch::Device device = values.device(); - for (const torch::Tensor& t : {values, rewards, dones, importance, vs, advantages}) { + for (const torch::Tensor& t : {values, rewards, dones, importance, advantages}) { TORCH_CHECK(t.dim() == 2, "Tensor must be 2D"); TORCH_CHECK(t.device() == device, "All tensors must be on same device"); TORCH_CHECK(t.size(0) == num_steps, "First dimension must match num_steps"); @@ -278,21 +42,21 @@ void vtrace_check_cuda(torch::Tensor values, torch::Tensor rewards, // [num_steps, horizon] -__global__ void puff_advantage_kernel(float* values, float* rewards, float* dones, float* importance, - float* vs, float* advantages, float gamma, float lambda, - float rho_clip, float c_clip, int num_steps, int horizon) { +__global__ void puff_advantage_kernel(float* values, float* rewards, + float* dones, float* importance, float* advantages, float gamma, + float lambda, float rho_clip, float c_clip, int num_steps, int horizon) { int row = blockIdx.x*blockDim.x + threadIdx.x; int offset = row*horizon; puff_advantage_row_cuda(values + offset, rewards + offset, dones + offset, - importance + offset, vs + offset, advantages + offset, gamma, lambda, rho_clip, c_clip, horizon); + importance + offset, advantages + offset, gamma, lambda, rho_clip, c_clip, horizon); } void compute_puff_advantage_cuda(torch::Tensor values, torch::Tensor rewards, - torch::Tensor dones, torch::Tensor importance, torch::Tensor vs, torch::Tensor advantages, + torch::Tensor dones, torch::Tensor importance, torch::Tensor advantages, double gamma, double lambda, double rho_clip, double c_clip) { int num_steps = values.size(0); int horizon = values.size(1); - vtrace_check_cuda(values, rewards, dones, importance, vs, advantages, num_steps, horizon); + vtrace_check_cuda(values, rewards, dones, importance, advantages, num_steps, horizon); TORCH_CHECK(values.is_cuda(), "All tensors must be on GPU"); assert(horizon <= max_horizon); @@ -308,7 +72,6 @@ void compute_puff_advantage_cuda(torch::Tensor values, torch::Tensor rewards, rewards.data_ptr(), dones.data_ptr(), importance.data_ptr(), - vs.data_ptr(), advantages.data_ptr(), gamma, lambda, diff --git a/setup.py b/setup.py index 4887b8ae56..79e748c241 100644 --- a/setup.py +++ b/setup.py @@ -442,8 +442,8 @@ def run(self): 'common': common, **environments, }, - ext_modules = c_extensions, - #cmdclass={"build_ext": cpp_extension.BuildExtension}, + ext_modules = torch_extensions, + cmdclass={"build_ext": cpp_extension.BuildExtension}, include_dirs=[numpy.get_include(), RAYLIB_NAME + '/include'], python_requires=">=3.9", license="MIT", From dd0a986c6a51a57d95029dd0ec0314d59480ea58 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 01:07:41 +0000 Subject: [PATCH 49/63] more refactor: --- clean_pufferl.py | 209 ++++++++++++++++++++--------------------------- 1 file changed, 90 insertions(+), 119 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index e270484352..a608a84e2c 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -83,7 +83,6 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.truncations = torch.zeros(segments, horizon, device=device) self.ratio = torch.ones(segments, horizon, device=device) self.importance = torch.ones(segments, horizon, device=device) - self.ep_uses = torch.zeros(segments, device=device, dtype=torch.int32) self.ep_lengths = torch.zeros(total_agents, device=device, dtype=torch.int32) self.ep_indices = torch.arange(total_agents, device=device, dtype=torch.int32) self.free_idx = total_agents @@ -168,23 +167,34 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.run_id = str(int(random.random() * 1e8)) # Initializations - self.global_step = 0 - self.uptime = 0 - self.epoch = 0 self.config = config self.vecenv = vecenv + self.epoch = 0 + self.global_step = 0 + self.last_log_step = 0 + self.last_log_time = time.time() self.start_time = time.time() self.utilization = Utilization() self.profile = Profile() self.stats = defaultdict(list) + self.last_stats = defaultdict(list) self.losses = {} # Dashboard - num_params = sum(p.numel() for p in policy.parameters() if p.requires_grad) - params, unit = abbreviate(num_params) - self.msg = f'Model Size: {params} {unit} parameters' + self.model_size = sum(p.numel() for p in policy.parameters() if p.requires_grad) self.print_dashboard(clear=True) + @property + def uptime(self): + return time.time() - self.start_time + + @property + def sps(self): + if self.global_step == self.last_log_step: + return 0 + + return (self.global_step - self.last_log_step) / (time.time() - self.last_log_time) + def evaluate(self): profile = self.profile epoch = self.epoch @@ -282,7 +292,6 @@ def evaluate(self): self.free_idx = self.total_agents self.ep_indices = torch.arange(self.total_agents, device=device, dtype=torch.int32) self.ep_lengths.zero_() - self.ep_uses.zero_() profile.end() return self.stats @@ -299,6 +308,7 @@ def train(self): clip_coef = config['clip_coef'] vf_clip = config['vf_clip_coef'] anneal_beta = b0 + (1 - b0)*a*self.epoch/self.total_epochs + self.ratio[:] = 1 for mb in range(self.total_minibatches): profile('train_misc', epoch, nest=True) @@ -318,7 +328,6 @@ def train(self): prio_probs = (prio_weights + 1e-6)/(prio_weights.sum() + 1e-6) idx = torch.multinomial(prio_probs, self.minibatch_segments) mb_prio = (self.segments*prio_probs[idx, None])**-anneal_beta - self.ep_uses[idx] += 1 mb_obs = self.observations[idx] mb_actions = self.actions[idx] mb_logprobs = self.logprobs[idx] @@ -331,15 +340,15 @@ def train(self): mb_advantages = advantages[idx] profile('train_forward', epoch) + if not config['use_rnn']: + mb_obs = mb_obs.reshape(-1, *self.vecenv.single_observation_space.shape) + state = dict( action=mb_actions, lstm_h=None, lstm_c=None, ) - if not config['use_rnn']: - mb_obs = mb_obs.reshape(-1, *self.vecenv.single_observation_space.shape) - # TODO: Currently only returning traj shaped value as a hack logits, newvalue = self.policy.forward_train(mb_obs, state) actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, @@ -353,20 +362,17 @@ def train(self): # TODO: Only do this if we are KL clipping? Saves 1-2% compute with torch.no_grad(): - # calculate approx_kl http://joschu.net/blog/kl-approx.html old_approx_kl = (-logratio).mean() approx_kl = ((ratio - 1) - logratio).mean() clipfrac = ((ratio - 1.0).abs() > config['clip_coef']).float().mean() # TODO: Do you need to do this? Policy hasn't changed - with torch.no_grad(): - adv = advantages[idx] - torch.ops.pufferlib.compute_puff_advantage(mb_values, mb_rewards, mb_terminals, - ratio, adv, config['gamma'], config['gae_lambda'], - config['vtrace_rho_clip'], config['vtrace_c_clip']) - + adv = advantages[idx] + torch.ops.pufferlib.compute_puff_advantage(mb_values, mb_rewards, mb_terminals, + ratio, adv, config['gamma'], config['gae_lambda'], + config['vtrace_rho_clip'], config['vtrace_c_clip']) adv = mb_advantages - adv = mb_prio * (adv - adv.mean()) / (adv.std() + 1e-8) + adv = mb_prio * (adv - adv.mean()) / (adv.std() + 1e-8) # TODO: Norm by full batch # Losses pg_loss1 = -adv * ratio @@ -385,9 +391,9 @@ def train(self): self.amp_context.__enter__() # TODO: Debug # This breaks vloss clipping? - with torch.no_grad(): - self.values[idx] = newvalue.float() + self.values[idx] = newvalue.detach().float() + # Logging profile('train_misc', epoch) losses['policy_loss'] += pg_loss.item() / self.total_minibatches losses['value_loss'] += v_loss.item() / self.total_minibatches @@ -397,6 +403,7 @@ def train(self): losses['clipfrac'] += clipfrac.item() / self.total_minibatches losses['importance'] += ratio.mean().item() / self.total_minibatches + # Learn on accumulated minibatches profile('learn', epoch) loss.backward() if (mb + 1) % self.accumulate_minibatches == 0: @@ -406,32 +413,28 @@ def train(self): # Reprioritize experience profile('train_misc', epoch) - self.max_uses = self.ep_uses.max().item() - self.mean_uses = self.ep_uses.float().mean().item() - self.ratio[:] = 1 - if config['anneal_lr']: self.scheduler.step() y_pred = self.values.flatten() - # TODO: Probably not updated y_true = advantages.flatten() + self.values.flatten() - var_y = y_true.var() explained_var = torch.nan if var_y == 0 else 1 - (y_true - y_pred).var() / var_y losses['explained_variance'] = explained_var.item() profile.end() - profile.clear() logs = None self.epoch += 1 done_training = self.global_step >= config['total_timesteps'] - if done_training or self.global_step == 0 or time.time() - self.start_time - self.uptime > 1: - self.uptime = time.time() - self.start_time + if done_training or self.global_step == 0 or time.time() > self.last_log_time + 0.25: logs = self.mean_and_log() self.losses = losses self.print_dashboard() + self.last_stats = self.stats self.stats = defaultdict(list) + self.last_log_time = time.time() + self.last_log_step = self.global_step + profile.clear() if self.epoch % config['checkpoint_interval'] == 0 or done_training: self.save_checkpoint() @@ -453,13 +456,11 @@ def mean_and_log(self): device = config['device'] agent_steps = int(dist_sum(self.global_step, device)) logs = { - #'SPS': dist_sum(self.profile.SPS, device), + 'SPS': dist_sum(self.sps, device), 'agent_steps': agent_steps, 'uptime': time.time() - self.start_time, 'epoch': int(dist_sum(self.epoch, device)), 'learning_rate': self.optimizer.param_groups[0]["lr"], - 'max_uses': self.max_uses, - 'mean_uses': self.mean_uses, **{f'environment/{k}': dist_mean(v, device) for k, v in self.stats.items()}, **{f'losses/{k}': dist_mean(v, device) for k, v in self.losses.items()}, **{f'performance/{k}': dist_sum(v['elapsed'], device) for k, v in self.profile}, @@ -467,8 +468,7 @@ def mean_and_log(self): if torch.distributed.is_initialized() and torch.distributed.get_rank() != 0: return logs - - if self.wandb: + elif self.wandb: self.wandb.log(logs) elif self.neptune: for k, v in logs.items(): @@ -531,62 +531,48 @@ def try_load_checkpoint(self): self.optimizer.load_state_dict(resume_state['optimizer_state_dict']) print(f'Loaded checkpoint {resume_state["model_name"]}') - def print_dashboard(self, clear=False, max_stats=[0]): - utilization = self.utilization + def print_dashboard(self, clear=False, idx=[0], + c1='[cyan]', c2='[white]', b1='[bright_cyan]', b2='[bright_white]'): profile = self.profile config = self.config console = Console() - if clear: - console.clear() - dashboard = Table(box=rich.box.ROUNDED, expand=True, show_header=False, border_style='bright_cyan') - - c1 = '[cyan]' - c2 = '[white]' - b1 = '[bright_cyan]' - b2 = '[bright_white]' - table = Table(box=None, expand=True, show_header=False) dashboard.add_row(table) - cpu_percent = np.mean(utilization.cpu_util) - dram_percent = np.mean(utilization.cpu_mem) - gpu_percent = np.mean(utilization.gpu_util) - vram_percent = np.mean(utilization.gpu_mem) + table.add_column(justify="left", width=30) table.add_column(justify="center", width=12) table.add_column(justify="center", width=12) table.add_column(justify="center", width=13) table.add_column(justify="right", width=13) + table.add_row( - f':blowfish: {b1}PufferLib {b2}2.0.0', - f'{c1}CPU: {b2}{cpu_percent:.1f}{c2}%', - f'{c1}GPU: {b2}{gpu_percent:.1f}{c2}%', - f'{c1}DRAM: {b2}{dram_percent:.1f}{c2}%', - f'{c1}VRAM: {b2}{vram_percent:.1f}{c2}%', + f'{b1}PufferLib {b2}2.0.0 {idx[0]*" "}:blowfish:', + f'{c1}CPU: {b2}{np.mean(self.utilization.cpu_util):.1f}{c2}%', + f'{c1}GPU: {b2}{np.mean(self.utilization.gpu_util):.1f}{c2}%', + f'{c1}DRAM: {b2}{np.mean(self.utilization.cpu_mem):.1f}{c2}%', + f'{c1}VRAM: {b2}{np.mean(self.utilization.gpu_mem):.1f}{c2}%', ) + idx[0] = (idx[0] - 1) % 10 s = Table(box=None, expand=True) - SPS = 0 - delta = profile.eval['delta'] + profile.train['delta'] + sps = self.sps remaining = 'A hair past a freckle' - if delta != 0: - SPS = config['batch_size'] / delta - remaining = duration((config['total_timesteps'] - self.global_step)/SPS, b2, c2) + if sps != 0: + remaining = duration((config['total_timesteps'] - self.global_step)/sps, b2, c2) - uptime = time.time() - self.start_time s.add_column(f"{c1}Summary", justify='left', vertical='top', width=10) s.add_column(f"{c1}Value", justify='right', vertical='top', width=14) s.add_row(f'{c2}Env', f'{b2}{config["env"]}') - steps, unit = abbreviate(self.global_step) - s.add_row(f'{c2}Steps', f'{b2}{steps:.1f}{c2}{unit}') - sps, unit = abbreviate(SPS) - s.add_row(f'{c2}SPS', f'{b2}{sps:.1f}{c2}{unit}') - epoch, unit = abbreviate(self.epoch) - s.add_row(f'{c2}Epoch', f'{b2}{epoch}{c2}{unit}') - s.add_row(f'{c2}Uptime', duration(uptime, b2, c2)) + s.add_row(f'{c2}Params', abbreviate(self.model_size, b2, c2)) + s.add_row(f'{c2}Steps', abbreviate(self.global_step, b2, c2)) + s.add_row(f'{c2}SPS', abbreviate(sps, b2, c2)) + s.add_row(f'{c2}Epoch', f'{b2}{self.epoch}') + s.add_row(f'{c2}Uptime', duration(self.uptime, b2, c2)) s.add_row(f'{c2}Remaining', remaining) + delta = profile.eval['delta'] + profile.train['delta'] p = Table(box=None, expand=True, show_header=False) p.add_column(f"{c1}Performance", justify="left", width=10) p.add_column(f"{c1}Time", justify="right", width=8) @@ -601,8 +587,6 @@ def print_dashboard(self, clear=False, max_stats=[0]): p.add_row(*fmt_perf(' Learn', c2, delta, profile.learn, b2, c2)) p.add_row(*fmt_perf(' Copy', c2, delta, profile.train_copy, b2, c2)) p.add_row(*fmt_perf(' Misc', c2, delta, profile.train_misc, b2, c2)) - if 'custom' in profile.profiles: - p.add_row(*fmt_perf(' Custom', c2, uptime, profile.custom, b2, c2)) l = Table(box=None, expand=True, ) l.add_column(f'{c1}Losses', justify="left", width=16) @@ -624,7 +608,7 @@ def print_dashboard(self, clear=False, max_stats=[0]): right.add_column(f"{c1}User Stats", justify="left", width=20) right.add_column(f"{c1}Value", justify="right", width=10) i = 0 - for metric, value in self.stats.items(): + for metric, value in (self.stats or self.last_stats).items(): try: # Discard non-numeric values int(value) except: @@ -636,21 +620,37 @@ def print_dashboard(self, clear=False, max_stats=[0]): if i == 30: break - for i in range(max_stats[0] - i): - u = left if i % 2 == 0 else right - u.add_row('', '') - - max_stats[0] = max(max_stats[0], i) - - table = Table(box=None, expand=True, pad_edge=False) - dashboard.add_row(table) - table.add_row(f' {c1}Message: {c2}{self.msg}') + if clear: + console.clear() with console.capture() as capture: console.print(dashboard) print('\033[0;0H' + capture.get()) +def abbreviate(num, b2, c2): + if num < 1e3: + return str(num) + elif num < 1e6: + return f'{num/1e3:.1f}K' + elif num < 1e9: + return f'{num/1e6:.1f}M' + elif num < 1e12: + return f'{num/1e9:.1f}B' + else: + return f'{num/1e12:.2f}T' + +def duration(seconds, b2, c2): + seconds = int(seconds) + h = seconds // 3600 + m = (seconds % 3600) // 60 + s = seconds % 60 + return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" + +def fmt_perf(name, color, delta_ref, prof, b2, c2): + percent = 0 if delta_ref == 0 else int(100*prof['delta']/delta_ref - 1e-5) + return f'{color}{name}', duration(prof['elapsed'], b2, c2), f'{b2}{percent:2d}{c2}%' + def dist_sum(value, device): if not torch.distributed.is_initialized(): return value @@ -683,7 +683,6 @@ def __call__(self, name, epoch, nest=False): torch.cuda.synchronize() tick = time.time() - if len(self.stack) != 0 and not nest: self.pop(tick) @@ -693,23 +692,18 @@ def __call__(self, name, epoch, nest=False): def pop(self, end): profile = self.profiles[self.stack.pop()] delta = end - profile['start'] - profile['buffer'] += delta profile['elapsed'] += delta - profile['calls'] += 1 + profile['delta'] += delta def end(self): torch.cuda.synchronize() end = time.time() - for i in range(len(self.stack)): self.pop(end) def clear(self): - for v in self.profiles.values(): - if v['buffer'] != 0: - v['delta'] = v['buffer'] - - v['buffer'] = 0 + for prof in self.profiles.values(): + prof['delta'] = 0 class Utilization(Thread): def __init__(self, delay=1, maxlen=20): @@ -718,9 +712,8 @@ def __init__(self, delay=1, maxlen=20): self.cpu_util = deque(maxlen=maxlen) self.gpu_util = deque(maxlen=maxlen) self.gpu_mem = deque(maxlen=maxlen) - - self.delay = delay self.stopped = False + self.delay = delay self.start() def run(self): @@ -735,35 +728,12 @@ def run(self): else: self.gpu_util.append(0) self.gpu_mem.append(0) + time.sleep(self.delay) def stop(self): self.stopped = True -def abbreviate(num): - if num < 1e3: - return num, '' - elif num < 1e6: - return num/1e3, 'k' - elif num < 1e9: - return num/1e6, 'm' - elif num < 1e12: - return num/1e9, 'b' - else: - return num/1e12, 't' - -def duration(seconds, b2, c2): - seconds = int(seconds) - h = seconds // 3600 - m = (seconds % 3600) // 60 - s = seconds % 60 - return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" - -def fmt_perf(name, color, delta_ref, prof, b2, c2): - percent = 0 if delta_ref == 0 else int(100*prof['delta']/delta_ref - 1e-5) - return f'{color}{name}', duration(prof['elapsed'], b2, c2), f'{b2}{percent:2d}{c2}%' - - def init_wandb(args, id=None, resume=True, tag=None): import wandb wandb.init( @@ -795,24 +765,25 @@ def init_neptune(args, id=None, resume=True, tag=None, mode="async"): ) except neptune.exceptions.NeptuneConnectionLostException: print("couldn't connect to neptune, logging in offline mode") - return init_neptune(args, name, id, resume, tag, mode="offline") + return init_neptune(args, id, resume, tag, mode="offline") return run +# TODO: Do we need this? def make_policy(env, policy_cls, rnn_cls, args): policy = policy_cls(env, **args['policy']) - args['rnn']['input_size'] = policy.hidden_size - args['rnn']['hidden_size'] = policy.hidden_size if rnn_cls is not None: policy = rnn_cls(env, policy, **args['rnn']) return policy.to(args['train']['device']) +# TODO: Is there a simpler interp def downsample_linear(arr, m): n = len(arr) x_old = np.linspace(0, 1, n) # Original indices normalized x_new = np.linspace(0, 1, m) # New indices normalized return np.interp(x_new, x_old, arr) +# TODO: All logs? def experiment(vecenv, policy, args): train_config = dict(**args['train'], env=env_name, tag=args['tag']) pufferl = CleanPuffeRL(train_config, vecenv, policy, neptune=args['neptune'], wandb=args['wandb']) From 3c28dd86d90d3d06887930a5a53f7fd37b298fa5 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 13:05:04 +0000 Subject: [PATCH 50/63] cleanup --- clean_pufferl.py | 10 ++++++---- config/ocean/pong.ini | 1 - 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index a608a84e2c..1b9ef2fa14 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -572,7 +572,7 @@ def print_dashboard(self, clear=False, idx=[0], s.add_row(f'{c2}Uptime', duration(self.uptime, b2, c2)) s.add_row(f'{c2}Remaining', remaining) - delta = profile.eval['delta'] + profile.train['delta'] + delta = profile.eval['buffer'] + profile.train['buffer'] p = Table(box=None, expand=True, show_header=False) p.add_column(f"{c1}Performance", justify="left", width=10) p.add_column(f"{c1}Time", justify="right", width=8) @@ -648,7 +648,7 @@ def duration(seconds, b2, c2): return f"{b2}{h}{c2}h {b2}{m}{c2}m {b2}{s}{c2}s" if h else f"{b2}{m}{c2}m {b2}{s}{c2}s" if m else f"{b2}{s}{c2}s" def fmt_perf(name, color, delta_ref, prof, b2, c2): - percent = 0 if delta_ref == 0 else int(100*prof['delta']/delta_ref - 1e-5) + percent = 0 if delta_ref == 0 else int(100*prof['buffer']/delta_ref - 1e-5) return f'{color}{name}', duration(prof['elapsed'], b2, c2), f'{b2}{percent:2d}{c2}%' def dist_sum(value, device): @@ -703,7 +703,9 @@ def end(self): def clear(self): for prof in self.profiles.values(): - prof['delta'] = 0 + if prof['delta'] > 0: + prof['buffer'] = prof['delta'] + prof['delta'] = 0 class Utilization(Thread): def __init__(self, delay=1, maxlen=20): @@ -860,7 +862,7 @@ def experiment(vecenv, policy, args): # Unpack to nested dict parsed = vars(parser.parse_args()) env_name = parsed.pop('env') - args = {} + args = defaultdict(dict) for key, value in parsed.items(): next = args for subkey in key.split('.'): diff --git a/config/ocean/pong.ini b/config/ocean/pong.ini index b5599f96f5..69d151c710 100644 --- a/config/ocean/pong.ini +++ b/config/ocean/pong.ini @@ -3,7 +3,6 @@ package = ocean env_name = puffer_pong policy_name = Policy rnn_name = Recurrent -vec = multiprocessing [env] num_envs = 4096 From 1865d6fdb269aaebeecfe3e81c2acc8515bfa977 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 15:10:46 +0000 Subject: [PATCH 51/63] temp --- clean_pufferl.py | 4 +++- config/ocean/breakout.ini | 13 ++++++++++--- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 1b9ef2fa14..d07d54f583 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -217,9 +217,11 @@ def evaluate(self): done_mask = d + t self.global_step += mask.sum() - profile('eval_copy', epoch) o = torch.as_tensor(o) + o = o.pin_memory() + profile('eval_copy', epoch) o_device = o.to(device, non_blocking=True) + profile('eval_misc', epoch) r = torch.as_tensor(r).to(device, non_blocking=True) d = torch.as_tensor(d).to(device, non_blocking=True) diff --git a/config/ocean/breakout.ini b/config/ocean/breakout.ini index 7ee6ef1c97..049bf5d822 100644 --- a/config/ocean/breakout.ini +++ b/config/ocean/breakout.ini @@ -4,8 +4,13 @@ env_name = puffer_breakout policy_name = Policy rnn_name = Recurrent +[vec] +num_workers = 16 +num_envs = 16 +batch_size = 8 + [env] -num_envs = 4096 +num_envs = 16384 [policy] hidden_size = 128 @@ -22,7 +27,7 @@ adam_beta1 = 0.99 adam_beta2 = 0.9999 adam_eps = 1e-14 -batch_size = 524288 +batch_size = 4194304 ent_coef = 0.025 gae_lambda = 0.85 @@ -31,7 +36,9 @@ gamma = 0.975 learning_rate = 0.01 max_grad_norm = 1.5 -minibatch_size = 16384 +#minibatch_size = 16384 +minibatch_size = 131072 +max_minibatch_size = 131072 prio_alpha = 0.0 # Doesn't matter From 98f6ac6cb459f7a169dea4e3bcd460fc23b090a2 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 17:14:51 +0000 Subject: [PATCH 52/63] nmmo3 test config --- config/ocean/nmmo3.ini | 15 +++++++++------ pufferlib/models.py | 1 - 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/config/ocean/nmmo3.ini b/config/ocean/nmmo3.ini index 4780006bc4..f5a33a5ee2 100644 --- a/config/ocean/nmmo3.ini +++ b/config/ocean/nmmo3.ini @@ -1,17 +1,21 @@ [base] package = ocean env_name = puffer_nmmo3 -vec = multiprocessing policy_name = NMMO3 rnn_name = NMMO3LSTM +[vec] +num_workers = 8 +num_envs = 8 +batch_size = 4 + [env] reward_combat_level = 1.0 reward_prof_level = 1.0 reward_item_level = 1.0 reward_market = 0.0 reward_death = -1.0 -num_envs = 4 +num_envs = 1 [train] total_timesteps = 107000000000 @@ -22,11 +26,10 @@ gae_lambda = 0.996005622445478 ent_coef = 0.01210084358004069 max_grad_norm = 0.6075578331947327 vf_coef = 0.3979089612467003 -# todo: run 500k, 64 horz -bptt_horizon = 32 -batch_size = 262144 +bptt_horizon = 64 +batch_size = 524288 minibatch_size = 32768 -compile = False +max_minibatch_size = 32768 [sweep] metric = min_comb_prof diff --git a/pufferlib/models.py b/pufferlib/models.py index 73eb55e214..0b76fe310e 100644 --- a/pufferlib/models.py +++ b/pufferlib/models.py @@ -62,7 +62,6 @@ def __init__(self, env, hidden_size=128): def forward(self, observations, state=None): hidden = self.encode_observations(observations, state=state) - state.hidden = hidden logits, values = self.decode_actions(hidden) return logits, values From 547caaa43c0bf6108177a88cbbf206ad578b41ea Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 17:19:04 +0000 Subject: [PATCH 53/63] nmmo3 policy --- pufferlib/ocean/torch.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pufferlib/ocean/torch.py b/pufferlib/ocean/torch.py index a3baa8e7b2..305314bea0 100644 --- a/pufferlib/ocean/torch.py +++ b/pufferlib/ocean/torch.py @@ -37,9 +37,9 @@ def __init__(self, env, hidden_size=512, output_size=512, **kwargs): self.is_continuous = False self.map_2d = nn.Sequential( - pufferlib.pytorch.layer_init(nn.Conv2d(self.multihot_dim, 256, 5, stride=3)), + pufferlib.pytorch.layer_init(nn.Conv2d(self.multihot_dim, 128, 5, stride=3)), nn.ReLU(), - pufferlib.pytorch.layer_init(nn.Conv2d(256, 256, 3, stride=1)), + pufferlib.pytorch.layer_init(nn.Conv2d(128, 128, 3, stride=1)), nn.Flatten(), ) @@ -49,7 +49,7 @@ def __init__(self, env, hidden_size=512, output_size=512, **kwargs): ) self.proj = nn.Sequential( - pufferlib.pytorch.layer_init(nn.Linear(2073, hidden_size)), + pufferlib.pytorch.layer_init(nn.Linear(1817, hidden_size)), nn.ReLU(), ) From 625349300eb9c5065ad560d179ff946e5d74687f Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 18:05:32 +0000 Subject: [PATCH 54/63] Auto batch size --- clean_pufferl.py | 10 +++++++--- config/default.ini | 4 ++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index d07d54f583..41a3b235b5 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -57,10 +57,14 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): self.total_agents = total_agents # Experience - batch_size = config['batch_size'] - if config['bptt_horizon'] == 'auto': - config['bptt_horizon'] = batch_size // total_agents + if config['batch_size'] == 'auto' and config['bptt_horizon'] == 'auto': + raise pufferlib.APIUsageError('Must specify batch_size or bptt_horizon') + elif config['batch_size'] == 'auto': + config['batch_size'] = total_agents * config['bptt_horizon'] + elif config['bptt_horizon'] == 'auto': + config['bptt_horizon'] = config['batch_size'] // total_agents + batch_size = config['batch_size'] horizon = config['bptt_horizon'] segments = batch_size // horizon self.segments = segments diff --git a/config/default.ini b/config/default.ini index 24460e1c7c..00d73bb796 100644 --- a/config/default.ini +++ b/config/default.ini @@ -47,12 +47,12 @@ adam_eps = 1e-12 data_dir = experiments checkpoint_interval = 200 -batch_size = 524288 +batch_size = auto minibatch_size = 8192 replay_factor = 0.0 # Accumulate gradients above this size max_minibatch_size = 32768 -bptt_horizon = auto +bptt_horizon = 64 compile = False compile_mode = max-autotune-no-cudagraphs compile_fullgraph = True From 2d1b8748519a3a5ce84afde439c30c3d7d662684 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 18:26:28 +0000 Subject: [PATCH 55/63] speedrun --- config/default.ini | 56 +++++++++++++++------------- config/ocean/breakout.ini | 12 ++---- config/ocean/pong.ini | 7 +++- pufferlib/ocean/breakout/breakout.c | 37 +++++++++++++++++- pufferlib/ocean/breakout/breakout.py | 2 +- pufferlib/ocean/pong/pong.c | 40 +++++++++++++++++++- pufferlib/ocean/pong/pong.py | 4 +- pufferlib/vector.py | 13 ++++++- setup.py | 4 +- 9 files changed, 129 insertions(+), 46 deletions(-) diff --git a/config/default.ini b/config/default.ini index 00d73bb796..2816695409 100644 --- a/config/default.ini +++ b/config/default.ini @@ -8,8 +8,8 @@ max_suggestion_cost = 3600 [vec] backend = Multiprocessing num_envs = 2 -num_workers = 2 -batch_size = 1 +num_workers = auto +batch_size = auto zero_copy = True seed = 42 @@ -49,7 +49,7 @@ data_dir = experiments checkpoint_interval = 200 batch_size = auto minibatch_size = 8192 -replay_factor = 0.0 + # Accumulate gradients above this size max_minibatch_size = 32768 bptt_horizon = 64 @@ -57,12 +57,9 @@ compile = False compile_mode = max-autotune-no-cudagraphs compile_fullgraph = True -use_vtrace = False vtrace_rho_clip = 1.0 vtrace_c_clip = 1.0 -use_puff_advantage = True - prio_alpha = 0.6 prio_beta0 = 0.4 @@ -71,6 +68,14 @@ method = Protein metric = score goal = maximize +[sweep.vec.num_envs] +distribution = uniform_pow2 +min = 1 +max = 8 +mean = 2 +scale = auto + +# TODO: Elim from base [sweep.train.total_timesteps] distribution = log_normal min = 5e7 @@ -78,20 +83,19 @@ max = 1e10 mean = 1e8 scale = time -#[sweep.train.batch_size] -#distribution = uniform_pow2 -#min = 131072 -#max = 2097152 -#mean = 524288 -#scale = auto - -# TODO: Mini minibatch optim is lower -#[sweep.train.minibatch_size] -#distribution = uniform_pow2 -#min = 16384 -#max = 131072 -#mean = 8192 -#scale = auto +[sweep.train.bptt_horizon] +distribution = int_uniform +min = 16 +max = 64 +mean = 64 +scale = auto + +[sweep.train.minibatch_size] +distribution = uniform_pow2 +min = 8192 +max = 131072 +mean = 32768 +scale = auto [sweep.train.learning_rate] distribution = log_normal @@ -121,12 +125,12 @@ mean = 0.95 max = 0.995 scale = auto -#[sweep.train.update_epochs] -#distribution = int_uniform -#min = 1 -#max = 4 -#mean = 1 -#scale = 1.0 +[sweep.train.update_epochs] +distribution = int_uniform +min = 1 +max = 4 +mean = 1 +scale = 1.0 [sweep.train.clip_coef] distribution = uniform diff --git a/config/ocean/breakout.ini b/config/ocean/breakout.ini index 049bf5d822..7c6bd186f6 100644 --- a/config/ocean/breakout.ini +++ b/config/ocean/breakout.ini @@ -5,12 +5,10 @@ policy_name = Policy rnn_name = Recurrent [vec] -num_workers = 16 -num_envs = 16 -batch_size = 8 +num_envs = 2 [env] -num_envs = 16384 +num_envs = 4096 [policy] hidden_size = 128 @@ -36,9 +34,7 @@ gamma = 0.975 learning_rate = 0.01 max_grad_norm = 1.5 -#minibatch_size = 16384 -minibatch_size = 131072 -max_minibatch_size = 131072 +minibatch_size = 16384 prio_alpha = 0.0 # Doesn't matter @@ -52,6 +48,6 @@ vf_coef = 1.3 [sweep.train.total_timesteps] distribution = log_normal min = 2e7 -max = 1e8 +max = 5e8 mean = 8e7 scale = auto diff --git a/config/ocean/pong.ini b/config/ocean/pong.ini index 69d151c710..bcacb4f9ef 100644 --- a/config/ocean/pong.ini +++ b/config/ocean/pong.ini @@ -4,13 +4,16 @@ env_name = puffer_pong policy_name = Policy rnn_name = Recurrent +[vec] +num_envs = 2 + [env] num_envs = 4096 [train] -total_timesteps = 80_000_000 +total_timesteps = 500_000_000 learning_rate = 0.05 -minibatch_size = 32768 +batch_size = auto [sweep.train.total_timesteps] distribution = log_normal diff --git a/pufferlib/ocean/breakout/breakout.c b/pufferlib/ocean/breakout/breakout.c index 1785ebd215..a7c79ada53 100644 --- a/pufferlib/ocean/breakout/breakout.c +++ b/pufferlib/ocean/breakout/breakout.c @@ -2,7 +2,7 @@ #include "breakout.h" #include "puffernet.h" -int main() { +void demo() { Weights* weights = load_weights("resources/breakout_weights.bin", 147972); LinearLSTM* net = make_linearlstm(weights, 1, 119, 3); @@ -51,3 +51,38 @@ int main() { free_allocated(&env); close_client(env.client); } + +void test_performance(int timeout) { + Breakout env = { + .width = 512, + .height = 512, + .paddle_width = 20, + .paddle_height = 70, + .ball_width = 10, + .ball_height = 15, + .brick_width = 10, + .brick_height = 10, + .brick_rows = 5, + .brick_cols = 10, + .continuous = 0, + }; + allocate(&env); + c_reset(&env); + + int start = time(NULL); + int num_steps = 0; + while (time(NULL) - start < timeout) { + env.actions[0] = rand() % 3; + c_step(&env); + num_steps++; + } + + int end = time(NULL); + float sps = num_steps / (end - start); + printf("Test Environment SPS: %f\n", sps); + free_allocated(&env); +} + +int main() { + test_performance(10); +} diff --git a/pufferlib/ocean/breakout/breakout.py b/pufferlib/ocean/breakout/breakout.py index 59300888c9..f482701a8c 100644 --- a/pufferlib/ocean/breakout/breakout.py +++ b/pufferlib/ocean/breakout/breakout.py @@ -46,7 +46,7 @@ def __init__(self, num_envs=1, render_mode=None, brick_cols=brick_cols, continuous=continuous ) - def reset(self, seed=None): + def reset(self, seed=0): binding.vec_reset(self.c_envs, seed) self.tick = 0 return self.observations, [] diff --git a/pufferlib/ocean/pong/pong.c b/pufferlib/ocean/pong/pong.c index e3c104ab0c..fa1c6f5981 100644 --- a/pufferlib/ocean/pong/pong.c +++ b/pufferlib/ocean/pong/pong.c @@ -1,7 +1,8 @@ +#include #include "pong.h" #include "puffernet.h" -int main() { +void demo() { Weights* weights = load_weights("resources/pong_weights.bin", 133764); LinearLSTM* net = make_linearlstm(weights, 1, 8, 3); @@ -56,3 +57,40 @@ int main() { close_client(env.client); } +void test_performance(int timeout) { + Pong env = { + .width = 500, + .height = 640, + .paddle_width = 20, + .paddle_height = 70, + .ball_width = 32, + .ball_height = 32, + .paddle_speed = 8, + .ball_initial_speed_x = 10, + .ball_initial_speed_y = 1, + .ball_speed_y_increment = 3, + .ball_max_speed_y = 13, + .max_score = 21, + .frameskip = 1, + .continuous = 0, + }; + allocate(&env); + c_reset(&env); + + int start = time(NULL); + int num_steps = 0; + while (time(NULL) - start < timeout) { + env.actions[0] = rand() % 3; + c_step(&env); + num_steps++; + } + + int end = time(NULL); + float sps = num_steps / (end - start); + printf("Test Environment SPS: %f\n", sps); + free_allocated(&env); +} + +int main() { + test_performance(10); +} diff --git a/pufferlib/ocean/pong/pong.py b/pufferlib/ocean/pong/pong.py index affcd86cd2..1a37a693be 100644 --- a/pufferlib/ocean/pong/pong.py +++ b/pufferlib/ocean/pong/pong.py @@ -53,7 +53,7 @@ def __init__(self, num_envs=1, render_mode=None, max_score=max_score, frameskip=frameskip, continuous=continuous ) - def reset(self, seed=None): + def reset(self, seed=0): binding.vec_reset(self.c_envs, seed) self.tick = 0 return self.observations, [] @@ -80,7 +80,6 @@ def render(self): def close(self): binding.vec_close(self.c_envs) -from pufferlib.ocean.pong.cy_pong import CyPong #from cy_pong import CyPong class CythonPong(pufferlib.PufferEnv): def __init__(self, num_envs=1, render_mode=None, @@ -158,4 +157,3 @@ def test_performance(cls, timeout=10, atn_cache=1024): if __name__ == '__main__': test_performance(Pong) - test_performance(CythonPong) diff --git a/pufferlib/vector.py b/pufferlib/vector.py index d7d767ced6..295f575302 100644 --- a/pufferlib/vector.py +++ b/pufferlib/vector.py @@ -650,13 +650,22 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer return vecenv if 'num_workers' in kwargs: - num_workers = kwargs['num_workers'] + if kwargs['num_workers'] == 'auto': + kwargs['num_workers'] = num_envs + + # TODO: None? - envs_per_worker = num_envs / num_workers + envs_per_worker = num_envs / kwargs['num_workers'] if envs_per_worker != int(envs_per_worker): raise pufferlib.APIUsageError('num_envs must be divisible by num_workers') if 'batch_size' in kwargs: + if kwargs['batch_size'] == 'auto': + if num_envs == 1: + kwargs['batch_size'] = 1 + else: + kwargs['batch_size'] = num_envs // 2 + batch_size = kwargs['batch_size'] if batch_size is None: batch_size = num_envs diff --git a/setup.py b/setup.py index 79e748c241..4887b8ae56 100644 --- a/setup.py +++ b/setup.py @@ -442,8 +442,8 @@ def run(self): 'common': common, **environments, }, - ext_modules = torch_extensions, - cmdclass={"build_ext": cpp_extension.BuildExtension}, + ext_modules = c_extensions, + #cmdclass={"build_ext": cpp_extension.BuildExtension}, include_dirs=[numpy.get_include(), RAYLIB_NAME + '/include'], python_requires=">=3.9", license="MIT", From 8557e4da3e7b1920ac87a1d8cbd7888f645486f1 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 18:37:43 +0000 Subject: [PATCH 56/63] minor --- clean_pufferl.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 41a3b235b5..cb01c03135 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -804,8 +804,11 @@ def experiment(vecenv, policy, args): all_logs.append(logs) vecenv.async_reset(train_config['seed']) - for _ in range(10): + i = 0 + stats = {} + while i < 10 and not stats: stats = pufferl.evaluate() + i += 1 logs = pufferl.mean_and_log() if logs is not None: From 3bbef95a2bead541b3f9eee83e6950943b1ba25a Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 18:49:01 +0000 Subject: [PATCH 57/63] remove namespace --- config/ocean/breakout.ini | 13 +++--- pufferlib/emulation.py | 8 ++-- pufferlib/pufferlib.py | 70 +++-------------------------- pufferlib/pytorch.py | 2 +- pufferlib/vector.py | 92 ++++++++++++++------------------------- 5 files changed, 49 insertions(+), 136 deletions(-) diff --git a/config/ocean/breakout.ini b/config/ocean/breakout.ini index 049bf5d822..d39030e877 100644 --- a/config/ocean/breakout.ini +++ b/config/ocean/breakout.ini @@ -5,12 +5,12 @@ policy_name = Policy rnn_name = Recurrent [vec] -num_workers = 16 -num_envs = 16 -batch_size = 8 +num_workers = 2 +num_envs = 2 +batch_size = 1 [env] -num_envs = 16384 +num_envs = 4096 [policy] hidden_size = 128 @@ -27,7 +27,6 @@ adam_beta1 = 0.99 adam_beta2 = 0.9999 adam_eps = 1e-14 -batch_size = 4194304 ent_coef = 0.025 gae_lambda = 0.85 @@ -36,9 +35,7 @@ gamma = 0.975 learning_rate = 0.01 max_grad_norm = 1.5 -#minibatch_size = 16384 -minibatch_size = 131072 -max_minibatch_size = 131072 +minibatch_size = 16384 prio_alpha = 0.0 # Doesn't matter diff --git a/pufferlib/emulation.py b/pufferlib/emulation.py index f52261771b..844104fc68 100644 --- a/pufferlib/emulation.py +++ b/pufferlib/emulation.py @@ -158,9 +158,9 @@ def __init__(self, env=None, env_creator=None, env_args=[], env_kwargs={}, buf=N self.is_obs_emulated = self.single_observation_space is not self.env.observation_space self.is_atn_emulated = self.single_action_space is not self.env.action_space - self.emulated = pufferlib.namespace( - observation_dtype = self.observation_space.dtype, - emulated_observation_dtype = self.obs_dtype, + self.emulated = dict( + observation_dtype=self.observation_space.dtype, + emulated_observation_dtype=self.obs_dtype, ) self.render_modes = 'human rgb_array'.split() @@ -260,7 +260,7 @@ def __init__(self, env=None, env_creator=None, env_args=[], buf=None, env_kwargs emulate_action_space(self.env_single_action_space)) self.is_obs_emulated = self.single_observation_space is not self.env_single_observation_space self.is_atn_emulated = self.single_action_space is not self.env_single_action_space - self.emulated = pufferlib.namespace( + self.emulated = dict( observation_dtype = self.single_observation_space.dtype, emulated_observation_dtype = self.obs_dtype, ) diff --git a/pufferlib/pufferlib.py b/pufferlib/pufferlib.py index cdfed44f90..8c1689071a 100644 --- a/pufferlib/pufferlib.py +++ b/pufferlib/pufferlib.py @@ -35,12 +35,12 @@ def set_buffers(env, buf=None): else: env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=np.int32) else: - env.observations = buf.observations - env.rewards = buf.rewards - env.terminals = buf.terminals - env.truncations = buf.truncations - env.masks = buf.masks - env.actions = buf.actions + env.observations = buf['observations'] + env.rewards = buf['rewards'] + env.terminals = buf['terminals'] + env.truncations = buf['truncations'] + env.masks = buf['masks'] + env.actions = buf['actions'] class PufferEnv: def __init__(self, buf=None): @@ -363,64 +363,6 @@ def step(self, action): def close(self): self.env.close() -### Namespace -def __getitem__(self, key): - return self.__dict__[key] - -def __setitem__(self, key, value): - self.__dict__[key] = value - -def keys(self): - return self.__dict__.keys() - -def values(self): - return self.__dict__.values() - -def items(self): - return self.__dict__.items() - -def __iter__(self): - return iter(self.__dict__) - -def __len__(self): - return len(self.__dict__) - -class Namespace(SimpleNamespace, Mapping): - __getitem__ = __getitem__ - __setitem__ = __setitem__ - __iter__ = __iter__ - __len__ = __len__ - keys = keys - values = values - items = items - -def dataclass(cls): - # Safely get annotations - annotations = getattr(cls, '__annotations__', {}) - - # Combine both annotated and non-annotated fields - all_fields = {**{k: None for k in annotations.keys()}, **cls.__dict__} - all_fields = {k: v for k, v in all_fields.items() if not callable(v) and not k.startswith('__')} - - def __init__(self, **kwargs): - for field, default_value in all_fields.items(): - setattr(self, field, kwargs.get(field, default_value)) - - cls.__init__ = __init__ - setattr(cls, "__getitem__", __getitem__) - setattr(cls, "__setitem__", __setitem__) - setattr(cls, "__iter__", __iter__) - setattr(cls, "__len__", __len__) - setattr(cls, "keys", keys) - setattr(cls, "values", values) - setattr(cls, "items", items) - return cls - -def namespace(self=None, **kwargs): - if self is None: - return Namespace(**kwargs) - self.__dict__.update(kwargs) - ### Wrappers class PettingZooTruncatedWrapper: def __init__(self, env): diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index 67e70f186c..1a7084191d 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -47,7 +47,7 @@ # TODO: handle discrete obs # Spend some time trying to break this fn with differnt obs -def nativize_dtype(emulated: pufferlib.namespace) -> NativeDType: +def nativize_dtype(emulated) -> NativeDType: # sample dtype - the dtype of what we obtain from the environment (usually bytes) sample_dtype: np.dtype = emulated.observation_dtype # structured dtype - the gym.Space converted numpy dtype diff --git a/pufferlib/vector.py b/pufferlib/vector.py index d7d767ced6..687a43f65a 100644 --- a/pufferlib/vector.py +++ b/pufferlib/vector.py @@ -74,7 +74,7 @@ def __init__(self, env_creators, env_args, env_kwargs, num_envs, buf=None, seed= ptr = 0 for i in range(num_envs): end = ptr + self.driver_env.num_agents - buf_i = pufferlib.namespace( + buf_i = dict( observations=self.observations[ptr:end], rewards=self.rewards[ptr:end], terminals=self.terminals[ptr:end], @@ -175,25 +175,25 @@ def _worker_process(env_creators, env_args, env_kwargs, obs_shape, obs_dtype, at # Environments read and write directly to shared memory shape = (num_workers, num_envs*num_agents) atn_arr = np.ndarray((*shape, *atn_shape), - dtype=atn_dtype, buffer=shm.actions)[worker_idx] - buf = pufferlib.namespace( + dtype=atn_dtype, buffer=shm['actions'])[worker_idx] + buf = dict( observations=np.ndarray((*shape, *obs_shape), - dtype=obs_dtype, buffer=shm.observations)[worker_idx], - rewards=np.ndarray(shape, dtype=np.float32, buffer=shm.rewards)[worker_idx], - terminals=np.ndarray(shape, dtype=bool, buffer=shm.terminals)[worker_idx], - truncations=np.ndarray(shape, dtype=bool, buffer=shm.truncateds)[worker_idx], - masks=np.ndarray(shape, dtype=bool, buffer=shm.masks)[worker_idx], + dtype=obs_dtype, buffer=shm['observations'])[worker_idx], + rewards=np.ndarray(shape, dtype=np.float32, buffer=shm['rewards'])[worker_idx], + terminals=np.ndarray(shape, dtype=bool, buffer=shm['terminals'])[worker_idx], + truncations=np.ndarray(shape, dtype=bool, buffer=shm['truncateds'])[worker_idx], + masks=np.ndarray(shape, dtype=bool, buffer=shm['masks'])[worker_idx], actions=atn_arr, ) - buf.masks[:] = True + buf['masks'][:] = True if is_native and num_envs == 1: envs = env_creators[0](*env_args[0], **env_kwargs[0], buf=buf, seed=seed) else: envs = Serial(env_creators, env_args, env_kwargs, num_envs, buf=buf, seed=seed*num_envs) - semaphores=np.ndarray(num_workers, dtype=np.uint8, buffer=shm.semaphores) - notify=np.ndarray(num_workers, dtype=bool, buffer=shm.notify) + semaphores=np.ndarray(num_workers, dtype=np.uint8, buffer=shm['semaphores']) + notify=np.ndarray(num_workers, dtype=bool, buffer=shm['notify']) start = time.time() while True: if notify[worker_idx]: @@ -297,7 +297,7 @@ def __init__(self, env_creators, env_args, env_kwargs, from multiprocessing import RawArray, set_start_method # Mac breaks without setting fork... but setting it breaks sweeps on 2nd run #set_start_method('fork') - self.shm = pufferlib.namespace( + self.shm = dict( observations=RawArray(obs_ctype, num_agents * int(np.prod(obs_shape))), actions=RawArray(atn_ctype, num_agents * int(np.prod(atn_shape))), rewards=RawArray('f', num_agents), @@ -311,18 +311,18 @@ def __init__(self, env_creators, env_args, env_kwargs, self.obs_batch_shape = (self.agents_per_batch, *obs_shape) self.atn_batch_shape = (self.workers_per_batch, agents_per_worker, *atn_shape) self.actions = np.ndarray((*shape, *atn_shape), - dtype=atn_dtype, buffer=self.shm.actions) - self.buf = pufferlib.namespace( + dtype=atn_dtype, buffer=self.shm['actions']) + self.buf = dict( observations=np.ndarray((*shape, *obs_shape), - dtype=obs_dtype, buffer=self.shm.observations), - rewards=np.ndarray(shape, dtype=np.float32, buffer=self.shm.rewards), - terminals=np.ndarray(shape, dtype=bool, buffer=self.shm.terminals), - truncations=np.ndarray(shape, dtype=bool, buffer=self.shm.truncateds), - masks=np.ndarray(shape, dtype=bool, buffer=self.shm.masks), - semaphores=np.ndarray(num_workers, dtype=np.uint8, buffer=self.shm.semaphores), - notify=np.ndarray(num_workers, dtype=bool, buffer=self.shm.notify), + dtype=obs_dtype, buffer=self.shm['observations']), + rewards=np.ndarray(shape, dtype=np.float32, buffer=self.shm['rewards']), + terminals=np.ndarray(shape, dtype=bool, buffer=self.shm['terminals']), + truncations=np.ndarray(shape, dtype=bool, buffer=self.shm['truncateds']), + masks=np.ndarray(shape, dtype=bool, buffer=self.shm['masks']), + semaphores=np.ndarray(num_workers, dtype=np.uint8, buffer=self.shm['semaphores']), + notify=np.ndarray(num_workers, dtype=bool, buffer=self.shm['notify']), ) - self.buf.semaphores[:] = MAIN + self.buf['semaphores'][:] = MAIN from multiprocessing import Pipe, Process self.send_pipes, w_recv_pipes = zip(*[Pipe() for _ in range(num_workers)]) @@ -356,13 +356,13 @@ def recv(self): # Bandaid patch for new experience buffer desync if self.sync_traj: worker = self.waiting_workers[0] - sem = self.buf.semaphores[worker] + sem = self.buf['semaphores'][worker] if sem >= MAIN: self.waiting_workers.pop(0) self.ready_workers.append(worker) else: worker = self.waiting_workers.pop(0) - sem = self.buf.semaphores[worker] + sem = self.buf['semaphores'][worker] if sem >= MAIN: self.ready_workers.append(worker) else: @@ -424,10 +424,10 @@ def recv(self): self.w_slice = w_slice buf = self.buf - o = buf.observations[w_slice].reshape(self.obs_batch_shape) - r = buf.rewards[w_slice].ravel() - d = buf.terminals[w_slice].ravel() - t = buf.truncations[w_slice].ravel() + o = buf['observations'][w_slice].reshape(self.obs_batch_shape) + r = buf['rewards'][w_slice].ravel() + d = buf['terminals'][w_slice].ravel() + t = buf['truncations'][w_slice].ravel() infos = [] for i in s_range: @@ -436,7 +436,7 @@ def recv(self): self.infos[i] = [] agent_ids = self.agent_ids[w_slice].ravel() - m = buf.masks[w_slice].ravel() + m = buf['masks'][w_slice].ravel() self.batch_mask = m return o, r, d, t, infos, agent_ids, m @@ -447,7 +447,7 @@ def send(self, actions): idxs = self.w_slice self.actions[idxs] = actions - self.buf.semaphores[idxs] = STEP + self.buf['semaphores'][idxs] = STEP def async_reset(self, seed=0): self.flag = RECV @@ -459,42 +459,16 @@ def async_reset(self, seed=0): self.waiting_workers = list(range(self.num_workers)) self.infos = [[] for _ in range(self.num_workers)] - self.buf.semaphores[:] = RESET + self.buf['semaphores'][:] = RESET for i in range(self.num_workers): start = i*self.envs_per_worker end = (i+1)*self.envs_per_worker self.send_pipes[i].send(seed+i) def notify(self): - self.buf.notify[:] = True + self.buf['notify'][:] = True def close(self): - ''' - while self.waiting_workers: - worker = self.waiting_workers.pop(0) - sem = self.buf.semaphores[worker] - if sem >= MAIN: - self.ready_workers.append(worker) - if sem == INFO: - self.recv_pipes[worker].recv() - else: - self.waiting_workers.append(worker) - - self.buf.semaphores[:] = CLOSE - self.waiting_workers = list(range(self.num_workers)) - - while self.waiting_workers: - worker = self.waiting_workers.pop(0) - sem = self.buf.semaphores[worker] - if sem >= MAIN: - self.ready_workers.append(worker) - if sem == INFO: - self.recv_pipes[worker].recv() - - else: - self.waiting_workers.append(worker) - ''' - for p in self.processes: p.terminate() @@ -691,7 +665,7 @@ def make(env_creator_or_creators, env_args=None, env_kwargs=None, backend=Puffer raise pufferlib.APIUsageError('env_creators must be a list of callables') if not isinstance(env_args[i], (list, tuple)): raise pufferlib.APIUsageError('env_args must be a list of lists or tuples') - if not isinstance(env_kwargs[i], (dict, pufferlib.Namespace)): + if not isinstance(env_kwargs[i], dict): raise pufferlib.APIUsageError('env_kwargs must be a list of dictionaries') # Keeps batch size consistent when debugging with Serial backend From 223a132c2b7418795f0bd3649e2c803619af7abc Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 20:12:15 +0000 Subject: [PATCH 58/63] Initial env bind tests --- pufferlib/ocean/env_binding.h | 49 ++++++++++---- tests/test_env_binding.py | 116 ++++++++++++++++++++++++++++++++++ 2 files changed, 154 insertions(+), 11 deletions(-) create mode 100644 tests/test_env_binding.py diff --git a/pufferlib/ocean/env_binding.h b/pufferlib/ocean/env_binding.h index 2230c53692..e270b48305 100644 --- a/pufferlib/ocean/env_binding.h +++ b/pufferlib/ocean/env_binding.h @@ -142,18 +142,22 @@ static PyObject* env_init(PyObject* self, PyObject* args, PyObject* kwargs) { Py_DECREF(py_seed); PyObject* empty_args = PyTuple_New(0); - if (my_init(env, empty_args, kwargs)) { - //PyErr_SetString(PyExc_TypeError, "env_init failed"); - Py_DECREF(kwargs); + my_init(env, empty_args, kwargs); + Py_DECREF(kwargs); + if (PyErr_Occurred()) { return NULL; } - Py_DECREF(kwargs); return PyLong_FromVoidPtr(env); } // Python function to reset the environment static PyObject* env_reset(PyObject* self, PyObject* args) { + if (PyTuple_Size(args) != 2) { + PyErr_SetString(PyExc_TypeError, "env_reset requires 2 arguments"); + return NULL; + } + Env* env = unpack_env(args); if (!env){ return NULL; @@ -165,6 +169,12 @@ static PyObject* env_reset(PyObject* self, PyObject* args) { // Python function to step the environment static PyObject* env_step(PyObject* self, PyObject* args) { + int num_args = PyTuple_Size(args); + if (num_args != 1) { + PyErr_SetString(PyExc_TypeError, "vec_render requires 1 argument"); + return NULL; + } + Env* env = unpack_env(args); if (!env){ return NULL; @@ -208,7 +218,12 @@ static VecEnv* unpack_vecenv(PyObject* args) { VecEnv* vec = (VecEnv*)PyLong_AsVoidPtr(handle_obj); if (!vec) { - PyErr_SetString(PyExc_ValueError, "Invalid vec env handle"); + PyErr_SetString(PyExc_ValueError, "Missing or invalid vec env handle"); + return NULL; + } + + if (vec->num_envs <= 0) { + PyErr_SetString(PyExc_ValueError, "Missing or invalid vec env handle"); return NULL; } @@ -361,9 +376,9 @@ static PyObject* vec_init(PyObject* self, PyObject* args, PyObject* kwargs) { Py_DECREF(py_seed); PyObject* empty_args = PyTuple_New(0); - if (my_init(env, empty_args, kwargs)) { - PyErr_SetString(PyExc_TypeError, "env_init failed"); - Py_DECREF(kwargs); + my_init(env, empty_args, kwargs); + Py_DECREF(kwargs); + if (PyErr_Occurred()) { return NULL; } } @@ -407,6 +422,11 @@ static PyObject* vectorize(PyObject* self, PyObject* args) { } static PyObject* vec_reset(PyObject* self, PyObject* args) { + if (PyTuple_Size(args) != 2) { + PyErr_SetString(PyExc_TypeError, "vec_reset requires 2 arguments"); + return NULL; + } + VecEnv* vec = unpack_vecenv(args); if (!vec) { return NULL; @@ -428,6 +448,12 @@ static PyObject* vec_reset(PyObject* self, PyObject* args) { } static PyObject* vec_step(PyObject* self, PyObject* arg) { + int num_args = PyTuple_Size(arg); + if (num_args != 1) { + PyErr_SetString(PyExc_TypeError, "vec_step requires 1 argument"); + return NULL; + } + VecEnv* vec = unpack_vecenv(arg); if (!vec) { return NULL; @@ -530,9 +556,10 @@ static PyObject* vec_close(PyObject* self, PyObject* args) { static double unpack(PyObject* kwargs, char* key) { PyObject* val = PyDict_GetItemString(kwargs, key); if (val == NULL) { - // If the key doesn't exist, don't set an error - this allows optional parameters - // Just return a default value that the caller can check for - return 0.0; + char error_msg[100]; + snprintf(error_msg, sizeof(error_msg), "Missing required keyword argument '%s'", key); + PyErr_SetString(PyExc_TypeError, error_msg); + return 1; } if (PyLong_Check(val)) { long out = PyLong_AsLong(val); diff --git a/tests/test_env_binding.py b/tests/test_env_binding.py new file mode 100644 index 0000000000..cefcef857d --- /dev/null +++ b/tests/test_env_binding.py @@ -0,0 +1,116 @@ +from pufferlib.ocean.breakout import breakout + +kwargs = dict( + frameskip=1, + width=576, + height=330, + paddle_width=62, + paddle_height=8, + ball_width=32, + ball_height=32, + brick_width=32, + brick_height=12, + brick_rows=6, + brick_cols=18, + continuous=False, +) + +def test_env_binding(): + reference = breakout.Breakout() + + # Correct usage + c_env = breakout.binding.env_init( + reference.observations, + reference.actions, + reference.rewards, + reference.terminals, + reference.truncations, + 0, + **kwargs + ) + c_envs = breakout.binding.vectorize(c_env) + breakout.binding.vec_reset(c_envs, 0) + breakout.binding.vec_step(c_envs) + breakout.binding.vec_close(c_envs) + + # Correct vec usage + c_envs = breakout.binding.vec_init( + reference.observations, + reference.actions, + reference.rewards, + reference.terminals, + reference.truncations, + reference.num_agents, + 0, + **kwargs + ) + + # Correct vec usage + c_envs = breakout.binding.vec_init( + reference.observations, + reference.actions, + reference.rewards, + reference.terminals, + reference.truncations, + reference.num_agents, + 0, + **kwargs + ) + breakout.binding.vec_reset(c_envs, 0) + breakout.binding.vec_step(c_envs) + breakout.binding.vec_close(c_envs) + + try: + c_env = breakout.binding.env_init() + raise Exception('init missing args. Should have thrown TypeError') + except TypeError: + pass + + try: + c_env = breakout.binding.env_init( + reference.observations, + reference.actions, + reference.rewards, + reference.terminals, + reference.truncations, + reference.num_agents, + 0, + ) + raise Exception('init missing kwarg. Should have thrown TypeError') + except TypeError: + pass + + try: + c_envs = breakout.binding.vec_init() + raise Exception('vec_init missing args. Should have thrown TypeError') + except TypeError: + pass + + try: + c_envs = breakout.binding.vec_init( + reference.observations, + reference.actions, + reference.rewards, + reference.terminals, + reference.truncations, + reference.num_agents, + 0, + ) + raise Exception('vec_init missing kwarg. Should have thrown TypeError') + except TypeError: + pass + + try: + breakout.binding.vec_reset() + raise Exception('vec_reset missing arg. Should have thrown TypeError') + except TypeError: + pass + + try: + breakout.binding.vec_step() + raise Exception('vec_step missing arg. Should have thrown TypeError') + except TypeError: + pass + +if __name__ == '__main__': + test_env_binding() From 3bb3076ecfd245f8f960873bfacc4a33a0c59dac Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 20:37:14 +0000 Subject: [PATCH 59/63] API fixes --- clean_pufferl.py | 24 ++++++++++++---------- pufferlib/ocean/breakout/breakout.h | 32 +++++++++++++---------------- pufferlib/pufferlib.py | 4 ++++ pufferlib/pytorch.py | 6 ++---- 4 files changed, 33 insertions(+), 33 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index cb01c03135..78a0354250 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -93,7 +93,6 @@ def __init__(self, config, vecenv, policy, neptune=False, wandb=False): # LSTM if config['use_rnn']: - # TODO: Doesn't exist in native envs n = vecenv.agents_per_batch h = policy.hidden_size self.lstm_h = {i*n: torch.zeros(n, h, device=device) for i in range(total_agents//n)} @@ -243,8 +242,7 @@ def evaluate(self): state['lstm_c'] = self.lstm_c[env_id.start] logits, value = self.policy(o_device, state) - action, logprob, _ = pufferlib.pytorch.sample_logits( - logits, is_continuous=self.policy.is_continuous) + action, logprob, _ = pufferlib.pytorch.sample_logits(logits) r = torch.clamp(r, -1, 1) profile('eval_copy', epoch) @@ -280,6 +278,8 @@ def evaluate(self): self.full_rows += num_full action = action.cpu().numpy() + if isinstance(logits, torch.distributions.Normal): + action = np.clip(action, vecenv.action_space.low, vecenv.action_space.high) profile('eval_misc', epoch) for i in info: @@ -357,8 +357,8 @@ def train(self): # TODO: Currently only returning traj shaped value as a hack logits, newvalue = self.policy.forward_train(mb_obs, state) - actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, - action=mb_actions, is_continuous=self.policy.is_continuous) + # TODO: Redundant actions? + actions, newlogprob, entropy = pufferlib.pytorch.sample_logits(logits, action=mb_actions) profile('train_misc', epoch) newlogprob = newlogprob.reshape(mb_logprobs.shape) @@ -985,10 +985,13 @@ def experiment(vecenv, policy, args): ob, info = vecenv.reset() driver = vecenv.driver_env num_agents = vecenv.observation_space.shape[0] - state = dict( - lstm_h=torch.zeros(num_agents, policy.hidden_size, device=device), - lstm_c=torch.zeros(num_agents, policy.hidden_size, device=device), - ) + + state = {} + if args['train']['use_rnn']: + state = dict( + lstm_h=torch.zeros(num_agents, policy.hidden_size, device=device), + lstm_c=torch.zeros(num_agents, policy.hidden_size, device=device), + ) frames = [] while True: @@ -1010,8 +1013,7 @@ def experiment(vecenv, policy, args): with torch.no_grad(): ob = torch.as_tensor(ob).to(args['train']['device']) logits, value = policy(ob, state) - action, logprob, _ = pufferlib.pytorch.sample_logits( - logits, is_continuous=policy.is_continuous) + action, logprob, _ = pufferlib.pytorch.sample_logits(logits) action = action.cpu().numpy().reshape(vecenv.action_space.shape) ob = vecenv.step(action)[0] diff --git a/pufferlib/ocean/breakout/breakout.h b/pufferlib/ocean/breakout/breakout.h index 6b89a1ea6e..c85e5564a0 100644 --- a/pufferlib/ocean/breakout/breakout.h +++ b/pufferlib/ocean/breakout/breakout.h @@ -19,18 +19,25 @@ #define BRICK_INDEX_BACKWALL_COLLISION -2 #define BRICK_INDEX_PADDLE_COLLISION -1 -typedef struct Log Log; -struct Log { +typedef struct Log { float perf; float score; float episode_return; float episode_length; float n; -}; +} Log; + +typedef struct Client { + float width; + float height; + float paddle_width; + float paddle_height; + float ball_width; + float ball_height; + Texture2D ball; +} Client; -typedef struct Client Client; -typedef struct Breakout Breakout; -struct Breakout { +typedef struct Breakout { Client* client; Log log; float* observations; @@ -68,7 +75,7 @@ struct Breakout { int frameskip; unsigned char hit_brick; int continuous; -}; +} Breakout; typedef struct CollisionInfo CollisionInfo; struct CollisionInfo { @@ -470,17 +477,6 @@ void c_step(Breakout* env) { Color BRICK_COLORS[6] = {RED, ORANGE, YELLOW, GREEN, SKYBLUE, BLUE}; -typedef struct Client Client; -struct Client { - float width; - float height; - float paddle_width; - float paddle_height; - float ball_width; - float ball_height; - Texture2D ball; -}; - static inline bool file_exists(const char* path) { return access(path, F_OK) != -1; } diff --git a/pufferlib/pufferlib.py b/pufferlib/pufferlib.py index 8c1689071a..41a7929a04 100644 --- a/pufferlib/pufferlib.py +++ b/pufferlib/pufferlib.py @@ -70,6 +70,10 @@ def __init__(self, buf=None): self.observation_space = pufferlib.spaces.joint_space(self.single_observation_space, self.num_agents) self.agent_ids = np.arange(self.num_agents) + @property + def agent_per_batch(self): + return self.num_agents + @property def emulated(self): '''Native envs do not use emulation''' diff --git a/pufferlib/pytorch.py b/pufferlib/pytorch.py index 1a7084191d..fd95fd714d 100644 --- a/pufferlib/pytorch.py +++ b/pufferlib/pytorch.py @@ -270,11 +270,9 @@ def entropy_probs(logits, probs): p_log_p = logits * probs return -p_log_p.sum(-1) - -def sample_logits(logits: Union[torch.Tensor, List[torch.Tensor]], - action=None, is_continuous=False): +def sample_logits(logits, action=None): is_discrete = isinstance(logits, torch.Tensor) - if is_continuous: + if isinstance(logits, torch.distributions.Normal): batch = logits.loc.shape[0] if action is None: action = logits.sample().view(batch, -1) From 688f4a1489d8b580877de24a9564d78e61626dba Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sat, 10 May 2025 21:22:21 +0000 Subject: [PATCH 60/63] Continuous atn space flatten --- clean_pufferl.py | 2 +- config/ocean/cartpole.ini | 1 - pufferlib/ocean/cartpole/cartpole.py | 14 ++++++-------- pufferlib/ocean/env_binding.h | 8 ++++++++ pufferlib/pufferlib.py | 6 +++--- pufferlib/spaces.py | 9 +++++---- 6 files changed, 23 insertions(+), 17 deletions(-) diff --git a/clean_pufferl.py b/clean_pufferl.py index 78a0354250..96bd3264e6 100644 --- a/clean_pufferl.py +++ b/clean_pufferl.py @@ -277,7 +277,7 @@ def evaluate(self): self.free_idx += num_full self.full_rows += num_full - action = action.cpu().numpy() + action = action.squeeze(-1).cpu().numpy() if isinstance(logits, torch.distributions.Normal): action = np.clip(action, vecenv.action_space.low, vecenv.action_space.high) diff --git a/config/ocean/cartpole.ini b/config/ocean/cartpole.ini index b1ea10cef3..6ecfb7db00 100644 --- a/config/ocean/cartpole.ini +++ b/config/ocean/cartpole.ini @@ -1,7 +1,6 @@ [base] package = ocean env_name = puffer_cartpole -vec = multiprocessing policy_name = Policy rnn_name = Recurrent diff --git a/pufferlib/ocean/cartpole/cartpole.py b/pufferlib/ocean/cartpole/cartpole.py index 9b3eecca3c..62e3ba6d3d 100644 --- a/pufferlib/ocean/cartpole/cartpole.py +++ b/pufferlib/ocean/cartpole/cartpole.py @@ -4,7 +4,7 @@ from pufferlib.ocean.cartpole import binding class Cartpole(pufferlib.PufferEnv): - def __init__(self, num_envs=1, render_mode='human', report_interval=1, continuous=False, buf=None, seed=0): + def __init__(self, num_envs=1, render_mode='human', report_interval=1, continuous=True, buf=None, seed=0): self.render_mode = render_mode self.num_agents = num_envs self.report_interval = report_interval @@ -18,17 +18,14 @@ def __init__(self, num_envs=1, render_mode='human', report_interval=1, continuou ) if self.continuous: self.single_action_space = gymnasium.spaces.Box( - low=-1.0, high=1.0, shape=(1,), dtype=np.float32 + low=-1.0, high=1.0, shape=(1,) ) else: self.single_action_space = gymnasium.spaces.Discrete(2) super().__init__(buf) - - self.actions = np.zeros(self.num_agents, dtype=np.float32) - self.terminals = np.zeros(self.num_agents, dtype=np.uint8) - self.truncations = np.zeros(self.num_agents, dtype=np.uint8) + self.actions = np.zeros(num_envs, dtype=np.float32) self.c_envs = binding.vec_init( self.observations, @@ -37,7 +34,8 @@ def __init__(self, num_envs=1, render_mode='human', report_interval=1, continuou self.terminals, self.truncations, num_envs, - int(self.continuous), + seed, + continuous=int(self.continuous), ) def reset(self, seed=None): @@ -98,4 +96,4 @@ def test_performance(timeout=10, atn_cache=8192, continuous=True): if __name__ == '__main__': test_performance() - \ No newline at end of file + diff --git a/pufferlib/ocean/env_binding.h b/pufferlib/ocean/env_binding.h index e270b48305..487cc9f230 100644 --- a/pufferlib/ocean/env_binding.h +++ b/pufferlib/ocean/env_binding.h @@ -64,6 +64,10 @@ static PyObject* env_init(PyObject* self, PyObject* args, PyObject* kwargs) { return NULL; } env->actions = PyArray_DATA(actions); + if (PyArray_STRIDE(actions, 0) == sizeof(double)) { + PyErr_SetString(PyExc_ValueError, "Action tensor passed as float64 (pass np.float32 buffer)"); + return NULL; + } PyObject* rew = PyTuple_GetItem(args, 2); if (!PyObject_TypeCheck(rew, &PyArray_Type)) { @@ -290,6 +294,10 @@ static PyObject* vec_init(PyObject* self, PyObject* args, PyObject* kwargs) { PyErr_SetString(PyExc_ValueError, "Actions must be contiguous"); return NULL; } + if (PyArray_STRIDE(actions, 0) == sizeof(double)) { + PyErr_SetString(PyExc_ValueError, "Action tensor passed as float64 (pass np.float32 buffer)"); + return NULL; + } PyObject* rew = PyTuple_GetItem(args, 2); if (!PyObject_TypeCheck(rew, &PyArray_Type)) { diff --git a/pufferlib/pufferlib.py b/pufferlib/pufferlib.py index 41a7929a04..e3f95643ba 100644 --- a/pufferlib/pufferlib.py +++ b/pufferlib/pufferlib.py @@ -29,11 +29,11 @@ def set_buffers(env, buf=None): env.masks = np.ones(env.num_agents, dtype=bool) # TODO: Major kerfuffle on inferring action space dtype. This needs some asserts? - atn_space = env.single_action_space + atn_space = pufferlib.spaces.joint_space(env.single_action_space, env.num_agents) if isinstance(env.single_action_space, pufferlib.spaces.Box): - env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=atn_space.dtype) + env.actions = np.zeros(atn_space.shape, dtype=atn_space.dtype) else: - env.actions = np.zeros((env.num_agents, *atn_space.shape), dtype=np.int32) + env.actions = np.zeros(atn_space.shape, dtype=np.int32) else: env.observations = buf['observations'] env.rewards = buf['rewards'] diff --git a/pufferlib/spaces.py b/pufferlib/spaces.py index b5bab9e6cc..178513c02c 100644 --- a/pufferlib/spaces.py +++ b/pufferlib/spaces.py @@ -17,9 +17,10 @@ def joint_space(space, n): high=np.repeat(space.nvec[None] - 1, n, axis=0), shape=(n, len(space)), dtype=space.dtype) elif isinstance(space, Box): - return gymnasium.spaces.Box( - low=np.repeat(space.low[None], n, axis=0), - high=np.repeat(space.high[None], n, axis=0), - shape=(n, *space.shape), dtype=space.dtype) + low = np.repeat(space.low[None], n, axis=0).squeeze() + high = np.repeat(space.high[None], n, axis=0).squeeze() + shape = [n, *[e for e in space.shape if e != 1]] + return gymnasium.spaces.Box(low=low, high=high, + shape=shape, dtype=space.dtype) else: raise ValueError(f'Unsupported space: {space}') From 9d480b04b6ea73e3c18ff299a5be1322c8eb1499 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Sun, 11 May 2025 00:38:29 +0000 Subject: [PATCH 61/63] Initial build sys --- setup.py | 74 +++++++++++++++++++------------------------------------- 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/setup.py b/setup.py index 4887b8ae56..1bb7a981ae 100644 --- a/setup.py +++ b/setup.py @@ -11,53 +11,26 @@ import tarfile import platform -from setuptools.command.build_ext import build_ext as DefaultBuildExt - -# python3 setup.py built_ext --inplace +from setuptools.command.build_ext import build_ext +from torch.utils import cpp_extension -class CustomBuildExt(DefaultBuildExt): + +class BuildExt(build_ext): def run(self): - # Split extensions into PyTorch and non-PyTorch - pytorch_extensions = [ext for ext in self.extensions if ext.name == 'pufferlib._C'] - other_extensions = [ext for ext in self.extensions if ext.name != 'pufferlib._C'] - - # Build PyTorch extensions with cpp_extension.BuildExtension - if pytorch_extensions: - pytorch_build_ext = cpp_extension.BuildExtension(self.distribution) - # Temporarily set extensions to only PyTorch ones - original_extensions = self.extensions - self.extensions = pytorch_extensions - pytorch_build_ext.run() - self.extensions = original_extensions # Restore original extensions - - # Build other extensions with default setuptools build_ext - if other_extensions: - self.extensions = other_extensions - super().run() - -''' - + cythonize( - [ - "pufferlib/extensions.pyx", - "c_advantage.pyx", - "pufferlib/puffernet.pyx", - *extensions, - ], - compiler_directives={ - 'language_level': 3, - 'boundscheck': False, - 'initializedcheck': False, - 'wraparound': False, - 'cdivision': True, - 'nonecheck': False, - 'profile': False, - }, - #nthreads=6, - #annotate=True, - #compiler_directives={'profile': True},# annotate=True - ), -''' - + self.run_command('build_torch') + self.run_command('build_c') + +class CBuildExt(build_ext): + def run(self): + self.extensions = [e for e in self.extensions if e.name != "pufferlib._C"] + super().run() + +class TorchBuildExt(cpp_extension.BuildExtension): + def run(self): + self.extensions = [e for e in self.extensions if e.name == "pufferlib._C"] + super().run() + + VERSION = '2.0.6' RAYLIB_BASE = 'https://github.com/raysan5/raylib/releases/download/5.5/' @@ -378,9 +351,8 @@ def run(self): for name in pure_c_extensions ] -from torch.utils import cpp_extension torch_extensions = [ - cpp_extension.CUDAExtension( + cpp_extension.CUDAExtension( "pufferlib._C", ["pufferlib.cpp", "pufferlib/pufferlib.cu"], extra_compile_args = { @@ -442,8 +414,12 @@ def run(self): 'common': common, **environments, }, - ext_modules = c_extensions, - #cmdclass={"build_ext": cpp_extension.BuildExtension}, + ext_modules = c_extensions + torch_extensions, + cmdclass={ + "build_ext": BuildExt, + "build_torch": TorchBuildExt, + "build_c": CBuildExt, + }, include_dirs=[numpy.get_include(), RAYLIB_NAME + '/include'], python_requires=">=3.9", license="MIT", From d0f032e093529311599cc3bde8f6cf2bf8d2b031 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 12 May 2025 16:07:01 +0000 Subject: [PATCH 62/63] Much cleaner setup.py --- pufferlib/__init__.py | 3 +- pufferlib/version.py | 1 - setup.py | 319 +++++++++++++++++++++++++----------------- 3 files changed, 188 insertions(+), 135 deletions(-) delete mode 100644 pufferlib/version.py diff --git a/pufferlib/__init__.py b/pufferlib/__init__.py index 9bc5c3e56e..cbdc7a463b 100644 --- a/pufferlib/__init__.py +++ b/pufferlib/__init__.py @@ -1,5 +1,4 @@ -from pufferlib import version -__version__ = version.__version__ +__version__ = '2.0.6' import os import sys diff --git a/pufferlib/version.py b/pufferlib/version.py deleted file mode 100644 index 13ce17d8e8..0000000000 --- a/pufferlib/version.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = '2.0.6' diff --git a/setup.py b/setup.py index 1bb7a981ae..4b6a8414a7 100644 --- a/setup.py +++ b/setup.py @@ -10,64 +10,143 @@ import zipfile import tarfile import platform +import shutil from setuptools.command.build_ext import build_ext from torch.utils import cpp_extension - - -class BuildExt(build_ext): - def run(self): - self.run_command('build_torch') - self.run_command('build_c') - -class CBuildExt(build_ext): - def run(self): - self.extensions = [e for e in self.extensions if e.name != "pufferlib._C"] - super().run() - -class TorchBuildExt(cpp_extension.BuildExtension): - def run(self): - self.extensions = [e for e in self.extensions if e.name == "pufferlib._C"] - super().run() +from torch.utils.cpp_extension import ( + CppExtension, + CUDAExtension, + BuildExtension, + CUDA_HOME, +) -VERSION = '2.0.6' +import pufferlib +VERSION = pufferlib.__version__ + +# Build with DEBUG=1 to enable debug symbols +DEBUG = os.getenv("DEBUG", "0") == "1" + +# Put C env names here. PufferLib will look for +# pufferlib/ocean//binding.c +c_extensions_names = [ + 'gpudrive', + 'squared', + 'pong', + 'breakout', + 'enduro', + 'blastar', + 'grid', + 'nmmo3', + 'tactical', + 'go', + 'cartpole' +] +# Put full paths to Cython extension here +# Note we are trying to move away from Cython, +# because our C envs are lighter weigh and +# easier to debug (you can run gdb --args python ...) +cython_extension_paths = [ + 'pufferlib/ocean/moba/cy_moba', + 'pufferlib/ocean/snake/cy_snake', + 'pufferlib/ocean/connect4/cy_connect4', + 'pufferlib/ocean/tripletriad/cy_tripletriad', + 'pufferlib/ocean/rware/cy_rware', + 'pufferlib/ocean/trash_pickup/cy_trash_pickup', + 'pufferlib/ocean/cpr/cy_cpr', + 'pufferlib/ocean/tower_climb/cy_tower_climb', +] + +# Build raylib for your platform RAYLIB_BASE = 'https://github.com/raysan5/raylib/releases/download/5.5/' RAYLIB_NAME = 'raylib-5.5_macos' if platform.system() == "Darwin" else 'raylib-5.5_linux_amd64' - -RAYLIB_LINUX = 'raylib-5.5_linux_amd64' -RAYLIB_LINUX_URL = RAYLIB_BASE + RAYLIB_LINUX + '.tar.gz' RLIGHTS_URL = 'https://raw.githubusercontent.com/raysan5/raylib/refs/heads/master/examples/shaders/rlights.h' -if not os.path.exists(RAYLIB_LINUX): - urllib.request.urlretrieve(RAYLIB_LINUX_URL, RAYLIB_LINUX + '.tar.gz') - with tarfile.open(RAYLIB_LINUX + '.tar.gz', 'r') as tar_ref: - tar_ref.extractall() - - os.remove(RAYLIB_LINUX + '.tar.gz') - urllib.request.urlretrieve(RLIGHTS_URL, 'raylib-5.5_linux_amd64/include/rlights.h') - -RAYLIB_MACOS = 'raylib-5.5_macos' -RAYLIB_MACOS_URL = RAYLIB_BASE + RAYLIB_MACOS + '.tar.gz' -if not os.path.exists(RAYLIB_MACOS): - urllib.request.urlretrieve(RAYLIB_MACOS_URL, RAYLIB_MACOS + '.tar.gz') - with tarfile.open(RAYLIB_MACOS + '.tar.gz', 'r') as tar_ref: - tar_ref.extractall() +def download_raylib(platform, url): + if not os.path.exists(platform): + urllib.request.urlretrieve(url, platform + '.tar.gz') + with tarfile.open(platform + '.tar.gz', 'r') as tar_ref: + tar_ref.extractall() - os.remove(RAYLIB_MACOS + '.tar.gz') - urllib.request.urlretrieve(RLIGHTS_URL, 'raylib-5.5_macos/include/rlights.h') + os.remove(platform + '.tar.gz') + urllib.request.urlretrieve(RLIGHTS_URL, platform + '/include/rlights.h') RAYLIB_WASM = 'raylib-5.5_webassembly' RAYLIB_WASM_URL = RAYLIB_BASE + RAYLIB_WASM + '.zip' -if not os.path.exists(RAYLIB_WASM): - urllib.request.urlretrieve(RAYLIB_WASM_URL, RAYLIB_WASM + '.zip') - with zipfile.ZipFile(RAYLIB_WASM + '.zip', 'r') as zip_ref: - zip_ref.extractall() +download_raylib(RAYLIB_WASM, RAYLIB_WASM_URL) - os.remove(RAYLIB_WASM + '.zip') - urllib.request.urlretrieve(RLIGHTS_URL, 'raylib-5.5_webassembly/include/rlights.h') +# Shared compile args for all platforms +extra_compile_args = [ + '-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION', + '-DPLATFORM_DESKTOP', +] +extra_link_args = [ + '-fwrapv' +] +cxx_args = [ + '-fdiagnostics-color=always', +] +nvcc_args = [] + +if DEBUG: + extra_compile_args += [ + '-O0', + '-g', + '-fsanitize=address,undefined,bounds,pointer-overflow,leak', + ] + extra_link_args += [ + '-g', + ] + cxx_args += [ + '-O0', + '-g', + ] + nvcc_args += [ + '-O0', + '-g', + ] +else: + extra_compile_args += [ + '-O2', + ] + extra_link_args += [ + '-O2', + ] + cxx_args += [ + '-O3', + ] + nvcc_args += [ + '-O3', + ] + +system = platform.system() +if system == 'Linux': + extra_compile_args += [ + '-Wno-alloc-size-larger-than', + '-fmax-errors=3', + ] + extra_link_args += [ + '-Bsymbolic-functions', + ] + RAYLIB_LINUX = 'raylib-5.5_linux_amd64' + RAYLIB_LINUX_URL = RAYLIB_BASE + RAYLIB_LINUX + '.tar.gz' + download_raylib(RAYLIB_LINUX, RAYLIB_LINUX_URL) +elif system == 'Darwin': + extra_compile_args += [ + ] + extra_link_args += [ + '-framework', 'Cocoa', + '-framework', 'OpenGL', + '-framework', 'IOKit', + ] + RAYLIB_MACOS = 'raylib-5.5_macos' + RAYLIB_MACOS_URL = RAYLIB_BASE + RAYLIB_MACOS + '.tar.gz' + download_raylib(RAYLIB_MACOS, RAYLIB_MACOS_URL) +else: + raise ValueError(f'Unsupported system: {system}') # Default Gym/Gymnasium/PettingZoo versions # Gym: @@ -83,31 +162,6 @@ def run(self): GYM_VERSION = '0.23' PETTINGZOO_VERSION = '1.24.1' -docs = [ - 'sphinx==5.0.0', - 'sphinx-rtd-theme==0.5.1', - 'sphinxcontrib-youtube==1.0.1', - 'sphinx-rtd-theme==0.5.1', - 'sphinx-design==0.4.1', - 'furo==2023.3.27', -] - -cleanrl = [ - 'stable_baselines3==2.1.0', - 'tensorboard==2.11.2', - 'torch', - 'tyro==0.8.6', - 'wandb==0.19.1', - 'scipy', - 'pyro-ppl', - 'neptune', - 'heavyball', -] - -ray = [ - 'ray==2.23.0', -] - environments = { 'avalon': [ f'gym=={GYM_VERSION}', @@ -259,6 +313,30 @@ def run(self): ], } +docs = [ + 'sphinx==5.0.0', + 'sphinx-rtd-theme==0.5.1', + 'sphinxcontrib-youtube==1.0.1', + 'sphinx-rtd-theme==0.5.1', + 'sphinx-design==0.4.1', + 'furo==2023.3.27', +] + +cleanrl = [ + 'stable_baselines3==2.1.0', + 'tensorboard==2.11.2', + 'torch', + 'tyro==0.8.6', + 'wandb==0.19.1', + 'scipy', + 'pyro-ppl', + 'neptune', + 'heavyball', +] + +ray = [ + 'ray==2.23.0', +] # These are the environments that PufferLib has made # compatible with the latest version of Gym/Gymnasium/PettingZoo @@ -286,84 +364,63 @@ def run(self): 'vizdoom', ]] -extension_paths = [ - #'pufferlib/ocean/nmmo3/cy_nmmo3', - 'pufferlib/ocean/moba/cy_moba', - # 'pufferlib/ocean/tactical/c_tactical', - #'pufferlib/ocean/squared/cy_squared', - 'pufferlib/ocean/snake/cy_snake', - #'pufferlib/ocean/gpudrive/cy_gpudrive', - #'pufferlib/ocean/pong/cy_pong', - # 'pufferlib/ocean/breakout/cy_breakout', - # 'pufferlib/ocean/cartpole/cy_cartpole', - 'pufferlib/ocean/connect4/cy_connect4', - #'pufferlib/ocean/grid/cy_grid', - 'pufferlib/ocean/tripletriad/cy_tripletriad', - # 'pufferlib/ocean/go/cy_go', - 'pufferlib/ocean/rware/cy_rware', - 'pufferlib/ocean/trash_pickup/cy_trash_pickup', - 'pufferlib/ocean/cpr/cy_cpr', - 'pufferlib/ocean/tower_climb/cy_tower_climb', -] +# Extensions +class BuildExt(build_ext): + def run(self): + self.run_command('build_torch') + self.run_command('build_c') -system = platform.system() -if system == 'Darwin': - # On macOS, use @loader_path. - # The extension “.so” is typically in pufferlib/ocean/..., - # and “raylib/lib” is (maybe) two directories up from ocean/. - # So @loader_path/../../raylib/lib is common. - extra_compile_args = ['-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION','-DPLATFORM_DESKTOP', '-O2'] - extra_link_args=['-fwrapv', '-framework', 'Cocoa', '-framework', 'OpenGL', '-framework', 'IOKit'] - -elif system == 'Linux': - extra_compile_args = ['-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION', '-DPLATFORM_DESKTOP', '-O2', '-Wno-alloc-size-larger-than', '-fmax-errors=3', '-g'] - extra_link_args=['-fwrapv', '-Bsymbolic-functions', '-O2'] - - # On Linux, $ORIGIN works -else: - raise ValueError(f'Unsupported system: {system}') +class CBuildExt(build_ext): + def run(self): + self.extensions = [e for e in self.extensions if e.name != "pufferlib._C"] + super().run() + +class TorchBuildExt(cpp_extension.BuildExtension): + def run(self): + self.extensions = [e for e in self.extensions if e.name == "pufferlib._C"] + super().run() -extensions = [Extension( - path.replace('/', '.'), - [path + '.pyx'], - include_dirs=[numpy.get_include(), 'raylib/include'], +RAYLIB_A = f'{RAYLIB_NAME}/lib/libraylib.a' +INCLUDE = [numpy.get_include(), 'raylib/include'] +extension_kwargs = dict( + include_dirs=INCLUDE, extra_compile_args=extra_compile_args, extra_link_args=extra_link_args, - extra_objects=[f'{RAYLIB_NAME}/lib/libraylib.a'], -) for path in extension_paths] - -#c_args = ['-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION', '-DPLATFORM_DESKTOP', '-O0', '-Wno-alloc-size-larger-than', '-g'] -#c_args = ['-DNPY_NO_DEPRECATED_API=NPY_1_7_API_VERSION', '-DPLATFORM_DESKTOP', '-O2'] -#c_args += "-Wsign-compare -DNDEBUG -g -O2 -Wall -g -fstack-protector-strong -Wformat -Werror=format-security -g -fwrapv -O2 -fPIC".split() - - -pure_c_extensions = ['gpudrive', 'squared', 'pong', 'breakout', 'enduro', 'blastar', 'grid', 'nmmo3', 'tactical', 'go', 'cartpole'] + extra_objects=[RAYLIB_A], +) c_extensions = [ Extension( f'pufferlib.ocean.{name}.binding', sources=[f'pufferlib/ocean/{name}/binding.c'], - include_dirs=[numpy.get_include(), 'raylib/include'], - extra_compile_args=extra_compile_args, # + ['-fsanitize=address,undefined,bounds,pointer-overflow,leak', '-static-libasan'], - extra_link_args=extra_link_args,# + ['-fsanitize=address,undefined,bounds,pointer-overflow,leak', '-g'], - extra_objects=[f'{RAYLIB_NAME}/lib/libraylib.a'], + **extension_kwargs, ) - for name in pure_c_extensions + for name in c_extensions_names ] +cython_extensions = cythonize([ + Extension( + path.replace('/', '.'), + [path + '.pyx'], + **extension_kwargs, + ) + for path in cython_extension_paths +]) + +# Check if CUDA compiler is available. You need cuda dev, not just runtime. +if shutil.which("nvcc"): + extension = CUDAExtension +else: + extension = CppExtension + torch_extensions = [ - cpp_extension.CUDAExtension( + extension( "pufferlib._C", ["pufferlib.cpp", "pufferlib/pufferlib.cu"], extra_compile_args = { - "cxx": [ - "-fdiagnostics-color=always", - #"-DPy_LIMITED_API=0x03090000", # min CPython version 3.9 - ], - "nvcc": [ - ], - }, - #py_limited_api=True, + "cxx": cxx_args, + "nvcc": nvcc_args, + } ), ] @@ -414,7 +471,7 @@ def run(self): 'common': common, **environments, }, - ext_modules = c_extensions + torch_extensions, + ext_modules = cython_extensions + c_extensions + torch_extensions, cmdclass={ "build_ext": BuildExt, "build_torch": TorchBuildExt, @@ -431,13 +488,11 @@ def run(self): "Intended Audience :: Science/Research", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - #options={"bdist_wheel": {"py_limited_api": "cp39"}}, ) #stable_baselines3 #supersuit==3.3.5 From b7ae6d92a8c11776c10d9ad66d43414aad626877 Mon Sep 17 00:00:00 2001 From: Joseph Suarez Date: Mon, 12 May 2025 16:29:01 +0000 Subject: [PATCH 63/63] env get binding --- pufferlib/ocean/env_binding.h | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/pufferlib/ocean/env_binding.h b/pufferlib/ocean/env_binding.h index 487cc9f230..12f54d4990 100644 --- a/pufferlib/ocean/env_binding.h +++ b/pufferlib/ocean/env_binding.h @@ -12,6 +12,13 @@ static PyObject* my_shared(PyObject* self, PyObject* args, PyObject* kwargs) { } #endif +static PyObject* my_get(PyObject* dict, Env* env); +#ifndef MY_GET +static PyObject* my_get(PyObject* dict, Env* env) { + return NULL; +} +#endif + static Env* unpack_env(PyObject* args) { PyObject* handle_obj = PyTuple_GetItem(args, 0); if (!PyObject_TypeCheck(handle_obj, &PyLong_Type)) { @@ -170,7 +177,6 @@ static PyObject* env_reset(PyObject* self, PyObject* args) { Py_RETURN_NONE; } - // Python function to step the environment static PyObject* env_step(PyObject* self, PyObject* args) { int num_args = PyTuple_Size(args); @@ -208,6 +214,19 @@ static PyObject* env_close(PyObject* self, PyObject* args) { Py_RETURN_NONE; } +static PyObject* env_get(PyObject* self, PyObject* args) { + Env* env = unpack_env(args); + if (!env){ + return NULL; + } + PyObject* dict = PyDict_New(); + my_get(dict, env); + if (PyErr_Occurred()) { + return NULL; + } + return dict; +} + typedef struct { Env** envs; int num_envs; @@ -596,6 +615,7 @@ static PyMethodDef methods[] = { {"env_step", env_step, METH_VARARGS, "Step the environment"}, {"env_render", env_render, METH_VARARGS, "Render the environment"}, {"env_close", env_close, METH_VARARGS, "Close the environment"}, + {"env_get", env_get, METH_VARARGS, "Get the environment state"}, {"vectorize", vectorize, METH_VARARGS, "Make a vector of environment handles"}, {"vec_init", (PyCFunction)vec_init, METH_VARARGS | METH_KEYWORDS, "Initialize a vector of environments"}, {"vec_reset", (PyCFunction)vec_reset, METH_VARARGS, "Reset the vector of environments"},