From 71219df6c600889294e59c54f88d1d70efd6af1a Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Fri, 10 Sep 2021 16:39:16 -0700 Subject: [PATCH] notebook example (#189) * config in result * value can be float * pytorch notebook example * docker, pre-commit * max_failure (#192); early_stop * extend starting_points (#196) Co-authored-by: Chi Wang (MSR) Co-authored-by: Qingyun Wu --- .devcontainer/Dockerfile | 23 + .devcontainer/devcontainer.json | 12 + .pre-commit-config.yaml | 18 + Dockerfile | 24 + README.md | 21 +- flaml/automl.py | 1569 +++++++++++++---------- flaml/searcher/blendsearch.py | 18 +- flaml/training_log.py | 10 +- flaml/tune/tune.py | 172 ++- flaml/version.py | 2 +- notebook/flaml_autovw.ipynb | 346 ++--- notebook/flaml_pytorch_cifar10.ipynb | 405 ++++++ setup.py | 17 +- test/test_automl.py | 526 +++++--- test/test_forecast.py | 98 +- test/{ => tune}/test_pytorch_cifar10.py | 37 +- 16 files changed, 2148 insertions(+), 1150 deletions(-) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json create mode 100644 .pre-commit-config.yaml create mode 100644 Dockerfile create mode 100644 notebook/flaml_pytorch_cifar10.ipynb rename test/{ => tune}/test_pytorch_cifar10.py (93%) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000..75077296c --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,23 @@ +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE file in the project root for license information. +#------------------------------------------------------------------------------------------------------------- + +FROM mcr.microsoft.com/vscode/devcontainers/python:0-3.7 + +# +# Update the OS and maybe install packages +# +ENV DEBIAN_FRONTEND=noninteractive +RUN apt-get update \ + && apt-get upgrade -y \ + && apt-get -y install --no-install-recommends build-essential \ + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* +ENV DEBIAN_FRONTEND=dialog + +# +# Install extras for development +# +RUN pip3 --disable-pip-version-check --no-cache-dir install flaml[test,notebook] \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000..632f97d72 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,12 @@ +{ + "extensions": ["ms-python.python", "visualstudioexptteam.vscodeintellicode"], + "dockerFile": "Dockerfile", + "settings": { + "terminal.integrated.profiles.linux": { + "bash": { + "path": "/bin/bash" + } + }, + "terminal.integrated.defaultProfile.linux": "bash" + } +} \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 000000000..cd5922ab2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,18 @@ + +repos: + - repo: https://github.com/psf/black + rev: stable + hooks: + - id: black + language_version: python3 + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v1.2.3 + hooks: + - id: flake8 + - id: check-added-large-files + - id: check-ast + - id: check-byte-order-marker + - id: check-merge-conflict + - id: detect-private-key + - id: trailing-whitespace + - id: no-commit-to-branch \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..5200a1f31 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,24 @@ +# basic setup +FROM python:3.7 +RUN apt-get update && apt-get -y update +RUN apt-get install -y sudo git + +# Setup user to not run as root +RUN adduser --disabled-password --gecos '' hb-dev +RUN adduser hb-dev sudo +RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers +USER flaml-dev + +# Pull repo +RUN cd /home/flaml-dev && git clone https://github.com/microsoft/FLAML.git +WORKDIR /home/flaml-dev/FLAML + +# Install FLAML (Note: extra components can be installed if needed) +RUN sudo pip install -e .[test,notebook] + +# Install precommit hooks +RUN pre-commit install + +# override default image starting point +CMD /bin/bash +ENTRYPOINT [] \ No newline at end of file diff --git a/README.md b/README.md index 3119917fd..4732e4b48 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ FLAML is a lightweight Python library that finds accurate machine learning models automatically, efficiently and economically. It frees users from selecting -learners and hyperparameters for each learner. It is fast and economical. +learners and hyperparameters for each learner. It is fast and economical. The simple and lightweight design makes it easy to extend, such as adding customized learners or metrics. FLAML is powered by a new, [cost-effective hyperparameter optimization](https://github.com/microsoft/FLAML/tree/main/flaml/tune) @@ -146,7 +146,7 @@ print(automl.predict(X_train[72:])) ```python from sklearn.datasets import fetch_openml from flaml import AutoML -X, y = fetch_openml(name="credit-g", return_X_y=True) +X, y = fetch_openml(name="credit-g", return_X_y=True) # not a real learning to rank dataaset groups = [200] * 4 + [100] * 2, # group counts automl = AutoML() @@ -206,9 +206,24 @@ git clone https://github.com/microsoft/FLAML.git pip install -e .[test,notebook] ``` +### Docker +We provide a simple [Dockerfile](https://github.com/microsoft/FLAML/blob/main/Dockerfile). +``` +docker build git://github.com/microsoft/FLAML -t flaml-dev +docker run -it flaml-dev +``` + +### Develop in Remote Container +If you use vscode, you can open the FLAML folder in a [Container](https://code.visualstudio.com/docs/remote/containers). +We have provided the configuration in (.devcontainer)[(https://github.com/microsoft/FLAML/blob/main/.devcontainer)]. + +### Pre-commit +Run `pre-commit install` to install pre-commit into your git hooks. Before you commit, run +`pre-commit run` to check if you meet the pre-commit requirements. If you use Windows (without WSL) and can't commit after installing pre-commit, you can run `pre-commit uninstall` to uninstall the hook. In WSL or Linux this is supposed to work. + ### Coverage -Any code you commit should generally not significantly impact coverage. To run all unit tests: +Any code you commit should not decrease coverage. To run all unit tests: ```bash coverage run -m pytest test diff --git a/flaml/automl.py b/flaml/automl.py index f330d9b8b..31ef63ac4 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -1,33 +1,49 @@ -'''! - * Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved. +"""! + * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE file in the * project root for license information. -''' +""" import time from typing import Callable, Optional from functools import partial import numpy as np from scipy.sparse import issparse -from sklearn.model_selection import train_test_split, RepeatedStratifiedKFold, \ - RepeatedKFold, GroupKFold, TimeSeriesSplit, GroupShuffleSplit +from sklearn.model_selection import ( + train_test_split, + RepeatedStratifiedKFold, + RepeatedKFold, + GroupKFold, + TimeSeriesSplit, + GroupShuffleSplit, +) from sklearn.utils import shuffle import pandas as pd import logging -from .ml import compute_estimator, train_estimator, get_estimator_class, \ - get_classification_objective +from .ml import ( + compute_estimator, + train_estimator, + get_estimator_class, + get_classification_objective, +) from .config import ( - MIN_SAMPLE_TRAIN, MEM_THRES, RANDOM_SEED, - SMALL_LARGE_THRES, CV_HOLDOUT_THRESHOLD, SPLIT_RATIO, N_SPLITS, - SAMPLE_MULTIPLY_FACTOR) + MIN_SAMPLE_TRAIN, + MEM_THRES, + RANDOM_SEED, + SMALL_LARGE_THRES, + CV_HOLDOUT_THRESHOLD, + SPLIT_RATIO, + N_SPLITS, + SAMPLE_MULTIPLY_FACTOR, +) from .data import concat from . import tune from .training_log import training_log_reader, training_log_writer logger = logging.getLogger(__name__) logger_formatter = logging.Formatter( - '[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s', - '%m-%d %H:%M:%S') + "[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s", "%m-%d %H:%M:%S" +) try: import mlflow @@ -36,15 +52,16 @@ except ImportError: class SearchState: - @property def search_space(self): return self._search_space_domain @property def estimated_cost4improvement(self): - return max(self.time_best_found - self.time_best_found_old, - self.total_time_used - self.time_best_found) + return max( + self.time_best_found - self.time_best_found_old, + self.total_time_used - self.time_best_found, + ) def __init__(self, learner_class, data_size, task, starting_point=None): self.init_eci = learner_class.cost_relative2lgbm() @@ -55,22 +72,22 @@ class SearchState: self.data_size = data_size self.ls_ever_converged = False self.learner_class = learner_class - search_space = learner_class.search_space( - data_size=data_size, task=task) + search_space = learner_class.search_space(data_size=data_size, task=task) for name, space in search_space.items(): - assert 'domain' in space - self._search_space_domain[name] = space['domain'] - if 'init_value' in space: - self.init_config[name] = space['init_value'] - if 'low_cost_init_value' in space: - self.low_cost_partial_config[name] = space[ - 'low_cost_init_value'] - if 'cat_hp_cost' in space: - self.cat_hp_cost[name] = space['cat_hp_cost'] + assert "domain" in space + self._search_space_domain[name] = space["domain"] + if "init_value" in space: + self.init_config[name] = space["init_value"] + if "low_cost_init_value" in space: + self.low_cost_partial_config[name] = space["low_cost_init_value"] + if "cat_hp_cost" in space: + self.cat_hp_cost[name] = space["cat_hp_cost"] # if a starting point is provided, set the init config to be # the starting point provided - if starting_point is not None and starting_point.get(name) is not None: + if isinstance(starting_point, dict) and starting_point.get(name) is not None: self.init_config[name] = starting_point[name] + if isinstance(starting_point, list): + self.init_config = starting_point self._hp_names = list(self._search_space_domain.keys()) self.search_alg = None self.best_config = None @@ -87,16 +104,16 @@ class SearchState: def update(self, result, time_used, save_model_history=False): if result: - config = result['config'] - if config and 'FLAML_sample_size' in config: - self.sample_size = config['FLAML_sample_size'] + config = result["config"] + if config and "FLAML_sample_size" in config: + self.sample_size = config["FLAML_sample_size"] else: self.sample_size = self.data_size - obj = result['val_loss'] - metric_for_logging = result['metric_for_logging'] - time2eval = result['time_total_s'] - trained_estimator = result['trained_estimator'] - del result['trained_estimator'] # free up RAM + obj = result["val_loss"] + metric_for_logging = result["metric_for_logging"] + time2eval = result["time_total_s"] + trained_estimator = result["trained_estimator"] + del result["trained_estimator"] # free up RAM else: obj, time2eval, trained_estimator = np.inf, 0.0, None metric_for_logging = config = None @@ -107,8 +124,7 @@ class SearchState: if self.base_eci is None: self.base_eci = time_used if (obj is not None) and (self.best_loss is None or obj < self.best_loss): - self.best_loss_old = self.best_loss if self.best_loss < np.inf \ - else 2 * obj + self.best_loss_old = self.best_loss if self.best_loss < np.inf else 2 * obj self.best_loss = obj self.time_best_found_old = self.time_best_found self.time_best_found = self.total_time_used @@ -119,29 +135,31 @@ class SearchState: if time2eval: self.time2eval_best_old = self.time2eval_best self.time2eval_best = time2eval - if self.trained_estimator and trained_estimator and \ - self.trained_estimator != trained_estimator and \ - not save_model_history: + if ( + self.trained_estimator + and trained_estimator + and self.trained_estimator != trained_estimator + and not save_model_history + ): self.trained_estimator.cleanup() if trained_estimator: self.trained_estimator = trained_estimator - self.metric_for_logging, self.val_loss, self.config = \ - metric_for_logging, obj, config + self.metric_for_logging = metric_for_logging + self.val_loss, self.config = obj, config def get_hist_config_sig(self, sample_size, config): config_values = tuple([config[k] for k in self._hp_names]) - config_sig = str(sample_size) + '_' + str(config_values) + config_sig = str(sample_size) + "_" + str(config_values) return config_sig def est_retrain_time(self, retrain_sample_size): - assert self.best_config_sample_size is not None, \ - 'need to first get best_config_sample_size' - return (self.time2eval_best * retrain_sample_size - / self.best_config_sample_size) + assert ( + self.best_config_sample_size is not None + ), "need to first get best_config_sample_size" + return self.time2eval_best * retrain_sample_size / self.best_config_sample_size class AutoMLState: - def _prepare_sample_train_data(self, sample_size): sampled_weight = groups = None if sample_size <= self.data_size: @@ -150,7 +168,7 @@ class AutoMLState: else: sampled_X_train = self.X_train[:sample_size] sampled_y_train = self.y_train[:sample_size] - weight = self.fit_kwargs.get('sample_weight') + weight = self.fit_kwargs.get("sample_weight") if weight is not None: sampled_weight = weight[:sample_size] if self.groups is not None: @@ -158,89 +176,106 @@ class AutoMLState: else: sampled_X_train = self.X_train_all sampled_y_train = self.y_train_all - if 'sample_weight' in self.fit_kwargs: + if "sample_weight" in self.fit_kwargs: sampled_weight = self.sample_weight_all if self.groups is not None: groups = self.groups_all return sampled_X_train, sampled_y_train, sampled_weight, groups - def _compute_with_config_base(self, - estimator, - config_w_resource): - if 'FLAML_sample_size' in config_w_resource: - sample_size = int(config_w_resource['FLAML_sample_size']) + def _compute_with_config_base(self, estimator, config_w_resource): + if "FLAML_sample_size" in config_w_resource: + sample_size = int(config_w_resource["FLAML_sample_size"]) else: sample_size = self.data_size - sampled_X_train, sampled_y_train, sampled_weight, groups = \ - self._prepare_sample_train_data(sample_size) + ( + sampled_X_train, + sampled_y_train, + sampled_weight, + groups, + ) = self._prepare_sample_train_data(sample_size) if sampled_weight is not None: - weight = self.fit_kwargs['sample_weight'] - self.fit_kwargs['sample_weight'] = sampled_weight + weight = self.fit_kwargs["sample_weight"] + self.fit_kwargs["sample_weight"] = sampled_weight else: weight = None if groups is not None: - self.fit_kwargs['groups'] = groups + self.fit_kwargs["groups"] = groups config = config_w_resource.copy() - if 'FLAML_sample_size' in config: - del config['FLAML_sample_size'] + if "FLAML_sample_size" in config: + del config["FLAML_sample_size"] time_left = self.time_budget - self.time_from_start - budget = time_left if sample_size == self.data_size else \ - time_left / 2 * sample_size / self.data_size + budget = ( + time_left + if sample_size == self.data_size + else time_left / 2 * sample_size / self.data_size + ) - trained_estimator, val_loss, metric_for_logging, _, pred_time = \ - compute_estimator( - sampled_X_train, - sampled_y_train, - self.X_val, - self.y_val, - self.weight_val, - self.groups_val, - min(budget, self.train_time_limit), - self.kf, - config, - self.task, - estimator, - self.eval_method, - self.metric, - self.best_loss, - self.n_jobs, - self.learner_classes.get(estimator), - self.log_training_metric, - self.fit_kwargs) + ( + trained_estimator, + val_loss, + metric_for_logging, + _, + pred_time, + ) = compute_estimator( + sampled_X_train, + sampled_y_train, + self.X_val, + self.y_val, + self.weight_val, + self.groups_val, + min(budget, self.train_time_limit), + self.kf, + config, + self.task, + estimator, + self.eval_method, + self.metric, + self.best_loss, + self.n_jobs, + self.learner_classes.get(estimator), + self.log_training_metric, + self.fit_kwargs, + ) result = { - 'pred_time': pred_time, - 'wall_clock_time': time.time() - self._start_time_flag, - 'metric_for_logging': metric_for_logging, - 'val_loss': val_loss, - 'trained_estimator': trained_estimator + "pred_time": pred_time, + "wall_clock_time": time.time() - self._start_time_flag, + "metric_for_logging": metric_for_logging, + "val_loss": val_loss, + "trained_estimator": trained_estimator, } if sampled_weight is not None: - self.fit_kwargs['sample_weight'] = weight + self.fit_kwargs["sample_weight"] = weight # tune.report(**result) return result - def _train_with_config( - self, estimator, config_w_resource, sample_size=None - ): + def _train_with_config(self, estimator, config_w_resource, sample_size=None): if not sample_size: sample_size = config_w_resource.get( - 'FLAML_sample_size', len(self.y_train_all)) - config = config_w_resource.get('ml', config_w_resource).copy() - if 'FLAML_sample_size' in config: - del config['FLAML_sample_size'] + "FLAML_sample_size", len(self.y_train_all) + ) + config = config_w_resource.get("ml", config_w_resource).copy() + if "FLAML_sample_size" in config: + del config["FLAML_sample_size"] if "learner" in config: del config["learner"] - sampled_X_train, sampled_y_train, sampled_weight, groups = \ - self._prepare_sample_train_data(sample_size) + ( + sampled_X_train, + sampled_y_train, + sampled_weight, + groups, + ) = self._prepare_sample_train_data(sample_size) if sampled_weight is not None: - weight = self.fit_kwargs['sample_weight'] - self.fit_kwargs['sample_weight'] = sampled_weight + weight = self.fit_kwargs["sample_weight"] + self.fit_kwargs["sample_weight"] = sampled_weight else: weight = None if groups is not None: - self.fit_kwargs['groups'] = groups - budget = None if self.time_budget is None else ( - self.time_budget - self.time_from_start) + self.fit_kwargs["groups"] = groups + budget = ( + None + if self.time_budget is None + else self.time_budget - self.time_from_start + ) estimator, train_time = train_estimator( sampled_X_train, sampled_y_train, @@ -250,26 +285,27 @@ class AutoMLState: self.n_jobs, self.learner_classes.get(estimator), budget, - self.fit_kwargs) + self.fit_kwargs, + ) if sampled_weight is not None: - self.fit_kwargs['sample_weight'] = weight + self.fit_kwargs["sample_weight"] = weight return estimator, train_time def size(state: AutoMLState, config: dict) -> float: - '''Size function + """Size function Returns: The mem size in bytes for a config - ''' - config = config.get('ml', config) - estimator = config['learner'] + """ + config = config.get("ml", config) + estimator = config["learner"] learner_class = state.learner_classes.get(estimator) return learner_class.size(config) class AutoML: - '''The AutoML class + """The AutoML class Example: @@ -285,7 +321,7 @@ class AutoML: automl.fit(X_train = X_train, y_train = y_train, **automl_settings) - ''' + """ from .version import __version__ @@ -296,28 +332,28 @@ class AutoML: @property def model_history(self): - '''A dictionary of iter->model, storing the models when + """A dictionary of iter->model, storing the models when the best model is updated each time. - ''' + """ return self._model_history @property def config_history(self): - '''A dictionary of iter->(estimator, config, time), + """A dictionary of iter->(estimator, config, time), storing the best estimator, config, and the time when the best model is updated each time. - ''' + """ return self._config_history @property def model(self): - '''An object with `predict()` and `predict_proba()` method (for + """An object with `predict()` and `predict_proba()` method (for classification), storing the best trained model. - ''' - return self.__dict__.get('_trained_estimator') + """ + return self.__dict__.get("_trained_estimator") def best_model_for_estimator(self, estimator_name): - '''Return the best model found for a particular estimator + """Return the best model found for a particular estimator Args: estimator_name: a str of the estimator's name @@ -325,47 +361,48 @@ class AutoML: Returns: An object with `predict()` and `predict_proba()` method (for classification), storing the best trained model for estimator_name. - ''' + """ state = self._search_states.get(estimator_name) - return state and getattr(state, 'trained_estimator', None) + return state and getattr(state, "trained_estimator", None) @property def best_estimator(self): - '''A string indicating the best estimator found.''' + """A string indicating the best estimator found.""" return self._best_estimator @property def best_iteration(self): - '''An integer of the iteration number where the best - config is found.''' + """An integer of the iteration number where the best + config is found.""" return self._best_iteration @property def best_config(self): - '''A dictionary of the best configuration.''' + """A dictionary of the best configuration.""" return self._search_states[self._best_estimator].best_config @property def best_config_per_estimator(self): - '''A dictionary of all estimators' best configuration.''' - return {e: e_search_state.best_config for e, e_search_state in - self._search_states.items()} + """A dictionary of all estimators' best configuration.""" + return { + e: e_search_state.best_config + for e, e_search_state in self._search_states.items() + } @property def best_loss(self): - '''A float of the best loss found - ''' + """A float of the best loss found""" return self._state.best_loss @property def best_config_train_time(self): - '''A float of the seconds taken by training the - best config.''' + """A float of the seconds taken by training the + best config.""" return self._search_states[self._best_estimator].best_config_train_time @property def classes_(self): - '''A list of n_classes elements for class labels.''' + """A list of n_classes elements for class labels.""" attr = getattr(self, "_label_transformer", None) if attr: return attr.classes_.tolist() @@ -375,7 +412,7 @@ class AutoML: return None def predict(self, X_test): - '''Predict label from features. + """Predict label from features. Args: X_test: A numpy array of featurized instances, shape n * m, @@ -387,24 +424,24 @@ class AutoML: Returns: A array-like of shape n * 1 - - each element is a predicted label for an instance. - ''' + """ estimator = getattr(self, "_trained_estimator", None) if estimator is None: logger.warning( - "No estimator is trained. Please run fit with enough budget.") + "No estimator is trained. Please run fit with enough budget." + ) return None X_test = self._preprocess(X_test) y_pred = estimator.predict(X_test) if y_pred.ndim > 1 and isinstance(y_pred, np.ndarray): y_pred = y_pred.flatten() if self._label_transformer: - return self._label_transformer.inverse_transform(pd.Series( - y_pred)) + return self._label_transformer.inverse_transform(pd.Series(y_pred)) else: return y_pred def predict_proba(self, X_test): - '''Predict the probability of each class from features, only works for + """Predict the probability of each class from features, only works for classification problems. Args: @@ -413,7 +450,7 @@ class AutoML: Returns: A numpy array of shape n * c. c is the # classes. Each element at (i, j) is the probability for instance i to be in class j. - ''' + """ X_test = self._preprocess(X_test) proba = self._trained_estimator.predict_proba(X_test) return proba @@ -421,9 +458,9 @@ class AutoML: def _preprocess(self, X): if isinstance(X, int): return X - if self._state.task == 'forecast': + if self._state.task == "forecast": X = pd.DataFrame(X) - X = X.rename(columns={X.columns[0]: 'ds'}) + X = X.rename(columns={X.columns[0]: "ds"}) else: if issparse(X): X = X.tocsr() @@ -431,81 +468,101 @@ class AutoML: X = self._transformer.transform(X) return X - def _validate_data(self, X_train_all, y_train_all, dataframe, label, - X_val=None, y_val=None, groups_val=None, groups=None): - if self._state.task == 'forecast': + def _validate_data( + self, + X_train_all, + y_train_all, + dataframe, + label, + X_val=None, + y_val=None, + groups_val=None, + groups=None, + ): + if self._state.task == "forecast": if dataframe is not None and label is not None: dataframe = dataframe.copy() - dataframe = dataframe.rename(columns={label[0]: 'ds', label[1]: 'y'}) + dataframe = dataframe.rename(columns={label[0]: "ds", label[1]: "y"}) elif dataframe is not None: - assert 'ds' in dataframe and 'y' in dataframe, ( - 'For forecasting task, dataframe must have columns ' - '"ds" and "y" with the dates and values respectively.') + assert "ds" in dataframe and "y" in dataframe, ( + "For forecasting task, dataframe must have columns " + '"ds" and "y" with the dates and values respectively.' + ) elif (X_train_all is not None) and (y_train_all is not None): dataframe = pd.DataFrame(X_train_all) - dataframe = dataframe.rename(columns={dataframe.columns[0]: 'ds'}) - dataframe['y'] = pd.Series(y_train_all) + dataframe = dataframe.rename(columns={dataframe.columns[0]: "ds"}) + dataframe["y"] = pd.Series(y_train_all) X_train_all = None y_train_all = None - label = 'y' + label = "y" if X_train_all is not None and y_train_all is not None: assert ( - isinstance(X_train_all, np.ndarray) or issparse(X_train_all) - or isinstance(X_train_all, pd.DataFrame)), ( + isinstance(X_train_all, np.ndarray) + or issparse(X_train_all) + or isinstance(X_train_all, pd.DataFrame) + ), ( "X_train_all must be a numpy array, a pandas dataframe, " - "or Scipy sparse matrix.") + "or Scipy sparse matrix." + ) + assert isinstance(y_train_all, np.ndarray) or isinstance( + y_train_all, pd.Series + ), "y_train_all must be a numpy array or a pandas series." assert ( - isinstance(y_train_all, np.ndarray) - or isinstance(y_train_all, pd.Series)), ( - "y_train_all must be a numpy array or a pandas series.") - assert X_train_all.size != 0 and y_train_all.size != 0, ( - "Input data must not be empty.") + X_train_all.size != 0 and y_train_all.size != 0 + ), "Input data must not be empty." if isinstance(y_train_all, np.ndarray): y_train_all = y_train_all.flatten() - assert X_train_all.shape[0] == y_train_all.shape[0], ( - "# rows in X_train must match length of y_train.") + assert ( + X_train_all.shape[0] == y_train_all.shape[0] + ), "# rows in X_train must match length of y_train." self._df = isinstance(X_train_all, pd.DataFrame) self._nrow, self._ndim = X_train_all.shape X, y = X_train_all, y_train_all elif dataframe is not None and label is not None: - assert isinstance(dataframe, pd.DataFrame), ( - "dataframe must be a pandas DataFrame") - assert label in dataframe.columns, ( - "label must a column name in dataframe") + assert isinstance( + dataframe, pd.DataFrame + ), "dataframe must be a pandas DataFrame" + assert label in dataframe.columns, "label must a column name in dataframe" self._df = True X = dataframe.drop(columns=label) self._nrow, self._ndim = X.shape y = dataframe[label] else: - raise ValueError( - "either X_train+y_train or dataframe+label are required") - if issparse(X_train_all) or self._state.task == 'forecast': + raise ValueError("either X_train+y_train or dataframe+label are required") + if issparse(X_train_all) or self._state.task == "forecast": self._transformer = self._label_transformer = False self._X_train_all, self._y_train_all = X, y else: from .data import DataTransformer + self._transformer = DataTransformer() - self._X_train_all, self._y_train_all = \ - self._transformer.fit_transform(X, y, self._state.task) + self._X_train_all, self._y_train_all = self._transformer.fit_transform( + X, y, self._state.task + ) self._label_transformer = self._transformer.label_transformer - self._sample_weight_full = self._state.fit_kwargs.get('sample_weight') + self._sample_weight_full = self._state.fit_kwargs.get("sample_weight") if X_val is not None and y_val is not None: assert ( - isinstance(X_val, np.ndarray) or issparse(X_val) - or isinstance(X_val, pd.DataFrame)), ( + isinstance(X_val, np.ndarray) + or issparse(X_val) + or isinstance(X_val, pd.DataFrame) + ), ( "X_val must be None, a numpy array, a pandas dataframe, " - "or Scipy sparse matrix.") - assert ( - isinstance(y_val, np.ndarray) or isinstance(y_val, pd.Series) + "or Scipy sparse matrix." + ) + assert isinstance(y_val, np.ndarray) or isinstance( + y_val, pd.Series ), "y_val must be None, a numpy array or a pandas series." assert X_val.size != 0 and y_val.size != 0, ( "Validation data are expected to be nonempty. " - "Use None for X_val and y_val if no validation data.") + "Use None for X_val and y_val if no validation data." + ) if isinstance(y_val, np.ndarray): y_val = y_val.flatten() - assert X_val.shape[0] == y_val.shape[0], ( - "# rows in X_val must match length of y_val.") + assert ( + X_val.shape[0] == y_val.shape[0] + ), "# rows in X_val must match length of y_val." if self._transformer: self._state.X_val = self._transformer.transform(X_val) else: @@ -518,30 +575,31 @@ class AutoML: self._state.X_val = self._state.y_val = None if groups is not None and len(groups) != self._nrow: # groups is given as group counts - self._state.groups = np.concatenate( - [[i] * c for i, c in enumerate(groups)]) - assert len(self._state.groups) == self._nrow, \ - "the sum of group counts must match the number of examples" - self._state.groups_val = np.concatenate( - [[i] * c for i, c in enumerate(groups_val)] - ) if groups_val is not None else None + self._state.groups = np.concatenate([[i] * c for i, c in enumerate(groups)]) + assert ( + len(self._state.groups) == self._nrow + ), "the sum of group counts must match the number of examples" + self._state.groups_val = ( + np.concatenate([[i] * c for i, c in enumerate(groups_val)]) + if groups_val is not None + else None + ) else: self._state.groups_val = groups_val self._state.groups = groups - def _prepare_data(self, - eval_method, - split_ratio, - n_splits): + def _prepare_data(self, eval_method, split_ratio, n_splits): X_val, y_val = self._state.X_val, self._state.y_val if issparse(X_val): X_val = X_val.tocsr() X_train_all, y_train_all = self._X_train_all, self._y_train_all if issparse(X_train_all): X_train_all = X_train_all.tocsr() - if self._state.task in ('binary', 'multi') \ - and self._state.fit_kwargs.get('sample_weight') is None \ - and self._split_type != 'time': + if ( + self._state.task in ("binary", "multi") + and self._state.fit_kwargs.get("sample_weight") is None + and self._split_type != "time" + ): # logger.info(f"label {pd.unique(y_train_all)}") label_set, counts = np.unique(y_train_all, return_counts=True) # augment rare classes @@ -554,31 +612,37 @@ class AutoML: n = len(y_train_all) while count < rare_threshld: if self._df: - X_train_all = concat(X_train_all, - X_train_all.iloc[:n].loc[rare_index]) + X_train_all = concat( + X_train_all, X_train_all.iloc[:n].loc[rare_index] + ) else: - X_train_all = concat(X_train_all, - X_train_all[:n][rare_index, :]) + X_train_all = concat( + X_train_all, X_train_all[:n][rare_index, :] + ) if isinstance(y_train_all, pd.Series): - y_train_all = concat(y_train_all, - y_train_all.iloc[:n].loc[rare_index]) + y_train_all = concat( + y_train_all, y_train_all.iloc[:n].loc[rare_index] + ) else: - y_train_all = np.concatenate([y_train_all, - y_train_all[:n][rare_index]]) + y_train_all = np.concatenate( + [y_train_all, y_train_all[:n][rare_index]] + ) count += rare_count - logger.info( - f"class {label} augmented from {rare_count} to {count}") - SHUFFLE_SPLIT_TYPES = ['uniform', 'stratified'] + logger.info(f"class {label} augmented from {rare_count} to {count}") + SHUFFLE_SPLIT_TYPES = ["uniform", "stratified"] if self._split_type in SHUFFLE_SPLIT_TYPES: if self._sample_weight_full is not None: - X_train_all, y_train_all, self._state.sample_weight_all = \ - shuffle(X_train_all, y_train_all, self._sample_weight_full, - random_state=RANDOM_SEED) - self._state.fit_kwargs[ - 'sample_weight'] = self._state.sample_weight_all + X_train_all, y_train_all, self._state.sample_weight_all = shuffle( + X_train_all, + y_train_all, + self._sample_weight_full, + random_state=RANDOM_SEED, + ) + self._state.fit_kwargs["sample_weight"] = self._state.sample_weight_all else: X_train_all, y_train_all = shuffle( - X_train_all, y_train_all, random_state=RANDOM_SEED) + X_train_all, y_train_all, random_state=RANDOM_SEED + ) if self._df: X_train_all.reset_index(drop=True, inplace=True) if isinstance(y_train_all, pd.Series): @@ -586,50 +650,59 @@ class AutoML: X_train, y_train = X_train_all, y_train_all self._state.groups_all = self._state.groups - if X_val is None and eval_method == 'holdout': + if X_val is None and eval_method == "holdout": # if eval_method = holdout, make holdout data - if self._split_type == 'time': - if self._state.task == 'forecast': + if self._split_type == "time": + if self._state.task == "forecast": num_samples = X_train_all.shape[0] - period = self._state.fit_kwargs['period'] - assert period < num_samples, ( - f"period={period}>#examples={num_samples}") + period = self._state.fit_kwargs["period"] + assert ( + period < num_samples + ), f"period={period}>#examples={num_samples}" split_idx = num_samples - period X_train = X_train_all[:split_idx] y_train = y_train_all[:split_idx] X_val = X_train_all[split_idx:] y_val = y_train_all[split_idx:] else: - if 'sample_weight' in self._state.fit_kwargs: - X_train, X_val, y_train, y_val, self._state.fit_kwargs[ - 'sample_weight'], self._state.weight_val = \ - train_test_split( - X_train_all, - y_train_all, - self._state.fit_kwargs['sample_weight'], - test_size=split_ratio, - shuffle=False) + if "sample_weight" in self._state.fit_kwargs: + ( + X_train, + X_val, + y_train, + y_val, + self._state.fit_kwargs["sample_weight"], + self._state.weight_val, + ) = train_test_split( + X_train_all, + y_train_all, + self._state.fit_kwargs["sample_weight"], + test_size=split_ratio, + shuffle=False, + ) else: X_train, X_val, y_train, y_val = train_test_split( X_train_all, y_train_all, test_size=split_ratio, - shuffle=False) - elif self._state.task == 'rank': - gss = GroupShuffleSplit(n_splits=1, test_size=split_ratio, - random_state=RANDOM_SEED) - for train_idx, val_idx in gss.split(X_train_all, y_train_all, - self._state.groups): + shuffle=False, + ) + elif self._state.task == "rank": + gss = GroupShuffleSplit( + n_splits=1, test_size=split_ratio, random_state=RANDOM_SEED + ) + for train_idx, val_idx in gss.split( + X_train_all, y_train_all, self._state.groups + ): if self._df: - X_train, X_val = X_train_all.iloc[ - train_idx], X_train_all.iloc[val_idx] + X_train = X_train_all.iloc[train_idx] + X_val = X_train_all.iloc[val_idx] else: - X_train, X_val = X_train_all[ - train_idx], X_train_all[val_idx] + X_train, X_val = X_train_all[train_idx], X_train_all[val_idx] y_train, y_val = y_train_all[train_idx], y_train_all[val_idx] - self._state.groups, self._state.groups_val = self._state.groups[ - train_idx], self._state.groups[val_idx] - elif self._state.task in ('binary', 'multi'): + self._state.groups = self._state.groups[train_idx] + self._state.groups_val = self._state.groups[val_idx] + elif self._state.task in ("binary", "multi"): # for classification, make sure the labels are complete in both # training and validation data label_set, first = np.unique(y_train_all, return_index=True) @@ -640,111 +713,133 @@ class AutoML: rest.extend(range(last, first[i])) last = first[i] + 1 rest.extend(range(last, len(y_train_all))) - X_first = X_train_all.iloc[first] if self._df else X_train_all[ - first] + X_first = X_train_all.iloc[first] if self._df else X_train_all[first] X_rest = X_train_all.iloc[rest] if self._df else X_train_all[rest] y_rest = y_train_all[rest] - stratify = y_rest if self._split_type == 'stratified' else \ - None - if 'sample_weight' in self._state.fit_kwargs: - X_train, X_val, y_train, y_val, weight_train, weight_val = \ - train_test_split( - X_rest, - y_rest, - self._state.fit_kwargs['sample_weight'][rest], - test_size=split_ratio, - random_state=RANDOM_SEED) - weight1 = self._state.fit_kwargs['sample_weight'][first] + stratify = y_rest if self._split_type == "stratified" else None + if "sample_weight" in self._state.fit_kwargs: + ( + X_train, + X_val, + y_train, + y_val, + weight_train, + weight_val, + ) = train_test_split( + X_rest, + y_rest, + self._state.fit_kwargs["sample_weight"][rest], + test_size=split_ratio, + random_state=RANDOM_SEED, + ) + weight1 = self._state.fit_kwargs["sample_weight"][first] self._state.weight_val = concat(weight1, weight_val) - self._state.fit_kwargs['sample_weight'] = concat( - weight1, weight_train) + self._state.fit_kwargs["sample_weight"] = concat( + weight1, weight_train + ) else: X_train, X_val, y_train, y_val = train_test_split( X_rest, y_rest, test_size=split_ratio, stratify=stratify, - random_state=RANDOM_SEED) + random_state=RANDOM_SEED, + ) X_train = concat(X_first, X_train) - y_train = concat( - label_set, y_train) if self._df else np.concatenate( - [label_set, y_train]) + y_train = ( + concat(label_set, y_train) + if self._df + else np.concatenate([label_set, y_train]) + ) X_val = concat(X_first, X_val) - y_val = concat(label_set, y_val) if self._df else \ - np.concatenate([label_set, y_val]) - elif self._state.task == 'regression': - if 'sample_weight' in self._state.fit_kwargs: - X_train, X_val, y_train, y_val, self._state.fit_kwargs[ - 'sample_weight'], self._state.weight_val = \ - train_test_split( - X_train_all, - y_train_all, - self._state.fit_kwargs['sample_weight'], - test_size=split_ratio, - random_state=RANDOM_SEED) + y_val = ( + concat(label_set, y_val) + if self._df + else np.concatenate([label_set, y_val]) + ) + elif self._state.task == "regression": + if "sample_weight" in self._state.fit_kwargs: + ( + X_train, + X_val, + y_train, + y_val, + self._state.fit_kwargs["sample_weight"], + self._state.weight_val, + ) = train_test_split( + X_train_all, + y_train_all, + self._state.fit_kwargs["sample_weight"], + test_size=split_ratio, + random_state=RANDOM_SEED, + ) else: X_train, X_val, y_train, y_val = train_test_split( X_train_all, y_train_all, test_size=split_ratio, - random_state=RANDOM_SEED) + random_state=RANDOM_SEED, + ) self._state.data_size = X_train.shape[0] self.data_size_full = len(y_train_all) - self._state.X_train, self._state.y_train, self._state.X_val, \ - self._state.y_val = (X_train, y_train, X_val, y_val) + self._state.X_train, self._state.y_train = X_train, y_train + self._state.X_val, self._state.y_val = X_val, y_val self._state.X_train_all = X_train_all self._state.y_train_all = y_train_all - if self._split_type == 'group': + if self._split_type == "group": # logger.info("Using GroupKFold") - assert len(self._state.groups_all) == y_train_all.size, \ - "the length of groups must match the number of examples" - assert len(np.unique(self._state.groups_all)) >= n_splits, \ - "the number of groups must be equal or larger than n_splits" + assert ( + len(self._state.groups_all) == y_train_all.size + ), "the length of groups must match the number of examples" + assert ( + len(np.unique(self._state.groups_all)) >= n_splits + ), "the number of groups must be equal or larger than n_splits" self._state.kf = GroupKFold(n_splits) self._state.kf.groups = self._state.groups_all elif self._split_type == "stratified": # logger.info("Using StratifiedKFold") assert y_train_all.size >= n_splits, ( f"{n_splits}-fold cross validation" - f" requires input data with at least {n_splits} examples.") + f" requires input data with at least {n_splits} examples." + ) assert y_train_all.size >= 2 * n_splits, ( f"{n_splits}-fold cross validation with metric=r2 " - f"requires input data with at least {n_splits*2} examples.") + f"requires input data with at least {n_splits*2} examples." + ) self._state.kf = RepeatedStratifiedKFold( - n_splits=n_splits, n_repeats=1, random_state=RANDOM_SEED) + n_splits=n_splits, n_repeats=1, random_state=RANDOM_SEED + ) elif self._split_type == "time": # logger.info("Using TimeSeriesSplit") - if self._state.task == 'forecast': - period = self._state.fit_kwargs['period'] + if self._state.task == "forecast": + period = self._state.fit_kwargs["period"] if period * (n_splits + 1) > y_train_all.size: n_splits = int(y_train_all.size / period - 1) assert n_splits >= 2, ( f"cross validation for forecasting period={period}" - f" requires input data with at least {3 * period} examples.") - logger.info( - f"Using nsplits={n_splits} due to data size limit.") - self._state.kf = TimeSeriesSplit( - n_splits=n_splits, test_size=period) + f" requires input data with at least {3 * period} examples." + ) + logger.info(f"Using nsplits={n_splits} due to data size limit.") + self._state.kf = TimeSeriesSplit(n_splits=n_splits, test_size=period) else: self._state.kf = TimeSeriesSplit(n_splits=n_splits) else: # logger.info("Using RepeatedKFold") self._state.kf = RepeatedKFold( - n_splits=n_splits, n_repeats=1, random_state=RANDOM_SEED) + n_splits=n_splits, n_repeats=1, random_state=RANDOM_SEED + ) - def add_learner(self, - learner_name, - learner_class): - '''Add a customized learner + def add_learner(self, learner_name, learner_class): + """Add a customized learner Args: learner_name: A string of the learner's name learner_class: A subclass of flaml.model.BaseEstimator - ''' + """ self._state.learner_classes[learner_name] = learner_class def get_estimator_from_log(self, log_file_name, record_id, task): - '''Get the estimator from log file + """Get the estimator from log file Args: log_file_name: A string of the log file name @@ -755,7 +850,7 @@ class AutoML: Returns: An estimator object for the given configuration - ''' + """ with training_log_reader(log_file_name) as reader: record = reader.get_record(record_id) @@ -763,29 +858,36 @@ class AutoML: config = record.config estimator, _ = train_estimator( - None, None, config, task, estimator, - estimator_class=self._state.learner_classes.get(estimator)) + None, + None, + config, + task, + estimator, + estimator_class=self._state.learner_classes.get(estimator), + ) return estimator - def retrain_from_log(self, - log_file_name, - X_train=None, - y_train=None, - dataframe=None, - label=None, - time_budget=0, - task='classification', - eval_method='auto', - split_ratio=SPLIT_RATIO, - n_splits=N_SPLITS, - split_type=None, - groups=None, - n_jobs=1, - train_best=True, - train_full=False, - record_id=-1, - **fit_kwargs): - '''Retrain from log file + def retrain_from_log( + self, + log_file_name, + X_train=None, + y_train=None, + dataframe=None, + label=None, + time_budget=0, + task="classification", + eval_method="auto", + split_ratio=SPLIT_RATIO, + n_splits=N_SPLITS, + split_type=None, + groups=None, + n_jobs=1, + train_best=True, + train_full=False, + record_id=-1, + **fit_kwargs, + ): + """Retrain from log file Args: log_file_name: A string of the log file name @@ -829,15 +931,15 @@ class AutoML: when `record_id >= 0`, `time_budget` will be ignored. **fit_kwargs: Other key word arguments to pass to fit() function of the searched learners, such as sample_weight. - ''' + """ self._state.task = task self._state.fit_kwargs = fit_kwargs self._validate_data(X_train, y_train, dataframe, label, groups=groups) - logger.info('log file name {}'.format(log_file_name)) + logger.info("log file name {}".format(log_file_name)) best_config = None - best_val_loss = float('+inf') + best_val_loss = float("+inf") best_estimator = None sample_size = None time_used = 0.0 @@ -867,93 +969,102 @@ class AutoML: sample_size = size if not training_duration: logger.warning( - f"No estimator found within time_budget={time_budget}") + f"No estimator found within time_budget={time_budget}" + ) from .model import BaseEstimator as Estimator + self._trained_estimator = Estimator() return training_duration if not best: return best_estimator = best.learner best_config = best.config - sample_size = len(self._y_train_all) if train_full \ - else best.sample_size + sample_size = len(self._y_train_all) if train_full else best.sample_size logger.info( - 'estimator = {}, config = {}, #training instances = {}'.format( - best_estimator, best_config, sample_size)) + "estimator = {}, config = {}, #training instances = {}".format( + best_estimator, best_config, sample_size + ) + ) # Partially copied from fit() function # Initilize some attributes required for retrain_from_log self._state.task = task self._decide_split_type(split_type) if record_id >= 0: - eval_method = 'cv' - elif eval_method == 'auto': + eval_method = "cv" + elif eval_method == "auto": eval_method = self._decide_eval_method(time_budget) self.modelcount = 0 self._prepare_data(eval_method, split_ratio, n_splits) self._state.time_budget = None self._state.n_jobs = n_jobs self._trained_estimator = self._state._train_with_config( - best_estimator, best_config, sample_size)[0] - logger.info('retrain from log succeeded') + best_estimator, best_config, sample_size + )[0] + logger.info("retrain from log succeeded") return training_duration def _decide_split_type(self, split_type): - if self._state.task == 'classification': + if self._state.task == "classification": self._state.task = get_classification_objective( - len(np.unique(self._y_train_all))) - if self._state.task in ('binary', 'multi'): + len(np.unique(self._y_train_all)) + ) + if self._state.task in ("binary", "multi"): assert split_type in [None, "stratified", "uniform", "time"] self._split_type = split_type or "stratified" - elif self._state.task == 'regression': + elif self._state.task == "regression": assert split_type in [None, "uniform", "time"] self._split_type = split_type or "uniform" - elif self._state.task == 'forecast': + elif self._state.task == "forecast": assert split_type in [None, "time"] self._split_type = "time" - assert isinstance(self._state.fit_kwargs.get('period'), int), ( - "missing a required integer 'period' for forecast.") - elif self._state.task == 'rank': - assert self._state.groups is not None, \ - 'groups must be specified for ranking task.' + assert isinstance( + self._state.fit_kwargs.get("period"), int + ), "missing a required integer 'period' for forecast." + elif self._state.task == "rank": + assert ( + self._state.groups is not None + ), "groups must be specified for ranking task." assert split_type in [None, "group"] - self._split_type = 'group' + self._split_type = "group" def _decide_eval_method(self, time_budget): if self._state.X_val is not None: - return 'holdout' + return "holdout" nrow, dim = self._nrow, self._ndim - if nrow * dim / 0.9 < SMALL_LARGE_THRES * ( - time_budget / 3600) and nrow < CV_HOLDOUT_THRESHOLD: + if ( + nrow * dim / 0.9 < SMALL_LARGE_THRES * (time_budget / 3600) + and nrow < CV_HOLDOUT_THRESHOLD + ): # time allows or sampling can be used and cv is necessary - return 'cv' + return "cv" else: - return 'holdout' + return "holdout" @property def search_space(self) -> dict: - '''Search space + """Search space Must be called after fit(...) (use max_iter=0 to prevent actual fitting) Returns: A dict of the search space - ''' + """ estimator_list = self.estimator_list if len(estimator_list) == 1: estimator = estimator_list[0] space = self._search_states[estimator].search_space.copy() - space['learner'] = estimator + space["learner"] = estimator return space choices = [] for estimator in estimator_list: space = self._search_states[estimator].search_space.copy() - space['learner'] = estimator + space["learner"] = estimator choices.append(space) - return {'ml': tune.choice(choices)} + return {"ml": tune.choice(choices)} @property def low_cost_partial_config(self) -> dict: - '''Low cost partial config + """Low cost partial config Returns: A dict. @@ -965,7 +1076,7 @@ class AutoML: an integer corresponding to the cheapest learner is appeneded to the list at the end. - ''' + """ if len(self.estimator_list) == 1: estimator = self.estimator_list[0] c = self._search_states[estimator].low_cost_partial_config @@ -975,15 +1086,20 @@ class AutoML: for estimator in self.estimator_list: c = self._search_states[estimator].low_cost_partial_config configs.append(c) - configs.append(np.argmin([ - self._state.learner_classes.get(estimator).cost_relative2lgbm() - for estimator in self.estimator_list])) - config = {'ml': configs} + configs.append( + np.argmin( + [ + self._state.learner_classes.get(estimator).cost_relative2lgbm() + for estimator in self.estimator_list + ] + ) + ) + config = {"ml": configs} return config @property def cat_hp_cost(self) -> dict: - '''Categorical hyperparameter cost + """Categorical hyperparameter cost Returns: A dict. @@ -994,7 +1110,7 @@ class AutoML: to each learner's cat_hp_cost; the cost relative to lgbm for each learner (as a list itself) is appended to the list at the end. - ''' + """ if len(self.estimator_list) == 1: estimator = self.estimator_list[0] c = self._search_states[estimator].cat_hp_cost @@ -1004,145 +1120,157 @@ class AutoML: for estimator in self.estimator_list: c = self._search_states[estimator].cat_hp_cost configs.append(c) - configs.append([ - self._state.learner_classes.get(estimator).cost_relative2lgbm() - for estimator in self.estimator_list]) - config = {'ml': configs} + configs.append( + [ + self._state.learner_classes.get(estimator).cost_relative2lgbm() + for estimator in self.estimator_list + ] + ) + config = {"ml": configs} return config @property def points_to_evaluate(self) -> dict: - '''Initial points to evaluate + """Initial points to evaluate Returns: A list of dicts. Each dict is the initial point for each learner - ''' + """ points = [] for estimator in self.estimator_list: - config = self._search_states[estimator].init_config - config['learner'] = estimator - if len(self.estimator_list) > 1: - points.append({'ml': config}) + if isinstance(self._search_states[estimator].init_config, list): + configs = self._search_states[estimator].init_config else: - points.append(config) + configs = [self._search_states[estimator].init_config] + for config in configs: + config['learner'] = estimator + if len(self.estimator_list) > 1: + points.append({'ml': config}) + else: + points.append(config) return points @property def prune_attr(self) -> Optional[str]: - '''Attribute for pruning + """Attribute for pruning Returns: A string for the sample size attribute or None - ''' - return 'FLAML_sample_size' if self._sample else None + """ + return "FLAML_sample_size" if self._sample else None @property def min_resource(self) -> Optional[float]: - '''Attribute for pruning + """Attribute for pruning Returns: A float for the minimal sample size or None - ''' + """ return MIN_SAMPLE_TRAIN if self._sample else None @property def max_resource(self) -> Optional[float]: - '''Attribute for pruning + """Attribute for pruning Returns: A float for the maximal sample size or None - ''' + """ return self._state.data_size if self._sample else None @property def trainable(self) -> Callable[[dict], Optional[float]]: - '''Training function + """Training function Returns: A function that evaluates each config and returns the loss - ''' + """ self._state.time_from_start = 0 for estimator in self.estimator_list: search_state = self._search_states[estimator] - if not hasattr(search_state, 'training_function'): + if not hasattr(search_state, "training_function"): search_state.training_function = partial( - AutoMLState._compute_with_config_base, - self._state, estimator) + AutoMLState._compute_with_config_base, self._state, estimator + ) states = self._search_states mem_res = self._mem_thres def train(config: dict): - sample_size = config.get('FLAML_sample_size') - config = config.get('ml', config).copy() + sample_size = config.get("FLAML_sample_size") + config = config.get("ml", config).copy() if sample_size: - config['FLAML_sample_size'] = sample_size - estimator = config['learner'] + config["FLAML_sample_size"] = sample_size + estimator = config["learner"] # check memory constraints before training if states[estimator].learner_class.size(config) <= mem_res: - del config['learner'] + del config["learner"] result = states[estimator].training_function(config) return result else: - return {'pred_time': 0, - 'wall_clock_time': None, - 'metric_for_logging': np.inf, - 'val_loss': np.inf, - 'trained_estimator': None - } + return { + "pred_time": 0, + "wall_clock_time": None, + "metric_for_logging": np.inf, + "val_loss": np.inf, + "trained_estimator": None, + } + return train @property def metric_constraints(self) -> list: - '''Metric constraints + """Metric constraints Returns: A list of the metric constraints - ''' + """ constraints = [] if np.isfinite(self._pred_time_limit): - constraints.append( - ('pred_time', '<=', self._pred_time_limit)) + constraints.append(("pred_time", "<=", self._pred_time_limit)) return constraints - def fit(self, - X_train=None, - y_train=None, - dataframe=None, - label=None, - metric='auto', - task='classification', - n_jobs=-1, - log_file_name='flaml.log', - estimator_list='auto', - time_budget=60, - max_iter=1000000, - sample=True, - ensemble=False, - eval_method='auto', - log_type='better', - model_history=False, - split_ratio=SPLIT_RATIO, - n_splits=N_SPLITS, - log_training_metric=False, - mem_thres=MEM_THRES, - pred_time_limit=np.inf, - train_time_limit=np.inf, - X_val=None, - y_val=None, - sample_weight_val=None, - groups_val=None, - groups=None, - verbose=1, - retrain_full=True, - split_type=None, - learner_selector='sample', - hpo_method=None, - starting_points={}, - seed=None, - n_concurrent_trials=1, - keep_search_state=False, - **fit_kwargs): - '''Find a model for a given task + def fit( + self, + X_train=None, + y_train=None, + dataframe=None, + label=None, + metric="auto", + task="classification", + n_jobs=-1, + log_file_name="flaml.log", + estimator_list="auto", + time_budget=60, + max_iter=1000000, + sample=True, + ensemble=False, + eval_method="auto", + log_type="better", + model_history=False, + split_ratio=SPLIT_RATIO, + n_splits=N_SPLITS, + log_training_metric=False, + mem_thres=MEM_THRES, + pred_time_limit=np.inf, + train_time_limit=np.inf, + X_val=None, + y_val=None, + sample_weight_val=None, + groups_val=None, + groups=None, + verbose=1, + retrain_full=True, + split_type=None, + learner_selector="sample", + hpo_method=None, + starting_points={}, + seed=None, + n_concurrent_trials=1, + keep_search_state=False, + append_log=False, + early_stop=False, + **fit_kwargs, + ): + """Find a model for a given task Args: X_train: A numpy array or a pandas dataframe of training data in @@ -1249,6 +1377,8 @@ class AutoML: config for the estimators. Keys are the name of the estimators, and values are the starting hyperparamter configurations for the corresponding estimators. + The value can be a single hyperparamter configuration dict or a list + of hyperparamter configuration dicts. seed: int or None, default=None | The random seed for np.random. n_concurrent_trials: [Experimental] int, default=1 | The number of concurrent trials. For n_concurrent_trials > 1, installation of @@ -1256,18 +1386,23 @@ class AutoML: keep_search_state: boolean, default=False | Whether to keep search state after fit(). By default the state is deleted for space saving. + early_stop: boolean, default=False | Whether to stop early if the + search is considered to converge. + append_log: boolean, default=False | whetehr to directly append the log + records to the input log file if it exists. **fit_kwargs: Other key word arguments to pass to fit() function of the searched learners, such as sample_weight. Include period as a key word argument for 'forecast' task. - ''' + """ self._state._start_time_flag = self._start_time_flag = time.time() self._state.task = task self._state.log_training_metric = log_training_metric self._state.fit_kwargs = fit_kwargs self._state.weight_val = sample_weight_val - self._validate_data(X_train, y_train, dataframe, label, X_val, y_val, - groups_val, groups) + self._validate_data( + X_train, y_train, dataframe, label, X_val, y_val, groups_val, groups + ) self._search_states = {} # key: estimator name; value: SearchState self._random = np.random.RandomState(RANDOM_SEED) if seed is not None: @@ -1278,7 +1413,7 @@ class AutoML: if verbose == 0: logger.setLevel(logging.WARNING) self._decide_split_type(split_type) - if eval_method == 'auto' or self._state.X_val is not None: + if eval_method == "auto" or self._state.X_val is not None: eval_method = self._decide_eval_method(time_budget) self._state.eval_method = eval_method if (not mlflow or not mlflow.active_run()) and not logger.handlers: @@ -1288,64 +1423,80 @@ class AutoML: logger.addHandler(_ch) logger.info("Evaluation method: {}".format(eval_method)) - self._retrain_in_budget = retrain_full == 'budget' and ( - eval_method == 'holdout' and self._state.X_val is None) - self._retrain_final = retrain_full is True and ( - eval_method == 'holdout' and self._state.X_val is None) or ( - eval_method == 'cv') + self._retrain_in_budget = retrain_full == "budget" and ( + eval_method == "holdout" and self._state.X_val is None + ) + self._retrain_final = ( + retrain_full is True + and (eval_method == "holdout" and self._state.X_val is None) + or (eval_method == "cv") + ) self._prepare_data(eval_method, split_ratio, n_splits) - self._sample = sample and task != 'rank' and eval_method != 'cv' and ( - MIN_SAMPLE_TRAIN * SAMPLE_MULTIPLY_FACTOR < self._state.data_size) - if 'auto' == metric: - if 'binary' in self._state.task: - metric = 'roc_auc' - elif 'multi' in self._state.task: - metric = 'log_loss' - elif self._state.task == 'forecast': - metric = 'mape' - elif self._state.task == 'rank': - metric = 'ndcg' + self._sample = ( + sample + and task != "rank" + and eval_method != "cv" + and (MIN_SAMPLE_TRAIN * SAMPLE_MULTIPLY_FACTOR < self._state.data_size) + ) + if "auto" == metric: + if "binary" in self._state.task: + metric = "roc_auc" + elif "multi" in self._state.task: + metric = "log_loss" + elif self._state.task == "forecast": + metric = "mape" + elif self._state.task == "rank": + metric = "ndcg" else: - metric = 'r2' + metric = "r2" self._state.metric = metric - if metric in ['r2', 'accuracy', 'roc_auc', 'roc_auc_ovr', 'roc_auc_ovo', - 'f1', 'ap', 'micro_f1', 'macro_f1', 'ndcg']: + if metric in [ + "r2", + "accuracy", + "roc_auc", + "roc_auc_ovr", + "roc_auc_ovo", + "f1", + "ap", + "micro_f1", + "macro_f1", + "ndcg", + ]: error_metric = f"1-{metric}" elif isinstance(metric, str): error_metric = metric else: - error_metric = 'customized metric' - logger.info(f'Minimizing error metric: {error_metric}') + error_metric = "customized metric" + logger.info(f"Minimizing error metric: {error_metric}") - if 'auto' == estimator_list: - if self._state.task == 'forecast': - estimator_list = ['fbprophet', 'arima', 'sarimax'] - elif self._state.task == 'rank': - estimator_list = ['lgbm', 'xgboost'] + if "auto" == estimator_list: + if self._state.task == "forecast": + estimator_list = ["fbprophet", "arima", "sarimax"] + elif self._state.task == "rank": + estimator_list = ["lgbm", "xgboost"] else: - estimator_list = [ - 'lgbm', 'rf', 'catboost', 'xgboost', 'extra_tree'] - if 'regression' != self._state.task: - estimator_list += ['lrl1'] + estimator_list = ["lgbm", "rf", "catboost", "xgboost", "extra_tree"] + if "regression" != self._state.task: + estimator_list += ["lrl1"] for estimator_name in estimator_list: if estimator_name not in self._state.learner_classes: self.add_learner( estimator_name, - get_estimator_class(self._state.task, estimator_name)) + get_estimator_class(self._state.task, estimator_name), + ) # set up learner search space for estimator_name in estimator_list: estimator_class = self._state.learner_classes[estimator_name] estimator_class.init() self._search_states[estimator_name] = SearchState( learner_class=estimator_class, - data_size=self._state.data_size, task=self._state.task, - starting_point=starting_points.get(estimator_name) + data_size=self._state.data_size, + task=self._state.task, + starting_point=starting_points.get(estimator_name), ) - logger.info("List of ML learners in AutoML Run: {}".format( - estimator_list)) + logger.info("List of ML learners in AutoML Run: {}".format(estimator_list)) self.estimator_list = estimator_list - self._hpo_method = hpo_method or ( - 'cfo' if n_concurrent_trials == 1 else 'bs') + self._hpo_method = hpo_method or ("cfo" if n_concurrent_trials == 1 else "bs") self._state.time_budget = time_budget self._active_estimators = estimator_list.copy() self._ensemble = ensemble @@ -1358,8 +1509,9 @@ class AutoML: self._save_model_history = model_history self._state.n_jobs = n_jobs self._n_concurrent_trials = n_concurrent_trials + self._early_stop = early_stop if log_file_name: - with training_log_writer(log_file_name) as save_helper: + with training_log_writer(log_file_name, append_log) as save_helper: self._training_log = save_helper self._search() else: @@ -1367,17 +1519,24 @@ class AutoML: self._search() if self._best_estimator: logger.info("fit succeeded") - logger.info(f"Time taken to find the best model: {self._time_taken_best_iter}") - if self._hpo_method in ('cfo', 'bs') and ( - self._time_taken_best_iter >= time_budget * 0.7) and not all( - state.search_alg and state.search_alg.searcher.is_ls_ever_converged - for state in self._search_states.values() + logger.info( + f"Time taken to find the best model: {self._time_taken_best_iter}" + ) + if ( + self._hpo_method in ("cfo", "bs") + and (self._time_taken_best_iter >= time_budget * 0.7) + and not all( + state.search_alg and state.search_alg.searcher.is_ls_ever_converged + for state in self._search_states.values() + ) ): logger.warning( "Time taken to find the best model is {0:.0f}% of the " "provided time budget and not all estimators' hyperparameter " "search converged. Consider increasing the time budget.".format( - self._time_taken_best_iter / time_budget * 100)) + self._time_taken_best_iter / time_budget * 100 + ) + ) if not keep_search_state: # release space @@ -1395,26 +1554,29 @@ class AutoML: def _search_parallel(self): try: from ray import __version__ as ray_version - assert ray_version >= '1.0.0' + + assert ray_version >= "1.0.0" import ray from ray.tune.suggest import ConcurrencyLimiter except (ImportError, AssertionError): raise ImportError( "n_concurrent_trial > 1 requires installation of ray. " - "Please run pip install flaml[ray]") - if self._hpo_method in ('cfo', 'grid'): + "Please run pip install flaml[ray]" + ) + if self._hpo_method in ("cfo", "grid"): from flaml import CFO as SearchAlgo - elif 'bs' == self._hpo_method: + elif "bs" == self._hpo_method: from flaml import BlendSearch as SearchAlgo - elif 'random' == self._hpo_method: + elif "random" == self._hpo_method: from ray.tune.suggest import BasicVariantGenerator as SearchAlgo from ray.tune.sample import Domain else: raise NotImplementedError( f"hpo_method={self._hpo_method} is not recognized. " - "'cfo' and 'bs' are supported.") + "'cfo' and 'bs' are supported." + ) space = self.search_space - if self._hpo_method == 'random': + if self._hpo_method == "random": # Any point in points_to_evaluate must consist of hyperparamters # that are tunable, which can be identified by checking whether # the corresponding value in the search space is an instance of @@ -1430,44 +1592,63 @@ class AutoML: del p[k] search_alg = SearchAlgo( max_concurrent=self._n_concurrent_trials, - points_to_evaluate=points_to_evaluate) + points_to_evaluate=points_to_evaluate, + ) else: search_alg = SearchAlgo( - metric='val_loss', space=space, + metric="val_loss", + space=space, low_cost_partial_config=self.low_cost_partial_config, points_to_evaluate=self.points_to_evaluate, cat_hp_cost=self.cat_hp_cost, prune_attr=self.prune_attr, min_resource=self.min_resource, max_resource=self.max_resource, - config_constraints=[(partial(size, self._state), '<=', self._mem_thres)], - metric_constraints=self.metric_constraints) + config_constraints=[ + (partial(size, self._state), "<=", self._mem_thres) + ], + metric_constraints=self.metric_constraints, + ) search_alg = ConcurrencyLimiter(search_alg, self._n_concurrent_trials) self._state.time_from_start = time.time() - self._start_time_flag time_left = self._state.time_budget - self._state.time_from_start - search_alg.set_search_properties(None, None, config={ - 'time_budget_s': time_left}) - resources_per_trial = { - "cpu": self._state.n_jobs} if self._state.n_jobs > 1 else None + search_alg.set_search_properties( + None, None, config={"time_budget_s": time_left} + ) + resources_per_trial = ( + {"cpu": self._state.n_jobs} if self._state.n_jobs > 1 else None + ) analysis = ray.tune.run( - self.trainable, search_alg=search_alg, config=space, - metric='val_loss', mode='min', resources_per_trial=resources_per_trial, - time_budget_s=self._state.time_budget, num_samples=self._max_iter, - verbose=self.verbose) + self.trainable, + search_alg=search_alg, + config=space, + metric="val_loss", + mode="min", + resources_per_trial=resources_per_trial, + time_budget_s=self._state.time_budget, + num_samples=self._max_iter, + verbose=self.verbose, + ) # logger.info([trial.last_result for trial in analysis.trials]) - trials = sorted((trial for trial in analysis.trials if trial.last_result - and trial.last_result['wall_clock_time'] is not None), - key=lambda x: x.last_result['wall_clock_time']) + trials = sorted( + ( + trial + for trial in analysis.trials + if trial.last_result + and trial.last_result["wall_clock_time"] is not None + ), + key=lambda x: x.last_result["wall_clock_time"], + ) for _track_iter, trial in enumerate(trials): result = trial.last_result better = False if result: - config = result['config'] - estimator = config.get('ml', config)['learner'] + config = result["config"] + estimator = config.get("ml", config)["learner"] search_state = self._search_states[estimator] search_state.update(result, 0, self._save_model_history) - if result['wall_clock_time'] is not None: - self._state.time_from_start = result['wall_clock_time'] + if result["wall_clock_time"] is not None: + self._state.time_from_start = result["wall_clock_time"] if search_state.sample_size == self._state.data_size: self._iter_per_learner[estimator] += 1 if not self._fullsize_reached: @@ -1476,15 +1657,20 @@ class AutoML: self._state.best_loss = search_state.best_loss self._best_estimator = estimator self._config_history[_track_iter] = ( - self._best_estimator, config, self._time_taken_best_iter) + self._best_estimator, + config, + self._time_taken_best_iter, + ) if self._save_model_history: - self._model_history[_track_iter] = search_state.trained_estimator + self._model_history[ + _track_iter + ] = search_state.trained_estimator self._trained_estimator = search_state.trained_estimator self._best_iteration = _track_iter self._time_taken_best_iter = self._state.time_from_start better = True self._search_states[estimator].best_config = config - if (better or self._log_type == 'all') and self._training_log: + if (better or self._log_type == "all") and self._training_log: self._training_log.append( self._iter_per_learner[estimator], search_state.metric_for_logging, @@ -1495,32 +1681,36 @@ class AutoML: self._state.best_loss, search_state.best_config, estimator, - search_state.sample_size) + search_state.sample_size, + ) def _search_sequential(self): try: from ray import __version__ as ray_version - assert ray_version >= '1.0.0' + + assert ray_version >= "1.0.0" from ray.tune.suggest import ConcurrencyLimiter except (ImportError, AssertionError): from .searcher.suggestion import ConcurrencyLimiter - if self._hpo_method in ('cfo', 'grid'): + if self._hpo_method in ("cfo", "grid"): from flaml import CFO as SearchAlgo - elif 'optuna' == self._hpo_method: + elif "optuna" == self._hpo_method: try: from ray import __version__ as ray_version - assert ray_version >= '1.0.0' + + assert ray_version >= "1.0.0" from ray.tune.suggest.optuna import OptunaSearch as SearchAlgo except (ImportError, AssertionError): from .searcher.suggestion import OptunaSearch as SearchAlgo - elif 'bs' == self._hpo_method: + elif "bs" == self._hpo_method: from flaml import BlendSearch as SearchAlgo - elif 'cfocat' == self._hpo_method: + elif "cfocat" == self._hpo_method: from flaml.searcher.cfo_cat import CFOCat as SearchAlgo else: raise NotImplementedError( f"hpo_method={self._hpo_method} is not recognized. " - "'cfo' and 'bs' are supported.") + "'cfo' and 'bs' are supported." + ) est_retrain_time = next_trial_time = 0 best_config_sig = None @@ -1534,46 +1724,55 @@ class AutoML: estimator = self._select_estimator(self._active_estimators) if not estimator: break - logger.info( - f"iteration {self._track_iter}, current learner {estimator}") + logger.info(f"iteration {self._track_iter}, current learner {estimator}") search_state = self._search_states[estimator] self._state.time_from_start = time.time() - self._start_time_flag time_left = self._state.time_budget - self._state.time_from_start - budget_left = time_left if not self._retrain_in_budget or better or ( - not self.best_estimator) or self._search_states[ - self.best_estimator].sample_size < self._state.data_size \ + budget_left = ( + time_left + if not self._retrain_in_budget + or better + or (not self.best_estimator) + or self._search_states[self.best_estimator].sample_size + < self._state.data_size else time_left - est_retrain_time + ) if not search_state.search_alg: search_state.training_function = partial( - AutoMLState._compute_with_config_base, - self._state, estimator) + AutoMLState._compute_with_config_base, self._state, estimator + ) search_space = search_state.search_space if self._sample: - prune_attr = 'FLAML_sample_size' + prune_attr = "FLAML_sample_size" min_resource = MIN_SAMPLE_TRAIN max_resource = self._state.data_size else: prune_attr = min_resource = max_resource = None learner_class = self._state.learner_classes.get(estimator) - if 'grid' == self._hpo_method: # for synthetic exp only + if "grid" == self._hpo_method: # for synthetic exp only points_to_evaluate = [] space = search_space keys = list(space.keys()) domain0, domain1 = space[keys[0]], space[keys[1]] for x1 in range(domain0.lower, domain0.upper + 1): for x2 in range(domain1.lower, domain1.upper + 1): - points_to_evaluate.append({ - keys[0]: x1, - keys[1]: x2, - }) + points_to_evaluate.append( + { + keys[0]: x1, + keys[1]: x2, + } + ) self._max_iter_per_learner = len(points_to_evaluate) low_cost_partial_config = None else: - points_to_evaluate = [search_state.init_config] + points_to_evaluate = search_state.init_config if isinstance( + search_state.init_config, list) else [search_state.init_config] low_cost_partial_config = search_state.low_cost_partial_config - if self._hpo_method in ('bs', 'cfo', 'grid', 'cfocat'): + if self._hpo_method in ("bs", "cfo", "grid", "cfocat"): algo = SearchAlgo( - metric='val_loss', mode='min', space=search_space, + metric="val_loss", + mode="min", + space=search_space, points_to_evaluate=points_to_evaluate, low_cost_partial_config=low_cost_partial_config, cat_hp_cost=search_state.cat_hp_cost, @@ -1581,27 +1780,29 @@ class AutoML: min_resource=min_resource, max_resource=max_resource, config_constraints=[ - (learner_class.size, '<=', self._mem_thres) + (learner_class.size, "<=", self._mem_thres) ], metric_constraints=self.metric_constraints, ) else: algo = SearchAlgo( - metric='val_loss', mode='min', space=search_space, + metric="val_loss", + mode="min", + space=search_space, points_to_evaluate=points_to_evaluate - if len(search_state.init_config) == len( - search_space) else None, + if len(search_state.init_config) == len(search_space) + else None, ) - search_state.search_alg = ConcurrencyLimiter(algo, - max_concurrent=1) + search_state.search_alg = ConcurrencyLimiter(algo, max_concurrent=1) # search_state.search_alg = algo else: search_space = None - if self._hpo_method in ('bs', 'cfo', 'cfocat'): + if self._hpo_method in ("bs", "cfo", "cfocat"): search_state.search_alg.set_search_properties( - metric=None, mode=None, + metric=None, + mode=None, config={ - 'metric_target': self._state.best_loss, + "metric_target": self._state.best_loss, }, ) start_run_time = time.time() @@ -1610,44 +1811,52 @@ class AutoML: search_alg=search_state.search_alg, time_budget_s=min(budget_left, self._state.train_time_limit), verbose=max(self.verbose - 1, 0), - use_ray=False) + use_ray=False, + ) time_used = time.time() - start_run_time better = False if analysis.trials: result = analysis.trials[-1].last_result - search_state.update(result, - time_used=time_used, - save_model_history=self._save_model_history) + search_state.update( + result, + time_used=time_used, + save_model_history=self._save_model_history, + ) if self._estimator_index is None: + # update init eci estimate eci_base = search_state.init_eci self._eci.append(search_state.estimated_cost4improvement) for e in self.estimator_list[1:]: - self._eci.append(self._search_states[e].init_eci - / eci_base * self._eci[0]) + self._eci.append( + self._search_states[e].init_eci / eci_base * self._eci[0] + ) self._estimator_index = 0 - if result['wall_clock_time'] is not None: - self._state.time_from_start = result['wall_clock_time'] + if result["wall_clock_time"] is not None: + self._state.time_from_start = result["wall_clock_time"] # logger.info(f"{self._search_states[estimator].sample_size}, {data_size}") if search_state.sample_size == self._state.data_size: self._iter_per_learner[estimator] += 1 - if not self._fullsize_reached: - self._fullsize_reached = True + self._fullsize_reached = True if search_state.best_loss < self._state.best_loss: best_config_sig = estimator + search_state.get_hist_config_sig( - self.data_size_full, - search_state.best_config) + self.data_size_full, search_state.best_config + ) self._state.best_loss = search_state.best_loss self._best_estimator = estimator - est_retrain_time = search_state.est_retrain_time( - self.data_size_full) if ( - best_config_sig not in self._retrained_config) else 0 + est_retrain_time = ( + search_state.est_retrain_time(self.data_size_full) + if (best_config_sig not in self._retrained_config) + else 0 + ) self._config_history[self._track_iter] = ( estimator, search_state.best_config, - self._state.time_from_start) + self._state.time_from_start, + ) if self._save_model_history: self._model_history[ - self._track_iter] = search_state.trained_estimator + self._track_iter + ] = search_state.trained_estimator elif self._trained_estimator: del self._trained_estimator self._trained_estimator = None @@ -1656,7 +1865,7 @@ class AutoML: self._time_taken_best_iter = self._state.time_from_start better = True next_trial_time = search_state.time2eval_best - if better or self._log_type == 'all': + if better or self._log_type == "all": if self._training_log: self._training_log.append( self._iter_per_learner[estimator], @@ -1668,84 +1877,107 @@ class AutoML: search_state.best_loss, search_state.best_config, estimator, - search_state.sample_size) + search_state.sample_size, + ) if mlflow is not None and mlflow.active_run(): with mlflow.start_run(nested=True): - mlflow.log_metric('iter_counter', - self._iter_per_learner[estimator]) - mlflow.log_param('metric_for_logging', - search_state.metric_for_logging) - mlflow.log_metric('trial_time', - search_state.trial_time) - mlflow.log_metric('wall_clock_time', - self._state.time_from_start) - mlflow.log_metric('validation_loss', - search_state.val_loss) - mlflow.log_param('config', - search_state.config) - mlflow.log_param('learner', - estimator) - mlflow.log_param('sample_size', - search_state.sample_size) - mlflow.log_metric('best_validation_loss', - search_state.best_loss) - mlflow.log_param('best_config', - search_state.best_config) - mlflow.log_param('best_learner', - self._best_estimator) + mlflow.log_metric( + "iter_counter", self._iter_per_learner[estimator] + ) + mlflow.log_param( + "metric_for_logging", search_state.metric_for_logging + ) + mlflow.log_metric("trial_time", search_state.trial_time) + mlflow.log_metric( + "wall_clock_time", self._state.time_from_start + ) + mlflow.log_metric("validation_loss", search_state.val_loss) + mlflow.log_param("config", search_state.config) + mlflow.log_param("learner", estimator) + mlflow.log_param("sample_size", search_state.sample_size) + mlflow.log_metric( + "best_validation_loss", search_state.best_loss + ) + mlflow.log_param("best_config", search_state.best_config) + mlflow.log_param("best_learner", self._best_estimator) logger.info( " at {:.1f}s,\tbest {}'s error={:.4f},\tbest {}'s error={:.4f}".format( self._state.time_from_start, estimator, search_state.best_loss, self._best_estimator, - self._state.best_loss)) - if self._hpo_method in ('cfo', 'bs') and all( - state.search_alg and state.search_alg.searcher.is_ls_ever_converged - for state in self._search_states.values()) and ( + self._state.best_loss, + ) + ) + if ( + self._hpo_method in ("cfo", "bs") + and all( + state.search_alg + and state.search_alg.searcher.is_ls_ever_converged + for state in self._search_states.values() + ) + and ( self._state.time_from_start - > self._warn_threshold * self._time_taken_best_iter): + > self._warn_threshold * self._time_taken_best_iter + ) + ): logger.warning( "All estimator hyperparameters local search has " "converged at least once, and the total search time " f"exceeds {self._warn_threshold} times the time taken " - "to find the best model.") + "to find the best model." + ) self._warn_threshold *= 10 + if self._early_stop: + logger.warning("Stopping search as early_stop is set to True.") + break else: - logger.info(f"no enough budget for learner {estimator}") + logger.info(f"stop trying learner {estimator}") if self._estimator_index is not None: self._active_estimators.remove(estimator) self._estimator_index -= 1 - if self._retrain_in_budget and best_config_sig and est_retrain_time \ - and not better and self._search_states[ - self._best_estimator].sample_size == self._state.data_size and ( - est_retrain_time - <= self._state.time_budget - self._state.time_from_start - <= est_retrain_time + next_trial_time): - self._trained_estimator, \ - retrain_time = self._state._train_with_config( - self._best_estimator, - self._search_states[self._best_estimator].best_config, - self.data_size_full) - logger.info("retrain {} for {:.1f}s".format( - self._best_estimator, retrain_time)) + self._state.time_from_start = time.time() - self._start_time_flag + if self._state.time_budget > self._state.time_from_start: + search_state.search_alg.searcher._is_ls_ever_converged = True + if ( + self._retrain_in_budget + and best_config_sig + and est_retrain_time + and not better + and self._search_states[self._best_estimator].sample_size + == self._state.data_size + and ( + est_retrain_time + <= self._state.time_budget - self._state.time_from_start + <= est_retrain_time + next_trial_time + ) + ): + self._trained_estimator, retrain_time = self._state._train_with_config( + self._best_estimator, + self._search_states[self._best_estimator].best_config, + self.data_size_full, + ) + logger.info( + "retrain {} for {:.1f}s".format(self._best_estimator, retrain_time) + ) self._retrained_config[best_config_sig] = retrain_time est_retrain_time = 0 self._state.time_from_start = time.time() - self._start_time_flag - if (self._state.time_from_start >= self._state.time_budget - or not self._active_estimators): + if ( + self._state.time_from_start >= self._state.time_budget + or not self._active_estimators + ): break if self._ensemble and self._best_estimator: time_left = self._state.time_budget - self._state.time_from_start - time_ensemble = self._search_states[ - self._best_estimator].time2eval_best + time_ensemble = self._search_states[self._best_estimator].time2eval_best if time_left < time_ensemble < 2 * time_left: break def _search(self): # initialize the search_states self._eci = [] - self._state.best_loss = float('+inf') + self._state.best_loss = float("+inf") self._state.time_from_start = 0 self._estimator_index = None self._best_iteration = 0 @@ -1772,78 +2004,93 @@ class AutoML: if self._best_estimator: self._selected = self._search_states[self._best_estimator] self.modelcount = sum( - search_state.total_iter - for search_state in self._search_states.values()) + search_state.total_iter for search_state in self._search_states.values() + ) if self._trained_estimator: - logger.info(f'selected model: {self._trained_estimator.model}') + logger.info(f"selected model: {self._trained_estimator.model}") if self._ensemble and self._state.task in ( - 'binary', 'multi', 'regression', + "binary", + "multi", + "regression", ): - search_states = list(x for x in self._search_states.items() - if x[1].trained_estimator) + search_states = list( + x for x in self._search_states.items() if x[1].trained_estimator + ) search_states.sort(key=lambda x: x[1].best_loss) - estimators = [(x[0], x[1].trained_estimator) - for x in search_states[:2]] + estimators = [(x[0], x[1].trained_estimator) for x in search_states[:2]] estimators += [ - (x[0], x[1].trained_estimator) for x in search_states[2:] - if x[1].best_loss < 4 * self._selected.best_loss] + (x[0], x[1].trained_estimator) + for x in search_states[2:] + if x[1].best_loss < 4 * self._selected.best_loss + ] logger.info(estimators) if len(estimators) <= 1: return - if self._state.task in ('binary', 'multi'): + if self._state.task in ("binary", "multi"): from sklearn.ensemble import StackingClassifier as Stacker else: from sklearn.ensemble import StackingRegressor as Stacker if isinstance(self._ensemble, dict): final_estimator = self._ensemble.get( - 'final_estimator', self._trained_estimator) - passthrough = self._ensemble.get('passthrough', True) + "final_estimator", self._trained_estimator + ) + passthrough = self._ensemble.get("passthrough", True) else: final_estimator = self._trained_estimator passthrough = True stacker = Stacker( - estimators, final_estimator, n_jobs=self._state.n_jobs, - passthrough=passthrough) + estimators, + final_estimator, + n_jobs=self._state.n_jobs, + passthrough=passthrough, + ) if self._sample_weight_full is not None: - self._state.fit_kwargs[ - 'sample_weight'] = self._sample_weight_full - stacker.fit(self._X_train_all, self._y_train_all, - **self._state.fit_kwargs) - logger.info(f'ensemble: {stacker}') + self._state.fit_kwargs["sample_weight"] = self._sample_weight_full + stacker.fit( + self._X_train_all, self._y_train_all, **self._state.fit_kwargs + ) + logger.info(f"ensemble: {stacker}") self._trained_estimator = stacker self._trained_estimator.model = stacker elif self._retrain_final: # reset time budget for retraining self._state.time_from_start -= self._state.time_budget - if self._state.task == 'forecast' or ( + if self._state.task == "forecast" or ( self._state.time_budget - self._state.time_from_start > self._selected.est_retrain_time(self.data_size_full) and self._selected.best_config_sample_size == self._state.data_size ): - self._trained_estimator, \ - retrain_time = self._state._train_with_config( - self._best_estimator, - self._search_states[self._best_estimator].best_config, - self.data_size_full) - logger.info("retrain {} for {:.1f}s".format( - self._best_estimator, retrain_time)) - if self._trained_estimator: - logger.info( - f'retrained model: {self._trained_estimator.model}') - else: + ( + self._trained_estimator, + retrain_time, + ) = self._state._train_with_config( + self._best_estimator, + self._search_states[self._best_estimator].best_config, + self.data_size_full, + ) logger.info( - "not retraining because the time budget is too small.") + "retrain {} for {:.1f}s".format( + self._best_estimator, retrain_time + ) + ) + if self._trained_estimator: + logger.info(f"retrained model: {self._trained_estimator.model}") + else: + logger.info("not retraining because the time budget is too small.") if self.model and mlflow is not None and mlflow.active_run(): - mlflow.sklearn.log_model(self.model, 'best_model') + mlflow.sklearn.log_model(self.model, "best_model") def __del__(self): - if hasattr(self, '_trained_estimator') and self._trained_estimator \ - and hasattr(self._trained_estimator, 'cleanup'): + if ( + hasattr(self, "_trained_estimator") + and self._trained_estimator + and hasattr(self._trained_estimator, "cleanup") + ): self._trained_estimator.cleanup() del self._trained_estimator def _select_estimator(self, estimator_list): - if self._learner_selector == 'roundrobin': + if self._learner_selector == "roundrobin": self._estimator_index += 1 if self._estimator_index == len(estimator_list): self._estimator_index = 0 @@ -1856,25 +2103,31 @@ class AutoML: self._search_states[estimator].sample_size ): # sample_size=None meaning no result search_state = self._search_states[estimator] - if (self._search_states[estimator].time2eval_best + if ( + self._search_states[estimator].time2eval_best > self._state.time_budget - self._state.time_from_start - or self._iter_per_learner[estimator] - >= self._max_iter_per_learner): + or self._iter_per_learner[estimator] >= self._max_iter_per_learner + ): inv.append(0) continue estimated_cost = search_state.estimated_cost4improvement if search_state.sample_size < self._state.data_size: estimated_cost = min( estimated_cost, - search_state.time2eval_best * min( + search_state.time2eval_best + * min( SAMPLE_MULTIPLY_FACTOR, - self._state.data_size / search_state.sample_size)) + self._state.data_size / search_state.sample_size, + ), + ) gap = search_state.best_loss - self._state.best_loss if gap > 0 and not self._ensemble: - delta_loss = (search_state.best_loss_old - - search_state.best_loss) or search_state.best_loss - delta_time = (search_state.total_time_used - - search_state.time_best_found_old) or 1e-10 + delta_loss = ( + search_state.best_loss_old - search_state.best_loss + ) or search_state.best_loss + delta_time = ( + search_state.total_time_used - search_state.time_best_found_old + ) or 1e-10 speed = delta_loss / delta_time if speed: estimated_cost = max(2 * gap / speed, estimated_cost) diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py index d1654b255..d9d02a898 100644 --- a/flaml/searcher/blendsearch.py +++ b/flaml/searcher/blendsearch.py @@ -761,16 +761,14 @@ class BlendSearchTuner(BlendSearch, NNITuner): parameters: object created by 'generate_parameters()' value: final metrics of the trial, including default metric ''' - result = {} - for k, v in parameters.items(): - result['config/' + k] = v - reward = extract_scalar_reward(value) - result[self._metric] = reward - # if nni does not report training cost, - # using sequence as an approximation. - # if no sequence, using a constant 1 - result[self.cost_attr] = value.get(self.cost_attr, value.get( - 'sequence', 1)) + result = { + 'config': parameters, self._metric: extract_scalar_reward(value), + self.cost_attr: 1 if isinstance(value, float) else value.get( + self.cost_attr, value.get('sequence', 1)) + # if nni does not report training cost, + # using sequence as an approximation. + # if no sequence, using a constant 1 + } self.on_trial_complete(str(parameter_id), result) ... diff --git a/flaml/training_log.py b/flaml/training_log.py index b58850c18..b4061a9d8 100644 --- a/flaml/training_log.py +++ b/flaml/training_log.py @@ -67,6 +67,9 @@ class TrainingLogWriter(object): def open(self): self.file = open(self.output_filename, 'w') + def append_open(self): + self.file = open(self.output_filename, 'a') + def append(self, it_counter: int, train_loss: float, @@ -157,10 +160,13 @@ class TrainingLogReader(object): @contextmanager -def training_log_writer(filename: str): +def training_log_writer(filename: str, append: bool = False): try: w = TrainingLogWriter(filename) - w.open() + if not append: + w.open() + else: + w.append_open() yield w finally: w.close() diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py index b5bf8bfe3..67a32bb08 100644 --- a/flaml/tune/tune.py +++ b/flaml/tune/tune.py @@ -1,20 +1,23 @@ -'''! - * Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved. +"""! + * Copyright (c) Microsoft Corporation. All rights reserved. * Licensed under the MIT License. See LICENSE file in the * project root for license information. -''' +""" from typing import Optional, Union, List, Callable, Tuple import numpy as np import datetime import time + try: from ray import __version__ as ray_version - assert ray_version >= '1.0.0' + + assert ray_version >= "1.0.0" from ray.tune.analysis import ExperimentAnalysis as EA except (ImportError, AssertionError): from .analysis import ExperimentAnalysis as EA from .result import DEFAULT_METRIC import logging + logger = logging.getLogger(__name__) @@ -26,8 +29,7 @@ _training_iteration = 0 class ExperimentAnalysis(EA): - '''Class for storing the experiment results - ''' + """Class for storing the experiment results""" def __init__(self, trials, metric, mode): try: @@ -39,7 +41,7 @@ class ExperimentAnalysis(EA): def report(_metric=None, **kwargs): - '''A function called by the HPO application to report final or intermediate + """A function called by the HPO application to report final or intermediate results. Example: @@ -70,13 +72,14 @@ def report(_metric=None, **kwargs): _metric: Optional default anonymous metric for ``tune.report(value)``. (For compatibility with ray.tune.report) **kwargs: Any key value pair to be reported. - ''' + """ global _use_ray global _verbose global _running_trial global _training_iteration if _use_ray: from ray import tune + return tune.report(_metric, **kwargs) else: result = kwargs @@ -91,11 +94,11 @@ def report(_metric=None, **kwargs): _training_iteration = 0 _running_trial = trial result["training_iteration"] = _training_iteration - result['config'] = trial.config + result["config"] = trial.config for key, value in trial.config.items(): - result['config/' + key] = value + result["config/" + key] = value _runner.process_trial_result(_runner.running_trial, result) - result['time_total_s'] = trial.last_update_time - trial.start_time + result["time_total_s"] = trial.last_update_time - trial.start_time if _verbose > 2: logger.info(f"result: {result}") if _runner.running_trial.is_finished(): @@ -104,31 +107,34 @@ def report(_metric=None, **kwargs): return True -def run(training_function, - config: Optional[dict] = None, - low_cost_partial_config: Optional[dict] = None, - cat_hp_cost: Optional[dict] = None, - metric: Optional[str] = None, - mode: Optional[str] = None, - time_budget_s: Union[int, float, datetime.timedelta] = None, - points_to_evaluate: Optional[List[dict]] = None, - evaluated_rewards: Optional[List] = None, - prune_attr: Optional[str] = None, - min_resource: Optional[float] = None, - max_resource: Optional[float] = None, - reduction_factor: Optional[float] = None, - report_intermediate_result: Optional[bool] = False, - search_alg=None, - verbose: Optional[int] = 2, - local_dir: Optional[str] = None, - num_samples: Optional[int] = 1, - resources_per_trial: Optional[dict] = None, - config_constraints: Optional[ - List[Tuple[Callable[[dict], float], str, float]]] = None, - metric_constraints: Optional[ - List[Tuple[str, str, float]]] = None, - use_ray: Optional[bool] = False): - '''The trigger for HPO. +def run( + training_function, + config: Optional[dict] = None, + low_cost_partial_config: Optional[dict] = None, + cat_hp_cost: Optional[dict] = None, + metric: Optional[str] = None, + mode: Optional[str] = None, + time_budget_s: Union[int, float, datetime.timedelta] = None, + points_to_evaluate: Optional[List[dict]] = None, + evaluated_rewards: Optional[List] = None, + prune_attr: Optional[str] = None, + min_resource: Optional[float] = None, + max_resource: Optional[float] = None, + reduction_factor: Optional[float] = None, + report_intermediate_result: Optional[bool] = False, + search_alg=None, + verbose: Optional[int] = 2, + local_dir: Optional[str] = None, + num_samples: Optional[int] = 1, + resources_per_trial: Optional[dict] = None, + config_constraints: Optional[ + List[Tuple[Callable[[dict], float], str, float]] + ] = None, + metric_constraints: Optional[List[Tuple[str, str, float]]] = None, + max_failure: Optional[int] = 100, + use_ray: Optional[bool] = False, +): + """The trigger for HPO. Example: @@ -236,25 +242,35 @@ def run(training_function, needed for a config. It is used to skip configs which do not fit in memory. metric_constraints: A list of metric constraints to be satisfied. - e.g., `['precision', '>=', 0.9]` - use_ray: A boolean of whether to use ray as the backend - ''' + e.g., `['precision', '>=', 0.9]`. + max_failure: int | the maximal consecutive number of failures to sample + a trial before the tuning is terminated. + use_ray: A boolean of whether to use ray as the backend. + """ global _use_ray global _verbose if not use_ray: _verbose = verbose if verbose > 0: import os + if local_dir: os.makedirs(local_dir, exist_ok=True) - logger.addHandler(logging.FileHandler(local_dir + '/tune_' + str( - datetime.datetime.now()).replace(':', '-') + '.log')) + logger.addHandler( + logging.FileHandler( + local_dir + + "/tune_" + + str(datetime.datetime.now()).replace(":", "-") + + ".log" + ) + ) elif not logger.handlers: # Add the console handler. _ch = logging.StreamHandler() logger_formatter = logging.Formatter( - '[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s', - '%m-%d %H:%M:%S') + "[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s", + "%m-%d %H:%M:%S", + ) _ch.setFormatter(logger_formatter) logger.addHandler(_ch) if verbose <= 2: @@ -266,42 +282,49 @@ def run(training_function, if search_alg is None: from ..searcher.blendsearch import BlendSearch + search_alg = BlendSearch( - metric=metric or DEFAULT_METRIC, mode=mode, + metric=metric or DEFAULT_METRIC, + mode=mode, space=config, points_to_evaluate=points_to_evaluate, evaluated_rewards=evaluated_rewards, low_cost_partial_config=low_cost_partial_config, cat_hp_cost=cat_hp_cost, prune_attr=prune_attr, - min_resource=min_resource, max_resource=max_resource, + min_resource=min_resource, + max_resource=max_resource, reduction_factor=reduction_factor, config_constraints=config_constraints, - metric_constraints=metric_constraints) + metric_constraints=metric_constraints, + ) else: search_alg.set_search_properties(metric, mode, config) if metric is None or mode is None: metric = metric or search_alg.metric mode = mode or search_alg.mode if time_budget_s: - search_alg.set_search_properties(None, None, config={ - 'time_budget_s': time_budget_s}) + search_alg.set_search_properties( + None, None, config={"time_budget_s": time_budget_s} + ) scheduler = None if report_intermediate_result: params = {} # scheduler resource_dimension=prune_attr if prune_attr: - params['time_attr'] = prune_attr + params["time_attr"] = prune_attr if max_resource: - params['max_t'] = max_resource + params["max_t"] = max_resource if min_resource: - params['grace_period'] = min_resource + params["grace_period"] = min_resource if reduction_factor: - params['reduction_factor'] = reduction_factor + params["reduction_factor"] = reduction_factor try: from ray import __version__ as ray_version - assert ray_version >= '1.0.0' + + assert ray_version >= "1.0.0" from ray.tune.schedulers import ASHAScheduler + scheduler = ASHAScheduler(**params) except (ImportError, AssertionError): pass @@ -309,17 +332,23 @@ def run(training_function, try: from ray import tune except ImportError: - raise ImportError("Failed to import ray tune. " - "Please install ray[tune] or set use_ray=False") + raise ImportError( + "Failed to import ray tune. " + "Please install ray[tune] or set use_ray=False" + ) _use_ray = True - return tune.run(training_function, - metric=metric, mode=mode, - search_alg=search_alg, - scheduler=scheduler, - time_budget_s=time_budget_s, - verbose=verbose, local_dir=local_dir, - num_samples=num_samples, - resources_per_trial=resources_per_trial) + return tune.run( + training_function, + metric=metric, + mode=mode, + search_alg=search_alg, + scheduler=scheduler, + time_budget_s=time_budget_s, + verbose=verbose, + local_dir=local_dir, + num_samples=num_samples, + resources_per_trial=resources_per_trial, + ) # simple sequential run without using tune.run() from ray time_start = time.time() @@ -327,6 +356,7 @@ def run(training_function, if scheduler: scheduler.set_search_properties(metric=metric, mode=mode) from .trial_runner import SequentialTrialRunner + global _runner _runner = SequentialTrialRunner( search_alg=search_alg, @@ -337,13 +367,18 @@ def run(training_function, num_trials = 0 if time_budget_s is None: time_budget_s = np.inf - while time.time() - time_start < time_budget_s and ( - num_samples < 0 or num_trials < num_samples): + fail = 0 + ub = (len(evaluated_rewards) if evaluated_rewards else 0) + max_failure + while ( + time.time() - time_start < time_budget_s + and (num_samples < 0 or num_trials < num_samples) + and fail < ub + ): trial_to_run = _runner.step() if trial_to_run: num_trials += 1 if verbose: - logger.info(f'trial {num_trials} config: {trial_to_run.config}') + logger.info(f"trial {num_trials} config: {trial_to_run.config}") result = training_function(trial_to_run.config) if result is not None: if isinstance(result, dict): @@ -351,6 +386,11 @@ def run(training_function, else: report(_metric=result) _runner.stop_trial(trial_to_run) + fail = 0 + else: + fail += 1 # break with ub consecutive failures + if fail == ub: + logger.warning("fail to sample a trial for 10 times in a row, stopping.") if verbose > 0: logger.handlers.clear() return ExperimentAnalysis(_runner.get_trials(), metric=metric, mode=mode) diff --git a/flaml/version.py b/flaml/version.py index 22049ab2c..63af88769 100644 --- a/flaml/version.py +++ b/flaml/version.py @@ -1 +1 @@ -__version__ = "0.6.2" +__version__ = "0.6.3" diff --git a/notebook/flaml_autovw.ipynb b/notebook/flaml_autovw.ipynb index f9a1bb91d..eec560d78 100644 --- a/notebook/flaml_autovw.ipynb +++ b/notebook/flaml_autovw.ipynb @@ -2,11 +2,6 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, "source": [ "Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved. \n", "\n", @@ -22,53 +17,40 @@ "\n", "*ChaCha for online AutoML. Qingyun Wu, Chi Wang, John Langford, Paul Mineiro and Marco Rossi. To appear in ICML 2021.*\n", "\n", - "AutoVW is implemented in FLAML. FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` option:\n", - "```bash\n", - "pip install flaml[notebook]\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install flaml[notebook];" - ] - }, - { - "cell_type": "markdown", + "AutoVW is implemented in FLAML. FLAML requires `Python>=3.6`. To run this notebook example, please install:" + ], "metadata": { "slideshow": { "slide_type": "slide" } - }, + } + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "!pip install flaml[notebook,vw];" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", "source": [ "## 2. Online regression with AutoVW\n", "### Load data from openml and preprocess\n", "\n", "Download [NewFuelCar](https://www.openml.org/d/41506) from OpenML." - ] + ], + "metadata": { + "slideshow": { + "slide_type": "slide" + } + } }, { "cell_type": "code", "execution_count": 1, - "metadata": { - "slideshow": { - "slide_type": "subslide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "(36203, 17) (36203,)\n" - ] - } - ], "source": [ "import openml\n", "# did = 42183\n", @@ -78,31 +60,34 @@ "data = ds.get_data(target=target_attribute, dataset_format='array')\n", "X, y = data[0], data[1]\n", "print(X.shape, y.shape)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Convert the openml dataset into vowpalwabbit examples:\n", - "Sequentially group features into up to 10 namespaces and convert the original data examples into vowpal wabbit format." - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "tags": [] - }, + ], "outputs": [ { "output_type": "stream", "name": "stdout", "text": [ - "openml example: 8.170000076293945 [1.0000e+01 7.0000e+00 3.0000e+00 4.0000e+00 nan 6.3300e+00\n 1.3600e-01 7.3300e+00 7.0100e+00 6.9800e+00 3.0000e-03 7.0000e+00\n 9.7000e+00 1.2300e+01 1.0217e+03 0.0000e+00 5.8000e+01]\nvw example: 8.170000076293945 |a 0:10.000000 1:7.000000|b 2:3.000000 3:4.000000|c 4:nan 5:6.330000|d 6:0.136000 7:7.330000|e 8:7.010000 9:6.980000|f 10:0.003000 11:7.000000|g 12:9.700000 13:12.300000|h 14:1021.700012 15:0.000000|i 16:58.000000\n" + "(36203, 17) (36203,)\n" ] } ], + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + } + }, + { + "cell_type": "markdown", + "source": [ + "Convert the openml dataset into vowpalwabbit examples:\n", + "Sequentially group features into up to 10 namespaces and convert the original data examples into vowpal wabbit format." + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 3, "source": [ "import numpy as np\n", "import string\n", @@ -128,24 +113,37 @@ " vw_examples.append(ns_line)\n", "print('openml example:', y[0], X[0])\n", "print('vw example:', vw_examples[0])" - ] + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "openml example: 8.170000076293945 [1.0000e+01 7.0000e+00 3.0000e+00 4.0000e+00 nan 6.3300e+00\n", + " 1.3600e-01 7.3300e+00 7.0100e+00 6.9800e+00 3.0000e-03 7.0000e+00\n", + " 9.7000e+00 1.2300e+01 1.0217e+03 0.0000e+00 5.8000e+01]\n", + "vw example: 8.170000076293945 |a 0:10.000000 1:7.000000|b 2:3.000000 3:4.000000|c 4:nan 5:6.330000|d 6:0.136000 7:7.330000|e 8:7.010000 9:6.980000|f 10:0.003000 11:7.000000|g 12:9.700000 13:12.300000|h 14:1021.700012 15:0.000000|i 16:58.000000\n" + ] + } + ], + "metadata": { + "tags": [] + } }, { "cell_type": "markdown", + "source": [ + "### Set up the online learning loop\n" + ], "metadata": { "slideshow": { "slide_type": "slide" } - }, - "source": [ - "### Set up the online learning loop\n" - ] + } }, { "cell_type": "code", "execution_count": 4, - "metadata": {}, - "outputs": [], "source": [ "from sklearn.metrics import mean_squared_error\n", "def online_learning_loop(iter_num, vw_examples, vw_alg):\n", @@ -166,22 +164,30 @@ " return loss_list\n", "\n", "max_iter_num = 10000 # or len(vw_examples)" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "### Vanilla Vowpal Wabbit (VW)\n", "Create and run a vanilla vowpal wabbit learner." - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 5, - "metadata": { - "tags": [] - }, + "source": [ + "from vowpalwabbit import pyvw\n", + "''' create a vanilla vw instance '''\n", + "vanilla_vw = pyvw.vw('--quiet')\n", + "\n", + "# online learning with vanilla VW\n", + "loss_list_vanilla = online_learning_loop(max_iter_num, vw_examples, vanilla_vw)\n", + "print('Final progressive validation loss of vanilla vw:', sum(loss_list_vanilla)/len(loss_list_vanilla))" + ], "outputs": [ { "output_type": "stream", @@ -192,33 +198,34 @@ ] } ], - "source": [ - "from vowpalwabbit import pyvw\n", - "''' create a vanilla vw instance '''\n", - "vanilla_vw = pyvw.vw('--quiet')\n", - "\n", - "# online learning with vanilla VW\n", - "loss_list_vanilla = online_learning_loop(max_iter_num, vw_examples, vanilla_vw)\n", - "print('Final progressive validation loss of vanilla vw:', sum(loss_list_vanilla)/len(loss_list_vanilla))" - ] + "metadata": { + "tags": [] + } }, { "cell_type": "markdown", - "metadata": {}, "source": [ "### AutoVW which tunes namespace interactions \n", "Create and run an AutoVW instance which tunes namespace interactions. Each AutoVW instance allows ```max_live_model_num``` of VW models (each associated with its own hyperaparameter configurations that are tuned online) to run concurrently in each step of the online learning loop." - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 6, - "metadata": { - "slideshow": { - "slide_type": "slide" - }, - "tags": [] - }, + "source": [ + "''' import AutoVW class from flaml package '''\n", + "from flaml import AutoVW\n", + "\n", + "'''create an AutoVW instance for tuning namespace interactions'''\n", + "# configure both hyperparamters to tune, e.g., 'interactions', and fixed arguments about the online learner,\n", + "# e.g., 'quiet' in the search_space argument.\n", + "autovw_ni = AutoVW(max_live_model_num=5, search_space={'interactions': AutoVW.AUTOMATIC, 'quiet': ''})\n", + "\n", + "# online learning with AutoVW\n", + "loss_list_autovw_ni = online_learning_loop(max_iter_num, vw_examples, autovw_ni)\n", + "print('Final progressive validation loss of autovw:', sum(loss_list_autovw_ni)/len(loss_list_autovw_ni))" + ], "outputs": [ { "output_type": "stream", @@ -235,44 +242,23 @@ ] } ], - "source": [ - "''' import AutoVW class from flaml package '''\n", - "from flaml import AutoVW\n", - "\n", - "'''create an AutoVW instance for tuning namespace interactions'''\n", - "# configure both hyperparamters to tune, e.g., 'interactions', and fixed arguments about the online learner,\n", - "# e.g., 'quiet' in the search_space argument.\n", - "autovw_ni = AutoVW(max_live_model_num=5, search_space={'interactions': AutoVW.AUTOMATIC, 'quiet': ''})\n", - "\n", - "# online learning with AutoVW\n", - "loss_list_autovw_ni = online_learning_loop(max_iter_num, vw_examples, autovw_ni)\n", - "print('Final progressive validation loss of autovw:', sum(loss_list_autovw_ni)/len(loss_list_autovw_ni))" - ] + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + } }, { "cell_type": "markdown", - "metadata": {}, "source": [ "### Online performance comparison between vanilla VW and AutoVW" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAFzCAYAAADIY/vqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeZxcVZ3//9ep6qV637N2VrKRlSyEVdkEghACwgABGQWFrwvCoN8ZcXRUGOcn+kUGcRlBGWHUSWBQgYAMi2yGBEgChKxkXztL7/tadX5/nNud7qzV6a6+Vd3v5+NRj6q6tX36UuRd59xzzzHWWkRERCSxBfwuQERERHpOgS4iItIPKNBFRET6AQW6iIhIP6BAFxER6QcU6CIiIv1Akt8F9ERhYaEdPXq032WIiIj0iVWrVpVZa4uO9lhCB/ro0aNZuXKl32WIiIj0CWPMzmM9pi53ERGRfkCBLiIi0g8o0EVERPqBhD6GLiIi/mhtbWXPnj00NTX5XUq/FAqFKC4uJjk5OerXKNBFRKTb9uzZQ1ZWFqNHj8YY43c5/Yq1lvLycvbs2cOYMWOifp263EVEpNuampooKChQmMeAMYaCgoJu934o0EVE5KQozGPnZPatAl1ERBLOBRdcwEsvvdRl20MPPcSXv/zlbr3Pc889x/333w/A97//fR544AEAPv/5z/P0008f83VPPPEECxcu7LKtrKyMoqIinn32Wa666qqO7T/84Q8ZN25cx/0lS5Zw5ZVXdqvOaCjQRUQk4SxcuJDFixd32bZ48eIjQvZErrzySu65555uf/7VV1/NK6+8QkNDQ8e2p59+mvnz53P22WfzzjvvdGxfvnw52dnZHDx4EIBly5Zx9tlnd/szT0SBLiIiCefaa6/lhRdeoKWlBYAdO3ZQUlLCokWLmDNnDlOmTOF73/tex/NHjx7N9773PWbNmsW0adPYuHEjAI8//jh33HHHcT/rvvvu4/TTT2fq1KncfvvtWGvJzs7mvPPOY8mSJR3Pa/9BUVRURHZ2Nlu2bAFg7969XHPNNSxbtgxwgX7OOef06v4AjXIXEZEeunfJOtaX1PTqe04els335k855uP5+fnMnTuXF198kQULFrB48WKuu+46/vmf/5n8/HzC4TAXXXQRH330EdOnTwegsLCQ999/n1/+8pc88MAD/OY3v4mqljvuuIPvfve7ANx88808//zzzJ8/n4ULF/KHP/yB66+/npKSEjZt2sSFF14IwDnnnMOyZcsIh8OMHz+eM888k5deeokrrriC1atXc/rpp/dwDx1JLXTP1jXvsO7tF/wuQ0REotS52729dfzUU08xa9YsZs6cybp161i/fn3H8z/zmc8AMHv2bHbs2BH157z++uucccYZTJs2jddee41169YBcPnll/P2229TU1PDU089xTXXXEMwGATg7LPPZtmyZSxbtoyzzjqLuXPn8u677/LBBx8wadIkQqFQL+2FQ9RC91S+8gDDaj+Ccy73uxQRkYRyvJZ0LC1YsIC7776b999/n4aGBvLz83nggQdYsWIFeXl5fP7zn+9y6ldqaioAwWCQtra2qD6jqamJr3zlK6xcuZIRI0bw/e9/v+M909LSmDdvHn/+859ZvHgxDz74YMfrzjnnHH72s58RDoe57bbbyMrKoqmpiTfeeCMmx89BLfQO1gQwNuJ3GSIiEqXMzEwuuOACbr31VhYuXEhNTQ0ZGRnk5ORw4MABXnzxxR5/Rnt4FxYWUldXd8TI94ULF/Lggw9y4MABzjrrrI7tp556KiUlJSxdupSZM2cCcNppp/GrX/0qJsfPQYHewZogBut3GSIi0g0LFy5k9erVLFy4kBkzZjBz5kwmTZrEjTfe2CvBmZuby2233cbUqVO59NJLjzj2ffHFF1NSUsL111/f5dxxYwxnnHEGBQUFHdO3nnXWWWzbti1mLXRjbeKG2Jw5c2xvrYf+3k9vZHTlcgZ9f3uvvJ+ISH+2YcMGTj31VL/L6NeOto+NMaustXOO9ny10NuZAAHU5S4iIolJge5Rl7uIiCQyBXo7EyCgQBcRkQSlQO9gCBD2uwgREZGTokD32IC63EVEJHEp0NuZAIEEHvEvIiIDmwK9g9EodxGRBPPMM89gjOlYbOV4HnrooS6rox3NLbfcwiOPPHLEZ1x22WXcfffdPPTQQx3bL730Ur74xS923P/GN77RZbY4cEuypqend6y0Bm5CnKPd7ikFuscGggp0EZEEs2jRIs4991wWLVp0wudGE+jHW5a1fcEVgEgkQllZWce87nDsZVELCwv5yU9+Es2f0yMK9HYa5S4iklDq6upYunQpjz32WEcIv/HGG1xxxRUdz7njjjt4/PHHefjhhykpKeGCCy7gggsuANyPgWnTpjF16lS++c1vAnDRRRexceNG9u3bB0B9fT2vvvoqV111FWeffTbLly8HYN26dUydOpWsrCwqKytpbm5mw4YNzJo164g6b731Vp588kkqKipiuj+0OEs7TSwjInJyXrwH9q/p3fccMg0uu/+4T3n22WeZN28eEyZMoKCggFWrVh3zuXfeeScPPvggr7/+OoWFhZSUlPDNb36TVatWkZeXxyWXXMIzzzzDVVddxTXXXMNTTz3FXXfdxZIlSzj//PPJzs4mOzubpKQkdu3a1bGK2t69e1m+fDk5OTlMmzaNlJQUvvvd7zJnzhyuvPJKwHWr33rrrfz0pz/l3nvv7dXd1Jla6B5jgmqhi4gkkEWLFnHDDTcAcMMNN0TV7d5uxYoVnH/++RQVFZGUlMRNN93EW2+9BRx9WdZ2hy+LetZZZ3Xcb587/r777usI83Z33nknTzzxBLW1tT36m49HLXSPNQECxmIjEUxAv3NERKJ2gpZ0LFRUVPDaa6+xZs0ajDGEw2GMMSxYsIBI5FBva+flU6N19tlns2/fPlavXs2yZcu6HFNvP46+Zs0apk6dyogRI/jJT35CdnY2t9xyyzHfMzc3lxtvvJFf/OIX3a4nWkqudsbtis5fBBERiU9PP/00N998Mzt37mTHjh3s3r2bMWPGEIlEWL9+Pc3NzVRVVfHXv/614zVZWVkdLeS5c+fy5ptvUlZWRjgcZtGiRZx33nmAWynt+uuv53Of+xyXXXYZoVCo4z3OPvtsnn/+efLz8wkGg+Tn51NVVcXy5ctPuIra17/+dR555JGo12LvLgV6u0B7oGu2OBGReLdo0SKuvvrqLtuuueYaFi9ezHXXXcfUqVO57rrrOtYiB7j99tuZN28eF1xwAUOHDuX+++/nggsuYMaMGcyePZsFCxZ0PLfzsqydTZs2jbKyMs4888wu23JycigsLATgu9/9Ls8999wRNRcWFnL11VfT3NzcK/vgcFo+1bP8iX/mrO2/oPmefaSG0nvlPUVE+istnxp7Wj71ZJkgAJGwWugiIpJ4FOgeoy53ERFJYAr0dhoUJyIiCUyB3q490NXlLiISlUQegxXvTmbfKtDbecfQrbrcRUROKBQKUV5erlCPAWst5eXlXU6Xi4YmlvHoGLqISPSKi4vZs2cPpaWlfpfSL4VCIYqLi7v1GgV6O6NAFxGJVnJyMmPGjPG7DOlEXe7tOrrcNShOREQSjwLdoy53ERFJZAp0j+kY5R6bOXZFRERiSYHusV6g24hGbIqISOJRoHtMwDuGbtXlLiIiiUeB7jGaKU5ERBKYAr1dQDPFiYhI4lKge9q73NEodxERSUAKdE9Hl7tVl7uIiCQeBbrHqMtdREQSmAK9nRZnERGRBKZA9xw6bU1d7iIikngU6B4TMIBa6CIikpgU6B5j3MJzmstdREQSkQLd0z4oDk0sIyIiCUiB3s4bFKcWuoiIJCIFuscE2xdnUQtdREQSjwLdY4wWZxERkcSlQPe0H0O3YbXQRUQk8ST5XUA7Y8xVwOVANvCYtfblvvz8QPtc7mqhi4hIAoppC90Y85/GmIPGmLWHbZ9njPnYGLPFGHMPgLX2GWvtbcCXgOtjWddRBXQMXUREElesu9wfB+Z13mDcwepfAJcBk4GFxpjJnZ7yHe/xPhXomCmura8/WkREpMdiGujW2reAisM2zwW2WGu3WWtbgMXAAuP8CHjRWvv+sd7TGHO7MWalMWZlaWlpr9XaMfVrxPbae4qIiPQVPwbFDQd2d7q/x9v2NeBTwLXGmC8d68XW2kettXOstXOKiop6raj25VM19auIiCSiuBkUZ619GHjYr883GhQnIiIJzI8W+l5gRKf7xd42XwWC7V3uGhQnIiKJx49AXwGMN8aMMcakADcAz/lQRxeB9i53LZ8qIiIJKNanrS0ClgMTjTF7jDFfsG4Y+R3AS8AG4Clr7bpY1hGVjkFx6nIXEZHEE9Nj6NbahcfY/hfgL7H87O5q73LXamsiIpKINPWrJxBQl7uIiCQuBbqnY3EWdbmLiEgCUqB7TMC4a7XQRUQkASnQPYGgG06gLncREUlECRnoxpj5xphHq6ure+09O+Zy16A4ERFJQAkZ6NbaJdba23NycnrtPdvXQ9dMcSIikogSMtBj4dB66Gqhi4hI4lGge9TlLiIiiUyB7tHiLCIiksgU6J72meLO3PSAz5WIiIh0nwLd0z5TnIiISCJSink6BsWJiIgkIAW6p2NxFhERkQSkQPeohS4iIolMge5RoIuISCJToHs0KE5ERBJZQqZYLOZyNwp0ERFJYAmZYrGYy73L+2u2OBERSTAJGeix1tbW6ncJIiIi3aJAP4q21ha/SxAREekWBfpRtLQ0+12CiIhItyjQj6KtpcnvEkRERLpFgX4UYR1DFxGRBKNAP4o2dbmLiEiCUaAfRVurutxFRCSxKNCPQl3uIiKSaBToRxFuVZe7iIgkFgV6Jytn/xiANgW6iIgkGAV6J6HcIQBE1OUuIiIJRoHeSSA5FYCIWugiIpJgThjoxpgfG2OyjTHJxpi/GmNKjTGf7YvijlNTr6+2BhBMSgEg3KapX0VEJLFE00K/xFpbA1wB7ADGAf8Yy6JOJFarrQWTXaBbdbmLiEiCiSbQk7zry4H/sdb2brM4jgTbu9zb1OUuIiKJJenET+F5Y8xGoBH4sjGmCOiXM68keS30iLrcRUQkwZywhW6tvQc4G5hjrW0F6oEFsS7MD8HkEAA2rC53ERFJLNEMivs7oNVaGzbGfAf4PTAs5pX5IKnjGLpa6CIikliiOYb+L9baWmPMucCngMeA/4htWf5oD/TJa37scyUiIiLdE02gh73ry4FHrbUvACmxK8k/SSmuyz3TNPpciYiISPdEE+h7jTGPANcDfzHGpEb5uoSTnNwvf6eIiMgAEE0wXwe8BFxqra0C8vH5PPRYSU5J9bsEERGRkxLNKPcGYCtwqTHmDmCQtfblmFfmg2AwmrP4RERE4k80o9zvAv4ADPIuvzfGfC3WhfnBBPrlkQQRERkAommSfgE4w1pbD2CM+RGwHPhZLAvzy+bgOMaHt/hdhoiISLdE0yQ1HBrpjnfbxKYc/5UNOpNmm+x3GSIiIt0STQv9t8C7xpg/e/evwp2L3j8lpZFqWrGRiLrgRUQkYZww0K21Dxpj3gDO9TbdYq39IKZVnYAxZj4wf9y4cb3/5klupHtzcyOhtIzef38REZEYOGYT1BiT337BLZv6e++y09vmm1gtnwpgvPncmxsbev29RUREYuV4LfRVgOXQ8XLrXRvv9tgY1uUbk5wGQGuTAl1ERBLHMQPdWjumLwuJFwGvhd7SrOlfRUQkcWjU12HaA721WS10ERFJHAr0w7QHes2BHf4WIiIi0g0K9MMkpaYDMO31W3yuREREJHpRTV5ujAkCgzs/31q7K1ZF+Snc2ux3CSIiIt12wkD35m3/HnAAiHibLTA9hnX5ZtzcebDU7ypERES6J5oW+l3ARGtteayLiQeZ2Xl+lyAiItJt0RxD3w1Ux7qQeLIi5xL2UeR3GSIiIlGLpoW+DXjDGPMC0HGA2Vr7YMyq8lkkKZ0QOpYuIiKJI5pA3+VdUrxLv2eTMwjZJr/LEBERiVo0i7PcC2CMyfTu18W6KL/Z5HTSTAuRcJhAMOh3OSIiIid0wmPoxpipxpgPgHXAOmPMKmPMlNiX5h+T4s5Fb2yo9bkSERGR6EQzKO5R4OvW2lHW2lHAN4Bfx7Ysf5kUt2xqY70CXUREEkM0gZ5hrX29/Y619g2gXy8UHkh1f15zQ78/uiAiIv1EVKPcjTH/AvzOu/9Z3Mj3fivoBXpLY43PlYiIiEQnmhb6rUAR8CfvUuRt67eCoUwAmhvVQhcRkcQQzSj3SuDOPqglasaY+cD8cePGxeT9k71Ab1Wgi4hIgjhmoBtjHrLW/oMxZglu7vYurLVXxrSy47DWLgGWzJkz57ZYvH9ymgv0tiYFuoiIJIbjtdDbj5k/0BeFxJOUtCwAwgp0ERFJEMcMdGvtKu/madban3Z+zBhzF/BmLAvzU2q6a6FHmut9rkRERCQ60QyK+9xRtn2+l+uIK2nproVeWVXlcyUiIiLROd4x9IXAjcAYY8xznR7KAipiXZifQhku0C/d81PgPn+LERERicLxjqEvA/YBhcBPOm2vBT6KZVF+S0kJ+V2CiIhItxzvGPpOYCdwVt+VEx9MIECZyacsUMQkv4sRERGJQjSLs5xpjFlhjKkzxrQYY8LGmH4/hdr+tHEYG/a7DBERkahEMyju58BCYDOQBnwR+EUsi4oHbcmZhCINfpchIiISlWgCHWvtFiBorQ1ba38LzIttWf6LJGeRbhXoIiKSGKJZnKXBGJMCfGiM+TFuoFxUPwQSmU3JIJ1G9lY1Mjw3ze9yREREjiuaYL4ZCAJ3APXACOCaWBYVD+pJJ8M084n7X/W7FBERkROKZnGWnd7NRuDe2JYTP5oD6QDkUetzJSIiIid2vIll1nCURVnaWWunx6SiOBEKu4H8j6f8CDe/joiISPw6Xgv9Cu/6q951+2Itn+U4Qd9fFGe4P3FaYIe/hYiIiEThmMfQrbU7ve72i621/2StXeNdvglc0ncl+mPM6ZcBEMH4XImIiMiJRTMozhhjzul05+woX5fYJl7GgYyJvB8ZTyTS7zskREQkwUVz2toXgP80xuQABqgEbo1pVXGiIX04ObUfU9fSRnYo2e9yREREjimaUe6rgBleoGOtrY55VXHChvLINXVUN7Qq0EVEJK4db5T7Z621vzfGfP2w7QBYax+McW3+S88nhzoONLQwIj/d72pERESO6Xgt9AzvOqsvColHSRn5pJgwtbXVQK7f5YiIiBzT8ZZPfcS7HjCTyRwuOasQgMaaMmCUv8WIiIgcx/G63B8+3guttXf2fjnRMcbMB+aPGzcupp+Tml0AQEXp/ph+joiISE8d7/SzVSe4+MZau8Rae3tOTk5MPycjpwiA599dF9PPERER6anjdbk/0ZeFxKNUr8t99iBNLiMiIvHthKetGWOKgG8Ck4FQ+3Zr7YUxrCs+pOUBEGodMGfqiYhIgopmxrc/ABuAMbjV1nYAK2JYU/zwAr2q/ADrS2p8LkZEROTYogn0AmvtY0CrtfZNa+2tQP9vnQMkh6gN5lJsSvn0w3/zuxoREZFjimbq11bvep8x5nKgBMiPXUnxpSpYQKFR61xEROJbNIH+A2/a128APwOygbtjWlUcqU/KpcDoGLqIiMS3aAL9XW/+9mrgghjXE3fScocQqtsDQGs4QnKw/y80JyIiiSeadHrbGPOyMeYLxpi8mFcUZ0aOGMGQpFoA/u5Xy32uRkRE5OhOGOjW2gnAd4ApwCpjzPPGmM/GvLI4YZLTCEUaeCblX/hwd5Xf5YiIiBxVVP3H1tr3rLVfB+YCFcDAmXSmyR0/Py2w1edCREREju2EgW6MyTbGfM4Y8yKwDNiHC/aBYdDkjpvZoWiGHIiIiPS9aBJqNfAMcJ+1duAdRJ7zBXj3V9TW1VJT3UZTa5hQctDvqkRERLqIpst9rLX27gEZ5gCBAJw6n/SWcsCy+UCd3xWJiIgcIZpBcbYvColrmYMJ2jaGUsH8ny/1uxoREZEj6KTqaKS5ifGWh77mcyEiIiJHp0CPRuG4LncfenWTT4WIiIgcXTSj3CcYY/5qjFnr3Z9ujPlO7EuLI8NnA1BDBgAPvbrZz2pERESOEE0L/dfAt/AWabHWfgTcEMui4tKcL2ACbnR7MGB8LkZERKSraAI93Vr73mHb2mJRTFzLGkpWpIYUWslM1fnoIiISX6IJ9DJjzCmABTDGXIubXGZgyRoMwHfOK6C6sZX65oH3m0ZEROJXNIH+VeARYJIxZi/wD8CXYlpVPMoaCsDM1vcBeHXDAT+rERER6SKaQN9prf0UUARMstaea63dGeO64k96AQDT3v8uAHct/tDPakRERLqIJtC3G2MeBc4EBu40afljj9i0ZHWJD4WIiIgcKZpAnwS8iut6326M+bkx5tzYlhWH0nJh8lWQnI43nICvLfrA35pEREQ80Uz92mCtfcpa+xlgJpANvBnzyuJR9R5obeBPGT8G4JSiDJ8LEhERcaKaKc4Yc54x5pfAKiAEXBfTquJV9jAAZkbWANDYEvazGhERkQ7RzBS3Azey/W/ANGvtddbaP8a6sLh0/rcAMMNnc/enJlBS3cRL6/ZTWd/ic2EiIjLQRdNCn26tvdpau8haWx/ziqJgjJlvjHm0urq6bz948GSYsRBqSijMSgHg//xuFbf/bmXf1iEiInKYYwa6MeafvJs/MMY8fPilj+o7KmvtEmvt7Tk5OX3/4XljoGYvF43L7ti0Ykclv35rW9/XIiIi4jneHKYbvOtVfVFIwig4BYAhbV0ny/u3v2zgtk8eeWqbiIhIXzhmoFtrl3jXT7RvM8YEgExrbU0f1Baf8se46+fv5o4LHubnr2/peOhbf1rDJ8YX8ulpQ30qTkREBqpoBsX9tzEm2xiTAawF1htj/jH2pcWpAm9t9N3v8I1LJrDxX+dx5Qw3+n3Re7v4yh/e97E4EREZqKIZFDfZa5FfBbwIjAFujmlV8SyUA8WnA2BqSgglBynKSu3ylDk/eJWd5XExflBERAaIaAI92RiTjAv056y1rbRPlTZQ7Vnhrn//GQDuvGh8l4fL6pp5csXuvq5KREQGsGgC/RFgB5ABvGWMGQUM3GPoAKff5q69Fdhy0pIZnpvW5Sm/fGMrY7/1Aj94fj2t4UhfVygiIgOMsbb7jW1jTJK11vcFwefMmWNXrvThHPC2ZvjFGRDKhv/zFgDldc0crG1m5Y4K/uXZdV2efueF4/j6JRP7vk4REelXjDGrrLVzjvZYNIPi7vIGxRljzGPGmPeBC3u9ykSSlArjL4HybeD9ICrITOXUodncfNZotv/w012e/vBrW/jt29vZU9ngR7UiIjIARNPlfqs3KO4SIA83IO7+mFaVCApOgZZaqN13xEPGGH587XTOm1DUse3eJes590evM/qeF3h3WzlLN5dxMr0jIiIiR3O8iWXaGe/608DvrLXrjDHmeC8YEIomuetfnAlfWwWZRV0evm7OCK6bM4LqxlZm3Ptyl8euf/SdjtvjBmVy1WnDGD84ixnFuQzJCcW8dBER6X9OeAzdGPNbYDjudLUZQBB4w1o7O/blHZ9vx9ABIhG4L8/dHv0J+Pzzx3zqX9bs494l6zhQ0xz1208akkXAGL74iTFcPn0oqUnBnlYsIiIJ7njH0KMJ9ABwGrDNWltljCkAhltrP+r9UrvH10AH+P+KXbc7wD273DnqJ7C1tI7fLd/JC2v2UVobfcA/9rk5XHTq4JOtVERE+oGeBroBbgLGWmvvM8aMBIZYa9/r/VK7x/dAX/ZzePnb7nbxXPjiK916eU1TK9bCyh0VjCrIYPOBWh5bup2VOyuP+vykgOGXN83ikilDelq5iIgkoJ4G+n8AEeBCa+2pxpg84GVr7em9X2r3+B7okQg8+xVYvcjd/37vLecaiVgCAUNrOMK/vbCBx5ft6PL44tvPZExhBgUZKSQFA7SFIyQFoxnjKCIiiaqngf6+tXaWMeYDa+1Mb9tqa+2MGNTaLb4HOkBbCzzyCXdu+l0fxuxjGlra+MZTq3lx7f7jPm/GiFzuvHAcZ44tICM1mjGPIiKSKHp0HjrQaowJ4k33aowpwrXYBSApBU69Eiq3w4v3wL7VMfmY9JQk/uOzs9n4r/OYOjz7mM9bvbuKLzyxkinfe4nR97zAi2v2sbeqkUhEp8iJiPRn0bTQbwKuB2YBTwDXAt+x1v5P7Ms7vrhooQOsfw6e6rReTS92vR/PgZomahpb+WB3FdmhJFKTgjz74V6e+bDkqM//P58cy7ypQ5hRnEsgoDMPRUQSzUl3uXsj3M8EKoCLcOek/9VauyEWhXZX3AS6tXBv7qH739oLqZn+1YM7Bv/mplJueXzFEY8FA4Znv3oOU4efeFS+iIjEj54eQ+84dh5v4ibQAR6aBlW7Dt2/ez3kDPevnk6stWzcX8u/vbCBpVvKjvvcKcOyueH0Edx4xiiCasWLiMSVngb6A8By4E82zuYqjatAr9gGJR/A07e6+5/8J7jw2/7WdBTWWp7/aB9fW/TBCZ87tiiDa2YVc+WMYRTnpaEJAkVE/NXTQK/FLZ3aBjThut2ttfbYI7P6SFwFervvd+rG7qNj6SerqTXM/uomyuubaWmzTB6WzbbSOu5/cSPvbq846mt+cNVUFpw2jKxQch9XKyIiPQr0eBaXgV69B/59yqH7d6+DnGL/6jlJreEIf9tcymNLt/Pe9gpaw12/J1mpSSQnBRhVkM6UYdlcMX0Yp4/OJxgw1Da1sr6khjc2lbL1YB0t4QjDc9PYX93E0NwQm/bXceVpw9heVs+cUXnMHZNPKDlIekpQvQAiIsfR0xb6rKNsrgZ2+r0melwGOkDlTvjp9EP3/+9myBzkXz29oDUcoaSqkf9ZuYc/vLuTyobWXv+MjJQg04pzuGDiIGaPymPWyDyNxhcR6aSngf4O7pS1Nd6macBaIAf4srX25WO9NtbiNtAB/uMcOLDW3Z7zBbjiQX/r6WVVDS2s2lnJ5GHZbD5Qx9ItZTz61rYuz7l8+lDOn1DEsNw0ahpbCQYMw3LTWLqljBnFuQVkuxMAACAASURBVDy9ag9bDtbSErZUN7QQsbC/pqnj9cNyQswYkcvU4TksnDuS/IyUvv4zRUTiSk8D/U/Av1hr13n3JwP3Af+EGyh3Wi/XG7W4DvTqvfC3n8DKx9z9fymHYP+fuW3LwTpGFaSTFDAn1X0ejlje217B6x8f5INdlazYcWhe+6nDs7l82jA+M2s4g7O1zKyIDDw9DfS11tqpR9tmjPlQgX4CD06Bmj3u9s3PwCkX+FtPgqluaGXpljJW7qxgyep9lNU1EzDwyQlFnDGmgLlj8pk8NJtQckDH30Wk3+tpoD+Jm1hmsbfpeqAQuBlY6uciLQkR6OE2+NeCQ/eveAjm3OJfPQnMWsvmg3X85m/beG1jKWV1h5afzQol8empQ5k/YxjnjCtQuItIv9TTQE8DvgKc6216G/gl7hS2dGttXS/W2i0JEegAq56AJXceuj/nVrjkB5CS4V9NCa4tHOG97RW8s72C3RUNvLxuP2FraWqNkJuezDmnFDI0J8TMkXl8YkIh2TrNTkT6gR6ftmaMSQEm4hZo+dha2/tDnE9CwgQ6uElnlvwD7Ou0ItvVj8CMG/yrqZ9pbAnz/EclLF6xm80HaqlpOnQSxuiCdM6fOIi/m1PM5KHZasGLSELqaQv9fNyiLDtwk8qMAD5nrX2rd8vsvoQK9HYvfhPe/dWh+yYAX14Ogyb5V1M/VdXQwrKt5azcUcmKHRWsLanGWhiem8YZY/KZNDSL3RWNDMkJMWukOx9e092KSDzraaCvAm601n7s3Z8ALLLWzu71SrspIQMdYMur8Ptrum4bPgeu+TXkj/WnpgHgYE0Tz3y4lxfW7OejPVUc/tUflhNi0tBspg7L5hMTipit8+BFJM70NNA/stZOP9E2PyRsoANEIm7J1Y3Pd90+YR5c/4cBcYqbn8IRy5aDdQzJCdHcGuad7RU8uWIXb28p73hObnoyM0fkMmd0PudNKGLKMHXVi4i/ehrovwXCwO+9TTcBQWvtrb1a5UlI6EBvF26D1/4V3n6o6/ZbX4IRZ4ACpE/VNrXy4e4qDtQ089rGA7yzrYKK+hYAkgKGUQXpzB1TwLypQ5g9Ko/MVP3wEpG+09NATwW+yqFR7n8DfmmtbT72q/pGvwj0zpb9HF4+bIW2r7wDg071px7BWsu2snre+LiU93dV8tqGgzS2hjsenzEil0+OL2RUQQZThmUzcXCWuulFJGZOOtCNMUFgnbU2Lkds9btABzca/tHzu2675jGYdq0v5ciR6prbeHtLGcu3lvP6xwfZWd7Q5fFpw3M4bUQuc0bncdYpBQzK0qx2ItI7etpCfxb4mrV2VyyK64l+Gejtdr0Df/4SVG539z/5j3Dhd/ytSY6qrK6Z9SU17Cyv58Pd1awrqWbzwTrCEff/Vn5GChMHZzFuUCaDs1OZPSqf6cU5ZKi7XkS6qaeB/hYwE3gPqG/fbq29sjeLPBn9OtDbbXwBFt/obp91B1x8HwSC/tYkJ9TQ0saKHZW8vaWMAzVNrNpZyZ7Kxo7HQ8kBzhxbwMwRecwcmcvsUXkKeBE5oZ4G+nlH226tfbMXauuRARHoAFW74aFO0+l/YxNkDfavHjlp+6ub+GBXJW9uKuWNj0u7rC43Ij+N6cNzmTUqj9NH5zEqP4OcdM1wJyKHnFSgG2NCwJeAcbilUx/ze/3zww2YQAeoL4f/1+kc9S+8AiPm+leP9IqDtU2s3FHJh7ur+Hh/LR/tqeqy1vyognTOGVfIqUOyOG/CIEYWpPtYrYj47WQD/UmgFTeq/TJgp7X2rphV2Q3GmPnA/HHjxt22efNmv8vpO+FW+NfCQ/fTC+Cmp2HYTJ3e1o9sL6tnzd5qNh+oZdnWctbsqaYlHAFgTGEGM0fmMnNkHpOHZjOuKFOteJEB5GQDfY21dpp3Owl4z1o7K3Zldt+AaqG3sxZe+ja884uu21Nz3Exz4y9RuPcz1lq2l9Xz1qZS3txUyge7q6jq1IofU5jB+EGZnDIok3FFmYwtyqAgI5XivDSdQifSz5xsoL/fOcAPvx8PBmSgt6svh2e/CptePPKxf1gDuSP7vibpE+3nxq/eXcWO8gY27qth/b6aLoPuwC0pe+rQbGaPyuPccYVMHZaj1rxIgjvZQA9zaFS7AdKABu+2tdZmx6DWbhnQgd6urQXW/Qne+zXs7bQvgqnwleVQcIp/tUmfsdZS19zGttJ6DtQ0UVbXwpq91by/s5LNB2uJWAgYmDM6n0lDshg/KJPxg7OYNCSL3PQUv8sXkSj1ePnUeKVAP0wk4qaQ/eu9h7bNvgUu/4lOdRvAaptaWbWzkve2V/C/a/ezrazj7FOMganDcjhnXCFjizL45PgihuRoIhyReKVAH2jaWmD1IvjLP0LYm6F31ufgsh9Dsv6xHujCEcu20jp2ljewfl8NSzeX8f6uStq8iXCK89KYXpzD7FH5nDYih/GDs8gOqateJB4o0Aeq5jp4835Y9rND2xb8EmbcoBa7dNHYEmZ7WT2vf3yQtXurWbO3uuOYfDBgmFGcw+xReeRnpDJhcCaThmYzLCek1edE+pgCfaBrrITFn4WdS939vDHw1fcgScdO5dgO1DSxencVq/dUsXxrOWv2VtMaPvTvRWFmCtOLc5k5IpfTRuYyIi+dobkhUpP0Y1EkVhTo4hxYD7+5CFobIC0fzvsmnP4FCKo7VU4sErHUNrexdm81W0vrWL27mtV7qthysK7jOUkBw6ShWZxSlMnQnDTGD8pkdGEGOWlJBIwhPyOFnLRktexFTpICXbp67Qfw1v87dH/YLPjk/4VTLtIxdum20tpmPt5fy+7KBnaU1bN6TxVbS+sprT36Csvt68oXZaWSk5ZMcZ67PTw3jVOHZjO2MEPnz4scgwJdjlS6Cf50G+z7sOv2oafB5Q9C8Wx/6pJ+o6GljU0H6jhY00RpXTNBY6hqbKWkqpH91U3sqmigrrmNsrpmmlojHa8LBlxLfkh2iCE5IYrz0hiVn86EwVmcMiiTwsxUggp8GaAU6HJsrY1uRbePnoSKbVC+xW0fOgNu+iNkFvlbn/R71lrqW8JsL61nw/4athyso7K+hdK6ZvZVNbGzor5L4KcEA2SnJVOYmcKYwgyK89IYnpvG4OwQg7JD5KQlMbogg6RgwMe/SiQ2FOgSHWth97uw6AY3kA6gaBJc/wcoHOdvbTJghSOWg7VNbCutZ8vBOkqqGimvb+FgbTN7KhvYW9lIc1uky2tSkwJMGpLF5GHZTBuey8yRuYzMT9cStZLwFOjSfRuWwJOfPXQ/ZyQs+BmMPd+vikSOylpLaV0zWw7UUVbfQktbpGM63PX7arrMe1+QkcKEwVmMLsygKDOFUwZlMrYwkzFFGWQq7CUBKNDl5ITb3LSyS+5yI+MBhs+GKx6CodP9rU0kCtZadpQ3sGZvNbsrGthV3sDGA7XsKq+nurEVby4dAgaGZIeYOSqP4tw0xhZlMDw3nfyMFE4ZlKFT8SRuKNCl53Yug7cegK1/dfdTc+DqX8GgSZA/9vivFYlDzW1htpXWs6uigQ37athaWs/7OyspqW6k8z+LxsCwnDRmj8pj1shcphXnMnloNmkpCnnpewp06T0V2+GPX+y6EMzwOXD+PTD+Yv/qEuklreEI+6ub2FPZyL7qRnaU1bO1tJ73dlR0ORWvKMstUZsSDFCQmUJqUpCirFQKMlLIDCWRkZJEbnoySYEAxrhVraoaW2kNR2hujdDcFqaxNUxuegoZKUkcqGmivrmN5KQAycEAI/LSGJ6XxpjCDMIRS3pKkkb3iwJdYqBiuzuXvXQj7F3ltmUUwY1PQcE4CLdASgYkhbQ+u/QL1lr21zSxZk81a/dWs7OigdLaZmqb2jhY20Q44hbCOXyAXm9KDhpaw5a0ZO/HQ2YKycEAw3JCFGWlMigrRGYoiYIMtz0vI4VhOSGy05IJJatHoT9QoEtsNdfC0ofgbw8c+VggGSKtcO7XYfQ5bhKb9Py+r1GkD1hraWgJU1HfQl1zGzWNrbSGLcGAwRhISQqQEgyQkhQglBQkJSlAS1uEyoYWMlKTyEt3wdvSFmFnRQMlVY1sPVhHxEJDaxt1TW20tEXYXdnAoKwQJVWNtIbdczsP/jtcwEBhZiqZoSRG5KWTkRqkIMPdH57rTvsbmhtidEEGAMnBgHoD4pQCXfrG/rWu1b7xBRfixzPvRzBkqgv4lPS+qU+kH4tEbMfx/8qGFraW1tHQEqa01p3Pb7HUNrWxu7KB3RWNNLWGj+hNMMadvZqWHGTikCxG5qdjgXFFmYwuTKc4L53huWkUZWlyH78o0MVfbc3w0VOw/U3YtxrKNnV9/OyvQeFEmHYtJKf5U6PIAGOtm5u/qr6Vjw/UUt/cxvayeqoaWjDGsGFfDetLaqhtbjvite2z+eWnpzA4J0RhRgp1zW2MyE9nbFEG4wdlMSTbHQbQ4MHepUCX+BIJw54V8PbDsGMpNFe77VlD4dT5MPsWGDzZ3xpFBGstLeEI1sKuigb2VjWy1xssuL+6mbK6ZqoaWiirayEYMByoaTqi1Z8dSmJwdsjN01+UwYi8dEbkpzMiP43BWSHN299NCnSJbxXb4d1HoOQD2P2O21Z8OnziGzD+UghoCk+RRBCJWPZWNbK9rJ591Y2U1bWwv7qJfdWNbNhXe8QpgeCW4Z0wOMvN2V+QQV56CkNzQuSmJ5MVSqI4L10D+jpRoEviqNwJ7z0KH/wOmqohczCc/kWYcytkFPpdnYj0QHNbmL2VjeyubGR3RQM7y+upqG9la2kduysaKK9vOeI1xpv0Jy89hVBygKRAgKSgITuUTHFeGiML0hmcHWJ4bhqFmakUZqb063n8FeiSeFqbYO3TsPTf3YIxJggzboBz7oKiiX5XJyK9zFpLdWMrDS1h9tc0UVnfQk1TKzvKGthT2UhVQwvNbRHaIhFaw5aaxlZ2VzZ0WbgH3A+AosxUhuaEGJrjRu8XZro5A4rz0hmUlUpRVmrCtvoV6JK4rIUDa2Hlb12rPRKGc+6EM78CmYP8rk5EfBSJWMrqmtlf00RJVRNldc0crGlif00T+6q9S1Uj9S3hI15bkJHCiHw3an9Qdiq5aSkMyk5lSE6IIdkhhuaEyElLxsTZPBoKdOkfKrbBq/fC+mfc+e2z/h5mf17zyovIcTW0tLG7opGSqkZK65o7juvvqmigpKqJ0tpm6o4ymj+UHGBIdqgj5IfkpDEkO9Vd57jQL8zs21P4FOjSv+xeAe8/Dh8uAhuG7GKYdo071p470u/qRCQBtYYjHKxtZr83gn9fdSMHvJZ++/XBmmZawl27+IMBw6CsVAZnu9AvzEohJy2ZQVkhBmWlMrYok4lDsnqtTgW69E8V22Ddn91SryUfQFIazPuha7XHWTeZiCS+SMRS0eBG7u+vdl37na8P1Lhu/5qmNsLeUn4XTx7Mr//+qPl7UhTo0v8d3Aj/ew9sex2KToXTboTJCyBvlN+VicgAE4lYyutbOFjbRFIgoBZ6NBTo0kUkAh/8l5tXvnK7Gxk//hLXFX/KhTqfXUQS3vECPamvixGJmUDAdbfP/jyUb4XlP4fVi2HTi5A9HE67yYV71mC/KxUR6XVqoUv/1tLgzmd//3ew5z13nH3Wza5LfthMv6sTEekWdbmLABzcAG/+CNY9A1gYPNWt1z5oEpx6peue12A6EYljCnSRzmr3w9o/wkdPgo24ZV+xMGiKW/Ft1ucgo8DvKkVEjqBAFzmecKsL+KX/DqUb3bYx58Hc21yrPSnV3/pERDwaFCdyPMFkN0/8tL9z57XvWAqbXoInPwvBVBh/sQv3MeepS15E4pYCXaRdIOi63KddC23NsOVVN2nNx3+Bjc9DarY71n7ajTDqbIW7iMQVBbrI0SSlwqTL3aW1Cdb8D2z6X3f94e8h/xSYcjXMvAnyx/pdrYiIjqGLdEtLg+uW//APsPNtwMDoc+HU+TDuU1Bwit8Vikg/pkFxIrFQsw9W/MaNlq/e7bYVjHPd8qdeAcNmqVteRHqVAl0klqyFqp3w8Yuw/lnY/a47HS5nBEy9xs0pP2ymwl1EekyBLtKXakpg88vw0VOwcxlgIWsYjD4HRp7lWvCZRX5XKSIJSIEu4pfqvbD1NdjyCmx+BVob3KIx4z7lTpObeBmkZvpdpYgkCJ2HLuKXnOFu7vhZN0O4Dco2uWPuq34Lm19yc8uPvximX+dWhEvJ8LtiEUlQaqGL+CHc5haLWfM0rF50qOU+4gwY/ymYdAUUTtBxdxHpQl3uIvEs3AY7/uYmstn0EpRvdttzR7kBdVOuhiHTIagONZGBToEukkgqd7hg3/yKO/5uw5AxCCbOg+k3uFa8wl1kQFKgiySq2v1u+tkdS71BdfXudLhJV7hZ7EbM1eIxIgOIAl2kP2iudS33j56EbW9CuBmS07vOVJc9zO8qRSSGNMpdpD9IzTq0eExTNWx9Hba97nXPv+yekz8WTrnITWQz+lzIG+VvzSLSZ9RCF0l01kLJ+7D5Vdi1HHa9A22N7rGiU91sddOuhfwx/tYpIj2mLneRgSTcBqUbXav94xfd6XEAxae7pV/HnOda8jolTiThKNBFBrLqPW7Z14+egoPr3bbcUW4im/EXu+vkNH9rFJGoKNBFxHXN7/8Idr/nRszvWg7NNZCSCRM/DeMvgUmf1mx1InFMg+JExHWxD53hLnNvg3ArbH8L1j/jTo1b85SbinbcRS7gT7kQsof6XbWIREktdBGBSNi12Nc9Ax//BWr2AsatDjfuwkMj53XcXcRX6nIXkei1d81vegnWPwcH1rjt2cNh1DlucN34T7mBdSLSpxToInLyag+45V83veQmtGmudtsLJ7rlXyd+GornQCDob50iA4ACXUR6h7VQvsWdErf5ZTclbaQNQjlQPBdGnuHmmh8+W4PrRGJAg+JEpHcYA4Xj3eWsr7oZ6za/4laL2/UuvPaKe14gyQX7qLNd0A+bCekFEAj4W79IP6ZAF5GTF8o5NB0tQGMl7FkJO9+GLX+Ftx4AOvUCFk2CIdNg6Gkw7DQYPAXS8nwpXaS/UZe7iMROc52binbfh27luPItULbJG0XvyR7ugn3wFBj9CTfwLjnkX80icUzH0EUkvtQegL2rYO9KKN/qgr50ozsen5QGRRNh+CwYPNVdhp2mZWJFUKCLSCJoqnat+a2vQ+kG2Pu+m8kODoX8qHPc8fihM6BgnI7Jy4CjQXEiEv9COTDhUncBiESgYhscXOemq923Glb8GsIt7vFgqhucVzTJnTY36mw3R31arn9/g4iPFOgiEp8CASgc5y6TF7ht4VYo/diF+4F1UPIB7FwGa58+9LqckTDyTK8lP91d6xQ6GQDiJtCNMWOBbwM51tpr/a5HROJQMBmGTHWXziq2uXPi6w7A/rWw/U03Nz2ACcCgKVA82x2PLz7dnUKXXgAp6X3/N4jESEwD3Rjzn8AVwEFr7dRO2+cBPwWCwG+stfdba7cBXzDGPH30dxMROYb8sV2norUWave5cN+7EvasgHV/hlWPH3qOCbjZ7rKGuJAfPBkGTYacYrXoJSHFuoX+OPBz4L/aNxhjgsAvgIuBPcAKY8xz1tr1Ma5FRAYKYyB7mLtMuMRti0SgaqcbeFe9xx2L37sSKra7Fr2NeK8NQP4pbuBdRqH7oTB0BgyZrha9xLWYBrq19i1jzOjDNs8Ftngtcowxi4EFgAJdRGInEID8Me5yuMZKKN0E1buhbLO3bvy7bntLnXuOCbqR9iPmuvPlB02GjCL3AyAQcD8Y0vOjX5GurRnqS93gvswi9yOjahe0NUFTjRskOHgKZA46+mt1Gp8cxo9j6MOB3Z3u7wHOMMYUAP8GzDTGfMta+8OjvdgYcztwO8DIkSNjXauIDARpeW4ees7our29677kA3ca3b7VsPawrvsu75PvfjC0tbiQD3UacZ9TDDUl7nh/QwW01B56LJAMkdZj15ec4X44mMChxXEyitx1eqG7zhvl/o7MwW7kP7gFc0zA9VTkjYFgSvd+dEhCiZtBcdbacuBLUTzvUeBRcOehx7ouERnAOnfdT7rcbQu3uVPpyja7FnZLnWudJ4fcBDnl29xc9hjXwsdCUshNhdtcC4MmuS78UDakZEH2UKjZB0kpLoiT01zwmoAL/30fuR8Ugye7wwKBJPfDIdzsPqtim/uMTRsgZ4Sbke94Pw5Sc9yPgsFTXf3DZ7tegeLTXa+DMd7sfVM1Y1+C8SPQ9wIjOt0v9raJiMS/YJIL5KEz/K7k6MKtULnDnd6XXnBovED5Fhf0LfVuvv1wK7Q0wLY3oLUBtr525HslpUFbowv73FHuh8bwWe7QQ+5Idz9rqFr8ccKPQF8BjDfGjMEF+Q3AjT7UISLS/wSTD62IF61I2IV+6UYo2+J+tBzc4H4UbH3NPb7lFTeb3+r/7vrapJA7ZJCW6z5z8BQX/tnDYNCpbl6AYNx0BvdrsT5tbRFwPlBojNkDfM9a+5gx5g7gJdxpa/9prV0XyzpEROQ4AkEIpLlJeIbNdNumXN31Oda6YK/zFtkp3woH1rpT/NoHFDaUw7uPHJrND1zY5412U/UWjnOHFbKGuLkBMopc6z4Q7LM/tT/TXO4iItJ7wq1ufEFzLZRv9sLfG1tQvsUd++8sKc216odOd4cIMge74/fpBW5cQc4IBX4nmstdRET6RjDZDeAD78yBTsKtbvncuoMu9A+sdQPyakpgzR8PjeDv8n4pbgKgogmQMcj9UEhKhZzhLvwHTXZd+8lpsf/b4pwCXURE+kYw2bXGB0+BUy7o+pi17qyAtiY3w1/ldre9eg8cXO8W6KkpARt25+g3dQp/E3ATABVNcpfCCe70wcLx7lS+AUKBLiIi/jPGnSMPbkDd0bS1uOPzqZlutH7tftfKP7DOXUo/ho9fdKHfLnOwG5VfONFdt4d+RmG/G52vQBcRkcSQlOIu4AbjFZziLu2r8YGbRa9yh5vSt2yTC/nSjbB6cdfJfNLyvYBvD3nvOoFPw1Ogi4hI/5GUeiioJ847tN1a12VfutEL+o0u7Nc9A01Vh56Xmu267DuHfNFEb3BeoO//nm5IyFHuxpj5wPxx48bdtnnzZr/LERGRRGWtmzGvvSXf+br+4KHnJae7Y/Kdg75wojslrw/Psz/eKPeEDPR2Om1NRERipqGia2u+/bqm0+SmwRQoGN+pNT/BjbzPH+sGAfYynbYmIiLSXen5MPJMd+msqcaddle68VDI710F6/4MeI1kE3St91OvgIvv65NyFegiIiLdEcqG4tnu0llLw6EWfdlmN7FOcnqflaVAFxER6Q0p6TDsNHfxQXwP2RMREZGoKNBFRET6AQW6iIhIP6BAFxER6QcU6CIiIv2AAl1ERKQfUKCLiIj0AwkZ6MaY+caYR6urq0/8ZBERkQEgIQPdWrvEWnt7Tk6O36WIiIjEhYQMdBEREelKgS4iItIPKNBFRET6AQW6iIhIP2CstX7XcNKMMaXAzl58y0KgrBffbyDSPuw57cOe0z7sHdqPPdfb+3CUtbboaA8kdKD3NmPMSmvtHL/rSGTahz2nfdhz2oe9Q/ux5/pyH6rLXUREpB9QoIuIiPQDCvSuHvW7gH5A+7DntA97Tvuwd2g/9lyf7UMdQxcREekH1EIXERHpBxTogDFmnjHmY2PMFmPMPX7XE0+MMSOMMa8bY9YbY9YZY+7ytucbY14xxmz2rvO87cYY87C3Lz8yxszq9F6f856/2RjzOb/+Jr8YY4LGmA+MMc9798cYY9719tWTxpgUb3uqd3+L9/joTu/xLW/7x8aYS/35S/xjjMk1xjxtjNlojNlgjDlL38XuMcbc7f2/vNYYs8gYE9J38fiMMf9pjDlojFnbaVuvfe+MMbONMWu81zxsjDEnVai1dkBfgCCwFRgLpACrgcl+1xUvF2AoMMu7nQVsAiYDPwbu8bbfA/zIu/1p4EXAAGcC73rb84Ft3nWedzvP77+vj/fl14H/Bp737j8F3ODd/hXwZe/2V4BfebdvAJ70bk/2vp+pwBjvexv0++/q4334BPBF73YKkKvvYrf233BgO5DW6Tv4eX0XT7jfPgnMAtZ22tZr3zvgPe+5xnvtZSdTp1roMBfYYq3dZq1tARYDC3yuKW5Ya/dZa9/3btcCG3D/KCzA/eOKd32Vd3sB8F/WeQfINcYMBS4FXrHWVlhrK4FXgHl9+Kf4yhhTDFwO/Ma7b4ALgae9pxy+D9v37dPARd7zFwCLrbXN1trtwBbc93dAMMbk4P5hfQzAWttira1C38XuSgLSjDFJQDqwD30Xj8ta+xZQcdjmXvneeY9lW2vfsS7d/6vTe3WLAt2F0+5O9/d42+QwXnfbTOBdYLC1dp/30H5gsHf7WPtzoO/nh4B/AiLe/QKgylrb5t3vvD869pX3eLX3/IG+D8cApcBvvUMXvzHGZKDvYtSstXuBB4BduCCvBlah7+LJ6K3v3XDv9uHbu02BLlExxmQCfwT+wVpb0/kx71elTpc4BmPMFcBBa+0qv2tJcEm4bs//sNbOBOpxXZ0d9F08Pu847wLcj6NhQAYDq3ciJuLle6dAh73AiE73i71t4jHGJOPC/A/W2j95mw94XUV41we97cfanwN5P58DXGmM2YE7pHMh8FNcV1yS95zO+6NjX3mP5wDlDOx9CK7lssda+653/2lcwOu7GL1PAduttaXW2lbgT7jvp76L3ddb37u93u3Dt3ebAh1WAOO9UZ4puIEfz/lcU9zwjpc9Bmyw1j7Y6aHngPZRmp8Dnu20/e+9kZ5nAtVet9RLwCXGmDyvlXCJt63fs9Z+y1pbbK0djft+vWatvQl4HbjWe9rh+7B9317rPd9622/wRh6PAcbjBtMMCNba/cBuY8xEb9NFwHr0XeyOXcCZxph07//t9n2o72L3Ha6QGgAABNlJREFU9cr3znusxhhzpvff5O87vVf3+D16MB4uuFGJm3AjNb/tdz3xdAHOxXUlfQR86F0+jTuO9ldgM/AqkO893wC/8PblGmBOp/e6FTd4Zgtwi99/m0/783wOjXIfi/tHcAvwP0Cqtz3k3d/iPT620+u/7e3bjznJkbCJfAFOA1Z638dncKOF9V3s3j68F9gIrAV+hxupru/i8ffZItyYg9b/v737C62yjuM4/v64Bhn5B3GFBCmFtHZRS9PwoloU2D/oD+KIiuwP4k1E5E0QQXUhFBShZFHEJCkIJEF2kaGOmRou2tbCgrB2WwQSu0hv/Hbx/Z12Om7HZtnZHj4veNhznuf7POe3c8b57vnt2fdLzhQ9/V/+3AE3l/fjJLCDUvRtposrxZmZmVWAp9zNzMwqwAndzMysApzQzczMKsAJ3czMrAKc0M3MzCrACd1slpC0TdIdkh6U9OIMj+0o3bCGJd3aJK5Hpdtbk5huSffO5Pn/b5LGJS1t9TjMZhMndLPZ4xbgK+B2YHCGx94JjEXETRFx+F+Oo5usNWBmc4gTulmLSXpD0rfAGuAY8AywU9LLU8SukHSw9Fk+IOlqSd1kK8cHJI1Imt9wzN3K/uHfAA/XbV8r6Vi5qj8q6bpSLfFVoLecq3equCnGtUzSYDnmu9osgaSdkr5W9t9+pS5+vMxIjJT9qyR9LumkpC0lpqecs1/Zc/tdSed8Zkl6TNLxcq73lH3n2yT1lbGMSXr+gt4cs7mk1RV4vHjxEpDJfDvQDhxpErcPeKKsPwXsLeubgB1TxF9KdnhaSVaw+pTJSnULgUvK+l3AnqnONV1cw/O8QKmyCLQBC8r6krptA8AN5fE4kz233yIrvy0AOoBfyvYe4DRZxayNbDe5oe74pcD15TVpL9vfIUtnriZbVdbGt7jV77EXLxd7qRXjN7PWWgWMAp1kz/nprGPyKvsj8sq8mU6yGcePAJJ2A5vLvkXALkkryfK+7dOc45/EDQEfKhv57I2IkbJ9o6TNZKe0ZUAXmbxhsmfCGHB5REwAE5LOSFpc9h2PiJ/K2D8hSxHX+nZD/qlhNTCUZbCZTzbJ2AdcI2k70A/sb/IamVWCE7pZC5Xp8j6yw9JvwGW5WSPAuoj44yI+/WvAoYh4SNnrfuBC4yJiUNJtwH1An6Q3gcPAVmBNRJyS1EfOGNScKV/P1q3XHtc+mxprUzc+FrArIs65iVDSjcB6YAuwkZzRMKss/w3drIUiYiQiusnmQF3AQWB9RHRPk8yPkh3bAB4lk2YzPwArJF1bHj9St28Rk20aN9VtnyCnv88X9xdJy8mp8veBD8gZh4Vkz/LfJV0J3HOesU5lrbIT4jygF/iyYf8BYIOkK8o4lkhaXu6AnxcRe4CXynjMKs0J3azFJHUApyLiLNAZESeahD8LPFluonsceK7ZuSPiNDnF3l9uivu1bvfrwDZJw/x9tu4Q0FW7Ka5JXL0eYLTE9AJvR8QoMEz+UvExcKTZWKcxRHaf+h74Gfis4fs7QSbs/eU1+YKc2r8KGCgzHbuBGf0boNlc5G5rZjYrSeoBtkbE/a0ei9lc4Ct0MzOzCvAVupmZWQX4Ct3MzKwCnNDNzMwqwAndzMysApzQzczMKsAJ3czMrAKc0M3MzCrgTwX/Tqy41DS7AAAAAElFTkSuQmCC\n" - }, - "metadata": { - "needs_background": "light" - } - } - ], "source": [ "import matplotlib.pyplot as plt\n", "def plot_progressive_loss(obj_list, alias, result_interval=1,):\n", @@ -290,22 +276,48 @@ "plot_progressive_loss(loss_list_vanilla, 'VanillaVW')\n", "plot_progressive_loss(loss_list_autovw_ni, 'AutoVW:NI')\n", "plt.show()" - ] + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAFzCAYAAADIY/vqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdeZxcVZ3//9ep6qV637N2VrKRlSyEVdkEghACwgABGQWFrwvCoN8ZcXRUGOcn+kUGcRlBGWHUSWBQgYAMi2yGBEgChKxkXztL7/tadX5/nNud7qzV6a6+Vd3v5+NRj6q6tX36UuRd59xzzzHWWkRERCSxBfwuQERERHpOgS4iItIPKNBFRET6AQW6iIhIP6BAFxER6QcU6CIiIv1Akt8F9ERhYaEdPXq032WIiIj0iVWrVpVZa4uO9lhCB/ro0aNZuXKl32WIiIj0CWPMzmM9pi53ERGRfkCBLiIi0g8o0EVERPqBhD6GLiIi/mhtbWXPnj00NTX5XUq/FAqFKC4uJjk5OerXKNBFRKTb9uzZQ1ZWFqNHj8YY43c5/Yq1lvLycvbs2cOYMWOifp263EVEpNuampooKChQmMeAMYaCgoJu934o0EVE5KQozGPnZPatAl1ERBLOBRdcwEsvvdRl20MPPcSXv/zlbr3Pc889x/333w/A97//fR544AEAPv/5z/P0008f83VPPPEECxcu7LKtrKyMoqIinn32Wa666qqO7T/84Q8ZN25cx/0lS5Zw5ZVXdqvOaCjQRUQk4SxcuJDFixd32bZ48eIjQvZErrzySu65555uf/7VV1/NK6+8QkNDQ8e2p59+mvnz53P22WfzzjvvdGxfvnw52dnZHDx4EIBly5Zx9tlnd/szT0SBLiIiCefaa6/lhRdeoKWlBYAdO3ZQUlLCokWLmDNnDlOmTOF73/tex/NHjx7N9773PWbNmsW0adPYuHEjAI8//jh33HHHcT/rvvvu4/TTT2fq1KncfvvtWGvJzs7mvPPOY8mSJR3Pa/9BUVRURHZ2Nlu2bAFg7969XHPNNSxbtgxwgX7OOef06v4AjXIXEZEeunfJOtaX1PTqe04els335k855uP5+fnMnTuXF198kQULFrB48WKuu+46/vmf/5n8/HzC4TAXXXQRH330EdOnTwegsLCQ999/n1/+8pc88MAD/OY3v4mqljvuuIPvfve7ANx88808//zzzJ8/n4ULF/KHP/yB66+/npKSEjZt2sSFF14IwDnnnMOyZcsIh8OMHz+eM888k5deeokrrriC1atXc/rpp/dwDx1JLXTP1jXvsO7tF/wuQ0REotS52729dfzUU08xa9YsZs6cybp161i/fn3H8z/zmc8AMHv2bHbs2BH157z++uucccYZTJs2jddee41169YBcPnll/P2229TU1PDU089xTXXXEMwGATg7LPPZtmyZSxbtoyzzjqLuXPn8u677/LBBx8wadIkQqFQL+2FQ9RC91S+8gDDaj+Ccy73uxQRkYRyvJZ0LC1YsIC7776b999/n4aGBvLz83nggQdYsWIFeXl5fP7zn+9y6ldqaioAwWCQtra2qD6jqamJr3zlK6xcuZIRI0bw/e9/v+M909LSmDdvHn/+859ZvHgxDz74YMfrzjnnHH72s58RDoe57bbbyMrKoqmpiTfeeCMmx89BLfQO1gQwNuJ3GSIiEqXMzEwuuOACbr31VhYuXEhNTQ0ZGRnk5ORw4MABXnzxxR5/Rnt4FxYWUldXd8TI94ULF/Lggw9y4MABzjrrrI7tp556KiUlJSxdupSZM2cCcNppp/GrX/0qJsfPQYHewZogBut3GSIi0g0LFy5k9erVLFy4kBkzZjBz5kwmTZrEjTfe2CvBmZuby2233cbUqVO59NJLjzj2ffHFF1NSUsL111/f5dxxYwxnnHEGBQUFHdO3nnXWWWzbti1mLXRjbeKG2Jw5c2xvrYf+3k9vZHTlcgZ9f3uvvJ+ISH+2YcMGTj31VL/L6NeOto+NMaustXOO9ny10NuZAAHU5S4iIolJge5Rl7uIiCQyBXo7EyCgQBcRkQSlQO9gCBD2uwgREZGTokD32IC63EVEJHEp0NuZAIEEHvEvIiIDmwK9g9EodxGRBPPMM89gjOlYbOV4HnrooS6rox3NLbfcwiOPPHLEZ1x22WXcfffdPPTQQx3bL730Ur74xS923P/GN77RZbY4cEuypqend6y0Bm5CnKPd7ikFuscGggp0EZEEs2jRIs4991wWLVp0wudGE+jHW5a1fcEVgEgkQllZWce87nDsZVELCwv5yU9+Es2f0yMK9HYa5S4iklDq6upYunQpjz32WEcIv/HGG1xxxRUdz7njjjt4/PHHefjhhykpKeGCCy7gggsuANyPgWnTpjF16lS++c1vAnDRRRexceNG9u3bB0B9fT2vvvoqV111FWeffTbLly8HYN26dUydOpWsrCwqKytpbm5mw4YNzJo164g6b731Vp588kkqKipiuj+0OEs7TSwjInJyXrwH9q/p3fccMg0uu/+4T3n22WeZN28eEyZMoKCggFWrVh3zuXfeeScPPvggr7/+OoWFhZSUlPDNb36TVatWkZeXxyWXXMIzzzzDVVddxTXXXMNTTz3FXXfdxZIlSzj//PPJzs4mOzubpKQkdu3a1bGK2t69e1m+fDk5OTlMmzaNlJQUvvvd7zJnzhyuvPJKwHWr33rrrfz0pz/l3nvv7dXd1Jla6B5jgmqhi4gkkEWLFnHDDTcAcMMNN0TV7d5uxYoVnH/++RQVFZGUlMRNN93EW2+9BRx9WdZ2hy+LetZZZ3Xcb587/r777usI83Z33nknTzzxBLW1tT36m49HLXSPNQECxmIjEUxAv3NERKJ2gpZ0LFRUVPDaa6+xZs0ajDGEw2GMMSxYsIBI5FBva+flU6N19tlns2/fPlavXs2yZcu6HFNvP46+Zs0apk6dyogRI/jJT35CdnY2t9xyyzHfMzc3lxtvvJFf/OIX3a4nWkqudsbtis5fBBERiU9PP/00N998Mzt37mTHjh3s3r2bMWPGEIlEWL9+Pc3NzVRVVfHXv/614zVZWVkdLeS5c+fy5ptvUlZWRjgcZtGiRZx33nmAWynt+uuv53Of+xyXXXYZoVCo4z3OPvtsnn/+efLz8wkGg+Tn51NVVcXy5ctPuIra17/+dR555JGo12LvLgV6u0B7oGu2OBGReLdo0SKuvvrqLtuuueYaFi9ezHXXXcfUqVO57rrrOtYiB7j99tuZN28eF1xwAUOHDuX+++/nggsuYMaMGcyePZsFCxZ0PLfzsqydTZs2jbKyMs4888wu23JycigsLATgu9/9Ls8999wRNRcWFnL11VfT3NzcK/vgcFo+1bP8iX/mrO2/oPmefaSG0nvlPUVE+istnxp7Wj71ZJkgAJGwWugiIpJ4FOgeoy53ERFJYAr0dhoUJyIiCUyB3q490NXlLiISlUQegxXvTmbfKtDbecfQrbrcRUROKBQKUV5erlCPAWst5eXlXU6Xi4YmlvHoGLqISPSKi4vZs2cPpaWlfpfSL4VCIYqLi7v1GgV6O6NAFxGJVnJyMmPGjPG7DOlEXe7tOrrcNShOREQSjwLdoy53ERFJZAp0j+kY5R6bOXZFRERiSYHusV6g24hGbIqISOJRoHtMwDuGbtXlLiIiiUeB7jGaKU5ERBKYAr1dQDPFiYhI4lKge9q73NEodxERSUAKdE9Hl7tVl7uIiCQeBbrHqMtdREQSmAK9nRZnERGRBKZA9xw6bU1d7iIikngU6B4TMIBa6CIikpgU6B5j3MJzmstdREQSkQLd0z4oDk0sIyIiCUiB3s4bFKcWuoiIJCIFuscE2xdnUQtdREQSjwLdY4wWZxERkcSlQPe0H0O3YbXQRUQk8ST5XUA7Y8xVwOVANvCYtfblvvz8QPtc7mqhi4hIAoppC90Y85/GmIPGmLWHbZ9njPnYGLPFGHMPgLX2GWvtbcCXgOtjWddRBXQMXUREElesu9wfB+Z13mDcwepfAJcBk4GFxpjJnZ7yHe/xPhXomCmura8/WkREpMdiGujW2reAisM2zwW2WGu3WWtbgMXAAuP8CHjRWvv+sd7TGHO7MWalMWZlaWlpr9XaMfVrxPbae4qIiPQVPwbFDQd2d7q/x9v2NeBTwLXGmC8d68XW2kettXOstXOKiop6raj25VM19auIiCSiuBkUZ619GHjYr883GhQnIiIJzI8W+l5gRKf7xd42XwWC7V3uGhQnIiKJx49AXwGMN8aMMcakADcAz/lQRxeB9i53LZ8qIiIJKNanrS0ClgMTjTF7jDFfsG4Y+R3AS8AG4Clr7bpY1hGVjkFx6nIXEZHEE9Nj6NbahcfY/hfgL7H87O5q73LXamsiIpKINPWrJxBQl7uIiCQuBbqnY3EWdbmLiEgCUqB7TMC4a7XQRUQkASnQPYGgG06gLncREUlECRnoxpj5xphHq6ure+09O+Zy16A4ERFJQAkZ6NbaJdba23NycnrtPdvXQ9dMcSIikogSMtBj4dB66Gqhi4hI4lGge9TlLiIiiUyB7tHiLCIiksgU6J72meLO3PSAz5WIiIh0nwLd0z5TnIiISCJSink6BsWJiIgkIAW6p2NxFhERkQSkQPeohS4iIolMge5RoIuISCJToHs0KE5ERBJZQqZYLOZyNwp0ERFJYAmZYrGYy73L+2u2OBERSTAJGeix1tbW6ncJIiIi3aJAP4q21ha/SxAREekWBfpRtLQ0+12CiIhItyjQj6KtpcnvEkRERLpFgX4UYR1DFxGRBKNAP4o2dbmLiEiCUaAfRVurutxFRCSxKNCPQl3uIiKSaBToRxFuVZe7iIgkFgV6Jytn/xiANgW6iIgkGAV6J6HcIQBE1OUuIiIJRoHeSSA5FYCIWugiIpJgThjoxpgfG2OyjTHJxpi/GmNKjTGf7YvijlNTr6+2BhBMSgEg3KapX0VEJLFE00K/xFpbA1wB7ADGAf8Yy6JOJFarrQWTXaBbdbmLiEiCiSbQk7zry4H/sdb2brM4jgTbu9zb1OUuIiKJJenET+F5Y8xGoBH4sjGmCOiXM68keS30iLrcRUQkwZywhW6tvQc4G5hjrW0F6oEFsS7MD8HkEAA2rC53ERFJLNEMivs7oNVaGzbGfAf4PTAs5pX5IKnjGLpa6CIikliiOYb+L9baWmPMucCngMeA/4htWf5oD/TJa37scyUiIiLdE02gh73ry4FHrbUvACmxK8k/SSmuyz3TNPpciYiISPdEE+h7jTGPANcDfzHGpEb5uoSTnNwvf6eIiMgAEE0wXwe8BFxqra0C8vH5PPRYSU5J9bsEERGRkxLNKPcGYCtwqTHmDmCQtfblmFfmg2AwmrP4RERE4k80o9zvAv4ADPIuvzfGfC3WhfnBBPrlkQQRERkAommSfgE4w1pbD2CM+RGwHPhZLAvzy+bgOMaHt/hdhoiISLdE0yQ1HBrpjnfbxKYc/5UNOpNmm+x3GSIiIt0STQv9t8C7xpg/e/evwp2L3j8lpZFqWrGRiLrgRUQkYZww0K21Dxpj3gDO9TbdYq39IKZVnYAxZj4wf9y4cb3/5klupHtzcyOhtIzef38REZEYOGYT1BiT337BLZv6e++y09vmm1gtnwpgvPncmxsbev29RUREYuV4LfRVgOXQ8XLrXRvv9tgY1uUbk5wGQGuTAl1ERBLHMQPdWjumLwuJFwGvhd7SrOlfRUQkcWjU12HaA721WS10ERFJHAr0w7QHes2BHf4WIiIi0g0K9MMkpaYDMO31W3yuREREJHpRTV5ujAkCgzs/31q7K1ZF+Snc2ux3CSIiIt12wkD35m3/HnAAiHibLTA9hnX5ZtzcebDU7ypERES6J5oW+l3ARGtteayLiQeZ2Xl+lyAiItJt0RxD3w1Ux7qQeLIi5xL2UeR3GSIiIlGLpoW+DXjDGPMC0HGA2Vr7YMyq8lkkKZ0QOpYuIiKJI5pA3+VdUrxLv2eTMwjZJr/LEBERiVo0i7PcC2CMyfTu18W6KL/Z5HTSTAuRcJhAMOh3OSIiIid0wmPoxpipxpgPgHXAOmPMKmPMlNiX5h+T4s5Fb2yo9bkSERGR6EQzKO5R4OvW2lHW2lHAN4Bfx7Ysf5kUt2xqY70CXUREEkM0gZ5hrX29/Y619g2gXy8UHkh1f15zQ78/uiAiIv1EVKPcjTH/AvzOu/9Z3Mj3fivoBXpLY43PlYiIiEQnmhb6rUAR8CfvUuRt67eCoUwAmhvVQhcRkcQQzSj3SuDOPqglasaY+cD8cePGxeT9k71Ab1Wgi4hIgjhmoBtjHrLW/oMxZglu7vYurLVXxrSy47DWLgGWzJkz57ZYvH9ymgv0tiYFuoiIJIbjtdDbj5k/0BeFxJOUtCwAwgp0ERFJEMcMdGvtKu/madban3Z+zBhzF/BmLAvzU2q6a6FHmut9rkRERCQ60QyK+9xRtn2+l+uIK2nproVeWVXlcyUiIiLROd4x9IXAjcAYY8xznR7KAipiXZifQhku0C/d81PgPn+LERERicLxjqEvA/YBhcBPOm2vBT6KZVF+S0kJ+V2CiIhItxzvGPpOYCdwVt+VEx9MIECZyacsUMQkv4sRERGJQjSLs5xpjFlhjKkzxrQYY8LGmH4/hdr+tHEYG/a7DBERkahEMyju58BCYDOQBnwR+EUsi4oHbcmZhCINfpchIiISlWgCHWvtFiBorQ1ba38LzIttWf6LJGeRbhXoIiKSGKJZnKXBGJMCfGiM+TFuoFxUPwQSmU3JIJ1G9lY1Mjw3ze9yREREjiuaYL4ZCAJ3APXACOCaWBYVD+pJJ8M084n7X/W7FBERkROKZnGWnd7NRuDe2JYTP5oD6QDkUetzJSIiIid2vIll1nCURVnaWWunx6SiOBEKu4H8j6f8CDe/joiISPw6Xgv9Cu/6q951+2Itn+U4Qd9fFGe4P3FaYIe/hYiIiEThmMfQrbU7ve72i621/2StXeNdvglc0ncl+mPM6ZcBEMH4XImIiMiJRTMozhhjzul05+woX5fYJl7GgYyJvB8ZTyTS7zskREQkwUVz2toXgP80xuQABqgEbo1pVXGiIX04ObUfU9fSRnYo2e9yREREjimaUe6rgBleoGOtrY55VXHChvLINXVUN7Qq0EVEJK4db5T7Z621vzfGfP2w7QBYax+McW3+S88nhzoONLQwIj/d72pERESO6Xgt9AzvOqsvColHSRn5pJgwtbXVQK7f5YiIiBzT8ZZPfcS7HjCTyRwuOasQgMaaMmCUv8WIiIgcx/G63B8+3guttXf2fjnRMcbMB+aPGzcupp+Tml0AQEXp/ph+joiISE8d7/SzVSe4+MZau8Rae3tOTk5MPycjpwiA599dF9PPERER6anjdbk/0ZeFxKNUr8t99iBNLiMiIvHthKetGWOKgG8Ck4FQ+3Zr7YUxrCs+pOUBEGodMGfqiYhIgopmxrc/ABuAMbjV1nYAK2JYU/zwAr2q/ADrS2p8LkZEROTYogn0AmvtY0CrtfZNa+2tQP9vnQMkh6gN5lJsSvn0w3/zuxoREZFjimbq11bvep8x5nKgBMiPXUnxpSpYQKFR61xEROJbNIH+A2/a128APwOygbtjWlUcqU/KpcDoGLqIiMS3aAL9XW/+9mrgghjXE3fScocQqtsDQGs4QnKw/y80JyIiiSeadHrbGPOyMeYLxpi8mFcUZ0aOGMGQpFoA/u5Xy32uRkRE5OhOGOjW2gnAd4ApwCpjzPPGmM/GvLI4YZLTCEUaeCblX/hwd5Xf5YiIiBxVVP3H1tr3rLVfB+YCFcDAmXSmyR0/Py2w1edCREREju2EgW6MyTbGfM4Y8yKwDNiHC/aBYdDkjpvZoWiGHIiIiPS9aBJqNfAMcJ+1duAdRJ7zBXj3V9TW1VJT3UZTa5hQctDvqkRERLqIpst9rLX27gEZ5gCBAJw6n/SWcsCy+UCd3xWJiIgcIZpBcbYvColrmYMJ2jaGUsH8ny/1uxoREZEj6KTqaKS5ifGWh77mcyEiIiJHp0CPRuG4LncfenWTT4WIiIgcXTSj3CcYY/5qjFnr3Z9ujPlO7EuLI8NnA1BDBgAPvbrZz2pERESOEE0L/dfAt/AWabHWfgTcEMui4tKcL2ACbnR7MGB8LkZERKSraAI93Vr73mHb2mJRTFzLGkpWpIYUWslM1fnoIiISX6IJ9DJjzCmABTDGXIubXGZgyRoMwHfOK6C6sZX65oH3m0ZEROJXNIH+VeARYJIxZi/wD8CXYlpVPMoaCsDM1vcBeHXDAT+rERER6SKaQN9prf0UUARMstaea63dGeO64k96AQDT3v8uAHct/tDPakRERLqIJtC3G2MeBc4EBu40afljj9i0ZHWJD4WIiIgcKZpAnwS8iut6326M+bkx5tzYlhWH0nJh8lWQnI43nICvLfrA35pEREQ80Uz92mCtfcpa+xlgJpANvBnzyuJR9R5obeBPGT8G4JSiDJ8LEhERcaKaKc4Yc54x5pfAKiAEXBfTquJV9jAAZkbWANDYEvazGhERkQ7RzBS3Azey/W/ANGvtddbaP8a6sLh0/rcAMMNnc/enJlBS3cRL6/ZTWd/ic2EiIjLQRdNCn26tvdpau8haWx/ziqJgjJlvjHm0urq6bz948GSYsRBqSijMSgHg//xuFbf/bmXf1iEiInKYYwa6MeafvJs/MMY8fPilj+o7KmvtEmvt7Tk5OX3/4XljoGYvF43L7ti0Ykclv35rW9/XIiIi4jneHKYbvOtVfVFIwig4BYAhbV0ny/u3v2zgtk8eeWqbiIhIXzhmoFtrl3jXT7RvM8YEgExrbU0f1Baf8se46+fv5o4LHubnr2/peOhbf1rDJ8YX8ulpQ30qTkREBqpoBsX9tzEm2xiTAawF1htj/jH2pcWpAm9t9N3v8I1LJrDxX+dx5Qw3+n3Re7v4yh/e97E4EREZqKIZFDfZa5FfBbwIjAFujmlV8SyUA8WnA2BqSgglBynKSu3ylDk/eJWd5XExflBERAaIaAI92RiTjAv056y1rbRPlTZQ7Vnhrn//GQDuvGh8l4fL6pp5csXuvq5KREQGsGgC/RFgB5ABvGWMGQUM3GPoAKff5q69Fdhy0pIZnpvW5Sm/fGMrY7/1Aj94fj2t4UhfVygiIgOMsbb7jW1jTJK11vcFwefMmWNXrvThHPC2ZvjFGRDKhv/zFgDldc0crG1m5Y4K/uXZdV2efueF4/j6JRP7vk4REelXjDGrrLVzjvZYNIPi7vIGxRljzGPGmPeBC3u9ykSSlArjL4HybeD9ICrITOXUodncfNZotv/w012e/vBrW/jt29vZU9ngR7UiIjIARNPlfqs3KO4SIA83IO7+mFaVCApOgZZaqN13xEPGGH587XTOm1DUse3eJes590evM/qeF3h3WzlLN5dxMr0jIiIiR3O8iWXaGe/608DvrLXrjDHmeC8YEIomuetfnAlfWwWZRV0evm7OCK6bM4LqxlZm3Ptyl8euf/SdjtvjBmVy1WnDGD84ixnFuQzJCcW8dBER6X9OeAzdGPNbYDjudLUZQBB4w1o7O/blHZ9vx9ABIhG4L8/dHv0J+Pzzx3zqX9bs494l6zhQ0xz1208akkXAGL74iTFcPn0oqUnBnlYsIiIJ7njH0KMJ9ABwGrDNWltljCkAhltrP+r9UrvH10AH+P+KXbc7wD273DnqJ7C1tI7fLd/JC2v2UVobfcA/9rk5XHTq4JOtVERE+oGeBroBbgLGWmvvM8aMBIZYa9/r/VK7x/dAX/ZzePnb7nbxXPjiK916eU1TK9bCyh0VjCrIYPOBWh5bup2VOyuP+vykgOGXN83ikilDelq5iIgkoJ4G+n8AEeBCa+2pxpg84GVr7em9X2r3+B7okQg8+xVYvcjd/37vLecaiVgCAUNrOMK/vbCBx5ft6PL44tvPZExhBgUZKSQFA7SFIyQFoxnjKCIiiaqngf6+tXaWMeYDa+1Mb9tqa+2MGNTaLb4HOkBbCzzyCXdu+l0fxuxjGlra+MZTq3lx7f7jPm/GiFzuvHAcZ44tICM1mjGPIiKSKHp0HjrQaowJ4k33aowpwrXYBSApBU69Eiq3w4v3wL7VMfmY9JQk/uOzs9n4r/OYOjz7mM9bvbuKLzyxkinfe4nR97zAi2v2sbeqkUhEp8iJiPRn0bTQbwKuB2YBTwDXAt+x1v5P7Ms7vrhooQOsfw6e6rReTS92vR/PgZomahpb+WB3FdmhJFKTgjz74V6e+bDkqM//P58cy7ypQ5hRnEsgoDMPRUQSzUl3uXsj3M8EKoCLcOek/9VauyEWhXZX3AS6tXBv7qH739oLqZn+1YM7Bv/mplJueXzFEY8FA4Znv3oOU4efeFS+iIjEj54eQ+84dh5v4ibQAR6aBlW7Dt2/ez3kDPevnk6stWzcX8u/vbCBpVvKjvvcKcOyueH0Edx4xiiCasWLiMSVngb6A8By4E82zuYqjatAr9gGJR/A07e6+5/8J7jw2/7WdBTWWp7/aB9fW/TBCZ87tiiDa2YVc+WMYRTnpaEJAkVE/NXTQK/FLZ3aBjThut2ttfbYI7P6SFwFervvd+rG7qNj6SerqTXM/uomyuubaWmzTB6WzbbSOu5/cSPvbq846mt+cNVUFpw2jKxQch9XKyIiPQr0eBaXgV69B/59yqH7d6+DnGL/6jlJreEIf9tcymNLt/Pe9gpaw12/J1mpSSQnBRhVkM6UYdlcMX0Yp4/OJxgw1Da1sr6khjc2lbL1YB0t4QjDc9PYX93E0NwQm/bXceVpw9heVs+cUXnMHZNPKDlIekpQvQAiIsfR0xb6rKNsrgZ2+r0melwGOkDlTvjp9EP3/+9myBzkXz29oDUcoaSqkf9ZuYc/vLuTyobWXv+MjJQg04pzuGDiIGaPymPWyDyNxhcR6aSngf4O7pS1Nd6macBaIAf4srX25WO9NtbiNtAB/uMcOLDW3Z7zBbjiQX/r6WVVDS2s2lnJ5GHZbD5Qx9ItZTz61rYuz7l8+lDOn1DEsNw0ahpbCQYMw3LTWLqljBnFuQVkuxMAACAASURBVDy9ag9bDtbSErZUN7QQsbC/pqnj9cNyQswYkcvU4TksnDuS/IyUvv4zRUTiSk8D/U/Av1hr13n3JwP3Af+EGyh3Wi/XG7W4DvTqvfC3n8DKx9z9fymHYP+fuW3LwTpGFaSTFDAn1X0ejlje217B6x8f5INdlazYcWhe+6nDs7l82jA+M2s4g7O1zKyIDDw9DfS11tqpR9tmjPlQgX4CD06Bmj3u9s3PwCkX+FtPgqluaGXpljJW7qxgyep9lNU1EzDwyQlFnDGmgLlj8pk8NJtQckDH30Wk3+tpoD+Jm1hmsbfpeqAQuBlY6uciLQkR6OE2+NeCQ/eveAjm3OJfPQnMWsvmg3X85m/beG1jKWV1h5afzQol8empQ5k/YxjnjCtQuItIv9TTQE8DvgKc6216G/gl7hS2dGttXS/W2i0JEegAq56AJXceuj/nVrjkB5CS4V9NCa4tHOG97RW8s72C3RUNvLxuP2FraWqNkJuezDmnFDI0J8TMkXl8YkIh2TrNTkT6gR6ftmaMSQEm4hZo+dha2/tDnE9CwgQ6uElnlvwD7Ou0ItvVj8CMG/yrqZ9pbAnz/EclLF6xm80HaqlpOnQSxuiCdM6fOIi/m1PM5KHZasGLSELqaQv9fNyiLDtwk8qMAD5nrX2rd8vsvoQK9HYvfhPe/dWh+yYAX14Ogyb5V1M/VdXQwrKt5azcUcmKHRWsLanGWhiem8YZY/KZNDSL3RWNDMkJMWukOx9e092KSDzraaCvAm601n7s3Z8ALLLWzu71SrspIQMdYMur8Ptrum4bPgeu+TXkj/WnpgHgYE0Tz3y4lxfW7OejPVUc/tUflhNi0tBspg7L5hMTipit8+BFJM70NNA/stZOP9E2PyRsoANEIm7J1Y3Pd90+YR5c/4cBcYqbn8IRy5aDdQzJCdHcGuad7RU8uWIXb28p73hObnoyM0fkMmd0PudNKGLKMHXVi4i/ehrovwXCwO+9TTcBQWvtrb1a5UlI6EBvF26D1/4V3n6o6/ZbX4IRZ4ACpE/VNrXy4e4qDtQ089rGA7yzrYKK+hYAkgKGUQXpzB1TwLypQ5g9Ko/MVP3wEpG+09NATwW+yqFR7n8DfmmtbT72q/pGvwj0zpb9HF4+bIW2r7wDg071px7BWsu2snre+LiU93dV8tqGgzS2hjsenzEil0+OL2RUQQZThmUzcXCWuulFJGZOOtCNMUFgnbU2Lkds9btABzca/tHzu2675jGYdq0v5ciR6prbeHtLGcu3lvP6xwfZWd7Q5fFpw3M4bUQuc0bncdYpBQzK0qx2ItI7etpCfxb4mrV2VyyK64l+Gejtdr0Df/4SVG539z/5j3Dhd/ytSY6qrK6Z9SU17Cyv58Pd1awrqWbzwTrCEff/Vn5GChMHZzFuUCaDs1OZPSqf6cU5ZKi7XkS6qaeB/hYwE3gPqG/fbq29sjeLPBn9OtDbbXwBFt/obp91B1x8HwSC/tYkJ9TQ0saKHZW8vaWMAzVNrNpZyZ7Kxo7HQ8kBzhxbwMwRecwcmcvsUXkKeBE5oZ4G+nlH226tfbMXauuRARHoAFW74aFO0+l/YxNkDfavHjlp+6ub+GBXJW9uKuWNj0u7rC43Ij+N6cNzmTUqj9NH5zEqP4OcdM1wJyKHnFSgG2NCwJeAcbilUx/ze/3zww2YQAeoL4f/1+kc9S+8AiPm+leP9IqDtU2s3FHJh7ur+Hh/LR/tqeqy1vyognTOGVfIqUOyOG/CIEYWpPtYrYj47WQD/UmgFTeq/TJgp7X2rphV2Q3GmPnA/HHjxt22efNmv8vpO+FW+NfCQ/fTC+Cmp2HYTJ3e1o9sL6tnzd5qNh+oZdnWctbsqaYlHAFgTGEGM0fmMnNkHpOHZjOuKFOteJEB5GQDfY21dpp3Owl4z1o7K3Zldt+AaqG3sxZe+ja884uu21Nz3Exz4y9RuPcz1lq2l9Xz1qZS3txUyge7q6jq1IofU5jB+EGZnDIok3FFmYwtyqAgI5XivDSdQifSz5xsoL/fOcAPvx8PBmSgt6svh2e/CptePPKxf1gDuSP7vibpE+3nxq/eXcWO8gY27qth/b6aLoPuwC0pe+rQbGaPyuPccYVMHZaj1rxIgjvZQA9zaFS7AdKABu+2tdZmx6DWbhnQgd6urQXW/Qne+zXs7bQvgqnwleVQcIp/tUmfsdZS19zGttJ6DtQ0UVbXwpq91by/s5LNB2uJWAgYmDM6n0lDshg/KJPxg7OYNCSL3PQUv8sXkSj1ePnUeKVAP0wk4qaQ/eu9h7bNvgUu/4lOdRvAaptaWbWzkve2V/C/a/ezrazj7FOMganDcjhnXCFjizL45PgihuRoIhyReKVAH2jaWmD1IvjLP0LYm6F31ufgsh9Dsv6xHujCEcu20jp2ljewfl8NSzeX8f6uStq8iXCK89KYXpzD7FH5nDYih/GDs8gOqateJB4o0Aeq5jp4835Y9rND2xb8EmbcoBa7dNHYEmZ7WT2vf3yQtXurWbO3uuOYfDBgmFGcw+xReeRnpDJhcCaThmYzLCek1edE+pgCfaBrrITFn4WdS939vDHw1fcgScdO5dgO1DSxencVq/dUsXxrOWv2VtMaPvTvRWFmCtOLc5k5IpfTRuYyIi+dobkhUpP0Y1EkVhTo4hxYD7+5CFobIC0fzvsmnP4FCKo7VU4sErHUNrexdm81W0vrWL27mtV7qthysK7jOUkBw6ShWZxSlMnQnDTGD8pkdGEGOWlJBIwhPyOFnLRktexFTpICXbp67Qfw1v87dH/YLPjk/4VTLtIxdum20tpmPt5fy+7KBnaU1bN6TxVbS+sprT36Csvt68oXZaWSk5ZMcZ67PTw3jVOHZjO2MEPnz4scgwJdjlS6Cf50G+z7sOv2oafB5Q9C8Wx/6pJ+o6GljU0H6jhY00RpXTNBY6hqbKWkqpH91U3sqmigrrmNsrpmmlojHa8LBlxLfkh2iCE5IYrz0hiVn86EwVmcMiiTwsxUggp8GaAU6HJsrY1uRbePnoSKbVC+xW0fOgNu+iNkFvlbn/R71lrqW8JsL61nw/4athyso7K+hdK6ZvZVNbGzor5L4KcEA2SnJVOYmcKYwgyK89IYnpvG4OwQg7JD5KQlMbogg6RgwMe/SiQ2FOgSHWth97uw6AY3kA6gaBJc/wcoHOdvbTJghSOWg7VNbCutZ8vBOkqqGimvb+FgbTN7KhvYW9lIc1uky2tSkwJMGpLF5GHZTBuey8yRuYzMT9cStZLwFOjSfRuWwJOfPXQ/ZyQs+BmMPd+vikSOylpLaV0zWw7UUVbfQktbpGM63PX7arrMe1+QkcKEwVmMLsygKDOFUwZlMrYwkzFFGWQq7CUBKNDl5ITb3LSyS+5yI+MBhs+GKx6CodP9rU0kCtZadpQ3sGZvNbsrGthV3sDGA7XsKq+nurEVby4dAgaGZIeYOSqP4tw0xhZlMDw3nfyMFE4ZlKFT8SRuKNCl53Yug7cegK1/dfdTc+DqX8GgSZA/9vivFYlDzW1htpXWs6uigQ37athaWs/7OyspqW6k8z+LxsCwnDRmj8pj1shcphXnMnloNmkpCnnpewp06T0V2+GPX+y6EMzwOXD+PTD+Yv/qEuklreEI+6ub2FPZyL7qRnaU1bO1tJ73dlR0ORWvKMstUZsSDFCQmUJqUpCirFQKMlLIDCWRkZJEbnoySYEAxrhVraoaW2kNR2hujdDcFqaxNUxuegoZKUkcqGmivrmN5KQAycEAI/LSGJ6XxpjCDMIRS3pKkkb3iwJdYqBiuzuXvXQj7F3ltmUUwY1PQcE4CLdASgYkhbQ+u/QL1lr21zSxZk81a/dWs7OigdLaZmqb2jhY20Q44hbCOXyAXm9KDhpaw5a0ZO/HQ2YKycEAw3JCFGWlMigrRGYoiYIMtz0vI4VhOSGy05IJJatHoT9QoEtsNdfC0ofgbw8c+VggGSKtcO7XYfQ5bhKb9Py+r1GkD1hraWgJU1HfQl1zGzWNrbSGLcGAwRhISQqQEgyQkhQglBQkJSlAS1uEyoYWMlKTyEt3wdvSFmFnRQMlVY1sPVhHxEJDaxt1TW20tEXYXdnAoKwQJVWNtIbdczsP/jtcwEBhZiqZoSRG5KWTkRqkIMPdH57rTvsbmhtidEEGAMnBgHoD4pQCXfrG/rWu1b7xBRfixzPvRzBkqgv4lPS+qU+kH4tEbMfx/8qGFraW1tHQEqa01p3Pb7HUNrWxu7KB3RWNNLWGj+hNMMadvZqWHGTikCxG5qdjgXFFmYwuTKc4L53huWkUZWlyH78o0MVfbc3w0VOw/U3YtxrKNnV9/OyvQeFEmHYtJKf5U6PIAGOtm5u/qr6Vjw/UUt/cxvayeqoaWjDGsGFfDetLaqhtbjvite2z+eWnpzA4J0RhRgp1zW2MyE9nbFEG4wdlMSTbHQbQ4MHepUCX+BIJw54V8PbDsGMpNFe77VlD4dT5MPsWGDzZ3xpFBGstLeEI1sKuigb2VjWy1xssuL+6mbK6ZqoaWiirayEYMByoaTqi1Z8dSmJwdsjN01+UwYi8dEbkpzMiP43BWSHN299NCnSJbxXb4d1HoOQD2P2O21Z8OnziGzD+UghoCk+RRBCJWPZWNbK9rJ591Y2U1bWwv7qJfdWNbNhXe8QpgeCW4Z0wOMvN2V+QQV56CkNzQuSmJ5MVSqI4L10D+jpRoEviqNwJ7z0KH/wOmqohczCc/kWYcytkFPpdnYj0QHNbmL2VjeyubGR3RQM7y+upqG9la2kduysaKK9vOeI1xpv0Jy89hVBygKRAgKSgITuUTHFeGiML0hmcHWJ4bhqFmakUZqb063n8FeiSeFqbYO3TsPTf3YIxJggzboBz7oKiiX5XJyK9zFpLdWMrDS1h9tc0UVnfQk1TKzvKGthT2UhVQwvNbRHaIhFaw5aaxlZ2VzZ0WbgH3A+AosxUhuaEGJrjRu8XZro5A4rz0hmUlUpRVmrCtvoV6JK4rIUDa2Hlb12rPRKGc+6EM78CmYP8rk5EfBSJWMrqmtlf00RJVRNldc0crGlif00T+6q9S1Uj9S3hI15bkJHCiHw3an9Qdiq5aSkMyk5lSE6IIdkhhuaEyElLxsTZPBoKdOkfKrbBq/fC+mfc+e2z/h5mf17zyovIcTW0tLG7opGSqkZK65o7juvvqmigpKqJ0tpm6o4ymj+UHGBIdqgj5IfkpDEkO9Vd57jQL8zs21P4FOjSv+xeAe8/Dh8uAhuG7GKYdo071p470u/qRCQBtYYjHKxtZr83gn9fdSMHvJZ++/XBmmZawl27+IMBw6CsVAZnu9AvzEohJy2ZQVkhBmWlMrYok4lDsnqtTgW69E8V22Ddn91SryUfQFIazPuha7XHWTeZiCS+SMRS0eBG7u+vdl37na8P1Lhu/5qmNsLeUn4XTx7Mr//+qPl7UhTo0v8d3Aj/ew9sex2KToXTboTJCyBvlN+VicgAE4lYyutbOFjbRFIgoBZ6NBTo0kUkAh/8l5tXvnK7Gxk//hLXFX/KhTqfXUQS3vECPamvixGJmUDAdbfP/jyUb4XlP4fVi2HTi5A9HE67yYV71mC/KxUR6XVqoUv/1tLgzmd//3ew5z13nH3Wza5LfthMv6sTEekWdbmLABzcAG/+CNY9A1gYPNWt1z5oEpx6peue12A6EYljCnSRzmr3w9o/wkdPgo24ZV+xMGiKW/Ft1ucgo8DvKkVEjqBAFzmecKsL+KX/DqUb3bYx58Hc21yrPSnV3/pERDwaFCdyPMFkN0/8tL9z57XvWAqbXoInPwvBVBh/sQv3MeepS15E4pYCXaRdIOi63KddC23NsOVVN2nNx3+Bjc9DarY71n7ajTDqbIW7iMQVBbrI0SSlwqTL3aW1Cdb8D2z6X3f94e8h/xSYcjXMvAnyx/pdrYiIjqGLdEtLg+uW//APsPNtwMDoc+HU+TDuU1Bwit8Vikg/pkFxIrFQsw9W/MaNlq/e7bYVjHPd8qdeAcNmqVteRHqVAl0klqyFqp3w8Yuw/lnY/a47HS5nBEy9xs0pP2ymwl1EekyBLtKXakpg88vw0VOwcxlgIWsYjD4HRp7lWvCZRX5XKSIJSIEu4pfqvbD1NdjyCmx+BVob3KIx4z7lTpObeBmkZvpdpYgkCJ2HLuKXnOFu7vhZN0O4Dco2uWPuq34Lm19yc8uPvximX+dWhEvJ8LtiEUlQaqGL+CHc5haLWfM0rF50qOU+4gwY/ymYdAUUTtBxdxHpQl3uIvEs3AY7/uYmstn0EpRvdttzR7kBdVOuhiHTIagONZGBToEukkgqd7hg3/yKO/5uw5AxCCbOg+k3uFa8wl1kQFKgiySq2v1u+tkdS71BdfXudLhJV7hZ7EbM1eIxIgOIAl2kP2iudS33j56EbW9CuBmS07vOVJc9zO8qRSSGNMpdpD9IzTq0eExTNWx9Hba97nXPv+yekz8WTrnITWQz+lzIG+VvzSLSZ9RCF0l01kLJ+7D5Vdi1HHa9A22N7rGiU91sddOuhfwx/tYpIj2mLneRgSTcBqUbXav94xfd6XEAxae7pV/HnOda8jolTiThKNBFBrLqPW7Z14+egoPr3bbcUW4im/EXu+vkNH9rFJGoKNBFxHXN7/8Idr/nRszvWg7NNZCSCRM/DeMvgUmf1mx1InFMg+JExHWxD53hLnNvg3ArbH8L1j/jTo1b85SbinbcRS7gT7kQsof6XbWIREktdBGBSNi12Nc9Ax//BWr2AsatDjfuwkMj53XcXcRX6nIXkei1d81vegnWPwcH1rjt2cNh1DlucN34T7mBdSLSpxToInLyag+45V83veQmtGmudtsLJ7rlXyd+GornQCDob50iA4ACXUR6h7VQvsWdErf5ZTclbaQNQjlQPBdGnuHmmh8+W4PrRGJAg+JEpHcYA4Xj3eWsr7oZ6za/4laL2/UuvPaKe14gyQX7qLNd0A+bCekFEAj4W79IP6ZAF5GTF8o5NB0tQGMl7FkJO9+GLX+Ftx4AOvUCFk2CIdNg6Gkw7DQYPAXS8nwpXaS/UZe7iMROc52binbfh27luPItULbJG0XvyR7ugn3wFBj9CTfwLjnkX80icUzH0EUkvtQegL2rYO9KKN/qgr50ozsen5QGRRNh+CwYPNVdhp2mZWJFUKCLSCJoqnat+a2vQ+kG2Pu+m8kODoX8qHPc8fihM6BgnI7Jy4CjQXEiEv9COTDhUncBiESgYhscXOemq923Glb8GsIt7vFgqhucVzTJnTY36mw3R31arn9/g4iPFOgiEp8CASgc5y6TF7ht4VYo/diF+4F1UPIB7FwGa58+9LqckTDyTK8lP91d6xQ6GQDiJtCNMWOBbwM51tpr/a5HROJQMBmGTHWXziq2uXPi6w7A/rWw/U03Nz2ACcCgKVA82x2PLz7dnUKXXgAp6X3/N4jESEwD3Rjzn8AVwEFr7dRO2+cBPwWCwG+stfdba7cBXzDGPH30dxMROYb8sV2norUWave5cN+7EvasgHV/hlWPH3qOCbjZ7rKGuJAfPBkGTYacYrXoJSHFuoX+OPBz4L/aNxhjgsAvgIuBPcAKY8xz1tr1Ma5FRAYKYyB7mLtMuMRti0SgaqcbeFe9xx2L37sSKra7Fr2NeK8NQP4pbuBdRqH7oTB0BgyZrha9xLWYBrq19i1jzOjDNs8Ftngtcowxi4EFgAJdRGInEID8Me5yuMZKKN0E1buhbLO3bvy7bntLnXuOCbqR9iPmuvPlB02GjCL3AyAQcD8Y0vOjX5GurRnqS93gvswi9yOjahe0NUFTjRskOHgKZA46+mt1Gp8cxo9j6MOB3Z3u7wHOMMYUAP8GzDTGfMta+8OjvdgYcztwO8DIkSNjXauIDARpeW4ees7our29677kA3ca3b7VsPawrvsu75PvfjC0tbiQD3UacZ9TDDUl7nh/QwW01B56LJAMkdZj15ec4X44mMChxXEyitx1eqG7zhvl/o7MwW7kP7gFc0zA9VTkjYFgSvd+dEhCiZtBcdbacuBLUTzvUeBRcOehx7ouERnAOnfdT7rcbQu3uVPpyja7FnZLnWudJ4fcBDnl29xc9hjXwsdCUshNhdtcC4MmuS78UDakZEH2UKjZB0kpLoiT01zwmoAL/30fuR8Ugye7wwKBJPfDIdzsPqtim/uMTRsgZ4Sbke94Pw5Sc9yPgsFTXf3DZ7tegeLTXa+DMd7sfVM1Y1+C8SPQ9wIjOt0v9raJiMS/YJIL5KEz/K7k6MKtULnDnd6XXnBovED5Fhf0LfVuvv1wK7Q0wLY3oLUBtr525HslpUFbowv73FHuh8bwWe7QQ+5Idz9rqFr8ccKPQF8BjDfGjMEF+Q3AjT7UISLS/wSTD62IF61I2IV+6UYo2+J+tBzc4H4UbH3NPb7lFTeb3+r/7vrapJA7ZJCW6z5z8BQX/tnDYNCpbl6AYNx0BvdrsT5tbRFwPlBojNkDfM9a+5gx5g7gJdxpa/9prV0XyzpEROQ4AkEIpLlJeIbNdNumXN31Oda6YK/zFtkp3woH1rpT/NoHFDaUw7uPHJrND1zY5412U/UWjnOHFbKGuLkBMopc6z4Q7LM/tT/TXO4iItJ7wq1ufEFzLZRv9sLfG1tQvsUd++8sKc216odOd4cIMge74/fpBW5cQc4IBX4nmstdRET6RjDZDeAD78yBTsKtbvncuoMu9A+sdQPyakpgzR8PjeDv8n4pbgKgogmQMcj9UEhKhZzhLvwHTXZd+8lpsf/b4pwCXURE+kYw2bXGB0+BUy7o+pi17qyAtiY3w1/ldre9eg8cXO8W6KkpARt25+g3dQp/E3ATABVNcpfCCe70wcLx7lS+AUKBLiIi/jPGnSMPbkDd0bS1uOPzqZlutH7tftfKP7DOXUo/ho9fdKHfLnOwG5VfONFdt4d+RmG/G52vQBcRkcSQlOIu4AbjFZziLu2r8YGbRa9yh5vSt2yTC/nSjbB6cdfJfNLyvYBvD3nvOoFPw1Ogi4hI/5GUeiioJ847tN1a12VfutEL+o0u7Nc9A01Vh56Xmu267DuHfNFEb3BeoO//nm5IyFHuxpj5wPxx48bdtnnzZr/LERGRRGWtmzGvvSXf+br+4KHnJae7Y/Kdg75wojslrw/Psz/eKPeEDPR2Om1NRERipqGia2u+/bqm0+SmwRQoGN+pNT/BjbzPH+sGAfYynbYmIiLSXen5MPJMd+msqcaddle68VDI710F6/4MeI1kE3St91OvgIvv65NyFegiIiLdEcqG4tnu0llLw6EWfdlmN7FOcnqflaVAFxER6Q0p6TDsNHfxQXwP2RMREZGoKNBFRET6AQW6iIhIP6BAFxER6QcU6CIiIv2AAl1ERKQfUKCLiIj0AwkZ6MaY+caYR6urq0/8ZBERkQEgIQPdWrvEWnt7Tk6O36WIiIjEhYQMdBEREelKgS4iItIPKNBFRET6AQW6iIhIP2CstX7XcNKMMaXAzl58y0KgrBffbyDSPuw57cOe0z7sHdqPPdfb+3CUtbboaA8kdKD3NmPMSmvtHL/rSGTahz2nfdhz2oe9Q/ux5/pyH6rLXUREpB9QoIuIiPQDCvSuHvW7gH5A+7DntA97Tvuwd2g/9lyf7UMdQxcREekH1EIXERHpBxTogDFmnjHmY2PMFmPMPX7XE0+MMSOMMa8bY9YbY9YZY+7ytucbY14xxmz2rvO87cYY87C3Lz8yxszq9F6f856/2RjzOb/+Jr8YY4LGmA+MMc9798cYY9719tWTxpgUb3uqd3+L9/joTu/xLW/7x8aYS/35S/xjjMk1xjxtjNlojNlgjDlL38XuMcbc7f2/vNYYs8gYE9J38fiMMf9pjDlojFnbaVuvfe+MMbONMWu81zxsjDEnVai1dkBfgCCwFRgLpACrgcl+1xUvF2AoMMu7nQVsAiYDPwbu8bbfA/zIu/1p4EXAAGcC73rb84Ft3nWedzvP77+vj/fl14H/Bp737j8F3ODd/hXwZe/2V4BfebdvAJ70bk/2vp+pwBjvexv0++/q4334BPBF73YKkKvvYrf233BgO5DW6Tv4eX0XT7jfPgnMAtZ22tZr3zvgPe+5xnvtZSdTp1roMBfYYq3dZq1tARYDC3yuKW5Ya/dZa9/3btcCG3D/KCzA/eOKd32Vd3sB8F/WeQfINcYMBS4FXrHWVlhrK4FXgHl9+Kf4yhhTDFwO/Ma7b4ALgae9pxy+D9v37dPARd7zFwCLrbXN1trtwBbc93dAMMbk4P5hfQzAWttira1C38XuSgLSjDFJQDqwD30Xj8ta+xZQcdjmXvneeY9lW2vfsS7d/6vTe3WLAt2F0+5O9/d42+QwXnfbTOBdYLC1dp/30H5gsHf7WPtzoO/nh4B/AiLe/QKgylrb5t3vvD869pX3eLX3/IG+D8cApcBvvUMXvzHGZKDvYtSstXuBB4BduCCvBlah7+LJ6K3v3XDv9uHbu02BLlExxmQCfwT+wVpb0/kx71elTpc4BmPMFcBBa+0qv2tJcEm4bs//sNbOBOpxXZ0d9F08Pu847wLcj6NhQAYDq3ciJuLle6dAh73AiE73i71t4jHGJOPC/A/W2j95mw94XUV41we97cfanwN5P58DXGmM2YE7pHMh8FNcV1yS95zO+6NjX3mP5wDlDOx9CK7lssda+653/2lcwOu7GL1PAduttaXW2lbgT7jvp76L3ddb37u93u3Dt3ebAh1WAOO9UZ4puIEfz/lcU9zwjpc9Bmyw1j7Y6aHngPZRmp8Dnu20/e+9kZ5nAtVet9RLwCXGmDyvlXCJt63fs9Z+y1pbbK0djft+vWatvQl4HbjWe9rh+7B9317rPd9622/wRh6PAcbjBtMMCNba/cBuY8xEb9NFwHr0XeyOXcCZxph07//t9n2o72L3Ha6QGgAABNlJREFU9cr3znusxhhzpvff5O87vVf3+D16MB4uuFGJm3AjNb/tdz3xdAHOxXUlfQR86F0+jTuO9ldgM/AqkO893wC/8PblGmBOp/e6FTd4Zgtwi99/m0/783wOjXIfi/tHcAvwP0Cqtz3k3d/iPT620+u/7e3bjznJkbCJfAFOA1Z638dncKOF9V3s3j68F9gIrAV+hxupru/i8ffZItyYg9b/v737C62yjuM4/v64Bhn5B3GFBCmFtHZRS9PwoloU2D/oD+KIiuwP4k1E5E0QQXUhFBShZFHEJCkIJEF2kaGOmRou2tbCgrB2WwQSu0hv/Hbx/Z12Om7HZtnZHj4veNhznuf7POe3c8b57vnt2fdLzhQ9/V/+3AE3l/fjJLCDUvRtposrxZmZmVWAp9zNzMwqwAndzMysApzQzczMKsAJ3czMrAKc0M3MzCrACd1slpC0TdIdkh6U9OIMj+0o3bCGJd3aJK5Hpdtbk5huSffO5Pn/b5LGJS1t9TjMZhMndLPZ4xbgK+B2YHCGx94JjEXETRFx+F+Oo5usNWBmc4gTulmLSXpD0rfAGuAY8AywU9LLU8SukHSw9Fk+IOlqSd1kK8cHJI1Imt9wzN3K/uHfAA/XbV8r6Vi5qj8q6bpSLfFVoLecq3equCnGtUzSYDnmu9osgaSdkr5W9t9+pS5+vMxIjJT9qyR9LumkpC0lpqecs1/Zc/tdSed8Zkl6TNLxcq73lH3n2yT1lbGMSXr+gt4cs7mk1RV4vHjxEpDJfDvQDhxpErcPeKKsPwXsLeubgB1TxF9KdnhaSVaw+pTJSnULgUvK+l3AnqnONV1cw/O8QKmyCLQBC8r6krptA8AN5fE4kz233yIrvy0AOoBfyvYe4DRZxayNbDe5oe74pcD15TVpL9vfIUtnriZbVdbGt7jV77EXLxd7qRXjN7PWWgWMAp1kz/nprGPyKvsj8sq8mU6yGcePAJJ2A5vLvkXALkkryfK+7dOc45/EDQEfKhv57I2IkbJ9o6TNZKe0ZUAXmbxhsmfCGHB5REwAE5LOSFpc9h2PiJ/K2D8hSxHX+nZD/qlhNTCUZbCZTzbJ2AdcI2k70A/sb/IamVWCE7pZC5Xp8j6yw9JvwGW5WSPAuoj44yI+/WvAoYh4SNnrfuBC4yJiUNJtwH1An6Q3gcPAVmBNRJyS1EfOGNScKV/P1q3XHtc+mxprUzc+FrArIs65iVDSjcB6YAuwkZzRMKss/w3drIUiYiQiusnmQF3AQWB9RHRPk8yPkh3bAB4lk2YzPwArJF1bHj9St28Rk20aN9VtnyCnv88X9xdJy8mp8veBD8gZh4Vkz/LfJV0J3HOesU5lrbIT4jygF/iyYf8BYIOkK8o4lkhaXu6AnxcRe4CXynjMKs0J3azFJHUApyLiLNAZESeahD8LPFluonsceK7ZuSPiNDnF3l9uivu1bvfrwDZJw/x9tu4Q0FW7Ka5JXL0eYLTE9AJvR8QoMEz+UvExcKTZWKcxRHaf+h74Gfis4fs7QSbs/eU1+YKc2r8KGCgzHbuBGf0boNlc5G5rZjYrSeoBtkbE/a0ei9lc4Ct0MzOzCvAVupmZWQX4Ct3MzKwCnNDNzMwqwAndzMysApzQzczMKsAJ3czMrAKc0M3MzCrgTwX/Tqy41DS7AAAAAElFTkSuQmCC" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "### AutoVW which tunes both namespace interactions and learning rate\n", "Create and run an AutoVW instance which tunes both namespace interactions and learning rate." - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 8, - "metadata": { - "tags": [] - }, + "source": [ + "from flaml.tune import loguniform\n", + "''' create another AutoVW instance for tuning namespace interactions and learning rate'''\n", + "# set up the search space and init config\n", + "search_space_nilr = {'interactions': AutoVW.AUTOMATIC, 'learning_rate': loguniform(lower=2e-10, upper=1.0), 'quiet': ''}\n", + "init_config_nilr = {'interactions': set(), 'learning_rate': 0.5}\n", + "# create an AutoVW instance\n", + "autovw_nilr = AutoVW(max_live_model_num=5, search_space=search_space_nilr, init_config=init_config_nilr)\n", + "\n", + "# online learning with AutoVW\n", + "loss_list_autovw_nilr = online_learning_loop(max_iter_num, vw_examples, autovw_nilr)\n", + "print('Final progressive validation loss of autovw_nilr:', sum(loss_list_autovw_nilr)/len(loss_list_autovw_nilr))\n" + ], "outputs": [ { "output_type": "stream", @@ -324,69 +336,69 @@ ] } ], - "source": [ - "from flaml.tune import loguniform\n", - "''' create another AutoVW instance for tuning namespace interactions and learning rate'''\n", - "# set up the search space and init config\n", - "search_space_nilr = {'interactions': AutoVW.AUTOMATIC, 'learning_rate': loguniform(lower=2e-10, upper=1.0), 'quiet': ''}\n", - "init_config_nilr = {'interactions': set(), 'learning_rate': 0.5}\n", - "# create an AutoVW instance\n", - "autovw_nilr = AutoVW(max_live_model_num=5, search_space=search_space_nilr, init_config=init_config_nilr)\n", - "\n", - "# online learning with AutoVW\n", - "loss_list_autovw_nilr = online_learning_loop(max_iter_num, vw_examples, autovw_nilr)\n", - "print('Final progressive validation loss of autovw_nilr:', sum(loss_list_autovw_nilr)/len(loss_list_autovw_nilr))\n" - ] + "metadata": { + "tags": [] + } }, { "cell_type": "markdown", - "metadata": {}, "source": [ "### Online performance comparison between vanilla VW and two AutoVW instances\n", "Compare the online progressive validation loss from the vanilla VW and two AutoVW instances." - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 10, - "metadata": { - "tags": [] - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAFzCAYAAADIY/vqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdd3hc1Z3/8feZURn1LlmWZUvuveNOMSU2HQIBnA4hbLLLkpBkFzYbCMnubze7SViSDdmEkAQ2ydoQJxBq6MYGG3DBveIiW5at3stoNHN+f9yRbGPZHlsaj0b6vJ5HjzR37p35jnjwR+fcU4y1FhEREYlurkgXICIiIj2nQBcREekHFOgiIiL9gAJdRESkH1Cgi4iI9AMKdBERkX4gJtIF9ER2drYtKiqKdBkiIiLnxfr166ustTndPRfVgV5UVMS6desiXYaIiMh5YYwpOdVz6nIXERHpBxToIiIi/YACXUREpB+I6nvoIiISGp/PR2lpKW1tbZEuRULg8XgYMmQIsbGxIV+jQBcRGQBKS0tJSUmhqKgIY0yky5HTsNZSXV1NaWkpxcXFIV+nLncRkQGgra2NrKwshXkUMMaQlZV11r0pCnQRkQFCYR49zuW/lQJdRETCbuHChbzyyisnHHvkkUf46le/elav89xzz/GDH/wAgIceeogf/ehHAHzxi19k+fLlp7zuySefZMmSJSccq6qqIicnh7/85S/ccMMNXcf//d//nZEjR3Y9fv7557nuuuvOqs5IUKCLiEjYLVmyhGXLlp1wbNmyZSeF7Jlcd9113H///Wf9/jfeeCOvvfYaLS0tXceWL1/Otddey7x583jvvfe6jq9Zs4bU1FQqKioAWL16NfPmzTvr9zzfFOgiIhJ2N998My+++CLt7e0AHDhwgLKyMpYuXcrMmTOZMGEC3/3ud7vOLyoq4rvf/S7Tp09n0qRJ7Ny5E4AnnniCu++++7Tv9f3vf58LLriAiRMnctddd2GtJTU1lYsvvpjnn3++67zOPyhycnJITU3lo48+AuDw4cPcdNNNrF69GnACff78+b36+wgHjXIXERlgvvf8NraXNfTqa44fnMp3r51wyuczMzOZNWsWL7/8Mtdffz3Lli3jlltu4dvf/jaZmZn4/X4uu+wyNm/ezOTJkwHIzs5mw4YN/PznP+dHP/oRjz/+eEi13H333Tz44IMAfO5zn+OFF17g2muvZcmSJfzhD3/g1ltvpaysjN27d3PppZcCMH/+fFavXo3f72fUqFHMmTOHV155hWuuuYZNmzZxwQUX9PA3FH5qoQe9/v4fefq1n0a6DBGRfuv4bvfO1vHTTz/N9OnTmTZtGtu2bWP79u1d53/yk58EYMaMGRw4cCDk93nrrbeYPXs2kyZN4s0332Tbtm0AXH311bz77rs0NDTw9NNPc9NNN+F2uwGYN28eq1evZvXq1cydO5dZs2bx/vvv8+GHHzJ27Fg8Hk8v/RbCRy30oKc3PcJ+Vz23cE+kSxERCavTtaTD6frrr+fee+9lw4YNtLS0kJmZyY9+9CPWrl1LRkYGX/ziF0+YqhUfHw+A2+2mo6MjpPdoa2vjb//2b1m3bh2FhYU89NBDXa+ZkJDA4sWLeeaZZ1i2bBkPP/xw13Xz58/nv//7v/H7/Xz5y18mJSWFtrY2VqxYERX3z0Et9C4uDBYb6TJERPqt5ORkFi5cyB133MGSJUtoaGggKSmJtLQ0ysvLefnll3v8Hp3hnZ2dTVNT00kj35csWcLDDz9MeXk5c+fO7To+btw4ysrKeOedd5g2bRoAU6dO5Re/+EVU3D8HBfpxDAFN0RQRCaslS5awadMmlixZwpQpU5g2bRpjx47l05/+dK8EZ3p6Ol/+8peZOHEiixYtOune9xVXXEFZWRm33nrrCXO9jTHMnj2brKysruVW586dy759+6KmhW6sjd5W6cyZM21v7Yf+t49dxHZ3NSu+tK1XXk9EpC/ZsWMH48aNi3QZcha6+29mjFlvrZ3Z3flqoQcZXOpwFxGRqKVADzJGXe4iIhK9FOhBzqA4ERGR6KRADzK4CES6CBERkXOkQA9yGZe63EVEJGop0IMMRi10ERGJWgr0ICfQ1UQXEQmnZ599FmNM12Yrp/PII4+csDtad26//XZ++ctfnvQeV155Jffeey+PPPJI1/FFixZx5513dj3+5je/ecJqceBsyZqYmNi10xo4C+J093Nfo0APchlNWxMRCbelS5eyYMECli5desZzQwn0023L2rnhCkAgEKCqqqprXXc49bao2dnZ/PjHPw7l4/QpCvQgo5XiRETCqqmpiXfeeYdf//rXXSG8YsUKrrnmmq5z7r77bp544gl++tOfUlZWxsKFC1m4cCHg/DEwadIkJk6cyH333QfAZZddxs6dOzly5AgAzc3NvP7669xwww3MmzePNWvWALBt2zYmTpxISkoKtbW1eL1eduzYwfTp00+q84477uCpp56ipqYmrL+P3qbNWYJcxq176CIyMLx8Pxzd0ruvOWgSXPmD057yl7/8hcWLFzN69GiysrJYv379Kc+95557ePjhh3nrrbfIzs6mrKyM++67j/Xr15ORkcEnPvEJnn32WW644QZuuukmnn76ab72ta/x/PPPc8kll5CamkpqaioxMTEcPHiwaxe1w4cPs2bNGtLS0pg0aRJxcXE8+OCDzJw5k+uuuw5wutXvuOMOfvKTn/C9732vV39N4aQWepCzsIwh4PdHuhQRkX5p6dKl3HbbbQDcdtttIXW7d1q7di2XXHIJOTk5xMTE8JnPfIaVK1cC3W/L2unj26LOnTu363Hn2vHf//73u8K80z333MOTTz5JY2Njjz7z+aQWepAr+LdNwAZw4Y5wNSIiYXSGlnQ41NTU8Oabb7JlyxaMMfj9fowxXH/99QQCx/pHj98+NVTz5s3jyJEjbNq0idWrV59wT73zPvqWLVuYOHEihYWF/PjHPyY1NZXbb7/9lK+Znp7Opz/9aR599NGzridS1EIPMsFfRXtHe4QrERHpf5YvX87nPvc5SkpKOHDgAIcOHaK4uJhAIMD27dvxer3U1dXxxhtvdF2TkpLS1UKeNWsWb7/9NlVVVfj9fpYuXcrFF18MOD2st956K1/4whe48sor8Xg8Xa8xb948XnjhBTIzM3G73WRmZlJXV8eaNWvOuIvaN77xDX75y1+GvBd7pCnQg4xxfhV+vy/ClYiI9D9Lly7lxhtvPOHYTTfdxLJly7jllluYOHEit9xyS9de5AB33XUXixcvZuHCheTn5/ODH/yAhQsXMmXKFGbMmMH111/fde7x27Ieb9KkSVRVVTFnzpwTjqWlpZGdnQ3Agw8+yHPPPXdSzdnZ2dx44414vd5e+R2Em7ZPDfruk7fyZ7az6pNvkZ6S3SuvKSLSV2j71Oij7VPPkSu40b1PXe4iIhKFFOhBxjgD4fwa5S4iIlFIgR7UOcrd16F76CIiEn0U6EGdLfRAIDpGM4qIiBxPgR6ke+giIhLNFOhBndPWAlYtdBERiT4K9CCXBsWJiITdQNo+9ZJLLuHjU6tXrFhBWloaU6dOZezYsXzrW98K+fXORIEe5Aq20H1aWEZEJGz64/apTzzxBA899FDI51944YVs3LiRDz/8kBdeeIF333035GtPR4Ee5NKgOBGRsNL2qSdKSEhg6tSpHD58uFdeT5uzBHW20KNlzV4RkXP1Hx/8BztrztzlfTbGZo7lvln3nfYcbZ96otraWvbs2cNFF13UK6+nFnqQWugiIuHVn7ZPra6uZurUqUydOpUHH3yQX/ziF12Pt2w5/V7zq1atYsqUKRQUFLBo0SIGDRoU8u/hdNRCD+oc5d4R0D10EenfztSSDof+tn1qVlYWGzduBJx76AcOHAj5PvqFF17ICy+8wP79+5kzZw633HILU6dODf0Dn4Ja6EEuV2cLXaPcRUR6m7ZPPVlxcTH3338///Ef/9Err6dAD3IZp7PCr0AXEel1A3X71KuvvpohQ4YwZMgQPvWpT530/Fe+8hVWrlzJgQMHzvk9Omn71KBfPvttflb/PD8Z910unXVzr7ymiEhfoe1To4+2Tz1Hbpda6CIiEr0U6EFdK8VplLuIiEQhBXqQyxVcy10tdBERiUIK9KDOFrqmrYlIfxXNY6YGmnP5b6VAD+oMdKsWuoj0Qx6Ph+rqaoV6FLDWUl1dfcL0u1BoYZkgV3BQXMAGznCmiEj0GTJkCKWlpVRWVka6FAmBx+NhyJAhZ3WNAj2oay13vwbFiUj/ExsbS3FxcaTLkDBSl3uQ2xULQMAq0EVEJPoo0IM6R7lrHrqIiESjPtPlboy5AbgaSAV+ba199Xy+f0zwHrq1CnQREYk+YW2hG2N+Y4ypMMZs/djxxcaYXcaYj4wx9wNYa5+11n4Z+Apwazjr6rZWlxaWERGR6BXuLvcngMXHHzDGuIFHgSuB8cASY8z44075TvD588rduduaWugiIhKFwhro1tqVQM3HDs8CPrLW7rPWtgPLgOuN4z+Al621G071msaYu4wx64wx63pz+kXXoLiA5miKiEj0icSguALg0HGPS4PH/h64HLjZGPOVU11srX3MWjvTWjszJyen14rqnLamUe4iIhKN+sygOGvtT4GfRur9Y9yd09bU5S4iItEnEi30w0DhcY+HBI9FlKtrUJxWihMRkegTiUBfC4wyxhQbY+KA24DnIlDHCTpb6Jq2JiIi0Sjc09aWAmuAMcaYUmPMl6y1HcDdwCvADuBpa+22cNYRCpemrYmISBQL6z10a+2SUxx/CXgpnO99tjoD3WpzFhERiUJa+jUoxhUHaLc1ERGJTgr0IJdbC8uIiEj0UqAHHRsUpxa6iIhEHwV6kDu4OYtfLXQREYlCURnoxphrjTGP1dfX99prxsZ07ramFrqIiESfqAx0a+3z1tq70tLSeu01XcYJdN1DFxGRaBSVgR4ObndnoGtzFhERiT4K9KAYtzNtTV3uIiISjRToQe7gtLVV3lPu3CoiItJnKdCDYmOcaWtlsSbClYiIiJw9BXpQjCs20iWIiIicMwV6UEyMAl1ERKKXAj3I7Vagi4hI9FKgB8W4wrrxnIiISFgp0INi3Ap0ERGJXlEZ6OFY+rVztzUREZFoFJWBHo6lX48X8Gv5VxERiS5RGejh1uptiXQJIiIiZ0WB3o0Wb2OkSxARETkrCvRutLU1R7oEERGRs6JA70ZruwJdRESiiwK9G226hy4iIlFGgd4NBbqIiEQbBXo3vD4FuoiIRBcFejfa2lsjXYKIiMhZUaAfZ77XWaimvUOBLiIi0UWBfpyLht0IQLuvLcKViIiInB0F+nHiYjwAtHco0EVEJLpEZaCHY3MWgLhYJ9B9Hd5efV0REZFwi8pAD9fmLPGxCQC0K9BFRCTKRGWgh0tnoPv86nIXEZHookA/TnxcZ6C3R7gSERGRs3PGQDfG/KcxJtUYE2uMecMYU2mM+ez5KO58i49LAqDDry53ERGJLqG00D9hrW0ArgEOACOBfwhnUZFyrMtdLXQREYkuoQR6TPD71cAfrbW9O7S8D0kIdrn/b/vqCFciIiJydmLOfAovGGN2Aq3AV40xOUC/HDXm8SRHugQREZFzcsYWurX2fmAeMNNa6wOagevDXVgkJHqSIl2CiIjIOQllUNynAJ+11m+M+Q7we2Bw2CuLgMT4lEiXICIick5CuYf+gLW20RizALgc+DXwP+EtKzLi4uIjXYKIiMg5CSXQ/cHvVwOPWWtfBOLCV5KIiIicrVAC/bAx5pfArcBLxpj4EK+LSgvbc0gMBCJdhoiIyFkJJZhvAV4BFllr64BM+uk8dACPK4F2YyJdhoiIyFkJZZR7C7AXWGSMuRvItda+GvbKIiTWFU+HMbR5WyJdioiISMhCGeX+NeAPQG7w6/fGmL8Pd2FnqCks26cCxLmdgXH1TbW9/toiIiLhEkqX+5eA2dbaB621DwJzgC+Ht6zTC9f2qXAs0Jtb6nr9tUVERMIllEA3HBvpTvDnfnuTOc7tLP/a1NpvV7gVEZF+KJSlX38LvG+MeSb4+Aacuej9UlxMAngV6CIiEl3OGOjW2oeNMSuABcFDt1trPwxrVRHUueNaS1tThCsREREJ3Sm73I0xmZ1fONum/j74VRI81i95Yp313P9l60ORLUREROQsnK6Fvh6wHLtfboPfTfDn4WGsK2KsdT5mTUy/XTtHRET6oVMGurW2+HwW0ldMG30ZVC2LdBkiIiJnRc3Qj5k8ak6kSxARETlrCvRuzPWmMshnz3yiiIhIH6FA70aciaPNpUAXEZHoEco8dIwxbiDv+POttQfDVVSkxZt42vrt0jkiItIfnTHQg+u2fxcoBzr3FbXA5DDWFVFxbg9tLhc+Xzuxsdr6XURE+r5QWuhfA8ZYa6vDXUxf4XEngIXaxkpyMwsiXY6IiMgZhXIP/RAwoNZBjY9JBKCusTLClYiIiIQmlBb6PmCFMeZFwNt50Fr7cNiqijBPbDL4oK5hwHRKiIhIlAsl0A8Gv+KCX/1eYlwKtEBDc1WkSxEREQlJKJuzfA/AGJMcfNzvdy1JjE8FoLlVe6KLiEh0OOM9dGPMRGPMh8A2YJsxZr0xZkL4SzttTdcaYx6rrw/Prf1kTzoAjW0KdBERiQ6hDIp7DPiGtXaYtXYY8E3gV+Et6/Sstc9ba+9KS0sLy+snJTiB3tLeEJbXFxER6W2hBHqStfatzgfW2hVAUtgq6gPSk7MAaPX2+7sLIiLST4Q0yt0Y8wDwu+Djz+KMfO+3UjsD3adAFxGR6BBKC/0OIAf4c/ArJ3is38pIyQWgrnxnhCsREREJTSij3GuBe85DLX1GRko2AJWJFRGuREREJDSnDHRjzCPW2q8bY57HWbv9BNba68JaWQR1rt8eE9BmdCIiEh1O10LvvGf+o/NRSF8zqj2GRpcCXUREosMpA91auz7441Rr7U+Of84Y8zXg7XAWFmkJxNLqaol0GSIiIiEJpQn6hW6OfbGX6+hzPCaONnPSnQYREZE+6XT30JcAnwaKjTHPHfdUClAT7sIiLcF4aDGWNp8fT6w70uWIiIic1unuoa8GjgDZwI+PO94IbA5nUX1BrImn2RjGPfAS+39wbaTLEREROa3T3UMvAUqAueevnL7DTSJtLhcpNEa6FBERkTMKZXOWOcaYtcaYJmNMuzHGb4zp94ucJwYCAIwo+s8IVyIiInJmoQyK+xmwBNgDJAB3Ao+Gs6i+wMY6I9w/SvBHuBIREZEzC2mitbX2I8BtrfVba38LLA5vWZGXkj880iWIiIiELJRAbzHGxAEbjTH/aYy5N8TrotrtCx4CIL0DAgFNXxMRkb4tlGD+HOAG7gaagULgpnAW1RfkJOZwhS8Ng6WxrSPS5YiIiJxWKJuzlAR/bAW+F95y+paUmCSaqKO2xUtaYmykyxERETml0y0ss4VuNmXpZK2dHJaK+pCU2FR8HWVUNDZQlJ0c6XJERERO6XQt9GuC3/8u+L1zs5bPcpqg70/SPOnQBBW1B6F4cKTLEREROaVT3kO31pYEu9uvsNb+o7V2S/DrPuAT56/EkxljrjXGPFZfXx/W90lPdPZFr6k9GNb3ERER6alQBsUZY8z84x7MC/G6sLHWPm+tvSstLS2s75OdMgiATSUfhfV9REREeuqMg+KALwG/McakAQaoBe4Ia1V9RG7GEACO1hyOcCUiIiKnF8oo9/XAlGCgY60Nbz93H5KZVgBAdnJrhCsRERE5vdONcv+stfb3xphvfOw4ANbah8NcW8SlpQ0FYKdbXe4iItK3ne5eeFLwe8opvvq9xCTnHrrpaOdTv1gd4WpERERO7XTbp/4y+H1ALSZzApeL8V6LFx9rD9RGuhoREZFTOl2X+09Pd6G19p7eL6fvSbax1Lu19KuIiPRtpxsUt/68VdGHpboSWBfbAC5vpEsRERE5pdN1uT95Pgvpq4amZBHwNpI0/GEeXzWVOy/UtqoiItL3nHGBGGNMjjHmR8aYl4wxb3Z+nY/i+gJ/bBwArth6/vXFHRGuRkREpHuhrPj2B2AHUIyz29oBYG0Ya+pTCqyJdAkiIiJnFEqgZ1lrfw34rLVvW2vvAC4Nc119xs25sxnrbccTCJAQG9EVb0VERE4plITyBb8fMcZcbYyZBmSGsaY+JXbe17i6qYU2l4tWfwveDn+kSxIRETlJKIH+r8FlX78JfAt4HLg3rFX1JTFx5Iy4HICEIf/LnvKmCBckIiJyslAC/X1rbb21dqu1dqG1doa19rmwV9aH5CQ7e6HHJO3jmv9+J8LViIiInCyUQH/XGPOqMeZLxpiMsFfUB41ILT7hsbU2QpWIiIh074yBbq0dDXwHmACsN8a8YIz5bNgr60Oyhs4jxR/oevzNP26KYDUiIiInC2nYtrX2A2vtN4BZQA0wsBadyRvP7a3BwXDGx583aH90ERHpW0JZWCbVGPMFY8zLwGrgCE6wDyg5+dMBMDENxLo1N11ERPqWUFrom4CpwPettaOttfdZawfcOu+5wYFxiakf4olx6z66iIj0KafbnKXTcKv0IiUY6K7c12msvpzq5nayk+MjXJWIiIgjlEFxAz7MAUblTgYgwTq/skde3x3JckRERE6gtUxD5EnJ59LmFlpNAHfCAX7/3kFa27VqnIiI9A0K9FDljGOjx+liTyz6BQC3PrYmkhWJiIh0CWWU+2hjzBvGmK3Bx5ONMd8Jf2l9jMvFzLhs58cOJ9gPVDVHsiIREZEuobTQfwX8E8FNWqy1m4HbwllUX/VPtQ0AFFpnPffMpLhIliMiItIllEBPtNZ+8LFjHeEopq/LXvAtbq9r4HBMLLddUMDRhjYCAY0ZFBGRyAsl0KuMMSMAC2CMuRlncZmIMcZca4x5rL6+/vy+8ZRPU5g8hA4DQ3LaafMFGP7tl3hs5d7zW4eIiMjHhBLofwf8EhhrjDkMfB34SlirOgNr7fPW2rvS0tLO7xu7XAzLnQjA9pY/dx3+t5d2sq3sPP9xISIicpxQAr3EWns5kAOMtdYusNaWhLmuPmt4zhQAVpW/eMLxGx59NxLliIiIAKEF+n5jzGPAHKApzPX0edn5U8nvcIYQzBzl7Tru81ueWnuQrYfVUhcRkfMvlEAfC7yO0/W+3xjzM2PMgvCW1YdljezaSjUx/zmevGMWeanONLb7/rSFa/77nUhWJyIiA1QoS7+2WGufttZ+EpgGpAJvh72yvio5l0VeJ9Ctbefi0TlcPi7vhFPueGItFY1tkahOREQGqJBWijPGXGyM+TmwHvAAt4S1qj7uzuoK5ra2Ulu5HYB7Lht1wvNv7qxg2QeHIlGaiIgMUKGsFHcAZ2T7KmCStfYWa+2fwl1YX+aa+hkmetspxY8v4CMv1UNaQuwJ5zz82m6m/8trPPL6bjqCXfQiIiLhEkoLfbK19kZr7VJrrdY6BbjqhxQaD34Drx14jYAN8NevX8ifvjqP+68c23VaTXM7j7y+h5+v0Dx1EREJL3Oq3VGNMf9orf1PY8xPu3veWntPWCsLwcyZM+26desi8t4lz/0d19SuBOD2ibfzjRnf6HrOWkvxP710wvlfvrCYG6YVMGHweZ47LyIi/YYxZr21dmZ3z52uhb4j+H39Kb4GtGGDpnb9/Nutvz3hOWMMP7ltKtdOGdx17Fer9nP1T9+h6P4XeWrtQZavL9WysSIi0mtO2ULv9mRjXECytbYhfCWFLpItdA6+zxde/iwbPB4Avjnjm3xx4hdPOq213c+4B/962peaPzKLqYXpXDw6l+lD0+kIWDyx7nBULSIiUex0LfQzBrox5v9wlnr1A2txpq39xFr7w94u9GxFNNCtpez/ZbGosKDr0JYvbOn21LUHavjVyn0crGlh59HGs3qbm6YP4WuXjWJoVmKPyhURkeh3ukCPCeH68dbaBmPMZ4CXgftxutwjHugRZQyDU4ay7HAptxXk4zanvntxQVEmFxRlAlDf4uPdvVVsLq3nF2+febDcnzaU8qcNpdwxv5hvfmI0SfGh/CcTEZGBJpQW+jZgKvB/wM+stW8bYzZZa6ecjwJPJ6ItdICdL8GyJfwkI43H09NYfu1yClMKSYw9u9b0oZoWclLiOVrfxps7K/jhK7to9fm5cFQ2q/ZUnXDuVZMG8cA148lPS+jNTyIiIlGgp13u9wD3AZuAq4GhwO+ttRf2dqFnK+KBDrD2cf6w6rv8IMtpgS8oWMD/XP4/vfoW60tquOl/1pxwbFJBGnOGZ3LR6BxG5iZT3uAlMc7NsKxE4mN0/11EpD/qUaCf4gVjrLUdPa6sh/pEoAO+v9zN9Lpjq+Ge6l56T63cXckXfvsBZ/pPlp0cx6dmFvK5OcMYnK6WvIhIf9HTFvrXgN8CjcDjOOu532+tfbW3Cz1bfSXQWflDJu3/3xMOvXPbO6TFh2fO+e7yRv7rtd28tr2cjjNMfUv1xDCrOIslswq5aHQOse6QVvsVEZE+qKeBvslaO8UYswj4G+AB4HfW2um9X+rZ6TOBXlvCrp/P4JXkRH6V7oT430z+G+6edvd5eXufP0CMy2CMoa6lnec3lfHsxjLWl9SedG5mUhzL7prD6LyU81KbiIj0np4G+mZr7WRjzE+AFdbaZ4wxH1prp4Wj2LPRZwId4CEnyCcVDwXAYNj0+U0YYyJZFTuONLD0g4P875qSE47HuV0smVXIyNxkphZmsLu8kb2VTVw2LpcZwzIjVK2IiJxOTwP9t0ABUAxMAdw4wT6jtws9W30q0D/4Fbz0LXbExfKF/DxaXU7X9jPXPcPIjJERLs7R7O3gnY+q+JvfnXmhv9F5yfzz1eO5aFR2xP8oERERR08D3YUzbW2ftbbOGJMFFFhrN/d+qWenTwV6p4fSqHC7uWzosQVn+lKoA3T4A+w82shLW46csHHMRaNz2FPeyJH6E/dyT4xzs2BkNldOGsTVkwYTF6P78CIikdDTQDfAZ4Dh1trvG2OGAoOstR/0fqlnp08G+pbl8Kcv8fXcbN5Icuajzx40m8cXPR7hwkLX5vOzZl819y3fTEWj96Tn89M8XDM5n5lFmVQ1eYZZcO0AACAASURBVJlVlMmwrKQTgt7b4ae+xcfRhjY6Apac5Hiqm9vJSYnH77ekJsTQ7g+QmRhHjAbqiYiEpKeB/j9AALjUWjvOGJMBvGqtvaD3Sz07fTLQAVY9jPfN7/FOQgJfz8sBYMnYJdx3wX24XdE1R9xaS12Lj1UfVfHmjnK2ljXwUUXTSee5DEwsSOPCUdksX19KecPJfwicSkp8DInxbuYMz2JaYTpzRmQxJi9FXf0iIh/T00DfYK2dfvxAOK0UF4LgILnvZ2Xwx9RjI8rfXfIuqXGpkaqqV1Q0tvH2rkpe3HKEzMQ4mts72FPRxL7K5pPOzU6OoyA9gVi3C7+1DE5LYOXuSi4ek8MLm4+c8j1S4mO4bFwuM4symVqYzoTBqQp4ERnwehro7wPzgLXBYM/BaaFrlPvp7H0LfncDAIuHDOZw7LE12PtDqHfnw4O1rD1Qw4icZKYUppOdHB/ytYGApaalnT3lTazYXcF7e6vZVd5Imy8AQGFmAgtG5rB44iAWjMzG7VK4i8jA09NA/wxwKzAdeBK4GfiOtfaPvV3o2erTgQ5gLXwvHa+BX6andc1RB7hz0p3cPfXuqOuCP59a2jtYtaeKFbsq2F3e1DWvPjs5jpumD2HBqGwmDE4jMykuwpWKiJwf5xzowRHuc4Aa4DLAAG9Ya3eEo9Cz1ecDHaCtHn7gzE2vdxkWDCvsempyzmSeXPwkMS7toBaKykYvq/dW8Yf3D7K+pBZ/cJU8T6yLCYPTmD8ii6sm5zN2UP/r/RARgZ630PvEIjLdiYpAB1j5Q3jzXwGocrtYOHTICU8/e/2zjEgfEYnKolZ1k5c3dlSwrqSGgzUt1Ld2sOtoAwELxdlJzBmeyaDUBKYUpjFneBaeWPWEiEj062mg/whYA/zZnstOLmEUNYEOsH8lLP8SNFcA8ERqCj/Oyuh6en7BfB6c8yCDkwdHqsKoV9HQxrMbD7N8fSn7KptPWOd+dF4yF47K4YapBUws0AA7EYlOPQ30RiAJ6ADacLrdrbU24v2aURXonV78Jqx15qQ/n5TIt3OzT3j6hxf9kMXFiyNRWb8SCFjK6lt5b18Nmw7Vsb6klu1HGgBnHv30YRmMzUvhYE0LuanxzBiWwbwR2WrJi0if1uvbp/YVURno4IyAf+qz0O7M5340PY1fZBwbMDc+azz/d9X/acBcL6toaOO5TWW8ur2cTYfq8HYEiI9x4e1wRtJnJcUxqziT7OR4Lhqdw0Wjs7W3vIj0KT1toXe3q1o9UBLpPdGjNtDBGQH/6ndgzc+6Dv06LYVHMo91w//XJf/F5cMuj0R1/Z61ltLaVrKS4+gIWNYfqOUP75fw+o6KE84bk5fC9GEZLBiZzezhmWQmxuHSlDkRiZCeBvp7OFPWtgQPTQK2AmnAVyO5L3pUB/rx1j4O7/wE6g/y16RE/uG4bvhbRt/CA3MfiGBxA4u1ln1VzRysaWHV7irWldSw80gj7X6nFe92GWYMy+CysbksGJXNuEGpCngROW96Guh/Bh6w1m4LPh4PfB/4R5yBclN7ud6Q9ZtA77T3TfjdjTQbwycL8ikLLkYzKmMUT13zFLGu2AgXODA1ezv4YH8NHx6qY3NpHSXVLeyvclbFi4txMbs4k6GZiYzNT2XcoBTGD04lMU5TEUWk9/U00Ldaayd2d8wYs1GB3suaKuCFe2HnC5TGuLmy0Nm1bXDSYJ65/hkSYxMjXKAA7Kt0FrpZuaeKDSW1HK5rPeH5ETlJTC3MYMawDMblpzA6L4WkeIW8iPRMTwP9KZyFZZYFD90KZAOfA96J5CYt/TLQO7XWwpbl+F76FtOLh3YdfnLxk0zP625Yg0SSt8PPvspmtpTWs/1IA7uONvLhodqupWvdLsOwzEQGpycwsSCNKUPSWDAqmxSPel1EJHQ9DfQE4G+BBcFD7wI/x5nClmitPXnrrfOkXwd6p/Zm7A9H8Y/pHv6anATAA3Me4JYxt0S4MDkTb4efrYcbOFDVzKbSOo7Ut1FW18ru8kZ8fkuMyzA8J4nx+anMHp7F6LwURuYmk5agkBeR7vV42poxJg4YA1hgl7XW17slnpsBEegA/g7s72/gqy07eTcxAYAbRt7AA3MeIM6tdcyjTWu7n3UlNazZW822sgbWHaihud3f9fyo3GSmD82gMDOBSUPSmV2cqfnxIgL0vIV+Cc6mLAdwFpUpBL5grV3Zu2WevQET6ACBAPzhJv5S/j7fycnqOvzt2d/mxpE34onxRLA46YlAwBlZ/1FFE9vL6ll7wFkEp77V+bvZGCjKSmLByGwWjMpm+tAMspI0fU5kIOppoK8HPm2t3RV8PBpYaq2d0euVhsgYcy1w7ciRI7+8Z8+eSJURGaXrqPvNFXyicDCtLtcJTz1y8cNcOuxyLWvaT1Q0tPHe/hq2ldWz+2gj7++voSXYks9P8zBmUApFWUlMHpLG6LwUxgxKIdbtOsOrikg062mgb7bWTj7TsUgYUC3047W3YP/vUxwqfY8783M5EnPi6OnXb36dvKS8CBUn4dLeEWDDwVo2HqpjS2k9Ww7Xc6S+FZ//2P/DOSnxDM9OIiMxjnH5qYzNT2FqYTp5qerBEekPehrovwX8wO+Dhz4DuK21d/RqledgwAZ6J18rrH+Sqlf/iV+np/L7tGPL69+WN5d/uPQR4uI0za0/a+8IsL+qmZ1HG9h4qI6qpnaO1LVS3dzOgepmOv/3HpGTxMSCNBaOyWVkbjJF2UkkaxqdSNTpaaDHA3/HsVHuq4CfW2u9vVrlORjwgX68QAC7/S88tOIb/DklGYAYa3ly3r8zefS1ES5OIqGxzdfVkv9gfw0bD9VR3dwOONPoBqV6GDsohbkjspgzPIuhWYmkahqdSJ92zoFujHED26y1Y8NVXE8o0Lvh76Bu61P82zsP8nKy0zof7OvgN5f+jILhl0W4OImkDn+ADw/Vsbu8kZLqFg7VtLDraCP7gqveGQOTh6QzY2gGY/NTmDwkjTF5KRqTIdKH9LSF/hfg7621B8NRXE8o0E9v7/Y/seSD79Ia/Af5cX8Ws5c8AwkZZ7hSBpIj9a28v6+GnUcbeX9/NdsON3StXZ/iiWF0nnMfflJBGiNykhmRm6SlbUUipKeBvhKYBnwANHcet9Ze15tFngsF+pnVNh7lb178NDu8lQAMb/fxaNY8hnzyNxGuTPoqf8Cyv6qZ9SU1bD3cwObD9ew80tC1zSzA4DQPEwvSmFiQxtTCdCYWpJGRGKvWvEiY9TTQL+7uuLX27V6orUcU6KHbV7Wdr79yJ/s7GgF4mgLGXf5vUKBlZOXMvB1+9pQ3sbeyidLaVnYdbWTr4fqu7nqApDg3+ekJjMlzNqgZnO6hODuZYZmJpCXEYnHu3YvIuTunQDfGeICvACNxtk79daT3P/84BfrZsdbyXskb3PX2vQAsaGnl9voGZo79FK7LHoCUQRGuUKJNQ5uP9SW1fFTeRGltCwdrWthyuIGqpu7HzKYlxJKVHEdRVhLZyXFkJceTkxxPXqqHQWnxjMpL0cA8kdM410B/CvDhjGq/Eiix1n4tbFWeAwX6uSlpKOGfX/97NjXu7zr26fpGrmtqYkLeDLj1d5CcG8EKJdo1tvn4qKKJA9XNHK33UtPsxeUyNLT6OFrfxtEGL2V1rV2r4R0vLzWezKR4clPiyU/zMDg9gaGZiYzMTaY4O0m71smAdq6BvsVaOyn4cwzwgbW2T/XPKtDPnbWWdUfXcserXzrh+PB2H4+WVzCkcD4seQo0j13CxFqLtyNAdXM7tc3tHKxpYV9lE3srm6lraXfm1Ne3Ud3s5fh/ptISYklPjCUjMY7h2UkUZCRQkJ5ATko82cnxpCbEMjQzUd370i+da6BvOD7AP/64L1Cg946ShhJWlq7klf1/ZVPVZgBur2vga7V1uIfMgmt/AnnjI1ylDFRN3g4O1bRQUt3M3spmjtS3Utvso7LRS2ltC0cb2gh87J+xhFg3Y/NTGJ+fyvjBqYwdlEJhRiI5KfEauCdR7VwD3c+xUe0GSABagj9ba21qtxeeRwr03rerZhffevtbHGg4gMvCP1fXcEtjE8SlwLWPwKSbI12iyAl8/gBH69vYV+W07L2+ADuONrC9rIHtRxpobDs29CfFE8P4/FSyU5x798XZSRRnJzEqL5lBqR6FvfR5Pd4+ta9SoIeHP+Dn0Y2P8qstvwLAWMv/q6zmmuYWTM5YuOJfYORl4NKWntK3WWsprW1lT0Ujh2qcvei3lTVQ0dBGQ1sHTd5jYZ+RGMvI3GQK0hMYnJ7A6LwU0hJiKcpOYkhGgja+kT5BgS7npKq1irvfuJtt1dsAcGP4YWU1VzQ1OSfMvRuGzoVRV0BMfAQrFTl71loqm7zsq2xmd3kj28sa2FvZRFldG+UNbXQc14/vdhkmDE5lYoGzet6ovGRG5iSTnRyvbWzlvFKgS480tDfwX+v/i+W7lwNQ6ErgW0cOcmlLq3OCJw0uuBPmfx08Eb8TI9JjPn+AA1XNVDZ5Ka1tZX9VMxtKatl5tPGEkfkuAymeWDKT4nC7DFlJcWQkxpGRFEtCbAxJ8W5i3S7iYpzWfV2Ljw5/AG9HAG+HnzZfgPTEWBLjYqhobKPZ20FCrJu4GBcZiXHkpDhT+nKD9/4T49zEB18rIymOzMQ4jEG3CgYQBbr0iraONn615Vf8dutv8QWcf9SuTBzG3x4poagmuDLwgnth0CRoa4DETCiYCUk5EBMXwcpFeoe1lspGL7vKGzlQ1eyMwm9qp7LJiwEa2zqoaWmnstFLa7u/awnd47kMxLpdeGLddP7729DmBHmyJ4b4GBcNrT7afIFur/+4GJdhUJqHrGRnql9yfAwZiXFdr1WYmUiqJ4aRucnEx7jJSopTr0IUU6BLrzrSdIQfrvshr5W81nVscvJQ/mXnewz3nWLtofwpkD4Uhl8Ck2+D+OTzUqtIJHX4A3QELIHgv7PxMW5c3bSoO/8dPv64tZaG1g6ONrRxtKGNFm8HLe1+fP4AvoDF7w9wpL4Nn99ysKaF+lZnml+bz0+z10+rz99tTTEuQ6zbRW5qPCmeGAalJpCf5mFQmofi7CTaOwJkJccxOi+lq2dA+g4FuoTF0eajBGyA32z9DU/vehqLZZQnh8+7cxnV4WeYdZPc4YW9b558ccEMp+U+/QswejG4NOBIpLd0/jHgdht2HW2ksc3HgapmfH5LbUs7h+taaWn3U9HQRl2rj5Lqlm5fJynOTUKcm/y0BLKT48hOjmd4TjKD0z1kJcWTmeTcFkhNiKHDb0mMc+sPgDBToEvYlTSU8M0V32RX7a4TjucmOivOfXvmPzI/NhNPyRpY/wQ0V4G33jkpMQvSh8GCr8OoRRDrOc/ViwxsgYCl1ednf1Uz9a0+rIV9VU3sOtrIwZoWjDHUNrdT3tBGRWP3y/oCpCfGUpydxLj8VDISY8lMiicvNZ6hmYkMzUwkPVG33npKgS7nzebKzTy39zlWlq7kSPORk56fmTeT0Rmj+fy4z1Lg98PuV2Hrcjj0vnOCccG0z8HUz0DhLGeTbhHpM+pbfVQEg72y0Utjm4/6Vh/+ABxtaGNfZRM7jjTQ6O3g4/GSnRzP6DxndsDg4Op+mUnOyn4F6YnkpmjWwJko0CWidtXs4oltT7CzZieHmw7T2uGMjk+MSeT6kddz56Q7yY1Jhp0vwtY/wZ5XwfqhcA5MuQ2mLFGrXSTKBAKW+lYfRxvaOBhc6W9PeRN7KpqobvZyNHj//3hxbheD0z0MyUgkxRMT7O73UJiRyKA0D+mJcXT4AxgD6YlxZCXFkZYwsLbtVaBLn9Hub+el/S+xs2Ynz+99nob2BgAKUwq5dcytXD/ietLbGuHD3ztfDaXOhTO+CLO/CrljI1e8iPSaQMBZB6C+1cfhulZKa1sprW3hcK3zc2Wjl4C1VDR68X98bd/jJMS6KchIYEhGgtPyT/OQn55AqieW3FSnJyArKQ5PbP9YCEuBLn3W9urtPL3raZ7b+9yxqXDFV/L58Z9nQvpozJ5XYeMf4KPXwd/u3GO/6j8hoyiyhYvIedHhDzgj/evbqGtx/o2IcRvqWnxUNTkt/UO1LRyua6W6ybnP313+ZybFkZ0cx+D0BAaleijMTGRYViK5KR4yEmNJT4zrWk+gL1OgS5/nD/hZW76WxzY/xpbKLbT525iSM4XPjPsMlw29jLi2Bnjnv+CDxyDQAeNvgPn3wOBpkS5dRPqQ9o4AVU1eGtp8lAe36a1u8lJW30ZVo5fDda2UN7RR1dR+0rVulyEn2RnIl5fqCX4d/7OHQakeUhNiItbNr0CXqFLRUsEL+17g6V1Pc7jpMAD3zriX28bcRmJLDaz4AWxa6gR75ggYexVMvEnhLiIh69zFr6rJS22Lj9pmZ0Gg8uC8/4oGL+WNx3oFjueJdTkBn+IhNzWeQcGwz0yKIz0xlpyUeHJTPGQlx/X6HgAKdIlKHYEOVpet5h/e/gdaOlrI9GRy3Yjr+NToTzHU5YENT8K2v0D5FueCghlw/c91n11Eek2bz98V7kfrnXX+nS9v189HG9po83W/qt/Vk/N59NO9t/O4Al2imrWWdeXr+N3237GydCUBG2B2/mw+N/5zXFhwIaahDLY/C6t+DO3NzmI1Ez8JQy7QjnAiEnbWWhraOqhtbqeu1UdlcEpfRWMbhRmJ3DRjSK+9lwJd+o2jzUf5w44/8Nze56hpq2FoylBuGXMLlxZeSqErHl75Z9j2Z6c7PiUfpn/emdeeXhjp0kVEekyBLv1Oa0crL+9/mT/t/hObqzYDMDt/NteNuI6rcy/AvW8lfPBLOLzeWaymaIEzn33CJzWnXUSilgJd+rVt1dt4o+QNlu5cSpOviYLkAhYWLmRR0SKmEu/MZ9+0DFqqIDkPZt3lDKLLLI506SIiZ0WBLgOCtZY3Dr7BU7ueYn35enwBHyPTR3LbmNu4qvhKUkreg3cehoNrwLhh6Bxwx0H+ZBh5ORRdqKVmRaRPU6DLgFPvref5vc/z9O6n2V+/n+TYZBYXL+az4z7LCK/XWWJ22zNOgFd/5Fw0aDLMvB3GXQdJ2ZH9ACIi3VCgy4BlrWVDxQae2vkUrx18DX/AzyWFl3Db2NuYmz/XWRzC2whb/wxrHoWq4G5xQ+c6y82OuQo8qRH9DCIinRToIkBVaxVPbnuSp3Y9RWtHK0WpRczIm8GUnClcNOQisuLTYd9b8NEbzkYxdSXOhUPnwbTPOAPq4hIj+yFEZEBToIscp7Wjlb/u/yvLdi1jb91evH4vca44rh95PV+e9GXyk/PB3+Hca9/1Eux4AeoPOhcXXQiTb4UJN0J8cmQ/iIgMOAp0kVMI2ACbKjfx3N7neO6j52gPtDMlZwrXDr+W60ZeR0JMAgT8cGAVbFkO+9+GuoPgioGRVzjbu45erKlwInJeKNBFQnCk6Qi/3fZb3jz4JuUt5STEJHBJ4SUsKlrExUMuJsYVA9bCwfdg5wtOwDcdBU+aM0p+2DwYew2kDIr0RxGRfkqBLnIWOgfSPb/3ed44+AZ13jryEvO4bOhlfHLUJxmTOcY5MeCHfStg81Ow720n3DFQMN0ZTDfmKsgbH8mPIiL9jAJd5Bz5/D5Wlq7kj7v/yLtl7wIwLnMcN4++mcXFi0mNC46ADwTg6CbY+ZLTeq/Y7hzPHgNjr3a65YfOjtCnEJH+QoEu0gvqvfU8s+cZnvnoGfbV78Pj9nDFsCu4ZsQ1zB40G/fxG8E0lDld8tufhbIPwQYgPg2KL4Tii2DSpyAxM3IfRkSikgJdpBdZa9lWvY3lu5fz6oFXafQ1khCTwFXFV3Hp0EuZN3iec7+9U0uNs4jN4Q3OoLr6Q85KdcUXwvjrYfwNCncRCYkCXSRMvH4vbx96mzcPvckbJW/Q5m8j05PJRUMu4saRNzItd5qzeM3xjm5xFrLZ/heo2QsYKJwFwxfCiEude/Du2Ih8HhHp2xToIudBU3sT75a9yysHXuHtQ2/THmgnPymf2fmzWVS0iLn5c0/slgenO373q8589yObAOt0zY9Y6IycH70IknMj8nlEpO9RoIucZ03tTbxW8horDq1g1eFV+AI+XMbFjLwZXDb0Mq4qvooMT8aJFzVXw/4VsOd12PMKtFSDKxYKZ8OIS5xR87njtYGMyACmQBeJoGZfM6tKV7GufB0rS1dypPkIMa4YFhQs4Jrh13BJ4SXEu+NPvMhap8W+dbmzUl3tfud4RrGzr/vYqyFvgsJdZIBRoIv0ITuqd/Dc3ud4cd+L1HprSYlN4dKhl7JgyALm5s8lLT7t5IvqD8Puvzqj5vevdI6lD3O2gB1xKYz6hAbWiQwACnSRPsgX8LGmbA0v73+Zt0vfprG9EbdxMy13GgsLF7KoaBF5SXknX9h41Nk8Zt9bcOBdaK1xRs3nT3FWqyu+CPKnQko314pIVFOgi/RxHYEOtlZtZWXpSt44+Ab76vcBcMGgC7hg0AUsGLyA8VnjTx5U5/dB6TrnnvveN+HIZiD4/3TmCGdBm3HXQOEccLnO74cSkV6nQBeJMvvq9/HKgVf46/6/doV7XmIeVxZfyTXDr2F0xuiTp8MB+Frh0AfO6Pn9K51NZfztkFoA466DyZ+CwdN1710kSinQRaLYoYZDrDmyhjcPvsl7R97Db/0UphSysHAhc/LnMCt/1smD6jp5m2DXy7D1T04L3u+F2CQYNtdpvRdfBNmjFfAiUUKBLtJP1LbV8lrJa7x56E0+OPJB13S46bnTGZs5lqm5U5mRN4PshOyTL25rgG1/dlrv+1ZA7QHneEYxTLjBCfghs9Q1L9KHKdBF+qGm9iY2VGzg3cPv8v6R99lbv7frueFpw5meN525+XOZXzCfpNikEy+21gn0vW84G8rsWwHWD6lDnHvuwy9xWu9xH7tORCJKgS4yAHj9XjZWbOTDig/ZWLmRjRUbafY1E+OKYWHhQj4x7BMsHLqw++75lhr46A3Y8kdnvfmONnDHOaPmR14OxRfDoEnqmheJMAW6yADk8/vYVLmJ1w++zkv7Xuqa8z45ZzIzB81kZt5MJmRPINb1sXXjO7xwcA3seQ32vApVu53jaYXO1Lihc50WvBa2ETnvFOgiA1zABvjg6Af8ec+f2Vy5mcNNhwFIjk1mfsF8Li28lIsLLz65ax6g4YgT7HtedUbQN1c4x1MGw5jFzqI2RRdCfPJ5/EQiA5MCXUROUNtWy9qja3nn8DusOryKqtYq4t3xXFJ4CZcPvZz5BfNJiUvp/uK6g7D3LfjoNfjoTfA1AwayRzmD6govcOa9Z4/WADuRXqZAF5FTCtgAH1Z8yMv7X+aVA69Q563DZVxMzp7MgoIFXFx4MaMzRuMy3YRzhxdK3oWD78PB1c7WsK21znMJGU73/LD5kDMGUgY5IR9ziil2InJGCnQRCYk/4GdT5SZWl61mZelKdtbsxGK79nifmDWRaXnTGJE24uRV68AZPV+9Fw6959yH378K6kqOOyHYks+bCPmTne85Y5z787ofL3JGCnQROSeVLZWsObKGVaWreLv0bVo7WgHI9GRywaALuHjIxVw05KLuN5TpVF/qtNybyqFmH1R95DyuP3jsnIQMp/U+bL6zF3zeRG02I9INBbqI9Ji1lj11e9hYsZENFRt4/8j7VLVW4TZuJmRPYErOFCZkTaAotYihqUNPfQ++U2O5s8hN7QGo3AlHN0PZRmc+PEDaUMgbDzljnV3l8iZC2hC15GVAU6CLSK8L2ADbqrbx1qG3+ODoB2yt2oo/GMYGw8iMkUzLmcbMQTMpTismIz6D3MTc7teg7+RthJI1TsAfXu+05Gv3gw04z6cWODvJ5U859pUySCEvA4YCXUTCzh/ws6NmB+vL19Pka2Jz5WY2V26mydfUdU6mJ5Oi1CLGZ41nQvYERqWPYkT6CGJcMad+YV+r05I/utW5L1++Far20LWrXFKuE+w5YyA5F3InwJCZkJAe3g8sEgEKdBGJCH/Az7bqbeyp3UNDewP76vdxoP4AO2t20uZvA5zWfH5SPmMyxzAjbwbzBs9jWOow4txxp35hb5MT7Ec2Hfuq2HGsux4go8jpph88DQqmO98TMsL7gUXCLCoC3RgzHPhnIM1ae3Mo1yjQRaKTL+Bjd81udtbs5EjzEQ40HGBjxUbKW8oBcBs3I9JHMDZzLEWpRYzJHMPk7Mmke07T6rYW2uqcbvpD7zv34yt2QM2xNe5JGey05uOSnMDPnwzJg5xV77QwjkSBiAW6MeY3wDVAhbV24nHHFwM/AdzA49baHxz33HIFusjAY63laPNR3i17l7KmMrZVb2NnzU5q2mq6zhmeNpxpudMYlzmOEekjKEorIiM+o/spdJ1aa51wL9vgdNuXrnVG3hvXsRa9cUHOuGArPh1yxzmD8eKSIXM4xJymt0DkPIpkoF8ENAH/2xnoxhg3sBu4AigF1gJLrLXbg88r0EWkS21bLbtqd7G1aisbyjewsXIjje2NXc8nxyYzPms8U3OnUpRaxPC04YzKGHXqLvvOf/N8rU63fW0JVO+B0nXOQLz2Zgj4jp3vioXMYiiY6ayCN3i6s1HN6f6IEAmTiHa5G2OKgBeOC/S5wEPW2kXBx/8EYK399+Dj0wa6MeYu4C6AoUOHzigpKTnVqSLSD1lrKW8p56O6j9hXt4+DjQf5sOJDdtfu7jon1hXL2MyxTMmZwsxBM5mQNYG8xLzTj7DvFAg48+UrtjnhXrnL2aDm0AfQUuWcE5cCOaOde/KBDojxOMezRjpd+f52p9XvSccZvGcgJc953f/f3r0Hx3mVdxz/PpJWV+u+si35KlvyRZKN7VxIQktdYEgIpKGdlKSlEKCUoZ3p0LRMB4ZOGdphmNIO5dZCaaDhUgI0UApkOpBCmECA3IhjyZItyZJtRb5pJXklW9bF1ukfhfsxnQAAEuBJREFU5+xq7diKZcva1er3mXln3/e877579vi1nj3nPe85Y0NwJuZbCUZe9OewHJ82ecbPdDdxGgrLfEtBSQ1EiiEnz/fmH4/7HyPlq6Gg1M+KV1jmz1NY7jsJVjdAbgSmz6t1IctkWkC/B7jDOffusP024JXAh4GP4mvuDyYC/GxUQxeRhPhEnBNjJ+iN97Ivto/WWCutsVYmzk8AUJpfSmNFI42VjWwo30BDRQNbq7e+/PPyCYk55F981o+EF+uCkX7/SN30eR/UTx32AflK5ESgJArLVvgAPdgNa272zfyW4wPyUK9/hC+1sx/4Y85P+uWSjOQPCRyU1vr8RxuheiOMj8DyJn87om4n5OZBZb1/UiAn4lsf9ChgRpotoM/yrMjCcs4NAu9Ndz5EZHEqLyinvKCcTZWbuH397QBMnp9k3+A+DgwdoGu4i65TXTza8+gFj9IV5RWxLbqNLVVb2LF8B+vK1lFfVk8k96JpZc1803tVPWz//Utn4vw5GD3mg3xuxNemnfPj148eg3g/rLvVB9hEjfvlnAtBOycXMP8eM/8jYvSY/yFxsh3OT0GkCEaO+h8HA/thcsy/b/SYD+JDvX4An/E47PvO5T+zuNoH/Iq1vkVh+dYw2U6uHwegukET72SgjGtynwvV0EVkrpxznBw7SfepbtoH2+k/3U/7YDs98Z5kbT6SE6GxspHKwko2VWyiKdrEjpodrCxZmebczwPnYGIELNe3OLjz0P9r33wf64LRozA17vsVxLohpb/CDPOtCwVlULHG1+qLKn0HwuoG/wOgsBzK6tTXYJ5lWg39GaDRzOqBfuA+4A/TkA8RWYLMjBUlK1hRsoJXrXpVMn3q/BStsVb6T/fTOdxJ53AnsbEYTx97mqnQSa6upI5dK3Zx44obWV++PjkC3hXdm88UZj7YAqwMDx/VvuLSxzrnZ9Qz8zX+WJffHurxI/idPunv9w92+0cGk039QW6+H91v2XJYud037yf6BWgY33l3vXu5PwzsBqLACeDDzrkvmtmdwCfxj619yTn30as5v2roInK9TZ2fonO4kz0De3juxHM8d+K5Cx6ly8/JpyRSAkBDZQMNFQ1UF1ZTlFfEsvxlFOQWUFFQwdqytURyIsQn4uRaLjk5OZybPkdtSe2V38fPVIk4cn7SB/djL/h+AcOH/I+A0yd8x8LJmVsdFJT5VoFIsa/VV2/0HQor1vl7/fklvlOhOvVdYFEMLHM1FNBFZKE55zg8cpgjo0foG+3jxJkTxM763u+HRw7TE++54B79lSjMLUy2ApRESqgtqWV16Wrqy+vZXLWZsvwyKgsqqSqsojS/lEhO5KX3+DPd9Hkf7E+0+c54J/f7bTfte/gPHXxph0LL8TX5qo2+Ob9y3Uy/hNJa/wMg2ujv9S+Rpn0FdBGRBTQ1PcXY1BjxiThnz51ldHKUvtE+4hNxygrKKMgtYGh8iEhOhNHJUY6eOcqyyDJOTZxi2k0Tn4jTN9rHkZEjnHPnLvkZq5atYkP5BpYXL6e+vJ66ZXVsqtxESaSE6sLqxXUbAPzjgqeP+1r9UI/v0DcW8+uDB33AH4/7Yy33wp7/eYW+lh9thOjm8LrJp+UXp+XrXC8K6CIii9D4uXE6hzsZGBvgzLkzDJ4dZOzcGJ1DnRTkFtA+1M7JsZPJeeoTKgoqks/iF+YVsqNmBzesuIGygjJWFK+YfZz8TDY+4p8eyCv09++HenxTfqzTN+3HOv2jg4nZ+cBPw1uzyQf4RKCPbvL38Rfbjx4U0EVEspZzjqNnjnLizAl64j3EJ+IcHjnMnoE9HD19NNlzP8EwaopqWFmyktWlq2moaGBN2ZrkKHuLNtgnTI372nxqkE+sT43NHFdYMRPcE4G+ZrO/h5+bMU90v4QCuojIEjU1PcXQ2SH2DOxhbGqM42eO03+6nyOjR9gX28fk9MzgNHmWR3lBOatLV9MSbaG5upnivGLWlq2lvrx+9mluM930tB8I6CWBvtN32kvIifgOetFNfjz/5VtmBt3JL0lf/oOsC+hmdhdwV0NDw590dXWlOzsiIotWfCKebNY/MHwgWcvfP7T/gqb8wtxCokVRGiobaKpqYkPFBmqKalhbtnZx3rNPdfaU76AX6wxD/Xb5x/SGey9svi9bNfOsfc1mH/CjjX4WvwUaaCfrAnqCaugiItfHuelz9MR76BvtY2xqjI6hDvpG+jg0cojDI4dxKc+bF+UVUVlQyabKTbREW2iJtrC+fD11JXWLO9BPjftAP9TjA3yig17swEwHPbiwU17NFr9esdZvF1XOa5YU0EVEZN6MTY1xZPQIg2cHOTRyiBdHXyR2NkbncCc98Z7kcZUFlTRFm2ip9s33LdEWaopr0pjzeeKc75Q3sN/frx88ONN8P3yYCwbX2XoX3Pu1eftoBXQREVkQo5OjtA+20xvvpX2wnbbBNg6eOsh0aLpeXryclmpfi2+qbmJt2VpWL1u9uGvyqRID6gwf9gG+tPbyY/9fBQV0ERFJm7GpseSc9m2xNvYN7uPwyMzU15UFlbREW9gW3ZZ8rSisSGOOM1emjeUuIiJLSHGkmJ3Ld7Jz+c5kWnwiTvtgO32jfbTF2miNtfLz/p8n782vKV1DS7SF7dHttERbks/Uy+Wphi4iIhnhzNQZ2gfbaY21JoP88TPHAf9IXWNlIxsrNvphccvqaY42s7Z0bfY0118BNbmLiMiiNDA2kAzwe2N76Y33EjsbS96TL8svSzbVb6/ZTnN1M9VF1WnO9fWjgC4iIllj8vwkvfHeZC1+b2zvJTveba/ZngzyxZHsGNNdAV1ERLJa4ln5tlib710fa+PI6BEAciyHjRUbaaluYXPVZpqrmxftPXkFdBERWXKGx4dpjbX6ZaCVjqGO5Fz2eZbH5qrNbK/ZzrboNpqqm1hXti7jh7fNuoCuoV9FRGSunHMMnB3w9+MH9rI3tpe2WFtyiNvC3EK2Vm9lW3Qb22q2sT26ndqS2ozqdJd1AT1BNXQREbkWiSFuDwwdSDbVdwx1JGepK42UsrlqM03VTTRXN7Mtuo3VpekbCEcBXURE5ApNTU/ROdxJ60Ar3ae66Rjs4MDwgWSQryiouGAgnIXsWa+BZURERK5QJCdCc3UzzdXNybSp6SkOnjqYvB/fGmvlyf4nkwPh1JXU0RxtTg5pu7VqK+UF5Quab9XQRURErkJiIJx9sX20DfphbftP9yf315XUcUf9HTxwwwPz9pmqoYuIiMyzkkgJN628iZtW3pRMGx4fpmOog/1D++kY7KAwd+EejVNAFxERmSeVhZXcVncbt9XdtuCfnbPgnygiIiLzTgFdREQkCyigi4iIZAEFdBERkSyggC4iIpIFFNBFRESywKIM6GZ2l5l9IR6PpzsrIiIiGWFRBnTn3Pedc+8pL1/YYfVEREQy1aIM6CIiInIhBXQREZEsoIAuIiKSBRTQRUREsoACuoiISBZY1POhm9kAcHgeTxkFYvN4vqVIZXjtVIbXTmU4P1SO126+y3Cdc67mUjsWdUCfb2b27OUmjpcrozK8dirDa6cynB8qx2u3kGWoJncREZEsoIAuIiKSBRTQL/SFdGcgC6gMr53K8NqpDOeHyvHaLVgZ6h66iIhIFlANXUREJAsooANmdoeZHTCzbjP7QLrzk0nMbI2ZPW5m7Wa2z8zeF9KrzOwxM+sKr5Uh3czs06Es95rZrpRz3R+O7zKz+9P1ndLFzHLN7Hkz+0HYrjezp0JZfdPM8kN6QdjuDvvXp5zjgyH9gJndnp5vkj5mVmFmj5jZfjPrMLNbdS3OjZk9EP4vt5nZw2ZWqGtxdmb2JTM7aWZtKWnzdt2Z2Q1m1hre82kzs6vKqHNuSS9ALnAQ2ADkAy8ATenOV6YsQC2wK6yXAp1AE/Bx4AMh/QPAP4T1O4H/BQy4BXgqpFcBPeG1MqxXpvv7LXBZ/iXwdeAHYftbwH1h/fPAn4b1PwM+H9bvA74Z1pvC9VkA1IfrNjfd32uBy/DLwLvDej5QoWtxTuW3CugFilKuwXfoWnzZcns1sAtoS0mbt+sOeDoca+G9b7iafKqGDjcD3c65HufcJPAN4O405yljOOeOOed+HdZHgQ78H4W78X9cCa9vDut3A19x3q+ACjOrBW4HHnPODTnnhoHHgDsW8KuklZmtBt4IPBi2DXgN8Eg45OIyTJTtI8Brw/F3A99wzk0453qBbvz1uySYWTn+D+sXAZxzk865U+hanKs8oMjM8oBi4Bi6FmflnHsCGLooeV6uu7CvzDn3K+ej+1dSzjUnCug+OPWlbL8Y0uQiobltJ/AUsMI5dyzsOg6sCOuXK8+lXs6fBP4amA7b1cAp59y5sJ1aHsmyCvvj4filXob1wADwH+HWxYNmVoKuxSvmnOsH/gk4gg/kceA5dC1ejfm67laF9YvT50wBXa6ImS0Dvg38hXNuJHVf+FWpxyUuw8zeBJx0zj2X7rwscnn4Zs/POed2AmfwTZ1JuhZnF+7z3o3/cVQHlLC0Wieui0y57hTQoR9Yk7K9OqRJYGYRfDD/T+fcd0LyidBURHg9GdIvV55LuZxfBfyOmR3C39J5DfApfFNcXjgmtTySZRX2lwODLO0yBF9zedE591TYfgQf4HUtXrnXAb3OuQHn3BTwHfz1qWtx7ubruusP6xenz5kCOjwDNIZenvn4jh/fS3OeMka4X/ZFoMM594mUXd8DEr007wf+JyX97aGn5y1APDRL/RB4vZlVhlrC60Na1nPOfdA5t9o5tx5/ff3EOfdW4HHgnnDYxWWYKNt7wvEupN8Xeh7XA434zjRLgnPuONBnZptD0muBdnQtzsUR4BYzKw7/txNlqGtx7ublugv7RszslvBv8vaUc81NunsPZsKC75XYie+p+aF05yeTFuA38E1Je4E9YbkTfx/tx0AX8H9AVTjegH8JZdkK3JhyrnfhO890A+9M93dLU3nuZqaX+wb8H8Fu4L+AgpBeGLa7w/4NKe//UCjbA1xlT9jFvAA7gGfD9fhdfG9hXYtzK8OPAPuBNuCr+J7quhZnL7OH8X0OpvAtRX88n9cdcGP49zgIfJYw6NtcF40UJyIikgXU5C4iIpIFFNBFRESygAK6iIhIFlBAFxERyQIK6CIiIllAAV0kQ5jZx8zst83szWb2wTm+tybMhvW8mf3mLMfttjDb2yzH7DCzO+fy+QvNzA6ZWTTd+RDJJAroIpnjlcCvgN8Cnpjje18LtDrndjrnfnaN+diBH2tARBYRBXSRNDOzfzSzvcBNwC+BdwOfM7O/vcSx683sJ2Ge5R+b2Voz24GfyvFuM9tjZkUXvecO8/OH/xr4vZT0m83sl6FW/wsz2xxGS/w74N5wrnsvddwl8lVrZk+E97QlWgnM7HNm9qz5+bc/knL8odAisSfs32VmPzSzg2b23nDM7nDOR83Puf15M3vJ3ywz+yMzezqc69/Mzzufa2YPhby0mtkDV/WPI7KYpHsEHi1atDjwwfwzQAR4cpbjvg/cH9bfBXw3rL8D+Owlji/Ez/DUiB/B6lvMjFRXBuSF9dcB377UuS533EWf81eEURaBXKA0rFelpP0U2B62DzEz5/Y/40d+KwVqgBMhfTcwjh/FLBc/3eQ9Ke+PAltDmURC+r/ih868AT9VZSJ/Fen+N9ai5XovicH4RSS9dgEvAFvwc85fzq3M1LK/iq+Zz2YLfjKOLgAz+xrwnrCvHPiymTXih/eNXOYcV3LcM8CXzE/k813n3J6Q/hYzew9+prRaoAkfvGFmzoRWYJlzbhQYNbMJM6sI+552zvWEvD+MH4o4MW83+FsNNwDP+GGwKcJPkvF9YIOZfQZ4FPjRLGUkkhUU0EXSKDSXP4SfYSkGFPtk2wPc6pw7ex0//u+Bx51zv2t+rvufXu1xzrknzOzVwBuBh8zsE8DPgPcDNznnhs3sIXyLQcJEeJ1OWU9sJ/42XTw29cXbBnzZOfeSToRm9grgduC9wFvwLRoiWUv30EXSyDm3xzm3Az85UBPwE+B259yOywTzX+BnbAN4Kz5ozmY/sN7MNobtP0jZV87MNI3vSEkfxTd/v9xxSWa2Dt9U/u/Ag/gWhzL8nOVxM1sBvOFl8nopN5ufCTEHuBf4+UX7fwzcY2bLQz6qzGxd6AGf45z7NvA3IT8iWU0BXSTNzKwGGHbOTQNbnHPtsxz+58A7Qye6twHvm+3czrlxfBP7o6FT3MmU3R8HPmZmz3Nha93jQFOiU9wsx6XaDbwQjrkX+JRz7gXgefyPiq8DT86W18t4Bj/7VAfQC/z3Rd+vHR+wfxTK5DF80/4q4KehpeNrwJweAxRZjDTbmohkJDPbDbzfOfemdOdFZDFQDV1ERCQLqIYuIiKSBVRDFxERyQIK6CIiIllAAV1ERCQLKKCLiIhkAQV0ERGRLKCALiIikgX+H/lJboJZbH5nAAAAAElFTkSuQmCC\n" - }, - "metadata": { - "needs_background": "light" - } - } - ], "source": [ "plt.figure(figsize=(8, 6))\n", "plot_progressive_loss(loss_list_vanilla, 'VanillaVW')\n", "plot_progressive_loss(loss_list_autovw_ni, 'AutoVW:NI')\n", "plot_progressive_loss(loss_list_autovw_nilr, 'AutoVW:NI+LR')\n", "plt.show()" - ] + ], + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": [ + "
" + ], + "image/svg+xml": "\n\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfQAAAFzCAYAAADIY/vqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nOzdd3hc1Z3/8feZURn1LlmWZUvuveNOMSU2HQIBnA4hbLLLkpBkFzYbCMnubze7SViSDdmEkAQ2ydoQJxBq6MYGG3DBveIiW5at3stoNHN+f9yRbGPZHlsaj0b6vJ5HjzR37p35jnjwR+fcU4y1FhEREYlurkgXICIiIj2nQBcREekHFOgiIiL9gAJdRESkH1Cgi4iI9AMKdBERkX4gJtIF9ER2drYtKiqKdBkiIiLnxfr166ustTndPRfVgV5UVMS6desiXYaIiMh5YYwpOdVz6nIXERHpBxToIiIi/YACXUREpB+I6nvoIiISGp/PR2lpKW1tbZEuRULg8XgYMmQIsbGxIV+jQBcRGQBKS0tJSUmhqKgIY0yky5HTsNZSXV1NaWkpxcXFIV+nLncRkQGgra2NrKwshXkUMMaQlZV11r0pCnQRkQFCYR49zuW/lQJdRETCbuHChbzyyisnHHvkkUf46le/elav89xzz/GDH/wAgIceeogf/ehHAHzxi19k+fLlp7zuySefZMmSJSccq6qqIicnh7/85S/ccMMNXcf//d//nZEjR3Y9fv7557nuuuvOqs5IUKCLiEjYLVmyhGXLlp1wbNmyZSeF7Jlcd9113H///Wf9/jfeeCOvvfYaLS0tXceWL1/Otddey7x583jvvfe6jq9Zs4bU1FQqKioAWL16NfPmzTvr9zzfFOgiIhJ2N998My+++CLt7e0AHDhwgLKyMpYuXcrMmTOZMGEC3/3ud7vOLyoq4rvf/S7Tp09n0qRJ7Ny5E4AnnniCu++++7Tv9f3vf58LLriAiRMnctddd2GtJTU1lYsvvpjnn3++67zOPyhycnJITU3lo48+AuDw4cPcdNNNrF69GnACff78+b36+wgHjXIXERlgvvf8NraXNfTqa44fnMp3r51wyuczMzOZNWsWL7/8Mtdffz3Lli3jlltu4dvf/jaZmZn4/X4uu+wyNm/ezOTJkwHIzs5mw4YN/PznP+dHP/oRjz/+eEi13H333Tz44IMAfO5zn+OFF17g2muvZcmSJfzhD3/g1ltvpaysjN27d3PppZcCMH/+fFavXo3f72fUqFHMmTOHV155hWuuuYZNmzZxwQUX9PA3FH5qoQe9/v4fefq1n0a6DBGRfuv4bvfO1vHTTz/N9OnTmTZtGtu2bWP79u1d53/yk58EYMaMGRw4cCDk93nrrbeYPXs2kyZN4s0332Tbtm0AXH311bz77rs0NDTw9NNPc9NNN+F2uwGYN28eq1evZvXq1cydO5dZs2bx/vvv8+GHHzJ27Fg8Hk8v/RbCRy30oKc3PcJ+Vz23cE+kSxERCavTtaTD6frrr+fee+9lw4YNtLS0kJmZyY9+9CPWrl1LRkYGX/ziF0+YqhUfHw+A2+2mo6MjpPdoa2vjb//2b1m3bh2FhYU89NBDXa+ZkJDA4sWLeeaZZ1i2bBkPP/xw13Xz58/nv//7v/H7/Xz5y18mJSWFtrY2VqxYERX3z0Et9C4uDBYb6TJERPqt5ORkFi5cyB133MGSJUtoaGggKSmJtLQ0ysvLefnll3v8Hp3hnZ2dTVNT00kj35csWcLDDz9MeXk5c+fO7To+btw4ysrKeOedd5g2bRoAU6dO5Re/+EVU3D8HBfpxDAFN0RQRCaslS5awadMmlixZwpQpU5g2bRpjx47l05/+dK8EZ3p6Ol/+8peZOHEiixYtOune9xVXXEFZWRm33nrrCXO9jTHMnj2brKysruVW586dy759+6KmhW6sjd5W6cyZM21v7Yf+t49dxHZ3NSu+tK1XXk9EpC/ZsWMH48aNi3QZcha6+29mjFlvrZ3Z3flqoQcZXOpwFxGRqKVADzJGXe4iIhK9FOhBzqA4ERGR6KRADzK4CES6CBERkXOkQA9yGZe63EVEJGop0IMMRi10ERGJWgr0ICfQ1UQXEQmnZ599FmNM12Yrp/PII4+csDtad26//XZ++ctfnvQeV155Jffeey+PPPJI1/FFixZx5513dj3+5je/ecJqceBsyZqYmNi10xo4C+J093Nfo0APchlNWxMRCbelS5eyYMECli5desZzQwn0023L2rnhCkAgEKCqqqprXXc49bao2dnZ/PjHPw7l4/QpCvQgo5XiRETCqqmpiXfeeYdf//rXXSG8YsUKrrnmmq5z7r77bp544gl++tOfUlZWxsKFC1m4cCHg/DEwadIkJk6cyH333QfAZZddxs6dOzly5AgAzc3NvP7669xwww3MmzePNWvWALBt2zYmTpxISkoKtbW1eL1eduzYwfTp00+q84477uCpp56ipqYmrL+P3qbNWYJcxq176CIyMLx8Pxzd0ruvOWgSXPmD057yl7/8hcWLFzN69GiysrJYv379Kc+95557ePjhh3nrrbfIzs6mrKyM++67j/Xr15ORkcEnPvEJnn32WW644QZuuukmnn76ab72ta/x/PPPc8kll5CamkpqaioxMTEcPHiwaxe1w4cPs2bNGtLS0pg0aRJxcXE8+OCDzJw5k+uuuw5wutXvuOMOfvKTn/C9732vV39N4aQWepCzsIwh4PdHuhQRkX5p6dKl3HbbbQDcdtttIXW7d1q7di2XXHIJOTk5xMTE8JnPfIaVK1cC3W/L2unj26LOnTu363Hn2vHf//73u8K80z333MOTTz5JY2Njjz7z+aQWepAr+LdNwAZw4Y5wNSIiYXSGlnQ41NTU8Oabb7JlyxaMMfj9fowxXH/99QQCx/pHj98+NVTz5s3jyJEjbNq0idWrV59wT73zPvqWLVuYOHEihYWF/PjHPyY1NZXbb7/9lK+Znp7Opz/9aR599NGzridS1EIPMsFfRXtHe4QrERHpf5YvX87nPvc5SkpKOHDgAIcOHaK4uJhAIMD27dvxer3U1dXxxhtvdF2TkpLS1UKeNWsWb7/9NlVVVfj9fpYuXcrFF18MOD2st956K1/4whe48sor8Xg8Xa8xb948XnjhBTIzM3G73WRmZlJXV8eaNWvOuIvaN77xDX75y1+GvBd7pCnQg4xxfhV+vy/ClYiI9D9Lly7lxhtvPOHYTTfdxLJly7jllluYOHEit9xyS9de5AB33XUXixcvZuHCheTn5/ODH/yAhQsXMmXKFGbMmMH111/fde7x27Ieb9KkSVRVVTFnzpwTjqWlpZGdnQ3Agw8+yHPPPXdSzdnZ2dx44414vd5e+R2Em7ZPDfruk7fyZ7az6pNvkZ6S3SuvKSLSV2j71Oij7VPPkSu40b1PXe4iIhKFFOhBxjgD4fwa5S4iIlFIgR7UOcrd16F76CIiEn0U6EGdLfRAIDpGM4qIiBxPgR6ke+giIhLNFOhBndPWAlYtdBERiT4K9CCXBsWJiITdQNo+9ZJLLuHjU6tXrFhBWloaU6dOZezYsXzrW98K+fXORIEe5Aq20H1aWEZEJGz64/apTzzxBA899FDI51944YVs3LiRDz/8kBdeeIF333035GtPR4Ee5NKgOBGRsNL2qSdKSEhg6tSpHD58uFdeT5uzBHW20KNlzV4RkXP1Hx/8BztrztzlfTbGZo7lvln3nfYcbZ96otraWvbs2cNFF13UK6+nFnqQWugiIuHVn7ZPra6uZurUqUydOpUHH3yQX/ziF12Pt2w5/V7zq1atYsqUKRQUFLBo0SIGDRoU8u/hdNRCD+oc5d4R0D10EenfztSSDof+tn1qVlYWGzduBJx76AcOHAj5PvqFF17ICy+8wP79+5kzZw633HILU6dODf0Dn4Ja6EEuV2cLXaPcRUR6m7ZPPVlxcTH3338///Ef/9Err6dAD3IZp7PCr0AXEel1A3X71KuvvpohQ4YwZMgQPvWpT530/Fe+8hVWrlzJgQMHzvk9Omn71KBfPvttflb/PD8Z910unXVzr7ymiEhfoe1To4+2Tz1Hbpda6CIiEr0U6EFdK8VplLuIiEQhBXqQyxVcy10tdBERiUIK9KDOFrqmrYlIfxXNY6YGmnP5b6VAD+oMdKsWuoj0Qx6Ph+rqaoV6FLDWUl1dfcL0u1BoYZkgV3BQXMAGznCmiEj0GTJkCKWlpVRWVka6FAmBx+NhyJAhZ3WNAj2oay13vwbFiUj/ExsbS3FxcaTLkDBSl3uQ2xULQMAq0EVEJPoo0IM6R7lrHrqIiESjPtPlboy5AbgaSAV+ba199Xy+f0zwHrq1CnQREYk+YW2hG2N+Y4ypMMZs/djxxcaYXcaYj4wx9wNYa5+11n4Z+Apwazjr6rZWlxaWERGR6BXuLvcngMXHHzDGuIFHgSuB8cASY8z44075TvD588rduduaWugiIhKFwhro1tqVQM3HDs8CPrLW7rPWtgPLgOuN4z+Al621G071msaYu4wx64wx63pz+kXXoLiA5miKiEj0icSguALg0HGPS4PH/h64HLjZGPOVU11srX3MWjvTWjszJyen14rqnLamUe4iIhKN+sygOGvtT4GfRur9Y9yd09bU5S4iItEnEi30w0DhcY+HBI9FlKtrUJxWihMRkegTiUBfC4wyxhQbY+KA24DnIlDHCTpb6Jq2JiIi0Sjc09aWAmuAMcaYUmPMl6y1HcDdwCvADuBpa+22cNYRCpemrYmISBQL6z10a+2SUxx/CXgpnO99tjoD3WpzFhERiUJa+jUoxhUHaLc1ERGJTgr0IJdbC8uIiEj0UqAHHRsUpxa6iIhEHwV6kDu4OYtfLXQREYlCURnoxphrjTGP1dfX99prxsZ07ramFrqIiESfqAx0a+3z1tq70tLSeu01XcYJdN1DFxGRaBSVgR4ObndnoGtzFhERiT4K9KAYtzNtTV3uIiISjRToQe7gtLVV3lPu3CoiItJnKdCDYmOcaWtlsSbClYiIiJw9BXpQjCs20iWIiIicMwV6UEyMAl1ERKKXAj3I7Vagi4hI9FKgB8W4wrrxnIiISFgp0INi3Ap0ERGJXlEZ6OFY+rVztzUREZFoFJWBHo6lX48X8Gv5VxERiS5RGejh1uptiXQJIiIiZ0WB3o0Wb2OkSxARETkrCvRutLU1R7oEERGRs6JA70ZruwJdRESiiwK9G226hy4iIlFGgd4NBbqIiEQbBXo3vD4FuoiIRBcFejfa2lsjXYKIiMhZUaAfZ77XWaimvUOBLiIi0UWBfpyLht0IQLuvLcKViIiInB0F+nHiYjwAtHco0EVEJLpEZaCHY3MWgLhYJ9B9Hd5efV0REZFwi8pAD9fmLPGxCQC0K9BFRCTKRGWgh0tnoPv86nIXEZHookA/TnxcZ6C3R7gSERGRs3PGQDfG/KcxJtUYE2uMecMYU2mM+ez5KO58i49LAqDDry53ERGJLqG00D9hrW0ArgEOACOBfwhnUZFyrMtdLXQREYkuoQR6TPD71cAfrbW9O7S8D0kIdrn/b/vqCFciIiJydmLOfAovGGN2Aq3AV40xOUC/HDXm8SRHugQREZFzcsYWurX2fmAeMNNa6wOagevDXVgkJHqSIl2CiIjIOQllUNynAJ+11m+M+Q7we2Bw2CuLgMT4lEiXICIick5CuYf+gLW20RizALgc+DXwP+EtKzLi4uIjXYKIiMg5CSXQ/cHvVwOPWWtfBOLCV5KIiIicrVAC/bAx5pfArcBLxpj4EK+LSgvbc0gMBCJdhoiIyFkJJZhvAV4BFllr64BM+uk8dACPK4F2YyJdhoiIyFkJZZR7C7AXWGSMuRvItda+GvbKIiTWFU+HMbR5WyJdioiISMhCGeX+NeAPQG7w6/fGmL8Pd2FnqCks26cCxLmdgXH1TbW9/toiIiLhEkqX+5eA2dbaB621DwJzgC+Ht6zTC9f2qXAs0Jtb6nr9tUVERMIllEA3HBvpTvDnfnuTOc7tLP/a1NpvV7gVEZF+KJSlX38LvG+MeSb4+Aacuej9UlxMAngV6CIiEl3OGOjW2oeNMSuABcFDt1trPwxrVRHUueNaS1tThCsREREJ3Sm73I0xmZ1fONum/j74VRI81i95Yp313P9l60ORLUREROQsnK6Fvh6wHLtfboPfTfDn4WGsK2KsdT5mTUy/XTtHRET6oVMGurW2+HwW0ldMG30ZVC2LdBkiIiJnRc3Qj5k8ak6kSxARETlrCvRuzPWmMshnz3yiiIhIH6FA70aciaPNpUAXEZHoEco8dIwxbiDv+POttQfDVVSkxZt42vrt0jkiItIfnTHQg+u2fxcoBzr3FbXA5DDWFVFxbg9tLhc+Xzuxsdr6XURE+r5QWuhfA8ZYa6vDXUxf4XEngIXaxkpyMwsiXY6IiMgZhXIP/RAwoNZBjY9JBKCusTLClYiIiIQmlBb6PmCFMeZFwNt50Fr7cNiqijBPbDL4oK5hwHRKiIhIlAsl0A8Gv+KCX/1eYlwKtEBDc1WkSxEREQlJKJuzfA/AGJMcfNzvdy1JjE8FoLlVe6KLiEh0OOM9dGPMRGPMh8A2YJsxZr0xZkL4SzttTdcaYx6rrw/Prf1kTzoAjW0KdBERiQ6hDIp7DPiGtXaYtXYY8E3gV+Et6/Sstc9ba+9KS0sLy+snJTiB3tLeEJbXFxER6W2hBHqStfatzgfW2hVAUtgq6gPSk7MAaPX2+7sLIiLST4Q0yt0Y8wDwu+Djz+KMfO+3UjsD3adAFxGR6BBKC/0OIAf4c/ArJ3is38pIyQWgrnxnhCsREREJTSij3GuBe85DLX1GRko2AJWJFRGuREREJDSnDHRjzCPW2q8bY57HWbv9BNba68JaWQR1rt8eE9BmdCIiEh1O10LvvGf+o/NRSF8zqj2GRpcCXUREosMpA91auz7441Rr7U+Of84Y8zXg7XAWFmkJxNLqaol0GSIiIiEJpQn6hW6OfbGX6+hzPCaONnPSnQYREZE+6XT30JcAnwaKjTHPHfdUClAT7sIiLcF4aDGWNp8fT6w70uWIiIic1unuoa8GjgDZwI+PO94IbA5nUX1BrImn2RjGPfAS+39wbaTLEREROa3T3UMvAUqAueevnL7DTSJtLhcpNEa6FBERkTMKZXOWOcaYtcaYJmNMuzHGb4zp94ucJwYCAIwo+s8IVyIiInJmoQyK+xmwBNgDJAB3Ao+Gs6i+wMY6I9w/SvBHuBIREZEzC2mitbX2I8BtrfVba38LLA5vWZGXkj880iWIiIiELJRAbzHGxAEbjTH/aYy5N8TrotrtCx4CIL0DAgFNXxMRkb4tlGD+HOAG7gaagULgpnAW1RfkJOZwhS8Ng6WxrSPS5YiIiJxWKJuzlAR/bAW+F95y+paUmCSaqKO2xUtaYmykyxERETml0y0ss4VuNmXpZK2dHJaK+pCU2FR8HWVUNDZQlJ0c6XJERERO6XQt9GuC3/8u+L1zs5bPcpqg70/SPOnQBBW1B6F4cKTLEREROaVT3kO31pYEu9uvsNb+o7V2S/DrPuAT56/EkxljrjXGPFZfXx/W90lPdPZFr6k9GNb3ERER6alQBsUZY8z84x7MC/G6sLHWPm+tvSstLS2s75OdMgiATSUfhfV9REREeuqMg+KALwG/McakAQaoBe4Ia1V9RG7GEACO1hyOcCUiIiKnF8oo9/XAlGCgY60Nbz93H5KZVgBAdnJrhCsRERE5vdONcv+stfb3xphvfOw4ANbah8NcW8SlpQ0FYKdbXe4iItK3ne5eeFLwe8opvvq9xCTnHrrpaOdTv1gd4WpERERO7XTbp/4y+H1ALSZzApeL8V6LFx9rD9RGuhoREZFTOl2X+09Pd6G19p7eL6fvSbax1Lu19KuIiPRtpxsUt/68VdGHpboSWBfbAC5vpEsRERE5pdN1uT95Pgvpq4amZBHwNpI0/GEeXzWVOy/UtqoiItL3nHGBGGNMjjHmR8aYl4wxb3Z+nY/i+gJ/bBwArth6/vXFHRGuRkREpHuhrPj2B2AHUIyz29oBYG0Ya+pTCqyJdAkiIiJnFEqgZ1lrfw34rLVvW2vvAC4Nc119xs25sxnrbccTCJAQG9EVb0VERE4plITyBb8fMcZcbYyZBmSGsaY+JXbe17i6qYU2l4tWfwveDn+kSxIRETlJKIH+r8FlX78JfAt4HLg3rFX1JTFx5Iy4HICEIf/LnvKmCBckIiJyslAC/X1rbb21dqu1dqG1doa19rmwV9aH5CQ7e6HHJO3jmv9+J8LViIiInCyUQH/XGPOqMeZLxpiMsFfUB41ILT7hsbU2QpWIiIh074yBbq0dDXwHmACsN8a8YIz5bNgr60Oyhs4jxR/oevzNP26KYDUiIiInC2nYtrX2A2vtN4BZQA0wsBadyRvP7a3BwXDGx583aH90ERHpW0JZWCbVGPMFY8zLwGrgCE6wDyg5+dMBMDENxLo1N11ERPqWUFrom4CpwPettaOttfdZawfcOu+5wYFxiakf4olx6z66iIj0KafbnKXTcKv0IiUY6K7c12msvpzq5nayk+MjXJWIiIgjlEFxAz7MAUblTgYgwTq/skde3x3JckRERE6gtUxD5EnJ59LmFlpNAHfCAX7/3kFa27VqnIiI9A0K9FDljGOjx+liTyz6BQC3PrYmkhWJiIh0CWWU+2hjzBvGmK3Bx5ONMd8Jf2l9jMvFzLhs58cOJ9gPVDVHsiIREZEuobTQfwX8E8FNWqy1m4HbwllUX/VPtQ0AFFpnPffMpLhIliMiItIllEBPtNZ+8LFjHeEopq/LXvAtbq9r4HBMLLddUMDRhjYCAY0ZFBGRyAsl0KuMMSMAC2CMuRlncZmIMcZca4x5rL6+/vy+8ZRPU5g8hA4DQ3LaafMFGP7tl3hs5d7zW4eIiMjHhBLofwf8EhhrjDkMfB34SlirOgNr7fPW2rvS0tLO7xu7XAzLnQjA9pY/dx3+t5d2sq3sPP9xISIicpxQAr3EWns5kAOMtdYusNaWhLmuPmt4zhQAVpW/eMLxGx59NxLliIiIAKEF+n5jzGPAHKApzPX0edn5U8nvcIYQzBzl7Tru81ueWnuQrYfVUhcRkfMvlEAfC7yO0/W+3xjzM2PMgvCW1YdljezaSjUx/zmevGMWeanONLb7/rSFa/77nUhWJyIiA1QoS7+2WGufttZ+EpgGpAJvh72yvio5l0VeJ9Ctbefi0TlcPi7vhFPueGItFY1tkahOREQGqJBWijPGXGyM+TmwHvAAt4S1qj7uzuoK5ra2Ulu5HYB7Lht1wvNv7qxg2QeHIlGaiIgMUKGsFHcAZ2T7KmCStfYWa+2fwl1YX+aa+hkmetspxY8v4CMv1UNaQuwJ5zz82m6m/8trPPL6bjqCXfQiIiLhEkoLfbK19kZr7VJrrdY6BbjqhxQaD34Drx14jYAN8NevX8ifvjqP+68c23VaTXM7j7y+h5+v0Dx1EREJL3Oq3VGNMf9orf1PY8xPu3veWntPWCsLwcyZM+26desi8t4lz/0d19SuBOD2ibfzjRnf6HrOWkvxP710wvlfvrCYG6YVMGHweZ47LyIi/YYxZr21dmZ3z52uhb4j+H39Kb4GtGGDpnb9/Nutvz3hOWMMP7ltKtdOGdx17Fer9nP1T9+h6P4XeWrtQZavL9WysSIi0mtO2ULv9mRjXECytbYhfCWFLpItdA6+zxde/iwbPB4Avjnjm3xx4hdPOq213c+4B/962peaPzKLqYXpXDw6l+lD0+kIWDyx7nBULSIiUex0LfQzBrox5v9wlnr1A2txpq39xFr7w94u9GxFNNCtpez/ZbGosKDr0JYvbOn21LUHavjVyn0crGlh59HGs3qbm6YP4WuXjWJoVmKPyhURkeh3ukCPCeH68dbaBmPMZ4CXgftxutwjHugRZQyDU4ay7HAptxXk4zanvntxQVEmFxRlAlDf4uPdvVVsLq3nF2+febDcnzaU8qcNpdwxv5hvfmI0SfGh/CcTEZGBJpQW+jZgKvB/wM+stW8bYzZZa6ecjwJPJ6ItdICdL8GyJfwkI43H09NYfu1yClMKSYw9u9b0oZoWclLiOVrfxps7K/jhK7to9fm5cFQ2q/ZUnXDuVZMG8cA148lPS+jNTyIiIlGgp13u9wD3AZuAq4GhwO+ttRf2dqFnK+KBDrD2cf6w6rv8IMtpgS8oWMD/XP4/vfoW60tquOl/1pxwbFJBGnOGZ3LR6BxG5iZT3uAlMc7NsKxE4mN0/11EpD/qUaCf4gVjrLUdPa6sh/pEoAO+v9zN9Lpjq+Ge6l56T63cXckXfvsBZ/pPlp0cx6dmFvK5OcMYnK6WvIhIf9HTFvrXgN8CjcDjOOu532+tfbW3Cz1bfSXQWflDJu3/3xMOvXPbO6TFh2fO+e7yRv7rtd28tr2cjjNMfUv1xDCrOIslswq5aHQOse6QVvsVEZE+qKeBvslaO8UYswj4G+AB4HfW2um9X+rZ6TOBXlvCrp/P4JXkRH6V7oT430z+G+6edvd5eXufP0CMy2CMoa6lnec3lfHsxjLWl9SedG5mUhzL7prD6LyU81KbiIj0np4G+mZr7WRjzE+AFdbaZ4wxH1prp4Wj2LPRZwId4CEnyCcVDwXAYNj0+U0YYyJZFTuONLD0g4P875qSE47HuV0smVXIyNxkphZmsLu8kb2VTVw2LpcZwzIjVK2IiJxOTwP9t0ABUAxMAdw4wT6jtws9W30q0D/4Fbz0LXbExfKF/DxaXU7X9jPXPcPIjJERLs7R7O3gnY+q+JvfnXmhv9F5yfzz1eO5aFR2xP8oERERR08D3YUzbW2ftbbOGJMFFFhrN/d+qWenTwV6p4fSqHC7uWzosQVn+lKoA3T4A+w82shLW46csHHMRaNz2FPeyJH6E/dyT4xzs2BkNldOGsTVkwYTF6P78CIikdDTQDfAZ4Dh1trvG2OGAoOstR/0fqlnp08G+pbl8Kcv8fXcbN5Icuajzx40m8cXPR7hwkLX5vOzZl819y3fTEWj96Tn89M8XDM5n5lFmVQ1eYZZcO0AACAASURBVJlVlMmwrKQTgt7b4ae+xcfRhjY6Apac5Hiqm9vJSYnH77ekJsTQ7g+QmRhHjAbqiYiEpKeB/j9AALjUWjvOGJMBvGqtvaD3Sz07fTLQAVY9jPfN7/FOQgJfz8sBYMnYJdx3wX24XdE1R9xaS12Lj1UfVfHmjnK2ljXwUUXTSee5DEwsSOPCUdksX19KecPJfwicSkp8DInxbuYMz2JaYTpzRmQxJi9FXf0iIh/T00DfYK2dfvxAOK0UF4LgILnvZ2Xwx9RjI8rfXfIuqXGpkaqqV1Q0tvH2rkpe3HKEzMQ4mts72FPRxL7K5pPOzU6OoyA9gVi3C7+1DE5LYOXuSi4ek8MLm4+c8j1S4mO4bFwuM4symVqYzoTBqQp4ERnwehro7wPzgLXBYM/BaaFrlPvp7H0LfncDAIuHDOZw7LE12PtDqHfnw4O1rD1Qw4icZKYUppOdHB/ytYGApaalnT3lTazYXcF7e6vZVd5Imy8AQGFmAgtG5rB44iAWjMzG7VK4i8jA09NA/wxwKzAdeBK4GfiOtfaPvV3o2erTgQ5gLXwvHa+BX6andc1RB7hz0p3cPfXuqOuCP59a2jtYtaeKFbsq2F3e1DWvPjs5jpumD2HBqGwmDE4jMykuwpWKiJwf5xzowRHuc4Aa4DLAAG9Ya3eEo9Cz1ecDHaCtHn7gzE2vdxkWDCvsempyzmSeXPwkMS7toBaKykYvq/dW8Yf3D7K+pBZ/cJU8T6yLCYPTmD8ii6sm5zN2UP/r/RARgZ630PvEIjLdiYpAB1j5Q3jzXwGocrtYOHTICU8/e/2zjEgfEYnKolZ1k5c3dlSwrqSGgzUt1Ld2sOtoAwELxdlJzBmeyaDUBKYUpjFneBaeWPWEiEj062mg/whYA/zZnstOLmEUNYEOsH8lLP8SNFcA8ERqCj/Oyuh6en7BfB6c8yCDkwdHqsKoV9HQxrMbD7N8fSn7KptPWOd+dF4yF47K4YapBUws0AA7EYlOPQ30RiAJ6ADacLrdrbU24v2aURXonV78Jqx15qQ/n5TIt3OzT3j6hxf9kMXFiyNRWb8SCFjK6lt5b18Nmw7Vsb6klu1HGgBnHv30YRmMzUvhYE0LuanxzBiWwbwR2WrJi0if1uvbp/YVURno4IyAf+qz0O7M5340PY1fZBwbMDc+azz/d9X/acBcL6toaOO5TWW8ur2cTYfq8HYEiI9x4e1wRtJnJcUxqziT7OR4Lhqdw0Wjs7W3vIj0KT1toXe3q1o9UBLpPdGjNtDBGQH/6ndgzc+6Dv06LYVHMo91w//XJf/F5cMuj0R1/Z61ltLaVrKS4+gIWNYfqOUP75fw+o6KE84bk5fC9GEZLBiZzezhmWQmxuHSlDkRiZCeBvp7OFPWtgQPTQK2AmnAVyO5L3pUB/rx1j4O7/wE6g/y16RE/uG4bvhbRt/CA3MfiGBxA4u1ln1VzRysaWHV7irWldSw80gj7X6nFe92GWYMy+CysbksGJXNuEGpCngROW96Guh/Bh6w1m4LPh4PfB/4R5yBclN7ud6Q9ZtA77T3TfjdjTQbwycL8ikLLkYzKmMUT13zFLGu2AgXODA1ezv4YH8NHx6qY3NpHSXVLeyvclbFi4txMbs4k6GZiYzNT2XcoBTGD04lMU5TEUWk9/U00Ldaayd2d8wYs1GB3suaKuCFe2HnC5TGuLmy0Nm1bXDSYJ65/hkSYxMjXKAA7Kt0FrpZuaeKDSW1HK5rPeH5ETlJTC3MYMawDMblpzA6L4WkeIW8iPRMTwP9KZyFZZYFD90KZAOfA96J5CYt/TLQO7XWwpbl+F76FtOLh3YdfnLxk0zP625Yg0SSt8PPvspmtpTWs/1IA7uONvLhodqupWvdLsOwzEQGpycwsSCNKUPSWDAqmxSPel1EJHQ9DfQE4G+BBcFD7wI/x5nClmitPXnrrfOkXwd6p/Zm7A9H8Y/pHv6anATAA3Me4JYxt0S4MDkTb4efrYcbOFDVzKbSOo7Ut1FW18ru8kZ8fkuMyzA8J4nx+anMHp7F6LwURuYmk5agkBeR7vV42poxJg4YA1hgl7XW17slnpsBEegA/g7s72/gqy07eTcxAYAbRt7AA3MeIM6tdcyjTWu7n3UlNazZW822sgbWHaihud3f9fyo3GSmD82gMDOBSUPSmV2cqfnxIgL0vIV+Cc6mLAdwFpUpBL5grV3Zu2WevQET6ACBAPzhJv5S/j7fycnqOvzt2d/mxpE34onxRLA46YlAwBlZ/1FFE9vL6ll7wFkEp77V+bvZGCjKSmLByGwWjMpm+tAMspI0fU5kIOppoK8HPm2t3RV8PBpYaq2d0euVhsgYcy1w7ciRI7+8Z8+eSJURGaXrqPvNFXyicDCtLtcJTz1y8cNcOuxyLWvaT1Q0tPHe/hq2ldWz+2gj7++voSXYks9P8zBmUApFWUlMHpLG6LwUxgxKIdbtOsOrikg062mgb7bWTj7TsUgYUC3047W3YP/vUxwqfY8783M5EnPi6OnXb36dvKS8CBUn4dLeEWDDwVo2HqpjS2k9Ww7Xc6S+FZ//2P/DOSnxDM9OIiMxjnH5qYzNT2FqYTp5qerBEekPehrovwX8wO+Dhz4DuK21d/RqledgwAZ6J18rrH+Sqlf/iV+np/L7tGPL69+WN5d/uPQR4uI0za0/a+8IsL+qmZ1HG9h4qI6qpnaO1LVS3dzOgepmOv/3HpGTxMSCNBaOyWVkbjJF2UkkaxqdSNTpaaDHA3/HsVHuq4CfW2u9vVrlORjwgX68QAC7/S88tOIb/DklGYAYa3ly3r8zefS1ES5OIqGxzdfVkv9gfw0bD9VR3dwOONPoBqV6GDsohbkjspgzPIuhWYmkahqdSJ92zoFujHED26y1Y8NVXE8o0Lvh76Bu61P82zsP8nKy0zof7OvgN5f+jILhl0W4OImkDn+ADw/Vsbu8kZLqFg7VtLDraCP7gqveGQOTh6QzY2gGY/NTmDwkjTF5KRqTIdKH9LSF/hfg7621B8NRXE8o0E9v7/Y/seSD79Ia/Af5cX8Ws5c8AwkZZ7hSBpIj9a28v6+GnUcbeX9/NdsON3StXZ/iiWF0nnMfflJBGiNykhmRm6SlbUUipKeBvhKYBnwANHcet9Ze15tFngsF+pnVNh7lb178NDu8lQAMb/fxaNY8hnzyNxGuTPoqf8Cyv6qZ9SU1bD3cwObD9ew80tC1zSzA4DQPEwvSmFiQxtTCdCYWpJGRGKvWvEiY9TTQL+7uuLX27V6orUcU6KHbV7Wdr79yJ/s7GgF4mgLGXf5vUKBlZOXMvB1+9pQ3sbeyidLaVnYdbWTr4fqu7nqApDg3+ekJjMlzNqgZnO6hODuZYZmJpCXEYnHu3YvIuTunQDfGeICvACNxtk79daT3P/84BfrZsdbyXskb3PX2vQAsaGnl9voGZo79FK7LHoCUQRGuUKJNQ5uP9SW1fFTeRGltCwdrWthyuIGqpu7HzKYlxJKVHEdRVhLZyXFkJceTkxxPXqqHQWnxjMpL0cA8kdM410B/CvDhjGq/Eiix1n4tbFWeAwX6uSlpKOGfX/97NjXu7zr26fpGrmtqYkLeDLj1d5CcG8EKJdo1tvn4qKKJA9XNHK33UtPsxeUyNLT6OFrfxtEGL2V1rV2r4R0vLzWezKR4clPiyU/zMDg9gaGZiYzMTaY4O0m71smAdq6BvsVaOyn4cwzwgbW2T/XPKtDPnbWWdUfXcserXzrh+PB2H4+WVzCkcD4seQo0j13CxFqLtyNAdXM7tc3tHKxpYV9lE3srm6lraXfm1Ne3Ud3s5fh/ptISYklPjCUjMY7h2UkUZCRQkJ5ATko82cnxpCbEMjQzUd370i+da6BvOD7AP/64L1Cg946ShhJWlq7klf1/ZVPVZgBur2vga7V1uIfMgmt/AnnjI1ylDFRN3g4O1bRQUt3M3spmjtS3Utvso7LRS2ltC0cb2gh87J+xhFg3Y/NTGJ+fyvjBqYwdlEJhRiI5KfEauCdR7VwD3c+xUe0GSABagj9ba21qtxeeRwr03rerZhffevtbHGg4gMvCP1fXcEtjE8SlwLWPwKSbI12iyAl8/gBH69vYV+W07L2+ADuONrC9rIHtRxpobDs29CfFE8P4/FSyU5x798XZSRRnJzEqL5lBqR6FvfR5Pd4+ta9SoIeHP+Dn0Y2P8qstvwLAWMv/q6zmmuYWTM5YuOJfYORl4NKWntK3WWsprW1lT0Ujh2qcvei3lTVQ0dBGQ1sHTd5jYZ+RGMvI3GQK0hMYnJ7A6LwU0hJiKcpOYkhGgja+kT5BgS7npKq1irvfuJtt1dsAcGP4YWU1VzQ1OSfMvRuGzoVRV0BMfAQrFTl71loqm7zsq2xmd3kj28sa2FvZRFldG+UNbXQc14/vdhkmDE5lYoGzet6ovGRG5iSTnRyvbWzlvFKgS480tDfwX+v/i+W7lwNQ6ErgW0cOcmlLq3OCJw0uuBPmfx08Eb8TI9JjPn+AA1XNVDZ5Ka1tZX9VMxtKatl5tPGEkfkuAymeWDKT4nC7DFlJcWQkxpGRFEtCbAxJ8W5i3S7iYpzWfV2Ljw5/AG9HAG+HnzZfgPTEWBLjYqhobKPZ20FCrJu4GBcZiXHkpDhT+nKD9/4T49zEB18rIymOzMQ4jEG3CgYQBbr0iraONn615Vf8dutv8QWcf9SuTBzG3x4poagmuDLwgnth0CRoa4DETCiYCUk5EBMXwcpFeoe1lspGL7vKGzlQ1eyMwm9qp7LJiwEa2zqoaWmnstFLa7u/awnd47kMxLpdeGLddP7729DmBHmyJ4b4GBcNrT7afIFur/+4GJdhUJqHrGRnql9yfAwZiXFdr1WYmUiqJ4aRucnEx7jJSopTr0IUU6BLrzrSdIQfrvshr5W81nVscvJQ/mXnewz3nWLtofwpkD4Uhl8Ck2+D+OTzUqtIJHX4A3QELIHgv7PxMW5c3bSoO/8dPv64tZaG1g6ONrRxtKGNFm8HLe1+fP4AvoDF7w9wpL4Nn99ysKaF+lZnml+bz0+z10+rz99tTTEuQ6zbRW5qPCmeGAalJpCf5mFQmofi7CTaOwJkJccxOi+lq2dA+g4FuoTF0eajBGyA32z9DU/vehqLZZQnh8+7cxnV4WeYdZPc4YW9b558ccEMp+U+/QswejG4NOBIpLd0/jHgdht2HW2ksc3HgapmfH5LbUs7h+taaWn3U9HQRl2rj5Lqlm5fJynOTUKcm/y0BLKT48hOjmd4TjKD0z1kJcWTmeTcFkhNiKHDb0mMc+sPgDBToEvYlTSU8M0V32RX7a4TjucmOivOfXvmPzI/NhNPyRpY/wQ0V4G33jkpMQvSh8GCr8OoRRDrOc/ViwxsgYCl1ednf1Uz9a0+rIV9VU3sOtrIwZoWjDHUNrdT3tBGRWP3y/oCpCfGUpydxLj8VDISY8lMiicvNZ6hmYkMzUwkPVG33npKgS7nzebKzTy39zlWlq7kSPORk56fmTeT0Rmj+fy4z1Lg98PuV2Hrcjj0vnOCccG0z8HUz0DhLGeTbhHpM+pbfVQEg72y0Utjm4/6Vh/+ABxtaGNfZRM7jjTQ6O3g4/GSnRzP6DxndsDg4Op+mUnOyn4F6YnkpmjWwJko0CWidtXs4oltT7CzZieHmw7T2uGMjk+MSeT6kddz56Q7yY1Jhp0vwtY/wZ5XwfqhcA5MuQ2mLFGrXSTKBAKW+lYfRxvaOBhc6W9PeRN7KpqobvZyNHj//3hxbheD0z0MyUgkxRMT7O73UJiRyKA0D+mJcXT4AxgD6YlxZCXFkZYwsLbtVaBLn9Hub+el/S+xs2Ynz+99nob2BgAKUwq5dcytXD/ietLbGuHD3ztfDaXOhTO+CLO/CrljI1e8iPSaQMBZB6C+1cfhulZKa1sprW3hcK3zc2Wjl4C1VDR68X98bd/jJMS6KchIYEhGgtPyT/OQn55AqieW3FSnJyArKQ5PbP9YCEuBLn3W9urtPL3raZ7b+9yxqXDFV/L58Z9nQvpozJ5XYeMf4KPXwd/u3GO/6j8hoyiyhYvIedHhDzgj/evbqGtx/o2IcRvqWnxUNTkt/UO1LRyua6W6ybnP313+ZybFkZ0cx+D0BAaleijMTGRYViK5KR4yEmNJT4zrWk+gL1OgS5/nD/hZW76WxzY/xpbKLbT525iSM4XPjPsMlw29jLi2Bnjnv+CDxyDQAeNvgPn3wOBpkS5dRPqQ9o4AVU1eGtp8lAe36a1u8lJW30ZVo5fDda2UN7RR1dR+0rVulyEn2RnIl5fqCX4d/7OHQakeUhNiItbNr0CXqFLRUsEL+17g6V1Pc7jpMAD3zriX28bcRmJLDaz4AWxa6gR75ggYexVMvEnhLiIh69zFr6rJS22Lj9pmZ0Gg8uC8/4oGL+WNx3oFjueJdTkBn+IhNzWeQcGwz0yKIz0xlpyUeHJTPGQlx/X6HgAKdIlKHYEOVpet5h/e/gdaOlrI9GRy3Yjr+NToTzHU5YENT8K2v0D5FueCghlw/c91n11Eek2bz98V7kfrnXX+nS9v189HG9po83W/qt/Vk/N59NO9t/O4Al2imrWWdeXr+N3237GydCUBG2B2/mw+N/5zXFhwIaahDLY/C6t+DO3NzmI1Ez8JQy7QjnAiEnbWWhraOqhtbqeu1UdlcEpfRWMbhRmJ3DRjSK+9lwJd+o2jzUf5w44/8Nze56hpq2FoylBuGXMLlxZeSqErHl75Z9j2Z6c7PiUfpn/emdeeXhjp0kVEekyBLv1Oa0crL+9/mT/t/hObqzYDMDt/NteNuI6rcy/AvW8lfPBLOLzeWaymaIEzn33CJzWnXUSilgJd+rVt1dt4o+QNlu5cSpOviYLkAhYWLmRR0SKmEu/MZ9+0DFqqIDkPZt3lDKLLLI506SIiZ0WBLgOCtZY3Dr7BU7ueYn35enwBHyPTR3LbmNu4qvhKUkreg3cehoNrwLhh6Bxwx0H+ZBh5ORRdqKVmRaRPU6DLgFPvref5vc/z9O6n2V+/n+TYZBYXL+az4z7LCK/XWWJ22zNOgFd/5Fw0aDLMvB3GXQdJ2ZH9ACIi3VCgy4BlrWVDxQae2vkUrx18DX/AzyWFl3Db2NuYmz/XWRzC2whb/wxrHoWq4G5xQ+c6y82OuQo8qRH9DCIinRToIkBVaxVPbnuSp3Y9RWtHK0WpRczIm8GUnClcNOQisuLTYd9b8NEbzkYxdSXOhUPnwbTPOAPq4hIj+yFEZEBToIscp7Wjlb/u/yvLdi1jb91evH4vca44rh95PV+e9GXyk/PB3+Hca9/1Eux4AeoPOhcXXQiTb4UJN0J8cmQ/iIgMOAp0kVMI2ACbKjfx3N7neO6j52gPtDMlZwrXDr+W60ZeR0JMAgT8cGAVbFkO+9+GuoPgioGRVzjbu45erKlwInJeKNBFQnCk6Qi/3fZb3jz4JuUt5STEJHBJ4SUsKlrExUMuJsYVA9bCwfdg5wtOwDcdBU+aM0p+2DwYew2kDIr0RxGRfkqBLnIWOgfSPb/3ed44+AZ13jryEvO4bOhlfHLUJxmTOcY5MeCHfStg81Ow720n3DFQMN0ZTDfmKsgbH8mPIiL9jAJd5Bz5/D5Wlq7kj7v/yLtl7wIwLnMcN4++mcXFi0mNC46ADwTg6CbY+ZLTeq/Y7hzPHgNjr3a65YfOjtCnEJH+QoEu0gvqvfU8s+cZnvnoGfbV78Pj9nDFsCu4ZsQ1zB40G/fxG8E0lDld8tufhbIPwQYgPg2KL4Tii2DSpyAxM3IfRkSikgJdpBdZa9lWvY3lu5fz6oFXafQ1khCTwFXFV3Hp0EuZN3iec7+9U0uNs4jN4Q3OoLr6Q85KdcUXwvjrYfwNCncRCYkCXSRMvH4vbx96mzcPvckbJW/Q5m8j05PJRUMu4saRNzItd5qzeM3xjm5xFrLZ/heo2QsYKJwFwxfCiEude/Du2Ih8HhHp2xToIudBU3sT75a9yysHXuHtQ2/THmgnPymf2fmzWVS0iLn5c0/slgenO373q8589yObAOt0zY9Y6IycH70IknMj8nlEpO9RoIucZ03tTbxW8horDq1g1eFV+AI+XMbFjLwZXDb0Mq4qvooMT8aJFzVXw/4VsOd12PMKtFSDKxYKZ8OIS5xR87njtYGMyACmQBeJoGZfM6tKV7GufB0rS1dypPkIMa4YFhQs4Jrh13BJ4SXEu+NPvMhap8W+dbmzUl3tfud4RrGzr/vYqyFvgsJdZIBRoIv0ITuqd/Dc3ud4cd+L1HprSYlN4dKhl7JgyALm5s8lLT7t5IvqD8Puvzqj5vevdI6lD3O2gB1xKYz6hAbWiQwACnSRPsgX8LGmbA0v73+Zt0vfprG9EbdxMy13GgsLF7KoaBF5SXknX9h41Nk8Zt9bcOBdaK1xRs3nT3FWqyu+CPKnQko314pIVFOgi/RxHYEOtlZtZWXpSt44+Ab76vcBcMGgC7hg0AUsGLyA8VnjTx5U5/dB6TrnnvveN+HIZiD4/3TmCGdBm3HXQOEccLnO74cSkV6nQBeJMvvq9/HKgVf46/6/doV7XmIeVxZfyTXDr2F0xuiTp8MB+Frh0AfO6Pn9K51NZfztkFoA466DyZ+CwdN1710kSinQRaLYoYZDrDmyhjcPvsl7R97Db/0UphSysHAhc/LnMCt/1smD6jp5m2DXy7D1T04L3u+F2CQYNtdpvRdfBNmjFfAiUUKBLtJP1LbV8lrJa7x56E0+OPJB13S46bnTGZs5lqm5U5mRN4PshOyTL25rgG1/dlrv+1ZA7QHneEYxTLjBCfghs9Q1L9KHKdBF+qGm9iY2VGzg3cPv8v6R99lbv7frueFpw5meN525+XOZXzCfpNikEy+21gn0vW84G8rsWwHWD6lDnHvuwy9xWu9xH7tORCJKgS4yAHj9XjZWbOTDig/ZWLmRjRUbafY1E+OKYWHhQj4x7BMsHLqw++75lhr46A3Y8kdnvfmONnDHOaPmR14OxRfDoEnqmheJMAW6yADk8/vYVLmJ1w++zkv7Xuqa8z45ZzIzB81kZt5MJmRPINb1sXXjO7xwcA3seQ32vApVu53jaYXO1Lihc50WvBa2ETnvFOgiA1zABvjg6Af8ec+f2Vy5mcNNhwFIjk1mfsF8Li28lIsLLz65ax6g4YgT7HtedUbQN1c4x1MGw5jFzqI2RRdCfPJ5/EQiA5MCXUROUNtWy9qja3nn8DusOryKqtYq4t3xXFJ4CZcPvZz5BfNJiUvp/uK6g7D3LfjoNfjoTfA1AwayRzmD6govcOa9Z4/WADuRXqZAF5FTCtgAH1Z8yMv7X+aVA69Q563DZVxMzp7MgoIFXFx4MaMzRuMy3YRzhxdK3oWD78PB1c7WsK21znMJGU73/LD5kDMGUgY5IR9ziil2InJGCnQRCYk/4GdT5SZWl61mZelKdtbsxGK79nifmDWRaXnTGJE24uRV68AZPV+9Fw6959yH378K6kqOOyHYks+bCPmTne85Y5z787ofL3JGCnQROSeVLZWsObKGVaWreLv0bVo7WgHI9GRywaALuHjIxVw05KLuN5TpVF/qtNybyqFmH1R95DyuP3jsnIQMp/U+bL6zF3zeRG02I9INBbqI9Ji1lj11e9hYsZENFRt4/8j7VLVW4TZuJmRPYErOFCZkTaAotYihqUNPfQ++U2O5s8hN7QGo3AlHN0PZRmc+PEDaUMgbDzljnV3l8iZC2hC15GVAU6CLSK8L2ADbqrbx1qG3+ODoB2yt2oo/GMYGw8iMkUzLmcbMQTMpTismIz6D3MTc7teg7+RthJI1TsAfXu+05Gv3gw04z6cWODvJ5U859pUySCEvA4YCXUTCzh/ws6NmB+vL19Pka2Jz5WY2V26mydfUdU6mJ5Oi1CLGZ41nQvYERqWPYkT6CGJcMad+YV+r05I/utW5L1++Far20LWrXFKuE+w5YyA5F3InwJCZkJAe3g8sEgEKdBGJCH/Az7bqbeyp3UNDewP76vdxoP4AO2t20uZvA5zWfH5SPmMyxzAjbwbzBs9jWOow4txxp35hb5MT7Ec2Hfuq2HGsux4go8jpph88DQqmO98TMsL7gUXCLCoC3RgzHPhnIM1ae3Mo1yjQRaKTL+Bjd81udtbs5EjzEQ40HGBjxUbKW8oBcBs3I9JHMDZzLEWpRYzJHMPk7Mmke07T6rYW2uqcbvpD7zv34yt2QM2xNe5JGey05uOSnMDPnwzJg5xV77QwjkSBiAW6MeY3wDVAhbV24nHHFwM/AdzA49baHxz33HIFusjAY63laPNR3i17l7KmMrZVb2NnzU5q2mq6zhmeNpxpudMYlzmOEekjKEorIiM+o/spdJ1aa51wL9vgdNuXrnVG3hvXsRa9cUHOuGArPh1yxzmD8eKSIXM4xJymt0DkPIpkoF8ENAH/2xnoxhg3sBu4AigF1gJLrLXbg88r0EWkS21bLbtqd7G1aisbyjewsXIjje2NXc8nxyYzPms8U3OnUpRaxPC04YzKGHXqLvvOf/N8rU63fW0JVO+B0nXOQLz2Zgj4jp3vioXMYiiY6ayCN3i6s1HN6f6IEAmTiHa5G2OKgBeOC/S5wEPW2kXBx/8EYK399+Dj0wa6MeYu4C6AoUOHzigpKTnVqSLSD1lrKW8p56O6j9hXt4+DjQf5sOJDdtfu7jon1hXL2MyxTMmZwsxBM5mQNYG8xLzTj7DvFAg48+UrtjnhXrnL2aDm0AfQUuWcE5cCOaOde/KBDojxOMezRjpd+f52p9XvSccZvGcgJc953f/f3r0Hx3mVdxz/PpJWV+u+si35KlvyRZKN7VxIQktdYEgIpKGdlKSlEKCUoZ3p0LRMB4ZOGdphmNIO5dZCaaDhUgI0UApkOpBCmECA3IhjyZItyZJtRb5pJXklW9bF1ukfhfsxnQAAEuBJREFU5+xq7diKZcva1er3mXln3/e877579vi1nj3nPe85Y0NwJuZbCUZe9OewHJ82ecbPdDdxGgrLfEtBSQ1EiiEnz/fmH4/7HyPlq6Gg1M+KV1jmz1NY7jsJVjdAbgSmz6t1IctkWkC/B7jDOffusP024JXAh4GP4mvuDyYC/GxUQxeRhPhEnBNjJ+iN97Ivto/WWCutsVYmzk8AUJpfSmNFI42VjWwo30BDRQNbq7e+/PPyCYk55F981o+EF+uCkX7/SN30eR/UTx32AflK5ESgJArLVvgAPdgNa272zfyW4wPyUK9/hC+1sx/4Y85P+uWSjOQPCRyU1vr8RxuheiOMj8DyJn87om4n5OZBZb1/UiAn4lsf9ChgRpotoM/yrMjCcs4NAu9Ndz5EZHEqLyinvKCcTZWbuH397QBMnp9k3+A+DgwdoGu4i65TXTza8+gFj9IV5RWxLbqNLVVb2LF8B+vK1lFfVk8k96JpZc1803tVPWz//Utn4vw5GD3mg3xuxNemnfPj148eg3g/rLvVB9hEjfvlnAtBOycXMP8eM/8jYvSY/yFxsh3OT0GkCEaO+h8HA/thcsy/b/SYD+JDvX4An/E47PvO5T+zuNoH/Iq1vkVh+dYw2U6uHwegukET72SgjGtynwvV0EVkrpxznBw7SfepbtoH2+k/3U/7YDs98Z5kbT6SE6GxspHKwko2VWyiKdrEjpodrCxZmebczwPnYGIELNe3OLjz0P9r33wf64LRozA17vsVxLohpb/CDPOtCwVlULHG1+qLKn0HwuoG/wOgsBzK6tTXYJ5lWg39GaDRzOqBfuA+4A/TkA8RWYLMjBUlK1hRsoJXrXpVMn3q/BStsVb6T/fTOdxJ53AnsbEYTx97mqnQSa6upI5dK3Zx44obWV++PjkC3hXdm88UZj7YAqwMDx/VvuLSxzrnZ9Qz8zX+WJffHurxI/idPunv9w92+0cGk039QW6+H91v2XJYud037yf6BWgY33l3vXu5PwzsBqLACeDDzrkvmtmdwCfxj619yTn30as5v2roInK9TZ2fonO4kz0De3juxHM8d+K5Cx6ly8/JpyRSAkBDZQMNFQ1UF1ZTlFfEsvxlFOQWUFFQwdqytURyIsQn4uRaLjk5OZybPkdtSe2V38fPVIk4cn7SB/djL/h+AcOH/I+A0yd8x8LJmVsdFJT5VoFIsa/VV2/0HQor1vl7/fklvlOhOvVdYFEMLHM1FNBFZKE55zg8cpgjo0foG+3jxJkTxM763u+HRw7TE++54B79lSjMLUy2ApRESqgtqWV16Wrqy+vZXLWZsvwyKgsqqSqsojS/lEhO5KX3+DPd9Hkf7E+0+c54J/f7bTfte/gPHXxph0LL8TX5qo2+Ob9y3Uy/hNJa/wMg2ujv9S+Rpn0FdBGRBTQ1PcXY1BjxiThnz51ldHKUvtE+4hNxygrKKMgtYGh8iEhOhNHJUY6eOcqyyDJOTZxi2k0Tn4jTN9rHkZEjnHPnLvkZq5atYkP5BpYXL6e+vJ66ZXVsqtxESaSE6sLqxXUbAPzjgqeP+1r9UI/v0DcW8+uDB33AH4/7Yy33wp7/eYW+lh9thOjm8LrJp+UXp+XrXC8K6CIii9D4uXE6hzsZGBvgzLkzDJ4dZOzcGJ1DnRTkFtA+1M7JsZPJeeoTKgoqks/iF+YVsqNmBzesuIGygjJWFK+YfZz8TDY+4p8eyCv09++HenxTfqzTN+3HOv2jg4nZ+cBPw1uzyQf4RKCPbvL38Rfbjx4U0EVEspZzjqNnjnLizAl64j3EJ+IcHjnMnoE9HD19NNlzP8EwaopqWFmyktWlq2moaGBN2ZrkKHuLNtgnTI372nxqkE+sT43NHFdYMRPcE4G+ZrO/h5+bMU90v4QCuojIEjU1PcXQ2SH2DOxhbGqM42eO03+6nyOjR9gX28fk9MzgNHmWR3lBOatLV9MSbaG5upnivGLWlq2lvrx+9mluM930tB8I6CWBvtN32kvIifgOetFNfjz/5VtmBt3JL0lf/oOsC+hmdhdwV0NDw590dXWlOzsiIotWfCKebNY/MHwgWcvfP7T/gqb8wtxCokVRGiobaKpqYkPFBmqKalhbtnZx3rNPdfaU76AX6wxD/Xb5x/SGey9svi9bNfOsfc1mH/CjjX4WvwUaaCfrAnqCaugiItfHuelz9MR76BvtY2xqjI6hDvpG+jg0cojDI4dxKc+bF+UVUVlQyabKTbREW2iJtrC+fD11JXWLO9BPjftAP9TjA3yig17swEwHPbiwU17NFr9esdZvF1XOa5YU0EVEZN6MTY1xZPQIg2cHOTRyiBdHXyR2NkbncCc98Z7kcZUFlTRFm2ip9s33LdEWaopr0pjzeeKc75Q3sN/frx88ONN8P3yYCwbX2XoX3Pu1eftoBXQREVkQo5OjtA+20xvvpX2wnbbBNg6eOsh0aLpeXryclmpfi2+qbmJt2VpWL1u9uGvyqRID6gwf9gG+tPbyY/9fBQV0ERFJm7GpseSc9m2xNvYN7uPwyMzU15UFlbREW9gW3ZZ8rSisSGOOM1emjeUuIiJLSHGkmJ3Ld7Jz+c5kWnwiTvtgO32jfbTF2miNtfLz/p8n782vKV1DS7SF7dHttERbks/Uy+Wphi4iIhnhzNQZ2gfbaY21JoP88TPHAf9IXWNlIxsrNvphccvqaY42s7Z0bfY0118BNbmLiMiiNDA2kAzwe2N76Y33EjsbS96TL8svSzbVb6/ZTnN1M9VF1WnO9fWjgC4iIllj8vwkvfHeZC1+b2zvJTveba/ZngzyxZHsGNNdAV1ERLJa4ln5tlib710fa+PI6BEAciyHjRUbaaluYXPVZpqrmxftPXkFdBERWXKGx4dpjbX6ZaCVjqGO5Fz2eZbH5qrNbK/ZzrboNpqqm1hXti7jh7fNuoCuoV9FRGSunHMMnB3w9+MH9rI3tpe2WFtyiNvC3EK2Vm9lW3Qb22q2sT26ndqS2ozqdJd1AT1BNXQREbkWiSFuDwwdSDbVdwx1JGepK42UsrlqM03VTTRXN7Mtuo3VpekbCEcBXURE5ApNTU/ROdxJ60Ar3ae66Rjs4MDwgWSQryiouGAgnIXsWa+BZURERK5QJCdCc3UzzdXNybSp6SkOnjqYvB/fGmvlyf4nkwPh1JXU0RxtTg5pu7VqK+UF5Quab9XQRURErkJiIJx9sX20DfphbftP9yf315XUcUf9HTxwwwPz9pmqoYuIiMyzkkgJN628iZtW3pRMGx4fpmOog/1D++kY7KAwd+EejVNAFxERmSeVhZXcVncbt9XdtuCfnbPgnygiIiLzTgFdREQkCyigi4iIZAEFdBERkSyggC4iIpIFFNBFRESywKIM6GZ2l5l9IR6PpzsrIiIiGWFRBnTn3Pedc+8pL1/YYfVEREQy1aIM6CIiInIhBXQREZEsoIAuIiKSBRTQRUREsoACuoiISBZY1POhm9kAcHgeTxkFYvN4vqVIZXjtVIbXTmU4P1SO126+y3Cdc67mUjsWdUCfb2b27OUmjpcrozK8dirDa6cynB8qx2u3kGWoJncREZEsoIAuIiKSBRTQL/SFdGcgC6gMr53K8NqpDOeHyvHaLVgZ6h66iIhIFlANXUREJAsooANmdoeZHTCzbjP7QLrzk0nMbI2ZPW5m7Wa2z8zeF9KrzOwxM+sKr5Uh3czs06Es95rZrpRz3R+O7zKz+9P1ndLFzHLN7Hkz+0HYrjezp0JZfdPM8kN6QdjuDvvXp5zjgyH9gJndnp5vkj5mVmFmj5jZfjPrMLNbdS3OjZk9EP4vt5nZw2ZWqGtxdmb2JTM7aWZtKWnzdt2Z2Q1m1hre82kzs6vKqHNuSS9ALnAQ2ADkAy8ATenOV6YsQC2wK6yXAp1AE/Bx4AMh/QPAP4T1O4H/BQy4BXgqpFcBPeG1MqxXpvv7LXBZ/iXwdeAHYftbwH1h/fPAn4b1PwM+H9bvA74Z1pvC9VkA1IfrNjfd32uBy/DLwLvDej5QoWtxTuW3CugFilKuwXfoWnzZcns1sAtoS0mbt+sOeDoca+G9b7iafKqGDjcD3c65HufcJPAN4O405yljOOeOOed+HdZHgQ78H4W78X9cCa9vDut3A19x3q+ACjOrBW4HHnPODTnnhoHHgDsW8KuklZmtBt4IPBi2DXgN8Eg45OIyTJTtI8Brw/F3A99wzk0453qBbvz1uySYWTn+D+sXAZxzk865U+hanKs8oMjM8oBi4Bi6FmflnHsCGLooeV6uu7CvzDn3K+ej+1dSzjUnCug+OPWlbL8Y0uQiobltJ/AUsMI5dyzsOg6sCOuXK8+lXs6fBP4amA7b1cAp59y5sJ1aHsmyCvvj4filXob1wADwH+HWxYNmVoKuxSvmnOsH/gk4gg/kceA5dC1ejfm67laF9YvT50wBXa6ImS0Dvg38hXNuJHVf+FWpxyUuw8zeBJx0zj2X7rwscnn4Zs/POed2AmfwTZ1JuhZnF+7z3o3/cVQHlLC0Wieui0y57hTQoR9Yk7K9OqRJYGYRfDD/T+fcd0LyidBURHg9GdIvV55LuZxfBfyOmR3C39J5DfApfFNcXjgmtTySZRX2lwODLO0yBF9zedE591TYfgQf4HUtXrnXAb3OuQHn3BTwHfz1qWtx7ubruusP6xenz5kCOjwDNIZenvn4jh/fS3OeMka4X/ZFoMM594mUXd8DEr007wf+JyX97aGn5y1APDRL/RB4vZlVhlrC60Na1nPOfdA5t9o5tx5/ff3EOfdW4HHgnnDYxWWYKNt7wvEupN8Xeh7XA434zjRLgnPuONBnZptD0muBdnQtzsUR4BYzKw7/txNlqGtx7ublugv7RszslvBv8vaUc81NunsPZsKC75XYie+p+aF05yeTFuA38E1Je4E9YbkTfx/tx0AX8H9AVTjegH8JZdkK3JhyrnfhO890A+9M93dLU3nuZqaX+wb8H8Fu4L+AgpBeGLa7w/4NKe//UCjbA1xlT9jFvAA7gGfD9fhdfG9hXYtzK8OPAPuBNuCr+J7quhZnL7OH8X0OpvAtRX88n9cdcGP49zgIfJYw6NtcF40UJyIikgXU5C4iIpIFFNBFRESygAK6iIhIFlBAFxERyQIK6CIiIllAAV0kQ5jZx8zst83szWb2wTm+tybMhvW8mf3mLMfttjDb2yzH7DCzO+fy+QvNzA6ZWTTd+RDJJAroIpnjlcCvgN8Cnpjje18LtDrndjrnfnaN+diBH2tARBYRBXSRNDOzfzSzvcBNwC+BdwOfM7O/vcSx683sJ2Ge5R+b2Voz24GfyvFuM9tjZkUXvecO8/OH/xr4vZT0m83sl6FW/wsz2xxGS/w74N5wrnsvddwl8lVrZk+E97QlWgnM7HNm9qz5+bc/knL8odAisSfs32VmPzSzg2b23nDM7nDOR83Puf15M3vJ3ywz+yMzezqc69/Mzzufa2YPhby0mtkDV/WPI7KYpHsEHi1atDjwwfwzQAR4cpbjvg/cH9bfBXw3rL8D+Owlji/Ez/DUiB/B6lvMjFRXBuSF9dcB377UuS533EWf81eEURaBXKA0rFelpP0U2B62DzEz5/Y/40d+KwVqgBMhfTcwjh/FLBc/3eQ9Ke+PAltDmURC+r/ih868AT9VZSJ/Fen+N9ai5XovicH4RSS9dgEvAFvwc85fzq3M1LK/iq+Zz2YLfjKOLgAz+xrwnrCvHPiymTXih/eNXOYcV3LcM8CXzE/k813n3J6Q/hYzew9+prRaoAkfvGFmzoRWYJlzbhQYNbMJM6sI+552zvWEvD+MH4o4MW83+FsNNwDP+GGwKcJPkvF9YIOZfQZ4FPjRLGUkkhUU0EXSKDSXP4SfYSkGFPtk2wPc6pw7ex0//u+Bx51zv2t+rvufXu1xzrknzOzVwBuBh8zsE8DPgPcDNznnhs3sIXyLQcJEeJ1OWU9sJ/42XTw29cXbBnzZOfeSToRm9grgduC9wFvwLRoiWUv30EXSyDm3xzm3Az85UBPwE+B259yOywTzX+BnbAN4Kz5ozmY/sN7MNobtP0jZV87MNI3vSEkfxTd/v9xxSWa2Dt9U/u/Ag/gWhzL8nOVxM1sBvOFl8nopN5ufCTEHuBf4+UX7fwzcY2bLQz6qzGxd6AGf45z7NvA3IT8iWU0BXSTNzKwGGHbOTQNbnHPtsxz+58A7Qye6twHvm+3czrlxfBP7o6FT3MmU3R8HPmZmz3Nha93jQFOiU9wsx6XaDbwQjrkX+JRz7gXgefyPiq8DT86W18t4Bj/7VAfQC/z3Rd+vHR+wfxTK5DF80/4q4KehpeNrwJweAxRZjDTbmohkJDPbDbzfOfemdOdFZDFQDV1ERCQLqIYuIiKSBVRDFxERyQIK6CIiIllAAV1ERCQLKKCLiIhkAQV0ERGRLKCALiIikgX+H/lJboJZbH5nAAAAAElFTkSuQmCC" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "metadata": { + "tags": [] + } }, { "cell_type": "markdown", - "metadata": {}, "source": [ "### AutoVW based on customized VW arguments\n", "You can easily create an AutoVW instance based on customized VW arguments (For now only arguments that are compatible with supervised regression task are well supported). The customized arguments can be passed to AutoVW through init_config and search space." - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": 11, - "metadata": { - "tags": [] - }, + "source": [ + "''' create an AutoVW instance with ustomized VW arguments'''\n", + "# parse the customized VW arguments\n", + "fixed_vw_hp_config = {'alg': 'supervised', 'loss_function': 'classic', 'quiet': ''}\n", + "search_space = fixed_vw_hp_config.copy()\n", + "search_space.update({'interactions': AutoVW.AUTOMATIC,})\n", + "\n", + "autovw_custom = AutoVW(max_live_model_num=5, search_space=search_space) \n", + "loss_list_custom = online_learning_loop(max_iter_num, vw_examples, autovw_custom)\n", + "print('Average final loss of the AutoVW (tuning namespaces) based on customized vw arguments:', sum(loss_list_custom)/len(loss_list_custom))\n" + ], "outputs": [ { "output_type": "stream", @@ -403,24 +415,16 @@ ] } ], - "source": [ - "''' create an AutoVW instance with ustomized VW arguments'''\n", - "# parse the customized VW arguments\n", - "fixed_vw_hp_config = {'alg': 'supervised', 'loss_function': 'classic', 'quiet': ''}\n", - "search_space = fixed_vw_hp_config.copy()\n", - "search_space.update({'interactions': AutoVW.AUTOMATIC,})\n", - "\n", - "autovw_custom = AutoVW(max_live_model_num=5, search_space=search_space) \n", - "loss_list_custom = online_learning_loop(max_iter_num, vw_examples, autovw_custom)\n", - "print('Average final loss of the AutoVW (tuning namespaces) based on customized vw arguments:', sum(loss_list_custom)/len(loss_list_custom))\n" - ] + "metadata": { + "tags": [] + } }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "source": [], "outputs": [], - "source": [] + "metadata": {} } ], "metadata": { diff --git a/notebook/flaml_pytorch_cifar10.ipynb b/notebook/flaml_pytorch_cifar10.ipynb new file mode 100644 index 000000000..9af8cb216 --- /dev/null +++ b/notebook/flaml_pytorch_cifar10.ipynb @@ -0,0 +1,405 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "source": [ + "# Pytorch model tuning example on CIFAR10\n", + "This notebook uses flaml to tune a pytorch model on CIFAR10. It is modified based on [this example](https://docs.ray.io/en/master/tune/examples/cifar10_pytorch.html).\n", + "\n", + "**Requirements.** This notebook requires:" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "!pip install torchvision flaml[blendsearch,ray];" + ], + "outputs": [], + "metadata": { + "tags": [] + } + }, + { + "cell_type": "markdown", + "source": [ + "## Network Specification" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import torch\n", + "import torch.nn as nn\n", + "import torch.nn.functional as F\n", + "import torch.optim as optim\n", + "from torch.utils.data import random_split\n", + "import torchvision\n", + "import torchvision.transforms as transforms\n", + "\n", + "\n", + "class Net(nn.Module):\n", + "\n", + " def __init__(self, l1=120, l2=84):\n", + " super(Net, self).__init__()\n", + " self.conv1 = nn.Conv2d(3, 6, 5)\n", + " self.pool = nn.MaxPool2d(2, 2)\n", + " self.conv2 = nn.Conv2d(6, 16, 5)\n", + " self.fc1 = nn.Linear(16 * 5 * 5, l1)\n", + " self.fc2 = nn.Linear(l1, l2)\n", + " self.fc3 = nn.Linear(l2, 10)\n", + "\n", + " def forward(self, x):\n", + " x = self.pool(F.relu(self.conv1(x)))\n", + " x = self.pool(F.relu(self.conv2(x)))\n", + " x = x.view(-1, 16 * 5 * 5)\n", + " x = F.relu(self.fc1(x))\n", + " x = F.relu(self.fc2(x))\n", + " x = self.fc3(x)\n", + " return x" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "## Data" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "def load_data(data_dir=\"data\"):\n", + " transform = transforms.Compose([\n", + " transforms.ToTensor(),\n", + " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))\n", + " ])\n", + "\n", + " trainset = torchvision.datasets.CIFAR10(\n", + " root=data_dir, train=True, download=True, transform=transform)\n", + "\n", + " testset = torchvision.datasets.CIFAR10(\n", + " root=data_dir, train=False, download=True, transform=transform)\n", + "\n", + " return trainset, testset" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "## Training" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "from ray import tune\n", + "\n", + "def train_cifar(config, checkpoint_dir=None, data_dir=None):\n", + " if \"l1\" not in config:\n", + " logger.warning(config)\n", + " net = Net(2**config[\"l1\"], 2**config[\"l2\"])\n", + "\n", + " device = \"cpu\"\n", + " if torch.cuda.is_available():\n", + " device = \"cuda:0\"\n", + " if torch.cuda.device_count() > 1:\n", + " net = nn.DataParallel(net)\n", + " net.to(device)\n", + "\n", + " criterion = nn.CrossEntropyLoss()\n", + " optimizer = optim.SGD(net.parameters(), lr=config[\"lr\"], momentum=0.9)\n", + "\n", + " # The `checkpoint_dir` parameter gets passed by Ray Tune when a checkpoint\n", + " # should be restored.\n", + " if checkpoint_dir:\n", + " checkpoint = os.path.join(checkpoint_dir, \"checkpoint\")\n", + " model_state, optimizer_state = torch.load(checkpoint)\n", + " net.load_state_dict(model_state)\n", + " optimizer.load_state_dict(optimizer_state)\n", + "\n", + " trainset, testset = load_data(data_dir)\n", + "\n", + " test_abs = int(len(trainset) * 0.8)\n", + " train_subset, val_subset = random_split(\n", + " trainset, [test_abs, len(trainset) - test_abs])\n", + "\n", + " trainloader = torch.utils.data.DataLoader(\n", + " train_subset,\n", + " batch_size=int(2**config[\"batch_size\"]),\n", + " shuffle=True,\n", + " num_workers=4)\n", + " valloader = torch.utils.data.DataLoader(\n", + " val_subset,\n", + " batch_size=int(2**config[\"batch_size\"]),\n", + " shuffle=True,\n", + " num_workers=4)\n", + "\n", + " for epoch in range(int(round(config[\"num_epochs\"]))): # loop over the dataset multiple times\n", + " running_loss = 0.0\n", + " epoch_steps = 0\n", + " for i, data in enumerate(trainloader, 0):\n", + " # get the inputs; data is a list of [inputs, labels]\n", + " inputs, labels = data\n", + " inputs, labels = inputs.to(device), labels.to(device)\n", + "\n", + " # zero the parameter gradients\n", + " optimizer.zero_grad()\n", + "\n", + " # forward + backward + optimize\n", + " outputs = net(inputs)\n", + " loss = criterion(outputs, labels)\n", + " loss.backward()\n", + " optimizer.step()\n", + "\n", + " # print statistics\n", + " running_loss += loss.item()\n", + " epoch_steps += 1\n", + " if i % 2000 == 1999: # print every 2000 mini-batches\n", + " print(\"[%d, %5d] loss: %.3f\" % (epoch + 1, i + 1,\n", + " running_loss / epoch_steps))\n", + " running_loss = 0.0\n", + "\n", + " # Validation loss\n", + " val_loss = 0.0\n", + " val_steps = 0\n", + " total = 0\n", + " correct = 0\n", + " for i, data in enumerate(valloader, 0):\n", + " with torch.no_grad():\n", + " inputs, labels = data\n", + " inputs, labels = inputs.to(device), labels.to(device)\n", + "\n", + " outputs = net(inputs)\n", + " _, predicted = torch.max(outputs.data, 1)\n", + " total += labels.size(0)\n", + " correct += (predicted == labels).sum().item()\n", + "\n", + " loss = criterion(outputs, labels)\n", + " val_loss += loss.cpu().numpy()\n", + " val_steps += 1\n", + "\n", + " # Here we save a checkpoint. It is automatically registered with\n", + " # Ray Tune and will potentially be passed as the `checkpoint_dir`\n", + " # parameter in future iterations.\n", + " with tune.checkpoint_dir(step=epoch) as checkpoint_dir:\n", + " path = os.path.join(checkpoint_dir, \"checkpoint\")\n", + " torch.save(\n", + " (net.state_dict(), optimizer.state_dict()), path)\n", + "\n", + " tune.report(loss=(val_loss / val_steps), accuracy=correct / total)\n", + " print(\"Finished Training\")" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "## Test Accuracy" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "def _test_accuracy(net, device=\"cpu\"):\n", + " trainset, testset = load_data()\n", + "\n", + " testloader = torch.utils.data.DataLoader(\n", + " testset, batch_size=4, shuffle=False, num_workers=2)\n", + "\n", + " correct = 0\n", + " total = 0\n", + " with torch.no_grad():\n", + " for data in testloader:\n", + " images, labels = data\n", + " images, labels = images.to(device), labels.to(device)\n", + " outputs = net(images)\n", + " _, predicted = torch.max(outputs.data, 1)\n", + " total += labels.size(0)\n", + " correct += (predicted == labels).sum().item()\n", + "\n", + " return correct / total" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "## Hyperparameter Optimization" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import numpy as np\n", + "import flaml\n", + "import ray\n", + "\n", + "data_dir = os.path.abspath(\"data\")\n", + "load_data(data_dir) # Download data for all trials before starting the run" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "### Search space" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "max_num_epoch = 100\n", + "config = {\n", + " \"l1\": tune.randint(2, 9), # log transformed with base 2\n", + " \"l2\": tune.randint(2, 9), # log transformed with base 2\n", + " \"lr\": tune.loguniform(1e-4, 1e-1),\n", + " \"num_epochs\": tune.loguniform(1, max_num_epoch),\n", + " \"batch_size\": tune.randint(1, 5) # log transformed with base 2\n", + "}" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "time_budget_s = 600 # time budget in seconds\n", + "gpus_per_trial = 0.5 # number of gpus for each trial; 0.5 means two training jobs can share one gpu\n", + "num_samples = 500 # maximal number of trials\n", + "np.random.seed(7654321)" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "markdown", + "source": [ + "### Launch the tuning" + ], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "source": [ + "import time\n", + "start_time = time.time()\n", + "result = flaml.tune.run(\n", + " tune.with_parameters(train_cifar, data_dir=data_dir),\n", + " config=config,\n", + " metric=\"loss\",\n", + " mode=\"min\",\n", + " low_cost_partial_config={\"num_epochs\": 1},\n", + " max_resource=max_num_epoch,\n", + " min_resource=1,\n", + " report_intermediate_result=True, # only set to True when intermediate results are reported by tune.report\n", + " resources_per_trial={\"cpu\": 1, \"gpu\": gpus_per_trial},\n", + " local_dir='logs/',\n", + " num_samples=num_samples,\n", + " time_budget_s=time_budget_s,\n", + " use_ray=True)\n", + "\n", + "ray.shutdown()" + ], + "outputs": [], + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 13, + "source": [ + "print(f\"#trials={len(result.trials)}\")\n", + "print(f\"time={time.time()-start_time}\")\n", + "best_trial = result.get_best_trial(\"loss\", \"min\", \"all\")\n", + "print(\"Best trial config: {}\".format(best_trial.config))\n", + "print(\"Best trial final validation loss: {}\".format(\n", + " best_trial.metric_analysis[\"loss\"][\"min\"]))\n", + "print(\"Best trial final validation accuracy: {}\".format(\n", + " best_trial.metric_analysis[\"accuracy\"][\"max\"]))\n", + "\n", + "best_trained_model = Net(2**best_trial.config[\"l1\"],\n", + " 2**best_trial.config[\"l2\"])\n", + "device = \"cpu\"\n", + "if torch.cuda.is_available():\n", + " device = \"cuda:0\"\n", + " if gpus_per_trial > 1:\n", + " best_trained_model = nn.DataParallel(best_trained_model)\n", + "best_trained_model.to(device)\n", + "\n", + "checkpoint_path = os.path.join(best_trial.checkpoint.value, \"checkpoint\")\n", + "\n", + "model_state, optimizer_state = torch.load(checkpoint_path)\n", + "best_trained_model.load_state_dict(model_state)\n", + "\n", + "test_acc = _test_accuracy(best_trained_model, device)\n", + "print(\"Best trial test set accuracy: {}\".format(test_acc))" + ], + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "#trials=44\n", + "time=1193.913584947586\n", + "Best trial config: {'l1': 8, 'l2': 8, 'lr': 0.0008818671030627281, 'num_epochs': 55.9513429004283, 'batch_size': 3}\n", + "Best trial final validation loss: 1.0694482081472874\n", + "Best trial final validation accuracy: 0.6389\n", + "Files already downloaded and verified\n", + "Files already downloaded and verified\n", + "Best trial test set accuracy: 0.6294\n" + ] + } + ], + "metadata": {} + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "display_name": "Python 3.8.10 64-bit ('venv': venv)" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.10" + }, + "metadata": { + "interpreter": { + "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" + } + }, + "interpreter": { + "hash": "f7771e6a3915580179405189f5aa4eb9047494cbe4e005b29b851351b54902f6" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} \ No newline at end of file diff --git a/setup.py b/setup.py index 3352f45dc..73757e85b 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,7 @@ install_requires = [ "scipy>=1.4.1", "catboost>=0.23", "scikit-learn>=0.24", -], +] setuptools.setup( @@ -40,12 +40,12 @@ setuptools.setup( "jupyter", "matplotlib==3.2.0", "rgf-python", - "vowpalwabbit", ], "test": [ "flake8>=3.8.4", "pytest>=6.1.1", "coverage>=5.3", + "pre-commit", "xgboost<1.3", "rgf-python", "optuna==2.8.0", @@ -56,11 +56,9 @@ setuptools.setup( "torch==1.8.1", "datasets==1.4.1", "azure-storage-blob", - "statsmodels>=0.12.2" - ], - "blendsearch": [ - "optuna==2.8.0" + "statsmodels>=0.12.2", ], + "blendsearch": ["optuna==2.8.0"], "ray": [ "ray[tune]==1.6.0", "pyyaml<5.3.1", @@ -79,12 +77,9 @@ setuptools.setup( "transformers", "datasets==1.4.1", "tensorboardX<=2.2", - "torch" + "torch", ], - "forecast": [ - "prophet>=1.0.1", - "statsmodels>=0.12.2" - ] + "forecast": ["prophet>=1.0.1", "statsmodels>=0.12.2"], }, classifiers=[ "Programming Language :: Python :: 3", diff --git a/test/test_automl.py b/test/test_automl.py index abe7fef13..adbbb8f38 100644 --- a/test/test_automl.py +++ b/test/test_automl.py @@ -13,17 +13,26 @@ from flaml.data import get_output_from_log from flaml.model import LGBMEstimator, SKLearnEstimator, XGBoostEstimator from rgf.sklearn import RGFClassifier, RGFRegressor from flaml import tune +from flaml.training_log import training_log_reader class MyRegularizedGreedyForest(SKLearnEstimator): - - def __init__(self, task='binary', n_jobs=1, max_leaf=4, - n_iter=1, n_tree_search=1, opt_interval=1, learning_rate=1.0, - min_samples_leaf=1, **params): + def __init__( + self, + task="binary", + n_jobs=1, + max_leaf=4, + n_iter=1, + n_tree_search=1, + opt_interval=1, + learning_rate=1.0, + min_samples_leaf=1, + **params + ): super().__init__(task, **params) - if 'regression' in task: + if "regression" in task: self.estimator_class = RGFRegressor else: self.estimator_class = RGFClassifier @@ -31,36 +40,45 @@ class MyRegularizedGreedyForest(SKLearnEstimator): # round integer hyperparameters self.params = { "n_jobs": n_jobs, - 'max_leaf': int(round(max_leaf)), - 'n_iter': int(round(n_iter)), - 'n_tree_search': int(round(n_tree_search)), - 'opt_interval': int(round(opt_interval)), - 'learning_rate': learning_rate, - 'min_samples_leaf': int(round(min_samples_leaf)) + "max_leaf": int(round(max_leaf)), + "n_iter": int(round(n_iter)), + "n_tree_search": int(round(n_tree_search)), + "opt_interval": int(round(opt_interval)), + "learning_rate": learning_rate, + "min_samples_leaf": int(round(min_samples_leaf)), } @classmethod def search_space(cls, data_size, task): space = { - 'max_leaf': {'domain': tune.qloguniform( - lower=4, upper=data_size, q=1), 'init_value': 4}, - 'n_iter': {'domain': tune.qloguniform( - lower=1, upper=data_size, q=1), 'init_value': 1}, - 'n_tree_search': {'domain': tune.qloguniform( - lower=1, upper=32768, q=1), 'init_value': 1}, - 'opt_interval': {'domain': tune.qloguniform( - lower=1, upper=10000, q=1), 'init_value': 100}, - 'learning_rate': {'domain': tune.loguniform( - lower=0.01, upper=20.0)}, - 'min_samples_leaf': {'domain': tune.qloguniform( - lower=1, upper=20, q=1), 'init_value': 20}, + "max_leaf": { + "domain": tune.qloguniform(lower=4, upper=data_size, q=1), + "init_value": 4, + }, + "n_iter": { + "domain": tune.qloguniform(lower=1, upper=data_size, q=1), + "init_value": 1, + }, + "n_tree_search": { + "domain": tune.qloguniform(lower=1, upper=32768, q=1), + "init_value": 1, + }, + "opt_interval": { + "domain": tune.qloguniform(lower=1, upper=10000, q=1), + "init_value": 100, + }, + "learning_rate": {"domain": tune.loguniform(lower=0.01, upper=20.0)}, + "min_samples_leaf": { + "domain": tune.qloguniform(lower=1, upper=20, q=1), + "init_value": 20, + }, } return space @classmethod def size(cls, config): - max_leaves = int(round(config['max_leaf'])) - n_estimators = int(round(config['n_iter'])) + max_leaves = int(round(config["max_leaf"])) + n_estimators = int(round(config["n_iter"])) return (max_leaves * 3 + (max_leaves - 1) * 4 + 1.0) * n_estimators * 8 @classmethod @@ -77,89 +95,97 @@ def logregobj(preds, dtrain): class MyXGB1(XGBoostEstimator): - '''XGBoostEstimator with logregobj as the objective function - ''' + """XGBoostEstimator with logregobj as the objective function""" def __init__(self, **params): super().__init__(objective=logregobj, **params) class MyXGB2(XGBoostEstimator): - '''XGBoostEstimator with 'reg:squarederror' as the objective function - ''' + """XGBoostEstimator with 'reg:squarederror' as the objective function""" def __init__(self, **params): - super().__init__(objective='reg:squarederror', **params) + super().__init__(objective="reg:squarederror", **params) class MyLargeLGBM(LGBMEstimator): - @classmethod def search_space(cls, **params): return { - 'n_estimators': { - 'domain': tune.lograndint(lower=4, upper=32768), - 'init_value': 32768, - 'low_cost_init_value': 4, + "n_estimators": { + "domain": tune.lograndint(lower=4, upper=32768), + "init_value": 32768, + "low_cost_init_value": 4, }, - 'num_leaves': { - 'domain': tune.lograndint(lower=4, upper=32768), - 'init_value': 32768, - 'low_cost_init_value': 4, + "num_leaves": { + "domain": tune.lograndint(lower=4, upper=32768), + "init_value": 32768, + "low_cost_init_value": 4, }, } -def custom_metric(X_test, y_test, estimator, labels, X_train, y_train, - weight_test=None, weight_train=None, config=None, - groups_test=None, groups_train=None): +def custom_metric( + X_test, + y_test, + estimator, + labels, + X_train, + y_train, + weight_test=None, + weight_train=None, + config=None, + groups_test=None, + groups_train=None, +): from sklearn.metrics import log_loss import time + start = time.time() y_pred = estimator.predict_proba(X_test) pred_time = (time.time() - start) / len(X_test) - test_loss = log_loss(y_test, y_pred, labels=labels, - sample_weight=weight_test) + test_loss = log_loss(y_test, y_pred, labels=labels, sample_weight=weight_test) y_pred = estimator.predict_proba(X_train) - train_loss = log_loss(y_train, y_pred, labels=labels, - sample_weight=weight_train) + train_loss = log_loss(y_train, y_pred, labels=labels, sample_weight=weight_train) alpha = 0.5 return test_loss * (1 + alpha) - alpha * train_loss, { - "test_loss": test_loss, "train_loss": train_loss, "pred_time": pred_time + "test_loss": test_loss, + "train_loss": train_loss, + "pred_time": pred_time, } class TestAutoML(unittest.TestCase): - def test_custom_learner(self): automl = AutoML() - automl.add_learner(learner_name='RGF', - learner_class=MyRegularizedGreedyForest) + automl.add_learner(learner_name="RGF", learner_class=MyRegularizedGreedyForest) X_train, y_train = load_wine(return_X_y=True) settings = { - "time_budget": 10, # total running time in seconds - "estimator_list": ['RGF', 'lgbm', 'rf', 'xgboost'], - "task": 'classification', # task type + "time_budget": 8, # total running time in seconds + "estimator_list": ["RGF", "lgbm", "rf", "xgboost"], + "task": "classification", # task type "sample": True, # whether to subsample training data "log_file_name": "test/wine.log", "log_training_metric": True, # whether to log training metric "n_jobs": 1, } - '''The main flaml automl API''' + """The main flaml automl API""" automl.fit(X_train=X_train, y_train=y_train, **settings) # print the best model found for RGF print(automl.best_model_for_estimator("RGF")) + MyRegularizedGreedyForest.search_space = lambda data_size, task: {} + automl.fit(X_train=X_train, y_train=y_train, **settings) + def test_ensemble(self): automl = AutoML() - automl.add_learner(learner_name='RGF', - learner_class=MyRegularizedGreedyForest) + automl.add_learner(learner_name="RGF", learner_class=MyRegularizedGreedyForest) X_train, y_train = load_wine(return_X_y=True) settings = { "time_budget": 5, # total running time in seconds - "estimator_list": ['rf', 'xgboost', 'catboost'], - "task": 'classification', # task type + "estimator_list": ["rf", "xgboost", "catboost"], + "task": "classification", # task type "sample": True, # whether to subsample training data "log_file_name": "test/wine.log", "log_training_metric": True, # whether to log training metric @@ -170,25 +196,72 @@ class TestAutoML(unittest.TestCase): "n_jobs": 1, } - '''The main flaml automl API''' + """The main flaml automl API""" automl.fit(X_train=X_train, y_train=y_train, **settings) def test_preprocess(self): automl = AutoML() - X = pd.DataFrame({ - 'f1': [1, -2, 3, -4, 5, -6, -7, 8, -9, -10, -11, -12, -13, -14], - 'f2': [3., 16., 10., 12., 3., 14., 11., 12., 5., 14., 20., 16., 15., 11.], - 'f3': ['a', 'b', 'a', 'c', 'c', 'b', 'b', 'b', 'b', 'a', 'b', 1.0, 1.0, 'a'], - 'f4': [True, True, False, True, True, False, False, False, True, True, False, False, True, True], - }) + X = pd.DataFrame( + { + "f1": [1, -2, 3, -4, 5, -6, -7, 8, -9, -10, -11, -12, -13, -14], + "f2": [ + 3.0, + 16.0, + 10.0, + 12.0, + 3.0, + 14.0, + 11.0, + 12.0, + 5.0, + 14.0, + 20.0, + 16.0, + 15.0, + 11.0, + ], + "f3": [ + "a", + "b", + "a", + "c", + "c", + "b", + "b", + "b", + "b", + "a", + "b", + 1.0, + 1.0, + "a", + ], + "f4": [ + True, + True, + False, + True, + True, + False, + False, + False, + True, + True, + False, + False, + True, + True, + ], + } + ) y = pd.Series([0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1]) automl = AutoML() automl_settings = { "time_budget": 6, - "task": 'classification', + "task": "classification", "n_jobs": 1, - "estimator_list": ['catboost', 'lrl2'], + "estimator_list": ["catboost", "lrl2"], "eval_method": "cv", "n_splits": 3, "metric": "accuracy", @@ -201,9 +274,9 @@ class TestAutoML(unittest.TestCase): automl = AutoML() automl_settings = { "time_budget": 2, - "task": 'classification', + "task": "classification", "n_jobs": 1, - "estimator_list": ['lrl2', 'kneighbor'], + "estimator_list": ["lrl2", "kneighbor"], "eval_method": "cv", "n_splits": 3, "metric": "accuracy", @@ -216,9 +289,9 @@ class TestAutoML(unittest.TestCase): automl = AutoML() automl_settings = { "time_budget": 3, - "task": 'classification', + "task": "classification", "n_jobs": 1, - "estimator_list": ['xgboost', 'catboost', 'kneighbor'], + "estimator_list": ["xgboost", "catboost", "kneighbor"], "eval_method": "cv", "n_splits": 3, "metric": "accuracy", @@ -231,9 +304,9 @@ class TestAutoML(unittest.TestCase): automl = AutoML() automl_settings = { "time_budget": 3, - "task": 'classification', + "task": "classification", "n_jobs": 1, - "estimator_list": ['lgbm', 'catboost', 'kneighbor'], + "estimator_list": ["lgbm", "catboost", "kneighbor"], "eval_method": "cv", "n_splits": 3, "metric": "accuracy", @@ -248,18 +321,18 @@ class TestAutoML(unittest.TestCase): def test_custom_metric(self): df, y = load_iris(return_X_y=True, as_frame=True) - df['label'] = y + df["label"] = y automl_experiment = AutoML() automl_settings = { "dataframe": df, - "label": 'label', + "label": "label", "time_budget": 5, - 'eval_method': 'cv', + "eval_method": "cv", "metric": custom_metric, - "task": 'classification', + "task": "classification", "log_file_name": "test/iris_custom.log", "log_training_metric": True, - 'log_type': 'all', + "log_type": "all", "n_jobs": 1, "model_history": True, "sample_weight": np.ones(len(y)), @@ -275,23 +348,29 @@ class TestAutoML(unittest.TestCase): print(automl_experiment.best_estimator) automl_experiment = AutoML() estimator = automl_experiment.get_estimator_from_log( - automl_settings["log_file_name"], record_id=0, - task='multi') + automl_settings["log_file_name"], record_id=0, task="multi" + ) print(estimator) - time_history, best_valid_loss_history, valid_loss_history, \ - config_history, metric_history = get_output_from_log( - filename=automl_settings['log_file_name'], time_budget=6) + ( + time_history, + best_valid_loss_history, + valid_loss_history, + config_history, + metric_history, + ) = get_output_from_log( + filename=automl_settings["log_file_name"], time_budget=6 + ) print(metric_history) def test_binary(self): automl_experiment = AutoML() automl_settings = { "time_budget": 1, - "task": 'binary', + "task": "binary", "log_file_name": "test/breast_cancer.log", "log_training_metric": True, "n_jobs": 1, - "model_history": True + "model_history": True, } X_train, y_train = load_breast_cancer(return_X_y=True) automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) @@ -301,20 +380,19 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 4, - "metric": 'accuracy', - "task": 'classification', + "metric": "accuracy", + "task": "classification", "log_file_name": "test/iris.log", "log_training_metric": True, "n_jobs": 1, - "model_history": True + "model_history": True, } X_train, y_train = load_iris(return_X_y=True, as_frame=as_frame) if as_frame: # test drop column X_train.columns = range(X_train.shape[1]) X_train[X_train.shape[1]] = np.zeros(len(y_train)) - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.classes_) print(automl_experiment.predict(X_train)[:5]) print(automl_experiment.model) @@ -328,8 +406,11 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() duration = automl_experiment.retrain_from_log( log_file_name=automl_settings["log_file_name"], - X_train=X_train, y_train=y_train, - train_full=True, record_id=0) + X_train=X_train, + y_train=y_train, + train_full=True, + record_id=0, + ) print(duration) print(automl_experiment.model) print(automl_experiment.predict_proba(X_train)[:5]) @@ -343,15 +424,34 @@ class TestAutoML(unittest.TestCase): "n_jobs": 1, "model_history": True, } - fake_df = pd.DataFrame({'A': [datetime(1900, 2, 3), datetime(1900, 3, 4), - datetime(1900, 3, 4), datetime(1900, 3, 4), - datetime(1900, 7, 2), datetime(1900, 8, 9)], - 'B': [datetime(1900, 1, 1), datetime(1900, 1, 1), - datetime(1900, 1, 1), datetime(1900, 1, 1), - datetime(1900, 1, 1), datetime(1900, 1, 1)], - 'year_A': [datetime(1900, 1, 2), datetime(1900, 8, 1), - datetime(1900, 1, 4), datetime(1900, 6, 1), - datetime(1900, 1, 5), datetime(1900, 4, 1)]}) + fake_df = pd.DataFrame( + { + "A": [ + datetime(1900, 2, 3), + datetime(1900, 3, 4), + datetime(1900, 3, 4), + datetime(1900, 3, 4), + datetime(1900, 7, 2), + datetime(1900, 8, 9), + ], + "B": [ + datetime(1900, 1, 1), + datetime(1900, 1, 1), + datetime(1900, 1, 1), + datetime(1900, 1, 1), + datetime(1900, 1, 1), + datetime(1900, 1, 1), + ], + "year_A": [ + datetime(1900, 1, 2), + datetime(1900, 8, 1), + datetime(1900, 1, 4), + datetime(1900, 6, 1), + datetime(1900, 1, 5), + datetime(1900, 4, 1), + ], + } + ) y = np.array([0, 1, 0, 1, 0, 0]) automl_experiment.fit(X_train=fake_df, y_train=y, **automl_settings) _ = automl_experiment.predict(fake_df) @@ -361,23 +461,27 @@ class TestAutoML(unittest.TestCase): automl_experiment_macro = AutoML() automl_settings = { "time_budget": 2, - "task": 'classification', + "task": "classification", "log_file_name": "test/micro_macro_f1.log", "log_training_metric": True, "n_jobs": 1, - "model_history": True + "model_history": True, } X_train, y_train = load_iris(return_X_y=True) automl_experiment_micro.fit( - X_train=X_train, y_train=y_train, metric='micro_f1', **automl_settings) + X_train=X_train, y_train=y_train, metric="micro_f1", **automl_settings + ) automl_experiment_macro.fit( - X_train=X_train, y_train=y_train, metric='macro_f1', **automl_settings) + X_train=X_train, y_train=y_train, metric="macro_f1", **automl_settings + ) estimator = automl_experiment_macro.model y_pred = estimator.predict(X_train) y_pred_proba = estimator.predict_proba(X_train) from flaml.ml import norm_confusion_matrix, multi_class_curves + print(norm_confusion_matrix(y_train, y_pred)) from sklearn.metrics import roc_curve, precision_recall_curve + print(multi_class_curves(y_train, y_pred_proba, roc_curve)) print(multi_class_curves(y_train, y_pred_proba, precision_recall_curve)) @@ -393,10 +497,9 @@ class TestAutoML(unittest.TestCase): "n_jobs": 1, "sample_weight": np.ones(len(y_train)), "eval_method": "holdout", - "model_history": True + "model_history": True, } - automl_experiment.fit( - X_train=X_train, y_train=y_train, **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) def test_roc_auc_ovo(self): automl_experiment = AutoML() @@ -407,28 +510,31 @@ class TestAutoML(unittest.TestCase): "log_file_name": "test/roc_auc_ovo.log", "log_training_metric": True, "n_jobs": 1, - "model_history": True + "model_history": True, } X_train, y_train = load_iris(return_X_y=True) - automl_experiment.fit( - X_train=X_train, y_train=y_train, **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) def test_regression(self): automl_experiment = AutoML() automl_settings = { "time_budget": 2, - "task": 'regression', + "task": "regression", "log_file_name": "test/boston.log", "log_training_metric": True, "n_jobs": 1, - "model_history": True + "model_history": True, } X_train, y_train = load_boston(return_X_y=True) n = int(len(y_train) * 9 // 10) - automl_experiment.fit(X_train=X_train[:n], y_train=y_train[:n], - X_val=X_train[n:], y_val=y_train[n:], - **automl_settings) - assert automl_experiment._state.eval_method == 'holdout' + automl_experiment.fit( + X_train=X_train[:n], + y_train=y_train[:n], + X_val=X_train[n:], + y_val=y_train[n:], + **automl_settings + ) + assert automl_experiment._state.eval_method == "holdout" print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -439,29 +545,34 @@ class TestAutoML(unittest.TestCase): automl_experiment.retrain_from_log( task="regression", log_file_name=automl_settings["log_file_name"], - X_train=X_train, y_train=y_train, - train_full=True, time_budget=1) + X_train=X_train, + y_train=y_train, + train_full=True, + time_budget=1, + ) automl_experiment.retrain_from_log( task="regression", log_file_name=automl_settings["log_file_name"], - X_train=X_train, y_train=y_train, - train_full=True, time_budget=0) + X_train=X_train, + y_train=y_train, + train_full=True, + time_budget=0, + ) def test_sparse_matrix_classification(self): automl_experiment = AutoML() automl_settings = { "time_budget": 2, - "metric": 'auto', - "task": 'classification', + "metric": "auto", + "task": "classification", "log_file_name": "test/sparse_classification.log", "split_type": "uniform", "n_jobs": 1, - "model_history": True + "model_history": True, } X_train = scipy.sparse.random(1554, 21, dtype=int) y_train = np.random.randint(3, size=1554) - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.classes_) print(automl_experiment.predict_proba(X_train)) print(automl_experiment.model) @@ -478,17 +589,22 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 2, - "metric": 'mae', - "task": 'regression', + "metric": "mae", + "task": "regression", "log_file_name": "test/sparse_regression.log", "n_jobs": 1, "model_history": True, "keep_search_state": True, "verbose": 0, + "early_stop": True, } - automl_experiment.fit(X_train=X_train, y_train=y_train, - X_val=X_val, y_val=y_val, - **automl_settings) + automl_experiment.fit( + X_train=X_train, + y_train=y_train, + X_val=X_val, + y_val=y_val, + **automl_settings + ) assert automl_experiment._state.X_val.shape == X_val.shape print(automl_experiment.predict(X_train)) print(automl_experiment.model) @@ -504,8 +620,8 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 3, - "metric": 'ap', - "task": 'classification', + "metric": "ap", + "task": "classification", "log_file_name": "test/sparse_classification.log", "estimator_list": ["xgboost"], "log_type": "all", @@ -513,8 +629,7 @@ class TestAutoML(unittest.TestCase): } X_train = scipy.sparse.eye(900000) y_train = np.random.randint(2, size=900000) - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -526,7 +641,7 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 10, - "task": 'regression', + "task": "regression", "log_file_name": "test/boston.log", "log_type": "all", "n_jobs": 1, @@ -535,8 +650,7 @@ class TestAutoML(unittest.TestCase): } X_train, y_train = load_boston(return_X_y=True) try: - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -550,8 +664,8 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 10, - "metric": 'ap', - "task": 'classification', + "metric": "ap", + "task": "classification", "log_file_name": "test/sparse_classification.log", "estimator_list": ["xgboost"], "log_type": "all", @@ -562,8 +676,7 @@ class TestAutoML(unittest.TestCase): X_train = scipy.sparse.eye(900000) y_train = np.random.randint(2, size=900000) try: - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -575,29 +688,29 @@ class TestAutoML(unittest.TestCase): def test_parallel_xgboost_others(self): # use random search as the hpo_method - self.test_parallel_xgboost(hpo_method='random') + self.test_parallel_xgboost(hpo_method="random") def test_random_out_of_memory(self): automl_experiment = AutoML() automl_experiment.add_learner( - learner_name='large_lgbm', learner_class=MyLargeLGBM) + learner_name="large_lgbm", learner_class=MyLargeLGBM + ) automl_settings = { "time_budget": 2, - "metric": 'ap', - "task": 'classification', + "metric": "ap", + "task": "classification", "log_file_name": "test/sparse_classification_oom.log", "estimator_list": ["large_lgbm"], "log_type": "all", "n_jobs": 1, "n_concurrent_trials": 2, - "hpo_method": 'random', + "hpo_method": "random", } X_train = scipy.sparse.eye(900000) y_train = np.random.randint(2, size=900000) try: - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -611,8 +724,8 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 2, - "metric": 'f1', - "task": 'classification', + "metric": "f1", + "task": "classification", "log_file_name": "test/sparse_classification.log", "estimator_list": ["lrl1", "lrl2"], "log_type": "all", @@ -620,8 +733,7 @@ class TestAutoML(unittest.TestCase): } X_train = scipy.sparse.random(3000, 900, density=0.1) y_train = np.random.randint(2, size=3000) - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -635,16 +747,16 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { "time_budget": 1, - 'eval_method': 'holdout', - "task": 'regression', + "eval_method": "holdout", + "task": "regression", "log_file_name": "test/sparse_regression.log", "n_jobs": 1, "model_history": True, "metric": "mse", "sample_weight": np.ones(len(y_train)), + "early_stop": True, } - automl_experiment.fit(X_train=X_train, y_train=y_train, - **automl_settings) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) print(automl_experiment.predict(X_train)) print(automl_experiment.model) print(automl_experiment.config_history) @@ -658,20 +770,25 @@ class TestAutoML(unittest.TestCase): X_val = scipy.sparse.random(100, 900, density=0.0001) y_val = np.random.uniform(size=100) automl_experiment = AutoML() - automl_experiment.add_learner(learner_name='my_xgb1', learner_class=MyXGB1) - automl_experiment.add_learner(learner_name='my_xgb2', learner_class=MyXGB2) + automl_experiment.add_learner(learner_name="my_xgb1", learner_class=MyXGB1) + automl_experiment.add_learner(learner_name="my_xgb2", learner_class=MyXGB2) automl_settings = { "time_budget": 2, - "estimator_list": ['my_xgb1', 'my_xgb2'], - "task": 'regression', - "log_file_name": 'test/regression_xgboost.log', + "estimator_list": ["my_xgb1", "my_xgb2"], + "task": "regression", + "log_file_name": "test/regression_xgboost.log", "n_jobs": 1, "model_history": True, "keep_search_state": True, + "early_stop": True, } - automl_experiment.fit(X_train=X_train, y_train=y_train, - X_val=X_val, y_val=y_val, - **automl_settings) + automl_experiment.fit( + X_train=X_train, + y_train=y_train, + X_val=X_val, + y_val=y_val, + **automl_settings + ) assert automl_experiment._state.X_val.shape == X_val.shape print(automl_experiment.predict(X_train)) print(automl_experiment.model) @@ -684,6 +801,63 @@ class TestAutoML(unittest.TestCase): print(automl_experiment.best_config_train_time) def test_fit_w_starting_point(self, as_frame=True): + automl_experiment = AutoML() + automl_settings = { + "time_budget": 3, + "metric": "accuracy", + "task": "classification", + "log_file_name": "test/iris.log", + "log_training_metric": True, + "n_jobs": 1, + "model_history": True, + } + X_train, y_train = load_iris(return_X_y=True, as_frame=as_frame) + if as_frame: + # test drop column + X_train.columns = range(X_train.shape[1]) + X_train[X_train.shape[1]] = np.zeros(len(y_train)) + automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings) + automl_val_accuracy = 1.0 - automl_experiment.best_loss + print("Best ML leaner:", automl_experiment.best_estimator) + print("Best hyperparmeter config:", automl_experiment.best_config) + print("Best accuracy on validation data: {0:.4g}".format(automl_val_accuracy)) + print( + "Training duration of best run: {0:.4g} s".format( + automl_experiment.best_config_train_time + ) + ) + + starting_points = automl_experiment.best_config_per_estimator + print("starting_points", starting_points) + automl_settings_resume = { + "time_budget": 2, + "metric": "accuracy", + "task": "classification", + "log_file_name": "test/iris_resume.log", + "log_training_metric": True, + "n_jobs": 1, + "model_history": True, + "log_type": "all", + "starting_points": starting_points, + } + new_automl_experiment = AutoML() + new_automl_experiment.fit( + X_train=X_train, y_train=y_train, **automl_settings_resume + ) + + new_automl_val_accuracy = 1.0 - new_automl_experiment.best_loss + print("Best ML leaner:", new_automl_experiment.best_estimator) + print("Best hyperparmeter config:", new_automl_experiment.best_config) + print( + "Best accuracy on validation data: {0:.4g}".format(new_automl_val_accuracy) + ) + print( + "Training duration of best run: {0:.4g} s".format( + new_automl_experiment.best_config_train_time + ) + ) + + def test_fit_w_starting_points_list(self, as_frame=True): automl_experiment = AutoML() automl_settings = { "time_budget": 3, @@ -707,28 +881,38 @@ class TestAutoML(unittest.TestCase): print('Best accuracy on validation data: {0:.4g}'.format(automl_val_accuracy)) print('Training duration of best run: {0:.4g} s'.format(automl_experiment.best_config_train_time)) - starting_points = automl_experiment.best_config_per_estimator - print('starting_points', starting_points) + starting_points = {} + log_file_name = automl_settings['log_file_name'] + with training_log_reader(log_file_name) as reader: + for record in reader.records(): + config = record.config + learner = record.learner + if learner not in starting_points: + starting_points[learner] = [] + starting_points[learner].append(config) + max_iter = sum([len(s) for k, s in starting_points.items()]) automl_settings_resume = { "time_budget": 2, "metric": 'accuracy', "task": 'classification', - "log_file_name": "test/iris_resume.log", + "log_file_name": "test/iris_resume_all.log", "log_training_metric": True, "n_jobs": 1, + "max_iter": max_iter, "model_history": True, "log_type": 'all', "starting_points": starting_points, + "append_log": True, } new_automl_experiment = AutoML() new_automl_experiment.fit(X_train=X_train, y_train=y_train, **automl_settings_resume) new_automl_val_accuracy = 1.0 - new_automl_experiment.best_loss - print('Best ML leaner:', new_automl_experiment.best_estimator) - print('Best hyperparmeter config:', new_automl_experiment.best_config) + # print('Best ML leaner:', new_automl_experiment.best_estimator) + # print('Best hyperparmeter config:', new_automl_experiment.best_config) print('Best accuracy on validation data: {0:.4g}'.format(new_automl_val_accuracy)) - print('Training duration of best run: {0:.4g} s'.format(new_automl_experiment.best_config_train_time)) + # print('Training duration of best run: {0:.4g} s'.format(new_automl_experiment.best_config_train_time)) if __name__ == "__main__": diff --git a/test/test_forecast.py b/test/test_forecast.py index ace4c0eed..8ca2b6059 100644 --- a/test/test_forecast.py +++ b/test/test_forecast.py @@ -5,70 +5,92 @@ from flaml import AutoML def test_forecast_automl(budget=5): # using dataframe import statsmodels.api as sm - data = sm.datasets.co2.load_pandas().data['co2'].resample('MS').mean() - data = data.fillna(data.bfill()).to_frame().reset_index().rename( - columns={'index': 'ds', 'co2': 'y'}) + + data = sm.datasets.co2.load_pandas().data["co2"].resample("MS").mean() + data = ( + data.fillna(data.bfill()) + .to_frame() + .reset_index() + .rename(columns={"index": "ds", "co2": "y"}) + ) num_samples = data.shape[0] time_horizon = 12 split_idx = num_samples - time_horizon df = data[:split_idx] - X_test = data[split_idx:]['ds'] - y_test = data[split_idx:]['y'] + X_test = data[split_idx:]["ds"] + y_test = data[split_idx:]["y"] automl = AutoML() settings = { "time_budget": budget, # total running time in seconds - "metric": 'mape', # primary metric - "task": 'forecast', # task type - "log_file_name": 'CO2_forecast.log', # flaml log file + "metric": "mape", # primary metric + "task": "forecast", # task type + "log_file_name": "test/CO2_forecast.log", # flaml log file "eval_method": "holdout", - "label": ('ds', 'y'), + "label": ("ds", "y"), } - '''The main flaml automl API''' + """The main flaml automl API""" try: automl.fit(dataframe=df, **settings, period=time_horizon) except ImportError: print("not using FBProphet due to ImportError") - automl.fit(dataframe=df, **settings, estimator_list=[ - 'arima', 'sarimax'], period=time_horizon) - ''' retrieve best config and best learner''' - print('Best ML leaner:', automl.best_estimator) - print('Best hyperparmeter config:', automl.best_config) - print(f'Best mape on validation data: {automl.best_loss}') - print(f'Training duration of best run: {automl.best_config_train_time}s') + automl.fit( + dataframe=df, + **settings, + estimator_list=["arima", "sarimax"], + period=time_horizon, + ) + """ retrieve best config and best learner""" + print("Best ML leaner:", automl.best_estimator) + print("Best hyperparmeter config:", automl.best_config) + print(f"Best mape on validation data: {automl.best_loss}") + print(f"Training duration of best run: {automl.best_config_train_time}s") print(automl.model.estimator) - ''' pickle and save the automl object ''' + """ pickle and save the automl object """ import pickle - with open('automl.pkl', 'wb') as f: + + with open("automl.pkl", "wb") as f: pickle.dump(automl, f, pickle.HIGHEST_PROTOCOL) - ''' compute predictions of testing dataset ''' + """ compute predictions of testing dataset """ y_pred = automl.predict(X_test) - print('Predicted labels', y_pred) - print('True labels', y_test) - ''' compute different metric values on testing dataset''' + print("Predicted labels", y_pred) + print("True labels", y_test) + """ compute different metric values on testing dataset""" from flaml.ml import sklearn_metric_loss_score - print('mape', '=', sklearn_metric_loss_score('mape', y_pred, y_test)) + + print("mape", "=", sklearn_metric_loss_score("mape", y_pred, y_test)) from flaml.data import get_output_from_log - time_history, best_valid_loss_history, valid_loss_history, config_history, metric_history = \ - get_output_from_log(filename=settings['log_file_name'], time_budget=budget) + + ( + time_history, + best_valid_loss_history, + valid_loss_history, + config_history, + metric_history, + ) = get_output_from_log(filename=settings["log_file_name"], time_budget=budget) for config in config_history: print(config) print(automl.prune_attr) print(automl.max_resource) print(automl.min_resource) - X_train = df['ds'] - y_train = df['y'] + X_train = df["ds"] + y_train = df["y"] automl = AutoML() try: automl.fit(X_train=X_train, y_train=y_train, **settings, period=time_horizon) except ImportError: print("not using FBProphet due to ImportError") - automl.fit(X_train=X_train, y_train=y_train, **settings, estimator_list=[ - 'arima', 'sarimax'], period=time_horizon) + automl.fit( + X_train=X_train, + y_train=y_train, + **settings, + estimator_list=["arima", "sarimax"], + period=time_horizon, + ) def test_numpy(): - X_train = np.arange('2014-01', '2021-01', dtype='datetime64[M]') + X_train = np.arange("2014-01", "2021-01", dtype="datetime64[M]") y_train = np.random.random(size=72) automl = AutoML() try: @@ -76,8 +98,10 @@ def test_numpy(): X_train=X_train[:60], # a single column of timestamp y_train=y_train, # value for each timestamp period=12, # time horizon to forecast, e.g., 12 months - task='forecast', time_budget=3, # time budget in seconds - log_file_name="test/forecast.log") + task="forecast", + time_budget=3, # time budget in seconds + log_file_name="test/forecast.log", + ) print(automl.predict(X_train[60:])) print(automl.predict(12)) except ValueError: @@ -89,9 +113,11 @@ def test_numpy(): X_train=X_train[:72], # a single column of timestamp y_train=y_train, # value for each timestamp period=12, # time horizon to forecast, e.g., 12 months - task='forecast', time_budget=1, # time budget in seconds - estimator_list=['arima', 'sarimax'], - log_file_name="test/forecast.log") + task="forecast", + time_budget=1, # time budget in seconds + estimator_list=["arima", "sarimax"], + log_file_name="test/forecast.log", + ) print(automl.predict(X_train[72:])) # an alternative way to specify predict steps for arima/sarimax print(automl.predict(12)) diff --git a/test/test_pytorch_cifar10.py b/test/tune/test_pytorch_cifar10.py similarity index 93% rename from test/test_pytorch_cifar10.py rename to test/tune/test_pytorch_cifar10.py index bdc679898..0c4a2ccf5 100644 --- a/test/test_pytorch_cifar10.py +++ b/test/tune/test_pytorch_cifar10.py @@ -1,12 +1,14 @@ '''Require: pip install torchvision ray flaml[blendsearch] ''' -import unittest import os import time +import numpy as np import logging logger = logging.getLogger(__name__) -logger.addHandler(logging.FileHandler('test/tune_pytorch_cifar10.log')) +os.makedirs('logs', exist_ok=True) +logger.addHandler(logging.FileHandler('logs/tune_pytorch_cifar10.log')) +logger.setLevel(logging.INFO) try: @@ -184,7 +186,7 @@ def _test_accuracy(net, device="cpu"): # __main_begin__ def cifar10_main( - method='BlendSearch', num_samples=10, max_num_epochs=100, gpus_per_trial=2 + method='BlendSearch', num_samples=10, max_num_epochs=100, gpus_per_trial=1 ): data_dir = os.path.abspath("test/data") load_data(data_dir) # Download data for all trials before starting the run @@ -192,7 +194,7 @@ def cifar10_main( from flaml import tune else: from ray import tune - if method in ['BlendSearch', 'BOHB', 'Optuna']: + if method in ['BOHB']: config = { "l1": tune.randint(2, 8), "l2": tune.randint(2, 8), @@ -205,28 +207,24 @@ def cifar10_main( "l1": tune.randint(2, 9), "l2": tune.randint(2, 9), "lr": tune.loguniform(1e-4, 1e-1), - "num_epochs": tune.qloguniform(1, max_num_epochs + 1, q=1), + "num_epochs": tune.loguniform(1, max_num_epochs), "batch_size": tune.randint(1, 5) } import ray - time_budget_s = 3600 + time_budget_s = 600 + np.random.seed(7654321) start_time = time.time() if method == 'BlendSearch': result = tune.run( ray.tune.with_parameters(train_cifar, data_dir=data_dir), config=config, - low_cost_partial_config={ - "l1": 2, - "l2": 2, - "num_epochs": 1, - "batch_size": 4, - }, metric="loss", mode="min", + low_cost_partial_config={"num_epochs": 1}, max_resource=max_num_epochs, min_resource=1, report_intermediate_result=True, - resources_per_trial={"cpu": 2, "gpu": gpus_per_trial}, + resources_per_trial={"cpu": 1, "gpu": gpus_per_trial}, local_dir='logs/', num_samples=num_samples, time_budget_s=time_budget_s, @@ -241,14 +239,11 @@ def cifar10_main( scheduler = HyperBandForBOHB(max_t=max_num_epochs) elif 'Optuna' == method: from ray.tune.suggest.optuna import OptunaSearch - algo = OptunaSearch() + algo = OptunaSearch(seed=10) elif 'CFO' == method: from flaml import CFO algo = CFO(low_cost_partial_config={ - "l1": 2, - "l2": 2, "num_epochs": 1, - "batch_size": 4, }) elif 'Nevergrad' == method: from ray.tune.suggest.nevergrad import NevergradSearch @@ -261,7 +256,7 @@ def cifar10_main( grace_period=1) result = tune.run( tune.with_parameters(train_cifar, data_dir=data_dir), - resources_per_trial={"cpu": 2, "gpu": gpus_per_trial}, + resources_per_trial={"cpu": 1, "gpu": gpus_per_trial}, config=config, metric="loss", mode="min", @@ -271,7 +266,7 @@ def cifar10_main( ray.shutdown() logger.info(f"method={method}") - logger.info(f"n_samples={num_samples}") + logger.info(f"#trials={len(result.trials)}") logger.info(f"time={time.time()-start_time}") best_trial = result.get_best_trial("loss", "min", "all") logger.info("Best trial config: {}".format(best_trial.config)) @@ -299,7 +294,7 @@ def cifar10_main( # __main_end__ -gpus_per_trial = 0 # 0.5 on GPU server +gpus_per_trial = 0.5 # on GPU server num_samples = 500 @@ -333,4 +328,4 @@ def _test_cifar10_nevergrad(): if __name__ == "__main__": - unittest.main() + _test_cifar10_bs()