From 6ff0ed434b8f617a2cfe31a1a259bc7c929e7116 Mon Sep 17 00:00:00 2001 From: Chi Wang Date: Mon, 22 Feb 2021 22:10:41 -0800 Subject: [PATCH] v0.2.5 (#30) * test distillbert * import check * complete partial config * None check * init config is not suggested by bo * badge * notebook for lightgbm --- .github/workflows/python-package.yml | 2 +- .gitignore | 4 +- README.md | 5 + flaml/__init__.py | 9 +- flaml/automl.py | 13 + flaml/searcher/blendsearch.py | 10 +- flaml/searcher/flow2.py | 18 +- flaml/version.py | 2 +- notebook/flaml_automl.ipynb | 969 ++++++++++++++ notebook/flaml_azureml.ipynb | 378 +++--- notebook/flaml_demo.ipynb | 1191 ----------------- ...ipynb => flaml_finetune_transformer.ipynb} | 12 +- notebook/flaml_lightgbm.ipynb | 649 +++++++++ test/test_automl.py | 2 +- test/test_distillbert.py | 217 +++ test/test_pytorch_cifar10.py | 3 +- test/test_xgboost2d.py | 11 +- 17 files changed, 2048 insertions(+), 1447 deletions(-) create mode 100644 notebook/flaml_automl.ipynb delete mode 100644 notebook/flaml_demo.ipynb rename notebook/{finetune_transformer_demo.ipynb => flaml_finetune_transformer.ipynb} (98%) create mode 100644 notebook/flaml_lightgbm.ipynb create mode 100644 test/test_distillbert.py diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 7a1862b8e..79b56f42e 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: Python package +name: Build on: push: diff --git a/.gitignore b/.gitignore index e260173d3..5c95aa067 100644 --- a/.gitignore +++ b/.gitignore @@ -146,6 +146,8 @@ dmypy.json # Cython debug symbols cython_debug/ -/catboost_info + +catboost_info notebook/*.pkl notebook/.azureml +mlruns diff --git a/README.md b/README.md index 989f08be0..21a37dd54 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,8 @@ +[![PyPI version](https://badge.fury.io/py/FLAML.svg)](https://badge.fury.io/py/FLAML) +[![Build](https://github.com/microsoft/FLAML/actions/workflows/python-package.yml/badge.svg)](https://github.com/microsoft/FLAML/actions/workflows/python-package.yml) +![Python Version](https://img.shields.io/badge/3.6%20%7C%203.7%20%7C%203.8-blue) +[![Downloads](https://pepy.tech/badge/flaml/month)](https://pepy.tech/project/flaml) + # FLAML - Fast and Lightweight AutoML

diff --git a/flaml/__init__.py b/flaml/__init__.py index 89b9eec72..9f46a8102 100644 --- a/flaml/__init__.py +++ b/flaml/__init__.py @@ -1,5 +1,5 @@ from flaml.searcher import CFO, BlendSearch, FLOW2 -from flaml.automl import AutoML +from flaml.automl import AutoML, logger_formatter from flaml.version import __version__ import logging @@ -7,10 +7,3 @@ import logging logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -# Add the console handler. -_ch = logging.StreamHandler() -logger_formatter = logging.Formatter( - '[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s', - '%m-%d %H:%M:%S') -_ch.setFormatter(logger_formatter) -logger.addHandler(_ch) \ No newline at end of file diff --git a/flaml/automl.py b/flaml/automl.py index 6b43bf125..b6f1c434c 100644 --- a/flaml/automl.py +++ b/flaml/automl.py @@ -25,6 +25,10 @@ from .training_log import training_log_reader, training_log_writer import logging logger = logging.getLogger(__name__) +logger_formatter = logging.Formatter( + '[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s', + '%m-%d %H:%M:%S') + try: import mlflow except: @@ -326,6 +330,10 @@ class AutoML: A numpy array of shape n * 1 - - each element is a predicted class label for an instance. ''' + if self._trained_estimator is None: + warnings.warn( + "No estimator is trained. Please run fit with enough budget.") + return None X_test = self._preprocess(X_test) y_pred = self._trained_estimator.predict(X_test) if y_pred.ndim > 1: y_pred = y_pred.flatten() @@ -837,6 +845,11 @@ class AutoML: if eval_method == 'auto' or self._state.X_val is not None: eval_method = self._decide_eval_method(time_budget) self._state.eval_method = eval_method + if not mlflow or not mlflow.active_run() and not logger.handler: + # Add the console handler. + _ch = logging.StreamHandler() + _ch.setFormatter(logger_formatter) + logger.addHandler(_ch) logger.info("Evaluation method: {}".format(eval_method)) self._retrain_full = retrain_full and (eval_method == 'holdout' and diff --git a/flaml/searcher/blendsearch.py b/flaml/searcher/blendsearch.py index 202e475a1..62701e435 100644 --- a/flaml/searcher/blendsearch.py +++ b/flaml/searcher/blendsearch.py @@ -113,8 +113,9 @@ class BlendSearch(Searcher): self._deadline = config.get('time_budget_s') + time.time() if 'metric_target' in config: self._metric_target = config.get('metric_target') - else: - self._metric, self._mode = metric, mode + else: + if metric: self._metric = metric + if mode: self._mode = mode self._ls.set_search_properties(metric, mode, config) if self._gs is not None: self._gs.set_search_properties(metric, mode, config) @@ -300,11 +301,9 @@ class BlendSearch(Searcher): else: # use init config init_config = self._points_to_evaluate.pop( 0) if self._points_to_evaluate else self._ls.init_config - if init_config==self._ls.init_config: - config = self._ls.complete_config(init_config, + config = self._ls.complete_config(init_config, self._admissible_min, self._admissible_max) # logger.info(f"reset config to {config}") - else: config = init_config config_signature = self._ls.config_signature(config) result = self._result.get(config_signature) if result: # tried before @@ -314,7 +313,6 @@ class BlendSearch(Searcher): self._result[config_signature] = {} else: return None # running but no result yet self._init_used = True - self._trial_proposed_by[trial_id] = 0 # logger.info(f"config={config}") return config diff --git a/flaml/searcher/flow2.py b/flaml/searcher/flow2.py index 912e0961c..3784ef957 100644 --- a/flaml/searcher/flow2.py +++ b/flaml/searcher/flow2.py @@ -190,6 +190,8 @@ class FLOW2(Searcher): self._K = 0 self._iter_best_config = self.trial_count = 1 self._reset_times = 0 + # record intermediate trial cost + self._trial_cost = {} @property def step_lower_bound(self) -> float: @@ -237,7 +239,8 @@ class FLOW2(Searcher): ''' generate a complete config from the partial config input add minimal resource to config if available ''' - if self._reset_times: # not the first time, use random gaussian + if self._reset_times and partial_config==self.init_config: + # not the first time to complete init_config, use random gaussian normalized = self.normalize(partial_config) for key in normalized: # don't change unordered cat choice @@ -258,21 +261,22 @@ class FLOW2(Searcher): normalized[key] = max(l, min(u, normalized[key] + delta)) # use best config for unordered cat choice config = self.denormalize(normalized) + self._reset_times += 1 else: + # first time init_config, or other configs, take as is config = partial_config.copy() for key, value in self.space.items(): if key not in config: config[key] = value - logger.debug(f'before random {config}') + # logger.debug(f'before random {config}') for _, generated in generate_variants({'config': config}): config = generated['config'] break - logger.debug(f'after random {config}') + # logger.debug(f'after random {config}') if self._resource: config[self.prune_attr] = self.min_resource - self._reset_times += 1 return config def create(self, init_config: Dict, obj: float, cost: float) -> Searcher: @@ -442,7 +446,8 @@ class FLOW2(Searcher): if proposed_by == self.incumbent: # proposed by current incumbent and no better self._num_complete4incumbent += 1 - cost = result.get(self.cost_attr) + cost = result.get( + self.cost_attr) if result else self._trial_cost.get(trial_id) if cost: self._cost_complete4incumbent += cost if self._num_complete4incumbent >= 2*self.dim and \ self._num_allowed4incumbent == 0: @@ -483,6 +488,9 @@ class FLOW2(Searcher): self._num_allowed4incumbent = 2 * self.dim self._proposed_by.clear() self._iter_best_config = self.trial_count + cost = result.get(self.cost_attr) + # record the cost in case it is pruned and cost info is lost + self._trial_cost[trial_id] = cost def rand_vector_unit_sphere(self, dim) -> np.ndarray: vec = self._random.normal(0, 1, dim) diff --git a/flaml/version.py b/flaml/version.py index 788da1fb3..fe404ae57 100644 --- a/flaml/version.py +++ b/flaml/version.py @@ -1 +1 @@ -__version__ = "0.2.4" +__version__ = "0.2.5" diff --git a/notebook/flaml_automl.ipynb b/notebook/flaml_automl.ipynb new file mode 100644 index 000000000..dee251790 --- /dev/null +++ b/notebook/flaml_automl.ipynb @@ -0,0 +1,969 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved. \n", + "\n", + "Licensed under the MIT License.\n", + "\n", + "# AutoML with FLAML Library\n", + "\n", + "\n", + "## 1. Introduction\n", + "\n", + "FLAML is a Python library (https://github.com/microsoft/FLAML) designed to automatically produce accurate machine learning models \n", + "with low computational cost. It is fast and cheap. The simple and lightweight design makes it easy \n", + "to use and extend, such as adding new learners. FLAML can \n", + "- serve as an economical AutoML engine,\n", + "- be used as a fast hyperparameter tuning tool, or \n", + "- be embedded in self-tuning software that requires low latency & resource in repetitive\n", + " tuning tasks.\n", + "\n", + "In this notebook, we use one real data example (binary classification) to showcase how to use FLAML library.\n", + "\n", + "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` option:\n", + "```bash\n", + "pip install flaml[notebook]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install flaml[notebook];" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## 2. Classification Example\n", + "### Load data and preprocess\n", + "\n", + "Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "load dataset from ./openml_ds1169.pkl\n", + "Dataset name: airlines\n", + "X_train.shape: (404537, 7), y_train.shape: (404537,);\n", + "X_test.shape: (134846, 7), y_test.shape: (134846,)\n" + ] + } + ], + "source": [ + "from flaml.data import load_openml_dataset\n", + "X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id = 1169, data_dir = './')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Run FLAML\n", + "In the FLAML automl run configuration, users can specify the task type, time budget, error metric, learner list, whether to subsample, resampling strategy type, and so on. All these arguments have default values which will be used if users do not provide them. For example, the default ML learners of FLAML are `['lgbm', 'xgboost', 'catboost', 'rf', 'extra_tree', 'lrl1']`. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "''' import AutoML class from flaml package '''\n", + "from flaml import AutoML\n", + "automl = AutoML()" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "settings = {\n", + " \"time_budget\": 300, # total running time in seconds\n", + " \"metric\": 'accuracy', # primary metrics can be chosen from: ['accuracy','roc_auc','f1','log_loss','mae','mse','r2']\n", + " \"task\": 'classification', # task type \n", + " \"log_file_name\": 'airlines_experiment.log', # flaml log file\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[flaml.automl: 02-22 14:02:59] {844} INFO - Evaluation method: holdout\n", + "[flaml.automl: 02-22 14:02:59] {569} INFO - Using StratifiedKFold\n", + "[flaml.automl: 02-22 14:02:59] {865} INFO - Minimizing error metric: 1-accuracy\n", + "[flaml.automl: 02-22 14:02:59] {885} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'catboost', 'xgboost', 'extra_tree', 'lrl1']\n", + "[flaml.automl: 02-22 14:02:59] {944} INFO - iteration 0 current learner lgbm\n", + "[flaml.automl: 02-22 14:02:59] {1098} INFO - at 0.6s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", + "[flaml.automl: 02-22 14:02:59] {944} INFO - iteration 1 current learner lgbm\n", + "[flaml.automl: 02-22 14:02:59] {1098} INFO - at 0.7s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", + "[flaml.automl: 02-22 14:02:59] {944} INFO - iteration 2 current learner lgbm\n", + "[flaml.automl: 02-22 14:02:59] {1098} INFO - at 0.8s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:02:59] {944} INFO - iteration 3 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:01] {1098} INFO - at 2.2s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:03:01] {944} INFO - iteration 4 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:01] {1098} INFO - at 2.6s,\tbest extra_tree's error=0.4039,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:03:01] {944} INFO - iteration 5 current learner rf\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.0s,\tbest rf's error=0.3982,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 6 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.1s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 7 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.2s,\tbest lgbm's error=0.3662,\tbest lgbm's error=0.3662\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 8 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.4s,\tbest lgbm's error=0.3636,\tbest lgbm's error=0.3636\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 9 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.5s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 10 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 11 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.7s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 12 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:02] {1098} INFO - at 3.8s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:02] {944} INFO - iteration 13 current learner rf\n", + "[flaml.automl: 02-22 14:03:03] {1098} INFO - at 4.1s,\tbest rf's error=0.3982,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:03] {944} INFO - iteration 14 current learner rf\n", + "[flaml.automl: 02-22 14:03:03] {1098} INFO - at 4.6s,\tbest rf's error=0.3941,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:03] {944} INFO - iteration 15 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:04] {1098} INFO - at 4.9s,\tbest extra_tree's error=0.4039,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:04] {944} INFO - iteration 16 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:04] {1098} INFO - at 5.0s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:04] {944} INFO - iteration 17 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:04] {1098} INFO - at 5.4s,\tbest extra_tree's error=0.3964,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:04] {944} INFO - iteration 18 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:04] {1098} INFO - at 5.5s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:04] {944} INFO - iteration 19 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:04] {1098} INFO - at 5.5s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:04] {944} INFO - iteration 20 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:04] {1098} INFO - at 5.7s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:04] {944} INFO - iteration 21 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:06] {1098} INFO - at 6.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:03:06] {944} INFO - iteration 22 current learner catboost\n", + "[flaml.automl: 02-22 14:03:07] {1098} INFO - at 7.9s,\tbest catboost's error=0.3600,\tbest catboost's error=0.3600\n", + "[flaml.automl: 02-22 14:03:07] {944} INFO - iteration 23 current learner catboost\n", + "[flaml.automl: 02-22 14:03:08] {1098} INFO - at 8.9s,\tbest catboost's error=0.3600,\tbest catboost's error=0.3600\n", + "[flaml.automl: 02-22 14:03:08] {944} INFO - iteration 24 current learner catboost\n", + "[flaml.automl: 02-22 14:03:08] {1098} INFO - at 9.6s,\tbest catboost's error=0.3599,\tbest catboost's error=0.3599\n", + "[flaml.automl: 02-22 14:03:08] {944} INFO - iteration 25 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:08] {1098} INFO - at 9.7s,\tbest xgboost's error=0.3787,\tbest catboost's error=0.3599\n", + "[flaml.automl: 02-22 14:03:08] {944} INFO - iteration 26 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:09] {1098} INFO - at 10.2s,\tbest extra_tree's error=0.3964,\tbest catboost's error=0.3599\n", + "[flaml.automl: 02-22 14:03:09] {944} INFO - iteration 27 current learner catboost\n", + "[flaml.automl: 02-22 14:03:09] {1098} INFO - at 10.8s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", + "[flaml.automl: 02-22 14:03:09] {944} INFO - iteration 28 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:10] {1098} INFO - at 10.9s,\tbest xgboost's error=0.3757,\tbest catboost's error=0.3598\n", + "[flaml.automl: 02-22 14:03:10] {944} INFO - iteration 29 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:10] {1098} INFO - at 11.0s,\tbest xgboost's error=0.3756,\tbest catboost's error=0.3598\n", + "[flaml.automl: 02-22 14:03:10] {944} INFO - iteration 30 current learner catboost\n", + "[flaml.automl: 02-22 14:03:10] {1098} INFO - at 11.6s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", + "[flaml.automl: 02-22 14:03:10] {944} INFO - iteration 31 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:11] {1098} INFO - at 12.3s,\tbest lgbm's error=0.3621,\tbest catboost's error=0.3598\n", + "[flaml.automl: 02-22 14:03:11] {944} INFO - iteration 32 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:12] {1098} INFO - at 13.7s,\tbest lgbm's error=0.3568,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:03:12] {944} INFO - iteration 33 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:14] {1098} INFO - at 15.0s,\tbest lgbm's error=0.3568,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:03:14] {944} INFO - iteration 34 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:14] {1098} INFO - at 15.5s,\tbest extra_tree's error=0.3941,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:03:14] {944} INFO - iteration 35 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:16] {1098} INFO - at 17.6s,\tbest lgbm's error=0.3397,\tbest lgbm's error=0.3397\n", + "[flaml.automl: 02-22 14:03:16] {944} INFO - iteration 36 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:19] {1098} INFO - at 20.8s,\tbest lgbm's error=0.3397,\tbest lgbm's error=0.3397\n", + "[flaml.automl: 02-22 14:03:19] {944} INFO - iteration 37 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:20] {1098} INFO - at 21.3s,\tbest extra_tree's error=0.3893,\tbest lgbm's error=0.3397\n", + "[flaml.automl: 02-22 14:03:20] {944} INFO - iteration 38 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:24] {1098} INFO - at 25.1s,\tbest lgbm's error=0.3336,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:24] {944} INFO - iteration 39 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:24] {1098} INFO - at 25.2s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:24] {944} INFO - iteration 40 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:24] {1098} INFO - at 25.4s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:24] {944} INFO - iteration 41 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:24] {1098} INFO - at 25.5s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:24] {944} INFO - iteration 42 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:24] {1098} INFO - at 25.6s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:24] {944} INFO - iteration 43 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:26] {1098} INFO - at 27.4s,\tbest lgbm's error=0.3336,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:26] {944} INFO - iteration 44 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:26] {1098} INFO - at 27.5s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:26] {944} INFO - iteration 45 current learner extra_tree\n", + "[flaml.automl: 02-22 14:03:27] {1098} INFO - at 28.0s,\tbest extra_tree's error=0.3889,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:27] {944} INFO - iteration 46 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:27] {1098} INFO - at 28.1s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:27] {944} INFO - iteration 47 current learner rf\n", + "[flaml.automl: 02-22 14:03:27] {1098} INFO - at 28.5s,\tbest rf's error=0.3941,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:27] {944} INFO - iteration 48 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:27] {1098} INFO - at 28.6s,\tbest xgboost's error=0.3598,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:27] {944} INFO - iteration 49 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:33] {1098} INFO - at 34.8s,\tbest lgbm's error=0.3336,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:33] {944} INFO - iteration 50 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:36] {1098} INFO - at 37.1s,\tbest lgbm's error=0.3336,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:36] {944} INFO - iteration 51 current learner xgboost\n", + "[flaml.automl: 02-22 14:03:36] {1098} INFO - at 37.2s,\tbest xgboost's error=0.3598,\tbest lgbm's error=0.3336\n", + "[flaml.automl: 02-22 14:03:36] {944} INFO - iteration 52 current learner lgbm\n", + "[flaml.automl: 02-22 14:03:55] {1098} INFO - at 56.8s,\tbest lgbm's error=0.3277,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:03:55] {944} INFO - iteration 53 current learner lrl1\n", + "[flaml.automl: 02-22 14:03:56] {1098} INFO - at 57.2s,\tbest lrl1's error=0.4338,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:03:56] {944} INFO - iteration 54 current learner lrl1\n", + "[flaml.automl: 02-22 14:03:56] {1098} INFO - at 57.6s,\tbest lrl1's error=0.4338,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:03:56] {944} INFO - iteration 55 current learner lgbm\n", + "[flaml.automl: 02-22 14:04:27] {1098} INFO - at 88.3s,\tbest lgbm's error=0.3277,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:04:27] {944} INFO - iteration 56 current learner lgbm\n", + "[flaml.automl: 02-22 14:04:39] {1098} INFO - at 100.8s,\tbest lgbm's error=0.3277,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:04:39] {944} INFO - iteration 57 current learner xgboost\n", + "[flaml.automl: 02-22 14:04:40] {1098} INFO - at 101.3s,\tbest xgboost's error=0.3540,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:04:40] {944} INFO - iteration 58 current learner rf\n", + "[flaml.automl: 02-22 14:04:41] {1098} INFO - at 101.9s,\tbest rf's error=0.3894,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:04:41] {944} INFO - iteration 59 current learner rf\n", + "[flaml.automl: 02-22 14:04:41] {1098} INFO - at 102.6s,\tbest rf's error=0.3875,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:04:41] {944} INFO - iteration 60 current learner xgboost\n", + "[flaml.automl: 02-22 14:04:42] {1098} INFO - at 103.1s,\tbest xgboost's error=0.3526,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:04:42] {944} INFO - iteration 61 current learner lgbm\n", + "[flaml.automl: 02-22 14:05:35] {1098} INFO - at 156.0s,\tbest lgbm's error=0.3277,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:05:35] {944} INFO - iteration 62 current learner xgboost\n", + "[flaml.automl: 02-22 14:05:35] {1098} INFO - at 156.3s,\tbest xgboost's error=0.3526,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:05:35] {944} INFO - iteration 63 current learner rf\n", + "[flaml.automl: 02-22 14:05:35] {1098} INFO - at 156.8s,\tbest rf's error=0.3875,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:05:35] {944} INFO - iteration 64 current learner xgboost\n", + "[flaml.automl: 02-22 14:05:37] {1098} INFO - at 158.4s,\tbest xgboost's error=0.3485,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:05:37] {944} INFO - iteration 65 current learner xgboost\n", + "[flaml.automl: 02-22 14:05:38] {1098} INFO - at 159.4s,\tbest xgboost's error=0.3485,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:05:38] {944} INFO - iteration 66 current learner lgbm\n", + "[flaml.automl: 02-22 14:06:26] {1098} INFO - at 207.1s,\tbest lgbm's error=0.3277,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:06:26] {944} INFO - iteration 67 current learner xgboost\n", + "[flaml.automl: 02-22 14:06:33] {1098} INFO - at 214.0s,\tbest xgboost's error=0.3485,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:06:33] {944} INFO - iteration 68 current learner xgboost\n", + "[flaml.automl: 02-22 14:06:38] {1098} INFO - at 219.8s,\tbest xgboost's error=0.3374,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:06:38] {944} INFO - iteration 69 current learner xgboost\n", + "[flaml.automl: 02-22 14:06:49] {1098} INFO - at 230.7s,\tbest xgboost's error=0.3374,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:06:49] {944} INFO - iteration 70 current learner lgbm\n", + "[flaml.automl: 02-22 14:07:01] {1098} INFO - at 242.8s,\tbest lgbm's error=0.3277,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:07:01] {944} INFO - iteration 71 current learner xgboost\n", + "[flaml.automl: 02-22 14:07:12] {1098} INFO - at 253.4s,\tbest xgboost's error=0.3281,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:07:12] {944} INFO - iteration 72 current learner xgboost\n", + "[flaml.automl: 02-22 14:07:16] {1098} INFO - at 257.2s,\tbest xgboost's error=0.3281,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:07:16] {944} INFO - iteration 73 current learner xgboost\n", + "[flaml.automl: 02-22 14:07:34] {1098} INFO - at 275.1s,\tbest xgboost's error=0.3281,\tbest lgbm's error=0.3277\n", + "[flaml.automl: 02-22 14:08:07] {1115} INFO - retrain xgboost for 33.5s\n", + "[flaml.automl: 02-22 14:08:07] {1139} INFO - selected model: LGBMClassifier(colsample_bytree=0.9866769216052313,\n", + " learning_rate=0.07800162809924845, max_bin=1023,\n", + " min_child_weight=13.44078628238488, n_estimators=469,\n", + " num_leaves=203, objective='binary',\n", + " reg_alpha=1.112259151174658e-10, reg_lambda=0.027996096138662233,\n", + " subsample=0.9565303465007353)\n", + "[flaml.automl: 02-22 14:08:07] {899} INFO - fit succeeded\n" + ] + } + ], + "source": [ + "'''The main flaml automl API'''\n", + "automl.fit(X_train = X_train, y_train = y_train, **settings)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Best model and metric" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Best ML leaner: lgbm\nBest hyperparmeter config: {'n_estimators': 469.0, 'max_leaves': 203.0, 'min_child_weight': 13.44078628238488, 'learning_rate': 0.07800162809924845, 'subsample': 0.9565303465007353, 'log_max_bin': 10.0, 'colsample_bytree': 0.9866769216052313, 'reg_alpha': 1.112259151174658e-10, 'reg_lambda': 0.027996096138662233, 'FLAML_sample_size': 364083}\nBest accuracy on validation data: 0.6723\nTraining duration of best run: 19.56 s\n" + ] + } + ], + "source": [ + "''' retrieve best config and best learner'''\n", + "print('Best ML leaner:', automl.best_estimator)\n", + "print('Best hyperparmeter config:', automl.best_config)\n", + "print('Best accuracy on validation data: {0:.4g}'.format(1-automl.best_loss))\n", + "print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "LGBMClassifier(colsample_bytree=0.9866769216052313,\n", + " learning_rate=0.07800162809924845, max_bin=1023,\n", + " min_child_weight=13.44078628238488, n_estimators=469,\n", + " num_leaves=203, objective='binary',\n", + " reg_alpha=1.112259151174658e-10, reg_lambda=0.027996096138662233,\n", + " subsample=0.9565303465007353)" + ] + }, + "metadata": {}, + "execution_count": 6 + } + ], + "source": [ + "automl.model" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "''' pickle and save the best model '''\n", + "import pickle\n", + "with open('best_model.pkl', 'wb') as f:\n", + " pickle.dump(automl.model, f, pickle.HIGHEST_PROTOCOL)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Predicted labels [1 0 1 ... 1 0 0]\nTrue labels [0 0 0 ... 0 1 0]\n" + ] + } + ], + "source": [ + "''' compute predictions of testing dataset ''' \n", + "y_pred = automl.predict(X_test)\n", + "print('Predicted labels', y_pred)\n", + "print('True labels', y_test)\n", + "y_pred_proba = automl.predict_proba(X_test)[:,1]" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "accuracy = 0.6722186790857719\n", + "roc_auc = 0.7254791013665546\n", + "log_loss = 0.6031746076041535\n", + "f1 = 0.5926492544191104\n" + ] + } + ], + "source": [ + "''' compute different metric values on testing dataset'''\n", + "from flaml.ml import sklearn_metric_loss_score\n", + "print('accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))\n", + "print('roc_auc', '=', 1 - sklearn_metric_loss_score('roc_auc', y_pred_proba, y_test))\n", + "print('log_loss', '=', sklearn_metric_loss_score('log_loss', y_pred_proba, y_test))\n", + "print('f1', '=', 1 - sklearn_metric_loss_score('f1', y_pred, y_test))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "See Section 4 for an accuracy comparison with default LightGBM and XGBoost.\n", + "\n", + "### Log history" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 4.0, 'min_child_weight': 2.776007506782275, 'learning_rate': 0.7179196339383696, 'subsample': 0.8746997476758036, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 9.69511928836042e-10, 'reg_lambda': 0.17744769739709204, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 4.0, 'min_child_weight': 2.776007506782275, 'learning_rate': 0.7179196339383696, 'subsample': 0.8746997476758036, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 9.69511928836042e-10, 'reg_lambda': 0.17744769739709204, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 10, 'learning_rate': 0.1, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 10, 'learning_rate': 0.1, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.2, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.2, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.09293377774381106, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.09293377774381106, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 18.0, 'max_leaves': 7.0, 'min_child_weight': 3.6614972724384676, 'learning_rate': 0.34175229608719143, 'subsample': 0.9346496568879574, 'log_max_bin': 8.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.1118080322587527e-09, 'reg_lambda': 0.1584101587833998, 'FLAML_sample_size': 364083}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 18.0, 'max_leaves': 7.0, 'min_child_weight': 3.6614972724384676, 'learning_rate': 0.34175229608719143, 'subsample': 0.9346496568879574, 'log_max_bin': 8.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.1118080322587527e-09, 'reg_lambda': 0.1584101587833998, 'FLAML_sample_size': 364083}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 62.0, 'max_leaves': 44.0, 'min_child_weight': 8.784469833660507, 'learning_rate': 0.17709445119696665, 'subsample': 0.9826350012458567, 'log_max_bin': 8.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 0.21025319396124098, 'FLAML_sample_size': 364083}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 62.0, 'max_leaves': 44.0, 'min_child_weight': 8.784469833660507, 'learning_rate': 0.17709445119696665, 'subsample': 0.9826350012458567, 'log_max_bin': 8.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 0.21025319396124098, 'FLAML_sample_size': 364083}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 209.0, 'max_leaves': 52.0, 'min_child_weight': 3.620089453949904, 'learning_rate': 0.4310996636372654, 'subsample': 0.9303385363256778, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 0.0018042679316819382, 'FLAML_sample_size': 364083}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 209.0, 'max_leaves': 52.0, 'min_child_weight': 3.620089453949904, 'learning_rate': 0.4310996636372654, 'subsample': 0.9303385363256778, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 0.0018042679316819382, 'FLAML_sample_size': 364083}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 469.0, 'max_leaves': 203.0, 'min_child_weight': 13.44078628238488, 'learning_rate': 0.07800162809924845, 'subsample': 0.9565303465007353, 'log_max_bin': 10.0, 'colsample_bytree': 0.9866769216052313, 'reg_alpha': 1.112259151174658e-10, 'reg_lambda': 0.027996096138662233, 'FLAML_sample_size': 364083}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 469.0, 'max_leaves': 203.0, 'min_child_weight': 13.44078628238488, 'learning_rate': 0.07800162809924845, 'subsample': 0.9565303465007353, 'log_max_bin': 10.0, 'colsample_bytree': 0.9866769216052313, 'reg_alpha': 1.112259151174658e-10, 'reg_lambda': 0.027996096138662233, 'FLAML_sample_size': 364083}}\n" + ] + } + ], + "source": [ + "from flaml.data import get_output_from_log\n", + "time_history, best_valid_loss_history, valid_loss_history, config_history, train_loss_history = \\\n", + " get_output_from_log(filename = settings['log_file_name'], time_budget = 60)\n", + "\n", + "for config in config_history:\n", + " print(config)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "

", + "image/svg+xml": "\r\n\r\n\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n\r\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3df7xVdZ3v8dfbA+SxRDRxBo4aNgllmqCk/bLUcqAyJXPKvOWMPSZy+jFWNxxpxqYf11sz59ZkpTHGtbKyHxIizSXIyZDJ8QcQxs9wzAw5mMAYaXQSOXzuH+u7cbNZZ7OEs84+e+/38/E4j7PXd33XWp/lj/053+93re9XEYGZmVmtgxodgJmZDU1OEGZmlssJwszMcjlBmJlZLicIMzPL5QRhZma5nCDM9oOkMyStb3QcZmVygrCmI+khSa9rZAwR8R8RMaGs80uaImmJpCckbZF0h6TzyrqeWR4nCLMckjoaeO0LgZuBG4GjgT8BPga8aT/OJUn+/9z2i//DsZYh6SBJV0r6paT/lvQ9SUdU7b9Z0m8k/S79df7iqn1fk/RlSQskbQfOSi2Vj0hamY75rqSDU/0zJW2sOr7fumn/FZIekbRJ0l9LCkkvyLkHAZ8DPhURsyPidxGxKyLuiIh3pzofl/TNqmPGpfMNS9uLJV0t6U7gD8BHJS2ruc6HJM1Pn58l6f9I2iDpUUmzJHUe4L8OawFOENZK/haYBrwGGAv8Fri2av8PgeOBo4CfAd+qOf5i4GrgUOCnqeytwFTgOOAlwF/VuX5uXUlTgQ8DrwNekOLrzwTgGGBOnTpFvBOYTnYvXwQmSDq+av/FwE3p8z8B44GJKb4ushaLtTknCGsl7wH+PiI2RsSTwMeBCyt/WUfEDRHxRNW+kyUdVnX8rRFxZ/qL/Y+p7AsRsSkiHgN+QPYl2p/+6r4V+GpErImIPwCfqHOO56bfjxS+63xfS9fbGRG/A24F3g6QEsULgfmpxfJu4EMR8VhEPAH8b+CiA7y+tQAnCGslzwNukbRN0jZgHdAH/ImkDkmfSd1PjwMPpWOOrDr+4Zxz/qbq8x+A59S5fn91x9acO+86Ff+dfo+pU6eI2mvcREoQZK2HeSlZjQYOAZZX/XNbmMqtzTlBWCt5GHh9RIyq+jk4InrIvhTPJ+vmOQwYl45R1fFlTW38CNlgc8UxdequJ7uPt9Sps53sS73iT3Pq1N7Lj4AjJU0kSxSV7qWtQC/w4qp/ZodFRL1EaG3CCcKa1XBJB1f9DANmAVdLeh6ApNGSzk/1DwWeJPsL/RCybpTB8j3gUkkvknQIdfr3I5t//8PAVZIulTQyDb6/StL1qdp9wKslHZu6yGbuK4CI2Ek2rtENHAHclsp3AV8B/kXSUQCSuiRN2e+7tZbhBGHNagHZX76Vn48D1wDzgR9JegK4Gzg91b8R+DXQA6xN+wZFRPwQ+ALwE+AB4K6068l+6s8B3ga8C9gEPAr8L7JxBCLiNuC7wEpgOfBvBUO5iawFdXNKGBV/l+K6O3W//TvZYLm1OXnBILPBJelFwGrgWTVf1GZDilsQZoNA0psljZB0ONljpT9wcrChzgnCbHC8B9gC/JLsyaq/aWw4ZvvmLiYzM8vlFoSZmeUa1ugABtKRRx4Z48aNa3QYZmZNY/ny5VsjIvfFyJZKEOPGjWPZsmX7rmhmZgBI+nV/+9zFZGZmuZwgzMwslxOEmZnlcoIwM7NcThBmZparpZ5iMjNrJ/NW9NC9aD2btvUydlQnM6ZMYNqkrgE7vxOEmVkTmreih5lzV9H7VB8APdt6mTl3FcCAJQl3MZmZNaHuRet3J4eK3qf66F60fsCu4QRhZtaENm3rfUbl+8MJwsysCY0d1fmMyveHE4SZWROaMWUCncM79ijrHN7BjCkDtxigB6nNzJpQZSD6ijkr2dG3iy4/xWRmZhXTJnXx7Xs3APDd97x8wM/vLiYzM8tVaoKQNFXSekkPSLqynzpnSrpP0hpJd6SyCams8vO4pA+WGauZme2ptC4mSR3AtcA5wEZgqaT5EbG2qs4o4DpgakRskHQUQESsByZWnacHuKWsWM3MbG9ltiBOAx6IiAcjYgfwHeD8mjoXA3MjYgNARGzOOc9rgV9GRL+LWpiZ2cArM0F0AQ9XbW9MZdXGA4dLWixpuaRLcs5zEfDt/i4iabqkZZKWbdmy5YCDNjOzTJkJQjllUbM9DDgVeCMwBbhK0vjdJ5BGAOcBN/d3kYi4PiImR8Tk0aNzl1U1M7P9UOZjrhuBY6q2jwY25dTZGhHbge2SlgAnA/en/a8HfhYRj5YYp5mZ5SizBbEUOF7ScaklcBEwv6bOrcAZkoZJOgQ4HVhXtf/t1OleMjOz8pTWgoiInZLeDywCOoAbImKNpMvS/lkRsU7SQmAlsAuYHRGrAVLCOAd4T1kxmplZ/0p9kzoiFgALaspm1Wx3A905x/4BeG6Z8ZmZWf/8JrWZmeVygjAzs1xOEGZmlssJwszMcjlBmJlZLicIMzPL5QWDzKrMW9FD96L1bNrWy9gSVugyayZOEGbJvBU9zJy7it6n+gDo2dbLzLmrAJwkrC05QZgl3YvW704OFb1P9XHFnJW7l3U0G2rWPvI4J4wZWcq5PQZhlmza1ptbvqNv1yBHYlbcCWNGcv7Eclq4bkGYJWNHddKTkyS6RnWWsiC82VDnFoRZMmPKBDqHd+xR1jm8gxlTJjQoIrPGcgvCLKkMRF8xZyU7+nbR5aeYrM05QZhVmTapa/eAtLuVrN25i8nMzHI5QZiZWS4nCDMzy+UEYWZmuZwgzMwslxOEmZnlcoIwM7NcThBmZpbLCcLMzHI5QZiZWS4nCDMzy+UEYWZmuZwgzMwslxOEmZnlcoIwM7NcXg/CBtW8FT10L1rPpm29jPWCPGZDmhOEDZp5K3qYOXcVvU/1AdCzrZeZc1cBOEmYDUGlJghJU4FrgA5gdkR8JqfOmcDngeHA1oh4TSofBcwGTgQCeFdE3FVmvFau7kXrdyeHit6n+rhizsrdq7gNBWsfeZwTxoxsdBhmDVdagpDUAVwLnANsBJZKmh8Ra6vqjAKuA6ZGxAZJR1Wd4hpgYURcKGkEcEhZsdrg2LStN7d8R9+uQY6kvhPGjOT8iW7RmJXZgjgNeCAiHgSQ9B3gfGBtVZ2LgbkRsQEgIjanuiOBVwN/lcp3ADtKjNUGwdhRnfTkJImuUZ1e/9lsCCrzKaYu4OGq7Y2prNp44HBJiyUtl3RJKn8+sAX4qqQVkmZLenbeRSRNl7RM0rItW7YM9D3YAJoxZQKdwzv2KOsc3sGMKRMaFJGZ1VNmglBOWdRsDwNOBd4ITAGukjQ+lZ8CfDkiJgHbgSvzLhIR10fE5IiYPHr06AEL3gbetEldfPqCkxjRkf1n1zWqk09fcJIHqM2GqH12MUk6IiIe249zbwSOqdo+GtiUU2drRGwHtktaApwM/AewMSLuSfXm0E+CsOYybVLX7gFpdyuZDW1FWhD3SLpZ0hsk5bUK+rMUOF7ScWmQ+SJgfk2dW4EzJA2TdAhwOrAuIn4DPCyp0vfwWvYcuzAzs5IVGaQeD7wOeBfwRUnfBb4WEffXOygidkp6P7CI7DHXGyJijaTL0v5ZEbFO0kJgJbCL7FHY1ekUHwC+lZLLg8Cl+3F/Zma2n/aZICIigNuA2ySdBXwTeK+knwNX1ns3ISIWAAtqymbVbHcD3TnH3gdMLnITZmY28IqMQTwXeAfwTuBRsr/s5wMTgZuB48oM0MzMGqNIF9NdwDeAaRGxsap8maRZ/RxjZmZNrkiCmJC6mfYSEf80wPGYmdkQUeQpph+lKTEAkHS4pEUlxmRmZkNAkQQxOiK2VTYi4rfAUXXqm5lZCyiSIPokHVvZkPQ89n4j2szMWkyRMYi/B34q6Y60/WpgenkhmZnZUFDkPYiFkk4BXkY2v9KHImJr6ZGZmVlDFZ3uuw/YDBwMnCCJiFhSXlhmZtZoRV6U+2vgcrLJ9u4ja0ncBZxdbmhmZtZIRQapLwdeCvw6Is4CJpGt1WBmZi2sSIL4Y0T8EUDSsyLiF4BXeDEza3FFxiA2phfl5pFN2Pdb9l7XwczMWkyRp5jenD5+XNJPgMOAhaVGZWZmDVc3QUg6CFgZEScCRMQd9eqbmVnrqDsGERG7gJ9Xv0ltZmbtocgYxBhgjaR7ge2Vwog4r7SozMys4YokiE+UHoWZmQ05RQapPe5gZtaGirxJ/QRPz946AhgObI+IkWUGZmZmjVWkBXFo9bakacBppUVkZmZDQpE3qfcQEfPwPExmZi2vSBfTBVWbBwGT8YJBZmYtr8hTTG+q+rwTeAg4v5RozMxsyCgyBnHpYARiZmZDyz7HICR9PU3WV9k+XNIN5YZlZmaNVmSQ+iURsa2yERG/JVsTwszMWliRBHGQpMMrG5KOoPhSpWZm1qSKfNF/FvhPSXPInl56K3B1qVGZmVnDFRmkvlHSMrJ3HwRcEBFrS4/MzMwaqsgg9cuAhyPiSxHxReBhSacXObmkqZLWS3pA0pX91DlT0n2S1ki6o6r8IUmr0r5lRW/IzMwGRpEupi8Dp1Rtb88p24ukDuBa4BxgI7BU0vzq1kd6Ouo6YGpEbJB0VM1pzoqIrQViNDOzAVZkkFoRsfvN6bSIUJHEchrwQEQ8GBE7gO+w9wt2FwNzI2JDOvfmYmGbmVnZiiSIByX9raTh6edy4MECx3UBD1dtb0xl1cYDh0taLGm5pEuq9gXwo1Q+vb+LSJouaZmkZVu2bCkQlpmZFVEkQVwGvALoIfuSPx14d4HjlFNWO4fTMOBU4I3AFOAqSePTvldGxCnA64H3SXp13kUi4vqImBwRk0ePHl0gLDMzK6LIU0ybgYsq25I6gXOBm/dx6EbgmKrto4FNOXW2RsR2YLukJcDJwP0RsalyfUm3kHVZLdlXvGZmNjAKTfctqUPS6yXdCPwKeFuBw5YCx0s6TtIIsiQzv6bOrcAZkoZJOoSsdbJO0rMlHZqu/Wzgz4HVxW7JzMwGQt0WROrWuZisC+he4JXA8yPiD/s6cUTslPR+YBHQAdwQEWskXZb2z4qIdZIWAiuBXcDsiFgt6fnALZIqMd4UEQv3+y7NzOwZ6zdBSNoIbCB7pHVGRDwh6VdFkkNFRCwAFtSUzarZ7ga6a8oeJOtqMjOzBqnXxfR9sqeO3ga8KXX1eKEgM7M20W+CiIjLgXHA54CzgPuB0ZLeKuk5gxOemZk1St1B6sjcHhHvJksWFwPTyFaVMzOzFlZ42u6IeAr4AfCD9KirmZm1sEKPudaKiN6BDsTMzIaW/UoQZmbW+pwgzMws1z7HINLcSDOA51XXj4izS4zLzMwarMgg9c3ALOArQF+54ZiZ2VBRJEHsjIgvlx6JmZkNKUXGIH4g6b2Sxkg6ovJTemRmZtZQRVoQf5l+z6gqC+D5Ax+OmZkNFUXWgzhuMAIxM7OhpchTTMOBvwEqK7otBv41vVltZmYtqkgX05eB4cB1afudqeyvywrKzMwar0iCeGlEVK/NcLukn5cVkJmZDQ1FnmLqk/RnlY202pvfhzAza3FFWhAzgJ9IehAQ2RvVl5YalZmZNVyRp5h+LOl4YAJZgvhFRDxZemTWcPNW9NC9aD2btvUydlQnM6ZMYNqkrkaHZWaDpN6a1GdHxO2SLqjZ9WeSiIi5JcdmDTRvRQ8z566i96msN7FnWy8z564CcJIwaxP1WhCvAW4H3pSzLwAniBbWvWj97uRQ0ftUH1fMWcm3791wQOde+8jjnDBm5AGdw8zK12+CiIh/TB8/GRG/qt4nyS/PtbhN2/LXhNrRt+uAz33CmJGcP9GtELOhrsgg9feBU2rK5gCnDnw4NlSMHdVJT06S6BrVyXff8/IGRGRmg63eGMQLgRcDh9WMQ4wEDi47MGusGVMm7DEGAdA5vIMZUyY0MCozG0z1WhATgHOBUew5DvEE8O4yg7LGqwxEXzFnJTv6dtHlp5jM2k69MYhbgVslvTwi7hrEmGyImDapa/eAtLuVzNpPkTGIFZLeR9bdtLtrKSLeVVpUZmbWcEWm2vgG8KfAFOAO4GiybiYzM2thRRLECyLiKmB7RHwdeCNwUrlhmZlZoxVJEJV1H7ZJOhE4DBhXWkRmZjYkFBmDuF7S4cBVwHzgOcDHSo3KCqs3X5LnUjKzA1Fksr7Z6eMdPMN1qCVNBa4BOoDZEfGZnDpnAp8nW5Roa0S8pmpfB7AM6ImIc5/JtVvBvr7g682XBHguJTM7IPVelPtwvQMj4nP19qcv92uBc4CNwFJJ8yNibVWdUWQr1U2NiA2Sjqo5zeXAOrKX89pKkcny6s2XBHtPi7E/cyl53iSz9lWvBXFo+j0BeClZ9xJkL80tKXDu04AHIuJBAEnfAc4H1lbVuRiYGxEbACJic2WHpKPJBsSvBuomq1ZUZLK8vKkwoP58Sc90LiXPm2TWvuq9KPcJAEk/Ak6JiCfS9seBmwucuwt4uGp7I3B6TZ3xwHBJi8kS0jURcWPa93ngCp5OVLkkTQemAxx77LEFwmoORSbLG9FxUO4XfteoTiA/gXguJTMrqsgg9bHAjqrtHRR7ikk5ZZFz/VOB1wKdwF2S7iZLHJsjYnkao+hXRFwPXA8wefLk2vM3rSKT5dV2Q8Ge8yV5LiUzOxBFEsQ3gHsl3UL2Bf9m4Mb6hwBZi+GYqu2jgU05dbZGxHZgu6QlwMlks8eeJ+kNZG9vj5T0zYh4R4HrtoQik+VVj0X0N5Dtp5jMbH8pYt9/dEs6BTgjbS6JiBUFjhkG3E/WOugBlgIXR8SaqjovAr5E9pb2COBe4KKIWF1V50zgI0WeYpo8eXIsW7Zsn/fTLOat6PFkeWZWKknLI2Jy3r56TzGNjIjHJR0BPJR+KvuOiIjH6l00InZKej+wiOwx1xsiYo2ky9L+WRGxTtJCYCWwi+xR2NX9n7W9eLI8M2ukel1MN5FN972cPccOlLb3+U5ERCwAFtSUzarZ7ga665xjMbB4X9dqBL+IZmatrN5TTOem315eNEeR9xTMzJpZvS6m2mVG9xARPxv4cJpHkfcUBoJfVDOzRqnXxfTZOvsCOHuAY2kqRd5TGAh+Uc3MGqVeF9NZgxlIsynynoKZWTMrMt03kk6U9FZJl1R+yg5sqJsxZQKdwzv2KPOLaGbWSvaZICT9I/DF9HMW8M/AeSXHNeRNm9TFW059uuunQ+Itp3Z5gNrMWkaRFsSFZC+7/SYiLiV70/lZpUbVBOat6OH7y3t2b/dF8P3lPcxb0VPnKDOz5lEkQfRGxC5gp6SRwGae4boQrai/p5i6F61vUERmZgOryFxMy9K6DV8he2nu92RTYrS1/p5i6q/czKzZ1HsP4kvATRHx3lQ0K02LMTIiVg5KdENYf08xjU1TbZuZNbt6XUz/BXxW0kOS/knSxIh4yMkh46eYzKzV9ZsgIuKaiHg58BrgMeCrktZJ+pik8YMW4RA1bVIXn77gJEZ0ZP8Iu0Z18ukLTvJTTGbWMgpN9727sjQJuAF4SUR07Kv+YGvEdN9v+9e7AM+2ambNqd5030Xegxgu6U2SvgX8kGyNh7cMcIxmZjbE1BukPgd4O/BGsqeWvgNMT6u/mZlZi6v3mOtHydaE+Mi+FgdqR5W1IHq29TKi4yDmrejx+IOZtRRP1rcfateC2NG3y2tBmFnLKTRZn+3Jb1GbWTso8iZ1y3umS4f6LWozawdt34KodBf1bOsleHrp0HqT7vX3trTfojazVtL2LYj9WTr04OEHcZBgV9UrJH6L2sxaTdsniP1ZOvTI52SznT/8WC87+nbRVaBbysys2bR9gvDSoWZm+dp+DMKT7pmZ5Wv7FkSlW+iKOSvdXWRmVqXtEwRkSaIyIO1uJTOzTNt3MZmZWT4nCDMzy+UEYWZmuZwgzMwsV6kJQtJUSeslPSDpyn7qnCnpPklrJN2Ryg6WdK+kn6fyT5QZp5mZ7a20p5gkdQDXAucAG4GlkuZHxNqqOqOA64CpEbFB0lFp15PA2RHxe0nDgZ9K+mFE3F1WvGZmtqcyWxCnAQ9ExIMRsYNsRbrza+pcDMyNiA0AEbE5/Y6I+H2qMzz9FF8828zMDliZCaILeLhqe2MqqzYeOFzSYknLJV1S2SGpQ9J9wGbgtoi4p8RYzcysRpkJQjllta2AYcCpZOteTwGukjQeICL6ImIicDRwmqQTcy8iTZe0TNKyLVu2DFz0ZmZtrswEsRE4pmr7aGBTTp2FEbE9IrYCS4CTqytExDZgMTA17yIRcX1ETI6IyaNHjx6o2M3M2l6ZCWIpcLyk4ySNAC4C5tfUuRU4Q9IwSYcApwPrJI1OA9hI6gReB/yixFjNzKxGaU8xRcROSe8HFgEdwA0RsUbSZWn/rIhYJ2khsBLYBcyOiNWSXgJ8PT0JdRDwvYj4t7JiNTOzvZU6WV9ELAAW1JTNqtnuBrprylYCk8qMzczM6vOb1GZmlssJwszMcjlBmJlZLicIMzPL5QRhZma5nCDMzCyXE4SZmeVygjAzs1xOEGZmlssJwszMcjlBmJlZLicIMzPL5QRhZma5nCDMzCyXE4SZmeVygjAzs1xOEGZmlssJwszMcjlBmJlZLicIMzPL5QRhZma5nCDMzCxX2yeIeSt6eOVnbueeXz3Gig3bmLeip9EhmZkNCcMaHUAjzVvRw8y5q+h9qg+AHX27mDl3FQDTJnU1MjQzs4Zr6xZE96L1u5NDRe9TfXQvWt+giMzMho62ThCbtvU+o3Izs3bS1gli7KjOZ1RuZtZO2jpBzJgygc7hHXuUdQ7vYMaUCQ2KyMxs6GjrQerKQHT3ovVs2tbL2FGdzJgywQPUZma0eYKALEk4IZiZ7a2tu5jMzKx/pSYISVMlrZf0gKQr+6lzpqT7JK2RdEcqO0bSTyStS+WXlxmnmZntrbQuJkkdwLXAOcBGYKmk+RGxtqrOKOA6YGpEbJB0VNq1E/ifEfEzSYcCyyXdVn2smZmVq8wWxGnAAxHxYETsAL4DnF9T52JgbkRsAIiIzen3IxHxs/T5CWAd4IECM7NBVGaC6AIertreyN5f8uOBwyUtlrRc0iW1J5E0DpgE3JN3EUnTJS2TtGzLli0DEriZmZX7FJNyyiLn+qcCrwU6gbsk3R0R9wNIeg7wfeCDEfF43kUi4nrg+lR/i6RfF4zvSGBrwbrNplXvzffVXHxfzeF5/e0oM0FsBI6p2j4a2JRTZ2tEbAe2S1oCnAzcL2k4WXL4VkTMLXLBiBhdNDhJyyJictH6zaRV78331Vx8X82vzC6mpcDxko6TNAK4CJhfU+dW4AxJwyQdApwOrJMk4P8C6yLicyXGaGZm/SitBREROyW9H1gEdAA3RMQaSZel/bMiYp2khcBKYBcwOyJWS3oV8E5glaT70ik/GhELyorXzMz2VOqb1OkLfUFN2aya7W6gu6bsp+SPYQyk60s+fyO16r35vpqL76vJKaJ23NjMzMxTbZiZWT+cIMzMLFdbJogic0Q1A0k3SNosaXVV2RGSbpP0X+n34Y2McX/0NxdXs9+bpIMl3Svp5+m+PpHKm/q+KiR1SFoh6d/Sdqvc10OSVqU545alspa4t31puwRRNUfU64ETgLdLOqGxUe23rwFTa8quBH4cEccDP07bzaYyF9eLgJcB70v/jpr93p4Ezo6Ik4GJwFRJL6P576vicrJpcSpa5b4AzoqIiVXvP7TSvfWr7RIExeaIagoRsQR4rKb4fODr6fPXgWmDGtQAqDMXV1PfW2R+nzaHp5+gye8LQNLRwBuB2VXFTX9fdbTyve3WjgmiyBxRzexPIuIRyL5ogaP2UX9Iq5mLq+nvLXXD3AdsBm6LiJa4L+DzwBVk7zNVtMJ9QZbEf5Tmi5ueylrl3upqxxXliswRZUNA7Vxc2Qv2zS0i+oCJaar7WySd2OiYDpSkc4HNEbFc0pmNjqcEr4yITWk5gtsk/aLRAQ2WdmxBFJkjqpk9KmkMQPq9ucHx7Jd+5uJqiXsDiIhtwGKyMaRmv69XAudJeoisy/ZsSd+k+e8LgIjYlH5vBm4h66ZuiXvbl3ZMEEXmiGpm84G/TJ//kmy+q6ZSZy6upr43SaNTywFJncDrgF/Q5PcVETMj4uiIGEf2/9PtEfEOmvy+ACQ9Oy1ahqRnA38OrKYF7q2ItnyTWtIbyPpMK3NEXd3gkPaLpG8DZ5JNP/wo8I/APOB7wLHABuAvIqJ2IHtIS3Nx/Qewiqf7tD9KNg7RtPcm6SVkA5odZH+cfS8iPinpuTTxfVVLXUwfiYhzW+G+JD2frNUAWZf8TRFxdSvcWxFtmSDMzGzf2rGLyczMCnCCMDOzXE4QZmaWywnCzMxyOUGYmVkuJwhrGpL+RdIHq7YXSZpdtf1ZSR+uc/zXJF2YPi+WtNfC85KGS/pMmqVzdZp99fVp30OSjtyPuHdft5/916aZQtdK6k2f75N0oaQFlXcnBpKkMZVZV/vZP0LSEkntONuCJU4Q1kz+E3gFgKSDyN7/eHHV/lcAdx7gNT4FjAFOjIgTgTcBhx7gOeuKiPdFxETgDcAv06yhEyNiTkS8Ib11PdA+DHylTkw7yGYpfVsJ17Ym4QRhzeROUoIgSwyrgSckHS7pWcCLgBWSPiZpaWoBXK+CkzhJOgR4N/CBiHgSICIejYjv5dT9cDr/6ppWzSWSVqY1H76Rc9ynUoui0P97lVaLpHGSfiFpdrrmtyS9TtKdqbVzWqr/bGXrhCxVtjZDfzMVvwVYmI55cWop3ZdiPz7VmQf8jyJxWmty89GaRpowbaekY8kSxV1kM/G+HPgdsDIidkj6UkR8EiB9SZ8L/KDAJV4AbIiIx+tVknQqcClwOtnkj/dIugPYAfw92eRuWyUdUXPcPwOHAZfG/r2h+gLgL4DpZFPGXAy8CjiP7E3zaen6t0fEu1LX1L2S/j0itlfFcRzw20oSBC4DromIb6XpZzpS+WrgpfsRp7UItyCs2VRaEZUEcVfV9n+mOmdJukfSKuBs9uyGGgivAm6JiO1pfYe5wBnpWnMiYitAzdQLVwGjIuI9+5kcAH4VEasiYhewhmzBmuoe1JAAAAHYSURBVCCbkmRcqvPnwJXKphRfDBxMNh1EtTHAlqrtu4CPSvo74HkR0Zvi7wN2VOYisvbjBGHNpjIOcRLZX7h3k7UgXgHcKelg4Drgwog4iayf/eCC534AOLbAF2J/XVai/6njlwKn1rYqnqEnqz7vqtrexdO9AQLeUjWOcWxEVK/yBtBL1T+TiLiJrBXSCyySdHZV3WcBfzyAmK2JOUFYs7mTrMvosYjoS3+ljyJLEnfx9BffVmXrSfT79FCtiPgD2SyyX0hdLZWnfd5RU3UJME3SIWmGzzeTTS74Y+CtaSI3apLBQuAzwP8r+S/yRcAHKuMukibl1Lmfp1sclQnpHoyIL5DNUvqSVP5cYEtEPFVivDaEOUFYs1lF9vTS3TVlv4uIremJn6+ksnlkf7k/E/9A1v2yVtLqdI7q7hjScqhfA+4lm2F2dkSsiIg1wNXAHZJ+Dnyu5ribU2zz03TfZfgU2VKmK1P8n6qtkMYjfinpBanobcDq1C31QuDGVH4WsKCkOK0JeDZXszYk6c3AqRHxD3XqzAVmRsT6wYvMhhI/xWTWhiLilkpXWJ7UxTbPyaG9uQVhZma5PAZhZma5nCDMzCyXE4SZmeVygjAzs1xOEGZmluv/A+ioR9g8hVqYAAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "plt.title('Learning Curve')\n", + "plt.xlabel('Wall Clock Time (s)')\n", + "plt.ylabel('Validation Accuracy')\n", + "plt.scatter(time_history, 1-np.array(valid_loss_history))\n", + "plt.step(time_history, 1-np.array(best_valid_loss_history), where='post')\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## 3. Customized Learner" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Some experienced automl users may have a preferred model to tune or may already have a reasonably by-hand-tuned model before launching the automl experiment. They need to select optimal configurations for the customized model mixed with standard built-in learners. \n", + "\n", + "FLAML can easily incorporate customized/new learners (preferably with sklearn API) provided by users in a real-time manner, as demonstrated below." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Example of Regularized Greedy Forest\n", + "\n", + "[Regularized Greedy Forest](https://arxiv.org/abs/1109.0887) (RGF) is a machine learning method currently not included in FLAML. The RGF has many tuning parameters, the most critical of which are: `[max_leaf, n_iter, n_tree_search, opt_interval, min_samples_leaf]`. To run a customized/new learner, the user needs to provide the following information:\n", + "* an implementation of the customized/new learner\n", + "* a list of hyperparameter names and types\n", + "* rough ranges of hyperparameters (i.e., upper/lower bounds)\n", + "* choose initial value corresponding to low cost for cost-related hyperparameters (e.g., initial value for max_leaf and n_iter should be small)\n", + "\n", + "In this example, the above information for RGF is wrapped in a python class called *MyRegularizedGreedyForest* that exposes the hyperparameters." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "''' SKLearnEstimator is the super class for a sklearn learner '''\n", + "from flaml.model import SKLearnEstimator\n", + "from flaml import tune\n", + "from rgf.sklearn import RGFClassifier, RGFRegressor\n", + "\n", + "\n", + "class MyRegularizedGreedyForest(SKLearnEstimator):\n", + "\n", + "\n", + " def __init__(self, task = 'binary:logistic', n_jobs = 1, **params):\n", + " '''Constructor\n", + " \n", + " Args:\n", + " task: A string of the task type, one of\n", + " 'binary:logistic', 'multi:softmax', 'regression'\n", + " n_jobs: An integer of the number of parallel threads\n", + " params: A dictionary of the hyperparameter names and values\n", + " '''\n", + "\n", + " super().__init__(task, **params)\n", + "\n", + " '''task=regression for RGFRegressor; \n", + " binary:logistic and multiclass:softmax for RGFClassifier'''\n", + " if 'regression' in task:\n", + " self.estimator_class = RGFRegressor\n", + " else:\n", + " self.estimator_class = RGFClassifier\n", + "\n", + " # convert to int for integer hyperparameters\n", + " self.params = {\n", + " \"n_jobs\": n_jobs,\n", + " 'max_leaf': int(params['max_leaf']),\n", + " 'n_iter': int(params['n_iter']),\n", + " 'n_tree_search': int(params['n_tree_search']),\n", + " 'opt_interval': int(params['opt_interval']),\n", + " 'learning_rate': params['learning_rate'],\n", + " 'min_samples_leaf':int(params['min_samples_leaf'])\n", + " } \n", + "\n", + " @classmethod\n", + " def search_space(cls, data_size, task):\n", + " '''[required method] search space\n", + "\n", + " Returns:\n", + " A dictionary of the search space. \n", + " Each key is the name of a hyperparameter, and value is a dict with\n", + " its domain and init_value (optional), cat_hp_cost (optional) \n", + " e.g., \n", + " {'domain': tune.randint(lower=1, upper=10), 'init_value': 1}\n", + " '''\n", + " space = { \n", + " 'max_leaf': {'domain': tune.qloguniform(lower = 4, upper = data_size, q = 1), 'init_value': 4},\n", + " 'n_iter': {'domain': tune.qloguniform(lower = 1, upper = data_size, q = 1), 'init_value': 1},\n", + " 'n_tree_search': {'domain': tune.qloguniform(lower = 1, upper = 32768, q = 1), 'init_value': 1},\n", + " 'opt_interval': {'domain': tune.qloguniform(lower = 1, upper = 10000, q = 1), 'init_value': 100},\n", + " 'learning_rate': {'domain': tune.loguniform(lower = 0.01, upper = 20.0)},\n", + " 'min_samples_leaf': {'domain': tune.qloguniform(lower = 1, upper = 20, q = 1), 'init_value': 20},\n", + " }\n", + " return space\n", + "\n", + " @classmethod\n", + " def size(cls, config):\n", + " '''[optional method] memory size of the estimator in bytes\n", + " \n", + " Args:\n", + " config - the dict of the hyperparameter config\n", + "\n", + " Returns:\n", + " A float of the memory size required by the estimator to train the\n", + " given config\n", + " '''\n", + " max_leaves = int(round(config['max_leaf']))\n", + " n_estimators = int(round(config['n_iter']))\n", + " return (max_leaves*3 + (max_leaves-1)*4 + 1.0)*n_estimators*8\n", + "\n", + " @classmethod\n", + " def cost_relative2lgbm(cls):\n", + " '''[optional method] relative cost compared to lightgbm\n", + " '''\n", + " return 1.0\n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Add Customized Learner and Run FLAML AutoML\n", + "\n", + "After adding RGF into the list of learners, we run automl by tuning hyperpameters of RGF as well as the default learners. " + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "automl = AutoML()\n", + "automl.add_learner(learner_name = 'RGF', learner_class = MyRegularizedGreedyForest)" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[flaml.automl: 02-22 14:08:16] {844} INFO - Evaluation method: holdout\n", + "[flaml.automl: 02-22 14:08:16] {569} INFO - Using StratifiedKFold\n", + "[flaml.automl: 02-22 14:08:17] {865} INFO - Minimizing error metric: 1-accuracy\n", + "[flaml.automl: 02-22 14:08:17] {885} INFO - List of ML learners in AutoML Run: ['RGF', 'lgbm', 'rf', 'xgboost']\n", + "[flaml.automl: 02-22 14:08:17] {944} INFO - iteration 0 current learner RGF\n", + "[flaml.automl: 02-22 14:08:19] {1098} INFO - at 2.8s,\tbest RGF's error=0.4333,\tbest RGF's error=0.4333\n", + "[flaml.automl: 02-22 14:08:19] {944} INFO - iteration 1 current learner RGF\n", + "[flaml.automl: 02-22 14:08:20] {1098} INFO - at 4.3s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", + "[flaml.automl: 02-22 14:08:20] {944} INFO - iteration 2 current learner RGF\n", + "[flaml.automl: 02-22 14:08:22] {1098} INFO - at 5.8s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", + "[flaml.automl: 02-22 14:08:22] {944} INFO - iteration 3 current learner RGF\n", + "[flaml.automl: 02-22 14:08:23] {1098} INFO - at 7.5s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", + "[flaml.automl: 02-22 14:08:23] {944} INFO - iteration 4 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:23] {1098} INFO - at 7.6s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", + "[flaml.automl: 02-22 14:08:23] {944} INFO - iteration 5 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 7.7s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", + "[flaml.automl: 02-22 14:08:24] {944} INFO - iteration 6 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 7.8s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:08:24] {944} INFO - iteration 7 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 7.9s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", + "[flaml.automl: 02-22 14:08:24] {944} INFO - iteration 8 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 8.1s,\tbest lgbm's error=0.3651,\tbest lgbm's error=0.3651\n", + "[flaml.automl: 02-22 14:08:24] {944} INFO - iteration 9 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 8.4s,\tbest lgbm's error=0.3632,\tbest lgbm's error=0.3632\n", + "[flaml.automl: 02-22 14:08:24] {944} INFO - iteration 10 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 8.5s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:24] {944} INFO - iteration 11 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:24] {1098} INFO - at 8.7s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:25] {944} INFO - iteration 12 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:25] {1098} INFO - at 8.8s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:25] {944} INFO - iteration 13 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:25] {1098} INFO - at 8.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:25] {944} INFO - iteration 14 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:25] {1098} INFO - at 9.0s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:25] {944} INFO - iteration 15 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:25] {1098} INFO - at 9.2s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:25] {944} INFO - iteration 16 current learner RGF\n", + "[flaml.automl: 02-22 14:08:26] {1098} INFO - at 10.6s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:26] {944} INFO - iteration 17 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:26] {1098} INFO - at 10.7s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:26] {944} INFO - iteration 18 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:27] {1098} INFO - at 10.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:27] {944} INFO - iteration 19 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:28] {1098} INFO - at 12.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:28] {944} INFO - iteration 20 current learner RGF\n", + "[flaml.automl: 02-22 14:08:30] {1098} INFO - at 13.9s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:30] {944} INFO - iteration 21 current learner RGF\n", + "[flaml.automl: 02-22 14:08:32] {1098} INFO - at 16.2s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:32] {944} INFO - iteration 22 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:33] {1098} INFO - at 17.2s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:33] {944} INFO - iteration 23 current learner xgboost\n", + "[flaml.automl: 02-22 14:08:33] {1098} INFO - at 17.4s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:33] {944} INFO - iteration 24 current learner xgboost\n", + "[flaml.automl: 02-22 14:08:33] {1098} INFO - at 17.5s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:33] {944} INFO - iteration 25 current learner xgboost\n", + "[flaml.automl: 02-22 14:08:33] {1098} INFO - at 17.6s,\tbest xgboost's error=0.3757,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:33] {944} INFO - iteration 26 current learner xgboost\n", + "[flaml.automl: 02-22 14:08:34] {1098} INFO - at 17.7s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3621\n", + "[flaml.automl: 02-22 14:08:34] {944} INFO - iteration 27 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:35] {1098} INFO - at 19.5s,\tbest lgbm's error=0.3568,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:35] {944} INFO - iteration 28 current learner rf\n", + "[flaml.automl: 02-22 14:08:36] {1098} INFO - at 20.1s,\tbest rf's error=0.4020,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:36] {944} INFO - iteration 29 current learner rf\n", + "[flaml.automl: 02-22 14:08:36] {1098} INFO - at 20.6s,\tbest rf's error=0.4005,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:36] {944} INFO - iteration 30 current learner RGF\n", + "[flaml.automl: 02-22 14:08:39] {1098} INFO - at 22.9s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:39] {944} INFO - iteration 31 current learner RGF\n", + "[flaml.automl: 02-22 14:08:51] {1098} INFO - at 35.1s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:51] {944} INFO - iteration 32 current learner lgbm\n", + "[flaml.automl: 02-22 14:08:52] {1098} INFO - at 36.5s,\tbest lgbm's error=0.3568,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:52] {944} INFO - iteration 33 current learner rf\n", + "[flaml.automl: 02-22 14:08:53] {1098} INFO - at 36.9s,\tbest rf's error=0.3983,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:08:53] {944} INFO - iteration 34 current learner RGF\n", + "[flaml.automl: 02-22 14:09:05] {1098} INFO - at 49.3s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3568\n", + "[flaml.automl: 02-22 14:09:05] {944} INFO - iteration 35 current learner lgbm\n", + "[flaml.automl: 02-22 14:09:09] {1098} INFO - at 52.7s,\tbest lgbm's error=0.3397,\tbest lgbm's error=0.3397\n", + "[flaml.automl: 02-22 14:09:09] {944} INFO - iteration 36 current learner xgboost\n", + "[flaml.automl: 02-22 14:09:09] {1098} INFO - at 52.8s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3397\n", + "[flaml.automl: 02-22 14:09:12] {1115} INFO - retrain xgboost for 3.4s\n", + "[flaml.automl: 02-22 14:09:12] {944} INFO - iteration 37 current learner lgbm\n", + "[flaml.automl: 02-22 14:09:17] {1098} INFO - at 61.4s,\tbest lgbm's error=0.3397,\tbest lgbm's error=0.3397\n", + "[flaml.automl: 02-22 14:09:17] {1139} INFO - selected model: LGBMClassifier(learning_rate=0.17709445119696665, max_bin=255,\n", + " min_child_weight=8.784469833660507, n_estimators=62,\n", + " num_leaves=44, objective='binary', reg_alpha=1e-10,\n", + " reg_lambda=0.21025319396124098, subsample=0.9826350012458567)\n", + "[flaml.automl: 02-22 14:09:17] {899} INFO - fit succeeded\n" + ] + } + ], + "source": [ + "settings = {\n", + " \"time_budget\": 60, # total running time in seconds\n", + " \"metric\": 'accuracy', \n", + " \"estimator_list\": ['RGF', 'lgbm', 'rf', 'xgboost'], # list of ML learners\n", + " \"task\": 'classification', # task type \n", + " \"log_file_name\": 'airlines_experiment_custom.log', # flaml log file \n", + " \"log_training_metric\": True, # whether to log training metric\n", + "}\n", + "\n", + "'''The main flaml automl API'''\n", + "automl.fit(X_train = X_train, y_train = y_train, **settings)" + ] + }, + { + "source": [ + "## 4. Comparison with alternatives\n", + "\n", + "### FLAML's accuracy" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "flaml accuracy = 0.6722186790857719\n" + ] + } + ], + "source": [ + "print('flaml accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))" + ] + }, + { + "source": [ + "### Default LightGBM" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "from lightgbm import LGBMClassifier\n", + "lgbm = LGBMClassifier()" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "LGBMClassifier()" + ] + }, + "metadata": {}, + "execution_count": 17 + } + ], + "source": [ + "lgbm.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "default lgbm accuracy = 0.6602123904305652\n" + ] + } + ], + "source": [ + "y_pred = lgbm.predict(X_test)\n", + "from flaml.ml import sklearn_metric_loss_score\n", + "print('default lgbm accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))" + ] + }, + { + "source": [ + "### Default XGBoost" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "from xgboost import XGBClassifier\n", + "xgb = XGBClassifier()" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n", + " colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n", + " importance_type='gain', interaction_constraints='',\n", + " learning_rate=0.300000012, max_delta_step=0, max_depth=6,\n", + " min_child_weight=1, missing=nan, monotone_constraints='()',\n", + " n_estimators=100, n_jobs=8, num_parallel_tree=1, random_state=0,\n", + " reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n", + " tree_method='exact', validate_parameters=1, verbosity=None)" + ] + }, + "metadata": {}, + "execution_count": 20 + } + ], + "source": [ + "xgb.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "default xgboost accuracy = 0.6676060098186078\n" + ] + } + ], + "source": [ + "y_pred = xgb.predict(X_test)\n", + "from flaml.ml import sklearn_metric_loss_score\n", + "print('default xgboost accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))" + ] + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "display_name": "Python 3", + "language": "python" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/notebook/flaml_azureml.ipynb b/notebook/flaml_azureml.ipynb index 57754adb4..8e4e94f4d 100644 --- a/notebook/flaml_azureml.ipynb +++ b/notebook/flaml_azureml.ipynb @@ -25,7 +25,7 @@ "- be embedded in self-tuning software that requires low latency & resource in repetitive\n", " tuning tasks.\n", "\n", - "In this notebook, we use one real data example (binary classification) to showcase how to ues FLAML library together with AzureML.\n", + "In this notebook, we use one real data example (binary classification) to showcase how to use FLAML library together with AzureML.\n", "\n", "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` and `azureml` option:\n", "```bash\n", @@ -42,9 +42,16 @@ "!pip install flaml[notebook,azureml]" ] }, + { + "source": [ + "### Enable mlflow in AzureML workspace" + ], + "cell_type": "markdown", + "metadata": {} + }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -63,7 +70,7 @@ } }, "source": [ - "## 2. Real Data Example\n", + "## 2. Classification Example\n", "### Load data and preprocess\n", "\n", "Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure." @@ -71,7 +78,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": { "slideshow": { "slide_type": "subslide" @@ -81,10 +88,9 @@ "outputs": [ { "output_type": "stream", - "name": "stderr", + "name": "stdout", "text": [ - "DEBUG - Data pickle file already exists and is up to date.\n", - "download dataset from openml\n", + "load dataset from ./openml_ds1169.pkl\n", "Dataset name: airlines\n", "X_train.shape: (404537, 7), y_train.shape: (404537,);\n", "X_test.shape: (134846, 7), y_test.shape: (134846,)\n" @@ -110,7 +116,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": { "slideshow": { "slide_type": "slide" @@ -125,7 +131,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": { "slideshow": { "slide_type": "slide" @@ -139,13 +145,13 @@ " \"estimator_list\": ['lgbm', 'rf', 'xgboost'], # list of ML learners\n", " \"task\": 'classification', # task type \n", " \"sample\": False, # whether to subsample training data\n", - " \"log_file_name\": 'airlines_experiment.log', # cache directory of flaml log files \n", + " \"log_file_name\": 'airlines_experiment.log', # flaml log file\n", "}" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": { "slideshow": { "slide_type": "slide" @@ -157,129 +163,67 @@ "output_type": "stream", "name": "stderr", "text": [ - "[flaml.automl: 02-17 13:46:26] {840} INFO - Evaluation method: holdout\n", "INFO - Evaluation method: holdout\n", - "[flaml.automl: 02-17 13:46:26] {565} INFO - Using StratifiedKFold\n", "INFO - Using StratifiedKFold\n", - "[flaml.automl: 02-17 13:46:26] {861} INFO - Minimizing error metric: 1-accuracy\n", "INFO - Minimizing error metric: 1-accuracy\n", - "[flaml.automl: 02-17 13:46:26] {881} INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost']\n", "INFO - List of ML learners in AutoML Run: ['lgbm', 'rf', 'xgboost']\n", - "[flaml.automl: 02-17 13:46:26] {940} INFO - iteration 0 current learner lgbm\n", "INFO - iteration 0 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:29] {1094} INFO - at 0.9s,\tbest lgbm's error=0.3771,\tbest lgbm's error=0.3771\n", - "INFO - at 0.9s,\tbest lgbm's error=0.3771,\tbest lgbm's error=0.3771\n", - "[flaml.automl: 02-17 13:46:29] {940} INFO - iteration 1 current learner lgbm\n", + "INFO - at 1.6s,\tbest lgbm's error=0.3771,\tbest lgbm's error=0.3771\n", "INFO - iteration 1 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:30] {1094} INFO - at 3.6s,\tbest lgbm's error=0.3771,\tbest lgbm's error=0.3771\n", - "INFO - at 3.6s,\tbest lgbm's error=0.3771,\tbest lgbm's error=0.3771\n", - "[flaml.automl: 02-17 13:46:30] {940} INFO - iteration 2 current learner lgbm\n", + "INFO - at 4.7s,\tbest lgbm's error=0.3771,\tbest lgbm's error=0.3771\n", "INFO - iteration 2 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:33] {1094} INFO - at 4.5s,\tbest lgbm's error=0.3751,\tbest lgbm's error=0.3751\n", - "INFO - at 4.5s,\tbest lgbm's error=0.3751,\tbest lgbm's error=0.3751\n", - "[flaml.automl: 02-17 13:46:33] {940} INFO - iteration 3 current learner xgboost\n", + "INFO - at 5.6s,\tbest lgbm's error=0.3751,\tbest lgbm's error=0.3751\n", "INFO - iteration 3 current learner xgboost\n", - "[flaml.automl: 02-17 13:46:37] {1094} INFO - at 8.8s,\tbest xgboost's error=0.3750,\tbest xgboost's error=0.3750\n", - "INFO - at 8.8s,\tbest xgboost's error=0.3750,\tbest xgboost's error=0.3750\n", - "[flaml.automl: 02-17 13:46:37] {940} INFO - iteration 4 current learner rf\n", - "INFO - iteration 4 current learner rf\n", - "[flaml.automl: 02-17 13:46:42] {1094} INFO - at 16.1s,\tbest rf's error=0.3877,\tbest xgboost's error=0.3750\n", - "INFO - at 16.1s,\tbest rf's error=0.3877,\tbest xgboost's error=0.3750\n", - "[flaml.automl: 02-17 13:46:42] {940} INFO - iteration 5 current learner lgbm\n", + "INFO - at 9.6s,\tbest xgboost's error=0.3750,\tbest xgboost's error=0.3750\n", + "INFO - iteration 4 current learner lgbm\n", + "INFO - at 12.5s,\tbest lgbm's error=0.3751,\tbest xgboost's error=0.3750\n", "INFO - iteration 5 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:43] {1094} INFO - at 16.8s,\tbest lgbm's error=0.3751,\tbest xgboost's error=0.3750\n", - "INFO - at 16.8s,\tbest lgbm's error=0.3751,\tbest xgboost's error=0.3750\n", - "[flaml.automl: 02-17 13:46:43] {940} INFO - iteration 6 current learner lgbm\n", - "INFO - iteration 6 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:46] {1094} INFO - at 18.2s,\tbest lgbm's error=0.3558,\tbest lgbm's error=0.3558\n", - "INFO - at 18.2s,\tbest lgbm's error=0.3558,\tbest lgbm's error=0.3558\n", - "[flaml.automl: 02-17 13:46:46] {940} INFO - iteration 7 current learner lgbm\n", + "INFO - at 14.3s,\tbest lgbm's error=0.3558,\tbest lgbm's error=0.3558\n", + "INFO - iteration 6 current learner rf\n", + "INFO - at 22.7s,\tbest rf's error=0.3876,\tbest lgbm's error=0.3558\n", "INFO - iteration 7 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:49] {1094} INFO - at 21.0s,\tbest lgbm's error=0.3492,\tbest lgbm's error=0.3492\n", - "INFO - at 21.0s,\tbest lgbm's error=0.3492,\tbest lgbm's error=0.3492\n", - "[flaml.automl: 02-17 13:46:49] {940} INFO - iteration 8 current learner lgbm\n", + "INFO - at 24.3s,\tbest lgbm's error=0.3492,\tbest lgbm's error=0.3492\n", "INFO - iteration 8 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:50] {1094} INFO - at 23.8s,\tbest lgbm's error=0.3492,\tbest lgbm's error=0.3492\n", - "INFO - at 23.8s,\tbest lgbm's error=0.3492,\tbest lgbm's error=0.3492\n", - "[flaml.automl: 02-17 13:46:50] {940} INFO - iteration 9 current learner lgbm\n", + "INFO - at 27.7s,\tbest lgbm's error=0.3492,\tbest lgbm's error=0.3492\n", "INFO - iteration 9 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:53] {1094} INFO - at 25.3s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "INFO - at 25.3s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:46:53] {940} INFO - iteration 10 current learner lgbm\n", - "INFO - iteration 10 current learner lgbm\n", - "[flaml.automl: 02-17 13:46:54] {1094} INFO - at 28.2s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "INFO - at 28.2s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:46:54] {940} INFO - iteration 11 current learner rf\n", + "INFO - at 29.5s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", + "INFO - iteration 10 current learner rf\n", + "INFO - at 35.9s,\tbest rf's error=0.3876,\tbest lgbm's error=0.3470\n", "INFO - iteration 11 current learner rf\n", - "[flaml.automl: 02-17 13:46:59] {1094} INFO - at 32.9s,\tbest rf's error=0.3847,\tbest lgbm's error=0.3470\n", - "INFO - at 32.9s,\tbest rf's error=0.3847,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:46:59] {940} INFO - iteration 12 current learner xgboost\n", + "INFO - at 43.4s,\tbest rf's error=0.3839,\tbest lgbm's error=0.3470\n", "INFO - iteration 12 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:00] {1094} INFO - at 33.6s,\tbest xgboost's error=0.3750,\tbest lgbm's error=0.3470\n", - "INFO - at 33.6s,\tbest xgboost's error=0.3750,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:47:00] {940} INFO - iteration 13 current learner lgbm\n", - "INFO - iteration 13 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:01] {1094} INFO - at 35.3s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "INFO - at 35.3s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:47:01] {940} INFO - iteration 14 current learner xgboost\n", + "INFO - at 44.3s,\tbest xgboost's error=0.3750,\tbest lgbm's error=0.3470\n", + "INFO - iteration 13 current learner xgboost\n", + "INFO - at 45.1s,\tbest xgboost's error=0.3750,\tbest lgbm's error=0.3470\n", "INFO - iteration 14 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:02] {1094} INFO - at 36.1s,\tbest xgboost's error=0.3750,\tbest lgbm's error=0.3470\n", - "INFO - at 36.1s,\tbest xgboost's error=0.3750,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:47:02] {940} INFO - iteration 15 current learner lgbm\n", + "INFO - at 45.9s,\tbest xgboost's error=0.3736,\tbest lgbm's error=0.3470\n", "INFO - iteration 15 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:04] {1094} INFO - at 37.5s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "INFO - at 37.5s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:47:04] {940} INFO - iteration 16 current learner xgboost\n", - "INFO - iteration 16 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:04] {1094} INFO - at 38.1s,\tbest xgboost's error=0.3736,\tbest lgbm's error=0.3470\n", - "INFO - at 38.1s,\tbest xgboost's error=0.3736,\tbest lgbm's error=0.3470\n", - "[flaml.automl: 02-17 13:47:04] {940} INFO - iteration 17 current learner lgbm\n", + "INFO - at 47.5s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", + "INFO - iteration 16 current learner lgbm\n", + "INFO - at 49.1s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", "INFO - iteration 17 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:08] {1094} INFO - at 39.7s,\tbest lgbm's error=0.3412,\tbest lgbm's error=0.3412\n", - "INFO - at 39.7s,\tbest lgbm's error=0.3412,\tbest lgbm's error=0.3412\n", - "[flaml.automl: 02-17 13:47:08] {940} INFO - iteration 18 current learner lgbm\n", + "INFO - at 50.6s,\tbest lgbm's error=0.3470,\tbest lgbm's error=0.3470\n", "INFO - iteration 18 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:12] {1094} INFO - at 43.6s,\tbest lgbm's error=0.3374,\tbest lgbm's error=0.3374\n", - "INFO - at 43.6s,\tbest lgbm's error=0.3374,\tbest lgbm's error=0.3374\n", - "[flaml.automl: 02-17 13:47:12] {940} INFO - iteration 19 current learner lgbm\n", + "INFO - at 52.5s,\tbest lgbm's error=0.3412,\tbest lgbm's error=0.3412\n", "INFO - iteration 19 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:12] {1094} INFO - at 46.0s,\tbest lgbm's error=0.3374,\tbest lgbm's error=0.3374\n", - "INFO - at 46.0s,\tbest lgbm's error=0.3374,\tbest lgbm's error=0.3374\n", - "[flaml.automl: 02-17 13:47:12] {940} INFO - iteration 20 current learner lgbm\n", - "INFO - iteration 20 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:19] {1094} INFO - at 50.8s,\tbest lgbm's error=0.3311,\tbest lgbm's error=0.3311\n", - "INFO - at 50.8s,\tbest lgbm's error=0.3311,\tbest lgbm's error=0.3311\n", - "[flaml.automl: 02-17 13:47:19] {940} INFO - iteration 21 current learner lgbm\n", - "INFO - iteration 21 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:22] {1094} INFO - at 56.2s,\tbest lgbm's error=0.3311,\tbest lgbm's error=0.3311\n", - "INFO - at 56.2s,\tbest lgbm's error=0.3311,\tbest lgbm's error=0.3311\n", - "[flaml.automl: 02-17 13:47:22] {940} INFO - iteration 22 current learner rf\n", - "INFO - iteration 22 current learner rf\n", - "[flaml.automl: 02-17 13:47:22] {1096} INFO - no enough budget for learner rf\n", + "INFO - at 56.7s,\tbest lgbm's error=0.3374,\tbest lgbm's error=0.3374\n", + "INFO - iteration 20 current learner xgboost\n", + "INFO - at 59.8s,\tbest xgboost's error=0.3696,\tbest lgbm's error=0.3374\n", + "INFO - iteration 21 current learner rf\n", "INFO - no enough budget for learner rf\n", - "[flaml.automl: 02-17 13:47:22] {940} INFO - iteration 23 current learner xgboost\n", - "INFO - iteration 23 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:22] {1096} INFO - no enough budget for learner xgboost\n", - "INFO - no enough budget for learner xgboost\n", - "[flaml.automl: 02-17 13:47:22] {1135} INFO - selected model: LGBMClassifier(colsample_bytree=0.9997863921359742,\n", - " learning_rate=0.1564464373197609, max_bin=511,\n", - " min_child_weight=7.427173668000723, n_estimators=18,\n", - " num_leaves=1846, objective='binary',\n", - " reg_alpha=6.349231150788211e-09, reg_lambda=0.8927146483558472)\n", - "INFO - selected model: LGBMClassifier(colsample_bytree=0.9997863921359742,\n", - " learning_rate=0.1564464373197609, max_bin=511,\n", - " min_child_weight=7.427173668000723, n_estimators=18,\n", - " num_leaves=1846, objective='binary',\n", - " reg_alpha=6.349231150788211e-09, reg_lambda=0.8927146483558472)\n", - "[flaml.automl: 02-17 13:47:22] {895} INFO - fit succeeded\n", + "INFO - selected model: LGBMClassifier(colsample_bytree=0.99236562733598,\n", + " learning_rate=0.32864729892819683, max_bin=1023,\n", + " min_child_weight=5.630999649172112, n_estimators=6,\n", + " num_leaves=1073, objective='binary',\n", + " reg_alpha=1.978160373587824e-09, reg_lambda=1.0)\n", "INFO - fit succeeded\n" ] } ], "source": [ - "'''The main flaml automl API'''\n", "mlflow.set_experiment(\"flaml\")\n", "with mlflow.start_run() as run:\n", + " '''The main flaml automl API'''\n", " automl.fit(X_train = X_train, y_train = y_train, **settings)" ] }, @@ -296,7 +240,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "slideshow": { "slide_type": "slide" @@ -308,7 +252,7 @@ "output_type": "stream", "name": "stdout", "text": [ - "Best ML leaner: lgbm\nBest hyperparmeter config: {'n_estimators': 18.0, 'max_leaves': 1846.0, 'min_child_weight': 7.427173668000723, 'learning_rate': 0.1564464373197609, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 0.9997863921359742, 'reg_alpha': 6.349231150788211e-09, 'reg_lambda': 0.8927146483558472}\nBest accuracy on validation data: 0.6689\nTraining duration of best run: 4.858 s\n" + "Best ML leaner: lgbm\nBest hyperparmeter config: {'n_estimators': 6.0, 'max_leaves': 1073.0, 'min_child_weight': 5.630999649172112, 'learning_rate': 0.32864729892819683, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.99236562733598, 'reg_alpha': 1.978160373587824e-09, 'reg_lambda': 1.0}\nBest accuracy on validation data: 0.6626\nTraining duration of best run: 2.253 s\n" ] } ], @@ -322,7 +266,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": { "slideshow": { "slide_type": "slide" @@ -333,15 +277,15 @@ "output_type": "execute_result", "data": { "text/plain": [ - "LGBMClassifier(colsample_bytree=0.9997863921359742,\n", - " learning_rate=0.1564464373197609, max_bin=511,\n", - " min_child_weight=7.427173668000723, n_estimators=18,\n", - " num_leaves=1846, objective='binary',\n", - " reg_alpha=6.349231150788211e-09, reg_lambda=0.8927146483558472)" + "LGBMClassifier(colsample_bytree=0.99236562733598,\n", + " learning_rate=0.32864729892819683, max_bin=1023,\n", + " min_child_weight=5.630999649172112, n_estimators=6,\n", + " num_leaves=1073, objective='binary',\n", + " reg_alpha=1.978160373587824e-09, reg_lambda=1.0)" ] }, "metadata": {}, - "execution_count": 8 + "execution_count": 7 } ], "source": [ @@ -350,7 +294,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "slideshow": { "slide_type": "slide" @@ -366,7 +310,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": { "slideshow": { "slide_type": "slide" @@ -392,7 +336,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 10, "metadata": { "slideshow": { "slide_type": "slide" @@ -404,10 +348,10 @@ "output_type": "stream", "name": "stdout", "text": [ - "accuracy = 0.6681918633107397\n", - "roc_auc = 0.7208412179342409\n", - "log_loss = 0.6064652793713222\n", - "f1 = 0.5838518559855651\n" + "accuracy = 0.6649585453035315\n", + "roc_auc = 0.7164009096685227\n", + "log_loss = 0.6109261780043986\n", + "f1 = 0.5708885406278198\n" ] } ], @@ -433,7 +377,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 11, "metadata": { "slideshow": { "slide_type": "subslide" @@ -445,7 +389,7 @@ "output_type": "stream", "name": "stdout", "text": [ - "{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482}}\n{'Current Learner': 'xgboost', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bylevel': 1.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}, 'Best Learner': 'xgboost', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bylevel': 1.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 7.0, 'max_leaves': 51.0, 'min_child_weight': 20.0, 'learning_rate': 0.8834537640176922, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.9837052481490312, 'reg_alpha': 4.482246955743696e-08, 'reg_lambda': 0.028657570201141073}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 7.0, 'max_leaves': 51.0, 'min_child_weight': 20.0, 'learning_rate': 0.8834537640176922, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.9837052481490312, 'reg_alpha': 4.482246955743696e-08, 'reg_lambda': 0.028657570201141073}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 15.0, 'max_leaves': 165.0, 'min_child_weight': 11.09973081317571, 'learning_rate': 1.0, 'subsample': 0.9847553005974036, 'log_max_bin': 9.0, 'colsample_bytree': 0.9508927355861483, 'reg_alpha': 2.031936014930936e-06, 'reg_lambda': 0.00624701632609755}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 15.0, 'max_leaves': 165.0, 'min_child_weight': 11.09973081317571, 'learning_rate': 1.0, 'subsample': 0.9847553005974036, 'log_max_bin': 9.0, 'colsample_bytree': 0.9508927355861483, 'reg_alpha': 2.031936014930936e-06, 'reg_lambda': 0.00624701632609755}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 1073.0, 'min_child_weight': 5.630999649172112, 'learning_rate': 0.32864729892819683, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.99236562733598, 'reg_alpha': 1.978160373587824e-09, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 1073.0, 'min_child_weight': 5.630999649172112, 'learning_rate': 0.32864729892819683, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.99236562733598, 'reg_alpha': 1.978160373587824e-09, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 18.0, 'max_leaves': 1846.0, 'min_child_weight': 7.427173668000723, 'learning_rate': 0.1564464373197609, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 0.9997863921359742, 'reg_alpha': 6.349231150788211e-09, 'reg_lambda': 0.8927146483558472}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 18.0, 'max_leaves': 1846.0, 'min_child_weight': 7.427173668000723, 'learning_rate': 0.1564464373197609, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 0.9997863921359742, 'reg_alpha': 6.349231150788211e-09, 'reg_lambda': 0.8927146483558472}}\n" + "{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482}}\n{'Current Learner': 'xgboost', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bylevel': 1.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}, 'Best Learner': 'xgboost', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'colsample_bylevel': 1.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 7.0, 'max_leaves': 51.0, 'min_child_weight': 20.0, 'learning_rate': 0.8834537640176922, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.9837052481490312, 'reg_alpha': 4.482246955743696e-08, 'reg_lambda': 0.028657570201141073}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 7.0, 'max_leaves': 51.0, 'min_child_weight': 20.0, 'learning_rate': 0.8834537640176922, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.9837052481490312, 'reg_alpha': 4.482246955743696e-08, 'reg_lambda': 0.028657570201141073}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 15.0, 'max_leaves': 165.0, 'min_child_weight': 11.09973081317571, 'learning_rate': 1.0, 'subsample': 0.9847553005974036, 'log_max_bin': 9.0, 'colsample_bytree': 0.9508927355861483, 'reg_alpha': 2.031936014930936e-06, 'reg_lambda': 0.00624701632609755}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 15.0, 'max_leaves': 165.0, 'min_child_weight': 11.09973081317571, 'learning_rate': 1.0, 'subsample': 0.9847553005974036, 'log_max_bin': 9.0, 'colsample_bytree': 0.9508927355861483, 'reg_alpha': 2.031936014930936e-06, 'reg_lambda': 0.00624701632609755}}\n{'Current Learner': 'lgbm', 'Current Sample': 364083, 'Current Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 1073.0, 'min_child_weight': 5.630999649172112, 'learning_rate': 0.32864729892819683, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.99236562733598, 'reg_alpha': 1.978160373587824e-09, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 1073.0, 'min_child_weight': 5.630999649172112, 'learning_rate': 0.32864729892819683, 'subsample': 1.0, 'log_max_bin': 10.0, 'colsample_bytree': 0.99236562733598, 'reg_alpha': 1.978160373587824e-09, 'reg_lambda': 1.0}}\n" ] } ], @@ -460,7 +404,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 12, "metadata": { "slideshow": { "slide_type": "slide" @@ -471,8 +415,8 @@ "output_type": "display_data", "data": { "text/plain": "
", - "image/svg+xml": "\r\n\r\n\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n\r\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEWCAYAAAB8LwAVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3df7xVdZ3v8dfbA+qxxIOJDRxEdFQKLUFJM7PUMrBUqKzU29i1R/6YqcbyBiM19vM6Y8OtqW4alxgry9+EiA2BTgZOpvIjiJ/iIBlyMIGxk4YngcPn/rHWxs12ncMCz9qbs/f7+Xicx9nru359lj/253y/3/X9fhURmJmZVdqv1gGYmdm+yQnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThNlekHSGpNW1jsOsSE4Q1utIekrSu2sZQ0T8Z0QMK+r6kkZLekjSC5I2SZon6YKi7meWxQnCLIOkphre+0LgbuAWYDDweuCLwPl7cS1J8v/ntlf8H47VDUn7SbpW0pOS/lvSXZIOLdt/t6Q/SPpT+tf58WX7fijpe5JmSdoCnJXWVD4naWl6zp2SDkyPP1PS+rLzuzw23T9B0jOSNkj6hKSQdEzGMwj4JvC1iJgaEX+KiB0RMS8iLk+P+bKkn5SdMzS9Xp90e66k6yU9DLwIfF7Swor7fFbSzPTzAZL+j6R1kp6VNFlS86v812F1wAnC6snfA+OAdwKDgD8CN5bt/zlwLHA48Bvg1orzLwGuBw4GfpWWfRgYAxwFvBn4n93cP/NYSWOAa4B3A8ek8XVlGHAEMK2bY/L4G+AKkmf5v8AwSceW7b8EuC39/HXgOGBEGl8rSY3FGpwThNWTK4EvRMT6iHgJ+DJwYekv64i4OSJeKNt3oqRDys6/NyIeTv9i/0ta9p2I2BARzwH3kXyJdqWrYz8M/CAiVkTEi8BXurnG69Lfz+R+6mw/TO+3PSL+BNwLXAyQJoo3ADPTGsvlwGcj4rmIeAH4J+CiV3l/qwNOEFZPjgTukdQuqR1YBXQCr5fUJOmGtPnpeeCp9JzDys5/OuOafyj7/CLw2m7u39WxgyqunXWfkv9Ofw/s5pg8Ku9xG2mCIKk9zEiT1QDgIGBR2T+32Wm5NTgnCKsnTwPnRkRL2c+BEdFG8qU4lqSZ5xBgaHqOys4vamrjZ0g6m0uO6ObY1STP8cFujtlC8qVe8lcZx1Q+y/3AYZJGkCSKUvPSZqADOL7sn9khEdFdIrQG4QRhvVVfSQeW/fQBJgPXSzoSQNIASWPT4w8GXiL5C/0gkmaUarkLuEzSGyUdRDft+5HMv38NcJ2kyyT1Szvf3y5pSnrYEuAdkoakTWQTdxdARGwn6deYBBwKPJCW7wC+D/yrpMMBJLVKGr3XT2t1wwnCeqtZJH/5ln6+DHwbmAncL+kF4FHg1PT4W4DfA23AynRfVUTEz4HvAL8E1gCPpLte6uL4acBHgI8DG4Bngf9N0o9ARDwA3AksBRYBP8sZym0kNai704RR8g9pXI+mzW//QdJZbg1OXjDIrLokvRFYDhxQ8UVttk9xDcKsCiS9X9L+kvqTvFZ6n5OD7eucIMyq40pgE/AkyZtVf1vbcMx2z01MZmaWyTUIMzPL1KfWAfSkww47LIYOHVrrMMzMeo1FixZtjojMgZF1lSCGDh3KwoULd3+gmZkBIOn3Xe0rtIlJ0hhJqyWtkXRtF8ecKWmJpBWS5qVlw9Ky0s/zkj5TZKxmZrarwmoQ6Xz6NwLnAOuBBZJmRsTKsmNagJuAMRGxrjSSMyJWk050ll6nDbinqFjNzOyViqxBnAKsiYi1EbEVuINkLpxylwDTI2IdQERszLjOu4AnI6LLapCZmfW8IhNEK7vOKLk+LSt3HNA/XeBkkaRLM65zEXB7QTGamVkXiuykVkZZ5aCLPsDJJLWEZuARSY9GxBMAkvYHLqCbycgkXUGyMApDhgzpgbDNzAyKTRDr2XVa48EkE49VHrM5IrYAWyQ9BJwIPJHuPxf4TUQ829VNImIKMAVg1KhRHvVnZg1jxuI2Js1ZzYb2Dga1NDN+9DDGjaxsqNl7RTYxLQCOlXRUWhO4iGSmzXL3AmdI6pNOg3wqySIvJRfj5iUzs1eYsbiNidOX0dbeQQBt7R1MnL6MGYvbeuwehSWIdCKyTwFzSL7074qIFZKuknRVeswqktWrlgLzgakRsRwgTRjnANOLitHMrLeaNGc1Hds6dynr2NbJpDmre+wehQ6Ui4hZJPP2l5dNrtieRLKISeW5L/Ly+rxmZlZmQ3vHHpXvDc/FZGbWCw1qad6j8r3hBGFm1guNHz2M5r5Nu5Q1921i/OieWwywruZiMjNrFKW3lSZMW8rWzh20FvAWkxOEmVkvNW5kK7fPXwfAnVee1uPXdxOTmZllcoIwM7NMThBmZpbJCcLMzDI5QZiZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoIwM7NMThBmZpbJCcLMzDI5QZiZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoIwM7NMThBmZpbJa1KbmaVmLG5j0pzVbGjvYFBLM+NHD2PcyNZah1UzThBmZiTJYeL0ZXRs6wSgrb2DidOXATRsknCCMDMDJs1ZvTM5lHRs62TCtKXcPn9djaLavZXPPM/wgf0Kubb7IMzMgA3tHZnlWzt3VDmSPTN8YD/GjiimhuMahJkZMKilmbaMJNHa0sydV55Wg4hqzzUIMzNg/OhhNPdt2qWsuW8T40cPq1FEtecahJkZL3dET5i2lK2dO2j1W0xOEGZmJeNGtu7skG7UZqVybmIyM7NMThBmZpbJCcLMzDIVmiAkjZG0WtIaSdd2ccyZkpZIWiFpXll5i6Rpkh6XtEqSGwTNzKqosE5qSU3AjcA5wHpggaSZEbGy7JgW4CZgTESsk3R42SW+DcyOiAsl7Q8cVFSsZmb2SkXWIE4B1kTE2ojYCtwBjK045hJgekSsA4iIjQCS+gHvAP4tLd8aEe0FxmpmZhWKTBCtwNNl2+vTsnLHAf0lzZW0SNKlafnRwCbgB5IWS5oq6TUFxmpmZhWKTBDKKIuK7T7AycD7gNHAdZKOS8tPAr4XESOBLUBXfRhXSFooaeGmTZt6LHgzs0ZXZIJYDxxRtj0Y2JBxzOyI2BIRm4GHgBPT8vUR8Vh63DSShPEKETElIkZFxKgBAwb06AOYmTWyIhPEAuBYSUelncwXATMrjrkXOENSH0kHAacCqyLiD8DTkkqToLwLWImZNbQZi9s4/YYHOeraf+f0Gx5kxuK2WodU1wp7iykitkv6FDAHaAJujogVkq5K90+OiFWSZgNLgR3A1IhYnl7i08CtaXJZC1xWVKxmtu/zgj7Vp4jKboHea9SoUbFw4cJah2FmBTj9hgczp+Pev2k/Rg5p6bH7lBbgaZS5mCQtiohRWfs8ktrMeoVqLehT5AI8vY1nczWzXsEL+lSfaxBm1it4QZ/qcw3CzHoFL+hTfU4QZtZreEGf6nITk5mZZXKCMDOzTLtNEJIOrUYgZma2b8lTg3hM0t2S3ispawI+MzOrQ3kSxHHAFOBvgDWS/imdcdXMzOrYbhNEJB6IiIuBTwAfA+ZLmudlQM3M6tduX3OV9DrgoyQ1iGdJJtGbCYwA7gaOKjJAMzOrjTzjIB4BfgyMi4j1ZeULJU0uJiwzM6u1PAliWHQx5WtEfL2H4zEzs31Enk7q+yXtnEtXUn9JcwqMyczM9gF5EsSAiGgvbUTEH4HDiwvJzMz2BXkSRKekIaUNSUcC9bPKkJmZZcrTB/EF4FeS5qXb7wCuKC4kMzPbF+w2QUTEbEknAW8FBHw2IjYXHpmZmdVU3um+O4GNwIHAcElExEPFhWVmZrWWZ6DcJ4CrgcHAEpKaxCPA2cWGZmZmtZSnk/pq4C3A7yPiLGAksKnQqMzMrObyNDH9JSL+IglJB0TE45K8CKxZhhmL25g0ZzUb2jsY5CUxrZfLkyDWpwPlZgAPSPojsKHYsMx6nxmL25g4fRkd2zoBaGvvYOL0ZQBOEtYr5XmL6f3pxy9L+iVwCDC70KjMeqFJc1bvTA4lHds6mTBt6c51lO3VW/nM8wwf2K/WYTSEbhOEpP2ApRFxAkBEzOvueLNGtqG9I7N8a+eOKkdS34YP7MfYEa6RVUO3CSIidkj6raQhEeE/gcy6MailmbaMJNHa0sydV3rpFOt98rzFNBBYIekXkmaWfooOzKy3GT96GM19m3Ypa+7bxPjRfqfDeqc8ndRfKTwKszpQ6oieMG0pWzt30Oq3mKyXy9NJ7X4Hs5zGjWzd2SHtZiXr7fKMpH6Bl2dv3R/oC2yJCL9GYGZWx/LUIA4u35Y0DjilsIjMCuABbGZ7Lk8n9S4iYgaeh8l6kdIAtrb2DoKXB7DNWNxW69DM9ml5mpg+ULa5HzAKLxhkvUi1B7B5IJfVizxvMZ1f9nk78BQwNs/FJY0Bvg00AVMj4oaMY84EvkXSt7E5It6Zlj8FvEAy1fj2iBiV555mlao9gM0Duaxe5OmDuGxvLiypCbgROAdYDyyQNDMiVpYd0wLcBIyJiHWSKte6PsuLE9mr5QFsZntnt30Qkn6UfpGXtvtLujnHtU8B1kTE2ojYCtzBK2selwDTS6O0I2Jj/tDN8vEANrO9k6eT+s0R0V7aiIg/kqwJsTutwNNl2+vTsnLHAf0lzZW0SNKlZfsCuD8t73INbElXSFooaeGmTV6mwl5p3MhW/vkDb2L/puQ/99aWZv75A2/yW0xmu5GnD2I/Sf3TxICkQ3Oep4yyys7tPsDJwLuAZuARSY9GxBPA6RGxIW12ekDS41nLnEbEFGAKwKhRo9x5bpk8gM1sz+X5ov8G8GtJ00i+4D8MXJ/jvPXAEWXbg3nlOhLrSTqmtwBbJD0EnAg8EREbIGl2knQPSZOV18E2M6uS3TYxRcQtwAeBZ0mWGv1ARPw4x7UXAMdKOkrS/sBFQOUkf/cCZ0jqI+kg4FRglaTXSDoYQNJrgPcAy/M+lJmZvXp5xkG8FVgREd9Ntw+WdGpEPNbdeRGxXdKngDkkr7neHBErJF2V7p8cEaskzQaWAjtIXoVdLulo4B5JpRhviwgvUmRmVkV5mpi+B5xUtr0loyxTRMwCZlWUTa7YngRMqihbS9LUZGZmNZLnLSZFxM7O34jYQb7EYmZmvVieBLFW0t9L6pv+XA2sLTowMzOrrTwJ4irgbUAbyVtHpwKXFxmUmZnVXp6pNjaSvIEEgKRm4Dzg7gLjMjOzGss13bekJknnSroF+B3wkWLDMjOzWuu2BiHpHSTzJb0PmA+cDhwdES9WITYzM6uhLhOEpPXAOpJXWsdHxAuSfufkYGbWGLprYvopyeR6HwHOT0c0e64jM7MG0WWCiIirgaHAN4GzgCeAAZI+LOm11QnPzMxqpdtO6kg8GBGXkySLS4BxJKvKmZlZHcs9IjoitgH3Afelr7qamVkdy/Waa6WIyF7k18zM6sZeJQgzM6t/ThBmZpYpz3oQxwHjgSPLj4+IswuMy8zMaixPJ/XdwGTg+0BnseGYmdm+Ik+C2B4R3ys8ErMuzFjcxqQ5q9nQ3sGglmbGjx7GuJGttQ7LrO7lSRD3Sfo74B7gpVJhRDxXWFRmqRmL25g4fRkd25LKa1t7BxOnLwNwkjArWJ4E8bH09/iysgCO7vlwzHY1ac7qncmhpGNbJxOmLeX2+ev26Forn3me4QP79WR4ZnUtz3oQR1UjELMsG9qzh9xs7dyxx9caPrAfY0e41mGWV563mPoCfwu8Iy2aC/y/dGS1WaEGtTTTlpEkWluaufPK02oQkVnjyDMO4nvAycBN6c/JaZlZ4caPHkZz36Zdypr7NjF+9LAaRWTWOPL0QbwlIk4s235Q0m+LCsisXKkjesK0pWzt3EGr32Iyq5o8CaJT0l9HxJMAko7G4yGsisaNbN3ZIe1mJbPqyZMgxgO/lLQWEMmI6ssKjcrMzGouz1tMv5B0LDCMJEE8HhEv7eY0MzPr5bpbk/rsiHhQ0gcqdv21JCJiesGxmZlZDXVXg3gn8CBwfsa+AJwgzMzqWJcJIiK+lH78akT8rnyfJA+eMzOrc3nGQfw0o2xaTwdiZmb7lu76IN4AHA8cUtEP0Q84sOjAzMystrrrgxgGnAe0sGs/xAvA5UUGZWZmtdddH8S9wL2STouIR6oYk5mZ7QPyDJRbLOmTJM1NO5uWIuLjhUVlZmY1l6eT+sfAXwGjgXnAYJJmpt2SNEbSaklrJF3bxTFnSloiaYWkeRX7miQtlvSzPPczM7OekydBHBMR1wFbIuJHwPuAN+3uJElNwI3AucBw4GJJwyuOaSGZIfaCiDge+FDFZa4GVuWI0czMelieBFFa96Fd0gnAIcDQHOedAqyJiLURsRW4AxhbccwlwPSIWAcQERtLOyQNJklGU3Pcy8zMelieBDFFUn/gOmAmsBL4lxzntQJPl22vT8vKHQf0lzRX0iJJl5bt+xYwAeh26TBJV0haKGnhpk2bcoRlZmZ55Jmsr/QX/Dz2bB1qZV0u4/4nA+8CmoFHJD1Kkjg2RsQiSWfuJr4pwBSAUaNGVV7fzMz2UncD5a7p7sSI+OZurr0eOKJsezCwIeOYzRGxBdgi6SHgROAk4AJJ7yV5c6qfpJ9ExEd3c08zM+sh3TUxHZz+jCJZk7o1/bmKpNN5dxYAx0o6StL+wEUkTVTl7gXOkNRH0kHAqcCqiJgYEYMjYmh63oNODmZm1dXdQLmvAEi6HzgpIl5It78M3L27C0fEdkmfAuYATcDNEbFC0lXp/skRsUrSbGApSV/D1IhY/iqfyczMekCegXJDgK1l21vJ9xYTETELmFVRNrliexIwqZtrzAXm5rmfmZn1nDwJ4sfAfEn3kHQyvx+4pdCozMys5vK8xXS9pJ8DZ6RFl0XE4mLDMjOzWuvuLaZ+EfG8pEOBp9Kf0r5DI+K54sMzM7Na6a4GcRvJdN+L2HX8gtLtPRkTYWZmvUx3bzGdl/728qJmZg2ouyamk7o7MSJ+0/PhmJnZvqK7JqZvdLMvgLN7OBYzM9uHdNfEdFY1AzEzs31LnnEQpNN8D2fXFeU8FsLMrI7tNkFI+hJwJkmCmEWyANCv8GA5M7O6lmc9iAtJpuP+Q0RcRjLb6gGFRmVmZjWXJ0F0RMQOYLukfsBGPAbCzKzu5emDWJiuHf19kkFzfwbmFxqVmZnVXHfjIL4L3BYRf5cWTU6n5u4XEUurEp2ZmdVMdzWI/wK+IWkgcCdwe0QsqU5YZmZWa132QUTEtyPiNOCdwHPADyStkvRFScdVLUIzM6uJ3XZSR8TvI+LrETESuIRkPYhVhUdmZmY1tdsEIamvpPMl3Qr8HHgC+GDhkZmZWU1110l9DnAx8D6St5buAK6IiC1Vis3MzGqou07qz5OsCfE5Lw5kZtZ4PFmfmZllyjOS2szMGpAThJmZZXKCMDOzTE4QZmaWyQnCzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLJMThJmZZXKCMDOzTE4QZmaWyQnCzMwyFZogJI2RtFrSGknXdnHMmZKWSFohaV5adqCk+ZJ+m5Z/pcg4zczslbpbD+JVkdQE3AicA6wHFkiaGREry45pAW4CxkTEOkmHp7teAs6OiD9L6gv8StLPI+LRouI1M7NdFVmDOAVYExFrI2IryYp0YyuOuQSYHhHrACJiY/o7IuLP6TF9058oMFYzM6tQZIJoBZ4u216flpU7Dugvaa6kRZIuLe2Q1CRpCbAReCAiHsu6iaQrJC2UtHDTpk09/AhmZo2ryAShjLLKWkAf4GSSda9HA9dJOg4gIjojYgQwGDhF0glZN4mIKRExKiJGDRgwoOeiNzNrcEUmiPXAEWXbg4ENGcfMjogtEbEZeAg4sfyAiGgH5gJjigvVzMwqFZkgFgDHSjpK0v7ARcDMimPuBc6Q1EfSQcCpwCpJA9IObCQ1A+8GHi8wVjMzq1DYW0wRsV3Sp4A5QBNwc0SskHRVun9yRKySNBtYCuwApkbEcklvBn6Uvgm1H3BXRPysqFjNzOyVCksQABExC5hVUTa5YnsSMKmibCkwssjYzMysex5JbWZmmZwgzMwskxOEmZllcoIwM7NMThBmZpbJCcLMzDI5QZiZWSYnCDMzy+QEYWZmmZwgzMwskxOEmZllcoIwM7NMThBmZpap0NlcG9mMxW1MmrOaDe0dDGppZvzoYYwbWbniau+5j5k1HieIAsxY3MbE6cvo2NYJQFt7BxOnLwPo0S/vat3HzBqTE0QBJs1ZvfNLu6RjWycTpi3l9vnreuw+i9e1s7VzR+H32ResfOZ5hg/sV+swzBqK+yAKsKG9I7O88sv81erqej19n33B8IH9GDvCtSKzanINogCDWpppy0gSrS3N3HnlaT12n9NveLAq9zGzxuQaRAHGjx5Gc9+mXcqa+zYxfvSwXnkfM2tMrkEUoNRBPGHaUrZ27qC1oLeLStfzW0xmVgQniIKMG9m6s6O4yOaecSNbnRDMrBBuYjIzs0wNX4PwQDMzs2wNnSA80MzMrGsNnSCKHtDmwV1m1ps1dB9E0QPaPLjLzHqzhq5BVGtAm5lZb9TQNQgPNDMz61pD1yA80MzMrGsNnSDAA83MzLrS0E1MZmbWNScIMzPL5ARhZmaZnCDMzCyTE4SZmWVSRNQ6hh4jaRPw+90cdhiwuQrh7Ev8zI3Bz9wYevqZj4yIAVk76ipB5CFpYUSMqnUc1eRnbgx+5sZQzWd2E5OZmWVygjAzs0yNmCCm1DqAGvAzNwY/c2Oo2jM3XB+EmZnl04g1CDMzy8EJwszMMjVUgpA0RtJqSWskXVvreIog6WZJGyUtLys7VNIDkv4r/d2/ljH2JElHSPqlpFWSVki6Oi2v52c+UNJ8Sb9Nn/kraXndPnOJpCZJiyX9LN2u62eW9JSkZZKWSFqYllXtmRsmQUhqAm4EzgWGAxdLGl7bqArxQ2BMRdm1wC8i4ljgF+l2vdgO/K+IeCPwVuCT6b/Xen7ml4CzI+JEYAQwRtJbqe9nLrkaWFW23QjPfFZEjCgb+1C1Z26YBAGcAqyJiLURsRW4Axhb45h6XEQ8BDxXUTwW+FH6+UfAuKoGVaCIeCYifpN+foHky6OV+n7miIg/p5t905+gjp8ZQNJg4H3A1LLiun7mLlTtmRspQbQCT5dtr0/LGsHrI+IZSL5QgcNrHE8hJA0FRgKPUefPnDa1LAE2Ag9ERN0/M/AtYAKwo6ys3p85gPslLZJ0RVpWtWdupBXllFHmd3zrhKTXAj8FPhMRz0tZ/7rrR0R0AiMktQD3SDqh1jEVSdJ5wMaIWCTpzFrHU0WnR8QGSYcDD0h6vJo3b6QaxHrgiLLtwcCGGsVSbc9KGgiQ/t5Y43h6lKS+JMnh1oiYnhbX9TOXREQ7MJek36men/l04AJJT5E0D58t6SfU9zMTERvS3xuBe0iayqv2zI2UIBYAx0o6StL+wEXAzBrHVC0zgY+lnz8G3FvDWHqUkqrCvwGrIuKbZbvq+ZkHpDUHJDUD7wYep46fOSImRsTgiBhK8v/ugxHxUer4mSW9RtLBpc/Ae4DlVPGZG2oktaT3krRjNgE3R8T1NQ6px0m6HTiTZErgZ4EvATOAu4AhwDrgQxFR2ZHdK0l6O/CfwDJebpv+PEk/RL0+85tJOiebSP7IuysivirpddTpM5dLm5g+FxHn1fMzSzqapNYASXfAbRFxfTWfuaEShJmZ5ddITUxmZrYHnCDMzCyTE4SZmWVygjAzs0xOEGZmlskJwnoNSf8q6TNl23MkTS3b/oaka7o5/4eSLkw/z5X0ioXfJfWVdEM6U+bydNbUc9N9T0k6bC/i3nnfLvbfmM7WuVJSR/p5iaQLJc0qjXnoSZIGlmZE7WL//pIektRIsy1YBScI601+DbwNQNJ+JGM9ji/b/zbg4Vd5j68BA4ETIuIE4Hzg4Fd5zW5FxCcjYgTwXuDJdObOERExLSLem46W7mnXAN/vJqatJDOFfqSAe1sv4QRhvcnDpAmCJDEsB16Q1F/SAcAbgcWSvihpQVoDmKKcEzNJOgi4HPh0RLwEEBHPRsRdGcdek15/eUWt5lJJS9O1Gn6ccd7X0hpFrv/3SrUWSUMlPS5panrPWyW9W9LDaW3nlPT41yhZE2SBknUTupqx+IPA7PSc49Oa0pI09mPTY2YA/yNPnFafXH20XiOdtGy7pCEkieIRkhl5TwP+BCyNiK2SvhsRXwVIv6TPA+7LcYtjgHUR8Xx3B0k6GbgMOJVkEsjHJM0DtgJfIJlgbbOkQyvO+xfgEOCy2LsRqscAHwKuIJk65hLg7cAFJKPHx6X3fzAiPp42Tc2X9B8RsaUsjqOAP5aSIHAV8O2IuDWdhqYpLV8OvGUv4rQ64RqE9TalWkQpQTxStv3r9JizJD0maRlwNrs2Q/WEtwP3RMSWdF2G6cAZ6b2mRcRmgIrpD64DWiLiyr1MDgC/i4hlEbEDWEGyaEyQTDMyND3mPcC1SqYCnwscSDIlQ7mBwKay7UeAz0v6B+DIiOhI4+8EtpbmA7LG4wRhvU2pH+JNJH/hPkpSg3gb8LCkA4GbgAsj4k0k7ewH5rz2GmBIji/ErpqsRNdTyC8ATq6sVeyhl8o+7yjb3sHLrQECPljWjzEkIspXYAPooOyfSUTcRlIL6QDmSDq77NgDgL+8ipitF3OCsN7mYZImo+ciojP9K72FJEk8wstffJuVrBHR5dtDlSLiRZKZYb+TNrWU3vb5aMWhDwHjJB2UzrL5fpIJA38BfDidTI2KZDAbuAH494L/Ip8DfLrU7yJpZMYxT/ByjaM0KdzaiPgOyUyhb07LXwdsiohtBcZr+zAnCOttlpG8vfRoRdmfImJz+sbP99OyGSR/ue+JfyRpflkpaXl6jfLmGNIlTn8IzCeZNXZqRCyOiBXA9cA8Sb8Fvllx3t1pbDPTabqL8DWSJUiXpvF/rfKAtD/iSUnHpEUfAZanzVJvAG5Jy88CZhUUp/UCns3VrAFJej9wckT8YzfHTAcmRsTq6kVm+xK/xWTWgCLinlJTWJa0iTco/qkAAAAuSURBVG2Gk0Njcw3CzMwyuQ/CzMwyOUGYmVkmJwgzM8vkBGFmZpmcIMzMLNP/ByWwIdumQhb/AAAAAElFTkSuQmCC\n" + "image/svg+xml": "\r\n\r\n\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n\r\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de7hdVX3u8e/LTpCgYKAGT0jARA1pg0giu6BFLOAl8UYC9QIchWNbgdPS0tpCgy0t1mOlxkttC6RAqWhFBBpCtJhIQaC1KCRNDAkxmlIMuZQENYIxNSR5zx9z7LBYrL33WmSvfX0/z7OeueaYY845Bpf122OMOceQbSIiIpq130AXICIihpYEjoiIaEkCR0REtCSBIyIiWpLAERERLUngiIiIliRwRPQxSSdJWjvQ5YholwSOGFYkPSrpTQNZBtv/antqu64vaaak+yQ9JWmrpHslndau+0XUS+CIaJGkjgG897uAW4DPAxOBlwJ/CrzzeVxLkvIbEC3LfzQxIkjaT9JcSf8p6YeSbpZ0aM3xWyT9t6SflL/mj6459jlJV0u6Q9J24JTSsvlDSSvLOV+WdEDJf7KkDTXnd5u3HL9E0mZJmyT9piRLemWDOgj4NPBR29fZ/ontPbbvtf3BkudySf9Yc86kcr1RZf8eSR+T9E3gZ8CHJS2tu8/vS1pUvr9A0iclrZf0uKT5ksbs47+OGOISOGKk+F1gDvCrwOHAj4Era45/DZgCHAb8B/DFuvPPBj4GHAT8W0l7DzALmAy8Gvg/Pdy/YV5Js4APAW8CXlnK152pwBHArT3kacb7gfOo6vI3wFRJU2qOnw3cWL7/JXAUML2UbwJVCydGsASOGCnOB/7Y9gbbPwcuB97V9Ze47ettP1Vz7FhJL645/3bb3yx/4f9PSftr25ts/wj4CtWPa3e6y/se4B9sr7b9M+AjPVzjF8p2c9O1buxz5X67bP8EuB04C6AEkF8EFpUWzgeB37f9I9tPAX8BnLmP948hLoEjRoqXAbdJ2iZpG7AG2A28VFKHpCtKN9aTwKPlnJfUnP9Yg2v+d833nwEv6uH+3eU9vO7aje7T5YdlO76HPM2ov8eNlMBB1dpYWILYOOBAYFnNP7fFJT1GsASOGCkeA95qe2zN5wDbG6l+LGdTdRe9GJhUzlHN+e2aRnoz1SB3lyN6yLuWqh6/1kOe7VQ/9l3+V4M89XX5OvASSdOpAkhXN9UTwA7g6Jp/Zi+23VOAjBEggSOGo9GSDqj5jALmAx+T9DIASeMkzS75DwJ+TvUX/YFU3TH95WbgA5J+SdKB9DB+4GoNhA8Bl0n6gKSDy6D/6yVdU7KtAN4g6cjS1XZpbwWwvYtq3GQecChwZ0nfA1wLfEbSYQCSJkia+bxrG8NCAkcMR3dQ/aXc9bkc+CywCPi6pKeAbwEnlPyfB34AbAQeLsf6he2vAX8NfANYB9xfDv28m/y3Au8Ffh3YBDwO/D+qcQps3wl8GVgJLAO+2mRRbqRqcd1SAkmXPyrl+lbpxvsXqkH6GMGUhZwiBg9JvwSsAl5Q9wMeMWikxRExwCSdLml/SYdQPf76lQSNGMwSOCIG3vnAVuA/qZ70+r8DW5yInqWrKiIiWpIWR0REtGTUQBegP7zkJS/xpEmTBroYERFDyrJly56w/ZwXPkdE4Jg0aRJLly7tPWNEROwl6QeN0tNVFRERLUngiIiIliRwRERESxI4IiKiJQkcERHRkhHxVFVExEizcPlG5i1Zy6ZtOzh87BgunjmVOTMm9Mm1EzgiIoaZhcs3cumCh9jx9G4ANm7bwaULHgLok+CRrqqIiGFm3pK1e4NGlx1P72bekrV9cv0EjoiIYWbTth0tpbcqgSMiYpg5fOyYltJblcARETHMXDxzKmNGdzwrbczoDi6e2TeLN2ZwPCJimOkaAL/k1pXs3L2HCXmqKiIiejNnxgS+9MB6AL58/uv69NrpqoqIiJYkcEREREsSOCIioiUJHBER0ZIEjoiIaElbA4ekWZLWSlonaW43eU6WtELSakn31qSPlXSrpO9KWiPpdSX9UEl3Svp+2R7SzjpERMSztS1wSOoArgTeCkwDzpI0rS7PWOAq4DTbRwPvrjn8WWCx7V8EjgXWlPS5wF22pwB3lf2IiOgn7WxxHA+ss/2I7Z3ATcDsujxnAwtsrwewvQVA0sHAG4C/L+k7bW8r58wGbijfbwDmtLEOERFRp52BYwLwWM3+hpJW6yjgEEn3SFom6ZyS/nJgK/APkpZLuk7SC8uxl9reDFC2hzW6uaTzJC2VtHTr1q19VaeIiBGvnYFDDdJctz8KOA54OzATuEzSUSX9NcDVtmcA22mxS8r2NbY7bXeOGzeu5cJHRERj7QwcG4AjavYnApsa5Flse7vtJ4D7qMYzNgAbbH+75LuVKpAAPC5pPEDZbmlT+SMiooF2Bo4HgSmSJkvaHzgTWFSX53bgJEmjJB0InACssf3fwGOSuqZyfCPwcPm+CDi3fD+3XCMiYkhZuHwjJ15xN5Pn/jMnXnE3C5dvHOgiNa1tkxza3iXpQmAJ0AFcb3u1pAvK8fm210haDKwE9gDX2V5VLvE7wBdL0HkE+EBJvwK4WdJvAOt59pNYERGDXruXdm032fXDDsNPZ2enly5dOtDFiIgA4MQr7mZjg9X49u/YjxlHju2z+zy8+UmmjT/4ec+OK2mZ7c769Lw5HhHRz7pbwnXn7j19ep9p4w9m9vS+b8FkPY6IiH52+NgxDVscE8aO6fO1M9ohLY6IiH7W7qVd2y0tjoiIftbupV3bLYEjImIAtHNp13ZLV1VERLQkgSMiIlqSwBERES1J4IiIiJYkcEREREsSOCIioiUJHBER0ZIEjoiIaEkCR0REtCSBIyIiWpLAERERLWlr4JA0S9JaSeskze0mz8mSVkhaLenemvRHJT1Uji2tSb9c0saSvkLS29pZh4iIeLa2TXIoqQO4EngzsAF4UNIi2w/X5BkLXAXMsr1e0mF1lznF9hMNLv8Z259sV9kjIqJ77WxxHA+ss/2I7Z3ATcDsujxnAwtsrwewvaWN5YmIiD7QzsAxAXisZn9DSat1FHCIpHskLZN0Ts0xA18v6efVnXehpJWSrpd0SKObSzpP0lJJS7du3bqvdYmIiKKdgUMN0ly3Pwo4Dng7MBO4TNJR5diJtl8DvBX4bUlvKOlXA68ApgObgU81urnta2x32u4cN27cvtUkIiL2amfg2AAcUbM/EdjUIM9i29vLWMZ9wLEAtjeV7RbgNqquL2w/bnu37T3AtV3pERHRP9oZOB4EpkiaLGl/4ExgUV2e24GTJI2SdCBwArBG0gslHQQg6YXAW4BVZX98zfmnd6VHRET/aNtTVbZ3SboQWAJ0ANfbXi3pgnJ8vu01khYDK4E9wHW2V0l6OXCbpK4y3mh7cbn0JyRNp+r2ehQ4v111iIiI52rrmuO27wDuqEubX7c/D5hXl/YIpcuqwTXf38fFjIiIFuTN8YiIaElbWxwR0bOFyzcyb8laNm3bweFjx3DxzKnMmVH/1HrE4JLAETFAFi7fyKULHmLH07sB2LhtB5cueAggwSMGtQSOiAEyb8navUGjy46nd3PJrSv50gPrB6hU0Z8e3vwk08YfPNDFaFnGOCIGyKZtOxqm79y9p59LEgNl2viDmT196LUu0+KIGCCHjx3DxgbBY8LYMXz5/NcNQIkimpMWR8QAuXjmVMaM7nhW2pjRHVw8c+oAlSiiOb0GDkmH9kdBIkaaOTMm8PEzjmH/jup/wwljx/DxM47JwHgMes10VX1b0grgH4Cv2a6fqDAinqc5MybsHQhP91QMFc10VR0FXAO8H1gn6S9qZrCNiIgRptfA4cqdts8CfhM4F3hA0r2S8idSRMQI02tXlaRfAN5H1eJ4HPgdqllupwO3AJPbWcCIiBhcmhnjuB/4AjDH9oaa9KWS5ndzTsSQlClAInrXTOCY2t2AuO2/7OPyRAyYTAES0ZxmAsfXJb3b9jaAssb3TbZntrdoEf1roKYAGarTTsTI1cxTVeO6ggaA7R8Dh7WvSBEDY6CmABmq007EyNVMi2O3pCNtrweQ9DKq1fd6JWkW8FmqFQCvs31FgzwnA38FjAaesP2rJf1R4ClgN7DLdmdJPxT4MjCJagXA95RgFrFPMgVIRHOaaXH8MfBvkr4g6QvAfcClvZ0kqQO4EngrMA04S9K0ujxjgauA02wfDby77jKn2J7eFTSKucBdtqcAd5X9iH2WKUAimtNri8P2YkmvAV4LCPh92080ce3jgXVlGVgk3QTMBh6uyXM2sKCrNWN7SxPXnQ2cXL7fANwD/FET50X0qGsA/JJbV7Jz9x4m5KmqiIaanR13N7AFOACYJgnb9/VyzgTgsZr9DcAJdXmOAkZLugc4CPis7c+XY6YamDfwd7avKekvtb0ZwPZmSQ3HWySdB5wHcOSRRzZRxYhMARLRjGZeAPxN4CJgIrCCquVxP3Bqb6c2SKsfGxkFHAe8ERgD3C/pW7a/B5xoe1MJDHdK+m4TweqZG1WB5hqAzs7OzK8VEdFHmhnjuAj4ZeAHtk8BZgBbmzhvA3BEzf5EYFODPIttby/dX/cBxwLY3lS2W4DbqLq+AB6XNB6gbJvp3oqIiD7STOD4H9v/AyDpBba/CzQzWvggMEXSZEn7A2dSTVVS63bgJEmjJB1I1ZW1RtILJR1U7vlC4C3AqnLOIqr5sijb25soS0RE9JFmxjg2lKefFlJ1Gf2Y57YcnsP2LkkXAkuoHse93vZqSReU4/Ntr5G0GFgJ7KF6ZHeVpJcDt0nqKuONtheXS18B3CzpN4D1PPdJrIiIaKNmnqo6vXy9XNI3gBcDi3s4pfbcO4A76tLm1+3PA+bVpT1C6bJqcM0fUo2JRETEAOgxcEjaD1hp+1UAtu/tl1JFRMSg1eMYh+09wHck5XnWiIgAmhvjGA+slvQAsL0r0fZpbStVREQMWs0Ejo+0vRQRETFkNDM4nnGNiIjYq5k3x5/imTe+96eaxXa77SwgEBExAjXT4jiodl/SHJ55izsiIkaYZt4cfxbbC+l9nqqIiBimmumqOqNmdz+gkyYXcoqIiOGnmaeq3lnzfRfVqnuz21KaiIgY9JoZ4/hAfxQkIiKGhl7HOCTdUCY57No/RNL17S1WREQMVs0Mjr/a9rauHds/plqTIyIiRqBmAsd+kg7p2pF0KM0vORsREcNMMwHgU8C/S7qV6mmq9wAfa2upIiJi0GpmcPzzkpZSvbsh4AzbD7e9ZBERMSg1Mzj+WuAx239r+2+AxySd0MzFJc2StFbSOklzu8lzsqQVklZLurfuWIek5ZK+WpN2uaSN5ZwVkt7WTFkiIqJvNNNVdTXwmpr97Q3SnkNSB3Al8GZgA/CgpEW1rZXytNZVwCzb6yUdVneZi4A1QP28WJ+x/ckmyh4REX2smcFx2d77pnhZ3KmZgHM8sM72I7Z3Ajfx3BcHzwYW2F5frr1l702licDbgeuauFdERPSTZgLHI5J+V9Lo8rkIeKSJ8yYAj9XsbyhptY4CDpF0j6Rlks6pOfZXwCXAngbXvlDSSknX1z7xVUvSeZKWSlq6devWJoobERHNaCZwXAD8CrCR6sf/BOCDTZynBmn1c1yNAo6jalnMBC6TdJSkdwBbbC9rcI2rgVcA04HNVE99PfdG9jW2O213jhs3roniRjMWLt/IiVfczeS5/8yJV9zNwuUbB7pIEdHPmnmqagtwZte+pDHAO4Bbejl1A3BEzf5EYFODPE/Y3g5sl3QfcCzV+MlpZeD7AOBgSf9o+322H68py7XAV4l+sXD5Ri5d8BA7nt4NwMZtO7h0wUMAzJlR35iMiOGqqRf5ykD3W4Czyvbf6D1wPAhMkTSZqrVyJtWYRq3bgb+VNIpqkagTqAa+bwEuLfc+GfhD2+8r++Ntby7nnw6saqYOse/mLVm7N2h02fH0bi65dSVfemD9AJWq7z28+Ummjc86ZRHd6TFwSHoD1Y/924EHgBOBl9v+WW8Xtr1L0oXAEqADuN72akkXlOPzba+RtBhYSTWWcZ3t3gLBJyRNp+r2ehQ4v7eyRN/YtG1Hw/SduxsNQw1d08YfzOzpaUFFdEc1D0w9+4C0AVhPNaaw0PZTkv7L9uT+LGBf6Ozs9NKlSwe6GEPeiVfczcYGwWPC2DF8c27W9ooYbiQts91Zn97T4Pg/UT0F9V7gnZJeSBZwGtEunjmVMaM7npU2ZnQHF8+cOkAlioiB0G3gsH0RMAn4NHAK8D1gnKT3SHpR/xQvBpM5Mybw8TOOYf+O6j+bCWPH8PEzjsnAeMQI0+MYR3nx727gbkmjgVlUA+RXAS9pf/FisJkzY8LegfAvn/+6AS5NRAyEpqdHt/008BXgK+WR3IiIGIGaeQHwOWw3frwmIiKGvecVOCIiYuRK4IiIiJb0OsYh6SjgYuBltflt58H9iIgRqJnB8VuA+cC1wO5e8kZExDDXTODYZfvqtpckIiKGhGbGOL4i6bckjZd0aNen7SWLiIhBqZkWx7lle3FNmoGX931xIiJisGtmPY4hN6lhRES0TzNPVY0G/i/whpJ0D/B35U3yiIgYYZrpqroaGE01PxXA+0vab7arUBERMXg1Ezh+2faxNft3S/pOuwoUERGDWzNPVe2W9IquHUkvp8n3OSTNkrRW0jpJc7vJc7KkFZJWS7q37liHpOWSvlqTdqikOyV9v2wPaaYsERHRN5oJHBcD35B0T/lhvxv4g95OKuuUXwm8FZgGnCVpWl2esVRdYKfZPhp4d91lLgLW1KXNBe6yPQW4q+xHREQ/6TVw2L4LmAL8bvlMtf2NJq59PLDO9iO2dwI3AbPr8pwNLLC9vtxrS9cBSROp1jq/ru6c2cAN5fsNwJwmyhIREX2k28Ah6dSyPYPqB/yVwCuAt5e03kwAHqvZ31DSah0FHFJaM8sknVNz7K+AS4A9dee81PZmgLI9rJvynydpqaSlW7dubaK4ERHRjJ4Gx3+VqlvqnQ2OGVjQy7XVzXn19z8OeCMwBrhf0reoAsoW28skndzLfRqyfQ1wDUBnZ2fWSo+I6CPdBg7bf1a+/rnt/6o9JqmZlwI3AEfU7E8ENjXI84Tt7cB2SfcBxwKvAU6T9DbgAOBgSf9o+33A45LG294saTywhYiI6DfNDI7/U4O0W5s470FgiqTJkvYHzgQW1eW5HThJ0ihJBwInAGtsX2p7ou1J5by7S9CgXKNrGpRzyzUiIqKfdNvikPSLwNHAi+vGNA6magX0yPYuSRcCS4AO4HrbqyVdUI7Pt71G0mJgJdVYxnW2V/Vy6SuAmyX9BrCe5z6JFRERbdTTGMdU4B3AWJ49zvEU8MFmLm77DuCOurT5dfvzgHk9XOMeqmlOuvZ/SDUmEhERA6CnMY7bgdslvc72/f1YpoiIGMSamXJkuaTfpuq22ttFZfvX21aqiIgYtJoZHP8C8L+AmcC9VE9HPdXOQkVExODVTOB4pe3LgO22b6B6GfCY9hYrIiIGq2YCR9e6G9skvQp4MTCpbSWKiIhBrZkxjmvKDLSXUb1D8SLgT9taqoiIGLSaWTq2a5LBe8k64xERI15PLwB+qKcTbX+674sTERGDXU8tjoPKdirwyzwzXcg7gfvaWaiIiBi8enoB8CMAkr4OvMb2U2X/cuCWfildREQMOs08VXUksLNmfyd5qioiYsRq5qmqLwAPSLqNaj2N04HPt7VUERExaDXzVNXHJH0NOKkkfcD28vYWKyIiBquenqo62PaTkg4FHi2frmOH2v5R+4sXERGDTU8tjhupplVfxrOXfFXZzzsdEREjULeD47bfUbaTbb+85jPZdlNBQ9IsSWslrZM0t5s8J0taIWm1pHtL2gGSHpD0nZL+kZr8l0vaWM5ZUZaXjYiIftJTV9VrejrR9n/0dFxSB3Al8GaqtcUflLTI9sM1ecYCVwGzbK+XdFg59HPgVNs/lTQa+DdJX7P9rXL8M7Y/2VvlIiKi7/XUVfWpHo4ZOLWXax8PrLP9CICkm4DZwMM1ec4GFtheD2B7S9ka+GnJM7p8arvLIiJigPT0AuAp+3jtCcBjNfsbgBPq8hwFjJZ0D9Wb6p+1/XnY22JZBrwSuNL2t2vOu1DSOcBS4A9s/3gfyxoREU1q5gVAJL1K0nskndP1aea0Bmn1rYZRwHFUa3zMBC6TdBSA7d22p1MtHHV8mdId4GrgFcB0YDPdtIwknSdpqaSlW7dubaK4ERHRjF4Dh6Q/A/6mfE4BPgGc1sS1NwBH1OxPBDY1yLPY9nbbT1DNgXVsbQbb24B7gFll//ESVPYA11J1iT2H7Wtsd9ruHDduXBPFjYiIZjTT4ngX8Ebgv21/gOqH/QVNnPcgMEXSZEn7A2fyzESJXW4HTpI0StKBVF1ZaySNKwPnSBoDvAn4btkfX3P+6cCqJsoSERF9pJkpR3bY3iNpl6SDgS008Q6H7V2SLgSWAB3A9bZXS7qgHJ9ve42kxcBKYA9wne1Vkl4N3FDGOfYDbrb91XLpT0iaTtXt9Shwfks1joiIfdJM4Fha/vq/lmqw+qfAA81c3PYdwB11afPr9ucB8+rSVgIzurnm+5u5d0REtEdP73H8LXCj7d8qSfNL6+Dg8sMeEREjUE8tju8DnypjCl8GvmR7Rf8UKyIiBquephz5rO3XAb8K/Aj4B0lrJP1p1yOzEREx8vT6VJXtH9j+S9szqN70Ph1Y0/aSRUTEoNTMexyjJb1T0heBrwHfA36t7SWLiIhBqafB8TcDZ1G91f0AcBNwnu3t/VS2iIgYhHoaHP8w1Zocf5hFmyIioks7JzmMiIhhqKlJDiMiIrokcEREREsSOCIioiUJHBER0ZIEjoiIaEkCR0REtCSBIyIiWpLAERERLWlr4JA0S9JaSeskze0mz8mSVkhaLeneknaApAckfaekf6Qm/6GS7pT0/bI9pJ11iIiIZ2tb4CjLvl4JvBWYBpwlaVpdnrHAVcBpto8G3l0O/Rw41faxwHRglqTXlmNzgbtsTwHuKvsREdFP2tniOB5YZ/sR2zupJkmcXZfnbGCB7fUAtreUrW3/tOQZXT4u+7OBG8r3G4A57atCRETUa2fgmAA8VrO/oaTVOgo4RNI9kpZJOqfrgKQOSSuALcCdtr9dDr3U9maAsj2s0c0lnSdpqaSlW7du7aMqRUREOwOHGqS5bn8UcBzV1O0zgcu6Vhe0vdv2dGAicLykV7Vyc9vX2O603Tlu3LjWSx8REQ21M3BsAI6o2Z8IbGqQZ7Ht7bafAO4Djq3NYHsbcA8wqyQ9XtZBp2y39H3RIyKiO+0MHA8CUyRNlrQ/cCawqC7P7cBJkkZJOhA4AVgjaVwZOEfSGOBNwHfLOYuAc8v3c8s1IiKin/S0kNM+sb1L0oXAEqADuN72akkXlOPzba+RtBhYCewBrrO9StKrgRvKk1n7ATfb/mq59BXAzZJ+A1jPM09iRUREP2hb4ACwfQdwR13a/Lr9ecC8urSVwIxurvlD4I19W9KIiGhW3hyPiIiWJHBERERLEjgiIqIlCRwREdGSBI6IiGhJAkdERLQkgSMiIlqSwBERES1J4IiIiJYkcEREREsSOCIioiUJHBER0ZIEjoiIaEkCR0REtCSBIyIiWpLAERERLWlr4JA0S9JaSeskze0mz8mSVkhaLeneknaEpG9IWlPSL6rJf7mkjeWcFZLe1s46DCULl2/kxCvuZvLcf+bEK+5m4fKNA12kiBiG2rYCYFn29UrgzcAG4EFJi2w/XJNnLHAVMMv2ekmHlUO7gD+w/R+SDgKWSbqz5tzP2P5ku8reTguXb2TekrVs2raDw8eO4eKZU5kzY0KfXPfSBQ+x4+ndAGzctoNLFzwE0CfXj4jo0s6lY48H1tl+BEDSTcBs4OGaPGcDC2yvB7C9pWw3A5vL96ckrQEm1J075LTzx33ekrV7r9tlx9O7ueTWlXzpgfX7dO16D29+kmnjD+7Ta0bE0NHOwDEBeKxmfwNwQl2eo4DRku4BDgI+a/vztRkkTaJaf/zbNckXSjoHWErVMvlx/c0lnQecB3DkkUfuSz36TDt/3Ddu29EwfefuPft03UamjT+Y2dPTiokYqdoZONQgzQ3ufxzwRmAMcL+kb9n+HoCkFwH/BPye7SfLOVcDHy3X+ijwKeDXn3Mj+xrgGoDOzs76+w6ITW38cd+/Y7+G15kwdgxfPv91+3z9iIgu7QwcG4AjavYnApsa5HnC9nZgu6T7gGOB70kaTRU0vmh7QdcJth/v+i7pWuCrbSp/nzt87JiGLYO++HGv7wYDGDO6g4tnTt2n60ZE1GvnU1UPAlMkTZa0P3AmsKguz+3ASZJGSTqQqitrjSQBfw+ssf3p2hMkja/ZPR1Y1bYa9LGLZ05lzOiOZ6X11Y/7nBkT+PgZxzBh7BhEFYw+fsYxGRiPiD7XthaH7V2SLgSWAB3A9bZXS7qgHJ9ve42kxcBKYA9wne1Vkl4PvB94SNKKcskP274D+ISk6VRdVY8C57erDn2t60f8kltXsnP3Hib04VNVXddPoIiIdpM9KLr/26qzs9NLly4d6GLs9d6/ux8gYw8RMahJWma7sz49b45HRERLEjgiIqIl7Xyqakhr1xveERFDXQJHA5m+IyKiewkcDbR7+o5M2RERQ1nGOBpo5xvekCk7ImJoS4ujgXa+4R0RMdSlxdFAO9/wjogY6tLiaKBrADxPVUVEPFcCRzcyfUdERGPpqoqIiJYkcEREREsSOCIioiUJHBER0ZIEjoiIaMmIWI9D0lbgBw0OvQR4op+L05+Gc/1St6FpONcNhl/9XmZ7XH3iiAgc3ZG0tNEiJcPFcK5f6jY0Dee6wfCvX5d0VUVEREsSOCIioiUjPXBcM9AFaLPhXL/UbWgaznWD4V8/YISPcUREROtGeosjIiJalMAREREtGbGBQ9IsSWslrZM0d6DLsy8kXS9pi6RVNWmHSrpT0vfL9pCBLOPzJekISd+QtEbSakkXlfThUr8DJD0g6Tulfh8p6cOlfh2Slkv6atkfFvUCkPSopIckrZC0tKQNm/r1ZPh2448AAAZuSURBVEQGDkkdwJXAW4FpwFmSpg1sqfbJ54BZdWlzgbtsTwHuKvtD0S7gD2z/EvBa4LfLv6vhUr+fA6faPhaYDsyS9FqGT/0uAtbU7A+XenU5xfb0mnc3hlv9GhqRgQM4Hlhn+xHbO4GbgNkDXKbnzfZ9wI/qkmcDN5TvNwBz+rVQfcT2Ztv/Ub4/RfUjNIHhUz/b/mnZHV0+ZhjUT9JE4O3AdTXJQ75evRju9QNGbuCYADxWs7+hpA0nL7W9GaofX+CwAS7PPpM0CZgBfJthVL/SnbMC2ALcaXu41O+vgEuAPTVpw6FeXQx8XdIySeeVtOFUv26N1BUA1SAtzyUPYpJeBPwT8Hu2n5Qa/SscmmzvBqZLGgvcJulVA12mfSXpHcAW28sknTzQ5WmTE21vknQYcKek7w50gfrLSG1xbACOqNmfCGwaoLK0y+OSxgOU7ZYBLs/zJmk0VdD4ou0FJXnY1K+L7W3APVTjVUO9ficCp0l6lKor+FRJ/8jQr9detjeV7RbgNqou8GFTv56M1MDxIDBF0mRJ+wNnAosGuEx9bRFwbvl+LnD7AJbleVPVtPh7YI3tT9ccGi71G1daGkgaA7wJ+C5DvH62L7U90fYkqv+/7rb9PoZ4vbpIeqGkg7q+A28BVjFM6tebEfvmuKS3UfXBdgDX2/7YABfpeZP0JeBkqimdHwf+DFgI3AwcCawH3m27fgB90JP0euBfgYd4pq/8w1TjHMOhfq+mGkTtoPpD7mbbfy7pFxgG9QMoXVV/aPsdw6Vekl5O1cqAqsv/RtsfGy71682IDRwREfH8jNSuqoiIeJ4SOCIioiUJHBER0ZIEjoiIaEkCR0REtCSBI4Y8SZ+R9Hs1+0skXVez/ylJH+rh/M9Jelf5fo+kzgZ5Rku6osx6uqrMaPvWcuxRSS95HuXee99ujl9ZZl59WNKO8n2FpHdJuqPr/Y++JGl810y23RzfX9J9kkbqrBNBAkcMD/8O/AqApP2o3mc5uub4rwDf3Md7fBQYD7zK9quAdwIH7eM1e2T7t21PB94G/GeZhXW67Vttv628ad7XPgRc20OZdlLN+vreNtw7hogEjhgOvkkJHFQBYxXwlKRDJL0A+CVguaQ/lfRgaTFcoyYnvJJ0IPBB4Hds/xzA9uO2b26Q90Pl+qvqWkHnSFpZ1t34QoPzPlpaIE39P9nVypE0SdJ3JV1X7vlFSW+S9M3SOjq+5H+hqnVbHlS1PkZ3s0H/GrC4nHN0aVmtKGWfUvIsBP53M+WM4SnNzRjyykRzuyQdSRVA7qea7fh1wE+AlbZ3Svpb238OUH683wF8pYlbvBJYb/vJnjJJOg74AHAC1USa35Z0L7AT+GOqSfGekHRo3XmfAF4MfMDP743cVwLvBs6jmk7nbOD1wGlUb9nPKfe/2/avly6uByT9i+3tNeWYDPy4KzgCFwCftf3FMjVPR0lfBfzy8yhnDBNpccRw0dXq6Aoc99fs/3vJc4qkb0t6CDiVZ3dn9YXXA7fZ3l7W2FgAnFTudavtJwDqpqC4DBhr+/znGTQA/sv2Q7b3AKupFhIy1TQtk0qetwBzVU3ffg9wANW0GLXGA1tr9u8HPizpj4CX2d5Ryr8b2Nk1V1OMPAkcMVx0jXMcQ/UX8beoWhy/AnxT0gHAVcC7bB9D1Y9/QJPXXgcc2cQPZXddX6L7afsfBI6rb4W06Oc13/fU7O/hmV4FAb9WM05ypO3alfkAdlDzz8T2jVStlh3AEkmn1uR9AfA/+1DmGMISOGK4+CZV19OPbO8uf9WPpQoe9/PMD+ITqtb26PZppnq2f0Y1Q+9fly6brqeP3leX9T5gjqQDy4ypp1NN0HgX8J4yAR51QWIxcAXwz23+C34J8Dtd4zqSZjTI8z2eaaF0TeT3iO2/ppr19dUl/ReArbafbmN5YxBL4Ijh4iGqp6m+VZf2E9tPlCeQri1pC6n+0m/Fn1B14zwsaVW5Rm23DmWJ288BD1DN3nud7eW2VwMfA+6V9B3g03Xn3VLKtqhMrd4OH6ValnZlKf9H6zOU8Y7/lPTKkvReYFXp3vpF4PMl/RTgjjaVM4aAzI4bEXtJOh04zvaf9JBnAXCp7bX9V7IYTPJUVUTsZfu2ri61RkpX3cIEjZEtLY6IiGhJxjgiIqIlCRwREdGSBI6IiGhJAkdERLQkgSMiIlry/wG4uIRVWEfmsAAAAABJRU5ErkJggg==\n" }, "metadata": { "needs_background": "light" @@ -536,7 +480,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 13, "metadata": { "slideshow": { "slide_type": "slide" @@ -641,7 +585,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 14, "metadata": { "slideshow": { "slide_type": "slide" @@ -655,7 +599,7 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 15, "metadata": { "slideshow": { "slide_type": "slide" @@ -667,115 +611,109 @@ "output_type": "stream", "name": "stderr", "text": [ - "[flaml.automl: 02-17 13:47:25] {840} INFO - Evaluation method: holdout\n", "INFO - Evaluation method: holdout\n", - "[flaml.automl: 02-17 13:47:26] {565} INFO - Using StratifiedKFold\n", "INFO - Using StratifiedKFold\n", - "[flaml.automl: 02-17 13:47:26] {861} INFO - Minimizing error metric: 1-accuracy\n", "INFO - Minimizing error metric: 1-accuracy\n", - "[flaml.automl: 02-17 13:47:26] {881} INFO - List of ML learners in AutoML Run: ['RGF', 'lgbm', 'rf', 'xgboost']\n", "INFO - List of ML learners in AutoML Run: ['RGF', 'lgbm', 'rf', 'xgboost']\n", - "[flaml.automl: 02-17 13:47:26] {940} INFO - iteration 0 current learner RGF\n", "INFO - iteration 0 current learner RGF\n", - "[flaml.automl: 02-17 13:47:28] {1094} INFO - at 1.4s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "INFO - at 1.4s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "[flaml.automl: 02-17 13:47:28] {940} INFO - iteration 1 current learner RGF\n", + "INFO - at 1.3s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", "INFO - iteration 1 current learner RGF\n", - "[flaml.automl: 02-17 13:47:29] {1094} INFO - at 4.0s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "INFO - at 4.0s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "[flaml.automl: 02-17 13:47:29] {940} INFO - iteration 2 current learner RGF\n", + "INFO - at 4.4s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", "INFO - iteration 2 current learner RGF\n", - "[flaml.automl: 02-17 13:47:30] {1094} INFO - at 4.9s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "INFO - at 4.9s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "[flaml.automl: 02-17 13:47:30] {940} INFO - iteration 3 current learner lgbm\n", + "INFO - at 5.3s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", "INFO - iteration 3 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:32] {1094} INFO - at 5.0s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", - "INFO - at 5.0s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", - "[flaml.automl: 02-17 13:47:32] {940} INFO - iteration 4 current learner RGF\n", + "INFO - at 5.3s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", "INFO - iteration 4 current learner RGF\n", - "[flaml.automl: 02-17 13:47:35] {1094} INFO - at 7.9s,\tbest RGF's error=0.3658,\tbest RGF's error=0.3658\n", - "INFO - at 7.9s,\tbest RGF's error=0.3658,\tbest RGF's error=0.3658\n", - "[flaml.automl: 02-17 13:47:35] {940} INFO - iteration 5 current learner lgbm\n", + "INFO - at 8.4s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3777\n", "INFO - iteration 5 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:35] {1094} INFO - at 9.8s,\tbest lgbm's error=0.3777,\tbest RGF's error=0.3658\n", - "INFO - at 9.8s,\tbest lgbm's error=0.3777,\tbest RGF's error=0.3658\n", - "[flaml.automl: 02-17 13:47:35] {940} INFO - iteration 6 current learner lgbm\n", + "INFO - at 8.4s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", "INFO - iteration 6 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:35] {1094} INFO - at 9.8s,\tbest lgbm's error=0.3669,\tbest RGF's error=0.3658\n", - "INFO - at 9.8s,\tbest lgbm's error=0.3669,\tbest RGF's error=0.3658\n", - "[flaml.automl: 02-17 13:47:35] {940} INFO - iteration 7 current learner lgbm\n", + "INFO - at 8.5s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", "INFO - iteration 7 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:35] {1094} INFO - at 9.9s,\tbest lgbm's error=0.3669,\tbest RGF's error=0.3658\n", - "INFO - at 9.9s,\tbest lgbm's error=0.3669,\tbest RGF's error=0.3658\n", - "[flaml.automl: 02-17 13:47:35] {940} INFO - iteration 8 current learner lgbm\n", + "INFO - at 10.4s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", "INFO - iteration 8 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:35] {1094} INFO - at 9.9s,\tbest lgbm's error=0.3662,\tbest RGF's error=0.3658\n", - "INFO - at 9.9s,\tbest lgbm's error=0.3662,\tbest RGF's error=0.3658\n", - "[flaml.automl: 02-17 13:47:35] {940} INFO - iteration 9 current learner lgbm\n", + "INFO - at 10.5s,\tbest lgbm's error=0.3662,\tbest lgbm's error=0.3662\n", "INFO - iteration 9 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:37] {1094} INFO - at 10.0s,\tbest lgbm's error=0.3636,\tbest lgbm's error=0.3636\n", - "INFO - at 10.0s,\tbest lgbm's error=0.3636,\tbest lgbm's error=0.3636\n", - "[flaml.automl: 02-17 13:47:37] {940} INFO - iteration 10 current learner lgbm\n", + "INFO - at 12.4s,\tbest lgbm's error=0.3636,\tbest lgbm's error=0.3636\n", "INFO - iteration 10 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:39] {1094} INFO - at 12.1s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 12.1s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:39] {940} INFO - iteration 11 current learner lgbm\n", - "INFO - iteration 11 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:39] {1094} INFO - at 14.0s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 14.0s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:39] {940} INFO - iteration 12 current learner lgbm\n", - "INFO - iteration 12 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:39] {1094} INFO - at 14.1s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 14.1s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:39] {940} INFO - iteration 13 current learner lgbm\n", - "INFO - iteration 13 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:39] {1094} INFO - at 14.2s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 14.2s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:39] {940} INFO - iteration 14 current learner lgbm\n", - "INFO - iteration 14 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:40] {1094} INFO - at 14.4s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", "INFO - at 14.4s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:40] {940} INFO - iteration 15 current learner lgbm\n", + "INFO - iteration 11 current learner lgbm\n", + "INFO - at 16.3s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "INFO - iteration 12 current learner lgbm\n", + "INFO - at 16.4s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "INFO - iteration 13 current learner lgbm\n", + "INFO - at 16.4s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "INFO - iteration 14 current learner lgbm\n", + "INFO - at 16.5s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", "INFO - iteration 15 current learner lgbm\n", - "[flaml.automl: 02-17 13:47:41] {1094} INFO - at 16.0s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 16.0s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:41] {940} INFO - iteration 16 current learner xgboost\n", - "INFO - iteration 16 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:41] {1094} INFO - at 16.1s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", - "INFO - at 16.1s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:41] {940} INFO - iteration 17 current learner RGF\n", - "INFO - iteration 17 current learner RGF\n", - "[flaml.automl: 02-17 13:47:42] {1094} INFO - at 17.1s,\tbest RGF's error=0.3658,\tbest lgbm's error=0.3621\n", - "INFO - at 17.1s,\tbest RGF's error=0.3658,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:42] {940} INFO - iteration 18 current learner xgboost\n", + "INFO - at 16.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", + "INFO - iteration 16 current learner RGF\n", + "INFO - at 17.5s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3621\n", + "INFO - iteration 17 current learner lgbm\n", + "INFO - at 18.8s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", "INFO - iteration 18 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:42] {1094} INFO - at 17.2s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", - "INFO - at 17.2s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:42] {940} INFO - iteration 19 current learner xgboost\n", + "INFO - at 18.9s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", "INFO - iteration 19 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:42] {1094} INFO - at 17.2s,\tbest xgboost's error=0.3757,\tbest lgbm's error=0.3621\n", - "INFO - at 17.2s,\tbest xgboost's error=0.3757,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:42] {940} INFO - iteration 20 current learner xgboost\n", + "INFO - at 19.0s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", "INFO - iteration 20 current learner xgboost\n", - "[flaml.automl: 02-17 13:47:42] {1094} INFO - at 17.3s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3621\n", - "INFO - at 17.3s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:42] {940} INFO - iteration 21 current learner RGF\n", + "INFO - at 19.1s,\tbest xgboost's error=0.3757,\tbest lgbm's error=0.3621\n", "INFO - iteration 21 current learner RGF\n", - "[flaml.automl: 02-17 13:47:44] {1094} INFO - at 18.4s,\tbest RGF's error=0.3658,\tbest lgbm's error=0.3621\n", - "INFO - at 18.4s,\tbest RGF's error=0.3658,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:47:44] {940} INFO - iteration 22 current learner RGF\n", - "INFO - iteration 22 current learner RGF\n", - "[flaml.automl: 02-17 13:48:30] {1094} INFO - at 65.2s,\tbest RGF's error=0.3658,\tbest lgbm's error=0.3621\n", - "INFO - at 65.2s,\tbest RGF's error=0.3658,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:48:30] {1135} INFO - selected model: LGBMClassifier(learning_rate=0.7179196339383696, max_bin=511,\n", - " min_child_weight=2.776007506782275, n_estimators=6, num_leaves=4,\n", - " objective='binary', reg_alpha=9.69511928836042e-10,\n", - " reg_lambda=0.17744769739709204, subsample=0.8746997476758036)\n", - "INFO - selected model: LGBMClassifier(learning_rate=0.7179196339383696, max_bin=511,\n", - " min_child_weight=2.776007506782275, n_estimators=6, num_leaves=4,\n", - " objective='binary', reg_alpha=9.69511928836042e-10,\n", - " reg_lambda=0.17744769739709204, subsample=0.8746997476758036)\n", - "[flaml.automl: 02-17 13:48:30] {895} INFO - fit succeeded\n", + "INFO - at 20.1s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3621\n", + "INFO - iteration 22 current learner lgbm\n", + "INFO - at 21.6s,\tbest lgbm's error=0.3611,\tbest lgbm's error=0.3611\n", + "INFO - iteration 23 current learner xgboost\n", + "INFO - at 23.5s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3611\n", + "INFO - iteration 24 current learner rf\n", + "INFO - at 24.0s,\tbest rf's error=0.4046,\tbest lgbm's error=0.3611\n", + "INFO - iteration 25 current learner xgboost\n", + "INFO - at 24.0s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3611\n", + "INFO - iteration 26 current learner xgboost\n", + "INFO - at 24.1s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3611\n", + "INFO - iteration 27 current learner xgboost\n", + "INFO - at 24.2s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3611\n", + "INFO - iteration 28 current learner xgboost\n", + "INFO - at 24.3s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3611\n", + "INFO - iteration 29 current learner xgboost\n", + "INFO - at 24.4s,\tbest xgboost's error=0.3620,\tbest lgbm's error=0.3611\n", + "INFO - iteration 30 current learner lgbm\n", + "INFO - at 25.4s,\tbest lgbm's error=0.3585,\tbest lgbm's error=0.3585\n", + "INFO - iteration 31 current learner lgbm\n", + "INFO - at 27.9s,\tbest lgbm's error=0.3585,\tbest lgbm's error=0.3585\n", + "INFO - iteration 32 current learner lgbm\n", + "INFO - at 29.8s,\tbest lgbm's error=0.3531,\tbest lgbm's error=0.3531\n", + "INFO - iteration 33 current learner xgboost\n", + "INFO - at 31.9s,\tbest xgboost's error=0.3598,\tbest lgbm's error=0.3531\n", + "INFO - iteration 34 current learner lgbm\n", + "INFO - at 33.1s,\tbest lgbm's error=0.3522,\tbest lgbm's error=0.3522\n", + "INFO - iteration 35 current learner rf\n", + "INFO - at 35.5s,\tbest rf's error=0.4046,\tbest lgbm's error=0.3522\n", + "INFO - iteration 36 current learner xgboost\n", + "INFO - at 35.6s,\tbest xgboost's error=0.3593,\tbest lgbm's error=0.3522\n", + "INFO - iteration 37 current learner rf\n", + "INFO - at 36.0s,\tbest rf's error=0.4046,\tbest lgbm's error=0.3522\n", + "INFO - iteration 38 current learner rf\n", + "INFO - at 36.5s,\tbest rf's error=0.4046,\tbest lgbm's error=0.3522\n", + "INFO - iteration 39 current learner lgbm\n", + "INFO - at 39.0s,\tbest lgbm's error=0.3359,\tbest lgbm's error=0.3359\n", + "INFO - iteration 40 current learner rf\n", + "INFO - at 41.8s,\tbest rf's error=0.3968,\tbest lgbm's error=0.3359\n", + "INFO - iteration 41 current learner lgbm\n", + "INFO - at 44.1s,\tbest lgbm's error=0.3359,\tbest lgbm's error=0.3359\n", + "INFO - iteration 42 current learner RGF\n", + "INFO - at 45.6s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3359\n", + "INFO - iteration 43 current learner lgbm\n", + "INFO - at 50.1s,\tbest lgbm's error=0.3359,\tbest lgbm's error=0.3359\n", + "INFO - iteration 44 current learner xgboost\n", + "INFO - at 50.3s,\tbest xgboost's error=0.3593,\tbest lgbm's error=0.3359\n", + "INFO - iteration 45 current learner lgbm\n", + "INFO - at 52.0s,\tbest lgbm's error=0.3359,\tbest lgbm's error=0.3359\n", + "INFO - iteration 46 current learner RGF\n", + "INFO - at 61.6s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3359\n", + "INFO - selected model: LGBMClassifier(colsample_bytree=0.9950430045845675,\n", + " learning_rate=0.23086971995751512, max_bin=1023,\n", + " min_child_weight=5.096453108920863, n_estimators=14,\n", + " num_leaves=453, objective='binary', reg_alpha=1e-10,\n", + " reg_lambda=1.0, subsample=0.9433718793194579)\n", "INFO - fit succeeded\n" ] } @@ -787,7 +725,7 @@ " \"estimator_list\": ['RGF', 'lgbm', 'rf', 'xgboost'], # list of ML learners\n", " \"task\": 'classification', # task type \n", " \"sample\": True, # whether to subsample training data\n", - " \"log_file_name\": 'airlines_experiment.log', # cache directory of flaml log files \n", + " \"log_file_name\": 'airlines_experiment_custom.log', # flaml log file\n", " \"log_training_metric\": True, # whether to log training metric\n", "}\n", "\n", diff --git a/notebook/flaml_demo.ipynb b/notebook/flaml_demo.ipynb deleted file mode 100644 index c330ba78c..000000000 --- a/notebook/flaml_demo.ipynb +++ /dev/null @@ -1,1191 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved. \n", - "\n", - "Licensed under the MIT License.\n", - "\n", - "# Demo of AutoML with FLAML Library\n", - "\n", - "\n", - "## 1. Introduction\n", - "\n", - "FLAML is a Python library (https://github.com/microsoft/FLAML) designed to automatically produce accurate machine learning models \n", - "with low computational cost. It is fast and cheap. The simple and lightweight design makes it easy \n", - "to use and extend, such as adding new learners. FLAML can \n", - "- serve as an economical AutoML engine,\n", - "- be used as a fast hyperparameter tuning tool, or \n", - "- be embedded in self-tuning software that requires low latency & resource in repetitive\n", - " tuning tasks.\n", - "\n", - "In this notebook, we use one real data example (binary classification) to showcase how to ues FLAML library.\n", - "\n", - "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` option:\n", - "```bash\n", - "pip install flaml[notebook]\n", - "```" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Collecting flaml[notebook]\n", - " Downloading FLAML-0.2.3-py3-none-any.whl (77 kB)\n", - "Requirement already satisfied: scipy>=1.4.1 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (1.4.1) WARNING: The script optuna.exe is installed in 'C:\\Users\\chiw\\Miniconda3\\envs\\flaml\\Scripts' which is not on PATH.\n", - " Consider adding this directory to PATH or, if you prefer to suppress this warning, use --no-warn-script-location.\n", - "\n", - "Processing c:\\users\\chiw\\appdata\\local\\pip\\cache\\wheels\\38\\61\\9e\\955ab1890f6cab231b1d756db63f36c711968a324296e0b649\\optuna-2.3.0-py3-none-any.whl\n", - "Requirement already satisfied: xgboost>=0.90 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (1.3.3)\n", - "Requirement already satisfied: catboost>=0.23 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (0.23.2)\n", - "Requirement already satisfied: NumPy>=1.16.2 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (1.18.4)\n", - "Requirement already satisfied: scikit-learn>=0.23.2 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (0.23.2)\n", - "Requirement already satisfied: lightgbm>=2.3.1 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (2.3.1)\n", - "Requirement already satisfied: matplotlib==3.2.0; extra == \"notebook\" in c:\\users\\chiw\\appdata\\roaming\\python\\python37\\site-packages (from flaml[notebook]) (3.2.0)\n", - "Requirement already satisfied: rgf-python; extra == \"notebook\" in c:\\users\\chiw\\appdata\\roaming\\python\\python37\\site-packages (from flaml[notebook]) (3.9.0)\n", - "Requirement already satisfied: openml==0.10.2; extra == \"notebook\" in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (0.10.2)\n", - "Requirement already satisfied: jupyter; extra == \"notebook\" in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from flaml[notebook]) (1.0.0)\n", - "Requirement already satisfied: packaging>=20.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (20.4)\n", - "Requirement already satisfied: alembic in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (1.4.1)\n", - "Requirement already satisfied: sqlalchemy>=1.1.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (1.3.20)\n", - "Requirement already satisfied: tqdm in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (4.56.1)\n", - "Requirement already satisfied: cliff in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (3.5.0)\n", - "Requirement already satisfied: joblib in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (0.14.1)\n", - "Requirement already satisfied: cmaes>=0.6.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (0.7.0)\n", - "Requirement already satisfied: colorlog in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from optuna==2.3.0->flaml[notebook]) (4.6.2)\n", - "Requirement already satisfied: graphviz in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from catboost>=0.23->flaml[notebook]) (0.14.1)\n", - "Requirement already satisfied: plotly in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from catboost>=0.23->flaml[notebook]) (4.9.0)\n", - "Requirement already satisfied: pandas>=0.24.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from catboost>=0.23->flaml[notebook]) (0.24.2)\n", - "Requirement already satisfied: six in c:\\users\\chiw\\appdata\\roaming\\python\\python37\\site-packages (from catboost>=0.23->flaml[notebook]) (1.14.0)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from scikit-learn>=0.23.2->flaml[notebook]) (2.0.0)\n", - "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from matplotlib==3.2.0; extra == \"notebook\"->flaml[notebook]) (2.4.7)\n", - "Requirement already satisfied: cycler>=0.10 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from matplotlib==3.2.0; extra == \"notebook\"->flaml[notebook]) (0.10.0)\n", - "Requirement already satisfied: kiwisolver>=1.0.1 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from matplotlib==3.2.0; extra == \"notebook\"->flaml[notebook]) (1.2.0)\n", - "Requirement already satisfied: python-dateutil>=2.1 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from matplotlib==3.2.0; extra == \"notebook\"->flaml[notebook]) (2.8.1)\n", - "Requirement already satisfied: requests in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from openml==0.10.2; extra == \"notebook\"->flaml[notebook]) (2.25.0)\n", - "Requirement already satisfied: xmltodict in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from openml==0.10.2; extra == \"notebook\"->flaml[notebook]) (0.12.0)\n", - "Requirement already satisfied: liac-arff>=2.4.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from openml==0.10.2; extra == \"notebook\"->flaml[notebook]) (2.4.0)\n", - "Requirement already satisfied: qtconsole in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from jupyter; extra == \"notebook\"->flaml[notebook]) (4.7.7)\n", - "Requirement already satisfied: notebook in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from jupyter; extra == \"notebook\"->flaml[notebook]) (6.1.3)\n", - "Requirement already satisfied: nbconvert in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from jupyter; extra == \"notebook\"->flaml[notebook]) (5.6.1)\n", - "Requirement already satisfied: ipykernel in c:\\users\\chiw\\appdata\\roaming\\python\\python37\\site-packages (from jupyter; extra == \"notebook\"->flaml[notebook]) (5.3.4)\n", - "Requirement already satisfied: jupyter-console in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from jupyter; extra == \"notebook\"->flaml[notebook]) (6.2.0)\n", - "Requirement already satisfied: ipywidgets in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from jupyter; extra == \"notebook\"->flaml[notebook]) (7.5.1)\n", - "Requirement already satisfied: python-editor>=0.3 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from alembic->optuna==2.3.0->flaml[notebook]) (1.0.4)\n", - "Requirement already satisfied: Mako in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from alembic->optuna==2.3.0->flaml[notebook]) (1.1.3)\n", - "Requirement already satisfied: cmd2!=0.8.3,>=0.8.0 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from cliff->optuna==2.3.0->flaml[notebook]) (1.4.0)\n", - "Requirement already satisfied: PrettyTable<0.8,>=0.7.2 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from cliff->optuna==2.3.0->flaml[notebook]) (0.7.2)\n", - "Requirement already satisfied: stevedore>=2.0.1 in c:\\users\\chiw\\miniconda3\\envs\\flaml\\lib\\site-packages (from cliff->optuna==2.3.0->flaml[notebook]) (3.2.2)" - ] - } - ], - "source": [ - "!pip install flaml[notebook];" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "## 2. Real Data Example\n", - "### Load data and preprocess\n", - "\n", - "Download [Airlines dataset](https://www.openml.org/d/1169) from OpenML. The task is to predict whether a given flight will be delayed, given the information of the scheduled departure." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "slideshow": { - "slide_type": "subslide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "load dataset from ./openml_ds1169.pkl\nDataset name: airlines\nX_train.shape: (404537, 7), y_train.shape: (404537,);\nX_test.shape: (134846, 7), y_test.shape: (134846,)\n" - ] - } - ], - "source": [ - "from flaml.data import load_openml_dataset\n", - "X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id = 1169, data_dir = './')" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "### Run FLAML\n", - "In the FLAML automl run configuration, users can specify the task type, time budget, error metric, learner list, whether to subsample, resampling strategy type, and so on. All these arguments have default values which will be used if users do not provide them. For example, the default ML learners of FLAML are `['lgbm', 'xgboost', 'catboost', 'rf', 'extra_tree', 'lrl1']`. " - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "''' import AutoML class from flaml package '''\n", - "from flaml import AutoML\n", - "automl = AutoML()" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "settings = {\n", - " \"time_budget\": 300, # total running time in seconds\n", - " \"metric\": 'accuracy', # primary metrics can be chosen from: ['accuracy','roc_auc','f1','log_loss','mae','mse','r2']\n", - " # \"estimator_list\": ['lgbm', 'rf', 'xgboost'], # list of ML learners\n", - " \"task\": 'classification', # task type \n", - " # \"sample\": False, # whether to subsample training data\n", - " \"log_file_name\": 'airlines_experiment.log', # cache directory of flaml log files \n", - "}" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "slideshow": { - "slide_type": "slide" - }, - "tags": [ - "outputPrepend" - ] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "error=0.3600,\tbest catboost's error=0.3600\n", - "[flaml.automl: 02-17 13:53:08] {939} INFO - iteration 22 current learner catboost\n", - "INFO - iteration 22 current learner catboost\n", - "[flaml.automl: 02-17 13:53:10] {1093} INFO - at 11.5s,\tbest catboost's error=0.3600,\tbest catboost's error=0.3600\n", - "INFO - at 11.5s,\tbest catboost's error=0.3600,\tbest catboost's error=0.3600\n", - "[flaml.automl: 02-17 13:53:10] {939} INFO - iteration 23 current learner rf\n", - "INFO - iteration 23 current learner rf\n", - "[flaml.automl: 02-17 13:53:10] {1093} INFO - at 12.0s,\tbest rf's error=0.4000,\tbest catboost's error=0.3600\n", - "INFO - at 12.0s,\tbest rf's error=0.4000,\tbest catboost's error=0.3600\n", - "[flaml.automl: 02-17 13:53:10] {939} INFO - iteration 24 current learner catboost\n", - "INFO - iteration 24 current learner catboost\n", - "[flaml.automl: 02-17 13:53:11] {1093} INFO - at 12.7s,\tbest catboost's error=0.3599,\tbest catboost's error=0.3599\n", - "INFO - at 12.7s,\tbest catboost's error=0.3599,\tbest catboost's error=0.3599\n", - "[flaml.automl: 02-17 13:53:11] {939} INFO - iteration 25 current learner xgboost\n", - "INFO - iteration 25 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:11] {1093} INFO - at 12.9s,\tbest xgboost's error=0.3787,\tbest catboost's error=0.3599\n", - "INFO - at 12.9s,\tbest xgboost's error=0.3787,\tbest catboost's error=0.3599\n", - "[flaml.automl: 02-17 13:53:11] {939} INFO - iteration 26 current learner extra_tree\n", - "INFO - iteration 26 current learner extra_tree\n", - "[flaml.automl: 02-17 13:53:12] {1093} INFO - at 13.4s,\tbest extra_tree's error=0.3967,\tbest catboost's error=0.3599\n", - "INFO - at 13.4s,\tbest extra_tree's error=0.3967,\tbest catboost's error=0.3599\n", - "[flaml.automl: 02-17 13:53:12] {939} INFO - iteration 27 current learner catboost\n", - "INFO - iteration 27 current learner catboost\n", - "[flaml.automl: 02-17 13:53:13] {1093} INFO - at 14.2s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", - "INFO - at 14.2s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", - "[flaml.automl: 02-17 13:53:13] {939} INFO - iteration 28 current learner xgboost\n", - "INFO - iteration 28 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:13] {1093} INFO - at 14.4s,\tbest xgboost's error=0.3757,\tbest catboost's error=0.3598\n", - "INFO - at 14.4s,\tbest xgboost's error=0.3757,\tbest catboost's error=0.3598\n", - "[flaml.automl: 02-17 13:53:13] {939} INFO - iteration 29 current learner xgboost\n", - "INFO - iteration 29 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:13] {1093} INFO - at 14.4s,\tbest xgboost's error=0.3756,\tbest catboost's error=0.3598\n", - "INFO - at 14.4s,\tbest xgboost's error=0.3756,\tbest catboost's error=0.3598\n", - "[flaml.automl: 02-17 13:53:13] {939} INFO - iteration 30 current learner catboost\n", - "INFO - iteration 30 current learner catboost\n", - "[flaml.automl: 02-17 13:53:13] {1093} INFO - at 15.1s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", - "INFO - at 15.1s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", - "[flaml.automl: 02-17 13:53:13] {939} INFO - iteration 31 current learner lgbm\n", - "INFO - iteration 31 current learner lgbm\n", - "[flaml.automl: 02-17 13:53:14] {1093} INFO - at 16.0s,\tbest lgbm's error=0.3618,\tbest catboost's error=0.3598\n", - "INFO - at 16.0s,\tbest lgbm's error=0.3618,\tbest catboost's error=0.3598\n", - "[flaml.automl: 02-17 13:53:14] {939} INFO - iteration 32 current learner catboost\n", - "INFO - iteration 32 current learner catboost\n", - "[flaml.automl: 02-17 13:53:15] {1093} INFO - at 17.2s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", - "INFO - at 17.2s,\tbest catboost's error=0.3598,\tbest catboost's error=0.3598\n", - "[flaml.automl: 02-17 13:53:15] {939} INFO - iteration 33 current learner catboost\n", - "INFO - iteration 33 current learner catboost\n", - "[flaml.automl: 02-17 13:53:17] {1093} INFO - at 19.0s,\tbest catboost's error=0.3592,\tbest catboost's error=0.3592\n", - "INFO - at 19.0s,\tbest catboost's error=0.3592,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:17] {939} INFO - iteration 34 current learner xgboost\n", - "INFO - iteration 34 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:17] {1093} INFO - at 19.2s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "INFO - at 19.2s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:17] {939} INFO - iteration 35 current learner xgboost\n", - "INFO - iteration 35 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:18] {1093} INFO - at 19.4s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "INFO - at 19.4s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:18] {939} INFO - iteration 36 current learner xgboost\n", - "INFO - iteration 36 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:18] {1093} INFO - at 19.5s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "INFO - at 19.5s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:18] {939} INFO - iteration 37 current learner xgboost\n", - "INFO - iteration 37 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:18] {1093} INFO - at 19.7s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "INFO - at 19.7s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:18] {939} INFO - iteration 38 current learner xgboost\n", - "INFO - iteration 38 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:18] {1093} INFO - at 19.9s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "INFO - at 19.9s,\tbest xgboost's error=0.3620,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:18] {939} INFO - iteration 39 current learner xgboost\n", - "INFO - iteration 39 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:18] {1093} INFO - at 20.2s,\tbest xgboost's error=0.3598,\tbest catboost's error=0.3592\n", - "INFO - at 20.2s,\tbest xgboost's error=0.3598,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:18] {939} INFO - iteration 40 current learner xgboost\n", - "INFO - iteration 40 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:19] {1093} INFO - at 20.4s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 20.4s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:19] {939} INFO - iteration 41 current learner xgboost\n", - "INFO - iteration 41 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:19] {1093} INFO - at 20.6s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 20.6s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:19] {939} INFO - iteration 42 current learner xgboost\n", - "INFO - iteration 42 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:19] {1093} INFO - at 21.0s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 21.0s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:19] {939} INFO - iteration 43 current learner catboost\n", - "INFO - iteration 43 current learner catboost\n", - "[flaml.automl: 02-17 13:53:20] {1093} INFO - at 22.1s,\tbest catboost's error=0.3592,\tbest catboost's error=0.3592\n", - "INFO - at 22.1s,\tbest catboost's error=0.3592,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:20] {939} INFO - iteration 44 current learner xgboost\n", - "INFO - iteration 44 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:21] {1093} INFO - at 22.3s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 22.3s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:21] {939} INFO - iteration 45 current learner extra_tree\n", - "INFO - iteration 45 current learner extra_tree\n", - "[flaml.automl: 02-17 13:53:21] {1093} INFO - at 22.8s,\tbest extra_tree's error=0.3915,\tbest catboost's error=0.3592\n", - "INFO - at 22.8s,\tbest extra_tree's error=0.3915,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:21] {939} INFO - iteration 46 current learner xgboost\n", - "INFO - iteration 46 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:21] {1093} INFO - at 23.1s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 23.1s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:21] {939} INFO - iteration 47 current learner xgboost\n", - "INFO - iteration 47 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:23] {1093} INFO - at 24.3s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 24.3s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:23] {939} INFO - iteration 48 current learner xgboost\n", - "INFO - iteration 48 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:24] {1093} INFO - at 25.6s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 25.6s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:24] {939} INFO - iteration 49 current learner catboost\n", - "INFO - iteration 49 current learner catboost\n", - "[flaml.automl: 02-17 13:53:25] {1093} INFO - at 26.8s,\tbest catboost's error=0.3592,\tbest catboost's error=0.3592\n", - "INFO - at 26.8s,\tbest catboost's error=0.3592,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:25] {939} INFO - iteration 50 current learner xgboost\n", - "INFO - iteration 50 current learner xgboost\n", - "[flaml.automl: 02-17 13:53:26] {1093} INFO - at 27.6s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "INFO - at 27.6s,\tbest xgboost's error=0.3593,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:26] {939} INFO - iteration 51 current learner extra_tree\n", - "INFO - iteration 51 current learner extra_tree\n", - "[flaml.automl: 02-17 13:53:26] {1093} INFO - at 28.2s,\tbest extra_tree's error=0.3910,\tbest catboost's error=0.3592\n", - "INFO - at 28.2s,\tbest extra_tree's error=0.3910,\tbest catboost's error=0.3592\n", - "[flaml.automl: 02-17 13:53:26] {939} INFO - iteration 52 current learner catboost\n", - "INFO - iteration 52 current learner catboost\n", - "[flaml.automl: 02-17 13:53:32] {1093} INFO - at 34.1s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "INFO - at 34.1s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:32] {939} INFO - iteration 53 current learner catboost\n", - "INFO - iteration 53 current learner catboost\n", - "[flaml.automl: 02-17 13:53:34] {1093} INFO - at 36.0s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "INFO - at 36.0s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:34] {939} INFO - iteration 54 current learner catboost\n", - "INFO - iteration 54 current learner catboost\n", - "[flaml.automl: 02-17 13:53:42] {1093} INFO - at 43.7s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "INFO - at 43.7s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:42] {939} INFO - iteration 55 current learner lrl1\n", - "INFO - iteration 55 current learner lrl1\n", - "[flaml.automl: 02-17 13:53:42] {1093} INFO - at 44.1s,\tbest lrl1's error=0.4338,\tbest catboost's error=0.3553\n", - "INFO - at 44.1s,\tbest lrl1's error=0.4338,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:42] {939} INFO - iteration 56 current learner catboost\n", - "INFO - iteration 56 current learner catboost\n", - "[flaml.automl: 02-17 13:53:47] {1093} INFO - at 48.3s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "INFO - at 48.3s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:47] {939} INFO - iteration 57 current learner lrl1\n", - "INFO - iteration 57 current learner lrl1\n", - "[flaml.automl: 02-17 13:53:47] {1093} INFO - at 48.7s,\tbest lrl1's error=0.4338,\tbest catboost's error=0.3553\n", - "INFO - at 48.7s,\tbest lrl1's error=0.4338,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:47] {939} INFO - iteration 58 current learner lrl1\n", - "INFO - iteration 58 current learner lrl1\n", - "[flaml.automl: 02-17 13:53:47] {1093} INFO - at 49.0s,\tbest lrl1's error=0.4338,\tbest catboost's error=0.3553\n", - "INFO - at 49.0s,\tbest lrl1's error=0.4338,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:47] {939} INFO - iteration 59 current learner catboost\n", - "INFO - iteration 59 current learner catboost\n", - "[flaml.automl: 02-17 13:53:54] {1093} INFO - at 55.4s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "INFO - at 55.4s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:53:54] {939} INFO - iteration 60 current learner catboost\n", - "INFO - iteration 60 current learner catboost\n", - "[flaml.automl: 02-17 13:54:00] {1093} INFO - at 61.8s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "INFO - at 61.8s,\tbest catboost's error=0.3553,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:54:00] {939} INFO - iteration 61 current learner lgbm\n", - "INFO - iteration 61 current learner lgbm\n", - "[flaml.automl: 02-17 13:54:01] {1093} INFO - at 62.6s,\tbest lgbm's error=0.3618,\tbest catboost's error=0.3553\n", - "INFO - at 62.6s,\tbest lgbm's error=0.3618,\tbest catboost's error=0.3553\n", - "[flaml.automl: 02-17 13:54:01] {939} INFO - iteration 62 current learner catboost\n", - "INFO - iteration 62 current learner catboost\n", - "[flaml.automl: 02-17 13:54:40] {1093} INFO - at 101.8s,\tbest catboost's error=0.3476,\tbest catboost's error=0.3476\n", - "INFO - at 101.8s,\tbest catboost's error=0.3476,\tbest catboost's error=0.3476\n", - "[flaml.automl: 02-17 13:54:40] {939} INFO - iteration 63 current learner catboost\n", - "INFO - iteration 63 current learner catboost\n", - "[flaml.automl: 02-17 13:54:48] {1093} INFO - at 109.9s,\tbest catboost's error=0.3476,\tbest catboost's error=0.3476\n", - "INFO - at 109.9s,\tbest catboost's error=0.3476,\tbest catboost's error=0.3476\n", - "[flaml.automl: 02-17 13:54:48] {939} INFO - iteration 64 current learner xgboost\n", - "INFO - iteration 64 current learner xgboost\n", - "[flaml.automl: 02-17 13:54:50] {1093} INFO - at 112.0s,\tbest xgboost's error=0.3424,\tbest xgboost's error=0.3424\n", - "INFO - at 112.0s,\tbest xgboost's error=0.3424,\tbest xgboost's error=0.3424\n", - "[flaml.automl: 02-17 13:54:50] {939} INFO - iteration 65 current learner xgboost\n", - "INFO - iteration 65 current learner xgboost\n", - "[flaml.automl: 02-17 13:54:56] {1093} INFO - at 117.6s,\tbest xgboost's error=0.3424,\tbest xgboost's error=0.3424\n", - "INFO - at 117.6s,\tbest xgboost's error=0.3424,\tbest xgboost's error=0.3424\n", - "[flaml.automl: 02-17 13:54:56] {939} INFO - iteration 66 current learner xgboost\n", - "INFO - iteration 66 current learner xgboost\n", - "[flaml.automl: 02-17 13:55:03] {1093} INFO - at 125.1s,\tbest xgboost's error=0.3400,\tbest xgboost's error=0.3400\n", - "INFO - at 125.1s,\tbest xgboost's error=0.3400,\tbest xgboost's error=0.3400\n", - "[flaml.automl: 02-17 13:55:03] {939} INFO - iteration 67 current learner xgboost\n", - "INFO - iteration 67 current learner xgboost\n", - "[flaml.automl: 02-17 13:55:06] {1093} INFO - at 127.4s,\tbest xgboost's error=0.3400,\tbest xgboost's error=0.3400\n", - "INFO - at 127.4s,\tbest xgboost's error=0.3400,\tbest xgboost's error=0.3400\n", - "[flaml.automl: 02-17 13:55:06] {939} INFO - iteration 68 current learner xgboost\n", - "INFO - iteration 68 current learner xgboost\n", - "[flaml.automl: 02-17 13:55:20] {1093} INFO - at 141.8s,\tbest xgboost's error=0.3366,\tbest xgboost's error=0.3366\n", - "INFO - at 141.8s,\tbest xgboost's error=0.3366,\tbest xgboost's error=0.3366\n", - "[flaml.automl: 02-17 13:55:20] {939} INFO - iteration 69 current learner xgboost\n", - "INFO - iteration 69 current learner xgboost\n", - "[flaml.automl: 02-17 13:55:25] {1093} INFO - at 147.0s,\tbest xgboost's error=0.3366,\tbest xgboost's error=0.3366\n", - "INFO - at 147.0s,\tbest xgboost's error=0.3366,\tbest xgboost's error=0.3366\n", - "[flaml.automl: 02-17 13:55:25] {939} INFO - iteration 70 current learner catboost\n", - "INFO - iteration 70 current learner catboost\n", - "[flaml.automl: 02-17 13:56:11] {1093} INFO - at 192.7s,\tbest catboost's error=0.3476,\tbest xgboost's error=0.3366\n", - "INFO - at 192.7s,\tbest catboost's error=0.3476,\tbest xgboost's error=0.3366\n", - "[flaml.automl: 02-17 13:56:11] {939} INFO - iteration 71 current learner xgboost\n", - "INFO - iteration 71 current learner xgboost\n", - "[flaml.automl: 02-17 13:56:29] {1093} INFO - at 210.7s,\tbest xgboost's error=0.3317,\tbest xgboost's error=0.3317\n", - "INFO - at 210.7s,\tbest xgboost's error=0.3317,\tbest xgboost's error=0.3317\n", - "[flaml.automl: 02-17 13:56:29] {939} INFO - iteration 72 current learner xgboost\n", - "INFO - iteration 72 current learner xgboost\n", - "[flaml.automl: 02-17 13:56:59] {1093} INFO - at 240.5s,\tbest xgboost's error=0.3268,\tbest xgboost's error=0.3268\n", - "INFO - at 240.5s,\tbest xgboost's error=0.3268,\tbest xgboost's error=0.3268\n", - "[flaml.automl: 02-17 13:56:59] {939} INFO - iteration 73 current learner xgboost\n", - "INFO - iteration 73 current learner xgboost\n", - "[flaml.automl: 02-17 13:57:14] {1093} INFO - at 255.9s,\tbest xgboost's error=0.3268,\tbest xgboost's error=0.3268\n", - "INFO - at 255.9s,\tbest xgboost's error=0.3268,\tbest xgboost's error=0.3268\n", - "[flaml.automl: 02-17 13:57:32] {1109} INFO - retrain xgboost for 18.0s\n", - "INFO - retrain xgboost for 18.0s\n", - "[flaml.automl: 02-17 13:57:32] {939} INFO - iteration 74 current learner extra_tree\n", - "INFO - iteration 74 current learner extra_tree\n", - "[flaml.automl: 02-17 13:57:32] {1093} INFO - at 274.2s,\tbest extra_tree's error=0.3910,\tbest xgboost's error=0.3268\n", - "INFO - at 274.2s,\tbest extra_tree's error=0.3910,\tbest xgboost's error=0.3268\n", - "[flaml.automl: 02-17 13:57:46] {1109} INFO - retrain extra_tree for 13.2s\n", - "INFO - retrain extra_tree for 13.2s\n", - "[flaml.automl: 02-17 13:57:46] {939} INFO - iteration 75 current learner extra_tree\n", - "INFO - iteration 75 current learner extra_tree\n", - "[flaml.automl: 02-17 13:57:46] {1093} INFO - at 287.8s,\tbest extra_tree's error=0.3910,\tbest xgboost's error=0.3268\n", - "INFO - at 287.8s,\tbest extra_tree's error=0.3910,\tbest xgboost's error=0.3268\n", - "[flaml.automl: 02-17 13:57:52] {1109} INFO - retrain extra_tree for 5.9s\n", - "INFO - retrain extra_tree for 5.9s\n", - "[flaml.automl: 02-17 13:57:52] {939} INFO - iteration 76 current learner extra_tree\n", - "INFO - iteration 76 current learner extra_tree\n", - "[flaml.automl: 02-17 13:57:52] {1093} INFO - at 293.9s,\tbest extra_tree's error=0.3910,\tbest xgboost's error=0.3268\n", - "INFO - at 293.9s,\tbest extra_tree's error=0.3910,\tbest xgboost's error=0.3268\n", - "[flaml.automl: 02-17 13:57:56] {1109} INFO - retrain extra_tree for 3.8s\n", - "INFO - retrain extra_tree for 3.8s\n", - "[flaml.automl: 02-17 13:57:56] {939} INFO - iteration 77 current learner lgbm\n", - "INFO - iteration 77 current learner lgbm\n", - "[flaml.automl: 02-17 13:57:57] {1093} INFO - at 299.0s,\tbest lgbm's error=0.3563,\tbest xgboost's error=0.3268\n", - "INFO - at 299.0s,\tbest lgbm's error=0.3563,\tbest xgboost's error=0.3268\n", - "[flaml.automl: 02-17 13:57:58] {1109} INFO - retrain lgbm for 0.9s\n", - "INFO - retrain lgbm for 0.9s\n", - "[flaml.automl: 02-17 13:57:58] {1133} INFO - selected model: XGBClassifier(base_score=0.5, booster='gbtree',\n", - " colsample_bylevel=0.8909660754557278, colsample_bynode=1,\n", - " colsample_bytree=0.9330310727361396, gamma=0, gpu_id=-1,\n", - " grow_policy='lossguide', importance_type='gain',\n", - " interaction_constraints='', learning_rate=0.16464534671449255,\n", - " max_delta_step=0, max_depth=0, max_leaves=28,\n", - " min_child_weight=20.0, missing=nan, monotone_constraints='()',\n", - " n_estimators=1221, n_jobs=-1, num_parallel_tree=1, random_state=0,\n", - " reg_alpha=1e-10, reg_lambda=0.003747467958239166,\n", - " scale_pos_weight=1, subsample=1.0, tree_method='hist',\n", - " validate_parameters=1, verbosity=0)\n", - "INFO - selected model: XGBClassifier(base_score=0.5, booster='gbtree',\n", - " colsample_bylevel=0.8909660754557278, colsample_bynode=1,\n", - " colsample_bytree=0.9330310727361396, gamma=0, gpu_id=-1,\n", - " grow_policy='lossguide', importance_type='gain',\n", - " interaction_constraints='', learning_rate=0.16464534671449255,\n", - " max_delta_step=0, max_depth=0, max_leaves=28,\n", - " min_child_weight=20.0, missing=nan, monotone_constraints='()',\n", - " n_estimators=1221, n_jobs=-1, num_parallel_tree=1, random_state=0,\n", - " reg_alpha=1e-10, reg_lambda=0.003747467958239166,\n", - " scale_pos_weight=1, subsample=1.0, tree_method='hist',\n", - " validate_parameters=1, verbosity=0)\n", - "[flaml.automl: 02-17 13:57:58] {894} INFO - fit succeeded\n", - "INFO - fit succeeded\n" - ] - } - ], - "source": [ - "'''The main flaml automl API'''\n", - "automl.fit(X_train = X_train, y_train = y_train, **settings)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "### Best model and metric" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "slideshow": { - "slide_type": "slide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Best ML leaner: xgboost\nBest hyperparmeter config: {'n_estimators': 1389.0, 'max_leaves': 28.0, 'min_child_weight': 20.0, 'learning_rate': 0.16464534671449255, 'subsample': 1.0, 'colsample_bylevel': 0.8909660754557278, 'colsample_bytree': 0.9330310727361396, 'reg_alpha': 1e-10, 'reg_lambda': 0.003747467958239166, 'FLAML_sample_size': 364083}\nBest accuracy on validation data: 0.6732\nTraining duration of best run: 29.74 s\n" - ] - } - ], - "source": [ - "''' retrieve best config and best learner'''\n", - "print('Best ML leaner:', automl.best_estimator)\n", - "print('Best hyperparmeter config:', automl.best_config)\n", - "print('Best accuracy on validation data: {0:.4g}'.format(1-automl.best_loss))\n", - "print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "XGBClassifier(base_score=0.5, booster='gbtree',\n", - " colsample_bylevel=0.8909660754557278, colsample_bynode=1,\n", - " colsample_bytree=0.9330310727361396, gamma=0, gpu_id=-1,\n", - " grow_policy='lossguide', importance_type='gain',\n", - " interaction_constraints='', learning_rate=0.16464534671449255,\n", - " max_delta_step=0, max_depth=0, max_leaves=28,\n", - " min_child_weight=20.0, missing=nan, monotone_constraints='()',\n", - " n_estimators=1221, n_jobs=-1, num_parallel_tree=1, random_state=0,\n", - " reg_alpha=1e-10, reg_lambda=0.003747467958239166,\n", - " scale_pos_weight=1, subsample=1.0, tree_method='hist',\n", - " validate_parameters=1, verbosity=0)" - ] - }, - "metadata": {}, - "execution_count": 7 - } - ], - "source": [ - "automl.model" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "''' pickle and save the best model '''\n", - "import pickle\n", - "with open('best_model.pkl', 'wb') as f:\n", - " pickle.dump(automl.model, f, pickle.HIGHEST_PROTOCOL)" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "slideshow": { - "slide_type": "slide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "Predicted labels [1 0 1 ... 1 0 0]\nTrue labels [0 0 0 ... 0 1 0]\n" - ] - } - ], - "source": [ - "''' compute predictions of testing dataset ''' \n", - "y_pred = automl.predict(X_test)\n", - "print('Predicted labels', y_pred)\n", - "print('True labels', y_test)\n", - "y_pred_proba = automl.predict_proba(X_test)[:,1]" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": { - "slideshow": { - "slide_type": "slide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "accuracy = 0.6721222728149148\nroc_auc = 0.7252473500166565\nlog_loss = 0.6035663268278709\nf1 = 0.5905710872605036\n" - ] - } - ], - "source": [ - "''' compute different metric values on testing dataset'''\n", - "from flaml.ml import sklearn_metric_loss_score\n", - "print('accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))\n", - "print('roc_auc', '=', 1 - sklearn_metric_loss_score('roc_auc', y_pred_proba, y_test))\n", - "print('log_loss', '=', sklearn_metric_loss_score('log_loss', y_pred_proba, y_test))\n", - "print('f1', '=', 1 - sklearn_metric_loss_score('f1', y_pred, y_test))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "See Section 4 for an accuracy comparison with default LightGBM and XGBoost.\n", - "\n", - "### Log history" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": { - "slideshow": { - "slide_type": "subslide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 23.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 17.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'lgbm', 'Current Sample': 10000, 'Current Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 4.0, 'min_child_weight': 2.776007506782275, 'learning_rate': 0.7179196339383696, 'subsample': 0.8746997476758036, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 9.69511928836042e-10, 'reg_lambda': 0.17744769739709204, 'FLAML_sample_size': 10000}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 6.0, 'max_leaves': 4.0, 'min_child_weight': 2.776007506782275, 'learning_rate': 0.7179196339383696, 'subsample': 0.8746997476758036, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 9.69511928836042e-10, 'reg_lambda': 0.17744769739709204, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 10, 'learning_rate': 0.1, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 10, 'learning_rate': 0.1, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.2, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.2, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.09293377774381106, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.09293377774381106, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 10000, 'Current Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.042438101461189835, 'FLAML_sample_size': 10000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.042438101461189835, 'FLAML_sample_size': 10000}}\n{'Current Learner': 'catboost', 'Current Sample': 40000, 'Current Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.042438101461189835, 'FLAML_sample_size': 40000}, 'Best Learner': 'catboost', 'Best Hyper-parameters': {'early_stopping_rounds': 11.0, 'learning_rate': 0.042438101461189835, 'FLAML_sample_size': 40000}}\n" - ] - } - ], - "source": [ - "from flaml.data import get_output_from_log\n", - "time_history, best_valid_loss_history, valid_loss_history, config_history, train_loss_history = \\\n", - " get_output_from_log(filename = settings['log_file_name'], time_budget = 60)\n", - "\n", - "for config in config_history:\n", - " print(config)" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [ - { - "output_type": "display_data", - "data": { - "text/plain": "
", - "image/svg+xml": "\r\n\r\n\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n\r\n", - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAgAElEQVR4nO3de5xdVX338c83k4EMQkiQYENCDCqJopVEUqjiBag2EdQE7/B4Ka0CbWmptMFgiwV96Cs14u0RyQOUCq0oF2OIGokUTCgI5GJCLsTYSDFkkpJEiEAcCSS//rHXSU4OZ87sncyeOTPzfb9e85qz1157n9/ZhPObtdfaaykiMDMzy2tQbwdgZmZ9ixOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGHWzSS9RdK63o7DrCxOHNavSHpM0tt7M4aI+M+IGF/W+SVNlnSvpGckbZW0SNJ7yno/s1pOHGYFSWrpxfd+P3AbcBMwGngZ8Fng3ftxLknyd4AV5n80NiBIGiRphqRfSvq1pFslHVG1/zZJ/yPpN+mv+ddW7fumpGskzZe0AzgttWz+TtLKdMwtkoak+qdK2lh1fKd10/5LJG2WtEnSJySFpFfV+QwCvgR8PiKuj4jfRMTuiFgUEZ9MdS6X9O9Vx4xN5xucthdKulLS/cBvgc9IWlrzPp+SNC+9PljSFyVtkPSEpNmS2g7wP4f1cU4cNlD8NTANeBtwNPAUcHXV/h8BxwFHAT8DvlVz/DnAlcBhwH2p7IPAFOBY4PXAnzR4/7p1JU0BLgbeDrwqxdeZ8cAxwO0N6uTxUeA8ss/y/4Dxko6r2n8OcHN6/c/AOGBCim8UWQvHBjAnDhsozgf+PiI2RsRzwOXA+yt/iUfEDRHxTNW+EyQdXnX8HRFxf/oL/3ep7GsRsSkingS+T/bl2pnO6n4Q+NeIWBMRvwWuaHCOl6bfm3N/6vq+md7vhYj4DXAHcDZASiCvBualFs4ngU9FxJMR8QzwT8CHD/D9rY9z4rCB4uXA9yRtl7QdWAvsAl4mqUXSzHQb62ngsXTMkVXHP17nnP9T9fq3wKEN3r+zukfXnLve+1T8Ov0e2aBOHrXvcTMpcZC1NuamJDYCOARYVnXd7kzlNoA5cdhA8TjwzogYVvUzJCLayb4sp5LdLjocGJuOUdXxZU0jvZmsk7vimAZ115F9jvc1qLOD7Mu+4vfq1Kn9LD8GjpQ0gSyBVG5TbQM6gNdWXbPDI6JRgrQBwInD+qNWSUOqfgYDs4ErJb0cQNIISVNT/cOA58j+oj+E7HZMT7kVOFfSayQdQoP+g8jWQLgYuEzSuZKGpk7/N0u6NlVbAbxV0ph0q+3SrgKIiBfI+k1mAUcAd6Xy3cB1wJclHQUgaZSkyfv9aa1fcOKw/mg+2V/KlZ/Lga8C84AfS3oGeBA4OdW/CfgV0A48kvb1iIj4EfA14CfAeuCBtOu5TurfDnwI+FNgE/AE8H/J+imIiLuAW4CVwDLgBzlDuZmsxXVbSiQVn05xPZhu4/0HWSe9DWDyQk5mzUPSa4DVwME1X+BmTcMtDrNeJuksSQdJGk42/PX7ThrWzJw4zHrf+cBW4JdkI73+vHfDMWvMt6rMzKwQtzjMzKyQwb0dQE848sgjY+zYsb0dhplZn7Js2bJtEfGiBz4HROIYO3YsS5cu7bqimZntIelX9cp9q8rMzApx4jAzs0KcOMzMrJBSE4ekKZLWSVovaUYndU6VtELSGkmLava1SFou6QdVZZdLak/HrJB0RpmfwczM9lVa53haXvNq4B3ARmCJpHkR8UhVnWHAN4ApEbGhMpFalYvIpr8eWlP+5Yj4Ylmxm5lZ58pscZwErI+IRyNiJ/Adsqmrq50DzImIDQARsaWyQ9Jo4Ezg+hJjNDPrl+Yub+eUmfdw7IwfcsrMe5i7vL3bzl1m4hjFvgvGbExl1cYBw9M6yMskfaxq31eAS4Dddc59YVq/+YY0v8+LSDpP0lJJS7du3XoAH8PMrG+Zu7ydS+eson17BwG0b+/g0jmrui15lJk4VKesdn6TwcCJZC2LyWTrDIyT9C5gS0Qsq3OOa4BXki29uRm4qt6bR8S1ETEpIiaNGOEFy8xs4Ji1YB0dz+/ap6zj+V3MWrCuW85f5gOAG9l3NbPRZOsH1NbZFhE7gB2S7gVOAN4AvCd1fA8Bhkr694j4SEQ8UTlY0nXkX2/AzGxA2LS9o1B5UWW2OJYAx0k6VtJBZAvcz6upcwfwFkmD0+pnJwNrI+LSiBgdEWPTcfdExEcAJFWvt3wW2doFZmaWHD2srVB5UaUljrSewIXAArKRUbdGxBpJF0i6INVZC9xJtlrZYuD6iOgqEXxB0ipJK4HTgE+V9RnMzPqi6ZPH09bask9ZW2sL0yd3z+KNA2Ja9UmTJoXnqjKzgWTu8nYuuX0lO3ftZtSwNqZPHs+0ibXjkxqTtCwiJtWWD4hJDs3MBpppE0fx7cUbALjl/Dd267k95YiZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSFOHGZmVogTh5mZFeLEYWZmhThxmJlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSGDezsAMytu7vJ2Zi1Yx6btHRw9rI3pk8czbeKo3g7LBggnDrM+Zu7ydi6ds4qO53cB0L69g0vnrAJw8rAe4cRh1sfMWrBuT9Ko6Hh+F5fcvpJvL97QS1FZM3pk89McP3Jot5/XfRxmfcym7R11y3fu2t3DkVizO37kUKZO6P5WqFscZn3M0cPaaK+TPEYNa+OW89/YCxHZQOMWh1kyd3k7p8y8h2Nn/JBTZt7D3OXtvR1SXdMnj6ettWWfsrbWFqZPHt9LEdlA4xaH9Tv7M+KoL3U4V+K55PaV7Ny1m1EeVWU9rNTEIWkK8FWgBbg+ImbWqXMq8BWgFdgWEW+r2tcCLAXaI+JdqewI4BZgLPAY8MGIeKrMz2EHpieHju5vAuiLHc4Htw5i4phhvj1lPa60xJG+9K8G3gFsBJZImhcRj1TVGQZ8A5gSERskHVVzmouAtUD1sIAZwN0RMVPSjLT96bI+hx2Ynv5Lfn8TQL0+A2juDueyOj7NulJmi+MkYH1EPAog6TvAVOCRqjrnAHMiYgNARGyp7JA0GjgTuBK4uOqYqcCp6fWNwEKcOJpWT/8lv78J4KCWQXXruMPZ7MXKTByjgMertjcCJ9fUGQe0SloIHAZ8NSJuSvu+AlySyqu9LCI2A0TE5jqtFAAknQecBzBmzJgD+Bh2IHp66Oj+JoDalhG4w9msM2UmDtUpizrvfyLwR0Ab8ICkB8kSypaIWJb6QAqLiGuBawEmTZpU+77WQ3p66Oj+JoDKbTNP42HWtTITx0bgmKrt0cCmOnW2RcQOYIeke4ETgDcA75F0BjAEGCrp3yPiI8ATkkam1sZIYAvWtKZPHt+jf8kfSAKYNnGUE4VZDooo549xSYOBX5C1JtqBJcA5EbGmqs5rgK8Dk4GDgMXAhyNidVWdU4G/qxpVNQv4dVXn+BERcUmjWCZNmhRLly7tzo9nBcxd3u6ho2Z9kKRlETGptry0FkdEvCDpQmAB2XDcGyJijaQL0v7ZEbFW0p3ASmA32ZDd1Z2fFYCZwK2S/gzYAHygrM9g3WPaxFF7OsLd0WzW95X6HEdEzAfm15TNrtmeBcxqcI6FZCOnKtu/JmvFmJlZL/CUI2ZmVogTh5mZFeLEYWZmhXSZONLcUGZmZkC+FsdDkm6TdIakeg/1mZnZAJIncYwjewL7o8B6Sf8kaVy5YZmZWbPqMnFE5q6IOBv4BPBxYLGkRZI8KN/MbIDp8jkOSS8FPkLW4ngC+CtgHjABuA04tswAzcysueR5APAB4N+AaRGxsap8qaTZnRxjZmb9VJ7EMT46mdAqIv65m+MxM7Mml6dz/MdppT4AJA2XtKDEmKzK3OXtnDLzHo6d8UNOmXkPc5e393ZIZjbA5WlxjIiI7ZWNiHiqs8WTrHv19LKrZmZ55EkcuySNqSzvKunlvHhBJitBTy+7WqZHNj/N8SOHdl3RzJpensTx98B9khal7beSlmS1cvX0sqtlOn7kUKZOcCvJrD/oMnFExJ2S3gD8IdlysJ+KiG2lR2Y9vuyqmVkeeSc53EW2ROtvgOMlvbW8kKxi+uTxtLW27FNW5rKrZmZ55HkA8BPARWRrhq8ga3k8AJxebmhW6QD3sqtm1kzy9HFcBPwB8GBEnCbp1cAV5YZlFV521cyaTZ5bVb+LiN8BSDo4In4O+F6JmdkAlafFsTE9ADgXuEvSU8CmcsMyM7NmlWdU1Vnp5eWSfgIcDtxZalRmZta0GiYOSYOAlRHxOoCIWNSovpmZ9X8N+zgiYjfwsKQxPRSPmZk1uTx9HCOBNZIWAzsqhRHxntKiGuDmLm9n1oJ1bNrewdHD2hjSOogjDz24t8MyMwPyJQ4Pve1B9SY2HOSV3s2sieTpHHe/Rg+qN7Hh7oDHn6w/b5WZWU/L8+T4M+ydDfcgoBXYERGe6rQE/WliQzPrn/K0OA6r3pY0DTiptIgGuEYTG5qZNYO8kxzuERFz8TxVpfHEhmbW7PLcqnpv1eYgYBJeyKk0ntjQzJpdnlFV7656/QLwGDC1lGgM8MSGZtbc8vRxnNsTgZiZWd/QZR+HpBvTJIeV7eGSbig3LDMza1Z5OsdfHxHbKxsR8RQwsbyQzMysmeVJHIMkDa9sSDqCfH0jZmbWD+VJHFcBP5X0eUmfA34KfCHPySVNkbRO0npJMzqpc6qkFZLWSFqUyoZIWizp4VR+RVX9yyW1p2NWSDojTyxmZtY98nSO3yRpKdmzGwLeGxGPdHWcpBbgauAdwEZgiaR51cemvpNvAFMiYoOko9Ku54DTI+JZSa3AfZJ+FBEPpv1fjogvFvicZmbWTfI8x/GHwJqI+HraPkzSyRHxUBeHngSsj4hH03HfIRvGW510zgHmRMQGgIjYkn4H8Gyq05p+/OyImVkTyHOr6hr2folDNrX6NTmOGwU8XrW9MZVVGwcMl7RQ0jJJH6vskNQiaQWwBbirJlFdKGmlpBuq+1+qSTpP0lJJS7du3ZojXDMzyyNP4lBqAQB7FnfK0zlebzLw2lbDYOBE4ExgMnCZpHHpfXZFxARgNHCSpNelY64BXglMADaT9cG8+I0iro2ISRExacSIETnCNTOzPPIkjkcl/bWk1vRzEfBojuM2AsdUbY8GNtWpc2dE7IiIbcC9wAnVFdJQ4IXAlLT9REoqu4Hr8ISLZmY9Kk/iuAB4E9BO9kV/MvDJHMctAY6TdKykg4APA/Nq6twBvEXSYEmHpHOvlTSi8tChpDbg7cDP0/bIquPPAlbniMXMzLpJnlFVW8i+9IE9X+TvAm7r4rgXJF0ILABagBsiYo2kC9L+2RGxVtKdwEpgN3B9RKyW9HrgxjQyaxBwa0T8IJ36C5ImkN32egw4v9AnNjOzA5LrQb70Bf7HwNnp9310kTgAImI+ML+mbHbN9ixgVk3ZSjp5Oj0iPponZjMzK0fDxCHprWRDZs8EFgOnAK+IiN/2QGxmZtaEOk0ckjYCG8hGMU2PiGck/beTRjnmLm9n1oJ1bNrewdHD2hjSOogjDz24t8MyM3uRRi2O7wLTgA8BuyTdgR/CK8Xc5e1cOmcVHc/vAqB9eweD6g1mNjNrAp2OqoqIi4CxwJeA04BfACMkfVDSoT0T3sAwa8G6PUmjYnfA40++eO1xM7Pe1nA4bmTuiYhPkiWRc8haIY+VH9rAsWl7/QSxc9fuHo7EzKxreZ7jACAino+I70fEOez7YJ8doKOHtdUtH9VJuZlZb8qdOKpFhO+hdKPpk8fT1tqyT1lbawvTJ4/vpYjMzDrnBZmawLSJ2dyPl9y+kp27djNqWBvTJ4/fU25m1kycOHKqHS7b3V/s0yaO4tuLNwBwy/lv7Lbzmpl1tzzrcYwDpgMvr64fEaeXGFdTqTdc9tI5qwDcKjCzASdPi+M2YDbZTLS7uqjbL9UbLtvx/C4uuX3lnlZCd3hk89McP3Jot53PzKwMeRLHCxGRZ+GmfqunhsseP3IoUye4BWNmzS1P4vi+pL8Avke2FjgAEfFkaVE1maOHtdFeJ3mMGtbm/ggzG3DyDMf9OFkfx0+BZelnaZlBNRsPlzUz2yvPehzH9kQgzczDZc3M9sozqqoV+HPgraloIfD/I+L5EuNqOh4ua2aWydPHcQ3QCnwjbX80lX2irKDMzKx55UkcfxARJ1Rt3yPp4bICMjOz5panc3yXpFdWNiS9ggH6PIeZmeVrcUwHfiLpUUBkT5CfW2pUZmbWtPKMqrpb0nHAeLLE8fOIeK6Lw8zMrJ9qtOb46RFxj6T31ux6pSQiYk7JsZmZWRNq1OJ4G3AP8O46+wJw4jAzG4A6TRwR8Y/p5eci4r+r90ka8A8FmpkNVHlGVX23Ttnt3R2ImZn1DY36OF4NvBY4vKafYygwpOzAzMysOTXq4xgPvAsYxr79HM8AnywzKDMza16N+jjuAO6Q9MaIeKAHYzIzsyaW5wHA5ZL+kuy21Z5bVBHxp6VFZWZmTStP5/i/Ab8HTAYWAaPJbleZmdkAlCdxvCoiLgN2RMSNwJnA75cblpmZNas8iaOy7sZ2Sa8DDgfGlhaRmZk1tTx9HNdKGg5cBswDDgU+W2pUZmbWtPJMcnh9erkIeEW54ZiZWbNr9ADgxY0OjIgvdX84ZmbW7Br1cRyWfiaRrTk+Kv1cAByf5+SSpkhaJ2m9pBmd1DlV0gpJayQtSmVDJC2W9HAqv6Kq/hGS7pL0X+n38Hwf1czMukOniSMiroiIK4AjgTdExN9GxN8CJ5INyW1IUgtwNfBOskRztqTja+oMI1vL/D0R8VrgA2nXc8DpacnaCcAUSX+Y9s0A7o6I44C707aZmfWQPKOqxgA7q7Z3km9U1UnA+oh4NCJ2At8BptbUOQeYExEbACJiS/odEfFsqtOafiJtTwVuTK9vBKbliMXMzLpJ3gcAF0u6XNI/Ag8BN+U4bhTweNX2xlRWbRwwXNJCScskfayyQ1KLpBXAFuCuiHgo7XpZRGwGSL+Pqvfmks6TtFTS0q1bt+YI18zM8ugycUTElWRrjD8FbAfOjYh/ynFu1TtdzfZgsltfZ5I9mX6ZpHHpfXdFxASy22InpWdIcouIayNiUkRMGjFiRJFDzcysgUajqoZGxNOSjgAeSz+VfUdExJNdnHsjcEzV9mhgU5062yJiB7BD0r3ACcAvKhUiYrukhcAUYDXwhKSREbFZ0kiyFomZmfWQRi2Om9PvZcDSqp/KdleWAMdJOlbSQcCHyR4grHYH8BZJgyUdApwMrJU0InWcI6kNeDvw83TMPODj6fXH0znMzKyHNJpW/V3p934tExsRL0i6EFgAtAA3RMQaSRek/bMjYq2kO4GVwG7g+ohYLen1wI1pZNYg4NaI+EE69UzgVkl/Bmxg70gsMzPrAY1uVb2h0YER8bOuTh4R84H5NWWza7ZnAbNqylYCEzs556+BP+rqvc3MrByNphy5qsG+AE7v5ljMzKwPaHSr6rSeDMTMzPqGPLPjkobCHs++KwDmeZbDzMz6mS4TR3ro71SyxDGfbAqR+8j3EKCZmfUzeZ4cfz9ZZ/T/RMS5ZM9ZHFxqVGZm1rTyJI6OiNgNvCBpKNkDd16Xw8xsgMrTx7E0PYx3HdnDf88Ci0uNyszMmlaj5zi+DtwcEX+Rimanh/WGpucszMxsAGrU4vgv4Ko0H9QtwLcjYkXPhGVmZs2q0UJOX42INwJvA54E/lXSWkmfrcxga2ZmA0+eadV/FRH/HBETyRZeOgtYW3pkZmbWlLpMHJJaJb1b0reAH5FNef6+0iMzM7Om1Khz/B3A2WSLLC0mW/r1vLR2hpmZDVCNOsc/Q7Ymx9/lWLTJzMwGCE9yaGZmheR5ctzMzGwPJw4zMyvEicPMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCnHiMDOzQpw4zMysECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4zMyvEicPMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCnHiMDOzQkpNHJKmSFonab2kGZ3UOVXSCklrJC1KZcdI+omktan8oqr6l0tqT8eskHRGmZ/BzMz2NbisE0tqAa4G3gFsBJZImhcRj1TVGQZ8A5gSERskHZV2vQD8bUT8TNJhwDJJd1Ud++WI+GJZsZuZWefKbHGcBKyPiEcjYifwHWBqTZ1zgDkRsQEgIrak35sj4mfp9TPAWmBUibGamVlOZSaOUcDjVdsbefGX/zhguKSFkpZJ+ljtSSSNBSYCD1UVXyhppaQbJA2v9+aSzpO0VNLSrVu3HsjnMDOzKmUmDtUpi5rtwcCJwJnAZOAySeP2nEA6FPgu8DcR8XQqvgZ4JTAB2AxcVe/NI+LaiJgUEZNGjBhxQB/EzMz2Kq2Pg6yFcUzV9mhgU5062yJiB7BD0r3ACcAvJLWSJY1vRcScygER8UTltaTrgB+UFL+ZmdVRZotjCXCcpGMlHQR8GJhXU+cO4C2SBks6BDgZWCtJwL8AayPiS9UHSBpZtXkWsLq0T2BmZi9SWosjIl6QdCGwAGgBboiINZIuSPtnR8RaSXcCK4HdwPURsVrSm4GPAqskrUin/ExEzAe+IGkC2W2vx4Dzy/oMZmb2YmXeqiJ90c+vKZtdsz0LmFVTdh/1+0iIiI92c5hmZlaAnxw3M7NCnDjMzKwQJw4zMyvEicPMzApx4jAzs0KcOMzMrBAnDjMzK8SJw8zMCnHiMDOzQpw4zMysECcOMzMrxInDzMwKceIwM7NCnDjMzKwQJw4zMyuk1PU4+rK5y9uZtWAdm7Z3cPSwNqZPHt/bIZmZNQW3OOqYu7ydS+eson17BwG0b+/g0jmr2Pbsc70dmplZr3PiqGPWgnV0PL9rn7KO53fx6NYdvRSRmVnzcOKoY9P2jrrlAUydMKpngzEzazJOHHUcPaytbvmoYW2cc/KYHo7GzKy5OHHUMX3yeNpaW/Ypa2ttcQe5mRkeVVXXtInZ7ajaUVWVcjOzgcyJoxPTJo5yojAzq8O3qszMrBAnDjMzK8SJw8zMCnHiMDOzQpw4zMysEEVEb8dQOklbgV8VOORIYFtJ4ZTFMfcMx9wzHHPP6Crml0fEiNrCAZE4ipK0NCIm9XYcRTjmnuGYe4Zj7hn7G7NvVZmZWSFOHGZmVogTR33X9nYA+8Ex9wzH3DMcc8/Yr5jdx2FmZoW4xWFmZoU4cZiZWSFOHFUkTZG0TtJ6STN6O548JD0maZWkFZKW9nY89Ui6QdIWSauryo6QdJek/0q/h/dmjPV0EvflktrT9V4h6YzejLGapGMk/UTSWklrJF2Uypv2WjeIuZmv8xBJiyU9nGK+IpU37XWGhnEXvtbu40gktQC/AN4BbASWAGdHxCO9GlgXJD0GTIqIpn3wSNJbgWeBmyLidansC8CTETEzJenhEfHp3oyzVidxXw48GxFf7M3Y6pE0EhgZET+TdBiwDJgG/AlNeq0bxPxBmvc6C3hJRDwrqRW4D7gIeC9Nep2hYdxTKHit3eLY6yRgfUQ8GhE7ge8AU3s5pn4hIu4FnqwpngrcmF7fSPZl0VQ6ibtpRcTmiPhZev0MsBYYRRNf6wYxN63IPJs2W9NP0MTXGRrGXZgTx16jgMertjfS5P+AkwB+LGmZpPN6O5gCXhYRmyH78gCO6uV4irhQ0sp0K6upbkdUSBoLTAQeoo9c65qYoYmvs6QWSSuALcBdEdEnrnMncUPBa+3EsZfqlPWF+3inRMQbgHcCf5lur1h5rgFeCUwANgNX9W44LybpUOC7wN9ExNO9HU8edWJu6uscEbsiYgIwGjhJ0ut6O6Y8Oom78LV24thrI3BM1fZoYFMvxZJbRGxKv7cA3yO75dYXPJHub1fuc2/p5XhyiYgn0v98u4HraLLrne5dfxf4VkTMScVNfa3rxdzs17kiIrYDC8n6CZr6Olerjnt/rrUTx15LgOMkHSvpIODDwLxejqkhSS9JHYpIegnwx8Dqxkc1jXnAx9PrjwN39GIsuVW+GJKzaKLrnTo//wVYGxFfqtrVtNe6s5ib/DqPkDQsvW4D3g78nCa+ztB53PtzrT2qqkoahvYVoAW4ISKu7OWQGpL0CrJWBsBg4OZmjFnSt4FTyaZwfgL4R2AucCswBtgAfCAimqojupO4TyVr0gfwGHB+5b52b5P0ZuA/gVXA7lT8GbI+g6a81g1iPpvmvc6vJ+v8biH74/vWiPicpJfSpNcZGsb9bxS81k4cZmZWiG9VmZlZIU4cZmZWiBOHmZkV4sRhZmaFOHGYmVkhThzW50n6sqS/qdpeIOn6qu2rJF3c4PhvSnp/er1Q0qQ6dVolzUwzn65Os4y+M+17TNKR+xH3nvftZP/VabbSRyR1VM1e+n5J8ytj8ruTpJGSftBg/0GS7pU0uLvf2/oOJw7rD34KvAlA0iCy5y5eW7X/TcD9B/genwdGAq9LM+W+GzjsAM/ZUET8ZZoe4gzglxExIf3cHhFnpKd/u9vFZE8PdxbTTuBu4EMlvLf1EU4c1h/cT0ocZAljNfCMpOGSDgZeAyyX9FlJS1KL4dr01HKXJB0CfBL4q4h4DvZMiXFrnboXp/OvrmkFfSxNIvdweuCq9rjPpxZIrv8nK60cSWMl/VzS9ek9vyXp7ZLuT62jk1L9l6QJ7JZIWi6ps5mf3wfcmY55bWpZrUixH5fqzAX+T544rX9yc9P6vIjYJOkFSWPIEsgDZDMbvxH4DbAyInZK+npEfA4gfXm/C/h+jrd4FbChqwkDJZ0InAucTDZp5kOSFgE7gb8nm5Bym6Qjao77AnA4cG7s3xO5rwI+AJxHNnXOOcCbgfeQPYU9Lb3/PRHxp+kW12JJ/xERO6riOBZ4qpIcgQuAr0bEt9I0PC2pfDXwB/sRp/UTbnFYf1FpdVQSx0HVZTAAAAIUSURBVANV2z9NdU6T9JCkVcDp7Hs7qzu8GfheROxI6x7MAd6S3uv2ymJbNdNQXAYMi4jz9zNpAPx3RKxKk9StAe5O51oFjE11/hiYoWxK7YXAELKpMaqNBLZWbT8AfEbSp4GXR0RHin8XsLMyT5oNPE4c1l9U+jl+n+wv4gfJWhxvAu6XNAT4BvD+iPh9svv4Q3Keez0wJscXZWe3vkTnU/QvAU6sbYUU9FzV691V27vZe1dBwPuq+knGRMTamvN0UHVNIuJmslZLB7BA0ulVdQ8GfncAMVsf5sRh/cX9ZLeenkxTRD8JDCNLHg+w9wtxm7K1HzodzVQrIn5LNoPr19Itm8roo4/UVL0XmCbpkDRb8VlkE/jdDXwwTYJHTZK4E5gJ/LDkv+AXAH9V6deRNLFOnV+wt4VSmUTz0Yj4GtnMr69P5S8FtkbE8yXGa03MicP6i1Vko6kerCn7TURsSyOQrktlc8n+0i/iH8hu4zwiaXU6R/VtHdISqN8EFpPNSHt9RCyPiDXAlcAiSQ8DX6o57rYU27w03XUZPk+2VOjKFP/nayuk/o5fSnpVKvoQsDrd3no1cFMqPw2YX1Kc1gd4dlwz20PSWcCJEfEPDerMAS6NiHU9F5k1E4+qMrM9IuJ7lVtq9aRbdXOdNAY2tzjMzKwQ93GYmVkhThxmZlaIE4eZmRXixGFmZoU4cZiZWSH/CwSPkN84qmWtAAAAAElFTkSuQmCC\n" - }, - "metadata": { - "needs_background": "light" - } - } - ], - "source": [ - "import matplotlib.pyplot as plt\n", - "import numpy as np\n", - "\n", - "plt.title('Learning Curve')\n", - "plt.xlabel('Wall Clock Time (s)')\n", - "plt.ylabel('Validation Accuracy')\n", - "plt.scatter(time_history, 1-np.array(valid_loss_history))\n", - "plt.step(time_history, 1-np.array(best_valid_loss_history), where='post')\n", - "plt.show()" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "## 3. Customized Learner" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "Some experienced automl users may have a preferred model to tune or may already have a reasonably by-hand-tuned model before launching the automl experiment. They need to select optimal configurations for the customized model mixed with standard built-in learners. \n", - "\n", - "FLAML can easily incorporate customized/new learners (preferably with sklearn API) provided by users in a real-time manner, as demonstrated below." - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "### Example of Regularized Greedy Forest\n", - "\n", - "[Regularized Greedy Forest](https://arxiv.org/abs/1109.0887) (RGF) is a machine learning method currently not included in FLAML. The RGF has many tuning parameters, the most critical of which are: `[max_leaf, n_iter, n_tree_search, opt_interval, min_samples_leaf]`. To run a customized/new learner, the user needs to provide the following information:\n", - "* an implementation of the customized/new learner\n", - "* a list of hyperparameter names and types\n", - "* rough ranges of hyperparameters (i.e., upper/lower bounds)\n", - "* choose initial value corresponding to low cost for cost-related hyperparameters (e.g., initial value for max_leaf and n_iter should be small)\n", - "\n", - "In this example, the above information for RGF is wrapped in a python class called *MyRegularizedGreedyForest* that exposes the hyperparameters." - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "''' SKLearnEstimator is the super class for a sklearn learner '''\n", - "from flaml.model import SKLearnEstimator\n", - "from flaml import tune\n", - "from rgf.sklearn import RGFClassifier, RGFRegressor\n", - "\n", - "\n", - "class MyRegularizedGreedyForest(SKLearnEstimator):\n", - "\n", - "\n", - " def __init__(self, task = 'binary:logistic', n_jobs = 1, **params):\n", - " '''Constructor\n", - " \n", - " Args:\n", - " task: A string of the task type, one of\n", - " 'binary:logistic', 'multi:softmax', 'regression'\n", - " n_jobs: An integer of the number of parallel threads\n", - " params: A dictionary of the hyperparameter names and values\n", - " '''\n", - "\n", - " super().__init__(task, **params)\n", - "\n", - " '''task=regression for RGFRegressor; \n", - " binary:logistic and multiclass:softmax for RGFClassifier'''\n", - " if 'regression' in task:\n", - " self.estimator_class = RGFRegressor\n", - " else:\n", - " self.estimator_class = RGFClassifier\n", - "\n", - " # convert to int for integer hyperparameters\n", - " self.params = {\n", - " \"n_jobs\": n_jobs,\n", - " 'max_leaf': int(params['max_leaf']),\n", - " 'n_iter': int(params['n_iter']),\n", - " 'n_tree_search': int(params['n_tree_search']),\n", - " 'opt_interval': int(params['opt_interval']),\n", - " 'learning_rate': params['learning_rate'],\n", - " 'min_samples_leaf':int(params['min_samples_leaf'])\n", - " } \n", - "\n", - " @classmethod\n", - " def search_space(cls, data_size, task):\n", - " '''[required method] search space\n", - "\n", - " Returns:\n", - " A dictionary of the search space. \n", - " Each key is the name of a hyperparameter, and value is a dict with\n", - " its domain and init_value (optional), cat_hp_cost (optional) \n", - " e.g., \n", - " {'domain': tune.randint(lower=1, upper=10), 'init_value': 1}\n", - " '''\n", - " space = { \n", - " 'max_leaf': {'domain': tune.qloguniform(lower = 4, upper = data_size, q = 1), 'init_value': 4},\n", - " 'n_iter': {'domain': tune.qloguniform(lower = 1, upper = data_size, q = 1), 'init_value': 1},\n", - " 'n_tree_search': {'domain': tune.qloguniform(lower = 1, upper = 32768, q = 1), 'init_value': 1},\n", - " 'opt_interval': {'domain': tune.qloguniform(lower = 1, upper = 10000, q = 1), 'init_value': 100},\n", - " 'learning_rate': {'domain': tune.loguniform(lower = 0.01, upper = 20.0)},\n", - " 'min_samples_leaf': {'domain': tune.qloguniform(lower = 1, upper = 20, q = 1), 'init_value': 20},\n", - " }\n", - " return space\n", - "\n", - " @classmethod\n", - " def size(cls, config):\n", - " '''[optional method] memory size of the estimator in bytes\n", - " \n", - " Args:\n", - " config - the dict of the hyperparameter config\n", - "\n", - " Returns:\n", - " A float of the memory size required by the estimator to train the\n", - " given config\n", - " '''\n", - " max_leaves = int(round(config['max_leaf']))\n", - " n_estimators = int(round(config['n_iter']))\n", - " return (max_leaves*3 + (max_leaves-1)*4 + 1.0)*n_estimators*8\n", - "\n", - " @classmethod\n", - " def cost_relative2lgbm(cls):\n", - " '''[optional method] relative cost compared to lightgbm\n", - " '''\n", - " return 1.0\n" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "source": [ - "### Add Customized Learner and Run FLAML AutoML\n", - "\n", - "After adding RGF into the list of learners, we run automl by tuning hyperpameters of RGF as well as the default learners. " - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": { - "slideshow": { - "slide_type": "slide" - } - }, - "outputs": [], - "source": [ - "automl = AutoML()\n", - "automl.add_learner(learner_name = 'RGF', learner_class = MyRegularizedGreedyForest)" - ] - }, - { - "cell_type": "code", - "execution_count": 15, - "metadata": { - "slideshow": { - "slide_type": "slide" - }, - "tags": [] - }, - "outputs": [ - { - "output_type": "stream", - "name": "stderr", - "text": [ - "[flaml.automl: 02-17 13:58:01] {839} INFO - Evaluation method: holdout\n", - "INFO - Evaluation method: holdout\n", - "[flaml.automl: 02-17 13:58:01] {564} INFO - Using StratifiedKFold\n", - "INFO - Using StratifiedKFold\n", - "[flaml.automl: 02-17 13:58:01] {860} INFO - Minimizing error metric: 1-accuracy\n", - "INFO - Minimizing error metric: 1-accuracy\n", - "[flaml.automl: 02-17 13:58:01] {880} INFO - List of ML learners in AutoML Run: ['RGF', 'lgbm', 'rf', 'xgboost']\n", - "INFO - List of ML learners in AutoML Run: ['RGF', 'lgbm', 'rf', 'xgboost']\n", - "[flaml.automl: 02-17 13:58:01] {939} INFO - iteration 0 current learner RGF\n", - "INFO - iteration 0 current learner RGF\n", - "[flaml.automl: 02-17 13:58:02] {1093} INFO - at 1.4s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "INFO - at 1.4s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "[flaml.automl: 02-17 13:58:02] {939} INFO - iteration 1 current learner RGF\n", - "INFO - iteration 1 current learner RGF\n", - "[flaml.automl: 02-17 13:58:04] {1093} INFO - at 2.9s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "INFO - at 2.9s,\tbest RGF's error=0.3787,\tbest RGF's error=0.3787\n", - "[flaml.automl: 02-17 13:58:04] {939} INFO - iteration 2 current learner lgbm\n", - "INFO - iteration 2 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:04] {1093} INFO - at 3.1s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", - "INFO - at 3.1s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", - "[flaml.automl: 02-17 13:58:04] {939} INFO - iteration 3 current learner lgbm\n", - "INFO - iteration 3 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:04] {1093} INFO - at 3.3s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", - "INFO - at 3.3s,\tbest lgbm's error=0.3777,\tbest lgbm's error=0.3777\n", - "[flaml.automl: 02-17 13:58:04] {939} INFO - iteration 4 current learner RGF\n", - "INFO - iteration 4 current learner RGF\n", - "[flaml.automl: 02-17 13:58:06] {1093} INFO - at 4.8s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3777\n", - "INFO - at 4.8s,\tbest RGF's error=0.3787,\tbest lgbm's error=0.3777\n", - "[flaml.automl: 02-17 13:58:06] {939} INFO - iteration 5 current learner lgbm\n", - "INFO - iteration 5 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:06] {1093} INFO - at 5.0s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", - "INFO - at 5.0s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", - "[flaml.automl: 02-17 13:58:06] {939} INFO - iteration 6 current learner lgbm\n", - "INFO - iteration 6 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:06] {1093} INFO - at 5.2s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", - "INFO - at 5.2s,\tbest lgbm's error=0.3669,\tbest lgbm's error=0.3669\n", - "[flaml.automl: 02-17 13:58:06] {939} INFO - iteration 7 current learner lgbm\n", - "INFO - iteration 7 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:06] {1093} INFO - at 5.3s,\tbest lgbm's error=0.3662,\tbest lgbm's error=0.3662\n", - "INFO - at 5.3s,\tbest lgbm's error=0.3662,\tbest lgbm's error=0.3662\n", - "[flaml.automl: 02-17 13:58:06] {939} INFO - iteration 8 current learner lgbm\n", - "INFO - iteration 8 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:07] {1093} INFO - at 5.7s,\tbest lgbm's error=0.3636,\tbest lgbm's error=0.3636\n", - "INFO - at 5.7s,\tbest lgbm's error=0.3636,\tbest lgbm's error=0.3636\n", - "[flaml.automl: 02-17 13:58:07] {939} INFO - iteration 9 current learner lgbm\n", - "INFO - iteration 9 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:07] {1093} INFO - at 5.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 5.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:07] {939} INFO - iteration 10 current learner lgbm\n", - "INFO - iteration 10 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:07] {1093} INFO - at 6.2s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 6.2s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:07] {939} INFO - iteration 11 current learner lgbm\n", - "INFO - iteration 11 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:07] {1093} INFO - at 6.3s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 6.3s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:07] {939} INFO - iteration 12 current learner lgbm\n", - "INFO - iteration 12 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:07] {1093} INFO - at 6.4s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 6.4s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:07] {939} INFO - iteration 13 current learner lgbm\n", - "INFO - iteration 13 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:07] {1093} INFO - at 6.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 6.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:07] {939} INFO - iteration 14 current learner lgbm\n", - "INFO - iteration 14 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:08] {1093} INFO - at 6.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 6.9s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:08] {939} INFO - iteration 15 current learner lgbm\n", - "INFO - iteration 15 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:09] {1093} INFO - at 8.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "INFO - at 8.6s,\tbest lgbm's error=0.3621,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:09] {939} INFO - iteration 16 current learner xgboost\n", - "INFO - iteration 16 current learner xgboost\n", - "[flaml.automl: 02-17 13:58:10] {1093} INFO - at 8.7s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", - "INFO - at 8.7s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3621\n", - "[flaml.automl: 02-17 13:58:10] {939} INFO - iteration 17 current learner lgbm\n", - "INFO - iteration 17 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:12] {1093} INFO - at 10.7s,\tbest lgbm's error=0.3611,\tbest lgbm's error=0.3611\n", - "INFO - at 10.7s,\tbest lgbm's error=0.3611,\tbest lgbm's error=0.3611\n", - "[flaml.automl: 02-17 13:58:12] {939} INFO - iteration 18 current learner xgboost\n", - "INFO - iteration 18 current learner xgboost\n", - "[flaml.automl: 02-17 13:58:12] {1093} INFO - at 10.9s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3611\n", - "INFO - at 10.9s,\tbest xgboost's error=0.3787,\tbest lgbm's error=0.3611\n", - "[flaml.automl: 02-17 13:58:12] {939} INFO - iteration 19 current learner xgboost\n", - "INFO - iteration 19 current learner xgboost\n", - "[flaml.automl: 02-17 13:58:12] {1093} INFO - at 11.0s,\tbest xgboost's error=0.3757,\tbest lgbm's error=0.3611\n", - "INFO - at 11.0s,\tbest xgboost's error=0.3757,\tbest lgbm's error=0.3611\n", - "[flaml.automl: 02-17 13:58:12] {939} INFO - iteration 20 current learner xgboost\n", - "INFO - iteration 20 current learner xgboost\n", - "[flaml.automl: 02-17 13:58:12] {1093} INFO - at 11.1s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3611\n", - "INFO - at 11.1s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3611\n", - "[flaml.automl: 02-17 13:58:12] {939} INFO - iteration 21 current learner rf\n", - "INFO - iteration 21 current learner rf\n", - "[flaml.automl: 02-17 13:58:13] {1093} INFO - at 11.8s,\tbest rf's error=0.4012,\tbest lgbm's error=0.3611\n", - "INFO - at 11.8s,\tbest rf's error=0.4012,\tbest lgbm's error=0.3611\n", - "[flaml.automl: 02-17 13:58:13] {939} INFO - iteration 22 current learner RGF\n", - "INFO - iteration 22 current learner RGF\n", - "[flaml.automl: 02-17 13:58:14] {1093} INFO - at 13.2s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3611\n", - "INFO - at 13.2s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3611\n", - "[flaml.automl: 02-17 13:58:14] {939} INFO - iteration 23 current learner lgbm\n", - "INFO - iteration 23 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:16] {1093} INFO - at 14.7s,\tbest lgbm's error=0.3585,\tbest lgbm's error=0.3585\n", - "INFO - at 14.7s,\tbest lgbm's error=0.3585,\tbest lgbm's error=0.3585\n", - "[flaml.automl: 02-17 13:58:16] {939} INFO - iteration 24 current learner rf\n", - "INFO - iteration 24 current learner rf\n", - "[flaml.automl: 02-17 13:58:16] {1093} INFO - at 15.3s,\tbest rf's error=0.3977,\tbest lgbm's error=0.3585\n", - "INFO - at 15.3s,\tbest rf's error=0.3977,\tbest lgbm's error=0.3585\n", - "[flaml.automl: 02-17 13:58:16] {939} INFO - iteration 25 current learner xgboost\n", - "INFO - iteration 25 current learner xgboost\n", - "[flaml.automl: 02-17 13:58:16] {1093} INFO - at 15.5s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3585\n", - "INFO - at 15.5s,\tbest xgboost's error=0.3756,\tbest lgbm's error=0.3585\n", - "[flaml.automl: 02-17 13:58:16] {939} INFO - iteration 26 current learner lgbm\n", - "INFO - iteration 26 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:18] {1093} INFO - at 16.9s,\tbest lgbm's error=0.3585,\tbest lgbm's error=0.3585\n", - "INFO - at 16.9s,\tbest lgbm's error=0.3585,\tbest lgbm's error=0.3585\n", - "[flaml.automl: 02-17 13:58:18] {939} INFO - iteration 27 current learner lgbm\n", - "INFO - iteration 27 current learner lgbm\n", - "[flaml.automl: 02-17 13:58:21] {1093} INFO - at 19.6s,\tbest lgbm's error=0.3531,\tbest lgbm's error=0.3531\n", - "INFO - at 19.6s,\tbest lgbm's error=0.3531,\tbest lgbm's error=0.3531\n", - "[flaml.automl: 02-17 13:58:21] {939} INFO - iteration 28 current learner rf\n", - "INFO - iteration 28 current learner rf\n", - "[flaml.automl: 02-17 13:58:21] {1093} INFO - at 20.3s,\tbest rf's error=0.3977,\tbest lgbm's error=0.3531\n", - "INFO - at 20.3s,\tbest rf's error=0.3977,\tbest lgbm's error=0.3531\n", - "[flaml.automl: 02-17 13:58:21] {939} INFO - iteration 29 current learner rf\n", - "INFO - iteration 29 current learner rf\n", - "[flaml.automl: 02-17 13:58:22] {1093} INFO - at 20.9s,\tbest rf's error=0.3977,\tbest lgbm's error=0.3531\n", - "INFO - at 20.9s,\tbest rf's error=0.3977,\tbest lgbm's error=0.3531\n", - "[flaml.automl: 02-17 13:58:22] {939} INFO - iteration 30 current learner RGF\n", - "INFO - iteration 30 current learner RGF\n", - "[flaml.automl: 02-17 13:58:23] {1093} INFO - at 21.9s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3531\n", - "INFO - at 21.9s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3531\n", - "[flaml.automl: 02-17 13:58:23] {939} INFO - iteration 31 current learner RGF\n", - "INFO - iteration 31 current learner RGF\n", - "[flaml.automl: 02-17 13:58:24] {1093} INFO - at 23.3s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3531\n", - "INFO - at 23.3s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3531\n", - "[flaml.automl: 02-17 13:58:24] {939} INFO - iteration 32 current learner RGF\n", - "INFO - iteration 32 current learner RGF\n", - "[flaml.automl: 02-17 13:59:08] {1093} INFO - at 67.1s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3531\n", - "INFO - at 67.1s,\tbest RGF's error=0.3674,\tbest lgbm's error=0.3531\n", - "[flaml.automl: 02-17 13:59:08] {1133} INFO - selected model: LGBMClassifier(learning_rate=0.1564464373197609, max_bin=511,\n", - " min_child_weight=1.4188300323104601, n_estimators=12,\n", - " num_leaves=45, objective='binary',\n", - " reg_alpha=3.209664512322882e-10, reg_lambda=0.8927146483558472,\n", - " subsample=0.96058565726185)\n", - "INFO - selected model: LGBMClassifier(learning_rate=0.1564464373197609, max_bin=511,\n", - " min_child_weight=1.4188300323104601, n_estimators=12,\n", - " num_leaves=45, objective='binary',\n", - " reg_alpha=3.209664512322882e-10, reg_lambda=0.8927146483558472,\n", - " subsample=0.96058565726185)\n", - "[flaml.automl: 02-17 13:59:08] {894} INFO - fit succeeded\n", - "INFO - fit succeeded\n" - ] - } - ], - "source": [ - "settings = {\n", - " \"time_budget\": 60, # total running time in seconds\n", - " \"metric\": 'accuracy', \n", - " \"estimator_list\": ['RGF', 'lgbm', 'rf', 'xgboost'], # list of ML learners\n", - " \"task\": 'classification', # task type \n", - " \"sample\": True, # whether to subsample training data\n", - " \"log_file_name\": 'airlines_experiment.log', # cache directory of flaml log files \n", - " \"log_training_metric\": True, # whether to log training metric\n", - "}\n", - "\n", - "'''The main flaml automl API'''\n", - "automl.fit(X_train = X_train, y_train = y_train, **settings)" - ] - }, - { - "source": [ - "## 4. Comparison with alternatives\n", - "\n", - "### FLAML's accuracy" - ], - "cell_type": "markdown", - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 16, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "flaml accuracy = 0.6721222728149148\n" - ] - } - ], - "source": [ - "print('flaml accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))" - ] - }, - { - "source": [ - "### Default LightGBM" - ], - "cell_type": "markdown", - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 17, - "metadata": {}, - "outputs": [], - "source": [ - "from lightgbm import LGBMClassifier\n", - "lgbm = LGBMClassifier()" - ] - }, - { - "cell_type": "code", - "execution_count": 18, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "LGBMClassifier()" - ] - }, - "metadata": {}, - "execution_count": 18 - } - ], - "source": [ - "lgbm.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 19, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "default lgbm accuracy = 0.6602123904305652\n" - ] - } - ], - "source": [ - "y_pred = lgbm.predict(X_test)\n", - "from flaml.ml import sklearn_metric_loss_score\n", - "print('default lgbm accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))" - ] - }, - { - "source": [ - "### Default XGBoost" - ], - "cell_type": "markdown", - "metadata": {} - }, - { - "cell_type": "code", - "execution_count": 20, - "metadata": {}, - "outputs": [], - "source": [ - "from xgboost import XGBClassifier\n", - "xgb = XGBClassifier()" - ] - }, - { - "cell_type": "code", - "execution_count": 21, - "metadata": {}, - "outputs": [ - { - "output_type": "execute_result", - "data": { - "text/plain": [ - "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=1,\n", - " colsample_bynode=1, colsample_bytree=1, gamma=0, gpu_id=-1,\n", - " importance_type='gain', interaction_constraints='',\n", - " learning_rate=0.300000012, max_delta_step=0, max_depth=6,\n", - " min_child_weight=1, missing=nan, monotone_constraints='()',\n", - " n_estimators=100, n_jobs=8, num_parallel_tree=1, random_state=0,\n", - " reg_alpha=0, reg_lambda=1, scale_pos_weight=1, subsample=1,\n", - " tree_method='exact', validate_parameters=1, verbosity=None)" - ] - }, - "metadata": {}, - "execution_count": 21 - } - ], - "source": [ - "xgb.fit(X_train, y_train)" - ] - }, - { - "cell_type": "code", - "execution_count": 22, - "metadata": {}, - "outputs": [ - { - "output_type": "stream", - "name": "stdout", - "text": [ - "default xgboost accuracy = 0.6676060098186078\n" - ] - } - ], - "source": [ - "y_pred = xgb.predict(X_test)\n", - "from flaml.ml import sklearn_metric_loss_score\n", - "print('default xgboost accuracy', '=', 1 - sklearn_metric_loss_score('accuracy', y_pred, y_test))" - ] - } - ], - "metadata": { - "kernelspec": { - "name": "python3", - "display_name": "Python 3", - "language": "python" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.7.7-final" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} \ No newline at end of file diff --git a/notebook/finetune_transformer_demo.ipynb b/notebook/flaml_finetune_transformer.ipynb similarity index 98% rename from notebook/finetune_transformer_demo.ipynb rename to notebook/flaml_finetune_transformer.ipynb index a7254516e..9bea4ef61 100644 --- a/notebook/finetune_transformer_demo.ipynb +++ b/notebook/flaml_finetune_transformer.ipynb @@ -4,7 +4,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This notebook uses the Huggingface transformers library to finetune a transformer model.\n", + "This notebook uses flaml to finetune a transformer model from Huggingface transformers library.\n", "\n", "**Requirements.** This notebook has additional requirements:" ] @@ -673,12 +673,14 @@ "metadata": {}, "outputs": [], "source": [ - "max_num_epoch = 4\n", + "max_num_epoch = 64\n", "search_space = {\n", " # You can mix constants with search space objects.\n", " \"num_train_epochs\": flaml.tune.loguniform(1, max_num_epoch),\n", " \"learning_rate\": flaml.tune.loguniform(1e-6, 1e-4),\n", " \"adam_epsilon\": flaml.tune.loguniform(1e-9, 1e-7),\n", + " \"adam_beta1\": flaml.tune.uniform(0.8, 0.99),\n", + " \"adam_beta2\": flaml.tune.loguniform(98e-2, 9999e-4),\n", " }" ] }, @@ -692,12 +694,12 @@ "HP_METRIC, MODE = \"matthews_correlation\", \"max\"\n", "\n", "# resources\n", - "num_cpus = 2\n", - "num_gpus = 2\n", + "num_cpus = 4\n", + "num_gpus = 4\n", "\n", "# constraints\n", "num_samples = -1 # number of trials, -1 means unlimited\n", - "time_budget_s = 3600 # time budget in seconds" + "time_budget_s = 10800 # time budget in seconds" ] }, { diff --git a/notebook/flaml_lightgbm.ipynb b/notebook/flaml_lightgbm.ipynb new file mode 100644 index 000000000..a9fce7ae2 --- /dev/null +++ b/notebook/flaml_lightgbm.ipynb @@ -0,0 +1,649 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "Copyright (c) 2020-2021 Microsoft Corporation. All rights reserved. \n", + "\n", + "Licensed under the MIT License.\n", + "\n", + "# Tune LightGBM with FLAML Library\n", + "\n", + "\n", + "## 1. Introduction\n", + "\n", + "FLAML is a Python library (https://github.com/microsoft/FLAML) designed to automatically produce accurate machine learning models \n", + "with low computational cost. It is fast and cheap. The simple and lightweight design makes it easy \n", + "to use and extend, such as adding new learners. FLAML can \n", + "- serve as an economical AutoML engine,\n", + "- be used as a fast hyperparameter tuning tool, or \n", + "- be embedded in self-tuning software that requires low latency & resource in repetitive\n", + " tuning tasks.\n", + "\n", + "In this notebook, we demonstrate how to use FLAML library to tune hyperparameters of LightGBM with a regression example.\n", + "\n", + "FLAML requires `Python>=3.6`. To run this notebook example, please install flaml with the `notebook` option:\n", + "```bash\n", + "pip install flaml[notebook]\n", + "```" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install flaml[notebook];" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "## 2. Regression Example\n", + "### Load data and preprocess\n", + "\n", + "Download [houses dataset](https://www.openml.org/d/537) from OpenML. The task is to predict median price of the house in the region based on demographic composition and a state of housing market in the region." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "load dataset from ./openml_ds537.pkl\nDataset name: houses\nX_train.shape: (15480, 8), y_train.shape: (15480,);\nX_test.shape: (5160, 8), y_test.shape: (5160,)\n" + ] + } + ], + "source": [ + "from flaml.data import load_openml_dataset\n", + "X_train, X_test, y_train, y_test = load_openml_dataset(dataset_id = 537, data_dir = './')" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Run FLAML\n", + "In the FLAML automl run configuration, users can specify the task type, time budget, error metric, learner list, whether to subsample, resampling strategy type, and so on. All these arguments have default values which will be used if users do not provide them. " + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "''' import AutoML class from flaml package '''\n", + "from flaml import AutoML\n", + "automl = AutoML()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "settings = {\n", + " \"time_budget\": 120, # total running time in seconds\n", + " \"metric\": 'r2', # primary metrics for regression can be chosen from: ['mae','mse','r2']\n", + " \"estimator_list\": ['lgbm'], # list of ML learners; we tune lightgbm in this example\n", + " \"task\": 'regression', # task type \n", + " \"log_file_name\": 'houses_experiment.log', # flaml log file\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "[flaml.automl: 02-22 14:37:41] {844} INFO - Evaluation method: cv\n", + "[flaml.automl: 02-22 14:37:41] {573} INFO - Using RepeatedKFold\n", + "[flaml.automl: 02-22 14:37:41] {865} INFO - Minimizing error metric: 1-r2\n", + "[flaml.automl: 02-22 14:37:41] {885} INFO - List of ML learners in AutoML Run: ['lgbm']\n", + "[flaml.automl: 02-22 14:37:41] {944} INFO - iteration 0 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:41] {1098} INFO - at 0.2s,\tbest lgbm's error=0.7383,\tbest lgbm's error=0.7383\n", + "[flaml.automl: 02-22 14:37:41] {944} INFO - iteration 1 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:41] {1098} INFO - at 0.3s,\tbest lgbm's error=0.7383,\tbest lgbm's error=0.7383\n", + "[flaml.automl: 02-22 14:37:41] {944} INFO - iteration 2 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:41] {1098} INFO - at 0.4s,\tbest lgbm's error=0.4578,\tbest lgbm's error=0.4578\n", + "[flaml.automl: 02-22 14:37:41] {944} INFO - iteration 3 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:41] {1098} INFO - at 0.5s,\tbest lgbm's error=0.4578,\tbest lgbm's error=0.4578\n", + "[flaml.automl: 02-22 14:37:41] {944} INFO - iteration 4 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:42] {1098} INFO - at 0.9s,\tbest lgbm's error=0.2637,\tbest lgbm's error=0.2637\n", + "[flaml.automl: 02-22 14:37:42] {944} INFO - iteration 5 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:42] {1098} INFO - at 1.2s,\tbest lgbm's error=0.2284,\tbest lgbm's error=0.2284\n", + "[flaml.automl: 02-22 14:37:42] {944} INFO - iteration 6 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:42] {1098} INFO - at 1.3s,\tbest lgbm's error=0.2284,\tbest lgbm's error=0.2284\n", + "[flaml.automl: 02-22 14:37:42] {944} INFO - iteration 7 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:42] {1098} INFO - at 1.6s,\tbest lgbm's error=0.2284,\tbest lgbm's error=0.2284\n", + "[flaml.automl: 02-22 14:37:42] {944} INFO - iteration 8 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:42] {1098} INFO - at 1.8s,\tbest lgbm's error=0.2284,\tbest lgbm's error=0.2284\n", + "[flaml.automl: 02-22 14:37:42] {944} INFO - iteration 9 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:43] {1098} INFO - at 1.9s,\tbest lgbm's error=0.2284,\tbest lgbm's error=0.2284\n", + "[flaml.automl: 02-22 14:37:43] {944} INFO - iteration 10 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:43] {1098} INFO - at 2.1s,\tbest lgbm's error=0.2284,\tbest lgbm's error=0.2284\n", + "[flaml.automl: 02-22 14:37:43] {944} INFO - iteration 11 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:43] {1098} INFO - at 2.6s,\tbest lgbm's error=0.2262,\tbest lgbm's error=0.2262\n", + "[flaml.automl: 02-22 14:37:43] {944} INFO - iteration 12 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:44] {1098} INFO - at 3.7s,\tbest lgbm's error=0.2009,\tbest lgbm's error=0.2009\n", + "[flaml.automl: 02-22 14:37:44] {944} INFO - iteration 13 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:45] {1098} INFO - at 3.9s,\tbest lgbm's error=0.2009,\tbest lgbm's error=0.2009\n", + "[flaml.automl: 02-22 14:37:45] {944} INFO - iteration 14 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:47] {1098} INFO - at 6.0s,\tbest lgbm's error=0.1854,\tbest lgbm's error=0.1854\n", + "[flaml.automl: 02-22 14:37:47] {944} INFO - iteration 15 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:48] {1098} INFO - at 7.0s,\tbest lgbm's error=0.1854,\tbest lgbm's error=0.1854\n", + "[flaml.automl: 02-22 14:37:48] {944} INFO - iteration 16 current learner lgbm\n", + "[flaml.automl: 02-22 14:37:59] {1098} INFO - at 18.0s,\tbest lgbm's error=0.1761,\tbest lgbm's error=0.1761\n", + "[flaml.automl: 02-22 14:37:59] {944} INFO - iteration 17 current learner lgbm\n", + "[flaml.automl: 02-22 14:38:35] {1098} INFO - at 53.9s,\tbest lgbm's error=0.1725,\tbest lgbm's error=0.1725\n", + "[flaml.automl: 02-22 14:38:35] {944} INFO - iteration 18 current learner lgbm\n", + "[flaml.automl: 02-22 14:39:10] {1098} INFO - at 88.9s,\tbest lgbm's error=0.1725,\tbest lgbm's error=0.1725\n", + "[flaml.automl: 02-22 14:39:10] {944} INFO - iteration 19 current learner lgbm\n", + "[flaml.automl: 02-22 14:39:14] {1098} INFO - at 92.9s,\tbest lgbm's error=0.1725,\tbest lgbm's error=0.1725\n", + "[flaml.automl: 02-22 14:39:14] {944} INFO - iteration 20 current learner lgbm\n", + "[flaml.automl: 02-22 14:39:31] {1098} INFO - at 110.7s,\tbest lgbm's error=0.1563,\tbest lgbm's error=0.1563\n", + "[flaml.automl: 02-22 14:39:31] {1139} INFO - selected model: LGBMRegressor(colsample_bytree=0.9046814915274195,\n", + " learning_rate=0.025065630491840726, max_bin=255,\n", + " min_child_weight=20.0, n_estimators=451, num_leaves=113,\n", + " objective='regression', reg_alpha=8.352751749829367e-10,\n", + " reg_lambda=0.13991138691596908)\n", + "[flaml.automl: 02-22 14:39:31] {899} INFO - fit succeeded\n" + ] + } + ], + "source": [ + "'''The main flaml automl API'''\n", + "automl.fit(X_train = X_train, y_train = y_train, **settings)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "### Best model and metric" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Best hyperparmeter config: {'n_estimators': 451.0, 'max_leaves': 113.0, 'min_child_weight': 20.0, 'learning_rate': 0.025065630491840726, 'subsample': 1.0, 'log_max_bin': 8.0, 'colsample_bytree': 0.9046814915274195, 'reg_alpha': 8.352751749829367e-10, 'reg_lambda': 0.13991138691596908}\nBest r2 on validation data: 0.8437\nTraining duration of best run: 17.86 s\n" + ] + } + ], + "source": [ + "''' retrieve best config'''\n", + "print('Best hyperparmeter config:', automl.best_config)\n", + "print('Best r2 on validation data: {0:.4g}'.format(1-automl.best_loss))\n", + "print('Training duration of best run: {0:.4g} s'.format(automl.best_config_train_time))" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "LGBMRegressor(colsample_bytree=0.9046814915274195,\n", + " learning_rate=0.025065630491840726, max_bin=255,\n", + " min_child_weight=20.0, n_estimators=451, num_leaves=113,\n", + " objective='regression', reg_alpha=8.352751749829367e-10,\n", + " reg_lambda=0.13991138691596908)" + ] + }, + "metadata": {}, + "execution_count": 10 + } + ], + "source": [ + "automl.model" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [], + "source": [ + "''' pickle and save the best model '''\n", + "import pickle\n", + "with open('best_model.pkl', 'wb') as f:\n", + " pickle.dump(automl.model, f, pickle.HIGHEST_PROTOCOL)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Predicted labels [147056.672508 246591.18821626 155253.69332074 ... 196516.76693923\n 235571.37776252 270133.77185961]\nTrue labels [136900. 241300. 200700. ... 160900. 227300. 265600.]\n" + ] + } + ], + "source": [ + "''' compute predictions of testing dataset ''' \n", + "y_pred = automl.predict(X_test)\n", + "print('Predicted labels', y_pred)\n", + "print('True labels', y_test)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": { + "slideshow": { + "slide_type": "slide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "r2 = 0.8503723727607084\nmse = 1977853769.4384706\nmae = 29258.487121555943\n" + ] + } + ], + "source": [ + "''' compute different metric values on testing dataset'''\n", + "from flaml.ml import sklearn_metric_loss_score\n", + "print('r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))\n", + "print('mse', '=', sklearn_metric_loss_score('mse', y_pred, y_test))\n", + "print('mae', '=', sklearn_metric_loss_score('mae', y_pred, y_test))" + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": { + "slideshow": { + "slide_type": "subslide" + }, + "tags": [] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4, 'max_leaves': 4, 'min_child_weight': 20.0, 'learning_rate': 0.1, 'subsample': 1.0, 'log_max_bin': 8, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 4.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 0.46335414315327306, 'subsample': 0.9339389930838808, 'log_max_bin': 10.0, 'colsample_bytree': 0.9904286645657556, 'reg_alpha': 2.841147337412889e-10, 'reg_lambda': 0.12000833497054482}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 20.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 20.0, 'max_leaves': 4.0, 'min_child_weight': 20.0, 'learning_rate': 1.0, 'subsample': 0.9917683183663918, 'log_max_bin': 10.0, 'colsample_bytree': 0.9858892907525497, 'reg_alpha': 3.8783982645515837e-10, 'reg_lambda': 0.36607431863072826}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 15.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 11.0, 'max_leaves': 15.0, 'min_child_weight': 14.947587304572773, 'learning_rate': 0.6092558236172073, 'subsample': 0.9659256891661986, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 3.816590663384559e-08, 'reg_lambda': 0.4482946615262561}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 22.0, 'max_leaves': 44.0, 'min_child_weight': 8.295709769360025, 'learning_rate': 0.8096645680737932, 'subsample': 0.9506809897636022, 'log_max_bin': 9.0, 'colsample_bytree': 0.9671874874371171, 'reg_alpha': 1.7301741960564346e-06, 'reg_lambda': 0.0977230117487556}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 22.0, 'max_leaves': 44.0, 'min_child_weight': 8.295709769360025, 'learning_rate': 0.8096645680737932, 'subsample': 0.9506809897636022, 'log_max_bin': 9.0, 'colsample_bytree': 0.9671874874371171, 'reg_alpha': 1.7301741960564346e-06, 'reg_lambda': 0.0977230117487556}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 9.0, 'max_leaves': 245.0, 'min_child_weight': 4.208492943400939, 'learning_rate': 0.26609407333531715, 'subsample': 0.9704154473613615, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 1.6843847487782941e-09, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 9.0, 'max_leaves': 245.0, 'min_child_weight': 4.208492943400939, 'learning_rate': 0.26609407333531715, 'subsample': 0.9704154473613615, 'log_max_bin': 10.0, 'colsample_bytree': 1.0, 'reg_alpha': 1.6843847487782941e-09, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 25.0, 'max_leaves': 403.0, 'min_child_weight': 5.55091634143324, 'learning_rate': 0.12666913709918798, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 5.406309953251589e-09, 'reg_lambda': 0.8927146483558472}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 25.0, 'max_leaves': 403.0, 'min_child_weight': 5.55091634143324, 'learning_rate': 0.12666913709918798, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 5.406309953251589e-09, 'reg_lambda': 0.8927146483558472}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 78.0, 'max_leaves': 2174.0, 'min_child_weight': 13.317463737456045, 'learning_rate': 0.06563935802336449, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 78.0, 'max_leaves': 2174.0, 'min_child_weight': 13.317463737456045, 'learning_rate': 0.06563935802336449, 'subsample': 1.0, 'log_max_bin': 9.0, 'colsample_bytree': 1.0, 'reg_alpha': 1e-10, 'reg_lambda': 1.0}}\n{'Current Learner': 'lgbm', 'Current Sample': 15480, 'Current Hyper-parameters': {'n_estimators': 200.0, 'max_leaves': 373.0, 'min_child_weight': 20.0, 'learning_rate': 0.05367311643039711, 'subsample': 0.9992325198304696, 'log_max_bin': 10.0, 'colsample_bytree': 0.957244844266689, 'reg_alpha': 1.4054731493830395e-08, 'reg_lambda': 1.0}, 'Best Learner': 'lgbm', 'Best Hyper-parameters': {'n_estimators': 200.0, 'max_leaves': 373.0, 'min_child_weight': 20.0, 'learning_rate': 0.05367311643039711, 'subsample': 0.9992325198304696, 'log_max_bin': 10.0, 'colsample_bytree': 0.957244844266689, 'reg_alpha': 1.4054731493830395e-08, 'reg_lambda': 1.0}}\n" + ] + } + ], + "source": [ + "from flaml.data import get_output_from_log\n", + "time_history, best_valid_loss_history, valid_loss_history, config_history, train_loss_history = \\\n", + " get_output_from_log(filename = settings['log_file_name'], time_budget = 60)\n", + "\n", + "for config in config_history:\n", + " print(config)" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "outputs": [ + { + "output_type": "display_data", + "data": { + "text/plain": "
", + "image/svg+xml": "\r\n\r\n\r\n\r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n \r\n\r\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEWCAYAAABrDZDcAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8GearUAAAcUElEQVR4nO3dfbxVZZ338c+XIyjmAxHY6AGEkvAhFRJtLJvUyUDTwHRMnYeGXhPaZNOME4bO2JO39+hw10zdWtzoOGTjsyJikWia2jiWYCBPSkNocsAUclDCkwj87j/WOrrY7L3P5pyz9j5nr+/79Tqvs9e1rr33b/Gwv3tda61rKSIwM7Pi6tfoAszMrLEcBGZmBecgMDMrOAeBmVnBOQjMzArOQWBmVnAOArMqJH1I0qpG12GWJweB9VqSnpP0kUbWEBE/jYgxeb2+pAmSHpW0WdIGSY9I+nhe72dWjoPACk1SSwPf+2zgDuBGYBjwTuDLwBldeC1J8v9n6xL/w7E+R1I/SdMl/UrSbyXdLmlwZv0dkn4j6ZX02/YRmXWzJX1X0nxJW4CT0j2PL0pamj7nNkl7pf1PlNSWeX7Fvun6SyS9IGm9pL+SFJIOKbMNAr4JXBER10fEKxGxIyIeiYjPpH2+Kuk/Ms8Zmb7eHunyw5KulPQY8BpwmaRFJe/zd5LmpY/3lPR/JD0v6UVJMyUN7OZfhzUBB4H1RX8DTAY+DBwE/A9wbWb9j4DRwAHAL4CbSp5/PnAlsC/wn2nbOcBEYBRwFPCXVd6/bF9JE4GLgY8Ah6T1VTIGGA7cWaVPLf4cmEqyLf8XGCNpdGb9+cDN6eOrgfcAY9P6Wkn2QKzgHATWF10A/ENEtEXE68BXgbM7vilHxA0RsTmz7mhJ+2eef09EPJZ+A/992vbtiFgfES8D95J8WFZSqe85wL9HxIqIeA34WpXXeEf6+4Wat7q82en7bYuIV4B7gPMA0kA4FJiX7oF8Bvi7iHg5IjYD/xs4t5vvb03AQWB90cHA3ZI2SdoEPA1sB94pqUXSVemw0avAc+lzhmSev7bMa/4m8/g1YJ8q71+p70Elr13ufTr8Nv19YJU+tSh9j5tJg4Bkb2BuGkpDgb2BJzN/bvel7VZwDgLri9YCp0bEoMzPXhGxjuTDbxLJ8Mz+wMj0Oco8P68pd18gOejbYXiVvqtItuOsKn22kHx4d/iDMn1Kt+V+YIiksSSB0DEstBFoB47I/JntHxHVAs8KwkFgvV1/SXtlfvYAZgJXSjoYQNJQSZPS/vsCr5N8496bZPijXm4Hpkg6TNLeVBl/j2T+94uByyVNkbRfehD8BEmz0m5LgD+SNCId2rq0swIiYhvJcYcZwGDggbR9B3Ad8C+SDgCQ1CppQpe31pqGg8B6u/kk32Q7fr4KfAuYB9wvaTPwM+D9af8bgV8D64CV6bq6iIgfAd8GfgKsBh5PV71eof+dwCeBTwPrgReB/0Uyzk9EPADcBiwFngR+UGMpN5PsEd2RBkOHL6V1/SwdNvsxyUFrKzj5xjRm+ZB0GLAc2LPkA9msV/EegVkPknSmpAGS3k5yuua9DgHr7RwEZj3rAmAD8CuSM5k+29hyzDrnoSEzs4LzHoGZWcHt0egCdteQIUNi5MiRjS7DzKxPefLJJzdGRNkLCPtcEIwcOZJFixZ13tHMzN4k6deV1nloyMys4BwEZmYF5yAwMys4B4GZWcE5CMzMCq7PnTVkZlY0cxevY8aCVazf1M5BgwYybcIYJo9r7bHXdxCYmfVicxev49I5y2h/YzsA6za1c+mcZQA9FgYOAmuYvL/lmDWDGQtWvRkCHdrf2M6MBascBNa31eNbjlkzWL+pfbfau8JBYA1R6VvOJXcu5ZYnnm9QVWa9T/+WfmzdvmOX9oMGDeyx93AQ1ImHQXZW6dtMuX/wZkU2fPBAnt24hR2ZiaIH9m9h2oSeu7mcg6AOPAyyq4MGDWRdmTBoHTSQ2y44vgEVmfVePmuoCXgYZFd79e9HP5HrtxyzZjF5XGuuXxodBHXgYZBdDdlnTwDWvtzO1u07aPVwmVnDOAjqwMMgZtab5TrFhKSJklZJWi1pepn1+0u6V9JTklZImpJnPT1l7uJ1fPCqhxg1/Yd88KqHmLt4XdX+0yaMYWD/lp3aPAxiZr1FbnsEklqAa4FTgDZgoaR5EbEy0+1zwMqIOEPSUGCVpJsiYmtedXVXVw78drRfcudSD4OYWa+T59DQccDqiFgDIOlWYBKQDYIA9pUkYB/gZWBbjjV1W3cO/O7Zvx/jRgzycJCZ9Sp5Dg21Amszy21pW9Y1wGHAemAZ8IWI2OUIqqSpkhZJWrRhw4a86q1Jdw78Hn7gfkwa670AM+td8twjUJm2KFmeACwBTgbeDTwg6acR8epOT4qYBcwCGD9+fOlr1JUP/JpZs8lzj6ANGJ5ZHkbyzT9rCjAnEquBZ4FDc6yp23zg18yaTZ5BsBAYLWmUpAHAucC8kj7PA38MIOmdwBhgTY41ddvkca380yeOZEBL8kfXOmgg//SJI33g18z6rNyGhiJim6SLgAVAC3BDRKyQdGG6fiZwBTBb0jKSoaQvRcTGvGrqKZPHtb55YNjDQWbW1+V6QVlEzAfml7TNzDxeD3w0zxrMzKw6X1lchWcMNbMiKHQQVPug94yhZlYUhQ2Czj7oO7twbOULr3L4gfvVvW4zs55W2CDo7IO+3LUC8NaFY744zMyaRWGDoLMrhAdUuD2cLxwzs2aT6+yjvVml+312fND/89lH+cIxMyuEwgZBZ1cId1w41jpoIMIXjplZ8yrs0FAtU0PnfXs4M7PeoLBBAL5C2MwMCjw0ZGZmCQeBmVnBOQjMzArOQWBmVnCFPFicnWOof0s/hg8uf02BmVkRFG6PoGOOoXWb2gmSK4mf3biFuYvXNbo0M7OGKFwQlJtjaEck7WZmRVS4IKg0x1CldjOzZle4IKg0x1CldjOzZle4IOhsjiEzs6Ip3FlDtcwxZGZWJIULAvAcQ2ZmWYUbGjIzs505CMzMCs5BYGZWcA4CM7OCcxCYmRVcrkEgaaKkVZJWS5peZv00SUvSn+WStksanGdNZma2s9yCQFILcC1wKnA4cJ6kw7N9ImJGRIyNiLHApcAjEfFyXjWZmdmu8twjOA5YHRFrImIrcCswqUr/84BbcqzHzMzKyDMIWoG1meW2tG0XkvYGJgJ3VVg/VdIiSYs2bNjQ44WamRVZnkGgMm1Roe8ZwGOVhoUiYlZEjI+I8UOHDu2xAs3MLN8gaAOGZ5aHAesr9D0XDwuZmTVEnkGwEBgtaZSkASQf9vNKO0naH/gwcE+OtZiZWQW5TToXEdskXQQsAFqAGyJihaQL0/Uz065nAvdHxJa8ajEzs8pynX00IuYD80vaZpYszwZm51mHmZlV5iuLzcwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBecgMDMrOAeBmVnBOQjMzArOQWBmVnAOAjOzgqsaBJL2k/TuMu1H5VeSmZnVU8UgkHQO8Axwl6QVko7NrJ6dd2FmZlYf1fYILgOOiYixwBTg+5I+ka5T7pWZmVld7FFlXUtEvAAQEU9IOgn4gaRhQNSlOjMzy121PYLN2eMDaSicCEwCjsi5LjMzq5NqewSfpWQIKCI2S5oInJNrVWZmVjcV9wgi4ingWUk/Lml/IyJuyr0yMzOri6qnj0bEduA1Sft35cUlTZS0StJqSdMr9DlR0pL0zKRHuvI+ZmbWddWGhjr8Hlgm6QFgS0djRPxNtSdJagGuBU4B2oCFkuZFxMpMn0HAd4CJEfG8pAO6sA1mZtYNtQTBD9Of3XUcsDoi1gBIupXkQPPKTJ/zgTkR8TxARLzUhfcxM7Nu6DQIIuJ7XXztVmBtZrkNeH9Jn/cA/SU9DOwLfCsibix9IUlTgakAI0aM6GI5ZmZWTp5zDZW76Kz0+oM9gGOAjwETgMslvWeXJ0XMiojxETF+6NChPV+pmVmB1TI01FVtwPDM8jBgfZk+GyNiC7BF0qPA0cAvc6zLzMwy8twjWAiMljRK0gDgXGBeSZ97gA9J2kPS3iRDR0/nWJOZmZXodI8gHaqZBhyc7R8RJ1d7XkRsk3QRsABoAW6IiBWSLkzXz4yIpyXdBywFdgDXR8TyLm+NmZnttlqGhu4AZgLXAdt358UjYj4wv6RtZsnyDGDG7ryumZn1nFqCYFtEfDf3SszMrCFqOUZwr6S/lnSgpMEdP7lXZmZmdVHLHsGn0t/TMm0BvKvnyzEzs3qr5YKyUfUoxMzMGqOWs4b6k0xJ/Udp08PA/4uIN3Ksy8zM6qSWoaHvAv1JJocD+PO07a/yKsrMzOqnliA4NiKOziw/JOmpvAoyM7P6quWsoe3ZW1ZKehe7eT2BmZn1XrXsEUwDfiJpDclEcgcDU3KtyszM6qaWs4YelDQaGEMSBM9ExOu5V2ZmZnVRMQgknRwRD0n6RMmqd0siIubkXJuZmdVBtT2CDwMPAWeUWRdAnwuCuYvXMWPBKtZvaqd/Sz+GDx7Y6JLMzBquYhBExFfSh1+PiGez6yT1uYvM5i5ex6VzltH+RnKce+v2HTy7cQtzF69j8rjWBldnZtY4tZw1dFeZtjt7upC8zViw6s0Q6LAjknYzsyKrdozgUOAIYP+S4wT7AXvlXVhPW7+pfbfazcyKotoxgjHA6cAgdj5OsBn4TJ5F5eGgQQNZV+ZD/6BBPk5gZsVW7RjBPcA9ko6PiMfrWFMupk0Ys9MxAoCB/VuYNmFMA6syM2u8Wi4oWyzpcyTDRG8OCUXEp3OrKgcdB4QvuXMpW7fvoHXQQKZNGOMDxWZWeLUEwfeBZ4AJwNeBP6WP3mB+8rhWbnnieQBuu+D4BldjZtY71HLW0CERcTmwJSK+B3wMODLfsszMrF5qCYKO+w5skvReYH9gZG4VmZlZXdUyNDRL0tuBy4F5wD7Al3OtyszM6qaWSeeuTx8+gu9TbGbWdKpdUHZxtSdGxDd7vhwzM6u3ansE+6a/xwDHkgwLQXJx2aN5FmVmZvVT7YKyrwFIuh94X0RsTpe/CtxRl+rMzCx3tZw1NALYmlneSo1nDUmaKGmVpNWSppdZf6KkVyQtSX98ENrMrM5qvaDsCUl3k9yH4Ezgxs6eJKkFuBY4BWgDFkqaFxErS7r+NCJO372yzcysp9Ry1tCVkn4EfChtmhIRi2t47eOA1RGxBkDSrcAkoDQIzMysgaqdNbRfRLwqaTDwXPrTsW5wRLzcyWu3Amszy23A+8v0O17SU8B64IsRsaJMLVOBqQAjRozo5G3NzGx3VNsjuJlkGuonSYaEOihd7uyaApVpi5LlXwAHR8TvJJ0GzAVG7/KkiFnALIDx48eXvoaZmXVDtbOGTk9/d/W2lG3A8MzyMJJv/dn3eDXzeL6k70gaEhEbu/ieZma2m6oNDb2v2hMj4hedvPZCYHR6f+N1wLnA+SXv8QfAixERko4jOYvpt7UUbmZmPaPa0NA3qqwL4ORqLxwR2yRdBCwAWoAbImKFpAvT9TOBs4HPStoGtAPnRoSHfszM6qja0NBJ3X3xiJgPzC9pm5l5fA1wTXffx8zMuq6W6whIp58+nJ3vUNbptQRmZtb7dRoEkr4CnEgSBPOBU4H/pIaLyszMrPerZYqJs4E/Bn4TEVOAo4E9c63KzMzqppYgaI+IHcA2SfsBL+H7EpiZNY1ajhEskjQIuI7k4rLfAU/kWpWZmdVNtesIrgFujoi/TptmSroP2C8iltalOjMzy121PYL/Br4h6UDgNuCWiFhSn7LMzKxeKh4jiIhvRcTxwIeBl4F/l/S0pC9Lek/dKjQzs1x1erA4In4dEVdHxDiSKSLOBJ7OvTIzM6uLToNAUn9JZ0i6CfgR8EvgrNwrMzOzuqh2sPgU4DzgYyRnCd0KTI2ILXWqzczM6qDaweLLSO5J8MUabkJjZmZ9VK6TzpmZWe9Xy5XFZmbWxBwEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBecgMDMrOAeBmVnB5RoEkiZKWiVptaTpVfodK2m7pLPzrMfMzHaVWxBIagGuBU4FDgfOk3R4hX5XAwvyqsXMzCrLc4/gOGB1RKyJiK0kN7aZVKbf54G7gJdyrMXMzCrIMwhagbWZ5ba07U2SWknugTwzxzrMzKyKPINAZdqiZPlfgS9FxPaqLyRNlbRI0qINGzb0WIFmZlb9VpXd1QYMzywPA9aX9BkP3CoJYAhwmqRtETE32ykiZgGzAMaPH18aJmZm1g15BsFCYLSkUcA64Fzg/GyHiBjV8VjSbOAHpSFgZmb5yi0IImKbpItIzgZqAW6IiBWSLkzX+7iAmVkvkOceARExH5hf0lY2ACLiL/OsxczMyvOVxWZmBecgMDMrOAeBmVnBOQjMzArOQWBmVnAOAjOzgnMQmJkVnIPAzKzgHARmZgXnIDAzKzgHgZlZwTkIzMwKzkFgZlZwDgIzs4JzEJiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcE5CMzMCs5BYGZWcA4CM7OCcxCYmRWcg8DMrOAcBGZmBecgMDMruFyDQNJESaskrZY0vcz6SZKWSloiaZGkE/Ksx8zMdrVHXi8sqQW4FjgFaAMWSpoXESsz3R4E5kVESDoKuB04NK+azMxsV3nuERwHrI6INRGxFbgVmJTtEBG/i4hIF98GBGZmVld5BkErsDaz3Ja27UTSmZKeAX4IfDrHeszMrIw8g0Bl2nb5xh8Rd0fEocBk4IqyLyRNTY8hLNqwYUMPl2lmVmx5BkEbMDyzPAxYX6lzRDwKvFvSkDLrZkXE+IgYP3To0J6v1MyswPIMgoXAaEmjJA0AzgXmZTtIOkSS0sfvAwYAv82xJjMzK5HbWUMRsU3SRcACoAW4ISJWSLowXT8TOAv4C0lvAO3AJzMHj83MrA5yCwKAiJgPzC9pm5l5fDVwdZ41mJlZdb6y2Mys4BwEZmYF5yAwMys4B4GZWcE5CMzMCi7Xs4Z6i7mL1zFjwSrWb2qnf0s/hg8e2OiSzMx6jabfI5i7eB2XzlnGuk3tBLB1+w6e3biFuYvXNbo0M7NeoemDYMaCVbS/sX2nth2RtJuZWQGCYP2m9t1qNzMrmqYPgoMGlT8eUKndzKxomj4Ipk0Yw8D+LTu1DezfwrQJYxpUkZlZ79L0Zw1NHpfcC6fjrKGDBg1k2oQxb7abmRVd0wcBJGHgD34zs/KafmjIzMyqcxCYmRWcg8DMrOAcBGZmBecgMDMrOPW1WwRL2gD8uotPHwJs7MFyeptm3z5o/m1s9u0Db2OjHBwRQ8ut6HNB0B2SFkXE+EbXkZdm3z5o/m1s9u0Db2Nv5KEhM7OCcxCYmRVc0YJgVqMLyFmzbx80/zY2+/aBt7HXKdQxAjMz21XR9gjMzKyEg8DMrOAKEQSSJkpaJWm1pOmNrqcnSLpB0kuSlmfaBkt6QNJ/p7/f3sgau0PScEk/kfS0pBWSvpC2N9M27iXpCUlPpdv4tbS9abYRQFKLpMWSfpAuN9v2PSdpmaQlkhalbX1qG5s+CCS1ANcCpwKHA+dJOryxVfWI2cDEkrbpwIMRMRp4MF3uq7YBfx8RhwF/CHwu/Xtrpm18HTg5Io4GxgITJf0hzbWNAF8Ans4sN9v2AZwUEWMz1w70qW1s+iAAjgNWR8SaiNgK3ApManBN3RYRjwIvlzRPAr6XPv4eMLmuRfWgiHghIn6RPt5M8kHSSnNtY0TE79LF/ulP0ETbKGkY8DHg+kxz02xfFX1qG4sQBK3A2sxyW9rWjN4ZES9A8kEKHNDgenqEpJHAOODnNNk2psMmS4CXgAciotm28V+BS4AdmbZm2j5Iwvt+SU9Kmpq29altLMIdylSmzefM9hGS9gHuAv42Il6Vyv119l0RsR0YK2kQcLek9za6pp4i6XTgpYh4UtKJja4nRx+MiPWSDgAekPRMowvaXUXYI2gDhmeWhwHrG1RL3l6UdCBA+vulBtfTLZL6k4TATRExJ21uqm3sEBGbgIdJjvs0yzZ+EPi4pOdIhmRPlvQfNM/2ARAR69PfLwF3kwxH96ltLEIQLARGSxolaQBwLjCvwTXlZR7wqfTxp4B7GlhLtyj56v9vwNMR8c3MqmbaxqHpngCSBgIfAZ6hSbYxIi6NiGERMZLk/91DEfFnNMn2AUh6m6R9Ox4DHwWW08e2sRBXFks6jWSssgW4ISKubHBJ3SbpFuBEkuluXwS+AswFbgdGAM8DfxIRpQeU+wRJJwA/BZbx1vjyZSTHCZplG48iOZDYQvKl7PaI+Lqkd9Ak29ghHRr6YkSc3kzbJ+ldJHsBkAy13xwRV/a1bSxEEJiZWWVFGBoyM7MqHARmZgXnIDAzKzgHgZlZwTkIzMwKzkFgvYqkf5H0t5nlBZKuzyx/Q9LFVZ4/W9LZ6eOHJe1yA3FJ/SVdlc4MuTydAfTUdN1zkoZ0oe4337fC+mvT2SlXSmpPHy+RdLak+R3XE/QkSQd2zPhZYf0ASY9KKsIMA1aFg8B6m/8CPgAgqR/JdRJHZNZ/AHism+9xBXAg8N6IeC9wBrBvN1+zqoj4XESMBU4DfpXOVDk2Iu6MiNPSK4t72sXAdVVq2koyM+Ync3hv60McBNbbPEYaBCQBsBzYLOntkvYEDgMWS/qypIXpN/pZqnESIkl7A58BPh8RrwNExIsRcXuZvhenr7+8ZC/lLyQtTe8j8P0yz7si3UOo6f9Xx16IpJGSnpF0ffqeN0n6iKTH0r2X49L+b1NyP4qFSub5rzSb7lnAfelzjkj3fJaktY9O+8wF/rSWOq15eZfQepV08q5tkkaQBMLjJLPFHg+8AiyNiK2SromIrwOkH8anA/fW8BaHAM9HxKvVOkk6BpgCvJ9k4sKfS3oE2Ar8A8lEYxslDS553j8D+wNTomtXax4C/AkwlWR6lPOBE4CPk1xZPTl9/4ci4tPpkNITkn4cEVsydYwC/qcj7IALgW9FxE3pVCstafty4Ngu1GlNxHsE1ht17BV0BMHjmeX/SvucJOnnkpYBJ7Pz8FFPOAG4OyK2pPcMmAN8KH2vOyNiI0DJtAGXA4Mi4oIuhgDAsxGxLCJ2ACtIbm4SJFNtjEz7fBSYrmT66oeBvUimMsg6ENiQWX4cuEzSl4CDI6I9rX87sLVjvhwrJgeB9UYdxwmOJPnG+jOSPYIPAI9J2gv4DnB2RBxJMg6+V42vvRoYUcMHX6WhJlF5GvOFwDGlewm76fXM4x2Z5R28tQcv4KzMcYYREZG9AxhAO5k/k4i4mWSvoh1YIOnkTN89gd93o2br4xwE1hs9RjLU83JEbE+/dQ8iCYPHeesDbqOS+xVUPFunVES8RjKr6bfTIZKOs2v+rKTro8BkSXuns0qeSTIJ3oPAOemkYpR86N8HXAX8MOdv2AuAz3ccF5E0rkyfX/LWHkTH5GhrIuLbJDNjHpW2vwPYEBFv5Fiv9XIOAuuNlpGcLfSzkrZXImJjeobNdWnbXJJv4rvjH0mGTVZKWp6+RnYYhfQ2mbOBJ0hmPL0+IhZHxArgSuARSU8B3yx53h1pbfPSqaXzcAXJbS2XpvVfUdohPV7wK0mHpE2fBJanw0mHAjem7ScB83Oq0/oIzz5q1qQknQkcExH/WKXPHODSiFhVv8qst/FZQ2ZNKiLu7hjCKicdGpvrEDDvEZiZFZyPEZiZFZyDwMys4BwEZmYF5yAwMys4B4GZWcH9f9JltgbZ0PHUAAAAAElFTkSuQmCC\n" + }, + "metadata": { + "needs_background": "light" + } + } + ], + "source": [ + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "plt.title('Learning Curve')\n", + "plt.xlabel('Wall Clock Time (s)')\n", + "plt.ylabel('Validation r2')\n", + "plt.scatter(time_history, 1-np.array(valid_loss_history))\n", + "plt.step(time_history, 1-np.array(best_valid_loss_history), where='post')\n", + "plt.show()" + ] + }, + { + "source": [ + "## 3. Comparison with alternatives\n", + "\n", + "### FLAML's accuracy" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "flaml r2 = 0.8503723727607084\n" + ] + } + ], + "source": [ + "print('flaml r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))" + ] + }, + { + "source": [ + "### Default LightGBM" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [], + "source": [ + "from lightgbm import LGBMRegressor\n", + "lgbm = LGBMRegressor()" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "output_type": "execute_result", + "data": { + "text/plain": [ + "LGBMRegressor()" + ] + }, + "metadata": {}, + "execution_count": 18 + } + ], + "source": [ + "lgbm.fit(X_train, y_train)" + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "default lgbm r2 = 0.8296179648694404\n" + ] + } + ], + "source": [ + "y_pred = lgbm.predict(X_test)\n", + "from flaml.ml import sklearn_metric_loss_score\n", + "print('default lgbm r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))" + ] + }, + { + "source": [ + "### Optuna LightGBM Tuner" + ], + "cell_type": "markdown", + "metadata": {} + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install optuna==2.5.0;" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "from sklearn.model_selection import train_test_split\n", + "train_x, val_x, train_y, val_y = train_test_split(X_train, y_train, test_size=0.1)\n", + "import optuna.integration.lightgbm as lgb\n", + "dtrain = lgb.Dataset(train_x, label=train_y)\n", + "dval = lgb.Dataset(val_x, label=val_y)\n", + "params = {\n", + " \"objective\": \"regression\",\n", + " \"metric\": \"regression\",\n", + " \"verbosity\": -1,\n", + "}\n" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": { + "tags": [ + "outputPrepend" + ] + }, + "outputs": [ + { + "output_type": "stream", + "name": "stderr", + "text": [ + "875128.6166067 and parameters: {'feature_fraction': 0.5}. Best is trial 0 with value: 2131729337.83384.\u001b[0m\n", + "feature_fraction, val_score: 1949307059.499325: 43%|####2 | 3/7 [00:06<00:08, 2.23s/it]\u001b[32m[I 2021-02-22 14:39:47,252]\u001b[0m Trial 2 finished with value: 1949307059.499325 and parameters: {'feature_fraction': 0.8999999999999999}. Best is trial 2 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction, val_score: 1949307059.499325: 57%|#####7 | 4/7 [00:09<00:06, 2.26s/it]\u001b[32m[I 2021-02-22 14:39:49,566]\u001b[0m Trial 3 finished with value: 1991236553.1444218 and parameters: {'feature_fraction': 0.7}. Best is trial 2 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction, val_score: 1949307059.499325: 71%|#######1 | 5/7 [00:11<00:04, 2.21s/it]\u001b[32m[I 2021-02-22 14:39:51,687]\u001b[0m Trial 4 finished with value: 1988181425.985298 and parameters: {'feature_fraction': 1.0}. Best is trial 2 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction, val_score: 1949307059.499325: 86%|########5 | 6/7 [00:13<00:02, 2.13s/it]\u001b[32m[I 2021-02-22 14:39:53,649]\u001b[0m Trial 5 finished with value: 1991236553.1444218 and parameters: {'feature_fraction': 0.8}. Best is trial 2 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction, val_score: 1949307059.499325: 100%|##########| 7/7 [00:14<00:00, 2.04s/it]\u001b[32m[I 2021-02-22 14:39:55,505]\u001b[0m Trial 6 finished with value: 1985494931.14108 and parameters: {'feature_fraction': 0.6}. Best is trial 2 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction, val_score: 1949307059.499325: 100%|##########| 7/7 [00:15<00:00, 2.14s/it]\n", + "num_leaves, val_score: 1949307059.499325: 5%|5 | 1/20 [00:00<00:13, 1.37it/s]\u001b[32m[I 2021-02-22 14:39:56,251]\u001b[0m Trial 7 finished with value: 2193886138.5860405 and parameters: {'num_leaves': 5}. Best is trial 7 with value: 2193886138.5860405.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 10%|# | 2/20 [00:10<01:48, 6.04s/it]\u001b[32m[I 2021-02-22 14:40:06,007]\u001b[0m Trial 8 finished with value: 2027098939.2209685 and parameters: {'num_leaves': 200}. Best is trial 8 with value: 2027098939.2209685.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 15%|#5 | 3/20 [00:20<02:10, 7.70s/it]\u001b[32m[I 2021-02-22 14:40:15,684]\u001b[0m Trial 9 finished with value: 2035885641.3675568 and parameters: {'num_leaves': 205}. Best is trial 8 with value: 2027098939.2209685.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 20%|## | 4/20 [00:21<01:21, 5.10s/it]\u001b[32m[I 2021-02-22 14:40:16,785]\u001b[0m Trial 10 finished with value: 2004366052.4067948 and parameters: {'num_leaves': 16}. Best is trial 10 with value: 2004366052.4067948.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 25%|##5 | 5/20 [00:21<00:52, 3.50s/it]\u001b[32m[I 2021-02-22 14:40:17,456]\u001b[0m Trial 11 finished with value: 2193886138.5860405 and parameters: {'num_leaves': 5}. Best is trial 10 with value: 2004366052.4067948.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 30%|### | 6/20 [00:23<00:41, 2.98s/it]\u001b[32m[I 2021-02-22 14:40:19,430]\u001b[0m Trial 12 finished with value: 2002345051.114594 and parameters: {'num_leaves': 42}. Best is trial 12 with value: 2002345051.114594.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 35%|###5 | 7/20 [00:33<01:05, 5.06s/it]\u001b[32m[I 2021-02-22 14:40:28,770]\u001b[0m Trial 13 finished with value: 2056130771.9329934 and parameters: {'num_leaves': 256}. Best is trial 12 with value: 2002345051.114594.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 40%|#### | 8/20 [00:44<01:22, 6.87s/it]\u001b[32m[I 2021-02-22 14:40:39,523]\u001b[0m Trial 14 finished with value: 2071021341.6650958 and parameters: {'num_leaves': 242}. Best is trial 12 with value: 2002345051.114594.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 45%|####5 | 9/20 [00:50<01:13, 6.64s/it]\u001b[32m[I 2021-02-22 14:40:45,668]\u001b[0m Trial 15 finished with value: 1987944577.546844 and parameters: {'num_leaves': 121}. Best is trial 15 with value: 1987944577.546844.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 50%|##### | 10/20 [00:59<01:15, 7.57s/it]\u001b[32m[I 2021-02-22 14:40:55,309]\u001b[0m Trial 16 finished with value: 2043161650.035548 and parameters: {'num_leaves': 222}. Best is trial 15 with value: 1987944577.546844.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 55%|#####5 | 11/20 [01:03<00:57, 6.43s/it]\u001b[32m[I 2021-02-22 14:40:59,172]\u001b[0m Trial 17 finished with value: 2029987579.182447 and parameters: {'num_leaves': 104}. Best is trial 15 with value: 1987944577.546844.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 60%|###### | 12/20 [01:07<00:45, 5.63s/it]\u001b[32m[I 2021-02-22 14:41:02,967]\u001b[0m Trial 18 finished with value: 2012583295.5343304 and parameters: {'num_leaves': 84}. Best is trial 15 with value: 1987944577.546844.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 65%|######5 | 13/20 [01:09<00:32, 4.64s/it]\u001b[32m[I 2021-02-22 14:41:05,314]\u001b[0m Trial 19 finished with value: 1981788985.8686044 and parameters: {'num_leaves': 56}. Best is trial 19 with value: 1981788985.8686044.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 70%|####### | 14/20 [01:15<00:30, 5.10s/it]\u001b[32m[I 2021-02-22 14:41:11,497]\u001b[0m Trial 20 finished with value: 2023959326.6503484 and parameters: {'num_leaves': 155}. Best is trial 19 with value: 1981788985.8686044.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 75%|#######5 | 15/20 [01:18<00:22, 4.43s/it]\u001b[32m[I 2021-02-22 14:41:14,365]\u001b[0m Trial 21 finished with value: 2017760258.9167733 and parameters: {'num_leaves': 69}. Best is trial 19 with value: 1981788985.8686044.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 80%|######## | 16/20 [01:24<00:18, 4.66s/it]\u001b[32m[I 2021-02-22 14:41:19,569]\u001b[0m Trial 22 finished with value: 2001975542.06975 and parameters: {'num_leaves': 141}. Best is trial 19 with value: 1981788985.8686044.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 85%|########5 | 17/20 [01:28<00:13, 4.46s/it]\u001b[32m[I 2021-02-22 14:41:23,567]\u001b[0m Trial 23 finished with value: 2003714379.9130254 and parameters: {'num_leaves': 111}. Best is trial 19 with value: 1981788985.8686044.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 90%|######### | 18/20 [01:33<00:09, 4.88s/it]\u001b[32m[I 2021-02-22 14:41:29,431]\u001b[0m Trial 24 finished with value: 2049456748.8392146 and parameters: {'num_leaves': 171}. Best is trial 19 with value: 1981788985.8686044.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 95%|#########5| 19/20 [01:36<00:04, 4.20s/it]\u001b[32m[I 2021-02-22 14:41:32,036]\u001b[0m Trial 25 finished with value: 1956603615.1475646 and parameters: {'num_leaves': 50}. Best is trial 25 with value: 1956603615.1475646.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 100%|##########| 20/20 [01:40<00:00, 4.03s/it]\u001b[32m[I 2021-02-22 14:41:35,682]\u001b[0m Trial 26 finished with value: 1956603615.1475646 and parameters: {'num_leaves': 50}. Best is trial 25 with value: 1956603615.1475646.\u001b[0m\n", + "num_leaves, val_score: 1949307059.499325: 100%|##########| 20/20 [01:40<00:00, 5.01s/it]\n", + "bagging, val_score: 1949307059.499325: 10%|# | 1/10 [00:02<00:25, 2.89s/it]\u001b[32m[I 2021-02-22 14:41:38,583]\u001b[0m Trial 27 finished with value: 2058040724.0398781 and parameters: {'bagging_fraction': 0.6789607888490847, 'bagging_freq': 1}. Best is trial 27 with value: 2058040724.0398781.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 20%|## | 2/10 [00:05<00:22, 2.77s/it]\u001b[32m[I 2021-02-22 14:41:41,271]\u001b[0m Trial 28 finished with value: 2058412265.7710927 and parameters: {'bagging_fraction': 0.6382384586067118, 'bagging_freq': 2}. Best is trial 27 with value: 2058040724.0398781.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 30%|### | 3/10 [00:08<00:20, 2.87s/it]\u001b[32m[I 2021-02-22 14:41:44,265]\u001b[0m Trial 29 finished with value: 2022566859.4286194 and parameters: {'bagging_fraction': 0.9689975648970396, 'bagging_freq': 3}. Best is trial 29 with value: 2022566859.4286194.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 40%|#### | 4/10 [00:11<00:17, 2.94s/it]\u001b[32m[I 2021-02-22 14:41:47,318]\u001b[0m Trial 30 finished with value: 2019125904.4126759 and parameters: {'bagging_fraction': 0.7947264008869785, 'bagging_freq': 5}. Best is trial 30 with value: 2019125904.4126759.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 50%|##### | 5/10 [00:14<00:14, 2.87s/it]\u001b[32m[I 2021-02-22 14:41:50,057]\u001b[0m Trial 31 finished with value: 1987043205.8620193 and parameters: {'bagging_fraction': 0.8589564136583515, 'bagging_freq': 7}. Best is trial 31 with value: 1987043205.8620193.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 60%|###### | 6/10 [00:16<00:11, 2.77s/it]\u001b[32m[I 2021-02-22 14:41:52,636]\u001b[0m Trial 32 finished with value: 1995986447.1995134 and parameters: {'bagging_fraction': 0.7967369529423736, 'bagging_freq': 1}. Best is trial 31 with value: 1987043205.8620193.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 70%|####### | 7/10 [00:19<00:08, 2.67s/it]\u001b[32m[I 2021-02-22 14:41:55,113]\u001b[0m Trial 33 finished with value: 1955997102.255711 and parameters: {'bagging_fraction': 0.8942338061809765, 'bagging_freq': 6}. Best is trial 33 with value: 1955997102.255711.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 80%|######## | 8/10 [00:22<00:05, 2.66s/it]\u001b[32m[I 2021-02-22 14:41:57,747]\u001b[0m Trial 34 finished with value: 2048654658.4711766 and parameters: {'bagging_fraction': 0.5987476817554606, 'bagging_freq': 2}. Best is trial 33 with value: 1955997102.255711.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 90%|######### | 9/10 [00:24<00:02, 2.66s/it]\u001b[32m[I 2021-02-22 14:42:00,415]\u001b[0m Trial 35 finished with value: 1972462964.5654855 and parameters: {'bagging_fraction': 0.9885365644176932, 'bagging_freq': 3}. Best is trial 33 with value: 1955997102.255711.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 100%|##########| 10/10 [00:27<00:00, 2.64s/it]\u001b[32m[I 2021-02-22 14:42:02,999]\u001b[0m Trial 36 finished with value: 2017973500.1911747 and parameters: {'bagging_fraction': 0.7252191625113514, 'bagging_freq': 3}. Best is trial 33 with value: 1955997102.255711.\u001b[0m\n", + "bagging, val_score: 1949307059.499325: 100%|##########| 10/10 [00:27<00:00, 2.73s/it]\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 17%|#6 | 1/6 [00:02<00:11, 2.20s/it]\u001b[32m[I 2021-02-22 14:42:05,220]\u001b[0m Trial 37 finished with value: 1949307059.499325 and parameters: {'feature_fraction': 0.9159999999999999}. Best is trial 37 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 33%|###3 | 2/6 [00:04<00:08, 2.16s/it]\u001b[32m[I 2021-02-22 14:42:07,344]\u001b[0m Trial 38 finished with value: 1949307059.499325 and parameters: {'feature_fraction': 0.8839999999999999}. Best is trial 37 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 50%|##### | 3/6 [00:06<00:06, 2.30s/it]\u001b[32m[I 2021-02-22 14:42:09,820]\u001b[0m Trial 39 finished with value: 1988181425.985298 and parameters: {'feature_fraction': 0.948}. Best is trial 37 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 67%|######6 | 4/6 [00:09<00:04, 2.30s/it]\u001b[32m[I 2021-02-22 14:42:12,114]\u001b[0m Trial 40 finished with value: 1988181425.985298 and parameters: {'feature_fraction': 0.9799999999999999}. Best is trial 37 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 83%|########3 | 5/6 [00:11<00:02, 2.33s/it]\u001b[32m[I 2021-02-22 14:42:14,504]\u001b[0m Trial 41 finished with value: 1949307059.499325 and parameters: {'feature_fraction': 0.852}. Best is trial 37 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 100%|##########| 6/6 [00:13<00:00, 2.28s/it]\u001b[32m[I 2021-02-22 14:42:16,674]\u001b[0m Trial 42 finished with value: 1949307059.499325 and parameters: {'feature_fraction': 0.82}. Best is trial 37 with value: 1949307059.499325.\u001b[0m\n", + "feature_fraction_stage2, val_score: 1949307059.499325: 100%|##########| 6/6 [00:13<00:00, 2.28s/it]\n", + "regularization_factors, val_score: 1949307038.475713: 5%|5 | 1/20 [00:02<00:41, 2.18s/it]\u001b[32m[I 2021-02-22 14:42:18,861]\u001b[0m Trial 43 finished with value: 1949307038.475713 and parameters: {'lambda_l1': 0.09101144524819704, 'lambda_l2': 1.7704703083488795e-08}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 10%|# | 2/20 [00:04<00:39, 2.21s/it]\u001b[32m[I 2021-02-22 14:42:21,099]\u001b[0m Trial 44 finished with value: 1995873001.2074113 and parameters: {'lambda_l1': 2.6557868173633803e-07, 'lambda_l2': 5.183750661759111}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 15%|#5 | 3/20 [00:06<00:37, 2.21s/it]\u001b[32m[I 2021-02-22 14:42:23,296]\u001b[0m Trial 45 finished with value: 1965414514.3746212 and parameters: {'lambda_l1': 0.7263427156399412, 'lambda_l2': 0.0013813379781067797}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 20%|## | 4/20 [00:08<00:34, 2.18s/it]\u001b[32m[I 2021-02-22 14:42:25,439]\u001b[0m Trial 46 finished with value: 1954274712.97174 and parameters: {'lambda_l1': 7.578182163631005e-05, 'lambda_l2': 0.00021084910200530688}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 25%|##5 | 5/20 [00:10<00:32, 2.16s/it]\u001b[32m[I 2021-02-22 14:42:27,566]\u001b[0m Trial 47 finished with value: 1965414460.2095408 and parameters: {'lambda_l1': 7.072150718391867e-06, 'lambda_l2': 0.0014472465664563316}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 30%|### | 6/20 [00:12<00:29, 2.14s/it]\u001b[32m[I 2021-02-22 14:42:29,671]\u001b[0m Trial 48 finished with value: 1949307057.598466 and parameters: {'lambda_l1': 4.951731906814475e-06, 'lambda_l2': 5.544032280720154e-07}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 35%|###5 | 7/20 [00:15<00:27, 2.14s/it]\u001b[32m[I 2021-02-22 14:42:31,794]\u001b[0m Trial 49 finished with value: 2021008326.2503476 and parameters: {'lambda_l1': 0.002348352296820926, 'lambda_l2': 0.143195505557829}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 40%|#### | 8/20 [00:17<00:25, 2.16s/it]\u001b[32m[I 2021-02-22 14:42:34,011]\u001b[0m Trial 50 finished with value: 1957224797.2084112 and parameters: {'lambda_l1': 1.5908622169872778e-07, 'lambda_l2': 0.7751981347764194}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949307038.475713: 45%|####5 | 9/20 [00:19<00:23, 2.16s/it]\u001b[32m[I 2021-02-22 14:42:36,177]\u001b[0m Trial 51 finished with value: 1958710530.5581698 and parameters: {'lambda_l1': 1.997720402454698e-06, 'lambda_l2': 0.3916021711591694}. Best is trial 43 with value: 1949307038.475713.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 50%|##### | 10/20 [00:21<00:21, 2.16s/it]\u001b[32m[I 2021-02-22 14:42:38,329]\u001b[0m Trial 52 finished with value: 1949011993.9609785 and parameters: {'lambda_l1': 1.9741002172068045e-06, 'lambda_l2': 0.0009225859572515588}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 55%|#####5 | 11/20 [00:25<00:23, 2.64s/it]\u001b[32m[I 2021-02-22 14:42:42,063]\u001b[0m Trial 53 finished with value: 1949306996.8346448 and parameters: {'lambda_l1': 1.4927989272260208e-08, 'lambda_l2': 1.815720293048345e-05}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 60%|###### | 12/20 [00:27<00:20, 2.52s/it]\u001b[32m[I 2021-02-22 14:42:44,299]\u001b[0m Trial 54 finished with value: 1949306999.1035354 and parameters: {'lambda_l1': 1.40977148038626e-08, 'lambda_l2': 1.7685279853730492e-05}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 65%|######5 | 13/20 [00:29<00:17, 2.46s/it]\u001b[32m[I 2021-02-22 14:42:46,634]\u001b[0m Trial 55 finished with value: 1949307044.7417297 and parameters: {'lambda_l1': 1.5463768519498943e-08, 'lambda_l2': 4.306626722644303e-06}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 70%|####### | 14/20 [00:32<00:14, 2.39s/it]\u001b[32m[I 2021-02-22 14:42:48,870]\u001b[0m Trial 56 finished with value: 1989218170.003291 and parameters: {'lambda_l1': 1.0022923369342419e-08, 'lambda_l2': 0.0045395189861971285}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 75%|#######5 | 15/20 [00:34<00:11, 2.34s/it]\u001b[32m[I 2021-02-22 14:42:51,095]\u001b[0m Trial 57 finished with value: 1949306886.7297008 and parameters: {'lambda_l1': 0.0002408003105743025, 'lambda_l2': 5.00826403625466e-05}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 80%|######## | 16/20 [00:36<00:08, 2.17s/it]\u001b[32m[I 2021-02-22 14:42:52,863]\u001b[0m Trial 58 finished with value: 1976046665.7503545 and parameters: {'lambda_l1': 0.0012739301862614585, 'lambda_l2': 0.015123687061631018}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 85%|########5 | 17/20 [00:37<00:05, 1.96s/it]\u001b[32m[I 2021-02-22 14:42:54,331]\u001b[0m Trial 59 finished with value: 1949307059.0756855 and parameters: {'lambda_l1': 0.000111924254335766, 'lambda_l2': 1.2760174203841509e-07}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 90%|######### | 18/20 [00:39<00:03, 1.80s/it]\u001b[32m[I 2021-02-22 14:42:55,768]\u001b[0m Trial 60 finished with value: 1954274496.2320116 and parameters: {'lambda_l1': 0.011232645128874983, 'lambda_l2': 0.00027341062705489946}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 95%|#########5| 19/20 [00:40<00:01, 1.76s/it]\u001b[32m[I 2021-02-22 14:42:57,426]\u001b[0m Trial 61 finished with value: 1949306916.0588307 and parameters: {'lambda_l1': 3.406557565523097e-05, 'lambda_l2': 4.1565414801737324e-05}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 100%|##########| 20/20 [00:42<00:00, 1.78s/it]\u001b[32m[I 2021-02-22 14:42:59,261]\u001b[0m Trial 62 finished with value: 1949868047.0339825 and parameters: {'lambda_l1': 7.935752792484713, 'lambda_l2': 1.1227100513172882e-06}. Best is trial 52 with value: 1949011993.9609785.\u001b[0m\n", + "regularization_factors, val_score: 1949011993.960979: 100%|##########| 20/20 [00:42<00:00, 2.13s/it]\n", + "min_data_in_leaf, val_score: 1949011993.960979: 20%|## | 1/5 [00:01<00:06, 1.62s/it]\u001b[32m[I 2021-02-22 14:43:00,901]\u001b[0m Trial 63 finished with value: 1989752975.1835222 and parameters: {'min_child_samples': 5}. Best is trial 63 with value: 1989752975.1835222.\u001b[0m\n", + "min_data_in_leaf, val_score: 1949011993.960979: 40%|#### | 2/5 [00:03<00:04, 1.64s/it]\u001b[32m[I 2021-02-22 14:43:02,562]\u001b[0m Trial 64 finished with value: 1970804604.9243653 and parameters: {'min_child_samples': 50}. Best is trial 64 with value: 1970804604.9243653.\u001b[0m\n", + "min_data_in_leaf, val_score: 1949011993.960979: 60%|###### | 3/5 [00:04<00:03, 1.63s/it]\u001b[32m[I 2021-02-22 14:43:04,166]\u001b[0m Trial 65 finished with value: 1998195068.5676072 and parameters: {'min_child_samples': 25}. Best is trial 64 with value: 1970804604.9243653.\u001b[0m\n", + "min_data_in_leaf, val_score: 1949011993.960979: 80%|######## | 4/5 [00:06<00:01, 1.68s/it]\u001b[32m[I 2021-02-22 14:43:05,928]\u001b[0m Trial 66 finished with value: 2013164530.6381345 and parameters: {'min_child_samples': 100}. Best is trial 64 with value: 1970804604.9243653.\u001b[0m\n", + "min_data_in_leaf, val_score: 1915185243.192194: 100%|##########| 5/5 [00:08<00:00, 1.61s/it]\u001b[32m[I 2021-02-22 14:43:07,426]\u001b[0m Trial 67 finished with value: 1915185243.1921945 and parameters: {'min_child_samples': 10}. Best is trial 67 with value: 1915185243.1921945.\u001b[0m\n", + "min_data_in_leaf, val_score: 1915185243.192194: 100%|##########| 5/5 [00:08<00:00, 1.63s/it]Wall time: 3min 26s\n", + "\n" + ] + } + ], + "source": [ + "%%time\n", + "model = lgb.train(params, dtrain, valid_sets=[dtrain, dval], verbose_eval=10000) \n" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "output_type": "stream", + "name": "stdout", + "text": [ + "Optuna LightGBM Tuner r2 = 0.8476245395516778\n" + ] + } + ], + "source": [ + "y_pred = model.predict(X_test)\n", + "from flaml.ml import sklearn_metric_loss_score\n", + "print('Optuna LightGBM Tuner r2', '=', 1 - sklearn_metric_loss_score('r2', y_pred, y_test))" + ] + } + ], + "metadata": { + "kernelspec": { + "name": "python3", + "display_name": "Python 3.7.7 64-bit ('flaml': conda)", + "metadata": { + "interpreter": { + "hash": "bfcd9a6a9254a5e160761a1fd7a9e444f011592c6770d9f4180dde058a9df5dd" + } + } + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.7-final" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} \ No newline at end of file diff --git a/test/test_automl.py b/test/test_automl.py index d502cb056..19e4352a4 100644 --- a/test/test_automl.py +++ b/test/test_automl.py @@ -274,7 +274,7 @@ class TestAutoML(unittest.TestCase): automl_experiment = AutoML() automl_settings = { - "time_budget": 2, + "time_budget": 3, "metric": 'ap', "task": 'classification', "log_file_name": "test/sparse_classification.log", diff --git a/test/test_distillbert.py b/test/test_distillbert.py new file mode 100644 index 000000000..c50687442 --- /dev/null +++ b/test/test_distillbert.py @@ -0,0 +1,217 @@ +'''Require: pip install torch transformers datasets flaml[blendsearch,ray] +''' +import time +import numpy as np + +try: + import ray + from datasets import ( + load_dataset, + load_metric, + ) + from transformers import ( + AutoModelForSequenceClassification, + AutoTokenizer, + Trainer, + TrainingArguments, + ) +except: + print("pip install torch transformers datasets flaml[blendsearch,ray]") + +import logging +logger = logging.getLogger(__name__) +logger.addHandler(logging.FileHandler('test/tune_distilbert.log')) +logger.setLevel(logging.INFO) + +import flaml + + +MODEL_CHECKPOINT = "distilbert-base-uncased" +TASK = "cola" +NUM_LABELS = 2 +COLUMN_NAME = "sentence" +METRIC_NAME = "matthews_correlation" + +# HP_METRIC, MODE = "loss", "min" +HP_METRIC, MODE = "matthews_correlation", "max" + +def train_distilbert(config: dict): + + # Define tokenize method + tokenizer = AutoTokenizer.from_pretrained(MODEL_CHECKPOINT, use_fast=True) + def tokenize(examples): + return tokenizer(examples[COLUMN_NAME], truncation=True) + # Load CoLA dataset and apply tokenizer + cola_raw = load_dataset("glue", TASK) + cola_encoded = cola_raw.map(tokenize, batched=True) + train_dataset, eval_dataset = cola_encoded["train"], cola_encoded["validation"] + + model = AutoModelForSequenceClassification.from_pretrained( + MODEL_CHECKPOINT, num_labels=NUM_LABELS + ) + + metric = load_metric("glue", TASK) + + def compute_metrics(eval_pred): + predictions, labels = eval_pred + predictions = np.argmax(predictions, axis=1) + return metric.compute(predictions=predictions, references=labels) + + training_args = TrainingArguments( + output_dir='.', + do_eval=False, + disable_tqdm=True, + logging_steps=20000, + save_total_limit=0, + **config, + ) + + trainer = Trainer( + model, + training_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + tokenizer=tokenizer, + compute_metrics=compute_metrics, + ) + + # train model + trainer.train() + + # evaluate model + eval_output = trainer.evaluate() + + flaml.tune.report( + loss=eval_output["eval_loss"], + matthews_correlation=eval_output["eval_matthews_correlation"], + ) + + +def _test_distillbert(method='BlendSearch'): + + max_num_epoch = 64 + num_samples = -1 + time_budget_s = 10800 + + search_space = { + # You can mix constants with search space objects. + "num_train_epochs": flaml.tune.loguniform(1, max_num_epoch), + "learning_rate": flaml.tune.loguniform(1e-6, 1e-4), + "adam_beta1": flaml.tune.uniform(0.8, 0.99), + "adam_beta2": flaml.tune.loguniform(98e-2, 9999e-4), + "adam_epsilon": flaml.tune.loguniform(1e-9, 1e-7), + } + + start_time = time.time() + ray.init(num_cpus=4, num_gpus=4) + if 'ASHA' == method: + algo = None + elif 'BOHB' == method: + from ray.tune.schedulers import HyperBandForBOHB + from ray.tune.suggest.bohb import tuneBOHB + algo = tuneBOHB(max_concurrent=4) + scheduler = HyperBandForBOHB(max_t=max_num_epoch) + elif 'Optuna' == method: + from ray.tune.suggest.optuna import OptunaSearch + algo = OptunaSearch() + elif 'CFO' == method: + from flaml import CFO + algo = CFO(points_to_evaluate=[{ + "num_train_epochs": 1, + }]) + elif 'BlendSearch' == method: + from flaml import BlendSearch + algo = BlendSearch(points_to_evaluate=[{ + "num_train_epochs": 1, + }]) + elif 'Dragonfly' == method: + from ray.tune.suggest.dragonfly import DragonflySearch + algo = DragonflySearch() + elif 'SkOpt' == method: + from ray.tune.suggest.skopt import SkOptSearch + algo = SkOptSearch() + elif 'Nevergrad' == method: + from ray.tune.suggest.nevergrad import NevergradSearch + import nevergrad as ng + algo = NevergradSearch(optimizer=ng.optimizers.OnePlusOne) + elif 'ZOOpt' == method: + from ray.tune.suggest.zoopt import ZOOptSearch + algo = ZOOptSearch(budget=num_samples) + elif 'Ax' == method: + from ray.tune.suggest.ax import AxSearch + algo = AxSearch() + elif 'HyperOpt' == method: + from ray.tune.suggest.hyperopt import HyperOptSearch + algo = HyperOptSearch() + scheduler = None + if method != 'BOHB': + from ray.tune.schedulers import ASHAScheduler + scheduler = ASHAScheduler( + max_t=max_num_epoch, + grace_period=1) + scheduler = None + analysis = ray.tune.run( + train_distilbert, + metric=HP_METRIC, + mode=MODE, + # You can add "gpu": 1 to allocate GPUs + resources_per_trial={"gpu": 1}, + config=search_space, local_dir='test/logs/', + num_samples=num_samples, time_budget_s=time_budget_s, + keep_checkpoints_num=1, checkpoint_score_attr=HP_METRIC, + scheduler=scheduler, search_alg=algo) + + ray.shutdown() + + best_trial = analysis.get_best_trial(HP_METRIC, MODE, "all") + metric = best_trial.metric_analysis[HP_METRIC][MODE] + + logger.info(f"method={method}") + logger.info(f"n_trials={len(analysis.trials)}") + logger.info(f"time={time.time()-start_time}") + logger.info(f"Best model eval {HP_METRIC}: {metric:.4f}") + logger.info(f"Best model parameters: {best_trial.config}") + + +def _test_distillbert_cfo(): + _test_distillbert('CFO') + + +def _test_distillbert_dragonfly(): + _test_distillbert('Dragonfly') + + +def _test_distillbert_skopt(): + _test_distillbert('SkOpt') + + +def _test_distillbert_nevergrad(): + _test_distillbert('Nevergrad') + + +def _test_distillbert_zoopt(): + _test_distillbert('ZOOpt') + + +def _test_distillbert_ax(): + _test_distillbert('Ax') + + +def __test_distillbert_hyperopt(): + _test_distillbert('HyperOpt') + + +def _test_distillbert_optuna(): + _test_distillbert('Optuna') + + +def _test_distillbert_asha(): + _test_distillbert('ASHA') + + +def _test_distillbert_bohb(): + _test_distillbert('BOHB') + + +if __name__ == "__main__": + _test_distillbert() \ No newline at end of file diff --git a/test/test_pytorch_cifar10.py b/test/test_pytorch_cifar10.py index 7841e2d93..7bb0e625f 100644 --- a/test/test_pytorch_cifar10.py +++ b/test/test_pytorch_cifar10.py @@ -1,4 +1,4 @@ -'''Require: pip install torchvision ray +'''Require: pip install torchvision ray flaml[blendsearch] ''' import unittest import os @@ -26,7 +26,6 @@ def load_data(data_dir="./data"): # __load_data_end__ -import numpy as np try: import torch import torch.nn as nn diff --git a/test/test_xgboost2d.py b/test/test_xgboost2d.py index d85a3663d..f1a7870e8 100644 --- a/test/test_xgboost2d.py +++ b/test/test_xgboost2d.py @@ -8,11 +8,7 @@ from flaml.model import XGBoostSklearnEstimator from flaml import tune -# dataset = "blood-transfusion-service-center" -# dataset = "Australian" dataset = "credit-g" -# dataset = "phoneme" -# dataset = "kc1" class XGBoost2D(XGBoostSklearnEstimator): @@ -50,8 +46,11 @@ def test_simple(method=None): "log_type": "all", "time_budget": 3#6000, } - - X, y = fetch_openml(name=dataset, return_X_y=True) + try: + X, y = fetch_openml(name=dataset, return_X_y=True) + except: + from sklearn.datasets import load_wine + X, y = load_wine(return_X_y=True) X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.33, random_state=42) automl.fit(X_train=X_train, y_train=y_train, **automl_settings)