source/FLAML
1
0
Fork 0
mirror of https://github.com/microsoft/FLAML.git synced 2026-02-09 02:09:16 +08:00
Code Issues Packages Projects Releases Wiki Activity

support string alg in tune (#1093)

Browse Source 
* support string alg in tune

* add test, enforce string feasible, support lexico in set_search_priorities in CFO

* fix bug

* fix bug

* fix bug

* fix bug

* fix bugs

* fix yiran

---------

Co-authored-by: “skzhang1” <“shaokunzhang529@gmail.com”>
This commit is contained in:
Shaokun
2023-07-01 11:01:14 +08:00
committed by GitHub
parent aa05434c87
commit 7a64148676
3 changed files with 149 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
flaml/tune/tune.py
View File

@@ -344,7 +344,7 @@ def run(
# do cleanup operation here
return
```
search_alg: An instance of BlendSearch as the search algorithm
search_alg: An instance/string of the search algorithm
to be used. The same instance can be used for iterative tuning.
e.g.,
@@ -481,12 +481,25 @@ def run(
else:
logger.setLevel(logging.CRITICAL)
from .searcher.blendsearch import BlendSearch, CFO
from .searcher.blendsearch import BlendSearch, CFO, RandomSearch
if lexico_objectives is not None:
logger.warning("If lexico_objectives is not None, search_alg is forced to be CFO")
search_alg = None
if search_alg is None:
if "modes" not in lexico_objectives.keys():
lexico_objectives["modes"] = ["min"] * len(lexico_objectives["metrics"])
for t_metric, t_mode in zip(lexico_objectives["metrics"], lexico_objectives["modes"]):
if t_metric not in lexico_objectives["tolerances"].keys():
lexico_objectives["tolerances"][t_metric] = 0
if t_metric not in lexico_objectives["targets"].keys():
lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
if search_alg is None or isinstance(search_alg, str):
if isinstance(search_alg, str):
assert search_alg in [
"BlendSearch",
"CFO",
"CFOCat",
"RandomSearch",
], f"search_alg={search_alg} is not recognized. 'BlendSearch', 'CFO', 'CFOcat' and 'RandomSearch' are supported."
flaml_scheduler_resource_attr = (
flaml_scheduler_min_resource
) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None
@@ -500,20 +513,30 @@ def run(
flaml_scheduler_max_resource = max_resource
flaml_scheduler_reduction_factor = reduction_factor
scheduler = None
if lexico_objectives is None:
try:
import optuna as _
SearchAlgorithm = BlendSearch
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
except ImportError:
SearchAlgorithm = CFO
logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
metric = metric or DEFAULT_METRIC
else:
if lexico_objectives:
# TODO: Modify after supporting BlendSearch in lexicographic optimization
SearchAlgorithm = CFO
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
logger.info(
f"Using search algorithm {SearchAlgorithm.__name__} for lexicographic optimization. Note that when providing other search algorithms, we use CFO instead temporarily."
)
metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
else:
if not search_alg or search_alg == "BlendSearch":
try:
import optuna as _
SearchAlgorithm = BlendSearch
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
except ImportError:
if search_alg == "BlendSearch":
raise ValueError("To use BlendSearch, run: pip install flaml[blendsearch]")
else:
SearchAlgorithm = CFO
logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
else:
SearchAlgorithm = locals()[search_alg]
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
metric = metric or DEFAULT_METRIC
search_alg = SearchAlgorithm(
metric=metric,
mode=mode,
@@ -535,8 +558,12 @@ def run(
)
else:
if metric is None or mode is None:
metric = metric or search_alg.metric or DEFAULT_METRIC
mode = mode or search_alg.mode
if lexico_objectives:
metric = lexico_objectives["metrics"][0] or metric or search_alg.metric or DEFAULT_METRIC
mode = lexico_objectives["modes"][0] or mode or search_alg.mode
else:
metric = metric or search_alg.metric or DEFAULT_METRIC
mode = mode or search_alg.mode
if ray_available and use_ray:
if ray_version.startswith("1."):
from ray.tune.suggest import ConcurrencyLimiter
@@ -555,6 +582,13 @@ def run(
):
search_alg.use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
searcher = search_alg.searcher if isinstance(search_alg, ConcurrencyLimiter) else search_alg
if lexico_objectives:
# TODO: Modify after supporting BlendSearch in lexicographic optimization
assert search_alg.__class__.__name__ in [
"CFO",
], "If lexico_objectives is not None, the search_alg must be CFO for now."
search_alg.lexico_objective = lexico_objectives
if isinstance(searcher, BlendSearch):
setting = {}
if time_budget_s:

4
setup.py
View File

@@ -85,7 +85,7 @@ setuptools.setup(
"tensorboardX==2.6", # test_forecast_panel
"requests<2.29.0", # https://github.com/docker/docker-py/issues/3113
"packaging",
"pydantic",
"pydantic==1.10.9",
"sympy",
"wolframalpha",
"joblib<1.3.0", # temp solution for joblib 1.3.0 issue, no need once https://github.com/joblib/joblib-spark/pull/48 is merged
@@ -137,7 +137,7 @@ setuptools.setup(
"benchmark": ["catboost>=0.26", "psutil==5.8.0", "xgboost==1.3.3", "pandas==1.1.4"],
"openai": ["openai==0.27.8", "diskcache"],
"autogen": ["openai==0.27.8", "diskcache", "docker"],
"mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic", "wolframalpha"],
"mathchat": ["openai==0.27.8", "diskcache", "docker", "sympy", "pydantic==1.10.9", "wolframalpha"],
"synapse": [
"joblibspark>=0.5.0",
"optuna==2.8.0",

95
test/tune/test_tune.py
View File

@@ -1,6 +1,6 @@
"""Require: pip install flaml[test,ray]
"""
from flaml import BlendSearch
from flaml import BlendSearch, CFO
import time
import os
from sklearn.model_selection import train_test_split
@@ -8,6 +8,7 @@ import sklearn.metrics
import sklearn.datasets
import xgboost as xgb
import logging
import math
try:
from ray.tune.integration.xgboost import TuneReportCheckpointCallback
@@ -20,6 +21,32 @@ logger.addHandler(logging.FileHandler("logs/tune.log"))
logger.setLevel(logging.INFO)
def _BraninCurrin(config):
# Rescale brain
x_1 = 15 * config["x1"] - 5
x_2 = 15 * config["x2"]
# Brain function
t1 = x_2 - 5.1 / (4 * math.pi**2) * x_1**2 + 5 / math.pi * x_1 - 6
t2 = 10 * (1 - 1 / (8 * math.pi)) * math.cos(x_1)
brain_result = t1**2 + t2 + 10
# Currin function
xc_1 = config["x1"]
xc_2 = config["x2"]
factor1 = 1 - math.exp(-1 / (2 * xc_2))
numer = 2300 * pow(xc_1, 3) + 1900 * pow(xc_1, 2) + 2092 * xc_1 + 60
denom = 100 * pow(xc_1, 3) + 500 * pow(xc_1, 2) + 4 * xc_1 + 20
currin_result = factor1 * numer / denom
return {"brain": brain_result, "currin": currin_result}
def _easy_objective(config):
# Hyperparameters
width, height, step = config["width"], config["height"], config["steps"]
# get_result
return {"mean_loss": (0.1 + width * step / 100) ** (-1) + height * 0.1}
def test_nested_run():
from flaml import AutoML, tune
@@ -352,6 +379,72 @@ def test_run_training_function_return_value():
)
def test_passing_search_alg():
from flaml import tune
# search_space
so_search_space = {
"steps": 100,
"width": tune.uniform(0, 20),
"height": tune.uniform(-100, 100),
}
mo_search_space = {
"x1": tune.uniform(lower=0.000001, upper=1.0),
"x2": tune.uniform(lower=0.000001, upper=1.0),
}
# lexicographic objectives
lexico_objectives = {}
lexico_objectives["metrics"] = ["brain", "currin"]
lexico_objectives["tolerances"] = {"brain": 10.0, "currin": 0.0}
lexico_objectives["targets"] = {"brain": 0.0, "currin": 0.0}
lexico_objectives["modes"] = ["min", "min"]
## Passing search_alg through string
# Non lexico tune
tune.run(
_easy_objective,
search_alg="BlendSearch",
metric="mean_loss",
mode="min",
num_samples=10,
config=so_search_space,
)
# lexico tune
tune.run(
_BraninCurrin, search_alg="CFO", num_samples=10, config=mo_search_space, lexico_objectives=lexico_objectives
)
tune.run(
_BraninCurrin,
search_alg="BlendSearch",
num_samples=10,
config=mo_search_space,
lexico_objectives=lexico_objectives,
)
## Passing search_alg through instance
so_bs = BlendSearch(time_budget_s=5, metric="mean_loss", mode="min")
# TODO: We will change CFO into blendsearch in the future
mo_bs = CFO(time_budget_s=5)
# Non lexico tune
tune.run(
_easy_objective,
search_alg=so_bs,
metric="mean_loss",
mode="min",
num_samples=10,
config=so_search_space,
)
# lexico tune
tune.run(
_BraninCurrin,
search_alg=mo_bs,
num_samples=10,
config=mo_search_space,
lexico_objectives=lexico_objectives,
)
def test_xgboost_bs():
_test_xgboost()