diff --git a/flaml/__init__.py b/flaml/__init__.py
index 87bb428b3..ecd80cf69 100644
--- a/flaml/__init__.py
+++ b/flaml/__init__.py
@@ -1,4 +1,4 @@
-from flaml.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearch
+from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearch
 from flaml.automl import AutoML, logger_formatter
 from flaml.onlineml.autovw import AutoVW
 from flaml.version import __version__
diff --git a/flaml/automl.py b/flaml/automl.py
index e830856e7..f91074cac 100644
--- a/flaml/automl.py
+++ b/flaml/automl.py
@@ -2961,7 +2961,7 @@ class AutoML(BaseEstimator):
                 else:
                     from ray.tune.search.optuna import OptunaSearch as SearchAlgo
             except (ImportError, AssertionError):
-                from .searcher.suggestion import OptunaSearch as SearchAlgo
+                from flaml.tune.searcher.suggestion import OptunaSearch as SearchAlgo
         else:
             raise NotImplementedError(
                 f"hpo_method={self._hpo_method} is not recognized. "
@@ -3125,7 +3125,7 @@ class AutoML(BaseEstimator):
             else:
                 from ray.tune.search import ConcurrencyLimiter
         except (ImportError, AssertionError):
-            from .searcher.suggestion import ConcurrencyLimiter
+            from flaml.tune.searcher.suggestion import ConcurrencyLimiter
         if self._hpo_method in ("cfo", "grid"):
             from flaml import CFO as SearchAlgo
         elif "optuna" == self._hpo_method:
@@ -3138,13 +3138,13 @@ class AutoML(BaseEstimator):
                 else:
                     from ray.tune.search.optuna import OptunaSearch as SearchAlgo
             except (ImportError, AssertionError):
-                from .searcher.suggestion import OptunaSearch as SearchAlgo
+                from flaml.tune.searcher.suggestion import OptunaSearch as SearchAlgo
         elif "bs" == self._hpo_method:
             from flaml import BlendSearch as SearchAlgo
         elif "random" == self._hpo_method:
-            from flaml.searcher import RandomSearch as SearchAlgo
+            from flaml.tune.searcher import RandomSearch as SearchAlgo
         elif "cfocat" == self._hpo_method:
-            from flaml.searcher.cfo_cat import CFOCat as SearchAlgo
+            from flaml.tune.searcher.cfo_cat import CFOCat as SearchAlgo
         else:
             raise NotImplementedError(
                 f"hpo_method={self._hpo_method} is not recognized. "
diff --git a/flaml/onlineml/autovw.py b/flaml/onlineml/autovw.py
index 1ec4e6cbb..da0e520b7 100644
--- a/flaml/onlineml/autovw.py
+++ b/flaml/onlineml/autovw.py
@@ -8,8 +8,8 @@ from flaml.tune import (
     polynomial_expansion_set,
 )
 from flaml.onlineml import OnlineTrialRunner
-from flaml.scheduler import ChaChaScheduler
-from flaml.searcher import ChampionFrontierSearcher
+from flaml.tune.scheduler import ChaChaScheduler
+from flaml.tune.searcher import ChampionFrontierSearcher
 from flaml.onlineml.trial import get_ns_feature_dim_from_vw_example
 
 logger = logging.getLogger(__name__)
diff --git a/flaml/onlineml/trial_runner.py b/flaml/onlineml/trial_runner.py
index 7ef7e119e..3860d0afb 100644
--- a/flaml/onlineml/trial_runner.py
+++ b/flaml/onlineml/trial_runner.py
@@ -1,7 +1,7 @@
 import numpy as np
 import math
 from flaml.tune import Trial
-from flaml.scheduler import TrialScheduler
+from flaml.tune.scheduler import TrialScheduler
 
 import logging
 
diff --git a/flaml/scheduler/__init__.py b/flaml/tune/scheduler/__init__.py
similarity index 100%
rename from flaml/scheduler/__init__.py
rename to flaml/tune/scheduler/__init__.py
diff --git a/flaml/scheduler/online_scheduler.py b/flaml/tune/scheduler/online_scheduler.py
similarity index 99%
rename from flaml/scheduler/online_scheduler.py
rename to flaml/tune/scheduler/online_scheduler.py
index 55f2563fe..bf235afcd 100644
--- a/flaml/scheduler/online_scheduler.py
+++ b/flaml/tune/scheduler/online_scheduler.py
@@ -1,7 +1,7 @@
 import numpy as np
 import logging
 from typing import Dict
-from flaml.scheduler import TrialScheduler
+from flaml.tune.scheduler import TrialScheduler
 from flaml.tune import Trial
 
 logger = logging.getLogger(__name__)
diff --git a/flaml/scheduler/trial_scheduler.py b/flaml/tune/scheduler/trial_scheduler.py
similarity index 100%
rename from flaml/scheduler/trial_scheduler.py
rename to flaml/tune/scheduler/trial_scheduler.py
diff --git a/flaml/searcher/__init__.py b/flaml/tune/searcher/__init__.py
similarity index 100%
rename from flaml/searcher/__init__.py
rename to flaml/tune/searcher/__init__.py
diff --git a/flaml/searcher/blendsearch.py b/flaml/tune/searcher/blendsearch.py
similarity index 97%
rename from flaml/searcher/blendsearch.py
rename to flaml/tune/searcher/blendsearch.py
index 269de2612..e22ae6055 100644
--- a/flaml/searcher/blendsearch.py
+++ b/flaml/tune/searcher/blendsearch.py
@@ -20,12 +20,12 @@ try:
 except (ImportError, AssertionError):
     from .suggestion import Searcher
     from .suggestion import OptunaSearch as GlobalSearch
-from ..tune.trial import unflatten_dict, flatten_dict
-from ..tune import INCUMBENT_RESULT
+from ..trial import unflatten_dict, flatten_dict
+from .. import INCUMBENT_RESULT
 from .search_thread import SearchThread
 from .flow2 import FLOW2
-from ..tune.space import add_cost_to_space, indexof, normalize, define_by_run_func
-from ..tune.result import TIME_TOTAL_S
+from ..space import add_cost_to_space, indexof, normalize, define_by_run_func
+from ..result import TIME_TOTAL_S
 
 import logging
 
@@ -939,7 +939,7 @@ try:
         qloguniform,
     )
 except (ImportError, AssertionError):
-    from ..tune.sample import (
+    from ..sample import (
         uniform,
         quniform,
         choice,
diff --git a/flaml/searcher/cfo_cat.py b/flaml/tune/searcher/cfo_cat.py
similarity index 100%
rename from flaml/searcher/cfo_cat.py
rename to flaml/tune/searcher/cfo_cat.py
diff --git a/flaml/searcher/flow2.py b/flaml/tune/searcher/flow2.py
similarity index 98%
rename from flaml/searcher/flow2.py
rename to flaml/tune/searcher/flow2.py
index d5722db21..226f624ac 100644
--- a/flaml/searcher/flow2.py
+++ b/flaml/tune/searcher/flow2.py
@@ -18,11 +18,10 @@ try:
     from ray.tune.utils.util import flatten_dict, unflatten_dict
 except (ImportError, AssertionError):
     from .suggestion import Searcher
-    from ..tune import sample
-    from ..tune.trial import flatten_dict, unflatten_dict
-from flaml.tune.sample import _BackwardsCompatibleNumpyRng
+    from flaml.tune import sample
+    from ..trial import flatten_dict, unflatten_dict
 from flaml.config import SAMPLE_MULTIPLY_FACTOR
-from ..tune.space import (
+from ..space import (
     complete_config,
     denormalize,
     normalize,
@@ -85,7 +84,7 @@ class FLOW2(Searcher):
         self.space = space or {}
         self._space = flatten_dict(self.space, prevent_delimiter=True)
         self._random = np.random.RandomState(seed)
-        self.rs_random = _BackwardsCompatibleNumpyRng(seed + 19823)
+        self.rs_random = sample._BackwardsCompatibleNumpyRng(seed + 19823)
         self.seed = seed
         self.init_config = init_config
         self.best_config = flatten_dict(init_config)
diff --git a/flaml/searcher/online_searcher.py b/flaml/tune/searcher/online_searcher.py
similarity index 99%
rename from flaml/searcher/online_searcher.py
rename to flaml/tune/searcher/online_searcher.py
index 81a98c458..d56710f09 100644
--- a/flaml/searcher/online_searcher.py
+++ b/flaml/tune/searcher/online_searcher.py
@@ -2,10 +2,9 @@ import numpy as np
 import logging
 import itertools
 from typing import Dict, Optional, List
-from flaml.tune import Categorical, Float, PolynomialExpansionSet
-from flaml.tune import Trial
+from flaml.tune import Categorical, Float, PolynomialExpansionSet, Trial
 from flaml.onlineml import VowpalWabbitTrial
-from flaml.searcher import CFO
+from flaml.tune.searcher import CFO
 
 logger = logging.getLogger(__name__)
 
diff --git a/flaml/searcher/search_thread.py b/flaml/tune/searcher/search_thread.py
similarity index 99%
rename from flaml/searcher/search_thread.py
rename to flaml/tune/searcher/search_thread.py
index 0d39507bd..d61089e49 100644
--- a/flaml/searcher/search_thread.py
+++ b/flaml/tune/searcher/search_thread.py
@@ -16,7 +16,7 @@ try:
 except (ImportError, AssertionError):
     from .suggestion import Searcher
 from .flow2 import FLOW2
-from ..tune.space import add_cost_to_space, unflatten_hierarchical
+from ..space import add_cost_to_space, unflatten_hierarchical
 import logging
 
 logger = logging.getLogger(__name__)
diff --git a/flaml/searcher/suggestion.py b/flaml/tune/searcher/suggestion.py
similarity index 97%
rename from flaml/searcher/suggestion.py
rename to flaml/tune/searcher/suggestion.py
index 30105dcae..a349c2f14 100644
--- a/flaml/searcher/suggestion.py
+++ b/flaml/tune/searcher/suggestion.py
@@ -23,7 +23,7 @@ import logging
 from typing import Any, Dict, Optional, Union, List, Tuple, Callable
 import pickle
 from .variant_generator import parse_spec_vars
-from ..tune.sample import (
+from ..sample import (
     Categorical,
     Domain,
     Float,
@@ -32,7 +32,7 @@ from ..tune.sample import (
     Quantized,
     Uniform,
 )
-from ..tune.trial import flatten_dict, unflatten_dict
+from ..trial import flatten_dict, unflatten_dict
 
 logger = logging.getLogger(__name__)
 
diff --git a/flaml/searcher/variant_generator.py b/flaml/tune/searcher/variant_generator.py
similarity index 96%
rename from flaml/searcher/variant_generator.py
rename to flaml/tune/searcher/variant_generator.py
index 388b0b574..fb0c8ed0e 100644
--- a/flaml/searcher/variant_generator.py
+++ b/flaml/tune/searcher/variant_generator.py
@@ -20,7 +20,7 @@ import logging
 from typing import Any, Dict, Generator, List, Tuple
 import numpy
 import random
-from ..tune.sample import Categorical, Domain, RandomState
+from ..sample import Categorical, Domain, RandomState
 
 try:
     from ray import __version__ as ray_version
diff --git a/flaml/tune/space.py b/flaml/tune/space.py
index 1c0b5c184..a6b5090e7 100644
--- a/flaml/tune/space.py
+++ b/flaml/tune/space.py
@@ -10,7 +10,7 @@ try:
         from ray.tune.search.variant_generator import generate_variants
 except (ImportError, AssertionError):
     from . import sample
-    from ..searcher.variant_generator import generate_variants
+    from .searcher.variant_generator import generate_variants
 from typing import Dict, Optional, Any, Tuple, Generator
 import numpy as np
 import logging
diff --git a/flaml/tune/tune.py b/flaml/tune/tune.py
index 9209dccd1..56e8dbaed 100644
--- a/flaml/tune/tune.py
+++ b/flaml/tune/tune.py
@@ -355,7 +355,7 @@ def run(
         else:
             logger.setLevel(logging.CRITICAL)
 
-    from ..searcher.blendsearch import BlendSearch, CFO
+    from .searcher.blendsearch import BlendSearch, CFO
 
     if search_alg is None:
         flaml_scheduler_resource_attr = (
@@ -409,7 +409,7 @@ def run(
             else:
                 from ray.tune.search import ConcurrencyLimiter
         else:
-            from flaml.searcher.suggestion import ConcurrencyLimiter
+            from flaml.tune.searcher.suggestion import ConcurrencyLimiter
         if (
             search_alg.__class__.__name__
             in [
diff --git a/test/nni/flaml_nni_wrap.py b/test/nni/flaml_nni_wrap.py
index f81d04164..bc76e05cf 100644
--- a/test/nni/flaml_nni_wrap.py
+++ b/test/nni/flaml_nni_wrap.py
@@ -1,4 +1,4 @@
-from flaml.searcher.blendsearch import BlendSearchTuner as BST
+from flaml.tune.searcher.blendsearch import BlendSearchTuner as BST
 
 
 class BlendSearchTuner(BST):
diff --git a/test/tune/example_scheduler.py b/test/tune/example_scheduler.py
index 5fc5e2854..e3d11320d 100644
--- a/test/tune/example_scheduler.py
+++ b/test/tune/example_scheduler.py
@@ -26,7 +26,7 @@ def easy_objective(use_raytune, config):
 
 def test_tune_scheduler(smoke_test=True, use_ray=True, use_raytune=False):
     import numpy as np
-    from flaml.searcher.blendsearch import BlendSearch
+    from flaml.tune.searcher.blendsearch import BlendSearch
 
     np.random.seed(100)
     easy_objective_custom_tune = partial(easy_objective, use_raytune)
diff --git a/test/tune/test_flaml_raytune_consistency.py b/test/tune/test_flaml_raytune_consistency.py
index 98b36e5df..736a64d9b 100644
--- a/test/tune/test_flaml_raytune_consistency.py
+++ b/test/tune/test_flaml_raytune_consistency.py
@@ -28,7 +28,7 @@ low_cost_partial_config = {"x": 1}
 
 
 def setup_searcher(searcher_name):
-    from flaml.searcher.blendsearch import BlendSearch, CFO, RandomSearch
+    from flaml.tune.searcher.blendsearch import BlendSearch, CFO, RandomSearch
 
     if "cfo" in searcher_name:
         searcher = CFO(
diff --git a/test/tune/test_record_incumbent.py b/test/tune/test_record_incumbent.py
index b267ee0f9..9cec3c1da 100644
--- a/test/tune/test_record_incumbent.py
+++ b/test/tune/test_record_incumbent.py
@@ -7,7 +7,7 @@ def rosenbrock_function(config: dict):
     funcLoss = 50
     for key, value in config.items():
         if key in ["x1", "x2", "x3", "x4", "x5"]:
-            funcLoss += value ** 2 - 10 * np.cos(2 * np.pi * value)
+            funcLoss += value**2 - 10 * np.cos(2 * np.pi * value)
     if INCUMBENT_RESULT in config.keys():
         print("----------------------------------------------")
         print("incumbent result", config[INCUMBENT_RESULT])
@@ -62,7 +62,7 @@ def test_record_incumbent(method="BlendSearch"):
             use_incumbent_result_in_evaluation=True,
         )
     elif method == "CFOCat":
-        from flaml.searcher.cfo_cat import CFOCat
+        from flaml.tune.searcher.cfo_cat import CFOCat
 
         algo = CFOCat(
             use_incumbent_result_in_evaluation=True,
diff --git a/test/tune/test_reproducibility.py b/test/tune/test_reproducibility.py
index c0229b8cf..974bdfd57 100644
--- a/test/tune/test_reproducibility.py
+++ b/test/tune/test_reproducibility.py
@@ -26,7 +26,7 @@ def _easy_objective(use_raytune, config):
 
 def test_tune(externally_setup_searcher=False, use_ray=False, use_raytune=False):
     from flaml import tune
-    from flaml.searcher.blendsearch import BlendSearch
+    from flaml.tune.searcher.blendsearch import BlendSearch
 
     easy_objective_custom_tune = partial(_easy_objective, use_raytune)
     search_space = {
diff --git a/test/tune/test_restore.py b/test/tune/test_restore.py
index 7a047e6d4..ef3c95425 100644
--- a/test/tune/test_restore.py
+++ b/test/tune/test_restore.py
@@ -3,7 +3,7 @@ import shutil
 import tempfile
 import unittest
 import numpy as np
-from flaml.searcher.suggestion import ConcurrencyLimiter
+from flaml.tune.searcher.suggestion import ConcurrencyLimiter
 from flaml import tune
 from flaml import CFO
 
diff --git a/test/tune/test_scheduler.py b/test/tune/test_scheduler.py
index 52bf50092..e12f0f6bb 100644
--- a/test/tune/test_scheduler.py
+++ b/test/tune/test_scheduler.py
@@ -1,6 +1,6 @@
 """Require: pip install flaml[test,ray]
 """
-from flaml.scheduler.trial_scheduler import TrialScheduler
+from flaml.tune.scheduler.trial_scheduler import TrialScheduler
 import numpy as np
 from flaml import tune
 
diff --git a/test/tune/test_searcher.py b/test/tune/test_searcher.py
index 7c1e70c98..440c08ab0 100644
--- a/test/tune/test_searcher.py
+++ b/test/tune/test_searcher.py
@@ -32,8 +32,12 @@ def wrong_define_search_space(trial):
 
 
 def test_searcher():
-    from flaml.searcher.suggestion import OptunaSearch, Searcher, ConcurrencyLimiter
-    from flaml.searcher.blendsearch import BlendSearch, CFO, RandomSearch
+    from flaml.tune.searcher.suggestion import (
+        OptunaSearch,
+        Searcher,
+        ConcurrencyLimiter,
+    )
+    from flaml.tune.searcher.blendsearch import BlendSearch, CFO, RandomSearch
     from flaml.tune import sample as flamlsample
 
     searcher = Searcher()
@@ -306,6 +310,6 @@ def test_no_optuna():
     import sys
 
     subprocess.check_call([sys.executable, "-m", "pip", "uninstall", "-y", "optuna"])
-    import flaml.searcher.suggestion
+    import flaml.tune.searcher.suggestion
 
     subprocess.check_call([sys.executable, "-m", "pip", "install", "optuna==2.8.0"])
diff --git a/test/tune/test_space.py b/test/tune/test_space.py
index f0c018b63..0081d1ffb 100644
--- a/test/tune/test_space.py
+++ b/test/tune/test_space.py
@@ -69,7 +69,7 @@ def test_define_by_run():
 
 
 def test_grid():
-    from flaml.searcher.variant_generator import (
+    from flaml.tune.searcher.variant_generator import (
         generate_variants,
         grid_search,
         TuneError,
diff --git a/test/tune/test_tune.py b/test/tune/test_tune.py
index 3ba47b9c8..8608461b9 100644
--- a/test/tune/test_tune.py
+++ b/test/tune/test_tune.py
@@ -1,6 +1,6 @@
 """Require: pip install flaml[test,ray]
 """
-from flaml.searcher.blendsearch import BlendSearch
+from flaml import BlendSearch
 import time
 import os
 from sklearn.model_selection import train_test_split
@@ -146,7 +146,7 @@ def _test_xgboost(method="BlendSearch"):
                         },
                     )
                 elif "CFOCat" == method:
-                    from flaml.searcher.cfo_cat import CFOCat
+                    from flaml.tune.searcher.cfo_cat import CFOCat
 
                     algo = CFOCat(
                         low_cost_partial_config={