wangtao
/
autogen
mirror of https://github.com/microsoft/autogen.git
1
0
Fork 1
Code Issues Packages Projects Releases Wiki Activity

cleanup

This commit is contained in:
Chi Wang 2023-09-16 15:30:28 +00:00
parent bc4473fe8a
commit 812db59d33
51 changed files with 64 additions and 7238 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.devcontainer/devcontainer.json
View File

@ -9,5 +9,5 @@
},
"terminal.integrated.defaultProfile.linux": "bash"
},
"updateContentCommand": "pip install -e .[notebook,openai] pre-commit && pre-commit install"
"updateContentCommand": "pip install -e . pre-commit && pre-commit install"
}

17
.github/workflows/openai.yml vendored
View File

@ -7,8 +7,11 @@ on:
pull_request:
branches: ['main']
paths:
- 'flaml/autogen/**'
- 'test/autogen/**'
- 'autogen/**'
- 'test/**'
- 'notebook/autogen_agentchat_auto_feedback_from_code_execution.ipynb'
- 'notebook/autogen_agentchat_function_call.ipynb'
- 'notebook/autogen_agentchat_MathChat.ipynb'
- 'notebook/autogen_openai_completion.ipynb'
- 'notebook/autogen_chatgpt_gpt4.ipynb'
- '.github/workflows/openai.yml'
@ -31,8 +34,8 @@ jobs:
run: |
docker --version
python -m pip install --upgrade pip wheel
pip install -e .[autogen,blendsearch]
python -c "import flaml"
pip install -e .[tune]
python -c "import autogen"
pip install coverage pytest datasets
- name: Install packages for test when needed
if: matrix.python-version == '3.9'
@ -54,7 +57,7 @@ jobs:
AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
run: |
coverage run -a -m pytest test/autogen
coverage run -a -m pytest test
coverage xml
- name: Coverage and check notebook outputs
if: matrix.python-version != '3.9'
@ -66,9 +69,9 @@ jobs:
OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }}
run: |
pip install nbconvert nbformat ipykernel
coverage run -a -m pytest test/autogen/test_notebook.py
coverage run -a -m pytest test/test_notebook.py
coverage xml
cat "$(pwd)/test/autogen/executed_openai_notebook_output.txt"
cat "$(pwd)/test/executed_openai_notebook_output.txt"
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v3
with:

66
.github/workflows/python-package.yml vendored
View File

@ -36,51 +36,12 @@ jobs:
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: On mac + python 3.10, install libomp to facilitate lgbm and xgboost install
if: matrix.os == 'macOS-latest' && matrix.python-version == '3.10'
run: |
# remove libomp version constraint after xgboost works with libomp>11.1.0 on python 3.10
wget https://raw.githubusercontent.com/Homebrew/homebrew-core/679923b4eb48a8dc7ecc1f05d06063cd79b3fc00/Formula/libomp.rb -O $(find $(brew --repository) -name libomp.rb)
brew unlink libomp
brew install libomp
export CC=/usr/bin/clang
export CXX=/usr/bin/clang++
export CPPFLAGS="$CPPFLAGS -Xpreprocessor -fopenmp"
export CFLAGS="$CFLAGS -I/usr/local/opt/libomp/include"
export CXXFLAGS="$CXXFLAGS -I/usr/local/opt/libomp/include"
export LDFLAGS="$LDFLAGS -Wl,-rpath,/usr/local/opt/libomp/lib -L/usr/local/opt/libomp/lib -lomp"
- name: Install packages and dependencies
run: |
python -m pip install --upgrade pip wheel
pip install -e .
python -c "import flaml"
python -c "import autogen"
pip install -e .[test]
- name: On Ubuntu python 3.8, install pyspark 3.2.3
if: matrix.python-version == '3.8' && matrix.os == 'ubuntu-latest'
run: |
pip install pyspark==3.2.3
pip list | grep "pyspark"
- name: If linux, install ray 2
if: matrix.os == 'ubuntu-latest'
run: |
pip install "ray[tune]<2.5.0"
- name: If mac, install ray
if: matrix.os == 'macOS-latest'
run: |
pip install -e .[ray]
- name: If linux or mac, install prophet on python < 3.9
if: (matrix.os == 'macOS-latest' || matrix.os == 'ubuntu-latest') && matrix.python-version != '3.9' && matrix.python-version != '3.10'
run: |
pip install -e .[forecast]
- name: Install vw on python < 3.10
if: matrix.python-version != '3.10'
run: |
pip install -e .[vw]
- name: Uninstall pyspark on (python 3.9) or (python 3.8 + windows)
if: matrix.python-version == '3.9' || (matrix.python-version == '3.8' && matrix.os == 'windows-2019')
run: |
# Uninstall pyspark to test env without pyspark
pip uninstall -y pyspark
- name: Test with pytest
if: matrix.python-version != '3.10'
run: |
@ -97,28 +58,3 @@ jobs:
with:
file: ./coverage.xml
flags: unittests
# docs:
# runs-on: ubuntu-latest
# steps:
# - uses: actions/checkout@v3
# - name: Setup Python
# uses: actions/setup-python@v4
# with:
# python-version: '3.8'
# - name: Compile documentation
# run: |
# pip install -e .
# python -m pip install sphinx sphinx_rtd_theme
# cd docs
# make html
# - name: Deploy to GitHub pages
# if: ${{ github.ref == 'refs/heads/main' }}
# uses: JamesIves/github-pages-deploy-action@3.6.2
# with:
# GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# BRANCH: gh-pages
# FOLDER: docs/_build/html
# CLEAN: true

19
README.md
View File

@ -5,25 +5,24 @@ This project welcomes contributions and suggestions. Most contributions require
Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us
the rights to use your contribution. For details, visit https://cla.opensource.microsoft.com.
[![PyPI version](https://badge.fury.io/py/pyautogen.svg)](https://badge.fury.io/py/pyautogen)
<!-- ![Conda version](https://img.shields.io/conda/vn/conda-forge/flaml) -->
<!-- ![Conda version](https://img.shields.io/conda/vn/conda-forge/pyautogen) -->
[![Build](https://github.com/microsoft/autogen/actions/workflows/python-package.yml/badge.svg)](https://github.com/microsoft/autogen/actions/workflows/python-package.yml)
![Python Version](https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10-blue)
<!-- [![Downloads](https://pepy.tech/badge/flaml)](https://pepy.tech/project/flaml) -->
<!-- [![Downloads](https://pepy.tech/badge/pyautogen)](https://pepy.tech/project/pyautogen) -->
[![](https://img.shields.io/discord/1025786666260111483?logo=discord&style=flat)](https://discord.gg/Cppx2vSPVP)
<!-- [![Join the chat at https://gitter.im/FLAMLer/community](https://badges.gitter.im/FLAMLer/community.svg)](https://gitter.im/FLAMLer/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) -->
This project is a spinoff from [FLAML](https://github.com/microsoft/FLAML).
# AutoGen
<!-- <p align="center">
<img src="https://github.com/microsoft/FLAML/blob/main/website/static/img/flaml.svg" width=200>
<img src="https://github.com/microsoft/autogen/blob/main/website/static/img/flaml.svg" width=200>
<br>
</p> -->
<!-- :fire: Heads-up: We're preparing to migrate [autogen](https://microsoft.github.io/FLAML/docs/Use-Cases/Autogen) into a dedicated github repository. Alongside this move, we'll also launch a dedicated Discord server and a website for comprehensive documentation.
:fire: autogen has graduated from [FLAML](https://github.com/microsoft/FLAML) into a new project.
:fire: The automated multi-agent chat framework in [autogen](https://microsoft.github.io/FLAML/docs/Use-Cases/Autogen) is in preview from v2.0.0.
<!-- :fire: Heads-up: We're preparing to migrate [autogen](https://microsoft.github.io/FLAML/docs/Use-Cases/Autogen) into a dedicated github repository. Alongside this move, we'll also launch a dedicated Discord server and a website for comprehensive documentation.
:fire: FLAML is highlighted in OpenAI's [cookbook](https://github.com/openai/openai-cookbook#related-resources-from-around-the-web).
@ -54,14 +53,14 @@ AutoGen requires **Python version >= 3.8**. It can be installed from pip:
pip install pyautogen
```
<!-- Minimal dependencies are installed without extra options. You can install extra options based on the feature you need.
For example, use the following to install the dependencies needed by the [`autogen`](https://microsoft.github.io/FLAML/docs/Use-Cases/Autogen) package.
Minimal dependencies are installed without extra options. You can install extra options based on the feature you need.
For example, use the following to install the dependencies needed by the [`blendsearch`](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function#blendsearch-economical-hyperparameter-optimization-with-blended-search-strategy) option.
```bash
pip install "flaml[autogen]"
pip install "pyautogen[blendsearch]"
```
Find more options in [Installation](https://microsoft.github.io/autogen/docs/Installation).
Each of the [`notebook examples`](https://github.com/microsoft/FLAML/tree/main/notebook) may require a specific option to be installed. -->
<!-- Each of the [`notebook examples`](https://github.com/microsoft/autogen/tree/main/notebook) may require a specific option to be installed. -->
## Quickstart

10
autogen/__init__.py Normal file
View File

@ -0,0 +1,10 @@
import logging
from .version import __version__
from .oai import *
from .agentchat import *
from .code_utils import DEFAULT_MODEL, FAST_MODEL
# Set the root logger.
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

0
flaml/autogen/agentchat/__init__.py → autogen/agentchat/__init__.py
View File

0
flaml/autogen/agentchat/agent.py → autogen/agentchat/agent.py
View File

0
flaml/autogen/agentchat/assistant_agent.py → autogen/agentchat/assistant_agent.py
View File

6
flaml/autogen/agentchat/contrib/math_user_proxy_agent.py → autogen/agentchat/contrib/math_user_proxy_agent.py
View File

@ -4,9 +4,9 @@ from pydantic import BaseModel, Extra, root_validator
from typing import Any, Callable, Dict, List, Optional, Union
from time import sleep
from flaml.autogen.agentchat import Agent, UserProxyAgent
from flaml.autogen.code_utils import UNKNOWN, extract_code, execute_code, infer_lang
from flaml.autogen.math_utils import get_answer
from autogen.agentchat import Agent, UserProxyAgent
from autogen.code_utils import UNKNOWN, extract_code, execute_code, infer_lang
from autogen.math_utils import get_answer
PROMPTS = {

4
flaml/autogen/agentchat/contrib/retrieve_assistant_agent.py → autogen/agentchat/contrib/retrieve_assistant_agent.py
View File

@ -1,5 +1,5 @@
from flaml.autogen.agentchat.agent import Agent
from flaml.autogen.agentchat.assistant_agent import AssistantAgent
from agentchat.agent import Agent
from agentchat.assistant_agent import AssistantAgent
from typing import Callable, Dict, Optional, Union, List, Tuple, Any

12
flaml/autogen/agentchat/contrib/retrieve_user_proxy_agent.py → autogen/agentchat/contrib/retrieve_user_proxy_agent.py
View File

@ -1,8 +1,8 @@
import chromadb
from flaml.autogen.agentchat.agent import Agent
from flaml.autogen.agentchat import UserProxyAgent
from flaml.autogen.retrieve_utils import create_vector_db_from_dir, query_vector_db, num_tokens_from_text
from flaml.autogen.code_utils import extract_code
from autogen.agentchat.agent import Agent
from autogen.agentchat import UserProxyAgent
from autogen.retrieve_utils import create_vector_db_from_dir, query_vector_db, num_tokens_from_text
from autogen.code_utils import extract_code
from typing import Callable, Dict, Optional, Union, List, Tuple, Any
from IPython import get_ipython
@ -106,7 +106,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):
- docs_path (Optional, str): the path to the docs directory. It can also be the path to a single file,
or the url to a single file. If key not provided, a default path `./docs` will be used.
- collection_name (Optional, str): the name of the collection.
If key not provided, a default name `flaml-docs` will be used.
If key not provided, a default name `autogen-docs` will be used.
- model (Optional, str): the model to use for the retrieve chat.
If key not provided, a default model `gpt-4` will be used.
- chunk_token_size (Optional, int): the chunk token size for the retrieve chat.
@ -135,7 +135,7 @@ class RetrieveUserProxyAgent(UserProxyAgent):
self._task = self._retrieve_config.get("task", "default")
self._client = self._retrieve_config.get("client", chromadb.Client())
self._docs_path = self._retrieve_config.get("docs_path", "./docs")
self._collection_name = self._retrieve_config.get("collection_name", "flaml-docs")
self._collection_name = self._retrieve_config.get("collection_name", "autogen-docs")
self._model = self._retrieve_config.get("model", "gpt-4")
self._max_tokens = self.get_max_tokens(self._model)
self._chunk_token_size = int(self._retrieve_config.get("chunk_token_size", self._max_tokens * 0.4))

6
flaml/autogen/agentchat/conversable_agent.py → autogen/agentchat/conversable_agent.py
View File

@ -3,9 +3,9 @@ from collections import defaultdict
import copy
import json
from typing import Any, Callable, Dict, List, Optional, Tuple, Type, Union
from flaml.autogen import oai
from autogen import oai
from .agent import Agent
from flaml.autogen.code_utils import (
from autogen.code_utils import (
DEFAULT_MODEL,
UNKNOWN,
execute_code,
@ -78,7 +78,7 @@ class ConversableAgent(Agent):
- work_dir (Optional, str): The working directory for the code execution.
If None, a default working directory will be used.
The default working directory is the "extensions" directory under
"path_to_flaml/autogen".
"path_to_autogen".
- use_docker (Optional, list, str or bool): The docker image to use for code execution.
If a list or a str of image name(s) is provided, the code will be executed in a docker container
with the first image successfully pulled.

0
flaml/autogen/agentchat/groupchat.py → autogen/agentchat/groupchat.py
View File

2
flaml/autogen/agentchat/user_proxy_agent.py → autogen/agentchat/user_proxy_agent.py
View File

@ -51,7 +51,7 @@ class UserProxyAgent(ConversableAgent):
- work_dir (Optional, str): The working directory for the code execution.
If None, a default working directory will be used.
The default working directory is the "extensions" directory under
"path_to_flaml/autogen".
"path_to_autogen".
- use_docker (Optional, list, str or bool): The docker image to use for code execution.
If a list or a str of image name(s) is provided, the code will be executed in a docker container
with the first image successfully pulled.

4
flaml/autogen/code_utils.py → autogen/code_utils.py
View File

@ -8,7 +8,7 @@ import re
import time
from hashlib import md5
import logging
from flaml.autogen import oai
from autogen import oai
try:
import docker
@ -205,7 +205,7 @@ def execute_code(
work_dir (Optional, str): The working directory for the code execution.
If None, a default working directory will be used.
The default working directory is the "extensions" directory under
"path_to_flaml/autogen".
"path_to_autogen".
use_docker (Optional, list, str or bool): The docker image to use for code execution.
If a list or a str of image name(s) is provided, the code will be executed in a docker container
with the first image successfully pulled.

0
flaml/autogen/extensions/__init__.py → autogen/extensions/__init__.py
View File

2
flaml/autogen/math_utils.py → autogen/math_utils.py
View File

@ -1,5 +1,5 @@
from typing import Optional
from flaml.autogen import oai, DEFAULT_MODEL
from autogen import oai, DEFAULT_MODEL
_MATH_PROMPT = "{problem} Solve the problem carefully. Simplify your answer as much as possible. Put the final answer in \\boxed{{}}."
_MATH_CONFIG = {

4
flaml/autogen/oai/__init__.py → autogen/oai/__init__.py
View File

@ -1,5 +1,5 @@
from flaml.autogen.oai.completion import Completion, ChatCompletion
from flaml.autogen.oai.openai_utils import (
from autogen.oai.completion import Completion, ChatCompletion
from autogen.oai.openai_utils import (
get_config_list,
config_list_gpt4_gpt35,
config_list_openai_aoai,

2
flaml/autogen/oai/completion.py → autogen/oai/completion.py
View File

@ -26,7 +26,7 @@ try:
ERROR = None
except ImportError:
ERROR = ImportError("please install flaml[openai] option to use the flaml.autogen.oai subpackage.")
ERROR = ImportError("please install openai and diskcache to use the autogen.oai subpackage.")
openai_Completion = object
logger = logging.getLogger(__name__)
if not logger.handlers:

0
flaml/autogen/oai/openai_utils.py → autogen/oai/openai_utils.py
View File

0
flaml/autogen/retrieve_utils.py → autogen/retrieve_utils.py
View File

1
autogen/version.py Normal file
View File

@ -0,0 +1 @@
__version__ = "0.1.0"

10
flaml/__init__.py
View File

@ -1,10 +0,0 @@
import logging
from flaml.automl import AutoML, logger_formatter
from flaml.tune.searcher import CFO, BlendSearch, FLOW2, BlendSearchTuner, RandomSearch
from flaml.onlineml.autovw import AutoVW
from flaml.version import __version__
# Set the root logger.
logger = logging.getLogger(__name__)
logger.setLevel(logging.INFO)

3
flaml/autogen/__init__.py
View File

@ -1,3 +0,0 @@
from .oai import *
from .agentchat import *
from .code_utils import DEFAULT_MODEL, FAST_MODEL

217
flaml/tune/README.md
View File

@ -1,217 +0,0 @@
# Economical Hyperparameter Optimization
`flaml.tune` is a module for economical hyperparameter tuning. It frees users from manually tuning many hyperparameters for a software, such as machine learning training procedures.
It can be used standalone, or together with ray tune or nni. Please find detailed guidelines and use cases about this module in our [documentation website](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function).
Below are some quick examples.
* Example for sequential tuning (recommended when compute resource is limited and each trial can consume all the resources):
```python
# require: pip install flaml[blendsearch]
from flaml import tune
import time
def evaluate_config(config):
'''evaluate a hyperparameter configuration'''
# we uss a toy example with 2 hyperparameters
metric = (round(config['x'])-85000)**2 - config['x']/config['y']
# usually the evaluation takes an non-neglible cost
# and the cost could be related to certain hyperparameters
# in this example, we assume it's proportional to x
time.sleep(config['x']/100000)
# use tune.report to report the metric to optimize
tune.report(metric=metric)
analysis = tune.run(
evaluate_config, # the function to evaluate a config
config={
'x': tune.lograndint(lower=1, upper=100000),
'y': tune.randint(lower=1, upper=100000)
}, # the search space
low_cost_partial_config={'x':1}, # a initial (partial) config with low cost
metric='metric', # the name of the metric used for optimization
mode='min', # the optimization mode, 'min' or 'max'
num_samples=-1, # the maximal number of configs to try, -1 means infinite
time_budget_s=60, # the time budget in seconds
local_dir='logs/', # the local directory to store logs
# verbose=0, # verbosity
# use_ray=True, # uncomment when performing parallel tuning using ray
)
print(analysis.best_trial.last_result) # the best trial's result
print(analysis.best_config) # the best config
```
* Example for using ray tune's API:
```python
# require: pip install flaml[blendsearch,ray]
from ray import tune as raytune
from flaml import CFO, BlendSearch
import time
def evaluate_config(config):
'''evaluate a hyperparameter configuration'''
# we use a toy example with 2 hyperparameters
metric = (round(config['x'])-85000)**2 - config['x']/config['y']
# usually the evaluation takes a non-neglible cost
# and the cost could be related to certain hyperparameters
# in this example, we assume it's proportional to x
time.sleep(config['x']/100000)
# use tune.report to report the metric to optimize
tune.report(metric=metric)
# provide a time budget (in seconds) for the tuning process
time_budget_s = 60
# provide the search space
config_search_space = {
'x': tune.lograndint(lower=1, upper=100000),
'y': tune.randint(lower=1, upper=100000)
}
# provide the low cost partial config
low_cost_partial_config={'x':1}
# set up CFO
cfo = CFO(low_cost_partial_config=low_cost_partial_config)
# set up BlendSearch
blendsearch = BlendSearch(
metric="metric", mode="min",
space=config_search_space,
low_cost_partial_config=low_cost_partial_config,
time_budget_s=time_budget_s
)
# NOTE: when using BlendSearch as a search_alg in ray tune, you need to
# configure the 'time_budget_s' for BlendSearch accordingly such that
# BlendSearch is aware of the time budget. This step is not needed when
# BlendSearch is used as the search_alg in flaml.tune as it is done
# automatically in flaml.
analysis = raytune.run(
evaluate_config, # the function to evaluate a config
config=config_search_space,
metric='metric', # the name of the metric used for optimization
mode='min', # the optimization mode, 'min' or 'max'
num_samples=-1, # the maximal number of configs to try, -1 means infinite
time_budget_s=time_budget_s, # the time budget in seconds
local_dir='logs/', # the local directory to store logs
search_alg=blendsearch # or cfo
)
print(analysis.best_trial.last_result) # the best trial's result
print(analysis.best_config) # the best config
```
* Example for using NNI: An example of using BlendSearch with NNI can be seen in [test](https://github.com/microsoft/FLAML/tree/main/test/nni). CFO can be used as well in a similar manner. To run the example, first make sure you have [NNI](https://nni.readthedocs.io/en/stable/) installed, then run:
```shell
$nnictl create --config ./config.yml
```
* For more examples, please check out
[notebooks](https://github.com/microsoft/FLAML/tree/main/notebook/).
`flaml` offers two HPO methods: CFO and BlendSearch.
`flaml.tune` uses BlendSearch by default.
## CFO: Frugal Optimization for Cost-related Hyperparameters
<p align="center">
<img src="https://github.com/microsoft/FLAML/blob/main/website/docs/Use-Cases/images/CFO.png" width=200>
<br>
</p>
CFO uses the randomized direct search method FLOW<sup>2</sup> with adaptive stepsize and random restart.
It requires a low-cost initial point as input if such point exists.
The search begins with the low-cost initial point and gradually move to
high cost region if needed. The local search method has a provable convergence
rate and bounded cost.
About FLOW<sup>2</sup>: FLOW<sup>2</sup> is a simple yet effective randomized direct search method.
It is an iterative optimization method that can optimize for black-box functions.
FLOW<sup>2</sup> only requires pairwise comparisons between function values to perform iterative update. Comparing to existing HPO methods, FLOW<sup>2</sup> has the following appealing properties:
1. It is applicable to general black-box functions with a good convergence rate in terms of loss.
1. It provides theoretical guarantees on the total evaluation cost incurred.
The GIFs attached below demonstrate an example search trajectory of FLOW<sup>2</sup> shown in the loss and evaluation cost (i.e., the training time ) space respectively. From the demonstration, we can see that (1) FLOW<sup>2</sup> can quickly move toward the low-loss region, showing good convergence property and (2) FLOW<sup>2</sup> tends to avoid exploring the high-cost region until necessary.
<p align="center">
<img align="center", src="https://github.com/microsoft/FLAML/blob/website/docs/Use-Cases/images/heatmap_loss_cfo_12s.gif" width=360> <img align="center", src="https://github.com/microsoft/FLAML/blob/main/website/docs/Use-Cases/images/heatmap_cost_cfo_12s.gif" width=360>
<br>
<figcaption>Figure 1. FLOW<sup>2</sup> in tuning the # of leaves and the # of trees for XGBoost. The two background heatmaps show the loss and cost distribution of all configurations. The black dots are the points evaluated in FLOW<sup>2</sup>. Black dots connected by lines are points that yield better loss performance when evaluated.</figcaption>
</p>
Example:
```python
from flaml import CFO
tune.run(...
search_alg = CFO(low_cost_partial_config=low_cost_partial_config),
)
```
Recommended scenario: there exist cost-related hyperparameters and a low-cost
initial point is known before optimization.
If the search space is complex and CFO gets trapped into local optima, consider
using BlendSearch.
## BlendSearch: Economical Hyperparameter Optimization With Blended Search Strategy
<p align="center">
<img src="https://github.com/microsoft/FLAML/blob/main/website/docs/Use-Cases/images/BlendSearch.png" width=200>
<br>
</p>
BlendSearch combines local search with global search. It leverages the frugality
of CFO and the space exploration ability of global search methods such as
Bayesian optimization. Like CFO, BlendSearch requires a low-cost initial point
as input if such point exists, and starts the search from there. Different from
CFO, BlendSearch will not wait for the local search to fully converge before
trying new start points. The new start points are suggested by the global search
method and filtered based on their distance to the existing points in the
cost-related dimensions. BlendSearch still gradually increases the trial cost.
It prioritizes among the global search thread and multiple local search threads
based on optimism in face of uncertainty.
Example:
```python
# require: pip install flaml[blendsearch]
from flaml import BlendSearch
tune.run(...
search_alg = BlendSearch(low_cost_partial_config=low_cost_partial_config),
)
```
* Recommended scenario: cost-related hyperparameters exist, a low-cost
initial point is known, and the search space is complex such that local search
is prone to be stuck at local optima.
* Suggestion about using larger search space in BlendSearch:
In hyperparameter optimization, a larger search space is desirable because it is more likely to include the optimal configuration (or one of the optimal configurations) in hindsight. However the performance (especially anytime performance) of most existing HPO methods is undesirable if the cost of the configurations in the search space has a large variation. Thus hand-crafted small search spaces (with relatively homogeneous cost) are often used in practice for these methods, which is subject to idiosyncrasy. BlendSearch combines the benefits of local search and global search, which enables a smart (economical) way of deciding where to explore in the search space even though it is larger than necessary. This allows users to specify a larger search space in BlendSearch, which is often easier and a better practice than narrowing down the search space by hand.
For more technical details, please check our papers.
* [Frugal Optimization for Cost-related Hyperparameters](https://arxiv.org/abs/2005.01571). Qingyun Wu, Chi Wang, Silu Huang. AAAI 2021.
```bibtex
@inproceedings{wu2021cfo,
title={Frugal Optimization for Cost-related Hyperparameters},
author={Qingyun Wu and Chi Wang and Silu Huang},
year={2021},
booktitle={AAAI'21},
}
```
* [Economical Hyperparameter Optimization With Blended Search Strategy](https://www.microsoft.com/en-us/research/publication/economical-hyperparameter-optimization-with-blended-search-strategy/). Chi Wang, Qingyun Wu, Silu Huang, Amin Saied. ICLR 2021.
```bibtex
@inproceedings{wang2021blendsearch,
title={Economical Hyperparameter Optimization With Blended Search Strategy},
author={Chi Wang and Qingyun Wu and Silu Huang and Amin Saied},
year={2021},
booktitle={ICLR'21},
}
```

40
flaml/tune/__init__.py
View File

@ -1,40 +0,0 @@
try:
from ray import __version__ as ray_version
assert ray_version >= "1.10.0"
from ray.tune import (
uniform,
quniform,
randint,
qrandint,
randn,
qrandn,
loguniform,
qloguniform,
lograndint,
qlograndint,
)
if ray_version.startswith("1."):
from ray.tune import sample
else:
from ray.tune.search import sample
except (ImportError, AssertionError):
from .sample import (
uniform,
quniform,
randint,
qrandint,
randn,
qrandn,
loguniform,
qloguniform,
lograndint,
qlograndint,
)
from . import sample
from .tune import run, report, INCUMBENT_RESULT
from .sample import polynomial_expansion_set
from .sample import PolynomialExpansionSet, Categorical, Float
from .trial import Trial
from .utils import choice

204
flaml/tune/analysis.py
View File

@ -1,204 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
from typing import Dict, Optional
import numpy as np
from .trial import Trial
import logging
logger = logging.getLogger(__name__)
def is_nan_or_inf(value):
return np.isnan(value) or np.isinf(value)
class ExperimentAnalysis:
"""Analyze results from a Tune experiment."""
@property
def best_trial(self) -> Trial:
"""Get the best trial of the experiment
The best trial is determined by comparing the last trial results
using the `metric` and `mode` parameters passed to `tune.run()`.
If you didn't pass these parameters, use
`get_best_trial(metric, mode, scope)` instead.
"""
if not self.default_metric or not self.default_mode:
raise ValueError(
"To fetch the `best_trial`, pass a `metric` and `mode` "
"parameter to `tune.run()`. Alternatively, use the "
"`get_best_trial(metric, mode)` method to set the metric "
"and mode explicitly."
)
return self.get_best_trial(self.default_metric, self.default_mode)
@property
def best_config(self) -> Dict:
"""Get the config of the best trial of the experiment
The best trial is determined by comparing the last trial results
using the `metric` and `mode` parameters passed to `tune.run()`.
If you didn't pass these parameters, use
`get_best_config(metric, mode, scope)` instead.
"""
if not self.default_metric or not self.default_mode:
raise ValueError(
"To fetch the `best_config`, pass a `metric` and `mode` "
"parameter to `tune.run()`. Alternatively, use the "
"`get_best_config(metric, mode)` method to set the metric "
"and mode explicitly."
)
return self.get_best_config(self.default_metric, self.default_mode)
@property
def results(self) -> Dict[str, Dict]:
"""Get the last result of all the trials of the experiment"""
return {trial.trial_id: trial.last_result for trial in self.trials}
def _validate_metric(self, metric: str) -> str:
if not metric and not self.default_metric:
raise ValueError(
"No `metric` has been passed and `default_metric` has "
"not been set. Please specify the `metric` parameter."
)
return metric or self.default_metric
def _validate_mode(self, mode: str) -> str:
if not mode and not self.default_mode:
raise ValueError(
"No `mode` has been passed and `default_mode` has "
"not been set. Please specify the `mode` parameter."
)
if mode and mode not in ["min", "max"]:
raise ValueError("If set, `mode` has to be one of [min, max]")
return mode or self.default_mode
def get_best_trial(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
scope: str = "last",
filter_nan_and_inf: bool = True,
) -> Optional[Trial]:
"""Retrieve the best trial object.
Compares all trials' scores on ``metric``.
If ``metric`` is not specified, ``self.default_metric`` will be used.
If `mode` is not specified, ``self.default_mode`` will be used.
These values are usually initialized by passing the ``metric`` and
``mode`` parameters to ``tune.run()``.
Args:
metric (str): Key for trial info to order on. Defaults to
``self.default_metric``.
mode (str): One of [min, max]. Defaults to ``self.default_mode``.
scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
If `scope=last`, only look at each trial's final step for
`metric`, and compare across trials based on `mode=[min,max]`.
If `scope=avg`, consider the simple average over all steps
for `metric` and compare across trials based on
`mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
consider the simple average over the last 5 or 10 steps for
`metric` and compare across trials based on `mode=[min,max]`.
If `scope=all`, find each trial's min/max score for `metric`
based on `mode`, and compare trials based on `mode=[min,max]`.
filter_nan_and_inf (bool): If True (default), NaN or infinite
values are disregarded and these trials are never selected as
the best trial.
"""
metric = self._validate_metric(metric)
mode = self._validate_mode(mode)
if scope not in ["all", "last", "avg", "last-5-avg", "last-10-avg"]:
raise ValueError(
"ExperimentAnalysis: attempting to get best trial for "
'metric {} for scope {} not in ["all", "last", "avg", '
'"last-5-avg", "last-10-avg"]. '
"If you didn't pass a `metric` parameter to `tune.run()`, "
"you have to pass one when fetching the best trial.".format(metric, scope)
)
best_trial = None
best_metric_score = None
for trial in self.trials:
if metric not in trial.metric_analysis:
continue
if scope in ["last", "avg", "last-5-avg", "last-10-avg"]:
metric_score = trial.metric_analysis[metric][scope]
else:
metric_score = trial.metric_analysis[metric][mode]
if filter_nan_and_inf and is_nan_or_inf(metric_score):
continue
if best_metric_score is None:
best_metric_score = metric_score
best_trial = trial
continue
if (mode == "max") and (best_metric_score < metric_score):
best_metric_score = metric_score
best_trial = trial
elif (mode == "min") and (best_metric_score > metric_score):
best_metric_score = metric_score
best_trial = trial
if not best_trial:
logger.warning("Could not find best trial. Did you pass the correct `metric` " "parameter?")
return best_trial
def get_best_config(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
scope: str = "last",
) -> Optional[Dict]:
"""Retrieve the best config corresponding to the trial.
Compares all trials' scores on `metric`.
If ``metric`` is not specified, ``self.default_metric`` will be used.
If `mode` is not specified, ``self.default_mode`` will be used.
These values are usually initialized by passing the ``metric`` and
``mode`` parameters to ``tune.run()``.
Args:
metric (str): Key for trial info to order on. Defaults to
``self.default_metric``.
mode (str): One of [min, max]. Defaults to ``self.default_mode``.
scope (str): One of [all, last, avg, last-5-avg, last-10-avg].
If `scope=last`, only look at each trial's final step for
`metric`, and compare across trials based on `mode=[min,max]`.
If `scope=avg`, consider the simple average over all steps
for `metric` and compare across trials based on
`mode=[min,max]`. If `scope=last-5-avg` or `scope=last-10-avg`,
consider the simple average over the last 5 or 10 steps for
`metric` and compare across trials based on `mode=[min,max]`.
If `scope=all`, find each trial's min/max score for `metric`
based on `mode`, and compare trials based on `mode=[min,max]`.
"""
best_trial = self.get_best_trial(metric, mode, scope)
return best_trial.config if best_trial else None
@property
def best_result(self) -> Dict:
"""Get the last result of the best trial of the experiment
The best trial is determined by comparing the last trial results
using the `metric` and `mode` parameters passed to `tune.run()`.
If you didn't pass these parameters, use
`get_best_trial(metric, mode, scope).last_result` instead.
"""
if not self.default_metric or not self.default_mode:
raise ValueError(
"To fetch the `best_result`, pass a `metric` and `mode` "
"parameter to `tune.run()`. Alternatively, use "
"`get_best_trial(metric, mode).last_result` to set "
"the metric and mode explicitly and fetch the last result."
)
return self.best_trial.last_result

12
flaml/tune/cgmanifest.json
View File

@ -1,12 +0,0 @@
{
"$schema": "https://json.schemastore.org/component-detection-manifest.json",
"Registrations": [
{
"Component": {
"Type": "pip",
"pip": { "Name": "ray[tune]", "Version": "1.5.1" }
},
"DevelopmentDependency": false
}
]
}

151
flaml/tune/result.py
View File

@ -1,151 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
import os
# yapf: disable
# __sphinx_doc_begin__
# (Optional/Auto-filled) training is terminated. Filled only if not provided.
DONE = "done"
# (Optional) Enum for user controlled checkpoint
SHOULD_CHECKPOINT = "should_checkpoint"
# (Auto-filled) The hostname of the machine hosting the training process.
HOSTNAME = "hostname"
# (Auto-filled) The auto-assigned id of the trial.
TRIAL_ID = "trial_id"
# (Auto-filled) The auto-assigned id of the trial.
EXPERIMENT_TAG = "experiment_tag"
# (Auto-filled) The node ip of the machine hosting the training process.
NODE_IP = "node_ip"
# (Auto-filled) The pid of the training process.
PID = "pid"
# (Optional) Default (anonymous) metric when using tune.report(x)
DEFAULT_METRIC = "_metric"
# (Optional) Mean reward for current training iteration
EPISODE_REWARD_MEAN = "episode_reward_mean"
# (Optional) Mean loss for training iteration
MEAN_LOSS = "mean_loss"
# (Optional) Mean loss for training iteration
NEG_MEAN_LOSS = "neg_mean_loss"
# (Optional) Mean accuracy for training iteration
MEAN_ACCURACY = "mean_accuracy"
# Number of episodes in this iteration.
EPISODES_THIS_ITER = "episodes_this_iter"
# (Optional/Auto-filled) Accumulated number of episodes for this trial.
EPISODES_TOTAL = "episodes_total"
# Number of timesteps in this iteration.
TIMESTEPS_THIS_ITER = "timesteps_this_iter"
# (Auto-filled) Accumulated number of timesteps for this entire trial.
TIMESTEPS_TOTAL = "timesteps_total"
# (Auto-filled) Time in seconds this iteration took to run.
# This may be overridden to override the system-computed time difference.
TIME_THIS_ITER_S = "time_this_iter_s"
# (Auto-filled) Accumulated time in seconds for this entire trial.
TIME_TOTAL_S = "time_total_s"
# (Auto-filled) The index of this training iteration.
TRAINING_ITERATION = "training_iteration"
# __sphinx_doc_end__
# yapf: enable
DEFAULT_EXPERIMENT_INFO_KEYS = ("trainable_name", EXPERIMENT_TAG, TRIAL_ID)
DEFAULT_RESULT_KEYS = (
TRAINING_ITERATION,
TIME_TOTAL_S,
TIMESTEPS_TOTAL,
MEAN_ACCURACY,
MEAN_LOSS,
)
# Make sure this doesn't regress
AUTO_RESULT_KEYS = (
TRAINING_ITERATION,
TIME_TOTAL_S,
EPISODES_TOTAL,
TIMESTEPS_TOTAL,
NODE_IP,
HOSTNAME,
PID,
TIME_TOTAL_S,
TIME_THIS_ITER_S,
"timestamp",
"experiment_id",
"date",
"time_since_restore",
"iterations_since_restore",
"timesteps_since_restore",
"config",
)
# __duplicate__ is a magic keyword used internally to
# avoid double-logging results when using the Function API.
RESULT_DUPLICATE = "__duplicate__"
# __trial_info__ is a magic keyword used internally to pass trial_info
# to the Trainable via the constructor.
TRIAL_INFO = "__trial_info__"
# __stdout_file__/__stderr_file__ are magic keywords used internally
# to pass log file locations to the Trainable via the constructor.
STDOUT_FILE = "__stdout_file__"
STDERR_FILE = "__stderr_file__"
# Where Tune writes result files by default
DEFAULT_RESULTS_DIR = (
os.environ.get("TEST_TMPDIR") or os.environ.get("TUNE_RESULT_DIR") or os.path.expanduser("~/ray_results")
)
# Meta file about status under each experiment directory, can be
# parsed by automlboard if exists.
JOB_META_FILE = "job_status.json"
# Meta file about status under each trial directory, can be parsed
# by automlboard if exists.
EXPR_META_FILE = "trial_status.json"
# File that stores parameters of the trial.
EXPR_PARAM_FILE = "params.json"
# Pickle File that stores parameters of the trial.
EXPR_PARAM_PICKLE_FILE = "params.pkl"
# File that stores the progress of the trial.
EXPR_PROGRESS_FILE = "progress.csv"
# File that stores results of the trial.
EXPR_RESULT_FILE = "result.json"
# Config prefix when using Analysis.
CONFIG_PREFIX = "config/"

612
flaml/tune/sample.py
View File

@ -1,612 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
import logging
from copy import copy
from math import isclose
from typing import Any, Dict, List, Optional, Sequence, Union
import numpy as np
# Backwards compatibility
try:
# Added in numpy>=1.17 but we require numpy>=1.16
np_random_generator = np.random.Generator
LEGACY_RNG = False
except AttributeError:
class np_random_generator:
pass
LEGACY_RNG = True
logger = logging.getLogger(__name__)
try:
from ray import __version__ as ray_version
if ray_version.startswith("1."):
from ray.tune.sample import _BackwardsCompatibleNumpyRng
else:
from ray.tune.search.sample import _BackwardsCompatibleNumpyRng
except ImportError:
class _BackwardsCompatibleNumpyRng:
"""Thin wrapper to ensure backwards compatibility between
new and old numpy randomness generators.
"""
_rng = None
def __init__(
self,
generator_or_seed: Optional[Union["np_random_generator", np.random.RandomState, int]] = None,
):
if generator_or_seed is None or isinstance(generator_or_seed, (np.random.RandomState, np_random_generator)):
self._rng = generator_or_seed
elif LEGACY_RNG:
self._rng = np.random.RandomState(generator_or_seed)
else:
self._rng = np.random.default_rng(generator_or_seed)
@property
def legacy_rng(self) -> bool:
return not isinstance(self._rng, np_random_generator)
@property
def rng(self):
# don't set self._rng to np.random to avoid picking issues
return self._rng if self._rng is not None else np.random
def __getattr__(self, name: str) -> Any:
# https://numpy.org/doc/stable/reference/random/new-or-different.html
if self.legacy_rng:
if name == "integers":
name = "randint"
elif name == "random":
name = "rand"
return getattr(self.rng, name)
RandomState = Union[None, _BackwardsCompatibleNumpyRng, np_random_generator, np.random.RandomState, int]
class Domain:
"""Base class to specify a type and valid range to sample parameters from.
This base class is implemented by parameter spaces, like float ranges
(``Float``), integer ranges (``Integer``), or categorical variables
(``Categorical``). The ``Domain`` object contains information about
valid values (e.g. minimum and maximum values), and exposes methods that
allow specification of specific samplers (e.g. ``uniform()`` or
``loguniform()``).
"""
sampler = None
default_sampler_cls = None
def cast(self, value):
"""Cast value to domain type"""
return value
def set_sampler(self, sampler, allow_override=False):
if self.sampler and not allow_override:
raise ValueError(
"You can only choose one sampler for parameter "
"domains. Existing sampler for parameter {}: "
"{}. Tried to add {}".format(self.__class__.__name__, self.sampler, sampler)
)
self.sampler = sampler
def get_sampler(self):
sampler = self.sampler
if not sampler:
sampler = self.default_sampler_cls()
return sampler
def sample(
self,
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
sampler = self.get_sampler()
return sampler.sample(self, spec=spec, size=size, random_state=random_state)
def is_grid(self):
return isinstance(self.sampler, Grid)
def is_function(self):
return False
def is_valid(self, value: Any):
"""Returns True if `value` is a valid value in this domain."""
raise NotImplementedError
@property
def domain_str(self):
return "(unknown)"
class Sampler:
def sample(
self,
domain: Domain,
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
raise NotImplementedError
class BaseSampler(Sampler):
def __str__(self):
return "Base"
class Uniform(Sampler):
def __str__(self):
return "Uniform"
class LogUniform(Sampler):
def __init__(self, base: float = 10):
self.base = base
assert self.base > 0, "Base has to be strictly greater than 0"
def __str__(self):
return "LogUniform"
class Normal(Sampler):
def __init__(self, mean: float = 0.0, sd: float = 0.0):
self.mean = mean
self.sd = sd
assert self.sd > 0, "SD has to be strictly greater than 0"
def __str__(self):
return "Normal"
class Grid(Sampler):
"""Dummy sampler used for grid search"""
def sample(
self,
domain: Domain,
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
return RuntimeError("Do not call `sample()` on grid.")
class Float(Domain):
class _Uniform(Uniform):
def sample(
self,
domain: "Float",
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
assert domain.lower > float("-inf"), "Uniform needs a lower bound"
assert domain.upper < float("inf"), "Uniform needs a upper bound"
items = random_state.uniform(domain.lower, domain.upper, size=size)
return items if len(items) > 1 else domain.cast(items[0])
class _LogUniform(LogUniform):
def sample(
self,
domain: "Float",
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
assert domain.lower > 0, "LogUniform needs a lower bound greater than 0"
assert 0 < domain.upper < float("inf"), "LogUniform needs a upper bound greater than 0"
logmin = np.log(domain.lower) / np.log(self.base)
logmax = np.log(domain.upper) / np.log(self.base)
items = self.base ** (random_state.uniform(logmin, logmax, size=size))
return items if len(items) > 1 else domain.cast(items[0])
class _Normal(Normal):
def sample(
self,
domain: "Float",
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
assert not domain.lower or domain.lower == float(
"-inf"
), "Normal sampling does not allow a lower value bound."
assert not domain.upper or domain.upper == float(
"inf"
), "Normal sampling does not allow a upper value bound."
items = random_state.normal(self.mean, self.sd, size=size)
return items if len(items) > 1 else domain.cast(items[0])
default_sampler_cls = _Uniform
def __init__(self, lower: Optional[float], upper: Optional[float]):
# Need to explicitly check for None
self.lower = lower if lower is not None else float("-inf")
self.upper = upper if upper is not None else float("inf")
def cast(self, value):
return float(value)
def uniform(self):
if not self.lower > float("-inf"):
raise ValueError("Uniform requires a lower bound. Make sure to set the " "`lower` parameter of `Float()`.")
if not self.upper < float("inf"):
raise ValueError("Uniform requires a upper bound. Make sure to set the " "`upper` parameter of `Float()`.")
new = copy(self)
new.set_sampler(self._Uniform())
return new
def loguniform(self, base: float = 10):
if not self.lower > 0:
raise ValueError(
"LogUniform requires a lower bound greater than 0."
f"Got: {self.lower}. Did you pass a variable that has "
"been log-transformed? If so, pass the non-transformed value "
"instead."
)
if not 0 < self.upper < float("inf"):
raise ValueError(
"LogUniform requires a upper bound greater than 0. "
f"Got: {self.lower}. Did you pass a variable that has "
"been log-transformed? If so, pass the non-transformed value "
"instead."
)
new = copy(self)
new.set_sampler(self._LogUniform(base))
return new
def normal(self, mean=0.0, sd=1.0):
new = copy(self)
new.set_sampler(self._Normal(mean, sd))
return new
def quantized(self, q: float):
if self.lower > float("-inf") and not isclose(self.lower / q, round(self.lower / q)):
raise ValueError(f"Your lower variable bound {self.lower} is not divisible by " f"quantization factor {q}.")
if self.upper < float("inf") and not isclose(self.upper / q, round(self.upper / q)):
raise ValueError(f"Your upper variable bound {self.upper} is not divisible by " f"quantization factor {q}.")
new = copy(self)
new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
return new
def is_valid(self, value: float):
return self.lower <= value <= self.upper
@property
def domain_str(self):
return f"({self.lower}, {self.upper})"
class Integer(Domain):
class _Uniform(Uniform):
def sample(
self,
domain: "Integer",
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
items = random_state.integers(domain.lower, domain.upper, size=size)
return items if len(items) > 1 else domain.cast(items[0])
class _LogUniform(LogUniform):
def sample(
self,
domain: "Integer",
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
assert domain.lower > 0, "LogUniform needs a lower bound greater than 0"
assert 0 < domain.upper < float("inf"), "LogUniform needs a upper bound greater than 0"
logmin = np.log(domain.lower) / np.log(self.base)
logmax = np.log(domain.upper) / np.log(self.base)
items = self.base ** (random_state.uniform(logmin, logmax, size=size))
items = np.floor(items).astype(int)
return items if len(items) > 1 else domain.cast(items[0])
default_sampler_cls = _Uniform
def __init__(self, lower, upper):
self.lower = lower
self.upper = upper
def cast(self, value):
return int(value)
def quantized(self, q: int):
new = copy(self)
new.set_sampler(Quantized(new.get_sampler(), q), allow_override=True)
return new
def uniform(self):
new = copy(self)
new.set_sampler(self._Uniform())
return new
def loguniform(self, base: float = 10):
if not self.lower > 0:
raise ValueError(
"LogUniform requires a lower bound greater than 0."
f"Got: {self.lower}. Did you pass a variable that has "
"been log-transformed? If so, pass the non-transformed value "
"instead."
)
if not 0 < self.upper < float("inf"):
raise ValueError(
"LogUniform requires a upper bound greater than 0. "
f"Got: {self.lower}. Did you pass a variable that has "
"been log-transformed? If so, pass the non-transformed value "
"instead."
)
new = copy(self)
new.set_sampler(self._LogUniform(base))
return new
def is_valid(self, value: int):
return self.lower <= value <= self.upper
@property
def domain_str(self):
return f"({self.lower}, {self.upper})"
class Categorical(Domain):
class _Uniform(Uniform):
def sample(
self,
domain: "Categorical",
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
# do not use .choice() directly on domain.categories
# as that will coerce them to a single dtype
indices = random_state.choice(np.arange(0, len(domain.categories)), size=size)
items = [domain.categories[index] for index in indices]
return items if len(items) > 1 else domain.cast(items[0])
default_sampler_cls = _Uniform
def __init__(self, categories: Sequence):
self.categories = list(categories)
def uniform(self):
new = copy(self)
new.set_sampler(self._Uniform())
return new
def grid(self):
new = copy(self)
new.set_sampler(Grid())
return new
def __len__(self):
return len(self.categories)
def __getitem__(self, item):
return self.categories[item]
def is_valid(self, value: Any):
return value in self.categories
@property
def domain_str(self):
return f"{self.categories}"
class Quantized(Sampler):
def __init__(self, sampler: Sampler, q: Union[float, int]):
self.sampler = sampler
self.q = q
assert self.sampler, "Quantized() expects a sampler instance"
def get_sampler(self):
return self.sampler
def sample(
self,
domain: Domain,
spec: Optional[Union[List[Dict], Dict]] = None,
size: int = 1,
random_state: "RandomState" = None,
):
if not isinstance(random_state, _BackwardsCompatibleNumpyRng):
random_state = _BackwardsCompatibleNumpyRng(random_state)
if self.q == 1:
return self.sampler.sample(domain, spec, size, random_state=random_state)
quantized_domain = copy(domain)
quantized_domain.lower = np.ceil(domain.lower / self.q) * self.q
quantized_domain.upper = np.floor(domain.upper / self.q) * self.q
values = self.sampler.sample(quantized_domain, spec, size, random_state=random_state)
quantized = np.round(np.divide(values, self.q)) * self.q
if not isinstance(quantized, np.ndarray):
return domain.cast(quantized)
return list(quantized)
class PolynomialExpansionSet:
def __init__(
self,
init_monomials: set = (),
highest_poly_order: int = None,
allow_self_inter: bool = False,
):
self._init_monomials = init_monomials
self._highest_poly_order = highest_poly_order if highest_poly_order is not None else len(self._init_monomials)
self._allow_self_inter = allow_self_inter
@property
def init_monomials(self):
return self._init_monomials
@property
def highest_poly_order(self):
return self._highest_poly_order
@property
def allow_self_inter(self):
return self._allow_self_inter
def __str__(self):
return "PolynomialExpansionSet"
def uniform(lower: float, upper: float):
"""Sample a float value uniformly between ``lower`` and ``upper``.
Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
``np.random.uniform(1, 10))``
"""
return Float(lower, upper).uniform()
def quniform(lower: float, upper: float, q: float):
"""Sample a quantized float value uniformly between ``lower`` and ``upper``.
Sampling from ``tune.uniform(1, 10)`` is equivalent to sampling from
``np.random.uniform(1, 10))``
The value will be quantized, i.e. rounded to an integer increment of ``q``.
Quantization makes the upper bound inclusive.
"""
return Float(lower, upper).uniform().quantized(q)
def loguniform(lower: float, upper: float, base: float = 10):
"""Sugar for sampling in different orders of magnitude.
Args:
lower (float): Lower boundary of the output interval (e.g. 1e-4)
upper (float): Upper boundary of the output interval (e.g. 1e-2)
base (int): Base of the log. Defaults to 10.
"""
return Float(lower, upper).loguniform(base)
def qloguniform(lower: float, upper: float, q: float, base: float = 10):
"""Sugar for sampling in different orders of magnitude.
The value will be quantized, i.e. rounded to an integer increment of ``q``.
Quantization makes the upper bound inclusive.
Args:
lower (float): Lower boundary of the output interval (e.g. 1e-4)
upper (float): Upper boundary of the output interval (e.g. 1e-2)
q (float): Quantization number. The result will be rounded to an
integer increment of this value.
base (int): Base of the log. Defaults to 10.
"""
return Float(lower, upper).loguniform(base).quantized(q)
def choice(categories: Sequence):
"""Sample a categorical value.
Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
``np.random.choice([1, 2])``
"""
return Categorical(categories).uniform()
def randint(lower: int, upper: int):
"""Sample an integer value uniformly between ``lower`` and ``upper``.
``lower`` is inclusive, ``upper`` is exclusive.
Sampling from ``tune.randint(10)`` is equivalent to sampling from
``np.random.randint(10)``
"""
return Integer(lower, upper).uniform()
def lograndint(lower: int, upper: int, base: float = 10):
"""Sample an integer value log-uniformly between ``lower`` and ``upper``,
with ``base`` being the base of logarithm.
``lower`` is inclusive, ``upper`` is exclusive.
"""
return Integer(lower, upper).loguniform(base)
def qrandint(lower: int, upper: int, q: int = 1):
"""Sample an integer value uniformly between ``lower`` and ``upper``.
``lower`` is inclusive, ``upper`` is also inclusive (!).
The value will be quantized, i.e. rounded to an integer increment of ``q``.
Quantization makes the upper bound inclusive.
"""
return Integer(lower, upper).uniform().quantized(q)
def qlograndint(lower: int, upper: int, q: int, base: float = 10):
"""Sample an integer value log-uniformly between ``lower`` and ``upper``,
with ``base`` being the base of logarithm.
``lower`` is inclusive, ``upper`` is also inclusive (!).
The value will be quantized, i.e. rounded to an integer increment of ``q``.
Quantization makes the upper bound inclusive.
"""
return Integer(lower, upper).loguniform(base).quantized(q)
def randn(mean: float = 0.0, sd: float = 1.0):
"""Sample a float value normally with ``mean`` and ``sd``.
Args:
mean (float): Mean of the normal distribution. Defaults to 0.
sd (float): SD of the normal distribution. Defaults to 1.
"""
return Float(None, None).normal(mean, sd)
def qrandn(mean: float, sd: float, q: float):
"""Sample a float value normally with ``mean`` and ``sd``.
The value will be quantized, i.e. rounded to an integer increment of ``q``.
Args:
mean: Mean of the normal distribution.
sd: SD of the normal distribution.
q: Quantization number. The result will be rounded to an
integer increment of this value.
"""
return Float(None, None).normal(mean, sd).quantized(q)
def polynomial_expansion_set(init_monomials: set, highest_poly_order: int = None, allow_self_inter: bool = False):
return PolynomialExpansionSet(init_monomials, highest_poly_order, allow_self_inter)

6
flaml/tune/scheduler/__init__.py
View File

@ -1,6 +0,0 @@
from .trial_scheduler import TrialScheduler
from .online_scheduler import (
OnlineScheduler,
OnlineSuccessiveDoublingScheduler,
ChaChaScheduler,
)

124
flaml/tune/scheduler/online_scheduler.py
View File

@ -1,124 +0,0 @@
import numpy as np
import logging
from typing import Dict
from flaml.tune.scheduler import TrialScheduler
from flaml.tune import Trial
logger = logging.getLogger(__name__)
class OnlineScheduler(TrialScheduler):
"""Class for the most basic OnlineScheduler."""
def on_trial_result(self, trial_runner, trial: Trial, result: Dict):
"""Report result and return a decision on the trial's status."""
# Always keep a trial running (return status TrialScheduler.CONTINUE).
return TrialScheduler.CONTINUE
def choose_trial_to_run(self, trial_runner) -> Trial:
"""Decide which trial to run next."""
# Trial prioritrization according to the status:
# PENDING (trials that have not been tried) > PAUSED (trials that have been ran).
# For trials with the same status, it chooses the ones with smaller resource lease.
for trial in trial_runner.get_trials():
if trial.status == Trial.PENDING:
return trial
min_paused_resource = np.inf
min_paused_resource_trial = None
for trial in trial_runner.get_trials():
# if there is a tie, prefer the earlier added ones
if trial.status == Trial.PAUSED and trial.resource_lease < min_paused_resource:
min_paused_resource = trial.resource_lease
min_paused_resource_trial = trial
if min_paused_resource_trial is not None:
return min_paused_resource_trial
class OnlineSuccessiveDoublingScheduler(OnlineScheduler):
"""class for the OnlineSuccessiveDoublingScheduler algorithm."""
def __init__(self, increase_factor: float = 2.0):
"""Constructor.
Args:
increase_factor: A float of multiplicative factor
used to increase resource lease. Default is 2.0.
"""
super().__init__()
self._increase_factor = increase_factor
def on_trial_result(self, trial_runner, trial: Trial, result: Dict):
"""Report result and return a decision on the trial's status."""
# 1. Returns TrialScheduler.CONTINUE (i.e., keep the trial running),
# if the resource consumed has not reached the current resource_lease.s.
# 2. otherwise double the current resource lease and return TrialScheduler.PAUSE.
if trial.result is None or trial.result.resource_used < trial.resource_lease:
return TrialScheduler.CONTINUE
else:
trial.set_resource_lease(trial.resource_lease * self._increase_factor)
logger.info(
"Doubled resource for trial %s, used: %s, current budget %s",
trial.trial_id,
trial.result.resource_used,
trial.resource_lease,
)
return TrialScheduler.PAUSE
class ChaChaScheduler(OnlineSuccessiveDoublingScheduler):
"""class for the ChaChaScheduler algorithm."""
def __init__(self, increase_factor: float = 2.0, **kwargs):
"""Constructor.
Args:
increase_factor: A float of multiplicative factor
used to increase resource lease. Default is 2.0.
"""
super().__init__(increase_factor)
self._keep_champion = kwargs.get("keep_champion", True)
self._keep_challenger_metric = kwargs.get("keep_challenger_metric", "ucb")
self._keep_challenger_ratio = kwargs.get("keep_challenger_ratio", 0.5)
self._pause_old_froniter = kwargs.get("pause_old_froniter", False)
logger.info("Using chacha scheduler with config %s", kwargs)
def on_trial_result(self, trial_runner, trial: Trial, result: Dict):
"""Report result and return a decision on the trial's status."""
# Make a decision according to: SuccessiveDoubling + champion check + performance check.
# Doubling scheduler makes a decision
decision = super().on_trial_result(trial_runner, trial, result)
# ***********Check whether the trial has been paused since a new champion is promoted**
# NOTE: This check is not enabled by default. Just keeping it for experimentation purpose.
## trial.is_checked_under_current_champion being False means the trial
# has not been paused since the new champion is promoted. If so, we need to
# tentatively pause it such that new trials can possiblly be taken into consideration
# NOTE: This may need to be changed. We need to do this because we only add trials.
# into the OnlineTrialRunner when there are avaialbe slots. Maybe we need to consider
# adding max_running_trial number of trials once a new champion is promoted.
if self._pause_old_froniter and not trial.is_checked_under_current_champion:
if decision == TrialScheduler.CONTINUE:
decision = TrialScheduler.PAUSE
trial.set_checked_under_current_champion(True)
logger.info("Tentitively set trial as paused")
# ****************Keep the champion always running******************
if (
self._keep_champion
and trial.trial_id == trial_runner.champion_trial.trial_id
and decision == TrialScheduler.PAUSE
):
return TrialScheduler.CONTINUE
# ****************Keep the trials with top performance always running******************
if self._keep_challenger_ratio is not None:
if decision == TrialScheduler.PAUSE:
logger.debug("champion, %s", trial_runner.champion_trial.trial_id)
# this can be inefficient when the # trials is large. TODO: need to improve efficiency.
top_trials = trial_runner.get_top_running_trials(
self._keep_challenger_ratio, self._keep_challenger_metric
)
logger.debug("top_learners: %s", top_trials)
if trial in top_trials:
logger.debug("top runner %s: set from PAUSE to CONTINUE", trial.trial_id)
return TrialScheduler.CONTINUE
return decision

33
flaml/tune/scheduler/trial_scheduler.py
View File

@ -1,33 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
from flaml.tune import trial_runner
from flaml.tune.trial import Trial
class TrialScheduler:
"""Interface for implementing a Trial Scheduler class."""
CONTINUE = "CONTINUE" #: Status for continuing trial execution
PAUSE = "PAUSE" #: Status for pausing trial execution
STOP = "STOP" #: Status for stopping trial execution
def on_trial_add(self, trial_runner: "trial_runner.TrialRunner", trial: Trial):
pass
def on_trial_remove(self, trial_runner: "trial_runner.TrialRunner", trial: Trial):
pass

3
flaml/tune/searcher/__init__.py
View File

@ -1,3 +0,0 @@
from .blendsearch import CFO, BlendSearch, BlendSearchTuner, RandomSearch
from .flow2 import FLOW2
from .online_searcher import ChampionFrontierSearcher

1121
flaml/tune/searcher/blendsearch.py
View File

File diff suppressed because it is too large Load Diff

28
flaml/tune/searcher/cfo_cat.py
View File

@ -1,28 +0,0 @@
# !
# * Copyright (c) Microsoft Corporation. All rights reserved.
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
from .flow2 import FLOW2
from .blendsearch import CFO
class FLOW2Cat(FLOW2):
"""Local search algorithm optimized for categorical variables."""
def _init_search(self):
super()._init_search()
self.step_ub = 1
self.step = self.STEPSIZE * self.step_ub
lb = self.step_lower_bound
if lb > self.step:
self.step = lb * 2
# upper bound
if self.step > self.step_ub:
self.step = self.step_ub
self._trunc = self.dim
class CFOCat(CFO):
"""CFO optimized for categorical variables."""
LocalSearch = FLOW2Cat

673
flaml/tune/searcher/flow2.py
View File

@ -1,673 +0,0 @@
# !
# * Copyright (c) Microsoft Corporation. All rights reserved.
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
from typing import Dict, Optional, Tuple
import numpy as np
import logging
from collections import defaultdict
try:
from ray import __version__ as ray_version
assert ray_version >= "1.0.0"
if ray_version.startswith("1."):
from ray.tune.suggest import Searcher
from ray.tune import sample
else:
from ray.tune.search import Searcher, sample
from ray.tune.utils.util import flatten_dict, unflatten_dict
except (ImportError, AssertionError):
from .suggestion import Searcher
from flaml.tune import sample
from ..trial import flatten_dict, unflatten_dict
from flaml.config import SAMPLE_MULTIPLY_FACTOR
from ..space import (
complete_config,
denormalize,
normalize,
generate_variants_compatible,
)
logger = logging.getLogger(__name__)
class FLOW2(Searcher):
"""Local search algorithm FLOW2, with adaptive step size."""
STEPSIZE = 0.1
STEP_LOWER_BOUND = 0.0001
def __init__(
self,
init_config: dict,
metric: Optional[str] = None,
mode: Optional[str] = None,
space: Optional[dict] = None,
resource_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
resource_multiple_factor: Optional[float] = None,
cost_attr: Optional[str] = "time_total_s",
seed: Optional[int] = 20,
lexico_objectives=None,
):
"""Constructor.
Args:
init_config: a dictionary of a partial or full initial config,
e.g., from a subset of controlled dimensions
to the initial low-cost values.
E.g., {'epochs': 1}.
metric: A string of the metric name to optimize for.
mode: A string in ['min', 'max'] to specify the objective as
minimization or maximization.
space: A dictionary to specify the search space.
resource_attr: A string to specify the resource dimension and the best
performance is assumed to be at the max_resource.
min_resource: A float of the minimal resource to use for the resource_attr.
max_resource: A float of the maximal resource to use for the resource_attr.
resource_multiple_factor: A float of the multiplicative factor
used for increasing resource.
cost_attr: A string of the attribute used for cost.
seed: An integer of the random seed.
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
optimization with lexicographic preferences. When lexico_objectives is not None, the arguments metric,
mode will be invalid. This dictionary shall contain the following fields of key-value pairs:
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
objectives.
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
metric names (provided in "metric"), and the values are the numerical target values.
- "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the absolute/percentage tolerance in the form of numeric/string.
E.g.,
```python
lexico_objectives = {
"metrics": ["error_rate", "pred_time"],
"modes": ["min", "min"],
"tolerances": {"error_rate": 0.01, "pred_time": 0.0},
"targets": {"error_rate": 0.0},
}
```
We also support percentage tolerance.
E.g.,
```python
lexico_objectives = {
"metrics": ["error_rate", "pred_time"],
"modes": ["min", "min"],
"tolerances": {"error_rate": "5%", "pred_time": "0%"},
"targets": {"error_rate": 0.0},
}
```
"""
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
else:
mode = "min"
super(FLOW2, self).__init__(metric=metric, mode=mode)
# internally minimizes, so "max" => -1
if mode == "max":
self.metric_op = -1.0
elif mode == "min":
self.metric_op = 1.0
self.space = space or {}
self._space = flatten_dict(self.space, prevent_delimiter=True)
self._random = np.random.RandomState(seed)
self.rs_random = sample._BackwardsCompatibleNumpyRng(seed + 19823)
self.seed = seed
self.init_config = init_config
self.best_config = flatten_dict(init_config)
self.resource_attr = resource_attr
self.min_resource = min_resource
self.lexico_objectives = lexico_objectives
if self.lexico_objectives is not None:
if "modes" not in self.lexico_objectives.keys():
self.lexico_objectives["modes"] = ["min"] * len(self.lexico_objectives["metrics"])
for t_metric, t_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
if t_metric not in self.lexico_objectives["tolerances"].keys():
self.lexico_objectives["tolerances"][t_metric] = 0
if t_metric not in self.lexico_objectives["targets"].keys():
self.lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
self.resource_multiple_factor = resource_multiple_factor or SAMPLE_MULTIPLY_FACTOR
self.cost_attr = cost_attr
self.max_resource = max_resource
self._resource = None
self._f_best = None # only use for lexico_comapre. It represent the best value achieved by lexico_flow.
self._step_lb = np.Inf
self._histories = None # only use for lexico_comapre. It records the result of historical configurations.
if space is not None:
self._init_search()
def _init_search(self):
self._tunable_keys = []
self._bounded_keys = []
self._unordered_cat_hp = {}
hier = False
for key, domain in self._space.items():
assert not (
isinstance(domain, dict) and "grid_search" in domain
), f"{key}'s domain is grid search, not supported in FLOW^2."
if callable(getattr(domain, "get_sampler", None)):
self._tunable_keys.append(key)
sampler = domain.get_sampler()
# the step size lower bound for uniform variables doesn't depend
# on the current config
if isinstance(sampler, sample.Quantized):
q = sampler.q
sampler = sampler.get_sampler()
if str(sampler) == "Uniform":
self._step_lb = min(self._step_lb, q / (domain.upper - domain.lower + 1))
elif isinstance(domain, sample.Integer) and str(sampler) == "Uniform":
self._step_lb = min(self._step_lb, 1.0 / (domain.upper - domain.lower))
if isinstance(domain, sample.Categorical):
if not domain.ordered:
self._unordered_cat_hp[key] = len(domain.categories)
if not hier:
for cat in domain.categories:
if isinstance(cat, dict):
hier = True
break
if str(sampler) != "Normal":
self._bounded_keys.append(key)
if not hier:
self._space_keys = sorted(self._tunable_keys)
self.hierarchical = hier
if self.resource_attr and self.resource_attr not in self._space and self.max_resource:
self.min_resource = self.min_resource or self._min_resource()
self._resource = self._round(self.min_resource)
if not hier:
self._space_keys.append(self.resource_attr)
else:
self._resource = None
self.incumbent = {}
self.incumbent = self.normalize(self.best_config) # flattened
self.best_obj = self.cost_incumbent = None
self.dim = len(self._tunable_keys) # total # tunable dimensions
self._direction_tried = None
self._num_complete4incumbent = self._cost_complete4incumbent = 0
self._num_allowed4incumbent = 2 * self.dim
self._proposed_by = {} # trial_id: int -> incumbent: Dict
self.step_ub = np.sqrt(self.dim)
self.step = self.STEPSIZE * self.step_ub
lb = self.step_lower_bound
if lb > self.step:
self.step = lb * 2
# upper bound
self.step = min(self.step, self.step_ub)
# maximal # consecutive no improvements
self.dir = 2 ** (min(9, self.dim))
self._configs = {} # dict from trial_id to (config, stepsize)
self._K = 0
self._iter_best_config = 1
self.trial_count_proposed = self.trial_count_complete = 1
self._num_proposedby_incumbent = 0
self._reset_times = 0
# record intermediate trial cost
self._trial_cost = {}
self._same = False # whether the proposed config is the same as best_config
self._init_phase = True # initial phase to increase initial stepsize
self._trunc = 0
# no truncation by default. when > 0, it means how many
# non-zero dimensions to keep in the random unit vector
@property
def step_lower_bound(self) -> float:
step_lb = self._step_lb
for key in self._tunable_keys:
if key not in self.best_config:
continue
domain = self._space[key]
sampler = domain.get_sampler()
# the stepsize lower bound for log uniform variables depends on the
# current config
if isinstance(sampler, sample.Quantized):
q = sampler.q
sampler_inner = sampler.get_sampler()
if str(sampler_inner) == "LogUniform":
step_lb = min(
step_lb,
np.log(1.0 + q / self.best_config[key]) / np.log(domain.upper / domain.lower),
)
elif isinstance(domain, sample.Integer) and str(sampler) == "LogUniform":
step_lb = min(
step_lb,
np.log(1.0 + 1.0 / self.best_config[key]) / np.log((domain.upper - 1) / domain.lower),
)
if np.isinf(step_lb):
step_lb = self.STEP_LOWER_BOUND
else:
step_lb *= self.step_ub
return step_lb
@property
def resource(self) -> float:
return self._resource
def _min_resource(self) -> float:
"""automatically decide minimal resource"""
return self.max_resource / np.pow(self.resource_multiple_factor, 5)
def _round(self, resource) -> float:
"""round the resource to self.max_resource if close to it"""
if resource * self.resource_multiple_factor > self.max_resource:
return self.max_resource
return resource
def rand_vector_gaussian(self, dim, std=1.0):
return self._random.normal(0, std, dim)
def complete_config(
self,
partial_config: Dict,
lower: Optional[Dict] = None,
upper: Optional[Dict] = None,
) -> Tuple[Dict, Dict]:
"""Generate a complete config from the partial config input.
Add minimal resource to config if available.
"""
disturb = self._reset_times and partial_config == self.init_config
# if not the first time to complete init_config, use random gaussian
config, space = complete_config(partial_config, self.space, self, disturb, lower, upper)
if partial_config == self.init_config:
self._reset_times += 1
if self._resource:
config[self.resource_attr] = self.min_resource
return config, space
def create(self, init_config: Dict, obj: float, cost: float, space: Dict) -> Searcher:
# space is the subspace where the init_config is located
flow2 = self.__class__(
init_config,
self.metric,
self.mode,
space,
self.resource_attr,
self.min_resource,
self.max_resource,
self.resource_multiple_factor,
self.cost_attr,
self.seed + 1,
self.lexico_objectives,
)
if self.lexico_objectives is not None:
flow2.best_obj = {}
for k, v in obj.items():
flow2.best_obj[k] = (
-v if self.lexico_objectives["modes"][self.lexico_objectives["metrics"].index(k)] == "max" else v
)
else:
flow2.best_obj = obj * self.metric_op # minimize internally
flow2.cost_incumbent = cost
self.seed += 1
return flow2
def normalize(self, config, recursive=False) -> Dict:
"""normalize each dimension in config to [0,1]."""
return normalize(config, self._space, self.best_config, self.incumbent, recursive)
def denormalize(self, config):
"""denormalize each dimension in config from [0,1]."""
return denormalize(config, self._space, self.best_config, self.incumbent, self._random)
def set_search_properties(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None,
) -> bool:
if metric:
self._metric = metric
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
self._mode = mode
if mode == "max":
self.metric_op = -1.0
elif mode == "min":
self.metric_op = 1.0
if config:
self.space = config
self._space = flatten_dict(self.space)
self._init_search()
return True
def update_fbest(
self,
):
obj_initial = self.lexico_objectives["metrics"][0]
feasible_index = np.array([*range(len(self._histories[obj_initial]))])
for k_metric in self.lexico_objectives["metrics"]:
k_values = np.array(self._histories[k_metric])
feasible_value = k_values.take(feasible_index)
self._f_best[k_metric] = np.min(feasible_value)
if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
else:
assert (
self.lexico_objectives["tolerances"][k_metric][-1] == "%"
), "String tolerance of {} should use %% as the suffix".format(k_metric)
tolerance_bound = self._f_best[k_metric] * (
1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
)
feasible_index_filter = np.where(
feasible_value
<= max(
tolerance_bound,
self.lexico_objectives["targets"][k_metric],
)
)[0]
feasible_index = feasible_index.take(feasible_index_filter)
def lexico_compare(self, result) -> bool:
if self._histories is None:
self._histories, self._f_best = defaultdict(list), {}
for k in self.lexico_objectives["metrics"]:
self._histories[k].append(result[k])
self.update_fbest()
return True
else:
for k in self.lexico_objectives["metrics"]:
self._histories[k].append(result[k])
self.update_fbest()
for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
k_target = (
self.lexico_objectives["targets"][k_metric]
if k_mode == "min"
else -self.lexico_objectives["targets"][k_metric]
)
if not isinstance(self.lexico_objectives["tolerances"][k_metric], str):
tolerance_bound = self._f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
else:
assert (
self.lexico_objectives["tolerances"][k_metric][-1] == "%"
), "String tolerance of {} should use %% as the suffix".format(k_metric)
tolerance_bound = self._f_best[k_metric] * (
1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))
)
if (result[k_metric] < max(tolerance_bound, k_target)) and (
self.best_obj[k_metric]
< max(
tolerance_bound,
k_target,
)
):
continue
elif result[k_metric] < self.best_obj[k_metric]:
return True
else:
return False
for k_metr in self.lexico_objectives["metrics"]:
if result[k_metr] == self.best_obj[k_metr]:
continue
elif result[k_metr] < self.best_obj[k_metr]:
return True
else:
return False
def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
"""
Compare with incumbent.
If better, move, reset num_complete and num_proposed.
If not better and num_complete >= 2*dim, num_allowed += 2.
"""
self.trial_count_complete += 1
if not error and result:
obj = (
result.get(self._metric)
if self.lexico_objectives is None
else {k: result[k] for k in self.lexico_objectives["metrics"]}
)
if obj:
obj = (
{
k: -obj[k] if m == "max" else obj[k]
for k, m in zip(
self.lexico_objectives["metrics"],
self.lexico_objectives["modes"],
)
}
if isinstance(obj, dict)
else obj * self.metric_op
)
if (
self.best_obj is None
or (self.lexico_objectives is None and obj < self.best_obj)
or (self.lexico_objectives is not None and self.lexico_compare(obj))
):
self.best_obj = obj
self.best_config, self.step = self._configs[trial_id]
self.incumbent = self.normalize(self.best_config)
self.cost_incumbent = result.get(self.cost_attr, 1)
if self._resource:
self._resource = self.best_config[self.resource_attr]
self._num_complete4incumbent = 0
self._cost_complete4incumbent = 0
self._num_proposedby_incumbent = 0
self._num_allowed4incumbent = 2 * self.dim
self._proposed_by.clear()
if self._K > 0:
self.step *= np.sqrt(self._K / self._oldK)
self.step = min(self.step, self.step_ub)
self._iter_best_config = self.trial_count_complete
if self._trunc:
self._trunc = min(self._trunc + 1, self.dim)
return
elif self._trunc:
self._trunc = max(self._trunc >> 1, 1)
proposed_by = self._proposed_by.get(trial_id)
if proposed_by == self.incumbent:
self._num_complete4incumbent += 1
cost = result.get(self.cost_attr, 1) if result else self._trial_cost.get(trial_id)
if cost:
self._cost_complete4incumbent += cost
if self._num_complete4incumbent >= 2 * self.dim and self._num_allowed4incumbent == 0:
self._num_allowed4incumbent = 2
if self._num_complete4incumbent == self.dir and (not self._resource or self._resource == self.max_resource):
self._num_complete4incumbent -= 2
self._num_allowed4incumbent = max(self._num_allowed4incumbent, 2)
def on_trial_result(self, trial_id: str, result: Dict):
"""Early update of incumbent."""
if result:
obj = (
result.get(self._metric)
if self.lexico_objectives is None
else {k: result[k] for k in self.lexico_objectives["metrics"]}
)
if obj:
obj = (
{
k: -obj[k] if m == "max" else obj[k]
for k, m in zip(
self.lexico_objectives["metrics"],
self.lexico_objectives["modes"],
)
}
if isinstance(obj, dict)
else obj * self.metric_op
)
if (
self.best_obj is None
or (self.lexico_objectives is None and obj < self.best_obj)
or (self.lexico_objectives is not None and self.lexico_compare(obj))
):
self.best_obj = obj
config = self._configs[trial_id][0]
if self.best_config != config:
self.best_config = config
if self._resource:
self._resource = config[self.resource_attr]
self.incumbent = self.normalize(self.best_config)
self.cost_incumbent = result.get(self.cost_attr, 1)
self._cost_complete4incumbent = 0
self._num_complete4incumbent = 0
self._num_proposedby_incumbent = 0
self._num_allowed4incumbent = 2 * self.dim
self._proposed_by.clear()
self._iter_best_config = self.trial_count_complete
cost = result.get(self.cost_attr, 1)
# record the cost in case it is pruned and cost info is lost
self._trial_cost[trial_id] = cost
def rand_vector_unit_sphere(self, dim, trunc=0) -> np.ndarray:
vec = self._random.normal(0, 1, dim)
if 0 < trunc < dim:
vec[np.abs(vec).argsort()[: dim - trunc]] = 0
mag = np.linalg.norm(vec)
return vec / mag
def suggest(self, trial_id: str) -> Optional[Dict]:
"""Suggest a new config, one of the following cases:
1. same incumbent, increase resource.
2. same resource, move from the incumbent to a random direction.
3. same resource, move from the incumbent to the opposite direction.
"""
# TODO: better decouple FLOW2 config suggestion and stepsize update
self.trial_count_proposed += 1
if (
self._num_complete4incumbent > 0
and self.cost_incumbent
and self._resource
and self._resource < self.max_resource
and (self._cost_complete4incumbent >= self.cost_incumbent * self.resource_multiple_factor)
):
return self._increase_resource(trial_id)
self._num_allowed4incumbent -= 1
move = self.incumbent.copy()
if self._direction_tried is not None:
# return negative direction
for i, key in enumerate(self._tunable_keys):
move[key] -= self._direction_tried[i]
self._direction_tried = None
else:
# propose a new direction
self._direction_tried = self.rand_vector_unit_sphere(self.dim, self._trunc) * self.step
for i, key in enumerate(self._tunable_keys):
move[key] += self._direction_tried[i]
self._project(move)
config = self.denormalize(move)
self._proposed_by[trial_id] = self.incumbent
self._configs[trial_id] = (config, self.step)
self._num_proposedby_incumbent += 1
best_config = self.best_config
if self._init_phase:
if self._direction_tried is None:
if self._same:
same = not any(key not in best_config or value != best_config[key] for key, value in config.items())
if same:
# increase step size
self.step += self.STEPSIZE
self.step = min(self.step, self.step_ub)
else:
same = not any(key not in best_config or value != best_config[key] for key, value in config.items())
self._same = same
if self._num_proposedby_incumbent == self.dir and (not self._resource or self._resource == self.max_resource):
# check stuck condition if using max resource
self._num_proposedby_incumbent -= 2
self._init_phase = False
if self.step < self.step_lower_bound:
return None
# decrease step size
self._oldK = self._K or self._iter_best_config
self._K = self.trial_count_proposed + 1
self.step *= np.sqrt(self._oldK / self._K)
if self._init_phase:
return unflatten_dict(config)
if self._trunc == 1 and self._direction_tried is not None:
# random
for i, key in enumerate(self._tunable_keys):
if self._direction_tried[i] != 0:
for _, generated in generate_variants_compatible(
{"config": {key: self._space[key]}}, random_state=self.rs_random
):
if generated["config"][key] != best_config[key]:
config[key] = generated["config"][key]
return unflatten_dict(config)
break
elif len(config) == len(best_config):
for key, value in best_config.items():
if value != config[key]:
return unflatten_dict(config)
# print('move to', move)
self.incumbent = move
return unflatten_dict(config)
def _increase_resource(self, trial_id):
# consider increasing resource using sum eval cost of complete
# configs
old_resource = self._resource
self._resource = self._round(self._resource * self.resource_multiple_factor)
self.cost_incumbent *= self._resource / old_resource
config = self.best_config.copy()
config[self.resource_attr] = self._resource
self._direction_tried = None
self._configs[trial_id] = (config, self.step)
return unflatten_dict(config)
def _project(self, config):
"""project normalized config in the feasible region and set resource_attr"""
for key in self._bounded_keys:
value = config[key]
config[key] = max(0, min(1, value))
if self._resource:
config[self.resource_attr] = self._resource
@property
def can_suggest(self) -> bool:
"""Can't suggest if 2*dim configs have been proposed for the incumbent
while fewer are completed.
"""
return self._num_allowed4incumbent > 0
def config_signature(self, config, space: Dict = None) -> tuple:
"""Return the signature tuple of a config."""
config = flatten_dict(config)
space = flatten_dict(space) if space else self._space
value_list = []
# self._space_keys doesn't contain keys with const values,
# e.g., "eval_metric": ["logloss", "error"].
keys = sorted(config.keys()) if self.hierarchical else self._space_keys
for key in keys:
value = config[key]
if key == self.resource_attr:
value_list.append(value)
else:
# key must be in space
domain = space[key]
if self.hierarchical and not (
domain is None or type(domain) in (str, int, float) or isinstance(domain, sample.Domain)
):
# not domain or hashable
# get rid of list type for hierarchical search space.
continue
if isinstance(domain, sample.Integer):
value_list.append(int(round(value)))
else:
value_list.append(value)
return tuple(value_list)
@property
def converged(self) -> bool:
"""Whether the local search has converged."""
if self._num_complete4incumbent < self.dir - 2:
return False
# check stepsize after enough configs are completed
return self.step < self.step_lower_bound
def reach(self, other: Searcher) -> bool:
"""whether the incumbent can reach the incumbent of other."""
config1, config2 = self.best_config, other.best_config
incumbent1, incumbent2 = self.incumbent, other.incumbent
if self._resource and config1[self.resource_attr] > config2[self.resource_attr]:
# resource will not decrease
return False
for key in self._unordered_cat_hp:
# unordered cat choice is hard to reach by chance
if config1[key] != config2.get(key):
return False
delta = np.array([incumbent1[key] - incumbent2.get(key, np.inf) for key in self._tunable_keys])
return np.linalg.norm(delta) <= self.step

388
flaml/tune/searcher/online_searcher.py
View File

@ -1,388 +0,0 @@
import numpy as np
import logging
import itertools
from typing import Dict, Optional, List
from flaml.tune import Categorical, Float, PolynomialExpansionSet, Trial
from flaml.onlineml import VowpalWabbitTrial
from flaml.tune.searcher import CFO
logger = logging.getLogger(__name__)
class BaseSearcher:
"""Abstract class for an online searcher."""
def __init__(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
):
pass
def set_search_properties(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = None,
):
if metric:
self._metric = metric
if mode:
assert mode in ["min", "max"], "`mode` must be 'min' or 'max'."
self._mode = mode
def next_trial(self):
NotImplementedError
def on_trial_result(self, trial_id: str, result: Dict):
pass
def on_trial_complete(self, trial):
pass
class ChampionFrontierSearcher(BaseSearcher):
"""The ChampionFrontierSearcher class.
NOTE about the correspondence about this code and the research paper:
[ChaCha for Online AutoML](https://arxiv.org/pdf/2106.04815.pdf).
This class serves the role of ConfigOralce as described in the paper.
"""
# **************************More notes***************************
# Every time we create an online trial, we generate a searcher_trial_id.
# At the same time, we also record the trial_id of the VW trial.
# Note that the trial_id is a unique signature of the configuration.
# So if two VWTrials are associated with the same config, they will have the same trial_id
# (although not the same searcher_trial_id).
# searcher_trial_id will be used in suggest().
# ****the following constants are used when generating new challengers in
# the _query_config_oracle function
# how many item to add when doing the expansion
# (i.e. how many interaction items to add at each time)
POLY_EXPANSION_ADDITION_NUM = 1
# the order of polynomial expansions to add based on the given seed interactions
EXPANSION_ORDER = 2
# the number of new challengers with new numerical hyperparamter configs
NUMERICAL_NUM = 2
# In order to use CFO, a loss name and loss values of configs are need
# since CFO in fact only requires relative loss order of two configs to perform
# the update, a pseudo loss can be used as long as the relative performance orders
# of different configs are perserved. We set the loss of the init config to be
# a large value (CFO_SEARCHER_LARGE_LOSS), and set the loss of the better config as
# 0.95 of the previous best config's loss.
# NOTE: this setting depends on the assumption that (and thus
# _query_config_oracle) is only triggered when a better champion is found.
CFO_SEARCHER_METRIC_NAME = "pseudo_loss"
CFO_SEARCHER_LARGE_LOSS = 1e6
# the random seed used in generating numerical hyperparamter configs (when CFO is not used)
NUM_RANDOM_SEED = 111
CHAMPION_TRIAL_NAME = "champion_trial"
TRIAL_CLASS = VowpalWabbitTrial
def __init__(
self,
init_config: Dict,
space: Optional[Dict] = None,
metric: Optional[str] = None,
mode: Optional[str] = None,
random_seed: Optional[int] = 2345,
online_trial_args: Optional[Dict] = {},
nonpoly_searcher_name: Optional[str] = "CFO",
):
"""Constructor.
Args:
init_config: A dictionary of initial configuration.
space: A dictionary to specify the search space.
metric: A string of the metric name to optimize for.
mode: A string in ['min', 'max'] to specify the objective as
minimization or maximization.
random_seed: An integer of the random seed.
online_trial_args: A dictionary to specify the online trial
arguments for experimental purpose.
nonpoly_searcher_name: A string to specify the search algorithm
for nonpoly hyperparameters.
"""
self._init_config = init_config
self._space = space
self._seed = random_seed
self._online_trial_args = online_trial_args
self._nonpoly_searcher_name = nonpoly_searcher_name
self._random_state = np.random.RandomState(self._seed)
self._searcher_for_nonpoly_hp = {}
# dicts to remember the mapping between searcher_trial_id and trial_id
self._space_of_nonpoly_hp = {}
# key: searcher_trial_id, value: trial_id
self._searcher_trialid_to_trialid = {}
# value: trial_id, key: searcher_trial_id
self._trialid_to_searcher_trial_id = {}
self._challenger_list = []
# initialize the search in set_search_properties
self.set_search_properties(setting={self.CHAMPION_TRIAL_NAME: None}, init_call=True)
logger.debug("using random seed %s in config oracle", self._seed)
def set_search_properties(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
config: Optional[Dict] = {},
setting: Optional[Dict] = {},
init_call: Optional[bool] = False,
):
"""Construct search space with the given config, and setup the search."""
super().set_search_properties(metric, mode, config)
# *********Use ConfigOralce (i.e, self._generate_new_space to generate list of new challengers)
logger.info("setting %s", setting)
champion_trial = setting.get(self.CHAMPION_TRIAL_NAME, None)
if champion_trial is None:
champion_trial = self._create_trial_from_config(self._init_config)
# generate a new list of challenger trials
new_challenger_list = self._query_config_oracle(
champion_trial.config,
champion_trial.trial_id,
self._trialid_to_searcher_trial_id[champion_trial.trial_id],
)
# add the newly generated challengers to existing challengers
# there can be duplicates and we check duplicates when calling next_trial()
self._challenger_list = self._challenger_list + new_challenger_list
# add the champion as part of the new_challenger_list when called initially
if init_call:
self._challenger_list.append(champion_trial)
logger.info(
"**Important** Created challengers from champion %s",
champion_trial.trial_id,
)
logger.info(
"New challenger size %s, %s",
len(self._challenger_list),
[t.trial_id for t in self._challenger_list],
)
def next_trial(self):
"""Return a trial from the _challenger_list."""
next_trial = None
if self._challenger_list:
next_trial = self._challenger_list.pop()
return next_trial
def _create_trial_from_config(self, config, searcher_trial_id=None):
if searcher_trial_id is None:
searcher_trial_id = Trial.generate_id()
trial = self.TRIAL_CLASS(config, **self._online_trial_args)
self._searcher_trialid_to_trialid[searcher_trial_id] = trial.trial_id
# only update the dict when the trial_id does not exist
if trial.trial_id not in self._trialid_to_searcher_trial_id:
self._trialid_to_searcher_trial_id[trial.trial_id] = searcher_trial_id
return trial
def _query_config_oracle(
self, seed_config, seed_config_trial_id, seed_config_searcher_trial_id=None
) -> List[Trial]:
"""Give the seed config, generate a list of new configs (which are supposed to include
at least one config that has better performance than the input seed_config).
"""
# group the hyperparameters according to whether the configs of them are independent
# with the other hyperparameters
hyperparameter_config_groups = []
searcher_trial_ids_groups = []
nonpoly_config = {}
for k, v in seed_config.items():
config_domain = self._space[k]
if isinstance(config_domain, PolynomialExpansionSet):
# get candidate configs for hyperparameters of the PolynomialExpansionSet type
partial_new_configs = self._generate_independent_hp_configs(k, v, config_domain)
if partial_new_configs:
hyperparameter_config_groups.append(partial_new_configs)
# does not have searcher_trial_ids
searcher_trial_ids_groups.append([])
elif isinstance(config_domain, Float) or isinstance(config_domain, Categorical):
# otherwise we need to deal with them in group
nonpoly_config[k] = v
if k not in self._space_of_nonpoly_hp:
self._space_of_nonpoly_hp[k] = self._space[k]
# -----------generate partial new configs for non-PolynomialExpansionSet hyperparameters
if nonpoly_config:
new_searcher_trial_ids = []
partial_new_nonpoly_configs = []
if "CFO" in self._nonpoly_searcher_name:
if seed_config_trial_id not in self._searcher_for_nonpoly_hp:
self._searcher_for_nonpoly_hp[seed_config_trial_id] = CFO(
space=self._space_of_nonpoly_hp,
points_to_evaluate=[nonpoly_config],
metric=self.CFO_SEARCHER_METRIC_NAME,
)
# initialize the search in set_search_properties
self._searcher_for_nonpoly_hp[seed_config_trial_id].set_search_properties(
setting={"metric_target": self.CFO_SEARCHER_LARGE_LOSS}
)
# We need to call this for once, such that the seed config in points_to_evaluate will be called
# to be tried
self._searcher_for_nonpoly_hp[seed_config_trial_id].suggest(seed_config_searcher_trial_id)
# assuming minimization
if self._searcher_for_nonpoly_hp[seed_config_trial_id].metric_target is None:
pseudo_loss = self.CFO_SEARCHER_LARGE_LOSS
else:
pseudo_loss = self._searcher_for_nonpoly_hp[seed_config_trial_id].metric_target * 0.95
pseudo_result_to_report = {}
for k, v in nonpoly_config.items():
pseudo_result_to_report["config/" + str(k)] = v
pseudo_result_to_report[self.CFO_SEARCHER_METRIC_NAME] = pseudo_loss
pseudo_result_to_report["time_total_s"] = 1
self._searcher_for_nonpoly_hp[seed_config_trial_id].on_trial_complete(
seed_config_searcher_trial_id, result=pseudo_result_to_report
)
while len(partial_new_nonpoly_configs) < self.NUMERICAL_NUM:
# suggest multiple times
new_searcher_trial_id = Trial.generate_id()
new_searcher_trial_ids.append(new_searcher_trial_id)
suggestion = self._searcher_for_nonpoly_hp[seed_config_trial_id].suggest(new_searcher_trial_id)
if suggestion is not None:
partial_new_nonpoly_configs.append(suggestion)
logger.info("partial_new_nonpoly_configs %s", partial_new_nonpoly_configs)
else:
raise NotImplementedError
if partial_new_nonpoly_configs:
hyperparameter_config_groups.append(partial_new_nonpoly_configs)
searcher_trial_ids_groups.append(new_searcher_trial_ids)
# ----------- coordinate generation of new challengers in the case of multiple groups
new_trials = []
for i in range(len(hyperparameter_config_groups)):
logger.info(
"hyperparameter_config_groups[i] %s %s",
len(hyperparameter_config_groups[i]),
hyperparameter_config_groups[i],
)
for j, new_partial_config in enumerate(hyperparameter_config_groups[i]):
new_seed_config = seed_config.copy()
new_seed_config.update(new_partial_config)
# For some groups of the hyperparameters, we may have already generated the
# searcher_trial_id. In that case, we only need to retrieve the searcher_trial_id
# instead of generating it again. So we do not generate searcher_trial_id and
# instead set the searcher_trial_id to be None. When creating a trial from a config,
# a searcher_trial_id will be generated if None is provided.
# TODO: An alternative option is to generate a searcher_trial_id for each partial config
if searcher_trial_ids_groups[i]:
new_searcher_trial_id = searcher_trial_ids_groups[i][j]
else:
new_searcher_trial_id = None
new_trial = self._create_trial_from_config(new_seed_config, new_searcher_trial_id)
new_trials.append(new_trial)
logger.info("new_configs %s", [t.trial_id for t in new_trials])
return new_trials
def _generate_independent_hp_configs(self, hp_name, current_config_value, config_domain) -> List:
if isinstance(config_domain, PolynomialExpansionSet):
seed_interactions = list(current_config_value) + list(config_domain.init_monomials)
logger.info(
"**Important** Seed namespaces (singletons and interactions): %s",
seed_interactions,
)
logger.info("current_config_value %s", current_config_value)
configs = self._generate_poly_expansion_sets(
seed_interactions,
self.EXPANSION_ORDER,
config_domain.allow_self_inter,
config_domain.highest_poly_order,
self.POLY_EXPANSION_ADDITION_NUM,
)
else:
raise NotImplementedError
configs_w_key = [{hp_name: hp_config} for hp_config in configs]
return configs_w_key
def _generate_poly_expansion_sets(
self,
seed_interactions,
order,
allow_self_inter,
highest_poly_order,
interaction_num_to_add,
):
champion_all_combinations = self._generate_all_comb(
seed_interactions, order, allow_self_inter, highest_poly_order
)
space = sorted(list(itertools.combinations(champion_all_combinations, interaction_num_to_add)))
self._random_state.shuffle(space)
candidate_configs = [set(seed_interactions) | set(item) for item in space]
final_candidate_configs = []
for c in candidate_configs:
new_c = set([e for e in c if len(e) > 1])
final_candidate_configs.append(new_c)
return final_candidate_configs
@staticmethod
def _generate_all_comb(
seed_interactions: list,
seed_interaction_order: int,
allow_self_inter: Optional[bool] = False,
highest_poly_order: Optional[int] = None,
):
"""Generate new interactions by doing up to seed_interaction_order on the seed_interactions
Args:
seed_interactions (List[str]): the see config which is a list of interactions string
(including the singletons)
seed_interaction_order (int): the maxmum order of interactions to perform on the seed_config
allow_self_inter (bool): whether self-interaction is allowed
e.g. if set False, 'aab' will be considered as 'ab', i.e. duplicates in the interaction
string are removed.
highest_poly_order (int): the highest polynomial order allowed for the resulting interaction.
e.g. if set 3, the interaction 'abcd' will be excluded.
"""
def get_interactions(list1, list2):
"""Get combinatorial list of tuples"""
new_list = []
for i in list1:
for j in list2:
# each interaction is sorted. E.g. after sorting
# 'abc' 'cba' 'bca' are all 'abc'
# this is done to ensure we can use the config as the signature
# of the trial, i.e., trial id.
new_interaction = "".join(sorted(i + j))
if new_interaction not in new_list:
new_list.append(new_interaction)
return new_list
def strip_self_inter(s):
"""Remove duplicates in an interaction string"""
if len(s) == len(set(s)):
return s
else:
# return ''.join(sorted(set(s)))
new_s = ""
char_list = []
for i in s:
if i not in char_list:
char_list.append(i)
new_s += i
return new_s
interactions = seed_interactions.copy()
all_interactions = []
while seed_interaction_order > 1:
interactions = get_interactions(interactions, seed_interactions)
seed_interaction_order -= 1
all_interactions += interactions
if not allow_self_inter:
all_interactions_no_self_inter = []
for s in all_interactions:
s_no_inter = strip_self_inter(s)
if len(s_no_inter) > 1 and s_no_inter not in all_interactions_no_self_inter:
all_interactions_no_self_inter.append(s_no_inter)
all_interactions = all_interactions_no_self_inter
if highest_poly_order is not None:
all_interactions = [c for c in all_interactions if len(c) <= highest_poly_order]
logger.info("all_combinations %s", all_interactions)
return all_interactions

169
flaml/tune/searcher/search_thread.py
View File

@ -1,169 +0,0 @@
# !
# * Copyright (c) Microsoft Corporation. All rights reserved.
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
from typing import Dict, Optional
import numpy as np
try:
from ray import __version__ as ray_version
assert ray_version >= "1.10.0"
if ray_version.startswith("1."):
from ray.tune.suggest import Searcher
else:
from ray.tune.search import Searcher
except (ImportError, AssertionError):
from .suggestion import Searcher
from .flow2 import FLOW2
from ..space import add_cost_to_space, unflatten_hierarchical
from ..result import TIME_TOTAL_S
import logging
logger = logging.getLogger(__name__)
class SearchThread:
"""Class of global or local search thread."""
def __init__(
self,
mode: str = "min",
search_alg: Optional[Searcher] = None,
cost_attr: Optional[str] = TIME_TOTAL_S,
eps: Optional[float] = 1.0,
):
"""When search_alg is omitted, use local search FLOW2."""
self._search_alg = search_alg
self._is_ls = isinstance(search_alg, FLOW2)
self._mode = mode
self._metric_op = 1 if mode == "min" else -1
self.cost_best = self.cost_last = self.cost_total = self.cost_best1 = getattr(search_alg, "cost_incumbent", 0)
self._eps = eps
self.cost_best2 = 0
self.obj_best1 = self.obj_best2 = getattr(search_alg, "best_obj", np.inf) # inherently minimize
self.best_result = None
# eci: estimated cost for improvement
self.eci = self.cost_best
self.priority = self.speed = 0
self._init_config = True
self.running = 0 # the number of running trials from the thread
self.cost_attr = cost_attr
if search_alg:
self.space = self._space = search_alg.space # unflattened space
if self.space and not isinstance(search_alg, FLOW2) and isinstance(search_alg._space, dict):
# remember const config
self._const = add_cost_to_space(self.space, {}, {})
def suggest(self, trial_id: str) -> Optional[Dict]:
"""Use the suggest() of the underlying search algorithm."""
if isinstance(self._search_alg, FLOW2):
config = self._search_alg.suggest(trial_id)
else:
try:
config = self._search_alg.suggest(trial_id)
if isinstance(self._search_alg._space, dict):
config.update(self._const)
else:
# define by run
config, self.space = unflatten_hierarchical(config, self._space)
except FloatingPointError:
logger.warning("The global search method raises FloatingPointError. " "Ignoring for this iteration.")
config = None
if config is not None:
self.running += 1
return config
def update_priority(self, eci: Optional[float] = 0):
# optimistic projection
self.priority = eci * self.speed - self.obj_best1
def update_eci(self, metric_target: float, max_speed: Optional[float] = np.inf):
# calculate eci: estimated cost for improvement over metric_target
best_obj = metric_target * self._metric_op
if not self.speed:
self.speed = max_speed
self.eci = max(self.cost_total - self.cost_best1, self.cost_best1 - self.cost_best2)
if self.obj_best1 > best_obj and self.speed > 0:
self.eci = max(self.eci, 2 * (self.obj_best1 - best_obj) / self.speed)
def _update_speed(self):
# calculate speed; use 0 for invalid speed temporarily
if self.obj_best2 > self.obj_best1:
# discount the speed if there are unfinished trials
self.speed = (
(self.obj_best2 - self.obj_best1) / self.running / (max(self.cost_total - self.cost_best2, self._eps))
)
else:
self.speed = 0
def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
"""Update the statistics of the thread."""
if not self._search_alg:
return
if not hasattr(self._search_alg, "_ot_trials") or (not error and trial_id in self._search_alg._ot_trials):
# optuna doesn't handle error
if self._is_ls or not self._init_config:
try:
self._search_alg.on_trial_complete(trial_id, result, error)
except RuntimeError as e:
# rs is used in place of optuna sometimes
if not str(e).endswith("has already finished and can not be updated."):
raise e
else:
# init config is not proposed by self._search_alg
# under this thread
self._init_config = False
if result:
self.cost_last = result.get(self.cost_attr, 1)
self.cost_total += self.cost_last
if self._search_alg.metric in result and (getattr(self._search_alg, "lexico_objectives", None) is None):
# TODO: Improve this behavior. When lexico_objectives is provided to CFO,
# related variables are not callable.
obj = result[self._search_alg.metric] * self._metric_op
if obj < self.obj_best1 or self.best_result is None:
self.cost_best2 = self.cost_best1
self.cost_best1 = self.cost_total
self.obj_best2 = obj if np.isinf(self.obj_best1) else self.obj_best1
self.obj_best1 = obj
self.cost_best = self.cost_last
self.best_result = result
if getattr(self._search_alg, "lexico_objectives", None) is None:
# TODO: Improve this behavior. When lexico_objectives is provided to CFO,
# related variables are not callable.
self._update_speed()
self.running -= 1
assert self.running >= 0
def on_trial_result(self, trial_id: str, result: Dict):
# TODO update the statistics of the thread with partial result?
if not self._search_alg:
return
if not hasattr(self._search_alg, "_ot_trials") or (trial_id in self._search_alg._ot_trials):
try:
self._search_alg.on_trial_result(trial_id, result)
except RuntimeError as e:
# rs is used in place of optuna sometimes
if not str(e).endswith("has already finished and can not be updated."):
raise e
new_cost = result.get(self.cost_attr, 1)
if self.cost_last < new_cost:
self.cost_last = new_cost
# self._update_speed()
@property
def converged(self) -> bool:
return self._search_alg.converged
@property
def resource(self) -> float:
return self._search_alg.resource
def reach(self, thread) -> bool:
"""Whether the incumbent can reach the incumbent of thread."""
return self._search_alg.reach(thread._search_alg)
@property
def can_suggest(self) -> bool:
"""Whether the thread can suggest new configs."""
return self._search_alg.can_suggest

897
flaml/tune/searcher/suggestion.py
View File

@ -1,897 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
import time
import functools
import warnings
import copy
import numpy as np
import logging
from typing import Any, Dict, Optional, Union, List, Tuple, Callable
import pickle
from .variant_generator import parse_spec_vars
from ..sample import (
Categorical,
Domain,
Float,
Integer,
LogUniform,
Quantized,
Uniform,
)
from ..trial import flatten_dict, unflatten_dict
from collections import defaultdict
logger = logging.getLogger(__name__)
UNRESOLVED_SEARCH_SPACE = str(
"You passed a `{par}` parameter to {cls} that contained unresolved search "
"space definitions. {cls} should however be instantiated with fully "
"configured search spaces only. To use Ray Tune's automatic search space "
"conversion, pass the space definition as part of the `config` argument "
"to `tune.run()` instead."
)
UNDEFINED_SEARCH_SPACE = str(
"Trying to sample a configuration from {cls}, but no search "
"space has been defined. Either pass the `{space}` argument when "
"instantiating the search algorithm, or pass a `config` to "
"`tune.run()`."
)
UNDEFINED_METRIC_MODE = str(
"Trying to sample a configuration from {cls}, but the `metric` "
"({metric}) or `mode` ({mode}) parameters have not been set. "
"Either pass these arguments when instantiating the search algorithm, "
"or pass them to `tune.run()`."
)
class Searcher:
"""Abstract class for wrapping suggesting algorithms.
Custom algorithms can extend this class easily by overriding the
`suggest` method provide generated parameters for the trials.
Any subclass that implements ``__init__`` must also call the
constructor of this class: ``super(Subclass, self).__init__(...)``.
To track suggestions and their corresponding evaluations, the method
`suggest` will be passed a trial_id, which will be used in
subsequent notifications.
Not all implementations support multi objectives.
Args:
metric (str or list): The training result objective value attribute. If
list then list of training result objective value attributes
mode (str or list): If string One of {min, max}. If list then
list of max and min, determines whether objective is minimizing
or maximizing the metric attribute. Must match type of metric.
```python
class ExampleSearch(Searcher):
def __init__(self, metric="mean_loss", mode="min", **kwargs):
super(ExampleSearch, self).__init__(
metric=metric, mode=mode, **kwargs)
self.optimizer = Optimizer()
self.configurations = {}
def suggest(self, trial_id):
configuration = self.optimizer.query()
self.configurations[trial_id] = configuration
def on_trial_complete(self, trial_id, result, **kwargs):
configuration = self.configurations[trial_id]
if result and self.metric in result:
self.optimizer.update(configuration, result[self.metric])
tune.run(trainable_function, search_alg=ExampleSearch())
```
"""
FINISHED = "FINISHED"
CKPT_FILE_TMPL = "searcher-state-{}.pkl"
def __init__(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
max_concurrent: Optional[int] = None,
use_early_stopped_trials: Optional[bool] = None,
):
self._metric = metric
self._mode = mode
if not mode or not metric:
# Early return to avoid assertions
return
assert isinstance(metric, type(mode)), "metric and mode must be of the same type"
if isinstance(mode, str):
assert mode in ["min", "max"], "if `mode` is a str must be 'min' or 'max'!"
elif isinstance(mode, list):
assert len(mode) == len(metric), "Metric and mode must be the same length"
assert all(mod in ["min", "max", "obs"] for mod in mode), "All of mode must be 'min' or 'max' or 'obs'!"
else:
raise ValueError("Mode must either be a list or string")
def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool:
"""Pass search properties to searcher.
This method acts as an alternative to instantiating search algorithms
with their own specific search spaces. Instead they can accept a
Tune config through this method. A searcher should return ``True``
if setting the config was successful, or ``False`` if it was
unsuccessful, e.g. when the search space has already been set.
Args:
metric (str): Metric to optimize
mode (str): One of ["min", "max"]. Direction to optimize.
config (dict): Tune config dict.
"""
return False
def on_trial_result(self, trial_id: str, result: Dict):
"""Optional notification for result during training.
Note that by default, the result dict may include NaNs or
may not include the optimization metric. It is up to the
subclass implementation to preprocess the result to
avoid breaking the optimization process.
Args:
trial_id (str): A unique string ID for the trial.
result (dict): Dictionary of metrics for current training progress.
Note that the result dict may include NaNs or
may not include the optimization metric. It is up to the
subclass implementation to preprocess the result to
avoid breaking the optimization process.
"""
pass
@property
def metric(self) -> str:
"""The training result objective value attribute."""
return self._metric
@property
def mode(self) -> str:
"""Specifies if minimizing or maximizing the metric."""
return self._mode
class ConcurrencyLimiter(Searcher):
"""A wrapper algorithm for limiting the number of concurrent trials.
Args:
searcher (Searcher): Searcher object that the
ConcurrencyLimiter will manage.
max_concurrent (int): Maximum concurrent samples from the underlying
searcher.
batch (bool): Whether to wait for all concurrent samples
to finish before updating the underlying searcher.
Example:
```python
from ray.tune.suggest import ConcurrencyLimiter # ray version < 2
search_alg = HyperOptSearch(metric="accuracy")
search_alg = ConcurrencyLimiter(search_alg, max_concurrent=2)
tune.run(trainable, search_alg=search_alg)
```
"""
def __init__(self, searcher: Searcher, max_concurrent: int, batch: bool = False):
assert type(max_concurrent) is int and max_concurrent > 0
self.searcher = searcher
self.max_concurrent = max_concurrent
self.batch = batch
self.live_trials = set()
self.cached_results = {}
super(ConcurrencyLimiter, self).__init__(metric=self.searcher.metric, mode=self.searcher.mode)
def suggest(self, trial_id: str) -> Optional[Dict]:
assert trial_id not in self.live_trials, f"Trial ID {trial_id} must be unique: already found in set."
if len(self.live_trials) >= self.max_concurrent:
logger.debug(
f"Not providing a suggestion for {trial_id} due to " "concurrency limit: %s/%s.",
len(self.live_trials),
self.max_concurrent,
)
return
suggestion = self.searcher.suggest(trial_id)
if suggestion not in (None, Searcher.FINISHED):
self.live_trials.add(trial_id)
return suggestion
def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
if trial_id not in self.live_trials:
return
elif self.batch:
self.cached_results[trial_id] = (result, error)
if len(self.cached_results) == self.max_concurrent:
# Update the underlying searcher once the
# full batch is completed.
for trial_id, (result, error) in self.cached_results.items():
self.searcher.on_trial_complete(trial_id, result=result, error=error)
self.live_trials.remove(trial_id)
self.cached_results = {}
else:
return
else:
self.searcher.on_trial_complete(trial_id, result=result, error=error)
self.live_trials.remove(trial_id)
def get_state(self) -> Dict:
state = self.__dict__.copy()
del state["searcher"]
return copy.deepcopy(state)
def set_state(self, state: Dict):
self.__dict__.update(state)
def save(self, checkpoint_path: str):
self.searcher.save(checkpoint_path)
def restore(self, checkpoint_path: str):
self.searcher.restore(checkpoint_path)
def on_pause(self, trial_id: str):
self.searcher.on_pause(trial_id)
def on_unpause(self, trial_id: str):
self.searcher.on_unpause(trial_id)
def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict) -> bool:
return self.searcher.set_search_properties(metric, mode, config)
try:
import optuna as ot
from optuna.distributions import BaseDistribution as OptunaDistribution
from optuna.samplers import BaseSampler
from optuna.trial import TrialState as OptunaTrialState
from optuna.trial import Trial as OptunaTrial
except ImportError:
ot = None
OptunaDistribution = None
BaseSampler = None
OptunaTrialState = None
OptunaTrial = None
DEFAULT_METRIC = "_metric"
TRAINING_ITERATION = "training_iteration"
DEFINE_BY_RUN_WARN_THRESHOLD_S = 1
def validate_warmstart(
parameter_names: List[str],
points_to_evaluate: List[Union[List, Dict]],
evaluated_rewards: List,
validate_point_name_lengths: bool = True,
):
"""Generic validation of a Searcher's warm start functionality.
Raises exceptions in case of type and length mismatches between
parameters.
If ``validate_point_name_lengths`` is False, the equality of lengths
between ``points_to_evaluate`` and ``parameter_names`` will not be
validated.
"""
if points_to_evaluate:
if not isinstance(points_to_evaluate, list):
raise TypeError("points_to_evaluate expected to be a list, got {}.".format(type(points_to_evaluate)))
for point in points_to_evaluate:
if not isinstance(point, (dict, list)):
raise TypeError(f"points_to_evaluate expected to include list or dict, " f"got {point}.")
if validate_point_name_lengths and (not len(point) == len(parameter_names)):
raise ValueError(
"Dim of point {}".format(point)
+ " and parameter_names {}".format(parameter_names)
+ " do not match."
)
if points_to_evaluate and evaluated_rewards:
if not isinstance(evaluated_rewards, list):
raise TypeError("evaluated_rewards expected to be a list, got {}.".format(type(evaluated_rewards)))
if not len(evaluated_rewards) == len(points_to_evaluate):
raise ValueError(
"Dim of evaluated_rewards {}".format(evaluated_rewards)
+ " and points_to_evaluate {}".format(points_to_evaluate)
+ " do not match."
)
class _OptunaTrialSuggestCaptor:
"""Utility to capture returned values from Optuna's suggest_ methods.
This will wrap around the ``optuna.Trial` object and decorate all
`suggest_` callables with a function capturing the returned value,
which will be saved in the ``captured_values`` dict.
"""
def __init__(self, ot_trial: OptunaTrial) -> None:
self.ot_trial = ot_trial
self.captured_values: Dict[str, Any] = {}
def _get_wrapper(self, func: Callable) -> Callable:
@functools.wraps(func)
def wrapper(*args, **kwargs):
# name is always the first arg for suggest_ methods
name = kwargs.get("name", args[0])
ret = func(*args, **kwargs)
self.captured_values[name] = ret
return ret
return wrapper
def __getattr__(self, item_name: str) -> Any:
item = getattr(self.ot_trial, item_name)
if item_name.startswith("suggest_") and callable(item):
return self._get_wrapper(item)
return item
class OptunaSearch(Searcher):
"""A wrapper around Optuna to provide trial suggestions.
`Optuna <https://optuna.org/>`_ is a hyperparameter optimization library.
In contrast to other libraries, it employs define-by-run style
hyperparameter definitions.
This Searcher is a thin wrapper around Optuna's search algorithms.
You can pass any Optuna sampler, which will be used to generate
hyperparameter suggestions.
Multi-objective optimization is supported.
Args:
space: Hyperparameter search space definition for
Optuna's sampler. This can be either a dict with
parameter names as keys and ``optuna.distributions`` as values,
or a Callable - in which case, it should be a define-by-run
function using ``optuna.trial`` to obtain the hyperparameter
values. The function should return either a dict of
constant values with names as keys, or None.
For more information, see https://optuna.readthedocs.io\
/en/stable/tutorial/10_key_features/002_configurations.html.
Warning - No actual computation should take place in the define-by-run
function. Instead, put the training logic inside the function
or class trainable passed to ``tune.run``.
metric: The training result objective value attribute. If
None but a mode was passed, the anonymous metric ``_metric``
will be used per default. Can be a list of metrics for
multi-objective optimization.
mode: One of {min, max}. Determines whether objective is
minimizing or maximizing the metric attribute. Can be a list of
modes for multi-objective optimization (corresponding to
``metric``).
points_to_evaluate: Initial parameter suggestions to be run
first. This is for when you already have some good parameters
you want to run first to help the algorithm make better suggestions
for future parameters. Needs to be a list of dicts containing the
configurations.
sampler: Optuna sampler used to
draw hyperparameter configurations. Defaults to ``MOTPESampler``
for multi-objective optimization with Optuna<2.9.0, and
``TPESampler`` in every other case.
Warning: Please note that with Optuna 2.10.0 and earlier
default ``MOTPESampler``/``TPESampler`` suffer
from performance issues when dealing with a large number of
completed trials (approx. >100). This will manifest as
a delay when suggesting new configurations.
This is an Optuna issue and may be fixed in a future
Optuna release.
seed: Seed to initialize sampler with. This parameter is only
used when ``sampler=None``. In all other cases, the sampler
you pass should be initialized with the seed already.
evaluated_rewards: If you have previously evaluated the
parameters passed in as points_to_evaluate you can avoid
re-running those trials by passing in the reward attributes
as a list so the optimiser can be told the results without
needing to re-compute the trial. Must be the same length as
points_to_evaluate.
Warning - When using ``evaluated_rewards``, the search space ``space``
must be provided as a dict with parameter names as
keys and ``optuna.distributions`` instances as values. The
define-by-run search space definition is not yet supported with
this functionality.
Tune automatically converts search spaces to Optuna's format:
```python
from ray.tune.suggest.optuna import OptunaSearch
config = {
"a": tune.uniform(6, 8)
"b": tune.loguniform(1e-4, 1e-2)
}
optuna_search = OptunaSearch(
metric="loss",
mode="min")
tune.run(trainable, config=config, search_alg=optuna_search)
```
If you would like to pass the search space manually, the code would
look like this:
```python
from ray.tune.suggest.optuna import OptunaSearch
import optuna
space = {
"a": optuna.distributions.UniformDistribution(6, 8),
"b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
}
optuna_search = OptunaSearch(
space,
metric="loss",
mode="min")
tune.run(trainable, search_alg=optuna_search)
# Equivalent Optuna define-by-run function approach:
def define_search_space(trial: optuna.Trial):
trial.suggest_float("a", 6, 8)
trial.suggest_float("b", 1e-4, 1e-2, log=True)
# training logic goes into trainable, this is just
# for search space definition
optuna_search = OptunaSearch(
define_search_space,
metric="loss",
mode="min")
tune.run(trainable, search_alg=optuna_search)
```
Multi-objective optimization is supported:
```python
from ray.tune.suggest.optuna import OptunaSearch
import optuna
space = {
"a": optuna.distributions.UniformDistribution(6, 8),
"b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
}
# Note you have to specify metric and mode here instead of
# in tune.run
optuna_search = OptunaSearch(
space,
metric=["loss1", "loss2"],
mode=["min", "max"])
# Do not specify metric and mode here!
tune.run(
trainable,
search_alg=optuna_search
)
```
You can pass configs that will be evaluated first using
``points_to_evaluate``:
```python
from ray.tune.suggest.optuna import OptunaSearch
import optuna
space = {
"a": optuna.distributions.UniformDistribution(6, 8),
"b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
}
optuna_search = OptunaSearch(
space,
points_to_evaluate=[{"a": 6.5, "b": 5e-4}, {"a": 7.5, "b": 1e-3}]
metric="loss",
mode="min")
tune.run(trainable, search_alg=optuna_search)
```
Avoid re-running evaluated trials by passing the rewards together with
`points_to_evaluate`:
```python
from ray.tune.suggest.optuna import OptunaSearch
import optuna
space = {
"a": optuna.distributions.UniformDistribution(6, 8),
"b": optuna.distributions.LogUniformDistribution(1e-4, 1e-2),
}
optuna_search = OptunaSearch(
space,
points_to_evaluate=[{"a": 6.5, "b": 5e-4}, {"a": 7.5, "b": 1e-3}]
evaluated_rewards=[0.89, 0.42]
metric="loss",
mode="min")
tune.run(trainable, search_alg=optuna_search)
```
"""
def __init__(
self,
space: Optional[
Union[
Dict[str, "OptunaDistribution"],
List[Tuple],
Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
]
] = None,
metric: Optional[Union[str, List[str]]] = None,
mode: Optional[Union[str, List[str]]] = None,
points_to_evaluate: Optional[List[Dict]] = None,
sampler: Optional["BaseSampler"] = None,
seed: Optional[int] = None,
evaluated_rewards: Optional[List] = None,
):
assert ot is not None, "Optuna must be installed! Run `pip install optuna`."
super(OptunaSearch, self).__init__(metric=metric, mode=mode)
if isinstance(space, dict) and space:
resolved_vars, domain_vars, grid_vars = parse_spec_vars(space)
if domain_vars or grid_vars:
logger.warning(UNRESOLVED_SEARCH_SPACE.format(par="space", cls=type(self).__name__))
space = self.convert_search_space(space)
else:
# Flatten to support nested dicts
space = flatten_dict(space, "/")
self._space = space
self._points_to_evaluate = points_to_evaluate or []
self._evaluated_rewards = evaluated_rewards
self._study_name = "optuna" # Fixed study name for in-memory storage
if sampler and seed:
logger.warning(
"You passed an initialized sampler to `OptunaSearch`. The "
"`seed` parameter has to be passed to the sampler directly "
"and will be ignored."
)
elif sampler:
assert isinstance(sampler, BaseSampler), (
"You can only pass an instance of " "`optuna.samplers.BaseSampler` " "as a sampler to `OptunaSearcher`."
)
self._sampler = sampler
self._seed = seed
self._completed_trials = set()
self._ot_trials = {}
self._ot_study = None
if self._space:
self._setup_study(mode)
def _setup_study(self, mode: Union[str, list]):
if self._metric is None and self._mode:
if isinstance(self._mode, list):
raise ValueError(
"If ``mode`` is a list (multi-objective optimization " "case), ``metric`` must be defined."
)
# If only a mode was passed, use anonymous metric
self._metric = DEFAULT_METRIC
pruner = ot.pruners.NopPruner()
storage = ot.storages.InMemoryStorage()
try:
from packaging import version
except ImportError:
raise ImportError("To use BlendSearch, run: pip install flaml[blendsearch]")
if self._sampler:
sampler = self._sampler
elif isinstance(mode, list) and version.parse(ot.__version__) < version.parse("2.9.0"):
# MOTPESampler deprecated in Optuna>=2.9.0
sampler = ot.samplers.MOTPESampler(seed=self._seed)
else:
sampler = ot.samplers.TPESampler(seed=self._seed)
if isinstance(mode, list):
study_direction_args = dict(
directions=["minimize" if m == "min" else "maximize" for m in mode],
)
else:
study_direction_args = dict(
direction="minimize" if mode == "min" else "maximize",
)
self._ot_study = ot.study.create_study(
storage=storage,
sampler=sampler,
pruner=pruner,
study_name=self._study_name,
load_if_exists=True,
**study_direction_args,
)
if self._points_to_evaluate:
validate_warmstart(
self._space,
self._points_to_evaluate,
self._evaluated_rewards,
validate_point_name_lengths=not callable(self._space),
)
if self._evaluated_rewards:
for point, reward in zip(self._points_to_evaluate, self._evaluated_rewards):
self.add_evaluated_point(point, reward)
else:
for point in self._points_to_evaluate:
self._ot_study.enqueue_trial(point)
def set_search_properties(self, metric: Optional[str], mode: Optional[str], config: Dict, **spec) -> bool:
if self._space:
return False
space = self.convert_search_space(config)
self._space = space
if metric:
self._metric = metric
if mode:
self._mode = mode
self._setup_study(self._mode)
return True
def _suggest_from_define_by_run_func(
self,
func: Callable[["OptunaTrial"], Optional[Dict[str, Any]]],
ot_trial: "OptunaTrial",
) -> Dict:
captor = _OptunaTrialSuggestCaptor(ot_trial)
time_start = time.time()
ret = func(captor)
time_taken = time.time() - time_start
if time_taken > DEFINE_BY_RUN_WARN_THRESHOLD_S:
warnings.warn(
"Define-by-run function passed in the `space` argument "
f"took {time_taken} seconds to "
"run. Ensure that actual computation, training takes "
"place inside Tune's train functions or Trainables "
"passed to `tune.run`."
)
if ret is not None:
if not isinstance(ret, dict):
raise TypeError(
"The return value of the define-by-run function "
"passed in the `space` argument should be "
"either None or a `dict` with `str` keys. "
f"Got {type(ret)}."
)
if not all(isinstance(k, str) for k in ret.keys()):
raise TypeError(
"At least one of the keys in the dict returned by the "
"define-by-run function passed in the `space` argument "
"was not a `str`."
)
return {**captor.captured_values, **ret} if ret else captor.captured_values
def suggest(self, trial_id: str) -> Optional[Dict]:
if not self._space:
raise RuntimeError(UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)
)
if callable(self._space):
# Define-by-run case
if trial_id not in self._ot_trials:
self._ot_trials[trial_id] = self._ot_study.ask()
ot_trial = self._ot_trials[trial_id]
params = self._suggest_from_define_by_run_func(self._space, ot_trial)
else:
# Use Optuna ask interface (since version 2.6.0)
if trial_id not in self._ot_trials:
self._ot_trials[trial_id] = self._ot_study.ask(fixed_distributions=self._space)
ot_trial = self._ot_trials[trial_id]
params = ot_trial.params
return unflatten_dict(params)
def on_trial_result(self, trial_id: str, result: Dict):
if isinstance(self.metric, list):
# Optuna doesn't support incremental results
# for multi-objective optimization
return
if trial_id in self._completed_trials:
logger.warning(
f"Received additional result for trial {trial_id}, but " f"it already finished. Result: {result}"
)
return
metric = result[self.metric]
step = result[TRAINING_ITERATION]
ot_trial = self._ot_trials[trial_id]
ot_trial.report(metric, step)
def on_trial_complete(self, trial_id: str, result: Optional[Dict] = None, error: bool = False):
if trial_id in self._completed_trials:
logger.warning(
f"Received additional completion for trial {trial_id}, but " f"it already finished. Result: {result}"
)
return
ot_trial = self._ot_trials[trial_id]
if result:
if isinstance(self.metric, list):
val = [result.get(metric, None) for metric in self.metric]
else:
val = result.get(self.metric, None)
else:
val = None
ot_trial_state = OptunaTrialState.COMPLETE
if val is None:
if error:
ot_trial_state = OptunaTrialState.FAIL
else:
ot_trial_state = OptunaTrialState.PRUNED
try:
self._ot_study.tell(ot_trial, val, state=ot_trial_state)
except Exception as exc:
logger.warning(exc) # E.g. if NaN was reported
self._completed_trials.add(trial_id)
def add_evaluated_point(
self,
parameters: Dict,
value: float,
error: bool = False,
pruned: bool = False,
intermediate_values: Optional[List[float]] = None,
):
if not self._space:
raise RuntimeError(UNDEFINED_SEARCH_SPACE.format(cls=self.__class__.__name__, space="space"))
if not self._metric or not self._mode:
raise RuntimeError(
UNDEFINED_METRIC_MODE.format(cls=self.__class__.__name__, metric=self._metric, mode=self._mode)
)
if callable(self._space):
raise TypeError(
"Define-by-run function passed in `space` argument is not "
"yet supported when using `evaluated_rewards`. Please provide "
"an `OptunaDistribution` dict or pass a Ray Tune "
"search space to `tune.run()`."
)
ot_trial_state = OptunaTrialState.COMPLETE
if error:
ot_trial_state = OptunaTrialState.FAIL
elif pruned:
ot_trial_state = OptunaTrialState.PRUNED
if intermediate_values:
intermediate_values_dict = {i: value for i, value in enumerate(intermediate_values)}
else:
intermediate_values_dict = None
trial = ot.trial.create_trial(
state=ot_trial_state,
value=value,
params=parameters,
distributions=self._space,
intermediate_values=intermediate_values_dict,
)
self._ot_study.add_trial(trial)
def save(self, checkpoint_path: str):
save_object = (
self._sampler,
self._ot_trials,
self._ot_study,
self._points_to_evaluate,
self._evaluated_rewards,
)
with open(checkpoint_path, "wb") as outputFile:
pickle.dump(save_object, outputFile)
def restore(self, checkpoint_path: str):
with open(checkpoint_path, "rb") as inputFile:
save_object = pickle.load(inputFile)
if len(save_object) == 5:
(
self._sampler,
self._ot_trials,
self._ot_study,
self._points_to_evaluate,
self._evaluated_rewards,
) = save_object
else:
# Backwards compatibility
(
self._sampler,
self._ot_trials,
self._ot_study,
self._points_to_evaluate,
) = save_object
@staticmethod
def convert_search_space(spec: Dict) -> Dict[str, Any]:
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
if not domain_vars and not grid_vars:
return {}
if grid_vars:
raise ValueError("Grid search parameters cannot be automatically converted " "to an Optuna search space.")
# Flatten and resolve again after checking for grid search.
spec = flatten_dict(spec, prevent_delimiter=True)
resolved_vars, domain_vars, grid_vars = parse_spec_vars(spec)
def resolve_value(domain: Domain) -> ot.distributions.BaseDistribution:
quantize = None
sampler = domain.get_sampler()
if isinstance(sampler, Quantized):
quantize = sampler.q
sampler = sampler.sampler
if isinstance(sampler, LogUniform):
logger.warning(
"Optuna does not handle quantization in loguniform "
"sampling. The parameter will be passed but it will "
"probably be ignored."
)
if isinstance(domain, Float):
if isinstance(sampler, LogUniform):
if quantize:
logger.warning(
"Optuna does not support both quantization and "
"sampling from LogUniform. Dropped quantization."
)
return ot.distributions.LogUniformDistribution(domain.lower, domain.upper)
elif isinstance(sampler, Uniform):
if quantize:
return ot.distributions.DiscreteUniformDistribution(domain.lower, domain.upper, quantize)
return ot.distributions.UniformDistribution(domain.lower, domain.upper)
elif isinstance(domain, Integer):
if isinstance(sampler, LogUniform):
return ot.distributions.IntLogUniformDistribution(
domain.lower, domain.upper - 1, step=quantize or 1
)
elif isinstance(sampler, Uniform):
# Upper bound should be inclusive for quantization and
# exclusive otherwise
return ot.distributions.IntUniformDistribution(
domain.lower,
domain.upper - int(bool(not quantize)),
step=quantize or 1,
)
elif isinstance(domain, Categorical):
if isinstance(sampler, Uniform):
return ot.distributions.CategoricalDistribution(domain.categories)
raise ValueError(
"Optuna search does not support parameters of type "
"`{}` with samplers of type `{}`".format(type(domain).__name__, type(domain.sampler).__name__)
)
# Parameter name is e.g. "a/b/c" for nested dicts
values = {"/".join(path): resolve_value(domain) for path, domain in domain_vars}
return values

318
flaml/tune/searcher/variant_generator.py
View File

@ -1,318 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
import copy
import logging
from typing import Any, Dict, Generator, List, Tuple
import numpy
import random
from ..sample import Categorical, Domain, RandomState
try:
from ray import __version__ as ray_version
if ray_version.startswith("1."):
from ray.tune.sample import Domain as RayDomain
else:
from ray.tune.search.sample import Domain as RayDomain
except ImportError:
RayDomain = Domain
logger = logging.getLogger(__name__)
class TuneError(Exception):
"""General error class raised by ray.tune."""
pass
def generate_variants(
unresolved_spec: Dict,
constant_grid_search: bool = False,
random_state: "RandomState" = None,
) -> Generator[Tuple[Dict, Dict], None, None]:
"""Generates variants from a spec (dict) with unresolved values.
There are two types of unresolved values:
Grid search: These define a grid search over values. For example, the
following grid search values in a spec will produce six distinct
variants in combination:
"activation": grid_search(["relu", "tanh"])
"learning_rate": grid_search([1e-3, 1e-4, 1e-5])
Lambda functions: These are evaluated to produce a concrete value, and
can express dependencies or conditional distributions between values.
They can also be used to express random search (e.g., by calling
into the `random` or `np` module).
"cpu": lambda spec: spec.config.num_workers
"batch_size": lambda spec: random.uniform(1, 1000)
Finally, to support defining specs in plain JSON / YAML, grid search
and lambda functions can also be defined alternatively as follows:
"activation": {"grid_search": ["relu", "tanh"]}
"cpu": {"eval": "spec.config.num_workers"}
Use `format_vars` to format the returned dict of hyperparameters.
Yields:
(Dict of resolved variables, Spec object)
"""
for resolved_vars, spec in _generate_variants(
unresolved_spec,
constant_grid_search=constant_grid_search,
random_state=random_state,
):
assert not _unresolved_values(spec)
yield resolved_vars, spec
def grid_search(values: List) -> Dict[str, List]:
"""Convenience method for specifying grid search over a value.
Arguments:
values: An iterable whose parameters will be gridded.
"""
return {"grid_search": values}
_STANDARD_IMPORTS = {
"random": random,
"np": numpy,
}
_MAX_RESOLUTION_PASSES = 20
def parse_spec_vars(
spec: Dict,
) -> Tuple[List[Tuple[Tuple, Any]], List[Tuple[Tuple, Any]], List[Tuple[Tuple, Any]]]:
resolved, unresolved = _split_resolved_unresolved_values(spec)
resolved_vars = list(resolved.items())
if not unresolved:
return resolved_vars, [], []
grid_vars = []
domain_vars = []
for path, value in unresolved.items():
if value.is_grid():
grid_vars.append((path, value))
else:
domain_vars.append((path, value))
grid_vars.sort()
return resolved_vars, domain_vars, grid_vars
def _generate_variants(
spec: Dict, constant_grid_search: bool = False, random_state: "RandomState" = None
) -> Tuple[Dict, Dict]:
spec = copy.deepcopy(spec)
_, domain_vars, grid_vars = parse_spec_vars(spec)
if not domain_vars and not grid_vars:
yield {}, spec
return
# Variables to resolve
to_resolve = domain_vars
all_resolved = True
if constant_grid_search:
# In this path, we first sample random variables and keep them constant
# for grid search.
# `_resolve_domain_vars` will alter `spec` directly
all_resolved, resolved_vars = _resolve_domain_vars(
spec, domain_vars, allow_fail=True, random_state=random_state
)
if not all_resolved:
# Not all variables have been resolved, but remove those that have
# from the `to_resolve` list.
to_resolve = [(r, d) for r, d in to_resolve if r not in resolved_vars]
grid_search = _grid_search_generator(spec, grid_vars)
for resolved_spec in grid_search:
if not constant_grid_search or not all_resolved:
# In this path, we sample the remaining random variables
_, resolved_vars = _resolve_domain_vars(resolved_spec, to_resolve, random_state=random_state)
for resolved, spec in _generate_variants(
resolved_spec,
constant_grid_search=constant_grid_search,
random_state=random_state,
):
for path, value in grid_vars:
resolved_vars[path] = _get_value(spec, path)
for k, v in resolved.items():
if k in resolved_vars and v != resolved_vars[k] and _is_resolved(resolved_vars[k]):
raise ValueError(
"The variable `{}` could not be unambiguously "
"resolved to a single value. Consider simplifying "
"your configuration.".format(k)
)
resolved_vars[k] = v
yield resolved_vars, spec
def assign_value(spec: Dict, path: Tuple, value: Any):
for k in path[:-1]:
spec = spec[k]
spec[path[-1]] = value
def _get_value(spec: Dict, path: Tuple) -> Any:
for k in path:
spec = spec[k]
return spec
def _resolve_domain_vars(
spec: Dict,
domain_vars: List[Tuple[Tuple, Domain]],
allow_fail: bool = False,
random_state: "RandomState" = None,
) -> Tuple[bool, Dict]:
resolved = {}
error = True
num_passes = 0
while error and num_passes < _MAX_RESOLUTION_PASSES:
num_passes += 1
error = False
for path, domain in domain_vars:
if path in resolved:
continue
try:
value = domain.sample(_UnresolvedAccessGuard(spec), random_state=random_state)
except RecursiveDependencyError as e:
error = e
# except Exception:
# raise ValueError(
# "Failed to evaluate expression: {}: {}".format(path, domain)
# )
else:
assign_value(spec, path, value)
resolved[path] = value
if error:
if not allow_fail:
raise error
else:
return False, resolved
return True, resolved
def _grid_search_generator(unresolved_spec: Dict, grid_vars: List) -> Generator[Dict, None, None]:
value_indices = [0] * len(grid_vars)
def increment(i):
value_indices[i] += 1
if value_indices[i] >= len(grid_vars[i][1]):
value_indices[i] = 0
if i + 1 < len(value_indices):
return increment(i + 1)
else:
return True
return False
if not grid_vars:
yield unresolved_spec
return
while value_indices[-1] < len(grid_vars[-1][1]):
spec = copy.deepcopy(unresolved_spec)
for i, (path, values) in enumerate(grid_vars):
assign_value(spec, path, values[value_indices[i]])
yield spec
if grid_vars:
done = increment(0)
if done:
break
def _is_resolved(v) -> bool:
resolved, _ = _try_resolve(v)
return resolved
def _try_resolve(v) -> Tuple[bool, Any]:
if isinstance(v, (Domain, RayDomain)):
# Domain to sample from
return False, v
elif isinstance(v, dict) and len(v) == 1 and "grid_search" in v:
# Grid search values
grid_values = v["grid_search"]
if not isinstance(grid_values, list):
raise TuneError("Grid search expected list of values, got: {}".format(grid_values))
return False, Categorical(grid_values).grid()
return True, v
def _split_resolved_unresolved_values(
spec: Dict,
) -> Tuple[Dict[Tuple, Any], Dict[Tuple, Any]]:
resolved_vars = {}
unresolved_vars = {}
for k, v in spec.items():
resolved, v = _try_resolve(v)
if not resolved:
unresolved_vars[(k,)] = v
elif isinstance(v, dict):
# Recurse into a dict
(
_resolved_children,
_unresolved_children,
) = _split_resolved_unresolved_values(v)
for path, value in _resolved_children.items():
resolved_vars[(k,) + path] = value
for path, value in _unresolved_children.items():
unresolved_vars[(k,) + path] = value
elif isinstance(v, list):
# Recurse into a list
for i, elem in enumerate(v):
(
_resolved_children,
_unresolved_children,
) = _split_resolved_unresolved_values({i: elem})
for path, value in _resolved_children.items():
resolved_vars[(k,) + path] = value
for path, value in _unresolved_children.items():
unresolved_vars[(k,) + path] = value
else:
resolved_vars[(k,)] = v
return resolved_vars, unresolved_vars
def _unresolved_values(spec: Dict) -> Dict[Tuple, Any]:
return _split_resolved_unresolved_values(spec)[1]
def has_unresolved_values(spec: Dict) -> bool:
return True if _unresolved_values(spec) else False
class _UnresolvedAccessGuard(dict):
def __init__(self, *args, **kwds):
super(_UnresolvedAccessGuard, self).__init__(*args, **kwds)
self.__dict__ = self
def __getattribute__(self, item):
value = dict.__getattribute__(self, item)
if not _is_resolved(value):
raise RecursiveDependencyError("`{}` recursively depends on {}".format(item, value))
elif isinstance(value, dict):
return _UnresolvedAccessGuard(value)
else:
return value
class RecursiveDependencyError(Exception):
def __init__(self, msg: str):
Exception.__init__(self, msg)

547
flaml/tune/space.py
View File

@ -1,547 +0,0 @@
try:
from ray import __version__ as ray_version
assert ray_version >= "1.10.0"
if ray_version.startswith("1."):
from ray.tune import sample
from ray.tune.suggest.variant_generator import generate_variants
else:
from ray.tune.search import sample
from ray.tune.search.variant_generator import generate_variants
except (ImportError, AssertionError):
from . import sample
from .searcher.variant_generator import generate_variants
from typing import Dict, Optional, Any, Tuple, Generator, List, Union
import numpy as np
import logging
logger = logging.getLogger(__name__)
def generate_variants_compatible(
unresolved_spec: Dict, constant_grid_search: bool = False, random_state=None
) -> Generator[Tuple[Dict, Dict], None, None]:
try:
return generate_variants(unresolved_spec, constant_grid_search, random_state)
except TypeError:
return generate_variants(unresolved_spec, constant_grid_search)
def is_constant(space: Union[Dict, List]) -> bool:
"""Whether the search space is all constant.
Returns:
A bool of whether the search space is all constant.
"""
if isinstance(space, dict):
for domain in space.values():
if isinstance(domain, (dict, list)):
if not is_constant(domain):
return False
continue
if isinstance(domain, sample.Domain):
return False
return True
elif isinstance(space, list):
for item in space:
if not is_constant(item):
return False
return True
return not isinstance(space, sample.Domain)
def define_by_run_func(trial, space: Dict, path: str = "") -> Optional[Dict[str, Any]]:
"""Define-by-run function to create the search space.
Returns:
A dict with constant values.
"""
config = {}
for key, domain in space.items():
if path:
key = path + "/" + key
if isinstance(domain, dict):
config.update(define_by_run_func(trial, domain, key))
continue
if not isinstance(domain, sample.Domain):
config[key] = domain
continue
sampler = domain.get_sampler()
quantize = None
if isinstance(sampler, sample.Quantized):
quantize = sampler.q
sampler = sampler.sampler
if isinstance(sampler, sample.LogUniform):
logger.warning(
"Optuna does not handle quantization in loguniform "
"sampling. The parameter will be passed but it will "
"probably be ignored."
)
if isinstance(domain, sample.Float):
if isinstance(sampler, sample.LogUniform):
if quantize:
logger.warning(
"Optuna does not support both quantization and "
"sampling from LogUniform. Dropped quantization."
)
trial.suggest_float(key, domain.lower, domain.upper, log=True)
elif isinstance(sampler, sample.Uniform):
if quantize:
trial.suggest_float(key, domain.lower, domain.upper, step=quantize)
else:
trial.suggest_float(key, domain.lower, domain.upper)
else:
raise ValueError(
"Optuna search does not support parameters of type "
"`{}` with samplers of type `{}`".format(type(domain).__name__, type(domain.sampler).__name__)
)
elif isinstance(domain, sample.Integer):
if isinstance(sampler, sample.LogUniform):
trial.suggest_int(key, domain.lower, domain.upper - int(bool(not quantize)), log=True)
elif isinstance(sampler, sample.Uniform):
# Upper bound should be inclusive for quantization and
# exclusive otherwise
trial.suggest_int(
key,
domain.lower,
domain.upper - int(bool(not quantize)),
step=quantize or 1,
)
elif isinstance(domain, sample.Categorical):
if isinstance(sampler, sample.Uniform):
if not hasattr(domain, "choices"):
domain.choices = list(range(len(domain.categories)))
choices = domain.choices
# This choice needs to be removed from the final config
index = trial.suggest_categorical(key + "_choice_", choices)
choice = domain.categories[index]
if isinstance(choice, dict):
key += f":{index}"
# the suffix needs to be removed from the final config
config.update(define_by_run_func(trial, choice, key))
else:
raise ValueError(
"Optuna search does not support parameters of type "
"`{}` with samplers of type `{}`".format(type(domain).__name__, type(domain.sampler).__name__)
)
# Return all constants in a dictionary.
return config
# def convert_key(
# conf: Dict, space: Dict, path: str = ""
# ) -> Optional[Dict[str, Any]]:
# """Convert config keys to define-by-run keys.
# Returns:
# A dict with converted keys.
# """
# config = {}
# for key, domain in space.items():
# value = conf[key]
# if path:
# key = path + '/' + key
# if isinstance(domain, dict):
# config.update(convert_key(conf[key], domain, key))
# elif isinstance(domain, sample.Categorical):
# index = indexof(domain, value)
# config[key + '_choice_'] = index
# if isinstance(value, dict):
# key += f":{index}"
# config.update(convert_key(value, domain.categories[index], key))
# else:
# config[key] = value
# return config
def unflatten_hierarchical(config: Dict, space: Dict) -> Tuple[Dict, Dict]:
"""Unflatten hierarchical config."""
hier = {}
subspace = {}
for key, value in config.items():
if "/" in key:
key = key[key.rfind("/") + 1 :]
if ":" in key:
pos = key.rfind(":")
true_key = key[:pos]
choice = int(key[pos + 1 :])
hier[true_key], subspace[true_key] = unflatten_hierarchical(value, space[true_key][choice])
else:
if key.endswith("_choice_"):
key = key[:-8]
domain = space.get(key)
if domain is not None:
if isinstance(domain, dict):
value, domain = unflatten_hierarchical(value, domain)
subspace[key] = domain
if isinstance(domain, sample.Domain):
sampler = domain.sampler
if isinstance(domain, sample.Categorical):
value = domain.categories[value]
if isinstance(value, dict):
continue
elif isinstance(sampler, sample.Quantized):
q = sampler.q
sampler = sampler.sampler
if isinstance(sampler, sample.LogUniform):
value = domain.cast(np.round(value / q) * q)
hier[key] = value
return hier, subspace
def add_cost_to_space(space: Dict, low_cost_point: Dict, choice_cost: Dict):
"""Update the space in place by adding low_cost_point and choice_cost.
Returns:
A dict with constant values.
"""
config = {}
for key in space:
domain = space[key]
if not isinstance(domain, sample.Domain):
if isinstance(domain, dict):
low_cost = low_cost_point.get(key, {})
choice_cost_list = choice_cost.get(key, {})
const = add_cost_to_space(domain, low_cost, choice_cost_list)
if const:
config[key] = const
else:
config[key] = domain
continue
low_cost = low_cost_point.get(key)
choice_cost_list = choice_cost.get(key)
if callable(getattr(domain, "get_sampler", None)):
sampler = domain.get_sampler()
if isinstance(sampler, sample.Quantized):
sampler = sampler.get_sampler()
domain.bounded = str(sampler) != "Normal"
if isinstance(domain, sample.Categorical):
domain.const = []
for i, cat in enumerate(domain.categories):
if isinstance(cat, dict):
if isinstance(low_cost, list):
low_cost_dict = low_cost[i]
else:
low_cost_dict = {}
if choice_cost_list:
choice_cost_dict = choice_cost_list[i]
else:
choice_cost_dict = {}
domain.const.append(add_cost_to_space(cat, low_cost_dict, choice_cost_dict))
else:
domain.const.append(None)
if choice_cost_list:
if len(choice_cost_list) == len(domain.categories):
domain.choice_cost = choice_cost_list
else:
domain.choice_cost = choice_cost_list[-1]
# sort the choices by cost
cost = np.array(domain.choice_cost)
ind = np.argsort(cost)
domain.categories = [domain.categories[i] for i in ind]
domain.choice_cost = cost[ind]
domain.const = [domain.const[i] for i in ind]
domain.ordered = True
else:
ordered = getattr(domain, "ordered", None)
if ordered is None:
# automatically decide whether to order the choices based on the value type
domain.ordered = ordered = all(isinstance(x, (int, float)) for x in domain.categories)
if ordered:
# sort the choices by value
ind = np.argsort(domain.categories)
domain.categories = [domain.categories[i] for i in ind]
if low_cost and low_cost not in domain.categories:
assert isinstance(low_cost, list), f"low cost {low_cost} not in domain {domain.categories}"
if domain.ordered:
sorted_points = [low_cost[i] for i in ind]
for i, point in enumerate(sorted_points):
low_cost[i] = point
if len(low_cost) > len(domain.categories):
if domain.ordered:
low_cost[-1] = int(np.where(ind == low_cost[-1])[0])
domain.low_cost_point = low_cost[-1]
return
if low_cost:
domain.low_cost_point = low_cost
return config
def normalize(
config: Dict,
space: Dict,
reference_config: Dict,
normalized_reference_config: Dict,
recursive: bool = False,
):
"""Normalize config in space according to reference_config.
Normalize each dimension in config to [0,1].
"""
config_norm = {}
for key, value in config.items():
domain = space.get(key)
if domain is None: # e.g., resource_attr
config_norm[key] = value
continue
if not callable(getattr(domain, "get_sampler", None)):
if recursive and isinstance(domain, dict):
config_norm[key] = normalize(value, domain, reference_config[key], {})
else:
config_norm[key] = value
continue
# domain: sample.Categorical/Integer/Float/Function
if isinstance(domain, sample.Categorical):
norm = None
# value is: a category, a nested dict, or a low_cost_point list
if value not in domain.categories:
# nested
if isinstance(value, list):
# low_cost_point list
norm = []
for i, cat in enumerate(domain.categories):
norm.append(normalize(value[i], cat, reference_config[key][i], {}) if recursive else value[i])
if len(value) > len(domain.categories):
# the low cost index was appended to low_cost_point list
index = value[-1]
value = domain.categories[index]
elif not recursive:
# no low cost index. randomly pick one as init point
continue
else:
# nested dict
config_norm[key] = value
continue
# normalize categorical
n = len(domain.categories)
if domain.ordered:
normalized = (domain.categories.index(value) + 0.5) / n
elif key in normalized_reference_config:
normalized = (
normalized_reference_config[key]
if value == reference_config[key]
else (normalized_reference_config[key] + 1 / n) % 1
)
else:
normalized = 0.5
if norm:
norm.append(normalized)
else:
norm = normalized
config_norm[key] = norm
continue
# Uniform/LogUniform/Normal/Base
sampler = domain.get_sampler()
if isinstance(sampler, sample.Quantized):
# sampler is sample.Quantized
quantize = sampler.q
sampler = sampler.get_sampler()
else:
quantize = None
if str(sampler) == "LogUniform":
upper = domain.upper - (isinstance(domain, sample.Integer) & (quantize is None))
config_norm[key] = np.log(value / domain.lower) / np.log(upper / domain.lower)
elif str(sampler) == "Uniform":
upper = domain.upper - (isinstance(domain, sample.Integer) & (quantize is None))
config_norm[key] = (value - domain.lower) / (upper - domain.lower)
elif str(sampler) == "Normal":
# N(mean, sd) -> N(0,1)
config_norm[key] = (value - sampler.mean) / sampler.sd
# else:
# config_norm[key] = value
return config_norm
def denormalize(
config: Dict,
space: Dict,
reference_config: Dict,
normalized_reference_config: Dict,
random_state,
):
config_denorm = {}
for key, value in config.items():
if key in space:
# domain: sample.Categorical/Integer/Float/Function
domain = space[key]
if isinstance(value, dict) or not callable(getattr(domain, "get_sampler", None)):
config_denorm[key] = value
else:
if isinstance(domain, sample.Categorical):
# denormalize categorical
n = len(domain.categories)
if isinstance(value, list):
# denormalize list
choice = min(n - 1, int(np.floor(value[-1] * n))) # max choice is n-1
config_denorm[key] = point = value[choice]
point["_choice_"] = choice
continue
if domain.ordered:
config_denorm[key] = domain.categories[min(n - 1, int(np.floor(value * n)))]
else:
assert key in normalized_reference_config
if min(n - 1, np.floor(value * n)) == min(
n - 1, np.floor(normalized_reference_config[key] * n)
):
config_denorm[key] = reference_config[key]
else: # ****random value each time!****
config_denorm[key] = random_state.choice(
[x for x in domain.categories if x != reference_config[key]]
)
continue
# Uniform/LogUniform/Normal/Base
sampler = domain.get_sampler()
if isinstance(sampler, sample.Quantized):
# sampler is sample.Quantized
quantize = sampler.q
sampler = sampler.get_sampler()
else:
quantize = None
# Handle Log/Uniform
if str(sampler) == "LogUniform":
upper = domain.upper - (isinstance(domain, sample.Integer) & (quantize is None))
config_denorm[key] = (upper / domain.lower) ** value * domain.lower
elif str(sampler) == "Uniform":
upper = domain.upper - (isinstance(domain, sample.Integer) & (quantize is None))
config_denorm[key] = value * (upper - domain.lower) + domain.lower
elif str(sampler) == "Normal":
# denormalization for 'Normal'
config_denorm[key] = value * sampler.sd + sampler.mean
# else:
# config_denorm[key] = value
# Handle quantized
if quantize is not None:
config_denorm[key] = np.round(np.divide(config_denorm[key], quantize)) * quantize
# Handle int (4.6 -> 5)
if isinstance(domain, sample.Integer):
config_denorm[key] = int(round(config_denorm[key]))
else: # resource_attr
config_denorm[key] = value
return config_denorm
def equal(config, const) -> bool:
if config == const:
return True
if not isinstance(config, Dict) or not isinstance(const, Dict):
return False
return all(equal(config[key], value) for key, value in const.items())
def indexof(domain: Dict, config: Dict) -> int:
"""Find the index of config in domain.categories."""
index = config.get("_choice_")
if index is not None:
return index
if config in domain.categories:
return domain.categories.index(config)
for i, cat in enumerate(domain.categories):
if not isinstance(cat, dict):
continue
# print(len(cat), len(config))
# if len(cat) != len(config):
# continue
# print(cat.keys())
if not set(config.keys()).issubset(set(cat.keys())):
continue
if equal(config, domain.const[i]):
# assumption: the concatenation of constants is a unique identifier
return i
return None
def complete_config(
partial_config: Dict,
space: Dict,
flow2,
disturb: bool = False,
lower: Optional[Dict] = None,
upper: Optional[Dict] = None,
) -> Tuple[Dict, Dict]:
"""Complete partial config in space.
Returns:
config, space.
"""
config = partial_config.copy()
normalized = normalize(config, space, partial_config, {})
# print("normalized", normalized)
if disturb:
for key, value in normalized.items():
domain = space.get(key)
if getattr(domain, "ordered", True) is False:
# don't change unordered cat choice
continue
if not callable(getattr(domain, "get_sampler", None)):
continue
if upper and lower:
up, low = upper[key], lower[key]
if isinstance(up, list):
gauss_std = (up[-1] - low[-1]) or flow2.STEPSIZE
up[-1] += flow2.STEPSIZE
low[-1] -= flow2.STEPSIZE
else:
gauss_std = (up - low) or flow2.STEPSIZE
# allowed bound
up += flow2.STEPSIZE
low -= flow2.STEPSIZE
elif domain.bounded:
up, low, gauss_std = 1, 0, 1.0
else:
up, low, gauss_std = np.Inf, -np.Inf, 1.0
if domain.bounded:
if isinstance(up, list):
up[-1] = min(up[-1], 1)
low[-1] = max(low[-1], 0)
else:
up = min(up, 1)
low = max(low, 0)
delta = flow2.rand_vector_gaussian(1, gauss_std)[0]
if isinstance(value, list):
# points + normalized index
value[-1] = max(low[-1], min(up[-1], value[-1] + delta))
else:
normalized[key] = max(low, min(up, value + delta))
config = denormalize(normalized, space, config, normalized, flow2._random)
# print("denormalized", config)
for key, value in space.items():
if key not in config:
config[key] = value
for _, generated in generate_variants_compatible({"config": config}, random_state=flow2.rs_random):
config = generated["config"]
break
subspace = {}
for key, domain in space.items():
value = config[key]
if isinstance(value, dict):
if isinstance(domain, sample.Categorical):
# nested space
index = indexof(domain, value)
# point = partial_config.get(key)
# if isinstance(point, list): # low cost point list
# point = point[index]
# else:
# point = {}
config[key], subspace[key] = complete_config(
value,
domain.categories[index],
flow2,
disturb,
lower and lower.get(key) and lower[key][index],
upper and upper.get(key) and upper[key][index],
)
assert "_choice_" not in subspace[key], "_choice_ is a reserved key for hierarchical search space"
subspace[key]["_choice_"] = index
else:
config[key], subspace[key] = complete_config(
value,
space[key],
flow2,
disturb,
lower and lower.get(key),
upper and upper.get(key),
)
continue
subspace[key] = domain
return config, subspace

8
flaml/tune/spark/__init__.py
View File

@ -1,8 +0,0 @@
from flaml.tune.spark.utils import (
check_spark,
get_n_cpus,
with_parameters,
broadcast_code,
)
__all__ = ["check_spark", "get_n_cpus", "with_parameters", "broadcast_code"]

301
flaml/tune/spark/utils.py
View File

@ -1,301 +0,0 @@
import logging
import os
import textwrap
import threading
import time
from functools import lru_cache, partial
logger = logging.getLogger(__name__)
logger_formatter = logging.Formatter(
"[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s", "%m-%d %H:%M:%S"
)
logger.propagate = False
os.environ["PYARROW_IGNORE_TIMEZONE"] = "1"
try:
import pyspark
from pyspark.sql import SparkSession
from pyspark.util import VersionUtils
import py4j
except ImportError:
_have_spark = False
py4j = None
_spark_major_minor_version = (0, 0)
else:
_have_spark = True
_spark_major_minor_version = VersionUtils.majorMinorVersion(pyspark.__version__)
@lru_cache(maxsize=2)
def check_spark():
"""Check if Spark is installed and running.
Result of the function will be cached since test once is enough. As lru_cache will not
cache exceptions, we don't raise exceptions here but only log a warning message.
Returns:
Return (True, None) if the check passes, otherwise log the exception message and
return (False, Exception(msg)). The exception can be raised by the caller.
"""
logger.debug("\nchecking Spark installation...This line should appear only once.\n")
if not _have_spark:
msg = """use_spark=True requires installation of PySpark. Please run pip install flaml[spark]
and check [here](https://spark.apache.org/docs/latest/api/python/getting_started/install.html)
for more details about installing Spark."""
return False, ImportError(msg)
if _spark_major_minor_version[0] < 3:
msg = "Spark version must be >= 3.0 to use flaml[spark]"
return False, ImportError(msg)
try:
SparkSession.builder.getOrCreate()
except RuntimeError as e:
return False, RuntimeError(e)
return True, None
def get_n_cpus(node="driver"):
"""Get the number of CPU cores of the given type of node.
Args:
node: string | The type of node to get the number of cores. Can be 'driver' or 'executor'.
Default is 'driver'.
Returns:
An int of the number of CPU cores.
"""
assert node in ["driver", "executor"]
try:
n_cpus = int(SparkSession.builder.getOrCreate().sparkContext.getConf().get(f"spark.{node}.cores"))
except (TypeError, RuntimeError):
n_cpus = os.cpu_count()
return n_cpus
def with_parameters(trainable, **kwargs):
"""Wrapper for trainables to pass arbitrary large data objects.
This wrapper function will store all passed parameters in the Spark
Broadcast variable.
Args:
trainable: Trainable to wrap.
**kwargs: parameters to store in object store.
Returns:
A new function with partial application of the given arguments
and keywords. The given arguments and keywords will be broadcasted
to all the executors.
```python
import pyspark
import flaml
from sklearn.datasets import load_iris
def train(config, data=None):
if isinstance(data, pyspark.broadcast.Broadcast):
data = data.value
print(config, data)
data = load_iris()
with_parameters_train = flaml.tune.spark.utils.with_parameters(train, data=data)
with_parameters_train(config=1)
train(config={"metric": "accuracy"})
```
"""
if not callable(trainable):
raise ValueError(
f"`with_parameters() only works with function trainables`. " f"Got type: " f"{type(trainable)}."
)
spark_available, spark_error_msg = check_spark()
if not spark_available:
raise spark_error_msg
spark = SparkSession.builder.getOrCreate()
bc_kwargs = dict()
for k, v in kwargs.items():
bc_kwargs[k] = spark.sparkContext.broadcast(v)
return partial(trainable, **bc_kwargs)
def broadcast_code(custom_code="", file_name="mylearner"):
"""Write customized learner/metric code contents to a file for importing.
It is necessary for using the customized learner/metric in spark backend.
The path of the learner/metric file will be returned.
Args:
custom_code: str, default="" | code contents of the custom learner/metric.
file_name: str, default="mylearner" | file name of the custom learner/metric.
Returns:
The path of the custom code file.
```python
from flaml.tune.spark.utils import broadcast_code
from flaml.automl.model import LGBMEstimator
custom_code = '''
from flaml.automl.model import LGBMEstimator
from flaml import tune
class MyLargeLGBM(LGBMEstimator):
@classmethod
def search_space(cls, **params):
return {
"n_estimators": {
"domain": tune.lograndint(lower=4, upper=32768),
"init_value": 32768,
"low_cost_init_value": 4,
},
"num_leaves": {
"domain": tune.lograndint(lower=4, upper=32768),
"init_value": 32768,
"low_cost_init_value": 4,
},
}
'''
broadcast_code(custom_code=custom_code)
from flaml.tune.spark.mylearner import MyLargeLGBM
assert isinstance(MyLargeLGBM(), LGBMEstimator)
```
"""
flaml_path = os.path.dirname(os.path.abspath(__file__))
custom_code = textwrap.dedent(custom_code)
custom_path = os.path.join(flaml_path, file_name + ".py")
with open(custom_path, "w") as f:
f.write(custom_code)
return custom_path
def get_broadcast_data(broadcast_data):
"""Get the broadcast data from the broadcast variable.
Args:
broadcast_data: pyspark.broadcast.Broadcast | the broadcast variable.
Returns:
The broadcast data.
"""
if _have_spark and isinstance(broadcast_data, pyspark.broadcast.Broadcast):
broadcast_data = broadcast_data.value
return broadcast_data
class PySparkOvertimeMonitor:
"""A context manager class to monitor if the PySpark job is overtime.
Example:
```python
with PySparkOvertimeMonitor(time_start, time_budget_s, force_cancel, parallel=parallel):
results = parallel(
delayed(evaluation_function)(trial_to_run.config)
for trial_to_run in trials_to_run
)
```
"""
def __init__(
self,
start_time,
time_budget_s,
force_cancel=False,
cancel_func=None,
parallel=None,
sc=None,
):
"""Constructor.
Specify the time budget and start time of the PySpark job, and specify how to cancel them.
Args:
Args relate to monitoring:
start_time: float | The start time of the PySpark job.
time_budget_s: float | The time budget of the PySpark job in seconds.
force_cancel: boolean, default=False | Whether to forcely cancel the PySpark job if overtime.
Args relate to how to cancel the PySpark job:
(Only one of the following args will work. Priorities from top to bottom)
cancel_func: function | A function to cancel the PySpark job.
parallel: joblib.parallel.Parallel | Specify this if using joblib_spark as a parallel backend. It will call parallel._backend.terminate() to cancel the jobs.
sc: pyspark.SparkContext object | You can pass a specific SparkContext.
If all three args is None, the monitor will call pyspark.SparkContext.getOrCreate().cancelAllJobs() to cancel the jobs.
"""
self._time_budget_s = time_budget_s
self._start_time = start_time
self._force_cancel = force_cancel
# TODO: add support for non-spark scenario
if self._force_cancel and _have_spark:
self._monitor_daemon = None
self._finished_flag = False
self._cancel_flag = False
self.sc = None
if cancel_func:
self.__cancel_func = cancel_func
elif parallel:
self.__cancel_func = parallel._backend.terminate
elif sc:
self.sc = sc
self.__cancel_func = self.sc.cancelAllJobs
else:
self.__cancel_func = pyspark.SparkContext.getOrCreate().cancelAllJobs
# logger.info(self.__cancel_func)
def _monitor_overtime(self):
"""The lifecycle function for monitor thread."""
if self._time_budget_s is None:
self.__cancel_func()
self._cancel_flag = True
return
while time.time() - self._start_time <= self._time_budget_s:
time.sleep(0.01)
if self._finished_flag:
return
self.__cancel_func()
self._cancel_flag = True
return
def _setLogLevel(self, level):
"""Set the log level of the spark context.
Set the level to OFF could block the warning message of Spark."""
if self.sc:
self.sc.setLogLevel(level)
else:
pyspark.SparkContext.getOrCreate().setLogLevel(level)
def __enter__(self):
"""Enter the context manager.
This will start a monitor thread if spark is available and force_cancel is True.
"""
if self._force_cancel and _have_spark:
self._monitor_daemon = threading.Thread(target=self._monitor_overtime)
# logger.setLevel("INFO")
logger.info("monitor started")
self._setLogLevel("OFF")
self._monitor_daemon.start()
def __exit__(self, exc_type, exc_value, exc_traceback):
"""Exit the context manager.
This will wait for the monitor thread to nicely exit."""
if self._force_cancel and _have_spark:
self._finished_flag = True
self._monitor_daemon.join()
if self._cancel_flag:
print()
logger.warning("Time exceeded, canceled jobs")
# self._setLogLevel("WARN")
if not exc_type:
return True
elif exc_type == py4j.protocol.Py4JJavaError:
return True
else:
return False

141
flaml/tune/trial.py
View File

@ -1,141 +0,0 @@
# Copyright 2020 The Ray Authors.
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# This source file is adapted here because ray does not fully support Windows.
# Copyright (c) Microsoft Corporation.
import uuid
import time
from numbers import Number
from collections import deque
def flatten_dict(dt, delimiter="/", prevent_delimiter=False):
dt = dt.copy()
if prevent_delimiter and any(delimiter in key for key in dt):
# Raise if delimiter is any of the keys
raise ValueError(
"Found delimiter `{}` in key when trying to flatten array."
"Please avoid using the delimiter in your specification."
)
while any(isinstance(v, dict) for v in dt.values()):
remove = []
add = {}
for key, value in dt.items():
if isinstance(value, dict):
for subkey, v in value.items():
if prevent_delimiter and delimiter in subkey:
# Raise if delimiter is in any of the subkeys
raise ValueError(
"Found delimiter `{}` in key when trying to "
"flatten array. Please avoid using the delimiter "
"in your specification."
)
add[delimiter.join([key, str(subkey)])] = v
remove.append(key)
dt.update(add)
for k in remove:
del dt[k]
return dt
def unflatten_dict(dt, delimiter="/"):
"""Unflatten dict. Does not support unflattening lists."""
dict_type = type(dt)
out = dict_type()
for key, val in dt.items():
path = key.split(delimiter)
item = out
for k in path[:-1]:
item = item.setdefault(k, dict_type())
item[path[-1]] = val
return out
class Trial:
"""A trial object holds the state for one model training run.
Trials are themselves managed by the TrialRunner class, which implements
the event loop for submitting trial runs to a Ray cluster.
Trials start in the PENDING state, and transition to RUNNING once started.
On error it transitions to ERROR, otherwise TERMINATED on success.
Attributes:
trainable_name (str): Name of the trainable object to be executed.
config (dict): Provided configuration dictionary with evaluated params.
trial_id (str): Unique identifier for the trial.
local_dir (str): Local_dir as passed to tune.run.
logdir (str): Directory where the trial logs are saved.
evaluated_params (dict): Evaluated parameters by search algorithm,
experiment_tag (str): Identifying trial name to show in the console.
resources (Resources): Amount of resources that this trial will use.
status (str): One of PENDING, RUNNING, PAUSED, TERMINATED, ERROR/
error_file (str): Path to the errors that this trial has raised.
"""
PENDING = "PENDING"
RUNNING = "RUNNING"
PAUSED = "PAUSED"
TERMINATED = "TERMINATED"
ERROR = "ERROR"
@classmethod
def generate_id(cls):
return str(uuid.uuid1().hex)[:8]
def update_last_result(self, result):
if self.experiment_tag:
result.update(experiment_tag=self.experiment_tag)
self.last_result = result
self.last_update_time = time.time()
for metric, value in flatten_dict(result).items():
if isinstance(value, Number):
if metric not in self.metric_analysis:
self.metric_analysis[metric] = {
"max": value,
"min": value,
"avg": value,
"last": value,
}
self.metric_n_steps[metric] = {}
for n in self.n_steps:
key = "last-{:d}-avg".format(n)
self.metric_analysis[metric][key] = value
# Store n as string for correct restore.
self.metric_n_steps[metric][str(n)] = deque([value], maxlen=n)
else:
step = result["training_iteration"] or 1
self.metric_analysis[metric]["max"] = max(value, self.metric_analysis[metric]["max"])
self.metric_analysis[metric]["min"] = min(value, self.metric_analysis[metric]["min"])
self.metric_analysis[metric]["avg"] = (
1 / step * (value + (step - 1) * self.metric_analysis[metric]["avg"])
)
self.metric_analysis[metric]["last"] = value
for n in self.n_steps:
key = "last-{:d}-avg".format(n)
self.metric_n_steps[metric][str(n)].append(value)
self.metric_analysis[metric][key] = sum(self.metric_n_steps[metric][str(n)]) / len(
self.metric_n_steps[metric][str(n)]
)
def set_status(self, status):
"""Sets the status of the trial."""
self.status = status
if status == Trial.RUNNING:
if self.start_time is None:
self.start_time = time.time()
def is_finished(self):
return self.status in [Trial.ERROR, Trial.TERMINATED]

171
flaml/tune/trial_runner.py
View File

@ -1,171 +0,0 @@
# !
# * Copyright (c) Microsoft Corporation. All rights reserved.
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
from typing import Optional
# try:
# from ray import __version__ as ray_version
# assert ray_version >= '1.0.0'
# from ray.tune.trial import Trial
# except (ImportError, AssertionError):
from .trial import Trial
import logging
logger = logging.getLogger(__name__)
class Nologger:
"""Logger without logging."""
def on_result(self, result):
pass
class SimpleTrial(Trial):
"""A simple trial class."""
def __init__(self, config, trial_id=None):
self.trial_id = Trial.generate_id() if trial_id is None else trial_id
self.config = config or {}
self.status = Trial.PENDING
self.start_time = None
self.last_result = None
self.last_update_time = -float("inf")
self.custom_trial_name = None
self.trainable_name = "trainable"
self.experiment_tag = "exp"
self.verbose = False
self.result_logger = Nologger()
self.metric_analysis = {}
self.n_steps = [5, 10]
self.metric_n_steps = {}
class BaseTrialRunner:
"""Implementation of a simple trial runner.
Note that the caller usually should not mutate trial state directly.
"""
def __init__(
self,
search_alg=None,
scheduler=None,
metric: Optional[str] = None,
mode: Optional[str] = "min",
):
self._search_alg = search_alg
self._scheduler_alg = scheduler
self._trials = []
self._metric = metric
self._mode = mode
def get_trials(self):
"""Returns the list of trials managed by this TrialRunner.
Note that the caller usually should not mutate trial state directly.
"""
return self._trials
def add_trial(self, trial):
"""Adds a new trial to this TrialRunner.
Trials may be added at any time.
Args:
trial (Trial): Trial to queue.
"""
self._trials.append(trial)
if self._scheduler_alg:
self._scheduler_alg.on_trial_add(self, trial)
def process_trial_result(self, trial, result):
trial.update_last_result(result)
if "time_total_s" not in result.keys():
result["time_total_s"] = trial.last_update_time - trial.start_time
self._search_alg.on_trial_result(trial.trial_id, result)
if self._scheduler_alg:
decision = self._scheduler_alg.on_trial_result(self, trial, result)
if decision == "STOP":
trial.set_status(Trial.TERMINATED)
elif decision == "PAUSE":
trial.set_status(Trial.PAUSED)
def stop_trial(self, trial):
"""Stops trial."""
if trial.status not in [Trial.ERROR, Trial.TERMINATED]:
if self._scheduler_alg:
self._scheduler_alg.on_trial_complete(self, trial.trial_id, trial.last_result)
self._search_alg.on_trial_complete(trial.trial_id, trial.last_result)
trial.set_status(Trial.TERMINATED)
elif self._scheduler_alg:
self._scheduler_alg.on_trial_remove(self, trial)
if trial.status == Trial.ERROR:
self._search_alg.on_trial_complete(trial.trial_id, trial.last_result, error=True)
class SequentialTrialRunner(BaseTrialRunner):
"""Implementation of the sequential trial runner."""
def step(self) -> Trial:
"""Runs one step of the trial event loop.
Callers should typically run this method repeatedly in a loop. They
may inspect or modify the runner's state in between calls to step().
Returns:
a trial to run.
"""
trial_id = Trial.generate_id()
config = self._search_alg.suggest(trial_id)
if config is not None:
trial = SimpleTrial(config, trial_id)
self.add_trial(trial)
trial.set_status(Trial.RUNNING)
else:
trial = None
self.running_trial = trial
return trial
def stop_trial(self, trial):
super().stop_trial(trial)
self.running_trial = None
class SparkTrialRunner(BaseTrialRunner):
"""Implementation of the spark trial runner."""
def __init__(
self,
search_alg=None,
scheduler=None,
metric: Optional[str] = None,
mode: Optional[str] = "min",
):
super().__init__(search_alg, scheduler, metric, mode)
self.running_trials = []
def step(self) -> Trial:
"""Runs one step of the trial event loop.
Callers should typically run this method repeatedly in a loop. They
may inspect or modify the runner's state in between calls to step().
Returns:
a trial to run.
"""
trial_id = Trial.generate_id()
config = self._search_alg.suggest(trial_id)
if config is not None:
trial = SimpleTrial(config, trial_id)
self.add_trial(trial)
trial.set_status(Trial.RUNNING)
self.running_trials.append(trial)
else:
trial = None
return trial
def stop_trial(self, trial):
super().stop_trial(trial)
self.running_trials.remove(trial)

926
flaml/tune/tune.py
View File

@ -1,926 +0,0 @@
# !
# * Copyright (c) FLAML authors. All rights reserved.
# * Licensed under the MIT License. See LICENSE file in the
# * project root for license information.
from typing import Optional, Union, List, Callable, Tuple, Dict
import numpy as np
import datetime
import time
import os
import sys
from collections import defaultdict
try:
from ray import __version__ as ray_version
assert ray_version >= "1.10.0"
from ray.tune.analysis import ExperimentAnalysis as EA
except (ImportError, AssertionError):
ray_available = False
from .analysis import ExperimentAnalysis as EA
else:
ray_available = True
from .trial import Trial
from .result import DEFAULT_METRIC
import logging
from flaml.tune.spark.utils import PySparkOvertimeMonitor, check_spark
logger = logging.getLogger(__name__)
logger.propagate = False
_use_ray = True
_runner = None
_verbose = 0
_running_trial = None
_training_iteration = 0
INCUMBENT_RESULT = "__incumbent_result__"
class ExperimentAnalysis(EA):
"""Class for storing the experiment results."""
def __init__(self, trials, metric, mode, lexico_objectives=None):
try:
super().__init__(self, None, trials, metric, mode)
self.lexico_objectives = lexico_objectives
except (TypeError, ValueError):
self.trials = trials
self.default_metric = metric or DEFAULT_METRIC
self.default_mode = mode
self.lexico_objectives = lexico_objectives
@property
def best_trial(self) -> Trial:
if self.lexico_objectives is None:
return super().best_trial
else:
return self.get_best_trial(self.default_metric, self.default_mode)
@property
def best_config(self) -> Dict:
if self.lexico_objectives is None:
return super().best_config
else:
return self.get_best_config(self.default_metric, self.default_mode)
def lexico_best(self, trials):
results = {index: trial.last_result for index, trial in enumerate(trials) if trial.last_result}
metrics = self.lexico_objectives["metrics"]
modes = self.lexico_objectives["modes"]
f_best = {}
keys = list(results.keys())
length = len(keys)
histories = defaultdict(list)
for time_index in range(length):
for objective, mode in zip(metrics, modes):
histories[objective].append(
results[keys[time_index]][objective] if mode == "min" else -results[keys[time_index]][objective]
)
obj_initial = self.lexico_objectives["metrics"][0]
feasible_index = np.array([*range(len(histories[obj_initial]))])
for k_metric, k_mode in zip(self.lexico_objectives["metrics"], self.lexico_objectives["modes"]):
k_values = np.array(histories[k_metric])
k_target = (
-self.lexico_objectives["targets"][k_metric]
if k_mode == "max"
else self.lexico_objectives["targets"][k_metric]
)
feasible_value = k_values.take(feasible_index)
f_best[k_metric] = np.min(feasible_value)
feasible_index_filter = np.where(
feasible_value
<= max(
f_best[k_metric] + self.lexico_objectives["tolerances"][k_metric]
if not isinstance(self.lexico_objectives["tolerances"][k_metric], str)
else f_best[k_metric]
* (1 + 0.01 * float(self.lexico_objectives["tolerances"][k_metric].replace("%", ""))),
k_target,
)
)[0]
feasible_index = feasible_index.take(feasible_index_filter)
best_trial = trials[feasible_index[-1]]
return best_trial
def get_best_trial(
self,
metric: Optional[str] = None,
mode: Optional[str] = None,
scope: str = "last",
filter_nan_and_inf: bool = True,
) -> Optional[Trial]:
if self.lexico_objectives is not None:
best_trial = self.lexico_best(self.trials)
else:
best_trial = super().get_best_trial(metric, mode, scope, filter_nan_and_inf)
return best_trial
@property
def best_result(self) -> Dict:
if self.lexico_best is None:
return super().best_result
else:
return self.best_trial.last_result
def report(_metric=None, **kwargs):
"""A function called by the HPO application to report final or intermediate
results.
Example:
```python
import time
from flaml import tune
def compute_with_config(config):
current_time = time.time()
metric2minimize = (round(config['x'])-95000)**2
time2eval = time.time() - current_time
tune.report(metric2minimize=metric2minimize, time2eval=time2eval)
analysis = tune.run(
compute_with_config,
config={
'x': tune.lograndint(lower=1, upper=1000000),
'y': tune.randint(lower=1, upper=1000000)
},
metric='metric2minimize', mode='min',
num_samples=1000000, time_budget_s=60, use_ray=False)
print(analysis.trials[-1].last_result)
```
Args:
_metric: Optional default anonymous metric for ``tune.report(value)``.
(For compatibility with ray.tune.report)
**kwargs: Any key value pair to be reported.
Raises:
StopIteration (when not using ray, i.e., _use_ray=False):
A StopIteration exception is raised if the trial has been signaled to stop.
SystemExit (when using ray):
A SystemExit exception is raised if the trial has been signaled to stop by ray.
"""
global _use_ray
global _verbose
global _running_trial
global _training_iteration
if _use_ray:
try:
from ray import tune
return tune.report(_metric, **kwargs)
except ImportError:
# calling tune.report() outside tune.run()
return
result = kwargs
if _metric:
result[DEFAULT_METRIC] = _metric
trial = getattr(_runner, "running_trial", None)
if not trial:
return None
if _running_trial == trial:
_training_iteration += 1
else:
_training_iteration = 0
_running_trial = trial
result["training_iteration"] = _training_iteration
result["config"] = trial.config
if INCUMBENT_RESULT in result["config"]:
del result["config"][INCUMBENT_RESULT]
for key, value in trial.config.items():
result["config/" + key] = value
_runner.process_trial_result(trial, result)
if _verbose > 2:
logger.info(f"result: {result}")
if trial.is_finished():
raise StopIteration
def run(
evaluation_function,
config: Optional[dict] = None,
low_cost_partial_config: Optional[dict] = None,
cat_hp_cost: Optional[dict] = None,
metric: Optional[str] = None,
mode: Optional[str] = None,
time_budget_s: Union[int, float] = None,
points_to_evaluate: Optional[List[dict]] = None,
evaluated_rewards: Optional[List] = None,
resource_attr: Optional[str] = None,
min_resource: Optional[float] = None,
max_resource: Optional[float] = None,
reduction_factor: Optional[float] = None,
scheduler=None,
search_alg=None,
verbose: Optional[int] = 2,
local_dir: Optional[str] = None,
num_samples: Optional[int] = 1,
resources_per_trial: Optional[dict] = None,
config_constraints: Optional[List[Tuple[Callable[[dict], float], str, float]]] = None,
metric_constraints: Optional[List[Tuple[str, str, float]]] = None,
max_failure: Optional[int] = 100,
use_ray: Optional[bool] = False,
use_spark: Optional[bool] = False,
use_incumbent_result_in_evaluation: Optional[bool] = None,
log_file_name: Optional[str] = None,
lexico_objectives: Optional[dict] = None,
force_cancel: Optional[bool] = False,
n_concurrent_trials: Optional[int] = 0,
**ray_args,
):
"""The function-based way of performing HPO.
Example:
```python
import time
from flaml import tune
def compute_with_config(config):
current_time = time.time()
metric2minimize = (round(config['x'])-95000)**2
time2eval = time.time() - current_time
tune.report(metric2minimize=metric2minimize, time2eval=time2eval)
# if the evaluation fails unexpectedly and the exception is caught,
# and it doesn't inform the goodness of the config,
# return {}
# if the failure indicates a config is bad,
# report a bad metric value like np.inf or -np.inf
# depending on metric mode being min or max
analysis = tune.run(
compute_with_config,
config={
'x': tune.lograndint(lower=1, upper=1000000),
'y': tune.randint(lower=1, upper=1000000)
},
metric='metric2minimize', mode='min',
num_samples=-1, time_budget_s=60, use_ray=False)
print(analysis.trials[-1].last_result)
```
Args:
evaluation_function: A user-defined evaluation function.
It takes a configuration as input, outputs a evaluation
result (can be a numerical value or a dictionary of string
and numerical value pairs) for the input configuration.
For machine learning tasks, it usually involves training and
scoring a machine learning model, e.g., through validation loss.
config: A dictionary to specify the search space.
low_cost_partial_config: A dictionary from a subset of
controlled dimensions to the initial low-cost values.
e.g., ```{'n_estimators': 4, 'max_leaves': 4}```
cat_hp_cost: A dictionary from a subset of categorical dimensions
to the relative cost of each choice.
e.g., ```{'tree_method': [1, 1, 2]}```
i.e., the relative cost of the
three choices of 'tree_method' is 1, 1 and 2 respectively
metric: A string of the metric name to optimize for.
mode: A string in ['min', 'max'] to specify the objective as
minimization or maximization.
time_budget_s: int or float | The time budget in seconds.
points_to_evaluate: A list of initial hyperparameter
configurations to run first.
evaluated_rewards (list): If you have previously evaluated the
parameters passed in as points_to_evaluate you can avoid
re-running those trials by passing in the reward attributes
as a list so the optimiser can be told the results without
needing to re-compute the trial. Must be the same or shorter length than
points_to_evaluate.
e.g.,
```python
points_to_evaluate = [
{"b": .99, "cost_related": {"a": 3}},
{"b": .99, "cost_related": {"a": 2}},
]
evaluated_rewards = [3.0]
```
means that you know the reward for the first config in
points_to_evaluate is 3.0 and want to inform run().
resource_attr: A string to specify the resource dimension used by
the scheduler via "scheduler".
min_resource: A float of the minimal resource to use for the resource_attr.
max_resource: A float of the maximal resource to use for the resource_attr.
reduction_factor: A float of the reduction factor used for incremental
pruning.
scheduler: A scheduler for executing the experiment. Can be None, 'flaml',
'asha' (or 'async_hyperband', 'asynchyperband') or a custom instance of the TrialScheduler class. Default is None:
in this case when resource_attr is provided, the 'flaml' scheduler will be
used, otherwise no scheduler will be used. When set 'flaml', an
authentic scheduler implemented in FLAML will be used. It does not
require users to report intermediate results in evaluation_function.
Find more details about this scheduler in this paper
https://arxiv.org/pdf/1911.04706.pdf).
When set 'asha', the input for arguments "resource_attr",
"min_resource", "max_resource" and "reduction_factor" will be passed
to ASHA's "time_attr", "max_t", "grace_period" and "reduction_factor"
respectively. You can also provide a self-defined scheduler instance
of the TrialScheduler class. When 'asha' or self-defined scheduler is
used, you usually need to report intermediate results in the evaluation
function via 'tune.report()'.
If you would like to do some cleanup opearation when the trial is stopped
by the scheduler, you can catch the `StopIteration` (when not using ray)
or `SystemExit` (when using ray) exception explicitly,
as shown in the following example.
Please find more examples using different types of schedulers
and how to set up the corresponding evaluation functions in
test/tune/test_scheduler.py, and test/tune/example_scheduler.py.
```python
def easy_objective(config):
width, height = config["width"], config["height"]
for step in range(config["steps"]):
intermediate_score = evaluation_fn(step, width, height)
try:
tune.report(iterations=step, mean_loss=intermediate_score)
except (StopIteration, SystemExit):
# do cleanup operation here
return
```
search_alg: An instance/string of the search algorithm
to be used. The same instance can be used for iterative tuning.
e.g.,
```python
from flaml import BlendSearch
algo = BlendSearch(metric='val_loss', mode='min',
space=search_space,
low_cost_partial_config=low_cost_partial_config)
for i in range(10):
analysis = tune.run(compute_with_config,
search_alg=algo, use_ray=False)
print(analysis.trials[-1].last_result)
```
verbose: 0, 1, 2, or 3. If ray or spark backend is used, their verbosity will be
affected by this argument. 0 = silent, 1 = only status updates,
2 = status and brief trial results, 3 = status and detailed trial results.
Defaults to 2.
local_dir: A string of the local dir to save ray logs if ray backend is
used; or a local dir to save the tuning log.
num_samples: An integer of the number of configs to try. Defaults to 1.
resources_per_trial: A dictionary of the hardware resources to allocate
per trial, e.g., `{'cpu': 1}`. It is only valid when using ray backend
(by setting 'use_ray = True'). It shall be used when you need to do
[parallel tuning](/docs/Use-Cases/Tune-User-Defined-Function#parallel-tuning).
config_constraints: A list of config constraints to be satisfied.
e.g., ```config_constraints = [(mem_size, '<=', 1024**3)]```
mem_size is a function which produces a float number for the bytes
needed for a config.
It is used to skip configs which do not fit in memory.
metric_constraints: A list of metric constraints to be satisfied.
e.g., `['precision', '>=', 0.9]`. The sign can be ">=" or "<=".
max_failure: int | the maximal consecutive number of failures to sample
a trial before the tuning is terminated.
use_ray: A boolean of whether to use ray as the backend.
use_spark: A boolean of whether to use spark as the backend.
log_file_name: A string of the log file name. Default to None.
When set to None:
if local_dir is not given, no log file is created;
if local_dir is given, the log file name will be autogenerated under local_dir.
Only valid when verbose > 0 or use_ray is True.
lexico_objectives: dict, default=None | It specifics information needed to perform multi-objective
optimization with lexicographic preferences. When lexico_objectives is not None, the arguments metric,
mode, will be invalid, and flaml's tune uses CFO
as the `search_alg`, which makes the input (if provided) `search_alg' invalid.
This dictionary shall contain the following fields of key-value pairs:
- "metrics": a list of optimization objectives with the orders reflecting the priorities/preferences of the
objectives.
- "modes" (optional): a list of optimization modes (each mode either "min" or "max") corresponding to the
objectives in the metric list. If not provided, we use "min" as the default mode for all the objectives.
- "targets" (optional): a dictionary to specify the optimization targets on the objectives. The keys are the
metric names (provided in "metric"), and the values are the numerical target values.
- "tolerances" (optional): a dictionary to specify the optimality tolerances on objectives. The keys are the metric names (provided in "metrics"), and the values are the absolute/percentage tolerance in the form of numeric/string.
E.g.,
```python
lexico_objectives = {
"metrics": ["error_rate", "pred_time"],
"modes": ["min", "min"],
"tolerances": {"error_rate": 0.01, "pred_time": 0.0},
"targets": {"error_rate": 0.0},
}
```
We also support percentage tolerance.
E.g.,
```python
lexico_objectives = {
"metrics": ["error_rate", "pred_time"],
"modes": ["min", "min"],
"tolerances": {"error_rate": "5%", "pred_time": "0%"},
"targets": {"error_rate": 0.0},
}
```
force_cancel: boolean, default=False | Whether to forcely cancel the PySpark job if overtime.
n_concurrent_trials: int, default=0 | The number of concurrent trials when perform hyperparameter
tuning with Spark. Only valid when use_spark=True and spark is required:
`pip install flaml[spark]`. Please check
[here](https://spark.apache.org/docs/latest/api/python/getting_started/install.html)
for more details about installing Spark. When tune.run() is called from AutoML, it will be
overwritten by the value of `n_concurrent_trials` in AutoML. When <= 0, the concurrent trials
will be set to the number of executors.
**ray_args: keyword arguments to pass to ray.tune.run().
Only valid when use_ray=True.
"""
global _use_ray
global _verbose
global _running_trial
global _training_iteration
old_use_ray = _use_ray
old_verbose = _verbose
old_running_trial = _running_trial
old_training_iteration = _training_iteration
if log_file_name:
dir_name = os.path.dirname(log_file_name)
if dir_name:
os.makedirs(dir_name, exist_ok=True)
elif local_dir and verbose > 0:
os.makedirs(local_dir, exist_ok=True)
log_file_name = os.path.join(local_dir, "tune_" + str(datetime.datetime.now()).replace(":", "-") + ".log")
if use_ray and use_spark:
raise ValueError("use_ray and use_spark cannot be both True.")
if not use_ray:
_use_ray = False
_verbose = verbose
old_handlers = logger.handlers
old_level = logger.getEffectiveLevel()
logger.handlers = []
global _runner
old_runner = _runner
assert not ray_args, "ray_args is only valid when use_ray=True"
if (
old_handlers
and isinstance(old_handlers[0], logging.StreamHandler)
and not isinstance(old_handlers[0], logging.FileHandler)
):
# Add the console handler.
logger.addHandler(old_handlers[0])
if verbose > 0:
if log_file_name:
logger.addHandler(logging.FileHandler(log_file_name))
elif not logger.hasHandlers():
# Add the console handler.
_ch = logging.StreamHandler(stream=sys.stdout)
logger_formatter = logging.Formatter(
"[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s",
"%m-%d %H:%M:%S",
)
_ch.setFormatter(logger_formatter)
logger.addHandler(_ch)
if verbose <= 2:
logger.setLevel(logging.INFO)
else:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.CRITICAL)
from .searcher.blendsearch import BlendSearch, CFO, RandomSearch
if lexico_objectives is not None:
if "modes" not in lexico_objectives.keys():
lexico_objectives["modes"] = ["min"] * len(lexico_objectives["metrics"])
for t_metric, t_mode in zip(lexico_objectives["metrics"], lexico_objectives["modes"]):
if t_metric not in lexico_objectives["tolerances"].keys():
lexico_objectives["tolerances"][t_metric] = 0
if t_metric not in lexico_objectives["targets"].keys():
lexico_objectives["targets"][t_metric] = -float("inf") if t_mode == "min" else float("inf")
if search_alg is None or isinstance(search_alg, str):
if isinstance(search_alg, str):
assert search_alg in [
"BlendSearch",
"CFO",
"CFOCat",
"RandomSearch",
], f"search_alg={search_alg} is not recognized. 'BlendSearch', 'CFO', 'CFOcat' and 'RandomSearch' are supported."
flaml_scheduler_resource_attr = (
flaml_scheduler_min_resource
) = flaml_scheduler_max_resource = flaml_scheduler_reduction_factor = None
if scheduler in (None, "flaml"):
# when scheduler is set 'flaml' or None, we will use a scheduler that is
# authentic to the search algorithms in flaml. After setting up
# the search algorithm accordingly, we need to set scheduler to
# None in case it is later used in the trial runner.
flaml_scheduler_resource_attr = resource_attr
flaml_scheduler_min_resource = min_resource
flaml_scheduler_max_resource = max_resource
flaml_scheduler_reduction_factor = reduction_factor
scheduler = None
if lexico_objectives:
# TODO: Modify after supporting BlendSearch in lexicographic optimization
SearchAlgorithm = CFO
logger.info(
f"Using search algorithm {SearchAlgorithm.__name__} for lexicographic optimization. Note that when providing other search algorithms, we use CFO instead temporarily."
)
metric = lexico_objectives["metrics"][0] or DEFAULT_METRIC
else:
if not search_alg or search_alg == "BlendSearch":
try:
import optuna as _
SearchAlgorithm = BlendSearch
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
except ImportError:
if search_alg == "BlendSearch":
raise ValueError("To use BlendSearch, run: pip install flaml[blendsearch]")
else:
SearchAlgorithm = CFO
logger.warning("Using CFO for search. To use BlendSearch, run: pip install flaml[blendsearch]")
else:
SearchAlgorithm = locals()[search_alg]
logger.info("Using search algorithm {}.".format(SearchAlgorithm.__name__))
metric = metric or DEFAULT_METRIC
search_alg = SearchAlgorithm(
metric=metric,
mode=mode,
space=config,
points_to_evaluate=points_to_evaluate,
evaluated_rewards=evaluated_rewards,
low_cost_partial_config=low_cost_partial_config,
cat_hp_cost=cat_hp_cost,
time_budget_s=time_budget_s,
num_samples=num_samples,
resource_attr=flaml_scheduler_resource_attr,
min_resource=flaml_scheduler_min_resource,
max_resource=flaml_scheduler_max_resource,
reduction_factor=flaml_scheduler_reduction_factor,
config_constraints=config_constraints,
metric_constraints=metric_constraints,
use_incumbent_result_in_evaluation=use_incumbent_result_in_evaluation,
lexico_objectives=lexico_objectives,
)
else:
if metric is None or mode is None:
if lexico_objectives:
metric = lexico_objectives["metrics"][0] or metric or search_alg.metric or DEFAULT_METRIC
mode = lexico_objectives["modes"][0] or mode or search_alg.mode
else:
metric = metric or search_alg.metric or DEFAULT_METRIC
mode = mode or search_alg.mode
if ray_available and use_ray:
if ray_version.startswith("1."):
from ray.tune.suggest import ConcurrencyLimiter
else:
from ray.tune.search import ConcurrencyLimiter
else:
from flaml.tune.searcher.suggestion import ConcurrencyLimiter
if (
search_alg.__class__.__name__
in [
"BlendSearch",
"CFO",
"CFOCat",
]
and use_incumbent_result_in_evaluation is not None
):
search_alg.use_incumbent_result_in_evaluation = use_incumbent_result_in_evaluation
searcher = search_alg.searcher if isinstance(search_alg, ConcurrencyLimiter) else search_alg
if lexico_objectives:
# TODO: Modify after supporting BlendSearch in lexicographic optimization
assert search_alg.__class__.__name__ in [
"CFO",
], "If lexico_objectives is not None, the search_alg must be CFO for now."
search_alg.lexico_objective = lexico_objectives
if isinstance(searcher, BlendSearch):
setting = {}
if time_budget_s:
setting["time_budget_s"] = time_budget_s
if num_samples > 0:
setting["num_samples"] = num_samples
searcher.set_search_properties(metric, mode, config, **setting)
else:
searcher.set_search_properties(metric, mode, config)
if scheduler in ("asha", "asynchyperband", "async_hyperband"):
params = {}
# scheduler resource_dimension=resource_attr
if resource_attr:
params["time_attr"] = resource_attr
if max_resource:
params["max_t"] = max_resource
if min_resource:
params["grace_period"] = min_resource
if reduction_factor:
params["reduction_factor"] = reduction_factor
if ray_available:
from ray.tune.schedulers import ASHAScheduler
scheduler = ASHAScheduler(**params)
if use_ray:
try:
from ray import tune
except ImportError:
raise ImportError("Failed to import ray tune. " "Please install ray[tune] or set use_ray=False")
_use_ray = True
try:
analysis = tune.run(
evaluation_function,
metric=metric,
mode=mode,
search_alg=search_alg,
scheduler=scheduler,
time_budget_s=time_budget_s,
verbose=verbose,
local_dir=local_dir,
num_samples=num_samples,
resources_per_trial=resources_per_trial,
**ray_args,
)
if log_file_name:
with open(log_file_name, "w") as f:
for trial in analysis.trials:
f.write(f"result: {trial.last_result}\n")
return analysis
finally:
_use_ray = old_use_ray
_verbose = old_verbose
_running_trial = old_running_trial
_training_iteration = old_training_iteration
if use_spark:
# parallel run with spark
spark_available, spark_error_msg = check_spark()
if not spark_available:
raise spark_error_msg
try:
from pyspark.sql import SparkSession
from joblib import Parallel, delayed, parallel_backend
from joblibspark import register_spark
except ImportError as e:
raise ImportError(f"{e}. Try pip install flaml[spark] or set use_spark=False.")
from flaml.tune.searcher.suggestion import ConcurrencyLimiter
from .trial_runner import SparkTrialRunner
register_spark()
spark = SparkSession.builder.getOrCreate()
sc = spark._jsc.sc()
num_executors = len([executor.host() for executor in sc.statusTracker().getExecutorInfos()]) - 1
"""
By default, the number of executors is the number of VMs in the cluster. And we can
launch one trial per executor. However, sometimes we can launch more trials than
the number of executors (e.g., local mode). In this case, we can set the environment
variable `FLAML_MAX_CONCURRENT` to override the detected `num_executors`.
`max_concurrent` is the maximum number of concurrent trials defined by `search_alg`,
`FLAML_MAX_CONCURRENT` will also be used to override `max_concurrent` if `search_alg`
is not an instance of `ConcurrencyLimiter`.
The final number of concurrent trials is the minimum of `max_concurrent` and
`num_executors` if `n_concurrent_trials<=0` (default, automl cases), otherwise the
minimum of `max_concurrent` and `n_concurrent_trials` (tuning cases).
"""
time_start = time.time()
try:
FLAML_MAX_CONCURRENT = int(os.getenv("FLAML_MAX_CONCURRENT", 0))
except ValueError:
FLAML_MAX_CONCURRENT = 0
num_executors = max(num_executors, FLAML_MAX_CONCURRENT, 1)
max_spark_parallelism = max(spark.sparkContext.defaultParallelism, FLAML_MAX_CONCURRENT)
if scheduler:
scheduler.set_search_properties(metric=metric, mode=mode)
if isinstance(search_alg, ConcurrencyLimiter):
max_concurrent = max(1, search_alg.max_concurrent)
else:
max_concurrent = max(1, max_spark_parallelism)
n_concurrent_trials = min(
n_concurrent_trials if n_concurrent_trials > 0 else num_executors,
max_concurrent,
)
with parallel_backend("spark"):
with Parallel(n_jobs=n_concurrent_trials, verbose=max(0, (verbose - 1) * 50)) as parallel:
try:
_runner = SparkTrialRunner(
search_alg=search_alg,
scheduler=scheduler,
metric=metric,
mode=mode,
)
num_trials = 0
if time_budget_s is None:
time_budget_s = np.inf
num_failures = 0
upperbound_num_failures = (len(evaluated_rewards) if evaluated_rewards else 0) + max_failure
while (
time.time() - time_start < time_budget_s
and (num_samples < 0 or num_trials < num_samples)
and num_failures < upperbound_num_failures
):
while len(_runner.running_trials) < n_concurrent_trials:
# suggest trials for spark
trial_next = _runner.step()
if trial_next:
num_trials += 1
else:
num_failures += 1 # break with upperbound_num_failures consecutive failures
logger.debug(f"consecutive failures is {num_failures}")
if num_failures >= upperbound_num_failures:
break
trials_to_run = _runner.running_trials
if not trials_to_run:
logger.warning(f"fail to sample a trial for {max_failure} times in a row, stopping.")
break
logger.info(
f"Number of trials: {num_trials}/{num_samples}, {len(_runner.running_trials)} RUNNING,"
f" {len(_runner._trials) - len(_runner.running_trials)} TERMINATED"
)
logger.debug(
f"Configs of Trials to run: {[trial_to_run.config for trial_to_run in trials_to_run]}"
)
results = None
with PySparkOvertimeMonitor(time_start, time_budget_s, force_cancel, parallel=parallel):
results = parallel(
delayed(evaluation_function)(trial_to_run.config) for trial_to_run in trials_to_run
)
# results = [evaluation_function(trial_to_run.config) for trial_to_run in trials_to_run]
while results:
result = results.pop(0)
trial_to_run = trials_to_run[0]
_runner.running_trial = trial_to_run
if result is not None:
if isinstance(result, dict):
if result:
logger.info(f"Brief result: {result}")
report(**result)
else:
# When the result returned is an empty dict, set the trial status to error
trial_to_run.set_status(Trial.ERROR)
else:
logger.info("Brief result: {}".format({metric: result}))
report(_metric=result)
_runner.stop_trial(trial_to_run)
num_failures = 0
analysis = ExperimentAnalysis(
_runner.get_trials(),
metric=metric,
mode=mode,
lexico_objectives=lexico_objectives,
)
return analysis
finally:
# recover the global variables in case of nested run
_use_ray = old_use_ray
_verbose = old_verbose
_running_trial = old_running_trial
_training_iteration = old_training_iteration
if not use_ray:
_runner = old_runner
logger.handlers = old_handlers
logger.setLevel(old_level)
# simple sequential run without using tune.run() from ray
time_start = time.time()
_use_ray = False
if scheduler:
scheduler.set_search_properties(metric=metric, mode=mode)
from .trial_runner import SequentialTrialRunner
try:
_runner = SequentialTrialRunner(
search_alg=search_alg,
scheduler=scheduler,
metric=metric,
mode=mode,
)
num_trials = 0
if time_budget_s is None:
time_budget_s = np.inf
num_failures = 0
upperbound_num_failures = (len(evaluated_rewards) if evaluated_rewards else 0) + max_failure
while (
time.time() - time_start < time_budget_s
and (num_samples < 0 or num_trials < num_samples)
and num_failures < upperbound_num_failures
):
trial_to_run = _runner.step()
if trial_to_run:
num_trials += 1
if verbose:
logger.info(f"trial {num_trials} config: {trial_to_run.config}")
result = None
with PySparkOvertimeMonitor(time_start, time_budget_s, force_cancel):
result = evaluation_function(trial_to_run.config)
if result is not None:
if isinstance(result, dict):
if result:
report(**result)
else:
# When the result returned is an empty dict, set the trial status to error
trial_to_run.set_status(Trial.ERROR)
else:
report(_metric=result)
_runner.stop_trial(trial_to_run)
num_failures = 0
if trial_to_run.last_result is None:
# application stops tuning by returning None
# TODO document this feature when it is finalized
break
else:
# break with upperbound_num_failures consecutive failures
num_failures += 1
if num_failures == upperbound_num_failures:
logger.warning(f"fail to sample a trial for {max_failure} times in a row, stopping.")
analysis = ExperimentAnalysis(
_runner.get_trials(),
metric=metric,
mode=mode,
lexico_objectives=lexico_objectives,
)
return analysis
finally:
# recover the global variables in case of nested run
_use_ray = old_use_ray
_verbose = old_verbose
_running_trial = old_running_trial
_training_iteration = old_training_iteration
if not use_ray:
_runner = old_runner
logger.handlers = old_handlers
logger.setLevel(old_level)
class Tuner:
"""Tuner is the class-based way of launching hyperparameter tuning jobs compatible with Ray Tune 2.
Args:
trainable: A user-defined evaluation function.
It takes a configuration as input, outputs a evaluation
result (can be a numerical value or a dictionary of string
and numerical value pairs) for the input configuration.
For machine learning tasks, it usually involves training and
scoring a machine learning model, e.g., through validation loss.
param_space: Search space of the tuning job.
One thing to note is that both preprocessor and dataset can be tuned here.
tune_config: Tuning algorithm specific configs.
Refer to ray.tune.tune_config.TuneConfig for more info.
run_config: Runtime configuration that is specific to individual trials.
If passed, this will overwrite the run config passed to the Trainer,
if applicable. Refer to ray.air.config.RunConfig for more info.
Usage pattern:
.. code-block:: python
from sklearn.datasets import load_breast_cancer
from ray import tune
from ray.data import from_pandas
from ray.air.config import RunConfig, ScalingConfig
from ray.train.xgboost import XGBoostTrainer
from ray.tune.tuner import Tuner
def get_dataset():
data_raw = load_breast_cancer(as_frame=True)
dataset_df = data_raw["data"]
dataset_df["target"] = data_raw["target"]
dataset = from_pandas(dataset_df)
return dataset
trainer = XGBoostTrainer(
label_column="target",
params={},
datasets={"train": get_dataset()},
)
param_space = {
"scaling_config": ScalingConfig(
num_workers=tune.grid_search([2, 4]),
resources_per_worker={
"CPU": tune.grid_search([1, 2]),
},
),
# You can even grid search various datasets in Tune.
# "datasets": {
# "train": tune.grid_search(
# [ds1, ds2]
# ),
# },
"params": {
"objective": "binary:logistic",
"tree_method": "approx",
"eval_metric": ["logloss", "error"],
"eta": tune.loguniform(1e-4, 1e-1),
"subsample": tune.uniform(0.5, 1.0),
"max_depth": tune.randint(1, 9),
},
}
tuner = Tuner(trainable=trainer, param_space=param_space,
run_config=RunConfig(name="my_tune_run"))
analysis = tuner.fit()
To retry a failed tune run, you can then do
.. code-block:: python
tuner = Tuner.restore(experiment_checkpoint_dir)
tuner.fit()
``experiment_checkpoint_dir`` can be easily located near the end of the
console output of your first failed run.
"""

27
flaml/tune/utils.py
View File

@ -1,27 +0,0 @@
from typing import Sequence
try:
from ray import __version__ as ray_version
assert ray_version >= "1.10.0"
if ray_version.startswith("1."):
from ray.tune import sample
else:
from ray.tune.search import sample
except (ImportError, AssertionError):
from . import sample
def choice(categories: Sequence, order=None):
"""Sample a categorical value.
Sampling from ``tune.choice([1, 2])`` is equivalent to sampling from
``np.random.choice([1, 2])``
Args:
categories (Sequence): Sequence of categories to sample from.
order (bool): Whether the categories have an order. If None, will be decided autoamtically:
Numerical categories have an order, while string categories do not.
"""
domain = sample.Categorical(categories).uniform()
domain.ordered = order if order is not None else all(isinstance(x, (int, float)) for x in categories)
return domain

1
flaml/version.py
View File

@ -1 +0,0 @@
__version__ = "2.1.0"

3
setup.py
View File

@ -17,6 +17,7 @@ install_requires = [
"openai",
"diskcache",
"termcolor",
"flaml",
]
@ -44,11 +45,11 @@ setuptools.setup(
"nbconvert",
"nbformat",
"ipykernel",
"packaging",
"pydantic==1.10.9",
"sympy",
"wolframalpha",
],
"blendsearch": ["flaml[blendsearch]"],
"mathchat": ["sympy", "pydantic==1.10.9", "wolframalpha"],
"retrievechat": [
"chromadb",

11
website/docs/Installation.md
View File

@ -5,10 +5,17 @@
AutoGen requires **Python version >= 3.8**. It can be installed from pip:
```bash
pip install "pyautogen"
pip install pyautogen
```
<!--
or conda:
```
conda install "flaml[autogen]" -c conda-forge
conda install pyautogen -c conda-forge
``` -->
### Optional Dependencies
* blendsearch
```bash
pip install "pyautogen[blendsearch]"
```