diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 180d8ecb..5a3b6e7f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,10 +18,11 @@ repos: entry: poetry run black language: system types: [python] + exclude: ^blackboxopt_client/examples/ - id: black-examples name: black-examples - entry: poetry run black --line-length=80 - files: ^blackboxopt/examples/ + entry: poetry run black --extend-exclude ^$ --line-length=80 + files: ^blackboxopt_client/examples/ language: system types: [python] - id: mypy diff --git a/blackboxopt/__init__.py b/blackboxopt/__init__.py index 8d048966..29b70ddc 100644 --- a/blackboxopt/__init__.py +++ b/blackboxopt/__init__.py @@ -2,7 +2,7 @@ from parameterspace import ParameterSpace -from . import io +from . import io, utils from .base import ( ConstraintsError, ContextError, @@ -14,4 +14,5 @@ OptimizerNotReady, ) from .evaluation import Evaluation, EvaluationSpecification -from .utils import sort_evaluations +from .logger import logger +from .utils import init_logger, sort_evaluations diff --git a/blackboxopt/examples/__init__.py b/blackboxopt/examples/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/blackboxopt/examples/multi_objective_multi_param.py b/blackboxopt/examples/multi_objective_multi_param.py new file mode 100644 index 00000000..6b2d2594 --- /dev/null +++ b/blackboxopt/examples/multi_objective_multi_param.py @@ -0,0 +1,90 @@ +# Copyright (c) 2020 - for information on the respective copyright owner +# see the NOTICE file and/or the repository https://github.com/boschresearch/blackboxopt +# +# SPDX-License-Identifier: Apache-2.0 + +import logging +import time + +import numpy as np +import parameterspace as ps +from sklearn.datasets import load_diabetes +from sklearn.ensemble import RandomForestRegressor +from sklearn.metrics import r2_score +from sklearn.model_selection import train_test_split + +import blackboxopt as bbo +from blackboxopt.optimization_loops.sequential import run_optimization_loop +from blackboxopt.optimizers.space_filling import SpaceFilling + +# Load a sklearn sample dataset +X_TRAIN, X_VALIDATE, Y_TRAIN, Y_VALIDATE = train_test_split( + *load_diabetes(return_X_y=True) +) + +# Set up search space with multiple ML model hyperparameters of different types +SPACE = ps.ParameterSpace() +SPACE.add(ps.IntegerParameter("n_estimators", bounds=(256, 2048), transformation="log")) +SPACE.add(ps.IntegerParameter("min_samples_leaf", bounds=(1, 32), transformation="log")) +SPACE.add(ps.ContinuousParameter("max_samples", bounds=(0.1, 1))) +SPACE.add(ps.ContinuousParameter("max_features", bounds=(0.1, 1))) +SPACE.add(ps.IntegerParameter("max_depth", bounds=(1, 128))) +SPACE.add(ps.CategoricalParameter("criterion", values=("squared_error", "poisson"))) + + +def evaluation_function( + eval_spec: bbo.EvaluationSpecification, +) -> bbo.Evaluation: + """Train and evaluate a random forest with given parameter configuration.""" + regr = RandomForestRegressor( + n_estimators=eval_spec.configuration["n_estimators"], + max_samples=eval_spec.configuration["max_samples"], + max_features=eval_spec.configuration["max_features"], + max_depth=eval_spec.configuration["max_depth"], + min_samples_leaf=eval_spec.configuration["min_samples_leaf"], + criterion=eval_spec.configuration["criterion"], + ) + + start = time.time() + regr.fit(X_TRAIN, Y_TRAIN) + fit_duration = time.time() - start + + y_pred = regr.predict(X_VALIDATE) + objectives = { + "R²": r2_score(Y_VALIDATE, y_pred), + "Fit Duration": fit_duration, + "Max Error": np.abs(Y_VALIDATE - y_pred).max(), + } + evaluation = eval_spec.create_evaluation(objectives=objectives) + return evaluation + + +def main(): + logger = bbo.init_logger(logging.INFO) + + # Create an optimization run based on a parameterspace and optimizer choice + optimizer = SpaceFilling( + search_space=SPACE, + objectives=[ + bbo.Objective("R²", greater_is_better=True), + bbo.Objective("Max Error", greater_is_better=False), + bbo.Objective("Fit Duration", greater_is_better=False), + ], + ) + + # Fetch new configurations to evaluate until the optimization is done or + # a given timeout is reached + evaluations = run_optimization_loop( + optimizer=optimizer, + evaluation_function=evaluation_function, + timeout_s=60.0, + ) + + logger.info(f"Evaluated {len(evaluations)} specifications") + + pareto_front = bbo.utils.filter_pareto_efficient(evaluations, optimizer.objectives) + logger.info(f"{len(pareto_front)} evaluation(s) are pareto efficient") + + +if __name__ == "__main__": + main() diff --git a/blackboxopt/logger.py b/blackboxopt/logger.py new file mode 100644 index 00000000..c7e8bf75 --- /dev/null +++ b/blackboxopt/logger.py @@ -0,0 +1,5 @@ +import logging +import os + +# Default logger used throughout the package +logger = logging.getLogger(os.environ.get("BBO_LOGGER_NAME", "blackboxopt")) diff --git a/blackboxopt/optimization_loops/sequential.py b/blackboxopt/optimization_loops/sequential.py index 06c1bb8d..2e2fd1d5 100644 --- a/blackboxopt/optimization_loops/sequential.py +++ b/blackboxopt/optimization_loops/sequential.py @@ -14,6 +14,7 @@ OptimizationComplete, OptimizerNotReady, ) +from blackboxopt import logger as default_logger from blackboxopt.base import MultiObjectiveOptimizer, SingleObjectiveOptimizer from blackboxopt.optimization_loops.utils import ( evaluation_function_wrapper, @@ -55,12 +56,13 @@ def run_optimization_loop( optimization loop. post_evaluation_callback: Reference to a callable that is invoked after each evaluation and takes a `blackboxopt.Evaluation` as its argument. - logger: The logger to use for logging progress. + logger: The logger to use for logging progress. Default: `blackboxopt.logger` Returns: - List of evluation specification and result for all evaluations. + List of evaluation specification and result for all evaluations. """ - logger = logging.getLogger("blackboxopt") if logger is None else logger + if logger is None: + logger = default_logger objectives = ( optimizer.objectives diff --git a/blackboxopt/utils.py b/blackboxopt/utils.py index 73db8e63..81cf5ccd 100644 --- a/blackboxopt/utils.py +++ b/blackboxopt/utils.py @@ -4,7 +4,10 @@ # SPDX-License-Identifier: Apache-2.0 import hashlib +import logging +import os import pickle +import sys from itertools import compress from typing import Dict, Iterable, List, Optional, Sequence @@ -12,6 +15,7 @@ from blackboxopt.base import Objective from blackboxopt.evaluation import Evaluation +from blackboxopt.logger import logger def get_loss_vector( @@ -99,3 +103,31 @@ def sort_evaluations(evaluations: Iterable[Evaluation]) -> Iterable[Evaluation]: ) ).hexdigest(), ) + + +def init_logger(level: int = None) -> logging.Logger: # pragma: no cover + """Initialize the default `blackboxopt.logger` as a nicely formatted stdout logger. + + Should no log level be given, the environment variable `BBO_LOG_LEVEL` is used + and if that is not present, the default is `logging.DEBUG`. + + Args: + level: the log level to set + + Returns: + The logger instance (equivalent to `blackboxopt.logger`) + """ + if level is None: + level_name = os.environ.get("BBO_LOG_LEVEL", "DEBUG") + level = getattr(logging, level_name) + + logger.setLevel(level) + handler = logging.StreamHandler(sys.stdout) + handler.setLevel(level) + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + ) + handler.setFormatter(formatter) + logger.addHandler(handler) + + return logger diff --git a/docs/examples/multi-objective-multi-param.md b/docs/examples/multi-objective-multi-param.md new file mode 100644 index 00000000..652fb32a --- /dev/null +++ b/docs/examples/multi-objective-multi-param.md @@ -0,0 +1,11 @@ +# Mixed Space & Multi Objective + +Aside from continuous parameters, different types are supported by `parameterspace`. +In the following example we use continuous, integer and categorical parameters. +Also, we are having a glance at optimizing multiple objectives at the same time. + +```python +--8<-- +blackboxopt/examples/multi_objective_multi_param.py +--8<-- +``` diff --git a/docs/examples/overview.md b/docs/examples/overview.md index 3036810c..15992a3e 100644 --- a/docs/examples/overview.md +++ b/docs/examples/overview.md @@ -9,3 +9,4 @@ pip install blackboxopt[examples] ## List of available examples - [Dask Distributed](dask-distributed.md) +- [Multi Objective Optimization](multi-objective-multi-param.md) diff --git a/mkdocs.yml b/mkdocs.yml index 62698bbc..3b433a65 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -30,6 +30,7 @@ nav: - Examples: - Overview: examples/overview.md - examples/dask-distributed.md + - examples/multi-objective-multi-param.md - ... plugins: diff --git a/poetry.lock b/poetry.lock index 5a255b42..17dd4332 100644 --- a/poetry.lock +++ b/poetry.lock @@ -13,13 +13,13 @@ files = [ [[package]] name = "alive-progress" -version = "3.1.2" +version = "3.1.4" description = "A new kind of Progress Bar, with real-time throughput, ETA, and very cool animations!" optional = true python-versions = ">=3.7, <4" files = [ - {file = "alive-progress-3.1.2.tar.gz", hash = "sha256:b22ba960151f582cdd6d489c56462d2f9adca821608075d0d8d2cd15d1b6845b"}, - {file = "alive_progress-3.1.2-py3-none-any.whl", hash = "sha256:d2b89b60fee2f112668117a9f361ceea44a685a37cafd009f396b87b9816efa3"}, + {file = "alive-progress-3.1.4.tar.gz", hash = "sha256:74a95d8d0d42bc99d3a3725dbd06ebb852245f1b64e301a7c375b92b22663f7b"}, + {file = "alive_progress-3.1.4-py3-none-any.whl", hash = "sha256:c80ad87ce9c1054b01135a87fae69ecebbfc2107497ae87cbe6aec7e534903db"}, ] [package.dependencies] @@ -362,62 +362,71 @@ test-no-images = ["pytest"] [[package]] name = "coverage" -version = "7.2.6" +version = "7.2.7" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.7" files = [ - {file = "coverage-7.2.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:496b86f1fc9c81a1cd53d8842ef712e950a4611bba0c42d33366a7b91ba969ec"}, - {file = "coverage-7.2.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:fbe6e8c0a9a7193ba10ee52977d4d5e7652957c1f56ccefed0701db8801a2a3b"}, - {file = "coverage-7.2.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d06b721c2550c01a60e5d3093f417168658fb454e5dfd9a23570e9bffe39a1"}, - {file = "coverage-7.2.6-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:77a04b84d01f0e12c66f16e69e92616442dc675bbe51b90bfb074b1e5d1c7fbd"}, - {file = "coverage-7.2.6-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:35db06450272473eab4449e9c2ad9bc6a0a68dab8e81a0eae6b50d9c2838767e"}, - {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:6727a0d929ff0028b1ed8b3e7f8701670b1d7032f219110b55476bb60c390bfb"}, - {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:aac1d5fdc5378f6bac2c0c7ebe7635a6809f5b4376f6cf5d43243c1917a67087"}, - {file = "coverage-7.2.6-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1c9e4a5eb1bbc3675ee57bc31f8eea4cd7fb0cbcbe4912cf1cb2bf3b754f4a80"}, - {file = "coverage-7.2.6-cp310-cp310-win32.whl", hash = "sha256:71f739f97f5f80627f1fee2331e63261355fd1e9a9cce0016394b6707ac3f4ec"}, - {file = "coverage-7.2.6-cp310-cp310-win_amd64.whl", hash = "sha256:fde5c7a9d9864d3e07992f66767a9817f24324f354caa3d8129735a3dc74f126"}, - {file = "coverage-7.2.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bc7b667f8654376e9353dd93e55e12ce2a59fb6d8e29fce40de682273425e044"}, - {file = "coverage-7.2.6-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:697f4742aa3f26c107ddcb2b1784a74fe40180014edbd9adaa574eac0529914c"}, - {file = "coverage-7.2.6-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:541280dde49ce74a4262c5e395b48ea1207e78454788887118c421cb4ffbfcac"}, - {file = "coverage-7.2.6-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6e7f1a8328eeec34c54f1d5968a708b50fc38d31e62ca8b0560e84a968fbf9a9"}, - {file = "coverage-7.2.6-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4bbd58eb5a2371bf160590f4262109f66b6043b0b991930693134cb617bc0169"}, - {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ae82c5f168d2a39a5d69a12a69d4dc23837a43cf2ca99be60dfe59996ea6b113"}, - {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:f5440cdaf3099e7ab17a5a7065aed59aff8c8b079597b61c1f8be6f32fe60636"}, - {file = "coverage-7.2.6-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:a6f03f87fea579d55e0b690d28f5042ec1368650466520fbc400e7aeaf09e995"}, - {file = "coverage-7.2.6-cp311-cp311-win32.whl", hash = "sha256:dc4d5187ef4d53e0d4c8eaf530233685667844c5fb0b855fea71ae659017854b"}, - {file = "coverage-7.2.6-cp311-cp311-win_amd64.whl", hash = "sha256:c93d52c3dc7b9c65e39473704988602300e3cc1bad08b5ab5b03ca98bbbc68c1"}, - {file = "coverage-7.2.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:42c692b55a647a832025a4c048007034fe77b162b566ad537ce65ad824b12a84"}, - {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7786b2fa7809bf835f830779ad285215a04da76293164bb6745796873f0942d"}, - {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:25bad4196104761bc26b1dae9b57383826542ec689ff0042f7f4f4dd7a815cba"}, - {file = "coverage-7.2.6-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2692306d3d4cb32d2cceed1e47cebd6b1d2565c993d6d2eda8e6e6adf53301e6"}, - {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:392154d09bd4473b9d11351ab5d63391f3d5d24d752f27b3be7498b0ee2b5226"}, - {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:fa079995432037b5e2ef5ddbb270bcd2ded9f52b8e191a5de11fe59a00ea30d8"}, - {file = "coverage-7.2.6-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d712cefff15c712329113b01088ba71bbcef0f7ea58478ca0bbec63a824844cb"}, - {file = "coverage-7.2.6-cp37-cp37m-win32.whl", hash = "sha256:004948e296149644d208964300cb3d98affc5211e9e490e9979af4030b0d6473"}, - {file = "coverage-7.2.6-cp37-cp37m-win_amd64.whl", hash = "sha256:c1d7a31603c3483ac49c1726723b0934f88f2c011c660e6471e7bd735c2fa110"}, - {file = "coverage-7.2.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3436927d1794fa6763b89b60c896f9e3bd53212001026ebc9080d23f0c2733c1"}, - {file = "coverage-7.2.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:44c9b9f1a245f3d0d202b1a8fa666a80b5ecbe4ad5d0859c0fb16a52d9763224"}, - {file = "coverage-7.2.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e3783a286d5a93a2921396d50ce45a909aa8f13eee964465012f110f0cbb611"}, - {file = "coverage-7.2.6-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3cff6980fe7100242170092bb40d2b1cdad79502cd532fd26b12a2b8a5f9aee0"}, - {file = "coverage-7.2.6-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c534431153caffc7c495c3eddf7e6a6033e7f81d78385b4e41611b51e8870446"}, - {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:3062fd5c62df988cea9f2972c593f77fed1182bfddc5a3b12b1e606cb7aba99e"}, - {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6284a2005e4f8061c58c814b1600ad0074ccb0289fe61ea709655c5969877b70"}, - {file = "coverage-7.2.6-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:97729e6828643f168a2a3f07848e1b1b94a366b13a9f5aba5484c2215724edc8"}, - {file = "coverage-7.2.6-cp38-cp38-win32.whl", hash = "sha256:dc11b42fa61ff1e788dd095726a0aed6aad9c03d5c5984b54cb9e1e67b276aa5"}, - {file = "coverage-7.2.6-cp38-cp38-win_amd64.whl", hash = "sha256:cbcc874f454ee51f158afd604a315f30c0e31dff1d5d5bf499fc529229d964dd"}, - {file = "coverage-7.2.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d3cacc6a665221108ecdf90517a8028d07a2783df3417d12dcfef1c517e67478"}, - {file = "coverage-7.2.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:272ab31228a9df857ab5df5d67936d8861464dc89c5d3fab35132626e9369379"}, - {file = "coverage-7.2.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9a8723ccec4e564d4b9a79923246f7b9a8de4ec55fa03ec4ec804459dade3c4f"}, - {file = "coverage-7.2.6-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5906f6a84b47f995cd1bf0aca1c72d591c55ee955f98074e93660d64dfc66eb9"}, - {file = "coverage-7.2.6-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52c139b7ab3f0b15f9aad0a3fedef5a1f8c0b2bdc291d88639ca2c97d3682416"}, - {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:a5ffd45c6b93c23a8507e2f436983015c6457aa832496b6a095505ca2f63e8f1"}, - {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:4f3c7c19581d471af0e9cb49d928172cd8492cd78a2b7a4e82345d33662929bb"}, - {file = "coverage-7.2.6-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2e8c0e79820cdd67978e1120983786422d279e07a381dbf89d03bbb23ec670a6"}, - {file = "coverage-7.2.6-cp39-cp39-win32.whl", hash = "sha256:13cde6bb0e58fb67d09e2f373de3899d1d1e866c5a9ff05d93615f2f54fbd2bb"}, - {file = "coverage-7.2.6-cp39-cp39-win_amd64.whl", hash = "sha256:6b9f64526286255735847aed0221b189486e0b9ed943446936e41b7e44b08783"}, - {file = "coverage-7.2.6-pp37.pp38.pp39-none-any.whl", hash = "sha256:6babcbf1e66e46052442f10833cfc4a0d3554d8276aa37af8531a83ed3c1a01d"}, - {file = "coverage-7.2.6.tar.gz", hash = "sha256:2025f913f2edb0272ef15d00b1f335ff8908c921c8eb2013536fcaf61f5a683d"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8"}, + {file = "coverage-7.2.7-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2"}, + {file = "coverage-7.2.7-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353"}, + {file = "coverage-7.2.7-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495"}, + {file = "coverage-7.2.7-cp310-cp310-win32.whl", hash = "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818"}, + {file = "coverage-7.2.7-cp310-cp310-win_amd64.whl", hash = "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f"}, + {file = "coverage-7.2.7-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f"}, + {file = "coverage-7.2.7-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97"}, + {file = "coverage-7.2.7-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a"}, + {file = "coverage-7.2.7-cp311-cp311-win32.whl", hash = "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a"}, + {file = "coverage-7.2.7-cp311-cp311-win_amd64.whl", hash = "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562"}, + {file = "coverage-7.2.7-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01"}, + {file = "coverage-7.2.7-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de"}, + {file = "coverage-7.2.7-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d"}, + {file = "coverage-7.2.7-cp312-cp312-win32.whl", hash = "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511"}, + {file = "coverage-7.2.7-cp312-cp312-win_amd64.whl", hash = "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3"}, + {file = "coverage-7.2.7-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9"}, + {file = "coverage-7.2.7-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959"}, + {file = "coverage-7.2.7-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02"}, + {file = "coverage-7.2.7-cp37-cp37m-win32.whl", hash = "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f"}, + {file = "coverage-7.2.7-cp37-cp37m-win_amd64.whl", hash = "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5"}, + {file = "coverage-7.2.7-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6"}, + {file = "coverage-7.2.7-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5"}, + {file = "coverage-7.2.7-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f"}, + {file = "coverage-7.2.7-cp38-cp38-win32.whl", hash = "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e"}, + {file = "coverage-7.2.7-cp38-cp38-win_amd64.whl", hash = "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9"}, + {file = "coverage-7.2.7-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e"}, + {file = "coverage-7.2.7-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250"}, + {file = "coverage-7.2.7-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2"}, + {file = "coverage-7.2.7-cp39-cp39-win32.whl", hash = "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb"}, + {file = "coverage-7.2.7-cp39-cp39-win_amd64.whl", hash = "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27"}, + {file = "coverage-7.2.7-pp37.pp38.pp39-none-any.whl", hash = "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d"}, + {file = "coverage-7.2.7.tar.gz", hash = "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59"}, ] [package.dependencies] @@ -468,20 +477,20 @@ test = ["pandas[test]", "pre-commit", "pytest", "pytest-rerunfailures", "pytest- [[package]] name = "deprecated" -version = "1.2.13" +version = "1.2.14" description = "Python @deprecated decorator to deprecate old python classes, functions or methods." optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ - {file = "Deprecated-1.2.13-py2.py3-none-any.whl", hash = "sha256:64756e3e14c8c5eea9795d93c524551432a0be75629f8f29e67ab8caf076c76d"}, - {file = "Deprecated-1.2.13.tar.gz", hash = "sha256:43ac5335da90c31c24ba028af536a91d41d53f9e6901ddb021bcc572ce44e38d"}, + {file = "Deprecated-1.2.14-py2.py3-none-any.whl", hash = "sha256:6fac8b097794a90302bdbb17b9b815e732d3c4720583ff1b198499d78470466c"}, + {file = "Deprecated-1.2.14.tar.gz", hash = "sha256:e5323eb936458dccc2582dc6f9c322c852a775a27065ff2b0c4970b9d53d01b3"}, ] [package.dependencies] wrapt = ">=1.10,<2" [package.extras] -dev = ["PyTest", "PyTest (<5)", "PyTest-Cov", "PyTest-Cov (<2.6)", "bump2version (<1)", "configparser (<5)", "importlib-metadata (<3)", "importlib-resources (<4)", "sphinx (<2)", "sphinxcontrib-websupport (<2)", "tox", "zipp (<2)"] +dev = ["PyTest", "PyTest-Cov", "bump2version (<1)", "sphinx (<2)", "tox"] [[package]] name = "dill" @@ -1529,13 +1538,13 @@ test = ["hypothesis (>=5.5.3)", "pytest (>=6.0)", "pytest-xdist (>=1.31)"] [[package]] name = "parameterspace" -version = "0.9.1" +version = "0.10.0" description = "Parametrized hierarchical spaces with flexible priors and transformations." optional = false python-versions = ">=3.8,<4.0" files = [ - {file = "parameterspace-0.9.1-py3-none-any.whl", hash = "sha256:4a74b9ecd281969c59228bcda24c95bded9cc48336c580be332916f7f154b26f"}, - {file = "parameterspace-0.9.1.tar.gz", hash = "sha256:b36049b59af3d5b093e0adb8de5fae85a6d02e880b0c872c06537129feaedd1c"}, + {file = "parameterspace-0.10.0-py3-none-any.whl", hash = "sha256:3b8b4c078f5ccfef69ff870da957b21ec548709da6efa78373585a82d4b05c70"}, + {file = "parameterspace-0.10.0.tar.gz", hash = "sha256:695693d4a253d6a9d2b3b5e985c94f9d4b29764624a0a6cbd8dcd321218e9629"}, ] [package.dependencies] @@ -1851,20 +1860,20 @@ test = ["flake8", "pytest (>=5.0)"] [[package]] name = "pyro-ppl" -version = "1.8.4" +version = "1.8.5" description = "A Python library for probabilistic modeling and inference" optional = true python-versions = ">=3.7" files = [ - {file = "pyro-ppl-1.8.4.tar.gz", hash = "sha256:766fad61e52df48885de96d41213da1f8e8c1b79ecf308ad53189fcd15c1cb41"}, - {file = "pyro_ppl-1.8.4-py3-none-any.whl", hash = "sha256:294f78f28f2fe7bbea2792bd6bd8c69b7cfe493cf8940cac97a9b5d0e7f194cd"}, + {file = "pyro-ppl-1.8.5.tar.gz", hash = "sha256:c2ac035995874f4adb8d07c92aa4594fe783ffde1614a59cce5e2e0d9cf19334"}, + {file = "pyro_ppl-1.8.5-py3-none-any.whl", hash = "sha256:09616269053f45b9c230ed6bcfbde199b9c308780d972b863ef7bdb707a1c346"}, ] [package.dependencies] numpy = ">=1.7" opt-einsum = ">=2.3.2" pyro-api = ">=0.1.1" -torch = ">=1.11.0" +torch = ">=2.0.1" tqdm = ">=4.36" [package.extras] @@ -1872,6 +1881,7 @@ dev = ["black (>=21.4b0)", "flake8", "graphviz (>=0.8)", "isort (>=5.0)", "jupyt extras = ["graphviz (>=0.8)", "jupyter (>=1.0.0)", "lap", "matplotlib (>=1.3)", "pandas", "pillow (==8.2.0)", "scikit-learn", "seaborn (>=0.11.0)", "torchvision (>=0.12.0)", "visdom (>=0.1.4,<0.2.2)", "wget"] funsor = ["funsor[torch] (==0.4.4)"] horovod = ["horovod[pytorch] (>=0.19)"] +lightning = ["pytorch-lightning"] profile = ["prettytable", "pytest-benchmark", "snakeviz"] test = ["black (>=21.4b0)", "flake8", "graphviz (>=0.8)", "jupyter (>=1.0.0)", "lap", "matplotlib (>=1.3)", "nbval", "pandas", "pillow (==8.2.0)", "pytest (>=5.0)", "pytest-cov", "scikit-learn", "scipy (>=1.1)", "seaborn (>=0.11.0)", "torchvision (>=0.12.0)", "visdom (>=0.1.4,<0.2.2)", "wget"] @@ -2440,13 +2450,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.6.1" +version = "4.6.2" description = "Backported and Experimental Type Hints for Python 3.7+" optional = false python-versions = ">=3.7" files = [ - {file = "typing_extensions-4.6.1-py3-none-any.whl", hash = "sha256:6bac751f4789b135c43228e72de18637e9a6c29d12777023a703fd1a6858469f"}, - {file = "typing_extensions-4.6.1.tar.gz", hash = "sha256:558bc0c4145f01e6405f4a5fdbd82050bd221b119f4bf72a961a1cfd471349d6"}, + {file = "typing_extensions-4.6.2-py3-none-any.whl", hash = "sha256:3a8b36f13dd5fdc5d1b16fe317f5668545de77fa0b8e02006381fd49d731ab98"}, + {file = "typing_extensions-4.6.2.tar.gz", hash = "sha256:06006244c70ac8ee83fa8282cb188f697b8db25bc8b4df07be1873c43897060c"}, ] [[package]] @@ -2682,6 +2692,7 @@ all = ["botorch", "dask", "distributed", "numpy", "pandas", "plotly", "pymoo", " bohb = ["numpy", "scipy", "statsmodels"] botorch = ["botorch", "sympy", "torch"] dask = ["dask", "distributed"] +examples = ["dask", "distributed"] hyperband = ["numpy"] space-fill = ["numpy", "scipy"] testing = ["numpy"] @@ -2690,4 +2701,4 @@ visualization = ["pandas", "plotly", "pymoo", "scipy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "214ad5797fb44bf764d0d5d91776ca9159ada96fb080459419665d161d125c04" +content-hash = "0b7eccfb2872232e2b3195806211ce1d86b2150c7cdc1d6a93bbe471d2598813" diff --git a/pyproject.toml b/pyproject.toml index bfa4bbb6..ba3842d6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,7 +54,7 @@ types-toml = "^0.10.7" kaleido = "0.2.1" # pin hard, because 0.2.1post1 fails to install [tool.poetry.extras] -all = ["numpy", "plotly", "scipy", "statsmodels", "dask", "distributed", "pandas", "sympy", "torch", "botorch", "pymoo"] +all = ["numpy", "plotly", "scipy", "statsmodels", "dask", "distributed", "pandas", "sympy", "torch", "botorch", "pymoo", "scikit-learn"] hyperband = ["numpy"] bohb = ["numpy", "scipy", "statsmodels"] botorch = ["sympy", "torch", "botorch"] @@ -62,6 +62,7 @@ space-fill = ["numpy", "scipy"] dask = ["dask", "distributed"] visualization = ["plotly", "scipy", "pandas", "pymoo"] testing = ["numpy"] +examples = ["scikit-learn", "dask", "distributed"] [tool.mypy] ignore_missing_imports = true @@ -70,7 +71,7 @@ no_strict_optional = true [tool.black] line-length = 88 # Examples should remain with line-length 80: -exclude = '^blackboxopt/examples/' +extend-exclude = 'blackboxopt/examples' [tool.ruff] # Enable pycodestyle, Pyflakes, isort diff --git a/tests/examples_test.py b/tests/examples_test.py new file mode 100644 index 00000000..423e76c4 --- /dev/null +++ b/tests/examples_test.py @@ -0,0 +1,30 @@ +from pathlib import Path + +import pytest + +from blackboxopt.examples import multi_objective_multi_param + + +@pytest.mark.parametrize("example_module", [multi_objective_multi_param]) +@pytest.mark.integration_test +def test_full_loop_examples(tmp_path, monkeypatch, example_module): + if not example_module: + return + + # Some examples run until a timeout of 60sec. As we are in a hurry, + # we add/overwrite that argument for the loop: + run_sequential = example_module.run_optimization_loop + + def run_sequential_mocked(*args, **kwargs): + kwargs["timeout_s"] = 5 + return run_sequential(*args, **kwargs) + + monkeypatch.setattr(example_module, "run_optimization_loop", run_sequential_mocked) + + # Some examples write output files. Let's redirect those to temporary path: + if getattr(example_module, "REPORT_PATH", None): + new_path = tmp_path / Path(example_module.REPORT_PATH).name + monkeypatch.setattr(example_module, "REPORT_PATH", str(new_path.absolute())) + + # Actually execute the example + example_module.main()