add multi objective example and default logger init (#106)

* add multi objective example and default logger init

---------

Signed-off-by: Grossberger Lukas (CR/AIR2.2) <Lukas.Grossberger@de.bosch.com>

.pre-commit-config.yaml

@@ -18,10 +18,11 @@ repos:
         entry: poetry run black
         language: system
         types: [python]
+        exclude: ^blackboxopt_client/examples/
       - id: black-examples
         name: black-examples
-        entry: poetry run black --line-length=80
-        files: ^blackboxopt/examples/
+        entry: poetry run black --extend-exclude ^$ --line-length=80
+        files: ^blackboxopt_client/examples/
         language: system
         types: [python]
       - id: mypy

blackboxopt/__init__.py

@@ -2,7 +2,7 @@
 
 from parameterspace import ParameterSpace
 
-from . import io
+from . import io, utils
 from .base import (
     ConstraintsError,
     ContextError,
@@ -14,4 +14,5 @@
     OptimizerNotReady,
 )
 from .evaluation import Evaluation, EvaluationSpecification
-from .utils import sort_evaluations
+from .logger import logger
+from .utils import init_logger, sort_evaluations

blackboxopt/examples/multi_objective_multi_param.py

@@ -0,0 +1,90 @@
+# Copyright (c) 2020 - for information on the respective copyright owner
+# see the NOTICE file and/or the repository https://github.com/boschresearch/blackboxopt
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import logging
+import time
+
+import numpy as np
+import parameterspace as ps
+from sklearn.datasets import load_diabetes
+from sklearn.ensemble import RandomForestRegressor
+from sklearn.metrics import r2_score
+from sklearn.model_selection import train_test_split
+
+import blackboxopt as bbo
+from blackboxopt.optimization_loops.sequential import run_optimization_loop
+from blackboxopt.optimizers.space_filling import SpaceFilling
+
+# Load a sklearn sample dataset
+X_TRAIN, X_VALIDATE, Y_TRAIN, Y_VALIDATE = train_test_split(
+    *load_diabetes(return_X_y=True)
+)
+
+# Set up search space with multiple ML model hyperparameters of different types
+SPACE = ps.ParameterSpace()
+SPACE.add(ps.IntegerParameter("n_estimators", bounds=(256, 2048), transformation="log"))
+SPACE.add(ps.IntegerParameter("min_samples_leaf", bounds=(1, 32), transformation="log"))
+SPACE.add(ps.ContinuousParameter("max_samples", bounds=(0.1, 1)))
+SPACE.add(ps.ContinuousParameter("max_features", bounds=(0.1, 1)))
+SPACE.add(ps.IntegerParameter("max_depth", bounds=(1, 128)))
+SPACE.add(ps.CategoricalParameter("criterion", values=("squared_error", "poisson")))
+
+
+def evaluation_function(
+    eval_spec: bbo.EvaluationSpecification,
+) -> bbo.Evaluation:
+    """Train and evaluate a random forest with given parameter configuration."""
+    regr = RandomForestRegressor(
+        n_estimators=eval_spec.configuration["n_estimators"],
+        max_samples=eval_spec.configuration["max_samples"],
+        max_features=eval_spec.configuration["max_features"],
+        max_depth=eval_spec.configuration["max_depth"],
+        min_samples_leaf=eval_spec.configuration["min_samples_leaf"],
+        criterion=eval_spec.configuration["criterion"],
+    )
+
+    start = time.time()
+    regr.fit(X_TRAIN, Y_TRAIN)
+    fit_duration = time.time() - start
+
+    y_pred = regr.predict(X_VALIDATE)
+    objectives = {
+        "R²": r2_score(Y_VALIDATE, y_pred),
+        "Fit Duration": fit_duration,
+        "Max Error": np.abs(Y_VALIDATE - y_pred).max(),
+    }
+    evaluation = eval_spec.create_evaluation(objectives=objectives)
+    return evaluation
+
+
+def main():
+    logger = bbo.init_logger(logging.INFO)
+
+    # Create an optimization run based on a parameterspace and optimizer choice
+    optimizer = SpaceFilling(
+        search_space=SPACE,
+        objectives=[
+            bbo.Objective("R²", greater_is_better=True),
+            bbo.Objective("Max Error", greater_is_better=False),
+            bbo.Objective("Fit Duration", greater_is_better=False),
+        ],
+    )
+
+    # Fetch new configurations to evaluate until the optimization is done or
+    # a given timeout is reached
+    evaluations = run_optimization_loop(
+        optimizer=optimizer,
+        evaluation_function=evaluation_function,
+        timeout_s=60.0,
+    )
+
+    logger.info(f"Evaluated {len(evaluations)} specifications")
+
+    pareto_front = bbo.utils.filter_pareto_efficient(evaluations, optimizer.objectives)
+    logger.info(f"{len(pareto_front)} evaluation(s) are pareto efficient")
+
+
+if __name__ == "__main__":
+    main()

blackboxopt/logger.py

@@ -0,0 +1,5 @@
+import logging
+import os
+
+# Default logger used throughout the package
+logger = logging.getLogger(os.environ.get("BBO_LOGGER_NAME", "blackboxopt"))

blackboxopt/optimization_loops/sequential.py

@@ -14,6 +14,7 @@
     OptimizationComplete,
     OptimizerNotReady,
 )
+from blackboxopt import logger as default_logger
 from blackboxopt.base import MultiObjectiveOptimizer, SingleObjectiveOptimizer
 from blackboxopt.optimization_loops.utils import (
     evaluation_function_wrapper,
@@ -55,12 +56,13 @@ def run_optimization_loop(
             optimization loop.
         post_evaluation_callback: Reference to a callable that is invoked after each
             evaluation and takes a `blackboxopt.Evaluation` as its argument.
-        logger: The logger to use for logging progress.
+        logger: The logger to use for logging progress. Default: `blackboxopt.logger`
 
     Returns:
-        List of evluation specification and result for all evaluations.
+        List of evaluation specification and result for all evaluations.
     """
-    logger = logging.getLogger("blackboxopt") if logger is None else logger
+    if logger is None:
+        logger = default_logger
 
     objectives = (
         optimizer.objectives

blackboxopt/utils.py

@@ -4,14 +4,18 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import hashlib
+import logging
+import os
 import pickle
+import sys
 from itertools import compress
 from typing import Dict, Iterable, List, Optional, Sequence
 
 import numpy as np
 
 from blackboxopt.base import Objective
 from blackboxopt.evaluation import Evaluation
+from blackboxopt.logger import logger
 
 
 def get_loss_vector(
@@ -99,3 +103,31 @@ def sort_evaluations(evaluations: Iterable[Evaluation]) -> Iterable[Evaluation]:
             )
         ).hexdigest(),
     )
+
+
+def init_logger(level: int = None) -> logging.Logger:  # pragma: no cover
+    """Initialize the default `blackboxopt.logger` as a nicely formatted stdout logger.
+
+    Should no log level be given, the environment variable `BBO_LOG_LEVEL` is used
+    and if that is not present, the default is `logging.DEBUG`.
+
+    Args:
+        level: the log level to set
+
+    Returns:
+        The logger instance (equivalent to `blackboxopt.logger`)
+    """
+    if level is None:
+        level_name = os.environ.get("BBO_LOG_LEVEL", "DEBUG")
+        level = getattr(logging, level_name)
+
+    logger.setLevel(level)
+    handler = logging.StreamHandler(sys.stdout)
+    handler.setLevel(level)
+    formatter = logging.Formatter(
+        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    )
+    handler.setFormatter(formatter)
+    logger.addHandler(handler)
+
+    return logger

docs/examples/multi-objective-multi-param.md

@@ -0,0 +1,11 @@
+# Mixed Space & Multi Objective
+
+Aside from continuous parameters, different types are supported by `parameterspace`.
+In the following example we use continuous, integer and categorical parameters.
+Also, we are having a glance at optimizing multiple objectives at the same time.
+
+```python
+--8<--
+blackboxopt/examples/multi_objective_multi_param.py
+--8<--
+```

docs/examples/overview.md

@@ -9,3 +9,4 @@ pip install blackboxopt[examples]
 ## List of available examples
 
 - [Dask Distributed](dask-distributed.md)
+- [Multi Objective Optimization](multi-objective-multi-param.md)

mkdocs.yml

@@ -30,6 +30,7 @@ nav:
   - Examples:
       - Overview: examples/overview.md
       - examples/dask-distributed.md
+      - examples/multi-objective-multi-param.md
   - ...
 
 plugins: