From 56185931e4d91ca9c5fb1186a9233afd17f6b41c Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Tue, 13 Aug 2024 23:48:29 -0400 Subject: [PATCH] Major improvements to testing design and coverage --- .flake8 | 7 +- obsidian/tests/param_configs.py | 25 ++++--- obsidian/tests/test_campaign.py | 99 +++++++++++++------------ obsidian/tests/test_constraints.py | 64 +++++++++++++++++ obsidian/tests/test_experiment.py | 2 + obsidian/tests/test_objectives.py | 82 +++++++++++++++++++++ obsidian/tests/test_optimizer_MOO.py | 103 +++++++-------------------- obsidian/tests/test_optimizer_SOO.py | 101 ++++++++++---------------- obsidian/tests/test_parameters.py | 62 ++++++++++++---- obsidian/tests/test_plotting.py | 3 + obsidian/tests/utils.py | 36 +++++++++- pyproject.toml | 3 +- 12 files changed, 376 insertions(+), 211 deletions(-) create mode 100644 obsidian/tests/test_constraints.py create mode 100644 obsidian/tests/test_objectives.py diff --git a/.flake8 b/.flake8 index fb314fa..8225b83 100644 --- a/.flake8 +++ b/.flake8 @@ -10,8 +10,11 @@ per-file-ignores = __init__.py: F401, F403 obsidian/dash/*: F401, F403 - # Often creating variables but not accessing them in testing - obsidian/tests/*: F841 + # Often importing and creating unaccessed objects during testing + obsidian/tests/*: F401, F841 + + # No good way around comparing types for recursive state-dict comparison + obsidian/tests/utils.py: E721 exclude = projects/ diff --git a/obsidian/tests/param_configs.py b/obsidian/tests/param_configs.py index 8b369a0..4c927ff 100644 --- a/obsidian/tests/param_configs.py +++ b/obsidian/tests/param_configs.py @@ -1,6 +1,9 @@ +"""Preset parameter configurations for unit testing""" + from obsidian.parameters import Param_Continuous, Param_Ordinal, Param_Categorical, \ Param_Observational, Param_Discrete_Numeric, ParamSpace +# Set up ap master list of parameter spaces for testing params = [ Param_Continuous('Parameter 1', 0, 10), Param_Continuous('Parameter 2', -20, 0), @@ -15,14 +18,25 @@ Param_Ordinal('Parameter 11', ['N']) ] +# Subset some default selections default = [params[i] for i in [0, 1, 2, 6]] # 2 continuous, 1 static, 1 categorical +X_sp_default = ParamSpace(params=default) + +# Numeric cont_small = [params[i] for i in [0, 1, 2]] # continuous including edge cases numeric = [params[i] for i in [0, 1, 2, 3, 4, 5]] # numeric including edge cases +X_sp_cont_small = ParamSpace(params=cont_small) +X_sp_numeric = ParamSpace(params=numeric) +# Nominal cat_small = [params[i] for i in [6, 7, 8]] # categorical including edge cases disc_small = [params[i] for i in [6, 9]] # 1 categorical, 1 ordinal disc_large = [params[i] for i in [6, 7, 8, 9, 10]] # discrete including edge cases +X_sp_cat_small = ParamSpace(params=cat_small) +X_sp_disc_small = ParamSpace(params=disc_small) +X_sp_disc_large = ParamSpace(params=disc_large) +# Set up a range of continuous parameters params_cont_large = [ Param_Continuous('Parameter 1', 0, 10), Param_Continuous('Parameter 2', 0, 10), @@ -37,15 +51,8 @@ Param_Continuous('Parameter 11', 0, 10), Param_Continuous('Parameter 12', 0, 10), ] - -X_sp_default = ParamSpace(params=default) -X_sp_cont_small = ParamSpace(params=cont_small) X_sp_cont_large = ParamSpace(params=params_cont_large) -X_sp_numeric = ParamSpace(params=numeric) -X_sp_cat_small = ParamSpace(params=cat_small) -X_sp_disc_small = ParamSpace(params=disc_small) -X_sp_disc_large = ParamSpace(params=disc_large) +X_sp_cont_ndims = [ParamSpace(params_cont_large[:i]) for i in range(len(params_cont_large))] +# Wrap everything for iteration during testing test_X_space = [X_sp_default, X_sp_cont_small, X_sp_numeric, X_sp_cat_small, X_sp_disc_small, X_sp_disc_large] - -X_sp_cont_ndims = [ParamSpace(params_cont_large[:i]) for i in range(len(params_cont_large))] diff --git a/obsidian/tests/test_campaign.py b/obsidian/tests/test_campaign.py index 8a55e23..31efd2e 100644 --- a/obsidian/tests/test_campaign.py +++ b/obsidian/tests/test_campaign.py @@ -1,21 +1,21 @@ +"""PyTests for obsidian.campaign""" -from obsidian.tests.param_configs import X_sp_cont_ndims, X_sp_default from obsidian.parameters import Target from obsidian.experiment import Simulator from obsidian.experiment.benchmark import two_leaves, shifted_parab from obsidian.campaign import Campaign, Explainer, calc_ofat_ranges -from obsidian.objectives import Identity_Objective, Scalar_WeightedNorm, Feature_Objective, \ - Objective_Sequence, Utopian_Distance, Index_Objective, Bounded_Target +from obsidian.objectives import Identity_Objective from obsidian.plotting import plot_interactions, plot_ofat_ranges from obsidian.exceptions import IncompatibleObjectiveError, UnfitError - -from obsidian.tests.utils import DEFAULT_MOO_PATH -import json +from obsidian.tests.param_configs import X_sp_cont_ndims, X_sp_default +from obsidian.tests.utils import DEFAULT_MOO_PATH, equal_state_dicts import pandas as pd import pytest - +import json + +# Avoid using TkAgg which causes Tcl issues during testing import matplotlib matplotlib.use('inline') @@ -29,79 +29,70 @@ @pytest.mark.parametrize('X_space, sim_fcn, target', [(X_sp_cont_ndims[2], two_leaves, target_test[0]), (X_sp_default, shifted_parab, target_test[1])]) -def test_campaign(X_space, sim_fcn, target): +def test_campaign_basics(X_space, sim_fcn, target): + # Standard usage campaign = Campaign(X_space, target) simulator = Simulator(X_space, sim_fcn, eps=0.05) X0 = campaign.suggest() y0 = simulator.simulate(X0) Z0 = pd.concat([X0, y0], axis=1) - campaign.m_exp - # Test some conditional usage + # Set an objective, suggest, clear + campaign.set_objective(Identity_Objective(mo=len(campaign.target) > 1)) + campaign.suggest() + campaign.clear_objective() + + # Add, fit, clear, examine campaign.add_data(Z0) campaign.fit() campaign.clear_data() campaign.y + campaign.__repr__() + # Add with iteration, examine, fit, analyze Z0['Iteration'] = 5 campaign.add_data(Z0) campaign.y campaign.fit() campaign.response_max + # Serialize, deserialize, re-serialize obj_dict = campaign.save_state() campaign2 = Campaign.load_state(obj_dict) - campaign2.__repr__() - - campaign2.set_objective(Identity_Objective(mo=len(campaign.target) > 1)) - campaign2.suggest() + obj_dict2 = campaign2.save_state() + assert equal_state_dicts(obj_dict, obj_dict2), 'Error during serialization' +# Load default with open(DEFAULT_MOO_PATH) as json_file: obj_dict = json.load(json_file) - campaign = Campaign.load_state(obj_dict) X_space = campaign.X_space target = campaign.target -test_objs = [Identity_Objective(mo=True), - Scalar_WeightedNorm(weights=[1, 1]), - Feature_Objective(X_space, indices=[0], coeff=[1]), - Objective_Sequence([Utopian_Distance([1], target[0]), Index_Objective()]), - Bounded_Target(bounds=[(0, 1), (0, 1)], targets=target), - None] - - -@pytest.mark.parametrize('obj', test_objs) -def test_campaign_objectives(obj): - campaign.set_objective(obj) - if campaign.objective: - campaign.objective.__repr__() - campaign.o - - obj_dict = campaign.save_state() - campaign2 = Campaign.load_state(obj_dict) - campaign2.save_state() - campaign2.__repr__() - campaign2.clear_objective() - def test_explain(): + # Standard usage exp = Explainer(campaign.optimizer) - exp.__repr__ exp.shap_explain(n=50) + exp.__repr__ + # Test SHAP plots exp.shap_summary() - fig = exp.shap_summary_bar() + exp.shap_summary_bar() + + # Test PDP-ICE, with options exp.shap_pdp_ice(ind=0, ice_color_var=None, npoints=10) exp.shap_pdp_ice(ind=0, npoints=10) exp.shap_pdp_ice(ind=(0, 1), npoints=5) + # Test pairwise SHAP analysis, with options X_new = campaign.X.iloc[0, :] X_ref = campaign.X.loc[1, :] df_shap_value_new, fig_bar, fig_line = exp.shap_single_point(X_new) df_shap_value_new, fig_bar, fig_line = exp.shap_single_point(X_new, X_ref=X_ref) + # Test sensitivity analysis, with options df_sens = exp.sensitivity() df_sens = exp.sensitivity(X_ref=X_ref) @@ -112,71 +103,89 @@ def test_explain(): @pytest.mark.parametrize('X_ref', X_ref_test) def test_analysis(X_ref): + # OFAT ranges with/out interactions and with/out X_ref ofat_ranges, _ = calc_ofat_ranges(campaign.optimizer, threshold=0.5, X_ref=X_ref, calc_interacts=False) ofat_ranges, cor = calc_ofat_ranges(campaign.optimizer, threshold=0.5, X_ref=X_ref) plot_interactions(campaign.optimizer, cor) plot_ofat_ranges(campaign.optimizer, ofat_ranges) + # OFAT ranges where all results should be NaN ofat_ranges, cor = calc_ofat_ranges(campaign.optimizer, threshold=9999, X_ref=X_ref) plot_interactions(campaign.optimizer, cor) plot_ofat_ranges(campaign.optimizer, ofat_ranges) +# VALIDATION TESTS - Force errors to be raised in object usage + @pytest.mark.fast def test_campaign_validation(): + # Missing X names random_data = pd.DataFrame(data={'A': [1, 2, 3], 'B': [4, 5, 6]}) with pytest.raises(KeyError): campaign.add_data(random_data) - + + # Missing Y names with pytest.raises(KeyError): campaign.add_data(campaign.X) - with pytest.raises(IncompatibleObjectiveError): - campaign.set_objective(Identity_Objective(mo=False)) - + # Missing data with pytest.raises(ValueError): campaign2 = Campaign(X_space, target) campaign2.fit() + # Bad objective + with pytest.raises(IncompatibleObjectiveError): + campaign.set_objective(Identity_Objective(mo=False)) + @pytest.mark.fast def test_explainer_validation(): + # Unfit optimizer campaign2 = Campaign(X_space, target) with pytest.raises(UnfitError): exp = Explainer(campaign2.optimizer) - + + # Unfit SHAP exp = Explainer(campaign.optimizer) with pytest.raises(UnfitError): exp.shap_summary() - + + # Unfit SHAP with pytest.raises(UnfitError): exp.shap_summary_bar() - + + # Unfit SHAP with pytest.raises(UnfitError): exp.shap_single_point(X_new=campaign.X_space.mean()) random_data = pd.DataFrame(data={'A': [1], 'B': [4]}) long_data = pd.DataFrame(data={'Parameter 1': [1, 2], 'Parameter 2': [1, 2]}) + # Missing X names with pytest.raises(ValueError): exp.shap_explain(n=50, X_ref=random_data) + # X_ref > 1 row with pytest.raises(ValueError): exp.shap_explain(n=50, X_ref=long_data) exp.shap_explain(n=50) + # Missing X names with pytest.raises(ValueError): exp.shap_single_point(X_new=random_data) + # Missing X names with pytest.raises(ValueError): exp.shap_single_point(X_new=campaign.X_space.mean(), X_ref=random_data) + # Missing X names with pytest.raises(ValueError): exp.sensitivity(X_ref=random_data) + # X_ref > 1 row with pytest.raises(ValueError): exp.sensitivity(X_ref=long_data) diff --git a/obsidian/tests/test_constraints.py b/obsidian/tests/test_constraints.py new file mode 100644 index 0000000..31b5049 --- /dev/null +++ b/obsidian/tests/test_constraints.py @@ -0,0 +1,64 @@ +"""PyTests for obsidian.constraints""" + +from obsidian.campaign import Campaign +from obsidian.constraints import ( + OutConstraint_Blank, + InConstraint_Generic, + InConstraint_ConstantDim, + OutConstraint_L1 +) +from obsidian.tests.utils import DEFAULT_MOO_PATH + +import pandas as pd +import pytest +import json + +# Load defaults +with open(DEFAULT_MOO_PATH) as json_file: + obj_dict = json.load(json_file) +campaign = Campaign.load_state(obj_dict) + +optimizer = campaign.optimizer +X_space = campaign.X_space +target = campaign.target + +test_ineq = [[InConstraint_Generic(X_space, indices=[0, 1], coeff=[1, 1], rhs=5)]] +test_nleq = [[InConstraint_ConstantDim(X_space, dim=0, tol=0.1)]] +test_out = [[OutConstraint_Blank(target)], [OutConstraint_L1(target, offset=1)]] + +# Run very short optimizations for testing +test_config = {'optim_samples': 2, 'optim_restarts': 2} + + +@pytest.mark.parametrize('ineq_constraints', test_ineq) +def test_ineq_constraints(ineq_constraints): + X_suggest, eval_suggest = optimizer.suggest(ineq_constraints=ineq_constraints, + **test_config) + df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) + + +@pytest.mark.parametrize('nleq_constraints', test_nleq) +def test_nleq_constraints(nleq_constraints): + X_suggest, eval_suggest = optimizer.suggest(nleq_constraints=nleq_constraints, + **test_config) + df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) + + +@pytest.mark.parametrize('out_constraints', test_out) +def test_out_constraints(out_constraints): + X_suggest, eval_suggest = optimizer.suggest(out_constraints=out_constraints, + **test_config) + df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) + + +@pytest.mark.slow +def test_combo_constraints(): + X_suggest, eval_suggest = optimizer.suggest(ineq_constraints=test_ineq[0], + nleq_constraints=test_nleq[0], + out_constraints=test_out[0], + **test_config) + df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) + + +if __name__ == '__main__': + pytest.main([__file__, '-m', 'not slow']) diff --git a/obsidian/tests/test_experiment.py b/obsidian/tests/test_experiment.py index aebec50..4be7ee2 100644 --- a/obsidian/tests/test_experiment.py +++ b/obsidian/tests/test_experiment.py @@ -1,3 +1,5 @@ +"""PyTests for obsidian.experiment""" + from obsidian.tests.param_configs import test_X_space from obsidian.experiment import ExpDesigner, Simulator diff --git a/obsidian/tests/test_objectives.py b/obsidian/tests/test_objectives.py new file mode 100644 index 0000000..23dd6a8 --- /dev/null +++ b/obsidian/tests/test_objectives.py @@ -0,0 +1,82 @@ +"""PyTests for obsidian.campaign""" + +from obsidian.campaign import Campaign +from obsidian.objectives import ( + Identity_Objective, + Scalar_WeightedNorm, + Scalar_WeightedSum, + Scalar_Chebyshev, + Feature_Objective, + Objective_Sequence, + Utopian_Distance, + Index_Objective, + Bounded_Target +) + +from obsidian.tests.utils import DEFAULT_MOO_PATH, equal_state_dicts +from obsidian.exceptions import IncompatibleObjectiveError + +import pandas as pd +import pytest +import json + +# Load default +with open(DEFAULT_MOO_PATH) as json_file: + obj_dict = json.load(json_file) +campaign = Campaign.load_state(obj_dict) +X_space = campaign.X_space +target = campaign.target + +# Run very short optimizations for testing +test_config = {'optim_samples': 2, 'optim_restarts': 2} + +test_objs = [Identity_Objective(mo=len(target) > 1), + Scalar_WeightedNorm(weights=[1, 1]), + Feature_Objective(X_space, indices=[0], coeff=[1]), + Objective_Sequence([Utopian_Distance([1], target[0]), Index_Objective()]), + Bounded_Target(bounds=[(0, 1)]*len(target), targets=target), + None] + +utopian = Utopian_Distance(utopian=[10, 10], targets=target) + +test_scalars = [Scalar_WeightedSum(weights=[0.5, 0.5]), + Objective_Sequence([utopian, Scalar_WeightedSum(weights=[0.5, 0.5])]), + Scalar_WeightedNorm(weights=[0.5, 0.5]), + Objective_Sequence([utopian, Scalar_WeightedNorm(weights=[0.5, 0.5], neg=True)]), + Scalar_Chebyshev(weights=[0.5, 0.5]), + Scalar_Chebyshev(weights=[0.5, 0.5], augment=False), + Objective_Sequence([utopian, Scalar_Chebyshev(weights=[0.5, 0.5])])] + + +@pytest.mark.parametrize('obj', test_objs + test_scalars) +def test_campaign_objectives(obj): + # Set objective, read, examine output + campaign.set_objective(obj) + if campaign.objective: + campaign.objective.__repr__() + campaign.o + + # Serialize, deserialize, re-serialize + obj_dict = campaign.save_state() + campaign2 = Campaign.load_state(obj_dict) + obj_dict2 = campaign2.save_state() + assert equal_state_dicts(obj_dict, obj_dict2), 'Error during serialization' + + +@pytest.mark.parametrize('m_batch', [pytest.param(1, marks=pytest.mark.fast), 2, pytest.param(5, marks=pytest.mark.slow)]) +@pytest.mark.parametrize('obj', test_objs) +def test_objective_suggestions(m_batch, obj): + optimizer = campaign.optimizer + if obj is not None: + if obj._is_mo and optimizer.n_response == 1 and not isinstance(obj, Feature_Objective): + with pytest.raises(IncompatibleObjectiveError): + X_suggest, eval_suggest = optimizer.suggest(m_batch=m_batch, objective=obj, **test_config) + else: + X_suggest, eval_suggest = optimizer.suggest(m_batch=m_batch, objective=obj, **test_config) + else: + X_suggest, eval_suggest = optimizer.suggest(m_batch=m_batch, **test_config) + df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) + + +if __name__ == '__main__': + pytest.main([__file__, '-m', 'not slow']) diff --git a/obsidian/tests/test_optimizer_MOO.py b/obsidian/tests/test_optimizer_MOO.py index ca06832..ba35068 100644 --- a/obsidian/tests/test_optimizer_MOO.py +++ b/obsidian/tests/test_optimizer_MOO.py @@ -1,13 +1,12 @@ +"""PyTests for obsidian.optimizer under multi-output usage""" from obsidian.tests.param_configs import X_sp_cont_ndims from obsidian.parameters import Target from obsidian.experiment import ExpDesigner, Simulator -from obsidian.experiment.benchmark import two_leaves from obsidian.optimizer import BayesianOptimizer -from obsidian.constraints import OutConstraint_Blank, InConstraint_Generic, InConstraint_ConstantDim -from obsidian.objectives import Identity_Objective, Index_Objective, Utopian_Distance, Objective_Sequence, Bounded_Target, \ - Scalar_WeightedSum, Scalar_WeightedNorm, Scalar_Chebyshev +from obsidian.experiment.benchmark import two_leaves +from obsidian.tests.utils import equal_state_dicts import pandas as pd import numpy as np @@ -22,7 +21,7 @@ def X_space(): @pytest.fixture() def Z0(X_space): designer = ExpDesigner(X_space, seed=0) - X0 = designer.initialize(m_initial=6, method='LHS') + X0 = designer.initialize(m_initial=len(X_space)*2, method='LHS') simulator = Simulator(X_space, two_leaves, eps=0.05) y0 = simulator.simulate(X0) Z0 = pd.concat([X0, y0], axis=1) @@ -37,6 +36,7 @@ def Z0(X_space): @pytest.mark.parametrize('surrogate', [pytest.param('GP', marks=pytest.mark.fast), 'GPflat', + 'GPprior', pytest.param('DKL', marks=pytest.mark.slow), 'DNN']) def test_optimizer_fit(X_space, surrogate, Z0, serial_test=True): @@ -46,8 +46,10 @@ def test_optimizer_fit(X_space, surrogate, Z0, serial_test=True): optimizer.fit(Z0, target=target) if serial_test: - save = optimizer.save_state() - optimizer_2 = BayesianOptimizer.load_state(save) + obj_dict = optimizer.save_state() + optimizer_2 = BayesianOptimizer.load_state(obj_dict) + obj_dict2 = optimizer_2.save_state() + assert equal_state_dicts(obj_dict, obj_dict2) optimizer_2.__repr__() y_pred = optimizer.predict(optimizer.X_train) y_pred_2 = optimizer_2.predict(optimizer.X_train) @@ -65,8 +67,8 @@ def test_optimizer_fit(X_space, surrogate, Z0, serial_test=True): Z0_base = pd.concat([X0, y0], axis=1) optimizer.fit(Z0_base, target=target) -# Run very short and bad optimizations for testing, but test all MOO aqs -test_config = {'optim_samples': 4, 'optim_restarts': 2} +# Run very short optimizations for testing +test_config = {'optim_samples': 2, 'optim_restarts': 2} def test_fit_nan(): @@ -77,6 +79,13 @@ def test_fit_nan(): optimizer_nan.fit(Z0_sample, target=target) +@pytest.mark.fast +def test_optimizer_pending(): + X_suggest, eval_suggest = optimizer.suggest(m_batch=2, **test_config) + X_suggest, eval_suggest = optimizer.suggest(m_batch=1, **test_config, X_pending=X_suggest) + X_suggest, eval_suggest = optimizer.suggest(m_batch=1, **test_config, X_pending=X_suggest, eval_pending=eval_suggest) + + @pytest.mark.parametrize('m_batch', [pytest.param(1, marks=pytest.mark.fast), 3]) @pytest.mark.parametrize('fixed_var', [None, {'Parameter 1': 5}]) def test_optimizer_suggest(m_batch, fixed_var): @@ -85,10 +94,16 @@ def test_optimizer_suggest(m_batch, fixed_var): df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) -test_aqs = ['NEHVI', {'NEHVI': {'ref_point': [0.1, 0.1]}}, - 'EHVI', {'EHVI': {'ref_point': [0.1, 0.1]}}, +test_aqs = ['NEHVI', + {'NEHVI': {'ref_point': [0.1, 0.1]}}, + 'EHVI', + {'EHVI': {'ref_point': [0.1, 0.1]}}, + 'NParEGO', {'NParEGO': {'scalarization_weights': [5, 1]}}, - 'Mean', 'SF', 'RS'] + 'Mean', + 'SF', + 'RS', + ] @pytest.mark.parametrize('aq', test_aqs) @@ -97,75 +112,11 @@ def test_optimizer_aqs(aq): df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) -utopian = Utopian_Distance(utopian=[10, 10], targets=target) - -test_scalars = [Scalar_WeightedSum(weights=[0.5, 0.5]), - Objective_Sequence([utopian, Scalar_WeightedSum(weights=[0.5, 0.5])]), - Scalar_WeightedNorm(weights=[0.5, 0.5]), - Objective_Sequence([utopian, Scalar_WeightedNorm(weights=[0.5, 0.5], neg=True)]), - Scalar_Chebyshev(weights=[0.5, 0.5]), - Scalar_Chebyshev(weights=[0.5, 0.5], augment=False), - Objective_Sequence([utopian, Scalar_Chebyshev(weights=[0.5, 0.5])])] - - -@pytest.mark.parametrize('scalar', test_scalars) -def test_optimizer_scalar(scalar): - X_suggest, eval_suggest = optimizer.suggest(m_batch=2, objective=scalar, **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - @pytest.mark.fast def test_optimizer_maximize(): X_suggest, eval_suggest = optimizer.maximize(**test_config) df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) -test_ineq = [[InConstraint_Generic(base_X_space, indices=[0, 1], coeff=[1, 1], rhs=5)]] -test_nleq = [[InConstraint_ConstantDim(base_X_space, dim=0, tol=0.1)]] -test_out = [[OutConstraint_Blank(target)]] - - -@pytest.mark.parametrize('ineq_constraints', test_ineq) -def test_ineq_constraints(ineq_constraints): - X_suggest, eval_suggest = optimizer.suggest(ineq_constraints=ineq_constraints, - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.parametrize('nleq_constraints', test_nleq) -def test_nleq_constraints(nleq_constraints): - X_suggest, eval_suggest = optimizer.suggest(nleq_constraints=nleq_constraints, - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.parametrize('out_constraints', test_out) -def test_out_constraints(out_constraints): - X_suggest, eval_suggest = optimizer.suggest(out_constraints=out_constraints, - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.slow -def test_combo_constraints(): - X_suggest, eval_suggest = optimizer.suggest(ineq_constraints=test_ineq[0], - nleq_constraints=test_nleq[0], - out_constraints=test_out[0], - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.parametrize('m_batch', [pytest.param(1, marks=pytest.mark.fast), 2, pytest.param(5, marks=pytest.mark.slow)]) -@pytest.mark.parametrize('obj', [Identity_Objective(mo=True), - Scalar_WeightedSum(weights=[0.5, 0.5]), - Utopian_Distance([10, 10], target), - Bounded_Target([(0.8, 1.0), None], target), - Bounded_Target([(0.8, 1.0), (0.8, 1.0)], target), - Objective_Sequence([Identity_Objective(mo=True), Index_Objective()])]) -def test_objective(m_batch, obj): - X_suggest, eval_suggest = optimizer.suggest(m_batch=m_batch, objective=obj, **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - if __name__ == '__main__': pytest.main([__file__, '-m', 'not slow']) diff --git a/obsidian/tests/test_optimizer_SOO.py b/obsidian/tests/test_optimizer_SOO.py index b76a17c..69afa27 100644 --- a/obsidian/tests/test_optimizer_SOO.py +++ b/obsidian/tests/test_optimizer_SOO.py @@ -1,18 +1,19 @@ +"""PyTests for obsidian.optimizer under single-output usage""" + from obsidian.tests.param_configs import X_sp_default, X_sp_cont_small, X_sp_cat_small from obsidian.parameters import Target from obsidian.experiment import ExpDesigner, Simulator -from obsidian.experiment.benchmark import shifted_parab from obsidian.optimizer import BayesianOptimizer -from obsidian.constraints import OutConstraint_Blank, InConstraint_Generic, InConstraint_ConstantDim, OutConstraint_L1 -from obsidian.objectives import Identity_Objective, Feature_Objective, Objective_Sequence, Utopian_Distance, Bounded_Target -from obsidian.tests.utils import approx_equal +from obsidian.experiment.benchmark import shifted_parab +from obsidian.tests.utils import approx_equal, equal_state_dicts import pandas as pd import numpy as np import pytest +# Test a variety of preset parameter spaces @pytest.fixture(params=[X_sp_default, X_sp_cont_small, X_sp_cat_small]) def X_space(request): return request.param @@ -21,7 +22,7 @@ def X_space(request): @pytest.fixture() def Z0(X_space): designer = ExpDesigner(X_space, seed=1) - X0 = designer.initialize(m_initial=8, method='LHS') + X0 = designer.initialize(m_initial=len(X_space)*2, method='LHS') simulator = Simulator(X_space, shifted_parab, eps=0.05) y0 = simulator.simulate(X0) Z0 = pd.concat([X0, y0], axis=1) @@ -31,15 +32,25 @@ def Z0(X_space): @pytest.mark.fast @pytest.mark.parametrize('f_transform', ['Standard', 'Logit_MinMax', 'Logit_Percentage', 'Identity']) def test_f_transform(X_space, Z0, f_transform): + # Test all of the transforms with fitting at min/max optimizer = BayesianOptimizer(X_space, surrogate='GP', seed=0, verbose=0) target = Target(name='Response', f_transform=f_transform, aim='max') + target.__repr__() optimizer.fit(Z0, target=target) - + + # Verify equivalence of f and f_inv y_train = optimizer.y_train f_train = target.transform_f(y_train) f_train_inv = target.transform_f(f_train, inverse=True) - target.__repr__() + assert approx_equal(y_train.values.flatten(), f_train_inv.values.flatten()) + target = Target(name='Response', f_transform=f_transform, aim='min') + optimizer.fit(Z0, target=target) + + # Verify equivalence of f and f_inv + y_train = optimizer.y_train + f_train = target.transform_f(y_train) + f_train_inv = target.transform_f(f_train, inverse=True) assert approx_equal(y_train.values.flatten(), f_train_inv.values.flatten()) @@ -52,7 +63,6 @@ def test_f_transform(X_space, Z0, f_transform): pytest.param('DKL', marks=pytest.mark.slow), 'DNN']) def test_optimizer_fit(X_space, surrogate, Z0, serial_test=True): - optimizer = BayesianOptimizer(X_space, surrogate=surrogate, seed=0, verbose=0) if surrogate == 'GPflat' and not X_space.X_cont: @@ -63,8 +73,10 @@ def test_optimizer_fit(X_space, surrogate, Z0, serial_test=True): optimizer.fit(Z0, target=target) if serial_test: - save = optimizer.save_state() - optimizer_2 = BayesianOptimizer.load_state(save) + obj_dict = optimizer.save_state() + optimizer_2 = BayesianOptimizer.load_state(obj_dict) + obj_dict2 = optimizer_2.save_state() + assert equal_state_dicts(obj_dict, obj_dict2) optimizer_2.__repr__() y_pred = optimizer.predict(optimizer.X_train) y_pred_2 = optimizer_2.predict(optimizer.X_train) @@ -82,7 +94,8 @@ def test_optimizer_fit(X_space, surrogate, Z0, serial_test=True): Z0_base = pd.concat([X0, y0], axis=1) optimizer.fit(Z0_base, target=target) -test_config = {'optim_samples': 4, 'optim_restarts': 2} +# Run very short optimizations for testing +test_config = {'optim_samples': 2, 'optim_restarts': 2} def test_fit_nan(): @@ -108,14 +121,25 @@ def test_optimizer_suggest(m_batch, fixed_var): df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) -test_aqs = ['NEI', 'EI', {'EI': {'inflate': 0.05}}, {'EI': {'inflate': -0.05}}, - 'PI', 'UCB', {'UCB': {'beta': 2}}, {'UCB': {'beta': 0}}, +test_aqs = ['NEI', + 'EI', + {'EI': {'inflate': 0.05}}, + {'EI': {'inflate': -0.05}}, + 'PI', + {'PI': {'inflate': 0.05}}, + {'PI': {'inflate': -0.05}}, + 'UCB', + {'UCB': {'beta': 2}}, + {'UCB': {'beta': 0}}, 'NIPV', - 'SF', 'RS', 'Mean', 'SR'] + 'SR', + 'Mean', + 'SF', + ] @pytest.mark.parametrize('aq', test_aqs) -def test_optimizer_aqs_SOO(aq): +def test_optimizer_aqs(aq): X_suggest, eval_suggest = optimizer.suggest(m_batch=2, acquisition=[aq], **test_config) df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) @@ -126,52 +150,5 @@ def test_optimizer_maximize(): df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) -test_ineq = [[InConstraint_Generic(base_X_space, indices=[0, 1], coeff=[1, 1], rhs=5)]] -test_nleq = [[InConstraint_ConstantDim(base_X_space, dim=0, tol=0.1)]] -test_out = [[OutConstraint_Blank(target)], [OutConstraint_L1(target, offset=1)]] - - -@pytest.mark.parametrize('ineq_constraints', test_ineq) -def test_ineq_constraints(ineq_constraints): - X_suggest, eval_suggest = optimizer.suggest(ineq_constraints=ineq_constraints, - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.parametrize('nleq_constraints', test_nleq) -def test_nleq_constraints(nleq_constraints): - with pytest.raises(Exception): - X_suggest, eval_suggest = optimizer.suggest(nleq_constraints=nleq_constraints, - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.parametrize('out_constraints', test_out) -def test_out_constraints(out_constraints): - X_suggest, eval_suggest = optimizer.suggest(out_constraints=out_constraints, - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.slow -def test_combo_constraints(): - X_suggest, eval_suggest = optimizer.suggest(ineq_constraints=test_ineq[0], - nleq_constraints=None, - out_constraints=test_out[0], - **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - -@pytest.mark.parametrize('m_batch', [pytest.param(1, marks=pytest.mark.fast), 2, pytest.param(5, marks=pytest.mark.slow)]) -@pytest.mark.parametrize('obj', [Identity_Objective(), - Feature_Objective(X_space=base_X_space, indices=[0, 1], coeff=[1, 1]), - Utopian_Distance([10], target), - Bounded_Target([(0.8, 1.0)], target), - Objective_Sequence([Identity_Objective(), Identity_Objective()])]) -def test_objective(m_batch, obj): - X_suggest, eval_suggest = optimizer.suggest(m_batch=m_batch, objective=obj, **test_config) - df_suggest = pd.concat([X_suggest, eval_suggest], axis=1) - - if __name__ == '__main__': pytest.main([__file__, '-m', 'not slow']) diff --git a/obsidian/tests/test_parameters.py b/obsidian/tests/test_parameters.py index df77f76..f38af40 100644 --- a/obsidian/tests/test_parameters.py +++ b/obsidian/tests/test_parameters.py @@ -1,3 +1,5 @@ +"""PyTests for obsidian.parameters""" + from obsidian.tests.param_configs import test_X_space from obsidian.experiment import ExpDesigner from obsidian.parameters import ( @@ -15,14 +17,16 @@ ) from obsidian.exceptions import UnsupportedError, UnfitError +from obsidian.tests.utils import equal_state_dicts import numpy as np import pandas as pd -import torch from pandas.testing import assert_frame_equal +import torch import pytest +# Iterate over several preset parameter spaces @pytest.fixture(params=test_X_space) def X_space(request): return request.param @@ -37,12 +41,13 @@ def X0(X_space): @pytest.mark.fast def test_param_loading(X_space): - X_space_dict = X_space.save_state() - X_space_new = ParamSpace.load_state(X_space_dict) - for param in X_space_new: + obj_dict = X_space.save_state() + X_space2 = ParamSpace.load_state(obj_dict) + for param in X_space2: param.__repr__() - X_space_new.__repr__() - assert X_space_dict == X_space_new.save_state(), 'Error during serialization of parameter space' + X_space2.__repr__() + obj_dict2 = X_space2.save_state() + assert equal_state_dicts(obj_dict, obj_dict2), 'Error during serialization' @pytest.mark.fast @@ -69,13 +74,11 @@ def test_param_encoding(X0, X_space): @pytest.mark.fast def test_param_transform_mapping(X_space): - print(X_space) - print(X_space.t_map) - print(X_space.tinv_map) assert len(X_space.t_map) == X_space.n_dim assert len(X_space.tinv_map) == X_space.n_tdim +# Set up a sampling of parameter types for testing encoding/decoding test_params = [ Param_Continuous('Parameter 1', 0, 10), Param_Observational('Parameter 2', 0, 10), @@ -85,18 +88,23 @@ def test_param_transform_mapping(X_space): Task('Parameter 6', ['A', 'B', 'C', 'D']), ] +# Set up a numbe of different data types for testing encoding/decoding test_type = [lambda x: list(x), lambda x: np.array(x), - lambda x: list(x)[0][-2]] + lambda x: list(x)[0][-2], # Single value + ] @pytest.mark.fast -@pytest.mark.parametrize('param, check_type', zip(test_params, test_type)) -def test_param_encoding_types(param, check_type): +@pytest.mark.parametrize('param, type_i', zip(test_params, test_type)) +def test_param_encoding_types(param, type_i): + + # Set up a variety of value types to test cont_vals = [0, 1, 2, 3, 4] cat_vals = ['A', 'B', 'C', 'D'] num_disc_vals = [-2, -1, 1, 2] + # Make sure that 2D arrays also work! if isinstance(param, Param_Continuous): d_2 = [cont_vals] * 3 elif isinstance(param, Param_Discrete) and not isinstance(param, Param_Discrete_Numeric): @@ -104,13 +112,17 @@ def test_param_encoding_types(param, check_type): elif isinstance(param, Param_Discrete_Numeric): d_2 = [num_disc_vals] * 3 - X = check_type(d_2) + X = type_i(d_2) + + # Unit map and demap X_u = param.unit_map(X) X_u_inv = param.unit_demap(X_u) + # Encode and decode X_t = param.encode(X) X_t_inv = param.decode(X_t) + # Check equivalence based on type if isinstance(X, np.ndarray): assert np.all(X_u_inv == X) if not isinstance(param, Param_Categorical): # Categorical params don't encode to the same shape @@ -131,32 +143,40 @@ def test_param_encoding_types(param, check_type): @pytest.mark.fast def test_numeric_param_validation(): + # Strings for numeric with pytest.raises(TypeError): param = Param_Continuous('test', min=string, max=string) + # Value outside of range with pytest.raises(ValueError): param = Param_Continuous('test', min=1, max=0) param._validate_value(2) + # Strings for numeric with pytest.raises(TypeError): Param_Observational('test', min=string, max=string) @pytest.mark.fast def test_discrete_param_validation(): + # Numeric for categoroical with pytest.raises(TypeError): param = Param_Categorical('test', categories=numeric_list) + # Value not in categories with pytest.raises(ValueError): param = Param_Categorical('test', categories=string_list) param._validate_value('E') + # Numeric for ordinal with pytest.raises(TypeError): param = Param_Ordinal('test', categories=numeric_list) + # Strings for discrete numeric with pytest.raises(TypeError): param = Param_Discrete_Numeric('test', categories=string_list) + # Value outside of range with pytest.raises(ValueError): param = Param_Discrete_Numeric('test', categories=numeric_list) param._validate_value(5) @@ -164,13 +184,16 @@ def test_discrete_param_validation(): @pytest.mark.fast def test_paramspace_validation(): + # Overlapping namespace with pytest.raises(ValueError): X_space = ParamSpace([test_params[0], test_params[0]]) + # Misuse of categorical separator cat_sep_param = Param_Continuous('Parameter^1', 0, 10) with pytest.raises(ValueError): X_space = ParamSpace([test_params[0], cat_sep_param]) - + + # >1 Task with pytest.raises(UnsupportedError): X_space = ParamSpace([Task('Parameter X', ['A', 'B', 'C', 'D']), Task('Parameter Y', ['A', 'B', 'C', 'D'])]) @@ -178,6 +201,8 @@ def test_paramspace_validation(): test_data = pd.DataFrame(np.random.uniform(0, 1, (10, 2)), columns=['Parameter X', 'Parameter Z']) X_space = ParamSpace([Param_Continuous('Parameter X', min=0, max=1), Param_Continuous('Parameter Y', min=0, max=1)]) + + # Missing X names with pytest.raises(KeyError): test_encoded = X_space.encode(test_data) @@ -185,27 +210,36 @@ def test_paramspace_validation(): @pytest.mark.fast def test_target_validation(): + # Invalid aim with pytest.raises(ValueError): Target('Response1', aim='maximize') + # Invalid f_transform with pytest.raises(KeyError): Target('Response1', f_transform='quadratic') test_response = torch.rand(10) + + # Transform before fit with pytest.raises(UnfitError): Target('Response1').transform_f(test_response) + # Transform non-arraylike with pytest.raises(TypeError): Target('Response1').transform_f('ABC') + # Transform non-numerical arraylike with pytest.raises(TypeError): Target('Response1').transform_f(['A', 'B', 'C']) + # Transform before fit with pytest.raises(UnfitError): transform_func = Standard_Scaler() transform_func.forward(test_response) test_neg_response = -0.5 - torch.rand(10) + + # Values outside of logit range, refitting with pytest.warns(UserWarning): transform_func = Logit_Scaler(range_response=100) transform_func.forward(test_neg_response, fit=False) diff --git a/obsidian/tests/test_plotting.py b/obsidian/tests/test_plotting.py index 9b97c49..bc2cab5 100644 --- a/obsidian/tests/test_plotting.py +++ b/obsidian/tests/test_plotting.py @@ -1,3 +1,5 @@ +"""PyTests for obsidian.plotting""" + from obsidian import Campaign from obsidian.plotting import ( parity_plot, @@ -13,6 +15,7 @@ from obsidian.tests.utils import DEFAULT_MOO_PATH import json +# Avoid using TkAgg which causes Tcl issues during testing import matplotlib matplotlib.use('inline') diff --git a/obsidian/tests/utils.py b/obsidian/tests/utils.py index e04317e..6154f93 100644 --- a/obsidian/tests/utils.py +++ b/obsidian/tests/utils.py @@ -1,4 +1,4 @@ -"""Utility functions for pytest tests""" +"""Utility functions for PyTest unit testing""" from obsidian.tests.param_configs import X_sp_default, X_sp_cont_ndims @@ -11,12 +11,46 @@ from typing import Callable import pandas as pd +import numpy as np import json +# Default campaigns for testing without having to re-run optimization DEFAULT_MOO_PATH = 'obsidian/tests/default_campaign_MOO.json' DEFAULT_SOO_PATH = 'obsidian/tests/default_campaign_SOO.json' +def equal_state_dicts(e1: dict | str | float | int, + e2: dict | str | float | int) -> bool: + """ + Recursively compare two dictionaries, and allow for floating-point error + """ + # If the values are equal, skip + if not e1 == e2: + # First, make sure we are comparing the same type + assert type(e1) == type(e2), f'Type mismatch at {e1} != {e2}' + + # If they are dictionaries, compare elements recursively + if isinstance(e1, dict): + assert e1.keys() == e2.keys(), f'Keys mismatch at {e1.keys()} != {e2.keys()}' + for k, v in e1.items(): + equal_state_dicts(v, e2[k]) + + # If they are lists, compare elements recursively + elif isinstance(e1, list): + assert len(e1) == len(e2), f'Length mismatch at {len(e1)} != {len(e2)}' + for e1_i, e2_i in zip(e1, e2): + equal_state_dicts(e1_i, e2_i) + + # Otherwise, if it is numerical, check for floating-point error + elif isinstance(e1, (float, int)): + if not (np.isnan(e1) and np.isnan(e2)): + assert (e1-e2)/e1 < 1e-6, f'{e1} != {e2}' + else: + raise ValueError(f'{e1} != {e2}') + + return True + + def approx_equal(x1: ArrayLike | float | int, x2: ArrayLike | float | int, tol: float = 1e-6): diff --git a/pyproject.toml b/pyproject.toml index 84d3e00..f60d9f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -62,12 +62,11 @@ sphinx = { version = "^7.3.7", optional = true} myst-parser = { version = "^3.0.1", optional = true} pydata-sphinx-theme = { version = "^0.15.4", optional = true} linkify-it-py = { version = "^2.0.3", optional = true} -tcl = { version = "^0.2", optional = true} [tool.poetry.extras] app = ["flask", "dash", "dash-daq", "dash-bootstrap-components"] -dev = ["pytest", "xlrd", "ipykernel", "jupyterlab", "flake8", "pytest-cov", "tcl"] +dev = ["pytest", "xlrd", "ipykernel", "jupyterlab", "flake8", "pytest-cov"] docs = ["sphinx", "myst-parser", "pydata-sphinx-theme", "linkify-it-py"]