From 3ab49f00879eabfb100ee0aaca32fdddddfc85cb Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Fri, 9 Aug 2024 16:50:16 -0400 Subject: [PATCH 1/3] Added default_MOO and default_SOO saves for testing --- obsidian/tests/default_campaign.json | 1 - obsidian/tests/default_campaign_MOO.json | 1 + obsidian/tests/default_campaign_SOO.json | 1 + obsidian/tests/test_campaign.py | 9 ++--- obsidian/tests/test_plotting.py | 4 +-- obsidian/tests/utils.py | 42 +++++++++++++++++------- 6 files changed, 40 insertions(+), 18 deletions(-) delete mode 100644 obsidian/tests/default_campaign.json create mode 100644 obsidian/tests/default_campaign_MOO.json create mode 100644 obsidian/tests/default_campaign_SOO.json diff --git a/obsidian/tests/default_campaign.json b/obsidian/tests/default_campaign.json deleted file mode 100644 index 7ed88f4..0000000 --- a/obsidian/tests/default_campaign.json +++ /dev/null @@ -1 +0,0 @@ -{"X_space": {"Parameter 1": {"state": {"name": "Parameter 1", "min": 0, "max": 10}, "class": "Param_Continuous"}, "Parameter 2": {"state": {"name": "Parameter 2", "min": -20, "max": 0}, "class": "Param_Continuous"}, "Parameter 4": {"state": {"name": "Parameter 4", "min": 0, "max": 24, "design_point": 24}, "class": "Param_Observational"}, "Parameter 7": {"state": {"name": "Parameter 7", "categories": ["A", "B", "C", "D"]}, "class": "Param_Categorical"}}, "optimizer": {"opt_attrs": {"X_train": {"Parameter 1": {"0": 3.75, "1": 8.25, "2": 4.25, "3": 9.25, "4": 1.75, "5": 0.75, "6": 6.25, "7": 2.75, "8": 7.75, "9": 9.75, "10": 8.75, "11": 7.25, "12": 5.75, "13": 5.25, "14": 0.25, "15": 3.25, "16": 4.75, "17": 1.25, "18": 2.25, "19": 6.75}, "Parameter 2": {"0": -11.5, "1": -16.5, "2": -0.5, "3": -13.5, "4": -14.5, "5": -2.5, "6": -1.5, "7": -6.5, "8": -15.5, "9": -7.5, "10": -3.5, "11": -4.5, "12": -19.5, "13": -10.5, "14": -8.5, "15": -12.5, "16": -17.5, "17": -18.5, "18": -5.5, "19": -9.5}, "Parameter 4": {"0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0}, "Parameter 7": {"0": "B", "1": "D", "2": "D", "3": "A", "4": "D", "5": "C", "6": "A", "7": "A", "8": "B", "9": "C", "10": "C", "11": "A", "12": "C", "13": "B", "14": "B", "15": "C", "16": "B", "17": "A", "18": "D", "19": "D"}}, "y_train": {"Response": {"0": 88.36007134508922, "1": 48.010521556839024, "2": 37.018588781657044, "3": 53.72724271417155, "4": 70.55695005978943, "5": 54.81260501294057, "6": 45.55499083534941, "7": 73.25307202957657, "8": 68.8148927558007, "9": 40.2021383094655, "10": 37.672115767998356, "11": 50.34605104443083, "12": 73.98927764290242, "13": 79.95149881336371, "14": 80.75502473506768, "15": 83.84083550458679, "16": 88.36007134508922, "17": 90.3707077873196, "18": 53.86172885859266, "19": 50.9793264077555}}, "y_names": ["Response"], "n_response": 1, "seed": null}, "X_space": {"Parameter 1": {"state": {"name": "Parameter 1", "min": 0, "max": 10}, "class": "Param_Continuous"}, "Parameter 2": {"state": {"name": "Parameter 2", "min": -20, "max": 0}, "class": "Param_Continuous"}, "Parameter 4": {"state": {"name": "Parameter 4", "min": 0, "max": 24, "design_point": 24}, "class": "Param_Observational"}, "Parameter 7": {"state": {"name": "Parameter 7", "categories": ["A", "B", "C", "D"]}, "class": "Param_Categorical"}}, "surrogate_spec": [{"GP": {}}], "target": [{"init_attrs": {"name": "Response", "aim": "max", "f_transform": "Standard"}, "f_raw": [[88.36007134508922], [48.010521556839024], [37.018588781657044], [53.72724271417155], [70.55695005978943], [54.81260501294057], [45.55499083534941], [73.25307202957657], [68.8148927558007], [40.2021383094655], [37.672115767998356], [50.34605104443083], [73.98927764290242], [79.95149881336371], [80.75502473506768], [83.84083550458679], [88.36007134508922], [90.3707077873196], [53.86172885859266], [50.9793264077555]]}], "model_states": [{"model_attrs": {"model_type": "GP", "seed": null, "hps": {}, "is_fit": true, "cat_dims": [3, 4, 5, 6], "task_feature": null, "X_order": ["Parameter 1", "Parameter 2", "Parameter 4", "Parameter 7^A", "Parameter 7^B", "Parameter 7^C", "Parameter 7^D"], "y_name": "Response Trans"}, "train_X": {"Parameter 1": {"0": 0.375, "1": 0.825, "2": 0.425, "3": 0.925, "4": 0.175, "5": 0.075, "6": 0.625, "7": 0.275, "8": 0.775, "9": 0.975, "10": 0.875, "11": 0.725, "12": 0.575, "13": 0.525, "14": 0.025, "15": 0.325, "16": 0.475, "17": 0.125, "18": 0.225, "19": 0.675}, "Parameter 2": {"0": 0.425, "1": 0.175, "2": 0.975, "3": 0.325, "4": 0.275, "5": 0.875, "6": 0.925, "7": 0.675, "8": 0.225, "9": 0.625, "10": 0.825, "11": 0.775, "12": 0.025, "13": 0.475, "14": 0.575, "15": 0.375, "16": 0.125, "17": 0.075, "18": 0.725, "19": 0.525}, "Parameter 4": {"0": 0.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0, "10": 0.0, "11": 0.0, "12": 0.0, "13": 0.0, "14": 0.0, "15": 0.0, "16": 0.0, "17": 0.0, "18": 0.0, "19": 0.0}, "Parameter 7^A": {"0": 0.0, "1": 0.0, "2": 0.0, "3": 1.0, "4": 0.0, "5": 0.0, "6": 1.0, "7": 1.0, "8": 0.0, "9": 0.0, "10": 0.0, "11": 1.0, "12": 0.0, "13": 0.0, "14": 0.0, "15": 0.0, "16": 0.0, "17": 1.0, "18": 0.0, "19": 0.0}, "Parameter 7^B": {"0": 1.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 1.0, "9": 0.0, "10": 0.0, "11": 0.0, "12": 0.0, "13": 1.0, "14": 1.0, "15": 0.0, "16": 1.0, "17": 0.0, "18": 0.0, "19": 0.0}, "Parameter 7^C": {"0": 0.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 1.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 1.0, "10": 1.0, "11": 0.0, "12": 1.0, "13": 0.0, "14": 0.0, "15": 1.0, "16": 0.0, "17": 0.0, "18": 0.0, "19": 0.0}, "Parameter 7^D": {"0": 0.0, "1": 1.0, "2": 1.0, "3": 0.0, "4": 1.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0, "10": 0.0, "11": 0.0, "12": 0.0, "13": 0.0, "14": 0.0, "15": 0.0, "16": 0.0, "17": 0.0, "18": 1.0, "19": 1.0}}, "train_Y": {"0": 1.365104829585948, "1": -0.8525034038050997, "2": -1.4566192056181526, "3": -0.5383128372924322, "4": 0.38664661754817886, "5": -0.47866140666697066, "6": -0.9874591882908472, "7": 0.5348252790153639, "8": 0.29090327863515375, "9": -1.2816515620799824, "10": -1.4207014114107992, "11": -0.7241428781439343, "12": 0.57528708436754, "13": 0.9029703132090072, "14": 0.9471320376660354, "15": 1.1167279663711698, "16": 1.365104829585948, "17": 1.4756092578148987, "18": -0.5309214888706929, "19": -0.6893381116203324}, "torch_params": {"likelihood.noise_covar.raw_noise": [9.999999974752427e-07], "likelihood.noise_covar.noise_prior.concentration": 0.8999999761581421, "likelihood.noise_covar.noise_prior.rate": 10.0, "likelihood.noise_covar.raw_noise_constraint.lower_bound": 9.999999974752427e-07, "likelihood.noise_covar.raw_noise_constraint.upper_bound": "inf", "mean_module.raw_constant": -2.8466494943370795, "covar_module.kernels.0.raw_outputscale": 4.767406485762148, "covar_module.kernels.0.base_kernel.kernels.0.raw_lengthscale": [[1.2328998016067925, 1.6273430353555187, 0.0]], "covar_module.kernels.0.base_kernel.kernels.0.active_dims": [0, 1, 2], "covar_module.kernels.0.base_kernel.kernels.0.raw_lengthscale_constraint.lower_bound": 9.999999747378752e-05, "covar_module.kernels.0.base_kernel.kernels.0.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.0.base_kernel.kernels.1.raw_outputscale": -4.712553674854026, "covar_module.kernels.0.base_kernel.kernels.1.active_dims": [3, 4, 5, 6], "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.raw_lengthscale": [[4.587132649586074, -9.685067920420769, 5.9754357044117, 6.0312096744800225]], "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.active_dims": [3, 4, 5, 6], "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.raw_lengthscale_constraint.lower_bound": 9.999999974752427e-07, "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.0.base_kernel.kernels.1.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.kernels.0.base_kernel.kernels.1.raw_outputscale_constraint.upper_bound": "inf", "covar_module.kernels.0.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.kernels.0.raw_outputscale_constraint.upper_bound": "inf", "covar_module.kernels.1.raw_outputscale": -1.3093327207595307, "covar_module.kernels.1.base_kernel.kernels.0.raw_lengthscale": [[2.083395723794677, 1.6581398762077824, 0.0]], "covar_module.kernels.1.base_kernel.kernels.0.active_dims": [0, 1, 2], "covar_module.kernels.1.base_kernel.kernels.0.raw_lengthscale_constraint.lower_bound": 9.999999747378752e-05, "covar_module.kernels.1.base_kernel.kernels.0.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.1.base_kernel.kernels.1.raw_lengthscale": [[22.87760531869039, 20.19337038131295, 10.021134653093318, -11.563254173355265]], "covar_module.kernels.1.base_kernel.kernels.1.active_dims": [3, 4, 5, 6], "covar_module.kernels.1.base_kernel.kernels.1.raw_lengthscale_constraint.lower_bound": 9.999999974752427e-07, "covar_module.kernels.1.base_kernel.kernels.1.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.1.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.kernels.1.raw_outputscale_constraint.upper_bound": "inf"}}]}, "data": {"Parameter 1": {"0": 3.75, "1": 8.25, "2": 4.25, "3": 9.25, "4": 1.75, "5": 0.75, "6": 6.25, "7": 2.75, "8": 7.75, "9": 9.75, "10": 8.75, "11": 7.25, "12": 5.75, "13": 5.25, "14": 0.25, "15": 3.25, "16": 4.75, "17": 1.25, "18": 2.25, "19": 6.75}, "Parameter 2": {"0": -11.5, "1": -16.5, "2": -0.5, "3": -13.5, "4": -14.5, "5": -2.5, "6": -1.5, "7": -6.5, "8": -15.5, "9": -7.5, "10": -3.5, "11": -4.5, "12": -19.5, "13": -10.5, "14": -8.5, "15": -12.5, "16": -17.5, "17": -18.5, "18": -5.5, "19": -9.5}, "Parameter 4": {"0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0}, "Parameter 7": {"0": "B", "1": "D", "2": "D", "3": "A", "4": "D", "5": "C", "6": "A", "7": "A", "8": "B", "9": "C", "10": "C", "11": "A", "12": "C", "13": "B", "14": "B", "15": "C", "16": "B", "17": "A", "18": "D", "19": "D"}, "Response": {"0": 88.36007134508922, "1": 48.010521556839024, "2": 37.018588781657044, "3": 53.72724271417155, "4": 70.55695005978943, "5": 54.81260501294057, "6": 45.55499083534941, "7": 73.25307202957657, "8": 68.8148927558007, "9": 40.2021383094655, "10": 37.672115767998356, "11": 50.34605104443083, "12": 73.98927764290242, "13": 79.95149881336371, "14": 80.75502473506768, "15": 83.84083550458679, "16": 88.36007134508922, "17": 90.3707077873196, "18": 53.86172885859266, "19": 50.9793264077555}, "Iteration": {"0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 0, "7": 0, "8": 0, "9": 0, "10": 0, "11": 0, "12": 0, "13": 0, "14": 0, "15": 0, "16": 0, "17": 0, "18": 0, "19": 0}}, "target": [{"init_attrs": {"name": "Response", "aim": "max", "f_transform": "Standard"}, "f_raw": [[88.36007134508922], [48.010521556839024], [37.018588781657044], [53.72724271417155], [70.55695005978943], [54.81260501294057], [45.55499083534941], [73.25307202957657], [68.8148927558007], [40.2021383094655], [37.672115767998356], [50.34605104443083], [73.98927764290242], [79.95149881336371], [80.75502473506768], [83.84083550458679], [88.36007134508922], [90.3707077873196], [53.86172885859266], [50.9793264077555]]}], "seed": null} \ No newline at end of file diff --git a/obsidian/tests/default_campaign_MOO.json b/obsidian/tests/default_campaign_MOO.json new file mode 100644 index 0000000..fc09166 --- /dev/null +++ b/obsidian/tests/default_campaign_MOO.json @@ -0,0 +1 @@ +{"X_space": {"Parameter 1": {"state": {"name": "Parameter 1", "min": 0, "max": 10}, "class": "Param_Continuous"}, "Parameter 2": {"state": {"name": "Parameter 2", "min": 0, "max": 10}, "class": "Param_Continuous"}}, "optimizer": {"opt_attrs": {"X_train": {"Parameter 1": {"0": 2.5, "1": 0.8333333333333333, "2": 7.5, "3": 5.833333333333334, "4": 9.166666666666666, "5": 4.166666666666667, "6": 0.0, "7": 9.561359020262923, "8": 0.0, "9": 0.0}, "Parameter 2": {"0": 2.5, "1": 5.833333333333334, "2": 7.5, "3": 0.8333333333333333, "4": 4.166666666666667, "5": 9.166666666666666, "6": 8.44723404661871, "7": 9.862823941909266, "8": 0.0, "9": 10.0}}, "y_train": {"Response 1": {"0": 0.013856171114995977, "1": 0.03432975568022848, "2": 0.4189098892895902, "3": 0.003431693206423828, "4": 0.1643399024276901, "5": 0.18170428213463904, "6": 0.0, "7": 0.9575375984495041, "8": 0.0, "9": 0.0}, "Response 2": {"0": 0.5352280996541698, "1": 0.7663162272889814, "2": 0.308476724543929, "3": 0.37918523689244016, "4": 0.1246219695813563, "5": 0.28181351107250613, "6": 0.8534158017742339, "7": 0.06793137056494765, "8": 0.8534158017742339, "9": 0.8534158017742339}}, "y_names": ["Response 1", "Response 2"], "n_response": 2, "seed": null}, "X_space": {"Parameter 1": {"state": {"name": "Parameter 1", "min": 0, "max": 10}, "class": "Param_Continuous"}, "Parameter 2": {"state": {"name": "Parameter 2", "min": 0, "max": 10}, "class": "Param_Continuous"}}, "surrogate_spec": [{"GP": {}}, {"GP": {}}], "target": [{"init_attrs": {"name": "Response 1", "aim": "max", "f_transform": "Standard"}, "f_raw": [[0.013856171114995977], [0.03432975568022848], [0.4189098892895902], [0.003431693206423828], [0.1643399024276901], [0.18170428213463904], [0.0], [0.9575375984495041], [0.0], [0.0]]}, {"init_attrs": {"name": "Response 2", "aim": "max", "f_transform": "Standard"}, "f_raw": [[0.5352280996541698], [0.7663162272889814], [0.308476724543929], [0.37918523689244016], [0.1246219695813563], [0.28181351107250613], [0.8534158017742339], [0.06793137056494765], [0.8534158017742339], [0.8534158017742339]]}], "model_states": [{"model_attrs": {"model_type": "GP", "seed": null, "hps": {}, "is_fit": true, "cat_dims": [], "task_feature": null, "X_order": ["Parameter 1", "Parameter 2"], "y_name": "Response 1 Trans"}, "train_X": {"Parameter 1": {"0": 0.25, "1": 0.08333333333333333, "2": 0.75, "3": 0.5833333333333334, "4": 0.9166666666666666, "5": 0.4166666666666667, "6": 0.0, "7": 0.9561359020262923, "8": 0.0, "9": 0.0}, "Parameter 2": {"0": 0.25, "1": 0.5833333333333334, "2": 0.75, "3": 0.08333333333333333, "4": 0.4166666666666667, "5": 0.9166666666666666, "6": 0.844723404661871, "7": 0.9862823941909266, "8": 0.0, "9": 1.0}}, "train_Y": {"0": -0.535669991872484, "1": -0.4686154775065579, "2": 0.7909506728074768, "3": -0.5698119517940827, "4": -0.042809863203040845, "5": 0.014061469943587476, "6": -0.5810513378760082, "7": 2.5550491552531267, "8": -0.5810513378760082, "9": -0.5810513378760082}, "torch_params": {"likelihood.noise_covar.raw_noise": [0.003398248045429932], "likelihood.noise_covar.noise_prior.concentration": 1.100000023841858, "likelihood.noise_covar.noise_prior.rate": 0.05000000074505806, "likelihood.noise_covar.raw_noise_constraint.lower_bound": 9.999999747378752e-05, "likelihood.noise_covar.raw_noise_constraint.upper_bound": "inf", "mean_module.raw_constant": 0.35957230077639185, "covar_module.raw_outputscale": 1.2891402290445992, "covar_module.base_kernel.raw_lengthscale": [[-0.22930632060853587, -0.1280487015578905]], "covar_module.base_kernel.lengthscale_prior.concentration": 3.0, "covar_module.base_kernel.lengthscale_prior.rate": 6.0, "covar_module.base_kernel.raw_lengthscale_constraint.lower_bound": 0.0, "covar_module.base_kernel.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.outputscale_prior.concentration": 2.0, "covar_module.outputscale_prior.rate": 0.15000000596046448, "covar_module.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.raw_outputscale_constraint.upper_bound": "inf"}}, {"model_attrs": {"model_type": "GP", "seed": null, "hps": {}, "is_fit": true, "cat_dims": [], "task_feature": null, "X_order": ["Parameter 1", "Parameter 2"], "y_name": "Response 2 Trans"}, "train_X": {"Parameter 1": {"0": 0.25, "1": 0.08333333333333333, "2": 0.75, "3": 0.5833333333333334, "4": 0.9166666666666666, "5": 0.4166666666666667, "6": 0.0, "7": 0.9561359020262923, "8": 0.0, "9": 0.0}, "Parameter 2": {"0": 0.25, "1": 0.5833333333333334, "2": 0.75, "3": 0.08333333333333333, "4": 0.4166666666666667, "5": 0.9166666666666666, "6": 0.844723404661871, "7": 0.9862823941909266, "8": 0.0, "9": 1.0}}, "train_Y": {"0": 0.1054446935687377, "1": 0.847300118950883, "2": -0.6224885826237413, "3": -0.3954951232741601, "4": -1.2127120997177576, "5": -0.7080847133049162, "6": 1.1269133234097133, "7": -1.394704263828187, "8": 1.1269133234097133, "9": 1.1269133234097133}, "torch_params": {"likelihood.noise_covar.raw_noise": [0.001348488460520536], "likelihood.noise_covar.noise_prior.concentration": 1.100000023841858, "likelihood.noise_covar.noise_prior.rate": 0.05000000074505806, "likelihood.noise_covar.raw_noise_constraint.lower_bound": 9.999999747378752e-05, "likelihood.noise_covar.raw_noise_constraint.upper_bound": "inf", "mean_module.raw_constant": -0.11554181160629078, "covar_module.raw_outputscale": 0.05017203782837847, "covar_module.base_kernel.raw_lengthscale": [[-1.0586594080313136, 0.5978649542247377]], "covar_module.base_kernel.lengthscale_prior.concentration": 3.0, "covar_module.base_kernel.lengthscale_prior.rate": 6.0, "covar_module.base_kernel.raw_lengthscale_constraint.lower_bound": 0.0, "covar_module.base_kernel.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.outputscale_prior.concentration": 2.0, "covar_module.outputscale_prior.rate": 0.15000000596046448, "covar_module.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.raw_outputscale_constraint.upper_bound": "inf"}}]}, "data": {"Parameter 1": {"0": 2.5, "1": 0.8333333333333333, "2": 7.5, "3": 5.833333333333334, "4": 9.166666666666666, "5": 4.166666666666667, "6": 0.0, "7": 9.561359020262923, "8": 0.0, "9": 0.0, "10": 8.05884810038377, "11": 6.8406804167176585}, "Parameter 2": {"0": 2.5, "1": 5.833333333333334, "2": 7.5, "3": 0.8333333333333333, "4": 4.166666666666667, "5": 9.166666666666666, "6": 8.44723404661871, "7": 9.862823941909266, "8": 0.0, "9": 10.0, "10": 10.0, "11": 10.0}, "Response 1": {"0": 0.013856171114995977, "1": 0.03432975568022848, "2": 0.4189098892895902, "3": 0.003431693206423828, "4": 0.1643399024276901, "5": 0.18170428213463904, "6": 0.0, "7": 0.9575375984495041, "8": 0.0, "9": 0.0, "10": 0.8191994146108325, "11": 0.6499071214288321}, "Response 2": {"0": 0.5352280996541698, "1": 0.7663162272889814, "2": 0.308476724543929, "3": 0.37918523689244016, "4": 0.1246219695813563, "5": 0.28181351107250613, "6": 0.8534158017742339, "7": 0.06793137056494765, "8": 0.8534158017742339, "9": 0.8534158017742339, "10": 0.25782685626481244, "11": 0.35190022724334613}, "Iteration": {"0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 1, "7": 1, "8": 2, "9": 2, "10": 3, "11": 3}, "Response 1 (pred)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.07260440423671881, "7": 0.2998109146830079, "8": 0.13553996363484694, "9": 0.0283643020909618, "10": 0.7895488620349673, "11": 0.5984855239765161}, "Response 1 lb": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": -0.059246292549191615, "7": 0.15146042241994814, "8": -0.13616102654512918, "9": -0.09360519411923673, "10": 0.7119357066023786, "11": 0.49972885756864527}, "Response 1 ub": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.2044551224696099, "7": 0.4481614310769234, "8": 0.40724099801000757, "9": 0.15033381814085398, "10": 0.8671620300921978, "11": 0.697242206448256}, "Response 2 (pred)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.582326398459104, "7": 0.3408065670773579, "8": 0.5919322389003672, "9": 0.8108402926111451, "10": 0.23918866328259236, "11": 0.32933785504134366}, "Response 2 lb": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.3938802797683256, "7": 0.12880245692356157, "8": 0.37133988630877496, "9": 0.7274598657587373, "10": 0.17670696544989506, "11": 0.24887720423658888}, "Response 2 ub": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.7707725478027373, "7": 0.552810711715978, "8": 0.8125246273737563, "9": 0.8942207330263057, "10": 0.30167037127863194, "11": 0.40979851893391717}, "f(Response 1)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": -0.4003359276706779, "7": 1.0322939538830744, "8": -0.2616209786702686, "9": -0.5868146567966284, "10": 2.0048571210574564, "11": 1.3790917937488958}, "f(Response 2)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.8168712025094681, "7": -0.2609086900401194, "8": 0.6239738174840637, "9": 1.3943418515102426, "10": -0.844922009589592, "11": -0.5555187084138095}, "Objective 1": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": -0.4003359276706779, "7": 1.0322939538830744, "8": -0.2616209786702686, "9": -0.5868146567966284, "10": 2.0048571210574564, "11": 1.3790917937488958}, "Objective 2": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.8168712025094681, "7": -0.2609086900401194, "8": 0.6239738174840637, "9": 1.3943418515102426, "10": -0.844922009589592, "11": -0.5555187084138095}, "aq Value": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": -0.7452203824605368, "7": -0.2832722625598487, "8": -0.6039225341357639, "9": -1.4415657461280458, "10": -0.3903933850999888, "11": -0.40909204451189396}, "aq Value (joint)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 0.1673485517948059, "7": 0.1673485517948059, "8": -0.3577695711270721, "9": -0.3577695711270721, "10": 0.015390517435002415, "11": 0.015390517435002415}, "aq Method": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": "NEHVI", "7": "NEHVI", "8": "NEHVI", "9": "NEHVI", "10": "NEHVI", "11": "NEHVI"}, "Expected Hypervolume (joint)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 2.45926678524061, "7": 2.45926678524061, "8": 1.4727612897691176, "9": 1.4727612897691176, "10": 2.1340993059181317, "11": 2.1340993059181317}, "Expected Pareto": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": false, "7": false, "8": true, "9": true, "10": true, "11": false}, "Hypervolume (iter)": {"0": 0.09053399549195458, "1": 0.09053399549195458, "2": 0.09053399549195458, "3": 0.09053399549195458, "4": 0.09053399549195458, "5": 0.09053399549195458, "6": 0.1164843458744553, "7": 0.1164843458744553, "8": 0.1164843458744553, "9": 0.1164843458744553, "10": 0.23092802448987865, "11": 0.23092802448987865}, "Pareto Front": {"0": false, "1": true, "2": false, "3": false, "4": false, "5": false, "6": true, "7": true, "8": false, "9": false, "10": true, "11": true}, "Response 1 (max) (iter)": {"0": 0.4189098892895902, "1": 0.4189098892895902, "2": 0.4189098892895902, "3": 0.4189098892895902, "4": 0.4189098892895902, "5": 0.4189098892895902, "6": 0.9575375984495041, "7": 0.9575375984495041, "8": 0.9575375984495041, "9": 0.9575375984495041, "10": 0.9575375984495041, "11": 0.9575375984495041}, "Response 2 (max) (iter)": {"0": 0.7663162272889814, "1": 0.7663162272889814, "2": 0.7663162272889814, "3": 0.7663162272889814, "4": 0.7663162272889814, "5": 0.7663162272889814, "6": 0.8534158017742339, "7": 0.8534158017742339, "8": 0.8534158017742339, "9": 0.8534158017742339, "10": 0.8534158017742339, "11": 0.8534158017742339}}, "target": [{"init_attrs": {"name": "Response 1", "aim": "max", "f_transform": "Standard"}, "f_raw": [[0.013856171114995977], [0.03432975568022848], [0.4189098892895902], [0.003431693206423828], [0.1643399024276901], [0.18170428213463904], [0.0], [0.9575375984495041], [0.0], [0.0]]}, {"init_attrs": {"name": "Response 2", "aim": "max", "f_transform": "Standard"}, "f_raw": [[0.5352280996541698], [0.7663162272889814], [0.308476724543929], [0.37918523689244016], [0.1246219695813563], [0.28181351107250613], [0.8534158017742339], [0.06793137056494765], [0.8534158017742339], [0.8534158017742339]]}], "seed": null} \ No newline at end of file diff --git a/obsidian/tests/default_campaign_SOO.json b/obsidian/tests/default_campaign_SOO.json new file mode 100644 index 0000000..a4f3e03 --- /dev/null +++ b/obsidian/tests/default_campaign_SOO.json @@ -0,0 +1 @@ +{"X_space": {"Parameter 1": {"state": {"name": "Parameter 1", "min": 0, "max": 10}, "class": "Param_Continuous"}, "Parameter 2": {"state": {"name": "Parameter 2", "min": -20, "max": 0}, "class": "Param_Continuous"}, "Parameter 3": {"state": {"name": "Parameter 3", "min": 5, "max": 5}, "class": "Param_Continuous"}, "Parameter 7": {"state": {"name": "Parameter 7", "categories": ["A", "B", "C", "D"]}, "class": "Param_Categorical"}}, "optimizer": {"opt_attrs": {"X_train": {"Parameter 1": {"0": 5.833333333333334, "1": 7.5, "2": 9.166666666666666, "3": 2.5, "4": 4.166666666666667, "5": 0.8333333333333333, "6": 1.181027361160873, "7": 10.0, "8": 3.5671554309773494, "9": 3.8259819976910956}, "Parameter 2": {"0": -8.333333333333332, "1": -5.0, "2": -15.0, "3": -18.333333333333332, "4": -11.666666666666666, "5": -1.6666666666666679, "6": -1.3765871582395803, "7": -20.0, "8": -11.972662327181533, "9": -12.021715223130977}, "Parameter 3": {"0": 5.0, "1": 5.0, "2": 5.0, "3": 5.0, "4": 5.0, "5": 5.0, "6": 5.0, "7": 5.0, "8": 5.0, "9": 5.0}, "Parameter 7": {"0": "D", "1": "C", "2": "B", "3": "D", "4": "A", "5": "B", "6": "A", "7": "A", "8": "B", "9": "A"}}, "y_train": {"Response 1": {"0": 52.91999776955157, "1": 47.950545897489405, "2": 57.200296096064626, "3": 69.86459941931172, "4": 84.03902691676035, "5": 56.568255091930745, "6": 53.72360713169506, "7": 46.766642700990914, "8": 89.79791912501248, "9": 85.82078349307184}}, "y_names": ["Response 1"], "n_response": 1, "seed": null}, "X_space": {"Parameter 1": {"state": {"name": "Parameter 1", "min": 0, "max": 10}, "class": "Param_Continuous"}, "Parameter 2": {"state": {"name": "Parameter 2", "min": -20, "max": 0}, "class": "Param_Continuous"}, "Parameter 3": {"state": {"name": "Parameter 3", "min": 5, "max": 5}, "class": "Param_Continuous"}, "Parameter 7": {"state": {"name": "Parameter 7", "categories": ["A", "B", "C", "D"]}, "class": "Param_Categorical"}}, "surrogate_spec": [{"GP": {}}], "target": [{"init_attrs": {"name": "Response 1", "aim": "max", "f_transform": "Standard"}, "f_raw": [[52.91999776955157], [47.950545897489405], [57.200296096064626], [69.86459941931172], [84.03902691676035], [56.568255091930745], [53.72360713169506], [46.766642700990914], [89.79791912501248], [85.82078349307184]]}], "model_states": [{"model_attrs": {"model_type": "GP", "seed": null, "hps": {}, "is_fit": true, "cat_dims": [3, 4, 5, 6], "task_feature": null, "X_order": ["Parameter 1", "Parameter 2", "Parameter 3", "Parameter 7^A", "Parameter 7^B", "Parameter 7^C", "Parameter 7^D"], "y_name": "Response 1 Trans"}, "train_X": {"Parameter 1": {"0": 0.5833333333333334, "1": 0.75, "2": 0.9166666666666666, "3": 0.25, "4": 0.4166666666666667, "5": 0.08333333333333333, "6": 0.1181027361160873, "7": 1.0, "8": 0.35671554309773496, "9": 0.3825981997691096}, "Parameter 2": {"0": 0.5833333333333334, "1": 0.75, "2": 0.25, "3": 0.0833333333333334, "4": 0.4166666666666667, "5": 0.9166666666666666, "6": 0.931170642088021, "7": 0.0, "8": 0.4013668836409233, "9": 0.3989142388434511}, "Parameter 3": {"0": 0.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0}, "Parameter 7^A": {"0": 0.0, "1": 0.0, "2": 0.0, "3": 0.0, "4": 1.0, "5": 0.0, "6": 1.0, "7": 1.0, "8": 0.0, "9": 1.0}, "Parameter 7^B": {"0": 0.0, "1": 0.0, "2": 1.0, "3": 0.0, "4": 0.0, "5": 1.0, "6": 0.0, "7": 0.0, "8": 1.0, "9": 0.0}, "Parameter 7^C": {"0": 0.0, "1": 1.0, "2": 0.0, "3": 0.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0}, "Parameter 7^D": {"0": 1.0, "1": 0.0, "2": 0.0, "3": 1.0, "4": 0.0, "5": 0.0, "6": 0.0, "7": 0.0, "8": 0.0, "9": 0.0}}, "train_Y": {"0": -0.6983359161594589, "1": -0.9989245473995874, "2": -0.43943230900326985, "3": 0.32659696334884825, "4": 1.183969540800228, "5": -0.4776627507524526, "6": -0.6497277709480181, "7": -1.0705356325892745, "8": 1.5323092713990505, "9": 1.2917431513039397}, "torch_params": {"likelihood.noise_covar.raw_noise": [9.999999974752427e-07], "likelihood.noise_covar.noise_prior.concentration": 0.8999999761581421, "likelihood.noise_covar.noise_prior.rate": 10.0, "likelihood.noise_covar.raw_noise_constraint.lower_bound": 9.999999974752427e-07, "likelihood.noise_covar.raw_noise_constraint.upper_bound": "inf", "mean_module.raw_constant": -0.2779732334765344, "covar_module.kernels.0.raw_outputscale": -0.09046509840992041, "covar_module.kernels.0.base_kernel.kernels.0.raw_lengthscale": [[-0.7876875598798336, -1.4605040475112987, 0.0]], "covar_module.kernels.0.base_kernel.kernels.0.active_dims": [0, 1, 2], "covar_module.kernels.0.base_kernel.kernels.0.raw_lengthscale_constraint.lower_bound": 9.999999747378752e-05, "covar_module.kernels.0.base_kernel.kernels.0.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.0.base_kernel.kernels.1.raw_outputscale": -3.411109706915788, "covar_module.kernels.0.base_kernel.kernels.1.active_dims": [3, 4, 5, 6], "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.raw_lengthscale": [[2188.6187430747477, -7630.060517320062, 648.5338363819815, 693.3724753872003]], "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.active_dims": [3, 4, 5, 6], "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.raw_lengthscale_constraint.lower_bound": 9.999999974752427e-07, "covar_module.kernels.0.base_kernel.kernels.1.base_kernel.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.0.base_kernel.kernels.1.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.kernels.0.base_kernel.kernels.1.raw_outputscale_constraint.upper_bound": "inf", "covar_module.kernels.0.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.kernels.0.raw_outputscale_constraint.upper_bound": "inf", "covar_module.kernels.1.raw_outputscale": -3786.33741273289, "covar_module.kernels.1.base_kernel.kernels.0.raw_lengthscale": [[384.169413551668, 1250.3450776680072, 0.0]], "covar_module.kernels.1.base_kernel.kernels.0.active_dims": [0, 1, 2], "covar_module.kernels.1.base_kernel.kernels.0.raw_lengthscale_constraint.lower_bound": 9.999999747378752e-05, "covar_module.kernels.1.base_kernel.kernels.0.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.1.base_kernel.kernels.1.raw_lengthscale": [[1082.3676483903282, 615.165514751988, 172.3691505963996, -64.28745326902173]], "covar_module.kernels.1.base_kernel.kernels.1.active_dims": [3, 4, 5, 6], "covar_module.kernels.1.base_kernel.kernels.1.raw_lengthscale_constraint.lower_bound": 9.999999974752427e-07, "covar_module.kernels.1.base_kernel.kernels.1.raw_lengthscale_constraint.upper_bound": "inf", "covar_module.kernels.1.raw_outputscale_constraint.lower_bound": 0.0, "covar_module.kernels.1.raw_outputscale_constraint.upper_bound": "inf"}}]}, "data": {"Parameter 1": {"0": 5.833333333333334, "1": 7.5, "2": 9.166666666666666, "3": 2.5, "4": 4.166666666666667, "5": 0.8333333333333333, "6": 1.181027361160873, "7": 10.0, "8": 3.5671554309773494, "9": 3.8259819976910956, "10": 2.5436867612524203, "11": 1.943079972685201}, "Parameter 2": {"0": -8.333333333333332, "1": -5.0, "2": -15.0, "3": -18.333333333333332, "4": -11.666666666666666, "5": -1.6666666666666679, "6": -1.3765871582395803, "7": -20.0, "8": -11.972662327181533, "9": -12.021715223130977, "10": -12.190951336683044, "11": -11.695168668363625}, "Parameter 3": {"0": 5.0, "1": 5.0, "2": 5.0, "3": 5.0, "4": 5.0, "5": 5.0, "6": 5.0, "7": 5.0, "8": 5.0, "9": 5.0, "10": 5.0, "11": 5.0}, "Parameter 7": {"0": "D", "1": "C", "2": "B", "3": "D", "4": "A", "5": "B", "6": "A", "7": "A", "8": "B", "9": "A", "10": "B", "11": "B"}, "Response 1": {"0": 52.91999776955157, "1": 47.950545897489405, "2": 57.200296096064626, "3": 69.86459941931172, "4": 84.03902691676035, "5": 56.568255091930745, "6": 53.72360713169506, "7": 46.766642700990914, "8": 89.79791912501248, "9": 85.82078349307184, "10": 92.15225029636682, "11": 91.4972443201605}, "Iteration": {"0": 0, "1": 0, "2": 0, "3": 0, "4": 0, "5": 0, "6": 1, "7": 1, "8": 2, "9": 2, "10": 3, "11": 3}, "Response 1 (pred)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 82.87450053515634, "7": 82.73777771636757, "8": 83.20085855107834, "9": 84.41553343886545, "10": 89.68405430731173, "11": 87.53569795614557}, "Response 1 lb": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 74.82059925402734, "7": 74.25211765306716, "8": 78.12376026030807, "9": 81.59498033002811, "10": 86.43560965382999, "11": 81.72003667435482}, "Response 1 ub": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 90.92840312634186, "7": 91.22343915995488, "8": 88.27795766769508, "9": 87.2360870064971, "10": 92.9324994891891, "11": 93.35136018391825}, "f(Response 1)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 1.6187921375701637, "7": 1.608474260622717, "8": 1.9694820573716958, "9": 2.0668410833202255, "10": 1.525421898187632, "11": 1.3954736641291545}, "Objective 1": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": 1.6187921375701637, "7": 1.608474260622717, "8": 1.9694820573716958, "9": 2.0668410833202255, "10": 1.525421898187632, "11": 1.3954736641291545}, "aq Value": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": -1.6494456924070198, "7": -1.6094291851799092, "8": -2.0767808380705333, "9": -2.273620548613846, "10": -2.6296665707757936, "11": -2.5557710737282764}, "aq Value (joint)": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": -0.9759695445936654, "7": -0.9759695445936654, "8": -1.8489609014435588, "9": -1.8489609014435588, "10": -2.339551238036062, "11": -2.339551238036062}, "aq Method": {"0": NaN, "1": NaN, "2": NaN, "3": NaN, "4": NaN, "5": NaN, "6": "NEI", "7": "NEI", "8": "NEI", "9": "NEI", "10": "NEI", "11": "NEI"}, "Response 1 (max) (iter)": {"0": 84.03902691676035, "1": 84.03902691676035, "2": 84.03902691676035, "3": 84.03902691676035, "4": 84.03902691676035, "5": 84.03902691676035, "6": 84.03902691676035, "7": 84.03902691676035, "8": 89.79791912501248, "9": 89.79791912501248, "10": 92.15225029636682, "11": 92.15225029636682}}, "target": [{"init_attrs": {"name": "Response 1", "aim": "max", "f_transform": "Standard"}, "f_raw": [[52.91999776955157], [47.950545897489405], [57.200296096064626], [69.86459941931172], [84.03902691676035], [56.568255091930745], [53.72360713169506], [46.766642700990914], [89.79791912501248], [85.82078349307184]]}], "seed": null} \ No newline at end of file diff --git a/obsidian/tests/test_campaign.py b/obsidian/tests/test_campaign.py index 073d345..5f2db57 100644 --- a/obsidian/tests/test_campaign.py +++ b/obsidian/tests/test_campaign.py @@ -7,7 +7,7 @@ from obsidian.objectives import Identity_Objective, Scalar_WeightedNorm, Feature_Objective, \ Objective_Sequence, Utopian_Distance, Index_Objective, Bounded_Target -from obsidian.tests.utils import DEFAULT_PATH +from obsidian.tests.utils import DEFAULT_MOO_PATH import json import pandas as pd @@ -35,6 +35,7 @@ def test_campaign(X_space, sim_fcn, target): campaign.clear_data() Z0['Iteration'] = 5 campaign.add_data(Z0) + campaign.y campaign.fit() obj_dict = campaign.save_state() @@ -45,7 +46,7 @@ def test_campaign(X_space, sim_fcn, target): campaign2.suggest() -with open(DEFAULT_PATH) as json_file: +with open(DEFAULT_MOO_PATH) as json_file: obj_dict = json.load(json_file) campaign = Campaign.load_state(obj_dict) @@ -53,8 +54,8 @@ def test_campaign(X_space, sim_fcn, target): target = campaign.target test_objs = [Identity_Objective(), Scalar_WeightedNorm(weights=[1, 1]), Feature_Objective(X_space, indices=[0], coeff=[1]), - Objective_Sequence([Utopian_Distance([1], target), Index_Objective()]), - Bounded_Target(bounds=[(0, 1)], targets=target)] + Objective_Sequence([Utopian_Distance([1], target[0]), Index_Objective()]), + Bounded_Target(bounds=[(0, 1)], targets=target[0])] @pytest.mark.parametrize('obj', test_objs) diff --git a/obsidian/tests/test_plotting.py b/obsidian/tests/test_plotting.py index 432b2ec..e53c716 100644 --- a/obsidian/tests/test_plotting.py +++ b/obsidian/tests/test_plotting.py @@ -2,11 +2,11 @@ from obsidian.plotting import parity_plot, factor_plot, surface_plot, visualize_inputs import pytest -from obsidian.tests.utils import DEFAULT_PATH +from obsidian.tests.utils import DEFAULT_MOO_PATH import json -with open(DEFAULT_PATH) as json_file: +with open(DEFAULT_MOO_PATH) as json_file: obj_dict = json.load(json_file) campaign = Campaign.load_state(obj_dict) diff --git a/obsidian/tests/utils.py b/obsidian/tests/utils.py index b0839a7..e04317e 100644 --- a/obsidian/tests/utils.py +++ b/obsidian/tests/utils.py @@ -1,10 +1,11 @@ """Utility functions for pytest tests""" -from .param_configs import X_sp_default +from obsidian.tests.param_configs import X_sp_default, X_sp_cont_ndims from obsidian import Campaign, ParamSpace, Target from obsidian.experiment import Simulator -from obsidian.experiment.benchmark import shifted_parab +from obsidian.experiment.benchmark import shifted_parab, two_leaves +from obsidian.objectives import Identity_Objective from numpy.typing import ArrayLike from typing import Callable @@ -12,7 +13,8 @@ import pandas as pd import json -DEFAULT_PATH = 'obsidian/tests/default_campaign.json' +DEFAULT_MOO_PATH = 'obsidian/tests/default_campaign_MOO.json' +DEFAULT_SOO_PATH = 'obsidian/tests/default_campaign_SOO.json' def approx_equal(x1: ArrayLike | float | int, @@ -27,28 +29,46 @@ def approx_equal(x1: ArrayLike | float | int, tol (float, optional): The tolerance value. Defaults to 1e-6. Returns: - bool: True if the numbers are approximately equal within the tolerance, False otherwise. + bool: True if the numbers are approximately equal within the tolerance, + False otherwise. """ diff = abs((x1-x2)/x1) return diff.max() < tol def save_default_campaign(X_space: ParamSpace, - response_func: Callable): # pragma: no cover + response_func: Callable, + path: str, + m_initial: int = 6, + n_response: int = 1): # pragma: no cover y_name = 'Response' - target = Target(name=y_name, f_transform='Standard', aim='max') + y_names = [y_name+' '+str(i+1) for i in range(n_response)] + target = [Target(name=n, f_transform='Standard', aim='max') for n in y_names] campaign = Campaign(X_space, target) - simulator = Simulator(X_space, response_func, y_name) - X0 = campaign.designer.initialize(m_initial=20, method='LHS') + simulator = Simulator(X_space, response_func, y_names) + + X0 = campaign.designer.initialize(m_initial=m_initial, method='LHS') y0 = simulator.simulate(X0) Z0 = pd.concat([X0, y0], axis=1) campaign.add_data(Z0) - campaign.fit() + + for i in range(3): + print(f'Simulating iteration {i+1}') + campaign.fit() + + X_suggest, eval_suggest = campaign.optimizer.suggest( + m_batch=2, objective=Identity_Objective(mo=n_response > 1)) + y_i = simulator.simulate(X_suggest) + Z_i = pd.concat([X_suggest, y_i, eval_suggest], axis=1) + campaign.add_data(Z_i) + obj_dict = campaign.save_state() - with open(DEFAULT_PATH, 'w') as outfile: + with open(path, 'w') as outfile: json.dump(obj_dict, outfile) if __name__ == '__main__': - save_default_campaign(X_sp_default, shifted_parab) + + save_default_campaign(X_sp_default, shifted_parab, DEFAULT_SOO_PATH, n_response=1) + save_default_campaign(X_sp_cont_ndims[2], two_leaves, DEFAULT_MOO_PATH, n_response=2) From 59ecb3495ef638b41bac85a1d99a699038724af1 Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Fri, 9 Aug 2024 16:50:36 -0400 Subject: [PATCH 2/3] Removed observational param from default X_space for testing --- obsidian/tests/param_configs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/obsidian/tests/param_configs.py b/obsidian/tests/param_configs.py index ad2e965..8b369a0 100644 --- a/obsidian/tests/param_configs.py +++ b/obsidian/tests/param_configs.py @@ -15,7 +15,7 @@ Param_Ordinal('Parameter 11', ['N']) ] -default = [params[i] for i in [0, 1, 3, 6]] # 2 continuous, 1 observational, 1 categorical +default = [params[i] for i in [0, 1, 2, 6]] # 2 continuous, 1 static, 1 categorical cont_small = [params[i] for i in [0, 1, 2]] # continuous including edge cases numeric = [params[i] for i in [0, 1, 2, 3, 4, 5]] # numeric including edge cases From b631f724a834f7ac53bbdf3aa5925feedb31d2a6 Mon Sep 17 00:00:00 2001 From: Kevin Stone Date: Fri, 9 Aug 2024 17:19:38 -0400 Subject: [PATCH 3/3] Fixed and improved campaign analysis --- obsidian/campaign/campaign.py | 98 +++++++++++++++++++++++++-------- obsidian/tests/test_campaign.py | 16 ++++-- 2 files changed, 86 insertions(+), 28 deletions(-) diff --git a/obsidian/campaign/campaign.py b/obsidian/campaign/campaign.py index 2ae7198..bc542a9 100644 --- a/obsidian/campaign/campaign.py +++ b/obsidian/campaign/campaign.py @@ -4,6 +4,7 @@ from obsidian.optimizer import Optimizer, BayesianOptimizer from obsidian.experiment import ExpDesigner from obsidian.objectives import Objective, Objective_Sequence, obj_class_dict +from obsidian.exceptions import IncompatibleObjectiveError import obsidian import pandas as pd @@ -52,8 +53,8 @@ def __init__(self, designer = ExpDesigner(X_space, seed=seed) if designer is None else designer self.set_designer(designer) - self.set_objective(objective) self.set_target(target) + self.set_objective(objective) # Non-object attributes self.iter = 0 @@ -87,6 +88,10 @@ def add_data(self, df: pd.DataFrame): self.iter += 1 self.data = pd.concat([self.data, new_data], axis=0, ignore_index=True) self.data.index.name = 'Observation ID' + self.data.index = self.data.index.astype('int') + + if self.optimizer.is_fit: + self._analyze() def clear_data(self): """Clears campaign data""" @@ -115,10 +120,23 @@ def objective(self) -> Objective | None: """Campaign Objective function""" return self._objective + def _eval_objective(self): + """Evaluates objective and appends it to campaign data""" + df_o = self.o + for col in df_o.columns: + self.data[col] = df_o[col].values + def set_objective(self, objective: Objective | None): - """Sets the campaign objective function""" + """(Re)sets the campaign objective function""" self._objective = objective - + if not self.data.empty: + # Remove previous objective evaluations + self.data = self.data.drop( + columns=[col for col in self.data.columns if 'Objective' in col] + ) + if self.optimizer.is_fit: + self._analyze() + @property def target(self): """Campaign experimental target(s)""" @@ -156,10 +174,7 @@ def m_exp(self) -> int: """ Number of observations in training data """ - if self.data is not None: - return self.data.shape[0] - else: - return 0 + return self.data.shape[0] @property def y(self) -> pd.Series | pd.DataFrame: @@ -188,6 +203,22 @@ def f(self) -> pd.Series | pd.DataFrame: f = pd.concat([t.transform_f(self.y[t.name]) for t in self.target], axis=1) return f + @property + def o(self) -> pd.Series | pd.DataFrame: + if self.objective: + try: + x = self.X_space.encode(self.data[list(self.X_space.X_names)]).values + o = self.objective(torch.tensor(self.f.values).unsqueeze(0), + X=torch.tensor(x)).squeeze(0) + if o.ndim < 2: + o = o.unsqueeze(1) # Rearrange into m x o + return pd.DataFrame(o.detach().cpu().numpy(), + columns=[f'Objective {o_i+1}' for o_i in range(o.shape[1])]) + except Exception: + raise IncompatibleObjectiveError('Objective(s) did not successfully execute on sample') + else: + return None + @property def X(self) -> pd.DataFrame: """ @@ -242,6 +273,7 @@ def load_state(cls, objective=new_objective, seed=obj_dict['seed']) new_campaign.data = pd.DataFrame(obj_dict['data']) + new_campaign.data.index = new_campaign.data.index.astype('int') try: new_campaign.iter = new_campaign.data['Iteration'].astype('int').max() @@ -273,7 +305,6 @@ def fit(self): raise ValueError('Must register data before fitting') self.optimizer.fit(self.data, target=self.target) - # self._analyze() def suggest(self, optim_kwargs={}): @@ -301,14 +332,21 @@ def _profile_hv(self): """ iters = self.data['Iteration'].unique() hv = {} + + if self.objective: + out = self.o + else: + out = self.y + for i in iters: iter_index = self.data.query(f'Iteration <= {i}').index - y_iter = self.y.loc[iter_index, :] - y_iter = torch.tensor(y_iter.values).to(self.optimizer.device) - hv[i] = self.optimizer.hypervolume(y_iter) + out_iter = out.loc[iter_index, :] + out_iter = torch.tensor(out_iter.values).to(self.optimizer.device) + hv[i] = self.optimizer.hypervolume(out_iter) + + self.data['Hypervolume (iter)'] = self.data.apply(lambda x: hv[x['Iteration']], axis=1) + self.data['Pareto Front'] = self.optimizer.pareto(torch.tensor(out.values).to(self.optimizer.device)) - self.data['Hypervolume'] = self.data.apply(lambda x: hv[x['Iteration']], axis=1) - return def _profile_max(self): @@ -318,13 +356,21 @@ def _profile_max(self): Returns: None """ - y_max = self.data.groupby('Iteration', observed=True).max()[self.y_names].reset_index() - y_max = y_max.rename(columns={name: name+' (max)' for name in self.y_names}) - - # Reset aggregate columns if previously calculated - if any([name+' (max)' in self.data.columns for name in self.y_names]): - self.data = self.data.copy().drop(columns={name+' (max)' for name in self.y_names}) - self.data = self.data.merge(y_max, on='Iteration') + + # Remove previous max-profiling + self.data = self.data.drop( + columns=[col for col in self.data.columns if '(max) (iter)' in col] + ) + + if self.objective: + out_names = [col for col in self.data.columns if 'Objective' in col] + else: + out_names = self.y_names + + for out in out_names: + self.data[out+' (max) (iter)'] = self.data.apply( + lambda x: self.data.query(f'Iteration<={x["Iteration"]}')[out].max(), axis=1 + ) return @@ -335,8 +381,16 @@ def _analyze(self): Returns: None """ + if self.objective: + self._eval_objective() self._profile_max() - if self.n_response > 1: + if self._is_mo: self._profile_hv() - self.data['Pareto Front'] = self.optimizer.pareto(torch.tensor(self.y.values).to(self.optimizer.device)) + else: + # Remove previous HV-profiling + self.data = self.data.drop( + columns=[col for col in self.data.columns + if 'Hypervolume' in col or 'Pareto' in col] + ) + return diff --git a/obsidian/tests/test_campaign.py b/obsidian/tests/test_campaign.py index 5f2db57..d5dac53 100644 --- a/obsidian/tests/test_campaign.py +++ b/obsidian/tests/test_campaign.py @@ -42,7 +42,7 @@ def test_campaign(X_space, sim_fcn, target): campaign2 = Campaign.load_state(obj_dict) campaign2.__repr__() - campaign2.set_objective(Identity_Objective()) + campaign2.set_objective(Identity_Objective(mo=len(campaign.target) > 1)) campaign2.suggest() @@ -53,21 +53,25 @@ def test_campaign(X_space, sim_fcn, target): X_space = campaign.X_space target = campaign.target -test_objs = [Identity_Objective(), Scalar_WeightedNorm(weights=[1, 1]), Feature_Objective(X_space, indices=[0], coeff=[1]), +test_objs = [Identity_Objective(mo=True), + Scalar_WeightedNorm(weights=[1, 1]), + Feature_Objective(X_space, indices=[0], coeff=[1]), Objective_Sequence([Utopian_Distance([1], target[0]), Index_Objective()]), - Bounded_Target(bounds=[(0, 1)], targets=target[0])] + Bounded_Target(bounds=[(0, 1), (0, 1)], targets=target), + None] @pytest.mark.parametrize('obj', test_objs) def test_campaign_objectives(obj): campaign.set_objective(obj) - campaign.objective.__repr__() - + if campaign.objective: + campaign.objective.__repr__() + obj_dict = campaign.save_state() campaign2 = Campaign.load_state(obj_dict) campaign2.save_state() campaign2.__repr__() - + if __name__ == '__main__': pytest.main([__file__, '-m', 'not slow'])