From fab1d9584510c2294816fda79916b143c8af79f6 Mon Sep 17 00:00:00 2001 From: Lenz Fiedler Date: Tue, 5 Oct 2021 15:39:09 +0200 Subject: [PATCH 1/2] Small bugfix --- mala/network/no_training_pruner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mala/network/no_training_pruner.py b/mala/network/no_training_pruner.py index bb24f5c84..47c93f52b 100644 --- a/mala/network/no_training_pruner.py +++ b/mala/network/no_training_pruner.py @@ -60,7 +60,7 @@ def prune(self, study: "optuna.study.Study", trial: objective = ObjectiveNoTraining(self._params, self._data_handler, self._trial_type) surrogate_loss = objective(trial) - if surrogate_loss > self._params.hyperparameters.no_training_cutoff: + if surrogate_loss < self._params.hyperparameters.no_training_cutoff: return True else: return False From 58a701115073b8430e76374b38af64bd05fb1f05 Mon Sep 17 00:00:00 2001 From: Lenz Fiedler Date: Tue, 5 Oct 2021 15:53:07 +0200 Subject: [PATCH 2/2] Ensemble "Training" for NoTraining --- mala/network/objective_no_training.py | 72 ++++++++++++++++++--------- 1 file changed, 48 insertions(+), 24 deletions(-) diff --git a/mala/network/objective_no_training.py b/mala/network/objective_no_training.py index 3fd225b03..7deb4b04e 100644 --- a/mala/network/objective_no_training.py +++ b/mala/network/objective_no_training.py @@ -53,30 +53,54 @@ def __call__(self, trial): super(ObjectiveNoTraining, self).parse_trial(trial) # Build the network. - net = Network(self.params) - device = "cuda" if self.params.use_gpu else "cpu" - - # Load the batchesand get the jacobian. - do_shuffle = self.params.running.use_shuffling_for_samplers - if self.data_handler.parameters.use_lazy_loading or \ - self.params.use_horovod: - do_shuffle = False - if self.params.running.use_shuffling_for_samplers: - self.data_handler.mix_datasets() - loader = DataLoader(self.data_handler.training_data_set, - batch_size=self.params.running.mini_batch_size, - shuffle=do_shuffle) - jac = ObjectiveNoTraining.__get_batch_jacobian(net, loader, device) - - # Loss = - score! - surrogate_loss = float('inf') - try: - surrogate_loss = - ObjectiveNoTraining.__calc_score(jac) - surrogate_loss = surrogate_loss.cpu().detach().numpy().astype( - np.float64) - except RuntimeError: - printout("Got a NaN, ignoring sample.") - return surrogate_loss + surrogate_losses = [] + for i in range(0, self.params.hyperparameters. + number_training_per_trial): + net = Network(self.params) + device = "cuda" if self.params.use_gpu else "cpu" + + # Load the batchesand get the jacobian. + do_shuffle = self.params.running.use_shuffling_for_samplers + if self.data_handler.parameters.use_lazy_loading or \ + self.params.use_horovod: + do_shuffle = False + if self.params.running.use_shuffling_for_samplers: + self.data_handler.mix_datasets() + loader = DataLoader(self.data_handler.training_data_set, + batch_size=self.params.running.mini_batch_size, + shuffle=do_shuffle) + jac = ObjectiveNoTraining.__get_batch_jacobian(net, loader, device) + + # Loss = - score! + surrogate_loss = float('inf') + try: + surrogate_loss = - ObjectiveNoTraining.__calc_score(jac) + surrogate_loss = surrogate_loss.cpu().detach().numpy().astype( + np.float64) + except RuntimeError: + printout("Got a NaN, ignoring sample.") + surrogate_losses.append(surrogate_loss) + + if self.params.hyperparameters.number_training_per_trial > 1: + printout("Losses from multiple runs are: ") + printout(surrogate_losses) + + if self.params.hyperparameters.trial_ensemble_evaluation == "mean": + return np.mean(surrogate_losses) + + elif self.params.hyperparameters.trial_ensemble_evaluation == \ + "mean_std": + mean = np.mean(surrogate_losses) + + # Cannot calculate the standar deviation of a bunch of infinities. + if np.isinf(mean): + return mean + else: + return np.mean(surrogate_losses) + \ + np.std(surrogate_losses) + else: + raise Exception("No way to estimate the trial metric from ensemble" + " training provided.") @staticmethod def __get_batch_jacobian(net: Network, loader: DataLoader, device) \