From f4bdb63b5bdb86860b5f15da12a8f246a0c3e9c5 Mon Sep 17 00:00:00 2001 From: carlo Date: Thu, 16 Oct 2014 22:48:33 +0200 Subject: [PATCH 01/32] dbm uploaded --- pylearn2/datasets/augment_input.py | 27 +++ pylearn2/datasets/gtsrb.py | 164 ++++++++++++++ pylearn2/datasets/mnistaugmented.py | 61 ++++++ pylearn2/scripts/papers/dbm/README | 16 ++ pylearn2/scripts/papers/dbm/dbm_mnist.yaml | 76 +++++++ pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml | 39 ++++ pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml | 42 ++++ .../scripts/papers/dbm/dbm_mnist_mlp.yaml | 55 +++++ .../papers/dbm/dbm_mnist_mlp_dropout.yaml | 63 ++++++ .../scripts/papers/dbm/dbm_mnist_softmax.yaml | 82 +++++++ pylearn2/scripts/papers/dbm/test_dbm_mnist.py | 207 ++++++++++++++++++ pylearn2/training_algorithms/sgd.py | 18 ++ 12 files changed, 850 insertions(+) create mode 100644 pylearn2/datasets/augment_input.py create mode 100644 pylearn2/datasets/gtsrb.py create mode 100644 pylearn2/datasets/mnistaugmented.py create mode 100644 pylearn2/scripts/papers/dbm/README create mode 100644 pylearn2/scripts/papers/dbm/dbm_mnist.yaml create mode 100644 pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml create mode 100644 pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml create mode 100644 pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml create mode 100644 pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml create mode 100644 pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml create mode 100644 pylearn2/scripts/papers/dbm/test_dbm_mnist.py diff --git a/pylearn2/datasets/augment_input.py b/pylearn2/datasets/augment_input.py new file mode 100644 index 0000000000..8cc5d11ecb --- /dev/null +++ b/pylearn2/datasets/augment_input.py @@ -0,0 +1,27 @@ +from pylearn2.utils import sharedX +from theano import function +import numpy + +def augment_input(X, model, mf_steps): + + print '\nAugmenting data...\n' + + i = 0 + init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') + + for x in X[:]: + init_data[0] = x + data = sharedX(init_data, name = 'v') + marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) + mp = function([], marginal_posterior) + mp = mp()[0][0] + if i == 0: + final_data = numpy.asarray([numpy.concatenate((mp, x))]) + else: + final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) + + i += 1 + + print 'Data augmentation complete!' + + return final_data \ No newline at end of file diff --git a/pylearn2/datasets/gtsrb.py b/pylearn2/datasets/gtsrb.py new file mode 100644 index 0000000000..97cf0e1e0f --- /dev/null +++ b/pylearn2/datasets/gtsrb.py @@ -0,0 +1,164 @@ +from PIL import Image +import numpy +import csv +import cPickle + +from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix +from pylearn2.datasets.augment_input import augment_input + +# User can select whether he wants to select all images or just a smaller set of them in order not to add too much noise +# after the resizing +bound_train = 1000 +bound_test = 1000 + +class GTSRB(DenseDesignMatrix): + + def __init__(self, which_set, model = None, mf_steps = None, one_hot = True, + start = None, stop = None, img_size = None): + + #path = "${PYLEARN2_DATA_PATH}/gtsrb" + first_path = "/home/deramo/workspace/datasets/gtsrb" + if which_set == 'train': + path = first_path + "/Final_Training/Images" + else: + path = first_path + "/Final_Test/Images" + self.path = path + self.which_set = which_set + self.delimiter = ';' + self.img_size = img_size + self.one_hot = one_hot + self.start = start + self.stop = stop + + try: + if which_set == 'train': + datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'train_dump.pkl.gz') + X, y = datasets[0], datasets[1] + else: + datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'test_dump.pkl.gz') + X, y = datasets[0], datasets[1] + except: + try: + if which_set == 'train': + datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'noaug_train_dump.pkl.gz') + X, y = datasets[0], datasets[1] + else: + datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'noaug_test_dump.pkl.gz') + X, y = datasets[0], datasets[1] + X, y = X[0:12600], y[0:12600] # temporaneo + + except: + X, y = self.load_data() + print "\ndata loaded!\n" + + noaug_datasets = X, y # not augmented datasets is saved in order not to waste time reloading gtsrb each time + if which_set == 'train': + save_to_dump(var_to_dump = noaug_datasets, dump_data_dir = self.path, dump_filename = 'noaug_train_dump.pkl.gz') + else: + save_to_dump(var_to_dump = noaug_datasets, dump_data_dir = self.path, dump_filename = 'noaug_test_dump.pkl.gz') + + X, y = X.astype(float), y.astype(float) + + # BUILD AUGMENTED INPUT FOR FINETUNING + if mf_steps is not None: + augmented_X = augment_input(X, model, mf_steps) + X = augmented_X + + datasets = augmented_X, y + if which_set == 'train': + save_to_dump(var_to_dump = datasets, dump_data_dir = self.path, dump_filename = 'train_dump.pkl.gz') + else: + save_to_dump(var_to_dump = datasets, dump_data_dir = self.path, dump_filename = 'test_dump.pkl.gz') + + X, y = X[self.start:self.stop], y[self.start:self.stop] + super(GTSRB, self).__init__(X = X, y = y) + + def load_data(self): + + print "\nloading data...\n" + + if self.which_set == 'train': + + first = True + + # loop over all 43 classes + for c in xrange(43): #43 + prefix = self.path + '/' + format(c, '05d') + '/' # subdirectory for class + f = open(prefix + 'GT-'+ format(c, '05d') + '.csv') # annotations file + reader = csv.reader(f, delimiter = self.delimiter) # csv parser for annotations file + reader.next() # skip header + for row in reader: + img = Image.open(prefix + '/' + row[0]) + if img.size[0] + bound_train >= self.img_size[0]: + img = img.convert('L') # grayscale + img = img.resize(self.img_size, Image.ANTIALIAS) #resize + if first: + X = numpy.asarray([img.getdata()]) + X /= 255. + y = numpy.asarray(row[7]) + first = False + else: + X = numpy.append(X, [img.getdata()], axis = 0) + X /= 255. + y = numpy.append(y, row[7]) + f.close() + + # shuffle + assert X.shape[0] == y.shape[0] + + indices = numpy.arange(X.shape[0]) + rng = numpy.random.RandomState() # if given an int argument will give reproducible results + rng.shuffle(indices) + # shuffle both the arrays consistently + i = 0 + temp_X = X + temp_y = y + for idx in indices: + X[i] = temp_X[idx] + y[i] = temp_y[idx] + i += 1 + + else: + + first = True + + f = open(self.path + '/' + "GT-final_test.csv") + reader = csv.reader(f, delimiter = self.delimiter) # csv parser for annotations file + reader.next() # skip header + + for c in xrange(12630): + for row in reader: + img = Image.open(self.path + '/' + row[0]) + if img.size[0] + bound_train >= self.img_size[0]: + img = img.convert('L') # grayscale + img = img.resize(self.img_size, Image.ANTIALIAS) #resize + if first: + X = numpy.asarray([img.getdata()]) + X /= 255. + y = row[7] + first = False + else: + X = numpy.append(X, [img.getdata()], axis = 0) + X /= 255. + y = numpy.append(y, row[7]) + f.close() + + # build the one_hot matrix used to specify labels + if self.one_hot: + one_hot = numpy.zeros((y.shape[0], 43)) + for i in xrange(y.shape[0]): + one_hot[i,y[i]] = 1. + y = one_hot + + return X, y + +def load_from_dump(dump_data_dir, dump_filename): + load_file = open(dump_data_dir + "/" + dump_filename) + unpickled_var = cPickle.load(load_file) + load_file.close() + return unpickled_var + +def save_to_dump(var_to_dump, dump_data_dir, dump_filename): + save_file = open(dump_data_dir + "/" + dump_filename, 'wb') # this will overwrite current contents + cPickle.dump(var_to_dump, save_file, -1) # the -1 is for HIGHEST_PROTOCOL + save_file.close() \ No newline at end of file diff --git a/pylearn2/datasets/mnistaugmented.py b/pylearn2/datasets/mnistaugmented.py new file mode 100644 index 0000000000..40cb78d383 --- /dev/null +++ b/pylearn2/datasets/mnistaugmented.py @@ -0,0 +1,61 @@ +import os +import cPickle +import numpy as np + +from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix +from pylearn2.datasets.augment_input import augment_input +from pylearn2.utils import serial + +''' + Loads MNIST dataset and build augmented dataset + for DBM discriminative finetuning +''' + +class MNISTAUGMENTED(DenseDesignMatrix): + + def __init__(self, dataset, which_set, model, mf_steps, one_hot = True, + start = None, stop = None): + + path = os.path.join('${PYLEARN2_DATA_PATH}', 'mnist') + path = serial.preprocess(path) + + try: + if which_set == 'train': + datasets = load_from_dump(dump_data_dir = path, dump_filename = 'aug_train_dump.pkl.gz') + augmented_X, y = datasets[0], datasets[1] + else: + datasets = load_from_dump(dump_data_dir = path, dump_filename = 'aug_test_dump.pkl.gz') + augmented_X, y = datasets[0], datasets[1] + except: + X = dataset.X + if one_hot: + one_hot = np.zeros((dataset.y.shape[0], 10), dtype='float32') + for i in xrange(dataset.y.shape[0]): + label = dataset.y[i] + one_hot[i, label] = 1. + y = one_hot + else: + y = dataset.y + + # BUILD AUGMENTED INPUT FOR FINETUNING + augmented_X = augment_input(X, model, mf_steps) + + datasets = augmented_X, y + if which_set == 'train': + save_to_dump(var_to_dump = datasets, dump_data_dir = path, dump_filename = 'aug_train_dump.pkl.gz') + else: + save_to_dump(var_to_dump = datasets, dump_data_dir = path, dump_filename = 'aug_test_dump.pkl.gz') + + augmented_X, y = augmented_X[start:stop], y[start:stop] + super(MNISTAUGMENTED, self).__init__(X = augmented_X, y = y) + +def load_from_dump(dump_data_dir, dump_filename): + load_file = open(dump_data_dir + "/" + dump_filename) + unpickled_var = cPickle.load(load_file) + load_file.close() + return unpickled_var + +def save_to_dump(var_to_dump, dump_data_dir, dump_filename): + save_file = open(dump_data_dir + "/" + dump_filename, 'wb') # this will overwrite current contents + cPickle.dump(var_to_dump, save_file, -1) # the -1 is for HIGHEST_PROTOCOL + save_file.close() \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/README b/pylearn2/scripts/papers/dbm/README new file mode 100644 index 0000000000..dc55e9b031 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/README @@ -0,0 +1,16 @@ +The files in this directory recreate the experiment reported in the +paper + +An efficient learning procedure for Deep Boltzmann Machines. G. Hinton, and R. Salakhutdinov. + +The procedure is divided in three phases: pretraining of RBMs, training and finetuning. The test_dbm_mnist script allows to enable each phase of training and to select whether the DBM +is composed of a softmax layer or not, and whether the MLP has to do finetuning with dropout or not. + +As explained in the paper, the finetuning procedure uses an augmented input to feed the MLP and this implementation creates it using augment_input.py and +mnistaugmented.py in pylearn2/datasets/. The latter takes the mnist dataset and augment it. Eventually, it saves .pkl files of the augmented dataset +because data augmentation is a time-consuming operation. + +NO DROPOUT RESULTS: +The procedure without dropout does not use a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0094%. +DROPOUT RESULTS: +The procedure with dropout uses a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0084%. \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml new file mode 100644 index 0000000000..c3c7ae0538 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml @@ -0,0 +1,76 @@ +!obj:pylearn2.train.Train { + dataset: &data !obj:pylearn2.datasets.binarizer.Binarizer { + raw: &raw_train !obj:pylearn2.datasets.mnist.MNIST { + which_set: "train", + one_hot: 1, + start: 0, + stop: %(train_stop)i + } + }, + model: !obj:pylearn2.models.dbm.DBM { + batch_size: %(batch_size)i, + niter: 10, + inference_procedure: !obj:pylearn2.models.dbm.WeightDoubling {}, + visible_layer: !obj:pylearn2.models.dbm.BinaryVector { + nvis: 784, + }, + hidden_layers: [ + !obj:pylearn2.models.dbm.BinaryVectorMaxPool { + layer_name: 'h1', + detector_layer_dim: %(detector_layer_1_dim)i, + pool_size: 1, + irange: 0.001, + }, + !obj:pylearn2.models.dbm.BinaryVectorMaxPool { + layer_name: 'h2', + detector_layer_dim: %(detector_layer_2_dim)i, + pool_size: 1, + irange: 0.001, + }, + ] + }, + algorithm: !obj:pylearn2.training_algorithms.sgd.SGD { + learning_rate: 0.005, + learning_rule: !obj:pylearn2.training_algorithms.learning_rule.Momentum { + init_momentum: 0.5, + }, + monitoring_batches: %(monitoring_batches)i, + monitoring_dataset: { + 'test': !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'test', + one_hot: 1 + } + }, + cost : !obj:pylearn2.costs.cost.SumOfCosts { + costs: [ + !obj:pylearn2.costs.dbm.VariationalPCD { + num_chains: 100, + num_gibbs_steps: 5, + }, + !obj:pylearn2.costs.dbm.WeightDecay { + coeffs: [ .0002, .0002 ], + }, + !obj:pylearn2.costs.dbm.TorontoSparsity { + targets: [ .2, .1 ], + coeffs: [ .001, .001 ], + } + ] + }, + termination_criterion: !obj:pylearn2.termination_criteria.EpochCounter { + max_epochs: %(max_epochs)i + }, + update_callbacks: [ + !obj:pylearn2.training_algorithms.sgd.LRAssigning { + } + ] + }, + extensions: [ + !obj:pylearn2.training_algorithms.learning_rule.MomentumAdjustor { + final_momentum: 0.9, + start: 1, + saturate: 6, + }, + ], + save_path: "%(save_path)s/dbm_mnist.pkl", + save_freq : %(max_epochs)i +} \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml new file mode 100644 index 0000000000..e718aa98f7 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml @@ -0,0 +1,39 @@ +!obj:pylearn2.train.Train { + dataset: &data !obj:pylearn2.datasets.binarizer.Binarizer { + raw: &raw_train !obj:pylearn2.datasets.mnist.MNIST { + which_set: "train", + one_hot: 1, + start: 0, + stop: %(train_stop)i + } + }, + model: !obj:pylearn2.models.rbm.RBM { + nvis : 784, + nhid : %(nhid)i, + irange : 0.001, + }, + algorithm: !obj:pylearn2.training_algorithms.sgd.SGD { + learning_rate : 0.05, + learning_rule: !obj:pylearn2.training_algorithms.learning_rule.Momentum { + init_momentum: 0.5, + }, + batch_size : %(batch_size)i, + monitoring_batches : %(monitoring_batches)i, + monitoring_dataset : *data, + cost: !obj:pylearn2.costs.ebm_estimation.CDk { + nsteps : 1, + }, + termination_criterion : !obj:pylearn2.termination_criteria.EpochCounter { + max_epochs: %(max_epochs)i, + }, + }, + extensions: [ + !obj:pylearn2.training_algorithms.learning_rule.MomentumAdjustor { + start: 1, + saturate: 6, + final_momentum: 0.9, + }, + ], + save_path: "%(save_path)s/dbm_mnist_l1.pkl", + save_freq: %(max_epochs)i +} \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml new file mode 100644 index 0000000000..6d19ff323a --- /dev/null +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml @@ -0,0 +1,42 @@ +!obj:pylearn2.train.Train { + dataset: &train !obj:pylearn2.datasets.binarizer.Binarizer { + raw: !obj:pylearn2.datasets.transformer_dataset.TransformerDataset { + raw: !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'train', + one_hot: 1, + start: 0, + stop: %(train_stop)i + }, + transformer: !pkl: "%(save_path)s/dbm_mnist_l1.pkl" + }, + }, + model: !obj:pylearn2.models.rbm.RBM { + nvis : %(nvis)i, + nhid : %(nhid)i, + irange : 0.01, + }, + algorithm: !obj:pylearn2.training_algorithms.sgd.SGD { + learning_rate : 0.05, + learning_rule: !obj:pylearn2.training_algorithms.learning_rule.Momentum { + init_momentum: 0.5, + }, + batch_size : %(batch_size)i, + monitoring_batches : %(monitoring_batches)i, + monitoring_dataset : *train, + cost : !obj:pylearn2.costs.ebm_estimation.CDk { + nsteps : 5, + }, + termination_criterion : !obj:pylearn2.termination_criteria.EpochCounter { + max_epochs: %(max_epochs)i, + }, + }, + extensions: [ + !obj:pylearn2.training_algorithms.learning_rule.MomentumAdjustor { + start: 1, + saturate: 6, + final_momentum: 0.9, + }, + ], + save_path: "%(save_path)s/dbm_mnist_l2.pkl", + save_freq: %(max_epochs)i +} \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml new file mode 100644 index 0000000000..0ee9ca836c --- /dev/null +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml @@ -0,0 +1,55 @@ +!obj:pylearn2.train.Train { + dataset: &train !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'train', + one_hot: 1, + start: 0, + stop: %(train_stop)i + }, + model: !obj:pylearn2.models.mlp.MLP { + batch_size: %(batch_size)i, + layers: [ + !obj:pylearn2.models.mlp.Sigmoid { + layer_name: 'h0', + dim: %(n_h0)i, + sparse_init: 15, + }, + !obj:pylearn2.models.mlp.Sigmoid { + layer_name: 'h1', + dim: %(n_h1)i, + sparse_init: 15, + }, + !obj:pylearn2.models.mlp.Softmax { + layer_name: 'y', + n_classes: 10, + irange: 0.05 + } + ], + nvis: %(nvis)i, + }, + algorithm: !obj:pylearn2.training_algorithms.bgd.BGD { + conjugate: 1, + line_search_mode: 'exhaustive', + updates_per_batch: 6, + monitoring_dataset: { + 'test': !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'test', + one_hot: 1 + }, + }, + cost: !obj:pylearn2.costs.mlp.Default {}, + termination_criterion: !obj:pylearn2.termination_criteria.And { + criteria: [ + !obj:pylearn2.termination_criteria.EpochCounter { + max_epochs: %(max_epochs)i + } + ] + }, + }, + extensions: [ + !obj:pylearn2.train_extensions.best_params.MonitorBasedSaveBest { + channel_name: 'test_y_misclass', + save_path: "%(save_path)s/dbm_mnist_mlp.pkl", + store_best_model: True + }, + ] +} \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml new file mode 100644 index 0000000000..6ad38d0605 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml @@ -0,0 +1,63 @@ +!obj:pylearn2.train.Train { + dataset: !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'train', + one_hot: 1, + start: 0, + stop: %(train_stop)i, + }, + model: !obj:pylearn2.models.mlp.MLP { + batch_size: %(batch_size)i, + layers: [ + !obj:pylearn2.models.mlp.Sigmoid { + layer_name: 'h0', + dim: %(n_h0)i, + sparse_init: 15, + }, + !obj:pylearn2.models.mlp.Sigmoid { + layer_name: 'h1', + dim: %(n_h1)i, + sparse_init: 15, + }, + !obj:pylearn2.models.mlp.Softmax { + layer_name: 'y', + n_classes: 10, + istdev: 0.01, + } + ], + nvis: %(nvis)i, + }, + algorithm: !obj:pylearn2.training_algorithms.sgd.SGD { + learning_rate: 1, + learning_rule: !obj:pylearn2.training_algorithms.learning_rule.Momentum { + init_momentum: 0.5, + }, + monitoring_dataset: { + 'test': !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'test', + one_hot: 1 + }, + }, + cost: !obj:pylearn2.costs.mlp.dropout.Dropout { + input_include_probs: {'h0': .8, 'h1': 0.5, 'y': 0.5}, + }, + termination_criterion: !obj:pylearn2.termination_criteria.And { + criteria: [ + !obj:pylearn2.termination_criteria.EpochCounter { + max_epochs: %(max_epochs)i + } + ] + } + }, + extensions: [ + !obj:pylearn2.training_algorithms.sgd.MomentumAdjustor { + start: 1, + saturate: 500, + final_momentum: 0.99, + }, + !obj:pylearn2.train_extensions.best_params.MonitorBasedSaveBest { + channel_name: 'test_y_misclass', + save_path: "%(save_path)s/dbm_mnist_mlp_dropout.pkl", + store_best_model: True + }, + ] +} \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml new file mode 100644 index 0000000000..b9435815f8 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml @@ -0,0 +1,82 @@ +!obj:pylearn2.train.Train { + dataset: &data !obj:pylearn2.datasets.binarizer.Binarizer { + raw: &raw_train !obj:pylearn2.datasets.mnist.MNIST { + which_set: "train", + one_hot: 1, + start: 0, + stop: %(train_stop)i + } + }, + model: !obj:pylearn2.models.dbm.DBM { + batch_size: %(batch_size)i, + niter: 10, + inference_procedure: !obj:pylearn2.models.dbm.WeightDoubling {}, + visible_layer: !obj:pylearn2.models.dbm.BinaryVector { + nvis: 784, + }, + hidden_layers: [ + !obj:pylearn2.models.dbm.BinaryVectorMaxPool { + layer_name: 'h1', + detector_layer_dim: %(detector_layer_1_dim)i, + pool_size: 1, + irange: 0.001, + }, + !obj:pylearn2.models.dbm.BinaryVectorMaxPool { + layer_name: 'h2', + detector_layer_dim: %(detector_layer_2_dim)i, + pool_size: 1, + irange: 0.001, + }, + !obj:pylearn2.models.dbm.Softmax { + layer_name: 'y', + irange: 0.001, + n_classes: 10 + } + ] + }, + algorithm: !obj:pylearn2.training_algorithms.sgd.SGD { + learning_rate: 0.005, + learning_rule: !obj:pylearn2.training_algorithms.learning_rule.Momentum { + init_momentum: 0.5, + }, + monitoring_batches: %(monitoring_batches)i, + monitoring_dataset: { + 'test': !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'test', + one_hot: 1 + } + }, + cost : !obj:pylearn2.costs.cost.SumOfCosts { + costs: [ + !obj:pylearn2.costs.dbm.VariationalPCD { + num_chains: 100, + num_gibbs_steps: 5, + supervised: 1, + toronto_neg: 1 + }, + !obj:pylearn2.costs.dbm.WeightDecay { + coeffs: [ .0002, .0002, 0.0002 ], + }, + ] + }, + termination_criterion: !obj:pylearn2.termination_criteria.EpochCounter { + max_epochs: %(max_epochs)i + }, + update_callbacks: [ + !obj:pylearn2.training_algorithms.sgd.LRAssigning { + } + ] + }, + extensions: [ + !obj:pylearn2.training_algorithms.learning_rule.MomentumAdjustor { + final_momentum: 0.9, + start: 1, + saturate: 6, + }, + !obj:pylearn2.train_extensions.best_params.MonitorBasedSaveBest { + channel_name: 'test_misclass', + save_path: "%(save_path)s/dbm_mnist_softmax.pkl", + store_best_model: True + }, + ], +} \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py new file mode 100644 index 0000000000..63568a1a79 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py @@ -0,0 +1,207 @@ +__authors__ = "Carlo D'Eramo, Francesco Visin, Matteo Matteucci" +__copyright__ = "Copyright 2014-2015, Politecnico di Milano" +__credits__ = ["Carlo D'Eramo, Francesco Visin, Matteo Matteucci"] +__license__ = "3-clause BSD" +__maintainer__ = "AIR-lab" +__email__ = "carlo.deramo@mail.polimi.it, francesco.visin@polimi.it, matteo.matteucci@polimi.it" + +import os +import numpy + +from pylearn2.utils import serial +from pylearn2.config import yaml_parse +from pylearn2.testing import no_debug_mode +from pylearn2.datasets import mnistaugmented +from theano import function + +################## +### PARAMETERS ### +################## + +N_HIDDEN_0 = 500 +N_HIDDEN_1 = 1000 + +PRETRAINING = 1 +TRAINING = 1 +FINETUNING = 1 + +# PRETRAINING +MAX_EPOCHS_L1 = 100 +MAX_EPOCHS_L2 = 200 + +# TRAINING +MAX_EPOCHS_DBM = 500 +SOFTMAX = 0 + +# FINETUNING +MAX_EPOCHS_MLP = 500 +DROPOUT = 0 +MF_STEPS = 1 # mf_steps for data augmentation + +@no_debug_mode +def test_train_example(): + + # path definition + cwd = os.getcwd() + train_path = cwd # change this if you don't want to use the current working directory + try: + os.chdir(train_path) + + # START PRETRAINING + # load and train first layer + train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') + layer1_yaml = open(train_yaml_path, 'r').read() + hyper_params_l1 = {'train_stop' : 60000, + 'batch_size' : 100, + 'monitoring_batches' : 5, + 'nhid' : N_HIDDEN_0, + 'max_epochs' : MAX_EPOCHS_L1, + 'save_path' : train_path + } + + if PRETRAINING: + + layer1_yaml = layer1_yaml % (hyper_params_l1) + train = yaml_parse.load(layer1_yaml) + + print '\n-----------------------------------' + print ' Unsupervised pre-training' + print '-----------------------------------\n' + + print '\nPre-Training first layer...\n' + train.main_loop() + + # load and train second layer + train_yaml_path = os.path.join(train_path, 'dbm_mnist_l2.yaml') + layer2_yaml = open(train_yaml_path, 'r').read() + hyper_params_l2 = {'train_stop' : 60000, + 'batch_size' : 100, + 'monitoring_batches' : 5, + 'nvis' : hyper_params_l1['nhid'], + 'nhid' : N_HIDDEN_1, + 'max_epochs' : MAX_EPOCHS_L2, + 'save_path' : train_path + } + + if PRETRAINING: + + layer2_yaml = layer2_yaml % (hyper_params_l2) + train = yaml_parse.load(layer2_yaml) + + print '\n...Pre-training second layer...\n' + train.main_loop() + + if TRAINING: + + # START TRAINING + if SOFTMAX: + train_yaml_path = os.path.join(train_path, 'dbm_mnist_softmax.yaml') + else: + train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') + yaml = open(train_yaml_path, 'r').read() + hyper_params_dbm = {'train_stop' : 60000, + 'valid_stop' : 60000, + 'batch_size' : 100, + 'detector_layer_1_dim' : hyper_params_l1['nhid'], + 'detector_layer_2_dim' : hyper_params_l2['nhid'], + 'monitoring_batches' : 5, + 'max_epochs' : MAX_EPOCHS_DBM, + 'save_path' : train_path + } + + yaml = yaml % (hyper_params_dbm) + train = yaml_parse.load(yaml) + + rbm1 = serial.load(os.path.join(train_path, 'dbm_mnist_l1.pkl')) + rbm2 = serial.load(os.path.join(train_path, 'dbm_mnist_l2.pkl')) + pretrained_rbms = [rbm1, rbm2] + + # clamp pretrained weights into respective dbm layers + for h, l in zip(train.model.hidden_layers, pretrained_rbms): + h.set_weights(l.get_weights()) + + # clamp pretrained biases into respective dbm layers + bias_param = pretrained_rbms[0].get_params()[1] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.visible_layer.set_biases(bias) + bias_param = pretrained_rbms[-1].get_params()[1] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.hidden_layers[0].set_biases(bias) + bias_param = pretrained_rbms[-1].get_params()[2] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.hidden_layers[1].set_biases(bias) + + print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' + + print '\n-----------------------------------' + print ' Unsupervised training' + print '-----------------------------------\n' + + print '\nTraining phase...' + train.main_loop() + + if FINETUNING: + + # START SUPERVISED TRAINING WITH BACKPROPAGATION + print '\n-----------------------------------' + print ' Supervised training' + print '-----------------------------------' + + # load dbm as a mlp + if DROPOUT: + train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp_dropout.yaml') + else: + train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') + mlp_yaml = open(train_yaml_path, 'r').read() + hyper_params_mlp = {'train_stop': 60000, + 'valid_stop' : 60000, + 'batch_size' : 5000, + 'nvis' : 784 + hyper_params_l2['nhid'], + 'n_h0' : hyper_params_l1['nhid'], + 'n_h1' : hyper_params_l2['nhid'], + 'max_epochs' : MAX_EPOCHS_MLP, + 'save_path' : train_path + } + + mlp_yaml = mlp_yaml % (hyper_params_mlp) + train = yaml_parse.load(mlp_yaml) + + if SOFTMAX: + dbm = serial.load(os.path.join(train_path, 'dbm_mnist_softmax.pkl')) + else: + dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) + + # dataset & monitoring dataset insertion without .yaml file to avoid problems (don't know how to pass 'model' hyperparameter) + train.dataset = mnistaugmented.MNISTAUGMENTED(dataset = train.dataset, which_set = 'train', one_hot = 1, model = dbm, start = 0, stop = hyper_params_mlp['train_stop'], mf_steps = MF_STEPS) + train.algorithm.monitoring_dataset = {#'valid' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), + 'test' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} + + # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON + + # concatenate weights between first and second hidden layer & weights between visible and first hidden layer + train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) + + # then clamp all the others normally + for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): + l.set_weights(h.get_weights()) + + # clamp biases + for l, h in zip(train.model.layers, dbm.hidden_layers): + l.set_biases(h.get_biases()) + + print '\nDBM trained weights and biases have been clamped in the MLP.' + + print '\n...Finetuning...\n' + train.main_loop() + + finally: + os.chdir(cwd) + +if __name__ == '__main__': + test_train_example() \ No newline at end of file diff --git a/pylearn2/training_algorithms/sgd.py b/pylearn2/training_algorithms/sgd.py index ce24008e80..3836ef932a 100644 --- a/pylearn2/training_algorithms/sgd.py +++ b/pylearn2/training_algorithms/sgd.py @@ -1143,3 +1143,21 @@ def on_monitor(self, model, dataset, algorithm): for param in model.get_params(): param.set_value(saved_params[param]) self._count += 1 + +class LRAssigning(object): + def __init__(self): + self.__dict__.update(locals()) + del self.self + self._count = 0 + self.new_lr = 0 + + def __call__(self, algorithm): + if self._count == 0: + self._new_lr = algorithm.learning_rate.get_value() + else: + self.new_lr = 10 / (2000 + self._count) # new learning rate will have this value + + self._count += 1 + new_lr = np.cast[config.floatX](self.new_lr) + algorithm.learning_rate.set_value(new_lr) + From 1269ab1fa77d7a1cf76fd0a3931dbd8ddcf9fadf Mon Sep 17 00:00:00 2001 From: carlo Date: Fri, 17 Oct 2014 15:45:26 +0200 Subject: [PATCH 02/32] corrections for travis.sh --- pylearn2/datasets/augment_input.py | 13 +-- pylearn2/datasets/mnistaugmented.py | 32 +++---- pylearn2/scripts/papers/dbm/test_dbm_mnist.py | 83 ++++++++++--------- 3 files changed, 68 insertions(+), 60 deletions(-) diff --git a/pylearn2/datasets/augment_input.py b/pylearn2/datasets/augment_input.py index 8cc5d11ecb..c5b300154d 100644 --- a/pylearn2/datasets/augment_input.py +++ b/pylearn2/datasets/augment_input.py @@ -2,13 +2,14 @@ from theano import function import numpy + def augment_input(X, model, mf_steps): - + print '\nAugmenting data...\n' - + i = 0 init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') - + for x in X[:]: init_data[0] = x data = sharedX(init_data, name = 'v') @@ -19,9 +20,9 @@ def augment_input(X, model, mf_steps): final_data = numpy.asarray([numpy.concatenate((mp, x))]) else: final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) - + i += 1 - + print 'Data augmentation complete!' - + return final_data \ No newline at end of file diff --git a/pylearn2/datasets/mnistaugmented.py b/pylearn2/datasets/mnistaugmented.py index 40cb78d383..a6c863c4c5 100644 --- a/pylearn2/datasets/mnistaugmented.py +++ b/pylearn2/datasets/mnistaugmented.py @@ -12,19 +12,21 @@ ''' class MNISTAUGMENTED(DenseDesignMatrix): - - def __init__(self, dataset, which_set, model, mf_steps, one_hot = True, - start = None, stop = None): - + + def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, + start=None, stop=None): + path = os.path.join('${PYLEARN2_DATA_PATH}', 'mnist') path = serial.preprocess(path) - + try: if which_set == 'train': - datasets = load_from_dump(dump_data_dir = path, dump_filename = 'aug_train_dump.pkl.gz') + datasets = load_from_dump(dump_data_dir=path, + dump_filename='aug_train_dump.pkl.gz') augmented_X, y = datasets[0], datasets[1] else: - datasets = load_from_dump(dump_data_dir = path, dump_filename = 'aug_test_dump.pkl.gz') + datasets = load_from_dump(dump_data_dir=path, + dump_filename='aug_test_dump.pkl.gz') augmented_X, y = datasets[0], datasets[1] except: X = dataset.X @@ -36,18 +38,20 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot = True, y = one_hot else: y = dataset.y - + # BUILD AUGMENTED INPUT FOR FINETUNING augmented_X = augment_input(X, model, mf_steps) - + datasets = augmented_X, y if which_set == 'train': - save_to_dump(var_to_dump = datasets, dump_data_dir = path, dump_filename = 'aug_train_dump.pkl.gz') + save_to_dump(var_to_dump=datasets, dump_data_dir=path, + dump_filename='aug_train_dump.pkl.gz') else: - save_to_dump(var_to_dump = datasets, dump_data_dir = path, dump_filename = 'aug_test_dump.pkl.gz') + save_to_dump(var_to_dump=datasets, dump_data_dir=path, + dump_filename='aug_test_dump.pkl.gz') augmented_X, y = augmented_X[start:stop], y[start:stop] - super(MNISTAUGMENTED, self).__init__(X = augmented_X, y = y) + super(MNISTAUGMENTED, self).__init__(X=augmented_X, y=y) def load_from_dump(dump_data_dir, dump_filename): load_file = open(dump_data_dir + "/" + dump_filename) @@ -56,6 +60,6 @@ def load_from_dump(dump_data_dir, dump_filename): return unpickled_var def save_to_dump(var_to_dump, dump_data_dir, dump_filename): - save_file = open(dump_data_dir + "/" + dump_filename, 'wb') # this will overwrite current contents - cPickle.dump(var_to_dump, save_file, -1) # the -1 is for HIGHEST_PROTOCOL + save_file = open(dump_data_dir + "/" + dump_filename, 'wb') # this will overwrite current contents + cPickle.dump(var_to_dump, save_file, -1) # the -1 is for HIGHEST_PROTOCOL save_file.close() \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py index 63568a1a79..7241a108d8 100644 --- a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py @@ -3,7 +3,8 @@ __credits__ = ["Carlo D'Eramo, Francesco Visin, Matteo Matteucci"] __license__ = "3-clause BSD" __maintainer__ = "AIR-lab" -__email__ = "carlo.deramo@mail.polimi.it, francesco.visin@polimi.it, matteo.matteucci@polimi.it" +__email__ = "carlo.deramo@mail.polimi.it, francesco.visin@polimi.it, \ +matteo.matteucci@polimi.it" import os import numpy @@ -14,10 +15,7 @@ from pylearn2.datasets import mnistaugmented from theano import function -################## -### PARAMETERS ### -################## - +# PARAMETERS N_HIDDEN_0 = 500 N_HIDDEN_1 = 1000 @@ -36,14 +34,14 @@ # FINETUNING MAX_EPOCHS_MLP = 500 DROPOUT = 0 -MF_STEPS = 1 # mf_steps for data augmentation +MF_STEPS = 1 # mf_steps for data augmentation @no_debug_mode def test_train_example(): # path definition cwd = os.getcwd() - train_path = cwd # change this if you don't want to use the current working directory + train_path = cwd # change this if you don't want to use the current working directory try: os.chdir(train_path) @@ -51,12 +49,12 @@ def test_train_example(): # load and train first layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') layer1_yaml = open(train_yaml_path, 'r').read() - hyper_params_l1 = {'train_stop' : 60000, - 'batch_size' : 100, - 'monitoring_batches' : 5, - 'nhid' : N_HIDDEN_0, - 'max_epochs' : MAX_EPOCHS_L1, - 'save_path' : train_path + hyper_params_l1 = {'train_stop': 60000, + 'batch_size': 100, + 'monitoring_batches': 5, + 'nhid': N_HIDDEN_0, + 'max_epochs': MAX_EPOCHS_L1, + 'save_path': train_path } if PRETRAINING: @@ -74,13 +72,13 @@ def test_train_example(): # load and train second layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l2.yaml') layer2_yaml = open(train_yaml_path, 'r').read() - hyper_params_l2 = {'train_stop' : 60000, - 'batch_size' : 100, - 'monitoring_batches' : 5, - 'nvis' : hyper_params_l1['nhid'], - 'nhid' : N_HIDDEN_1, - 'max_epochs' : MAX_EPOCHS_L2, - 'save_path' : train_path + hyper_params_l2 = {'train_stop': 60000, + 'batch_size': 100, + 'monitoring_batches': 5, + 'nvis': hyper_params_l1['nhid'], + 'nhid': N_HIDDEN_1, + 'max_epochs': MAX_EPOCHS_L2, + 'save_path': train_path } if PRETRAINING: @@ -99,14 +97,14 @@ def test_train_example(): else: train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') yaml = open(train_yaml_path, 'r').read() - hyper_params_dbm = {'train_stop' : 60000, - 'valid_stop' : 60000, - 'batch_size' : 100, - 'detector_layer_1_dim' : hyper_params_l1['nhid'], - 'detector_layer_2_dim' : hyper_params_l2['nhid'], - 'monitoring_batches' : 5, - 'max_epochs' : MAX_EPOCHS_DBM, - 'save_path' : train_path + hyper_params_dbm = {'train_stop': 60000, + 'valid_stop': 60000, + 'batch_size': 100, + 'detector_layer_1_dim': hyper_params_l1['nhid'], + 'detector_layer_2_dim': hyper_params_l2['nhid'], + 'monitoring_batches': 5, + 'max_epochs': MAX_EPOCHS_DBM, + 'save_path': train_path } yaml = yaml % (hyper_params_dbm) @@ -160,13 +158,13 @@ def test_train_example(): train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') mlp_yaml = open(train_yaml_path, 'r').read() hyper_params_mlp = {'train_stop': 60000, - 'valid_stop' : 60000, - 'batch_size' : 5000, - 'nvis' : 784 + hyper_params_l2['nhid'], - 'n_h0' : hyper_params_l1['nhid'], - 'n_h1' : hyper_params_l2['nhid'], - 'max_epochs' : MAX_EPOCHS_MLP, - 'save_path' : train_path + 'valid_stop': 60000, + 'batch_size': 5000, + 'nvis': 784 + hyper_params_l2['nhid'], + 'n_h0': hyper_params_l1['nhid'], + 'n_h1': hyper_params_l2['nhid'], + 'max_epochs': MAX_EPOCHS_MLP, + 'save_path': train_path } mlp_yaml = mlp_yaml % (hyper_params_mlp) @@ -177,15 +175,20 @@ def test_train_example(): else: dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - # dataset & monitoring dataset insertion without .yaml file to avoid problems (don't know how to pass 'model' hyperparameter) - train.dataset = mnistaugmented.MNISTAUGMENTED(dataset = train.dataset, which_set = 'train', one_hot = 1, model = dbm, start = 0, stop = hyper_params_mlp['train_stop'], mf_steps = MF_STEPS) - train.algorithm.monitoring_dataset = {#'valid' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), - 'test' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} + # dataset & monitoring dataset insertion without .yaml file to avoid problems + # (don't know how to pass 'model' hyperparameter) + train.dataset = mnistaugmented.MNISTAUGMENTED(dataset = train.dataset, + which_set = 'train', one_hot = 1, model = dbm, start = 0, + stop = hyper_params_mlp['train_stop'], mf_steps = MF_STEPS) + train.algorithm.monitoring_dataset = { #'valid' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), + 'test' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], + which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON # concatenate weights between first and second hidden layer & weights between visible and first hidden layer - train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) + train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): From 7cfcf8edd0d88db4e2b20fb6fb2cb7a00584708c Mon Sep 17 00:00:00 2001 From: carlo Date: Fri, 17 Oct 2014 23:56:17 +0200 Subject: [PATCH 03/32] corrections for travis.sh --- pylearn2/scripts/papers/dbm/test_dbm_mnist.py | 67 ++++++++++--------- 1 file changed, 34 insertions(+), 33 deletions(-) diff --git a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py index 7241a108d8..2e3b81996e 100644 --- a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py @@ -24,7 +24,7 @@ FINETUNING = 1 # PRETRAINING -MAX_EPOCHS_L1 = 100 +MAX_EPOCHS_L1 = 100 MAX_EPOCHS_L2 = 200 # TRAINING @@ -36,17 +36,18 @@ DROPOUT = 0 MF_STEPS = 1 # mf_steps for data augmentation + @no_debug_mode def test_train_example(): - + # path definition cwd = os.getcwd() train_path = cwd # change this if you don't want to use the current working directory try: os.chdir(train_path) - + # START PRETRAINING - # load and train first layer + # load and train first layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') layer1_yaml = open(train_yaml_path, 'r').read() hyper_params_l1 = {'train_stop': 60000, @@ -56,19 +57,19 @@ def test_train_example(): 'max_epochs': MAX_EPOCHS_L1, 'save_path': train_path } - + if PRETRAINING: - + layer1_yaml = layer1_yaml % (hyper_params_l1) train = yaml_parse.load(layer1_yaml) - + print '\n-----------------------------------' print ' Unsupervised pre-training' print '-----------------------------------\n' - + print '\nPre-Training first layer...\n' train.main_loop() - + # load and train second layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l2.yaml') layer2_yaml = open(train_yaml_path, 'r').read() @@ -80,17 +81,17 @@ def test_train_example(): 'max_epochs': MAX_EPOCHS_L2, 'save_path': train_path } - + if PRETRAINING: - + layer2_yaml = layer2_yaml % (hyper_params_l2) train = yaml_parse.load(layer2_yaml) - + print '\n...Pre-training second layer...\n' train.main_loop() - + if TRAINING: - + # START TRAINING if SOFTMAX: train_yaml_path = os.path.join(train_path, 'dbm_mnist_softmax.yaml') @@ -106,18 +107,18 @@ def test_train_example(): 'max_epochs': MAX_EPOCHS_DBM, 'save_path': train_path } - + yaml = yaml % (hyper_params_dbm) train = yaml_parse.load(yaml) rbm1 = serial.load(os.path.join(train_path, 'dbm_mnist_l1.pkl')) rbm2 = serial.load(os.path.join(train_path, 'dbm_mnist_l2.pkl')) pretrained_rbms = [rbm1, rbm2] - + # clamp pretrained weights into respective dbm layers for h, l in zip(train.model.hidden_layers, pretrained_rbms): h.set_weights(l.get_weights()) - + # clamp pretrained biases into respective dbm layers bias_param = pretrained_rbms[0].get_params()[1] fun = function([], bias_param) @@ -134,23 +135,23 @@ def test_train_example(): cuda_bias = fun() bias = numpy.asarray(cuda_bias) train.model.hidden_layers[1].set_biases(bias) - + print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' print '\n-----------------------------------' print ' Unsupervised training' print '-----------------------------------\n' - + print '\nTraining phase...' train.main_loop() - + if FINETUNING: - + # START SUPERVISED TRAINING WITH BACKPROPAGATION print '\n-----------------------------------' print ' Supervised training' print '-----------------------------------' - + # load dbm as a mlp if DROPOUT: train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp_dropout.yaml') @@ -166,15 +167,15 @@ def test_train_example(): 'max_epochs': MAX_EPOCHS_MLP, 'save_path': train_path } - + mlp_yaml = mlp_yaml % (hyper_params_mlp) train = yaml_parse.load(mlp_yaml) - + if SOFTMAX: dbm = serial.load(os.path.join(train_path, 'dbm_mnist_softmax.pkl')) else: dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - + # dataset & monitoring dataset insertion without .yaml file to avoid problems # (don't know how to pass 'model' hyperparameter) train.dataset = mnistaugmented.MNISTAUGMENTED(dataset = train.dataset, @@ -182,27 +183,27 @@ def test_train_example(): stop = hyper_params_mlp['train_stop'], mf_steps = MF_STEPS) train.algorithm.monitoring_dataset = { #'valid' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), 'test' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], - which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} - + which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} + # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON - + # concatenate weights between first and second hidden layer & weights between visible and first hidden layer train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) - + # then clamp all the others normally for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): l.set_weights(h.get_weights()) - + # clamp biases for l, h in zip(train.model.layers, dbm.hidden_layers): l.set_biases(h.get_biases()) - + print '\nDBM trained weights and biases have been clamped in the MLP.' - + print '\n...Finetuning...\n' train.main_loop() - + finally: os.chdir(cwd) From e72ebd453e9fa1159addf523efc654b4c18fa264 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 18 Oct 2014 23:31:20 +0200 Subject: [PATCH 04/32] corrections for travis.sh --- pylearn2/datasets/augment_input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/datasets/augment_input.py b/pylearn2/datasets/augment_input.py index c5b300154d..8b2eafb412 100644 --- a/pylearn2/datasets/augment_input.py +++ b/pylearn2/datasets/augment_input.py @@ -13,7 +13,7 @@ def augment_input(X, model, mf_steps): for x in X[:]: init_data[0] = x data = sharedX(init_data, name = 'v') - marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) + marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) mp = function([], marginal_posterior) mp = mp()[0][0] if i == 0: From 0a1b1f752b56ed4db28c1c9aa04b5769a38451c9 Mon Sep 17 00:00:00 2001 From: carlo Date: Fri, 24 Oct 2014 19:05:38 +0200 Subject: [PATCH 05/32] corrections for pr --- pylearn2/datasets/augment_input.py | 28 ----- .../{mnistaugmented.py => mnist_augmented.py} | 32 ++--- pylearn2/scripts/papers/dbm/README | 6 +- pylearn2/scripts/papers/dbm/augment_input.py | 119 ++++++++++++++++++ pylearn2/scripts/papers/dbm/dbm_mnist.yaml | 5 +- pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml | 3 +- pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml | 3 +- .../scripts/papers/dbm/dbm_mnist_mlp.yaml | 3 +- .../papers/dbm/dbm_mnist_mlp_dropout.yaml | 3 +- .../scripts/papers/dbm/dbm_mnist_softmax.yaml | 5 +- pylearn2/scripts/papers/dbm/test_dbm_mnist.py | 8 +- pylearn2/training_algorithms/sgd.py | 7 +- 12 files changed, 157 insertions(+), 65 deletions(-) delete mode 100644 pylearn2/datasets/augment_input.py rename pylearn2/datasets/{mnistaugmented.py => mnist_augmented.py} (51%) create mode 100644 pylearn2/scripts/papers/dbm/augment_input.py diff --git a/pylearn2/datasets/augment_input.py b/pylearn2/datasets/augment_input.py deleted file mode 100644 index 8b2eafb412..0000000000 --- a/pylearn2/datasets/augment_input.py +++ /dev/null @@ -1,28 +0,0 @@ -from pylearn2.utils import sharedX -from theano import function -import numpy - - -def augment_input(X, model, mf_steps): - - print '\nAugmenting data...\n' - - i = 0 - init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') - - for x in X[:]: - init_data[0] = x - data = sharedX(init_data, name = 'v') - marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) - mp = function([], marginal_posterior) - mp = mp()[0][0] - if i == 0: - final_data = numpy.asarray([numpy.concatenate((mp, x))]) - else: - final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) - - i += 1 - - print 'Data augmentation complete!' - - return final_data \ No newline at end of file diff --git a/pylearn2/datasets/mnistaugmented.py b/pylearn2/datasets/mnist_augmented.py similarity index 51% rename from pylearn2/datasets/mnistaugmented.py rename to pylearn2/datasets/mnist_augmented.py index a6c863c4c5..9cd8f54ee0 100644 --- a/pylearn2/datasets/mnistaugmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -1,9 +1,8 @@ import os -import cPickle import numpy as np from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix -from pylearn2.datasets.augment_input import augment_input +from pylearn2.scripts.papers.dbm import augment_input from pylearn2.utils import serial ''' @@ -11,7 +10,7 @@ for DBM discriminative finetuning ''' -class MNISTAUGMENTED(DenseDesignMatrix): +class MNIST_AUGMENTED(DenseDesignMatrix): def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, start=None, stop=None): @@ -21,12 +20,10 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, try: if which_set == 'train': - datasets = load_from_dump(dump_data_dir=path, - dump_filename='aug_train_dump.pkl.gz') + datasets = serial.load(filepath=path + 'aug_train_dump.pkl.gz') augmented_X, y = datasets[0], datasets[1] else: - datasets = load_from_dump(dump_data_dir=path, - dump_filename='aug_test_dump.pkl.gz') + datasets = serial.load(filepath=path + 'aug_test_dump.pkl.gz') augmented_X, y = datasets[0], datasets[1] except: X = dataset.X @@ -44,22 +41,11 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, datasets = augmented_X, y if which_set == 'train': - save_to_dump(var_to_dump=datasets, dump_data_dir=path, - dump_filename='aug_train_dump.pkl.gz') + serial.save(filepath=path + 'aug_train_dump.pkl.gz', + obj=datasets) else: - save_to_dump(var_to_dump=datasets, dump_data_dir=path, - dump_filename='aug_test_dump.pkl.gz') + serial.save(filepath=path + 'aug_test_dump.pkl.gz', + obj=datasets) augmented_X, y = augmented_X[start:stop], y[start:stop] - super(MNISTAUGMENTED, self).__init__(X=augmented_X, y=y) - -def load_from_dump(dump_data_dir, dump_filename): - load_file = open(dump_data_dir + "/" + dump_filename) - unpickled_var = cPickle.load(load_file) - load_file.close() - return unpickled_var - -def save_to_dump(var_to_dump, dump_data_dir, dump_filename): - save_file = open(dump_data_dir + "/" + dump_filename, 'wb') # this will overwrite current contents - cPickle.dump(var_to_dump, save_file, -1) # the -1 is for HIGHEST_PROTOCOL - save_file.close() \ No newline at end of file + super(MNIST_AUGMENTED, self).__init__(X=augmented_X, y=y) diff --git a/pylearn2/scripts/papers/dbm/README b/pylearn2/scripts/papers/dbm/README index dc55e9b031..dfb3b34ac7 100644 --- a/pylearn2/scripts/papers/dbm/README +++ b/pylearn2/scripts/papers/dbm/README @@ -13,4 +13,8 @@ because data augmentation is a time-consuming operation. NO DROPOUT RESULTS: The procedure without dropout does not use a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0094%. DROPOUT RESULTS: -The procedure with dropout uses a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0084%. \ No newline at end of file +The procedure with dropout uses a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0084%. + +Experiments have been performed on Ubuntu 14.04 LTS using a NVIDIA Tesla C1060 GPU and a 8-core Intel(R) Core(TM) i7 CPU 920 @ 2.67GHz. I used openblas-base and numpy version 1.9.0, +scipy version 0.13.3, theano version 0.6.0 and pylearn2 with 6264 commits. + \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/augment_input.py b/pylearn2/scripts/papers/dbm/augment_input.py new file mode 100644 index 0000000000..4751880328 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/augment_input.py @@ -0,0 +1,119 @@ +from pylearn2.utils import sharedX +from theano import function +import numpy + +""" +This module augments the dataset in order to make it suitable for +DBM discriminative finetuning. +For each example in the dataset, using the provided trained DBM, +it performs n mean-field updates initializing the state of the second +hidden layer of the DBM and augments the example with this state. +It returns a dataset where each example is composed of its previous +value concatenated with the respective initialization of the second +hidden layer of the DBM. + +Parameters +---------- +X : ndarray, 2-dimensional + a matrix containing the initial dataset +model : DBM + the DBM model to be finetuned. It is used for + mean field updates +mf_steps : int + the number of mean field updates + +Returns +------- +final_data : ndarray, 2-dimensional + the final augmented dataset + +References +---------- +Salakhutdinov Ruslan and Hinton Geoffrey. "An efficient +procedure for deep boltzmann machines". 2012. +""" + +'''def augment_input(X, model, mf_steps): + + print '\nAugmenting data...\n' + + i = 0 + init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') + + for x in X[:]: + init_data[0] = x + data = sharedX(init_data, name = 'v') + marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) + mp = function([], marginal_posterior) + mp = mp()[0][0] + if i == 0: + final_data = numpy.asarray([numpy.concatenate((mp, x))]) + else: + final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) + + i += 1 + + print 'Data augmentation complete!' + + return final_data''' + +from multiprocessing import Process, Queue + +def augment(X, model, mf_steps, q): + + print '\nAugmenting data...\n' + + i = 0 + init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') + + for x in X[:]: + init_data[0] = x + data = sharedX(init_data, name = 'v') + marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) + mp = function([], marginal_posterior) + mp = mp()[0][0] + if i == 0: + final_data = numpy.asarray([numpy.concatenate((mp, x))]) + else: + final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) + + i += 1 + + print 'Data augmentation complete!' + + q.put(final_data) + +def augment_input(X, model, mf_steps): + + q1 = Queue() + q2 = Queue() + q3 = Queue() + q4 = Queue() + q5 = Queue() + q6 = Queue() + p1 = Process(target = augment, args = (X, model, mf_steps, q1)) + p2 = Process(target = augment, args = (X, model, mf_steps, q2)) + p3 = Process(target = augment, args = (X, model, mf_steps, q3)) + p4 = Process(target = augment, args = (X, model, mf_steps, q4)) + p5 = Process(target = augment, args = (X, model, mf_steps, q5)) + p6 = Process(target = augment, args = (X, model, mf_steps, q6)) + processes = [p1, p2, p3, p4, p5, p6] + + for p in processes: + p.start() + + p1.join() + p2.join() + p3.join() + p4.join() + p5.join() + p6.join() + + d1 = q1.get() + d2 = q2.get() + d3 = q3.get() + d4 = q4.get() + d5 = q5.get() + d6 = q6.get() + + return numpy.concatenate((d1,d2,d3,d4,d5,d6), axis = 0) \ No newline at end of file diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml index c3c7ae0538..87b44d1538 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml @@ -60,7 +60,7 @@ max_epochs: %(max_epochs)i }, update_callbacks: [ - !obj:pylearn2.training_algorithms.sgd.LRAssigning { + !obj:pylearn2.training_algorithms.sgd.CustomizedLROverEpoch { } ] }, @@ -73,4 +73,5 @@ ], save_path: "%(save_path)s/dbm_mnist.pkl", save_freq : %(max_epochs)i -} \ No newline at end of file +} + diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml index e718aa98f7..ccfe01e050 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml @@ -36,4 +36,5 @@ ], save_path: "%(save_path)s/dbm_mnist_l1.pkl", save_freq: %(max_epochs)i -} \ No newline at end of file +} + diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml index 6d19ff323a..2c3cd63249 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml @@ -39,4 +39,5 @@ ], save_path: "%(save_path)s/dbm_mnist_l2.pkl", save_freq: %(max_epochs)i -} \ No newline at end of file +} + diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml index 0ee9ca836c..448e15c9bb 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml @@ -52,4 +52,5 @@ store_best_model: True }, ] -} \ No newline at end of file +} + diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml index 6ad38d0605..84a32e23d5 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml @@ -60,4 +60,5 @@ store_best_model: True }, ] -} \ No newline at end of file +} + diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml index b9435815f8..8276672e40 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml @@ -63,7 +63,7 @@ max_epochs: %(max_epochs)i }, update_callbacks: [ - !obj:pylearn2.training_algorithms.sgd.LRAssigning { + !obj:pylearn2.training_algorithms.sgd.CustomizedLROverEpoch { } ] }, @@ -79,4 +79,5 @@ store_best_model: True }, ], -} \ No newline at end of file +} + diff --git a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py index 2e3b81996e..7ba2169cd4 100644 --- a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/test_dbm_mnist.py @@ -12,7 +12,7 @@ from pylearn2.utils import serial from pylearn2.config import yaml_parse from pylearn2.testing import no_debug_mode -from pylearn2.datasets import mnistaugmented +from pylearn2.datasets import mnist_augmented from theano import function # PARAMETERS @@ -178,11 +178,11 @@ def test_train_example(): # dataset & monitoring dataset insertion without .yaml file to avoid problems # (don't know how to pass 'model' hyperparameter) - train.dataset = mnistaugmented.MNISTAUGMENTED(dataset = train.dataset, + train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset = train.dataset, which_set = 'train', one_hot = 1, model = dbm, start = 0, stop = hyper_params_mlp['train_stop'], mf_steps = MF_STEPS) - train.algorithm.monitoring_dataset = { #'valid' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), - 'test' : mnistaugmented.MNISTAUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], + train.algorithm.monitoring_dataset = { #'valid' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), + 'test' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON diff --git a/pylearn2/training_algorithms/sgd.py b/pylearn2/training_algorithms/sgd.py index 3836ef932a..a42d3738b1 100644 --- a/pylearn2/training_algorithms/sgd.py +++ b/pylearn2/training_algorithms/sgd.py @@ -1144,7 +1144,12 @@ def on_monitor(self, model, dataset, algorithm): param.set_value(saved_params[param]) self._count += 1 -class LRAssigning(object): +class CustomizedLROverEpoch(object): + """ + Assigns a different value to learning rate at each + training epoch. This is used when the value to be assigned + is not reproducible with linear or exponential decay. + """ def __init__(self): self.__dict__.update(locals()) del self.self From d75713b89fe47cd21625787e975033bdd831607c Mon Sep 17 00:00:00 2001 From: carlo Date: Fri, 24 Oct 2014 19:12:17 +0200 Subject: [PATCH 06/32] corrections for pr --- pylearn2/datasets/mnist_augmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 9cd8f54ee0..404ab680a6 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -2,7 +2,7 @@ import numpy as np from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix -from pylearn2.scripts.papers.dbm import augment_input +from pylearn2.scripts.papers.dbm.augment_input import augment_input from pylearn2.utils import serial ''' From d4d505c1d336d5fa3c29d60c473094d11cef959e Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 10:21:54 +0200 Subject: [PATCH 07/32] Corrections for pr. This corrections include adding of new lines, README update, some files moved in other directories --- pylearn2/datasets/mnist_augmented.py | 2 +- pylearn2/scripts/{papers => }/dbm/augment_input.py | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename pylearn2/scripts/{papers => }/dbm/augment_input.py (100%) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 404ab680a6..ea197df149 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -2,7 +2,7 @@ import numpy as np from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix -from pylearn2.scripts.papers.dbm.augment_input import augment_input +from pylearn2.scripts.dbm.augment_input import augment_input from pylearn2.utils import serial ''' diff --git a/pylearn2/scripts/papers/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py similarity index 100% rename from pylearn2/scripts/papers/dbm/augment_input.py rename to pylearn2/scripts/dbm/augment_input.py From 875a6bdc85b2efa3a016755bb50b9f71afc1f37c Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 11:43:21 +0200 Subject: [PATCH 08/32] added new line --- pylearn2/scripts/dbm/augment_input.py | 65 +-------------------------- 1 file changed, 2 insertions(+), 63 deletions(-) diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index 4751880328..aa3506fb76 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -33,33 +33,7 @@ procedure for deep boltzmann machines". 2012. """ -'''def augment_input(X, model, mf_steps): - - print '\nAugmenting data...\n' - - i = 0 - init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') - - for x in X[:]: - init_data[0] = x - data = sharedX(init_data, name = 'v') - marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) - mp = function([], marginal_posterior) - mp = mp()[0][0] - if i == 0: - final_data = numpy.asarray([numpy.concatenate((mp, x))]) - else: - final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) - - i += 1 - - print 'Data augmentation complete!' - - return final_data''' - -from multiprocessing import Process, Queue - -def augment(X, model, mf_steps, q): +def augment_input(X, model, mf_steps): print '\nAugmenting data...\n' @@ -81,39 +55,4 @@ def augment(X, model, mf_steps, q): print 'Data augmentation complete!' - q.put(final_data) - -def augment_input(X, model, mf_steps): - - q1 = Queue() - q2 = Queue() - q3 = Queue() - q4 = Queue() - q5 = Queue() - q6 = Queue() - p1 = Process(target = augment, args = (X, model, mf_steps, q1)) - p2 = Process(target = augment, args = (X, model, mf_steps, q2)) - p3 = Process(target = augment, args = (X, model, mf_steps, q3)) - p4 = Process(target = augment, args = (X, model, mf_steps, q4)) - p5 = Process(target = augment, args = (X, model, mf_steps, q5)) - p6 = Process(target = augment, args = (X, model, mf_steps, q6)) - processes = [p1, p2, p3, p4, p5, p6] - - for p in processes: - p.start() - - p1.join() - p2.join() - p3.join() - p4.join() - p5.join() - p6.join() - - d1 = q1.get() - d2 = q2.get() - d3 = q3.get() - d4 = q4.get() - d5 = q5.get() - d6 = q6.get() - - return numpy.concatenate((d1,d2,d3,d4,d5,d6), axis = 0) \ No newline at end of file + return final_data From f37c42acdc4cda1a0f94ccdf24eda877347b36b3 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 13:02:30 +0200 Subject: [PATCH 09/32] fast version of test added --- pylearn2/scripts/papers/dbm/README | 4 +- .../papers/dbm/{ => tests}/test_dbm_mnist.py | 0 .../papers/dbm/tests/test_dbm_mnist_fast.py | 195 ++++++++++++++++++ 3 files changed, 197 insertions(+), 2 deletions(-) rename pylearn2/scripts/papers/dbm/{ => tests}/test_dbm_mnist.py (100%) create mode 100644 pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py diff --git a/pylearn2/scripts/papers/dbm/README b/pylearn2/scripts/papers/dbm/README index dfb3b34ac7..3716801621 100644 --- a/pylearn2/scripts/papers/dbm/README +++ b/pylearn2/scripts/papers/dbm/README @@ -11,9 +11,9 @@ mnistaugmented.py in pylearn2/datasets/. The latter takes the mnist dataset and because data augmentation is a time-consuming operation. NO DROPOUT RESULTS: -The procedure without dropout does not use a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0094%. + DROPOUT RESULTS: -The procedure with dropout uses a softmax layer for the DBM and using all the provided parameters it should give you a test_y_misclass of 0.0084%. + Experiments have been performed on Ubuntu 14.04 LTS using a NVIDIA Tesla C1060 GPU and a 8-core Intel(R) Core(TM) i7 CPU 920 @ 2.67GHz. I used openblas-base and numpy version 1.9.0, scipy version 0.13.3, theano version 0.6.0 and pylearn2 with 6264 commits. diff --git a/pylearn2/scripts/papers/dbm/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py similarity index 100% rename from pylearn2/scripts/papers/dbm/test_dbm_mnist.py rename to pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py new file mode 100644 index 0000000000..cadde27221 --- /dev/null +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -0,0 +1,195 @@ +__authors__ = "Carlo D'Eramo, Francesco Visin, Matteo Matteucci" +__copyright__ = "Copyright 2014-2015, Politecnico di Milano" +__credits__ = ["Carlo D'Eramo, Francesco Visin, Matteo Matteucci"] +__license__ = "3-clause BSD" +__maintainer__ = "AIR-lab" +__email__ = "carlo.deramo@mail.polimi.it, francesco.visin@polimi.it, \ +matteo.matteucci@polimi.it" + +import os +import numpy + +from pylearn2.utils import serial +from pylearn2.config import yaml_parse +from pylearn2.testing import no_debug_mode +from pylearn2.datasets import mnist_augmented +from theano import function + +PRETRAINING = 1 +TRAINING = 1 +FINETUNING = 1 +SOFTMAX = 0 +DROPOUT = 0 + +@no_debug_mode +def test_train_example(): + + # path definition + cwd = os.getcwd() + train_path = cwd # change this if you don't want to use the current working directory + try: + os.chdir(train_path) + + # START PRETRAINING + # load and train first layer + train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') + layer1_yaml = open(train_yaml_path, 'r').read() + hyper_params_l1 = {'train_stop': 5000, + 'batch_size': 100, + 'monitoring_batches': 5, + 'nhid': 100, + 'max_epochs': 10, + 'save_path': train_path + } + + if PRETRAINING: + + layer1_yaml = layer1_yaml % (hyper_params_l1) + train = yaml_parse.load(layer1_yaml) + + print '\n-----------------------------------' + print ' Unsupervised pre-training' + print '-----------------------------------\n' + + print '\nPre-Training first layer...\n' + train.main_loop() + + # load and train second layer + train_yaml_path = os.path.join(train_path, 'dbm_mnist_l2.yaml') + layer2_yaml = open(train_yaml_path, 'r').read() + hyper_params_l2 = {'train_stop': 1000, + 'batch_size': 100, + 'monitoring_batches': 5, + 'nvis': hyper_params_l1['nhid'], + 'nhid': 200, + 'max_epochs': 20, + 'save_path': train_path + } + + if PRETRAINING: + + layer2_yaml = layer2_yaml % (hyper_params_l2) + train = yaml_parse.load(layer2_yaml) + + print '\n...Pre-training second layer...\n' + train.main_loop() + + if TRAINING: + + # START TRAINING + if SOFTMAX: + train_yaml_path = os.path.join(train_path, 'dbm_mnist_softmax.yaml') + else: + train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') + yaml = open(train_yaml_path, 'r').read() + hyper_params_dbm = {'train_stop': 1000, + 'valid_stop': 1000, + 'batch_size': 100, + 'detector_layer_1_dim': hyper_params_l1['nhid'], + 'detector_layer_2_dim': hyper_params_l2['nhid'], + 'monitoring_batches': 5, + 'max_epochs': 20, + 'save_path': train_path + } + + yaml = yaml % (hyper_params_dbm) + train = yaml_parse.load(yaml) + + rbm1 = serial.load(os.path.join(train_path, 'dbm_mnist_l1.pkl')) + rbm2 = serial.load(os.path.join(train_path, 'dbm_mnist_l2.pkl')) + pretrained_rbms = [rbm1, rbm2] + + # clamp pretrained weights into respective dbm layers + for h, l in zip(train.model.hidden_layers, pretrained_rbms): + h.set_weights(l.get_weights()) + + # clamp pretrained biases into respective dbm layers + bias_param = pretrained_rbms[0].get_params()[1] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.visible_layer.set_biases(bias) + bias_param = pretrained_rbms[-1].get_params()[1] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.hidden_layers[0].set_biases(bias) + bias_param = pretrained_rbms[-1].get_params()[2] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.hidden_layers[1].set_biases(bias) + + print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' + + print '\n-----------------------------------' + print ' Unsupervised training' + print '-----------------------------------\n' + + print '\nTraining phase...' + train.main_loop() + + if FINETUNING: + + # START SUPERVISED TRAINING WITH BACKPROPAGATION + print '\n-----------------------------------' + print ' Supervised training' + print '-----------------------------------' + + # load dbm as a mlp + if DROPOUT: + train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp_dropout.yaml') + else: + train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') + mlp_yaml = open(train_yaml_path, 'r').read() + hyper_params_mlp = {'train_stop': 1000, + 'valid_stop': 1000, + 'batch_size': 200, + 'nvis': 784 + hyper_params_l2['nhid'], + 'n_h0': hyper_params_l1['nhid'], + 'n_h1': hyper_params_l2['nhid'], + 'max_epochs': 100, + 'save_path': train_path + } + + mlp_yaml = mlp_yaml % (hyper_params_mlp) + train = yaml_parse.load(mlp_yaml) + + if SOFTMAX: + dbm = serial.load(os.path.join(train_path, 'dbm_mnist_softmax.pkl')) + else: + dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) + + # dataset & monitoring dataset insertion without .yaml file to avoid problems + # (don't know how to pass 'model' hyperparameter) + train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset = train.dataset, + which_set = 'train', one_hot = 1, model = dbm, start = 0, + stop = hyper_params_mlp['train_stop'], mf_steps = 1) + train.algorithm.monitoring_dataset = { #'valid' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = 1), + 'test' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], + which_set = 'test', one_hot = 1, model = dbm, mf_steps = 1)} + + # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON + + # concatenate weights between first and second hidden layer & weights between visible and first hidden layer + train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) + + # then clamp all the others normally + for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): + l.set_weights(h.get_weights()) + + # clamp biases + for l, h in zip(train.model.layers, dbm.hidden_layers): + l.set_biases(h.get_biases()) + + print '\nDBM trained weights and biases have been clamped in the MLP.' + + print '\n...Finetuning...\n' + train.main_loop() + + finally: + os.chdir(cwd) + +if __name__ == '__main__': + test_train_example() \ No newline at end of file From ae20246069d546ccb1ee529b6d69ecaa0d181f80 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 14:16:58 +0200 Subject: [PATCH 10/32] correction to fast version of test script --- .../papers/dbm/tests/test_dbm_mnist_fast.py | 258 ++++++++---------- 1 file changed, 118 insertions(+), 140 deletions(-) diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index cadde27221..7e06ec39cf 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -15,12 +15,6 @@ from pylearn2.datasets import mnist_augmented from theano import function -PRETRAINING = 1 -TRAINING = 1 -FINETUNING = 1 -SOFTMAX = 0 -DROPOUT = 0 - @no_debug_mode def test_train_example(): @@ -34,31 +28,29 @@ def test_train_example(): # load and train first layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') layer1_yaml = open(train_yaml_path, 'r').read() - hyper_params_l1 = {'train_stop': 5000, - 'batch_size': 100, + hyper_params_l1 = {'train_stop': 20, + 'batch_size': 5, 'monitoring_batches': 5, 'nhid': 100, 'max_epochs': 10, 'save_path': train_path } - if PRETRAINING: - - layer1_yaml = layer1_yaml % (hyper_params_l1) - train = yaml_parse.load(layer1_yaml) + layer1_yaml = layer1_yaml % (hyper_params_l1) + train = yaml_parse.load(layer1_yaml) - print '\n-----------------------------------' - print ' Unsupervised pre-training' - print '-----------------------------------\n' + print '\n-----------------------------------' + print ' Unsupervised pre-training' + print '-----------------------------------\n' - print '\nPre-Training first layer...\n' - train.main_loop() + print '\nPre-Training first layer...\n' + train.main_loop() # load and train second layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l2.yaml') layer2_yaml = open(train_yaml_path, 'r').read() - hyper_params_l2 = {'train_stop': 1000, - 'batch_size': 100, + hyper_params_l2 = {'train_stop': 20, + 'batch_size': 5, 'monitoring_batches': 5, 'nvis': hyper_params_l1['nhid'], 'nhid': 200, @@ -66,127 +58,113 @@ def test_train_example(): 'save_path': train_path } - if PRETRAINING: - - layer2_yaml = layer2_yaml % (hyper_params_l2) - train = yaml_parse.load(layer2_yaml) - - print '\n...Pre-training second layer...\n' - train.main_loop() - - if TRAINING: - - # START TRAINING - if SOFTMAX: - train_yaml_path = os.path.join(train_path, 'dbm_mnist_softmax.yaml') - else: - train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') - yaml = open(train_yaml_path, 'r').read() - hyper_params_dbm = {'train_stop': 1000, - 'valid_stop': 1000, - 'batch_size': 100, - 'detector_layer_1_dim': hyper_params_l1['nhid'], - 'detector_layer_2_dim': hyper_params_l2['nhid'], - 'monitoring_batches': 5, - 'max_epochs': 20, - 'save_path': train_path - } - - yaml = yaml % (hyper_params_dbm) - train = yaml_parse.load(yaml) - - rbm1 = serial.load(os.path.join(train_path, 'dbm_mnist_l1.pkl')) - rbm2 = serial.load(os.path.join(train_path, 'dbm_mnist_l2.pkl')) - pretrained_rbms = [rbm1, rbm2] - - # clamp pretrained weights into respective dbm layers - for h, l in zip(train.model.hidden_layers, pretrained_rbms): - h.set_weights(l.get_weights()) - - # clamp pretrained biases into respective dbm layers - bias_param = pretrained_rbms[0].get_params()[1] - fun = function([], bias_param) - cuda_bias = fun() - bias = numpy.asarray(cuda_bias) - train.model.visible_layer.set_biases(bias) - bias_param = pretrained_rbms[-1].get_params()[1] - fun = function([], bias_param) - cuda_bias = fun() - bias = numpy.asarray(cuda_bias) - train.model.hidden_layers[0].set_biases(bias) - bias_param = pretrained_rbms[-1].get_params()[2] - fun = function([], bias_param) - cuda_bias = fun() - bias = numpy.asarray(cuda_bias) - train.model.hidden_layers[1].set_biases(bias) - - print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' - - print '\n-----------------------------------' - print ' Unsupervised training' - print '-----------------------------------\n' - - print '\nTraining phase...' - train.main_loop() - - if FINETUNING: - - # START SUPERVISED TRAINING WITH BACKPROPAGATION - print '\n-----------------------------------' - print ' Supervised training' - print '-----------------------------------' - - # load dbm as a mlp - if DROPOUT: - train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp_dropout.yaml') - else: - train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') - mlp_yaml = open(train_yaml_path, 'r').read() - hyper_params_mlp = {'train_stop': 1000, - 'valid_stop': 1000, - 'batch_size': 200, - 'nvis': 784 + hyper_params_l2['nhid'], - 'n_h0': hyper_params_l1['nhid'], - 'n_h1': hyper_params_l2['nhid'], - 'max_epochs': 100, - 'save_path': train_path - } - - mlp_yaml = mlp_yaml % (hyper_params_mlp) - train = yaml_parse.load(mlp_yaml) - - if SOFTMAX: - dbm = serial.load(os.path.join(train_path, 'dbm_mnist_softmax.pkl')) - else: - dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - - # dataset & monitoring dataset insertion without .yaml file to avoid problems - # (don't know how to pass 'model' hyperparameter) - train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset = train.dataset, - which_set = 'train', one_hot = 1, model = dbm, start = 0, - stop = hyper_params_mlp['train_stop'], mf_steps = 1) - train.algorithm.monitoring_dataset = { #'valid' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = 1), - 'test' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], - which_set = 'test', one_hot = 1, model = dbm, mf_steps = 1)} - - # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON - - # concatenate weights between first and second hidden layer & weights between visible and first hidden layer - train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), - dbm.hidden_layers[0].get_weights()))) - - # then clamp all the others normally - for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): - l.set_weights(h.get_weights()) - - # clamp biases - for l, h in zip(train.model.layers, dbm.hidden_layers): - l.set_biases(h.get_biases()) - - print '\nDBM trained weights and biases have been clamped in the MLP.' - - print '\n...Finetuning...\n' - train.main_loop() + layer2_yaml = layer2_yaml % (hyper_params_l2) + train = yaml_parse.load(layer2_yaml) + + print '\n...Pre-training second layer...\n' + train.main_loop() + + # START TRAINING + train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') + yaml = open(train_yaml_path, 'r').read() + hyper_params_dbm = {'train_stop': 20, + 'valid_stop': 20, + 'batch_size': 5, + 'detector_layer_1_dim': hyper_params_l1['nhid'], + 'detector_layer_2_dim': hyper_params_l2['nhid'], + 'monitoring_batches': 5, + 'max_epochs': 5, + 'save_path': train_path + } + + yaml = yaml % (hyper_params_dbm) + train = yaml_parse.load(yaml) + + rbm1 = serial.load(os.path.join(train_path, 'dbm_mnist_l1.pkl')) + rbm2 = serial.load(os.path.join(train_path, 'dbm_mnist_l2.pkl')) + pretrained_rbms = [rbm1, rbm2] + + # clamp pretrained weights into respective dbm layers + for h, l in zip(train.model.hidden_layers, pretrained_rbms): + h.set_weights(l.get_weights()) + + # clamp pretrained biases into respective dbm layers + bias_param = pretrained_rbms[0].get_params()[1] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.visible_layer.set_biases(bias) + bias_param = pretrained_rbms[-1].get_params()[1] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.hidden_layers[0].set_biases(bias) + bias_param = pretrained_rbms[-1].get_params()[2] + fun = function([], bias_param) + cuda_bias = fun() + bias = numpy.asarray(cuda_bias) + train.model.hidden_layers[1].set_biases(bias) + + print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' + + print '\n-----------------------------------' + print ' Unsupervised training' + print '-----------------------------------\n' + + print '\nTraining phase...' + train.main_loop() + + # START SUPERVISED TRAINING WITH BACKPROPAGATION + print '\n-----------------------------------' + print ' Supervised training' + print '-----------------------------------' + + # load dbm as a mlp + train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') + + mlp_yaml = open(train_yaml_path, 'r').read() + hyper_params_mlp = {'train_stop': 20, + 'valid_stop': 20, + 'batch_size': 5, + 'nvis': 784 + hyper_params_l2['nhid'], + 'n_h0': hyper_params_l1['nhid'], + 'n_h1': hyper_params_l2['nhid'], + 'max_epochs': 20, + 'save_path': train_path + } + + mlp_yaml = mlp_yaml % (hyper_params_mlp) + train = yaml_parse.load(mlp_yaml) + + dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) + + # dataset & monitoring dataset insertion without .yaml file to avoid problems + # (don't know how to pass 'model' hyperparameter) + train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset = train.dataset, + which_set = 'train', one_hot = 1, model = dbm, start = 0, + stop = hyper_params_mlp['train_stop'], mf_steps = 1) + train.algorithm.monitoring_dataset = { #'valid' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = 1), + 'test' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], + which_set = 'test', one_hot = 1, model = dbm, start = 0, stop = 20, mf_steps = 1)} + + # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON + + # concatenate weights between first and second hidden layer & weights between visible and first hidden layer + train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) + + # then clamp all the others normally + for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): + l.set_weights(h.get_weights()) + + # clamp biases + for l, h in zip(train.model.layers, dbm.hidden_layers): + l.set_biases(h.get_biases()) + + print '\nDBM trained weights and biases have been clamped in the MLP.' + + print '\n...Finetuning...\n' + train.main_loop() finally: os.chdir(cwd) From 266e4063e552049b29bb460d257860e8d44f7aff Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 14:36:22 +0200 Subject: [PATCH 11/32] added save_aug flag to mnist_augmented to choose whether or not to save augmented datasets. corrections on fast test script --- pylearn2/datasets/mnist_augmented.py | 18 ++++++++++-------- .../papers/dbm/tests/test_dbm_mnist_fast.py | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index ea197df149..23fabdb512 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -13,7 +13,7 @@ class MNIST_AUGMENTED(DenseDesignMatrix): def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, - start=None, stop=None): + start=None, stop=None, save_aug=False): path = os.path.join('${PYLEARN2_DATA_PATH}', 'mnist') path = serial.preprocess(path) @@ -25,6 +25,7 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, else: datasets = serial.load(filepath=path + 'aug_test_dump.pkl.gz') augmented_X, y = datasets[0], datasets[1] + augmented_X, y = augmented_X[start:stop], y[start:stop] except: X = dataset.X if one_hot: @@ -37,15 +38,16 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, y = dataset.y # BUILD AUGMENTED INPUT FOR FINETUNING + X, y = X[start:stop], y[start:stop] augmented_X = augment_input(X, model, mf_steps) datasets = augmented_X, y - if which_set == 'train': - serial.save(filepath=path + 'aug_train_dump.pkl.gz', - obj=datasets) - else: - serial.save(filepath=path + 'aug_test_dump.pkl.gz', - obj=datasets) + if save_aug == True: + if which_set == 'train': + serial.save(filepath=path + 'aug_train_dump.pkl.gz', + obj=datasets) + else: + serial.save(filepath=path + 'aug_test_dump.pkl.gz', + obj=datasets) - augmented_X, y = augmented_X[start:stop], y[start:stop] super(MNIST_AUGMENTED, self).__init__(X=augmented_X, y=y) diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index 7e06ec39cf..12f0bcfde0 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -26,7 +26,7 @@ def test_train_example(): # START PRETRAINING # load and train first layer - train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') + train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist_l1.yaml') layer1_yaml = open(train_yaml_path, 'r').read() hyper_params_l1 = {'train_stop': 20, 'batch_size': 5, @@ -47,7 +47,7 @@ def test_train_example(): train.main_loop() # load and train second layer - train_yaml_path = os.path.join(train_path, 'dbm_mnist_l2.yaml') + train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist_l2.yaml') layer2_yaml = open(train_yaml_path, 'r').read() hyper_params_l2 = {'train_stop': 20, 'batch_size': 5, @@ -65,7 +65,7 @@ def test_train_example(): train.main_loop() # START TRAINING - train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') + train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist.yaml') yaml = open(train_yaml_path, 'r').read() hyper_params_dbm = {'train_stop': 20, 'valid_stop': 20, @@ -120,7 +120,7 @@ def test_train_example(): print '-----------------------------------' # load dbm as a mlp - train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') + train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist_mlp.yaml') mlp_yaml = open(train_yaml_path, 'r').read() hyper_params_mlp = {'train_stop': 20, From d30f6f3ca5211cb070b3f8ff6b298831185f7e15 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 14:44:47 +0200 Subject: [PATCH 12/32] corrections on mnist_augmented --- pylearn2/datasets/mnist_augmented.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 23fabdb512..00f612dec4 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -20,9 +20,11 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, try: if which_set == 'train': - datasets = serial.load(filepath=path + 'aug_train_dump.pkl.gz') + path = os.path.join(path, 'aug_train_dump.pkl.gz') + datasets = serial.load(filepath=path) augmented_X, y = datasets[0], datasets[1] else: + path = os.path.join(path, 'aug_test_dump.pkl.gz') datasets = serial.load(filepath=path + 'aug_test_dump.pkl.gz') augmented_X, y = datasets[0], datasets[1] augmented_X, y = augmented_X[start:stop], y[start:stop] @@ -44,10 +46,10 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, datasets = augmented_X, y if save_aug == True: if which_set == 'train': - serial.save(filepath=path + 'aug_train_dump.pkl.gz', - obj=datasets) + path = os.path.join(path, 'aug_train_dump.pkl.gz') + serial.save(filepath=path, obj=datasets) else: - serial.save(filepath=path + 'aug_test_dump.pkl.gz', - obj=datasets) + path = os.path.join(path, 'aug_test_dump.pkl.gz') + serial.save(filepath=path, obj=datasets) super(MNIST_AUGMENTED, self).__init__(X=augmented_X, y=y) From 64be5e91f3c4e0559129220c2887250dad1df457 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 14:51:01 +0200 Subject: [PATCH 13/32] readme completed --- pylearn2/scripts/papers/dbm/README | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pylearn2/scripts/papers/dbm/README b/pylearn2/scripts/papers/dbm/README index 3716801621..7015d2ef6d 100644 --- a/pylearn2/scripts/papers/dbm/README +++ b/pylearn2/scripts/papers/dbm/README @@ -10,10 +10,13 @@ As explained in the paper, the finetuning procedure uses an augmented input to f mnistaugmented.py in pylearn2/datasets/. The latter takes the mnist dataset and augment it. Eventually, it saves .pkl files of the augmented dataset because data augmentation is a time-consuming operation. -NO DROPOUT RESULTS: +There are two tests in /tests. The script to run the whole procedure, with all the right parameters, reaches the result published by Hinton & Salakhutdinov. The fast version of it, is +suitable to be run on travis. It does not perform well because it uses a very small training set and a very small number of epochs. +NO DROPOUT RESULTS: +The test returns a 0.094% test error WITHOUT softmax on the top of the DBM and dropout. DROPOUT RESULTS: - +The test returns a 0.084% test error WITH softmax on the top of the DBM and dropout. Experiments have been performed on Ubuntu 14.04 LTS using a NVIDIA Tesla C1060 GPU and a 8-core Intel(R) Core(TM) i7 CPU 920 @ 2.67GHz. I used openblas-base and numpy version 1.9.0, scipy version 0.13.3, theano version 0.6.0 and pylearn2 with 6264 commits. From 8d629209045054fc3cf7472b6760d8ef46b23531 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 25 Oct 2014 15:05:49 +0200 Subject: [PATCH 14/32] removed useless files --- pylearn2/datasets/gtsrb.py | 164 ------------------------------------- 1 file changed, 164 deletions(-) delete mode 100644 pylearn2/datasets/gtsrb.py diff --git a/pylearn2/datasets/gtsrb.py b/pylearn2/datasets/gtsrb.py deleted file mode 100644 index 97cf0e1e0f..0000000000 --- a/pylearn2/datasets/gtsrb.py +++ /dev/null @@ -1,164 +0,0 @@ -from PIL import Image -import numpy -import csv -import cPickle - -from pylearn2.datasets.dense_design_matrix import DenseDesignMatrix -from pylearn2.datasets.augment_input import augment_input - -# User can select whether he wants to select all images or just a smaller set of them in order not to add too much noise -# after the resizing -bound_train = 1000 -bound_test = 1000 - -class GTSRB(DenseDesignMatrix): - - def __init__(self, which_set, model = None, mf_steps = None, one_hot = True, - start = None, stop = None, img_size = None): - - #path = "${PYLEARN2_DATA_PATH}/gtsrb" - first_path = "/home/deramo/workspace/datasets/gtsrb" - if which_set == 'train': - path = first_path + "/Final_Training/Images" - else: - path = first_path + "/Final_Test/Images" - self.path = path - self.which_set = which_set - self.delimiter = ';' - self.img_size = img_size - self.one_hot = one_hot - self.start = start - self.stop = stop - - try: - if which_set == 'train': - datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'train_dump.pkl.gz') - X, y = datasets[0], datasets[1] - else: - datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'test_dump.pkl.gz') - X, y = datasets[0], datasets[1] - except: - try: - if which_set == 'train': - datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'noaug_train_dump.pkl.gz') - X, y = datasets[0], datasets[1] - else: - datasets = load_from_dump(dump_data_dir = self.path, dump_filename = 'noaug_test_dump.pkl.gz') - X, y = datasets[0], datasets[1] - X, y = X[0:12600], y[0:12600] # temporaneo - - except: - X, y = self.load_data() - print "\ndata loaded!\n" - - noaug_datasets = X, y # not augmented datasets is saved in order not to waste time reloading gtsrb each time - if which_set == 'train': - save_to_dump(var_to_dump = noaug_datasets, dump_data_dir = self.path, dump_filename = 'noaug_train_dump.pkl.gz') - else: - save_to_dump(var_to_dump = noaug_datasets, dump_data_dir = self.path, dump_filename = 'noaug_test_dump.pkl.gz') - - X, y = X.astype(float), y.astype(float) - - # BUILD AUGMENTED INPUT FOR FINETUNING - if mf_steps is not None: - augmented_X = augment_input(X, model, mf_steps) - X = augmented_X - - datasets = augmented_X, y - if which_set == 'train': - save_to_dump(var_to_dump = datasets, dump_data_dir = self.path, dump_filename = 'train_dump.pkl.gz') - else: - save_to_dump(var_to_dump = datasets, dump_data_dir = self.path, dump_filename = 'test_dump.pkl.gz') - - X, y = X[self.start:self.stop], y[self.start:self.stop] - super(GTSRB, self).__init__(X = X, y = y) - - def load_data(self): - - print "\nloading data...\n" - - if self.which_set == 'train': - - first = True - - # loop over all 43 classes - for c in xrange(43): #43 - prefix = self.path + '/' + format(c, '05d') + '/' # subdirectory for class - f = open(prefix + 'GT-'+ format(c, '05d') + '.csv') # annotations file - reader = csv.reader(f, delimiter = self.delimiter) # csv parser for annotations file - reader.next() # skip header - for row in reader: - img = Image.open(prefix + '/' + row[0]) - if img.size[0] + bound_train >= self.img_size[0]: - img = img.convert('L') # grayscale - img = img.resize(self.img_size, Image.ANTIALIAS) #resize - if first: - X = numpy.asarray([img.getdata()]) - X /= 255. - y = numpy.asarray(row[7]) - first = False - else: - X = numpy.append(X, [img.getdata()], axis = 0) - X /= 255. - y = numpy.append(y, row[7]) - f.close() - - # shuffle - assert X.shape[0] == y.shape[0] - - indices = numpy.arange(X.shape[0]) - rng = numpy.random.RandomState() # if given an int argument will give reproducible results - rng.shuffle(indices) - # shuffle both the arrays consistently - i = 0 - temp_X = X - temp_y = y - for idx in indices: - X[i] = temp_X[idx] - y[i] = temp_y[idx] - i += 1 - - else: - - first = True - - f = open(self.path + '/' + "GT-final_test.csv") - reader = csv.reader(f, delimiter = self.delimiter) # csv parser for annotations file - reader.next() # skip header - - for c in xrange(12630): - for row in reader: - img = Image.open(self.path + '/' + row[0]) - if img.size[0] + bound_train >= self.img_size[0]: - img = img.convert('L') # grayscale - img = img.resize(self.img_size, Image.ANTIALIAS) #resize - if first: - X = numpy.asarray([img.getdata()]) - X /= 255. - y = row[7] - first = False - else: - X = numpy.append(X, [img.getdata()], axis = 0) - X /= 255. - y = numpy.append(y, row[7]) - f.close() - - # build the one_hot matrix used to specify labels - if self.one_hot: - one_hot = numpy.zeros((y.shape[0], 43)) - for i in xrange(y.shape[0]): - one_hot[i,y[i]] = 1. - y = one_hot - - return X, y - -def load_from_dump(dump_data_dir, dump_filename): - load_file = open(dump_data_dir + "/" + dump_filename) - unpickled_var = cPickle.load(load_file) - load_file.close() - return unpickled_var - -def save_to_dump(var_to_dump, dump_data_dir, dump_filename): - save_file = open(dump_data_dir + "/" + dump_filename, 'wb') # this will overwrite current contents - cPickle.dump(var_to_dump, save_file, -1) # the -1 is for HIGHEST_PROTOCOL - save_file.close() \ No newline at end of file From e466b44de01a515db2c3b8755858d35e4c0b13f8 Mon Sep 17 00:00:00 2001 From: carlo Date: Tue, 28 Oct 2014 12:21:39 +0100 Subject: [PATCH 15/32] path correction --- pylearn2/datasets/mnist_augmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 00f612dec4..5fd3f7e6a9 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -25,7 +25,7 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, augmented_X, y = datasets[0], datasets[1] else: path = os.path.join(path, 'aug_test_dump.pkl.gz') - datasets = serial.load(filepath=path + 'aug_test_dump.pkl.gz') + datasets = serial.load(filepath=path) augmented_X, y = datasets[0], datasets[1] augmented_X, y = augmented_X[start:stop], y[start:stop] except: From 4852189c2a6a7787c2ce421b492abd9a5eb11612 Mon Sep 17 00:00:00 2001 From: carlo Date: Tue, 28 Oct 2014 12:38:16 +0100 Subject: [PATCH 16/32] code optimization --- pylearn2/datasets/mnist_augmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 5fd3f7e6a9..311d733ba6 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -43,8 +43,8 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, X, y = X[start:stop], y[start:stop] augmented_X = augment_input(X, model, mf_steps) - datasets = augmented_X, y if save_aug == True: + datasets = augmented_X, y if which_set == 'train': path = os.path.join(path, 'aug_train_dump.pkl.gz') serial.save(filepath=path, obj=datasets) From 9f1cf00b30168faaa8284d340aae5275d17e0138 Mon Sep 17 00:00:00 2001 From: carlo Date: Tue, 28 Oct 2014 17:14:39 +0100 Subject: [PATCH 17/32] corrections on paths for saving and loading --- pylearn2/datasets/mnist_augmented.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 311d733ba6..66bf0c4a60 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -15,16 +15,16 @@ class MNIST_AUGMENTED(DenseDesignMatrix): def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, start=None, stop=None, save_aug=False): - path = os.path.join('${PYLEARN2_DATA_PATH}', 'mnist') - path = serial.preprocess(path) + self.path = os.path.join('${PYLEARN2_DATA_PATH}', 'mnist') + self.path = serial.preprocess(self.path) try: if which_set == 'train': - path = os.path.join(path, 'aug_train_dump.pkl.gz') + path = os.path.join(self.path, 'aug_train_dump.pkl.gz') datasets = serial.load(filepath=path) augmented_X, y = datasets[0], datasets[1] else: - path = os.path.join(path, 'aug_test_dump.pkl.gz') + path = os.path.join(self.path, 'aug_test_dump.pkl.gz') datasets = serial.load(filepath=path) augmented_X, y = datasets[0], datasets[1] augmented_X, y = augmented_X[start:stop], y[start:stop] @@ -44,12 +44,13 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, augmented_X = augment_input(X, model, mf_steps) if save_aug == True: - datasets = augmented_X, y + datasets = augmented_X, y if which_set == 'train': - path = os.path.join(path, 'aug_train_dump.pkl.gz') + path = os.path.join(self.path, 'aug_train_dump.pkl.gz') serial.save(filepath=path, obj=datasets) else: - path = os.path.join(path, 'aug_test_dump.pkl.gz') + path = os.path.join(self.path, 'aug_test_dump.pkl.gz') serial.save(filepath=path, obj=datasets) super(MNIST_AUGMENTED, self).__init__(X=augmented_X, y=y) + From b686b29aa348c22b69ea4a1f60138b14e1e7746a Mon Sep 17 00:00:00 2001 From: carlo Date: Fri, 14 Nov 2014 12:48:50 +0100 Subject: [PATCH 18/32] one_hot flag removed. binarized test set in dbm softmax added --- pylearn2/scripts/papers/dbm/dbm_mnist.yaml | 8 +------- pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml | 1 - pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml | 11 ++++++----- pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml | 2 -- .../scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml | 4 +--- pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml | 11 ++++++----- 6 files changed, 14 insertions(+), 23 deletions(-) diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml index 87b44d1538..cdd07e2481 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml @@ -2,7 +2,6 @@ dataset: &data !obj:pylearn2.datasets.binarizer.Binarizer { raw: &raw_train !obj:pylearn2.datasets.mnist.MNIST { which_set: "train", - one_hot: 1, start: 0, stop: %(train_stop)i } @@ -35,12 +34,7 @@ init_momentum: 0.5, }, monitoring_batches: %(monitoring_batches)i, - monitoring_dataset: { - 'test': !obj:pylearn2.datasets.mnist.MNIST { - which_set: 'test', - one_hot: 1 - } - }, + monitoring_dataset: *data, cost : !obj:pylearn2.costs.cost.SumOfCosts { costs: [ !obj:pylearn2.costs.dbm.VariationalPCD { diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml index ccfe01e050..95bbd04c85 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l1.yaml @@ -2,7 +2,6 @@ dataset: &data !obj:pylearn2.datasets.binarizer.Binarizer { raw: &raw_train !obj:pylearn2.datasets.mnist.MNIST { which_set: "train", - one_hot: 1, start: 0, stop: %(train_stop)i } diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml index 2c3cd63249..f0459e06df 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml @@ -1,11 +1,12 @@ !obj:pylearn2.train.Train { dataset: &train !obj:pylearn2.datasets.binarizer.Binarizer { raw: !obj:pylearn2.datasets.transformer_dataset.TransformerDataset { - raw: !obj:pylearn2.datasets.mnist.MNIST { - which_set: 'train', - one_hot: 1, - start: 0, - stop: %(train_stop)i + raw: !obj:pylearn2.datasets.binarizer.Binarizer { + raw: !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'train', + start: 0, + stop: %(train_stop)i + }, }, transformer: !pkl: "%(save_path)s/dbm_mnist_l1.pkl" }, diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml index 448e15c9bb..e467a9127b 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp.yaml @@ -1,7 +1,6 @@ !obj:pylearn2.train.Train { dataset: &train !obj:pylearn2.datasets.mnist.MNIST { which_set: 'train', - one_hot: 1, start: 0, stop: %(train_stop)i }, @@ -33,7 +32,6 @@ monitoring_dataset: { 'test': !obj:pylearn2.datasets.mnist.MNIST { which_set: 'test', - one_hot: 1 }, }, cost: !obj:pylearn2.costs.mlp.Default {}, diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml index 84a32e23d5..1ac524dd21 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_mlp_dropout.yaml @@ -1,7 +1,6 @@ !obj:pylearn2.train.Train { dataset: !obj:pylearn2.datasets.mnist.MNIST { which_set: 'train', - one_hot: 1, start: 0, stop: %(train_stop)i, }, @@ -33,8 +32,7 @@ }, monitoring_dataset: { 'test': !obj:pylearn2.datasets.mnist.MNIST { - which_set: 'test', - one_hot: 1 + which_set: 'test', }, }, cost: !obj:pylearn2.costs.mlp.dropout.Dropout { diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml index 8276672e40..045c0e94fd 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml @@ -2,7 +2,6 @@ dataset: &data !obj:pylearn2.datasets.binarizer.Binarizer { raw: &raw_train !obj:pylearn2.datasets.mnist.MNIST { which_set: "train", - one_hot: 1, start: 0, stop: %(train_stop)i } @@ -41,10 +40,12 @@ }, monitoring_batches: %(monitoring_batches)i, monitoring_dataset: { - 'test': !obj:pylearn2.datasets.mnist.MNIST { - which_set: 'test', - one_hot: 1 - } + 'test': + !obj:pylearn2.datasets.binarizer.Binarizer { + raw: !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'test', + }, + } }, cost : !obj:pylearn2.costs.cost.SumOfCosts { costs: [ From 3e9d06eec164ea7e69383e8abbfb9386d0fe12a5 Mon Sep 17 00:00:00 2001 From: carlo Date: Fri, 14 Nov 2014 13:06:38 +0100 Subject: [PATCH 19/32] fixed dataset for second layer --- pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml index f0459e06df..76d765e78c 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_l2.yaml @@ -1,12 +1,10 @@ !obj:pylearn2.train.Train { dataset: &train !obj:pylearn2.datasets.binarizer.Binarizer { raw: !obj:pylearn2.datasets.transformer_dataset.TransformerDataset { - raw: !obj:pylearn2.datasets.binarizer.Binarizer { - raw: !obj:pylearn2.datasets.mnist.MNIST { - which_set: 'train', - start: 0, - stop: %(train_stop)i - }, + raw: !obj:pylearn2.datasets.mnist.MNIST { + which_set: 'train', + start: 0, + stop: %(train_stop)i }, transformer: !pkl: "%(save_path)s/dbm_mnist_l1.pkl" }, From b1d46d2742dc52aefe9cb8e478e61445fb7395be Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 22 Nov 2014 13:13:04 +0100 Subject: [PATCH 20/32] corrections to meet pep8 requirements --- pylearn2/datasets/mnist_augmented.py | 6 +- pylearn2/scripts/dbm/augment_input.py | 18 +++-- .../papers/dbm/tests/test_dbm_mnist.py | 74 ++++++++++++------- .../papers/dbm/tests/test_dbm_mnist_fast.py | 52 ++++++++----- 4 files changed, 93 insertions(+), 57 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 66bf0c4a60..2e0ad89416 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -10,6 +10,7 @@ for DBM discriminative finetuning ''' + class MNIST_AUGMENTED(DenseDesignMatrix): def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, @@ -30,7 +31,7 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, augmented_X, y = augmented_X[start:stop], y[start:stop] except: X = dataset.X - if one_hot: + if one_hot is True: one_hot = np.zeros((dataset.y.shape[0], 10), dtype='float32') for i in xrange(dataset.y.shape[0]): label = dataset.y[i] @@ -51,6 +52,5 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, else: path = os.path.join(self.path, 'aug_test_dump.pkl.gz') serial.save(filepath=path, obj=datasets) - - super(MNIST_AUGMENTED, self).__init__(X=augmented_X, y=y) + super(MNIST_AUGMENTED, self).__init__(X=augmented_X, y=y) diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index aa3506fb76..4b0093c2a1 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -5,8 +5,8 @@ """ This module augments the dataset in order to make it suitable for DBM discriminative finetuning. -For each example in the dataset, using the provided trained DBM, -it performs n mean-field updates initializing the state of the second +For each example in the dataset, using the provided trained DBM, +it performs n mean-field updates initializing the state of the second hidden layer of the DBM and augments the example with this state. It returns a dataset where each example is composed of its previous value concatenated with the respective initialization of the second @@ -29,27 +29,31 @@ References ---------- -Salakhutdinov Ruslan and Hinton Geoffrey. "An efficient +Salakhutdinov Ruslan and Hinton Geoffrey. "An efficient procedure for deep boltzmann machines". 2012. """ + def augment_input(X, model, mf_steps): print '\nAugmenting data...\n' i = 0 - init_data = model.visible_layer.space.get_origin_batch(batch_size = 1, dtype = 'float32') + init_data = model.visible_layer.space.get_origin_batch(batch_size=1, + dtype='float32') for x in X[:]: init_data[0] = x - data = sharedX(init_data, name = 'v') - marginal_posterior = model.mf(V = data, niter = mf_steps)[1] # mean field inference of second hidden layer (niter: number of mean field updates) + data = sharedX(init_data, name='v') + # mean field inference of second hidden layer + # (niter: number of mean field updates) + marginal_posterior = model.mf(V=data, niter=mf_steps)[1] mp = function([], marginal_posterior) mp = mp()[0][0] if i == 0: final_data = numpy.asarray([numpy.concatenate((mp, x))]) else: - final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis = 0) + final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis=0) i += 1 diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index 7ba2169cd4..60e2d497fe 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -42,12 +42,12 @@ def test_train_example(): # path definition cwd = os.getcwd() - train_path = cwd # change this if you don't want to use the current working directory + train_path = cwd # training path is the current working directory try: os.chdir(train_path) # START PRETRAINING - # load and train first layer + # load and train first layer train_yaml_path = os.path.join(train_path, 'dbm_mnist_l1.yaml') layer1_yaml = open(train_yaml_path, 'r').read() hyper_params_l1 = {'train_stop': 60000, @@ -94,15 +94,16 @@ def test_train_example(): # START TRAINING if SOFTMAX: - train_yaml_path = os.path.join(train_path, 'dbm_mnist_softmax.yaml') + train_yaml_path = os.path.join(train_path, + 'dbm_mnist_softmax.yaml') else: train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') yaml = open(train_yaml_path, 'r').read() hyper_params_dbm = {'train_stop': 60000, 'valid_stop': 60000, 'batch_size': 100, - 'detector_layer_1_dim': hyper_params_l1['nhid'], - 'detector_layer_2_dim': hyper_params_l2['nhid'], + 'detector_layer_1_dim':hyper_params_l1['nhid'], + 'detector_layer_2_dim':hyper_params_l2['nhid'], 'monitoring_batches': 5, 'max_epochs': MAX_EPOCHS_DBM, 'save_path': train_path @@ -136,7 +137,8 @@ def test_train_example(): bias = numpy.asarray(cuda_bias) train.model.hidden_layers[1].set_biases(bias) - print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' + print '\nAll layers weights and biases have been clamped\ + to the respective layers of the DBM' print '\n-----------------------------------' print ' Unsupervised training' @@ -154,9 +156,11 @@ def test_train_example(): # load dbm as a mlp if DROPOUT: - train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp_dropout.yaml') + train_yaml_path = os.path.join(train_path, + 'dbm_mnist_mlp_dropout.yaml') else: - train_yaml_path = os.path.join(train_path, 'dbm_mnist_mlp.yaml') + train_yaml_path = os.path.join(train_path, + 'dbm_mnist_mlp.yaml') mlp_yaml = open(train_yaml_path, 'r').read() hyper_params_mlp = {'train_stop': 60000, 'valid_stop': 60000, @@ -172,24 +176,39 @@ def test_train_example(): train = yaml_parse.load(mlp_yaml) if SOFTMAX: - dbm = serial.load(os.path.join(train_path, 'dbm_mnist_softmax.pkl')) + dbm = serial.load(os.path.join(train_path, + 'dbm_mnist_softmax.pkl')) else: - dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - - # dataset & monitoring dataset insertion without .yaml file to avoid problems - # (don't know how to pass 'model' hyperparameter) - train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset = train.dataset, - which_set = 'train', one_hot = 1, model = dbm, start = 0, - stop = hyper_params_mlp['train_stop'], mf_steps = MF_STEPS) - train.algorithm.monitoring_dataset = { #'valid' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = mf_steps), - 'test' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], - which_set = 'test', one_hot = 1, model = dbm, mf_steps = MF_STEPS)} - - # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON - - # concatenate weights between first and second hidden layer & weights between visible and first hidden layer - train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), - dbm.hidden_layers[0].get_weights()))) + dbm = serial.load(os.path.join(train_path, + 'dbm_mnist.pkl')) + + train.dataset = mnist_augmented.MNIST_AUGMENTED( + dataset=train.dataset, + which_set='train', + one_hot=1, + model=dbm, start=0, + stop=hyper_params_mlp['train_stop'], + mf_steps=MF_STEPS) + train.algorithm.monitoring_dataset = { + # 'valid' : mnist_augmented.MNIST_AUGMENTED( + # dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], + # mf_steps=mf_steps), + 'test' : mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, + mf_steps=MF_STEPS)} + + # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS + # EXPLAINED BY HINTON + + # concatenate weights between first and second hidden + # layer & weights between visible and first hidden layer + train.model.layers[0].set_weights(numpy.concatenate(( + dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): @@ -199,7 +218,8 @@ def test_train_example(): for l, h in zip(train.model.layers, dbm.hidden_layers): l.set_biases(h.get_biases()) - print '\nDBM trained weights and biases have been clamped in the MLP.' + print '\nDBM trained weights and biases have been + clamped in the MLP.' print '\n...Finetuning...\n' train.main_loop() @@ -208,4 +228,4 @@ def test_train_example(): os.chdir(cwd) if __name__ == '__main__': - test_train_example() \ No newline at end of file + test_train_example() diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index 12f0bcfde0..ade9214dc2 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -15,17 +15,18 @@ from pylearn2.datasets import mnist_augmented from theano import function + @no_debug_mode def test_train_example(): # path definition cwd = os.getcwd() - train_path = cwd # change this if you don't want to use the current working directory + train_path = cwd # train path is the current working directory try: os.chdir(train_path) # START PRETRAINING - # load and train first layer + # load and train first layer train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist_l1.yaml') layer1_yaml = open(train_yaml_path, 'r').read() hyper_params_l1 = {'train_stop': 20, @@ -105,7 +106,8 @@ def test_train_example(): bias = numpy.asarray(cuda_bias) train.model.hidden_layers[1].set_biases(bias) - print '\nAll layers weights and biases have been clamped to the respective layers of the DBM' + print '\nAll layers weights and biases have been clamped\ + to the respective layers of the DBM' print '\n-----------------------------------' print ' Unsupervised training' @@ -121,7 +123,7 @@ def test_train_example(): # load dbm as a mlp train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist_mlp.yaml') - + mlp_yaml = open(train_yaml_path, 'r').read() hyper_params_mlp = {'train_stop': 20, 'valid_stop': 20, @@ -138,26 +140,36 @@ def test_train_example(): dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - # dataset & monitoring dataset insertion without .yaml file to avoid problems - # (don't know how to pass 'model' hyperparameter) - train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset = train.dataset, - which_set = 'train', one_hot = 1, model = dbm, start = 0, - stop = hyper_params_mlp['train_stop'], mf_steps = 1) - train.algorithm.monitoring_dataset = { #'valid' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['valid'], which_set = 'train', one_hot = 1, model = dbm, start = hyper_params_mlp['train_stop'], stop = hyper_params_mlp['valid_stop'], mf_steps = 1), - 'test' : mnist_augmented.MNIST_AUGMENTED(dataset = train.algorithm.monitoring_dataset['test'], - which_set = 'test', one_hot = 1, model = dbm, start = 0, stop = 20, mf_steps = 1)} - - # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS EXPLAINED BY HINTON - - # concatenate weights between first and second hidden layer & weights between visible and first hidden layer - train.model.layers[0].set_weights(numpy.concatenate((dbm.hidden_layers[1].get_weights().transpose(), - dbm.hidden_layers[0].get_weights()))) + train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset=train.dataset, + which_set='train', one_hot=1, + model=dbm, start=0, + stop=hyper_params_mlp['train_stop'], + mf_steps=1) + train.algorithm.monitoring_dataset = { + # 'valid' : mnist_augmented.MNIST_AUGMENTED( + # dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], mf_steps=1), + 'test' : mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, start=0, + stop=20, mf_steps=1)} + + # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS + # EXPLAINED BY HINTON + + # concatenate weights between first and second hidden layer & + # weights between visible and first hidden layer + train.model.layers[0].set_weights(numpy.concatenate(( + dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): l.set_weights(h.get_weights()) - # clamp biases + # clamp biases for l, h in zip(train.model.layers, dbm.hidden_layers): l.set_biases(h.get_biases()) @@ -170,4 +182,4 @@ def test_train_example(): os.chdir(cwd) if __name__ == '__main__': - test_train_example() \ No newline at end of file + test_train_example() From bd3a8f9ea38640f021c594eb5c4d06d20f696aa5 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 22 Nov 2014 16:15:01 +0100 Subject: [PATCH 21/32] other corrections for pep8 --- pylearn2/datasets/mnist_augmented.py | 2 +- pylearn2/scripts/dbm/augment_input.py | 12 ++-- pylearn2/scripts/papers/dbm/dbm_mnist.yaml | 4 +- .../scripts/papers/dbm/dbm_mnist_softmax.yaml | 4 +- .../papers/dbm/tests/test_dbm_mnist.py | 58 +++++++++---------- .../papers/dbm/tests/test_dbm_mnist_fast.py | 34 +++++------ 6 files changed, 58 insertions(+), 56 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 2e0ad89416..718ad93a76 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -44,7 +44,7 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, X, y = X[start:stop], y[start:stop] augmented_X = augment_input(X, model, mf_steps) - if save_aug == True: + if save_aug is True: datasets = augmented_X, y if which_set == 'train': path = os.path.join(self.path, 'aug_train_dump.pkl.gz') diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index 4b0093c2a1..92b29415da 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -39,21 +39,23 @@ def augment_input(X, model, mf_steps): print '\nAugmenting data...\n' i = 0 - init_data = model.visible_layer.space.get_origin_batch(batch_size=1, - dtype='float32') + init_data = model.visible_layer.space.get_origin_batch(batch_size=1, + dtype='float32') for x in X[:]: init_data[0] = x data = sharedX(init_data, name='v') - # mean field inference of second hidden layer - # (niter: number of mean field updates) + # mean field inference of second hidden layer + # (niter: number of mean field updates) marginal_posterior = model.mf(V=data, niter=mf_steps)[1] mp = function([], marginal_posterior) mp = mp()[0][0] if i == 0: final_data = numpy.asarray([numpy.concatenate((mp, x))]) else: - final_data = numpy.append(final_data, [numpy.concatenate((mp, x))], axis=0) + final_data = numpy.append(final_data, + [numpy.concatenate((mp, x))], + axis=0) i += 1 diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml index cdd07e2481..9e402b610b 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist.yaml @@ -16,13 +16,13 @@ hidden_layers: [ !obj:pylearn2.models.dbm.BinaryVectorMaxPool { layer_name: 'h1', - detector_layer_dim: %(detector_layer_1_dim)i, + detector_layer_dim: %(n_h1)i, pool_size: 1, irange: 0.001, }, !obj:pylearn2.models.dbm.BinaryVectorMaxPool { layer_name: 'h2', - detector_layer_dim: %(detector_layer_2_dim)i, + detector_layer_dim: %(n_h2)i, pool_size: 1, irange: 0.001, }, diff --git a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml index 045c0e94fd..eb46913176 100644 --- a/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml +++ b/pylearn2/scripts/papers/dbm/dbm_mnist_softmax.yaml @@ -16,13 +16,13 @@ hidden_layers: [ !obj:pylearn2.models.dbm.BinaryVectorMaxPool { layer_name: 'h1', - detector_layer_dim: %(detector_layer_1_dim)i, + detector_layer_dim: %(n_h1)i, pool_size: 1, irange: 0.001, }, !obj:pylearn2.models.dbm.BinaryVectorMaxPool { layer_name: 'h2', - detector_layer_dim: %(detector_layer_2_dim)i, + detector_layer_dim: %(n_h2)i, pool_size: 1, irange: 0.001, }, diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index 60e2d497fe..52e3ad7fcb 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -95,15 +95,15 @@ def test_train_example(): # START TRAINING if SOFTMAX: train_yaml_path = os.path.join(train_path, - 'dbm_mnist_softmax.yaml') + 'dbm_mnist_softmax.yaml') else: train_yaml_path = os.path.join(train_path, 'dbm_mnist.yaml') yaml = open(train_yaml_path, 'r').read() hyper_params_dbm = {'train_stop': 60000, 'valid_stop': 60000, 'batch_size': 100, - 'detector_layer_1_dim':hyper_params_l1['nhid'], - 'detector_layer_2_dim':hyper_params_l2['nhid'], + 'n_h1': hyper_params_l1['nhid'], + 'n_h2': hyper_params_l2['nhid'], 'monitoring_batches': 5, 'max_epochs': MAX_EPOCHS_DBM, 'save_path': train_path @@ -137,8 +137,8 @@ def test_train_example(): bias = numpy.asarray(cuda_bias) train.model.hidden_layers[1].set_biases(bias) - print '\nAll layers weights and biases have been clamped\ - to the respective layers of the DBM' + print ("\nAll layers weights and biases have been clamped " + "to the respective layers of the DBM") print '\n-----------------------------------' print ' Unsupervised training' @@ -156,11 +156,11 @@ def test_train_example(): # load dbm as a mlp if DROPOUT: - train_yaml_path = os.path.join(train_path, - 'dbm_mnist_mlp_dropout.yaml') + train_yaml_path = os.path.join(train_path, + 'dbm_mnist_mlp_dropout.yaml') else: - train_yaml_path = os.path.join(train_path, - 'dbm_mnist_mlp.yaml') + train_yaml_path = os.path.join(train_path, + 'dbm_mnist_mlp.yaml') mlp_yaml = open(train_yaml_path, 'r').read() hyper_params_mlp = {'train_stop': 60000, 'valid_stop': 60000, @@ -183,31 +183,31 @@ def test_train_example(): 'dbm_mnist.pkl')) train.dataset = mnist_augmented.MNIST_AUGMENTED( - dataset=train.dataset, + dataset=train.dataset, which_set='train', - one_hot=1, - model=dbm, start=0, - stop=hyper_params_mlp['train_stop'], - mf_steps=MF_STEPS) + one_hot=1, + model=dbm, start=0, + stop=hyper_params_mlp['train_stop'], + mf_steps=MF_STEPS) train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.MNIST_AUGMENTED( - # dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], - # mf_steps=mf_steps), + # 'valid' : mnist_augmented.MNIST_AUGMENTED( + # dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], + # mf_steps=mf_steps), 'test' : mnist_augmented.MNIST_AUGMENTED( - dataset=train.algorithm.monitoring_dataset['test'], - which_set='test', one_hot=1, model=dbm, - mf_steps=MF_STEPS)} + dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, + mf_steps=MF_STEPS)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS - # EXPLAINED BY HINTON + # EXPLAINED BY HINTON - # concatenate weights between first and second hidden - # layer & weights between visible and first hidden layer + # Concatenate weights between first and second hidden + # layer & weights between visible and first hidden layer train.model.layers[0].set_weights(numpy.concatenate(( - dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally @@ -218,8 +218,8 @@ def test_train_example(): for l, h in zip(train.model.layers, dbm.hidden_layers): l.set_biases(h.get_biases()) - print '\nDBM trained weights and biases have been - clamped in the MLP.' + print ("\nDBM trained weights and biases have been " + "clamped in the MLP.") print '\n...Finetuning...\n' train.main_loop() diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index ade9214dc2..88cd6c248c 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -71,8 +71,8 @@ def test_train_example(): hyper_params_dbm = {'train_stop': 20, 'valid_stop': 20, 'batch_size': 5, - 'detector_layer_1_dim': hyper_params_l1['nhid'], - 'detector_layer_2_dim': hyper_params_l2['nhid'], + 'n_h1': hyper_params_l1['nhid'], + 'n_h2': hyper_params_l2['nhid'], 'monitoring_batches': 5, 'max_epochs': 5, 'save_path': train_path @@ -106,8 +106,8 @@ def test_train_example(): bias = numpy.asarray(cuda_bias) train.model.hidden_layers[1].set_biases(bias) - print '\nAll layers weights and biases have been clamped\ - to the respective layers of the DBM' + print("\nAll layers weights and biases have been clamped " + "to the respective layers of the DBM") print '\n-----------------------------------' print ' Unsupervised training' @@ -142,27 +142,27 @@ def test_train_example(): train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset=train.dataset, which_set='train', one_hot=1, - model=dbm, start=0, + model=dbm, start=0, stop=hyper_params_mlp['train_stop'], - mf_steps=1) + mf_steps=1) train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.MNIST_AUGMENTED( - # dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], mf_steps=1), + # 'valid' : mnist_augmented.MNIST_AUGMENTED( + # dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], mf_steps=1), 'test' : mnist_augmented.MNIST_AUGMENTED( - dataset=train.algorithm.monitoring_dataset['test'], + dataset=train.algorithm.monitoring_dataset['test'], which_set='test', one_hot=1, model=dbm, start=0, - stop=20, mf_steps=1)} + stop=20, mf_steps=1)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS - # EXPLAINED BY HINTON + # EXPLAINED BY HINTON - # concatenate weights between first and second hidden layer & - # weights between visible and first hidden layer + # Concatenate weights between first and second hidden layer & + # weights between visible and first hidden layer train.model.layers[0].set_weights(numpy.concatenate(( - dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally From 70ad04a3d78aa4a89b3c09ef71a8dec326e272cb Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 22 Nov 2014 17:37:40 +0100 Subject: [PATCH 22/32] docstring corrections --- pylearn2/datasets/mnist_augmented.py | 2 +- pylearn2/scripts/dbm/augment_input.py | 4 +- .../papers/dbm/tests/test_dbm_mnist.py | 81 ++++++++++--------- .../papers/dbm/tests/test_dbm_mnist_fast.py | 54 +++++++------ 4 files changed, 72 insertions(+), 69 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 718ad93a76..f6e4330e1d 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -33,7 +33,7 @@ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, X = dataset.X if one_hot is True: one_hot = np.zeros((dataset.y.shape[0], 10), dtype='float32') - for i in xrange(dataset.y.shape[0]): + for i in range(dataset.y.shape[0]): label = dataset.y[i] one_hot[i, label] = 1. y = one_hot diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index 92b29415da..92959dbbc2 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -36,7 +36,7 @@ def augment_input(X, model, mf_steps): - print '\nAugmenting data...\n' + print("\nAugmenting data...\n") i = 0 init_data = model.visible_layer.space.get_origin_batch(batch_size=1, @@ -59,6 +59,6 @@ def augment_input(X, model, mf_steps): i += 1 - print 'Data augmentation complete!' + print("Data augmentation complete!") return final_data diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index 52e3ad7fcb..ef6b87dce9 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -63,11 +63,11 @@ def test_train_example(): layer1_yaml = layer1_yaml % (hyper_params_l1) train = yaml_parse.load(layer1_yaml) - print '\n-----------------------------------' - print ' Unsupervised pre-training' - print '-----------------------------------\n' + print("\n-----------------------------------" + " Unsupervised pre-training " + "-----------------------------------\n") - print '\nPre-Training first layer...\n' + print("\nPre-Training first layer...\n") train.main_loop() # load and train second layer @@ -87,7 +87,7 @@ def test_train_example(): layer2_yaml = layer2_yaml % (hyper_params_l2) train = yaml_parse.load(layer2_yaml) - print '\n...Pre-training second layer...\n' + print("\n...Pre-training second layer...\n") train.main_loop() if TRAINING: @@ -137,22 +137,22 @@ def test_train_example(): bias = numpy.asarray(cuda_bias) train.model.hidden_layers[1].set_biases(bias) - print ("\nAll layers weights and biases have been clamped " + print("\nAll layers weights and biases have been clamped " "to the respective layers of the DBM") - print '\n-----------------------------------' - print ' Unsupervised training' - print '-----------------------------------\n' + print("\n-----------------------------------" + " Unsupervised training " + "-----------------------------------\n") - print '\nTraining phase...' + print("\nTraining phase...") train.main_loop() if FINETUNING: # START SUPERVISED TRAINING WITH BACKPROPAGATION - print '\n-----------------------------------' - print ' Supervised training' - print '-----------------------------------' + print("\n-----------------------------------" + " Supervised training " + "-----------------------------------\n" # load dbm as a mlp if DROPOUT: @@ -176,37 +176,38 @@ def test_train_example(): train = yaml_parse.load(mlp_yaml) if SOFTMAX: - dbm = serial.load(os.path.join(train_path, - 'dbm_mnist_softmax.pkl')) + dbm = serial.load(os.path.join(train_path, + 'dbm_mnist_softmax.pkl')) else: - dbm = serial.load(os.path.join(train_path, - 'dbm_mnist.pkl')) - - train.dataset = mnist_augmented.MNIST_AUGMENTED( - dataset=train.dataset, - which_set='train', - one_hot=1, - model=dbm, start=0, - stop=hyper_params_mlp['train_stop'], - mf_steps=MF_STEPS) + dbm = serial.load(os.path.join(train_path, + 'dbm_mnist.pkl')) + + train.dataset = mnist_augmented.\ + MNIST_AUGMENTED(dataset=train.dataset, + which_set='train', + one_hot=1, + model=dbm, start=0, + stop=hyper_params_mlp['train_stop'], + mf_steps=MF_STEPS) train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.MNIST_AUGMENTED( - # dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], - # mf_steps=mf_steps), - 'test' : mnist_augmented.MNIST_AUGMENTED( - dataset=train.algorithm.monitoring_dataset['test'], - which_set='test', one_hot=1, model=dbm, - mf_steps=MF_STEPS)} + # 'valid' : mnist_augmented.\ + # MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], + # mf_steps=mf_steps), + 'test' : mnist_augmented.\ + MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, + mf_steps=MF_STEPS)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS # EXPLAINED BY HINTON # Concatenate weights between first and second hidden # layer & weights between visible and first hidden layer - train.model.layers[0].set_weights(numpy.concatenate(( + train.model.layers[0].set_weights( + numpy.concatenate(( dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) @@ -214,14 +215,14 @@ def test_train_example(): for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): l.set_weights(h.get_weights()) - # clamp biases + # clamp biases for l, h in zip(train.model.layers, dbm.hidden_layers): l.set_biases(h.get_biases()) - print ("\nDBM trained weights and biases have been " - "clamped in the MLP.") + print("\nDBM trained weights and biases have been " + "clamped in the MLP.") - print '\n...Finetuning...\n' + print("\n...Finetuning...\n") train.main_loop() finally: diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index 88cd6c248c..c9ae6979b9 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -40,11 +40,11 @@ def test_train_example(): layer1_yaml = layer1_yaml % (hyper_params_l1) train = yaml_parse.load(layer1_yaml) - print '\n-----------------------------------' - print ' Unsupervised pre-training' - print '-----------------------------------\n' + print("\n-----------------------------------" + " Unsupervised pre-training' " + "-----------------------------------\n") - print '\nPre-Training first layer...\n' + print("\nPre-Training first layer...\n") train.main_loop() # load and train second layer @@ -62,7 +62,7 @@ def test_train_example(): layer2_yaml = layer2_yaml % (hyper_params_l2) train = yaml_parse.load(layer2_yaml) - print '\n...Pre-training second layer...\n' + print("\n...Pre-training second layer...\n") train.main_loop() # START TRAINING @@ -109,17 +109,17 @@ def test_train_example(): print("\nAll layers weights and biases have been clamped " "to the respective layers of the DBM") - print '\n-----------------------------------' - print ' Unsupervised training' - print '-----------------------------------\n' + print("\n-----------------------------------" + " Unsupervised training' " + "-----------------------------------\n") - print '\nTraining phase...' + print("\nTraining phase...") train.main_loop() # START SUPERVISED TRAINING WITH BACKPROPAGATION - print '\n-----------------------------------' - print ' Supervised training' - print '-----------------------------------' + print("\n-----------------------------------" + " Supervised training' " + "-----------------------------------\n") # load dbm as a mlp train_yaml_path = os.path.join(train_path, '..', 'dbm_mnist_mlp.yaml') @@ -140,28 +140,30 @@ def test_train_example(): dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - train.dataset = mnist_augmented.MNIST_AUGMENTED(dataset=train.dataset, + train.dataset = mnist_augmented.\ + MNIST_AUGMENTED(dataset=train.dataset, which_set='train', one_hot=1, model=dbm, start=0, stop=hyper_params_mlp['train_stop'], mf_steps=1) - train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.MNIST_AUGMENTED( - # dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], mf_steps=1), - 'test' : mnist_augmented.MNIST_AUGMENTED( - dataset=train.algorithm.monitoring_dataset['test'], - which_set='test', one_hot=1, model=dbm, start=0, - stop=20, mf_steps=1)} + train.algorithm.monitoring_dataset = { + # 'valid' : mnist_augmented.\ + # MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], mf_steps=1), + 'test' : mnist_augmented.\ + MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, start=0, + stop=20, mf_steps=1)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS # EXPLAINED BY HINTON # Concatenate weights between first and second hidden layer & # weights between visible and first hidden layer - train.model.layers[0].set_weights(numpy.concatenate(( + train.model.layers[0].set_weights( + numpy.concatenate(( dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) @@ -173,9 +175,9 @@ def test_train_example(): for l, h in zip(train.model.layers, dbm.hidden_layers): l.set_biases(h.get_biases()) - print '\nDBM trained weights and biases have been clamped in the MLP.' + print("\nDBM trained weights and biases have been clamped in the MLP.") - print '\n...Finetuning...\n' + print("\n...Finetuning...\n") train.main_loop() finally: From 058151eec42e68851c819b30cdad00233fc405c4 Mon Sep 17 00:00:00 2001 From: carlo Date: Sun, 23 Nov 2014 15:08:29 +0100 Subject: [PATCH 23/32] corrections to docstrings and indentations --- pylearn2/datasets/mnist_augmented.py | 30 +++++++++++--- pylearn2/scripts/dbm/augment_input.py | 39 ++++++++++--------- .../papers/dbm/tests/test_dbm_mnist.py | 22 +++++------ .../papers/dbm/tests/test_dbm_mnist_fast.py | 35 ++++++++++------- 4 files changed, 77 insertions(+), 49 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index f6e4330e1d..cbfcdedcd8 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -5,13 +5,33 @@ from pylearn2.scripts.dbm.augment_input import augment_input from pylearn2.utils import serial -''' - Loads MNIST dataset and build augmented dataset - for DBM discriminative finetuning -''' - class MNIST_AUGMENTED(DenseDesignMatrix): + + """ + Loads MNIST dataset and build augmented dataset + for DBM discriminative finetuning. + + Parameters + ---------- + dataset : `pylearn2.datasets.dataset.Dataset` + which_set : str + Select between training or test set. + model : `pylearn2.models.model.Model` + The DBM to be finetuned. + mf_steps : int + Number of mean field updates for data augmentation. + one_hot : bool, optional + Enable or disable one-hot configuration for + label matrix. + start : int, optional + First index of dataset to be finetuned. + stop : int, optional + Last index of dataset to be finetuned. + save_aug : bool, optional + Select whether to save the augmented dataset + in a pkl file or not. + """ def __init__(self, dataset, which_set, model, mf_steps, one_hot=True, start=None, stop=None, save_aug=False): diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index 92959dbbc2..c5b55fcd42 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -12,29 +12,30 @@ value concatenated with the respective initialization of the second hidden layer of the DBM. -Parameters ----------- -X : ndarray, 2-dimensional - a matrix containing the initial dataset -model : DBM - the DBM model to be finetuned. It is used for - mean field updates -mf_steps : int - the number of mean field updates -Returns -------- -final_data : ndarray, 2-dimensional - the final augmented dataset +def augment_input(X, model, mf_steps): -References ----------- -Salakhutdinov Ruslan and Hinton Geoffrey. "An efficient -procedure for deep boltzmann machines". 2012. -""" + """ + Parameters + ---------- + X : ndarray, 2-dimensional + A matrix containing the initial dataset. + model : DBM + The DBM model to be finetuned. It is used for + mean field updates. + mf_steps : int + The number of mean field updates. + Returns + ------- + final_data : ndarray, 2-dimensional + The final augmented dataset. -def augment_input(X, model, mf_steps): + References + ---------- + Salakhutdinov Ruslan and Hinton Geoffrey. "An efficient + procedure for deep boltzmann machines". 2012. + """ print("\nAugmenting data...\n") diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index ef6b87dce9..d2bf76a483 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -138,7 +138,7 @@ def test_train_example(): train.model.hidden_layers[1].set_biases(bias) print("\nAll layers weights and biases have been clamped " - "to the respective layers of the DBM") + "to the respective layers of the DBM") print("\n-----------------------------------" " Unsupervised training " @@ -182,22 +182,22 @@ def test_train_example(): dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - train.dataset = mnist_augmented.\ - MNIST_AUGMENTED(dataset=train.dataset, + train.dataset = mnist_augmented.MNIST_AUGMENTED( + dataset=train.dataset, which_set='train', one_hot=1, model=dbm, start=0, stop=hyper_params_mlp['train_stop'], mf_steps=MF_STEPS) train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.\ - # MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], - # mf_steps=mf_steps), - 'test' : mnist_augmented.\ - MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['test'], + # 'valid' : mnist_augmented.MNIST_AUGMENTED( + # dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], + # mf_steps=mf_steps), + 'test' : mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['test'], which_set='test', one_hot=1, model=dbm, mf_steps=MF_STEPS)} diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index c9ae6979b9..cb2f9def70 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -18,6 +18,10 @@ @no_debug_mode def test_train_example(): + + """ + Fast version of test script + """ # path definition cwd = os.getcwd() @@ -140,22 +144,25 @@ def test_train_example(): dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) - train.dataset = mnist_augmented.\ - MNIST_AUGMENTED(dataset=train.dataset, - which_set='train', one_hot=1, - model=dbm, start=0, + train.dataset = mnist_augmented.MNIST_AUGMENTED( + dataset=train.dataset, + which_set='train', + one_hot=1, + model=dbm, + start=0, stop=hyper_params_mlp['train_stop'], mf_steps=1) train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.\ - # MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], mf_steps=1), - 'test' : mnist_augmented.\ - MNIST_AUGMENTED(dataset=train.algorithm.monitoring_dataset['test'], - which_set='test', one_hot=1, model=dbm, start=0, - stop=20, mf_steps=1)} + # 'valid': mnist_augmented.MNIST_AUGMENTED( + # dataset=train.algorithm.monitoring_dataset['valid'], + # which_set='train', one_hot=1, model=dbm, + # start=hyper_params_mlp['train_stop'], + # stop=hyper_params_mlp['valid_stop'], mf_steps=1), + 'test': mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, start=0, + stop=20, mf_steps=1) + } # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS # EXPLAINED BY HINTON @@ -164,7 +171,7 @@ def test_train_example(): # weights between visible and first hidden layer train.model.layers[0].set_weights( numpy.concatenate(( - dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[1].get_weights().transpose(), dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally From 98501f287d4beecd8a1563bfdb2c185830c32311 Mon Sep 17 00:00:00 2001 From: carlo Date: Sun, 23 Nov 2014 17:05:21 +0100 Subject: [PATCH 24/32] corrections in docstrings --- pylearn2/datasets/mnist_augmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index cbfcdedcd8..2cc41052cc 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -9,7 +9,7 @@ class MNIST_AUGMENTED(DenseDesignMatrix): """ - Loads MNIST dataset and build augmented dataset + Loads MNIST dataset and builds augmented dataset for DBM discriminative finetuning. Parameters From 4dd7f5ed517196f540574bb88ee0eda98f49291e Mon Sep 17 00:00:00 2001 From: carlo Date: Mon, 24 Nov 2014 12:12:26 +0100 Subject: [PATCH 25/32] docstring corrected --- pylearn2/scripts/dbm/augment_input.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index c5b55fcd42..fc5639d82d 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -11,7 +11,7 @@ It returns a dataset where each example is composed of its previous value concatenated with the respective initialization of the second hidden layer of the DBM. - +""" def augment_input(X, model, mf_steps): From 4d6dbf92efe75779cb3bcacf330e6b0f04b9d778 Mon Sep 17 00:00:00 2001 From: carlo Date: Mon, 24 Nov 2014 13:45:05 +0100 Subject: [PATCH 26/32] indentation errors fixed --- pylearn2/datasets/mnist_augmented.py | 5 ++- pylearn2/scripts/dbm/augment_input.py | 1 + .../papers/dbm/tests/test_dbm_mnist.py | 42 ++++++++++--------- .../papers/dbm/tests/test_dbm_mnist_fast.py | 42 ++++++++++--------- 4 files changed, 49 insertions(+), 41 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 2cc41052cc..b8e460bed9 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -5,9 +5,12 @@ from pylearn2.scripts.dbm.augment_input import augment_input from pylearn2.utils import serial +""" +Augmented MNIST wrapper class +""" class MNIST_AUGMENTED(DenseDesignMatrix): - + """ Loads MNIST dataset and builds augmented dataset for DBM discriminative finetuning. diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index fc5639d82d..11ae477ac0 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -13,6 +13,7 @@ hidden layer of the DBM. """ + def augment_input(X, model, mf_steps): """ diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index d2bf76a483..11fc81127f 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -152,7 +152,7 @@ def test_train_example(): # START SUPERVISED TRAINING WITH BACKPROPAGATION print("\n-----------------------------------" " Supervised training " - "-----------------------------------\n" + "-----------------------------------\n") # load dbm as a mlp if DROPOUT: @@ -183,23 +183,25 @@ def test_train_example(): 'dbm_mnist.pkl')) train.dataset = mnist_augmented.MNIST_AUGMENTED( - dataset=train.dataset, - which_set='train', - one_hot=1, - model=dbm, start=0, - stop=hyper_params_mlp['train_stop'], - mf_steps=MF_STEPS) + dataset=train.dataset, + which_set='train', + one_hot=1, + model=dbm, start=0, + stop=hyper_params_mlp['train_stop'], + mf_steps=MF_STEPS) train.algorithm.monitoring_dataset = { - # 'valid' : mnist_augmented.MNIST_AUGMENTED( - # dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], - # mf_steps=mf_steps), - 'test' : mnist_augmented.MNIST_AUGMENTED( - dataset=train.algorithm.monitoring_dataset['test'], - which_set='test', one_hot=1, model=dbm, - mf_steps=MF_STEPS)} + ''' + 'valid': mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['valid'], + which_set='train', one_hot=1, model=dbm, + start=hyper_params_mlp['train_stop'], + stop=hyper_params_mlp['valid_stop'], + mf_steps=mf_steps), + ''' + 'test': mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, + mf_steps=MF_STEPS)} # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS # EXPLAINED BY HINTON @@ -207,9 +209,9 @@ def test_train_example(): # Concatenate weights between first and second hidden # layer & weights between visible and first hidden layer train.model.layers[0].set_weights( - numpy.concatenate(( - dbm.hidden_layers[1].get_weights().transpose(), - dbm.hidden_layers[0].get_weights()))) + numpy.concatenate(( + dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index cb2f9def70..4866b94cf9 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -18,7 +18,7 @@ @no_debug_mode def test_train_example(): - + """ Fast version of test script """ @@ -145,23 +145,25 @@ def test_train_example(): dbm = serial.load(os.path.join(train_path, 'dbm_mnist.pkl')) train.dataset = mnist_augmented.MNIST_AUGMENTED( - dataset=train.dataset, - which_set='train', - one_hot=1, - model=dbm, - start=0, - stop=hyper_params_mlp['train_stop'], - mf_steps=1) + dataset=train.dataset, + which_set='train', + one_hot=1, + model=dbm, + start=0, + stop=hyper_params_mlp['train_stop'], + mf_steps=1) train.algorithm.monitoring_dataset = { - # 'valid': mnist_augmented.MNIST_AUGMENTED( - # dataset=train.algorithm.monitoring_dataset['valid'], - # which_set='train', one_hot=1, model=dbm, - # start=hyper_params_mlp['train_stop'], - # stop=hyper_params_mlp['valid_stop'], mf_steps=1), - 'test': mnist_augmented.MNIST_AUGMENTED( - dataset=train.algorithm.monitoring_dataset['test'], - which_set='test', one_hot=1, model=dbm, start=0, - stop=20, mf_steps=1) + ''' + 'valid': mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['valid'], + which_set='train', one_hot=1, model=dbm, + start=hyper_params_mlp['train_stop'], + stop=hyper_params_mlp['valid_stop'], mf_steps=1), + ''' + 'test': mnist_augmented.MNIST_AUGMENTED( + dataset=train.algorithm.monitoring_dataset['test'], + which_set='test', one_hot=1, model=dbm, start=0, + stop=20, mf_steps=1) } # DBM TRAINED WEIGHTS CLAMPED FOR FINETUNING AS @@ -170,9 +172,9 @@ def test_train_example(): # Concatenate weights between first and second hidden layer & # weights between visible and first hidden layer train.model.layers[0].set_weights( - numpy.concatenate(( - dbm.hidden_layers[1].get_weights().transpose(), - dbm.hidden_layers[0].get_weights()))) + numpy.concatenate(( + dbm.hidden_layers[1].get_weights().transpose(), + dbm.hidden_layers[0].get_weights()))) # then clamp all the others normally for l, h in zip(train.model.layers[1:], dbm.hidden_layers[1:]): From bb6886a9cc38357250a75020375d9f5c2b5deb2d Mon Sep 17 00:00:00 2001 From: carlo Date: Mon, 24 Nov 2014 14:51:42 +0100 Subject: [PATCH 27/32] module-level docstrings added --- pylearn2/datasets/mnist_augmented.py | 7 ++++--- pylearn2/scripts/dbm/augment_input.py | 8 ++++---- pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py | 10 ++++++++++ .../scripts/papers/dbm/tests/test_dbm_mnist_fast.py | 8 +++++++- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index b8e460bed9..0594452bfa 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -1,3 +1,7 @@ +""" +Augmented MNIST wrapper class +""" + import os import numpy as np @@ -5,9 +9,6 @@ from pylearn2.scripts.dbm.augment_input import augment_input from pylearn2.utils import serial -""" -Augmented MNIST wrapper class -""" class MNIST_AUGMENTED(DenseDesignMatrix): diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index 11ae477ac0..f7852950c9 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -1,7 +1,3 @@ -from pylearn2.utils import sharedX -from theano import function -import numpy - """ This module augments the dataset in order to make it suitable for DBM discriminative finetuning. @@ -13,6 +9,10 @@ hidden layer of the DBM. """ +from pylearn2.utils import sharedX +from theano import function +import numpy + def augment_input(X, model, mf_steps): diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index 11fc81127f..7c004cc13a 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -1,3 +1,8 @@ +""" +This is the test version that achieves the +Hinton & Salakhutdinov's results. +""" + __authors__ = "Carlo D'Eramo, Francesco Visin, Matteo Matteucci" __copyright__ = "Copyright 2014-2015, Politecnico di Milano" __credits__ = ["Carlo D'Eramo, Francesco Visin, Matteo Matteucci"] @@ -40,6 +45,11 @@ @no_debug_mode def test_train_example(): + """ + Parameters + ---------- + """ + # path definition cwd = os.getcwd() train_path = cwd # training path is the current working directory diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index 4866b94cf9..3d6eee9152 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -1,3 +1,8 @@ +""" +This is a fast version of test script +for DBM training. +""" + __authors__ = "Carlo D'Eramo, Francesco Visin, Matteo Matteucci" __copyright__ = "Copyright 2014-2015, Politecnico di Milano" __credits__ = ["Carlo D'Eramo, Francesco Visin, Matteo Matteucci"] @@ -20,7 +25,8 @@ def test_train_example(): """ - Fast version of test script + Parameters + ---------- """ # path definition From a84be21832a911089c6b522f07ff9a5b7fdd8ce7 Mon Sep 17 00:00:00 2001 From: carlo Date: Mon, 24 Nov 2014 15:49:09 +0100 Subject: [PATCH 28/32] function summary lines added --- pylearn2/scripts/dbm/augment_input.py | 2 ++ pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py | 3 +-- pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py | 3 +-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pylearn2/scripts/dbm/augment_input.py b/pylearn2/scripts/dbm/augment_input.py index f7852950c9..a2c52dd2b9 100644 --- a/pylearn2/scripts/dbm/augment_input.py +++ b/pylearn2/scripts/dbm/augment_input.py @@ -17,6 +17,8 @@ def augment_input(X, model, mf_steps): """ + Input augmentation script. + Parameters ---------- X : ndarray, 2-dimensional diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index 7c004cc13a..7c14358067 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -46,8 +46,7 @@ def test_train_example(): """ - Parameters - ---------- + Test script. """ # path definition diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index 3d6eee9152..a2c8296033 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -25,8 +25,7 @@ def test_train_example(): """ - Parameters - ---------- + Fast test script. """ # path definition From f994a69ea8c6d6c5b67d67ec351314fe16a5b9d7 Mon Sep 17 00:00:00 2001 From: carlo Date: Mon, 24 Nov 2014 17:44:45 +0100 Subject: [PATCH 29/32] parameter docstring corrected --- pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py | 4 ++++ pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py index 7c14358067..204014406d 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist.py @@ -47,6 +47,10 @@ def test_train_example(): """ Test script. + + Parameters + ---------- + WRITEME """ # path definition diff --git a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py index a2c8296033..9a441ba75c 100644 --- a/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py +++ b/pylearn2/scripts/papers/dbm/tests/test_dbm_mnist_fast.py @@ -26,6 +26,10 @@ def test_train_example(): """ Fast test script. + + Parameters + ---------- + WRITEME """ # path definition From 660aff08e89508e18c4c6d7fb15322c36be4bbdc Mon Sep 17 00:00:00 2001 From: carlo Date: Mon, 24 Nov 2014 23:19:30 +0100 Subject: [PATCH 30/32] parameter docstring corrected --- pylearn2/datasets/mnist_augmented.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pylearn2/datasets/mnist_augmented.py b/pylearn2/datasets/mnist_augmented.py index 0594452bfa..cba3c07168 100644 --- a/pylearn2/datasets/mnist_augmented.py +++ b/pylearn2/datasets/mnist_augmented.py @@ -20,7 +20,7 @@ class MNIST_AUGMENTED(DenseDesignMatrix): ---------- dataset : `pylearn2.datasets.dataset.Dataset` which_set : str - Select between training or test set. + Select between training and test set. model : `pylearn2.models.model.Model` The DBM to be finetuned. mf_steps : int From 0c86ba34f37af6ddce627298f358042ad12a4662 Mon Sep 17 00:00:00 2001 From: carlo Date: Sat, 17 Jan 2015 14:37:47 +0100 Subject: [PATCH 31/32] README corrected --- pylearn2/scripts/papers/dbm/README | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pylearn2/scripts/papers/dbm/README b/pylearn2/scripts/papers/dbm/README index 7015d2ef6d..4f7be45e61 100644 --- a/pylearn2/scripts/papers/dbm/README +++ b/pylearn2/scripts/papers/dbm/README @@ -14,10 +14,10 @@ There are two tests in /tests. The script to run the whole procedure, with all t suitable to be run on travis. It does not perform well because it uses a very small training set and a very small number of epochs. NO DROPOUT RESULTS: -The test returns a 0.094% test error WITHOUT softmax on the top of the DBM and dropout. +The test returns a 0.94% test error WITHOUT softmax on the top of the DBM and dropout. DROPOUT RESULTS: -The test returns a 0.084% test error WITH softmax on the top of the DBM and dropout. +The test returns a 0.84% test error WITH softmax on the top of the DBM and dropout. Experiments have been performed on Ubuntu 14.04 LTS using a NVIDIA Tesla C1060 GPU and a 8-core Intel(R) Core(TM) i7 CPU 920 @ 2.67GHz. I used openblas-base and numpy version 1.9.0, scipy version 0.13.3, theano version 0.6.0 and pylearn2 with 6264 commits. - \ No newline at end of file + From e5ddec46d3b433b44c8964c8a44a29ad200e9341 Mon Sep 17 00:00:00 2001 From: carlo Date: Wed, 11 Feb 2015 23:51:38 +0100 Subject: [PATCH 32/32] README updated --- pylearn2/scripts/papers/dbm/README | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pylearn2/scripts/papers/dbm/README b/pylearn2/scripts/papers/dbm/README index 4f7be45e61..677c92f903 100644 --- a/pylearn2/scripts/papers/dbm/README +++ b/pylearn2/scripts/papers/dbm/README @@ -5,6 +5,8 @@ An efficient learning procedure for Deep Boltzmann Machines. G. Hinton, and R. S The procedure is divided in three phases: pretraining of RBMs, training and finetuning. The test_dbm_mnist script allows to enable each phase of training and to select whether the DBM is composed of a softmax layer or not, and whether the MLP has to do finetuning with dropout or not. +This implementation works only for DBMs with 2 hidden layers: the stacking of RBMs to compose the DBM needs some changes to Contrastive Divergence algorithm that have not been implemented here. +However, it has been shown that using more than 2 layers in a DBM, does not guarantee to improve performances. As explained in the paper, the finetuning procedure uses an augmented input to feed the MLP and this implementation creates it using augment_input.py and mnistaugmented.py in pylearn2/datasets/. The latter takes the mnist dataset and augment it. Eventually, it saves .pkl files of the augmented dataset