change output of smad to sigmoid and rewrite perfromance metrics with…

… numpy instead of tensorflow

experiments/training/train_smad.py

@@ -29,7 +29,7 @@ def parse_args():
 	parser.add_argument('-dense_sizes', nargs='+', type=int, help="The sizes of each (dense) hidden layer in the network.")
 	parser.add_argument("-n_net", type=int, default=10, help="The number of distinct networks to be trained and saved.")
 	parser.add_argument("-n_step", type=int, default=300, help="The number of training steps.")
-	parser.add_argument("-decay_step", type=int, default=100, help="The number of training steps after which the learning rate is decayed")
+	parser.add_argument("-decay_step", type=int, default=300, help="The number of training steps after which the learning rate is decayed")
 	parser.add_argument("-lr_decay", type=float, default=0.5, help="The factor by which the learning rate is multiplied every 'decay_step' steps")
 	return parser.parse_args()
 
@@ -40,7 +40,7 @@ def get_save_path(antipattern, net_number):
 def build_dataset(antipattern, systems):
 	input_size = {'god_class':8, 'feature_envy':9}
 	X = np.empty(shape=[0, input_size[antipattern]])
-	Y = np.empty(shape=[0, 2])
+	Y = np.empty(shape=[0, 1])
 	for systemName in systems:
 		X = np.concatenate((X, nnUtils.getInstances(systemName, antipattern)), axis=0)
 		Y = np.concatenate((Y, nnUtils.getLabels(systemName, antipattern)), axis=0)
@@ -60,13 +60,14 @@ def train(session, model, x_train, y_train, x_test, y_test, num_step, start_lr,
 		feed_dict_train = {
 					model.input_x: x_train,
 					model.input_y: y_train,
+					model.training: True,
 					model.learning_rate:learning_rate,
 					model.beta:beta}
 
 		session.run(model.learning_step, feed_dict=feed_dict_train)
 
-		loss_train = session.run(model.loss, feed_dict={model.input_x:x_train, model.input_y:y_train})
-		loss_test  = session.run(model.loss, feed_dict={model.input_x:x_test, model.input_y:y_test})
+		loss_train = session.run(model.loss, feed_dict={model.input_x:x_train, model.input_y:y_train, model.training: False})
+		loss_test  = session.run(model.loss, feed_dict={model.input_x:x_test, model.input_y:y_test, model.training: False})
 		losses_train.append(loss_train)
 		losses_test.append(loss_test)
 	return losses_train, losses_test
@@ -126,10 +127,9 @@ def train(session, model, x_train, y_train, x_test, y_test, num_step, start_lr,
 
 	# Print Ensemble performances
 	print("\nPerformances on " + args.test_system + ": ")
-	print('Precision :')
-	print('Recall    :')
-	print('F-Mesure  :')
-	print('Accuracy  :')
+	print('Precision: ' + str(nnUtils.precision(ensemble_prediction, y_test)))
+	print('Recall   : ' + str(nnUtils.recall(ensemble_prediction, y_test)))
+	print('F-Mesure : ' + str(nnUtils.f_measure(ensemble_prediction, y_test)))
 
 	# Plot learning curves
 	nnUtils.plot_learning_curves(all_losses_train, all_losses_test)

neural_networks/customLoss.py → neural_networks/loss.py

@@ -2,23 +2,22 @@
 
 import tensorflow as tf
 
-
 def f_measure_approx(logits, labels, gamma):
     ''' 
     This function implements the Differentiable approximation of the f-measure from:
     Martin Jansche (2005):
         [Maximum Expected F-Measure Training of Logistic Regression Models]
 
-    true_positive: sum(sigmoid(gamma*logits)) for label = +1
+    true_positive:  sum(sigmoid(gamma*logits)) for label = +1
     detected: sum(sigmoid(gamma*logits))
     gamma > 0
     '''
-    true_positive = tf.reduce_sum(tf.multiply(labels, tf.nn.softmax(gamma*logits)), 0)[0]
-    positive = tf.reduce_sum(labels, 0)[0]
-    detected = tf.reduce_sum(tf.nn.softmax(gamma*logits), 0)[0]
+    true_positive = tf.reduce_sum(tf.multiply(labels, tf.nn.sigmoid(gamma*logits)))
+    positive = tf.reduce_sum(labels)
+    detected = tf.reduce_sum(tf.nn.sigmoid(gamma*logits))
 
     return 2*true_positive/(positive+detected)
 
 # Custom loss function that maximizes f-measure  
-def loss(logits, labels):
-	return 1 - f_measure_approx(logits, labels, 4)
+def compute(logits, labels):
+    return 1 - f_measure_approx(logits, labels, 4)

neural_networks/smad/context.py

@@ -4,4 +4,4 @@
 ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..'))
 sys.path.insert(0, ROOT_DIR)
 
-import neural_networks.customLoss as customLoss
+import neural_networks.loss as loss

neural_networks/smad/model.py

@@ -1,4 +1,4 @@
-from context import customLoss
+from context import loss
 
 import tensorflow as tf
 
@@ -8,11 +8,12 @@ def __init__(self, shape, input_size):
 
 		# Placeholders for instances and labels
 		self.input_x = tf.placeholder(tf.float32,[None, input_size], name="input_x")
-		self.input_y = tf.placeholder(tf.float32,[None, 2], name="input_y")
+		self.input_y = tf.placeholder(tf.float32,[None, 1], name="input_y")
 
-		# Placeholders for learning parameters
-		self.learning_rate     = tf.placeholder(tf.float32, name="learning_rate")
-		self.beta              = tf.placeholder(tf.float32, name="beta")
+		# Placeholders for training parameters
+		self.training      = tf.placeholder(tf.bool, name="training")
+		self.learning_rate = tf.placeholder(tf.float32, name="learning_rate")
+		self.beta          = tf.placeholder(tf.float32, name="beta")
 
 		# L2 regularization & initialization
 		l2_reg = tf.contrib.layers.l2_regularizer(scale=self.beta)
@@ -22,25 +23,27 @@ def __init__(self, shape, input_size):
 		h_in = self.input_x
 		for size in shape:
 			with tf.name_scope("hidden-%s" % size):
-				h_in = tf.layers.dense(h_in,
-									size,
-									activation=tf.tanh,
-									kernel_initializer=xavier,
-									kernel_regularizer=l2_reg,
-									bias_regularizer=l2_reg)
+				h_in = tf.layers.dense(
+					inputs=h_in,
+					units=size,
+					activation=tf.tanh,
+					kernel_initializer=xavier,
+					kernel_regularizer=l2_reg,
+					bias_regularizer=l2_reg)
 
 		# Output layer
 		with tf.name_scope("output"):
-			self.logits = tf.layers.dense(h_in,
-										2,
-										kernel_initializer=xavier,
-										kernel_regularizer=l2_reg,
-										bias_regularizer=l2_reg)
-			self.inference = tf.nn.softmax(self.logits)
+			self.logits = tf.layers.dense(
+				inputs=h_in,
+				units=1,
+				kernel_initializer=xavier,
+				kernel_regularizer=l2_reg,
+				bias_regularizer=l2_reg)
+			self.inference = tf.nn.sigmoid(self.logits)
 
 		# Loss function
 		with tf.name_scope("loss"):
-			self.loss = customLoss.loss(self.logits, self.input_y)
+			self.loss = loss.compute(self.logits, self.input_y)
 			l2_loss = tf.losses.get_regularization_loss()
 			loss_reg = self.loss + l2_loss
 

utils/nnUtils.py

@@ -10,33 +10,26 @@
 import sys
 
 ### EVALUATION ####
-def true_positive(output, labels):
-	tp = tf.cast(tf.equal(tf.argmax(output,1) + tf.argmax(labels,1), 0), tf.float32)
+def detected(output):
+	return np.sum((output > 0.5).astype(float))
 
-	return tp
+def positive(labels):
+	return np.sum(labels)
 
-def precision(output, labels):
-	tp = true_positive(output, labels)
-	detected = tf.cast(tf.equal(tf.argmax(output,1), 0), tf.float32)
+def true_positive(output, labels):
+	return np.sum((output + labels > 1.5).astype(float))
 
-	return tf.reduce_sum(tp)/tf.reduce_sum(detected)
+def precision(output, labels):
+	return true_positive(output, labels)/detected(output)
 
 def recall(output, labels):
-	tp = true_positive(output, labels)
-	positive = tf.cast(tf.equal(tf.argmax(labels,1), 0), tf.float32)
-
-	return tf.reduce_sum(tp)/tf.reduce_sum(positive)
+	return true_positive(output, labels)/positive(labels)
 
 def f_measure(output, labels):
-	prec = precision(output, labels)
-	rec = recall(output, labels)
-
-	return 2*prec*rec/(prec+rec)
+	p = precision(output, labels)
+	r = recall(output, labels)
 
-def accuracy(output, labels):
-	correct_prediction = tf.equal(tf.argmax(output, 1), tf.argmax(labels,1))
-
-	return tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+	return 2*p*r/(p+r)
 
 
 ### UTILS ###
@@ -58,7 +51,7 @@ def ensemble_prediction(model, save_paths, input_x):
 	with tf.Session() as session:
 		for save_path in save_paths:
 			saver.restore(sess=session, save_path=save_path)
-			prediction = session.run(model.inference, feed_dict={model.input_x: input_x})
+			prediction = session.run(model.inference, feed_dict={model.input_x: input_x, model.training: False})
 			predictions.append(prediction)
 
 	return np.mean(np.array(predictions), axis=0)
@@ -104,9 +97,9 @@ def getLabels(systemName, antipattern):
 	true = dataUtils.getAntipatterns(systemName, antipattern)
 	for entity in entities:
 		if entity in true:
-			labels.append([1, 0])
+			labels.append([1.])
 		else:
-			labels.append([0, 1])
+			labels.append([0.])
 
 	return np.array(labels)
 
@@ -126,7 +119,6 @@ def getInstances(systemName, antipattern, normalized=True):
 		metrics.append(dataUtils.getFEInCodeMetrics(systemName))
 		metrics.append(dataUtils.getFEJDeodorantMetrics(systemName))
 
-
 	instances = []
 	for entity in entities:
 		instance = []