new NN with loss based on f-mesure maximization

historical_anti-pattern_detection/MLP.py

@@ -5,90 +5,111 @@
 import argparse
 import sys
 import reader
-import shuffle
 import math
 
+import transformData as td
 import tensorflow as tf
 import matplotlib.pyplot as plt
 import numpy as np
 
-learning_rate = 0.05
-num_nodes_1= 40
-num_nodes_2 = 20
-beta = 0.01
+learning_rate = 0.1
+beta = 0.0
+layers = [40,20,8]
+
 
-input_size = 5
 num_labels = 2
 
+def layer(x, input_size, output_size):
+    w = tf.Variable(tf.truncated_normal(shape=[input_size, output_size]))
+    b = tf.Variable(tf.zeros([output_size]))
+    return w, tf.sigmoid(tf.matmul(x,w) + b)
+
+
 graph = tf.Graph()
 with graph.as_default():
 
 
     instances , labels = reader.constructDataset3()
 
-    dataset_x , dataset_y = shuffle.shuffle(instances , labels)
+    dataset_x , dataset_y = td.shuffle(instances , labels)
 
-    data_x, data_y = shuffle.rebalanceData(2,dataset_x,dataset_y)
+    '''data_x, data_y = shuffle.rebalanceData(2,dataset_x,dataset_y)
 
     validSet_start_idx = int(math.ceil(len(data_x)*0.7))
 
     x_train = data_x[:validSet_start_idx,:]
     y_train = data_y[:validSet_start_idx,:]
     x_valid = data_x[validSet_start_idx:,:]
-    y_valid = data_y[validSet_start_idx:,:]
+    y_valid = data_y[validSet_start_idx:,:]'''
 
-    '''validSet_start_idx = int(math.ceil(len(dataset_x)*0.7))
+    validSet_start_idx = int(math.ceil(len(dataset_x)*0.7))
 
     train_data_x = dataset_x[:validSet_start_idx,:]
     train_data_y = dataset_y[:validSet_start_idx,:]
     x_valid = dataset_x[validSet_start_idx:,:]
     y_valid = dataset_y[validSet_start_idx:,:]
 
-    x_train, y_train = shuffle.rebalanceData(8, train_data_x, train_data_y)'''
-
+    #x_train, y_train = shuffle.rebalanceData(2, train_data_x, train_data_y)
+    x_train, y_train = train_data_x, train_data_y
 
     # Input data. For the training data, we use a placeholder that will be fed
     # at run time with a training minibatch.
-    x = tf.placeholder(tf.float32,[None,input_size])
+    x = tf.placeholder(tf.float32,[None,len(x_train[0])])
     y_ = tf.placeholder(tf.float32,[None,num_labels])
 
-    # Variables.
-    weights_1 = tf.Variable(tf.truncated_normal([input_size, num_nodes_1]))
-    biases_1 = tf.Variable(tf.zeros([num_nodes_1]))
-    weights_2 = tf.Variable(tf.truncated_normal([num_nodes_1, num_nodes_2]))
-    biases_2 = tf.Variable(tf.zeros([num_nodes_2]))
-    weights_3 = tf.Variable(tf.truncated_normal([num_nodes_2, num_labels]))
-    biases_3 = tf.Variable(tf.zeros([num_labels]))
-
-    # Training computation.
-    logits_1 = tf.matmul(x, weights_1) + biases_1
-    relu_layer_1= tf.nn.tanh(logits_1)
-    logits_2 = tf.matmul(relu_layer_1, weights_2) + biases_2
-    relu_layer_2= tf.nn.tanh(logits_2)
-    logits_3 = tf.matmul(relu_layer_2, weights_3) + biases_3
+
+    h = x
+    regularizers = 0
+    input_size = len(x_train[0])
+    for size in layers:
+        w, h = layer(h, input_size, size)
+        regularizers = regularizers + tf.nn.l2_loss(w)
+        input_size = size
+
+    weight = tf.Variable(tf.truncated_normal(shape=[input_size, num_labels]))
+    biases = tf.Variable(tf.zeros([num_labels]))
+    regularizers = regularizers + tf.nn.l2_loss(weight)
+    logits = tf.matmul(h, weight) + biases 
+
     # Normal loss function
-    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits_3, labels=y_))
-    # Loss function with L2 Regularization with beta=0.01
-    regularizers = tf.nn.l2_loss(weights_1) + tf.nn.l2_loss(weights_2) + tf.nn.l2_loss(weights_3)
-    loss = tf.reduce_mean(loss + beta * regularizers)
+    #loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=y_))
+    # Loss function with L2 Regularization
+    #loss = tf.reduce_mean(loss + beta * regularizers)
 
+
+    param = tf.constant([1,0], tf.float32,shape=[1,2])
+    A = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(tf.multiply(y_,tf.nn.softmax(3*logits)),0,keep_dims=True),transpose_b=True))
+    npos = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(y_,0,keep_dims=True),transpose_b=True))
+    mpos = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(tf.nn.softmax(3*logits),0,keep_dims=True),transpose_b=True))
+
+
+    loss = 1 - 2*A/(npos+mpos)
     # Optimizer.
-    optimizer = tf.train.GradientDescentOptimizer(0.1).minimize(loss)
+    optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)
 
     # Predictions for the training
-    train_prediction = tf.nn.softmax(logits_3)
+    train_prediction = tf.nn.softmax(logits)
 
-    detected = tf.cast(tf.equal(tf.argmax(logits_3,1), 0), tf.float32)
+    detected = tf.cast(tf.equal(tf.argmax(logits,1), 0), tf.float32)
     correct = tf.cast(tf.equal(tf.argmax(y_,1), 0), tf.float32)
-    true_positive = tf.cast(tf.equal(tf.argmax(logits_3,1) + tf.argmax(y_,1), 0), tf.float32)
+    true_positive = tf.cast(tf.equal(tf.argmax(logits,1) + tf.argmax(y_,1), 0), tf.float32)
 
     precision = tf.reduce_sum(true_positive)/tf.reduce_sum(detected)
     recall = tf.reduce_sum(true_positive)/tf.reduce_sum(correct)
     f_mesure = 2*precision*recall/(precision+recall)
 
+    #tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)), reduction_indices=[1]))
+    T = tf.nn.softmax(logits)[0]
+    #param = tf.constant([1,0], tf.float32,shape=[1,2])
+    #TP = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(tf.multiply(y_,tf.nn.softmax(3*logits)),0,keep_dims=True),transpose_b=True))
+    #npos = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(y_,0,keep_dims=True),transpose_b=True))
+    #mpos = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(tf.nn.softmax(3*logits),0,keep_dims=True),transpose_b=True))
+    TP2 = tf.reduce_sum(true_positive)
+    TP3 = tf.reduce_sum(detected)
 
 
-num_steps = 4000
+
+num_steps = 1000
 losses = []
 fm = []
 bestLossStep = 0
@@ -100,7 +121,7 @@
     session.run(tf.global_variables_initializer())
     print("Initialized")
     for step in range(num_steps):
-        batch_data, batch_labels = shuffle.shuffle(x_train, y_train)
+        batch_data, batch_labels = td.shuffle(x_train, y_train)
 
         feed_dict = {x: batch_data, y_: batch_labels}
         session.run(optimizer, feed_dict=feed_dict)
@@ -116,7 +137,8 @@
             bestFM = f
             bestFMStep = step
 
-
+    print(session.run([detected, correct, TP2,A,mpos,TP3], feed_dict={x: x_valid,
+                                        y_: y_valid}))
     print(session.run([precision, recall, f_mesure], feed_dict={x: x_valid,
                                         y_: y_valid}))
 

historical_anti-pattern_detection/NN.py

@@ -0,0 +1,163 @@
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import sys
+import reader
+import math
+
+import transformData as td
+import tensorflow as tf
+import matplotlib.pyplot as plt
+import numpy as np
+
+starter_learning_rate = 0.30
+beta = 0.0
+layers = [32,16,8]
+
+
+def layer(x, n_in, n_out):
+    low = -4*np.sqrt(6.0/(n_in + n_out))
+    high = 4*np.sqrt(6.0/(n_in + n_out))
+    w = tf.Variable(tf.random_uniform(shape=[n_in, n_out], minval=low, maxval=high, dtype=tf.float32))
+    #w = tf.Variable(tf.truncated_normal(shape=[n_in, n_out]))
+    b = tf.Variable(tf.zeros([n_out]))
+    return w, tf.matmul(x,w) + b, tf.sigmoid(tf.matmul(x,w) + b)
+
+
+''' Differentiable approximation of the f-mesure performed by the model on a given dataset.
+    Based on the article of Martin Jansche "Maximum Expected F-Measure Training of Logistic Regression Models"
+
+    true_positive ≈ sum(sigmoid(gamma*logits)) for label = +1
+    detected ≈ sum(sigmoid(gamma*logits))
+
+    for gamma -> +∞
+'''
+def f_mesure_approx(logits, labels, gamma):
+    param = tf.constant([1,0], tf.float32,shape=[1,2])
+    true_positive = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(tf.multiply(labels,tf.nn.softmax(gamma*logits)),0,keep_dims=True),transpose_b=True))
+    positive = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(labels,0,keep_dims=True),transpose_b=True))
+    detected = tf.reduce_mean(tf.matmul(param, tf.reduce_sum(tf.nn.softmax(gamma*logits),0,keep_dims=True),transpose_b=True))
+
+    return 2*true_positive/(positive+detected)
+
+
+def evaluate_model(logits, labels):
+    true_positive = tf.cast(tf.equal(tf.argmax(logits,1) + tf.argmax(y_,1), 0), tf.float32)
+    positive = tf.cast(tf.equal(tf.argmax(y_,1), 0), tf.float32)
+    detected = tf.cast(tf.equal(tf.argmax(logits,1), 0), tf.float32)
+    correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(y_,1))
+
+    precision = tf.reduce_sum(true_positive)/tf.reduce_sum(detected)
+    recall = tf.reduce_sum(true_positive)/tf.reduce_sum(positive)
+    f_mesure = 2*precision*recall/(precision+recall)
+    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
+
+    return precision, recall, f_mesure, accuracy    
+
+graph = tf.Graph()
+with graph.as_default():
+
+    instances , labels = reader.constructDataset2()
+    dataset_x , dataset_y = td.shuffle(instances , labels)
+
+    validSet_start_idx = int(math.ceil(len(dataset_x)*0.7))
+
+    x_train = dataset_x[:validSet_start_idx,:]
+    y_train = dataset_y[:validSet_start_idx,:]
+    x_valid = dataset_x[validSet_start_idx:,:]
+    y_valid = dataset_y[validSet_start_idx:,:]
+
+    nonZeroTotal = np.nonzero(dataset_y[:,0])[0].size
+    nonZeroTrain = np.nonzero(y_train[:,0])[0].size
+    nonZeroValid = np.nonzero(y_valid[:,0])[0].size
+
+    print('nonzero :', nonZeroTotal, nonZeroTrain, nonZeroValid)
+
+    input_size = len(x_train[0]) #4
+    output_size = len(y_train[0]) #2
+    layers.append(output_size)
+
+    # Input data. For the training data, we use a placeholder that will be fed
+    # at run time with a training minibatch.
+    x = tf.placeholder(tf.float32,[None,input_size])
+    y_ = tf.placeholder(tf.float32,[None,output_size])
+
+
+    # Construct model
+    # Construct model
+    h = x
+    logits = x
+    regularizers = 0
+    n_in = input_size
+    for size in layers:
+        w, logits, h = layer(h, n_in, size)
+        regularizers = regularizers + tf.nn.l2_loss(w)
+        n_in = size
+
+
+    # Loss function with L2 Regularization
+    loss = 1 - f_mesure_approx(logits, y_, 2)
+    loss = tf.reduce_mean(loss + beta * regularizers)
+
+
+    # Learning mecanism 
+    global_step = tf.Variable(0, trainable=False)
+    learning_rate = tf.train.exponential_decay(starter_learning_rate, global_step,
+                                           400, 0.9, staircase=True)
+    # Passing global_step to minimize() will increment it at each step.
+    learning_step = (
+        tf.train.GradientDescentOptimizer(learning_rate)
+        .minimize(loss, global_step=global_step)
+    )
+
+    # Predictions for the training
+    train_prediction = tf.nn.softmax(logits)
+
+    precision, recall, f_mesure, accuracy = evaluate_model(logits, y_)
+
+
+
+num_steps = 4000
+losses = []
+fm = []
+lrates = []
+bestLossStep = 0
+bestLoss = 100
+bestFMStep = 0
+bestFM = 0
+
+with tf.Session(graph=graph) as session:
+    session.run(tf.global_variables_initializer())
+    print("Initialized")
+
+    for step in range(num_steps):
+        batch_data, batch_labels = td.shuffle(x_train, y_train)
+        feed_dict = {x: batch_data, y_: batch_labels}
+
+        session.run(learning_step, feed_dict=feed_dict)
+        l, f, lr = session.run([loss, f_mesure, learning_rate], feed_dict={x:x_valid, y_:y_valid})
+        losses.append(l)
+        fm.append(f)
+        lrates.append(lr)
+
+        if l < bestLoss:
+            bestLoss = l
+            bestLossStep = step
+
+        if f > bestFM:
+            bestFM = f
+            bestFMStep = step
+
+    precision, recall, f_mesure, accuracy = session.run([precision, recall, f_mesure, accuracy], feed_dict={x: x_valid, y_: y_valid})
+    print('Precision :', precision)
+    print('Recall :', recall)
+    print('F-Mesure :', f_mesure)
+    print('Accuracy :', accuracy)
+    print('\n')
+    print('Best loss :',bestLoss,' at step :',bestLossStep)
+    print('Best f-mesure :',bestFM,' at step :',bestFMStep)
+
+    plt.plot(range(num_steps), losses,range(num_steps), fm, range(num_steps), lrates)
+    plt.show()

historical_anti-pattern_detection/data/anti-pattern_occurences/blob/apache-log4j.csv

@@ -0,0 +1,4 @@
+src.java.org.apache.log4j.xml.DOMConfigurator
+src.java.org.apache.log4j.net.SyslogAppender
+src.java.org.apache.log4j.net.SMTPAppender
+src.java.org.apache.log4j.Category

historical_anti-pattern_detection/data/anti-pattern_occurences/blob/apache-pig.csv

@@ -0,0 +1,5 @@
+src.org.apache.pig.tools.pigstats.ScriptState
+src.org.apache.pig.PigServer
+src.org.apache.pig.impl.plan.OperatorPlan
+src.org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange
+src.org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MultiQueryOptimizer

historical_anti-pattern_detection/data/anti-pattern_occurences/blob/apache-struts.csv

@@ -0,0 +1,3 @@
+java.com.opensymphony.xwork2.config.providers.XmlConfigurationProvider
+java.org.apache.struts2.dispatcher.Dispatcher
+java.org.apache.struts2.components.UIBean