Add files via upload

CalculateEntropReducerDate.py

@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+
+from operator import itemgetter
+import sys
+
+en_score = {}
+
+for line in sys.stdin:
+    line = line.strip()
+    if line!="" and line is not None:
+        splitLine = line.split()
+        topicId = splitLine[0]
+        score = splitLine[1]
+        date = splitLine[2]
+        dateTopicId = date + " " + topicId
+
+        if dateTopicId in en_score:
+                en_score[dateTopicId].append(float(score))
+        else:
+                en_score[dateTopicId] = []
+                en_score[dateTopicId].append(float(score))
+
+
+for topic in en_score.keys():
+    avg_score = sum(en_score[topic])/len(en_score[topic])
+    print '%s\t%s'% (topic, avg_score)

CalculateEntropyMapperDate.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python
+
+
+import sys, math, nltk
+sys.path.append('./')
+
+
+def readFileandReturnAnArray(fileName, readMode, isLower):
+    myArray=[]
+    with open(fileName, readMode) as readHandle:
+        for line in readHandle.readlines():
+            lineRead = line
+            if isLower:
+                lineRead = lineRead.lower()
+            myArray.append(lineRead.strip().lstrip())
+    readHandle.close()
+    return myArray
+
+
+for line in sys.stdin:
+    try:
+        line = line.strip()
+        date = line[-12:]
+        line = line[:-12]
+        Tokens = nltk.word_tokenize(line)
+        length = len(Tokens)
+        topicfiles =["foodtopic1", "foodtopic2", "foodtopic3"]
+        for i in topicfiles:
+            alphabet = readFileandReturnAnArray(i, "r", True)
+            topicId = alphabet.pop(0)
+            freqList = []
+            for symbol in alphabet:
+                words = nltk.word_tokenize(symbol)
+                if words.__len__() != 1:
+                     cntList=[]
+                     newcnt = 0
+                     for word in words:
+                         newcnt = Tokens.count(word)
+                         cntList.append(newcnt)
+                     if all(map(lambda x: x == cntList[0], cntList)) == True and len(cntList) != 0:
+                         cnt = cntList[0]
+                     else:
+                         cnt=0
+                else:
+                     cnt = Tokens.count(symbol)
+                if cnt != 0 and length != 0:
+                    wordProb = float(cnt)/length
+                    freqList.append(wordProb)
+            # Shannon entropy
+            ent = 0.0
+            for freq in freqList:
+                ent = ent + float(freq) * math.log(freq, 2)
+            if ent != 0.0:
+                ent = -ent
+                print '%s\t%s\t%s' % (topicId,ent,date)
+    except ValueError:
+        continue