TREC-IS_V3_2019AEvents_2019AFormat_Evaluation_Notebook (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading 2019A-assr1.json\n",
      "Reading 2019A-assr2.json\n",
      "Reading 2019A-assr3.json\n",
      "Reading 2019A-assr4.json\n",
      "Reading 2019A-assr5.json\n",
      "Reading 2019A-assr6.json\n",
      "Reading 2019-assr2.json\n"
     ]
    }
   ],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC IS 2019 Evaluation Script\n",
    "# Configured for 2019-A Events\n",
    "# Used to evaluate TREC-IS runs\n",
    "# --------------------------------------------------\n",
    "version = 2.0 # Notebook Version Number\n",
    "\n",
    "# Configuration (Change this to match your setting)\n",
    "# System output file to evaluate:\n",
    "runFile = \"mmyrun.run\"\n",
    "runName = \"myrun\"\n",
    "\n",
    "# The location of the ground truth data against which to compare the run\n",
    "classificationLabelFiles = [\n",
    "    \"2019A-assr1.json\",\n",
    "    \"2019A-assr2.json\",\n",
    "    \"2019A-assr3.json\",\n",
    "    \"2019A-assr4.json\",\n",
    "    \"2019A-assr5.json\",\n",
    "    \"2019A-assr6.json\",\n",
    "    \"2019-assr2.json\"\n",
    "]\n",
    "\n",
    "# The location of the ontology file\n",
    "ontologyFile = \"ITR-H.types.v4.json\"\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Static data for the 2019 edition\n",
    "# --------------------------------------------------\n",
    "# Identifiers for the test events\n",
    "eventIdentifiers = [\n",
    "    \"floodChoco2019\",\n",
    "    \"earthquakeCalifornia2014\",\n",
    "    \"earthquakeBohol2013\",\n",
    "    \"hurricaneFlorence2018\",\n",
    "    \"shootingDallas2017\",\n",
    "    \"fireYMM2016\"\n",
    "]\n",
    "\n",
    "# Mapping of prority labels (by assessors) into numerical values [0-1]\n",
    "# We use this to calculate error against the participant priority scores\n",
    "priorityMapping = {\n",
    "    \"Critical\" : 1.0,\n",
    "    \"High\" : 0.75,\n",
    "    \"Medium\" : 0.5,\n",
    "    \"Low\" : 0.25\n",
    "}\n",
    "\n",
    "# What we consider to be highly important categories of information\n",
    "highImportCategories = [\n",
    "    \"Request-GoodsServices\",\n",
    "    \"Request-SearchAndRescue\",\n",
    "    \"CallToAction-MovePeople\",\n",
    "    \"Report-EmergingThreats\",\n",
    "    \"Report-NewSubEvent\",\n",
    "    \"Report-ServiceAvailable\"\n",
    "]\n",
    "\n",
    "highImportCategoriesShort = [\n",
    "    \"GoodsServices\",\n",
    "    \"SearchAndRescue\",\n",
    "    \"MovePeople\",\n",
    "    \"EmergingThreats\",\n",
    "    \"NewSubEvent\",\n",
    "    \"ServiceAvailable\"\n",
    "]\n",
    "\n",
    "# Parameters\n",
    "var_lambda = 0.75 # weight to place on actionable information categories in comparison to non actionable categoriee\n",
    "var_alpha = 0.3 # Flat gain for providing a correct alert, regardless of the categories selected\n",
    "alertPriorityThreshold = 0.7\n",
    "\n",
    "resultsFile = open(runName+\".results.overall.txt\",\"w+\")\n",
    "resultsFile.write(\"TREC-IS 2019-A Notebook Evaluator v\"+str(version)+\"\\n\")\n",
    "resultsFile.write(\"Run: \"+runName+\" (\"+runFile+\")\"+\"\\n\")\n",
    "resultsFile.write(\"\"+\"\\n\")\n",
    "\n",
    "perTopicFile = open(runName+\".results.pertopic.txt\",\"w+\")\n",
    "perTopicFile.write(\"TREC-IS 2019-A Notebook Evaluator v\"+str(version)+\"\\n\")\n",
    "perTopicFile.write(\"Run: \"+runName+\" (\"+runFile+\")\"+\"\\n\")\n",
    "perTopicFile.write(\"\"+\"\\n\")\n",
    "\n",
    "perEventFile = open(runName+\".results.perevent.txt\",\"w+\")\n",
    "perEventFile.write(\"TREC-IS 2019-A Notebook Evaluator v\"+str(version)+\"\\n\")\n",
    "perEventFile.write(\"Run: \"+runName+\" (\"+runFile+\")\"+\"\\n\")\n",
    "perEventFile.write(\"\"+\"\\n\")\n",
    "\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Processing Starts Here\n",
    "# --------------------------------------------------\n",
    "import json\n",
    "from pprint import pprint\n",
    "import gzip\n",
    "import math\n",
    "import numpy as np\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Stage 1: Load the ground truth dataset \n",
    "# --------------------------------------------------\n",
    "\n",
    "groundtruthJSON = []\n",
    "for groundtruthFile in classificationLabelFiles:\n",
    "    print(\"Reading \"+groundtruthFile)\n",
    "    with open(groundtruthFile) as groundtruthJSONFile:    \n",
    "        groundtruthJSON.append(json.load(groundtruthJSONFile))\n",
    "#pprint(groundtruthJSON[\"events\"])\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Stage 2: Load run file \n",
    "# --------------------------------------------------\n",
    "with open(runFile, encoding='utf-8') as openRunFile:\n",
    "    runContents = openRunFile.readlines() # lines not yet decoded\n",
    "#pprint(runContents[0])\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Stage 3: Load the categories \n",
    "# --------------------------------------------------\n",
    "with open(ontologyFile, encoding='utf-8') as ontologyJSONFile:    \n",
    "    ontologyJSON = json.load(ontologyJSONFile)\n",
    "\n",
    "informationTypes2Index = {} # category -> numerical index\n",
    "informationTypesShort2Index = {} # category short form (e.g. Report-EmergingThreats vs. EmergingThreats) -> numerical index\n",
    "informationTypeIndex = 0\n",
    "for informationTypeJSON in ontologyJSON[\"informationTypes\"]:\n",
    "    informationTypeId = informationTypeJSON[\"id\"]\n",
    "    informationTypes2Index[informationTypeId] = informationTypeIndex\n",
    "    informationTypesShort2Index[informationTypeId.split(\"-\")[1]] = informationTypeIndex\n",
    "    informationTypeIndex = informationTypeIndex + 1\n",
    "    \n",
    "# -----------------------------------------------------------\n",
    "# Stage 4: Produce ground truth maps between tweetIds and categories\n",
    "# -----------------------------------------------------------\n",
    "# Notes: Ground truth is used as a base, if a run includes tweets\n",
    "#        not in the ground truth they will be ignored\n",
    "# Assumptions: A tweet will not be returned for multiple events\n",
    "\n",
    "tweetId2TRECInfoCategories = {} # tweet id -> Array of categories selected by assessors\n",
    "tweetId2TRECHighImportInfoCategories = {} # tweet id -> Array of categories selected by assessors\n",
    "tweetId2TRECLowImportInfoCategories = {} # tweet id -> Array of categories selected by assessors\n",
    "tweetId2TRECPriorityCategory = {} # tweet id -> priority label (Critical,High,Medium,Low)\n",
    "index2TweetId = {} # ordered tweets\n",
    "event2tweetIds = {} # event -> tweet ids for tweets within that event\n",
    "countHighCriticalImport = 0\n",
    "countLowMediumImport = 0\n",
    "\n",
    "tweetIndex = 0\n",
    "for groundtruth in groundtruthJSON:\n",
    "    for eventJSON in groundtruth[\"events\"]:\n",
    "        eventid = eventJSON[\"eventid\"]\n",
    "        #print(eventid)\n",
    "        # two events were split and assessed in parts, re-name these so they are correctly read\n",
    "        if eventid.endswith(\"A\") | eventid.endswith(\"B\") | eventid.endswith(\"C\") | eventid.endswith(\"D\") | eventid.endswith(\"E\"):\n",
    "            eventid = eventid[:-1]\n",
    "        \n",
    "        if not event2tweetIds.get(eventid):\n",
    "            event2tweetIds[eventid] = []\n",
    "        \n",
    "        if any(eventid in s for s in eventIdentifiers):\n",
    "            # iterate over tweets in the event\n",
    "            for tweetJSON in eventJSON[\"tweets\"]:\n",
    "                tweetid = tweetJSON[\"postID\"]\n",
    "                categories = tweetJSON[\"categories\"]\n",
    "                priority = tweetJSON[\"priority\"]\n",
    "                \n",
    "                if priority == \"High\" or priority == \"Critical\":\n",
    "                    countHighCriticalImport = countHighCriticalImport + 1\n",
    "                \n",
    "                if priority == \"Low\" or priority == \"Medium\":\n",
    "                    countLowMediumImport = countLowMediumImport + 1\n",
    "                \n",
    "                event2tweetIds[eventid].append(tweetid)\n",
    "                \n",
    "                # check categories for name issues and correct if possible\n",
    "                cleanedCategories = []\n",
    "                highImportCats = []\n",
    "                lowImportCats = []\n",
    "                for categoryId in categories:\n",
    "                    if not any(categoryId in s for s in informationTypesShort2Index.keys()):\n",
    "                        print(\"Found unknown category \"+categoryId)\n",
    "                    else:\n",
    "                        cleanedCategories.append(categoryId)\n",
    "                        if any(categoryId in s for s in highImportCategoriesShort):\n",
    "                            highImportCats.append(categoryId)\n",
    "                        else:\n",
    "                            lowImportCats.append(categoryId)\n",
    "    \n",
    "                tweetId2TRECInfoCategories[tweetid] = cleanedCategories\n",
    "                tweetId2TRECHighImportInfoCategories[tweetid] = highImportCats\n",
    "                tweetId2TRECLowImportInfoCategories[tweetid] = lowImportCats\n",
    "                tweetId2TRECPriorityCategory[tweetid] = priority\n",
    "                index2TweetId[tweetIndex] = tweetid;\n",
    "                tweetIndex = tweetIndex + 1\n",
    "\n",
    "                \n",
    "        else:\n",
    "            print(\"WARN: Found ground truth data for event not in the 2019 topic set \"+eventid+\", ignoring...\")\n",
    "        \n",
    "# -----------------------------------------------------------\n",
    "# Stage 5: Produce run predicted maps between tweetIds and categories\n",
    "# -----------------------------------------------------------\n",
    "tweetId2RunInfoCategories = {} # tweet id -> predicted category by participant system\n",
    "tweetId2RunHighImportInfoCategories = {} # tweet id -> predicted category by participant system\n",
    "tweetId2RunLowImportInfoCategories = {} # tweet id -> predicted category by participant system\n",
    "tweetId2RunPriorityScore = {} # tweet id -> importance score from participant system\n",
    "tweetId2RunPriorityScoreNorm = {} # tweet id -> importance score from participant system\n",
    "event2TweetIdRank = {} # event -> (rank,tweetid)\n",
    "\n",
    "maxPrediction = -999999\n",
    "minPrediction = 999999\n",
    "\n",
    "for runLine in runContents:\n",
    "    predictionParts = runLine.strip().replace(\", \", \",\").replace(\"\\\",\\\" \", \"\\\",\\\"\").replace(\" \",\"\\t\").replace(\"'\",\"\\\"\").split('\\t')\n",
    "    \n",
    "    #print(runLine)\n",
    "    if (len(predictionParts)<6 ):\n",
    "        continue\n",
    "    else:\n",
    "        eventId = predictionParts[0]\n",
    "        tweetId = predictionParts[2]\n",
    "        rank = float(predictionParts[3])\n",
    "        #print(predictionParts[5])\n",
    "        categories = json.loads(predictionParts[5])\n",
    "        priority = predictionParts[4]\n",
    "        \n",
    "        priorityNum = float(priority)\n",
    "        if (maxPrediction<priorityNum): \n",
    "            maxPrediction = priorityNum\n",
    "        if (minPrediction>priorityNum):\n",
    "            minPrediction = priorityNum\n",
    "        \n",
    "        tweetId2RunInfoCategories[tweetId] = categories\n",
    "        tweetId2RunPriorityScore[tweetId] = priority\n",
    "        \n",
    "        if not event2TweetIdRank.get(eventId):\n",
    "            event2TweetIdRank[eventId] = []\n",
    "        rankTuple = (tweetId,rank)\n",
    "        event2TweetIdRank.get(eventId).append(rankTuple)\n",
    "        \n",
    "        highImportCats = []\n",
    "        lowImportCats = []\n",
    "        for categoryId in categories:\n",
    "            cleanedCategories.append(categoryId)\n",
    "            if any(categoryId in s for s in highImportCategories):\n",
    "                highImportCats.append(categoryId)\n",
    "            else:\n",
    "                lowImportCats.append(categoryId)\n",
    "                \n",
    "        tweetId2RunHighImportInfoCategories[tweetId] = highImportCats\n",
    "        tweetId2RunLowImportInfoCategories[tweetId] = lowImportCats\n",
    "\n",
    "for eventId in event2TweetIdRank.keys():\n",
    "    tweetsSorted = sorted(event2TweetIdRank.get(eventId), key=lambda tup: tup[1])\n",
    "    event2TweetIdRank[eventId] = tweetsSorted\n",
    "    \n",
    "for i in range(len(index2TweetId)):\n",
    "    tweetId = index2TweetId[i]\n",
    "    if tweetId2RunPriorityScore.get(tweetId):\n",
    "        \n",
    "        priority = float(tweetId2RunPriorityScore.get(tweetId))\n",
    "                \n",
    "        # scale to between 0 and 1\n",
    "        if (maxPrediction-minPrediction)>0:\n",
    "            tweetId2RunPriorityScoreNorm[tweetId] = (priority-minPrediction)/(maxPrediction-minPrediction)\n",
    "        else:\n",
    "            tweetId2RunPriorityScoreNorm[tweetId] = 0.0\n",
    "    else:\n",
    "        tweetId2RunPriorityScoreNorm[tweetId] = 0.0\n",
    "    \n",
    "# --------------------------------------------------\n",
    "# Stage 6: Create ground truth vectors per category\n",
    "# --------------------------------------------------\n",
    "\n",
    "category2GroundTruth = {} # category -> tweet vector with binary 1 vs all ground truth category labels\n",
    "\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    categoryIdShort = categoryId.split(\"-\")[1]\n",
    "    categoryVector = []\n",
    "    for i in range(len(index2TweetId)):\n",
    "        tweetId = index2TweetId[i]\n",
    "        categories = tweetId2TRECInfoCategories.get(tweetId)\n",
    "        #pprint(categories)\n",
    "        if any(categoryIdShort in s for s in categories):\n",
    "            categoryVector.append(1)\n",
    "        else:\n",
    "            categoryVector.append(0)\n",
    "    category2GroundTruth[categoryId] = categoryVector\n",
    "            \n",
    "#pprint(category2GroundTruth)\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Stage 7: Create run vectors per category \n",
    "# --------------------------------------------------\n",
    "# Assumptions: If run misses a tweet, we assume id has\n",
    "#              no categories\n",
    "category2Predicted = {} # category -> tweet vector with binary 1 vs all predicted by system labels\n",
    "\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    categoryIdShort = categoryId.split(\"-\")[1]\n",
    "    categoryVector = []\n",
    "    for i in range(len(index2TweetId)):\n",
    "        tweetId = index2TweetId[i]\n",
    "        \n",
    "        if tweetId2RunInfoCategories.get(tweetId):\n",
    "            categories = tweetId2RunInfoCategories.get(tweetId)\n",
    "            if any(categoryIdShort in s for s in categories):\n",
    "                categoryVector.append(1)\n",
    "            else:\n",
    "                categoryVector.append(0)\n",
    "        else:\n",
    "            categoryVector.append(0)\n",
    "\n",
    "    category2Predicted[categoryId] = categoryVector\n",
    "\n",
    "#pprint(category2Predicted)\n",
    "\n",
    "# --------------------------------------------------\n",
    "# Stage 8: Make event category vectors \n",
    "# --------------------------------------------------\n",
    "\n",
    "event2groundtruth = {} # event -> category -> tweet vector with binary 1 vs all ground truth category labels\n",
    "for eventId in eventIdentifiers:\n",
    "    eventCategories = {}\n",
    "    for categoryId in informationTypes2Index.keys():\n",
    "        categoryIdShort = categoryId.split(\"-\")[1]\n",
    "        categoryVector = []\n",
    "        #print(eventId)\n",
    "        for tweetId in event2tweetIds.get(eventId):\n",
    "            categories = tweetId2TRECInfoCategories.get(tweetId)\n",
    "            if any(categoryIdShort in s for s in categories):\n",
    "                categoryVector.append(1)\n",
    "            else:\n",
    "                categoryVector.append(0)\n",
    "            \n",
    "        eventCategories[categoryId] = categoryVector\n",
    "    event2groundtruth[eventId] = eventCategories\n",
    "    \n",
    "\n",
    "event2prediction = {} # event -> category -> tweet vector with binary 1 vs all predicted by system labels\n",
    "for eventId in eventIdentifiers:\n",
    "    eventCategories = {}\n",
    "    for categoryId in informationTypes2Index.keys():\n",
    "        categoryIdShort = categoryId.split(\"-\")[1]\n",
    "        categoryVector = []\n",
    "        for tweetId in event2tweetIds.get(eventId):\n",
    "            #print(tweetId)\n",
    "            #print(categories)\n",
    "            categories = tweetId2RunInfoCategories.get(tweetId)\n",
    "            #print(categories)\n",
    "            \n",
    "            #if categories==None:\n",
    "            #    categories=['Request-GoodsServices']\n",
    "            \n",
    "            if any(categoryId in s for s in categories):\n",
    "                categoryVector.append(1)\n",
    "            else:\n",
    "                categoryVector.append(0)\n",
    "            \n",
    "        eventCategories[categoryId] = categoryVector\n",
    "    event2prediction[eventId] = eventCategories\n",
    "\n",
    "\n",
    "# -----------------------------------------------------------\n",
    "# Stage 9: Make priority classification vectors\n",
    "# -----------------------------------------------------------\n",
    "\n",
    "category2GroundTruthPriority = {} # category -> tweet vector with binary 1 vs all ground truth priority labels\n",
    "\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    categoryIdShort = categoryId.split(\"-\")[1]\n",
    "    priorityVector = []\n",
    "    for i in range(len(index2TweetId)):\n",
    "        tweetId = index2TweetId[i]\n",
    "        categories = tweetId2TRECInfoCategories.get(tweetId)\n",
    "        if any(categoryIdShort in s for s in categories):\n",
    "            priority = tweetId2TRECPriorityCategory.get(tweetId)\n",
    "            priorityAsNumber = priorityMapping[priority]\n",
    "            priorityVector.append(priorityAsNumber)\n",
    "    category2GroundTruthPriority[categoryId] = priorityVector\n",
    "\n",
    "category2PredictedPriority = {} # category -> tweet vector with binary 1 vs all predicted by system labels\n",
    "\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    categoryIdShort = categoryId.split(\"-\")[1]\n",
    "    categoryVector = []\n",
    "    for i in range(len(index2TweetId)):\n",
    "        tweetId = index2TweetId[i]\n",
    "        categories = tweetId2TRECInfoCategories.get(tweetId)\n",
    "        #print (categories)\n",
    "        if any(categoryIdShort in s for s in categories):\n",
    "            if tweetId2RunPriorityScore.get(tweetId):\n",
    "                priority = float(tweetId2RunPriorityScore.get(tweetId))\n",
    "                \n",
    "                # scale to between 0 and 1\n",
    "                if (maxPrediction-minPrediction)>0:\n",
    "                    normPriority = (priority-minPrediction)/(maxPrediction-minPrediction)\n",
    "                else:\n",
    "                    normPriority = 0.0;\n",
    "                # bound by min and max on ground truth\n",
    "                if (normPriority<priorityMapping[\"Low\"]): \n",
    "                    normPriority = priorityMapping[\"Low\"]\n",
    "                if (normPriority>priorityMapping[\"Critical\"]): \n",
    "                    normPriority = priorityMapping[\"Critical\"]\n",
    "                \n",
    "                categoryVector.append(normPriority)\n",
    "            else:\n",
    "                categoryVector.append(priorityMapping[\"Low\"]) # default to low priority\n",
    "\n",
    "    category2PredictedPriority[categoryId] = categoryVector\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# --------------------------------------------------\n",
    "# Disable Warnings (comment this out when debugging!)\n",
    "# --------------------------------------------------\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\") # ignore warnings about 0-score categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "High Importance Alert Worth: -0.7023054755043228\n",
      "Low Importance Alert Worth: 0.0\n",
      "Accumulated Alert Worth: -0.3511527377521614\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Mertic: Accumulated Alert Worth\n",
    "# Measures system effectiveness from the perspective\n",
    "# of end-user alerting of important information\n",
    "# --------------------------------------------------\n",
    "\n",
    "totalHighImportWorth = 0.0\n",
    "totalLowImportWorth = 0.0\n",
    "AccumulatedAlertWorth = 0.0\n",
    "\n",
    "\n",
    "for eventId in event2TweetIdRank.keys():\n",
    "    numConsecutiveFalseAlerts = 0\n",
    "    for tweetIdRank in event2TweetIdRank[eventId]:\n",
    "        \n",
    "        tweetId = tweetIdRank[0]\n",
    "        try:\n",
    "            trecPriority = tweetId2TRECPriorityCategory[tweetId]\n",
    "        except:\n",
    "            #print(tweetId)\n",
    "            continue # skip tweets not in the ground truth\n",
    "        runPriority = float(tweetId2RunPriorityScore[tweetId])\n",
    "        \n",
    "        trecHighImportCats = set(tweetId2TRECHighImportInfoCategories[tweetid])\n",
    "        trecLowImportCats = set(tweetId2TRECHighImportInfoCategories[tweetid])\n",
    "        runHighImportCats = set(tweetId2RunHighImportInfoCategories[tweetId])\n",
    "        runLowImportCats = set(tweetId2RunLowImportInfoCategories[tweetId])\n",
    "        \n",
    "        gamma = 0\n",
    "        if len(trecHighImportCats)>0:\n",
    "            gamma = var_lambda\n",
    "        \n",
    "        ActCScore = 0.0\n",
    "        if len(trecHighImportCats | runHighImportCats) > 0: ActCScore = gamma*(len(trecHighImportCats & runHighImportCats) / len(trecHighImportCats | runHighImportCats))\n",
    "        NActCScore = 0.0\n",
    "        if len(trecLowImportCats | runLowImportCats)>0: NActCScore = (1-gamma)*(len(trecLowImportCats & runLowImportCats) / len(trecLowImportCats | runLowImportCats))\n",
    "        \n",
    "        worth = 0.0\n",
    "        if trecPriority == \"High\" or trecPriority == \"Critical\":\n",
    "            # calculate highImportWorth\n",
    "            if runPriority >= alertPriorityThreshold:\n",
    "                worth = var_alpha + ((1 - var_alpha) * (ActCScore+NActCScore))\n",
    "                numConsecutiveFalseAlerts = 0\n",
    "            else:\n",
    "                worth = -1\n",
    "            \n",
    "            #print(eventId+\" \"+tweetId+\" (High) \"+str(worth))\n",
    "            \n",
    "            \n",
    "            totalHighImportWorth = totalHighImportWorth + worth\n",
    "        else:\n",
    "            if runPriority >= alertPriorityThreshold:\n",
    "                worth = max(-math.log10((numConsecutiveFalseAlerts/2)+1),-1)\n",
    "                numConsecutiveFalseAlerts = numConsecutiveFalseAlerts + 1\n",
    "            else:\n",
    "                worth = ActCScore+NActCScore\n",
    "            \n",
    "            #print(eventId+\" \"+tweetId+\" (Low) \"+str(worth)+\" (\"+str(numConsecutiveFalseAlerts)+\")\")\n",
    "            \n",
    "            totalLowImportWorth = totalLowImportWorth + worth\n",
    "            \n",
    "        AccumulatedAlertWorth = AccumulatedAlertWorth + worth\n",
    "        \n",
    "    \n",
    "totalHighImportWorth = totalHighImportWorth / countHighCriticalImport\n",
    "totalLowImportWorth = totalLowImportWorth / countLowMediumImport\n",
    "AccumulatedAlertWorth = totalHighImportWorth + totalLowImportWorth\n",
    "        \n",
    "AccumulatedAlertWorth = (AccumulatedAlertWorth/2)\n",
    "totalHighImportWorth = totalHighImportWorth\n",
    "totalLowImportWorth = totalLowImportWorth\n",
    "\n",
    "print(\"High Importance Alert Worth: \"+str(totalHighImportWorth))\n",
    "print(\"Low Importance Alert Worth: \"+str(totalLowImportWorth))\n",
    "print(\"Accumulated Alert Worth: \"+str(AccumulatedAlertWorth))\n",
    "       \n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"EVALUATON: Alertng Performance\"+\"\\n\")\n",
    "resultsFile.write(\"Overall performance\"+\"\\n\")\n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"> High Importance Alert Worth:\"+\"\\t\"+str(totalHighImportWorth)+\"\\n\")\n",
    "resultsFile.write(\"> Low Importance Alert Worth:\"+\"\\t\"+str(totalLowImportWorth)+\"\\n\")\n",
    "resultsFile.write(\"> Accumulated Alert Worth:\"+\"\\t\"+str(AccumulatedAlertWorth)+\"\\n\")\n",
    "resultsFile.write(\"\"+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Information Type Precision (positive class, multi-type, macro): 0.000110562685094\n",
      "Information Type Recall (positive class, multi-type, macro): 0.024347826087\n",
      "Information Type F1 (positive class, multi-type, macro): 0.000220125786164\n",
      "Information Type Accuracy (overall, multi-type, macro): 0.876342677495\n",
      "High Importance Information Type Precision (positive class, multi-type, macro): 0.000460677854557\n",
      "High Importance Information Type Recall (positive class, multi-type, macro): 0.101449275362\n",
      "High Importance Information Type F1 (positive class, multi-type, macro): 0.000917190775681\n",
      "High Importance Information Type Accuracy (overall, multi-type, macro): 0.870338835036\n",
      "Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "Low Importance Information Type Accuracy (overall, multi-type, macro): 0.878238627746\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Type Categorization\n",
    "# Overall performance\n",
    "# --------------------------------------------------\n",
    "# Average performance over information types\n",
    "# Macro averaged (information types have equal weight)\n",
    "# Does not average across events (larger events have more impact)\n",
    "# Positive class is the target class\n",
    "# Precision, recall and F1 only consider the positive class\n",
    "# Accuracy is an overall metric\n",
    "# We report performance for all categories, high importance categories and low importance categories\n",
    "\n",
    "from sklearn.metrics import precision_score\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import f1_score\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "avgPrecision = 0.0\n",
    "avgRecall = 0.0\n",
    "avgF1 = 0.0\n",
    "avgAccuracy = 0.0\n",
    "\n",
    "avgPrecisionHigh = 0.0\n",
    "avgRecallHigh = 0.0\n",
    "avgF1High = 0.0\n",
    "avgAccuracyHigh = 0.0\n",
    "\n",
    "avgPrecisionLow = 0.0\n",
    "avgRecallLow = 0.0\n",
    "avgF1Low = 0.0\n",
    "avgAccuracyLow = 0.0\n",
    "\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    categoryPrecision = precision_score(category2GroundTruth[categoryId], category2Predicted[categoryId], average='binary')\n",
    "    categoryRecall = recall_score(category2GroundTruth[categoryId], category2Predicted[categoryId], average='binary')\n",
    "    categoryF1 = f1_score(category2GroundTruth[categoryId], category2Predicted[categoryId], average='binary')\n",
    "    categoryAccuracy = accuracy_score(category2GroundTruth[categoryId], category2Predicted[categoryId])\n",
    "    \n",
    "    avgPrecision = avgPrecision + categoryPrecision\n",
    "    avgRecall = avgRecall + categoryRecall\n",
    "    avgF1 = avgF1 + categoryF1\n",
    "    avgAccuracy = avgAccuracy + categoryAccuracy\n",
    "    \n",
    "    if any(categoryId in s for s in highImportCategories):\n",
    "        avgPrecisionHigh = avgPrecisionHigh + categoryPrecision\n",
    "        avgRecallHigh = avgRecallHigh + categoryRecall\n",
    "        avgF1High = avgF1High + categoryF1\n",
    "        avgAccuracyHigh = avgAccuracyHigh + categoryAccuracy\n",
    "    else:\n",
    "        avgPrecisionLow = avgPrecisionLow + categoryPrecision\n",
    "        avgRecallLow = avgRecallLow + categoryRecall\n",
    "        avgF1Low = avgF1Low + categoryF1\n",
    "        avgAccuracyLow = avgAccuracyLow + categoryAccuracy\n",
    "\n",
    "numInformationTypes = len(informationTypes2Index)\n",
    "numHighInformationTypes = len(highImportCategories)\n",
    "numLowInformationTypes = numInformationTypes - numHighInformationTypes\n",
    "        \n",
    "print(\"Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecision/numInformationTypes))\n",
    "print(\"Information Type Recall (positive class, multi-type, macro): \"+str(avgRecall/numInformationTypes))\n",
    "print(\"Information Type F1 (positive class, multi-type, macro): \"+str(avgF1/numInformationTypes))\n",
    "print(\"Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracy/numInformationTypes))\n",
    "\n",
    "print(\"High Importance Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecisionHigh/numHighInformationTypes))\n",
    "print(\"High Importance Information Type Recall (positive class, multi-type, macro): \"+str(avgRecallHigh/numHighInformationTypes))\n",
    "print(\"High Importance Information Type F1 (positive class, multi-type, macro): \"+str(avgF1High/numHighInformationTypes))\n",
    "print(\"High Importance Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracyHigh/numHighInformationTypes))\n",
    "\n",
    "print(\"Low Importance Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecisionLow/numLowInformationTypes))\n",
    "print(\"Low Importance Information Type Recall (positive class, multi-type, macro): \"+str(avgRecallLow/numLowInformationTypes))\n",
    "print(\"Low Importance Information Type F1 (positive class, multi-type, macro): \"+str(avgF1Low/numLowInformationTypes))\n",
    "print(\"Low Importance Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracyLow/numLowInformationTypes))\n",
    "\n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"EVALUATON: Information Type Categorization\"+\"\\n\")\n",
    "resultsFile.write(\"Overall performance\"+\"\\n\")\n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"> Information Type Precision (positive class, multi-type, macro):\"+\"\\t\"+str(avgPrecision/len(informationTypes2Index))+\"\\n\")\n",
    "resultsFile.write(\"> Information Type Recall (positive class, multi-type, macro):\"+\"\\t\"+str(avgRecall/len(informationTypes2Index))+\"\\n\")\n",
    "resultsFile.write(\"> Information Type F1 (positive class, multi-type, macro):\"+\"\\t\"+str(avgF1/len(informationTypes2Index))+\"\\n\")\n",
    "resultsFile.write(\"> Information Type Accuracy (overall, multi-type, macro):\"+\"\\t\"+str(avgAccuracy/len(informationTypes2Index))+\"\\n\")\n",
    "resultsFile.write(\"> High Importance Information Type Precision (positive class, multi-type, macro):\"+\"\\t\"+str(avgPrecisionHigh/numHighInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> High Importance Information Type Recall (positive class, multi-type, macro):\"+\"\\t\"+str(avgRecallHigh/numHighInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> High Importance Information Type F1 (positive class, multi-type, macro):\"+\"\\t\"+str(avgF1High/numHighInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> High Importance Information Type Accuracy (overall, multi-type, macro):\"+\"\\t\"+str(avgAccuracyHigh/numHighInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> Low Importance Information Type Precision (positive class, multi-type, macro):\"+\"\\t\"+str(avgPrecisionLow/numLowInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> Low Importance Information Type Recall (positive class, multi-type, macro):\"+\"\\t\"+str(avgRecallLow/numLowInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> Low Importance Information Type F1 (positive class, multi-type, macro):\"+\"\\t\"+str(avgF1Low/numLowInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> Low Importance Information Type Accuracy (overall, multi-type, macro):\"+\"\\t\"+str(avgAccuracyLow/numLowInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"\"+\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Report-ThirdPartyObservation\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "               Other Classes       0.96      1.00      0.98      7318\n",
      "Report-ThirdPartyObservation       0.00      0.00      0.00       316\n",
      "\n",
      "                 avg / total       0.92      0.96      0.94      7634\n",
      "\n",
      "Report-Official\n",
      "                 precision    recall  f1-score   support\n",
      "\n",
      "  Other Classes       0.97      1.00      0.98      7405\n",
      "Report-Official       0.00      0.00      0.00       229\n",
      "\n",
      "    avg / total       0.94      0.97      0.96      7634\n",
      "\n",
      "CallToAction-Volunteer\n",
      "                        precision    recall  f1-score   support\n",
      "\n",
      "         Other Classes       1.00      1.00      1.00      7611\n",
      "CallToAction-Volunteer       0.00      0.00      0.00        23\n",
      "\n",
      "           avg / total       0.99      1.00      1.00      7634\n",
      "\n",
      "Report-CleanUp\n",
      "                precision    recall  f1-score   support\n",
      "\n",
      " Other Classes       0.99      1.00      1.00      7560\n",
      "Report-CleanUp       0.00      0.00      0.00        74\n",
      "\n",
      "   avg / total       0.98      0.99      0.99      7634\n",
      "\n",
      "Report-OriginalEvent\n",
      "                      precision    recall  f1-score   support\n",
      "\n",
      "       Other Classes       0.88      1.00      0.94      6751\n",
      "Report-OriginalEvent       0.00      0.00      0.00       883\n",
      "\n",
      "         avg / total       0.78      0.88      0.83      7634\n",
      "\n",
      "Request-GoodsServices\n",
      "                       precision    recall  f1-score   support\n",
      "\n",
      "        Other Classes       1.00      0.34      0.50      7611\n",
      "Request-GoodsServices       0.00      0.61      0.01        23\n",
      "\n",
      "          avg / total       0.99      0.34      0.50      7634\n",
      "\n",
      "Report-Hashtags\n",
      "                 precision    recall  f1-score   support\n",
      "\n",
      "  Other Classes       0.50      1.00      0.66      3790\n",
      "Report-Hashtags       0.00      0.00      0.00      3844\n",
      "\n",
      "    avg / total       0.25      0.50      0.33      7634\n",
      "\n",
      "Report-Location\n",
      "                 precision    recall  f1-score   support\n",
      "\n",
      "  Other Classes       0.79      1.00      0.88      6052\n",
      "Report-Location       0.00      0.00      0.00      1582\n",
      "\n",
      "    avg / total       0.63      0.79      0.70      7634\n",
      "\n",
      "Report-FirstPartyObservation\n",
      "                              precision    recall  f1-score   support\n",
      "\n",
      "               Other Classes       0.94      1.00      0.97      7151\n",
      "Report-FirstPartyObservation       0.00      0.00      0.00       483\n",
      "\n",
      "                 avg / total       0.88      0.94      0.91      7634\n",
      "\n",
      "Report-ServiceAvailable\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "          Other Classes       0.98      1.00      0.99      7484\n",
      "Report-ServiceAvailable       0.00      0.00      0.00       150\n",
      "\n",
      "            avg / total       0.96      0.98      0.97      7634\n",
      "\n",
      "Request-SearchAndRescue\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "          Other Classes       1.00      1.00      1.00      7624\n",
      "Request-SearchAndRescue       0.00      0.00      0.00        10\n",
      "\n",
      "            avg / total       1.00      1.00      1.00      7634\n",
      "\n",
      "Report-Weather\n",
      "                precision    recall  f1-score   support\n",
      "\n",
      " Other Classes       0.96      1.00      0.98      7335\n",
      "Report-Weather       0.00      0.00      0.00       299\n",
      "\n",
      "   avg / total       0.92      0.96      0.94      7634\n",
      "\n",
      "Report-News\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "Other Classes       0.81      1.00      0.89      6158\n",
      "  Report-News       0.00      0.00      0.00      1476\n",
      "\n",
      "  avg / total       0.65      0.81      0.72      7634\n",
      "\n",
      "CallToAction-MovePeople\n",
      "                         precision    recall  f1-score   support\n",
      "\n",
      "          Other Classes       0.99      1.00      0.99      7526\n",
      "CallToAction-MovePeople       0.00      0.00      0.00       108\n",
      "\n",
      "            avg / total       0.97      0.99      0.98      7634\n",
      "\n",
      "Other-Irrelevant\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "   Other Classes       0.66      1.00      0.79      5014\n",
      "Other-Irrelevant       0.00      0.00      0.00      2620\n",
      "\n",
      "     avg / total       0.43      0.66      0.52      7634\n",
      "\n",
      "Other-Discussion\n",
      "                  precision    recall  f1-score   support\n",
      "\n",
      "   Other Classes       0.93      1.00      0.96      7065\n",
      "Other-Discussion       0.00      0.00      0.00       569\n",
      "\n",
      "     avg / total       0.86      0.93      0.89      7634\n",
      "\n",
      "Other-Advice\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "Other Classes       0.94      1.00      0.97      7184\n",
      " Other-Advice       0.00      0.00      0.00       450\n",
      "\n",
      "  avg / total       0.89      0.94      0.91      7634\n",
      "\n",
      "Report-MultimediaShare\n",
      "                        precision    recall  f1-score   support\n",
      "\n",
      "         Other Classes       0.79      1.00      0.88      6035\n",
      "Report-MultimediaShare       0.00      0.00      0.00      1599\n",
      "\n",
      "           avg / total       0.62      0.79      0.70      7634\n",
      "\n",
      "CallToAction-Donations\n",
      "                        precision    recall  f1-score   support\n",
      "\n",
      "         Other Classes       0.99      1.00      0.99      7557\n",
      "CallToAction-Donations       0.00      0.00      0.00        77\n",
      "\n",
      "           avg / total       0.98      0.99      0.98      7634\n",
      "\n",
      "Report-Factoid\n",
      "                precision    recall  f1-score   support\n",
      "\n",
      " Other Classes       0.87      1.00      0.93      6625\n",
      "Report-Factoid       0.00      0.00      0.00      1009\n",
      "\n",
      "   avg / total       0.75      0.87      0.81      7634\n",
      "\n",
      "Report-NewSubEvent\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "     Other Classes       0.98      1.00      0.99      7492\n",
      "Report-NewSubEvent       0.00      0.00      0.00       142\n",
      "\n",
      "       avg / total       0.96      0.98      0.97      7634\n",
      "\n",
      "Other-ContextualInformation\n",
      "                             precision    recall  f1-score   support\n",
      "\n",
      "              Other Classes       0.96      1.00      0.98      7365\n",
      "Other-ContextualInformation       0.00      0.00      0.00       269\n",
      "\n",
      "                avg / total       0.93      0.96      0.95      7634\n",
      "\n",
      "Report-EmergingThreats\n",
      "                        precision    recall  f1-score   support\n",
      "\n",
      "         Other Classes       0.94      1.00      0.97      7168\n",
      "Report-EmergingThreats       0.00      0.00      0.00       466\n",
      "\n",
      "           avg / total       0.88      0.94      0.91      7634\n",
      "\n",
      "Request-InformationWanted\n",
      "                           precision    recall  f1-score   support\n",
      "\n",
      "            Other Classes       0.99      1.00      1.00      7564\n",
      "Request-InformationWanted       0.00      0.00      0.00        70\n",
      "\n",
      "              avg / total       0.98      0.99      0.99      7634\n",
      "\n",
      "Other-Sentiment\n",
      "                 precision    recall  f1-score   support\n",
      "\n",
      "  Other Classes       0.77      1.00      0.87      5845\n",
      "Other-Sentiment       0.00      0.00      0.00      1789\n",
      "\n",
      "    avg / total       0.59      0.77      0.66      7634\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Type Categorization\n",
    "# Per Information Type Performance\n",
    "# --------------------------------------------------\n",
    "# Per Category Classification Performance with confusion matrices\n",
    "# Performance on the target class is what we care about here, \n",
    "# primaraly with respect to recall, as we want the user to \n",
    "# see all of the information for a given category. A small\n",
    "# amount of noise being added to the feed is an acceptable\n",
    "# cost for good recall.\n",
    "#\n",
    "# Does not average across events (larger events have more impact)\n",
    "\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "perTopicFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "perTopicFile.write(\"EVALUATON: Information Type Categorization (Multi-type)\"+\"\\n\")\n",
    "perTopicFile.write(\"Per Information Type Performance\"+\"\\n\")\n",
    "perTopicFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    target_names = ['Other Classes', categoryId]\n",
    "    print(categoryId)\n",
    "    print(classification_report(category2GroundTruth[categoryId], category2Predicted[categoryId], target_names=target_names))\n",
    "\n",
    "\n",
    "    perTopicFile.write(categoryId+\"\\n\")\n",
    "    perTopicFile.write(classification_report(category2GroundTruth[categoryId], category2Predicted[categoryId], target_names=target_names)+\"\\n\")\n",
    "    perTopicFile.write(\"\"+\"\\n\")\n",
    "\n",
    "perTopicFile.write(\"\"+\"\\n\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Type Categorization\n",
    "# Per Information Type F1 Graph\n",
    "# --------------------------------------------------\n",
    "# Per Category Classification Performance\n",
    "# F1 scores for each information type, graphed\n",
    "# Does not average across events (larger events have more impact)\n",
    "\n",
    "\n",
    "\n",
    "N = len(informationTypes2Index)\n",
    "ind = np.arange(N)\n",
    "\n",
    "scoresPerCategoryF1 = []\n",
    "categoryLabels = []\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    scoresPerCategoryF1.append(f1_score(category2GroundTruth[categoryId], category2Predicted[categoryId], average='binary'))\n",
    "    categoryLabels.append(categoryId)\n",
    "    \n",
    "width = 0.90       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "p1 = plt.bar(ind, scoresPerCategoryF1, width)\n",
    "\n",
    "plt.ylabel('F1 Scores')\n",
    "plt.title('F1 Scores by Information Type')\n",
    "plt.xticks(ind, categoryLabels, rotation='vertical')\n",
    "plt.yticks(np.arange(0, 1, 0.1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "floodChoco2019\n",
      "  Information Type Precision (positive class, multi-type, macro): 0.000103626943005\n",
      "  Information Type Recall (positive class, multi-type, macro): 0.04\n",
      "  Information Type F1 (positive class, multi-type, macro): 0.000206718346253\n",
      "  Information Type Accuracy (overall, multi-type, macro): 0.913007712082\n",
      "  High Importance Information Type Precision (positive class, multi-type, macro): 0.000431778929188\n",
      "  High Importance Information Type Recall (positive class, multi-type, macro): 0.166666666667\n",
      "  High Importance Information Type F1 (positive class, multi-type, macro): 0.000861326442722\n",
      "  High Importance Information Type Accuracy (overall, multi-type, macro): 0.830762639246\n",
      "  Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Accuracy (overall, multi-type, macro): 0.938979840346\n",
      "\n",
      "earthquakeCalifornia2014\n",
      "  Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Information Type Accuracy (overall, multi-type, macro): 0.84125984252\n",
      "  High Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  High Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  High Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  High Importance Information Type Accuracy (overall, multi-type, macro): 0.816272965879\n",
      "  Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Accuracy (overall, multi-type, macro): 0.849150435143\n",
      "\n",
      "earthquakeBohol2013\n",
      "  Information Type Precision (positive class, multi-type, macro): 0.000129240710824\n",
      "  Information Type Recall (positive class, multi-type, macro): 0.04\n",
      "  Information Type F1 (positive class, multi-type, macro): 0.000257648953301\n",
      "  Information Type Accuracy (overall, multi-type, macro): 0.860743134087\n",
      "  High Importance Information Type Precision (positive class, multi-type, macro): 0.000538502961766\n",
      "  High Importance Information Type Recall (positive class, multi-type, macro): 0.166666666667\n",
      "  High Importance Information Type F1 (positive class, multi-type, macro): 0.00107353730542\n",
      "  High Importance Information Type Accuracy (overall, multi-type, macro): 0.830102315563\n",
      "  Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Accuracy (overall, multi-type, macro): 0.870419182042\n",
      "\n",
      "hurricaneFlorence2018\n",
      "  Information Type Precision (positive class, multi-type, macro): 0.000256102440976\n",
      "  Information Type Recall (positive class, multi-type, macro): 0.04\n",
      "  Information Type F1 (positive class, multi-type, macro): 0.000508946322068\n",
      "  Information Type Accuracy (overall, multi-type, macro): 0.862713085234\n",
      "  High Importance Information Type Precision (positive class, multi-type, macro): 0.00106709350407\n",
      "  High Importance Information Type Recall (positive class, multi-type, macro): 0.166666666667\n",
      "  High Importance Information Type F1 (positive class, multi-type, macro): 0.00212060967528\n",
      "  High Importance Information Type Accuracy (overall, multi-type, macro): 0.81179138322\n",
      "  Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Accuracy (overall, multi-type, macro): 0.878793622712\n",
      "\n",
      "shootingDallas2017\n",
      "  Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Information Type Accuracy (overall, multi-type, macro): 0.88836\n",
      "  High Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  High Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  High Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  High Importance Information Type Accuracy (overall, multi-type, macro): 0.8315\n",
      "  Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Accuracy (overall, multi-type, macro): 0.906315789474\n",
      "\n",
      "fireYMM2016\n",
      "  Information Type Precision (positive class, multi-type, macro): 8e-05\n",
      "  Information Type Recall (positive class, multi-type, macro): 0.04\n",
      "  Information Type F1 (positive class, multi-type, macro): 0.000159680638723\n",
      "  Information Type Accuracy (overall, multi-type, macro): 0.83032\n",
      "  High Importance Information Type Precision (positive class, multi-type, macro): 0.000333333333333\n",
      "  High Importance Information Type Recall (positive class, multi-type, macro): 0.166666666667\n",
      "  High Importance Information Type F1 (positive class, multi-type, macro): 0.000665335994677\n",
      "  High Importance Information Type Accuracy (overall, multi-type, macro): 0.793583333333\n",
      "  Low Importance Information Type Precision (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Recall (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type F1 (positive class, multi-type, macro): 0.0\n",
      "  Low Importance Information Type Accuracy (overall, multi-type, macro): 0.841921052632\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Type Categorization\n",
    "# Per Event Performance\n",
    "# --------------------------------------------------\n",
    "# Categorization performance for each event\n",
    "# Precision, recall and F1 only consider the positive class\n",
    "# Accuracy is an overall metric\n",
    "# We report performance for all categories, high importance categories and low importance categories\n",
    "# Macro average (categories have equal weight)\n",
    "\n",
    "perEventFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "perEventFile.write(\"EVALUATON: Information Type Categorization (Multi-type)\"+\"\\n\")\n",
    "perEventFile.write(\"Per Event Performance\"+\"\\n\")\n",
    "perEventFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "\n",
    "for eventId in eventIdentifiers:\n",
    "    avgPrecision = 0.0\n",
    "    avgRecall = 0.0\n",
    "    avgF1 = 0.0\n",
    "    avgAccuracy = 0.0\n",
    "    \n",
    "    avgPrecisionHigh = 0.0\n",
    "    avgRecallHigh = 0.0\n",
    "    avgF1High = 0.0\n",
    "    avgAccuracyHigh = 0.0\n",
    "\n",
    "    avgPrecisionLow = 0.0\n",
    "    avgRecallLow = 0.0\n",
    "    avgF1Low = 0.0\n",
    "    avgAccuracyLow = 0.0\n",
    "\n",
    "    for categoryId in informationTypes2Index.keys():\n",
    "        \n",
    "        categoryPrecision = precision_score(event2groundtruth[eventId].get(categoryId), event2prediction[eventId].get(categoryId), average='binary')\n",
    "        categoryRecall = recall_score(event2groundtruth[eventId].get(categoryId), event2prediction[eventId].get(categoryId), average='binary')\n",
    "        categoryF1 = f1_score(event2groundtruth[eventId].get(categoryId), event2prediction[eventId].get(categoryId), average='binary')\n",
    "        categoryAccuracy = accuracy_score(event2groundtruth[eventId].get(categoryId), event2prediction[eventId].get(categoryId))\n",
    "        \n",
    "        avgPrecision = avgPrecision + categoryPrecision\n",
    "        avgRecall = avgRecall + categoryRecall\n",
    "        avgF1 = avgF1 + categoryF1\n",
    "        avgAccuracy = avgAccuracy + categoryAccuracy\n",
    "        \n",
    "        if any(categoryId in s for s in highImportCategories):\n",
    "            avgPrecisionHigh = avgPrecisionHigh + categoryPrecision\n",
    "            avgRecallHigh = avgRecallHigh + categoryRecall\n",
    "            avgF1High = avgF1High + categoryF1\n",
    "            avgAccuracyHigh = avgAccuracyHigh + categoryAccuracy\n",
    "        else:\n",
    "            avgPrecisionLow = avgPrecisionLow + categoryPrecision\n",
    "            avgRecallLow = avgRecallLow + categoryRecall\n",
    "            avgF1Low = avgF1Low + categoryF1\n",
    "            avgAccuracyLow = avgAccuracyLow + categoryAccuracy\n",
    "        \n",
    "    print(eventId)\n",
    "    print(\"  Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecision/len(informationTypes2Index)))\n",
    "    print(\"  Information Type Recall (positive class, multi-type, macro): \"+str(avgRecall/len(informationTypes2Index)))\n",
    "    print(\"  Information Type F1 (positive class, multi-type, macro): \"+str(avgF1/len(informationTypes2Index)))\n",
    "    print(\"  Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracy/len(informationTypes2Index)))\n",
    "    print(\"  High Importance Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecisionHigh/numHighInformationTypes))\n",
    "    print(\"  High Importance Information Type Recall (positive class, multi-type, macro): \"+str(avgRecallHigh/numHighInformationTypes))\n",
    "    print(\"  High Importance Information Type F1 (positive class, multi-type, macro): \"+str(avgF1High/numHighInformationTypes))\n",
    "    print(\"  High Importance Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracyHigh/numHighInformationTypes))\n",
    "    print(\"  Low Importance Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecisionLow/numLowInformationTypes))\n",
    "    print(\"  Low Importance Information Type Recall (positive class, multi-type, macro): \"+str(avgRecallLow/numLowInformationTypes))\n",
    "    print(\"  Low Importance Information Type F1 (positive class, multi-type, macro): \"+str(avgF1Low/numLowInformationTypes))\n",
    "    print(\"  Low Importance Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracyLow/numLowInformationTypes))\n",
    "    print(\"\")\n",
    "    \n",
    "    perEventFile.write(eventId+\"\\n\")\n",
    "    perEventFile.write(\"  Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecision/len(informationTypes2Index))+\"\\n\")\n",
    "    perEventFile.write(\"  Information Type Recall (positive class, multi-type, macro): \"+str(avgRecall/len(informationTypes2Index))+\"\\n\")\n",
    "    perEventFile.write(\"  Information Type F1 (positive class, multi-type, macro): \"+str(avgF1/len(informationTypes2Index))+\"\\n\")\n",
    "    perEventFile.write(\"  Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracy/len(informationTypes2Index))+\"\\n\")\n",
    "    perEventFile.write(\"  High Importance Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecisionHigh/numHighInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  High Importance Information Type Recall (positive class, multi-type, macro): \"+str(avgRecallHigh/numHighInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  High Importance Information Type F1 (positive class, multi-type, macro): \"+str(avgF1High/numHighInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  High Importance Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracyHigh/numHighInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  Low Importance Information Type Precision (positive class, multi-type, macro): \"+str(avgPrecisionLow/numLowInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  Low Importance Information Type Recall (positive class, multi-type, macro): \"+str(avgRecallLow/numLowInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  Low Importance Information Type F1 (positive class, multi-type, macro): \"+str(avgF1Low/numLowInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"  Low Importance Information Type Accuracy (overall, multi-type, macro): \"+str(avgAccuracyLow/numLowInformationTypes)+\"\\n\")\n",
    "    perEventFile.write(\"\\n\")\n",
    "    \n",
    "perEventFile.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Type Categorization\n",
    "# Per Event F1 Graph\n",
    "# --------------------------------------------------\n",
    "# Multi-type (1 vs All): Tweets have multiple information types, aim: predict all of them\n",
    "# Macro average (categories have equal weight)\n",
    "\n",
    "N = len(eventIdentifiers)\n",
    "ind = np.arange(N)\n",
    "\n",
    "scoresPerEventF1 = []\n",
    "for eventId in eventIdentifiers:\n",
    "    avgF1 = 0.0\n",
    "\n",
    "    for categoryId in informationTypes2Index.keys():\n",
    "        avgF1 = avgF1 + f1_score(event2groundtruth[eventId].get(categoryId), event2prediction[eventId].get(categoryId), average='binary')\n",
    "        \n",
    "    scoresPerEventF1.append(avgF1/len(informationTypes2Index))\n",
    "    \n",
    "width = 0.90       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "p1 = plt.bar(ind, scoresPerEventF1, width)\n",
    "\n",
    "plt.ylabel('F1 Scores')\n",
    "plt.title('F1 Category Scores by Event')\n",
    "plt.xticks(ind, eventIdentifiers, rotation='vertical')\n",
    "plt.yticks(np.arange(0, 1, 0.1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Priority Estimation Error (mean squared error, macro): 0.078461885739\n",
      "Priority Estimation Error High Importance (mean squared error, macro): 0.190765443044\n",
      "Priority Estimation Error Low Importance (mean squared error, macro): 0.0429976044849\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Priority Level\n",
    "# Overall Performance\n",
    "# --------------------------------------------------\n",
    "# How divergent is the system from the human priority labels?\n",
    "# Converts the human labels into numerical values, see priorityMapping\n",
    "# Average error over information types, lower is better\n",
    "# Macro average (categories have equal weight)\n",
    "\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "avgMSE = 0.0;\n",
    "avgMSEHigh = 0.0;\n",
    "avgMSELow = 0.0;\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    groundTruthPriorities = category2GroundTruthPriority[categoryId]\n",
    "    predictedPriorities = category2PredictedPriority[categoryId]\n",
    "    \n",
    "    error = mean_squared_error(groundTruthPriorities, predictedPriorities)\n",
    "    avgMSE = avgMSE + error;\n",
    "    \n",
    "    if any(categoryId in s for s in highImportCategories):\n",
    "        avgMSEHigh = avgMSEHigh + error\n",
    "    else:\n",
    "        avgMSELow = avgMSELow + error\n",
    "    \n",
    "    \n",
    "    \n",
    "print(\"Priority Estimation Error (mean squared error, macro): \"+str(avgMSE/len(informationTypes2Index)))\n",
    "print(\"Priority Estimation Error High Importance (mean squared error, macro): \"+str(avgMSEHigh/numHighInformationTypes))\n",
    "print(\"Priority Estimation Error Low Importance (mean squared error, macro): \"+str(avgMSELow/numLowInformationTypes))\n",
    "    \n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"EVALUATON: Information Priority Level\"+\"\\n\")\n",
    "resultsFile.write(\"Overall Performance\"+\"\\n\")\n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"> Priority Estimation Error (mean squared error, macro): \"+str(avgMSE/len(informationTypes2Index))+\"\\n\")\n",
    "resultsFile.write(\"> Priority Estimation Error High Importance (mean squared error, macro): \"+str(avgMSEHigh/numHighInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"> Priority Estimation Error Low Importance (mean squared error, macro): \"+str(avgMSELow/numLowInformationTypes)+\"\\n\")\n",
    "resultsFile.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# --------------------------------------------------\n",
    "# TREC-IS 2019-A\n",
    "# Information Priority Level\n",
    "# Per Information Type Performance\n",
    "# --------------------------------------------------\n",
    "# Error per information type, lower is better\n",
    "\n",
    "N = len(informationTypes2Index)\n",
    "ind = np.arange(N)\n",
    "\n",
    "mseValues = []\n",
    "categoryLabels = []\n",
    "for categoryId in informationTypes2Index.keys():\n",
    "    groundTruthPriorities = category2GroundTruthPriority[categoryId]\n",
    "    predictedPriorities = category2PredictedPriority[categoryId]\n",
    "    error = mean_squared_error(groundTruthPriorities, predictedPriorities)\n",
    "    categoryLabels.append(categoryId)\n",
    "    mseValues.append(error);\n",
    "    \n",
    "width = 0.90       # the width of the bars: can also be len(x) sequence\n",
    "\n",
    "p1 = plt.bar(ind, mseValues, width)\n",
    "\n",
    "plt.ylabel('MSE Scores (lower is better)')\n",
    "plt.title('Mean Squared Error on Predicted Priority Levels Per Type')\n",
    "plt.xticks(ind, categoryLabels, rotation='vertical')\n",
    "plt.yticks(np.arange(0, 1, 0.1))\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Run & AW-H & AW-A & CF1-H & CF1-A & CAcc & PErr-H & PErr-A \\\\\n",
      "myrun & -0.7023 & -0.3512 & 0.0007 & 0.0002 & 0.8303 & 0.1908 & 0.0785 \\\\\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "73"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Print the evaluation table row in latex\n",
    "print(\"Run & AW-H & AW-A & CF1-H & CF1-A & CAcc & PErr-H & PErr-A \\\\\\\\\")\n",
    "\n",
    "resultLine = (str.format('{0:.4f}', totalHighImportWorth)+\n",
    "     \" & \"+\n",
    "     str.format('{0:.4f}', AccumulatedAlertWorth)+\n",
    "     \" & \"+\n",
    "     str.format('{0:.4f}',avgF1High/numHighInformationTypes)+\n",
    "     \" & \"+\n",
    "     str.format('{0:.4f}',avgF1/numInformationTypes)+\n",
    "     \" & \"+\n",
    "     str.format('{0:.4f}',avgAccuracy/numInformationTypes)+\n",
    "     \" & \"+\n",
    "     str.format('{0:.4f}',avgMSEHigh/numHighInformationTypes)+\n",
    "     \" & \"+\n",
    "     str.format('{0:.4f}',avgMSE/len(informationTypes2Index))+\n",
    "     \" \\\\\\\\\")\n",
    "\n",
    "print(runName+\" & \"+resultLine)\n",
    "\n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(\"LATEX\"+\"\\n\")\n",
    "resultsFile.write(\"--------------------------------------------------\"+\"\\n\")\n",
    "resultsFile.write(runName+\" & \"+resultLine)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Done\n",
    "resultsFile.close() \n",
    "perTopicFile.close()\n",
    "perEventFile.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}