Full Fact Claim Matching.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "import json\n",
    "import pandas\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import sklearn.neighbors\n",
    "from sklearn import cluster, covariance, manifold\n",
    "import lxml\n",
    "import lxml.objectify\n",
    "from lxml import etree\n",
    "import nltk\n",
    "import nltk.corpus\n",
    "from nltk.corpus import stopwords\n",
    "STOPS = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = gensim.models.KeyedVectors.load_word2vec_format('data/GoogleNews-vectors-negative300.bin', binary=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "claims_file = 'data/claim_conclusion.json'\n",
    "with open(claims_file) as fh:\n",
    "    claims_df = pandas.read_json(fh)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>claim</th>\n",
       "      <th>conclusion</th>\n",
       "      <th>url</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>Trade union sign up for those aged between 25 ...</td>\n",
       "      <td>This is not quite correct. 13% of the UK popul...</td>\n",
       "      <td>economy/union-membership-among-young-people</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670</th>\n",
       "      <td>£1 in every £10 that goes to the NHS is to pay...</td>\n",
       "      <td>This isn’t correct according to the latest cos...</td>\n",
       "      <td>health/what-nhs-paying-private-finance-initiat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>The Treasury loses £40 billion each year due t...</td>\n",
       "      <td>That’s a long way off from HMRC’s estimates, ...</td>\n",
       "      <td>economy/does-treasury-lose-40-billion-each-yea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672</th>\n",
       "      <td>A man who assaulted his wife was spared jail b...</td>\n",
       "      <td>We don’t know for sure, but it seems unlikely ...</td>\n",
       "      <td>law/domestic-violence-vulnerable-mustafa-bashir</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>Putting a doctor through medical school “costs...</td>\n",
       "      <td>That’s wrong. About £64,300 of that comes in s...</td>\n",
       "      <td>health/cost-training-doctor</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 claim  \\\n",
       "669  Trade union sign up for those aged between 25 ...   \n",
       "670  £1 in every £10 that goes to the NHS is to pay...   \n",
       "671  The Treasury loses £40 billion each year due t...   \n",
       "672  A man who assaulted his wife was spared jail b...   \n",
       "673  Putting a doctor through medical school “costs...   \n",
       "\n",
       "                                            conclusion  \\\n",
       "669  This is not quite correct. 13% of the UK popul...   \n",
       "670  This isn’t correct according to the latest cos...   \n",
       "671   That’s a long way off from HMRC’s estimates, ...   \n",
       "672  We don’t know for sure, but it seems unlikely ...   \n",
       "673  That’s wrong. About £64,300 of that comes in s...   \n",
       "\n",
       "                                                   url  \n",
       "669        economy/union-membership-among-young-people  \n",
       "670  health/what-nhs-paying-private-finance-initiat...  \n",
       "671  economy/does-treasury-lose-40-billion-each-yea...  \n",
       "672    law/domestic-violence-vulnerable-mustafa-bashir  \n",
       "673                        health/cost-training-doctor  "
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "claims_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def claim2vec(text):\n",
    "    word_vecs = []\n",
    "    for word in text.split():\n",
    "        if word not in STOPS and word in model:\n",
    "            word_vecs.append(model[word])\n",
    "    \n",
    "    if word_vecs:\n",
    "        result = np.mean(word_vecs, axis=0)\n",
    "        assert len(result) == 300\n",
    "        return result\n",
    "    else:\n",
    "        return np.zeros(300, dtype=np.float32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "claims_df['vec'] = claims_df['claim'].apply(claim2vec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>claim</th>\n",
       "      <th>conclusion</th>\n",
       "      <th>url</th>\n",
       "      <th>vec</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>669</th>\n",
       "      <td>Trade union sign up for those aged between 25 ...</td>\n",
       "      <td>This is not quite correct. 13% of the UK popul...</td>\n",
       "      <td>economy/union-membership-among-young-people</td>\n",
       "      <td>[-0.0107422, 0.00634766, -0.0569946, -0.090368...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>670</th>\n",
       "      <td>£1 in every £10 that goes to the NHS is to pay...</td>\n",
       "      <td>This isn’t correct according to the latest cos...</td>\n",
       "      <td>health/what-nhs-paying-private-finance-initiat...</td>\n",
       "      <td>[-0.00469971, 0.0252609, 0.0643616, 0.0410461,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>671</th>\n",
       "      <td>The Treasury loses £40 billion each year due t...</td>\n",
       "      <td>That’s a long way off from HMRC’s estimates, ...</td>\n",
       "      <td>economy/does-treasury-lose-40-billion-each-yea...</td>\n",
       "      <td>[0.141553, 0.127283, -0.0599688, 0.110437, -0....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>672</th>\n",
       "      <td>A man who assaulted his wife was spared jail b...</td>\n",
       "      <td>We don’t know for sure, but it seems unlikely ...</td>\n",
       "      <td>law/domestic-violence-vulnerable-mustafa-bashir</td>\n",
       "      <td>[0.000762939, 0.0124207, 0.0871887, -0.0160217...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>673</th>\n",
       "      <td>Putting a doctor through medical school “costs...</td>\n",
       "      <td>That’s wrong. About £64,300 of that comes in s...</td>\n",
       "      <td>health/cost-training-doctor</td>\n",
       "      <td>[-0.0274658, 0.0705261, 0.0146942, 0.0796051, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 claim  \\\n",
       "669  Trade union sign up for those aged between 25 ...   \n",
       "670  £1 in every £10 that goes to the NHS is to pay...   \n",
       "671  The Treasury loses £40 billion each year due t...   \n",
       "672  A man who assaulted his wife was spared jail b...   \n",
       "673  Putting a doctor through medical school “costs...   \n",
       "\n",
       "                                            conclusion  \\\n",
       "669  This is not quite correct. 13% of the UK popul...   \n",
       "670  This isn’t correct according to the latest cos...   \n",
       "671   That’s a long way off from HMRC’s estimates, ...   \n",
       "672  We don’t know for sure, but it seems unlikely ...   \n",
       "673  That’s wrong. About £64,300 of that comes in s...   \n",
       "\n",
       "                                                   url  \\\n",
       "669        economy/union-membership-among-young-people   \n",
       "670  health/what-nhs-paying-private-finance-initiat...   \n",
       "671  economy/does-treasury-lose-40-billion-each-yea...   \n",
       "672    law/domestic-violence-vulnerable-mustafa-bashir   \n",
       "673                        health/cost-training-doctor   \n",
       "\n",
       "                                                   vec  \n",
       "669  [-0.0107422, 0.00634766, -0.0569946, -0.090368...  \n",
       "670  [-0.00469971, 0.0252609, 0.0643616, 0.0410461,...  \n",
       "671  [0.141553, 0.127283, -0.0599688, 0.110437, -0....  \n",
       "672  [0.000762939, 0.0124207, 0.0871887, -0.0160217...  \n",
       "673  [-0.0274658, 0.0705261, 0.0146942, 0.0796051, ...  "
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "claims_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array(claims_df['vec'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Original ***:\n",
      " The EU referendum outcome is \"on a knife edge\".\n",
      "\n",
      "*** Top 5 ***:\n",
      " ['The EU referendum was “advisory” only.'\n",
      " 'The government’s EU leaflet distributed before the referendum said that leaving the EU meant leaving the single market.'\n",
      " 'Campaigners on both sides of the EU referendum made false claims.'\n",
      " 'The Vote Leave campaign includes three completely untrue claims on its EU referendum leaflet: Turkey becoming a member, an EU army and the £350 million a week cost of membership.'\n",
      " 'If we vote ‘remain’... The EU will continue to control… trade']\n",
      "\n",
      "*** Bottom 5 ***:\n",
      " ['Free schools improve neighbouring schools. '\n",
      " 'Low-paid workers are trapped in poverty.'\n",
      " '44.2% of Southern Mainline and Coast trains were not at terminus on time.'\n",
      " '95% of new workers are foreigners.'\n",
      " '58.8% of graduates are in non-graduate jobs. ']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dsc/anaconda/envs/p3/lib/python3.5/site-packages/sklearn/utils/validation.py:395: DeprecationWarning: Passing 1d arrays as data is deprecated in 0.17 and will raise ValueError in 0.19. Reshape your data either using X.reshape(-1, 1) if your data has a single feature or X.reshape(1, -1) if it contains a single sample.\n",
      "  DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "for example in [200, 300, 400]:\n",
    "    dists, neighbours = nn.kneighbors(X[example], n_neighbors=673)\n",
    "    dists = dists[0]\n",
    "    neighbours = neighbours[0]\n",
    "    print('*** Original ***:\\n', claims_df.iloc[neighbours[0], 0])\n",
    "    print('\\n*** Top 5 ***:\\n', claims_df.iloc[neighbours[1:], 0].head(5).values)\n",
    "    print('\\n*** Bottom 5 ***:\\n', claims_df.iloc[neighbours, 0].tail(5).values)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "*** Politician said ***:\n",
      "  The TUC estimates suggest that tax avoidance costs us £25 billion a year, while tax evasion costs us £70 billion a year\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['Tax evasion and tax avoidance costs the government £34 billion a year.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  The TUC estimates suggest that tax avoidance costs us £25 billion a year, while tax evasion costs us £70 billion a year\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['Tax evasion and tax avoidance costs the government £34 billion a year.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  In 2009, Greece's budget deficit was running at 13\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['In 2010 the budget deficit was running at over 10% of the UK’s GDP.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  At the same time the gulf between rich and poor has got wider, with the attainment gap between students in fee-paying schools and those in state schools doubling\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'The attainment gap between rich and poor pupils in grammar schools is virtually zero.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  If they see public expenditure cut so that their local schools are not refurbished, and if they see a tax on welfare benefits, they will expect us at least to maximise the revenue from the tax that people and organisations should be paying\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'Self-employed people will only pay more tax after this budget if they have taxable profits over £32,000, once you take every change to tax and welfare policies into account.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  Of this, £29 billion will come from tax measures, including the increase in VAT, higher capital gains tax and a new permanent levy on banks\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'Conservative-led changes to corporation tax, inheritance tax, capital gains tax and the bank levy will lose the government £70 billion between April 2016 and April 2022.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  However, if this is the Government's main priority, they would do better to look at the state of the UK tax system where the top five retail banks stand to cut around £19 billion from their tax bills in the future because of huge losses during the economic downturn, despite being saved by the UK Government through an £850 billion bail-out\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'Further measures that cut UK corporation tax to 12.5%, matching Ireland, would cost the UK government £120 billion over the next five years.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  In that sense, people in England have an interest in what happens in Wales as much as Welsh people do\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['94% of people in Wales don’t want independence.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  A referendum on that would be the same as a referendum on taking part in all the EU's institutions-in other words, being in the EU at all\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['The EU referendum was “advisory” only.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  We were promised last year:\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['We were promised a ‘punishment budget’ before the referendum.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      " The Government could simply have said, \"There will be a referendum on each new EU treaty-period\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['The EU referendum was “advisory” only.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      " The Chairman of the Communities and Local Government Committee is absolutely right: my colleagues and I have been reflecting over the past few weeks, when the representatives of many local authorities throughout the country have been to see us, on what it must have been like to have been in a previous Administration, when one could believe that money was no object-that one could simply get this thing from the money tree and spend it as one wished by giving higher and higher settlements to every authority that came to visit\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'The local Government Association said the spending review would allow some councils to go some of the way to raise the money they need for social care.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  As was said earlier, £120 billion of tax has not been paid as a result of tax evasion and avoidance\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['Tax evasion and tax avoidance costs the government £34 billion a year.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  As was said earlier, £120 billion of tax has not been paid as a result of tax evasion and avoidance\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['Tax evasion and tax avoidance costs the government £34 billion a year.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  The deficit was expected to be £149 billion for last year\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'The NHS had a deficit of over £800 million last year and is predicted to have a £2 billion deficit this year. ']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  The deficit was expected to be £149 billion for last year\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'The NHS had a deficit of over £800 million last year and is predicted to have a £2 billion deficit this year. ']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  The corporation tax cut from the Chancellor is one way of proceeding for a Budget strategy, but I believe he could have done something far more radical: instead of giving away that corporation tax cut, he could have spent the cash on a massive programme of employment, training and support for young people in the economy\n",
      "\n",
      "*** Relevant claims ***:\n",
      " [ 'Self-employed people will only pay more tax after this budget if they have taxable profits over £32,000, once you take every change to tax and welfare policies into account.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  House building under Labour fell to its lowest level since 1946\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['House building has fallen to its lowest level since the 1920s.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  There was a £38 billion black hole\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['There was a £38 billion black hole in the defence budget in 2010.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      "  In 2010-11, just 150 people under the age of 19 started an apprenticeship, as against 200 people aged between 19 and 24 and 250 people aged 25 and over\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['600,000 people aged 18-24 are unemployed.']\n",
      "--------------------------------------------------\n",
      "*** Politician said ***:\n",
      " Tax evasion and tax fraud cost the Exchequer billions every year\n",
      "\n",
      "*** Relevant claims ***:\n",
      " ['Tax evasion and tax avoidance costs the government £34 billion a year.']\n",
      "--------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "xml_file = 'data/hansard/src/debates2017-01-09a.xml'\n",
    "\n",
    "import os\n",
    "base = 'data/hansard/src/'\n",
    "for fname in os.listdir(base):\n",
    "    xml_file = os.path.join(base, fname)\n",
    "    \n",
    "    try:\n",
    "        with open(xml_file) as fh:\n",
    "            xml = etree.parse(fh)\n",
    "    except UnicodeDecodeError:\n",
    "        continue\n",
    "\n",
    "    for i, p in enumerate(xml.findall('.//p')):\n",
    "        if not p.text or len(p.text) < 140:\n",
    "            continue\n",
    "        \n",
    "        for sent in p.text.split('.'):\n",
    "            vec = claim2vec(sent)\n",
    "            dists, neighbours = nn.kneighbors([vec], n_neighbors=5)\n",
    "            dists = dists[0]\n",
    "            dists = dists[dists <= 0.15]\n",
    "            if len(dists) == 0:\n",
    "                continue\n",
    "            neighbours = neighbours[0][:len(dists)].tolist()\n",
    "            print('*** Politician said ***:\\n', sent)\n",
    "            print('\\n*** Relevant claims ***:\\n', claims_df.iloc[neighbours, 0].head(5).values)\n",
    "\n",
    "            print('-'*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:p3]",
   "language": "python",
   "name": "conda-env-p3-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}