iirc_generate_and_evaluate.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "\n",
    "from transformers import GPT2Tokenizer\n",
    "tokenizer = GPT2Tokenizer.from_pretrained('gpt2')\n",
    "\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "def generate(prompt,max_tokens=4000, temperature=0):\n",
    "    response = openai.Completion.create(\n",
    "        engine=\"text-davinci-002\",\n",
    "        prompt=prompt,\n",
    "        temperature=temperature,\n",
    "        max_tokens=256,\n",
    "        top_p=1,\n",
    "        frequency_penalty=0,\n",
    "        presence_penalty=0\n",
    "    )\n",
    "\n",
    "    return response[\"choices\"][0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "test_set = json.load(open('data/iirc_reranked.json','r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "dev = json.load(open(\"./data/explained_dev.json\",'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-10-21 11:47:58.298767: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libcudart.so.11.0'; dlerror: libcudart.so.11.0: cannot open shared object file: No such file or directory\n",
      "2022-10-21 11:47:58.298809: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer, util\n",
    "model = SentenceTransformer('sentence-transformers/msmarco-bert-base-dot-v5')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef7c8291f6ff4e29b7def67831111a3e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/18 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "passages = []\n",
    "for q in dev:\n",
    "    text = \"{0}\".format(q['question'])\n",
    "    passages.append(text)\n",
    "passages_embeddings = model.encode(passages, show_progress_bar=True, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 150/150 [00:17<00:00,  8.41it/s]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "import random\n",
    "\n",
    "RANDOM_PROMPT = False\n",
    "KSHOT = 4\n",
    "USE_STATIC_PROMPT = False\n",
    "STATIC_PROMPT = \"For each example, use the documents to provide an answer to the question and cite evidence from the documents to support the answer. If there is not enough information in the documents to answer the question, then state \"not enough information\".\\n\\nExample 1:\\n\\nDocument 1: Title: Fred J. Shields. Content: Ollie Murphy\\'s first-half goal gave \\'the Royals\\' a huge boost at half-time.\\n\\nDocument 2: Title: Ollie Murphy. Content: He plays club football for Carnaross\\n\\nDocument 3: Title: Ollie Murphy. Content: He came to national prominence in 1999 when he was one of Meath\\'s best player\\'s\\n\\nQuestion: Based on the above documents, Did Ollie Murphy play for any teams other than \\'the Royals\\'?\\n\\nEvidence: Document 1 says that Ollie Murphy\\'s first-half goal gave \\'the Royals\\' a huge boost at half-time. However, this does not necessarily mean that Ollie only played for \\'the Royals\\'. Document 2 states that Ollie plays club football for Carnaross. This suggests that Ollie may have played for other teams in addition to \\'the Royals\\'. Document 3 says that Ollie came to national prominence in 1999 when he was one of Meath\\'s best players. This also suggests that Ollie has played for other teams. Therefore, it is likely that Ollie has played for teams other than \\'the Royals\\'.\\n\\nAnswer: yes.\\n\\nExample 2:\\n\\nDocument 1: Title: Don Rendell. Content: The club played in the Brunei Premier League in the early 2000s, winning the league title in 2002 and 2004.\\n\\nQuestion: Based on the above documents, What club came in second at the 2004 Brunei Premier League?\\n\\nEvidence:  There is not enough information in the documents to answer the question.\\n\\nAnswer: not enough information.\\n\\nExample 3:\\n\\nDocument 1: Title: Stacy Compton. Content: Despicable Me, the first film in the series, and the first film from Illumination, was released on July 9, 2010.\\n\\nDocument 2: Title: Miranda Cosgrove. Content: Cosgrove\\'s first television appearance (aside from commercials) was in 2001 as the voice of young Lana Lang in the pilot episode of Smallville.\\n\\nDocument 3: Title: Miranda Cosgrove. Content: In 2004, Cosgrove soon landed her first major role in a television series when she was awarded a main role in the Nickelodeon series Drake & Josh\\n\\nDocument 4: Title: Miranda Cosgrove. Content: Also in 2004, Cosgrove guest-starred in a special episode of the animated series What\\'s New, Scooby-Doo?, as well as guest-starring in a season five episode of Grounded For Life\\n\\nDocument 5: Title: Miranda Cosgrove. Content: The television series, which aired on Disney, is a spin-off of the original film, Lilo & Stitch\\n\\nDocument 6: Title: Miranda Cosgrove. Content: The first of these appearances was in Zoey 101. Cosgrove later guest starred on an episode of Unfabulous,\\n\\nDocument 7: Title: Miranda Cosgrove. Content: However, Cosgrove was already in the works of starring in her own sitcom, titled iCarly, released on September 8, 2007.\\n\\nQuestion: Based on the above documents, How many TV shows had Miranda Cosgrove been featured in by the year Despicable Me was released?\\n\\nEvidence: According to document 1, Despicable Me was released on July 9, 2010.Document 2 states that Cosgrove\\'s first television appearance was in 2001 as the voice of young Lana Lang in the pilot episode of Smallville.Document 3 says that, in 2004, Cosgrove landed her first major role in a television series when she was awarded a main role in the Nickelodeon series Drake & Josh.Document 4 states that, also in 2004, Cosgrove guest-starred in a special episode of the animated series What\\'s New, Scooby-Doo?, as well as guest-starring in a season five episode of Grounded For Life.Document 5 says that the television series, which aired on Disney, is a spin-off of the original film, Lilo & Stitch.Document 6 states that the first of these appearances was in Zoey 101. Cosgrove later guest starred on an episode of Unfabulous.Document 7 says that, however, Cosgrove was already in the works of starring in her own sitcom, titled iCarly, released on September 8, 2007.Therefore, Miranda Cosgrove had been featured in 8 TV shows by the year Despicable Me was released.\\n\\nAnswer: 8 TV shows.\\n\\nExample 4:\\n\\n\"\n",
    "\n",
    "test = []\n",
    "\n",
    "for q in tqdm(test_set):\n",
    "    if USE_STATIC_PROMPT:\n",
    "        prompt = STATIC_PROMPT\n",
    "    else:\n",
    "        if RANDOM_PROMPT:\n",
    "            hits = random.sample(dev, k=KSHOT)\n",
    "        else:\n",
    "            item_passage = \"\"\n",
    "            for i,c in enumerate(q['context']):\n",
    "                item_passage+= \"Document {0}: {1}\\n\\n\".format(i+1, c['text'])\n",
    "            item_passage += \"{0}\".format(q['question'])\n",
    "            \n",
    "            item_embedding = model.encode(item_passage)\n",
    "\n",
    "            all_top = util.dot_score(item_embedding, passages_embeddings)[0].topk(KSHOT)\n",
    "            if len(all_top.indices) == 1:\n",
    "                hits = np.array(dev)[[all_top.indices]].tolist()\n",
    "            else:\n",
    "                hits = np.array(dev)[all_top.indices].tolist()\n",
    "\n",
    "            hits.reverse()\n",
    "        \n",
    "        prompt = \"For each example, use the documents to create an \\\"Answer\\\" and an \\\"Evidence\\\" to the \\\"Question\\\". Answer \\\"not enough information\\\" when not enough information is provided in the documents.\\n\\n\"\n",
    "        for i, hit in enumerate(hits):\n",
    "            prompt += \"Example {0}:\\n\\n\".format(i+1)\n",
    "            for j,c in enumerate(hit['context']):\n",
    "                text = \"\"\n",
    "                if c['passage'] == \"main\":\n",
    "                    text = \"Title: {0}. Content: {1}\".format(hit['title'],c['text'])\n",
    "                else:\n",
    "                    text = \"Title: {0}. Content: {1}\".format(c['passage'],c['text'])\n",
    "                prompt+= \"Document {0}: {1}\\n\\n\".format(j+1, text)\n",
    "            answer = \"\"\n",
    "            if hit['answer']['type'] == \"span\":\n",
    "                answer = \", \".join([a['text'] for a in hit['answer'][\"answer_spans\"]])\n",
    "            elif hit['answer']['type'] == \"value\":\n",
    "                answer = \"{0} {1}\".format(hit['answer']['answer_value'],hit['answer']['answer_unit'])\n",
    "            elif hit['answer']['type'] == \"binary\":\n",
    "                answer = hit['answer']['answer_value']\n",
    "            elif hit['answer']['type'] == \"none\":\n",
    "                answer = \"Not enough information.\"\n",
    "            prompt += \"Question: Based on the above documents, {0}\\n\\nEvidence: {1}\\n\\nAnswer: {2}.\\n\\n\".format(hit['question'], hit['explanation'].replace('\\n',''), answer)\n",
    "        \n",
    "        prompt += \"Example {0}:\\n\\n\".format(i+2)\n",
    "\n",
    "    \n",
    "    limit_per_query = 3\n",
    "    min_total = 3\n",
    "    chosen = []\n",
    "    if len(q['decomposition']) < 2:\n",
    "        chosen = q['decomposition'][0]['documents'][:min_total]\n",
    "    else:\n",
    "        for d in q['decomposition']:\n",
    "            chosen = chosen +d['documents'][:limit_per_query]\n",
    "    for i, c in enumerate(chosen):\n",
    "        text = \"\"\n",
    "        text = \"Title: {0}. Content: {1}\".format(c['title'], c['text'])\n",
    "        prompt+= \"Document {0}: {1}\\n\\n\".format(i+1, text)\n",
    "    \n",
    "    prompt += \"Question: Based on the above documents, {0}\\n\\nEvidence:\".format(q['question'])\n",
    "    \n",
    "    q['prompt'] = prompt\n",
    "    answers = []\n",
    "    if q['answer']['type'] == \"span\":\n",
    "        at = \", \".join([a['text'] for a in q['answer'][\"answer_spans\"]])\n",
    "        answers.append(at)\n",
    "    elif q['answer']['type'] == \"value\":\n",
    "        at = \"{0} {1}\".format(q['answer']['answer_value'],q['answer']['answer_unit'])\n",
    "        answers.append(at)\n",
    "    elif q['answer']['type'] == \"binary\":\n",
    "        answers.append(q['answer']['answer_value'])\n",
    "    elif q['answer']['type'] == \"none\":\n",
    "        answers.append(\"Not enough information\")\n",
    "    q['clean_answers'] = answers\n",
    "    test.append(q)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm\n",
    "import json\n",
    "import re\n",
    "\n",
    "temperature = 0\n",
    "attempts = 1\n",
    "pattern = \"(?<=Answer:)(.*)$\"\n",
    "\n",
    "for item in tqdm(test):\n",
    "    item['responses'] = []\n",
    "    item['completions'] = []\n",
    "    for i in range(attempts):\n",
    "        res = generate(item['prompt'],temperature=temperature)\n",
    "        if \"Answer\" not in res:\n",
    "            item[\"new_prompt\"] = \"{0}{1}\\n\\nAnswer:\".format(item[\"prompt\"], res)\n",
    "            res2= generate(item[\"new_prompt\"])\n",
    "            item['results'] = \"{0}\\n\\nAnswer: {1}\".format(res, res2)\n",
    "            item['responses'].append(res2)\n",
    "            item['completions'].append(\"{0}\\n\\nAnswer: {1}\".format(res, res2))\n",
    "            item[\"asked_twice\"] = True\n",
    "        else:\n",
    "            matches = re.findall(pattern, res)\n",
    "            if len(matches) > 0: \n",
    "                response = matches[0]\n",
    "                item['responses'].append(response)\n",
    "            item['results'] = res \n",
    "            item['completions'].append(res)\n",
    "            item[\"asked_twice\"] = False\n",
    "        \n",
    "json.dump(test, open(\"results/iirc.json\",'w'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import argparse\n",
    "import collections\n",
    "import numpy as np\n",
    "import os\n",
    "import re\n",
    "import string\n",
    "import sys\n",
    "import unicodedata\n",
    "\n",
    "def normalize_answer(s):\n",
    "  \"\"\"Lower text and remove punctuation, articles and extra whitespace.\"\"\"\n",
    "  def remove_articles(text):\n",
    "    regex = re.compile(r'\\b(a|an|the)\\b', re.UNICODE)\n",
    "    return re.sub(regex, ' ', text)\n",
    "  def white_space_fix(text):\n",
    "    return ' '.join(text.split())\n",
    "  def remove_punc(text):\n",
    "    exclude = set(string.punctuation)\n",
    "    return ''.join(ch for ch in text if ch not in exclude)\n",
    "  def lower(text):\n",
    "    return text.lower()\n",
    "  def remove_accents(input_str):\n",
    "      nfkd_form = unicodedata.normalize('NFKD', input_str)\n",
    "      only_ascii = nfkd_form.encode('ASCII', 'ignore')\n",
    "      return only_ascii.decode(\"utf-8\")\n",
    "\n",
    "  return white_space_fix(remove_articles(remove_punc(lower(remove_accents(s)))))\n",
    "\n",
    "def get_tokens(s):\n",
    "  if not s: return []\n",
    "  return normalize_answer(s).split()\n",
    "\n",
    "def compute_exact(a_gold, a_pred):\n",
    "  return int(normalize_answer(a_gold) == normalize_answer(a_pred))\n",
    "\n",
    "def compute_f1(a_gold, a_pred):\n",
    "  gold_toks = get_tokens(a_gold)\n",
    "  pred_toks = get_tokens(a_pred)\n",
    "  common = collections.Counter(gold_toks) & collections.Counter(pred_toks)\n",
    "  num_same = sum(common.values())\n",
    "  if len(gold_toks) == 0 or len(pred_toks) == 0:\n",
    "    # If either is no-answer, then F1 is 1 if they agree, 0 otherwise\n",
    "    return int(gold_toks == pred_toks)\n",
    "  if num_same == 0:\n",
    "    return 0\n",
    "  precision = 1.0 * num_same / len(pred_toks)\n",
    "  recall = 1.0 * num_same / len(gold_toks)\n",
    "  f1 = (2 * precision * recall) / (precision + recall)\n",
    "  return f1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "from tqdm import tqdm\n",
    "\n",
    "f1s = []\n",
    "ems = []\n",
    "\n",
    "for item in tqdm(test):\n",
    "    normalised = [normalize_answer(a.replace('\\n','')) for a in item['responses']]\n",
    "    c = Counter(normalised)\n",
    "    response = c.most_common(1)[0][0]\n",
    "    if \"Not enough information provided in the documents.\" == item['clean_answers'][0]:\n",
    "        item['clean_answers'][0] = \"Not enough information\"\n",
    "    f1 = compute_f1(item['clean_answers'][0], response)\n",
    "    f1s.append(f1)\n",
    "    ems.append(compute_exact(item['clean_answers'][0], response))\n",
    "\n",
    "print(\"F1:\",np.mean(f1s))\n",
    "print(\"EM:\",np.mean(ems))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}