experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext cudf.pandas\n",
    "import pandas as pd\n",
    "from llama_cpp import Llama\n",
    "import re\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "tqdm.pandas()\n",
    "from sentence_transformers import SentenceTransformer, util\n",
    "from transformers import AutoTokenizer, AutoModelForTokenClassification, pipeline\n",
    "from concurrent.futures import ThreadPoolExecutor\n",
    "import multiprocessing\n",
    "from scipy.stats import spearmanr, kendalltau"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cerbero = Llama(\n",
    "    model_path='ggml-model-Q8_0.gguf',\n",
    "    n_ctx=4086,\n",
    "    n_gpu_layers=21\n",
    ")\n",
    "\n",
    "bugiardini = pd.read_table('ground_truth.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_prompt1 = []\n",
    "for i, farmaco in enumerate(bugiardini['farmaco']):\n",
    "    print('Sample n.', i)\n",
    "    #prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "#[|Umano|] Cos'è il farmaco {farmaco} e a cosa serve?\n",
    "#[|Assistente|]'''\n",
    "    prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "[|Umano|] A cosa serve {farmaco}?\n",
    "[|Assistente|]'''\n",
    "    output = cerbero(prompt, max_tokens=None, stop='[|Umano|]')\n",
    "    answers_prompt1.append(output['choices'][0]['text'].strip(' ').strip('\\n'))\n",
    "\n",
    "answers1 = pd.DataFrame({'answers_prompt_1':answers_prompt1\n",
    "                        })\n",
    "#answers1.to_csv('/home/luca/Scrivania/answers1_ZSP1.tsv', sep='\\t')\n",
    "answers1.to_csv('/home/luca/Scrivania/answers1_ZSP2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_prompt2 = []\n",
    "for i, farmaco in enumerate(bugiardini['farmaco']):\n",
    "    print('Sample n.', i)\n",
    "    #prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "#[|Umano|] Ci sono particolari avvertenze, precauzioni o potenziali interazioni con altri farmaci per il farmaco {farmaco}?\n",
    "#[|Assistente|]'''\n",
    "    prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "[|Umano|] Ci sono controindicazioni per {farmaco}?\n",
    "[|Assistente|]'''\n",
    "    output = cerbero(prompt, max_tokens=None, stop='[|Umano|]')\n",
    "    answers_prompt2.append(output['choices'][0]['text'].strip(' ').strip('\\n'))\n",
    "\n",
    "answers2 = pd.DataFrame({'answers_prompt_2':answers_prompt2\n",
    "                        })\n",
    "#answers2.to_csv('/home/luca/Scrivania/answers2_ZSP1.tsv', sep='\\t')\n",
    "answers2.to_csv('/home/luca/Scrivania/answers2_ZSP2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_prompt3 = []\n",
    "for i, farmaco in enumerate(bugiardini['farmaco']):\n",
    "    print('Sample n.', i)\n",
    "#    prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "#[|Umano|] Quando, in quali dosi e in che modo devo prendere il farmaco {farmaco}?\n",
    "#[|Assistente|]'''\n",
    "    prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "[|Umano|] Come devo assumere {farmaco}?\n",
    "[|Assistente|]'''\n",
    "    output = cerbero(prompt, max_tokens=None, stop='[|Umano|]')\n",
    "    answers_prompt3.append(output['choices'][0]['text'].strip(' ').strip('\\n'))\n",
    "\n",
    "answers3 = pd.DataFrame({'answers_prompt_3':answers_prompt3\n",
    "                        })\n",
    "#answers3.to_csv('/home/luca/Scrivania/answers3_ZSP1.tsv', sep='\\t')\n",
    "answers3.to_csv('/home/luca/Scrivania/answers3_ZSP2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "answers_prompt4 = []\n",
    "for i, farmaco in enumerate(bugiardini['farmaco']):\n",
    "    print('Sample n.', i)\n",
    "    #prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "#[|Umano|] Quali sono i possibili effetti indesiderati del farmaco {farmaco}?\n",
    "#[|Assistente|]'''\n",
    "    prompt = f'''Questa è una conversazione tra un umano ed un assistente AI. L'assistente AI risponde con parole semplici alle domande dei pazienti sui farmaci.\n",
    "[|Umano|] Ci sono effetti collaterali per {farmaco}?\n",
    "[|Assistente|]'''\n",
    "    output = cerbero(prompt, max_tokens=None, stop='[|Umano|]')\n",
    "    answers_prompt4.append(output['choices'][0]['text'].strip(' ').strip('\\n'))\n",
    "\n",
    "answers4 = pd.DataFrame({'answers_prompt_4':answers_prompt4\n",
    "                        })\n",
    "#answers4.to_csv('/home/luca/Scrivania/answers4_ZSP1.tsv', sep='\\t')\n",
    "answers4.to_csv('/home/luca/Scrivania/answers4_ZSP2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''answers_ZSP1 = pd.concat([\n",
    "    answers1,\n",
    "    answers2,\n",
    "    answers3,\n",
    "    answers4\n",
    "],axis=1)'''\n",
    "\n",
    "answers_ZSP2 = pd.concat([\n",
    "    answers1,\n",
    "    answers2,\n",
    "    answers3,\n",
    "    answers4\n",
    "],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_answers(txt):\n",
    "    if '[|Umano|]' in txt:\n",
    "        txt = txt.split('[|Umano|]')[0].strip()\n",
    "    elif '[| Umano |]' in txt:\n",
    "        txt = txt.split('[| Umano |]')[0].strip()\n",
    "    elif '[| Umano|]' in txt:\n",
    "        txt = txt.split('[| Umano|]')[0].strip()\n",
    "    elif '[|Umano |]' in txt:\n",
    "        txt = txt.split('[|Umano |]')[0].strip()\n",
    "\n",
    "    elif '[|Human|]' in txt:\n",
    "        txt = txt.split('[|Human|]')[0].strip()\n",
    "    elif '[| Human |]' in txt:\n",
    "        txt = txt.split('[| Human |]')[0].strip()\n",
    "    elif '[| Human|]' in txt:\n",
    "        txt = txt.split('[| Human|]')[0].strip()\n",
    "    elif '[|Human |]' in txt:\n",
    "        txt = txt.split('[|Human |]')[0].strip()\n",
    "\n",
    "    elif '[|Humano|]' in txt:\n",
    "        txt = txt.split('[|Humano|]')[0].strip()\n",
    "    elif '[| Humano |]' in txt:\n",
    "        txt = txt.split('[| Humano |]')[0].strip()\n",
    "    elif '[| Humano|]' in txt:\n",
    "        txt = txt.split('[| Humano|]')[0].strip()\n",
    "    elif '[|Humano |]' in txt:\n",
    "        txt = txt.split('[|Humano |]')[0].strip()\n",
    "\n",
    "    if '[|Assistente|]' in txt:\n",
    "        txt = txt.split('[|Assistente|]')[0].strip()\n",
    "    elif '[| Assistente |]' in txt:\n",
    "        txt = txt.split('[| Assistente |]')[0].strip()\n",
    "    elif '[| Assistente|]' in txt:\n",
    "        txt = txt.split('[| Assistente|]')[0].strip()\n",
    "    elif '[|Assistente |]' in txt:\n",
    "        txt = txt.split('[|Assistente |]')[0].strip()\n",
    "    \n",
    "    return txt\n",
    "\n",
    "#answers_ZSP1['answers_prompt_1'] = answers_ZSP1['answers_prompt_1'].progress_apply(clean_answers)\n",
    "#answers_ZSP1['answers_prompt_2'] = answers_ZSP1['answers_prompt_2'].progress_apply(clean_answers)\n",
    "#answers_ZSP1['answers_prompt_3'] = answers_ZSP1['answers_prompt_3'].progress_apply(clean_answers)\n",
    "#answers_ZSP1['answers_prompt_4'] = answers_ZSP1['answers_prompt_4'].progress_apply(clean_answers)\n",
    "\n",
    "answers_ZSP2['answers_prompt_1'] = answers_ZSP2['answers_prompt_1'].progress_apply(clean_answers)\n",
    "answers_ZSP2['answers_prompt_2'] = answers_ZSP2['answers_prompt_2'].progress_apply(clean_answers)\n",
    "answers_ZSP2['answers_prompt_3'] = answers_ZSP2['answers_prompt_3'].progress_apply(clean_answers)\n",
    "answers_ZSP2['answers_prompt_4'] = answers_ZSP2['answers_prompt_4'].progress_apply(clean_answers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###############################\n",
    "### EVALUATION - SIMILARITY ###\n",
    "###############################\n",
    "\n",
    "def get_text_embeddings(text):\n",
    "    sentences = re.findall('[^!?。.？！]+[!?。.？！]?', text)\n",
    "    sentence_embeddings = sbert.encode(sentences)\n",
    "    text_embeddings = np.mean(sentence_embeddings, axis=0)\n",
    "\n",
    "    return text_embeddings\n",
    "\n",
    "sbert = SentenceTransformer('sentence-transformers/paraphrase-multilingual-mpnet-base-v2', device='cuda')\n",
    "\n",
    "### PROMPT 1\n",
    "\n",
    "df = pd.DataFrame()\n",
    "df['introduzione_embeddings'] = bugiardini['introduzione'].progress_apply(get_text_embeddings)\n",
    "#df['answers_prompt_1_embeddings'] = answers_ZSP1['answers_prompt_1'].progress_apply(get_text_embeddings)\n",
    "df['answers_prompt_1_embeddings'] = answers_ZSP2['answers_prompt_1'].progress_apply(get_text_embeddings)\n",
    "\n",
    "cosine_similarities1 = [util.cos_sim(np.array([emb1]), np.array([emb2]))[0][0] for emb1, emb2 in zip(df['introduzione_embeddings'], df['answers_prompt_1_embeddings'])]\n",
    "\n",
    "### PROMPT 2\n",
    "\n",
    "df['avv_prec_inter_embeddings'] = bugiardini['avv_prec_inter'].progress_apply(get_text_embeddings)\n",
    "#df['answers_prompt_2_embeddings'] = answers_ZSP1['answers_prompt_2'].progress_apply(get_text_embeddings)\n",
    "df['answers_prompt_2_embeddings'] = answers_ZSP2['answers_prompt_2'].progress_apply(get_text_embeddings)\n",
    "\n",
    "cosine_similarities2 = [util.cos_sim(np.array([emb1]), np.array([emb2]))[0][0] for emb1, emb2 in zip(df['avv_prec_inter_embeddings'], df['answers_prompt_2_embeddings'])]\n",
    "\n",
    "### PROMPT 3\n",
    "\n",
    "df['dosaggio_embeddings'] = bugiardini['dosaggio'].progress_apply(get_text_embeddings)\n",
    "#df['answers_prompt_3_embeddings'] = answers_ZSP1['answers_prompt_3'].progress_apply(get_text_embeddings)\n",
    "df['answers_prompt_3_embeddings'] = answers_ZSP2['answers_prompt_3'].progress_apply(get_text_embeddings)\n",
    "\n",
    "cosine_similarities3 = [util.cos_sim(np.array([emb1]), np.array([emb2]))[0][0] for emb1, emb2 in zip(df['dosaggio_embeddings'], df['answers_prompt_3_embeddings'])]\n",
    "\n",
    "### PROMPT 4\n",
    "df = pd.DataFrame()\n",
    "df['effetti_indesiderati_embeddings'] = bugiardini['effetti_indesiderati'].progress_apply(get_text_embeddings)\n",
    "#df['answers_prompt_4_embeddings'] = answers_ZSP1['answers_prompt_4'].progress_apply(get_text_embeddings)\n",
    "df['answers_prompt_4_embeddings'] = answers_ZSP2['answers_prompt_4'].progress_apply(get_text_embeddings)\n",
    "\n",
    "cosine_similarities4 = [util.cos_sim(np.array([emb1]), np.array([emb2]))[0][0] for emb1, emb2 in zip(df['effetti_indesiderati_embeddings'], df['answers_prompt_4_embeddings'])]\n",
    "\n",
    "cos_simi = pd.DataFrame({\n",
    "    'sim1':cosine_similarities1,\n",
    "    'sim2':cosine_similarities2,\n",
    "    'sim3':cosine_similarities3,\n",
    "    'sim4':cosine_similarities4\n",
    "    })\n",
    "\n",
    "#cos_simi.to_csv('cosine_similarities_ZSP1.tsv', sep='\\t')\n",
    "cos_simi.to_csv('cosine_similarities_ZSP2.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#answers = pd.concat([answers_ZSP1, cos_simi], axis=1)\n",
    "#answers.to_csv('answers_ZSP1_clean_\\&_similarity.tsv')\n",
    "\n",
    "answers = pd.concat([answers_ZSP2, cos_simi], axis=1)\n",
    "answers.to_csv('answers_ZSP2_clean_\\&_similarity.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "###################################\n",
    "### EVALUATION - NER OVERLAP F1 ###\n",
    "###################################\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"DeepMount00/Italian_NER_XXL\")\n",
    "model = AutoModelForTokenClassification.from_pretrained(\"DeepMount00/Italian_NER_XXL\", ignore_mismatched_sizes=True)\n",
    "nlp = pipeline(\"ner\", model=model, tokenizer=tokenizer)\n",
    "\n",
    "tag_list = ['B-ETA', 'I-ETA', 'B-MALATTIA', 'I-MALATTIA', 'B-MEDICINA', 'I-MEDICINA',\n",
    "            'B-STRENGTH', 'I-STRENGTH', 'B-FREQUENZA', 'I-FREQUENZA', 'B-DURATION',\n",
    "            'I-DURATION', 'B-DOSAGGIO', 'I-DOSAGGIO', 'B-FORM', 'I-FORM']\n",
    "\n",
    "ground_truth = pd.read_table('ground_truth.tsv')\n",
    "#answers = pd.read_table('answers_ZSP1_clean_&_similarity.tsv')\n",
    "answers = pd.read_table('answers_ZSP2_clean_&_similarity.tsv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_metrics_single_pair(gold, answer):\n",
    "    ent_goldstandard = {d['word'] for d in nlp(gold.lower()) if d['word'] != '.' and d['entity'] in tag_list}\n",
    "    ent_answer = {d['word'] for d in nlp(answer.lower()) if d['word'] != '.' and d['entity'] in tag_list}\n",
    "    \n",
    "    common_entities = ent_goldstandard.intersection(ent_answer)\n",
    "\n",
    "    precision = len(common_entities) / len(ent_answer) if len(ent_answer) != 0 else 0\n",
    "    recall = len(common_entities) / len(ent_goldstandard) if len(ent_goldstandard) != 0 else 0\n",
    "    f1_score = 2 * (precision * recall) / (precision + recall) if (precision + recall) != 0 else 0\n",
    "    \n",
    "    return precision, recall, f1_score, ent_goldstandard, ent_answer\n",
    "\n",
    "def calculate_metrics_parallel(gold_standard, answers):\n",
    "    metrics_list = []\n",
    "    with ThreadPoolExecutor(max_workers=multiprocessing.cpu_count()) as executor:\n",
    "        for metrics in tqdm(executor.map(calculate_metrics_single_pair, gold_standard, answers), total=len(gold_standard)):\n",
    "            metrics_list.append(metrics)\n",
    "    return metrics_list\n",
    "\n",
    "metrics = calculate_metrics_parallel(ground_truth['introduzione'], answers['answers_prompt_1'])\n",
    "answers[['Precision_1', 'Recall_1', 'F1_1', 'ent_gold_1', 'ent_output_1']] = pd.DataFrame(metrics, columns=['Precision', 'Recall', 'F1', 'ent_gold', 'ent_output'])\n",
    "\n",
    "metrics = calculate_metrics_parallel(ground_truth['avv_prec_inter'], answers['answers_prompt_2'])\n",
    "answers[['Precision_2', 'Recall_2', 'F1_2', 'ent_gold_2', 'ent_output_2']] = pd.DataFrame(metrics, columns=['Precision', 'Recall', 'F1', 'ent_gold', 'ent_output'])\n",
    "\n",
    "metrics = calculate_metrics_parallel(ground_truth['dosaggio'], answers['answers_prompt_3'])\n",
    "answers[['Precision_3', 'Recall_3', 'F1_3', 'ent_gold_3', 'ent_output_3']] = pd.DataFrame(metrics, columns=['Precision', 'Recall', 'F1', 'ent_gold', 'ent_output'])\n",
    "\n",
    "metrics = calculate_metrics_parallel(ground_truth['effetti_indesiderati'], answers['answers_prompt_4'])\n",
    "answers[['Precision_4', 'Recall_4', 'F1_4', 'ent_gold_4', 'ent_output_4']] = pd.DataFrame(metrics, columns=['Precision', 'Recall', 'F1', 'ent_gold', 'ent_output'])\n",
    "\n",
    "#answers.to_csv('ZSP1_NER_f1_precision_recall_ents.tsv', sep='\\t')\n",
    "answers.to_csv('ZSP2_NER_f1_precision_recall_ents.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i,j in zip(['sim1', 'sim2', 'sim3', 'sim4'], ['F1_1', 'F1_2', 'F1_3', 'F1_4']):\n",
    "    #sim = pd.read_table('answers_ZSP1_clean_&_similarity.tsv')[i]\n",
    "    sim = pd.read_table('answers_ZSP2_clean_&_similarity.tsv')[i]\n",
    "    f1 = pd.read_table('ZSP2_NER_f1_precision_recall_ents.tsv')[j]\n",
    "\n",
    "    correlation_spearman, p_value_spearman = spearmanr(sim, f1)\n",
    "    correlation_kendalltau, p_value_kendalltau = kendalltau(sim, f1)\n",
    "\n",
    "    print(f\"Spearman's rank correlation coefficient: {correlation_spearman}\")\n",
    "    print(f\"P-value: {p_value_spearman}\")\n",
    "\n",
    "    print(f\"Kendall's Tau: {correlation_kendalltau}\")\n",
    "    print(f\"P-value: {p_value_kendalltau}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}