topic_diversity_experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from diversity_metrics import *\n",
    "\n",
    "from gensim.test.utils import datapath\n",
    "from gensim.models import KeyedVectors\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "#load fasttext English embeddings\n",
    "wv = gensim.models.fasttext.load_facebook_model('cc.en.300.bin.gz') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "puw: 1.0\n",
      "jd: 1.0\n",
      "we-pd: 0.8854947608266958\n",
      "we-cd: 0.7022003047740666\n",
      "irbo p=0.5: 1.0\n",
      "irbo p=0.9: 1.0\n",
      "we-irbo p=0.5: 0.8863432257652993\n",
      "we-irbo p=0.9: 0.8473274547767722\n"
     ]
    }
   ],
   "source": [
    "#case 1: semantically different topic indicators\n",
    "word_list1 =['key', 'encryption', 'chip','keys', 'clipper', 'privacy','government', 'security', 'system','secure']\n",
    "word_list2= ['space',  'nasa', 'air', 'earth', 'shuttle', 'energy', 'orbit',  'launch','satellite', 'spacecraft']\n",
    "topics = [word_list1, word_list2]\n",
    "\n",
    "print(\"puw:\",proportion_unique_words(topics, topk=10))\n",
    "print(\"jd:\", pairwise_jaccard_diversity(topics, topk=10))\n",
    "print(\"we-pd:\", pairwise_word_embedding_distance(topics, wv, topk=10))\n",
    "print(\"we-cd:\", centroid_distance(topics, wv, topk=10))\n",
    "print(\"irbo p=0.5:\",irbo(topics, weight=0.5, topk=10))\n",
    "print(\"irbo p=0.9:\",irbo(topics, weight=0.9, topk=10))\n",
    "print(\"we-irbo p=0.5:\",word_embedding_irbo(topics,wv, weight=0.5, topk=10))\n",
    "print(\"we-irbo p=0.9:\",word_embedding_irbo(topics,wv, weight=0.9, topk=10))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "puw: 0.95\n",
      "jd: 0.9473684210526316\n",
      "we-pd: 0.7917604124860372\n",
      "we-cd: 0.40876076228943925\n",
      "irbo p=0.5: 0.9995876736111111\n",
      "irbo p=0.9: 0.9564749821\n",
      "we-irbo p=0.5: 0.8081312898605358\n",
      "we-irbo p=0.9: 0.7122522800966331\n"
     ]
    }
   ],
   "source": [
    "#case 1: semantically different topic indicators\n",
    "word_list1 =['key', 'encryption', 'chip','keys', 'clipper', 'privacy','government', 'security', 'system','secure']\n",
    "word_list2 = ['systems','dos', 'windows', 'data', 'system', 'keyboard', 'standard', 'software', 'phone', 'computer']\n",
    "topics = [word_list1, word_list2]\n",
    "\n",
    "print(\"puw:\",proportion_unique_words(topics, topk=10))\n",
    "print(\"jd:\", pairwise_jaccard_diversity(topics, topk=10))\n",
    "print(\"we-pd:\", pairwise_word_embedding_distance(topics, wv, topk=10))\n",
    "print(\"we-cd:\", centroid_distance(topics, wv, topk=10))\n",
    "print(\"irbo p=0.5:\",irbo(topics, weight=0.5, topk=10))\n",
    "print(\"irbo p=0.9:\",irbo(topics, weight=0.9, topk=10))\n",
    "print(\"we-irbo p=0.5:\",word_embedding_irbo(topics,wv, weight=0.5, topk=10))\n",
    "print(\"we-irbo p=0.9:\",word_embedding_irbo(topics,wv, weight=0.9, topk=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "puw: 0.5\n",
      "jd: 0.0\n",
      "we-pd: 0.7034720531478524\n",
      "we-cd: 0.0\n",
      "irbo p=0.5: 0.949259052579365\n",
      "irbo p=0.9: 0.41664842007142844\n",
      "we-irbo p=0.5: 0.7495700333813494\n",
      "we-irbo p=0.9: 0.3196630142224881\n"
     ]
    }
   ],
   "source": [
    "#case 2: topics are equal to each other but words arein different rankings. \n",
    "#Here, we expect a scoreclose to 0, but different from 0\n",
    "\n",
    "word_list1 =['key', 'encryption', 'chip','keys', 'clipper', 'privacy','government', 'security', 'system','secure']\n",
    "word_list2 = ['system', 'privacy', 'government', 'keys', 'clipper', 'key', 'secure', 'chip', 'encryption', 'security']\n",
    "topics = [word_list1, word_list2]\n",
    "\n",
    "print(\"puw:\",proportion_unique_words(topics, topk=10))\n",
    "print(\"jd:\", pairwise_jaccard_diversity(topics, topk=10))\n",
    "print(\"we-pd:\", pairwise_word_embedding_distance(topics, wv, topk=10))\n",
    "print(\"we-cd:\", centroid_distance(topics, wv, topk=10))\n",
    "print(\"irbo p=0.5:\",irbo(topics, weight=0.5, topk=10))\n",
    "print(\"irbo p=0.9:\",irbo(topics, weight=0.9, topk=10))\n",
    "print(\"we-irbo p=0.5:\",word_embedding_irbo(topics,wv, weight=0.5, topk=10))\n",
    "print(\"we-irbo p=0.9:\",word_embedding_irbo(topics,wv, weight=0.9, topk=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "puw: 0.5\n",
      "jd: 0.0\n",
      "we-pd: 0.6388633799180389\n",
      "we-cd: 0.0\n",
      "irbo p=0.5: 0.6971757192460317\n",
      "irbo p=0.9: 0.25495442007142843\n",
      "we-irbo p=0.5: 0.4235902222746173\n",
      "we-irbo p=0.9: 0.16493039447169933\n"
     ]
    }
   ],
   "source": [
    "#case 2: topics are equal to each other but words arein different rankings. \n",
    "#Here, we expect a scoreclose to 0, but different from 0\n",
    "word_list1= ['space',  'nasa', 'air', 'earth', 'shuttle', 'energy', 'orbit',  'launch','satellite', 'spacecraft']\n",
    "word_list2= ['shuttle', 'nasa', 'air', 'space', 'orbit', 'spacecraft', 'satellite', 'earth', 'energy', 'launch']\n",
    "topics = [word_list1, word_list2]\n",
    "\n",
    "print(\"puw:\",proportion_unique_words(topics, topk=10))\n",
    "print(\"jd:\", pairwise_jaccard_diversity(topics, topk=10))\n",
    "print(\"we-pd:\", pairwise_word_embedding_distance(topics, wv, topk=10))\n",
    "print(\"we-cd:\", centroid_distance(topics, wv, topk=10))\n",
    "print(\"irbo p=0.5:\",irbo(topics, weight=0.5, topk=10))\n",
    "print(\"irbo p=0.9:\",irbo(topics, weight=0.9, topk=10))\n",
    "print(\"we-irbo p=0.5:\",word_embedding_irbo(topics,wv, weight=0.5, topk=10))\n",
    "print(\"we-irbo p=0.9:\",word_embedding_irbo(topics,wv, weight=0.9, topk=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "puw: 0.9\n",
      "jd: 0.8888888888888888\n",
      "we-pd: 0.7660228164051659\n",
      "we-cd: 0.08938503333446868\n",
      "irbo p=0.5: 0.9907583085317461\n",
      "irbo p=0.9: 0.8661255131285714\n",
      "we-irbo p=0.5: 0.3064155106536216\n",
      "we-irbo p=0.9: 0.292981129801001\n"
     ]
    }
   ],
   "source": [
    "#case 3: topics  are  composed  of  different  words  butthey are related to the same theme. \n",
    "#Here, we expect a score close to 0\n",
    "word_list1 =['system','privacy','government','keys','clipper','key', 'secure','chip','encryption','security']\n",
    "word_list2= ['systems', 'private', 'governments', 'key', 'clippers', 'keys', 'secured', 'chips','encrypt', 'securities']\n",
    "topics = [word_list1, word_list2]\n",
    "\n",
    "print(\"puw:\",proportion_unique_words(topics, topk=10))\n",
    "print(\"jd:\", pairwise_jaccard_diversity(topics, topk=10))\n",
    "print(\"we-pd:\", pairwise_word_embedding_distance(topics, wv, topk=10))\n",
    "print(\"we-cd:\", centroid_distance(topics, wv, topk=10))\n",
    "print(\"irbo p=0.5:\",irbo(topics, weight=0.5, topk=10))\n",
    "print(\"irbo p=0.9:\",irbo(topics, weight=0.9, topk=10))\n",
    "print(\"we-irbo p=0.5:\",word_embedding_irbo(topics,wv, weight=0.5, topk=10))\n",
    "print(\"we-irbo p=0.9:\",word_embedding_irbo(topics,wv, weight=0.9, topk=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "puw: 1.0\n",
      "jd: 1.0\n",
      "we-pd: 0.7028714663162827\n",
      "we-cd: 0.2611359886942879\n",
      "irbo p=0.5: 1.0\n",
      "irbo p=0.9: 1.0\n",
      "we-irbo p=0.5: 0.6124324284498543\n",
      "we-irbo p=0.9: 0.4547453124316556\n"
     ]
    }
   ],
   "source": [
    "#case 3: topics  are  composed  of  different  words  butthey are related to the same theme. \n",
    "#Here, we expect a score close to 0\n",
    "word_list1= ['space',  'nasa', 'air', 'earth', 'shuttle', 'energy', 'orbit',  'launch','satellite', 'spacecraft']\n",
    "word_list2= ['universe',  'esa', 'aircraft', 'planet', 'shuttles', 'energies', 'orbits',  'launches','satellites', 'spacecrafts']\n",
    "topics = [word_list1, word_list2]\n",
    "\n",
    "print(\"puw:\",proportion_unique_words(topics, topk=10))\n",
    "print(\"jd:\", pairwise_jaccard_diversity(topics, topk=10))\n",
    "print(\"we-pd:\", pairwise_word_embedding_distance(topics, wv, topk=10))\n",
    "print(\"we-cd:\", centroid_distance(topics, wv, topk=10))\n",
    "print(\"irbo p=0.5:\",irbo(topics, weight=0.5, topk=10))\n",
    "print(\"irbo p=0.9:\",irbo(topics, weight=0.9, topk=10))\n",
    "print(\"we-irbo p=0.5:\",word_embedding_irbo(topics,wv, weight=0.5, topk=10))\n",
    "print(\"we-irbo p=0.9:\",word_embedding_irbo(topics,wv, weight=0.9, topk=10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}