similarity_analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import re, string\n",
    "import pickle\n",
    "from nltk.corpus import stopwords\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from gensim.models import Word2Vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./docs/UG-Manual.txt', 'r') as f:\n",
    "    text = f.readlines()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), norm='l2', preprocessor=None, smooth_idf=True,\n",
       "        stop_words='english', strip_accents=None, sublinear_tf=False,\n",
       "        token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b', tokenizer=None, use_idf=True,\n",
       "        vocabulary=None)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "({'undergraduate': 1343,\n",
       "  'programmes': 1027,\n",
       "  'tech': 1299,\n",
       "  'bachelors': 188,\n",
       "  'masters': 822,\n",
       "  'dual': 465,\n",
       "  'degree': 413,\n",
       "  'sc': 1166,\n",
       "  'year': 1406,\n",
       "  'ph': 979,\n",
       "  'mspd': 877,\n",
       "  'manual': 817,\n",
       "  'procedures': 1019,\n",
       "  'requirements': 1115,\n",
       "  'indian': 718,\n",
       "  'institute': 733,\n",
       "  'technology': 1301,\n",
       "  'kanpur': 765,\n",
       "  'ug': 1340,\n",
       "  'version': 1366,\n",
       "  'sept': 1195,\n",
       "  '13': 11,\n",
       "  '2017': 26,\n",
       "  'table': 1290,\n",
       "  'contents': 347,\n",
       "  'introduction': 749,\n",
       "  '05': 1,\n",
       "  'study': 1263,\n",
       "  '06': 2,\n",
       "  'new': 891,\n",
       "  'students': 1261,\n",
       "  'admission': 99,\n",
       "  'jee': 761,\n",
       "  'jam': 760,\n",
       "  'options': 936,\n",
       "  'enrolled': 508,\n",
       "  'branch': 213,\n",
       "  'change': 255,\n",
       "  'double': 460,\n",
       "  'major': 803,\n",
       "  'minors': 858,\n",
       "  '07': 3,\n",
       "  'procedure': 1018,\n",
       "  'rules': 1151,\n",
       "  '08': 4,\n",
       "  'bs': 219,\n",
       "  'msc': 875,\n",
       "  'non': 894,\n",
       "  '09': 5,\n",
       "  'validity': 1364,\n",
       "  'cancellation': 233,\n",
       "  'academic': 70,\n",
       "  'session': 1197,\n",
       "  '10': 6,\n",
       "  'dates': 391,\n",
       "  'duration': 470,\n",
       "  'calendar': 228,\n",
       "  'curriculum': 389,\n",
       "  '11': 8,\n",
       "  'programme': 1026,\n",
       "  '12': 9,\n",
       "  'list': 795,\n",
       "  'courses': 371,\n",
       "  'registration': 1088,\n",
       "  'pre': 1001,\n",
       "  'final': 590,\n",
       "  'add': 90,\n",
       "  'drop': 462,\n",
       "  '14': 12,\n",
       "  'course': 370,\n",
       "  'load': 797,\n",
       "  'regular': 1089,\n",
       "  'semesters': 1188,\n",
       "  'exceptions': 545,\n",
       "  'regarding': 1084,\n",
       "  'summer': 1281,\n",
       "  'term': 1306,\n",
       "  '15': 13,\n",
       "  'administrative': 98,\n",
       "  'late': 774,\n",
       "  'teaching': 1297,\n",
       "  'evaluation': 530,\n",
       "  '16': 15,\n",
       "  'medium': 840,\n",
       "  'instruction': 737,\n",
       "  'offering': 918,\n",
       "  'offerings': 919,\n",
       "  'given': 630,\n",
       "  'semester': 1187,\n",
       "  'conduct': 320,\n",
       "  'attendance': 173,\n",
       "  'class': 278,\n",
       "  'work': 1403,\n",
       "  'week': 1384,\n",
       "  'timings': 1323,\n",
       "  '17': 16,\n",
       "  'performance': 967,\n",
       "  'feedback': 582,\n",
       "  'examinations': 537,\n",
       "  'quizzes': 1061,\n",
       "  'make': 806,\n",
       "  'examination': 536,\n",
       "  'results': 1134,\n",
       "  '18': 17,\n",
       "  'letter': 788,\n",
       "  'grades': 642,\n",
       "  'weights': 1389,\n",
       "  'index': 716,\n",
       "  '19': 19,\n",
       "  'cumulative': 384,\n",
       "  'declaration': 406,\n",
       "  'result': 1132,\n",
       "  'withholding': 1400,\n",
       "  'awarded': 184,\n",
       "  'grade': 640,\n",
       "  'eligibility': 489,\n",
       "  '20': 20,\n",
       "  'minimum': 856,\n",
       "  'maximum': 829,\n",
       "  'graduation': 646,\n",
       "  'distinction': 449,\n",
       "  'award': 183,\n",
       "  'degrees': 414,\n",
       "  'withdrawal': 1396,\n",
       "  '21': 27,\n",
       "  'inadequate': 706,\n",
       "  '22': 28,\n",
       "  'mechanism': 837,\n",
       "  'address': 95,\n",
       "  'warning': 1380,\n",
       "  'probation': 1016,\n",
       "  'termination': 1308,\n",
       "  '23': 29,\n",
       "  'appeal': 139,\n",
       "  'governing': 637,\n",
       "  'addition': 93,\n",
       "  '24': 30,\n",
       "  'application': 147,\n",
       "  'process': 1021,\n",
       "  'road': 1146,\n",
       "  'map': 819,\n",
       "  '25': 31,\n",
       "  '26': 33,\n",
       "  'pg': 978,\n",
       "  '27': 34,\n",
       "  'minor': 857,\n",
       "  'retrospective': 1137,\n",
       "  '28': 35,\n",
       "  'roadmap': 1147,\n",
       "  '29': 36,\n",
       "  'calculation': 227,\n",
       "  'seat': 1176,\n",
       "  'availability': 179,\n",
       "  '30': 38,\n",
       "  'leave': 782,\n",
       "  'absence': 66,\n",
       "  '31': 40,\n",
       "  'mid': 852,\n",
       "  'recess': 1068,\n",
       "  'vacation': 1361,\n",
       "  'short': 1210,\n",
       "  'temporary': 1304,\n",
       "  'penalty': 964,\n",
       "  'unsanctioned': 1351,\n",
       "  'excessive': 547,\n",
       "  'leaves': 783,\n",
       "  'permission': 973,\n",
       "  'proceed': 1020,\n",
       "  'institutions': 736,\n",
       "  '32': 41,\n",
       "  'transfer': 1329,\n",
       "  'credits': 379,\n",
       "  'waiver': 1377,\n",
       "  'lieu': 792,\n",
       "  'thereof': 1314,\n",
       "  'scholarships': 1171,\n",
       "  'awards': 186,\n",
       "  'medals': 838,\n",
       "  '33': 42,\n",
       "  'discipline': 444,\n",
       "  '34': 43,\n",
       "  'code': 287,\n",
       "  'disciplinary': 443,\n",
       "  'actions': 87,\n",
       "  'related': 1095,\n",
       "  'matter': 826,\n",
       "  'quick': 1058,\n",
       "  'guide': 656,\n",
       "  '36': 45,\n",
       "  'amendments': 131,\n",
       "  '38': 48,\n",
       "  'chapter': 259,\n",
       "  'objectives': 910,\n",
       "  'iit': 691,\n",
       "  'provide': 1039,\n",
       "  'highest': 673,\n",
       "  'level': 790,\n",
       "  'education': 478,\n",
       "  'science': 1172,\n",
       "  'produce': 1024,\n",
       "  'competent': 300,\n",
       "  'creative': 376,\n",
       "  'imaginative': 697,\n",
       "  'engineers': 504,\n",
       "  'scientists': 1175,\n",
       "  'promote': 1033,\n",
       "  'spirit': 1236,\n",
       "  'free': 616,\n",
       "  'objective': 909,\n",
       "  'enquiry': 507,\n",
       "  'development': 434,\n",
       "  'knowledge': 766,\n",
       "  'different': 438,\n",
       "  'disciplines': 445,\n",
       "  'highly': 674,\n",
       "  'skilled': 1218,\n",
       "  'technologists': 1300,\n",
       "  'honed': 679,\n",
       "  'managerial': 812,\n",
       "  'entrepreneurial': 519,\n",
       "  'skills': 1219,\n",
       "  'having': 667,\n",
       "  'team': 1298,\n",
       "  'leadership': 780,\n",
       "  'qualities': 1056,\n",
       "  'increase': 715,\n",
       "  'student': 1260,\n",
       "  'participation': 953,\n",
       "  'nation': 883,\n",
       "  'building': 222,\n",
       "  'sensitive': 1193,\n",
       "  'local': 798,\n",
       "  'needs': 889,\n",
       "  'sets': 1200,\n",
       "  'fall': 576,\n",
       "  'purview': 1052,\n",
       "  'senate': 1191,\n",
       "  'graduate': 644,\n",
       "  'committee': 294,\n",
       "  'sugc': 1278,\n",
       "  'include': 708,\n",
       "  'following': 600,\n",
       "  'committees': 295,\n",
       "  'units': 1348,\n",
       "  'directly': 441,\n",
       "  'concerned': 316,\n",
       "  'departmental': 418,\n",
       "  'dugc': 467,\n",
       "  'department': 417,\n",
       "  'constitutes': 338,\n",
       "  'consists': 334,\n",
       "  'convener': 353,\n",
       "  'nominated': 892,\n",
       "  'head': 668,\n",
       "  'consultation': 342,\n",
       "  'faculty': 569,\n",
       "  'members': 843,\n",
       "  'representatives': 1107,\n",
       "  'period': 969,\n",
       "  'tenure': 1305,\n",
       "  'years': 1407,\n",
       "  'half': 660,\n",
       "  'retiring': 1136,\n",
       "  'decided': 403,\n",
       "  'adopted': 104,\n",
       "  'advises': 112,\n",
       "  'opportunities': 931,\n",
       "  'monitors': 870,\n",
       "  'progress': 1029,\n",
       "  'academically': 71,\n",
       "  'weak': 1381,\n",
       "  'handles': 664,\n",
       "  'problem': 1017,\n",
       "  'faced': 563,\n",
       "  'standing': 1243,\n",
       "  'formed': 607,\n",
       "  'look': 799,\n",
       "  'issues': 758,\n",
       "  'wide': 1392,\n",
       "  'makes': 807,\n",
       "  'recommendations': 1073,\n",
       "  'including': 711,\n",
       "  'policy': 994,\n",
       "  'matters': 827,\n",
       "  'specific': 1230,\n",
       "  'instances': 732,\n",
       "  'constituents': 336,\n",
       "  'conveners': 354,\n",
       "  'various': 1365,\n",
       "  'dugcs': 468,\n",
       "  'pugcs': 1045,\n",
       "  'applicable': 145,\n",
       "  'chairperson': 253,\n",
       "  'ex': 533,\n",
       "  'officio': 924,\n",
       "  'nominee': 893,\n",
       "  'elected': 484,\n",
       "  'constituent': 335,\n",
       "  'subcommittees': 1264,\n",
       "  'apec': 138,\n",
       "  'core': 364,\n",
       "  'ccc': 244,\n",
       "  'chairpersons': 254,\n",
       "  'turn': 1332,\n",
       "  'constitute': 337,\n",
       "  'member': 842,\n",
       "  'coordinates': 361,\n",
       "  'oversees': 947,\n",
       "  'facets': 564,\n",
       "  'evaluates': 528,\n",
       "  'end': 496,\n",
       "  'studies': 1262,\n",
       "  'action': 86,\n",
       "  'taken': 1291,\n",
       "  'case': 239,\n",
       "  'deficient': 412,\n",
       "  'currently': 387,\n",
       "  'admitted': 102,\n",
       "  'added': 91,\n",
       "  'approved': 157,\n",
       "  'ii': 689,\n",
       "  'bachelor': 187,\n",
       "  'aerospace': 115,\n",
       "  'engineering': 503,\n",
       "  'ae': 114,\n",
       "  'biological': 207,\n",
       "  'sciences': 1173,\n",
       "  'bio': 206,\n",
       "  'bsbe': 220,\n",
       "  'chemical': 264,\n",
       "  'che': 262,\n",
       "  'civil': 275,\n",
       "  'ce': 245,\n",
       "  'computer': 313,\n",
       "  'cse': 382,\n",
       "  'electrical': 487,\n",
       "  'ee': 479,\n",
       "  'mechanical': 835,\n",
       "  'material': 823,\n",
       "  'mse': 876,\n",
       "  'earth': 475,\n",
       "  'es': 524,\n",
       "  'chemistry': 265,\n",
       "  'chm': 267,\n",
       "  'economics': 477,\n",
       "  'eco': 476,\n",
       "  'mathematics': 825,\n",
       "  'scientific': 1174,\n",
       "  'computing': 314,\n",
       "  'mth': 878,\n",
       "  'physics': 985,\n",
       "  'phy': 983,\n",
       "  'master': 821,\n",
       "  'stat': 1246,\n",
       "  'doctor': 453,\n",
       "  'philosophy': 981,\n",
       "  'offered': 917,\n",
       "  'described': 422,\n",
       "  'section': 1179,\n",
       "  'switch': 1288,\n",
       "  'multidisciplinary': 880,\n",
       "  'enhance': 506,\n",
       "  'existing': 551,\n",
       "  'changes': 257,\n",
       "  'choose': 269,\n",
       "  'apply': 150,\n",
       "  'program': 1025,\n",
       "  'streams': 1255,\n",
       "  'mechanics': 836,\n",
       "  'energy': 501,\n",
       "  'environment': 521,\n",
       "  'climate': 285,\n",
       "  'majors': 805,\n",
       "  'departments': 419,\n",
       "  'offer': 916,\n",
       "  'parent': 949,\n",
       "  'second': 1178,\n",
       "  'purpose': 1048,\n",
       "  '4th': 52,\n",
       "  'earns': 474,\n",
       "  'categories': 241,\n",
       "  'category': 242,\n",
       "  'combinations': 289,\n",
       "  'ms': 874,\n",
       "  'possible': 997,\n",
       "  'mdes': 831,\n",
       "  'bsmdes': 221,\n",
       "  'mba': 830,\n",
       "  'combining': 291,\n",
       "  'phd': 980,\n",
       "  'allowed': 127,\n",
       "  'specialization': 1229,\n",
       "  'called': 229,\n",
       "  'acknowledged': 82,\n",
       "  'sheet': 1205,\n",
       "  'completed': 304,\n",
       "  'time': 1319,\n",
       "  'required': 1113,\n",
       "  'available': 180,\n",
       "  'templates': 1303,\n",
       "  'general': 627,\n",
       "  'month': 871,\n",
       "  'july': 763,\n",
       "  'joint': 762,\n",
       "  'entrance': 518,\n",
       "  'conducted': 321,\n",
       "  'india': 717,\n",
       "  'iits': 693,\n",
       "  'specified': 1232,\n",
       "  'information': 727,\n",
       "  'brochure': 217,\n",
       "  'brought': 218,\n",
       "  'reserved': 1121,\n",
       "  'seats': 1177,\n",
       "  'reservation': 1120,\n",
       "  'shall': 1204,\n",
       "  'prescribed': 1006,\n",
       "  'board': 208,\n",
       "  'governors': 639,\n",
       "  'subject': 1265,\n",
       "  'current': 386,\n",
       "  'formulated': 610,\n",
       "  'government': 638,\n",
       "  'details': 429,\n",
       "  'breakup': 215,\n",
       "  'office': 920,\n",
       "  'stated': 1248,\n",
       "  'st': 1239,\n",
       "  'candidates': 235,\n",
       "  'filled': 589,\n",
       "  'basis': 196,\n",
       "  'qualifying': 1055,\n",
       "  'norms': 898,\n",
       "  'remain': 1099,\n",
       "  'vacant': 1360,\n",
       "  'appeared': 144,\n",
       "  'satisfy': 1162,\n",
       "  'certain': 248,\n",
       "  'relaxed': 1097,\n",
       "  'conditions': 319,\n",
       "  'preparatory': 1003,\n",
       "  'english': 505,\n",
       "  'completion': 307,\n",
       "  'passing': 958,\n",
       "  'appearance': 143,\n",
       "  'obc': 908,\n",
       "  'belonging': 204,\n",
       "  'creamy': 374,\n",
       "  'layer': 778,\n",
       "  'pwd': 1053,\n",
       "  'person': 975,\n",
       "  'disability': 442,\n",
       "  'respective': 1126,\n",
       "  'iii': 690,\n",
       "  'advanced': 108,\n",
       "  'normally': 897,\n",
       "  'admissions': 100,\n",
       "  'exceptional': 544,\n",
       "  'circumstances': 274,\n",
       "  'admit': 101,\n",
       "  'partial': 951,\n",
       "  'similar': 1215,\n",
       "  'test': 1310,\n",
       "  'qualification': 1054,\n",
       "  'equivalent': 522,\n",
       "  'recognized': 1070,\n",
       "  'university': 1349,\n",
       "  'backward': 191,\n",
       "  'classes': 279,\n",
       "  'registered': 1086,\n",
       "  'abroad': 65,\n",
       "  'officially': 923,\n",
       "  'sponsored': 1237,\n",
       "  'complete': 303,\n",
       "  'carry': 238,\n",
       "  'research': 1119,\n",
       "  'credit': 377,\n",
       "  'use': 1354,\n",
       "  'facilities': 565,\n",
       "  'official': 922,\n",
       "  'transcript': 1328,\n",
       "  'obtained': 912,\n",
       "  'deem': 409,\n",
       "  'appropriate': 154,\n",
       "  'earned': 473,\n",
       "  'counted': 368,\n",
       "  'dean': 397,\n",
       "  'affairs': 116,\n",
       "  'recommend': 1071,\n",
       "  'approval': 155,\n",
       "  'decision': 405,\n",
       "  'based': 194,\n",
       "  'advice': 110,\n",
       "  'strength': 1256,\n",
       "  'pay': 959,\n",
       "  'fees': 585,\n",
       "  'depending': 420,\n",
       "  'status': 1252,\n",
       "  'nationality': 884,\n",
       "  'governed': 636,\n",
       "  'regulations': 1092,\n",
       "  'requires': 1116,\n",
       "  'applicant': 146,\n",
       "  'fulfill': 619,\n",
       "  'eligible': 490,\n",
       "  'laid': 771,\n",
       "  'formally': 605,\n",
       "  'provisionally': 1043,\n",
       "  'submit': 1267,\n",
       "  'copies': 362,\n",
       "  'mark': 820,\n",
       "  'sheets': 1206,\n",
       "  'provisional': 1042,\n",
       "  'certificates': 250,\n",
       "  'documents': 455,\n",
       "  'date': 390,\n",
       "  'cancel': 232,\n",
       "  'fails': 573,\n",
       "  'meet': 841,\n",
       "  'stipulated': 1254,\n",
       "  'requirement': 1114,\n",
       "  'later': 775,\n",
       "  'supplied': 1284,\n",
       "  'false': 578,\n",
       "  'suppressed': 1286,\n",
       "  'relevant': 1098,\n",
       "  'seeking': 1182,\n",
       "  'runs': 1153,\n",
       "  'middle': 853,\n",
       "  'divided': 450,\n",
       "  'parts': 955,\n",
       "  'fourth': 615,\n",
       "  'november': 903,\n",
       "  'december': 401,\n",
       "  'april': 160,\n",
       "  'eighteen': 482,\n",
       "  'weeks': 1385,\n",
       "  'working': 1404,\n",
       "  'days': 393,\n",
       "  'used': 1355,\n",
       "  'utilized': 1358,\n",
       "  'day': 392,\n",
       "  'monday': 869,\n",
       "  'fourteen': 614,\n",
       "  'devoted': 435,\n",
       "  'excludes': 549,\n",
       "  'holidays': 678,\n",
       "  'spent': 1234,\n",
       "  'activities': 88,\n",
       "  'document': 454,\n",
       "  'submission': 1266,\n",
       "  'deadline': 395,\n",
       "  'conversion': 356,\n",
       "  'published': 1044,\n",
       "  'doaa': 452,\n",
       "  'website': 1383,\n",
       "  'entire': 515,\n",
       "  'ic': 686,\n",
       "  'compulsory': 310,\n",
       "  'set': 1199,\n",
       "  'includes': 710,\n",
       "  'basic': 195,\n",
       "  'electronics': 488,\n",
       "  'graphics': 650,\n",
       "  'manufacturing': 818,\n",
       "  'processes': 1023,\n",
       "  'communication': 297,\n",
       "  'physical': 984,\n",
       "  'eso': 525,\n",
       "  'electives': 486,\n",
       "  'elective': 485,\n",
       "  'basket': 197,\n",
       "  'hss': 684,\n",
       "  'humanities': 685,\n",
       "  'social': 1223,\n",
       "  'open': 929,\n",
       "  'oe': 914,\n",
       "  'condition': 318,\n",
       "  'topic': 1325,\n",
       "  'coverage': 372,\n",
       "  'meant': 833,\n",
       "  'widen': 1393,\n",
       "  'dc': 394,\n",
       "  'opt': 933,\n",
       "  'cases': 240,\n",
       "  'completing': 306,\n",
       "  'base': 193,\n",
       "  'comparable': 298,\n",
       "  'post': 998,\n",
       "  'gain': 626,\n",
       "  'strong': 1258,\n",
       "  'foundation': 613,\n",
       "  'chosen': 271,\n",
       "  'seminar': 1189,\n",
       "  'thesis': 1315,\n",
       "  'project': 1031,\n",
       "  'experience': 553,\n",
       "  'template': 1302,\n",
       "  'gives': 631,\n",
       "  'designed': 426,\n",
       "  'lay': 777,\n",
       "  'credited': 378,\n",
       "  'develop': 433,\n",
       "  'pursue': 1049,\n",
       "  'area': 161,\n",
       "  'projects': 1032,\n",
       "  'replaced': 1103,\n",
       "  'seminars': 1190,\n",
       "  'included': 709,\n",
       "  'initiated': 729,\n",
       "  'methodology': 850,\n",
       "  'quite': 1059,\n",
       "  'early': 472,\n",
       "  'aimed': 120,\n",
       "  'young': 1408,\n",
       "  'motivated': 873,\n",
       "  'individuals': 724,\n",
       "  'rigorous': 1145,\n",
       "  'training': 1327,\n",
       "  'desired': 427,\n",
       "  'understanding': 1344,\n",
       "  'maturity': 828,\n",
       "  'solid': 1224,\n",
       "  'stage': 1241,\n",
       "  'enable': 494,\n",
       "  'career': 236,\n",
       "  'contained': 345,\n",
       "  'bulletin': 223,\n",
       "  'periodically': 970,\n",
       "  'register': 1085,\n",
       "  'commencement': 292,\n",
       "  'await': 182,\n",
       "  'certificate': 249,\n",
       "  'institution': 734,\n",
       "  'stating': 1250,\n",
       "  'theory': 1313,\n",
       "  'practical': 999,\n",
       "  'passed': 957,\n",
       "  'failing': 572,\n",
       "  'cancelled': 234,\n",
       "  'responsibility': 1128,\n",
       "  'rests': 1130,\n",
       "  'terminated': 1307,\n",
       "  'involves': 754,\n",
       "  'selection': 1185,\n",
       "  'consistent': 333,\n",
       "  'detailed': 428,\n",
       "  'flexibility': 597,\n",
       "  'choice': 268,\n",
       "  'gets': 629,\n",
       "  'approves': 158,\n",
       "  'form': 603,\n",
       "  'considered': 331,\n",
       "  'valid': 1363,\n",
       "  'conflict': 325,\n",
       "  'entirely': 516,\n",
       "  'online': 928,\n",
       "  'request': 1110,\n",
       "  'instructor': 738,\n",
       "  'accepts': 73,\n",
       "  'select': 1183,\n",
       "  'number': 905,\n",
       "  'permitted': 974,\n",
       "  'ensuring': 511,\n",
       "  'requisites': 1118,\n",
       "  'timetable': 1322,\n",
       "  'clash': 277,\n",
       "  'dropped': 463,\n",
       "  'automatically': 177,\n",
       "  'plan': 990,\n",
       "  'accepted': 72,\n",
       "  'instructors': 739,\n",
       "  'need': 887,\n",
       "  'unavoidable': 1342,\n",
       "  'reason': 1064,\n",
       "  'unable': 1341,\n",
       "  'beginning': 201,\n",
       "  'submitted': 1268,\n",
       "  'option': 935,\n",
       "  'did': 437,\n",
       "  'previous': 1008,\n",
       "  'prior': 1012,\n",
       "  'fine': 594,\n",
       "  'avail': 178,\n",
       "  'using': 1356,\n",
       "  'accordingly': 78,\n",
       "  'exact': 534,\n",
       "  'dropping': 464,\n",
       "  'net': 890,\n",
       "  'charge': 261,\n",
       "  'total': 1326,\n",
       "  'exceed': 540,\n",
       "  '44': 51,\n",
       "  'applies': 149,\n",
       "  'adding': 92,\n",
       "  'does': 456,\n",
       "  'preventing': 1007,\n",
       "  'attending': 174,\n",
       "  'reasons': 1065,\n",
       "  'proper': 1034,\n",
       "  'authorization': 175,\n",
       "  'send': 1192,\n",
       "  'sent': 1194,\n",
       "  'conveyed': 359,\n",
       "  'carries': 237,\n",
       "  'weightage': 1386,\n",
       "  'terms': 1309,\n",
       "  'indicating': 721,\n",
       "  'approximate': 159,\n",
       "  'contact': 344,\n",
       "  'hours': 683,\n",
       "  'lectures': 786,\n",
       "  'tutorials': 1336,\n",
       "  'laboratory': 769,\n",
       "  'self': 1186,\n",
       "  'formula': 609,\n",
       "  '3l': 49,\n",
       "  '2t': 37,\n",
       "  'lecture': 785,\n",
       "  'tutorial': 1335,\n",
       "  'additional': 94,\n",
       "  'needed': 888,\n",
       "  'assignments': 166,\n",
       "  'normal': 896,\n",
       "  'percent': 966,\n",
       "  '35': 44,\n",
       "  '65': 58,\n",
       "  'identified': 688,\n",
       "  'special': 1228,\n",
       "  'extra': 561,\n",
       "  'upper': 1353,\n",
       "  'limit': 793,\n",
       "  'follows': 601,\n",
       "  '100': 7,\n",
       "  'consent': 326,\n",
       "  'declare': 407,\n",
       "  'pass': 956,\n",
       "  'fail': 570,\n",
       "  'shown': 1211,\n",
       "  'chooses': 270,\n",
       "  'received': 1067,\n",
       "  'spi': 1235,\n",
       "  'cpi': 373,\n",
       "  'higher': 672,\n",
       "  'intends': 743,\n",
       "  'registers': 1087,\n",
       "  '70': 60,\n",
       "  'count': 367,\n",
       "  'satisfying': 1163,\n",
       "  'overload': 945,\n",
       "  'advised': 111,\n",
       "  'low': 800,\n",
       "  'till': 1318,\n",
       "  'point': 992,\n",
       "  'eighth': 483,\n",
       "  'failed': 571,\n",
       "  'backlog': 189,\n",
       "  'vacancies': 1359,\n",
       "  'left': 787,\n",
       "  'fulfilled': 620,\n",
       "  'advance': 107,\n",
       "  'absent': 67,\n",
       "  'necessarily': 886,\n",
       "  'contiguous': 348,\n",
       "  'sanction': 1156,\n",
       "  'corresponding': 365,\n",
       "  'forced': 602,\n",
       "  'steps': 1253,\n",
       "  'payment': 961,\n",
       "  'clearance': 282,\n",
       "  'outstanding': 943,\n",
       "  'dues': 466,\n",
       "  'signing': 1214,\n",
       "  'roll': 1148,\n",
       "  'dosa': 459,\n",
       "  'expected': 552,\n",
       "  'paying': 960,\n",
       "  'fee': 581,\n",
       "  'waived': 1376,\n",
       "  'unexpected': 1346,\n",
       "  'events': 532,\n",
       "  'illness': 695,\n",
       "  'family': 579,\n",
       "  'emergency': 493,\n",
       "  'diagnostic': 436,\n",
       "  'selected': 1184,\n",
       "  'language': 772,\n",
       "  'submitting': 1269,\n",
       "  'proposal': 1036,\n",
       "  'format': 606,\n",
       "  'circulate': 272,\n",
       "  'staff': 1240,\n",
       "  'email': 491,\n",
       "  'requesting': 1111,\n",
       "  'proposer': 1038,\n",
       "  'circulated': 273,\n",
       "  'suitable': 1279,\n",
       "  'finally': 592,\n",
       "  'modified': 866,\n",
       "  'addressed': 96,\n",
       "  'resubmitted': 1131,\n",
       "  'forward': 611,\n",
       "  'comments': 293,\n",
       "  'begins': 202,\n",
       "  'finalised': 591,\n",
       "  'taking': 1292,\n",
       "  'consideration': 329,\n",
       "  'typically': 1339,\n",
       "  '40': 50,\n",
       "  '50': 53,\n",
       "  'minutes': 860,\n",
       "  'combination': 288,\n",
       "  'modular': 868,\n",
       "  'taught': 1294,\n",
       "  'assistance': 167,\n",
       "  'tutors': 1337,\n",
       "  'assistants': 168,\n",
       "  'responsible': 1129,\n",
       "  'conducting': 322,\n",
       "  'holding': 676,\n",
       "  'evaluating': 529,\n",
       "  'awarding': 185,\n",
       "  'remains': 1100,\n",
       "  'sanctioned': 1157,\n",
       "  'deregistration': 421,\n",
       "  'resulting': 1133,\n",
       "  'appear': 142,\n",
       "  'compelling': 299,\n",
       "  'operates': 930,\n",
       "  'schedule': 1168,\n",
       "  'held': 670,\n",
       "  'friday': 618,\n",
       "  '8am': 63,\n",
       "  '30pm': 39,\n",
       "  'scheduled': 1169,\n",
       "  'outside': 942,\n",
       "  'usually': 1357,\n",
       "  'minute': 859,\n",
       "  'slots': 1221,\n",
       "  '75': 62,\n",
       "  'lab': 767,\n",
       "  '180': 18,\n",
       "  'saturdays': 1164,\n",
       "  'sundays': 1282,\n",
       "  'unless': 1350,\n",
       "  'announced': 133,\n",
       "  'holiday': 677,\n",
       "  'allow': 125,\n",
       "  'miss': 864,\n",
       "  'regularly': 1091,\n",
       "  'mutually': 882,\n",
       "  'convenient': 355,\n",
       "  'hold': 675,\n",
       "  'quiz': 1060,\n",
       "  'inconveniently': 713,\n",
       "  'continuous': 352,\n",
       "  'evaluated': 527,\n",
       "  'tests': 1311,\n",
       "  'component': 308,\n",
       "  'determine': 431,\n",
       "  'inform': 726,\n",
       "  'weightages': 1387,\n",
       "  'start': 1245,\n",
       "  'periods': 971,\n",
       "  'exam': 535,\n",
       "  'ensure': 510,\n",
       "  'principle': 1010,\n",
       "  'recommended': 1074,\n",
       "  'bona': 211,\n",
       "  'fide': 587,\n",
       "  'site': 1216,\n",
       "  'giving': 632,\n",
       "  'failure': 574,\n",
       "  'supporting': 1285,\n",
       "  'chief': 266,\n",
       "  'medical': 839,\n",
       "  'officer': 921,\n",
       "  'health': 669,\n",
       "  'center': 247,\n",
       "  'assignment': 165,\n",
       "  'decide': 402,\n",
       "  'opportunity': 932,\n",
       "  'rule': 1150,\n",
       "  'inability': 705,\n",
       "  '72': 61,\n",
       "  '96': 64,\n",
       "  '120': 10,\n",
       "  '51': 54,\n",
       "  '150': 14,\n",
       "  'respectively': 1127,\n",
       "  'graded': 641,\n",
       "  'answer': 134,\n",
       "  'books': 212,\n",
       "  'soon': 1225,\n",
       "  'exams': 539,\n",
       "  'indicated': 719,\n",
       "  'endsemester': 500,\n",
       "  'returned': 1140,\n",
       "  'saved': 1165,\n",
       "  'months': 872,\n",
       "  'incharge': 707,\n",
       "  'account': 80,\n",
       "  'regularity': 1090,\n",
       "  'satisfactory': 1159,\n",
       "  'unsatisfactory': 1352,\n",
       "  'implies': 700,\n",
       "  'seven': 1201,\n",
       "  'descriptions': 424,\n",
       "  'numerical': 906,\n",
       "  'equivalents': 523,\n",
       "  'scale': 1167,\n",
       "  'points': 993,\n",
       "  'description': 423,\n",
       "  'excellent': 543,\n",
       "  'good': 635,\n",
       "  'fair': 575,\n",
       "  'exposure': 557,\n",
       "  'intended': 742,\n",
       "  'recognize': 1069,\n",
       "  'encourage': 495,\n",
       "  'sparingly': 1227,\n",
       "  'indicates': 720,\n",
       "  'requisite': 1117,\n",
       "  'repeated': 1102,\n",
       "  'facility': 566,\n",
       "  'letters': 789,\n",
       "  'stand': 1242,\n",
       "  'incomplete': 712,\n",
       "  'missed': 865,\n",
       "  'genuine': 628,\n",
       "  'satisfactorily': 1158,\n",
       "  'converted': 357,\n",
       "  'communicated': 296,\n",
       "  'mere': 845,\n",
       "  'lack': 770,\n",
       "  'decides': 404,\n",
       "  'waive': 1375,\n",
       "  'granted': 648,\n",
       "  'computation': 311,\n",
       "  'weighted': 1388,\n",
       "  'average': 181,\n",
       "  'associated': 171,\n",
       "  'g1': 624,\n",
       "  'g2': 625,\n",
       "  'c1': 224,\n",
       "  'c2': 225,\n",
       "  'calculated': 226,\n",
       "  'multiplying': 881,\n",
       "  'dividing': 451,\n",
       "  'sum': 1280,\n",
       "  'ሻ൘': 1410,\n",
       "  'overall': 944,\n",
       "  'computed': 312,\n",
       "  'manner': 816,\n",
       "  'consider': 328,\n",
       "  'latest': 776,\n",
       "  'repeat': 1101,\n",
       "  'substitute': 1272,\n",
       "  'replaces': 1104,\n",
       "  'old': 925,\n",
       "  'mentioned': 844,\n",
       "  'report': 1105,\n",
       "  'printed': 1011,\n",
       "  'copy': 363,\n",
       "  'issued': 757,\n",
       "  'duplicate': 469,\n",
       "  'withheld': 1399,\n",
       "  'paid': 948,\n",
       "  'indiscipline': 722,\n",
       "  'pending': 965,\n",
       "  'directives': 440,\n",
       "  'changed': 256,\n",
       "  'tutor': 1334,\n",
       "  'records': 1078,\n",
       "  'justifications': 764,\n",
       "  'iitk': 692,\n",
       "  'spend': 1233,\n",
       "  'pursuing': 1050,\n",
       "  'said': 1155,\n",
       "  'grant': 647,\n",
       "  'relaxation': 1096,\n",
       "  'residence': 1123,\n",
       "  'view': 1369,\n",
       "  'extent': 560,\n",
       "  'according': 77,\n",
       "  'merit': 846,\n",
       "  'continue': 351,\n",
       "  'merits': 847,\n",
       "  'order': 937,\n",
       "  'clear': 281,\n",
       "  'achieve': 81,\n",
       "  'integrated': 740,\n",
       "  'discontinued': 446,\n",
       "  '2011': 23,\n",
       "  'deemed': 410,\n",
       "  'iv': 759,\n",
       "  'met': 849,\n",
       "  'outlined': 941,\n",
       "  'sections': 1180,\n",
       "  'satisfied': 1160,\n",
       "  ...},\n",
       " array([7.67708346, 6.98393628, 6.06764555, ..., 7.67708346, 7.27161835,\n",
       "        7.27161835]))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vectorizer.vocabulary_, vectorizer.idf_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/robsr/Study/6_sem/cs671/ass2/word2vec/vocab_wiki', 'rb') as f:\n",
    "    w2v_vocab = pickle.load(f)\n",
    "\n",
    "with open('/home/robsr/Study/6_sem/cs671/ass2/word2vec/word_embd_wiki', 'rb') as f:\n",
    "    w2v_emb = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1411, 306561)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vectorizer.vocabulary_), len(w2v_vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict_map = dict(zip(vectorizer.vocabulary_.keys(), vectorizer.vocabulary_.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.499029630899191"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf_list[dict_map['minor']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1320"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(vectorizer.vocabulary_).intersection(set(w2v_vocab)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf_list=vectorizer.idf_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "dict2_map = dict(zip(w2v_vocab,w2v_emb[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.15477984, -2.0337992 ,  2.4290586 , -3.056886  , -1.057775  ,\n",
       "       -4.109627  , -5.4036374 , -0.960159  , -0.4067876 , -0.7150726 ,\n",
       "        1.433462  ,  0.07999339,  0.9158702 , -1.3823819 , -1.9065577 ,\n",
       "        0.10502692,  2.4295835 ,  0.87741166, -1.4756235 , -0.97669715,\n",
       "       -1.0124223 ,  1.4910471 ,  0.28479797,  2.9853632 , -0.33701995,\n",
       "       -3.0471282 , -3.2314794 ,  1.5150688 ,  0.66872895,  2.6914747 ,\n",
       "       -1.1656333 , -2.2341485 , -1.9307367 ,  0.57268447, -0.566805  ,\n",
       "       -0.21795437,  2.027495  ,  1.0977734 , -2.2751606 , -0.56245667,\n",
       "        1.8409363 ,  1.2036233 ,  0.3940309 , -1.6139008 , -0.16043401,\n",
       "        1.7434422 ,  2.8763914 ,  1.2819798 , -0.35282567,  1.5693824 ,\n",
       "       -1.1952358 ,  0.4316564 , -0.6656108 , -2.079122  , -1.3494529 ,\n",
       "       -1.2576004 ,  0.9210248 , -1.7706712 ,  3.2506306 , -1.3288287 ,\n",
       "        0.26579046,  0.92098695,  1.0769811 ,  3.8884976 ,  1.7446189 ,\n",
       "       -1.7932181 , -1.0333136 ,  2.1613014 , -1.9965231 , -1.696254  ,\n",
       "       -1.0704104 , -0.76798844, -0.77775836,  0.26896882, -0.24041495,\n",
       "       -1.1707941 , -4.622861  , -3.5021687 ,  3.9092197 , -0.32678112,\n",
       "       -1.5373727 ,  3.1433082 ,  1.4017816 ,  0.48829332, -0.22894092,\n",
       "        2.5554621 , -2.4959462 ,  2.2216938 ,  3.0923884 ,  4.4724984 ,\n",
       "       -1.0148011 ,  3.6243155 ,  0.131896  ,  1.7820964 , -0.53017604,\n",
       "        2.0218825 , -0.24876663, -2.4008691 , -0.2776787 ,  0.65065396,\n",
       "       -3.1064103 ,  0.5571169 , -1.549919  , -3.1912487 , -2.0252123 ,\n",
       "       -2.2461605 ,  0.5468806 ,  0.7378824 ,  4.4393325 ,  1.8606704 ,\n",
       "       -1.3276052 , -1.3606598 , -1.1355265 , -1.4822305 , -0.7390368 ,\n",
       "       -1.7845926 ,  2.435608  ,  1.290958  ,  0.7046712 , -2.0786552 ,\n",
       "        1.6273241 ,  0.68629766, -1.8233433 ,  0.26095602,  1.2677264 ,\n",
       "       -1.6654451 ,  0.4865034 ,  0.8250731 , -1.4266161 , -1.011606  ,\n",
       "        1.8296902 , -3.756431  ,  0.57943773,  5.2635303 , -2.420878  ,\n",
       "        0.2103727 , -3.8119977 ,  2.135929  ,  1.0418748 , -0.05409392,\n",
       "       -4.2821164 , -2.7273571 ,  2.3812401 ,  1.0908139 , -3.4786015 ,\n",
       "       -1.1609436 ,  0.8434747 , -5.1984773 ,  1.9266434 , -0.7542096 ,\n",
       "        1.8920239 ,  1.7752013 ,  1.9241534 , -0.6922262 ,  0.67629135,\n",
       "       -2.0849562 ,  1.7340668 ,  0.14864784,  0.43043515,  1.2734364 ,\n",
       "        5.0476923 ,  2.161562  , -3.1837423 , -0.72694653, -2.2633781 ,\n",
       "       -3.0212827 ,  1.5943966 , -2.946028  , -0.6401108 , -0.4354179 ,\n",
       "        1.944641  , -1.3057958 ,  3.5159445 , -0.53110147, -1.4454387 ,\n",
       "       -1.2192084 , -1.5373276 ,  0.6191488 ,  3.4783113 , -2.132027  ,\n",
       "       -0.47131404, -0.31746504, -2.6386423 ,  0.52657866, -0.89270717,\n",
       "       -0.68266875, -0.67592347, -0.08366153, -0.7178982 , -1.6342    ,\n",
       "       -2.7102537 ,  2.1848779 , -2.4301088 ,  1.0625861 ,  4.8372836 ,\n",
       "        2.8484035 , -0.41863942,  5.964426  ,  0.8913847 ,  3.566219  ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dict2_map['word']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "text.split('.').split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def simw2v(n, raw_text_file, sections, index_list, query):\n",
    "    \n",
    "    with open(raw_text_file, 'r') as f:\n",
    "        text = f.readlines()\n",
    "    \n",
    "    vectorizer = TfidfVectorizer(stop_words='english')\n",
    "    vectorizer.fit(text)\n",
    "    idf_list=vectorizer.idf_\n",
    "    \n",
    "    titles_avg = [para.split() for para in sections]\n",
    "    avg_vec=[]\n",
    "    \n",
    "    dict2_map = dict(zip(w2v_vocab,w2v_emb[0]))\n",
    "    \n",
    "    for i in titles_avg:\n",
    "        temp=[]\n",
    "        summ=0\n",
    "        for j in i:\n",
    "            try:\n",
    "                print('')\n",
    "                temp+=idf_list[dict_map[j]]*dict2_map[j]\n",
    "                summ+=idf_list[dict_map[j]]\n",
    "            except:\n",
    "                pass\n",
    "        avg_vec.append(temp/summ)\n",
    "    \n",
    "    ttemp=[]\n",
    "    summm=0\n",
    "    for j in query.split():\n",
    "        try:\n",
    "            ttemp+=idf_list[dict_map[j]]*dict2_map[j]\n",
    "            summm+=idf_list[dict_map[j]]\n",
    "        except:\n",
    "            pass\n",
    "    ttemp=(ttemp/summm)\n",
    "    \n",
    "    sim_mat = cosine_similarity(avg_vec, ttemp)\n",
    "\n",
    "    top_n = [i[0] for i in sorted(enumerate(sim_mat), key=lambda x:x[1], reverse=True)][:n]\n",
    "    top_sim = [i[1][0] for i in sorted(enumerate(sim_mat), key=lambda x:x[1], reverse=True)][:n]\n",
    "    top_n_ids = [index_list[top_n[i]][2] for i in range(n)]    \n",
    "    top_n_titles = [index_list[top_n[i]][0] for i in range(n)]\n",
    "    top_n_sim = [sim_mat[top_n[i]][0] for i in range(n)]\n",
    "    # print(top_n_sim)\n",
    "    # print(top_n_ids)\n",
    "    print(top_n_titles)\n",
    "    # print('\\n\\n\\n')\n",
    "    # print(sections[top_n[0]])\n",
    "    \n",
    "    return top_n_ids, top_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "simw2v(3,'./docs/UG-Manual.txt', )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def top_w2v(n, raw_text_file,  index_map, query):\n",
    "    \n",
    "    with open(raw_text_file, 'r') as f:\n",
    "        text = f.readlines()\n",
    "    \n",
    "    vectorizer = TfidfVectorizer(stop_words='english')\n",
    "    vectorizer.fit(text)\n",
    "    \n",
    "    titles = [dict_['title'] for dict_ in index_map]\n",
    "    \n",
    "    titles_vector = vectorizer.transform(titles)\n",
    "    titles_arr = titles_vector.toarray()\n",
    "\n",
    "    query_vector = vectorizer.transform([query])\n",
    "    query_arr = query_vector.toarray()\n",
    "\n",
    "    sim_mat = cosine_similarity(titles_arr, query_arr)\n",
    "    top_n = [i[0] for i in sorted(enumerate(sim_mat), key=lambda x:x[1], reverse=True)][:n]\n",
    "    top_n_ids = [index_map[top_n[i]]['id'] for i in range(n)]    \n",
    "    top_n_titles = [index_map[top_n[i]]['title'] for i in range(n)]\n",
    "    top_n_sim = [sim_mat[top_n[i]][0] for i in range(n)]\n",
    "    print(top_n_sim)\n",
    "    print(top_n_ids)\n",
    "    print(top_n_titles)\n",
    "    \n",
    "    return top_n_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_idf(3, './index_sim1.txt', index_map, 'maximum no of credits in a semester')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/home/robsr/Study/6_sem/cs671/ass2/word2vec/vocab_wiki', 'rb') as f:\n",
    "    w2v_vocab = pickle.load(f)\n",
    "\n",
    "with open('/home/robsr/Study/6_sem/cs671/ass2/word2vec/word_embd_wiki', 'rb') as f:\n",
    "    w2v_emb = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_map = dict(zip(w2v_vocab, w2v_emb[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepro_words(data):\n",
    "    ques_str = data\n",
    "    ques_str = re.sub(\"'\\w+\",'', ques_str)\n",
    "\n",
    "    for char in string.punctuation:\n",
    "        ques_str = ques_str.replace(char, '')\n",
    "\n",
    "    words = re.findall(\"[^\\s]+\", ques_str)\n",
    "    stopWords = set(stopwords.words('english'))\n",
    "    wordsFiltered = []\n",
    "    for w in words:\n",
    "        if w not in stopWords:\n",
    "            wordsFiltered.append(w)\n",
    "    return wordsFiltered"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prepro_words('can i change my branch in iitk ?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weighted_emb_string(list_):\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def word2vec(n, raw_text_file, emb_map, index_map, query):\n",
    "    \n",
    "    with open(raw_text_file, 'r') as f:\n",
    "        text = f.readlines()\n",
    "    \n",
    "    vectorizer = TfidfVectorizer(stop_words='english')\n",
    "    vectorizer.fit(text)\n",
    "    \n",
    "#     query_words = prepro_words(query)\n",
    "    raw_vocab = vectorizer.vocabulary_.keys()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "# list of text documents\n",
    "text = [\"The quick brown fox jumped over the lazy dog.\",\"The dog.\",\n",
    "\t\t\"The fox\"]\n",
    "# create the transform\n",
    "vectorizer = TfidfVectorizer()\n",
    "# tokenize and build vocab\n",
    "vectorizer.fit(text)\n",
    "# summarize\n",
    "print(vectorizer.vocabulary_)\n",
    "print(vectorizer.vocabulary_.keys())\n",
    "print(vectorizer.idf_)\n",
    "# encode document\n",
    "vector = vectorizer.transform([text[0]])\n",
    "# summarize encoded vector\n",
    "print(vector.shape)\n",
    "print(vector.toarray())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentences = index_list\n",
    "model = Word2Vec(sentences, workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "words = list(model.wv.vocab)\n",
    "print(words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}