NoDublication.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Connect to MongoDB\n",
    "from pymongo.mongo_client import MongoClient\n",
    "from pymongo.server_api import ServerApi\n",
    "from dotenv import load_dotenv\n",
    "import os\n",
    "\n",
    "# Load environment variables from .env file\n",
    "load_dotenv()\n",
    "\n",
    "# Access an environment variable\n",
    "password = os.getenv('MONGO')\n",
    "\n",
    "uri = f\"mongodb+srv://baderalotaibi3:{password}@cluster0.od393y9.mongodb.net/?retryWrites=true&w=majority\"\n",
    "# Create a new client and connect to the server\n",
    "client = MongoClient(uri, server_api=ServerApi('1'))\n",
    "db = client[\"WIKIQUIZ\"]\n",
    "collection = db[\"WikiQuizEnApi\"]\n",
    "FinalQuestions = db[\"FinalQuestions\"]\n",
    "\n",
    "# Send a ping to confirm a successful connection\n",
    "try:\n",
    "    client.admin.command('ping')\n",
    "    print(\"Pinged your deployment. You successfully connected to MongoDB!\")\n",
    "\n",
    "except Exception as e:\n",
    "    print(e)\n",
    "\n",
    "\n",
    "def process_question(text, new_text=None, dictionary=False, temp_dict=None):\n",
    "    lines = text.strip().split('\\n')\n",
    "    question = lines[0]  # The question line\n",
    "    choices = lines[1:]  # The choice lines\n",
    "    \n",
    "    parsed_choices = []\n",
    "    for choice in choices:\n",
    "        parts = choice.split('] ')\n",
    "        label = parts[0][1:]  # Remove '[' for dictionary keys\n",
    "        value = parts[1].replace(\"[ANSWER]\", \"\").strip()\n",
    "        is_answer = \"[ANSWER]\" in parts[1]\n",
    "        parsed_choices.append([label, value, is_answer])\n",
    "    \n",
    "    # Identify duplicates\n",
    "    seen = {}\n",
    "    duplicates = set()\n",
    "    for _, value, _ in parsed_choices:\n",
    "        if value in seen:\n",
    "            duplicates.add(value)\n",
    "        else:\n",
    "            seen[value] = True\n",
    "    \n",
    "    # Replace the first non-answer duplicate if new_text is provided\n",
    "    if new_text:\n",
    "        for value in duplicates:\n",
    "            for i, (label, choice_value, is_answer) in enumerate(parsed_choices):\n",
    "                if value == choice_value and not is_answer:\n",
    "                    parsed_choices[i][1] = new_text\n",
    "                    break  # Stop after replacing the first non-answer duplicate for this value\n",
    "\n",
    "    # Construct the new choices text and optionally create a dictionary\n",
    "    if dictionary:\n",
    "        # Initialize or use the provided dictionary\n",
    "        question_dict = temp_dict if temp_dict is not None else {}\n",
    "        question_dict[\"question\"] = question\n",
    "        for label, value, is_answer in parsed_choices:\n",
    "            if is_answer:\n",
    "                value += \" [ANSWER]\"\n",
    "            question_dict[label] = value\n",
    "        return question_dict\n",
    "    else:\n",
    "        new_choices = [f\"[{label}] {value}{' [ANSWER]' if is_answer else ''}\" for label, value, is_answer in parsed_choices]\n",
    "        updated_text = question + \"\\n\" + \"\\n\".join(new_choices)\n",
    "        return updated_text if new_text else (list(duplicates) if duplicates else [])\n",
    "\n",
    "def removeDublication():\n",
    "    completed_docs = FinalQuestions.count_documents({})\n",
    "    All_docs = collection.count_documents({})\n",
    "    print(f\"Total documents: {All_docs}\")\n",
    "    print(f\"Documents processed: {completed_docs}\")\n",
    "    for index,i in enumerate(collection.find().skip(completed_docs)):\n",
    "        index+=completed_docs\n",
    "        index+=1 \n",
    "        percentage_processed = ( index / All_docs) * 100\n",
    "        print(f\"Processed {index} out of {All_docs} documents.\")\n",
    "        print(f\"Percentage of documents processed: {percentage_processed:.2f}%\")\n",
    "        try:\n",
    "            question = i[\"question\"]\n",
    "        except:\n",
    "            print(\"No question found\")\n",
    "            continue\n",
    "        print(\"Question\")\n",
    "        print(question)\n",
    "        response = process_question(question)\n",
    "        print(\"Response\")\n",
    "        print(response)\n",
    "        if not response:\n",
    "            print(\"No duplicates found\")\n",
    "            response = process_question(question, dictionary=True, temp_dict=i)\n",
    "            print(response)\n",
    "            try:\n",
    "                FinalQuestions.insert_one(i)\n",
    "            except:\n",
    "                print(\"Error inserting document\")\n",
    "                print(i)\n",
    "        else:\n",
    "            print(\"Duplicates found\")\n",
    "            new_question = process_question(question, new_text=\"None Of The Above\", dictionary=True, temp_dict=i)\n",
    "            print(new_question)\n",
    "            try:\n",
    "                FinalQuestions.insert_one(new_question)\n",
    "            except:\n",
    "                print(\"Error inserting document\")\n",
    "                print(new_question)\n",
    "    print(\"****************Done****************\")\n",
    "    print(f\"Finshed processing {All_docs-completed_docs} documents.\")\n",
    "    \n",
    "    \n",
    "import time\n",
    "while True:\n",
    "\n",
    "    removeDublication()\n",
    "    print(\"****************Sleeping****************\")\n",
    "    time.sleep(60)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}