RAGAS_Synthetic_Data.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "7e3598afba6a4872b9b815ee297a43d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_24261048611f4a9ca43425d91238c387",
              "IPY_MODEL_fbf5a3db794340e8bb1b2afc15501f24",
              "IPY_MODEL_3bcef8cb1629437bb13c9a6cc28421a2"
            ],
            "layout": "IPY_MODEL_11763382958d4e34a21f9d568133b8a1"
          }
        },
        "24261048611f4a9ca43425d91238c387": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_32c5f0df6df84cd9aa2c1e9c825ce7ac",
            "placeholder": "​",
            "style": "IPY_MODEL_cfa3f842ca5749319c3d4d6252c60e2c",
            "value": "embedding nodes: 100%"
          }
        },
        "fbf5a3db794340e8bb1b2afc15501f24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c906d60ff7045fabec1dbe2db54899e",
            "max": 10,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_96a3dcfc85024c8fbb6d400f539d4f13",
            "value": 10
          }
        },
        "3bcef8cb1629437bb13c9a6cc28421a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a775c12cef66497db55eca27a22bf090",
            "placeholder": "​",
            "style": "IPY_MODEL_60cb58f404d8429cb3c137535fa0416e",
            "value": " 10/10 [00:05&lt;00:00,  1.08s/it]"
          }
        },
        "11763382958d4e34a21f9d568133b8a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": null
          }
        },
        "32c5f0df6df84cd9aa2c1e9c825ce7ac": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cfa3f842ca5749319c3d4d6252c60e2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1c906d60ff7045fabec1dbe2db54899e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96a3dcfc85024c8fbb6d400f539d4f13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a775c12cef66497db55eca27a22bf090": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60cb58f404d8429cb3c137535fa0416e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "001ceb38b1334a03bbb0f3d62e4cbe22": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_447b4cf3723b42ac867c5e7eb1f55c8a",
              "IPY_MODEL_a338e4943050446ea1f3fa62560d3268",
              "IPY_MODEL_904ef338f44b46c2b12ea76b4d5adb1b"
            ],
            "layout": "IPY_MODEL_8b781225a9af4e9da54660aebe68a4bd"
          }
        },
        "447b4cf3723b42ac867c5e7eb1f55c8a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e005362264d04f9bb26fa304c7d4bb04",
            "placeholder": "​",
            "style": "IPY_MODEL_8cc4747404b2493ba46782f8e50092bf",
            "value": "Generating: 100%"
          }
        },
        "a338e4943050446ea1f3fa62560d3268": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6996a365bbcf4031aa220a1a3ead2657",
            "max": 5,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_44a62dc1d2c7401d96337054d3f24972",
            "value": 5
          }
        },
        "904ef338f44b46c2b12ea76b4d5adb1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d720c55ed8c6455386fdc41714d1bf27",
            "placeholder": "​",
            "style": "IPY_MODEL_2a088dda6d3041c8af604163d3ae5a09",
            "value": " 5/5 [04:47&lt;00:00, 45.72s/it]"
          }
        },
        "8b781225a9af4e9da54660aebe68a4bd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e005362264d04f9bb26fa304c7d4bb04": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8cc4747404b2493ba46782f8e50092bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6996a365bbcf4031aa220a1a3ead2657": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "44a62dc1d2c7401d96337054d3f24972": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d720c55ed8c6455386fdc41714d1bf27": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a088dda6d3041c8af604163d3ae5a09": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "Evaluating RAG (Retrieval-Augmented Generation) augmented pipelines is crucial for assessing their performance. However, manually creating hundreds of QA (Question-Context-Answer) samples from documents can be time-consuming and labor-intensive. Additionally, human-generated questions may struggle to reach the level of complexity required for a thorough evaluation, ultimately impacting the quality of the assessment. By using synthetic data generation developer time in data aggregation process can be reduced by 90%."
      ],
      "metadata": {
        "id": "yMxtXjZUq5vu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jCnmg6WKVO7g",
        "outputId": "ac5a5d7d-9adb-4e6e-d6f0-fd9a80856bf8"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: ragas in /usr/local/lib/python3.10/dist-packages (0.1.13)\n",
            "Requirement already satisfied: langchain-openai in /usr/local/lib/python3.10/dist-packages (0.1.21)\n",
            "Collecting sentence_transformers\n",
            "  Downloading sentence_transformers-3.0.1-py3-none-any.whl.metadata (10 kB)\n",
            "Requirement already satisfied: xmltodict in /usr/local/lib/python3.10/dist-packages (0.13.0)\n",
            "Requirement already satisfied: python-dotenv in /usr/local/lib/python3.10/dist-packages (1.0.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from ragas) (1.26.4)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.10/dist-packages (from ragas) (2.20.0)\n",
            "Requirement already satisfied: tiktoken in /usr/local/lib/python3.10/dist-packages (from ragas) (0.7.0)\n",
            "Requirement already satisfied: langchain in /usr/local/lib/python3.10/dist-packages (from ragas) (0.2.13)\n",
            "Requirement already satisfied: langchain-core in /usr/local/lib/python3.10/dist-packages (from ragas) (0.2.30)\n",
            "Requirement already satisfied: langchain-community in /usr/local/lib/python3.10/dist-packages (from ragas) (0.2.12)\n",
            "Requirement already satisfied: openai>1 in /usr/local/lib/python3.10/dist-packages (from ragas) (1.40.6)\n",
            "Requirement already satisfied: pysbd>=0.3.4 in /usr/local/lib/python3.10/dist-packages (from ragas) (0.3.4)\n",
            "Requirement already satisfied: nest-asyncio in /usr/local/lib/python3.10/dist-packages (from ragas) (1.6.0)\n",
            "Requirement already satisfied: appdirs in /usr/local/lib/python3.10/dist-packages (from ragas) (1.4.4)\n",
            "Requirement already satisfied: transformers<5.0.0,>=4.34.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.42.4)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (4.66.5)\n",
            "Requirement already satisfied: torch>=1.11.0 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (2.3.1+cu121)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.3.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (1.13.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (0.23.5)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from sentence_transformers) (9.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (3.15.4)\n",
            "Requirement already satisfied: fsspec>=2023.5.0 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2024.5.0)\n",
            "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (24.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (6.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (2.32.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.15.1->sentence_transformers) (4.12.2)\n",
            "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /usr/local/lib/python3.10/dist-packages (from langchain-core->ragas) (1.33)\n",
            "Requirement already satisfied: langsmith<0.2.0,>=0.1.75 in /usr/local/lib/python3.10/dist-packages (from langchain-core->ragas) (0.1.99)\n",
            "Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain-core->ragas) (2.8.2)\n",
            "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /usr/local/lib/python3.10/dist-packages (from langchain-core->ragas) (8.5.0)\n",
            "Requirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai>1->ragas) (3.7.1)\n",
            "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai>1->ragas) (1.7.0)\n",
            "Requirement already satisfied: httpx<1,>=0.23.0 in /usr/local/lib/python3.10/dist-packages (from openai>1->ragas) (0.27.0)\n",
            "Requirement already satisfied: jiter<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from openai>1->ragas) (0.5.0)\n",
            "Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai>1->ragas) (1.3.1)\n",
            "Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken->ragas) (2024.5.15)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (1.13.1)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.3)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (3.1.4)\n",
            "Collecting nvidia-cuda-nvrtc-cu12==12.1.105 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-runtime-cu12==12.1.105 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cuda-cupti-cu12==12.1.105 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cudnn-cu12==8.9.2.26 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cublas-cu12==12.1.3.1 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cufft-cu12==11.0.2.54 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-curand-cu12==10.3.2.106 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl.metadata (1.5 kB)\n",
            "Collecting nvidia-cusolver-cu12==11.4.5.107 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-cusparse-cu12==12.1.0.106 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl.metadata (1.6 kB)\n",
            "Collecting nvidia-nccl-cu12==2.20.5 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl.metadata (1.8 kB)\n",
            "Collecting nvidia-nvtx-cu12==12.1.105 (from torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl.metadata (1.7 kB)\n",
            "Requirement already satisfied: triton==2.3.1 in /usr/local/lib/python3.10/dist-packages (from torch>=1.11.0->sentence_transformers) (2.3.1)\n",
            "Collecting nvidia-nvjitlink-cu12 (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.11.0->sentence_transformers)\n",
            "  Using cached nvidia_nvjitlink_cu12-12.6.20-py3-none-manylinux2014_x86_64.whl.metadata (1.5 kB)\n",
            "Requirement already satisfied: safetensors>=0.4.1 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence_transformers) (0.4.4)\n",
            "Requirement already satisfied: tokenizers<0.20,>=0.19 in /usr/local/lib/python3.10/dist-packages (from transformers<5.0.0,>=4.34.0->sentence_transformers) (0.19.1)\n",
            "Requirement already satisfied: pyarrow>=15.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (17.0.0)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (0.6)\n",
            "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (0.3.8)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (2.1.4)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (3.4.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (0.70.16)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets->ragas) (3.10.1)\n",
            "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain->ragas) (2.0.32)\n",
            "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain->ragas) (4.0.3)\n",
            "Requirement already satisfied: langchain-text-splitters<0.3.0,>=0.2.0 in /usr/local/lib/python3.10/dist-packages (from langchain->ragas) (0.2.2)\n",
            "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /usr/local/lib/python3.10/dist-packages (from langchain-community->ragas) (0.6.7)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (1.4.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn->sentence_transformers) (3.5.0)\n",
            "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ragas) (2.3.4)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ragas) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ragas) (24.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ragas) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ragas) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets->ragas) (1.9.4)\n",
            "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>1->ragas) (3.7)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai>1->ragas) (1.2.2)\n",
            "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community->ragas) (3.21.3)\n",
            "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from dataclasses-json<0.7,>=0.5.7->langchain-community->ragas) (0.9.0)\n",
            "Requirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>1->ragas) (2024.7.4)\n",
            "Requirement already satisfied: httpcore==1.* in /usr/local/lib/python3.10/dist-packages (from httpx<1,>=0.23.0->openai>1->ragas) (1.0.5)\n",
            "Requirement already satisfied: h11<0.15,>=0.13 in /usr/local/lib/python3.10/dist-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>1->ragas) (0.14.0)\n",
            "Requirement already satisfied: jsonpointer>=1.9 in /usr/local/lib/python3.10/dist-packages (from jsonpatch<2.0,>=1.33->langchain-core->ragas) (3.0.0)\n",
            "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /usr/local/lib/python3.10/dist-packages (from langsmith<0.2.0,>=0.1.75->langchain-core->ragas) (3.10.7)\n",
            "Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain-core->ragas) (0.7.0)\n",
            "Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain-core->ragas) (2.20.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (3.3.2)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->huggingface-hub>=0.15.1->sentence_transformers) (2.0.7)\n",
            "Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain->ragas) (3.0.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.11.0->sentence_transformers) (2.1.5)\n",
            "Requirement already satisfied: python-dateutil>=2.8.2 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->ragas) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->ragas) (2024.1)\n",
            "Requirement already satisfied: tzdata>=2022.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets->ragas) (2024.1)\n",
            "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.11.0->sentence_transformers) (1.3.0)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.2->pandas->datasets->ragas) (1.16.0)\n",
            "Requirement already satisfied: mypy-extensions>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain-community->ragas) (1.0.0)\n",
            "Downloading sentence_transformers-3.0.1-py3-none-any.whl (227 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m227.1/227.1 kB\u001b[0m \u001b[31m4.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hUsing cached nvidia_cublas_cu12-12.1.3.1-py3-none-manylinux1_x86_64.whl (410.6 MB)\n",
            "Using cached nvidia_cuda_cupti_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (14.1 MB)\n",
            "Using cached nvidia_cuda_nvrtc_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (23.7 MB)\n",
            "Using cached nvidia_cuda_runtime_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (823 kB)\n",
            "Using cached nvidia_cudnn_cu12-8.9.2.26-py3-none-manylinux1_x86_64.whl (731.7 MB)\n",
            "Using cached nvidia_cufft_cu12-11.0.2.54-py3-none-manylinux1_x86_64.whl (121.6 MB)\n",
            "Using cached nvidia_curand_cu12-10.3.2.106-py3-none-manylinux1_x86_64.whl (56.5 MB)\n",
            "Using cached nvidia_cusolver_cu12-11.4.5.107-py3-none-manylinux1_x86_64.whl (124.2 MB)\n",
            "Using cached nvidia_cusparse_cu12-12.1.0.106-py3-none-manylinux1_x86_64.whl (196.0 MB)\n",
            "Using cached nvidia_nccl_cu12-2.20.5-py3-none-manylinux2014_x86_64.whl (176.2 MB)\n",
            "Using cached nvidia_nvtx_cu12-12.1.105-py3-none-manylinux1_x86_64.whl (99 kB)\n",
            "Using cached nvidia_nvjitlink_cu12-12.6.20-py3-none-manylinux2014_x86_64.whl (19.7 MB)\n",
            "Installing collected packages: nvidia-nvtx-cu12, nvidia-nvjitlink-cu12, nvidia-nccl-cu12, nvidia-curand-cu12, nvidia-cufft-cu12, nvidia-cuda-runtime-cu12, nvidia-cuda-nvrtc-cu12, nvidia-cuda-cupti-cu12, nvidia-cublas-cu12, nvidia-cusparse-cu12, nvidia-cudnn-cu12, nvidia-cusolver-cu12, sentence_transformers\n",
            "Successfully installed nvidia-cublas-cu12-12.1.3.1 nvidia-cuda-cupti-cu12-12.1.105 nvidia-cuda-nvrtc-cu12-12.1.105 nvidia-cuda-runtime-cu12-12.1.105 nvidia-cudnn-cu12-8.9.2.26 nvidia-cufft-cu12-11.0.2.54 nvidia-curand-cu12-10.3.2.106 nvidia-cusolver-cu12-11.4.5.107 nvidia-cusparse-cu12-12.1.0.106 nvidia-nccl-cu12-2.20.5 nvidia-nvjitlink-cu12-12.6.20 nvidia-nvtx-cu12-12.1.105 sentence_transformers-3.0.1\n"
          ]
        }
      ],
      "source": [
        "!pip install ragas langchain-openai sentence_transformers xmltodict python-dotenv"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from google.colab import userdata\n",
        "import pandas as pd\n",
        "from langchain_community.document_loaders import PubMedLoader\n",
        "from langchain_community.embeddings import HuggingFaceBgeEmbeddings\n",
        "from langchain_openai import ChatOpenAI\n",
        "from ragas.testset.generator import TestsetGenerator\n",
        "from ragas.testset.evolutions import simple, reasoning, multi_context"
      ],
      "metadata": {
        "id": "GGOnCa9VVlF3"
      },
      "execution_count": 37,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "os.environ[\"OPENAI_API_KEY\"] = userdata.get('OPENAI_API_KEY')"
      ],
      "metadata": {
        "id": "sRiUXaXVVqAh"
      },
      "execution_count": 38,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_generation_model = ChatOpenAI(model='gpt-4o-mini')"
      ],
      "metadata": {
        "id": "xOyWmReAlzr5"
      },
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "critic_model = ChatOpenAI(model='gpt-4o')"
      ],
      "metadata": {
        "id": "AwhTTkuzl5qj"
      },
      "execution_count": 40,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model_name = \"BAAI/bge-small-en\"\n",
        "model_kwargs = {\"device\": \"cpu\"}\n",
        "encode_kwargs = {\"normalize_embeddings\": True}\n",
        "embeddings = HuggingFaceBgeEmbeddings(\n",
        "    model_name=model_name, model_kwargs=model_kwargs, encode_kwargs=encode_kwargs\n",
        ")"
      ],
      "metadata": {
        "id": "tq2_Y2uTmc-U"
      },
      "execution_count": 46,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loader = PubMedLoader(\"cancer\", load_max_docs=5)"
      ],
      "metadata": {
        "id": "LgsJDhQIVp9O"
      },
      "execution_count": 42,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "loader"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "srHFe8kNV-bN",
        "outputId": "cd446feb-a402-45ba-9a7b-83d2b29c19f8"
      },
      "execution_count": 43,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<langchain_community.document_loaders.pubmed.PubMedLoader at 0x7dcae911eb90>"
            ]
          },
          "metadata": {},
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "documents = loader.load()"
      ],
      "metadata": {
        "id": "JmbStmXSV_QF"
      },
      "execution_count": 44,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "documents"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pOxORmXvWBds",
        "outputId": "d444f7ca-3822-4f3b-bfb8-f35e39dbb178"
      },
      "execution_count": 45,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[Document(metadata={'uid': '39137027', 'Title': 'Standardization of scan protocols for RT CT simulator from different vendors using quantitative image quality technique.', 'Published': '2024-08-13', 'Copyright Information': '© 2024 The Author(s). Journal of Applied Clinical Medical Physics published by Wiley Periodicals, LLC on behalf of The American Association of Physicists in Medicine.'}, page_content=\"OBJECTIVE: To investigate the feasibility of standardizing RT simulation CT scanner protocols between vendors using target-based image quality (IQ) metrics.\\nMETHOD AND MATERIALS: A systematic assessment process in phantom was developed to standardize clinical scan protocols for scanners from different vendors following these steps: (a) images were acquired by varying CTDI and using an iterative reconstruction (IR) method (IR: iDose and model-based iterative reconstruction [IMR] of CT-Philips Big Bore scanner, SAFIRE of CT-Siemens biograph PETCT scanner), (b) CT exams were classified into body and brain protocols, (c) the rescaled noise power spectrum (NPS) was calculated, (d) quantified the IQ change due to varied CTDI and IR, and (e) matched the IR strength level. IQ metrics included noise and texture from NPS, contrast, and contrast-to-noise ratio (CNR), low contrast detectability (d'). Area under curve (AUC) of the receiver operation characteristic curve of d' was calculated and compared.\\nRESULTS: The level of change in the IQ ratio was significant (>0.6) when using IMR. The IQ ratio change was relatively low to moderate when using either iDose in CTp (0.1-0.5) or SAFIRE in CT (0.1-0.6). SAFIRE-2 in CT showed a closer match to the reference body protocol when compared to iDose-3 in CT. In the brain protocol, iDose-3 in CT could be matched to the low to moderate level of SAFIRE in CT. The AUC of d' was highest when using IMR in CT with lower CTDI, and SAFIRE in CT performed better than iDose in CT CONCLUSION: It is possible to use target-based IQ metrics to evaluate the performance of the system and operations across various scanners in a phantom. This can serve as an initial reference to convert clinical scanned protocols from one CT simulation scanner to another.\"),\n",
              " Document(metadata={'uid': '39137008', 'Title': 'Time structures of proton pencil beam scanning delivery on a microsecond scale measured with a pixelated semiconductor detector Timepix3.', 'Published': '2024-08-13', 'Copyright Information': '© 2024 The Author(s). Journal of Applied Clinical Medical Physics published by Wiley Periodicals LLC on behalf of American Association of Physicists in Medicine.'}, page_content='PURPOSE: The time structures of proton spot delivery in proton pencil beam scanning (PBS) radiation therapy are essential in many clinical applications. This study aims to characterize the time structures of proton PBS delivered by both synchrotron and synchrocyclotron accelerators using a non-invasive technique based on scattered particle tracking.\\nMETHODS: A pixelated semiconductor detector, AdvaPIX-Timepix3, with a temporal resolution of 1.56\\xa0ns, was employed to measure time of arrival of secondary particles generated by a proton beam. The detector was placed laterally to the high-flux area of the beam in order to allow for single particle detection and not interfere with the treatment. The detector recorded counts of radiation events, their deposited energy and the timestamp associated with the single events. Individual recorded events and their temporal characteristics were used to analyze beam time structures, including energy layer switch time, magnet switch time, spot switch time, and the scanning speeds in the x and y directions. All the measurements were repeated 30 times on three dates, reducing statistical uncertainty.\\nRESULTS: The uncertainty of the measured energy layer switch times, magnet switch time, and the spot switch time were all within 1% of average values. The scanning speeds uncertainties were within 1.5% and are more precise than previously reported results. The measurements also revealed continuous sub-milliseconds proton spills at a low dose rate for the synchrotron accelerator and radiofrequency pulses at 7\\xa0µs and 1\\xa0ms repetition time for the synchrocyclotron accelerator.\\nCONCLUSION: The AdvaPIX-Timepix3 detector can be used to directly measure and monitor time structures on microseconds scale of the PBS proton beam delivery. This method yielded results with high precision and is completely independent of the machine log files.'),\n",
              " Document(metadata={'uid': '39137006', 'Title': 'Tumor-Associated Antigen Burden Correlates with Immune Checkpoint Blockade Benefit in Tumors with Low Levels of T-cell Exhaustion.', 'Published': '2024-08-13', 'Copyright Information': ''}, page_content='Tumor-associated antigens (TAAs) are important targets for cancer vaccines. However, TAA-based vaccines have not yet achieved their full potential in clinical trials. In contrast, immune checkpoint blockade (ICB) has emerged as an effective therapy, leading to durable responses in selected cancer patients. To date, few generalizable associations between TAAs and ICB benefit have been reported, with most studies focusing on melanoma that has the highest mutation rate in cancer. In this study, we developed a TAA burden (TAB) algorithm based on known and putative TAAs and investigated the association of TAB with ICB benefit. Analysis of the IMVigor210 patient cohort of urothelial carcinoma treated with anti-PD-L1 revealed that high tumor mutation burden (TMB) weakened the association of TAB with ICB benefit. Furthermore, TAB correlated with ICB efficacy in tumors characterized by negative PD-L1 staining on immune cells, while high levels of PD-L1 staining on immune cells were linked to T-cell exhaustion. Validation across independent clinical datasets-including urothelial carcinoma cohorts treated with anti-PD1/PD-L1 agents and neoadjuvant anti-PD1 trials for head and neck cancers-corroborated the finding that TAB correlates with ICB benefit in tumors with low T-cell exhaustion. Pan-cancer analyses revealed that in most cancer entities, tumors with higher T-cell exhaustion exhibited lower TAB levels, implying possible immunoediting of TAAs in tumors with established antitumor immunity. Our study challenges the prevailing notion of a lack of association between TAAs and ICB response. It also underscores the need for future investigations into the immunogenicity of TAAs and TAA-based vaccine strategies in tumors with low levels of T-cell exhaustion.'),\n",
              " Document(metadata={'uid': '39136995', 'Title': 'Phase separation of PML/RARα and BRD4 coassembled microspeckles governs transcriptional dysregulation in acute promyelocytic leukemia.', 'Published': '2024-08-13', 'Copyright Information': ''}, page_content='In acute promyelocytic leukemia (APL), the promyelocytic leukemia-retinoic acid receptor alpha (PML/RARα) fusion protein destroys PML nuclear bodies (NBs), leading to the formation of microspeckles. However, our understanding, largely learned from morphological observations, lacks insight into the mechanisms behind PML/RARα-mediated microspeckle formation and its role in APL leukemogenesis. This study presents evidence uncovering liquid-liquid phase separation (LLPS) as a key mechanism in the formation of PML/RARα-mediated microspeckles. This process is facilitated by the intrinsically disordered region containing a large portion of PML and a smaller segment of RARα. We demonstrate the coassembly of bromodomain-containing protein 4 (BRD4) within PML/RARα-mediated condensates, differing from wild-type PML-formed NBs. In the absence of PML/RARα, PML NBs and BRD4 puncta exist as two independent phases, but the presence of PML/RARα disrupts PML NBs and redistributes PML and BRD4 into a distinct phase, forming PML/RARα-assembled microspeckles. Genome-wide profiling reveals a PML/RARα-induced BRD4 redistribution across the genome, with preferential binding to super-enhancers and broad-promoters (SEBPs). Mechanistically, BRD4 is recruited by PML/RARα into nuclear condensates, facilitating BRD4 chromatin binding to exert transcriptional activation essential for APL survival. Perturbing LLPS through chemical inhibition (1, 6-hexanediol) significantly reduces chromatin co-occupancy of PML/RARα and BRD4, attenuating their target gene activation. Finally, a series of experimental validations in primary APL patient samples confirm that PML/RARα forms microspeckles through condensates, recruits BRD4 to coassemble condensates, and co-occupies SEBP regions. Our findings elucidate the biophysical, pathological, and transcriptional dynamics of PML/RARα-assembled microspeckles, underscoring the importance of BRD4 in mediating transcriptional activation that enables PML/RARα to initiate APL.'),\n",
              " Document(metadata={'uid': '39136986', 'Title': 'Race adjustments in clinical algorithms can help correct for racial disparities in data quality.', 'Published': '2024-08-13', 'Copyright Information': ''}, page_content=\"['P', 'P']\\nDespite ethical and historical arguments for removing race from clinical algorithms, the consequences of removal remain unclear. Here, we highlight a largely undiscussed consideration in this debate: varying data quality of input features across race groups. For example, family history of cancer is an essential predictor in cancer risk prediction algorithms but is less reliably documented for Black participants and may therefore be less predictive of cancer outcomes. Using data from the Southern Community Cohort Study, we assessed whether race adjustments could allow risk prediction models to capture varying data quality by race, focusing on colorectal cancer risk prediction. We analyzed 77,836 adults with no history of colorectal cancer at baseline. The predictive value of self-reported family history was greater for White participants than for Black participants. We compared two cancer risk prediction algorithms-a race-blind algorithm which included standard colorectal cancer risk factors but not race, and a race-adjusted algorithm which additionally included race. Relative to the race-blind algorithm, the race-adjusted algorithm improved predictive performance, as measured by goodness of fit in a likelihood ratio test (-value: <0.001) and area under the receiving operating characteristic curve among Black participants (-value: 0.006). Because the race-blind algorithm underpredicted risk for Black participants, the race-adjusted algorithm increased the fraction of Black participants among the predicted high-risk group, potentially increasing access to screening. More broadly, this study shows that race adjustments may be beneficial when the data quality of key predictors in clinical algorithms differs by race group.\")]"
            ]
          },
          "metadata": {},
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "generator = TestsetGenerator.from_langchain(\n",
        "    data_generation_model,\n",
        "    critic_model,\n",
        "    embeddings\n",
        ")"
      ],
      "metadata": {
        "id": "w0LyO7_YYVES"
      },
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "distributions = {\n",
        "    simple: 0.5,\n",
        "    multi_context: 0.4,\n",
        "    reasoning: 0.1\n",
        "}"
      ],
      "metadata": {
        "id": "nJGlv7eSWWH4"
      },
      "execution_count": 48,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "testset = generator.generate_with_langchain_docs(documents, 5, distributions)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "7e3598afba6a4872b9b815ee297a43d3",
            "24261048611f4a9ca43425d91238c387",
            "fbf5a3db794340e8bb1b2afc15501f24",
            "3bcef8cb1629437bb13c9a6cc28421a2",
            "11763382958d4e34a21f9d568133b8a1",
            "32c5f0df6df84cd9aa2c1e9c825ce7ac",
            "cfa3f842ca5749319c3d4d6252c60e2c",
            "1c906d60ff7045fabec1dbe2db54899e",
            "96a3dcfc85024c8fbb6d400f539d4f13",
            "a775c12cef66497db55eca27a22bf090",
            "60cb58f404d8429cb3c137535fa0416e",
            "001ceb38b1334a03bbb0f3d62e4cbe22",
            "447b4cf3723b42ac867c5e7eb1f55c8a",
            "a338e4943050446ea1f3fa62560d3268",
            "904ef338f44b46c2b12ea76b4d5adb1b",
            "8b781225a9af4e9da54660aebe68a4bd",
            "e005362264d04f9bb26fa304c7d4bb04",
            "8cc4747404b2493ba46782f8e50092bf",
            "6996a365bbcf4031aa220a1a3ead2657",
            "44a62dc1d2c7401d96337054d3f24972",
            "d720c55ed8c6455386fdc41714d1bf27",
            "2a088dda6d3041c8af604163d3ae5a09"
          ]
        },
        "id": "5sFRGZvaWY31",
        "outputId": "252e201c-8107-4eba-aa99-b70adf6ef5bc"
      },
      "execution_count": 49,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "embedding nodes:   0%|          | 0/10 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "7e3598afba6a4872b9b815ee297a43d3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:ragas.testset.docstore:Filename and doc_id are the same for all nodes.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Generating:   0%|          | 0/5 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "001ceb38b1334a03bbb0f3d62e4cbe22"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df = testset.to_pandas()"
      ],
      "metadata": {
        "id": "VL3DbSodWcr8"
      },
      "execution_count": 50,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(test_df)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "JAvzirfyWegI",
        "outputId": "337ba99b-a997-48ce-e01f-0fa669561712"
      },
      "execution_count": 51,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "                                            question  \\\n",
            "0  How does the level of T-cell exhaustion affect...   \n",
            "1  What is the role of liquid-liquid phase separa...   \n",
            "2  How does LLPS affect PML/RARα microspeckles in...   \n",
            "3  How does LLPS affect PML/RARα microspeckles in...   \n",
            "4  How did the development of the TAA burden (TAB...   \n",
            "\n",
            "                                            contexts  \\\n",
            "0  [Tumor-associated antigens (TAAs) are importan...   \n",
            "1  [In acute promyelocytic leukemia (APL), the pr...   \n",
            "2  [Tumor-associated antigens (TAAs) are importan...   \n",
            "3  [Tumor-associated antigens (TAAs) are importan...   \n",
            "4  [Tumor-associated antigens (TAAs) are importan...   \n",
            "\n",
            "                                        ground_truth evolution_type  \\\n",
            "0  The level of T-cell exhaustion affects the ass...         simple   \n",
            "1  Liquid-liquid phase separation (LLPS) is a key...         simple   \n",
            "2  The answer to given question is not present in...  multi_context   \n",
            "3  The answer to given question is not present in...  multi_context   \n",
            "4  The development of the TAA burden (TAB) algori...         simple   \n",
            "\n",
            "                                            metadata  episode_done  \n",
            "0  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n",
            "1  [{'uid': '39136995', 'Title': 'Phase separatio...          True  \n",
            "2  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n",
            "3  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n",
            "4  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "test_df"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1370
        },
        "id": "h6LSOPYfqVlZ",
        "outputId": "b46babd1-5dc5-4f59-ff79-a3082ed6c4d6"
      },
      "execution_count": 52,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                            question  \\\n",
              "0  How does the level of T-cell exhaustion affect...   \n",
              "1  What is the role of liquid-liquid phase separa...   \n",
              "2  How does LLPS affect PML/RARα microspeckles in...   \n",
              "3  How does LLPS affect PML/RARα microspeckles in...   \n",
              "4  How did the development of the TAA burden (TAB...   \n",
              "\n",
              "                                            contexts  \\\n",
              "0  [Tumor-associated antigens (TAAs) are importan...   \n",
              "1  [In acute promyelocytic leukemia (APL), the pr...   \n",
              "2  [Tumor-associated antigens (TAAs) are importan...   \n",
              "3  [Tumor-associated antigens (TAAs) are importan...   \n",
              "4  [Tumor-associated antigens (TAAs) are importan...   \n",
              "\n",
              "                                        ground_truth evolution_type  \\\n",
              "0  The level of T-cell exhaustion affects the ass...         simple   \n",
              "1  Liquid-liquid phase separation (LLPS) is a key...         simple   \n",
              "2  The answer to given question is not present in...  multi_context   \n",
              "3  The answer to given question is not present in...  multi_context   \n",
              "4  The development of the TAA burden (TAB) algori...         simple   \n",
              "\n",
              "                                            metadata  episode_done  \n",
              "0  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n",
              "1  [{'uid': '39136995', 'Title': 'Phase separatio...          True  \n",
              "2  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n",
              "3  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  \n",
              "4  [{'uid': '39137006', 'Title': 'Tumor-Associate...          True  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-56b7ee54-d545-4ba6-8bec-0256d0da7846\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>question</th>\n",
              "      <th>contexts</th>\n",
              "      <th>ground_truth</th>\n",
              "      <th>evolution_type</th>\n",
              "      <th>metadata</th>\n",
              "      <th>episode_done</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>How does the level of T-cell exhaustion affect...</td>\n",
              "      <td>[Tumor-associated antigens (TAAs) are importan...</td>\n",
              "      <td>The level of T-cell exhaustion affects the ass...</td>\n",
              "      <td>simple</td>\n",
              "      <td>[{'uid': '39137006', 'Title': 'Tumor-Associate...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>What is the role of liquid-liquid phase separa...</td>\n",
              "      <td>[In acute promyelocytic leukemia (APL), the pr...</td>\n",
              "      <td>Liquid-liquid phase separation (LLPS) is a key...</td>\n",
              "      <td>simple</td>\n",
              "      <td>[{'uid': '39136995', 'Title': 'Phase separatio...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>How does LLPS affect PML/RARα microspeckles in...</td>\n",
              "      <td>[Tumor-associated antigens (TAAs) are importan...</td>\n",
              "      <td>The answer to given question is not present in...</td>\n",
              "      <td>multi_context</td>\n",
              "      <td>[{'uid': '39137006', 'Title': 'Tumor-Associate...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>How does LLPS affect PML/RARα microspeckles in...</td>\n",
              "      <td>[Tumor-associated antigens (TAAs) are importan...</td>\n",
              "      <td>The answer to given question is not present in...</td>\n",
              "      <td>multi_context</td>\n",
              "      <td>[{'uid': '39137006', 'Title': 'Tumor-Associate...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>How did the development of the TAA burden (TAB...</td>\n",
              "      <td>[Tumor-associated antigens (TAAs) are importan...</td>\n",
              "      <td>The development of the TAA burden (TAB) algori...</td>\n",
              "      <td>simple</td>\n",
              "      <td>[{'uid': '39137006', 'Title': 'Tumor-Associate...</td>\n",
              "      <td>True</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-56b7ee54-d545-4ba6-8bec-0256d0da7846')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-56b7ee54-d545-4ba6-8bec-0256d0da7846 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-56b7ee54-d545-4ba6-8bec-0256d0da7846');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-c1eebac7-0576-4fb6-b67b-97f78778b0ec\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-c1eebac7-0576-4fb6-b67b-97f78778b0ec')\"\n",
              "            title=\"Suggest charts\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-c1eebac7-0576-4fb6-b67b-97f78778b0ec button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "  <div id=\"id_aff62e49-d92c-4746-9a6e-c6a0f5d0f9cc\">\n",
              "    <style>\n",
              "      .colab-df-generate {\n",
              "        background-color: #E8F0FE;\n",
              "        border: none;\n",
              "        border-radius: 50%;\n",
              "        cursor: pointer;\n",
              "        display: none;\n",
              "        fill: #1967D2;\n",
              "        height: 32px;\n",
              "        padding: 0 0 0 0;\n",
              "        width: 32px;\n",
              "      }\n",
              "\n",
              "      .colab-df-generate:hover {\n",
              "        background-color: #E2EBFA;\n",
              "        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "        fill: #174EA6;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate {\n",
              "        background-color: #3B4455;\n",
              "        fill: #D2E3FC;\n",
              "      }\n",
              "\n",
              "      [theme=dark] .colab-df-generate:hover {\n",
              "        background-color: #434B5C;\n",
              "        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "        fill: #FFFFFF;\n",
              "      }\n",
              "    </style>\n",
              "    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('test_df')\"\n",
              "            title=\"Generate code using this dataframe.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "    <script>\n",
              "      (() => {\n",
              "      const buttonEl =\n",
              "        document.querySelector('#id_aff62e49-d92c-4746-9a6e-c6a0f5d0f9cc button.colab-df-generate');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      buttonEl.onclick = () => {\n",
              "        google.colab.notebook.generateWithVariable('test_df');\n",
              "      }\n",
              "      })();\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "dataframe",
              "variable_name": "test_df",
              "summary": "{\n  \"name\": \"test_df\",\n  \"rows\": 5,\n  \"fields\": [\n    {\n      \"column\": \"question\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"What is the role of liquid-liquid phase separation (LLPS) in the formation of PML/RAR\\u03b1-mediated microspeckles in acute promyelocytic leukemia (APL)?\",\n          \"How did the development of the TAA burden (TAB) algorithm help investigate the association between tumor-associated antigens and immune checkpoint blockade benefit?\",\n          \"How does the level of T-cell exhaustion affect the association between tumor-associated antigen burden (TAB) and immune checkpoint blockade (ICB) benefit?\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"contexts\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"ground_truth\",\n      \"properties\": {\n        \"dtype\": \"string\",\n        \"num_unique_values\": 4,\n        \"samples\": [\n          \"Liquid-liquid phase separation (LLPS) is a key mechanism in the formation of PML/RAR\\u03b1-mediated microspeckles in acute promyelocytic leukemia (APL). The intrinsically disordered region containing a large portion of PML and a smaller segment of RAR\\u03b1 facilitates this process.\",\n          \"The development of the TAA burden (TAB) algorithm based on known and putative tumor-associated antigens (TAAs) allowed the study to investigate the association between TAAs and immune checkpoint blockade (ICB) benefit. The algorithm was used to analyze patient cohorts, revealing that TAB correlated with ICB efficacy in tumors with low PD-L1 staining on immune cells and low T-cell exhaustion.\",\n          \"The level of T-cell exhaustion affects the association between tumor-associated antigen burden (TAB) and immune checkpoint blockade (ICB) benefit. In tumors with low T-cell exhaustion, TAB correlates with ICB efficacy. However, high levels of PD-L1 staining on immune cells, which is linked to T-cell exhaustion, weakens the association between TAB and ICB benefit.\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"evolution_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 2,\n        \"samples\": [\n          \"multi_context\",\n          \"simple\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"metadata\",\n      \"properties\": {\n        \"dtype\": \"object\",\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"episode_done\",\n      \"properties\": {\n        \"dtype\": \"boolean\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          true\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"
            }
          },
          "metadata": {},
          "execution_count": 52
        }
      ]
    },
    {
      "source": [
        "from google.colab import sheets\n",
        "sheet = sheets.InteractiveSheet(df=test_df)"
      ],
      "cell_type": "code",
      "execution_count": 53,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "https://docs.google.com/spreadsheets/d/18mP55CFzzr7Tmj-OcGsmCw5_nKT6yPjskDpasLesKqs#gid=0\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/usr/local/lib/python3.10/dist-packages/google/colab/sheets.py:31: FutureWarning: DataFrame.applymap has been deprecated. Use DataFrame.map instead.\n",
            "  return frame.applymap(_clean_val).replace({np.nan: None})\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.lib.display.IFrame at 0x7dc9c058a860>"
            ],
            "text/html": [
              "\n",
              "        <iframe\n",
              "            width=\"100%\"\n",
              "            height=\"600\"\n",
              "            src=\"https://docs.google.com/spreadsheets/d/18mP55CFzzr7Tmj-OcGsmCw5_nKT6yPjskDpasLesKqs#gid=0/edit?rm=embedded?usp=sharing?widget=true&amp;headers=false\"\n",
              "            frameborder=\"0\"\n",
              "            allowfullscreen\n",
              "            \n",
              "        ></iframe>\n",
              "        "
            ]
          },
          "metadata": {}
        }
      ],
      "metadata": {
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 673
        },
        "id": "UXN5BumNqZs9",
        "outputId": "8d4061c3-8e07-485a-c9cf-63327ae73f8c"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Ragas takes a novel approach to evaluation data generation. An ideal evaluation dataset should encompass various types of questions encountered in production, including questions of varying difficulty levels. LLMs by default are not good at creating diverse samples as it tends to follow common paths. Inspired by works like Evol-Instruct, Ragas achieves this by employing an evolutionary generation paradigm, where questions with different characteristics such as reasoning, conditioning, multi-context, and more are systematically crafted from the provided set of documents."
      ],
      "metadata": {
        "id": "8KXRBQ4fq2mJ"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "SJHyn5WnqYT-"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}