AA test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b2b57b54-a47a-43d7-aa7d-63379f45c73a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import pandahouse as ph\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import scipy.stats as st\n",
    "import matplotlib.pyplot as plt\n",
    "from typing import List"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b39bbe81-4048-4468-af35-92d6b733975e",
   "metadata": {},
   "outputs": [],
   "source": [
    " connection = {\n",
    "    'host': 'https://clickhouse.lab.karpov.courses',\n",
    "    'password': 'dpo_python_2020',\n",
    "    'user': 'student',\n",
    "    'database': 'simulator_20221020'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "31cf6d7f-049c-4bd2-b27a-79dea272b577",
   "metadata": {},
   "outputs": [],
   "source": [
    " q = \"\"\"\n",
    "    select count(distinct user_id)\n",
    "    from {db}.feed_actions \n",
    "    where toDate(time) >= '2022-09-26' and toDate(time) <= '2022-10-02' \n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "27f76694-faab-43bf-af38-27fb692b854f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>uniqExact(user_id)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>42585</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   uniqExact(user_id)\n",
       "0               42585"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # кол-во пользоватей для A/A теста\n",
    "ph.read_clickhouse(q, connection=connection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "2e115bbc-9677-4730-b568-7f8900072662",
   "metadata": {},
   "outputs": [],
   "source": [
    " q = \"\"\"\n",
    "    select\n",
    "      user_id,\n",
    "      exp_group, \n",
    "      countIf(action='like') as likes,\n",
    "      countIf(action='view') as views, \n",
    "      likes / views as ctr\n",
    "    from {db}.feed_actions\n",
    "    where toDate(time) >= '2022-09-26' and toDate(time) <= '2022-10-02'\n",
    "      and exp_group in (2, 3)\n",
    "    group by 2, 1\n",
    "    \"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "20c9ebcf-47e7-46a1-935d-0e1778872981",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_aa = ph.read_clickhouse(q, connection=connection)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "56254396-2e1d-4169-bb7e-c47c27bc44fa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>exp_group</th>\n",
       "      <th>likes</th>\n",
       "      <th>views</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>115383</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>36</td>\n",
       "      <td>0.305556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>123580</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>92</td>\n",
       "      <td>0.163043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>131473</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>148</td>\n",
       "      <td>0.175676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>23985</td>\n",
       "      <td>2</td>\n",
       "      <td>12</td>\n",
       "      <td>66</td>\n",
       "      <td>0.181818</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>24523</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>120</td>\n",
       "      <td>0.141667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17044</th>\n",
       "      <td>122008</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>31</td>\n",
       "      <td>0.129032</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17045</th>\n",
       "      <td>25383</td>\n",
       "      <td>2</td>\n",
       "      <td>16</td>\n",
       "      <td>57</td>\n",
       "      <td>0.280702</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17046</th>\n",
       "      <td>122190</td>\n",
       "      <td>3</td>\n",
       "      <td>25</td>\n",
       "      <td>74</td>\n",
       "      <td>0.337838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17047</th>\n",
       "      <td>108927</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>33</td>\n",
       "      <td>0.242424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17048</th>\n",
       "      <td>17310</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>0.142857</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>17049 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       user_id  exp_group  likes  views       ctr\n",
       "0       115383          3     11     36  0.305556\n",
       "1       123580          3     15     92  0.163043\n",
       "2       131473          2     26    148  0.175676\n",
       "3        23985          2     12     66  0.181818\n",
       "4        24523          2     17    120  0.141667\n",
       "...        ...        ...    ...    ...       ...\n",
       "17044   122008          3      4     31  0.129032\n",
       "17045    25383          2     16     57  0.280702\n",
       "17046   122190          3     25     74  0.337838\n",
       "17047   108927          2      8     33  0.242424\n",
       "17048    17310          2      2     14  0.142857\n",
       "\n",
       "[17049 rows x 5 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # датафрейм для A/A теста с ctr для каждого пользователя\n",
    "df_aa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "b13c5f49-adbb-4a3a-a027-815587e9f6d7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>likes</th>\n",
       "      <th>views</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>exp_group</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8480</td>\n",
       "      <td>8480</td>\n",
       "      <td>8480</td>\n",
       "      <td>8480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>8569</td>\n",
       "      <td>8569</td>\n",
       "      <td>8569</td>\n",
       "      <td>8569</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           user_id  likes  views   ctr\n",
       "exp_group                             \n",
       "2             8480   8480   8480  8480\n",
       "3             8569   8569   8569  8569"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " # проверим группы на размеры\n",
    "df_aa\\\n",
    "    .groupby('exp_group')\\\n",
    "    .count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "37f83570-cfe5-46bb-8d82-9019e7ec80f9",
   "metadata": {},
   "outputs": [],
   "source": [
    " # Установим размер графиков\n",
    "sns.set(rc={'figure.figsize':(10, 10)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a4b10ac8-6319-430f-b481-777b6986b886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAJSCAYAAACcOYKBAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAA1nUlEQVR4nO3de3RU5b3/8c+eWxIIMRACSbhjrVKltTYWldIiNIcoBKyXSqm2Sqlt1XNszylKvYDWqgWsWqsc9dfWHo9devR4q2gjVpZLYykVq1IKUstFREKAQFAgk8nM3r8/PKRsZiaZwMzeTzLv11quRb57z8w3j5PkM/vyPJbjOI4AAABgpIDfDQAAACA9whoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMMIaAACAwQhrAAAABgv53UCu7dmzX7adu6nkysqK1dy8L2fP35MxNukxNukxNukxNukxNukxNumZNDaBgKX+/fum3Nbrw5ptOzkNawdfA6kxNukxNukxNukxNukxNukxNun1hLHhNCgAAIDBCGsAAAAG6/WnQQEAgPcSibj27NmpeDzmdytp7dgRkG3bnr5mKBRR//7lCgYzj2CENQAAkHV79uxUYWEf9e1bIcuy/G4npVAooHjcu7DmOI727/9Qe/bs1MCBlRk/jtOgAAAg6+LxmPr2LTE2qPnBsiz17VvS7aONhDUAAJATBLVkRzImhDUAAACDEdYAAAAMRlgDAAAwGGENAADkvXg87ncLaTF1BwAA8MyuXTt1552L9Pbbb6qoqI+++tVZmjLlLH3zm1/Tf/zHPH3hC1/UgQMHdOmls3TJJXN01lnTdMstNyoSieiDD7bqb39bo+OPP0HXX3+TKio6n/7iz3/+k+68c5F2725WTc1Z2rx5o6ZMOVt1defo+eef1bPPPq0xYz6l+vrndc4552nWrG/ozjsXaeXKP6qgoFB1defoG9+YrUAgoF/96n598MFWzZ9/sySpsXGbLrhgul5++U8KhUK68srLdNJJn9Ybb/xZ7733nk455XO69toFKik55qjHjCNrAADAE7Zt6+qrf6BPfOKTeuqp3+uuu5boscce0bp1a/WjH83XokW3aM+e3frFL+7Qccd9UmedNa3jscuW1euSS+bouef+oE984pO66abrO32tlpYWXX/9Nfrud6/Uc8+9pOHDR+ivf33btc/atWtUVTVEv/vdC/rmN2frzjsXaf/+fXrssWd0zz0PqL7+eT3//O8y/v7q65/Tj340X888U69gMKi77rq9ewOUBmENAAB4Yt26tWpp2aNLL/22wuGwhgwZqunTz9FLLy3T5z9/ms48c7Kuuup7WrHiNc2de63rsWecMV4nn3yKIpGILrvscv3tb39VU9P2tK+1YkWDRo0arS99aZJCoZAuuGCmysoGuvYZOLBc558/U6FQSKFQWC+9tEzf+c6V6tOnryorqzRz5tdVX/98xt/flClna/ToT6ioqEhz5nxPy5e/qEQi0b1BSoHToAAAwBPbtzequXmXamsndtQSCVuf+czJkqTp08/VE088posvvlTHHFPqeuygQRUd/+7Tp49KSkq0a9cuDR5coVR27dqlQYMGd3xtWZbKywcd9pz/3L53b4vi8bjr1GpFRaV27dqZ8fd36PNVVFQqHo9r794WDRhQlvFzpEJYAwAAnhg8eLAqK6v06KNPJW1LJBJatOgW1dZO1VNP/a+mTp2uoUOHdWzfseOfR9EOHDigDz/8UAMHDkx6noMGDhyo117b0fG14zjauXOHa59DJ6g95phShUIhbd/eqFGjRkuSmpq2a+DAcklSUVGRotFox/7Nzc1Jr7ljR1PHv5uatisUCiWFziPBaVAAAOCJMWNOVJ8+ffTww79RW1tUiURCGzf+Q+vW/U3//d8PyrIs/ehH8zVr1sX6yU8WuE4hrljxR7399ltqb2/XL3/5nzrxxJPSHlWTpNNP/4I2bPiHXnnlZcXjcT355GNqbt6Vdv9gMKhJk2r0wANLdODAfm3f3qj/+Z/fasqUsyVJxx33Sb399l+0fft27du3Tw8//GDScyxb9ntt2rRR0WhUv/zlfZo4cbKCweBRjNjHCGsAAMATwWBQixbdpXff/bsuuGCGpk79sn7605/ojTde16OP/lbXX3+TgsGgvv71b8qypIcf/k3HY2tqpujBBx/Q2WdP0vr17+iGG27u9LVKS0t1880/1ZIld2vq1MnatGmTTjjhUwqHw2kf8/3vz1VRUZG++tUZuvzyOaqpqdXUqdMlSaeeepomTfoXXXLJTH3rWxfpjDMmJD1+ypSzdeutN2r69CmKxWL6/vfnHtlAHcZyHMfJyjMZqrl5n2w7d99ieXk/7dz5Uc6evydjbNJjbNJjbNJjbNJjbNLza2y2b39PFRUjsvJct9xyo8rLB+myyy4/4uewbVtf+crZWrDgJzrllGpJUigUUDxuZ6XHK6+8rGNakK6kGptAwFJZWXHK/TmyBgAAeqWVK1foo48+UiwW00MP/VqSoxNPPMnvtrqNGwwAAECP9Pbbb+qHP/y3lNtefPFVrVmzWjfddL3a29s1cuQo3Xrrz1RQUOhxl0eP06BHiUPv6TE26TE26TE26TE26TE26fWG06C5ks3ToN3BaVAAAIBehLAGAABgMK5ZA3KsMGLLcdpctbbWuAojAUVjfF4CAHSOsAbkmOO0qWnLSletb58CFQ88WVKRLz0BAHoOwhoAADBS//5FCoWyH1Xi8bj27GnN+vPmCmENAAAYKRQK6d3Vz2b9eY/7dF2X++zd26Kbb56vDz7YqnA4rKFDh2vu3GvVv3//rPfTFS6YAQAAOIxlWZo16xt65JEn9dBD/6MhQ4bqvvt+4UsvhDUAAIDDlJQc07EslSSdeOJJ2r59uy+9ENYAAAA6Ydu2nnrqCX3hC1/05fUJawAAAJ24887F6tOnSOed91VfXp8bDAAAANK45567tHXrFi1ceKcCAX+OcRHWAAAAUrj//nu1fv06LV78c0UiEd/6IKwBAAAjxePxjKbZOJLn7crGjRv03//9oIYNG67vfne2JKmyskq33XZ71vvpCmENAAAYyc+Ja0ePPlYNDat8e/1DcYMBAACAwQhrAAAABiOsAQAAGIywBgAAYDDCGgAAgMG4GxTINctS3HZcpf3RdhVblk8NAQB6EsIakGOJhKMNW1tctXA4pIFVTuoHAAAkSceU9lEkHMz688baE9rbcqDL/X70o//Qtm3bFAhYKirqox/8YK6OO+74rPfTFcIaAAAwUiQc1P978u2sP++3z/1MRvtdd91NKi4uliS9+urLuu22H+vXv/5t1vvpCtesAQAApHAwqEnSvn37ZFmsDQoAAGCUn/70Zv35z3+SJN1++92+9MCRNQAAgDTmzbtBTz75nC677HItWfJzX3ogrAEAAHShtnaq/vKXN7R3b4vnr01YAwAAOMyBAwfU1LS94+uGhldUUlKikpJjPO+Fa9YAAAAOE4226oYb5ikabVUgEFRJSYkWLrxTlg9zZBLWAACAkWLtiYyn2eju83ZlwIAyPfDAb7L+2keCsAYAAIyUycS1+YBr1gAAAAxGWAMAADAYYQ0AAOSE47AG8uGOZEwIawAAIOtCoYj27/+QwHYIx3G0f/+HCoUi3XocNxgAAICs69+/XHv27NS+fS1+t5JWIBCQbduevmYoFFH//uXde0yOegEAAHksGAxp4MBKv9voVHl5P+3c+ZHfbXSJ06AAAAAGI6wBAAAYjLAGAABgMMIaAACAwQhrAAAABiOsAQAAGIywBgAAYDDCGgAAgMEIawAAAAYjrAEAABiMsAYAAGAwwhoAAIDBCGsAAAAGI6wBAAAYLOR3A0CPELAUjSVcpcJIULIdnxoCAOQLwhqQgWgsofoVm1212tNHqjDEwWkAQG7xlwYAAMBgnoe1e+65R8cff7z+/ve/S5LeeustTZ8+XVOmTNHs2bPV3NzcsW9n2wAAAPKBp2Htb3/7m9566y0NGTJEkmTbtubOnav58+frhRdeUHV1tW6//fYutwEAAOQLz8JaLBbTj3/8Y914440dtTVr1qigoEDV1dWSpJkzZ6q+vr7LbQAAAPnCsxsMfv7zn2v69OkaOnRoR62xsVFVVVUdXw8YMEC2baulpaXTbaWlpRm/bllZcVb670x5eb+cv0ZP1VvGZseeA+rbt8BVKygMq7x/ny4fu3dvVOFw8o9aIGipfEDvGJ9s6y3vm1xgbNJjbNJjbNLrCWPjSVh78803tWbNGv3whz/04uVcmpv3yc7h9Arl5f20c+dHOXv+nqw3jU1b3Nb+/W3uWrQ9o+8vFHTU3h531cLhkOyE02vGJ5t60/sm2xib9Bib9Bib9Ewam0DASnuAyZOw9vrrr2vDhg2aPHmyJGn79u361re+pYsvvljbtm3r2G/37t0KBAIqLS1VZWVl2m0AAAD5wpNr1i677DI1NDRo+fLlWr58uSoqKvSrX/1Kc+bMUTQa1apVqyRJjz76qGprayVJJ510UtptAAAA+cLXSXEDgYAWLVqkBQsWqK2tTUOGDNHixYu73AYAAJAvfAlry5cv7/j3KaecomeffTblfp1tAwAAyAesYAAAAGAwwhoAAIDBWMgdyEBxkaPxY/sl1eLtPjUEAMgbhDUgE3ZMG9c3uEqVZbWSivzpBwCQNzgNCgAAYDDCGgAAgMEIawAAAAYjrAEAABiMsAYAAGAw7gYFsilgKRpLuErF/JQBAI4Cf0aAIxQISNE22120LNWv2OwqnTex0rumAAC9DmENOEIJW0nBrOa0kb70AgDovbhmDQAAwGCENQAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAAACDMXUH8lZhxJbjtLlqllWgaIzPMAAAcxDWkLccp01NW1a6aoOHj5NU5E9DAACkQFgDjlAkHNT4sf1ctZIix6duAAC9FWENOEKOHdPG9Q2uWkVZrU/dAAB6Ky7OAQAAMBhH1oBDhEJBSa1JdcvyvhcAACTCGuBiJ2Jq2rIqqV4xotqHbgAA4DQoAACA0TiyBhzKshS3uaMTAGAOwhpwCMdxtGFrS1J96CjvewEAQOI0KAAAgNEIawAAAAYjrAEAABiMsAYAAGAwwhoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMMIaAACAwVjBADBJwFI0lnCVCiNBiSWwACBvEdaALIqEgxo/tp+rFg5m/vhoLKH6FZtdtdrTR6owxEFwAMhXhDUgixw7po3rG1y1YYNqfOoGANAb8HEdAADAYIQ1AAAAgxHWAAAADEZYAwAAMBhhDQAAwGCENQAAAIMR1gAAAAxGWAMAADAYk+ICPgkEpGib7S5alj/NAACMRVgDfJKwlbS0VM1pI33pBQBgLsIa0IsURmw5TpurZlkFisa44gEAeirCGtCLOE6bmrasdNUGDx8nqcifhgAAR42P2wAAAAYjrAEAABiMsAYAAGAwwhoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMOZZQ/6yLMVtx+8uAADoFGENeSuRcLRha4urNnSUP70AAJAOp0EBAAAMRlgDAAAwGGENAADAYFyzBvgkEg5q/Nh+rlpJETc8AADcCGuATxw7po3rG1y1irJan7oBAJiK06AAAAAGI6wBAAAYjLAGAABgMMIaAACAwQhrAAAABiOsAQAAGIywBgAAYDDCGgAAgMEIawAAAAYjrAEAABiM5aaA3sSyFLedpBoAoOcirAG9SCLhaMPWFletYjiLwwNAT8ZpUAAAAINxZA0wXHGRo6Bak+qWVaBojM9bANDbEdYA09kxNW1dmVQePHycpCLv+wEAeIqP5QAAAAYjrAEAABiMsAYAAGAwrlkDDBIJBzV+bD9XLRT0qRkAgBEIa4BBHDumjesbXLVhg2p86gYAYALCGmA4S0pelUBiZQIAyBOENcBwjpS0KoHEygQAkC+4wQAAAMBghDUAAACDEdYAAAAMxjVryA8BS9FYwlUq5t0PAOgB+HOFvBCNJVS/YrOrdt7ESn+aAQCgGzgNCgAAYDDCGgAAgMEIawAAAAYjrAEAABiMsAYAAGAwwhoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMMIaAACAwQhrAAAABiOsAQAAGIywBgAAYLCQVy90+eWXa+vWrQoEAurTp49uuOEGjRkzRps2bdK8efPU0tKi0tJSLVy4UCNHjpSkTrcBAADkA8+OrC1cuFC/+93v9PTTT2v27Nm69tprJUkLFizQrFmz9MILL2jWrFmaP39+x2M62wYAAJAPPAtr/fr16/j3vn37ZFmWmpubtXbtWk2bNk2SNG3aNK1du1a7d+/udBsAAEC+8Ow0qCRdd911eu211+Q4jn75y1+qsbFRgwcPVjAYlCQFg0ENGjRIjY2Nchwn7bYBAwZ42TYAAIBvPA1rt9xyiyTp6aef1qJFi3TVVVfl/DXLyopz/hrl5f263ilPmTI2O/YcUN++Ba6aZUnhcPKPQKpauvrR1I728YGgpfIB7vHduzeatG+q/UxnyvvGRIxNeoxNeoxNej1hbDwNawedc845mj9/vioqKtTU1KREIqFgMKhEIqEdO3aosrJSjuOk3dYdzc37ZNtOjr6Tj/8n79z5Uc6evyczaWza4rb2729z1RxHam+PJ+2bqpaufqS1g4HqaJ7TTjhJ4xsKOkn7ptrPZCa9b0zD2KTH2KTH2KRn0tgEAlbaA0yeXLO2f/9+NTY2dny9fPlyHXPMMSorK9OYMWO0dOlSSdLSpUs1ZswYDRgwoNNtAAAA+cKTI2utra266qqr1NraqkAgoGOOOUb33XefLMvSjTfeqHnz5mnJkiUqKSnRwoULOx7X2TYAAIB84ElYGzhwoB577LGU24499lg9/vjj3d4GdEdxkaPxY93XJYSDPjUDAEA3+HLNGuA5O6aN6xtcpWGDanxqBgCAzLHcFAAAgMEIawAAAAYjrAEAABiMsAYAAGAwwhoAAIDBuBsUvUvAUjSWSCoX804HAPRQ/AlDrxKNJVS/YnNS/byJ3VumDAAAU3AaFAAAwGCENQAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAAACDEdYAAAAMRlgDAAAwGGENAADAYIQ1AAAAgxHWAAAADEZYAwAAMBhhDQAAwGAhvxsAsqm4yNH4sf2S6uGgD80AAJAFhDX0LnZMG9c3JJWHDarxoRkAAI4ep0EBAAAMRlgDAAAwGGENAADAYIQ1AAAAgxHWAAAADMbdoEAPFQ4HFQy0umqW5VMzAICcIawBPZRjx9S0ZZWrVjGi2qduAAC5wmlQAAAAgxHWAAAADMZpUCAfBSxFY4mkcmEkKNmODw0BANIhrAF5KBpLqH7F5qR67ekjVRjigDsAmITfygAAAAYjrAEAABiMsAYAAGAwwhoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMMIaAACAwQhrAAAABiOsAQAAGIywBgAAYDDCGgAAgMEIawAAAAYL+d0AgNwKBKRom+0uWpY/zQAAuo2wBvRyCVuqX7HZVTvvzBEaP7Zf0r7FRY7i7R41BgDICGENyEdOTBvXNySVK8tqJRV53w8AIC2uWQMAADAYYQ0AAMBghDUAAACDEdYAAAAMRlgDAAAwWMZh7fe//33Ken19fdaaAQAAgFvGYe26665LWZ8/f37WmgGQOUtS3HZc/wEAep8u51l7//33JUmO43T8+9BtkUgkN50B6JQjacPWFldt6ChfWgEA5FCXYa2mpkaWZclxHNXU1Li2DRw4UP/6r/+as+YAAADyXZdh7Z133pEkXXTRRXr44Ydz3hAAwwQsRWMJV6kwEpQ47QoAnsh4uSmCGpCforFE0tqitaePVGGIm8kBwAsZh7X3339fd911l9atW6cDBw64tr388svZ7gsAAADqRlj74Q9/qGHDhumaa65RURELPQMAAHgh47D27rvv6pFHHlEgwKkPAAAAr2ScvE499VStXbs2l70AAADgMBkfWRsyZIjmzJmjmpoaDRw40LXtqquuynpjAAAA6EZYa21t1Zlnnql4PK7t27fnsicAAAD8n4zD2m233ZbLPgAAAJBCt6buSGfYsGFZaQYAAABuGYe1Q5edOsiyLEnSunXrst8ZAAAAMg9rB5edOmjnzp265557VF1dnfWmAAAA8LEjnjStvLxc1113ne64445s9gMAAIBDZHxkLZWNGzeqtbU1W70AyIFIOKjxY/u5auGgT80AALot47A2a9asjmvUpI+n8vjHP/6hK664IieNAcgOx45p4/oGV23YoBqfugEAdFfGYe2CCy5wfV1UVKQTTjhBI0eOzHZPAAAA+D8Zh7WvfOUruewDAAAAKWR8g0F7e7vuvvtuTZ48WWPHjtXkyZN19913KxaL5bI/AACAvJbxkbXFixdr9erVuummm1RVVaVt27ZpyZIl2rdvn6699tpc9ggAAJC3Mg5r9fX1euaZZ9S/f39J0ujRo/WpT31KM2bMIKwBAADkSManQQ9duSCTOgAAAI5exmGttrZW3/ve9/Tqq69qw4YNeuWVV3TFFVeotrY2l/0BAADktYxPg86dO1f/+Z//qR//+MfasWOHBg8erKlTp+p73/teLvsDAADIa10eWXvjjTe0ePFiRSIRXXXVVXrxxRf19ttva9myZYrFYlq7dq0XfQIAAOSlLsPa/fffr1NPPTXltnHjxum+++7LelMAAAD4WJdhbd26dZowYULKbWeccYbWrFmT9aYAAADwsS7D2r59+9Te3p5yWzwe1/79+7PeFAAAAD7WZVgbPXq0GhoaUm5raGjQ6NGjs94UAAAAPtZlWLvkkku0YMECLVu2TLZtS5Js29ayZct044036tJLL815kwAAAPmqy6k76urqtGvXLl1zzTVqb29XaWmpWlpaFA6H9W//9m+aNm2aF30CAADkpYzmWbv00kt1wQUX6M0331RLS4tKS0v12c9+VsXFxbnuD0gvYCkaS7hKxRnPHAgAQM+Q8Z+24uLitHeFAn6IxhKqX7HZVTtvYqU/zQAAkCMZLzcFAAAA73HSCECnioscjR/bL6kWTz2jDwAgywhrADpnx7RxvXv6nsqyWklF/vQDAHmG06AAAAAGI6wBAAAYjLAGAABgMMIaAACAwbjBAECHQECKttmuGhMNA4C/+DWMHivVlBLhoE/N9BIJW0w0DACGIayh50oxpcSwQTU+NQMAQG5wzRoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMG4wANAhEg5yhy0AGIawBqCDwx22AGAcT06D7tmzR9/+9rc1ZcoU1dXV6corr9Tu3bslSW+99ZamT5+uKVOmaPbs2Wpubu54XGfbAAAA8oEnYc2yLM2ZM0cvvPCCnn32WQ0bNky33367bNvW3LlzNX/+fL3wwguqrq7W7bffLkmdbgMAAMgXnoS10tJSjRs3ruPrk08+Wdu2bdOaNWtUUFCg6upqSdLMmTNVX18vSZ1uAwAAyBee3w1q27YeeeQRTZo0SY2NjaqqqurYNmDAANm2rZaWlk63AQAA5AvPbzC4+eab1adPH1100UV68cUXc/56ZWXFOX+N8vJ+Xe+Up3I5Nnv3RhUOJ7+FU9XS1b16fG/rKRC0VD4gd/9v+ZlKj7FJj7FJj7FJryeMjadhbeHChXrvvfd03333KRAIqLKyUtu2bevYvnv3bgUCAZWWlna6rTuam/fJtp1sfQtJysv7aefOj3L2/D1ZNsemMGLLcdpctYAltbfHk/ZNVUtX9+rxh9cOhh+TeurOvnbCydn7np+p9Bib9Bib9Bib9Ewam0DASnuAybPToHfccYfWrFmje++9V5FIRJJ00kknKRqNatWqVZKkRx99VLW1tV1uQ/5xnDY1bVnp+k9K+N0WAAA558mRtXfffVf333+/Ro4cqZkzZ0qShg4dqnvvvVeLFi3SggUL1NbWpiFDhmjx4sWSpEAgkHYbAABAvvAkrB133HFav359ym2nnHKKnn322W5vAwAAyAesDQoAAGAwwhoAAIDBWBsUQLeFw0EFA61JdcsqUDTGZ0AAyCbCGoBuc+yYmrasSqoPHj5OUpH3DQFAL8ZHYAAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAAACDEdYAAAAMRlgDAAAwGPOswTiFEVuO0+aqBflYAQDIU4Q1GMdx2tS0ZaWrVjmi2qduAADwF8crAAAADEZYAwAAMBhhDQAAwGBcs4aewbIUtx2/uwAAwHOENfQIjuNow9YWV23oKH96AQDAS5wGBQAAMBhhDQAAwGCENQAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAAACDMc8agOyxLEXjtqtUGAlKTGgMAEeMsAYgaxIJR/UrNrtqtaePVGGIg/gAcKT4DQoAAGAwjqwByJpwOKjxY/u5asVFjuLtPjUEAL0AYQ1A1jh2TBvXN7hqlWW1kor8aQgAegFOgwIAABiMsAYAAGAwToMCyKlwOKhgoNVVs6wCRWN8VgSATBDWAOSUY8fUtGWVqzZ4+DhxHRsAZIawBiCnLEnxwyfFtSxfegGAnoiwBiCnHEkbtra4ahXDWdEAADJFWAPQbSmPlgEAcoKwBqDbUh0tk6ShozxvBQB6PW7HAgAAMBhhDQAAwGCENQAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAAACDEdYAAAAMRlgDAAAwGGENAADAYIQ1AAAAg7E2KIBerTBiy3HaXDXLKlA0xmdVAD0DYQ1Ar+Y4bWrastJVGzx8nKQifxoCgG4irME8lqW47fjdBQAARiCswTiJhKMNW1tctaGj/OkFAAC/cdEGAACAwQhrAAAABuM0KIDeLdU1kJblTy8AcAQIawA8Fw4HFQy0umq5mk4j1TWQFcO5gQVAz0FYA+A5x46pacsqV43pNAAgNa5ZAwAAMBhhDQAAwGCENQAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAAACDEdYAAAAMxqS4ADxnSSwBBQAZIqwB8JwjsQQUAGSIsAag5wlYisYSrlJhJCgdfrQOAHoBwhqAHicaS6h+xWZXrfb0kSoMcRkugN6H32wAAAAGI6wBAAAYjLAGAABgMK5ZA2CESCQoxVq1d29UoeA/bxQIhQoUbeNzJYD8RVgDYAQ7EdNrf3xR4XBI7e3xjvr4M2olFfnXGAD4jI+rAAAABiOsAQAAGIywBgAAYDDCGgAAgMEIawAAAAYjrAEAABiMsAYAAGAwwhoAAIDBCGsAAAAGI6wBAAAYjLAGAABgMMIaAACAwQhrAAAABiOsAQAAGIywBgAAYLCQ3w0gjwUsRWOJpHIx70oAADrwZxG+icYSql+xOal+3sRK75sBAMBQnAYFAAAwGGENAADAYIQ1AAAAgxHWAAAADEZYAwAAMBh3g8IbqabpsCx/ekGPV1zkaPzYfkm1eLtPDQFADhHW4IlU03TUnDbSl17QswQCUrTNdtWKQzFtXN/gqlWW1Uoq8rAzAPAGYQ2A0RK2koI+c/EByCeENXgi1WmrkiLHp24AAOg5CGvwhp182qqirNanZgAA6Dm4GxQAAMBghDUAAACDEdYAAAAMRlgDAAAwGGENAADAYJ6EtYULF2rSpEk6/vjj9fe//72jvmnTJl144YWaMmWKLrzwQm3evDmjbQAAAPnCk7A2efJk/fa3v9WQIUNc9QULFmjWrFl64YUXNGvWLM2fPz+jbQAAAPnCk7BWXV2tykr3jOPNzc1au3atpk2bJkmaNm2a1q5dq927d3e6DQAAIJ/4NiluY2OjBg8erGAwKEkKBoMaNGiQGhsb5ThO2m0DBgzo1uuUlRVnvffDlZf363qnPHVwbPbujSocdr/dLEvq27cg6TGWpaR9paOr+f14eupe7dBtqd4nqd4jgaCl8gHJP4up3nvp9u0J+H2THmOTHmOTXk8Ym16/gkFz8z7Zdu6WNSov76edOz/K2fP3ZIeOTSjoqL097truONL+/W1Jj3McJe0rHV3N78cfXjsYHkzqyYTHt7fHFQ6HXNtSvU9SvUfshJPyZzHVey/dvqbj9016jE16jE16Jo1NIGClPcDkW1irrKxUU1OTEomEgsGgEomEduzYocrKSjmOk3YbAABAPvFt6o6ysjKNGTNGS5culSQtXbpUY8aM0YABAzrdBgAAkE88ObL2k5/8RMuWLdOuXbt06aWXqrS0VM8995xuvPFGzZs3T0uWLFFJSYkWLlzY8ZjOtgEAAOQLT8La9ddfr+uvvz6pfuyxx+rxxx9P+ZjOtgEAAOQLVjAAAAAwGGENAADAYIQ1AAAAgxHWAAAADNbrJ8UF0LNFwkGNH+ueYTwc9KkZAPABYQ2A0Rw7po3rG1y1YYNqfOoGALzHaVAAAACDcWQNQK8QCEjRNjupXsxvOQA9HL/GAPQKCVuqX7E5qX7eRNYUBtCzcRoUAADAYIQ1AAAAg3EaFADSKIzYcpy2pLplFSga47MuAG8Q1gAgDcdpU9OWlUn1wcPHSSryviEAeYmPhgAAAAbjyBp8k2pmeonZ6XFkeD8B6K0Ia/BNqpnpJWanx5Hh/QSgt+I0KAAAgMEIawAAAAYjrAEAABiMa9aQdQfnpmprjasg3C5JsiyfmwIAoIcirCHrDs5N1bdPgfYf+HhC0YoR1T53BQBAz8RpUAAAAIMR1gAAAAxGWAMAADAYYQ0AAMBg3GAAAJIUsBSNJVylgoiluO0k78vtzQA8RFgDAEnRWEL1Kza7audNrNSGrS1J+1YMTxHgACBHOA0KAABgMMIaAACAwQhrAAAABiOsAQAAGIywBgAAYDDCGgAAgMEIawAAAAYjrAEAABiMSXGRfdbHs77vj7annv0dAABkjLCGrEskHG3Y2qJwOKT29rgkaegon5sCAKCH4jQoAACAwQhrAAAABiOsAQAAGIywBgAAYDDCGgAAgMEIawAAAAYjrAEAABiMedZwxAojthynLaluWT40AwBAL0VYwxFznDY1bVmZVK8YUe1DN8DRKS5yNH5sP1ctHPSpGQA4BGENACTJjmnj+gZXadigGp+aAYB/4po1AAAAgxHWAAAADEZYAwAAMBhhDQAAwGDcYICMpJqmIxiQ4rbjU0cAAOQHwhoykmqajooR1dqwtSVp36GjPGoKOELhcFDBQKurxvyAAExFWAOQdxw7pqYtq1w15gcEYCquWQMAADAYYQ0AAMBgnAYFgBxJv35ugaIxPisDyAxhDQByJN36uYOHj5NU5H1DAHokPtoBAAAYjLAGAABgMMIaAACAwQhrAAAABuMGAwDwWCgUVMJuPaxWoGgbn58BJCOsAUA2BCxFYwlXqSCSeg2rRCKm1/74oqs2/oxacYcogFQIawDQTanWFk0oovoV77lq50+sPKrnZD42ABJhDZmyLMVtx+8uADPYMX2w+XVXadix4zV+bD9XLRTM/ClTrVfKfGwAJMIaMpRIONqwtcVVGzrKn14AvzlSip+HmDaub3DVhg2qyfg5LSnpA1EoFJT0z6Ntba1xFUYCHG0D8gxhDQAMkCoADhvlPoIXCgU1eNjnJRV62hsAfxHWAMBQhwe4cDikgVVcjgDkG8LaUYq3H1BBuAdcFJziTjVJKowEpcOvRUuxbzHvFPQiqU45AoCp+BN8lBLx5IWaTbwoOBpLqH7F5qR67ekjVRgKdLnved24qw0wXeprznxpBQC6ZNjhHwAAAByKI2sA0EsVRmw5TpurZuRlGgA6RVgDgBzx+9o4x+kZl2kA6BxhDQByJNW1cRLXx2VdwNKOPQfUFrc7SilvngJ6KMIaAKBHi8YSenX1Vu3f/89TvqlungJ6KsIaAPQggYAUbbOT6hxJAnovwtpRisft5GtSLMufZgD0eglbGU/DA6B3IKwdpbhtJ12TUjGcT7cAvFVc5Cgo9wTdQbIb0CsQ1pCkuMjR+LH9XLVw0KdmAGTGjqlpq/vOz8oR1T41AyCbCGt5rqSPZCfcn8ZDlrRxfYOrNmxQjZdtAUDGioscVR9XpHgi4qrF231sCsgiwlqeSyTa9Mc/1rtqX5pAMAPQg9gxbXr3NbW3xztKlWW1Yj459BZc0QAAAGAwjqz1QqmWmCkqCiddhyZxLRoAAKYjrPVCqZaYqRhRnXQdmsS1aEBPEwkHU37wCqX64GVZRze1UMBSNJZwlZjPDfAeYQ0AehDHjqX84DV8UE1SMHMc56imForGEklzujGfG+A9whoA9AKp1iFlDVKgdyCsAQByg9OoQFYQ1gAAOcFpVCA7CGu9UaqLigFALAQP9ESEtV4okUi+qJhrVwBILAQP9ET8ZAIAABiMI2sAgJSKi5ykOd1YcxPwHmENAJBaijndWHMT8B5hDQDySLoVEEr6SB8ecN94UJziL0SqGxS4OQHILcJaD1dYYCsed68DmnLZGQBQ+hUQKspqVb+i0VU7b2Jl0n6pblA4f9IIBYOxpH1D4cjRNQtAEmGtx4vH2/TaH+tdtS9NYL1PAB6yY2raujKpPHDoOB+aAXof7gYFAAAwGGENAADAYJwG7SlSrLEnpb4AGABMFQoFFI1zgwLQHfyp7yFSrbEnpb4AGAC8FAgo9RJ3VnKprd3Wi3/a7Kqlu0HBsgoUjXECCCCsAQBSTukRzvDO8oStpCXuJGnYscnPWVKUItSluUFh8PBxYk43gLAGAFDqKT2GDTq6O8tTPWdFWe1RPedRS3FJCadhYTrCmoEKI7Yc57C505ivCIABjuYIXDrpTqOGQkFJra7a0Z4aTXVJCYvYw3SENQM5TpuatrhPCTBfEQAT5OIIXLrTqENHxdS0ZZWrxqlR5CPCmoksK/lTZooLdQGgp8nFkbmj1a07VDmNCh8Q1gyUSDhJnzIHD/enFwDIplwcmTta3blDNRAKa+fuPa5acfkAxW3W+UPuGB/WNm3apHnz5qmlpUWlpaVauHChRo4c6Xdbnku3+LLfn0gBIFcspbiWzcr+aYaSIifp92vIimn7e8l3qFaMqE4Km0PKaxWNF7hqPeVoW6prpCWmTTGN8WFtwYIFmjVrlmbMmKFnnnlG8+fP10MPPeR3W55Lt/iy359IASBXHCVfy1YxPLMAFIkEpVhrUr1/UTj5NGwg+ffr8EE1qeeOSyHV4vbTJoxWPMunVgsLbMXjycEqFCpQtM0drA4NYW2tcRWE21MGsFTXSEuprw1MFex6Y6gz8fs0Oqw1Nzdr7dq1evDBByVJ06ZN080336zdu3drwIABGT1HIJDbi70sK6Cior6HvWZAASfD1w1Yamt3/5BGgsnPaVnBpFq6eqa1o318V88ZCoUUCsWN6smPx6eqhUIh43ry+/Gp3jem9JTN5zyax6d73/j9PXnZUzAUUGvcHWIiwYAKC/u63jeOndBf3nw16TnHnTpBjVvedNWGD56Q/PpWUNua25MeP2x0ck8FkbAmfra/q1ZSZGvbzr2u2jF9S7UvmvSUki01vP2BqzT1jKFybPdpWEtK+T2dNm6SCgvcvQYtadv7b3/872BQiURCQ0ZUK+K49wsFA7JChUnPaQWCyX8/rbh2fPC2qzRkRLUKLfdzWlZEHybnZJUUSY5z+PcUUVu7OwQVhG05SjFJcop9U0n1+M4em8n3OXjoKQoE3EdPs62zvGI5jmPscdo1a9bommuu0XPPPddRO/vss7V48WKdeOKJPnYGAADgjd517BIAAKCXMTqsVVZWqqmpSYnEx6cJE4mEduzYocpK1sMEAAD5weiwVlZWpjFjxmjp0qWSpKVLl2rMmDEZX68GAADQ0xl9zZokbdiwQfPmzdOHH36okpISLVy4UKNHj/a7LQAAAE8YH9YAAADymdGnQQEAAPIdYQ0AAMBghDUAAACDEdYAAAAMRljLwKZNm3ThhRdqypQpuvDCC7V58+akfRKJhG666SZ9+ctfVk1NjR5//HHvG/VBJmPT0NCgc889VyeddJIWLlzofZM+yWRs7r33Xk2dOlV1dXU699xz9eqryUvJ9EaZjM0TTzyhuro6zZgxQ3V1dXmzJnAmY3PQxo0b9ZnPfCZvfq4yGZtf/OIXOv300zVjxgzNmDFDN910k/eN+iDT983zzz+vuro6TZs2TXV1ddq1a5e3jfogk7G5+uqrO94zM2bM0AknnKCXXnrJ+2bTcdCliy++2Hn66acdx3Gcp59+2rn44ouT9nnqqaec2bNnO4lEwmlubnYmTJjgvP/++1636rlMxmbz5s3O2rVrnTvuuMP56U9/6nWLvslkbF555RXnwIEDjuM4zrp165zPfe5zTmtrq6d9+iGTsfnoo48c27Y7/j1x4kRn3bp1nvbph0zGxnEcJx6POxdddJHz7//+73nzc5XJ2Nx99915Mx6HymRsVq9e7Zx11lnOjh07HMdxnA8//NCJRqOe9umHTH+mDlq3bp3z+c9/3mlra/OivYxwZK0LBxeTnzZtmqSPF5Nfu3atdu/e7drv+eef1wUXXKBAIKABAwboy1/+surr6/1o2TOZjs2IESM0ZswYhUIhP9r0RaZjM2HCBBUVFUmSjj/+eDmOo5aWFq/b9VSmY1NcXCzL+nhh42g0qvb29o6ve6tMx0aSHnjgAU2cOFEjR470uEt/dGds8k2mY/Ob3/xGs2fPVnl5uSSpX79+KijI7eLkfjuS983//u//qq6uTpFIxKs2u0RY60JjY6MGDx6sYDAoSQoGgxo0aJAaGxuT9quqqur4urKyUtu3b/e0V69lOjb56EjG5umnn9bw4cNVUVHhVZu+6M7YvPTSS5o6darOPPNMzZkzR8cff7zX7Xoq07F555131NDQoEsuucSHLv3RnffNc889p7q6Os2ePVtvvvmm1616LtOx2bBhg95//319/etf11e+8hUtWbJETi+farW7v4tjsZieffZZnXfeeV622aX8OdQBGOzPf/6zfv7zn+vXv/61360YZfLkyZo8ebK2bdumK664Ql/84hfzfgWT9vZ23XDDDbrttts6/gDhn2bOnKnvfve7CofDeu2113T55Zfr+eefV//+/f1uzXeJRELr16/Xgw8+qFgspjlz5qiqqkrnnHOO360Z4w9/+IOqqqo0ZswYv1tx4chaFzJdTL6yslLbtm3r+LqxsbHXHyHJdGzyUXfG5s0339TcuXN177335kUQOZL3TVVVlcaOHauXX37Zoy79kcnY7Ny5U1u2bNFll12mSZMm6b/+67/02GOP6YYbbvCrbU9k+r4pLy9XOByWJI0fP16VlZV69913Pe/XS5mOTVVVlWpraxWJRFRcXKzJkydr9erVfrTsme7+vnniiSeMO6omEda6lOli8rW1tXr88cdl27Z2796tP/zhD5oyZYofLXsm07HJR5mOzerVq/WDH/xAd999t0488UQ/WvVcpmOzYcOGjn/v3r1bK1eu1Cc/+UlPe/VaJmNTVVWllStXavny5Vq+fLm++c1v6qtf/apuvvlmv9r2RKbvm6ampo5/r1u3Th988IFGjRrlaa9ey3Rspk2bpoaGBjmOo/b2dv3pT3/SCSec4EfLnunO36nt27frjTfeUF1dnddtds3nGxx6hH/84x/O+eef7/zLv/yLc/755zsbNmxwHMdx5syZ46xevdpxnI/vzJo/f74zefJkZ/Lkyc6jjz7qZ8ueyWRsXn/9dWfChAnOZz/7Wefkk092JkyY4Lzyyit+tu2JTMbm3HPPdcaNG+dMnz6947933nnHz7Y9kcnY3HLLLc7ZZ5/tTJ8+3amrq3MeeughP1v2TCZjc6h8uvsxk7G5+uqrnalTpzp1dXXOueee67z88st+tuyZTMYmkUg4t956q1NbW+ucffbZzq233uokEgk/2/ZEpj9TS5Yscb7//e/71WanWMgdAADAYJwGBQAAMBhhDQAAwGCENQAAAIMR1gAAAAxGWAMAADAYYQ0AAMBghDUAyNCTTz6pr33ta363ASDPENYAIIsOLmsDANnCpLgAkEJjY6NuueUWrVq1So7j6IQTTtBf/vIXxeNxFRYWKhgMatWqVZo3b54KCgq0bds2vf7661qyZInOOOMMv9sH0IuE/G4AAEyTSCT0ne98R6eddpqWL1+uYDCov/71r9qyZYsef/xxPfLII679ly5dqgceeED333+/2tvbfeoaQG/FaVAAOMzq1au1Y8cOXX311erTp48KCgpUXV2ddv/Jkyfrc5/7nAKBgAoKCjzsFEA+IKwBwGEaGxtVVVWlUCizkw+VlZU57ghAPiOsAcBhKisr1djYqHg87qpbluVTRwDyGWENAA7z6U9/WuXl5frZz36mAwcOqK2tTW+88YbKysrU1NSkWCzmd4sA8ghhDQAOEwwGdd999+m9997TmWeeqS9+8Yv6/e9/r9NOO02f+MQn9IUvfEHjxo3zu00AeYKpOwAAAAzGkTUAAACDEdYAAAAMRlgDAAAwGGENAADAYIQ1AAAAgxHWAAAADEZYAwAAMBhhDQAAwGCENQAAAIP9f6vz6nv5NXooAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    " groups = sns.histplot(data=df_aa,\n",
    "                      x         = 'ctr',\n",
    "                      hue       = 'exp_group',\n",
    "                      palette   = ['y', 'b'],\n",
    "                      kde       = False,\n",
    "                      alpha     = 0.5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f1c6c755",
   "metadata": {},
   "source": [
    "Расределение экспериментальных групп практически совпадает"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "5dabafab-6afd-4927-84e1-2104847df5f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Проведем бутстрап\n",
    "\n",
    "def bootstrap_aa(n_bootstrap: int = 10_000, \n",
    "                 B: int = 500,\n",
    ") -> List:\n",
    "    pvalues = []\n",
    "    for _ in range(n_bootstrap):\n",
    "        group_2 = df_aa[df_aa['exp_group'] == 2]['ctr'].sample(B, replace=False).tolist()\n",
    "        group_3 = df_aa[df_aa['exp_group'] == 3]['ctr'].sample(B, replace=False).tolist()\n",
    "        pvalues.append(st.ttest_ind(group_2, group_3, equal_var=False)[1])\n",
    "    \n",
    "    return pvalues"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "d41fc988-ff8e-4f79-9418-fd63275242aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "pvalues = bootstrap_aa()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "83baafe5-cafa-4ca0-9033-b09e8779924f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<AxesSubplot:ylabel='Count'>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmsAAAJBCAYAAAAZe/K0AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfRElEQVR4nO3df2zdZd3w8U/P2i14r1A6yugGhkDMnEJUqCHxYbllc9mUDkXFkQkqE0jAH4uEZYQfK798tBsaQZhoNGiEQOAPhwzchlm854iiGJDMISiioitsa92Dg8HanvP8AfRmuLVno+ecT3ter4Rk7XV6fa+di7bvfc+Pb0OpVCoFAAApFWq9AAAA9k+sAQAkJtYAABITawAAiYk1AIDExBoAQGJiDQAgscZaL6DS/vWvF6NYrNxbyU2ZMjl6e3dVbH4OnD3Jyb7kY09ysi/5VGNPCoWGOPzw/9rn2LiPtWKxVNFYe/0Y5GJPcrIv+diTnOxLPrXcEw+DAgAkJtYAABITawAAiYk1AIDExBoAQGJiDQAgMbEGAJCYWAMASEysAQAkJtYAABITawAAiYk1AIDExBoAQGJiDQAgMbEGAJCYWAMASEysAQAkJtYAABJrrNaBZs+eHRMnToxJkyZFRMSll14as2bNisceeyyWL18er7zySkyfPj1WrlwZU6ZMiYgYdgwAoB5U9czaTTfdFPfee2/ce++9MWvWrCgWi7F06dJYvnx5rFu3Ljo6OuKGG26IiBh2DACgXtT0YdDNmzfHpEmToqOjIyIizj777Fi7du2IYwAA9aJqD4NGvPrQZ6lUipNPPjkuueSS6OnpiWnTpg2Nt7a2RrFYjJ07dw471tLSUs1lAwDUTNVi7Y477oj29vbYs2dPfPWrX41rr7025s6dW/HjTpkyueLHaGtrrvgxODD2JCf7ko89ycm+5FPLPalarLW3t0dExMSJE2PRokVx0UUXxWc+85nYunXr0G36+vqiUChES0tLtLe373fsQPT27opisTQqf4d9aWtrju3b/12x+dnboYcdEpMmVuZ/21f2DMQL/293RebG90pG9iQn+5JPNfakUGjY7wmmqsTaSy+9FIODg9Hc3BylUikeeOCBmDlzZpxwwgnx8ssvxyOPPBIdHR1x1113xfz58yMihh2jfk2a2BhLb/yfYW/T1NQY/f0DBzz3yiX/fbDLAoCKqUqs9fb2xpe+9KUYHByMYrEYxx9/fHR1dUWhUIgVK1ZEV1fXXm/PERHDjgEA1IuqxNoxxxwTq1ev3ufYSSedFPfdd98BjwEA1ANXMAAASEysAQAkJtYAABITawAAiYk1AIDExBoAQGJiDQAgMbEGAJBY1a4NCgDZuN4wY4FYA6BulXO94YPlesOMFrEGr+kfKEZbW3NF5vYvbAAOlliD1zQ1FvwLG4B0vMAAACAxZ9YS88RXAECsJeaJrwCAh0EBABITawAAiYk1AIDExBoAQGJiDQAgMbEGAJCYWAMASEysAQAkJtYAABJzBQMAYIhLHeYj1gCAIS51mI9YA6gT+zpj0tbWPCpzO2MClSPWAOrEm8+YNDU1Rn//wKjM7YwJVI4XGAAAJCbWAAAS8zAoMC55RRswXog1YFzyijZgvBBrQM1U8uxXJfUPFEftVZRv5qwd8GZj76ckMG6M1bNfTY2FMbluYGzyAgMAgMScWatTHsYBgLFBrNUpD+MAwNjgYVAAgMTEGgBAYh4GBQDGvEq+FVD/QLEi85ZLrAHDGs0fgJV6UQvAWH0roHKINWBYo/UDsKmpMfr7B/b6XK1/AALVVcl3IhjPxBoAUBXeieDgiDWARJx5AN5MrAEk4swD8GbeugMAIDFn1gBIrZJvyQBjgf/7YYzzi4zxbjy/JQOUw094GOMq+Ysswi8zgFrznDUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJeVNcABhjXLmkvthpABhjXIKrvog1AKiA/oFitLU1H9TXHuzXMT6JNQCogKbGwkGd/Wpqaoz+/oFhb+PsV33xAgMAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACTWWOsFMP70DxSjra251ssAgHFBrDHqmhoLsfTG/6nI3CuX/HdF5gWArDwMCgCQmFgDAEhMrAEAJOY5a1AFXnQBwMESa1AFXnQBwMHyMCgAQGJiDQAgMbEGAJCYWAMASEysAQAk5tWgb5G3ZAAAKkmsvUXekgEAqCQPgwIAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AILGqx9rNN98cM2bMiKeeeioiIh577LE444wzYt68ebF48eLo7e0duu1wYwAA9aCqsfaHP/whHnvssZg+fXpERBSLxVi6dGksX7481q1bFx0dHXHDDTeMOAYAUC+q9qa4e/bsiWuvvTa+8Y1vxGc+85mIiNi8eXNMmjQpOjo6IiLi7LPPjjlz5sTXvva1YccAyMXVXKByqhZrN954Y5xxxhlx9NFHD32up6cnpk2bNvRxa2trFIvF2Llz57BjLS0tZR93ypTJo7L+4TQ1Ve5uNPfBzX2wx6/1ujPOPZrz72uesXq/jJe5R21vGwtx+aqHRmWuN/u/F/+fcXN/j+bXZVz3eJ67lv8YqUqsPfroo7F58+a49NJLq3G4vfT27opisVSx+dvamqO/f6Bi85v7wOduamo86OOP1/skw/z725exer+Mh7nfyvfKSHOPtnqau9x9ybbu8T739u3/rtjcERGFQsN+TzBVJdZ++9vfxtNPPx1z5syJiIjnnnsuPv/5z8e5554bW7duHbpdX19fFAqFaGlpifb29v2OAQDUi6q8wODCCy+MTZs2xYYNG2LDhg1x1FFHxQ9+8IM4//zz4+WXX45HHnkkIiLuuuuumD9/fkREnHDCCfsdAwCoF1V7ztq+FAqFWLFiRXR1dcUrr7wS06dPj5UrV444BgBQL2oSaxs2bBj680knnRT33XffPm833BgAQD1wBQMAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEmus1oEuvvji+Mc//hGFQiHe9ra3xVVXXRUzZ86MZ555Ji677LLYuXNntLS0RHd3dxx77LEREcOOAQDUg6qdWevu7o6f/vSnsXr16li8eHFcfvnlERHR1dUVixYtinXr1sWiRYti+fLlQ18z3BgAQD2oWqw1NzcP/XnXrl3R0NAQvb29sWXLlujs7IyIiM7OztiyZUv09fUNOwYAUC+q9jBoRMQVV1wRDz30UJRKpfj+978fPT09MXXq1JgwYUJEREyYMCGOPPLI6OnpiVKptN+x1tbWso85Zcrkivxd3qipqXJ3o7kPbu6DPX6t151x7tGcf1/zjNX7ZbzMPZrHGi/3SYa5K/lzbrSOX29zt7U1j3yjCqlqrH31q1+NiIjVq1fHihUrYsmSJRU/Zm/vrigWSxWbv62tOfr7Byo2v7kPfO6mpsaDPv54vU8yzL+/fRmr98t4mPutfK+MNPdoq6e5y92XbOse73Nv3/7vis0dEVEoNOz3BFNNXg36sY99LB5++OE46qij4vnnn4/BwcGIiBgcHIxt27ZFe3t7tLe373cMAKBeVCXWXnzxxejp6Rn6eMOGDXHYYYfFlClTYubMmbFmzZqIiFizZk3MnDkzWltbhx0DAKgXVXkYdPfu3bFkyZLYvXt3FAqFOOyww+LWW2+NhoaGuPrqq+Oyyy6LVatWxaGHHhrd3d1DXzfcGABAPahKrB1xxBFx991373Ps+OOPj3vuueeAxwAA6oErGAAAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASKzvWfvazn+3z82vXrh21xQAAsLeyY+2KK67Y5+eXL18+aosBAGBvjSPd4Nlnn42IiFKpNPTnN45NnDixMisDAGDkWJs7d240NDREqVSKuXPn7jV2xBFHxJe+9KWKLQ4AoN6NGGt//OMfIyLinHPOidtvv73iCwIA4H+V/Zw1oQYAUH0jnll73bPPPhvf+ta34oknnoiXXnppr7Ff/OIXo70uAADiAGLt0ksvjWOOOSaWLVsWhxxySCXXBADAa8qOtT/96U9x5513RqHgfXQBAKql7PJ6//vfH1u2bKnkWgAAeJOyz6xNnz49zj///Jg7d24cccQRe40tWbJk1BcGAMABxNru3bvjtNNOi4GBgXjuuecquSYAAF5Tdqx97Wtfq+Q6AADYhwN66479OeaYY0ZlMQAA7K3sWHvjZade19DQEBERTzzxxOivDACA8mPt9ctOvW779u1x8803R0dHx6gvCgCAVx30m6a1tbXFFVdcEd/85jdHcz0AALzBW3qH27/85S+xe/fu0VoLAABvUvbDoIsWLRp6jlrEq2/l8ec//zm+8IUvVGRhAAAcQKydddZZe318yCGHxDvf+c449thjR3tNAAC8puxYO/PMMyu5DgAA9qHs56z19/fHTTfdFHPmzIkTTzwx5syZEzfddFPs2bOnkusDAKhrZZ9ZW7lyZTz++ONxzTXXxLRp02Lr1q2xatWq2LVrV1x++eWVXCMAQN0qO9bWrl0b9957bxx++OEREXHcccfFu971rvjoRz8q1gAAKqTsh0HfeOWCcj4PAMBbV3aszZ8/Py666KL45S9/GU8//XRs3LgxvvCFL8T8+fMruT4AgLpW9sOgS5cuje985ztx7bXXxrZt22Lq1Klx+umnx0UXXVTJ9QEA1LURz6z97ne/i5UrV8bEiRNjyZIl8eCDD8bvf//7WL9+fezZsye2bNlSjXUCANSlEWPtu9/9brz//e/f59gpp5wSt95666gvCgCAV40Ya0888UTMmjVrn2Mf+MAHYvPmzaO+KAAAXjVirO3atSv6+/v3OTYwMBAvvvjiqC8KAIBXjRhrxx13XGzatGmfY5s2bYrjjjtu1BcFAMCrRoy1z33uc9HV1RXr16+PYrEYERHFYjHWr18fV199dZx33nkVXyQAQL0a8a07FixYEDt27Ihly5ZFf39/tLS0xM6dO6OpqSm+/OUvR2dnZzXWCQBQl8p6n7XzzjsvzjrrrHj00Udj586d0dLSEu973/ti8uTJlV4fAEBdK/tNcSdPnrzfV4UCAFAZZV9uCgCA6hNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACRWlVj717/+FRdccEHMmzcvFixYEF/84hejr68vIiIee+yxOOOMM2LevHmxePHi6O3tHfq64cYAAOpBVWKtoaEhzj///Fi3bl3cd999ccwxx8QNN9wQxWIxli5dGsuXL49169ZFR0dH3HDDDRERw44BANSLqsRaS0tLnHLKKUMfv/e9742tW7fG5s2bY9KkSdHR0REREWeffXasXbs2ImLYMQCAelH156wVi8W48847Y/bs2dHT0xPTpk0bGmttbY1isRg7d+4cdgwAoF40VvuA1113XbztbW+Lc845Jx588MGKH2/KlMkVP0ZTU+XuRnMf3NwHe/xarzvj3KM5/77mGav3y3iZezSPNV7ukwxzV/Ln3Ggdv97mbmtrrtjcI6lqrHV3d8ff/va3uPXWW6NQKER7e3ts3bp1aLyvry8KhUK0tLQMO3Ygent3RbFYGq2/wn9oa2uO/v6Bis1v7gOfu6mp8aCPP17vkwzz729fxur9Mh7mfivfKyPNPdrqae5y9yXbusf73Nu3/7tic0dEFAoN+z3BVLWHQb/5zW/G5s2b45ZbbomJEydGRMQJJ5wQL7/8cjzyyCMREXHXXXfF/PnzRxwDAKgXVTmz9qc//Sm++93vxrHHHhtnn312REQcffTRccstt8SKFSuiq6srXnnllZg+fXqsXLkyIiIKhcJ+xwAA6kVVYu0d73hHPPnkk/scO+mkk+K+++474DEAgHrgCgYAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJCbWAAASE2sAAImJNQCAxMQaAEBiYg0AIDGxBgCQmFgDAEhMrAEAJFaVWOvu7o7Zs2fHjBkz4qmnnhr6/DPPPBMLFy6MefPmxcKFC+Ovf/1rWWMAAPWiKrE2Z86cuOOOO2L69Ol7fb6rqysWLVoU69ati0WLFsXy5cvLGgMAqBdVibWOjo5ob2/f63O9vb2xZcuW6OzsjIiIzs7O2LJlS/T19Q07BgBQTxprdeCenp6YOnVqTJgwISIiJkyYEEceeWT09PREqVTa71hra2utlgwAUHU1i7VqmTJlcsWP0dRUubvR3Ac398Eev9brzjj3aM6/r3nG6v0yXuYezWONl/skw9yV/Dk3Wsevt7nb2porNvdIahZr7e3t8fzzz8fg4GBMmDAhBgcHY9u2bdHe3h6lUmm/Yweqt3dXFIulCvwNXtXW1hz9/QMVm9/cBz53U1PjQR9/vN4nGebf376M1ftlPMz9Vr5XRpp7tNXT3OXuS7Z1j/e5t2//d8XmjogoFBr2e4KpZm/dMWXKlJg5c2asWbMmIiLWrFkTM2fOjNbW1mHHAADqSVXOrF1//fWxfv362LFjR5x33nnR0tIS999/f1x99dVx2WWXxapVq+LQQw+N7u7uoa8ZbgwAoF5UJdauvPLKuPLKK//j88cff3zcc889+/ya4cYAAOqFKxgAACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkJhYAwBITKwBACQm1gAAEhNrAACJiTUAgMTEGgBAYmINACAxsQYAkFj6WHvmmWdi4cKFMW/evFi4cGH89a9/rfWSAACqJn2sdXV1xaJFi2LdunWxaNGiWL58ea2XBABQNY21XsBwent7Y8uWLXHbbbdFRERnZ2dcd9110dfXF62trWXNUSg0VHKJERFxePMkcyeau7GpMQb6J1Rk7rdirM49WvPvb1/G6v0yHuZ+K98rI8092upp7nL3Jdu6x/vcle6J4eZvKJVKpYoe/S3YvHlzLFu2LO6///6hz33kIx+JlStXxrvf/e4argwAoDrSPwwKAFDPUsdae3t7PP/88zE4OBgREYODg7Ft27Zob2+v8coAAKojdaxNmTIlZs6cGWvWrImIiDVr1sTMmTPLfr4aAMBYl/o5axERTz/9dFx22WXxwgsvxKGHHhrd3d1x3HHH1XpZAABVkT7WAADqWeqHQQEA6p1YAwBITKwBACQm1gAAEhNrZSjnYvKDg4NxzTXXxIc+9KGYO3du3HPPPdVfaJ0pZ19uueWWOP3002PBggXx8Y9/PH75y19Wf6F1pJw9ed1f/vKXeM973hPd3d3VW2CdKndfHnjggViwYEF0dnbGggULYseOHdVdaB0pZ096e3vjwgsvjAULFsSHP/zhuPrqq2NgYKD6i60T3d3dMXv27JgxY0Y89dRT+7xNzX7XlxjRueeeW1q9enWpVCqVVq9eXTr33HP/4zY/+clPSosXLy4NDg6Went7S7NmzSo9++yz1V5qXSlnXzZu3Fh66aWXSqVSqfTEE0+UTj755NLu3burus56Us6elEql0sDAQOmcc84pXXLJJaWvf/3r1VxiXSpnXx5//PHShz/84dK2bdtKpVKp9MILL5Refvnlqq6znpSzJ9dff/3Q98eePXtKn/zkJ0v3339/VddZT37729+Wtm7dWjrttNNKTz755D5vU6vf9c6sjeD1i8l3dnZGxKsXk9+yZUv09fXtdbsHHnggzjrrrCgUCtHa2hof+tCHYu3atbVYcl0od19mzZoVhxxySEREzJgxI0qlUuzcubPay60L5e5JRMT3vve9+OAHPxjHHntslVdZf8rdlx/+8IexePHiaGtri4iI5ubmmDSpchfFrmfl7klDQ0O8+OKLUSwWY8+ePdHf3x9Tp06txZLrQkdHx4hXSKrV73qxNoKenp6YOnVqTJgwISIiJkyYEEceeWT09PT8x+2mTZs29HF7e3s899xzVV1rPSl3X95o9erV8fa3vz2OOuqoai2zrpS7J3/84x9j06ZN8bnPfa4Gq6w/5e7L008/Hc8++2x8+tOfjjPPPDNWrVoVJW/DWRHl7snFF18czzzzTJx66qlD/5188sm1WDKvqdXverFGXfjNb34TN954Y3zjG9+o9VLqWn9/f1x11VVxzTXXDP2iIofBwcF48skn47bbbosf//jHsXHjxrj33ntrvay6tnbt2pgxY0Zs2rQpNm7cGI888ohHbOqUWBtBuReTb29vj61btw593NPT4wxOBZW7LxERjz76aCxdujRuueUWlyqroHL2ZPv27fH3v/89Lrzwwpg9e3b86Ec/irvvvjuuuuqqWi173Cv3e2XatGkxf/78mDhxYkyePDnmzJkTjz/+eC2WPO6Vuye33357nHHGGVEoFKK5uTlmz54dDz/8cC2WzGtq9bterI2g3IvJz58/P+65554oFovR19cXP//5z2PevHm1WHJdKHdfHn/88fjKV74SN910U7z73e+uxVLrRjl7Mm3atHj44Ydjw4YNsWHDhvjsZz8bn/rUp+K6666r1bLHvXK/Vzo7O2PTpk1RKpWiv78/fv3rX8c73/nOWix53Ct3T44++ujYuHFjRETs2bMnfvWrX8U73vGOqq+X/1Wr3/WuDVqG/V1M/oILLogvf/nLceKJJ8bg4GBce+218dBDD0VExAUXXBALFy6s8crHt3L25ROf+ET885//3OtJuStWrIgZM2bUcOXjVzl78kbf/va346WXXoply5bVaMX1oZx9KRaL0d3dHRs3boxCoRCnnnpqLFu2LAoF/6avhHL25O9//3t0dXXFjh07YnBwME455ZS44oororGxsdbLH5euv/76WL9+fezYsSMOP/zwaGlpifvvvz/F73qxBgCQmH8yAQAkJtYAABITawAAiYk1AIDExBoAQGJiDQAgMbEGAJCYWAMASOz/A0UlwpdaQKrYAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.histplot(pvalues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "b9be2886-4d3a-4c79-9163-bb6f447a42a6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4.569999999999999% is the percent of p-values which are less than 0.05 \n"
     ]
    }
   ],
   "source": [
    "perc = sum([1 for i in pvalues if i <= 0.05]) / 10000 * 100\n",
    "print(f'{perc}% is the percent of p-values which are less than 0.05 ')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "76080f67",
   "metadata": {},
   "source": [
    "#### Вывод: \n",
    "Статистически значимые различия получаем в 4,6% , что соответствует ошибке I рода. \n",
    "\n",
    "Система сплитования на группы работает корректно."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}