sick_arivale_participants.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "9ddfc8cd-99c1-4b7d-bc0d-00be3352e2fe",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns \n",
    "import matplotlib.pyplot as plt\n",
    "from statannot import add_stat_annotation\n",
    "import matplotlib\n",
    "import warnings\n",
    "%matplotlib inline\n",
    "warnings.simplefilter('ignore')\n",
    "from arivale_data_interface import *\n",
    "frozen_ss_path='/proj/arivale/snapshots/arivale_snapshot_ISB_2020-03-16_2156'\n",
    "sn=list_snapshot_contents() #list all content\n",
    "def get_frozen_snapshot(ss_name, ss_path=frozen_ss_path): #collect data\n",
    "    return get_snapshot(ss_name, path=ss_path)\n",
    "from statsmodels.stats.anova import anova_lm\n",
    "import statsmodels.formula.api as smf\n",
    "import scipy.stats\n",
    "warnings.simplefilter('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5df5ce20-4a4d-4611-a84e-e2ae06027c8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "genpath = '/proj/gibbons/kramos/github/IBT-and-the-Gut-Microbiome'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8be108d7-7619-4947-9e40-110421f1134e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>id</th>\n",
       "      <th>simpson</th>\n",
       "      <th>inv_simpson</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>days_in_program</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>BMI_CALC</th>\n",
       "      <th>HEIGHT_CALC</th>\n",
       "      <th>height_cm</th>\n",
       "      <th>assessment:digestion:bowel-movements:enum</th>\n",
       "      <th>bowel_movement_frequency</th>\n",
       "      <th>vegetable_frequency</th>\n",
       "      <th>b_vegetable_frequency</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5216</td>\n",
       "      <td>AV15-4372|AKE035</td>\n",
       "      <td>0.973573</td>\n",
       "      <td>37.839776</td>\n",
       "      <td>01602320</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>M</td>\n",
       "      <td>47.0</td>\n",
       "      <td>37.229730</td>\n",
       "      <td>74.0</td>\n",
       "      <td>187.96</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5081</td>\n",
       "      <td>AV15-3716|AKE025</td>\n",
       "      <td>0.977155</td>\n",
       "      <td>43.774041</td>\n",
       "      <td>01228476</td>\n",
       "      <td>0.0</td>\n",
       "      <td>F</td>\n",
       "      <td>50.0</td>\n",
       "      <td>22.462722</td>\n",
       "      <td>65.0</td>\n",
       "      <td>165.10</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4956</td>\n",
       "      <td>AV15-3489|AKE025</td>\n",
       "      <td>0.874632</td>\n",
       "      <td>7.976489</td>\n",
       "      <td>01826852</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>35.0</td>\n",
       "      <td>24.541709</td>\n",
       "      <td>61.0</td>\n",
       "      <td>154.94</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5150</td>\n",
       "      <td>AV15-3840|AKE025</td>\n",
       "      <td>0.957654</td>\n",
       "      <td>23.615008</td>\n",
       "      <td>01602518</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>49.0</td>\n",
       "      <td>34.154541</td>\n",
       "      <td>64.0</td>\n",
       "      <td>162.56</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>low</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5020</td>\n",
       "      <td>AV15-3636|AKE025</td>\n",
       "      <td>0.981430</td>\n",
       "      <td>53.849198</td>\n",
       "      <td>01447446</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>56.0</td>\n",
       "      <td>21.950816</td>\n",
       "      <td>70.0</td>\n",
       "      <td>177.80</td>\n",
       "      <td>(2) 3-6 times per week</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>high</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.1                id   simpson  inv_simpson sample_id  \\\n",
       "0          5216  AV15-4372|AKE035  0.973573    37.839776  01602320   \n",
       "1          5081  AV15-3716|AKE025  0.977155    43.774041  01228476   \n",
       "2          4956  AV15-3489|AKE025  0.874632     7.976489  01826852   \n",
       "3          5150  AV15-3840|AKE025  0.957654    23.615008  01602518   \n",
       "4          5020  AV15-3636|AKE025  0.981430    53.849198  01447446   \n",
       "\n",
       "   days_in_program sex   age   BMI_CALC  HEIGHT_CALC  height_cm  \\\n",
       "0             -2.0   M  47.0  37.229730         74.0     187.96   \n",
       "1              0.0   F  50.0  22.462722         65.0     165.10   \n",
       "2              1.0   F  35.0  24.541709         61.0     154.94   \n",
       "3              1.0   F  49.0  34.154541         64.0     162.56   \n",
       "4              1.0   F  56.0  21.950816         70.0     177.80   \n",
       "\n",
       "  assessment:digestion:bowel-movements:enum  bowel_movement_frequency  \\\n",
       "0                       (3) 1-3 times daily                       3.0   \n",
       "1                       (3) 1-3 times daily                       3.0   \n",
       "2                       (3) 1-3 times daily                       3.0   \n",
       "3                       (3) 1-3 times daily                       3.0   \n",
       "4                    (2) 3-6 times per week                       2.0   \n",
       "\n",
       "   vegetable_frequency b_vegetable_frequency  \n",
       "0                  1.0                   low  \n",
       "1                  4.0                  high  \n",
       "2                  4.0                  high  \n",
       "3                  2.0                   low  \n",
       "4                  3.0                  high  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "######### Import, clean, and format Arivale dataset #########\n",
    "\n",
    "arivale=pd.read_csv(genpath+'/arivale_cohort/alpha-diversity.csv').rename(columns={'Unnamed: 0':'id'})\n",
    "\n",
    "# Get sample metadata and merge\n",
    "metadata = pd.read_csv('/proj/arivale/microbiome/16S_processed/metadata.csv')[['public_client_id', 'id', 'days_in_program','sex', 'age']]\\\n",
    "                        .rename(columns = {'public_client_id':'sample_id'})\n",
    "arivale = pd.merge(arivale, metadata, on = 'id')\n",
    "\n",
    "# Get weight data and merge\n",
    "weights = get_frozen_snapshot('weight')[['public_client_id', 'days_in_program', 'BMI_CALC', 'HEIGHT_CALC']]\\\n",
    "                     .dropna().rename(columns = {'public_client_id':'sample_id'})\n",
    "arivale = pd.merge_asof(\n",
    "    arivale[arivale.sample_id.isin(weights.sample_id)].sort_values(by='days_in_program'),\n",
    "    weights.sort_values(by='days_in_program'), \n",
    "    by = 'sample_id', on = 'days_in_program', direction='nearest')\n",
    "\n",
    "# convert height from in to cm \n",
    "arivale['height_cm']=arivale['HEIGHT_CALC'].apply(lambda x: x*2.54)\n",
    "\n",
    "# Finally, I need bowel movement frequency \n",
    "digestion = get_frozen_snapshot('assessments_digestive_health')[['public_client_id', 'days_in_program', 'assessment:digestion:bowel-movements:enum']].dropna().rename(columns = {'public_client_id':'sample_id'})\n",
    "\n",
    "# Get the bowel movement number \n",
    "digestion['bowel_movement_frequency'] = digestion['assessment:digestion:bowel-movements:enum'].astype(str).apply(lambda x: x[1])\n",
    "digestion['bowel_movement_frequency'] = digestion['bowel_movement_frequency'].astype(int)\n",
    "\n",
    "arivale = pd.merge_asof(arivale.sort_values(by='days_in_program'),\n",
    "    digestion.sort_values(by='days_in_program'),\n",
    "    by='sample_id', on='days_in_program', direction='nearest')\n",
    "\n",
    "# Add in dietary data\n",
    "diet=get_frozen_snapshot('assessments')[['public_client_id', 'days_in_program', 'assessment:lifestyle:vegetables:enum']].dropna().reset_index(drop=True)\\\n",
    ".rename(columns={'assessment:lifestyle:vegetables:enum':'vegetable_frequency', 'public_client_id':'sample_id'})\n",
    "\n",
    "# Make vegetable frequency numeric \n",
    "diet_num = list()\n",
    "for i in range(0, len(diet)): \n",
    "    check = diet.iloc[i, 2]\n",
    "    if check == '(0) Zero/less than 1 per day':\n",
    "        diet_num.append(1)\n",
    "    elif check == '(1) 1': \n",
    "        diet_num.append(2)\n",
    "    elif check == '(2) 2-3': \n",
    "        diet_num.append(3)\n",
    "    elif check == '(3) 4-5':\n",
    "        diet_num.append(4)\n",
    "    elif check == '(4) 6 or more': \n",
    "        diet_num.append(5) \n",
    "    else: \n",
    "        float('nan')\n",
    "diet['vegetable_frequency']=diet_num\n",
    "high = [5,4,3]\n",
    "diet['b_vegetable_frequency'] = diet['vegetable_frequency'].apply(lambda x: 'high' if x in high else 'low')\n",
    "\n",
    "arivale=pd.merge_asof(\n",
    "    arivale.sort_values(by='days_in_program'), \n",
    "    diet[diet.sample_id.isin(arivale.sample_id)].sort_values(by='days_in_program'), \n",
    "    by='sample_id', on='days_in_program', direction='nearest')\n",
    "\n",
    "arivale.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fa39f421-1104-4659-8441-ece707c6717e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>id</th>\n",
       "      <th>simpson</th>\n",
       "      <th>inv_simpson</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>days_in_program</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>BMI_CALC</th>\n",
       "      <th>HEIGHT_CALC</th>\n",
       "      <th>height_cm</th>\n",
       "      <th>assessment:digestion:bowel-movements:enum</th>\n",
       "      <th>bowel_movement_frequency</th>\n",
       "      <th>vegetable_frequency</th>\n",
       "      <th>b_vegetable_frequency</th>\n",
       "      <th>LDL-CHOL CALCULATION</th>\n",
       "      <th>CRP HIGH SENSITIVITY</th>\n",
       "      <th>GLYCOHEMOGLOBIN A1C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5216</td>\n",
       "      <td>AV15-4372|AKE035</td>\n",
       "      <td>0.973573</td>\n",
       "      <td>37.839776</td>\n",
       "      <td>01602320</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>M</td>\n",
       "      <td>47.0</td>\n",
       "      <td>37.229730</td>\n",
       "      <td>74.0</td>\n",
       "      <td>187.96</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>low</td>\n",
       "      <td>151.0</td>\n",
       "      <td>6.54</td>\n",
       "      <td>5.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5081</td>\n",
       "      <td>AV15-3716|AKE025</td>\n",
       "      <td>0.977155</td>\n",
       "      <td>43.774041</td>\n",
       "      <td>01228476</td>\n",
       "      <td>0.0</td>\n",
       "      <td>F</td>\n",
       "      <td>50.0</td>\n",
       "      <td>22.462722</td>\n",
       "      <td>65.0</td>\n",
       "      <td>165.10</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>5.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1277</td>\n",
       "      <td>22001612562257|GFM-1079-006</td>\n",
       "      <td>0.982218</td>\n",
       "      <td>56.237055</td>\n",
       "      <td>01360278</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>43.0</td>\n",
       "      <td>20.938965</td>\n",
       "      <td>64.0</td>\n",
       "      <td>162.56</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>82.0</td>\n",
       "      <td>0.22</td>\n",
       "      <td>5.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4831</td>\n",
       "      <td>AV15-2608|AKE034</td>\n",
       "      <td>0.980011</td>\n",
       "      <td>50.027191</td>\n",
       "      <td>01788725</td>\n",
       "      <td>1.0</td>\n",
       "      <td>M</td>\n",
       "      <td>32.0</td>\n",
       "      <td>27.006658</td>\n",
       "      <td>69.0</td>\n",
       "      <td>175.26</td>\n",
       "      <td>(2) 3-6 times per week</td>\n",
       "      <td>2.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>137.0</td>\n",
       "      <td>1.12</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4037</td>\n",
       "      <td>AV15-1295|AKE009</td>\n",
       "      <td>0.979111</td>\n",
       "      <td>47.871679</td>\n",
       "      <td>01421882</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>19.0</td>\n",
       "      <td>23.793846</td>\n",
       "      <td>65.0</td>\n",
       "      <td>165.10</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>low</td>\n",
       "      <td>91.0</td>\n",
       "      <td>0.20</td>\n",
       "      <td>5.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.1                           id   simpson  inv_simpson sample_id  \\\n",
       "0          5216             AV15-4372|AKE035  0.973573    37.839776  01602320   \n",
       "1          5081             AV15-3716|AKE025  0.977155    43.774041  01228476   \n",
       "2          1277  22001612562257|GFM-1079-006  0.982218    56.237055  01360278   \n",
       "3          4831             AV15-2608|AKE034  0.980011    50.027191  01788725   \n",
       "4          4037             AV15-1295|AKE009  0.979111    47.871679  01421882   \n",
       "\n",
       "   days_in_program sex   age   BMI_CALC  HEIGHT_CALC  height_cm  \\\n",
       "0             -2.0   M  47.0  37.229730         74.0     187.96   \n",
       "1              0.0   F  50.0  22.462722         65.0     165.10   \n",
       "2              1.0   F  43.0  20.938965         64.0     162.56   \n",
       "3              1.0   M  32.0  27.006658         69.0     175.26   \n",
       "4              1.0   F  19.0  23.793846         65.0     165.10   \n",
       "\n",
       "  assessment:digestion:bowel-movements:enum  bowel_movement_frequency  \\\n",
       "0                       (3) 1-3 times daily                       3.0   \n",
       "1                       (3) 1-3 times daily                       3.0   \n",
       "2                       (3) 1-3 times daily                       3.0   \n",
       "3                    (2) 3-6 times per week                       2.0   \n",
       "4                                       NaN                       NaN   \n",
       "\n",
       "   vegetable_frequency b_vegetable_frequency  LDL-CHOL CALCULATION  \\\n",
       "0                  1.0                   low                 151.0   \n",
       "1                  4.0                  high                  87.0   \n",
       "2                  4.0                  high                  82.0   \n",
       "3                  4.0                  high                 137.0   \n",
       "4                  2.0                   low                  91.0   \n",
       "\n",
       "   CRP HIGH SENSITIVITY  GLYCOHEMOGLOBIN A1C  \n",
       "0                  6.54                  5.9  \n",
       "1                  0.16                  5.9  \n",
       "2                  0.22                  5.6  \n",
       "3                  1.12                  5.5  \n",
       "4                  0.20                  5.6  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Get health markers from blood chemistries\n",
    "chemistries = get_frozen_snapshot(\"chemistries\")[['public_client_id', 'days_in_program', 'LDL-CHOL CALCULATION','CRP HIGH SENSITIVITY','GLYCOHEMOGLOBIN A1C']].sort_values(by='days_in_program')\n",
    "chemistries = chemistries.rename(columns={'public_client_id':'sample_id'})\n",
    "chemistries['days_in_program']=chemistries['days_in_program'].astype(float)\n",
    "\n",
    "arivale=pd.merge_asof(\n",
    "    arivale.sort_values(by='days_in_program'), \n",
    "    chemistries[chemistries.sample_id.isin(arivale.sample_id)].sort_values(by='days_in_program'), \n",
    "    by='sample_id', on='days_in_program', direction='nearest')\n",
    "\n",
    "arivale.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1a9fbaee-93e0-41f1-807f-fbf23416d6d2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0.1</th>\n",
       "      <th>id</th>\n",
       "      <th>simpson</th>\n",
       "      <th>inv_simpson</th>\n",
       "      <th>sample_id</th>\n",
       "      <th>days_in_program</th>\n",
       "      <th>sex</th>\n",
       "      <th>age</th>\n",
       "      <th>BMI_CALC</th>\n",
       "      <th>HEIGHT_CALC</th>\n",
       "      <th>height_cm</th>\n",
       "      <th>assessment:digestion:bowel-movements:enum</th>\n",
       "      <th>bowel_movement_frequency</th>\n",
       "      <th>vegetable_frequency</th>\n",
       "      <th>b_vegetable_frequency</th>\n",
       "      <th>LDL-CHOL CALCULATION</th>\n",
       "      <th>CRP HIGH SENSITIVITY</th>\n",
       "      <th>GLYCOHEMOGLOBIN A1C</th>\n",
       "      <th>meds_antibiotics_last_3_months</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5216</td>\n",
       "      <td>AV15-4372|AKE035</td>\n",
       "      <td>0.973573</td>\n",
       "      <td>37.839776</td>\n",
       "      <td>01602320</td>\n",
       "      <td>-2.0</td>\n",
       "      <td>M</td>\n",
       "      <td>47.0</td>\n",
       "      <td>37.229730</td>\n",
       "      <td>74.0</td>\n",
       "      <td>187.96</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>low</td>\n",
       "      <td>151.0</td>\n",
       "      <td>6.54</td>\n",
       "      <td>5.9</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5081</td>\n",
       "      <td>AV15-3716|AKE025</td>\n",
       "      <td>0.977155</td>\n",
       "      <td>43.774041</td>\n",
       "      <td>01228476</td>\n",
       "      <td>0.0</td>\n",
       "      <td>F</td>\n",
       "      <td>50.0</td>\n",
       "      <td>22.462722</td>\n",
       "      <td>65.0</td>\n",
       "      <td>165.10</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>5.9</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5104</td>\n",
       "      <td>AV15-3759|AKE025</td>\n",
       "      <td>0.918389</td>\n",
       "      <td>12.253256</td>\n",
       "      <td>01250245</td>\n",
       "      <td>1.0</td>\n",
       "      <td>M</td>\n",
       "      <td>32.0</td>\n",
       "      <td>46.955261</td>\n",
       "      <td>69.0</td>\n",
       "      <td>175.26</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>high</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5150</td>\n",
       "      <td>AV15-3840|AKE025</td>\n",
       "      <td>0.957654</td>\n",
       "      <td>23.615008</td>\n",
       "      <td>01602518</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>49.0</td>\n",
       "      <td>34.154541</td>\n",
       "      <td>64.0</td>\n",
       "      <td>162.56</td>\n",
       "      <td>(3) 1-3 times daily</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>low</td>\n",
       "      <td>95.0</td>\n",
       "      <td>5.88</td>\n",
       "      <td>6.2</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5020</td>\n",
       "      <td>AV15-3636|AKE025</td>\n",
       "      <td>0.981430</td>\n",
       "      <td>53.849198</td>\n",
       "      <td>01447446</td>\n",
       "      <td>1.0</td>\n",
       "      <td>F</td>\n",
       "      <td>56.0</td>\n",
       "      <td>21.950816</td>\n",
       "      <td>70.0</td>\n",
       "      <td>177.80</td>\n",
       "      <td>(2) 3-6 times per week</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>high</td>\n",
       "      <td>137.0</td>\n",
       "      <td>0.61</td>\n",
       "      <td>5.4</td>\n",
       "      <td>No</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Unnamed: 0.1                id   simpson  inv_simpson sample_id  \\\n",
       "0          5216  AV15-4372|AKE035  0.973573    37.839776  01602320   \n",
       "1          5081  AV15-3716|AKE025  0.977155    43.774041  01228476   \n",
       "2          5104  AV15-3759|AKE025  0.918389    12.253256  01250245   \n",
       "3          5150  AV15-3840|AKE025  0.957654    23.615008  01602518   \n",
       "4          5020  AV15-3636|AKE025  0.981430    53.849198  01447446   \n",
       "\n",
       "   days_in_program sex   age   BMI_CALC  HEIGHT_CALC  height_cm  \\\n",
       "0             -2.0   M  47.0  37.229730         74.0     187.96   \n",
       "1              0.0   F  50.0  22.462722         65.0     165.10   \n",
       "2              1.0   M  32.0  46.955261         69.0     175.26   \n",
       "3              1.0   F  49.0  34.154541         64.0     162.56   \n",
       "4              1.0   F  56.0  21.950816         70.0     177.80   \n",
       "\n",
       "  assessment:digestion:bowel-movements:enum  bowel_movement_frequency  \\\n",
       "0                       (3) 1-3 times daily                       3.0   \n",
       "1                       (3) 1-3 times daily                       3.0   \n",
       "2                       (3) 1-3 times daily                       3.0   \n",
       "3                       (3) 1-3 times daily                       3.0   \n",
       "4                    (2) 3-6 times per week                       2.0   \n",
       "\n",
       "   vegetable_frequency b_vegetable_frequency  LDL-CHOL CALCULATION  \\\n",
       "0                  1.0                   low                 151.0   \n",
       "1                  4.0                  high                  87.0   \n",
       "2                  3.0                  high                   NaN   \n",
       "3                  2.0                   low                  95.0   \n",
       "4                  3.0                  high                 137.0   \n",
       "\n",
       "   CRP HIGH SENSITIVITY  GLYCOHEMOGLOBIN A1C meds_antibiotics_last_3_months  \n",
       "0                  6.54                  5.9                             No  \n",
       "1                  0.16                  5.9                             No  \n",
       "2                   NaN                  NaN                             No  \n",
       "3                  5.88                  6.2                             No  \n",
       "4                  0.61                  5.4                             No  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove individuals with abx use\n",
    "abx = get_frozen_snapshot('assessments_medications')[['public_client_id', 'days_in_program', 'meds_antibiotics_last_3_months']].rename(columns = {'public_client_id':'sample_id'})\n",
    "                                                                                                                                                  \n",
    "arivale=pd.merge_asof(\n",
    "    arivale.sort_values(by='days_in_program'), \n",
    "    abx[abx.sample_id.isin(arivale.sample_id)].sort_values(by='days_in_program'), \n",
    "    by='sample_id', on='days_in_program', direction='nearest')\n",
    "\n",
    "arivale = arivale[arivale['meds_antibiotics_last_3_months']=='No']\n",
    "arivale.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a5d4eaad-a508-42b4-847e-cbb02047bc62",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean dataframe\n",
    "arivale=arivale.sort_values(by='days_in_program').drop_duplicates(subset='sample_id', keep='first')\n",
    "arivale=arivale[['sample_id', 'age', 'sex', 'BMI_CALC', 'bowel_movement_frequency', 'height_cm', 'inv_simpson','vegetable_frequency', 'b_vegetable_frequency', 'LDL-CHOL CALCULATION', 'CRP HIGH SENSITIVITY', 'GLYCOHEMOGLOBIN A1C']].dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "76c4c4ab-6b11-4f3f-8dd5-8487d52d437d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sample_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>BMI_CALC</th>\n",
       "      <th>bowel_movement_frequency</th>\n",
       "      <th>height_cm</th>\n",
       "      <th>inv_simpson</th>\n",
       "      <th>vegetable_frequency</th>\n",
       "      <th>b_vegetable_frequency</th>\n",
       "      <th>LDL</th>\n",
       "      <th>CRP</th>\n",
       "      <th>HBA1C</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>01602320</td>\n",
       "      <td>47.0</td>\n",
       "      <td>M</td>\n",
       "      <td>37.229730</td>\n",
       "      <td>3.0</td>\n",
       "      <td>187.96</td>\n",
       "      <td>37.839776</td>\n",
       "      <td>1.0</td>\n",
       "      <td>low</td>\n",
       "      <td>151.0</td>\n",
       "      <td>6.54</td>\n",
       "      <td>5.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>01228476</td>\n",
       "      <td>50.0</td>\n",
       "      <td>F</td>\n",
       "      <td>22.462722</td>\n",
       "      <td>3.0</td>\n",
       "      <td>165.10</td>\n",
       "      <td>43.774041</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>87.0</td>\n",
       "      <td>0.16</td>\n",
       "      <td>5.9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>01360278</td>\n",
       "      <td>43.0</td>\n",
       "      <td>F</td>\n",
       "      <td>20.938965</td>\n",
       "      <td>3.0</td>\n",
       "      <td>162.56</td>\n",
       "      <td>56.237055</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>82.0</td>\n",
       "      <td>0.22</td>\n",
       "      <td>5.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>01048590</td>\n",
       "      <td>47.0</td>\n",
       "      <td>F</td>\n",
       "      <td>22.459158</td>\n",
       "      <td>3.0</td>\n",
       "      <td>160.02</td>\n",
       "      <td>28.621199</td>\n",
       "      <td>3.0</td>\n",
       "      <td>high</td>\n",
       "      <td>76.0</td>\n",
       "      <td>0.68</td>\n",
       "      <td>5.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>01826852</td>\n",
       "      <td>35.0</td>\n",
       "      <td>F</td>\n",
       "      <td>24.541709</td>\n",
       "      <td>3.0</td>\n",
       "      <td>154.94</td>\n",
       "      <td>7.976489</td>\n",
       "      <td>4.0</td>\n",
       "      <td>high</td>\n",
       "      <td>93.0</td>\n",
       "      <td>1.94</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sample_id   age sex   BMI_CALC  bowel_movement_frequency  height_cm  \\\n",
       "0   01602320  47.0   M  37.229730                       3.0     187.96   \n",
       "1   01228476  50.0   F  22.462722                       3.0     165.10   \n",
       "11  01360278  43.0   F  20.938965                       3.0     162.56   \n",
       "8   01048590  47.0   F  22.459158                       3.0     160.02   \n",
       "7   01826852  35.0   F  24.541709                       3.0     154.94   \n",
       "\n",
       "    inv_simpson  vegetable_frequency b_vegetable_frequency    LDL   CRP  HBA1C  \n",
       "0     37.839776                  1.0                   low  151.0  6.54    5.9  \n",
       "1     43.774041                  4.0                  high   87.0  0.16    5.9  \n",
       "11    56.237055                  4.0                  high   82.0  0.22    5.6  \n",
       "8     28.621199                  3.0                  high   76.0  0.68    5.5  \n",
       "7      7.976489                  4.0                  high   93.0  1.94    6.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Rename variables for regression\n",
    "arivale = arivale.rename(columns = {'LDL-CHOL CALCULATION':'LDL', 'CRP HIGH SENSITIVITY':'CRP', 'GLYCOHEMOGLOBIN A1C':'HBA1C'})\n",
    "arivale.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a6090bba-cf9c-4886-863c-07920c7d3378",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Log transform Simpson's Diversity and height\n",
    "features = ['inv_simpson', 'height_cm']\n",
    "arivale[features] = arivale[features].apply(lambda x: np.log(x))\n",
    "\n",
    "# # Standardize data\n",
    "features=['age', 'BMI_CALC', 'height_cm', 'inv_simpson', 'LDL', 'CRP', 'HBA1C'] \n",
    "arivale[features]=arivale[features].apply(lambda x:(x-x.mean()) / x.std())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ae59e062-fb75-4115-a5a6-f652c6a09c46",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c570d7a9-b1b9-491b-935a-b9f3cc6e7ff9",
   "metadata": {},
   "outputs": [],
   "source": [
    "#### Define dictionary with each covariate and formulas excluding each to determine individual R2 for every covar\n",
    "covars = ['C(sex)', 'age', 'BMI_CALC', 'bowel_movement_frequency', 'vegetable_frequency', 'height_cm', 'height_cm:vegetable_frequency', 'LDL', 'CRP', 'HBA1C']\n",
    "d_formula=dict()\n",
    "d_formula[0] = {\n",
    "    'formula':'inv_simpson ~ age + C(sex) + BMI_CALC + bowel_movement_frequency + vegetable_frequency + height_cm + height_cm:vegetable_frequency + LDL + CRP + HBA1C',\n",
    "    'excluded':'none',\n",
    "    # 'covariates':list(map(lambda x: x.replace('C(sex)', 'C(sex)[T.M]').replace('C(vegetable_frequency)','C(vegetable_frequency)[T.low]'), covars))}\n",
    "    'covariates':list(map(lambda x: x.replace('C(sex)', 'C(sex)[T.M]'), covars))}\n",
    "d_formula[1] = {\n",
    "    'formula':'inv_simpson ~ height_cm',\n",
    "    'excluded':'all but height',\n",
    "    'covariates':float('NaN')}\n",
    "\n",
    "for i in range(0,len(covars)):\n",
    "    excluded=covars[i]\n",
    "    temp_covars=[i for i in covars if i != excluded]\n",
    "    formula=' + '.join(temp_covars)\n",
    "    formula = 'inv_simpson ~ '+formula\n",
    "    temp_covars=list(map(lambda x: x.replace('C(sex)', 'C(sex)[T.M]'),temp_covars))\n",
    "    \n",
    "    d_formula[i+2] = {\n",
    "        'formula':formula,\n",
    "        'excluded':excluded,\n",
    "        'covariates':temp_covars}\n",
    "\n",
    "# Formula for determining indivudal R2 for every covariate\n",
    "def find_covar_rsq(dataframe, f_dict):\n",
    "    big_squared=pd.DataFrame()\n",
    "    \n",
    "    for i in range(0, len(f_dict)): # for every formula combination in my dictionary \n",
    "        if i==0: # first entry in the dict is the full formula \n",
    "            results = smf.ols(formula=f_dict[i]['formula'], data=dataframe).fit() \n",
    "            ref_r=results.rsquared # this is going to be used to derive r_squared of indiviudal covariates\n",
    "            betas=pd.DataFrame(results.params).reset_index().rename(columns={'index':'covar', 0:'Beta'}) # get beta coefficients\n",
    "            betas['formula']=f_dict[i]['formula'] # record the formula used\n",
    "            betas['rsquared_model']=results.rsquared # get the rsquared of the model\n",
    "            ps = pd.DataFrame(results.pvalues.loc[f_dict[i]['covariates']]).reset_index().rename(columns={0:'p_covar', 'index':'covar'}) # get pvalues for each covar\n",
    "            betas=pd.merge(betas, ps, on = 'covar') #merge ps with beta coefficients for each covar\n",
    "            \n",
    "            ### F-test to compare model with and without height as a covar\n",
    "            model_reduced = smf.ols(formula = 'inv_simpson ~ age + C(sex) + BMI_CALC + bowel_movement_frequency + vegetable_frequency', data=dataframe).fit()\n",
    "            F_test = anova_lm(model_reduced, results)\n",
    "            betas['F']=F_test.loc[1,'F']\n",
    "            betas['F_pval']=F_test.loc[1,'Pr(>F)']\n",
    "\n",
    "        \n",
    "        elif i==1: # the second entry in my dict is just simpson vs height\n",
    "            # like before, get beta coefficient and r^2 values\n",
    "            results=smf.ols(formula=f_dict[i]['formula'], data=dataframe).fit()\n",
    "            temp=pd.DataFrame(results.params).reset_index().rename(columns={'index':'covar', 0:'Beta'})\n",
    "            temp['formula']=f_dict[i]['formula']\n",
    "            temp['rsquared_model']=results.rsquared\n",
    "            temp['p_covar']=results.pvalues['height_cm']\n",
    "            \n",
    "            F_test = anova_lm(results)\n",
    "            temp['F']=F_test.loc['height_cm','F']\n",
    "            temp['F_pval']=F_test.loc['height_cm','PR(>F)']\n",
    "            betas=pd.concat([betas, temp])\n",
    "        \n",
    "        else: # for every other formula, we excluded one covar so we can derive the r^2 for indiviudal covars\n",
    "            results = smf.ols(formula=f_dict[i]['formula'], data=dataframe).fit() #ols with the formula[i]\n",
    "            squared_df=pd.DataFrame(columns=['covar', 'rsquared_covar', 'formula', 'rsquared_model'])\n",
    "            squared_df.loc[i,'covar']=f_dict[i]['excluded'] # the excluded covar is the one we are getting an r^2 value for\n",
    "            squared_df.loc[i,'rsquared_covar']=ref_r-results.rsquared\n",
    "            squared_df.loc[i,'formula']=f_dict[i]['formula']\n",
    "            squared_df.loc[i,'rsquared_model']=ref_r\n",
    "            big_squared=pd.concat([big_squared, squared_df])\n",
    "    \n",
    "    #Now we have everything we need to make one big df with my regression info \n",
    "    big_squared.loc[2,'covar']='C(sex)[T.M]' #so I can merge my dfs on covars\n",
    "    # big_squared.loc[6,'covar']='C(vegetable_frequency)[T.low]'\n",
    "    complete=pd.merge(big_squared[['covar', 'rsquared_covar', 'rsquared_model']], \\\n",
    "                      betas[['covar', 'Beta', 'formula', 'rsquared_model', 'p_covar', 'F', 'F_pval']], \\\n",
    "                      on=['covar', 'rsquared_model'], how='right')\n",
    "    return(complete, F_test)\n",
    "\n",
    "arivale_results, arivale_F= find_covar_rsq(arivale, d_formula)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2eedb742-51e0-458a-a1d8-697eaa78ebfc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>covar</th>\n",
       "      <th>rsquared_covar</th>\n",
       "      <th>rsquared_model</th>\n",
       "      <th>Beta</th>\n",
       "      <th>formula</th>\n",
       "      <th>p_covar</th>\n",
       "      <th>F</th>\n",
       "      <th>F_pval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C(sex)[T.M]</td>\n",
       "      <td>0.002542</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-1.521272e-01</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>7.777428e-03</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>age</td>\n",
       "      <td>0.007594</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>9.017540e-02</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>4.348971e-06</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BMI_CALC</td>\n",
       "      <td>0.008416</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-1.072950e-01</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>1.330580e-06</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bowel_movement_frequency</td>\n",
       "      <td>0.009784</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-1.874314e-01</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>1.874410e-07</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>vegetable_frequency</td>\n",
       "      <td>0.001516</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>4.605855e-02</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>3.977037e-02</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>height_cm</td>\n",
       "      <td>0.003014</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>1.919596e-01</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>3.760615e-03</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>height_cm:vegetable_frequency</td>\n",
       "      <td>0.000168</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-1.507022e-02</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>4.933511e-01</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>LDL</td>\n",
       "      <td>0.000121</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-1.127072e-02</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>5.609913e-01</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>CRP</td>\n",
       "      <td>0.002118</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-5.228354e-02</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>1.510933e-02</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>HBA1C</td>\n",
       "      <td>0.004648</td>\n",
       "      <td>0.06477</td>\n",
       "      <td>-7.384013e-02</td>\n",
       "      <td>inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...</td>\n",
       "      <td>3.223246e-04</td>\n",
       "      <td>10.95402</td>\n",
       "      <td>1.875719e-10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Intercept</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.008008</td>\n",
       "      <td>-2.189691e-15</td>\n",
       "      <td>inv_simpson ~ height_cm</td>\n",
       "      <td>4.469390e-06</td>\n",
       "      <td>21.14109</td>\n",
       "      <td>4.469390e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>height_cm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.008008</td>\n",
       "      <td>8.948497e-02</td>\n",
       "      <td>inv_simpson ~ height_cm</td>\n",
       "      <td>4.469390e-06</td>\n",
       "      <td>21.14109</td>\n",
       "      <td>4.469390e-06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            covar rsquared_covar rsquared_model          Beta  \\\n",
       "0                     C(sex)[T.M]       0.002542        0.06477 -1.521272e-01   \n",
       "1                             age       0.007594        0.06477  9.017540e-02   \n",
       "2                        BMI_CALC       0.008416        0.06477 -1.072950e-01   \n",
       "3        bowel_movement_frequency       0.009784        0.06477 -1.874314e-01   \n",
       "4             vegetable_frequency       0.001516        0.06477  4.605855e-02   \n",
       "5                       height_cm       0.003014        0.06477  1.919596e-01   \n",
       "6   height_cm:vegetable_frequency       0.000168        0.06477 -1.507022e-02   \n",
       "7                             LDL       0.000121        0.06477 -1.127072e-02   \n",
       "8                             CRP       0.002118        0.06477 -5.228354e-02   \n",
       "9                           HBA1C       0.004648        0.06477 -7.384013e-02   \n",
       "10                      Intercept            NaN       0.008008 -2.189691e-15   \n",
       "11                      height_cm            NaN       0.008008  8.948497e-02   \n",
       "\n",
       "                                              formula       p_covar         F  \\\n",
       "0   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  7.777428e-03  10.95402   \n",
       "1   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  4.348971e-06  10.95402   \n",
       "2   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  1.330580e-06  10.95402   \n",
       "3   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  1.874410e-07  10.95402   \n",
       "4   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  3.977037e-02  10.95402   \n",
       "5   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  3.760615e-03  10.95402   \n",
       "6   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  4.933511e-01  10.95402   \n",
       "7   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  5.609913e-01  10.95402   \n",
       "8   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  1.510933e-02  10.95402   \n",
       "9   inv_simpson ~ age + C(sex) + BMI_CALC + bowel_...  3.223246e-04  10.95402   \n",
       "10                            inv_simpson ~ height_cm  4.469390e-06  21.14109   \n",
       "11                            inv_simpson ~ height_cm  4.469390e-06  21.14109   \n",
       "\n",
       "          F_pval  \n",
       "0   1.875719e-10  \n",
       "1   1.875719e-10  \n",
       "2   1.875719e-10  \n",
       "3   1.875719e-10  \n",
       "4   1.875719e-10  \n",
       "5   1.875719e-10  \n",
       "6   1.875719e-10  \n",
       "7   1.875719e-10  \n",
       "8   1.875719e-10  \n",
       "9   1.875719e-10  \n",
       "10  4.469390e-06  \n",
       "11  4.469390e-06  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "arivale_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "04a3ba9b-8d55-405e-8608-7438572474d3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:            inv_simpson   R-squared:                       0.065\n",
      "Model:                            OLS   Adj. R-squared:                  0.061\n",
      "Method:                 Least Squares   F-statistic:                     18.08\n",
      "Date:                Mon, 24 Jun 2024   Prob (F-statistic):           2.51e-32\n",
      "Time:                        14:23:47   Log-Likelihood:                -3630.8\n",
      "No. Observations:                2621   AIC:                             7284.\n",
      "Df Residuals:                    2610   BIC:                             7348.\n",
      "Df Model:                          10                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "=================================================================================================\n",
      "                                    coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------------------------\n",
      "Intercept                         0.4334      0.111      3.892      0.000       0.215       0.652\n",
      "C(sex)[T.M]                      -0.1521      0.057     -2.664      0.008      -0.264      -0.040\n",
      "age                               0.0902      0.020      4.604      0.000       0.052       0.129\n",
      "BMI_CALC                         -0.1073      0.022     -4.846      0.000      -0.151      -0.064\n",
      "bowel_movement_frequency         -0.1874      0.036     -5.225      0.000      -0.258      -0.117\n",
      "vegetable_frequency               0.0461      0.022      2.057      0.040       0.002       0.090\n",
      "height_cm                         0.1920      0.066      2.900      0.004       0.062       0.322\n",
      "vegetable_frequency:height_cm    -0.0151      0.022     -0.685      0.493      -0.058       0.028\n",
      "LDL                              -0.0113      0.019     -0.581      0.561      -0.049       0.027\n",
      "CRP                              -0.0523      0.022     -2.431      0.015      -0.094      -0.010\n",
      "HBA1C                            -0.0738      0.021     -3.601      0.000      -0.114      -0.034\n",
      "==============================================================================\n",
      "Omnibus:                      278.024   Durbin-Watson:                   1.934\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              427.634\n",
      "Skew:                          -0.773   Prob(JB):                     1.38e-93\n",
      "Kurtosis:                       4.236   Cond. No.                         25.0\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
     ]
    }
   ],
   "source": [
    "ari_ols=smf.ols(formula='inv_simpson ~ age + C(sex) + BMI_CALC + bowel_movement_frequency + vegetable_frequency + height_cm + vegetable_frequency:height_cm + LDL + CRP + HBA1C', data=arivale).fit()\n",
    "print(ari_ols.summary())"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}