notebooks/mwas/bootstrapping-bulk_apply_confidence_intervals_to_results.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bulk-apply confidence intervals calculations to results\n",
    "Apply the calculation of confidence intervals through the `confidence_intervals` package to all model results in respective output folders. This is done by automatically loading the predictions DataFrames and feeding them into the package. The results are saves as aggregated .csv files. Optionally, in the last cell, the column `lower_bound_larger_null` is added to check if the lower bound of the confidence interval is larger than 0."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import audbenchmark\n",
    "from tqdm import tqdm\n",
    "\n",
    "# Import bootstrapping\n",
    "from confidence_intervals import evaluate_with_conf_int"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1) Load the compiled results DataFrames\n",
    "2) Calculate confidence intervals on all \"Path\" given\n",
    "   1) Sort the results acc. to the best declared performance\n",
    "   2) Maybe also highlight models with significant lower bounds\n",
    "3) Save the results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add a meaningful term to summarise the conditions compiled\n",
    "file_out_composed = \"compiled-outer_cv_loso\"\n",
    "\n",
    "lst_conditions = [\n",
    "    \"outer_cv_loso-surveys_all\",\n",
    "    \"outer_cv_loso-survey_daily\",\n",
    "    \"outer_cv_loso-surveys_weekly\",\n",
    "    #\n",
    "    # \"outer_cv_loso-noisy-surveys_all\",\n",
    "    # \"outer_cv_loso-noisy-surveys_daily\",\n",
    "    # \"outer_cv_loso-noisy-surveys_weekly\",\n",
    "    # \"outer_cv_loso-noisy-surveys_weekly-stress_current_raw\",\n",
    "]\n",
    "path_composed = \"../../results/mwas/composed\"\n",
    "path_results_base = \"/data/phecker/project/audiary/projects/prompt-performance/modelling-static/results/mwas/modelling\"\n",
    "metric = audbenchmark.metric.concordance_cc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Conditions:   0%|          | 0/1 [00:00<?, ?it/s]\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "\u001b[A\n",
      "Conditions: 100%|██████████| 1/1 [59:40<00:00, 3580.46s/it]\n"
     ]
    }
   ],
   "source": [
    "# Initialize a list to store the rows with the highest ccc_conf_mean\n",
    "meta_data = []\n",
    "\n",
    "for cur_condition in tqdm(lst_conditions, desc=\"Conditions\"):\n",
    "    path_condition = os.path.join(path_composed, cur_condition)\n",
    "\n",
    "    # Get list of all .csv files in path_condition\n",
    "    csv_files = [f for f in os.listdir(path_condition) if f.endswith(\".csv\")]\n",
    "\n",
    "    # Iterate over all .csv files in path_condition with a progress bar\n",
    "    for file_name in tqdm(csv_files, desc=f\"Targets {cur_condition}\", leave=False):\n",
    "        if file_name.endswith(\".csv\"):\n",
    "            # Construct the full file path\n",
    "            file_path = os.path.join(path_condition, file_name)\n",
    "\n",
    "            # Load the .csv file\n",
    "            df_results = pd.read_csv(file_path)\n",
    "\n",
    "            # Extract the target from the file name\n",
    "            target = file_name.replace(\"results-\", \"\").replace(\".csv\", \"\")\n",
    "\n",
    "            # Initialize lists to store the results\n",
    "            ccc_conf_mean = []\n",
    "            ccc_conf_low = []\n",
    "            ccc_conf_high = []\n",
    "            lower_bound_larger_null = []\n",
    "            tasks = []\n",
    "            features = []\n",
    "\n",
    "            # Iterate over every row in the column \"path\"\n",
    "            for str_path in tqdm(\n",
    "                df_results[\"path\"],\n",
    "                desc=f\"individual results {cur_condition}\",\n",
    "                leave=False,\n",
    "            ):\n",
    "                # Remove the leading forward slash\n",
    "                str_path = str_path.lstrip(\"/\")\n",
    "\n",
    "                # Extract the task and features\n",
    "                path_parts = str_path.split(\"/\")\n",
    "                task = path_parts[2] if len(path_parts) > 2 else \"\"\n",
    "                feature = path_parts[7] if len(path_parts) > 7 else \"\"\n",
    "\n",
    "                # Remove the trailing part starting from \"models/results-compiled.yaml\"\n",
    "                str_path = str_path.split(\"models/results-compiled.yaml\")[0]\n",
    "\n",
    "                # Construct the path to the parquet file\n",
    "                path = os.path.join(\n",
    "                    path_results_base, str_path, \"data/df_results_test.parquet.zstd\"\n",
    "                )\n",
    "\n",
    "                # Load the parquet file\n",
    "                df_test = pd.read_parquet(path)\n",
    "\n",
    "                # Evaluate with confidence intervals\n",
    "                tpl_result = evaluate_with_conf_int(\n",
    "                    samples=df_test[\"predictions\"].values,\n",
    "                    metric=metric,\n",
    "                    labels=df_test[target].values,\n",
    "                    conditions=df_test[\"participant_code\"].values,\n",
    "                    num_bootstraps=1000,\n",
    "                    alpha=5,\n",
    "                )\n",
    "\n",
    "                # Extract the mean and confidence interval values\n",
    "                mean_value = tpl_result[0]\n",
    "                conf_int_low = tpl_result[1][0]\n",
    "                conf_int_high = tpl_result[1][1]\n",
    "\n",
    "                # Append the results to the lists\n",
    "                ccc_conf_mean.append(mean_value)\n",
    "                ccc_conf_low.append(conf_int_low)\n",
    "                ccc_conf_high.append(conf_int_high)\n",
    "                lower_bound_larger_null.append(conf_int_low > 0)\n",
    "                tasks.append(task)\n",
    "                features.append(feature)\n",
    "\n",
    "            # Add the results as new columns to df_results at the beginning\n",
    "            df_results.insert(0, \"ccc_conf_mean\", ccc_conf_mean)\n",
    "            df_results.insert(1, \"ccc_conf_low\", ccc_conf_low)\n",
    "            df_results.insert(2, \"ccc_conf_high\", ccc_conf_high)\n",
    "            df_results.insert(3, \"lower_bound_larger_null\", lower_bound_larger_null)\n",
    "            df_results.insert(4, \"task\", tasks)\n",
    "            df_results.insert(5, \"features\", features)\n",
    "\n",
    "            # Sort df_results by ccc_conf_mean\n",
    "            df_results = df_results.sort_values(by=\"ccc_conf_mean\", ascending=False)\n",
    "\n",
    "            # Find the row with the highest ccc_conf_mean\n",
    "            best_row = df_results.iloc[0].copy()\n",
    "            best_row[\"survey\"] = cur_condition\n",
    "            best_row[\"target\"] = target\n",
    "\n",
    "            # Append the best row to the meta_data list\n",
    "            meta_data.append(best_row)\n",
    "\n",
    "            # Construct the new file path with \"-conf\" appended\n",
    "            new_file_path = file_path.replace(\".csv\", \"-conf.csv\")\n",
    "\n",
    "            # Write the updated DataFrame back to the new file path\n",
    "            df_results.to_csv(new_file_path, index=False)\n",
    "\n",
    "# Convert the meta_data list to a DataFrame\n",
    "df_meta = pd.DataFrame(meta_data)\n",
    "\n",
    "# Ensure the columns \"survey\" and \"target\" are at the beginning\n",
    "columns = [\"survey\", \"target\"] + [col for col in df_meta.columns if col not in [\"survey\", \"target\"]]\n",
    "df_meta = df_meta[columns]\n",
    "\n",
    "# Save the meta DataFrame to path_composed as \"compiled.csv\"\n",
    "compiled_path = os.path.join(path_composed, file_out_composed + \".csv\")\n",
    "df_meta.to_csv(compiled_path, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Conditions: 100%|██████████| 1/1 [00:00<00:00, 26.16it/s]\n"
     ]
    }
   ],
   "source": [
    "# Initialize a list to store the rows with the highest ccc_conf_mean\n",
    "meta_data = []\n",
    "\n",
    "for cur_condition in tqdm(lst_conditions, desc=\"Conditions\"):\n",
    "    path_condition = os.path.join(path_composed, cur_condition)\n",
    "\n",
    "    # Get list of all .csv files that are already processed (\"-conf\") in path_condition\n",
    "    csv_files = [f for f in os.listdir(path_condition) if f.endswith(\"-conf.csv\")]\n",
    "\n",
    "    # Iterate over all .csv files in path_condition with a progress bar\n",
    "    for file_name in tqdm(csv_files, desc=f\"Targets {cur_condition}\", leave=False):\n",
    "        # Construct the full file path\n",
    "        file_path = os.path.join(path_condition, file_name)\n",
    "\n",
    "        # Load the .csv file\n",
    "        df_results = pd.read_csv(file_path)\n",
    "\n",
    "        # Extract the target from the file name\n",
    "        target = file_name.replace(\"results-\", \"\").replace(\"-conf.csv\", \"\")\n",
    "\n",
    "        # Find the first row where \"lower_bound_larger_null\" is True\n",
    "        matching_row = df_results[df_results[\"lower_bound_larger_null\"] == True].head(1)\n",
    "\n",
    "        if not matching_row.empty:\n",
    "            # Add cur_condition and target to the matching row\n",
    "            matching_row.insert(0, \"survey\", cur_condition)\n",
    "            matching_row.insert(1, \"target\", target)\n",
    "            # Append the matching row to meta_data\n",
    "            meta_data.append(matching_row.iloc[0])\n",
    "        else:\n",
    "            # Append an empty row with cur_condition and target\n",
    "            empty_row = pd.Series({\"survey\": cur_condition, \"target\": target})\n",
    "            meta_data.append(empty_row)\n",
    "\n",
    "# Convert the meta_data list to a DataFrame\n",
    "df_meta = pd.DataFrame(meta_data)\n",
    "\n",
    "# Ensure the columns \"survey\" and \"target\" are at the beginning\n",
    "columns = [\"survey\", \"target\"] + [col for col in df_meta.columns if col not in [\"survey\", \"target\"]]\n",
    "df_meta = df_meta[columns]\n",
    "\n",
    "# Save the meta DataFrame to path_composed as \"compiled.csv\"\n",
    "compiled_path = os.path.join(path_composed, file_out_composed + \"-significant.csv\")\n",
    "df_meta.to_csv(compiled_path, index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "prompt_performance-static_modelling-test-devaice",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}