VAE_Debias.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8db87925-51b4-498f-9b63-f0f3b1dd4cb8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "##load NLST data set with train and tune split\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "# we use train for trainining, and tune for validation\n",
    "tune_path = 'nlst_tune_with_labels.npz' \n",
    "train_path = 'nlst_train_with_labels.npz' \n",
    "\n",
    "def read_embeddings(f):\n",
    "    data = np.load(f,allow_pickle=True)\n",
    "    key = data.files[0]\n",
    "    return pd.DataFrame.from_dict(data[key].item(), orient='index')\n",
    "\n",
    "df_tune = read_embeddings(tune_path)\n",
    "df_train = read_embeddings(train_path)\n",
    "\n",
    "demo=pd.read_csv('nlst_780_prsn_idc_20210527.csv')\n",
    "\n",
    "def get_pid(df):\n",
    "    pid=[int(i.split('/')[1]) for i in df.index]\n",
    "    df['pid']=pid\n",
    "    return df\n",
    "df_tune=get_pid(df_tune)\n",
    "df_train=get_pid(df_train)\n",
    "\n",
    "\n",
    "def add_demo(df):\n",
    "    columns=['age','gender','race']\n",
    "    pid=list(df['pid'])\n",
    "    demo_pid=list(demo['pid'])\n",
    "    indices = [demo_pid.index(x) for x in pid if x in demo_pid]\n",
    "    #print(indices)\n",
    "    selected_rows = demo.iloc[indices]\n",
    "    #print(selected_rows)\n",
    "    selected_columns = selected_rows[columns]\n",
    "    selected_columns.reset_index(drop=True, inplace=True)\n",
    "    print(len(selected_columns),len(df))\n",
    "    df['age']=list(selected_columns['age'])\n",
    "    df['gender']=list(selected_columns['gender'])\n",
    "    df['race']=list(selected_columns['race'])\n",
    "\n",
    "    return df\n",
    "\n",
    "df_tune=add_demo(df_tune)\n",
    "df_train=add_demo(df_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34a0cdf9-3d25-4e3c-a740-e6c71eb88309",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Put sex and age into dataloader\n",
    "import torch\n",
    "from torch.utils.data import DataLoader, TensorDataset\n",
    "    \n",
    "batch_size = 32\n",
    "num_epochs = 100\n",
    "lambda_adv = 0.1  # Trade-off hyperparameter\n",
    "n=100\n",
    "\n",
    "\n",
    "X=np.array(list(df_train['embedding']))[:n]\n",
    "y_sex=np.array(list(df_train['gender']))[:n]-1\n",
    "y_age=np.array(list(df_train['age']))[:n]\n",
    "\n",
    "def get_dataloader(X):\n",
    "    X = torch.Tensor(X)\n",
    "    y=np.vstack((y_sex,y_age)).T\n",
    "    sensitive_attr = torch.Tensor(y) # Sensitive attribute\n",
    "    dataset = TensorDataset(X, sensitive_attr)\n",
    "    dataloader = DataLoader(dataset, batch_size=batch_size, shuffle=True)\n",
    "    return dataloader\n",
    "dataloader=get_dataloader(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "083cd23b-24dd-4340-9deb-7d57cbf7a7f7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "##define model and set training loop\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.distributions.normal import Normal\n",
    "# Reparameterization trick\n",
    "def reparameterize(mean, log_var):\n",
    "    std = torch.exp(0.5*log_var)\n",
    "    epsilon = torch.randn_like(std)\n",
    "    return mean + epsilon*std\n",
    "\n",
    "def train(num_epochs=10,dataloader=dataloader,latent_dim=100,num_sensitives=2):\n",
    "    # Encoder\n",
    "    class Encoder(nn.Module):\n",
    "        def __init__(self, input_dim, latent_dim):\n",
    "            super(Encoder, self).__init__()\n",
    "            self.fc = nn.Linear(input_dim, 2 * latent_dim)  # Mean and log-variance\n",
    "\n",
    "        def forward(self, x):\n",
    "            params = self.fc(x)\n",
    "            mean, log_var = torch.chunk(params, 2, dim=-1)\n",
    "            return mean, log_var\n",
    "\n",
    "    # Decoder\n",
    "    class Decoder(nn.Module):\n",
    "        def __init__(self, latent_dim, output_dim):\n",
    "            super(Decoder, self).__init__()\n",
    "            self.fc = nn.Linear(latent_dim, output_dim)\n",
    "\n",
    "        def forward(self, z):\n",
    "            return self.fc(z)\n",
    "\n",
    "    class MixedAdversary(nn.Module):\n",
    "        def __init__(self, latent_dim, attribute_types):\n",
    "            super(MixedAdversary, self).__init__()\n",
    "            self.branches = nn.ModuleList()\n",
    "            for attr_type in attribute_types:\n",
    "                if attr_type == \"binary\":\n",
    "                    self.branches.append(nn.Linear(latent_dim, 1))  # Binary output\n",
    "                elif attr_type == \"regression\":  # Multiclass with 'attr_type' classes\n",
    "                    self.branches.append(nn.Linear(latent_dim, 1))  # regression output\n",
    "\n",
    "        def forward(self, z):\n",
    "            outputs = []\n",
    "            for branch in self.branches:\n",
    "                outputs.append(branch(z))\n",
    "            return outputs\n",
    "    \n",
    "    # Loss function for VAE with fairness constraint\n",
    "    def vae_loss(recon_x, x, mean, log_var):\n",
    "        recon_loss = nn.MSELoss()(recon_x, x)\n",
    "        kl_div = -0.5 * torch.sum(1 + log_var - mean.pow(2) - log_var.exp())\n",
    "        return recon_loss + kl_div\n",
    "\n",
    "    # Training Loop\n",
    "    encoder = Encoder(input_dim=1408, latent_dim=latent_dim)\n",
    "    decoder = Decoder(latent_dim=latent_dim, output_dim=1408)\n",
    "    attribute_types=['binary',20]\n",
    "    mixed_adversary = MixedAdversary(latent_dim=latent_dim, attribute_types=['binary','regression'])\n",
    "\n",
    "    encoder_optimizer = optim.Adam(encoder.parameters(), lr=0.0005)\n",
    "    decoder_optimizer = optim.Adam(decoder.parameters(), lr=0.0005)\n",
    "    adv_optimizer = optim.Adam(mixed_adversary.parameters(), lr=0.002)\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        for X_batch, sensitive_batch in dataloader:\n",
    "            # Encoder forward pass\n",
    "            mean, log_var = encoder(X_batch)\n",
    "            z = reparameterize(mean, log_var)\n",
    "\n",
    "            # Decoder forward pass\n",
    "            recon_batch = decoder(z)\n",
    "\n",
    "            # Compute VAE loss\n",
    "            vae_loss_value = vae_loss(recon_batch, X_batch, mean, log_var)\n",
    "\n",
    "            # Train adversary (predict sensitive attributes from z)\n",
    "            \n",
    "            adv_preds = mixed_adversary(z.detach())  # Detach z to avoid backprop through adversary\n",
    "\n",
    "            # Compute adversary loss\n",
    "            adv_loss = 0\n",
    "            for i, pred in enumerate(adv_preds):\n",
    "                #print(pred.shape,sensitive_batch[:,i].shape)\n",
    "                if attribute_types[i] == \"binary\":\n",
    "                    adv_loss += nn.BCELoss()(torch.sigmoid(pred), torch.unsqueeze(sensitive_batch[:,i],1))\n",
    "                else:  \n",
    "                    adv_loss += nn.MSELoss()(pred, torch.unsqueeze(sensitive_batch[:,i],1))\n",
    "            #print('--------')\n",
    "            encoder_loss = -adv_loss  # Maximize adversary's loss\n",
    "\n",
    "            # Backpropagate\n",
    "            encoder_optimizer.zero_grad()\n",
    "            decoder_optimizer.zero_grad()\n",
    "            adv_optimizer.zero_grad()\n",
    "\n",
    "            vae_loss_value.backward(retain_graph=True)\n",
    "            encoder_loss.backward()\n",
    "\n",
    "            encoder_optimizer.step()\n",
    "            decoder_optimizer.step()\n",
    "            adv_optimizer.step()\n",
    "\n",
    "        if epoch % 10 == 0:\n",
    "            print(f\"Epoch {epoch}, VAE Loss: {vae_loss_value.item():.4f}, Adv Loss: {adv_loss.item():.4f}\")\n",
    "            \n",
    "    return encoder,decoder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b92c617a-b240-4754-bdd9-0fb734395910",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## train model\n",
    "encoder,decoder=train(num_epochs=100,dataloader=dataloader,latent_dim=500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "09cce67b-ef8c-4fd4-b073-67d47052366e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "## methods for evaluation\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "# Reparameterization trick\n",
    "def reparameterize(mean, log_var):\n",
    "    std = torch.exp(0.5*log_var)\n",
    "    epsilon = torch.randn_like(std)\n",
    "    return mean + epsilon*std\n",
    "\n",
    "def get_classification_res(original_X, recon_X, i):\n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, i)\n",
    "    one=clf1.score(original_X,i)\n",
    "    \n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, i)\n",
    "    two=clf1.score(recon_X,i)\n",
    "    return np.round(one,3),np.round(two,3)\n",
    "\n",
    "def get_regression_res(original_X, recon_X, i):\n",
    "    clf1 = LinearRegression().fit(original_X, i)\n",
    "    y_pred=clf1.predict(original_X)\n",
    "    mae1=mean_absolute_error(i,y_pred)\n",
    "    \n",
    "    clf1 = LinearRegression().fit(recon_X, i)\n",
    "    y_pred=clf1.predict(recon_X)\n",
    "    mae2=mean_absolute_error(i,y_pred)\n",
    "    return np.round(mae1,3),np.round(mae2,3)\n",
    "\n",
    "\n",
    "def evaluate(original_X, recon_X, sensitive_label,label):\n",
    "    attr=['binary','regression']\n",
    "    sl=[]\n",
    "    for index,i in enumerate(sensitive_label):\n",
    "        if attr[index]=='binary':\n",
    "            one,two=get_classification_res(original_X, recon_X, i)\n",
    "            if two<one:\n",
    "                sl.append(['good',one,two])\n",
    "            else:\n",
    "                sl.append(['bad',one,two])\n",
    "        elif attr[index]=='regression':\n",
    "            one,two=get_regression_res(original_X, recon_X, i)\n",
    "            if two>one:\n",
    "                sl.append(['good',one,two])\n",
    "            else:\n",
    "                sl.append(['bad',one,two])\n",
    "\n",
    "    print(sl)\n",
    "    l=[]\n",
    "    for i in label:\n",
    "        one,two=get_classification_res(original_X, recon_X, i)\n",
    "        l.append([one,two])\n",
    "    print(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7aab7143-7f23-456a-90db-6ffd9777fb6e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score,accuracy_score\n",
    "X_test=np.array(list(df_tune['embedding']))\n",
    "y_test=np.array(list(df_tune['gender']))-1\n",
    "y_test_age=np.array(list(df_tune['age']))\n",
    "\n",
    "# Testing the VAE Output\n",
    "encoder.eval()\n",
    "decoder.eval()\n",
    "#mixed_adversary.eval()\n",
    "test_data = torch.Tensor(X_test)\n",
    "with torch.no_grad():\n",
    "    # Test on some data\n",
    "    mean, log_var = encoder(test_data)\n",
    "    z = reparameterize(mean, log_var)\n",
    "    \n",
    "    # Reconstructed data\n",
    "    recon_test_data = decoder(z)\n",
    "\n",
    "y_test_1=np.array(list(df_tune['cancer_in_1']))\n",
    "y_test_2=np.array(list(df_tune['cancer_in_2']))\n",
    "\n",
    "evaluate(X_test, recon_test_data, [y_test,y_test_age],[y_test_1,y_test_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4a64981-f4ed-4e96-9190-421f95ec319e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "#age before and after\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "i=y_test_age\n",
    "clf1 = LinearRegression().fit(original_X, i)\n",
    "pred1=clf1.predict(original_X)\n",
    "    \n",
    "clf1 = LinearRegression().fit(recon_X, i)\n",
    "pred2=clf1.predict(recon_X)\n",
    "\n",
    "plt.rcParams['font.size'] = 20  # Set default font size\n",
    "\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.scatter(i, pred1, alpha=0.5, label='Before', color='b')\n",
    "plt.scatter(i, pred2, alpha=0.5, label='After', color='r')\n",
    "\n",
    "plt.plot([40, 80], [40, 80], 'r--')  # Reference line\n",
    "plt.xlabel('Actual Age')\n",
    "plt.ylabel('Predicted Age')\n",
    "plt.legend()\n",
    "\n",
    "plt.grid(True)\n",
    "plt.savefig('age_scatter_before_and_after.png', dpi=300,bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0202b882-101b-45d6-99c5-51fb4acf98fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score,accuracy_score,roc_curve,auc\n",
    "\n",
    "#sex before and after\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "i=y_test\n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, i)\n",
    "pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "y_pred1=clf1.predict(original_X)\n",
    "\n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, i)\n",
    "pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "y_pred2=clf1.predict(original_X)\n",
    "\n",
    "plt.rcParams['font.size'] = 20  # Set default font size\n",
    "\n",
    "# Calculate ROC curve\n",
    "fpr1, tpr1, thresholds = roc_curve(i, pred1) \n",
    "roc_auc1 = auc(fpr1, tpr1)\n",
    "fpr2, tpr2, thresholds = roc_curve(i, pred2) \n",
    "roc_auc2 = auc(fpr2, tpr2)\n",
    "from sklearn.metrics import roc_auc_score,classification_report,confusion_matrix\n",
    "conf_matrix = confusion_matrix(i, y_pred2)\n",
    "# Per-class accuracy\n",
    "per_class_accuracy = conf_matrix.diagonal() / conf_matrix.sum(axis=1)\n",
    "\n",
    "# Display results\n",
    "for i, acc in enumerate(per_class_accuracy):\n",
    "    print(f\"Accuracy for class {i}: {acc:.2f}\")\n",
    "\n",
    "\n",
    "# Plot the ROC curve\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.plot(fpr1, tpr1, label='ROC curve before (AUC = %0.3f)' % roc_auc1,linewidth=5.0,color=\"blue\")\n",
    "plt.plot(fpr2, tpr2, label='ROC curve after (AUC = %0.3f)' % roc_auc2,linewidth=5.0,color=\"red\")\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'k--', label='Random')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.savefig('sex_roc_before_and_after.png', dpi=300,bbox_inches='tight')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a49153f4-74af-4d58-9d3f-5e4eaeee9bde",
   "metadata": {},
   "outputs": [],
   "source": [
    "#c1 before and after\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "i=y_test_1\n",
    "print(np.unique(i))\n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, i)\n",
    "#print(clf1.score(original_X,i))\n",
    "pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "    \n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, i)\n",
    "#print(clf1.score(recon_X,i))\n",
    "pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "   \n",
    "plt.rcParams['font.size'] = 20  # Set default font size\n",
    "\n",
    "# Calculate ROC curve\n",
    "fpr1, tpr1, thresholds = roc_curve(i, pred1) \n",
    "roc_auc1 = auc(fpr1, tpr1)\n",
    "fpr2, tpr2, thresholds = roc_curve(i, pred2)\n",
    "roc_auc2 = auc(fpr2, tpr2)\n",
    "print(roc_auc1,roc_auc2)\n",
    "# Plot the ROC curve\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.plot(fpr1, tpr1, label='ROC curve before (AUC = %0.3f)' % roc_auc1,linewidth=5.0,color=\"blue\")\n",
    "plt.plot(fpr2, tpr2, label='ROC curve after (AUC = %0.3f)' % roc_auc2,linewidth=5.0,color=\"red\")\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'k--', label='Random')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.savefig('cancer_in_1_roc_before_and_after.png', dpi=300,bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "147b62b5-2860-4390-917e-e2894bc9f624",
   "metadata": {},
   "outputs": [],
   "source": [
    "#c2 before and after\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "i=y_test_2\n",
    "print(np.unique(i))\n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, i)\n",
    "#print(clf1.score(original_X,i))\n",
    "pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "    \n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, i)\n",
    "#print(clf1.score(recon_X,i))\n",
    "pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "   \n",
    "plt.rcParams['font.size'] = 20  # Set default font size\n",
    "\n",
    "# Calculate ROC curve\n",
    "fpr1, tpr1, thresholds = roc_curve(i, pred1) \n",
    "roc_auc1 = auc(fpr1, tpr1)\n",
    "fpr2, tpr2, thresholds = roc_curve(i, pred2) \n",
    "roc_auc2 = auc(fpr2, tpr2)\n",
    "print(roc_auc1,roc_auc2)\n",
    "\n",
    "# Plot the ROC curve\n",
    "plt.figure(figsize=(10, 10))\n",
    "plt.plot(fpr1, tpr1, label='ROC curve before (AUC = %0.3f)' % roc_auc1,linewidth=5.0,color=\"blue\")\n",
    "plt.plot(fpr2, tpr2, label='ROC curve after (AUC = %0.3f)' % roc_auc2,linewidth=5.0,color=\"red\")\n",
    "\n",
    "plt.plot([0, 1], [0, 1], 'k--', label='Random')\n",
    "plt.xlim([0.0, 1.0])\n",
    "plt.ylim([0.0, 1.05])\n",
    "plt.xlabel('False Positive Rate')\n",
    "plt.ylabel('True Positive Rate')\n",
    "plt.legend()\n",
    "plt.savefig('cancer_in_2_roc_before_and_after.png', dpi=300,bbox_inches='tight')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2b27681c-58e1-466e-bd9e-74285192676f",
   "metadata": {},
   "source": [
    "## Additional Experiment: Equal Opportunity Difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0e456d56-36d9-4497-a481-3d1789038971",
   "metadata": {},
   "outputs": [],
   "source": [
    "#c1 before and after\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "i=y_test_1\n",
    "sex=y_test\n",
    "print(np.unique(i))\n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, i)\n",
    "#print(clf1.score(original_X,i))\n",
    "pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "y_pred1=clf1.predict(original_X)\n",
    "  \n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, i)\n",
    "#print(clf1.score(recon_X,i))\n",
    "pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "y_pred2=clf1.predict(recon_X)\n",
    "        \n",
    "print('--------')\n",
    "    \n",
    "from fairlearn.metrics import equal_opportunity_difference,equal_opportunity_ratio\n",
    "\n",
    "print('sex before',equal_opportunity_difference(i,\n",
    "                                y_pred1,\n",
    "                                sensitive_features=sex))\n",
    "print('sex after',equal_opportunity_difference(i,\n",
    "                                y_pred2,\n",
    "                                sensitive_features=sex))\n",
    "print('--------')\n",
    "\n",
    "print('age before',equal_opportunity_difference(i,\n",
    "                                y_pred1,\n",
    "                                sensitive_features=y_test_age))\n",
    "print('age after',equal_opportunity_difference(i,\n",
    "                                y_pred2,\n",
    "                                sensitive_features=y_test_age))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68389cb7-9db2-4792-83d3-d01704cba23f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#c2 before and after\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "i=y_test_2\n",
    "sex=y_test\n",
    "print(np.unique(i))\n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, i)\n",
    "#print(clf1.score(original_X,i))\n",
    "pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "y_pred1=clf1.predict(original_X)\n",
    "  \n",
    "clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, i)\n",
    "#print(clf1.score(recon_X,i))\n",
    "pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "y_pred2=clf1.predict(recon_X)\n",
    "        \n",
    "print('--------')\n",
    "    \n",
    "from fairlearn.metrics import equal_opportunity_difference,equal_opportunity_ratio\n",
    "\n",
    "print('sex before',equal_opportunity_difference(i,\n",
    "                                y_pred1,\n",
    "                                sensitive_features=sex))\n",
    "print('sex after',equal_opportunity_difference(i,\n",
    "                                y_pred2,\n",
    "                                sensitive_features=sex))\n",
    "print('--------')\n",
    "\n",
    "print('age before',equal_opportunity_difference(i,\n",
    "                                y_pred1,\n",
    "                                sensitive_features=y_test_age))\n",
    "print('age after',equal_opportunity_difference(i,\n",
    "                                y_pred2,\n",
    "                                sensitive_features=y_test_age))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b440f9ce-9afc-4474-868a-3753d2432871",
   "metadata": {},
   "source": [
    "## Additioanl Experiment: Data Poisoning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e26f3fb1-2fc7-4198-a9f9-2304e6ca62d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define flipping percentages, male\n",
    "flipping_percentages = [0, 25, 50, 75, 100]\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "sex=y_test\n",
    "i=y_test_1\n",
    "import copy\n",
    "\n",
    "# Function to flip labels\n",
    "def flip_labels(y, indices, flip_percentage):\n",
    "    n = len(indices)\n",
    "    n_to_flip = int(n * flip_percentage / 100)  # Number of labels to flip\n",
    "    flip_indices = np.random.choice(indices, size=n_to_flip, replace=False)  # Randomly select indices\n",
    "    flipped_y = copy.deepcopy(y)\n",
    "    flipped_y[flip_indices] = 1 - flipped_y[flip_indices]  # Flip labels (0 <-> 1)\n",
    "    return flipped_y, flip_indices\n",
    "\n",
    "# Get male indices\n",
    "male_indices = np.where(sex == 0)[0]\n",
    "female_indices = np.where(sex == 1)[0]\n",
    "\n",
    "\n",
    "# Perform flipping for each percentage\n",
    "for percentage in flipping_percentages:\n",
    "    flipped_y, flipped_indices = flip_labels(i, male_indices, percentage)\n",
    "    print(flipped_y.shape,i.shape)\n",
    "    \n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, flipped_y)\n",
    "    pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "    y_pred1=clf1.predict(original_X)\n",
    "\n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, flipped_y)\n",
    "    pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "    y_pred2=clf1.predict(recon_X)\n",
    "\n",
    "    #print(roc_auc_score(i, pred1),roc_auc_score(i, pred2))\n",
    "    from fairlearn.metrics import equal_opportunity_difference,equal_opportunity_ratio\n",
    "\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred1,\n",
    "                                    sensitive_features=sex))\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred2,\n",
    "                                    sensitive_features=sex))\n",
    "    print('------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1104992-a1c2-4113-9eb8-224fc207dc69",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define flipping percentages, female\n",
    "flipping_percentages = [0, 25, 50, 75, 100]\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "sex=y_test\n",
    "i=y_test_1\n",
    "import copy\n",
    "\n",
    "# Function to flip labels\n",
    "def flip_labels(y, indices, flip_percentage):\n",
    "    n = len(indices)\n",
    "    n_to_flip = int(n * flip_percentage / 100)  # Number of labels to flip\n",
    "    flip_indices = np.random.choice(indices, size=n_to_flip, replace=False)  # Randomly select indices\n",
    "    flipped_y = copy.deepcopy(y)\n",
    "    flipped_y[flip_indices] = 1 - flipped_y[flip_indices]  # Flip labels (0 <-> 1)\n",
    "    return flipped_y, flip_indices\n",
    "\n",
    "# Get male indices\n",
    "male_indices = np.where(sex == 0)[0]\n",
    "female_indices = np.where(sex == 1)[0]\n",
    "\n",
    "\n",
    "# Perform flipping for each percentage\n",
    "for percentage in flipping_percentages:\n",
    "    flipped_y, flipped_indices = flip_labels(i, female_indices, percentage)\n",
    "    print(flipped_y.shape,i.shape)\n",
    "    \n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, flipped_y)\n",
    "    pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "    y_pred1=clf1.predict(original_X)\n",
    "\n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, flipped_y)\n",
    "    pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "    y_pred2=clf1.predict(recon_X)\n",
    "\n",
    "    #print(roc_auc_score(i, pred1),roc_auc_score(i, pred2))\n",
    "    from fairlearn.metrics import equal_opportunity_difference,equal_opportunity_ratio\n",
    "\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred1,\n",
    "                                    sensitive_features=sex))\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred2,\n",
    "                                    sensitive_features=sex))\n",
    "    print('------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ca7f129c-a05a-41dc-b541-a58c4ec3e279",
   "metadata": {},
   "source": [
    "## cancer in 2 years"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9101668-0655-4ad4-a8de-5b6a950957ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define flipping percentages, male\n",
    "flipping_percentages = [0, 25, 50, 75, 100]\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "sex=y_test\n",
    "i=y_test_2\n",
    "import copy\n",
    "\n",
    "# Function to flip labels\n",
    "def flip_labels(y, indices, flip_percentage):\n",
    "    n = len(indices)\n",
    "    n_to_flip = int(n * flip_percentage / 100)  # Number of labels to flip\n",
    "    flip_indices = np.random.choice(indices, size=n_to_flip, replace=False)  # Randomly select indices\n",
    "    flipped_y = copy.deepcopy(y)\n",
    "    flipped_y[flip_indices] = 1 - flipped_y[flip_indices]  # Flip labels (0 <-> 1)\n",
    "    return flipped_y, flip_indices\n",
    "\n",
    "# Get male indices\n",
    "male_indices = np.where(sex == 0)[0]\n",
    "female_indices = np.where(sex == 1)[0]\n",
    "\n",
    "\n",
    "# Perform flipping for each percentage\n",
    "for percentage in flipping_percentages:\n",
    "    flipped_y, flipped_indices = flip_labels(i, male_indices, percentage)\n",
    "    print(flipped_y.shape,i.shape)\n",
    "    \n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, flipped_y)\n",
    "    pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "    y_pred1=clf1.predict(original_X)\n",
    "\n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, flipped_y)\n",
    "    pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "    y_pred2=clf1.predict(recon_X)\n",
    "\n",
    "    from fairlearn.metrics import equal_opportunity_difference,equal_opportunity_ratio\n",
    "\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred1,\n",
    "                                    sensitive_features=sex))\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred2,\n",
    "                                    sensitive_features=sex))\n",
    "    print('------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a179bce6-66f7-4993-8b0a-49a799d17cff",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define flipping percentages, female\n",
    "flipping_percentages = [0, 25, 50, 75, 100]\n",
    "original_X=X_test\n",
    "recon_X=recon_test_data\n",
    "sex=y_test\n",
    "i=y_test_2\n",
    "import copy\n",
    "\n",
    "# Function to flip labels\n",
    "def flip_labels(y, indices, flip_percentage):\n",
    "    n = len(indices)\n",
    "    n_to_flip = int(n * flip_percentage / 100)  # Number of labels to flip\n",
    "    flip_indices = np.random.choice(indices, size=n_to_flip, replace=False)  # Randomly select indices\n",
    "    flipped_y = copy.deepcopy(y)\n",
    "    flipped_y[flip_indices] = 1 - flipped_y[flip_indices]  # Flip labels (0 <-> 1)\n",
    "    return flipped_y, flip_indices\n",
    "\n",
    "# Get male indices\n",
    "male_indices = np.where(sex == 0)[0]\n",
    "female_indices = np.where(sex == 1)[0]\n",
    "\n",
    "\n",
    "# Perform flipping for each percentage\n",
    "for percentage in flipping_percentages:\n",
    "    flipped_y, flipped_indices = flip_labels(i, female_indices, percentage)\n",
    "    print(flipped_y.shape,i.shape)\n",
    "    \n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(original_X, flipped_y)\n",
    "    pred1=clf1.predict_proba(original_X)[:, 1]\n",
    "    y_pred1=clf1.predict(original_X)\n",
    "\n",
    "    clf1 = LogisticRegression(random_state=0,max_iter=1000).fit(recon_X, flipped_y)\n",
    "    pred2=clf1.predict_proba(recon_X)[:, 1]\n",
    "    y_pred2=clf1.predict(recon_X)\n",
    "\n",
    "\n",
    "    from fairlearn.metrics import equal_opportunity_difference,equal_opportunity_ratio\n",
    "\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred1,\n",
    "                                    sensitive_features=sex))\n",
    "    print(equal_opportunity_difference(i,\n",
    "                                    y_pred2,\n",
    "                                    sensitive_features=sex))\n",
    "    print('------')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "421cb301-eed5-4555-9c60-aa14a898c87b",
   "metadata": {},
   "source": [
    "## TSNE Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a36ad68-7ed6-45ba-a02c-2bba2d2771f6",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.manifold import TSNE\n",
    "X_embedded = TSNE(n_components=2, learning_rate='auto',\n",
    "                  init='random', perplexity=100).fit_transform(recon_test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c4a49807-2c1e-43b0-8eda-80a0775785e0",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "y_train=np.array(list(df_train['gender']))-1\n",
    "\n",
    "x=X_embedded[:,0]\n",
    "y=X_embedded[:,1]\n",
    "labels=y_test\n",
    "# Create a figure and axis\n",
    "plt.figure(figsize=(10, 10))\n",
    "\n",
    "# Create a scatter plot for each label category\n",
    "unique_labels = np.unique(labels)\n",
    "colors = plt.cm.viridis(np.linspace(0, 1, len(unique_labels)))  # Generate colors\n",
    "\n",
    "for label, color in zip(unique_labels, colors):\n",
    "    if label==0:\n",
    "        t='Male'\n",
    "    else:\n",
    "        t='Female'\n",
    "    plt.scatter(x[labels == label], y[labels == label], color=color, label=t, edgecolor='k', s=100)\n",
    "\n",
    "\n",
    "# Add titles and labels\n",
    "plt.title('Gender')\n",
    "plt.xlabel('feature 1')\n",
    "plt.ylabel('feature 2')\n",
    "plt.legend()\n",
    "\n",
    "# Show plot\n",
    "plt.savefig('gender.png', dpi=300,bbox_inches='tight')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "57871a87-c00d-4a79-9233-270643c98996",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}