TabularAE.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 390
    },
    "id": "I4PczoTMNXi3",
    "outputId": "c9842093-2082-44ee-fac6-c191946c1e77"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Name: fastai\n",
      "Version: 2.0.8\n",
      "Summary: fastai simplifies training fast and accurate neural nets using modern best practices\n",
      "Home-page: https://github.com/fastai/fastai\n",
      "Author: Jeremy Howard, Sylvain Gugger, and contributors\n",
      "Author-email: info@fast.ai\n",
      "License: Apache Software License 2.0\n",
      "Location: /mnt/c/work/ml/fastai\n",
      "Requires: pip, packaging, fastcore, torchvision, matplotlib, pandas, requests, pyyaml, fastprogress, pillow, scikit-learn, scipy, spacy, pandas, torch\n",
      "Required-by: \n",
      "---\n",
      "Name: fastcore\n",
      "Version: 1.0.1\n",
      "Summary: Python supercharged for fastai development\n",
      "Home-page: https://github.com/fastai/fastcore\n",
      "Author: Jeremy Howard and Sylvain Gugger\n",
      "Author-email: infos@fast.ai\n",
      "License: Apache Software License 2.0\n",
      "Location: /home/etienne/miniconda3/envs/fastai/lib/python3.8/site-packages\n",
      "Requires: packaging, wheel, pip, numpy, dataclasses\n",
      "Required-by: nbdev, fastai2, fastai\n"
     ]
    }
   ],
   "source": [
    "!pip show fastai fastcore"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "pS3C4jJLEdOV"
   },
   "outputs": [],
   "source": [
    "from matplotlib import cm\n",
    "from fastai.tabular.all import *\n",
    "\n",
    "pd.set_option('display.float_format', lambda x: '%.3f' % x)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Pu3gjQ7CNMSv"
   },
   "source": [
    "We'll use the `Adult Sample` dataset:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "UmzTzqPTEttQ"
   },
   "outputs": [],
   "source": [
    "path = untar_data(URLs.ADULT_SAMPLE)\n",
    "df = pd.read_csv(path/'adult.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "8bCOcb3DNSVd"
   },
   "source": [
    "And declare the relevent information:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "2YqxE0AVEdOX"
   },
   "outputs": [],
   "source": [
    "cat_names = ['workclass', 'education', 'marital-status', 'occupation', 'relationship', 'race']\n",
    "cont_names = ['age', 'fnlwgt', 'education-num']\n",
    "procs = [Categorify, FillMissing, Normalize]\n",
    "y_names = 'salary'\n",
    "y_block = CategoryBlock()\n",
    "splits = RandomSplitter()(range_of(df))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fUIkMhCENnLd"
   },
   "source": [
    "Next we need our own version of `ReadTabBatch` that will return our inputs\n",
    "\n",
    "> The continous variables are still normalized if we used `Normalize`. Couldn't figure out an easy way to de-norm it, but it's okay that we do not"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "08quVRZuEdOc"
   },
   "outputs": [],
   "source": [
    "class ReadTabBatchIdentity(ItemTransform):\n",
    "    \"Read a batch of data and return the inputs as both `x` and `y`\"\n",
    "    def __init__(self, to): store_attr()\n",
    "\n",
    "    def encodes(self, to):\n",
    "        if not to.with_cont: res = (tensor(to.cats).long(),) + (tensor(to.cats).long(),)\n",
    "        else: res = (tensor(to.cats).long(),tensor(to.conts).float()) + (tensor(to.cats).long(), tensor(to.conts).float())\n",
    "        if to.device is not None: res = to_device(res, to.device)\n",
    "        return res\n",
    "    \n",
    "class TabularPandasIdentity(TabularPandas): pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "itUJ_YaoN6uT"
   },
   "source": [
    "Next we need to make a new `TabDataLoader` that uses our `RadTabBatchIdentity`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "C0YUhcd3EdOe"
   },
   "outputs": [],
   "source": [
    "@delegates()\n",
    "class TabDataLoaderIdentity(TabDataLoader):\n",
    "    \"A transformed `DataLoader` for AutoEncoder problems with Tabular data\"\n",
    "    do_item = noops\n",
    "    def __init__(self, dataset, bs=16, shuffle=False, after_batch=None, num_workers=0, **kwargs):\n",
    "        if after_batch is None: after_batch = L(TransformBlock().batch_tfms)+ReadTabBatchIdentity(dataset)\n",
    "        super().__init__(dataset, bs=bs, shuffle=shuffle, after_batch=after_batch, num_workers=num_workers, **kwargs)\n",
    "\n",
    "    def create_batch(self, b): return self.dataset.iloc[b]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PX9Vv9OPOAVl"
   },
   "source": [
    "And make `TabularPandasIdentity`'s `dl_type` to `TabDataLoaderIdentity`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "q5EEsDr-1S5_"
   },
   "outputs": [],
   "source": [
    "TabularPandasIdentity._dl_type = TabDataLoaderIdentity"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tXmzs8Hx1WQd"
   },
   "source": [
    "To start we'll make a very basic `to` object using our new `TabularPandasIdentity`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "4sWPDJNy1aK1"
   },
   "outputs": [],
   "source": [
    "to = TabularPandasIdentity(df, [Categorify, FillMissing, Normalize], cat_names, cont_names, splits=RandomSplitter(seed=32)(df))\n",
    "dls = to.dataloaders(bs=1024)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "-kcUFTYoOGO0"
   },
   "source": [
    "Set the `n_inp` to 2:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "uQDN7OpH1b9B"
   },
   "outputs": [],
   "source": [
    "dls.n_inp = 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ojWg3NDI1iXx"
   },
   "source": [
    "And then we'll calculate the embedding sizes:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "id": "NERxSEdK1jyv"
   },
   "outputs": [],
   "source": [
    "emb_szs = get_emb_sz(to.train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "L5neKs1q1kuF"
   },
   "source": [
    "For each categorical variable we need to know the total possible values it can have:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 134
    },
    "id": "Uv7-oVlPEdOg",
    "outputId": "bdfd5add-2f30-423a-aedc-42d564110157"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'workclass': 10,\n",
       " 'education': 17,\n",
       " 'marital-status': 8,\n",
       " 'occupation': 16,\n",
       " 'relationship': 7,\n",
       " 'race': 6,\n",
       " 'education-num_na': 3}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_cats = {k:len(v) for k,v in to.classes.items()}\n",
    "total_cats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tKNiQtro13Dk"
   },
   "source": [
    "We will need this dictionary in our loss function to figure out where to apply our `CrossEntropyLossFlat` for each categorical variables"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "mpyLIJsO2DwA"
   },
   "source": [
    "Next we need to know the total number ouf outputs possible for our categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "u56gAbBYEdOi",
    "outputId": "e8a87efe-6dc2-48c8-87b8-15a2c088922d"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "67"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum([v for k,v in total_cats.items()])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2BjAP1OVOYsQ"
   },
   "source": [
    "And let's keep a batch of our data for later"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "id": "RAqcnZI2EdOl"
   },
   "outputs": [],
   "source": [
    "batch = dls.one_batch()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "t5c7Vi-f2aLc"
   },
   "source": [
    "Next we need to know the means and standard deviations:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "id": "rvbLzGcV2cDP",
    "outputId": "f17df585-8136-4869-ec7c-d263e3052d26"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'age': 38.5793696495067,\n",
       " 'fnlwgt': 190006.02011593536,\n",
       " 'education-num': 10.079158782958984}"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to.means"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QhHUG2Sk2p42"
   },
   "source": [
    "We can store them in a `DataFrame` for easy adjustments:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "ttuLY8iWEdOn"
   },
   "outputs": [],
   "source": [
    "means = pd.DataFrame.from_dict({k:[v] for k,v in to.means.items()})\n",
    "stds = pd.DataFrame.from_dict({k:[v] for k,v in to.stds.items()})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k5rHaP4N2yIl"
   },
   "source": [
    "We'll also use a SigmoidRange based on the un-normalized data to reduce the range our values can be:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "lqoao9U5EdOo"
   },
   "outputs": [],
   "source": [
    "low = (df[cont_names].min().to_frame().T.values - means.values) / stds.values\n",
    "high = (df[cont_names].max().to_frame().T.values - means.values) / stds.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "id": "2DcGJCze3Kqa",
    "outputId": "77bf0d20-aef1-426b-eeda-7c969c73cac7"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([[-1.57952443, -1.67843578, -3.55622464]]),\n",
       " array([[ 3.76378659, 12.22741736,  2.31914013]]))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "low, high"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "hu8pZ9oSEdOq"
   },
   "source": [
    "## Batch Swap Noise\n",
    "Used in the winning solution for the Kaggle competition [Puerto Seguro Safe Driver Prediction](https://www.kaggle.com/c/porto-seguro-safe-driver-prediction/discussion/44629#250927)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "NZb6cetaEdOr"
   },
   "outputs": [],
   "source": [
    "class BatchSwapNoise(Module):\n",
    "    \"Swap Noise Module\"\n",
    "    def __init__(self, p): store_attr()\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.training:\n",
    "            mask = torch.rand(x.size()) > (1 - self.p)\n",
    "            l1 = torch.floor(torch.rand(x.size()) * x.size(0)).type(torch.LongTensor)\n",
    "            l2 = (mask.type(torch.LongTensor) * x.size(1))\n",
    "            res = (l1 * l2).view(-1)\n",
    "            idx = torch.arange(x.nelement()) + res\n",
    "            idx[idx>=x.nelement()] = idx[idx>=x.nelement()]-x.nelement()\n",
    "            return x.flatten()[idx].view(x.size())\n",
    "        else:\n",
    "            return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "QjduDB3JOpfi"
   },
   "source": [
    "We'll make a custom `TabularAE` model (AutoEncoder) for us to use."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "0moqukV_EdOs"
   },
   "outputs": [],
   "source": [
    "class TabularAE(TabularModel):\n",
    "    \"A simple AutoEncoder model\"\n",
    "    def __init__(self, emb_szs, n_cont, hidden_size, cats, low, high, ps=0.2, embed_p=0.01, bswap=None):\n",
    "        super().__init__(emb_szs, n_cont, layers=[1024, 512, 256], out_sz=hidden_size, embed_p=embed_p, act_cls=Mish())\n",
    "        \n",
    "        self.bswap = bswap\n",
    "        self.cats = cats\n",
    "        self.activation_cats = sum([v for k,v in cats.items()])\n",
    "        \n",
    "        self.layers = nn.Sequential(*L(self.layers.children())[:-1] + nn.Sequential(LinBnDrop(256, hidden_size, p=ps, act=Mish())))\n",
    "        \n",
    "        if(bswap != None): self.noise = BatchSwapNoise(bswap)\n",
    "        self.decoder = nn.Sequential(\n",
    "            LinBnDrop(hidden_size, 256, p=ps, act=Mish()),\n",
    "            LinBnDrop(256, 512, p=ps, act=Mish()),\n",
    "            LinBnDrop(512, 1024, p=ps, act=Mish())\n",
    "        )\n",
    "        \n",
    "        self.decoder_cont = nn.Sequential(\n",
    "            LinBnDrop(1024, n_cont, p=ps, bn=False, act=None),\n",
    "            SigmoidRange(low=low, high=high)\n",
    "        )\n",
    "        \n",
    "        self.decoder_cat = LinBnDrop(1024, self.activation_cats, p=ps, bn=False, act=None)\n",
    "        \n",
    "    def forward(self, x_cat, x_cont=None, encode=False):\n",
    "        if(self.bswap != None):\n",
    "            x_cat = self.noise(x_cat)\n",
    "            x_cont = self.noise(x_cont)\n",
    "        encoded = super().forward(x_cat, x_cont)\n",
    "        if encode: return encoded # return the representation\n",
    "        decoded_trunk = self.decoder(encoded)\n",
    "        decoded_cats = self.decoder_cat(decoded_trunk)\n",
    "        decoded_conts = self.decoder_cont(decoded_trunk)\n",
    "        return decoded_cats, decoded_conts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vO2gBJNoO3hP"
   },
   "source": [
    "We'll also need a loss function that can grade how well our features represent the original dataset. \n",
    "\n",
    "The categorical features will be graded on `CrossEntropyLossFlat` and the continous with `MSELossFlat`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "24aYC4fk4Qev"
   },
   "outputs": [],
   "source": [
    "class RecreatedLoss(Module):\n",
    "    \"Measures how well we have created the original tabular inputs\"\n",
    "    def __init__(self, cat_dict, reduction='mean'):\n",
    "        ce = CrossEntropyLossFlat(reduction='none')\n",
    "        mse = MSELossFlat(reduction='none')\n",
    "        store_attr('cat_dict,ce,mse,reduction')\n",
    "\n",
    "    def forward(self, preds, cat_targs, cont_targs):\n",
    "        cats, conts = preds\n",
    "        tot_ce, pos = [], 0\n",
    "        for i, (k,v) in enumerate(self.cat_dict.items()):\n",
    "            tot_ce += [self.ce(cats[:, pos:pos+v], cat_targs[:,i])]\n",
    "            pos += v\n",
    "            \n",
    "        tot_ce = torch.stack(tot_ce, axis=1).mean(axis=1)\n",
    "        cont_loss = self.mse(conts, cont_targs).view(conts.shape).mean(axis=1)\n",
    "        total_loss = torch.stack([tot_ce, cont_loss], axis=1).sum(axis=1)\n",
    "        \n",
    "        if self.reduction == 'mean':\n",
    "            return total_loss.mean()\n",
    "        elif self.reduction == 'sum':\n",
    "            return total_loss.sum()\n",
    "        \n",
    "        return total_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RV7j8WoePBJO"
   },
   "source": [
    "All we need to do is pass in our `total_cats` dictionary:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "zbsDGXPX_VUV"
   },
   "outputs": [],
   "source": [
    "loss_func = RecreatedLoss(total_cats)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "# b = dls.one_batch()\n",
    "\n",
    "# out = learn.model(*b[:2])\n",
    "# loss = loss_func(out, *b[-2:])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "WrhU2fLP7N-U"
   },
   "source": [
    "We'll make an config dictionary for us to use:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "id": "ThV0ch4x7R58"
   },
   "outputs": [],
   "source": [
    "config = {\n",
    "    'hidden_size': 128,\n",
    "    'dropout': 0.1,\n",
    "    'embed_p': 0.01,\n",
    "    'wd': 0.01,\n",
    "    'bswap': 0.1,\n",
    "    'lr': 1e-3,\n",
    "    'epochs': 100\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "PYgESCPk7TeP"
   },
   "source": [
    "And make our model & `Learner`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "id": "i7xmhROaEdOw"
   },
   "outputs": [],
   "source": [
    "model = TabularAE(emb_szs, len(cont_names), config['hidden_size'], ps=config['dropout'], cats=total_cats, embed_p=config['embed_p'], bswap=config['bswap'], low=tensor(low).cuda(), high=tensor(high).cuda())\n",
    "learn = Learner(dls, model, lr=config['lr'], loss_func=loss_func, wd=config['wd'], opt_func=ranger).to_fp16()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "J76BTkMP7Ws2"
   },
   "source": [
    "Finally we'll fit for a few epochs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 271
    },
    "id": "dcFlPHLAGBZ4",
    "outputId": "07d78f9a-b513-4b89-fc0c-cbfcc02db820"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>7.849534</td>\n",
       "      <td>3.430095</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>3.788902</td>\n",
       "      <td>1.078850</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>2.278958</td>\n",
       "      <td>0.546157</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>1.537396</td>\n",
       "      <td>0.235419</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>1.130815</td>\n",
       "      <td>0.188008</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.889883</td>\n",
       "      <td>0.135979</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.736168</td>\n",
       "      <td>0.142726</td>\n",
       "      <td>00:03</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement since epoch 5: early stopping\n"
     ]
    }
   ],
   "source": [
    "learn.fit_flat_cos(config['epochs'], cbs=[EarlyStoppingCallback()], lr=4e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "k6NIV2T7EdO2"
   },
   "source": [
    "# Getting the compressed representations"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9E9LxZUPPJWX"
   },
   "source": [
    "Next we're going to grade our compressed representations and then attempt to train on them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {
    "id": "bnnQC-jbEdO2"
   },
   "outputs": [],
   "source": [
    "dl = learn.dls.test_dl(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "l5vf0sGyPTlS"
   },
   "source": [
    "Let's predict over all the data manually using PyTorch:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {
    "id": "LG7l2jCVPV5k"
   },
   "outputs": [],
   "source": [
    "outs = []\n",
    "learn.model.eval()\n",
    "learn.model.cuda()\n",
    "for batch in dl:\n",
    "    with torch.no_grad():\n",
    "        out = learn.model(*batch[:2], True).cpu().numpy()\n",
    "        outs.append(out)\n",
    "outs = np.concatenate(outs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(32561, 128)"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outs.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YIjJ0MOjP8ge"
   },
   "source": [
    "As well as get the actual preds and targs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 17
    },
    "id": "_sLsBpFfQADo",
    "outputId": "5a65abdb-6edc-4a94-f4b8-71ae3a9c3de7"
   },
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "(cat_preds, cont_preds), (cat_targs, cont_targs), losses = learn.get_preds(dl=dl, with_loss=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0.3237, 0.0433, 0.2750,  ..., 0.0423, 0.1769, 0.1015])"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ykgzRNmcEdO-"
   },
   "source": [
    "# Measuring accuracy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "SsEY3yF_EdO-"
   },
   "source": [
    "## Continuous"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "fejwMQRbEdO_",
    "outputId": "ce6d5ba6-17a6-4028-9fff-4bbb2f1911a5"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>GroupBy</th>\n",
       "      <th>age</th>\n",
       "      <th>fnlwgt</th>\n",
       "      <th>education-num</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Min</td>\n",
       "      <td>0.000</td>\n",
       "      <td>4.563</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Max</td>\n",
       "      <td>33.577</td>\n",
       "      <td>221926.921</td>\n",
       "      <td>3.517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Mean</td>\n",
       "      <td>2.360</td>\n",
       "      <td>28601.768</td>\n",
       "      <td>0.309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Median</td>\n",
       "      <td>1.939</td>\n",
       "      <td>23106.977</td>\n",
       "      <td>0.251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>R2</td>\n",
       "      <td>0.950</td>\n",
       "      <td>0.873</td>\n",
       "      <td>0.975</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  GroupBy    age     fnlwgt  education-num\n",
       "0     Min  0.000      4.563          0.000\n",
       "0     Max 33.577 221926.921          3.517\n",
       "0    Mean  2.360  28601.768          0.309\n",
       "0  Median  1.939  23106.977          0.251\n",
       "0      R2  0.950      0.873          0.975"
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import r2_score\n",
    "\n",
    "cont_preds = pd.DataFrame(cont_preds, columns=cont_names)\n",
    "cont_targs = pd.DataFrame(cont_targs, columns=cont_names)\n",
    "\n",
    "preds = pd.DataFrame((cont_preds.values * stds.values) + means.values, columns=cont_preds.columns)\n",
    "targets = pd.DataFrame((cont_targs.values * stds.values) + means.values, columns=cont_targs.columns)\n",
    "\n",
    "mi = (np.abs(targets-preds)).min().to_frame().T\n",
    "ma = (np.abs(targets-preds)).max().to_frame().T\n",
    "mean = (np.abs(targets-preds)).mean().to_frame().T\n",
    "median = (np.abs(targets-preds)).median().to_frame().T\n",
    "r2 = pd.DataFrame.from_dict({c:[r2_score(targets[c], preds[c])] for c in preds.columns})\n",
    "\n",
    "\n",
    "for d,name in zip([mi,ma,mean,median,r2], ['Min', 'Max', 'Mean', 'Median', 'R2']):\n",
    "    d = d.insert(0, 'GroupBy', name)\n",
    "    \n",
    "data = pd.concat([mi,ma,mean,median,r2])\n",
    "data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Xgg2KiaTQIre"
   },
   "source": [
    "We can also grab the R2:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "id": "nUVqCx3YEdPB",
    "outputId": "25dcd1b9-7142-4b09-988f-16bba3ea2e6f"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0   0.933\n",
       "dtype: float64"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r2.mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "fXB3fiBrEdPC"
   },
   "source": [
    "## Categorical"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "id": "nOEzxfpREdPF"
   },
   "outputs": [],
   "source": [
    "cat_reduced = torch.zeros_like(cat_targs)\n",
    "pos=0\n",
    "for i, (k,v) in enumerate(total_cats.items()):\n",
    "    cat_reduced[:,i] = cat_preds[:,pos:pos+v].argmax(dim=1)\n",
    "    pos += v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "id": "zlH1IcACEdPH"
   },
   "outputs": [],
   "source": [
    "cat_preds = pd.DataFrame(cat_reduced, columns=cat_names)\n",
    "cat_targs = pd.DataFrame(cat_targs, columns=cat_names)\n",
    "\n",
    "from sklearn.metrics import balanced_accuracy_score, f1_score\n",
    "\n",
    "accuracy = pd.DataFrame.from_dict({c:[balanced_accuracy_score(cat_targs[c], cat_preds[c])] for c in cat_preds.columns})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "id": "ZMqj8zcKEdPJ"
   },
   "outputs": [],
   "source": [
    "f1 = pd.DataFrame.from_dict({c:[f1_score(cat_targs[c], cat_preds[c], average='weighted')] for c in cat_preds.columns})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 143
    },
    "id": "vuBk02t5EdPL",
    "outputId": "432517e9-1e98-44c8-e82b-00dba4c73947"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MetricName</th>\n",
       "      <th>workclass</th>\n",
       "      <th>education</th>\n",
       "      <th>marital-status</th>\n",
       "      <th>occupation</th>\n",
       "      <th>relationship</th>\n",
       "      <th>race</th>\n",
       "      <th>education-num_na</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Accuracy</td>\n",
       "      <td>0.767</td>\n",
       "      <td>0.876</td>\n",
       "      <td>0.756</td>\n",
       "      <td>0.882</td>\n",
       "      <td>0.988</td>\n",
       "      <td>0.736</td>\n",
       "      <td>0.957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>F1</td>\n",
       "      <td>0.995</td>\n",
       "      <td>0.988</td>\n",
       "      <td>0.984</td>\n",
       "      <td>0.986</td>\n",
       "      <td>0.995</td>\n",
       "      <td>0.982</td>\n",
       "      <td>0.999</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  MetricName  workclass  education  ...  relationship  race  education-num_na\n",
       "0   Accuracy      0.767      0.876  ...         0.988 0.736             0.957\n",
       "0         F1      0.995      0.988  ...         0.995 0.982             0.999\n",
       "\n",
       "[2 rows x 8 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for d,name in zip([accuracy, f1], ['Accuracy', 'F1']):\n",
    "    d = d.insert(0, 'MetricName', name)\n",
    "pd.concat([accuracy, f1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "3GcWeUSqQXIG"
   },
   "source": [
    "And check it's overall accuracy:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "id": "2rOtbaScEdPN",
    "outputId": "c685c93a-9a39-4d72-fa6c-f42fae03aa63"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0   0.852\n",
       "dtype: float64"
      ]
     },
     "execution_count": 39,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy.mean(axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "u9K317948SvR"
   },
   "source": [
    "## Predicting\n",
    "\n",
    "Now that we have our compressed representations, let's use them to train a new model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "id": "HAM08-DsJsjh"
   },
   "outputs": [],
   "source": [
    "ys = df['salary'].to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "id": "RRbccF9zKBV5"
   },
   "outputs": [],
   "source": [
    "test_eq(len(outs), len(ys))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "id": "OX0IPAG8KhbL"
   },
   "outputs": [],
   "source": [
    "df_outs = pd.DataFrame(columns=['salary'] + list(range(0,128)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "id": "9KjjmrbfKqIN"
   },
   "outputs": [],
   "source": [
    "df_outs['salary'] = ys"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "id": "gfO5CknNKtXl"
   },
   "outputs": [],
   "source": [
    "df_outs[list(range(0,128))] = outs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "id": "v6mUmbz5Lkqb"
   },
   "outputs": [],
   "source": [
    "pd.options.mode.chained_assignment=None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "id": "l7lAP34pMAvN"
   },
   "outputs": [],
   "source": [
    "splits = RandomSplitter()(range_of(df))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "id": "FGezIL33MVXx"
   },
   "outputs": [],
   "source": [
    "df_outs[list(range(0,128))] = df_outs[list(range(0,128))].astype(np.float16)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "id": "M1MWujHOKwBf"
   },
   "outputs": [],
   "source": [
    "cont_names = list(range(0,128))\n",
    "to = TabularPandas(df_outs, procs = [Normalize], cont_names=cont_names, splits=splits, y_names=['salary'], reduce_memory=False, \n",
    "                   y_block=CategoryBlock())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "id": "G4AKuKfbLgwQ"
   },
   "outputs": [],
   "source": [
    "dls = to.dataloaders(bs=1024)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "id": "NoYN1wi3NaYE"
   },
   "outputs": [],
   "source": [
    "def accuracy(inp, targ, axis=-1):\n",
    "    \"Compute accuracy with `targ` when `pred` is bs * n_classes\"\n",
    "    pred,targ = flatten_check(inp.argmax(dim=axis), targ)\n",
    "    return (pred == targ).float().mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "id": "lhOMRBYCLAZy"
   },
   "outputs": [],
   "source": [
    "learn = tabular_learner(dls, layers=[200,100], metrics=[accuracy])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "id": "UuvlsOh4MlN_",
    "outputId": "64b00442-2801-4bfe-a9ca-d823c3140590"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0.385410</td>\n",
       "      <td>0.361070</td>\n",
       "      <td>0.834152</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.365105</td>\n",
       "      <td>0.351813</td>\n",
       "      <td>0.840602</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.355500</td>\n",
       "      <td>0.351886</td>\n",
       "      <td>0.837684</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.350068</td>\n",
       "      <td>0.350590</td>\n",
       "      <td>0.837377</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.346437</td>\n",
       "      <td>0.354474</td>\n",
       "      <td>0.840909</td>\n",
       "      <td>00:00</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learn.fit(5, 1e-2)"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "TabularAE.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}