notebooks/3_machine_learning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. General Code\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   ## 1.1 Libraries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries to exploratory data analysis\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from packages.common_functions import load_csv_into_dataframe, load_dataframe_to_csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries to preprocessing and enrichment data\n",
    "#Used to scale data\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "#Used to reduction dimension of data set\n",
    "from sklearn.decomposition import PCA\n",
    "#used to creates pipelines from grid search\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import libraries to machine learning\n",
    "#Used on regression models\n",
    "from sklearn.linear_model import LinearRegression, LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "# Used on classification models\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "\n",
    "# Used to get metrics of ml models\n",
    "from sklearn.metrics import mean_squared_error, accuracy_score, r2_score\n",
    "\n",
    "# Used to get bes params for ml models\n",
    "from sklearn.model_selection import GridSearchCV, train_test_split\n",
    "\n",
    "# Used to export bests ml models\n",
    "import joblib"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   ## 1.2 Machine learning functions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "def grid_search_function(model, param_grid, X_train, y_train, pca_components=[0.50, 0.99], cv=2):\n",
    "        \"\"\"\n",
    "        Perandom_forestorm a grid search with cross-validation to find the best\n",
    "        parameters for a given machine learning model.\n",
    "\n",
    "        Parameters:\n",
    "        - model: Machine learning model.\n",
    "        - param_grid (dict): Dictionary of parameters to try in the grid search.\n",
    "        - X (array-like): Training data.\n",
    "        - y (array-like): Training labels.\n",
    "        - pca_components (list, optional): List of principal components to try in PCA.\n",
    "        Default: [0.50, 0.99]\n",
    "        - cv (int, optional): Number of splits for cross-validation. Default: 2\n",
    "\n",
    "        Returns:\n",
    "        tuple: A tuple containing the best parameters, best score, best estimator, and\n",
    "        cross-validation results.\n",
    "        \"\"\"\n",
    "        # Set uo the pipeline with function of PCA and scaling and ml model\n",
    "        pipeline = Pipeline([\n",
    "            ('scaler', StandardScaler()),\n",
    "            ('pca', PCA()),\n",
    "            ('classify', model)\n",
    "        ])\n",
    "\n",
    "        # used grid search with cross validation using previous pipeline and grid params of specific model\n",
    "        grid_search = GridSearchCV(pipeline, param_grid, cv=cv, n_jobs=-1)\n",
    "        grid_search.fit(X_train, y_train)\n",
    "        \n",
    "        # Obtain bes params from specific model\n",
    "        best_params = grid_search.best_params_\n",
    "\n",
    "        # delete 'classify__' prefix from all keys \n",
    "        best_params = {key.replace('classify__', '', 1): value for key, value in best_params.items()}\n",
    "        \n",
    "        return best_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, best_params = None):\n",
    "        # grid search for obtain best params to specific model\n",
    "        if best_params != None:\n",
    "            # establish best params into specific model\n",
    "            model.set_params(**best_params)\n",
    "        # train the model\n",
    "\n",
    "        return model.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, is_classification=False):\n",
    "        # evaluation classification or regression model\n",
    "        y_pred = model.predict(X_test)\n",
    "        if is_classification:\n",
    "            acc = accuracy_score(y_test, y_pred)\n",
    "            print(f\"Precisión del modelo: {acc}\")\n",
    "        else:\n",
    "            mse = mean_squared_error(y_test, y_pred)\n",
    "            r2 = r2_score(y_test, y_pred)\n",
    "            print(f'Mean Squared Error en el conjunto de prueba: {mse:.2f}')\n",
    "            print(f'Coeficiente de Determinación (R²): {r2:.2f}')\n",
    "        \n",
    "        return y_pred"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "   ## 1.3 Load Data Frames\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Path of codec df\n",
    "CODEC_DF_PATH = '../data/processed_csv/2014_census_codec.csv'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSV file successfully load in DataFrame.\n"
     ]
    }
   ],
   "source": [
    "# load codec df in variable 'df'\n",
    "df = load_csv_into_dataframe(CODEC_DF_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter by category in 'CIUDAD' column is different of NULL value and your equivalent convention used before codec column\n",
    "df = df[(df['CIUDAD'].notnull()) & (df['CIUDAD'] != 4)].copy()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Machine Learning Stage"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.1 Target Column: City"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1.1 Data Separation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate the columns on X and y (target column)\n",
    "target_column = 'CIUDAD'\n",
    "X = df.copy().drop(target_column, axis='columns')\n",
    "y = df[target_column].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate data on train and test per X and y dfs\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.35, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1.2 Regression Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.2.1 Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create linear regression model object from ModeloML class\n",
    "linear_regression_model = LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params from linear regression model\n",
    "linear_regression_model_params_grid = {\n",
    "        'classify__fit_intercept': [True, False],  # Indica si debe ajustarse o no el intercepto\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtains best params from linear regression model using grid search\n",
    "linear_regression_model_best_params_grid = grid_search_function(linear_model, param_grid=linear_regression_model_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.2.1.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Not specify best params from linear regression model\n",
    "linear_model = train(linear_model, best_params=linear_regression_model_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.2.1.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error en el conjunto de prueba: 0.00\n",
      "Coeficiente de Determinación (R²): 1.00\n"
     ]
    }
   ],
   "source": [
    "# Evaluation of the lineal regression model\n",
    "y_pred = test(linear_model, is_classification=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.2.2 Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create logistic regression model object\n",
    "logistic_reg_model = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hyper params from linear regression model\n",
    "logistic_reg_params_grid = {\n",
    "    'classify__penalty': ['l1', 'l2'],\n",
    "    'classify__C': [0.1, 1, 10],\n",
    "    'classify__fit_intercept': [True, False],\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/model_selection/_validation.py:425: FitFailedWarning: \n",
      "12 fits failed out of a total of 24.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "12 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 732, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/base.py\", line 1151, in wrapper\n",
      "    return fit_method(estimator, *args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/pipeline.py\", line 420, in fit\n",
      "    self._final_estimator.fit(Xt, y, **fit_params_last_step)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/base.py\", line 1151, in wrapper\n",
      "    return fit_method(estimator, *args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1168, in fit\n",
      "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 56, in _check_solver\n",
      "    raise ValueError(\n",
      "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/model_selection/_search.py:976: UserWarning: One or more of the test scores are non-finite: [nan  1. nan  1. nan  1. nan  1. nan  1. nan  1.]\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# obtains best params from linear regression model using grid search\n",
    "logistic_reg_best_params_grid = grid_search_function(logistic_reg_model, param_grid=logistic_reg_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.2.2.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train lineal regression model with best params previous obtains with grid search \n",
    "logistic_reg_model = train(logistic_reg_model, best_params=logistic_reg_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.2.2.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error en el conjunto de prueba: 3.36\n",
      "Coeficiente de Determinación (R²): -1.88\n"
     ]
    }
   ],
   "source": [
    "# evaluation lineal regression model\n",
    "y_pred = test(logistic_reg_model, is_classification=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1.3 Classification Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " #### 2.1.3.1 Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create support vector machine model object from ModeloML class\n",
    "svm_model = SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params from support vector machine model\n",
    "svm_params_grid = {\n",
    "    'classify__C': [0.1, 1, 10],  \n",
    "    'classify__kernel': ['linear', 'rbf']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain best params from support vector machine model using grid search\n",
    "svm_best_params = grid_search_function(svm_model, param_grid=svm_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.1.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train support vector machine model with best params \n",
    "svm_model = train(svm_model, svm_best_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.1.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 1.0\n"
     ]
    }
   ],
   "source": [
    "# testing support vector machine model\n",
    "y_pred = test(svm_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.2 Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create decision tree model object by ModeloML class\n",
    "decision_tree_model = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params for decision tree model\n",
    "decision_tree_model_params_grid = {\n",
    "    'classify__max_depth': [None, 10, 20],  # Ajusta estos valores\n",
    "    'classify__min_samples_split': [2, 5, 10],\n",
    "    'classify__min_samples_leaf': [1, 2, 4]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [],
   "source": [
    "#obtain best params grid to decision tree model using grid search\n",
    "decision_tree_model_best_params_grid = grid_search_function(decision_tree_model, param_grid=decision_tree_model_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.2.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 179,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train decision tree model with best params\n",
    "decision_tree_model = train(decision_tree_model, decision_tree_model_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.2.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 1.0\n"
     ]
    }
   ],
   "source": [
    "# test decision tree model\n",
    "y_pred = test(decision_tree_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.3 Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create random forest model object by ModeloML class\n",
    "random_forest_model = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params for random forest model\n",
    "random_forest_params_grid = {\n",
    "    'classify__max_depth': [None, 5, 10],  # Reducir la profundidad del árbol\n",
    "    'classify__min_samples_split': [2, 5, 10],  # Aumentar min_samples_split\n",
    "    'classify__min_samples_leaf': [1, 2, 4],  # Aumentar min_samples_leaf\n",
    "    'classify__ccp_alpha': [0.0, 0.1, 0.2]  # Añadir ccp_alpha\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 183,
   "metadata": {},
   "outputs": [],
   "source": [
    "#obtain best params from random forest model with grid search\n",
    "random_forest_best_params_grid = grid_search_function(model= random_forest_model, param_grid=random_forest_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.3.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 184,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train random forest model with best params\n",
    "random_forest_model = train(random_forest_model, random_forest_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.3.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 1.0\n"
     ]
    }
   ],
   "source": [
    "# test random forest model\n",
    "y_pred = test(random_forest_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.4 K-Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create knn model by ModeloML class\n",
    "knn_model = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params for knn model\n",
    "knn_params_grid = {\n",
    "    'classify__n_neighbors': [3, 5, 7],  # Número de vecinos a considerar\n",
    "    'classify__weights': ['uniform', 'distance'],  # Peso dado a los vecinos (uniforme o ponderado por la inversa de la distancia)\n",
    "    'classify__metric': ['euclidean', 'manhattan'],  # Métrica de distancia utilizada\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain best params from knn model with grid search\n",
    "knn_best_params_grid = grid_search_function(model=knn_model, param_grid=knn_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.4.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train knn model with best params\n",
    "knn_model = train(knn_model, best_params=knn_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.4.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 1.0\n"
     ]
    }
   ],
   "source": [
    "# test knn model\n",
    "y_pred = test(knn_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1.4 Best ML Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.4.1 Saved Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# No model obtained the necessary precision to be used in imputation."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.2 Target Column: Education Level"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CSV file successfully load in DataFrame.\n"
     ]
    }
   ],
   "source": [
    "# load dataset pre-encode into data frame 'df'\n",
    "df = load_csv_into_dataframe(CODEC_DF_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter df without null values \n",
    "df = df[df['ULTIMO_NIVEL_EDUCATIVO'] != -1]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2.1 Data Separation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate columns \n",
    "# target column\n",
    "target_column = 'ULTIMO_NIVEL_EDUCATIVO'\n",
    "y = df[target_column].copy()\n",
    "# features columns\n",
    "X = df.copy().drop(target_column, axis='columns')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate feature columns and target column into train and test\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2.2 Regression Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.2.1 Linear Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create logistic regression instance from ModeloML class\n",
    "linear_regression_model = LinearRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params from logistic regression model\n",
    "linear_regression_model_params_grid = {\n",
    "        'classify__fit_intercept': [True, False],  # Indica si debe ajustarse o no el intercepto\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain the best params from logistic regression model with grid search\n",
    "linear_regression_model_best_params_grid = grid_search_function(linear_regression_model, param_grid=linear_regression_model_params_grid,  X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.2.2.1.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train logistic regression model with best params\n",
    "linear_regression_model_best_params_grid = train(linear_regression_model, best_params=linear_regression_model_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.2.2.1.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error en el conjunto de prueba: 3.01\n",
      "Coeficiente de Determinación (R²): 0.12\n"
     ]
    }
   ],
   "source": [
    "# test logistic regression model\n",
    "linear_regression_model = test(linear_regression_model, is_classification=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.2.2.2 Logistic Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create logistic regression model instance from ModeloML class\n",
    "logistic_regression_model = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "# logistic regression params\n",
    "logistic_regression_params_grid = {\n",
    "    'classify__penalty': ['l1', 'l2'], \n",
    "    'classify__C': [0.1, 1, 10], \n",
    "    'classify__fit_intercept': [True, False],  \n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/model_selection/_validation.py:425: FitFailedWarning: \n",
      "12 fits failed out of a total of 24.\n",
      "The score on these train-test partitions for these parameters will be set to nan.\n",
      "If these failures are not expected, you can try to debug them by setting error_score='raise'.\n",
      "\n",
      "Below are more details about the failures:\n",
      "--------------------------------------------------------------------------------\n",
      "12 fits failed with the following error:\n",
      "Traceback (most recent call last):\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/model_selection/_validation.py\", line 732, in _fit_and_score\n",
      "    estimator.fit(X_train, y_train, **fit_params)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/base.py\", line 1151, in wrapper\n",
      "    return fit_method(estimator, *args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/pipeline.py\", line 420, in fit\n",
      "    self._final_estimator.fit(Xt, y, **fit_params_last_step)\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/base.py\", line 1151, in wrapper\n",
      "    return fit_method(estimator, *args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 1168, in fit\n",
      "    solver = _check_solver(self.solver, self.penalty, self.dual)\n",
      "             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py\", line 56, in _check_solver\n",
      "    raise ValueError(\n",
      "ValueError: Solver lbfgs supports only 'l2' or 'none' penalties, got l1 penalty.\n",
      "\n",
      "  warnings.warn(some_fits_failed_message, FitFailedWarning)\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/model_selection/_search.py:976: UserWarning: One or more of the test scores are non-finite: [       nan 0.71696192        nan 0.61042812        nan 0.71954541\n",
      "        nan 0.6106572         nan 0.71990175        nan 0.61064447]\n",
      "  warnings.warn(\n",
      "/Library/Frameworks/Python.framework/Versions/3.11/lib/python3.11/site-packages/sklearn/linear_model/_logistic.py:460: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "source": [
    "# obtain logistic regression best params with grid search\n",
    "logistic_regression_best_params_grid = grid_search_function(logistic_regression_model, logistic_regression_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.2.2.2.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train logistic regression model with best params\n",
    "logistic_regression_model = train(logistic_regression_model, logistic_regression_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.2.2.2.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Squared Error en el conjunto de prueba: 3.57\n",
      "Coeficiente de Determinación (R²): -0.04\n"
     ]
    }
   ],
   "source": [
    "# test logistic regression model\n",
    "y_pred = test(logistic_regression_model, is_classification=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2.3 Classification Models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " #### 2.1.3.1 Support Vector Machines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create support vector machine model object from ModeloML class\n",
    "svm_model = SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params from support vector machine model\n",
    "svm_params_grid = {\n",
    "    'classify__C': [0.1, 1, 10],  \n",
    "    'classify__kernel': ['linear', 'rbf']\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain best params from support vector machine model using grid search\n",
    "svm_best_params = grid_search_function(svm_model, param_grid=svm_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.1.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train support vector machine model with best params \n",
    "svm_model = train(svm_model, svm_best_params)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.1.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 210,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 0.31158055484856195\n"
     ]
    }
   ],
   "source": [
    "# testing support vector machine model\n",
    "y_pred = test(svm_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.2 Decision Tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create decision tree model object by ModeloML class\n",
    "decision_tree_model = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params for decision tree model\n",
    "decision_tree_model_params_grid = {\n",
    "    'classify__max_depth': [None, 10, 20],  # Ajusta estos valores\n",
    "    'classify__min_samples_split': [2, 5, 10],\n",
    "    'classify__min_samples_leaf': [1, 2, 4]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "#obtain best params grid to decision tree model using grid search\n",
    "decision_tree_model_best_params_grid = grid_search_function(decision_tree_model, param_grid=decision_tree_model_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.2.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 214,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train decision tree model with best params\n",
    "decision_tree_model = train(decision_tree_model, decision_tree_model_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.2.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 0.7640621023161109\n"
     ]
    }
   ],
   "source": [
    "# test decision tree model\n",
    "y_pred = test(decision_tree_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.3 Random Forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "#create random forest model object by ModeloML class\n",
    "random_forest_model = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params for random forest model\n",
    "random_forest_params_grid = {\n",
    "    'classify__max_depth': [None, 5, 10],  # Reducir la profundidad del árbol\n",
    "    'classify__min_samples_split': [2, 5, 10],  # Aumentar min_samples_split\n",
    "    'classify__min_samples_leaf': [1, 2, 4],  # Aumentar min_samples_leaf\n",
    "    'classify__ccp_alpha': [0.0, 0.1, 0.2]  # Añadir ccp_alpha\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 218,
   "metadata": {},
   "outputs": [],
   "source": [
    "#obtain best params from random forest model with grid search\n",
    "random_forest_best_params_grid = grid_search_function(model= random_forest_model, param_grid=random_forest_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.3.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 219,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train random forest model with best params\n",
    "random_forest_model = train(random_forest_model, random_forest_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.3.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 220,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 0.780198523797404\n"
     ]
    }
   ],
   "source": [
    "# test random forest model\n",
    "y_pred = test(random_forest_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 2.1.3.4 K-Neighbors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 221,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create knn model by ModeloML class\n",
    "knn_model = KNeighborsClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 222,
   "metadata": {},
   "outputs": [],
   "source": [
    "# params for knn model\n",
    "knn_params_grid = {\n",
    "    'classify__n_neighbors': [3, 5, 7],  # Número de vecinos a considerar\n",
    "    'classify__weights': ['uniform', 'distance'],  # Peso dado a los vecinos (uniforme o ponderado por la inversa de la distancia)\n",
    "    'classify__metric': ['euclidean', 'manhattan'],  # Métrica de distancia utilizada\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 223,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obtain best params from knn model with grid search\n",
    "knn_best_params_grid = grid_search_function(model=knn_model, param_grid=knn_params_grid, X_train=X_train, y_train=y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.4.1 Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train knn model with best params\n",
    "knn_model = train(knn_model, best_params=knn_best_params_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### 2.1.3.4.2 Testing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Precisión del modelo: 0.6786459658946297\n"
     ]
    }
   ],
   "source": [
    "# test knn model\n",
    "y_pred = test(knn_model, is_classification=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2.4 Best ML Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " #### 2.2.4.1 Saved Best Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 227,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['../data/ml_best_model/ultimo_nivel_educativo_random_forest.pkl']"
      ]
     },
     "execution_count": 227,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# export the model that obtain best precision value for this column\n",
    "joblib.dump(random_forest_model, '../data/ml_best_model/ultimo_nivel_educativo_random_forest.pkl')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}