machinelearning/classificationPerformanceMeasures/03-PrecisionRecallTradeoff/precisionRecallTradeoff.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 精准度和召回率的平衡"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.036128Z",
     "start_time": "2024-09-16T09:44:47.732636Z"
    }
   },
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.516299Z",
     "start_time": "2024-09-16T09:44:48.037763Z"
    }
   },
   "source": [
    "from sklearn import datasets\n",
    "\n",
    "digits = datasets.load_digits()\n",
    "X = digits.data\n",
    "y = digits.target.copy()\n",
    "\n",
    "y[digits.target==9] = 1\n",
    "y[digits.target!=9] = 0"
   ],
   "outputs": [],
   "execution_count": 2
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.557796Z",
     "start_time": "2024-09-16T09:44:48.517192Z"
    }
   },
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)"
   ],
   "outputs": [],
   "execution_count": 3
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.621060Z",
     "start_time": "2024-09-16T09:44:48.558958Z"
    }
   },
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "log_reg = LogisticRegression()\n",
    "log_reg.fit(X_train, y_train)\n",
    "y_predict = log_reg.predict(X_test)"
   ],
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/envs/myenv3.10/lib/python3.10/site-packages/sklearn/linear_model/_logistic.py:469: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    }
   ],
   "execution_count": 4
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.630035Z",
     "start_time": "2024-09-16T09:44:48.623437Z"
    }
   },
   "source": [
    "from sklearn.metrics import f1_score\n",
    "\n",
    "f1_score(y_test, y_predict)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.8674698795180723)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 5
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.635696Z",
     "start_time": "2024-09-16T09:44:48.630895Z"
    }
   },
   "source": [
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "confusion_matrix(y_test, y_predict)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[403,   2],\n",
       "       [  9,  36]])"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.641379Z",
     "start_time": "2024-09-16T09:44:48.636644Z"
    }
   },
   "source": [
    "from sklearn.metrics import precision_score\n",
    "\n",
    "precision_score(y_test, y_predict)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.9473684210526315)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 7
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.647216Z",
     "start_time": "2024-09-16T09:44:48.642367Z"
    }
   },
   "source": [
    "from sklearn.metrics import recall_score\n",
    "\n",
    "recall_score(y_test, y_predict)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.8)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.651854Z",
     "start_time": "2024-09-16T09:44:48.648306Z"
    }
   },
   "source": [
    "log_reg.decision_function(X_test)[:10]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-21.39632803, -32.89836755, -16.42503064, -79.82740073,\n",
       "       -48.03006253, -24.18016394, -44.61773888, -24.2382381 ,\n",
       "        -1.1442306 , -19.00687291])"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 9
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.655747Z",
     "start_time": "2024-09-16T09:44:48.652693Z"
    }
   },
   "source": [
    "log_reg.predict(X_test)[:10]"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 0, 0, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.661423Z",
     "start_time": "2024-09-16T09:44:48.656830Z"
    }
   },
   "source": [
    "decision_scores = log_reg.decision_function(X_test)\n",
    "print(decision_scores) # 每个样本的 score 值 "
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-21.39632803 -32.89836755 -16.42503064 -79.82740073 -48.03006253\n",
      " -24.18016394 -44.61773888 -24.2382381   -1.1442306  -19.00687291\n",
      " -65.82579222 -50.97384945 -30.91383804 -45.95001961 -37.36079287\n",
      " -29.52191392 -36.92525489 -82.81936974 -37.64655242  -9.87455985\n",
      "  -9.27181223 -85.26072092 -16.75200555 -45.34187288  -5.02565958\n",
      " -48.30720246 -11.65686706 -37.35115796 -25.08294084 -13.59519777\n",
      " -16.59094522 -28.7868136  -34.37187546 -28.52250796  -8.12118919\n",
      "  -4.61236199 -21.9213878  -21.88101143 -31.15032516 -23.37622916\n",
      " -26.90704067 -62.23845924 -37.68292314 -66.36540349 -20.10762829\n",
      " -16.67567208 -18.17120489 -21.54706516 -28.96499959 -19.61221275\n",
      "   2.42106669   7.72579943 -34.86440796 -42.71126562 -25.63790717\n",
      " -34.75654341  -7.59704356 -49.51935699 -51.52679169  19.66635111\n",
      " -10.10862009 -32.01145468 -11.49933661  -1.44233748 -48.69944223\n",
      " -43.86134828 -24.84126264 -19.60577522 -36.64996545  -3.52583948\n",
      "  -4.45221101 -19.2151373  -20.35656157 -40.89711366 -11.85824617\n",
      " -32.75336536 -35.76186227 -28.58890488 -55.41752828 -18.83061949\n",
      "   4.57091465 -16.463646   -76.78227343 -58.24102588 -30.23791246\n",
      " -29.423417   -33.41427822  -8.41635365 -47.91948007 -65.49892246\n",
      " -16.91630447 -22.17290895 -11.28295144 -18.67186697 -69.22923712\n",
      " -46.39154589 -39.45401269 -35.9335525  -17.7303169  -62.96898278\n",
      " -16.86305036 -55.15268001 -28.77455676 -68.48799082 -68.86724305\n",
      "  -6.50066996 -25.5166632  -38.3237795  -27.4657971  -15.54534665\n",
      " -27.47967996 -20.34240725  12.07585494 -23.09377024 -35.9749607\n",
      " -29.87345938 -68.96188671 -27.31294693 -54.25065148 -24.62891432\n",
      " -11.84731319 -47.36810768  -2.7552612  -59.69176582 -30.99290537\n",
      "  -8.99688404 -70.8437001  -56.98391867 -20.07087563 -21.50466085\n",
      " -68.2885106  -18.91402081 -38.59546834 -57.36443298  -0.91127195\n",
      " -22.5230999  -22.66313156 -29.00758695 -32.76608118 -20.44146722\n",
      " -11.35208442   4.63883739   6.26793819   1.48978716  -7.63611926\n",
      " -39.24568643  12.15963601 -74.54758286 -75.08976825 -49.97540869\n",
      " -11.63726915 -47.61549493 -75.41844924 -29.90338777 -63.94066399\n",
      "  -7.26363237  -6.64195494 -18.2155006  -32.47359246 -17.94096338\n",
      " -43.32139635 -32.70198994 -34.305205   -72.74642491 -15.19453349\n",
      "  11.47711832 -56.41444251  -6.03679672 -48.38174285 -16.44342301\n",
      "  -2.13532057 -11.85856169 -33.2615532  -51.35612964 -10.3854679\n",
      " -17.18931257  -5.24186043 -25.20231775 -15.70408888   3.5512389\n",
      " -45.02515953 -12.57784626 -25.37484849 -16.5760627  -22.16247652\n",
      " -82.49793825  -5.87782175 -20.26924964 -20.46798451 -26.81797463\n",
      " -25.98205563 -40.46368473 -38.00370623 -26.96407202 -23.7556394\n",
      " -20.14896398  -9.68706827 -19.68267804 -42.50630152 -44.1474587\n",
      " -15.65592717 -64.03595061 -24.55492892 -56.30748473 -13.0153844\n",
      " -29.66152689   3.89274429 -44.3372052   -7.91496387   1.14288279\n",
      "  -2.82200976 -11.9298259    7.50919056  -7.17677196 -46.38967231\n",
      " -48.65790438  -4.59703312 -19.04918093 -24.07425902 -48.76420724\n",
      " -15.02118533 -24.92911043 -16.69731748 -18.68076058 -15.70026039\n",
      " -16.86891478 -38.53342161 -31.09474143  -9.38184941 -71.45269781\n",
      " -22.77486554 -14.43346467 -23.07982404 -34.32201506  -0.88829297\n",
      " -32.74800984 -11.22610884 -18.67835381  -8.21183045 -45.4430361\n",
      " -22.3048531  -62.4046439  -46.77074565 -65.15482912 -33.22979898\n",
      " -23.47223796 -28.50164243 -64.79125993   1.45396815  -4.09350604\n",
      " -25.6560368  -22.31931787 -54.69329449 -16.34265394 -12.07699182\n",
      " -35.28803117  -5.74561325 -13.50697395 -72.29504081  -6.16616975\n",
      "  -1.16745264 -35.57074284 -24.16464549 -68.32001285  14.76664432\n",
      " -63.07044866   9.9114665  -24.14614137 -32.45946469 -14.39403155\n",
      " -85.73589062 -12.78449394   8.99700957 -16.51275081 -36.68112125\n",
      " -16.5142318  -19.35489432 -32.59144615  -5.64199251   7.68831562\n",
      "   9.39582472   5.86052677 -35.64855143 -12.98554895 -54.42604517\n",
      " -41.10680277   5.62961814 -79.49290143 -15.82339505 -19.23222952\n",
      " -10.86523966 -42.52431569 -19.82259121 -15.70234888 -17.99450127\n",
      " -18.02770039  -6.75709263 -20.78875363 -16.58248998 -70.42953085\n",
      "  -9.20996399 -31.69729569 -19.68492641 -21.96674718 -24.76965523\n",
      " -16.38407636 -13.36979769 -22.93073434  11.05839158 -15.38201515\n",
      " -32.94035842 -13.74990956 -50.36070675 -20.46520927 -56.27361211\n",
      " -28.68660012 -21.86436781 -30.41316875 -69.26213003 -59.34758793\n",
      "  14.34666127   8.58023128 -25.67055443   2.74201107   4.9349716\n",
      " -19.67145243 -58.83290391 -10.0106847  -28.80840929 -27.20569285\n",
      "   6.29303171 -80.47836457 -34.4594727  -50.2950448  -35.95426284\n",
      " -48.638095   -17.99381876 -62.34571387  -3.09533027 -25.26144012\n",
      " -64.10550641  -9.62433703 -21.75093177  19.90895049 -18.75278074\n",
      "  -4.47036571 -13.15064794 -21.64150504 -43.10209091 -52.11204298\n",
      " -28.5267845  -14.55989702  -2.47300226  -6.12313765   3.69927708\n",
      " -15.00324649 -40.85549694 -26.65311131  14.10932393 -17.69128675\n",
      "  15.19041957 -33.09240839   5.26736074 -14.27034185 -53.58823639\n",
      " -50.03864289 -30.66777869 -38.04959657 -23.29430945 -24.69534374\n",
      " -13.56655803 -22.61684297 -27.22295453 -19.64547188 -28.17465092\n",
      " -19.93511498 -29.83095565 -11.29200116 -17.24466233 -24.03020454\n",
      " -24.35711168  10.3927233  -17.22134301 -38.02469802 -16.08880146\n",
      " -37.58627833 -16.33661862 -69.12719912 -33.68695612 -43.62833897\n",
      " -26.59250139 -10.31848973 -66.36421923 -31.89660769 -45.56274349\n",
      " -14.58211052 -36.12438586 -14.95239118 -70.02153238 -11.35930794\n",
      " -40.86598947 -32.66642135 -19.76342546 -27.57777649 -15.73319443\n",
      " -31.58141356  -8.5131452  -21.37778539 -34.07581735 -11.68164353\n",
      " -36.4319518  -34.77814158 -22.22229751   4.77640054 -21.31951689\n",
      "  -4.45913305 -20.83062634 -32.25695828 -41.12662624 -25.08183831\n",
      " -19.7625071  -47.87267406 -30.91359757 -45.56398272 -71.52302451\n",
      "  -6.25666324 -32.55674222   2.28031179  11.94364146   7.115445\n",
      " -31.37474547 -63.96093128 -23.79015443  -5.7376593  -32.42331007\n",
      " -24.72419264 -67.70498722 -32.82676878 -33.60753273 -31.53836834\n",
      " -51.97739661 -22.54373289  -7.74506607 -17.29730481 -25.78028594\n",
      " -32.38263295 -29.49540535 -66.43502146 -45.69850615 -16.05293553]\n"
     ]
    }
   ],
   "execution_count": 11
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.665244Z",
     "start_time": "2024-09-16T09:44:48.662351Z"
    }
   },
   "source": [
    "np.min(decision_scores)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(-85.73589062374965)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 12
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.670779Z",
     "start_time": "2024-09-16T09:44:48.666938Z"
    }
   },
   "source": [
    "np.max(decision_scores)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(19.908950489969612)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 13
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.676526Z",
     "start_time": "2024-09-16T09:44:48.674396Z"
    }
   },
   "source": [
    "# 使用新的预测值， 默认这个decision_scores =0 ，>=5分类为1，<5分类为0 \n",
    "y_predict_2 = np.array(decision_scores >= 5, dtype='int')"
   ],
   "outputs": [],
   "execution_count": 14
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.681893Z",
     "start_time": "2024-09-16T09:44:48.677709Z"
    }
   },
   "source": [
    "confusion_matrix(y_test, y_predict_2)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[404,   1],\n",
       "       [ 21,  24]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 15
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.687884Z",
     "start_time": "2024-09-16T09:44:48.683104Z"
    }
   },
   "source": [
    "precision_score(y_test, y_predict_2)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.96)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 16
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.693952Z",
     "start_time": "2024-09-16T09:44:48.688880Z"
    }
   },
   "source": [
    "recall_score(y_test, y_predict_2)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.5333333333333333)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 17
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.697095Z",
     "start_time": "2024-09-16T09:44:48.694898Z"
    }
   },
   "source": [
    "# >=-5分类为1  小于-5 分类为0\n",
    "y_predict_3 = np.array(decision_scores >= -5, dtype='int')"
   ],
   "outputs": [],
   "execution_count": 18
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.702119Z",
     "start_time": "2024-09-16T09:44:48.698324Z"
    }
   },
   "source": [
    "confusion_matrix(y_test, y_predict_3)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[390,  15],\n",
       "       [  5,  40]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 19
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.707863Z",
     "start_time": "2024-09-16T09:44:48.703023Z"
    }
   },
   "source": [
    "precision_score(y_test, y_predict_3)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.7272727272727273)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 20
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-09-16T09:44:48.713875Z",
     "start_time": "2024-09-16T09:44:48.708779Z"
    }
   },
   "source": [
    "recall_score(y_test, y_predict_3)"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "np.float64(0.8888888888888888)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 21
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}