best_roster.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import gzip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with gzip.GzipFile('data.json.gz','r') as fp:\n",
    "    data = json.loads(fp.read())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data['teams'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['players'][0].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getdiff(x):\n",
    "    res = {}\n",
    "    try:\n",
    "        for k in x:\n",
    "            ok = 'opp' + k[0].upper() + k[1:] \n",
    "            if ok in x:\n",
    "                if k[:2] in ['fg','ft','tp'] and (len(k) < 3 or k[2] != 'a'):\n",
    "                    res[k + '%'] = 100*(x[k]/max(1,x[k[:2]+'a' + k[2:]])) #- x[ok]/max(1,x[ok[:5]+'a' + ok[5:]]))\n",
    "                elif k == 'pts':\n",
    "                    res[k] = x[k] - x[ok]\n",
    "                else:\n",
    "                    res[k] = x[k] #- x[ok]\n",
    "    except:\n",
    "        print(k)\n",
    "        raise\n",
    "    return res\n",
    "data2 = sum([[getdiff(_) for _ in team['stats'] if not _['playoffs']] for team in data['teams']],[])\n",
    "d2 = sum([[_['won'] for _ in team['seasons']] for team in data['teams']],[])\n",
    "for i in range(len(d2)):\n",
    "    data2[i]['wins'] = d2[i]\n",
    "getdiff(data['teams'][0]['stats'][1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('fivethirtyeight')\n",
    "plt.scatter(df[df.wins>0].pts/82,df[df.wins>0].wins/82,s=8,alpha=0.1)\n",
    "plt.plot([-20,20],[-20* 0.0221+0.5,20* 0.0221+0.5],c='k',label='2.21% per Pt')\n",
    "plt.grid(True)\n",
    "plt.title('BBGM 750 Seasons')\n",
    "plt.xlabel('Avg MOV')\n",
    "plt.ylabel('Win%')\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.api as sm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sm.OLS(df[df.wins>0].wins/82,df[df.wins>0].pts/82).fit().summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = df[[_ for _ in df.columns if _ not in ['wins','pts']]]\n",
    "y = df['pts']/50\n",
    "X = (X-X.mean())/X.std()\n",
    "X = sm.add_constant(X)\n",
    "clf = sm.OLS(y,X)\n",
    "clf_res = clf.fit()\n",
    "clf_res.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coeff = clf_res.params\n",
    "coeff.iloc[coeff.abs().argsort()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['players'][0]['ratings']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['players'][0]['stats'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "skillset = set()\n",
    "for p in data['players']:\n",
    "    for s in p['ratings']:\n",
    "        skillset = skillset.union(set(s['skills']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data3 = {}\n",
    "for p in data['players']:\n",
    "    pdata = {}\n",
    "    #pdata['pid'] = p['pid']\n",
    "    for s in p['stats']:\n",
    "        if s['playoffs']:\n",
    "            continue\n",
    "        pdata[s['season']] = {'team':s['tid']}\n",
    "        pdata[s['season']]['min'] = s['min']\n",
    "        pdata[s['season']]['per'] = s['per']\n",
    "        pdata[s['season']]['ows'] = s['ows']\n",
    "        pdata[s['season']]['dws'] = s['dws']\n",
    "        #pdata[s['season']]['ewa'] = s['ewa']\n",
    "        #pdata[s['season']]['drtg'] = s['drtg']\n",
    "        #pdata[s['season']]['ortg'] = s['ortg']\n",
    "\n",
    "    for r in p['ratings']:\n",
    "        ud = pdata.get(r['season'],{})\n",
    "        ud.update(r)\n",
    "        pdata[r['season']] = ud\n",
    "    for s in pdata:\n",
    "        for c in ['G','C','F']:\n",
    "            pdata[s][c+'_min'] = float(c in pdata[s]['pos'])\n",
    "        for skill in skillset:\n",
    "            pdata[s][skill+'_smin'] = float(skill in pdata[s]['skills'])\n",
    "        del pdata[s]['fuzz']\n",
    "        del pdata[s]['pos']\n",
    "        del pdata[s]['pot']\n",
    "        del pdata[s]['season']\n",
    "        del pdata[s]['skills']\n",
    "    data3[p['pid']] = pdata"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_n = list(data3[0][2020].keys())\n",
    "data4=[]\n",
    "for p in data3:\n",
    "    for s in data3[p]:\n",
    "        data4.append([s]+[data3[p][s].get(_,np.nan) for _ in data_n])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data4,columns=['season'] + data_n)\n",
    "df = df.dropna()\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for col in df.columns:\n",
    "    if col in ['season','team','min']:\n",
    "        continue\n",
    "    df[col] = df[col]*df['min']\n",
    "df_r = df.groupby(['season','team']).sum()#/df['min']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_r2 = df_r.reset_index()\n",
    "for col in df_r2:\n",
    "    if col in ['season','team','min']:\n",
    "        continue\n",
    "    df_r2[col] = df_r2[col]/np.maximum(1,df_r2['min'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#df_pr = df_r2[df_r2 >0].dropna(how='all')\n",
    "df_r2 = df_r2[df_r2.season < df_r2.season.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#data['teams'][0]#['seasons'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d2 = sum([[_['won'] for _ in team['seasons']] for team in data['teams']],[])\n",
    "d3 = sum([[team['tid'] for _ in team['seasons']] for team in data['teams']],[])\n",
    "d4 = sum([[_['season'] for _ in team['seasons']] for team in data['teams']],[])\n",
    "\n",
    "for i in range(len(d2)):\n",
    "    data2[i]['wins'] = d2[i]\n",
    "    data2[i]['team'] = d3[i]\n",
    "    data2[i]['season'] = d4[i]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df[df.season < df.season.max()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_r2['season'] = df_r2['season'].astype(int)\n",
    "df_r2['team'] = df_r2['team'].astype(int)\n",
    "dt = df_r2.set_index(['season','team']).join(df.set_index(['season','team']),rsuffix='_r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = dt[[_ for _ in dt.columns if 'min' not in _ and _ not in ['ovr','per','dws','ows','min','err','pred','wins','pts','fg%', 'fga','fgAtRim%', 'fgaAtRim', 'fgLowPost%', 'fgaLowPost', 'fgMidRange%',  'fgaMidRange', 'tp%', 'tpa', 'ft%', 'fta', 'orb', 'drb_r', 'ast', 'tov',  'stl', 'blk', 'pf']]]\n",
    "y = dt['pts']/50\n",
    "if False:\n",
    "    ss = preprocessing.PolynomialFeatures()\n",
    "    X2 = np.nan_to_num(ss.fit_transform(X))\n",
    "    X2 = (X2-X2.mean())/X2.std()\n",
    "\n",
    "    X2 = pd.DataFrame(X2,columns=ss.get_feature_names(X.columns))\n",
    "    X2.index = X.index\n",
    "    X = X2\n",
    "else:\n",
    "    X = (X-X.mean())/X.std()\n",
    "    X = sm.add_constant(X)\n",
    "\n",
    "\n",
    "clf = sm.OLS(y,X)\n",
    "clf_res = clf.fit()\n",
    "\n",
    "legit_vals = []\n",
    "for v in clf_res.pvalues.iteritems():\n",
    "    if v[1] < 0.05:\n",
    "        legit_vals.append(v[0])\n",
    "#clf_res.summary()\n",
    "if False:\n",
    "    clf = sm.RLM(y,X[legit_vals])\n",
    "    clf_res = clf.fit()\n",
    "clf_res.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import scipy.stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('fivethirtyeight')\n",
    "plt.style.use('seaborn-white')\n",
    "plt.scatter(dt['ovr'],dt['pts']/50,s=1,alpha=0.4)\n",
    "plt.xlabel('overall (min weighted)')\n",
    "plt.ylabel('point margin')\n",
    "plt.title('r squared: {:.3f}'.format(scipy.stats.pearsonr(dt['ovr']/50,dt['pts'])[0]**2))\n",
    "plt.grid(True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.style.use('fivethirtyeight')\n",
    "plt.style.use('seaborn-white')\n",
    "plt.scatter(dt['pts']/50,clf_res.predict(),s=1,alpha=0.4)\n",
    "plt.xlabel('actual margin')\n",
    "plt.ylabel('predicted margin')\n",
    "plt.title('r squared: {:.3f}'.format(scipy.stats.pearsonr(dt['pts']/50,clf_res.predict())[0]**2))\n",
    "plt.grid(True)\n",
    "plt.xlim(-40,40)\n",
    "plt.ylim(-40,40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt['pred'] = clf_res.predict()\n",
    "dt['err'] = (dt['pts']/50- clf_res.predict())#/np.maximum(1,abs(dt['pts']/50))\n",
    "dt['err'] = (dt['err']-dt['err'].mean())/dt['err'].std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt[dt.err > 3.4][['ovr','pts','err','pred']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['teams'][21]['name']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dt.iloc[:,:-21]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coeff = clf_res.params\n",
    "coeff.iloc[coeff.abs().argsort()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"display.precision\", 2)\n",
    "clf_res.params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_roster(filename):\n",
    "    base = json.load(open(filename,'rb'))\n",
    "\n",
    "    \n",
    "    for i in range(len(base['players'])):\n",
    "        player = base['players'][i]\n",
    "        if 'name' in player and player['name'] !='':\n",
    "            sname = base['players'][i]['name'].split()\n",
    "            base['players'][i]['firstName'] = sname[0]\n",
    "            base['players'][i]['lastName'] = ' '.join(sname[1:])\n",
    "            del base['players'][i]['name']\n",
    "    players = base['players']\n",
    "    print(base['version'])\n",
    "    base['version'] = 33\n",
    "\n",
    "    return players,base"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "players,base = load_roster('alexnoob.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rt = [_['ratings'][0] for _ in players]\n",
    "name = [_['firstName'] + ' ' + _['lastName'] for _ in players]\n",
    "for r,n in zip(rt,name):\n",
    "    r['name'] = n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN = pd.DataFrame(rt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r =0.159 * (dfAN.hgt - 47.5) + \\\n",
    "    0.0777 * (dfAN.stre - 50.2) + \\\n",
    "    0.123 * (dfAN.spd - 50.8) + \\\n",
    "    0.051 * (dfAN.jmp - 48.7) + \\\n",
    "    0.0632 * (dfAN.endu - 39.9) + \\\n",
    "    0.0126 * (dfAN.ins - 42.4) + \\\n",
    "    0.0286 * (dfAN.dnk - 49.5) + \\\n",
    "    0.0202 * (dfAN.ft - 47.0) + \\\n",
    "    0.0726 * (dfAN.tp - 47.1) + \\\n",
    "    0.133 * (dfAN.oiq - 46.8) + \\\n",
    "    0.159 * (dfAN.diq - 46.7) + \\\n",
    "    0.059 * (dfAN.drb - 54.8) + \\\n",
    "    0.062 * (dfAN.pss - 51.3) + \\\n",
    "    0.01 * (dfAN.fg - 47.0) + \\\n",
    "    0.01 * (dfAN.reb - 51.4) +48.5;\n",
    "dfAN['ovr'] = r"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN.sort_values('ovr',0,False).head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = dfAN['hgt'] * coeff['hgt']\n",
    "for k,v in coeff.items():\n",
    "    if k == 'const':\n",
    "        tmp += v\n",
    "    elif k == 'hgt':\n",
    "        continue\n",
    "    else:\n",
    "        tmp += dfAN[k]*v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN['new_ovr'] = tmp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf_oc = sm.OLS(dfAN.ovr,sm.add_constant(dfAN.new_ovr)).fit()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN['new_ovr'] = clf_oc.predict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(4,4))\n",
    "plt.scatter(dfAN.ovr,dfAN.new_ovr)\n",
    "plt.grid(True)\n",
    "plt.plot([35,70],[35,70],c='k')\n",
    "plt.xlim(30,75)\n",
    "plt.xlabel('current ovr')\n",
    "plt.ylabel('team MOV ovr')\n",
    "plt.title('ovr tests (alexnoob roster)\\n non-scaled ovr')\n",
    "plt.ylim(30,75)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN['diff'] = dfAN['new_ovr'] - dfAN['ovr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN.sort_values('diff').tail(50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(dfAN['diff'],20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = dfAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4 = df3.set_index('name').join(dfAN.set_index('name'),rsuffix='_auto',how='outer')#.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4['AUTOdiff'] =df4['ovr_auto']- df4['ovr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4.to_csv('testlol.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df4.sort_values('AUTOdiff',0,False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}