performance-vs-time.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Performance Variation over Time\n",
    "\n",
    "This notebook contains a series of plots designed to demonstrate the performance variation over long periods of time.  This analysis is the same method used to generate Figure 5 in the SC'18 paper, but this notebook is a little more flexible in its ingestion of datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import time\n",
    "import datetime\n",
    "import warnings\n",
    "import matplotlib\n",
    "matplotlib.rcParams.update({'font.size': 18})\n",
    "import pandas\n",
    "import numpy\n",
    "import scipy.stats\n",
    "import abcutils"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load and Synthesize Data from CSV\n",
    "\n",
    "This process loads each summary CSV file, creates a few derived metrics, and then merges each system's CSV into a single global dataset that can be sliced and diced by system, benchmark, or any other way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe = abcutils.sc18paper.load_raw_datasets(\n",
    "    input_datasets={\n",
    "        'cori': 'summaries/cori-summaries_2017-02-14-2019-03-31.csv'\n",
    "    })\n",
    "\n",
    "filtered_df = abcutils.sc18paper.clean_sc18_dataframe(dataframe)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Input Parameters\n",
    "\n",
    "Look at one combination of (compute system, file system) to show what this UMAMI analysis can do."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "TEST_PLATFORM = 'cscratch@cori-knl'\n",
    "\n",
    "plot_metric = 'darshan_normalized_perf_by_max'\n",
    "\n",
    "group_by = ['_test_platform', '_benchmark_id']\n",
    "\n",
    "print(\"test_platform =\", TEST_PLATFORM)\n",
    "print(\"plot_metric =\", abcutils.CONFIG['metric_labels'].get(plot_metric, plot_metric))\n",
    "print(\"date_start =\", abcutils.sc18paper.DATE_START.isoformat())\n",
    "print(\"date_end =\", abcutils.sc18paper.DATE_END.isoformat())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Width of simple moving average (SMA) short/long windows\n",
    "SHORT_WINDOW = pandas.Timedelta(days=14)\n",
    "LONG_WINDOW = (filtered_df['_datetime_start'].max() - filtered_df['_datetime_start'].min()) * 3\n",
    "\n",
    "print(\"Short window will average over %s at a time\" % SHORT_WINDOW)\n",
    "print(\"Long window will average over %s at a time\" % LONG_WINDOW)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Baseline SMA vs Global Mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def draw_fig5(filtered_df, plot_metric, benchmark_ids, labels, draw_region_bounds=0):\n",
    "    \"\"\"Draw Figure 5 from the Year in the Life paper\n",
    "\n",
    "    Args:\n",
    "        filtered_df (pandas.DataFrame): Feature vectors to be plotted\n",
    "        plot_metric (str): Column name of dependent variable (usually performance)\n",
    "        benchmark_ids (list): Benchmark ID strings for benchmarks/systems to draw\n",
    "        labels (list): Logical names to be used to label each benchmark_ids' pane\n",
    "        draw_region_bounds (int): Time in seconds below which region boundaries\n",
    "            should not be drawn.  If zero, do not draw region boundaries.\n",
    "\n",
    "    Returns:\n",
    "        matplotlib.figure.Figure: Figure containing one or more axes with the\n",
    "            requested plots\n",
    "        \"\"\"\n",
    "    nrows = len(benchmark_ids)\n",
    "    fig, axes = matplotlib.pyplot.subplots(nrows=nrows, ncols=1, figsize=(8, 3*nrows), sharex=True)\n",
    "\n",
    "    sma_intercepts_list = []\n",
    "    abs_ymax = None\n",
    "\n",
    "    for index, _benchmark_id in enumerate(benchmark_ids):\n",
    "        _example_df = filtered_df.groupby(by=group_by).get_group((TEST_PLATFORM, _benchmark_id))\n",
    "\n",
    "        sma_short = abcutils.features.calculate_sma(_example_df,\n",
    "                                                    '_datetime_start',\n",
    "                                                    plot_metric,\n",
    "                                                    window=SHORT_WINDOW)\n",
    "        if LONG_WINDOW.days > 365*2:\n",
    "            # use the global mean rather than rely on a sufficiently long window to calculate it--just to be safe!\n",
    "            sma_long = pandas.Series(_example_df[plot_metric].mean(),\n",
    "                                     index=sma_short.index)\n",
    "        else:\n",
    "            sma_long = abcutils.features.calculate_sma(_example_df,\n",
    "                                                       '_datetime_start',\n",
    "                                                       plot_metric,\n",
    "                                                       window=LONG_WINDOW)\n",
    "\n",
    "        sma_intercepts = abcutils.features.find_sma_intercepts(sma_short, sma_long, _example_df['_datetime_start'])\n",
    "        sma_intercepts_list.append(sma_intercepts)\n",
    "\n",
    "        if len(fig.axes) > 1:\n",
    "            ax = axes[index]\n",
    "        else:\n",
    "            ax = axes\n",
    "        abcutils.plot.sma_overlaps(dataframe=_example_df,\n",
    "                                   plot_metric=plot_metric,\n",
    "                                   short_window=SHORT_WINDOW,\n",
    "                                   long_window=LONG_WINDOW,\n",
    "                                   sma_intercepts=sma_intercepts,\n",
    "                                   ax=ax,\n",
    "                                   raw_data_interval=86400)\n",
    "        # Add intercepts\n",
    "        y_min, y_max = ax.get_ylim()\n",
    "        if abs_ymax is None:\n",
    "            abs_ymax = y_max\n",
    "        else:\n",
    "            abs_ymax = max(abs_ymax, y_max)\n",
    "\n",
    "        if draw_region_bounds:\n",
    "            intercept_list = set([])\n",
    "            last_x_val = None\n",
    "            for row in sma_intercepts.itertuples():\n",
    "                x_val = abcutils.core.pd2epoch(row[1])\n",
    "                if last_x_val is not None:\n",
    "                    # only draw intercepts that \n",
    "                    if (x_val - last_x_val) > draw_region_bounds:\n",
    "                        intercept_list.add(last_x_val)\n",
    "                        intercept_list.add(x_val)\n",
    "                last_x_val = x_val\n",
    "            for intercept_x in intercept_list:\n",
    "                        ax.plot([intercept_x, intercept_x], [y_min, y_max], color='black', linestyle='--')\n",
    "\n",
    "    for index, ax in enumerate(fig.axes):\n",
    "        # Restyle the SMA lines\n",
    "        ax.get_lines()[1].set_color('C2')\n",
    "        if LONG_WINDOW.days > 365*2:\n",
    "            ax.get_lines()[1].set_label(\"$SMA_\\infty$\")\n",
    "            ax.get_lines()[0].set_label(\"SMA\")\n",
    "        else:\n",
    "            ax.get_lines()[1].set_label(\"$SMA_{long}$\")# % LONG_WINDOW.days)\n",
    "            ax.get_lines()[0].set_label(\"$SMA_{short}$\")# % SHORT_WINDOW.days)\n",
    "\n",
    "        # Set x ticks\n",
    "        abcutils.plot.fix_xticks_timeseries(ax,\n",
    "                                            format=\"%b %Y\",\n",
    "                                            criteria=lambda x: x.day == 1 and x.month % 2 == 0)\n",
    "\n",
    "        # Set pane label\n",
    "        ax.set_title(labels[index],\n",
    "                     x=0.025,\n",
    "                     y=(0.025),\n",
    "                     fontsize=matplotlib.rcParams.get('font.size'),\n",
    "                     ha='left',\n",
    "                     backgroundcolor='#FFFFFFDD')\n",
    "\n",
    "        # Fix y labels and limits\n",
    "        if nrows > 1:\n",
    "            ax.set_ylabel(\"\")\n",
    "\n",
    "        if 'normalized' in plot_metric:\n",
    "            YMAX = 1\n",
    "            YSTEP = 0.2\n",
    "            ax.set_ylim(0, YMAX)\n",
    "            if index == 0:\n",
    "                ax.set_yticks(numpy.arange(0, YMAX + YSTEP, YSTEP))\n",
    "            else:\n",
    "                ax.set_yticks(numpy.arange(0, YMAX, YSTEP))\n",
    "        else:\n",
    "            ax.set_ylim(0, abs_ymax)\n",
    "            ax.yaxis.set_major_locator(matplotlib.ticker.AutoLocator())\n",
    "            print(\"Setting ymax to %f\" % abs_ymax)\n",
    "\n",
    "        # Recolor bars and remove patches\n",
    "        for bar in [rect for rect in ax.get_children() if isinstance(rect, matplotlib.patches.Rectangle)]:\n",
    "            if bar.get_width() == 86400:\n",
    "                pass\n",
    "            else:\n",
    "                bar.set_visible(False)\n",
    "\n",
    "    print(\"Showing\", abcutils.CONFIG['metric_labels'].get(plot_metric, plot_metric))\n",
    "    print(\"Test platform:\", TEST_PLATFORM)\n",
    "    print(\"SMA window:\", SHORT_WINDOW)\n",
    "    fig.subplots_adjust(hspace=0.0, wspace=0.0)\n",
    "    \n",
    "    # Draw the legend\n",
    "    fig.axes[-1].legend()\n",
    "\n",
    "    if nrows > 1:\n",
    "        fig.text(0.02, 0.5,\n",
    "                 \"Fraction Peak Performance\",\n",
    "                 verticalalignment='center',\n",
    "                 horizontalalignment='center',\n",
    "                 rotation='vertical',\n",
    "                 fontsize=matplotlib.rcParams.get('font.size'))\n",
    "\n",
    "    return fig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "benchmark_ids_list = [\n",
    "    ['ior_shared_write', 'ior_shared_read', '(a) IOR/s write', '(b) IOR/s read'],\n",
    "    ['ior_fpp_write', 'ior_fpp_read', '(a) IOR/fpp write', '(b) IOR/fpp read'],\n",
    "    ['vpicio_uni_shared_write', 'dbscan_read_shared_read', '(a) VPIC write', '(b) BDCATS read'],\n",
    "    ['hacc_io_write_fpp_write', 'hacc_io_read_fpp_read', '(a) HACC write', '(b) HACC read'],\n",
    "]\n",
    "for benchmark_ids in benchmark_ids_list:\n",
    "    fig = draw_fig5(filtered_df=filtered_df,\n",
    "                     plot_metric='darshan_agg_perf_by_slowest_posix_gibs',\n",
    "                     benchmark_ids=benchmark_ids[0:2],\n",
    "                     labels=benchmark_ids[2:4])\n",
    "\n",
    "    output_file = \"figs/longterm-%s-%s.pdf\" % (TEST_PLATFORM.split('@', 1)[0], benchmark_ids[0].split('_')[0])\n",
    "    fig.savefig(output_file, bbox_inches='tight')\n",
    "    print(\"Saved to\", output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = draw_fig5(filtered_df=filtered_df,\n",
    "                plot_metric=plot_metric,\n",
    "                benchmark_ids=['hacc_io_write_fpp_write'],\n",
    "                labels=[\"HACC write on Cori\"],\n",
    "                draw_region_bounds=30*86400)\n",
    "fig.set_size_inches(8, 4)\n",
    "ax = fig.axes[0]\n",
    "ax.set_ylabel(ax.get_ylabel().replace(\"\\n\", \" \"))\n",
    "\n",
    "output_file = \"figs/longterm-%s-hacc-write.pdf\" % (TEST_PLATFORM.split('@', 1)[0])\n",
    "fig.savefig(output_file, bbox_inches='tight')\n",
    "print(\"Saved to\", output_file)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Difference between SMAs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(12,4))\n",
    "\n",
    "#for index, _benchmark_id in enumerate(['hacc_io_write_fpp_write', 'hacc_io_read_fpp_read']):\n",
    "for index, _benchmark_id in enumerate(filtered_df['_benchmark_id'].unique()):\n",
    "    _example_df = filtered_df.groupby(by=group_by).get_group((TEST_PLATFORM, _benchmark_id))\n",
    "\n",
    "    sma_short = abcutils.features.calculate_sma(_example_df,\n",
    "                                                '_datetime_start',\n",
    "                                                plot_metric,\n",
    "                                                window=SHORT_WINDOW)\n",
    "    if LONG_WINDOW.days > 365*2:\n",
    "        # use the global mean rather than rely on a sufficiently long window to calculate it--just to be safe!\n",
    "        sma_long = pandas.Series(_example_df[plot_metric].mean(),\n",
    "                                 index=sma_short.index)\n",
    "    else:\n",
    "        sma_long = abcutils.features.calculate_sma(_example_df,\n",
    "                                                   '_datetime_start',\n",
    "                                                   plot_metric,\n",
    "                                                   window=LONG_WINDOW)\n",
    "\n",
    "    x = sma_short.index.map(lambda x: int(time.mktime(x.timetuple()) / 86400) * 86400)\n",
    "    y = sma_short - sma_long\n",
    "\n",
    "    ax.grid()\n",
    "    ax.plot(x, y, label=abcutils.config.CONFIG['benchmark_labels_short'].get(_benchmark_id, _benchmark_id))\n",
    "    ax.set_ylabel(\"Fraction Peak Performance\\n$SMA_{short} - SMA_{long}$\")\n",
    "    abcutils.plot.fix_xticks_timeseries(ax, format=\"%b %d, %Y\")\n",
    "\n",
    "ax.legend(bbox_to_anchor=(1.05, 1.00), loc='upper left')\n",
    "ax.grid()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Simple Moving Deviation\n",
    "\n",
    "The following plot answers the question of if performance _variation_ changes over time."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, ax = matplotlib.pyplot.subplots(figsize=(12,4))\n",
    "\n",
    "for index, _benchmark_id in enumerate(filtered_df['_benchmark_id'].unique()):\n",
    "    _example_df = filtered_df.groupby(by=group_by).get_group((TEST_PLATFORM, _benchmark_id))\n",
    "\n",
    "    # calculate the simple moving standard deviation (SMSD?)\n",
    "    x_column = '_datetime_start'\n",
    "    y_column = plot_metric\n",
    "    window = SHORT_WINDOW\n",
    "    half_window = window / 2.0\n",
    "    indices = []\n",
    "    values = []\n",
    "    for index, start, _ in _example_df[[x_column, y_column]].itertuples():\n",
    "        window_start = start - half_window\n",
    "        window_end = start + half_window\n",
    "        window_df = _example_df[_example_df[x_column] >= window_start]\n",
    "        window_df = window_df[window_df[x_column] < window_end]\n",
    "        indices.append(window_df.loc[index][x_column])\n",
    "        values.append(window_df[y_column].std())\n",
    "    smsd_short = pandas.Series(values, index=indices, name=y_column).sort_index()\n",
    "\n",
    "    x = smsd_short.index.map(lambda x: int(time.mktime(x.timetuple()) / 86400) * 86400)\n",
    "    y_filt = _example_df['_datetime_start'] > _example_df['_datetime_start'].max() - SHORT_WINDOW\n",
    "    y = smsd_short - _example_df[y_filt][plot_metric].std()\n",
    "#   y = smsd_short\n",
    "\n",
    "    ax.grid()\n",
    "    ax.plot(x, y, label=abcutils.config.CONFIG['benchmark_labels_short'].get(_benchmark_id, _benchmark_id))\n",
    "    ax.set_ylabel(\"Sliding stdev relative to\\nstdev over last %s-day window$\" % SHORT_WINDOW.days)\n",
    "    abcutils.plot.fix_xticks_timeseries(ax, format=\"%b %d, %Y\")\n",
    "\n",
    "ax.legend(bbox_to_anchor=(1.05, 1.00), loc='upper left')\n",
    "ax.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}