src/benchmarks/gc/GC.Infrastructure/Notebooks/VolatilityReport.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Volatility Report: To Quantify Volatility of Multiple Runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "#r \"nuget: Microsoft.Diagnostics.Tracing.TraceEvent, 3.1.13\"\n",
    "#r \"nuget: YamlDotnet\" \n",
    "#r \"nuget: XPlot.Plotly\"\n",
    "#r \"nuget: XPlot.Plotly.Interactive\"\n",
    "#r \"nuget: Microsoft.Data.Analysis, 0.19.1\"\n",
    "#r \"nuget: Newtonsoft.Json\"\n",
    "\n",
    "using Etlx = Microsoft.Diagnostics.Tracing.Etlx;\n",
    "using Microsoft.Data.Analysis;\n",
    "using Microsoft.Diagnostics.Tracing.Analysis.GC;\n",
    "using Microsoft.Diagnostics.Tracing.Analysis;\n",
    "using Microsoft.Diagnostics.Tracing.Parsers.Clr;\n",
    "using Microsoft.Diagnostics.Tracing;\n",
    "using XPlot.Plotly;\n",
    "\n",
    "using System.IO;\n",
    "using Newtonsoft.Json;"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "pwsh"
    },
    "polyglot_notebook": {
     "kernelName": "pwsh"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "dotnet build -c Release \"..\\GC.Analysis.API\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "#r \"C:\\performance\\artifacts\\bin\\GC.Analysis.API\\Release\\net7.0\\GC.Analysis.API.dll\" \n",
    "\n",
    "using GC.Analysis.API;"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Helper Methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "// Consolidation of the Runs from the Infrastructure.\n",
    "public void MoveFiles(string basePath)\n",
    "{\n",
    "    var etlZipFiles = Directory.GetFiles(basePath, \"*.etl.zip\");\n",
    "\n",
    "    foreach (var etlZipFile in etlZipFiles)\n",
    "    {\n",
    "        string newPath = \"\";\n",
    "        if (etlZipFile.Contains(\"baseline\"))\n",
    "        {\n",
    "            newPath = etlZipFile.Replace(\"baseline.\", \"\");\n",
    "        }\n",
    "        \n",
    "        else\n",
    "        {\n",
    "            string replaced = Path.GetFileName(etlZipFile).Replace(\"run.\", \"\");\n",
    "            string[] split = replaced.Split(\".\");\n",
    "            string newVal = (int.Parse(split[1]) + 10).ToString();\n",
    "            newPath = basePath + split[0] + \".\" + newVal + \".\" + split[2] + \".\" + split[3];\n",
    "        }\n",
    "\n",
    "        File.Move(etlZipFile, newPath);\n",
    "    }\n",
    "}\n",
    "\n",
    "public List<GCProcessData> GetTraces(string basePath)\n",
    "{\n",
    "    Dictionary<string, Analyzer> traces = AnalyzerManager.GetAllAnalyzers(basePath);\n",
    "    List<GCProcessData> data = new();\n",
    "    foreach (var t in traces)\n",
    "    {\n",
    "        GCProcessData processData = t.Value.GetProcessGCData(\"corerun\").First();\n",
    "        data.Add(processData);\n",
    "    }\n",
    "\n",
    "    return data;\n",
    "}\n",
    "\n",
    "public class MinMaxDiff\n",
    "{\n",
    "    public MinMaxDiff(IEnumerable<double> data)\n",
    "    {\n",
    "        Max = data.Max();\n",
    "        Min = data.Min();\n",
    "        All = data.ToList();\n",
    "        var rest = data.ToList();\n",
    "        rest.Remove(Min);\n",
    "        rest.Remove(Max);\n",
    "        Rest = rest;\n",
    "    }\n",
    "\n",
    "    public double Max { get; }\n",
    "    public double Min { get; }\n",
    "    public double VolPercent => Min != 0 ? Vol / Min * 100 : double.PositiveInfinity;\n",
    "    public double Vol => (Max - Min);\n",
    "    public double Mean => All.Average();\n",
    "    public IReadOnlyList<double> All { get; }\n",
    "    public IReadOnlyList<double> Rest { get; }\n",
    "}\n",
    "\n",
    "public void ShowVolatility(IEnumerable<double> data)\n",
    "{\n",
    "    var layout = new Layout.Layout\n",
    "    {\n",
    "        xaxis = new Xaxis { title = \"Iteration #\" },\n",
    "        yaxis = new Yaxis { title = \"Execution Time Diff %\" },\n",
    "        title = \"Execution Time Diff % vs. Iterations\",\n",
    "    };\n",
    "\n",
    "    var first = data.First(); \n",
    "\n",
    "    var scatter = new Scatter\n",
    "    {\n",
    "        x = Enumerable.Range(0, data.Count()), \n",
    "        y = data.Select(g =>  (g - first) / first * 100),\n",
    "        mode = \"markers+lines\",\n",
    "    };\n",
    "\n",
    "    Chart.Plot(scatter, layout).Display();\n",
    "}\n",
    "\n",
    "public void ShowVolatilityComparison(IEnumerable<GCProcessData> baseline, IEnumerable<GCProcessData> comparand)\n",
    "{\n",
    "    var layout = new Layout.Layout\n",
    "    {\n",
    "        xaxis = new Xaxis { title = \"Iteration #\" },\n",
    "        yaxis = new Yaxis { title = \"Execution Time Diff %\" },\n",
    "        title = \"Execution Time Diff % vs. Iterations\",\n",
    "    };\n",
    "\n",
    "    List<Scatter> scatters = new();\n",
    "\n",
    "    // Baseline.\n",
    "    Scatter GetScatter(IEnumerable<GCProcessData> data, string name)\n",
    "    {\n",
    "        var first = data.First(); \n",
    "\n",
    "        var scatter = new Scatter\n",
    "        {\n",
    "            x = Enumerable.Range(0, data.Count()), \n",
    "            y = data.Select(g =>  (g.DurationMSec - first.DurationMSec) / first.DurationMSec * 100),\n",
    "            mode = \"markers+lines\",\n",
    "            name = name\n",
    "        };\n",
    "\n",
    "        return scatter;\n",
    "    }\n",
    "\n",
    "    Chart.Plot(new [] { GetScatter(baseline, \"baseline\"), GetScatter(comparand, \"comparand\")}, layout).Display();\n",
    "}\n",
    "\n",
    "public void SummarizeVolatility(List<GCProcessData> gcData)\n",
    "{\n",
    "    StringDataFrameColumn criteriaColumn   = new(\"Criteria\"); \n",
    "    StringDataFrameColumn volPercentColumn = new(\"Vol %\"); \n",
    "    StringDataFrameColumn meanColumn       = new(\"Mean\"); \n",
    "    StringDataFrameColumn volColumn        = new(\"Vol\"); \n",
    "    StringDataFrameColumn minColumn        = new(\"Min\"); \n",
    "    StringDataFrameColumn maxColumn        = new(\"Max\"); \n",
    "\n",
    "    // Duration MSec.\n",
    "    MinMaxDiff durationMinMaxDiff    = new MinMaxDiff(gcData.Select(gc => gc.DurationMSec));\n",
    "    criteriaColumn.Append(\"Execution Time (MSec)\");\n",
    "    volPercentColumn.Append(durationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    meanColumn.Append(durationMinMaxDiff.Mean.ToString(\"N2\"));\n",
    "    volColumn.Append(durationMinMaxDiff.Vol.ToString(\"N2\"));\n",
    "    minColumn.Append(durationMinMaxDiff.Min.ToString(\"N2\"));\n",
    "    maxColumn.Append(durationMinMaxDiff.Max.ToString(\"N2\"));\n",
    "\n",
    "    // Total Allocations MB.\n",
    "    MinMaxDiff allocationMinMaxDiff = new MinMaxDiff(gcData.Select(gc => gc.Stats.TotalAllocatedMB));\n",
    "    criteriaColumn.Append(\"Total Allocations (MB)\");\n",
    "    volPercentColumn.Append(allocationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    meanColumn.Append(allocationMinMaxDiff.Mean.ToString(\"N2\"));\n",
    "    volColumn.Append(allocationMinMaxDiff.Vol.ToString(\"N2\"));\n",
    "    minColumn.Append(allocationMinMaxDiff.Min.ToString(\"N2\"));\n",
    "    maxColumn.Append(allocationMinMaxDiff.Max.ToString(\"N2\"));\n",
    "\n",
    "    // Total GC Pause Time. \n",
    "    MinMaxDiff totalGCPauseTime = new MinMaxDiff(gcData.Select(gc => gc.Stats.TotalPauseTimeMSec)); \n",
    "    criteriaColumn.Append(\"Total GC Pause Time (MSec)\");\n",
    "    volPercentColumn.Append(totalGCPauseTime.VolPercent.ToString(\"N2\"));\n",
    "    meanColumn.Append(totalGCPauseTime.Mean.ToString(\"N2\"));\n",
    "    volColumn.Append(totalGCPauseTime.Vol.ToString(\"N2\"));\n",
    "    minColumn.Append(totalGCPauseTime.Min.ToString(\"N2\"));\n",
    "    maxColumn.Append(totalGCPauseTime.Max.ToString(\"N2\"));\n",
    "\n",
    "    // GC Pause Contribution %.\n",
    "    MinMaxDiff contributionPercentByGC = new MinMaxDiff(gcData.Select(gc => gc.Stats.TotalPauseTimeMSec / gc.DurationMSec * 100));\n",
    "    criteriaColumn.Append(\"GC Pause Contribution %\");\n",
    "    volPercentColumn.Append(contributionPercentByGC.VolPercent.ToString(\"N2\"));\n",
    "    meanColumn.Append(contributionPercentByGC.Mean.ToString(\"N2\"));\n",
    "    volColumn.Append(contributionPercentByGC.Vol.ToString(\"N2\"));\n",
    "    minColumn.Append(contributionPercentByGC.Min.ToString(\"N2\"));\n",
    "    maxColumn.Append(contributionPercentByGC.Max.ToString(\"N2\"));\n",
    "\n",
    "    var volDataframe = new DataFrame(criteriaColumn, volPercentColumn, meanColumn, volColumn, minColumn, maxColumn);\n",
    "    volDataframe.Display();\n",
    "\n",
    "    ShowVolatility(gcData.Select(gc => gc.DurationMSec));\n",
    "\n",
    "    GCProcessData min = gcData.MinBy(gc => gc.DurationMSec);\n",
    "    GCProcessData max = gcData.MaxBy(gc => gc.DurationMSec);\n",
    "    min.CompareNormalizedByMaxTotalAllocations(new []  { max }).Display();\n",
    "}\n",
    "\n",
    "public void SummarizeComparativeVolatility(List<GCProcessData> baseline, List<GCProcessData> comparand)\n",
    "{\n",
    "    StringDataFrameColumn criteriaColumn   = new(\"Criteria\"); \n",
    "    StringDataFrameColumn baselineVolPercentageColumn = new(\"Baseline Vol %\"); \n",
    "    StringDataFrameColumn comparandVolPercentageColumn = new(\"Comparand Vol %\"); \n",
    "    StringDataFrameColumn volPercentColumn = new(\"Δ Vol %\"); \n",
    "\n",
    "    // Duration MSec.\n",
    "    MinMaxDiff baselineDurationMinMaxDiff  = new MinMaxDiff(baseline.Select(gc => gc.DurationMSec));\n",
    "    MinMaxDiff comparandDurationMinMaxDiff = new MinMaxDiff(comparand.Select(gc => gc.DurationMSec));\n",
    "\n",
    "    criteriaColumn.Append(\"Execution Time (MSec)\");\n",
    "    baselineVolPercentageColumn.Append(baselineDurationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    comparandVolPercentageColumn.Append(comparandDurationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    volPercentColumn.Append((comparandDurationMinMaxDiff.VolPercent - baselineDurationMinMaxDiff.VolPercent).ToString(\"N2\"));\n",
    "\n",
    "    // Total Allocation.\n",
    "    MinMaxDiff baselineTotalAllocationMinMaxDiff  = new MinMaxDiff(baseline.Select(gc => gc.Stats.TotalAllocatedMB));\n",
    "    MinMaxDiff comparandTotalAllocationMinMaxDiff = new MinMaxDiff(comparand.Select(gc => gc.Stats.TotalAllocatedMB));\n",
    "    criteriaColumn.Append(\"Total Allocations (MB)\");\n",
    "    baselineVolPercentageColumn.Append(baselineTotalAllocationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    comparandVolPercentageColumn.Append(comparandTotalAllocationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    volPercentColumn.Append((comparandTotalAllocationMinMaxDiff.VolPercent - baselineTotalAllocationMinMaxDiff.VolPercent).ToString(\"N2\"));\n",
    "\n",
    "    // Total GC Pause Time.\n",
    "    MinMaxDiff baselineTotalPauseTimeMinMaxDiff  = new MinMaxDiff(baseline.Select(gc => gc.Stats.TotalAllocatedMB));\n",
    "    MinMaxDiff comparandTotalPauseTimeMinMaxDiff = new MinMaxDiff(comparand.Select(gc => gc.Stats.TotalAllocatedMB));\n",
    "    criteriaColumn.Append(\"Total Pause Time (MSec)\");\n",
    "    baselineVolPercentageColumn.Append(baselineTotalPauseTimeMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    comparandVolPercentageColumn.Append(comparandTotalPauseTimeMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    volPercentColumn.Append((comparandTotalPauseTimeMinMaxDiff.VolPercent - baselineTotalAllocationMinMaxDiff.VolPercent).ToString(\"N2\"));\n",
    "\n",
    "    // GC Pause Contribution % \n",
    "    MinMaxDiff baselineGCPauseContributionMinMaxDiff = new MinMaxDiff(baseline.Select(gc => gc.Stats.TotalPauseTimeMSec / gc.DurationMSec * 100));\n",
    "    MinMaxDiff comparandGCPauseContributionMinMaxDiff = new MinMaxDiff(comparand.Select(gc => gc.Stats.TotalPauseTimeMSec / gc.DurationMSec * 100));\n",
    "    criteriaColumn.Append(\"GC Pause Contribution %\");\n",
    "    baselineVolPercentageColumn.Append(baselineGCPauseContributionMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    comparandVolPercentageColumn.Append(comparandGCPauseContributionMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "    volPercentColumn.Append((baselineGCPauseContributionMinMaxDiff.VolPercent - comparandGCPauseContributionMinMaxDiff.VolPercent).ToString(\"N2\"));\n",
    "\n",
    "    DataFrame dataFrame = new DataFrame(criteriaColumn, baselineVolPercentageColumn, comparandVolPercentageColumn, volPercentColumn);\n",
    "    dataFrame.Display();\n",
    "\n",
    "    ShowVolatilityComparison(baseline, comparand);\n",
    "}\n",
    "\n",
    "// File based.\n",
    "\n",
    "public List<double> GetExecutionTimesFromFiles(string path)\n",
    "{\n",
    "    List<double> executionTimes = new();\n",
    "\n",
    "    string[] files = Directory.GetFiles(path);\n",
    "    foreach (var f in files)\n",
    "    {\n",
    "        string text = File.ReadAllText(f);\n",
    "        string secondLine = text.Split(\"\\n\")[1];\n",
    "        executionTimes.Add(long.Parse(secondLine.Split(\" \")[1].Replace(\"ms\", \"\")));\n",
    "    }\n",
    "\n",
    "    return executionTimes;\n",
    "}\n",
    "\n",
    "public Dictionary<string, List<double>> GetAllExecutionTimesFromFiles(string basePath)\n",
    "{\n",
    "    Dictionary<string, List<double>> executionTimes = new();\n",
    "    string[] directories = Directory.GetDirectories(basePath);\n",
    "    foreach (var d in directories)\n",
    "    {\n",
    "        var dir = new DirectoryInfo(d);\n",
    "        var dirName = dir.Name;\n",
    "        List<double> times = GetExecutionTimesFromFiles(d);\n",
    "        executionTimes[dirName] = times; \n",
    "    }\n",
    "\n",
    "    return executionTimes;\n",
    "}\n",
    "\n",
    "public void SummarizeFileVolatility(string basePath)\n",
    "{\n",
    "    Dictionary<string, List<double>> data = GetAllExecutionTimesFromFiles(basePath);\n",
    "    StringDataFrameColumn scenarioColumn   = new(\"Scenario\"); \n",
    "    StringDataFrameColumn volPercentColumn = new(\"Vol %\"); \n",
    "    StringDataFrameColumn meanColumn       = new(\"Mean\"); \n",
    "    StringDataFrameColumn volColumn        = new(\"Vol\"); \n",
    "    StringDataFrameColumn minColumn        = new(\"Min\"); \n",
    "    StringDataFrameColumn maxColumn        = new(\"Max\"); \n",
    "\n",
    "    foreach (var d in data)\n",
    "    {\n",
    "        MinMaxDiff durationMinMaxDiff = new MinMaxDiff(d.Value);\n",
    "\n",
    "        scenarioColumn.Append(d.Key);\n",
    "        volPercentColumn.Append(durationMinMaxDiff.VolPercent.ToString(\"N2\"));\n",
    "        meanColumn.Append(durationMinMaxDiff.Mean.ToString(\"N2\"));\n",
    "        volColumn.Append(durationMinMaxDiff.Vol.ToString(\"N2\"));\n",
    "        minColumn.Append(durationMinMaxDiff.Min.ToString(\"N2\"));\n",
    "        maxColumn.Append(durationMinMaxDiff.Max.ToString(\"N2\"));\n",
    "    }\n",
    "\n",
    "    DataFrame dataFrame = new(scenarioColumn, volPercentColumn, meanColumn, volColumn, minColumn, maxColumn); \n",
    "    dataFrame.Display();\n",
    "}"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Volatility Based on Trace Duration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "string BASE_PATH = @\"ADD YOUR PATH.\";\n",
    "\n",
    "var traces = GetTraces(BASE_PATH);"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarize For A Single Set of Runs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "SummarizeVolatility(traces)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Summarize Comparison"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "var baselinePath  = @\"SOME BASELINE PATH\";\n",
    "var comparandPath = @\"SOME COMPARAND PATH\";\n",
    "\n",
    "List<GCProcessData> baselineTraces  = GetTraces(baselinePath);\n",
    "List<GCProcessData> comparandTraces = GetTraces(comparandPath);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "SummarizeComparativeVolatility(baselineTraces, comparandTraces)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Debugging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "System.Diagnostics.Process.GetCurrentProcess().Id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "dotnet_interactive": {
     "language": "csharp"
    },
    "polyglot_notebook": {
     "kernelName": "csharp"
    },
    "vscode": {
     "languageId": "polyglot-notebook"
    }
   },
   "outputs": [],
   "source": [
    "#!about"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}