From 949bab85bd30fba9c06ab39e94018b87395badf4 Mon Sep 17 00:00:00 2001 From: Domenic Barbuzzi Date: Wed, 24 Jul 2024 14:11:43 +0000 Subject: [PATCH] Add filtering Changes are duplicated from Varun in neuralmagic/nm-vllm https://github.com/neuralmagic/nm-vllm/pull/132 --- dev/bench/index.html | 221 +++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 211 insertions(+), 10 deletions(-) diff --git a/dev/bench/index.html b/dev/bench/index.html index 6c88780..e9b33e7 100644 --- a/dev/bench/index.html +++ b/dev/bench/index.html @@ -71,6 +71,14 @@ word-break: break-word; text-align: center; } + .query-tool { + display: flex; + flex-direction: row; + justify-content: space-around; + align-items: center; + flex-wrap: wrap; + width: 50%; + } .benchmark-graphs { display: flex; flex-direction: row; @@ -162,10 +170,30 @@ }; // Prepare data points for charts - return Object.keys(data.entries).map(name => ({ - name, - dataSet: collectBenchesPerTestCase(data.entries[name]), - })); + let allDataSets = Object.keys(data.entries).map(name => + collectBenchesPerTestCase(data.entries[name]) + ); + + // Combine all datasets + let allDataSetsMap = new Map() + for (const DataSet of allDataSets) { + for (const [name, bench] of DataSet) { + allDataSetsMap.set(name, bench) + } + } + return new Array({name: "Benchmarks", dataSet: allDataSetsMap}); + } + + function shortPythonVersion(long_py_version_str) { + // long_py_version_str = "3.10.12 (main, Mar 7 2024, 18:39:53) [GCC 9.4.0]" + // to shortPythonVersion 3.10.12 + return long_py_version_str.slice(0, long_py_version_str.indexOf(" ")).trim() + } + + function shortDate(long_date_str) { + // long_date_str = 2024-03-19 03:26:26 UTC + // to shortDate 2024-03-19 + return long_date_str.slice(0, long_date_str.indexOf(" ")).trim() } function renderAllChars(dataSets) { @@ -180,14 +208,32 @@ labels: dataset.map(d => d.commit.id.slice(0, 7)), datasets: [ { - label: name, + label: JSON.parse(name).name, data: dataset.map(d => d.bench.value), borderColor: color, backgroundColor: color + '60', // Add alpha for #rrggbbaa } ], }; + + function chartTitle() { + const name_json = JSON.parse(name) + const metric_name = name_json.name + const description = name_json.description + const gpu_description = name_json.gpu_description + const vllm_version = name_json.vllm_version + const python_version = shortPythonVersion(name_json.python_version) + const torch_version = name_json.torch_version + + const title = [metric_name, description, `GPU : ${gpu_description}`, `vllm : ${vllm_version}`, `py_version : ${python_version}`, `torch : ${torch_version}`] + return title + } + const options = { + title : { + display: true, + text: chartTitle(data) + }, scales: { xAxes: [ { @@ -226,8 +272,13 @@ return label; }, afterLabel: item => { - const { extra } = dataset[item.index].bench; - return extra ? '\n' + extra : ''; + const extra = dataset[item.index].bench.extra; + if (!extra) { + return '' + } + const extra_json = JSON.parse(extra) + const extra_str = `\n script-name ${extra_json.script_name} \n script-args : ${JSON.stringify(extra_json.script_args, null, 2)})` + return extra_str; } } }, @@ -250,8 +301,15 @@ } function renderBenchSet(name, benchSet, main) { + + const prev_bench_set = document.getElementById('benchmark-set') + if (prev_bench_set != null) { + prev_bench_set.remove() + } + const setElem = document.createElement('div'); setElem.className = 'benchmark-set'; + setElem.id = 'benchmark-set' main.appendChild(setElem); const nameElem = document.createElement('h1'); @@ -268,10 +326,153 @@ } } - const main = document.getElementById('main'); - for (const {name, dataSet} of dataSets) { - renderBenchSet(name, dataSet, main); + function prepareQueryTool(main, dataSets) { + + function addDropDown(div, dropDownName, dropDownOptions) { + const name = document.createElement('h5'); + name.textContent = dropDownName; + div.appendChild(name); + + const s = document.createElement('select') + s.id = dropDownName + for (const item of dropDownOptions) { + s.options.add(new Option(item)) + } + + div.appendChild(s) + } + + function processOption(option_str) { + if (option_str === null) { + return "null" + } + option_str = option_str.trim() + if (option_str === "") { + return "null" + } + return option_str + } + + const ANY_STR = "any" + + function compareOption(option, to_compare) { + return option === ANY_STR || option === processOption(to_compare); + } + + let metrics = new Set([ANY_STR]) + let dates = new Set([ANY_STR]) + let models = new Set([ANY_STR]) + let datasets = new Set([ANY_STR]) + let gpus = new Set([ANY_STR]) + let script_names = new Set([ANY_STR]) + let vllm_versions = new Set([ANY_STR]) + let py_versions = new Set([ANY_STR]) + let torch_versions = new Set([ANY_STR]) + for (const {name, dataSet} of dataSets) { + for (const [key, values] of dataSet.entries()) { + const extra = JSON.parse(values[0].bench.extra) + metrics.add(processOption(JSON.parse(key).name)) + models.add(processOption(extra.model)) + datasets.add(processOption(extra.dataset)) + gpus.add(processOption(extra.gpu_description)) + script_names.add(processOption(extra.script_name)) + vllm_versions.add(processOption(extra.benchmarking_context.vllm_version)) + py_versions.add(processOption(shortPythonVersion(extra.benchmarking_context.python_version))) + torch_versions.add(processOption(extra.benchmarking_context.torch_version)) + + // Each value is has a different date + for (const value of values) { + dates.add(processOption(shortDate(JSON.parse(value.bench.extra).date))) + } + } + } + + function filter_bench_by_date(benches, date_query) { + if (date_query == ANY_STR) { + return benches + } + + let filtered_benches = new Array() + for (const item of benches) { + if (compareOption(date_query, shortDate(JSON.parse(item.bench.extra).date))) { + filtered_benches.push(item) + } + } + return filtered_benches + } + + // == Query ==== + const queryElem = document.createElement('div'); + queryElem.className = 'query-tool' + queryElem.textContent = 'query-tool' + queryElem.id = 'query-tool' + main.appendChild(queryElem) + + addDropDown(queryElem, "dates", dates) + addDropDown(queryElem, "metrics", metrics) + addDropDown(queryElem, "models", models) + addDropDown(queryElem, "datasets", datasets) + addDropDown(queryElem, "GPU", gpus) + addDropDown(queryElem, "script_name", script_names) + addDropDown(queryElem, "vllm_version", vllm_versions) + addDropDown(queryElem, "python_version", py_versions) + addDropDown(queryElem, "torch_version", torch_versions) + + function filterCharts() { + const date = queryElem.querySelector('[id=dates]').value + const metric = queryElem.querySelector('[id=metrics]').value + const model = queryElem.querySelector('[id=models]').value + const dataset = queryElem.querySelector('[id=datasets]').value + const gpu = queryElem.querySelector('[id=GPU]').value + const script_name = queryElem.querySelector('[id=script_name]').value + const vllm_version = queryElem.querySelector('[id=vllm_version]').value + const python_version = queryElem.querySelector('[id=python_version]').value + const torch_version = queryElem.querySelector('[id=torch_version]').value + + let filtered_datasets = new Array() + + for (const {name, dataSet} of dataSets) { + let filtered_dataset_map = new Map() + filtered_datasets.push({name: name, dataSet: filtered_dataset_map}) + + for (const [key, values] of dataSet.entries()) { + const extra = JSON.parse(values[0].bench.extra) + let put_values = ( + compareOption(metric, JSON.parse(key).name) && + compareOption(model, extra.model) && + compareOption(dataset, extra.dataset) && + compareOption(gpu, extra.gpu_description) && + compareOption(script_name, extra.script_name) && + compareOption(vllm_version, extra.benchmarking_context.vllm_version) && + compareOption(python_version, shortPythonVersion(extra.benchmarking_context.python_version)) && + compareOption(torch_version, extra.benchmarking_context.torch_version)) + + if (!put_values) { + continue + } + + filtered_dataset_map.set(key, filter_bench_by_date(values, date)) + } + } + renderDatasets(filtered_datasets) + } + + const b = document.createElement('button') + b.textContent = 'filter' + b.onclick= filterCharts + queryElem.appendChild(b) + + } + + function renderDatasets(dataSets) { + for (const {name, dataSet} of dataSets) { + renderBenchSet(name, dataSet, main); + } } + + const main = document.getElementById('main'); + prepareQueryTool(main, dataSets) + renderDatasets(dataSets) } renderAllChars(init()); // Start