Skip to content

Commit

Permalink
Add filtering
Browse files Browse the repository at this point in the history 
Changes are duplicated from Varun in neuralmagic/nm-vllm
neuralmagic/nm-vllm#132
  • Loading branch information
@dbarbuzzi
dbarbuzzi committed Jul 24, 2024
1 parent ebb2f26 commit 949bab8
Showing 1 changed file with 211 additions and 10 deletions.
221 changes: 211 additions & 10 deletions dev/bench/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,14 @@
word-break: break-word;
text-align: center;
}
.query-tool {
display: flex;
flex-direction: row;
justify-content: space-around;
align-items: center;
flex-wrap: wrap;
width: 50%;
}
.benchmark-graphs {
display: flex;
flex-direction: row;
Expand Down Expand Up @@ -162,10 +170,30 @@
};

// Prepare data points for charts
return Object.keys(data.entries).map(name => ({
name,
dataSet: collectBenchesPerTestCase(data.entries[name]),
}));
let allDataSets = Object.keys(data.entries).map(name =>
collectBenchesPerTestCase(data.entries[name])
);

// Combine all datasets
let allDataSetsMap = new Map()
for (const DataSet of allDataSets) {
for (const [name, bench] of DataSet) {
allDataSetsMap.set(name, bench)
}
}
return new Array({name: "Benchmarks", dataSet: allDataSetsMap});
}

function shortPythonVersion(long_py_version_str) {
// long_py_version_str = "3.10.12 (main, Mar 7 2024, 18:39:53) [GCC 9.4.0]"
// to shortPythonVersion 3.10.12
return long_py_version_str.slice(0, long_py_version_str.indexOf(" ")).trim()
}

function shortDate(long_date_str) {
// long_date_str = 2024-03-19 03:26:26 UTC
// to shortDate 2024-03-19
return long_date_str.slice(0, long_date_str.indexOf(" ")).trim()
}

function renderAllChars(dataSets) {
Expand All @@ -180,14 +208,32 @@
labels: dataset.map(d => d.commit.id.slice(0, 7)),
datasets: [
{
label: name,
label: JSON.parse(name).name,
data: dataset.map(d => d.bench.value),
borderColor: color,
backgroundColor: color + '60', // Add alpha for #rrggbbaa
}
],
};

function chartTitle() {
const name_json = JSON.parse(name)
const metric_name = name_json.name
const description = name_json.description
const gpu_description = name_json.gpu_description
const vllm_version = name_json.vllm_version
const python_version = shortPythonVersion(name_json.python_version)
const torch_version = name_json.torch_version

const title = [metric_name, description, `GPU : ${gpu_description}`, `vllm : ${vllm_version}`, `py_version : ${python_version}`, `torch : ${torch_version}`]
return title
}

const options = {
title : {
display: true,
text: chartTitle(data)
},
scales: {
xAxes: [
{
Expand Down Expand Up @@ -226,8 +272,13 @@
return label;
},
afterLabel: item => {
const { extra } = dataset[item.index].bench;
return extra ? '\n' + extra : '';
const extra = dataset[item.index].bench.extra;
if (!extra) {
return ''
}
const extra_json = JSON.parse(extra)
const extra_str = `\n script-name ${extra_json.script_name} \n script-args : ${JSON.stringify(extra_json.script_args, null, 2)})`
return extra_str;
}
}
},
Expand All @@ -250,8 +301,15 @@
}

function renderBenchSet(name, benchSet, main) {

const prev_bench_set = document.getElementById('benchmark-set')
if (prev_bench_set != null) {
prev_bench_set.remove()
}

const setElem = document.createElement('div');
setElem.className = 'benchmark-set';
setElem.id = 'benchmark-set'
main.appendChild(setElem);

const nameElem = document.createElement('h1');
Expand All @@ -268,10 +326,153 @@
}
}

const main = document.getElementById('main');
for (const {name, dataSet} of dataSets) {
renderBenchSet(name, dataSet, main);
function prepareQueryTool(main, dataSets) {

function addDropDown(div, dropDownName, dropDownOptions) {
const name = document.createElement('h5');
name.textContent = dropDownName;
div.appendChild(name);

const s = document.createElement('select')
s.id = dropDownName
for (const item of dropDownOptions) {
s.options.add(new Option(item))
}

div.appendChild(s)
}

function processOption(option_str) {
if (option_str === null) {
return "null"
}
option_str = option_str.trim()
if (option_str === "") {
return "null"
}
return option_str
}

const ANY_STR = "any"

function compareOption(option, to_compare) {
return option === ANY_STR || option === processOption(to_compare);
}

let metrics = new Set([ANY_STR])
let dates = new Set([ANY_STR])
let models = new Set([ANY_STR])
let datasets = new Set([ANY_STR])
let gpus = new Set([ANY_STR])
let script_names = new Set([ANY_STR])
let vllm_versions = new Set([ANY_STR])
let py_versions = new Set([ANY_STR])
let torch_versions = new Set([ANY_STR])
for (const {name, dataSet} of dataSets) {
for (const [key, values] of dataSet.entries()) {
const extra = JSON.parse(values[0].bench.extra)
metrics.add(processOption(JSON.parse(key).name))
models.add(processOption(extra.model))
datasets.add(processOption(extra.dataset))
gpus.add(processOption(extra.gpu_description))
script_names.add(processOption(extra.script_name))
vllm_versions.add(processOption(extra.benchmarking_context.vllm_version))
py_versions.add(processOption(shortPythonVersion(extra.benchmarking_context.python_version)))
torch_versions.add(processOption(extra.benchmarking_context.torch_version))

// Each value is has a different date
for (const value of values) {
dates.add(processOption(shortDate(JSON.parse(value.bench.extra).date)))
}
}
}

function filter_bench_by_date(benches, date_query) {
if (date_query == ANY_STR) {
return benches
}

let filtered_benches = new Array()
for (const item of benches) {
if (compareOption(date_query, shortDate(JSON.parse(item.bench.extra).date))) {
filtered_benches.push(item)
}
}
return filtered_benches
}

// == Query ====
const queryElem = document.createElement('div');
queryElem.className = 'query-tool'
queryElem.textContent = 'query-tool'
queryElem.id = 'query-tool'
main.appendChild(queryElem)

addDropDown(queryElem, "dates", dates)
addDropDown(queryElem, "metrics", metrics)
addDropDown(queryElem, "models", models)
addDropDown(queryElem, "datasets", datasets)
addDropDown(queryElem, "GPU", gpus)
addDropDown(queryElem, "script_name", script_names)
addDropDown(queryElem, "vllm_version", vllm_versions)
addDropDown(queryElem, "python_version", py_versions)
addDropDown(queryElem, "torch_version", torch_versions)

function filterCharts() {
const date = queryElem.querySelector('[id=dates]').value
const metric = queryElem.querySelector('[id=metrics]').value
const model = queryElem.querySelector('[id=models]').value
const dataset = queryElem.querySelector('[id=datasets]').value
const gpu = queryElem.querySelector('[id=GPU]').value
const script_name = queryElem.querySelector('[id=script_name]').value
const vllm_version = queryElem.querySelector('[id=vllm_version]').value
const python_version = queryElem.querySelector('[id=python_version]').value
const torch_version = queryElem.querySelector('[id=torch_version]').value

let filtered_datasets = new Array()

for (const {name, dataSet} of dataSets) {
let filtered_dataset_map = new Map()
filtered_datasets.push({name: name, dataSet: filtered_dataset_map})

for (const [key, values] of dataSet.entries()) {
const extra = JSON.parse(values[0].bench.extra)
let put_values = (
compareOption(metric, JSON.parse(key).name) &&
compareOption(model, extra.model) &&
compareOption(dataset, extra.dataset) &&
compareOption(gpu, extra.gpu_description) &&
compareOption(script_name, extra.script_name) &&
compareOption(vllm_version, extra.benchmarking_context.vllm_version) &&
compareOption(python_version, shortPythonVersion(extra.benchmarking_context.python_version)) &&
compareOption(torch_version, extra.benchmarking_context.torch_version))

if (!put_values) {
continue
}

filtered_dataset_map.set(key, filter_bench_by_date(values, date))
}
}
renderDatasets(filtered_datasets)
}

const b = document.createElement('button')
b.textContent = 'filter'
b.onclick= filterCharts
queryElem.appendChild(b)

}

function renderDatasets(dataSets) {
for (const {name, dataSet} of dataSets) {
renderBenchSet(name, dataSet, main);
}
}

const main = document.getElementById('main');
prepareQueryTool(main, dataSets)
renderDatasets(dataSets)
}

renderAllChars(init()); // Start
Expand Down

0 comments on commit 949bab8

Please sign in to comment.