From 97f089cda344b279b90623ce93d4c18af264e770 Mon Sep 17 00:00:00 2001
From: github-action-benchmark <github@users.noreply.github.com>
Date: Wed, 28 Aug 2024 02:14:02 +0000
Subject: [PATCH] add smaller_is_better (customSmallerIsBetter) benchmark
 result for 0cba85481b49397693147757cc5a8d42e37fe5e3

---
 dev/bench/data.js | 94 +++++++++++++++++++++++------------------------
 1 file changed, 47 insertions(+), 47 deletions(-)

diff --git a/dev/bench/data.js b/dev/bench/data.js
index 13de387..70051e4 100644
--- a/dev/bench/data.js
+++ b/dev/bench/data.js
@@ -1,54 +1,8 @@
 window.BENCHMARK_DATA = {
-  "lastUpdate": 1724724736873,
+  "lastUpdate": 1724811242971,
   "repoUrl": "https://github.com/neuralmagic/nm-vllm-ent",
   "entries": {
     "smaller_is_better": [
-      {
-        "commit": {
-          "author": {
-            "name": "Robert Shaw",
-            "username": "robertgshaw2-neuralmagic",
-            "email": "114415538+robertgshaw2-neuralmagic@users.noreply.github.com"
-          },
-          "committer": {
-            "name": "GitHub",
-            "username": "web-flow",
-            "email": "noreply@github.com"
-          },
-          "id": "831e1fbec511819cffa7dcd0b74c968e9fd803ec",
-          "message": "[ Upstream Sync ] Upstream sync 2024 07 21 (`v0.5.2`) (#24)\n\nSUMMARY:\r\n- Upstream sync from\r\nhttps://github.com/vllm-project/vllm/commit/79d406e9183aa12cdef6f1876eb9a15385662587\r\n(`v0.5.1`) to\r\nhttps://github.com/vllm-project/vllm/commit/4cf256ae7f8b0be8f06f6b85821e55d4f5bdaa13\r\n(`v0.5.2`)\r\n- Comprare\r\nhttps://github.com/neuralmagic/nm-vllm-ent/compare/upstream-sync-2024-07-21..upstream-v0.5.2?expand=1\r\n\r\nTEST PLAN:\r\n- Automation\r\n\r\n---------\r\n\r\nSigned-off-by: kevin <kevin@anyscale.com>\r\nSigned-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>\r\nSigned-off-by: Thomas Parnell <tpa@zurich.ibm.com>\r\nSigned-off-by: sangjune.park <sangjune.park@navercorp.com>\r\nSigned-off-by: yatta zhang <ytzhang01@foxmail.com>\r\nSigned-off-by: zhangyuntao.dev <zhangyuntao.dev@bytedance.com>\r\nSigned-off-by: Xiaowei Jiang <xwjiang2010@gmail.com>\r\nCo-authored-by: Simon Mo <simon.mo@hey.com>\r\nCo-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com>\r\nCo-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>\r\nCo-authored-by: Woosuk Kwon <woosuk.kwon@berkeley.edu>\r\nCo-authored-by: Haichuan <1778876540@qq.com>\r\nCo-authored-by: Roger Wang <ywang@roblox.com>\r\nCo-authored-by: Robert Shaw <rshaw@neuralmagic>\r\nCo-authored-by: youkaichao <youkaichao@gmail.com>\r\nCo-authored-by: Cody Yu <hao.yu.cody@gmail.com>\r\nCo-authored-by: kczimm <4733573+kczimm@users.noreply.github.com>\r\nCo-authored-by: Avshalom Manevich <12231371+avshalomman@users.noreply.github.com>\r\nCo-authored-by: afeldman-nm <156691304+afeldman-nm@users.noreply.github.com>\r\nCo-authored-by: Eric <ericperfectttt@gmail.com>\r\nCo-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com>\r\nCo-authored-by: Kevin H. Luu <kevin@anyscale.com>\r\nCo-authored-by: Swapnil Parekh <swapnilbp100@gmail.com>\r\nCo-authored-by: Swapnil Parekh <swapnilp@ibm.com>\r\nCo-authored-by: Joe G <joseph.granados@h2o.ai>\r\nCo-authored-by: Antoni Baum <antoni.baum@protonmail.com>\r\nCo-authored-by: Murali Andoorveedu <37849411+andoorve@users.noreply.github.com>\r\nCo-authored-by: Baoyuan Qi <qibaoyuan@126.com>\r\nCo-authored-by: Abhinav Goyal <abhinav.goyal@flipkart.com>\r\nCo-authored-by: Thomas Parnell <tpa@zurich.ibm.com>\r\nCo-authored-by: Benjamin Muskalla <bmuskalla@github.com>\r\nCo-authored-by: sangjune.park <park12sj@gmail.com>\r\nCo-authored-by: sroy745 <142070531+sroy745@users.noreply.github.com>\r\nCo-authored-by: daquexian <daquexian566@gmail.com>\r\nCo-authored-by: Jie Fu (傅杰) <jiefu@tencent.com>\r\nCo-authored-by: Lim Xiang Yang <xiangyang95@gmail.com>\r\nCo-authored-by: aniaan <hi@aniaan.dev>\r\nCo-authored-by: pushan <62173185+pushan01@users.noreply.github.com>\r\nCo-authored-by: zhangyuntao.dev <zhangyuntao.dev@bytedance.com>\r\nCo-authored-by: Travis Johnson <tsjohnso@us.ibm.com>\r\nCo-authored-by: Mor Zusman <mor.zusmann@gmail.com>\r\nCo-authored-by: xwjiang2010 <87673679+xwjiang2010@users.noreply.github.com>\r\nCo-authored-by: Kuntai Du <kuntai@uchicago.edu>\r\nCo-authored-by: Zifei Tong <zifeitong@gmail.com>\r\nCo-authored-by: Lily Liu <lilyliupku@gmail.com>\r\nCo-authored-by: Helena Kloosterman <helena.kloosterman@intel.com>\r\nCo-authored-by: Michael Goin <michael@neuralmagic.com>\r\nCo-authored-by: adityagoel14 <aditya.goel@amd.com>\r\nCo-authored-by: Hongxia Yang <62075498+hongxiayang@users.noreply.github.com>\r\nCo-authored-by: Alexei-V-Ivanov-AMD <156011006+Alexei-V-Ivanov-AMD@users.noreply.github.com>\r\nCo-authored-by: Yihuan Bu <88394319+kevinbu233@users.noreply.github.com>\r\nCo-authored-by: Saliya Ekanayake <esaliya@gmail.com>\r\nCo-authored-by: Saliya Ekanayake <esaliya@d-matrix.ai>\r\nCo-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>\r\nCo-authored-by: Noam Gat <noamgat@gmail.com>\r\nCo-authored-by: Isotr0py <2037008807@qq.com>\r\nCo-authored-by: Yuan Tang <terrytangyuan@gmail.com>\r\nCo-authored-by: Tyler Michael Smith <tyler@neuralmagic.com>\r\nCo-authored-by: Ethan Xu <70482605+EthanqX@users.noreply.github.com>\r\nCo-authored-by: Robert Cohn <rscohn2@gmail.com>\r\nCo-authored-by: Fish <45708320+lxline@users.noreply.github.com>\r\nCo-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>\r\nCo-authored-by: DefTruth <31974251+DefTruth@users.noreply.github.com>\r\nCo-authored-by: Pernekhan Utemuratov <bestkhang@gmail.com>\r\nCo-authored-by: Pernekhan Utemuratov <pernekhan@deepinfra.com>\r\nCo-authored-by: Chih-Chieh-Yang <chih.chieh.yang@ibm.com>\r\nCo-authored-by: derekk-nm <derek@neuralmagic.com>",
-          "timestamp": "2024-08-08T00:44:15Z",
-          "url": "https://github.com/neuralmagic/nm-vllm-ent/commit/831e1fbec511819cffa7dcd0b74c968e9fd803ec"
-        },
-        "date": 1723084037232,
-        "tool": "customSmallerIsBetter",
-        "benches": [
-          {
-            "name": "{\"name\": \"mean_ttft_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.2.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
-            "value": 177.43016637666489,
-            "unit": "ms",
-            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.2.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"tokenizer\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'meta-llama/Meta-Llama-3-8B-Instruct', 'tokenizer': 'meta-llama/Meta-Llama-3-8B-Instruct', 'max-model-len': 4096, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-08 02:26:12 UTC\",\n  \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n  \"dataset\": \"sharegpt\"\n}"
-          },
-          {
-            "name": "{\"name\": \"mean_tpot_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.2.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
-            "value": 83.44827355641736,
-            "unit": "ms",
-            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.2.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"tokenizer\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'meta-llama/Meta-Llama-3-8B-Instruct', 'tokenizer': 'meta-llama/Meta-Llama-3-8B-Instruct', 'max-model-len': 4096, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-08 02:26:12 UTC\",\n  \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n  \"dataset\": \"sharegpt\"\n}"
-          },
-          {
-            "name": "{\"name\": \"mean_ttft_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.2.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
-            "value": 23.403049120004578,
-            "unit": "ms",
-            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.2.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"facebook/opt-350m\",\n    \"tokenizer\": \"facebook/opt-350m\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'facebook/opt-350m', 'tokenizer': 'facebook/opt-350m', 'max-model-len': 2048, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-08 02:18:20 UTC\",\n  \"model\": \"facebook/opt-350m\",\n  \"dataset\": \"sharegpt\"\n}"
-          },
-          {
-            "name": "{\"name\": \"mean_tpot_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.2.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
-            "value": 6.071583097552319,
-            "unit": "ms",
-            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.2.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"facebook/opt-350m\",\n    \"tokenizer\": \"facebook/opt-350m\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'facebook/opt-350m', 'tokenizer': 'facebook/opt-350m', 'max-model-len': 2048, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-08 02:18:20 UTC\",\n  \"model\": \"facebook/opt-350m\",\n  \"dataset\": \"sharegpt\"\n}"
-          }
-        ]
-      },
       {
         "commit": {
           "author": {
@@ -2218,6 +2172,52 @@ window.BENCHMARK_DATA = {
             "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.3.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"tokenizer\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'meta-llama/Meta-Llama-3-8B-Instruct', 'tokenizer': 'meta-llama/Meta-Llama-3-8B-Instruct', 'max-model-len': 4096, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-27 02:11:10 UTC\",\n  \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n  \"dataset\": \"sharegpt\"\n}"
           }
         ]
+      },
+      {
+        "commit": {
+          "author": {
+            "name": "Domenic Barbuzzi",
+            "username": "dbarbuzzi",
+            "email": "domenic@neuralmagic.com"
+          },
+          "committer": {
+            "name": "GitHub",
+            "username": "web-flow",
+            "email": "noreply@github.com"
+          },
+          "id": "0cba85481b49397693147757cc5a8d42e37fe5e3",
+          "message": "Add annotations to job names (#65)\n\nSUMMARY:\r\nThis PR updates the 'TEST' and 'LM-EVAL' job definitions in their\r\nrespective workflows so they will have additional information in their\r\nnames in the left sidebar when viewing a run (see TEST PLAN for what\r\nthat looks like).\r\n\r\nTEST PLAN:\r\nFrom the remote-push run on this job (nightly/anything else running\r\nthose workflows will be identically updated):\r\nhttps://github.com/neuralmagic/nm-vllm-ent/actions/runs/10531500526\r\n![Screenshot 2024-08-23\r\n164314-fs8](https://github.com/user-attachments/assets/b9db163e-4b9c-469a-8622-bc5a5cf8c94b)",
+          "timestamp": "2024-08-27T15:42:45Z",
+          "url": "https://github.com/neuralmagic/nm-vllm-ent/commit/0cba85481b49397693147757cc5a8d42e37fe5e3"
+        },
+        "date": 1724811242474,
+        "tool": "customSmallerIsBetter",
+        "benches": [
+          {
+            "name": "{\"name\": \"mean_ttft_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.3.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
+            "value": 183.0294290466918,
+            "unit": "ms",
+            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.3.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"tokenizer\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'meta-llama/Meta-Llama-3-8B-Instruct', 'tokenizer': 'meta-llama/Meta-Llama-3-8B-Instruct', 'max-model-len': 4096, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-28 02:12:55 UTC\",\n  \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n  \"dataset\": \"sharegpt\"\n}"
+          },
+          {
+            "name": "{\"name\": \"mean_tpot_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.3.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
+            "value": 86.7150696858631,
+            "unit": "ms",
+            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.3.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - meta-llama/Meta-Llama-3-8B-Instruct\\nmax-model-len - 4096\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"tokenizer\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'meta-llama/Meta-Llama-3-8B-Instruct', 'tokenizer': 'meta-llama/Meta-Llama-3-8B-Instruct', 'max-model-len': 4096, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-28 02:12:55 UTC\",\n  \"model\": \"meta-llama/Meta-Llama-3-8B-Instruct\",\n  \"dataset\": \"sharegpt\"\n}"
+          },
+          {
+            "name": "{\"name\": \"mean_ttft_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.3.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
+            "value": 27.99905281333243,
+            "unit": "ms",
+            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.3.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"facebook/opt-350m\",\n    \"tokenizer\": \"facebook/opt-350m\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'facebook/opt-350m', 'tokenizer': 'facebook/opt-350m', 'max-model-len': 2048, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-28 02:03:43 UTC\",\n  \"model\": \"facebook/opt-350m\",\n  \"dataset\": \"sharegpt\"\n}"
+          },
+          {
+            "name": "{\"name\": \"mean_tpot_ms\", \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\", \"gpu_description\": \"NVIDIA L4 x 1\", \"vllm_version\": \"0.5.3.0\", \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\", \"torch_version\": \"2.3.1+cu121\"}",
+            "value": 7.135941599949365,
+            "unit": "ms",
+            "extra": "{\n  \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n  \"benchmarking_context\": {\n    \"vllm_version\": \"0.5.3.0\",\n    \"python_version\": \"3.10.12 (main, Jun  7 2023, 13:43:11) [GCC 11.3.0]\",\n    \"torch_version\": \"2.3.1+cu121\",\n    \"torch_cuda_version\": \"12.1\",\n    \"cuda_devices\": \"[_CudaDeviceProperties(name='NVIDIA L4', major=8, minor=9, total_memory=22478MB, multi_processor_count=58)]\",\n    \"cuda_device_names\": [\n      \"NVIDIA L4\"\n    ]\n  },\n  \"gpu_description\": \"NVIDIA L4 x 1\",\n  \"script_name\": \"benchmark_serving.py\",\n  \"script_args\": {\n    \"description\": \"VLLM Serving - Dense\\nmodel - facebook/opt-350m\\nmax-model-len - 2048\\nsparsity - None\\nbenchmark_serving {\\n  \\\"nr-qps-pair_\\\": \\\"300,1\\\",\\n  \\\"dataset\\\": \\\"sharegpt\\\"\\n}\",\n    \"backend\": \"vllm\",\n    \"version\": \"N/A\",\n    \"base_url\": null,\n    \"host\": \"127.0.0.1\",\n    \"port\": 9000,\n    \"endpoint\": \"/generate\",\n    \"dataset\": \"sharegpt\",\n    \"num_input_tokens\": null,\n    \"num_output_tokens\": null,\n    \"model\": \"facebook/opt-350m\",\n    \"tokenizer\": \"facebook/opt-350m\",\n    \"best_of\": 1,\n    \"use_beam_search\": false,\n    \"log_model_io\": false,\n    \"seed\": 0,\n    \"trust_remote_code\": false,\n    \"disable_tqdm\": false,\n    \"save_directory\": \"benchmark-results\",\n    \"num_prompts_\": null,\n    \"request_rate_\": null,\n    \"nr_qps_pair_\": [\n      300,\n      \"1.0\"\n    ],\n    \"server_tensor_parallel_size\": 1,\n    \"server_args\": \"{'model': 'facebook/opt-350m', 'tokenizer': 'facebook/opt-350m', 'max-model-len': 2048, 'host': '127.0.0.1', 'port': 9000, 'tensor-parallel-size': 1, 'disable-log-requests': ''}\"\n  },\n  \"date\": \"2024-08-28 02:03:43 UTC\",\n  \"model\": \"facebook/opt-350m\",\n  \"dataset\": \"sharegpt\"\n}"
+          }
+        ]
       }
     ]
   }