{
 "count": 11,
 "note": "cluster and multi-node runs (measured, source-cited) out of record scope: local inference means single-machine hardware",
 "records": [
  {
   "backend": "vLLM",
   "batch": 1,
   "ctx": null,
   "date": "2026-09",
   "flags": [],
   "hardware": "GB200",
   "id": "githubissues-gb200-glm53-nvfp4-vllm-mnnvl",
   "model": "GLM-5.3",
   "notes": "vLLM P/D disaggregation on GB200 (DEP8 prefill + TP4 decode, AgentX C64 agentic load); NIXL KV transfer over MNNVL cuda_ipc; output tok/s/user mean under agentic load",
   "params": "MLA + DSA indexer (DeepSeek-V3.2 architecture)",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "NVFP4",
   "quote": "GLM-5.3 (NVFP4), DEP8 prefill + TP4 decode, AgentX C64. ; with P/D disaggregation on GB200, NIXL KV transfer is dominated by ; Output tok/s/user mean / p50 | 102.68 / 91.10 | 136.77 / 129.06 | +33% / +42%",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #55434 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/55434",
   "tps": 102.68,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 1,
   "ctx": null,
   "date": "2026-09",
   "flags": [],
   "hardware": "GB200",
   "id": "githubissues-gb200-glm53-nvfp4-vllm-ib4",
   "model": "GLM-5.3",
   "notes": "same GB200 P/D deployment (DEP8 prefill + TP4 decode, AgentX C64 agentic load) with NIXL over native IB 4-rail rc_x; output tok/s/user mean under agentic load",
   "params": "MLA + DSA indexer (DeepSeek-V3.2 architecture)",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "NVFP4",
   "quote": "GLM-5.3 (NVFP4), DEP8 prefill + TP4 decode, AgentX C64. ; with P/D disaggregation on GB200, NIXL KV transfer is dominated by ; Output tok/s/user mean / p50 | 102.68 / 91.10 | 136.77 / 129.06 | +33% / +42%",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #55434 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/55434",
   "tps": 136.77,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 1024,
   "ctx": null,
   "date": "2026-05",
   "flags": [],
   "hardware": "GB200",
   "id": "vllm-41603-gb200-dsv4-pro-fp4-ref",
   "model": "DeepSeek-V4-Pro",
   "notes": "2P/1D disaggregated MegaMOE on GB200 (16 prefill + 8 decode GPUs, Dynamo + vLLM); tok/s/user at concurrency 1024, pre-merge PR container (good reference); total ~179,747 tok/s across 24 GPUs; config=reference",
   "params": "FP4 weights + FP4 indexer, MTP2",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP4",
   "quote": "Model: deepseek-ai/DeepSeek-V4-Pro; Hardware: GB200, 24 GPUs total; Workload: ISL 8192 / OSL 1024, concurrency 1024, 10,240 completed requests; tok/s/user = 1000 / median TPOT_ms; PR container: vLLM 0.20.1rc1.dev38+g61c3a50f4 | completed | ~7,489 | ~22.9 | ~43.7 ms | 2.358 | ~179,747",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #41603 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/41603",
   "tps": 22.9,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 1024,
   "ctx": null,
   "date": "2026-05",
   "flags": [],
   "hardware": "GB200",
   "id": "vllm-41603-gb200-dsv4-pro-fp4-v0201",
   "model": "DeepSeek-V4-Pro",
   "notes": "same 24-GPU 2P/1D workload on the official v0.20.1 release image (InferenceX/GHA run); the pre-merge PR container reference on the same workload is ~22.9 tok/s/user; config=v0.20.1",
   "params": "FP4 weights + FP4 indexer, MTP2",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP4",
   "quote": "Model: deepseek-ai/DeepSeek-V4-Pro; Hardware: GB200, 24 GPUs total; Workload: ISL 8192 / OSL 1024, concurrency 1024, 10,240 completed requests; Official vllm/vllm-openai:v0.20.1-ubuntu2404 | completed | 5,860 | 16.71 | 59.86 ms | 2.359 | ~140,634",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #41603 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/41603",
   "tps": 16.71,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 1024,
   "ctx": null,
   "date": "2026-05",
   "flags": [],
   "hardware": "GB200",
   "id": "vllm-41603-gb200-dsv4-pro-fp4-nightly",
   "model": "DeepSeek-V4-Pro",
   "notes": "same workload on the nightly build with the #41015 FP32->FP4 cvt regression; reverting #41015 on the nightly recovers to 21.75 tok/s/user in the same table; config=nightly",
   "params": "FP4 weights + FP4 indexer, MTP2",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP4",
   "quote": "Model: deepseek-ai/DeepSeek-V4-Pro; Hardware: GB200, 24 GPUs total; Workload: ISL 8192 / OSL 1024, concurrency 1024, 10,240 completed requests; Nightly: vLLM 0.20.1rc1.dev91+ga749a33d8 | completed | 5,305 | 15.0 | 66.83 ms | 2.360 | 127,326",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #41603 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/41603",
   "tps": 15.0,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 32,
   "ctx": null,
   "date": "2026-09",
   "flags": [],
   "hardware": "GB200",
   "id": "vllm-56975-gb200-glm53-flash-fp8-c32",
   "model": "GLM-5.3-Flash",
   "notes": "aggregate output tok/s at c=32, default --max-num-batched-tokens 16384, TP4; c=64/128 in the same row are 2607/3377; p99 ITL 199 ms at c=32 (SGLang on the same tray: 13 ms, per the issue)",
   "params": "MoE, FP8 checkpoint",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP8",
   "quote": "1 tray of GB200 NVL72 (4\u00d7 GB200, SM100), zai-org/GLM-5.3-Flash FP8, recipe single_node_tp; Chunk size does not bound it. Same tray, one job, fresh server per column; 16384 (default) 1892 2607 3377 199 337 342 14.6",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #56975 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/56975",
   "tps": 1892,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 64,
   "ctx": null,
   "date": "2026-09",
   "flags": [],
   "hardware": "GB200",
   "id": "vllm-56975-gb200-glm53-flash-fp8-c64",
   "model": "GLM-5.3-Flash",
   "notes": "aggregate output tok/s at c=64, default --max-num-batched-tokens 16384, TP4; p99 ITL 337 ms at c=64; raising --max-cudagraph-capture-size to the chunk size cuts the p99 ITL tail by about half at -7% throughput per the same issue",
   "params": "MoE, FP8 checkpoint",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP8",
   "quote": "1 tray of GB200 NVL72 (4\u00d7 GB200, SM100), zai-org/GLM-5.3-Flash FP8, recipe single_node_tp; Chunk size does not bound it. Same tray, one job, fresh server per column; 16384 (default) 1892 2607 3377 199 337 342 14.6",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #56975 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/56975",
   "tps": 2607,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 128,
   "ctx": null,
   "date": "2026-09",
   "flags": [],
   "hardware": "GB200",
   "id": "vllm-56975-gb200-glm53-flash-fp8-c128",
   "model": "GLM-5.3-Flash",
   "notes": "aggregate output tok/s at c=128, default --max-num-batched-tokens 16384, TP4; p99 ITL 342 ms at c=128; every slow step carries prefill tokens (decode-only steps run 14.1 ms flat per the issue)",
   "params": "MoE, FP8 checkpoint",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP8",
   "quote": "1 tray of GB200 NVL72 (4\u00d7 GB200, SM100), zai-org/GLM-5.3-Flash FP8, recipe single_node_tp; Chunk size does not bound it. Same tray, one job, fresh server per column; 16384 (default) 1892 2607 3377 199 337 342 14.6",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #56975 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/56975",
   "tps": 3377,
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": null,
   "ctx": null,
   "date": "2025-12",
   "flags": [],
   "hardware": "H20",
   "id": "vllm-30832-h20-dsv32",
   "model": "DeepSeek-V3.2",
   "notes": "quoted in the issue from the official vLLM recipe (docs.vllm.ai, DeepSeek-V3_2.html, TP8 benchmark output); vllm bench serve, random ISL/OSL 2048/1024, request-rate 10, 100 prompts; vLLM 0.13.0rc2; same block: peak output 1300.00, total 2375.18 tok/s",
   "params": "MoE",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "as-published",
   "quote": "We used the following script to benchmark deepseek-ai/DeepSeek-V3.2 on 8xH20; Output token throughput (tok/s): 791.73",
   "ram_gb": null,
   "retrieved": "2026-09-26",
   "scope": "cluster",
   "source_name": "vLLM issue #30832 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/30832",
   "tps": "791.73",
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 120,
   "ctx": null,
   "date": "2026-05",
   "flags": [],
   "hardware": "H200",
   "id": "vllm-43648-h200-dsv4flash-fp8-dp4",
   "model": "DeepSeek-V4-Flash",
   "notes": "DP4 + expert parallel across 4x H200; crash-report run: server died mid-benchmark, 135 of 600 requests completed; vLLM v0.20.1, --kv-cache-dtype fp8, --block-size 256; max request concurrency 120",
   "params": "V4-Flash MoE",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "FP8",
   "quote": "GPU model: NVIDIA H200; Model: DeepSeek-V4-Flash-FP8; --enable-expert-parallel --data-parallel-size 4; Successful requests: 135; Failed requests: 465; Output token throughput (tok/s): 1118.18",
   "ram_gb": null,
   "retrieved": "2026-09-27",
   "scope": "cluster",
   "source_name": "vLLM issue #43648 (community report)",
   "source_url": "https://github.com/vllm-project/vllm/issues/43648",
   "tps": "1118.18",
   "ttft_s": null
  },
  {
   "backend": "vLLM",
   "batch": 512,
   "ctx": null,
   "date": "2026-04",
   "flags": [],
   "hardware": "B300",
   "id": "vllm-40350-b300-qwen35-397b-nvfp4-c512",
   "model": "Qwen3.5-397B-A17B",
   "notes": "4x B300 node, TP=4 EP=4, MTP3 + async scheduling, FP8 KV; engine-log snapshot during the c=512 bench (bug: the last request then hangs at 0.0 tok/s); first ~1000 of 1536 requests completed at full throughput; scope=cluster (multi-GPU node); config=mtp3-ep4",
   "params": "397B MoE A17B",
   "power_w": null,
   "pp_tps": null,
   "provenance": "community",
   "quant": "NVFP4",
   "quote": "GPU: NVIDIA B300 \u00d7 4 (single node, TP=4, EP=4); --model nvidia/Qwen3.5-397B-A17B-NVFP4; --max-concurrency 512; Avg generation throughput: 12921.2 tokens/s, Running: 493 reqs",
   "retrieved": "2026-09-27",
   "scope": "cluster",
   "source_name": "https://github.com/vllm-project/vllm/issues/40350",
   "source_url": "https://github.com/vllm-project/vllm/issues/40350",
   "tps": "12921.2",
   "ttft_s": null
  }
 ],
 "retrieved": "2026-10-22"
}