136 lines
5.2 KiB
JSON
136 lines
5.2 KiB
JSON
{
|
|
"run_id": "20260831T132015Z",
|
|
"created_at": "2026-08-31T13:20:15.702115+00:00",
|
|
"host": {
|
|
"platform": "macOS-15.6-arm64-arm-64bit",
|
|
"machine": "arm64",
|
|
"python": "3.12.8"
|
|
},
|
|
"arguments": {
|
|
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results",
|
|
"runtime": "vllm",
|
|
"ollama_url": "http://127.0.0.1:11434",
|
|
"ollama_model": "qwen3.8:27b-mlx",
|
|
"vllm_url": "http://127.0.0.1:8001",
|
|
"vllm_model": "qwen3.8-27b-mxfp8",
|
|
"warmups": 2,
|
|
"rounds": 5,
|
|
"concurrency": [
|
|
1,
|
|
4
|
|
],
|
|
"max_tokens": 128,
|
|
"seed": 42,
|
|
"timeout": 600
|
|
},
|
|
"runtimes": [
|
|
{
|
|
"name": "vllm-metal",
|
|
"kind": "openai",
|
|
"base_url": "http://127.0.0.1:8001",
|
|
"model": "qwen3.8-27b-mxfp8"
|
|
}
|
|
],
|
|
"versions": {
|
|
"ollama": "ollama version is 0.32.14",
|
|
"vllm": "INFO 08-31 15:20:16 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 15:20:16 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 15:20:16 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 15:20:18 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 15:20:19 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu"
|
|
},
|
|
"model_metadata": {
|
|
"vllm-metal": {
|
|
"object": "list",
|
|
"data": [
|
|
{
|
|
"id": "qwen3.8-27b-mxfp8",
|
|
"object": "model",
|
|
"created": 1788182421,
|
|
"owned_by": "vllm",
|
|
"root": "mlx-community/Qwen3.8-27B-mxfp8",
|
|
"parent": null,
|
|
"max_model_len": 8192,
|
|
"permission": [
|
|
{
|
|
"id": "modelperm-a3c90f9b6d588379",
|
|
"object": "model_permission",
|
|
"created": 1788182421,
|
|
"allow_create_engine": false,
|
|
"allow_sampling": true,
|
|
"allow_logprobs": true,
|
|
"allow_search_indices": false,
|
|
"allow_view": true,
|
|
"allow_fine_tuning": false,
|
|
"organization": "*",
|
|
"group": null,
|
|
"is_blocking": false
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
},
|
|
"prompts": {
|
|
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
|
|
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
|
|
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
|
|
},
|
|
"measurement_notes": {
|
|
"ttft": "Client wall time from request start until first non-empty generated text chunk.",
|
|
"total": "Client wall time from request start until stream completion.",
|
|
"decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).",
|
|
"aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time."
|
|
},
|
|
"completed_at": "2026-08-31T13:22:48.218681+00:00",
|
|
"summary": [
|
|
{
|
|
"runtime": "vllm-metal",
|
|
"model": "qwen3.8-27b-mxfp8",
|
|
"concurrency": 1,
|
|
"requests": 5,
|
|
"successes": 5,
|
|
"failures": 0,
|
|
"ttft_s_mean": 0.37155269999057056,
|
|
"ttft_s_p50": 0.4171057919738814,
|
|
"ttft_s_p95": 0.4531440335791558,
|
|
"total_s_mean": 7.930883799993898,
|
|
"total_s_p50": 8.838838333962485,
|
|
"total_s_p95": 8.998963149765041,
|
|
"prompt_tps_mean": null,
|
|
"prompt_tps_p50": null,
|
|
"prompt_tps_p95": null,
|
|
"decode_tps_mean": 15.079897408512505,
|
|
"decode_tps_p50": 15.126147490283795,
|
|
"decode_tps_p95": 15.196162811046243,
|
|
"request_tps_mean": 14.380896425252896,
|
|
"request_tps_p50": 14.386982882152838,
|
|
"request_tps_p95": 14.53502856074605,
|
|
"aggregate_output_tps_mean": 14.380548747737752,
|
|
"aggregate_output_tps_p50": 14.386712701914696,
|
|
"aggregate_output_tps_p95": 14.534638049200744
|
|
},
|
|
{
|
|
"runtime": "vllm-metal",
|
|
"model": "qwen3.8-27b-mxfp8",
|
|
"concurrency": 4,
|
|
"requests": 20,
|
|
"successes": 20,
|
|
"failures": 0,
|
|
"ttft_s_mean": 1.255672066603438,
|
|
"ttft_s_p50": 1.3144360205042176,
|
|
"ttft_s_p95": 1.419064954231726,
|
|
"total_s_mean": 12.251293779097614,
|
|
"total_s_p50": 12.702340687508695,
|
|
"total_s_p95": 13.641910566360457,
|
|
"prompt_tps_mean": null,
|
|
"prompt_tps_p50": null,
|
|
"prompt_tps_p95": null,
|
|
"decode_tps_mean": 10.500220268362742,
|
|
"decode_tps_p50": 10.485677148644122,
|
|
"decode_tps_p95": 11.071162546140778,
|
|
"request_tps_mean": 9.417789343799708,
|
|
"request_tps_p50": 9.430282582501674,
|
|
"request_tps_p95": 10.077752113288362,
|
|
"aggregate_output_tps_mean": 35.09087333775396,
|
|
"aggregate_output_tps_p50": 34.96408475660248,
|
|
"aggregate_output_tps_p95": 35.52678209465127
|
|
}
|
|
]
|
|
} |