docs: record Qwen MLX benchmark results
This commit is contained in:
Christian Meyer
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"run_id": "20260831T133636Z",
|
||||
"created_at": "2026-08-31T13:37:14.110488+00:00",
|
||||
"host": {
|
||||
"platform": "macOS-15.6-arm64-arm-64bit",
|
||||
"machine": "arm64",
|
||||
"python": "3.12.8"
|
||||
},
|
||||
"arguments": {
|
||||
"model": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"label": "qwen3.8-27b-nvfp4",
|
||||
"warmups": 2,
|
||||
"rounds": 5,
|
||||
"max_tokens": 128,
|
||||
"seed": 42,
|
||||
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results"
|
||||
},
|
||||
"load_s": 3.066661207994912,
|
||||
"mlx_version": "0.32.0",
|
||||
"device": "Device(gpu, 0)",
|
||||
"prompts": {
|
||||
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
|
||||
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
|
||||
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
|
||||
},
|
||||
"summary": {
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"requests": 5,
|
||||
"successes": 5,
|
||||
"ttft_s_mean": 0.4445558497915044,
|
||||
"ttft_s_p50": 0.48620820802170783,
|
||||
"ttft_s_p95": 0.518837732996326,
|
||||
"total_s_mean": 4.9355954667902555,
|
||||
"total_s_p50": 4.966667582979426,
|
||||
"total_s_p95": 5.034370008390397,
|
||||
"prompt_tps_mean": 107.94239735907567,
|
||||
"prompt_tps_p50": 107.53363787342614,
|
||||
"prompt_tps_p95": 135.27062378239268,
|
||||
"decode_tps_mean": 28.570950408778582,
|
||||
"decode_tps_p50": 28.53361993265224,
|
||||
"decode_tps_p95": 28.826484879691197,
|
||||
"request_tps_mean": 25.943347049499913,
|
||||
"request_tps_p50": 25.771807325831702,
|
||||
"request_tps_p95": 26.64316963019791,
|
||||
"peak_memory_gb": 15.479377792
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
[
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "warmup",
|
||||
"round": 1,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.6381363340187818,
|
||||
"total_s": 5.067599917005282,
|
||||
"prompt_tps": 46.64172306513033,
|
||||
"decode_tps": 28.964400213729846,
|
||||
"request_tps": 25.258505425906254,
|
||||
"peak_memory_gb": 15.399916034,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "warmup",
|
||||
"round": 2,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.46635333297308534,
|
||||
"total_s": 4.885915416001808,
|
||||
"prompt_tps": 122.76309757245242,
|
||||
"decode_tps": 29.032294926274737,
|
||||
"request_tps": 26.19775192603388,
|
||||
"peak_memory_gb": 15.453767792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 1,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.3432638749945909,
|
||||
"total_s": 4.786967416992411,
|
||||
"prompt_tps": 90.81617646556052,
|
||||
"decode_tps": 28.873430911706816,
|
||||
"request_tps": 26.73926702438696,
|
||||
"peak_memory_gb": 15.453767792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 2,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.48620820802170783,
|
||||
"total_s": 4.966667582979426,
|
||||
"prompt_tps": 118.1052752991172,
|
||||
"decode_tps": 28.63870075162873,
|
||||
"request_tps": 25.771807325831702,
|
||||
"peak_memory_gb": 15.453784176,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 3,
|
||||
"prompt_name": "code",
|
||||
"prompt_tokens": 54,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.5125948329805396,
|
||||
"total_s": 5.009093041997403,
|
||||
"prompt_tps": 139.56196090321154,
|
||||
"decode_tps": 28.53361993265224,
|
||||
"request_tps": 25.553528139089888,
|
||||
"peak_memory_gb": 15.479377792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 4,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.36031387496041134,
|
||||
"total_s": 4.874560041993391,
|
||||
"prompt_tps": 83.69493625406292,
|
||||
"decode_tps": 28.424103574919425,
|
||||
"request_tps": 26.258780053441704,
|
||||
"peak_memory_gb": 15.479377792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 5,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.5203984580002725,
|
||||
"total_s": 5.040689249988645,
|
||||
"prompt_tps": 107.53363787342614,
|
||||
"decode_tps": 28.384896872985717,
|
||||
"request_tps": 25.393352704749322,
|
||||
"peak_memory_gb": 15.479377792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4",
|
||||
"requests": 5,
|
||||
"successes": 5,
|
||||
"ttft_s_mean": 0.4445558497915044,
|
||||
"ttft_s_p50": 0.48620820802170783,
|
||||
"ttft_s_p95": 0.518837732996326,
|
||||
"total_s_mean": 4.9355954667902555,
|
||||
"total_s_p50": 4.966667582979426,
|
||||
"total_s_p95": 5.034370008390397,
|
||||
"prompt_tps_mean": 107.94239735907567,
|
||||
"prompt_tps_p50": 107.53363787342614,
|
||||
"prompt_tps_p95": 135.27062378239268,
|
||||
"decode_tps_mean": 28.570950408778582,
|
||||
"decode_tps_p50": 28.53361993265224,
|
||||
"decode_tps_p95": 28.826484879691197,
|
||||
"request_tps_mean": 25.943347049499913,
|
||||
"request_tps_p50": 25.771807325831702,
|
||||
"request_tps_p95": 26.64316963019791,
|
||||
"peak_memory_gb": 15.479377792
|
||||
}
|
||||
Reference in New Issue
Block a user