docs: record Qwen MLX benchmark results
This commit is contained in:
Christian Meyer
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"run_id": "20260831T133721Z",
|
||||
"created_at": "2026-08-31T13:38:28.661722+00:00",
|
||||
"host": {
|
||||
"platform": "macOS-15.6-arm64-arm-64bit",
|
||||
"machine": "arm64",
|
||||
"python": "3.12.8"
|
||||
},
|
||||
"arguments": {
|
||||
"model": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"label": "qwen3.8-27b-mxfp8",
|
||||
"warmups": 2,
|
||||
"rounds": 5,
|
||||
"max_tokens": 128,
|
||||
"seed": 42,
|
||||
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results"
|
||||
},
|
||||
"load_s": 5.387068083975464,
|
||||
"mlx_version": "0.32.0",
|
||||
"device": "Device(gpu, 0)",
|
||||
"prompts": {
|
||||
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
|
||||
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
|
||||
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
|
||||
},
|
||||
"summary": {
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"requests": 5,
|
||||
"successes": 5,
|
||||
"ttft_s_mean": 0.6405866249930113,
|
||||
"ttft_s_p50": 0.6006166669540107,
|
||||
"ttft_s_p95": 0.993492508213967,
|
||||
"total_s_mean": 8.97830267478712,
|
||||
"total_s_p50": 8.952084249991458,
|
||||
"total_s_p95": 9.543902474397328,
|
||||
"prompt_tps_mean": 77.25518114106583,
|
||||
"prompt_tps_p50": 69.803885982221,
|
||||
"prompt_tps_p95": 108.54351756783554,
|
||||
"decode_tps_mean": 15.141694584402668,
|
||||
"decode_tps_p50": 15.417877788139895,
|
||||
"decode_tps_p95": 15.90376325022775,
|
||||
"request_tps_mean": 14.036390549836813,
|
||||
"request_tps_p50": 14.298346220336581,
|
||||
"request_tps_p95": 14.998850485179887,
|
||||
"peak_memory_gb": 28.05746427
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
[
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "warmup",
|
||||
"round": 1,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 122,
|
||||
"ttft_s": 1.4950010409811512,
|
||||
"total_s": 8.873082707985304,
|
||||
"prompt_tps": 65.2268651572191,
|
||||
"decode_tps": 16.569121716474896,
|
||||
"request_tps": 13.749449206667087,
|
||||
"peak_memory_gb": 27.989241503,
|
||||
"finish_reason": "stop",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "warmup",
|
||||
"round": 2,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.5392710830201395,
|
||||
"total_s": 8.420522083004471,
|
||||
"prompt_tps": 104.9724920224018,
|
||||
"decode_tps": 16.272217242824215,
|
||||
"request_tps": 15.200957700514595,
|
||||
"peak_memory_gb": 28.031514318,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "measured",
|
||||
"round": 1,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 122,
|
||||
"ttft_s": 0.41662891599116847,
|
||||
"total_s": 8.074155374953989,
|
||||
"prompt_tps": 69.803885982221,
|
||||
"decode_tps": 15.96651063653789,
|
||||
"request_tps": 15.109939595470719,
|
||||
"peak_memory_gb": 28.031514318,
|
||||
"finish_reason": "stop",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "measured",
|
||||
"round": 2,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.6006166669540107,
|
||||
"total_s": 8.794534499989823,
|
||||
"prompt_tps": 94.30109779838891,
|
||||
"decode_tps": 15.652773704987183,
|
||||
"request_tps": 14.554494044016558,
|
||||
"peak_memory_gb": 28.031531726,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "measured",
|
||||
"round": 3,
|
||||
"prompt_name": "code",
|
||||
"prompt_tokens": 54,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.6342117090243846,
|
||||
"total_s": 8.952084249991458,
|
||||
"prompt_tps": 112.1041225101972,
|
||||
"decode_tps": 15.417877788139895,
|
||||
"request_tps": 14.298346220336581,
|
||||
"peak_memory_gb": 28.05746427,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "measured",
|
||||
"round": 4,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 122,
|
||||
"ttft_s": 0.46816312498413026,
|
||||
"total_s": 9.549591040995438,
|
||||
"prompt_tps": 61.572468489546665,
|
||||
"decode_tps": 13.468195538794124,
|
||||
"request_tps": 12.775416190731752,
|
||||
"peak_memory_gb": 28.05746427,
|
||||
"finish_reason": "stop",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"source": "mlx-community/Qwen3.8-27B-mxfp8",
|
||||
"phase": "measured",
|
||||
"round": 5,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 1.0833127080113627,
|
||||
"total_s": 9.521148208004888,
|
||||
"prompt_tps": 48.494330924975344,
|
||||
"decode_tps": 15.203115253554254,
|
||||
"request_tps": 13.443756698628452,
|
||||
"peak_memory_gb": 28.05746427,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-mxfp8",
|
||||
"requests": 5,
|
||||
"successes": 5,
|
||||
"ttft_s_mean": 0.6405866249930113,
|
||||
"ttft_s_p50": 0.6006166669540107,
|
||||
"ttft_s_p95": 0.993492508213967,
|
||||
"total_s_mean": 8.97830267478712,
|
||||
"total_s_p50": 8.952084249991458,
|
||||
"total_s_p95": 9.543902474397328,
|
||||
"prompt_tps_mean": 77.25518114106583,
|
||||
"prompt_tps_p50": 69.803885982221,
|
||||
"prompt_tps_p95": 108.54351756783554,
|
||||
"decode_tps_mean": 15.141694584402668,
|
||||
"decode_tps_p50": 15.417877788139895,
|
||||
"decode_tps_p95": 15.90376325022775,
|
||||
"request_tps_mean": 14.036390549836813,
|
||||
"request_tps_p50": 14.298346220336581,
|
||||
"request_tps_p95": 14.998850485179887,
|
||||
"peak_memory_gb": 28.05746427
|
||||
}
|
||||
Reference in New Issue
Block a user