docs: record Qwen MLX benchmark results
This commit is contained in:
Christian Meyer
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"run_id": "20260831T134407Z",
|
||||
"created_at": "2026-08-31T13:44:45.718463+00:00",
|
||||
"host": {
|
||||
"platform": "macOS-15.6-arm64-arm-64bit",
|
||||
"machine": "arm64",
|
||||
"python": "3.12.8"
|
||||
},
|
||||
"arguments": {
|
||||
"model": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"label": "qwen3.8-27b-nvfp4-rerun",
|
||||
"warmups": 2,
|
||||
"rounds": 5,
|
||||
"max_tokens": 128,
|
||||
"seed": 42,
|
||||
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results"
|
||||
},
|
||||
"load_s": 3.153125083015766,
|
||||
"mlx_version": "0.32.0",
|
||||
"device": "Device(gpu, 0)",
|
||||
"prompts": {
|
||||
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
|
||||
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
|
||||
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
|
||||
},
|
||||
"summary": {
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"requests": 5,
|
||||
"successes": 5,
|
||||
"ttft_s_mean": 0.46268870839849113,
|
||||
"ttft_s_p50": 0.5028267079615034,
|
||||
"ttft_s_p95": 0.5507681754184887,
|
||||
"total_s_mean": 5.0599139665835535,
|
||||
"total_s_p50": 5.047160332964268,
|
||||
"total_s_p95": 5.222982274997048,
|
||||
"prompt_tps_mean": 103.80325950757997,
|
||||
"prompt_tps_p50": 102.6553925326314,
|
||||
"prompt_tps_p95": 128.99882432098076,
|
||||
"decode_tps_mean": 27.91928963114296,
|
||||
"decode_tps_p50": 27.87385628056727,
|
||||
"decode_tps_p95": 28.51407145210707,
|
||||
"request_tps_mean": 25.315032232739288,
|
||||
"request_tps_p50": 25.360795289977208,
|
||||
"request_tps_p95": 26.258409367849122,
|
||||
"peak_memory_gb": 15.479377792
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
[
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "warmup",
|
||||
"round": 1,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.46846333297435194,
|
||||
"total_s": 4.887791082961485,
|
||||
"prompt_tps": 84.84884256090237,
|
||||
"decode_tps": 29.034596267228054,
|
||||
"request_tps": 26.18769866130316,
|
||||
"peak_memory_gb": 15.399948802,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "warmup",
|
||||
"round": 2,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.476554416003637,
|
||||
"total_s": 4.906530833046418,
|
||||
"prompt_tps": 122.50184376172285,
|
||||
"decode_tps": 28.96545575207384,
|
||||
"request_tps": 26.0876787195335,
|
||||
"peak_memory_gb": 15.453767792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 1,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.34607333299936727,
|
||||
"total_s": 4.835503207985312,
|
||||
"prompt_tps": 90.02367046337802,
|
||||
"decode_tps": 28.58315107058229,
|
||||
"request_tps": 26.470874797192113,
|
||||
"peak_memory_gb": 15.453767792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 2,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.5028267079615034,
|
||||
"total_s": 5.047160332964268,
|
||||
"prompt_tps": 113.11164547028159,
|
||||
"decode_tps": 28.2377529782062,
|
||||
"request_tps": 25.360795289977208,
|
||||
"peak_memory_gb": 15.453784176,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 3,
|
||||
"prompt_name": "code",
|
||||
"prompt_tokens": 54,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.5305347089888528,
|
||||
"total_s": 5.134004874969833,
|
||||
"prompt_tps": 132.97061903365554,
|
||||
"decode_tps": 27.87385628056727,
|
||||
"request_tps": 24.93180336155254,
|
||||
"peak_memory_gb": 15.479377792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 4,
|
||||
"prompt_name": "short",
|
||||
"prompt_tokens": 20,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.3781822500168346,
|
||||
"total_s": 5.037674791994505,
|
||||
"prompt_tps": 80.25497003795336,
|
||||
"decode_tps": 27.538176565956825,
|
||||
"request_tps": 25.408547650477164,
|
||||
"peak_memory_gb": 15.479377792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
},
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"source": "mlx-community/Qwen3.8-27B-nvfp4",
|
||||
"phase": "measured",
|
||||
"round": 5,
|
||||
"prompt_name": "medium",
|
||||
"prompt_tokens": 43,
|
||||
"output_tokens": 128,
|
||||
"ttft_s": 0.5558265420258977,
|
||||
"total_s": 5.245226625003852,
|
||||
"prompt_tps": 102.6553925326314,
|
||||
"decode_tps": 27.363511260402223,
|
||||
"request_tps": 24.403140064497403,
|
||||
"peak_memory_gb": 15.479377792,
|
||||
"finish_reason": "length",
|
||||
"error": null
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"runtime": "mlx-lm-direct",
|
||||
"model": "qwen3.8-27b-nvfp4-rerun",
|
||||
"requests": 5,
|
||||
"successes": 5,
|
||||
"ttft_s_mean": 0.46268870839849113,
|
||||
"ttft_s_p50": 0.5028267079615034,
|
||||
"ttft_s_p95": 0.5507681754184887,
|
||||
"total_s_mean": 5.0599139665835535,
|
||||
"total_s_p50": 5.047160332964268,
|
||||
"total_s_p95": 5.222982274997048,
|
||||
"prompt_tps_mean": 103.80325950757997,
|
||||
"prompt_tps_p50": 102.6553925326314,
|
||||
"prompt_tps_p95": 128.99882432098076,
|
||||
"decode_tps_mean": 27.91928963114296,
|
||||
"decode_tps_p50": 27.87385628056727,
|
||||
"decode_tps_p95": 28.51407145210707,
|
||||
"request_tps_mean": 25.315032232739288,
|
||||
"request_tps_p50": 25.360795289977208,
|
||||
"request_tps_p95": 26.258409367849122,
|
||||
"peak_memory_gb": 15.479377792
|
||||
}
|
||||
Reference in New Issue
Block a user