christian.meyer/lai_automatisierung
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

docs: record Qwen MLX benchmark results
Some checks failed
CI / test (push) Has been cancelled
Details
CI / test (pull_request) Has been cancelled
Details

Browse Source
This commit is contained in:
Christian Meyer
2026-08-31 16:39:37 +02:00
parent 75b2c8958c
commit 407af8099b
40 changed files with 6091 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json Normal file
View File

@@ -0,0 +1,48 @@
{
"run_id": "20260831T133636Z",
"created_at": "2026-08-31T13:37:14.110488+00:00",
"host": {
"platform": "macOS-15.6-arm64-arm-64bit",
"machine": "arm64",
"python": "3.12.8"
},
"arguments": {
"model": "mlx-community/Qwen3.8-27B-nvfp4",
"label": "qwen3.8-27b-nvfp4",
"warmups": 2,
"rounds": 5,
"max_tokens": 128,
"seed": 42,
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results"
},
"load_s": 3.066661207994912,
"mlx_version": "0.32.0",
"device": "Device(gpu, 0)",
"prompts": {
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
},
"summary": {
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"requests": 5,
"successes": 5,
"ttft_s_mean": 0.4445558497915044,
"ttft_s_p50": 0.48620820802170783,
"ttft_s_p95": 0.518837732996326,
"total_s_mean": 4.9355954667902555,
"total_s_p50": 4.966667582979426,
"total_s_p95": 5.034370008390397,
"prompt_tps_mean": 107.94239735907567,
"prompt_tps_p50": 107.53363787342614,
"prompt_tps_p95": 135.27062378239268,
"decode_tps_mean": 28.570950408778582,
"decode_tps_p50": 28.53361993265224,
"decode_tps_p95": 28.826484879691197,
"request_tps_mean": 25.943347049499913,
"request_tps_p50": 25.771807325831702,
"request_tps_p95": 26.64316963019791,
"peak_memory_gb": 15.479377792
}
}

128
benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json Normal file
View File

@@ -0,0 +1,128 @@
[
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "warmup",
"round": 1,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 128,
"ttft_s": 0.6381363340187818,
"total_s": 5.067599917005282,
"prompt_tps": 46.64172306513033,
"decode_tps": 28.964400213729846,
"request_tps": 25.258505425906254,
"peak_memory_gb": 15.399916034,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "warmup",
"round": 2,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.46635333297308534,
"total_s": 4.885915416001808,
"prompt_tps": 122.76309757245242,
"decode_tps": 29.032294926274737,
"request_tps": 26.19775192603388,
"peak_memory_gb": 15.453767792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 1,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 128,
"ttft_s": 0.3432638749945909,
"total_s": 4.786967416992411,
"prompt_tps": 90.81617646556052,
"decode_tps": 28.873430911706816,
"request_tps": 26.73926702438696,
"peak_memory_gb": 15.453767792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 2,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.48620820802170783,
"total_s": 4.966667582979426,
"prompt_tps": 118.1052752991172,
"decode_tps": 28.63870075162873,
"request_tps": 25.771807325831702,
"peak_memory_gb": 15.453784176,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 3,
"prompt_name": "code",
"prompt_tokens": 54,
"output_tokens": 128,
"ttft_s": 0.5125948329805396,
"total_s": 5.009093041997403,
"prompt_tps": 139.56196090321154,
"decode_tps": 28.53361993265224,
"request_tps": 25.553528139089888,
"peak_memory_gb": 15.479377792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 4,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 128,
"ttft_s": 0.36031387496041134,
"total_s": 4.874560041993391,
"prompt_tps": 83.69493625406292,
"decode_tps": 28.424103574919425,
"request_tps": 26.258780053441704,
"peak_memory_gb": 15.479377792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 5,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.5203984580002725,
"total_s": 5.040689249988645,
"prompt_tps": 107.53363787342614,
"decode_tps": 28.384896872985717,
"request_tps": 25.393352704749322,
"peak_memory_gb": 15.479377792,
"finish_reason": "length",
"error": null
}
]

22
benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json Normal file
View File

@@ -0,0 +1,22 @@
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4",
"requests": 5,
"successes": 5,
"ttft_s_mean": 0.4445558497915044,
"ttft_s_p50": 0.48620820802170783,
"ttft_s_p95": 0.518837732996326,
"total_s_mean": 4.9355954667902555,
"total_s_p50": 4.966667582979426,
"total_s_p95": 5.034370008390397,
"prompt_tps_mean": 107.94239735907567,
"prompt_tps_p50": 107.53363787342614,
"prompt_tps_p95": 135.27062378239268,
"decode_tps_mean": 28.570950408778582,
"decode_tps_p50": 28.53361993265224,
"decode_tps_p95": 28.826484879691197,
"request_tps_mean": 25.943347049499913,
"request_tps_p50": 25.771807325831702,
"request_tps_p95": 26.64316963019791,
"peak_memory_gb": 15.479377792
}

48
benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json Normal file
View File

@@ -0,0 +1,48 @@
{
"run_id": "20260831T133721Z",
"created_at": "2026-08-31T13:38:28.661722+00:00",
"host": {
"platform": "macOS-15.6-arm64-arm-64bit",
"machine": "arm64",
"python": "3.12.8"
},
"arguments": {
"model": "mlx-community/Qwen3.8-27B-mxfp8",
"label": "qwen3.8-27b-mxfp8",
"warmups": 2,
"rounds": 5,
"max_tokens": 128,
"seed": 42,
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results"
},
"load_s": 5.387068083975464,
"mlx_version": "0.32.0",
"device": "Device(gpu, 0)",
"prompts": {
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
},
"summary": {
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"requests": 5,
"successes": 5,
"ttft_s_mean": 0.6405866249930113,
"ttft_s_p50": 0.6006166669540107,
"ttft_s_p95": 0.993492508213967,
"total_s_mean": 8.97830267478712,
"total_s_p50": 8.952084249991458,
"total_s_p95": 9.543902474397328,
"prompt_tps_mean": 77.25518114106583,
"prompt_tps_p50": 69.803885982221,
"prompt_tps_p95": 108.54351756783554,
"decode_tps_mean": 15.141694584402668,
"decode_tps_p50": 15.417877788139895,
"decode_tps_p95": 15.90376325022775,
"request_tps_mean": 14.036390549836813,
"request_tps_p50": 14.298346220336581,
"request_tps_p95": 14.998850485179887,
"peak_memory_gb": 28.05746427
}
}

128
benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json Normal file
View File

@@ -0,0 +1,128 @@
[
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "warmup",
"round": 1,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 122,
"ttft_s": 1.4950010409811512,
"total_s": 8.873082707985304,
"prompt_tps": 65.2268651572191,
"decode_tps": 16.569121716474896,
"request_tps": 13.749449206667087,
"peak_memory_gb": 27.989241503,
"finish_reason": "stop",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "warmup",
"round": 2,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.5392710830201395,
"total_s": 8.420522083004471,
"prompt_tps": 104.9724920224018,
"decode_tps": 16.272217242824215,
"request_tps": 15.200957700514595,
"peak_memory_gb": 28.031514318,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "measured",
"round": 1,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 122,
"ttft_s": 0.41662891599116847,
"total_s": 8.074155374953989,
"prompt_tps": 69.803885982221,
"decode_tps": 15.96651063653789,
"request_tps": 15.109939595470719,
"peak_memory_gb": 28.031514318,
"finish_reason": "stop",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "measured",
"round": 2,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.6006166669540107,
"total_s": 8.794534499989823,
"prompt_tps": 94.30109779838891,
"decode_tps": 15.652773704987183,
"request_tps": 14.554494044016558,
"peak_memory_gb": 28.031531726,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "measured",
"round": 3,
"prompt_name": "code",
"prompt_tokens": 54,
"output_tokens": 128,
"ttft_s": 0.6342117090243846,
"total_s": 8.952084249991458,
"prompt_tps": 112.1041225101972,
"decode_tps": 15.417877788139895,
"request_tps": 14.298346220336581,
"peak_memory_gb": 28.05746427,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "measured",
"round": 4,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 122,
"ttft_s": 0.46816312498413026,
"total_s": 9.549591040995438,
"prompt_tps": 61.572468489546665,
"decode_tps": 13.468195538794124,
"request_tps": 12.775416190731752,
"peak_memory_gb": 28.05746427,
"finish_reason": "stop",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"source": "mlx-community/Qwen3.8-27B-mxfp8",
"phase": "measured",
"round": 5,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 1.0833127080113627,
"total_s": 9.521148208004888,
"prompt_tps": 48.494330924975344,
"decode_tps": 15.203115253554254,
"request_tps": 13.443756698628452,
"peak_memory_gb": 28.05746427,
"finish_reason": "length",
"error": null
}
]

22
benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json Normal file
View File

@@ -0,0 +1,22 @@
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-mxfp8",
"requests": 5,
"successes": 5,
"ttft_s_mean": 0.6405866249930113,
"ttft_s_p50": 0.6006166669540107,
"ttft_s_p95": 0.993492508213967,
"total_s_mean": 8.97830267478712,
"total_s_p50": 8.952084249991458,
"total_s_p95": 9.543902474397328,
"prompt_tps_mean": 77.25518114106583,
"prompt_tps_p50": 69.803885982221,
"prompt_tps_p95": 108.54351756783554,
"decode_tps_mean": 15.141694584402668,
"decode_tps_p50": 15.417877788139895,
"decode_tps_p95": 15.90376325022775,
"request_tps_mean": 14.036390549836813,
"request_tps_p50": 14.298346220336581,
"request_tps_p95": 14.998850485179887,
"peak_memory_gb": 28.05746427
}

48
benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json Normal file
View File

@@ -0,0 +1,48 @@
{
"run_id": "20260831T134407Z",
"created_at": "2026-08-31T13:44:45.718463+00:00",
"host": {
"platform": "macOS-15.6-arm64-arm-64bit",
"machine": "arm64",
"python": "3.12.8"
},
"arguments": {
"model": "mlx-community/Qwen3.8-27B-nvfp4",
"label": "qwen3.8-27b-nvfp4-rerun",
"warmups": 2,
"rounds": 5,
"max_tokens": 128,
"seed": 42,
"output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results"
},
"load_s": 3.153125083015766,
"mlx_version": "0.32.0",
"device": "Device(gpu, 0)",
"prompts": {
"short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.",
"medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.",
"code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code."
},
"summary": {
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"requests": 5,
"successes": 5,
"ttft_s_mean": 0.46268870839849113,
"ttft_s_p50": 0.5028267079615034,
"ttft_s_p95": 0.5507681754184887,
"total_s_mean": 5.0599139665835535,
"total_s_p50": 5.047160332964268,
"total_s_p95": 5.222982274997048,
"prompt_tps_mean": 103.80325950757997,
"prompt_tps_p50": 102.6553925326314,
"prompt_tps_p95": 128.99882432098076,
"decode_tps_mean": 27.91928963114296,
"decode_tps_p50": 27.87385628056727,
"decode_tps_p95": 28.51407145210707,
"request_tps_mean": 25.315032232739288,
"request_tps_p50": 25.360795289977208,
"request_tps_p95": 26.258409367849122,
"peak_memory_gb": 15.479377792
}
}

128
benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json Normal file
View File

@@ -0,0 +1,128 @@
[
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "warmup",
"round": 1,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 128,
"ttft_s": 0.46846333297435194,
"total_s": 4.887791082961485,
"prompt_tps": 84.84884256090237,
"decode_tps": 29.034596267228054,
"request_tps": 26.18769866130316,
"peak_memory_gb": 15.399948802,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "warmup",
"round": 2,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.476554416003637,
"total_s": 4.906530833046418,
"prompt_tps": 122.50184376172285,
"decode_tps": 28.96545575207384,
"request_tps": 26.0876787195335,
"peak_memory_gb": 15.453767792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 1,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 128,
"ttft_s": 0.34607333299936727,
"total_s": 4.835503207985312,
"prompt_tps": 90.02367046337802,
"decode_tps": 28.58315107058229,
"request_tps": 26.470874797192113,
"peak_memory_gb": 15.453767792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 2,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.5028267079615034,
"total_s": 5.047160332964268,
"prompt_tps": 113.11164547028159,
"decode_tps": 28.2377529782062,
"request_tps": 25.360795289977208,
"peak_memory_gb": 15.453784176,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 3,
"prompt_name": "code",
"prompt_tokens": 54,
"output_tokens": 128,
"ttft_s": 0.5305347089888528,
"total_s": 5.134004874969833,
"prompt_tps": 132.97061903365554,
"decode_tps": 27.87385628056727,
"request_tps": 24.93180336155254,
"peak_memory_gb": 15.479377792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 4,
"prompt_name": "short",
"prompt_tokens": 20,
"output_tokens": 128,
"ttft_s": 0.3781822500168346,
"total_s": 5.037674791994505,
"prompt_tps": 80.25497003795336,
"decode_tps": 27.538176565956825,
"request_tps": 25.408547650477164,
"peak_memory_gb": 15.479377792,
"finish_reason": "length",
"error": null
},
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"source": "mlx-community/Qwen3.8-27B-nvfp4",
"phase": "measured",
"round": 5,
"prompt_name": "medium",
"prompt_tokens": 43,
"output_tokens": 128,
"ttft_s": 0.5558265420258977,
"total_s": 5.245226625003852,
"prompt_tps": 102.6553925326314,
"decode_tps": 27.363511260402223,
"request_tps": 24.403140064497403,
"peak_memory_gb": 15.479377792,
"finish_reason": "length",
"error": null
}
]

22
benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json Normal file
View File

@@ -0,0 +1,22 @@
{
"runtime": "mlx-lm-direct",
"model": "qwen3.8-27b-nvfp4-rerun",
"requests": 5,
"successes": 5,
"ttft_s_mean": 0.46268870839849113,
"ttft_s_p50": 0.5028267079615034,
"ttft_s_p95": 0.5507681754184887,
"total_s_mean": 5.0599139665835535,
"total_s_p50": 5.047160332964268,
"total_s_p95": 5.222982274997048,
"prompt_tps_mean": 103.80325950757997,
"prompt_tps_p50": 102.6553925326314,
"prompt_tps_p95": 128.99882432098076,
"decode_tps_mean": 27.91928963114296,
"decode_tps_p50": 27.87385628056727,
"decode_tps_p95": 28.51407145210707,
"request_tps_mean": 25.315032232739288,
"request_tps_p50": 25.360795289977208,
"request_tps_p95": 26.258409367849122,
"peak_memory_gb": 15.479377792
}