From 407af8099b35e7e729c5c1b31f4cc2b1eda123fa Mon Sep 17 00:00:00 2001 From: Christian Meyer Date: Mon, 31 Aug 2026 16:39:37 +0200 Subject: [PATCH] docs: record Qwen MLX benchmark results --- benchmarks/inference/REPORT.md | 193 ++++ .../results.json | 111 +++ .../summary.json | 53 ++ .../results.json | 111 +++ .../summary.json | 53 ++ .../results.json | 111 +++ .../summary.json | 53 ++ .../results.json | 111 +++ .../summary.json | 53 ++ .../metadata.json | 48 + .../raw.json | 128 +++ .../summary.json | 22 + .../metadata.json | 48 + .../raw.json | 128 +++ .../summary.json | 22 + .../metadata.json | 48 + .../raw.json | 128 +++ .../summary.json | 22 + .../results/20260831T114727Z/metadata.json | 136 +++ .../results/20260831T114727Z/raw.csv | 36 + .../results/20260831T114727Z/raw.json | 842 ++++++++++++++++++ .../results/20260831T114727Z/summary.json | 54 ++ .../results/20260831T114928Z/metadata.json | 135 +++ .../results/20260831T114928Z/raw.csv | 36 + .../results/20260831T114928Z/raw.json | 842 ++++++++++++++++++ .../results/20260831T114928Z/summary.json | 54 ++ .../results/20260831T121855Z/metadata.json | 135 +++ .../results/20260831T121855Z/raw.csv | 36 + .../results/20260831T121855Z/raw.json | 842 ++++++++++++++++++ .../results/20260831T121855Z/summary.json | 54 ++ .../results/20260831T132015Z/metadata.json | 136 +++ .../results/20260831T132015Z/raw.csv | 36 + .../results/20260831T132015Z/raw.json | 842 ++++++++++++++++++ .../results/20260831T132015Z/summary.json | 54 ++ .../results/20260831T134334Z/metadata.json | 108 +++ .../results/20260831T134334Z/raw.csv | 8 + .../results/20260831T134334Z/raw.json | 170 ++++ .../results/20260831T134334Z/summary.json | 28 + .../results/backend_verification.json | 37 + benchmarks/inference/results/cold_start.json | 27 + 40 files changed, 6091 insertions(+) create mode 100644 benchmarks/inference/REPORT.md create mode 100644 benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json create mode 100644 benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json create mode 100644 benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json create mode 100644 benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json create mode 100644 benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json create mode 100644 benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json create mode 100644 benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json create mode 100644 benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json create mode 100644 benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json create mode 100644 benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json create mode 100644 benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json create mode 100644 benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json create mode 100644 benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json create mode 100644 benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json create mode 100644 benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json create mode 100644 benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json create mode 100644 benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json create mode 100644 benchmarks/inference/results/20260831T114727Z/metadata.json create mode 100644 benchmarks/inference/results/20260831T114727Z/raw.csv create mode 100644 benchmarks/inference/results/20260831T114727Z/raw.json create mode 100644 benchmarks/inference/results/20260831T114727Z/summary.json create mode 100644 benchmarks/inference/results/20260831T114928Z/metadata.json create mode 100644 benchmarks/inference/results/20260831T114928Z/raw.csv create mode 100644 benchmarks/inference/results/20260831T114928Z/raw.json create mode 100644 benchmarks/inference/results/20260831T114928Z/summary.json create mode 100644 benchmarks/inference/results/20260831T121855Z/metadata.json create mode 100644 benchmarks/inference/results/20260831T121855Z/raw.csv create mode 100644 benchmarks/inference/results/20260831T121855Z/raw.json create mode 100644 benchmarks/inference/results/20260831T121855Z/summary.json create mode 100644 benchmarks/inference/results/20260831T132015Z/metadata.json create mode 100644 benchmarks/inference/results/20260831T132015Z/raw.csv create mode 100644 benchmarks/inference/results/20260831T132015Z/raw.json create mode 100644 benchmarks/inference/results/20260831T132015Z/summary.json create mode 100644 benchmarks/inference/results/20260831T134334Z/metadata.json create mode 100644 benchmarks/inference/results/20260831T134334Z/raw.csv create mode 100644 benchmarks/inference/results/20260831T134334Z/raw.json create mode 100644 benchmarks/inference/results/20260831T134334Z/summary.json create mode 100644 benchmarks/inference/results/backend_verification.json create mode 100644 benchmarks/inference/results/cold_start.json diff --git a/benchmarks/inference/REPORT.md b/benchmarks/inference/REPORT.md new file mode 100644 index 0000000..ea7fbee --- /dev/null +++ b/benchmarks/inference/REPORT.md @@ -0,0 +1,193 @@ +# Benchmark report: Ollama, vLLM-Metal, NVFP4, and MXFP8 + +Date: 2026-08-31 + +## Result + +For this Apple M4 Max and the tested Qwen3.8-27B packages: + +- **Interactive/single-request use: Ollama NVFP4 wins.** It delivered 1.83x the vLLM-Metal NVFP4 median decode speed. It was also 1.50x faster than vLLM-Metal for MXFP8. +- **Four concurrent requests: vLLM-Metal wins.** NVFP4 delivered 1.62x and MXFP8 1.73x Ollama's median aggregate output throughput, with much lower tail TTFT. +- **Quantization: NVFP4 wins decisively for speed and memory.** MXFP8 approximately doubles weight storage and reduced median single-stream decode speed by 50% in Ollama and 39% in vLLM-Metal. +- **Complex batch-1 coding: Ollama wins every measured task.** Depending on quantization and task, vLLM-Metal took 1.53x to 2.37x as long. +- **Practical choice:** use Ollama NVFP4 for interactive chat, coding, and one agent at a time. Use vLLM-Metal NVFP4 when several users or jobs generate concurrently and aggregate capacity or tail TTFT matters. MXFP8 is not attractive for speed or memory on this machine; use it only if its output quality is proven better for the target workload. + +All 50 measured requests succeeded. Warmups are excluded from the table. + +Both serving stacks were conclusively verified as native Apple GPU implementations. Ollama logged an MLX runner, MLX engine `0.32.0-213-gadf21de`, `device=gpu`, and Metal. vLLM-Metal activated its Metal plugin, set MLX to `Device(gpu, 0)`, set PyTorch to MPS, and loaded native paged-attention Metal kernels. The `0.28.0+cpu` vLLM version string describes the portable core wheel, not the active compute backend. + +## Measurements + +| Workload and metric | Ollama | vLLM-Metal | Result | +|---|---:|---:|---| +| Concurrency 1, TTFT p50 | 0.119 s | 0.372 s | Ollama 3.12x faster | +| Concurrency 1, TTFT p95 | 0.393 s | 0.420 s | Ollama 1.07x faster | +| Concurrency 1, total latency p50 | 2.754 s | 5.455 s | Ollama 1.98x faster | +| Concurrency 1, total latency p95 | 3.451 s | 5.562 s | Ollama 1.61x faster | +| Concurrency 1, decode p50 | 45.73 tok/s | 24.96 tok/s | Ollama 1.83x faster | +| Concurrency 1, aggregate output p50 | 44.19 tok/s | 23.46 tok/s | Ollama 1.88x faster | +| Concurrency 4, TTFT p50 | 5.078 s | 1.169 s | vLLM-Metal 4.34x faster | +| Concurrency 4, TTFT p95 | 10.691 s | 1.236 s | vLLM-Metal 8.65x faster | +| Concurrency 4, total latency p50 | 8.619 s | 9.273 s | Ollama 1.08x faster | +| Concurrency 4, total latency p95 | 14.521 s | 9.591 s | vLLM-Metal 1.51x faster | +| Concurrency 4, aggregate output p50 | 33.92 tok/s | 54.89 tok/s | vLLM-Metal 1.62x faster | + +The concurrency-4 median individual latency does not represent group completion: Ollama returned some requests early while queueing others. Its p95 TTFT reached 10.69 seconds. Aggregate output rate and p95 latency are the relevant capacity metrics here. + +## MXFP8 measurements + +All 50 measured MXFP8 requests succeeded. The protocol is identical to the NVFP4 test. + +| Workload and metric | Ollama | vLLM-Metal | Result | +|---|---:|---:|---| +| Concurrency 1, TTFT p50 | 0.259 s | 0.417 s | Ollama 1.61x faster | +| Concurrency 1, TTFT p95 | 0.583 s | 0.453 s | vLLM-Metal 1.29x faster | +| Concurrency 1, total latency p50 | 5.272 s | 8.839 s | Ollama 1.68x faster | +| Concurrency 1, total latency p95 | 5.728 s | 8.999 s | Ollama 1.57x faster | +| Concurrency 1, decode p50 | 22.74 tok/s | 15.13 tok/s | Ollama 1.50x faster | +| Concurrency 1, aggregate output p50 | 22.22 tok/s | 14.39 tok/s | Ollama 1.54x faster | +| Concurrency 4, TTFT p50 | 9.394 s | 1.314 s | vLLM-Metal 7.15x faster | +| Concurrency 4, TTFT p95 | 18.583 s | 1.419 s | vLLM-Metal 13.10x faster | +| Concurrency 4, total latency p50 | 14.902 s | 12.702 s | vLLM-Metal 1.17x faster | +| Concurrency 4, total latency p95 | 24.082 s | 13.642 s | vLLM-Metal 1.77x faster | +| Concurrency 4, aggregate output p50 | 20.18 tok/s | 34.96 tok/s | vLLM-Metal 1.73x faster | + +## Complex coding tasks + +These runs used native chat templates, thinking disabled, temperature 0, seed 42, an 8,192-token context, and a 3,072-token output cap. Complete generated responses are retained with their hashes. + +### NVFP4 + +| Task | Ollama total | Ollama decode | vLLM total | vLLM decode | Result by total time | +|---|---:|---:|---:|---:|---| +| Python async TTL cache | 56.41 s | 55.38 tok/s | 133.44 s | 23.20 tok/s | Ollama 2.37x faster | +| Laravel idempotent transfer | 78.72 s | 39.58 tok/s | 166.01 s | 18.92 tok/s | Ollama 2.11x faster* | +| TypeScript DAG scheduler | 96.54 s | 32.30 tok/s | 152.22 s | 20.38 tok/s | Ollama 1.58x faster | + +\* The vLLM PHP response stopped naturally at 2,840 tokens; the Ollama response reached 3,072. Its normalized decode rate was still 2.09x lower. + +### MXFP8 + +| Task | Ollama total | Ollama decode | vLLM total | vLLM decode | Result by total time | +|---|---:|---:|---:|---:|---| +| Python async TTL cache | 149.82 s | 20.64 tok/s | 247.71 s | 12.46 tok/s | Ollama 1.65x faster | +| Laravel idempotent transfer | 149.77 s | 20.72 tok/s | 242.90 s | 12.73 tok/s | Ollama 1.62x faster | +| TypeScript DAG scheduler | 147.28 s | 21.06 tok/s | 224.71 s | 13.76 tok/s | Ollama 1.53x faster | + +Except for the vLLM NVFP4 PHP response, every coding run reached the 3,072-token cap. These figures therefore measure time to produce a fixed amount of long-form code, not time to a guaranteed complete or correct implementation. The generated code was retained as evidence but was not compiled as a cross-model quality benchmark because the model artifacts produced different implementations. + +## Exact-checkpoint direct MLX control + +The direct control removes HTTP, vLLM scheduling, paged-cache management, and request-state overhead. It loads the exact same Hugging Face artifact used by vLLM through MLX-LM 0.31.3. It uses the same raw prompts, two warmups, five measured requests, temperature 0, seed 42, and 128 output tokens. + +| Quantization | Direct MLX decode p50 | vLLM decode p50 | Difference | Direct MLX peak memory | +|---|---:|---:|---:|---:| +| NVFP4 | 28.53 tok/s | 24.96 tok/s | vLLM 12.5% lower | 15.48 GB | +| MXFP8 | 15.42 tok/s | 15.13 tok/s | vLLM 1.9% lower | 28.06 GB | + +This isolates part of the NVFP4 gap as serving overhead. It also shows that short MXFP8 batch-1 decode is almost identical between direct MLX-LM and vLLM-Metal. It does not make the Ollama comparison artifact-identical: Ollama's packed weights differ. Long hybrid-GDN sequences still incur additional vLLM state and paging work that this short control does not represent. + +A fresh back-to-back NVFP4 rerun, with Ollama unloaded before direct MLX-LM started, confirmed the practical batch-1 comparison: + +| Metric | Ollama | Direct MLX-LM | Result | +|---|---:|---:|---| +| TTFT p50 | 0.120 s | 0.503 s | Ollama 4.20x faster | +| Decode p50 | 45.28 tok/s | 27.87 tok/s | Ollama 1.62x faster | +| Request throughput p50 | 44.57 tok/s | 25.36 tok/s | Ollama 1.76x faster | + +The direct MLX-LM rerun was 38.4% below Ollama's median decode throughput. This remains a practical-package comparison rather than a byte-identical runtime comparison. + +## Memory and package sizes + +The previously observed approximately 32 GB for vLLM NVFP4 was not a 32-GB model. It was model memory plus a large cache reservation. + +| Item | NVFP4 | MXFP8 | +|---|---:|---:| +| Ollama package label | about 18 GB | about 31 GB | +| Hugging Face checkpoint | 16.1 GB | 28.7 GB | +| Direct MLX peak memory | 15.48 GB | 28.06 GB | +| vLLM logged model memory | 15.13 GB | 27.74 GB | +| vLLM runtime overhead | 1.16 GB | 1.17 GB | +| vLLM logical KV/cache budget | 15.05 GB | 18.51 GB | + +For MXFP8, vLLM reported 8.94 GB as the concrete MPS KV-cache capacity inside the larger 18.51-GB logical cache budget. Its default 0.92 memory-utilization target provisioned capacity for roughly ten concurrent 8K requests, although the benchmark used at most four short requests. That reservation is serving capacity, not weight size. + +## Why vLLM-Metal is slower at batch 1 + +- Ollama uses a tightly integrated Qwen-specific MLX runner and a newer/custom MLX engine revision. +- vLLM-Metal adds scheduling, paged KV-cache management, request state, block-table translation, and hybrid Gated DeltaNet state management. +- Qwen3.8 has 16 SDPA and 48 linear/GDN layers. vLLM-Metal labels this hybrid paged-attention path experimental and translates its large logical blocks into Metal-kernel-compatible blocks. +- Some shapes can use lazy/fused Metal kernels; unsupported cases use less favorable convolution or recurrent fallback paths. +- These costs buy batching: vLLM loses single-stream latency but wins aggregate throughput and tail TTFT at concurrency four. +- The direct exact-checkpoint result confirms measurable vLLM overhead for NVFP4. The much larger Ollama advantage in long coding runs also includes different packed artifacts and different runtime integration, so it cannot be assigned exclusively to the scheduler. + +## Token-count check + +The runtimes did not always stop at the same token count because the packed artifacts generate different output and encounter EOS differently: + +| Workload | Ollama output tokens | vLLM-Metal output tokens | +|---|---:|---:| +| Five measured requests, concurrency 1 | 576 | 640 | +| Twenty measured requests, concurrency 4 | 2,340 | 2,556 | + +The fixed short prompt ended at 96–98 tokens in Ollama and at 124–128 tokens in vLLM-Metal. Therefore raw total latency alone is not a fair speed metric. Decode tokens per second and aggregate output tokens per second normalize for this difference and support the same conclusions. + +## Test protocol + +- Host: MacBook Pro, Apple M4 Max, 64 GB unified memory, macOS 15.6, arm64 +- Ollama: 0.32.14, native MLX execution +- vLLM core: 0.28.0+cpu with native `vllm-metal` 0.4.0 development plugin; logs confirmed `Device(gpu, 0)` and MPS +- Context limit: 8,192 tokens +- Sampling: temperature 0, seed 42 +- Output limit: 128 tokens +- Two warmup groups, then five measured groups at concurrency 1 and 4 +- Three fixed raw-completion prompts +- Ollama speculative drafting explicitly disabled +- Runtimes loaded and tested sequentially to prevent unified-memory contention + +The `+cpu` suffix belongs to the vLLM core wheel. In this setup, the Metal plugin supplied GPU execution; this was not a CPU benchmark. + +## Model-equivalence limitation + +- Ollama package: `qwen3.8:27b-mlx` +- vLLM-Metal package: `mlx-community/Qwen3.8-27B-nvfp4` +- Hugging Face revision: `5ff8ef173ad0d7c3aae92f0be43031a6ab8067c6` +- Both are based on Qwen3.8-27B and declare NVFP4 group-size-16 quantization. +- They are not byte-identical. A corresponding 44,564,480-byte packed weight tensor had different SHA-256 hashes. + +This is a comparison of the closest practical native runtime/model packages, not a pure runtime-overhead comparison using identical packed weights. Quantization/conversion differences can account for some of the speed difference. + +## Cold start + +Cold startup was measured separately after both model packages were cached locally: + +| Measurement | Ollama | vLLM-Metal | +|---|---:|---:| +| Model-weight load | 3.245 s | 3.25 s | +| Operational readiness | 3.455 s for a cold one-token request | approximately 28.5 s from process launch to API ready | + +The model-weight load itself was effectively tied. vLLM-Metal additionally spent 9.71 seconds profiling, allocating its 15-GB KV cache, and warming kernels; Python process and plugin startup account for the remaining launch time. Ollama's daemon was already running, so its value measures unloaded-model-to-response rather than daemon process launch. These operational measurements are useful but not perfectly symmetric. + +The first vLLM setup took about 12 minutes 45 seconds to download and reconstruct the 16.1-GB checkpoint. That one-time network cost is excluded from the cached cold-start and all warm-inference results. + +## Caveats + +- Prompt-throughput values from Ollama were excluded because repeated short prompts produced cache-hit durations near zero; vLLM's completion stream did not expose equivalent per-request prefill duration. +- Five groups establish a clear directional result, but longer multi-run tests are appropriate before capacity planning. +- Both runtimes used prefix caching as shipped. That reflects practical warmed operation, not cache-disabled kernel isolation. +- Total latency is influenced by differing EOS positions; normalized token rates are more comparable. +- Ollama and Hugging Face model pairs share the Qwen3.8-27B base and nominal quantization mode but are not byte-identical. Results compare practical deployable packages, not only runtime code. +- Direct MLX controls are batch-1 controls only; MLX-LM does not reproduce vLLM's concurrent serving objective. + +## Evidence + +- vLLM-Metal: `results/20260831T114727Z/metadata.json`, `raw.json`, `raw.csv`, and `summary.json` +- Ollama: `results/20260831T114928Z/metadata.json`, `raw.json`, `raw.csv`, and `summary.json` +- Cold start: `results/cold_start.json` +- Backend proof: `results/backend_verification.json` +- Ollama MXFP8: `results/20260831T121855Z/` +- vLLM-Metal MXFP8: `results/20260831T132015Z/` +- NVFP4 coding: `coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/` and `coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/` +- MXFP8 coding: `coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/` and `coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/` +- Direct MLX controls: `direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/` and `direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/` +- Back-to-back rerun: `results/20260831T134334Z/` and `direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/` diff --git a/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json new file mode 100644 index 0000000..0e4189f --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T120914Z", + "created_at": "2026-08-31T12:09:14.858871+00:00", + "runtime": { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mlx" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:05:22.804254+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.38745154201751575, + "total_s": 0.3874947500298731, + "prompt_tokens": 41, + "output_tokens": 1, + "decode_tps": 14.664458922599563, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:05:23.192111+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.021859125001356, + "total_s": 56.40731879102532, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 55.37840341105521, + "finish_reason": "length", + "output_chars": 12165, + "output_sha256": "a3355a481519820f9021b6846e3f784fba8aa922d3317c5a84d6762447d9dd97", + "output": "import asyncio\nimport time\nfrom collections import OrderedDict\nfrom dataclasses import dataclass, field\nfrom typing import (\n Any,\n Awaitable,\n Callable,\n Generic,\n Optional,\n TypeVar,\n)\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n factory_calls: int\n factory_errors: int\n invalidations: int\n\n\nclass _Entry(Generic[V]):\n __slots__ = (\"value\", \"expires_at\", \"weight\")\n\n def __init__(self, value: V, expires_at: float, weight: int) -> None:\n self.value = value\n self.expires_at = expires_at\n self.weight = weight\n\n\nclass _Inflight(Generic[V]):\n __slots__ = (\"event\", \"value\", \"error\", \"cancelled\")\n\n def __init__(self) -> None:\n self.event = asyncio.Event()\n self.value: Optional[V] = None\n self.error: Optional[BaseException] = None\n self.cancelled = False\n\n\nclass AsyncTTLCache(Generic[K, V]):\n def __init__(\n self,\n max_weight: int,\n default_ttl: float,\n *,\n weight_fn: Optional[Callable[[V], int]] = None,\n ) -> None:\n if max_weight <= 0:\n raise ValueError(\"max_weight must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n self._max_weight = max_weight\n self._default_ttl = default_ttl\n self._weight_fn = weight_fn or (lambda _v: 1)\n self._lock = asyncio.Lock()\n self._store: \"OrderedDict[K, _Entry[V]]\" = OrderedDict()\n self._inflight: \"dict[K, _Inflight[V]]\" = {}\n self._current_weight = 0\n self._stats = CacheStats(0, 0, 0, 0, 0, 0)\n\n @property\n def stats(self) -> CacheStats:\n return self._stats\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n weight: Optional[int] = None,\n ) -> V:\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None:\n if time.monotonic() < entry.expires_at:\n self._store.move_to_end(key)\n self._stats = CacheStats(\n self._stats.hits + 1,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n return entry.value\n self._remove_entry(key)\n\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses + 1,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n\n inflight = self._inflight.get(key)\n if inflight is not None:\n waiter = inflight\n else:\n waiter = _Inflight()\n self._inflight[key] = waiter\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls + 1,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n self._run_factory(key, factory, ttl, weight, waiter)\n\n await waiter.event.wait()\n\n if waiter.cancelled:\n raise asyncio.CancelledError()\n if waiter.error is not None:\n raise waiter.error\n return waiter.value\n\n def _run_factory(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: Optional[float],\n weight: Optional[int],\n waiter: _Inflight[V],\n ) -> None:\n async def runner() -> None:\n try:\n value = await factory()\n except asyncio.CancelledError:\n waiter.cancelled = True\n waiter.event.set()\n return\n except BaseException as exc:\n waiter.error = exc\n waiter.event.set()\n async with self._lock:\n self._inflight.pop(key, None)\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors + 1,\n self._stats.invalidations,\n )\n return\n\n async with self._lock:\n current = self._inflight.get(key)\n if current is not waiter:\n return\n self._inflight.pop(key, None)\n self._put_entry(key, value, ttl, weight)\n waiter.value = value\n waiter.event.set()\n\n asyncio.create_task(runner())\n\n def _put_entry(\n self,\n key: K,\n value: V,\n ttl: Optional[float],\n weight: Optional[int],\n ) -> None:\n if weight is None:\n weight = self._weight_fn(value)\n if weight <= 0:\n raise ValueError(\"weight must be positive\")\n if ttl is None:\n ttl = self._default_ttl\n if ttl <= 0:\n raise ValueError(\"ttl must be positive\")\n\n expires_at = time.monotonic() + ttl\n entry = _Entry(value, expires_at, weight)\n\n if self._current_weight + weight > self._max_weight:\n while self._current_weight > 0 and self._current_weight + weight > self._max_weight:\n evict_key, evict_entry = next(iter(self._store.items()))\n self._remove_entry(evict_key)\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions + 1,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n\n self._store[key] = entry\n self._store.move_to_end(key)\n self._current_weight += weight\n\n def _remove_entry(self, key: K) -> None:\n entry = self._store.pop(key, None)\n if entry is not None:\n self._current_weight -= entry.weight\n\n async def invalidate(self, key: K) -> bool:\n async with self._lock:\n removed = self._remove_entry(key)\n inflight = self._inflight.pop(key, None)\n if inflight is not None:\n inflight.cancelled = True\n inflight.event.set()\n if removed or inflight is not None:\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations + 1,\n )\n return True\n return False\n\n async def clear(self) -> None:\n async with self._lock:\n self._store.clear()\n self._current_weight = 0\n for inflight in self._inflight.values():\n inflight.cancelled = True\n inflight.event.set()\n self._inflight.clear()\n\n async def __aenter__(self) -> \"AsyncTTLCache[K, V]\":\n return self\n\n async def __aexit__(self, *args: Any) -> None:\n await self.clear()\n\n\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_basic_hit_and_miss() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> str:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.01)\n return \"value\"\n\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert calls == 1\n assert cache.stats.misses == 1\n assert cache.stats.hits == 0\n\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert calls == 1\n assert cache.stats.hits == 1\n\n\n@pytest.mark.asyncio\nasync def test_single_flight() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.05)\n return 42\n\n results = await asyncio.gather(\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n )\n assert all(r == 42 for r in results)\n assert calls == 1\n assert cache.stats.factory_calls == 1\n\n\n@pytest.mark.asyncio\nasync def test_cancellation_does_not_cancel_shared_factory() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.05)\n return 99\n\n task1 = asyncio.create_task(cache.get_or_create(1, factory))\n task2 = asyncio.create_task(cache.get_or_create(1, factory))\n await asyncio.sleep(0.01)\n task1.cancel()\n with pytest.raises(asyncio.CancelledError):\n await task1\n result = await task2\n assert result == 99\n assert calls == 1\n\n\n@pytest.mark.asyncio\nasync def test_expiry() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=0.05)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n return calls\n\n r1 = await cache.get_or_create(1, factory)\n assert r1 == 1\n await asyncio.sleep(0.06)\n r2 = await cache.get_or_create(1, factory)\n assert r2 == 2\n assert cache.stats.misses == 2\n\n\n@pytest.mark.asyncio\nasync def test_weighted_capacity_eviction() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n return calls\n\n await cache.get_or_create(1, factory, weight=6)\n await cache.get_or_create(2, factory, weight=5)\n assert cache.stats.evictions == 1\n assert 1 not in cache._store\n assert 2 in cache._store\n\n\n@pytest.mark.asyncio\nasync def test_factory_failure_not_cached() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n raise ValueError(\"fail\")\n\n with pytest.raises(ValueError):\n await cache.get_or_create(1, factory)\n assert cache.stats.factory_errors == 1\n assert 1 not in cache._store\n\n async def good_factory() -> int:\n return 42\n\n result = await cache.get_or_create(1, good_factory)\n assert result == 42\n assert cache.stats.factory_calls == 2\n\n\n@pytest.mark.asyncio\nasync def test_invalidation_during_creation_prevents_stale_publication() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.05)\n return 100\n\n task = asyncio.create_task(cache.get_or_create(1, factory))\n await asyncio.sleep(0.01)\n invalidated = await cache.invalidate(1)\n assert invalidated is True\n with pytest.raises(asyncio.CancelledError):\n await task\n assert 1 not in cache._store\n assert calls == 1\n\n async def new_factory() -> int:\n return 200\n\n result = await cache.get_or_create(1, new_factory)\n assert result == 200\n assert cache.stats.factory_calls == 2\n\n\n@pytest.mark.asyncio\nasync def test_clear_cancels_inflight() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n\n async def factory() -> int:\n await asyncio.sleep(0.05)\n return 1\n\n task = asyncio.create_task(cache.get_or_create(1, factory))\n await asyncio.sleep(0.01)\n await cache.clear()\n with pytest.raises(asyncio.CancelledError):\n await task", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:06:19.598953+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.2263128339545801, + "total_s": 78.72023624996655, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 39.58218973701569, + "finish_reason": "length", + "output_chars": 12654, + "output_sha256": "070acfea480b76f0913ce21686f2031f7a32421f5645f28eebe0c2ef9d2728ec", + "output": "```php\nexecuteTransfer($fromAccountId, $toAccountId, $amountCents, $idempotencyKey);\n } catch (\\Throwable $e) {\n if ($this->isDeadlock($e)) {\n $lastException = $e;\n // Exponential backoff with jitter\n $sleepMs = (int) (10 * pow(2, $attempt) + random_int(0, 10));\n usleep($sleepMs * 1000);\n continue;\n }\n throw $e;\n }\n }\n\n // All retries exhausted\n throw new \\RuntimeException(\n 'Transfer failed after ' . $maxRetries . ' deadlock retries.',\n 0,\n $lastException\n );\n }\n\n /**\n * Execute the transfer inside a database transaction.\n */\n private function executeTransfer(\n int $fromAccountId,\n int $toAccountId,\n int $amountCents,\n string $idempotencyKey\n ): array {\n return DB::transaction(function () use ($fromAccountId, $toAccountId, $amountCents, $idempotencyKey) {\n // 1. Check idempotency: if a transfer with this key already exists, return it.\n $existing = DB::table('transfers')\n ->where('idempotency_key', $idempotencyKey)\n ->lockForUpdate()\n ->first();\n\n if ($existing !== null) {\n return [\n 'transfer_id' => (int) $existing->id,\n 'idempotency_key' => $existing->idempotency_key,\n 'from_account_id' => (int) $existing->from_account_id,\n 'to_account_id' => (int) $existing->to_account_id,\n 'amount_cents' => (int) $existing->amount_cents,\n 'status' => $existing->status,\n 'was_duplicate' => true,\n ];\n }\n\n // 2. Lock both account rows in deterministic ID order to avoid deadlocks.\n $lockOrder = [$fromAccountId, $toAccountId];\n sort($lockOrder);\n\n $lockedAccounts = [];\n foreach ($lockOrder as $accountId) {\n $row = DB::table('accounts')\n ->where('id', $accountId)\n ->lockForUpdate()\n ->first();\n\n if ($row === null) {\n throw new InvalidTransferException(\"Account {$accountId} does not exist.\");\n }\n $lockedAccounts[$accountId] = $row;\n }\n\n // 3. Check sufficient funds on the source account.\n $fromAccount = $lockedAccounts[$fromAccountId];\n if ($fromAccount->balance_cents < $amountCents) {\n throw new InsufficientFundsException(\n (int) $fromAccount->balance_cents,\n $amountCents\n );\n }\n\n // 4. Perform the transfer: debit source, credit destination.\n DB::table('accounts')\n ->where('id', $fromAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents - ' . $amountCents),\n 'version' => DB::raw('version + 1'),\n ]);\n\n DB::table('accounts')\n ->where('id', $toAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents + ' . $amountCents),\n 'version' => DB::raw('version + 1'),\n ]);\n\n // 5. Persist the transfer record.\n $transferId = DB::table('transfers')->insertGetId([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'status' => 'completed',\n 'created_at' => now(),\n 'updated_at' => now(),\n ]);\n\n // 6. Persist the outbox event in the same transaction.\n DB::table('outbox_events')->insert([\n 'transfer_id' => $transferId,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transferId,\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n ]),\n 'created_at' => now(),\n ]);\n\n return [\n 'transfer_id' => (int) $transferId,\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'status' => 'completed',\n 'was_duplicate' => false,\n ];\n });\n }\n\n /**\n * Detect whether a Throwable represents a database deadlock.\n */\n private function isDeadlock(\\Throwable $e): bool\n {\n // MySQL/InnoDB deadlock error code is 1213\n // PostgreSQL deadlock error code is 40P01\n $message = $e->getMessage();\n $code = $e->getCode();\n\n if ($code === 1213 || $code === 40P01) {\n return true;\n }\n\n // Check SQLSTATE or message for deadlock indicators\n if (str_contains($message, 'Deadlock') || str_contains($message, 'deadlock')) {\n return true;\n }\n\n // Check for SQLSTATE 40001 (serialization failure)\n if (str_contains($message, '40001')) {\n return true;\n }\n\n return false;\n }\n}\n\n// ---------------------------------------------------------------------------\n// PHPUnit Tests\n// ---------------------------------------------------------------------------\n\nclass MoneyTransferServiceTest extends TestCase\n{\n use RefreshDatabase;\n\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Create the schema for testing\n Schema::create('accounts', function (Blueprint $table) {\n $table->id();\n $table->bigInteger('balance_cents')->default(0);\n $table->unsignedInteger('version')->default(0);\n $table->timestamps();\n });\n\n Schema::create('transfers', function (Blueprint $table) {\n $table->id();\n $table->string('idempotency_key')->unique();\n $table->unsignedBigInteger('from_account_id');\n $table->unsignedBigInteger('to_account_id');\n $table->bigInteger('amount_cents');\n $table->string('status')->default('completed');\n $table->timestamps();\n });\n\n Schema::create('outbox_events', function (Blueprint $table) {\n $table->id();\n $table->unsignedBigInteger('transfer_id');\n $table->string('event_type');\n $table->text('payload');\n $table->timestamps();\n });\n\n $this->service = new MoneyTransferService();\n }\n\n /**\n * Test that a valid transfer moves money correctly.\n */\n public function test_valid_transfer_moves_money(): void\n {\n $fromId = $this->createAccount(10000); // 100.00\n $toId = $this->createAccount(0); // 0.00\n\n $result = $this->service->transfer($fromId, $toId, 5000, 'key-001');\n\n $this->assertFalse($result['was_duplicate']);\n $this->assertEquals('completed', $result['status']);\n $this->assertEquals(5000, $result['amount_cents']);\n\n $fromBalance = DB::table('accounts')->where('id', $fromId)->value('balance_cents');\n $toBalance = DB::table('accounts')->where('id', $toId)->value('balance_cents');\n\n $this->assertEquals(5000, (int) $fromBalance);\n $this->assertEquals(5000, (int) $toBalance);\n }\n\n /**\n * Test that a duplicate idempotency key returns the original transfer\n * without moving money again.\n */\n public function test_duplicate_idempotency_key_returns_original_transfer(): void\n {\n $fromId = $this->createAccount(10000);\n $toId = $this->createAccount(0);\n\n $key = 'key-dup-001';\n\n $result1 = $this->service->transfer($fromId, $toId, 3000, $key);\n $this->assertFalse($result1['was_duplicate']);\n\n $result2 = $this->service->transfer($fromId, $toId, 3000, $key);\n $this->assertTrue($result2['was_duplicate']);\n\n // Balances should reflect only one transfer\n $fromBalance = DB::table('accounts')->where('id', $fromId)->value('balance_cents');\n $toBalance =", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:07:38.318694+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.5556320829782635, + "total_s": 96.54028320801444, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 32.29928081838954, + "finish_reason": "length", + "output_chars": 11211, + "output_sha256": "f97ad0b443b4fe7ae9a5b2a3f0cc3c8ad2b71bc2ba4d5849bd679b498488c654", + "output": "```typescript\n// strict-mode, dependency-free DAG task scheduler\n\nexport interface Task {\n id: string;\n deps: string[];\n run: (signal: AbortSignal) => Promise;\n}\n\nexport interface ScheduleOptions {\n maxConcurrency?: number;\n signal?: AbortSignal;\n}\n\nexport type TaskResults = Map;\n\nexport class CycleError extends Error {\n constructor(public readonly path: string[]) {\n super(`Cycle detected: ${path.join(\" -> \")}`);\n this.name = \"CycleError\";\n }\n}\n\nexport class ValidationError extends Error {\n constructor(message: string) {\n super(message);\n this.name = \"ValidationError\";\n }\n}\n\nexport class CancellationError extends Error {\n constructor() {\n super(\"Schedule cancelled\");\n this.name = \"CancellationError\";\n }\n}\n\nexport class TaskError extends Error {\n constructor(\n public readonly taskId: string,\n public readonly cause: unknown\n ) {\n super(`Task \"${taskId}\" failed: ${cause instanceof Error ? cause.message : String(cause)}`);\n this.name = \"TaskError\";\n }\n}\n\n// Internal node representation\ninterface Node {\n id: string;\n deps: string[];\n run: (signal: AbortSignal) => Promise;\n inDegree: number;\n dependents: string[];\n result?: unknown;\n error?: unknown;\n status: \"pending\" | \"running\" | \"done\" | \"failed\" | \"skipped\";\n promise?: Promise;\n}\n\n// Topological sort with cycle detection using DFS\nfunction topologicalSort(nodes: Map, order: string[]): string[] {\n const visited = new Set();\n const inStack = new Set();\n const sorted: string[] = [];\n\n function dfs(id: string, stack: string[]): void {\n if (inStack.has(id)) {\n // Found a cycle; extract the cycle path\n const cycleStart = stack.indexOf(id);\n const cyclePath = stack.slice(cycleStart).concat(id);\n throw new CycleError(cyclePath);\n }\n if (visited.has(id)) return;\n\n inStack.add(id);\n stack.push(id);\n\n const node = nodes.get(id)!;\n // Process dependencies in the order they appear in the input\n for (const dep of node.deps) {\n dfs(dep, stack);\n }\n\n stack.pop();\n inStack.delete(id);\n visited.add(id);\n sorted.push(id);\n }\n\n // Process nodes in input order for stable scheduling\n for (const id of order) {\n if (!visited.has(id)) {\n dfs(id, []);\n }\n }\n\n return sorted;\n}\n\n// Kahn's algorithm variant that preserves input order for stable scheduling\nfunction kahnSort(nodes: Map, inputOrder: string[]): string[] {\n // Build in-degree map\n const inDegree = new Map();\n const dependents = new Map();\n\n for (const [id, node] of nodes) {\n inDegree.set(id, node.deps.length);\n dependents.set(id, []);\n }\n\n for (const [id, node] of nodes) {\n for (const dep of node.deps) {\n const deps = dependents.get(dep);\n if (deps) {\n deps.push(id);\n }\n }\n }\n\n // Use a queue that preserves input order\n // Initialize with nodes that have in-degree 0, in input order\n const queue: string[] = [];\n for (const id of inputOrder) {\n if (inDegree.get(id) === 0) {\n queue.push(id);\n }\n }\n\n const sorted: string[] = [];\n let head = 0;\n\n while (head < queue.length) {\n const id = queue[head++];\n sorted.push(id);\n\n const deps = dependents.get(id) || [];\n for (const dep of deps) {\n const deg = inDegree.get(dep)!;\n inDegree.set(dep, deg - 1);\n if (inDegree.get(dep) === 0) {\n queue.push(dep);\n }\n }\n }\n\n if (sorted.length !== nodes.size) {\n // Cycle exists; use DFS to find the cycle path\n const visited = new Set();\n const inStack = new Set();\n\n function dfs(id: string, stack: string[]): void {\n if (inStack.has(id)) {\n const cycleStart = stack.indexOf(id);\n const cyclePath = stack.slice(cycleStart).concat(id);\n throw new CycleError(cyclePath);\n }\n if (visited.has(id)) return;\n\n inStack.add(id);\n stack.push(id);\n\n const node = nodes.get(id)!;\n for (const dep of node.deps) {\n dfs(dep, stack);\n }\n\n stack.pop();\n inStack.delete(id);\n visited.add(id);\n }\n\n for (const id of inputOrder) {\n if (!visited.has(id)) {\n dfs(id, []);\n }\n }\n // Should not reach here\n throw new CycleError([\"unknown\"]);\n }\n\n return sorted;\n}\n\nexport async function schedule(\n tasks: Task[],\n options: ScheduleOptions = {}\n): Promise {\n const maxConcurrency = options.maxConcurrency ?? Infinity;\n const externalSignal = options.signal;\n\n // Validate: check for duplicate IDs\n const idSet = new Set();\n for (const task of tasks) {\n if (idSet.has(task.id)) {\n throw new ValidationError(`Duplicate task ID: \"${task.id}\"`);\n }\n idSet.add(task.id);\n }\n\n // Validate: check for unknown dependencies\n for (const task of tasks) {\n for (const dep of task.deps) {\n if (!idSet.has(dep)) {\n throw new ValidationError(\n `Task \"${task.id}\" depends on unknown task \"${dep}\"`\n );\n }\n }\n }\n\n // Build node map\n const nodes = new Map();\n const inputOrder: string[] = [];\n\n for (const task of tasks) {\n nodes.set(task.id, {\n id: task.id,\n deps: task.deps,\n run: task.run,\n inDegree: task.deps.length,\n dependents: [],\n status: \"pending\",\n });\n inputOrder.push(task.id);\n }\n\n // Build dependents map\n for (const [id, node] of nodes) {\n for (const dep of node.deps) {\n const depNode = nodes.get(dep);\n if (depNode) {\n depNode.dependents.push(id);\n }\n }\n }\n\n // Detect cycles and get topological order\n const topoOrder = kahnSort(nodes, inputOrder);\n\n // Create an internal AbortController that we can abort on failure or external signal\n const controller = new AbortController();\n const internalSignal = controller.signal;\n\n // Link external signal to internal controller\n if (externalSignal) {\n if (externalSignal.aborted) {\n controller.abort();\n } else {\n externalSignal.addEventListener(\"abort\", () => {\n controller.abort();\n }, { once: true });\n }\n }\n\n // Track running tasks and their promises\n const runningPromises = new Set>();\n const results = new Map();\n let failed = false;\n let failureError: unknown = null;\n\n // Track which tasks have been started (to avoid starting dependents of failed tasks)\n const started = new Set();\n const skipped = new Set();\n\n // Helper: mark all transitive dependents as skipped\n function skipDependents(id: string): void {\n const node = nodes.get(id)!;\n for (const dep of node.dependents) {\n if (!skipped.has(dep) && !started.has(dep)) {\n skipped.add(dep);\n skipDependents(dep);\n }\n }\n }\n\n // Helper: check if all dependencies of a task are done (not failed/skipped)\n function depsReady(id: string): boolean {\n const node = nodes.get(id)!;\n for (const dep of node.deps) {\n const depNode = nodes.get(dep)!;\n if (depNode.status === \"failed\" || depNode.status === \"skipped\") {\n return false;\n }\n if (depNode.status !== \"done\") {\n return false;\n }\n }\n return true;\n }\n\n // Helper: check if a task should be skipped (any dep failed or skipped)\n function shouldSkip(id: string): boolean {\n const node = nodes.get(id)!;\n for (const dep of node.deps) {\n const depNode = nodes.get(dep)!;\n if (depNode.status === \"failed\" || depNode.status === \"skipped\") {\n return true;\n }\n }\n return false;\n }\n\n // Main scheduling loop\n // We process tasks in topological order, respecting concurrency limits\n // Use a pointer to the next task to consider\n let nextIndex = 0;\n const readyQueue: string[] = [];\n\n // Initialize ready queue with tasks that have no dependencies\n for (const id of topoOrder) {\n const node = nodes.get(id)!;\n if (node.deps.length === 0) {\n readyQueue.push(id);\n }\n }\n\n // We need to process tasks as they become ready\n // Use a more dynamic approach: after each task completes, check if new tasks are ready\n\n // Process the schedule\n async function runTask(id: string): Promise {\n const node = nodes.get(id)!;\n if (node.status !== \"pending\") return;\n\n // Check if we should skip this task\n if (shouldSkip(id)) {\n node.status = \"skipped\";\n skipDependents(id);\n return;\n }\n\n if (internalSignal.aborted) {\n node.status = \"skipped\";\n skipDependents(id);\n return;\n }\n\n started.add(id);\n node.status = \"running\";\n\n const taskPromise = node.run(internalSignal);\n node.promise = taskPromise;\n runningPromises.add(taskPromise);\n\n try {\n const result = await taskPromise;\n node.result = result;\n node.status = \"done\";\n results.set(id, result);\n } catch (err) {\n node.error = err;\n node.status = \"failed\";\n failed = true;\n failureError = new TaskError(id, err);\n // Abort all running tasks\n controller.abort();\n // Skip all dependents\n skipDependents(id);\n } finally {\n runningPromises.delete(taskPromise);\n }\n }\n\n // Process tasks in waves, respecting concurrency\n // We use a simple approach: maintain a set of running promises and a queue of ready tasks\n const running = new Set>();\n\n // Add initial ready tasks\n for (const id of readyQueue) {\n if (running.size >= maxConcurrency) break;\n if (skipped.has(id)) continue;\n const p = runTask(id);\n running.add(p);\n }\n\n // Process remaining tasks as they become ready\n // We need to wait for tasks to complete and then check for new ready tasks\n let processed = 0;\n const totalTasks = nodes.size;\n\n while (processed < totalTasks) {\n // Wait for at least one running task to complete\n if (running.size === 0) {\n // No more running tasks; check if there are more ready tasks\n // Find new ready tasks\n let foundNew = false;\n for (const id of topoOrder) {\n const node = nodes.get(id)!;\n if (node.status === \"pending\" && !skipped.has(id)) {\n if (depsReady(id)) {\n if (running.size >= maxConcurrency) break;\n const p = runTask(id);\n running.add(p);\n foundNew = true;\n }\n }\n }\n if (!foundNew) {\n // No more tasks to run\n break;\n }\n }\n\n // Wait for one task to complete\n if (running.size > 0) {\n // Use Promise.race but handle rejections\n const promises = Array.from(running);\n // We need to wait for one to settle without unhandled rejections\n // Create a race that catches rejections\n const settled = await Promise.race(\n promises.map((p) =>\n p.then(\n () => ({ type: \"fulfilled\" as const, promise: p }),\n () => ({ type: \"rejected\" as const, promise: p })\n )\n )\n );\n running.delete(settled.promise);\n processed++;\n }\n\n // After a task completes, check for new ready tasks\n if (!failed && !internalSignal.aborted) {\n for (const id of topo", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json new file mode 100644 index 0000000..b08f87e --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:05:23.192111+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.021859125001356, + "total_s": 56.40731879102532, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 55.37840341105521, + "finish_reason": "length", + "output_chars": 12165, + "output_sha256": "a3355a481519820f9021b6846e3f784fba8aa922d3317c5a84d6762447d9dd97", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:06:19.598953+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.2263128339545801, + "total_s": 78.72023624996655, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 39.58218973701569, + "finish_reason": "length", + "output_chars": 12654, + "output_sha256": "070acfea480b76f0913ce21686f2031f7a32421f5645f28eebe0c2ef9d2728ec", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:07:38.318694+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.5556320829782635, + "total_s": 96.54028320801444, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 32.29928081838954, + "finish_reason": "length", + "output_chars": 11211, + "output_sha256": "f97ad0b443b4fe7ae9a5b2a3f0cc3c8ad2b71bc2ba4d5849bd679b498488c654", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json new file mode 100644 index 0000000..6b2ad8e --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T121823Z", + "created_at": "2026-08-31T12:18:23.136973+00:00", + "runtime": { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-nvfp4" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:10:51.048391+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.38219533301889896, + "total_s": 0.41809329099487513, + "prompt_tokens": 41, + "output_tokens": 2, + "decode_tps": 55.71347543886622, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:10:51.466896+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.0361592500121333, + "total_s": 133.43742824997753, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 23.20219453486358, + "finish_reason": "length", + "output_chars": 14058, + "output_sha256": "a18e80f1852ecc0c70b987f244870b8a2dc20db8ad42505280162a4b17dbd057", + "output": "from __future__ import annotations\n\nimport asyncio\nimport time\nfrom dataclasses import dataclass, field\nfrom typing import (\n Any,\n Awaitable,\n Callable,\n Generic,\n Hashable,\n Optional,\n TypeVar,\n)\n\nK = TypeVar(\"K\", bound=Hashable)\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n\n\n@dataclass\nclass _Entry:\n value: V\n expires_at: float\n weight: int\n last_access: float\n\n\nclass _Inflight:\n __slots__ = (\"task\", \"cancelled\")\n\n def __init__(self, task: \"asyncio.Task[V]\") -> None:\n self.task = task\n self.cancelled = False\n\n\nclass AsyncTTLCache(Generic[K, V]):\n \"\"\"\n A generic async TTL cache with weighted LRU eviction, per-entry TTL,\n single-flight get_or_create, and safe cancellation semantics.\n\n - Weighted LRU: each entry has a weight; total weight <= capacity.\n - TTL: per-entry expiration based on time.monotonic.\n - Single-flight: concurrent get_or_create for the same key share one factory call.\n - Cancellation: cancelling one waiter does not cancel the shared factory.\n - Exceptions are never cached.\n - Invalidation during creation prevents stale publication.\n - The cache lock is never held while awaiting user code.\n \"\"\"\n\n def __init__(\n self,\n capacity: int,\n default_ttl: float,\n *,\n weight_fn: Optional[Callable[[V], int]] = None,\n ) -> None:\n if capacity <= 0:\n raise ValueError(\"capacity must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n\n self._capacity = capacity\n self._default_ttl = default_ttl\n self._weight_fn = weight_fn or (lambda _v: 1)\n\n # LRU order: most recently used at front (index 0)\n self._entries: dict[K, _Entry] = {}\n self._lru_order: list[K] = [] # index 0 = MRU, last = LRU\n self._total_weight: int = 0\n\n # Single-flight tracking\n self._inflight: dict[K, _Inflight] = {}\n\n # Statistics\n self._hits: int = 0\n self._misses: int = 0\n self._evictions: int = 0\n\n self._lock = asyncio.Lock()\n\n @property\n def stats(self) -> CacheStats:\n return CacheStats(\n hits=self._hits,\n misses=self._misses,\n evictions=self._evictions,\n )\n\n def clear(self) -> None:\n \"\"\"Clear all entries and cancel any in-flight factories.\"\"\"\n self._entries.clear()\n self._lru_order.clear()\n self._total_weight = 0\n for key in list(self._inflight.keys()):\n inflight = self._inflight.pop(key)\n if not inflight.task.done():\n inflight.cancelled = True\n inflight.task.cancel()\n\n def invalidate(self, key: K) -> None:\n \"\"\"\n Invalidate a key. If a factory is in-flight for this key,\n mark it as cancelled so its result is not published.\n \"\"\"\n self._entries.pop(key, None)\n if key in self._lru_order:\n self._lru_order.remove(key)\n # Adjust weight\n entry = self._entries.get(key)\n if entry is None and key in self._lru_order:\n pass # already removed\n # Recompute weight if entry was removed\n # Actually we need to track weight separately\n # Let's restructure: store weight in entry and recompute on removal\n # For simplicity, we track _total_weight and adjust on removal\n # But we already popped from _entries, so we need the weight before popping.\n # Let's fix: get weight before popping.\n # Actually, let's restructure invalidate to handle weight properly.\n # We'll fix this in the implementation below.\n\n inflight = self._inflight.get(key)\n if inflight is not None:\n inflight.cancelled = True\n if not inflight.task.done():\n inflight.task.cancel()\n\n def _remove_key(self, key: K) -> None:\n \"\"\"Remove a key from entries and LRU order, adjusting total weight.\"\"\"\n entry = self._entries.pop(key, None)\n if entry is not None:\n self._total_weight -= entry.weight\n if key in self._lru_order:\n self._lru_order.remove(key)\n\n def invalidate(self, key: K) -> None: # noqa: F811\n \"\"\"\n Invalidate a key. If a factory is in-flight for this key,\n mark it as cancelled so its result is not published.\n \"\"\"\n self._remove_key(key)\n inflight = self._inflight.get(key)\n if inflight is not None:\n inflight.cancelled = True\n if not inflight.task.done():\n inflight.task.cancel()\n\n def _evict_lru(self, needed_weight: int) -> None:\n \"\"\"Evict LRU entries until total_weight + needed_weight <= capacity.\"\"\"\n while self._total_weight + needed_weight > self._capacity:\n if not self._lru_order:\n break\n lru_key = self._lru_order[-1]\n entry = self._entries.get(lru_key)\n if entry is None:\n self._lru_order.pop()\n continue\n self._remove_key(lru_key)\n self._evictions += 1\n\n def _touch(self, key: K) -> None:\n \"\"\"Move key to MRU position.\"\"\"\n if key in self._lru_order:\n self._lru_order.remove(key)\n self._lru_order.insert(0, key)\n\n async def get(self, key: K) -> Optional[V]:\n \"\"\"\n Get a value from the cache. Returns None if not present or expired.\n Does not trigger factory creation.\n \"\"\"\n async with self._lock:\n entry = self._entries.get(key)\n if entry is None:\n self._misses += 1\n return None\n now = time.monotonic()\n if now >= entry.expires_at:\n # Expired\n self._remove_key(key)\n self._misses += 1\n return None\n # Hit\n self._hits += 1\n entry.last_access = now\n self._touch(key)\n return entry.value\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n weight: Optional[int] = None,\n ) -> V:\n \"\"\"\n Get a value from the cache, or create it using the async factory.\n Single-flight: concurrent calls for the same key share one factory invocation.\n Exceptions from the factory are never cached.\n \"\"\"\n effective_ttl = ttl if ttl is not None else self._default_ttl\n if effective_ttl <= 0:\n raise ValueError(\"ttl must be positive\")\n\n # First, check if we already have a valid entry\n async with self._lock:\n entry = self._entries.get(key)\n if entry is not None:\n now = time.monotonic()\n if now < entry.expires_at:\n self._hits += 1\n entry.last_access = now\n self._touch(key)\n return entry.value\n else:\n # Expired, remove it\n self._remove_key(key)\n self._misses += 1\n\n # Check if there's an in-flight factory\n inflight = self._inflight.get(key)\n if inflight is not None and not inflight.cancelled:\n # Wait for the existing factory\n pass # We'll wait outside the lock\n else:\n # No valid entry and no active in-flight (or it was cancelled)\n # We need to start a new factory\n self._misses += 1\n # Create the task outside the lock\n task = asyncio.ensure_future(factory())\n inflight = _Inflight(task)\n self._inflight[key] = inflight\n # Release lock, then await\n # We need to break out of the lock context\n # Let's restructure: use a flag\n break_out = True\n else:\n break_out = False\n\n # If we found an existing in-flight, wait for it\n if not break_out:\n assert inflight is not None\n try:\n value = await inflight.task\n except asyncio.CancelledError:\n # Our task was cancelled, but the shared factory may still be running\n # We should not propagate cancellation to the factory\n # Re-raise to indicate our caller was cancelled\n raise\n except Exception:\n # Factory failed; do not cache\n self._inflight.pop(key, None)\n raise\n else:\n # Factory succeeded\n self._inflight.pop(key, None)\n # Check if it was invalidated during creation\n if inflight.cancelled:\n # Do not publish\n return await self.get_or_create(\n key, factory, ttl=ttl, weight=weight\n )\n # Publish the result\n async with self._lock:\n # Re-check: maybe another task already published\n existing = self._entries.get(key)\n if existing is not None:\n now = time.monotonic()\n if now < existing.expires_at:\n self._hits += 1\n existing.last_access = now\n self._touch(key)\n return existing.value\n else:\n self._remove_key(key)\n\n # Check if invalidated\n if inflight.cancelled:\n # Do not publish\n pass\n else:\n # Insert into cache\n effective_weight = weight if weight is not None else self._weight_fn(value)\n if effective_weight <= 0:\n raise ValueError(\"weight must be positive\")\n if effective_weight > self._capacity:\n raise ValueError(\n f\"weight {effective_weight} exceeds capacity {self._capacity}\"\n )\n\n now = time.monotonic()\n new_entry = _Entry(\n value=value,\n expires_at=now + effective_ttl,\n weight=effective_weight,\n last_access=now,\n )\n self._evict_lru(effective_weight)\n self._entries[key] = new_entry\n self._lru_order.insert(0, key)\n self._total_weight += effective_weight\n return value\n # If invalidated, fall through to retry\n return await self.get_or_create(key, factory, ttl=ttl, weight=weight)\n\n # We need to start a new factory\n assert inflight is not None\n try:\n value = await inflight.task\n except asyncio.CancelledError:\n # Our task was cancelled\n # The shared factory may still be running\n # Do not cancel the shared factory\n raise\n except Exception:\n # Factory failed; do not cache\n async with self._lock:\n self._inflight.pop(key, None)\n raise\n else:\n # Factory succeeded\n async with self._lock:\n self._inflight.pop(key, None)\n # Check if it was invalidated during creation\n if inflight.cancelled:\n # Do not publish; retry\n return await self.get_or_create(key, factory, ttl=ttl, weight=weight)\n # Publish the result\n async with self._lock:\n # Re-check: maybe another task already published\n existing = self._entries.get(key)\n if existing is not None:\n now = time.monotonic()\n if now < existing.expires_at:\n self._hits += 1\n existing.last_access = now\n self._touch(key)\n return existing.value\n else:\n self._remove_key(key)\n\n # Check if invalidated\n if inflight.cancelled:\n pass\n else:\n # Insert into cache\n effective_weight = weight if weight is not None else self._weight_fn(value)\n if effective_weight <= 0:\n raise ValueError(\"weight must be positive\")\n if effective_weight > self._capacity:\n raise ValueError(\n f\"weight {effective_weight} exceeds capacity {self._capacity}\"\n )\n\n now = time.monotonic()\n new_entry = _Entry(\n value=value,\n expires_at=now + effective_ttl,\n weight=effective_weight,\n last_access=now,\n )\n self._evict_lru(effective_weight)\n self._entries[key] = new_entry\n self._lru_order.insert(0, key)\n self._total_weight += effective_weight\n return value\n # If invalidated, fall through to retry\n return await self.get_or_create(key, factory, ttl=ttl, weight=weight)\n\n async def set(\n self,\n key: K,\n value: V,\n *,\n ttl: Optional[float] = None,\n weight: Optional[int] = None,\n ) -> None:\n \"\"\"\n Set a value in the cache. Evicts LRU entries if needed", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:13:04.907109+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 15.930970875022467, + "total_s": 166.00547937501688, + "prompt_tokens": 182, + "output_tokens": 2840, + "decode_tps": 18.923933374068692, + "finish_reason": "stop", + "output_chars": 11403, + "output_sha256": "bb758808663b88bb3fbacc52df49598eb21aa8bc7fa0ee8466410f0ae13e8b0d", + "output": "where('idempotency_key', $idempotencyKey)\n ->first();\n\n if ($existingTransfer) {\n return [\n 'transfer_id' => $existingTransfer->id,\n 'idempotency_key' => $existingTransfer->idempotency_key,\n 'from_account_id' => $existingTransfer->from_account_id,\n 'to_account_id' => $existingTransfer->to_account_id,\n 'amount_cents' => $existingTransfer->amount_cents,\n 'status' => 'duplicate',\n ];\n }\n\n // Retry loop for deadlocks\n $maxRetries = 5;\n $attempt = 0;\n\n while (true) {\n $attempt++;\n try {\n $result = DB::transaction(function () use ($idempotencyKey, $fromAccountId, $toAccountId, $amountCents) {\n // Lock both account rows in deterministic ID order\n $minId = min($fromAccountId, $toAccountId);\n $maxId = max($fromAccountId, $toAccountId);\n\n $accounts = DB::table('accounts')\n ->whereIn('id', [$minId, $maxId])\n ->orderBy('id')\n ->lockForUpdate()\n ->get();\n\n if ($accounts->count() !== 2) {\n throw new RuntimeException('One or both accounts do not exist.');\n }\n\n $fromAccount = $accounts->firstWhere('id', $fromAccountId);\n $toAccount = $accounts->firstWhere('id', $toAccountId);\n\n // Reject insufficient funds\n if ($fromAccount->balance_cents < $amountCents) {\n throw new RuntimeException('Insufficient funds.');\n }\n\n // Perform transfer\n DB::table('accounts')\n ->where('id', $fromAccountId)\n ->decrement('balance_cents', $amountCents);\n\n DB::table('accounts')\n ->where('id', $toAccountId)\n ->increment('balance_cents', $amountCents);\n\n // Insert transfer\n $transferId = DB::table('transfers')->insertGetId([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'created_at' => now(),\n ]);\n\n // Persist outbox event in the same transaction\n DB::table('outbox_events')->insert([\n 'transfer_id' => $transferId,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transferId,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n ]),\n 'created_at' => now(),\n ]);\n\n return [\n 'transfer_id' => $transferId,\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'status' => 'created',\n ];\n });\n\n return $result;\n\n } catch (Illuminate\\Database\\QueryException $e) {\n // Detect deadlock (MySQL error 1213)\n if ($e->getCode() === 1213 || str_contains($e->getMessage(), 'Deadlock')) {\n if ($attempt >= $maxRetries) {\n throw new RuntimeException('Transfer failed after maximum retries due to deadlock.', 0, $e);\n }\n usleep(100000 * $attempt); // Exponential backoff\n continue;\n }\n throw $e;\n }\n }\n }\n}\n\nclass MoneyTransferServiceTest extends TestCase\n{\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Create test database schema\n Schema::dropIfExists('accounts');\n Schema::dropIfExists('transfers');\n Schema::dropIfExists('outbox_events');\n\n Schema::create('accounts', function (Blueprint $table) {\n $table->id();\n $table->unsignedBigInteger('balance_cents')->default(0);\n });\n\n Schema::create('transfers', function (Blueprint $table) {\n $table->id();\n $table->string('idempotency_key')->unique();\n $table->unsignedBigInteger('from_account_id');\n $table->unsignedBigInteger('to_account_id');\n $table->unsignedBigInteger('amount_cents');\n $table->timestamp('created_at');\n });\n\n Schema::create('outbox_events', function (Blueprint $table) {\n $table->id();\n $table->unsignedBigInteger('transfer_id');\n $table->string('event_type');\n $table->json('payload');\n $table->timestamp('created_at');\n });\n\n $this->service = new MoneyTransferService();\n }\n\n public function test_transfer_moves_funds_correctly(): void\n {\n // Setup: Account 1 has 10000 cents, Account 2 has 0 cents\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $result = $this->service->transfer('key-1', 1, 2, 5000);\n\n $this->assertSame('created', $result['status']);\n $this->assertSame(5000, $result['amount_cents']);\n\n $account1 = DB::table('accounts')->where('id', 1)->first();\n $account2 = DB::table('accounts')->where('id', 2)->first();\n\n $this->assertSame(5000, $account1->balance_cents);\n $this->assertSame(5000, $account2->balance_cents);\n\n $transferCount = DB::table('transfers')->count();\n $this->assertSame(1, $transferCount);\n\n $outboxCount = DB::table('outbox_events')->count();\n $this->assertSame(1, $outboxCount);\n }\n\n public function test_duplicate_request_returns_original_transfer(): void\n {\n // Setup: Account 1 has 10000 cents, Account 2 has 0 cents\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $firstResult = $this->service->transfer('key-dup', 1, 2, 3000);\n $this->assertSame('created', $firstResult['status']);\n\n $secondResult = $this->service->transfer('key-dup', 1, 2, 3000);\n $this->assertSame('duplicate', $secondResult['status']);\n $this->assertSame($firstResult['transfer_id'], $secondResult['transfer_id']);\n\n // Balances should only reflect one transfer\n $account1 = DB::table('accounts')->where('id', 1)->first();\n $account2 = DB::table('accounts')->where('id', 2)->first();\n\n $this->assertSame(7000, $account1->balance_cents);\n $this->assertSame(3000, $account2->balance_cents);\n\n $transferCount = DB::table('transfers')->count();\n $this->assertSame(1, $transferCount);\n\n $outboxCount = DB::table('outbox_events')->count();\n $this->assertSame(1, $outboxCount);\n }\n\n public function test_competing_requests_preserve_balances_and_create_one_transfer(): void\n {\n // Setup: Account 1 has 10000 cents, Account 2 has 0 cents\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n // Simulate competing requests with same idempotency key\n $results = [];\n for ($i = 0; $i < 5; $i++) {\n $results[] = $this->service->transfer('key-compete', 1, 2, 2000);\n }\n\n // All should return the same transfer ID\n $transferIds = array_unique(array_column($results, 'transfer_id'));\n $this->assertCount(1, $transferIds);\n\n // Only one transfer should be created\n $transferCount = DB::table('transfers')->count();\n $this->assertSame(1, $transferCount);\n\n // Only one outbox event\n $outboxCount = DB::table('outbox_events')->count();\n $this->assertSame(1, $outboxCount);\n\n // Balances should reflect only one transfer of 2000\n $account1 = DB::table('accounts')->where('id', 1)->first();\n $account2 = DB::table('accounts')->where('id', 2)->first();\n\n $this->assertSame(8000, $account1->balance_cents);\n $this->assertSame(2000, $account2->balance_cents);\n }\n\n public function test_rejects_insufficient_funds(): void\n {\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 1000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $this->expectException(RuntimeException::class);\n $this->expectExceptionMessage('Insufficient funds.');\n\n $this->service->transfer('key-insufficient', 1, 2, 5000);\n }\n\n public function test_rejects_non_positive_amount(): void\n {\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $this->expectException(InvalidArgumentException::class);\n $this->expectExceptionMessage('Amount must be positive.');\n\n $this->service->transfer('key-zero', 1, 2, 0);\n }\n\n public function test_rejects_same_account(): void\n {\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ]);\n\n $this->expectException(InvalidArgumentException::class);\n $this->expectExceptionMessage('Source and destination accounts must be distinct.');\n\n $this->service->transfer('key-same', 1, 1, 1000);\n }\n}", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:15:50.913910+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4814700839924626, + "total_s": 152.22010587499244, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 20.379645761550783, + "finish_reason": "length", + "output_chars": 10869, + "output_sha256": "f700d51c471a59acfcf85016d416f0c5354662552fea44b48f56d80c3afdcb59", + "output": "type TaskResult = { ok: true; value: T } | { ok: false; error: unknown };\n\ninterface Task {\n id: string;\n deps: readonly string[];\n run: (signal: AbortSignal) => Promise;\n}\n\ninterface SchedulerOptions {\n maxConcurrency?: number;\n signal?: AbortSignal;\n}\n\nclass SchedulerError extends Error {\n constructor(message: string) {\n super(message);\n this.name = \"SchedulerError\";\n }\n}\n\nclass CycleError extends SchedulerError {\n readonly path: readonly string[];\n constructor(path: readonly string[]) {\n super(`Dependency cycle detected: ${path.join(\" -> \")}`);\n this.name = \"CycleError\";\n this.path = path;\n }\n}\n\nclass DuplicateIdError extends SchedulerError {\n constructor(id: string) {\n super(`Duplicate task ID: ${id}`);\n this.name = \"DuplicateIdError\";\n }\n}\n\nclass UnknownDependencyError extends SchedulerError {\n constructor(taskId: string, depId: string) {\n super(`Task \"${taskId}\" depends on unknown task \"${depId}\"`);\n this.name = \"UnknownDependencyError\";\n }\n}\n\nclass CancellationError extends SchedulerError {\n constructor() {\n super(\"Scheduler aborted\");\n this.name = \"CancellationError\";\n }\n}\n\nclass TaskFailureError extends SchedulerError {\n readonly taskId: string;\n readonly cause: unknown;\n constructor(taskId: string, cause: unknown) {\n super(`Task \"${taskId}\" failed`);\n this.name = \"TaskFailureError\";\n this.taskId = taskId;\n this.cause = cause;\n }\n}\n\nfunction isAbortSignal(value: unknown): value is AbortSignal {\n return (\n typeof value === \"object\" &&\n value !== null &&\n \"aborted\" in value &&\n \"addEventListener\" in value &&\n \"removeEventListener\" in value\n );\n}\n\nfunction createCombinedSignal(external: AbortSignal | undefined): {\n signal: AbortSignal;\n cancel: () => void;\n} {\n const controller = new AbortController();\n const onExternalAbort = () => controller.abort(external?.reason);\n if (external) {\n if (external.aborted) {\n controller.abort(external.reason);\n } else {\n external.addEventListener(\"abort\", onExternalAbort, { once: true });\n }\n }\n const cancel = () => {\n if (external) {\n external.removeEventListener(\"abort\", onExternalAbort);\n }\n if (!controller.signal.aborted) {\n controller.abort(new CancellationError());\n }\n };\n return { signal: controller.signal, cancel };\n}\n\nfunction topologicalOrder(\n tasks: readonly Task[],\n idToTask: Map>\n): string[] {\n const inDegree = new Map();\n const dependents = new Map();\n\n for (const task of tasks) {\n inDegree.set(task.id, 0);\n dependents.set(task.id, []);\n }\n\n for (const task of tasks) {\n for (const dep of task.deps) {\n inDegree.set(task.id, (inDegree.get(task.id) ?? 0) + 1);\n const list = dependents.get(dep);\n if (list) {\n list.push(task.id);\n }\n }\n }\n\n const queue: string[] = [];\n for (const task of tasks) {\n if ((inDegree.get(task.id) ?? 0) === 0) {\n queue.push(task.id);\n }\n }\n\n const order: string[] = [];\n let head = 0;\n while (head < queue.length) {\n const id = queue[head++];\n order.push(id);\n const next = dependents.get(id) ?? [];\n for (const dependent of next) {\n const remaining = (inDegree.get(dependent) ?? 0) - 1;\n inDegree.set(dependent, remaining);\n if (remaining === 0) {\n queue.push(dependent);\n }\n }\n }\n\n if (order.length !== tasks.length) {\n const visited = new Set();\n const inStack = new Set();\n const path: string[] = [];\n let cyclePath: string[] | undefined;\n\n const dfs = (id: string): void => {\n if (cyclePath !== undefined) return;\n if (inStack.has(id)) {\n const start = path.indexOf(id);\n cyclePath = [...path.slice(start), id];\n return;\n }\n if (visited.has(id)) return;\n visited.add(id);\n inStack.add(id);\n path.push(id);\n const task = idToTask.get(id);\n if (task) {\n for (const dep of task.deps) {\n if (idToTask.has(dep)) {\n dfs(dep);\n if (cyclePath !== undefined) return;\n }\n }\n }\n path.pop();\n inStack.delete(id);\n };\n\n for (const task of tasks) {\n if (cyclePath !== undefined) break;\n dfs(task.id);\n }\n\n throw new CycleError(cyclePath ?? tasks.map((t) => t.id));\n }\n\n return order;\n}\n\nasync function schedule(\n tasks: readonly Task[],\n options: SchedulerOptions = {}\n): Promise> {\n const maxConcurrency = options.maxConcurrency ?? 1;\n if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {\n throw new SchedulerError(\"maxConcurrency must be a positive integer\");\n }\n\n const idToTask = new Map>();\n for (const task of tasks) {\n if (idToTask.has(task.id)) {\n throw new DuplicateIdError(task.id);\n }\n idToTask.set(task.id, task);\n }\n\n for (const task of tasks) {\n for (const dep of task.deps) {\n if (!idToTask.has(dep)) {\n throw new UnknownDependencyError(task.id, dep);\n }\n }\n }\n\n const order = topologicalOrder(tasks, idToTask);\n const { signal, cancel } = createCombinedSignal(options.signal);\n\n const results = new Map();\n const failed = new Set();\n const cancelled = new Set();\n const running = new Set();\n const promises = new Map>();\n const readyQueue: string[] = [];\n const readySet = new Set();\n const remainingDeps = new Map();\n\n for (const id of order) {\n const task = idToTask.get(id)!;\n remainingDeps.set(id, task.deps.length);\n }\n\n for (const id of order) {\n const task = idToTask.get(id)!;\n if (task.deps.length === 0) {\n readyQueue.push(id);\n readySet.add(id);\n }\n }\n\n let activeCount = 0;\n let firstError: unknown = undefined;\n let hasError = false;\n let isCancelled = false;\n let isDone = false;\n\n const onAbort = () => {\n if (isCancelled || isDone) return;\n isCancelled = true;\n for (const id of readyQueue.splice(0)) {\n cancelled.add(id);\n }\n for (const id of readySet) {\n if (!running.has(id) && !cancelled.has(id)) {\n cancelled.add(id);\n }\n }\n readySet.clear();\n };\n\n signal.addEventListener(\"abort\", onAbort, { once: true });\n\n const markFailed = (id: string, error: unknown): void => {\n if (failed.has(id)) return;\n failed.add(id);\n if (!hasError) {\n hasError = true;\n firstError = error;\n }\n const dependents = new Set();\n const collectDependents = (taskId: string): void => {\n if (dependents.has(taskId)) return;\n dependents.add(taskId);\n const task = idToTask.get(taskId);\n if (task) {\n for (const dep of task.deps) {\n collectDependents(dep);\n }\n }\n };\n const task = idToTask.get(id);\n if (task) {\n for (const dep of task.deps) {\n collectDependents(dep);\n }\n }\n for (const depId of dependents) {\n if (!failed.has(depId) && !cancelled.has(depId) && !running.has(depId)) {\n cancelled.add(depId);\n }\n }\n };\n\n const tryStartNext = (): void => {\n if (isDone || hasError || isCancelled) return;\n while (activeCount < maxConcurrency && readyQueue.length > 0) {\n const id = readyQueue.shift()!;\n if (readySet.has(id)) {\n readySet.delete(id);\n }\n if (failed.has(id) || cancelled.has(id) || running.has(id)) {\n continue;\n }\n const task = idToTask.get(id)!;\n running.add(id);\n activeCount += 1;\n\n const promise = (async (): Promise => {\n try {\n if (signal.aborted) {\n throw new CancellationError();\n }\n const value = await task.run(signal);\n results.set(id, value);\n } catch (error) {\n if (isCancelled) {\n cancelled.add(id);\n } else {\n markFailed(id, error);\n }\n } finally {\n running.delete(id);\n activeCount -= 1;\n const dependents = new Set();\n const collectDependents = (taskId: string): void => {\n if (dependents.has(taskId)) return;\n dependents.add(taskId);\n const t = idToTask.get(taskId);\n if (t) {\n for (const dep of t.deps) {\n collectDependents(dep);\n }\n }\n };\n for (const [otherId, otherTask] of idToTask) {\n if (otherTask.deps.includes(id)) {\n collectDependents(otherId);\n }\n }\n for (const depId of dependents) {\n if (failed.has(depId) || cancelled.has(depId) || running.has(depId)) {\n continue;\n }\n const remaining = (remainingDeps.get(depId) ?? 0) - 1;\n remainingDeps.set(depId, remaining);\n if (remaining === 0 && !readySet.has(depId) && !readyQueue.includes(depId)) {\n readyQueue.push(depId);\n readySet.add(depId);\n }\n }\n tryStartNext();\n }\n })();\n\n promises.set(id, promise);\n }\n };\n\n const allSettled = (async (): Promise => {\n try {\n tryStartNext();\n while (!isDone) {\n if (hasError || isCancelled) {\n isDone = true;\n break;\n }\n if (activeCount === 0 && readyQueue.length === 0) {\n isDone = true;\n break;\n }\n const pending = [...promises.values()];\n if (pending.length === 0) {\n isDone = true;\n break;\n }\n await Promise.race(pending);\n }\n } finally {\n signal.removeEventListener(\"abort\", onAbort);\n }\n })();\n\n const resultPromise = allSettled.then(\n () => {\n if (isCancelled) {\n throw new CancellationError();\n }\n if (hasError) {\n throw new TaskFailureError(\n [...failed][0] ?? \"unknown\",\n firstError\n );\n }\n return results;\n },\n (error) => {\n if (isCancelled) {\n throw new CancellationError();\n }\n throw error;\n }\n );\n\n resultPromise.catch(() => undefined);\n\n return resultPromise;\n}\n\nfunction assertEqual(actual: T, expected: T, message: string): void {\n if (actual !== expected) {\n throw new Error(`${message}: expected ${String(expected)}, got ${String(actual)}`);\n }\n}\n\nfunction assertDeepEqual(actual: T, expected: T, message: string): void {\n const a = JSON.stringify(actual);\n const e = JSON.stringify(expected);\n if (a !== e) {\n throw new Error(`${message}: expected ${e}, got ${a}`);\n }\n}\n\nfunction assertThrows(\n fn: () => unknown,\n errorClass: new (...args: never[]) => Error,\n message: string\n): void {\n try {\n fn();\n throw new Error(`${message}: expected throw, but did not throw`);\n }", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json new file mode 100644 index 0000000..136941d --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:10:51.466896+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.0361592500121333, + "total_s": 133.43742824997753, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 23.20219453486358, + "finish_reason": "length", + "output_chars": 14058, + "output_sha256": "a18e80f1852ecc0c70b987f244870b8a2dc20db8ad42505280162a4b17dbd057", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:13:04.907109+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 15.930970875022467, + "total_s": 166.00547937501688, + "prompt_tokens": 182, + "output_tokens": 2840, + "decode_tps": 18.923933374068692, + "finish_reason": "stop", + "output_chars": 11403, + "output_sha256": "bb758808663b88bb3fbacc52df49598eb21aa8bc7fa0ee8466410f0ae13e8b0d", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:15:50.913910+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4814700839924626, + "total_s": 152.22010587499244, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 20.379645761550783, + "finish_reason": "length", + "output_chars": 10869, + "output_sha256": "f700d51c471a59acfcf85016d416f0c5354662552fea44b48f56d80c3afdcb59", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json new file mode 100644 index 0000000..dd86822 --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T123006Z", + "created_at": "2026-08-31T12:30:06.184278+00:00", + "runtime": { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mxfp8" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:22:38.815059+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.44539537496166304, + "total_s": 0.44552537496201694, + "prompt_tokens": 41, + "output_tokens": 1, + "decode_tps": 8.241328039906357, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:22:39.260933+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1107976250350475, + "total_s": 149.8235780420364, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 20.63948064778098, + "finish_reason": "length", + "output_chars": 13492, + "output_sha256": "3e5ca7654fc69293040b7cc1b554ccddbd1a99b82710d361074fee260803182f", + "output": "import asyncio\nimport time\nfrom collections import OrderedDict\nfrom dataclasses import dataclass\nfrom typing import (\n Any,\n Awaitable,\n Callable,\n Generic,\n Optional,\n TypeVar,\n)\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n\n\nclass _Entry:\n __slots__ = (\"value\", \"expires_at\", \"weight\")\n\n def __init__(self, value: Any, expires_at: float, weight: int) -> None:\n self.value = value\n self.expires_at = expires_at\n self.weight = weight\n\n\nclass _Inflight:\n __slots__ = (\"event\", \"value\", \"error\", \"cancelled\")\n\n def __init__(self) -> None:\n self.event = asyncio.Event()\n self.value: Any = None\n self.error: Optional[BaseException] = None\n self.cancelled: bool = False\n\n\nclass AsyncTTLCache(Generic[K, V]):\n \"\"\"\n A generic async cache with:\n - Weighted LRU capacity (evict least-recently-used entries until total weight <= max_weight).\n - Per-entry TTL based on time.monotonic.\n - Single-flight get_or_create: concurrent misses for the same key invoke the async factory\n exactly once.\n - Cancellation of one waiter does not cancel the shared factory.\n - Exceptions are never cached.\n - Invalidating a key while its factory is running prevents stale publication.\n - The cache lock is never held while awaiting user code.\n - clear(), invalidate(), and immutable hit/miss/eviction statistics.\n \"\"\"\n\n def __init__(\n self,\n max_weight: int,\n default_ttl: float,\n weight_fn: Optional[Callable[[V], int]] = None,\n ) -> None:\n if max_weight <= 0:\n raise ValueError(\"max_weight must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n\n self._max_weight: int = max_weight\n self._default_ttl: float = default_ttl\n self._weight_fn: Callable[[V], int] = weight_fn or (lambda _v: 1)\n\n self._lock: asyncio.Lock = asyncio.Lock()\n self._store: OrderedDict[K, _Entry] = OrderedDict()\n self._current_weight: int = 0\n self._inflight: dict[K, _Inflight] = {}\n\n self._hits: int = 0\n self._misses: int = 0\n self._evictions: int = 0\n\n # ------------------------------------------------------------------\n # Public API\n # ------------------------------------------------------------------\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n ) -> V:\n \"\"\"\n Return the cached value for *key*, or invoke *factory* to create it.\n\n If multiple coroutines call get_or_create for the same key concurrently\n and the key is not yet cached, the factory is invoked exactly once.\n All waiters receive the same result (or the same exception).\n \"\"\"\n effective_ttl = ttl if ttl is not None else self._default_ttl\n\n # Fast path: check cache under lock\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None:\n if time.monotonic() < entry.expires_at:\n # Move to end (most recently used)\n self._store.move_to_end(key)\n self._hits += 1\n return entry.value\n else:\n # Expired: remove and fall through to factory\n self._store.pop(key, None)\n self._current_weight -= entry.weight\n\n # Check if there is already an in-flight factory for this key\n inflight = self._inflight.get(key)\n if inflight is not None:\n # Join the existing in-flight operation\n pass\n else:\n # Start a new in-flight operation\n inflight = _Inflight()\n self._inflight[key] = inflight\n self._misses += 1\n\n # If we joined an existing in-flight, wait for it\n if inflight.cancelled:\n # The in-flight was invalidated; we need to start our own\n # Re-acquire lock to check again\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None and time.monotonic() < entry.expires_at:\n self._store.move_to_end(key)\n self._hits += 1\n return entry.value\n # Check if another in-flight started\n inflight = self._inflight.get(key)\n if inflight is not None and not inflight.cancelled:\n pass # fall through to wait\n else:\n # Start a new in-flight\n inflight = _Inflight()\n self._inflight[key] = inflight\n self._misses += 1\n\n # Wait for the in-flight to complete\n await inflight.event.wait()\n\n if inflight.error is not None:\n raise inflight.error\n return inflight.value\n\n async def _run_factory(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: float,\n inflight: _Inflight,\n ) -> None:\n \"\"\"\n Run the factory outside the lock. This is called by get_or_create\n after acquiring the lock and setting up the inflight entry.\n \"\"\"\n try:\n value = await factory()\n except BaseException as exc:\n # Mark the inflight as failed\n inflight.error = exc\n inflight.event.set()\n # Remove inflight entry\n async with self._lock:\n # Only remove if it's still the same inflight\n if self._inflight.get(key) is inflight:\n del self._inflight[key]\n return\n\n # Check if the key was invalidated while the factory was running\n async with self._lock:\n if self._inflight.get(key) is not inflight:\n # The inflight was replaced (e.g., by invalidate), so this result is stale\n return\n if inflight.cancelled:\n # The inflight was cancelled, so this result is stale\n return\n\n # Check if the key is already in the cache (another coroutine may have\n # published a value, though this shouldn't happen with single-flight)\n if key in self._store:\n # Already cached, don't overwrite\n del self._inflight[key]\n return\n\n # Publish the value\n weight = self._weight_fn(value)\n if weight > self._max_weight:\n # Single item exceeds max weight; don't cache it\n del self._inflight[key]\n inflight.value = value\n inflight.event.set()\n return\n\n # Evict if necessary\n self._evict_for_weight(weight)\n\n expires_at = time.monotonic() + ttl\n self._store[key] = _Entry(value, expires_at, weight)\n self._current_weight += weight\n\n # Mark the inflight as complete\n del self._inflight[key]\n inflight.value = value\n inflight.event.set()\n\n async def invalidate(self, key: K) -> None:\n \"\"\"\n Invalidate a key. If a factory is currently running for this key,\n the result will not be published.\n \"\"\"\n async with self._lock:\n # Remove from cache if present\n entry = self._store.pop(key, None)\n if entry is not None:\n self._current_weight -= entry.weight\n\n # Cancel any in-flight factory for this key\n inflight = self._inflight.get(key)\n if inflight is not None:\n inflight.cancelled = True\n inflight.event.set()\n del self._inflight[key]\n\n async def clear(self) -> None:\n \"\"\"\n Clear all entries and cancel all in-flight factories.\n \"\"\"\n async with self._lock:\n self._store.clear()\n self._current_weight = 0\n # Cancel all in-flight factories\n for inflight in self._inflight.values():\n inflight.cancelled = True\n inflight.event.set()\n self._inflight.clear()\n\n @property\n def stats(self) -> CacheStats:\n \"\"\"\n Return an immutable snapshot of cache statistics.\n \"\"\"\n return CacheStats(\n hits=self._hits,\n misses=self._misses,\n evictions=self._evictions,\n )\n\n # ------------------------------------------------------------------\n # Internal helpers\n # ------------------------------------------------------------------\n\n def _evict_for_weight(self, incoming_weight: int) -> None:\n \"\"\"\n Evict least-recently-used entries until there is room for incoming_weight.\n This is called while holding the lock.\n \"\"\"\n while self._current_weight + incoming_weight > self._max_weight:\n if not self._store:\n break\n # Pop the least-recently-used entry (first in OrderedDict)\n key, entry = self._store.popitem(last=False)\n self._current_weight -= entry.weight\n self._evictions += 1\n\n # ------------------------------------------------------------------\n # Override get_or_create to properly handle the factory execution\n # ------------------------------------------------------------------\n\n async def get_or_create( # type: ignore[override]\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n ) -> V:\n \"\"\"\n Return the cached value for *key*, or invoke *factory* to create it.\n\n If multiple coroutines call get_or_create for the same key concurrently\n and the key is not yet cached, the factory is invoked exactly once.\n All waiters receive the same result (or the same exception).\n \"\"\"\n effective_ttl = ttl if ttl is not None else self._default_ttl\n\n # Fast path: check cache under lock\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None:\n if time.monotonic() < entry.expires_at:\n # Move to end (most recently used)\n self._store.move_to_end(key)\n self._hits += 1\n return entry.value\n else:\n # Expired: remove and fall through to factory\n self._store.pop(key, None)\n self._current_weight -= entry.weight\n\n # Check if there is already an in-flight factory for this key\n inflight = self._inflight.get(key)\n if inflight is not None and not inflight.cancelled:\n # Join the existing in-flight operation\n pass\n else:\n # Start a new in-flight operation\n inflight = _Inflight()\n self._inflight[key] = inflight\n self._misses += 1\n # We need to run the factory\n should_run = True\n else:\n should_run = False\n\n if should_run:\n # Run the factory outside the lock\n await self._run_factory(key, factory, effective_ttl, inflight)\n\n # Wait for the in-flight to complete\n await inflight.event.wait()\n\n if inflight.error is not None:\n raise inflight.error\n return inflight.value\n\n\n# ------------------------------------------------------------------\n# Tests\n# ------------------------------------------------------------------\n\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_basic_get_or_create() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=10.0)\n call_count = 0\n\n async def factory() -> str:\n nonlocal call_count\n call_count += 1\n return \"value\"\n\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert call_count == 1\n\n # Second call should hit the cache\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert call_count == 1\n assert cache.stats.hits == 1\n assert cache.stats.misses == 1\n\n\n@pytest.mark.asyncio\nasync def test_single_flight() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=10.0)\n call_count = 0\n\n async def factory() -> str:\n nonlocal call_count\n call_count += 1\n await asyncio.sleep(0.05)\n return \"value\"\n\n # Launch multiple concurrent get_or_create for the same key\n results = await asyncio.gather(\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n )\n\n assert all(r == \"value\" for r in results)\n assert call_count == 1\n assert cache.stats.misses == 1\n\n\n@pytest.mark.asyncio\nasync def test_cancellation_does_not_cancel_factory() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=10.0)\n call_count = 0\n factory_completed = asyncio.Event()\n\n async def factory() -> str:\n nonlocal call_count\n call_count += 1\n await asyncio.sleep(0.05)\n factory_completed.set()\n return \"value\"\n\n # Start a get", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:25:09.083177+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6688512500259094, + "total_s": 149.76781020802446, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 20.71737613932144, + "finish_reason": "length", + "output_chars": 12642, + "output_sha256": "71ee449c3f573828250542e04f7a63db5a8d23087bdec89f0a9a0bb31d52a3b5", + "output": " 0)\n * @param string $idempotencyKey Unique client-supplied key for idempotency\n *\n * @return array{\n * transfer_id: int,\n * from_account_id: int,\n * to_account_id: int,\n * amount_cents: int,\n * idempotency_key: string,\n * created_at: string,\n * was_duplicate: bool\n * }\n *\n * @throws \\InvalidArgumentException If amount is not positive or accounts are the same\n * @throws \\RuntimeException If insufficient funds or after exhausting deadlock retries\n */\n public function transfer(\n int $fromAccountId,\n int $toAccountId,\n int $amountCents,\n string $idempotencyKey\n ): array {\n // Validate positive amount\n if ($amountCents <= 0) {\n throw new \\InvalidArgumentException('Transfer amount must be a positive integer number of cents.');\n }\n\n // Validate distinct accounts\n if ($fromAccountId === $toAccountId) {\n throw new \\InvalidArgumentException('Source and destination accounts must be distinct.');\n }\n\n // Validate idempotency key is non-empty\n if (trim($idempotencyKey) === '') {\n throw new \\InvalidArgumentException('Idempotency key must be a non-empty string.');\n }\n\n $lastException = null;\n\n for ($attempt = 0; $attempt < self::MAX_DEADLOCK_RETRIES; $attempt++) {\n try {\n return $this->executeTransfer(\n $fromAccountId,\n $toAccountId,\n $amountCents,\n $idempotencyKey\n );\n } catch (\\Throwable $e) {\n // Detect deadlock (MySQL error 1213, PostgreSQL 40P01, etc.)\n if ($this->isDeadlockException($e)) {\n $lastException = $e;\n // Exponential backoff with jitter\n $delayMs = self::DEADLOCK_RETRY_BASE_DELAY_MS * (2 ** $attempt) + random_int(0, 20);\n usleep($delayMs * 1000);\n continue;\n }\n // Non-deadlock exception: rethrow immediately\n throw $e;\n }\n }\n\n // All retries exhausted due to deadlocks\n throw new \\RuntimeException(\n 'Transfer failed after ' . self::MAX_DEADLOCK_RETRIES . ' deadlock retries.',\n 0,\n $lastException\n );\n }\n\n /**\n * Execute the transfer inside a database transaction.\n *\n * @return array{\n * transfer_id: int,\n * from_account_id: int,\n * to_account_id: int,\n * amount_cents: int,\n * idempotency_key: string,\n * created_at: string,\n * was_duplicate: bool\n * }\n */\n private function executeTransfer(\n int $fromAccountId,\n int $toAccountId,\n int $amountCents,\n string $idempotencyKey\n ): array {\n return DB::transaction(function () use ($fromAccountId, $toAccountId, $amountCents, $idempotencyKey) {\n // Check for existing transfer with this idempotency key (idempotency check)\n $existingTransfer = DB::table('transfers')\n ->where('idempotency_key', $idempotencyKey)\n ->first();\n\n if ($existingTransfer !== null) {\n // Return the original transfer result; no state changes\n return [\n 'transfer_id' => (int) $existingTransfer->id,\n 'from_account_id' => (int) $existingTransfer->from_account_id,\n 'to_account_id' => (int) $existingTransfer->to_account_id,\n 'amount_cents' => (int) $existingTransfer->amount_cents,\n 'idempotency_key' => $existingTransfer->idempotency_key,\n 'created_at' => $existingTransfer->created_at,\n 'was_duplicate' => true,\n ];\n }\n\n // Lock both account rows in deterministic ID order to prevent deadlocks.\n // We always lock the lower ID first, then the higher ID.\n $lowerId = min($fromAccountId, $toAccountId);\n $higherId = max($fromAccountId, $toAccountId);\n\n // Lock the lower-ID account row\n $lowerAccount = DB::table('accounts')\n ->where('id', $lowerId)\n ->lockForUpdate()\n ->first();\n\n if ($lowerAccount === null) {\n throw new \\RuntimeException(\"Account with ID {$lowerId} not found.\");\n }\n\n // Lock the higher-ID account row\n $higherAccount = DB::table('accounts')\n ->where('id', $higherId)\n ->lockForUpdate()\n ->first();\n\n if ($higherAccount === null) {\n throw new \\RuntimeException(\"Account with ID {$higherId} not found.\");\n }\n\n // Identify which locked row is the source and which is the destination\n if ($fromAccountId === $lowerId) {\n $fromAccount = $lowerAccount;\n $toAccount = $higherAccount;\n } else {\n $fromAccount = $higherAccount;\n $toAccount = $lowerAccount;\n }\n\n // Check sufficient funds\n if ($fromAccount->balance_cents < $amountCents) {\n throw new \\RuntimeException(\n \"Insufficient funds: account {$fromAccountId} has \" .\n $fromAccount->balance_cents . \" cents, but {$amountCents} cents are required.\"\n );\n }\n\n // Perform the transfer: debit source, credit destination\n DB::table('accounts')\n ->where('id', $fromAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents - ' . $amountCents),\n ]);\n\n DB::table('accounts')\n ->where('id', $toAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents + ' . $amountCents),\n ]);\n\n // Insert the transfer record\n $now = date('Y-m-d H:i:s');\n $transferId = DB::table('transfers')->insertGetId([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'created_at' => $now,\n ]);\n\n // Persist an outbox event in the same transaction\n DB::table('outbox_events')->insert([\n 'transfer_id' => $transferId,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transferId,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'idempotency_key' => $idempotencyKey,\n ]),\n 'created_at' => $now,\n ]);\n\n return [\n 'transfer_id' => (int) $transferId,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'idempotency_key' => $idempotencyKey,\n 'created_at' => $now,\n 'was_duplicate' => false,\n ];\n });\n }\n\n /**\n * Determine whether a Throwable represents a database deadlock.\n *\n * @param \\Throwable $e\n * @return bool\n */\n private function isDeadlockException(\\Throwable $e): bool\n {\n // MySQL/InnoDB deadlock: SQLSTATE 40001, error code 1213\n // PostgreSQL deadlock: SQLSTATE 40P01\n // We check the message and any nested exception for common deadlock indicators.\n\n $message = $e->getMessage();\n\n // Check for common deadlock error codes/messages\n if (\n str_contains($message, '1213') ||\n str_contains($message, 'Deadlock found') ||\n str_contains($message, 'deadlock') ||\n str_contains($message, '40P01') ||\n str_contains($message, '40001')\n ) {\n return true;\n }\n\n // Check if it's a PDOException with a deadlock SQLSTATE\n if ($e instanceof \\PDOException) {\n $sqlState = $e->getCode();\n if ($sqlState === '40001' || $sqlState === '40P01') {\n return true;\n }\n }\n\n // Check previous exception in the chain\n if ($e->getPrevious() !== null) {\n return $this->isDeadlockException($e->getPrevious());\n }\n\n return false;\n }\n}\n\n/**\n * PHPUnit test suite for MoneyTransferService.\n *\n * These tests use an in-memory SQLite database to verify:\n * 1. Duplicate requests (same idempotency key) return the original transfer without\n * duplicating effects.\n * 2. Competing requests (concurrent transfers with different keys) preserve balances\n * and create exactly one transfer and one outbox event per successful transfer.\n * 3. Insufficient funds are rejected.\n * 4. Invalid inputs (zero/negative amount, same account) are rejected.\n */\nclass MoneyTransferServiceTest extends TestCase\n{\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Configure an in-memory SQLite database for testing\n $this->configureInMemoryDatabase();\n\n $this->service = new MoneyTransferService();\n }\n\n /**\n * Configure Laravel's DB facade to use an in-memory SQLite database\n * and create the required tables.\n */\n private function configureInMemoryDatabase(): void\n {\n // We need to bootstrap a minimal Laravel container for the DB facade.\n // In a real Laravel 12 app, this would be handled by the framework.\n // Here we simulate it using the Illuminate\\Database Capsule.\n\n $capsule = new \\Illuminate\\Database\\Capsule\\Manager();\n $capsule->addConnection([\n 'driver' => 'sqlite',\n 'database' => ':memory:',\n 'prefix' => '',\n ]);\n $capsule->setAsGlobal();\n $capsule->bootEloquent();\n\n // Create tables\n $schema = $capsule->schema();\n\n $schema->create('accounts', function ($table) {\n $table->bigIncrements('id');\n $table->big", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:27:38.904048+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.6032058749697171, + "total_s": 147.27845333400182, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 21.060449812696582, + "finish_reason": "length", + "output_chars": 11368, + "output_sha256": "a3d74aa8a8183babda562bc53b32dda3b1b3ad318788d2384cc8babff3df3223", + "output": "// dag-scheduler.ts\n// Strict-mode, dependency-free DAG task scheduler.\n\n// ---------------------------------------------------------------------------\n// Types\n// ---------------------------------------------------------------------------\n\nexport interface Task {\n readonly id: string;\n readonly deps: readonly string[];\n readonly run: (signal: AbortSignal) => Promise;\n}\n\nexport interface ScheduleOptions {\n readonly maxConcurrency?: number;\n readonly signal?: AbortSignal;\n}\n\nexport type ScheduleResult = Map;\n\n// ---------------------------------------------------------------------------\n// Errors\n// ---------------------------------------------------------------------------\n\nexport class CycleError extends Error {\n readonly path: readonly string[];\n constructor(path: readonly string[]) {\n super(`Cycle detected: ${path.join(\" -> \")}`);\n this.name = \"CycleError\";\n this.path = path;\n }\n}\n\nexport class UnknownDependencyError extends Error {\n readonly taskId: string;\n readonly depId: string;\n constructor(taskId: string, depId: string) {\n super(`Task \"${taskId}\" has unknown dependency \"${depId}\"`);\n this.name = \"UnknownDependencyError\";\n this.taskId = taskId;\n this.depId = depId;\n }\n}\n\nexport class DuplicateIdError extends Error {\n readonly id: string;\n constructor(id: string) {\n super(`Duplicate task id: \"${id}\"`);\n this.name = \"DuplicateIdError\";\n this.id = id;\n }\n}\n\n// ---------------------------------------------------------------------------\n// Internal helpers\n// ---------------------------------------------------------------------------\n\n/**\n * Detect a cycle in a directed graph using iterative DFS.\n * Returns the cycle path (list of node ids) or null if acyclic.\n */\nfunction detectCycle(\n nodes: readonly string[],\n adj: Map\n): string[] | null {\n const WHITE = 0;\n const GRAY = 1;\n const BLACK = 2;\n\n const color = new Map();\n for (const n of nodes) color.set(n, WHITE);\n\n // parent map for path reconstruction\n const parent = new Map();\n\n for (const start of nodes) {\n if (color.get(start) !== WHITE) continue;\n\n // Iterative DFS with explicit stack of [node, nextChildIndex]\n const stack: Array<{ node: string; idx: number }> = [{ node: start, idx: 0 }];\n parent.set(start, null);\n\n while (stack.length > 0) {\n const frame = stack[stack.length - 1];\n const node = frame.node;\n const neighbors = adj.get(node) ?? [];\n\n if (frame.idx < neighbors.length) {\n const next = neighbors[frame.idx];\n frame.idx++;\n\n const c = color.get(next);\n if (c === GRAY) {\n // Found a cycle: reconstruct path from `next` back to `node`\n const path: string[] = [next, node];\n let cur: string | null = node;\n while (cur !== null && cur !== next) {\n const p = parent.get(cur);\n if (p === undefined || p === null) break;\n path.push(p);\n cur = p;\n }\n path.reverse();\n return path;\n }\n if (c === WHITE) {\n color.set(next, GRAY);\n parent.set(next, node);\n stack.push({ node: next, idx: 0 });\n }\n } else {\n color.set(node, BLACK);\n stack.pop();\n }\n }\n }\n\n return null;\n}\n\n/**\n * Compute a stable topological order using Kahn's algorithm.\n * Ties are broken by original input order (stable).\n */\nfunction topologicalOrder(\n tasks: readonly Task[]\n): string[] {\n const idToIndex = new Map();\n for (let i = 0; i < tasks.length; i++) {\n idToIndex.set(tasks[i].id, i);\n }\n\n const inDegree = new Map();\n const adj = new Map();\n\n for (const t of tasks) {\n inDegree.set(t.id, 0);\n adj.set(t.id, []);\n }\n\n for (const t of tasks) {\n for (const dep of t.deps) {\n // dep -> t edge\n adj.get(dep)!.push(t.id);\n inDegree.set(t.id, (inDegree.get(t.id) ?? 0) + 1);\n }\n }\n\n // Use a min-heap-like approach: pick the ready node with the smallest\n // original index to ensure stable input-order scheduling.\n const ready: string[] = [];\n for (const t of tasks) {\n if ((inDegree.get(t.id) ?? 0) === 0) {\n ready.push(t.id);\n }\n }\n // Sort ready by original index for stability\n ready.sort((a, b) => (idToIndex.get(a)! - idToIndex.get(b)!));\n\n const order: string[] = [];\n while (ready.length > 0) {\n // Pick the first (smallest original index)\n const node = ready.shift()!;\n order.push(node);\n\n const children = adj.get(node) ?? [];\n for (const child of children) {\n const deg = (inDegree.get(child) ?? 0) - 1;\n inDegree.set(child, deg);\n if (deg === 0) {\n // Insert into ready maintaining sorted order by original index\n const idx = idToIndex.get(child)!;\n // Binary search for insertion point\n let lo = 0;\n let hi = ready.length;\n while (lo < hi) {\n const mid = (lo + hi) >> 1;\n if (idToIndex.get(ready[mid])! < idx) {\n lo = mid + 1;\n } else {\n hi = mid;\n }\n }\n ready.splice(lo, 0, child);\n }\n }\n }\n\n return order;\n}\n\n// ---------------------------------------------------------------------------\n// Scheduler\n// ---------------------------------------------------------------------------\n\nexport async function schedule(\n tasks: readonly Task[],\n options: ScheduleOptions = {}\n): Promise> {\n const maxConcurrency = options.maxConcurrency ?? Infinity;\n const externalSignal = options.signal;\n\n // 1. Validate: duplicate IDs\n const seen = new Set();\n for (const t of tasks) {\n if (seen.has(t.id)) {\n throw new DuplicateIdError(t.id);\n }\n seen.add(t.id);\n }\n\n // 2. Validate: unknown dependencies\n for (const t of tasks) {\n for (const dep of t.deps) {\n if (!seen.has(dep)) {\n throw new UnknownDependencyError(t.id, dep);\n }\n }\n }\n\n // 3. Build adjacency and detect cycles\n const ids = tasks.map((t) => t.id);\n const adj = new Map();\n for (const t of tasks) {\n adj.set(t.id, t.deps);\n }\n\n const cycle = detectCycle(ids, adj);\n if (cycle !== null) {\n throw new CycleError(cycle);\n }\n\n // 4. Compute stable topological order\n const topo = topologicalOrder(tasks);\n\n // 5. Build a lookup from id to task\n const taskById = new Map>();\n for (const t of tasks) {\n taskById.set(t.id, t);\n }\n\n // 6. Compute in-degree for scheduling\n const inDegree = new Map();\n const dependents = new Map(); // dep -> list of task ids that depend on it\n\n for (const t of tasks) {\n inDegree.set(t.id, t.deps.length);\n for (const dep of t.deps) {\n if (!dependents.has(dep)) {\n dependents.set(dep, []);\n }\n dependents.get(dep)!.push(t.id);\n }\n }\n\n // 7. Scheduling state\n const results = new Map();\n const running = new Set>();\n const failed = new Set(); // tasks that failed or were skipped due to failure\n let firstError: unknown = null;\n let hasError = false;\n\n // AbortController for internal cancellation\n const internalAbort = new AbortController();\n const onExternalAbort = () => {\n internalAbort.abort();\n };\n if (externalSignal) {\n if (externalSignal.aborted) {\n internalAbort.abort();\n } else {\n externalSignal.addEventListener(\"abort\", onExternalAbort, { once: true });\n }\n }\n\n // Track which tasks have been \"started\" (their run() has been called)\n const started = new Set();\n\n // Helper: check if a task should be skipped because a dependency failed\n function shouldSkip(taskId: string): boolean {\n const t = taskById.get(taskId)!;\n for (const dep of t.deps) {\n if (failed.has(dep)) {\n return true;\n }\n }\n return false;\n }\n\n // Helper: mark a task and all its transitive dependents as failed/skipped\n function markFailedAndDependents(taskId: string): void {\n if (failed.has(taskId)) return;\n failed.add(taskId);\n const queue: string[] = [taskId];\n while (queue.length > 0) {\n const cur = queue.shift()!;\n const deps = dependents.get(cur) ?? [];\n for (const dep of deps) {\n if (!failed.has(dep)) {\n failed.add(dep);\n queue.push(dep);\n }\n }\n }\n }\n\n // Helper: try to start a task if all deps are done and it's not failed\n function tryStart(taskId: string): void {\n if (started.has(taskId) || failed.has(taskId)) return;\n const t = taskById.get(taskId)!;\n // Check all deps are completed (in results) and none failed\n for (const dep of t.deps) {\n if (!results.has(dep)) return; // dep not yet done\n if (failed.has(dep)) {\n markFailedAndDependents(taskId);\n return;\n }\n }\n // All deps satisfied and none failed\n started.add(taskId);\n\n const p = (async () => {\n try {\n const result = await t.run(internalAbort.signal);\n if (hasError) return; // already failed, don't record\n results.set(taskId, result);\n } catch (err) {\n if (!hasError) {\n hasError = true;\n firstError = err;\n markFailedAndDependents(taskId);\n // Abort all running tasks\n internalAbort.abort();\n }\n // Re-throw to let the outer catch handle it, but we don't want\n // unhandled rejection. We'll handle it in the running set.\n throw err;\n }\n })();\n\n // Attach a catch to prevent unhandled rejection\n p.catch(() => {\n // Already handled above; this is just to prevent unhandled rejection\n });\n\n running.add(p);\n p.finally(() => {\n running.delete(p);\n });\n }\n\n // 8. Initial pass: start all tasks with no dependencies\n for (const taskId of topo) {\n const t = taskById.get(taskId)!;\n if (t.deps.length === 0) {\n tryStart(taskId);\n }\n }\n\n // 9. Main loop: wait for tasks to complete, then start newly-ready tasks\n // We use a promise-based approach: wait for at least one running task to finish,\n // then check for newly ready tasks.\n\n // We need to handle the case where no tasks are running but there are still\n // tasks to start (shouldn't happen in a valid DAG, but be safe).\n\n // Use a \"wake\" mechanism: when a task completes, we check for newly ready tasks.\n // We'll use a simple approach: poll the running set.\n\n // Actually, a cleaner approach: use a \"completion\" event.\n // Let's restructure: we'll use a queue of \"ready\" tasks and a running set.\n\n // Reset and use a cleaner scheduling loop\n // Clear previous state\n running.clear();\n started.clear();\n results.clear();\n failed.clear();\n hasError = false;\n firstError = null;\n\n // Ready queue: tasks whose deps are all satisfied\n const readyQueue: string[] = [];\n for (const taskId of topo) {\n const t = taskById.get(taskId)!;\n if (t.deps.length === 0) {\n readyQueue.push(taskId);\n }\n }\n\n // Process ready queue: start tasks up to maxConcurrency\n function processReady(): void {\n while (readyQueue.length > 0 && running.size < maxConcurrency && !hasError) {\n const taskId = readyQueue.shift()!;\n if (failed.has(taskId)) continue;\n if (started.has", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json new file mode 100644 index 0000000..5fff934 --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:22:39.260933+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1107976250350475, + "total_s": 149.8235780420364, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 20.63948064778098, + "finish_reason": "length", + "output_chars": 13492, + "output_sha256": "3e5ca7654fc69293040b7cc1b554ccddbd1a99b82710d361074fee260803182f", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:25:09.083177+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6688512500259094, + "total_s": 149.76781020802446, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 20.71737613932144, + "finish_reason": "length", + "output_chars": 12642, + "output_sha256": "71ee449c3f573828250542e04f7a63db5a8d23087bdec89f0a9a0bb31d52a3b5", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:27:38.904048+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.6032058749697171, + "total_s": 147.27845333400182, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 21.060449812696582, + "finish_reason": "length", + "output_chars": 11368, + "output_sha256": "a3d74aa8a8183babda562bc53b32dda3b1b3ad318788d2384cc8babff3df3223", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json new file mode 100644 index 0000000..2baad10 --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T133452Z", + "created_at": "2026-08-31T13:34:52.367128+00:00", + "runtime": { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-mxfp8" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:22:56.588802+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.38669666700297967, + "total_s": 0.44958637503441423, + "prompt_tokens": 41, + "output_tokens": 2, + "decode_tps": 31.801705916655347, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:22:57.038624+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1419034169521183, + "total_s": 247.70563520898577, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 12.459253344652916, + "finish_reason": "length", + "output_chars": 13028, + "output_sha256": "699045091c4f5bc19d0f8f0bbc676b164512c8e3e7dec0f95acb13b5bef2f4ba", + "output": "from __future__ import annotations\n\nimport asyncio\nimport time\nfrom collections import OrderedDict\nfrom dataclasses import dataclass, field\nfrom typing import Any, Awaitable, Callable, Generic, TypeVar\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n\n\n@dataclass\nclass _Entry:\n value: Any\n expires_at: float\n weight: int\n key: Any\n\n\nclass AsyncTTLCache(Generic[K, V]):\n \"\"\"\n A generic async TTL cache with weighted LRU capacity, per-entry TTL,\n single-flight get_or_create, and safe cancellation semantics.\n \"\"\"\n\n def __init__(\n self,\n max_weight: int,\n default_ttl: float,\n weight_fn: Callable[[K, V], int] | None = None,\n ) -> None:\n if max_weight <= 0:\n raise ValueError(\"max_weight must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n\n self._max_weight: int = max_weight\n self._default_ttl: float = default_ttl\n self._weight_fn: Callable[[K, V], int] | None = weight_fn\n\n self._lock = asyncio.Lock()\n self._entries: OrderedDict[K, _Entry] = OrderedDict()\n self._current_weight: int = 0\n\n # Single-flight: key -> (future, task)\n self._inflight: dict[K, tuple[asyncio.Future[V], asyncio.Task[V]]] = {}\n\n # Statistics\n self._hits: int = 0\n self._misses: int = 0\n self._evictions: int = 0\n\n def _compute_weight(self, key: K, value: V) -> int:\n if self._weight_fn is not None:\n return self._weight_fn(key, value)\n return 1\n\n def _evict_if_needed(self, required_weight: int) -> None:\n \"\"\"Evict LRU entries until we have room for required_weight.\"\"\"\n while self._current_weight + required_weight > self._max_weight and self._entries:\n # Pop the least recently used (first item in OrderedDict)\n key, entry = self._entries.popitem(last=False)\n self._current_weight -= entry.weight\n self._evictions += 1\n\n async def get(self, key: K) -> V | None:\n \"\"\"\n Retrieve a value from the cache. Returns None if not present or expired.\n Does not trigger factory creation.\n \"\"\"\n async with self._lock:\n entry = self._entries.get(key)\n if entry is None:\n self._misses += 1\n return None\n\n # Check TTL\n if time.monotonic() >= entry.expires_at:\n # Expired: remove and count as miss\n del self._entries[key]\n self._current_weight -= entry.weight\n self._misses += 1\n return None\n\n # Hit: move to most recently used\n self._entries.move_to_end(key)\n self._hits += 1\n return entry.value\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: float | None = None,\n ) -> V:\n \"\"\"\n Get a value from cache, or create it using the async factory if missing/expired.\n Single-flight: concurrent calls for the same key share one factory invocation.\n \"\"\"\n # First, try to get from cache\n value = await self.get(key)\n if value is not None:\n return value\n\n # Check if there's already an in-flight creation for this key\n async with self._lock:\n if key in self._inflight:\n future, _ = self._inflight[key]\n else:\n # Start a new single-flight creation\n future: asyncio.Future[V] = asyncio.get_event_loop().create_future()\n task = asyncio.ensure_future(self._run_factory(key, factory, ttl, future))\n self._inflight[key] = (future, task)\n\n # Wait for the result. If this task is cancelled, we must NOT cancel the shared factory.\n try:\n return await future\n except asyncio.CancelledError:\n # Our await was cancelled, but the factory task continues.\n # We should re-raise to propagate cancellation to the caller.\n raise\n\n async def _run_factory(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: float | None,\n future: asyncio.Future[V],\n ) -> None:\n \"\"\"\n Run the factory, publish result to future, and update cache.\n Never holds the cache lock while awaiting user code.\n \"\"\"\n try:\n value = await factory()\n except Exception as exc:\n # Exceptions are never cached. Set exception on future.\n if not future.done():\n future.set_exception(exc)\n # Clean up inflight entry\n async with self._lock:\n self._inflight.pop(key, None)\n return\n\n # Check if the key was invalidated while factory was running.\n # If invalidated, we must NOT publish the stale value.\n async with self._lock:\n # Check if key is still in inflight (i.e., not invalidated)\n if key not in self._inflight:\n # Key was invalidated during creation; do not publish\n if not future.done():\n future.set_exception(\n RuntimeError(f\"Key {key!r} was invalidated during creation\")\n )\n return\n\n # Remove from inflight\n del self._inflight[key]\n\n # Check if key already exists in cache (e.g., another path added it)\n if key in self._entries:\n # Already present; don't overwrite\n if not future.done():\n future.set_result(self._entries[key].value)\n return\n\n # Compute weight and evict if needed\n weight = self._compute_weight(key, value)\n self._evict_if_needed(weight)\n\n # Insert new entry\n effective_ttl = ttl if ttl is not None else self._default_ttl\n expires_at = time.monotonic() + effective_ttl\n self._entries[key] = _Entry(\n value=value,\n expires_at=expires_at,\n weight=weight,\n key=key,\n )\n self._current_weight += weight\n\n # Set result on future\n if not future.done():\n future.set_result(value)\n\n async def invalidate(self, key: K) -> bool:\n \"\"\"\n Invalidate a key. Returns True if the key was present (in cache or inflight).\n If inflight, prevents stale publication.\n \"\"\"\n async with self._lock:\n found = False\n\n # Remove from cache if present\n if key in self._entries:\n entry = self._entries.pop(key)\n self._current_weight -= entry.weight\n found = True\n\n # Remove from inflight if present\n if key in self._inflight:\n future, task = self._inflight.pop(key)\n # Cancel the task if it's still running\n if not task.done():\n task.cancel()\n # Set exception on future to unblock waiters\n if not future.done():\n future.set_exception(\n RuntimeError(f\"Key {key!r} was invalidated\")\n )\n found = True\n\n return found\n\n async def clear(self) -> None:\n \"\"\"Clear all entries and cancel all in-flight factories.\"\"\"\n async with self._lock:\n # Cancel all in-flight tasks\n for key, (future, task) in list(self._inflight.items()):\n if not task.done():\n task.cancel()\n if not future.done():\n future.set_exception(\n RuntimeError(\"Cache cleared\")\n )\n self._inflight.clear()\n\n # Clear all entries\n self._entries.clear()\n self._current_weight = 0\n\n @property\n def stats(self) -> CacheStats:\n \"\"\"Return immutable snapshot of cache statistics.\"\"\"\n return CacheStats(\n hits=self._hits,\n misses=self._misses,\n evictions=self._evictions,\n )\n\n @property\n def size(self) -> int:\n \"\"\"Return the number of entries currently in the cache.\"\"\"\n return len(self._entries)\n\n @property\n def current_weight(self) -> int:\n \"\"\"Return the current total weight of cached entries.\"\"\"\n return self._current_weight\n\n\n# ---------------------------------------------------------------------------\n# Pytest-asyncio tests\n# ---------------------------------------------------------------------------\n\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_basic_get_and_set() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n async def factory() -> int:\n return 42\n\n result = await cache.get_or_create(\"key\", factory)\n assert result == 42\n\n # Second call should hit cache\n result2 = await cache.get_or_create(\"key\", factory)\n assert result2 == 42\n\n stats = cache.stats\n assert stats.hits == 1\n assert stats.misses == 1\n\n\n@pytest.mark.asyncio\nasync def test_ttl_expiry() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=0.05)\n\n call_count = 0\n\n async def factory() -> int:\n nonlocal call_count\n call_count += 1\n return call_count\n\n result1 = await cache.get_or_create(\"key\", factory)\n assert result1 == 1\n\n # Wait for TTL to expire\n await asyncio.sleep(0.1)\n\n result2 = await cache.get_or_create(\"key\", factory)\n assert result2 == 2\n assert call_count == 2\n\n\n@pytest.mark.asyncio\nasync def test_single_flight_concurrent_misses() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n call_count = 0\n\n async def factory() -> int:\n nonlocal call_count\n call_count += 1\n await asyncio.sleep(0.05)\n return call_count\n\n # Launch multiple concurrent get_or_create calls for the same key\n results = await asyncio.gather(\n cache.get_or_create(\"key\", factory),\n cache.get_or_create(\"key\", factory),\n cache.get_or_create(\"key\", factory),\n )\n\n # All should get the same value\n assert all(r == results[0] for r in results)\n # Factory should have been called exactly once\n assert call_count == 1\n\n\n@pytest.mark.asyncio\nasync def test_cancellation_does_not_cancel_shared_factory() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n factory_started = asyncio.Event()\n factory_done = asyncio.Event()\n result_value: list[int] = []\n\n async def factory() -> int:\n factory_started.set()\n await asyncio.sleep(0.1)\n val = 99\n result_value.append(val)\n factory_done.set()\n return val\n\n # Start a get_or_create\n task1 = asyncio.ensure_future(cache.get_or_create(\"key\", factory))\n\n # Wait for factory to start\n await factory_started.wait()\n\n # Cancel the first waiter\n task1.cancel()\n\n try:\n await task1\n except asyncio.CancelledError:\n pass\n\n # The factory should still complete\n await factory_done.wait()\n\n # Now a new get_or_create should get the cached value\n result = await cache.get_or_create(\"key\", factory)\n assert result == 99\n\n\n@pytest.mark.asyncio\nasync def test_factory_failure_not_cached() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n call_count = 0\n\n async def failing_factory() -> int:\n nonlocal call_count\n call_count += 1\n raise ValueError(\"factory failed\")\n\n with pytest.raises(ValueError, match=\"factory failed\"):\n await cache.get_or_create(\"key\", failing_factory)\n\n # Factory should not be cached; next call should invoke factory again\n with pytest.raises(ValueError, match=\"factory failed\"):\n await cache.get_or_create(\"key\", failing_factory)\n\n assert call_count == 2\n\n\n@pytest.mark.asyncio\nasync def test_invalidation_during_creation_prevents_stale_publication() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n factory_started = asyncio.Event()\n\n async def factory() -> int:\n factory_started.set()\n await asyncio.sleep(0.1)\n return 123\n\n # Start get_or_create\n task = asyncio.ensure_future(cache.get_or_create(\"key\", factory))\n\n # Wait for factory to start\n await factory_started.wait()\n\n # Invalidate while factory is running\n await cache.invalidate(\"key\")\n\n # The original task should raise an exception\n with pytest.raises(RuntimeError, match=\"invalidated\"):\n await task\n\n # A new get_or_create should invoke the factory again\n result = await cache.get_or_create(\"key\", factory)\n assert result == 123\n\n\n@pytest.mark.asyncio\nasync def test_weighted_lru_eviction()", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:27:04.745948+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6236594580113888, + "total_s": 242.9034315830213, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 12.73210751545455, + "finish_reason": "length", + "output_chars": 11719, + "output_sha256": "64e2a1896826c7e6a38540d8e7d2650e66be4c3df39a3a63ca49261d90eeea17", + "output": "first();\n if ($existing !== null) {\n return $existing;\n }\n\n $lastException = null;\n\n for ($attempt = 0; $attempt < self::MAX_DEADLOCK_RETRIES; $attempt++) {\n try {\n return DB::transaction(function () use ($fromAccountId, $toAccountId, $amountCents, $idempotencyKey) {\n // Lock both account rows in deterministic ID order to prevent deadlocks\n $minId = min($fromAccountId, $toAccountId);\n $maxId = max($fromAccountId, $toAccountId);\n\n $accounts = Account::whereIn('id', [$minId, $maxId])\n ->orderBy('id')\n ->lockForUpdate()\n ->get();\n\n if ($accounts->count() !== 2) {\n throw new RuntimeException('One or both accounts do not exist.');\n }\n\n $fromAccount = $accounts->firstWhere('id', $fromAccountId);\n $toAccount = $accounts->firstWhere('id', $toAccountId);\n\n // Check sufficient funds\n if ($fromAccount->balance_cents < $amountCents) {\n throw new RuntimeException('Insufficient funds for transfer.');\n }\n\n // Perform the transfer\n $fromAccount->balance_cents -= $amountCents;\n $fromAccount->save();\n\n $toAccount->balance_cents += $amountCents;\n $toAccount->save();\n\n // Create the transfer record\n $transfer = Transfer::create([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'created_at' => now(),\n ]);\n\n // Persist outbox event in the same transaction\n OutboxEvent::create([\n 'transfer_id' => $transfer->id,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transfer->id,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n ]),\n 'created_at' => now(),\n ]);\n\n return $transfer;\n });\n } catch (\\Illuminate\\Database\\QueryException $e) {\n // Detect deadlock (MySQL error 1213) and retry\n if ($this->isDeadlock($e)) {\n $lastException = $e;\n continue;\n }\n throw $e;\n }\n }\n\n throw new RuntimeException('Transfer failed after maximum deadlock retries.', 0, $lastException);\n }\n\n /**\n * Determine if a QueryException represents a deadlock.\n */\n private function isDeadlock(\\Illuminate\\Database\\QueryException $e): bool\n {\n // MySQL deadlock error code is 1213\n return $e->getCode() === 1213 || str_contains($e->getMessage(), '1213');\n }\n}\n\nnamespace Tests\\Unit;\n\nuse App\\Models\\Account;\nuse App\\Models\\Transfer;\nuse App\\Models\\OutboxEvent;\nuse App\\Services\\MoneyTransferService;\nuse Illuminate\\Support\\Facades\\DB;\nuse PHPUnit\\Framework\\TestCase;\n\nclass MoneyTransferServiceTest extends TestCase\n{\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Reset tables\n DB::table('outbox_events')->delete();\n DB::table('transfers')->delete();\n DB::table('accounts')->delete();\n\n $this->service = new MoneyTransferService();\n }\n\n /**\n * Test that a valid transfer moves funds correctly.\n */\n public function testValidTransferMovesFunds(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $transfer = $this->service->transfer(1, 2, 2500, 'key-001');\n\n $this->assertNotNull($transfer->id);\n $this->assertEquals(7500, Account::find(1)->balance_cents);\n $this->assertEquals(7500, Account::find(2)->balance_cents);\n $this->assertEquals(1, Transfer::count());\n $this->assertEquals(1, OutboxEvent::count());\n }\n\n /**\n * Test that duplicate requests with the same idempotency key return the original transfer\n * without duplicating effects.\n */\n public function testDuplicateRequestReturnsOriginalTransfer(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $first = $this->service->transfer(1, 2, 2500, 'key-dup-001');\n $second = $this->service->transfer(1, 2, 2500, 'key-dup-001');\n\n // Same transfer returned\n $this->assertEquals($first->id, $second->id);\n\n // Balances reflect only one transfer\n $this->assertEquals(7500, Account::find(1)->balance_cents);\n $this->assertEquals(7500, Account::find(2)->balance_cents);\n\n // Only one transfer and one outbox event\n $this->assertEquals(1, Transfer::count());\n $this->assertEquals(1, OutboxEvent::count());\n }\n\n /**\n * Test that competing requests (simulated by sequential calls with same key)\n * preserve balances and create exactly one transfer and one outbox event.\n */\n public function testCompetingRequestsPreserveBalances(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n // Simulate two competing requests with the same idempotency key\n $result1 = $this->service->transfer(1, 2, 3000, 'key-compete-001');\n $result2 = $this->service->transfer(1, 2, 3000, 'key-compete-001');\n\n $this->assertEquals($result1->id, $result2->id);\n $this->assertEquals(7000, Account::find(1)->balance_cents);\n $this->assertEquals(8000, Account::find(2)->balance_cents);\n $this->assertEquals(1, Transfer::count());\n $this->assertEquals(1, OutboxEvent::count());\n }\n\n /**\n * Test that insufficient funds are rejected.\n */\n public function testInsufficientFundsAreRejected(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 1000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $this->expectException(\\RuntimeException::class);\n $this->expectExceptionMessage('Insufficient funds');\n\n $this->service->transfer(1, 2, 5000, 'key-insufficient-001');\n }\n\n /**\n * Test that zero or negative amounts are rejected.\n */\n public function testNonPositiveAmountIsRejected(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $this->expectException(\\RuntimeException::class);\n $this->expectExceptionMessage('positive');\n\n $this->service->transfer(1, 2, 0, 'key-zero-001');\n }\n\n /**\n * Test that same source and destination accounts are rejected.\n */\n public function testSameAccountIsRejected(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n\n $this->expectException(\\RuntimeException::class);\n $this->expectExceptionMessage('distinct');\n\n $this->service->transfer(1, 1, 1000, 'key-same-001');\n }\n\n /**\n * Test that different idempotency keys create separate transfers.\n */\n public function testDifferentKeysCreateSeparateTransfers(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $this->service->transfer(1, 2, 1000, 'key-a-001');\n $this->service->transfer(1, 2, 2000, 'key-b-001');\n\n $this->assertEquals(7000, Account::find(1)->balance_cents);\n $this->assertEquals(8000, Account::find(2)->balance_cents);\n $this->assertEquals(2, Transfer::count());\n $this->assertEquals(2, OutboxEvent::count());\n }\n\n /**\n * Test that the outbox event contains correct payload.\n */\n public function testOutboxEventHasCorrectPayload(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $transfer = $this->service->transfer(1, 2, 2500, 'key-payload-001');\n\n $event = OutboxEvent::first();\n $this->assertEquals($transfer->id, $event->transfer_id);\n $this->assertEquals", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:31:07.651240+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4923370420001447, + "total_s": 224.71278991695726, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 13.76217976638934, + "finish_reason": "length", + "output_chars": 10877, + "output_sha256": "bb305dfdb203bcf6d40071cb30ae92561f0b39bf38f52973d3f27e45cd5ff0a7", + "output": "type TaskResult = unknown;\n\ninterface Task {\n id: string;\n deps: readonly string[];\n run: (signal: AbortSignal) => Promise;\n}\n\ninterface SchedulerOptions {\n maxConcurrency?: number;\n}\n\ntype TaskStatus = 'pending' | 'running' | 'succeeded' | 'failed' | 'skipped';\n\ninterface TaskState {\n task: Task;\n status: TaskStatus;\n result?: T;\n error?: unknown;\n}\n\nclass CycleError extends Error {\n readonly path: readonly string[];\n constructor(path: readonly string[]) {\n super(`Dependency cycle detected: ${path.join(' -> ')}`);\n this.name = 'CycleError';\n this.path = path;\n }\n}\n\nclass DuplicateIdError extends Error {\n constructor(id: string) {\n super(`Duplicate task ID: \"${id}\"`);\n this.name = 'DuplicateIdError';\n }\n}\n\nclass UnknownDependencyError extends Error {\n constructor(taskId: string, depId: string) {\n super(`Task \"${taskId}\" depends on unknown task \"${depId}\"`);\n this.name = 'UnknownDependencyError';\n }\n}\n\nclass TaskExecutionError extends Error {\n readonly taskId: string;\n readonly cause: unknown;\n constructor(taskId: string, cause: unknown) {\n super(`Task \"${taskId}\" failed: ${cause instanceof Error ? cause.message : String(cause)}`);\n this.name = 'TaskExecutionError';\n this.taskId = taskId;\n this.cause = cause;\n }\n}\n\nfunction detectCycle(\n tasks: readonly Task[],\n idToIndex: Map\n): string[] | null {\n const n = tasks.length;\n const color = new Array(n).fill(0);\n const stack: number[] = [];\n\n const dfs = (u: number): boolean => {\n color[u] = 1;\n stack.push(u);\n const deps = tasks[u].deps;\n for (const depId of deps) {\n const v = idToIndex.get(depId);\n if (v === undefined) continue;\n if (color[v] === 1) {\n const start = stack.indexOf(v);\n return true;\n }\n if (color[v] === 0 && dfs(v)) {\n return true;\n }\n }\n stack.pop();\n color[u] = 2;\n return false;\n };\n\n for (let i = 0; i < n; i++) {\n if (color[i] === 0) {\n if (dfs(i)) {\n const start = stack.indexOf(i);\n const path = stack.slice(start).map((idx) => tasks[idx].id);\n path.push(tasks[i].id);\n return path;\n }\n }\n }\n return null;\n}\n\nfunction buildTopologicalOrder(\n tasks: readonly Task[],\n idToIndex: Map\n): number[] {\n const n = tasks.length;\n const inDegree = new Array(n).fill(0);\n const dependents = new Array>(n);\n for (let i = 0; i < n; i++) {\n dependents[i] = new Set();\n }\n\n for (let i = 0; i < n; i++) {\n const deps = tasks[i].deps;\n for (const depId of deps) {\n const depIdx = idToIndex.get(depId);\n if (depIdx === undefined) continue;\n inDegree[i]++;\n dependents[depIdx].add(i);\n }\n }\n\n const queue: number[] = [];\n for (let i = 0; i < n; i++) {\n if (inDegree[i] === 0) {\n queue.push(i);\n }\n }\n\n const order: number[] = [];\n while (queue.length > 0) {\n const u = queue.shift()!;\n order.push(u);\n const deps = tasks[u].deps;\n for (const depId of deps) {\n const depIdx = idToIndex.get(depId);\n if (depIdx === undefined) continue;\n inDegree[depIdx]--;\n if (inDegree[depIdx] === 0) {\n queue.push(depIdx);\n }\n }\n }\n\n return order;\n}\n\nexport async function runDag(\n tasks: readonly Task[],\n options: SchedulerOptions = {},\n externalSignal?: AbortSignal\n): Promise> {\n const maxConcurrency = options.maxConcurrency ?? Infinity;\n\n if (tasks.length === 0) {\n return new Map();\n }\n\n const idToIndex = new Map();\n for (let i = 0; i < tasks.length; i++) {\n const id = tasks[i].id;\n if (idToIndex.has(id)) {\n throw new DuplicateIdError(id);\n }\n idToIndex.set(id, i);\n }\n\n for (let i = 0; i < tasks.length; i++) {\n const task = tasks[i];\n for (const depId of task.deps) {\n if (!idToIndex.has(depId)) {\n throw new UnknownDependencyError(task.id, depId);\n }\n }\n }\n\n const cyclePath = detectCycle(tasks, idToIndex);\n if (cyclePath !== null) {\n throw new CycleError(cyclePath);\n }\n\n const topoOrder = buildTopologicalOrder(tasks, idToIndex);\n\n const states: TaskState[] = tasks.map((task) => ({\n task,\n status: 'pending' as TaskStatus,\n }));\n\n const results = new Map();\n const failedTaskIds = new Set();\n const skippedTaskIds = new Set();\n let firstError: unknown = null;\n let firstErrorTaskId: string | null = null;\n\n const abortController = new AbortController();\n const signal = abortController.signal;\n\n if (externalSignal) {\n if (externalSignal.aborted) {\n abortController.abort(externalSignal.reason);\n } else {\n externalSignal.addEventListener('abort', () => {\n abortController.abort(externalSignal.reason);\n }, { once: true });\n }\n }\n\n const runningPromises = new Set>();\n let activeCount = 0;\n let completedCount = 0;\n const totalTasks = tasks.length;\n\n const onAbort = () => {\n for (const state of states) {\n if (state.status === 'pending') {\n state.status = 'skipped';\n skippedTaskIds.add(state.task.id);\n }\n }\n };\n\n if (signal.aborted) {\n onAbort();\n } else {\n signal.addEventListener('abort', onAbort, { once: true });\n }\n\n const tryStartTask = (idx: number): void => {\n const state = states[idx];\n if (state.status !== 'pending') return;\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(state.task.id);\n return;\n }\n if (activeCount >= maxConcurrency) return;\n\n const task = state.task;\n state.status = 'running';\n activeCount++;\n\n const promise = (async () => {\n try {\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n return;\n }\n\n const result = await task.run(signal);\n\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n return;\n }\n\n state.status = 'succeeded';\n state.result = result;\n results.set(task.id, result);\n } catch (err) {\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n return;\n }\n\n state.status = 'failed';\n state.error = err;\n failedTaskIds.add(task.id);\n\n if (firstError === null) {\n firstError = err;\n firstErrorTaskId = task.id;\n }\n } finally {\n activeCount--;\n completedCount++;\n runningPromises.delete(promise);\n scheduleNext();\n }\n })();\n\n runningPromises.add(promise);\n };\n\n const scheduleNext = (): void => {\n if (signal.aborted) return;\n\n for (const idx of topoOrder) {\n const state = states[idx];\n if (state.status !== 'pending') continue;\n\n const task = state.task;\n const deps = task.deps;\n\n let canStart = true;\n for (const depId of deps) {\n const depIdx = idToIndex.get(depId);\n if (depIdx === undefined) continue;\n const depState = states[depIdx];\n if (depState.status === 'failed') {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n canStart = false;\n break;\n }\n if (depState.status === 'skipped') {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n canStart = false;\n break;\n }\n if (depState.status !== 'succeeded') {\n canStart = false;\n break;\n }\n }\n\n if (!canStart) continue;\n\n if (activeCount >= maxConcurrency) break;\n\n tryStartTask(idx);\n }\n };\n\n scheduleNext();\n\n const allDone = new Promise((resolve) => {\n const check = () => {\n if (completedCount >= totalTasks) {\n resolve();\n }\n };\n\n const interval = setInterval(check, 1);\n const onAbortCheck = () => {\n clearInterval(interval);\n check();\n };\n\n if (signal.aborted) {\n clearInterval(interval);\n onAbortCheck();\n } else {\n signal.addEventListener('abort', onAbortCheck, { once: true });\n }\n\n const originalCheck = check;\n const wrappedCheck = () => {\n originalCheck();\n if (completedCount >= totalTasks) {\n clearInterval(interval);\n }\n };\n\n const originalScheduleNext = scheduleNext;\n // We need to hook into completion. Let's use a different approach.\n });\n\n // The above approach is flawed. Let me restructure.\n\n // Actually, let me redo the scheduling with a cleaner approach.\n\n // Reset and use a proper completion tracking.\n // I'll restructure the whole function.\n\n // Let me just return a proper implementation.\n\n // The issue is I'm trying to track completion. Let me use a counter that increments\n // in the finally block of each task, and resolve when it reaches totalTasks.\n\n // But I already have completedCount. The problem is the allDone promise setup.\n // Let me fix it.\n\n // Actually, the code above has a bug in the allDone setup. Let me rewrite the whole thing cleanly.\n\n // I'll just return the correct implementation.\n\n // Let me restart the function body properly.\n\n // OK I realize I'm going in circles. Let me just write the correct code.\n\n // The key insight: I need a promise that resolves when all tasks are done.\n // I can track this with a counter and a resolve function.\n\n // Let me rewrite from scratch in the actual output.\n\n // For now, let me just make sure the logic is correct and output the final version.\n\n // I'll output the corrected version below.\n\n // Actually, I should just write the correct code. Let me do that.\n\n // The allDone promise should be created with a resolve that's called when completedCount reaches totalTasks.\n // I can do this by having a variable that holds the resolve function.\n\n // Let me restructure:\n\n // const allDoneResolve: (() => void) | null = ...\n // No, I'll use a different pattern.\n\n // I'll create the promise and store the resolve in a closure.\n\n // Here's the correct approach:\n\n // let resolveAllDone: () => void;\n // const allDone = new Promise((r) => { resolveAllDone = r; });\n // Then in the finally block: if (completedCount >= totalTasks) resolveAllDone();\n\n // That's clean. Let me use that.\n\n // OK, I'll write the final correct version now.\n\n // I realize I've been overthinking this. Let me just output the correct code.\n\n // The final answer will be a clean, correct implementation.\n\n // Let me write it out properly.\n\n // I'll use the pattern:\n // let resolveAllDone: () => void;\n // const allDone = new Promise((r) => { resolveAllDone = r; });\n // And call resolveAllDone() when completedCount === totalTasks.\n\n // That's the cleanest approach.\n\n // Now let me write the", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json new file mode 100644 index 0000000..b74496a --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:22:57.038624+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1419034169521183, + "total_s": 247.70563520898577, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 12.459253344652916, + "finish_reason": "length", + "output_chars": 13028, + "output_sha256": "699045091c4f5bc19d0f8f0bbc676b164512c8e3e7dec0f95acb13b5bef2f4ba", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:27:04.745948+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6236594580113888, + "total_s": 242.9034315830213, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 12.73210751545455, + "finish_reason": "length", + "output_chars": 11719, + "output_sha256": "64e2a1896826c7e6a38540d8e7d2650e66be4c3df39a3a63ca49261d90eeea17", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:31:07.651240+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4923370420001447, + "total_s": 224.71278991695726, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 13.76217976638934, + "finish_reason": "length", + "output_chars": 10877, + "output_sha256": "bb305dfdb203bcf6d40071cb30ae92561f0b39bf38f52973d3f27e45cd5ff0a7", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json new file mode 100644 index 0000000..59a8aad --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json @@ -0,0 +1,48 @@ +{ + "run_id": "20260831T133636Z", + "created_at": "2026-08-31T13:37:14.110488+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "model": "mlx-community/Qwen3.8-27B-nvfp4", + "label": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "max_tokens": 128, + "seed": 42, + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results" + }, + "load_s": 3.066661207994912, + "mlx_version": "0.32.0", + "device": "Device(gpu, 0)", + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "summary": { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.4445558497915044, + "ttft_s_p50": 0.48620820802170783, + "ttft_s_p95": 0.518837732996326, + "total_s_mean": 4.9355954667902555, + "total_s_p50": 4.966667582979426, + "total_s_p95": 5.034370008390397, + "prompt_tps_mean": 107.94239735907567, + "prompt_tps_p50": 107.53363787342614, + "prompt_tps_p95": 135.27062378239268, + "decode_tps_mean": 28.570950408778582, + "decode_tps_p50": 28.53361993265224, + "decode_tps_p95": 28.826484879691197, + "request_tps_mean": 25.943347049499913, + "request_tps_p50": 25.771807325831702, + "request_tps_p95": 26.64316963019791, + "peak_memory_gb": 15.479377792 + } +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json new file mode 100644 index 0000000..2464f98 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json @@ -0,0 +1,128 @@ +[ + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.6381363340187818, + "total_s": 5.067599917005282, + "prompt_tps": 46.64172306513033, + "decode_tps": 28.964400213729846, + "request_tps": 25.258505425906254, + "peak_memory_gb": 15.399916034, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.46635333297308534, + "total_s": 4.885915416001808, + "prompt_tps": 122.76309757245242, + "decode_tps": 29.032294926274737, + "request_tps": 26.19775192603388, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.3432638749945909, + "total_s": 4.786967416992411, + "prompt_tps": 90.81617646556052, + "decode_tps": 28.873430911706816, + "request_tps": 26.73926702438696, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.48620820802170783, + "total_s": 4.966667582979426, + "prompt_tps": 118.1052752991172, + "decode_tps": 28.63870075162873, + "request_tps": 25.771807325831702, + "peak_memory_gb": 15.453784176, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 3, + "prompt_name": "code", + "prompt_tokens": 54, + "output_tokens": 128, + "ttft_s": 0.5125948329805396, + "total_s": 5.009093041997403, + "prompt_tps": 139.56196090321154, + "decode_tps": 28.53361993265224, + "request_tps": 25.553528139089888, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 4, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.36031387496041134, + "total_s": 4.874560041993391, + "prompt_tps": 83.69493625406292, + "decode_tps": 28.424103574919425, + "request_tps": 26.258780053441704, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 5, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5203984580002725, + "total_s": 5.040689249988645, + "prompt_tps": 107.53363787342614, + "decode_tps": 28.384896872985717, + "request_tps": 25.393352704749322, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json new file mode 100644 index 0000000..ba73797 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json @@ -0,0 +1,22 @@ +{ + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.4445558497915044, + "ttft_s_p50": 0.48620820802170783, + "ttft_s_p95": 0.518837732996326, + "total_s_mean": 4.9355954667902555, + "total_s_p50": 4.966667582979426, + "total_s_p95": 5.034370008390397, + "prompt_tps_mean": 107.94239735907567, + "prompt_tps_p50": 107.53363787342614, + "prompt_tps_p95": 135.27062378239268, + "decode_tps_mean": 28.570950408778582, + "decode_tps_p50": 28.53361993265224, + "decode_tps_p95": 28.826484879691197, + "request_tps_mean": 25.943347049499913, + "request_tps_p50": 25.771807325831702, + "request_tps_p95": 26.64316963019791, + "peak_memory_gb": 15.479377792 +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json new file mode 100644 index 0000000..27bdef5 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json @@ -0,0 +1,48 @@ +{ + "run_id": "20260831T133721Z", + "created_at": "2026-08-31T13:38:28.661722+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "model": "mlx-community/Qwen3.8-27B-mxfp8", + "label": "qwen3.8-27b-mxfp8", + "warmups": 2, + "rounds": 5, + "max_tokens": 128, + "seed": 42, + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results" + }, + "load_s": 5.387068083975464, + "mlx_version": "0.32.0", + "device": "Device(gpu, 0)", + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "summary": { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.6405866249930113, + "ttft_s_p50": 0.6006166669540107, + "ttft_s_p95": 0.993492508213967, + "total_s_mean": 8.97830267478712, + "total_s_p50": 8.952084249991458, + "total_s_p95": 9.543902474397328, + "prompt_tps_mean": 77.25518114106583, + "prompt_tps_p50": 69.803885982221, + "prompt_tps_p95": 108.54351756783554, + "decode_tps_mean": 15.141694584402668, + "decode_tps_p50": 15.417877788139895, + "decode_tps_p95": 15.90376325022775, + "request_tps_mean": 14.036390549836813, + "request_tps_p50": 14.298346220336581, + "request_tps_p95": 14.998850485179887, + "peak_memory_gb": 28.05746427 + } +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json new file mode 100644 index 0000000..252b13a --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json @@ -0,0 +1,128 @@ +[ + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "warmup", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 122, + "ttft_s": 1.4950010409811512, + "total_s": 8.873082707985304, + "prompt_tps": 65.2268651572191, + "decode_tps": 16.569121716474896, + "request_tps": 13.749449206667087, + "peak_memory_gb": 27.989241503, + "finish_reason": "stop", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "warmup", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5392710830201395, + "total_s": 8.420522083004471, + "prompt_tps": 104.9724920224018, + "decode_tps": 16.272217242824215, + "request_tps": 15.200957700514595, + "peak_memory_gb": 28.031514318, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 122, + "ttft_s": 0.41662891599116847, + "total_s": 8.074155374953989, + "prompt_tps": 69.803885982221, + "decode_tps": 15.96651063653789, + "request_tps": 15.109939595470719, + "peak_memory_gb": 28.031514318, + "finish_reason": "stop", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.6006166669540107, + "total_s": 8.794534499989823, + "prompt_tps": 94.30109779838891, + "decode_tps": 15.652773704987183, + "request_tps": 14.554494044016558, + "peak_memory_gb": 28.031531726, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 3, + "prompt_name": "code", + "prompt_tokens": 54, + "output_tokens": 128, + "ttft_s": 0.6342117090243846, + "total_s": 8.952084249991458, + "prompt_tps": 112.1041225101972, + "decode_tps": 15.417877788139895, + "request_tps": 14.298346220336581, + "peak_memory_gb": 28.05746427, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 4, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 122, + "ttft_s": 0.46816312498413026, + "total_s": 9.549591040995438, + "prompt_tps": 61.572468489546665, + "decode_tps": 13.468195538794124, + "request_tps": 12.775416190731752, + "peak_memory_gb": 28.05746427, + "finish_reason": "stop", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 5, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 1.0833127080113627, + "total_s": 9.521148208004888, + "prompt_tps": 48.494330924975344, + "decode_tps": 15.203115253554254, + "request_tps": 13.443756698628452, + "peak_memory_gb": 28.05746427, + "finish_reason": "length", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json new file mode 100644 index 0000000..54446c6 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json @@ -0,0 +1,22 @@ +{ + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.6405866249930113, + "ttft_s_p50": 0.6006166669540107, + "ttft_s_p95": 0.993492508213967, + "total_s_mean": 8.97830267478712, + "total_s_p50": 8.952084249991458, + "total_s_p95": 9.543902474397328, + "prompt_tps_mean": 77.25518114106583, + "prompt_tps_p50": 69.803885982221, + "prompt_tps_p95": 108.54351756783554, + "decode_tps_mean": 15.141694584402668, + "decode_tps_p50": 15.417877788139895, + "decode_tps_p95": 15.90376325022775, + "request_tps_mean": 14.036390549836813, + "request_tps_p50": 14.298346220336581, + "request_tps_p95": 14.998850485179887, + "peak_memory_gb": 28.05746427 +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json new file mode 100644 index 0000000..a294dc3 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json @@ -0,0 +1,48 @@ +{ + "run_id": "20260831T134407Z", + "created_at": "2026-08-31T13:44:45.718463+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "model": "mlx-community/Qwen3.8-27B-nvfp4", + "label": "qwen3.8-27b-nvfp4-rerun", + "warmups": 2, + "rounds": 5, + "max_tokens": 128, + "seed": 42, + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results" + }, + "load_s": 3.153125083015766, + "mlx_version": "0.32.0", + "device": "Device(gpu, 0)", + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "summary": { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.46268870839849113, + "ttft_s_p50": 0.5028267079615034, + "ttft_s_p95": 0.5507681754184887, + "total_s_mean": 5.0599139665835535, + "total_s_p50": 5.047160332964268, + "total_s_p95": 5.222982274997048, + "prompt_tps_mean": 103.80325950757997, + "prompt_tps_p50": 102.6553925326314, + "prompt_tps_p95": 128.99882432098076, + "decode_tps_mean": 27.91928963114296, + "decode_tps_p50": 27.87385628056727, + "decode_tps_p95": 28.51407145210707, + "request_tps_mean": 25.315032232739288, + "request_tps_p50": 25.360795289977208, + "request_tps_p95": 26.258409367849122, + "peak_memory_gb": 15.479377792 + } +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json new file mode 100644 index 0000000..2d74af1 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json @@ -0,0 +1,128 @@ +[ + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.46846333297435194, + "total_s": 4.887791082961485, + "prompt_tps": 84.84884256090237, + "decode_tps": 29.034596267228054, + "request_tps": 26.18769866130316, + "peak_memory_gb": 15.399948802, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.476554416003637, + "total_s": 4.906530833046418, + "prompt_tps": 122.50184376172285, + "decode_tps": 28.96545575207384, + "request_tps": 26.0876787195335, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.34607333299936727, + "total_s": 4.835503207985312, + "prompt_tps": 90.02367046337802, + "decode_tps": 28.58315107058229, + "request_tps": 26.470874797192113, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5028267079615034, + "total_s": 5.047160332964268, + "prompt_tps": 113.11164547028159, + "decode_tps": 28.2377529782062, + "request_tps": 25.360795289977208, + "peak_memory_gb": 15.453784176, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 3, + "prompt_name": "code", + "prompt_tokens": 54, + "output_tokens": 128, + "ttft_s": 0.5305347089888528, + "total_s": 5.134004874969833, + "prompt_tps": 132.97061903365554, + "decode_tps": 27.87385628056727, + "request_tps": 24.93180336155254, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 4, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.3781822500168346, + "total_s": 5.037674791994505, + "prompt_tps": 80.25497003795336, + "decode_tps": 27.538176565956825, + "request_tps": 25.408547650477164, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 5, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5558265420258977, + "total_s": 5.245226625003852, + "prompt_tps": 102.6553925326314, + "decode_tps": 27.363511260402223, + "request_tps": 24.403140064497403, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json new file mode 100644 index 0000000..400ae9c --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json @@ -0,0 +1,22 @@ +{ + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.46268870839849113, + "ttft_s_p50": 0.5028267079615034, + "ttft_s_p95": 0.5507681754184887, + "total_s_mean": 5.0599139665835535, + "total_s_p50": 5.047160332964268, + "total_s_p95": 5.222982274997048, + "prompt_tps_mean": 103.80325950757997, + "prompt_tps_p50": 102.6553925326314, + "prompt_tps_p95": 128.99882432098076, + "decode_tps_mean": 27.91928963114296, + "decode_tps_p50": 27.87385628056727, + "decode_tps_p95": 28.51407145210707, + "request_tps_mean": 25.315032232739288, + "request_tps_p50": 25.360795289977208, + "request_tps_p95": 26.258409367849122, + "peak_memory_gb": 15.479377792 +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114727Z/metadata.json b/benchmarks/inference/results/20260831T114727Z/metadata.json new file mode 100644 index 0000000..471c894 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/metadata.json @@ -0,0 +1,136 @@ +{ + "run_id": "20260831T114727Z", + "created_at": "2026-08-31T11:47:27.622260+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "vllm", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-nvfp4" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 13:47:28 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 13:47:28 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 13:47:28 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 13:47:30 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 13:47:30 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "vllm-metal": { + "object": "list", + "data": [ + { + "id": "qwen3.8-27b-nvfp4", + "object": "model", + "created": 1788176852, + "owned_by": "vllm", + "root": "mlx-community/Qwen3.8-27B-nvfp4", + "parent": null, + "max_model_len": 8192, + "permission": [ + { + "id": "modelperm-956fc45cb39100d2", + "object": "model_permission", + "created": 1788176852, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + } + ] + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T11:49:15.455115+00:00", + "summary": [ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.3338013501954265, + "ttft_s_p50": 0.37227433401858434, + "ttft_s_p95": 0.4201282836031169, + "total_s_mean": 5.444907250197138, + "total_s_p50": 5.454897999996319, + "total_s_p95": 5.562144016823732, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 25.04504255080693, + "decode_tps_p50": 24.962500989743333, + "decode_tps_p95": 25.32382313060655, + "request_tps_mean": 23.517575676354507, + "request_tps_p50": 23.465150035818517, + "request_tps_p95": 24.196276605031606, + "aggregate_output_tps_mean": 23.516827822391825, + "aggregate_output_tps_p50": 23.464397444683396, + "aggregate_output_tps_p95": 24.195480643338048 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.1358489938022103, + "ttft_s_p50": 1.1691114795103204, + "ttft_s_p95": 1.2360623297834537, + "total_s_mean": 9.343862856202758, + "total_s_p50": 9.272835896001197, + "total_s_p95": 9.591099269420374, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.572524640705884, + "decode_tps_p50": 15.580143217705817, + "decode_tps_p95": 15.920671319893613, + "request_tps_mean": 13.679827878866371, + "request_tps_p50": 13.803600866803595, + "request_tps_p95": 13.831739950302437, + "aggregate_output_tps_mean": 54.66256367620249, + "aggregate_output_tps_p50": 54.89183177619698, + "aggregate_output_tps_p95": 55.24148726503248 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114727Z/raw.csv b/benchmarks/inference/results/20260831T114727Z/raw.csv new file mode 100644 index 0000000..2e7c106 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,1,0,0,8b712536-91f2-4ded-b334-6db253435ba3,short,106,128,2026-08-31T11:47:32.567050+00:00,0.561656582984142,5.602756957989186,20,128,,25.39128175956463,22.845895504619364,5.6035958750289865,22.842475234589944, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,1,1,0,be5a9e94-abd6-4cbc-8029-7a55e3ad129c,medium,252,128,2026-08-31T11:47:38.170087+00:00,0.3621360409888439,5.426666249986738,43,128,,25.273815086064428,23.587225398339694,5.426845958980266,23.586444311762243, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,0,0,e0ac985d-818e-45ec-a2c1-7e077fe0f1cd,short,106,128,2026-08-31T11:47:43.597014+00:00,0.21451262495247647,5.262066499970388,20,128,,25.358817987761604,24.325044162919706,5.262241083022673,24.324237141654443, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,1,0,428bad62-8fbf-41a1-9b36-07912506da11,medium,252,128,2026-08-31T11:47:48.859322+00:00,0.37227433401858434,5.454897999996319,43,128,,25.18384370198632,23.465150035818517,5.455072959011886,23.464397444683396, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,2,0,b2de9565-6ff0-47a1-91e6-847b83994f6c,code,267,128,2026-08-31T11:47:54.314442+00:00,0.40532424999400973,5.533015583991073,54,128,,24.962500989743333,23.13385857259254,5.533179917023517,23.13317150707355, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,3,0,dc02daab-d1f2-49ac-8e29-01ee86148e0a,short,106,128,2026-08-31T11:47:59.847681+00:00,0.25306625000666827,5.405130041996017,20,128,,24.844412873734196,23.681206373479203,5.405301624967251,23.680454650072466, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,4,0,677f2a25-31e5-43e7-b8fd-5e607453b904,medium,252,128,2026-08-31T11:48:05.253048+00:00,0.42382929200539365,5.569426125031896,43,128,,24.875637200809187,22.982619236962577,5.569605667027645,22.981878368475286, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,0,7109b7b9-3690-49d6-8b20-969734c3dcad,short,106,128,2026-08-31T11:48:10.822944+00:00,1.1026008339831606,8.694552083965391,20,128,,16.859960738064487,14.721862467884838,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,1,b34a5a99-e633-4f3a-892d-67af2b425779,medium,252,128,2026-08-31T11:48:10.823149+00:00,1.1023455830290914,8.694302666990552,43,128,,16.859947782161328,14.722284799904022,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,2,1260b39f-a4b1-4d94-9406-70028f614ce0,code,267,128,2026-08-31T11:48:10.823017+00:00,1.0163157919887453,8.694556291971821,54,128,,16.670485901070972,14.721855342772315,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,3,b43d525a-50de-4e41-b00c-d1159d1ddfd4,short,106,128,2026-08-31T11:48:10.822845+00:00,1.1026693329913542,8.694680040993262,20,128,,16.85982869664412,14.721645810600473,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,0,1da340a9-b91e-4f9b-8217-2af7c2fbc030,medium,252,128,2026-08-31T11:48:19.518058+00:00,1.245261833013501,9.168966500030365,43,128,,16.154059922603068,13.960133892906699,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,1,23e076e6-7b50-4d08-a3ff-f79ddc53bafe,code,267,128,2026-08-31T11:48:19.518183+00:00,1.2451274580089375,9.168911040993407,54,128,,16.153899038190183,13.96021833211415,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,2,c777d58b-0520-4f3f-8537-6a163ab350f5,short,106,128,2026-08-31T11:48:19.518319+00:00,1.2449699160060845,9.168749333010055,20,128,,16.153907531211612,13.960464546583719,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,3,5609655c-e54a-499f-99c6-c77c19c1a3e0,medium,252,128,2026-08-31T11:48:19.518407+00:00,1.174761500034947,9.16857587499544,43,128,,16.012380822970083,13.960728661152478,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,0,d2615398-7368-4f9a-9424-7a2e76d8955d,short,106,128,2026-08-31T11:48:28.687706+00:00,1.0508670000126585,9.266459833015688,20,128,,15.580129468662143,13.813257954666332,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,1,b307dcc2-b325-4005-a63f-54123aa28e31,medium,252,128,2026-08-31T11:48:28.687614+00:00,1.0509756249957718,9.266665957984515,43,128,,15.579944570943383,13.812950696653772,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,2,09c7607f-96f5-4a6d-a9e1-3637c8afc257,code,267,128,2026-08-31T11:48:28.687778+00:00,1.0507545830332674,9.266332915984094,54,128,,15.580156966749492,13.813447149001583,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,3,0328d6c0-e3f4-4a86-917d-dbeb5b8deee7,short,106,128,2026-08-31T11:48:28.687501+00:00,0.9898037080420181,9.266721500025596,20,128,,15.464693889308833,13.81286790583341,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,0,74110182-284c-4250-9395-e3d0f2772eef,medium,252,128,2026-08-31T11:48:37.954714+00:00,1.1083811249700375,9.379890666983556,43,128,,15.474805336299134,13.646214496992963,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,1,fca02813-8ad6-4e99-bc76-9e922ea9a63a,code,267,128,2026-08-31T11:48:37.954992+00:00,1.1690869170124643,9.379730832995847,54,128,,15.589520299476966,13.646447033397155,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,2,5eb13719-9063-4ddd-92c8-73332709ecfd,short,106,128,2026-08-31T11:48:37.954812+00:00,1.1692542500095442,9.37985170801403,20,128,,15.589608509574806,13.646271176189105,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,3,9d9a5b23-6950-4d5c-bad2-72f896d38009,medium,252,128,2026-08-31T11:48:37.954917+00:00,1.1691360420081764,9.379842250025831,43,128,,15.58940202671112,13.646284936150979,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,0,caf79143-9d5b-4fb2-830e-3bd2d38cc5d3,code,267,128,2026-08-31T11:48:47.335341+00:00,1.172440624970477,9.591207291989122,54,128,,15.204127286417465,13.345556623190662,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,1,d77a5e81-922c-49ce-96c5-1fb443746984,short,106,128,2026-08-31T11:48:47.335566+00:00,1.2360592089826241,9.591093584022019,20,128,,15.320104532711328,13.345714842490702,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,2,9ccc94c9-d569-4785-9f48-e48e760b0e15,medium,252,128,2026-08-31T11:48:47.335447+00:00,1.236121624999214,9.591066916007549,43,128,,15.320267882275031,13.345751950324445,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,3,e7c5be37-35a8-47a5-869d-a3bee16750d4,code,267,128,2026-08-31T11:48:47.335653+00:00,1.2359632499865256,9.590960291970987,54,128,,15.32017298830757,13.345900316900948,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,0,49d569ca-4eef-46fd-88f9-58cf68e01898,short,106,128,2026-08-31T11:48:56.927355+00:00,1.067110000003595,9.073982625035569,20,124,,15.486695718417879,13.665443843575128,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,1,a34ede89-ca43-4a1c-a7c8-c916bfec24a5,medium,252,128,2026-08-31T11:48:56.927278+00:00,1.0672226250171661,9.253953957988415,43,128,,15.635055652124883,13.831925313341854,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,2,af0dfbe9-f1a5-434a-a714-98fad2af83a3,code,267,128,2026-08-31T11:48:56.927205+00:00,1.067305459000636,9.254084500018507,54,128,,15.634964539617723,13.831730194352993,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,3,80dc0643-6f40-4ab5-805d-ace4e9f149bb,short,106,128,2026-08-31T11:48:56.927082+00:00,1.0054587500053458,9.254091374983545,20,128,,15.517723460297553,13.831719918610334,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,0,12e42bf2-6a75-4f89-a598-ac14d5b67a78,medium,252,128,2026-08-31T11:49:06.181834+00:00,1.2329518750193529,9.273020750028081,43,128,,15.920261628338482,13.803484695061464,9.273481999989599,55.21119251652985, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,1,5f2cf183-3654-45be-81ed-4eebfb57bce1,code,267,128,2026-08-31T11:49:06.182126+00:00,1.1721314580063336,9.272628374979831,54,128,,15.801499748959015,13.804068795141205,9.273481999989599,55.21119251652985, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,2,746da721-ed00-4c80-bc92-ed115313463e,short,106,128,2026-08-31T11:49:06.182026+00:00,1.2329447079682723,9.272807125002146,20,128,,15.920670449385964,13.80380269690667,9.273481999989599,55.21119251652985, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,3,5c45d61d-9d4a-4fc4-931d-5193cdcaa598,medium,252,128,2026-08-31T11:49:06.181940+00:00,1.2330110420007259,9.272864667000249,43,128,,15.920687859538933,13.803717038545729,9.273481999989599,55.21119251652985, diff --git a/benchmarks/inference/results/20260831T114727Z/raw.json b/benchmarks/inference/results/20260831T114727Z/raw.json new file mode 100644 index 0000000..1d28313 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "8b712536-91f2-4ded-b334-6db253435ba3", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:32.567050+00:00", + "ttft_s": 0.561656582984142, + "total_s": 5.602756957989186, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.39128175956463, + "request_tps": 22.845895504619364, + "group_elapsed_s": 5.6035958750289865, + "group_output_tps": 22.842475234589944, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "be5a9e94-abd6-4cbc-8029-7a55e3ad129c", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:38.170087+00:00", + "ttft_s": 0.3621360409888439, + "total_s": 5.426666249986738, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.273815086064428, + "request_tps": 23.587225398339694, + "group_elapsed_s": 5.426845958980266, + "group_output_tps": 23.586444311762243, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "e0ac985d-818e-45ec-a2c1-7e077fe0f1cd", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:43.597014+00:00", + "ttft_s": 0.21451262495247647, + "total_s": 5.262066499970388, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.358817987761604, + "request_tps": 24.325044162919706, + "group_elapsed_s": 5.262241083022673, + "group_output_tps": 24.324237141654443, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "428bad62-8fbf-41a1-9b36-07912506da11", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:48.859322+00:00", + "ttft_s": 0.37227433401858434, + "total_s": 5.454897999996319, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.18384370198632, + "request_tps": 23.465150035818517, + "group_elapsed_s": 5.455072959011886, + "group_output_tps": 23.464397444683396, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "b2de9565-6ff0-47a1-91e6-847b83994f6c", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:54.314442+00:00", + "ttft_s": 0.40532424999400973, + "total_s": 5.533015583991073, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 24.962500989743333, + "request_tps": 23.13385857259254, + "group_elapsed_s": 5.533179917023517, + "group_output_tps": 23.13317150707355, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "dc02daab-d1f2-49ac-8e29-01ee86148e0a", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:59.847681+00:00", + "ttft_s": 0.25306625000666827, + "total_s": 5.405130041996017, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 24.844412873734196, + "request_tps": 23.681206373479203, + "group_elapsed_s": 5.405301624967251, + "group_output_tps": 23.680454650072466, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "677f2a25-31e5-43e7-b8fd-5e607453b904", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:05.253048+00:00", + "ttft_s": 0.42382929200539365, + "total_s": 5.569426125031896, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 24.875637200809187, + "request_tps": 22.982619236962577, + "group_elapsed_s": 5.569605667027645, + "group_output_tps": 22.981878368475286, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "7109b7b9-3690-49d6-8b20-969734c3dcad", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.822944+00:00", + "ttft_s": 1.1026008339831606, + "total_s": 8.694552083965391, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.859960738064487, + "request_tps": 14.721862467884838, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "b34a5a99-e633-4f3a-892d-67af2b425779", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.823149+00:00", + "ttft_s": 1.1023455830290914, + "total_s": 8.694302666990552, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.859947782161328, + "request_tps": 14.722284799904022, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "1260b39f-a4b1-4d94-9406-70028f614ce0", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.823017+00:00", + "ttft_s": 1.0163157919887453, + "total_s": 8.694556291971821, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.670485901070972, + "request_tps": 14.721855342772315, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "b43d525a-50de-4e41-b00c-d1159d1ddfd4", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.822845+00:00", + "ttft_s": 1.1026693329913542, + "total_s": 8.694680040993262, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.85982869664412, + "request_tps": 14.721645810600473, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "1da340a9-b91e-4f9b-8217-2af7c2fbc030", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518058+00:00", + "ttft_s": 1.245261833013501, + "total_s": 9.168966500030365, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.154059922603068, + "request_tps": 13.960133892906699, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "23e076e6-7b50-4d08-a3ff-f79ddc53bafe", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518183+00:00", + "ttft_s": 1.2451274580089375, + "total_s": 9.168911040993407, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.153899038190183, + "request_tps": 13.96021833211415, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "c777d58b-0520-4f3f-8537-6a163ab350f5", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518319+00:00", + "ttft_s": 1.2449699160060845, + "total_s": 9.168749333010055, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.153907531211612, + "request_tps": 13.960464546583719, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "5609655c-e54a-499f-99c6-c77c19c1a3e0", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518407+00:00", + "ttft_s": 1.174761500034947, + "total_s": 9.16857587499544, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.012380822970083, + "request_tps": 13.960728661152478, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "d2615398-7368-4f9a-9424-7a2e76d8955d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687706+00:00", + "ttft_s": 1.0508670000126585, + "total_s": 9.266459833015688, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.580129468662143, + "request_tps": 13.813257954666332, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "b307dcc2-b325-4005-a63f-54123aa28e31", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687614+00:00", + "ttft_s": 1.0509756249957718, + "total_s": 9.266665957984515, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.579944570943383, + "request_tps": 13.812950696653772, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "09c7607f-96f5-4a6d-a9e1-3637c8afc257", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687778+00:00", + "ttft_s": 1.0507545830332674, + "total_s": 9.266332915984094, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.580156966749492, + "request_tps": 13.813447149001583, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "0328d6c0-e3f4-4a86-917d-dbeb5b8deee7", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687501+00:00", + "ttft_s": 0.9898037080420181, + "total_s": 9.266721500025596, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.464693889308833, + "request_tps": 13.81286790583341, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "74110182-284c-4250-9395-e3d0f2772eef", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954714+00:00", + "ttft_s": 1.1083811249700375, + "total_s": 9.379890666983556, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.474805336299134, + "request_tps": 13.646214496992963, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "fca02813-8ad6-4e99-bc76-9e922ea9a63a", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954992+00:00", + "ttft_s": 1.1690869170124643, + "total_s": 9.379730832995847, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.589520299476966, + "request_tps": 13.646447033397155, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "5eb13719-9063-4ddd-92c8-73332709ecfd", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954812+00:00", + "ttft_s": 1.1692542500095442, + "total_s": 9.37985170801403, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.589608509574806, + "request_tps": 13.646271176189105, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "9d9a5b23-6950-4d5c-bad2-72f896d38009", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954917+00:00", + "ttft_s": 1.1691360420081764, + "total_s": 9.379842250025831, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.58940202671112, + "request_tps": 13.646284936150979, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "caf79143-9d5b-4fb2-830e-3bd2d38cc5d3", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335341+00:00", + "ttft_s": 1.172440624970477, + "total_s": 9.591207291989122, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.204127286417465, + "request_tps": 13.345556623190662, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "d77a5e81-922c-49ce-96c5-1fb443746984", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335566+00:00", + "ttft_s": 1.2360592089826241, + "total_s": 9.591093584022019, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.320104532711328, + "request_tps": 13.345714842490702, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "9ccc94c9-d569-4785-9f48-e48e760b0e15", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335447+00:00", + "ttft_s": 1.236121624999214, + "total_s": 9.591066916007549, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.320267882275031, + "request_tps": 13.345751950324445, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "e7c5be37-35a8-47a5-869d-a3bee16750d4", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335653+00:00", + "ttft_s": 1.2359632499865256, + "total_s": 9.590960291970987, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.32017298830757, + "request_tps": 13.345900316900948, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "49d569ca-4eef-46fd-88f9-58cf68e01898", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927355+00:00", + "ttft_s": 1.067110000003595, + "total_s": 9.073982625035569, + "prompt_tokens": 20, + "output_tokens": 124, + "prompt_tps": null, + "decode_tps": 15.486695718417879, + "request_tps": 13.665443843575128, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "a34ede89-ca43-4a1c-a7c8-c916bfec24a5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927278+00:00", + "ttft_s": 1.0672226250171661, + "total_s": 9.253953957988415, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.635055652124883, + "request_tps": 13.831925313341854, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "af0dfbe9-f1a5-434a-a714-98fad2af83a3", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927205+00:00", + "ttft_s": 1.067305459000636, + "total_s": 9.254084500018507, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.634964539617723, + "request_tps": 13.831730194352993, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "80dc0643-6f40-4ab5-805d-ace4e9f149bb", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927082+00:00", + "ttft_s": 1.0054587500053458, + "total_s": 9.254091374983545, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.517723460297553, + "request_tps": 13.831719918610334, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "12e42bf2-6a75-4f89-a598-ac14d5b67a78", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.181834+00:00", + "ttft_s": 1.2329518750193529, + "total_s": 9.273020750028081, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.920261628338482, + "request_tps": 13.803484695061464, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "5f2cf183-3654-45be-81ed-4eebfb57bce1", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.182126+00:00", + "ttft_s": 1.1721314580063336, + "total_s": 9.272628374979831, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.801499748959015, + "request_tps": 13.804068795141205, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "746da721-ed00-4c80-bc92-ed115313463e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.182026+00:00", + "ttft_s": 1.2329447079682723, + "total_s": 9.272807125002146, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.920670449385964, + "request_tps": 13.80380269690667, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "5c45d61d-9d4a-4fc4-931d-5193cdcaa598", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.181940+00:00", + "ttft_s": 1.2330110420007259, + "total_s": 9.272864667000249, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.920687859538933, + "request_tps": 13.803717038545729, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114727Z/summary.json b/benchmarks/inference/results/20260831T114727Z/summary.json new file mode 100644 index 0000000..47cf7b8 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.3338013501954265, + "ttft_s_p50": 0.37227433401858434, + "ttft_s_p95": 0.4201282836031169, + "total_s_mean": 5.444907250197138, + "total_s_p50": 5.454897999996319, + "total_s_p95": 5.562144016823732, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 25.04504255080693, + "decode_tps_p50": 24.962500989743333, + "decode_tps_p95": 25.32382313060655, + "request_tps_mean": 23.517575676354507, + "request_tps_p50": 23.465150035818517, + "request_tps_p95": 24.196276605031606, + "aggregate_output_tps_mean": 23.516827822391825, + "aggregate_output_tps_p50": 23.464397444683396, + "aggregate_output_tps_p95": 24.195480643338048 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.1358489938022103, + "ttft_s_p50": 1.1691114795103204, + "ttft_s_p95": 1.2360623297834537, + "total_s_mean": 9.343862856202758, + "total_s_p50": 9.272835896001197, + "total_s_p95": 9.591099269420374, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.572524640705884, + "decode_tps_p50": 15.580143217705817, + "decode_tps_p95": 15.920671319893613, + "request_tps_mean": 13.679827878866371, + "request_tps_p50": 13.803600866803595, + "request_tps_p95": 13.831739950302437, + "aggregate_output_tps_mean": 54.66256367620249, + "aggregate_output_tps_p50": 54.89183177619698, + "aggregate_output_tps_p95": 55.24148726503248 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114928Z/metadata.json b/benchmarks/inference/results/20260831T114928Z/metadata.json new file mode 100644 index 0000000..68b26b1 --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/metadata.json @@ -0,0 +1,135 @@ +{ + "run_id": "20260831T114928Z", + "created_at": "2026-08-31T11:49:28.271574+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "ollama", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mlx" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 13:49:28 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 13:49:28 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 13:49:28 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 13:49:30 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 13:49:31 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "ollama": { + "license": "\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n", + "modelfile": "# Modelfile generated by \"ollama show\"\nFROM qwen3.8:27b-mlx\nTEMPLATE {{ .Prompt }}\nRENDERER qwen3.8\nPARSER qwen3.5\nPARAMETER min_p 0\nPARAMETER presence_penalty 0\nPARAMETER repeat_penalty 1\nPARAMETER temperature 1\nPARAMETER top_k 20\nPARAMETER top_p 0.95\nLICENSE \"\"\"\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\"\"\"\n", + "parameters": "top_k 20\ntop_p 0.95\nmin_p 0\npresence_penalty 0\nrepeat_penalty 1\ntemperature 1", + "template": "{{ .Prompt }}", + "details": { + "parent_model": "", + "format": "safetensors", + "family": "qwen3_5", + "families": null, + "parameter_size": "27.8B", + "quantization_level": "nvfp4" + }, + "model_info": { + "general.architecture": "qwen3_5", + "general.parameter_count": 27781081984, + "qwen3_5.block_count": 64, + "qwen3_5.context_length": 262144, + "qwen3_5.embedding_length": 5120 + }, + "capabilities": [ + "completion", + "vision", + "tools", + "thinking" + ], + "modified_at": "2026-08-20T17:42:35.895756165+02:00", + "requires": "0.32.12" + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T11:51:31.347428+00:00", + "summary": [ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.18287834140937775, + "ttft_s_p50": 0.11936691601295024, + "ttft_s_p95": 0.3934195415815338, + "total_s_mean": 2.769790649821516, + "total_s_p50": 2.7544727920321748, + "total_s_p95": 3.450829833233729, + "prompt_tps_mean": 1201063.1140147406, + "prompt_tps_p50": 816.5668113066018, + "prompt_tps_p95": 3147950.5530253737, + "decode_tps_mean": 44.301349906835995, + "decode_tps_p50": 45.72630609321048, + "decode_tps_p95": 52.87492274071074, + "request_tps_mean": 42.02276635113332, + "request_tps_p50": 44.19526212378514, + "request_tps_p95": 46.37707708697445, + "aggregate_output_tps_mean": 42.01338924160581, + "aggregate_output_tps_p50": 44.18819214792146, + "aggregate_output_tps_p95": 46.368391249450006 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 5.296346539442311, + "ttft_s_p50": 5.078280103509314, + "ttft_s_p95": 10.691154320855276, + "total_s_mean": 8.731022270751419, + "total_s_p50": 8.619212541496381, + "total_s_p95": 14.520918775268365, + "prompt_tps_mean": 4873110.971654413, + "prompt_tps_p50": 3843030.8359358422, + "prompt_tps_p95": 9861582.229537027, + "decode_tps_mean": 33.61516109813652, + "decode_tps_p50": 34.997484039567254, + "decode_tps_p95": 40.1951363429803, + "request_tps_mean": 18.258369295759792, + "request_tps_p50": 12.924863350205007, + "request_tps_p95": 36.94528376333951, + "aggregate_output_tps_mean": 33.27170984729996, + "aggregate_output_tps_p50": 33.918007597024484, + "aggregate_output_tps_p95": 35.45560236969114 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114928Z/raw.csv b/benchmarks/inference/results/20260831T114928Z/raw.csv new file mode 100644 index 0000000..ed4b33d --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,1,0,0,0bd09e14-c217-44ad-b0a1-98600e1bf40e,short,106,128,2026-08-31T11:49:33.251646+00:00,3.627692041976843,5.577126249962021,20,96,116.69505030650684,46.54721674973356,17.213166010121025,5.578097124991473,17.210170036282175, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,1,1,0,ada11e2f-0b97-42ec-a5be-5175f0bb29b1,medium,252,128,2026-08-31T11:49:38.829357+00:00,0.3621791250188835,2.7543381249997765,43,128,157.23847751774449,52.253166030964316,46.472144737135494,2.7547111669555306,46.46585149667936, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,0,0,a3bd0047-d992-4553-9b17-4472b6290acc,short,106,128,2026-08-31T11:49:41.584102+00:00,0.11936691601295024,2.1721785410190932,20,96,448.5805263612496,45.72630609321048,44.19526212378514,2.172526082955301,44.18819214792146, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,1,0,a8033a77-d60f-42b4-a2a9-8f6a97ef3496,medium,252,128,2026-08-31T11:49:43.757773+00:00,0.14179470797535032,2.78225216595456,43,128,816.5668113066018,47.377908815894095,46.00589463683088,2.7835853330325335,45.98386062788756, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,2,0,9cc954fc-95f5-474e-b219-8a89d7091934,code,267,128,2026-08-31T11:49:46.540349+00:00,0.45632574998307973,2.7544727920321748,54,128,146.71421293937848,54.2491762219149,46.46987269951035,2.754789875005372,46.46452390484062, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,3,0,e8ee5060-7c14-4c34-bd5b-76d7e51fef07,short,106,128,2026-08-31T11:49:49.295554+00:00,0.0983880830463022,2.5220755000482313,20,96,2758620.689655172,38.503422116977305,38.063888253212134,2.522709292010404,38.054325285929174, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,4,0,edd132c8-7aaa-4eb0-98e8-2b2d711f2cf9,medium,252,128,2026-08-31T11:49:51.817976+00:00,0.09851625002920628,3.6179742500535212,43,128,3245283.0188679243,35.649936286183205,35.3789140423281,3.6182677499600686,35.37604424145024, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,0,f0df217a-d72c-4b5c-b4a7-87c267716da0,short,106,128,2026-08-31T11:49:55.436979+00:00,0.11441224999725819,2.9100067079998553,20,96,4138216.428719222,33.47564745836445,32.989614675487815,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,1,9565d3c0-e753-4a7b-be43-115556beec5b,medium,252,128,2026-08-31T11:49:55.437104+00:00,6.277438874996733,9.582254499953706,43,128,3865168.539325843,37.881971614565146,13.358025504396528,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,2,42b7e290-9d92-4647-bbdb-3b3ba99f4c3e,code,267,128,2026-08-31T11:49:55.436744+00:00,3.0367828750167973,6.193101375014521,54,128,771.3321549092078,39.94659431201623,20.66815836672783,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,3,c22aaea0-7394-47e8-b6ab-25444fc113d1,short,106,128,2026-08-31T11:49:55.437235+00:00,9.661167457990814,12.522873707988765,20,96,3966679.888932963,32.71048444655768,7.665972063485584,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,0,84f401af-117d-4956-b939-fdbb1e1ceecf,medium,252,128,2026-08-31T11:50:07.961234+00:00,3.091476584027987,6.373854791978374,43,128,5375000.0,38.1244834091544,20.08203891953902,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,1,b3c9f50e-fcf6-4ad6-b00f-9de925c9092a,code,267,128,2026-08-31T11:50:07.961465+00:00,9.849772959016263,12.85736475000158,54,128,11076923.076923078,41.56311628897905,9.955383742223248,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,2,5f528a81-943d-445a-a2f2-9b986c01c96e,short,106,128,2026-08-31T11:50:07.961041+00:00,0.11284091597190127,3.010363540961407,20,96,1476995.7905619969,32.336156802028626,31.889836125686298,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,3,45991adb-aae4-4122-bb0d-21d34b549b46,medium,252,128,2026-08-31T11:50:07.960837+00:00,6.452098417037632,9.774627042002976,43,128,7319148.936170213,37.70880560635668,13.095128791100224,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,0,1cd3b269-0ed7-4470-9bf5-9aba6dbdda3b,short,106,128,2026-08-31T11:50:20.819745+00:00,9.799659290991258,12.57556495803874,20,96,2461538.461538462,33.69646737394496,7.633851864335801,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,1,b2258258-f465-4131-bb56-cdead87e67c5,medium,252,128,2026-08-31T11:50:20.820473+00:00,2.9737471669795923,6.33726870897226,43,128,4755585.04755585,37.255703188397476,20.19797579654126,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,2,9bb209a2-19a6-4172-a166-c29a75683a37,code,267,128,2026-08-31T11:50:20.819981+00:00,0.11236062500393018,2.89358854200691,54,128,3661016.9491525427,44.88156766233007,44.23572949014475,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,3,c74fbeef-16a3-40b2-ade6-0b24b3ee0147,short,106,128,2026-08-31T11:50:20.820727+00:00,6.417795290995855,9.71814520796761,20,98,2608582.235554976,29.056692998778793,10.084228821735737,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,0,844d6c49-9989-400d-bb5d-31f8490da904,medium,252,128,2026-08-31T11:50:33.396890+00:00,10.48226895800326,14.052842833043542,43,128,5083944.194845118,35.11947732750843,9.108477303896379,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,1,213efda7-5da6-47a7-9517-f63ad082e62e,code,267,128,2026-08-31T11:50:33.397263+00:00,0.11672520800493658,3.508996458025649,54,128,3630252.100840336,36.93011583511385,36.47766577456727,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,2,d03d55d9-a506-44a9-887e-e0ab77392c8d,short,106,128,2026-08-31T11:50:33.397083+00:00,3.606067124987021,6.893670374993235,20,96,3356267.830172848,28.527554720530254,13.925818145909567,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,3,55be3260-b3a0-422a-b918-a81261da6a84,medium,252,128,2026-08-31T11:50:33.396391+00:00,6.9750645000021905,10.400937125028577,43,128,6745098.039215687,36.57719818551596,12.30658338391295,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,0,930b484d-3177-4ee9-9d72-5442ee8cd7c9,code,267,128,2026-08-31T11:50:47.451672+00:00,0.11444733297685161,3.540263083006721,54,128,4248957.431741286,36.58630699945727,36.15550511327833,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,1,231f3cbb-61f2-4042-8a99-def6a8698a7d,short,106,128,2026-08-31T11:50:47.451387+00:00,10.67886250000447,14.209862457995769,20,98,2840102.243680773,27.132075592063398,6.896618478165229,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,2,32437705-bf67-438f-bad7-0bb10d0e2030,medium,252,128,2026-08-31T11:50:47.451856+00:00,3.622358249966055,7.2201729999505915,43,128,4777777.777777778,34.87549075162609,17.72810706902396,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,3,a16a724c-155d-4e2d-ba28-3a8683af4d40,code,267,128,2026-08-31T11:50:47.451136+00:00,7.304396207968239,10.594903749995865,54,128,17515407.071034707,38.04511611184116,12.081280115456448,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,0,f6ab47e7-5bdf-4b5c-be3f-0e639727a2bb,short,106,128,2026-08-31T11:51:01.661760+00:00,10.42098737496417,14.485269291966688,20,96,2105263.1578947366,23.198198961620538,6.627422525947802,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,1,8e015249-8b0c-4560-b93d-603afb968f5b,medium,252,128,2026-08-31T11:51:01.662046+00:00,0.11227079102536663,3.6567685829941183,43,128,3571131.9657835728,35.39152606628793,35.00358228717748,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,2,d2848890-f070-45de-8a2f-4bb2b8788a87,code,267,128,2026-08-31T11:51:01.661862+00:00,7.207546124991495,10.33818608301226,54,128,9458749.343142407,39.94848206301452,12.381282264819164,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,3,aba512eb-9825-4e49-9ab8-0c75678ff654,short,106,128,2026-08-31T11:51:01.661952+00:00,3.738764916022774,7.12777191598434,20,96,2652168.1474605487,27.731643965337334,13.468444435590849,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,0,4c24ae09-884b-4b94-ae8e-37346f1408f2,medium,252,128,2026-08-31T11:51:16.148220+00:00,10.924698917020578,15.198258958000224,43,128,7068880.486602006,29.434480556585335,8.422017308280036,15.199551291007083,31.579879616840746, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,1,c94ceb91-b83d-4d47-a36d-71741993f366,code,267,128,2026-08-31T11:51:16.148464+00:00,0.12565479200566188,3.500943167018704,54,128,4025044.7227191413,37.03108618648992,36.56157609350765,15.199551291007083,31.579879616840746, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,2,0672967f-d135-44b0-b89f-7995f59b2d8c,short,106,128,2026-08-31T11:51:16.148378+00:00,7.600669999956153,10.846751042001415,20,96,2666666.6666666665,28.898123213254255,8.850576511645125,15.199551291007083,31.579879616840746, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,3,8185344b-c228-457a-8c3d-6a7b5f803594,medium,252,128,2026-08-31T11:51:16.147977+00:00,3.5925854169763625,7.520279875025153,43,128,4229785.559708833,31.98591420303284,17.02064313126004,15.199551291007083,31.579879616840746, diff --git a/benchmarks/inference/results/20260831T114928Z/raw.json b/benchmarks/inference/results/20260831T114928Z/raw.json new file mode 100644 index 0000000..705b8ad --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "0bd09e14-c217-44ad-b0a1-98600e1bf40e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:33.251646+00:00", + "ttft_s": 3.627692041976843, + "total_s": 5.577126249962021, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 116.69505030650684, + "decode_tps": 46.54721674973356, + "request_tps": 17.213166010121025, + "group_elapsed_s": 5.578097124991473, + "group_output_tps": 17.210170036282175, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "ada11e2f-0b97-42ec-a5be-5175f0bb29b1", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:38.829357+00:00", + "ttft_s": 0.3621791250188835, + "total_s": 2.7543381249997765, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 157.23847751774449, + "decode_tps": 52.253166030964316, + "request_tps": 46.472144737135494, + "group_elapsed_s": 2.7547111669555306, + "group_output_tps": 46.46585149667936, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "a3bd0047-d992-4553-9b17-4472b6290acc", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:41.584102+00:00", + "ttft_s": 0.11936691601295024, + "total_s": 2.1721785410190932, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 448.5805263612496, + "decode_tps": 45.72630609321048, + "request_tps": 44.19526212378514, + "group_elapsed_s": 2.172526082955301, + "group_output_tps": 44.18819214792146, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "a8033a77-d60f-42b4-a2a9-8f6a97ef3496", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:43.757773+00:00", + "ttft_s": 0.14179470797535032, + "total_s": 2.78225216595456, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 816.5668113066018, + "decode_tps": 47.377908815894095, + "request_tps": 46.00589463683088, + "group_elapsed_s": 2.7835853330325335, + "group_output_tps": 45.98386062788756, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "9cc954fc-95f5-474e-b219-8a89d7091934", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:46.540349+00:00", + "ttft_s": 0.45632574998307973, + "total_s": 2.7544727920321748, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 146.71421293937848, + "decode_tps": 54.2491762219149, + "request_tps": 46.46987269951035, + "group_elapsed_s": 2.754789875005372, + "group_output_tps": 46.46452390484062, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "e8ee5060-7c14-4c34-bd5b-76d7e51fef07", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:49.295554+00:00", + "ttft_s": 0.0983880830463022, + "total_s": 2.5220755000482313, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2758620.689655172, + "decode_tps": 38.503422116977305, + "request_tps": 38.063888253212134, + "group_elapsed_s": 2.522709292010404, + "group_output_tps": 38.054325285929174, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "edd132c8-7aaa-4eb0-98e8-2b2d711f2cf9", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:51.817976+00:00", + "ttft_s": 0.09851625002920628, + "total_s": 3.6179742500535212, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3245283.0188679243, + "decode_tps": 35.649936286183205, + "request_tps": 35.3789140423281, + "group_elapsed_s": 3.6182677499600686, + "group_output_tps": 35.37604424145024, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "f0df217a-d72c-4b5c-b4a7-87c267716da0", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.436979+00:00", + "ttft_s": 0.11441224999725819, + "total_s": 2.9100067079998553, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 4138216.428719222, + "decode_tps": 33.47564745836445, + "request_tps": 32.989614675487815, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "9565d3c0-e753-4a7b-be43-115556beec5b", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.437104+00:00", + "ttft_s": 6.277438874996733, + "total_s": 9.582254499953706, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3865168.539325843, + "decode_tps": 37.881971614565146, + "request_tps": 13.358025504396528, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "42b7e290-9d92-4647-bbdb-3b3ba99f4c3e", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.436744+00:00", + "ttft_s": 3.0367828750167973, + "total_s": 6.193101375014521, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 771.3321549092078, + "decode_tps": 39.94659431201623, + "request_tps": 20.66815836672783, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "c22aaea0-7394-47e8-b6ab-25444fc113d1", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.437235+00:00", + "ttft_s": 9.661167457990814, + "total_s": 12.522873707988765, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 3966679.888932963, + "decode_tps": 32.71048444655768, + "request_tps": 7.665972063485584, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "84f401af-117d-4956-b939-fdbb1e1ceecf", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.961234+00:00", + "ttft_s": 3.091476584027987, + "total_s": 6.373854791978374, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 5375000.0, + "decode_tps": 38.1244834091544, + "request_tps": 20.08203891953902, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "b3c9f50e-fcf6-4ad6-b00f-9de925c9092a", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.961465+00:00", + "ttft_s": 9.849772959016263, + "total_s": 12.85736475000158, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 11076923.076923078, + "decode_tps": 41.56311628897905, + "request_tps": 9.955383742223248, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "5f528a81-943d-445a-a2f2-9b986c01c96e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.961041+00:00", + "ttft_s": 0.11284091597190127, + "total_s": 3.010363540961407, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 1476995.7905619969, + "decode_tps": 32.336156802028626, + "request_tps": 31.889836125686298, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "45991adb-aae4-4122-bb0d-21d34b549b46", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.960837+00:00", + "ttft_s": 6.452098417037632, + "total_s": 9.774627042002976, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7319148.936170213, + "decode_tps": 37.70880560635668, + "request_tps": 13.095128791100224, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "1cd3b269-0ed7-4470-9bf5-9aba6dbdda3b", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.819745+00:00", + "ttft_s": 9.799659290991258, + "total_s": 12.57556495803874, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2461538.461538462, + "decode_tps": 33.69646737394496, + "request_tps": 7.633851864335801, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "b2258258-f465-4131-bb56-cdead87e67c5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.820473+00:00", + "ttft_s": 2.9737471669795923, + "total_s": 6.33726870897226, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4755585.04755585, + "decode_tps": 37.255703188397476, + "request_tps": 20.19797579654126, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "9bb209a2-19a6-4172-a166-c29a75683a37", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.819981+00:00", + "ttft_s": 0.11236062500393018, + "total_s": 2.89358854200691, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 3661016.9491525427, + "decode_tps": 44.88156766233007, + "request_tps": 44.23572949014475, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "c74fbeef-16a3-40b2-ade6-0b24b3ee0147", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.820727+00:00", + "ttft_s": 6.417795290995855, + "total_s": 9.71814520796761, + "prompt_tokens": 20, + "output_tokens": 98, + "prompt_tps": 2608582.235554976, + "decode_tps": 29.056692998778793, + "request_tps": 10.084228821735737, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "844d6c49-9989-400d-bb5d-31f8490da904", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.396890+00:00", + "ttft_s": 10.48226895800326, + "total_s": 14.052842833043542, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 5083944.194845118, + "decode_tps": 35.11947732750843, + "request_tps": 9.108477303896379, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "213efda7-5da6-47a7-9517-f63ad082e62e", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.397263+00:00", + "ttft_s": 0.11672520800493658, + "total_s": 3.508996458025649, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 3630252.100840336, + "decode_tps": 36.93011583511385, + "request_tps": 36.47766577456727, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "d03d55d9-a506-44a9-887e-e0ab77392c8d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.397083+00:00", + "ttft_s": 3.606067124987021, + "total_s": 6.893670374993235, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 3356267.830172848, + "decode_tps": 28.527554720530254, + "request_tps": 13.925818145909567, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "55be3260-b3a0-422a-b918-a81261da6a84", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.396391+00:00", + "ttft_s": 6.9750645000021905, + "total_s": 10.400937125028577, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 6745098.039215687, + "decode_tps": 36.57719818551596, + "request_tps": 12.30658338391295, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "930b484d-3177-4ee9-9d72-5442ee8cd7c9", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451672+00:00", + "ttft_s": 0.11444733297685161, + "total_s": 3.540263083006721, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 4248957.431741286, + "decode_tps": 36.58630699945727, + "request_tps": 36.15550511327833, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "231f3cbb-61f2-4042-8a99-def6a8698a7d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451387+00:00", + "ttft_s": 10.67886250000447, + "total_s": 14.209862457995769, + "prompt_tokens": 20, + "output_tokens": 98, + "prompt_tps": 2840102.243680773, + "decode_tps": 27.132075592063398, + "request_tps": 6.896618478165229, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "32437705-bf67-438f-bad7-0bb10d0e2030", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451856+00:00", + "ttft_s": 3.622358249966055, + "total_s": 7.2201729999505915, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4777777.777777778, + "decode_tps": 34.87549075162609, + "request_tps": 17.72810706902396, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "a16a724c-155d-4e2d-ba28-3a8683af4d40", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451136+00:00", + "ttft_s": 7.304396207968239, + "total_s": 10.594903749995865, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 17515407.071034707, + "decode_tps": 38.04511611184116, + "request_tps": 12.081280115456448, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "f6ab47e7-5bdf-4b5c-be3f-0e639727a2bb", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.661760+00:00", + "ttft_s": 10.42098737496417, + "total_s": 14.485269291966688, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2105263.1578947366, + "decode_tps": 23.198198961620538, + "request_tps": 6.627422525947802, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "8e015249-8b0c-4560-b93d-603afb968f5b", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.662046+00:00", + "ttft_s": 0.11227079102536663, + "total_s": 3.6567685829941183, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3571131.9657835728, + "decode_tps": 35.39152606628793, + "request_tps": 35.00358228717748, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "d2848890-f070-45de-8a2f-4bb2b8788a87", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.661862+00:00", + "ttft_s": 7.207546124991495, + "total_s": 10.33818608301226, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 9458749.343142407, + "decode_tps": 39.94848206301452, + "request_tps": 12.381282264819164, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "aba512eb-9825-4e49-9ab8-0c75678ff654", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.661952+00:00", + "ttft_s": 3.738764916022774, + "total_s": 7.12777191598434, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2652168.1474605487, + "decode_tps": 27.731643965337334, + "request_tps": 13.468444435590849, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "4c24ae09-884b-4b94-ae8e-37346f1408f2", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.148220+00:00", + "ttft_s": 10.924698917020578, + "total_s": 15.198258958000224, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7068880.486602006, + "decode_tps": 29.434480556585335, + "request_tps": 8.422017308280036, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "c94ceb91-b83d-4d47-a36d-71741993f366", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.148464+00:00", + "ttft_s": 0.12565479200566188, + "total_s": 3.500943167018704, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 4025044.7227191413, + "decode_tps": 37.03108618648992, + "request_tps": 36.56157609350765, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "0672967f-d135-44b0-b89f-7995f59b2d8c", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.148378+00:00", + "ttft_s": 7.600669999956153, + "total_s": 10.846751042001415, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2666666.6666666665, + "decode_tps": 28.898123213254255, + "request_tps": 8.850576511645125, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "8185344b-c228-457a-8c3d-6a7b5f803594", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.147977+00:00", + "ttft_s": 3.5925854169763625, + "total_s": 7.520279875025153, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4229785.559708833, + "decode_tps": 31.98591420303284, + "request_tps": 17.02064313126004, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114928Z/summary.json b/benchmarks/inference/results/20260831T114928Z/summary.json new file mode 100644 index 0000000..157f7c4 --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.18287834140937775, + "ttft_s_p50": 0.11936691601295024, + "ttft_s_p95": 0.3934195415815338, + "total_s_mean": 2.769790649821516, + "total_s_p50": 2.7544727920321748, + "total_s_p95": 3.450829833233729, + "prompt_tps_mean": 1201063.1140147406, + "prompt_tps_p50": 816.5668113066018, + "prompt_tps_p95": 3147950.5530253737, + "decode_tps_mean": 44.301349906835995, + "decode_tps_p50": 45.72630609321048, + "decode_tps_p95": 52.87492274071074, + "request_tps_mean": 42.02276635113332, + "request_tps_p50": 44.19526212378514, + "request_tps_p95": 46.37707708697445, + "aggregate_output_tps_mean": 42.01338924160581, + "aggregate_output_tps_p50": 44.18819214792146, + "aggregate_output_tps_p95": 46.368391249450006 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 5.296346539442311, + "ttft_s_p50": 5.078280103509314, + "ttft_s_p95": 10.691154320855276, + "total_s_mean": 8.731022270751419, + "total_s_p50": 8.619212541496381, + "total_s_p95": 14.520918775268365, + "prompt_tps_mean": 4873110.971654413, + "prompt_tps_p50": 3843030.8359358422, + "prompt_tps_p95": 9861582.229537027, + "decode_tps_mean": 33.61516109813652, + "decode_tps_p50": 34.997484039567254, + "decode_tps_p95": 40.1951363429803, + "request_tps_mean": 18.258369295759792, + "request_tps_p50": 12.924863350205007, + "request_tps_p95": 36.94528376333951, + "aggregate_output_tps_mean": 33.27170984729996, + "aggregate_output_tps_p50": 33.918007597024484, + "aggregate_output_tps_p95": 35.45560236969114 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T121855Z/metadata.json b/benchmarks/inference/results/20260831T121855Z/metadata.json new file mode 100644 index 0000000..92f8df9 --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/metadata.json @@ -0,0 +1,135 @@ +{ + "run_id": "20260831T121855Z", + "created_at": "2026-08-31T12:18:55.849116+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "ollama", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mxfp8", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mxfp8" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 14:18:56 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 14:18:56 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 14:18:56 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 14:18:58 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 14:18:59 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "ollama": { + "license": "\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n", + "modelfile": "# Modelfile generated by \"ollama show\"\nFROM qwen3.8:27b-mxfp8\nTEMPLATE {{ .Prompt }}\nRENDERER qwen3.8\nPARSER qwen3.5\nPARAMETER min_p 0\nPARAMETER presence_penalty 0\nPARAMETER repeat_penalty 1\nPARAMETER temperature 1\nPARAMETER top_k 20\nPARAMETER top_p 0.95\nLICENSE \"\"\"\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\"\"\"\n", + "parameters": "min_p 0\npresence_penalty 0\nrepeat_penalty 1\ntemperature 1\ntop_k 20\ntop_p 0.95", + "template": "{{ .Prompt }}", + "details": { + "parent_model": "", + "format": "safetensors", + "family": "qwen3_5", + "families": null, + "parameter_size": "27.8B", + "quantization_level": "mxfp8" + }, + "model_info": { + "general.architecture": "qwen3_5", + "general.parameter_count": 27781081984, + "qwen3_5.block_count": 64, + "qwen3_5.context_length": 262144, + "qwen3_5.embedding_length": 5120 + }, + "capabilities": [ + "completion", + "vision", + "tools", + "thinking" + ], + "modified_at": "2026-08-31T01:00:34.465875846+02:00", + "requires": "0.32.12" + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T12:22:34.295676+00:00", + "summary": [ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.296576766599901, + "ttft_s_p50": 0.25913220801157877, + "ttft_s_p95": 0.5828672000090591, + "total_s_mean": 5.365511283196975, + "total_s_p50": 5.271547083975747, + "total_s_p95": 5.7278931244160045, + "prompt_tps_mean": 1579505.471393669, + "prompt_tps_p50": 423.79232394227944, + "prompt_tps_p95": 4040897.1304585882, + "decode_tps_mean": 22.074969591997284, + "decode_tps_p50": 22.737809958094928, + "decode_tps_p95": 26.029319910172887, + "request_tps_mean": 21.326224707051917, + "request_tps_p50": 22.22186664913243, + "request_tps_p95": 23.99720340744444, + "aggregate_output_tps_mean": 21.3238226965937, + "aggregate_output_tps_p50": 22.217896609947385, + "aggregate_output_tps_p95": 23.99517949790225 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 9.275126714701765, + "ttft_s_p50": 9.394405500497669, + "ttft_s_p95": 18.583187756550615, + "total_s_mean": 15.170045258343453, + "total_s_p50": 14.90246445898083, + "total_s_p95": 24.08200175591046, + "prompt_tps_mean": 7067890.927047749, + "prompt_tps_p50": 6856543.23965746, + "prompt_tps_p95": 11845450.362447295, + "decode_tps_mean": 19.377795978907, + "decode_tps_p50": 19.039894932933024, + "decode_tps_p95": 22.784262218881533, + "request_tps_mean": 10.048677383124767, + "request_tps_p50": 7.0969869715907645, + "request_tps_p95": 21.739669344814565, + "aggregate_output_tps_mean": 19.35053924595546, + "aggregate_output_tps_p50": 20.179477104341757, + "aggregate_output_tps_p95": 20.562657512957074 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T121855Z/raw.csv b/benchmarks/inference/results/20260831T121855Z/raw.csv new file mode 100644 index 0000000..0656eb5 --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,1,0,0,556d36df-74f6-4dba-866f-134c99f385ac,short,106,128,2026-08-31T12:19:01.706610+00:00,8.133006000018213,13.098198042018339,20,95,80.39981477169073,18.60021913626931,7.252906063509266,13.099143458006438,7.252382593148428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,1,1,0,456b9efb-0afa-48e4-8093-041758c70c7d,medium,252,128,2026-08-31T12:19:14.805394+00:00,0.6389443330117501,6.043248832982499,43,128,91.99219534791703,23.084069916067204,21.180660194134138,6.043762709014118,21.178859290602404, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,0,0,78330ae3-a01a-4bf4-b1b6-e4ae861145a8,short,106,128,2026-08-31T12:19:20.848831+00:00,0.25939199997810647,5.077765540976543,20,95,194.8949698826367,19.20226982827757,18.709016639970706,5.077990833029617,18.708186588694836, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,1,0,f8373947-d690-42a4-b613-33f71746b2f9,medium,252,128,2026-08-31T12:19:25.927037+00:00,0.25913220801157877,5.760092166019604,43,128,423.79232394227944,22.737809958094928,22.22186664913243,5.761121416988317,22.217896609947385, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,2,0,9e12f0f4-50d6-444c-8752-4565aa55084b,code,267,128,2026-08-31T12:19:31.688098+00:00,0.6637360000167973,5.271547083975747,54,128,115.32507388679936,26.793290081008884,24.281296924974765,5.271936749981251,24.27950221528269, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,3,0,583bec1f-0983-4208-8635-439a9a0c5424,short,106,128,2026-08-31T12:19:36.960047+00:00,0.15107908402569592,5.119054667011369,20,95,4102564.1025641025,18.66803886577614,18.558113983858536,5.11973733396735,18.555639440663118, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,4,0,641937dd-533c-493e-b7b3-3fa2cc6e8879,medium,252,128,2026-08-31T12:19:42.079662+00:00,0.1495445409673266,5.59909695800161,43,128,3794229.2420365307,22.9734392268289,22.86082933732315,5.599817292008083,22.85788862838049, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,0,3dbdfe4c-7d43-4575-a1e6-6b74bd5a10a7,short,106,128,2026-08-31T12:19:47.680109+00:00,0.16311579098692164,4.8760897910106,20,95,1791151.7105498835,19.64848684540538,19.482824162741817,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,1,1ea04fa9-7f60-4936-ba53-6829df8e5714,medium,252,128,2026-08-31T12:19:47.680573+00:00,10.314829125010874,16.516400250024162,43,128,6370370.37037037,20.216323480613685,7.749872736331438,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,2,046681e1-e7be-4c58-8284-88f821e4a8a0,code,267,128,2026-08-31T12:19:47.680452+00:00,5.110771542007569,10.15410141699249,54,128,514.3377603686088,24.759396813702583,12.605743703307615,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,3,f496e50f-a2db-4fad-a2e1-2a7965fd862d,short,106,128,2026-08-31T12:19:47.680317+00:00,16.645131999976,22.03015966597013,20,95,6666666.666666666,17.25091779989875,4.312270153300159,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,0,cb3d0e9e-0a6f-4f36-aec9-4b20cbc8f87d,medium,252,128,2026-08-31T12:20:09.710819+00:00,17.595250790996943,23.510470708017237,43,128,19843101.06137517,21.163606353137514,5.444382700357892,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,1,fa3bb29e-7370-4255-877e-376076d7aa76,code,267,128,2026-08-31T12:20:09.711021+00:00,5.7327396660111845,11.353798499971163,54,128,9000000.0,22.18879858004337,11.273760054868431,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,2,93ec5b28-1a29-426a-96ff-b3b525fb3c71,short,106,128,2026-08-31T12:20:09.710927+00:00,0.1655573329771869,5.57563720800681,20,95,1563477.1732332706,17.170644798585553,17.038411298277563,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,3,dcac6e9a-3580-4bb1-8787-9cef1c64bfd5,medium,252,128,2026-08-31T12:20:09.711149+00:00,11.527663582994137,17.459117082995363,43,128,6331909.880724489,21.03978753266432,7.331413117371669,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,0,d42caf5b-fb6a-4e96-8e42-fd0b8fc1945b,short,106,128,2026-08-31T12:20:33.222727+00:00,6.285906166012865,10.864910541044082,20,95,8421052.631578946,20.156958877874334,8.743744335594945,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,1,87483860-d4c6-4f13-928c-48de78b6ef70,medium,252,128,2026-08-31T12:20:33.222336+00:00,0.17068937502335757,6.147177167003974,43,128,4669345.205776958,20.965937740191077,20.822565630133763,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,2,ee57b9cf-06bc-468b-bd1e-f936abf292df,code,267,128,2026-08-31T12:20:33.222603+00:00,16.19913299998734,21.62435016600648,54,128,11675675.675675677,22.963769927247082,5.919253018812849,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,3,13164aa4-836c-4761-8f99-95bfe18d33a1,short,106,128,2026-08-31T12:20:33.222066+00:00,11.00553745898651,16.043328458967153,20,95,10214504.596527068,18.355407643172423,5.9214645042626,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,0,51389c06-2a1c-4dcb-a847-5f0c914e298c,medium,252,128,2026-08-31T12:20:54.847154+00:00,0.16574266698444262,6.310333041998092,43,128,5402688.779997488,20.423563700405246,20.284190889466956,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,1,1952ea53-9c68-4536-8e9d-089790a8bdee,code,267,128,2026-08-31T12:20:54.847267+00:00,18.0774574590032,23.57407404197147,54,128,6513086.47931492,22.77481444475703,5.4296936444717945,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,2,d71abf03-e17c-42b0-b6bb-c57bf6961bc2,short,106,128,2026-08-31T12:20:54.847373+00:00,12.635340875014663,17.94654095900478,20,95,9229349.330872172,17.481471270830138,5.293499188339869,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,3,4e10d58d-5d6d-46e7-8254-46dd35cfeb9f,medium,252,128,2026-08-31T12:20:54.847461+00:00,6.461538125004154,12.505505541979801,43,128,3739130.434782609,20.730070992395465,10.235491845596812,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,0,63b32437-7995-44f4-b5ca-f0f52da2bda7,code,267,128,2026-08-31T12:21:18.421714+00:00,17.908610125014093,23.736398082983214,54,128,15071169.411108011,21.428265908373906,5.392562070812423,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,1,3b8fa85d-5574-4ac2-b33c-7208ef99a34f,short,106,128,2026-08-31T12:21:18.421631+00:00,6.036650207999628,11.081229583011009,20,95,4571428.571428572,18.37088402458498,8.573055840810985,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,2,76178335-d531-49f7-83d5-84a414d73ce5,medium,252,128,2026-08-31T12:21:18.421520+00:00,11.268355667009018,17.757357084017713,43,128,7533286.615276804,19.182821184166418,7.208279891786645,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,3,728d2a73-194b-41bb-9dda-216960415fcf,code,267,128,2026-08-31T12:21:18.421818+00:00,0.17636983300326392,5.888466249976773,54,128,7200000.0,21.89716979543998,21.737409125934093,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,0,dfcbb66f-3170-41e1-a4e1-a9b7449517ce,short,106,128,2026-08-31T12:21:42.158434+00:00,6.034018917009234,11.539304499980062,20,95,3200000.0,16.88203353475361,8.232731877399036,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,1,04f9942b-bc73-4665-9830-40b7a50beb96,medium,252,128,2026-08-31T12:21:42.158640+00:00,11.68165541702183,18.32316145801451,43,128,9921550.530687585,18.896968681699633,6.985694051394885,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,2,e2e252a5-26bf-4277-a744-cf7456cdfac7,code,267,128,2026-08-31T12:21:42.158549+00:00,0.17531787499319762,5.8762462079757825,54,128,5355548.943766736,21.95473700787165,21.782613503543573,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,3,61a1f98f-fb9a-49a8-8f00-04bb52b494cc,short,106,128,2026-08-31T12:21:42.158328+00:00,18.46465341700241,23.86128708301112,20,95,5782017.924255566,17.187946685051212,3.981344328556299,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,0,ae555d62-2572-4a62-8cd6-7ae714aace26,medium,252,128,2026-08-31T12:22:06.019994+00:00,20.83534020796651,28.275580540997908,43,128,7937973.047812442,16.887931275751107,4.526874340012493,28.276262458995916,16.940004029691313, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,1,c0147505-dd4a-4ad8-bb9d-0955114a8ee0,code,267,128,2026-08-31T12:22:06.020061+00:00,13.962619374971837,20.66803166596219,54,128,10125632.852053255,18.57306514141657,6.193139340443381,28.276262458995916,16.940004029691313, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,2,3a3d625f-5d64-42d0-9145-3056ea9864db,short,106,128,2026-08-31T12:22:06.019916+00:00,7.783273542008828,13.761600458994508,20,95,1927710.843373494,15.539155324201145,6.903266831722942,28.276262458995916,16.940004029691313, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,3,8ea78898-3c59-49f2-9734-f01bb668d810,medium,252,128,2026-08-31T12:22:06.019812+00:00,0.17432458401890472,7.616022333968431,43,128,2866666.6666666665,16.90294641795698,16.806673403399,28.276262458995916,16.940004029691313, diff --git a/benchmarks/inference/results/20260831T121855Z/raw.json b/benchmarks/inference/results/20260831T121855Z/raw.json new file mode 100644 index 0000000..9d4472d --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "556d36df-74f6-4dba-866f-134c99f385ac", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:01.706610+00:00", + "ttft_s": 8.133006000018213, + "total_s": 13.098198042018339, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 80.39981477169073, + "decode_tps": 18.60021913626931, + "request_tps": 7.252906063509266, + "group_elapsed_s": 13.099143458006438, + "group_output_tps": 7.252382593148428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "456b9efb-0afa-48e4-8093-041758c70c7d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:14.805394+00:00", + "ttft_s": 0.6389443330117501, + "total_s": 6.043248832982499, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 91.99219534791703, + "decode_tps": 23.084069916067204, + "request_tps": 21.180660194134138, + "group_elapsed_s": 6.043762709014118, + "group_output_tps": 21.178859290602404, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "78330ae3-a01a-4bf4-b1b6-e4ae861145a8", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:20.848831+00:00", + "ttft_s": 0.25939199997810647, + "total_s": 5.077765540976543, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 194.8949698826367, + "decode_tps": 19.20226982827757, + "request_tps": 18.709016639970706, + "group_elapsed_s": 5.077990833029617, + "group_output_tps": 18.708186588694836, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "f8373947-d690-42a4-b613-33f71746b2f9", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:25.927037+00:00", + "ttft_s": 0.25913220801157877, + "total_s": 5.760092166019604, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 423.79232394227944, + "decode_tps": 22.737809958094928, + "request_tps": 22.22186664913243, + "group_elapsed_s": 5.761121416988317, + "group_output_tps": 22.217896609947385, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "9e12f0f4-50d6-444c-8752-4565aa55084b", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:31.688098+00:00", + "ttft_s": 0.6637360000167973, + "total_s": 5.271547083975747, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 115.32507388679936, + "decode_tps": 26.793290081008884, + "request_tps": 24.281296924974765, + "group_elapsed_s": 5.271936749981251, + "group_output_tps": 24.27950221528269, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "583bec1f-0983-4208-8635-439a9a0c5424", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:36.960047+00:00", + "ttft_s": 0.15107908402569592, + "total_s": 5.119054667011369, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 4102564.1025641025, + "decode_tps": 18.66803886577614, + "request_tps": 18.558113983858536, + "group_elapsed_s": 5.11973733396735, + "group_output_tps": 18.555639440663118, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "641937dd-533c-493e-b7b3-3fa2cc6e8879", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:42.079662+00:00", + "ttft_s": 0.1495445409673266, + "total_s": 5.59909695800161, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3794229.2420365307, + "decode_tps": 22.9734392268289, + "request_tps": 22.86082933732315, + "group_elapsed_s": 5.599817292008083, + "group_output_tps": 22.85788862838049, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "3dbdfe4c-7d43-4575-a1e6-6b74bd5a10a7", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680109+00:00", + "ttft_s": 0.16311579098692164, + "total_s": 4.8760897910106, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 1791151.7105498835, + "decode_tps": 19.64848684540538, + "request_tps": 19.482824162741817, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "1ea04fa9-7f60-4936-ba53-6829df8e5714", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680573+00:00", + "ttft_s": 10.314829125010874, + "total_s": 16.516400250024162, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 6370370.37037037, + "decode_tps": 20.216323480613685, + "request_tps": 7.749872736331438, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "046681e1-e7be-4c58-8284-88f821e4a8a0", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680452+00:00", + "ttft_s": 5.110771542007569, + "total_s": 10.15410141699249, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 514.3377603686088, + "decode_tps": 24.759396813702583, + "request_tps": 12.605743703307615, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "f496e50f-a2db-4fad-a2e1-2a7965fd862d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680317+00:00", + "ttft_s": 16.645131999976, + "total_s": 22.03015966597013, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 6666666.666666666, + "decode_tps": 17.25091779989875, + "request_tps": 4.312270153300159, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "cb3d0e9e-0a6f-4f36-aec9-4b20cbc8f87d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.710819+00:00", + "ttft_s": 17.595250790996943, + "total_s": 23.510470708017237, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 19843101.06137517, + "decode_tps": 21.163606353137514, + "request_tps": 5.444382700357892, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "fa3bb29e-7370-4255-877e-376076d7aa76", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.711021+00:00", + "ttft_s": 5.7327396660111845, + "total_s": 11.353798499971163, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 9000000.0, + "decode_tps": 22.18879858004337, + "request_tps": 11.273760054868431, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "93ec5b28-1a29-426a-96ff-b3b525fb3c71", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.710927+00:00", + "ttft_s": 0.1655573329771869, + "total_s": 5.57563720800681, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 1563477.1732332706, + "decode_tps": 17.170644798585553, + "request_tps": 17.038411298277563, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "dcac6e9a-3580-4bb1-8787-9cef1c64bfd5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.711149+00:00", + "ttft_s": 11.527663582994137, + "total_s": 17.459117082995363, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 6331909.880724489, + "decode_tps": 21.03978753266432, + "request_tps": 7.331413117371669, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "d42caf5b-fb6a-4e96-8e42-fd0b8fc1945b", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222727+00:00", + "ttft_s": 6.285906166012865, + "total_s": 10.864910541044082, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 8421052.631578946, + "decode_tps": 20.156958877874334, + "request_tps": 8.743744335594945, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "87483860-d4c6-4f13-928c-48de78b6ef70", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222336+00:00", + "ttft_s": 0.17068937502335757, + "total_s": 6.147177167003974, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4669345.205776958, + "decode_tps": 20.965937740191077, + "request_tps": 20.822565630133763, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "ee57b9cf-06bc-468b-bd1e-f936abf292df", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222603+00:00", + "ttft_s": 16.19913299998734, + "total_s": 21.62435016600648, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 11675675.675675677, + "decode_tps": 22.963769927247082, + "request_tps": 5.919253018812849, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "13164aa4-836c-4761-8f99-95bfe18d33a1", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222066+00:00", + "ttft_s": 11.00553745898651, + "total_s": 16.043328458967153, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 10214504.596527068, + "decode_tps": 18.355407643172423, + "request_tps": 5.9214645042626, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "51389c06-2a1c-4dcb-a847-5f0c914e298c", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847154+00:00", + "ttft_s": 0.16574266698444262, + "total_s": 6.310333041998092, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 5402688.779997488, + "decode_tps": 20.423563700405246, + "request_tps": 20.284190889466956, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "1952ea53-9c68-4536-8e9d-089790a8bdee", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847267+00:00", + "ttft_s": 18.0774574590032, + "total_s": 23.57407404197147, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 6513086.47931492, + "decode_tps": 22.77481444475703, + "request_tps": 5.4296936444717945, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "d71abf03-e17c-42b0-b6bb-c57bf6961bc2", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847373+00:00", + "ttft_s": 12.635340875014663, + "total_s": 17.94654095900478, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 9229349.330872172, + "decode_tps": 17.481471270830138, + "request_tps": 5.293499188339869, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "4e10d58d-5d6d-46e7-8254-46dd35cfeb9f", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847461+00:00", + "ttft_s": 6.461538125004154, + "total_s": 12.505505541979801, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3739130.434782609, + "decode_tps": 20.730070992395465, + "request_tps": 10.235491845596812, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "63b32437-7995-44f4-b5ca-f0f52da2bda7", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421714+00:00", + "ttft_s": 17.908610125014093, + "total_s": 23.736398082983214, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 15071169.411108011, + "decode_tps": 21.428265908373906, + "request_tps": 5.392562070812423, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "3b8fa85d-5574-4ac2-b33c-7208ef99a34f", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421631+00:00", + "ttft_s": 6.036650207999628, + "total_s": 11.081229583011009, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 4571428.571428572, + "decode_tps": 18.37088402458498, + "request_tps": 8.573055840810985, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "76178335-d531-49f7-83d5-84a414d73ce5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421520+00:00", + "ttft_s": 11.268355667009018, + "total_s": 17.757357084017713, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7533286.615276804, + "decode_tps": 19.182821184166418, + "request_tps": 7.208279891786645, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "728d2a73-194b-41bb-9dda-216960415fcf", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421818+00:00", + "ttft_s": 0.17636983300326392, + "total_s": 5.888466249976773, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 7200000.0, + "decode_tps": 21.89716979543998, + "request_tps": 21.737409125934093, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "dfcbb66f-3170-41e1-a4e1-a9b7449517ce", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158434+00:00", + "ttft_s": 6.034018917009234, + "total_s": 11.539304499980062, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 3200000.0, + "decode_tps": 16.88203353475361, + "request_tps": 8.232731877399036, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "04f9942b-bc73-4665-9830-40b7a50beb96", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158640+00:00", + "ttft_s": 11.68165541702183, + "total_s": 18.32316145801451, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 9921550.530687585, + "decode_tps": 18.896968681699633, + "request_tps": 6.985694051394885, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "e2e252a5-26bf-4277-a744-cf7456cdfac7", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158549+00:00", + "ttft_s": 0.17531787499319762, + "total_s": 5.8762462079757825, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 5355548.943766736, + "decode_tps": 21.95473700787165, + "request_tps": 21.782613503543573, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "61a1f98f-fb9a-49a8-8f00-04bb52b494cc", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158328+00:00", + "ttft_s": 18.46465341700241, + "total_s": 23.86128708301112, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 5782017.924255566, + "decode_tps": 17.187946685051212, + "request_tps": 3.981344328556299, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "ae555d62-2572-4a62-8cd6-7ae714aace26", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.019994+00:00", + "ttft_s": 20.83534020796651, + "total_s": 28.275580540997908, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7937973.047812442, + "decode_tps": 16.887931275751107, + "request_tps": 4.526874340012493, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "c0147505-dd4a-4ad8-bb9d-0955114a8ee0", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.020061+00:00", + "ttft_s": 13.962619374971837, + "total_s": 20.66803166596219, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 10125632.852053255, + "decode_tps": 18.57306514141657, + "request_tps": 6.193139340443381, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "3a3d625f-5d64-42d0-9145-3056ea9864db", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.019916+00:00", + "ttft_s": 7.783273542008828, + "total_s": 13.761600458994508, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 1927710.843373494, + "decode_tps": 15.539155324201145, + "request_tps": 6.903266831722942, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "8ea78898-3c59-49f2-9734-f01bb668d810", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.019812+00:00", + "ttft_s": 0.17432458401890472, + "total_s": 7.616022333968431, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 2866666.6666666665, + "decode_tps": 16.90294641795698, + "request_tps": 16.806673403399, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T121855Z/summary.json b/benchmarks/inference/results/20260831T121855Z/summary.json new file mode 100644 index 0000000..d97729d --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.296576766599901, + "ttft_s_p50": 0.25913220801157877, + "ttft_s_p95": 0.5828672000090591, + "total_s_mean": 5.365511283196975, + "total_s_p50": 5.271547083975747, + "total_s_p95": 5.7278931244160045, + "prompt_tps_mean": 1579505.471393669, + "prompt_tps_p50": 423.79232394227944, + "prompt_tps_p95": 4040897.1304585882, + "decode_tps_mean": 22.074969591997284, + "decode_tps_p50": 22.737809958094928, + "decode_tps_p95": 26.029319910172887, + "request_tps_mean": 21.326224707051917, + "request_tps_p50": 22.22186664913243, + "request_tps_p95": 23.99720340744444, + "aggregate_output_tps_mean": 21.3238226965937, + "aggregate_output_tps_p50": 22.217896609947385, + "aggregate_output_tps_p95": 23.99517949790225 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 9.275126714701765, + "ttft_s_p50": 9.394405500497669, + "ttft_s_p95": 18.583187756550615, + "total_s_mean": 15.170045258343453, + "total_s_p50": 14.90246445898083, + "total_s_p95": 24.08200175591046, + "prompt_tps_mean": 7067890.927047749, + "prompt_tps_p50": 6856543.23965746, + "prompt_tps_p95": 11845450.362447295, + "decode_tps_mean": 19.377795978907, + "decode_tps_p50": 19.039894932933024, + "decode_tps_p95": 22.784262218881533, + "request_tps_mean": 10.048677383124767, + "request_tps_p50": 7.0969869715907645, + "request_tps_p95": 21.739669344814565, + "aggregate_output_tps_mean": 19.35053924595546, + "aggregate_output_tps_p50": 20.179477104341757, + "aggregate_output_tps_p95": 20.562657512957074 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T132015Z/metadata.json b/benchmarks/inference/results/20260831T132015Z/metadata.json new file mode 100644 index 0000000..6e2ae60 --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/metadata.json @@ -0,0 +1,136 @@ +{ + "run_id": "20260831T132015Z", + "created_at": "2026-08-31T13:20:15.702115+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "vllm", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-mxfp8", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-mxfp8" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 15:20:16 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 15:20:16 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 15:20:16 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 15:20:18 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 15:20:19 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "vllm-metal": { + "object": "list", + "data": [ + { + "id": "qwen3.8-27b-mxfp8", + "object": "model", + "created": 1788182421, + "owned_by": "vllm", + "root": "mlx-community/Qwen3.8-27B-mxfp8", + "parent": null, + "max_model_len": 8192, + "permission": [ + { + "id": "modelperm-a3c90f9b6d588379", + "object": "model_permission", + "created": 1788182421, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + } + ] + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T13:22:48.218681+00:00", + "summary": [ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.37155269999057056, + "ttft_s_p50": 0.4171057919738814, + "ttft_s_p95": 0.4531440335791558, + "total_s_mean": 7.930883799993898, + "total_s_p50": 8.838838333962485, + "total_s_p95": 8.998963149765041, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.079897408512505, + "decode_tps_p50": 15.126147490283795, + "decode_tps_p95": 15.196162811046243, + "request_tps_mean": 14.380896425252896, + "request_tps_p50": 14.386982882152838, + "request_tps_p95": 14.53502856074605, + "aggregate_output_tps_mean": 14.380548747737752, + "aggregate_output_tps_p50": 14.386712701914696, + "aggregate_output_tps_p95": 14.534638049200744 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.255672066603438, + "ttft_s_p50": 1.3144360205042176, + "ttft_s_p95": 1.419064954231726, + "total_s_mean": 12.251293779097614, + "total_s_p50": 12.702340687508695, + "total_s_p95": 13.641910566360457, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 10.500220268362742, + "decode_tps_p50": 10.485677148644122, + "decode_tps_p95": 11.071162546140778, + "request_tps_mean": 9.417789343799708, + "request_tps_p50": 9.430282582501674, + "request_tps_p95": 10.077752113288362, + "aggregate_output_tps_mean": 35.09087333775396, + "aggregate_output_tps_p50": 34.96408475660248, + "aggregate_output_tps_p95": 35.52678209465127 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T132015Z/raw.csv b/benchmarks/inference/results/20260831T132015Z/raw.csv new file mode 100644 index 0000000..8d2ffed --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,1,0,0,4c059f2d-caa2-44ae-ab1d-539d608c86d8,short,106,128,2026-08-31T13:20:21.117908+00:00,0.3135264579905197,6.390581874991767,20,93,,15.303464197450301,14.552665440988472,6.39150691695977,14.550559235604652, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,1,1,0,1bf28507-8f2f-4127-ba2f-2f5637e722bf,medium,252,128,2026-08-31T13:20:27.508796+00:00,0.3822946660220623,8.7317384579801,43,128,,15.330362499509992,14.659165596401756,8.731932666967623,14.658839558419446, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,0,0,81b82bc5-db99-4c10-9ec1-1f684485db59,short,106,128,2026-08-31T13:20:36.240817+00:00,0.2682818329776637,6.39245533302892,20,93,,15.185722612075187,14.548400443172758,6.392639332974795,14.547981695179217, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,1,0,1de32693-20bd-450c-8341-17773d6c57b2,medium,252,128,2026-08-31T13:20:42.633525+00:00,0.4171057919738814,8.838838333962485,43,128,,15.198772860789006,14.481541031039212,8.839007750037126,14.481263465286855, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,2,0,e821a6d4-8b00-477b-a66f-3579d33c583b,code,267,128,2026-08-31T13:20:51.472629+00:00,0.4347635000012815,8.896931416995358,54,128,,15.126147490283795,14.386982882152838,8.897098499990534,14.386712701914696, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,3,0,6415410d-9d1c-4521-b2e2-04c40ce2fd72,short,106,128,2026-08-31T13:21:00.369851+00:00,0.2798732080264017,6.501722833025269,20,93,,14.947323642528074,14.303901041060966,6.501933082996402,14.303438502498576, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,4,0,44eda129-4834-4a55-a14a-33a4a510922d,medium,252,128,2026-08-31T13:21:06.871859+00:00,0.4577391669736244,9.024471082957461,43,128,,14.94152043688646,14.183656728838715,9.024667916994076,14.183347373809413, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,0,a696baea-c3a4-422a-9d29-c4fff0e6d70e,short,106,128,2026-08-31T13:21:15.897040+00:00,1.1126187500194646,9.980102250003256,20,93,,10.48775562989996,9.318541801510065,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,1,2a75391e-8c1c-4244-8cea-f8f97ce3c40d,medium,252,128,2026-08-31T13:21:15.896746+00:00,1.1128792079980485,12.400287791970186,43,128,,11.340069693388886,10.322341073639151,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,2,21360267-b5fe-4bd2-8b87-d5bbdf4a9c49,code,267,128,2026-08-31T13:21:15.896944+00:00,1.1127408749889582,12.400139749981463,54,128,,11.340079447673899,10.322464309339042,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,3,106f9109-b093-475b-85c2-dc0847a3f3e3,short,106,128,2026-08-31T13:21:15.896863+00:00,1.0121074999915436,9.893255082948599,20,93,,10.471619701317342,9.400343892910335,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,0,fd6e44d9-a975-4760-853a-c85d6547b8f2,medium,252,128,2026-08-31T13:21:28.297798+00:00,1.4682920000050217,13.96469266602071,43,128,,10.242949423676817,9.165973291446168,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,1,a5541448-562a-4c84-aeff-3e2e5b8aba34,code,267,128,2026-08-31T13:21:28.297877+00:00,1.4681764579727314,13.9645455830032,54,128,,10.242975277003744,9.166069833005798,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,2,bb897f87-b4a0-4483-9946-4f6483212386,short,106,128,2026-08-31T13:21:28.297713+00:00,1.3568928329623304,13.431885249970946,20,122,,10.103526013660515,9.082864968658356,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,3,576c3ce6-5566-4a17-b311-3c2f8f3a5a17,medium,252,128,2026-08-31T13:21:28.297622+00:00,1.4684889169875532,13.964902083971538,43,128,,10.242939176993687,9.16583583832745,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,0,75886439-869f-49d6-881d-4bc7cf650b04,short,106,128,2026-08-31T13:21:42.263079+00:00,1.1142227500095032,10.27476887498051,20,93,,10.152233145411333,9.051298489687577,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,1,2ecba00e-62dd-4ea7-a1e1-c7b0c12aae0e,medium,252,128,2026-08-31T13:21:42.263219+00:00,1.1140572919975966,12.70125433296198,43,128,,11.046675011003936,10.077744815157159,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,2,c1df76bd-2c64-4e34-a5be-10d74b64ce17,code,267,128,2026-08-31T13:21:42.263452+00:00,1.1138802919886075,12.701070374983829,54,128,,11.046681644400257,10.077890777781237,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,3,1bf72731-e2e3-4f8d-bf42-627b9fce292a,short,106,128,2026-08-31T13:21:42.263323+00:00,1.0123424580087885,10.185485332971439,20,93,,10.138291888359873,9.130640019572722,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,0,a04682c5-d585-4466-8fad-17ed2b9ba80b,medium,252,128,2026-08-31T13:21:54.962444+00:00,1.4147373330197297,13.505381957977079,43,128,,10.586697729563905,9.477703066694506,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,1,bf6552cc-7c18-4617-8260-b6a63ff65b3c,code,267,128,2026-08-31T13:21:54.962650+00:00,1.4145112080150284,13.505133582977578,54,128,,10.586717211933143,9.477877372596776,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,2,36abe2cb-3ddf-4311-853a-4151aef2b328,short,106,128,2026-08-31T13:21:54.962724+00:00,1.4144840000080876,10.538147750019561,20,93,,10.193273508121454,8.825080289828673,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,3,110efa2a-5fb4-48e9-835b-3092cd0df4fb,medium,252,128,2026-08-31T13:21:54.962552+00:00,1.3138721249997616,13.505327666993253,43,128,,10.49915652475652,9.477741166756687,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,0,4d78c4f5-e05f-4ffe-afb6-748b8db712f6,code,267,128,2026-08-31T13:22:08.467061+00:00,1.2189033330068924,13.40631824999582,54,128,,10.502637423262865,9.547736941127733,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,1,9beb5379-ecf9-45d3-8725-13dd100b245d,short,106,128,2026-08-31T13:22:08.467363+00:00,1.3166250829817727,10.428931292030029,20,93,,10.205978362278222,8.917500498931489,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,2,e863870c-69a1-44bd-9d57-7d28f9a30c17,medium,252,128,2026-08-31T13:22:08.467157+00:00,1.3168415420223027,13.406341000052635,43,128,,10.58770054495327,9.547720738976986,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,3,21ffaa25-b50a-4b5f-b562-4a7810b56506,code,267,128,2026-08-31T13:22:08.467234+00:00,1.316743416013196,13.406194041017443,54,128,,10.587743311946818,9.547825401331103,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,0,00728b55-60a4-46d3-8000-146e38d0b3a6,short,106,128,2026-08-31T13:22:21.873446+00:00,1.0375588749884628,10.190524249977898,20,93,,10.160641517788692,9.126125184403708,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,1,531d5ceb-d88b-49ab-ad8d-7bea678c3bb8,medium,252,128,2026-08-31T13:22:21.873549+00:00,1.140847125032451,12.702382000046782,43,128,,11.07119438584415,10.076850152949941,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,2,a4b2f7bb-ab5c-405b-9514-d64bf34153ea,code,267,128,2026-08-31T13:22:21.873705+00:00,1.1407294999808073,12.702299374970607,54,128,,11.071160870366917,10.076915700177802,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,3,1b77ace8-b3ef-47f2-a2cd-06b6f9400a6e,short,106,128,2026-08-31T13:22:21.873633+00:00,1.1407265829620883,10.277835665969178,20,93,,10.178274020276117,9.048597683647658,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,0,1143b997-b26b-4ca6-a4ad-dd1014020849,medium,252,128,2026-08-31T13:22:34.576247+00:00,1.3149999160086736,13.642209500016179,43,128,,10.383534012925246,9.382644358294614,13.642570749972947,34.96408475660248, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,1,907e97b8-fec4-47ff-b971-45716a15b6fb,code,267,128,2026-08-31T13:22:34.576465+00:00,1.419054625032004,13.641894833010156,54,128,,10.472197772531725,9.38286078047386,13.642570749972947,34.96408475660248, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,2,d5875acf-e267-4393-84c1-a01dac48c4a5,short,106,128,2026-08-31T13:22:34.576349+00:00,1.4192612090264447,10.662482584011741,20,93,,10.061427312742246,8.722171339295066,13.642570749972947,34.96408475660248, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,3,d2828df9-67d3-44b0-9809-bdb8c75f5282,medium,252,128,2026-08-31T13:22:34.576542+00:00,1.4190426669665612,13.641892916988581,43,128,,10.47218916878814,9.382862098308841,13.642570749972947,34.96408475660248, diff --git a/benchmarks/inference/results/20260831T132015Z/raw.json b/benchmarks/inference/results/20260831T132015Z/raw.json new file mode 100644 index 0000000..252c974 --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "4c059f2d-caa2-44ae-ab1d-539d608c86d8", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:21.117908+00:00", + "ttft_s": 0.3135264579905197, + "total_s": 6.390581874991767, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 15.303464197450301, + "request_tps": 14.552665440988472, + "group_elapsed_s": 6.39150691695977, + "group_output_tps": 14.550559235604652, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "1bf28507-8f2f-4127-ba2f-2f5637e722bf", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:27.508796+00:00", + "ttft_s": 0.3822946660220623, + "total_s": 8.7317384579801, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.330362499509992, + "request_tps": 14.659165596401756, + "group_elapsed_s": 8.731932666967623, + "group_output_tps": 14.658839558419446, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "81b82bc5-db99-4c10-9ec1-1f684485db59", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:36.240817+00:00", + "ttft_s": 0.2682818329776637, + "total_s": 6.39245533302892, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 15.185722612075187, + "request_tps": 14.548400443172758, + "group_elapsed_s": 6.392639332974795, + "group_output_tps": 14.547981695179217, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "1de32693-20bd-450c-8341-17773d6c57b2", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:42.633525+00:00", + "ttft_s": 0.4171057919738814, + "total_s": 8.838838333962485, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.198772860789006, + "request_tps": 14.481541031039212, + "group_elapsed_s": 8.839007750037126, + "group_output_tps": 14.481263465286855, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "e821a6d4-8b00-477b-a66f-3579d33c583b", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:51.472629+00:00", + "ttft_s": 0.4347635000012815, + "total_s": 8.896931416995358, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.126147490283795, + "request_tps": 14.386982882152838, + "group_elapsed_s": 8.897098499990534, + "group_output_tps": 14.386712701914696, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "6415410d-9d1c-4521-b2e2-04c40ce2fd72", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:00.369851+00:00", + "ttft_s": 0.2798732080264017, + "total_s": 6.501722833025269, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 14.947323642528074, + "request_tps": 14.303901041060966, + "group_elapsed_s": 6.501933082996402, + "group_output_tps": 14.303438502498576, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "44eda129-4834-4a55-a14a-33a4a510922d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:06.871859+00:00", + "ttft_s": 0.4577391669736244, + "total_s": 9.024471082957461, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 14.94152043688646, + "request_tps": 14.183656728838715, + "group_elapsed_s": 9.024667916994076, + "group_output_tps": 14.183347373809413, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "a696baea-c3a4-422a-9d29-c4fff0e6d70e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.897040+00:00", + "ttft_s": 1.1126187500194646, + "total_s": 9.980102250003256, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.48775562989996, + "request_tps": 9.318541801510065, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "2a75391e-8c1c-4244-8cea-f8f97ce3c40d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.896746+00:00", + "ttft_s": 1.1128792079980485, + "total_s": 12.400287791970186, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.340069693388886, + "request_tps": 10.322341073639151, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "21360267-b5fe-4bd2-8b87-d5bbdf4a9c49", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.896944+00:00", + "ttft_s": 1.1127408749889582, + "total_s": 12.400139749981463, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.340079447673899, + "request_tps": 10.322464309339042, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "106f9109-b093-475b-85c2-dc0847a3f3e3", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.896863+00:00", + "ttft_s": 1.0121074999915436, + "total_s": 9.893255082948599, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.471619701317342, + "request_tps": 9.400343892910335, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "fd6e44d9-a975-4760-853a-c85d6547b8f2", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297798+00:00", + "ttft_s": 1.4682920000050217, + "total_s": 13.96469266602071, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.242949423676817, + "request_tps": 9.165973291446168, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "a5541448-562a-4c84-aeff-3e2e5b8aba34", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297877+00:00", + "ttft_s": 1.4681764579727314, + "total_s": 13.9645455830032, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.242975277003744, + "request_tps": 9.166069833005798, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "bb897f87-b4a0-4483-9946-4f6483212386", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297713+00:00", + "ttft_s": 1.3568928329623304, + "total_s": 13.431885249970946, + "prompt_tokens": 20, + "output_tokens": 122, + "prompt_tps": null, + "decode_tps": 10.103526013660515, + "request_tps": 9.082864968658356, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "576c3ce6-5566-4a17-b311-3c2f8f3a5a17", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297622+00:00", + "ttft_s": 1.4684889169875532, + "total_s": 13.964902083971538, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.242939176993687, + "request_tps": 9.16583583832745, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "75886439-869f-49d6-881d-4bc7cf650b04", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263079+00:00", + "ttft_s": 1.1142227500095032, + "total_s": 10.27476887498051, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.152233145411333, + "request_tps": 9.051298489687577, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "2ecba00e-62dd-4ea7-a1e1-c7b0c12aae0e", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263219+00:00", + "ttft_s": 1.1140572919975966, + "total_s": 12.70125433296198, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.046675011003936, + "request_tps": 10.077744815157159, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "c1df76bd-2c64-4e34-a5be-10d74b64ce17", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263452+00:00", + "ttft_s": 1.1138802919886075, + "total_s": 12.701070374983829, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.046681644400257, + "request_tps": 10.077890777781237, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "1bf72731-e2e3-4f8d-bf42-627b9fce292a", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263323+00:00", + "ttft_s": 1.0123424580087885, + "total_s": 10.185485332971439, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.138291888359873, + "request_tps": 9.130640019572722, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "a04682c5-d585-4466-8fad-17ed2b9ba80b", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962444+00:00", + "ttft_s": 1.4147373330197297, + "total_s": 13.505381957977079, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.586697729563905, + "request_tps": 9.477703066694506, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "bf6552cc-7c18-4617-8260-b6a63ff65b3c", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962650+00:00", + "ttft_s": 1.4145112080150284, + "total_s": 13.505133582977578, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.586717211933143, + "request_tps": 9.477877372596776, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "36abe2cb-3ddf-4311-853a-4151aef2b328", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962724+00:00", + "ttft_s": 1.4144840000080876, + "total_s": 10.538147750019561, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.193273508121454, + "request_tps": 8.825080289828673, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "110efa2a-5fb4-48e9-835b-3092cd0df4fb", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962552+00:00", + "ttft_s": 1.3138721249997616, + "total_s": 13.505327666993253, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.49915652475652, + "request_tps": 9.477741166756687, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "4d78c4f5-e05f-4ffe-afb6-748b8db712f6", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467061+00:00", + "ttft_s": 1.2189033330068924, + "total_s": 13.40631824999582, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.502637423262865, + "request_tps": 9.547736941127733, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "9beb5379-ecf9-45d3-8725-13dd100b245d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467363+00:00", + "ttft_s": 1.3166250829817727, + "total_s": 10.428931292030029, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.205978362278222, + "request_tps": 8.917500498931489, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "e863870c-69a1-44bd-9d57-7d28f9a30c17", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467157+00:00", + "ttft_s": 1.3168415420223027, + "total_s": 13.406341000052635, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.58770054495327, + "request_tps": 9.547720738976986, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "21ffaa25-b50a-4b5f-b562-4a7810b56506", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467234+00:00", + "ttft_s": 1.316743416013196, + "total_s": 13.406194041017443, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.587743311946818, + "request_tps": 9.547825401331103, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "00728b55-60a4-46d3-8000-146e38d0b3a6", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873446+00:00", + "ttft_s": 1.0375588749884628, + "total_s": 10.190524249977898, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.160641517788692, + "request_tps": 9.126125184403708, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "531d5ceb-d88b-49ab-ad8d-7bea678c3bb8", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873549+00:00", + "ttft_s": 1.140847125032451, + "total_s": 12.702382000046782, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.07119438584415, + "request_tps": 10.076850152949941, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "a4b2f7bb-ab5c-405b-9514-d64bf34153ea", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873705+00:00", + "ttft_s": 1.1407294999808073, + "total_s": 12.702299374970607, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.071160870366917, + "request_tps": 10.076915700177802, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "1b77ace8-b3ef-47f2-a2cd-06b6f9400a6e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873633+00:00", + "ttft_s": 1.1407265829620883, + "total_s": 10.277835665969178, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.178274020276117, + "request_tps": 9.048597683647658, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "1143b997-b26b-4ca6-a4ad-dd1014020849", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576247+00:00", + "ttft_s": 1.3149999160086736, + "total_s": 13.642209500016179, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.383534012925246, + "request_tps": 9.382644358294614, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "907e97b8-fec4-47ff-b971-45716a15b6fb", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576465+00:00", + "ttft_s": 1.419054625032004, + "total_s": 13.641894833010156, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.472197772531725, + "request_tps": 9.38286078047386, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "d5875acf-e267-4393-84c1-a01dac48c4a5", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576349+00:00", + "ttft_s": 1.4192612090264447, + "total_s": 10.662482584011741, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.061427312742246, + "request_tps": 8.722171339295066, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "d2828df9-67d3-44b0-9809-bdb8c75f5282", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576542+00:00", + "ttft_s": 1.4190426669665612, + "total_s": 13.641892916988581, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.47218916878814, + "request_tps": 9.382862098308841, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T132015Z/summary.json b/benchmarks/inference/results/20260831T132015Z/summary.json new file mode 100644 index 0000000..c554507 --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.37155269999057056, + "ttft_s_p50": 0.4171057919738814, + "ttft_s_p95": 0.4531440335791558, + "total_s_mean": 7.930883799993898, + "total_s_p50": 8.838838333962485, + "total_s_p95": 8.998963149765041, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.079897408512505, + "decode_tps_p50": 15.126147490283795, + "decode_tps_p95": 15.196162811046243, + "request_tps_mean": 14.380896425252896, + "request_tps_p50": 14.386982882152838, + "request_tps_p95": 14.53502856074605, + "aggregate_output_tps_mean": 14.380548747737752, + "aggregate_output_tps_p50": 14.386712701914696, + "aggregate_output_tps_p95": 14.534638049200744 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.255672066603438, + "ttft_s_p50": 1.3144360205042176, + "ttft_s_p95": 1.419064954231726, + "total_s_mean": 12.251293779097614, + "total_s_p50": 12.702340687508695, + "total_s_p95": 13.641910566360457, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 10.500220268362742, + "decode_tps_p50": 10.485677148644122, + "decode_tps_p95": 11.071162546140778, + "request_tps_mean": 9.417789343799708, + "request_tps_p50": 9.430282582501674, + "request_tps_p95": 10.077752113288362, + "aggregate_output_tps_mean": 35.09087333775396, + "aggregate_output_tps_p50": 34.96408475660248, + "aggregate_output_tps_p95": 35.52678209465127 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T134334Z/metadata.json b/benchmarks/inference/results/20260831T134334Z/metadata.json new file mode 100644 index 0000000..33946ef --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/metadata.json @@ -0,0 +1,108 @@ +{ + "run_id": "20260831T134334Z", + "created_at": "2026-08-31T13:43:34.324346+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "ollama", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mlx" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 15:43:34 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 15:43:34 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 15:43:34 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 15:43:36 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 15:43:37 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "ollama": { + "license": "\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n", + "modelfile": "# Modelfile generated by \"ollama show\"\nFROM qwen3.8:27b-mlx\nTEMPLATE {{ .Prompt }}\nRENDERER qwen3.8\nPARSER qwen3.5\nPARAMETER min_p 0\nPARAMETER presence_penalty 0\nPARAMETER repeat_penalty 1\nPARAMETER temperature 1\nPARAMETER top_k 20\nPARAMETER top_p 0.95\nLICENSE \"\"\"\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\"\"\"\n", + "parameters": "top_k 20\ntop_p 0.95\nmin_p 0\npresence_penalty 0\nrepeat_penalty 1\ntemperature 1", + "template": "{{ .Prompt }}", + "details": { + "parent_model": "", + "format": "safetensors", + "family": "qwen3_5", + "families": null, + "parameter_size": "27.8B", + "quantization_level": "nvfp4" + }, + "model_info": { + "general.architecture": "qwen3_5", + "general.parameter_count": 27781081984, + "qwen3_5.block_count": 64, + "qwen3_5.context_length": 262144, + "qwen3_5.embedding_length": 5120 + }, + "capabilities": [ + "completion", + "vision", + "tools", + "thinking" + ], + "modified_at": "2026-08-20T17:42:35.895756165+02:00", + "requires": "0.32.12" + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T13:44:00.378796+00:00", + "summary": [ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.1723672749940306, + "ttft_s_p50": 0.11980204097926617, + "ttft_s_p95": 0.36265889178030186, + "total_s_mean": 2.551861874992028, + "total_s_p50": 2.646599249972496, + "total_s_p95": 2.8519752004183827, + "prompt_tps_mean": 1253549.1340794961, + "prompt_tps_p50": 831.312776076798, + "prompt_tps_p95": 3513050.713847771, + "decode_tps_mean": 47.324053520964824, + "decode_tps_p50": 45.28260592046214, + "decode_tps_p95": 54.383553646467476, + "request_tps_mean": 45.013305014436966, + "request_tps_p50": 44.56718732154595, + "request_tps_p95": 47.927692834210475, + "aggregate_output_tps_mean": 45.00738945185462, + "aggregate_output_tps_p50": 44.56227530114212, + "aggregate_output_tps_p95": 47.92094820555752 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T134334Z/raw.csv b/benchmarks/inference/results/20260831T134334Z/raw.csv new file mode 100644 index 0000000..205cb75 --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/raw.csv @@ -0,0 +1,8 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T134334Z,ollama,qwen3.8:27b-mlx,warmup,1,0,0,c3b04207-976b-4a0d-9d3c-80ef8dc4225f,short,106,128,2026-08-31T13:43:39.472249+00:00,3.4916902080294676,5.4140321660088375,20,96,114.72541286980045,48.19996255746575,17.731701079044402,5.4149448340176605,17.728712469407014, +20260831T134334Z,ollama,qwen3.8:27b-mlx,warmup,1,1,0,47d529a2-f167-46a0-af05-a6ca0e1929a4,medium,252,128,2026-08-31T13:43:44.886979+00:00,0.35504300001775846,2.7301927499938756,43,128,157.93211765391837,52.61516193511847,46.88313673101913,2.730699084000662,46.87444352618714, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,0,0,5b638c8b-a185-48d7-8625-f6ebacc06853,short,106,128,2026-08-31T13:43:47.617556+00:00,0.11980204097926617,2.1940376659622416,20,96,441.616869764425,45.28260592046214,43.7549461840698,2.1943347500055097,43.749022340260055, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,1,0,fd1c62de-2412-4243-85a0-0851e2364f7c,medium,252,128,2026-08-31T13:43:49.811979+00:00,0.12685995898209512,2.771603333996609,43,128,831.312776076798,47.52723693214773,46.18265479404875,2.771931625029538,46.17718519613052, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,2,0,50050b20-0fd0-4e15-a4bb-ddfc74ada7e1,code,267,128,2026-08-31T13:43:52.584034+00:00,0.42160862497985363,2.646599249972496,54,128,159.3484419263456,56.09763282504741,48.363952344250905,2.6469858330092393,48.35688895791427, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,3,0,22721212-cd1e-460f-9aa3-f2a1bb6f2f96,short,106,128,2026-08-31T13:43:55.231045+00:00,0.09599920798791572,2.2750009580049664,20,96,2500000.0,42.69653289142422,42.19778442826943,2.2753357499605045,42.19157546382611, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,4,0,d95f9110-8465-4bf1-8b6d-511699a58906,medium,252,128,2026-08-31T13:43:57.506343+00:00,0.09756654204102233,2.8720681670238264,43,128,3766313.3923097136,45.01625903574264,44.56718732154595,2.8723847499932162,44.56227530114212, diff --git a/benchmarks/inference/results/20260831T134334Z/raw.json b/benchmarks/inference/results/20260831T134334Z/raw.json new file mode 100644 index 0000000..fd3d13e --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/raw.json @@ -0,0 +1,170 @@ +[ + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "c3b04207-976b-4a0d-9d3c-80ef8dc4225f", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:39.472249+00:00", + "ttft_s": 3.4916902080294676, + "total_s": 5.4140321660088375, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 114.72541286980045, + "decode_tps": 48.19996255746575, + "request_tps": 17.731701079044402, + "group_elapsed_s": 5.4149448340176605, + "group_output_tps": 17.728712469407014, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "47d529a2-f167-46a0-af05-a6ca0e1929a4", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:44.886979+00:00", + "ttft_s": 0.35504300001775846, + "total_s": 2.7301927499938756, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 157.93211765391837, + "decode_tps": 52.61516193511847, + "request_tps": 46.88313673101913, + "group_elapsed_s": 2.730699084000662, + "group_output_tps": 46.87444352618714, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "5b638c8b-a185-48d7-8625-f6ebacc06853", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:47.617556+00:00", + "ttft_s": 0.11980204097926617, + "total_s": 2.1940376659622416, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 441.616869764425, + "decode_tps": 45.28260592046214, + "request_tps": 43.7549461840698, + "group_elapsed_s": 2.1943347500055097, + "group_output_tps": 43.749022340260055, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "fd1c62de-2412-4243-85a0-0851e2364f7c", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:49.811979+00:00", + "ttft_s": 0.12685995898209512, + "total_s": 2.771603333996609, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 831.312776076798, + "decode_tps": 47.52723693214773, + "request_tps": 46.18265479404875, + "group_elapsed_s": 2.771931625029538, + "group_output_tps": 46.17718519613052, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "50050b20-0fd0-4e15-a4bb-ddfc74ada7e1", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:52.584034+00:00", + "ttft_s": 0.42160862497985363, + "total_s": 2.646599249972496, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 159.3484419263456, + "decode_tps": 56.09763282504741, + "request_tps": 48.363952344250905, + "group_elapsed_s": 2.6469858330092393, + "group_output_tps": 48.35688895791427, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "22721212-cd1e-460f-9aa3-f2a1bb6f2f96", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:55.231045+00:00", + "ttft_s": 0.09599920798791572, + "total_s": 2.2750009580049664, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2500000.0, + "decode_tps": 42.69653289142422, + "request_tps": 42.19778442826943, + "group_elapsed_s": 2.2753357499605045, + "group_output_tps": 42.19157546382611, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "d95f9110-8465-4bf1-8b6d-511699a58906", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:57.506343+00:00", + "ttft_s": 0.09756654204102233, + "total_s": 2.8720681670238264, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3766313.3923097136, + "decode_tps": 45.01625903574264, + "request_tps": 44.56718732154595, + "group_elapsed_s": 2.8723847499932162, + "group_output_tps": 44.56227530114212, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T134334Z/summary.json b/benchmarks/inference/results/20260831T134334Z/summary.json new file mode 100644 index 0000000..b7eed98 --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/summary.json @@ -0,0 +1,28 @@ +[ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.1723672749940306, + "ttft_s_p50": 0.11980204097926617, + "ttft_s_p95": 0.36265889178030186, + "total_s_mean": 2.551861874992028, + "total_s_p50": 2.646599249972496, + "total_s_p95": 2.8519752004183827, + "prompt_tps_mean": 1253549.1340794961, + "prompt_tps_p50": 831.312776076798, + "prompt_tps_p95": 3513050.713847771, + "decode_tps_mean": 47.324053520964824, + "decode_tps_p50": 45.28260592046214, + "decode_tps_p95": 54.383553646467476, + "request_tps_mean": 45.013305014436966, + "request_tps_p50": 44.56718732154595, + "request_tps_p95": 47.927692834210475, + "aggregate_output_tps_mean": 45.00738945185462, + "aggregate_output_tps_p50": 44.56227530114212, + "aggregate_output_tps_p95": 47.92094820555752 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/backend_verification.json b/benchmarks/inference/results/backend_verification.json new file mode 100644 index 0000000..e435a67 --- /dev/null +++ b/benchmarks/inference/results/backend_verification.json @@ -0,0 +1,37 @@ +{ + "verified_at": "2026-08-31", + "host": "Apple M4 Max, arm64, macOS 15.6", + "ollama": { + "version": "0.32.14", + "mlx_version": "0.32.0-213-gadf21de", + "device": "gpu", + "metal_library": true, + "nvfp4_model": { + "name": "qwen3.8:27b-mlx", + "id": "5642e97495e1", + "format": "safetensors", + "quantization": "nvfp4", + "parameter_size": "27.8B", + "runner_log": "starting mlx runner subprocess" + }, + "mxfp8_model": { + "name": "qwen3.8:27b-mxfp8", + "id": "464021588235", + "format": "safetensors", + "quantization": "mxfp8", + "parameter_size": "27.8B", + "runner_log": "starting mlx runner subprocess" + } + }, + "vllm_metal": { + "vllm_version": "0.28.0+cpu", + "plugin_version": "0.4.0.dev20260831112704", + "mlx_version": "0.32.0", + "mlx_lm_version": "0.31.3", + "mlx_vlm_version": "0.6.4", + "platform_plugin": "metal", + "compute_log": "MLX device set to: Device(gpu, 0)", + "pytorch_device_log": "PyTorch device set to: mps", + "note": "The +cpu suffix identifies the portable vLLM core wheel; the Metal plugin supplies MLX GPU execution." + } +} diff --git a/benchmarks/inference/results/cold_start.json b/benchmarks/inference/results/cold_start.json new file mode 100644 index 0000000..bd9bfa5 --- /dev/null +++ b/benchmarks/inference/results/cold_start.json @@ -0,0 +1,27 @@ +{ + "measured_at": "2026-08-31", + "notes": [ + "Cold starts were measured separately from warm inference.", + "The vLLM model was already present in the Hugging Face cache; zero bytes were downloaded.", + "vLLM launch-to-ready is derived from the recorded epoch and second-resolution API startup log.", + "Ollama's daemon was already running, but the model was fully unloaded before the request.", + "The Ollama request used one output token and keep_alive=0 so generation time was negligible and the model was unloaded afterward." + ], + "vllm_metal": { + "process_start_epoch": 1788177244.465068, + "api_ready_local_time": "2026-08-31T13:54:33+02:00", + "approx_launch_to_api_ready_s": 28.535, + "model_load_s": 3.25, + "engine_profile_cache_warmup_s": 9.71, + "download_bytes": 0 + }, + "ollama": { + "client_total_s": 3.455159667, + "server_total_s": 3.448873125, + "model_load_s": 3.244982292, + "prompt_eval_s": 0.13364925, + "decode_s": 0.069530292, + "prompt_tokens": 7, + "output_tokens": 1 + } +}