diff --git a/.dockerignore b/.dockerignore index e913f96..1089dc5 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,8 @@ .git .env .venv -.docker-tmp -__pycache__ .pytest_cache .ruff_cache -workspaces +old_2026-08-31 +tests +__pycache__ \ No newline at end of file diff --git a/.env.example b/.env.example index ee4a771..08de28b 100644 --- a/.env.example +++ b/.env.example @@ -1,27 +1,36 @@ -FAILURE_DB_HOST=host.docker.internal -FAILURE_DB_PORT=33060 -FAILURE_DB_NAME=laravel_ai_chat -FAILURE_DB_USER=laravel -FAILURE_DB_PASSWORD= - -SOURCE_REPOSITORY=https://gitea.tisinfra.de/christian.meyer/lai.git +SOURCE_REPOSITORY=https://gitea.example.test/owner/lai.git SOURCE_BRANCH=main -GITEA_URL=https://gitea.tisinfra.de -GITEA_OWNER=christian.meyer +GITEA_URL=https://gitea.example.test +GITEA_OWNER=owner GITEA_REPOSITORY=lai GITEA_USERNAME=automation-bot GITEA_TOKEN= -# dry-run, pr, direct-main -AUTOMATION_MODE=dry-run -ALLOW_DIRECT_MAIN=false +GITEA_QUEUED_LABEL=ai:queued +GITEA_RUNNING_LABEL=ai:running +GITEA_READY_LABEL=ai:ready +GITEA_FAILED_LABEL=ai:failed +POLL_SECONDS=60 +LOG_LEVEL=INFO -AGENT_COMMAND=aider AGENT_MODEL=mistral/devstral-latest MISTRAL_API_KEY= +# Optional OpenAI-compatible endpoint instead of Mistral: +AGENT_MODEL_API_BASE= +AGENT_COMMAND_TIMEOUT_SECONDS=180 +AGENT_TIMEOUT_SECONDS=1200 +AGENT_STEP_LIMIT=50 +AGENT_COST_LIMIT=5 +MAX_AGENT_ATTEMPTS=3 -POLL_SECONDS=60 -STALE_CLAIM_MINUTES=90 +SETUP_COMMANDS=composer install --no-interaction --prefer-dist;;npm ci +TEST_COMMANDS=npm run build;;php artisan test +TEST_DB_HOST=laravel-ai-chat-mysql +TEST_DB_PORT=3306 +TEST_DB_NAME=laravel_ai_chat_agent_test +TEST_DB_USER=lai_test_runner +TEST_DB_PASSWORD=agent_test_secret +TEST_CUSTOMER_DB=customer_agent_test +TEST_CONFIG_DB=cockpit_agent_test MAX_CHANGED_FILES=12 MAX_CHANGED_LINES=800 -TEST_COMMANDS=composer install --no-interaction --prefer-dist;;php artisan test diff --git a/.gitignore b/.gitignore index a6928d8..f4df433 100644 --- a/.gitignore +++ b/.gitignore @@ -1,9 +1,9 @@ .env .venv/ -__pycache__/ .pytest_cache/ .ruff_cache/ -.docker-tmp/ -.docker-buildx/ -*.pyc -workspaces/ +__pycache__/ +*.py[cod] +*.egg-info/ +.coverage +htmlcov/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index c8537be..7d37b8c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,17 +2,18 @@ FROM python:3.12-slim RUN apt-get update \ && apt-get install -y --no-install-recommends \ - composer git nodejs npm openssh-client php-cli php-curl php-mbstring php-mysql php-xml php-zip unzip \ + composer git nodejs npm php-cli php-curl php-mbstring php-mysql php-xml php-zip unzip \ && rm -rf /var/lib/apt/lists/* WORKDIR /app COPY pyproject.toml README.md ./ COPY src ./src -RUN pip install --no-cache-dir ".[agent]" +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --retries 10 --timeout 120 . RUN useradd --create-home --uid 10001 coder \ - && mkdir -p /workspaces \ - && chmod 700 /home/coder + && install -d -m 0755 /workspaces \ + && chmod 0700 /home/coder ENTRYPOINT ["lai-agent"] CMD ["worker"] diff --git a/README.md b/README.md index 89e2051..d96cde1 100644 --- a/README.md +++ b/README.md @@ -1,170 +1,58 @@ -# LAI Automatisierung +# LAI Coding Automation -Kontrollierter Coding-Agent fuer Failure Reports aus -[`lai`](https://gitea.tisinfra.de/christian.meyer/lai). Der Orchestrator verwendet -[Aider](https://aider.chat/) headless als austauschbaren Coding-Kern. Git-, Gitea-, -Policy- und Freigabelogik bleiben in diesem Projekt. +This service turns explicitly queued Gitea issues into reviewed pull requests. It no longer reads the +Laravel production database and does not merge changes automatically. -## Warum Aider? +## Workflow -- scriptbarer Einmal-Auftrag mit `--message-file` -- gute Git- und Repository-Map-Unterstuetzung -- Mistral ueber LiteLLM nutzbar -- automatische Commits koennen deaktiviert werden -- deutlich kleinerer Betriebsumfang als OpenHands +1. LAI creates one unlabeled Gitea issue for a wrong-answer report. +2. An administrator reviews it and explicitly applies `ai:queued`, either through the LAI admin dashboard or directly in Gitea. +3. This worker replaces the label with `ai:running`, clones `main`, and creates `ai/issue-N`. +4. mini-SWE-agent 2.4.6 implements the issue in a credential-free, unprivileged command environment. +5. The worker deterministically removes trailing spaces from newly changed text, then rejects any + remaining whitespace errors, protected paths, symlinks, empty patches, and oversized diffs. +6. Deterministic setup and test commands run with isolated test database credentials. +7. A pull request containing `Fixes #N`, the original problem, changed files, and checks is opened. +8. The issue is labeled `ai:ready` or `ai:failed` and receives a status comment. -OpenHands bleibt eine moegliche spaetere Alternative fuer komplexere, mehrstufige -Agentenlaeufe. Sein Headless-Modus arbeitet jedoch immer mit `always-approve`; deshalb -muessen auch dort unsere externen Policy-Gates erhalten bleiben. +Normal reports are never selected automatically. The manual `ai:queued` label is the explicit approval and the only worker trigger. -## Ablauf +Human review and merge are always required. -```text -failure_reports(open) - -> transaktionales Claiming (SKIP LOCKED) - -> anonymisierter Agentenauftrag - -> frischer Clone von lai.git - -> Aider-Aenderung ohne Commit - -> Pfad-, Groessen- und Sicherheitsgates - -> Tests - -> Commit - -> PR oder kontrollierter Push auf main - -> failure_reports(resolved/agent_failed) -``` +## Security boundary -Der Coding-Agent erhaelt keinen Zugang zur ERP- oder Kundendatenbank. Rohdaten und -`.env`-Dateien sind ausgeschlossen. Repository-Pushes verwenden `GIT_ASKPASS`; Tokens -werden nicht in Remote-URLs oder Prozessargumenten gespeichert. Der Orchestrator laeuft -als privilegierter Containerprozess, startet Aider aber als separaten Benutzer `coder`. -Dieser Unterprozess erhaelt nur Modell-Credentials, nicht die Failure-DB- oder -Gitea-Zugangsdaten. +- Gitea and model credentials exist only in the root worker process. +- Agent-issued commands receive a strict environment without inherited worker variables. +- The agent runs as the unprivileged `coder` user. +- Git metadata lives outside the writable worktree, and Git hooks are disabled for commits. +- The Docker socket is not mounted. Repository cloning and pushing use HTTPS with a temporary + `GIT_ASKPASS` credential file that is deleted immediately. +- `.env`, CI, Docker/deployment files, dependencies, generated storage, and symlinks are rejected. -Die nachgelagerten Testprozesse erhalten separat konfigurierte `TEST_DB_*`-Zugangsdaten. -Dieser Benutzer darf ausschliesslich die isolierten Testschemas bearbeiten. Fuer Laravel -werden daraus `DB_*`, `ERP_DB_*` und `COCKPIT_CONFIG_DATABASE` gesetzt; Aider selbst sieht -diese Werte nie. Produktive ERP- oder Kundenzugangsdaten duerfen hier nicht verwendet werden. -Vor jedem Testlauf erstellt `npm run build` die Vite-Assets und initialisiert -`php artisan migrate:fresh --force` das isolierte Laravel-Schema neu. Deshalb muessen alle -`TEST_DB_*`-Schemas dediziert und vollstaendig entbehrlich sein. +The container is the intended execution boundary. Do not run the worker directly on a developer host. -Laravel-Tests laufen mit PHP und Composer direkt im isolierten Worker. Der Docker-Socket -des Hosts wird nicht eingebunden. Abgebrochene Claims werden nach standardmaessig 90 -Minuten wieder freigegeben; `STALE_CLAIM_MINUTES` passt dieses Intervall an. -Schlaegt die isolierte Validierung fehl, erhaelt Aider standardmaessig genau einen -Korrekturversuch mit redigierter Testausgabe. `MAX_AGENT_ATTEMPTS` ist auf 1 bis 3 begrenzt. +## Configuration -## Schnellstart +Copy `.env.example` to `.env` and set `GITEA_TOKEN` plus `MISTRAL_API_KEY`. Existing test database +settings can be retained. Commands are separated with `;;`. Laravel's default connection uses +`TEST_DB_NAME`, the ERP connection uses `TEST_CUSTOMER_DB`, and cross-schema cockpit fixtures use +`TEST_CONFIG_DB`. The worker enables database integration tests and supplies a non-secret +placeholder AI key so mocked provider-discovery tests execute deterministically. Real Gitea and +model credentials are never copied into the test or agent command environment. -```bash -cp .env.example .env -# FAILURE_DB_PASSWORD und bei Bedarf MISTRAL_API_KEY eintragen -# TEST_DB_PASSWORD fuer den Benutzer der isolierten Testschemas eintragen -docker compose build -docker compose run --rm automation check -docker compose run --rm automation once -``` +Start the worker with Docker Compose. Use `lai-agent check` inside the image for configuration +validation, `lai-agent preflight` for clean-clone setup and tests without a model call, or +`lai-agent once` for a single polling cycle. -Der Default `AUTOMATION_MODE=dry-run` claimt einen offenen Report, erzeugt den -anonymisierten Auftrag und setzt ihn danach wieder auf `open`. Es wird weder ein Agent -gestartet noch gepusht. +Set `LOG_LEVEL=DEBUG` to include redacted output from setup and test commands. Agent runs always +log their model, configured limits, call count, cost, exit status, and trajectory path. `INFO` is +the default for normal operation. -## Fortschritt beobachten +`MAX_AGENT_ATTEMPTS` controls validation-driven self-repair. When an agent submission fails policy, +diff, or test validation, the worker preserves the workspace and gives the redacted failure output +to a fresh agent attempt. Only the final failed attempt marks the issue as `ai:failed` and removes +the workspace. -```bash -docker compose ps automation -docker compose exec automation lai-agent check -docker compose logs -f automation -``` - -Der Worker protokolliert nur sichere Metadaten, keine Prompts, SQL-Ergebnisse, Secrets oder -Modellausgaben. Fuer jeden Report erscheinen Workspace-Vorbereitung, Agentenversuch, Anzahl -und Pfade geaenderter Dateien, Policy-Pruefung, jeder Validierungsbefehl, Commit, PR-URL und -Cleanup. Bei laengeren Agenten- oder Testphasen wird alle 30 Sekunden Laufzeitfortschritt -ausgegeben. `agent_running` ohne Heartbeat oder Unterprozess deutet auf einen unterbrochenen -Container hin; normale Claims werden nach `STALE_CLAIM_MINUTES` wieder freigegeben. - -## PR-Modus - -```dotenv -AUTOMATION_MODE=pr -GITEA_USERNAME=automation-bot -GITEA_TOKEN=... -MISTRAL_API_KEY=... -``` - -Der Bot benoetigt Schreibzugriff auf `christian.meyer/lai` und die Berechtigung, -Branches sowie Pull Requests anzulegen. Jeder Lauf erstellt -`ai/failure-{report_id}` und einen PR gegen `main`. - -## Direkter main-Modus - -```dotenv -AUTOMATION_MODE=direct-main -ALLOW_DIRECT_MAIN=true -GITEA_TOKEN=... -``` - -Der zweite Schalter ist ein Kill-Switch. In diesem Modus sind Migrationen, Routes, -Auth, Middleware, PermissionService und SQLValidatorService zusaetzlich blockiert. -Branch Protection muss den Bot fuer direkte Pushes explizit zulassen. PR-Modus bleibt -die empfohlene Betriebsart. - -## Modelle - -Das Modell wird ausschliesslich ueber `AGENT_MODEL` gewaehlt. Devstral ist der -Default und ein Coding-Agent, nicht das Chat-Modell der Laravel-Anwendung: - -```dotenv -AGENT_MODEL=mistral/devstral-latest -MISTRAL_API_KEY=... -``` - -Codestral verwendet denselben Mistral-Zugang: - -```dotenv -AGENT_MODEL=mistral/codestral-latest -MISTRAL_API_KEY=... -``` - -Qwen 3.5 27B ueber OpenRouter: - -```dotenv -AGENT_MODEL=openrouter/qwen/qwen3.5-27b -OPENROUTER_API_KEY=... -``` - -Oder mit einem lokalen Ollama auf dem Docker-Host: - -```dotenv -AGENT_MODEL=ollama_chat/qwen3.5:27b -OLLAMA_API_BASE=http://host.docker.internal:11434 -``` - -Nach einer Aenderung wird der Worker neu erstellt, damit Compose die neue Umgebung -uebernimmt: - -```bash -docker compose up -d --force-recreate automation -docker compose exec -T automation lai-agent check -``` - -Aider waehlt das passende Edit-Format anhand des Modells. Das Chat-Modell und -Coding-Modell sind bewusst getrennt. Fuer Coding-Agenten sollte ein -agentisch/coding-optimiertes Modell verwendet werden. Ein Wechsel zu OpenHands oder -einem anderen Backend erfordert einen neuen Adapter neben `AiderAgent`. - -## Tests - -```bash -python -m venv .venv -. .venv/bin/activate -pip install ".[dev]" -ruff check src tests -pytest -q -``` - -## Repositorys - -- Chat: `https://gitea.tisinfra.de/christian.meyer/lai.git` -- Automatisierung: `https://gitea.tisinfra.de/christian.meyer/lai_automatisierung.git` +## Archived implementation +The previous database-polling Aider implementation is preserved unchanged in `old_2026-08-31/`. diff --git a/benchmarks/.gitignore b/benchmarks/.gitignore new file mode 100644 index 0000000..43ae0e2 --- /dev/null +++ b/benchmarks/.gitignore @@ -0,0 +1,2 @@ +__pycache__/ +*.py[cod] diff --git a/benchmarks/inference/README.md b/benchmarks/inference/README.md new file mode 100644 index 0000000..e4ca47b --- /dev/null +++ b/benchmarks/inference/README.md @@ -0,0 +1,76 @@ +# Ollama versus vLLM-Metal benchmark + +This benchmark compares native Apple-Silicon execution through Ollama and vLLM-Metal. It does not change the Laravel application, the automation worker, or their model configuration. + +The completed 2026-08-31 comparison and recommendation are documented in `REPORT.md`. + +## Model classification + +- Ollama: `qwen3.8:27b-mlx` +- vLLM-Metal: `mlx-community/Qwen3.8-27B-nvfp4`, revision `5ff8ef173ad0d7c3aae92f0be43031a6ab8067c6` +- Both derive from Qwen3.8-27B and use nominal NVFP4 group-size-16 quantization. +- They are **not byte-identical artifacts**. A complete corresponding 44,564,480-byte packed weight tensor produced different SHA-256 hashes. Results therefore compare deployable runtime/model packages, not runtime overhead with identical bytes. + +The MXFP8 comparison uses Ollama `qwen3.8:27b-mxfp8` and +`mlx-community/Qwen3.8-27B-mxfp8`, revision +`d48d163bcdf24acaf656474854ab88ea17d65bd1`. Both declare group-size-32 MXFP8 +but are also not byte-identical. + +## Controlled protocol + +1. Run both services natively on the same otherwise-idle Apple Silicon host. +2. Use a context limit of 8,192, temperature 0, seed 42, and 128 maximum output tokens. + Use raw text completions and disable Ollama speculative drafting so neither runtime has a draft model. +3. Run two warmup rounds before measured rounds. +4. Run five measured rounds at concurrency 1 and five groups at concurrency 4. +5. Alternate three fixed prompts to reduce prefix-cache bias. +6. Compare client-observed TTFT and total latency, warmed decode speed, and aggregate output throughput. +7. Preserve metadata, prompts, per-request measurements, errors, and summaries as JSON and CSV. +8. Treat startup/model loading as a separate cold-load observation; do not mix it into warm inference percentiles. + +`benchmark.py` uses only the Python standard library. It expects Ollama at `127.0.0.1:11434` and the vLLM OpenAI-compatible API at `127.0.0.1:8001`. + +## Complex coding suite + +`coding_benchmark.py` sends the same three complex implementation tasks through each runtime's +native chat API: an asynchronous Python cache, an idempotent Laravel money transfer, and a +concurrent TypeScript DAG scheduler. Thinking and speculative drafting are disabled, temperature is +zero, and full generated responses are retained for inspection. The summary records TTFT, total +latency, output tokens, decode rate, and finish reason. + +When every response reaches the configured output limit, the suite measures time for a fixed amount +of long-form coding output rather than time to a naturally completed solution. This distinction must +be preserved in reports. + +## Direct MLX control + +`direct_mlx_benchmark.py` loads a Hugging Face checkpoint directly with MLX-LM and measures the +same raw prompts at concurrency one. Running it against the exact NVFP4 or MXFP8 checkpoint used by +vLLM isolates short-generation MLX performance from HTTP, scheduling, paged-cache, and request-state +overhead. It is not a concurrent-serving benchmark and cannot isolate Ollama because Ollama's packed +artifact is different. + +## Metrics + +- **TTFT:** request start to the first non-empty generated text chunk at the client. +- **Total latency:** request start to completion of the response stream. +- **Decode tokens/s:** Ollama's server-reported evaluation rate; for vLLM, output tokens divided by post-TTFT client time. +- **Request tokens/s:** output tokens divided by total client latency, including prefill and network overhead. +- **Aggregate output tokens/s:** all successful generated tokens in a concurrent group divided by group wall time. +- **p50/p95:** linearly interpolated percentiles over measured requests or groups; warmups are excluded. + +Prompt throughput is reported when the server exposes both prompt token count and prompt duration. A missing value is not estimated. + +## Result files + +Each run creates a timestamped directory below `results/` containing: + +- `metadata.json`: environment, arguments, model metadata, prompts, and summary +- `raw.json`: complete per-request evidence +- `raw.csv`: tabular per-request evidence +- `summary.json`: grouped statistics + +The coding suite stores equivalent evidence below `coding-results/`. Direct MLX controls store +`metadata.json`, `raw.json`, and `summary.json` below `direct-results/`. + +Do not compare runs performed under materially different memory pressure, thermal conditions, model revisions, or runtime versions. diff --git a/benchmarks/inference/REPORT.md b/benchmarks/inference/REPORT.md new file mode 100644 index 0000000..ea7fbee --- /dev/null +++ b/benchmarks/inference/REPORT.md @@ -0,0 +1,193 @@ +# Benchmark report: Ollama, vLLM-Metal, NVFP4, and MXFP8 + +Date: 2026-08-31 + +## Result + +For this Apple M4 Max and the tested Qwen3.8-27B packages: + +- **Interactive/single-request use: Ollama NVFP4 wins.** It delivered 1.83x the vLLM-Metal NVFP4 median decode speed. It was also 1.50x faster than vLLM-Metal for MXFP8. +- **Four concurrent requests: vLLM-Metal wins.** NVFP4 delivered 1.62x and MXFP8 1.73x Ollama's median aggregate output throughput, with much lower tail TTFT. +- **Quantization: NVFP4 wins decisively for speed and memory.** MXFP8 approximately doubles weight storage and reduced median single-stream decode speed by 50% in Ollama and 39% in vLLM-Metal. +- **Complex batch-1 coding: Ollama wins every measured task.** Depending on quantization and task, vLLM-Metal took 1.53x to 2.37x as long. +- **Practical choice:** use Ollama NVFP4 for interactive chat, coding, and one agent at a time. Use vLLM-Metal NVFP4 when several users or jobs generate concurrently and aggregate capacity or tail TTFT matters. MXFP8 is not attractive for speed or memory on this machine; use it only if its output quality is proven better for the target workload. + +All 50 measured requests succeeded. Warmups are excluded from the table. + +Both serving stacks were conclusively verified as native Apple GPU implementations. Ollama logged an MLX runner, MLX engine `0.32.0-213-gadf21de`, `device=gpu`, and Metal. vLLM-Metal activated its Metal plugin, set MLX to `Device(gpu, 0)`, set PyTorch to MPS, and loaded native paged-attention Metal kernels. The `0.28.0+cpu` vLLM version string describes the portable core wheel, not the active compute backend. + +## Measurements + +| Workload and metric | Ollama | vLLM-Metal | Result | +|---|---:|---:|---| +| Concurrency 1, TTFT p50 | 0.119 s | 0.372 s | Ollama 3.12x faster | +| Concurrency 1, TTFT p95 | 0.393 s | 0.420 s | Ollama 1.07x faster | +| Concurrency 1, total latency p50 | 2.754 s | 5.455 s | Ollama 1.98x faster | +| Concurrency 1, total latency p95 | 3.451 s | 5.562 s | Ollama 1.61x faster | +| Concurrency 1, decode p50 | 45.73 tok/s | 24.96 tok/s | Ollama 1.83x faster | +| Concurrency 1, aggregate output p50 | 44.19 tok/s | 23.46 tok/s | Ollama 1.88x faster | +| Concurrency 4, TTFT p50 | 5.078 s | 1.169 s | vLLM-Metal 4.34x faster | +| Concurrency 4, TTFT p95 | 10.691 s | 1.236 s | vLLM-Metal 8.65x faster | +| Concurrency 4, total latency p50 | 8.619 s | 9.273 s | Ollama 1.08x faster | +| Concurrency 4, total latency p95 | 14.521 s | 9.591 s | vLLM-Metal 1.51x faster | +| Concurrency 4, aggregate output p50 | 33.92 tok/s | 54.89 tok/s | vLLM-Metal 1.62x faster | + +The concurrency-4 median individual latency does not represent group completion: Ollama returned some requests early while queueing others. Its p95 TTFT reached 10.69 seconds. Aggregate output rate and p95 latency are the relevant capacity metrics here. + +## MXFP8 measurements + +All 50 measured MXFP8 requests succeeded. The protocol is identical to the NVFP4 test. + +| Workload and metric | Ollama | vLLM-Metal | Result | +|---|---:|---:|---| +| Concurrency 1, TTFT p50 | 0.259 s | 0.417 s | Ollama 1.61x faster | +| Concurrency 1, TTFT p95 | 0.583 s | 0.453 s | vLLM-Metal 1.29x faster | +| Concurrency 1, total latency p50 | 5.272 s | 8.839 s | Ollama 1.68x faster | +| Concurrency 1, total latency p95 | 5.728 s | 8.999 s | Ollama 1.57x faster | +| Concurrency 1, decode p50 | 22.74 tok/s | 15.13 tok/s | Ollama 1.50x faster | +| Concurrency 1, aggregate output p50 | 22.22 tok/s | 14.39 tok/s | Ollama 1.54x faster | +| Concurrency 4, TTFT p50 | 9.394 s | 1.314 s | vLLM-Metal 7.15x faster | +| Concurrency 4, TTFT p95 | 18.583 s | 1.419 s | vLLM-Metal 13.10x faster | +| Concurrency 4, total latency p50 | 14.902 s | 12.702 s | vLLM-Metal 1.17x faster | +| Concurrency 4, total latency p95 | 24.082 s | 13.642 s | vLLM-Metal 1.77x faster | +| Concurrency 4, aggregate output p50 | 20.18 tok/s | 34.96 tok/s | vLLM-Metal 1.73x faster | + +## Complex coding tasks + +These runs used native chat templates, thinking disabled, temperature 0, seed 42, an 8,192-token context, and a 3,072-token output cap. Complete generated responses are retained with their hashes. + +### NVFP4 + +| Task | Ollama total | Ollama decode | vLLM total | vLLM decode | Result by total time | +|---|---:|---:|---:|---:|---| +| Python async TTL cache | 56.41 s | 55.38 tok/s | 133.44 s | 23.20 tok/s | Ollama 2.37x faster | +| Laravel idempotent transfer | 78.72 s | 39.58 tok/s | 166.01 s | 18.92 tok/s | Ollama 2.11x faster* | +| TypeScript DAG scheduler | 96.54 s | 32.30 tok/s | 152.22 s | 20.38 tok/s | Ollama 1.58x faster | + +\* The vLLM PHP response stopped naturally at 2,840 tokens; the Ollama response reached 3,072. Its normalized decode rate was still 2.09x lower. + +### MXFP8 + +| Task | Ollama total | Ollama decode | vLLM total | vLLM decode | Result by total time | +|---|---:|---:|---:|---:|---| +| Python async TTL cache | 149.82 s | 20.64 tok/s | 247.71 s | 12.46 tok/s | Ollama 1.65x faster | +| Laravel idempotent transfer | 149.77 s | 20.72 tok/s | 242.90 s | 12.73 tok/s | Ollama 1.62x faster | +| TypeScript DAG scheduler | 147.28 s | 21.06 tok/s | 224.71 s | 13.76 tok/s | Ollama 1.53x faster | + +Except for the vLLM NVFP4 PHP response, every coding run reached the 3,072-token cap. These figures therefore measure time to produce a fixed amount of long-form code, not time to a guaranteed complete or correct implementation. The generated code was retained as evidence but was not compiled as a cross-model quality benchmark because the model artifacts produced different implementations. + +## Exact-checkpoint direct MLX control + +The direct control removes HTTP, vLLM scheduling, paged-cache management, and request-state overhead. It loads the exact same Hugging Face artifact used by vLLM through MLX-LM 0.31.3. It uses the same raw prompts, two warmups, five measured requests, temperature 0, seed 42, and 128 output tokens. + +| Quantization | Direct MLX decode p50 | vLLM decode p50 | Difference | Direct MLX peak memory | +|---|---:|---:|---:|---:| +| NVFP4 | 28.53 tok/s | 24.96 tok/s | vLLM 12.5% lower | 15.48 GB | +| MXFP8 | 15.42 tok/s | 15.13 tok/s | vLLM 1.9% lower | 28.06 GB | + +This isolates part of the NVFP4 gap as serving overhead. It also shows that short MXFP8 batch-1 decode is almost identical between direct MLX-LM and vLLM-Metal. It does not make the Ollama comparison artifact-identical: Ollama's packed weights differ. Long hybrid-GDN sequences still incur additional vLLM state and paging work that this short control does not represent. + +A fresh back-to-back NVFP4 rerun, with Ollama unloaded before direct MLX-LM started, confirmed the practical batch-1 comparison: + +| Metric | Ollama | Direct MLX-LM | Result | +|---|---:|---:|---| +| TTFT p50 | 0.120 s | 0.503 s | Ollama 4.20x faster | +| Decode p50 | 45.28 tok/s | 27.87 tok/s | Ollama 1.62x faster | +| Request throughput p50 | 44.57 tok/s | 25.36 tok/s | Ollama 1.76x faster | + +The direct MLX-LM rerun was 38.4% below Ollama's median decode throughput. This remains a practical-package comparison rather than a byte-identical runtime comparison. + +## Memory and package sizes + +The previously observed approximately 32 GB for vLLM NVFP4 was not a 32-GB model. It was model memory plus a large cache reservation. + +| Item | NVFP4 | MXFP8 | +|---|---:|---:| +| Ollama package label | about 18 GB | about 31 GB | +| Hugging Face checkpoint | 16.1 GB | 28.7 GB | +| Direct MLX peak memory | 15.48 GB | 28.06 GB | +| vLLM logged model memory | 15.13 GB | 27.74 GB | +| vLLM runtime overhead | 1.16 GB | 1.17 GB | +| vLLM logical KV/cache budget | 15.05 GB | 18.51 GB | + +For MXFP8, vLLM reported 8.94 GB as the concrete MPS KV-cache capacity inside the larger 18.51-GB logical cache budget. Its default 0.92 memory-utilization target provisioned capacity for roughly ten concurrent 8K requests, although the benchmark used at most four short requests. That reservation is serving capacity, not weight size. + +## Why vLLM-Metal is slower at batch 1 + +- Ollama uses a tightly integrated Qwen-specific MLX runner and a newer/custom MLX engine revision. +- vLLM-Metal adds scheduling, paged KV-cache management, request state, block-table translation, and hybrid Gated DeltaNet state management. +- Qwen3.8 has 16 SDPA and 48 linear/GDN layers. vLLM-Metal labels this hybrid paged-attention path experimental and translates its large logical blocks into Metal-kernel-compatible blocks. +- Some shapes can use lazy/fused Metal kernels; unsupported cases use less favorable convolution or recurrent fallback paths. +- These costs buy batching: vLLM loses single-stream latency but wins aggregate throughput and tail TTFT at concurrency four. +- The direct exact-checkpoint result confirms measurable vLLM overhead for NVFP4. The much larger Ollama advantage in long coding runs also includes different packed artifacts and different runtime integration, so it cannot be assigned exclusively to the scheduler. + +## Token-count check + +The runtimes did not always stop at the same token count because the packed artifacts generate different output and encounter EOS differently: + +| Workload | Ollama output tokens | vLLM-Metal output tokens | +|---|---:|---:| +| Five measured requests, concurrency 1 | 576 | 640 | +| Twenty measured requests, concurrency 4 | 2,340 | 2,556 | + +The fixed short prompt ended at 96–98 tokens in Ollama and at 124–128 tokens in vLLM-Metal. Therefore raw total latency alone is not a fair speed metric. Decode tokens per second and aggregate output tokens per second normalize for this difference and support the same conclusions. + +## Test protocol + +- Host: MacBook Pro, Apple M4 Max, 64 GB unified memory, macOS 15.6, arm64 +- Ollama: 0.32.14, native MLX execution +- vLLM core: 0.28.0+cpu with native `vllm-metal` 0.4.0 development plugin; logs confirmed `Device(gpu, 0)` and MPS +- Context limit: 8,192 tokens +- Sampling: temperature 0, seed 42 +- Output limit: 128 tokens +- Two warmup groups, then five measured groups at concurrency 1 and 4 +- Three fixed raw-completion prompts +- Ollama speculative drafting explicitly disabled +- Runtimes loaded and tested sequentially to prevent unified-memory contention + +The `+cpu` suffix belongs to the vLLM core wheel. In this setup, the Metal plugin supplied GPU execution; this was not a CPU benchmark. + +## Model-equivalence limitation + +- Ollama package: `qwen3.8:27b-mlx` +- vLLM-Metal package: `mlx-community/Qwen3.8-27B-nvfp4` +- Hugging Face revision: `5ff8ef173ad0d7c3aae92f0be43031a6ab8067c6` +- Both are based on Qwen3.8-27B and declare NVFP4 group-size-16 quantization. +- They are not byte-identical. A corresponding 44,564,480-byte packed weight tensor had different SHA-256 hashes. + +This is a comparison of the closest practical native runtime/model packages, not a pure runtime-overhead comparison using identical packed weights. Quantization/conversion differences can account for some of the speed difference. + +## Cold start + +Cold startup was measured separately after both model packages were cached locally: + +| Measurement | Ollama | vLLM-Metal | +|---|---:|---:| +| Model-weight load | 3.245 s | 3.25 s | +| Operational readiness | 3.455 s for a cold one-token request | approximately 28.5 s from process launch to API ready | + +The model-weight load itself was effectively tied. vLLM-Metal additionally spent 9.71 seconds profiling, allocating its 15-GB KV cache, and warming kernels; Python process and plugin startup account for the remaining launch time. Ollama's daemon was already running, so its value measures unloaded-model-to-response rather than daemon process launch. These operational measurements are useful but not perfectly symmetric. + +The first vLLM setup took about 12 minutes 45 seconds to download and reconstruct the 16.1-GB checkpoint. That one-time network cost is excluded from the cached cold-start and all warm-inference results. + +## Caveats + +- Prompt-throughput values from Ollama were excluded because repeated short prompts produced cache-hit durations near zero; vLLM's completion stream did not expose equivalent per-request prefill duration. +- Five groups establish a clear directional result, but longer multi-run tests are appropriate before capacity planning. +- Both runtimes used prefix caching as shipped. That reflects practical warmed operation, not cache-disabled kernel isolation. +- Total latency is influenced by differing EOS positions; normalized token rates are more comparable. +- Ollama and Hugging Face model pairs share the Qwen3.8-27B base and nominal quantization mode but are not byte-identical. Results compare practical deployable packages, not only runtime code. +- Direct MLX controls are batch-1 controls only; MLX-LM does not reproduce vLLM's concurrent serving objective. + +## Evidence + +- vLLM-Metal: `results/20260831T114727Z/metadata.json`, `raw.json`, `raw.csv`, and `summary.json` +- Ollama: `results/20260831T114928Z/metadata.json`, `raw.json`, `raw.csv`, and `summary.json` +- Cold start: `results/cold_start.json` +- Backend proof: `results/backend_verification.json` +- Ollama MXFP8: `results/20260831T121855Z/` +- vLLM-Metal MXFP8: `results/20260831T132015Z/` +- NVFP4 coding: `coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/` and `coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/` +- MXFP8 coding: `coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/` and `coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/` +- Direct MLX controls: `direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/` and `direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/` +- Back-to-back rerun: `results/20260831T134334Z/` and `direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/` diff --git a/benchmarks/inference/benchmark.py b/benchmarks/inference/benchmark.py new file mode 100644 index 0000000..f239ad1 --- /dev/null +++ b/benchmarks/inference/benchmark.py @@ -0,0 +1,500 @@ +#!/usr/bin/env python3 +"""Reproducible HTTP benchmark for Ollama and OpenAI-compatible runtimes.""" + +from __future__ import annotations + +import argparse +import concurrent.futures +import csv +import json +import math +import platform +import statistics +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +PROMPTS = { + "short": ( + "In exactly four concise bullet points, explain why database transactions are useful. " + "Do not use a heading." + ), + "medium": ( + "You are reviewing a Laravel service that transfers money between two accounts. " + "Explain the failure modes involving concurrency, partial writes, retries, and audit logs. " + "Then provide a robust implementation plan. Keep the response technical and concise." + ), + "code": ( + "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, " + "validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. " + "Include type hints, a short docstring, and five assert-based examples. Output only code." + ), +} + +CSV_FIELDS = ( + "run_id", + "runtime", + "model", + "phase", + "concurrency", + "round", + "slot", + "request_id", + "prompt_name", + "prompt_chars", + "max_tokens", + "started_at", + "ttft_s", + "total_s", + "prompt_tokens", + "output_tokens", + "prompt_tps", + "decode_tps", + "request_tps", + "group_elapsed_s", + "group_output_tps", + "error", +) + + +@dataclass(frozen=True) +class RuntimeConfig: + name: str + kind: str + base_url: str + model: str + + +@dataclass +class Measurement: + run_id: str + runtime: str + model: str + phase: str + concurrency: int + round: int + slot: int + request_id: str + prompt_name: str + prompt_chars: int + max_tokens: int + started_at: str + ttft_s: float | None = None + total_s: float | None = None + prompt_tokens: int | None = None + output_tokens: int | None = None + prompt_tps: float | None = None + decode_tps: float | None = None + request_tps: float | None = None + group_elapsed_s: float | None = None + group_output_tps: float | None = None + error: str | None = None + + +def utc_now() -> str: + return datetime.now(UTC).isoformat() + + +def percentile(values: list[float], fraction: float) -> float | None: + if not values: + return None + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def safe_version(command: list[str]) -> str | None: + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=15, check=False) + except (OSError, subprocess.SubprocessError): + return None + value = (result.stdout or result.stderr).strip() + return value or None + + +def http_json(url: str, timeout: float = 15) -> dict[str, Any]: + request = urllib.request.Request(url, headers={"Accept": "application/json"}) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + + +def read_ndjson(response: Any, started: float) -> tuple[float | None, dict[str, Any], int]: + first_token_at: float | None = None + output_chars = 0 + terminal: dict[str, Any] = {} + for raw_line in response: + if not raw_line.strip(): + continue + event = json.loads(raw_line) + content = event.get("response", "") + if content: + output_chars += len(content) + if first_token_at is None: + first_token_at = time.perf_counter() + if event.get("done"): + terminal = event + return (first_token_at - started if first_token_at else None, terminal, output_chars) + + +def read_sse(response: Any, started: float) -> tuple[float | None, dict[str, Any], int]: + first_token_at: float | None = None + output_chars = 0 + terminal: dict[str, Any] = {} + for raw_line in response: + line = raw_line.decode("utf-8").strip() + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + break + event = json.loads(payload) + terminal = event + choices = event.get("choices") or [] + if choices: + text = choices[0].get("text") or choices[0].get("delta", {}).get("content") or "" + if text: + output_chars += len(text) + if first_token_at is None: + first_token_at = time.perf_counter() + return (first_token_at - started if first_token_at else None, terminal, output_chars) + + +def benchmark_request( + runtime: RuntimeConfig, + prompt_name: str, + prompt: str, + max_tokens: int, + seed: int, + timeout: float, + run_id: str, + phase: str, + concurrency: int, + round_number: int, + slot: int, +) -> Measurement: + measurement = Measurement( + run_id=run_id, + runtime=runtime.name, + model=runtime.model, + phase=phase, + concurrency=concurrency, + round=round_number, + slot=slot, + request_id=str(uuid.uuid4()), + prompt_name=prompt_name, + prompt_chars=len(prompt), + max_tokens=max_tokens, + started_at=utc_now(), + ) + headers = {"Content-Type": "application/json", "Accept": "text/event-stream"} + if runtime.kind == "ollama": + url = f"{runtime.base_url.rstrip('/')}/api/generate" + payload = { + "model": runtime.model, + "prompt": prompt, + "stream": True, + "raw": True, + "keep_alive": "30m", + "options": { + "temperature": 0, + "seed": seed, + "num_predict": max_tokens, + "num_ctx": 8192, + "draft_num_predict": 0, + }, + } + else: + url = f"{runtime.base_url.rstrip('/')}/v1/completions" + payload = { + "model": runtime.model, + "prompt": prompt, + "stream": True, + "stream_options": {"include_usage": True}, + "temperature": 0, + "seed": seed, + "max_tokens": max_tokens, + } + + started = time.perf_counter() + try: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers=headers, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + if runtime.kind == "ollama": + ttft, terminal, output_chars = read_ndjson(response, started) + measurement.prompt_tokens = terminal.get("prompt_eval_count") + measurement.output_tokens = terminal.get("eval_count") + prompt_ns = terminal.get("prompt_eval_duration") + decode_ns = terminal.get("eval_duration") + if prompt_ns and measurement.prompt_tokens: + measurement.prompt_tps = measurement.prompt_tokens / (prompt_ns / 1e9) + if decode_ns and measurement.output_tokens: + measurement.decode_tps = measurement.output_tokens / (decode_ns / 1e9) + else: + ttft, terminal, output_chars = read_sse(response, started) + usage = terminal.get("usage") or {} + measurement.prompt_tokens = usage.get("prompt_tokens") + measurement.output_tokens = usage.get("completion_tokens") + measurement.ttft_s = ttft + measurement.total_s = time.perf_counter() - started + if measurement.output_tokens and measurement.total_s: + measurement.request_tps = measurement.output_tokens / measurement.total_s + if measurement.decode_tps is None and measurement.ttft_s is not None: + decode_time = measurement.total_s - measurement.ttft_s + if decode_time > 0: + measurement.decode_tps = measurement.output_tokens / decode_time + if output_chars == 0: + measurement.error = "stream contained no generated text" + except urllib.error.HTTPError as error: + detail = error.read().decode("utf-8", errors="replace")[:1000] + measurement.error = f"HTTP {error.code}: {detail}" + measurement.total_s = time.perf_counter() - started + except (OSError, ValueError, json.JSONDecodeError) as error: + measurement.error = f"{type(error).__name__}: {error}" + measurement.total_s = time.perf_counter() - started + return measurement + + +def run_round( + runtime: RuntimeConfig, + prompt_items: list[tuple[str, str]], + max_tokens: int, + seed: int, + timeout: float, + run_id: str, + phase: str, + concurrency: int, + round_number: int, +) -> tuple[list[Measurement], float]: + barrier = threading.Barrier(concurrency) + + def execute(slot: int) -> Measurement: + prompt_name, prompt = prompt_items[(round_number * concurrency + slot) % len(prompt_items)] + barrier.wait() + return benchmark_request( + runtime, + prompt_name, + prompt, + max_tokens, + seed, + timeout, + run_id, + phase, + concurrency, + round_number, + slot, + ) + + group_started = time.perf_counter() + with concurrent.futures.ThreadPoolExecutor(max_workers=concurrency) as executor: + measurements = list(executor.map(execute, range(concurrency))) + group_elapsed = time.perf_counter() - group_started + successful_tokens = sum(item.output_tokens or 0 for item in measurements if not item.error) + group_tps = successful_tokens / group_elapsed if group_elapsed else 0.0 + for item in measurements: + item.group_elapsed_s = group_elapsed + item.group_output_tps = group_tps + return measurements, group_elapsed + + +def summarize(measurements: list[Measurement]) -> list[dict[str, Any]]: + groups: dict[tuple[str, int], list[Measurement]] = {} + for measurement in measurements: + if measurement.phase != "measured": + continue + groups.setdefault((measurement.runtime, measurement.concurrency), []).append(measurement) + + summaries: list[dict[str, Any]] = [] + for (runtime, concurrency), items in sorted(groups.items()): + successful = [item for item in items if not item.error] + summary: dict[str, Any] = { + "runtime": runtime, + "model": items[0].model, + "concurrency": concurrency, + "requests": len(items), + "successes": len(successful), + "failures": len(items) - len(successful), + } + for field in ("ttft_s", "total_s", "prompt_tps", "decode_tps", "request_tps"): + values = [ + float(value) + for item in successful + if (value := getattr(item, field)) is not None + ] + summary[f"{field}_mean"] = statistics.fmean(values) if values else None + summary[f"{field}_p50"] = percentile(values, 0.50) + summary[f"{field}_p95"] = percentile(values, 0.95) + group_tps = [ + float(item.group_output_tps) + for item in successful + if item.slot == 0 and item.group_output_tps is not None + ] + summary["aggregate_output_tps_mean"] = statistics.fmean(group_tps) if group_tps else None + summary["aggregate_output_tps_p50"] = percentile(group_tps, 0.50) + summary["aggregate_output_tps_p95"] = percentile(group_tps, 0.95) + summaries.append(summary) + return summaries + + +def write_csv(path: Path, measurements: list[Measurement]) -> None: + with path.open("w", newline="", encoding="utf-8") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_FIELDS, lineterminator="\n") + writer.writeheader() + for measurement in measurements: + writer.writerow(asdict(measurement)) + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, default=Path(__file__).parent / "results") + parser.add_argument("--runtime", choices=("all", "ollama", "vllm"), default="all") + parser.add_argument("--ollama-url", default="http://127.0.0.1:11434") + parser.add_argument("--ollama-model", default="qwen3.8:27b-mlx") + parser.add_argument("--vllm-url", default="http://127.0.0.1:8001") + parser.add_argument("--vllm-model", default="qwen3.8-27b-nvfp4") + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--rounds", type=int, default=5) + parser.add_argument("--concurrency", type=int, nargs="+", default=(1, 4)) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--timeout", type=float, default=600) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.warmups < 0 or args.rounds < 1 or any(value < 1 for value in args.concurrency): + raise SystemExit("warmups must be >= 0; rounds and concurrency must be >= 1") + + runtimes = { + "ollama": RuntimeConfig("ollama", "ollama", args.ollama_url, args.ollama_model), + "vllm": RuntimeConfig("vllm-metal", "openai", args.vllm_url, args.vllm_model), + } + selected = list(runtimes.values()) if args.runtime == "all" else [runtimes[args.runtime]] + for runtime in selected: + health_url = ( + f"{runtime.base_url.rstrip('/')}/api/tags" + if runtime.kind == "ollama" + else f"{runtime.base_url.rstrip('/')}/v1/models" + ) + try: + http_json(health_url) + except Exception as error: + raise SystemExit(f"{runtime.name} is not ready at {health_url}: {error}") from error + + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + output_dir = args.output_dir / run_id + output_dir.mkdir(parents=True, exist_ok=False) + metadata = { + "run_id": run_id, + "created_at": utc_now(), + "host": { + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "arguments": vars(args) | {"output_dir": str(args.output_dir)}, + "runtimes": [asdict(runtime) for runtime in selected], + "versions": { + "ollama": safe_version(["ollama", "--version"]), + "vllm": safe_version([str(Path.home() / ".venv-vllm-metal/bin/vllm"), "--version"]), + }, + "model_metadata": {}, + "prompts": PROMPTS, + "measurement_notes": { + "ttft": ( + "Client wall time from request start until first non-empty generated text chunk." + ), + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time.", + }, + } + for runtime in selected: + url = ( + f"{runtime.base_url.rstrip('/')}/api/show" + if runtime.kind == "ollama" + else f"{runtime.base_url.rstrip('/')}/v1/models" + ) + try: + if runtime.kind == "ollama": + request = urllib.request.Request( + url, + data=json.dumps({"model": runtime.model}).encode(), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: + metadata["model_metadata"][runtime.name] = json.load(response) + else: + metadata["model_metadata"][runtime.name] = http_json(url) + except Exception as error: + metadata["model_metadata"][runtime.name] = {"metadata_error": str(error)} + + prompt_items = list(PROMPTS.items()) + measurements: list[Measurement] = [] + total_groups = len(selected) * sum(args.warmups + args.rounds for _ in args.concurrency) + completed_groups = 0 + for runtime in selected: + for concurrency in args.concurrency: + for phase, rounds in (("warmup", args.warmups), ("measured", args.rounds)): + for round_number in range(rounds): + items, elapsed = run_round( + runtime, + prompt_items, + args.max_tokens, + args.seed, + args.timeout, + run_id, + phase, + concurrency, + round_number, + ) + measurements.extend(items) + completed_groups += 1 + errors = sum(bool(item.error) for item in items) + print( + f"[{completed_groups}/{total_groups}] {runtime.name} phase={phase} " + f"concurrency={concurrency} round={round_number + 1}/{rounds} " + f"elapsed={elapsed:.3f}s errors={errors}", + flush=True, + ) + + summaries = summarize(measurements) + metadata["completed_at"] = utc_now() + metadata["summary"] = summaries + (output_dir / "raw.json").write_text( + json.dumps([asdict(item) | { + "group_elapsed_s": getattr(item, "group_elapsed_s", None), + "group_output_tps": getattr(item, "group_output_tps", None), + } for item in measurements], indent=2), + encoding="utf-8", + ) + (output_dir / "metadata.json").write_text( + json.dumps(metadata, indent=2, default=str), encoding="utf-8" + ) + (output_dir / "summary.json").write_text(json.dumps(summaries, indent=2), encoding="utf-8") + write_csv(output_dir / "raw.csv", measurements) + print(json.dumps(summaries, indent=2)) + print(f"Results: {output_dir}") + return 1 if any(item.error for item in measurements if item.phase == "measured") else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json new file mode 100644 index 0000000..0e4189f --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T120914Z", + "created_at": "2026-08-31T12:09:14.858871+00:00", + "runtime": { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mlx" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:05:22.804254+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.38745154201751575, + "total_s": 0.3874947500298731, + "prompt_tokens": 41, + "output_tokens": 1, + "decode_tps": 14.664458922599563, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:05:23.192111+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.021859125001356, + "total_s": 56.40731879102532, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 55.37840341105521, + "finish_reason": "length", + "output_chars": 12165, + "output_sha256": "a3355a481519820f9021b6846e3f784fba8aa922d3317c5a84d6762447d9dd97", + "output": "import asyncio\nimport time\nfrom collections import OrderedDict\nfrom dataclasses import dataclass, field\nfrom typing import (\n Any,\n Awaitable,\n Callable,\n Generic,\n Optional,\n TypeVar,\n)\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n factory_calls: int\n factory_errors: int\n invalidations: int\n\n\nclass _Entry(Generic[V]):\n __slots__ = (\"value\", \"expires_at\", \"weight\")\n\n def __init__(self, value: V, expires_at: float, weight: int) -> None:\n self.value = value\n self.expires_at = expires_at\n self.weight = weight\n\n\nclass _Inflight(Generic[V]):\n __slots__ = (\"event\", \"value\", \"error\", \"cancelled\")\n\n def __init__(self) -> None:\n self.event = asyncio.Event()\n self.value: Optional[V] = None\n self.error: Optional[BaseException] = None\n self.cancelled = False\n\n\nclass AsyncTTLCache(Generic[K, V]):\n def __init__(\n self,\n max_weight: int,\n default_ttl: float,\n *,\n weight_fn: Optional[Callable[[V], int]] = None,\n ) -> None:\n if max_weight <= 0:\n raise ValueError(\"max_weight must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n self._max_weight = max_weight\n self._default_ttl = default_ttl\n self._weight_fn = weight_fn or (lambda _v: 1)\n self._lock = asyncio.Lock()\n self._store: \"OrderedDict[K, _Entry[V]]\" = OrderedDict()\n self._inflight: \"dict[K, _Inflight[V]]\" = {}\n self._current_weight = 0\n self._stats = CacheStats(0, 0, 0, 0, 0, 0)\n\n @property\n def stats(self) -> CacheStats:\n return self._stats\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n weight: Optional[int] = None,\n ) -> V:\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None:\n if time.monotonic() < entry.expires_at:\n self._store.move_to_end(key)\n self._stats = CacheStats(\n self._stats.hits + 1,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n return entry.value\n self._remove_entry(key)\n\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses + 1,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n\n inflight = self._inflight.get(key)\n if inflight is not None:\n waiter = inflight\n else:\n waiter = _Inflight()\n self._inflight[key] = waiter\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls + 1,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n self._run_factory(key, factory, ttl, weight, waiter)\n\n await waiter.event.wait()\n\n if waiter.cancelled:\n raise asyncio.CancelledError()\n if waiter.error is not None:\n raise waiter.error\n return waiter.value\n\n def _run_factory(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: Optional[float],\n weight: Optional[int],\n waiter: _Inflight[V],\n ) -> None:\n async def runner() -> None:\n try:\n value = await factory()\n except asyncio.CancelledError:\n waiter.cancelled = True\n waiter.event.set()\n return\n except BaseException as exc:\n waiter.error = exc\n waiter.event.set()\n async with self._lock:\n self._inflight.pop(key, None)\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors + 1,\n self._stats.invalidations,\n )\n return\n\n async with self._lock:\n current = self._inflight.get(key)\n if current is not waiter:\n return\n self._inflight.pop(key, None)\n self._put_entry(key, value, ttl, weight)\n waiter.value = value\n waiter.event.set()\n\n asyncio.create_task(runner())\n\n def _put_entry(\n self,\n key: K,\n value: V,\n ttl: Optional[float],\n weight: Optional[int],\n ) -> None:\n if weight is None:\n weight = self._weight_fn(value)\n if weight <= 0:\n raise ValueError(\"weight must be positive\")\n if ttl is None:\n ttl = self._default_ttl\n if ttl <= 0:\n raise ValueError(\"ttl must be positive\")\n\n expires_at = time.monotonic() + ttl\n entry = _Entry(value, expires_at, weight)\n\n if self._current_weight + weight > self._max_weight:\n while self._current_weight > 0 and self._current_weight + weight > self._max_weight:\n evict_key, evict_entry = next(iter(self._store.items()))\n self._remove_entry(evict_key)\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions + 1,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations,\n )\n\n self._store[key] = entry\n self._store.move_to_end(key)\n self._current_weight += weight\n\n def _remove_entry(self, key: K) -> None:\n entry = self._store.pop(key, None)\n if entry is not None:\n self._current_weight -= entry.weight\n\n async def invalidate(self, key: K) -> bool:\n async with self._lock:\n removed = self._remove_entry(key)\n inflight = self._inflight.pop(key, None)\n if inflight is not None:\n inflight.cancelled = True\n inflight.event.set()\n if removed or inflight is not None:\n self._stats = CacheStats(\n self._stats.hits,\n self._stats.misses,\n self._stats.evictions,\n self._stats.factory_calls,\n self._stats.factory_errors,\n self._stats.invalidations + 1,\n )\n return True\n return False\n\n async def clear(self) -> None:\n async with self._lock:\n self._store.clear()\n self._current_weight = 0\n for inflight in self._inflight.values():\n inflight.cancelled = True\n inflight.event.set()\n self._inflight.clear()\n\n async def __aenter__(self) -> \"AsyncTTLCache[K, V]\":\n return self\n\n async def __aexit__(self, *args: Any) -> None:\n await self.clear()\n\n\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_basic_hit_and_miss() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> str:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.01)\n return \"value\"\n\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert calls == 1\n assert cache.stats.misses == 1\n assert cache.stats.hits == 0\n\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert calls == 1\n assert cache.stats.hits == 1\n\n\n@pytest.mark.asyncio\nasync def test_single_flight() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.05)\n return 42\n\n results = await asyncio.gather(\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n )\n assert all(r == 42 for r in results)\n assert calls == 1\n assert cache.stats.factory_calls == 1\n\n\n@pytest.mark.asyncio\nasync def test_cancellation_does_not_cancel_shared_factory() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.05)\n return 99\n\n task1 = asyncio.create_task(cache.get_or_create(1, factory))\n task2 = asyncio.create_task(cache.get_or_create(1, factory))\n await asyncio.sleep(0.01)\n task1.cancel()\n with pytest.raises(asyncio.CancelledError):\n await task1\n result = await task2\n assert result == 99\n assert calls == 1\n\n\n@pytest.mark.asyncio\nasync def test_expiry() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=0.05)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n return calls\n\n r1 = await cache.get_or_create(1, factory)\n assert r1 == 1\n await asyncio.sleep(0.06)\n r2 = await cache.get_or_create(1, factory)\n assert r2 == 2\n assert cache.stats.misses == 2\n\n\n@pytest.mark.asyncio\nasync def test_weighted_capacity_eviction() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n return calls\n\n await cache.get_or_create(1, factory, weight=6)\n await cache.get_or_create(2, factory, weight=5)\n assert cache.stats.evictions == 1\n assert 1 not in cache._store\n assert 2 in cache._store\n\n\n@pytest.mark.asyncio\nasync def test_factory_failure_not_cached() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n raise ValueError(\"fail\")\n\n with pytest.raises(ValueError):\n await cache.get_or_create(1, factory)\n assert cache.stats.factory_errors == 1\n assert 1 not in cache._store\n\n async def good_factory() -> int:\n return 42\n\n result = await cache.get_or_create(1, good_factory)\n assert result == 42\n assert cache.stats.factory_calls == 2\n\n\n@pytest.mark.asyncio\nasync def test_invalidation_during_creation_prevents_stale_publication() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n calls = 0\n\n async def factory() -> int:\n nonlocal calls\n calls += 1\n await asyncio.sleep(0.05)\n return 100\n\n task = asyncio.create_task(cache.get_or_create(1, factory))\n await asyncio.sleep(0.01)\n invalidated = await cache.invalidate(1)\n assert invalidated is True\n with pytest.raises(asyncio.CancelledError):\n await task\n assert 1 not in cache._store\n assert calls == 1\n\n async def new_factory() -> int:\n return 200\n\n result = await cache.get_or_create(1, new_factory)\n assert result == 200\n assert cache.stats.factory_calls == 2\n\n\n@pytest.mark.asyncio\nasync def test_clear_cancels_inflight() -> None:\n cache = AsyncTTLCache[int, int](max_weight=10, default_ttl=1.0)\n\n async def factory() -> int:\n await asyncio.sleep(0.05)\n return 1\n\n task = asyncio.create_task(cache.get_or_create(1, factory))\n await asyncio.sleep(0.01)\n await cache.clear()\n with pytest.raises(asyncio.CancelledError):\n await task", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:06:19.598953+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.2263128339545801, + "total_s": 78.72023624996655, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 39.58218973701569, + "finish_reason": "length", + "output_chars": 12654, + "output_sha256": "070acfea480b76f0913ce21686f2031f7a32421f5645f28eebe0c2ef9d2728ec", + "output": "```php\nexecuteTransfer($fromAccountId, $toAccountId, $amountCents, $idempotencyKey);\n } catch (\\Throwable $e) {\n if ($this->isDeadlock($e)) {\n $lastException = $e;\n // Exponential backoff with jitter\n $sleepMs = (int) (10 * pow(2, $attempt) + random_int(0, 10));\n usleep($sleepMs * 1000);\n continue;\n }\n throw $e;\n }\n }\n\n // All retries exhausted\n throw new \\RuntimeException(\n 'Transfer failed after ' . $maxRetries . ' deadlock retries.',\n 0,\n $lastException\n );\n }\n\n /**\n * Execute the transfer inside a database transaction.\n */\n private function executeTransfer(\n int $fromAccountId,\n int $toAccountId,\n int $amountCents,\n string $idempotencyKey\n ): array {\n return DB::transaction(function () use ($fromAccountId, $toAccountId, $amountCents, $idempotencyKey) {\n // 1. Check idempotency: if a transfer with this key already exists, return it.\n $existing = DB::table('transfers')\n ->where('idempotency_key', $idempotencyKey)\n ->lockForUpdate()\n ->first();\n\n if ($existing !== null) {\n return [\n 'transfer_id' => (int) $existing->id,\n 'idempotency_key' => $existing->idempotency_key,\n 'from_account_id' => (int) $existing->from_account_id,\n 'to_account_id' => (int) $existing->to_account_id,\n 'amount_cents' => (int) $existing->amount_cents,\n 'status' => $existing->status,\n 'was_duplicate' => true,\n ];\n }\n\n // 2. Lock both account rows in deterministic ID order to avoid deadlocks.\n $lockOrder = [$fromAccountId, $toAccountId];\n sort($lockOrder);\n\n $lockedAccounts = [];\n foreach ($lockOrder as $accountId) {\n $row = DB::table('accounts')\n ->where('id', $accountId)\n ->lockForUpdate()\n ->first();\n\n if ($row === null) {\n throw new InvalidTransferException(\"Account {$accountId} does not exist.\");\n }\n $lockedAccounts[$accountId] = $row;\n }\n\n // 3. Check sufficient funds on the source account.\n $fromAccount = $lockedAccounts[$fromAccountId];\n if ($fromAccount->balance_cents < $amountCents) {\n throw new InsufficientFundsException(\n (int) $fromAccount->balance_cents,\n $amountCents\n );\n }\n\n // 4. Perform the transfer: debit source, credit destination.\n DB::table('accounts')\n ->where('id', $fromAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents - ' . $amountCents),\n 'version' => DB::raw('version + 1'),\n ]);\n\n DB::table('accounts')\n ->where('id', $toAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents + ' . $amountCents),\n 'version' => DB::raw('version + 1'),\n ]);\n\n // 5. Persist the transfer record.\n $transferId = DB::table('transfers')->insertGetId([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'status' => 'completed',\n 'created_at' => now(),\n 'updated_at' => now(),\n ]);\n\n // 6. Persist the outbox event in the same transaction.\n DB::table('outbox_events')->insert([\n 'transfer_id' => $transferId,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transferId,\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n ]),\n 'created_at' => now(),\n ]);\n\n return [\n 'transfer_id' => (int) $transferId,\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'status' => 'completed',\n 'was_duplicate' => false,\n ];\n });\n }\n\n /**\n * Detect whether a Throwable represents a database deadlock.\n */\n private function isDeadlock(\\Throwable $e): bool\n {\n // MySQL/InnoDB deadlock error code is 1213\n // PostgreSQL deadlock error code is 40P01\n $message = $e->getMessage();\n $code = $e->getCode();\n\n if ($code === 1213 || $code === 40P01) {\n return true;\n }\n\n // Check SQLSTATE or message for deadlock indicators\n if (str_contains($message, 'Deadlock') || str_contains($message, 'deadlock')) {\n return true;\n }\n\n // Check for SQLSTATE 40001 (serialization failure)\n if (str_contains($message, '40001')) {\n return true;\n }\n\n return false;\n }\n}\n\n// ---------------------------------------------------------------------------\n// PHPUnit Tests\n// ---------------------------------------------------------------------------\n\nclass MoneyTransferServiceTest extends TestCase\n{\n use RefreshDatabase;\n\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Create the schema for testing\n Schema::create('accounts', function (Blueprint $table) {\n $table->id();\n $table->bigInteger('balance_cents')->default(0);\n $table->unsignedInteger('version')->default(0);\n $table->timestamps();\n });\n\n Schema::create('transfers', function (Blueprint $table) {\n $table->id();\n $table->string('idempotency_key')->unique();\n $table->unsignedBigInteger('from_account_id');\n $table->unsignedBigInteger('to_account_id');\n $table->bigInteger('amount_cents');\n $table->string('status')->default('completed');\n $table->timestamps();\n });\n\n Schema::create('outbox_events', function (Blueprint $table) {\n $table->id();\n $table->unsignedBigInteger('transfer_id');\n $table->string('event_type');\n $table->text('payload');\n $table->timestamps();\n });\n\n $this->service = new MoneyTransferService();\n }\n\n /**\n * Test that a valid transfer moves money correctly.\n */\n public function test_valid_transfer_moves_money(): void\n {\n $fromId = $this->createAccount(10000); // 100.00\n $toId = $this->createAccount(0); // 0.00\n\n $result = $this->service->transfer($fromId, $toId, 5000, 'key-001');\n\n $this->assertFalse($result['was_duplicate']);\n $this->assertEquals('completed', $result['status']);\n $this->assertEquals(5000, $result['amount_cents']);\n\n $fromBalance = DB::table('accounts')->where('id', $fromId)->value('balance_cents');\n $toBalance = DB::table('accounts')->where('id', $toId)->value('balance_cents');\n\n $this->assertEquals(5000, (int) $fromBalance);\n $this->assertEquals(5000, (int) $toBalance);\n }\n\n /**\n * Test that a duplicate idempotency key returns the original transfer\n * without moving money again.\n */\n public function test_duplicate_idempotency_key_returns_original_transfer(): void\n {\n $fromId = $this->createAccount(10000);\n $toId = $this->createAccount(0);\n\n $key = 'key-dup-001';\n\n $result1 = $this->service->transfer($fromId, $toId, 3000, $key);\n $this->assertFalse($result1['was_duplicate']);\n\n $result2 = $this->service->transfer($fromId, $toId, 3000, $key);\n $this->assertTrue($result2['was_duplicate']);\n\n // Balances should reflect only one transfer\n $fromBalance = DB::table('accounts')->where('id', $fromId)->value('balance_cents');\n $toBalance =", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:07:38.318694+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.5556320829782635, + "total_s": 96.54028320801444, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 32.29928081838954, + "finish_reason": "length", + "output_chars": 11211, + "output_sha256": "f97ad0b443b4fe7ae9a5b2a3f0cc3c8ad2b71bc2ba4d5849bd679b498488c654", + "output": "```typescript\n// strict-mode, dependency-free DAG task scheduler\n\nexport interface Task {\n id: string;\n deps: string[];\n run: (signal: AbortSignal) => Promise;\n}\n\nexport interface ScheduleOptions {\n maxConcurrency?: number;\n signal?: AbortSignal;\n}\n\nexport type TaskResults = Map;\n\nexport class CycleError extends Error {\n constructor(public readonly path: string[]) {\n super(`Cycle detected: ${path.join(\" -> \")}`);\n this.name = \"CycleError\";\n }\n}\n\nexport class ValidationError extends Error {\n constructor(message: string) {\n super(message);\n this.name = \"ValidationError\";\n }\n}\n\nexport class CancellationError extends Error {\n constructor() {\n super(\"Schedule cancelled\");\n this.name = \"CancellationError\";\n }\n}\n\nexport class TaskError extends Error {\n constructor(\n public readonly taskId: string,\n public readonly cause: unknown\n ) {\n super(`Task \"${taskId}\" failed: ${cause instanceof Error ? cause.message : String(cause)}`);\n this.name = \"TaskError\";\n }\n}\n\n// Internal node representation\ninterface Node {\n id: string;\n deps: string[];\n run: (signal: AbortSignal) => Promise;\n inDegree: number;\n dependents: string[];\n result?: unknown;\n error?: unknown;\n status: \"pending\" | \"running\" | \"done\" | \"failed\" | \"skipped\";\n promise?: Promise;\n}\n\n// Topological sort with cycle detection using DFS\nfunction topologicalSort(nodes: Map, order: string[]): string[] {\n const visited = new Set();\n const inStack = new Set();\n const sorted: string[] = [];\n\n function dfs(id: string, stack: string[]): void {\n if (inStack.has(id)) {\n // Found a cycle; extract the cycle path\n const cycleStart = stack.indexOf(id);\n const cyclePath = stack.slice(cycleStart).concat(id);\n throw new CycleError(cyclePath);\n }\n if (visited.has(id)) return;\n\n inStack.add(id);\n stack.push(id);\n\n const node = nodes.get(id)!;\n // Process dependencies in the order they appear in the input\n for (const dep of node.deps) {\n dfs(dep, stack);\n }\n\n stack.pop();\n inStack.delete(id);\n visited.add(id);\n sorted.push(id);\n }\n\n // Process nodes in input order for stable scheduling\n for (const id of order) {\n if (!visited.has(id)) {\n dfs(id, []);\n }\n }\n\n return sorted;\n}\n\n// Kahn's algorithm variant that preserves input order for stable scheduling\nfunction kahnSort(nodes: Map, inputOrder: string[]): string[] {\n // Build in-degree map\n const inDegree = new Map();\n const dependents = new Map();\n\n for (const [id, node] of nodes) {\n inDegree.set(id, node.deps.length);\n dependents.set(id, []);\n }\n\n for (const [id, node] of nodes) {\n for (const dep of node.deps) {\n const deps = dependents.get(dep);\n if (deps) {\n deps.push(id);\n }\n }\n }\n\n // Use a queue that preserves input order\n // Initialize with nodes that have in-degree 0, in input order\n const queue: string[] = [];\n for (const id of inputOrder) {\n if (inDegree.get(id) === 0) {\n queue.push(id);\n }\n }\n\n const sorted: string[] = [];\n let head = 0;\n\n while (head < queue.length) {\n const id = queue[head++];\n sorted.push(id);\n\n const deps = dependents.get(id) || [];\n for (const dep of deps) {\n const deg = inDegree.get(dep)!;\n inDegree.set(dep, deg - 1);\n if (inDegree.get(dep) === 0) {\n queue.push(dep);\n }\n }\n }\n\n if (sorted.length !== nodes.size) {\n // Cycle exists; use DFS to find the cycle path\n const visited = new Set();\n const inStack = new Set();\n\n function dfs(id: string, stack: string[]): void {\n if (inStack.has(id)) {\n const cycleStart = stack.indexOf(id);\n const cyclePath = stack.slice(cycleStart).concat(id);\n throw new CycleError(cyclePath);\n }\n if (visited.has(id)) return;\n\n inStack.add(id);\n stack.push(id);\n\n const node = nodes.get(id)!;\n for (const dep of node.deps) {\n dfs(dep, stack);\n }\n\n stack.pop();\n inStack.delete(id);\n visited.add(id);\n }\n\n for (const id of inputOrder) {\n if (!visited.has(id)) {\n dfs(id, []);\n }\n }\n // Should not reach here\n throw new CycleError([\"unknown\"]);\n }\n\n return sorted;\n}\n\nexport async function schedule(\n tasks: Task[],\n options: ScheduleOptions = {}\n): Promise {\n const maxConcurrency = options.maxConcurrency ?? Infinity;\n const externalSignal = options.signal;\n\n // Validate: check for duplicate IDs\n const idSet = new Set();\n for (const task of tasks) {\n if (idSet.has(task.id)) {\n throw new ValidationError(`Duplicate task ID: \"${task.id}\"`);\n }\n idSet.add(task.id);\n }\n\n // Validate: check for unknown dependencies\n for (const task of tasks) {\n for (const dep of task.deps) {\n if (!idSet.has(dep)) {\n throw new ValidationError(\n `Task \"${task.id}\" depends on unknown task \"${dep}\"`\n );\n }\n }\n }\n\n // Build node map\n const nodes = new Map();\n const inputOrder: string[] = [];\n\n for (const task of tasks) {\n nodes.set(task.id, {\n id: task.id,\n deps: task.deps,\n run: task.run,\n inDegree: task.deps.length,\n dependents: [],\n status: \"pending\",\n });\n inputOrder.push(task.id);\n }\n\n // Build dependents map\n for (const [id, node] of nodes) {\n for (const dep of node.deps) {\n const depNode = nodes.get(dep);\n if (depNode) {\n depNode.dependents.push(id);\n }\n }\n }\n\n // Detect cycles and get topological order\n const topoOrder = kahnSort(nodes, inputOrder);\n\n // Create an internal AbortController that we can abort on failure or external signal\n const controller = new AbortController();\n const internalSignal = controller.signal;\n\n // Link external signal to internal controller\n if (externalSignal) {\n if (externalSignal.aborted) {\n controller.abort();\n } else {\n externalSignal.addEventListener(\"abort\", () => {\n controller.abort();\n }, { once: true });\n }\n }\n\n // Track running tasks and their promises\n const runningPromises = new Set>();\n const results = new Map();\n let failed = false;\n let failureError: unknown = null;\n\n // Track which tasks have been started (to avoid starting dependents of failed tasks)\n const started = new Set();\n const skipped = new Set();\n\n // Helper: mark all transitive dependents as skipped\n function skipDependents(id: string): void {\n const node = nodes.get(id)!;\n for (const dep of node.dependents) {\n if (!skipped.has(dep) && !started.has(dep)) {\n skipped.add(dep);\n skipDependents(dep);\n }\n }\n }\n\n // Helper: check if all dependencies of a task are done (not failed/skipped)\n function depsReady(id: string): boolean {\n const node = nodes.get(id)!;\n for (const dep of node.deps) {\n const depNode = nodes.get(dep)!;\n if (depNode.status === \"failed\" || depNode.status === \"skipped\") {\n return false;\n }\n if (depNode.status !== \"done\") {\n return false;\n }\n }\n return true;\n }\n\n // Helper: check if a task should be skipped (any dep failed or skipped)\n function shouldSkip(id: string): boolean {\n const node = nodes.get(id)!;\n for (const dep of node.deps) {\n const depNode = nodes.get(dep)!;\n if (depNode.status === \"failed\" || depNode.status === \"skipped\") {\n return true;\n }\n }\n return false;\n }\n\n // Main scheduling loop\n // We process tasks in topological order, respecting concurrency limits\n // Use a pointer to the next task to consider\n let nextIndex = 0;\n const readyQueue: string[] = [];\n\n // Initialize ready queue with tasks that have no dependencies\n for (const id of topoOrder) {\n const node = nodes.get(id)!;\n if (node.deps.length === 0) {\n readyQueue.push(id);\n }\n }\n\n // We need to process tasks as they become ready\n // Use a more dynamic approach: after each task completes, check if new tasks are ready\n\n // Process the schedule\n async function runTask(id: string): Promise {\n const node = nodes.get(id)!;\n if (node.status !== \"pending\") return;\n\n // Check if we should skip this task\n if (shouldSkip(id)) {\n node.status = \"skipped\";\n skipDependents(id);\n return;\n }\n\n if (internalSignal.aborted) {\n node.status = \"skipped\";\n skipDependents(id);\n return;\n }\n\n started.add(id);\n node.status = \"running\";\n\n const taskPromise = node.run(internalSignal);\n node.promise = taskPromise;\n runningPromises.add(taskPromise);\n\n try {\n const result = await taskPromise;\n node.result = result;\n node.status = \"done\";\n results.set(id, result);\n } catch (err) {\n node.error = err;\n node.status = \"failed\";\n failed = true;\n failureError = new TaskError(id, err);\n // Abort all running tasks\n controller.abort();\n // Skip all dependents\n skipDependents(id);\n } finally {\n runningPromises.delete(taskPromise);\n }\n }\n\n // Process tasks in waves, respecting concurrency\n // We use a simple approach: maintain a set of running promises and a queue of ready tasks\n const running = new Set>();\n\n // Add initial ready tasks\n for (const id of readyQueue) {\n if (running.size >= maxConcurrency) break;\n if (skipped.has(id)) continue;\n const p = runTask(id);\n running.add(p);\n }\n\n // Process remaining tasks as they become ready\n // We need to wait for tasks to complete and then check for new ready tasks\n let processed = 0;\n const totalTasks = nodes.size;\n\n while (processed < totalTasks) {\n // Wait for at least one running task to complete\n if (running.size === 0) {\n // No more running tasks; check if there are more ready tasks\n // Find new ready tasks\n let foundNew = false;\n for (const id of topoOrder) {\n const node = nodes.get(id)!;\n if (node.status === \"pending\" && !skipped.has(id)) {\n if (depsReady(id)) {\n if (running.size >= maxConcurrency) break;\n const p = runTask(id);\n running.add(p);\n foundNew = true;\n }\n }\n }\n if (!foundNew) {\n // No more tasks to run\n break;\n }\n }\n\n // Wait for one task to complete\n if (running.size > 0) {\n // Use Promise.race but handle rejections\n const promises = Array.from(running);\n // We need to wait for one to settle without unhandled rejections\n // Create a race that catches rejections\n const settled = await Promise.race(\n promises.map((p) =>\n p.then(\n () => ({ type: \"fulfilled\" as const, promise: p }),\n () => ({ type: \"rejected\" as const, promise: p })\n )\n )\n );\n running.delete(settled.promise);\n processed++;\n }\n\n // After a task completes, check for new ready tasks\n if (!failed && !internalSignal.aborted) {\n for (const id of topo", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json new file mode 100644 index 0000000..b08f87e --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T120914Z-ollama-qwen3.8_27b-mlx/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:05:23.192111+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.021859125001356, + "total_s": 56.40731879102532, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 55.37840341105521, + "finish_reason": "length", + "output_chars": 12165, + "output_sha256": "a3355a481519820f9021b6846e3f784fba8aa922d3317c5a84d6762447d9dd97", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:06:19.598953+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.2263128339545801, + "total_s": 78.72023624996655, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 39.58218973701569, + "finish_reason": "length", + "output_chars": 12654, + "output_sha256": "070acfea480b76f0913ce21686f2031f7a32421f5645f28eebe0c2ef9d2728ec", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "started_at": "2026-08-31T12:07:38.318694+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.5556320829782635, + "total_s": 96.54028320801444, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 32.29928081838954, + "finish_reason": "length", + "output_chars": 11211, + "output_sha256": "f97ad0b443b4fe7ae9a5b2a3f0cc3c8ad2b71bc2ba4d5849bd679b498488c654", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json new file mode 100644 index 0000000..6b2ad8e --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T121823Z", + "created_at": "2026-08-31T12:18:23.136973+00:00", + "runtime": { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-nvfp4" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:10:51.048391+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.38219533301889896, + "total_s": 0.41809329099487513, + "prompt_tokens": 41, + "output_tokens": 2, + "decode_tps": 55.71347543886622, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:10:51.466896+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.0361592500121333, + "total_s": 133.43742824997753, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 23.20219453486358, + "finish_reason": "length", + "output_chars": 14058, + "output_sha256": "a18e80f1852ecc0c70b987f244870b8a2dc20db8ad42505280162a4b17dbd057", + "output": "from __future__ import annotations\n\nimport asyncio\nimport time\nfrom dataclasses import dataclass, field\nfrom typing import (\n Any,\n Awaitable,\n Callable,\n Generic,\n Hashable,\n Optional,\n TypeVar,\n)\n\nK = TypeVar(\"K\", bound=Hashable)\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n\n\n@dataclass\nclass _Entry:\n value: V\n expires_at: float\n weight: int\n last_access: float\n\n\nclass _Inflight:\n __slots__ = (\"task\", \"cancelled\")\n\n def __init__(self, task: \"asyncio.Task[V]\") -> None:\n self.task = task\n self.cancelled = False\n\n\nclass AsyncTTLCache(Generic[K, V]):\n \"\"\"\n A generic async TTL cache with weighted LRU eviction, per-entry TTL,\n single-flight get_or_create, and safe cancellation semantics.\n\n - Weighted LRU: each entry has a weight; total weight <= capacity.\n - TTL: per-entry expiration based on time.monotonic.\n - Single-flight: concurrent get_or_create for the same key share one factory call.\n - Cancellation: cancelling one waiter does not cancel the shared factory.\n - Exceptions are never cached.\n - Invalidation during creation prevents stale publication.\n - The cache lock is never held while awaiting user code.\n \"\"\"\n\n def __init__(\n self,\n capacity: int,\n default_ttl: float,\n *,\n weight_fn: Optional[Callable[[V], int]] = None,\n ) -> None:\n if capacity <= 0:\n raise ValueError(\"capacity must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n\n self._capacity = capacity\n self._default_ttl = default_ttl\n self._weight_fn = weight_fn or (lambda _v: 1)\n\n # LRU order: most recently used at front (index 0)\n self._entries: dict[K, _Entry] = {}\n self._lru_order: list[K] = [] # index 0 = MRU, last = LRU\n self._total_weight: int = 0\n\n # Single-flight tracking\n self._inflight: dict[K, _Inflight] = {}\n\n # Statistics\n self._hits: int = 0\n self._misses: int = 0\n self._evictions: int = 0\n\n self._lock = asyncio.Lock()\n\n @property\n def stats(self) -> CacheStats:\n return CacheStats(\n hits=self._hits,\n misses=self._misses,\n evictions=self._evictions,\n )\n\n def clear(self) -> None:\n \"\"\"Clear all entries and cancel any in-flight factories.\"\"\"\n self._entries.clear()\n self._lru_order.clear()\n self._total_weight = 0\n for key in list(self._inflight.keys()):\n inflight = self._inflight.pop(key)\n if not inflight.task.done():\n inflight.cancelled = True\n inflight.task.cancel()\n\n def invalidate(self, key: K) -> None:\n \"\"\"\n Invalidate a key. If a factory is in-flight for this key,\n mark it as cancelled so its result is not published.\n \"\"\"\n self._entries.pop(key, None)\n if key in self._lru_order:\n self._lru_order.remove(key)\n # Adjust weight\n entry = self._entries.get(key)\n if entry is None and key in self._lru_order:\n pass # already removed\n # Recompute weight if entry was removed\n # Actually we need to track weight separately\n # Let's restructure: store weight in entry and recompute on removal\n # For simplicity, we track _total_weight and adjust on removal\n # But we already popped from _entries, so we need the weight before popping.\n # Let's fix: get weight before popping.\n # Actually, let's restructure invalidate to handle weight properly.\n # We'll fix this in the implementation below.\n\n inflight = self._inflight.get(key)\n if inflight is not None:\n inflight.cancelled = True\n if not inflight.task.done():\n inflight.task.cancel()\n\n def _remove_key(self, key: K) -> None:\n \"\"\"Remove a key from entries and LRU order, adjusting total weight.\"\"\"\n entry = self._entries.pop(key, None)\n if entry is not None:\n self._total_weight -= entry.weight\n if key in self._lru_order:\n self._lru_order.remove(key)\n\n def invalidate(self, key: K) -> None: # noqa: F811\n \"\"\"\n Invalidate a key. If a factory is in-flight for this key,\n mark it as cancelled so its result is not published.\n \"\"\"\n self._remove_key(key)\n inflight = self._inflight.get(key)\n if inflight is not None:\n inflight.cancelled = True\n if not inflight.task.done():\n inflight.task.cancel()\n\n def _evict_lru(self, needed_weight: int) -> None:\n \"\"\"Evict LRU entries until total_weight + needed_weight <= capacity.\"\"\"\n while self._total_weight + needed_weight > self._capacity:\n if not self._lru_order:\n break\n lru_key = self._lru_order[-1]\n entry = self._entries.get(lru_key)\n if entry is None:\n self._lru_order.pop()\n continue\n self._remove_key(lru_key)\n self._evictions += 1\n\n def _touch(self, key: K) -> None:\n \"\"\"Move key to MRU position.\"\"\"\n if key in self._lru_order:\n self._lru_order.remove(key)\n self._lru_order.insert(0, key)\n\n async def get(self, key: K) -> Optional[V]:\n \"\"\"\n Get a value from the cache. Returns None if not present or expired.\n Does not trigger factory creation.\n \"\"\"\n async with self._lock:\n entry = self._entries.get(key)\n if entry is None:\n self._misses += 1\n return None\n now = time.monotonic()\n if now >= entry.expires_at:\n # Expired\n self._remove_key(key)\n self._misses += 1\n return None\n # Hit\n self._hits += 1\n entry.last_access = now\n self._touch(key)\n return entry.value\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n weight: Optional[int] = None,\n ) -> V:\n \"\"\"\n Get a value from the cache, or create it using the async factory.\n Single-flight: concurrent calls for the same key share one factory invocation.\n Exceptions from the factory are never cached.\n \"\"\"\n effective_ttl = ttl if ttl is not None else self._default_ttl\n if effective_ttl <= 0:\n raise ValueError(\"ttl must be positive\")\n\n # First, check if we already have a valid entry\n async with self._lock:\n entry = self._entries.get(key)\n if entry is not None:\n now = time.monotonic()\n if now < entry.expires_at:\n self._hits += 1\n entry.last_access = now\n self._touch(key)\n return entry.value\n else:\n # Expired, remove it\n self._remove_key(key)\n self._misses += 1\n\n # Check if there's an in-flight factory\n inflight = self._inflight.get(key)\n if inflight is not None and not inflight.cancelled:\n # Wait for the existing factory\n pass # We'll wait outside the lock\n else:\n # No valid entry and no active in-flight (or it was cancelled)\n # We need to start a new factory\n self._misses += 1\n # Create the task outside the lock\n task = asyncio.ensure_future(factory())\n inflight = _Inflight(task)\n self._inflight[key] = inflight\n # Release lock, then await\n # We need to break out of the lock context\n # Let's restructure: use a flag\n break_out = True\n else:\n break_out = False\n\n # If we found an existing in-flight, wait for it\n if not break_out:\n assert inflight is not None\n try:\n value = await inflight.task\n except asyncio.CancelledError:\n # Our task was cancelled, but the shared factory may still be running\n # We should not propagate cancellation to the factory\n # Re-raise to indicate our caller was cancelled\n raise\n except Exception:\n # Factory failed; do not cache\n self._inflight.pop(key, None)\n raise\n else:\n # Factory succeeded\n self._inflight.pop(key, None)\n # Check if it was invalidated during creation\n if inflight.cancelled:\n # Do not publish\n return await self.get_or_create(\n key, factory, ttl=ttl, weight=weight\n )\n # Publish the result\n async with self._lock:\n # Re-check: maybe another task already published\n existing = self._entries.get(key)\n if existing is not None:\n now = time.monotonic()\n if now < existing.expires_at:\n self._hits += 1\n existing.last_access = now\n self._touch(key)\n return existing.value\n else:\n self._remove_key(key)\n\n # Check if invalidated\n if inflight.cancelled:\n # Do not publish\n pass\n else:\n # Insert into cache\n effective_weight = weight if weight is not None else self._weight_fn(value)\n if effective_weight <= 0:\n raise ValueError(\"weight must be positive\")\n if effective_weight > self._capacity:\n raise ValueError(\n f\"weight {effective_weight} exceeds capacity {self._capacity}\"\n )\n\n now = time.monotonic()\n new_entry = _Entry(\n value=value,\n expires_at=now + effective_ttl,\n weight=effective_weight,\n last_access=now,\n )\n self._evict_lru(effective_weight)\n self._entries[key] = new_entry\n self._lru_order.insert(0, key)\n self._total_weight += effective_weight\n return value\n # If invalidated, fall through to retry\n return await self.get_or_create(key, factory, ttl=ttl, weight=weight)\n\n # We need to start a new factory\n assert inflight is not None\n try:\n value = await inflight.task\n except asyncio.CancelledError:\n # Our task was cancelled\n # The shared factory may still be running\n # Do not cancel the shared factory\n raise\n except Exception:\n # Factory failed; do not cache\n async with self._lock:\n self._inflight.pop(key, None)\n raise\n else:\n # Factory succeeded\n async with self._lock:\n self._inflight.pop(key, None)\n # Check if it was invalidated during creation\n if inflight.cancelled:\n # Do not publish; retry\n return await self.get_or_create(key, factory, ttl=ttl, weight=weight)\n # Publish the result\n async with self._lock:\n # Re-check: maybe another task already published\n existing = self._entries.get(key)\n if existing is not None:\n now = time.monotonic()\n if now < existing.expires_at:\n self._hits += 1\n existing.last_access = now\n self._touch(key)\n return existing.value\n else:\n self._remove_key(key)\n\n # Check if invalidated\n if inflight.cancelled:\n pass\n else:\n # Insert into cache\n effective_weight = weight if weight is not None else self._weight_fn(value)\n if effective_weight <= 0:\n raise ValueError(\"weight must be positive\")\n if effective_weight > self._capacity:\n raise ValueError(\n f\"weight {effective_weight} exceeds capacity {self._capacity}\"\n )\n\n now = time.monotonic()\n new_entry = _Entry(\n value=value,\n expires_at=now + effective_ttl,\n weight=effective_weight,\n last_access=now,\n )\n self._evict_lru(effective_weight)\n self._entries[key] = new_entry\n self._lru_order.insert(0, key)\n self._total_weight += effective_weight\n return value\n # If invalidated, fall through to retry\n return await self.get_or_create(key, factory, ttl=ttl, weight=weight)\n\n async def set(\n self,\n key: K,\n value: V,\n *,\n ttl: Optional[float] = None,\n weight: Optional[int] = None,\n ) -> None:\n \"\"\"\n Set a value in the cache. Evicts LRU entries if needed", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:13:04.907109+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 15.930970875022467, + "total_s": 166.00547937501688, + "prompt_tokens": 182, + "output_tokens": 2840, + "decode_tps": 18.923933374068692, + "finish_reason": "stop", + "output_chars": 11403, + "output_sha256": "bb758808663b88bb3fbacc52df49598eb21aa8bc7fa0ee8466410f0ae13e8b0d", + "output": "where('idempotency_key', $idempotencyKey)\n ->first();\n\n if ($existingTransfer) {\n return [\n 'transfer_id' => $existingTransfer->id,\n 'idempotency_key' => $existingTransfer->idempotency_key,\n 'from_account_id' => $existingTransfer->from_account_id,\n 'to_account_id' => $existingTransfer->to_account_id,\n 'amount_cents' => $existingTransfer->amount_cents,\n 'status' => 'duplicate',\n ];\n }\n\n // Retry loop for deadlocks\n $maxRetries = 5;\n $attempt = 0;\n\n while (true) {\n $attempt++;\n try {\n $result = DB::transaction(function () use ($idempotencyKey, $fromAccountId, $toAccountId, $amountCents) {\n // Lock both account rows in deterministic ID order\n $minId = min($fromAccountId, $toAccountId);\n $maxId = max($fromAccountId, $toAccountId);\n\n $accounts = DB::table('accounts')\n ->whereIn('id', [$minId, $maxId])\n ->orderBy('id')\n ->lockForUpdate()\n ->get();\n\n if ($accounts->count() !== 2) {\n throw new RuntimeException('One or both accounts do not exist.');\n }\n\n $fromAccount = $accounts->firstWhere('id', $fromAccountId);\n $toAccount = $accounts->firstWhere('id', $toAccountId);\n\n // Reject insufficient funds\n if ($fromAccount->balance_cents < $amountCents) {\n throw new RuntimeException('Insufficient funds.');\n }\n\n // Perform transfer\n DB::table('accounts')\n ->where('id', $fromAccountId)\n ->decrement('balance_cents', $amountCents);\n\n DB::table('accounts')\n ->where('id', $toAccountId)\n ->increment('balance_cents', $amountCents);\n\n // Insert transfer\n $transferId = DB::table('transfers')->insertGetId([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'created_at' => now(),\n ]);\n\n // Persist outbox event in the same transaction\n DB::table('outbox_events')->insert([\n 'transfer_id' => $transferId,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transferId,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n ]),\n 'created_at' => now(),\n ]);\n\n return [\n 'transfer_id' => $transferId,\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'status' => 'created',\n ];\n });\n\n return $result;\n\n } catch (Illuminate\\Database\\QueryException $e) {\n // Detect deadlock (MySQL error 1213)\n if ($e->getCode() === 1213 || str_contains($e->getMessage(), 'Deadlock')) {\n if ($attempt >= $maxRetries) {\n throw new RuntimeException('Transfer failed after maximum retries due to deadlock.', 0, $e);\n }\n usleep(100000 * $attempt); // Exponential backoff\n continue;\n }\n throw $e;\n }\n }\n }\n}\n\nclass MoneyTransferServiceTest extends TestCase\n{\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Create test database schema\n Schema::dropIfExists('accounts');\n Schema::dropIfExists('transfers');\n Schema::dropIfExists('outbox_events');\n\n Schema::create('accounts', function (Blueprint $table) {\n $table->id();\n $table->unsignedBigInteger('balance_cents')->default(0);\n });\n\n Schema::create('transfers', function (Blueprint $table) {\n $table->id();\n $table->string('idempotency_key')->unique();\n $table->unsignedBigInteger('from_account_id');\n $table->unsignedBigInteger('to_account_id');\n $table->unsignedBigInteger('amount_cents');\n $table->timestamp('created_at');\n });\n\n Schema::create('outbox_events', function (Blueprint $table) {\n $table->id();\n $table->unsignedBigInteger('transfer_id');\n $table->string('event_type');\n $table->json('payload');\n $table->timestamp('created_at');\n });\n\n $this->service = new MoneyTransferService();\n }\n\n public function test_transfer_moves_funds_correctly(): void\n {\n // Setup: Account 1 has 10000 cents, Account 2 has 0 cents\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $result = $this->service->transfer('key-1', 1, 2, 5000);\n\n $this->assertSame('created', $result['status']);\n $this->assertSame(5000, $result['amount_cents']);\n\n $account1 = DB::table('accounts')->where('id', 1)->first();\n $account2 = DB::table('accounts')->where('id', 2)->first();\n\n $this->assertSame(5000, $account1->balance_cents);\n $this->assertSame(5000, $account2->balance_cents);\n\n $transferCount = DB::table('transfers')->count();\n $this->assertSame(1, $transferCount);\n\n $outboxCount = DB::table('outbox_events')->count();\n $this->assertSame(1, $outboxCount);\n }\n\n public function test_duplicate_request_returns_original_transfer(): void\n {\n // Setup: Account 1 has 10000 cents, Account 2 has 0 cents\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $firstResult = $this->service->transfer('key-dup', 1, 2, 3000);\n $this->assertSame('created', $firstResult['status']);\n\n $secondResult = $this->service->transfer('key-dup', 1, 2, 3000);\n $this->assertSame('duplicate', $secondResult['status']);\n $this->assertSame($firstResult['transfer_id'], $secondResult['transfer_id']);\n\n // Balances should only reflect one transfer\n $account1 = DB::table('accounts')->where('id', 1)->first();\n $account2 = DB::table('accounts')->where('id', 2)->first();\n\n $this->assertSame(7000, $account1->balance_cents);\n $this->assertSame(3000, $account2->balance_cents);\n\n $transferCount = DB::table('transfers')->count();\n $this->assertSame(1, $transferCount);\n\n $outboxCount = DB::table('outbox_events')->count();\n $this->assertSame(1, $outboxCount);\n }\n\n public function test_competing_requests_preserve_balances_and_create_one_transfer(): void\n {\n // Setup: Account 1 has 10000 cents, Account 2 has 0 cents\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n // Simulate competing requests with same idempotency key\n $results = [];\n for ($i = 0; $i < 5; $i++) {\n $results[] = $this->service->transfer('key-compete', 1, 2, 2000);\n }\n\n // All should return the same transfer ID\n $transferIds = array_unique(array_column($results, 'transfer_id'));\n $this->assertCount(1, $transferIds);\n\n // Only one transfer should be created\n $transferCount = DB::table('transfers')->count();\n $this->assertSame(1, $transferCount);\n\n // Only one outbox event\n $outboxCount = DB::table('outbox_events')->count();\n $this->assertSame(1, $outboxCount);\n\n // Balances should reflect only one transfer of 2000\n $account1 = DB::table('accounts')->where('id', 1)->first();\n $account2 = DB::table('accounts')->where('id', 2)->first();\n\n $this->assertSame(8000, $account1->balance_cents);\n $this->assertSame(2000, $account2->balance_cents);\n }\n\n public function test_rejects_insufficient_funds(): void\n {\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 1000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $this->expectException(RuntimeException::class);\n $this->expectExceptionMessage('Insufficient funds.');\n\n $this->service->transfer('key-insufficient', 1, 2, 5000);\n }\n\n public function test_rejects_non_positive_amount(): void\n {\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ['id' => 2, 'balance_cents' => 0],\n ]);\n\n $this->expectException(InvalidArgumentException::class);\n $this->expectExceptionMessage('Amount must be positive.');\n\n $this->service->transfer('key-zero', 1, 2, 0);\n }\n\n public function test_rejects_same_account(): void\n {\n DB::table('accounts')->insert([\n ['id' => 1, 'balance_cents' => 10000],\n ]);\n\n $this->expectException(InvalidArgumentException::class);\n $this->expectExceptionMessage('Source and destination accounts must be distinct.');\n\n $this->service->transfer('key-same', 1, 1, 1000);\n }\n}", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:15:50.913910+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4814700839924626, + "total_s": 152.22010587499244, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 20.379645761550783, + "finish_reason": "length", + "output_chars": 10869, + "output_sha256": "f700d51c471a59acfcf85016d416f0c5354662552fea44b48f56d80c3afdcb59", + "output": "type TaskResult = { ok: true; value: T } | { ok: false; error: unknown };\n\ninterface Task {\n id: string;\n deps: readonly string[];\n run: (signal: AbortSignal) => Promise;\n}\n\ninterface SchedulerOptions {\n maxConcurrency?: number;\n signal?: AbortSignal;\n}\n\nclass SchedulerError extends Error {\n constructor(message: string) {\n super(message);\n this.name = \"SchedulerError\";\n }\n}\n\nclass CycleError extends SchedulerError {\n readonly path: readonly string[];\n constructor(path: readonly string[]) {\n super(`Dependency cycle detected: ${path.join(\" -> \")}`);\n this.name = \"CycleError\";\n this.path = path;\n }\n}\n\nclass DuplicateIdError extends SchedulerError {\n constructor(id: string) {\n super(`Duplicate task ID: ${id}`);\n this.name = \"DuplicateIdError\";\n }\n}\n\nclass UnknownDependencyError extends SchedulerError {\n constructor(taskId: string, depId: string) {\n super(`Task \"${taskId}\" depends on unknown task \"${depId}\"`);\n this.name = \"UnknownDependencyError\";\n }\n}\n\nclass CancellationError extends SchedulerError {\n constructor() {\n super(\"Scheduler aborted\");\n this.name = \"CancellationError\";\n }\n}\n\nclass TaskFailureError extends SchedulerError {\n readonly taskId: string;\n readonly cause: unknown;\n constructor(taskId: string, cause: unknown) {\n super(`Task \"${taskId}\" failed`);\n this.name = \"TaskFailureError\";\n this.taskId = taskId;\n this.cause = cause;\n }\n}\n\nfunction isAbortSignal(value: unknown): value is AbortSignal {\n return (\n typeof value === \"object\" &&\n value !== null &&\n \"aborted\" in value &&\n \"addEventListener\" in value &&\n \"removeEventListener\" in value\n );\n}\n\nfunction createCombinedSignal(external: AbortSignal | undefined): {\n signal: AbortSignal;\n cancel: () => void;\n} {\n const controller = new AbortController();\n const onExternalAbort = () => controller.abort(external?.reason);\n if (external) {\n if (external.aborted) {\n controller.abort(external.reason);\n } else {\n external.addEventListener(\"abort\", onExternalAbort, { once: true });\n }\n }\n const cancel = () => {\n if (external) {\n external.removeEventListener(\"abort\", onExternalAbort);\n }\n if (!controller.signal.aborted) {\n controller.abort(new CancellationError());\n }\n };\n return { signal: controller.signal, cancel };\n}\n\nfunction topologicalOrder(\n tasks: readonly Task[],\n idToTask: Map>\n): string[] {\n const inDegree = new Map();\n const dependents = new Map();\n\n for (const task of tasks) {\n inDegree.set(task.id, 0);\n dependents.set(task.id, []);\n }\n\n for (const task of tasks) {\n for (const dep of task.deps) {\n inDegree.set(task.id, (inDegree.get(task.id) ?? 0) + 1);\n const list = dependents.get(dep);\n if (list) {\n list.push(task.id);\n }\n }\n }\n\n const queue: string[] = [];\n for (const task of tasks) {\n if ((inDegree.get(task.id) ?? 0) === 0) {\n queue.push(task.id);\n }\n }\n\n const order: string[] = [];\n let head = 0;\n while (head < queue.length) {\n const id = queue[head++];\n order.push(id);\n const next = dependents.get(id) ?? [];\n for (const dependent of next) {\n const remaining = (inDegree.get(dependent) ?? 0) - 1;\n inDegree.set(dependent, remaining);\n if (remaining === 0) {\n queue.push(dependent);\n }\n }\n }\n\n if (order.length !== tasks.length) {\n const visited = new Set();\n const inStack = new Set();\n const path: string[] = [];\n let cyclePath: string[] | undefined;\n\n const dfs = (id: string): void => {\n if (cyclePath !== undefined) return;\n if (inStack.has(id)) {\n const start = path.indexOf(id);\n cyclePath = [...path.slice(start), id];\n return;\n }\n if (visited.has(id)) return;\n visited.add(id);\n inStack.add(id);\n path.push(id);\n const task = idToTask.get(id);\n if (task) {\n for (const dep of task.deps) {\n if (idToTask.has(dep)) {\n dfs(dep);\n if (cyclePath !== undefined) return;\n }\n }\n }\n path.pop();\n inStack.delete(id);\n };\n\n for (const task of tasks) {\n if (cyclePath !== undefined) break;\n dfs(task.id);\n }\n\n throw new CycleError(cyclePath ?? tasks.map((t) => t.id));\n }\n\n return order;\n}\n\nasync function schedule(\n tasks: readonly Task[],\n options: SchedulerOptions = {}\n): Promise> {\n const maxConcurrency = options.maxConcurrency ?? 1;\n if (!Number.isInteger(maxConcurrency) || maxConcurrency < 1) {\n throw new SchedulerError(\"maxConcurrency must be a positive integer\");\n }\n\n const idToTask = new Map>();\n for (const task of tasks) {\n if (idToTask.has(task.id)) {\n throw new DuplicateIdError(task.id);\n }\n idToTask.set(task.id, task);\n }\n\n for (const task of tasks) {\n for (const dep of task.deps) {\n if (!idToTask.has(dep)) {\n throw new UnknownDependencyError(task.id, dep);\n }\n }\n }\n\n const order = topologicalOrder(tasks, idToTask);\n const { signal, cancel } = createCombinedSignal(options.signal);\n\n const results = new Map();\n const failed = new Set();\n const cancelled = new Set();\n const running = new Set();\n const promises = new Map>();\n const readyQueue: string[] = [];\n const readySet = new Set();\n const remainingDeps = new Map();\n\n for (const id of order) {\n const task = idToTask.get(id)!;\n remainingDeps.set(id, task.deps.length);\n }\n\n for (const id of order) {\n const task = idToTask.get(id)!;\n if (task.deps.length === 0) {\n readyQueue.push(id);\n readySet.add(id);\n }\n }\n\n let activeCount = 0;\n let firstError: unknown = undefined;\n let hasError = false;\n let isCancelled = false;\n let isDone = false;\n\n const onAbort = () => {\n if (isCancelled || isDone) return;\n isCancelled = true;\n for (const id of readyQueue.splice(0)) {\n cancelled.add(id);\n }\n for (const id of readySet) {\n if (!running.has(id) && !cancelled.has(id)) {\n cancelled.add(id);\n }\n }\n readySet.clear();\n };\n\n signal.addEventListener(\"abort\", onAbort, { once: true });\n\n const markFailed = (id: string, error: unknown): void => {\n if (failed.has(id)) return;\n failed.add(id);\n if (!hasError) {\n hasError = true;\n firstError = error;\n }\n const dependents = new Set();\n const collectDependents = (taskId: string): void => {\n if (dependents.has(taskId)) return;\n dependents.add(taskId);\n const task = idToTask.get(taskId);\n if (task) {\n for (const dep of task.deps) {\n collectDependents(dep);\n }\n }\n };\n const task = idToTask.get(id);\n if (task) {\n for (const dep of task.deps) {\n collectDependents(dep);\n }\n }\n for (const depId of dependents) {\n if (!failed.has(depId) && !cancelled.has(depId) && !running.has(depId)) {\n cancelled.add(depId);\n }\n }\n };\n\n const tryStartNext = (): void => {\n if (isDone || hasError || isCancelled) return;\n while (activeCount < maxConcurrency && readyQueue.length > 0) {\n const id = readyQueue.shift()!;\n if (readySet.has(id)) {\n readySet.delete(id);\n }\n if (failed.has(id) || cancelled.has(id) || running.has(id)) {\n continue;\n }\n const task = idToTask.get(id)!;\n running.add(id);\n activeCount += 1;\n\n const promise = (async (): Promise => {\n try {\n if (signal.aborted) {\n throw new CancellationError();\n }\n const value = await task.run(signal);\n results.set(id, value);\n } catch (error) {\n if (isCancelled) {\n cancelled.add(id);\n } else {\n markFailed(id, error);\n }\n } finally {\n running.delete(id);\n activeCount -= 1;\n const dependents = new Set();\n const collectDependents = (taskId: string): void => {\n if (dependents.has(taskId)) return;\n dependents.add(taskId);\n const t = idToTask.get(taskId);\n if (t) {\n for (const dep of t.deps) {\n collectDependents(dep);\n }\n }\n };\n for (const [otherId, otherTask] of idToTask) {\n if (otherTask.deps.includes(id)) {\n collectDependents(otherId);\n }\n }\n for (const depId of dependents) {\n if (failed.has(depId) || cancelled.has(depId) || running.has(depId)) {\n continue;\n }\n const remaining = (remainingDeps.get(depId) ?? 0) - 1;\n remainingDeps.set(depId, remaining);\n if (remaining === 0 && !readySet.has(depId) && !readyQueue.includes(depId)) {\n readyQueue.push(depId);\n readySet.add(depId);\n }\n }\n tryStartNext();\n }\n })();\n\n promises.set(id, promise);\n }\n };\n\n const allSettled = (async (): Promise => {\n try {\n tryStartNext();\n while (!isDone) {\n if (hasError || isCancelled) {\n isDone = true;\n break;\n }\n if (activeCount === 0 && readyQueue.length === 0) {\n isDone = true;\n break;\n }\n const pending = [...promises.values()];\n if (pending.length === 0) {\n isDone = true;\n break;\n }\n await Promise.race(pending);\n }\n } finally {\n signal.removeEventListener(\"abort\", onAbort);\n }\n })();\n\n const resultPromise = allSettled.then(\n () => {\n if (isCancelled) {\n throw new CancellationError();\n }\n if (hasError) {\n throw new TaskFailureError(\n [...failed][0] ?? \"unknown\",\n firstError\n );\n }\n return results;\n },\n (error) => {\n if (isCancelled) {\n throw new CancellationError();\n }\n throw error;\n }\n );\n\n resultPromise.catch(() => undefined);\n\n return resultPromise;\n}\n\nfunction assertEqual(actual: T, expected: T, message: string): void {\n if (actual !== expected) {\n throw new Error(`${message}: expected ${String(expected)}, got ${String(actual)}`);\n }\n}\n\nfunction assertDeepEqual(actual: T, expected: T, message: string): void {\n const a = JSON.stringify(actual);\n const e = JSON.stringify(expected);\n if (a !== e) {\n throw new Error(`${message}: expected ${e}, got ${a}`);\n }\n}\n\nfunction assertThrows(\n fn: () => unknown,\n errorClass: new (...args: never[]) => Error,\n message: string\n): void {\n try {\n fn();\n throw new Error(`${message}: expected throw, but did not throw`);\n }", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json new file mode 100644 index 0000000..136941d --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T121823Z-vllm-metal-qwen3.8-27b-nvfp4/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:10:51.466896+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.0361592500121333, + "total_s": 133.43742824997753, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 23.20219453486358, + "finish_reason": "length", + "output_chars": 14058, + "output_sha256": "a18e80f1852ecc0c70b987f244870b8a2dc20db8ad42505280162a4b17dbd057", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:13:04.907109+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 15.930970875022467, + "total_s": 166.00547937501688, + "prompt_tokens": 182, + "output_tokens": 2840, + "decode_tps": 18.923933374068692, + "finish_reason": "stop", + "output_chars": 11403, + "output_sha256": "bb758808663b88bb3fbacc52df49598eb21aa8bc7fa0ee8466410f0ae13e8b0d", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "started_at": "2026-08-31T12:15:50.913910+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4814700839924626, + "total_s": 152.22010587499244, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 20.379645761550783, + "finish_reason": "length", + "output_chars": 10869, + "output_sha256": "f700d51c471a59acfcf85016d416f0c5354662552fea44b48f56d80c3afdcb59", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json new file mode 100644 index 0000000..dd86822 --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T123006Z", + "created_at": "2026-08-31T12:30:06.184278+00:00", + "runtime": { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mxfp8" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:22:38.815059+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.44539537496166304, + "total_s": 0.44552537496201694, + "prompt_tokens": 41, + "output_tokens": 1, + "decode_tps": 8.241328039906357, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:22:39.260933+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1107976250350475, + "total_s": 149.8235780420364, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 20.63948064778098, + "finish_reason": "length", + "output_chars": 13492, + "output_sha256": "3e5ca7654fc69293040b7cc1b554ccddbd1a99b82710d361074fee260803182f", + "output": "import asyncio\nimport time\nfrom collections import OrderedDict\nfrom dataclasses import dataclass\nfrom typing import (\n Any,\n Awaitable,\n Callable,\n Generic,\n Optional,\n TypeVar,\n)\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n\n\nclass _Entry:\n __slots__ = (\"value\", \"expires_at\", \"weight\")\n\n def __init__(self, value: Any, expires_at: float, weight: int) -> None:\n self.value = value\n self.expires_at = expires_at\n self.weight = weight\n\n\nclass _Inflight:\n __slots__ = (\"event\", \"value\", \"error\", \"cancelled\")\n\n def __init__(self) -> None:\n self.event = asyncio.Event()\n self.value: Any = None\n self.error: Optional[BaseException] = None\n self.cancelled: bool = False\n\n\nclass AsyncTTLCache(Generic[K, V]):\n \"\"\"\n A generic async cache with:\n - Weighted LRU capacity (evict least-recently-used entries until total weight <= max_weight).\n - Per-entry TTL based on time.monotonic.\n - Single-flight get_or_create: concurrent misses for the same key invoke the async factory\n exactly once.\n - Cancellation of one waiter does not cancel the shared factory.\n - Exceptions are never cached.\n - Invalidating a key while its factory is running prevents stale publication.\n - The cache lock is never held while awaiting user code.\n - clear(), invalidate(), and immutable hit/miss/eviction statistics.\n \"\"\"\n\n def __init__(\n self,\n max_weight: int,\n default_ttl: float,\n weight_fn: Optional[Callable[[V], int]] = None,\n ) -> None:\n if max_weight <= 0:\n raise ValueError(\"max_weight must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n\n self._max_weight: int = max_weight\n self._default_ttl: float = default_ttl\n self._weight_fn: Callable[[V], int] = weight_fn or (lambda _v: 1)\n\n self._lock: asyncio.Lock = asyncio.Lock()\n self._store: OrderedDict[K, _Entry] = OrderedDict()\n self._current_weight: int = 0\n self._inflight: dict[K, _Inflight] = {}\n\n self._hits: int = 0\n self._misses: int = 0\n self._evictions: int = 0\n\n # ------------------------------------------------------------------\n # Public API\n # ------------------------------------------------------------------\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n ) -> V:\n \"\"\"\n Return the cached value for *key*, or invoke *factory* to create it.\n\n If multiple coroutines call get_or_create for the same key concurrently\n and the key is not yet cached, the factory is invoked exactly once.\n All waiters receive the same result (or the same exception).\n \"\"\"\n effective_ttl = ttl if ttl is not None else self._default_ttl\n\n # Fast path: check cache under lock\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None:\n if time.monotonic() < entry.expires_at:\n # Move to end (most recently used)\n self._store.move_to_end(key)\n self._hits += 1\n return entry.value\n else:\n # Expired: remove and fall through to factory\n self._store.pop(key, None)\n self._current_weight -= entry.weight\n\n # Check if there is already an in-flight factory for this key\n inflight = self._inflight.get(key)\n if inflight is not None:\n # Join the existing in-flight operation\n pass\n else:\n # Start a new in-flight operation\n inflight = _Inflight()\n self._inflight[key] = inflight\n self._misses += 1\n\n # If we joined an existing in-flight, wait for it\n if inflight.cancelled:\n # The in-flight was invalidated; we need to start our own\n # Re-acquire lock to check again\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None and time.monotonic() < entry.expires_at:\n self._store.move_to_end(key)\n self._hits += 1\n return entry.value\n # Check if another in-flight started\n inflight = self._inflight.get(key)\n if inflight is not None and not inflight.cancelled:\n pass # fall through to wait\n else:\n # Start a new in-flight\n inflight = _Inflight()\n self._inflight[key] = inflight\n self._misses += 1\n\n # Wait for the in-flight to complete\n await inflight.event.wait()\n\n if inflight.error is not None:\n raise inflight.error\n return inflight.value\n\n async def _run_factory(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: float,\n inflight: _Inflight,\n ) -> None:\n \"\"\"\n Run the factory outside the lock. This is called by get_or_create\n after acquiring the lock and setting up the inflight entry.\n \"\"\"\n try:\n value = await factory()\n except BaseException as exc:\n # Mark the inflight as failed\n inflight.error = exc\n inflight.event.set()\n # Remove inflight entry\n async with self._lock:\n # Only remove if it's still the same inflight\n if self._inflight.get(key) is inflight:\n del self._inflight[key]\n return\n\n # Check if the key was invalidated while the factory was running\n async with self._lock:\n if self._inflight.get(key) is not inflight:\n # The inflight was replaced (e.g., by invalidate), so this result is stale\n return\n if inflight.cancelled:\n # The inflight was cancelled, so this result is stale\n return\n\n # Check if the key is already in the cache (another coroutine may have\n # published a value, though this shouldn't happen with single-flight)\n if key in self._store:\n # Already cached, don't overwrite\n del self._inflight[key]\n return\n\n # Publish the value\n weight = self._weight_fn(value)\n if weight > self._max_weight:\n # Single item exceeds max weight; don't cache it\n del self._inflight[key]\n inflight.value = value\n inflight.event.set()\n return\n\n # Evict if necessary\n self._evict_for_weight(weight)\n\n expires_at = time.monotonic() + ttl\n self._store[key] = _Entry(value, expires_at, weight)\n self._current_weight += weight\n\n # Mark the inflight as complete\n del self._inflight[key]\n inflight.value = value\n inflight.event.set()\n\n async def invalidate(self, key: K) -> None:\n \"\"\"\n Invalidate a key. If a factory is currently running for this key,\n the result will not be published.\n \"\"\"\n async with self._lock:\n # Remove from cache if present\n entry = self._store.pop(key, None)\n if entry is not None:\n self._current_weight -= entry.weight\n\n # Cancel any in-flight factory for this key\n inflight = self._inflight.get(key)\n if inflight is not None:\n inflight.cancelled = True\n inflight.event.set()\n del self._inflight[key]\n\n async def clear(self) -> None:\n \"\"\"\n Clear all entries and cancel all in-flight factories.\n \"\"\"\n async with self._lock:\n self._store.clear()\n self._current_weight = 0\n # Cancel all in-flight factories\n for inflight in self._inflight.values():\n inflight.cancelled = True\n inflight.event.set()\n self._inflight.clear()\n\n @property\n def stats(self) -> CacheStats:\n \"\"\"\n Return an immutable snapshot of cache statistics.\n \"\"\"\n return CacheStats(\n hits=self._hits,\n misses=self._misses,\n evictions=self._evictions,\n )\n\n # ------------------------------------------------------------------\n # Internal helpers\n # ------------------------------------------------------------------\n\n def _evict_for_weight(self, incoming_weight: int) -> None:\n \"\"\"\n Evict least-recently-used entries until there is room for incoming_weight.\n This is called while holding the lock.\n \"\"\"\n while self._current_weight + incoming_weight > self._max_weight:\n if not self._store:\n break\n # Pop the least-recently-used entry (first in OrderedDict)\n key, entry = self._store.popitem(last=False)\n self._current_weight -= entry.weight\n self._evictions += 1\n\n # ------------------------------------------------------------------\n # Override get_or_create to properly handle the factory execution\n # ------------------------------------------------------------------\n\n async def get_or_create( # type: ignore[override]\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n *,\n ttl: Optional[float] = None,\n ) -> V:\n \"\"\"\n Return the cached value for *key*, or invoke *factory* to create it.\n\n If multiple coroutines call get_or_create for the same key concurrently\n and the key is not yet cached, the factory is invoked exactly once.\n All waiters receive the same result (or the same exception).\n \"\"\"\n effective_ttl = ttl if ttl is not None else self._default_ttl\n\n # Fast path: check cache under lock\n async with self._lock:\n entry = self._store.get(key)\n if entry is not None:\n if time.monotonic() < entry.expires_at:\n # Move to end (most recently used)\n self._store.move_to_end(key)\n self._hits += 1\n return entry.value\n else:\n # Expired: remove and fall through to factory\n self._store.pop(key, None)\n self._current_weight -= entry.weight\n\n # Check if there is already an in-flight factory for this key\n inflight = self._inflight.get(key)\n if inflight is not None and not inflight.cancelled:\n # Join the existing in-flight operation\n pass\n else:\n # Start a new in-flight operation\n inflight = _Inflight()\n self._inflight[key] = inflight\n self._misses += 1\n # We need to run the factory\n should_run = True\n else:\n should_run = False\n\n if should_run:\n # Run the factory outside the lock\n await self._run_factory(key, factory, effective_ttl, inflight)\n\n # Wait for the in-flight to complete\n await inflight.event.wait()\n\n if inflight.error is not None:\n raise inflight.error\n return inflight.value\n\n\n# ------------------------------------------------------------------\n# Tests\n# ------------------------------------------------------------------\n\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_basic_get_or_create() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=10.0)\n call_count = 0\n\n async def factory() -> str:\n nonlocal call_count\n call_count += 1\n return \"value\"\n\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert call_count == 1\n\n # Second call should hit the cache\n result = await cache.get_or_create(1, factory)\n assert result == \"value\"\n assert call_count == 1\n assert cache.stats.hits == 1\n assert cache.stats.misses == 1\n\n\n@pytest.mark.asyncio\nasync def test_single_flight() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=10.0)\n call_count = 0\n\n async def factory() -> str:\n nonlocal call_count\n call_count += 1\n await asyncio.sleep(0.05)\n return \"value\"\n\n # Launch multiple concurrent get_or_create for the same key\n results = await asyncio.gather(\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n cache.get_or_create(1, factory),\n )\n\n assert all(r == \"value\" for r in results)\n assert call_count == 1\n assert cache.stats.misses == 1\n\n\n@pytest.mark.asyncio\nasync def test_cancellation_does_not_cancel_factory() -> None:\n cache = AsyncTTLCache[int, str](max_weight=10, default_ttl=10.0)\n call_count = 0\n factory_completed = asyncio.Event()\n\n async def factory() -> str:\n nonlocal call_count\n call_count += 1\n await asyncio.sleep(0.05)\n factory_completed.set()\n return \"value\"\n\n # Start a get", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:25:09.083177+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6688512500259094, + "total_s": 149.76781020802446, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 20.71737613932144, + "finish_reason": "length", + "output_chars": 12642, + "output_sha256": "71ee449c3f573828250542e04f7a63db5a8d23087bdec89f0a9a0bb31d52a3b5", + "output": " 0)\n * @param string $idempotencyKey Unique client-supplied key for idempotency\n *\n * @return array{\n * transfer_id: int,\n * from_account_id: int,\n * to_account_id: int,\n * amount_cents: int,\n * idempotency_key: string,\n * created_at: string,\n * was_duplicate: bool\n * }\n *\n * @throws \\InvalidArgumentException If amount is not positive or accounts are the same\n * @throws \\RuntimeException If insufficient funds or after exhausting deadlock retries\n */\n public function transfer(\n int $fromAccountId,\n int $toAccountId,\n int $amountCents,\n string $idempotencyKey\n ): array {\n // Validate positive amount\n if ($amountCents <= 0) {\n throw new \\InvalidArgumentException('Transfer amount must be a positive integer number of cents.');\n }\n\n // Validate distinct accounts\n if ($fromAccountId === $toAccountId) {\n throw new \\InvalidArgumentException('Source and destination accounts must be distinct.');\n }\n\n // Validate idempotency key is non-empty\n if (trim($idempotencyKey) === '') {\n throw new \\InvalidArgumentException('Idempotency key must be a non-empty string.');\n }\n\n $lastException = null;\n\n for ($attempt = 0; $attempt < self::MAX_DEADLOCK_RETRIES; $attempt++) {\n try {\n return $this->executeTransfer(\n $fromAccountId,\n $toAccountId,\n $amountCents,\n $idempotencyKey\n );\n } catch (\\Throwable $e) {\n // Detect deadlock (MySQL error 1213, PostgreSQL 40P01, etc.)\n if ($this->isDeadlockException($e)) {\n $lastException = $e;\n // Exponential backoff with jitter\n $delayMs = self::DEADLOCK_RETRY_BASE_DELAY_MS * (2 ** $attempt) + random_int(0, 20);\n usleep($delayMs * 1000);\n continue;\n }\n // Non-deadlock exception: rethrow immediately\n throw $e;\n }\n }\n\n // All retries exhausted due to deadlocks\n throw new \\RuntimeException(\n 'Transfer failed after ' . self::MAX_DEADLOCK_RETRIES . ' deadlock retries.',\n 0,\n $lastException\n );\n }\n\n /**\n * Execute the transfer inside a database transaction.\n *\n * @return array{\n * transfer_id: int,\n * from_account_id: int,\n * to_account_id: int,\n * amount_cents: int,\n * idempotency_key: string,\n * created_at: string,\n * was_duplicate: bool\n * }\n */\n private function executeTransfer(\n int $fromAccountId,\n int $toAccountId,\n int $amountCents,\n string $idempotencyKey\n ): array {\n return DB::transaction(function () use ($fromAccountId, $toAccountId, $amountCents, $idempotencyKey) {\n // Check for existing transfer with this idempotency key (idempotency check)\n $existingTransfer = DB::table('transfers')\n ->where('idempotency_key', $idempotencyKey)\n ->first();\n\n if ($existingTransfer !== null) {\n // Return the original transfer result; no state changes\n return [\n 'transfer_id' => (int) $existingTransfer->id,\n 'from_account_id' => (int) $existingTransfer->from_account_id,\n 'to_account_id' => (int) $existingTransfer->to_account_id,\n 'amount_cents' => (int) $existingTransfer->amount_cents,\n 'idempotency_key' => $existingTransfer->idempotency_key,\n 'created_at' => $existingTransfer->created_at,\n 'was_duplicate' => true,\n ];\n }\n\n // Lock both account rows in deterministic ID order to prevent deadlocks.\n // We always lock the lower ID first, then the higher ID.\n $lowerId = min($fromAccountId, $toAccountId);\n $higherId = max($fromAccountId, $toAccountId);\n\n // Lock the lower-ID account row\n $lowerAccount = DB::table('accounts')\n ->where('id', $lowerId)\n ->lockForUpdate()\n ->first();\n\n if ($lowerAccount === null) {\n throw new \\RuntimeException(\"Account with ID {$lowerId} not found.\");\n }\n\n // Lock the higher-ID account row\n $higherAccount = DB::table('accounts')\n ->where('id', $higherId)\n ->lockForUpdate()\n ->first();\n\n if ($higherAccount === null) {\n throw new \\RuntimeException(\"Account with ID {$higherId} not found.\");\n }\n\n // Identify which locked row is the source and which is the destination\n if ($fromAccountId === $lowerId) {\n $fromAccount = $lowerAccount;\n $toAccount = $higherAccount;\n } else {\n $fromAccount = $higherAccount;\n $toAccount = $lowerAccount;\n }\n\n // Check sufficient funds\n if ($fromAccount->balance_cents < $amountCents) {\n throw new \\RuntimeException(\n \"Insufficient funds: account {$fromAccountId} has \" .\n $fromAccount->balance_cents . \" cents, but {$amountCents} cents are required.\"\n );\n }\n\n // Perform the transfer: debit source, credit destination\n DB::table('accounts')\n ->where('id', $fromAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents - ' . $amountCents),\n ]);\n\n DB::table('accounts')\n ->where('id', $toAccountId)\n ->update([\n 'balance_cents' => DB::raw('balance_cents + ' . $amountCents),\n ]);\n\n // Insert the transfer record\n $now = date('Y-m-d H:i:s');\n $transferId = DB::table('transfers')->insertGetId([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'created_at' => $now,\n ]);\n\n // Persist an outbox event in the same transaction\n DB::table('outbox_events')->insert([\n 'transfer_id' => $transferId,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transferId,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'idempotency_key' => $idempotencyKey,\n ]),\n 'created_at' => $now,\n ]);\n\n return [\n 'transfer_id' => (int) $transferId,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'idempotency_key' => $idempotencyKey,\n 'created_at' => $now,\n 'was_duplicate' => false,\n ];\n });\n }\n\n /**\n * Determine whether a Throwable represents a database deadlock.\n *\n * @param \\Throwable $e\n * @return bool\n */\n private function isDeadlockException(\\Throwable $e): bool\n {\n // MySQL/InnoDB deadlock: SQLSTATE 40001, error code 1213\n // PostgreSQL deadlock: SQLSTATE 40P01\n // We check the message and any nested exception for common deadlock indicators.\n\n $message = $e->getMessage();\n\n // Check for common deadlock error codes/messages\n if (\n str_contains($message, '1213') ||\n str_contains($message, 'Deadlock found') ||\n str_contains($message, 'deadlock') ||\n str_contains($message, '40P01') ||\n str_contains($message, '40001')\n ) {\n return true;\n }\n\n // Check if it's a PDOException with a deadlock SQLSTATE\n if ($e instanceof \\PDOException) {\n $sqlState = $e->getCode();\n if ($sqlState === '40001' || $sqlState === '40P01') {\n return true;\n }\n }\n\n // Check previous exception in the chain\n if ($e->getPrevious() !== null) {\n return $this->isDeadlockException($e->getPrevious());\n }\n\n return false;\n }\n}\n\n/**\n * PHPUnit test suite for MoneyTransferService.\n *\n * These tests use an in-memory SQLite database to verify:\n * 1. Duplicate requests (same idempotency key) return the original transfer without\n * duplicating effects.\n * 2. Competing requests (concurrent transfers with different keys) preserve balances\n * and create exactly one transfer and one outbox event per successful transfer.\n * 3. Insufficient funds are rejected.\n * 4. Invalid inputs (zero/negative amount, same account) are rejected.\n */\nclass MoneyTransferServiceTest extends TestCase\n{\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Configure an in-memory SQLite database for testing\n $this->configureInMemoryDatabase();\n\n $this->service = new MoneyTransferService();\n }\n\n /**\n * Configure Laravel's DB facade to use an in-memory SQLite database\n * and create the required tables.\n */\n private function configureInMemoryDatabase(): void\n {\n // We need to bootstrap a minimal Laravel container for the DB facade.\n // In a real Laravel 12 app, this would be handled by the framework.\n // Here we simulate it using the Illuminate\\Database Capsule.\n\n $capsule = new \\Illuminate\\Database\\Capsule\\Manager();\n $capsule->addConnection([\n 'driver' => 'sqlite',\n 'database' => ':memory:',\n 'prefix' => '',\n ]);\n $capsule->setAsGlobal();\n $capsule->bootEloquent();\n\n // Create tables\n $schema = $capsule->schema();\n\n $schema->create('accounts', function ($table) {\n $table->bigIncrements('id');\n $table->big", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:27:38.904048+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.6032058749697171, + "total_s": 147.27845333400182, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 21.060449812696582, + "finish_reason": "length", + "output_chars": 11368, + "output_sha256": "a3d74aa8a8183babda562bc53b32dda3b1b3ad318788d2384cc8babff3df3223", + "output": "// dag-scheduler.ts\n// Strict-mode, dependency-free DAG task scheduler.\n\n// ---------------------------------------------------------------------------\n// Types\n// ---------------------------------------------------------------------------\n\nexport interface Task {\n readonly id: string;\n readonly deps: readonly string[];\n readonly run: (signal: AbortSignal) => Promise;\n}\n\nexport interface ScheduleOptions {\n readonly maxConcurrency?: number;\n readonly signal?: AbortSignal;\n}\n\nexport type ScheduleResult = Map;\n\n// ---------------------------------------------------------------------------\n// Errors\n// ---------------------------------------------------------------------------\n\nexport class CycleError extends Error {\n readonly path: readonly string[];\n constructor(path: readonly string[]) {\n super(`Cycle detected: ${path.join(\" -> \")}`);\n this.name = \"CycleError\";\n this.path = path;\n }\n}\n\nexport class UnknownDependencyError extends Error {\n readonly taskId: string;\n readonly depId: string;\n constructor(taskId: string, depId: string) {\n super(`Task \"${taskId}\" has unknown dependency \"${depId}\"`);\n this.name = \"UnknownDependencyError\";\n this.taskId = taskId;\n this.depId = depId;\n }\n}\n\nexport class DuplicateIdError extends Error {\n readonly id: string;\n constructor(id: string) {\n super(`Duplicate task id: \"${id}\"`);\n this.name = \"DuplicateIdError\";\n this.id = id;\n }\n}\n\n// ---------------------------------------------------------------------------\n// Internal helpers\n// ---------------------------------------------------------------------------\n\n/**\n * Detect a cycle in a directed graph using iterative DFS.\n * Returns the cycle path (list of node ids) or null if acyclic.\n */\nfunction detectCycle(\n nodes: readonly string[],\n adj: Map\n): string[] | null {\n const WHITE = 0;\n const GRAY = 1;\n const BLACK = 2;\n\n const color = new Map();\n for (const n of nodes) color.set(n, WHITE);\n\n // parent map for path reconstruction\n const parent = new Map();\n\n for (const start of nodes) {\n if (color.get(start) !== WHITE) continue;\n\n // Iterative DFS with explicit stack of [node, nextChildIndex]\n const stack: Array<{ node: string; idx: number }> = [{ node: start, idx: 0 }];\n parent.set(start, null);\n\n while (stack.length > 0) {\n const frame = stack[stack.length - 1];\n const node = frame.node;\n const neighbors = adj.get(node) ?? [];\n\n if (frame.idx < neighbors.length) {\n const next = neighbors[frame.idx];\n frame.idx++;\n\n const c = color.get(next);\n if (c === GRAY) {\n // Found a cycle: reconstruct path from `next` back to `node`\n const path: string[] = [next, node];\n let cur: string | null = node;\n while (cur !== null && cur !== next) {\n const p = parent.get(cur);\n if (p === undefined || p === null) break;\n path.push(p);\n cur = p;\n }\n path.reverse();\n return path;\n }\n if (c === WHITE) {\n color.set(next, GRAY);\n parent.set(next, node);\n stack.push({ node: next, idx: 0 });\n }\n } else {\n color.set(node, BLACK);\n stack.pop();\n }\n }\n }\n\n return null;\n}\n\n/**\n * Compute a stable topological order using Kahn's algorithm.\n * Ties are broken by original input order (stable).\n */\nfunction topologicalOrder(\n tasks: readonly Task[]\n): string[] {\n const idToIndex = new Map();\n for (let i = 0; i < tasks.length; i++) {\n idToIndex.set(tasks[i].id, i);\n }\n\n const inDegree = new Map();\n const adj = new Map();\n\n for (const t of tasks) {\n inDegree.set(t.id, 0);\n adj.set(t.id, []);\n }\n\n for (const t of tasks) {\n for (const dep of t.deps) {\n // dep -> t edge\n adj.get(dep)!.push(t.id);\n inDegree.set(t.id, (inDegree.get(t.id) ?? 0) + 1);\n }\n }\n\n // Use a min-heap-like approach: pick the ready node with the smallest\n // original index to ensure stable input-order scheduling.\n const ready: string[] = [];\n for (const t of tasks) {\n if ((inDegree.get(t.id) ?? 0) === 0) {\n ready.push(t.id);\n }\n }\n // Sort ready by original index for stability\n ready.sort((a, b) => (idToIndex.get(a)! - idToIndex.get(b)!));\n\n const order: string[] = [];\n while (ready.length > 0) {\n // Pick the first (smallest original index)\n const node = ready.shift()!;\n order.push(node);\n\n const children = adj.get(node) ?? [];\n for (const child of children) {\n const deg = (inDegree.get(child) ?? 0) - 1;\n inDegree.set(child, deg);\n if (deg === 0) {\n // Insert into ready maintaining sorted order by original index\n const idx = idToIndex.get(child)!;\n // Binary search for insertion point\n let lo = 0;\n let hi = ready.length;\n while (lo < hi) {\n const mid = (lo + hi) >> 1;\n if (idToIndex.get(ready[mid])! < idx) {\n lo = mid + 1;\n } else {\n hi = mid;\n }\n }\n ready.splice(lo, 0, child);\n }\n }\n }\n\n return order;\n}\n\n// ---------------------------------------------------------------------------\n// Scheduler\n// ---------------------------------------------------------------------------\n\nexport async function schedule(\n tasks: readonly Task[],\n options: ScheduleOptions = {}\n): Promise> {\n const maxConcurrency = options.maxConcurrency ?? Infinity;\n const externalSignal = options.signal;\n\n // 1. Validate: duplicate IDs\n const seen = new Set();\n for (const t of tasks) {\n if (seen.has(t.id)) {\n throw new DuplicateIdError(t.id);\n }\n seen.add(t.id);\n }\n\n // 2. Validate: unknown dependencies\n for (const t of tasks) {\n for (const dep of t.deps) {\n if (!seen.has(dep)) {\n throw new UnknownDependencyError(t.id, dep);\n }\n }\n }\n\n // 3. Build adjacency and detect cycles\n const ids = tasks.map((t) => t.id);\n const adj = new Map();\n for (const t of tasks) {\n adj.set(t.id, t.deps);\n }\n\n const cycle = detectCycle(ids, adj);\n if (cycle !== null) {\n throw new CycleError(cycle);\n }\n\n // 4. Compute stable topological order\n const topo = topologicalOrder(tasks);\n\n // 5. Build a lookup from id to task\n const taskById = new Map>();\n for (const t of tasks) {\n taskById.set(t.id, t);\n }\n\n // 6. Compute in-degree for scheduling\n const inDegree = new Map();\n const dependents = new Map(); // dep -> list of task ids that depend on it\n\n for (const t of tasks) {\n inDegree.set(t.id, t.deps.length);\n for (const dep of t.deps) {\n if (!dependents.has(dep)) {\n dependents.set(dep, []);\n }\n dependents.get(dep)!.push(t.id);\n }\n }\n\n // 7. Scheduling state\n const results = new Map();\n const running = new Set>();\n const failed = new Set(); // tasks that failed or were skipped due to failure\n let firstError: unknown = null;\n let hasError = false;\n\n // AbortController for internal cancellation\n const internalAbort = new AbortController();\n const onExternalAbort = () => {\n internalAbort.abort();\n };\n if (externalSignal) {\n if (externalSignal.aborted) {\n internalAbort.abort();\n } else {\n externalSignal.addEventListener(\"abort\", onExternalAbort, { once: true });\n }\n }\n\n // Track which tasks have been \"started\" (their run() has been called)\n const started = new Set();\n\n // Helper: check if a task should be skipped because a dependency failed\n function shouldSkip(taskId: string): boolean {\n const t = taskById.get(taskId)!;\n for (const dep of t.deps) {\n if (failed.has(dep)) {\n return true;\n }\n }\n return false;\n }\n\n // Helper: mark a task and all its transitive dependents as failed/skipped\n function markFailedAndDependents(taskId: string): void {\n if (failed.has(taskId)) return;\n failed.add(taskId);\n const queue: string[] = [taskId];\n while (queue.length > 0) {\n const cur = queue.shift()!;\n const deps = dependents.get(cur) ?? [];\n for (const dep of deps) {\n if (!failed.has(dep)) {\n failed.add(dep);\n queue.push(dep);\n }\n }\n }\n }\n\n // Helper: try to start a task if all deps are done and it's not failed\n function tryStart(taskId: string): void {\n if (started.has(taskId) || failed.has(taskId)) return;\n const t = taskById.get(taskId)!;\n // Check all deps are completed (in results) and none failed\n for (const dep of t.deps) {\n if (!results.has(dep)) return; // dep not yet done\n if (failed.has(dep)) {\n markFailedAndDependents(taskId);\n return;\n }\n }\n // All deps satisfied and none failed\n started.add(taskId);\n\n const p = (async () => {\n try {\n const result = await t.run(internalAbort.signal);\n if (hasError) return; // already failed, don't record\n results.set(taskId, result);\n } catch (err) {\n if (!hasError) {\n hasError = true;\n firstError = err;\n markFailedAndDependents(taskId);\n // Abort all running tasks\n internalAbort.abort();\n }\n // Re-throw to let the outer catch handle it, but we don't want\n // unhandled rejection. We'll handle it in the running set.\n throw err;\n }\n })();\n\n // Attach a catch to prevent unhandled rejection\n p.catch(() => {\n // Already handled above; this is just to prevent unhandled rejection\n });\n\n running.add(p);\n p.finally(() => {\n running.delete(p);\n });\n }\n\n // 8. Initial pass: start all tasks with no dependencies\n for (const taskId of topo) {\n const t = taskById.get(taskId)!;\n if (t.deps.length === 0) {\n tryStart(taskId);\n }\n }\n\n // 9. Main loop: wait for tasks to complete, then start newly-ready tasks\n // We use a promise-based approach: wait for at least one running task to finish,\n // then check for newly ready tasks.\n\n // We need to handle the case where no tasks are running but there are still\n // tasks to start (shouldn't happen in a valid DAG, but be safe).\n\n // Use a \"wake\" mechanism: when a task completes, we check for newly ready tasks.\n // We'll use a simple approach: poll the running set.\n\n // Actually, a cleaner approach: use a \"completion\" event.\n // Let's restructure: we'll use a queue of \"ready\" tasks and a running set.\n\n // Reset and use a cleaner scheduling loop\n // Clear previous state\n running.clear();\n started.clear();\n results.clear();\n failed.clear();\n hasError = false;\n firstError = null;\n\n // Ready queue: tasks whose deps are all satisfied\n const readyQueue: string[] = [];\n for (const taskId of topo) {\n const t = taskById.get(taskId)!;\n if (t.deps.length === 0) {\n readyQueue.push(taskId);\n }\n }\n\n // Process ready queue: start tasks up to maxConcurrency\n function processReady(): void {\n while (readyQueue.length > 0 && running.size < maxConcurrency && !hasError) {\n const taskId = readyQueue.shift()!;\n if (failed.has(taskId)) continue;\n if (started.has", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json new file mode 100644 index 0000000..5fff934 --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T123006Z-ollama-qwen3.8_27b-mxfp8/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:22:39.260933+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1107976250350475, + "total_s": 149.8235780420364, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 20.63948064778098, + "finish_reason": "length", + "output_chars": 13492, + "output_sha256": "3e5ca7654fc69293040b7cc1b554ccddbd1a99b82710d361074fee260803182f", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:25:09.083177+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6688512500259094, + "total_s": 149.76781020802446, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 20.71737613932144, + "finish_reason": "length", + "output_chars": 12642, + "output_sha256": "71ee449c3f573828250542e04f7a63db5a8d23087bdec89f0a9a0bb31d52a3b5", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "started_at": "2026-08-31T12:27:38.904048+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.6032058749697171, + "total_s": 147.27845333400182, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 21.060449812696582, + "finish_reason": "length", + "output_chars": 11368, + "output_sha256": "a3d74aa8a8183babda562bc53b32dda3b1b3ad318788d2384cc8babff3df3223", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json new file mode 100644 index 0000000..2baad10 --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/results.json @@ -0,0 +1,111 @@ +{ + "run_id": "20260831T133452Z", + "created_at": "2026-08-31T13:34:52.367128+00:00", + "runtime": { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-mxfp8" + }, + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "package_versions": { + "vllm": "0.28.0+cpu", + "vllm-metal": "0.4.0.dev20260831112704", + "mlx": "0.32.0", + "mlx-lm": "0.31.3", + "mlx-vlm": "0.6.4" + }, + "settings": { + "temperature": 0, + "seed": 42, + "max_tokens": 3072, + "context_tokens": 8192, + "raw_completion": false, + "chat_template": true, + "thinking": false, + "ollama_speculative_drafting": false + }, + "warmup": { + "task": "warmup", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:22:56.588802+00:00", + "prompt_chars": 27, + "max_tokens": 8, + "ttft_s": 0.38669666700297967, + "total_s": 0.44958637503441423, + "prompt_tokens": 41, + "output_tokens": 2, + "decode_tps": 31.801705916655347, + "finish_reason": "stop", + "output_chars": 5, + "output_sha256": "c2e3ac47f4a325469c1a2d5f117e463ec943c721986d5d9f09ac4540b7d80526", + "output": "READY", + "error": null + }, + "tasks": { + "python_async_cache": "Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module\ncontaining a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based\non time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async\nfactory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are\nnever cached; invalidating a key while its factory is running must prevent stale publication; never\nhold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction\nstatistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted\ncapacity, factory failure, and invalidation during creation. Use only the standard library plus\npytest. Output only the complete module, without Markdown fences or prose.", + "php_idempotent_transfer": "Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an\nidempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements:\nvalidate positive amount and distinct accounts; use a database transaction; lock both account rows\nin deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries\nreturn the original transfer; retry deadlocks without duplicating effects; persist an outbox event\nin the same transaction; and prove with tests that duplicate and competing requests preserve\nbalances, create one transfer, and create one outbox event. Include minimal model/table assumptions\nas comments.\nOutput only valid PHP, without Markdown fences or prose.", + "typescript_dag_scheduler": "Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in\none TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function.\nRequirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable\ninput-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the\nfirst error while awaiting already-running tasks; never start dependents of failed tasks; return a\nMap of typed results on success; and avoid unhandled promise rejections. Include a compact test\nharness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external\ncancellation, and rejection. Output only TypeScript, without Markdown fences or prose." + }, + "results": [ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:22:57.038624+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1419034169521183, + "total_s": 247.70563520898577, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 12.459253344652916, + "finish_reason": "length", + "output_chars": 13028, + "output_sha256": "699045091c4f5bc19d0f8f0bbc676b164512c8e3e7dec0f95acb13b5bef2f4ba", + "output": "from __future__ import annotations\n\nimport asyncio\nimport time\nfrom collections import OrderedDict\nfrom dataclasses import dataclass, field\nfrom typing import Any, Awaitable, Callable, Generic, TypeVar\n\nK = TypeVar(\"K\")\nV = TypeVar(\"V\")\n\n\n@dataclass(frozen=True)\nclass CacheStats:\n hits: int\n misses: int\n evictions: int\n\n\n@dataclass\nclass _Entry:\n value: Any\n expires_at: float\n weight: int\n key: Any\n\n\nclass AsyncTTLCache(Generic[K, V]):\n \"\"\"\n A generic async TTL cache with weighted LRU capacity, per-entry TTL,\n single-flight get_or_create, and safe cancellation semantics.\n \"\"\"\n\n def __init__(\n self,\n max_weight: int,\n default_ttl: float,\n weight_fn: Callable[[K, V], int] | None = None,\n ) -> None:\n if max_weight <= 0:\n raise ValueError(\"max_weight must be positive\")\n if default_ttl <= 0:\n raise ValueError(\"default_ttl must be positive\")\n\n self._max_weight: int = max_weight\n self._default_ttl: float = default_ttl\n self._weight_fn: Callable[[K, V], int] | None = weight_fn\n\n self._lock = asyncio.Lock()\n self._entries: OrderedDict[K, _Entry] = OrderedDict()\n self._current_weight: int = 0\n\n # Single-flight: key -> (future, task)\n self._inflight: dict[K, tuple[asyncio.Future[V], asyncio.Task[V]]] = {}\n\n # Statistics\n self._hits: int = 0\n self._misses: int = 0\n self._evictions: int = 0\n\n def _compute_weight(self, key: K, value: V) -> int:\n if self._weight_fn is not None:\n return self._weight_fn(key, value)\n return 1\n\n def _evict_if_needed(self, required_weight: int) -> None:\n \"\"\"Evict LRU entries until we have room for required_weight.\"\"\"\n while self._current_weight + required_weight > self._max_weight and self._entries:\n # Pop the least recently used (first item in OrderedDict)\n key, entry = self._entries.popitem(last=False)\n self._current_weight -= entry.weight\n self._evictions += 1\n\n async def get(self, key: K) -> V | None:\n \"\"\"\n Retrieve a value from the cache. Returns None if not present or expired.\n Does not trigger factory creation.\n \"\"\"\n async with self._lock:\n entry = self._entries.get(key)\n if entry is None:\n self._misses += 1\n return None\n\n # Check TTL\n if time.monotonic() >= entry.expires_at:\n # Expired: remove and count as miss\n del self._entries[key]\n self._current_weight -= entry.weight\n self._misses += 1\n return None\n\n # Hit: move to most recently used\n self._entries.move_to_end(key)\n self._hits += 1\n return entry.value\n\n async def get_or_create(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: float | None = None,\n ) -> V:\n \"\"\"\n Get a value from cache, or create it using the async factory if missing/expired.\n Single-flight: concurrent calls for the same key share one factory invocation.\n \"\"\"\n # First, try to get from cache\n value = await self.get(key)\n if value is not None:\n return value\n\n # Check if there's already an in-flight creation for this key\n async with self._lock:\n if key in self._inflight:\n future, _ = self._inflight[key]\n else:\n # Start a new single-flight creation\n future: asyncio.Future[V] = asyncio.get_event_loop().create_future()\n task = asyncio.ensure_future(self._run_factory(key, factory, ttl, future))\n self._inflight[key] = (future, task)\n\n # Wait for the result. If this task is cancelled, we must NOT cancel the shared factory.\n try:\n return await future\n except asyncio.CancelledError:\n # Our await was cancelled, but the factory task continues.\n # We should re-raise to propagate cancellation to the caller.\n raise\n\n async def _run_factory(\n self,\n key: K,\n factory: Callable[[], Awaitable[V]],\n ttl: float | None,\n future: asyncio.Future[V],\n ) -> None:\n \"\"\"\n Run the factory, publish result to future, and update cache.\n Never holds the cache lock while awaiting user code.\n \"\"\"\n try:\n value = await factory()\n except Exception as exc:\n # Exceptions are never cached. Set exception on future.\n if not future.done():\n future.set_exception(exc)\n # Clean up inflight entry\n async with self._lock:\n self._inflight.pop(key, None)\n return\n\n # Check if the key was invalidated while factory was running.\n # If invalidated, we must NOT publish the stale value.\n async with self._lock:\n # Check if key is still in inflight (i.e., not invalidated)\n if key not in self._inflight:\n # Key was invalidated during creation; do not publish\n if not future.done():\n future.set_exception(\n RuntimeError(f\"Key {key!r} was invalidated during creation\")\n )\n return\n\n # Remove from inflight\n del self._inflight[key]\n\n # Check if key already exists in cache (e.g., another path added it)\n if key in self._entries:\n # Already present; don't overwrite\n if not future.done():\n future.set_result(self._entries[key].value)\n return\n\n # Compute weight and evict if needed\n weight = self._compute_weight(key, value)\n self._evict_if_needed(weight)\n\n # Insert new entry\n effective_ttl = ttl if ttl is not None else self._default_ttl\n expires_at = time.monotonic() + effective_ttl\n self._entries[key] = _Entry(\n value=value,\n expires_at=expires_at,\n weight=weight,\n key=key,\n )\n self._current_weight += weight\n\n # Set result on future\n if not future.done():\n future.set_result(value)\n\n async def invalidate(self, key: K) -> bool:\n \"\"\"\n Invalidate a key. Returns True if the key was present (in cache or inflight).\n If inflight, prevents stale publication.\n \"\"\"\n async with self._lock:\n found = False\n\n # Remove from cache if present\n if key in self._entries:\n entry = self._entries.pop(key)\n self._current_weight -= entry.weight\n found = True\n\n # Remove from inflight if present\n if key in self._inflight:\n future, task = self._inflight.pop(key)\n # Cancel the task if it's still running\n if not task.done():\n task.cancel()\n # Set exception on future to unblock waiters\n if not future.done():\n future.set_exception(\n RuntimeError(f\"Key {key!r} was invalidated\")\n )\n found = True\n\n return found\n\n async def clear(self) -> None:\n \"\"\"Clear all entries and cancel all in-flight factories.\"\"\"\n async with self._lock:\n # Cancel all in-flight tasks\n for key, (future, task) in list(self._inflight.items()):\n if not task.done():\n task.cancel()\n if not future.done():\n future.set_exception(\n RuntimeError(\"Cache cleared\")\n )\n self._inflight.clear()\n\n # Clear all entries\n self._entries.clear()\n self._current_weight = 0\n\n @property\n def stats(self) -> CacheStats:\n \"\"\"Return immutable snapshot of cache statistics.\"\"\"\n return CacheStats(\n hits=self._hits,\n misses=self._misses,\n evictions=self._evictions,\n )\n\n @property\n def size(self) -> int:\n \"\"\"Return the number of entries currently in the cache.\"\"\"\n return len(self._entries)\n\n @property\n def current_weight(self) -> int:\n \"\"\"Return the current total weight of cached entries.\"\"\"\n return self._current_weight\n\n\n# ---------------------------------------------------------------------------\n# Pytest-asyncio tests\n# ---------------------------------------------------------------------------\n\nimport pytest\n\n\n@pytest.mark.asyncio\nasync def test_basic_get_and_set() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n async def factory() -> int:\n return 42\n\n result = await cache.get_or_create(\"key\", factory)\n assert result == 42\n\n # Second call should hit cache\n result2 = await cache.get_or_create(\"key\", factory)\n assert result2 == 42\n\n stats = cache.stats\n assert stats.hits == 1\n assert stats.misses == 1\n\n\n@pytest.mark.asyncio\nasync def test_ttl_expiry() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=0.05)\n\n call_count = 0\n\n async def factory() -> int:\n nonlocal call_count\n call_count += 1\n return call_count\n\n result1 = await cache.get_or_create(\"key\", factory)\n assert result1 == 1\n\n # Wait for TTL to expire\n await asyncio.sleep(0.1)\n\n result2 = await cache.get_or_create(\"key\", factory)\n assert result2 == 2\n assert call_count == 2\n\n\n@pytest.mark.asyncio\nasync def test_single_flight_concurrent_misses() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n call_count = 0\n\n async def factory() -> int:\n nonlocal call_count\n call_count += 1\n await asyncio.sleep(0.05)\n return call_count\n\n # Launch multiple concurrent get_or_create calls for the same key\n results = await asyncio.gather(\n cache.get_or_create(\"key\", factory),\n cache.get_or_create(\"key\", factory),\n cache.get_or_create(\"key\", factory),\n )\n\n # All should get the same value\n assert all(r == results[0] for r in results)\n # Factory should have been called exactly once\n assert call_count == 1\n\n\n@pytest.mark.asyncio\nasync def test_cancellation_does_not_cancel_shared_factory() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n factory_started = asyncio.Event()\n factory_done = asyncio.Event()\n result_value: list[int] = []\n\n async def factory() -> int:\n factory_started.set()\n await asyncio.sleep(0.1)\n val = 99\n result_value.append(val)\n factory_done.set()\n return val\n\n # Start a get_or_create\n task1 = asyncio.ensure_future(cache.get_or_create(\"key\", factory))\n\n # Wait for factory to start\n await factory_started.wait()\n\n # Cancel the first waiter\n task1.cancel()\n\n try:\n await task1\n except asyncio.CancelledError:\n pass\n\n # The factory should still complete\n await factory_done.wait()\n\n # Now a new get_or_create should get the cached value\n result = await cache.get_or_create(\"key\", factory)\n assert result == 99\n\n\n@pytest.mark.asyncio\nasync def test_factory_failure_not_cached() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n call_count = 0\n\n async def failing_factory() -> int:\n nonlocal call_count\n call_count += 1\n raise ValueError(\"factory failed\")\n\n with pytest.raises(ValueError, match=\"factory failed\"):\n await cache.get_or_create(\"key\", failing_factory)\n\n # Factory should not be cached; next call should invoke factory again\n with pytest.raises(ValueError, match=\"factory failed\"):\n await cache.get_or_create(\"key\", failing_factory)\n\n assert call_count == 2\n\n\n@pytest.mark.asyncio\nasync def test_invalidation_during_creation_prevents_stale_publication() -> None:\n cache = AsyncTTLCache[str, int](max_weight=10, default_ttl=10.0)\n\n factory_started = asyncio.Event()\n\n async def factory() -> int:\n factory_started.set()\n await asyncio.sleep(0.1)\n return 123\n\n # Start get_or_create\n task = asyncio.ensure_future(cache.get_or_create(\"key\", factory))\n\n # Wait for factory to start\n await factory_started.wait()\n\n # Invalidate while factory is running\n await cache.invalidate(\"key\")\n\n # The original task should raise an exception\n with pytest.raises(RuntimeError, match=\"invalidated\"):\n await task\n\n # A new get_or_create should invoke the factory again\n result = await cache.get_or_create(\"key\", factory)\n assert result == 123\n\n\n@pytest.mark.asyncio\nasync def test_weighted_lru_eviction()", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:27:04.745948+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6236594580113888, + "total_s": 242.9034315830213, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 12.73210751545455, + "finish_reason": "length", + "output_chars": 11719, + "output_sha256": "64e2a1896826c7e6a38540d8e7d2650e66be4c3df39a3a63ca49261d90eeea17", + "output": "first();\n if ($existing !== null) {\n return $existing;\n }\n\n $lastException = null;\n\n for ($attempt = 0; $attempt < self::MAX_DEADLOCK_RETRIES; $attempt++) {\n try {\n return DB::transaction(function () use ($fromAccountId, $toAccountId, $amountCents, $idempotencyKey) {\n // Lock both account rows in deterministic ID order to prevent deadlocks\n $minId = min($fromAccountId, $toAccountId);\n $maxId = max($fromAccountId, $toAccountId);\n\n $accounts = Account::whereIn('id', [$minId, $maxId])\n ->orderBy('id')\n ->lockForUpdate()\n ->get();\n\n if ($accounts->count() !== 2) {\n throw new RuntimeException('One or both accounts do not exist.');\n }\n\n $fromAccount = $accounts->firstWhere('id', $fromAccountId);\n $toAccount = $accounts->firstWhere('id', $toAccountId);\n\n // Check sufficient funds\n if ($fromAccount->balance_cents < $amountCents) {\n throw new RuntimeException('Insufficient funds for transfer.');\n }\n\n // Perform the transfer\n $fromAccount->balance_cents -= $amountCents;\n $fromAccount->save();\n\n $toAccount->balance_cents += $amountCents;\n $toAccount->save();\n\n // Create the transfer record\n $transfer = Transfer::create([\n 'idempotency_key' => $idempotencyKey,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n 'created_at' => now(),\n ]);\n\n // Persist outbox event in the same transaction\n OutboxEvent::create([\n 'transfer_id' => $transfer->id,\n 'event_type' => 'transfer.completed',\n 'payload' => json_encode([\n 'transfer_id' => $transfer->id,\n 'from_account_id' => $fromAccountId,\n 'to_account_id' => $toAccountId,\n 'amount_cents' => $amountCents,\n ]),\n 'created_at' => now(),\n ]);\n\n return $transfer;\n });\n } catch (\\Illuminate\\Database\\QueryException $e) {\n // Detect deadlock (MySQL error 1213) and retry\n if ($this->isDeadlock($e)) {\n $lastException = $e;\n continue;\n }\n throw $e;\n }\n }\n\n throw new RuntimeException('Transfer failed after maximum deadlock retries.', 0, $lastException);\n }\n\n /**\n * Determine if a QueryException represents a deadlock.\n */\n private function isDeadlock(\\Illuminate\\Database\\QueryException $e): bool\n {\n // MySQL deadlock error code is 1213\n return $e->getCode() === 1213 || str_contains($e->getMessage(), '1213');\n }\n}\n\nnamespace Tests\\Unit;\n\nuse App\\Models\\Account;\nuse App\\Models\\Transfer;\nuse App\\Models\\OutboxEvent;\nuse App\\Services\\MoneyTransferService;\nuse Illuminate\\Support\\Facades\\DB;\nuse PHPUnit\\Framework\\TestCase;\n\nclass MoneyTransferServiceTest extends TestCase\n{\n private MoneyTransferService $service;\n\n protected function setUp(): void\n {\n parent::setUp();\n\n // Reset tables\n DB::table('outbox_events')->delete();\n DB::table('transfers')->delete();\n DB::table('accounts')->delete();\n\n $this->service = new MoneyTransferService();\n }\n\n /**\n * Test that a valid transfer moves funds correctly.\n */\n public function testValidTransferMovesFunds(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $transfer = $this->service->transfer(1, 2, 2500, 'key-001');\n\n $this->assertNotNull($transfer->id);\n $this->assertEquals(7500, Account::find(1)->balance_cents);\n $this->assertEquals(7500, Account::find(2)->balance_cents);\n $this->assertEquals(1, Transfer::count());\n $this->assertEquals(1, OutboxEvent::count());\n }\n\n /**\n * Test that duplicate requests with the same idempotency key return the original transfer\n * without duplicating effects.\n */\n public function testDuplicateRequestReturnsOriginalTransfer(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $first = $this->service->transfer(1, 2, 2500, 'key-dup-001');\n $second = $this->service->transfer(1, 2, 2500, 'key-dup-001');\n\n // Same transfer returned\n $this->assertEquals($first->id, $second->id);\n\n // Balances reflect only one transfer\n $this->assertEquals(7500, Account::find(1)->balance_cents);\n $this->assertEquals(7500, Account::find(2)->balance_cents);\n\n // Only one transfer and one outbox event\n $this->assertEquals(1, Transfer::count());\n $this->assertEquals(1, OutboxEvent::count());\n }\n\n /**\n * Test that competing requests (simulated by sequential calls with same key)\n * preserve balances and create exactly one transfer and one outbox event.\n */\n public function testCompetingRequestsPreserveBalances(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n // Simulate two competing requests with the same idempotency key\n $result1 = $this->service->transfer(1, 2, 3000, 'key-compete-001');\n $result2 = $this->service->transfer(1, 2, 3000, 'key-compete-001');\n\n $this->assertEquals($result1->id, $result2->id);\n $this->assertEquals(7000, Account::find(1)->balance_cents);\n $this->assertEquals(8000, Account::find(2)->balance_cents);\n $this->assertEquals(1, Transfer::count());\n $this->assertEquals(1, OutboxEvent::count());\n }\n\n /**\n * Test that insufficient funds are rejected.\n */\n public function testInsufficientFundsAreRejected(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 1000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $this->expectException(\\RuntimeException::class);\n $this->expectExceptionMessage('Insufficient funds');\n\n $this->service->transfer(1, 2, 5000, 'key-insufficient-001');\n }\n\n /**\n * Test that zero or negative amounts are rejected.\n */\n public function testNonPositiveAmountIsRejected(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $this->expectException(\\RuntimeException::class);\n $this->expectExceptionMessage('positive');\n\n $this->service->transfer(1, 2, 0, 'key-zero-001');\n }\n\n /**\n * Test that same source and destination accounts are rejected.\n */\n public function testSameAccountIsRejected(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n\n $this->expectException(\\RuntimeException::class);\n $this->expectExceptionMessage('distinct');\n\n $this->service->transfer(1, 1, 1000, 'key-same-001');\n }\n\n /**\n * Test that different idempotency keys create separate transfers.\n */\n public function testDifferentKeysCreateSeparateTransfers(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $this->service->transfer(1, 2, 1000, 'key-a-001');\n $this->service->transfer(1, 2, 2000, 'key-b-001');\n\n $this->assertEquals(7000, Account::find(1)->balance_cents);\n $this->assertEquals(8000, Account::find(2)->balance_cents);\n $this->assertEquals(2, Transfer::count());\n $this->assertEquals(2, OutboxEvent::count());\n }\n\n /**\n * Test that the outbox event contains correct payload.\n */\n public function testOutboxEventHasCorrectPayload(): void\n {\n $from = Account::create(['id' => 1, 'balance_cents' => 10000]);\n $to = Account::create(['id' => 2, 'balance_cents' => 5000]);\n\n $transfer = $this->service->transfer(1, 2, 2500, 'key-payload-001');\n\n $event = OutboxEvent::first();\n $this->assertEquals($transfer->id, $event->transfer_id);\n $this->assertEquals", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:31:07.651240+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4923370420001447, + "total_s": 224.71278991695726, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 13.76217976638934, + "finish_reason": "length", + "output_chars": 10877, + "output_sha256": "bb305dfdb203bcf6d40071cb30ae92561f0b39bf38f52973d3f27e45cd5ff0a7", + "output": "type TaskResult = unknown;\n\ninterface Task {\n id: string;\n deps: readonly string[];\n run: (signal: AbortSignal) => Promise;\n}\n\ninterface SchedulerOptions {\n maxConcurrency?: number;\n}\n\ntype TaskStatus = 'pending' | 'running' | 'succeeded' | 'failed' | 'skipped';\n\ninterface TaskState {\n task: Task;\n status: TaskStatus;\n result?: T;\n error?: unknown;\n}\n\nclass CycleError extends Error {\n readonly path: readonly string[];\n constructor(path: readonly string[]) {\n super(`Dependency cycle detected: ${path.join(' -> ')}`);\n this.name = 'CycleError';\n this.path = path;\n }\n}\n\nclass DuplicateIdError extends Error {\n constructor(id: string) {\n super(`Duplicate task ID: \"${id}\"`);\n this.name = 'DuplicateIdError';\n }\n}\n\nclass UnknownDependencyError extends Error {\n constructor(taskId: string, depId: string) {\n super(`Task \"${taskId}\" depends on unknown task \"${depId}\"`);\n this.name = 'UnknownDependencyError';\n }\n}\n\nclass TaskExecutionError extends Error {\n readonly taskId: string;\n readonly cause: unknown;\n constructor(taskId: string, cause: unknown) {\n super(`Task \"${taskId}\" failed: ${cause instanceof Error ? cause.message : String(cause)}`);\n this.name = 'TaskExecutionError';\n this.taskId = taskId;\n this.cause = cause;\n }\n}\n\nfunction detectCycle(\n tasks: readonly Task[],\n idToIndex: Map\n): string[] | null {\n const n = tasks.length;\n const color = new Array(n).fill(0);\n const stack: number[] = [];\n\n const dfs = (u: number): boolean => {\n color[u] = 1;\n stack.push(u);\n const deps = tasks[u].deps;\n for (const depId of deps) {\n const v = idToIndex.get(depId);\n if (v === undefined) continue;\n if (color[v] === 1) {\n const start = stack.indexOf(v);\n return true;\n }\n if (color[v] === 0 && dfs(v)) {\n return true;\n }\n }\n stack.pop();\n color[u] = 2;\n return false;\n };\n\n for (let i = 0; i < n; i++) {\n if (color[i] === 0) {\n if (dfs(i)) {\n const start = stack.indexOf(i);\n const path = stack.slice(start).map((idx) => tasks[idx].id);\n path.push(tasks[i].id);\n return path;\n }\n }\n }\n return null;\n}\n\nfunction buildTopologicalOrder(\n tasks: readonly Task[],\n idToIndex: Map\n): number[] {\n const n = tasks.length;\n const inDegree = new Array(n).fill(0);\n const dependents = new Array>(n);\n for (let i = 0; i < n; i++) {\n dependents[i] = new Set();\n }\n\n for (let i = 0; i < n; i++) {\n const deps = tasks[i].deps;\n for (const depId of deps) {\n const depIdx = idToIndex.get(depId);\n if (depIdx === undefined) continue;\n inDegree[i]++;\n dependents[depIdx].add(i);\n }\n }\n\n const queue: number[] = [];\n for (let i = 0; i < n; i++) {\n if (inDegree[i] === 0) {\n queue.push(i);\n }\n }\n\n const order: number[] = [];\n while (queue.length > 0) {\n const u = queue.shift()!;\n order.push(u);\n const deps = tasks[u].deps;\n for (const depId of deps) {\n const depIdx = idToIndex.get(depId);\n if (depIdx === undefined) continue;\n inDegree[depIdx]--;\n if (inDegree[depIdx] === 0) {\n queue.push(depIdx);\n }\n }\n }\n\n return order;\n}\n\nexport async function runDag(\n tasks: readonly Task[],\n options: SchedulerOptions = {},\n externalSignal?: AbortSignal\n): Promise> {\n const maxConcurrency = options.maxConcurrency ?? Infinity;\n\n if (tasks.length === 0) {\n return new Map();\n }\n\n const idToIndex = new Map();\n for (let i = 0; i < tasks.length; i++) {\n const id = tasks[i].id;\n if (idToIndex.has(id)) {\n throw new DuplicateIdError(id);\n }\n idToIndex.set(id, i);\n }\n\n for (let i = 0; i < tasks.length; i++) {\n const task = tasks[i];\n for (const depId of task.deps) {\n if (!idToIndex.has(depId)) {\n throw new UnknownDependencyError(task.id, depId);\n }\n }\n }\n\n const cyclePath = detectCycle(tasks, idToIndex);\n if (cyclePath !== null) {\n throw new CycleError(cyclePath);\n }\n\n const topoOrder = buildTopologicalOrder(tasks, idToIndex);\n\n const states: TaskState[] = tasks.map((task) => ({\n task,\n status: 'pending' as TaskStatus,\n }));\n\n const results = new Map();\n const failedTaskIds = new Set();\n const skippedTaskIds = new Set();\n let firstError: unknown = null;\n let firstErrorTaskId: string | null = null;\n\n const abortController = new AbortController();\n const signal = abortController.signal;\n\n if (externalSignal) {\n if (externalSignal.aborted) {\n abortController.abort(externalSignal.reason);\n } else {\n externalSignal.addEventListener('abort', () => {\n abortController.abort(externalSignal.reason);\n }, { once: true });\n }\n }\n\n const runningPromises = new Set>();\n let activeCount = 0;\n let completedCount = 0;\n const totalTasks = tasks.length;\n\n const onAbort = () => {\n for (const state of states) {\n if (state.status === 'pending') {\n state.status = 'skipped';\n skippedTaskIds.add(state.task.id);\n }\n }\n };\n\n if (signal.aborted) {\n onAbort();\n } else {\n signal.addEventListener('abort', onAbort, { once: true });\n }\n\n const tryStartTask = (idx: number): void => {\n const state = states[idx];\n if (state.status !== 'pending') return;\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(state.task.id);\n return;\n }\n if (activeCount >= maxConcurrency) return;\n\n const task = state.task;\n state.status = 'running';\n activeCount++;\n\n const promise = (async () => {\n try {\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n return;\n }\n\n const result = await task.run(signal);\n\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n return;\n }\n\n state.status = 'succeeded';\n state.result = result;\n results.set(task.id, result);\n } catch (err) {\n if (signal.aborted) {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n return;\n }\n\n state.status = 'failed';\n state.error = err;\n failedTaskIds.add(task.id);\n\n if (firstError === null) {\n firstError = err;\n firstErrorTaskId = task.id;\n }\n } finally {\n activeCount--;\n completedCount++;\n runningPromises.delete(promise);\n scheduleNext();\n }\n })();\n\n runningPromises.add(promise);\n };\n\n const scheduleNext = (): void => {\n if (signal.aborted) return;\n\n for (const idx of topoOrder) {\n const state = states[idx];\n if (state.status !== 'pending') continue;\n\n const task = state.task;\n const deps = task.deps;\n\n let canStart = true;\n for (const depId of deps) {\n const depIdx = idToIndex.get(depId);\n if (depIdx === undefined) continue;\n const depState = states[depIdx];\n if (depState.status === 'failed') {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n canStart = false;\n break;\n }\n if (depState.status === 'skipped') {\n state.status = 'skipped';\n skippedTaskIds.add(task.id);\n canStart = false;\n break;\n }\n if (depState.status !== 'succeeded') {\n canStart = false;\n break;\n }\n }\n\n if (!canStart) continue;\n\n if (activeCount >= maxConcurrency) break;\n\n tryStartTask(idx);\n }\n };\n\n scheduleNext();\n\n const allDone = new Promise((resolve) => {\n const check = () => {\n if (completedCount >= totalTasks) {\n resolve();\n }\n };\n\n const interval = setInterval(check, 1);\n const onAbortCheck = () => {\n clearInterval(interval);\n check();\n };\n\n if (signal.aborted) {\n clearInterval(interval);\n onAbortCheck();\n } else {\n signal.addEventListener('abort', onAbortCheck, { once: true });\n }\n\n const originalCheck = check;\n const wrappedCheck = () => {\n originalCheck();\n if (completedCount >= totalTasks) {\n clearInterval(interval);\n }\n };\n\n const originalScheduleNext = scheduleNext;\n // We need to hook into completion. Let's use a different approach.\n });\n\n // The above approach is flawed. Let me restructure.\n\n // Actually, let me redo the scheduling with a cleaner approach.\n\n // Reset and use a proper completion tracking.\n // I'll restructure the whole function.\n\n // Let me just return a proper implementation.\n\n // The issue is I'm trying to track completion. Let me use a counter that increments\n // in the finally block of each task, and resolve when it reaches totalTasks.\n\n // But I already have completedCount. The problem is the allDone promise setup.\n // Let me fix it.\n\n // Actually, the code above has a bug in the allDone setup. Let me rewrite the whole thing cleanly.\n\n // I'll just return the correct implementation.\n\n // Let me restart the function body properly.\n\n // OK I realize I'm going in circles. Let me just write the correct code.\n\n // The key insight: I need a promise that resolves when all tasks are done.\n // I can track this with a counter and a resolve function.\n\n // Let me rewrite from scratch in the actual output.\n\n // For now, let me just make sure the logic is correct and output the final version.\n\n // I'll output the corrected version below.\n\n // Actually, I should just write the correct code. Let me do that.\n\n // The allDone promise should be created with a resolve that's called when completedCount reaches totalTasks.\n // I can do this by having a variable that holds the resolve function.\n\n // Let me restructure:\n\n // const allDoneResolve: (() => void) | null = ...\n // No, I'll use a different pattern.\n\n // I'll create the promise and store the resolve in a closure.\n\n // Here's the correct approach:\n\n // let resolveAllDone: () => void;\n // const allDone = new Promise((r) => { resolveAllDone = r; });\n // Then in the finally block: if (completedCount >= totalTasks) resolveAllDone();\n\n // That's clean. Let me use that.\n\n // OK, I'll write the final correct version now.\n\n // I realize I've been overthinking this. Let me just output the correct code.\n\n // The final answer will be a clean, correct implementation.\n\n // Let me write it out properly.\n\n // I'll use the pattern:\n // let resolveAllDone: () => void;\n // const allDone = new Promise((r) => { resolveAllDone = r; });\n // And call resolveAllDone() when completedCount === totalTasks.\n\n // That's the cleanest approach.\n\n // Now let me write the", + "error": null + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json new file mode 100644 index 0000000..b74496a --- /dev/null +++ b/benchmarks/inference/coding-results/20260831T133452Z-vllm-metal-qwen3.8-27b-mxfp8/summary.json @@ -0,0 +1,53 @@ +[ + { + "task": "python_async_cache", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:22:57.038624+00:00", + "prompt_chars": 857, + "max_tokens": 3072, + "ttft_s": 1.1419034169521183, + "total_s": 247.70563520898577, + "prompt_tokens": 211, + "output_tokens": 3072, + "decode_tps": 12.459253344652916, + "finish_reason": "length", + "output_chars": 13028, + "output_sha256": "699045091c4f5bc19d0f8f0bbc676b164512c8e3e7dec0f95acb13b5bef2f4ba", + "error": null + }, + { + "task": "php_idempotent_transfer", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:27:04.745948+00:00", + "prompt_chars": 756, + "max_tokens": 3072, + "ttft_s": 1.6236594580113888, + "total_s": 242.9034315830213, + "prompt_tokens": 182, + "output_tokens": 3072, + "decode_tps": 12.73210751545455, + "finish_reason": "length", + "output_chars": 11719, + "output_sha256": "64e2a1896826c7e6a38540d8e7d2650e66be4c3df39a3a63ca49261d90eeea17", + "error": null + }, + { + "task": "typescript_dag_scheduler", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "started_at": "2026-08-31T13:31:07.651240+00:00", + "prompt_chars": 773, + "max_tokens": 3072, + "ttft_s": 1.4923370420001447, + "total_s": 224.71278991695726, + "prompt_tokens": 183, + "output_tokens": 3072, + "decode_tps": 13.76217976638934, + "finish_reason": "length", + "output_chars": 10877, + "output_sha256": "bb305dfdb203bcf6d40071cb30ae92561f0b39bf38f52973d3f27e45cd5ff0a7", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/coding_benchmark.py b/benchmarks/inference/coding_benchmark.py new file mode 100644 index 0000000..8b6e215 --- /dev/null +++ b/benchmarks/inference/coding_benchmark.py @@ -0,0 +1,364 @@ +#!/usr/bin/env python3 +"""Measure end-to-end latency for deterministic, complex programming tasks.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import platform +import subprocess +import sys +import time +import urllib.error +import urllib.request +from dataclasses import asdict, dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +TASKS = { + "python_async_cache": """ +Act as a senior Python engineer. Implement a production-ready, single-file Python 3.12 module +containing a generic AsyncTTLCache[K, V]. Requirements: weighted LRU capacity; per-entry TTL based +on time.monotonic; single-flight get_or_create so concurrent misses for one key invoke the async +factory exactly once; cancellation of one waiter must not cancel the shared factory; exceptions are +never cached; invalidating a key while its factory is running must prevent stale publication; never +hold the cache lock while awaiting user code; clear(), invalidate(), and immutable hit/miss/eviction +statistics. Include concise pytest-asyncio tests covering races, cancellation, expiry, weighted +capacity, factory failure, and invalidation during creation. Use only the standard library plus +pytest. Output only the complete module, without Markdown fences or prose. +""".strip(), + "php_idempotent_transfer": """ +Act as a senior Laravel 12 engineer. Produce one self-contained PHP file that demonstrates an +idempotent money-transfer service and its PHPUnit tests. Store money as integer cents. Requirements: +validate positive amount and distinct accounts; use a database transaction; lock both account rows +in deterministic ID order; reject insufficient funds; use a unique client idempotency key so retries +return the original transfer; retry deadlocks without duplicating effects; persist an outbox event +in the same transaction; and prove with tests that duplicate and competing requests preserve +balances, create one transfer, and create one outbox event. Include minimal model/table assumptions +as comments. +Output only valid PHP, without Markdown fences or prose. +""".strip(), + "typescript_dag_scheduler": """ +Act as a senior TypeScript engineer. Implement a strict-mode, dependency-free DAG task scheduler in +one TypeScript file. Each task has an ID, dependency IDs, and an async run(signal) function. +Requirements: validate unknown dependencies and duplicate IDs; report an explicit cycle path; stable +input-order scheduling; configurable maximum concurrency; AbortSignal support; fail fast after the +first error while awaiting already-running tasks; never start dependents of failed tasks; return a +Map of typed results on success; and avoid unhandled promise rejections. Include a compact test +harness covering a diamond graph, concurrency enforcement, stable order, cycle detection, external +cancellation, and rejection. Output only TypeScript, without Markdown fences or prose. +""".strip(), +} + + +@dataclass(frozen=True) +class Runtime: + name: str + kind: str + base_url: str + model: str + + +@dataclass +class Result: + task: str + runtime: str + model: str + started_at: str + prompt_chars: int + max_tokens: int + ttft_s: float | None = None + total_s: float | None = None + prompt_tokens: int | None = None + output_tokens: int | None = None + decode_tps: float | None = None + finish_reason: str | None = None + output_chars: int = 0 + output_sha256: str | None = None + output: str = "" + error: str | None = None + + +def utc_now() -> str: + return datetime.now(UTC).isoformat() + + +def post_json(url: str, payload: dict[str, Any], timeout: float) -> dict[str, Any]: + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=timeout) as response: + return json.load(response) + + +def health(runtime: Runtime) -> None: + path = "/api/tags" if runtime.kind == "ollama" else "/v1/models" + with urllib.request.urlopen(f"{runtime.base_url.rstrip('/')}{path}", timeout=15): + pass + + +def consume_ollama(response: Any, started: float) -> tuple[float | None, str, dict[str, Any]]: + first_at: float | None = None + chunks: list[str] = [] + terminal: dict[str, Any] = {} + for line in response: + if not line.strip(): + continue + event = json.loads(line) + text = event.get("message", {}).get("content") or event.get("response") or "" + if text: + if first_at is None: + first_at = time.perf_counter() + chunks.append(text) + if event.get("done"): + terminal = event + return (first_at - started if first_at else None, "".join(chunks), terminal) + + +def consume_sse( + response: Any, started: float +) -> tuple[float | None, str, dict[str, Any], str | None]: + first_at: float | None = None + chunks: list[str] = [] + terminal: dict[str, Any] = {} + finish_reason: str | None = None + for raw_line in response: + line = raw_line.decode("utf-8").strip() + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if payload == "[DONE]": + break + event = json.loads(payload) + terminal = event + for choice in event.get("choices") or []: + text = choice.get("text") or choice.get("delta", {}).get("content") or "" + if text: + if first_at is None: + first_at = time.perf_counter() + chunks.append(text) + finish_reason = choice.get("finish_reason") or finish_reason + return (first_at - started if first_at else None, "".join(chunks), terminal, finish_reason) + + +def request_payload( + runtime: Runtime, prompt: str, max_tokens: int, seed: int +) -> tuple[str, dict[str, Any]]: + if runtime.kind == "ollama": + return f"{runtime.base_url.rstrip('/')}/api/chat", { + "model": runtime.model, + "messages": [ + { + "role": "system", + "content": ( + "You are a senior software engineer. Follow every output constraint " + "exactly and return a complete solution." + ), + }, + {"role": "user", "content": prompt}, + ], + "think": False, + "stream": True, + "keep_alive": "30m", + "options": { + "temperature": 0, + "seed": seed, + "num_predict": max_tokens, + "num_ctx": 8192, + "draft_num_predict": 0, + }, + } + return f"{runtime.base_url.rstrip('/')}/v1/chat/completions", { + "model": runtime.model, + "messages": [ + { + "role": "system", + "content": ( + "You are a senior software engineer. Follow every output constraint exactly " + "and return a complete solution." + ), + }, + {"role": "user", "content": prompt}, + ], + "stream": True, + "stream_options": {"include_usage": True}, + "chat_template_kwargs": {"enable_thinking": False}, + "temperature": 0, + "seed": seed, + "max_tokens": max_tokens, + } + + +def run_task( + runtime: Runtime, + task_name: str, + prompt: str, + max_tokens: int, + seed: int, + timeout: float, +) -> Result: + result = Result( + task=task_name, + runtime=runtime.name, + model=runtime.model, + started_at=utc_now(), + prompt_chars=len(prompt), + max_tokens=max_tokens, + ) + url, payload = request_payload(runtime, prompt, max_tokens, seed) + request = urllib.request.Request( + url, + data=json.dumps(payload).encode("utf-8"), + headers={"Content-Type": "application/json", "Accept": "text/event-stream"}, + method="POST", + ) + started = time.perf_counter() + try: + with urllib.request.urlopen(request, timeout=timeout) as response: + if runtime.kind == "ollama": + result.ttft_s, result.output, terminal = consume_ollama(response, started) + result.prompt_tokens = terminal.get("prompt_eval_count") + result.output_tokens = terminal.get("eval_count") + result.finish_reason = terminal.get("done_reason") + decode_ns = terminal.get("eval_duration") + if decode_ns and result.output_tokens: + result.decode_tps = result.output_tokens / (decode_ns / 1e9) + else: + result.ttft_s, result.output, terminal, result.finish_reason = consume_sse( + response, started + ) + usage = terminal.get("usage") or {} + result.prompt_tokens = usage.get("prompt_tokens") + result.output_tokens = usage.get("completion_tokens") + result.total_s = time.perf_counter() - started + if result.decode_tps is None and result.output_tokens and result.ttft_s is not None: + decode_s = result.total_s - result.ttft_s + if decode_s > 0: + result.decode_tps = result.output_tokens / decode_s + result.output_chars = len(result.output) + result.output_sha256 = hashlib.sha256(result.output.encode()).hexdigest() + if not result.output: + result.error = "empty generated output" + except urllib.error.HTTPError as error: + detail = error.read().decode("utf-8", errors="replace")[:2000] + result.error = f"HTTP {error.code}: {detail}" + result.total_s = time.perf_counter() - started + except (OSError, ValueError, json.JSONDecodeError) as error: + result.error = f"{type(error).__name__}: {error}" + result.total_s = time.perf_counter() - started + return result + + +def package_versions() -> dict[str, str | None]: + script = ( + "import importlib.metadata as m,json; " + "print(json.dumps({p:m.version(p) for p in " + "['vllm','vllm-metal','mlx','mlx-lm','mlx-vlm']}))" + ) + executable = Path.home() / ".venv-vllm-metal/bin/python" + try: + process = subprocess.run( + [str(executable), "-c", script], capture_output=True, text=True, timeout=30, check=True + ) + return json.loads(process.stdout) + except (OSError, subprocess.SubprocessError, json.JSONDecodeError): + return {} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--runtime", choices=("ollama", "vllm"), required=True) + parser.add_argument("--ollama-url", default="http://127.0.0.1:11434") + parser.add_argument("--ollama-model", default="qwen3.8:27b-mlx") + parser.add_argument("--vllm-url", default="http://127.0.0.1:8001") + parser.add_argument("--vllm-model", default="qwen3.8-27b-nvfp4") + parser.add_argument("--max-tokens", type=int, default=1536) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--timeout", type=float, default=900) + parser.add_argument("--output-dir", type=Path, default=Path(__file__).parent / "coding-results") + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.max_tokens < 1: + raise SystemExit("max-tokens must be positive") + runtime = ( + Runtime("ollama", "ollama", args.ollama_url, args.ollama_model) + if args.runtime == "ollama" + else Runtime("vllm-metal", "openai", args.vllm_url, args.vllm_model) + ) + try: + health(runtime) + except Exception as error: + raise SystemExit(f"{runtime.name} is unavailable: {error}") from error + + warmup = run_task(runtime, "warmup", "Output only the word READY.", 8, args.seed, args.timeout) + if warmup.error: + raise SystemExit(f"warmup failed: {warmup.error}") + print( + f"warmup runtime={runtime.name} total={warmup.total_s:.3f}s " + f"tokens={warmup.output_tokens}", + flush=True, + ) + + results: list[Result] = [] + for index, (task_name, prompt) in enumerate(TASKS.items(), start=1): + result = run_task(runtime, task_name, prompt, args.max_tokens, args.seed, args.timeout) + results.append(result) + print( + f"[{index}/{len(TASKS)}] task={task_name} total={result.total_s:.3f}s " + f"ttft={result.ttft_s:.3f}s tokens={result.output_tokens} " + f"decode={result.decode_tps:.2f}tok/s finish={result.finish_reason} " + f"error={result.error or 'none'}", + flush=True, + ) + + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + output_dir = args.output_dir / f"{run_id}-{runtime.name}-{runtime.model.replace(':', '_')}" + output_dir.mkdir(parents=True, exist_ok=False) + raw = { + "run_id": run_id, + "created_at": utc_now(), + "runtime": asdict(runtime), + "host": { + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "package_versions": package_versions(), + "settings": { + "temperature": 0, + "seed": args.seed, + "max_tokens": args.max_tokens, + "context_tokens": 8192, + "raw_completion": False, + "chat_template": True, + "thinking": False, + "ollama_speculative_drafting": False, + }, + "warmup": asdict(warmup), + "tasks": TASKS, + "results": [asdict(result) for result in results], + } + (output_dir / "results.json").write_text(json.dumps(raw, indent=2), encoding="utf-8") + summary = [ + { + key: value + for key, value in asdict(result).items() + if key not in {"output"} + } + for result in results + ] + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(f"Results: {output_dir}") + return 1 if any(result.error for result in results) else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json new file mode 100644 index 0000000..59a8aad --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/metadata.json @@ -0,0 +1,48 @@ +{ + "run_id": "20260831T133636Z", + "created_at": "2026-08-31T13:37:14.110488+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "model": "mlx-community/Qwen3.8-27B-nvfp4", + "label": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "max_tokens": 128, + "seed": 42, + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results" + }, + "load_s": 3.066661207994912, + "mlx_version": "0.32.0", + "device": "Device(gpu, 0)", + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "summary": { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.4445558497915044, + "ttft_s_p50": 0.48620820802170783, + "ttft_s_p95": 0.518837732996326, + "total_s_mean": 4.9355954667902555, + "total_s_p50": 4.966667582979426, + "total_s_p95": 5.034370008390397, + "prompt_tps_mean": 107.94239735907567, + "prompt_tps_p50": 107.53363787342614, + "prompt_tps_p95": 135.27062378239268, + "decode_tps_mean": 28.570950408778582, + "decode_tps_p50": 28.53361993265224, + "decode_tps_p95": 28.826484879691197, + "request_tps_mean": 25.943347049499913, + "request_tps_p50": 25.771807325831702, + "request_tps_p95": 26.64316963019791, + "peak_memory_gb": 15.479377792 + } +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json new file mode 100644 index 0000000..2464f98 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/raw.json @@ -0,0 +1,128 @@ +[ + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.6381363340187818, + "total_s": 5.067599917005282, + "prompt_tps": 46.64172306513033, + "decode_tps": 28.964400213729846, + "request_tps": 25.258505425906254, + "peak_memory_gb": 15.399916034, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.46635333297308534, + "total_s": 4.885915416001808, + "prompt_tps": 122.76309757245242, + "decode_tps": 29.032294926274737, + "request_tps": 26.19775192603388, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.3432638749945909, + "total_s": 4.786967416992411, + "prompt_tps": 90.81617646556052, + "decode_tps": 28.873430911706816, + "request_tps": 26.73926702438696, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.48620820802170783, + "total_s": 4.966667582979426, + "prompt_tps": 118.1052752991172, + "decode_tps": 28.63870075162873, + "request_tps": 25.771807325831702, + "peak_memory_gb": 15.453784176, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 3, + "prompt_name": "code", + "prompt_tokens": 54, + "output_tokens": 128, + "ttft_s": 0.5125948329805396, + "total_s": 5.009093041997403, + "prompt_tps": 139.56196090321154, + "decode_tps": 28.53361993265224, + "request_tps": 25.553528139089888, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 4, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.36031387496041134, + "total_s": 4.874560041993391, + "prompt_tps": 83.69493625406292, + "decode_tps": 28.424103574919425, + "request_tps": 26.258780053441704, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 5, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5203984580002725, + "total_s": 5.040689249988645, + "prompt_tps": 107.53363787342614, + "decode_tps": 28.384896872985717, + "request_tps": 25.393352704749322, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json new file mode 100644 index 0000000..ba73797 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133636Z-qwen3.8-27b-nvfp4/summary.json @@ -0,0 +1,22 @@ +{ + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.4445558497915044, + "ttft_s_p50": 0.48620820802170783, + "ttft_s_p95": 0.518837732996326, + "total_s_mean": 4.9355954667902555, + "total_s_p50": 4.966667582979426, + "total_s_p95": 5.034370008390397, + "prompt_tps_mean": 107.94239735907567, + "prompt_tps_p50": 107.53363787342614, + "prompt_tps_p95": 135.27062378239268, + "decode_tps_mean": 28.570950408778582, + "decode_tps_p50": 28.53361993265224, + "decode_tps_p95": 28.826484879691197, + "request_tps_mean": 25.943347049499913, + "request_tps_p50": 25.771807325831702, + "request_tps_p95": 26.64316963019791, + "peak_memory_gb": 15.479377792 +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json new file mode 100644 index 0000000..27bdef5 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/metadata.json @@ -0,0 +1,48 @@ +{ + "run_id": "20260831T133721Z", + "created_at": "2026-08-31T13:38:28.661722+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "model": "mlx-community/Qwen3.8-27B-mxfp8", + "label": "qwen3.8-27b-mxfp8", + "warmups": 2, + "rounds": 5, + "max_tokens": 128, + "seed": 42, + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results" + }, + "load_s": 5.387068083975464, + "mlx_version": "0.32.0", + "device": "Device(gpu, 0)", + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "summary": { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.6405866249930113, + "ttft_s_p50": 0.6006166669540107, + "ttft_s_p95": 0.993492508213967, + "total_s_mean": 8.97830267478712, + "total_s_p50": 8.952084249991458, + "total_s_p95": 9.543902474397328, + "prompt_tps_mean": 77.25518114106583, + "prompt_tps_p50": 69.803885982221, + "prompt_tps_p95": 108.54351756783554, + "decode_tps_mean": 15.141694584402668, + "decode_tps_p50": 15.417877788139895, + "decode_tps_p95": 15.90376325022775, + "request_tps_mean": 14.036390549836813, + "request_tps_p50": 14.298346220336581, + "request_tps_p95": 14.998850485179887, + "peak_memory_gb": 28.05746427 + } +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json new file mode 100644 index 0000000..252b13a --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/raw.json @@ -0,0 +1,128 @@ +[ + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "warmup", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 122, + "ttft_s": 1.4950010409811512, + "total_s": 8.873082707985304, + "prompt_tps": 65.2268651572191, + "decode_tps": 16.569121716474896, + "request_tps": 13.749449206667087, + "peak_memory_gb": 27.989241503, + "finish_reason": "stop", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "warmup", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5392710830201395, + "total_s": 8.420522083004471, + "prompt_tps": 104.9724920224018, + "decode_tps": 16.272217242824215, + "request_tps": 15.200957700514595, + "peak_memory_gb": 28.031514318, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 122, + "ttft_s": 0.41662891599116847, + "total_s": 8.074155374953989, + "prompt_tps": 69.803885982221, + "decode_tps": 15.96651063653789, + "request_tps": 15.109939595470719, + "peak_memory_gb": 28.031514318, + "finish_reason": "stop", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.6006166669540107, + "total_s": 8.794534499989823, + "prompt_tps": 94.30109779838891, + "decode_tps": 15.652773704987183, + "request_tps": 14.554494044016558, + "peak_memory_gb": 28.031531726, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 3, + "prompt_name": "code", + "prompt_tokens": 54, + "output_tokens": 128, + "ttft_s": 0.6342117090243846, + "total_s": 8.952084249991458, + "prompt_tps": 112.1041225101972, + "decode_tps": 15.417877788139895, + "request_tps": 14.298346220336581, + "peak_memory_gb": 28.05746427, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 4, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 122, + "ttft_s": 0.46816312498413026, + "total_s": 9.549591040995438, + "prompt_tps": 61.572468489546665, + "decode_tps": 13.468195538794124, + "request_tps": 12.775416190731752, + "peak_memory_gb": 28.05746427, + "finish_reason": "stop", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "source": "mlx-community/Qwen3.8-27B-mxfp8", + "phase": "measured", + "round": 5, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 1.0833127080113627, + "total_s": 9.521148208004888, + "prompt_tps": 48.494330924975344, + "decode_tps": 15.203115253554254, + "request_tps": 13.443756698628452, + "peak_memory_gb": 28.05746427, + "finish_reason": "length", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json new file mode 100644 index 0000000..54446c6 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T133721Z-qwen3.8-27b-mxfp8/summary.json @@ -0,0 +1,22 @@ +{ + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-mxfp8", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.6405866249930113, + "ttft_s_p50": 0.6006166669540107, + "ttft_s_p95": 0.993492508213967, + "total_s_mean": 8.97830267478712, + "total_s_p50": 8.952084249991458, + "total_s_p95": 9.543902474397328, + "prompt_tps_mean": 77.25518114106583, + "prompt_tps_p50": 69.803885982221, + "prompt_tps_p95": 108.54351756783554, + "decode_tps_mean": 15.141694584402668, + "decode_tps_p50": 15.417877788139895, + "decode_tps_p95": 15.90376325022775, + "request_tps_mean": 14.036390549836813, + "request_tps_p50": 14.298346220336581, + "request_tps_p95": 14.998850485179887, + "peak_memory_gb": 28.05746427 +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json new file mode 100644 index 0000000..a294dc3 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/metadata.json @@ -0,0 +1,48 @@ +{ + "run_id": "20260831T134407Z", + "created_at": "2026-08-31T13:44:45.718463+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "model": "mlx-community/Qwen3.8-27B-nvfp4", + "label": "qwen3.8-27b-nvfp4-rerun", + "warmups": 2, + "rounds": 5, + "max_tokens": 128, + "seed": 42, + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/direct-results" + }, + "load_s": 3.153125083015766, + "mlx_version": "0.32.0", + "device": "Device(gpu, 0)", + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "summary": { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.46268870839849113, + "ttft_s_p50": 0.5028267079615034, + "ttft_s_p95": 0.5507681754184887, + "total_s_mean": 5.0599139665835535, + "total_s_p50": 5.047160332964268, + "total_s_p95": 5.222982274997048, + "prompt_tps_mean": 103.80325950757997, + "prompt_tps_p50": 102.6553925326314, + "prompt_tps_p95": 128.99882432098076, + "decode_tps_mean": 27.91928963114296, + "decode_tps_p50": 27.87385628056727, + "decode_tps_p95": 28.51407145210707, + "request_tps_mean": 25.315032232739288, + "request_tps_p50": 25.360795289977208, + "request_tps_p95": 26.258409367849122, + "peak_memory_gb": 15.479377792 + } +} \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json new file mode 100644 index 0000000..2d74af1 --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/raw.json @@ -0,0 +1,128 @@ +[ + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.46846333297435194, + "total_s": 4.887791082961485, + "prompt_tps": 84.84884256090237, + "decode_tps": 29.034596267228054, + "request_tps": 26.18769866130316, + "peak_memory_gb": 15.399948802, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "warmup", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.476554416003637, + "total_s": 4.906530833046418, + "prompt_tps": 122.50184376172285, + "decode_tps": 28.96545575207384, + "request_tps": 26.0876787195335, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 1, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.34607333299936727, + "total_s": 4.835503207985312, + "prompt_tps": 90.02367046337802, + "decode_tps": 28.58315107058229, + "request_tps": 26.470874797192113, + "peak_memory_gb": 15.453767792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 2, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5028267079615034, + "total_s": 5.047160332964268, + "prompt_tps": 113.11164547028159, + "decode_tps": 28.2377529782062, + "request_tps": 25.360795289977208, + "peak_memory_gb": 15.453784176, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 3, + "prompt_name": "code", + "prompt_tokens": 54, + "output_tokens": 128, + "ttft_s": 0.5305347089888528, + "total_s": 5.134004874969833, + "prompt_tps": 132.97061903365554, + "decode_tps": 27.87385628056727, + "request_tps": 24.93180336155254, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 4, + "prompt_name": "short", + "prompt_tokens": 20, + "output_tokens": 128, + "ttft_s": 0.3781822500168346, + "total_s": 5.037674791994505, + "prompt_tps": 80.25497003795336, + "decode_tps": 27.538176565956825, + "request_tps": 25.408547650477164, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + }, + { + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "source": "mlx-community/Qwen3.8-27B-nvfp4", + "phase": "measured", + "round": 5, + "prompt_name": "medium", + "prompt_tokens": 43, + "output_tokens": 128, + "ttft_s": 0.5558265420258977, + "total_s": 5.245226625003852, + "prompt_tps": 102.6553925326314, + "decode_tps": 27.363511260402223, + "request_tps": 24.403140064497403, + "peak_memory_gb": 15.479377792, + "finish_reason": "length", + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json new file mode 100644 index 0000000..400ae9c --- /dev/null +++ b/benchmarks/inference/direct-results/20260831T134407Z-qwen3.8-27b-nvfp4-rerun/summary.json @@ -0,0 +1,22 @@ +{ + "runtime": "mlx-lm-direct", + "model": "qwen3.8-27b-nvfp4-rerun", + "requests": 5, + "successes": 5, + "ttft_s_mean": 0.46268870839849113, + "ttft_s_p50": 0.5028267079615034, + "ttft_s_p95": 0.5507681754184887, + "total_s_mean": 5.0599139665835535, + "total_s_p50": 5.047160332964268, + "total_s_p95": 5.222982274997048, + "prompt_tps_mean": 103.80325950757997, + "prompt_tps_p50": 102.6553925326314, + "prompt_tps_p95": 128.99882432098076, + "decode_tps_mean": 27.91928963114296, + "decode_tps_p50": 27.87385628056727, + "decode_tps_p95": 28.51407145210707, + "request_tps_mean": 25.315032232739288, + "request_tps_p50": 25.360795289977208, + "request_tps_p95": 26.258409367849122, + "peak_memory_gb": 15.479377792 +} \ No newline at end of file diff --git a/benchmarks/inference/direct_mlx_benchmark.py b/benchmarks/inference/direct_mlx_benchmark.py new file mode 100644 index 0000000..aaa5861 --- /dev/null +++ b/benchmarks/inference/direct_mlx_benchmark.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +"""Measure direct MLX-LM generation without an HTTP serving layer.""" + +from __future__ import annotations + +import argparse +import json +import math +import platform +import statistics +import time +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import mlx.core as mx +from benchmark import PROMPTS +from mlx_lm import load, stream_generate +from mlx_lm.sample_utils import make_sampler + + +def percentile(values: list[float], fraction: float) -> float: + ordered = sorted(values) + position = (len(ordered) - 1) * fraction + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + return ordered[lower] + (ordered[upper] - ordered[lower]) * (position - lower) + + +def summarize(rows: list[dict[str, Any]]) -> dict[str, Any]: + measured = [row for row in rows if row["phase"] == "measured"] + result: dict[str, Any] = { + "runtime": "mlx-lm-direct", + "model": measured[0]["model"], + "requests": len(measured), + "successes": sum(row["error"] is None for row in measured), + } + for field in ("ttft_s", "total_s", "prompt_tps", "decode_tps", "request_tps"): + values = [float(row[field]) for row in measured if row[field] is not None] + result[f"{field}_mean"] = statistics.fmean(values) + result[f"{field}_p50"] = percentile(values, 0.5) + result[f"{field}_p95"] = percentile(values, 0.95) + result["peak_memory_gb"] = max(float(row["peak_memory_gb"]) for row in measured) + return result + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", required=True) + parser.add_argument("--label", required=True) + parser.add_argument("--warmups", type=int, default=2) + parser.add_argument("--rounds", type=int, default=5) + parser.add_argument("--max-tokens", type=int, default=128) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument( + "--output-dir", type=Path, default=Path(__file__).parent / "direct-results" + ) + return parser.parse_args() + + +def main() -> int: + args = parse_args() + if args.warmups < 0 or args.rounds < 1 or args.max_tokens < 1: + raise SystemExit("warmups must be >= 0; rounds and max-tokens must be >= 1") + + run_id = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") + output_dir = args.output_dir / f"{run_id}-{args.label}" + output_dir.mkdir(parents=True, exist_ok=False) + + load_started = time.perf_counter() + model, tokenizer = load(args.model, trust_remote_code=True) + load_s = time.perf_counter() - load_started + sampler = make_sampler(temp=0.0) + prompt_items = list(PROMPTS.items()) + rows: list[dict[str, Any]] = [] + + for phase, count in (("warmup", args.warmups), ("measured", args.rounds)): + for round_number in range(count): + prompt_name, prompt = prompt_items[round_number % len(prompt_items)] + mx.random.seed(args.seed) + started = time.perf_counter() + first_token_at: float | None = None + final = None + error = None + try: + for response in stream_generate( + model, + tokenizer, + prompt, + max_tokens=args.max_tokens, + sampler=sampler, + ): + if first_token_at is None: + first_token_at = time.perf_counter() + final = response + except Exception as exception: # Preserve evidence before failing the run. + error = f"{type(exception).__name__}: {exception}" + total_s = time.perf_counter() - started + tokens = final.generation_tokens if final is not None else 0 + row = { + "runtime": "mlx-lm-direct", + "model": args.label, + "source": args.model, + "phase": phase, + "round": round_number + 1, + "prompt_name": prompt_name, + "prompt_tokens": final.prompt_tokens if final is not None else None, + "output_tokens": tokens, + "ttft_s": first_token_at - started if first_token_at is not None else None, + "total_s": total_s, + "prompt_tps": final.prompt_tps if final is not None else None, + "decode_tps": final.generation_tps if final is not None else None, + "request_tps": tokens / total_s if tokens else None, + "peak_memory_gb": final.peak_memory if final is not None else None, + "finish_reason": final.finish_reason if final is not None else None, + "error": error, + } + rows.append(row) + print( + f"{phase} {round_number + 1}/{count} prompt={prompt_name} " + f"total={total_s:.3f}s tokens={tokens} " + f"decode={row['decode_tps'] or 0:.2f}tok/s error={error or 'none'}", + flush=True, + ) + + summary = summarize(rows) + metadata = { + "run_id": run_id, + "created_at": datetime.now(UTC).isoformat(), + "host": { + "platform": platform.platform(), + "machine": platform.machine(), + "python": platform.python_version(), + }, + "arguments": vars(args) | {"output_dir": str(args.output_dir)}, + "load_s": load_s, + "mlx_version": getattr(mx, "__version__", None), + "device": str(mx.default_device()), + "prompts": PROMPTS, + "summary": summary, + } + (output_dir / "raw.json").write_text(json.dumps(rows, indent=2), encoding="utf-8") + (output_dir / "metadata.json").write_text( + json.dumps(metadata, indent=2, default=str), encoding="utf-8" + ) + (output_dir / "summary.json").write_text(json.dumps(summary, indent=2), encoding="utf-8") + print(json.dumps(summary, indent=2)) + print(f"Results: {output_dir}") + return 1 if any(row["error"] for row in rows if row["phase"] == "measured") else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/benchmarks/inference/results/20260831T114727Z/metadata.json b/benchmarks/inference/results/20260831T114727Z/metadata.json new file mode 100644 index 0000000..471c894 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/metadata.json @@ -0,0 +1,136 @@ +{ + "run_id": "20260831T114727Z", + "created_at": "2026-08-31T11:47:27.622260+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "vllm", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-nvfp4" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 13:47:28 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 13:47:28 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 13:47:28 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 13:47:30 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 13:47:30 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "vllm-metal": { + "object": "list", + "data": [ + { + "id": "qwen3.8-27b-nvfp4", + "object": "model", + "created": 1788176852, + "owned_by": "vllm", + "root": "mlx-community/Qwen3.8-27B-nvfp4", + "parent": null, + "max_model_len": 8192, + "permission": [ + { + "id": "modelperm-956fc45cb39100d2", + "object": "model_permission", + "created": 1788176852, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + } + ] + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T11:49:15.455115+00:00", + "summary": [ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.3338013501954265, + "ttft_s_p50": 0.37227433401858434, + "ttft_s_p95": 0.4201282836031169, + "total_s_mean": 5.444907250197138, + "total_s_p50": 5.454897999996319, + "total_s_p95": 5.562144016823732, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 25.04504255080693, + "decode_tps_p50": 24.962500989743333, + "decode_tps_p95": 25.32382313060655, + "request_tps_mean": 23.517575676354507, + "request_tps_p50": 23.465150035818517, + "request_tps_p95": 24.196276605031606, + "aggregate_output_tps_mean": 23.516827822391825, + "aggregate_output_tps_p50": 23.464397444683396, + "aggregate_output_tps_p95": 24.195480643338048 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.1358489938022103, + "ttft_s_p50": 1.1691114795103204, + "ttft_s_p95": 1.2360623297834537, + "total_s_mean": 9.343862856202758, + "total_s_p50": 9.272835896001197, + "total_s_p95": 9.591099269420374, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.572524640705884, + "decode_tps_p50": 15.580143217705817, + "decode_tps_p95": 15.920671319893613, + "request_tps_mean": 13.679827878866371, + "request_tps_p50": 13.803600866803595, + "request_tps_p95": 13.831739950302437, + "aggregate_output_tps_mean": 54.66256367620249, + "aggregate_output_tps_p50": 54.89183177619698, + "aggregate_output_tps_p95": 55.24148726503248 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114727Z/raw.csv b/benchmarks/inference/results/20260831T114727Z/raw.csv new file mode 100644 index 0000000..2e7c106 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,1,0,0,8b712536-91f2-4ded-b334-6db253435ba3,short,106,128,2026-08-31T11:47:32.567050+00:00,0.561656582984142,5.602756957989186,20,128,,25.39128175956463,22.845895504619364,5.6035958750289865,22.842475234589944, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,1,1,0,be5a9e94-abd6-4cbc-8029-7a55e3ad129c,medium,252,128,2026-08-31T11:47:38.170087+00:00,0.3621360409888439,5.426666249986738,43,128,,25.273815086064428,23.587225398339694,5.426845958980266,23.586444311762243, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,0,0,e0ac985d-818e-45ec-a2c1-7e077fe0f1cd,short,106,128,2026-08-31T11:47:43.597014+00:00,0.21451262495247647,5.262066499970388,20,128,,25.358817987761604,24.325044162919706,5.262241083022673,24.324237141654443, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,1,0,428bad62-8fbf-41a1-9b36-07912506da11,medium,252,128,2026-08-31T11:47:48.859322+00:00,0.37227433401858434,5.454897999996319,43,128,,25.18384370198632,23.465150035818517,5.455072959011886,23.464397444683396, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,2,0,b2de9565-6ff0-47a1-91e6-847b83994f6c,code,267,128,2026-08-31T11:47:54.314442+00:00,0.40532424999400973,5.533015583991073,54,128,,24.962500989743333,23.13385857259254,5.533179917023517,23.13317150707355, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,3,0,dc02daab-d1f2-49ac-8e29-01ee86148e0a,short,106,128,2026-08-31T11:47:59.847681+00:00,0.25306625000666827,5.405130041996017,20,128,,24.844412873734196,23.681206373479203,5.405301624967251,23.680454650072466, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,1,4,0,677f2a25-31e5-43e7-b8fd-5e607453b904,medium,252,128,2026-08-31T11:48:05.253048+00:00,0.42382929200539365,5.569426125031896,43,128,,24.875637200809187,22.982619236962577,5.569605667027645,22.981878368475286, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,0,7109b7b9-3690-49d6-8b20-969734c3dcad,short,106,128,2026-08-31T11:48:10.822944+00:00,1.1026008339831606,8.694552083965391,20,128,,16.859960738064487,14.721862467884838,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,1,b34a5a99-e633-4f3a-892d-67af2b425779,medium,252,128,2026-08-31T11:48:10.823149+00:00,1.1023455830290914,8.694302666990552,43,128,,16.859947782161328,14.722284799904022,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,2,1260b39f-a4b1-4d94-9406-70028f614ce0,code,267,128,2026-08-31T11:48:10.823017+00:00,1.0163157919887453,8.694556291971821,54,128,,16.670485901070972,14.721855342772315,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,0,3,b43d525a-50de-4e41-b00c-d1159d1ddfd4,short,106,128,2026-08-31T11:48:10.822845+00:00,1.1026693329913542,8.694680040993262,20,128,,16.85982869664412,14.721645810600473,8.695109791995492,58.883672805527404, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,0,1da340a9-b91e-4f9b-8217-2af7c2fbc030,medium,252,128,2026-08-31T11:48:19.518058+00:00,1.245261833013501,9.168966500030365,43,128,,16.154059922603068,13.960133892906699,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,1,23e076e6-7b50-4d08-a3ff-f79ddc53bafe,code,267,128,2026-08-31T11:48:19.518183+00:00,1.2451274580089375,9.168911040993407,54,128,,16.153899038190183,13.96021833211415,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,2,c777d58b-0520-4f3f-8537-6a163ab350f5,short,106,128,2026-08-31T11:48:19.518319+00:00,1.2449699160060845,9.168749333010055,20,128,,16.153907531211612,13.960464546583719,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,warmup,4,1,3,5609655c-e54a-499f-99c6-c77c19c1a3e0,medium,252,128,2026-08-31T11:48:19.518407+00:00,1.174761500034947,9.16857587499544,43,128,,16.012380822970083,13.960728661152478,9.169412165996619,55.83782152346415, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,0,d2615398-7368-4f9a-9424-7a2e76d8955d,short,106,128,2026-08-31T11:48:28.687706+00:00,1.0508670000126585,9.266459833015688,20,128,,15.580129468662143,13.813257954666332,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,1,b307dcc2-b325-4005-a63f-54123aa28e31,medium,252,128,2026-08-31T11:48:28.687614+00:00,1.0509756249957718,9.266665957984515,43,128,,15.579944570943383,13.812950696653772,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,2,09c7607f-96f5-4a6d-a9e1-3637c8afc257,code,267,128,2026-08-31T11:48:28.687778+00:00,1.0507545830332674,9.266332915984094,54,128,,15.580156966749492,13.813447149001583,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,0,3,0328d6c0-e3f4-4a86-917d-dbeb5b8deee7,short,106,128,2026-08-31T11:48:28.687501+00:00,0.9898037080420181,9.266721500025596,20,128,,15.464693889308833,13.81286790583341,9.267125833022874,55.24906095215814, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,0,74110182-284c-4250-9395-e3d0f2772eef,medium,252,128,2026-08-31T11:48:37.954714+00:00,1.1083811249700375,9.379890666983556,43,128,,15.474805336299134,13.646214496992963,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,1,fca02813-8ad6-4e99-bc76-9e922ea9a63a,code,267,128,2026-08-31T11:48:37.954992+00:00,1.1690869170124643,9.379730832995847,54,128,,15.589520299476966,13.646447033397155,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,2,5eb13719-9063-4ddd-92c8-73332709ecfd,short,106,128,2026-08-31T11:48:37.954812+00:00,1.1692542500095442,9.37985170801403,20,128,,15.589608509574806,13.646271176189105,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,1,3,9d9a5b23-6950-4d5c-bad2-72f896d38009,medium,252,128,2026-08-31T11:48:37.954917+00:00,1.1691360420081764,9.379842250025831,43,128,,15.58940202671112,13.646284936150979,9.380457791965455,54.58155788926826, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,0,caf79143-9d5b-4fb2-830e-3bd2d38cc5d3,code,267,128,2026-08-31T11:48:47.335341+00:00,1.172440624970477,9.591207291989122,54,128,,15.204127286417465,13.345556623190662,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,1,d77a5e81-922c-49ce-96c5-1fb443746984,short,106,128,2026-08-31T11:48:47.335566+00:00,1.2360592089826241,9.591093584022019,20,128,,15.320104532711328,13.345714842490702,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,2,9ccc94c9-d569-4785-9f48-e48e760b0e15,medium,252,128,2026-08-31T11:48:47.335447+00:00,1.236121624999214,9.591066916007549,43,128,,15.320267882275031,13.345751950324445,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,2,3,e7c5be37-35a8-47a5-869d-a3bee16750d4,code,267,128,2026-08-31T11:48:47.335653+00:00,1.2359632499865256,9.590960291970987,54,128,,15.32017298830757,13.345900316900948,9.591755541972816,53.37917524685921, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,0,49d569ca-4eef-46fd-88f9-58cf68e01898,short,106,128,2026-08-31T11:48:56.927355+00:00,1.067110000003595,9.073982625035569,20,124,,15.486695718417879,13.665443843575128,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,1,a34ede89-ca43-4a1c-a7c8-c916bfec24a5,medium,252,128,2026-08-31T11:48:56.927278+00:00,1.0672226250171661,9.253953957988415,43,128,,15.635055652124883,13.831925313341854,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,2,af0dfbe9-f1a5-434a-a714-98fad2af83a3,code,267,128,2026-08-31T11:48:56.927205+00:00,1.067305459000636,9.254084500018507,54,128,,15.634964539617723,13.831730194352993,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,3,3,80dc0643-6f40-4ab5-805d-ace4e9f149bb,short,106,128,2026-08-31T11:48:56.927082+00:00,1.0054587500053458,9.254091374983545,20,128,,15.517723460297553,13.831719918610334,9.254564541974105,54.89183177619698, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,0,12e42bf2-6a75-4f89-a598-ac14d5b67a78,medium,252,128,2026-08-31T11:49:06.181834+00:00,1.2329518750193529,9.273020750028081,43,128,,15.920261628338482,13.803484695061464,9.273481999989599,55.21119251652985, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,1,5f2cf183-3654-45be-81ed-4eebfb57bce1,code,267,128,2026-08-31T11:49:06.182126+00:00,1.1721314580063336,9.272628374979831,54,128,,15.801499748959015,13.804068795141205,9.273481999989599,55.21119251652985, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,2,746da721-ed00-4c80-bc92-ed115313463e,short,106,128,2026-08-31T11:49:06.182026+00:00,1.2329447079682723,9.272807125002146,20,128,,15.920670449385964,13.80380269690667,9.273481999989599,55.21119251652985, +20260831T114727Z,vllm-metal,qwen3.8-27b-nvfp4,measured,4,4,3,5c45d61d-9d4a-4fc4-931d-5193cdcaa598,medium,252,128,2026-08-31T11:49:06.181940+00:00,1.2330110420007259,9.272864667000249,43,128,,15.920687859538933,13.803717038545729,9.273481999989599,55.21119251652985, diff --git a/benchmarks/inference/results/20260831T114727Z/raw.json b/benchmarks/inference/results/20260831T114727Z/raw.json new file mode 100644 index 0000000..1d28313 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "8b712536-91f2-4ded-b334-6db253435ba3", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:32.567050+00:00", + "ttft_s": 0.561656582984142, + "total_s": 5.602756957989186, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.39128175956463, + "request_tps": 22.845895504619364, + "group_elapsed_s": 5.6035958750289865, + "group_output_tps": 22.842475234589944, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "be5a9e94-abd6-4cbc-8029-7a55e3ad129c", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:38.170087+00:00", + "ttft_s": 0.3621360409888439, + "total_s": 5.426666249986738, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.273815086064428, + "request_tps": 23.587225398339694, + "group_elapsed_s": 5.426845958980266, + "group_output_tps": 23.586444311762243, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "e0ac985d-818e-45ec-a2c1-7e077fe0f1cd", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:43.597014+00:00", + "ttft_s": 0.21451262495247647, + "total_s": 5.262066499970388, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.358817987761604, + "request_tps": 24.325044162919706, + "group_elapsed_s": 5.262241083022673, + "group_output_tps": 24.324237141654443, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "428bad62-8fbf-41a1-9b36-07912506da11", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:48.859322+00:00", + "ttft_s": 0.37227433401858434, + "total_s": 5.454897999996319, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 25.18384370198632, + "request_tps": 23.465150035818517, + "group_elapsed_s": 5.455072959011886, + "group_output_tps": 23.464397444683396, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "b2de9565-6ff0-47a1-91e6-847b83994f6c", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:54.314442+00:00", + "ttft_s": 0.40532424999400973, + "total_s": 5.533015583991073, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 24.962500989743333, + "request_tps": 23.13385857259254, + "group_elapsed_s": 5.533179917023517, + "group_output_tps": 23.13317150707355, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "dc02daab-d1f2-49ac-8e29-01ee86148e0a", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:47:59.847681+00:00", + "ttft_s": 0.25306625000666827, + "total_s": 5.405130041996017, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 24.844412873734196, + "request_tps": 23.681206373479203, + "group_elapsed_s": 5.405301624967251, + "group_output_tps": 23.680454650072466, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "677f2a25-31e5-43e7-b8fd-5e607453b904", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:05.253048+00:00", + "ttft_s": 0.42382929200539365, + "total_s": 5.569426125031896, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 24.875637200809187, + "request_tps": 22.982619236962577, + "group_elapsed_s": 5.569605667027645, + "group_output_tps": 22.981878368475286, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "7109b7b9-3690-49d6-8b20-969734c3dcad", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.822944+00:00", + "ttft_s": 1.1026008339831606, + "total_s": 8.694552083965391, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.859960738064487, + "request_tps": 14.721862467884838, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "b34a5a99-e633-4f3a-892d-67af2b425779", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.823149+00:00", + "ttft_s": 1.1023455830290914, + "total_s": 8.694302666990552, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.859947782161328, + "request_tps": 14.722284799904022, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "1260b39f-a4b1-4d94-9406-70028f614ce0", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.823017+00:00", + "ttft_s": 1.0163157919887453, + "total_s": 8.694556291971821, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.670485901070972, + "request_tps": 14.721855342772315, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "b43d525a-50de-4e41-b00c-d1159d1ddfd4", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:10.822845+00:00", + "ttft_s": 1.1026693329913542, + "total_s": 8.694680040993262, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.85982869664412, + "request_tps": 14.721645810600473, + "group_elapsed_s": 8.695109791995492, + "group_output_tps": 58.883672805527404, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "1da340a9-b91e-4f9b-8217-2af7c2fbc030", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518058+00:00", + "ttft_s": 1.245261833013501, + "total_s": 9.168966500030365, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.154059922603068, + "request_tps": 13.960133892906699, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "23e076e6-7b50-4d08-a3ff-f79ddc53bafe", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518183+00:00", + "ttft_s": 1.2451274580089375, + "total_s": 9.168911040993407, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.153899038190183, + "request_tps": 13.96021833211415, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "c777d58b-0520-4f3f-8537-6a163ab350f5", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518319+00:00", + "ttft_s": 1.2449699160060845, + "total_s": 9.168749333010055, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.153907531211612, + "request_tps": 13.960464546583719, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "5609655c-e54a-499f-99c6-c77c19c1a3e0", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:19.518407+00:00", + "ttft_s": 1.174761500034947, + "total_s": 9.16857587499544, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 16.012380822970083, + "request_tps": 13.960728661152478, + "group_elapsed_s": 9.169412165996619, + "group_output_tps": 55.83782152346415, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "d2615398-7368-4f9a-9424-7a2e76d8955d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687706+00:00", + "ttft_s": 1.0508670000126585, + "total_s": 9.266459833015688, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.580129468662143, + "request_tps": 13.813257954666332, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "b307dcc2-b325-4005-a63f-54123aa28e31", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687614+00:00", + "ttft_s": 1.0509756249957718, + "total_s": 9.266665957984515, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.579944570943383, + "request_tps": 13.812950696653772, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "09c7607f-96f5-4a6d-a9e1-3637c8afc257", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687778+00:00", + "ttft_s": 1.0507545830332674, + "total_s": 9.266332915984094, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.580156966749492, + "request_tps": 13.813447149001583, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "0328d6c0-e3f4-4a86-917d-dbeb5b8deee7", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:28.687501+00:00", + "ttft_s": 0.9898037080420181, + "total_s": 9.266721500025596, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.464693889308833, + "request_tps": 13.81286790583341, + "group_elapsed_s": 9.267125833022874, + "group_output_tps": 55.24906095215814, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "74110182-284c-4250-9395-e3d0f2772eef", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954714+00:00", + "ttft_s": 1.1083811249700375, + "total_s": 9.379890666983556, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.474805336299134, + "request_tps": 13.646214496992963, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "fca02813-8ad6-4e99-bc76-9e922ea9a63a", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954992+00:00", + "ttft_s": 1.1690869170124643, + "total_s": 9.379730832995847, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.589520299476966, + "request_tps": 13.646447033397155, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "5eb13719-9063-4ddd-92c8-73332709ecfd", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954812+00:00", + "ttft_s": 1.1692542500095442, + "total_s": 9.37985170801403, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.589608509574806, + "request_tps": 13.646271176189105, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "9d9a5b23-6950-4d5c-bad2-72f896d38009", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:37.954917+00:00", + "ttft_s": 1.1691360420081764, + "total_s": 9.379842250025831, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.58940202671112, + "request_tps": 13.646284936150979, + "group_elapsed_s": 9.380457791965455, + "group_output_tps": 54.58155788926826, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "caf79143-9d5b-4fb2-830e-3bd2d38cc5d3", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335341+00:00", + "ttft_s": 1.172440624970477, + "total_s": 9.591207291989122, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.204127286417465, + "request_tps": 13.345556623190662, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "d77a5e81-922c-49ce-96c5-1fb443746984", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335566+00:00", + "ttft_s": 1.2360592089826241, + "total_s": 9.591093584022019, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.320104532711328, + "request_tps": 13.345714842490702, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "9ccc94c9-d569-4785-9f48-e48e760b0e15", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335447+00:00", + "ttft_s": 1.236121624999214, + "total_s": 9.591066916007549, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.320267882275031, + "request_tps": 13.345751950324445, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "e7c5be37-35a8-47a5-869d-a3bee16750d4", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:47.335653+00:00", + "ttft_s": 1.2359632499865256, + "total_s": 9.590960291970987, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.32017298830757, + "request_tps": 13.345900316900948, + "group_elapsed_s": 9.591755541972816, + "group_output_tps": 53.37917524685921, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "49d569ca-4eef-46fd-88f9-58cf68e01898", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927355+00:00", + "ttft_s": 1.067110000003595, + "total_s": 9.073982625035569, + "prompt_tokens": 20, + "output_tokens": 124, + "prompt_tps": null, + "decode_tps": 15.486695718417879, + "request_tps": 13.665443843575128, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "a34ede89-ca43-4a1c-a7c8-c916bfec24a5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927278+00:00", + "ttft_s": 1.0672226250171661, + "total_s": 9.253953957988415, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.635055652124883, + "request_tps": 13.831925313341854, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "af0dfbe9-f1a5-434a-a714-98fad2af83a3", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927205+00:00", + "ttft_s": 1.067305459000636, + "total_s": 9.254084500018507, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.634964539617723, + "request_tps": 13.831730194352993, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "80dc0643-6f40-4ab5-805d-ace4e9f149bb", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:48:56.927082+00:00", + "ttft_s": 1.0054587500053458, + "total_s": 9.254091374983545, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.517723460297553, + "request_tps": 13.831719918610334, + "group_elapsed_s": 9.254564541974105, + "group_output_tps": 54.89183177619698, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "12e42bf2-6a75-4f89-a598-ac14d5b67a78", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.181834+00:00", + "ttft_s": 1.2329518750193529, + "total_s": 9.273020750028081, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.920261628338482, + "request_tps": 13.803484695061464, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "5f2cf183-3654-45be-81ed-4eebfb57bce1", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.182126+00:00", + "ttft_s": 1.1721314580063336, + "total_s": 9.272628374979831, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.801499748959015, + "request_tps": 13.804068795141205, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "746da721-ed00-4c80-bc92-ed115313463e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.182026+00:00", + "ttft_s": 1.2329447079682723, + "total_s": 9.272807125002146, + "prompt_tokens": 20, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.920670449385964, + "request_tps": 13.80380269690667, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + }, + { + "run_id": "20260831T114727Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "5c45d61d-9d4a-4fc4-931d-5193cdcaa598", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:06.181940+00:00", + "ttft_s": 1.2330110420007259, + "total_s": 9.272864667000249, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.920687859538933, + "request_tps": 13.803717038545729, + "group_elapsed_s": 9.273481999989599, + "group_output_tps": 55.21119251652985, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114727Z/summary.json b/benchmarks/inference/results/20260831T114727Z/summary.json new file mode 100644 index 0000000..47cf7b8 --- /dev/null +++ b/benchmarks/inference/results/20260831T114727Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.3338013501954265, + "ttft_s_p50": 0.37227433401858434, + "ttft_s_p95": 0.4201282836031169, + "total_s_mean": 5.444907250197138, + "total_s_p50": 5.454897999996319, + "total_s_p95": 5.562144016823732, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 25.04504255080693, + "decode_tps_p50": 24.962500989743333, + "decode_tps_p95": 25.32382313060655, + "request_tps_mean": 23.517575676354507, + "request_tps_p50": 23.465150035818517, + "request_tps_p95": 24.196276605031606, + "aggregate_output_tps_mean": 23.516827822391825, + "aggregate_output_tps_p50": 23.464397444683396, + "aggregate_output_tps_p95": 24.195480643338048 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-nvfp4", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.1358489938022103, + "ttft_s_p50": 1.1691114795103204, + "ttft_s_p95": 1.2360623297834537, + "total_s_mean": 9.343862856202758, + "total_s_p50": 9.272835896001197, + "total_s_p95": 9.591099269420374, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.572524640705884, + "decode_tps_p50": 15.580143217705817, + "decode_tps_p95": 15.920671319893613, + "request_tps_mean": 13.679827878866371, + "request_tps_p50": 13.803600866803595, + "request_tps_p95": 13.831739950302437, + "aggregate_output_tps_mean": 54.66256367620249, + "aggregate_output_tps_p50": 54.89183177619698, + "aggregate_output_tps_p95": 55.24148726503248 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114928Z/metadata.json b/benchmarks/inference/results/20260831T114928Z/metadata.json new file mode 100644 index 0000000..68b26b1 --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/metadata.json @@ -0,0 +1,135 @@ +{ + "run_id": "20260831T114928Z", + "created_at": "2026-08-31T11:49:28.271574+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "ollama", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mlx" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 13:49:28 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 13:49:28 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 13:49:28 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 13:49:30 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 13:49:31 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "ollama": { + "license": "\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n", + "modelfile": "# Modelfile generated by \"ollama show\"\nFROM qwen3.8:27b-mlx\nTEMPLATE {{ .Prompt }}\nRENDERER qwen3.8\nPARSER qwen3.5\nPARAMETER min_p 0\nPARAMETER presence_penalty 0\nPARAMETER repeat_penalty 1\nPARAMETER temperature 1\nPARAMETER top_k 20\nPARAMETER top_p 0.95\nLICENSE \"\"\"\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\"\"\"\n", + "parameters": "top_k 20\ntop_p 0.95\nmin_p 0\npresence_penalty 0\nrepeat_penalty 1\ntemperature 1", + "template": "{{ .Prompt }}", + "details": { + "parent_model": "", + "format": "safetensors", + "family": "qwen3_5", + "families": null, + "parameter_size": "27.8B", + "quantization_level": "nvfp4" + }, + "model_info": { + "general.architecture": "qwen3_5", + "general.parameter_count": 27781081984, + "qwen3_5.block_count": 64, + "qwen3_5.context_length": 262144, + "qwen3_5.embedding_length": 5120 + }, + "capabilities": [ + "completion", + "vision", + "tools", + "thinking" + ], + "modified_at": "2026-08-20T17:42:35.895756165+02:00", + "requires": "0.32.12" + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T11:51:31.347428+00:00", + "summary": [ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.18287834140937775, + "ttft_s_p50": 0.11936691601295024, + "ttft_s_p95": 0.3934195415815338, + "total_s_mean": 2.769790649821516, + "total_s_p50": 2.7544727920321748, + "total_s_p95": 3.450829833233729, + "prompt_tps_mean": 1201063.1140147406, + "prompt_tps_p50": 816.5668113066018, + "prompt_tps_p95": 3147950.5530253737, + "decode_tps_mean": 44.301349906835995, + "decode_tps_p50": 45.72630609321048, + "decode_tps_p95": 52.87492274071074, + "request_tps_mean": 42.02276635113332, + "request_tps_p50": 44.19526212378514, + "request_tps_p95": 46.37707708697445, + "aggregate_output_tps_mean": 42.01338924160581, + "aggregate_output_tps_p50": 44.18819214792146, + "aggregate_output_tps_p95": 46.368391249450006 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 5.296346539442311, + "ttft_s_p50": 5.078280103509314, + "ttft_s_p95": 10.691154320855276, + "total_s_mean": 8.731022270751419, + "total_s_p50": 8.619212541496381, + "total_s_p95": 14.520918775268365, + "prompt_tps_mean": 4873110.971654413, + "prompt_tps_p50": 3843030.8359358422, + "prompt_tps_p95": 9861582.229537027, + "decode_tps_mean": 33.61516109813652, + "decode_tps_p50": 34.997484039567254, + "decode_tps_p95": 40.1951363429803, + "request_tps_mean": 18.258369295759792, + "request_tps_p50": 12.924863350205007, + "request_tps_p95": 36.94528376333951, + "aggregate_output_tps_mean": 33.27170984729996, + "aggregate_output_tps_p50": 33.918007597024484, + "aggregate_output_tps_p95": 35.45560236969114 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114928Z/raw.csv b/benchmarks/inference/results/20260831T114928Z/raw.csv new file mode 100644 index 0000000..ed4b33d --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,1,0,0,0bd09e14-c217-44ad-b0a1-98600e1bf40e,short,106,128,2026-08-31T11:49:33.251646+00:00,3.627692041976843,5.577126249962021,20,96,116.69505030650684,46.54721674973356,17.213166010121025,5.578097124991473,17.210170036282175, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,1,1,0,ada11e2f-0b97-42ec-a5be-5175f0bb29b1,medium,252,128,2026-08-31T11:49:38.829357+00:00,0.3621791250188835,2.7543381249997765,43,128,157.23847751774449,52.253166030964316,46.472144737135494,2.7547111669555306,46.46585149667936, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,0,0,a3bd0047-d992-4553-9b17-4472b6290acc,short,106,128,2026-08-31T11:49:41.584102+00:00,0.11936691601295024,2.1721785410190932,20,96,448.5805263612496,45.72630609321048,44.19526212378514,2.172526082955301,44.18819214792146, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,1,0,a8033a77-d60f-42b4-a2a9-8f6a97ef3496,medium,252,128,2026-08-31T11:49:43.757773+00:00,0.14179470797535032,2.78225216595456,43,128,816.5668113066018,47.377908815894095,46.00589463683088,2.7835853330325335,45.98386062788756, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,2,0,9cc954fc-95f5-474e-b219-8a89d7091934,code,267,128,2026-08-31T11:49:46.540349+00:00,0.45632574998307973,2.7544727920321748,54,128,146.71421293937848,54.2491762219149,46.46987269951035,2.754789875005372,46.46452390484062, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,3,0,e8ee5060-7c14-4c34-bd5b-76d7e51fef07,short,106,128,2026-08-31T11:49:49.295554+00:00,0.0983880830463022,2.5220755000482313,20,96,2758620.689655172,38.503422116977305,38.063888253212134,2.522709292010404,38.054325285929174, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,1,4,0,edd132c8-7aaa-4eb0-98e8-2b2d711f2cf9,medium,252,128,2026-08-31T11:49:51.817976+00:00,0.09851625002920628,3.6179742500535212,43,128,3245283.0188679243,35.649936286183205,35.3789140423281,3.6182677499600686,35.37604424145024, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,0,f0df217a-d72c-4b5c-b4a7-87c267716da0,short,106,128,2026-08-31T11:49:55.436979+00:00,0.11441224999725819,2.9100067079998553,20,96,4138216.428719222,33.47564745836445,32.989614675487815,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,1,9565d3c0-e753-4a7b-be43-115556beec5b,medium,252,128,2026-08-31T11:49:55.437104+00:00,6.277438874996733,9.582254499953706,43,128,3865168.539325843,37.881971614565146,13.358025504396528,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,2,42b7e290-9d92-4647-bbdb-3b3ba99f4c3e,code,267,128,2026-08-31T11:49:55.436744+00:00,3.0367828750167973,6.193101375014521,54,128,771.3321549092078,39.94659431201623,20.66815836672783,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,0,3,c22aaea0-7394-47e8-b6ab-25444fc113d1,short,106,128,2026-08-31T11:49:55.437235+00:00,9.661167457990814,12.522873707988765,20,96,3966679.888932963,32.71048444655768,7.665972063485584,12.524156374973245,35.77087243139417, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,0,84f401af-117d-4956-b939-fdbb1e1ceecf,medium,252,128,2026-08-31T11:50:07.961234+00:00,3.091476584027987,6.373854791978374,43,128,5375000.0,38.1244834091544,20.08203891953902,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,1,b3c9f50e-fcf6-4ad6-b00f-9de925c9092a,code,267,128,2026-08-31T11:50:07.961465+00:00,9.849772959016263,12.85736475000158,54,128,11076923.076923078,41.56311628897905,9.955383742223248,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,2,5f528a81-943d-445a-a2f2-9b986c01c96e,short,106,128,2026-08-31T11:50:07.961041+00:00,0.11284091597190127,3.010363540961407,20,96,1476995.7905619969,32.336156802028626,31.889836125686298,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,warmup,4,1,3,45991adb-aae4-4122-bb0d-21d34b549b46,medium,252,128,2026-08-31T11:50:07.960837+00:00,6.452098417037632,9.774627042002976,43,128,7319148.936170213,37.70880560635668,13.095128791100224,12.858689875050914,37.32884179214249, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,0,1cd3b269-0ed7-4470-9bf5-9aba6dbdda3b,short,106,128,2026-08-31T11:50:20.819745+00:00,9.799659290991258,12.57556495803874,20,96,2461538.461538462,33.69646737394496,7.633851864335801,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,1,b2258258-f465-4131-bb56-cdead87e67c5,medium,252,128,2026-08-31T11:50:20.820473+00:00,2.9737471669795923,6.33726870897226,43,128,4755585.04755585,37.255703188397476,20.19797579654126,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,2,9bb209a2-19a6-4172-a166-c29a75683a37,code,267,128,2026-08-31T11:50:20.819981+00:00,0.11236062500393018,2.89358854200691,54,128,3661016.9491525427,44.88156766233007,44.23572949014475,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,0,3,c74fbeef-16a3-40b2-ade6-0b24b3ee0147,short,106,128,2026-08-31T11:50:20.820727+00:00,6.417795290995855,9.71814520796761,20,98,2608582.235554976,29.056692998778793,10.084228821735737,12.576361375045963,35.781414558656905, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,0,844d6c49-9989-400d-bb5d-31f8490da904,medium,252,128,2026-08-31T11:50:33.396890+00:00,10.48226895800326,14.052842833043542,43,128,5083944.194845118,35.11947732750843,9.108477303896379,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,1,213efda7-5da6-47a7-9517-f63ad082e62e,code,267,128,2026-08-31T11:50:33.397263+00:00,0.11672520800493658,3.508996458025649,54,128,3630252.100840336,36.93011583511385,36.47766577456727,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,2,d03d55d9-a506-44a9-887e-e0ab77392c8d,short,106,128,2026-08-31T11:50:33.397083+00:00,3.606067124987021,6.893670374993235,20,96,3356267.830172848,28.527554720530254,13.925818145909567,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,1,3,55be3260-b3a0-422a-b918-a81261da6a84,medium,252,128,2026-08-31T11:50:33.396391+00:00,6.9750645000021905,10.400937125028577,43,128,6745098.039215687,36.57719818551596,12.30658338391295,14.054668250028044,34.152353613828076, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,0,930b484d-3177-4ee9-9d72-5442ee8cd7c9,code,267,128,2026-08-31T11:50:47.451672+00:00,0.11444733297685161,3.540263083006721,54,128,4248957.431741286,36.58630699945727,36.15550511327833,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,1,231f3cbb-61f2-4042-8a99-def6a8698a7d,short,106,128,2026-08-31T11:50:47.451387+00:00,10.67886250000447,14.209862457995769,20,98,2840102.243680773,27.132075592063398,6.896618478165229,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,2,32437705-bf67-438f-bad7-0bb10d0e2030,medium,252,128,2026-08-31T11:50:47.451856+00:00,3.622358249966055,7.2201729999505915,43,128,4777777.777777778,34.87549075162609,17.72810706902396,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,2,3,a16a724c-155d-4e2d-ba28-3a8683af4d40,code,267,128,2026-08-31T11:50:47.451136+00:00,7.304396207968239,10.594903749995865,54,128,17515407.071034707,38.04511611184116,12.081280115456448,14.210740375041496,33.918007597024484, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,0,f6ab47e7-5bdf-4b5c-be3f-0e639727a2bb,short,106,128,2026-08-31T11:51:01.661760+00:00,10.42098737496417,14.485269291966688,20,96,2105263.1578947366,23.198198961620538,6.627422525947802,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,1,8e015249-8b0c-4560-b93d-603afb968f5b,medium,252,128,2026-08-31T11:51:01.662046+00:00,0.11227079102536663,3.6567685829941183,43,128,3571131.9657835728,35.39152606628793,35.00358228717748,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,2,d2848890-f070-45de-8a2f-4bb2b8788a87,code,267,128,2026-08-31T11:51:01.661862+00:00,7.207546124991495,10.33818608301226,54,128,9458749.343142407,39.94848206301452,12.381282264819164,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,3,3,aba512eb-9825-4e49-9ab8-0c75678ff654,short,106,128,2026-08-31T11:51:01.661952+00:00,3.738764916022774,7.12777191598434,20,96,2652168.1474605487,27.731643965337334,13.468444435590849,14.485774166998453,30.926893850149572, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,0,4c24ae09-884b-4b94-ae8e-37346f1408f2,medium,252,128,2026-08-31T11:51:16.148220+00:00,10.924698917020578,15.198258958000224,43,128,7068880.486602006,29.434480556585335,8.422017308280036,15.199551291007083,31.579879616840746, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,1,c94ceb91-b83d-4d47-a36d-71741993f366,code,267,128,2026-08-31T11:51:16.148464+00:00,0.12565479200566188,3.500943167018704,54,128,4025044.7227191413,37.03108618648992,36.56157609350765,15.199551291007083,31.579879616840746, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,2,0672967f-d135-44b0-b89f-7995f59b2d8c,short,106,128,2026-08-31T11:51:16.148378+00:00,7.600669999956153,10.846751042001415,20,96,2666666.6666666665,28.898123213254255,8.850576511645125,15.199551291007083,31.579879616840746, +20260831T114928Z,ollama,qwen3.8:27b-mlx,measured,4,4,3,8185344b-c228-457a-8c3d-6a7b5f803594,medium,252,128,2026-08-31T11:51:16.147977+00:00,3.5925854169763625,7.520279875025153,43,128,4229785.559708833,31.98591420303284,17.02064313126004,15.199551291007083,31.579879616840746, diff --git a/benchmarks/inference/results/20260831T114928Z/raw.json b/benchmarks/inference/results/20260831T114928Z/raw.json new file mode 100644 index 0000000..705b8ad --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "0bd09e14-c217-44ad-b0a1-98600e1bf40e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:33.251646+00:00", + "ttft_s": 3.627692041976843, + "total_s": 5.577126249962021, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 116.69505030650684, + "decode_tps": 46.54721674973356, + "request_tps": 17.213166010121025, + "group_elapsed_s": 5.578097124991473, + "group_output_tps": 17.210170036282175, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "ada11e2f-0b97-42ec-a5be-5175f0bb29b1", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:38.829357+00:00", + "ttft_s": 0.3621791250188835, + "total_s": 2.7543381249997765, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 157.23847751774449, + "decode_tps": 52.253166030964316, + "request_tps": 46.472144737135494, + "group_elapsed_s": 2.7547111669555306, + "group_output_tps": 46.46585149667936, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "a3bd0047-d992-4553-9b17-4472b6290acc", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:41.584102+00:00", + "ttft_s": 0.11936691601295024, + "total_s": 2.1721785410190932, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 448.5805263612496, + "decode_tps": 45.72630609321048, + "request_tps": 44.19526212378514, + "group_elapsed_s": 2.172526082955301, + "group_output_tps": 44.18819214792146, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "a8033a77-d60f-42b4-a2a9-8f6a97ef3496", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:43.757773+00:00", + "ttft_s": 0.14179470797535032, + "total_s": 2.78225216595456, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 816.5668113066018, + "decode_tps": 47.377908815894095, + "request_tps": 46.00589463683088, + "group_elapsed_s": 2.7835853330325335, + "group_output_tps": 45.98386062788756, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "9cc954fc-95f5-474e-b219-8a89d7091934", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:46.540349+00:00", + "ttft_s": 0.45632574998307973, + "total_s": 2.7544727920321748, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 146.71421293937848, + "decode_tps": 54.2491762219149, + "request_tps": 46.46987269951035, + "group_elapsed_s": 2.754789875005372, + "group_output_tps": 46.46452390484062, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "e8ee5060-7c14-4c34-bd5b-76d7e51fef07", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:49.295554+00:00", + "ttft_s": 0.0983880830463022, + "total_s": 2.5220755000482313, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2758620.689655172, + "decode_tps": 38.503422116977305, + "request_tps": 38.063888253212134, + "group_elapsed_s": 2.522709292010404, + "group_output_tps": 38.054325285929174, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "edd132c8-7aaa-4eb0-98e8-2b2d711f2cf9", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:51.817976+00:00", + "ttft_s": 0.09851625002920628, + "total_s": 3.6179742500535212, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3245283.0188679243, + "decode_tps": 35.649936286183205, + "request_tps": 35.3789140423281, + "group_elapsed_s": 3.6182677499600686, + "group_output_tps": 35.37604424145024, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "f0df217a-d72c-4b5c-b4a7-87c267716da0", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.436979+00:00", + "ttft_s": 0.11441224999725819, + "total_s": 2.9100067079998553, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 4138216.428719222, + "decode_tps": 33.47564745836445, + "request_tps": 32.989614675487815, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "9565d3c0-e753-4a7b-be43-115556beec5b", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.437104+00:00", + "ttft_s": 6.277438874996733, + "total_s": 9.582254499953706, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3865168.539325843, + "decode_tps": 37.881971614565146, + "request_tps": 13.358025504396528, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "42b7e290-9d92-4647-bbdb-3b3ba99f4c3e", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.436744+00:00", + "ttft_s": 3.0367828750167973, + "total_s": 6.193101375014521, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 771.3321549092078, + "decode_tps": 39.94659431201623, + "request_tps": 20.66815836672783, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "c22aaea0-7394-47e8-b6ab-25444fc113d1", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:49:55.437235+00:00", + "ttft_s": 9.661167457990814, + "total_s": 12.522873707988765, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 3966679.888932963, + "decode_tps": 32.71048444655768, + "request_tps": 7.665972063485584, + "group_elapsed_s": 12.524156374973245, + "group_output_tps": 35.77087243139417, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "84f401af-117d-4956-b939-fdbb1e1ceecf", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.961234+00:00", + "ttft_s": 3.091476584027987, + "total_s": 6.373854791978374, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 5375000.0, + "decode_tps": 38.1244834091544, + "request_tps": 20.08203891953902, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "b3c9f50e-fcf6-4ad6-b00f-9de925c9092a", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.961465+00:00", + "ttft_s": 9.849772959016263, + "total_s": 12.85736475000158, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 11076923.076923078, + "decode_tps": 41.56311628897905, + "request_tps": 9.955383742223248, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "5f528a81-943d-445a-a2f2-9b986c01c96e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.961041+00:00", + "ttft_s": 0.11284091597190127, + "total_s": 3.010363540961407, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 1476995.7905619969, + "decode_tps": 32.336156802028626, + "request_tps": 31.889836125686298, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "45991adb-aae4-4122-bb0d-21d34b549b46", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:07.960837+00:00", + "ttft_s": 6.452098417037632, + "total_s": 9.774627042002976, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7319148.936170213, + "decode_tps": 37.70880560635668, + "request_tps": 13.095128791100224, + "group_elapsed_s": 12.858689875050914, + "group_output_tps": 37.32884179214249, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "1cd3b269-0ed7-4470-9bf5-9aba6dbdda3b", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.819745+00:00", + "ttft_s": 9.799659290991258, + "total_s": 12.57556495803874, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2461538.461538462, + "decode_tps": 33.69646737394496, + "request_tps": 7.633851864335801, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "b2258258-f465-4131-bb56-cdead87e67c5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.820473+00:00", + "ttft_s": 2.9737471669795923, + "total_s": 6.33726870897226, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4755585.04755585, + "decode_tps": 37.255703188397476, + "request_tps": 20.19797579654126, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "9bb209a2-19a6-4172-a166-c29a75683a37", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.819981+00:00", + "ttft_s": 0.11236062500393018, + "total_s": 2.89358854200691, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 3661016.9491525427, + "decode_tps": 44.88156766233007, + "request_tps": 44.23572949014475, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "c74fbeef-16a3-40b2-ade6-0b24b3ee0147", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:20.820727+00:00", + "ttft_s": 6.417795290995855, + "total_s": 9.71814520796761, + "prompt_tokens": 20, + "output_tokens": 98, + "prompt_tps": 2608582.235554976, + "decode_tps": 29.056692998778793, + "request_tps": 10.084228821735737, + "group_elapsed_s": 12.576361375045963, + "group_output_tps": 35.781414558656905, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "844d6c49-9989-400d-bb5d-31f8490da904", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.396890+00:00", + "ttft_s": 10.48226895800326, + "total_s": 14.052842833043542, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 5083944.194845118, + "decode_tps": 35.11947732750843, + "request_tps": 9.108477303896379, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "213efda7-5da6-47a7-9517-f63ad082e62e", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.397263+00:00", + "ttft_s": 0.11672520800493658, + "total_s": 3.508996458025649, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 3630252.100840336, + "decode_tps": 36.93011583511385, + "request_tps": 36.47766577456727, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "d03d55d9-a506-44a9-887e-e0ab77392c8d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.397083+00:00", + "ttft_s": 3.606067124987021, + "total_s": 6.893670374993235, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 3356267.830172848, + "decode_tps": 28.527554720530254, + "request_tps": 13.925818145909567, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "55be3260-b3a0-422a-b918-a81261da6a84", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:33.396391+00:00", + "ttft_s": 6.9750645000021905, + "total_s": 10.400937125028577, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 6745098.039215687, + "decode_tps": 36.57719818551596, + "request_tps": 12.30658338391295, + "group_elapsed_s": 14.054668250028044, + "group_output_tps": 34.152353613828076, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "930b484d-3177-4ee9-9d72-5442ee8cd7c9", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451672+00:00", + "ttft_s": 0.11444733297685161, + "total_s": 3.540263083006721, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 4248957.431741286, + "decode_tps": 36.58630699945727, + "request_tps": 36.15550511327833, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "231f3cbb-61f2-4042-8a99-def6a8698a7d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451387+00:00", + "ttft_s": 10.67886250000447, + "total_s": 14.209862457995769, + "prompt_tokens": 20, + "output_tokens": 98, + "prompt_tps": 2840102.243680773, + "decode_tps": 27.132075592063398, + "request_tps": 6.896618478165229, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "32437705-bf67-438f-bad7-0bb10d0e2030", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451856+00:00", + "ttft_s": 3.622358249966055, + "total_s": 7.2201729999505915, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4777777.777777778, + "decode_tps": 34.87549075162609, + "request_tps": 17.72810706902396, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "a16a724c-155d-4e2d-ba28-3a8683af4d40", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:50:47.451136+00:00", + "ttft_s": 7.304396207968239, + "total_s": 10.594903749995865, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 17515407.071034707, + "decode_tps": 38.04511611184116, + "request_tps": 12.081280115456448, + "group_elapsed_s": 14.210740375041496, + "group_output_tps": 33.918007597024484, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "f6ab47e7-5bdf-4b5c-be3f-0e639727a2bb", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.661760+00:00", + "ttft_s": 10.42098737496417, + "total_s": 14.485269291966688, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2105263.1578947366, + "decode_tps": 23.198198961620538, + "request_tps": 6.627422525947802, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "8e015249-8b0c-4560-b93d-603afb968f5b", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.662046+00:00", + "ttft_s": 0.11227079102536663, + "total_s": 3.6567685829941183, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3571131.9657835728, + "decode_tps": 35.39152606628793, + "request_tps": 35.00358228717748, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "d2848890-f070-45de-8a2f-4bb2b8788a87", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.661862+00:00", + "ttft_s": 7.207546124991495, + "total_s": 10.33818608301226, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 9458749.343142407, + "decode_tps": 39.94848206301452, + "request_tps": 12.381282264819164, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "aba512eb-9825-4e49-9ab8-0c75678ff654", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:01.661952+00:00", + "ttft_s": 3.738764916022774, + "total_s": 7.12777191598434, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2652168.1474605487, + "decode_tps": 27.731643965337334, + "request_tps": 13.468444435590849, + "group_elapsed_s": 14.485774166998453, + "group_output_tps": 30.926893850149572, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "4c24ae09-884b-4b94-ae8e-37346f1408f2", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.148220+00:00", + "ttft_s": 10.924698917020578, + "total_s": 15.198258958000224, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7068880.486602006, + "decode_tps": 29.434480556585335, + "request_tps": 8.422017308280036, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "c94ceb91-b83d-4d47-a36d-71741993f366", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.148464+00:00", + "ttft_s": 0.12565479200566188, + "total_s": 3.500943167018704, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 4025044.7227191413, + "decode_tps": 37.03108618648992, + "request_tps": 36.56157609350765, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "0672967f-d135-44b0-b89f-7995f59b2d8c", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.148378+00:00", + "ttft_s": 7.600669999956153, + "total_s": 10.846751042001415, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2666666.6666666665, + "decode_tps": 28.898123213254255, + "request_tps": 8.850576511645125, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + }, + { + "run_id": "20260831T114928Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "8185344b-c228-457a-8c3d-6a7b5f803594", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T11:51:16.147977+00:00", + "ttft_s": 3.5925854169763625, + "total_s": 7.520279875025153, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4229785.559708833, + "decode_tps": 31.98591420303284, + "request_tps": 17.02064313126004, + "group_elapsed_s": 15.199551291007083, + "group_output_tps": 31.579879616840746, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T114928Z/summary.json b/benchmarks/inference/results/20260831T114928Z/summary.json new file mode 100644 index 0000000..157f7c4 --- /dev/null +++ b/benchmarks/inference/results/20260831T114928Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.18287834140937775, + "ttft_s_p50": 0.11936691601295024, + "ttft_s_p95": 0.3934195415815338, + "total_s_mean": 2.769790649821516, + "total_s_p50": 2.7544727920321748, + "total_s_p95": 3.450829833233729, + "prompt_tps_mean": 1201063.1140147406, + "prompt_tps_p50": 816.5668113066018, + "prompt_tps_p95": 3147950.5530253737, + "decode_tps_mean": 44.301349906835995, + "decode_tps_p50": 45.72630609321048, + "decode_tps_p95": 52.87492274071074, + "request_tps_mean": 42.02276635113332, + "request_tps_p50": 44.19526212378514, + "request_tps_p95": 46.37707708697445, + "aggregate_output_tps_mean": 42.01338924160581, + "aggregate_output_tps_p50": 44.18819214792146, + "aggregate_output_tps_p95": 46.368391249450006 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 5.296346539442311, + "ttft_s_p50": 5.078280103509314, + "ttft_s_p95": 10.691154320855276, + "total_s_mean": 8.731022270751419, + "total_s_p50": 8.619212541496381, + "total_s_p95": 14.520918775268365, + "prompt_tps_mean": 4873110.971654413, + "prompt_tps_p50": 3843030.8359358422, + "prompt_tps_p95": 9861582.229537027, + "decode_tps_mean": 33.61516109813652, + "decode_tps_p50": 34.997484039567254, + "decode_tps_p95": 40.1951363429803, + "request_tps_mean": 18.258369295759792, + "request_tps_p50": 12.924863350205007, + "request_tps_p95": 36.94528376333951, + "aggregate_output_tps_mean": 33.27170984729996, + "aggregate_output_tps_p50": 33.918007597024484, + "aggregate_output_tps_p95": 35.45560236969114 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T121855Z/metadata.json b/benchmarks/inference/results/20260831T121855Z/metadata.json new file mode 100644 index 0000000..92f8df9 --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/metadata.json @@ -0,0 +1,135 @@ +{ + "run_id": "20260831T121855Z", + "created_at": "2026-08-31T12:18:55.849116+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "ollama", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mxfp8", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mxfp8" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 14:18:56 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 14:18:56 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 14:18:56 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 14:18:58 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 14:18:59 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "ollama": { + "license": "\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n", + "modelfile": "# Modelfile generated by \"ollama show\"\nFROM qwen3.8:27b-mxfp8\nTEMPLATE {{ .Prompt }}\nRENDERER qwen3.8\nPARSER qwen3.5\nPARAMETER min_p 0\nPARAMETER presence_penalty 0\nPARAMETER repeat_penalty 1\nPARAMETER temperature 1\nPARAMETER top_k 20\nPARAMETER top_p 0.95\nLICENSE \"\"\"\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\"\"\"\n", + "parameters": "min_p 0\npresence_penalty 0\nrepeat_penalty 1\ntemperature 1\ntop_k 20\ntop_p 0.95", + "template": "{{ .Prompt }}", + "details": { + "parent_model": "", + "format": "safetensors", + "family": "qwen3_5", + "families": null, + "parameter_size": "27.8B", + "quantization_level": "mxfp8" + }, + "model_info": { + "general.architecture": "qwen3_5", + "general.parameter_count": 27781081984, + "qwen3_5.block_count": 64, + "qwen3_5.context_length": 262144, + "qwen3_5.embedding_length": 5120 + }, + "capabilities": [ + "completion", + "vision", + "tools", + "thinking" + ], + "modified_at": "2026-08-31T01:00:34.465875846+02:00", + "requires": "0.32.12" + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T12:22:34.295676+00:00", + "summary": [ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.296576766599901, + "ttft_s_p50": 0.25913220801157877, + "ttft_s_p95": 0.5828672000090591, + "total_s_mean": 5.365511283196975, + "total_s_p50": 5.271547083975747, + "total_s_p95": 5.7278931244160045, + "prompt_tps_mean": 1579505.471393669, + "prompt_tps_p50": 423.79232394227944, + "prompt_tps_p95": 4040897.1304585882, + "decode_tps_mean": 22.074969591997284, + "decode_tps_p50": 22.737809958094928, + "decode_tps_p95": 26.029319910172887, + "request_tps_mean": 21.326224707051917, + "request_tps_p50": 22.22186664913243, + "request_tps_p95": 23.99720340744444, + "aggregate_output_tps_mean": 21.3238226965937, + "aggregate_output_tps_p50": 22.217896609947385, + "aggregate_output_tps_p95": 23.99517949790225 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 9.275126714701765, + "ttft_s_p50": 9.394405500497669, + "ttft_s_p95": 18.583187756550615, + "total_s_mean": 15.170045258343453, + "total_s_p50": 14.90246445898083, + "total_s_p95": 24.08200175591046, + "prompt_tps_mean": 7067890.927047749, + "prompt_tps_p50": 6856543.23965746, + "prompt_tps_p95": 11845450.362447295, + "decode_tps_mean": 19.377795978907, + "decode_tps_p50": 19.039894932933024, + "decode_tps_p95": 22.784262218881533, + "request_tps_mean": 10.048677383124767, + "request_tps_p50": 7.0969869715907645, + "request_tps_p95": 21.739669344814565, + "aggregate_output_tps_mean": 19.35053924595546, + "aggregate_output_tps_p50": 20.179477104341757, + "aggregate_output_tps_p95": 20.562657512957074 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T121855Z/raw.csv b/benchmarks/inference/results/20260831T121855Z/raw.csv new file mode 100644 index 0000000..0656eb5 --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,1,0,0,556d36df-74f6-4dba-866f-134c99f385ac,short,106,128,2026-08-31T12:19:01.706610+00:00,8.133006000018213,13.098198042018339,20,95,80.39981477169073,18.60021913626931,7.252906063509266,13.099143458006438,7.252382593148428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,1,1,0,456b9efb-0afa-48e4-8093-041758c70c7d,medium,252,128,2026-08-31T12:19:14.805394+00:00,0.6389443330117501,6.043248832982499,43,128,91.99219534791703,23.084069916067204,21.180660194134138,6.043762709014118,21.178859290602404, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,0,0,78330ae3-a01a-4bf4-b1b6-e4ae861145a8,short,106,128,2026-08-31T12:19:20.848831+00:00,0.25939199997810647,5.077765540976543,20,95,194.8949698826367,19.20226982827757,18.709016639970706,5.077990833029617,18.708186588694836, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,1,0,f8373947-d690-42a4-b613-33f71746b2f9,medium,252,128,2026-08-31T12:19:25.927037+00:00,0.25913220801157877,5.760092166019604,43,128,423.79232394227944,22.737809958094928,22.22186664913243,5.761121416988317,22.217896609947385, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,2,0,9e12f0f4-50d6-444c-8752-4565aa55084b,code,267,128,2026-08-31T12:19:31.688098+00:00,0.6637360000167973,5.271547083975747,54,128,115.32507388679936,26.793290081008884,24.281296924974765,5.271936749981251,24.27950221528269, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,3,0,583bec1f-0983-4208-8635-439a9a0c5424,short,106,128,2026-08-31T12:19:36.960047+00:00,0.15107908402569592,5.119054667011369,20,95,4102564.1025641025,18.66803886577614,18.558113983858536,5.11973733396735,18.555639440663118, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,1,4,0,641937dd-533c-493e-b7b3-3fa2cc6e8879,medium,252,128,2026-08-31T12:19:42.079662+00:00,0.1495445409673266,5.59909695800161,43,128,3794229.2420365307,22.9734392268289,22.86082933732315,5.599817292008083,22.85788862838049, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,0,3dbdfe4c-7d43-4575-a1e6-6b74bd5a10a7,short,106,128,2026-08-31T12:19:47.680109+00:00,0.16311579098692164,4.8760897910106,20,95,1791151.7105498835,19.64848684540538,19.482824162741817,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,1,1ea04fa9-7f60-4936-ba53-6829df8e5714,medium,252,128,2026-08-31T12:19:47.680573+00:00,10.314829125010874,16.516400250024162,43,128,6370370.37037037,20.216323480613685,7.749872736331438,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,2,046681e1-e7be-4c58-8284-88f821e4a8a0,code,267,128,2026-08-31T12:19:47.680452+00:00,5.110771542007569,10.15410141699249,54,128,514.3377603686088,24.759396813702583,12.605743703307615,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,0,3,f496e50f-a2db-4fad-a2e1-2a7965fd862d,short,106,128,2026-08-31T12:19:47.680317+00:00,16.645131999976,22.03015966597013,20,95,6666666.666666666,17.25091779989875,4.312270153300159,22.031325709016528,20.24390206429885, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,0,cb3d0e9e-0a6f-4f36-aec9-4b20cbc8f87d,medium,252,128,2026-08-31T12:20:09.710819+00:00,17.595250790996943,23.510470708017237,43,128,19843101.06137517,21.163606353137514,5.444382700357892,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,1,fa3bb29e-7370-4255-877e-376076d7aa76,code,267,128,2026-08-31T12:20:09.711021+00:00,5.7327396660111845,11.353798499971163,54,128,9000000.0,22.18879858004337,11.273760054868431,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,2,93ec5b28-1a29-426a-96ff-b3b525fb3c71,short,106,128,2026-08-31T12:20:09.710927+00:00,0.1655573329771869,5.57563720800681,20,95,1563477.1732332706,17.170644798585553,17.038411298277563,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,warmup,4,1,3,dcac6e9a-3580-4bb1-8787-9cef1c64bfd5,medium,252,128,2026-08-31T12:20:09.711149+00:00,11.527663582994137,17.459117082995363,43,128,6331909.880724489,21.03978753266432,7.331413117371669,23.51121658302145,20.37325454038428, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,0,d42caf5b-fb6a-4e96-8e42-fd0b8fc1945b,short,106,128,2026-08-31T12:20:33.222727+00:00,6.285906166012865,10.864910541044082,20,95,8421052.631578946,20.156958877874334,8.743744335594945,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,1,87483860-d4c6-4f13-928c-48de78b6ef70,medium,252,128,2026-08-31T12:20:33.222336+00:00,0.17068937502335757,6.147177167003974,43,128,4669345.205776958,20.965937740191077,20.822565630133763,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,2,ee57b9cf-06bc-468b-bd1e-f936abf292df,code,267,128,2026-08-31T12:20:33.222603+00:00,16.19913299998734,21.62435016600648,54,128,11675675.675675677,22.963769927247082,5.919253018812849,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,0,3,13164aa4-836c-4761-8f99-95bfe18d33a1,short,106,128,2026-08-31T12:20:33.222066+00:00,11.00553745898651,16.043328458967153,20,95,10214504.596527068,18.355407643172423,5.9214645042626,21.625605207984336,20.623700271534293, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,0,51389c06-2a1c-4dcb-a847-5f0c914e298c,medium,252,128,2026-08-31T12:20:54.847154+00:00,0.16574266698444262,6.310333041998092,43,128,5402688.779997488,20.423563700405246,20.284190889466956,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,1,1952ea53-9c68-4536-8e9d-089790a8bdee,code,267,128,2026-08-31T12:20:54.847267+00:00,18.0774574590032,23.57407404197147,54,128,6513086.47931492,22.77481444475703,5.4296936444717945,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,2,d71abf03-e17c-42b0-b6bb-c57bf6961bc2,short,106,128,2026-08-31T12:20:54.847373+00:00,12.635340875014663,17.94654095900478,20,95,9229349.330872172,17.481471270830138,5.293499188339869,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,1,3,4e10d58d-5d6d-46e7-8254-46dd35cfeb9f,medium,252,128,2026-08-31T12:20:54.847461+00:00,6.461538125004154,12.505505541979801,43,128,3739130.434782609,20.730070992395465,10.235491845596812,23.574590582982637,20.318486478648204, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,0,63b32437-7995-44f4-b5ca-f0f52da2bda7,code,267,128,2026-08-31T12:21:18.421714+00:00,17.908610125014093,23.736398082983214,54,128,15071169.411108011,21.428265908373906,5.392562070812423,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,1,3b8fa85d-5574-4ac2-b33c-7208ef99a34f,short,106,128,2026-08-31T12:21:18.421631+00:00,6.036650207999628,11.081229583011009,20,95,4571428.571428572,18.37088402458498,8.573055840810985,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,2,76178335-d531-49f7-83d5-84a414d73ce5,medium,252,128,2026-08-31T12:21:18.421520+00:00,11.268355667009018,17.757357084017713,43,128,7533286.615276804,19.182821184166418,7.208279891786645,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,2,3,728d2a73-194b-41bb-9dda-216960415fcf,code,267,128,2026-08-31T12:21:18.421818+00:00,0.17636983300326392,5.888466249976773,54,128,7200000.0,21.89716979543998,21.737409125934093,23.736987709009554,20.179477104341757, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,0,dfcbb66f-3170-41e1-a4e1-a9b7449517ce,short,106,128,2026-08-31T12:21:42.158434+00:00,6.034018917009234,11.539304499980062,20,95,3200000.0,16.88203353475361,8.232731877399036,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,1,04f9942b-bc73-4665-9830-40b7a50beb96,medium,252,128,2026-08-31T12:21:42.158640+00:00,11.68165541702183,18.32316145801451,43,128,9921550.530687585,18.896968681699633,6.985694051394885,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,2,e2e252a5-26bf-4277-a744-cf7456cdfac7,code,267,128,2026-08-31T12:21:42.158549+00:00,0.17531787499319762,5.8762462079757825,54,128,5355548.943766736,21.95473700787165,21.782613503543573,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,3,3,61a1f98f-fb9a-49a8-8f00-04bb52b494cc,short,106,128,2026-08-31T12:21:42.158328+00:00,18.46465341700241,23.86128708301112,20,95,5782017.924255566,17.187946685051212,3.981344328556299,23.861715458042454,18.69102834556173, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,0,ae555d62-2572-4a62-8cd6-7ae714aace26,medium,252,128,2026-08-31T12:22:06.019994+00:00,20.83534020796651,28.275580540997908,43,128,7937973.047812442,16.887931275751107,4.526874340012493,28.276262458995916,16.940004029691313, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,1,c0147505-dd4a-4ad8-bb9d-0955114a8ee0,code,267,128,2026-08-31T12:22:06.020061+00:00,13.962619374971837,20.66803166596219,54,128,10125632.852053255,18.57306514141657,6.193139340443381,28.276262458995916,16.940004029691313, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,2,3a3d625f-5d64-42d0-9145-3056ea9864db,short,106,128,2026-08-31T12:22:06.019916+00:00,7.783273542008828,13.761600458994508,20,95,1927710.843373494,15.539155324201145,6.903266831722942,28.276262458995916,16.940004029691313, +20260831T121855Z,ollama,qwen3.8:27b-mxfp8,measured,4,4,3,8ea78898-3c59-49f2-9734-f01bb668d810,medium,252,128,2026-08-31T12:22:06.019812+00:00,0.17432458401890472,7.616022333968431,43,128,2866666.6666666665,16.90294641795698,16.806673403399,28.276262458995916,16.940004029691313, diff --git a/benchmarks/inference/results/20260831T121855Z/raw.json b/benchmarks/inference/results/20260831T121855Z/raw.json new file mode 100644 index 0000000..9d4472d --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "556d36df-74f6-4dba-866f-134c99f385ac", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:01.706610+00:00", + "ttft_s": 8.133006000018213, + "total_s": 13.098198042018339, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 80.39981477169073, + "decode_tps": 18.60021913626931, + "request_tps": 7.252906063509266, + "group_elapsed_s": 13.099143458006438, + "group_output_tps": 7.252382593148428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "456b9efb-0afa-48e4-8093-041758c70c7d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:14.805394+00:00", + "ttft_s": 0.6389443330117501, + "total_s": 6.043248832982499, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 91.99219534791703, + "decode_tps": 23.084069916067204, + "request_tps": 21.180660194134138, + "group_elapsed_s": 6.043762709014118, + "group_output_tps": 21.178859290602404, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "78330ae3-a01a-4bf4-b1b6-e4ae861145a8", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:20.848831+00:00", + "ttft_s": 0.25939199997810647, + "total_s": 5.077765540976543, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 194.8949698826367, + "decode_tps": 19.20226982827757, + "request_tps": 18.709016639970706, + "group_elapsed_s": 5.077990833029617, + "group_output_tps": 18.708186588694836, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "f8373947-d690-42a4-b613-33f71746b2f9", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:25.927037+00:00", + "ttft_s": 0.25913220801157877, + "total_s": 5.760092166019604, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 423.79232394227944, + "decode_tps": 22.737809958094928, + "request_tps": 22.22186664913243, + "group_elapsed_s": 5.761121416988317, + "group_output_tps": 22.217896609947385, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "9e12f0f4-50d6-444c-8752-4565aa55084b", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:31.688098+00:00", + "ttft_s": 0.6637360000167973, + "total_s": 5.271547083975747, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 115.32507388679936, + "decode_tps": 26.793290081008884, + "request_tps": 24.281296924974765, + "group_elapsed_s": 5.271936749981251, + "group_output_tps": 24.27950221528269, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "583bec1f-0983-4208-8635-439a9a0c5424", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:36.960047+00:00", + "ttft_s": 0.15107908402569592, + "total_s": 5.119054667011369, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 4102564.1025641025, + "decode_tps": 18.66803886577614, + "request_tps": 18.558113983858536, + "group_elapsed_s": 5.11973733396735, + "group_output_tps": 18.555639440663118, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "641937dd-533c-493e-b7b3-3fa2cc6e8879", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:42.079662+00:00", + "ttft_s": 0.1495445409673266, + "total_s": 5.59909695800161, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3794229.2420365307, + "decode_tps": 22.9734392268289, + "request_tps": 22.86082933732315, + "group_elapsed_s": 5.599817292008083, + "group_output_tps": 22.85788862838049, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "3dbdfe4c-7d43-4575-a1e6-6b74bd5a10a7", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680109+00:00", + "ttft_s": 0.16311579098692164, + "total_s": 4.8760897910106, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 1791151.7105498835, + "decode_tps": 19.64848684540538, + "request_tps": 19.482824162741817, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "1ea04fa9-7f60-4936-ba53-6829df8e5714", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680573+00:00", + "ttft_s": 10.314829125010874, + "total_s": 16.516400250024162, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 6370370.37037037, + "decode_tps": 20.216323480613685, + "request_tps": 7.749872736331438, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "046681e1-e7be-4c58-8284-88f821e4a8a0", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680452+00:00", + "ttft_s": 5.110771542007569, + "total_s": 10.15410141699249, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 514.3377603686088, + "decode_tps": 24.759396813702583, + "request_tps": 12.605743703307615, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "f496e50f-a2db-4fad-a2e1-2a7965fd862d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:19:47.680317+00:00", + "ttft_s": 16.645131999976, + "total_s": 22.03015966597013, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 6666666.666666666, + "decode_tps": 17.25091779989875, + "request_tps": 4.312270153300159, + "group_elapsed_s": 22.031325709016528, + "group_output_tps": 20.24390206429885, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "cb3d0e9e-0a6f-4f36-aec9-4b20cbc8f87d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.710819+00:00", + "ttft_s": 17.595250790996943, + "total_s": 23.510470708017237, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 19843101.06137517, + "decode_tps": 21.163606353137514, + "request_tps": 5.444382700357892, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "fa3bb29e-7370-4255-877e-376076d7aa76", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.711021+00:00", + "ttft_s": 5.7327396660111845, + "total_s": 11.353798499971163, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 9000000.0, + "decode_tps": 22.18879858004337, + "request_tps": 11.273760054868431, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "93ec5b28-1a29-426a-96ff-b3b525fb3c71", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.710927+00:00", + "ttft_s": 0.1655573329771869, + "total_s": 5.57563720800681, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 1563477.1732332706, + "decode_tps": 17.170644798585553, + "request_tps": 17.038411298277563, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "dcac6e9a-3580-4bb1-8787-9cef1c64bfd5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:09.711149+00:00", + "ttft_s": 11.527663582994137, + "total_s": 17.459117082995363, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 6331909.880724489, + "decode_tps": 21.03978753266432, + "request_tps": 7.331413117371669, + "group_elapsed_s": 23.51121658302145, + "group_output_tps": 20.37325454038428, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "d42caf5b-fb6a-4e96-8e42-fd0b8fc1945b", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222727+00:00", + "ttft_s": 6.285906166012865, + "total_s": 10.864910541044082, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 8421052.631578946, + "decode_tps": 20.156958877874334, + "request_tps": 8.743744335594945, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "87483860-d4c6-4f13-928c-48de78b6ef70", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222336+00:00", + "ttft_s": 0.17068937502335757, + "total_s": 6.147177167003974, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 4669345.205776958, + "decode_tps": 20.965937740191077, + "request_tps": 20.822565630133763, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "ee57b9cf-06bc-468b-bd1e-f936abf292df", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222603+00:00", + "ttft_s": 16.19913299998734, + "total_s": 21.62435016600648, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 11675675.675675677, + "decode_tps": 22.963769927247082, + "request_tps": 5.919253018812849, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "13164aa4-836c-4761-8f99-95bfe18d33a1", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:33.222066+00:00", + "ttft_s": 11.00553745898651, + "total_s": 16.043328458967153, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 10214504.596527068, + "decode_tps": 18.355407643172423, + "request_tps": 5.9214645042626, + "group_elapsed_s": 21.625605207984336, + "group_output_tps": 20.623700271534293, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "51389c06-2a1c-4dcb-a847-5f0c914e298c", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847154+00:00", + "ttft_s": 0.16574266698444262, + "total_s": 6.310333041998092, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 5402688.779997488, + "decode_tps": 20.423563700405246, + "request_tps": 20.284190889466956, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "1952ea53-9c68-4536-8e9d-089790a8bdee", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847267+00:00", + "ttft_s": 18.0774574590032, + "total_s": 23.57407404197147, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 6513086.47931492, + "decode_tps": 22.77481444475703, + "request_tps": 5.4296936444717945, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "d71abf03-e17c-42b0-b6bb-c57bf6961bc2", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847373+00:00", + "ttft_s": 12.635340875014663, + "total_s": 17.94654095900478, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 9229349.330872172, + "decode_tps": 17.481471270830138, + "request_tps": 5.293499188339869, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "4e10d58d-5d6d-46e7-8254-46dd35cfeb9f", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:20:54.847461+00:00", + "ttft_s": 6.461538125004154, + "total_s": 12.505505541979801, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3739130.434782609, + "decode_tps": 20.730070992395465, + "request_tps": 10.235491845596812, + "group_elapsed_s": 23.574590582982637, + "group_output_tps": 20.318486478648204, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "63b32437-7995-44f4-b5ca-f0f52da2bda7", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421714+00:00", + "ttft_s": 17.908610125014093, + "total_s": 23.736398082983214, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 15071169.411108011, + "decode_tps": 21.428265908373906, + "request_tps": 5.392562070812423, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "3b8fa85d-5574-4ac2-b33c-7208ef99a34f", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421631+00:00", + "ttft_s": 6.036650207999628, + "total_s": 11.081229583011009, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 4571428.571428572, + "decode_tps": 18.37088402458498, + "request_tps": 8.573055840810985, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "76178335-d531-49f7-83d5-84a414d73ce5", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421520+00:00", + "ttft_s": 11.268355667009018, + "total_s": 17.757357084017713, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7533286.615276804, + "decode_tps": 19.182821184166418, + "request_tps": 7.208279891786645, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "728d2a73-194b-41bb-9dda-216960415fcf", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:18.421818+00:00", + "ttft_s": 0.17636983300326392, + "total_s": 5.888466249976773, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 7200000.0, + "decode_tps": 21.89716979543998, + "request_tps": 21.737409125934093, + "group_elapsed_s": 23.736987709009554, + "group_output_tps": 20.179477104341757, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "dfcbb66f-3170-41e1-a4e1-a9b7449517ce", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158434+00:00", + "ttft_s": 6.034018917009234, + "total_s": 11.539304499980062, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 3200000.0, + "decode_tps": 16.88203353475361, + "request_tps": 8.232731877399036, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "04f9942b-bc73-4665-9830-40b7a50beb96", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158640+00:00", + "ttft_s": 11.68165541702183, + "total_s": 18.32316145801451, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 9921550.530687585, + "decode_tps": 18.896968681699633, + "request_tps": 6.985694051394885, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "e2e252a5-26bf-4277-a744-cf7456cdfac7", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158549+00:00", + "ttft_s": 0.17531787499319762, + "total_s": 5.8762462079757825, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 5355548.943766736, + "decode_tps": 21.95473700787165, + "request_tps": 21.782613503543573, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "61a1f98f-fb9a-49a8-8f00-04bb52b494cc", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:21:42.158328+00:00", + "ttft_s": 18.46465341700241, + "total_s": 23.86128708301112, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 5782017.924255566, + "decode_tps": 17.187946685051212, + "request_tps": 3.981344328556299, + "group_elapsed_s": 23.861715458042454, + "group_output_tps": 18.69102834556173, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "ae555d62-2572-4a62-8cd6-7ae714aace26", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.019994+00:00", + "ttft_s": 20.83534020796651, + "total_s": 28.275580540997908, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 7937973.047812442, + "decode_tps": 16.887931275751107, + "request_tps": 4.526874340012493, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "c0147505-dd4a-4ad8-bb9d-0955114a8ee0", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.020061+00:00", + "ttft_s": 13.962619374971837, + "total_s": 20.66803166596219, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 10125632.852053255, + "decode_tps": 18.57306514141657, + "request_tps": 6.193139340443381, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "3a3d625f-5d64-42d0-9145-3056ea9864db", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.019916+00:00", + "ttft_s": 7.783273542008828, + "total_s": 13.761600458994508, + "prompt_tokens": 20, + "output_tokens": 95, + "prompt_tps": 1927710.843373494, + "decode_tps": 15.539155324201145, + "request_tps": 6.903266831722942, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + }, + { + "run_id": "20260831T121855Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "8ea78898-3c59-49f2-9734-f01bb668d810", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T12:22:06.019812+00:00", + "ttft_s": 0.17432458401890472, + "total_s": 7.616022333968431, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 2866666.6666666665, + "decode_tps": 16.90294641795698, + "request_tps": 16.806673403399, + "group_elapsed_s": 28.276262458995916, + "group_output_tps": 16.940004029691313, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T121855Z/summary.json b/benchmarks/inference/results/20260831T121855Z/summary.json new file mode 100644 index 0000000..d97729d --- /dev/null +++ b/benchmarks/inference/results/20260831T121855Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.296576766599901, + "ttft_s_p50": 0.25913220801157877, + "ttft_s_p95": 0.5828672000090591, + "total_s_mean": 5.365511283196975, + "total_s_p50": 5.271547083975747, + "total_s_p95": 5.7278931244160045, + "prompt_tps_mean": 1579505.471393669, + "prompt_tps_p50": 423.79232394227944, + "prompt_tps_p95": 4040897.1304585882, + "decode_tps_mean": 22.074969591997284, + "decode_tps_p50": 22.737809958094928, + "decode_tps_p95": 26.029319910172887, + "request_tps_mean": 21.326224707051917, + "request_tps_p50": 22.22186664913243, + "request_tps_p95": 23.99720340744444, + "aggregate_output_tps_mean": 21.3238226965937, + "aggregate_output_tps_p50": 22.217896609947385, + "aggregate_output_tps_p95": 23.99517949790225 + }, + { + "runtime": "ollama", + "model": "qwen3.8:27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 9.275126714701765, + "ttft_s_p50": 9.394405500497669, + "ttft_s_p95": 18.583187756550615, + "total_s_mean": 15.170045258343453, + "total_s_p50": 14.90246445898083, + "total_s_p95": 24.08200175591046, + "prompt_tps_mean": 7067890.927047749, + "prompt_tps_p50": 6856543.23965746, + "prompt_tps_p95": 11845450.362447295, + "decode_tps_mean": 19.377795978907, + "decode_tps_p50": 19.039894932933024, + "decode_tps_p95": 22.784262218881533, + "request_tps_mean": 10.048677383124767, + "request_tps_p50": 7.0969869715907645, + "request_tps_p95": 21.739669344814565, + "aggregate_output_tps_mean": 19.35053924595546, + "aggregate_output_tps_p50": 20.179477104341757, + "aggregate_output_tps_p95": 20.562657512957074 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T132015Z/metadata.json b/benchmarks/inference/results/20260831T132015Z/metadata.json new file mode 100644 index 0000000..6e2ae60 --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/metadata.json @@ -0,0 +1,136 @@ +{ + "run_id": "20260831T132015Z", + "created_at": "2026-08-31T13:20:15.702115+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "vllm", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-mxfp8", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1, + 4 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "vllm-metal", + "kind": "openai", + "base_url": "http://127.0.0.1:8001", + "model": "qwen3.8-27b-mxfp8" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 15:20:16 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 15:20:16 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 15:20:16 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 15:20:18 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 15:20:19 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "vllm-metal": { + "object": "list", + "data": [ + { + "id": "qwen3.8-27b-mxfp8", + "object": "model", + "created": 1788182421, + "owned_by": "vllm", + "root": "mlx-community/Qwen3.8-27B-mxfp8", + "parent": null, + "max_model_len": 8192, + "permission": [ + { + "id": "modelperm-a3c90f9b6d588379", + "object": "model_permission", + "created": 1788182421, + "allow_create_engine": false, + "allow_sampling": true, + "allow_logprobs": true, + "allow_search_indices": false, + "allow_view": true, + "allow_fine_tuning": false, + "organization": "*", + "group": null, + "is_blocking": false + } + ] + } + ] + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T13:22:48.218681+00:00", + "summary": [ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.37155269999057056, + "ttft_s_p50": 0.4171057919738814, + "ttft_s_p95": 0.4531440335791558, + "total_s_mean": 7.930883799993898, + "total_s_p50": 8.838838333962485, + "total_s_p95": 8.998963149765041, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.079897408512505, + "decode_tps_p50": 15.126147490283795, + "decode_tps_p95": 15.196162811046243, + "request_tps_mean": 14.380896425252896, + "request_tps_p50": 14.386982882152838, + "request_tps_p95": 14.53502856074605, + "aggregate_output_tps_mean": 14.380548747737752, + "aggregate_output_tps_p50": 14.386712701914696, + "aggregate_output_tps_p95": 14.534638049200744 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.255672066603438, + "ttft_s_p50": 1.3144360205042176, + "ttft_s_p95": 1.419064954231726, + "total_s_mean": 12.251293779097614, + "total_s_p50": 12.702340687508695, + "total_s_p95": 13.641910566360457, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 10.500220268362742, + "decode_tps_p50": 10.485677148644122, + "decode_tps_p95": 11.071162546140778, + "request_tps_mean": 9.417789343799708, + "request_tps_p50": 9.430282582501674, + "request_tps_p95": 10.077752113288362, + "aggregate_output_tps_mean": 35.09087333775396, + "aggregate_output_tps_p50": 34.96408475660248, + "aggregate_output_tps_p95": 35.52678209465127 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T132015Z/raw.csv b/benchmarks/inference/results/20260831T132015Z/raw.csv new file mode 100644 index 0000000..8d2ffed --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/raw.csv @@ -0,0 +1,36 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,1,0,0,4c059f2d-caa2-44ae-ab1d-539d608c86d8,short,106,128,2026-08-31T13:20:21.117908+00:00,0.3135264579905197,6.390581874991767,20,93,,15.303464197450301,14.552665440988472,6.39150691695977,14.550559235604652, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,1,1,0,1bf28507-8f2f-4127-ba2f-2f5637e722bf,medium,252,128,2026-08-31T13:20:27.508796+00:00,0.3822946660220623,8.7317384579801,43,128,,15.330362499509992,14.659165596401756,8.731932666967623,14.658839558419446, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,0,0,81b82bc5-db99-4c10-9ec1-1f684485db59,short,106,128,2026-08-31T13:20:36.240817+00:00,0.2682818329776637,6.39245533302892,20,93,,15.185722612075187,14.548400443172758,6.392639332974795,14.547981695179217, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,1,0,1de32693-20bd-450c-8341-17773d6c57b2,medium,252,128,2026-08-31T13:20:42.633525+00:00,0.4171057919738814,8.838838333962485,43,128,,15.198772860789006,14.481541031039212,8.839007750037126,14.481263465286855, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,2,0,e821a6d4-8b00-477b-a66f-3579d33c583b,code,267,128,2026-08-31T13:20:51.472629+00:00,0.4347635000012815,8.896931416995358,54,128,,15.126147490283795,14.386982882152838,8.897098499990534,14.386712701914696, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,3,0,6415410d-9d1c-4521-b2e2-04c40ce2fd72,short,106,128,2026-08-31T13:21:00.369851+00:00,0.2798732080264017,6.501722833025269,20,93,,14.947323642528074,14.303901041060966,6.501933082996402,14.303438502498576, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,1,4,0,44eda129-4834-4a55-a14a-33a4a510922d,medium,252,128,2026-08-31T13:21:06.871859+00:00,0.4577391669736244,9.024471082957461,43,128,,14.94152043688646,14.183656728838715,9.024667916994076,14.183347373809413, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,0,a696baea-c3a4-422a-9d29-c4fff0e6d70e,short,106,128,2026-08-31T13:21:15.897040+00:00,1.1126187500194646,9.980102250003256,20,93,,10.48775562989996,9.318541801510065,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,1,2a75391e-8c1c-4244-8cea-f8f97ce3c40d,medium,252,128,2026-08-31T13:21:15.896746+00:00,1.1128792079980485,12.400287791970186,43,128,,11.340069693388886,10.322341073639151,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,2,21360267-b5fe-4bd2-8b87-d5bbdf4a9c49,code,267,128,2026-08-31T13:21:15.896944+00:00,1.1127408749889582,12.400139749981463,54,128,,11.340079447673899,10.322464309339042,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,0,3,106f9109-b093-475b-85c2-dc0847a3f3e3,short,106,128,2026-08-31T13:21:15.896863+00:00,1.0121074999915436,9.893255082948599,20,93,,10.471619701317342,9.400343892910335,12.400747582956683,35.64301241059653, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,0,fd6e44d9-a975-4760-853a-c85d6547b8f2,medium,252,128,2026-08-31T13:21:28.297798+00:00,1.4682920000050217,13.96469266602071,43,128,,10.242949423676817,9.165973291446168,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,1,a5541448-562a-4c84-aeff-3e2e5b8aba34,code,267,128,2026-08-31T13:21:28.297877+00:00,1.4681764579727314,13.9645455830032,54,128,,10.242975277003744,9.166069833005798,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,2,bb897f87-b4a0-4483-9946-4f6483212386,short,106,128,2026-08-31T13:21:28.297713+00:00,1.3568928329623304,13.431885249970946,20,122,,10.103526013660515,9.082864968658356,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,warmup,4,1,3,576c3ce6-5566-4a17-b311-3c2f8f3a5a17,medium,252,128,2026-08-31T13:21:28.297622+00:00,1.4684889169875532,13.964902083971538,43,128,,10.242939176993687,9.16583583832745,13.965293916000519,36.232678169433896, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,0,75886439-869f-49d6-881d-4bc7cf650b04,short,106,128,2026-08-31T13:21:42.263079+00:00,1.1142227500095032,10.27476887498051,20,93,,10.152233145411333,9.051298489687577,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,1,2ecba00e-62dd-4ea7-a1e1-c7b0c12aae0e,medium,252,128,2026-08-31T13:21:42.263219+00:00,1.1140572919975966,12.70125433296198,43,128,,11.046675011003936,10.077744815157159,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,2,c1df76bd-2c64-4e34-a5be-10d74b64ce17,code,267,128,2026-08-31T13:21:42.263452+00:00,1.1138802919886075,12.701070374983829,54,128,,11.046681644400257,10.077890777781237,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,0,3,1bf72731-e2e3-4f8d-bf42-627b9fce292a,short,106,128,2026-08-31T13:21:42.263323+00:00,1.0123424580087885,10.185485332971439,20,93,,10.138291888359873,9.130640019572722,12.701848333003,34.79808516147676, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,0,a04682c5-d585-4466-8fad-17ed2b9ba80b,medium,252,128,2026-08-31T13:21:54.962444+00:00,1.4147373330197297,13.505381957977079,43,128,,10.586697729563905,9.477703066694506,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,1,bf6552cc-7c18-4617-8260-b6a63ff65b3c,code,267,128,2026-08-31T13:21:54.962650+00:00,1.4145112080150284,13.505133582977578,54,128,,10.586717211933143,9.477877372596776,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,2,36abe2cb-3ddf-4311-853a-4151aef2b328,short,106,128,2026-08-31T13:21:54.962724+00:00,1.4144840000080876,10.538147750019561,20,93,,10.193273508121454,8.825080289828673,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,1,3,110efa2a-5fb4-48e9-835b-3092cd0df4fb,medium,252,128,2026-08-31T13:21:54.962552+00:00,1.3138721249997616,13.505327666993253,43,128,,10.49915652475652,9.477741166756687,13.505815375014208,35.31811939932565, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,0,4d78c4f5-e05f-4ffe-afb6-748b8db712f6,code,267,128,2026-08-31T13:22:08.467061+00:00,1.2189033330068924,13.40631824999582,54,128,,10.502637423262865,9.547736941127733,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,1,9beb5379-ecf9-45d3-8725-13dd100b245d,short,106,128,2026-08-31T13:22:08.467363+00:00,1.3166250829817727,10.428931292030029,20,93,,10.205978362278222,8.917500498931489,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,2,e863870c-69a1-44bd-9d57-7d28f9a30c17,medium,252,128,2026-08-31T13:22:08.467157+00:00,1.3168415420223027,13.406341000052635,43,128,,10.58770054495327,9.547720738976986,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,2,3,21ffaa25-b50a-4b5f-b562-4a7810b56506,code,267,128,2026-08-31T13:22:08.467234+00:00,1.316743416013196,13.406194041017443,54,128,,10.587743311946818,9.547825401331103,13.406804582977202,35.57894776848267, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,0,00728b55-60a4-46d3-8000-146e38d0b3a6,short,106,128,2026-08-31T13:22:21.873446+00:00,1.0375588749884628,10.190524249977898,20,93,,10.160641517788692,9.126125184403708,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,1,531d5ceb-d88b-49ab-ad8d-7bea678c3bb8,medium,252,128,2026-08-31T13:22:21.873549+00:00,1.140847125032451,12.702382000046782,43,128,,11.07119438584415,10.076850152949941,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,2,a4b2f7bb-ab5c-405b-9514-d64bf34153ea,code,267,128,2026-08-31T13:22:21.873705+00:00,1.1407294999808073,12.702299374970607,54,128,,11.071160870366917,10.076915700177802,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,3,3,1b77ace8-b3ef-47f2-a2cd-06b6f9400a6e,short,106,128,2026-08-31T13:22:21.873633+00:00,1.1407265829620883,10.277835665969178,20,93,,10.178274020276117,9.048597683647658,12.702927250007633,34.795129602882234, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,0,1143b997-b26b-4ca6-a4ad-dd1014020849,medium,252,128,2026-08-31T13:22:34.576247+00:00,1.3149999160086736,13.642209500016179,43,128,,10.383534012925246,9.382644358294614,13.642570749972947,34.96408475660248, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,1,907e97b8-fec4-47ff-b971-45716a15b6fb,code,267,128,2026-08-31T13:22:34.576465+00:00,1.419054625032004,13.641894833010156,54,128,,10.472197772531725,9.38286078047386,13.642570749972947,34.96408475660248, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,2,d5875acf-e267-4393-84c1-a01dac48c4a5,short,106,128,2026-08-31T13:22:34.576349+00:00,1.4192612090264447,10.662482584011741,20,93,,10.061427312742246,8.722171339295066,13.642570749972947,34.96408475660248, +20260831T132015Z,vllm-metal,qwen3.8-27b-mxfp8,measured,4,4,3,d2828df9-67d3-44b0-9809-bdb8c75f5282,medium,252,128,2026-08-31T13:22:34.576542+00:00,1.4190426669665612,13.641892916988581,43,128,,10.47218916878814,9.382862098308841,13.642570749972947,34.96408475660248, diff --git a/benchmarks/inference/results/20260831T132015Z/raw.json b/benchmarks/inference/results/20260831T132015Z/raw.json new file mode 100644 index 0000000..252c974 --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/raw.json @@ -0,0 +1,842 @@ +[ + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "4c059f2d-caa2-44ae-ab1d-539d608c86d8", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:21.117908+00:00", + "ttft_s": 0.3135264579905197, + "total_s": 6.390581874991767, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 15.303464197450301, + "request_tps": 14.552665440988472, + "group_elapsed_s": 6.39150691695977, + "group_output_tps": 14.550559235604652, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "1bf28507-8f2f-4127-ba2f-2f5637e722bf", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:27.508796+00:00", + "ttft_s": 0.3822946660220623, + "total_s": 8.7317384579801, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.330362499509992, + "request_tps": 14.659165596401756, + "group_elapsed_s": 8.731932666967623, + "group_output_tps": 14.658839558419446, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "81b82bc5-db99-4c10-9ec1-1f684485db59", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:36.240817+00:00", + "ttft_s": 0.2682818329776637, + "total_s": 6.39245533302892, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 15.185722612075187, + "request_tps": 14.548400443172758, + "group_elapsed_s": 6.392639332974795, + "group_output_tps": 14.547981695179217, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "1de32693-20bd-450c-8341-17773d6c57b2", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:42.633525+00:00", + "ttft_s": 0.4171057919738814, + "total_s": 8.838838333962485, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.198772860789006, + "request_tps": 14.481541031039212, + "group_elapsed_s": 8.839007750037126, + "group_output_tps": 14.481263465286855, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "e821a6d4-8b00-477b-a66f-3579d33c583b", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:20:51.472629+00:00", + "ttft_s": 0.4347635000012815, + "total_s": 8.896931416995358, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 15.126147490283795, + "request_tps": 14.386982882152838, + "group_elapsed_s": 8.897098499990534, + "group_output_tps": 14.386712701914696, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "6415410d-9d1c-4521-b2e2-04c40ce2fd72", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:00.369851+00:00", + "ttft_s": 0.2798732080264017, + "total_s": 6.501722833025269, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 14.947323642528074, + "request_tps": 14.303901041060966, + "group_elapsed_s": 6.501933082996402, + "group_output_tps": 14.303438502498576, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "44eda129-4834-4a55-a14a-33a4a510922d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:06.871859+00:00", + "ttft_s": 0.4577391669736244, + "total_s": 9.024471082957461, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 14.94152043688646, + "request_tps": 14.183656728838715, + "group_elapsed_s": 9.024667916994076, + "group_output_tps": 14.183347373809413, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "a696baea-c3a4-422a-9d29-c4fff0e6d70e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.897040+00:00", + "ttft_s": 1.1126187500194646, + "total_s": 9.980102250003256, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.48775562989996, + "request_tps": 9.318541801510065, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "2a75391e-8c1c-4244-8cea-f8f97ce3c40d", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.896746+00:00", + "ttft_s": 1.1128792079980485, + "total_s": 12.400287791970186, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.340069693388886, + "request_tps": 10.322341073639151, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "21360267-b5fe-4bd2-8b87-d5bbdf4a9c49", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.896944+00:00", + "ttft_s": 1.1127408749889582, + "total_s": 12.400139749981463, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.340079447673899, + "request_tps": 10.322464309339042, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "106f9109-b093-475b-85c2-dc0847a3f3e3", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:15.896863+00:00", + "ttft_s": 1.0121074999915436, + "total_s": 9.893255082948599, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.471619701317342, + "request_tps": 9.400343892910335, + "group_elapsed_s": 12.400747582956683, + "group_output_tps": 35.64301241059653, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "fd6e44d9-a975-4760-853a-c85d6547b8f2", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297798+00:00", + "ttft_s": 1.4682920000050217, + "total_s": 13.96469266602071, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.242949423676817, + "request_tps": 9.165973291446168, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "a5541448-562a-4c84-aeff-3e2e5b8aba34", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297877+00:00", + "ttft_s": 1.4681764579727314, + "total_s": 13.9645455830032, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.242975277003744, + "request_tps": 9.166069833005798, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "bb897f87-b4a0-4483-9946-4f6483212386", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297713+00:00", + "ttft_s": 1.3568928329623304, + "total_s": 13.431885249970946, + "prompt_tokens": 20, + "output_tokens": 122, + "prompt_tps": null, + "decode_tps": 10.103526013660515, + "request_tps": 9.082864968658356, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "warmup", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "576c3ce6-5566-4a17-b311-3c2f8f3a5a17", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:28.297622+00:00", + "ttft_s": 1.4684889169875532, + "total_s": 13.964902083971538, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.242939176993687, + "request_tps": 9.16583583832745, + "group_elapsed_s": 13.965293916000519, + "group_output_tps": 36.232678169433896, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 0, + "request_id": "75886439-869f-49d6-881d-4bc7cf650b04", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263079+00:00", + "ttft_s": 1.1142227500095032, + "total_s": 10.27476887498051, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.152233145411333, + "request_tps": 9.051298489687577, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 1, + "request_id": "2ecba00e-62dd-4ea7-a1e1-c7b0c12aae0e", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263219+00:00", + "ttft_s": 1.1140572919975966, + "total_s": 12.70125433296198, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.046675011003936, + "request_tps": 10.077744815157159, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 2, + "request_id": "c1df76bd-2c64-4e34-a5be-10d74b64ce17", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263452+00:00", + "ttft_s": 1.1138802919886075, + "total_s": 12.701070374983829, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.046681644400257, + "request_tps": 10.077890777781237, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 0, + "slot": 3, + "request_id": "1bf72731-e2e3-4f8d-bf42-627b9fce292a", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:42.263323+00:00", + "ttft_s": 1.0123424580087885, + "total_s": 10.185485332971439, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.138291888359873, + "request_tps": 9.130640019572722, + "group_elapsed_s": 12.701848333003, + "group_output_tps": 34.79808516147676, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 0, + "request_id": "a04682c5-d585-4466-8fad-17ed2b9ba80b", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962444+00:00", + "ttft_s": 1.4147373330197297, + "total_s": 13.505381957977079, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.586697729563905, + "request_tps": 9.477703066694506, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 1, + "request_id": "bf6552cc-7c18-4617-8260-b6a63ff65b3c", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962650+00:00", + "ttft_s": 1.4145112080150284, + "total_s": 13.505133582977578, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.586717211933143, + "request_tps": 9.477877372596776, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 2, + "request_id": "36abe2cb-3ddf-4311-853a-4151aef2b328", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962724+00:00", + "ttft_s": 1.4144840000080876, + "total_s": 10.538147750019561, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.193273508121454, + "request_tps": 8.825080289828673, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 1, + "slot": 3, + "request_id": "110efa2a-5fb4-48e9-835b-3092cd0df4fb", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:21:54.962552+00:00", + "ttft_s": 1.3138721249997616, + "total_s": 13.505327666993253, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.49915652475652, + "request_tps": 9.477741166756687, + "group_elapsed_s": 13.505815375014208, + "group_output_tps": 35.31811939932565, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 0, + "request_id": "4d78c4f5-e05f-4ffe-afb6-748b8db712f6", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467061+00:00", + "ttft_s": 1.2189033330068924, + "total_s": 13.40631824999582, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.502637423262865, + "request_tps": 9.547736941127733, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 1, + "request_id": "9beb5379-ecf9-45d3-8725-13dd100b245d", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467363+00:00", + "ttft_s": 1.3166250829817727, + "total_s": 10.428931292030029, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.205978362278222, + "request_tps": 8.917500498931489, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 2, + "request_id": "e863870c-69a1-44bd-9d57-7d28f9a30c17", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467157+00:00", + "ttft_s": 1.3168415420223027, + "total_s": 13.406341000052635, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.58770054495327, + "request_tps": 9.547720738976986, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 2, + "slot": 3, + "request_id": "21ffaa25-b50a-4b5f-b562-4a7810b56506", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:08.467234+00:00", + "ttft_s": 1.316743416013196, + "total_s": 13.406194041017443, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.587743311946818, + "request_tps": 9.547825401331103, + "group_elapsed_s": 13.406804582977202, + "group_output_tps": 35.57894776848267, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 0, + "request_id": "00728b55-60a4-46d3-8000-146e38d0b3a6", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873446+00:00", + "ttft_s": 1.0375588749884628, + "total_s": 10.190524249977898, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.160641517788692, + "request_tps": 9.126125184403708, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 1, + "request_id": "531d5ceb-d88b-49ab-ad8d-7bea678c3bb8", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873549+00:00", + "ttft_s": 1.140847125032451, + "total_s": 12.702382000046782, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.07119438584415, + "request_tps": 10.076850152949941, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 2, + "request_id": "a4b2f7bb-ab5c-405b-9514-d64bf34153ea", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873705+00:00", + "ttft_s": 1.1407294999808073, + "total_s": 12.702299374970607, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 11.071160870366917, + "request_tps": 10.076915700177802, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 3, + "slot": 3, + "request_id": "1b77ace8-b3ef-47f2-a2cd-06b6f9400a6e", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:21.873633+00:00", + "ttft_s": 1.1407265829620883, + "total_s": 10.277835665969178, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.178274020276117, + "request_tps": 9.048597683647658, + "group_elapsed_s": 12.702927250007633, + "group_output_tps": 34.795129602882234, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 0, + "request_id": "1143b997-b26b-4ca6-a4ad-dd1014020849", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576247+00:00", + "ttft_s": 1.3149999160086736, + "total_s": 13.642209500016179, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.383534012925246, + "request_tps": 9.382644358294614, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 1, + "request_id": "907e97b8-fec4-47ff-b971-45716a15b6fb", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576465+00:00", + "ttft_s": 1.419054625032004, + "total_s": 13.641894833010156, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.472197772531725, + "request_tps": 9.38286078047386, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 2, + "request_id": "d5875acf-e267-4393-84c1-a01dac48c4a5", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576349+00:00", + "ttft_s": 1.4192612090264447, + "total_s": 10.662482584011741, + "prompt_tokens": 20, + "output_tokens": 93, + "prompt_tps": null, + "decode_tps": 10.061427312742246, + "request_tps": 8.722171339295066, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + }, + { + "run_id": "20260831T132015Z", + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "phase": "measured", + "concurrency": 4, + "round": 4, + "slot": 3, + "request_id": "d2828df9-67d3-44b0-9809-bdb8c75f5282", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:22:34.576542+00:00", + "ttft_s": 1.4190426669665612, + "total_s": 13.641892916988581, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": null, + "decode_tps": 10.47218916878814, + "request_tps": 9.382862098308841, + "group_elapsed_s": 13.642570749972947, + "group_output_tps": 34.96408475660248, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T132015Z/summary.json b/benchmarks/inference/results/20260831T132015Z/summary.json new file mode 100644 index 0000000..c554507 --- /dev/null +++ b/benchmarks/inference/results/20260831T132015Z/summary.json @@ -0,0 +1,54 @@ +[ + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.37155269999057056, + "ttft_s_p50": 0.4171057919738814, + "ttft_s_p95": 0.4531440335791558, + "total_s_mean": 7.930883799993898, + "total_s_p50": 8.838838333962485, + "total_s_p95": 8.998963149765041, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 15.079897408512505, + "decode_tps_p50": 15.126147490283795, + "decode_tps_p95": 15.196162811046243, + "request_tps_mean": 14.380896425252896, + "request_tps_p50": 14.386982882152838, + "request_tps_p95": 14.53502856074605, + "aggregate_output_tps_mean": 14.380548747737752, + "aggregate_output_tps_p50": 14.386712701914696, + "aggregate_output_tps_p95": 14.534638049200744 + }, + { + "runtime": "vllm-metal", + "model": "qwen3.8-27b-mxfp8", + "concurrency": 4, + "requests": 20, + "successes": 20, + "failures": 0, + "ttft_s_mean": 1.255672066603438, + "ttft_s_p50": 1.3144360205042176, + "ttft_s_p95": 1.419064954231726, + "total_s_mean": 12.251293779097614, + "total_s_p50": 12.702340687508695, + "total_s_p95": 13.641910566360457, + "prompt_tps_mean": null, + "prompt_tps_p50": null, + "prompt_tps_p95": null, + "decode_tps_mean": 10.500220268362742, + "decode_tps_p50": 10.485677148644122, + "decode_tps_p95": 11.071162546140778, + "request_tps_mean": 9.417789343799708, + "request_tps_p50": 9.430282582501674, + "request_tps_p95": 10.077752113288362, + "aggregate_output_tps_mean": 35.09087333775396, + "aggregate_output_tps_p50": 34.96408475660248, + "aggregate_output_tps_p95": 35.52678209465127 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T134334Z/metadata.json b/benchmarks/inference/results/20260831T134334Z/metadata.json new file mode 100644 index 0000000..33946ef --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/metadata.json @@ -0,0 +1,108 @@ +{ + "run_id": "20260831T134334Z", + "created_at": "2026-08-31T13:43:34.324346+00:00", + "host": { + "platform": "macOS-15.6-arm64-arm-64bit", + "machine": "arm64", + "python": "3.12.8" + }, + "arguments": { + "output_dir": "/Users/christianmeyer/Documents/git/lai_automatisierung/benchmarks/inference/results", + "runtime": "ollama", + "ollama_url": "http://127.0.0.1:11434", + "ollama_model": "qwen3.8:27b-mlx", + "vllm_url": "http://127.0.0.1:8001", + "vllm_model": "qwen3.8-27b-nvfp4", + "warmups": 2, + "rounds": 5, + "concurrency": [ + 1 + ], + "max_tokens": 128, + "seed": 42, + "timeout": 600 + }, + "runtimes": [ + { + "name": "ollama", + "kind": "ollama", + "base_url": "http://127.0.0.1:11434", + "model": "qwen3.8:27b-mlx" + } + ], + "versions": { + "ollama": "ollama version is 0.32.14", + "vllm": "INFO 08-31 15:43:34 [__init__.py:52] Available plugins for group vllm.platform_plugins:\nINFO 08-31 15:43:34 [__init__.py:54] - metal -> vllm_metal:register\nINFO 08-31 15:43:34 [__init__.py:57] All plugins in this group will be loaded. Set `VLLM_PLUGINS` to control which plugins to load.\nINFO 08-31 15:43:36 [__init__.py:254] Platform plugin metal is activated\nINFO 08-31 15:43:37 [importing.py:88] Triton not installed or not compatible; certain GPU-related functions will not be available.\n0.28.0+cpu" + }, + "model_metadata": { + "ollama": { + "license": "\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n", + "modelfile": "# Modelfile generated by \"ollama show\"\nFROM qwen3.8:27b-mlx\nTEMPLATE {{ .Prompt }}\nRENDERER qwen3.8\nPARSER qwen3.5\nPARAMETER min_p 0\nPARAMETER presence_penalty 0\nPARAMETER repeat_penalty 1\nPARAMETER temperature 1\nPARAMETER top_k 20\nPARAMETER top_p 0.95\nLICENSE \"\"\"\n\n Apache License\n Version 2.0, January 2004\n http://www.apache.org/licenses/\n\n TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION\n\n 1. Definitions.\n\n \"License\" shall mean the terms and conditions for use, reproduction,\n and distribution as defined by Sections 1 through 9 of this document.\n\n \"Licensor\" shall mean the copyright owner or entity authorized by\n the copyright owner that is granting the License.\n\n \"Legal Entity\" shall mean the union of the acting entity and all\n other entities that control, are controlled by, or are under common\n control with that entity. For the purposes of this definition,\n \"control\" means (i) the power, direct or indirect, to cause the\n direction or management of such entity, whether by contract or\n otherwise, or (ii) ownership of fifty percent (50%) or more of the\n outstanding shares, or (iii) beneficial ownership of such entity.\n\n \"You\" (or \"Your\") shall mean an individual or Legal Entity\n exercising permissions granted by this License.\n\n \"Source\" form shall mean the preferred form for making modifications,\n including but not limited to software source code, documentation\n source, and configuration files.\n\n \"Object\" form shall mean any form resulting from mechanical\n transformation or translation of a Source form, including but\n not limited to compiled object code, generated documentation,\n and conversions to other media types.\n\n \"Work\" shall mean the work of authorship, whether in Source or\n Object form, made available under the License, as indicated by a\n copyright notice that is included in or attached to the work\n (an example is provided in the Appendix below).\n\n \"Derivative Works\" shall mean any work, whether in Source or Object\n form, that is based on (or derived from) the Work and for which the\n editorial revisions, annotations, elaborations, or other modifications\n represent, as a whole, an original work of authorship. For the purposes\n of this License, Derivative Works shall not include works that remain\n separable from, or merely link (or bind by name) to the interfaces of,\n the Work and Derivative Works thereof.\n\n \"Contribution\" shall mean any work of authorship, including\n the original version of the Work and any modifications or additions\n to that Work or Derivative Works thereof, that is intentionally\n submitted to Licensor for inclusion in the Work by the copyright owner\n or by an individual or Legal Entity authorized to submit on behalf of\n the copyright owner. For the purposes of this definition, \"submitted\"\n means any form of electronic, verbal, or written communication sent\n to the Licensor or its representatives, including but not limited to\n communication on electronic mailing lists, source code control systems,\n and issue tracking systems that are managed by, or on behalf of, the\n Licensor for the purpose of discussing and improving the Work, but\n excluding communication that is conspicuously marked or otherwise\n designated in writing by the copyright owner as \"Not a Contribution.\"\n\n \"Contributor\" shall mean Licensor and any individual or Legal Entity\n on behalf of whom a Contribution has been received by Licensor and\n subsequently incorporated within the Work.\n\n 2. Grant of Copyright License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n copyright license to reproduce, prepare Derivative Works of,\n publicly display, publicly perform, sublicense, and distribute the\n Work and such Derivative Works in Source or Object form.\n\n 3. Grant of Patent License. Subject to the terms and conditions of\n this License, each Contributor hereby grants to You a perpetual,\n worldwide, non-exclusive, no-charge, royalty-free, irrevocable\n (except as stated in this section) patent license to make, have made,\n use, offer to sell, sell, import, and otherwise transfer the Work,\n where such license applies only to those patent claims licensable\n by such Contributor that are necessarily infringed by their\n Contribution(s) alone or by combination of their Contribution(s)\n with the Work to which such Contribution(s) was submitted. If You\n institute patent litigation against any entity (including a\n cross-claim or counterclaim in a lawsuit) alleging that the Work\n or a Contribution incorporated within the Work constitutes direct\n or contributory patent infringement, then any patent licenses\n granted to You under this License for that Work shall terminate\n as of the date such litigation is filed.\n\n 4. Redistribution. You may reproduce and distribute copies of the\n Work or Derivative Works thereof in any medium, with or without\n modifications, and in Source or Object form, provided that You\n meet the following conditions:\n\n (a) You must give any other recipients of the Work or\n Derivative Works a copy of this License; and\n\n (b) You must cause any modified files to carry prominent notices\n stating that You changed the files; and\n\n (c) You must retain, in the Source form of any Derivative Works\n that You distribute, all copyright, patent, trademark, and\n attribution notices from the Source form of the Work,\n excluding those notices that do not pertain to any part of\n the Derivative Works; and\n\n (d) If the Work includes a \"NOTICE\" text file as part of its\n distribution, then any Derivative Works that You distribute must\n include a readable copy of the attribution notices contained\n within such NOTICE file, excluding those notices that do not\n pertain to any part of the Derivative Works, in at least one\n of the following places: within a NOTICE text file distributed\n as part of the Derivative Works; within the Source form or\n documentation, if provided along with the Derivative Works; or,\n within a display generated by the Derivative Works, if and\n wherever such third-party notices normally appear. The contents\n of the NOTICE file are for informational purposes only and\n do not modify the License. You may add Your own attribution\n notices within Derivative Works that You distribute, alongside\n or as an addendum to the NOTICE text from the Work, provided\n that such additional attribution notices cannot be construed\n as modifying the License.\n\n You may add Your own copyright statement to Your modifications and\n may provide additional or different license terms and conditions\n for use, reproduction, or distribution of Your modifications, or\n for any such Derivative Works as a whole, provided Your use,\n reproduction, and distribution of the Work otherwise complies with\n the conditions stated in this License.\n\n 5. Submission of Contributions. Unless You explicitly state otherwise,\n any Contribution intentionally submitted for inclusion in the Work\n by You to the Licensor shall be under the terms and conditions of\n this License, without any additional terms or conditions.\n Notwithstanding the above, nothing herein shall supersede or modify\n the terms of any separate license agreement you may have executed\n with Licensor regarding such Contributions.\n\n 6. Trademarks. This License does not grant permission to use the trade\n names, trademarks, service marks, or product names of the Licensor,\n except as required for reasonable and customary use in describing the\n origin of the Work and reproducing the content of the NOTICE file.\n\n 7. Disclaimer of Warranty. Unless required by applicable law or\n agreed to in writing, Licensor provides the Work (and each\n Contributor provides its Contributions) on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or\n implied, including, without limitation, any warranties or conditions\n of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A\n PARTICULAR PURPOSE. You are solely responsible for determining the\n appropriateness of using or redistributing the Work and assume any\n risks associated with Your exercise of permissions under this License.\n\n 8. Limitation of Liability. In no event and under no legal theory,\n whether in tort (including negligence), contract, or otherwise,\n unless required by applicable law (such as deliberate and grossly\n negligent acts) or agreed to in writing, shall any Contributor be\n liable to You for damages, including any direct, indirect, special,\n incidental, or consequential damages of any character arising as a\n result of this License or out of the use or inability to use the\n Work (including but not limited to damages for loss of goodwill,\n work stoppage, computer failure or malfunction, or any and all\n other commercial damages or losses), even if such Contributor\n has been advised of the possibility of such damages.\n\n 9. Accepting Warranty or Additional Liability. While redistributing\n the Work or Derivative Works thereof, You may choose to offer,\n and charge a fee for, acceptance of support, warranty, indemnity,\n or other liability obligations and/or rights consistent with this\n License. However, in accepting such obligations, You may act only\n on Your own behalf and on Your sole responsibility, not on behalf\n of any other Contributor, and only if You agree to indemnify,\n defend, and hold each Contributor harmless for any liability\n incurred by, or claims asserted against, such Contributor by reason\n of your accepting any such warranty or additional liability.\n\n END OF TERMS AND CONDITIONS\n\n APPENDIX: How to apply the Apache License to your work.\n\n To apply the Apache License to your work, attach the following\n boilerplate notice, with the fields enclosed by brackets \"[]\"\n replaced with your own identifying information. (Don't include\n the brackets!) The text should be enclosed in the appropriate\n comment syntax for the file format. We also recommend that a\n file or class name and description of purpose be included on the\n same \"printed page\" as the copyright notice for easier\n identification within third-party archives.\n\n Copyright 2026 Alibaba Cloud\n\n Licensed under the Apache License, Version 2.0 (the \"License\");\n you may not use this file except in compliance with the License.\n You may obtain a copy of the License at\n\n http://www.apache.org/licenses/LICENSE-2.0\n\n Unless required by applicable law or agreed to in writing, software\n distributed under the License is distributed on an \"AS IS\" BASIS,\n WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n See the License for the specific language governing permissions and\n limitations under the License.\n\"\"\"\n", + "parameters": "top_k 20\ntop_p 0.95\nmin_p 0\npresence_penalty 0\nrepeat_penalty 1\ntemperature 1", + "template": "{{ .Prompt }}", + "details": { + "parent_model": "", + "format": "safetensors", + "family": "qwen3_5", + "families": null, + "parameter_size": "27.8B", + "quantization_level": "nvfp4" + }, + "model_info": { + "general.architecture": "qwen3_5", + "general.parameter_count": 27781081984, + "qwen3_5.block_count": 64, + "qwen3_5.context_length": 262144, + "qwen3_5.embedding_length": 5120 + }, + "capabilities": [ + "completion", + "vision", + "tools", + "thinking" + ], + "modified_at": "2026-08-20T17:42:35.895756165+02:00", + "requires": "0.32.12" + } + }, + "prompts": { + "short": "In exactly four concise bullet points, explain why database transactions are useful. Do not use a heading.", + "medium": "You are reviewing a Laravel service that transfers money between two accounts. Explain the failure modes involving concurrency, partial writes, retries, and audit logs. Then provide a robust implementation plan. Keep the response technical and concise.", + "code": "Write a Python 3.12 function named merge_intervals that accepts a list of integer pairs, validates each pair, merges overlapping or adjacent intervals, and returns sorted tuples. Include type hints, a short docstring, and five assert-based examples. Output only code." + }, + "measurement_notes": { + "ttft": "Client wall time from request start until first non-empty generated text chunk.", + "total": "Client wall time from request start until stream completion.", + "decode_tps": "Ollama server eval metric; otherwise output tokens / (total - TTFT).", + "aggregate_output_tps": "Sum of successful output tokens / concurrent group wall time." + }, + "completed_at": "2026-08-31T13:44:00.378796+00:00", + "summary": [ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.1723672749940306, + "ttft_s_p50": 0.11980204097926617, + "ttft_s_p95": 0.36265889178030186, + "total_s_mean": 2.551861874992028, + "total_s_p50": 2.646599249972496, + "total_s_p95": 2.8519752004183827, + "prompt_tps_mean": 1253549.1340794961, + "prompt_tps_p50": 831.312776076798, + "prompt_tps_p95": 3513050.713847771, + "decode_tps_mean": 47.324053520964824, + "decode_tps_p50": 45.28260592046214, + "decode_tps_p95": 54.383553646467476, + "request_tps_mean": 45.013305014436966, + "request_tps_p50": 44.56718732154595, + "request_tps_p95": 47.927692834210475, + "aggregate_output_tps_mean": 45.00738945185462, + "aggregate_output_tps_p50": 44.56227530114212, + "aggregate_output_tps_p95": 47.92094820555752 + } + ] +} \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T134334Z/raw.csv b/benchmarks/inference/results/20260831T134334Z/raw.csv new file mode 100644 index 0000000..205cb75 --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/raw.csv @@ -0,0 +1,8 @@ +run_id,runtime,model,phase,concurrency,round,slot,request_id,prompt_name,prompt_chars,max_tokens,started_at,ttft_s,total_s,prompt_tokens,output_tokens,prompt_tps,decode_tps,request_tps,group_elapsed_s,group_output_tps,error +20260831T134334Z,ollama,qwen3.8:27b-mlx,warmup,1,0,0,c3b04207-976b-4a0d-9d3c-80ef8dc4225f,short,106,128,2026-08-31T13:43:39.472249+00:00,3.4916902080294676,5.4140321660088375,20,96,114.72541286980045,48.19996255746575,17.731701079044402,5.4149448340176605,17.728712469407014, +20260831T134334Z,ollama,qwen3.8:27b-mlx,warmup,1,1,0,47d529a2-f167-46a0-af05-a6ca0e1929a4,medium,252,128,2026-08-31T13:43:44.886979+00:00,0.35504300001775846,2.7301927499938756,43,128,157.93211765391837,52.61516193511847,46.88313673101913,2.730699084000662,46.87444352618714, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,0,0,5b638c8b-a185-48d7-8625-f6ebacc06853,short,106,128,2026-08-31T13:43:47.617556+00:00,0.11980204097926617,2.1940376659622416,20,96,441.616869764425,45.28260592046214,43.7549461840698,2.1943347500055097,43.749022340260055, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,1,0,fd1c62de-2412-4243-85a0-0851e2364f7c,medium,252,128,2026-08-31T13:43:49.811979+00:00,0.12685995898209512,2.771603333996609,43,128,831.312776076798,47.52723693214773,46.18265479404875,2.771931625029538,46.17718519613052, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,2,0,50050b20-0fd0-4e15-a4bb-ddfc74ada7e1,code,267,128,2026-08-31T13:43:52.584034+00:00,0.42160862497985363,2.646599249972496,54,128,159.3484419263456,56.09763282504741,48.363952344250905,2.6469858330092393,48.35688895791427, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,3,0,22721212-cd1e-460f-9aa3-f2a1bb6f2f96,short,106,128,2026-08-31T13:43:55.231045+00:00,0.09599920798791572,2.2750009580049664,20,96,2500000.0,42.69653289142422,42.19778442826943,2.2753357499605045,42.19157546382611, +20260831T134334Z,ollama,qwen3.8:27b-mlx,measured,1,4,0,d95f9110-8465-4bf1-8b6d-511699a58906,medium,252,128,2026-08-31T13:43:57.506343+00:00,0.09756654204102233,2.8720681670238264,43,128,3766313.3923097136,45.01625903574264,44.56718732154595,2.8723847499932162,44.56227530114212, diff --git a/benchmarks/inference/results/20260831T134334Z/raw.json b/benchmarks/inference/results/20260831T134334Z/raw.json new file mode 100644 index 0000000..fd3d13e --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/raw.json @@ -0,0 +1,170 @@ +[ + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "c3b04207-976b-4a0d-9d3c-80ef8dc4225f", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:39.472249+00:00", + "ttft_s": 3.4916902080294676, + "total_s": 5.4140321660088375, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 114.72541286980045, + "decode_tps": 48.19996255746575, + "request_tps": 17.731701079044402, + "group_elapsed_s": 5.4149448340176605, + "group_output_tps": 17.728712469407014, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "warmup", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "47d529a2-f167-46a0-af05-a6ca0e1929a4", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:44.886979+00:00", + "ttft_s": 0.35504300001775846, + "total_s": 2.7301927499938756, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 157.93211765391837, + "decode_tps": 52.61516193511847, + "request_tps": 46.88313673101913, + "group_elapsed_s": 2.730699084000662, + "group_output_tps": 46.87444352618714, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 0, + "slot": 0, + "request_id": "5b638c8b-a185-48d7-8625-f6ebacc06853", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:47.617556+00:00", + "ttft_s": 0.11980204097926617, + "total_s": 2.1940376659622416, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 441.616869764425, + "decode_tps": 45.28260592046214, + "request_tps": 43.7549461840698, + "group_elapsed_s": 2.1943347500055097, + "group_output_tps": 43.749022340260055, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 1, + "slot": 0, + "request_id": "fd1c62de-2412-4243-85a0-0851e2364f7c", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:49.811979+00:00", + "ttft_s": 0.12685995898209512, + "total_s": 2.771603333996609, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 831.312776076798, + "decode_tps": 47.52723693214773, + "request_tps": 46.18265479404875, + "group_elapsed_s": 2.771931625029538, + "group_output_tps": 46.17718519613052, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 2, + "slot": 0, + "request_id": "50050b20-0fd0-4e15-a4bb-ddfc74ada7e1", + "prompt_name": "code", + "prompt_chars": 267, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:52.584034+00:00", + "ttft_s": 0.42160862497985363, + "total_s": 2.646599249972496, + "prompt_tokens": 54, + "output_tokens": 128, + "prompt_tps": 159.3484419263456, + "decode_tps": 56.09763282504741, + "request_tps": 48.363952344250905, + "group_elapsed_s": 2.6469858330092393, + "group_output_tps": 48.35688895791427, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 3, + "slot": 0, + "request_id": "22721212-cd1e-460f-9aa3-f2a1bb6f2f96", + "prompt_name": "short", + "prompt_chars": 106, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:55.231045+00:00", + "ttft_s": 0.09599920798791572, + "total_s": 2.2750009580049664, + "prompt_tokens": 20, + "output_tokens": 96, + "prompt_tps": 2500000.0, + "decode_tps": 42.69653289142422, + "request_tps": 42.19778442826943, + "group_elapsed_s": 2.2753357499605045, + "group_output_tps": 42.19157546382611, + "error": null + }, + { + "run_id": "20260831T134334Z", + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "phase": "measured", + "concurrency": 1, + "round": 4, + "slot": 0, + "request_id": "d95f9110-8465-4bf1-8b6d-511699a58906", + "prompt_name": "medium", + "prompt_chars": 252, + "max_tokens": 128, + "started_at": "2026-08-31T13:43:57.506343+00:00", + "ttft_s": 0.09756654204102233, + "total_s": 2.8720681670238264, + "prompt_tokens": 43, + "output_tokens": 128, + "prompt_tps": 3766313.3923097136, + "decode_tps": 45.01625903574264, + "request_tps": 44.56718732154595, + "group_elapsed_s": 2.8723847499932162, + "group_output_tps": 44.56227530114212, + "error": null + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/20260831T134334Z/summary.json b/benchmarks/inference/results/20260831T134334Z/summary.json new file mode 100644 index 0000000..b7eed98 --- /dev/null +++ b/benchmarks/inference/results/20260831T134334Z/summary.json @@ -0,0 +1,28 @@ +[ + { + "runtime": "ollama", + "model": "qwen3.8:27b-mlx", + "concurrency": 1, + "requests": 5, + "successes": 5, + "failures": 0, + "ttft_s_mean": 0.1723672749940306, + "ttft_s_p50": 0.11980204097926617, + "ttft_s_p95": 0.36265889178030186, + "total_s_mean": 2.551861874992028, + "total_s_p50": 2.646599249972496, + "total_s_p95": 2.8519752004183827, + "prompt_tps_mean": 1253549.1340794961, + "prompt_tps_p50": 831.312776076798, + "prompt_tps_p95": 3513050.713847771, + "decode_tps_mean": 47.324053520964824, + "decode_tps_p50": 45.28260592046214, + "decode_tps_p95": 54.383553646467476, + "request_tps_mean": 45.013305014436966, + "request_tps_p50": 44.56718732154595, + "request_tps_p95": 47.927692834210475, + "aggregate_output_tps_mean": 45.00738945185462, + "aggregate_output_tps_p50": 44.56227530114212, + "aggregate_output_tps_p95": 47.92094820555752 + } +] \ No newline at end of file diff --git a/benchmarks/inference/results/backend_verification.json b/benchmarks/inference/results/backend_verification.json new file mode 100644 index 0000000..e435a67 --- /dev/null +++ b/benchmarks/inference/results/backend_verification.json @@ -0,0 +1,37 @@ +{ + "verified_at": "2026-08-31", + "host": "Apple M4 Max, arm64, macOS 15.6", + "ollama": { + "version": "0.32.14", + "mlx_version": "0.32.0-213-gadf21de", + "device": "gpu", + "metal_library": true, + "nvfp4_model": { + "name": "qwen3.8:27b-mlx", + "id": "5642e97495e1", + "format": "safetensors", + "quantization": "nvfp4", + "parameter_size": "27.8B", + "runner_log": "starting mlx runner subprocess" + }, + "mxfp8_model": { + "name": "qwen3.8:27b-mxfp8", + "id": "464021588235", + "format": "safetensors", + "quantization": "mxfp8", + "parameter_size": "27.8B", + "runner_log": "starting mlx runner subprocess" + } + }, + "vllm_metal": { + "vllm_version": "0.28.0+cpu", + "plugin_version": "0.4.0.dev20260831112704", + "mlx_version": "0.32.0", + "mlx_lm_version": "0.31.3", + "mlx_vlm_version": "0.6.4", + "platform_plugin": "metal", + "compute_log": "MLX device set to: Device(gpu, 0)", + "pytorch_device_log": "PyTorch device set to: mps", + "note": "The +cpu suffix identifies the portable vLLM core wheel; the Metal plugin supplies MLX GPU execution." + } +} diff --git a/benchmarks/inference/results/cold_start.json b/benchmarks/inference/results/cold_start.json new file mode 100644 index 0000000..bd9bfa5 --- /dev/null +++ b/benchmarks/inference/results/cold_start.json @@ -0,0 +1,27 @@ +{ + "measured_at": "2026-08-31", + "notes": [ + "Cold starts were measured separately from warm inference.", + "The vLLM model was already present in the Hugging Face cache; zero bytes were downloaded.", + "vLLM launch-to-ready is derived from the recorded epoch and second-resolution API startup log.", + "Ollama's daemon was already running, but the model was fully unloaded before the request.", + "The Ollama request used one output token and keep_alive=0 so generation time was negligible and the model was unloaded afterward." + ], + "vllm_metal": { + "process_start_epoch": 1788177244.465068, + "api_ready_local_time": "2026-08-31T13:54:33+02:00", + "approx_launch_to_api_ready_s": 28.535, + "model_load_s": 3.25, + "engine_profile_cache_warmup_s": 9.71, + "download_bytes": 0 + }, + "ollama": { + "client_total_s": 3.455159667, + "server_total_s": 3.448873125, + "model_load_s": 3.244982292, + "prompt_eval_s": 0.13364925, + "decode_s": 0.069530292, + "prompt_tokens": 7, + "output_tokens": 1 + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 723bb06..db7288b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,42 +3,56 @@ services: build: . restart: unless-stopped environment: - FAILURE_DB_HOST: ${FAILURE_DB_HOST:-host.docker.internal} - FAILURE_DB_PORT: ${FAILURE_DB_PORT:-33060} - FAILURE_DB_NAME: ${FAILURE_DB_NAME:-laravel_ai_chat} - FAILURE_DB_USER: ${FAILURE_DB_USER:-laravel} - FAILURE_DB_PASSWORD: ${FAILURE_DB_PASSWORD:?FAILURE_DB_PASSWORD must be set} SOURCE_REPOSITORY: ${SOURCE_REPOSITORY:-https://gitea.tisinfra.de/christian.meyer/lai.git} SOURCE_BRANCH: ${SOURCE_BRANCH:-main} - AUTOMATION_MODE: ${AUTOMATION_MODE:-dry-run} - ALLOW_DIRECT_MAIN: ${ALLOW_DIRECT_MAIN:-false} GITEA_URL: ${GITEA_URL:-https://gitea.tisinfra.de} GITEA_OWNER: ${GITEA_OWNER:-christian.meyer} GITEA_REPOSITORY: ${GITEA_REPOSITORY:-lai} GITEA_USERNAME: ${GITEA_USERNAME:-automation-bot} - GITEA_TOKEN: ${GITEA_TOKEN:-} - AGENT_COMMAND: ${AGENT_COMMAND:-aider} + GITEA_TOKEN: ${GITEA_TOKEN:?GITEA_TOKEN must be set} + GITEA_QUEUED_LABEL: ${GITEA_QUEUED_LABEL:-ai:queued} + GITEA_RUNNING_LABEL: ${GITEA_RUNNING_LABEL:-ai:running} + GITEA_READY_LABEL: ${GITEA_READY_LABEL:-ai:ready} + GITEA_FAILED_LABEL: ${GITEA_FAILED_LABEL:-ai:failed} AGENT_MODEL: ${AGENT_MODEL:-mistral/devstral-latest} + AGENT_MODEL_API_BASE: ${AGENT_MODEL_API_BASE:-} + AGENT_COMMAND_TIMEOUT_SECONDS: ${AGENT_COMMAND_TIMEOUT_SECONDS:-180} + AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-1200} + AGENT_STEP_LIMIT: ${AGENT_STEP_LIMIT:-50} + AGENT_COST_LIMIT: ${AGENT_COST_LIMIT:-5} + MAX_AGENT_ATTEMPTS: ${MAX_AGENT_ATTEMPTS:-3} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + AGENT_USER: coder MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} - OPENAI_API_KEY: ${OPENAI_API_KEY:-} - OPENAI_API_BASE: ${OPENAI_API_BASE:-} - OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} - OLLAMA_API_BASE: ${OLLAMA_API_BASE:-} POLL_SECONDS: ${POLL_SECONDS:-60} - MAX_AGENT_ATTEMPTS: ${MAX_AGENT_ATTEMPTS:-2} - STALE_CLAIM_MINUTES: ${STALE_CLAIM_MINUTES:-90} - TEST_COMMANDS: ${TEST_COMMANDS:-composer install --no-interaction --prefer-dist;;npm ci;;npm run build;;php artisan migrate:fresh --force;;php artisan test} - TEST_DB_HOST: ${TEST_DB_HOST:-host.docker.internal} - TEST_DB_PORT: ${TEST_DB_PORT:-33060} + SETUP_COMMANDS: ${SETUP_COMMANDS:-composer install --no-interaction --prefer-dist;;npm ci} + TEST_COMMANDS: ${TEST_COMMANDS:-npm run build;;php artisan test} + TEST_DB_HOST: ${TEST_DB_HOST:-laravel-ai-chat-mysql} + TEST_DB_PORT: ${TEST_DB_PORT:-3306} TEST_DB_NAME: ${TEST_DB_NAME:-laravel_ai_chat_agent_test} TEST_DB_USER: ${TEST_DB_USER:-lai_test_runner} TEST_DB_PASSWORD: ${TEST_DB_PASSWORD:-agent_test_secret} TEST_CUSTOMER_DB: ${TEST_CUSTOMER_DB:-customer_agent_test} TEST_CONFIG_DB: ${TEST_CONFIG_DB:-cockpit_agent_test} + MAX_CHANGED_FILES: ${MAX_CHANGED_FILES:-12} + MAX_CHANGED_LINES: ${MAX_CHANGED_LINES:-800} extra_hosts: - host.docker.internal:host-gateway volumes: - automation-workspaces:/workspaces + networks: + - automation + - lai_laravel_network + security_opt: + - no-new-privileges:true + pids_limit: 512 volumes: automation-workspaces: + +networks: + automation: + driver: bridge + lai_laravel_network: + external: true + name: lai_laravel-network diff --git a/old_2026-08-31/.dockerignore b/old_2026-08-31/.dockerignore new file mode 100644 index 0000000..e913f96 --- /dev/null +++ b/old_2026-08-31/.dockerignore @@ -0,0 +1,8 @@ +.git +.env +.venv +.docker-tmp +__pycache__ +.pytest_cache +.ruff_cache +workspaces diff --git a/old_2026-08-31/.env.example b/old_2026-08-31/.env.example new file mode 100644 index 0000000..cf9fb9a --- /dev/null +++ b/old_2026-08-31/.env.example @@ -0,0 +1,29 @@ +FAILURE_DB_HOST=host.docker.internal +FAILURE_DB_PORT=33060 +FAILURE_DB_NAME=laravel_ai_chat +FAILURE_DB_USER=laravel +FAILURE_DB_PASSWORD= + +SOURCE_REPOSITORY=https://gitea.tisinfra.de/christian.meyer/lai.git +SOURCE_BRANCH=main +GITEA_URL=https://gitea.tisinfra.de +GITEA_OWNER=christian.meyer +GITEA_REPOSITORY=lai +GITEA_USERNAME=automation-bot +GITEA_TOKEN= + +# dry-run, pr, direct-main +AUTOMATION_MODE=dry-run +ALLOW_DIRECT_MAIN=false + +AGENT_COMMAND=aider +AGENT_MODEL=mistral/devstral-latest +AGENT_TIMEOUT_SECONDS=900 +AGENT_IDLE_TIMEOUT_SECONDS=300 +MISTRAL_API_KEY= + +POLL_SECONDS=60 +STALE_CLAIM_MINUTES=90 +MAX_CHANGED_FILES=12 +MAX_CHANGED_LINES=800 +TEST_COMMANDS=composer install --no-interaction --prefer-dist;;php artisan test diff --git a/old_2026-08-31/.gitea/workflows/ci.yml b/old_2026-08-31/.gitea/workflows/ci.yml new file mode 100644 index 0000000..58fa6bf --- /dev/null +++ b/old_2026-08-31/.gitea/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ".[dev]" + - run: ruff check src tests + - run: pytest -q + - run: docker build -t lai-automation:test . diff --git a/old_2026-08-31/.gitignore b/old_2026-08-31/.gitignore new file mode 100644 index 0000000..a6928d8 --- /dev/null +++ b/old_2026-08-31/.gitignore @@ -0,0 +1,9 @@ +.env +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +.docker-tmp/ +.docker-buildx/ +*.pyc +workspaces/ diff --git a/old_2026-08-31/Dockerfile b/old_2026-08-31/Dockerfile new file mode 100644 index 0000000..9f0852c --- /dev/null +++ b/old_2026-08-31/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + composer git nodejs npm openssh-client php-cli php-curl php-mbstring php-mysql php-xml php-zip unzip \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src ./src +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --retries 10 --timeout 120 ".[agent]" + +RUN useradd --create-home --uid 10001 coder \ + && mkdir -p /workspaces \ + && chmod 700 /home/coder + +ENTRYPOINT ["lai-agent"] +CMD ["worker"] diff --git a/old_2026-08-31/README.md b/old_2026-08-31/README.md new file mode 100644 index 0000000..0f5b378 --- /dev/null +++ b/old_2026-08-31/README.md @@ -0,0 +1,202 @@ +# LAI Automatisierung + +Kontrollierter Coding-Agent fuer Failure Reports aus +[`lai`](https://gitea.tisinfra.de/christian.meyer/lai). Der Orchestrator verwendet +[Aider](https://aider.chat/) headless als austauschbaren Coding-Kern. Git-, Gitea-, +Policy- und Freigabelogik bleiben in diesem Projekt. + +## Warum Aider? + +- scriptbarer Einmal-Auftrag mit `--message-file` +- gute Git- und Repository-Map-Unterstuetzung +- Mistral ueber LiteLLM nutzbar +- automatische Commits koennen deaktiviert werden +- deutlich kleinerer Betriebsumfang als OpenHands + +OpenHands bleibt eine moegliche spaetere Alternative fuer komplexere, mehrstufige +Agentenlaeufe. Sein Headless-Modus arbeitet jedoch immer mit `always-approve`; deshalb +muessen auch dort unsere externen Policy-Gates erhalten bleiben. + +## Ablauf + +```text +failure_reports(open) + -> transaktionales Claiming (SKIP LOCKED) + -> anonymisierter Agentenauftrag + -> frischer Clone von lai.git + -> Aider-Aenderung ohne Commit + -> Pfad-, Groessen- und Sicherheitsgates + -> Tests + -> Commit + -> PR oder kontrollierter Push auf main + -> failure_reports(resolved/agent_failed) +``` + +Der Coding-Agent erhaelt keinen Zugang zur ERP- oder Kundendatenbank. Rohdaten und +`.env`-Dateien sind ausgeschlossen. Repository-Pushes verwenden `GIT_ASKPASS`; Tokens +werden nicht in Remote-URLs oder Prozessargumenten gespeichert. Der Orchestrator laeuft +als privilegierter Containerprozess, startet Aider aber als separaten Benutzer `coder`. +Dieser Unterprozess erhaelt nur Modell-Credentials, nicht die Failure-DB- oder +Gitea-Zugangsdaten. + +Die Workspace-Setup- und Testprozesse erhalten separat konfigurierte `TEST_DB_*`-Zugangsdaten. +Dieser Benutzer darf ausschliesslich die isolierten Testschemas bearbeiten. Fuer Laravel +werden daraus `DB_*`, `ERP_DB_*` und `COCKPIT_CONFIG_DATABASE` gesetzt; Aider selbst sieht +diese Werte nie. Produktive ERP- oder Kundenzugangsdaten duerfen hier nicht verwendet werden. +Vor Aider und erneut vor der abschliessenden Validierung installiert `SETUP_COMMANDS` die +Abhaengigkeiten, erstellt die Vite-Assets und initialisiert mit +`php artisan migrate:fresh --force` das isolierte Laravel-Schema neu. `TEST_COMMANDS` enthaelt +die eigentlichen Pruefungen. Deshalb muessen alle +`TEST_DB_*`-Schemas dediziert und vollstaendig entbehrlich sein. + +Lokale Ollama-Modelle laufen fuer Produktionsdateien in einem einzelnen Whole-File-Durchgang. +Devstral und andere externe Coding-Modelle verwenden direkt das Diff-Format statt desselben +Modells zusaetzlich als Architect und Editor aufzurufen. Das halbiert Modellrunden fuer kleine +Reparaturen. Bei gezielt gerouteten Dateien ist die zusaetzliche Repo-Map deaktiviert. +Automatische URL-Erkennung und Web-Scraping sind deaktiviert, damit URLs in Anwendungscode oder +Tests niemals als externe Rechercheauftraege interpretiert werden. +`AGENT_TIMEOUT_SECONDS` begrenzt Agentenprozess und Modell-API-Aufruf. +`AGENT_IDLE_TIMEOUT_SECONDS` beendet zusaetzlich einen Agenten, der trotz laufendem Prozess +standardmaessig fuenf Minuten keine Ausgabe mehr erzeugt; vorhandene Teilpatches werden danach +weiterhin validiert. +Setup-Fehler werden als Infrastrukturfehler beendet und nicht als Codekorrektur an das Modell +zurueckgegeben. Hat ein lokales Modell vor einem Timeout bereits Dateien geaendert, durchlaufen +diese Teilresultate trotzdem Policy, Syntaxpruefung und Tests; nur ein Timeout ohne Diff verwirft +den Versuch sofort. + +Laravel-Tests laufen mit PHP und Composer direkt im isolierten Worker. Der Docker-Socket +des Hosts wird nicht eingebunden. Abgebrochene Claims werden nach standardmaessig 90 +Minuten wieder freigegeben; `STALE_CLAIM_MINUTES` passt dieses Intervall an. +Explizite Nur-Entfernen-Auftraege werden als negativer SQL-Vertrag geprueft: Der verlangte +Ausdruck muss aus der passenden Referenzabfrage verschwinden, in einer negativen Test-Assertion +vorkommen, und alle anderen Felder, Joins, Filter, Gruppierungen sowie bestehenden Tests muessen +unveraendert bleiben. Ist der Ausdruck in einer eindeutig zur Benutzerfrage passenden, getrackten +Referenzabfrage bereits entfernt, wird ein veralteter Report ohne Agentenlauf und ohne PR als +`already_fixed` abgeschlossen. + +Schlaegt Policy, Syntax oder isolierte Validierung fehl, erhaelt Aider kontrollierte +Korrekturversuche mit redigierter Diagnose. Syntaxgueltige Teilpatches werden als Checkpoint +erhalten; unerwartete Dateien werden verworfen. Nach einem Testfehler bleiben alle gerouteten +Produktions- und Testdateien sichtbar, damit der Agent die Ursache korrigieren kann, statt nur +Assertions an fehlerhaftes Verhalten anzupassen. Mehrteilige Feedbacks zu Antwortformat und +expliziten Routing-Praefixen werden an ihre jeweiligen Anwendungseigentuemer geroutet und muessen +vollstaendig abgedeckt sein. Zwei identische Policy-Fehler hintereinander brechen frueh ab. +`MAX_AGENT_ATTEMPTS` ist auf 1 bis 7 begrenzt. + +## Schnellstart + +```bash +cp .env.example .env +# FAILURE_DB_PASSWORD und bei Bedarf MISTRAL_API_KEY eintragen +# TEST_DB_PASSWORD fuer den Benutzer der isolierten Testschemas eintragen +docker compose build +docker compose run --rm automation check +docker compose run --rm automation once +``` + +Der Default `AUTOMATION_MODE=dry-run` claimt einen offenen Report, erzeugt den +anonymisierten Auftrag und setzt ihn danach wieder auf `open`. Es wird weder ein Agent +gestartet noch gepusht. + +## Fortschritt beobachten + +```bash +docker compose ps automation +docker compose exec automation lai-agent check +docker compose logs -f automation +``` + +Der Worker protokolliert keine unredigierten Prompts, SQL-Ergebnisse oder Secrets. Aider-Ausgabe +wird ohne Terminalfarben live, redigiert und pro Zeile begrenzt protokolliert; in Report-Metadaten +landet nur ein begrenzter Ausgabetail. Fuer jeden Versuch erscheinen Modell, Edit-Strategie, +Dateien, PID, Laufzeit, Ausgabeaktivitaet und Ergebnisstatus. Zusaetzlich erscheinen +Workspace-Vorbereitung, Policy-Pruefung, jeder Validierungsbefehl, Commit, PR-URL und Cleanup. +Bei laengeren Agenten- oder Testphasen wird alle 30 Sekunden Laufzeitfortschritt ausgegeben und +`updated_at` als Datenbank-Lease erneuert. Nur wirklich verwaiste Claims werden nach +`STALE_CLAIM_MINUTES` wieder freigegeben. + +## PR-Modus + +```dotenv +AUTOMATION_MODE=pr +GITEA_USERNAME=automation-bot +GITEA_TOKEN=... +MISTRAL_API_KEY=... +``` + +Der Bot benoetigt Schreibzugriff auf `christian.meyer/lai` und die Berechtigung, +Branches sowie Pull Requests anzulegen. Jeder Lauf erstellt +`ai/failure-{report_id}` und einen PR gegen `main`. + +## Direkter main-Modus + +```dotenv +AUTOMATION_MODE=direct-main +ALLOW_DIRECT_MAIN=true +GITEA_TOKEN=... +``` + +Der zweite Schalter ist ein Kill-Switch. In diesem Modus sind Migrationen, Routes, +Auth, Middleware, PermissionService und SQLValidatorService zusaetzlich blockiert. +Branch Protection muss den Bot fuer direkte Pushes explizit zulassen. PR-Modus bleibt +die empfohlene Betriebsart. + +## Modelle + +Das Modell wird ausschliesslich ueber `AGENT_MODEL` gewaehlt. Devstral ist der +Default und ein Coding-Agent, nicht das Chat-Modell der Laravel-Anwendung: + +```dotenv +AGENT_MODEL=mistral/devstral-latest +MISTRAL_API_KEY=... +``` + +Codestral verwendet denselben Mistral-Zugang: + +```dotenv +AGENT_MODEL=mistral/codestral-latest +MISTRAL_API_KEY=... +``` + +Qwen 3.5 27B ueber OpenRouter: + +```dotenv +AGENT_MODEL=openrouter/qwen/qwen3.5-27b +OPENROUTER_API_KEY=... +``` + +Oder mit einem lokalen Ollama auf dem Docker-Host: + +```dotenv +AGENT_MODEL=ollama_chat/qwen3.5:27b +OLLAMA_API_BASE=http://host.docker.internal:11434 +``` + +Nach einer Aenderung wird der Worker neu erstellt, damit Compose die neue Umgebung +uebernimmt: + +```bash +docker compose up -d --force-recreate automation +docker compose exec -T automation lai-agent check +``` + +Aider verwendet fuer lokale Produktionsreparaturen `whole` und sonst das praezise +`diff`-Format. Das Chat-Modell und Coding-Modell sind bewusst getrennt. Fuer Coding-Agenten sollte ein +agentisch/coding-optimiertes Modell verwendet werden. Ein Wechsel zu OpenHands oder +einem anderen Backend erfordert einen neuen Adapter neben `AiderAgent`. + +## Tests + +```bash +python -m venv .venv +. .venv/bin/activate +pip install ".[dev]" +ruff check src tests +pytest -q +``` + +## Repositorys + +- Chat: `https://gitea.tisinfra.de/christian.meyer/lai.git` +- Automatisierung: `https://gitea.tisinfra.de/christian.meyer/lai_automatisierung.git` + diff --git a/old_2026-08-31/docker-compose.yml b/old_2026-08-31/docker-compose.yml new file mode 100644 index 0000000..a808f94 --- /dev/null +++ b/old_2026-08-31/docker-compose.yml @@ -0,0 +1,57 @@ +services: + automation: + build: . + restart: unless-stopped + environment: + FAILURE_DB_HOST: ${FAILURE_DB_HOST:-laravel-ai-chat-mysql} + FAILURE_DB_PORT: ${FAILURE_DB_PORT:-3306} + FAILURE_DB_NAME: ${FAILURE_DB_NAME:-laravel_ai_chat} + FAILURE_DB_USER: ${FAILURE_DB_USER:-laravel} + FAILURE_DB_PASSWORD: ${FAILURE_DB_PASSWORD:?FAILURE_DB_PASSWORD must be set} + SOURCE_REPOSITORY: ${SOURCE_REPOSITORY:-https://gitea.tisinfra.de/christian.meyer/lai.git} + SOURCE_BRANCH: ${SOURCE_BRANCH:-main} + AUTOMATION_MODE: ${AUTOMATION_MODE:-dry-run} + ALLOW_DIRECT_MAIN: ${ALLOW_DIRECT_MAIN:-false} + GITEA_URL: ${GITEA_URL:-https://gitea.tisinfra.de} + GITEA_OWNER: ${GITEA_OWNER:-christian.meyer} + GITEA_REPOSITORY: ${GITEA_REPOSITORY:-lai} + GITEA_USERNAME: ${GITEA_USERNAME:-automation-bot} + GITEA_TOKEN: ${GITEA_TOKEN:-} + AGENT_COMMAND: ${AGENT_COMMAND:-aider} + AGENT_MODEL: ${AGENT_MODEL:-mistral/devstral-latest} + AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-900} + AGENT_IDLE_TIMEOUT_SECONDS: ${AGENT_IDLE_TIMEOUT_SECONDS:-300} + MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_API_BASE: ${OPENAI_API_BASE:-} + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} + OLLAMA_API_BASE: ${OLLAMA_API_BASE:-} + POLL_SECONDS: ${POLL_SECONDS:-60} + MAX_AGENT_ATTEMPTS: ${MAX_AGENT_ATTEMPTS:-2} + STALE_CLAIM_MINUTES: ${STALE_CLAIM_MINUTES:-90} + SETUP_COMMANDS: ${SETUP_COMMANDS:-composer install --no-interaction --prefer-dist;;npm ci;;npm run build;;php artisan migrate:fresh --force} + TEST_COMMANDS: ${TEST_COMMANDS:-php artisan test} + TEST_DB_HOST: ${TEST_DB_HOST:-laravel-ai-chat-mysql} + TEST_DB_PORT: ${TEST_DB_PORT:-3306} + TEST_DB_NAME: ${TEST_DB_NAME:-laravel_ai_chat_agent_test} + TEST_DB_USER: ${TEST_DB_USER:-lai_test_runner} + TEST_DB_PASSWORD: ${TEST_DB_PASSWORD:-agent_test_secret} + TEST_CUSTOMER_DB: ${TEST_CUSTOMER_DB:-customer_agent_test} + TEST_CONFIG_DB: ${TEST_CONFIG_DB:-cockpit_agent_test} + extra_hosts: + - host.docker.internal:host-gateway + volumes: + - automation-workspaces:/workspaces + networks: + - automation + - lai_laravel_network + +volumes: + automation-workspaces: + +networks: + automation: + driver: bridge + lai_laravel_network: + external: true + name: lai_laravel-network diff --git a/old_2026-08-31/pyproject.toml b/old_2026-08-31/pyproject.toml new file mode 100644 index 0000000..4c6a342 --- /dev/null +++ b/old_2026-08-31/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "lai-automation" +version = "0.1.0" +description = "Controlled coding-agent automation for Laravel AI failure reports" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "PyMySQL>=1.1,<2", + "requests>=2.32,<3", +] + +[project.optional-dependencies] +dev = ["pytest>=8.3,<9", "ruff>=0.9,<1"] +agent = ["aider-chat>=0.86,<1"] + +[project.scripts] +lai-agent = "lai_automation.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/lai_automation"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py312" diff --git a/old_2026-08-31/src/lai_automation/__init__.py b/old_2026-08-31/src/lai_automation/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/old_2026-08-31/src/lai_automation/agent.py b/old_2026-08-31/src/lai_automation/agent.py new file mode 100644 index 0000000..a68c042 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/agent.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import logging +import os +import pwd +import re +import shutil +import signal +import subprocess +import threading +import time +from collections import deque +from pathlib import Path + +from .config import Settings +from .prompt import redact + +LOG = logging.getLogger(__name__) +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +class AgentTimeoutError(RuntimeError): + pass + + +class AgentProviderError(RuntimeError): + pass + + +_PROVIDER_FAILURE = re.compile( + r"(?:litellm\.(?:InternalServerError|APIConnectionError|RateLimitError)|" + r"MistralException|The API provider's servers are down or overloaded|" + r"Name or service not known|Temporary failure in name resolution|" + r"Connection refused|Connection timed out)", + re.IGNORECASE, +) + + +class _OutputCapture: + def __init__(self, secret_values: tuple[str, ...] = (), limit: int = 24_000) -> None: + self.secret_values = tuple(value for value in secret_values if len(value) >= 8) + self.limit = limit + self.lines: deque[tuple[str, str]] = deque() + self.characters = 0 + self.line_count = 0 + self.last_activity = time.monotonic() + self.lock = threading.Lock() + + def add(self, stream_name: str, value: str) -> str: + safe_value = redact(_ANSI_ESCAPE.sub("", value.rstrip("\r\n")))[:4_000] + for secret_value in self.secret_values: + safe_value = safe_value.replace(secret_value, "[REDACTED]") + with self.lock: + self.lines.append((stream_name, safe_value)) + self.characters += len(safe_value) + self.line_count += 1 + self.last_activity = time.monotonic() + while self.lines and self.characters > self.limit: + _, removed = self.lines.popleft() + self.characters -= len(removed) + return safe_value + + def tail(self, stream_name: str | None = None, limit: int = 12_000) -> str: + with self.lock: + values = [ + value + for source, value in self.lines + if stream_name is None or source == stream_name + ] + result = "\n".join(values) + return result[-limit:] + + def activity(self) -> tuple[int, float]: + with self.lock: + return self.line_count, time.monotonic() - self.last_activity + + +class AiderAgent: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.last_run_metadata: dict = {} + + def run( + self, + workspace: Path, + task: str, + files: tuple[str, ...] = (), + log_context: str = "Agent", + ) -> str: + task_file = workspace / ".git" / "failure-task.txt" + task_file.write_text(task) + task_file.chmod(0o600) + command = self.command(task_file, files) + coder = pwd.getpwnam("coder") + git_config = Path(coder.pw_dir) / ".gitconfig" + git_config.write_text( + "[user]\n\tname = LAI Automation Coder\n\temail = coder@localhost\n" + f"[safe]\n\tdirectory = {workspace.resolve()}\n" + ) + git_config.chmod(0o600) + os.chown(git_config, coder.pw_uid, coder.pw_gid) + os.chown(task_file, coder.pw_uid, coder.pw_gid) + for path in workspace.rglob("*"): + if ".git" not in path.parts: + os.chown(path, coder.pw_uid, coder.pw_gid, follow_symlinks=False) + os.chown(workspace, coder.pw_uid, coder.pw_gid, follow_symlinks=False) + + environment = { + "HOME": coder.pw_dir, + "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), + } + for name in ( + "MISTRAL_API_KEY", + "OPENAI_API_KEY", + "OPENAI_API_BASE", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "OLLAMA_API_BASE", + ): + if os.getenv(name): + environment[name] = os.environ[name] + + capture = _OutputCapture(tuple( + environment[name] + for name in ( + "MISTRAL_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + ) + if environment.get(name) + )) + process: subprocess.Popen[str] | None = None + readers: list[threading.Thread] = [] + monitor_stopped = threading.Event() + idle_timed_out = threading.Event() + started = time.monotonic() + self.last_run_metadata = { + "model": self.settings.agent_model, + "strategy": self.edit_strategy(files), + "editable_files": list(files), + "status": "starting", + } + + def read_stream(stream, stream_name: str) -> None: + try: + for line in iter(stream.readline, ""): + safe_line = capture.add(stream_name, line) + if safe_line.strip(): + LOG.info( + "%s aider_%s %s", + log_context, + stream_name, + safe_line[:1_000], + ) + finally: + stream.close() + + def monitor() -> None: + while not monitor_stopped.wait(30): + lines, idle_seconds = capture.activity() + LOG.info( + "%s Aider pid=%s running elapsed=%ss output_lines=%s output_idle=%ss", + log_context, + process.pid if process else "pending", + round(time.monotonic() - started), + lines, + round(idle_seconds), + ) + if process and self._output_stalled(idle_seconds): + idle_timed_out.set() + LOG.warning( + "%s Aider pid=%s produced no output for %ss; terminating process group", + log_context, + process.pid, + round(idle_seconds), + ) + self._terminate_process_group(process) + return + + monitor_thread = threading.Thread(target=monitor, daemon=True) + monitor_thread.start() + try: + try: + process = subprocess.Popen( + command, + cwd=workspace, + env=environment, + text=True, + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=1, + start_new_session=True, + user=coder.pw_uid, + group=coder.pw_gid, + extra_groups=(), + ) + self.last_run_metadata.update({ + "pid": process.pid, + "status": "running", + }) + LOG.info( + "%s Aider started pid=%s model=%s strategy=%s files=%s", + log_context, + process.pid, + self.settings.agent_model, + self.edit_strategy(files), + ",".join(files) or "repository-map", + ) + assert process.stdout is not None + assert process.stderr is not None + readers = [ + threading.Thread( + target=read_stream, + args=(process.stdout, "stdout"), + daemon=True, + ), + threading.Thread( + target=read_stream, + args=(process.stderr, "stderr"), + daemon=True, + ), + ] + for reader in readers: + reader.start() + process.wait(timeout=self.settings.agent_timeout_seconds + 60) + if idle_timed_out.is_set(): + self.last_run_metadata["status"] = "idle_timeout" + output = capture.tail(limit=4_000) + detail = f"\nLast agent output:\n{output}" if output else "" + raise AgentTimeoutError( + "Aider produced no output for " + f"{self.settings.agent_idle_timeout_seconds} seconds.{detail}" + ) + except subprocess.TimeoutExpired as exception: + self.last_run_metadata["status"] = "timeout" + if process: + LOG.warning( + "%s Aider pid=%s exceeded process timeout; terminating process group", + log_context, + process.pid, + ) + self._terminate_process_group(process) + for reader in readers: + reader.join(timeout=2) + output = capture.tail(limit=4_000) + detail = f"\nLast agent output:\n{output}" if output else "" + raise AgentTimeoutError( + f"Aider exceeded the {self.settings.agent_timeout_seconds + 60}-second process " + f"timeout.{detail}" + ) from exception + finally: + monitor_stopped.set() + monitor_thread.join(timeout=1) + for reader in readers: + reader.join(timeout=1) + lines, idle_seconds = capture.activity() + self.last_run_metadata.update({ + "elapsed_seconds": round(time.monotonic() - started, 1), + "output_lines": lines, + "output_idle_seconds": round(idle_seconds, 1), + "output_tail": capture.tail(limit=4_000), + }) + shutil.rmtree(workspace / ".aider.tags.cache.v4", ignore_errors=True) + for artifact in (".aider.chat.history.md", ".aider.input.history"): + (workspace / artifact).unlink(missing_ok=True) + os.chown(workspace, 0, coder.pw_gid, follow_symlinks=False) + workspace.chmod(0o775) + assert process is not None + if process.returncode != 0: + self.last_run_metadata["status"] = "failed" + raise RuntimeError( + f"Aider failed ({process.returncode}): {capture.tail(limit=4_000)}" + ) + output = capture.tail() + if _PROVIDER_FAILURE.search(output): + self.last_run_metadata["status"] = "provider_error" + raise AgentProviderError( + "Model provider failed before producing a usable repair. Last provider output:\n" + + capture.tail(limit=4_000) + ) + self.last_run_metadata["status"] = "completed" + return output + + @staticmethod + def _terminate_process_group(process: subprocess.Popen[str]) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + LOG.error("Aider process group %s did not exit after SIGKILL", process.pid) + + def edit_strategy(self, files: tuple[str, ...]) -> str: + if self.settings.agent_model.startswith("ollama") and any( + not filename.startswith("tests/") for filename in files + ): + return "whole" + return "diff" + + def _output_stalled(self, idle_seconds: float) -> bool: + return idle_seconds >= self.settings.agent_idle_timeout_seconds + + def command(self, task_file: Path, files: tuple[str, ...] = ()) -> list[str]: + command = [ + self.settings.agent_command, + "--model", + self.settings.agent_model, + "--message-file", + str(task_file), + "--yes", + "--no-auto-commits", + "--no-dirty-commits", + "--no-git", + "--stream", + "--no-pretty", + "--no-gitignore", + "--no-check-update", + "--no-show-release-notes", + "--no-suggest-shell-commands", + "--no-detect-urls", + "--disable-playwright", + "--chat-history-file", + "/dev/null", + "--input-history-file", + "/dev/null", + "--no-analytics", + "--map-tokens", + "0" if files else "2048", + "--no-auto-lint", + "--timeout", + str(self.settings.agent_timeout_seconds), + ] + for filename in files: + if (task_file.parent.parent / filename).is_file(): + command.extend(["--file", filename]) + if self.settings.agent_model.startswith("ollama"): + command.extend(["--edit-format", self.edit_strategy(files)]) + elif files: + command.extend(["--edit-format", "diff"]) + return command diff --git a/old_2026-08-31/src/lai_automation/cli.py b/old_2026-08-31/src/lai_automation/cli.py new file mode 100644 index 0000000..bff5253 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/cli.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import argparse +import logging + +from .config import Settings +from .worker import AutomationWorker + + +def main() -> None: + parser = argparse.ArgumentParser(description="LAI controlled coding-agent automation") + parser.add_argument("command", choices=["once", "worker", "check"]) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + settings = Settings.from_env() + + if args.command == "check": + print(f"mode={settings.mode}") + print(f"source={settings.source_repository}") + print(f"model={settings.agent_model}") + print(f"direct_main_enabled={settings.allow_direct_main}") + return + + worker = AutomationWorker(settings) + if args.command == "once": + worker.run_once() + else: + worker.run_forever() + + +if __name__ == "__main__": + main() diff --git a/old_2026-08-31/src/lai_automation/config.py b/old_2026-08-31/src/lai_automation/config.py new file mode 100644 index 0000000..543ebba --- /dev/null +++ b/old_2026-08-31/src/lai_automation/config.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +_DEFAULT_SETUP_COMMANDS = ( + "composer install --no-interaction --prefer-dist;;" + "npm ci;;" + "npm run build;;" + "php artisan migrate:fresh --force" +) + + +def _bool(name: str, default: bool = False) -> bool: + return os.getenv(name, str(default)).lower() in {"1", "true", "yes", "on"} + + +def _commands(value: str) -> tuple[str, ...]: + return tuple(command.strip() for command in value.split(";;") if command.strip()) + + +@dataclass(frozen=True) +class Settings: + database_host: str + database_port: int + database_name: str + database_user: str + database_password: str + source_repository: str + source_branch: str + workspace_root: Path + mode: str + allow_direct_main: bool + gitea_url: str + gitea_owner: str + gitea_repository: str + gitea_token: str + gitea_username: str + agent_command: str + agent_model: str + agent_timeout_seconds: int + agent_idle_timeout_seconds: int + setup_commands: tuple[str, ...] + test_commands: tuple[str, ...] + test_database_host: str + test_database_port: int + test_database_name: str + test_database_user: str + test_database_password: str + test_customer_database: str + test_config_database: str + poll_seconds: int + max_changed_files: int + max_changed_lines: int + max_agent_attempts: int + stale_claim_minutes: int + dry_run: bool + + @classmethod + def from_env(cls) -> Settings: + mode = os.getenv("AUTOMATION_MODE", "dry-run").lower() + if mode not in {"dry-run", "pr", "direct-main"}: + raise ValueError("AUTOMATION_MODE must be dry-run, pr, or direct-main") + + settings = cls( + database_host=os.getenv("FAILURE_DB_HOST", "host.docker.internal"), + database_port=int(os.getenv("FAILURE_DB_PORT", "33060")), + database_name=os.getenv("FAILURE_DB_NAME", "laravel_ai_chat"), + database_user=os.getenv("FAILURE_DB_USER", "laravel"), + database_password=os.getenv("FAILURE_DB_PASSWORD", ""), + source_repository=os.getenv( + "SOURCE_REPOSITORY", "https://gitea.tisinfra.de/christian.meyer/lai.git" + ), + source_branch=os.getenv("SOURCE_BRANCH", "main"), + workspace_root=Path(os.getenv("WORKSPACE_ROOT", "/workspaces")), + mode=mode, + allow_direct_main=_bool("ALLOW_DIRECT_MAIN"), + gitea_url=os.getenv("GITEA_URL", "https://gitea.tisinfra.de"), + gitea_owner=os.getenv("GITEA_OWNER", "christian.meyer"), + gitea_repository=os.getenv("GITEA_REPOSITORY", "lai"), + gitea_token=os.getenv("GITEA_TOKEN", ""), + gitea_username=os.getenv("GITEA_USERNAME", "automation-bot"), + agent_command=os.getenv("AGENT_COMMAND", "aider"), + agent_model=os.getenv("AGENT_MODEL", "mistral/devstral-latest"), + agent_timeout_seconds=int(os.getenv("AGENT_TIMEOUT_SECONDS", "900")), + agent_idle_timeout_seconds=int(os.getenv("AGENT_IDLE_TIMEOUT_SECONDS", "300")), + setup_commands=_commands( + os.getenv("SETUP_COMMANDS", _DEFAULT_SETUP_COMMANDS) + ), + test_commands=_commands( + os.getenv("TEST_COMMANDS", "php artisan test") + ), + test_database_host=os.getenv("TEST_DB_HOST", "host.docker.internal"), + test_database_port=int(os.getenv("TEST_DB_PORT", "33060")), + test_database_name=os.getenv("TEST_DB_NAME", "laravel_ai_chat_agent_test"), + test_database_user=os.getenv("TEST_DB_USER", "lai_test_runner"), + test_database_password=os.getenv("TEST_DB_PASSWORD", ""), + test_customer_database=os.getenv("TEST_CUSTOMER_DB", "customer_agent_test"), + test_config_database=os.getenv("TEST_CONFIG_DB", "cockpit_agent_test"), + poll_seconds=int(os.getenv("POLL_SECONDS", "60")), + max_changed_files=int(os.getenv("MAX_CHANGED_FILES", "12")), + max_changed_lines=int(os.getenv("MAX_CHANGED_LINES", "800")), + max_agent_attempts=int(os.getenv("MAX_AGENT_ATTEMPTS", "2")), + stale_claim_minutes=int(os.getenv("STALE_CLAIM_MINUTES", "90")), + dry_run=mode == "dry-run", + ) + settings.validate() + return settings + + def validate(self) -> None: + if self.mode == "direct-main" and not self.allow_direct_main: + raise ValueError("direct-main requires ALLOW_DIRECT_MAIN=true") + if self.mode in {"pr", "direct-main"} and not self.gitea_token: + raise ValueError(f"{self.mode} requires GITEA_TOKEN") + if not self.database_password: + raise ValueError("FAILURE_DB_PASSWORD is required") + if not self.test_database_password: + raise ValueError("TEST_DB_PASSWORD is required") + if not self.test_commands: + raise ValueError("TEST_COMMANDS must contain at least one command") + test_databases = { + "TEST_DB_NAME": self.test_database_name, + "TEST_CUSTOMER_DB": self.test_customer_database, + "TEST_CONFIG_DB": self.test_config_database, + } + for setting_name, database_name in test_databases.items(): + if "test" not in database_name.lower(): + raise ValueError(f"{setting_name} must name an isolated test database") + if database_name == self.database_name: + raise ValueError(f"{setting_name} must differ from FAILURE_DB_NAME") + if len(set(test_databases.values())) != len(test_databases): + raise ValueError("TEST_DB_NAME, TEST_CUSTOMER_DB, and TEST_CONFIG_DB must be distinct") + if not 60 <= self.agent_timeout_seconds <= 3600: + raise ValueError("AGENT_TIMEOUT_SECONDS must be between 60 and 3600") + if not 60 <= self.agent_idle_timeout_seconds <= self.agent_timeout_seconds: + raise ValueError( + "AGENT_IDLE_TIMEOUT_SECONDS must be between 60 and AGENT_TIMEOUT_SECONDS" + ) + if not 1 <= self.max_agent_attempts <= 7: + raise ValueError("MAX_AGENT_ATTEMPTS must be between 1 and 7") diff --git a/src/lai_automation/database.py b/old_2026-08-31/src/lai_automation/database.py similarity index 86% rename from src/lai_automation/database.py rename to old_2026-08-31/src/lai_automation/database.py index 6e59aeb..80011f9 100644 --- a/src/lai_automation/database.py +++ b/old_2026-08-31/src/lai_automation/database.py @@ -76,12 +76,28 @@ class FailureRepository: self._finish(report_id, "open", {"last_dry_run": reason}) def complete(self, report_id: int, metadata: dict) -> None: - self._finish(report_id, "resolved", metadata) + self._finish(report_id, "resolved", {**metadata, "agent_error": None}) def fail(self, report_id: int, error: str, metadata: dict | None = None) -> None: payload = {"agent_error": error, **(metadata or {})} self._finish(report_id, "agent_failed", payload) + def heartbeat(self, report_id: int) -> None: + with self.connection() as connection, connection.cursor() as cursor: + try: + cursor.execute( + """ + UPDATE failure_reports + SET updated_at = NOW() + WHERE id = %s AND status = 'agent_running' + """, + (report_id,), + ) + connection.commit() + except Exception: + connection.rollback() + raise + def _finish(self, report_id: int, status: str, additions: dict) -> None: with self.connection() as connection, connection.cursor() as cursor: try: diff --git a/old_2026-08-31/src/lai_automation/git.py b/old_2026-08-31/src/lai_automation/git.py new file mode 100644 index 0000000..284cf7a --- /dev/null +++ b/old_2026-08-31/src/lai_automation/git.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import os +import pwd +import shlex +import shutil +import subprocess +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlparse + +from .config import Settings + + +@dataclass(frozen=True) +class ChangeSummary: + files: tuple[str, ...] + added_lines: int + deleted_lines: int + + @property + def changed_lines(self) -> int: + return self.added_lines + self.deleted_lines + + +class GitWorkspace: + def __init__(self, settings: Settings, report_id: int) -> None: + self.settings = settings + self.path = settings.workspace_root / f"failure-{report_id}" + self.branch = f"ai/failure-{report_id}" + + def prepare(self) -> None: + repository = urlparse(self.settings.source_repository) + gitea = urlparse(self.settings.gitea_url) + if repository.scheme != "https" or repository.hostname != gitea.hostname: + raise ValueError("SOURCE_REPOSITORY must use HTTPS on the configured Gitea host") + shutil.rmtree(self.path, ignore_errors=True) + self.path.parent.mkdir(parents=True, exist_ok=True) + clone = self._git_authenticated if self.settings.gitea_token else self._git + try: + clone( + "clone", + "--branch", + self.settings.source_branch, + "--single-branch", + self.settings.source_repository, + str(self.path), + cwd=self.path.parent, + ) + except subprocess.CalledProcessError as exception: + output = "\n".join(filter(None, (exception.stdout, exception.stderr))).strip() + if len(output) > 4_000: + output = output[-4_000:] + raise RuntimeError( + "Source repository clone failed" + + (f":\n{output}" if output else " without diagnostic output") + ) from exception + self._git("config", "user.name", "LAI Automation") + self._git("config", "user.email", "automation@localhost") + self._git("checkout", "-b", self.branch) + + def run( + self, + command: str, + check: bool = True, + username: str | None = None, + extra_environment: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + return self.run_args( + shlex.split(command), + check=check, + username=username, + extra_environment=extra_environment, + ) + + def run_args( + self, + command: Sequence[str], + check: bool = True, + username: str | None = None, + extra_environment: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + launch_identity = {} + if username: + account = pwd.getpwnam(username) + environment = { + "HOME": account.pw_dir, + "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), + } + launch_identity = { + "user": account.pw_uid, + "group": account.pw_gid, + "extra_groups": (), + } + + environment.update(extra_environment or {}) + + return subprocess.run( + command, + cwd=self.path, + env=environment, + text=True, + capture_output=True, + check=check, + timeout=1800, + **launch_identity, + ) + + def discard_changes(self) -> None: + self._git("reset", "--hard", "HEAD") + self._git("clean", "-fd") + + def checkpoint(self) -> str: + self._git("add", "--all") + return self._git("stash", "create").stdout.strip() + + def restore_checkpoint(self, checkpoint: str | None) -> None: + self.discard_changes() + if checkpoint: + self._git("stash", "apply", checkpoint) + + def summary(self) -> ChangeSummary: + self._git("add", "-N", "--all") + try: + output = self._git("diff", "HEAD", "--numstat").stdout + except subprocess.CalledProcessError: + output = self._git("diff", "--numstat").stdout + files: list[str] = [] + added = deleted = 0 + for line in output.splitlines(): + add, remove, filename = line.split("\t", 2) + files.append(filename) + added += int(add) if add.isdigit() else 0 + deleted += int(remove) if remove.isdigit() else 0 + return ChangeSummary(tuple(files), added, deleted) + + def diff(self, limit: int = 12_000) -> str: + try: + output = self._git("diff", "HEAD", "--no-ext-diff", "--unified=3").stdout + except subprocess.CalledProcessError: + output = self._git("diff", "--no-ext-diff", "--unified=3").stdout + if len(output) <= limit: + return output + half = limit // 2 + return output[:half] + "\n...[diff middle omitted]...\n" + output[-half:] + + def added_text(self) -> str: + output = self._git("diff", "HEAD", "--no-ext-diff", "--unified=0").stdout + return "\n".join( + line[1:] + for line in output.splitlines() + if line.startswith("+") and not line.startswith("+++") + ) + + def head_file(self, filename: str) -> str | None: + result = subprocess.run( + ["git", "show", f"HEAD:{filename}"], + cwd=self.path, + text=True, + capture_output=True, + check=False, + timeout=300, + ) + return result.stdout if result.returncode == 0 else None + + def commit(self, report_id: int) -> str: + self._git("add", "--all") + self._git("commit", "-m", f"Fix failure report #{report_id}") + return self._git("rev-parse", "HEAD").stdout.strip() + + def push_branch(self) -> None: + self._git_authenticated("push", "origin", f"HEAD:{self.branch}") + + def push_main(self) -> None: + self._git_authenticated("push", "origin", f"HEAD:{self.settings.source_branch}") + + def cleanup(self) -> None: + if self.path.exists(): + shutil.rmtree(self.path) + + def _git(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=cwd or self.path, + text=True, + capture_output=True, + check=True, + timeout=300, + ) + + def _git_authenticated( + self, + *args: str, + cwd: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + working_directory = cwd or self.path + auth_directory = Path(tempfile.mkdtemp( + prefix=".lai-git-auth-", + dir=working_directory, + )) + askpass = auth_directory / "askpass.sh" + username_file = auth_directory / "username" + token_file = auth_directory / "token" + try: + username_file.write_text(self.settings.gitea_username) + token_file.write_text(self.settings.gitea_token) + username_file.chmod(0o600) + token_file.chmod(0o600) + askpass.write_text( + "#!/bin/sh\ncase \"$1\" in *Username*) cat \"$GIT_BOT_USER_FILE\";; *) cat \"$GIT_BOT_TOKEN_FILE\";; esac\n" + ) + askpass.chmod(0o700) + env = { + "HOME": os.environ.get("HOME", "/root"), + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "GIT_ASKPASS": str(askpass), + "GIT_TERMINAL_PROMPT": "0", + "GIT_BOT_USER_FILE": str(username_file), + "GIT_BOT_TOKEN_FILE": str(token_file), + } + return subprocess.run( + ["git", *args], + cwd=working_directory, + env=env, + text=True, + capture_output=True, + check=True, + timeout=300, + ) + finally: + shutil.rmtree(auth_directory, ignore_errors=True) diff --git a/old_2026-08-31/src/lai_automation/gitea.py b/old_2026-08-31/src/lai_automation/gitea.py new file mode 100644 index 0000000..02e037f --- /dev/null +++ b/old_2026-08-31/src/lai_automation/gitea.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .config import Settings + + +class GiteaClient: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.session = requests.Session() + self.session.headers.update({ + "Authorization": f"token {settings.gitea_token}", + "Accept": "application/json", + }) + retry = Retry( + total=3, + backoff_factor=1, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("POST",), + ) + self.session.mount("https://", HTTPAdapter(max_retries=retry)) + + def create_pull_request(self, branch: str, title: str, body: str) -> str: + url = ( + f"{self.settings.gitea_url.rstrip('/')}/api/v1/repos/" + f"{self.settings.gitea_owner}/{self.settings.gitea_repository}/pulls" + ) + response = self.session.post( + url, + json={ + "base": self.settings.source_branch, + "head": branch, + "title": title, + "body": body, + }, + timeout=30, + ) + response.raise_for_status() + payload = response.json() + pull_request_url = payload.get("html_url") or payload.get("url") + if not pull_request_url: + raise ValueError("Gitea response did not contain a pull request URL") + return pull_request_url diff --git a/old_2026-08-31/src/lai_automation/models.py b/old_2026-08-31/src/lai_automation/models.py new file mode 100644 index 0000000..4dfc726 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/models.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class FailureReport: + id: int + chat_message_id: int | None + source: str + category: str + user_feedback: str | None + feedback_type: str | None + expected_behavior: str | None + prompt: str | None + response: str | None + sql_query: str | None + error_message: str | None + context: dict[str, Any] + + @classmethod + def from_row(cls, row: dict[str, Any]) -> FailureReport: + context = row.get("context") or {} + if isinstance(context, str): + context = json.loads(context) + return cls( + id=row["id"], + chat_message_id=row.get("chat_message_id"), + source=row["source"], + category=row["category"], + user_feedback=row.get("user_feedback"), + feedback_type=row.get("feedback_type"), + expected_behavior=row.get("expected_behavior"), + prompt=row.get("prompt"), + response=row.get("response"), + sql_query=row.get("sql_query"), + error_message=row.get("error_message"), + context=context, + ) diff --git a/old_2026-08-31/src/lai_automation/policy.py b/old_2026-08-31/src/lai_automation/policy.py new file mode 100644 index 0000000..29024df --- /dev/null +++ b/old_2026-08-31/src/lai_automation/policy.py @@ -0,0 +1,788 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from difflib import SequenceMatcher +from pathlib import Path, PurePosixPath + +from .config import Settings +from .git import ChangeSummary, GitWorkspace +from .models import FailureReport +from .prompt import ( + php_negative_assertion, + requires_explicit_route_repair, + requires_table_output_repair, +) +from .removal import ( + find_matching_reference, + is_removal_only, + normalize_source_layout, + normalize_sql, + removal_directives, + remove_sql_fragment, + sql_contains_fragment, + without_reference_sql, +) + +_ALWAYS_DENIED = ( + ".env", + "vendor/", + "storage/", + "bootstrap/cache/", + "docker/", + "docker-compose", + ".gitea/", + ".github/", + ".aider", +) + +_DIRECT_MAIN_DENIED = ( + "database/migrations/", + "app/Http/Middleware/", + "app/Models/User.php", + "app/Services/PermissionService.php", + "app/Services/SQLValidatorService.php", + "config/auth.php", + "config/database.php", + "routes/", +) + + +@dataclass(frozen=True) +class PolicyResult: + allowed: bool + errors: tuple[str, ...] + + +def evaluate_changes( + summary: ChangeSummary, + settings: Settings, + workspace: Path | None = None, +) -> PolicyResult: + errors: list[str] = [] + if not summary.files: + errors.append("Agent produced no file changes") + elif all(filename.startswith("tests/") for filename in summary.files): + errors.append("Agent changed only tests; production behavior was not fixed") + if len(summary.files) > settings.max_changed_files: + errors.append(f"Changed {len(summary.files)} files; limit is {settings.max_changed_files}") + if summary.changed_lines > settings.max_changed_lines: + errors.append(f"Changed {summary.changed_lines} lines; limit is {settings.max_changed_lines}") + + for filename in summary.files: + if PurePosixPath(filename).is_absolute() or ".." in PurePosixPath(filename).parts: + errors.append(f"Invalid changed path: {filename}") + continue + if workspace and (workspace / filename).is_symlink(): + errors.append(f"Symbolic links are not allowed: {filename}") + continue + if any(filename == prefix or filename.startswith(prefix) for prefix in _ALWAYS_DENIED): + errors.append(f"Denied path changed: {filename}") + if settings.mode == "direct-main" and any( + filename == prefix or filename.startswith(prefix) for prefix in _DIRECT_MAIN_DENIED + ): + errors.append(f"Direct-main denied path changed: {filename}") + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_report_scope(report: FailureReport, summary: ChangeSummary) -> PolicyResult: + expected = f"{report.user_feedback or ''}\n{report.expected_behavior or ''}".lower() + requires_cross_database_orchestration = ( + "cockpit_config" in expected and ("fk_" in expected or "erp" in expected) + ) + has_application_change = any(filename.startswith("app/") for filename in summary.files) + errors = [] + if requires_cross_database_orchestration and not has_application_change: + errors.append( + "Cross-database report changed no application code; inspect routing and implement staged orchestration" + ) + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_routed_files( + summary: ChangeSummary, + routed_files: tuple[str, ...], +) -> PolicyResult: + if not routed_files: + return PolicyResult(True, ()) + unexpected = sorted(set(summary.files) - set(routed_files)) + if not unexpected: + return PolicyResult(True, ()) + return PolicyResult(False, ( + "Agent changed files outside the routed repair scope: " + ", ".join(unexpected), + )) + + +def evaluate_requested_behavior_scope( + report: FailureReport, + summary: ChangeSummary, + added_text: str, +) -> PolicyResult: + errors: list[str] = [] + changed_files = set(summary.files) + added_lower = added_text.casefold() + + if requires_table_output_repair(report): + required_files = { + "app/Services/AIService.php", + "tests/Feature/AIPrivacyBoundaryTest.php", + } + missing_files = sorted(required_files - changed_files) + if missing_files: + errors.append( + "Table-presentation requirement must change the real answer formatter and its " + "provider-backed regression test; missing: " + ", ".join(missing_files) + ) + if report.prompt and report.prompt.casefold() not in added_lower: + errors.append( + "Table-presentation regression test must use the exact reported user prompt so " + "grouped output is exercised without an explicit table keyword" + ) + if "markdown-tabelle" not in added_lower and "markdown table" not in added_lower: + errors.append( + "Table-presentation repair must assert a Markdown-table answer instruction" + ) + if not re.search(r"count\s*\(\s*\$rows\s*\)\s*>\s*1", added_text, re.IGNORECASE): + errors.append( + "Grouped-series table detection must require multiple rows with count($rows) > 1; " + "do not turn every multi-column scalar/detail result into a table" + ) + if not re.search( + r"single[_\s-]row|one[_\s-]row|einzelne[nr]?\s+(?:zeile|datensatz)", + added_text, + re.IGNORECASE, + ): + errors.append( + "Add a negative regression test proving one ordinary multi-column result row keeps " + "the sentence-format instruction" + ) + if re.search( + r"assertStringContainsString\s*\(\s*['\"]alle bereitgestellten Zeilen", + added_text, + re.IGNORECASE, + ): + errors.append( + "The table regression must assert the exact grammatical substring " + "'allen bereitgestellten Zeilen', not 'alle bereitgestellten Zeilen'" + ) + row_count_assertions = ( + ( + r"assertSame\s*\(\s*3\s*,\s*substr_count\s*\(\s*\$userPrompt\s*,\s*" + r"['\"](?:\\?['\"])?Monat(?:\\?['\"])?['\"]\s*\)\s*\)" + ), + ( + r"assertSame\s*\(\s*3\s*,\s*substr_count\s*\(\s*\$userPrompt\s*,\s*" + r"['\"](?:\\?['\"])?Anzahl_Lieferscheine(?:\\?['\"])?['\"]\s*\)\s*\)" + ), + ) + if any(not re.search(pattern, added_text, re.IGNORECASE) for pattern in row_count_assertions): + errors.append( + "Grouped-table provider regression must assert every supplied row reaches " + "$userPrompt; a static mocked Markdown response is vacuous" + ) + + if requires_explicit_route_repair(report): + desired = "\n".join(filter(None, ( + report.user_feedback, + report.expected_behavior, + ))).casefold() + required_files = { + "app/Http/Controllers/ChatController.php", + "tests/Unit/ChatControllerSourceResolutionTest.php", + } + if "www:" in desired: + required_files.update({ + "app/Services/WebSearchService.php", + "tests/Unit/WebSearchServiceTest.php", + }) + missing_files = sorted(required_files - changed_files) + if missing_files: + errors.append( + "Explicit route-prefix requirement must change ChatController routing and its " + "deterministic regression test; missing: " + ", ".join(missing_files) + ) + missing_prefixes = [ + prefix + for prefix in ("cockpit:", "www:", "direct:") + if prefix in desired and prefix not in added_lower + ] + if missing_prefixes: + errors.append( + "Explicit route-prefix regression coverage is missing: " + + ", ".join(missing_prefixes) + ) + required_route_fragments = ( + ("resolveRoutePrefix", r"resolveRoutePrefix"), + ("database tuple", r"\[\s*['\"]database['\"]\s*,"), + ("research tuple", r"\[\s*['\"]research['\"]\s*,"), + ("direct tuple", r"\[\s*['\"]direct['\"]\s*,"), + ("no-prefix tuple", r"\[\s*null\s*,"), + ("web research call", r"generateWebResearchResponse\s*\("), + ("direct model call", r"generateDirectResponse\s*\("), + ( + "destructured processMessage call", + ( + r"\[\s*\$explicitRoute\s*,\s*\$message\s*\]\s*=\s*" + r"\$this->resolveRoutePrefix\(\$[A-Za-z_]\w*\)" + ), + ), + ("explicit route fallback", r"\$route\s*=\s*\$explicitRoute\s*\?\?"), + ) + missing_route_fragments = [ + label + for label, pattern in required_route_fragments + if not re.search(pattern, added_text, re.IGNORECASE) + ] + if missing_route_fragments: + errors.append( + "Explicit route-prefix repair must use one tested resolveRoutePrefix helper that " + "is destructured by processMessage before route fallback; missing: " + + ", ".join(missing_route_fragments) + ) + process_message_calls = len(re.findall(r"->processMessage\s*\(", added_text)) + never_classifier_expectations = len(re.findall( + r"expects\s*\([^\n;]*never\s*\(\s*\)[^;]*\)\s*" + r"->method\s*\(\s*['\"]determineRoute['\"]\s*\)", + added_text, + re.IGNORECASE, + )) + if process_message_calls < 3 or never_classifier_expectations < 3: + errors.append( + "Explicit prefix regression tests must call processMessage() for cockpit:, www:, " + "and direct: and assert AIService::determineRoute() is never called for each path; " + "helper-only tests do not prove request routing" + ) + cockpit_test = re.search( + r"function\s+test[^\{]*cockpit[^\{]*\(\)\s*:\s*void\s*\{(?P.*?)\n\s*\}", + added_text, + re.IGNORECASE | re.DOTALL, + ) + cockpit_body = cockpit_test.group("body") if cockpit_test else "" + required_false_heuristics = ( + "requiresCustomerDatabase", + "requiresConfigDatabase", + "requiresErpDatabase", + ) + if not cockpit_test or any( + not re.search( + rf"method\s*\(\s*['\"]{method}['\"]\s*\).*?willReturn\s*\(\s*false\s*\)", + cockpit_body, + re.IGNORECASE | re.DOTALL, + ) + for method in required_false_heuristics + ): + errors.append( + "The cockpit processMessage regression must force every normal database heuristic " + "false so only the explicit prefix can select the database route" + ) + forbidden_route_patterns = ( + r"resolveSource\s*\(\s*['\"](?:cockpit|www|direct):", + r"function\s+determineRouteForMessage\b", + r"\$this->defaultModel\b", + r"\$this->determineRoute\s*\(", + r"\$route\s*=\s*\$prefixRoute\s*\?\?", + r"return\s+\$this->resolveRoutePrefix\s*\(", + r"['\"]message['\"]\s*=>\s*\$originalMessage", + r"assertStringContainsString\s*\(\s*['\"](?:cockpit|www|direct):", + r"->json\s*\(", + ) + if any(re.search(pattern, added_text, re.IGNORECASE) for pattern in forbidden_route_patterns): + errors.append( + "Explicit prefix tests must exercise resolveRoutePrefix and the real request route; " + "do not test resolveDatabaseSource directly or add wrappers for nonexistent " + "ChatController model-routing members" + ) + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_removal_scope( + report: FailureReport, + summary: ChangeSummary, + workspace: GitWorkspace, +) -> PolicyResult: + if not is_removal_only(report): + return PolicyResult(True, ()) + + errors: list[str] = [] + production_files = [filename for filename in summary.files if not filename.startswith("tests/")] + test_files = [filename for filename in summary.files if filename.startswith("tests/")] + if len(production_files) != 1: + errors.append( + "Removal-only repair must change exactly one owning production file" + ) + return PolicyResult(False, tuple(errors)) + if not test_files: + errors.append("Removal-only repair must add a focused negative regression assertion") + + filename = production_files[0] + original = workspace.head_file(filename) + path = workspace.path / filename + if original is None or not path.is_file(): + errors.append(f"Could not compare removal-only source file with HEAD: {filename}") + return PolicyResult(False, tuple(errors)) + current = path.read_text(encoding="utf-8") + original_example, _ = find_matching_reference(original, report.prompt) + current_example, _ = find_matching_reference(current, report.prompt) + if not original_example or not current_example: + errors.append( + "Removal-only repair must update the uniquely matching tracked reference SQL" + ) + return PolicyResult(False, tuple(errors)) + + expected_sql = original_example.sql + for directive in removal_directives(report): + if not sql_contains_fragment(original_example.sql, directive.fragment): + errors.append( + f"Forbidden SQL fragment was already absent before the agent edit: {directive.fragment}" + ) + continue + if sql_contains_fragment(current_example.sql, directive.fragment): + errors.append( + f"Removal-only repair retained forbidden SQL fragment: {directive.fragment}" + ) + expected_sql = remove_sql_fragment(expected_sql, directive.fragment) + + if normalize_sql(current_example.sql) != normalize_sql(expected_sql): + errors.append( + "Removal-only repair changed joins, fields, filters, grouping, ordering, or other SQL " + "outside the explicitly forbidden fragment" + ) + original_outside_sql = without_reference_sql(original, original_example) + current_outside_sql = without_reference_sql(current, current_example) + if normalize_source_layout(original_outside_sql) != normalize_source_layout(current_outside_sql): + errors.append( + "Removal-only repair changed production content outside the matching reference SQL" + ) + + for test_file in test_files: + previous_test = workspace.head_file(test_file) + current_test_path = workspace.path / test_file + if previous_test is None or not current_test_path.is_file(): + continue + current_test = current_test_path.read_text(encoding="utf-8") + if not _lines_are_subsequence(previous_test, current_test): + errors.append( + f"Removal-only repair removed or rewrote existing regression coverage: {test_file}" + ) + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_removal_prevention( + report: FailureReport, + summary: ChangeSummary, + workspace: GitWorkspace, +) -> PolicyResult: + directives = removal_directives(report) + if not directives or is_removal_only(report): + return PolicyResult(True, ()) + + protected_references: list[tuple[str, str, str]] = [] + production_additions: list[str] = [] + test_additions: list[str] = [] + for filename in summary.files: + original = workspace.head_file(filename) + path = workspace.path / filename + if original is None or not path.is_file(): + continue + current = path.read_text(encoding="utf-8") + additions = "\n".join(_added_lines(original, current)) + if filename.startswith("tests/"): + test_additions.append(additions) + continue + production_additions.append(additions) + original_example, _ = find_matching_reference(original, report.prompt) + if original_example and all( + not sql_contains_fragment(original_example.sql, directive.fragment) + for directive in directives + ): + current_example, _ = find_matching_reference(current, report.prompt) + protected_references.append(( + filename, + original_example.sql, + current_example.sql if current_example else "", + )) + + if not protected_references: + return PolicyResult(True, ()) + + errors: list[str] = [] + for filename, original_sql, current_sql in protected_references: + if not current_sql or normalize_sql(current_sql) != normalize_sql(original_sql): + errors.append( + "The matching reference SQL was already corrected before this report and is a " + f"protected baseline; preserve it exactly instead of rewriting it: {filename}" + ) + + added_production = "\n".join(production_additions) + added_tests = "\n".join(test_additions) + production_lines = added_production.splitlines() + directive_identifiers = { + _directive_identifier(directive.fragment).casefold() + for directive in directives + } + unrelated_rules = [ + line.strip() + for line in production_lines + if _looks_like_php_rule(line) + and not any(identifier in line.casefold() for identifier in directive_identifiers) + ] + if unrelated_rules: + errors.append( + "Removal-prevention repair added unrelated business guidance. Add only the narrow rule " + "that prohibits the reported identifier; preserve existing date, document-code, lookup, " + "status, and exclusion semantics. Remove: " + " | ".join(unrelated_rules) + ) + for directive in directives: + identifier = _directive_identifier(directive.fragment) + related_rules = [ + line + for line in production_lines + if identifier.casefold() in line.casefold() and _has_negative_language(line) + ] + if not related_rules: + errors.append( + "Add narrow reusable production guidance that explicitly prohibits using " + f"{identifier} for this business case" + ) + if any(sql_contains_fragment(line, directive.fragment) for line in related_rules): + errors.append( + "Reusable production guidance must not embed the complete forbidden predicate " + f"{directive.fragment}; mention {identifier} and the prohibition without reproducing " + "the full predicate, otherwise a real context-level negative assertion cannot pass" + ) + + positive_context_literals = _positive_context_assertion_literals(added_tests) + if not any( + len(literal.strip()) >= 12 and literal in added_production + for literal in positive_context_literals + ): + errors.append( + "The focused context-builder test is vacuous: it only checks text that was already " + "absent from the old context. Add assertStringContainsString for unique text from the " + "new production prohibition and pass every table declared by that rule to " + "businessContext(), while retaining the exact negative assertion" + ) + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_expected_behavior(report: FailureReport, added_text: str) -> PolicyResult: + expected = report.expected_behavior or "" + if not expected.strip(): + return PolicyResult(True, ()) + if not isinstance(added_text, str): + added_text = "" + + directives = removal_directives(report) + forbidden_text = "\n".join(item.fragment for item in directives).lower() + + technical_terms = set(re.findall( + r"\b(?:[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+|[A-Za-z]+[A-Z][A-Za-z]+|\d+)\b", + expected, + )) + technical_terms = { + term + for term in technical_terms + if term.lower() not in forbidden_text and not term.isupper() + } + expected_lower = expected.lower() + for status_filter in ("obsolet", "storniert", "gebucht"): + if status_filter in expected_lower and status_filter not in forbidden_text: + technical_terms.add(status_filter) + + added_lower = added_text.lower() + missing = sorted(term for term in technical_terms if term.lower() not in added_lower) + errors = [] + if missing: + errors.append( + "Repair does not cover explicit expected-behavior terms in added code and assertions: " + + ", ".join(missing) + ) + + expected_identifiers = [ + identifier + for identifier in re.findall( + r"\b[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+\b|\b[A-Za-z]+[A-Z][A-Za-z]+\b", + expected, + ) + if identifier.lower() not in forbidden_text and not identifier.isupper() + ] + for identifier in expected_identifiers: + escaped = re.escape(identifier) + if re.search( + rf"(?:never|niemals)\s+(?:\w+\s+){{0,6}}{escaped}", + added_text, + re.IGNORECASE, + ): + errors.append(f"Repair contradicts expected identifier {identifier}") + + numeric_filters = [ + pair + for pair in re.findall( + r"\b([A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+|[A-Za-z]+[A-Z][A-Za-z]+)\s+(\d+)\b", + expected, + ) + if pair[0].lower() not in forbidden_text + ] + document_codes = () if "auftragskennung" in forbidden_text else re.finditer( + r"\bauftragskennung\s+(\d+)\b", + expected, + re.IGNORECASE, + ) + for match in document_codes: + pair = ("AuftragsKennung", match.group(1)) + if not any( + identifier.lower() == pair[0].lower() and value == pair[1] + for identifier, value in numeric_filters + ): + numeric_filters.append(pair) + technical_terms.add(pair[0]) + for identifier, value in numeric_filters: + if not re.search( + rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b", + added_text, + re.IGNORECASE, + ): + errors.append(f"Repair must express {identifier} = {value} explicitly") + + if "obsolet" in expected_lower and "obsolet" not in forbidden_text and not re.search( + r"COALESCE\s*\([^)]*(? bool: + if any( + not re.search( + rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b", + candidate, + re.IGNORECASE, + ) + for identifier, value in numeric_filters + ): + return False + if any(identifier.lower() not in candidate.lower() for identifier in expected_sql_identifiers): + return False + if ( + "obsolet" in expected_lower + and "obsolet" not in forbidden_text + and not re.search( + r"COALESCE\s*\([^)]*(? bool: + return bool(re.search( + r"assert\w*(?:not|false)|notcontains|doesnotcontain|\bnot\s+in\b", + line, + re.IGNORECASE, + )) + + +def _directive_identifier(fragment: str) -> str: + identifiers = re.findall( + r"(?:[A-Za-z_][A-Za-z0-9_]*\.)?([A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+)", + fragment, + ) + return identifiers[-1] if identifiers else fragment + + +def _has_negative_language(line: str) -> bool: + return bool(re.search( + r"\b(?:nicht|nie|niemals|kein\w*|ohne|verboten|" + r"never|not|without|forbid\w*|prohibit\w*)\b", + line, + re.IGNORECASE, + )) + + +def _looks_like_php_rule(line: str) -> bool: + stripped = line.strip() + return bool( + "=>" not in stripped + and re.fullmatch(r"['\"].{8,}['\"],", stripped) + ) + + +def _added_lines(original: str, current: str) -> tuple[str, ...]: + before = original.splitlines() + after = current.splitlines() + additions: list[str] = [] + for tag, _before_start, _before_end, after_start, after_end in SequenceMatcher( + None, + before, + after, + autojunk=False, + ).get_opcodes(): + if tag in {"insert", "replace"}: + additions.extend(after[after_start:after_end]) + return tuple(additions) + + +def _positive_context_assertion_literals(text: str) -> tuple[str, ...]: + literals: list[str] = [] + patterns = ( + r"assertStringContainsString\(\s*'((?:\\.|[^'\\])*)'\s*,\s*\$\w*context\w*", + r'assertStringContainsString\(\s*"((?:\\.|[^"\\])*)"\s*,\s*\$\w*context\w*', + ) + for pattern in patterns: + for match in re.finditer(pattern, text, re.IGNORECASE): + literals.append( + match.group(1).replace("\\'", "'").replace('\\"', '"').replace("\\\\", "\\") + ) + return tuple(literals) + + +def _without_identifier_qualifiers(fragment: str) -> str: + return re.sub( + r"\b[A-Za-z_][A-Za-z0-9_]*\.(?=[A-Za-z_][A-Za-z0-9_]*)", + "", + fragment, + ) + + +def _lines_are_subsequence(original: str, current: str) -> bool: + original_lines = [line.strip() for line in original.splitlines() if line.strip()] + current_lines = iter(line.strip() for line in current.splitlines() if line.strip()) + return all(any(candidate == expected for candidate in current_lines) for expected in original_lines) diff --git a/old_2026-08-31/src/lai_automation/prompt.py b/old_2026-08-31/src/lai_automation/prompt.py new file mode 100644 index 0000000..e64d7c0 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/prompt.py @@ -0,0 +1,451 @@ +from __future__ import annotations + +import re + +from .models import FailureReport +from .removal import is_removal_only, removal_directives + +_SECRET_PATTERNS = ( + ( + re.compile(r"(?i)\b(?:mysql|mariadb|postgres(?:ql)?|sqlserver)://[^\s]+"), + "[DATABASE_URL]", + ), + (re.compile(r"(?i)(password|passwd|secret|token|api[_ -]?key)\s*[:=]\s*\S+"), r"\1=[REDACTED]"), + (re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "[EMAIL]"), + (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "[IP]"), + (re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer [REDACTED]"), + (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[AWS_KEY]"), + (re.compile(r"-----BEGIN [A-Z ]+KEY-----[\s\S]+?-----END [A-Z ]+KEY-----"), "[PRIVATE_KEY]"), + (re.compile(r"\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), "[JWT]"), +) + +_REPOSITORY_PLAYBOOK = """ +Repository ownership map: +- app/Services/SQLGeneratorService.php assembles live schema and business context for SQL generation. +- config/erp_semantics.php owns ERP joins, metrics, formulas, filters, and reference examples. +- config/cockpit_semantics.php owns device and customer-database semantics. +- app/Services/AIService.php owns provider calls, pseudonymization boundaries, and response parsing. +- app/Services/TenantDatabaseService.php owns ERP/config/customer source selection and tenant metadata. +- Cross-database workflows are orchestrated in ChatController/TenantDatabaseService; one SQL query + cannot join tables exposed through different Laravel connections unless the tracked architecture + explicitly supports qualified access. +- app/Services/SQLValidatorService.php owns read-only SQL safety; do not weaken it to accept a fix. +- app/Http/Controllers/ChatController.php orchestrates routing, execution, answers, and diagnostics. +- app/Services/FailureTrackingService.php and FailureAutomationService.php own failure handoff. +- tests/Unit is preferred for deterministic service/context rules; tests/Feature is for HTTP, + persistence, authorization, or real isolated-database behavior. + +Repair workflow: +1. Inspect the owning implementation, its configuration, and the nearest existing test first. +2. State one root-cause hypothesis and make the smallest change that can prove it. +3. Put reusable business knowledge in tracked semantics configuration, not question-specific code. +4. A prompt/context fix must test the real generated context. A deterministic SQL builder must test + its exact SQL and execute it against synthetic fixtures when useful. +5. Tests must contain assertions that execute; reject empty loops and broad unrelated fallbacks. +6. Do not expose protected production methods publicly for tests. Use a test-only subclass. +7. Do not duplicate one rule in service code and semantics config, or add explanatory comments for + self-evident test setup. +8. Inspect the final diff for malformed edits, unrelated files, hardcoded report values, and tests + that pass without exercising the changed behavior. Before retrying an edit, reread the current + file and never leave conflict markers or partial methods behind. +9. PHP config files are executable code. Preserve brackets, commas, quotes, and array nesting; run + `php -l` on every changed PHP file before broader tests. +10. The orchestrator prepares dependencies, frontend assets, migrations, and isolated synthetic test + schemas. Never require or copy production rows. Build deterministic fixtures for data-dependent + regressions and do not edit generated assets, vendor files, node_modules, or environment files. +""" + + +def redact(value: str | None) -> str: + result = value or "" + for pattern, replacement in _SECRET_PATTERNS: + result = pattern.sub(replacement, result) + return result[:12_000] + + +def php_negative_assertion(fragment: str) -> str: + escaped = fragment.replace("\\", "\\\\").replace("'", "\\'") + return f"$this->assertStringNotContainsString('{escaped}', $context);" + + +def requires_table_output_repair(report: FailureReport) -> bool: + desired = _desired_report_text(report) + return bool(re.search( + r"\b(?:markdown[- ]?tabelle|tabelle|tabellar(?:isch|ische|ischer|ischen|er)?)\b", + desired, + re.IGNORECASE, + )) + + +def requires_explicit_route_repair(report: FailureReport) -> bool: + desired = _desired_report_text(report).casefold() + return any(prefix in desired for prefix in ("cockpit:", "www:", "direct:")) + + +def _desired_report_text(report: FailureReport) -> str: + return "\n".join(filter(None, ( + report.prompt, + report.user_feedback, + report.expected_behavior, + ))) + + +def _declares_data_correct(report: FailureReport) -> bool: + desired = _desired_report_text(report) + return bool(re.search( + r"\b(?:die\s+)?daten\s+(?:sind|waren)\s+korrekt\b|" + r"\b(?:the\s+)?data\s+(?:is|are|was|were)\s+correct\b", + desired, + re.IGNORECASE, + )) + + +def repair_files(report: FailureReport) -> tuple[str, ...]: + expected = f"{report.user_feedback or ''}\n{report.expected_behavior or ''}".lower() + report_text = "\n".join(filter(None, ( + report.prompt, + report.user_feedback, + report.expected_behavior, + report.sql_query, + report.error_message, + ))).lower() + if "cockpit_config" in expected and ("fk_" in expected or "erp" in expected): + return ( + "app/Http/Controllers/ChatController.php", + "app/Services/TenantDatabaseService.php", + "tests/Unit/ChatControllerSourceResolutionTest.php", + ) + if '__LAI_' in (report.response or ''): + return ( + "app/Services/AIService.php", + "app/Http/Controllers/ChatController.php", + "tests/Feature/AIPrivacyBoundaryTest.php", + ) + + production_files: list[str] = [] + test_files: list[str] = [] + + def add_file(filename: str) -> None: + target = test_files if filename.startswith("tests/") else production_files + if filename not in target: + target.append(filename) + + if requires_table_output_repair(report): + add_file("app/Services/AIService.php") + add_file("tests/Feature/AIPrivacyBoundaryTest.php") + + if requires_explicit_route_repair(report): + add_file("app/Http/Controllers/ChatController.php") + add_file("tests/Unit/ChatControllerSourceResolutionTest.php") + if "www:" in report_text: + add_file("app/Services/WebSearchService.php") + add_file("tests/Unit/WebSearchServiceTest.php") + + if report.category in {"incorrect_answer", "sql_execution"} and ( + report.context.get("database_type") == "erp" + or "fk_" in report_text + or any(term in report_text for term in ( + "auftrag", "rechnung", "lieferschein", "umsatz", "gutschrift" + )) + ) and not _declares_data_correct(report): + add_file("config/erp_semantics.php") + add_file("tests/Unit/SQLGeneratorBusinessContextTest.php") + + return (*production_files, *test_files) + + +def build_task(report: FailureReport) -> str: + prompt = redact(report.prompt) + feedback = redact(report.user_feedback) + expected_behavior = redact(report.expected_behavior) + response = redact(report.response) + error = redact(report.error_message) + sql = redact(report.sql_query) + tenant_scope = '' + if report.context.get('database_type') == 'customer': + tenant_scope = """ +Tenant routing fact: +The customer has already selected one tenant database. SQL inside that database must not +invent an additional customer discriminator column. Use only columns present in tracked +schema/semantics, and fix SQL-generation guidance rather than rewriting one unknown column +to another guessed column. TenantDatabaseService records the explicit source type in +database.connections..tenant_type; use that metadata instead of database-name +patterns or heuristics. +""" + sql_semantics = '' + if report.category in {'incorrect_answer', 'sql_execution'}: + sql_semantics = """ +SQL repair guidance: +When the SQL is syntactically valid but uses the wrong business calculation, prefer adding or +correcting tracked domain knowledge in config/*_semantics.php so the schema prompt teaches the +model the right joins, filters, and formula. Do not hardcode one user's question or article ID +in SQLGeneratorService, and do not add post-generation replacement of guessed SQL. + +Translate positive expected behavior literally into both a reusable rule and a reference SQL example. +If it gives a numeric document code, use direct equality with that code rather than a lookup label. +If it names a date column, use that exact column even when another date is common elsewhere. Add +every requested exclusion explicitly. Do not copy additional date, booking, lookup, or status +filters from the known-bad SQL when expected behavior replaces or does not request them. Regression +assertions must require the corrected fields and filters and must never assert the opposite rule. +The same single executable reference SELECT must demonstrate all requested identifiers, numeric +codes, date columns, and exclusions together; listing them only in separate prose rules is not proof. +Use an expected bare status name as that exact column token; do not invent bStatus_ or other prefixes. +The focused test must assert every required filter, including obsolete and cancellation exclusions. +Do not normalize capitalization or wording in unrelated existing rules. + +For regression coverage of prompt guidance, expose the relevant protected schema/business- +context builder through a test subclass and assert that its real output contains the required +formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt +guidance to rewrite that mocked response. Keep database fixture tests only when production code +itself deterministically builds and executes the SQL under test. + +For an unknown-column failure, compare every alias.column in the observed SQL with the table +schema and identify the owning table before editing. Teach the exact ownership, join, grouping, +and real column names in semantics config, then add a reference SQL example for the reported +question shape. Never replace one missing column with an unverified synonym. + +When expected behavior names tables from multiple databases or connections, first inspect source +routing and existing staged-query workflows. Do not put an impossible cross-connection join only +in ERP semantics. Implement deterministic orchestration that resolves identifiers in one source +and passes only the required identifier into the next source, preserving tenant isolation. +For this cross-database report, editing only config/*_semantics.php or tests is an invalid repair. +You MUST implement production behavior in ChatController or TenantDatabaseService and cover that +exact application method in ChatControllerSourceResolutionTest. Before editing, describe the data +flow, connection used at each stage, and where the final result is assembled. +""" + placeholder_guidance = '' + if '__LAI_' in response: + placeholder_guidance = """ +Privacy placeholder repair guidance: +Trace the actual answer path through AIService::generateDatabaseResponse, issued token +restoration, and ChatController's local fallback. A token can only be restored when it exists +in the exact token map returned by protect(); never test a made-up token against an unrelated +map. Cover model-invented or malformed __LAI_...__ output with an HTTP-backed AIService test +and verify the user receives a real local database-result fallback, not a raw placeholder. +""" + + directives = removal_directives(report) + forbidden_lower = "\n".join(item.fragment for item in directives).lower() + expected_lower = expected_behavior.lower() + sql_literal_contract: list[str] = [] + document_code = re.search(r"\bauftragskennung\s+(\d+)\b", expected_behavior, re.IGNORECASE) + if document_code and "auftragskennung" not in forbidden_lower: + sql_literal_contract.append(f"a.AuftragsKennung = {document_code.group(1)}") + if "system_created" in expected_lower and "system_created" not in forbidden_lower: + sql_literal_contract.append("a.System_created") + if "obsolet" in expected_lower and "obsolet" not in forbidden_lower: + sql_literal_contract.append("COALESCE(a.obsolet, 0) = 0") + if "storniert" in expected_lower and "storniert" not in forbidden_lower: + sql_literal_contract.append("COALESCE(a.bStatus_storniert, 0) = 0") + literal_guidance = "" + if sql_literal_contract: + literal_guidance = ( + "\nExact SQL literal contract derived from expected behavior:\n- " + + "\n- ".join(sql_literal_contract) + + "\nOne added reference SELECT and the focused regression assertions must contain " + "every literal above. Preserve spelling and capitalization exactly. Do not add an LS " + "lookup or bStatus_gebucht unless it appears in this contract. Assertions must quote " + "these exact SQL literals; do not replace them with descriptive prose.\n" + ) + removal_guidance = "" + if directives: + exact_assertions = "\n- ".join( + php_negative_assertion(item.fragment) for item in directives + ) + removal_guidance = ( + "\nExact SQL removal contract derived from expected behavior:\n- " + + "\n- ".join(item.fragment for item in directives) + + "\nEvery matching production reference SELECT must omit every fragment above. " + "The following PHP regression assertions are mandatory; copy them exactly, including " + "aliases and qualifiers:\n- " + + exact_assertions + + "\nA forbidden identifier is a negative requirement: do not add it positively merely " + "because it appears in expected behavior. An unqualified substring is not equivalent " + "to an alias-qualified fragment.\n" + ) + if is_removal_only(report): + removal_guidance += ( + "This is explicitly a removal-only repair. Preserve every other selected field, " + "table, join, date column, predicate, grouping, ordering, limit, rule, and existing " + "test assertion byte-for-byte. Change only the matching reference SQL fragment and " + "add focused negative regression coverage.\n" + ) + else: + removal_guidance += ( + "Before editing, inspect the matching tracked reference SELECT. If it already omits " + "the forbidden fragment, treat that SELECT as a protected correct baseline and " + "preserve it exactly. Add or clarify the narrow reusable business rule that prevents " + "the reported recurrence instead of regressing the reference merely to create a diff. " + "Add only that narrow prohibition rule. Do not add or restate date-column choices, " + "document codes, lookup strategies, joins, cancellation/obsolete rules, or other " + "business guidance already covered by the protected reference. " + "Phrase the rule without embedding the complete forbidden predicate: mention its " + "column identifier and say it must not be used, so the exact context-level negative " + "assertion above remains true. The focused context-builder test must positively assert " + "the new rule with assertStringContainsString for unique text from that rule as well as use the exact " + "negative assertion above; a negative assertion alone is vacuous when the old reference " + "already omitted the fragment. A configured rule is exercised only when every " + "table in its tables list is supplied to businessContext(); align the test's allowed " + "tables with the rule so the test cannot pass while omitting the production change.\n" + ) + + interaction_guidance = "" + if requires_table_output_repair(report): + interaction_guidance += """ +Answer-presentation repair guidance: +This report requires a real answer-formatting change, not ERP SQL or semantics changes when the +report says the data is already correct. Inspect AIService::generateDatabaseResponse(). Grouped +series requests such as counts per month must request a compact Markdown table even when the +original user prompt does not literally contain the word "Tabelle". Add a provider-backed test +using the exact reported user prompt. It must prove that the answer-generation system instruction +requests a Markdown table and that all supplied grouped rows remain available. Do not add static +example result values or output-format metadata to config/*_semantics.php. Grouped-series detection +must require multiple result rows and grouping/value columns; one ordinary result row with multiple +columns must remain sentence-formatted. Add that negative provider-backed regression test as well. +The positive grouped test uses columns named exactly Monat and Anzahl_Lieferscheine. Ensure the +production grouping/value-column detection recognizes those exact names, and confirm the positive +test receives Markdown-Tabelle before finishing; do not merely add the test without satisfying it. +Inside the provider fake, prove every supplied row reached $userPrompt without exposing protected +values: for three rows, assert substr_count($userPrompt, '"Monat"') and +substr_count($userPrompt, '"Anzahl_Lieferscheine"') both equal 3. A mocked Markdown response alone +is vacuous because it can pass even if no grouped rows reached the provider. +Keep the existing exact table instruction text: "Die Benutzerfrage verlangt eine Tabelle. Gib eine +kurze Einleitung und danach eine kompakte Markdown-Tabelle mit allen bereitgestellten Zeilen aus." +Do not move or duplicate this instruction inside shouldUseTableFormat(); that helper returns only bool. +The regression assertion must use the exact substring "allen bereitgestellten Zeilen". Never assert +"alle bereitgestellten Zeilen": that grammatically different substring is not present in the correct +production instruction. If Markdown-Tabelle passes but this assertion fails, fix the assertion—not +the already-correct production instruction. +""" + if requires_explicit_route_repair(report): + interaction_guidance += """ +Explicit route-prefix repair guidance: +Implement one protected resolveRoutePrefix(string $message): array helper that always returns exactly +[explicit route or null, cleaned message]. Call it with exact destructuring at the start of +processMessage(): [$explicitRoute, $message] = $this->resolveRoutePrefix($message); Then compute +$route = $explicitRoute ?? (...) around the existing database-keyword/model-classifier expression, +before database keyword heuristics or model classification: +"cockpit:" must force the database path, "www:" must force public web research, and "direct:" must +force the selected model without web or database access, even when the remaining text contains +keywords for another route. Web retrieval must be provider-independent: WebSearchService retrieves +bounded public search snippets, then AIService::generateWebResearchResponse() passes only the clean +question and those untrusted snippets to the selected model, including Ollama/Qwen, and returns +clickable source URLs. Do not pass private conversation/database context into web research, fetch +arbitrary result URLs, or obey instructions embedded in search snippets. Remove the control prefix +before passing the user's actual question downstream. Add deterministic ChatController routing tests +for all three prefixes that prove the classifier cannot override them. For every explicit prefix, +AIService::determineRoute() must not be called; never write a test expecting that classifier call. +Helper-only tests are insufficient: add three tests that call processMessage() itself, one for each +prefix, assert determineRoute() is never called, and assert the cleaned question reaches the forced +database/research/direct response path. Mock only external collaborators, not processMessage or +prefix logic. +This is feasible in the existing test file: switch it from PHPUnit\\Framework\\TestCase to +Tests\\TestCase, use Illuminate\\Foundation\\Testing\\RefreshDatabase, build an +Illuminate\\Http\\Request with Request::create(), and attach app('session')->driver() using +setLaravelSession(). For www:, mock generateWebResearchResponse() with the exact cleaned question +and selected ollama model, assert the JSON route is research, and assert source metadata is persisted. +For direct:, mock generateDirectResponse(), assert generateWebResearchResponse() is never called, +and assert the JSON route is direct. For cockpit:, make the tenant mock return true from +requiresCustomerDatabase() and null from resolveFromPrompt(), then assert the JSON route is +customer_clarification and the persisted ChatMessage.message equals the cleaned question exactly. +For that cockpit test, all requiresCustomerDatabase/requiresConfigDatabase/requiresErpDatabase +heuristics must return false; otherwise the database route would be selected without the prefix and +the test is vacuous. The explicit cockpit route itself must be the only reason determineRoute() is +bypassed. Use a cleaned question such as "Erkläre Quantenphysik" that has no database keywords. +Each prefix is a control token and must not be stored or passed downstream. Never assert that a +persisted message contains cockpit:, www:, or direct:, and never change ChatMessage 'message' fields +to $originalMessage. In all three tests put +$aiService->expects($this->never())->method('determineRoute') before processMessage(). Use an +explicit model in each request-path test so mocked response methods do not throw an argument-mismatch +exception that processMessage() catches and converts into an error JSON response. Assert response +status/success before reading route. +Do not write “we can't fully test processMessage”; Laravel's application test case provides the +HTTP, session, response, and database context needed here. +JsonResponse has no json() method in this codebase; read it once with $payload = +$response->getData(true), then assert $payload['success'] and $payload['route']. +The helper must accept leading whitespace and case-insensitive prefixes, and trim the cleaned +question. Test the helper returns ['database', cleaned question], ['research', cleaned question], +['direct', cleaned question], and [null, unchanged trimmed question]. Do not assign the whole helper +result array to $route. Then prove +processMessage() uses that result rather than only testing resolveRoutePrefix() or +resolveDatabaseSource(). +Do not add prefix handling to resolveDatabaseSource(), do not change its default-source behavior, and +do not add wrappers that reference nonexistent properties or methods such as defaultModel or +determineRoute on ChatController. Inspect the existing TestableChatController before editing: +expose resolveRoutePrefix() exactly once, and replace an existing wrapper when necessary—never +append a duplicate method declaration. The public test-subclass wrapper must call +parent::resolveRoutePrefix($message), never $this->resolveRoutePrefix($message), which recursively +calls itself. Do not reinterpret "cockpit:" as a new SQL business rule. +All three prefix behaviors and any independent presentation requirement in this report are mandatory. +""" + + return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository. + +{_REPOSITORY_PLAYBOOK} + +Rules: +- First add a regression test that reproduces the reported behavior. +- The regression test must execute the relevant production behavior; never mock or stub the behavior being tested. +- Make the smallest production change that fixes the root cause. +- Treat the user's expected behavior as authoritative, while verifying table and column names against tracked schema and semantics. +- Every positively required technical identifier, numeric code, and status filter named in expected + behavior must appear in the production repair and focused regression assertions. Identifiers + explicitly requested for removal must instead be absent from matching SQL and appear in an exact + negative assertion. Do not substitute a pre-existing rule. +- Preserve read-only SQL validation and tenant isolation. +- Never read or modify .env files, credentials, deployment secrets, or production data. +- Do not depend on production row contents. Use synthetic fixtures or assert generated business context and SQL guidance. +- Do not weaken tests, authentication, authorization, SQL validation, or safety checks. +- Do not commit or push; the orchestrator owns Git publication. + +Failure category: {report.category} +Feedback type: {report.feedback_type or 'not specified'} +Source: {report.source} +Model: {redact(str(report.context.get('model', 'unknown')))} +Database type: {redact(str(report.context.get('database_type', 'unknown')))} + +User request: +{prompt or '[not available]'} + +User feedback: +{feedback or '[not available]'} + +Expected behavior: +{expected_behavior or '[not available]'} + +Observed answer: +{response or '[not available]'} + +Observed error: +{error or '[not available]'} + +Generated SQL, if relevant: +{sql or '[not available]'} +{tenant_scope} +{sql_semantics} +{literal_guidance} +{removal_guidance} +{interaction_guidance} +{placeholder_guidance} + +Run focused tests while working. Leave the repository with the fix and regression test applied. +""" + + +def build_retry_task(report: FailureReport, exception: Exception) -> str: + failure = redact(str(exception)) + + return build_task(report) + f""" + +The previous implementation failed isolated validation. + +Correct the existing changes using this test output: + +{failure} + +Keep the regression test meaningful. Do not remove or weaken tests, safety checks, +authentication, authorization, SQL validation, or tenant isolation. If a new test calls an +unrelated specialized code path that correctly returns no result, repair the test to exercise +the owning production behavior; never broaden unrelated production routing merely to make the +test pass. Do not commit or push. +""" diff --git a/old_2026-08-31/src/lai_automation/removal.py b/old_2026-08-31/src/lai_automation/removal.py new file mode 100644 index 0000000..5491247 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/removal.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass +from difflib import SequenceMatcher +from pathlib import Path + +from .models import FailureReport + +_REMOVAL_INTENT = re.compile( + r"\b(?:entfern\w*|lösch\w*|loesch\w*|weglass\w*|streichen|" + r"remove\w*|delet\w*|omit\w*|drop\w*|ohne|without)\b|" + r"\b(?:darf|soll|muss)\b.{0,50}\b(?:nicht|nie|niemals)\b|" + r"\b(?:must|should)\s+not\b|\bdo\s+not\b", + re.IGNORECASE | re.DOTALL, +) +_NEGATED_REMOVAL = re.compile( + r"\b(?:nicht|nie|niemals)\s+(?:entfern\w*|lösch\w*|loesch\w*|weglass\w*)\b|" + r"\b(?:do\s+not|don't|never)\s+(?:remove\w*|delet\w*|omit\w*|drop\w*)\b", + re.IGNORECASE, +) +_ONLY_REMOVAL = re.compile( + r"\b(?:einfach\s+)?nur\b.{0,120}\b(?:entfern\w*|lösch\w*|loesch\w*|" + r"weglass\w*|streichen)\b|" + r"\b(?:just|only)\b.{0,120}\b(?:remove\w*|delet\w*|omit\w*|drop\w*)\b", + re.IGNORECASE | re.DOTALL, +) +_QUOTED_FRAGMENT = re.compile( + r'"(?P[^"\n]{1,500})"|' + r"'(?P[^'\n]{1,500})'|" + r"`(?P[^`\n]{1,500})`|" + r"„(?P[^”\n]{1,500})”" +) +_TECHNICAL_IDENTIFIER = re.compile( + r"\b(?:[A-Za-z_][A-Za-z0-9_]*\.)?[A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+\b" +) +_QUESTION_KEY = re.compile(r"['\"]question['\"]\s*=>", re.IGNORECASE) +_SQL_HEREDOC = re.compile( + r"['\"]sql['\"]\s*=>\s*<<<(?P['\"]?)(?P