chore: archive previous Aider worker
This commit is contained in:
8
old_2026-08-31/.dockerignore
Normal file
8
old_2026-08-31/.dockerignore
Normal file
@@ -0,0 +1,8 @@
|
|||||||
|
.git
|
||||||
|
.env
|
||||||
|
.venv
|
||||||
|
.docker-tmp
|
||||||
|
__pycache__
|
||||||
|
.pytest_cache
|
||||||
|
.ruff_cache
|
||||||
|
workspaces
|
||||||
29
old_2026-08-31/.env.example
Normal file
29
old_2026-08-31/.env.example
Normal file
@@ -0,0 +1,29 @@
|
|||||||
|
FAILURE_DB_HOST=host.docker.internal
|
||||||
|
FAILURE_DB_PORT=33060
|
||||||
|
FAILURE_DB_NAME=laravel_ai_chat
|
||||||
|
FAILURE_DB_USER=laravel
|
||||||
|
FAILURE_DB_PASSWORD=
|
||||||
|
|
||||||
|
SOURCE_REPOSITORY=https://gitea.tisinfra.de/christian.meyer/lai.git
|
||||||
|
SOURCE_BRANCH=main
|
||||||
|
GITEA_URL=https://gitea.tisinfra.de
|
||||||
|
GITEA_OWNER=christian.meyer
|
||||||
|
GITEA_REPOSITORY=lai
|
||||||
|
GITEA_USERNAME=automation-bot
|
||||||
|
GITEA_TOKEN=
|
||||||
|
|
||||||
|
# dry-run, pr, direct-main
|
||||||
|
AUTOMATION_MODE=dry-run
|
||||||
|
ALLOW_DIRECT_MAIN=false
|
||||||
|
|
||||||
|
AGENT_COMMAND=aider
|
||||||
|
AGENT_MODEL=mistral/devstral-latest
|
||||||
|
AGENT_TIMEOUT_SECONDS=900
|
||||||
|
AGENT_IDLE_TIMEOUT_SECONDS=300
|
||||||
|
MISTRAL_API_KEY=
|
||||||
|
|
||||||
|
POLL_SECONDS=60
|
||||||
|
STALE_CLAIM_MINUTES=90
|
||||||
|
MAX_CHANGED_FILES=12
|
||||||
|
MAX_CHANGED_LINES=800
|
||||||
|
TEST_COMMANDS=composer install --no-interaction --prefer-dist;;php artisan test
|
||||||
18
old_2026-08-31/.gitea/workflows/ci.yml
Normal file
18
old_2026-08-31/.gitea/workflows/ci.yml
Normal file
@@ -0,0 +1,18 @@
|
|||||||
|
name: CI
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
pull_request:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: "3.12"
|
||||||
|
- run: python -m pip install ".[dev]"
|
||||||
|
- run: ruff check src tests
|
||||||
|
- run: pytest -q
|
||||||
|
- run: docker build -t lai-automation:test .
|
||||||
9
old_2026-08-31/.gitignore
vendored
Normal file
9
old_2026-08-31/.gitignore
vendored
Normal file
@@ -0,0 +1,9 @@
|
|||||||
|
.env
|
||||||
|
.venv/
|
||||||
|
__pycache__/
|
||||||
|
.pytest_cache/
|
||||||
|
.ruff_cache/
|
||||||
|
.docker-tmp/
|
||||||
|
.docker-buildx/
|
||||||
|
*.pyc
|
||||||
|
workspaces/
|
||||||
19
old_2026-08-31/Dockerfile
Normal file
19
old_2026-08-31/Dockerfile
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
FROM python:3.12-slim
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
composer git nodejs npm openssh-client php-cli php-curl php-mbstring php-mysql php-xml php-zip unzip \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
COPY pyproject.toml README.md ./
|
||||||
|
COPY src ./src
|
||||||
|
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||||
|
pip install --retries 10 --timeout 120 ".[agent]"
|
||||||
|
|
||||||
|
RUN useradd --create-home --uid 10001 coder \
|
||||||
|
&& mkdir -p /workspaces \
|
||||||
|
&& chmod 700 /home/coder
|
||||||
|
|
||||||
|
ENTRYPOINT ["lai-agent"]
|
||||||
|
CMD ["worker"]
|
||||||
202
old_2026-08-31/README.md
Normal file
202
old_2026-08-31/README.md
Normal file
@@ -0,0 +1,202 @@
|
|||||||
|
# LAI Automatisierung
|
||||||
|
|
||||||
|
Kontrollierter Coding-Agent fuer Failure Reports aus
|
||||||
|
[`lai`](https://gitea.tisinfra.de/christian.meyer/lai). Der Orchestrator verwendet
|
||||||
|
[Aider](https://aider.chat/) headless als austauschbaren Coding-Kern. Git-, Gitea-,
|
||||||
|
Policy- und Freigabelogik bleiben in diesem Projekt.
|
||||||
|
|
||||||
|
## Warum Aider?
|
||||||
|
|
||||||
|
- scriptbarer Einmal-Auftrag mit `--message-file`
|
||||||
|
- gute Git- und Repository-Map-Unterstuetzung
|
||||||
|
- Mistral ueber LiteLLM nutzbar
|
||||||
|
- automatische Commits koennen deaktiviert werden
|
||||||
|
- deutlich kleinerer Betriebsumfang als OpenHands
|
||||||
|
|
||||||
|
OpenHands bleibt eine moegliche spaetere Alternative fuer komplexere, mehrstufige
|
||||||
|
Agentenlaeufe. Sein Headless-Modus arbeitet jedoch immer mit `always-approve`; deshalb
|
||||||
|
muessen auch dort unsere externen Policy-Gates erhalten bleiben.
|
||||||
|
|
||||||
|
## Ablauf
|
||||||
|
|
||||||
|
```text
|
||||||
|
failure_reports(open)
|
||||||
|
-> transaktionales Claiming (SKIP LOCKED)
|
||||||
|
-> anonymisierter Agentenauftrag
|
||||||
|
-> frischer Clone von lai.git
|
||||||
|
-> Aider-Aenderung ohne Commit
|
||||||
|
-> Pfad-, Groessen- und Sicherheitsgates
|
||||||
|
-> Tests
|
||||||
|
-> Commit
|
||||||
|
-> PR oder kontrollierter Push auf main
|
||||||
|
-> failure_reports(resolved/agent_failed)
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Coding-Agent erhaelt keinen Zugang zur ERP- oder Kundendatenbank. Rohdaten und
|
||||||
|
`.env`-Dateien sind ausgeschlossen. Repository-Pushes verwenden `GIT_ASKPASS`; Tokens
|
||||||
|
werden nicht in Remote-URLs oder Prozessargumenten gespeichert. Der Orchestrator laeuft
|
||||||
|
als privilegierter Containerprozess, startet Aider aber als separaten Benutzer `coder`.
|
||||||
|
Dieser Unterprozess erhaelt nur Modell-Credentials, nicht die Failure-DB- oder
|
||||||
|
Gitea-Zugangsdaten.
|
||||||
|
|
||||||
|
Die Workspace-Setup- und Testprozesse erhalten separat konfigurierte `TEST_DB_*`-Zugangsdaten.
|
||||||
|
Dieser Benutzer darf ausschliesslich die isolierten Testschemas bearbeiten. Fuer Laravel
|
||||||
|
werden daraus `DB_*`, `ERP_DB_*` und `COCKPIT_CONFIG_DATABASE` gesetzt; Aider selbst sieht
|
||||||
|
diese Werte nie. Produktive ERP- oder Kundenzugangsdaten duerfen hier nicht verwendet werden.
|
||||||
|
Vor Aider und erneut vor der abschliessenden Validierung installiert `SETUP_COMMANDS` die
|
||||||
|
Abhaengigkeiten, erstellt die Vite-Assets und initialisiert mit
|
||||||
|
`php artisan migrate:fresh --force` das isolierte Laravel-Schema neu. `TEST_COMMANDS` enthaelt
|
||||||
|
die eigentlichen Pruefungen. Deshalb muessen alle
|
||||||
|
`TEST_DB_*`-Schemas dediziert und vollstaendig entbehrlich sein.
|
||||||
|
|
||||||
|
Lokale Ollama-Modelle laufen fuer Produktionsdateien in einem einzelnen Whole-File-Durchgang.
|
||||||
|
Devstral und andere externe Coding-Modelle verwenden direkt das Diff-Format statt desselben
|
||||||
|
Modells zusaetzlich als Architect und Editor aufzurufen. Das halbiert Modellrunden fuer kleine
|
||||||
|
Reparaturen. Bei gezielt gerouteten Dateien ist die zusaetzliche Repo-Map deaktiviert.
|
||||||
|
Automatische URL-Erkennung und Web-Scraping sind deaktiviert, damit URLs in Anwendungscode oder
|
||||||
|
Tests niemals als externe Rechercheauftraege interpretiert werden.
|
||||||
|
`AGENT_TIMEOUT_SECONDS` begrenzt Agentenprozess und Modell-API-Aufruf.
|
||||||
|
`AGENT_IDLE_TIMEOUT_SECONDS` beendet zusaetzlich einen Agenten, der trotz laufendem Prozess
|
||||||
|
standardmaessig fuenf Minuten keine Ausgabe mehr erzeugt; vorhandene Teilpatches werden danach
|
||||||
|
weiterhin validiert.
|
||||||
|
Setup-Fehler werden als Infrastrukturfehler beendet und nicht als Codekorrektur an das Modell
|
||||||
|
zurueckgegeben. Hat ein lokales Modell vor einem Timeout bereits Dateien geaendert, durchlaufen
|
||||||
|
diese Teilresultate trotzdem Policy, Syntaxpruefung und Tests; nur ein Timeout ohne Diff verwirft
|
||||||
|
den Versuch sofort.
|
||||||
|
|
||||||
|
Laravel-Tests laufen mit PHP und Composer direkt im isolierten Worker. Der Docker-Socket
|
||||||
|
des Hosts wird nicht eingebunden. Abgebrochene Claims werden nach standardmaessig 90
|
||||||
|
Minuten wieder freigegeben; `STALE_CLAIM_MINUTES` passt dieses Intervall an.
|
||||||
|
Explizite Nur-Entfernen-Auftraege werden als negativer SQL-Vertrag geprueft: Der verlangte
|
||||||
|
Ausdruck muss aus der passenden Referenzabfrage verschwinden, in einer negativen Test-Assertion
|
||||||
|
vorkommen, und alle anderen Felder, Joins, Filter, Gruppierungen sowie bestehenden Tests muessen
|
||||||
|
unveraendert bleiben. Ist der Ausdruck in einer eindeutig zur Benutzerfrage passenden, getrackten
|
||||||
|
Referenzabfrage bereits entfernt, wird ein veralteter Report ohne Agentenlauf und ohne PR als
|
||||||
|
`already_fixed` abgeschlossen.
|
||||||
|
|
||||||
|
Schlaegt Policy, Syntax oder isolierte Validierung fehl, erhaelt Aider kontrollierte
|
||||||
|
Korrekturversuche mit redigierter Diagnose. Syntaxgueltige Teilpatches werden als Checkpoint
|
||||||
|
erhalten; unerwartete Dateien werden verworfen. Nach einem Testfehler bleiben alle gerouteten
|
||||||
|
Produktions- und Testdateien sichtbar, damit der Agent die Ursache korrigieren kann, statt nur
|
||||||
|
Assertions an fehlerhaftes Verhalten anzupassen. Mehrteilige Feedbacks zu Antwortformat und
|
||||||
|
expliziten Routing-Praefixen werden an ihre jeweiligen Anwendungseigentuemer geroutet und muessen
|
||||||
|
vollstaendig abgedeckt sein. Zwei identische Policy-Fehler hintereinander brechen frueh ab.
|
||||||
|
`MAX_AGENT_ATTEMPTS` ist auf 1 bis 7 begrenzt.
|
||||||
|
|
||||||
|
## Schnellstart
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp .env.example .env
|
||||||
|
# FAILURE_DB_PASSWORD und bei Bedarf MISTRAL_API_KEY eintragen
|
||||||
|
# TEST_DB_PASSWORD fuer den Benutzer der isolierten Testschemas eintragen
|
||||||
|
docker compose build
|
||||||
|
docker compose run --rm automation check
|
||||||
|
docker compose run --rm automation once
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Default `AUTOMATION_MODE=dry-run` claimt einen offenen Report, erzeugt den
|
||||||
|
anonymisierten Auftrag und setzt ihn danach wieder auf `open`. Es wird weder ein Agent
|
||||||
|
gestartet noch gepusht.
|
||||||
|
|
||||||
|
## Fortschritt beobachten
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose ps automation
|
||||||
|
docker compose exec automation lai-agent check
|
||||||
|
docker compose logs -f automation
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Worker protokolliert keine unredigierten Prompts, SQL-Ergebnisse oder Secrets. Aider-Ausgabe
|
||||||
|
wird ohne Terminalfarben live, redigiert und pro Zeile begrenzt protokolliert; in Report-Metadaten
|
||||||
|
landet nur ein begrenzter Ausgabetail. Fuer jeden Versuch erscheinen Modell, Edit-Strategie,
|
||||||
|
Dateien, PID, Laufzeit, Ausgabeaktivitaet und Ergebnisstatus. Zusaetzlich erscheinen
|
||||||
|
Workspace-Vorbereitung, Policy-Pruefung, jeder Validierungsbefehl, Commit, PR-URL und Cleanup.
|
||||||
|
Bei laengeren Agenten- oder Testphasen wird alle 30 Sekunden Laufzeitfortschritt ausgegeben und
|
||||||
|
`updated_at` als Datenbank-Lease erneuert. Nur wirklich verwaiste Claims werden nach
|
||||||
|
`STALE_CLAIM_MINUTES` wieder freigegeben.
|
||||||
|
|
||||||
|
## PR-Modus
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
AUTOMATION_MODE=pr
|
||||||
|
GITEA_USERNAME=automation-bot
|
||||||
|
GITEA_TOKEN=...
|
||||||
|
MISTRAL_API_KEY=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Der Bot benoetigt Schreibzugriff auf `christian.meyer/lai` und die Berechtigung,
|
||||||
|
Branches sowie Pull Requests anzulegen. Jeder Lauf erstellt
|
||||||
|
`ai/failure-{report_id}` und einen PR gegen `main`.
|
||||||
|
|
||||||
|
## Direkter main-Modus
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
AUTOMATION_MODE=direct-main
|
||||||
|
ALLOW_DIRECT_MAIN=true
|
||||||
|
GITEA_TOKEN=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Der zweite Schalter ist ein Kill-Switch. In diesem Modus sind Migrationen, Routes,
|
||||||
|
Auth, Middleware, PermissionService und SQLValidatorService zusaetzlich blockiert.
|
||||||
|
Branch Protection muss den Bot fuer direkte Pushes explizit zulassen. PR-Modus bleibt
|
||||||
|
die empfohlene Betriebsart.
|
||||||
|
|
||||||
|
## Modelle
|
||||||
|
|
||||||
|
Das Modell wird ausschliesslich ueber `AGENT_MODEL` gewaehlt. Devstral ist der
|
||||||
|
Default und ein Coding-Agent, nicht das Chat-Modell der Laravel-Anwendung:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
AGENT_MODEL=mistral/devstral-latest
|
||||||
|
MISTRAL_API_KEY=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Codestral verwendet denselben Mistral-Zugang:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
AGENT_MODEL=mistral/codestral-latest
|
||||||
|
MISTRAL_API_KEY=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Qwen 3.5 27B ueber OpenRouter:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
AGENT_MODEL=openrouter/qwen/qwen3.5-27b
|
||||||
|
OPENROUTER_API_KEY=...
|
||||||
|
```
|
||||||
|
|
||||||
|
Oder mit einem lokalen Ollama auf dem Docker-Host:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
AGENT_MODEL=ollama_chat/qwen3.5:27b
|
||||||
|
OLLAMA_API_BASE=http://host.docker.internal:11434
|
||||||
|
```
|
||||||
|
|
||||||
|
Nach einer Aenderung wird der Worker neu erstellt, damit Compose die neue Umgebung
|
||||||
|
uebernimmt:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
docker compose up -d --force-recreate automation
|
||||||
|
docker compose exec -T automation lai-agent check
|
||||||
|
```
|
||||||
|
|
||||||
|
Aider verwendet fuer lokale Produktionsreparaturen `whole` und sonst das praezise
|
||||||
|
`diff`-Format. Das Chat-Modell und Coding-Modell sind bewusst getrennt. Fuer Coding-Agenten sollte ein
|
||||||
|
agentisch/coding-optimiertes Modell verwendet werden. Ein Wechsel zu OpenHands oder
|
||||||
|
einem anderen Backend erfordert einen neuen Adapter neben `AiderAgent`.
|
||||||
|
|
||||||
|
## Tests
|
||||||
|
|
||||||
|
```bash
|
||||||
|
python -m venv .venv
|
||||||
|
. .venv/bin/activate
|
||||||
|
pip install ".[dev]"
|
||||||
|
ruff check src tests
|
||||||
|
pytest -q
|
||||||
|
```
|
||||||
|
|
||||||
|
## Repositorys
|
||||||
|
|
||||||
|
- Chat: `https://gitea.tisinfra.de/christian.meyer/lai.git`
|
||||||
|
- Automatisierung: `https://gitea.tisinfra.de/christian.meyer/lai_automatisierung.git`
|
||||||
|
|
||||||
57
old_2026-08-31/docker-compose.yml
Normal file
57
old_2026-08-31/docker-compose.yml
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
services:
|
||||||
|
automation:
|
||||||
|
build: .
|
||||||
|
restart: unless-stopped
|
||||||
|
environment:
|
||||||
|
FAILURE_DB_HOST: ${FAILURE_DB_HOST:-laravel-ai-chat-mysql}
|
||||||
|
FAILURE_DB_PORT: ${FAILURE_DB_PORT:-3306}
|
||||||
|
FAILURE_DB_NAME: ${FAILURE_DB_NAME:-laravel_ai_chat}
|
||||||
|
FAILURE_DB_USER: ${FAILURE_DB_USER:-laravel}
|
||||||
|
FAILURE_DB_PASSWORD: ${FAILURE_DB_PASSWORD:?FAILURE_DB_PASSWORD must be set}
|
||||||
|
SOURCE_REPOSITORY: ${SOURCE_REPOSITORY:-https://gitea.tisinfra.de/christian.meyer/lai.git}
|
||||||
|
SOURCE_BRANCH: ${SOURCE_BRANCH:-main}
|
||||||
|
AUTOMATION_MODE: ${AUTOMATION_MODE:-dry-run}
|
||||||
|
ALLOW_DIRECT_MAIN: ${ALLOW_DIRECT_MAIN:-false}
|
||||||
|
GITEA_URL: ${GITEA_URL:-https://gitea.tisinfra.de}
|
||||||
|
GITEA_OWNER: ${GITEA_OWNER:-christian.meyer}
|
||||||
|
GITEA_REPOSITORY: ${GITEA_REPOSITORY:-lai}
|
||||||
|
GITEA_USERNAME: ${GITEA_USERNAME:-automation-bot}
|
||||||
|
GITEA_TOKEN: ${GITEA_TOKEN:-}
|
||||||
|
AGENT_COMMAND: ${AGENT_COMMAND:-aider}
|
||||||
|
AGENT_MODEL: ${AGENT_MODEL:-mistral/devstral-latest}
|
||||||
|
AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-900}
|
||||||
|
AGENT_IDLE_TIMEOUT_SECONDS: ${AGENT_IDLE_TIMEOUT_SECONDS:-300}
|
||||||
|
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
|
||||||
|
OPENAI_API_KEY: ${OPENAI_API_KEY:-}
|
||||||
|
OPENAI_API_BASE: ${OPENAI_API_BASE:-}
|
||||||
|
OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-}
|
||||||
|
OLLAMA_API_BASE: ${OLLAMA_API_BASE:-}
|
||||||
|
POLL_SECONDS: ${POLL_SECONDS:-60}
|
||||||
|
MAX_AGENT_ATTEMPTS: ${MAX_AGENT_ATTEMPTS:-2}
|
||||||
|
STALE_CLAIM_MINUTES: ${STALE_CLAIM_MINUTES:-90}
|
||||||
|
SETUP_COMMANDS: ${SETUP_COMMANDS:-composer install --no-interaction --prefer-dist;;npm ci;;npm run build;;php artisan migrate:fresh --force}
|
||||||
|
TEST_COMMANDS: ${TEST_COMMANDS:-php artisan test}
|
||||||
|
TEST_DB_HOST: ${TEST_DB_HOST:-laravel-ai-chat-mysql}
|
||||||
|
TEST_DB_PORT: ${TEST_DB_PORT:-3306}
|
||||||
|
TEST_DB_NAME: ${TEST_DB_NAME:-laravel_ai_chat_agent_test}
|
||||||
|
TEST_DB_USER: ${TEST_DB_USER:-lai_test_runner}
|
||||||
|
TEST_DB_PASSWORD: ${TEST_DB_PASSWORD:-agent_test_secret}
|
||||||
|
TEST_CUSTOMER_DB: ${TEST_CUSTOMER_DB:-customer_agent_test}
|
||||||
|
TEST_CONFIG_DB: ${TEST_CONFIG_DB:-cockpit_agent_test}
|
||||||
|
extra_hosts:
|
||||||
|
- host.docker.internal:host-gateway
|
||||||
|
volumes:
|
||||||
|
- automation-workspaces:/workspaces
|
||||||
|
networks:
|
||||||
|
- automation
|
||||||
|
- lai_laravel_network
|
||||||
|
|
||||||
|
volumes:
|
||||||
|
automation-workspaces:
|
||||||
|
|
||||||
|
networks:
|
||||||
|
automation:
|
||||||
|
driver: bridge
|
||||||
|
lai_laravel_network:
|
||||||
|
external: true
|
||||||
|
name: lai_laravel-network
|
||||||
32
old_2026-08-31/pyproject.toml
Normal file
32
old_2026-08-31/pyproject.toml
Normal file
@@ -0,0 +1,32 @@
|
|||||||
|
[build-system]
|
||||||
|
requires = ["hatchling"]
|
||||||
|
build-backend = "hatchling.build"
|
||||||
|
|
||||||
|
[project]
|
||||||
|
name = "lai-automation"
|
||||||
|
version = "0.1.0"
|
||||||
|
description = "Controlled coding-agent automation for Laravel AI failure reports"
|
||||||
|
readme = "README.md"
|
||||||
|
requires-python = ">=3.12"
|
||||||
|
dependencies = [
|
||||||
|
"PyMySQL>=1.1,<2",
|
||||||
|
"requests>=2.32,<3",
|
||||||
|
]
|
||||||
|
|
||||||
|
[project.optional-dependencies]
|
||||||
|
dev = ["pytest>=8.3,<9", "ruff>=0.9,<1"]
|
||||||
|
agent = ["aider-chat>=0.86,<1"]
|
||||||
|
|
||||||
|
[project.scripts]
|
||||||
|
lai-agent = "lai_automation.cli:main"
|
||||||
|
|
||||||
|
[tool.hatch.build.targets.wheel]
|
||||||
|
packages = ["src/lai_automation"]
|
||||||
|
|
||||||
|
[tool.pytest.ini_options]
|
||||||
|
pythonpath = ["src"]
|
||||||
|
testpaths = ["tests"]
|
||||||
|
|
||||||
|
[tool.ruff]
|
||||||
|
line-length = 100
|
||||||
|
target-version = "py312"
|
||||||
1
old_2026-08-31/src/lai_automation/__init__.py
Normal file
1
old_2026-08-31/src/lai_automation/__init__.py
Normal file
@@ -0,0 +1 @@
|
|||||||
|
__version__ = "0.1.0"
|
||||||
353
old_2026-08-31/src/lai_automation/agent.py
Normal file
353
old_2026-08-31/src/lai_automation/agent.py
Normal file
@@ -0,0 +1,353 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import pwd
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections import deque
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
from .prompt import redact
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
_ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]")
|
||||||
|
|
||||||
|
|
||||||
|
class AgentTimeoutError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class AgentProviderError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
_PROVIDER_FAILURE = re.compile(
|
||||||
|
r"(?:litellm\.(?:InternalServerError|APIConnectionError|RateLimitError)|"
|
||||||
|
r"MistralException|The API provider's servers are down or overloaded|"
|
||||||
|
r"Name or service not known|Temporary failure in name resolution|"
|
||||||
|
r"Connection refused|Connection timed out)",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class _OutputCapture:
|
||||||
|
def __init__(self, secret_values: tuple[str, ...] = (), limit: int = 24_000) -> None:
|
||||||
|
self.secret_values = tuple(value for value in secret_values if len(value) >= 8)
|
||||||
|
self.limit = limit
|
||||||
|
self.lines: deque[tuple[str, str]] = deque()
|
||||||
|
self.characters = 0
|
||||||
|
self.line_count = 0
|
||||||
|
self.last_activity = time.monotonic()
|
||||||
|
self.lock = threading.Lock()
|
||||||
|
|
||||||
|
def add(self, stream_name: str, value: str) -> str:
|
||||||
|
safe_value = redact(_ANSI_ESCAPE.sub("", value.rstrip("\r\n")))[:4_000]
|
||||||
|
for secret_value in self.secret_values:
|
||||||
|
safe_value = safe_value.replace(secret_value, "[REDACTED]")
|
||||||
|
with self.lock:
|
||||||
|
self.lines.append((stream_name, safe_value))
|
||||||
|
self.characters += len(safe_value)
|
||||||
|
self.line_count += 1
|
||||||
|
self.last_activity = time.monotonic()
|
||||||
|
while self.lines and self.characters > self.limit:
|
||||||
|
_, removed = self.lines.popleft()
|
||||||
|
self.characters -= len(removed)
|
||||||
|
return safe_value
|
||||||
|
|
||||||
|
def tail(self, stream_name: str | None = None, limit: int = 12_000) -> str:
|
||||||
|
with self.lock:
|
||||||
|
values = [
|
||||||
|
value
|
||||||
|
for source, value in self.lines
|
||||||
|
if stream_name is None or source == stream_name
|
||||||
|
]
|
||||||
|
result = "\n".join(values)
|
||||||
|
return result[-limit:]
|
||||||
|
|
||||||
|
def activity(self) -> tuple[int, float]:
|
||||||
|
with self.lock:
|
||||||
|
return self.line_count, time.monotonic() - self.last_activity
|
||||||
|
|
||||||
|
|
||||||
|
class AiderAgent:
|
||||||
|
def __init__(self, settings: Settings) -> None:
|
||||||
|
self.settings = settings
|
||||||
|
self.last_run_metadata: dict = {}
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
workspace: Path,
|
||||||
|
task: str,
|
||||||
|
files: tuple[str, ...] = (),
|
||||||
|
log_context: str = "Agent",
|
||||||
|
) -> str:
|
||||||
|
task_file = workspace / ".git" / "failure-task.txt"
|
||||||
|
task_file.write_text(task)
|
||||||
|
task_file.chmod(0o600)
|
||||||
|
command = self.command(task_file, files)
|
||||||
|
coder = pwd.getpwnam("coder")
|
||||||
|
git_config = Path(coder.pw_dir) / ".gitconfig"
|
||||||
|
git_config.write_text(
|
||||||
|
"[user]\n\tname = LAI Automation Coder\n\temail = coder@localhost\n"
|
||||||
|
f"[safe]\n\tdirectory = {workspace.resolve()}\n"
|
||||||
|
)
|
||||||
|
git_config.chmod(0o600)
|
||||||
|
os.chown(git_config, coder.pw_uid, coder.pw_gid)
|
||||||
|
os.chown(task_file, coder.pw_uid, coder.pw_gid)
|
||||||
|
for path in workspace.rglob("*"):
|
||||||
|
if ".git" not in path.parts:
|
||||||
|
os.chown(path, coder.pw_uid, coder.pw_gid, follow_symlinks=False)
|
||||||
|
os.chown(workspace, coder.pw_uid, coder.pw_gid, follow_symlinks=False)
|
||||||
|
|
||||||
|
environment = {
|
||||||
|
"HOME": coder.pw_dir,
|
||||||
|
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
||||||
|
}
|
||||||
|
for name in (
|
||||||
|
"MISTRAL_API_KEY",
|
||||||
|
"OPENAI_API_KEY",
|
||||||
|
"OPENAI_API_BASE",
|
||||||
|
"ANTHROPIC_API_KEY",
|
||||||
|
"OPENROUTER_API_KEY",
|
||||||
|
"OLLAMA_API_BASE",
|
||||||
|
):
|
||||||
|
if os.getenv(name):
|
||||||
|
environment[name] = os.environ[name]
|
||||||
|
|
||||||
|
capture = _OutputCapture(tuple(
|
||||||
|
environment[name]
|
||||||
|
for name in (
|
||||||
|
"MISTRAL_API_KEY",
|
||||||
|
"OPENAI_API_KEY",
|
||||||
|
"ANTHROPIC_API_KEY",
|
||||||
|
"OPENROUTER_API_KEY",
|
||||||
|
)
|
||||||
|
if environment.get(name)
|
||||||
|
))
|
||||||
|
process: subprocess.Popen[str] | None = None
|
||||||
|
readers: list[threading.Thread] = []
|
||||||
|
monitor_stopped = threading.Event()
|
||||||
|
idle_timed_out = threading.Event()
|
||||||
|
started = time.monotonic()
|
||||||
|
self.last_run_metadata = {
|
||||||
|
"model": self.settings.agent_model,
|
||||||
|
"strategy": self.edit_strategy(files),
|
||||||
|
"editable_files": list(files),
|
||||||
|
"status": "starting",
|
||||||
|
}
|
||||||
|
|
||||||
|
def read_stream(stream, stream_name: str) -> None:
|
||||||
|
try:
|
||||||
|
for line in iter(stream.readline, ""):
|
||||||
|
safe_line = capture.add(stream_name, line)
|
||||||
|
if safe_line.strip():
|
||||||
|
LOG.info(
|
||||||
|
"%s aider_%s %s",
|
||||||
|
log_context,
|
||||||
|
stream_name,
|
||||||
|
safe_line[:1_000],
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
stream.close()
|
||||||
|
|
||||||
|
def monitor() -> None:
|
||||||
|
while not monitor_stopped.wait(30):
|
||||||
|
lines, idle_seconds = capture.activity()
|
||||||
|
LOG.info(
|
||||||
|
"%s Aider pid=%s running elapsed=%ss output_lines=%s output_idle=%ss",
|
||||||
|
log_context,
|
||||||
|
process.pid if process else "pending",
|
||||||
|
round(time.monotonic() - started),
|
||||||
|
lines,
|
||||||
|
round(idle_seconds),
|
||||||
|
)
|
||||||
|
if process and self._output_stalled(idle_seconds):
|
||||||
|
idle_timed_out.set()
|
||||||
|
LOG.warning(
|
||||||
|
"%s Aider pid=%s produced no output for %ss; terminating process group",
|
||||||
|
log_context,
|
||||||
|
process.pid,
|
||||||
|
round(idle_seconds),
|
||||||
|
)
|
||||||
|
self._terminate_process_group(process)
|
||||||
|
return
|
||||||
|
|
||||||
|
monitor_thread = threading.Thread(target=monitor, daemon=True)
|
||||||
|
monitor_thread.start()
|
||||||
|
try:
|
||||||
|
try:
|
||||||
|
process = subprocess.Popen(
|
||||||
|
command,
|
||||||
|
cwd=workspace,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
errors="replace",
|
||||||
|
stdout=subprocess.PIPE,
|
||||||
|
stderr=subprocess.PIPE,
|
||||||
|
bufsize=1,
|
||||||
|
start_new_session=True,
|
||||||
|
user=coder.pw_uid,
|
||||||
|
group=coder.pw_gid,
|
||||||
|
extra_groups=(),
|
||||||
|
)
|
||||||
|
self.last_run_metadata.update({
|
||||||
|
"pid": process.pid,
|
||||||
|
"status": "running",
|
||||||
|
})
|
||||||
|
LOG.info(
|
||||||
|
"%s Aider started pid=%s model=%s strategy=%s files=%s",
|
||||||
|
log_context,
|
||||||
|
process.pid,
|
||||||
|
self.settings.agent_model,
|
||||||
|
self.edit_strategy(files),
|
||||||
|
",".join(files) or "repository-map",
|
||||||
|
)
|
||||||
|
assert process.stdout is not None
|
||||||
|
assert process.stderr is not None
|
||||||
|
readers = [
|
||||||
|
threading.Thread(
|
||||||
|
target=read_stream,
|
||||||
|
args=(process.stdout, "stdout"),
|
||||||
|
daemon=True,
|
||||||
|
),
|
||||||
|
threading.Thread(
|
||||||
|
target=read_stream,
|
||||||
|
args=(process.stderr, "stderr"),
|
||||||
|
daemon=True,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
for reader in readers:
|
||||||
|
reader.start()
|
||||||
|
process.wait(timeout=self.settings.agent_timeout_seconds + 60)
|
||||||
|
if idle_timed_out.is_set():
|
||||||
|
self.last_run_metadata["status"] = "idle_timeout"
|
||||||
|
output = capture.tail(limit=4_000)
|
||||||
|
detail = f"\nLast agent output:\n{output}" if output else ""
|
||||||
|
raise AgentTimeoutError(
|
||||||
|
"Aider produced no output for "
|
||||||
|
f"{self.settings.agent_idle_timeout_seconds} seconds.{detail}"
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired as exception:
|
||||||
|
self.last_run_metadata["status"] = "timeout"
|
||||||
|
if process:
|
||||||
|
LOG.warning(
|
||||||
|
"%s Aider pid=%s exceeded process timeout; terminating process group",
|
||||||
|
log_context,
|
||||||
|
process.pid,
|
||||||
|
)
|
||||||
|
self._terminate_process_group(process)
|
||||||
|
for reader in readers:
|
||||||
|
reader.join(timeout=2)
|
||||||
|
output = capture.tail(limit=4_000)
|
||||||
|
detail = f"\nLast agent output:\n{output}" if output else ""
|
||||||
|
raise AgentTimeoutError(
|
||||||
|
f"Aider exceeded the {self.settings.agent_timeout_seconds + 60}-second process "
|
||||||
|
f"timeout.{detail}"
|
||||||
|
) from exception
|
||||||
|
finally:
|
||||||
|
monitor_stopped.set()
|
||||||
|
monitor_thread.join(timeout=1)
|
||||||
|
for reader in readers:
|
||||||
|
reader.join(timeout=1)
|
||||||
|
lines, idle_seconds = capture.activity()
|
||||||
|
self.last_run_metadata.update({
|
||||||
|
"elapsed_seconds": round(time.monotonic() - started, 1),
|
||||||
|
"output_lines": lines,
|
||||||
|
"output_idle_seconds": round(idle_seconds, 1),
|
||||||
|
"output_tail": capture.tail(limit=4_000),
|
||||||
|
})
|
||||||
|
shutil.rmtree(workspace / ".aider.tags.cache.v4", ignore_errors=True)
|
||||||
|
for artifact in (".aider.chat.history.md", ".aider.input.history"):
|
||||||
|
(workspace / artifact).unlink(missing_ok=True)
|
||||||
|
os.chown(workspace, 0, coder.pw_gid, follow_symlinks=False)
|
||||||
|
workspace.chmod(0o775)
|
||||||
|
assert process is not None
|
||||||
|
if process.returncode != 0:
|
||||||
|
self.last_run_metadata["status"] = "failed"
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Aider failed ({process.returncode}): {capture.tail(limit=4_000)}"
|
||||||
|
)
|
||||||
|
output = capture.tail()
|
||||||
|
if _PROVIDER_FAILURE.search(output):
|
||||||
|
self.last_run_metadata["status"] = "provider_error"
|
||||||
|
raise AgentProviderError(
|
||||||
|
"Model provider failed before producing a usable repair. Last provider output:\n"
|
||||||
|
+ capture.tail(limit=4_000)
|
||||||
|
)
|
||||||
|
self.last_run_metadata["status"] = "completed"
|
||||||
|
return output
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _terminate_process_group(process: subprocess.Popen[str]) -> None:
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGTERM)
|
||||||
|
except ProcessLookupError:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
process.wait(timeout=5)
|
||||||
|
return
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
pass
|
||||||
|
try:
|
||||||
|
os.killpg(process.pid, signal.SIGKILL)
|
||||||
|
except ProcessLookupError:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
process.wait(timeout=5)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
LOG.error("Aider process group %s did not exit after SIGKILL", process.pid)
|
||||||
|
|
||||||
|
def edit_strategy(self, files: tuple[str, ...]) -> str:
|
||||||
|
if self.settings.agent_model.startswith("ollama") and any(
|
||||||
|
not filename.startswith("tests/") for filename in files
|
||||||
|
):
|
||||||
|
return "whole"
|
||||||
|
return "diff"
|
||||||
|
|
||||||
|
def _output_stalled(self, idle_seconds: float) -> bool:
|
||||||
|
return idle_seconds >= self.settings.agent_idle_timeout_seconds
|
||||||
|
|
||||||
|
def command(self, task_file: Path, files: tuple[str, ...] = ()) -> list[str]:
|
||||||
|
command = [
|
||||||
|
self.settings.agent_command,
|
||||||
|
"--model",
|
||||||
|
self.settings.agent_model,
|
||||||
|
"--message-file",
|
||||||
|
str(task_file),
|
||||||
|
"--yes",
|
||||||
|
"--no-auto-commits",
|
||||||
|
"--no-dirty-commits",
|
||||||
|
"--no-git",
|
||||||
|
"--stream",
|
||||||
|
"--no-pretty",
|
||||||
|
"--no-gitignore",
|
||||||
|
"--no-check-update",
|
||||||
|
"--no-show-release-notes",
|
||||||
|
"--no-suggest-shell-commands",
|
||||||
|
"--no-detect-urls",
|
||||||
|
"--disable-playwright",
|
||||||
|
"--chat-history-file",
|
||||||
|
"/dev/null",
|
||||||
|
"--input-history-file",
|
||||||
|
"/dev/null",
|
||||||
|
"--no-analytics",
|
||||||
|
"--map-tokens",
|
||||||
|
"0" if files else "2048",
|
||||||
|
"--no-auto-lint",
|
||||||
|
"--timeout",
|
||||||
|
str(self.settings.agent_timeout_seconds),
|
||||||
|
]
|
||||||
|
for filename in files:
|
||||||
|
if (task_file.parent.parent / filename).is_file():
|
||||||
|
command.extend(["--file", filename])
|
||||||
|
if self.settings.agent_model.startswith("ollama"):
|
||||||
|
command.extend(["--edit-format", self.edit_strategy(files)])
|
||||||
|
elif files:
|
||||||
|
command.extend(["--edit-format", "diff"])
|
||||||
|
return command
|
||||||
36
old_2026-08-31/src/lai_automation/cli.py
Normal file
36
old_2026-08-31/src/lai_automation/cli.py
Normal file
@@ -0,0 +1,36 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import logging
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
from .worker import AutomationWorker
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
parser = argparse.ArgumentParser(description="LAI controlled coding-agent automation")
|
||||||
|
parser.add_argument("command", choices=["once", "worker", "check"])
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
logging.basicConfig(
|
||||||
|
level=logging.INFO,
|
||||||
|
format="%(asctime)s %(levelname)s %(name)s %(message)s",
|
||||||
|
)
|
||||||
|
settings = Settings.from_env()
|
||||||
|
|
||||||
|
if args.command == "check":
|
||||||
|
print(f"mode={settings.mode}")
|
||||||
|
print(f"source={settings.source_repository}")
|
||||||
|
print(f"model={settings.agent_model}")
|
||||||
|
print(f"direct_main_enabled={settings.allow_direct_main}")
|
||||||
|
return
|
||||||
|
|
||||||
|
worker = AutomationWorker(settings)
|
||||||
|
if args.command == "once":
|
||||||
|
worker.run_once()
|
||||||
|
else:
|
||||||
|
worker.run_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
141
old_2026-08-31/src/lai_automation/config.py
Normal file
141
old_2026-08-31/src/lai_automation/config.py
Normal file
@@ -0,0 +1,141 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
_DEFAULT_SETUP_COMMANDS = (
|
||||||
|
"composer install --no-interaction --prefer-dist;;"
|
||||||
|
"npm ci;;"
|
||||||
|
"npm run build;;"
|
||||||
|
"php artisan migrate:fresh --force"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _bool(name: str, default: bool = False) -> bool:
|
||||||
|
return os.getenv(name, str(default)).lower() in {"1", "true", "yes", "on"}
|
||||||
|
|
||||||
|
|
||||||
|
def _commands(value: str) -> tuple[str, ...]:
|
||||||
|
return tuple(command.strip() for command in value.split(";;") if command.strip())
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class Settings:
|
||||||
|
database_host: str
|
||||||
|
database_port: int
|
||||||
|
database_name: str
|
||||||
|
database_user: str
|
||||||
|
database_password: str
|
||||||
|
source_repository: str
|
||||||
|
source_branch: str
|
||||||
|
workspace_root: Path
|
||||||
|
mode: str
|
||||||
|
allow_direct_main: bool
|
||||||
|
gitea_url: str
|
||||||
|
gitea_owner: str
|
||||||
|
gitea_repository: str
|
||||||
|
gitea_token: str
|
||||||
|
gitea_username: str
|
||||||
|
agent_command: str
|
||||||
|
agent_model: str
|
||||||
|
agent_timeout_seconds: int
|
||||||
|
agent_idle_timeout_seconds: int
|
||||||
|
setup_commands: tuple[str, ...]
|
||||||
|
test_commands: tuple[str, ...]
|
||||||
|
test_database_host: str
|
||||||
|
test_database_port: int
|
||||||
|
test_database_name: str
|
||||||
|
test_database_user: str
|
||||||
|
test_database_password: str
|
||||||
|
test_customer_database: str
|
||||||
|
test_config_database: str
|
||||||
|
poll_seconds: int
|
||||||
|
max_changed_files: int
|
||||||
|
max_changed_lines: int
|
||||||
|
max_agent_attempts: int
|
||||||
|
stale_claim_minutes: int
|
||||||
|
dry_run: bool
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_env(cls) -> Settings:
|
||||||
|
mode = os.getenv("AUTOMATION_MODE", "dry-run").lower()
|
||||||
|
if mode not in {"dry-run", "pr", "direct-main"}:
|
||||||
|
raise ValueError("AUTOMATION_MODE must be dry-run, pr, or direct-main")
|
||||||
|
|
||||||
|
settings = cls(
|
||||||
|
database_host=os.getenv("FAILURE_DB_HOST", "host.docker.internal"),
|
||||||
|
database_port=int(os.getenv("FAILURE_DB_PORT", "33060")),
|
||||||
|
database_name=os.getenv("FAILURE_DB_NAME", "laravel_ai_chat"),
|
||||||
|
database_user=os.getenv("FAILURE_DB_USER", "laravel"),
|
||||||
|
database_password=os.getenv("FAILURE_DB_PASSWORD", ""),
|
||||||
|
source_repository=os.getenv(
|
||||||
|
"SOURCE_REPOSITORY", "https://gitea.tisinfra.de/christian.meyer/lai.git"
|
||||||
|
),
|
||||||
|
source_branch=os.getenv("SOURCE_BRANCH", "main"),
|
||||||
|
workspace_root=Path(os.getenv("WORKSPACE_ROOT", "/workspaces")),
|
||||||
|
mode=mode,
|
||||||
|
allow_direct_main=_bool("ALLOW_DIRECT_MAIN"),
|
||||||
|
gitea_url=os.getenv("GITEA_URL", "https://gitea.tisinfra.de"),
|
||||||
|
gitea_owner=os.getenv("GITEA_OWNER", "christian.meyer"),
|
||||||
|
gitea_repository=os.getenv("GITEA_REPOSITORY", "lai"),
|
||||||
|
gitea_token=os.getenv("GITEA_TOKEN", ""),
|
||||||
|
gitea_username=os.getenv("GITEA_USERNAME", "automation-bot"),
|
||||||
|
agent_command=os.getenv("AGENT_COMMAND", "aider"),
|
||||||
|
agent_model=os.getenv("AGENT_MODEL", "mistral/devstral-latest"),
|
||||||
|
agent_timeout_seconds=int(os.getenv("AGENT_TIMEOUT_SECONDS", "900")),
|
||||||
|
agent_idle_timeout_seconds=int(os.getenv("AGENT_IDLE_TIMEOUT_SECONDS", "300")),
|
||||||
|
setup_commands=_commands(
|
||||||
|
os.getenv("SETUP_COMMANDS", _DEFAULT_SETUP_COMMANDS)
|
||||||
|
),
|
||||||
|
test_commands=_commands(
|
||||||
|
os.getenv("TEST_COMMANDS", "php artisan test")
|
||||||
|
),
|
||||||
|
test_database_host=os.getenv("TEST_DB_HOST", "host.docker.internal"),
|
||||||
|
test_database_port=int(os.getenv("TEST_DB_PORT", "33060")),
|
||||||
|
test_database_name=os.getenv("TEST_DB_NAME", "laravel_ai_chat_agent_test"),
|
||||||
|
test_database_user=os.getenv("TEST_DB_USER", "lai_test_runner"),
|
||||||
|
test_database_password=os.getenv("TEST_DB_PASSWORD", ""),
|
||||||
|
test_customer_database=os.getenv("TEST_CUSTOMER_DB", "customer_agent_test"),
|
||||||
|
test_config_database=os.getenv("TEST_CONFIG_DB", "cockpit_agent_test"),
|
||||||
|
poll_seconds=int(os.getenv("POLL_SECONDS", "60")),
|
||||||
|
max_changed_files=int(os.getenv("MAX_CHANGED_FILES", "12")),
|
||||||
|
max_changed_lines=int(os.getenv("MAX_CHANGED_LINES", "800")),
|
||||||
|
max_agent_attempts=int(os.getenv("MAX_AGENT_ATTEMPTS", "2")),
|
||||||
|
stale_claim_minutes=int(os.getenv("STALE_CLAIM_MINUTES", "90")),
|
||||||
|
dry_run=mode == "dry-run",
|
||||||
|
)
|
||||||
|
settings.validate()
|
||||||
|
return settings
|
||||||
|
|
||||||
|
def validate(self) -> None:
|
||||||
|
if self.mode == "direct-main" and not self.allow_direct_main:
|
||||||
|
raise ValueError("direct-main requires ALLOW_DIRECT_MAIN=true")
|
||||||
|
if self.mode in {"pr", "direct-main"} and not self.gitea_token:
|
||||||
|
raise ValueError(f"{self.mode} requires GITEA_TOKEN")
|
||||||
|
if not self.database_password:
|
||||||
|
raise ValueError("FAILURE_DB_PASSWORD is required")
|
||||||
|
if not self.test_database_password:
|
||||||
|
raise ValueError("TEST_DB_PASSWORD is required")
|
||||||
|
if not self.test_commands:
|
||||||
|
raise ValueError("TEST_COMMANDS must contain at least one command")
|
||||||
|
test_databases = {
|
||||||
|
"TEST_DB_NAME": self.test_database_name,
|
||||||
|
"TEST_CUSTOMER_DB": self.test_customer_database,
|
||||||
|
"TEST_CONFIG_DB": self.test_config_database,
|
||||||
|
}
|
||||||
|
for setting_name, database_name in test_databases.items():
|
||||||
|
if "test" not in database_name.lower():
|
||||||
|
raise ValueError(f"{setting_name} must name an isolated test database")
|
||||||
|
if database_name == self.database_name:
|
||||||
|
raise ValueError(f"{setting_name} must differ from FAILURE_DB_NAME")
|
||||||
|
if len(set(test_databases.values())) != len(test_databases):
|
||||||
|
raise ValueError("TEST_DB_NAME, TEST_CUSTOMER_DB, and TEST_CONFIG_DB must be distinct")
|
||||||
|
if not 60 <= self.agent_timeout_seconds <= 3600:
|
||||||
|
raise ValueError("AGENT_TIMEOUT_SECONDS must be between 60 and 3600")
|
||||||
|
if not 60 <= self.agent_idle_timeout_seconds <= self.agent_timeout_seconds:
|
||||||
|
raise ValueError(
|
||||||
|
"AGENT_IDLE_TIMEOUT_SECONDS must be between 60 and AGENT_TIMEOUT_SECONDS"
|
||||||
|
)
|
||||||
|
if not 1 <= self.max_agent_attempts <= 7:
|
||||||
|
raise ValueError("MAX_AGENT_ATTEMPTS must be between 1 and 7")
|
||||||
123
old_2026-08-31/src/lai_automation/database.py
Normal file
123
old_2026-08-31/src/lai_automation/database.py
Normal file
@@ -0,0 +1,123 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from contextlib import contextmanager
|
||||||
|
|
||||||
|
import pymysql
|
||||||
|
from pymysql.cursors import DictCursor
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
from .models import FailureReport
|
||||||
|
|
||||||
|
|
||||||
|
class FailureRepository:
|
||||||
|
def __init__(self, settings: Settings) -> None:
|
||||||
|
self.settings = settings
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def connection(self) -> Iterator[pymysql.Connection]:
|
||||||
|
connection = pymysql.connect(
|
||||||
|
host=self.settings.database_host,
|
||||||
|
port=self.settings.database_port,
|
||||||
|
user=self.settings.database_user,
|
||||||
|
password=self.settings.database_password,
|
||||||
|
database=self.settings.database_name,
|
||||||
|
cursorclass=DictCursor,
|
||||||
|
autocommit=False,
|
||||||
|
connect_timeout=10,
|
||||||
|
read_timeout=30,
|
||||||
|
write_timeout=30,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
yield connection
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
def claim_next(self) -> FailureReport | None:
|
||||||
|
with self.connection() as connection, connection.cursor() as cursor:
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
UPDATE failure_reports
|
||||||
|
SET status = 'open', updated_at = NOW()
|
||||||
|
WHERE status = 'agent_running'
|
||||||
|
AND updated_at < DATE_SUB(NOW(), INTERVAL %s MINUTE)
|
||||||
|
""",
|
||||||
|
(self.settings.stale_claim_minutes,),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
cursor.execute("SET TRANSACTION ISOLATION LEVEL READ COMMITTED")
|
||||||
|
cursor.execute("START TRANSACTION")
|
||||||
|
except Exception:
|
||||||
|
connection.rollback()
|
||||||
|
raise
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
SELECT * FROM failure_reports
|
||||||
|
WHERE status = 'open'
|
||||||
|
ORDER BY id
|
||||||
|
LIMIT 1
|
||||||
|
FOR UPDATE SKIP LOCKED
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
row = cursor.fetchone()
|
||||||
|
if not row:
|
||||||
|
connection.commit()
|
||||||
|
return None
|
||||||
|
cursor.execute(
|
||||||
|
"UPDATE failure_reports SET status = 'agent_running', updated_at = NOW() WHERE id = %s",
|
||||||
|
(row["id"],),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
return FailureReport.from_row(row)
|
||||||
|
|
||||||
|
def release(self, report_id: int, reason: str) -> None:
|
||||||
|
self._finish(report_id, "open", {"last_dry_run": reason})
|
||||||
|
|
||||||
|
def complete(self, report_id: int, metadata: dict) -> None:
|
||||||
|
self._finish(report_id, "resolved", {**metadata, "agent_error": None})
|
||||||
|
|
||||||
|
def fail(self, report_id: int, error: str, metadata: dict | None = None) -> None:
|
||||||
|
payload = {"agent_error": error, **(metadata or {})}
|
||||||
|
self._finish(report_id, "agent_failed", payload)
|
||||||
|
|
||||||
|
def heartbeat(self, report_id: int) -> None:
|
||||||
|
with self.connection() as connection, connection.cursor() as cursor:
|
||||||
|
try:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
UPDATE failure_reports
|
||||||
|
SET updated_at = NOW()
|
||||||
|
WHERE id = %s AND status = 'agent_running'
|
||||||
|
""",
|
||||||
|
(report_id,),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
except Exception:
|
||||||
|
connection.rollback()
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _finish(self, report_id: int, status: str, additions: dict) -> None:
|
||||||
|
with self.connection() as connection, connection.cursor() as cursor:
|
||||||
|
try:
|
||||||
|
cursor.execute("SELECT context FROM failure_reports WHERE id = %s", (report_id,))
|
||||||
|
row = cursor.fetchone() or {}
|
||||||
|
context = row.get("context") or {}
|
||||||
|
if isinstance(context, str):
|
||||||
|
context = json.loads(context)
|
||||||
|
context.update(additions)
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
UPDATE failure_reports
|
||||||
|
SET status = %s, context = %s, updated_at = NOW()
|
||||||
|
WHERE id = %s
|
||||||
|
""",
|
||||||
|
(status, json.dumps(context, ensure_ascii=True), report_id),
|
||||||
|
)
|
||||||
|
connection.commit()
|
||||||
|
except Exception as exception:
|
||||||
|
connection.rollback()
|
||||||
|
raise RuntimeError(
|
||||||
|
f"Could not finalize failure report {report_id} as {status}"
|
||||||
|
) from exception
|
||||||
234
old_2026-08-31/src/lai_automation/git.py
Normal file
234
old_2026-08-31/src/lai_automation/git.py
Normal file
@@ -0,0 +1,234 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import os
|
||||||
|
import pwd
|
||||||
|
import shlex
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
from collections.abc import Mapping, Sequence
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import urlparse
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ChangeSummary:
|
||||||
|
files: tuple[str, ...]
|
||||||
|
added_lines: int
|
||||||
|
deleted_lines: int
|
||||||
|
|
||||||
|
@property
|
||||||
|
def changed_lines(self) -> int:
|
||||||
|
return self.added_lines + self.deleted_lines
|
||||||
|
|
||||||
|
|
||||||
|
class GitWorkspace:
|
||||||
|
def __init__(self, settings: Settings, report_id: int) -> None:
|
||||||
|
self.settings = settings
|
||||||
|
self.path = settings.workspace_root / f"failure-{report_id}"
|
||||||
|
self.branch = f"ai/failure-{report_id}"
|
||||||
|
|
||||||
|
def prepare(self) -> None:
|
||||||
|
repository = urlparse(self.settings.source_repository)
|
||||||
|
gitea = urlparse(self.settings.gitea_url)
|
||||||
|
if repository.scheme != "https" or repository.hostname != gitea.hostname:
|
||||||
|
raise ValueError("SOURCE_REPOSITORY must use HTTPS on the configured Gitea host")
|
||||||
|
shutil.rmtree(self.path, ignore_errors=True)
|
||||||
|
self.path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
clone = self._git_authenticated if self.settings.gitea_token else self._git
|
||||||
|
try:
|
||||||
|
clone(
|
||||||
|
"clone",
|
||||||
|
"--branch",
|
||||||
|
self.settings.source_branch,
|
||||||
|
"--single-branch",
|
||||||
|
self.settings.source_repository,
|
||||||
|
str(self.path),
|
||||||
|
cwd=self.path.parent,
|
||||||
|
)
|
||||||
|
except subprocess.CalledProcessError as exception:
|
||||||
|
output = "\n".join(filter(None, (exception.stdout, exception.stderr))).strip()
|
||||||
|
if len(output) > 4_000:
|
||||||
|
output = output[-4_000:]
|
||||||
|
raise RuntimeError(
|
||||||
|
"Source repository clone failed"
|
||||||
|
+ (f":\n{output}" if output else " without diagnostic output")
|
||||||
|
) from exception
|
||||||
|
self._git("config", "user.name", "LAI Automation")
|
||||||
|
self._git("config", "user.email", "automation@localhost")
|
||||||
|
self._git("checkout", "-b", self.branch)
|
||||||
|
|
||||||
|
def run(
|
||||||
|
self,
|
||||||
|
command: str,
|
||||||
|
check: bool = True,
|
||||||
|
username: str | None = None,
|
||||||
|
extra_environment: Mapping[str, str] | None = None,
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
return self.run_args(
|
||||||
|
shlex.split(command),
|
||||||
|
check=check,
|
||||||
|
username=username,
|
||||||
|
extra_environment=extra_environment,
|
||||||
|
)
|
||||||
|
|
||||||
|
def run_args(
|
||||||
|
self,
|
||||||
|
command: Sequence[str],
|
||||||
|
check: bool = True,
|
||||||
|
username: str | None = None,
|
||||||
|
extra_environment: Mapping[str, str] | None = None,
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
environment = dict(os.environ)
|
||||||
|
launch_identity = {}
|
||||||
|
if username:
|
||||||
|
account = pwd.getpwnam(username)
|
||||||
|
environment = {
|
||||||
|
"HOME": account.pw_dir,
|
||||||
|
"PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"),
|
||||||
|
}
|
||||||
|
launch_identity = {
|
||||||
|
"user": account.pw_uid,
|
||||||
|
"group": account.pw_gid,
|
||||||
|
"extra_groups": (),
|
||||||
|
}
|
||||||
|
|
||||||
|
environment.update(extra_environment or {})
|
||||||
|
|
||||||
|
return subprocess.run(
|
||||||
|
command,
|
||||||
|
cwd=self.path,
|
||||||
|
env=environment,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=check,
|
||||||
|
timeout=1800,
|
||||||
|
**launch_identity,
|
||||||
|
)
|
||||||
|
|
||||||
|
def discard_changes(self) -> None:
|
||||||
|
self._git("reset", "--hard", "HEAD")
|
||||||
|
self._git("clean", "-fd")
|
||||||
|
|
||||||
|
def checkpoint(self) -> str:
|
||||||
|
self._git("add", "--all")
|
||||||
|
return self._git("stash", "create").stdout.strip()
|
||||||
|
|
||||||
|
def restore_checkpoint(self, checkpoint: str | None) -> None:
|
||||||
|
self.discard_changes()
|
||||||
|
if checkpoint:
|
||||||
|
self._git("stash", "apply", checkpoint)
|
||||||
|
|
||||||
|
def summary(self) -> ChangeSummary:
|
||||||
|
self._git("add", "-N", "--all")
|
||||||
|
try:
|
||||||
|
output = self._git("diff", "HEAD", "--numstat").stdout
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
output = self._git("diff", "--numstat").stdout
|
||||||
|
files: list[str] = []
|
||||||
|
added = deleted = 0
|
||||||
|
for line in output.splitlines():
|
||||||
|
add, remove, filename = line.split("\t", 2)
|
||||||
|
files.append(filename)
|
||||||
|
added += int(add) if add.isdigit() else 0
|
||||||
|
deleted += int(remove) if remove.isdigit() else 0
|
||||||
|
return ChangeSummary(tuple(files), added, deleted)
|
||||||
|
|
||||||
|
def diff(self, limit: int = 12_000) -> str:
|
||||||
|
try:
|
||||||
|
output = self._git("diff", "HEAD", "--no-ext-diff", "--unified=3").stdout
|
||||||
|
except subprocess.CalledProcessError:
|
||||||
|
output = self._git("diff", "--no-ext-diff", "--unified=3").stdout
|
||||||
|
if len(output) <= limit:
|
||||||
|
return output
|
||||||
|
half = limit // 2
|
||||||
|
return output[:half] + "\n...[diff middle omitted]...\n" + output[-half:]
|
||||||
|
|
||||||
|
def added_text(self) -> str:
|
||||||
|
output = self._git("diff", "HEAD", "--no-ext-diff", "--unified=0").stdout
|
||||||
|
return "\n".join(
|
||||||
|
line[1:]
|
||||||
|
for line in output.splitlines()
|
||||||
|
if line.startswith("+") and not line.startswith("+++")
|
||||||
|
)
|
||||||
|
|
||||||
|
def head_file(self, filename: str) -> str | None:
|
||||||
|
result = subprocess.run(
|
||||||
|
["git", "show", f"HEAD:{filename}"],
|
||||||
|
cwd=self.path,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=False,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
return result.stdout if result.returncode == 0 else None
|
||||||
|
|
||||||
|
def commit(self, report_id: int) -> str:
|
||||||
|
self._git("add", "--all")
|
||||||
|
self._git("commit", "-m", f"Fix failure report #{report_id}")
|
||||||
|
return self._git("rev-parse", "HEAD").stdout.strip()
|
||||||
|
|
||||||
|
def push_branch(self) -> None:
|
||||||
|
self._git_authenticated("push", "origin", f"HEAD:{self.branch}")
|
||||||
|
|
||||||
|
def push_main(self) -> None:
|
||||||
|
self._git_authenticated("push", "origin", f"HEAD:{self.settings.source_branch}")
|
||||||
|
|
||||||
|
def cleanup(self) -> None:
|
||||||
|
if self.path.exists():
|
||||||
|
shutil.rmtree(self.path)
|
||||||
|
|
||||||
|
def _git(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]:
|
||||||
|
return subprocess.run(
|
||||||
|
["git", *args],
|
||||||
|
cwd=cwd or self.path,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=True,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _git_authenticated(
|
||||||
|
self,
|
||||||
|
*args: str,
|
||||||
|
cwd: Path | None = None,
|
||||||
|
) -> subprocess.CompletedProcess[str]:
|
||||||
|
working_directory = cwd or self.path
|
||||||
|
auth_directory = Path(tempfile.mkdtemp(
|
||||||
|
prefix=".lai-git-auth-",
|
||||||
|
dir=working_directory,
|
||||||
|
))
|
||||||
|
askpass = auth_directory / "askpass.sh"
|
||||||
|
username_file = auth_directory / "username"
|
||||||
|
token_file = auth_directory / "token"
|
||||||
|
try:
|
||||||
|
username_file.write_text(self.settings.gitea_username)
|
||||||
|
token_file.write_text(self.settings.gitea_token)
|
||||||
|
username_file.chmod(0o600)
|
||||||
|
token_file.chmod(0o600)
|
||||||
|
askpass.write_text(
|
||||||
|
"#!/bin/sh\ncase \"$1\" in *Username*) cat \"$GIT_BOT_USER_FILE\";; *) cat \"$GIT_BOT_TOKEN_FILE\";; esac\n"
|
||||||
|
)
|
||||||
|
askpass.chmod(0o700)
|
||||||
|
env = {
|
||||||
|
"HOME": os.environ.get("HOME", "/root"),
|
||||||
|
"PATH": os.environ.get("PATH", "/usr/bin:/bin"),
|
||||||
|
"GIT_ASKPASS": str(askpass),
|
||||||
|
"GIT_TERMINAL_PROMPT": "0",
|
||||||
|
"GIT_BOT_USER_FILE": str(username_file),
|
||||||
|
"GIT_BOT_TOKEN_FILE": str(token_file),
|
||||||
|
}
|
||||||
|
return subprocess.run(
|
||||||
|
["git", *args],
|
||||||
|
cwd=working_directory,
|
||||||
|
env=env,
|
||||||
|
text=True,
|
||||||
|
capture_output=True,
|
||||||
|
check=True,
|
||||||
|
timeout=300,
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
shutil.rmtree(auth_directory, ignore_errors=True)
|
||||||
46
old_2026-08-31/src/lai_automation/gitea.py
Normal file
46
old_2026-08-31/src/lai_automation/gitea.py
Normal file
@@ -0,0 +1,46 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import requests
|
||||||
|
from requests.adapters import HTTPAdapter
|
||||||
|
from urllib3.util.retry import Retry
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
class GiteaClient:
|
||||||
|
def __init__(self, settings: Settings) -> None:
|
||||||
|
self.settings = settings
|
||||||
|
self.session = requests.Session()
|
||||||
|
self.session.headers.update({
|
||||||
|
"Authorization": f"token {settings.gitea_token}",
|
||||||
|
"Accept": "application/json",
|
||||||
|
})
|
||||||
|
retry = Retry(
|
||||||
|
total=3,
|
||||||
|
backoff_factor=1,
|
||||||
|
status_forcelist=(429, 500, 502, 503, 504),
|
||||||
|
allowed_methods=("POST",),
|
||||||
|
)
|
||||||
|
self.session.mount("https://", HTTPAdapter(max_retries=retry))
|
||||||
|
|
||||||
|
def create_pull_request(self, branch: str, title: str, body: str) -> str:
|
||||||
|
url = (
|
||||||
|
f"{self.settings.gitea_url.rstrip('/')}/api/v1/repos/"
|
||||||
|
f"{self.settings.gitea_owner}/{self.settings.gitea_repository}/pulls"
|
||||||
|
)
|
||||||
|
response = self.session.post(
|
||||||
|
url,
|
||||||
|
json={
|
||||||
|
"base": self.settings.source_branch,
|
||||||
|
"head": branch,
|
||||||
|
"title": title,
|
||||||
|
"body": body,
|
||||||
|
},
|
||||||
|
timeout=30,
|
||||||
|
)
|
||||||
|
response.raise_for_status()
|
||||||
|
payload = response.json()
|
||||||
|
pull_request_url = payload.get("html_url") or payload.get("url")
|
||||||
|
if not pull_request_url:
|
||||||
|
raise ValueError("Gitea response did not contain a pull request URL")
|
||||||
|
return pull_request_url
|
||||||
41
old_2026-08-31/src/lai_automation/models.py
Normal file
41
old_2026-08-31/src/lai_automation/models.py
Normal file
@@ -0,0 +1,41 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import json
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class FailureReport:
|
||||||
|
id: int
|
||||||
|
chat_message_id: int | None
|
||||||
|
source: str
|
||||||
|
category: str
|
||||||
|
user_feedback: str | None
|
||||||
|
feedback_type: str | None
|
||||||
|
expected_behavior: str | None
|
||||||
|
prompt: str | None
|
||||||
|
response: str | None
|
||||||
|
sql_query: str | None
|
||||||
|
error_message: str | None
|
||||||
|
context: dict[str, Any]
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def from_row(cls, row: dict[str, Any]) -> FailureReport:
|
||||||
|
context = row.get("context") or {}
|
||||||
|
if isinstance(context, str):
|
||||||
|
context = json.loads(context)
|
||||||
|
return cls(
|
||||||
|
id=row["id"],
|
||||||
|
chat_message_id=row.get("chat_message_id"),
|
||||||
|
source=row["source"],
|
||||||
|
category=row["category"],
|
||||||
|
user_feedback=row.get("user_feedback"),
|
||||||
|
feedback_type=row.get("feedback_type"),
|
||||||
|
expected_behavior=row.get("expected_behavior"),
|
||||||
|
prompt=row.get("prompt"),
|
||||||
|
response=row.get("response"),
|
||||||
|
sql_query=row.get("sql_query"),
|
||||||
|
error_message=row.get("error_message"),
|
||||||
|
context=context,
|
||||||
|
)
|
||||||
788
old_2026-08-31/src/lai_automation/policy.py
Normal file
788
old_2026-08-31/src/lai_automation/policy.py
Normal file
@@ -0,0 +1,788 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
from pathlib import Path, PurePosixPath
|
||||||
|
|
||||||
|
from .config import Settings
|
||||||
|
from .git import ChangeSummary, GitWorkspace
|
||||||
|
from .models import FailureReport
|
||||||
|
from .prompt import (
|
||||||
|
php_negative_assertion,
|
||||||
|
requires_explicit_route_repair,
|
||||||
|
requires_table_output_repair,
|
||||||
|
)
|
||||||
|
from .removal import (
|
||||||
|
find_matching_reference,
|
||||||
|
is_removal_only,
|
||||||
|
normalize_source_layout,
|
||||||
|
normalize_sql,
|
||||||
|
removal_directives,
|
||||||
|
remove_sql_fragment,
|
||||||
|
sql_contains_fragment,
|
||||||
|
without_reference_sql,
|
||||||
|
)
|
||||||
|
|
||||||
|
_ALWAYS_DENIED = (
|
||||||
|
".env",
|
||||||
|
"vendor/",
|
||||||
|
"storage/",
|
||||||
|
"bootstrap/cache/",
|
||||||
|
"docker/",
|
||||||
|
"docker-compose",
|
||||||
|
".gitea/",
|
||||||
|
".github/",
|
||||||
|
".aider",
|
||||||
|
)
|
||||||
|
|
||||||
|
_DIRECT_MAIN_DENIED = (
|
||||||
|
"database/migrations/",
|
||||||
|
"app/Http/Middleware/",
|
||||||
|
"app/Models/User.php",
|
||||||
|
"app/Services/PermissionService.php",
|
||||||
|
"app/Services/SQLValidatorService.php",
|
||||||
|
"config/auth.php",
|
||||||
|
"config/database.php",
|
||||||
|
"routes/",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class PolicyResult:
|
||||||
|
allowed: bool
|
||||||
|
errors: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_changes(
|
||||||
|
summary: ChangeSummary,
|
||||||
|
settings: Settings,
|
||||||
|
workspace: Path | None = None,
|
||||||
|
) -> PolicyResult:
|
||||||
|
errors: list[str] = []
|
||||||
|
if not summary.files:
|
||||||
|
errors.append("Agent produced no file changes")
|
||||||
|
elif all(filename.startswith("tests/") for filename in summary.files):
|
||||||
|
errors.append("Agent changed only tests; production behavior was not fixed")
|
||||||
|
if len(summary.files) > settings.max_changed_files:
|
||||||
|
errors.append(f"Changed {len(summary.files)} files; limit is {settings.max_changed_files}")
|
||||||
|
if summary.changed_lines > settings.max_changed_lines:
|
||||||
|
errors.append(f"Changed {summary.changed_lines} lines; limit is {settings.max_changed_lines}")
|
||||||
|
|
||||||
|
for filename in summary.files:
|
||||||
|
if PurePosixPath(filename).is_absolute() or ".." in PurePosixPath(filename).parts:
|
||||||
|
errors.append(f"Invalid changed path: {filename}")
|
||||||
|
continue
|
||||||
|
if workspace and (workspace / filename).is_symlink():
|
||||||
|
errors.append(f"Symbolic links are not allowed: {filename}")
|
||||||
|
continue
|
||||||
|
if any(filename == prefix or filename.startswith(prefix) for prefix in _ALWAYS_DENIED):
|
||||||
|
errors.append(f"Denied path changed: {filename}")
|
||||||
|
if settings.mode == "direct-main" and any(
|
||||||
|
filename == prefix or filename.startswith(prefix) for prefix in _DIRECT_MAIN_DENIED
|
||||||
|
):
|
||||||
|
errors.append(f"Direct-main denied path changed: {filename}")
|
||||||
|
|
||||||
|
return PolicyResult(not errors, tuple(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_report_scope(report: FailureReport, summary: ChangeSummary) -> PolicyResult:
|
||||||
|
expected = f"{report.user_feedback or ''}\n{report.expected_behavior or ''}".lower()
|
||||||
|
requires_cross_database_orchestration = (
|
||||||
|
"cockpit_config" in expected and ("fk_" in expected or "erp" in expected)
|
||||||
|
)
|
||||||
|
has_application_change = any(filename.startswith("app/") for filename in summary.files)
|
||||||
|
errors = []
|
||||||
|
if requires_cross_database_orchestration and not has_application_change:
|
||||||
|
errors.append(
|
||||||
|
"Cross-database report changed no application code; inspect routing and implement staged orchestration"
|
||||||
|
)
|
||||||
|
|
||||||
|
return PolicyResult(not errors, tuple(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_routed_files(
|
||||||
|
summary: ChangeSummary,
|
||||||
|
routed_files: tuple[str, ...],
|
||||||
|
) -> PolicyResult:
|
||||||
|
if not routed_files:
|
||||||
|
return PolicyResult(True, ())
|
||||||
|
unexpected = sorted(set(summary.files) - set(routed_files))
|
||||||
|
if not unexpected:
|
||||||
|
return PolicyResult(True, ())
|
||||||
|
return PolicyResult(False, (
|
||||||
|
"Agent changed files outside the routed repair scope: " + ", ".join(unexpected),
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_requested_behavior_scope(
|
||||||
|
report: FailureReport,
|
||||||
|
summary: ChangeSummary,
|
||||||
|
added_text: str,
|
||||||
|
) -> PolicyResult:
|
||||||
|
errors: list[str] = []
|
||||||
|
changed_files = set(summary.files)
|
||||||
|
added_lower = added_text.casefold()
|
||||||
|
|
||||||
|
if requires_table_output_repair(report):
|
||||||
|
required_files = {
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
}
|
||||||
|
missing_files = sorted(required_files - changed_files)
|
||||||
|
if missing_files:
|
||||||
|
errors.append(
|
||||||
|
"Table-presentation requirement must change the real answer formatter and its "
|
||||||
|
"provider-backed regression test; missing: " + ", ".join(missing_files)
|
||||||
|
)
|
||||||
|
if report.prompt and report.prompt.casefold() not in added_lower:
|
||||||
|
errors.append(
|
||||||
|
"Table-presentation regression test must use the exact reported user prompt so "
|
||||||
|
"grouped output is exercised without an explicit table keyword"
|
||||||
|
)
|
||||||
|
if "markdown-tabelle" not in added_lower and "markdown table" not in added_lower:
|
||||||
|
errors.append(
|
||||||
|
"Table-presentation repair must assert a Markdown-table answer instruction"
|
||||||
|
)
|
||||||
|
if not re.search(r"count\s*\(\s*\$rows\s*\)\s*>\s*1", added_text, re.IGNORECASE):
|
||||||
|
errors.append(
|
||||||
|
"Grouped-series table detection must require multiple rows with count($rows) > 1; "
|
||||||
|
"do not turn every multi-column scalar/detail result into a table"
|
||||||
|
)
|
||||||
|
if not re.search(
|
||||||
|
r"single[_\s-]row|one[_\s-]row|einzelne[nr]?\s+(?:zeile|datensatz)",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"Add a negative regression test proving one ordinary multi-column result row keeps "
|
||||||
|
"the sentence-format instruction"
|
||||||
|
)
|
||||||
|
if re.search(
|
||||||
|
r"assertStringContainsString\s*\(\s*['\"]alle bereitgestellten Zeilen",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"The table regression must assert the exact grammatical substring "
|
||||||
|
"'allen bereitgestellten Zeilen', not 'alle bereitgestellten Zeilen'"
|
||||||
|
)
|
||||||
|
row_count_assertions = (
|
||||||
|
(
|
||||||
|
r"assertSame\s*\(\s*3\s*,\s*substr_count\s*\(\s*\$userPrompt\s*,\s*"
|
||||||
|
r"['\"](?:\\?['\"])?Monat(?:\\?['\"])?['\"]\s*\)\s*\)"
|
||||||
|
),
|
||||||
|
(
|
||||||
|
r"assertSame\s*\(\s*3\s*,\s*substr_count\s*\(\s*\$userPrompt\s*,\s*"
|
||||||
|
r"['\"](?:\\?['\"])?Anzahl_Lieferscheine(?:\\?['\"])?['\"]\s*\)\s*\)"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if any(not re.search(pattern, added_text, re.IGNORECASE) for pattern in row_count_assertions):
|
||||||
|
errors.append(
|
||||||
|
"Grouped-table provider regression must assert every supplied row reaches "
|
||||||
|
"$userPrompt; a static mocked Markdown response is vacuous"
|
||||||
|
)
|
||||||
|
|
||||||
|
if requires_explicit_route_repair(report):
|
||||||
|
desired = "\n".join(filter(None, (
|
||||||
|
report.user_feedback,
|
||||||
|
report.expected_behavior,
|
||||||
|
))).casefold()
|
||||||
|
required_files = {
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
}
|
||||||
|
if "www:" in desired:
|
||||||
|
required_files.update({
|
||||||
|
"app/Services/WebSearchService.php",
|
||||||
|
"tests/Unit/WebSearchServiceTest.php",
|
||||||
|
})
|
||||||
|
missing_files = sorted(required_files - changed_files)
|
||||||
|
if missing_files:
|
||||||
|
errors.append(
|
||||||
|
"Explicit route-prefix requirement must change ChatController routing and its "
|
||||||
|
"deterministic regression test; missing: " + ", ".join(missing_files)
|
||||||
|
)
|
||||||
|
missing_prefixes = [
|
||||||
|
prefix
|
||||||
|
for prefix in ("cockpit:", "www:", "direct:")
|
||||||
|
if prefix in desired and prefix not in added_lower
|
||||||
|
]
|
||||||
|
if missing_prefixes:
|
||||||
|
errors.append(
|
||||||
|
"Explicit route-prefix regression coverage is missing: "
|
||||||
|
+ ", ".join(missing_prefixes)
|
||||||
|
)
|
||||||
|
required_route_fragments = (
|
||||||
|
("resolveRoutePrefix", r"resolveRoutePrefix"),
|
||||||
|
("database tuple", r"\[\s*['\"]database['\"]\s*,"),
|
||||||
|
("research tuple", r"\[\s*['\"]research['\"]\s*,"),
|
||||||
|
("direct tuple", r"\[\s*['\"]direct['\"]\s*,"),
|
||||||
|
("no-prefix tuple", r"\[\s*null\s*,"),
|
||||||
|
("web research call", r"generateWebResearchResponse\s*\("),
|
||||||
|
("direct model call", r"generateDirectResponse\s*\("),
|
||||||
|
(
|
||||||
|
"destructured processMessage call",
|
||||||
|
(
|
||||||
|
r"\[\s*\$explicitRoute\s*,\s*\$message\s*\]\s*=\s*"
|
||||||
|
r"\$this->resolveRoutePrefix\(\$[A-Za-z_]\w*\)"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
("explicit route fallback", r"\$route\s*=\s*\$explicitRoute\s*\?\?"),
|
||||||
|
)
|
||||||
|
missing_route_fragments = [
|
||||||
|
label
|
||||||
|
for label, pattern in required_route_fragments
|
||||||
|
if not re.search(pattern, added_text, re.IGNORECASE)
|
||||||
|
]
|
||||||
|
if missing_route_fragments:
|
||||||
|
errors.append(
|
||||||
|
"Explicit route-prefix repair must use one tested resolveRoutePrefix helper that "
|
||||||
|
"is destructured by processMessage before route fallback; missing: "
|
||||||
|
+ ", ".join(missing_route_fragments)
|
||||||
|
)
|
||||||
|
process_message_calls = len(re.findall(r"->processMessage\s*\(", added_text))
|
||||||
|
never_classifier_expectations = len(re.findall(
|
||||||
|
r"expects\s*\([^\n;]*never\s*\(\s*\)[^;]*\)\s*"
|
||||||
|
r"->method\s*\(\s*['\"]determineRoute['\"]\s*\)",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
))
|
||||||
|
if process_message_calls < 3 or never_classifier_expectations < 3:
|
||||||
|
errors.append(
|
||||||
|
"Explicit prefix regression tests must call processMessage() for cockpit:, www:, "
|
||||||
|
"and direct: and assert AIService::determineRoute() is never called for each path; "
|
||||||
|
"helper-only tests do not prove request routing"
|
||||||
|
)
|
||||||
|
cockpit_test = re.search(
|
||||||
|
r"function\s+test[^\{]*cockpit[^\{]*\(\)\s*:\s*void\s*\{(?P<body>.*?)\n\s*\}",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
cockpit_body = cockpit_test.group("body") if cockpit_test else ""
|
||||||
|
required_false_heuristics = (
|
||||||
|
"requiresCustomerDatabase",
|
||||||
|
"requiresConfigDatabase",
|
||||||
|
"requiresErpDatabase",
|
||||||
|
)
|
||||||
|
if not cockpit_test or any(
|
||||||
|
not re.search(
|
||||||
|
rf"method\s*\(\s*['\"]{method}['\"]\s*\).*?willReturn\s*\(\s*false\s*\)",
|
||||||
|
cockpit_body,
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
for method in required_false_heuristics
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"The cockpit processMessage regression must force every normal database heuristic "
|
||||||
|
"false so only the explicit prefix can select the database route"
|
||||||
|
)
|
||||||
|
forbidden_route_patterns = (
|
||||||
|
r"resolveSource\s*\(\s*['\"](?:cockpit|www|direct):",
|
||||||
|
r"function\s+determineRouteForMessage\b",
|
||||||
|
r"\$this->defaultModel\b",
|
||||||
|
r"\$this->determineRoute\s*\(",
|
||||||
|
r"\$route\s*=\s*\$prefixRoute\s*\?\?",
|
||||||
|
r"return\s+\$this->resolveRoutePrefix\s*\(",
|
||||||
|
r"['\"]message['\"]\s*=>\s*\$originalMessage",
|
||||||
|
r"assertStringContainsString\s*\(\s*['\"](?:cockpit|www|direct):",
|
||||||
|
r"->json\s*\(",
|
||||||
|
)
|
||||||
|
if any(re.search(pattern, added_text, re.IGNORECASE) for pattern in forbidden_route_patterns):
|
||||||
|
errors.append(
|
||||||
|
"Explicit prefix tests must exercise resolveRoutePrefix and the real request route; "
|
||||||
|
"do not test resolveDatabaseSource directly or add wrappers for nonexistent "
|
||||||
|
"ChatController model-routing members"
|
||||||
|
)
|
||||||
|
|
||||||
|
return PolicyResult(not errors, tuple(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_removal_scope(
|
||||||
|
report: FailureReport,
|
||||||
|
summary: ChangeSummary,
|
||||||
|
workspace: GitWorkspace,
|
||||||
|
) -> PolicyResult:
|
||||||
|
if not is_removal_only(report):
|
||||||
|
return PolicyResult(True, ())
|
||||||
|
|
||||||
|
errors: list[str] = []
|
||||||
|
production_files = [filename for filename in summary.files if not filename.startswith("tests/")]
|
||||||
|
test_files = [filename for filename in summary.files if filename.startswith("tests/")]
|
||||||
|
if len(production_files) != 1:
|
||||||
|
errors.append(
|
||||||
|
"Removal-only repair must change exactly one owning production file"
|
||||||
|
)
|
||||||
|
return PolicyResult(False, tuple(errors))
|
||||||
|
if not test_files:
|
||||||
|
errors.append("Removal-only repair must add a focused negative regression assertion")
|
||||||
|
|
||||||
|
filename = production_files[0]
|
||||||
|
original = workspace.head_file(filename)
|
||||||
|
path = workspace.path / filename
|
||||||
|
if original is None or not path.is_file():
|
||||||
|
errors.append(f"Could not compare removal-only source file with HEAD: {filename}")
|
||||||
|
return PolicyResult(False, tuple(errors))
|
||||||
|
current = path.read_text(encoding="utf-8")
|
||||||
|
original_example, _ = find_matching_reference(original, report.prompt)
|
||||||
|
current_example, _ = find_matching_reference(current, report.prompt)
|
||||||
|
if not original_example or not current_example:
|
||||||
|
errors.append(
|
||||||
|
"Removal-only repair must update the uniquely matching tracked reference SQL"
|
||||||
|
)
|
||||||
|
return PolicyResult(False, tuple(errors))
|
||||||
|
|
||||||
|
expected_sql = original_example.sql
|
||||||
|
for directive in removal_directives(report):
|
||||||
|
if not sql_contains_fragment(original_example.sql, directive.fragment):
|
||||||
|
errors.append(
|
||||||
|
f"Forbidden SQL fragment was already absent before the agent edit: {directive.fragment}"
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
if sql_contains_fragment(current_example.sql, directive.fragment):
|
||||||
|
errors.append(
|
||||||
|
f"Removal-only repair retained forbidden SQL fragment: {directive.fragment}"
|
||||||
|
)
|
||||||
|
expected_sql = remove_sql_fragment(expected_sql, directive.fragment)
|
||||||
|
|
||||||
|
if normalize_sql(current_example.sql) != normalize_sql(expected_sql):
|
||||||
|
errors.append(
|
||||||
|
"Removal-only repair changed joins, fields, filters, grouping, ordering, or other SQL "
|
||||||
|
"outside the explicitly forbidden fragment"
|
||||||
|
)
|
||||||
|
original_outside_sql = without_reference_sql(original, original_example)
|
||||||
|
current_outside_sql = without_reference_sql(current, current_example)
|
||||||
|
if normalize_source_layout(original_outside_sql) != normalize_source_layout(current_outside_sql):
|
||||||
|
errors.append(
|
||||||
|
"Removal-only repair changed production content outside the matching reference SQL"
|
||||||
|
)
|
||||||
|
|
||||||
|
for test_file in test_files:
|
||||||
|
previous_test = workspace.head_file(test_file)
|
||||||
|
current_test_path = workspace.path / test_file
|
||||||
|
if previous_test is None or not current_test_path.is_file():
|
||||||
|
continue
|
||||||
|
current_test = current_test_path.read_text(encoding="utf-8")
|
||||||
|
if not _lines_are_subsequence(previous_test, current_test):
|
||||||
|
errors.append(
|
||||||
|
f"Removal-only repair removed or rewrote existing regression coverage: {test_file}"
|
||||||
|
)
|
||||||
|
return PolicyResult(not errors, tuple(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_removal_prevention(
|
||||||
|
report: FailureReport,
|
||||||
|
summary: ChangeSummary,
|
||||||
|
workspace: GitWorkspace,
|
||||||
|
) -> PolicyResult:
|
||||||
|
directives = removal_directives(report)
|
||||||
|
if not directives or is_removal_only(report):
|
||||||
|
return PolicyResult(True, ())
|
||||||
|
|
||||||
|
protected_references: list[tuple[str, str, str]] = []
|
||||||
|
production_additions: list[str] = []
|
||||||
|
test_additions: list[str] = []
|
||||||
|
for filename in summary.files:
|
||||||
|
original = workspace.head_file(filename)
|
||||||
|
path = workspace.path / filename
|
||||||
|
if original is None or not path.is_file():
|
||||||
|
continue
|
||||||
|
current = path.read_text(encoding="utf-8")
|
||||||
|
additions = "\n".join(_added_lines(original, current))
|
||||||
|
if filename.startswith("tests/"):
|
||||||
|
test_additions.append(additions)
|
||||||
|
continue
|
||||||
|
production_additions.append(additions)
|
||||||
|
original_example, _ = find_matching_reference(original, report.prompt)
|
||||||
|
if original_example and all(
|
||||||
|
not sql_contains_fragment(original_example.sql, directive.fragment)
|
||||||
|
for directive in directives
|
||||||
|
):
|
||||||
|
current_example, _ = find_matching_reference(current, report.prompt)
|
||||||
|
protected_references.append((
|
||||||
|
filename,
|
||||||
|
original_example.sql,
|
||||||
|
current_example.sql if current_example else "",
|
||||||
|
))
|
||||||
|
|
||||||
|
if not protected_references:
|
||||||
|
return PolicyResult(True, ())
|
||||||
|
|
||||||
|
errors: list[str] = []
|
||||||
|
for filename, original_sql, current_sql in protected_references:
|
||||||
|
if not current_sql or normalize_sql(current_sql) != normalize_sql(original_sql):
|
||||||
|
errors.append(
|
||||||
|
"The matching reference SQL was already corrected before this report and is a "
|
||||||
|
f"protected baseline; preserve it exactly instead of rewriting it: {filename}"
|
||||||
|
)
|
||||||
|
|
||||||
|
added_production = "\n".join(production_additions)
|
||||||
|
added_tests = "\n".join(test_additions)
|
||||||
|
production_lines = added_production.splitlines()
|
||||||
|
directive_identifiers = {
|
||||||
|
_directive_identifier(directive.fragment).casefold()
|
||||||
|
for directive in directives
|
||||||
|
}
|
||||||
|
unrelated_rules = [
|
||||||
|
line.strip()
|
||||||
|
for line in production_lines
|
||||||
|
if _looks_like_php_rule(line)
|
||||||
|
and not any(identifier in line.casefold() for identifier in directive_identifiers)
|
||||||
|
]
|
||||||
|
if unrelated_rules:
|
||||||
|
errors.append(
|
||||||
|
"Removal-prevention repair added unrelated business guidance. Add only the narrow rule "
|
||||||
|
"that prohibits the reported identifier; preserve existing date, document-code, lookup, "
|
||||||
|
"status, and exclusion semantics. Remove: " + " | ".join(unrelated_rules)
|
||||||
|
)
|
||||||
|
for directive in directives:
|
||||||
|
identifier = _directive_identifier(directive.fragment)
|
||||||
|
related_rules = [
|
||||||
|
line
|
||||||
|
for line in production_lines
|
||||||
|
if identifier.casefold() in line.casefold() and _has_negative_language(line)
|
||||||
|
]
|
||||||
|
if not related_rules:
|
||||||
|
errors.append(
|
||||||
|
"Add narrow reusable production guidance that explicitly prohibits using "
|
||||||
|
f"{identifier} for this business case"
|
||||||
|
)
|
||||||
|
if any(sql_contains_fragment(line, directive.fragment) for line in related_rules):
|
||||||
|
errors.append(
|
||||||
|
"Reusable production guidance must not embed the complete forbidden predicate "
|
||||||
|
f"{directive.fragment}; mention {identifier} and the prohibition without reproducing "
|
||||||
|
"the full predicate, otherwise a real context-level negative assertion cannot pass"
|
||||||
|
)
|
||||||
|
|
||||||
|
positive_context_literals = _positive_context_assertion_literals(added_tests)
|
||||||
|
if not any(
|
||||||
|
len(literal.strip()) >= 12 and literal in added_production
|
||||||
|
for literal in positive_context_literals
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"The focused context-builder test is vacuous: it only checks text that was already "
|
||||||
|
"absent from the old context. Add assertStringContainsString for unique text from the "
|
||||||
|
"new production prohibition and pass every table declared by that rule to "
|
||||||
|
"businessContext(), while retaining the exact negative assertion"
|
||||||
|
)
|
||||||
|
|
||||||
|
return PolicyResult(not errors, tuple(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def evaluate_expected_behavior(report: FailureReport, added_text: str) -> PolicyResult:
|
||||||
|
expected = report.expected_behavior or ""
|
||||||
|
if not expected.strip():
|
||||||
|
return PolicyResult(True, ())
|
||||||
|
if not isinstance(added_text, str):
|
||||||
|
added_text = ""
|
||||||
|
|
||||||
|
directives = removal_directives(report)
|
||||||
|
forbidden_text = "\n".join(item.fragment for item in directives).lower()
|
||||||
|
|
||||||
|
technical_terms = set(re.findall(
|
||||||
|
r"\b(?:[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+|[A-Za-z]+[A-Z][A-Za-z]+|\d+)\b",
|
||||||
|
expected,
|
||||||
|
))
|
||||||
|
technical_terms = {
|
||||||
|
term
|
||||||
|
for term in technical_terms
|
||||||
|
if term.lower() not in forbidden_text and not term.isupper()
|
||||||
|
}
|
||||||
|
expected_lower = expected.lower()
|
||||||
|
for status_filter in ("obsolet", "storniert", "gebucht"):
|
||||||
|
if status_filter in expected_lower and status_filter not in forbidden_text:
|
||||||
|
technical_terms.add(status_filter)
|
||||||
|
|
||||||
|
added_lower = added_text.lower()
|
||||||
|
missing = sorted(term for term in technical_terms if term.lower() not in added_lower)
|
||||||
|
errors = []
|
||||||
|
if missing:
|
||||||
|
errors.append(
|
||||||
|
"Repair does not cover explicit expected-behavior terms in added code and assertions: "
|
||||||
|
+ ", ".join(missing)
|
||||||
|
)
|
||||||
|
|
||||||
|
expected_identifiers = [
|
||||||
|
identifier
|
||||||
|
for identifier in re.findall(
|
||||||
|
r"\b[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+\b|\b[A-Za-z]+[A-Z][A-Za-z]+\b",
|
||||||
|
expected,
|
||||||
|
)
|
||||||
|
if identifier.lower() not in forbidden_text and not identifier.isupper()
|
||||||
|
]
|
||||||
|
for identifier in expected_identifiers:
|
||||||
|
escaped = re.escape(identifier)
|
||||||
|
if re.search(
|
||||||
|
rf"(?:never|niemals)\s+(?:\w+\s+){{0,6}}{escaped}",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
errors.append(f"Repair contradicts expected identifier {identifier}")
|
||||||
|
|
||||||
|
numeric_filters = [
|
||||||
|
pair
|
||||||
|
for pair in re.findall(
|
||||||
|
r"\b([A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+|[A-Za-z]+[A-Z][A-Za-z]+)\s+(\d+)\b",
|
||||||
|
expected,
|
||||||
|
)
|
||||||
|
if pair[0].lower() not in forbidden_text
|
||||||
|
]
|
||||||
|
document_codes = () if "auftragskennung" in forbidden_text else re.finditer(
|
||||||
|
r"\bauftragskennung\s+(\d+)\b",
|
||||||
|
expected,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
for match in document_codes:
|
||||||
|
pair = ("AuftragsKennung", match.group(1))
|
||||||
|
if not any(
|
||||||
|
identifier.lower() == pair[0].lower() and value == pair[1]
|
||||||
|
for identifier, value in numeric_filters
|
||||||
|
):
|
||||||
|
numeric_filters.append(pair)
|
||||||
|
technical_terms.add(pair[0])
|
||||||
|
for identifier, value in numeric_filters:
|
||||||
|
if not re.search(
|
||||||
|
rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
errors.append(f"Repair must express {identifier} = {value} explicitly")
|
||||||
|
|
||||||
|
if "obsolet" in expected_lower and "obsolet" not in forbidden_text and not re.search(
|
||||||
|
r"COALESCE\s*\([^)]*(?<![A-Za-z0-9_])obsolet(?![A-Za-z0-9_])[^)]*,\s*0\s*\)\s*=\s*0",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"Repair must explicitly exclude obsolete rows with the exact SQL form COALESCE(a.obsolet, 0) = 0"
|
||||||
|
)
|
||||||
|
if "storniert" in expected_lower and "storniert" not in forbidden_text and not re.search(
|
||||||
|
r"COALESCE\s*\([^)]*storniert[^)]*,\s*0\s*\)\s*=\s*0",
|
||||||
|
added_text,
|
||||||
|
re.IGNORECASE,
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"Repair must explicitly exclude cancelled rows with the exact SQL form COALESCE(a.bStatus_storniert, 0) = 0"
|
||||||
|
)
|
||||||
|
observed_sql = report.sql_query or ""
|
||||||
|
if (
|
||||||
|
not directives
|
||||||
|
and "gebucht" not in expected_lower
|
||||||
|
and re.search(r"bStatus_gebucht", observed_sql, re.IGNORECASE)
|
||||||
|
and re.search(r"bStatus_gebucht", added_text, re.IGNORECASE)
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"Repair copied bStatus_gebucht from the known-bad SQL although expected behavior does not request it"
|
||||||
|
)
|
||||||
|
|
||||||
|
sql_candidates = [
|
||||||
|
line.strip()
|
||||||
|
for line in added_text.splitlines()
|
||||||
|
if re.match(r"\s*SELECT\b", line, re.IGNORECASE)
|
||||||
|
]
|
||||||
|
if numeric_filters and report.category in {"incorrect_answer", "sql_execution"}:
|
||||||
|
expected_sql_identifiers = [
|
||||||
|
identifier
|
||||||
|
for identifier in expected_identifiers
|
||||||
|
if identifier.lower() != "fk_auftrag"
|
||||||
|
]
|
||||||
|
|
||||||
|
def demonstrates_expected_sql(candidate: str) -> bool:
|
||||||
|
if any(
|
||||||
|
not re.search(
|
||||||
|
rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b",
|
||||||
|
candidate,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
for identifier, value in numeric_filters
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
if any(identifier.lower() not in candidate.lower() for identifier in expected_sql_identifiers):
|
||||||
|
return False
|
||||||
|
if (
|
||||||
|
"obsolet" in expected_lower
|
||||||
|
and "obsolet" not in forbidden_text
|
||||||
|
and not re.search(
|
||||||
|
r"COALESCE\s*\([^)]*(?<![A-Za-z0-9_])obsolet(?![A-Za-z0-9_])[^)]*,\s*0\s*\)\s*=\s*0",
|
||||||
|
candidate,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
return not (
|
||||||
|
"storniert" in expected_lower
|
||||||
|
and "storniert" not in forbidden_text
|
||||||
|
and not re.search(
|
||||||
|
r"COALESCE\s*\([^)]*storniert[^)]*,\s*0\s*\)\s*=\s*0",
|
||||||
|
candidate,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
if not any(demonstrates_expected_sql(candidate) for candidate in sql_candidates):
|
||||||
|
errors.append(
|
||||||
|
"Repair must add one reference SELECT that demonstrates all expected identifiers, "
|
||||||
|
"numeric filters, and status exclusions together"
|
||||||
|
)
|
||||||
|
|
||||||
|
expects_direct_document_code = any(
|
||||||
|
identifier.lower() == "auftragskennung" for identifier, _ in numeric_filters
|
||||||
|
)
|
||||||
|
if expects_direct_document_code and any(
|
||||||
|
re.search(r"Bezeichnung_Kurz\s*=\s*['\"]LS['\"]", candidate, re.IGNORECASE)
|
||||||
|
for candidate in sql_candidates
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"Repair used an LS lookup in its reference SQL instead of relying on the authoritative direct document code"
|
||||||
|
)
|
||||||
|
if expects_direct_document_code and any(
|
||||||
|
re.search(r"\bJOIN\s+FK_Nummerkreis\b", candidate, re.IGNORECASE)
|
||||||
|
for candidate in sql_candidates
|
||||||
|
):
|
||||||
|
errors.append(
|
||||||
|
"Repair joined FK_Nummerkreis in its reference SQL although the authoritative direct document code needs no lookup"
|
||||||
|
)
|
||||||
|
|
||||||
|
required_assertion_patterns = [
|
||||||
|
(identifier, rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b")
|
||||||
|
for identifier, value in numeric_filters
|
||||||
|
]
|
||||||
|
if "obsolet" in expected_lower and "obsolet" not in forbidden_text:
|
||||||
|
required_assertion_patterns.append((
|
||||||
|
"COALESCE(obsolet, 0) = 0",
|
||||||
|
r"COALESCE.*(?<![A-Za-z0-9_])obsolet(?![A-Za-z0-9_]).*,\s*0.*=\s*0",
|
||||||
|
))
|
||||||
|
if "storniert" in expected_lower and "storniert" not in forbidden_text:
|
||||||
|
required_assertion_patterns.append((
|
||||||
|
"COALESCE(storniert, 0) = 0",
|
||||||
|
r"COALESCE.*storniert.*,\s*0.*=\s*0",
|
||||||
|
))
|
||||||
|
if required_assertion_patterns and re.search(r"assert", added_text, re.IGNORECASE):
|
||||||
|
test_assertions = "\n".join(
|
||||||
|
line for line in added_text.splitlines() if "assert" in line.lower()
|
||||||
|
)
|
||||||
|
missing_assertions = [
|
||||||
|
label
|
||||||
|
for label, pattern in required_assertion_patterns
|
||||||
|
if not re.search(pattern, test_assertions, re.IGNORECASE)
|
||||||
|
]
|
||||||
|
if missing_assertions:
|
||||||
|
errors.append(
|
||||||
|
"Focused regression assertions do not cover: " + ", ".join(missing_assertions)
|
||||||
|
)
|
||||||
|
|
||||||
|
assertion_lines = [
|
||||||
|
line for line in added_text.splitlines() if "assert" in line.lower()
|
||||||
|
]
|
||||||
|
for directive in directives:
|
||||||
|
if any(sql_contains_fragment(candidate, directive.fragment) for candidate in sql_candidates):
|
||||||
|
errors.append(
|
||||||
|
f"Reference SQL still contains explicitly forbidden fragment: {directive.fragment}"
|
||||||
|
)
|
||||||
|
negative_assertions = [
|
||||||
|
line for line in assertion_lines if _is_negative_assertion(line)
|
||||||
|
]
|
||||||
|
has_negative_assertion = any(
|
||||||
|
sql_contains_fragment(line, directive.fragment)
|
||||||
|
for line in negative_assertions
|
||||||
|
)
|
||||||
|
if not has_negative_assertion:
|
||||||
|
required_assertion = php_negative_assertion(directive.fragment)
|
||||||
|
message = (
|
||||||
|
"Focused regression test must negatively assert the exact forbidden SQL fragment: "
|
||||||
|
+ directive.fragment
|
||||||
|
+ ". Required PHP assertion (copy exactly): "
|
||||||
|
+ required_assertion
|
||||||
|
)
|
||||||
|
unqualified_fragment = _without_identifier_qualifiers(directive.fragment)
|
||||||
|
near_match = next((
|
||||||
|
line.strip()
|
||||||
|
for line in negative_assertions
|
||||||
|
if unqualified_fragment != directive.fragment
|
||||||
|
and sql_contains_fragment(line, unqualified_fragment)
|
||||||
|
), None)
|
||||||
|
if near_match:
|
||||||
|
message += (
|
||||||
|
" Found a near-match with a missing qualifier: "
|
||||||
|
+ near_match
|
||||||
|
+ " Replace it with the required assertion; qualifiers are mandatory."
|
||||||
|
)
|
||||||
|
errors.append(message)
|
||||||
|
|
||||||
|
return PolicyResult(not errors, tuple(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def _is_negative_assertion(line: str) -> bool:
|
||||||
|
return bool(re.search(
|
||||||
|
r"assert\w*(?:not|false)|notcontains|doesnotcontain|\bnot\s+in\b",
|
||||||
|
line,
|
||||||
|
re.IGNORECASE,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def _directive_identifier(fragment: str) -> str:
|
||||||
|
identifiers = re.findall(
|
||||||
|
r"(?:[A-Za-z_][A-Za-z0-9_]*\.)?([A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+)",
|
||||||
|
fragment,
|
||||||
|
)
|
||||||
|
return identifiers[-1] if identifiers else fragment
|
||||||
|
|
||||||
|
|
||||||
|
def _has_negative_language(line: str) -> bool:
|
||||||
|
return bool(re.search(
|
||||||
|
r"\b(?:nicht|nie|niemals|kein\w*|ohne|verboten|"
|
||||||
|
r"never|not|without|forbid\w*|prohibit\w*)\b",
|
||||||
|
line,
|
||||||
|
re.IGNORECASE,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def _looks_like_php_rule(line: str) -> bool:
|
||||||
|
stripped = line.strip()
|
||||||
|
return bool(
|
||||||
|
"=>" not in stripped
|
||||||
|
and re.fullmatch(r"['\"].{8,}['\"],", stripped)
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _added_lines(original: str, current: str) -> tuple[str, ...]:
|
||||||
|
before = original.splitlines()
|
||||||
|
after = current.splitlines()
|
||||||
|
additions: list[str] = []
|
||||||
|
for tag, _before_start, _before_end, after_start, after_end in SequenceMatcher(
|
||||||
|
None,
|
||||||
|
before,
|
||||||
|
after,
|
||||||
|
autojunk=False,
|
||||||
|
).get_opcodes():
|
||||||
|
if tag in {"insert", "replace"}:
|
||||||
|
additions.extend(after[after_start:after_end])
|
||||||
|
return tuple(additions)
|
||||||
|
|
||||||
|
|
||||||
|
def _positive_context_assertion_literals(text: str) -> tuple[str, ...]:
|
||||||
|
literals: list[str] = []
|
||||||
|
patterns = (
|
||||||
|
r"assertStringContainsString\(\s*'((?:\\.|[^'\\])*)'\s*,\s*\$\w*context\w*",
|
||||||
|
r'assertStringContainsString\(\s*"((?:\\.|[^"\\])*)"\s*,\s*\$\w*context\w*',
|
||||||
|
)
|
||||||
|
for pattern in patterns:
|
||||||
|
for match in re.finditer(pattern, text, re.IGNORECASE):
|
||||||
|
literals.append(
|
||||||
|
match.group(1).replace("\\'", "'").replace('\\"', '"').replace("\\\\", "\\")
|
||||||
|
)
|
||||||
|
return tuple(literals)
|
||||||
|
|
||||||
|
|
||||||
|
def _without_identifier_qualifiers(fragment: str) -> str:
|
||||||
|
return re.sub(
|
||||||
|
r"\b[A-Za-z_][A-Za-z0-9_]*\.(?=[A-Za-z_][A-Za-z0-9_]*)",
|
||||||
|
"",
|
||||||
|
fragment,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _lines_are_subsequence(original: str, current: str) -> bool:
|
||||||
|
original_lines = [line.strip() for line in original.splitlines() if line.strip()]
|
||||||
|
current_lines = iter(line.strip() for line in current.splitlines() if line.strip())
|
||||||
|
return all(any(candidate == expected for candidate in current_lines) for expected in original_lines)
|
||||||
451
old_2026-08-31/src/lai_automation/prompt.py
Normal file
451
old_2026-08-31/src/lai_automation/prompt.py
Normal file
@@ -0,0 +1,451 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
|
||||||
|
from .models import FailureReport
|
||||||
|
from .removal import is_removal_only, removal_directives
|
||||||
|
|
||||||
|
_SECRET_PATTERNS = (
|
||||||
|
(
|
||||||
|
re.compile(r"(?i)\b(?:mysql|mariadb|postgres(?:ql)?|sqlserver)://[^\s]+"),
|
||||||
|
"[DATABASE_URL]",
|
||||||
|
),
|
||||||
|
(re.compile(r"(?i)(password|passwd|secret|token|api[_ -]?key)\s*[:=]\s*\S+"), r"\1=[REDACTED]"),
|
||||||
|
(re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "[EMAIL]"),
|
||||||
|
(re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "[IP]"),
|
||||||
|
(re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer [REDACTED]"),
|
||||||
|
(re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[AWS_KEY]"),
|
||||||
|
(re.compile(r"-----BEGIN [A-Z ]+KEY-----[\s\S]+?-----END [A-Z ]+KEY-----"), "[PRIVATE_KEY]"),
|
||||||
|
(re.compile(r"\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), "[JWT]"),
|
||||||
|
)
|
||||||
|
|
||||||
|
_REPOSITORY_PLAYBOOK = """
|
||||||
|
Repository ownership map:
|
||||||
|
- app/Services/SQLGeneratorService.php assembles live schema and business context for SQL generation.
|
||||||
|
- config/erp_semantics.php owns ERP joins, metrics, formulas, filters, and reference examples.
|
||||||
|
- config/cockpit_semantics.php owns device and customer-database semantics.
|
||||||
|
- app/Services/AIService.php owns provider calls, pseudonymization boundaries, and response parsing.
|
||||||
|
- app/Services/TenantDatabaseService.php owns ERP/config/customer source selection and tenant metadata.
|
||||||
|
- Cross-database workflows are orchestrated in ChatController/TenantDatabaseService; one SQL query
|
||||||
|
cannot join tables exposed through different Laravel connections unless the tracked architecture
|
||||||
|
explicitly supports qualified access.
|
||||||
|
- app/Services/SQLValidatorService.php owns read-only SQL safety; do not weaken it to accept a fix.
|
||||||
|
- app/Http/Controllers/ChatController.php orchestrates routing, execution, answers, and diagnostics.
|
||||||
|
- app/Services/FailureTrackingService.php and FailureAutomationService.php own failure handoff.
|
||||||
|
- tests/Unit is preferred for deterministic service/context rules; tests/Feature is for HTTP,
|
||||||
|
persistence, authorization, or real isolated-database behavior.
|
||||||
|
|
||||||
|
Repair workflow:
|
||||||
|
1. Inspect the owning implementation, its configuration, and the nearest existing test first.
|
||||||
|
2. State one root-cause hypothesis and make the smallest change that can prove it.
|
||||||
|
3. Put reusable business knowledge in tracked semantics configuration, not question-specific code.
|
||||||
|
4. A prompt/context fix must test the real generated context. A deterministic SQL builder must test
|
||||||
|
its exact SQL and execute it against synthetic fixtures when useful.
|
||||||
|
5. Tests must contain assertions that execute; reject empty loops and broad unrelated fallbacks.
|
||||||
|
6. Do not expose protected production methods publicly for tests. Use a test-only subclass.
|
||||||
|
7. Do not duplicate one rule in service code and semantics config, or add explanatory comments for
|
||||||
|
self-evident test setup.
|
||||||
|
8. Inspect the final diff for malformed edits, unrelated files, hardcoded report values, and tests
|
||||||
|
that pass without exercising the changed behavior. Before retrying an edit, reread the current
|
||||||
|
file and never leave conflict markers or partial methods behind.
|
||||||
|
9. PHP config files are executable code. Preserve brackets, commas, quotes, and array nesting; run
|
||||||
|
`php -l` on every changed PHP file before broader tests.
|
||||||
|
10. The orchestrator prepares dependencies, frontend assets, migrations, and isolated synthetic test
|
||||||
|
schemas. Never require or copy production rows. Build deterministic fixtures for data-dependent
|
||||||
|
regressions and do not edit generated assets, vendor files, node_modules, or environment files.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def redact(value: str | None) -> str:
|
||||||
|
result = value or ""
|
||||||
|
for pattern, replacement in _SECRET_PATTERNS:
|
||||||
|
result = pattern.sub(replacement, result)
|
||||||
|
return result[:12_000]
|
||||||
|
|
||||||
|
|
||||||
|
def php_negative_assertion(fragment: str) -> str:
|
||||||
|
escaped = fragment.replace("\\", "\\\\").replace("'", "\\'")
|
||||||
|
return f"$this->assertStringNotContainsString('{escaped}', $context);"
|
||||||
|
|
||||||
|
|
||||||
|
def requires_table_output_repair(report: FailureReport) -> bool:
|
||||||
|
desired = _desired_report_text(report)
|
||||||
|
return bool(re.search(
|
||||||
|
r"\b(?:markdown[- ]?tabelle|tabelle|tabellar(?:isch|ische|ischer|ischen|er)?)\b",
|
||||||
|
desired,
|
||||||
|
re.IGNORECASE,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def requires_explicit_route_repair(report: FailureReport) -> bool:
|
||||||
|
desired = _desired_report_text(report).casefold()
|
||||||
|
return any(prefix in desired for prefix in ("cockpit:", "www:", "direct:"))
|
||||||
|
|
||||||
|
|
||||||
|
def _desired_report_text(report: FailureReport) -> str:
|
||||||
|
return "\n".join(filter(None, (
|
||||||
|
report.prompt,
|
||||||
|
report.user_feedback,
|
||||||
|
report.expected_behavior,
|
||||||
|
)))
|
||||||
|
|
||||||
|
|
||||||
|
def _declares_data_correct(report: FailureReport) -> bool:
|
||||||
|
desired = _desired_report_text(report)
|
||||||
|
return bool(re.search(
|
||||||
|
r"\b(?:die\s+)?daten\s+(?:sind|waren)\s+korrekt\b|"
|
||||||
|
r"\b(?:the\s+)?data\s+(?:is|are|was|were)\s+correct\b",
|
||||||
|
desired,
|
||||||
|
re.IGNORECASE,
|
||||||
|
))
|
||||||
|
|
||||||
|
|
||||||
|
def repair_files(report: FailureReport) -> tuple[str, ...]:
|
||||||
|
expected = f"{report.user_feedback or ''}\n{report.expected_behavior or ''}".lower()
|
||||||
|
report_text = "\n".join(filter(None, (
|
||||||
|
report.prompt,
|
||||||
|
report.user_feedback,
|
||||||
|
report.expected_behavior,
|
||||||
|
report.sql_query,
|
||||||
|
report.error_message,
|
||||||
|
))).lower()
|
||||||
|
if "cockpit_config" in expected and ("fk_" in expected or "erp" in expected):
|
||||||
|
return (
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"app/Services/TenantDatabaseService.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
)
|
||||||
|
if '__LAI_' in (report.response or ''):
|
||||||
|
return (
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
production_files: list[str] = []
|
||||||
|
test_files: list[str] = []
|
||||||
|
|
||||||
|
def add_file(filename: str) -> None:
|
||||||
|
target = test_files if filename.startswith("tests/") else production_files
|
||||||
|
if filename not in target:
|
||||||
|
target.append(filename)
|
||||||
|
|
||||||
|
if requires_table_output_repair(report):
|
||||||
|
add_file("app/Services/AIService.php")
|
||||||
|
add_file("tests/Feature/AIPrivacyBoundaryTest.php")
|
||||||
|
|
||||||
|
if requires_explicit_route_repair(report):
|
||||||
|
add_file("app/Http/Controllers/ChatController.php")
|
||||||
|
add_file("tests/Unit/ChatControllerSourceResolutionTest.php")
|
||||||
|
if "www:" in report_text:
|
||||||
|
add_file("app/Services/WebSearchService.php")
|
||||||
|
add_file("tests/Unit/WebSearchServiceTest.php")
|
||||||
|
|
||||||
|
if report.category in {"incorrect_answer", "sql_execution"} and (
|
||||||
|
report.context.get("database_type") == "erp"
|
||||||
|
or "fk_" in report_text
|
||||||
|
or any(term in report_text for term in (
|
||||||
|
"auftrag", "rechnung", "lieferschein", "umsatz", "gutschrift"
|
||||||
|
))
|
||||||
|
) and not _declares_data_correct(report):
|
||||||
|
add_file("config/erp_semantics.php")
|
||||||
|
add_file("tests/Unit/SQLGeneratorBusinessContextTest.php")
|
||||||
|
|
||||||
|
return (*production_files, *test_files)
|
||||||
|
|
||||||
|
|
||||||
|
def build_task(report: FailureReport) -> str:
|
||||||
|
prompt = redact(report.prompt)
|
||||||
|
feedback = redact(report.user_feedback)
|
||||||
|
expected_behavior = redact(report.expected_behavior)
|
||||||
|
response = redact(report.response)
|
||||||
|
error = redact(report.error_message)
|
||||||
|
sql = redact(report.sql_query)
|
||||||
|
tenant_scope = ''
|
||||||
|
if report.context.get('database_type') == 'customer':
|
||||||
|
tenant_scope = """
|
||||||
|
Tenant routing fact:
|
||||||
|
The customer has already selected one tenant database. SQL inside that database must not
|
||||||
|
invent an additional customer discriminator column. Use only columns present in tracked
|
||||||
|
schema/semantics, and fix SQL-generation guidance rather than rewriting one unknown column
|
||||||
|
to another guessed column. TenantDatabaseService records the explicit source type in
|
||||||
|
database.connections.<connection>.tenant_type; use that metadata instead of database-name
|
||||||
|
patterns or heuristics.
|
||||||
|
"""
|
||||||
|
sql_semantics = ''
|
||||||
|
if report.category in {'incorrect_answer', 'sql_execution'}:
|
||||||
|
sql_semantics = """
|
||||||
|
SQL repair guidance:
|
||||||
|
When the SQL is syntactically valid but uses the wrong business calculation, prefer adding or
|
||||||
|
correcting tracked domain knowledge in config/*_semantics.php so the schema prompt teaches the
|
||||||
|
model the right joins, filters, and formula. Do not hardcode one user's question or article ID
|
||||||
|
in SQLGeneratorService, and do not add post-generation replacement of guessed SQL.
|
||||||
|
|
||||||
|
Translate positive expected behavior literally into both a reusable rule and a reference SQL example.
|
||||||
|
If it gives a numeric document code, use direct equality with that code rather than a lookup label.
|
||||||
|
If it names a date column, use that exact column even when another date is common elsewhere. Add
|
||||||
|
every requested exclusion explicitly. Do not copy additional date, booking, lookup, or status
|
||||||
|
filters from the known-bad SQL when expected behavior replaces or does not request them. Regression
|
||||||
|
assertions must require the corrected fields and filters and must never assert the opposite rule.
|
||||||
|
The same single executable reference SELECT must demonstrate all requested identifiers, numeric
|
||||||
|
codes, date columns, and exclusions together; listing them only in separate prose rules is not proof.
|
||||||
|
Use an expected bare status name as that exact column token; do not invent bStatus_ or other prefixes.
|
||||||
|
The focused test must assert every required filter, including obsolete and cancellation exclusions.
|
||||||
|
Do not normalize capitalization or wording in unrelated existing rules.
|
||||||
|
|
||||||
|
For regression coverage of prompt guidance, expose the relevant protected schema/business-
|
||||||
|
context builder through a test subclass and assert that its real output contains the required
|
||||||
|
formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt
|
||||||
|
guidance to rewrite that mocked response. Keep database fixture tests only when production code
|
||||||
|
itself deterministically builds and executes the SQL under test.
|
||||||
|
|
||||||
|
For an unknown-column failure, compare every alias.column in the observed SQL with the table
|
||||||
|
schema and identify the owning table before editing. Teach the exact ownership, join, grouping,
|
||||||
|
and real column names in semantics config, then add a reference SQL example for the reported
|
||||||
|
question shape. Never replace one missing column with an unverified synonym.
|
||||||
|
|
||||||
|
When expected behavior names tables from multiple databases or connections, first inspect source
|
||||||
|
routing and existing staged-query workflows. Do not put an impossible cross-connection join only
|
||||||
|
in ERP semantics. Implement deterministic orchestration that resolves identifiers in one source
|
||||||
|
and passes only the required identifier into the next source, preserving tenant isolation.
|
||||||
|
For this cross-database report, editing only config/*_semantics.php or tests is an invalid repair.
|
||||||
|
You MUST implement production behavior in ChatController or TenantDatabaseService and cover that
|
||||||
|
exact application method in ChatControllerSourceResolutionTest. Before editing, describe the data
|
||||||
|
flow, connection used at each stage, and where the final result is assembled.
|
||||||
|
"""
|
||||||
|
placeholder_guidance = ''
|
||||||
|
if '__LAI_' in response:
|
||||||
|
placeholder_guidance = """
|
||||||
|
Privacy placeholder repair guidance:
|
||||||
|
Trace the actual answer path through AIService::generateDatabaseResponse, issued token
|
||||||
|
restoration, and ChatController's local fallback. A token can only be restored when it exists
|
||||||
|
in the exact token map returned by protect(); never test a made-up token against an unrelated
|
||||||
|
map. Cover model-invented or malformed __LAI_...__ output with an HTTP-backed AIService test
|
||||||
|
and verify the user receives a real local database-result fallback, not a raw placeholder.
|
||||||
|
"""
|
||||||
|
|
||||||
|
directives = removal_directives(report)
|
||||||
|
forbidden_lower = "\n".join(item.fragment for item in directives).lower()
|
||||||
|
expected_lower = expected_behavior.lower()
|
||||||
|
sql_literal_contract: list[str] = []
|
||||||
|
document_code = re.search(r"\bauftragskennung\s+(\d+)\b", expected_behavior, re.IGNORECASE)
|
||||||
|
if document_code and "auftragskennung" not in forbidden_lower:
|
||||||
|
sql_literal_contract.append(f"a.AuftragsKennung = {document_code.group(1)}")
|
||||||
|
if "system_created" in expected_lower and "system_created" not in forbidden_lower:
|
||||||
|
sql_literal_contract.append("a.System_created")
|
||||||
|
if "obsolet" in expected_lower and "obsolet" not in forbidden_lower:
|
||||||
|
sql_literal_contract.append("COALESCE(a.obsolet, 0) = 0")
|
||||||
|
if "storniert" in expected_lower and "storniert" not in forbidden_lower:
|
||||||
|
sql_literal_contract.append("COALESCE(a.bStatus_storniert, 0) = 0")
|
||||||
|
literal_guidance = ""
|
||||||
|
if sql_literal_contract:
|
||||||
|
literal_guidance = (
|
||||||
|
"\nExact SQL literal contract derived from expected behavior:\n- "
|
||||||
|
+ "\n- ".join(sql_literal_contract)
|
||||||
|
+ "\nOne added reference SELECT and the focused regression assertions must contain "
|
||||||
|
"every literal above. Preserve spelling and capitalization exactly. Do not add an LS "
|
||||||
|
"lookup or bStatus_gebucht unless it appears in this contract. Assertions must quote "
|
||||||
|
"these exact SQL literals; do not replace them with descriptive prose.\n"
|
||||||
|
)
|
||||||
|
removal_guidance = ""
|
||||||
|
if directives:
|
||||||
|
exact_assertions = "\n- ".join(
|
||||||
|
php_negative_assertion(item.fragment) for item in directives
|
||||||
|
)
|
||||||
|
removal_guidance = (
|
||||||
|
"\nExact SQL removal contract derived from expected behavior:\n- "
|
||||||
|
+ "\n- ".join(item.fragment for item in directives)
|
||||||
|
+ "\nEvery matching production reference SELECT must omit every fragment above. "
|
||||||
|
"The following PHP regression assertions are mandatory; copy them exactly, including "
|
||||||
|
"aliases and qualifiers:\n- "
|
||||||
|
+ exact_assertions
|
||||||
|
+ "\nA forbidden identifier is a negative requirement: do not add it positively merely "
|
||||||
|
"because it appears in expected behavior. An unqualified substring is not equivalent "
|
||||||
|
"to an alias-qualified fragment.\n"
|
||||||
|
)
|
||||||
|
if is_removal_only(report):
|
||||||
|
removal_guidance += (
|
||||||
|
"This is explicitly a removal-only repair. Preserve every other selected field, "
|
||||||
|
"table, join, date column, predicate, grouping, ordering, limit, rule, and existing "
|
||||||
|
"test assertion byte-for-byte. Change only the matching reference SQL fragment and "
|
||||||
|
"add focused negative regression coverage.\n"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
removal_guidance += (
|
||||||
|
"Before editing, inspect the matching tracked reference SELECT. If it already omits "
|
||||||
|
"the forbidden fragment, treat that SELECT as a protected correct baseline and "
|
||||||
|
"preserve it exactly. Add or clarify the narrow reusable business rule that prevents "
|
||||||
|
"the reported recurrence instead of regressing the reference merely to create a diff. "
|
||||||
|
"Add only that narrow prohibition rule. Do not add or restate date-column choices, "
|
||||||
|
"document codes, lookup strategies, joins, cancellation/obsolete rules, or other "
|
||||||
|
"business guidance already covered by the protected reference. "
|
||||||
|
"Phrase the rule without embedding the complete forbidden predicate: mention its "
|
||||||
|
"column identifier and say it must not be used, so the exact context-level negative "
|
||||||
|
"assertion above remains true. The focused context-builder test must positively assert "
|
||||||
|
"the new rule with assertStringContainsString for unique text from that rule as well as use the exact "
|
||||||
|
"negative assertion above; a negative assertion alone is vacuous when the old reference "
|
||||||
|
"already omitted the fragment. A configured rule is exercised only when every "
|
||||||
|
"table in its tables list is supplied to businessContext(); align the test's allowed "
|
||||||
|
"tables with the rule so the test cannot pass while omitting the production change.\n"
|
||||||
|
)
|
||||||
|
|
||||||
|
interaction_guidance = ""
|
||||||
|
if requires_table_output_repair(report):
|
||||||
|
interaction_guidance += """
|
||||||
|
Answer-presentation repair guidance:
|
||||||
|
This report requires a real answer-formatting change, not ERP SQL or semantics changes when the
|
||||||
|
report says the data is already correct. Inspect AIService::generateDatabaseResponse(). Grouped
|
||||||
|
series requests such as counts per month must request a compact Markdown table even when the
|
||||||
|
original user prompt does not literally contain the word "Tabelle". Add a provider-backed test
|
||||||
|
using the exact reported user prompt. It must prove that the answer-generation system instruction
|
||||||
|
requests a Markdown table and that all supplied grouped rows remain available. Do not add static
|
||||||
|
example result values or output-format metadata to config/*_semantics.php. Grouped-series detection
|
||||||
|
must require multiple result rows and grouping/value columns; one ordinary result row with multiple
|
||||||
|
columns must remain sentence-formatted. Add that negative provider-backed regression test as well.
|
||||||
|
The positive grouped test uses columns named exactly Monat and Anzahl_Lieferscheine. Ensure the
|
||||||
|
production grouping/value-column detection recognizes those exact names, and confirm the positive
|
||||||
|
test receives Markdown-Tabelle before finishing; do not merely add the test without satisfying it.
|
||||||
|
Inside the provider fake, prove every supplied row reached $userPrompt without exposing protected
|
||||||
|
values: for three rows, assert substr_count($userPrompt, '"Monat"') and
|
||||||
|
substr_count($userPrompt, '"Anzahl_Lieferscheine"') both equal 3. A mocked Markdown response alone
|
||||||
|
is vacuous because it can pass even if no grouped rows reached the provider.
|
||||||
|
Keep the existing exact table instruction text: "Die Benutzerfrage verlangt eine Tabelle. Gib eine
|
||||||
|
kurze Einleitung und danach eine kompakte Markdown-Tabelle mit allen bereitgestellten Zeilen aus."
|
||||||
|
Do not move or duplicate this instruction inside shouldUseTableFormat(); that helper returns only bool.
|
||||||
|
The regression assertion must use the exact substring "allen bereitgestellten Zeilen". Never assert
|
||||||
|
"alle bereitgestellten Zeilen": that grammatically different substring is not present in the correct
|
||||||
|
production instruction. If Markdown-Tabelle passes but this assertion fails, fix the assertion—not
|
||||||
|
the already-correct production instruction.
|
||||||
|
"""
|
||||||
|
if requires_explicit_route_repair(report):
|
||||||
|
interaction_guidance += """
|
||||||
|
Explicit route-prefix repair guidance:
|
||||||
|
Implement one protected resolveRoutePrefix(string $message): array helper that always returns exactly
|
||||||
|
[explicit route or null, cleaned message]. Call it with exact destructuring at the start of
|
||||||
|
processMessage(): [$explicitRoute, $message] = $this->resolveRoutePrefix($message); Then compute
|
||||||
|
$route = $explicitRoute ?? (...) around the existing database-keyword/model-classifier expression,
|
||||||
|
before database keyword heuristics or model classification:
|
||||||
|
"cockpit:" must force the database path, "www:" must force public web research, and "direct:" must
|
||||||
|
force the selected model without web or database access, even when the remaining text contains
|
||||||
|
keywords for another route. Web retrieval must be provider-independent: WebSearchService retrieves
|
||||||
|
bounded public search snippets, then AIService::generateWebResearchResponse() passes only the clean
|
||||||
|
question and those untrusted snippets to the selected model, including Ollama/Qwen, and returns
|
||||||
|
clickable source URLs. Do not pass private conversation/database context into web research, fetch
|
||||||
|
arbitrary result URLs, or obey instructions embedded in search snippets. Remove the control prefix
|
||||||
|
before passing the user's actual question downstream. Add deterministic ChatController routing tests
|
||||||
|
for all three prefixes that prove the classifier cannot override them. For every explicit prefix,
|
||||||
|
AIService::determineRoute() must not be called; never write a test expecting that classifier call.
|
||||||
|
Helper-only tests are insufficient: add three tests that call processMessage() itself, one for each
|
||||||
|
prefix, assert determineRoute() is never called, and assert the cleaned question reaches the forced
|
||||||
|
database/research/direct response path. Mock only external collaborators, not processMessage or
|
||||||
|
prefix logic.
|
||||||
|
This is feasible in the existing test file: switch it from PHPUnit\\Framework\\TestCase to
|
||||||
|
Tests\\TestCase, use Illuminate\\Foundation\\Testing\\RefreshDatabase, build an
|
||||||
|
Illuminate\\Http\\Request with Request::create(), and attach app('session')->driver() using
|
||||||
|
setLaravelSession(). For www:, mock generateWebResearchResponse() with the exact cleaned question
|
||||||
|
and selected ollama model, assert the JSON route is research, and assert source metadata is persisted.
|
||||||
|
For direct:, mock generateDirectResponse(), assert generateWebResearchResponse() is never called,
|
||||||
|
and assert the JSON route is direct. For cockpit:, make the tenant mock return true from
|
||||||
|
requiresCustomerDatabase() and null from resolveFromPrompt(), then assert the JSON route is
|
||||||
|
customer_clarification and the persisted ChatMessage.message equals the cleaned question exactly.
|
||||||
|
For that cockpit test, all requiresCustomerDatabase/requiresConfigDatabase/requiresErpDatabase
|
||||||
|
heuristics must return false; otherwise the database route would be selected without the prefix and
|
||||||
|
the test is vacuous. The explicit cockpit route itself must be the only reason determineRoute() is
|
||||||
|
bypassed. Use a cleaned question such as "Erkläre Quantenphysik" that has no database keywords.
|
||||||
|
Each prefix is a control token and must not be stored or passed downstream. Never assert that a
|
||||||
|
persisted message contains cockpit:, www:, or direct:, and never change ChatMessage 'message' fields
|
||||||
|
to $originalMessage. In all three tests put
|
||||||
|
$aiService->expects($this->never())->method('determineRoute') before processMessage(). Use an
|
||||||
|
explicit model in each request-path test so mocked response methods do not throw an argument-mismatch
|
||||||
|
exception that processMessage() catches and converts into an error JSON response. Assert response
|
||||||
|
status/success before reading route.
|
||||||
|
Do not write “we can't fully test processMessage”; Laravel's application test case provides the
|
||||||
|
HTTP, session, response, and database context needed here.
|
||||||
|
JsonResponse has no json() method in this codebase; read it once with $payload =
|
||||||
|
$response->getData(true), then assert $payload['success'] and $payload['route'].
|
||||||
|
The helper must accept leading whitespace and case-insensitive prefixes, and trim the cleaned
|
||||||
|
question. Test the helper returns ['database', cleaned question], ['research', cleaned question],
|
||||||
|
['direct', cleaned question], and [null, unchanged trimmed question]. Do not assign the whole helper
|
||||||
|
result array to $route. Then prove
|
||||||
|
processMessage() uses that result rather than only testing resolveRoutePrefix() or
|
||||||
|
resolveDatabaseSource().
|
||||||
|
Do not add prefix handling to resolveDatabaseSource(), do not change its default-source behavior, and
|
||||||
|
do not add wrappers that reference nonexistent properties or methods such as defaultModel or
|
||||||
|
determineRoute on ChatController. Inspect the existing TestableChatController before editing:
|
||||||
|
expose resolveRoutePrefix() exactly once, and replace an existing wrapper when necessary—never
|
||||||
|
append a duplicate method declaration. The public test-subclass wrapper must call
|
||||||
|
parent::resolveRoutePrefix($message), never $this->resolveRoutePrefix($message), which recursively
|
||||||
|
calls itself. Do not reinterpret "cockpit:" as a new SQL business rule.
|
||||||
|
All three prefix behaviors and any independent presentation requirement in this report are mandatory.
|
||||||
|
"""
|
||||||
|
|
||||||
|
return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository.
|
||||||
|
|
||||||
|
{_REPOSITORY_PLAYBOOK}
|
||||||
|
|
||||||
|
Rules:
|
||||||
|
- First add a regression test that reproduces the reported behavior.
|
||||||
|
- The regression test must execute the relevant production behavior; never mock or stub the behavior being tested.
|
||||||
|
- Make the smallest production change that fixes the root cause.
|
||||||
|
- Treat the user's expected behavior as authoritative, while verifying table and column names against tracked schema and semantics.
|
||||||
|
- Every positively required technical identifier, numeric code, and status filter named in expected
|
||||||
|
behavior must appear in the production repair and focused regression assertions. Identifiers
|
||||||
|
explicitly requested for removal must instead be absent from matching SQL and appear in an exact
|
||||||
|
negative assertion. Do not substitute a pre-existing rule.
|
||||||
|
- Preserve read-only SQL validation and tenant isolation.
|
||||||
|
- Never read or modify .env files, credentials, deployment secrets, or production data.
|
||||||
|
- Do not depend on production row contents. Use synthetic fixtures or assert generated business context and SQL guidance.
|
||||||
|
- Do not weaken tests, authentication, authorization, SQL validation, or safety checks.
|
||||||
|
- Do not commit or push; the orchestrator owns Git publication.
|
||||||
|
|
||||||
|
Failure category: {report.category}
|
||||||
|
Feedback type: {report.feedback_type or 'not specified'}
|
||||||
|
Source: {report.source}
|
||||||
|
Model: {redact(str(report.context.get('model', 'unknown')))}
|
||||||
|
Database type: {redact(str(report.context.get('database_type', 'unknown')))}
|
||||||
|
|
||||||
|
User request:
|
||||||
|
{prompt or '[not available]'}
|
||||||
|
|
||||||
|
User feedback:
|
||||||
|
{feedback or '[not available]'}
|
||||||
|
|
||||||
|
Expected behavior:
|
||||||
|
{expected_behavior or '[not available]'}
|
||||||
|
|
||||||
|
Observed answer:
|
||||||
|
{response or '[not available]'}
|
||||||
|
|
||||||
|
Observed error:
|
||||||
|
{error or '[not available]'}
|
||||||
|
|
||||||
|
Generated SQL, if relevant:
|
||||||
|
{sql or '[not available]'}
|
||||||
|
{tenant_scope}
|
||||||
|
{sql_semantics}
|
||||||
|
{literal_guidance}
|
||||||
|
{removal_guidance}
|
||||||
|
{interaction_guidance}
|
||||||
|
{placeholder_guidance}
|
||||||
|
|
||||||
|
Run focused tests while working. Leave the repository with the fix and regression test applied.
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def build_retry_task(report: FailureReport, exception: Exception) -> str:
|
||||||
|
failure = redact(str(exception))
|
||||||
|
|
||||||
|
return build_task(report) + f"""
|
||||||
|
|
||||||
|
The previous implementation failed isolated validation.
|
||||||
|
|
||||||
|
Correct the existing changes using this test output:
|
||||||
|
|
||||||
|
{failure}
|
||||||
|
|
||||||
|
Keep the regression test meaningful. Do not remove or weaken tests, safety checks,
|
||||||
|
authentication, authorization, SQL validation, or tenant isolation. If a new test calls an
|
||||||
|
unrelated specialized code path that correctly returns no result, repair the test to exercise
|
||||||
|
the owning production behavior; never broaden unrelated production routing merely to make the
|
||||||
|
test pass. Do not commit or push.
|
||||||
|
"""
|
||||||
307
old_2026-08-31/src/lai_automation/removal.py
Normal file
307
old_2026-08-31/src/lai_automation/removal.py
Normal file
@@ -0,0 +1,307 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from difflib import SequenceMatcher
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .models import FailureReport
|
||||||
|
|
||||||
|
_REMOVAL_INTENT = re.compile(
|
||||||
|
r"\b(?:entfern\w*|lösch\w*|loesch\w*|weglass\w*|streichen|"
|
||||||
|
r"remove\w*|delet\w*|omit\w*|drop\w*|ohne|without)\b|"
|
||||||
|
r"\b(?:darf|soll|muss)\b.{0,50}\b(?:nicht|nie|niemals)\b|"
|
||||||
|
r"\b(?:must|should)\s+not\b|\bdo\s+not\b",
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_NEGATED_REMOVAL = re.compile(
|
||||||
|
r"\b(?:nicht|nie|niemals)\s+(?:entfern\w*|lösch\w*|loesch\w*|weglass\w*)\b|"
|
||||||
|
r"\b(?:do\s+not|don't|never)\s+(?:remove\w*|delet\w*|omit\w*|drop\w*)\b",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
_ONLY_REMOVAL = re.compile(
|
||||||
|
r"\b(?:einfach\s+)?nur\b.{0,120}\b(?:entfern\w*|lösch\w*|loesch\w*|"
|
||||||
|
r"weglass\w*|streichen)\b|"
|
||||||
|
r"\b(?:just|only)\b.{0,120}\b(?:remove\w*|delet\w*|omit\w*|drop\w*)\b",
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
_QUOTED_FRAGMENT = re.compile(
|
||||||
|
r'"(?P<double>[^"\n]{1,500})"|'
|
||||||
|
r"'(?P<single>[^'\n]{1,500})'|"
|
||||||
|
r"`(?P<backtick>[^`\n]{1,500})`|"
|
||||||
|
r"„(?P<german>[^”\n]{1,500})”"
|
||||||
|
)
|
||||||
|
_TECHNICAL_IDENTIFIER = re.compile(
|
||||||
|
r"\b(?:[A-Za-z_][A-Za-z0-9_]*\.)?[A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+\b"
|
||||||
|
)
|
||||||
|
_QUESTION_KEY = re.compile(r"['\"]question['\"]\s*=>", re.IGNORECASE)
|
||||||
|
_SQL_HEREDOC = re.compile(
|
||||||
|
r"['\"]sql['\"]\s*=>\s*<<<(?P<quote>['\"]?)(?P<label>[A-Za-z_][A-Za-z0-9_]*)"
|
||||||
|
r"(?P=quote)[^\S\r\n]*\r?\n",
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class RemovalDirective:
|
||||||
|
fragment: str
|
||||||
|
origin: str
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ReferenceExample:
|
||||||
|
question: str
|
||||||
|
sql: str
|
||||||
|
sql_start: int
|
||||||
|
sql_end: int
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class AlreadyFixedResult:
|
||||||
|
filename: str
|
||||||
|
question: str
|
||||||
|
match_score: float
|
||||||
|
forbidden_fragments: tuple[str, ...]
|
||||||
|
|
||||||
|
|
||||||
|
def removal_directives(report: FailureReport) -> tuple[RemovalDirective, ...]:
|
||||||
|
expected = report.expected_behavior or ""
|
||||||
|
if not expected.strip() or not _REMOVAL_INTENT.search(expected):
|
||||||
|
return ()
|
||||||
|
negated_removal = _NEGATED_REMOVAL.search(expected)
|
||||||
|
if negated_removal:
|
||||||
|
remaining = expected[:negated_removal.start()] + expected[negated_removal.end():]
|
||||||
|
if not _REMOVAL_INTENT.search(remaining):
|
||||||
|
return ()
|
||||||
|
|
||||||
|
observed_sql = report.sql_query or ""
|
||||||
|
directives: list[RemovalDirective] = []
|
||||||
|
for match in _QUOTED_FRAGMENT.finditer(expected):
|
||||||
|
candidate = next(value for value in match.groupdict().values() if value is not None)
|
||||||
|
candidate = clean_fragment(candidate)
|
||||||
|
if _looks_like_sql_fragment(candidate):
|
||||||
|
directives.append(RemovalDirective(candidate, "quoted expected behavior"))
|
||||||
|
|
||||||
|
if not directives:
|
||||||
|
for sentence in re.split(r"[\n.!?;]+", expected):
|
||||||
|
if not _REMOVAL_INTENT.search(sentence):
|
||||||
|
continue
|
||||||
|
for identifier in _TECHNICAL_IDENTIFIER.findall(sentence):
|
||||||
|
predicate = _predicate_containing(observed_sql, identifier)
|
||||||
|
directives.append(RemovalDirective(predicate or identifier, "removal sentence"))
|
||||||
|
|
||||||
|
deduplicated: dict[str, RemovalDirective] = {}
|
||||||
|
for directive in directives:
|
||||||
|
key = normalize_sql(directive.fragment)
|
||||||
|
if key:
|
||||||
|
deduplicated.setdefault(key, directive)
|
||||||
|
return tuple(deduplicated.values())
|
||||||
|
|
||||||
|
|
||||||
|
def is_removal_only(report: FailureReport) -> bool:
|
||||||
|
expected = report.expected_behavior or ""
|
||||||
|
return bool(removal_directives(report) and _ONLY_REMOVAL.search(expected))
|
||||||
|
|
||||||
|
|
||||||
|
def clean_fragment(fragment: str) -> str:
|
||||||
|
result = re.sub(r"\s+", " ", fragment).strip().strip(";")
|
||||||
|
result = re.sub(r"^(?:AND|OR|WHERE)\s+", "", result, flags=re.IGNORECASE)
|
||||||
|
return result.strip()
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_sql(sql: str) -> str:
|
||||||
|
return _canonical_sql(sql).strip(";").casefold()
|
||||||
|
|
||||||
|
|
||||||
|
def sql_contains_fragment(sql: str, fragment: str) -> bool:
|
||||||
|
normalized_fragment = normalize_sql(clean_fragment(fragment))
|
||||||
|
if not normalized_fragment:
|
||||||
|
return False
|
||||||
|
return normalized_fragment in normalize_sql(sql)
|
||||||
|
|
||||||
|
|
||||||
|
def remove_sql_fragment(sql: str, fragment: str) -> str:
|
||||||
|
result = _canonical_sql(sql)
|
||||||
|
target = _canonical_sql(clean_fragment(fragment))
|
||||||
|
if not target:
|
||||||
|
return result
|
||||||
|
|
||||||
|
while True:
|
||||||
|
index = result.casefold().find(target.casefold())
|
||||||
|
if index < 0:
|
||||||
|
break
|
||||||
|
before = result[:index]
|
||||||
|
after = result[index + len(target):]
|
||||||
|
leading_connector = re.search(r"\s+(?:AND|OR)\s*$", before, re.IGNORECASE)
|
||||||
|
trailing_connector = re.match(r"^\s+(?:AND|OR)\s+", after, re.IGNORECASE)
|
||||||
|
leading_where = re.search(r"\bWHERE\s*$", before, re.IGNORECASE)
|
||||||
|
if leading_connector:
|
||||||
|
result = before[:leading_connector.start()] + after
|
||||||
|
elif trailing_connector:
|
||||||
|
result = before + after[trailing_connector.end():]
|
||||||
|
elif leading_where:
|
||||||
|
result = before[:leading_where.start()] + after
|
||||||
|
else:
|
||||||
|
result = before + after
|
||||||
|
result = re.sub(r"\s+", " ", result).strip()
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def _canonical_sql(sql: str) -> str:
|
||||||
|
result = re.sub(r"\s+", " ", sql).strip()
|
||||||
|
return re.sub(r"\s*([(),.=<>])\s*", r"\1", result)
|
||||||
|
|
||||||
|
|
||||||
|
def extract_reference_examples(text: str) -> tuple[ReferenceExample, ...]:
|
||||||
|
markers = list(_QUESTION_KEY.finditer(text))
|
||||||
|
examples: list[ReferenceExample] = []
|
||||||
|
for index, marker in enumerate(markers):
|
||||||
|
block_end = markers[index + 1].start() if index + 1 < len(markers) else len(text)
|
||||||
|
question = _php_string_after(text, marker.end(), block_end)
|
||||||
|
if question is None:
|
||||||
|
continue
|
||||||
|
heredoc = _SQL_HEREDOC.search(text, marker.end(), block_end)
|
||||||
|
if not heredoc:
|
||||||
|
continue
|
||||||
|
terminator = re.compile(
|
||||||
|
rf"^{re.escape(heredoc.group('label'))}\s*,?\s*$",
|
||||||
|
re.MULTILINE,
|
||||||
|
).search(text, heredoc.end(), block_end)
|
||||||
|
if not terminator:
|
||||||
|
continue
|
||||||
|
examples.append(ReferenceExample(
|
||||||
|
question=question,
|
||||||
|
sql=text[heredoc.end():terminator.start()].strip(),
|
||||||
|
sql_start=heredoc.end(),
|
||||||
|
sql_end=terminator.start(),
|
||||||
|
))
|
||||||
|
return tuple(examples)
|
||||||
|
|
||||||
|
|
||||||
|
def find_matching_reference(
|
||||||
|
text: str,
|
||||||
|
prompt: str | None,
|
||||||
|
) -> tuple[ReferenceExample | None, float]:
|
||||||
|
if not prompt:
|
||||||
|
return None, 0.0
|
||||||
|
scored = sorted(
|
||||||
|
((_question_score(prompt, example.question), example)
|
||||||
|
for example in extract_reference_examples(text)),
|
||||||
|
key=lambda item: item[0],
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
if not scored or scored[0][0] < 0.92:
|
||||||
|
return None, 0.0
|
||||||
|
if len(scored) > 1 and scored[1][0] >= scored[0][0] - 0.05:
|
||||||
|
return None, 0.0
|
||||||
|
return scored[0][1], scored[0][0]
|
||||||
|
|
||||||
|
|
||||||
|
def detect_already_fixed(
|
||||||
|
report: FailureReport,
|
||||||
|
workspace: Path,
|
||||||
|
routed_files: tuple[str, ...],
|
||||||
|
) -> AlreadyFixedResult | None:
|
||||||
|
directives = removal_directives(report)
|
||||||
|
observed_sql = report.sql_query or ""
|
||||||
|
if not is_removal_only(report) or not observed_sql:
|
||||||
|
return None
|
||||||
|
if not all(sql_contains_fragment(observed_sql, item.fragment) for item in directives):
|
||||||
|
return None
|
||||||
|
|
||||||
|
matches: list[tuple[str, ReferenceExample, float]] = []
|
||||||
|
for filename in routed_files:
|
||||||
|
if filename.startswith("tests/"):
|
||||||
|
continue
|
||||||
|
path = workspace / filename
|
||||||
|
if not path.is_file():
|
||||||
|
continue
|
||||||
|
example, score = find_matching_reference(path.read_text(encoding="utf-8"), report.prompt)
|
||||||
|
if example:
|
||||||
|
matches.append((filename, example, score))
|
||||||
|
if len(matches) != 1:
|
||||||
|
return None
|
||||||
|
|
||||||
|
filename, example, score = matches[0]
|
||||||
|
if any(sql_contains_fragment(example.sql, item.fragment) for item in directives):
|
||||||
|
return None
|
||||||
|
return AlreadyFixedResult(
|
||||||
|
filename=filename,
|
||||||
|
question=example.question,
|
||||||
|
match_score=score,
|
||||||
|
forbidden_fragments=tuple(item.fragment for item in directives),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def without_reference_sql(text: str, example: ReferenceExample) -> str:
|
||||||
|
return text[:example.sql_start] + "<REFERENCE_SQL>" + text[example.sql_end:]
|
||||||
|
|
||||||
|
|
||||||
|
def normalize_source_layout(text: str) -> str:
|
||||||
|
lines = [line.rstrip() for line in text.replace("\r\n", "\n").splitlines()]
|
||||||
|
return "\n".join(lines).strip()
|
||||||
|
|
||||||
|
|
||||||
|
def _looks_like_sql_fragment(value: str) -> bool:
|
||||||
|
return bool(
|
||||||
|
_TECHNICAL_IDENTIFIER.search(value)
|
||||||
|
or re.search(r"\b(?:WHERE|AND|OR|JOIN|GROUP\s+BY|ORDER\s+BY)\b", value, re.IGNORECASE)
|
||||||
|
or (re.search(r"[=<>]", value) and re.search(r"[A-Za-z_]", value))
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _predicate_containing(sql: str, identifier: str) -> str | None:
|
||||||
|
clauses = re.finditer(
|
||||||
|
r"\b(?:WHERE|AND|OR)\b\s*(?P<predicate>.*?)"
|
||||||
|
r"(?=\s+\b(?:AND|OR|GROUP\s+BY|ORDER\s+BY|HAVING|LIMIT)\b|;|$)",
|
||||||
|
sql,
|
||||||
|
re.IGNORECASE | re.DOTALL,
|
||||||
|
)
|
||||||
|
for clause in clauses:
|
||||||
|
predicate = clause.group("predicate").strip()
|
||||||
|
if re.search(rf"(?<![A-Za-z0-9_]){re.escape(identifier)}(?![A-Za-z0-9_])", predicate, re.IGNORECASE):
|
||||||
|
return predicate
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _php_string_after(text: str, start: int, end: int) -> str | None:
|
||||||
|
position = start
|
||||||
|
while position < end and text[position].isspace():
|
||||||
|
position += 1
|
||||||
|
if position >= end or text[position] not in {"'", '"'}:
|
||||||
|
return None
|
||||||
|
quote = text[position]
|
||||||
|
position += 1
|
||||||
|
value: list[str] = []
|
||||||
|
while position < end:
|
||||||
|
character = text[position]
|
||||||
|
if character == "\\" and position + 1 < end:
|
||||||
|
value.append(text[position + 1])
|
||||||
|
position += 2
|
||||||
|
continue
|
||||||
|
if character == quote:
|
||||||
|
return "".join(value)
|
||||||
|
value.append(character)
|
||||||
|
position += 1
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _normalize_question(value: str) -> str:
|
||||||
|
normalized = unicodedata.normalize("NFKD", value.casefold().replace("ß", "ss"))
|
||||||
|
normalized = "".join(character for character in normalized if not unicodedata.combining(character))
|
||||||
|
return " ".join(re.findall(r"[a-z0-9]+", normalized))
|
||||||
|
|
||||||
|
|
||||||
|
def _question_score(left: str, right: str) -> float:
|
||||||
|
normalized_left = _normalize_question(left)
|
||||||
|
normalized_right = _normalize_question(right)
|
||||||
|
if normalized_left == normalized_right:
|
||||||
|
return 1.0
|
||||||
|
left_tokens = set(normalized_left.split())
|
||||||
|
right_tokens = set(normalized_right.split())
|
||||||
|
if min(len(left_tokens), len(right_tokens)) < 4:
|
||||||
|
return 0.0
|
||||||
|
overlap = len(left_tokens & right_tokens) / len(left_tokens | right_tokens)
|
||||||
|
sequence = SequenceMatcher(None, normalized_left, normalized_right).ratio()
|
||||||
|
return (0.7 * overlap) + (0.3 * sequence)
|
||||||
875
old_2026-08-31/src/lai_automation/worker.py
Normal file
875
old_2026-08-31/src/lai_automation/worker.py
Normal file
@@ -0,0 +1,875 @@
|
|||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import logging
|
||||||
|
import re
|
||||||
|
import signal
|
||||||
|
import threading
|
||||||
|
import time
|
||||||
|
from collections.abc import Iterator
|
||||||
|
from contextlib import contextmanager
|
||||||
|
from dataclasses import asdict
|
||||||
|
|
||||||
|
import pymysql
|
||||||
|
|
||||||
|
from .agent import AgentProviderError, AgentTimeoutError, AiderAgent
|
||||||
|
from .config import Settings
|
||||||
|
from .database import FailureRepository
|
||||||
|
from .git import ChangeSummary, GitWorkspace
|
||||||
|
from .gitea import GiteaClient
|
||||||
|
from .models import FailureReport
|
||||||
|
from .policy import (
|
||||||
|
evaluate_changes,
|
||||||
|
evaluate_expected_behavior,
|
||||||
|
evaluate_removal_prevention,
|
||||||
|
evaluate_removal_scope,
|
||||||
|
evaluate_report_scope,
|
||||||
|
evaluate_requested_behavior_scope,
|
||||||
|
evaluate_routed_files,
|
||||||
|
)
|
||||||
|
from .prompt import build_retry_task, build_task, redact, repair_files
|
||||||
|
from .removal import detect_already_fixed
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
_TEST_APP_KEY = "base64:MDEyMzQ1Njc4OWFiY2RlZjAxMjM0NTY3ODlhYmNkZWY="
|
||||||
|
_OPERATIONAL_ERROR_PATTERNS = (
|
||||||
|
re.compile(r"access denied for user", re.IGNORECASE),
|
||||||
|
re.compile(r"authentication failed", re.IGNORECASE),
|
||||||
|
re.compile(r"(?:can't|cannot|could not) connect to (?:mysql|mariadb|database)", re.IGNORECASE),
|
||||||
|
re.compile(r"connection refused", re.IGNORECASE),
|
||||||
|
re.compile(r"network is unreachable", re.IGNORECASE),
|
||||||
|
re.compile(r"could not resolve host", re.IGNORECASE),
|
||||||
|
re.compile(r"no address associated with hostname", re.IGNORECASE),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class ValidationSetupError(RuntimeError):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class AutomationWorker:
|
||||||
|
def __init__(self, settings: Settings) -> None:
|
||||||
|
self.settings = settings
|
||||||
|
self.reports = FailureRepository(settings)
|
||||||
|
self.agent = AiderAgent(settings)
|
||||||
|
self.gitea = GiteaClient(settings)
|
||||||
|
self.stop_event = threading.Event()
|
||||||
|
|
||||||
|
def run_once(self) -> bool:
|
||||||
|
report = self.reports.claim_next()
|
||||||
|
if not report:
|
||||||
|
LOG.info("No open failure reports")
|
||||||
|
return False
|
||||||
|
|
||||||
|
LOG.info(
|
||||||
|
"Claimed failure report %s category=%s source=%s database_type=%s",
|
||||||
|
report.id,
|
||||||
|
report.category,
|
||||||
|
report.source,
|
||||||
|
report.context.get("database_type", "unknown"),
|
||||||
|
)
|
||||||
|
if self.settings.dry_run:
|
||||||
|
task = build_task(report)
|
||||||
|
LOG.info("Dry-run task for report %s:\n%s", report.id, task)
|
||||||
|
self.reports.release(report.id, "Task generated successfully; no agent executed")
|
||||||
|
return True
|
||||||
|
|
||||||
|
operational_reason = self._operational_failure_reason(report)
|
||||||
|
if operational_reason:
|
||||||
|
LOG.warning("Failure report %s skipped: %s", report.id, operational_reason)
|
||||||
|
self.reports.fail(report.id, operational_reason, {
|
||||||
|
"classification": "operational",
|
||||||
|
"automation_action": "no_source_change",
|
||||||
|
})
|
||||||
|
return True
|
||||||
|
|
||||||
|
workspace = GitWorkspace(self.settings, report.id)
|
||||||
|
metadata: dict = {"mode": self.settings.mode, "model": self.settings.agent_model}
|
||||||
|
try:
|
||||||
|
with self._progress(report.id, "workspace clone"):
|
||||||
|
workspace.prepare()
|
||||||
|
files = repair_files(report)
|
||||||
|
already_fixed = detect_already_fixed(report, workspace.path, files)
|
||||||
|
if already_fixed:
|
||||||
|
metadata.update({
|
||||||
|
"classification": "already_fixed",
|
||||||
|
"automation_action": "no_source_change",
|
||||||
|
"matched_reference_file": already_fixed.filename,
|
||||||
|
"matched_reference_question": already_fixed.question,
|
||||||
|
"match_score": already_fixed.match_score,
|
||||||
|
"forbidden_fragments": list(already_fixed.forbidden_fragments),
|
||||||
|
})
|
||||||
|
self.reports.complete(report.id, metadata)
|
||||||
|
LOG.info(
|
||||||
|
"Resolved report %s as already fixed file=%s match_score=%.2f fragments=%s",
|
||||||
|
report.id,
|
||||||
|
already_fixed.filename,
|
||||||
|
already_fixed.match_score,
|
||||||
|
",".join(already_fixed.forbidden_fragments),
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
with self._progress(report.id, "workspace dependency setup"):
|
||||||
|
metadata["workspace_setup"] = self._prepare_workspace_for_agent(
|
||||||
|
workspace,
|
||||||
|
report.id,
|
||||||
|
)
|
||||||
|
LOG.info("Report %s branch=%s workspace ready", report.id, workspace.branch)
|
||||||
|
task = build_task(report)
|
||||||
|
if files:
|
||||||
|
LOG.info("Report %s routed editable files=%s", report.id, ",".join(files))
|
||||||
|
metadata["agent_attempts"] = []
|
||||||
|
valid_checkpoint: str | None = None
|
||||||
|
policy_constraints: list[str] = []
|
||||||
|
policy_failure_counts: dict[tuple[str, ...], int] = {}
|
||||||
|
rejected_candidate_counts: dict[tuple[tuple[str, ...], str], int] = {}
|
||||||
|
validation_failure_counts: dict[str, int] = {}
|
||||||
|
first_validation_checkpoints: dict[str, str] = {}
|
||||||
|
attempt_files = files
|
||||||
|
for attempt in range(1, self.settings.max_agent_attempts + 1):
|
||||||
|
LOG.info(
|
||||||
|
"Report %s agent attempt %s/%s starting model=%s strategy=%s files=%s",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
self.settings.max_agent_attempts,
|
||||||
|
self.settings.agent_model,
|
||||||
|
self._agent_strategy(attempt_files),
|
||||||
|
",".join(attempt_files) or "repository-map",
|
||||||
|
)
|
||||||
|
attempt_started = time.monotonic()
|
||||||
|
attempt_metadata = {
|
||||||
|
"attempt": attempt,
|
||||||
|
"model": self.settings.agent_model,
|
||||||
|
"strategy": self._agent_strategy(attempt_files),
|
||||||
|
"editable_files": list(attempt_files),
|
||||||
|
"status": "running",
|
||||||
|
}
|
||||||
|
metadata["agent_attempts"].append(attempt_metadata)
|
||||||
|
with self._progress(report.id, f"agent attempt {attempt}"):
|
||||||
|
try:
|
||||||
|
agent_output = self.agent.run(
|
||||||
|
workspace.path,
|
||||||
|
self._with_editable_scope(task, attempt_files),
|
||||||
|
attempt_files,
|
||||||
|
log_context=f"Report {report.id} attempt {attempt}",
|
||||||
|
)
|
||||||
|
attempt_metadata["status"] = "completed"
|
||||||
|
except AgentTimeoutError as exception:
|
||||||
|
attempt_metadata["status"] = "timeout"
|
||||||
|
partial_summary = workspace.summary()
|
||||||
|
if (
|
||||||
|
not isinstance(partial_summary, ChangeSummary)
|
||||||
|
or not partial_summary.files
|
||||||
|
):
|
||||||
|
raise
|
||||||
|
LOG.warning(
|
||||||
|
"Report %s attempt %s timed out with partial changes; validating them",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
)
|
||||||
|
agent_output = str(exception)
|
||||||
|
except AgentProviderError as exception:
|
||||||
|
attempt_metadata["status"] = "provider_error"
|
||||||
|
partial_summary = workspace.summary()
|
||||||
|
if (
|
||||||
|
not isinstance(partial_summary, ChangeSummary)
|
||||||
|
or not partial_summary.files
|
||||||
|
):
|
||||||
|
raise
|
||||||
|
LOG.warning(
|
||||||
|
"Report %s attempt %s provider failed with partial changes; "
|
||||||
|
"validating them",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
)
|
||||||
|
agent_output = str(exception)
|
||||||
|
except Exception:
|
||||||
|
attempt_metadata["status"] = "failed"
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
attempt_metadata["elapsed_seconds"] = round(
|
||||||
|
time.monotonic() - attempt_started,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
agent_details = getattr(self.agent, "last_run_metadata", None)
|
||||||
|
if isinstance(agent_details, dict):
|
||||||
|
attempt_metadata.update(agent_details)
|
||||||
|
attempt_metadata["output_tail"] = agent_output[-4_000:]
|
||||||
|
summary = workspace.summary()
|
||||||
|
metadata["changes"] = asdict(summary)
|
||||||
|
LOG.info(
|
||||||
|
"Report %s attempt %s changed files=%s lines=+%s/-%s paths=%s",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
len(summary.files),
|
||||||
|
summary.added_lines,
|
||||||
|
summary.deleted_lines,
|
||||||
|
",".join(summary.files) or "none",
|
||||||
|
)
|
||||||
|
candidate_checkpoint: str | None = None
|
||||||
|
if summary.files:
|
||||||
|
try:
|
||||||
|
self._lint_changed_php(workspace, summary, report.id, attempt)
|
||||||
|
except RuntimeError as exception:
|
||||||
|
if attempt >= self.settings.max_agent_attempts:
|
||||||
|
raise
|
||||||
|
LOG.warning(
|
||||||
|
"PHP syntax failed for report %s on attempt %s; discarding malformed attempt",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
)
|
||||||
|
discarded_diff = workspace.diff()
|
||||||
|
workspace.restore_checkpoint(valid_checkpoint)
|
||||||
|
restored_state = (
|
||||||
|
"The last syntax-valid changes were restored. Correct them without "
|
||||||
|
"recreating the malformed edit."
|
||||||
|
if valid_checkpoint
|
||||||
|
else "The malformed attempt was discarded. Start again from the clean repository."
|
||||||
|
)
|
||||||
|
syntax_guidance = ""
|
||||||
|
if "cannot redeclare" in str(exception).casefold():
|
||||||
|
syntax_guidance = (
|
||||||
|
" PHP reported a duplicate declaration. Search the entire owning "
|
||||||
|
"class for the named method, keep exactly one implementation and "
|
||||||
|
"one test-subclass wrapper, and edit those existing declarations. "
|
||||||
|
"Never append another method with the same name."
|
||||||
|
)
|
||||||
|
task = build_retry_task(
|
||||||
|
report,
|
||||||
|
RuntimeError(
|
||||||
|
f"{exception}\n{restored_state}{syntax_guidance} "
|
||||||
|
"Make smaller edits and run php -l on "
|
||||||
|
"each changed PHP file. Reconstruct the intended application fix from "
|
||||||
|
"this discarded patch excerpt, correcting its syntax. This is a "
|
||||||
|
"repair-only attempt: preserve the intended behavior, place methods "
|
||||||
|
"inside their owning class, and do not redesign or expand the solution:\n"
|
||||||
|
f"{discarded_diff}"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
policy = evaluate_changes(summary, self.settings, workspace.path)
|
||||||
|
scope_policy = evaluate_report_scope(report, summary)
|
||||||
|
if policy.allowed and not scope_policy.allowed:
|
||||||
|
policy = scope_policy
|
||||||
|
routed_policy = evaluate_routed_files(summary, files)
|
||||||
|
if policy.allowed and not routed_policy.allowed:
|
||||||
|
policy = routed_policy
|
||||||
|
added_text = workspace.added_text()
|
||||||
|
requested_behavior_policy = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
summary,
|
||||||
|
added_text,
|
||||||
|
)
|
||||||
|
if policy.allowed and not requested_behavior_policy.allowed:
|
||||||
|
policy = requested_behavior_policy
|
||||||
|
expected_policy = evaluate_expected_behavior(report, added_text)
|
||||||
|
if policy.allowed and not expected_policy.allowed:
|
||||||
|
policy = expected_policy
|
||||||
|
prevention_policy = evaluate_removal_prevention(report, summary, workspace)
|
||||||
|
if policy.allowed and not prevention_policy.allowed:
|
||||||
|
policy = prevention_policy
|
||||||
|
removal_policy = evaluate_removal_scope(report, summary, workspace)
|
||||||
|
if policy.allowed and not removal_policy.allowed:
|
||||||
|
policy = removal_policy
|
||||||
|
if not policy.allowed:
|
||||||
|
policy_constraints.extend(
|
||||||
|
error for error in policy.errors if error not in policy_constraints
|
||||||
|
)
|
||||||
|
signature = tuple(sorted(policy.errors))
|
||||||
|
policy_failure_counts[signature] = policy_failure_counts.get(signature, 0) + 1
|
||||||
|
candidate_fingerprint = hashlib.sha256(
|
||||||
|
str(workspace.diff()).encode("utf-8")
|
||||||
|
).hexdigest()[:12]
|
||||||
|
candidate_key = (signature, candidate_fingerprint)
|
||||||
|
rejected_candidate_counts[candidate_key] = (
|
||||||
|
rejected_candidate_counts.get(candidate_key, 0) + 1
|
||||||
|
)
|
||||||
|
attempt_metadata["validation_status"] = "policy_rejected"
|
||||||
|
attempt_metadata["policy_errors"] = list(policy.errors)
|
||||||
|
attempt_metadata["candidate_fingerprint"] = candidate_fingerprint
|
||||||
|
LOG.info(
|
||||||
|
"Report %s attempt %s policy signature occurrence=%s "
|
||||||
|
"candidate=%s identical_candidate_occurrence=%s",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
policy_failure_counts[signature],
|
||||||
|
candidate_fingerprint,
|
||||||
|
rejected_candidate_counts[candidate_key],
|
||||||
|
)
|
||||||
|
if rejected_candidate_counts[candidate_key] >= 2:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Repeated identical candidate rejection without semantic progress: "
|
||||||
|
+ "; ".join(policy.errors)
|
||||||
|
)
|
||||||
|
retryable = (
|
||||||
|
not summary.files
|
||||||
|
or all(filename.startswith("tests/") for filename in summary.files)
|
||||||
|
or not scope_policy.allowed
|
||||||
|
or not routed_policy.allowed
|
||||||
|
or not requested_behavior_policy.allowed
|
||||||
|
or not expected_policy.allowed
|
||||||
|
or not prevention_policy.allowed
|
||||||
|
or not removal_policy.allowed
|
||||||
|
)
|
||||||
|
if retryable and attempt < self.settings.max_agent_attempts:
|
||||||
|
LOG.warning(
|
||||||
|
"Policy rejected retryable changes for report %s on attempt %s: %s",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
"; ".join(policy.errors),
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
not scope_policy.allowed
|
||||||
|
or not routed_policy.allowed
|
||||||
|
or not requested_behavior_policy.allowed
|
||||||
|
or not prevention_policy.allowed
|
||||||
|
or not removal_policy.allowed
|
||||||
|
):
|
||||||
|
workspace.restore_checkpoint(None)
|
||||||
|
valid_checkpoint = None
|
||||||
|
attempt_files = files
|
||||||
|
elif not expected_policy.allowed:
|
||||||
|
workspace.restore_checkpoint(valid_checkpoint)
|
||||||
|
attempt_files = files
|
||||||
|
else:
|
||||||
|
valid_checkpoint = candidate_checkpoint
|
||||||
|
task = build_retry_task(
|
||||||
|
report,
|
||||||
|
RuntimeError(
|
||||||
|
"Policy rejected changes: " + "; ".join(policy.errors)
|
||||||
|
+ ". Modify the owning production code and add a focused regression test."
|
||||||
|
)
|
||||||
|
)
|
||||||
|
task += (
|
||||||
|
"\n\nAll policy constraints from earlier attempts remain mandatory:\n- "
|
||||||
|
+ "\n- ".join(policy_constraints)
|
||||||
|
+ "\nThe rejected files were reset unless explicitly described as a valid checkpoint. "
|
||||||
|
"Reapply a complete production fix and regression test; do not submit only one file."
|
||||||
|
)
|
||||||
|
continue
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected changes: " + "; ".join(policy.errors)
|
||||||
|
)
|
||||||
|
if any(not filename.startswith("tests/") for filename in summary.files):
|
||||||
|
candidate_checkpoint = workspace.checkpoint()
|
||||||
|
valid_checkpoint = candidate_checkpoint
|
||||||
|
try:
|
||||||
|
metadata["tests"] = self._run_tests(workspace, report.id, attempt)
|
||||||
|
attempt_metadata["validation_status"] = "passed"
|
||||||
|
LOG.info("Report %s attempt %s validation passed", report.id, attempt)
|
||||||
|
break
|
||||||
|
except ValidationSetupError:
|
||||||
|
raise
|
||||||
|
except RuntimeError as exception:
|
||||||
|
diagnostic = redact(str(exception))
|
||||||
|
validation_signature = self._validation_signature(diagnostic)
|
||||||
|
validation_failure_counts[validation_signature] = (
|
||||||
|
validation_failure_counts.get(validation_signature, 0) + 1
|
||||||
|
)
|
||||||
|
validation_occurrence = validation_failure_counts[validation_signature]
|
||||||
|
if candidate_checkpoint and validation_signature not in first_validation_checkpoints:
|
||||||
|
first_validation_checkpoints[validation_signature] = candidate_checkpoint
|
||||||
|
attempt_metadata["validation_status"] = "failed"
|
||||||
|
attempt_metadata["validation_error_tail"] = diagnostic[-4_000:]
|
||||||
|
attempt_metadata["validation_signature"] = validation_signature
|
||||||
|
attempt_metadata["validation_occurrence"] = validation_occurrence
|
||||||
|
if attempt >= self.settings.max_agent_attempts:
|
||||||
|
raise
|
||||||
|
LOG.warning(
|
||||||
|
"Tests failed for report %s on attempt %s; requesting correction. "
|
||||||
|
"Diagnostic tail:\n%s",
|
||||||
|
report.id,
|
||||||
|
attempt,
|
||||||
|
diagnostic[-2_000:],
|
||||||
|
)
|
||||||
|
task = build_retry_task(report, exception)
|
||||||
|
if policy_constraints:
|
||||||
|
task += (
|
||||||
|
"\n\nAll policy constraints from earlier attempts remain mandatory:\n- "
|
||||||
|
+ "\n- ".join(policy_constraints)
|
||||||
|
)
|
||||||
|
attempt_files = files
|
||||||
|
if validation_occurrence >= 3:
|
||||||
|
first_checkpoint = first_validation_checkpoints.get(validation_signature)
|
||||||
|
if first_checkpoint:
|
||||||
|
workspace.restore_checkpoint(first_checkpoint)
|
||||||
|
valid_checkpoint = first_checkpoint
|
||||||
|
task += (
|
||||||
|
"\n\nThe same executable failure survived multiple edits. Ineffective "
|
||||||
|
"later retries were discarded and the first policy-valid candidate was "
|
||||||
|
"restored. Re-read the complete failing test and owning implementation. "
|
||||||
|
"Correct a contradictory test expectation or the actual production call "
|
||||||
|
"path; do not add another helper, wrapper, fallback, or duplicate method."
|
||||||
|
)
|
||||||
|
elif validation_occurrence >= 2:
|
||||||
|
task += (
|
||||||
|
"\n\nThe same executable failure has repeated. The previous edit did not "
|
||||||
|
"address its root cause. Re-read the complete failing test and production "
|
||||||
|
"call path before editing; do not add duplicate methods or assertions."
|
||||||
|
)
|
||||||
|
task += (
|
||||||
|
"\n\nThe current routed candidate passed static policy but failed executable "
|
||||||
|
"validation. Keep the valid checkpoint, inspect every routed production and "
|
||||||
|
"test file, and fix the root cause shown by the failure. Do not merely rewrite "
|
||||||
|
"assertions to match broken production output. All routed files remain available, "
|
||||||
|
"but change only those needed for the correction."
|
||||||
|
)
|
||||||
|
|
||||||
|
summary = workspace.summary()
|
||||||
|
metadata["changes"] = asdict(summary)
|
||||||
|
policy = evaluate_changes(summary, self.settings, workspace.path)
|
||||||
|
if not policy.allowed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected final changes: " + "; ".join(policy.errors)
|
||||||
|
)
|
||||||
|
added_text = workspace.added_text()
|
||||||
|
requested_behavior_policy = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
summary,
|
||||||
|
added_text,
|
||||||
|
)
|
||||||
|
if not requested_behavior_policy.allowed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected final changes: "
|
||||||
|
+ "; ".join(requested_behavior_policy.errors)
|
||||||
|
)
|
||||||
|
expected_policy = evaluate_expected_behavior(report, added_text)
|
||||||
|
if not expected_policy.allowed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected final changes: " + "; ".join(expected_policy.errors)
|
||||||
|
)
|
||||||
|
prevention_policy = evaluate_removal_prevention(report, summary, workspace)
|
||||||
|
if not prevention_policy.allowed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected final changes: " + "; ".join(prevention_policy.errors)
|
||||||
|
)
|
||||||
|
removal_policy = evaluate_removal_scope(report, summary, workspace)
|
||||||
|
if not removal_policy.allowed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected final changes: " + "; ".join(removal_policy.errors)
|
||||||
|
)
|
||||||
|
routed_policy = evaluate_routed_files(summary, files)
|
||||||
|
if not routed_policy.allowed:
|
||||||
|
raise RuntimeError(
|
||||||
|
"Policy rejected final changes: " + "; ".join(routed_policy.errors)
|
||||||
|
)
|
||||||
|
commit = workspace.commit(report.id)
|
||||||
|
metadata["commit"] = commit
|
||||||
|
LOG.info("Report %s committed %s", report.id, commit)
|
||||||
|
|
||||||
|
if self.settings.mode == "pr":
|
||||||
|
with self._progress(report.id, "branch publication"):
|
||||||
|
workspace.push_branch()
|
||||||
|
metadata["pull_request"] = self.gitea.create_pull_request(
|
||||||
|
workspace.branch,
|
||||||
|
f"Fix failure report #{report.id}",
|
||||||
|
self._pull_request_body(report, summary, metadata["tests"]),
|
||||||
|
)
|
||||||
|
LOG.info("Report %s pull request=%s", report.id, metadata["pull_request"])
|
||||||
|
elif self.settings.mode == "direct-main":
|
||||||
|
workspace.push_main()
|
||||||
|
metadata["direct_main"] = True
|
||||||
|
|
||||||
|
self.reports.complete(report.id, metadata)
|
||||||
|
LOG.info("Completed failure report %s", report.id)
|
||||||
|
return True
|
||||||
|
except AgentProviderError as exception:
|
||||||
|
metadata.update({
|
||||||
|
"classification": "operational",
|
||||||
|
"automation_action": "no_source_change",
|
||||||
|
})
|
||||||
|
LOG.exception("Failure report %s model provider failed", report.id)
|
||||||
|
self.reports.fail(report.id, str(exception), metadata)
|
||||||
|
return True
|
||||||
|
except Exception as exception:
|
||||||
|
LOG.exception("Failure report %s failed", report.id)
|
||||||
|
self.reports.fail(report.id, str(exception), metadata)
|
||||||
|
return True
|
||||||
|
finally:
|
||||||
|
try:
|
||||||
|
workspace.cleanup()
|
||||||
|
LOG.info("Report %s workspace cleaned", report.id)
|
||||||
|
except OSError:
|
||||||
|
LOG.exception("Could not clean workspace for report %s", report.id)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _with_editable_scope(task: str, files: tuple[str, ...]) -> str:
|
||||||
|
if not files:
|
||||||
|
return task
|
||||||
|
return (
|
||||||
|
task
|
||||||
|
+ "\n\nStrict editable scope for this repair; use these exact repository-relative paths:\n- "
|
||||||
|
+ "\n- ".join(files)
|
||||||
|
+ "\nDo not add, edit, rename, or create any other path. Do not shorten these paths "
|
||||||
|
"to basenames. The orchestrator owns all Git operations; do not run git commands."
|
||||||
|
)
|
||||||
|
|
||||||
|
def _agent_strategy(self, files: tuple[str, ...]) -> str:
|
||||||
|
if self.settings.agent_model.startswith("ollama") and any(
|
||||||
|
not filename.startswith("tests/") for filename in files
|
||||||
|
):
|
||||||
|
return "whole"
|
||||||
|
return "diff"
|
||||||
|
|
||||||
|
def _lint_changed_php(
|
||||||
|
self,
|
||||||
|
workspace: GitWorkspace,
|
||||||
|
summary: ChangeSummary,
|
||||||
|
report_id: int,
|
||||||
|
attempt: int,
|
||||||
|
) -> None:
|
||||||
|
for filename in summary.files:
|
||||||
|
if not filename.endswith(".php"):
|
||||||
|
continue
|
||||||
|
result = workspace.run_args(
|
||||||
|
["php", "-l", filename],
|
||||||
|
check=False,
|
||||||
|
username="coder",
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
output = (result.stdout + "\n" + result.stderr)[-4_000:]
|
||||||
|
raise RuntimeError(f"PHP syntax check failed for {filename}:\n{output}")
|
||||||
|
LOG.info("Report %s attempt %s PHP syntax preflight passed", report_id, attempt)
|
||||||
|
|
||||||
|
def run_forever(self) -> None:
|
||||||
|
LOG.info(
|
||||||
|
"Worker started mode=%s model=%s poll_seconds=%s max_attempts=%s tests=%s direct_main=%s",
|
||||||
|
self.settings.mode,
|
||||||
|
self.settings.agent_model,
|
||||||
|
self.settings.poll_seconds,
|
||||||
|
self.settings.max_agent_attempts,
|
||||||
|
len(self.settings.test_commands),
|
||||||
|
self.settings.allow_direct_main,
|
||||||
|
)
|
||||||
|
signal.signal(signal.SIGTERM, self._stop)
|
||||||
|
signal.signal(signal.SIGINT, self._stop)
|
||||||
|
while not self.stop_event.is_set():
|
||||||
|
self.run_once()
|
||||||
|
self.stop_event.wait(self.settings.poll_seconds)
|
||||||
|
|
||||||
|
def _stop(self, signum: int, _frame) -> None:
|
||||||
|
LOG.info("Received signal %s; stopping after the current report", signum)
|
||||||
|
self.stop_event.set()
|
||||||
|
|
||||||
|
def _run_tests(
|
||||||
|
self,
|
||||||
|
workspace: GitWorkspace,
|
||||||
|
report_id: int | None = None,
|
||||||
|
attempt: int | None = None,
|
||||||
|
) -> list[dict]:
|
||||||
|
self._write_safe_test_env(workspace)
|
||||||
|
self._prepare_test_schema()
|
||||||
|
test_environment = self._test_environment()
|
||||||
|
results = self._run_commands(
|
||||||
|
workspace,
|
||||||
|
self.settings.setup_commands,
|
||||||
|
test_environment,
|
||||||
|
report_id,
|
||||||
|
attempt,
|
||||||
|
"validation setup",
|
||||||
|
"Validation setup command failed",
|
||||||
|
username="coder",
|
||||||
|
exception_type=ValidationSetupError,
|
||||||
|
)
|
||||||
|
results.extend(self._run_commands(
|
||||||
|
workspace,
|
||||||
|
self.settings.test_commands,
|
||||||
|
test_environment,
|
||||||
|
report_id,
|
||||||
|
attempt,
|
||||||
|
"validation command",
|
||||||
|
"Test command failed",
|
||||||
|
username="coder",
|
||||||
|
))
|
||||||
|
return results
|
||||||
|
|
||||||
|
def _prepare_workspace_for_agent(
|
||||||
|
self,
|
||||||
|
workspace: GitWorkspace,
|
||||||
|
report_id: int,
|
||||||
|
) -> list[dict]:
|
||||||
|
self._write_safe_test_env(workspace)
|
||||||
|
self._prepare_test_schema()
|
||||||
|
return self._run_commands(
|
||||||
|
workspace,
|
||||||
|
self.settings.setup_commands,
|
||||||
|
self._test_environment(),
|
||||||
|
report_id,
|
||||||
|
None,
|
||||||
|
"workspace setup",
|
||||||
|
"Workspace setup command failed",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _test_environment(self) -> dict[str, str]:
|
||||||
|
return {
|
||||||
|
"AGENT_TEST_DB_ENABLED": "true",
|
||||||
|
"APP_ENV": "testing",
|
||||||
|
"APP_KEY": _TEST_APP_KEY,
|
||||||
|
"APP_DEBUG": "false",
|
||||||
|
"CACHE_STORE": "array",
|
||||||
|
"SESSION_DRIVER": "array",
|
||||||
|
"QUEUE_CONNECTION": "sync",
|
||||||
|
"AI_MISTRAL_API_KEY": "automation-test-key",
|
||||||
|
"TEST_DB_HOST": self.settings.test_database_host,
|
||||||
|
"TEST_DB_PORT": str(self.settings.test_database_port),
|
||||||
|
"TEST_DB_NAME": self.settings.test_database_name,
|
||||||
|
"TEST_DB_USER": self.settings.test_database_user,
|
||||||
|
"TEST_DB_PASSWORD": self.settings.test_database_password,
|
||||||
|
"TEST_CUSTOMER_DB": self.settings.test_customer_database,
|
||||||
|
"TEST_CONFIG_DB": self.settings.test_config_database,
|
||||||
|
"DB_CONNECTION": "mysql",
|
||||||
|
"DB_HOST": self.settings.test_database_host,
|
||||||
|
"DB_PORT": str(self.settings.test_database_port),
|
||||||
|
"DB_DATABASE": self.settings.test_database_name,
|
||||||
|
"DB_USERNAME": self.settings.test_database_user,
|
||||||
|
"DB_PASSWORD": self.settings.test_database_password,
|
||||||
|
"ERP_DB_HOST": self.settings.test_database_host,
|
||||||
|
"ERP_DB_PORT": str(self.settings.test_database_port),
|
||||||
|
"ERP_DATABASE": self.settings.test_customer_database,
|
||||||
|
"ERP_DB_USERNAME": self.settings.test_database_user,
|
||||||
|
"ERP_DB_PASSWORD": self.settings.test_database_password,
|
||||||
|
"COCKPIT_CONFIG_DATABASE": self.settings.test_config_database,
|
||||||
|
}
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _write_safe_test_env(workspace: GitWorkspace) -> None:
|
||||||
|
env_file = workspace.path / ".env"
|
||||||
|
env_file.write_text(
|
||||||
|
"APP_ENV=testing\n"
|
||||||
|
f"APP_KEY={_TEST_APP_KEY}\n"
|
||||||
|
"APP_DEBUG=false\n"
|
||||||
|
"CACHE_STORE=array\n"
|
||||||
|
"SESSION_DRIVER=array\n"
|
||||||
|
"QUEUE_CONNECTION=sync\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
env_file.chmod(0o600)
|
||||||
|
|
||||||
|
def _run_commands(
|
||||||
|
self,
|
||||||
|
workspace: GitWorkspace,
|
||||||
|
commands: tuple[str, ...],
|
||||||
|
environment: dict[str, str],
|
||||||
|
report_id: int | None,
|
||||||
|
attempt: int | None,
|
||||||
|
stage: str,
|
||||||
|
failure_prefix: str,
|
||||||
|
username: str | None = None,
|
||||||
|
exception_type: type[RuntimeError] = RuntimeError,
|
||||||
|
) -> list[dict]:
|
||||||
|
results = []
|
||||||
|
for command in commands:
|
||||||
|
label = f"{stage}: {command}"
|
||||||
|
LOG.info("Report %s attempt %s %s starting", report_id, attempt, label)
|
||||||
|
with self._progress(report_id, label):
|
||||||
|
result = workspace.run(
|
||||||
|
command,
|
||||||
|
check=False,
|
||||||
|
username=username,
|
||||||
|
extra_environment=environment,
|
||||||
|
)
|
||||||
|
results.append({
|
||||||
|
"stage": stage,
|
||||||
|
"command": command,
|
||||||
|
"returncode": result.returncode,
|
||||||
|
"stdout": result.stdout[-6000:],
|
||||||
|
"stderr": result.stderr[-6000:],
|
||||||
|
})
|
||||||
|
LOG.info(
|
||||||
|
"Report %s attempt %s %s finished returncode=%s",
|
||||||
|
report_id,
|
||||||
|
attempt,
|
||||||
|
label,
|
||||||
|
result.returncode,
|
||||||
|
)
|
||||||
|
if result.returncode != 0:
|
||||||
|
output = result.stdout + "\n" + result.stderr
|
||||||
|
if len(output) > 12_000:
|
||||||
|
output = output[:6_000] + "\n...[middle omitted]...\n" + output[-6_000:]
|
||||||
|
raise exception_type(f"{failure_prefix}: {command}\n{output}")
|
||||||
|
return results
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _operational_failure_reason(report: FailureReport) -> str | None:
|
||||||
|
if report.category != "processing":
|
||||||
|
return None
|
||||||
|
error = report.error_message or ""
|
||||||
|
if any(pattern.search(error) for pattern in _OPERATIONAL_ERROR_PATTERNS):
|
||||||
|
return (
|
||||||
|
"Operational database or network failure; no source-code repair was attempted. "
|
||||||
|
"Verify service availability and runtime credentials, then retry the user request."
|
||||||
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _validation_signature(diagnostic: str) -> str:
|
||||||
|
decisive_lines = [
|
||||||
|
line.strip().casefold()
|
||||||
|
for line in diagnostic.splitlines()
|
||||||
|
if re.search(
|
||||||
|
r"\b(?:failed|failure|fatal error|expectation failed|expected:|"
|
||||||
|
r"cannot redeclare|called 0 times|call to protected method)\b",
|
||||||
|
line,
|
||||||
|
re.IGNORECASE,
|
||||||
|
)
|
||||||
|
]
|
||||||
|
normalized = "\n".join(decisive_lines) or diagnostic.casefold()
|
||||||
|
normalized = re.sub(r"\b\d+(?:\.\d+)?(?:ms|s)?\b", "#", normalized)
|
||||||
|
normalized = re.sub(r"\s+", " ", normalized).strip()
|
||||||
|
return hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:12]
|
||||||
|
|
||||||
|
def _prepare_test_schema(self) -> None:
|
||||||
|
database = self.settings.test_customer_database
|
||||||
|
if not re.fullmatch(r"[A-Za-z0-9_]+", database):
|
||||||
|
raise ValueError("TEST_CUSTOMER_DB contains invalid characters")
|
||||||
|
connection = pymysql.connect(
|
||||||
|
host=self.settings.test_database_host,
|
||||||
|
port=self.settings.test_database_port,
|
||||||
|
user=self.settings.test_database_user,
|
||||||
|
password=self.settings.test_database_password,
|
||||||
|
database=database,
|
||||||
|
autocommit=True,
|
||||||
|
connect_timeout=10,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
with connection.cursor() as cursor:
|
||||||
|
cursor.execute(
|
||||||
|
"""
|
||||||
|
CREATE TABLE IF NOT EXISTS FK_Nummerkreis (
|
||||||
|
Kennung INT PRIMARY KEY,
|
||||||
|
Bezeichnung VARCHAR(100) NOT NULL,
|
||||||
|
Bezeichnung_Kurz VARCHAR(20) NOT NULL
|
||||||
|
)
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
cursor.executemany(
|
||||||
|
"""
|
||||||
|
INSERT INTO FK_Nummerkreis (Kennung, Bezeichnung, Bezeichnung_Kurz)
|
||||||
|
VALUES (%s, %s, %s)
|
||||||
|
ON DUPLICATE KEY UPDATE
|
||||||
|
Bezeichnung = VALUES(Bezeichnung),
|
||||||
|
Bezeichnung_Kurz = VALUES(Bezeichnung_Kurz)
|
||||||
|
""",
|
||||||
|
(
|
||||||
|
(3, "Rechnung", "RG"),
|
||||||
|
(4, "Lieferschein", "LS"),
|
||||||
|
(8, "Gutschrift", "GS"),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
@contextmanager
|
||||||
|
def _progress(
|
||||||
|
self,
|
||||||
|
report_id: int | None,
|
||||||
|
stage: str,
|
||||||
|
interval: float = 30.0,
|
||||||
|
) -> Iterator[None]:
|
||||||
|
started = time.monotonic()
|
||||||
|
stopped = threading.Event()
|
||||||
|
|
||||||
|
def heartbeat() -> None:
|
||||||
|
while not stopped.wait(interval):
|
||||||
|
LOG.info(
|
||||||
|
"Report %s %s still running elapsed=%ss",
|
||||||
|
report_id,
|
||||||
|
stage,
|
||||||
|
round(time.monotonic() - started),
|
||||||
|
)
|
||||||
|
if report_id is not None and not self.settings.dry_run:
|
||||||
|
try:
|
||||||
|
self.reports.heartbeat(report_id)
|
||||||
|
except Exception:
|
||||||
|
LOG.warning(
|
||||||
|
"Report %s %s database heartbeat failed",
|
||||||
|
report_id,
|
||||||
|
stage,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
thread = threading.Thread(target=heartbeat, daemon=True)
|
||||||
|
thread.start()
|
||||||
|
try:
|
||||||
|
yield
|
||||||
|
finally:
|
||||||
|
stopped.set()
|
||||||
|
thread.join(timeout=0.1)
|
||||||
|
LOG.info(
|
||||||
|
"Report %s %s finished elapsed=%.1fs",
|
||||||
|
report_id,
|
||||||
|
stage,
|
||||||
|
time.monotonic() - started,
|
||||||
|
)
|
||||||
|
if report_id is not None and not self.settings.dry_run:
|
||||||
|
try:
|
||||||
|
self.reports.heartbeat(report_id)
|
||||||
|
except Exception:
|
||||||
|
LOG.warning(
|
||||||
|
"Report %s %s final database heartbeat failed",
|
||||||
|
report_id,
|
||||||
|
stage,
|
||||||
|
exc_info=True,
|
||||||
|
)
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _pull_request_body(report: FailureReport, summary, tests: list[dict]) -> str:
|
||||||
|
checks = "\n".join(
|
||||||
|
f"- {'PASS' if item['returncode'] == 0 else 'FAIL'} `{item['command']}`"
|
||||||
|
for item in tests
|
||||||
|
)
|
||||||
|
files = "\n".join(f"- `{filename}`" for filename in summary.files)
|
||||||
|
return f"""Automated fix for internal failure report #{report.id}.
|
||||||
|
|
||||||
|
## Original failure report
|
||||||
|
|
||||||
|
**Source**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.source)}
|
||||||
|
|
||||||
|
**Category**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.category)}
|
||||||
|
|
||||||
|
**Feedback type**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.feedback_type)}
|
||||||
|
|
||||||
|
**User prompt**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.prompt)}
|
||||||
|
|
||||||
|
**Observed response**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.response)}
|
||||||
|
|
||||||
|
**User feedback**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.user_feedback)}
|
||||||
|
|
||||||
|
**Expected behavior**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.expected_behavior)}
|
||||||
|
|
||||||
|
**Error message**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.error_message)}
|
||||||
|
|
||||||
|
**Generated SQL**
|
||||||
|
{AutomationWorker._quote_pull_request_value(report.sql_query)}
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
{files}
|
||||||
|
|
||||||
|
## Validation
|
||||||
|
{checks}
|
||||||
|
|
||||||
|
## Risk controls
|
||||||
|
- No production database access was provided to the coding agent.
|
||||||
|
- Secret and deployment paths are blocked by policy.
|
||||||
|
- Human review is required before merge.
|
||||||
|
"""
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _quote_pull_request_value(value: str | None, limit: int = 4_000) -> str:
|
||||||
|
raw_value = value or ""
|
||||||
|
safe_value = redact(raw_value).strip()
|
||||||
|
was_truncated = len(raw_value) > limit
|
||||||
|
safe_value = safe_value[:limit].rstrip()
|
||||||
|
if not safe_value:
|
||||||
|
return "> _Not provided._"
|
||||||
|
if was_truncated:
|
||||||
|
safe_value += "\n\n_[truncated]_"
|
||||||
|
return "\n".join(f"> {line}" if line else ">" for line in safe_value.splitlines())
|
||||||
201
old_2026-08-31/tests/test_agent.py
Normal file
201
old_2026-08-31/tests/test_agent.py
Normal file
@@ -0,0 +1,201 @@
|
|||||||
|
import io
|
||||||
|
import signal
|
||||||
|
import subprocess
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from test_config import settings
|
||||||
|
|
||||||
|
from lai_automation.agent import AgentProviderError, AgentTimeoutError, AiderAgent
|
||||||
|
|
||||||
|
|
||||||
|
def test_aider_command_cannot_mutate_git_configuration_or_send_analytics() -> None:
|
||||||
|
command = AiderAgent(settings()).command(Path("/repo/.git/task.txt"))
|
||||||
|
|
||||||
|
assert "--no-gitignore" in command
|
||||||
|
assert "--no-analytics" in command
|
||||||
|
assert "--no-auto-commits" in command
|
||||||
|
assert "--no-git" in command
|
||||||
|
assert "--stream" in command
|
||||||
|
assert "--no-pretty" in command
|
||||||
|
assert "--no-check-update" in command
|
||||||
|
assert "--no-suggest-shell-commands" in command
|
||||||
|
assert "--no-detect-urls" in command
|
||||||
|
assert "--disable-playwright" in command
|
||||||
|
assert "--no-auto-lint" in command
|
||||||
|
assert command[command.index("--map-tokens") + 1] == "2048"
|
||||||
|
assert command.count("/dev/null") == 2
|
||||||
|
assert "--edit-format" not in command
|
||||||
|
assert command[-2:] == ["--timeout", "900"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_aider_command_adds_existing_routed_files(tmp_path) -> None:
|
||||||
|
git_dir = tmp_path / ".git"
|
||||||
|
git_dir.mkdir()
|
||||||
|
controller = tmp_path / "app/Http/Controllers/ChatController.php"
|
||||||
|
controller.parent.mkdir(parents=True)
|
||||||
|
controller.write_text("<?php\n")
|
||||||
|
|
||||||
|
command = AiderAgent(settings()).command(
|
||||||
|
git_dir / "task.txt",
|
||||||
|
("app/Http/Controllers/ChatController.php", "missing.php"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert ["--file", "app/Http/Controllers/ChatController.php"] == command[
|
||||||
|
command.index("--file"):command.index("--file") + 2
|
||||||
|
]
|
||||||
|
assert "missing.php" not in command
|
||||||
|
assert command[command.index("--map-tokens") + 1] == "0"
|
||||||
|
assert command[command.index("--edit-format") + 1] == "diff"
|
||||||
|
assert "--architect" not in command
|
||||||
|
assert "--editor-model" not in command
|
||||||
|
|
||||||
|
|
||||||
|
def test_unrouted_aider_command_stays_in_direct_edit_mode() -> None:
|
||||||
|
command = AiderAgent(settings()).command(Path("/repo/.git/task.txt"))
|
||||||
|
|
||||||
|
assert "--architect" not in command
|
||||||
|
|
||||||
|
|
||||||
|
def test_aider_detects_output_stall_at_configured_limit() -> None:
|
||||||
|
agent = AiderAgent(settings(agent_idle_timeout_seconds=120))
|
||||||
|
|
||||||
|
assert not agent._output_stalled(119.9)
|
||||||
|
assert agent._output_stalled(120)
|
||||||
|
|
||||||
|
|
||||||
|
def test_ollama_aider_command_uses_single_whole_file_pass(tmp_path) -> None:
|
||||||
|
git_dir = tmp_path / ".git"
|
||||||
|
git_dir.mkdir()
|
||||||
|
source = tmp_path / "config/erp_semantics.php"
|
||||||
|
source.parent.mkdir(parents=True)
|
||||||
|
source.write_text("<?php\n")
|
||||||
|
|
||||||
|
command = AiderAgent(settings(agent_model="ollama_chat/qwen3-coder:30b")).command(
|
||||||
|
git_dir / "task.txt",
|
||||||
|
("config/erp_semantics.php",),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert command[command.index("--edit-format") + 1] == "whole"
|
||||||
|
assert "--no-auto-lint" in command
|
||||||
|
assert "--architect" not in command
|
||||||
|
assert "--editor-model" not in command
|
||||||
|
|
||||||
|
|
||||||
|
def test_ollama_test_only_correction_uses_precise_diff_format(tmp_path) -> None:
|
||||||
|
git_dir = tmp_path / ".git"
|
||||||
|
git_dir.mkdir()
|
||||||
|
test_file = tmp_path / "tests/Unit/FixTest.php"
|
||||||
|
test_file.parent.mkdir(parents=True)
|
||||||
|
test_file.write_text("<?php\n")
|
||||||
|
|
||||||
|
command = AiderAgent(settings(agent_model="ollama_chat/qwen3-coder:30b")).command(
|
||||||
|
git_dir / "task.txt",
|
||||||
|
("tests/Unit/FixTest.php",),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert command[command.index("--edit-format") + 1] == "diff"
|
||||||
|
assert "--no-auto-lint" in command
|
||||||
|
|
||||||
|
|
||||||
|
def test_aider_timeout_has_actionable_error(tmp_path, monkeypatch) -> None:
|
||||||
|
workspace = tmp_path / "repo"
|
||||||
|
(workspace / ".git").mkdir(parents=True)
|
||||||
|
|
||||||
|
class TimedOutProcess:
|
||||||
|
pid = 4321
|
||||||
|
stdout = io.StringIO("partial output\n")
|
||||||
|
stderr = io.StringIO("")
|
||||||
|
returncode = None
|
||||||
|
|
||||||
|
def wait(self, timeout=None):
|
||||||
|
raise subprocess.TimeoutExpired("aider", timeout)
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"lai_automation.agent.subprocess.Popen",
|
||||||
|
lambda *args, **kwargs: TimedOutProcess(),
|
||||||
|
)
|
||||||
|
signals = []
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"lai_automation.agent.os.killpg",
|
||||||
|
lambda pid, sent_signal: signals.append((pid, sent_signal)),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
|
||||||
|
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
|
||||||
|
)())
|
||||||
|
|
||||||
|
with pytest.raises(AgentTimeoutError, match="partial output"):
|
||||||
|
AiderAgent(settings(agent_timeout_seconds=60)).run(workspace, "fix it")
|
||||||
|
|
||||||
|
assert signals == [(4321, signal.SIGTERM), (4321, signal.SIGKILL)]
|
||||||
|
|
||||||
|
|
||||||
|
def test_aider_environment_does_not_include_database_credentials(tmp_path, monkeypatch) -> None:
|
||||||
|
config = settings(workspace_root=tmp_path)
|
||||||
|
workspace = tmp_path / "repo"
|
||||||
|
(workspace / ".git").mkdir(parents=True)
|
||||||
|
monkeypatch.setenv("TEST_DB_PASSWORD", "must-not-reach-aider")
|
||||||
|
monkeypatch.setenv("OPENROUTER_API_KEY", "qwen-provider-key")
|
||||||
|
monkeypatch.setenv("OLLAMA_API_BASE", "http://host.docker.internal:11434")
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
class SuccessfulProcess:
|
||||||
|
pid = 1234
|
||||||
|
stdout = io.StringIO("provider said qwen-provider-key\n")
|
||||||
|
stderr = io.StringIO("")
|
||||||
|
returncode = 0
|
||||||
|
|
||||||
|
def wait(self, timeout=None):
|
||||||
|
return self.returncode
|
||||||
|
|
||||||
|
def popen(command, **kwargs):
|
||||||
|
captured.update(kwargs["env"])
|
||||||
|
return SuccessfulProcess()
|
||||||
|
|
||||||
|
monkeypatch.setattr("lai_automation.agent.subprocess.Popen", popen)
|
||||||
|
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
|
||||||
|
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
|
||||||
|
)())
|
||||||
|
|
||||||
|
output = AiderAgent(config).run(workspace, "fix it")
|
||||||
|
|
||||||
|
assert "TEST_DB_PASSWORD" not in captured
|
||||||
|
assert "DB_PASSWORD" not in captured
|
||||||
|
assert captured["OPENROUTER_API_KEY"] == "qwen-provider-key"
|
||||||
|
assert captured["OLLAMA_API_BASE"] == "http://host.docker.internal:11434"
|
||||||
|
assert "qwen-provider-key" not in output
|
||||||
|
assert "[REDACTED]" in output
|
||||||
|
|
||||||
|
|
||||||
|
def test_successful_exit_with_only_provider_failure_is_rejected(tmp_path, monkeypatch) -> None:
|
||||||
|
workspace = tmp_path / "repo"
|
||||||
|
(workspace / ".git").mkdir(parents=True)
|
||||||
|
|
||||||
|
class ProviderFailedProcess:
|
||||||
|
pid = 1235
|
||||||
|
stdout = io.StringIO(
|
||||||
|
"litellm.InternalServerError: MistralException - Name or service not known\n"
|
||||||
|
"The API provider's servers are down or overloaded.\n"
|
||||||
|
)
|
||||||
|
stderr = io.StringIO("")
|
||||||
|
returncode = 0
|
||||||
|
|
||||||
|
def wait(self, timeout=None):
|
||||||
|
return self.returncode
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"lai_automation.agent.subprocess.Popen",
|
||||||
|
lambda *args, **kwargs: ProviderFailedProcess(),
|
||||||
|
)
|
||||||
|
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
|
||||||
|
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
|
||||||
|
)())
|
||||||
|
|
||||||
|
agent = AiderAgent(settings())
|
||||||
|
with pytest.raises(AgentProviderError, match="provider failed"):
|
||||||
|
agent.run(workspace, "fix it")
|
||||||
|
|
||||||
|
assert agent.last_run_metadata["status"] == "provider_error"
|
||||||
110
old_2026-08-31/tests/test_config.py
Normal file
110
old_2026-08-31/tests/test_config.py
Normal file
@@ -0,0 +1,110 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from lai_automation.config import Settings
|
||||||
|
|
||||||
|
|
||||||
|
def settings(**overrides) -> Settings:
|
||||||
|
values = {
|
||||||
|
"database_host": "db",
|
||||||
|
"database_port": 3306,
|
||||||
|
"database_name": "lai",
|
||||||
|
"database_user": "worker",
|
||||||
|
"database_password": "secret",
|
||||||
|
"source_repository": "https://example.test/lai.git",
|
||||||
|
"source_branch": "main",
|
||||||
|
"workspace_root": Path("/tmp/workspaces"),
|
||||||
|
"mode": "dry-run",
|
||||||
|
"allow_direct_main": False,
|
||||||
|
"gitea_url": "https://example.test",
|
||||||
|
"gitea_owner": "owner",
|
||||||
|
"gitea_repository": "lai",
|
||||||
|
"gitea_token": "",
|
||||||
|
"gitea_username": "bot",
|
||||||
|
"agent_command": "aider",
|
||||||
|
"agent_model": "mistral/test",
|
||||||
|
"agent_timeout_seconds": 900,
|
||||||
|
"agent_idle_timeout_seconds": 300,
|
||||||
|
"setup_commands": (),
|
||||||
|
"test_commands": ("php artisan test",),
|
||||||
|
"test_database_host": "test-db",
|
||||||
|
"test_database_port": 3306,
|
||||||
|
"test_database_name": "lai_agent_test",
|
||||||
|
"test_database_user": "test_runner",
|
||||||
|
"test_database_password": "test-secret",
|
||||||
|
"test_customer_database": "customer_agent_test",
|
||||||
|
"test_config_database": "cockpit_agent_test",
|
||||||
|
"poll_seconds": 60,
|
||||||
|
"max_changed_files": 12,
|
||||||
|
"max_changed_lines": 800,
|
||||||
|
"max_agent_attempts": 2,
|
||||||
|
"stale_claim_minutes": 90,
|
||||||
|
"dry_run": True,
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return Settings(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def test_direct_main_requires_explicit_kill_switch() -> None:
|
||||||
|
config = settings(mode="direct-main", dry_run=False, gitea_token="token")
|
||||||
|
with pytest.raises(ValueError, match="ALLOW_DIRECT_MAIN"):
|
||||||
|
config.validate()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pr_requires_gitea_token() -> None:
|
||||||
|
config = settings(mode="pr", dry_run=False)
|
||||||
|
with pytest.raises(ValueError, match="GITEA_TOKEN"):
|
||||||
|
config.validate()
|
||||||
|
|
||||||
|
|
||||||
|
def test_default_commands_prepare_workspace_and_run_tests(monkeypatch) -> None:
|
||||||
|
monkeypatch.setenv("FAILURE_DB_PASSWORD", "failure-secret")
|
||||||
|
monkeypatch.setenv("TEST_DB_PASSWORD", "test-secret")
|
||||||
|
monkeypatch.delenv("SETUP_COMMANDS", raising=False)
|
||||||
|
monkeypatch.delenv("TEST_COMMANDS", raising=False)
|
||||||
|
|
||||||
|
config = Settings.from_env()
|
||||||
|
|
||||||
|
assert "npm ci" in config.setup_commands
|
||||||
|
assert "npm run build" in config.setup_commands
|
||||||
|
assert "php artisan migrate:fresh --force" in config.setup_commands
|
||||||
|
assert config.test_commands == ("php artisan test",)
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_attempts_are_bounded() -> None:
|
||||||
|
with pytest.raises(ValueError, match="MAX_AGENT_ATTEMPTS"):
|
||||||
|
settings(max_agent_attempts=8).validate()
|
||||||
|
|
||||||
|
settings(max_agent_attempts=7).validate()
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_timeout_is_bounded() -> None:
|
||||||
|
with pytest.raises(ValueError, match="AGENT_TIMEOUT_SECONDS"):
|
||||||
|
settings(agent_timeout_seconds=59).validate()
|
||||||
|
|
||||||
|
settings(agent_timeout_seconds=60, agent_idle_timeout_seconds=60).validate()
|
||||||
|
|
||||||
|
|
||||||
|
def test_agent_idle_timeout_is_bounded_by_process_timeout() -> None:
|
||||||
|
with pytest.raises(ValueError, match="AGENT_IDLE_TIMEOUT_SECONDS"):
|
||||||
|
settings(agent_idle_timeout_seconds=59).validate()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="AGENT_IDLE_TIMEOUT_SECONDS"):
|
||||||
|
settings(agent_timeout_seconds=120, agent_idle_timeout_seconds=121).validate()
|
||||||
|
|
||||||
|
settings(agent_timeout_seconds=120, agent_idle_timeout_seconds=60).validate()
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_databases_must_be_distinct_and_explicitly_test_only() -> None:
|
||||||
|
with pytest.raises(ValueError, match="isolated test database"):
|
||||||
|
settings(test_customer_database="customer_production").validate()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="must be distinct"):
|
||||||
|
settings(test_config_database="customer_agent_test").validate()
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="FAILURE_DB_NAME"):
|
||||||
|
settings(
|
||||||
|
database_name="lai_agent_test",
|
||||||
|
test_database_name="lai_agent_test",
|
||||||
|
).validate()
|
||||||
35
old_2026-08-31/tests/test_database.py
Normal file
35
old_2026-08-31/tests/test_database.py
Normal file
@@ -0,0 +1,35 @@
|
|||||||
|
from unittest.mock import MagicMock, Mock
|
||||||
|
|
||||||
|
from test_config import settings
|
||||||
|
|
||||||
|
from lai_automation.database import FailureRepository
|
||||||
|
|
||||||
|
|
||||||
|
def test_heartbeat_refreshes_only_an_active_claim() -> None:
|
||||||
|
repository = FailureRepository(settings())
|
||||||
|
connection = MagicMock()
|
||||||
|
cursor = MagicMock()
|
||||||
|
repository.connection = MagicMock()
|
||||||
|
repository.connection.return_value.__enter__.return_value = connection
|
||||||
|
connection.cursor.return_value.__enter__.return_value = cursor
|
||||||
|
|
||||||
|
repository.heartbeat(17)
|
||||||
|
|
||||||
|
statement, parameters = cursor.execute.call_args.args
|
||||||
|
assert "SET updated_at = NOW()" in statement
|
||||||
|
assert "status = 'agent_running'" in statement
|
||||||
|
assert parameters == (17,)
|
||||||
|
connection.commit.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_complete_clears_previous_agent_error() -> None:
|
||||||
|
repository = FailureRepository(settings())
|
||||||
|
repository._finish = Mock()
|
||||||
|
|
||||||
|
repository.complete(17, {"classification": "already_fixed"})
|
||||||
|
|
||||||
|
repository._finish.assert_called_once_with(
|
||||||
|
17,
|
||||||
|
"resolved",
|
||||||
|
{"classification": "already_fixed", "agent_error": None},
|
||||||
|
)
|
||||||
239
old_2026-08-31/tests/test_git.py
Normal file
239
old_2026-08-31/tests/test_git.py
Normal file
@@ -0,0 +1,239 @@
|
|||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
from test_config import settings
|
||||||
|
|
||||||
|
from lai_automation.git import GitWorkspace
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepare_uses_temporary_authentication_for_private_clone(tmp_path, monkeypatch) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path, gitea_token="token"), 10)
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
calls.append((command, kwargs))
|
||||||
|
if command[1] == "clone":
|
||||||
|
askpass = Path(kwargs["env"]["GIT_ASKPASS"])
|
||||||
|
assert askpass.is_file()
|
||||||
|
assert Path(kwargs["env"]["GIT_BOT_USER_FILE"]).read_text() == "bot"
|
||||||
|
assert Path(kwargs["env"]["GIT_BOT_TOKEN_FILE"]).read_text() == "token"
|
||||||
|
workspace.path.mkdir()
|
||||||
|
return subprocess.CompletedProcess(command, 0, "", "")
|
||||||
|
|
||||||
|
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
|
||||||
|
|
||||||
|
workspace.prepare()
|
||||||
|
|
||||||
|
clone_command, clone_options = calls[0]
|
||||||
|
assert clone_command[:2] == ["git", "clone"]
|
||||||
|
assert "token" not in " ".join(clone_command)
|
||||||
|
assert clone_options["cwd"] == tmp_path
|
||||||
|
assert not tuple(tmp_path.glob(".lai-git-auth-*"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_prepare_surfaces_clone_stderr_without_credentials(tmp_path, monkeypatch) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path, gitea_token="token"), 11)
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
raise subprocess.CalledProcessError(
|
||||||
|
128,
|
||||||
|
command,
|
||||||
|
output="",
|
||||||
|
stderr="fatal: upstream returned 403",
|
||||||
|
)
|
||||||
|
|
||||||
|
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
|
||||||
|
|
||||||
|
try:
|
||||||
|
workspace.prepare()
|
||||||
|
except RuntimeError as exception:
|
||||||
|
message = str(exception)
|
||||||
|
else:
|
||||||
|
raise AssertionError("Expected clone failure")
|
||||||
|
|
||||||
|
assert "Source repository clone failed" in message
|
||||||
|
assert "upstream returned 403" in message
|
||||||
|
assert "token" not in message
|
||||||
|
assert not tuple(tmp_path.glob(".lai-git-auth-*"))
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_command_does_not_interpret_shell_metacharacters(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 1)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
|
||||||
|
result = workspace.run("printf safe ; touch injected", check=False)
|
||||||
|
|
||||||
|
assert result.stdout == "safe"
|
||||||
|
assert not (workspace.path / "injected").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_extra_environment_is_passed_without_privilege_drop(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 6)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
|
||||||
|
result = workspace.run_args(
|
||||||
|
[sys.executable, "-c", "import os; print(os.environ['SAFE_TEST_VALUE'])"],
|
||||||
|
extra_environment={"SAFE_TEST_VALUE": "available"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.stdout.strip() == "available"
|
||||||
|
|
||||||
|
|
||||||
|
def test_coder_commands_use_native_process_identity(tmp_path, monkeypatch) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 9)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
captured = {}
|
||||||
|
|
||||||
|
def run(command, **kwargs):
|
||||||
|
captured.update(kwargs)
|
||||||
|
return subprocess.CompletedProcess(command, 0, "ok", "")
|
||||||
|
|
||||||
|
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"lai_automation.git.pwd.getpwnam",
|
||||||
|
lambda _: Mock(pw_dir="/home/coder", pw_uid=10001, pw_gid=10001),
|
||||||
|
)
|
||||||
|
|
||||||
|
workspace.run_args(["php", "-v"], username="coder")
|
||||||
|
|
||||||
|
assert captured["user"] == 10001
|
||||||
|
assert captured["group"] == 10001
|
||||||
|
assert captured["extra_groups"] == ()
|
||||||
|
assert "preexec_fn" not in captured
|
||||||
|
|
||||||
|
|
||||||
|
def test_summary_includes_untracked_files(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 2)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
(workspace.path / "new.php").write_text("<?php\n")
|
||||||
|
|
||||||
|
summary = workspace.summary()
|
||||||
|
|
||||||
|
assert summary.files == ("new.php",)
|
||||||
|
assert summary.added_lines == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_summary_includes_checkpoint_staged_files(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 7)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
tracked = workspace.path / "config.php"
|
||||||
|
tracked.write_text("<?php\nreturn [];\n")
|
||||||
|
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test",
|
||||||
|
"commit", "-m", "base"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
tracked.write_text("<?php\nreturn ['fixed'];\n")
|
||||||
|
|
||||||
|
workspace.checkpoint()
|
||||||
|
summary = workspace.summary()
|
||||||
|
|
||||||
|
assert summary.files == ("config.php",)
|
||||||
|
assert summary.added_lines == 1
|
||||||
|
assert summary.deleted_lines == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_discard_changes_restores_clean_disposable_workspace(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 3)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "user.email", "test@example.invalid"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "user.name", "Test"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
tracked = workspace.path / "tracked.php"
|
||||||
|
tracked.write_text("<?php\nreturn [];\n")
|
||||||
|
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||||
|
subprocess.run(["git", "commit", "-m", "base"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
tracked.write_text("<?php\ninvalid\n")
|
||||||
|
untracked = workspace.path / "extra.php"
|
||||||
|
untracked.write_text("<?php\n")
|
||||||
|
|
||||||
|
workspace.discard_changes()
|
||||||
|
|
||||||
|
assert tracked.read_text() == "<?php\nreturn [];\n"
|
||||||
|
assert not untracked.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_checkpoint_restores_last_valid_changes(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 4)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "user.email", "test@example.invalid"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "config", "user.name", "Test"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
)
|
||||||
|
tracked = workspace.path / "config.php"
|
||||||
|
tracked.write_text("<?php\nreturn [];\n")
|
||||||
|
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||||
|
subprocess.run(["git", "commit", "-m", "base"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
tracked.write_text("<?php\nreturn ['valid'];\n")
|
||||||
|
added = workspace.path / "test.php"
|
||||||
|
added.write_text("<?php\n")
|
||||||
|
checkpoint = workspace.checkpoint()
|
||||||
|
tracked.write_text("<?php\nreturn [broken;\n")
|
||||||
|
|
||||||
|
workspace.restore_checkpoint(checkpoint)
|
||||||
|
|
||||||
|
assert tracked.read_text() == "<?php\nreturn ['valid'];\n"
|
||||||
|
assert added.exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_diff_returns_discarded_patch_excerpt(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 5)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
tracked = workspace.path / "service.php"
|
||||||
|
tracked.write_text("<?php\nreturn 'old';\n")
|
||||||
|
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test", "commit", "-m", "base"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
tracked.write_text("<?php\nreturn 'new';\n")
|
||||||
|
|
||||||
|
patch = workspace.diff()
|
||||||
|
|
||||||
|
assert "-return 'old';" in patch
|
||||||
|
assert "+return 'new';" in patch
|
||||||
|
|
||||||
|
|
||||||
|
def test_head_file_reads_original_tracked_content(tmp_path) -> None:
|
||||||
|
workspace = GitWorkspace(settings(workspace_root=tmp_path), 8)
|
||||||
|
workspace.path.mkdir()
|
||||||
|
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||||
|
tracked = workspace.path / "config.php"
|
||||||
|
tracked.write_text("<?php\nreturn 'original';\n")
|
||||||
|
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||||
|
subprocess.run(
|
||||||
|
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test",
|
||||||
|
"commit", "-m", "base"],
|
||||||
|
cwd=workspace.path,
|
||||||
|
check=True,
|
||||||
|
capture_output=True,
|
||||||
|
)
|
||||||
|
tracked.write_text("<?php\nreturn 'candidate';\n")
|
||||||
|
|
||||||
|
assert workspace.head_file("config.php") == "<?php\nreturn 'original';\n"
|
||||||
|
assert workspace.head_file("missing.php") is None
|
||||||
27
old_2026-08-31/tests/test_gitea.py
Normal file
27
old_2026-08-31/tests/test_gitea.py
Normal file
@@ -0,0 +1,27 @@
|
|||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from test_config import settings
|
||||||
|
|
||||||
|
from lai_automation.gitea import GiteaClient
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_pull_request_requires_url() -> None:
|
||||||
|
client = GiteaClient(settings(gitea_token="token"))
|
||||||
|
response = Mock()
|
||||||
|
response.json.return_value = {"number": 12}
|
||||||
|
client.session.post = Mock(return_value=response)
|
||||||
|
|
||||||
|
with pytest.raises(ValueError, match="pull request URL"):
|
||||||
|
client.create_pull_request("ai/failure-1", "Fix", "Body")
|
||||||
|
|
||||||
|
|
||||||
|
def test_create_pull_request_returns_html_url() -> None:
|
||||||
|
client = GiteaClient(settings(gitea_token="token"))
|
||||||
|
response = Mock()
|
||||||
|
response.json.return_value = {"html_url": "https://example.test/owner/lai/pulls/12"}
|
||||||
|
client.session.post = Mock(return_value=response)
|
||||||
|
|
||||||
|
result = client.create_pull_request("ai/failure-1", "Fix", "Body")
|
||||||
|
|
||||||
|
assert result.endswith("/pulls/12")
|
||||||
924
old_2026-08-31/tests/test_policy.py
Normal file
924
old_2026-08-31/tests/test_policy.py
Normal file
@@ -0,0 +1,924 @@
|
|||||||
|
from dataclasses import replace
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
from test_config import settings
|
||||||
|
|
||||||
|
from lai_automation.git import ChangeSummary
|
||||||
|
from lai_automation.models import FailureReport
|
||||||
|
from lai_automation.policy import (
|
||||||
|
evaluate_changes,
|
||||||
|
evaluate_expected_behavior,
|
||||||
|
evaluate_removal_prevention,
|
||||||
|
evaluate_removal_scope,
|
||||||
|
evaluate_report_scope,
|
||||||
|
evaluate_requested_behavior_scope,
|
||||||
|
evaluate_routed_files,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_accepts_small_application_change() -> None:
|
||||||
|
result = evaluate_changes(
|
||||||
|
ChangeSummary(("app/Services/Example.php", "tests/Feature/ExampleTest.php"), 40, 5),
|
||||||
|
settings(),
|
||||||
|
)
|
||||||
|
assert result.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_rejects_secrets_and_vendor_paths() -> None:
|
||||||
|
result = evaluate_changes(
|
||||||
|
ChangeSummary((".env", "vendor/foo.php", ".aider.tags.cache.v4/cache.db"), 3, 0),
|
||||||
|
settings(),
|
||||||
|
)
|
||||||
|
assert not result.allowed
|
||||||
|
assert len(result.errors) == 3
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_rejects_test_only_repair() -> None:
|
||||||
|
result = evaluate_changes(
|
||||||
|
ChangeSummary(("tests/Unit/ExampleTest.php",), 20, 2),
|
||||||
|
settings(),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert "production behavior was not fixed" in result.errors[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_direct_main_blocks_validator_and_migrations() -> None:
|
||||||
|
config = replace(
|
||||||
|
settings(),
|
||||||
|
mode="direct-main",
|
||||||
|
dry_run=False,
|
||||||
|
allow_direct_main=True,
|
||||||
|
gitea_token="token",
|
||||||
|
)
|
||||||
|
result = evaluate_changes(
|
||||||
|
ChangeSummary(("app/Services/SQLValidatorService.php", "database/migrations/new.php"), 10, 2),
|
||||||
|
config,
|
||||||
|
)
|
||||||
|
assert not result.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_rejects_symlinks(tmp_path) -> None:
|
||||||
|
(tmp_path / "link.php").symlink_to("/etc/passwd")
|
||||||
|
result = evaluate_changes(ChangeSummary(("link.php",), 1, 0), settings(), tmp_path)
|
||||||
|
assert not result.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_cross_database_report_requires_application_orchestration() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=56,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Use cockpit_config.time_registration",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Join cockpit_config manager_id and FK_AuftragPos",
|
||||||
|
prompt="Hours in April",
|
||||||
|
response="0 hours",
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
rejected = evaluate_report_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary(("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2),
|
||||||
|
)
|
||||||
|
accepted = evaluate_report_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary(("app/Http/Controllers/ChatController.php", "tests/Unit/FixTest.php"), 20, 2),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not rejected.allowed
|
||||||
|
assert "staged orchestration" in rejected.errors[0]
|
||||||
|
assert accepted.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_requires_explicit_technical_terms_in_added_code() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
"FK_Auftrag nach AuftragsKennung 2 filtern und System_created verwenden; "
|
||||||
|
"obsolet und storniert ausfiltern"
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
rejected = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"Use FK_Auftrag.Datum_erfassung and status storniert",
|
||||||
|
)
|
||||||
|
accepted = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"SELECT DATE_FORMAT(System_created, '%Y-%m') FROM FK_Auftrag "
|
||||||
|
"WHERE AuftragsKennung = 2 AND COALESCE(obsolet, 0) = 0 "
|
||||||
|
"AND COALESCE(bStatus_storniert, 0) = 0",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not rejected.allowed
|
||||||
|
assert "AuftragsKennung" in rejected.errors[0]
|
||||||
|
assert "System_created" in rejected.errors[0]
|
||||||
|
assert accepted.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_rejects_negated_required_date_column() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="System_created verwenden",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"Lieferscheine niemals mit System_created filtern",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert "contradicts expected identifier System_created" in result.errors[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_allows_rejecting_an_alternative_column_on_same_table() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="FK_Auftrag nach System_created filtern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"FK_Auftrag nach System_created filtern, nicht nach FK_Auftrag.Datum_erfassung",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_does_not_treat_later_never_as_table_negation() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="FK_Auftrag nach System_created filtern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"FK_Auftrag nach System_created filtern und Datum_erfassung niemals verwenden",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_rejects_unrequested_filter_copied_from_bad_sql() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="System_created verwenden und status storniert ausfiltern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query="WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"System_created AND bStatus_gebucht = 1 AND COALESCE(bStatus_storniert, 0) = 0",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert "copied bStatus_gebucht" in result.errors[-1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_requires_one_complete_reference_select() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
"FK_Auftrag nach Auftragskennung 2 filtern und System_created verwenden; "
|
||||||
|
"obsolet und status storniert ausfiltern"
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS' AND Datum_erfassung >= '2024-01-01'",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
split_rules_and_bad_sql = (
|
||||||
|
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
|
||||||
|
"COALESCE(bStatus_storniert, 0) = 0\n"
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE Bezeichnung_Kurz = 'LS' "
|
||||||
|
"AND a.System_created >= '2024-01-01' "
|
||||||
|
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
||||||
|
)
|
||||||
|
complete_sql = (
|
||||||
|
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
|
||||||
|
"COALESCE(bStatus_storniert, 0) = 0\n"
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||||
|
"AND a.System_created >= '2024-01-01' "
|
||||||
|
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
||||||
|
)
|
||||||
|
|
||||||
|
rejected = evaluate_expected_behavior(report, split_rules_and_bad_sql)
|
||||||
|
accepted = evaluate_expected_behavior(report, complete_sql)
|
||||||
|
|
||||||
|
assert not rejected.allowed
|
||||||
|
assert any("one reference SELECT" in error for error in rejected.errors)
|
||||||
|
assert any("LS lookup" in error for error in rejected.errors)
|
||||||
|
assert accepted.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_detects_lowercase_document_code_without_observed_sql() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
rejected = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"SELECT * FROM FK_Auftrag WHERE Bezeichnung_Kurz = 'LS'",
|
||||||
|
)
|
||||||
|
accepted = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not rejected.allowed
|
||||||
|
assert any("AuftragsKennung = 2" in error for error in rejected.errors)
|
||||||
|
assert accepted.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_rejects_lookup_join_for_authoritative_direct_code() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
|
||||||
|
result = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"SELECT * FROM FK_Auftrag a JOIN FK_Nummerkreis n ON n.Kennung = a.AuftragsKennung "
|
||||||
|
"WHERE a.AuftragsKennung = 2",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert any("joined FK_Nummerkreis" in error for error in result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_rejects_invented_obsolete_prefix_and_incomplete_assertions() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Auftragskennung 2, obsolet und status storniert ausfiltern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
invented = (
|
||||||
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
||||||
|
"AND COALESCE(bStatus_obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
|
||||||
|
"$this->assertStringContainsString('AuftragsKennung = 2', $context);"
|
||||||
|
)
|
||||||
|
complete = (
|
||||||
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
||||||
|
"AND COALESCE(obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
|
||||||
|
"$this->assertStringContainsString('AuftragsKennung = 2', $context);\n"
|
||||||
|
"$this->assertStringContainsString('COALESCE(obsolet, 0) = 0', $context);\n"
|
||||||
|
"$this->assertStringContainsString('COALESCE(bStatus_storniert, 0) = 0', $context);"
|
||||||
|
)
|
||||||
|
|
||||||
|
rejected = evaluate_expected_behavior(report, invented)
|
||||||
|
accepted = evaluate_expected_behavior(report, complete)
|
||||||
|
|
||||||
|
assert not rejected.allowed
|
||||||
|
assert any("obsolete rows" in error for error in rejected.errors)
|
||||||
|
assert any("Focused regression assertions" in error for error in rejected.errors)
|
||||||
|
assert accepted.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_routed_file_policy_rejects_unexpected_edits() -> None:
|
||||||
|
result = evaluate_routed_files(
|
||||||
|
ChangeSummary(("config/erp_semantics.php", "app/Services/AIService.php"), 10, 0),
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert "app/Services/AIService.php" in result.errors[0]
|
||||||
|
|
||||||
|
|
||||||
|
def test_requested_behavior_scope_requires_all_presentation_and_routing_owners() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
|
||||||
|
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
|
||||||
|
),
|
||||||
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
response="2024-01: 38",
|
||||||
|
sql_query="SELECT COUNT(*) FROM FK_Auftrag",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
incomplete = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 10, 0),
|
||||||
|
"cockpit: www: Markdown-Tabelle",
|
||||||
|
)
|
||||||
|
complete = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary((
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"app/Services/WebSearchService.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
"tests/Unit/WebSearchServiceTest.php",
|
||||||
|
), 30, 2),
|
||||||
|
(
|
||||||
|
"Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024\n"
|
||||||
|
"Markdown-Tabelle\ncount($rows) > 1\nsingle_row keeps sentence format\n"
|
||||||
|
"resolveRoutePrefix\n['database', 'Frage']\n['research', 'Frage']\n"
|
||||||
|
"['direct', 'Frage']\n"
|
||||||
|
"[null, 'Frage']\n[$explicitRoute, $message] = "
|
||||||
|
"$this->resolveRoutePrefix($message);\n$route = $explicitRoute ?? $fallback;\n"
|
||||||
|
"generateWebResearchResponse($message, 'ollama');\n"
|
||||||
|
"generateDirectResponse($message, 'ollama', []);\n"
|
||||||
|
"$controller->processMessage($cockpitRequest);\n"
|
||||||
|
"$ai->expects($this->never())->method('determineRoute');\n"
|
||||||
|
"function test_cockpit_prefix(): void {\n"
|
||||||
|
"$tenant->method('requiresCustomerDatabase')->willReturn(false);\n"
|
||||||
|
"$tenant->method('requiresConfigDatabase')->willReturn(false);\n"
|
||||||
|
"$tenant->method('requiresErpDatabase')->willReturn(false);\n}\n"
|
||||||
|
"$controller->processMessage($wwwRequest);\n"
|
||||||
|
"$ai->expects($this->never())->method('determineRoute');\n"
|
||||||
|
"$controller->processMessage($directRequest);\n"
|
||||||
|
"$ai->expects($this->never())->method('determineRoute');\n"
|
||||||
|
"$this->assertSame(3, substr_count($userPrompt, '\"Monat\"'));\n"
|
||||||
|
"$this->assertSame(3, substr_count($userPrompt, '\"Anzahl_Lieferscheine\"'));\n"
|
||||||
|
"cockpit:\nwww:\ndirect:"
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not incomplete.allowed
|
||||||
|
assert any("AIService.php" in error for error in incomplete.errors)
|
||||||
|
assert any("ChatController.php" in error for error in incomplete.errors)
|
||||||
|
assert not complete.errors
|
||||||
|
assert complete.allowed
|
||||||
|
|
||||||
|
|
||||||
|
def test_requested_behavior_scope_rejects_dead_prefix_source_tests() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Als Tabelle anzeigen",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
|
||||||
|
prompt="Anzahl pro Monat",
|
||||||
|
response="Text",
|
||||||
|
sql_query="SELECT 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
added = """
|
||||||
|
Markdown-Tabelle
|
||||||
|
count($rows) > 1
|
||||||
|
single_row keeps sentence format
|
||||||
|
resolveRoutePrefix
|
||||||
|
['database', 'Frage']
|
||||||
|
['direct', 'Frage']
|
||||||
|
[null, 'Frage']
|
||||||
|
[$explicitRoute, $message] = $this->resolveRoutePrefix($message);
|
||||||
|
$route = $explicitRoute ?? $fallback;
|
||||||
|
$controller->processMessage($cockpitRequest);
|
||||||
|
$ai->expects($this->never())->method('determineRoute');
|
||||||
|
$controller->processMessage($wwwRequest);
|
||||||
|
$ai->expects($this->never())->method('determineRoute');
|
||||||
|
$controller->resolveSource('cockpit: Frage', []);
|
||||||
|
return $this->resolveRoutePrefix($message);
|
||||||
|
public function determineRouteForMessage() { return $this->determineRoute($message, $this->defaultModel); }
|
||||||
|
www:
|
||||||
|
cockpit:
|
||||||
|
Anzahl pro Monat
|
||||||
|
"""
|
||||||
|
|
||||||
|
result = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary((
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
), 40, 2),
|
||||||
|
added,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert any("do not test resolveDatabaseSource" in error for error in result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_requested_behavior_scope_rejects_helper_only_prefix_tests() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Als Tabelle anzeigen",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
|
||||||
|
prompt="Anzahl pro Monat",
|
||||||
|
response="Text",
|
||||||
|
sql_query="SELECT 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
added = """
|
||||||
|
Anzahl pro Monat
|
||||||
|
Markdown-Tabelle
|
||||||
|
count($rows) > 1
|
||||||
|
single_row stays sentence formatted
|
||||||
|
resolveRoutePrefix
|
||||||
|
['database', 'Frage']
|
||||||
|
['direct', 'Frage']
|
||||||
|
[null, 'Frage']
|
||||||
|
[$explicitRoute, $message] = $this->resolveRoutePrefix($originalMessage);
|
||||||
|
$route = $explicitRoute ?? $fallback;
|
||||||
|
cockpit:
|
||||||
|
www:
|
||||||
|
"""
|
||||||
|
|
||||||
|
result = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary((
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
), 40, 2),
|
||||||
|
added,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert any("helper-only tests" in error for error in result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_requested_behavior_scope_rejects_incorrect_table_instruction_assertion() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Als Tabelle anzeigen",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="",
|
||||||
|
prompt="Anzahl pro Monat",
|
||||||
|
response="Text",
|
||||||
|
sql_query="SELECT 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
added = """
|
||||||
|
Anzahl pro Monat
|
||||||
|
Markdown-Tabelle
|
||||||
|
count($rows) > 1
|
||||||
|
single_row stays sentence formatted
|
||||||
|
$this->assertStringContainsString('alle bereitgestellten Zeilen', $systemPrompt);
|
||||||
|
"""
|
||||||
|
|
||||||
|
result = evaluate_requested_behavior_scope(
|
||||||
|
report,
|
||||||
|
ChangeSummary((
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
), 20, 2),
|
||||||
|
added,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert any("exact grammatical substring" in error for error in result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_expected_behavior_treats_removed_predicate_as_negative_requirement() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=5,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong delivery note count",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
|
||||||
|
"Einfach nur das aus der SQL entfernen"
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
accepted = (
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
|
||||||
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||||
|
)
|
||||||
|
still_present = (
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1\n"
|
||||||
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||||
|
)
|
||||||
|
|
||||||
|
assert evaluate_expected_behavior(report, accepted).allowed
|
||||||
|
rejected_sql = evaluate_expected_behavior(report, still_present)
|
||||||
|
rejected_test = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2",
|
||||||
|
)
|
||||||
|
near_match = evaluate_expected_behavior(
|
||||||
|
report,
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
|
||||||
|
"$this->assertStringNotContainsString('bStatus_gebucht = 1', $context);",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not rejected_sql.allowed
|
||||||
|
assert any("forbidden fragment" in error for error in rejected_sql.errors)
|
||||||
|
assert not rejected_test.allowed
|
||||||
|
assert any("negatively assert" in error for error in rejected_test.errors)
|
||||||
|
assert not near_match.allowed
|
||||||
|
assert any("near-match with a missing qualifier" in error for error in near_match.errors)
|
||||||
|
assert any(
|
||||||
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||||
|
in error
|
||||||
|
for error in near_match.errors
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_removal_prevention_rejects_vacuous_test_and_full_forbidden_rule(tmp_path) -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=6,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Die Abfrage stimmt nicht",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||||
|
"weil Lieferscheine nicht gebucht werden."
|
||||||
|
),
|
||||||
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
response="0 Ergebnisse",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
source_file = "config/erp_semantics.php"
|
||||||
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||||
|
reference_sql = (
|
||||||
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||||
|
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
||||||
|
)
|
||||||
|
|
||||||
|
def semantics(metric: str = "", sql: str = reference_sql) -> str:
|
||||||
|
return f"""<?php
|
||||||
|
return [
|
||||||
|
'metrics' => [
|
||||||
|
{metric}
|
||||||
|
],
|
||||||
|
'examples' => [[
|
||||||
|
'tables' => ['FK_Auftrag'],
|
||||||
|
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
||||||
|
'sql' => <<<'SQL'
|
||||||
|
{sql}
|
||||||
|
SQL,
|
||||||
|
]],
|
||||||
|
];
|
||||||
|
"""
|
||||||
|
|
||||||
|
original_source = semantics()
|
||||||
|
bad_metric = """ 'Lieferscheine' => [
|
||||||
|
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
|
||||||
|
'rules' => [
|
||||||
|
'Lieferscheine werden nicht gebucht, daher niemals a.bStatus_gebucht = 1 verwenden.',
|
||||||
|
],
|
||||||
|
],"""
|
||||||
|
original_test = """<?php
|
||||||
|
$context = $service->businessContext(['FK_Auftrag'], 'erp');
|
||||||
|
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
||||||
|
"""
|
||||||
|
bad_test = original_test + (
|
||||||
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
||||||
|
)
|
||||||
|
(tmp_path / "config").mkdir()
|
||||||
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||||
|
(tmp_path / source_file).write_text(semantics(bad_metric))
|
||||||
|
(tmp_path / test_file).write_text(bad_test)
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.head_file.side_effect = lambda filename: {
|
||||||
|
source_file: original_source,
|
||||||
|
test_file: original_test,
|
||||||
|
}.get(filename)
|
||||||
|
summary = ChangeSummary((source_file, test_file), 3, 0)
|
||||||
|
|
||||||
|
result = evaluate_removal_prevention(report, summary, workspace)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert any("must not embed the complete forbidden predicate" in error for error in result.errors)
|
||||||
|
assert any("context-builder test is vacuous" in error for error in result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_removal_prevention_rejects_unrelated_new_semantics(tmp_path) -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=6,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong delivery note count",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||||
|
"weil Lieferscheine nicht gebucht werden."
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
source_file = "config/erp_semantics.php"
|
||||||
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||||
|
original_source = """<?php
|
||||||
|
return ['examples' => [[
|
||||||
|
'question' => 'Delivery notes by month',
|
||||||
|
'sql' => <<<'SQL'
|
||||||
|
SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
|
||||||
|
SQL,
|
||||||
|
]]];
|
||||||
|
"""
|
||||||
|
current_source = original_source.replace(
|
||||||
|
"return [",
|
||||||
|
"""return [
|
||||||
|
'metrics' => ['Lieferscheine' => [
|
||||||
|
'tables' => ['FK_Auftrag'],
|
||||||
|
'rules' => [
|
||||||
|
'Lieferscheine werden nicht gebucht; bStatus_gebucht niemals verwenden.',
|
||||||
|
'Verwende Datum_erfassung oder System_created als Datumsspalte.',
|
||||||
|
],
|
||||||
|
]],""",
|
||||||
|
)
|
||||||
|
original_test = "<?php\n"
|
||||||
|
current_test = original_test + (
|
||||||
|
"$this->assertStringContainsString('Lieferscheine werden nicht gebucht; "
|
||||||
|
"bStatus_gebucht niemals verwenden.', $context);\n"
|
||||||
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
||||||
|
)
|
||||||
|
(tmp_path / "config").mkdir()
|
||||||
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||||
|
(tmp_path / source_file).write_text(current_source)
|
||||||
|
(tmp_path / test_file).write_text(current_test)
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.head_file.side_effect = lambda filename: {
|
||||||
|
source_file: original_source,
|
||||||
|
test_file: original_test,
|
||||||
|
}.get(filename)
|
||||||
|
|
||||||
|
result = evaluate_removal_prevention(
|
||||||
|
report,
|
||||||
|
ChangeSummary((source_file, test_file), 7, 0),
|
||||||
|
workspace,
|
||||||
|
)
|
||||||
|
|
||||||
|
assert not result.allowed
|
||||||
|
assert any("unrelated business guidance" in error for error in result.errors)
|
||||||
|
assert any("Datum_erfassung" in error for error in result.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_removal_prevention_accepts_exercised_rule_and_protects_reference(tmp_path) -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=6,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Die Abfrage stimmt nicht",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||||
|
"weil Lieferscheine nicht gebucht werden."
|
||||||
|
),
|
||||||
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
response="0 Ergebnisse",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
source_file = "config/erp_semantics.php"
|
||||||
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||||
|
reference_sql = (
|
||||||
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||||
|
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
||||||
|
)
|
||||||
|
|
||||||
|
def semantics(metric: str = "", sql: str = reference_sql) -> str:
|
||||||
|
return f"""<?php
|
||||||
|
return [
|
||||||
|
'metrics' => [
|
||||||
|
{metric}
|
||||||
|
],
|
||||||
|
'examples' => [[
|
||||||
|
'tables' => ['FK_Auftrag'],
|
||||||
|
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
||||||
|
'sql' => <<<'SQL'
|
||||||
|
{sql}
|
||||||
|
SQL,
|
||||||
|
]],
|
||||||
|
];
|
||||||
|
"""
|
||||||
|
|
||||||
|
rule = (
|
||||||
|
"Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine "
|
||||||
|
"niemals verwenden."
|
||||||
|
)
|
||||||
|
metric = f""" 'Lieferscheine' => [
|
||||||
|
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
|
||||||
|
'rules' => [
|
||||||
|
'{rule}',
|
||||||
|
],
|
||||||
|
],"""
|
||||||
|
original_source = semantics()
|
||||||
|
original_test = """<?php
|
||||||
|
$context = $service->businessContext(['FK_Auftrag'], 'erp');
|
||||||
|
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
||||||
|
"""
|
||||||
|
current_test = """<?php
|
||||||
|
$context = $service->businessContext(['FK_Auftrag', 'FK_Nummerkreis'], 'erp');
|
||||||
|
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
||||||
|
$this->assertStringContainsString('Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine niemals verwenden.', $context);
|
||||||
|
$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);
|
||||||
|
"""
|
||||||
|
(tmp_path / "config").mkdir()
|
||||||
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||||
|
(tmp_path / source_file).write_text(semantics(metric))
|
||||||
|
(tmp_path / test_file).write_text(current_test)
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.head_file.side_effect = lambda filename: {
|
||||||
|
source_file: original_source,
|
||||||
|
test_file: original_test,
|
||||||
|
}.get(filename)
|
||||||
|
summary = ChangeSummary((source_file, test_file), 4, 1)
|
||||||
|
|
||||||
|
accepted = evaluate_removal_prevention(report, summary, workspace)
|
||||||
|
|
||||||
|
assert accepted.allowed
|
||||||
|
|
||||||
|
(tmp_path / source_file).write_text(semantics(
|
||||||
|
metric,
|
||||||
|
reference_sql.replace("AuftragsKennung = 2", "AuftragsKennung = 4"),
|
||||||
|
))
|
||||||
|
regressed = evaluate_removal_prevention(report, summary, workspace)
|
||||||
|
|
||||||
|
assert not regressed.allowed
|
||||||
|
assert any("protected baseline" in error for error in regressed.errors)
|
||||||
|
|
||||||
|
|
||||||
|
def test_removal_only_scope_allows_only_exact_predicate_removal_and_added_test(tmp_path) -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=5,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong delivery note count",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
|
||||||
|
"Einfach nur das aus der SQL entfernen"
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month in 2024",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
original_sql = (
|
||||||
|
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
|
||||||
|
"WHERE a.AuftragsKennung = 2 AND a.bStatus_gebucht = 1 "
|
||||||
|
"AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
|
||||||
|
)
|
||||||
|
repaired_sql = (
|
||||||
|
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
|
||||||
|
"WHERE a.AuftragsKennung = 2 AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
|
||||||
|
)
|
||||||
|
|
||||||
|
def semantics(sql: str) -> str:
|
||||||
|
return f"""<?php
|
||||||
|
return [
|
||||||
|
'examples' => [[
|
||||||
|
'question' => 'Delivery notes by month in 2024',
|
||||||
|
'sql' => <<<'SQL'
|
||||||
|
{sql}
|
||||||
|
SQL,
|
||||||
|
]],
|
||||||
|
];
|
||||||
|
"""
|
||||||
|
|
||||||
|
source_file = "config/erp_semantics.php"
|
||||||
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||||
|
(tmp_path / "config").mkdir()
|
||||||
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||||
|
(tmp_path / source_file).write_text(semantics(repaired_sql))
|
||||||
|
previous_test = "<?php\n$this->assertStringContainsString('System_created', $context);\n"
|
||||||
|
(tmp_path / test_file).write_text(
|
||||||
|
previous_test
|
||||||
|
+ "$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
||||||
|
)
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.head_file.side_effect = lambda filename: {
|
||||||
|
source_file: semantics(original_sql),
|
||||||
|
test_file: previous_test,
|
||||||
|
}.get(filename)
|
||||||
|
summary = ChangeSummary((source_file, test_file), 2, 1)
|
||||||
|
|
||||||
|
result = evaluate_removal_scope(report, summary, workspace)
|
||||||
|
|
||||||
|
assert result.allowed
|
||||||
|
|
||||||
|
(tmp_path / source_file).write_text(semantics(repaired_sql.replace(
|
||||||
|
"System_created",
|
||||||
|
"Datum_erfassung",
|
||||||
|
)))
|
||||||
|
regression = evaluate_removal_scope(report, summary, workspace)
|
||||||
|
|
||||||
|
assert not regression.allowed
|
||||||
|
assert any("outside the explicitly forbidden" in error for error in regression.errors)
|
||||||
367
old_2026-08-31/tests/test_prompt.py
Normal file
367
old_2026-08-31/tests/test_prompt.py
Normal file
@@ -0,0 +1,367 @@
|
|||||||
|
from lai_automation.models import FailureReport
|
||||||
|
from lai_automation.prompt import build_retry_task, build_task, redact, repair_files
|
||||||
|
|
||||||
|
|
||||||
|
def test_redact_removes_common_secrets_and_personal_markers() -> None:
|
||||||
|
value = "password=hunter2 token: abc123 admin@example.com 10.2.3.4 Bearer secret.jwt"
|
||||||
|
result = redact(value)
|
||||||
|
assert "hunter2" not in result
|
||||||
|
assert "abc123" not in result
|
||||||
|
assert "admin@example.com" not in result
|
||||||
|
assert "10.2.3.4" not in result
|
||||||
|
assert "secret.jwt" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_contains_constraints_and_redacted_report() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=91,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="password=secret-value",
|
||||||
|
feedback_type="wrong_join",
|
||||||
|
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
|
||||||
|
prompt="List locations for admin@example.com",
|
||||||
|
response="The wrong product count was returned.",
|
||||||
|
sql_query="SELECT * FROM company",
|
||||||
|
error_message=None,
|
||||||
|
context={"model": "mistral", "database_type": "customer"},
|
||||||
|
)
|
||||||
|
task = build_task(report)
|
||||||
|
assert "failure report #7" in task
|
||||||
|
assert "secret-value" not in task
|
||||||
|
assert "admin@example.com" not in task
|
||||||
|
assert "Do not commit or push" in task
|
||||||
|
assert "never mock or stub the behavior being tested" in task
|
||||||
|
assert "Repository ownership map:" in task
|
||||||
|
assert "config/erp_semantics.php owns ERP joins" in task
|
||||||
|
assert "Tests must contain assertions that execute" in task
|
||||||
|
assert "Do not expose protected production methods publicly" in task
|
||||||
|
assert "Feedback type: wrong_join" in task
|
||||||
|
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
|
||||||
|
assert "The wrong product count was returned." in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_redact_removes_extended_credentials() -> None:
|
||||||
|
private_key = "-----BEGIN PRIVATE KEY-----\nprivate-material\n-----END PRIVATE KEY-----"
|
||||||
|
value = (
|
||||||
|
"AKIAIOSFODNN7EXAMPLE "
|
||||||
|
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.signature "
|
||||||
|
"mysql://admin:database-password@db.internal/lai "
|
||||||
|
f"{private_key}"
|
||||||
|
)
|
||||||
|
result = redact(value)
|
||||||
|
assert "AKIAIOSFODNN7EXAMPLE" not in result
|
||||||
|
assert "eyJhbGci" not in result
|
||||||
|
assert "database-password" not in result
|
||||||
|
assert "private-material" not in result
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_task_contains_failure_without_secrets() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=8,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use a weighted average",
|
||||||
|
prompt="Average sale price",
|
||||||
|
response="Zero",
|
||||||
|
sql_query="SELECT AVG(list_price)",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
task = build_retry_task(report, RuntimeError("Missing import; token=secret-value"))
|
||||||
|
|
||||||
|
assert "Missing import" in task
|
||||||
|
assert "secret-value" not in task
|
||||||
|
assert "Do not remove or weaken tests" in task
|
||||||
|
assert "unrelated production routing" in task
|
||||||
|
assert "Use a weighted average" in task
|
||||||
|
assert "SELECT AVG(list_price)" in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_incorrect_answer_task_guides_semantics_level_repair() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=42,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Sum net values and divide by quantity",
|
||||||
|
prompt="Average sale price for article 9031",
|
||||||
|
response="0.00",
|
||||||
|
sql_query="SELECT AVG(Vk_preis) FROM FK_Artikel",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "config/*_semantics.php" in task
|
||||||
|
assert "Do not hardcode" in task
|
||||||
|
assert "protected schema/business-" in task
|
||||||
|
assert "context builder" in task
|
||||||
|
assert "known-bad SQL" in task
|
||||||
|
assert "compare every alias.column" in task
|
||||||
|
assert "Never replace one missing column" in task
|
||||||
|
assert "Never require or copy production rows" in task
|
||||||
|
assert repair_files(report) == (
|
||||||
|
"config/erp_semantics.php",
|
||||||
|
"tests/Unit/SQLGeneratorBusinessContextTest.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_derives_exact_sql_literal_contract() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
"FK_Auftrag nach Auftragskennung 2 filtern, System_created nutzen, "
|
||||||
|
"obsolet und status storniert ausfiltern"
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT Datum_erfassung",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "Exact SQL literal contract" in task
|
||||||
|
assert "a.AuftragsKennung = 2" in task
|
||||||
|
assert "a.System_created" in task
|
||||||
|
assert "COALESCE(a.obsolet, 0) = 0" in task
|
||||||
|
assert "COALESCE(a.bStatus_storniert, 0) = 0" in task
|
||||||
|
assert "Do not add an LS lookup or bStatus_gebucht" in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_task_derives_exact_negative_contract_for_removal_only_report() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=5,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong delivery note count",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
|
||||||
|
"Einfach nur das aus der SQL entfernen"
|
||||||
|
),
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "Exact SQL removal contract" in task
|
||||||
|
assert "a.bStatus_gebucht = 1" in task
|
||||||
|
assert "negative assertion" in task
|
||||||
|
assert "forbidden identifier is a negative requirement" in task
|
||||||
|
assert (
|
||||||
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||||
|
in task
|
||||||
|
)
|
||||||
|
assert "copy them exactly, including aliases and qualifiers" in task
|
||||||
|
assert "explicitly a removal-only repair" in task
|
||||||
|
assert "Preserve every other selected field" in task
|
||||||
|
assert "Exact SQL literal contract" not in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_non_removal_only_contract_protects_correct_reference_and_exercises_rule() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=6,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Die Abfrage stimmt nicht",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||||
|
"weil Lieferscheine nicht gebucht werden."
|
||||||
|
),
|
||||||
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
response="0 Ergebnisse",
|
||||||
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "protected correct baseline" in task
|
||||||
|
assert "preserve it exactly" in task
|
||||||
|
assert "positively assert the new rule" in task
|
||||||
|
assert "negative assertion alone is vacuous" in task
|
||||||
|
assert "without embedding the complete forbidden predicate" in task
|
||||||
|
assert "Do not add or restate date-column choices" in task
|
||||||
|
assert "every table in its tables list" in task
|
||||||
|
assert "supplied to businessContext()" in task
|
||||||
|
assert "explicitly a removal-only repair" not in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_retry_task_keeps_failure_from_start_of_long_output() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=43,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="Top products",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT a.missing FROM orders a",
|
||||||
|
error_message="Unknown column",
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_retry_task(report, RuntimeError("DECISIVE ASSERTION\n" + "x" * 11_000))
|
||||||
|
|
||||||
|
assert "DECISIVE ASSERTION" in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_customer_database_task_explains_tenant_scope() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=13,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="What did this customer buy most?",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT * FROM invoice_row WHERE InvoiceCreator = 'tenant'",
|
||||||
|
error_message="Unknown column InvoiceCreator",
|
||||||
|
context={"database_type": "customer"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "already selected one tenant database" in task
|
||||||
|
assert "must not" in task
|
||||||
|
assert "guessed column" in task
|
||||||
|
assert "tenant_type" in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_unresolved_placeholder_task_guides_real_answer_path() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=49,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Token was not restored",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Return the database value",
|
||||||
|
prompt="Average price",
|
||||||
|
response="Price: __LAI_UNKNOWN_VALUE__",
|
||||||
|
sql_query="SELECT AVG(price)",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "AIService::generateDatabaseResponse" in task
|
||||||
|
assert "exact token map" in task
|
||||||
|
assert "HTTP-backed AIService test" in task
|
||||||
|
assert "local database-result fallback" in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_cross_database_report_guides_staged_source_routing() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=56,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong hours",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Join cockpit_config.time_registration to customer manager and ERP positions",
|
||||||
|
prompt="Hours in April",
|
||||||
|
response="0 hours",
|
||||||
|
sql_query="SELECT SUM(hours) FROM LX_AUFGABE",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "tables from multiple databases" in task
|
||||||
|
assert "staged-query workflows" in task
|
||||||
|
assert "Do not put an impossible cross-connection join" in task
|
||||||
|
assert "preserving tenant isolation" in task
|
||||||
|
assert "editing only config/*_semantics.php or tests is an invalid repair" in task
|
||||||
|
assert "describe the data" in task
|
||||||
|
assert "flow, connection used at each stage" in task
|
||||||
|
assert repair_files(report) == (
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"app/Services/TenantDatabaseService.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_presentation_and_route_feedback_is_routed_to_real_owners() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
|
||||||
|
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
|
||||||
|
),
|
||||||
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
response="2024-01: 38, 2024-02: 95",
|
||||||
|
sql_query="SELECT Monat, COUNT(*) FROM FK_Auftrag GROUP BY Monat",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert repair_files(report) == (
|
||||||
|
"app/Services/AIService.php",
|
||||||
|
"app/Http/Controllers/ChatController.php",
|
||||||
|
"app/Services/WebSearchService.php",
|
||||||
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||||
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||||
|
"tests/Unit/WebSearchServiceTest.php",
|
||||||
|
)
|
||||||
|
task = build_task(report)
|
||||||
|
assert "real answer-formatting change" in task
|
||||||
|
assert "exact reported user prompt" in task
|
||||||
|
assert '"cockpit:" must force the database path' in task
|
||||||
|
assert '"www:" must force public web research' in task
|
||||||
|
assert '"direct:" must' in task
|
||||||
|
assert "including Ollama/Qwen" in task
|
||||||
|
assert "Do not pass private conversation/database context" in task
|
||||||
|
assert "AIService::determineRoute() must not be called" in task
|
||||||
|
assert "resolveRoutePrefix(string $message): array" in task
|
||||||
|
assert "[$explicitRoute, $message] = $this->resolveRoutePrefix($message)" in task
|
||||||
|
assert "Do not add prefix handling to resolveDatabaseSource()" in task
|
||||||
|
assert "parent::resolveRoutePrefix($message)" in task
|
||||||
|
assert "three tests that call processMessage() itself" in task
|
||||||
|
assert "setLaravelSession()" in task
|
||||||
|
assert "generateWebResearchResponse()" in task
|
||||||
|
assert "columns named exactly Monat and Anzahl_Lieferscheine" in task
|
||||||
|
assert 'exact substring "allen bereitgestellten Zeilen"' in task
|
||||||
|
assert "substr_count($userPrompt" in task
|
||||||
|
assert "heuristics must return false" in task
|
||||||
|
assert "persisted ChatMessage.message equals the cleaned question exactly" in task
|
||||||
|
assert "$response->getData(true)" in task
|
||||||
|
assert "Do not write “we can't fully test processMessage”" in task
|
||||||
|
assert "one ordinary result row with multiple" in task
|
||||||
|
assert "duplicate method declaration" in task
|
||||||
|
assert "All three prefix behaviors" in task
|
||||||
|
assert "config/*_semantics.php" in task
|
||||||
145
old_2026-08-31/tests/test_removal.py
Normal file
145
old_2026-08-31/tests/test_removal.py
Normal file
@@ -0,0 +1,145 @@
|
|||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from lai_automation.models import FailureReport
|
||||||
|
from lai_automation.removal import (
|
||||||
|
detect_already_fixed,
|
||||||
|
extract_reference_examples,
|
||||||
|
find_matching_reference,
|
||||||
|
is_removal_only,
|
||||||
|
removal_directives,
|
||||||
|
remove_sql_fragment,
|
||||||
|
sql_contains_fragment,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def removal_report(**overrides) -> FailureReport:
|
||||||
|
values = {
|
||||||
|
"id": 5,
|
||||||
|
"chat_message_id": 1,
|
||||||
|
"source": "user_feedback",
|
||||||
|
"category": "incorrect_answer",
|
||||||
|
"user_feedback": "Die Anzahl der Lieferscheine ist falsch",
|
||||||
|
"feedback_type": "wrong_result",
|
||||||
|
"expected_behavior": (
|
||||||
|
'Es muss " a.bStatus_gebucht = 1 " entfernt werden! '
|
||||||
|
"Lieferscheine werden nicht gebucht! Einfach nur das aus der SQL entfernen"
|
||||||
|
),
|
||||||
|
"prompt": "Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
"response": "0 Ergebnisse",
|
||||||
|
"sql_query": (
|
||||||
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||||
|
"AND a.bStatus_gebucht = 1 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
||||||
|
),
|
||||||
|
"error_message": None,
|
||||||
|
"context": {"database_type": "erp"},
|
||||||
|
}
|
||||||
|
values.update(overrides)
|
||||||
|
return FailureReport(**values)
|
||||||
|
|
||||||
|
|
||||||
|
def semantics(sql: str, question: str | None = None) -> str:
|
||||||
|
question = question or "Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024"
|
||||||
|
return f"""<?php
|
||||||
|
|
||||||
|
return [
|
||||||
|
'examples' => [
|
||||||
|
[
|
||||||
|
'question' => '{question}',
|
||||||
|
'sql' => <<<'SQL'
|
||||||
|
{sql}
|
||||||
|
SQL,
|
||||||
|
],
|
||||||
|
],
|
||||||
|
];
|
||||||
|
"""
|
||||||
|
|
||||||
|
|
||||||
|
def test_extracts_exact_quoted_german_removal_contract() -> None:
|
||||||
|
directives = removal_directives(removal_report())
|
||||||
|
|
||||||
|
assert tuple(item.fragment for item in directives) == ("a.bStatus_gebucht = 1",)
|
||||||
|
assert is_removal_only(removal_report())
|
||||||
|
|
||||||
|
|
||||||
|
def test_infers_unquoted_predicate_from_observed_sql() -> None:
|
||||||
|
report = removal_report(
|
||||||
|
expected_behavior="Bitte den bStatus_gebucht Filter aus der SQL entfernen",
|
||||||
|
)
|
||||||
|
|
||||||
|
directives = removal_directives(report)
|
||||||
|
|
||||||
|
assert len(directives) == 1
|
||||||
|
assert sql_contains_fragment(directives[0].fragment, "a.bStatus_gebucht = 1")
|
||||||
|
|
||||||
|
|
||||||
|
def test_does_not_invert_do_not_remove_instruction() -> None:
|
||||||
|
report = removal_report(
|
||||||
|
expected_behavior="Do not remove `a.bStatus_gebucht = 1` from invoice SQL",
|
||||||
|
)
|
||||||
|
|
||||||
|
assert removal_directives(report) == ()
|
||||||
|
assert not is_removal_only(report)
|
||||||
|
|
||||||
|
|
||||||
|
def test_removes_only_the_requested_conjunct() -> None:
|
||||||
|
sql = (
|
||||||
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung=2 "
|
||||||
|
"AND a.bStatus_gebucht=1 AND COALESCE(a.obsolet,0)=0 GROUP BY a.SheetNr;"
|
||||||
|
)
|
||||||
|
|
||||||
|
repaired = remove_sql_fragment(sql, "a.bStatus_gebucht = 1")
|
||||||
|
|
||||||
|
assert not sql_contains_fragment(repaired, "a.bStatus_gebucht = 1")
|
||||||
|
assert "a.AuftragsKennung=2" in repaired
|
||||||
|
assert "COALESCE(a.obsolet,0)=0" in repaired
|
||||||
|
assert "GROUP BY a.SheetNr" in repaired
|
||||||
|
|
||||||
|
|
||||||
|
def test_extracts_and_uniquely_matches_reference_example() -> None:
|
||||||
|
content = semantics("SELECT 1") + semantics(
|
||||||
|
"SELECT 2",
|
||||||
|
"Welcher Kunde hatte im Januar den meisten Umsatz?",
|
||||||
|
)
|
||||||
|
|
||||||
|
examples = extract_reference_examples(content)
|
||||||
|
match, score = find_matching_reference(content, removal_report().prompt)
|
||||||
|
|
||||||
|
assert len(examples) == 2
|
||||||
|
assert match is not None
|
||||||
|
assert match.sql == "SELECT 1"
|
||||||
|
assert score == 1.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_detects_already_fixed_only_with_matching_reference(tmp_path: Path) -> None:
|
||||||
|
source = tmp_path / "config/erp_semantics.php"
|
||||||
|
source.parent.mkdir(parents=True)
|
||||||
|
source.write_text(semantics(
|
||||||
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||||
|
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
||||||
|
))
|
||||||
|
|
||||||
|
result = detect_already_fixed(
|
||||||
|
removal_report(),
|
||||||
|
tmp_path,
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result.filename == "config/erp_semantics.php"
|
||||||
|
assert result.match_score == 1.0
|
||||||
|
assert result.forbidden_fragments == ("a.bStatus_gebucht = 1",)
|
||||||
|
|
||||||
|
|
||||||
|
def test_does_not_mark_additive_or_mismatched_report_already_fixed(tmp_path: Path) -> None:
|
||||||
|
source = tmp_path / "config/erp_semantics.php"
|
||||||
|
source.parent.mkdir(parents=True)
|
||||||
|
source.write_text(semantics("SELECT COUNT(*) FROM FK_Auftrag a"))
|
||||||
|
additive = removal_report(
|
||||||
|
expected_behavior=(
|
||||||
|
'"a.bStatus_gebucht = 1" entfernen und zusätzlich System_created verwenden'
|
||||||
|
),
|
||||||
|
)
|
||||||
|
mismatched = removal_report(prompt="Zeige Rechnungen im Jahr 2024")
|
||||||
|
|
||||||
|
assert detect_already_fixed(additive, tmp_path, ("config/erp_semantics.php",)) is None
|
||||||
|
assert detect_already_fixed(mismatched, tmp_path, ("config/erp_semantics.php",)) is None
|
||||||
968
old_2026-08-31/tests/test_worker.py
Normal file
968
old_2026-08-31/tests/test_worker.py
Normal file
@@ -0,0 +1,968 @@
|
|||||||
|
import threading
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import MagicMock, Mock, patch
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from test_config import settings
|
||||||
|
|
||||||
|
from lai_automation.agent import AgentProviderError, AgentTimeoutError
|
||||||
|
from lai_automation.git import ChangeSummary
|
||||||
|
from lai_automation.models import FailureReport
|
||||||
|
from lai_automation.worker import AutomationWorker, ValidationSetupError
|
||||||
|
|
||||||
|
|
||||||
|
def test_dry_run_generates_task_without_agent_or_git() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=11,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="List locations",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message="Unknown column",
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
|
||||||
|
assert worker.run_once()
|
||||||
|
worker.reports.release.assert_called_once()
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_stop_signal_interrupts_worker_wait() -> None:
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
|
||||||
|
worker._stop(15, None)
|
||||||
|
|
||||||
|
assert worker.stop_event.is_set()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pull_request_body_contains_redacted_original_report() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=7,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Use a table\ntoken=super-secret-token",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Show every supplied row",
|
||||||
|
prompt="Count delivery notes by month",
|
||||||
|
response="January: 38, February: 95",
|
||||||
|
sql_query="SELECT month, COUNT(*) FROM deliveries GROUP BY month",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
summary = ChangeSummary(("app/Services/AIService.php",), 5, 1)
|
||||||
|
|
||||||
|
body = AutomationWorker._pull_request_body(
|
||||||
|
report,
|
||||||
|
summary,
|
||||||
|
[{"command": "php artisan test", "returncode": 0}],
|
||||||
|
)
|
||||||
|
|
||||||
|
assert "## Original failure report" in body
|
||||||
|
assert "**Source**\n> user_feedback" in body
|
||||||
|
assert "**Category**\n> incorrect_answer" in body
|
||||||
|
assert "**Feedback type**\n> wrong_result" in body
|
||||||
|
assert "**User prompt**\n> Count delivery notes by month" in body
|
||||||
|
assert "**Observed response**\n> January: 38, February: 95" in body
|
||||||
|
assert "**User feedback**\n> Use a table\n> token=[REDACTED]" in body
|
||||||
|
assert "**Expected behavior**\n> Show every supplied row" in body
|
||||||
|
assert "**Error message**\n> _Not provided._" in body
|
||||||
|
assert "**Generated SQL**\n> SELECT month, COUNT(*)" in body
|
||||||
|
assert "super-secret-token" not in body
|
||||||
|
assert "## Scope\n- `app/Services/AIService.php`" in body
|
||||||
|
assert "## Validation\n- PASS `php artisan test`" in body
|
||||||
|
|
||||||
|
|
||||||
|
def test_database_credentials_are_only_passed_to_test_process(tmp_path) -> None:
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
worker._prepare_test_schema = Mock()
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.run.return_value = Mock(returncode=0, stdout="ok", stderr="")
|
||||||
|
|
||||||
|
results = worker._run_tests(workspace)
|
||||||
|
|
||||||
|
assert results[0]["returncode"] == 0
|
||||||
|
environment = workspace.run.call_args.kwargs["extra_environment"]
|
||||||
|
assert environment["AGENT_TEST_DB_ENABLED"] == "true"
|
||||||
|
assert environment["APP_ENV"] == "testing"
|
||||||
|
assert environment["AI_MISTRAL_API_KEY"] == "automation-test-key"
|
||||||
|
assert environment["TEST_DB_USER"] == "test_runner"
|
||||||
|
assert environment["TEST_DB_PASSWORD"] == "test-secret"
|
||||||
|
assert environment["DB_DATABASE"] == "lai_agent_test"
|
||||||
|
assert environment["ERP_DATABASE"] == "customer_agent_test"
|
||||||
|
assert environment["COCKPIT_CONFIG_DATABASE"] == "cockpit_agent_test"
|
||||||
|
|
||||||
|
|
||||||
|
def test_workspace_setup_creates_safe_env_and_prepares_assets(tmp_path) -> None:
|
||||||
|
worker = AutomationWorker(settings(
|
||||||
|
workspace_root=tmp_path,
|
||||||
|
setup_commands=("npm run build",),
|
||||||
|
))
|
||||||
|
worker._prepare_test_schema = Mock()
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path / "failure-1"
|
||||||
|
workspace.path.mkdir()
|
||||||
|
workspace.run.return_value = Mock(returncode=0, stdout="built", stderr="")
|
||||||
|
|
||||||
|
results = worker._prepare_workspace_for_agent(workspace, 1)
|
||||||
|
|
||||||
|
env_contents = (workspace.path / ".env").read_text()
|
||||||
|
assert "APP_ENV=testing" in env_contents
|
||||||
|
assert "DB_PASSWORD" not in env_contents
|
||||||
|
assert results[0]["stage"] == "workspace setup"
|
||||||
|
assert workspace.run.call_args.kwargs["extra_environment"]["DB_PASSWORD"] == "test-secret"
|
||||||
|
|
||||||
|
|
||||||
|
def test_processing_connection_failure_is_not_sent_to_agent() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=15,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="processing",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="List customers",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message="Access denied for user 'runtime'@'host'",
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
worker.agent.run.assert_not_called()
|
||||||
|
worker.reports.fail.assert_called_once()
|
||||||
|
assert worker.reports.fail.call_args.args[2]["classification"] == "operational"
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_provider_failure_is_classified_as_operational() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=16,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use the correct rule",
|
||||||
|
prompt="List records",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = AgentProviderError("Mistral DNS failed")
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-16"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
worker.reports.fail.assert_called_once()
|
||||||
|
metadata = worker.reports.fail.call_args.args[2]
|
||||||
|
assert metadata["classification"] == "operational"
|
||||||
|
assert metadata["automation_action"] == "no_source_change"
|
||||||
|
workspace.commit.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_schema_creates_minimal_erp_lookup(monkeypatch) -> None:
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
cursor = Mock()
|
||||||
|
connection = MagicMock()
|
||||||
|
connection.cursor.return_value.__enter__.return_value = cursor
|
||||||
|
connect = Mock(return_value=connection)
|
||||||
|
monkeypatch.setattr("lai_automation.worker.pymysql.connect", connect)
|
||||||
|
|
||||||
|
worker._prepare_test_schema()
|
||||||
|
|
||||||
|
assert connect.call_args.kwargs["database"] == "customer_agent_test"
|
||||||
|
assert "CREATE TABLE IF NOT EXISTS FK_Nummerkreis" in cursor.execute.call_args.args[0]
|
||||||
|
assert cursor.executemany.call_args.args[1][0] == (3, "Rechnung", "RG")
|
||||||
|
connection.close.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_commands_are_logged_without_output(caplog, tmp_path) -> None:
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
worker._prepare_test_schema = Mock()
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.run.return_value = Mock(returncode=0, stdout="sensitive output", stderr="")
|
||||||
|
|
||||||
|
with caplog.at_level("INFO"):
|
||||||
|
worker._run_tests(workspace, report_id=7, attempt=2)
|
||||||
|
|
||||||
|
assert "Report 7 attempt 2 validation command: php artisan test starting" in caplog.text
|
||||||
|
assert "returncode=0" in caplog.text
|
||||||
|
assert "sensitive output" not in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_validation_preserves_start_and_end_of_long_output(tmp_path) -> None:
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
worker._prepare_test_schema = Mock()
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.run.return_value = Mock(
|
||||||
|
returncode=1,
|
||||||
|
stdout="DECISIVE ASSERTION\n" + "x" * 13_000,
|
||||||
|
stderr="FINAL ERROR",
|
||||||
|
)
|
||||||
|
|
||||||
|
try:
|
||||||
|
worker._run_tests(workspace)
|
||||||
|
except RuntimeError as exception:
|
||||||
|
failure = str(exception)
|
||||||
|
else:
|
||||||
|
raise AssertionError("Expected failed validation")
|
||||||
|
|
||||||
|
assert "DECISIVE ASSERTION" in failure
|
||||||
|
assert "...[middle omitted]..." in failure
|
||||||
|
assert "FINAL ERROR" in failure
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_signature_ignores_counts_timings_and_line_numbers() -> None:
|
||||||
|
first = """
|
||||||
|
FAILED Tests\\Unit\\RouteTest > prefixes are removed
|
||||||
|
Expectation failed for method determineRoute when invoked 1 time.
|
||||||
|
Method was expected to be called 1 time, actually called 0 times.
|
||||||
|
Tests: 1 failed, 50 passed (186 assertions)
|
||||||
|
Duration: 0.49s
|
||||||
|
at tests/Unit/RouteTest.php:229
|
||||||
|
"""
|
||||||
|
second = """
|
||||||
|
FAILED Tests\\Unit\\RouteTest > prefixes are removed
|
||||||
|
Expectation failed for method determineRoute when invoked 1 time.
|
||||||
|
Method was expected to be called 1 time, actually called 0 times.
|
||||||
|
Tests: 1 failed, 50 passed (191 assertions)
|
||||||
|
Duration: 0.50s
|
||||||
|
at tests/Unit/RouteTest.php:269
|
||||||
|
"""
|
||||||
|
|
||||||
|
assert AutomationWorker._validation_signature(first) == AutomationWorker._validation_signature(
|
||||||
|
second
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_validation_setup_failure_is_typed_and_not_a_test_failure(tmp_path) -> None:
|
||||||
|
worker = AutomationWorker(settings(setup_commands=("npm ci",)))
|
||||||
|
worker._prepare_test_schema = Mock()
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.run.return_value = Mock(
|
||||||
|
returncode=1,
|
||||||
|
stdout="network unavailable",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
|
||||||
|
with pytest.raises(ValidationSetupError, match="Validation setup command failed"):
|
||||||
|
worker._run_tests(workspace)
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_stage_emits_heartbeat(caplog) -> None:
|
||||||
|
worker = AutomationWorker(settings())
|
||||||
|
|
||||||
|
with caplog.at_level("INFO"), worker._progress(7, "agent attempt 1", interval=0.001):
|
||||||
|
threading.Event().wait(0.01)
|
||||||
|
|
||||||
|
assert "Report 7 agent attempt 1 still running" in caplog.text
|
||||||
|
assert "Report 7 agent attempt 1 finished" in caplog.text
|
||||||
|
|
||||||
|
|
||||||
|
def test_long_stage_refreshes_database_claim_lease() -> None:
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
|
||||||
|
with worker._progress(7, "agent attempt 1", interval=0.001):
|
||||||
|
threading.Event().wait(0.01)
|
||||||
|
|
||||||
|
assert worker.reports.heartbeat.call_count >= 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_timed_out_agent_partial_changes_are_validated_and_published() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use System_created",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = AgentTimeoutError("timed out after applying edits")
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-4"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
20,
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
workspace.added_text.return_value = (
|
||||||
|
"Use DATE_FORMAT(System_created, '%Y-%m') and AuftragsKennung = 2; "
|
||||||
|
"COALESCE(obsolet, 0) = 0 and COALESCE(bStatus_storniert, 0) = 0"
|
||||||
|
)
|
||||||
|
workspace.run_args.return_value = Mock(
|
||||||
|
returncode=0,
|
||||||
|
stdout="No syntax errors",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
workspace.push_branch.assert_called_once()
|
||||||
|
worker.reports.complete.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_provider_failure_partial_changes_are_validated_and_published() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use System_created",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = AgentProviderError("provider failed after edits")
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-4"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
20,
|
||||||
|
2,
|
||||||
|
)
|
||||||
|
workspace.added_text.return_value = (
|
||||||
|
"Use DATE_FORMAT(System_created, '%Y-%m') and AuftragsKennung = 2; "
|
||||||
|
"COALESCE(obsolet, 0) = 0 and COALESCE(bStatus_storniert, 0) = 0"
|
||||||
|
)
|
||||||
|
workspace.run_args.return_value = Mock(
|
||||||
|
returncode=0,
|
||||||
|
stdout="No syntax errors",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
workspace.push_branch.assert_called_once()
|
||||||
|
worker.reports.complete.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_php_syntax_failure_discards_attempt_before_full_validation() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=56,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong hours",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use time registration",
|
||||||
|
prompt="Hours in April",
|
||||||
|
response="0 hours",
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||||
|
worker = AutomationWorker(config)
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["broken PHP", "fixed PHP"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-56"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
|
||||||
|
)
|
||||||
|
workspace.run_args.side_effect = [
|
||||||
|
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
|
||||||
|
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||||
|
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||||
|
]
|
||||||
|
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+intended fix"
|
||||||
|
workspace.checkpoint.return_value = "valid-checkpoint"
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 2
|
||||||
|
assert "unexpected token" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
assert "clean repository" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
assert "intended fix" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
assert "repair-only attempt" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
workspace.restore_checkpoint.assert_called_once_with(None)
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_malformed_retry_restores_last_syntax_valid_attempt() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=56,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong hours",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use time registration",
|
||||||
|
prompt="Hours in April",
|
||||||
|
response="0 hours",
|
||||||
|
sql_query="SELECT 0",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
config = settings(mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3)
|
||||||
|
worker = AutomationWorker(config)
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["valid but failing", "broken retry", "corrected retry"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._run_tests = Mock(side_effect=[RuntimeError("Expected hour rule"), []])
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-56"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
|
||||||
|
)
|
||||||
|
workspace.run_args.side_effect = [
|
||||||
|
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||||
|
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||||
|
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
|
||||||
|
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||||
|
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||||
|
]
|
||||||
|
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+application repair"
|
||||||
|
workspace.checkpoint.side_effect = ["checkpoint-1", "checkpoint-3"]
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 3
|
||||||
|
workspace.restore_checkpoint.assert_called_once_with("checkpoint-1")
|
||||||
|
assert "last syntax-valid changes were restored" in worker.agent.run.call_args_list[2].args[1]
|
||||||
|
assert "application repair" in worker.agent.run.call_args_list[2].args[1]
|
||||||
|
assert worker._run_tests.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_validation_gets_one_correction_before_pr() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=13,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="List products",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT missing_column FROM invoice_row",
|
||||||
|
error_message="Unknown column",
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||||
|
worker = AutomationWorker(config)
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["first change", "corrected change"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._run_tests = Mock(side_effect=[RuntimeError("Missing import"), []])
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-13"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1
|
||||||
|
)
|
||||||
|
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 2
|
||||||
|
assert "Missing import" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
assert worker._run_tests.call_count == 2
|
||||||
|
workspace.push_branch.assert_called_once()
|
||||||
|
worker.reports.complete.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_failed_validation_retry_keeps_full_routed_context() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="AuftragsKennung 2 verwenden",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT Datum_erfassung",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["valid production with bad test", "fixed assertion"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
worker._run_tests = Mock(side_effect=[RuntimeError("case-sensitive assertion failed"), []])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-4"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
20,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
added = (
|
||||||
|
"AuftragsKennung = 2\n"
|
||||||
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2"
|
||||||
|
)
|
||||||
|
workspace.added_text.return_value = added
|
||||||
|
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||||
|
workspace.checkpoint.return_value = "valid-production"
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_args_list[1].args[2] == (
|
||||||
|
"config/erp_semantics.php",
|
||||||
|
"tests/Unit/SQLGeneratorBusinessContextTest.php",
|
||||||
|
)
|
||||||
|
retry_task = worker.agent.run.call_args_list[1].args[1]
|
||||||
|
assert "inspect every routed production and test file" in retry_task
|
||||||
|
assert "Do not merely rewrite assertions" in retry_task
|
||||||
|
assert "case-sensitive" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
attempts = worker.reports.complete.call_args.args[1]["agent_attempts"]
|
||||||
|
assert attempts[0]["validation_status"] == "failed"
|
||||||
|
assert "case-sensitive assertion failed" in attempts[0]["validation_error_tail"]
|
||||||
|
assert attempts[1]["validation_status"] == "passed"
|
||||||
|
|
||||||
|
|
||||||
|
def test_no_change_response_gets_one_correction() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=14,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="List products",
|
||||||
|
response=None,
|
||||||
|
sql_query=None,
|
||||||
|
error_message="Unknown column",
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||||
|
worker = AutomationWorker(config)
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["no edit", "applied edit"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-14"
|
||||||
|
workspace.summary.side_effect = [
|
||||||
|
ChangeSummary((), 0, 0),
|
||||||
|
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
|
||||||
|
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
|
||||||
|
]
|
||||||
|
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 2
|
||||||
|
assert "Agent produced no file changes" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
workspace.push_branch.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_test_only_change_gets_correction_before_validation() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=49,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Token was not restored",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Return the real value",
|
||||||
|
prompt="Average price",
|
||||||
|
response="__LAI_UNKNOWN__",
|
||||||
|
sql_query="SELECT AVG(price)",
|
||||||
|
error_message=None,
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||||
|
worker = AutomationWorker(config)
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["tests only", "production fix"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-49"
|
||||||
|
workspace.summary.side_effect = [
|
||||||
|
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 8, 0),
|
||||||
|
ChangeSummary(
|
||||||
|
("app/Services/AIService.php", "tests/Feature/AIPrivacyBoundaryTest.php"),
|
||||||
|
15,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
ChangeSummary(
|
||||||
|
("app/Services/AIService.php", "tests/Feature/AIPrivacyBoundaryTest.php"),
|
||||||
|
15,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 2
|
||||||
|
assert "changed only tests" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
assert "exact repository-relative paths" in worker.agent.run.call_args_list[1].args[1]
|
||||||
|
workspace.checkpoint.assert_called_once()
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_semantically_invalid_change_is_not_kept_as_retry_checkpoint() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="System_created verwenden und obsolet ausfiltern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query="SELECT Datum_erfassung",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["missing filter", "correct repair"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-4"
|
||||||
|
workspace.summary.return_value = ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
12,
|
||||||
|
1,
|
||||||
|
)
|
||||||
|
workspace.added_text.side_effect = [
|
||||||
|
"System_created",
|
||||||
|
"System_created AND COALESCE(obsolet, 0) = 0",
|
||||||
|
"System_created AND COALESCE(obsolet, 0) = 0",
|
||||||
|
]
|
||||||
|
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||||
|
workspace.checkpoint.return_value = "policy-valid-checkpoint"
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
workspace.restore_checkpoint.assert_called_once_with(None)
|
||||||
|
workspace.checkpoint.assert_called_once()
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
|
||||||
|
|
||||||
|
def test_policy_constraints_accumulate_across_retries() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=4,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="AuftragsKennung 2 und System_created verwenden; obsolet ausfiltern",
|
||||||
|
prompt="Delivery notes by month",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS'",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(
|
||||||
|
mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3
|
||||||
|
))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["incomplete", "test only", "complete"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-4"
|
||||||
|
workspace.summary.side_effect = [
|
||||||
|
ChangeSummary(("config/erp_semantics.php",), 5, 0),
|
||||||
|
ChangeSummary(("tests/Unit/SQLGeneratorBusinessContextTest.php",), 5, 0),
|
||||||
|
ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
20,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
ChangeSummary(
|
||||||
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||||
|
20,
|
||||||
|
1,
|
||||||
|
),
|
||||||
|
]
|
||||||
|
complete = (
|
||||||
|
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0\n"
|
||||||
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
||||||
|
"AND System_created IS NOT NULL AND COALESCE(obsolet, 0) = 0"
|
||||||
|
)
|
||||||
|
workspace.added_text.side_effect = ["AuftragsKennung = 2", "test", complete, complete]
|
||||||
|
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||||
|
workspace.checkpoint.return_value = "valid"
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
third_task = worker.agent.run.call_args_list[2].args[1]
|
||||||
|
assert "All policy constraints from earlier attempts remain mandatory" in third_task
|
||||||
|
assert "one reference SELECT" in third_task
|
||||||
|
assert "changed only tests" in third_task
|
||||||
|
|
||||||
|
|
||||||
|
def test_already_fixed_removal_report_skips_agent_and_dependency_setup(tmp_path) -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=5,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong delivery note count",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior=(
|
||||||
|
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
|
||||||
|
"Einfach nur das aus der SQL entfernen"
|
||||||
|
),
|
||||||
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||||
|
response="0 rows",
|
||||||
|
sql_query=(
|
||||||
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||||
|
"AND a.bStatus_gebucht = 1"
|
||||||
|
),
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
source = tmp_path / "config/erp_semantics.php"
|
||||||
|
source.parent.mkdir(parents=True)
|
||||||
|
source.write_text("""<?php
|
||||||
|
return ['examples' => [[
|
||||||
|
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
||||||
|
'sql' => <<<'SQL'
|
||||||
|
SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
|
||||||
|
SQL,
|
||||||
|
]]];
|
||||||
|
""")
|
||||||
|
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock()
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = tmp_path
|
||||||
|
workspace.branch = "ai/failure-5"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
workspace.prepare.assert_called_once()
|
||||||
|
worker._prepare_workspace_for_agent.assert_not_called()
|
||||||
|
worker.agent.run.assert_not_called()
|
||||||
|
workspace.commit.assert_not_called()
|
||||||
|
workspace.push_branch.assert_not_called()
|
||||||
|
worker.gitea.create_pull_request.assert_not_called()
|
||||||
|
worker.reports.complete.assert_called_once()
|
||||||
|
metadata = worker.reports.complete.call_args.args[1]
|
||||||
|
assert metadata["classification"] == "already_fixed"
|
||||||
|
assert metadata["automation_action"] == "no_source_change"
|
||||||
|
assert metadata["forbidden_fragments"] == ["a.bStatus_gebucht = 1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_identical_policy_failure_stops_after_two_attempts() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=18,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="List generic records",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT missing FROM records",
|
||||||
|
error_message="Unknown column",
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(
|
||||||
|
mode="pr",
|
||||||
|
dry_run=False,
|
||||||
|
gitea_token="token",
|
||||||
|
max_agent_attempts=5,
|
||||||
|
))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["tests only", "still tests only", "must not run"]
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-18"
|
||||||
|
workspace.summary.side_effect = [
|
||||||
|
ChangeSummary(("tests/Unit/FixTest.php",), 2, 0),
|
||||||
|
ChangeSummary(("tests/Unit/FixTest.php",), 3, 0),
|
||||||
|
]
|
||||||
|
workspace.added_text.return_value = ""
|
||||||
|
workspace.diff.return_value = "same rejected candidate"
|
||||||
|
workspace.run_args.return_value = Mock(
|
||||||
|
returncode=0,
|
||||||
|
stdout="No syntax errors",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 2
|
||||||
|
workspace.commit.assert_not_called()
|
||||||
|
worker.reports.fail.assert_called_once()
|
||||||
|
assert "Repeated identical candidate rejection" in worker.reports.fail.call_args.args[1]
|
||||||
|
|
||||||
|
|
||||||
|
def test_same_policy_error_with_changed_candidate_can_progress() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=18,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="automatic",
|
||||||
|
category="sql_execution",
|
||||||
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
|
prompt="List generic records",
|
||||||
|
response=None,
|
||||||
|
sql_query="SELECT missing FROM records",
|
||||||
|
error_message="Unknown column",
|
||||||
|
context={},
|
||||||
|
)
|
||||||
|
worker = AutomationWorker(settings(
|
||||||
|
mode="pr",
|
||||||
|
dry_run=False,
|
||||||
|
gitea_token="token",
|
||||||
|
max_agent_attempts=3,
|
||||||
|
))
|
||||||
|
worker.reports = Mock()
|
||||||
|
worker.reports.claim_next.return_value = report
|
||||||
|
worker.agent = Mock()
|
||||||
|
worker.agent.run.side_effect = ["first tests only", "improved tests only", "complete fix"]
|
||||||
|
worker.gitea = Mock()
|
||||||
|
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||||
|
worker._run_tests = Mock(return_value=[])
|
||||||
|
workspace = Mock()
|
||||||
|
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||||
|
workspace.branch = "ai/failure-18"
|
||||||
|
workspace.summary.side_effect = [
|
||||||
|
ChangeSummary(("tests/Unit/FixTest.php",), 2, 0),
|
||||||
|
ChangeSummary(("tests/Unit/FixTest.php",), 3, 0),
|
||||||
|
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 8, 1),
|
||||||
|
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 8, 1),
|
||||||
|
]
|
||||||
|
workspace.added_text.return_value = ""
|
||||||
|
workspace.diff.side_effect = ["first rejected candidate", "second rejected candidate"]
|
||||||
|
workspace.run_args.return_value = Mock(
|
||||||
|
returncode=0,
|
||||||
|
stdout="No syntax errors",
|
||||||
|
stderr="",
|
||||||
|
)
|
||||||
|
workspace.checkpoint.return_value = "valid"
|
||||||
|
workspace.commit.return_value = "commit-id"
|
||||||
|
|
||||||
|
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||||
|
assert worker.run_once()
|
||||||
|
|
||||||
|
assert worker.agent.run.call_count == 3
|
||||||
|
worker._run_tests.assert_called_once()
|
||||||
|
workspace.push_branch.assert_called_once()
|
||||||
|
worker.reports.complete.assert_called_once()
|
||||||
2629
old_2026-08-31/uv.lock
generated
Normal file
2629
old_2026-08-31/uv.lock
generated
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user