From 582db3a654a1c7828870115cd48f139bf537fd23 Mon Sep 17 00:00:00 2001 From: Christian Meyer Date: Mon, 31 Aug 2026 02:46:58 +0200 Subject: [PATCH 01/12] chore: archive previous Aider worker --- old_2026-08-31/.dockerignore | 8 + old_2026-08-31/.env.example | 29 + old_2026-08-31/.gitea/workflows/ci.yml | 18 + old_2026-08-31/.gitignore | 9 + old_2026-08-31/Dockerfile | 19 + old_2026-08-31/README.md | 202 ++ old_2026-08-31/docker-compose.yml | 57 + old_2026-08-31/pyproject.toml | 32 + old_2026-08-31/src/lai_automation/__init__.py | 1 + old_2026-08-31/src/lai_automation/agent.py | 353 +++ old_2026-08-31/src/lai_automation/cli.py | 36 + old_2026-08-31/src/lai_automation/config.py | 141 + old_2026-08-31/src/lai_automation/database.py | 123 + old_2026-08-31/src/lai_automation/git.py | 234 ++ old_2026-08-31/src/lai_automation/gitea.py | 46 + old_2026-08-31/src/lai_automation/models.py | 41 + old_2026-08-31/src/lai_automation/policy.py | 788 +++++ old_2026-08-31/src/lai_automation/prompt.py | 451 +++ old_2026-08-31/src/lai_automation/removal.py | 307 ++ old_2026-08-31/src/lai_automation/worker.py | 875 ++++++ old_2026-08-31/tests/test_agent.py | 201 ++ old_2026-08-31/tests/test_config.py | 110 + old_2026-08-31/tests/test_database.py | 35 + old_2026-08-31/tests/test_git.py | 239 ++ old_2026-08-31/tests/test_gitea.py | 27 + old_2026-08-31/tests/test_policy.py | 924 ++++++ old_2026-08-31/tests/test_prompt.py | 367 +++ old_2026-08-31/tests/test_removal.py | 145 + old_2026-08-31/tests/test_worker.py | 968 ++++++ old_2026-08-31/uv.lock | 2629 +++++++++++++++++ 30 files changed, 9415 insertions(+) create mode 100644 old_2026-08-31/.dockerignore create mode 100644 old_2026-08-31/.env.example create mode 100644 old_2026-08-31/.gitea/workflows/ci.yml create mode 100644 old_2026-08-31/.gitignore create mode 100644 old_2026-08-31/Dockerfile create mode 100644 old_2026-08-31/README.md create mode 100644 old_2026-08-31/docker-compose.yml create mode 100644 old_2026-08-31/pyproject.toml create mode 100644 old_2026-08-31/src/lai_automation/__init__.py create mode 100644 old_2026-08-31/src/lai_automation/agent.py create mode 100644 old_2026-08-31/src/lai_automation/cli.py create mode 100644 old_2026-08-31/src/lai_automation/config.py create mode 100644 old_2026-08-31/src/lai_automation/database.py create mode 100644 old_2026-08-31/src/lai_automation/git.py create mode 100644 old_2026-08-31/src/lai_automation/gitea.py create mode 100644 old_2026-08-31/src/lai_automation/models.py create mode 100644 old_2026-08-31/src/lai_automation/policy.py create mode 100644 old_2026-08-31/src/lai_automation/prompt.py create mode 100644 old_2026-08-31/src/lai_automation/removal.py create mode 100644 old_2026-08-31/src/lai_automation/worker.py create mode 100644 old_2026-08-31/tests/test_agent.py create mode 100644 old_2026-08-31/tests/test_config.py create mode 100644 old_2026-08-31/tests/test_database.py create mode 100644 old_2026-08-31/tests/test_git.py create mode 100644 old_2026-08-31/tests/test_gitea.py create mode 100644 old_2026-08-31/tests/test_policy.py create mode 100644 old_2026-08-31/tests/test_prompt.py create mode 100644 old_2026-08-31/tests/test_removal.py create mode 100644 old_2026-08-31/tests/test_worker.py create mode 100644 old_2026-08-31/uv.lock diff --git a/old_2026-08-31/.dockerignore b/old_2026-08-31/.dockerignore new file mode 100644 index 0000000..e913f96 --- /dev/null +++ b/old_2026-08-31/.dockerignore @@ -0,0 +1,8 @@ +.git +.env +.venv +.docker-tmp +__pycache__ +.pytest_cache +.ruff_cache +workspaces diff --git a/old_2026-08-31/.env.example b/old_2026-08-31/.env.example new file mode 100644 index 0000000..cf9fb9a --- /dev/null +++ b/old_2026-08-31/.env.example @@ -0,0 +1,29 @@ +FAILURE_DB_HOST=host.docker.internal +FAILURE_DB_PORT=33060 +FAILURE_DB_NAME=laravel_ai_chat +FAILURE_DB_USER=laravel +FAILURE_DB_PASSWORD= + +SOURCE_REPOSITORY=https://gitea.tisinfra.de/christian.meyer/lai.git +SOURCE_BRANCH=main +GITEA_URL=https://gitea.tisinfra.de +GITEA_OWNER=christian.meyer +GITEA_REPOSITORY=lai +GITEA_USERNAME=automation-bot +GITEA_TOKEN= + +# dry-run, pr, direct-main +AUTOMATION_MODE=dry-run +ALLOW_DIRECT_MAIN=false + +AGENT_COMMAND=aider +AGENT_MODEL=mistral/devstral-latest +AGENT_TIMEOUT_SECONDS=900 +AGENT_IDLE_TIMEOUT_SECONDS=300 +MISTRAL_API_KEY= + +POLL_SECONDS=60 +STALE_CLAIM_MINUTES=90 +MAX_CHANGED_FILES=12 +MAX_CHANGED_LINES=800 +TEST_COMMANDS=composer install --no-interaction --prefer-dist;;php artisan test diff --git a/old_2026-08-31/.gitea/workflows/ci.yml b/old_2026-08-31/.gitea/workflows/ci.yml new file mode 100644 index 0000000..58fa6bf --- /dev/null +++ b/old_2026-08-31/.gitea/workflows/ci.yml @@ -0,0 +1,18 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install ".[dev]" + - run: ruff check src tests + - run: pytest -q + - run: docker build -t lai-automation:test . diff --git a/old_2026-08-31/.gitignore b/old_2026-08-31/.gitignore new file mode 100644 index 0000000..a6928d8 --- /dev/null +++ b/old_2026-08-31/.gitignore @@ -0,0 +1,9 @@ +.env +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ +.docker-tmp/ +.docker-buildx/ +*.pyc +workspaces/ diff --git a/old_2026-08-31/Dockerfile b/old_2026-08-31/Dockerfile new file mode 100644 index 0000000..9f0852c --- /dev/null +++ b/old_2026-08-31/Dockerfile @@ -0,0 +1,19 @@ +FROM python:3.12-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + composer git nodejs npm openssh-client php-cli php-curl php-mbstring php-mysql php-xml php-zip unzip \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY pyproject.toml README.md ./ +COPY src ./src +RUN --mount=type=cache,target=/root/.cache/pip \ + pip install --retries 10 --timeout 120 ".[agent]" + +RUN useradd --create-home --uid 10001 coder \ + && mkdir -p /workspaces \ + && chmod 700 /home/coder + +ENTRYPOINT ["lai-agent"] +CMD ["worker"] diff --git a/old_2026-08-31/README.md b/old_2026-08-31/README.md new file mode 100644 index 0000000..0f5b378 --- /dev/null +++ b/old_2026-08-31/README.md @@ -0,0 +1,202 @@ +# LAI Automatisierung + +Kontrollierter Coding-Agent fuer Failure Reports aus +[`lai`](https://gitea.tisinfra.de/christian.meyer/lai). Der Orchestrator verwendet +[Aider](https://aider.chat/) headless als austauschbaren Coding-Kern. Git-, Gitea-, +Policy- und Freigabelogik bleiben in diesem Projekt. + +## Warum Aider? + +- scriptbarer Einmal-Auftrag mit `--message-file` +- gute Git- und Repository-Map-Unterstuetzung +- Mistral ueber LiteLLM nutzbar +- automatische Commits koennen deaktiviert werden +- deutlich kleinerer Betriebsumfang als OpenHands + +OpenHands bleibt eine moegliche spaetere Alternative fuer komplexere, mehrstufige +Agentenlaeufe. Sein Headless-Modus arbeitet jedoch immer mit `always-approve`; deshalb +muessen auch dort unsere externen Policy-Gates erhalten bleiben. + +## Ablauf + +```text +failure_reports(open) + -> transaktionales Claiming (SKIP LOCKED) + -> anonymisierter Agentenauftrag + -> frischer Clone von lai.git + -> Aider-Aenderung ohne Commit + -> Pfad-, Groessen- und Sicherheitsgates + -> Tests + -> Commit + -> PR oder kontrollierter Push auf main + -> failure_reports(resolved/agent_failed) +``` + +Der Coding-Agent erhaelt keinen Zugang zur ERP- oder Kundendatenbank. Rohdaten und +`.env`-Dateien sind ausgeschlossen. Repository-Pushes verwenden `GIT_ASKPASS`; Tokens +werden nicht in Remote-URLs oder Prozessargumenten gespeichert. Der Orchestrator laeuft +als privilegierter Containerprozess, startet Aider aber als separaten Benutzer `coder`. +Dieser Unterprozess erhaelt nur Modell-Credentials, nicht die Failure-DB- oder +Gitea-Zugangsdaten. + +Die Workspace-Setup- und Testprozesse erhalten separat konfigurierte `TEST_DB_*`-Zugangsdaten. +Dieser Benutzer darf ausschliesslich die isolierten Testschemas bearbeiten. Fuer Laravel +werden daraus `DB_*`, `ERP_DB_*` und `COCKPIT_CONFIG_DATABASE` gesetzt; Aider selbst sieht +diese Werte nie. Produktive ERP- oder Kundenzugangsdaten duerfen hier nicht verwendet werden. +Vor Aider und erneut vor der abschliessenden Validierung installiert `SETUP_COMMANDS` die +Abhaengigkeiten, erstellt die Vite-Assets und initialisiert mit +`php artisan migrate:fresh --force` das isolierte Laravel-Schema neu. `TEST_COMMANDS` enthaelt +die eigentlichen Pruefungen. Deshalb muessen alle +`TEST_DB_*`-Schemas dediziert und vollstaendig entbehrlich sein. + +Lokale Ollama-Modelle laufen fuer Produktionsdateien in einem einzelnen Whole-File-Durchgang. +Devstral und andere externe Coding-Modelle verwenden direkt das Diff-Format statt desselben +Modells zusaetzlich als Architect und Editor aufzurufen. Das halbiert Modellrunden fuer kleine +Reparaturen. Bei gezielt gerouteten Dateien ist die zusaetzliche Repo-Map deaktiviert. +Automatische URL-Erkennung und Web-Scraping sind deaktiviert, damit URLs in Anwendungscode oder +Tests niemals als externe Rechercheauftraege interpretiert werden. +`AGENT_TIMEOUT_SECONDS` begrenzt Agentenprozess und Modell-API-Aufruf. +`AGENT_IDLE_TIMEOUT_SECONDS` beendet zusaetzlich einen Agenten, der trotz laufendem Prozess +standardmaessig fuenf Minuten keine Ausgabe mehr erzeugt; vorhandene Teilpatches werden danach +weiterhin validiert. +Setup-Fehler werden als Infrastrukturfehler beendet und nicht als Codekorrektur an das Modell +zurueckgegeben. Hat ein lokales Modell vor einem Timeout bereits Dateien geaendert, durchlaufen +diese Teilresultate trotzdem Policy, Syntaxpruefung und Tests; nur ein Timeout ohne Diff verwirft +den Versuch sofort. + +Laravel-Tests laufen mit PHP und Composer direkt im isolierten Worker. Der Docker-Socket +des Hosts wird nicht eingebunden. Abgebrochene Claims werden nach standardmaessig 90 +Minuten wieder freigegeben; `STALE_CLAIM_MINUTES` passt dieses Intervall an. +Explizite Nur-Entfernen-Auftraege werden als negativer SQL-Vertrag geprueft: Der verlangte +Ausdruck muss aus der passenden Referenzabfrage verschwinden, in einer negativen Test-Assertion +vorkommen, und alle anderen Felder, Joins, Filter, Gruppierungen sowie bestehenden Tests muessen +unveraendert bleiben. Ist der Ausdruck in einer eindeutig zur Benutzerfrage passenden, getrackten +Referenzabfrage bereits entfernt, wird ein veralteter Report ohne Agentenlauf und ohne PR als +`already_fixed` abgeschlossen. + +Schlaegt Policy, Syntax oder isolierte Validierung fehl, erhaelt Aider kontrollierte +Korrekturversuche mit redigierter Diagnose. Syntaxgueltige Teilpatches werden als Checkpoint +erhalten; unerwartete Dateien werden verworfen. Nach einem Testfehler bleiben alle gerouteten +Produktions- und Testdateien sichtbar, damit der Agent die Ursache korrigieren kann, statt nur +Assertions an fehlerhaftes Verhalten anzupassen. Mehrteilige Feedbacks zu Antwortformat und +expliziten Routing-Praefixen werden an ihre jeweiligen Anwendungseigentuemer geroutet und muessen +vollstaendig abgedeckt sein. Zwei identische Policy-Fehler hintereinander brechen frueh ab. +`MAX_AGENT_ATTEMPTS` ist auf 1 bis 7 begrenzt. + +## Schnellstart + +```bash +cp .env.example .env +# FAILURE_DB_PASSWORD und bei Bedarf MISTRAL_API_KEY eintragen +# TEST_DB_PASSWORD fuer den Benutzer der isolierten Testschemas eintragen +docker compose build +docker compose run --rm automation check +docker compose run --rm automation once +``` + +Der Default `AUTOMATION_MODE=dry-run` claimt einen offenen Report, erzeugt den +anonymisierten Auftrag und setzt ihn danach wieder auf `open`. Es wird weder ein Agent +gestartet noch gepusht. + +## Fortschritt beobachten + +```bash +docker compose ps automation +docker compose exec automation lai-agent check +docker compose logs -f automation +``` + +Der Worker protokolliert keine unredigierten Prompts, SQL-Ergebnisse oder Secrets. Aider-Ausgabe +wird ohne Terminalfarben live, redigiert und pro Zeile begrenzt protokolliert; in Report-Metadaten +landet nur ein begrenzter Ausgabetail. Fuer jeden Versuch erscheinen Modell, Edit-Strategie, +Dateien, PID, Laufzeit, Ausgabeaktivitaet und Ergebnisstatus. Zusaetzlich erscheinen +Workspace-Vorbereitung, Policy-Pruefung, jeder Validierungsbefehl, Commit, PR-URL und Cleanup. +Bei laengeren Agenten- oder Testphasen wird alle 30 Sekunden Laufzeitfortschritt ausgegeben und +`updated_at` als Datenbank-Lease erneuert. Nur wirklich verwaiste Claims werden nach +`STALE_CLAIM_MINUTES` wieder freigegeben. + +## PR-Modus + +```dotenv +AUTOMATION_MODE=pr +GITEA_USERNAME=automation-bot +GITEA_TOKEN=... +MISTRAL_API_KEY=... +``` + +Der Bot benoetigt Schreibzugriff auf `christian.meyer/lai` und die Berechtigung, +Branches sowie Pull Requests anzulegen. Jeder Lauf erstellt +`ai/failure-{report_id}` und einen PR gegen `main`. + +## Direkter main-Modus + +```dotenv +AUTOMATION_MODE=direct-main +ALLOW_DIRECT_MAIN=true +GITEA_TOKEN=... +``` + +Der zweite Schalter ist ein Kill-Switch. In diesem Modus sind Migrationen, Routes, +Auth, Middleware, PermissionService und SQLValidatorService zusaetzlich blockiert. +Branch Protection muss den Bot fuer direkte Pushes explizit zulassen. PR-Modus bleibt +die empfohlene Betriebsart. + +## Modelle + +Das Modell wird ausschliesslich ueber `AGENT_MODEL` gewaehlt. Devstral ist der +Default und ein Coding-Agent, nicht das Chat-Modell der Laravel-Anwendung: + +```dotenv +AGENT_MODEL=mistral/devstral-latest +MISTRAL_API_KEY=... +``` + +Codestral verwendet denselben Mistral-Zugang: + +```dotenv +AGENT_MODEL=mistral/codestral-latest +MISTRAL_API_KEY=... +``` + +Qwen 3.5 27B ueber OpenRouter: + +```dotenv +AGENT_MODEL=openrouter/qwen/qwen3.5-27b +OPENROUTER_API_KEY=... +``` + +Oder mit einem lokalen Ollama auf dem Docker-Host: + +```dotenv +AGENT_MODEL=ollama_chat/qwen3.5:27b +OLLAMA_API_BASE=http://host.docker.internal:11434 +``` + +Nach einer Aenderung wird der Worker neu erstellt, damit Compose die neue Umgebung +uebernimmt: + +```bash +docker compose up -d --force-recreate automation +docker compose exec -T automation lai-agent check +``` + +Aider verwendet fuer lokale Produktionsreparaturen `whole` und sonst das praezise +`diff`-Format. Das Chat-Modell und Coding-Modell sind bewusst getrennt. Fuer Coding-Agenten sollte ein +agentisch/coding-optimiertes Modell verwendet werden. Ein Wechsel zu OpenHands oder +einem anderen Backend erfordert einen neuen Adapter neben `AiderAgent`. + +## Tests + +```bash +python -m venv .venv +. .venv/bin/activate +pip install ".[dev]" +ruff check src tests +pytest -q +``` + +## Repositorys + +- Chat: `https://gitea.tisinfra.de/christian.meyer/lai.git` +- Automatisierung: `https://gitea.tisinfra.de/christian.meyer/lai_automatisierung.git` + diff --git a/old_2026-08-31/docker-compose.yml b/old_2026-08-31/docker-compose.yml new file mode 100644 index 0000000..a808f94 --- /dev/null +++ b/old_2026-08-31/docker-compose.yml @@ -0,0 +1,57 @@ +services: + automation: + build: . + restart: unless-stopped + environment: + FAILURE_DB_HOST: ${FAILURE_DB_HOST:-laravel-ai-chat-mysql} + FAILURE_DB_PORT: ${FAILURE_DB_PORT:-3306} + FAILURE_DB_NAME: ${FAILURE_DB_NAME:-laravel_ai_chat} + FAILURE_DB_USER: ${FAILURE_DB_USER:-laravel} + FAILURE_DB_PASSWORD: ${FAILURE_DB_PASSWORD:?FAILURE_DB_PASSWORD must be set} + SOURCE_REPOSITORY: ${SOURCE_REPOSITORY:-https://gitea.tisinfra.de/christian.meyer/lai.git} + SOURCE_BRANCH: ${SOURCE_BRANCH:-main} + AUTOMATION_MODE: ${AUTOMATION_MODE:-dry-run} + ALLOW_DIRECT_MAIN: ${ALLOW_DIRECT_MAIN:-false} + GITEA_URL: ${GITEA_URL:-https://gitea.tisinfra.de} + GITEA_OWNER: ${GITEA_OWNER:-christian.meyer} + GITEA_REPOSITORY: ${GITEA_REPOSITORY:-lai} + GITEA_USERNAME: ${GITEA_USERNAME:-automation-bot} + GITEA_TOKEN: ${GITEA_TOKEN:-} + AGENT_COMMAND: ${AGENT_COMMAND:-aider} + AGENT_MODEL: ${AGENT_MODEL:-mistral/devstral-latest} + AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-900} + AGENT_IDLE_TIMEOUT_SECONDS: ${AGENT_IDLE_TIMEOUT_SECONDS:-300} + MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} + OPENAI_API_KEY: ${OPENAI_API_KEY:-} + OPENAI_API_BASE: ${OPENAI_API_BASE:-} + OPENROUTER_API_KEY: ${OPENROUTER_API_KEY:-} + OLLAMA_API_BASE: ${OLLAMA_API_BASE:-} + POLL_SECONDS: ${POLL_SECONDS:-60} + MAX_AGENT_ATTEMPTS: ${MAX_AGENT_ATTEMPTS:-2} + STALE_CLAIM_MINUTES: ${STALE_CLAIM_MINUTES:-90} + SETUP_COMMANDS: ${SETUP_COMMANDS:-composer install --no-interaction --prefer-dist;;npm ci;;npm run build;;php artisan migrate:fresh --force} + TEST_COMMANDS: ${TEST_COMMANDS:-php artisan test} + TEST_DB_HOST: ${TEST_DB_HOST:-laravel-ai-chat-mysql} + TEST_DB_PORT: ${TEST_DB_PORT:-3306} + TEST_DB_NAME: ${TEST_DB_NAME:-laravel_ai_chat_agent_test} + TEST_DB_USER: ${TEST_DB_USER:-lai_test_runner} + TEST_DB_PASSWORD: ${TEST_DB_PASSWORD:-agent_test_secret} + TEST_CUSTOMER_DB: ${TEST_CUSTOMER_DB:-customer_agent_test} + TEST_CONFIG_DB: ${TEST_CONFIG_DB:-cockpit_agent_test} + extra_hosts: + - host.docker.internal:host-gateway + volumes: + - automation-workspaces:/workspaces + networks: + - automation + - lai_laravel_network + +volumes: + automation-workspaces: + +networks: + automation: + driver: bridge + lai_laravel_network: + external: true + name: lai_laravel-network diff --git a/old_2026-08-31/pyproject.toml b/old_2026-08-31/pyproject.toml new file mode 100644 index 0000000..4c6a342 --- /dev/null +++ b/old_2026-08-31/pyproject.toml @@ -0,0 +1,32 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "lai-automation" +version = "0.1.0" +description = "Controlled coding-agent automation for Laravel AI failure reports" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "PyMySQL>=1.1,<2", + "requests>=2.32,<3", +] + +[project.optional-dependencies] +dev = ["pytest>=8.3,<9", "ruff>=0.9,<1"] +agent = ["aider-chat>=0.86,<1"] + +[project.scripts] +lai-agent = "lai_automation.cli:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/lai_automation"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] + +[tool.ruff] +line-length = 100 +target-version = "py312" diff --git a/old_2026-08-31/src/lai_automation/__init__.py b/old_2026-08-31/src/lai_automation/__init__.py new file mode 100644 index 0000000..3dc1f76 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/__init__.py @@ -0,0 +1 @@ +__version__ = "0.1.0" diff --git a/old_2026-08-31/src/lai_automation/agent.py b/old_2026-08-31/src/lai_automation/agent.py new file mode 100644 index 0000000..a68c042 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/agent.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +import logging +import os +import pwd +import re +import shutil +import signal +import subprocess +import threading +import time +from collections import deque +from pathlib import Path + +from .config import Settings +from .prompt import redact + +LOG = logging.getLogger(__name__) +_ANSI_ESCAPE = re.compile(r"\x1b\[[0-?]*[ -/]*[@-~]") + + +class AgentTimeoutError(RuntimeError): + pass + + +class AgentProviderError(RuntimeError): + pass + + +_PROVIDER_FAILURE = re.compile( + r"(?:litellm\.(?:InternalServerError|APIConnectionError|RateLimitError)|" + r"MistralException|The API provider's servers are down or overloaded|" + r"Name or service not known|Temporary failure in name resolution|" + r"Connection refused|Connection timed out)", + re.IGNORECASE, +) + + +class _OutputCapture: + def __init__(self, secret_values: tuple[str, ...] = (), limit: int = 24_000) -> None: + self.secret_values = tuple(value for value in secret_values if len(value) >= 8) + self.limit = limit + self.lines: deque[tuple[str, str]] = deque() + self.characters = 0 + self.line_count = 0 + self.last_activity = time.monotonic() + self.lock = threading.Lock() + + def add(self, stream_name: str, value: str) -> str: + safe_value = redact(_ANSI_ESCAPE.sub("", value.rstrip("\r\n")))[:4_000] + for secret_value in self.secret_values: + safe_value = safe_value.replace(secret_value, "[REDACTED]") + with self.lock: + self.lines.append((stream_name, safe_value)) + self.characters += len(safe_value) + self.line_count += 1 + self.last_activity = time.monotonic() + while self.lines and self.characters > self.limit: + _, removed = self.lines.popleft() + self.characters -= len(removed) + return safe_value + + def tail(self, stream_name: str | None = None, limit: int = 12_000) -> str: + with self.lock: + values = [ + value + for source, value in self.lines + if stream_name is None or source == stream_name + ] + result = "\n".join(values) + return result[-limit:] + + def activity(self) -> tuple[int, float]: + with self.lock: + return self.line_count, time.monotonic() - self.last_activity + + +class AiderAgent: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.last_run_metadata: dict = {} + + def run( + self, + workspace: Path, + task: str, + files: tuple[str, ...] = (), + log_context: str = "Agent", + ) -> str: + task_file = workspace / ".git" / "failure-task.txt" + task_file.write_text(task) + task_file.chmod(0o600) + command = self.command(task_file, files) + coder = pwd.getpwnam("coder") + git_config = Path(coder.pw_dir) / ".gitconfig" + git_config.write_text( + "[user]\n\tname = LAI Automation Coder\n\temail = coder@localhost\n" + f"[safe]\n\tdirectory = {workspace.resolve()}\n" + ) + git_config.chmod(0o600) + os.chown(git_config, coder.pw_uid, coder.pw_gid) + os.chown(task_file, coder.pw_uid, coder.pw_gid) + for path in workspace.rglob("*"): + if ".git" not in path.parts: + os.chown(path, coder.pw_uid, coder.pw_gid, follow_symlinks=False) + os.chown(workspace, coder.pw_uid, coder.pw_gid, follow_symlinks=False) + + environment = { + "HOME": coder.pw_dir, + "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), + } + for name in ( + "MISTRAL_API_KEY", + "OPENAI_API_KEY", + "OPENAI_API_BASE", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + "OLLAMA_API_BASE", + ): + if os.getenv(name): + environment[name] = os.environ[name] + + capture = _OutputCapture(tuple( + environment[name] + for name in ( + "MISTRAL_API_KEY", + "OPENAI_API_KEY", + "ANTHROPIC_API_KEY", + "OPENROUTER_API_KEY", + ) + if environment.get(name) + )) + process: subprocess.Popen[str] | None = None + readers: list[threading.Thread] = [] + monitor_stopped = threading.Event() + idle_timed_out = threading.Event() + started = time.monotonic() + self.last_run_metadata = { + "model": self.settings.agent_model, + "strategy": self.edit_strategy(files), + "editable_files": list(files), + "status": "starting", + } + + def read_stream(stream, stream_name: str) -> None: + try: + for line in iter(stream.readline, ""): + safe_line = capture.add(stream_name, line) + if safe_line.strip(): + LOG.info( + "%s aider_%s %s", + log_context, + stream_name, + safe_line[:1_000], + ) + finally: + stream.close() + + def monitor() -> None: + while not monitor_stopped.wait(30): + lines, idle_seconds = capture.activity() + LOG.info( + "%s Aider pid=%s running elapsed=%ss output_lines=%s output_idle=%ss", + log_context, + process.pid if process else "pending", + round(time.monotonic() - started), + lines, + round(idle_seconds), + ) + if process and self._output_stalled(idle_seconds): + idle_timed_out.set() + LOG.warning( + "%s Aider pid=%s produced no output for %ss; terminating process group", + log_context, + process.pid, + round(idle_seconds), + ) + self._terminate_process_group(process) + return + + monitor_thread = threading.Thread(target=monitor, daemon=True) + monitor_thread.start() + try: + try: + process = subprocess.Popen( + command, + cwd=workspace, + env=environment, + text=True, + errors="replace", + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + bufsize=1, + start_new_session=True, + user=coder.pw_uid, + group=coder.pw_gid, + extra_groups=(), + ) + self.last_run_metadata.update({ + "pid": process.pid, + "status": "running", + }) + LOG.info( + "%s Aider started pid=%s model=%s strategy=%s files=%s", + log_context, + process.pid, + self.settings.agent_model, + self.edit_strategy(files), + ",".join(files) or "repository-map", + ) + assert process.stdout is not None + assert process.stderr is not None + readers = [ + threading.Thread( + target=read_stream, + args=(process.stdout, "stdout"), + daemon=True, + ), + threading.Thread( + target=read_stream, + args=(process.stderr, "stderr"), + daemon=True, + ), + ] + for reader in readers: + reader.start() + process.wait(timeout=self.settings.agent_timeout_seconds + 60) + if idle_timed_out.is_set(): + self.last_run_metadata["status"] = "idle_timeout" + output = capture.tail(limit=4_000) + detail = f"\nLast agent output:\n{output}" if output else "" + raise AgentTimeoutError( + "Aider produced no output for " + f"{self.settings.agent_idle_timeout_seconds} seconds.{detail}" + ) + except subprocess.TimeoutExpired as exception: + self.last_run_metadata["status"] = "timeout" + if process: + LOG.warning( + "%s Aider pid=%s exceeded process timeout; terminating process group", + log_context, + process.pid, + ) + self._terminate_process_group(process) + for reader in readers: + reader.join(timeout=2) + output = capture.tail(limit=4_000) + detail = f"\nLast agent output:\n{output}" if output else "" + raise AgentTimeoutError( + f"Aider exceeded the {self.settings.agent_timeout_seconds + 60}-second process " + f"timeout.{detail}" + ) from exception + finally: + monitor_stopped.set() + monitor_thread.join(timeout=1) + for reader in readers: + reader.join(timeout=1) + lines, idle_seconds = capture.activity() + self.last_run_metadata.update({ + "elapsed_seconds": round(time.monotonic() - started, 1), + "output_lines": lines, + "output_idle_seconds": round(idle_seconds, 1), + "output_tail": capture.tail(limit=4_000), + }) + shutil.rmtree(workspace / ".aider.tags.cache.v4", ignore_errors=True) + for artifact in (".aider.chat.history.md", ".aider.input.history"): + (workspace / artifact).unlink(missing_ok=True) + os.chown(workspace, 0, coder.pw_gid, follow_symlinks=False) + workspace.chmod(0o775) + assert process is not None + if process.returncode != 0: + self.last_run_metadata["status"] = "failed" + raise RuntimeError( + f"Aider failed ({process.returncode}): {capture.tail(limit=4_000)}" + ) + output = capture.tail() + if _PROVIDER_FAILURE.search(output): + self.last_run_metadata["status"] = "provider_error" + raise AgentProviderError( + "Model provider failed before producing a usable repair. Last provider output:\n" + + capture.tail(limit=4_000) + ) + self.last_run_metadata["status"] = "completed" + return output + + @staticmethod + def _terminate_process_group(process: subprocess.Popen[str]) -> None: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + return + except subprocess.TimeoutExpired: + pass + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + return + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + LOG.error("Aider process group %s did not exit after SIGKILL", process.pid) + + def edit_strategy(self, files: tuple[str, ...]) -> str: + if self.settings.agent_model.startswith("ollama") and any( + not filename.startswith("tests/") for filename in files + ): + return "whole" + return "diff" + + def _output_stalled(self, idle_seconds: float) -> bool: + return idle_seconds >= self.settings.agent_idle_timeout_seconds + + def command(self, task_file: Path, files: tuple[str, ...] = ()) -> list[str]: + command = [ + self.settings.agent_command, + "--model", + self.settings.agent_model, + "--message-file", + str(task_file), + "--yes", + "--no-auto-commits", + "--no-dirty-commits", + "--no-git", + "--stream", + "--no-pretty", + "--no-gitignore", + "--no-check-update", + "--no-show-release-notes", + "--no-suggest-shell-commands", + "--no-detect-urls", + "--disable-playwright", + "--chat-history-file", + "/dev/null", + "--input-history-file", + "/dev/null", + "--no-analytics", + "--map-tokens", + "0" if files else "2048", + "--no-auto-lint", + "--timeout", + str(self.settings.agent_timeout_seconds), + ] + for filename in files: + if (task_file.parent.parent / filename).is_file(): + command.extend(["--file", filename]) + if self.settings.agent_model.startswith("ollama"): + command.extend(["--edit-format", self.edit_strategy(files)]) + elif files: + command.extend(["--edit-format", "diff"]) + return command diff --git a/old_2026-08-31/src/lai_automation/cli.py b/old_2026-08-31/src/lai_automation/cli.py new file mode 100644 index 0000000..bff5253 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/cli.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +import argparse +import logging + +from .config import Settings +from .worker import AutomationWorker + + +def main() -> None: + parser = argparse.ArgumentParser(description="LAI controlled coding-agent automation") + parser.add_argument("command", choices=["once", "worker", "check"]) + args = parser.parse_args() + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + settings = Settings.from_env() + + if args.command == "check": + print(f"mode={settings.mode}") + print(f"source={settings.source_repository}") + print(f"model={settings.agent_model}") + print(f"direct_main_enabled={settings.allow_direct_main}") + return + + worker = AutomationWorker(settings) + if args.command == "once": + worker.run_once() + else: + worker.run_forever() + + +if __name__ == "__main__": + main() diff --git a/old_2026-08-31/src/lai_automation/config.py b/old_2026-08-31/src/lai_automation/config.py new file mode 100644 index 0000000..543ebba --- /dev/null +++ b/old_2026-08-31/src/lai_automation/config.py @@ -0,0 +1,141 @@ +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +_DEFAULT_SETUP_COMMANDS = ( + "composer install --no-interaction --prefer-dist;;" + "npm ci;;" + "npm run build;;" + "php artisan migrate:fresh --force" +) + + +def _bool(name: str, default: bool = False) -> bool: + return os.getenv(name, str(default)).lower() in {"1", "true", "yes", "on"} + + +def _commands(value: str) -> tuple[str, ...]: + return tuple(command.strip() for command in value.split(";;") if command.strip()) + + +@dataclass(frozen=True) +class Settings: + database_host: str + database_port: int + database_name: str + database_user: str + database_password: str + source_repository: str + source_branch: str + workspace_root: Path + mode: str + allow_direct_main: bool + gitea_url: str + gitea_owner: str + gitea_repository: str + gitea_token: str + gitea_username: str + agent_command: str + agent_model: str + agent_timeout_seconds: int + agent_idle_timeout_seconds: int + setup_commands: tuple[str, ...] + test_commands: tuple[str, ...] + test_database_host: str + test_database_port: int + test_database_name: str + test_database_user: str + test_database_password: str + test_customer_database: str + test_config_database: str + poll_seconds: int + max_changed_files: int + max_changed_lines: int + max_agent_attempts: int + stale_claim_minutes: int + dry_run: bool + + @classmethod + def from_env(cls) -> Settings: + mode = os.getenv("AUTOMATION_MODE", "dry-run").lower() + if mode not in {"dry-run", "pr", "direct-main"}: + raise ValueError("AUTOMATION_MODE must be dry-run, pr, or direct-main") + + settings = cls( + database_host=os.getenv("FAILURE_DB_HOST", "host.docker.internal"), + database_port=int(os.getenv("FAILURE_DB_PORT", "33060")), + database_name=os.getenv("FAILURE_DB_NAME", "laravel_ai_chat"), + database_user=os.getenv("FAILURE_DB_USER", "laravel"), + database_password=os.getenv("FAILURE_DB_PASSWORD", ""), + source_repository=os.getenv( + "SOURCE_REPOSITORY", "https://gitea.tisinfra.de/christian.meyer/lai.git" + ), + source_branch=os.getenv("SOURCE_BRANCH", "main"), + workspace_root=Path(os.getenv("WORKSPACE_ROOT", "/workspaces")), + mode=mode, + allow_direct_main=_bool("ALLOW_DIRECT_MAIN"), + gitea_url=os.getenv("GITEA_URL", "https://gitea.tisinfra.de"), + gitea_owner=os.getenv("GITEA_OWNER", "christian.meyer"), + gitea_repository=os.getenv("GITEA_REPOSITORY", "lai"), + gitea_token=os.getenv("GITEA_TOKEN", ""), + gitea_username=os.getenv("GITEA_USERNAME", "automation-bot"), + agent_command=os.getenv("AGENT_COMMAND", "aider"), + agent_model=os.getenv("AGENT_MODEL", "mistral/devstral-latest"), + agent_timeout_seconds=int(os.getenv("AGENT_TIMEOUT_SECONDS", "900")), + agent_idle_timeout_seconds=int(os.getenv("AGENT_IDLE_TIMEOUT_SECONDS", "300")), + setup_commands=_commands( + os.getenv("SETUP_COMMANDS", _DEFAULT_SETUP_COMMANDS) + ), + test_commands=_commands( + os.getenv("TEST_COMMANDS", "php artisan test") + ), + test_database_host=os.getenv("TEST_DB_HOST", "host.docker.internal"), + test_database_port=int(os.getenv("TEST_DB_PORT", "33060")), + test_database_name=os.getenv("TEST_DB_NAME", "laravel_ai_chat_agent_test"), + test_database_user=os.getenv("TEST_DB_USER", "lai_test_runner"), + test_database_password=os.getenv("TEST_DB_PASSWORD", ""), + test_customer_database=os.getenv("TEST_CUSTOMER_DB", "customer_agent_test"), + test_config_database=os.getenv("TEST_CONFIG_DB", "cockpit_agent_test"), + poll_seconds=int(os.getenv("POLL_SECONDS", "60")), + max_changed_files=int(os.getenv("MAX_CHANGED_FILES", "12")), + max_changed_lines=int(os.getenv("MAX_CHANGED_LINES", "800")), + max_agent_attempts=int(os.getenv("MAX_AGENT_ATTEMPTS", "2")), + stale_claim_minutes=int(os.getenv("STALE_CLAIM_MINUTES", "90")), + dry_run=mode == "dry-run", + ) + settings.validate() + return settings + + def validate(self) -> None: + if self.mode == "direct-main" and not self.allow_direct_main: + raise ValueError("direct-main requires ALLOW_DIRECT_MAIN=true") + if self.mode in {"pr", "direct-main"} and not self.gitea_token: + raise ValueError(f"{self.mode} requires GITEA_TOKEN") + if not self.database_password: + raise ValueError("FAILURE_DB_PASSWORD is required") + if not self.test_database_password: + raise ValueError("TEST_DB_PASSWORD is required") + if not self.test_commands: + raise ValueError("TEST_COMMANDS must contain at least one command") + test_databases = { + "TEST_DB_NAME": self.test_database_name, + "TEST_CUSTOMER_DB": self.test_customer_database, + "TEST_CONFIG_DB": self.test_config_database, + } + for setting_name, database_name in test_databases.items(): + if "test" not in database_name.lower(): + raise ValueError(f"{setting_name} must name an isolated test database") + if database_name == self.database_name: + raise ValueError(f"{setting_name} must differ from FAILURE_DB_NAME") + if len(set(test_databases.values())) != len(test_databases): + raise ValueError("TEST_DB_NAME, TEST_CUSTOMER_DB, and TEST_CONFIG_DB must be distinct") + if not 60 <= self.agent_timeout_seconds <= 3600: + raise ValueError("AGENT_TIMEOUT_SECONDS must be between 60 and 3600") + if not 60 <= self.agent_idle_timeout_seconds <= self.agent_timeout_seconds: + raise ValueError( + "AGENT_IDLE_TIMEOUT_SECONDS must be between 60 and AGENT_TIMEOUT_SECONDS" + ) + if not 1 <= self.max_agent_attempts <= 7: + raise ValueError("MAX_AGENT_ATTEMPTS must be between 1 and 7") diff --git a/old_2026-08-31/src/lai_automation/database.py b/old_2026-08-31/src/lai_automation/database.py new file mode 100644 index 0000000..80011f9 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/database.py @@ -0,0 +1,123 @@ +from __future__ import annotations + +import json +from collections.abc import Iterator +from contextlib import contextmanager + +import pymysql +from pymysql.cursors import DictCursor + +from .config import Settings +from .models import FailureReport + + +class FailureRepository: + def __init__(self, settings: Settings) -> None: + self.settings = settings + + @contextmanager + def connection(self) -> Iterator[pymysql.Connection]: + connection = pymysql.connect( + host=self.settings.database_host, + port=self.settings.database_port, + user=self.settings.database_user, + password=self.settings.database_password, + database=self.settings.database_name, + cursorclass=DictCursor, + autocommit=False, + connect_timeout=10, + read_timeout=30, + write_timeout=30, + ) + try: + yield connection + finally: + connection.close() + + def claim_next(self) -> FailureReport | None: + with self.connection() as connection, connection.cursor() as cursor: + try: + cursor.execute( + """ + UPDATE failure_reports + SET status = 'open', updated_at = NOW() + WHERE status = 'agent_running' + AND updated_at < DATE_SUB(NOW(), INTERVAL %s MINUTE) + """, + (self.settings.stale_claim_minutes,), + ) + connection.commit() + cursor.execute("SET TRANSACTION ISOLATION LEVEL READ COMMITTED") + cursor.execute("START TRANSACTION") + except Exception: + connection.rollback() + raise + cursor.execute( + """ + SELECT * FROM failure_reports + WHERE status = 'open' + ORDER BY id + LIMIT 1 + FOR UPDATE SKIP LOCKED + """ + ) + row = cursor.fetchone() + if not row: + connection.commit() + return None + cursor.execute( + "UPDATE failure_reports SET status = 'agent_running', updated_at = NOW() WHERE id = %s", + (row["id"],), + ) + connection.commit() + return FailureReport.from_row(row) + + def release(self, report_id: int, reason: str) -> None: + self._finish(report_id, "open", {"last_dry_run": reason}) + + def complete(self, report_id: int, metadata: dict) -> None: + self._finish(report_id, "resolved", {**metadata, "agent_error": None}) + + def fail(self, report_id: int, error: str, metadata: dict | None = None) -> None: + payload = {"agent_error": error, **(metadata or {})} + self._finish(report_id, "agent_failed", payload) + + def heartbeat(self, report_id: int) -> None: + with self.connection() as connection, connection.cursor() as cursor: + try: + cursor.execute( + """ + UPDATE failure_reports + SET updated_at = NOW() + WHERE id = %s AND status = 'agent_running' + """, + (report_id,), + ) + connection.commit() + except Exception: + connection.rollback() + raise + + def _finish(self, report_id: int, status: str, additions: dict) -> None: + with self.connection() as connection, connection.cursor() as cursor: + try: + cursor.execute("SELECT context FROM failure_reports WHERE id = %s", (report_id,)) + row = cursor.fetchone() or {} + context = row.get("context") or {} + if isinstance(context, str): + context = json.loads(context) + context.update(additions) + cursor.execute( + """ + UPDATE failure_reports + SET status = %s, context = %s, updated_at = NOW() + WHERE id = %s + """, + (status, json.dumps(context, ensure_ascii=True), report_id), + ) + connection.commit() + except Exception as exception: + connection.rollback() + raise RuntimeError( + f"Could not finalize failure report {report_id} as {status}" + ) from exception diff --git a/old_2026-08-31/src/lai_automation/git.py b/old_2026-08-31/src/lai_automation/git.py new file mode 100644 index 0000000..284cf7a --- /dev/null +++ b/old_2026-08-31/src/lai_automation/git.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import os +import pwd +import shlex +import shutil +import subprocess +import tempfile +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from urllib.parse import urlparse + +from .config import Settings + + +@dataclass(frozen=True) +class ChangeSummary: + files: tuple[str, ...] + added_lines: int + deleted_lines: int + + @property + def changed_lines(self) -> int: + return self.added_lines + self.deleted_lines + + +class GitWorkspace: + def __init__(self, settings: Settings, report_id: int) -> None: + self.settings = settings + self.path = settings.workspace_root / f"failure-{report_id}" + self.branch = f"ai/failure-{report_id}" + + def prepare(self) -> None: + repository = urlparse(self.settings.source_repository) + gitea = urlparse(self.settings.gitea_url) + if repository.scheme != "https" or repository.hostname != gitea.hostname: + raise ValueError("SOURCE_REPOSITORY must use HTTPS on the configured Gitea host") + shutil.rmtree(self.path, ignore_errors=True) + self.path.parent.mkdir(parents=True, exist_ok=True) + clone = self._git_authenticated if self.settings.gitea_token else self._git + try: + clone( + "clone", + "--branch", + self.settings.source_branch, + "--single-branch", + self.settings.source_repository, + str(self.path), + cwd=self.path.parent, + ) + except subprocess.CalledProcessError as exception: + output = "\n".join(filter(None, (exception.stdout, exception.stderr))).strip() + if len(output) > 4_000: + output = output[-4_000:] + raise RuntimeError( + "Source repository clone failed" + + (f":\n{output}" if output else " without diagnostic output") + ) from exception + self._git("config", "user.name", "LAI Automation") + self._git("config", "user.email", "automation@localhost") + self._git("checkout", "-b", self.branch) + + def run( + self, + command: str, + check: bool = True, + username: str | None = None, + extra_environment: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + return self.run_args( + shlex.split(command), + check=check, + username=username, + extra_environment=extra_environment, + ) + + def run_args( + self, + command: Sequence[str], + check: bool = True, + username: str | None = None, + extra_environment: Mapping[str, str] | None = None, + ) -> subprocess.CompletedProcess[str]: + environment = dict(os.environ) + launch_identity = {} + if username: + account = pwd.getpwnam(username) + environment = { + "HOME": account.pw_dir, + "PATH": os.environ.get("PATH", "/usr/local/bin:/usr/bin:/bin"), + } + launch_identity = { + "user": account.pw_uid, + "group": account.pw_gid, + "extra_groups": (), + } + + environment.update(extra_environment or {}) + + return subprocess.run( + command, + cwd=self.path, + env=environment, + text=True, + capture_output=True, + check=check, + timeout=1800, + **launch_identity, + ) + + def discard_changes(self) -> None: + self._git("reset", "--hard", "HEAD") + self._git("clean", "-fd") + + def checkpoint(self) -> str: + self._git("add", "--all") + return self._git("stash", "create").stdout.strip() + + def restore_checkpoint(self, checkpoint: str | None) -> None: + self.discard_changes() + if checkpoint: + self._git("stash", "apply", checkpoint) + + def summary(self) -> ChangeSummary: + self._git("add", "-N", "--all") + try: + output = self._git("diff", "HEAD", "--numstat").stdout + except subprocess.CalledProcessError: + output = self._git("diff", "--numstat").stdout + files: list[str] = [] + added = deleted = 0 + for line in output.splitlines(): + add, remove, filename = line.split("\t", 2) + files.append(filename) + added += int(add) if add.isdigit() else 0 + deleted += int(remove) if remove.isdigit() else 0 + return ChangeSummary(tuple(files), added, deleted) + + def diff(self, limit: int = 12_000) -> str: + try: + output = self._git("diff", "HEAD", "--no-ext-diff", "--unified=3").stdout + except subprocess.CalledProcessError: + output = self._git("diff", "--no-ext-diff", "--unified=3").stdout + if len(output) <= limit: + return output + half = limit // 2 + return output[:half] + "\n...[diff middle omitted]...\n" + output[-half:] + + def added_text(self) -> str: + output = self._git("diff", "HEAD", "--no-ext-diff", "--unified=0").stdout + return "\n".join( + line[1:] + for line in output.splitlines() + if line.startswith("+") and not line.startswith("+++") + ) + + def head_file(self, filename: str) -> str | None: + result = subprocess.run( + ["git", "show", f"HEAD:{filename}"], + cwd=self.path, + text=True, + capture_output=True, + check=False, + timeout=300, + ) + return result.stdout if result.returncode == 0 else None + + def commit(self, report_id: int) -> str: + self._git("add", "--all") + self._git("commit", "-m", f"Fix failure report #{report_id}") + return self._git("rev-parse", "HEAD").stdout.strip() + + def push_branch(self) -> None: + self._git_authenticated("push", "origin", f"HEAD:{self.branch}") + + def push_main(self) -> None: + self._git_authenticated("push", "origin", f"HEAD:{self.settings.source_branch}") + + def cleanup(self) -> None: + if self.path.exists(): + shutil.rmtree(self.path) + + def _git(self, *args: str, cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["git", *args], + cwd=cwd or self.path, + text=True, + capture_output=True, + check=True, + timeout=300, + ) + + def _git_authenticated( + self, + *args: str, + cwd: Path | None = None, + ) -> subprocess.CompletedProcess[str]: + working_directory = cwd or self.path + auth_directory = Path(tempfile.mkdtemp( + prefix=".lai-git-auth-", + dir=working_directory, + )) + askpass = auth_directory / "askpass.sh" + username_file = auth_directory / "username" + token_file = auth_directory / "token" + try: + username_file.write_text(self.settings.gitea_username) + token_file.write_text(self.settings.gitea_token) + username_file.chmod(0o600) + token_file.chmod(0o600) + askpass.write_text( + "#!/bin/sh\ncase \"$1\" in *Username*) cat \"$GIT_BOT_USER_FILE\";; *) cat \"$GIT_BOT_TOKEN_FILE\";; esac\n" + ) + askpass.chmod(0o700) + env = { + "HOME": os.environ.get("HOME", "/root"), + "PATH": os.environ.get("PATH", "/usr/bin:/bin"), + "GIT_ASKPASS": str(askpass), + "GIT_TERMINAL_PROMPT": "0", + "GIT_BOT_USER_FILE": str(username_file), + "GIT_BOT_TOKEN_FILE": str(token_file), + } + return subprocess.run( + ["git", *args], + cwd=working_directory, + env=env, + text=True, + capture_output=True, + check=True, + timeout=300, + ) + finally: + shutil.rmtree(auth_directory, ignore_errors=True) diff --git a/old_2026-08-31/src/lai_automation/gitea.py b/old_2026-08-31/src/lai_automation/gitea.py new file mode 100644 index 0000000..02e037f --- /dev/null +++ b/old_2026-08-31/src/lai_automation/gitea.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import requests +from requests.adapters import HTTPAdapter +from urllib3.util.retry import Retry + +from .config import Settings + + +class GiteaClient: + def __init__(self, settings: Settings) -> None: + self.settings = settings + self.session = requests.Session() + self.session.headers.update({ + "Authorization": f"token {settings.gitea_token}", + "Accept": "application/json", + }) + retry = Retry( + total=3, + backoff_factor=1, + status_forcelist=(429, 500, 502, 503, 504), + allowed_methods=("POST",), + ) + self.session.mount("https://", HTTPAdapter(max_retries=retry)) + + def create_pull_request(self, branch: str, title: str, body: str) -> str: + url = ( + f"{self.settings.gitea_url.rstrip('/')}/api/v1/repos/" + f"{self.settings.gitea_owner}/{self.settings.gitea_repository}/pulls" + ) + response = self.session.post( + url, + json={ + "base": self.settings.source_branch, + "head": branch, + "title": title, + "body": body, + }, + timeout=30, + ) + response.raise_for_status() + payload = response.json() + pull_request_url = payload.get("html_url") or payload.get("url") + if not pull_request_url: + raise ValueError("Gitea response did not contain a pull request URL") + return pull_request_url diff --git a/old_2026-08-31/src/lai_automation/models.py b/old_2026-08-31/src/lai_automation/models.py new file mode 100644 index 0000000..4dfc726 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/models.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class FailureReport: + id: int + chat_message_id: int | None + source: str + category: str + user_feedback: str | None + feedback_type: str | None + expected_behavior: str | None + prompt: str | None + response: str | None + sql_query: str | None + error_message: str | None + context: dict[str, Any] + + @classmethod + def from_row(cls, row: dict[str, Any]) -> FailureReport: + context = row.get("context") or {} + if isinstance(context, str): + context = json.loads(context) + return cls( + id=row["id"], + chat_message_id=row.get("chat_message_id"), + source=row["source"], + category=row["category"], + user_feedback=row.get("user_feedback"), + feedback_type=row.get("feedback_type"), + expected_behavior=row.get("expected_behavior"), + prompt=row.get("prompt"), + response=row.get("response"), + sql_query=row.get("sql_query"), + error_message=row.get("error_message"), + context=context, + ) diff --git a/old_2026-08-31/src/lai_automation/policy.py b/old_2026-08-31/src/lai_automation/policy.py new file mode 100644 index 0000000..29024df --- /dev/null +++ b/old_2026-08-31/src/lai_automation/policy.py @@ -0,0 +1,788 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from difflib import SequenceMatcher +from pathlib import Path, PurePosixPath + +from .config import Settings +from .git import ChangeSummary, GitWorkspace +from .models import FailureReport +from .prompt import ( + php_negative_assertion, + requires_explicit_route_repair, + requires_table_output_repair, +) +from .removal import ( + find_matching_reference, + is_removal_only, + normalize_source_layout, + normalize_sql, + removal_directives, + remove_sql_fragment, + sql_contains_fragment, + without_reference_sql, +) + +_ALWAYS_DENIED = ( + ".env", + "vendor/", + "storage/", + "bootstrap/cache/", + "docker/", + "docker-compose", + ".gitea/", + ".github/", + ".aider", +) + +_DIRECT_MAIN_DENIED = ( + "database/migrations/", + "app/Http/Middleware/", + "app/Models/User.php", + "app/Services/PermissionService.php", + "app/Services/SQLValidatorService.php", + "config/auth.php", + "config/database.php", + "routes/", +) + + +@dataclass(frozen=True) +class PolicyResult: + allowed: bool + errors: tuple[str, ...] + + +def evaluate_changes( + summary: ChangeSummary, + settings: Settings, + workspace: Path | None = None, +) -> PolicyResult: + errors: list[str] = [] + if not summary.files: + errors.append("Agent produced no file changes") + elif all(filename.startswith("tests/") for filename in summary.files): + errors.append("Agent changed only tests; production behavior was not fixed") + if len(summary.files) > settings.max_changed_files: + errors.append(f"Changed {len(summary.files)} files; limit is {settings.max_changed_files}") + if summary.changed_lines > settings.max_changed_lines: + errors.append(f"Changed {summary.changed_lines} lines; limit is {settings.max_changed_lines}") + + for filename in summary.files: + if PurePosixPath(filename).is_absolute() or ".." in PurePosixPath(filename).parts: + errors.append(f"Invalid changed path: {filename}") + continue + if workspace and (workspace / filename).is_symlink(): + errors.append(f"Symbolic links are not allowed: {filename}") + continue + if any(filename == prefix or filename.startswith(prefix) for prefix in _ALWAYS_DENIED): + errors.append(f"Denied path changed: {filename}") + if settings.mode == "direct-main" and any( + filename == prefix or filename.startswith(prefix) for prefix in _DIRECT_MAIN_DENIED + ): + errors.append(f"Direct-main denied path changed: {filename}") + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_report_scope(report: FailureReport, summary: ChangeSummary) -> PolicyResult: + expected = f"{report.user_feedback or ''}\n{report.expected_behavior or ''}".lower() + requires_cross_database_orchestration = ( + "cockpit_config" in expected and ("fk_" in expected or "erp" in expected) + ) + has_application_change = any(filename.startswith("app/") for filename in summary.files) + errors = [] + if requires_cross_database_orchestration and not has_application_change: + errors.append( + "Cross-database report changed no application code; inspect routing and implement staged orchestration" + ) + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_routed_files( + summary: ChangeSummary, + routed_files: tuple[str, ...], +) -> PolicyResult: + if not routed_files: + return PolicyResult(True, ()) + unexpected = sorted(set(summary.files) - set(routed_files)) + if not unexpected: + return PolicyResult(True, ()) + return PolicyResult(False, ( + "Agent changed files outside the routed repair scope: " + ", ".join(unexpected), + )) + + +def evaluate_requested_behavior_scope( + report: FailureReport, + summary: ChangeSummary, + added_text: str, +) -> PolicyResult: + errors: list[str] = [] + changed_files = set(summary.files) + added_lower = added_text.casefold() + + if requires_table_output_repair(report): + required_files = { + "app/Services/AIService.php", + "tests/Feature/AIPrivacyBoundaryTest.php", + } + missing_files = sorted(required_files - changed_files) + if missing_files: + errors.append( + "Table-presentation requirement must change the real answer formatter and its " + "provider-backed regression test; missing: " + ", ".join(missing_files) + ) + if report.prompt and report.prompt.casefold() not in added_lower: + errors.append( + "Table-presentation regression test must use the exact reported user prompt so " + "grouped output is exercised without an explicit table keyword" + ) + if "markdown-tabelle" not in added_lower and "markdown table" not in added_lower: + errors.append( + "Table-presentation repair must assert a Markdown-table answer instruction" + ) + if not re.search(r"count\s*\(\s*\$rows\s*\)\s*>\s*1", added_text, re.IGNORECASE): + errors.append( + "Grouped-series table detection must require multiple rows with count($rows) > 1; " + "do not turn every multi-column scalar/detail result into a table" + ) + if not re.search( + r"single[_\s-]row|one[_\s-]row|einzelne[nr]?\s+(?:zeile|datensatz)", + added_text, + re.IGNORECASE, + ): + errors.append( + "Add a negative regression test proving one ordinary multi-column result row keeps " + "the sentence-format instruction" + ) + if re.search( + r"assertStringContainsString\s*\(\s*['\"]alle bereitgestellten Zeilen", + added_text, + re.IGNORECASE, + ): + errors.append( + "The table regression must assert the exact grammatical substring " + "'allen bereitgestellten Zeilen', not 'alle bereitgestellten Zeilen'" + ) + row_count_assertions = ( + ( + r"assertSame\s*\(\s*3\s*,\s*substr_count\s*\(\s*\$userPrompt\s*,\s*" + r"['\"](?:\\?['\"])?Monat(?:\\?['\"])?['\"]\s*\)\s*\)" + ), + ( + r"assertSame\s*\(\s*3\s*,\s*substr_count\s*\(\s*\$userPrompt\s*,\s*" + r"['\"](?:\\?['\"])?Anzahl_Lieferscheine(?:\\?['\"])?['\"]\s*\)\s*\)" + ), + ) + if any(not re.search(pattern, added_text, re.IGNORECASE) for pattern in row_count_assertions): + errors.append( + "Grouped-table provider regression must assert every supplied row reaches " + "$userPrompt; a static mocked Markdown response is vacuous" + ) + + if requires_explicit_route_repair(report): + desired = "\n".join(filter(None, ( + report.user_feedback, + report.expected_behavior, + ))).casefold() + required_files = { + "app/Http/Controllers/ChatController.php", + "tests/Unit/ChatControllerSourceResolutionTest.php", + } + if "www:" in desired: + required_files.update({ + "app/Services/WebSearchService.php", + "tests/Unit/WebSearchServiceTest.php", + }) + missing_files = sorted(required_files - changed_files) + if missing_files: + errors.append( + "Explicit route-prefix requirement must change ChatController routing and its " + "deterministic regression test; missing: " + ", ".join(missing_files) + ) + missing_prefixes = [ + prefix + for prefix in ("cockpit:", "www:", "direct:") + if prefix in desired and prefix not in added_lower + ] + if missing_prefixes: + errors.append( + "Explicit route-prefix regression coverage is missing: " + + ", ".join(missing_prefixes) + ) + required_route_fragments = ( + ("resolveRoutePrefix", r"resolveRoutePrefix"), + ("database tuple", r"\[\s*['\"]database['\"]\s*,"), + ("research tuple", r"\[\s*['\"]research['\"]\s*,"), + ("direct tuple", r"\[\s*['\"]direct['\"]\s*,"), + ("no-prefix tuple", r"\[\s*null\s*,"), + ("web research call", r"generateWebResearchResponse\s*\("), + ("direct model call", r"generateDirectResponse\s*\("), + ( + "destructured processMessage call", + ( + r"\[\s*\$explicitRoute\s*,\s*\$message\s*\]\s*=\s*" + r"\$this->resolveRoutePrefix\(\$[A-Za-z_]\w*\)" + ), + ), + ("explicit route fallback", r"\$route\s*=\s*\$explicitRoute\s*\?\?"), + ) + missing_route_fragments = [ + label + for label, pattern in required_route_fragments + if not re.search(pattern, added_text, re.IGNORECASE) + ] + if missing_route_fragments: + errors.append( + "Explicit route-prefix repair must use one tested resolveRoutePrefix helper that " + "is destructured by processMessage before route fallback; missing: " + + ", ".join(missing_route_fragments) + ) + process_message_calls = len(re.findall(r"->processMessage\s*\(", added_text)) + never_classifier_expectations = len(re.findall( + r"expects\s*\([^\n;]*never\s*\(\s*\)[^;]*\)\s*" + r"->method\s*\(\s*['\"]determineRoute['\"]\s*\)", + added_text, + re.IGNORECASE, + )) + if process_message_calls < 3 or never_classifier_expectations < 3: + errors.append( + "Explicit prefix regression tests must call processMessage() for cockpit:, www:, " + "and direct: and assert AIService::determineRoute() is never called for each path; " + "helper-only tests do not prove request routing" + ) + cockpit_test = re.search( + r"function\s+test[^\{]*cockpit[^\{]*\(\)\s*:\s*void\s*\{(?P.*?)\n\s*\}", + added_text, + re.IGNORECASE | re.DOTALL, + ) + cockpit_body = cockpit_test.group("body") if cockpit_test else "" + required_false_heuristics = ( + "requiresCustomerDatabase", + "requiresConfigDatabase", + "requiresErpDatabase", + ) + if not cockpit_test or any( + not re.search( + rf"method\s*\(\s*['\"]{method}['\"]\s*\).*?willReturn\s*\(\s*false\s*\)", + cockpit_body, + re.IGNORECASE | re.DOTALL, + ) + for method in required_false_heuristics + ): + errors.append( + "The cockpit processMessage regression must force every normal database heuristic " + "false so only the explicit prefix can select the database route" + ) + forbidden_route_patterns = ( + r"resolveSource\s*\(\s*['\"](?:cockpit|www|direct):", + r"function\s+determineRouteForMessage\b", + r"\$this->defaultModel\b", + r"\$this->determineRoute\s*\(", + r"\$route\s*=\s*\$prefixRoute\s*\?\?", + r"return\s+\$this->resolveRoutePrefix\s*\(", + r"['\"]message['\"]\s*=>\s*\$originalMessage", + r"assertStringContainsString\s*\(\s*['\"](?:cockpit|www|direct):", + r"->json\s*\(", + ) + if any(re.search(pattern, added_text, re.IGNORECASE) for pattern in forbidden_route_patterns): + errors.append( + "Explicit prefix tests must exercise resolveRoutePrefix and the real request route; " + "do not test resolveDatabaseSource directly or add wrappers for nonexistent " + "ChatController model-routing members" + ) + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_removal_scope( + report: FailureReport, + summary: ChangeSummary, + workspace: GitWorkspace, +) -> PolicyResult: + if not is_removal_only(report): + return PolicyResult(True, ()) + + errors: list[str] = [] + production_files = [filename for filename in summary.files if not filename.startswith("tests/")] + test_files = [filename for filename in summary.files if filename.startswith("tests/")] + if len(production_files) != 1: + errors.append( + "Removal-only repair must change exactly one owning production file" + ) + return PolicyResult(False, tuple(errors)) + if not test_files: + errors.append("Removal-only repair must add a focused negative regression assertion") + + filename = production_files[0] + original = workspace.head_file(filename) + path = workspace.path / filename + if original is None or not path.is_file(): + errors.append(f"Could not compare removal-only source file with HEAD: {filename}") + return PolicyResult(False, tuple(errors)) + current = path.read_text(encoding="utf-8") + original_example, _ = find_matching_reference(original, report.prompt) + current_example, _ = find_matching_reference(current, report.prompt) + if not original_example or not current_example: + errors.append( + "Removal-only repair must update the uniquely matching tracked reference SQL" + ) + return PolicyResult(False, tuple(errors)) + + expected_sql = original_example.sql + for directive in removal_directives(report): + if not sql_contains_fragment(original_example.sql, directive.fragment): + errors.append( + f"Forbidden SQL fragment was already absent before the agent edit: {directive.fragment}" + ) + continue + if sql_contains_fragment(current_example.sql, directive.fragment): + errors.append( + f"Removal-only repair retained forbidden SQL fragment: {directive.fragment}" + ) + expected_sql = remove_sql_fragment(expected_sql, directive.fragment) + + if normalize_sql(current_example.sql) != normalize_sql(expected_sql): + errors.append( + "Removal-only repair changed joins, fields, filters, grouping, ordering, or other SQL " + "outside the explicitly forbidden fragment" + ) + original_outside_sql = without_reference_sql(original, original_example) + current_outside_sql = without_reference_sql(current, current_example) + if normalize_source_layout(original_outside_sql) != normalize_source_layout(current_outside_sql): + errors.append( + "Removal-only repair changed production content outside the matching reference SQL" + ) + + for test_file in test_files: + previous_test = workspace.head_file(test_file) + current_test_path = workspace.path / test_file + if previous_test is None or not current_test_path.is_file(): + continue + current_test = current_test_path.read_text(encoding="utf-8") + if not _lines_are_subsequence(previous_test, current_test): + errors.append( + f"Removal-only repair removed or rewrote existing regression coverage: {test_file}" + ) + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_removal_prevention( + report: FailureReport, + summary: ChangeSummary, + workspace: GitWorkspace, +) -> PolicyResult: + directives = removal_directives(report) + if not directives or is_removal_only(report): + return PolicyResult(True, ()) + + protected_references: list[tuple[str, str, str]] = [] + production_additions: list[str] = [] + test_additions: list[str] = [] + for filename in summary.files: + original = workspace.head_file(filename) + path = workspace.path / filename + if original is None or not path.is_file(): + continue + current = path.read_text(encoding="utf-8") + additions = "\n".join(_added_lines(original, current)) + if filename.startswith("tests/"): + test_additions.append(additions) + continue + production_additions.append(additions) + original_example, _ = find_matching_reference(original, report.prompt) + if original_example and all( + not sql_contains_fragment(original_example.sql, directive.fragment) + for directive in directives + ): + current_example, _ = find_matching_reference(current, report.prompt) + protected_references.append(( + filename, + original_example.sql, + current_example.sql if current_example else "", + )) + + if not protected_references: + return PolicyResult(True, ()) + + errors: list[str] = [] + for filename, original_sql, current_sql in protected_references: + if not current_sql or normalize_sql(current_sql) != normalize_sql(original_sql): + errors.append( + "The matching reference SQL was already corrected before this report and is a " + f"protected baseline; preserve it exactly instead of rewriting it: {filename}" + ) + + added_production = "\n".join(production_additions) + added_tests = "\n".join(test_additions) + production_lines = added_production.splitlines() + directive_identifiers = { + _directive_identifier(directive.fragment).casefold() + for directive in directives + } + unrelated_rules = [ + line.strip() + for line in production_lines + if _looks_like_php_rule(line) + and not any(identifier in line.casefold() for identifier in directive_identifiers) + ] + if unrelated_rules: + errors.append( + "Removal-prevention repair added unrelated business guidance. Add only the narrow rule " + "that prohibits the reported identifier; preserve existing date, document-code, lookup, " + "status, and exclusion semantics. Remove: " + " | ".join(unrelated_rules) + ) + for directive in directives: + identifier = _directive_identifier(directive.fragment) + related_rules = [ + line + for line in production_lines + if identifier.casefold() in line.casefold() and _has_negative_language(line) + ] + if not related_rules: + errors.append( + "Add narrow reusable production guidance that explicitly prohibits using " + f"{identifier} for this business case" + ) + if any(sql_contains_fragment(line, directive.fragment) for line in related_rules): + errors.append( + "Reusable production guidance must not embed the complete forbidden predicate " + f"{directive.fragment}; mention {identifier} and the prohibition without reproducing " + "the full predicate, otherwise a real context-level negative assertion cannot pass" + ) + + positive_context_literals = _positive_context_assertion_literals(added_tests) + if not any( + len(literal.strip()) >= 12 and literal in added_production + for literal in positive_context_literals + ): + errors.append( + "The focused context-builder test is vacuous: it only checks text that was already " + "absent from the old context. Add assertStringContainsString for unique text from the " + "new production prohibition and pass every table declared by that rule to " + "businessContext(), while retaining the exact negative assertion" + ) + + return PolicyResult(not errors, tuple(errors)) + + +def evaluate_expected_behavior(report: FailureReport, added_text: str) -> PolicyResult: + expected = report.expected_behavior or "" + if not expected.strip(): + return PolicyResult(True, ()) + if not isinstance(added_text, str): + added_text = "" + + directives = removal_directives(report) + forbidden_text = "\n".join(item.fragment for item in directives).lower() + + technical_terms = set(re.findall( + r"\b(?:[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+|[A-Za-z]+[A-Z][A-Za-z]+|\d+)\b", + expected, + )) + technical_terms = { + term + for term in technical_terms + if term.lower() not in forbidden_text and not term.isupper() + } + expected_lower = expected.lower() + for status_filter in ("obsolet", "storniert", "gebucht"): + if status_filter in expected_lower and status_filter not in forbidden_text: + technical_terms.add(status_filter) + + added_lower = added_text.lower() + missing = sorted(term for term in technical_terms if term.lower() not in added_lower) + errors = [] + if missing: + errors.append( + "Repair does not cover explicit expected-behavior terms in added code and assertions: " + + ", ".join(missing) + ) + + expected_identifiers = [ + identifier + for identifier in re.findall( + r"\b[A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+\b|\b[A-Za-z]+[A-Z][A-Za-z]+\b", + expected, + ) + if identifier.lower() not in forbidden_text and not identifier.isupper() + ] + for identifier in expected_identifiers: + escaped = re.escape(identifier) + if re.search( + rf"(?:never|niemals)\s+(?:\w+\s+){{0,6}}{escaped}", + added_text, + re.IGNORECASE, + ): + errors.append(f"Repair contradicts expected identifier {identifier}") + + numeric_filters = [ + pair + for pair in re.findall( + r"\b([A-Za-z][A-Za-z0-9]*_[A-Za-z0-9_]+|[A-Za-z]+[A-Z][A-Za-z]+)\s+(\d+)\b", + expected, + ) + if pair[0].lower() not in forbidden_text + ] + document_codes = () if "auftragskennung" in forbidden_text else re.finditer( + r"\bauftragskennung\s+(\d+)\b", + expected, + re.IGNORECASE, + ) + for match in document_codes: + pair = ("AuftragsKennung", match.group(1)) + if not any( + identifier.lower() == pair[0].lower() and value == pair[1] + for identifier, value in numeric_filters + ): + numeric_filters.append(pair) + technical_terms.add(pair[0]) + for identifier, value in numeric_filters: + if not re.search( + rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b", + added_text, + re.IGNORECASE, + ): + errors.append(f"Repair must express {identifier} = {value} explicitly") + + if "obsolet" in expected_lower and "obsolet" not in forbidden_text and not re.search( + r"COALESCE\s*\([^)]*(? bool: + if any( + not re.search( + rf"{re.escape(identifier)}\s*=\s*{re.escape(value)}\b", + candidate, + re.IGNORECASE, + ) + for identifier, value in numeric_filters + ): + return False + if any(identifier.lower() not in candidate.lower() for identifier in expected_sql_identifiers): + return False + if ( + "obsolet" in expected_lower + and "obsolet" not in forbidden_text + and not re.search( + r"COALESCE\s*\([^)]*(? bool: + return bool(re.search( + r"assert\w*(?:not|false)|notcontains|doesnotcontain|\bnot\s+in\b", + line, + re.IGNORECASE, + )) + + +def _directive_identifier(fragment: str) -> str: + identifiers = re.findall( + r"(?:[A-Za-z_][A-Za-z0-9_]*\.)?([A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+)", + fragment, + ) + return identifiers[-1] if identifiers else fragment + + +def _has_negative_language(line: str) -> bool: + return bool(re.search( + r"\b(?:nicht|nie|niemals|kein\w*|ohne|verboten|" + r"never|not|without|forbid\w*|prohibit\w*)\b", + line, + re.IGNORECASE, + )) + + +def _looks_like_php_rule(line: str) -> bool: + stripped = line.strip() + return bool( + "=>" not in stripped + and re.fullmatch(r"['\"].{8,}['\"],", stripped) + ) + + +def _added_lines(original: str, current: str) -> tuple[str, ...]: + before = original.splitlines() + after = current.splitlines() + additions: list[str] = [] + for tag, _before_start, _before_end, after_start, after_end in SequenceMatcher( + None, + before, + after, + autojunk=False, + ).get_opcodes(): + if tag in {"insert", "replace"}: + additions.extend(after[after_start:after_end]) + return tuple(additions) + + +def _positive_context_assertion_literals(text: str) -> tuple[str, ...]: + literals: list[str] = [] + patterns = ( + r"assertStringContainsString\(\s*'((?:\\.|[^'\\])*)'\s*,\s*\$\w*context\w*", + r'assertStringContainsString\(\s*"((?:\\.|[^"\\])*)"\s*,\s*\$\w*context\w*', + ) + for pattern in patterns: + for match in re.finditer(pattern, text, re.IGNORECASE): + literals.append( + match.group(1).replace("\\'", "'").replace('\\"', '"').replace("\\\\", "\\") + ) + return tuple(literals) + + +def _without_identifier_qualifiers(fragment: str) -> str: + return re.sub( + r"\b[A-Za-z_][A-Za-z0-9_]*\.(?=[A-Za-z_][A-Za-z0-9_]*)", + "", + fragment, + ) + + +def _lines_are_subsequence(original: str, current: str) -> bool: + original_lines = [line.strip() for line in original.splitlines() if line.strip()] + current_lines = iter(line.strip() for line in current.splitlines() if line.strip()) + return all(any(candidate == expected for candidate in current_lines) for expected in original_lines) diff --git a/old_2026-08-31/src/lai_automation/prompt.py b/old_2026-08-31/src/lai_automation/prompt.py new file mode 100644 index 0000000..e64d7c0 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/prompt.py @@ -0,0 +1,451 @@ +from __future__ import annotations + +import re + +from .models import FailureReport +from .removal import is_removal_only, removal_directives + +_SECRET_PATTERNS = ( + ( + re.compile(r"(?i)\b(?:mysql|mariadb|postgres(?:ql)?|sqlserver)://[^\s]+"), + "[DATABASE_URL]", + ), + (re.compile(r"(?i)(password|passwd|secret|token|api[_ -]?key)\s*[:=]\s*\S+"), r"\1=[REDACTED]"), + (re.compile(r"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), "[EMAIL]"), + (re.compile(r"\b(?:\d{1,3}\.){3}\d{1,3}\b"), "[IP]"), + (re.compile(r"(?i)bearer\s+[A-Za-z0-9._~+/=-]+"), "Bearer [REDACTED]"), + (re.compile(r"\bAKIA[0-9A-Z]{16}\b"), "[AWS_KEY]"), + (re.compile(r"-----BEGIN [A-Z ]+KEY-----[\s\S]+?-----END [A-Z ]+KEY-----"), "[PRIVATE_KEY]"), + (re.compile(r"\b[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\b"), "[JWT]"), +) + +_REPOSITORY_PLAYBOOK = """ +Repository ownership map: +- app/Services/SQLGeneratorService.php assembles live schema and business context for SQL generation. +- config/erp_semantics.php owns ERP joins, metrics, formulas, filters, and reference examples. +- config/cockpit_semantics.php owns device and customer-database semantics. +- app/Services/AIService.php owns provider calls, pseudonymization boundaries, and response parsing. +- app/Services/TenantDatabaseService.php owns ERP/config/customer source selection and tenant metadata. +- Cross-database workflows are orchestrated in ChatController/TenantDatabaseService; one SQL query + cannot join tables exposed through different Laravel connections unless the tracked architecture + explicitly supports qualified access. +- app/Services/SQLValidatorService.php owns read-only SQL safety; do not weaken it to accept a fix. +- app/Http/Controllers/ChatController.php orchestrates routing, execution, answers, and diagnostics. +- app/Services/FailureTrackingService.php and FailureAutomationService.php own failure handoff. +- tests/Unit is preferred for deterministic service/context rules; tests/Feature is for HTTP, + persistence, authorization, or real isolated-database behavior. + +Repair workflow: +1. Inspect the owning implementation, its configuration, and the nearest existing test first. +2. State one root-cause hypothesis and make the smallest change that can prove it. +3. Put reusable business knowledge in tracked semantics configuration, not question-specific code. +4. A prompt/context fix must test the real generated context. A deterministic SQL builder must test + its exact SQL and execute it against synthetic fixtures when useful. +5. Tests must contain assertions that execute; reject empty loops and broad unrelated fallbacks. +6. Do not expose protected production methods publicly for tests. Use a test-only subclass. +7. Do not duplicate one rule in service code and semantics config, or add explanatory comments for + self-evident test setup. +8. Inspect the final diff for malformed edits, unrelated files, hardcoded report values, and tests + that pass without exercising the changed behavior. Before retrying an edit, reread the current + file and never leave conflict markers or partial methods behind. +9. PHP config files are executable code. Preserve brackets, commas, quotes, and array nesting; run + `php -l` on every changed PHP file before broader tests. +10. The orchestrator prepares dependencies, frontend assets, migrations, and isolated synthetic test + schemas. Never require or copy production rows. Build deterministic fixtures for data-dependent + regressions and do not edit generated assets, vendor files, node_modules, or environment files. +""" + + +def redact(value: str | None) -> str: + result = value or "" + for pattern, replacement in _SECRET_PATTERNS: + result = pattern.sub(replacement, result) + return result[:12_000] + + +def php_negative_assertion(fragment: str) -> str: + escaped = fragment.replace("\\", "\\\\").replace("'", "\\'") + return f"$this->assertStringNotContainsString('{escaped}', $context);" + + +def requires_table_output_repair(report: FailureReport) -> bool: + desired = _desired_report_text(report) + return bool(re.search( + r"\b(?:markdown[- ]?tabelle|tabelle|tabellar(?:isch|ische|ischer|ischen|er)?)\b", + desired, + re.IGNORECASE, + )) + + +def requires_explicit_route_repair(report: FailureReport) -> bool: + desired = _desired_report_text(report).casefold() + return any(prefix in desired for prefix in ("cockpit:", "www:", "direct:")) + + +def _desired_report_text(report: FailureReport) -> str: + return "\n".join(filter(None, ( + report.prompt, + report.user_feedback, + report.expected_behavior, + ))) + + +def _declares_data_correct(report: FailureReport) -> bool: + desired = _desired_report_text(report) + return bool(re.search( + r"\b(?:die\s+)?daten\s+(?:sind|waren)\s+korrekt\b|" + r"\b(?:the\s+)?data\s+(?:is|are|was|were)\s+correct\b", + desired, + re.IGNORECASE, + )) + + +def repair_files(report: FailureReport) -> tuple[str, ...]: + expected = f"{report.user_feedback or ''}\n{report.expected_behavior or ''}".lower() + report_text = "\n".join(filter(None, ( + report.prompt, + report.user_feedback, + report.expected_behavior, + report.sql_query, + report.error_message, + ))).lower() + if "cockpit_config" in expected and ("fk_" in expected or "erp" in expected): + return ( + "app/Http/Controllers/ChatController.php", + "app/Services/TenantDatabaseService.php", + "tests/Unit/ChatControllerSourceResolutionTest.php", + ) + if '__LAI_' in (report.response or ''): + return ( + "app/Services/AIService.php", + "app/Http/Controllers/ChatController.php", + "tests/Feature/AIPrivacyBoundaryTest.php", + ) + + production_files: list[str] = [] + test_files: list[str] = [] + + def add_file(filename: str) -> None: + target = test_files if filename.startswith("tests/") else production_files + if filename not in target: + target.append(filename) + + if requires_table_output_repair(report): + add_file("app/Services/AIService.php") + add_file("tests/Feature/AIPrivacyBoundaryTest.php") + + if requires_explicit_route_repair(report): + add_file("app/Http/Controllers/ChatController.php") + add_file("tests/Unit/ChatControllerSourceResolutionTest.php") + if "www:" in report_text: + add_file("app/Services/WebSearchService.php") + add_file("tests/Unit/WebSearchServiceTest.php") + + if report.category in {"incorrect_answer", "sql_execution"} and ( + report.context.get("database_type") == "erp" + or "fk_" in report_text + or any(term in report_text for term in ( + "auftrag", "rechnung", "lieferschein", "umsatz", "gutschrift" + )) + ) and not _declares_data_correct(report): + add_file("config/erp_semantics.php") + add_file("tests/Unit/SQLGeneratorBusinessContextTest.php") + + return (*production_files, *test_files) + + +def build_task(report: FailureReport) -> str: + prompt = redact(report.prompt) + feedback = redact(report.user_feedback) + expected_behavior = redact(report.expected_behavior) + response = redact(report.response) + error = redact(report.error_message) + sql = redact(report.sql_query) + tenant_scope = '' + if report.context.get('database_type') == 'customer': + tenant_scope = """ +Tenant routing fact: +The customer has already selected one tenant database. SQL inside that database must not +invent an additional customer discriminator column. Use only columns present in tracked +schema/semantics, and fix SQL-generation guidance rather than rewriting one unknown column +to another guessed column. TenantDatabaseService records the explicit source type in +database.connections..tenant_type; use that metadata instead of database-name +patterns or heuristics. +""" + sql_semantics = '' + if report.category in {'incorrect_answer', 'sql_execution'}: + sql_semantics = """ +SQL repair guidance: +When the SQL is syntactically valid but uses the wrong business calculation, prefer adding or +correcting tracked domain knowledge in config/*_semantics.php so the schema prompt teaches the +model the right joins, filters, and formula. Do not hardcode one user's question or article ID +in SQLGeneratorService, and do not add post-generation replacement of guessed SQL. + +Translate positive expected behavior literally into both a reusable rule and a reference SQL example. +If it gives a numeric document code, use direct equality with that code rather than a lookup label. +If it names a date column, use that exact column even when another date is common elsewhere. Add +every requested exclusion explicitly. Do not copy additional date, booking, lookup, or status +filters from the known-bad SQL when expected behavior replaces or does not request them. Regression +assertions must require the corrected fields and filters and must never assert the opposite rule. +The same single executable reference SELECT must demonstrate all requested identifiers, numeric +codes, date columns, and exclusions together; listing them only in separate prose rules is not proof. +Use an expected bare status name as that exact column token; do not invent bStatus_ or other prefixes. +The focused test must assert every required filter, including obsolete and cancellation exclusions. +Do not normalize capitalization or wording in unrelated existing rules. + +For regression coverage of prompt guidance, expose the relevant protected schema/business- +context builder through a test subclass and assert that its real output contains the required +formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt +guidance to rewrite that mocked response. Keep database fixture tests only when production code +itself deterministically builds and executes the SQL under test. + +For an unknown-column failure, compare every alias.column in the observed SQL with the table +schema and identify the owning table before editing. Teach the exact ownership, join, grouping, +and real column names in semantics config, then add a reference SQL example for the reported +question shape. Never replace one missing column with an unverified synonym. + +When expected behavior names tables from multiple databases or connections, first inspect source +routing and existing staged-query workflows. Do not put an impossible cross-connection join only +in ERP semantics. Implement deterministic orchestration that resolves identifiers in one source +and passes only the required identifier into the next source, preserving tenant isolation. +For this cross-database report, editing only config/*_semantics.php or tests is an invalid repair. +You MUST implement production behavior in ChatController or TenantDatabaseService and cover that +exact application method in ChatControllerSourceResolutionTest. Before editing, describe the data +flow, connection used at each stage, and where the final result is assembled. +""" + placeholder_guidance = '' + if '__LAI_' in response: + placeholder_guidance = """ +Privacy placeholder repair guidance: +Trace the actual answer path through AIService::generateDatabaseResponse, issued token +restoration, and ChatController's local fallback. A token can only be restored when it exists +in the exact token map returned by protect(); never test a made-up token against an unrelated +map. Cover model-invented or malformed __LAI_...__ output with an HTTP-backed AIService test +and verify the user receives a real local database-result fallback, not a raw placeholder. +""" + + directives = removal_directives(report) + forbidden_lower = "\n".join(item.fragment for item in directives).lower() + expected_lower = expected_behavior.lower() + sql_literal_contract: list[str] = [] + document_code = re.search(r"\bauftragskennung\s+(\d+)\b", expected_behavior, re.IGNORECASE) + if document_code and "auftragskennung" not in forbidden_lower: + sql_literal_contract.append(f"a.AuftragsKennung = {document_code.group(1)}") + if "system_created" in expected_lower and "system_created" not in forbidden_lower: + sql_literal_contract.append("a.System_created") + if "obsolet" in expected_lower and "obsolet" not in forbidden_lower: + sql_literal_contract.append("COALESCE(a.obsolet, 0) = 0") + if "storniert" in expected_lower and "storniert" not in forbidden_lower: + sql_literal_contract.append("COALESCE(a.bStatus_storniert, 0) = 0") + literal_guidance = "" + if sql_literal_contract: + literal_guidance = ( + "\nExact SQL literal contract derived from expected behavior:\n- " + + "\n- ".join(sql_literal_contract) + + "\nOne added reference SELECT and the focused regression assertions must contain " + "every literal above. Preserve spelling and capitalization exactly. Do not add an LS " + "lookup or bStatus_gebucht unless it appears in this contract. Assertions must quote " + "these exact SQL literals; do not replace them with descriptive prose.\n" + ) + removal_guidance = "" + if directives: + exact_assertions = "\n- ".join( + php_negative_assertion(item.fragment) for item in directives + ) + removal_guidance = ( + "\nExact SQL removal contract derived from expected behavior:\n- " + + "\n- ".join(item.fragment for item in directives) + + "\nEvery matching production reference SELECT must omit every fragment above. " + "The following PHP regression assertions are mandatory; copy them exactly, including " + "aliases and qualifiers:\n- " + + exact_assertions + + "\nA forbidden identifier is a negative requirement: do not add it positively merely " + "because it appears in expected behavior. An unqualified substring is not equivalent " + "to an alias-qualified fragment.\n" + ) + if is_removal_only(report): + removal_guidance += ( + "This is explicitly a removal-only repair. Preserve every other selected field, " + "table, join, date column, predicate, grouping, ordering, limit, rule, and existing " + "test assertion byte-for-byte. Change only the matching reference SQL fragment and " + "add focused negative regression coverage.\n" + ) + else: + removal_guidance += ( + "Before editing, inspect the matching tracked reference SELECT. If it already omits " + "the forbidden fragment, treat that SELECT as a protected correct baseline and " + "preserve it exactly. Add or clarify the narrow reusable business rule that prevents " + "the reported recurrence instead of regressing the reference merely to create a diff. " + "Add only that narrow prohibition rule. Do not add or restate date-column choices, " + "document codes, lookup strategies, joins, cancellation/obsolete rules, or other " + "business guidance already covered by the protected reference. " + "Phrase the rule without embedding the complete forbidden predicate: mention its " + "column identifier and say it must not be used, so the exact context-level negative " + "assertion above remains true. The focused context-builder test must positively assert " + "the new rule with assertStringContainsString for unique text from that rule as well as use the exact " + "negative assertion above; a negative assertion alone is vacuous when the old reference " + "already omitted the fragment. A configured rule is exercised only when every " + "table in its tables list is supplied to businessContext(); align the test's allowed " + "tables with the rule so the test cannot pass while omitting the production change.\n" + ) + + interaction_guidance = "" + if requires_table_output_repair(report): + interaction_guidance += """ +Answer-presentation repair guidance: +This report requires a real answer-formatting change, not ERP SQL or semantics changes when the +report says the data is already correct. Inspect AIService::generateDatabaseResponse(). Grouped +series requests such as counts per month must request a compact Markdown table even when the +original user prompt does not literally contain the word "Tabelle". Add a provider-backed test +using the exact reported user prompt. It must prove that the answer-generation system instruction +requests a Markdown table and that all supplied grouped rows remain available. Do not add static +example result values or output-format metadata to config/*_semantics.php. Grouped-series detection +must require multiple result rows and grouping/value columns; one ordinary result row with multiple +columns must remain sentence-formatted. Add that negative provider-backed regression test as well. +The positive grouped test uses columns named exactly Monat and Anzahl_Lieferscheine. Ensure the +production grouping/value-column detection recognizes those exact names, and confirm the positive +test receives Markdown-Tabelle before finishing; do not merely add the test without satisfying it. +Inside the provider fake, prove every supplied row reached $userPrompt without exposing protected +values: for three rows, assert substr_count($userPrompt, '"Monat"') and +substr_count($userPrompt, '"Anzahl_Lieferscheine"') both equal 3. A mocked Markdown response alone +is vacuous because it can pass even if no grouped rows reached the provider. +Keep the existing exact table instruction text: "Die Benutzerfrage verlangt eine Tabelle. Gib eine +kurze Einleitung und danach eine kompakte Markdown-Tabelle mit allen bereitgestellten Zeilen aus." +Do not move or duplicate this instruction inside shouldUseTableFormat(); that helper returns only bool. +The regression assertion must use the exact substring "allen bereitgestellten Zeilen". Never assert +"alle bereitgestellten Zeilen": that grammatically different substring is not present in the correct +production instruction. If Markdown-Tabelle passes but this assertion fails, fix the assertion—not +the already-correct production instruction. +""" + if requires_explicit_route_repair(report): + interaction_guidance += """ +Explicit route-prefix repair guidance: +Implement one protected resolveRoutePrefix(string $message): array helper that always returns exactly +[explicit route or null, cleaned message]. Call it with exact destructuring at the start of +processMessage(): [$explicitRoute, $message] = $this->resolveRoutePrefix($message); Then compute +$route = $explicitRoute ?? (...) around the existing database-keyword/model-classifier expression, +before database keyword heuristics or model classification: +"cockpit:" must force the database path, "www:" must force public web research, and "direct:" must +force the selected model without web or database access, even when the remaining text contains +keywords for another route. Web retrieval must be provider-independent: WebSearchService retrieves +bounded public search snippets, then AIService::generateWebResearchResponse() passes only the clean +question and those untrusted snippets to the selected model, including Ollama/Qwen, and returns +clickable source URLs. Do not pass private conversation/database context into web research, fetch +arbitrary result URLs, or obey instructions embedded in search snippets. Remove the control prefix +before passing the user's actual question downstream. Add deterministic ChatController routing tests +for all three prefixes that prove the classifier cannot override them. For every explicit prefix, +AIService::determineRoute() must not be called; never write a test expecting that classifier call. +Helper-only tests are insufficient: add three tests that call processMessage() itself, one for each +prefix, assert determineRoute() is never called, and assert the cleaned question reaches the forced +database/research/direct response path. Mock only external collaborators, not processMessage or +prefix logic. +This is feasible in the existing test file: switch it from PHPUnit\\Framework\\TestCase to +Tests\\TestCase, use Illuminate\\Foundation\\Testing\\RefreshDatabase, build an +Illuminate\\Http\\Request with Request::create(), and attach app('session')->driver() using +setLaravelSession(). For www:, mock generateWebResearchResponse() with the exact cleaned question +and selected ollama model, assert the JSON route is research, and assert source metadata is persisted. +For direct:, mock generateDirectResponse(), assert generateWebResearchResponse() is never called, +and assert the JSON route is direct. For cockpit:, make the tenant mock return true from +requiresCustomerDatabase() and null from resolveFromPrompt(), then assert the JSON route is +customer_clarification and the persisted ChatMessage.message equals the cleaned question exactly. +For that cockpit test, all requiresCustomerDatabase/requiresConfigDatabase/requiresErpDatabase +heuristics must return false; otherwise the database route would be selected without the prefix and +the test is vacuous. The explicit cockpit route itself must be the only reason determineRoute() is +bypassed. Use a cleaned question such as "Erkläre Quantenphysik" that has no database keywords. +Each prefix is a control token and must not be stored or passed downstream. Never assert that a +persisted message contains cockpit:, www:, or direct:, and never change ChatMessage 'message' fields +to $originalMessage. In all three tests put +$aiService->expects($this->never())->method('determineRoute') before processMessage(). Use an +explicit model in each request-path test so mocked response methods do not throw an argument-mismatch +exception that processMessage() catches and converts into an error JSON response. Assert response +status/success before reading route. +Do not write “we can't fully test processMessage”; Laravel's application test case provides the +HTTP, session, response, and database context needed here. +JsonResponse has no json() method in this codebase; read it once with $payload = +$response->getData(true), then assert $payload['success'] and $payload['route']. +The helper must accept leading whitespace and case-insensitive prefixes, and trim the cleaned +question. Test the helper returns ['database', cleaned question], ['research', cleaned question], +['direct', cleaned question], and [null, unchanged trimmed question]. Do not assign the whole helper +result array to $route. Then prove +processMessage() uses that result rather than only testing resolveRoutePrefix() or +resolveDatabaseSource(). +Do not add prefix handling to resolveDatabaseSource(), do not change its default-source behavior, and +do not add wrappers that reference nonexistent properties or methods such as defaultModel or +determineRoute on ChatController. Inspect the existing TestableChatController before editing: +expose resolveRoutePrefix() exactly once, and replace an existing wrapper when necessary—never +append a duplicate method declaration. The public test-subclass wrapper must call +parent::resolveRoutePrefix($message), never $this->resolveRoutePrefix($message), which recursively +calls itself. Do not reinterpret "cockpit:" as a new SQL business rule. +All three prefix behaviors and any independent presentation requirement in this report are mandatory. +""" + + return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository. + +{_REPOSITORY_PLAYBOOK} + +Rules: +- First add a regression test that reproduces the reported behavior. +- The regression test must execute the relevant production behavior; never mock or stub the behavior being tested. +- Make the smallest production change that fixes the root cause. +- Treat the user's expected behavior as authoritative, while verifying table and column names against tracked schema and semantics. +- Every positively required technical identifier, numeric code, and status filter named in expected + behavior must appear in the production repair and focused regression assertions. Identifiers + explicitly requested for removal must instead be absent from matching SQL and appear in an exact + negative assertion. Do not substitute a pre-existing rule. +- Preserve read-only SQL validation and tenant isolation. +- Never read or modify .env files, credentials, deployment secrets, or production data. +- Do not depend on production row contents. Use synthetic fixtures or assert generated business context and SQL guidance. +- Do not weaken tests, authentication, authorization, SQL validation, or safety checks. +- Do not commit or push; the orchestrator owns Git publication. + +Failure category: {report.category} +Feedback type: {report.feedback_type or 'not specified'} +Source: {report.source} +Model: {redact(str(report.context.get('model', 'unknown')))} +Database type: {redact(str(report.context.get('database_type', 'unknown')))} + +User request: +{prompt or '[not available]'} + +User feedback: +{feedback or '[not available]'} + +Expected behavior: +{expected_behavior or '[not available]'} + +Observed answer: +{response or '[not available]'} + +Observed error: +{error or '[not available]'} + +Generated SQL, if relevant: +{sql or '[not available]'} +{tenant_scope} +{sql_semantics} +{literal_guidance} +{removal_guidance} +{interaction_guidance} +{placeholder_guidance} + +Run focused tests while working. Leave the repository with the fix and regression test applied. +""" + + +def build_retry_task(report: FailureReport, exception: Exception) -> str: + failure = redact(str(exception)) + + return build_task(report) + f""" + +The previous implementation failed isolated validation. + +Correct the existing changes using this test output: + +{failure} + +Keep the regression test meaningful. Do not remove or weaken tests, safety checks, +authentication, authorization, SQL validation, or tenant isolation. If a new test calls an +unrelated specialized code path that correctly returns no result, repair the test to exercise +the owning production behavior; never broaden unrelated production routing merely to make the +test pass. Do not commit or push. +""" diff --git a/old_2026-08-31/src/lai_automation/removal.py b/old_2026-08-31/src/lai_automation/removal.py new file mode 100644 index 0000000..5491247 --- /dev/null +++ b/old_2026-08-31/src/lai_automation/removal.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import re +import unicodedata +from dataclasses import dataclass +from difflib import SequenceMatcher +from pathlib import Path + +from .models import FailureReport + +_REMOVAL_INTENT = re.compile( + r"\b(?:entfern\w*|lösch\w*|loesch\w*|weglass\w*|streichen|" + r"remove\w*|delet\w*|omit\w*|drop\w*|ohne|without)\b|" + r"\b(?:darf|soll|muss)\b.{0,50}\b(?:nicht|nie|niemals)\b|" + r"\b(?:must|should)\s+not\b|\bdo\s+not\b", + re.IGNORECASE | re.DOTALL, +) +_NEGATED_REMOVAL = re.compile( + r"\b(?:nicht|nie|niemals)\s+(?:entfern\w*|lösch\w*|loesch\w*|weglass\w*)\b|" + r"\b(?:do\s+not|don't|never)\s+(?:remove\w*|delet\w*|omit\w*|drop\w*)\b", + re.IGNORECASE, +) +_ONLY_REMOVAL = re.compile( + r"\b(?:einfach\s+)?nur\b.{0,120}\b(?:entfern\w*|lösch\w*|loesch\w*|" + r"weglass\w*|streichen)\b|" + r"\b(?:just|only)\b.{0,120}\b(?:remove\w*|delet\w*|omit\w*|drop\w*)\b", + re.IGNORECASE | re.DOTALL, +) +_QUOTED_FRAGMENT = re.compile( + r'"(?P[^"\n]{1,500})"|' + r"'(?P[^'\n]{1,500})'|" + r"`(?P[^`\n]{1,500})`|" + r"„(?P[^”\n]{1,500})”" +) +_TECHNICAL_IDENTIFIER = re.compile( + r"\b(?:[A-Za-z_][A-Za-z0-9_]*\.)?[A-Za-z_][A-Za-z0-9_]*_[A-Za-z0-9_]+\b" +) +_QUESTION_KEY = re.compile(r"['\"]question['\"]\s*=>", re.IGNORECASE) +_SQL_HEREDOC = re.compile( + r"['\"]sql['\"]\s*=>\s*<<<(?P['\"]?)(?P