chore: archive previous Aider worker

This commit is contained in:
Christian Meyer
2026-08-31 02:46:58 +02:00
parent c8740a0374
commit 582db3a654
30 changed files with 9415 additions and 0 deletions

View File

@@ -0,0 +1,201 @@
import io
import signal
import subprocess
from pathlib import Path
import pytest
from test_config import settings
from lai_automation.agent import AgentProviderError, AgentTimeoutError, AiderAgent
def test_aider_command_cannot_mutate_git_configuration_or_send_analytics() -> None:
command = AiderAgent(settings()).command(Path("/repo/.git/task.txt"))
assert "--no-gitignore" in command
assert "--no-analytics" in command
assert "--no-auto-commits" in command
assert "--no-git" in command
assert "--stream" in command
assert "--no-pretty" in command
assert "--no-check-update" in command
assert "--no-suggest-shell-commands" in command
assert "--no-detect-urls" in command
assert "--disable-playwright" in command
assert "--no-auto-lint" in command
assert command[command.index("--map-tokens") + 1] == "2048"
assert command.count("/dev/null") == 2
assert "--edit-format" not in command
assert command[-2:] == ["--timeout", "900"]
def test_aider_command_adds_existing_routed_files(tmp_path) -> None:
git_dir = tmp_path / ".git"
git_dir.mkdir()
controller = tmp_path / "app/Http/Controllers/ChatController.php"
controller.parent.mkdir(parents=True)
controller.write_text("<?php\n")
command = AiderAgent(settings()).command(
git_dir / "task.txt",
("app/Http/Controllers/ChatController.php", "missing.php"),
)
assert ["--file", "app/Http/Controllers/ChatController.php"] == command[
command.index("--file"):command.index("--file") + 2
]
assert "missing.php" not in command
assert command[command.index("--map-tokens") + 1] == "0"
assert command[command.index("--edit-format") + 1] == "diff"
assert "--architect" not in command
assert "--editor-model" not in command
def test_unrouted_aider_command_stays_in_direct_edit_mode() -> None:
command = AiderAgent(settings()).command(Path("/repo/.git/task.txt"))
assert "--architect" not in command
def test_aider_detects_output_stall_at_configured_limit() -> None:
agent = AiderAgent(settings(agent_idle_timeout_seconds=120))
assert not agent._output_stalled(119.9)
assert agent._output_stalled(120)
def test_ollama_aider_command_uses_single_whole_file_pass(tmp_path) -> None:
git_dir = tmp_path / ".git"
git_dir.mkdir()
source = tmp_path / "config/erp_semantics.php"
source.parent.mkdir(parents=True)
source.write_text("<?php\n")
command = AiderAgent(settings(agent_model="ollama_chat/qwen3-coder:30b")).command(
git_dir / "task.txt",
("config/erp_semantics.php",),
)
assert command[command.index("--edit-format") + 1] == "whole"
assert "--no-auto-lint" in command
assert "--architect" not in command
assert "--editor-model" not in command
def test_ollama_test_only_correction_uses_precise_diff_format(tmp_path) -> None:
git_dir = tmp_path / ".git"
git_dir.mkdir()
test_file = tmp_path / "tests/Unit/FixTest.php"
test_file.parent.mkdir(parents=True)
test_file.write_text("<?php\n")
command = AiderAgent(settings(agent_model="ollama_chat/qwen3-coder:30b")).command(
git_dir / "task.txt",
("tests/Unit/FixTest.php",),
)
assert command[command.index("--edit-format") + 1] == "diff"
assert "--no-auto-lint" in command
def test_aider_timeout_has_actionable_error(tmp_path, monkeypatch) -> None:
workspace = tmp_path / "repo"
(workspace / ".git").mkdir(parents=True)
class TimedOutProcess:
pid = 4321
stdout = io.StringIO("partial output\n")
stderr = io.StringIO("")
returncode = None
def wait(self, timeout=None):
raise subprocess.TimeoutExpired("aider", timeout)
monkeypatch.setattr(
"lai_automation.agent.subprocess.Popen",
lambda *args, **kwargs: TimedOutProcess(),
)
signals = []
monkeypatch.setattr(
"lai_automation.agent.os.killpg",
lambda pid, sent_signal: signals.append((pid, sent_signal)),
)
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
)())
with pytest.raises(AgentTimeoutError, match="partial output"):
AiderAgent(settings(agent_timeout_seconds=60)).run(workspace, "fix it")
assert signals == [(4321, signal.SIGTERM), (4321, signal.SIGKILL)]
def test_aider_environment_does_not_include_database_credentials(tmp_path, monkeypatch) -> None:
config = settings(workspace_root=tmp_path)
workspace = tmp_path / "repo"
(workspace / ".git").mkdir(parents=True)
monkeypatch.setenv("TEST_DB_PASSWORD", "must-not-reach-aider")
monkeypatch.setenv("OPENROUTER_API_KEY", "qwen-provider-key")
monkeypatch.setenv("OLLAMA_API_BASE", "http://host.docker.internal:11434")
captured = {}
class SuccessfulProcess:
pid = 1234
stdout = io.StringIO("provider said qwen-provider-key\n")
stderr = io.StringIO("")
returncode = 0
def wait(self, timeout=None):
return self.returncode
def popen(command, **kwargs):
captured.update(kwargs["env"])
return SuccessfulProcess()
monkeypatch.setattr("lai_automation.agent.subprocess.Popen", popen)
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
)())
output = AiderAgent(config).run(workspace, "fix it")
assert "TEST_DB_PASSWORD" not in captured
assert "DB_PASSWORD" not in captured
assert captured["OPENROUTER_API_KEY"] == "qwen-provider-key"
assert captured["OLLAMA_API_BASE"] == "http://host.docker.internal:11434"
assert "qwen-provider-key" not in output
assert "[REDACTED]" in output
def test_successful_exit_with_only_provider_failure_is_rejected(tmp_path, monkeypatch) -> None:
workspace = tmp_path / "repo"
(workspace / ".git").mkdir(parents=True)
class ProviderFailedProcess:
pid = 1235
stdout = io.StringIO(
"litellm.InternalServerError: MistralException - Name or service not known\n"
"The API provider's servers are down or overloaded.\n"
)
stderr = io.StringIO("")
returncode = 0
def wait(self, timeout=None):
return self.returncode
monkeypatch.setattr(
"lai_automation.agent.subprocess.Popen",
lambda *args, **kwargs: ProviderFailedProcess(),
)
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
)())
agent = AiderAgent(settings())
with pytest.raises(AgentProviderError, match="provider failed"):
agent.run(workspace, "fix it")
assert agent.last_run_metadata["status"] == "provider_error"

View File

@@ -0,0 +1,110 @@
from pathlib import Path
import pytest
from lai_automation.config import Settings
def settings(**overrides) -> Settings:
values = {
"database_host": "db",
"database_port": 3306,
"database_name": "lai",
"database_user": "worker",
"database_password": "secret",
"source_repository": "https://example.test/lai.git",
"source_branch": "main",
"workspace_root": Path("/tmp/workspaces"),
"mode": "dry-run",
"allow_direct_main": False,
"gitea_url": "https://example.test",
"gitea_owner": "owner",
"gitea_repository": "lai",
"gitea_token": "",
"gitea_username": "bot",
"agent_command": "aider",
"agent_model": "mistral/test",
"agent_timeout_seconds": 900,
"agent_idle_timeout_seconds": 300,
"setup_commands": (),
"test_commands": ("php artisan test",),
"test_database_host": "test-db",
"test_database_port": 3306,
"test_database_name": "lai_agent_test",
"test_database_user": "test_runner",
"test_database_password": "test-secret",
"test_customer_database": "customer_agent_test",
"test_config_database": "cockpit_agent_test",
"poll_seconds": 60,
"max_changed_files": 12,
"max_changed_lines": 800,
"max_agent_attempts": 2,
"stale_claim_minutes": 90,
"dry_run": True,
}
values.update(overrides)
return Settings(**values)
def test_direct_main_requires_explicit_kill_switch() -> None:
config = settings(mode="direct-main", dry_run=False, gitea_token="token")
with pytest.raises(ValueError, match="ALLOW_DIRECT_MAIN"):
config.validate()
def test_pr_requires_gitea_token() -> None:
config = settings(mode="pr", dry_run=False)
with pytest.raises(ValueError, match="GITEA_TOKEN"):
config.validate()
def test_default_commands_prepare_workspace_and_run_tests(monkeypatch) -> None:
monkeypatch.setenv("FAILURE_DB_PASSWORD", "failure-secret")
monkeypatch.setenv("TEST_DB_PASSWORD", "test-secret")
monkeypatch.delenv("SETUP_COMMANDS", raising=False)
monkeypatch.delenv("TEST_COMMANDS", raising=False)
config = Settings.from_env()
assert "npm ci" in config.setup_commands
assert "npm run build" in config.setup_commands
assert "php artisan migrate:fresh --force" in config.setup_commands
assert config.test_commands == ("php artisan test",)
def test_agent_attempts_are_bounded() -> None:
with pytest.raises(ValueError, match="MAX_AGENT_ATTEMPTS"):
settings(max_agent_attempts=8).validate()
settings(max_agent_attempts=7).validate()
def test_agent_timeout_is_bounded() -> None:
with pytest.raises(ValueError, match="AGENT_TIMEOUT_SECONDS"):
settings(agent_timeout_seconds=59).validate()
settings(agent_timeout_seconds=60, agent_idle_timeout_seconds=60).validate()
def test_agent_idle_timeout_is_bounded_by_process_timeout() -> None:
with pytest.raises(ValueError, match="AGENT_IDLE_TIMEOUT_SECONDS"):
settings(agent_idle_timeout_seconds=59).validate()
with pytest.raises(ValueError, match="AGENT_IDLE_TIMEOUT_SECONDS"):
settings(agent_timeout_seconds=120, agent_idle_timeout_seconds=121).validate()
settings(agent_timeout_seconds=120, agent_idle_timeout_seconds=60).validate()
def test_validation_databases_must_be_distinct_and_explicitly_test_only() -> None:
with pytest.raises(ValueError, match="isolated test database"):
settings(test_customer_database="customer_production").validate()
with pytest.raises(ValueError, match="must be distinct"):
settings(test_config_database="customer_agent_test").validate()
with pytest.raises(ValueError, match="FAILURE_DB_NAME"):
settings(
database_name="lai_agent_test",
test_database_name="lai_agent_test",
).validate()

View File

@@ -0,0 +1,35 @@
from unittest.mock import MagicMock, Mock
from test_config import settings
from lai_automation.database import FailureRepository
def test_heartbeat_refreshes_only_an_active_claim() -> None:
repository = FailureRepository(settings())
connection = MagicMock()
cursor = MagicMock()
repository.connection = MagicMock()
repository.connection.return_value.__enter__.return_value = connection
connection.cursor.return_value.__enter__.return_value = cursor
repository.heartbeat(17)
statement, parameters = cursor.execute.call_args.args
assert "SET updated_at = NOW()" in statement
assert "status = 'agent_running'" in statement
assert parameters == (17,)
connection.commit.assert_called_once()
def test_complete_clears_previous_agent_error() -> None:
repository = FailureRepository(settings())
repository._finish = Mock()
repository.complete(17, {"classification": "already_fixed"})
repository._finish.assert_called_once_with(
17,
"resolved",
{"classification": "already_fixed", "agent_error": None},
)

View File

@@ -0,0 +1,239 @@
import subprocess
import sys
from pathlib import Path
from unittest.mock import Mock
from test_config import settings
from lai_automation.git import GitWorkspace
def test_prepare_uses_temporary_authentication_for_private_clone(tmp_path, monkeypatch) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path, gitea_token="token"), 10)
calls = []
def run(command, **kwargs):
calls.append((command, kwargs))
if command[1] == "clone":
askpass = Path(kwargs["env"]["GIT_ASKPASS"])
assert askpass.is_file()
assert Path(kwargs["env"]["GIT_BOT_USER_FILE"]).read_text() == "bot"
assert Path(kwargs["env"]["GIT_BOT_TOKEN_FILE"]).read_text() == "token"
workspace.path.mkdir()
return subprocess.CompletedProcess(command, 0, "", "")
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
workspace.prepare()
clone_command, clone_options = calls[0]
assert clone_command[:2] == ["git", "clone"]
assert "token" not in " ".join(clone_command)
assert clone_options["cwd"] == tmp_path
assert not tuple(tmp_path.glob(".lai-git-auth-*"))
def test_prepare_surfaces_clone_stderr_without_credentials(tmp_path, monkeypatch) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path, gitea_token="token"), 11)
def run(command, **kwargs):
raise subprocess.CalledProcessError(
128,
command,
output="",
stderr="fatal: upstream returned 403",
)
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
try:
workspace.prepare()
except RuntimeError as exception:
message = str(exception)
else:
raise AssertionError("Expected clone failure")
assert "Source repository clone failed" in message
assert "upstream returned 403" in message
assert "token" not in message
assert not tuple(tmp_path.glob(".lai-git-auth-*"))
def test_test_command_does_not_interpret_shell_metacharacters(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 1)
workspace.path.mkdir()
result = workspace.run("printf safe ; touch injected", check=False)
assert result.stdout == "safe"
assert not (workspace.path / "injected").exists()
def test_extra_environment_is_passed_without_privilege_drop(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 6)
workspace.path.mkdir()
result = workspace.run_args(
[sys.executable, "-c", "import os; print(os.environ['SAFE_TEST_VALUE'])"],
extra_environment={"SAFE_TEST_VALUE": "available"},
)
assert result.stdout.strip() == "available"
def test_coder_commands_use_native_process_identity(tmp_path, monkeypatch) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 9)
workspace.path.mkdir()
captured = {}
def run(command, **kwargs):
captured.update(kwargs)
return subprocess.CompletedProcess(command, 0, "ok", "")
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
monkeypatch.setattr(
"lai_automation.git.pwd.getpwnam",
lambda _: Mock(pw_dir="/home/coder", pw_uid=10001, pw_gid=10001),
)
workspace.run_args(["php", "-v"], username="coder")
assert captured["user"] == 10001
assert captured["group"] == 10001
assert captured["extra_groups"] == ()
assert "preexec_fn" not in captured
def test_summary_includes_untracked_files(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 2)
workspace.path.mkdir()
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
(workspace.path / "new.php").write_text("<?php\n")
summary = workspace.summary()
assert summary.files == ("new.php",)
assert summary.added_lines == 1
def test_summary_includes_checkpoint_staged_files(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 7)
workspace.path.mkdir()
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
tracked = workspace.path / "config.php"
tracked.write_text("<?php\nreturn [];\n")
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
subprocess.run(
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test",
"commit", "-m", "base"],
cwd=workspace.path,
check=True,
capture_output=True,
)
tracked.write_text("<?php\nreturn ['fixed'];\n")
workspace.checkpoint()
summary = workspace.summary()
assert summary.files == ("config.php",)
assert summary.added_lines == 1
assert summary.deleted_lines == 1
def test_discard_changes_restores_clean_disposable_workspace(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 3)
workspace.path.mkdir()
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
subprocess.run(
["git", "config", "user.email", "test@example.invalid"],
cwd=workspace.path,
check=True,
)
subprocess.run(
["git", "config", "user.name", "Test"],
cwd=workspace.path,
check=True,
)
tracked = workspace.path / "tracked.php"
tracked.write_text("<?php\nreturn [];\n")
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
subprocess.run(["git", "commit", "-m", "base"], cwd=workspace.path, check=True, capture_output=True)
tracked.write_text("<?php\ninvalid\n")
untracked = workspace.path / "extra.php"
untracked.write_text("<?php\n")
workspace.discard_changes()
assert tracked.read_text() == "<?php\nreturn [];\n"
assert not untracked.exists()
def test_checkpoint_restores_last_valid_changes(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 4)
workspace.path.mkdir()
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
subprocess.run(
["git", "config", "user.email", "test@example.invalid"],
cwd=workspace.path,
check=True,
)
subprocess.run(
["git", "config", "user.name", "Test"],
cwd=workspace.path,
check=True,
)
tracked = workspace.path / "config.php"
tracked.write_text("<?php\nreturn [];\n")
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
subprocess.run(["git", "commit", "-m", "base"], cwd=workspace.path, check=True, capture_output=True)
tracked.write_text("<?php\nreturn ['valid'];\n")
added = workspace.path / "test.php"
added.write_text("<?php\n")
checkpoint = workspace.checkpoint()
tracked.write_text("<?php\nreturn [broken;\n")
workspace.restore_checkpoint(checkpoint)
assert tracked.read_text() == "<?php\nreturn ['valid'];\n"
assert added.exists()
def test_diff_returns_discarded_patch_excerpt(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 5)
workspace.path.mkdir()
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
tracked = workspace.path / "service.php"
tracked.write_text("<?php\nreturn 'old';\n")
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
subprocess.run(
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test", "commit", "-m", "base"],
cwd=workspace.path,
check=True,
capture_output=True,
)
tracked.write_text("<?php\nreturn 'new';\n")
patch = workspace.diff()
assert "-return 'old';" in patch
assert "+return 'new';" in patch
def test_head_file_reads_original_tracked_content(tmp_path) -> None:
workspace = GitWorkspace(settings(workspace_root=tmp_path), 8)
workspace.path.mkdir()
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
tracked = workspace.path / "config.php"
tracked.write_text("<?php\nreturn 'original';\n")
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
subprocess.run(
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test",
"commit", "-m", "base"],
cwd=workspace.path,
check=True,
capture_output=True,
)
tracked.write_text("<?php\nreturn 'candidate';\n")
assert workspace.head_file("config.php") == "<?php\nreturn 'original';\n"
assert workspace.head_file("missing.php") is None

View File

@@ -0,0 +1,27 @@
from unittest.mock import Mock
import pytest
from test_config import settings
from lai_automation.gitea import GiteaClient
def test_create_pull_request_requires_url() -> None:
client = GiteaClient(settings(gitea_token="token"))
response = Mock()
response.json.return_value = {"number": 12}
client.session.post = Mock(return_value=response)
with pytest.raises(ValueError, match="pull request URL"):
client.create_pull_request("ai/failure-1", "Fix", "Body")
def test_create_pull_request_returns_html_url() -> None:
client = GiteaClient(settings(gitea_token="token"))
response = Mock()
response.json.return_value = {"html_url": "https://example.test/owner/lai/pulls/12"}
client.session.post = Mock(return_value=response)
result = client.create_pull_request("ai/failure-1", "Fix", "Body")
assert result.endswith("/pulls/12")

View File

@@ -0,0 +1,924 @@
from dataclasses import replace
from unittest.mock import Mock
from test_config import settings
from lai_automation.git import ChangeSummary
from lai_automation.models import FailureReport
from lai_automation.policy import (
evaluate_changes,
evaluate_expected_behavior,
evaluate_removal_prevention,
evaluate_removal_scope,
evaluate_report_scope,
evaluate_requested_behavior_scope,
evaluate_routed_files,
)
def test_policy_accepts_small_application_change() -> None:
result = evaluate_changes(
ChangeSummary(("app/Services/Example.php", "tests/Feature/ExampleTest.php"), 40, 5),
settings(),
)
assert result.allowed
def test_policy_rejects_secrets_and_vendor_paths() -> None:
result = evaluate_changes(
ChangeSummary((".env", "vendor/foo.php", ".aider.tags.cache.v4/cache.db"), 3, 0),
settings(),
)
assert not result.allowed
assert len(result.errors) == 3
def test_policy_rejects_test_only_repair() -> None:
result = evaluate_changes(
ChangeSummary(("tests/Unit/ExampleTest.php",), 20, 2),
settings(),
)
assert not result.allowed
assert "production behavior was not fixed" in result.errors[0]
def test_direct_main_blocks_validator_and_migrations() -> None:
config = replace(
settings(),
mode="direct-main",
dry_run=False,
allow_direct_main=True,
gitea_token="token",
)
result = evaluate_changes(
ChangeSummary(("app/Services/SQLValidatorService.php", "database/migrations/new.php"), 10, 2),
config,
)
assert not result.allowed
def test_policy_rejects_symlinks(tmp_path) -> None:
(tmp_path / "link.php").symlink_to("/etc/passwd")
result = evaluate_changes(ChangeSummary(("link.php",), 1, 0), settings(), tmp_path)
assert not result.allowed
def test_cross_database_report_requires_application_orchestration() -> None:
report = FailureReport(
id=56,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Use cockpit_config.time_registration",
feedback_type="wrong_result",
expected_behavior="Join cockpit_config manager_id and FK_AuftragPos",
prompt="Hours in April",
response="0 hours",
sql_query="SELECT 0",
error_message=None,
context={},
)
rejected = evaluate_report_scope(
report,
ChangeSummary(("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2),
)
accepted = evaluate_report_scope(
report,
ChangeSummary(("app/Http/Controllers/ChatController.php", "tests/Unit/FixTest.php"), 20, 2),
)
assert not rejected.allowed
assert "staged orchestration" in rejected.errors[0]
assert accepted.allowed
def test_expected_behavior_requires_explicit_technical_terms_in_added_code() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior=(
"FK_Auftrag nach AuftragsKennung 2 filtern und System_created verwenden; "
"obsolet und storniert ausfiltern"
),
prompt="Delivery notes by month",
response=None,
sql_query=None,
error_message=None,
context={},
)
rejected = evaluate_expected_behavior(
report,
"Use FK_Auftrag.Datum_erfassung and status storniert",
)
accepted = evaluate_expected_behavior(
report,
"SELECT DATE_FORMAT(System_created, '%Y-%m') FROM FK_Auftrag "
"WHERE AuftragsKennung = 2 AND COALESCE(obsolet, 0) = 0 "
"AND COALESCE(bStatus_storniert, 0) = 0",
)
assert not rejected.allowed
assert "AuftragsKennung" in rejected.errors[0]
assert "System_created" in rejected.errors[0]
assert accepted.allowed
def test_expected_behavior_rejects_negated_required_date_column() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="System_created verwenden",
prompt="Delivery notes by month",
response=None,
sql_query=None,
error_message=None,
context={},
)
result = evaluate_expected_behavior(
report,
"Lieferscheine niemals mit System_created filtern",
)
assert not result.allowed
assert "contradicts expected identifier System_created" in result.errors[0]
def test_expected_behavior_allows_rejecting_an_alternative_column_on_same_table() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="FK_Auftrag nach System_created filtern",
prompt="Delivery notes by month",
response=None,
sql_query=None,
error_message=None,
context={},
)
result = evaluate_expected_behavior(
report,
"FK_Auftrag nach System_created filtern, nicht nach FK_Auftrag.Datum_erfassung",
)
assert result.allowed
def test_expected_behavior_does_not_treat_later_never_as_table_negation() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="FK_Auftrag nach System_created filtern",
prompt="Delivery notes by month",
response=None,
sql_query=None,
error_message=None,
context={},
)
result = evaluate_expected_behavior(
report,
"FK_Auftrag nach System_created filtern und Datum_erfassung niemals verwenden",
)
assert result.allowed
def test_expected_behavior_rejects_unrequested_filter_copied_from_bad_sql() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="System_created verwenden und status storniert ausfiltern",
prompt="Delivery notes by month",
response=None,
sql_query="WHERE a.bStatus_gebucht = 1",
error_message=None,
context={},
)
result = evaluate_expected_behavior(
report,
"System_created AND bStatus_gebucht = 1 AND COALESCE(bStatus_storniert, 0) = 0",
)
assert not result.allowed
assert "copied bStatus_gebucht" in result.errors[-1]
def test_expected_behavior_requires_one_complete_reference_select() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior=(
"FK_Auftrag nach Auftragskennung 2 filtern und System_created verwenden; "
"obsolet und status storniert ausfiltern"
),
prompt="Delivery notes by month",
response=None,
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS' AND Datum_erfassung >= '2024-01-01'",
error_message=None,
context={},
)
split_rules_and_bad_sql = (
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
"COALESCE(bStatus_storniert, 0) = 0\n"
"SELECT * FROM FK_Auftrag a WHERE Bezeichnung_Kurz = 'LS' "
"AND a.System_created >= '2024-01-01' "
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
)
complete_sql = (
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
"COALESCE(bStatus_storniert, 0) = 0\n"
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
"AND a.System_created >= '2024-01-01' "
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
)
rejected = evaluate_expected_behavior(report, split_rules_and_bad_sql)
accepted = evaluate_expected_behavior(report, complete_sql)
assert not rejected.allowed
assert any("one reference SELECT" in error for error in rejected.errors)
assert any("LS lookup" in error for error in rejected.errors)
assert accepted.allowed
def test_expected_behavior_detects_lowercase_document_code_without_observed_sql() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
prompt="Delivery notes by month",
response=None,
sql_query=None,
error_message=None,
context={},
)
rejected = evaluate_expected_behavior(
report,
"SELECT * FROM FK_Auftrag WHERE Bezeichnung_Kurz = 'LS'",
)
accepted = evaluate_expected_behavior(
report,
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2",
)
assert not rejected.allowed
assert any("AuftragsKennung = 2" in error for error in rejected.errors)
assert accepted.allowed
def test_expected_behavior_rejects_lookup_join_for_authoritative_direct_code() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
prompt="Delivery notes by month",
response=None,
sql_query=None,
error_message=None,
context={},
)
result = evaluate_expected_behavior(
report,
"SELECT * FROM FK_Auftrag a JOIN FK_Nummerkreis n ON n.Kennung = a.AuftragsKennung "
"WHERE a.AuftragsKennung = 2",
)
assert not result.allowed
assert any("joined FK_Nummerkreis" in error for error in result.errors)
def test_expected_behavior_rejects_invented_obsolete_prefix_and_incomplete_assertions() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback=None,
feedback_type="wrong_result",
expected_behavior="Auftragskennung 2, obsolet und status storniert ausfiltern",
prompt="Delivery notes by month",
response=None,
sql_query="SELECT 0",
error_message=None,
context={},
)
invented = (
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
"AND COALESCE(bStatus_obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
"$this->assertStringContainsString('AuftragsKennung = 2', $context);"
)
complete = (
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
"AND COALESCE(obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
"$this->assertStringContainsString('AuftragsKennung = 2', $context);\n"
"$this->assertStringContainsString('COALESCE(obsolet, 0) = 0', $context);\n"
"$this->assertStringContainsString('COALESCE(bStatus_storniert, 0) = 0', $context);"
)
rejected = evaluate_expected_behavior(report, invented)
accepted = evaluate_expected_behavior(report, complete)
assert not rejected.allowed
assert any("obsolete rows" in error for error in rejected.errors)
assert any("Focused regression assertions" in error for error in rejected.errors)
assert accepted.allowed
def test_routed_file_policy_rejects_unexpected_edits() -> None:
result = evaluate_routed_files(
ChangeSummary(("config/erp_semantics.php", "app/Services/AIService.php"), 10, 0),
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
)
assert not result.allowed
assert "app/Services/AIService.php" in result.errors[0]
def test_requested_behavior_scope_requires_all_presentation_and_routing_owners() -> None:
report = FailureReport(
id=7,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
feedback_type="wrong_result",
expected_behavior=(
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
),
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
response="2024-01: 38",
sql_query="SELECT COUNT(*) FROM FK_Auftrag",
error_message=None,
context={"database_type": "erp"},
)
incomplete = evaluate_requested_behavior_scope(
report,
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 10, 0),
"cockpit: www: Markdown-Tabelle",
)
complete = evaluate_requested_behavior_scope(
report,
ChangeSummary((
"app/Services/AIService.php",
"app/Http/Controllers/ChatController.php",
"app/Services/WebSearchService.php",
"tests/Feature/AIPrivacyBoundaryTest.php",
"tests/Unit/ChatControllerSourceResolutionTest.php",
"tests/Unit/WebSearchServiceTest.php",
), 30, 2),
(
"Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024\n"
"Markdown-Tabelle\ncount($rows) > 1\nsingle_row keeps sentence format\n"
"resolveRoutePrefix\n['database', 'Frage']\n['research', 'Frage']\n"
"['direct', 'Frage']\n"
"[null, 'Frage']\n[$explicitRoute, $message] = "
"$this->resolveRoutePrefix($message);\n$route = $explicitRoute ?? $fallback;\n"
"generateWebResearchResponse($message, 'ollama');\n"
"generateDirectResponse($message, 'ollama', []);\n"
"$controller->processMessage($cockpitRequest);\n"
"$ai->expects($this->never())->method('determineRoute');\n"
"function test_cockpit_prefix(): void {\n"
"$tenant->method('requiresCustomerDatabase')->willReturn(false);\n"
"$tenant->method('requiresConfigDatabase')->willReturn(false);\n"
"$tenant->method('requiresErpDatabase')->willReturn(false);\n}\n"
"$controller->processMessage($wwwRequest);\n"
"$ai->expects($this->never())->method('determineRoute');\n"
"$controller->processMessage($directRequest);\n"
"$ai->expects($this->never())->method('determineRoute');\n"
"$this->assertSame(3, substr_count($userPrompt, '\"Monat\"'));\n"
"$this->assertSame(3, substr_count($userPrompt, '\"Anzahl_Lieferscheine\"'));\n"
"cockpit:\nwww:\ndirect:"
),
)
assert not incomplete.allowed
assert any("AIService.php" in error for error in incomplete.errors)
assert any("ChatController.php" in error for error in incomplete.errors)
assert not complete.errors
assert complete.allowed
def test_requested_behavior_scope_rejects_dead_prefix_source_tests() -> None:
report = FailureReport(
id=7,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Als Tabelle anzeigen",
feedback_type="wrong_result",
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
prompt="Anzahl pro Monat",
response="Text",
sql_query="SELECT 1",
error_message=None,
context={"database_type": "erp"},
)
added = """
Markdown-Tabelle
count($rows) > 1
single_row keeps sentence format
resolveRoutePrefix
['database', 'Frage']
['direct', 'Frage']
[null, 'Frage']
[$explicitRoute, $message] = $this->resolveRoutePrefix($message);
$route = $explicitRoute ?? $fallback;
$controller->processMessage($cockpitRequest);
$ai->expects($this->never())->method('determineRoute');
$controller->processMessage($wwwRequest);
$ai->expects($this->never())->method('determineRoute');
$controller->resolveSource('cockpit: Frage', []);
return $this->resolveRoutePrefix($message);
public function determineRouteForMessage() { return $this->determineRoute($message, $this->defaultModel); }
www:
cockpit:
Anzahl pro Monat
"""
result = evaluate_requested_behavior_scope(
report,
ChangeSummary((
"app/Services/AIService.php",
"app/Http/Controllers/ChatController.php",
"tests/Feature/AIPrivacyBoundaryTest.php",
"tests/Unit/ChatControllerSourceResolutionTest.php",
), 40, 2),
added,
)
assert not result.allowed
assert any("do not test resolveDatabaseSource" in error for error in result.errors)
def test_requested_behavior_scope_rejects_helper_only_prefix_tests() -> None:
report = FailureReport(
id=7,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Als Tabelle anzeigen",
feedback_type="wrong_result",
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
prompt="Anzahl pro Monat",
response="Text",
sql_query="SELECT 1",
error_message=None,
context={"database_type": "erp"},
)
added = """
Anzahl pro Monat
Markdown-Tabelle
count($rows) > 1
single_row stays sentence formatted
resolveRoutePrefix
['database', 'Frage']
['direct', 'Frage']
[null, 'Frage']
[$explicitRoute, $message] = $this->resolveRoutePrefix($originalMessage);
$route = $explicitRoute ?? $fallback;
cockpit:
www:
"""
result = evaluate_requested_behavior_scope(
report,
ChangeSummary((
"app/Services/AIService.php",
"app/Http/Controllers/ChatController.php",
"tests/Feature/AIPrivacyBoundaryTest.php",
"tests/Unit/ChatControllerSourceResolutionTest.php",
), 40, 2),
added,
)
assert not result.allowed
assert any("helper-only tests" in error for error in result.errors)
def test_requested_behavior_scope_rejects_incorrect_table_instruction_assertion() -> None:
report = FailureReport(
id=7,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Als Tabelle anzeigen",
feedback_type="wrong_result",
expected_behavior="",
prompt="Anzahl pro Monat",
response="Text",
sql_query="SELECT 1",
error_message=None,
context={"database_type": "erp"},
)
added = """
Anzahl pro Monat
Markdown-Tabelle
count($rows) > 1
single_row stays sentence formatted
$this->assertStringContainsString('alle bereitgestellten Zeilen', $systemPrompt);
"""
result = evaluate_requested_behavior_scope(
report,
ChangeSummary((
"app/Services/AIService.php",
"tests/Feature/AIPrivacyBoundaryTest.php",
), 20, 2),
added,
)
assert not result.allowed
assert any("exact grammatical substring" in error for error in result.errors)
def test_expected_behavior_treats_removed_predicate_as_negative_requirement() -> None:
report = FailureReport(
id=5,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong delivery note count",
feedback_type="wrong_result",
expected_behavior=(
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
"Einfach nur das aus der SQL entfernen"
),
prompt="Delivery notes by month",
response="0 rows",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
accepted = (
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
)
still_present = (
"SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1\n"
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
)
assert evaluate_expected_behavior(report, accepted).allowed
rejected_sql = evaluate_expected_behavior(report, still_present)
rejected_test = evaluate_expected_behavior(
report,
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2",
)
near_match = evaluate_expected_behavior(
report,
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
"$this->assertStringNotContainsString('bStatus_gebucht = 1', $context);",
)
assert not rejected_sql.allowed
assert any("forbidden fragment" in error for error in rejected_sql.errors)
assert not rejected_test.allowed
assert any("negatively assert" in error for error in rejected_test.errors)
assert not near_match.allowed
assert any("near-match with a missing qualifier" in error for error in near_match.errors)
assert any(
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
in error
for error in near_match.errors
)
def test_removal_prevention_rejects_vacuous_test_and_full_forbidden_rule(tmp_path) -> None:
report = FailureReport(
id=6,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Die Abfrage stimmt nicht",
feedback_type="wrong_result",
expected_behavior=(
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
"weil Lieferscheine nicht gebucht werden."
),
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
response="0 Ergebnisse",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
source_file = "config/erp_semantics.php"
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
reference_sql = (
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
)
def semantics(metric: str = "", sql: str = reference_sql) -> str:
return f"""<?php
return [
'metrics' => [
{metric}
],
'examples' => [[
'tables' => ['FK_Auftrag'],
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
'sql' => <<<'SQL'
{sql}
SQL,
]],
];
"""
original_source = semantics()
bad_metric = """ 'Lieferscheine' => [
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
'rules' => [
'Lieferscheine werden nicht gebucht, daher niemals a.bStatus_gebucht = 1 verwenden.',
],
],"""
original_test = """<?php
$context = $service->businessContext(['FK_Auftrag'], 'erp');
$this->assertStringContainsString('AuftragsKennung = 2', $context);
"""
bad_test = original_test + (
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
)
(tmp_path / "config").mkdir()
(tmp_path / "tests/Unit").mkdir(parents=True)
(tmp_path / source_file).write_text(semantics(bad_metric))
(tmp_path / test_file).write_text(bad_test)
workspace = Mock()
workspace.path = tmp_path
workspace.head_file.side_effect = lambda filename: {
source_file: original_source,
test_file: original_test,
}.get(filename)
summary = ChangeSummary((source_file, test_file), 3, 0)
result = evaluate_removal_prevention(report, summary, workspace)
assert not result.allowed
assert any("must not embed the complete forbidden predicate" in error for error in result.errors)
assert any("context-builder test is vacuous" in error for error in result.errors)
def test_removal_prevention_rejects_unrelated_new_semantics(tmp_path) -> None:
report = FailureReport(
id=6,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong delivery note count",
feedback_type="wrong_result",
expected_behavior=(
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
"weil Lieferscheine nicht gebucht werden."
),
prompt="Delivery notes by month",
response="0 rows",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
source_file = "config/erp_semantics.php"
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
original_source = """<?php
return ['examples' => [[
'question' => 'Delivery notes by month',
'sql' => <<<'SQL'
SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
SQL,
]]];
"""
current_source = original_source.replace(
"return [",
"""return [
'metrics' => ['Lieferscheine' => [
'tables' => ['FK_Auftrag'],
'rules' => [
'Lieferscheine werden nicht gebucht; bStatus_gebucht niemals verwenden.',
'Verwende Datum_erfassung oder System_created als Datumsspalte.',
],
]],""",
)
original_test = "<?php\n"
current_test = original_test + (
"$this->assertStringContainsString('Lieferscheine werden nicht gebucht; "
"bStatus_gebucht niemals verwenden.', $context);\n"
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
)
(tmp_path / "config").mkdir()
(tmp_path / "tests/Unit").mkdir(parents=True)
(tmp_path / source_file).write_text(current_source)
(tmp_path / test_file).write_text(current_test)
workspace = Mock()
workspace.path = tmp_path
workspace.head_file.side_effect = lambda filename: {
source_file: original_source,
test_file: original_test,
}.get(filename)
result = evaluate_removal_prevention(
report,
ChangeSummary((source_file, test_file), 7, 0),
workspace,
)
assert not result.allowed
assert any("unrelated business guidance" in error for error in result.errors)
assert any("Datum_erfassung" in error for error in result.errors)
def test_removal_prevention_accepts_exercised_rule_and_protects_reference(tmp_path) -> None:
report = FailureReport(
id=6,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Die Abfrage stimmt nicht",
feedback_type="wrong_result",
expected_behavior=(
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
"weil Lieferscheine nicht gebucht werden."
),
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
response="0 Ergebnisse",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
source_file = "config/erp_semantics.php"
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
reference_sql = (
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
)
def semantics(metric: str = "", sql: str = reference_sql) -> str:
return f"""<?php
return [
'metrics' => [
{metric}
],
'examples' => [[
'tables' => ['FK_Auftrag'],
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
'sql' => <<<'SQL'
{sql}
SQL,
]],
];
"""
rule = (
"Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine "
"niemals verwenden."
)
metric = f""" 'Lieferscheine' => [
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
'rules' => [
'{rule}',
],
],"""
original_source = semantics()
original_test = """<?php
$context = $service->businessContext(['FK_Auftrag'], 'erp');
$this->assertStringContainsString('AuftragsKennung = 2', $context);
"""
current_test = """<?php
$context = $service->businessContext(['FK_Auftrag', 'FK_Nummerkreis'], 'erp');
$this->assertStringContainsString('AuftragsKennung = 2', $context);
$this->assertStringContainsString('Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine niemals verwenden.', $context);
$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);
"""
(tmp_path / "config").mkdir()
(tmp_path / "tests/Unit").mkdir(parents=True)
(tmp_path / source_file).write_text(semantics(metric))
(tmp_path / test_file).write_text(current_test)
workspace = Mock()
workspace.path = tmp_path
workspace.head_file.side_effect = lambda filename: {
source_file: original_source,
test_file: original_test,
}.get(filename)
summary = ChangeSummary((source_file, test_file), 4, 1)
accepted = evaluate_removal_prevention(report, summary, workspace)
assert accepted.allowed
(tmp_path / source_file).write_text(semantics(
metric,
reference_sql.replace("AuftragsKennung = 2", "AuftragsKennung = 4"),
))
regressed = evaluate_removal_prevention(report, summary, workspace)
assert not regressed.allowed
assert any("protected baseline" in error for error in regressed.errors)
def test_removal_only_scope_allows_only_exact_predicate_removal_and_added_test(tmp_path) -> None:
report = FailureReport(
id=5,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong delivery note count",
feedback_type="wrong_result",
expected_behavior=(
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
"Einfach nur das aus der SQL entfernen"
),
prompt="Delivery notes by month in 2024",
response="0 rows",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
original_sql = (
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
"WHERE a.AuftragsKennung = 2 AND a.bStatus_gebucht = 1 "
"AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
)
repaired_sql = (
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
"WHERE a.AuftragsKennung = 2 AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
)
def semantics(sql: str) -> str:
return f"""<?php
return [
'examples' => [[
'question' => 'Delivery notes by month in 2024',
'sql' => <<<'SQL'
{sql}
SQL,
]],
];
"""
source_file = "config/erp_semantics.php"
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
(tmp_path / "config").mkdir()
(tmp_path / "tests/Unit").mkdir(parents=True)
(tmp_path / source_file).write_text(semantics(repaired_sql))
previous_test = "<?php\n$this->assertStringContainsString('System_created', $context);\n"
(tmp_path / test_file).write_text(
previous_test
+ "$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
)
workspace = Mock()
workspace.path = tmp_path
workspace.head_file.side_effect = lambda filename: {
source_file: semantics(original_sql),
test_file: previous_test,
}.get(filename)
summary = ChangeSummary((source_file, test_file), 2, 1)
result = evaluate_removal_scope(report, summary, workspace)
assert result.allowed
(tmp_path / source_file).write_text(semantics(repaired_sql.replace(
"System_created",
"Datum_erfassung",
)))
regression = evaluate_removal_scope(report, summary, workspace)
assert not regression.allowed
assert any("outside the explicitly forbidden" in error for error in regression.errors)

View File

@@ -0,0 +1,367 @@
from lai_automation.models import FailureReport
from lai_automation.prompt import build_retry_task, build_task, redact, repair_files
def test_redact_removes_common_secrets_and_personal_markers() -> None:
value = "password=hunter2 token: abc123 admin@example.com 10.2.3.4 Bearer secret.jwt"
result = redact(value)
assert "hunter2" not in result
assert "abc123" not in result
assert "admin@example.com" not in result
assert "10.2.3.4" not in result
assert "secret.jwt" not in result
def test_task_contains_constraints_and_redacted_report() -> None:
report = FailureReport(
id=7,
chat_message_id=91,
source="user_feedback",
category="incorrect_answer",
user_feedback="password=secret-value",
feedback_type="wrong_join",
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
prompt="List locations for admin@example.com",
response="The wrong product count was returned.",
sql_query="SELECT * FROM company",
error_message=None,
context={"model": "mistral", "database_type": "customer"},
)
task = build_task(report)
assert "failure report #7" in task
assert "secret-value" not in task
assert "admin@example.com" not in task
assert "Do not commit or push" in task
assert "never mock or stub the behavior being tested" in task
assert "Repository ownership map:" in task
assert "config/erp_semantics.php owns ERP joins" in task
assert "Tests must contain assertions that execute" in task
assert "Do not expose protected production methods publicly" in task
assert "Feedback type: wrong_join" in task
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
assert "The wrong product count was returned." in task
def test_redact_removes_extended_credentials() -> None:
private_key = "-----BEGIN PRIVATE KEY-----\nprivate-material\n-----END PRIVATE KEY-----"
value = (
"AKIAIOSFODNN7EXAMPLE "
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.signature "
"mysql://admin:database-password@db.internal/lai "
f"{private_key}"
)
result = redact(value)
assert "AKIAIOSFODNN7EXAMPLE" not in result
assert "eyJhbGci" not in result
assert "database-password" not in result
assert "private-material" not in result
def test_retry_task_contains_failure_without_secrets() -> None:
report = FailureReport(
id=8,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="Use a weighted average",
prompt="Average sale price",
response="Zero",
sql_query="SELECT AVG(list_price)",
error_message=None,
context={"database_type": "erp"},
)
task = build_retry_task(report, RuntimeError("Missing import; token=secret-value"))
assert "Missing import" in task
assert "secret-value" not in task
assert "Do not remove or weaken tests" in task
assert "unrelated production routing" in task
assert "Use a weighted average" in task
assert "SELECT AVG(list_price)" in task
def test_incorrect_answer_task_guides_semantics_level_repair() -> None:
report = FailureReport(
id=42,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="Sum net values and divide by quantity",
prompt="Average sale price for article 9031",
response="0.00",
sql_query="SELECT AVG(Vk_preis) FROM FK_Artikel",
error_message=None,
context={"database_type": "erp"},
)
task = build_task(report)
assert "config/*_semantics.php" in task
assert "Do not hardcode" in task
assert "protected schema/business-" in task
assert "context builder" in task
assert "known-bad SQL" in task
assert "compare every alias.column" in task
assert "Never replace one missing column" in task
assert "Never require or copy production rows" in task
assert repair_files(report) == (
"config/erp_semantics.php",
"tests/Unit/SQLGeneratorBusinessContextTest.php",
)
def test_task_derives_exact_sql_literal_contract() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior=(
"FK_Auftrag nach Auftragskennung 2 filtern, System_created nutzen, "
"obsolet und status storniert ausfiltern"
),
prompt="Delivery notes by month",
response=None,
sql_query="SELECT Datum_erfassung",
error_message=None,
context={"database_type": "erp"},
)
task = build_task(report)
assert "Exact SQL literal contract" in task
assert "a.AuftragsKennung = 2" in task
assert "a.System_created" in task
assert "COALESCE(a.obsolet, 0) = 0" in task
assert "COALESCE(a.bStatus_storniert, 0) = 0" in task
assert "Do not add an LS lookup or bStatus_gebucht" in task
def test_task_derives_exact_negative_contract_for_removal_only_report() -> None:
report = FailureReport(
id=5,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong delivery note count",
feedback_type="wrong_result",
expected_behavior=(
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
"Einfach nur das aus der SQL entfernen"
),
prompt="Delivery notes by month",
response="0 rows",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
task = build_task(report)
assert "Exact SQL removal contract" in task
assert "a.bStatus_gebucht = 1" in task
assert "negative assertion" in task
assert "forbidden identifier is a negative requirement" in task
assert (
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
in task
)
assert "copy them exactly, including aliases and qualifiers" in task
assert "explicitly a removal-only repair" in task
assert "Preserve every other selected field" in task
assert "Exact SQL literal contract" not in task
def test_non_removal_only_contract_protects_correct_reference_and_exercises_rule() -> None:
report = FailureReport(
id=6,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Die Abfrage stimmt nicht",
feedback_type="wrong_result",
expected_behavior=(
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
"weil Lieferscheine nicht gebucht werden."
),
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
response="0 Ergebnisse",
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
error_message=None,
context={"database_type": "erp"},
)
task = build_task(report)
assert "protected correct baseline" in task
assert "preserve it exactly" in task
assert "positively assert the new rule" in task
assert "negative assertion alone is vacuous" in task
assert "without embedding the complete forbidden predicate" in task
assert "Do not add or restate date-column choices" in task
assert "every table in its tables list" in task
assert "supplied to businessContext()" in task
assert "explicitly a removal-only repair" not in task
def test_retry_task_keeps_failure_from_start_of_long_output() -> None:
report = FailureReport(
id=43,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="Top products",
response=None,
sql_query="SELECT a.missing FROM orders a",
error_message="Unknown column",
context={"database_type": "erp"},
)
task = build_retry_task(report, RuntimeError("DECISIVE ASSERTION\n" + "x" * 11_000))
assert "DECISIVE ASSERTION" in task
def test_customer_database_task_explains_tenant_scope() -> None:
report = FailureReport(
id=13,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="What did this customer buy most?",
response=None,
sql_query="SELECT * FROM invoice_row WHERE InvoiceCreator = 'tenant'",
error_message="Unknown column InvoiceCreator",
context={"database_type": "customer"},
)
task = build_task(report)
assert "already selected one tenant database" in task
assert "must not" in task
assert "guessed column" in task
assert "tenant_type" in task
def test_unresolved_placeholder_task_guides_real_answer_path() -> None:
report = FailureReport(
id=49,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Token was not restored",
feedback_type="wrong_result",
expected_behavior="Return the database value",
prompt="Average price",
response="Price: __LAI_UNKNOWN_VALUE__",
sql_query="SELECT AVG(price)",
error_message=None,
context={"database_type": "erp"},
)
task = build_task(report)
assert "AIService::generateDatabaseResponse" in task
assert "exact token map" in task
assert "HTTP-backed AIService test" in task
assert "local database-result fallback" in task
def test_cross_database_report_guides_staged_source_routing() -> None:
report = FailureReport(
id=56,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong hours",
feedback_type="wrong_result",
expected_behavior="Join cockpit_config.time_registration to customer manager and ERP positions",
prompt="Hours in April",
response="0 hours",
sql_query="SELECT SUM(hours) FROM LX_AUFGABE",
error_message=None,
context={"database_type": "erp"},
)
task = build_task(report)
assert "tables from multiple databases" in task
assert "staged-query workflows" in task
assert "Do not put an impossible cross-connection join" in task
assert "preserving tenant isolation" in task
assert "editing only config/*_semantics.php or tests is an invalid repair" in task
assert "describe the data" in task
assert "flow, connection used at each stage" in task
assert repair_files(report) == (
"app/Http/Controllers/ChatController.php",
"app/Services/TenantDatabaseService.php",
"tests/Unit/ChatControllerSourceResolutionTest.php",
)
def test_presentation_and_route_feedback_is_routed_to_real_owners() -> None:
report = FailureReport(
id=7,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
feedback_type="wrong_result",
expected_behavior=(
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
),
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
response="2024-01: 38, 2024-02: 95",
sql_query="SELECT Monat, COUNT(*) FROM FK_Auftrag GROUP BY Monat",
error_message=None,
context={"database_type": "erp"},
)
assert repair_files(report) == (
"app/Services/AIService.php",
"app/Http/Controllers/ChatController.php",
"app/Services/WebSearchService.php",
"tests/Feature/AIPrivacyBoundaryTest.php",
"tests/Unit/ChatControllerSourceResolutionTest.php",
"tests/Unit/WebSearchServiceTest.php",
)
task = build_task(report)
assert "real answer-formatting change" in task
assert "exact reported user prompt" in task
assert '"cockpit:" must force the database path' in task
assert '"www:" must force public web research' in task
assert '"direct:" must' in task
assert "including Ollama/Qwen" in task
assert "Do not pass private conversation/database context" in task
assert "AIService::determineRoute() must not be called" in task
assert "resolveRoutePrefix(string $message): array" in task
assert "[$explicitRoute, $message] = $this->resolveRoutePrefix($message)" in task
assert "Do not add prefix handling to resolveDatabaseSource()" in task
assert "parent::resolveRoutePrefix($message)" in task
assert "three tests that call processMessage() itself" in task
assert "setLaravelSession()" in task
assert "generateWebResearchResponse()" in task
assert "columns named exactly Monat and Anzahl_Lieferscheine" in task
assert 'exact substring "allen bereitgestellten Zeilen"' in task
assert "substr_count($userPrompt" in task
assert "heuristics must return false" in task
assert "persisted ChatMessage.message equals the cleaned question exactly" in task
assert "$response->getData(true)" in task
assert "Do not write “we can't fully test processMessage”" in task
assert "one ordinary result row with multiple" in task
assert "duplicate method declaration" in task
assert "All three prefix behaviors" in task
assert "config/*_semantics.php" in task

View File

@@ -0,0 +1,145 @@
from pathlib import Path
from lai_automation.models import FailureReport
from lai_automation.removal import (
detect_already_fixed,
extract_reference_examples,
find_matching_reference,
is_removal_only,
removal_directives,
remove_sql_fragment,
sql_contains_fragment,
)
def removal_report(**overrides) -> FailureReport:
values = {
"id": 5,
"chat_message_id": 1,
"source": "user_feedback",
"category": "incorrect_answer",
"user_feedback": "Die Anzahl der Lieferscheine ist falsch",
"feedback_type": "wrong_result",
"expected_behavior": (
'Es muss " a.bStatus_gebucht = 1 " entfernt werden! '
"Lieferscheine werden nicht gebucht! Einfach nur das aus der SQL entfernen"
),
"prompt": "Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
"response": "0 Ergebnisse",
"sql_query": (
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
"AND a.bStatus_gebucht = 1 AND COALESCE(a.bStatus_storniert, 0) = 0"
),
"error_message": None,
"context": {"database_type": "erp"},
}
values.update(overrides)
return FailureReport(**values)
def semantics(sql: str, question: str | None = None) -> str:
question = question or "Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024"
return f"""<?php
return [
'examples' => [
[
'question' => '{question}',
'sql' => <<<'SQL'
{sql}
SQL,
],
],
];
"""
def test_extracts_exact_quoted_german_removal_contract() -> None:
directives = removal_directives(removal_report())
assert tuple(item.fragment for item in directives) == ("a.bStatus_gebucht = 1",)
assert is_removal_only(removal_report())
def test_infers_unquoted_predicate_from_observed_sql() -> None:
report = removal_report(
expected_behavior="Bitte den bStatus_gebucht Filter aus der SQL entfernen",
)
directives = removal_directives(report)
assert len(directives) == 1
assert sql_contains_fragment(directives[0].fragment, "a.bStatus_gebucht = 1")
def test_does_not_invert_do_not_remove_instruction() -> None:
report = removal_report(
expected_behavior="Do not remove `a.bStatus_gebucht = 1` from invoice SQL",
)
assert removal_directives(report) == ()
assert not is_removal_only(report)
def test_removes_only_the_requested_conjunct() -> None:
sql = (
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung=2 "
"AND a.bStatus_gebucht=1 AND COALESCE(a.obsolet,0)=0 GROUP BY a.SheetNr;"
)
repaired = remove_sql_fragment(sql, "a.bStatus_gebucht = 1")
assert not sql_contains_fragment(repaired, "a.bStatus_gebucht = 1")
assert "a.AuftragsKennung=2" in repaired
assert "COALESCE(a.obsolet,0)=0" in repaired
assert "GROUP BY a.SheetNr" in repaired
def test_extracts_and_uniquely_matches_reference_example() -> None:
content = semantics("SELECT 1") + semantics(
"SELECT 2",
"Welcher Kunde hatte im Januar den meisten Umsatz?",
)
examples = extract_reference_examples(content)
match, score = find_matching_reference(content, removal_report().prompt)
assert len(examples) == 2
assert match is not None
assert match.sql == "SELECT 1"
assert score == 1.0
def test_detects_already_fixed_only_with_matching_reference(tmp_path: Path) -> None:
source = tmp_path / "config/erp_semantics.php"
source.parent.mkdir(parents=True)
source.write_text(semantics(
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
))
result = detect_already_fixed(
removal_report(),
tmp_path,
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
)
assert result is not None
assert result.filename == "config/erp_semantics.php"
assert result.match_score == 1.0
assert result.forbidden_fragments == ("a.bStatus_gebucht = 1",)
def test_does_not_mark_additive_or_mismatched_report_already_fixed(tmp_path: Path) -> None:
source = tmp_path / "config/erp_semantics.php"
source.parent.mkdir(parents=True)
source.write_text(semantics("SELECT COUNT(*) FROM FK_Auftrag a"))
additive = removal_report(
expected_behavior=(
'"a.bStatus_gebucht = 1" entfernen und zusätzlich System_created verwenden'
),
)
mismatched = removal_report(prompt="Zeige Rechnungen im Jahr 2024")
assert detect_already_fixed(additive, tmp_path, ("config/erp_semantics.php",)) is None
assert detect_already_fixed(mismatched, tmp_path, ("config/erp_semantics.php",)) is None

View File

@@ -0,0 +1,968 @@
import threading
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
import pytest
from test_config import settings
from lai_automation.agent import AgentProviderError, AgentTimeoutError
from lai_automation.git import ChangeSummary
from lai_automation.models import FailureReport
from lai_automation.worker import AutomationWorker, ValidationSetupError
def test_dry_run_generates_task_without_agent_or_git() -> None:
report = FailureReport(
id=11,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List locations",
response=None,
sql_query=None,
error_message="Unknown column",
context={},
)
worker = AutomationWorker(settings())
worker.reports = Mock()
worker.reports.claim_next.return_value = report
assert worker.run_once()
worker.reports.release.assert_called_once()
worker.agent = Mock()
worker.agent.run.assert_not_called()
def test_stop_signal_interrupts_worker_wait() -> None:
worker = AutomationWorker(settings())
worker._stop(15, None)
assert worker.stop_event.is_set()
def test_pull_request_body_contains_redacted_original_report() -> None:
report = FailureReport(
id=7,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Use a table\ntoken=super-secret-token",
feedback_type="wrong_result",
expected_behavior="Show every supplied row",
prompt="Count delivery notes by month",
response="January: 38, February: 95",
sql_query="SELECT month, COUNT(*) FROM deliveries GROUP BY month",
error_message=None,
context={},
)
summary = ChangeSummary(("app/Services/AIService.php",), 5, 1)
body = AutomationWorker._pull_request_body(
report,
summary,
[{"command": "php artisan test", "returncode": 0}],
)
assert "## Original failure report" in body
assert "**Source**\n> user_feedback" in body
assert "**Category**\n> incorrect_answer" in body
assert "**Feedback type**\n> wrong_result" in body
assert "**User prompt**\n> Count delivery notes by month" in body
assert "**Observed response**\n> January: 38, February: 95" in body
assert "**User feedback**\n> Use a table\n> token=[REDACTED]" in body
assert "**Expected behavior**\n> Show every supplied row" in body
assert "**Error message**\n> _Not provided._" in body
assert "**Generated SQL**\n> SELECT month, COUNT(*)" in body
assert "super-secret-token" not in body
assert "## Scope\n- `app/Services/AIService.php`" in body
assert "## Validation\n- PASS `php artisan test`" in body
def test_database_credentials_are_only_passed_to_test_process(tmp_path) -> None:
worker = AutomationWorker(settings())
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.path = tmp_path
workspace.run.return_value = Mock(returncode=0, stdout="ok", stderr="")
results = worker._run_tests(workspace)
assert results[0]["returncode"] == 0
environment = workspace.run.call_args.kwargs["extra_environment"]
assert environment["AGENT_TEST_DB_ENABLED"] == "true"
assert environment["APP_ENV"] == "testing"
assert environment["AI_MISTRAL_API_KEY"] == "automation-test-key"
assert environment["TEST_DB_USER"] == "test_runner"
assert environment["TEST_DB_PASSWORD"] == "test-secret"
assert environment["DB_DATABASE"] == "lai_agent_test"
assert environment["ERP_DATABASE"] == "customer_agent_test"
assert environment["COCKPIT_CONFIG_DATABASE"] == "cockpit_agent_test"
def test_workspace_setup_creates_safe_env_and_prepares_assets(tmp_path) -> None:
worker = AutomationWorker(settings(
workspace_root=tmp_path,
setup_commands=("npm run build",),
))
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.path = tmp_path / "failure-1"
workspace.path.mkdir()
workspace.run.return_value = Mock(returncode=0, stdout="built", stderr="")
results = worker._prepare_workspace_for_agent(workspace, 1)
env_contents = (workspace.path / ".env").read_text()
assert "APP_ENV=testing" in env_contents
assert "DB_PASSWORD" not in env_contents
assert results[0]["stage"] == "workspace setup"
assert workspace.run.call_args.kwargs["extra_environment"]["DB_PASSWORD"] == "test-secret"
def test_processing_connection_failure_is_not_sent_to_agent() -> None:
report = FailureReport(
id=15,
chat_message_id=1,
source="automatic",
category="processing",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List customers",
response=None,
sql_query=None,
error_message="Access denied for user 'runtime'@'host'",
context={},
)
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
assert worker.run_once()
worker.agent.run.assert_not_called()
worker.reports.fail.assert_called_once()
assert worker.reports.fail.call_args.args[2]["classification"] == "operational"
def test_model_provider_failure_is_classified_as_operational() -> None:
report = FailureReport(
id=16,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="Use the correct rule",
prompt="List records",
response="0 rows",
sql_query="SELECT 0",
error_message=None,
context={},
)
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = AgentProviderError("Mistral DNS failed")
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-16"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
worker.reports.fail.assert_called_once()
metadata = worker.reports.fail.call_args.args[2]
assert metadata["classification"] == "operational"
assert metadata["automation_action"] == "no_source_change"
workspace.commit.assert_not_called()
def test_test_schema_creates_minimal_erp_lookup(monkeypatch) -> None:
worker = AutomationWorker(settings())
cursor = Mock()
connection = MagicMock()
connection.cursor.return_value.__enter__.return_value = cursor
connect = Mock(return_value=connection)
monkeypatch.setattr("lai_automation.worker.pymysql.connect", connect)
worker._prepare_test_schema()
assert connect.call_args.kwargs["database"] == "customer_agent_test"
assert "CREATE TABLE IF NOT EXISTS FK_Nummerkreis" in cursor.execute.call_args.args[0]
assert cursor.executemany.call_args.args[1][0] == (3, "Rechnung", "RG")
connection.close.assert_called_once()
def test_validation_commands_are_logged_without_output(caplog, tmp_path) -> None:
worker = AutomationWorker(settings())
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.path = tmp_path
workspace.run.return_value = Mock(returncode=0, stdout="sensitive output", stderr="")
with caplog.at_level("INFO"):
worker._run_tests(workspace, report_id=7, attempt=2)
assert "Report 7 attempt 2 validation command: php artisan test starting" in caplog.text
assert "returncode=0" in caplog.text
assert "sensitive output" not in caplog.text
def test_failed_validation_preserves_start_and_end_of_long_output(tmp_path) -> None:
worker = AutomationWorker(settings())
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.path = tmp_path
workspace.run.return_value = Mock(
returncode=1,
stdout="DECISIVE ASSERTION\n" + "x" * 13_000,
stderr="FINAL ERROR",
)
try:
worker._run_tests(workspace)
except RuntimeError as exception:
failure = str(exception)
else:
raise AssertionError("Expected failed validation")
assert "DECISIVE ASSERTION" in failure
assert "...[middle omitted]..." in failure
assert "FINAL ERROR" in failure
def test_validation_signature_ignores_counts_timings_and_line_numbers() -> None:
first = """
FAILED Tests\\Unit\\RouteTest > prefixes are removed
Expectation failed for method determineRoute when invoked 1 time.
Method was expected to be called 1 time, actually called 0 times.
Tests: 1 failed, 50 passed (186 assertions)
Duration: 0.49s
at tests/Unit/RouteTest.php:229
"""
second = """
FAILED Tests\\Unit\\RouteTest > prefixes are removed
Expectation failed for method determineRoute when invoked 1 time.
Method was expected to be called 1 time, actually called 0 times.
Tests: 1 failed, 50 passed (191 assertions)
Duration: 0.50s
at tests/Unit/RouteTest.php:269
"""
assert AutomationWorker._validation_signature(first) == AutomationWorker._validation_signature(
second
)
def test_validation_setup_failure_is_typed_and_not_a_test_failure(tmp_path) -> None:
worker = AutomationWorker(settings(setup_commands=("npm ci",)))
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.path = tmp_path
workspace.run.return_value = Mock(
returncode=1,
stdout="network unavailable",
stderr="",
)
with pytest.raises(ValidationSetupError, match="Validation setup command failed"):
worker._run_tests(workspace)
def test_long_stage_emits_heartbeat(caplog) -> None:
worker = AutomationWorker(settings())
with caplog.at_level("INFO"), worker._progress(7, "agent attempt 1", interval=0.001):
threading.Event().wait(0.01)
assert "Report 7 agent attempt 1 still running" in caplog.text
assert "Report 7 agent attempt 1 finished" in caplog.text
def test_long_stage_refreshes_database_claim_lease() -> None:
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
with worker._progress(7, "agent attempt 1", interval=0.001):
threading.Event().wait(0.01)
assert worker.reports.heartbeat.call_count >= 2
def test_timed_out_agent_partial_changes_are_validated_and_published() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="Use System_created",
prompt="Delivery notes by month",
response="0 rows",
sql_query="SELECT 0",
error_message=None,
context={"database_type": "erp"},
)
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = AgentTimeoutError("timed out after applying edits")
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock(return_value=[])
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-4"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
20,
2,
)
workspace.added_text.return_value = (
"Use DATE_FORMAT(System_created, '%Y-%m') and AuftragsKennung = 2; "
"COALESCE(obsolet, 0) = 0 and COALESCE(bStatus_storniert, 0) = 0"
)
workspace.run_args.return_value = Mock(
returncode=0,
stdout="No syntax errors",
stderr="",
)
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
worker._run_tests.assert_called_once()
workspace.push_branch.assert_called_once()
worker.reports.complete.assert_called_once()
def test_provider_failure_partial_changes_are_validated_and_published() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="Use System_created",
prompt="Delivery notes by month",
response="0 rows",
sql_query="SELECT 0",
error_message=None,
context={"database_type": "erp"},
)
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = AgentProviderError("provider failed after edits")
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock(return_value=[])
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-4"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
20,
2,
)
workspace.added_text.return_value = (
"Use DATE_FORMAT(System_created, '%Y-%m') and AuftragsKennung = 2; "
"COALESCE(obsolet, 0) = 0 and COALESCE(bStatus_storniert, 0) = 0"
)
workspace.run_args.return_value = Mock(
returncode=0,
stdout="No syntax errors",
stderr="",
)
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
worker._run_tests.assert_called_once()
workspace.push_branch.assert_called_once()
worker.reports.complete.assert_called_once()
def test_php_syntax_failure_discards_attempt_before_full_validation() -> None:
report = FailureReport(
id=56,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong hours",
feedback_type="wrong_result",
expected_behavior="Use time registration",
prompt="Hours in April",
response="0 hours",
sql_query="SELECT 0",
error_message=None,
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["broken PHP", "fixed PHP"]
worker.gitea = Mock()
worker._run_tests = Mock(return_value=[])
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-56"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
)
workspace.run_args.side_effect = [
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
]
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+intended fix"
workspace.checkpoint.return_value = "valid-checkpoint"
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "unexpected token" in worker.agent.run.call_args_list[1].args[1]
assert "clean repository" in worker.agent.run.call_args_list[1].args[1]
assert "intended fix" in worker.agent.run.call_args_list[1].args[1]
assert "repair-only attempt" in worker.agent.run.call_args_list[1].args[1]
workspace.restore_checkpoint.assert_called_once_with(None)
worker._run_tests.assert_called_once()
def test_malformed_retry_restores_last_syntax_valid_attempt() -> None:
report = FailureReport(
id=56,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong hours",
feedback_type="wrong_result",
expected_behavior="Use time registration",
prompt="Hours in April",
response="0 hours",
sql_query="SELECT 0",
error_message=None,
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3)
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["valid but failing", "broken retry", "corrected retry"]
worker.gitea = Mock()
worker._run_tests = Mock(side_effect=[RuntimeError("Expected hour rule"), []])
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-56"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
)
workspace.run_args.side_effect = [
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
]
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+application repair"
workspace.checkpoint.side_effect = ["checkpoint-1", "checkpoint-3"]
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 3
workspace.restore_checkpoint.assert_called_once_with("checkpoint-1")
assert "last syntax-valid changes were restored" in worker.agent.run.call_args_list[2].args[1]
assert "application repair" in worker.agent.run.call_args_list[2].args[1]
assert worker._run_tests.call_count == 2
def test_failed_validation_gets_one_correction_before_pr() -> None:
report = FailureReport(
id=13,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List products",
response=None,
sql_query="SELECT missing_column FROM invoice_row",
error_message="Unknown column",
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["first change", "corrected change"]
worker.gitea = Mock()
worker._run_tests = Mock(side_effect=[RuntimeError("Missing import"), []])
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-13"
workspace.summary.return_value = ChangeSummary(
("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1
)
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "Missing import" in worker.agent.run.call_args_list[1].args[1]
assert worker._run_tests.call_count == 2
workspace.push_branch.assert_called_once()
worker.reports.complete.assert_called_once()
def test_failed_validation_retry_keeps_full_routed_context() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="AuftragsKennung 2 verwenden",
prompt="Delivery notes by month",
response=None,
sql_query="SELECT Datum_erfassung",
error_message=None,
context={"database_type": "erp"},
)
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["valid production with bad test", "fixed assertion"]
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock(return_value=[])
worker._run_tests = Mock(side_effect=[RuntimeError("case-sensitive assertion failed"), []])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-4"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
20,
1,
)
added = (
"AuftragsKennung = 2\n"
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2"
)
workspace.added_text.return_value = added
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.checkpoint.return_value = "valid-production"
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_args_list[1].args[2] == (
"config/erp_semantics.php",
"tests/Unit/SQLGeneratorBusinessContextTest.php",
)
retry_task = worker.agent.run.call_args_list[1].args[1]
assert "inspect every routed production and test file" in retry_task
assert "Do not merely rewrite assertions" in retry_task
assert "case-sensitive" in worker.agent.run.call_args_list[1].args[1]
attempts = worker.reports.complete.call_args.args[1]["agent_attempts"]
assert attempts[0]["validation_status"] == "failed"
assert "case-sensitive assertion failed" in attempts[0]["validation_error_tail"]
assert attempts[1]["validation_status"] == "passed"
def test_no_change_response_gets_one_correction() -> None:
report = FailureReport(
id=14,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List products",
response=None,
sql_query=None,
error_message="Unknown column",
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["no edit", "applied edit"]
worker.gitea = Mock()
worker._run_tests = Mock(return_value=[])
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-14"
workspace.summary.side_effect = [
ChangeSummary((), 0, 0),
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
]
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "Agent produced no file changes" in worker.agent.run.call_args_list[1].args[1]
worker._run_tests.assert_called_once()
workspace.push_branch.assert_called_once()
def test_test_only_change_gets_correction_before_validation() -> None:
report = FailureReport(
id=49,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Token was not restored",
feedback_type="wrong_result",
expected_behavior="Return the real value",
prompt="Average price",
response="__LAI_UNKNOWN__",
sql_query="SELECT AVG(price)",
error_message=None,
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["tests only", "production fix"]
worker.gitea = Mock()
worker._run_tests = Mock(return_value=[])
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-49"
workspace.summary.side_effect = [
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 8, 0),
ChangeSummary(
("app/Services/AIService.php", "tests/Feature/AIPrivacyBoundaryTest.php"),
15,
1,
),
ChangeSummary(
("app/Services/AIService.php", "tests/Feature/AIPrivacyBoundaryTest.php"),
15,
1,
),
]
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "changed only tests" in worker.agent.run.call_args_list[1].args[1]
assert "exact repository-relative paths" in worker.agent.run.call_args_list[1].args[1]
workspace.checkpoint.assert_called_once()
worker._run_tests.assert_called_once()
def test_semantically_invalid_change_is_not_kept_as_retry_checkpoint() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="System_created verwenden und obsolet ausfiltern",
prompt="Delivery notes by month",
response="0 rows",
sql_query="SELECT Datum_erfassung",
error_message=None,
context={"database_type": "erp"},
)
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["missing filter", "correct repair"]
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock(return_value=[])
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-4"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
12,
1,
)
workspace.added_text.side_effect = [
"System_created",
"System_created AND COALESCE(obsolet, 0) = 0",
"System_created AND COALESCE(obsolet, 0) = 0",
]
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.checkpoint.return_value = "policy-valid-checkpoint"
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
workspace.restore_checkpoint.assert_called_once_with(None)
workspace.checkpoint.assert_called_once()
worker._run_tests.assert_called_once()
def test_policy_constraints_accumulate_across_retries() -> None:
report = FailureReport(
id=4,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong result",
feedback_type="wrong_result",
expected_behavior="AuftragsKennung 2 und System_created verwenden; obsolet ausfiltern",
prompt="Delivery notes by month",
response=None,
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS'",
error_message=None,
context={"database_type": "erp"},
)
worker = AutomationWorker(settings(
mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3
))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["incomplete", "test only", "complete"]
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock(return_value=[])
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-4"
workspace.summary.side_effect = [
ChangeSummary(("config/erp_semantics.php",), 5, 0),
ChangeSummary(("tests/Unit/SQLGeneratorBusinessContextTest.php",), 5, 0),
ChangeSummary(
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
20,
1,
),
ChangeSummary(
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
20,
1,
),
]
complete = (
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0\n"
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
"AND System_created IS NOT NULL AND COALESCE(obsolet, 0) = 0"
)
workspace.added_text.side_effect = ["AuftragsKennung = 2", "test", complete, complete]
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.checkpoint.return_value = "valid"
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
third_task = worker.agent.run.call_args_list[2].args[1]
assert "All policy constraints from earlier attempts remain mandatory" in third_task
assert "one reference SELECT" in third_task
assert "changed only tests" in third_task
def test_already_fixed_removal_report_skips_agent_and_dependency_setup(tmp_path) -> None:
report = FailureReport(
id=5,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong delivery note count",
feedback_type="wrong_result",
expected_behavior=(
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
"Einfach nur das aus der SQL entfernen"
),
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
response="0 rows",
sql_query=(
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
"AND a.bStatus_gebucht = 1"
),
error_message=None,
context={"database_type": "erp"},
)
source = tmp_path / "config/erp_semantics.php"
source.parent.mkdir(parents=True)
source.write_text("""<?php
return ['examples' => [[
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
'sql' => <<<'SQL'
SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
SQL,
]]];
""")
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock()
workspace = Mock()
workspace.path = tmp_path
workspace.branch = "ai/failure-5"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
workspace.prepare.assert_called_once()
worker._prepare_workspace_for_agent.assert_not_called()
worker.agent.run.assert_not_called()
workspace.commit.assert_not_called()
workspace.push_branch.assert_not_called()
worker.gitea.create_pull_request.assert_not_called()
worker.reports.complete.assert_called_once()
metadata = worker.reports.complete.call_args.args[1]
assert metadata["classification"] == "already_fixed"
assert metadata["automation_action"] == "no_source_change"
assert metadata["forbidden_fragments"] == ["a.bStatus_gebucht = 1"]
def test_identical_policy_failure_stops_after_two_attempts() -> None:
report = FailureReport(
id=18,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List generic records",
response=None,
sql_query="SELECT missing FROM records",
error_message="Unknown column",
context={},
)
worker = AutomationWorker(settings(
mode="pr",
dry_run=False,
gitea_token="token",
max_agent_attempts=5,
))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["tests only", "still tests only", "must not run"]
worker._prepare_workspace_for_agent = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-18"
workspace.summary.side_effect = [
ChangeSummary(("tests/Unit/FixTest.php",), 2, 0),
ChangeSummary(("tests/Unit/FixTest.php",), 3, 0),
]
workspace.added_text.return_value = ""
workspace.diff.return_value = "same rejected candidate"
workspace.run_args.return_value = Mock(
returncode=0,
stdout="No syntax errors",
stderr="",
)
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
workspace.commit.assert_not_called()
worker.reports.fail.assert_called_once()
assert "Repeated identical candidate rejection" in worker.reports.fail.call_args.args[1]
def test_same_policy_error_with_changed_candidate_can_progress() -> None:
report = FailureReport(
id=18,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List generic records",
response=None,
sql_query="SELECT missing FROM records",
error_message="Unknown column",
context={},
)
worker = AutomationWorker(settings(
mode="pr",
dry_run=False,
gitea_token="token",
max_agent_attempts=3,
))
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["first tests only", "improved tests only", "complete fix"]
worker.gitea = Mock()
worker._prepare_workspace_for_agent = Mock(return_value=[])
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-18"
workspace.summary.side_effect = [
ChangeSummary(("tests/Unit/FixTest.php",), 2, 0),
ChangeSummary(("tests/Unit/FixTest.php",), 3, 0),
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 8, 1),
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 8, 1),
]
workspace.added_text.return_value = ""
workspace.diff.side_effect = ["first rejected candidate", "second rejected candidate"]
workspace.run_args.return_value = Mock(
returncode=0,
stdout="No syntax errors",
stderr="",
)
workspace.checkpoint.return_value = "valid"
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 3
worker._run_tests.assert_called_once()
workspace.push_branch.assert_called_once()
worker.reports.complete.assert_called_once()