chore: archive previous Aider worker
This commit is contained in:
201
old_2026-08-31/tests/test_agent.py
Normal file
201
old_2026-08-31/tests/test_agent.py
Normal file
@@ -0,0 +1,201 @@
|
||||
import io
|
||||
import signal
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.agent import AgentProviderError, AgentTimeoutError, AiderAgent
|
||||
|
||||
|
||||
def test_aider_command_cannot_mutate_git_configuration_or_send_analytics() -> None:
|
||||
command = AiderAgent(settings()).command(Path("/repo/.git/task.txt"))
|
||||
|
||||
assert "--no-gitignore" in command
|
||||
assert "--no-analytics" in command
|
||||
assert "--no-auto-commits" in command
|
||||
assert "--no-git" in command
|
||||
assert "--stream" in command
|
||||
assert "--no-pretty" in command
|
||||
assert "--no-check-update" in command
|
||||
assert "--no-suggest-shell-commands" in command
|
||||
assert "--no-detect-urls" in command
|
||||
assert "--disable-playwright" in command
|
||||
assert "--no-auto-lint" in command
|
||||
assert command[command.index("--map-tokens") + 1] == "2048"
|
||||
assert command.count("/dev/null") == 2
|
||||
assert "--edit-format" not in command
|
||||
assert command[-2:] == ["--timeout", "900"]
|
||||
|
||||
|
||||
def test_aider_command_adds_existing_routed_files(tmp_path) -> None:
|
||||
git_dir = tmp_path / ".git"
|
||||
git_dir.mkdir()
|
||||
controller = tmp_path / "app/Http/Controllers/ChatController.php"
|
||||
controller.parent.mkdir(parents=True)
|
||||
controller.write_text("<?php\n")
|
||||
|
||||
command = AiderAgent(settings()).command(
|
||||
git_dir / "task.txt",
|
||||
("app/Http/Controllers/ChatController.php", "missing.php"),
|
||||
)
|
||||
|
||||
assert ["--file", "app/Http/Controllers/ChatController.php"] == command[
|
||||
command.index("--file"):command.index("--file") + 2
|
||||
]
|
||||
assert "missing.php" not in command
|
||||
assert command[command.index("--map-tokens") + 1] == "0"
|
||||
assert command[command.index("--edit-format") + 1] == "diff"
|
||||
assert "--architect" not in command
|
||||
assert "--editor-model" not in command
|
||||
|
||||
|
||||
def test_unrouted_aider_command_stays_in_direct_edit_mode() -> None:
|
||||
command = AiderAgent(settings()).command(Path("/repo/.git/task.txt"))
|
||||
|
||||
assert "--architect" not in command
|
||||
|
||||
|
||||
def test_aider_detects_output_stall_at_configured_limit() -> None:
|
||||
agent = AiderAgent(settings(agent_idle_timeout_seconds=120))
|
||||
|
||||
assert not agent._output_stalled(119.9)
|
||||
assert agent._output_stalled(120)
|
||||
|
||||
|
||||
def test_ollama_aider_command_uses_single_whole_file_pass(tmp_path) -> None:
|
||||
git_dir = tmp_path / ".git"
|
||||
git_dir.mkdir()
|
||||
source = tmp_path / "config/erp_semantics.php"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text("<?php\n")
|
||||
|
||||
command = AiderAgent(settings(agent_model="ollama_chat/qwen3-coder:30b")).command(
|
||||
git_dir / "task.txt",
|
||||
("config/erp_semantics.php",),
|
||||
)
|
||||
|
||||
assert command[command.index("--edit-format") + 1] == "whole"
|
||||
assert "--no-auto-lint" in command
|
||||
assert "--architect" not in command
|
||||
assert "--editor-model" not in command
|
||||
|
||||
|
||||
def test_ollama_test_only_correction_uses_precise_diff_format(tmp_path) -> None:
|
||||
git_dir = tmp_path / ".git"
|
||||
git_dir.mkdir()
|
||||
test_file = tmp_path / "tests/Unit/FixTest.php"
|
||||
test_file.parent.mkdir(parents=True)
|
||||
test_file.write_text("<?php\n")
|
||||
|
||||
command = AiderAgent(settings(agent_model="ollama_chat/qwen3-coder:30b")).command(
|
||||
git_dir / "task.txt",
|
||||
("tests/Unit/FixTest.php",),
|
||||
)
|
||||
|
||||
assert command[command.index("--edit-format") + 1] == "diff"
|
||||
assert "--no-auto-lint" in command
|
||||
|
||||
|
||||
def test_aider_timeout_has_actionable_error(tmp_path, monkeypatch) -> None:
|
||||
workspace = tmp_path / "repo"
|
||||
(workspace / ".git").mkdir(parents=True)
|
||||
|
||||
class TimedOutProcess:
|
||||
pid = 4321
|
||||
stdout = io.StringIO("partial output\n")
|
||||
stderr = io.StringIO("")
|
||||
returncode = None
|
||||
|
||||
def wait(self, timeout=None):
|
||||
raise subprocess.TimeoutExpired("aider", timeout)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lai_automation.agent.subprocess.Popen",
|
||||
lambda *args, **kwargs: TimedOutProcess(),
|
||||
)
|
||||
signals = []
|
||||
monkeypatch.setattr(
|
||||
"lai_automation.agent.os.killpg",
|
||||
lambda pid, sent_signal: signals.append((pid, sent_signal)),
|
||||
)
|
||||
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
|
||||
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
|
||||
)())
|
||||
|
||||
with pytest.raises(AgentTimeoutError, match="partial output"):
|
||||
AiderAgent(settings(agent_timeout_seconds=60)).run(workspace, "fix it")
|
||||
|
||||
assert signals == [(4321, signal.SIGTERM), (4321, signal.SIGKILL)]
|
||||
|
||||
|
||||
def test_aider_environment_does_not_include_database_credentials(tmp_path, monkeypatch) -> None:
|
||||
config = settings(workspace_root=tmp_path)
|
||||
workspace = tmp_path / "repo"
|
||||
(workspace / ".git").mkdir(parents=True)
|
||||
monkeypatch.setenv("TEST_DB_PASSWORD", "must-not-reach-aider")
|
||||
monkeypatch.setenv("OPENROUTER_API_KEY", "qwen-provider-key")
|
||||
monkeypatch.setenv("OLLAMA_API_BASE", "http://host.docker.internal:11434")
|
||||
captured = {}
|
||||
|
||||
class SuccessfulProcess:
|
||||
pid = 1234
|
||||
stdout = io.StringIO("provider said qwen-provider-key\n")
|
||||
stderr = io.StringIO("")
|
||||
returncode = 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return self.returncode
|
||||
|
||||
def popen(command, **kwargs):
|
||||
captured.update(kwargs["env"])
|
||||
return SuccessfulProcess()
|
||||
|
||||
monkeypatch.setattr("lai_automation.agent.subprocess.Popen", popen)
|
||||
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
|
||||
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
|
||||
)())
|
||||
|
||||
output = AiderAgent(config).run(workspace, "fix it")
|
||||
|
||||
assert "TEST_DB_PASSWORD" not in captured
|
||||
assert "DB_PASSWORD" not in captured
|
||||
assert captured["OPENROUTER_API_KEY"] == "qwen-provider-key"
|
||||
assert captured["OLLAMA_API_BASE"] == "http://host.docker.internal:11434"
|
||||
assert "qwen-provider-key" not in output
|
||||
assert "[REDACTED]" in output
|
||||
|
||||
|
||||
def test_successful_exit_with_only_provider_failure_is_rejected(tmp_path, monkeypatch) -> None:
|
||||
workspace = tmp_path / "repo"
|
||||
(workspace / ".git").mkdir(parents=True)
|
||||
|
||||
class ProviderFailedProcess:
|
||||
pid = 1235
|
||||
stdout = io.StringIO(
|
||||
"litellm.InternalServerError: MistralException - Name or service not known\n"
|
||||
"The API provider's servers are down or overloaded.\n"
|
||||
)
|
||||
stderr = io.StringIO("")
|
||||
returncode = 0
|
||||
|
||||
def wait(self, timeout=None):
|
||||
return self.returncode
|
||||
|
||||
monkeypatch.setattr(
|
||||
"lai_automation.agent.subprocess.Popen",
|
||||
lambda *args, **kwargs: ProviderFailedProcess(),
|
||||
)
|
||||
monkeypatch.setattr("lai_automation.agent.os.chown", lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr("lai_automation.agent.pwd.getpwnam", lambda _: type(
|
||||
"Account", (), {"pw_dir": str(tmp_path), "pw_uid": 10001, "pw_gid": 10001}
|
||||
)())
|
||||
|
||||
agent = AiderAgent(settings())
|
||||
with pytest.raises(AgentProviderError, match="provider failed"):
|
||||
agent.run(workspace, "fix it")
|
||||
|
||||
assert agent.last_run_metadata["status"] == "provider_error"
|
||||
110
old_2026-08-31/tests/test_config.py
Normal file
110
old_2026-08-31/tests/test_config.py
Normal file
@@ -0,0 +1,110 @@
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from lai_automation.config import Settings
|
||||
|
||||
|
||||
def settings(**overrides) -> Settings:
|
||||
values = {
|
||||
"database_host": "db",
|
||||
"database_port": 3306,
|
||||
"database_name": "lai",
|
||||
"database_user": "worker",
|
||||
"database_password": "secret",
|
||||
"source_repository": "https://example.test/lai.git",
|
||||
"source_branch": "main",
|
||||
"workspace_root": Path("/tmp/workspaces"),
|
||||
"mode": "dry-run",
|
||||
"allow_direct_main": False,
|
||||
"gitea_url": "https://example.test",
|
||||
"gitea_owner": "owner",
|
||||
"gitea_repository": "lai",
|
||||
"gitea_token": "",
|
||||
"gitea_username": "bot",
|
||||
"agent_command": "aider",
|
||||
"agent_model": "mistral/test",
|
||||
"agent_timeout_seconds": 900,
|
||||
"agent_idle_timeout_seconds": 300,
|
||||
"setup_commands": (),
|
||||
"test_commands": ("php artisan test",),
|
||||
"test_database_host": "test-db",
|
||||
"test_database_port": 3306,
|
||||
"test_database_name": "lai_agent_test",
|
||||
"test_database_user": "test_runner",
|
||||
"test_database_password": "test-secret",
|
||||
"test_customer_database": "customer_agent_test",
|
||||
"test_config_database": "cockpit_agent_test",
|
||||
"poll_seconds": 60,
|
||||
"max_changed_files": 12,
|
||||
"max_changed_lines": 800,
|
||||
"max_agent_attempts": 2,
|
||||
"stale_claim_minutes": 90,
|
||||
"dry_run": True,
|
||||
}
|
||||
values.update(overrides)
|
||||
return Settings(**values)
|
||||
|
||||
|
||||
def test_direct_main_requires_explicit_kill_switch() -> None:
|
||||
config = settings(mode="direct-main", dry_run=False, gitea_token="token")
|
||||
with pytest.raises(ValueError, match="ALLOW_DIRECT_MAIN"):
|
||||
config.validate()
|
||||
|
||||
|
||||
def test_pr_requires_gitea_token() -> None:
|
||||
config = settings(mode="pr", dry_run=False)
|
||||
with pytest.raises(ValueError, match="GITEA_TOKEN"):
|
||||
config.validate()
|
||||
|
||||
|
||||
def test_default_commands_prepare_workspace_and_run_tests(monkeypatch) -> None:
|
||||
monkeypatch.setenv("FAILURE_DB_PASSWORD", "failure-secret")
|
||||
monkeypatch.setenv("TEST_DB_PASSWORD", "test-secret")
|
||||
monkeypatch.delenv("SETUP_COMMANDS", raising=False)
|
||||
monkeypatch.delenv("TEST_COMMANDS", raising=False)
|
||||
|
||||
config = Settings.from_env()
|
||||
|
||||
assert "npm ci" in config.setup_commands
|
||||
assert "npm run build" in config.setup_commands
|
||||
assert "php artisan migrate:fresh --force" in config.setup_commands
|
||||
assert config.test_commands == ("php artisan test",)
|
||||
|
||||
|
||||
def test_agent_attempts_are_bounded() -> None:
|
||||
with pytest.raises(ValueError, match="MAX_AGENT_ATTEMPTS"):
|
||||
settings(max_agent_attempts=8).validate()
|
||||
|
||||
settings(max_agent_attempts=7).validate()
|
||||
|
||||
|
||||
def test_agent_timeout_is_bounded() -> None:
|
||||
with pytest.raises(ValueError, match="AGENT_TIMEOUT_SECONDS"):
|
||||
settings(agent_timeout_seconds=59).validate()
|
||||
|
||||
settings(agent_timeout_seconds=60, agent_idle_timeout_seconds=60).validate()
|
||||
|
||||
|
||||
def test_agent_idle_timeout_is_bounded_by_process_timeout() -> None:
|
||||
with pytest.raises(ValueError, match="AGENT_IDLE_TIMEOUT_SECONDS"):
|
||||
settings(agent_idle_timeout_seconds=59).validate()
|
||||
|
||||
with pytest.raises(ValueError, match="AGENT_IDLE_TIMEOUT_SECONDS"):
|
||||
settings(agent_timeout_seconds=120, agent_idle_timeout_seconds=121).validate()
|
||||
|
||||
settings(agent_timeout_seconds=120, agent_idle_timeout_seconds=60).validate()
|
||||
|
||||
|
||||
def test_validation_databases_must_be_distinct_and_explicitly_test_only() -> None:
|
||||
with pytest.raises(ValueError, match="isolated test database"):
|
||||
settings(test_customer_database="customer_production").validate()
|
||||
|
||||
with pytest.raises(ValueError, match="must be distinct"):
|
||||
settings(test_config_database="customer_agent_test").validate()
|
||||
|
||||
with pytest.raises(ValueError, match="FAILURE_DB_NAME"):
|
||||
settings(
|
||||
database_name="lai_agent_test",
|
||||
test_database_name="lai_agent_test",
|
||||
).validate()
|
||||
35
old_2026-08-31/tests/test_database.py
Normal file
35
old_2026-08-31/tests/test_database.py
Normal file
@@ -0,0 +1,35 @@
|
||||
from unittest.mock import MagicMock, Mock
|
||||
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.database import FailureRepository
|
||||
|
||||
|
||||
def test_heartbeat_refreshes_only_an_active_claim() -> None:
|
||||
repository = FailureRepository(settings())
|
||||
connection = MagicMock()
|
||||
cursor = MagicMock()
|
||||
repository.connection = MagicMock()
|
||||
repository.connection.return_value.__enter__.return_value = connection
|
||||
connection.cursor.return_value.__enter__.return_value = cursor
|
||||
|
||||
repository.heartbeat(17)
|
||||
|
||||
statement, parameters = cursor.execute.call_args.args
|
||||
assert "SET updated_at = NOW()" in statement
|
||||
assert "status = 'agent_running'" in statement
|
||||
assert parameters == (17,)
|
||||
connection.commit.assert_called_once()
|
||||
|
||||
|
||||
def test_complete_clears_previous_agent_error() -> None:
|
||||
repository = FailureRepository(settings())
|
||||
repository._finish = Mock()
|
||||
|
||||
repository.complete(17, {"classification": "already_fixed"})
|
||||
|
||||
repository._finish.assert_called_once_with(
|
||||
17,
|
||||
"resolved",
|
||||
{"classification": "already_fixed", "agent_error": None},
|
||||
)
|
||||
239
old_2026-08-31/tests/test_git.py
Normal file
239
old_2026-08-31/tests/test_git.py
Normal file
@@ -0,0 +1,239 @@
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock
|
||||
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.git import GitWorkspace
|
||||
|
||||
|
||||
def test_prepare_uses_temporary_authentication_for_private_clone(tmp_path, monkeypatch) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path, gitea_token="token"), 10)
|
||||
calls = []
|
||||
|
||||
def run(command, **kwargs):
|
||||
calls.append((command, kwargs))
|
||||
if command[1] == "clone":
|
||||
askpass = Path(kwargs["env"]["GIT_ASKPASS"])
|
||||
assert askpass.is_file()
|
||||
assert Path(kwargs["env"]["GIT_BOT_USER_FILE"]).read_text() == "bot"
|
||||
assert Path(kwargs["env"]["GIT_BOT_TOKEN_FILE"]).read_text() == "token"
|
||||
workspace.path.mkdir()
|
||||
return subprocess.CompletedProcess(command, 0, "", "")
|
||||
|
||||
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
|
||||
|
||||
workspace.prepare()
|
||||
|
||||
clone_command, clone_options = calls[0]
|
||||
assert clone_command[:2] == ["git", "clone"]
|
||||
assert "token" not in " ".join(clone_command)
|
||||
assert clone_options["cwd"] == tmp_path
|
||||
assert not tuple(tmp_path.glob(".lai-git-auth-*"))
|
||||
|
||||
|
||||
def test_prepare_surfaces_clone_stderr_without_credentials(tmp_path, monkeypatch) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path, gitea_token="token"), 11)
|
||||
|
||||
def run(command, **kwargs):
|
||||
raise subprocess.CalledProcessError(
|
||||
128,
|
||||
command,
|
||||
output="",
|
||||
stderr="fatal: upstream returned 403",
|
||||
)
|
||||
|
||||
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
|
||||
|
||||
try:
|
||||
workspace.prepare()
|
||||
except RuntimeError as exception:
|
||||
message = str(exception)
|
||||
else:
|
||||
raise AssertionError("Expected clone failure")
|
||||
|
||||
assert "Source repository clone failed" in message
|
||||
assert "upstream returned 403" in message
|
||||
assert "token" not in message
|
||||
assert not tuple(tmp_path.glob(".lai-git-auth-*"))
|
||||
|
||||
|
||||
def test_test_command_does_not_interpret_shell_metacharacters(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 1)
|
||||
workspace.path.mkdir()
|
||||
|
||||
result = workspace.run("printf safe ; touch injected", check=False)
|
||||
|
||||
assert result.stdout == "safe"
|
||||
assert not (workspace.path / "injected").exists()
|
||||
|
||||
|
||||
def test_extra_environment_is_passed_without_privilege_drop(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 6)
|
||||
workspace.path.mkdir()
|
||||
|
||||
result = workspace.run_args(
|
||||
[sys.executable, "-c", "import os; print(os.environ['SAFE_TEST_VALUE'])"],
|
||||
extra_environment={"SAFE_TEST_VALUE": "available"},
|
||||
)
|
||||
|
||||
assert result.stdout.strip() == "available"
|
||||
|
||||
|
||||
def test_coder_commands_use_native_process_identity(tmp_path, monkeypatch) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 9)
|
||||
workspace.path.mkdir()
|
||||
captured = {}
|
||||
|
||||
def run(command, **kwargs):
|
||||
captured.update(kwargs)
|
||||
return subprocess.CompletedProcess(command, 0, "ok", "")
|
||||
|
||||
monkeypatch.setattr("lai_automation.git.subprocess.run", run)
|
||||
monkeypatch.setattr(
|
||||
"lai_automation.git.pwd.getpwnam",
|
||||
lambda _: Mock(pw_dir="/home/coder", pw_uid=10001, pw_gid=10001),
|
||||
)
|
||||
|
||||
workspace.run_args(["php", "-v"], username="coder")
|
||||
|
||||
assert captured["user"] == 10001
|
||||
assert captured["group"] == 10001
|
||||
assert captured["extra_groups"] == ()
|
||||
assert "preexec_fn" not in captured
|
||||
|
||||
|
||||
def test_summary_includes_untracked_files(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 2)
|
||||
workspace.path.mkdir()
|
||||
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||
(workspace.path / "new.php").write_text("<?php\n")
|
||||
|
||||
summary = workspace.summary()
|
||||
|
||||
assert summary.files == ("new.php",)
|
||||
assert summary.added_lines == 1
|
||||
|
||||
|
||||
def test_summary_includes_checkpoint_staged_files(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 7)
|
||||
workspace.path.mkdir()
|
||||
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||
tracked = workspace.path / "config.php"
|
||||
tracked.write_text("<?php\nreturn [];\n")
|
||||
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||
subprocess.run(
|
||||
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test",
|
||||
"commit", "-m", "base"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
tracked.write_text("<?php\nreturn ['fixed'];\n")
|
||||
|
||||
workspace.checkpoint()
|
||||
summary = workspace.summary()
|
||||
|
||||
assert summary.files == ("config.php",)
|
||||
assert summary.added_lines == 1
|
||||
assert summary.deleted_lines == 1
|
||||
|
||||
|
||||
def test_discard_changes_restores_clean_disposable_workspace(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 3)
|
||||
workspace.path.mkdir()
|
||||
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||
subprocess.run(
|
||||
["git", "config", "user.email", "test@example.invalid"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.name", "Test"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
)
|
||||
tracked = workspace.path / "tracked.php"
|
||||
tracked.write_text("<?php\nreturn [];\n")
|
||||
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||
subprocess.run(["git", "commit", "-m", "base"], cwd=workspace.path, check=True, capture_output=True)
|
||||
tracked.write_text("<?php\ninvalid\n")
|
||||
untracked = workspace.path / "extra.php"
|
||||
untracked.write_text("<?php\n")
|
||||
|
||||
workspace.discard_changes()
|
||||
|
||||
assert tracked.read_text() == "<?php\nreturn [];\n"
|
||||
assert not untracked.exists()
|
||||
|
||||
|
||||
def test_checkpoint_restores_last_valid_changes(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 4)
|
||||
workspace.path.mkdir()
|
||||
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||
subprocess.run(
|
||||
["git", "config", "user.email", "test@example.invalid"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
)
|
||||
subprocess.run(
|
||||
["git", "config", "user.name", "Test"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
)
|
||||
tracked = workspace.path / "config.php"
|
||||
tracked.write_text("<?php\nreturn [];\n")
|
||||
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||
subprocess.run(["git", "commit", "-m", "base"], cwd=workspace.path, check=True, capture_output=True)
|
||||
tracked.write_text("<?php\nreturn ['valid'];\n")
|
||||
added = workspace.path / "test.php"
|
||||
added.write_text("<?php\n")
|
||||
checkpoint = workspace.checkpoint()
|
||||
tracked.write_text("<?php\nreturn [broken;\n")
|
||||
|
||||
workspace.restore_checkpoint(checkpoint)
|
||||
|
||||
assert tracked.read_text() == "<?php\nreturn ['valid'];\n"
|
||||
assert added.exists()
|
||||
|
||||
|
||||
def test_diff_returns_discarded_patch_excerpt(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 5)
|
||||
workspace.path.mkdir()
|
||||
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||
tracked = workspace.path / "service.php"
|
||||
tracked.write_text("<?php\nreturn 'old';\n")
|
||||
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||
subprocess.run(
|
||||
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test", "commit", "-m", "base"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
tracked.write_text("<?php\nreturn 'new';\n")
|
||||
|
||||
patch = workspace.diff()
|
||||
|
||||
assert "-return 'old';" in patch
|
||||
assert "+return 'new';" in patch
|
||||
|
||||
|
||||
def test_head_file_reads_original_tracked_content(tmp_path) -> None:
|
||||
workspace = GitWorkspace(settings(workspace_root=tmp_path), 8)
|
||||
workspace.path.mkdir()
|
||||
subprocess.run(["git", "init"], cwd=workspace.path, check=True, capture_output=True)
|
||||
tracked = workspace.path / "config.php"
|
||||
tracked.write_text("<?php\nreturn 'original';\n")
|
||||
subprocess.run(["git", "add", "."], cwd=workspace.path, check=True)
|
||||
subprocess.run(
|
||||
["git", "-c", "user.email=test@example.invalid", "-c", "user.name=Test",
|
||||
"commit", "-m", "base"],
|
||||
cwd=workspace.path,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
)
|
||||
tracked.write_text("<?php\nreturn 'candidate';\n")
|
||||
|
||||
assert workspace.head_file("config.php") == "<?php\nreturn 'original';\n"
|
||||
assert workspace.head_file("missing.php") is None
|
||||
27
old_2026-08-31/tests/test_gitea.py
Normal file
27
old_2026-08-31/tests/test_gitea.py
Normal file
@@ -0,0 +1,27 @@
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.gitea import GiteaClient
|
||||
|
||||
|
||||
def test_create_pull_request_requires_url() -> None:
|
||||
client = GiteaClient(settings(gitea_token="token"))
|
||||
response = Mock()
|
||||
response.json.return_value = {"number": 12}
|
||||
client.session.post = Mock(return_value=response)
|
||||
|
||||
with pytest.raises(ValueError, match="pull request URL"):
|
||||
client.create_pull_request("ai/failure-1", "Fix", "Body")
|
||||
|
||||
|
||||
def test_create_pull_request_returns_html_url() -> None:
|
||||
client = GiteaClient(settings(gitea_token="token"))
|
||||
response = Mock()
|
||||
response.json.return_value = {"html_url": "https://example.test/owner/lai/pulls/12"}
|
||||
client.session.post = Mock(return_value=response)
|
||||
|
||||
result = client.create_pull_request("ai/failure-1", "Fix", "Body")
|
||||
|
||||
assert result.endswith("/pulls/12")
|
||||
924
old_2026-08-31/tests/test_policy.py
Normal file
924
old_2026-08-31/tests/test_policy.py
Normal file
@@ -0,0 +1,924 @@
|
||||
from dataclasses import replace
|
||||
from unittest.mock import Mock
|
||||
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.git import ChangeSummary
|
||||
from lai_automation.models import FailureReport
|
||||
from lai_automation.policy import (
|
||||
evaluate_changes,
|
||||
evaluate_expected_behavior,
|
||||
evaluate_removal_prevention,
|
||||
evaluate_removal_scope,
|
||||
evaluate_report_scope,
|
||||
evaluate_requested_behavior_scope,
|
||||
evaluate_routed_files,
|
||||
)
|
||||
|
||||
|
||||
def test_policy_accepts_small_application_change() -> None:
|
||||
result = evaluate_changes(
|
||||
ChangeSummary(("app/Services/Example.php", "tests/Feature/ExampleTest.php"), 40, 5),
|
||||
settings(),
|
||||
)
|
||||
assert result.allowed
|
||||
|
||||
|
||||
def test_policy_rejects_secrets_and_vendor_paths() -> None:
|
||||
result = evaluate_changes(
|
||||
ChangeSummary((".env", "vendor/foo.php", ".aider.tags.cache.v4/cache.db"), 3, 0),
|
||||
settings(),
|
||||
)
|
||||
assert not result.allowed
|
||||
assert len(result.errors) == 3
|
||||
|
||||
|
||||
def test_policy_rejects_test_only_repair() -> None:
|
||||
result = evaluate_changes(
|
||||
ChangeSummary(("tests/Unit/ExampleTest.php",), 20, 2),
|
||||
settings(),
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert "production behavior was not fixed" in result.errors[0]
|
||||
|
||||
|
||||
def test_direct_main_blocks_validator_and_migrations() -> None:
|
||||
config = replace(
|
||||
settings(),
|
||||
mode="direct-main",
|
||||
dry_run=False,
|
||||
allow_direct_main=True,
|
||||
gitea_token="token",
|
||||
)
|
||||
result = evaluate_changes(
|
||||
ChangeSummary(("app/Services/SQLValidatorService.php", "database/migrations/new.php"), 10, 2),
|
||||
config,
|
||||
)
|
||||
assert not result.allowed
|
||||
|
||||
|
||||
def test_policy_rejects_symlinks(tmp_path) -> None:
|
||||
(tmp_path / "link.php").symlink_to("/etc/passwd")
|
||||
result = evaluate_changes(ChangeSummary(("link.php",), 1, 0), settings(), tmp_path)
|
||||
assert not result.allowed
|
||||
|
||||
|
||||
def test_cross_database_report_requires_application_orchestration() -> None:
|
||||
report = FailureReport(
|
||||
id=56,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Use cockpit_config.time_registration",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Join cockpit_config manager_id and FK_AuftragPos",
|
||||
prompt="Hours in April",
|
||||
response="0 hours",
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
rejected = evaluate_report_scope(
|
||||
report,
|
||||
ChangeSummary(("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2),
|
||||
)
|
||||
accepted = evaluate_report_scope(
|
||||
report,
|
||||
ChangeSummary(("app/Http/Controllers/ChatController.php", "tests/Unit/FixTest.php"), 20, 2),
|
||||
)
|
||||
|
||||
assert not rejected.allowed
|
||||
assert "staged orchestration" in rejected.errors[0]
|
||||
assert accepted.allowed
|
||||
|
||||
|
||||
def test_expected_behavior_requires_explicit_technical_terms_in_added_code() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
"FK_Auftrag nach AuftragsKennung 2 filtern und System_created verwenden; "
|
||||
"obsolet und storniert ausfiltern"
|
||||
),
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
rejected = evaluate_expected_behavior(
|
||||
report,
|
||||
"Use FK_Auftrag.Datum_erfassung and status storniert",
|
||||
)
|
||||
accepted = evaluate_expected_behavior(
|
||||
report,
|
||||
"SELECT DATE_FORMAT(System_created, '%Y-%m') FROM FK_Auftrag "
|
||||
"WHERE AuftragsKennung = 2 AND COALESCE(obsolet, 0) = 0 "
|
||||
"AND COALESCE(bStatus_storniert, 0) = 0",
|
||||
)
|
||||
|
||||
assert not rejected.allowed
|
||||
assert "AuftragsKennung" in rejected.errors[0]
|
||||
assert "System_created" in rejected.errors[0]
|
||||
assert accepted.allowed
|
||||
|
||||
|
||||
def test_expected_behavior_rejects_negated_required_date_column() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="System_created verwenden",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
result = evaluate_expected_behavior(
|
||||
report,
|
||||
"Lieferscheine niemals mit System_created filtern",
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert "contradicts expected identifier System_created" in result.errors[0]
|
||||
|
||||
|
||||
def test_expected_behavior_allows_rejecting_an_alternative_column_on_same_table() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="FK_Auftrag nach System_created filtern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
result = evaluate_expected_behavior(
|
||||
report,
|
||||
"FK_Auftrag nach System_created filtern, nicht nach FK_Auftrag.Datum_erfassung",
|
||||
)
|
||||
|
||||
assert result.allowed
|
||||
|
||||
|
||||
def test_expected_behavior_does_not_treat_later_never_as_table_negation() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="FK_Auftrag nach System_created filtern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
result = evaluate_expected_behavior(
|
||||
report,
|
||||
"FK_Auftrag nach System_created filtern und Datum_erfassung niemals verwenden",
|
||||
)
|
||||
|
||||
assert result.allowed
|
||||
|
||||
|
||||
def test_expected_behavior_rejects_unrequested_filter_copied_from_bad_sql() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="System_created verwenden und status storniert ausfiltern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query="WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
result = evaluate_expected_behavior(
|
||||
report,
|
||||
"System_created AND bStatus_gebucht = 1 AND COALESCE(bStatus_storniert, 0) = 0",
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert "copied bStatus_gebucht" in result.errors[-1]
|
||||
|
||||
|
||||
def test_expected_behavior_requires_one_complete_reference_select() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
"FK_Auftrag nach Auftragskennung 2 filtern und System_created verwenden; "
|
||||
"obsolet und status storniert ausfiltern"
|
||||
),
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS' AND Datum_erfassung >= '2024-01-01'",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
split_rules_and_bad_sql = (
|
||||
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
|
||||
"COALESCE(bStatus_storniert, 0) = 0\n"
|
||||
"SELECT * FROM FK_Auftrag a WHERE Bezeichnung_Kurz = 'LS' "
|
||||
"AND a.System_created >= '2024-01-01' "
|
||||
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
||||
)
|
||||
complete_sql = (
|
||||
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
|
||||
"COALESCE(bStatus_storniert, 0) = 0\n"
|
||||
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||
"AND a.System_created >= '2024-01-01' "
|
||||
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
||||
)
|
||||
|
||||
rejected = evaluate_expected_behavior(report, split_rules_and_bad_sql)
|
||||
accepted = evaluate_expected_behavior(report, complete_sql)
|
||||
|
||||
assert not rejected.allowed
|
||||
assert any("one reference SELECT" in error for error in rejected.errors)
|
||||
assert any("LS lookup" in error for error in rejected.errors)
|
||||
assert accepted.allowed
|
||||
|
||||
|
||||
def test_expected_behavior_detects_lowercase_document_code_without_observed_sql() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
rejected = evaluate_expected_behavior(
|
||||
report,
|
||||
"SELECT * FROM FK_Auftrag WHERE Bezeichnung_Kurz = 'LS'",
|
||||
)
|
||||
accepted = evaluate_expected_behavior(
|
||||
report,
|
||||
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2",
|
||||
)
|
||||
|
||||
assert not rejected.allowed
|
||||
assert any("AuftragsKennung = 2" in error for error in rejected.errors)
|
||||
assert accepted.allowed
|
||||
|
||||
|
||||
def test_expected_behavior_rejects_lookup_join_for_authoritative_direct_code() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
|
||||
result = evaluate_expected_behavior(
|
||||
report,
|
||||
"SELECT * FROM FK_Auftrag a JOIN FK_Nummerkreis n ON n.Kennung = a.AuftragsKennung "
|
||||
"WHERE a.AuftragsKennung = 2",
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert any("joined FK_Nummerkreis" in error for error in result.errors)
|
||||
|
||||
|
||||
def test_expected_behavior_rejects_invented_obsolete_prefix_and_incomplete_assertions() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback=None,
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Auftragskennung 2, obsolet und status storniert ausfiltern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
invented = (
|
||||
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
||||
"AND COALESCE(bStatus_obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
|
||||
"$this->assertStringContainsString('AuftragsKennung = 2', $context);"
|
||||
)
|
||||
complete = (
|
||||
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
||||
"AND COALESCE(obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
|
||||
"$this->assertStringContainsString('AuftragsKennung = 2', $context);\n"
|
||||
"$this->assertStringContainsString('COALESCE(obsolet, 0) = 0', $context);\n"
|
||||
"$this->assertStringContainsString('COALESCE(bStatus_storniert, 0) = 0', $context);"
|
||||
)
|
||||
|
||||
rejected = evaluate_expected_behavior(report, invented)
|
||||
accepted = evaluate_expected_behavior(report, complete)
|
||||
|
||||
assert not rejected.allowed
|
||||
assert any("obsolete rows" in error for error in rejected.errors)
|
||||
assert any("Focused regression assertions" in error for error in rejected.errors)
|
||||
assert accepted.allowed
|
||||
|
||||
|
||||
def test_routed_file_policy_rejects_unexpected_edits() -> None:
|
||||
result = evaluate_routed_files(
|
||||
ChangeSummary(("config/erp_semantics.php", "app/Services/AIService.php"), 10, 0),
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert "app/Services/AIService.php" in result.errors[0]
|
||||
|
||||
|
||||
def test_requested_behavior_scope_requires_all_presentation_and_routing_owners() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
|
||||
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
|
||||
),
|
||||
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
response="2024-01: 38",
|
||||
sql_query="SELECT COUNT(*) FROM FK_Auftrag",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
incomplete = evaluate_requested_behavior_scope(
|
||||
report,
|
||||
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 10, 0),
|
||||
"cockpit: www: Markdown-Tabelle",
|
||||
)
|
||||
complete = evaluate_requested_behavior_scope(
|
||||
report,
|
||||
ChangeSummary((
|
||||
"app/Services/AIService.php",
|
||||
"app/Http/Controllers/ChatController.php",
|
||||
"app/Services/WebSearchService.php",
|
||||
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||
"tests/Unit/WebSearchServiceTest.php",
|
||||
), 30, 2),
|
||||
(
|
||||
"Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024\n"
|
||||
"Markdown-Tabelle\ncount($rows) > 1\nsingle_row keeps sentence format\n"
|
||||
"resolveRoutePrefix\n['database', 'Frage']\n['research', 'Frage']\n"
|
||||
"['direct', 'Frage']\n"
|
||||
"[null, 'Frage']\n[$explicitRoute, $message] = "
|
||||
"$this->resolveRoutePrefix($message);\n$route = $explicitRoute ?? $fallback;\n"
|
||||
"generateWebResearchResponse($message, 'ollama');\n"
|
||||
"generateDirectResponse($message, 'ollama', []);\n"
|
||||
"$controller->processMessage($cockpitRequest);\n"
|
||||
"$ai->expects($this->never())->method('determineRoute');\n"
|
||||
"function test_cockpit_prefix(): void {\n"
|
||||
"$tenant->method('requiresCustomerDatabase')->willReturn(false);\n"
|
||||
"$tenant->method('requiresConfigDatabase')->willReturn(false);\n"
|
||||
"$tenant->method('requiresErpDatabase')->willReturn(false);\n}\n"
|
||||
"$controller->processMessage($wwwRequest);\n"
|
||||
"$ai->expects($this->never())->method('determineRoute');\n"
|
||||
"$controller->processMessage($directRequest);\n"
|
||||
"$ai->expects($this->never())->method('determineRoute');\n"
|
||||
"$this->assertSame(3, substr_count($userPrompt, '\"Monat\"'));\n"
|
||||
"$this->assertSame(3, substr_count($userPrompt, '\"Anzahl_Lieferscheine\"'));\n"
|
||||
"cockpit:\nwww:\ndirect:"
|
||||
),
|
||||
)
|
||||
|
||||
assert not incomplete.allowed
|
||||
assert any("AIService.php" in error for error in incomplete.errors)
|
||||
assert any("ChatController.php" in error for error in incomplete.errors)
|
||||
assert not complete.errors
|
||||
assert complete.allowed
|
||||
|
||||
|
||||
def test_requested_behavior_scope_rejects_dead_prefix_source_tests() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Als Tabelle anzeigen",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
|
||||
prompt="Anzahl pro Monat",
|
||||
response="Text",
|
||||
sql_query="SELECT 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
added = """
|
||||
Markdown-Tabelle
|
||||
count($rows) > 1
|
||||
single_row keeps sentence format
|
||||
resolveRoutePrefix
|
||||
['database', 'Frage']
|
||||
['direct', 'Frage']
|
||||
[null, 'Frage']
|
||||
[$explicitRoute, $message] = $this->resolveRoutePrefix($message);
|
||||
$route = $explicitRoute ?? $fallback;
|
||||
$controller->processMessage($cockpitRequest);
|
||||
$ai->expects($this->never())->method('determineRoute');
|
||||
$controller->processMessage($wwwRequest);
|
||||
$ai->expects($this->never())->method('determineRoute');
|
||||
$controller->resolveSource('cockpit: Frage', []);
|
||||
return $this->resolveRoutePrefix($message);
|
||||
public function determineRouteForMessage() { return $this->determineRoute($message, $this->defaultModel); }
|
||||
www:
|
||||
cockpit:
|
||||
Anzahl pro Monat
|
||||
"""
|
||||
|
||||
result = evaluate_requested_behavior_scope(
|
||||
report,
|
||||
ChangeSummary((
|
||||
"app/Services/AIService.php",
|
||||
"app/Http/Controllers/ChatController.php",
|
||||
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||
), 40, 2),
|
||||
added,
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert any("do not test resolveDatabaseSource" in error for error in result.errors)
|
||||
|
||||
|
||||
def test_requested_behavior_scope_rejects_helper_only_prefix_tests() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Als Tabelle anzeigen",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
|
||||
prompt="Anzahl pro Monat",
|
||||
response="Text",
|
||||
sql_query="SELECT 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
added = """
|
||||
Anzahl pro Monat
|
||||
Markdown-Tabelle
|
||||
count($rows) > 1
|
||||
single_row stays sentence formatted
|
||||
resolveRoutePrefix
|
||||
['database', 'Frage']
|
||||
['direct', 'Frage']
|
||||
[null, 'Frage']
|
||||
[$explicitRoute, $message] = $this->resolveRoutePrefix($originalMessage);
|
||||
$route = $explicitRoute ?? $fallback;
|
||||
cockpit:
|
||||
www:
|
||||
"""
|
||||
|
||||
result = evaluate_requested_behavior_scope(
|
||||
report,
|
||||
ChangeSummary((
|
||||
"app/Services/AIService.php",
|
||||
"app/Http/Controllers/ChatController.php",
|
||||
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||
), 40, 2),
|
||||
added,
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert any("helper-only tests" in error for error in result.errors)
|
||||
|
||||
|
||||
def test_requested_behavior_scope_rejects_incorrect_table_instruction_assertion() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Als Tabelle anzeigen",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="",
|
||||
prompt="Anzahl pro Monat",
|
||||
response="Text",
|
||||
sql_query="SELECT 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
added = """
|
||||
Anzahl pro Monat
|
||||
Markdown-Tabelle
|
||||
count($rows) > 1
|
||||
single_row stays sentence formatted
|
||||
$this->assertStringContainsString('alle bereitgestellten Zeilen', $systemPrompt);
|
||||
"""
|
||||
|
||||
result = evaluate_requested_behavior_scope(
|
||||
report,
|
||||
ChangeSummary((
|
||||
"app/Services/AIService.php",
|
||||
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||
), 20, 2),
|
||||
added,
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert any("exact grammatical substring" in error for error in result.errors)
|
||||
|
||||
|
||||
def test_expected_behavior_treats_removed_predicate_as_negative_requirement() -> None:
|
||||
report = FailureReport(
|
||||
id=5,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong delivery note count",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
|
||||
"Einfach nur das aus der SQL entfernen"
|
||||
),
|
||||
prompt="Delivery notes by month",
|
||||
response="0 rows",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
accepted = (
|
||||
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
|
||||
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||
)
|
||||
still_present = (
|
||||
"SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1\n"
|
||||
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||
)
|
||||
|
||||
assert evaluate_expected_behavior(report, accepted).allowed
|
||||
rejected_sql = evaluate_expected_behavior(report, still_present)
|
||||
rejected_test = evaluate_expected_behavior(
|
||||
report,
|
||||
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2",
|
||||
)
|
||||
near_match = evaluate_expected_behavior(
|
||||
report,
|
||||
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
|
||||
"$this->assertStringNotContainsString('bStatus_gebucht = 1', $context);",
|
||||
)
|
||||
|
||||
assert not rejected_sql.allowed
|
||||
assert any("forbidden fragment" in error for error in rejected_sql.errors)
|
||||
assert not rejected_test.allowed
|
||||
assert any("negatively assert" in error for error in rejected_test.errors)
|
||||
assert not near_match.allowed
|
||||
assert any("near-match with a missing qualifier" in error for error in near_match.errors)
|
||||
assert any(
|
||||
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||
in error
|
||||
for error in near_match.errors
|
||||
)
|
||||
|
||||
|
||||
def test_removal_prevention_rejects_vacuous_test_and_full_forbidden_rule(tmp_path) -> None:
|
||||
report = FailureReport(
|
||||
id=6,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Die Abfrage stimmt nicht",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||
"weil Lieferscheine nicht gebucht werden."
|
||||
),
|
||||
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
response="0 Ergebnisse",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
source_file = "config/erp_semantics.php"
|
||||
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||
reference_sql = (
|
||||
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
||||
)
|
||||
|
||||
def semantics(metric: str = "", sql: str = reference_sql) -> str:
|
||||
return f"""<?php
|
||||
return [
|
||||
'metrics' => [
|
||||
{metric}
|
||||
],
|
||||
'examples' => [[
|
||||
'tables' => ['FK_Auftrag'],
|
||||
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
||||
'sql' => <<<'SQL'
|
||||
{sql}
|
||||
SQL,
|
||||
]],
|
||||
];
|
||||
"""
|
||||
|
||||
original_source = semantics()
|
||||
bad_metric = """ 'Lieferscheine' => [
|
||||
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
|
||||
'rules' => [
|
||||
'Lieferscheine werden nicht gebucht, daher niemals a.bStatus_gebucht = 1 verwenden.',
|
||||
],
|
||||
],"""
|
||||
original_test = """<?php
|
||||
$context = $service->businessContext(['FK_Auftrag'], 'erp');
|
||||
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
||||
"""
|
||||
bad_test = original_test + (
|
||||
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
||||
)
|
||||
(tmp_path / "config").mkdir()
|
||||
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||
(tmp_path / source_file).write_text(semantics(bad_metric))
|
||||
(tmp_path / test_file).write_text(bad_test)
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.head_file.side_effect = lambda filename: {
|
||||
source_file: original_source,
|
||||
test_file: original_test,
|
||||
}.get(filename)
|
||||
summary = ChangeSummary((source_file, test_file), 3, 0)
|
||||
|
||||
result = evaluate_removal_prevention(report, summary, workspace)
|
||||
|
||||
assert not result.allowed
|
||||
assert any("must not embed the complete forbidden predicate" in error for error in result.errors)
|
||||
assert any("context-builder test is vacuous" in error for error in result.errors)
|
||||
|
||||
|
||||
def test_removal_prevention_rejects_unrelated_new_semantics(tmp_path) -> None:
|
||||
report = FailureReport(
|
||||
id=6,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong delivery note count",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||
"weil Lieferscheine nicht gebucht werden."
|
||||
),
|
||||
prompt="Delivery notes by month",
|
||||
response="0 rows",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
source_file = "config/erp_semantics.php"
|
||||
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||
original_source = """<?php
|
||||
return ['examples' => [[
|
||||
'question' => 'Delivery notes by month',
|
||||
'sql' => <<<'SQL'
|
||||
SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
|
||||
SQL,
|
||||
]]];
|
||||
"""
|
||||
current_source = original_source.replace(
|
||||
"return [",
|
||||
"""return [
|
||||
'metrics' => ['Lieferscheine' => [
|
||||
'tables' => ['FK_Auftrag'],
|
||||
'rules' => [
|
||||
'Lieferscheine werden nicht gebucht; bStatus_gebucht niemals verwenden.',
|
||||
'Verwende Datum_erfassung oder System_created als Datumsspalte.',
|
||||
],
|
||||
]],""",
|
||||
)
|
||||
original_test = "<?php\n"
|
||||
current_test = original_test + (
|
||||
"$this->assertStringContainsString('Lieferscheine werden nicht gebucht; "
|
||||
"bStatus_gebucht niemals verwenden.', $context);\n"
|
||||
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
||||
)
|
||||
(tmp_path / "config").mkdir()
|
||||
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||
(tmp_path / source_file).write_text(current_source)
|
||||
(tmp_path / test_file).write_text(current_test)
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.head_file.side_effect = lambda filename: {
|
||||
source_file: original_source,
|
||||
test_file: original_test,
|
||||
}.get(filename)
|
||||
|
||||
result = evaluate_removal_prevention(
|
||||
report,
|
||||
ChangeSummary((source_file, test_file), 7, 0),
|
||||
workspace,
|
||||
)
|
||||
|
||||
assert not result.allowed
|
||||
assert any("unrelated business guidance" in error for error in result.errors)
|
||||
assert any("Datum_erfassung" in error for error in result.errors)
|
||||
|
||||
|
||||
def test_removal_prevention_accepts_exercised_rule_and_protects_reference(tmp_path) -> None:
|
||||
report = FailureReport(
|
||||
id=6,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Die Abfrage stimmt nicht",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||
"weil Lieferscheine nicht gebucht werden."
|
||||
),
|
||||
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
response="0 Ergebnisse",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
source_file = "config/erp_semantics.php"
|
||||
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||
reference_sql = (
|
||||
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
||||
)
|
||||
|
||||
def semantics(metric: str = "", sql: str = reference_sql) -> str:
|
||||
return f"""<?php
|
||||
return [
|
||||
'metrics' => [
|
||||
{metric}
|
||||
],
|
||||
'examples' => [[
|
||||
'tables' => ['FK_Auftrag'],
|
||||
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
||||
'sql' => <<<'SQL'
|
||||
{sql}
|
||||
SQL,
|
||||
]],
|
||||
];
|
||||
"""
|
||||
|
||||
rule = (
|
||||
"Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine "
|
||||
"niemals verwenden."
|
||||
)
|
||||
metric = f""" 'Lieferscheine' => [
|
||||
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
|
||||
'rules' => [
|
||||
'{rule}',
|
||||
],
|
||||
],"""
|
||||
original_source = semantics()
|
||||
original_test = """<?php
|
||||
$context = $service->businessContext(['FK_Auftrag'], 'erp');
|
||||
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
||||
"""
|
||||
current_test = """<?php
|
||||
$context = $service->businessContext(['FK_Auftrag', 'FK_Nummerkreis'], 'erp');
|
||||
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
||||
$this->assertStringContainsString('Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine niemals verwenden.', $context);
|
||||
$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);
|
||||
"""
|
||||
(tmp_path / "config").mkdir()
|
||||
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||
(tmp_path / source_file).write_text(semantics(metric))
|
||||
(tmp_path / test_file).write_text(current_test)
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.head_file.side_effect = lambda filename: {
|
||||
source_file: original_source,
|
||||
test_file: original_test,
|
||||
}.get(filename)
|
||||
summary = ChangeSummary((source_file, test_file), 4, 1)
|
||||
|
||||
accepted = evaluate_removal_prevention(report, summary, workspace)
|
||||
|
||||
assert accepted.allowed
|
||||
|
||||
(tmp_path / source_file).write_text(semantics(
|
||||
metric,
|
||||
reference_sql.replace("AuftragsKennung = 2", "AuftragsKennung = 4"),
|
||||
))
|
||||
regressed = evaluate_removal_prevention(report, summary, workspace)
|
||||
|
||||
assert not regressed.allowed
|
||||
assert any("protected baseline" in error for error in regressed.errors)
|
||||
|
||||
|
||||
def test_removal_only_scope_allows_only_exact_predicate_removal_and_added_test(tmp_path) -> None:
|
||||
report = FailureReport(
|
||||
id=5,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong delivery note count",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
|
||||
"Einfach nur das aus der SQL entfernen"
|
||||
),
|
||||
prompt="Delivery notes by month in 2024",
|
||||
response="0 rows",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
original_sql = (
|
||||
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
|
||||
"WHERE a.AuftragsKennung = 2 AND a.bStatus_gebucht = 1 "
|
||||
"AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
|
||||
)
|
||||
repaired_sql = (
|
||||
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
|
||||
"WHERE a.AuftragsKennung = 2 AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
|
||||
)
|
||||
|
||||
def semantics(sql: str) -> str:
|
||||
return f"""<?php
|
||||
return [
|
||||
'examples' => [[
|
||||
'question' => 'Delivery notes by month in 2024',
|
||||
'sql' => <<<'SQL'
|
||||
{sql}
|
||||
SQL,
|
||||
]],
|
||||
];
|
||||
"""
|
||||
|
||||
source_file = "config/erp_semantics.php"
|
||||
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
||||
(tmp_path / "config").mkdir()
|
||||
(tmp_path / "tests/Unit").mkdir(parents=True)
|
||||
(tmp_path / source_file).write_text(semantics(repaired_sql))
|
||||
previous_test = "<?php\n$this->assertStringContainsString('System_created', $context);\n"
|
||||
(tmp_path / test_file).write_text(
|
||||
previous_test
|
||||
+ "$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
||||
)
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.head_file.side_effect = lambda filename: {
|
||||
source_file: semantics(original_sql),
|
||||
test_file: previous_test,
|
||||
}.get(filename)
|
||||
summary = ChangeSummary((source_file, test_file), 2, 1)
|
||||
|
||||
result = evaluate_removal_scope(report, summary, workspace)
|
||||
|
||||
assert result.allowed
|
||||
|
||||
(tmp_path / source_file).write_text(semantics(repaired_sql.replace(
|
||||
"System_created",
|
||||
"Datum_erfassung",
|
||||
)))
|
||||
regression = evaluate_removal_scope(report, summary, workspace)
|
||||
|
||||
assert not regression.allowed
|
||||
assert any("outside the explicitly forbidden" in error for error in regression.errors)
|
||||
367
old_2026-08-31/tests/test_prompt.py
Normal file
367
old_2026-08-31/tests/test_prompt.py
Normal file
@@ -0,0 +1,367 @@
|
||||
from lai_automation.models import FailureReport
|
||||
from lai_automation.prompt import build_retry_task, build_task, redact, repair_files
|
||||
|
||||
|
||||
def test_redact_removes_common_secrets_and_personal_markers() -> None:
|
||||
value = "password=hunter2 token: abc123 admin@example.com 10.2.3.4 Bearer secret.jwt"
|
||||
result = redact(value)
|
||||
assert "hunter2" not in result
|
||||
assert "abc123" not in result
|
||||
assert "admin@example.com" not in result
|
||||
assert "10.2.3.4" not in result
|
||||
assert "secret.jwt" not in result
|
||||
|
||||
|
||||
def test_task_contains_constraints_and_redacted_report() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=91,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="password=secret-value",
|
||||
feedback_type="wrong_join",
|
||||
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
|
||||
prompt="List locations for admin@example.com",
|
||||
response="The wrong product count was returned.",
|
||||
sql_query="SELECT * FROM company",
|
||||
error_message=None,
|
||||
context={"model": "mistral", "database_type": "customer"},
|
||||
)
|
||||
task = build_task(report)
|
||||
assert "failure report #7" in task
|
||||
assert "secret-value" not in task
|
||||
assert "admin@example.com" not in task
|
||||
assert "Do not commit or push" in task
|
||||
assert "never mock or stub the behavior being tested" in task
|
||||
assert "Repository ownership map:" in task
|
||||
assert "config/erp_semantics.php owns ERP joins" in task
|
||||
assert "Tests must contain assertions that execute" in task
|
||||
assert "Do not expose protected production methods publicly" in task
|
||||
assert "Feedback type: wrong_join" in task
|
||||
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
|
||||
assert "The wrong product count was returned." in task
|
||||
|
||||
|
||||
def test_redact_removes_extended_credentials() -> None:
|
||||
private_key = "-----BEGIN PRIVATE KEY-----\nprivate-material\n-----END PRIVATE KEY-----"
|
||||
value = (
|
||||
"AKIAIOSFODNN7EXAMPLE "
|
||||
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.signature "
|
||||
"mysql://admin:database-password@db.internal/lai "
|
||||
f"{private_key}"
|
||||
)
|
||||
result = redact(value)
|
||||
assert "AKIAIOSFODNN7EXAMPLE" not in result
|
||||
assert "eyJhbGci" not in result
|
||||
assert "database-password" not in result
|
||||
assert "private-material" not in result
|
||||
|
||||
|
||||
def test_retry_task_contains_failure_without_secrets() -> None:
|
||||
report = FailureReport(
|
||||
id=8,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Use a weighted average",
|
||||
prompt="Average sale price",
|
||||
response="Zero",
|
||||
sql_query="SELECT AVG(list_price)",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
task = build_retry_task(report, RuntimeError("Missing import; token=secret-value"))
|
||||
|
||||
assert "Missing import" in task
|
||||
assert "secret-value" not in task
|
||||
assert "Do not remove or weaken tests" in task
|
||||
assert "unrelated production routing" in task
|
||||
assert "Use a weighted average" in task
|
||||
assert "SELECT AVG(list_price)" in task
|
||||
|
||||
|
||||
def test_incorrect_answer_task_guides_semantics_level_repair() -> None:
|
||||
report = FailureReport(
|
||||
id=42,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Sum net values and divide by quantity",
|
||||
prompt="Average sale price for article 9031",
|
||||
response="0.00",
|
||||
sql_query="SELECT AVG(Vk_preis) FROM FK_Artikel",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "config/*_semantics.php" in task
|
||||
assert "Do not hardcode" in task
|
||||
assert "protected schema/business-" in task
|
||||
assert "context builder" in task
|
||||
assert "known-bad SQL" in task
|
||||
assert "compare every alias.column" in task
|
||||
assert "Never replace one missing column" in task
|
||||
assert "Never require or copy production rows" in task
|
||||
assert repair_files(report) == (
|
||||
"config/erp_semantics.php",
|
||||
"tests/Unit/SQLGeneratorBusinessContextTest.php",
|
||||
)
|
||||
|
||||
|
||||
def test_task_derives_exact_sql_literal_contract() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
"FK_Auftrag nach Auftragskennung 2 filtern, System_created nutzen, "
|
||||
"obsolet und status storniert ausfiltern"
|
||||
),
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query="SELECT Datum_erfassung",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "Exact SQL literal contract" in task
|
||||
assert "a.AuftragsKennung = 2" in task
|
||||
assert "a.System_created" in task
|
||||
assert "COALESCE(a.obsolet, 0) = 0" in task
|
||||
assert "COALESCE(a.bStatus_storniert, 0) = 0" in task
|
||||
assert "Do not add an LS lookup or bStatus_gebucht" in task
|
||||
|
||||
|
||||
def test_task_derives_exact_negative_contract_for_removal_only_report() -> None:
|
||||
report = FailureReport(
|
||||
id=5,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong delivery note count",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
|
||||
"Einfach nur das aus der SQL entfernen"
|
||||
),
|
||||
prompt="Delivery notes by month",
|
||||
response="0 rows",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "Exact SQL removal contract" in task
|
||||
assert "a.bStatus_gebucht = 1" in task
|
||||
assert "negative assertion" in task
|
||||
assert "forbidden identifier is a negative requirement" in task
|
||||
assert (
|
||||
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
||||
in task
|
||||
)
|
||||
assert "copy them exactly, including aliases and qualifiers" in task
|
||||
assert "explicitly a removal-only repair" in task
|
||||
assert "Preserve every other selected field" in task
|
||||
assert "Exact SQL literal contract" not in task
|
||||
|
||||
|
||||
def test_non_removal_only_contract_protects_correct_reference_and_exercises_rule() -> None:
|
||||
report = FailureReport(
|
||||
id=6,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Die Abfrage stimmt nicht",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
||||
"weil Lieferscheine nicht gebucht werden."
|
||||
),
|
||||
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
response="0 Ergebnisse",
|
||||
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "protected correct baseline" in task
|
||||
assert "preserve it exactly" in task
|
||||
assert "positively assert the new rule" in task
|
||||
assert "negative assertion alone is vacuous" in task
|
||||
assert "without embedding the complete forbidden predicate" in task
|
||||
assert "Do not add or restate date-column choices" in task
|
||||
assert "every table in its tables list" in task
|
||||
assert "supplied to businessContext()" in task
|
||||
assert "explicitly a removal-only repair" not in task
|
||||
|
||||
|
||||
def test_retry_task_keeps_failure_from_start_of_long_output() -> None:
|
||||
report = FailureReport(
|
||||
id=43,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="Top products",
|
||||
response=None,
|
||||
sql_query="SELECT a.missing FROM orders a",
|
||||
error_message="Unknown column",
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_retry_task(report, RuntimeError("DECISIVE ASSERTION\n" + "x" * 11_000))
|
||||
|
||||
assert "DECISIVE ASSERTION" in task
|
||||
|
||||
|
||||
def test_customer_database_task_explains_tenant_scope() -> None:
|
||||
report = FailureReport(
|
||||
id=13,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="What did this customer buy most?",
|
||||
response=None,
|
||||
sql_query="SELECT * FROM invoice_row WHERE InvoiceCreator = 'tenant'",
|
||||
error_message="Unknown column InvoiceCreator",
|
||||
context={"database_type": "customer"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "already selected one tenant database" in task
|
||||
assert "must not" in task
|
||||
assert "guessed column" in task
|
||||
assert "tenant_type" in task
|
||||
|
||||
|
||||
def test_unresolved_placeholder_task_guides_real_answer_path() -> None:
|
||||
report = FailureReport(
|
||||
id=49,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Token was not restored",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Return the database value",
|
||||
prompt="Average price",
|
||||
response="Price: __LAI_UNKNOWN_VALUE__",
|
||||
sql_query="SELECT AVG(price)",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "AIService::generateDatabaseResponse" in task
|
||||
assert "exact token map" in task
|
||||
assert "HTTP-backed AIService test" in task
|
||||
assert "local database-result fallback" in task
|
||||
|
||||
|
||||
def test_cross_database_report_guides_staged_source_routing() -> None:
|
||||
report = FailureReport(
|
||||
id=56,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong hours",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Join cockpit_config.time_registration to customer manager and ERP positions",
|
||||
prompt="Hours in April",
|
||||
response="0 hours",
|
||||
sql_query="SELECT SUM(hours) FROM LX_AUFGABE",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
task = build_task(report)
|
||||
|
||||
assert "tables from multiple databases" in task
|
||||
assert "staged-query workflows" in task
|
||||
assert "Do not put an impossible cross-connection join" in task
|
||||
assert "preserving tenant isolation" in task
|
||||
assert "editing only config/*_semantics.php or tests is an invalid repair" in task
|
||||
assert "describe the data" in task
|
||||
assert "flow, connection used at each stage" in task
|
||||
assert repair_files(report) == (
|
||||
"app/Http/Controllers/ChatController.php",
|
||||
"app/Services/TenantDatabaseService.php",
|
||||
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||
)
|
||||
|
||||
|
||||
def test_presentation_and_route_feedback_is_routed_to_real_owners() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
|
||||
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
|
||||
),
|
||||
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
response="2024-01: 38, 2024-02: 95",
|
||||
sql_query="SELECT Monat, COUNT(*) FROM FK_Auftrag GROUP BY Monat",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
|
||||
assert repair_files(report) == (
|
||||
"app/Services/AIService.php",
|
||||
"app/Http/Controllers/ChatController.php",
|
||||
"app/Services/WebSearchService.php",
|
||||
"tests/Feature/AIPrivacyBoundaryTest.php",
|
||||
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
||||
"tests/Unit/WebSearchServiceTest.php",
|
||||
)
|
||||
task = build_task(report)
|
||||
assert "real answer-formatting change" in task
|
||||
assert "exact reported user prompt" in task
|
||||
assert '"cockpit:" must force the database path' in task
|
||||
assert '"www:" must force public web research' in task
|
||||
assert '"direct:" must' in task
|
||||
assert "including Ollama/Qwen" in task
|
||||
assert "Do not pass private conversation/database context" in task
|
||||
assert "AIService::determineRoute() must not be called" in task
|
||||
assert "resolveRoutePrefix(string $message): array" in task
|
||||
assert "[$explicitRoute, $message] = $this->resolveRoutePrefix($message)" in task
|
||||
assert "Do not add prefix handling to resolveDatabaseSource()" in task
|
||||
assert "parent::resolveRoutePrefix($message)" in task
|
||||
assert "three tests that call processMessage() itself" in task
|
||||
assert "setLaravelSession()" in task
|
||||
assert "generateWebResearchResponse()" in task
|
||||
assert "columns named exactly Monat and Anzahl_Lieferscheine" in task
|
||||
assert 'exact substring "allen bereitgestellten Zeilen"' in task
|
||||
assert "substr_count($userPrompt" in task
|
||||
assert "heuristics must return false" in task
|
||||
assert "persisted ChatMessage.message equals the cleaned question exactly" in task
|
||||
assert "$response->getData(true)" in task
|
||||
assert "Do not write “we can't fully test processMessage”" in task
|
||||
assert "one ordinary result row with multiple" in task
|
||||
assert "duplicate method declaration" in task
|
||||
assert "All three prefix behaviors" in task
|
||||
assert "config/*_semantics.php" in task
|
||||
145
old_2026-08-31/tests/test_removal.py
Normal file
145
old_2026-08-31/tests/test_removal.py
Normal file
@@ -0,0 +1,145 @@
|
||||
from pathlib import Path
|
||||
|
||||
from lai_automation.models import FailureReport
|
||||
from lai_automation.removal import (
|
||||
detect_already_fixed,
|
||||
extract_reference_examples,
|
||||
find_matching_reference,
|
||||
is_removal_only,
|
||||
removal_directives,
|
||||
remove_sql_fragment,
|
||||
sql_contains_fragment,
|
||||
)
|
||||
|
||||
|
||||
def removal_report(**overrides) -> FailureReport:
|
||||
values = {
|
||||
"id": 5,
|
||||
"chat_message_id": 1,
|
||||
"source": "user_feedback",
|
||||
"category": "incorrect_answer",
|
||||
"user_feedback": "Die Anzahl der Lieferscheine ist falsch",
|
||||
"feedback_type": "wrong_result",
|
||||
"expected_behavior": (
|
||||
'Es muss " a.bStatus_gebucht = 1 " entfernt werden! '
|
||||
"Lieferscheine werden nicht gebucht! Einfach nur das aus der SQL entfernen"
|
||||
),
|
||||
"prompt": "Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
"response": "0 Ergebnisse",
|
||||
"sql_query": (
|
||||
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||
"AND a.bStatus_gebucht = 1 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
||||
),
|
||||
"error_message": None,
|
||||
"context": {"database_type": "erp"},
|
||||
}
|
||||
values.update(overrides)
|
||||
return FailureReport(**values)
|
||||
|
||||
|
||||
def semantics(sql: str, question: str | None = None) -> str:
|
||||
question = question or "Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024"
|
||||
return f"""<?php
|
||||
|
||||
return [
|
||||
'examples' => [
|
||||
[
|
||||
'question' => '{question}',
|
||||
'sql' => <<<'SQL'
|
||||
{sql}
|
||||
SQL,
|
||||
],
|
||||
],
|
||||
];
|
||||
"""
|
||||
|
||||
|
||||
def test_extracts_exact_quoted_german_removal_contract() -> None:
|
||||
directives = removal_directives(removal_report())
|
||||
|
||||
assert tuple(item.fragment for item in directives) == ("a.bStatus_gebucht = 1",)
|
||||
assert is_removal_only(removal_report())
|
||||
|
||||
|
||||
def test_infers_unquoted_predicate_from_observed_sql() -> None:
|
||||
report = removal_report(
|
||||
expected_behavior="Bitte den bStatus_gebucht Filter aus der SQL entfernen",
|
||||
)
|
||||
|
||||
directives = removal_directives(report)
|
||||
|
||||
assert len(directives) == 1
|
||||
assert sql_contains_fragment(directives[0].fragment, "a.bStatus_gebucht = 1")
|
||||
|
||||
|
||||
def test_does_not_invert_do_not_remove_instruction() -> None:
|
||||
report = removal_report(
|
||||
expected_behavior="Do not remove `a.bStatus_gebucht = 1` from invoice SQL",
|
||||
)
|
||||
|
||||
assert removal_directives(report) == ()
|
||||
assert not is_removal_only(report)
|
||||
|
||||
|
||||
def test_removes_only_the_requested_conjunct() -> None:
|
||||
sql = (
|
||||
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung=2 "
|
||||
"AND a.bStatus_gebucht=1 AND COALESCE(a.obsolet,0)=0 GROUP BY a.SheetNr;"
|
||||
)
|
||||
|
||||
repaired = remove_sql_fragment(sql, "a.bStatus_gebucht = 1")
|
||||
|
||||
assert not sql_contains_fragment(repaired, "a.bStatus_gebucht = 1")
|
||||
assert "a.AuftragsKennung=2" in repaired
|
||||
assert "COALESCE(a.obsolet,0)=0" in repaired
|
||||
assert "GROUP BY a.SheetNr" in repaired
|
||||
|
||||
|
||||
def test_extracts_and_uniquely_matches_reference_example() -> None:
|
||||
content = semantics("SELECT 1") + semantics(
|
||||
"SELECT 2",
|
||||
"Welcher Kunde hatte im Januar den meisten Umsatz?",
|
||||
)
|
||||
|
||||
examples = extract_reference_examples(content)
|
||||
match, score = find_matching_reference(content, removal_report().prompt)
|
||||
|
||||
assert len(examples) == 2
|
||||
assert match is not None
|
||||
assert match.sql == "SELECT 1"
|
||||
assert score == 1.0
|
||||
|
||||
|
||||
def test_detects_already_fixed_only_with_matching_reference(tmp_path: Path) -> None:
|
||||
source = tmp_path / "config/erp_semantics.php"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text(semantics(
|
||||
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
||||
))
|
||||
|
||||
result = detect_already_fixed(
|
||||
removal_report(),
|
||||
tmp_path,
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
)
|
||||
|
||||
assert result is not None
|
||||
assert result.filename == "config/erp_semantics.php"
|
||||
assert result.match_score == 1.0
|
||||
assert result.forbidden_fragments == ("a.bStatus_gebucht = 1",)
|
||||
|
||||
|
||||
def test_does_not_mark_additive_or_mismatched_report_already_fixed(tmp_path: Path) -> None:
|
||||
source = tmp_path / "config/erp_semantics.php"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text(semantics("SELECT COUNT(*) FROM FK_Auftrag a"))
|
||||
additive = removal_report(
|
||||
expected_behavior=(
|
||||
'"a.bStatus_gebucht = 1" entfernen und zusätzlich System_created verwenden'
|
||||
),
|
||||
)
|
||||
mismatched = removal_report(prompt="Zeige Rechnungen im Jahr 2024")
|
||||
|
||||
assert detect_already_fixed(additive, tmp_path, ("config/erp_semantics.php",)) is None
|
||||
assert detect_already_fixed(mismatched, tmp_path, ("config/erp_semantics.php",)) is None
|
||||
968
old_2026-08-31/tests/test_worker.py
Normal file
968
old_2026-08-31/tests/test_worker.py
Normal file
@@ -0,0 +1,968 @@
|
||||
import threading
|
||||
from pathlib import Path
|
||||
from unittest.mock import MagicMock, Mock, patch
|
||||
|
||||
import pytest
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.agent import AgentProviderError, AgentTimeoutError
|
||||
from lai_automation.git import ChangeSummary
|
||||
from lai_automation.models import FailureReport
|
||||
from lai_automation.worker import AutomationWorker, ValidationSetupError
|
||||
|
||||
|
||||
def test_dry_run_generates_task_without_agent_or_git() -> None:
|
||||
report = FailureReport(
|
||||
id=11,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List locations",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message="Unknown column",
|
||||
context={},
|
||||
)
|
||||
worker = AutomationWorker(settings())
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
|
||||
assert worker.run_once()
|
||||
worker.reports.release.assert_called_once()
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.assert_not_called()
|
||||
|
||||
|
||||
def test_stop_signal_interrupts_worker_wait() -> None:
|
||||
worker = AutomationWorker(settings())
|
||||
|
||||
worker._stop(15, None)
|
||||
|
||||
assert worker.stop_event.is_set()
|
||||
|
||||
|
||||
def test_pull_request_body_contains_redacted_original_report() -> None:
|
||||
report = FailureReport(
|
||||
id=7,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Use a table\ntoken=super-secret-token",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Show every supplied row",
|
||||
prompt="Count delivery notes by month",
|
||||
response="January: 38, February: 95",
|
||||
sql_query="SELECT month, COUNT(*) FROM deliveries GROUP BY month",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
summary = ChangeSummary(("app/Services/AIService.php",), 5, 1)
|
||||
|
||||
body = AutomationWorker._pull_request_body(
|
||||
report,
|
||||
summary,
|
||||
[{"command": "php artisan test", "returncode": 0}],
|
||||
)
|
||||
|
||||
assert "## Original failure report" in body
|
||||
assert "**Source**\n> user_feedback" in body
|
||||
assert "**Category**\n> incorrect_answer" in body
|
||||
assert "**Feedback type**\n> wrong_result" in body
|
||||
assert "**User prompt**\n> Count delivery notes by month" in body
|
||||
assert "**Observed response**\n> January: 38, February: 95" in body
|
||||
assert "**User feedback**\n> Use a table\n> token=[REDACTED]" in body
|
||||
assert "**Expected behavior**\n> Show every supplied row" in body
|
||||
assert "**Error message**\n> _Not provided._" in body
|
||||
assert "**Generated SQL**\n> SELECT month, COUNT(*)" in body
|
||||
assert "super-secret-token" not in body
|
||||
assert "## Scope\n- `app/Services/AIService.php`" in body
|
||||
assert "## Validation\n- PASS `php artisan test`" in body
|
||||
|
||||
|
||||
def test_database_credentials_are_only_passed_to_test_process(tmp_path) -> None:
|
||||
worker = AutomationWorker(settings())
|
||||
worker._prepare_test_schema = Mock()
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.run.return_value = Mock(returncode=0, stdout="ok", stderr="")
|
||||
|
||||
results = worker._run_tests(workspace)
|
||||
|
||||
assert results[0]["returncode"] == 0
|
||||
environment = workspace.run.call_args.kwargs["extra_environment"]
|
||||
assert environment["AGENT_TEST_DB_ENABLED"] == "true"
|
||||
assert environment["APP_ENV"] == "testing"
|
||||
assert environment["AI_MISTRAL_API_KEY"] == "automation-test-key"
|
||||
assert environment["TEST_DB_USER"] == "test_runner"
|
||||
assert environment["TEST_DB_PASSWORD"] == "test-secret"
|
||||
assert environment["DB_DATABASE"] == "lai_agent_test"
|
||||
assert environment["ERP_DATABASE"] == "customer_agent_test"
|
||||
assert environment["COCKPIT_CONFIG_DATABASE"] == "cockpit_agent_test"
|
||||
|
||||
|
||||
def test_workspace_setup_creates_safe_env_and_prepares_assets(tmp_path) -> None:
|
||||
worker = AutomationWorker(settings(
|
||||
workspace_root=tmp_path,
|
||||
setup_commands=("npm run build",),
|
||||
))
|
||||
worker._prepare_test_schema = Mock()
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path / "failure-1"
|
||||
workspace.path.mkdir()
|
||||
workspace.run.return_value = Mock(returncode=0, stdout="built", stderr="")
|
||||
|
||||
results = worker._prepare_workspace_for_agent(workspace, 1)
|
||||
|
||||
env_contents = (workspace.path / ".env").read_text()
|
||||
assert "APP_ENV=testing" in env_contents
|
||||
assert "DB_PASSWORD" not in env_contents
|
||||
assert results[0]["stage"] == "workspace setup"
|
||||
assert workspace.run.call_args.kwargs["extra_environment"]["DB_PASSWORD"] == "test-secret"
|
||||
|
||||
|
||||
def test_processing_connection_failure_is_not_sent_to_agent() -> None:
|
||||
report = FailureReport(
|
||||
id=15,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="processing",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List customers",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message="Access denied for user 'runtime'@'host'",
|
||||
context={},
|
||||
)
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
|
||||
assert worker.run_once()
|
||||
|
||||
worker.agent.run.assert_not_called()
|
||||
worker.reports.fail.assert_called_once()
|
||||
assert worker.reports.fail.call_args.args[2]["classification"] == "operational"
|
||||
|
||||
|
||||
def test_model_provider_failure_is_classified_as_operational() -> None:
|
||||
report = FailureReport(
|
||||
id=16,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Use the correct rule",
|
||||
prompt="List records",
|
||||
response="0 rows",
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = AgentProviderError("Mistral DNS failed")
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-16"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
worker.reports.fail.assert_called_once()
|
||||
metadata = worker.reports.fail.call_args.args[2]
|
||||
assert metadata["classification"] == "operational"
|
||||
assert metadata["automation_action"] == "no_source_change"
|
||||
workspace.commit.assert_not_called()
|
||||
|
||||
|
||||
def test_test_schema_creates_minimal_erp_lookup(monkeypatch) -> None:
|
||||
worker = AutomationWorker(settings())
|
||||
cursor = Mock()
|
||||
connection = MagicMock()
|
||||
connection.cursor.return_value.__enter__.return_value = cursor
|
||||
connect = Mock(return_value=connection)
|
||||
monkeypatch.setattr("lai_automation.worker.pymysql.connect", connect)
|
||||
|
||||
worker._prepare_test_schema()
|
||||
|
||||
assert connect.call_args.kwargs["database"] == "customer_agent_test"
|
||||
assert "CREATE TABLE IF NOT EXISTS FK_Nummerkreis" in cursor.execute.call_args.args[0]
|
||||
assert cursor.executemany.call_args.args[1][0] == (3, "Rechnung", "RG")
|
||||
connection.close.assert_called_once()
|
||||
|
||||
|
||||
def test_validation_commands_are_logged_without_output(caplog, tmp_path) -> None:
|
||||
worker = AutomationWorker(settings())
|
||||
worker._prepare_test_schema = Mock()
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.run.return_value = Mock(returncode=0, stdout="sensitive output", stderr="")
|
||||
|
||||
with caplog.at_level("INFO"):
|
||||
worker._run_tests(workspace, report_id=7, attempt=2)
|
||||
|
||||
assert "Report 7 attempt 2 validation command: php artisan test starting" in caplog.text
|
||||
assert "returncode=0" in caplog.text
|
||||
assert "sensitive output" not in caplog.text
|
||||
|
||||
|
||||
def test_failed_validation_preserves_start_and_end_of_long_output(tmp_path) -> None:
|
||||
worker = AutomationWorker(settings())
|
||||
worker._prepare_test_schema = Mock()
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.run.return_value = Mock(
|
||||
returncode=1,
|
||||
stdout="DECISIVE ASSERTION\n" + "x" * 13_000,
|
||||
stderr="FINAL ERROR",
|
||||
)
|
||||
|
||||
try:
|
||||
worker._run_tests(workspace)
|
||||
except RuntimeError as exception:
|
||||
failure = str(exception)
|
||||
else:
|
||||
raise AssertionError("Expected failed validation")
|
||||
|
||||
assert "DECISIVE ASSERTION" in failure
|
||||
assert "...[middle omitted]..." in failure
|
||||
assert "FINAL ERROR" in failure
|
||||
|
||||
|
||||
def test_validation_signature_ignores_counts_timings_and_line_numbers() -> None:
|
||||
first = """
|
||||
FAILED Tests\\Unit\\RouteTest > prefixes are removed
|
||||
Expectation failed for method determineRoute when invoked 1 time.
|
||||
Method was expected to be called 1 time, actually called 0 times.
|
||||
Tests: 1 failed, 50 passed (186 assertions)
|
||||
Duration: 0.49s
|
||||
at tests/Unit/RouteTest.php:229
|
||||
"""
|
||||
second = """
|
||||
FAILED Tests\\Unit\\RouteTest > prefixes are removed
|
||||
Expectation failed for method determineRoute when invoked 1 time.
|
||||
Method was expected to be called 1 time, actually called 0 times.
|
||||
Tests: 1 failed, 50 passed (191 assertions)
|
||||
Duration: 0.50s
|
||||
at tests/Unit/RouteTest.php:269
|
||||
"""
|
||||
|
||||
assert AutomationWorker._validation_signature(first) == AutomationWorker._validation_signature(
|
||||
second
|
||||
)
|
||||
|
||||
|
||||
def test_validation_setup_failure_is_typed_and_not_a_test_failure(tmp_path) -> None:
|
||||
worker = AutomationWorker(settings(setup_commands=("npm ci",)))
|
||||
worker._prepare_test_schema = Mock()
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.run.return_value = Mock(
|
||||
returncode=1,
|
||||
stdout="network unavailable",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
with pytest.raises(ValidationSetupError, match="Validation setup command failed"):
|
||||
worker._run_tests(workspace)
|
||||
|
||||
|
||||
def test_long_stage_emits_heartbeat(caplog) -> None:
|
||||
worker = AutomationWorker(settings())
|
||||
|
||||
with caplog.at_level("INFO"), worker._progress(7, "agent attempt 1", interval=0.001):
|
||||
threading.Event().wait(0.01)
|
||||
|
||||
assert "Report 7 agent attempt 1 still running" in caplog.text
|
||||
assert "Report 7 agent attempt 1 finished" in caplog.text
|
||||
|
||||
|
||||
def test_long_stage_refreshes_database_claim_lease() -> None:
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
|
||||
with worker._progress(7, "agent attempt 1", interval=0.001):
|
||||
threading.Event().wait(0.01)
|
||||
|
||||
assert worker.reports.heartbeat.call_count >= 2
|
||||
|
||||
|
||||
def test_timed_out_agent_partial_changes_are_validated_and_published() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Use System_created",
|
||||
prompt="Delivery notes by month",
|
||||
response="0 rows",
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = AgentTimeoutError("timed out after applying edits")
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-4"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
20,
|
||||
2,
|
||||
)
|
||||
workspace.added_text.return_value = (
|
||||
"Use DATE_FORMAT(System_created, '%Y-%m') and AuftragsKennung = 2; "
|
||||
"COALESCE(obsolet, 0) = 0 and COALESCE(bStatus_storniert, 0) = 0"
|
||||
)
|
||||
workspace.run_args.return_value = Mock(
|
||||
returncode=0,
|
||||
stdout="No syntax errors",
|
||||
stderr="",
|
||||
)
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
worker._run_tests.assert_called_once()
|
||||
workspace.push_branch.assert_called_once()
|
||||
worker.reports.complete.assert_called_once()
|
||||
|
||||
|
||||
def test_provider_failure_partial_changes_are_validated_and_published() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Use System_created",
|
||||
prompt="Delivery notes by month",
|
||||
response="0 rows",
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = AgentProviderError("provider failed after edits")
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-4"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
20,
|
||||
2,
|
||||
)
|
||||
workspace.added_text.return_value = (
|
||||
"Use DATE_FORMAT(System_created, '%Y-%m') and AuftragsKennung = 2; "
|
||||
"COALESCE(obsolet, 0) = 0 and COALESCE(bStatus_storniert, 0) = 0"
|
||||
)
|
||||
workspace.run_args.return_value = Mock(
|
||||
returncode=0,
|
||||
stdout="No syntax errors",
|
||||
stderr="",
|
||||
)
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
worker._run_tests.assert_called_once()
|
||||
workspace.push_branch.assert_called_once()
|
||||
worker.reports.complete.assert_called_once()
|
||||
|
||||
|
||||
def test_php_syntax_failure_discards_attempt_before_full_validation() -> None:
|
||||
report = FailureReport(
|
||||
id=56,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong hours",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Use time registration",
|
||||
prompt="Hours in April",
|
||||
response="0 hours",
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||
worker = AutomationWorker(config)
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["broken PHP", "fixed PHP"]
|
||||
worker.gitea = Mock()
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-56"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
|
||||
)
|
||||
workspace.run_args.side_effect = [
|
||||
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
|
||||
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||
]
|
||||
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+intended fix"
|
||||
workspace.checkpoint.return_value = "valid-checkpoint"
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 2
|
||||
assert "unexpected token" in worker.agent.run.call_args_list[1].args[1]
|
||||
assert "clean repository" in worker.agent.run.call_args_list[1].args[1]
|
||||
assert "intended fix" in worker.agent.run.call_args_list[1].args[1]
|
||||
assert "repair-only attempt" in worker.agent.run.call_args_list[1].args[1]
|
||||
workspace.restore_checkpoint.assert_called_once_with(None)
|
||||
worker._run_tests.assert_called_once()
|
||||
|
||||
|
||||
def test_malformed_retry_restores_last_syntax_valid_attempt() -> None:
|
||||
report = FailureReport(
|
||||
id=56,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong hours",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Use time registration",
|
||||
prompt="Hours in April",
|
||||
response="0 hours",
|
||||
sql_query="SELECT 0",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
config = settings(mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3)
|
||||
worker = AutomationWorker(config)
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["valid but failing", "broken retry", "corrected retry"]
|
||||
worker.gitea = Mock()
|
||||
worker._run_tests = Mock(side_effect=[RuntimeError("Expected hour rule"), []])
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-56"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
|
||||
)
|
||||
workspace.run_args.side_effect = [
|
||||
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
|
||||
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||
Mock(returncode=0, stdout="No syntax errors", stderr=""),
|
||||
]
|
||||
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+application repair"
|
||||
workspace.checkpoint.side_effect = ["checkpoint-1", "checkpoint-3"]
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 3
|
||||
workspace.restore_checkpoint.assert_called_once_with("checkpoint-1")
|
||||
assert "last syntax-valid changes were restored" in worker.agent.run.call_args_list[2].args[1]
|
||||
assert "application repair" in worker.agent.run.call_args_list[2].args[1]
|
||||
assert worker._run_tests.call_count == 2
|
||||
|
||||
|
||||
def test_failed_validation_gets_one_correction_before_pr() -> None:
|
||||
report = FailureReport(
|
||||
id=13,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List products",
|
||||
response=None,
|
||||
sql_query="SELECT missing_column FROM invoice_row",
|
||||
error_message="Unknown column",
|
||||
context={},
|
||||
)
|
||||
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||
worker = AutomationWorker(config)
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["first change", "corrected change"]
|
||||
worker.gitea = Mock()
|
||||
worker._run_tests = Mock(side_effect=[RuntimeError("Missing import"), []])
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-13"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1
|
||||
)
|
||||
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 2
|
||||
assert "Missing import" in worker.agent.run.call_args_list[1].args[1]
|
||||
assert worker._run_tests.call_count == 2
|
||||
workspace.push_branch.assert_called_once()
|
||||
worker.reports.complete.assert_called_once()
|
||||
|
||||
|
||||
def test_failed_validation_retry_keeps_full_routed_context() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="AuftragsKennung 2 verwenden",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query="SELECT Datum_erfassung",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["valid production with bad test", "fixed assertion"]
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
worker._run_tests = Mock(side_effect=[RuntimeError("case-sensitive assertion failed"), []])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-4"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
20,
|
||||
1,
|
||||
)
|
||||
added = (
|
||||
"AuftragsKennung = 2\n"
|
||||
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2"
|
||||
)
|
||||
workspace.added_text.return_value = added
|
||||
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||
workspace.checkpoint.return_value = "valid-production"
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_args_list[1].args[2] == (
|
||||
"config/erp_semantics.php",
|
||||
"tests/Unit/SQLGeneratorBusinessContextTest.php",
|
||||
)
|
||||
retry_task = worker.agent.run.call_args_list[1].args[1]
|
||||
assert "inspect every routed production and test file" in retry_task
|
||||
assert "Do not merely rewrite assertions" in retry_task
|
||||
assert "case-sensitive" in worker.agent.run.call_args_list[1].args[1]
|
||||
attempts = worker.reports.complete.call_args.args[1]["agent_attempts"]
|
||||
assert attempts[0]["validation_status"] == "failed"
|
||||
assert "case-sensitive assertion failed" in attempts[0]["validation_error_tail"]
|
||||
assert attempts[1]["validation_status"] == "passed"
|
||||
|
||||
|
||||
def test_no_change_response_gets_one_correction() -> None:
|
||||
report = FailureReport(
|
||||
id=14,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List products",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
error_message="Unknown column",
|
||||
context={},
|
||||
)
|
||||
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||
worker = AutomationWorker(config)
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["no edit", "applied edit"]
|
||||
worker.gitea = Mock()
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-14"
|
||||
workspace.summary.side_effect = [
|
||||
ChangeSummary((), 0, 0),
|
||||
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
|
||||
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
|
||||
]
|
||||
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 2
|
||||
assert "Agent produced no file changes" in worker.agent.run.call_args_list[1].args[1]
|
||||
worker._run_tests.assert_called_once()
|
||||
workspace.push_branch.assert_called_once()
|
||||
|
||||
|
||||
def test_test_only_change_gets_correction_before_validation() -> None:
|
||||
report = FailureReport(
|
||||
id=49,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Token was not restored",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="Return the real value",
|
||||
prompt="Average price",
|
||||
response="__LAI_UNKNOWN__",
|
||||
sql_query="SELECT AVG(price)",
|
||||
error_message=None,
|
||||
context={},
|
||||
)
|
||||
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||
worker = AutomationWorker(config)
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["tests only", "production fix"]
|
||||
worker.gitea = Mock()
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-49"
|
||||
workspace.summary.side_effect = [
|
||||
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 8, 0),
|
||||
ChangeSummary(
|
||||
("app/Services/AIService.php", "tests/Feature/AIPrivacyBoundaryTest.php"),
|
||||
15,
|
||||
1,
|
||||
),
|
||||
ChangeSummary(
|
||||
("app/Services/AIService.php", "tests/Feature/AIPrivacyBoundaryTest.php"),
|
||||
15,
|
||||
1,
|
||||
),
|
||||
]
|
||||
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 2
|
||||
assert "changed only tests" in worker.agent.run.call_args_list[1].args[1]
|
||||
assert "exact repository-relative paths" in worker.agent.run.call_args_list[1].args[1]
|
||||
workspace.checkpoint.assert_called_once()
|
||||
worker._run_tests.assert_called_once()
|
||||
|
||||
|
||||
def test_semantically_invalid_change_is_not_kept_as_retry_checkpoint() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="System_created verwenden und obsolet ausfiltern",
|
||||
prompt="Delivery notes by month",
|
||||
response="0 rows",
|
||||
sql_query="SELECT Datum_erfassung",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["missing filter", "correct repair"]
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-4"
|
||||
workspace.summary.return_value = ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
12,
|
||||
1,
|
||||
)
|
||||
workspace.added_text.side_effect = [
|
||||
"System_created",
|
||||
"System_created AND COALESCE(obsolet, 0) = 0",
|
||||
"System_created AND COALESCE(obsolet, 0) = 0",
|
||||
]
|
||||
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||
workspace.checkpoint.return_value = "policy-valid-checkpoint"
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
workspace.restore_checkpoint.assert_called_once_with(None)
|
||||
workspace.checkpoint.assert_called_once()
|
||||
worker._run_tests.assert_called_once()
|
||||
|
||||
|
||||
def test_policy_constraints_accumulate_across_retries() -> None:
|
||||
report = FailureReport(
|
||||
id=4,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong result",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior="AuftragsKennung 2 und System_created verwenden; obsolet ausfiltern",
|
||||
prompt="Delivery notes by month",
|
||||
response=None,
|
||||
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS'",
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
worker = AutomationWorker(settings(
|
||||
mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3
|
||||
))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["incomplete", "test only", "complete"]
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-4"
|
||||
workspace.summary.side_effect = [
|
||||
ChangeSummary(("config/erp_semantics.php",), 5, 0),
|
||||
ChangeSummary(("tests/Unit/SQLGeneratorBusinessContextTest.php",), 5, 0),
|
||||
ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
20,
|
||||
1,
|
||||
),
|
||||
ChangeSummary(
|
||||
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
||||
20,
|
||||
1,
|
||||
),
|
||||
]
|
||||
complete = (
|
||||
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0\n"
|
||||
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
||||
"AND System_created IS NOT NULL AND COALESCE(obsolet, 0) = 0"
|
||||
)
|
||||
workspace.added_text.side_effect = ["AuftragsKennung = 2", "test", complete, complete]
|
||||
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
|
||||
workspace.checkpoint.return_value = "valid"
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
third_task = worker.agent.run.call_args_list[2].args[1]
|
||||
assert "All policy constraints from earlier attempts remain mandatory" in third_task
|
||||
assert "one reference SELECT" in third_task
|
||||
assert "changed only tests" in third_task
|
||||
|
||||
|
||||
def test_already_fixed_removal_report_skips_agent_and_dependency_setup(tmp_path) -> None:
|
||||
report = FailureReport(
|
||||
id=5,
|
||||
chat_message_id=1,
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="Wrong delivery note count",
|
||||
feedback_type="wrong_result",
|
||||
expected_behavior=(
|
||||
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
|
||||
"Einfach nur das aus der SQL entfernen"
|
||||
),
|
||||
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
||||
response="0 rows",
|
||||
sql_query=(
|
||||
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
||||
"AND a.bStatus_gebucht = 1"
|
||||
),
|
||||
error_message=None,
|
||||
context={"database_type": "erp"},
|
||||
)
|
||||
source = tmp_path / "config/erp_semantics.php"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text("""<?php
|
||||
return ['examples' => [[
|
||||
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
||||
'sql' => <<<'SQL'
|
||||
SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
|
||||
SQL,
|
||||
]]];
|
||||
""")
|
||||
worker = AutomationWorker(settings(mode="pr", dry_run=False, gitea_token="token"))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock()
|
||||
workspace = Mock()
|
||||
workspace.path = tmp_path
|
||||
workspace.branch = "ai/failure-5"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
workspace.prepare.assert_called_once()
|
||||
worker._prepare_workspace_for_agent.assert_not_called()
|
||||
worker.agent.run.assert_not_called()
|
||||
workspace.commit.assert_not_called()
|
||||
workspace.push_branch.assert_not_called()
|
||||
worker.gitea.create_pull_request.assert_not_called()
|
||||
worker.reports.complete.assert_called_once()
|
||||
metadata = worker.reports.complete.call_args.args[1]
|
||||
assert metadata["classification"] == "already_fixed"
|
||||
assert metadata["automation_action"] == "no_source_change"
|
||||
assert metadata["forbidden_fragments"] == ["a.bStatus_gebucht = 1"]
|
||||
|
||||
|
||||
def test_identical_policy_failure_stops_after_two_attempts() -> None:
|
||||
report = FailureReport(
|
||||
id=18,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List generic records",
|
||||
response=None,
|
||||
sql_query="SELECT missing FROM records",
|
||||
error_message="Unknown column",
|
||||
context={},
|
||||
)
|
||||
worker = AutomationWorker(settings(
|
||||
mode="pr",
|
||||
dry_run=False,
|
||||
gitea_token="token",
|
||||
max_agent_attempts=5,
|
||||
))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["tests only", "still tests only", "must not run"]
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-18"
|
||||
workspace.summary.side_effect = [
|
||||
ChangeSummary(("tests/Unit/FixTest.php",), 2, 0),
|
||||
ChangeSummary(("tests/Unit/FixTest.php",), 3, 0),
|
||||
]
|
||||
workspace.added_text.return_value = ""
|
||||
workspace.diff.return_value = "same rejected candidate"
|
||||
workspace.run_args.return_value = Mock(
|
||||
returncode=0,
|
||||
stdout="No syntax errors",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 2
|
||||
workspace.commit.assert_not_called()
|
||||
worker.reports.fail.assert_called_once()
|
||||
assert "Repeated identical candidate rejection" in worker.reports.fail.call_args.args[1]
|
||||
|
||||
|
||||
def test_same_policy_error_with_changed_candidate_can_progress() -> None:
|
||||
report = FailureReport(
|
||||
id=18,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List generic records",
|
||||
response=None,
|
||||
sql_query="SELECT missing FROM records",
|
||||
error_message="Unknown column",
|
||||
context={},
|
||||
)
|
||||
worker = AutomationWorker(settings(
|
||||
mode="pr",
|
||||
dry_run=False,
|
||||
gitea_token="token",
|
||||
max_agent_attempts=3,
|
||||
))
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["first tests only", "improved tests only", "complete fix"]
|
||||
worker.gitea = Mock()
|
||||
worker._prepare_workspace_for_agent = Mock(return_value=[])
|
||||
worker._run_tests = Mock(return_value=[])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-18"
|
||||
workspace.summary.side_effect = [
|
||||
ChangeSummary(("tests/Unit/FixTest.php",), 2, 0),
|
||||
ChangeSummary(("tests/Unit/FixTest.php",), 3, 0),
|
||||
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 8, 1),
|
||||
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 8, 1),
|
||||
]
|
||||
workspace.added_text.return_value = ""
|
||||
workspace.diff.side_effect = ["first rejected candidate", "second rejected candidate"]
|
||||
workspace.run_args.return_value = Mock(
|
||||
returncode=0,
|
||||
stdout="No syntax errors",
|
||||
stderr="",
|
||||
)
|
||||
workspace.checkpoint.return_value = "valid"
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 3
|
||||
worker._run_tests.assert_called_once()
|
||||
workspace.push_branch.assert_called_once()
|
||||
worker.reports.complete.assert_called_once()
|
||||
Reference in New Issue
Block a user