christian.meyer/lai_automatisierung
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replace Aider with secure mini-SWE-agent worker
Some checks failed
CI / test (push) Has been cancelled
Details
CI / test (pull_request) Has been cancelled
Details

Browse Source
This commit is contained in:
Christian Meyer
2026-08-31 02:46:58 +02:00
parent 582db3a654
commit 25ed6a8439
30 changed files with 4215 additions and 2148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

518
tests/test_worker.py
View File

@@ -1,351 +1,195 @@
import threading
from pathlib import Path
from unittest.mock import MagicMock, Mock, patch
from __future__ import annotations
from test_config import settings
from unittest.mock import Mock, call, patch
from lai_automation.git import ChangeSummary
from lai_automation.models import FailureReport
from lai_automation.worker import AutomationWorker
from lai_automation.models import (
AgentResult,
ChangeSummary,
Issue,
Label,
PullRequest,
)
from lai_automation.worker import Worker
def test_dry_run_generates_task_without_agent_or_git() -> None:
report = FailureReport(
id=11,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List locations",
response=None,
sql_query=None,
error_message="Unknown column",
context={},
def queued_issue() -> Issue:
return Issue(
number=7,
title="Wrong calculation",
body="The original report",
html_url="https://gitea.example.test/owner/lai/issues/7",
labels=(Label(1, "ai:queued"), Label(9, "bug")),
)
worker = AutomationWorker(settings())
worker.reports = Mock()
worker.reports.claim_next.return_value = report
assert worker.run_once()
worker.reports.release.assert_called_once()
worker.agent = Mock()
worker.agent.run.assert_not_called()
def test_stop_signal_interrupts_worker_wait() -> None:
worker = AutomationWorker(settings())
worker._stop(15, None)
assert worker.stop_event.is_set()
def labels() -> dict[str, Label]:
return {
"ai:queued": Label(1, "ai:queued"),
"ai:running": Label(2, "ai:running"),
"ai:ready": Label(3, "ai:ready"),
"ai:failed": Label(4, "ai:failed"),
}
def test_database_credentials_are_only_passed_to_test_process() -> None:
worker = AutomationWorker(settings())
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.run.return_value = Mock(returncode=0, stdout="ok", stderr="")
results = worker._run_tests(workspace)
assert results[0]["returncode"] == 0
environment = workspace.run.call_args.kwargs["extra_environment"]
assert environment["AGENT_TEST_DB_ENABLED"] == "true"
assert environment["APP_ENV"] == "testing"
assert environment["AI_MISTRAL_API_KEY"] == "automation-test-key"
assert environment["TEST_DB_USER"] == "test_runner"
assert environment["TEST_DB_PASSWORD"] == "test-secret"
assert environment["DB_DATABASE"] == "lai_agent_test"
assert environment["ERP_DATABASE"] == "customer_agent_test"
assert environment["COCKPIT_CONFIG_DATABASE"] == "cockpit_agent_test"
def test_test_schema_creates_minimal_erp_lookup(monkeypatch) -> None:
worker = AutomationWorker(settings())
cursor = Mock()
connection = MagicMock()
connection.cursor.return_value.__enter__.return_value = cursor
connect = Mock(return_value=connection)
monkeypatch.setattr("lai_automation.worker.pymysql.connect", connect)
worker._prepare_test_schema()
assert connect.call_args.kwargs["database"] == "customer_agent_test"
assert "CREATE TABLE IF NOT EXISTS FK_Nummerkreis" in cursor.execute.call_args.args[0]
assert cursor.executemany.call_args.args[1][0] == (3, "Rechnung", "RG")
connection.close.assert_called_once()
def test_validation_commands_are_logged_without_output(caplog) -> None:
worker = AutomationWorker(settings())
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.run.return_value = Mock(returncode=0, stdout="sensitive output", stderr="")
with caplog.at_level("INFO"):
worker._run_tests(workspace, report_id=7, attempt=2)
assert "Report 7 attempt 2 validation command: php artisan test starting" in caplog.text
assert "returncode=0" in caplog.text
assert "sensitive output" not in caplog.text
def test_failed_validation_preserves_start_and_end_of_long_output() -> None:
worker = AutomationWorker(settings())
worker._prepare_test_schema = Mock()
workspace = Mock()
workspace.run.return_value = Mock(
returncode=1,
stdout="DECISIVE ASSERTION\n" + "x" * 13_000,
stderr="FINAL ERROR",
@patch("lai_automation.worker.GitWorkspace")
def test_successful_issue_creates_pr_and_ready_label(
GitWorkspace,
settings,
) -> None:
issue = queued_issue()
running = Issue(
**{
**issue.__dict__,
"labels": (Label(2, "ai:running"), Label(9, "bug")),
}
)
gitea = Mock()
gitea.get_issue.return_value = issue
gitea.set_automation_state.side_effect = [running, running]
gitea.create_pull_request.return_value = PullRequest(
18,
"https://gitea.example.test/owner/lai/pulls/18",
)
agent = Mock()
agent.run.return_value = AgentResult(
"Submitted",
"Implemented and tested",
4,
0.2,
)
workspace = GitWorkspace.return_value
workspace.path = settings.workspace_root / "issue-7" / "repo"
workspace.branch = "ai/issue-7"
workspace.summary.return_value = ChangeSummary(
("app/Services/Fix.php", "tests/Feature/FixTest.php"),
20,
2,
)
workspace.has_symlink.return_value = False
workspace.has_path.return_value = False
workspace.commit.return_value = "abc123"
Worker(settings, gitea, agent).process(issue, labels())
workspace.prepare.assert_called_once()
workspace.run_trusted.assert_has_calls(
[
call(
"composer install",
extra_environment=settings.test_environment,
),
call(
"php artisan test",
extra_environment=settings.test_environment,
),
]
)
workspace.push.assert_called_once()
assert workspace.check_diff.call_count == 2
assert gitea.create_pull_request.call_args.args[0] == "ai/issue-7"
assert "Fixes #7" in gitea.create_pull_request.call_args.args[2]
assert "The original report" in gitea.create_pull_request.call_args.args[2]
assert gitea.set_automation_state.call_args_list[-1].args[1] == "ai:ready"
workspace.cleanup.assert_called_once()
@patch("lai_automation.worker.GitWorkspace")
def test_failed_tests_prevent_push_and_mark_issue_failed(
GitWorkspace,
settings,
) -> None:
issue = queued_issue()
running = Issue(
**{
**issue.__dict__,
"labels": (Label(2, "ai:running"),),
}
)
gitea = Mock()
gitea.get_issue.side_effect = [issue, running]
gitea.set_automation_state.return_value = running
agent = Mock()
agent.run.return_value = AgentResult("Submitted", "Done", 2, 0.1)
workspace = GitWorkspace.return_value
workspace.path = settings.workspace_root / "issue-7" / "repo"
workspace.summary.return_value = ChangeSummary(("app/Fix.php",), 2, 0)
workspace.has_symlink.return_value = False
workspace.has_path.return_value = False
workspace.run_trusted.side_effect = [
Mock(),
RuntimeError("tests failed with token=hidden"),
]
Worker(settings, gitea, agent).process(issue, labels())
workspace.push.assert_not_called()
gitea.create_pull_request.assert_not_called()
assert gitea.set_automation_state.call_args_list[-1].args[1] == "ai:failed"
failure_comment = gitea.comment.call_args_list[-1].args[1]
assert "tests failed" in failure_comment
assert "hidden" not in failure_comment
def test_run_once_only_processes_first_queued_issue(settings) -> None:
gitea = Mock()
issue = queued_issue()
second_issue = Issue(**{**issue.__dict__, "number": 8})
expected_labels = labels()
gitea.ensure_automation_labels.return_value = expected_labels
gitea.list_queued_issues.return_value = [issue, second_issue]
worker = Worker(settings, gitea, Mock())
worker.process = Mock()
assert worker.run_once() is True
worker.process.assert_called_once_with(issue, expected_labels)
def test_run_once_returns_false_without_work(settings) -> None:
gitea = Mock()
gitea.ensure_automation_labels.return_value = labels()
gitea.list_queued_issues.return_value = []
assert Worker(settings, gitea, Mock()).run_once() is False
@patch("lai_automation.worker.GitWorkspace")
def test_preflight_runs_setup_and_tests_without_model(
GitWorkspace,
settings,
) -> None:
workspace = GitWorkspace.return_value
agent = Mock()
Worker(settings, Mock(), agent).preflight()
GitWorkspace.assert_called_once_with(settings, 0)
workspace.prepare.assert_called_once()
workspace.run_trusted.assert_has_calls(
[
call(
"composer install",
extra_environment=settings.test_environment,
),
call(
"php artisan test",
extra_environment=settings.test_environment,
),
]
)
agent.run.assert_not_called()
workspace.cleanup.assert_called_once()
@patch("lai_automation.worker.GitWorkspace")
def test_preflight_cleans_up_after_failure(GitWorkspace, settings) -> None:
workspace = GitWorkspace.return_value
workspace.run_trusted.side_effect = RuntimeError("test failure")
try:
worker._run_tests(workspace)
except RuntimeError as exception:
failure = str(exception)
Worker(settings, Mock(), Mock()).preflight()
except RuntimeError:
pass
else:
raise AssertionError("Expected failed validation")
raise AssertionError("Expected preflight failure")
assert "DECISIVE ASSERTION" in failure
assert "...[middle omitted]..." in failure
assert "FINAL ERROR" in failure
def test_long_stage_emits_heartbeat(caplog) -> None:
worker = AutomationWorker(settings())
with caplog.at_level("INFO"), worker._progress(7, "agent attempt 1", interval=0.001):
threading.Event().wait(0.01)
assert "Report 7 agent attempt 1 still running" in caplog.text
assert "Report 7 agent attempt 1 finished" in caplog.text
def test_php_syntax_failure_discards_attempt_before_full_validation() -> None:
report = FailureReport(
id=56,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong hours",
feedback_type="wrong_result",
expected_behavior="Use time registration",
prompt="Hours in April",
response="0 hours",
sql_query="SELECT 0",
error_message=None,
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["broken PHP", "fixed PHP"]
worker.gitea = Mock()
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-56"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
)
workspace.run_args.side_effect = [
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
]
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+intended fix"
workspace.checkpoint.return_value = "valid-checkpoint"
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "unexpected token" in worker.agent.run.call_args_list[1].args[1]
assert "clean repository" in worker.agent.run.call_args_list[1].args[1]
assert "intended fix" in worker.agent.run.call_args_list[1].args[1]
assert "repair-only attempt" in worker.agent.run.call_args_list[1].args[1]
workspace.restore_checkpoint.assert_called_once_with(None)
worker._run_tests.assert_called_once()
def test_malformed_retry_restores_last_syntax_valid_attempt() -> None:
report = FailureReport(
id=56,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Wrong hours",
feedback_type="wrong_result",
expected_behavior="Use time registration",
prompt="Hours in April",
response="0 hours",
sql_query="SELECT 0",
error_message=None,
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token", max_agent_attempts=3)
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["valid but failing", "broken retry", "corrected retry"]
worker.gitea = Mock()
worker._run_tests = Mock(side_effect=[RuntimeError("Expected hour rule"), []])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-56"
workspace.summary.return_value = ChangeSummary(
("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2
)
workspace.run_args.side_effect = [
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=255, stdout="Parse error", stderr="unexpected token"),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
Mock(returncode=0, stdout="No syntax errors", stderr=""),
]
workspace.diff.return_value = "--- app/Service.php\n+++ app/Service.php\n+application repair"
workspace.checkpoint.side_effect = ["checkpoint-1", "checkpoint-3"]
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 3
workspace.restore_checkpoint.assert_called_once_with("checkpoint-1")
assert "last syntax-valid changes were restored" in worker.agent.run.call_args_list[2].args[1]
assert "application repair" in worker.agent.run.call_args_list[2].args[1]
assert worker._run_tests.call_count == 2
def test_failed_validation_gets_one_correction_before_pr() -> None:
report = FailureReport(
id=13,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List products",
response=None,
sql_query="SELECT missing_column FROM invoice_row",
error_message="Unknown column",
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["first change", "corrected change"]
worker.gitea = Mock()
worker._run_tests = Mock(side_effect=[RuntimeError("Missing import"), []])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-13"
workspace.summary.return_value = ChangeSummary(
("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1
)
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "Missing import" in worker.agent.run.call_args_list[1].args[1]
assert worker._run_tests.call_count == 2
workspace.push_branch.assert_called_once()
worker.reports.complete.assert_called_once()
def test_no_change_response_gets_one_correction() -> None:
report = FailureReport(
id=14,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List products",
response=None,
sql_query=None,
error_message="Unknown column",
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["no edit", "applied edit"]
worker.gitea = Mock()
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-14"
workspace.summary.side_effect = [
ChangeSummary((), 0, 0),
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1),
]
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "Agent produced no file changes" in worker.agent.run.call_args_list[1].args[1]
worker._run_tests.assert_called_once()
workspace.push_branch.assert_called_once()
def test_test_only_change_gets_correction_before_validation() -> None:
report = FailureReport(
id=49,
chat_message_id=1,
source="user_feedback",
category="incorrect_answer",
user_feedback="Token was not restored",
feedback_type="wrong_result",
expected_behavior="Return the real value",
prompt="Average price",
response="__LAI_UNKNOWN__",
sql_query="SELECT AVG(price)",
error_message=None,
context={},
)
config = settings(mode="pr", dry_run=False, gitea_token="token")
worker = AutomationWorker(config)
worker.reports = Mock()
worker.reports.claim_next.return_value = report
worker.agent = Mock()
worker.agent.run.side_effect = ["tests only", "production fix"]
worker.gitea = Mock()
worker._run_tests = Mock(return_value=[])
workspace = Mock()
workspace.path = Path("/tmp/nonexistent-workspace")
workspace.branch = "ai/failure-49"
workspace.summary.side_effect = [
ChangeSummary(("tests/Unit/FixTest.php",), 8, 0),
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 15, 1),
ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 15, 1),
]
workspace.run_args.return_value = Mock(returncode=0, stdout="No syntax errors", stderr="")
workspace.commit.return_value = "commit-id"
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
assert worker.run_once()
assert worker.agent.run.call_count == 2
assert "changed only tests" in worker.agent.run.call_args_list[1].args[1]
worker._run_tests.assert_called_once()
workspace.cleanup.assert_called_once()