From 6463394e7abb72070a767e7388cc8dc1846ccbc7 Mon Sep 17 00:00:00 2001 From: Christian Meyer Date: Thu, 27 Aug 2026 23:58:53 +0200 Subject: [PATCH] Improve agent repair retries --- src/lai_automation/agent.py | 2 +- src/lai_automation/prompt.py | 10 ++++++++-- src/lai_automation/worker.py | 4 +++- tests/test_agent.py | 2 +- tests/test_prompt.py | 23 +++++++++++++++++++++++ tests/test_worker.py | 21 +++++++++++++++++++++ 6 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/lai_automation/agent.py b/src/lai_automation/agent.py index 38fa79c..352b461 100644 --- a/src/lai_automation/agent.py +++ b/src/lai_automation/agent.py @@ -84,7 +84,7 @@ class AiderAgent: "/dev/null", "--no-analytics", "--edit-format", - "diff", + "udiff", "--timeout", "180", ] diff --git a/src/lai_automation/prompt.py b/src/lai_automation/prompt.py index 79ea697..773f0f9 100644 --- a/src/lai_automation/prompt.py +++ b/src/lai_automation/prompt.py @@ -42,7 +42,8 @@ Repair workflow: 7. Do not duplicate one rule in service code and semantics config, or add explanatory comments for self-evident test setup. 8. Inspect the final diff for malformed edits, unrelated files, hardcoded report values, and tests - that pass without exercising the changed behavior. + that pass without exercising the changed behavior. Before retrying an edit, reread the current + file and never leave conflict markers or partial methods behind. """ @@ -85,6 +86,11 @@ context builder through a test subclass and assert that its real output contains formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt guidance to rewrite that mocked response. Keep database fixture tests only when production code itself deterministically builds and executes the SQL under test. + +For an unknown-column failure, compare every alias.column in the observed SQL with the table +schema and identify the owning table before editing. Teach the exact ownership, join, grouping, +and real column names in semantics config, then add a reference SQL example for the reported +question shape. Never replace one missing column with an unverified synonym. """ return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository. @@ -131,7 +137,7 @@ Run focused tests while working. Leave the repository with the fix and regressio def build_retry_task(report: FailureReport, exception: Exception) -> str: - failure = redact(str(exception))[-6_000:] + failure = redact(str(exception)) return build_task(report) + f""" diff --git a/src/lai_automation/worker.py b/src/lai_automation/worker.py index 16697c7..461322f 100644 --- a/src/lai_automation/worker.py +++ b/src/lai_automation/worker.py @@ -221,7 +221,9 @@ class AutomationWorker: result.returncode, ) if result.returncode != 0: - output = (result.stdout + "\n" + result.stderr)[-4000:] + output = result.stdout + "\n" + result.stderr + if len(output) > 12_000: + output = output[:6_000] + "\n...[middle omitted]...\n" + output[-6_000:] raise RuntimeError(f"Test command failed: {command}\n{output}") return results diff --git a/tests/test_agent.py b/tests/test_agent.py index 8802b16..562be51 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -12,7 +12,7 @@ def test_aider_command_cannot_mutate_git_configuration_or_send_analytics() -> No assert "--no-analytics" in command assert "--no-auto-commits" in command assert command.count("/dev/null") == 2 - assert command[-4:] == ["--edit-format", "diff", "--timeout", "180"] + assert command[-4:] == ["--edit-format", "udiff", "--timeout", "180"] def test_aider_environment_does_not_include_database_credentials(tmp_path, monkeypatch) -> None: diff --git a/tests/test_prompt.py b/tests/test_prompt.py index 4904fbe..f40720a 100644 --- a/tests/test_prompt.py +++ b/tests/test_prompt.py @@ -105,6 +105,29 @@ def test_incorrect_answer_task_guides_semantics_level_repair() -> None: assert "protected schema/business-" in task assert "context builder" in task assert "known-bad SQL" in task + assert "compare every alias.column" in task + assert "Never replace one missing column" in task + + +def test_retry_task_keeps_failure_from_start_of_long_output() -> None: + report = FailureReport( + id=43, + chat_message_id=1, + source="automatic", + category="sql_execution", + user_feedback=None, + feedback_type=None, + expected_behavior=None, + prompt="Top products", + response=None, + sql_query="SELECT a.missing FROM orders a", + error_message="Unknown column", + context={"database_type": "erp"}, + ) + + task = build_retry_task(report, RuntimeError("DECISIVE ASSERTION\n" + "x" * 11_000)) + + assert "DECISIVE ASSERTION" in task def test_customer_database_task_explains_tenant_scope() -> None: diff --git a/tests/test_worker.py b/tests/test_worker.py index 9f16852..a19c549 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -74,6 +74,27 @@ def test_validation_commands_are_logged_without_output(caplog) -> None: assert "sensitive output" not in caplog.text +def test_failed_validation_preserves_start_and_end_of_long_output() -> None: + worker = AutomationWorker(settings()) + workspace = Mock() + workspace.run.return_value = Mock( + returncode=1, + stdout="DECISIVE ASSERTION\n" + "x" * 13_000, + stderr="FINAL ERROR", + ) + + try: + worker._run_tests(workspace) + except RuntimeError as exception: + failure = str(exception) + else: + raise AssertionError("Expected failed validation") + + assert "DECISIVE ASSERTION" in failure + assert "...[middle omitted]..." in failure + assert "FINAL ERROR" in failure + + def test_long_stage_emits_heartbeat(caplog) -> None: worker = AutomationWorker(settings())