Improve agent repair retries
Some checks failed
CI / test (push) Has been cancelled

This commit is contained in:
2026-08-27 23:58:53 +02:00
parent 67ec2131c5
commit 6463394e7a
6 changed files with 57 additions and 5 deletions

View File

@@ -84,7 +84,7 @@ class AiderAgent:
"/dev/null", "/dev/null",
"--no-analytics", "--no-analytics",
"--edit-format", "--edit-format",
"diff", "udiff",
"--timeout", "--timeout",
"180", "180",
] ]

View File

@@ -42,7 +42,8 @@ Repair workflow:
7. Do not duplicate one rule in service code and semantics config, or add explanatory comments for 7. Do not duplicate one rule in service code and semantics config, or add explanatory comments for
self-evident test setup. self-evident test setup.
8. Inspect the final diff for malformed edits, unrelated files, hardcoded report values, and tests 8. Inspect the final diff for malformed edits, unrelated files, hardcoded report values, and tests
that pass without exercising the changed behavior. that pass without exercising the changed behavior. Before retrying an edit, reread the current
file and never leave conflict markers or partial methods behind.
""" """
@@ -85,6 +86,11 @@ context builder through a test subclass and assert that its real output contains
formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt
guidance to rewrite that mocked response. Keep database fixture tests only when production code guidance to rewrite that mocked response. Keep database fixture tests only when production code
itself deterministically builds and executes the SQL under test. itself deterministically builds and executes the SQL under test.
For an unknown-column failure, compare every alias.column in the observed SQL with the table
schema and identify the owning table before editing. Teach the exact ownership, join, grouping,
and real column names in semantics config, then add a reference SQL example for the reported
question shape. Never replace one missing column with an unverified synonym.
""" """
return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository. return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository.
@@ -131,7 +137,7 @@ Run focused tests while working. Leave the repository with the fix and regressio
def build_retry_task(report: FailureReport, exception: Exception) -> str: def build_retry_task(report: FailureReport, exception: Exception) -> str:
failure = redact(str(exception))[-6_000:] failure = redact(str(exception))
return build_task(report) + f""" return build_task(report) + f"""

View File

@@ -221,7 +221,9 @@ class AutomationWorker:
result.returncode, result.returncode,
) )
if result.returncode != 0: if result.returncode != 0:
output = (result.stdout + "\n" + result.stderr)[-4000:] output = result.stdout + "\n" + result.stderr
if len(output) > 12_000:
output = output[:6_000] + "\n...[middle omitted]...\n" + output[-6_000:]
raise RuntimeError(f"Test command failed: {command}\n{output}") raise RuntimeError(f"Test command failed: {command}\n{output}")
return results return results

View File

@@ -12,7 +12,7 @@ def test_aider_command_cannot_mutate_git_configuration_or_send_analytics() -> No
assert "--no-analytics" in command assert "--no-analytics" in command
assert "--no-auto-commits" in command assert "--no-auto-commits" in command
assert command.count("/dev/null") == 2 assert command.count("/dev/null") == 2
assert command[-4:] == ["--edit-format", "diff", "--timeout", "180"] assert command[-4:] == ["--edit-format", "udiff", "--timeout", "180"]
def test_aider_environment_does_not_include_database_credentials(tmp_path, monkeypatch) -> None: def test_aider_environment_does_not_include_database_credentials(tmp_path, monkeypatch) -> None:

View File

@@ -105,6 +105,29 @@ def test_incorrect_answer_task_guides_semantics_level_repair() -> None:
assert "protected schema/business-" in task assert "protected schema/business-" in task
assert "context builder" in task assert "context builder" in task
assert "known-bad SQL" in task assert "known-bad SQL" in task
assert "compare every alias.column" in task
assert "Never replace one missing column" in task
def test_retry_task_keeps_failure_from_start_of_long_output() -> None:
report = FailureReport(
id=43,
chat_message_id=1,
source="automatic",
category="sql_execution",
user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="Top products",
response=None,
sql_query="SELECT a.missing FROM orders a",
error_message="Unknown column",
context={"database_type": "erp"},
)
task = build_retry_task(report, RuntimeError("DECISIVE ASSERTION\n" + "x" * 11_000))
assert "DECISIVE ASSERTION" in task
def test_customer_database_task_explains_tenant_scope() -> None: def test_customer_database_task_explains_tenant_scope() -> None:

View File

@@ -74,6 +74,27 @@ def test_validation_commands_are_logged_without_output(caplog) -> None:
assert "sensitive output" not in caplog.text assert "sensitive output" not in caplog.text
def test_failed_validation_preserves_start_and_end_of_long_output() -> None:
worker = AutomationWorker(settings())
workspace = Mock()
workspace.run.return_value = Mock(
returncode=1,
stdout="DECISIVE ASSERTION\n" + "x" * 13_000,
stderr="FINAL ERROR",
)
try:
worker._run_tests(workspace)
except RuntimeError as exception:
failure = str(exception)
else:
raise AssertionError("Expected failed validation")
assert "DECISIVE ASSERTION" in failure
assert "...[middle omitted]..." in failure
assert "FINAL ERROR" in failure
def test_long_stage_emits_heartbeat(caplog) -> None: def test_long_stage_emits_heartbeat(caplog) -> None:
worker = AutomationWorker(settings()) worker = AutomationWorker(settings())