diff --git a/src/lai_automation/policy.py b/src/lai_automation/policy.py index 9f107b7..9bfe3d4 100644 --- a/src/lai_automation/policy.py +++ b/src/lai_automation/policy.py @@ -44,6 +44,8 @@ def evaluate_changes( errors: list[str] = [] if not summary.files: errors.append("Agent produced no file changes") + elif all(filename.startswith("tests/") for filename in summary.files): + errors.append("Agent changed only tests; production behavior was not fixed") if len(summary.files) > settings.max_changed_files: errors.append(f"Changed {len(summary.files)} files; limit is {settings.max_changed_files}") if summary.changed_lines > settings.max_changed_lines: diff --git a/src/lai_automation/prompt.py b/src/lai_automation/prompt.py index 773f0f9..ed87f2e 100644 --- a/src/lai_automation/prompt.py +++ b/src/lai_automation/prompt.py @@ -91,6 +91,16 @@ For an unknown-column failure, compare every alias.column in the observed SQL wi schema and identify the owning table before editing. Teach the exact ownership, join, grouping, and real column names in semantics config, then add a reference SQL example for the reported question shape. Never replace one missing column with an unverified synonym. +""" + placeholder_guidance = '' + if '__LAI_' in response: + placeholder_guidance = """ +Privacy placeholder repair guidance: +Trace the actual answer path through AIService::generateDatabaseResponse, issued token +restoration, and ChatController's local fallback. A token can only be restored when it exists +in the exact token map returned by protect(); never test a made-up token against an unrelated +map. Cover model-invented or malformed __LAI_...__ output with an HTTP-backed AIService test +and verify the user receives a real local database-result fallback, not a raw placeholder. """ return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository. @@ -131,6 +141,7 @@ Generated SQL, if relevant: {sql or '[not available]'} {tenant_scope} {sql_semantics} + {placeholder_guidance} Run focused tests while working. Leave the repository with the fix and regression test applied. """ diff --git a/src/lai_automation/worker.py b/src/lai_automation/worker.py index 461322f..98a1d66 100644 --- a/src/lai_automation/worker.py +++ b/src/lai_automation/worker.py @@ -77,16 +77,21 @@ class AutomationWorker: ) policy = evaluate_changes(summary, self.settings, workspace.path) if not policy.allowed: - if not summary.files and attempt < self.settings.max_agent_attempts: + retryable = not summary.files or all( + filename.startswith("tests/") for filename in summary.files + ) + if retryable and attempt < self.settings.max_agent_attempts: LOG.warning( - "Agent produced no changes for report %s on attempt %s; requesting correction", + "Policy rejected retryable changes for report %s on attempt %s: %s", report.id, attempt, + "; ".join(policy.errors), ) task = build_retry_task( report, RuntimeError( - "No file changes were applied. Make the smallest concrete code and regression-test edits now." + "Policy rejected changes: " + "; ".join(policy.errors) + + ". Modify the owning production code and add a focused regression test." ) ) continue diff --git a/tests/test_policy.py b/tests/test_policy.py index b77f5e4..cbbdee0 100644 --- a/tests/test_policy.py +++ b/tests/test_policy.py @@ -23,6 +23,16 @@ def test_policy_rejects_secrets_and_vendor_paths() -> None: assert len(result.errors) == 3 +def test_policy_rejects_test_only_repair() -> None: + result = evaluate_changes( + ChangeSummary(("tests/Unit/ExampleTest.php",), 20, 2), + settings(), + ) + + assert not result.allowed + assert "production behavior was not fixed" in result.errors[0] + + def test_direct_main_blocks_validator_and_migrations() -> None: config = replace( settings(), diff --git a/tests/test_prompt.py b/tests/test_prompt.py index f40720a..3d075e4 100644 --- a/tests/test_prompt.py +++ b/tests/test_prompt.py @@ -152,3 +152,27 @@ def test_customer_database_task_explains_tenant_scope() -> None: assert "must not" in task assert "guessed column" in task assert "tenant_type" in task + + +def test_unresolved_placeholder_task_guides_real_answer_path() -> None: + report = FailureReport( + id=49, + chat_message_id=1, + source="user_feedback", + category="incorrect_answer", + user_feedback="Token was not restored", + feedback_type="wrong_result", + expected_behavior="Return the database value", + prompt="Average price", + response="Price: __LAI_UNKNOWN_VALUE__", + sql_query="SELECT AVG(price)", + error_message=None, + context={"database_type": "erp"}, + ) + + task = build_task(report) + + assert "AIService::generateDatabaseResponse" in task + assert "exact token map" in task + assert "HTTP-backed AIService test" in task + assert "local database-result fallback" in task diff --git a/tests/test_worker.py b/tests/test_worker.py index a19c549..182fa15 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -131,7 +131,9 @@ def test_failed_validation_gets_one_correction_before_pr() -> None: workspace = Mock() workspace.path = Path("/tmp/nonexistent-workspace") workspace.branch = "ai/failure-13" - workspace.summary.return_value = ChangeSummary(("tests/Feature/FixTest.php",), 5, 0) + workspace.summary.return_value = ChangeSummary( + ("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1 + ) workspace.commit.return_value = "commit-id" with patch("lai_automation.worker.GitWorkspace", return_value=workspace): @@ -172,8 +174,8 @@ def test_no_change_response_gets_one_correction() -> None: workspace.branch = "ai/failure-14" workspace.summary.side_effect = [ ChangeSummary((), 0, 0), - ChangeSummary(("tests/Feature/FixTest.php",), 5, 0), - ChangeSummary(("tests/Feature/FixTest.php",), 5, 0), + ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1), + ChangeSummary(("app/Services/Fix.php", "tests/Feature/FixTest.php"), 10, 1), ] workspace.commit.return_value = "commit-id" @@ -181,6 +183,47 @@ def test_no_change_response_gets_one_correction() -> None: assert worker.run_once() assert worker.agent.run.call_count == 2 - assert "No file changes" in worker.agent.run.call_args_list[1].args[1] + assert "Agent produced no file changes" in worker.agent.run.call_args_list[1].args[1] worker._run_tests.assert_called_once() workspace.push_branch.assert_called_once() + + +def test_test_only_change_gets_correction_before_validation() -> None: + report = FailureReport( + id=49, + chat_message_id=1, + source="user_feedback", + category="incorrect_answer", + user_feedback="Token was not restored", + feedback_type="wrong_result", + expected_behavior="Return the real value", + prompt="Average price", + response="__LAI_UNKNOWN__", + sql_query="SELECT AVG(price)", + error_message=None, + context={}, + ) + config = settings(mode="pr", dry_run=False, gitea_token="token") + worker = AutomationWorker(config) + worker.reports = Mock() + worker.reports.claim_next.return_value = report + worker.agent = Mock() + worker.agent.run.side_effect = ["tests only", "production fix"] + worker.gitea = Mock() + worker._run_tests = Mock(return_value=[]) + workspace = Mock() + workspace.path = Path("/tmp/nonexistent-workspace") + workspace.branch = "ai/failure-49" + workspace.summary.side_effect = [ + ChangeSummary(("tests/Unit/FixTest.php",), 8, 0), + ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 15, 1), + ChangeSummary(("app/Services/Fix.php", "tests/Unit/FixTest.php"), 15, 1), + ] + workspace.commit.return_value = "commit-id" + + with patch("lai_automation.worker.GitWorkspace", return_value=workspace): + assert worker.run_once() + + assert worker.agent.run.call_count == 2 + assert "changed only tests" in worker.agent.run.call_args_list[1].args[1] + worker._run_tests.assert_called_once()