Reject test-only agent repairs

src/lai_automation/policy.py

@@ -44,6 +44,8 @@ def evaluate_changes(
     errors: list[str] = []
     if not summary.files:
         errors.append("Agent produced no file changes")
+    elif all(filename.startswith("tests/") for filename in summary.files):
+        errors.append("Agent changed only tests; production behavior was not fixed")
     if len(summary.files) > settings.max_changed_files:
         errors.append(f"Changed {len(summary.files)} files; limit is {settings.max_changed_files}")
     if summary.changed_lines > settings.max_changed_lines:

src/lai_automation/prompt.py

@@ -91,6 +91,16 @@ For an unknown-column failure, compare every alias.column in the observed SQL wi
 schema and identify the owning table before editing. Teach the exact ownership, join, grouping,
 and real column names in semantics config, then add a reference SQL example for the reported
 question shape. Never replace one missing column with an unverified synonym.
+"""
+    placeholder_guidance = ''
+    if '__LAI_' in response:
+        placeholder_guidance = """
+Privacy placeholder repair guidance:
+Trace the actual answer path through AIService::generateDatabaseResponse, issued token
+restoration, and ChatController's local fallback. A token can only be restored when it exists
+in the exact token map returned by protect(); never test a made-up token against an unrelated
+map. Cover model-invented or malformed __LAI_...__ output with an HTTP-backed AIService test
+and verify the user receives a real local database-result fallback, not a raw placeholder.
 """
 
     return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository.
@@ -131,6 +141,7 @@ Generated SQL, if relevant:
 {sql or '[not available]'}
 {tenant_scope}
 {sql_semantics}
+    {placeholder_guidance}
 
 Run focused tests while working. Leave the repository with the fix and regression test applied.
 """

src/lai_automation/worker.py

@@ -77,16 +77,21 @@ class AutomationWorker:
                 )
                 policy = evaluate_changes(summary, self.settings, workspace.path)
                 if not policy.allowed:
-                    if not summary.files and attempt < self.settings.max_agent_attempts:
+                    retryable = not summary.files or all(
+                        filename.startswith("tests/") for filename in summary.files
+                    )
+                    if retryable and attempt < self.settings.max_agent_attempts:
                         LOG.warning(
-                            "Agent produced no changes for report %s on attempt %s; requesting correction",
+                            "Policy rejected retryable changes for report %s on attempt %s: %s",
                             report.id,
                             attempt,
+                            "; ".join(policy.errors),
                         )
                         task = build_retry_task(
                             report,
                             RuntimeError(
-                                "No file changes were applied. Make the smallest concrete code and regression-test edits now."
+                                "Policy rejected changes: " + "; ".join(policy.errors)
+                                + ". Modify the owning production code and add a focused regression test."
                             )
                         )
                         continue