diff --git a/.env.example b/.env.example index 8f657e6..08de28b 100644 --- a/.env.example +++ b/.env.example @@ -21,6 +21,7 @@ AGENT_COMMAND_TIMEOUT_SECONDS=180 AGENT_TIMEOUT_SECONDS=1200 AGENT_STEP_LIMIT=50 AGENT_COST_LIMIT=5 +MAX_AGENT_ATTEMPTS=3 SETUP_COMMANDS=composer install --no-interaction --prefer-dist;;npm ci TEST_COMMANDS=npm run build;;php artisan test diff --git a/README.md b/README.md index 86b9304..d96cde1 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,11 @@ Set `LOG_LEVEL=DEBUG` to include redacted output from setup and test commands. A log their model, configured limits, call count, cost, exit status, and trajectory path. `INFO` is the default for normal operation. +`MAX_AGENT_ATTEMPTS` controls validation-driven self-repair. When an agent submission fails policy, +diff, or test validation, the worker preserves the workspace and gives the redacted failure output +to a fresh agent attempt. Only the final failed attempt marks the issue as `ai:failed` and removes +the workspace. + ## Archived implementation The previous database-polling Aider implementation is preserved unchanged in `old_2026-08-31/`. diff --git a/docker-compose.yml b/docker-compose.yml index 091b919..db7288b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,6 +20,7 @@ services: AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-1200} AGENT_STEP_LIMIT: ${AGENT_STEP_LIMIT:-50} AGENT_COST_LIMIT: ${AGENT_COST_LIMIT:-5} + MAX_AGENT_ATTEMPTS: ${MAX_AGENT_ATTEMPTS:-3} LOG_LEVEL: ${LOG_LEVEL:-INFO} AGENT_USER: coder MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} diff --git a/src/lai_automation/agent.py b/src/lai_automation/agent.py index 697530d..00793d8 100644 --- a/src/lai_automation/agent.py +++ b/src/lai_automation/agent.py @@ -82,6 +82,7 @@ class MiniSWEAgentRunner: issue: Issue, workspace: Path, trajectory_path: Path, + validation_feedback: str = "", ) -> AgentResult: logger.info( "Starting agent for issue #%s: model=%s step_limit=%s cost_limit=%.2f " @@ -130,7 +131,7 @@ class MiniSWEAgentRunner: max_consecutive_format_errors=3, output_path=trajectory_path, ) - info = agent.run(self._task(issue)) + info = agent.run(self._task(issue, validation_feedback)) result = AgentResult( exit_status=str(info.get("exit_status") or "unknown"), submission=str(info.get("submission") or ""), @@ -149,12 +150,22 @@ class MiniSWEAgentRunner: return result @staticmethod - def _task(issue: Issue) -> str: - return "\n".join( + def _task(issue: Issue, validation_feedback: str = "") -> str: + parts = [ + f"Issue #{issue.number}: {issue.title}", + f"URL: {issue.html_url}", + "", + issue.body[:20_000], + ] + if validation_feedback: + parts.extend( [ - f"Issue #{issue.number}: {issue.title}", - f"URL: {issue.html_url}", "", - issue.body[:20_000], + "The previous attempt failed deterministic validation. Keep the existing", + "workspace changes, diagnose and repair this exact failure, then rerun the", + "smallest relevant check and submit again:", + "", + validation_feedback[:8_000], ] ) + return "\n".join(parts) diff --git a/src/lai_automation/config.py b/src/lai_automation/config.py index 0780f9b..415253f 100644 --- a/src/lai_automation/config.py +++ b/src/lai_automation/config.py @@ -33,6 +33,7 @@ class Settings: agent_wall_time_seconds: int agent_step_limit: int agent_cost_limit: float + max_agent_attempts: int agent_user: str setup_commands: tuple[str, ...] test_commands: tuple[str, ...] @@ -75,6 +76,7 @@ class Settings: agent_wall_time_seconds=int(os.getenv("AGENT_TIMEOUT_SECONDS", "1200")), agent_step_limit=int(os.getenv("AGENT_STEP_LIMIT", "50")), agent_cost_limit=float(os.getenv("AGENT_COST_LIMIT", "5")), + max_agent_attempts=int(os.getenv("MAX_AGENT_ATTEMPTS", "3")), agent_user=os.getenv("AGENT_USER", "coder"), setup_commands=_commands( os.getenv( @@ -156,6 +158,8 @@ class Settings: raise ValueError("AGENT_STEP_LIMIT must be between 1 and 200") if self.agent_cost_limit < 0: raise ValueError("AGENT_COST_LIMIT cannot be negative") + if not 1 <= self.max_agent_attempts <= 10: + raise ValueError("MAX_AGENT_ATTEMPTS must be between 1 and 10") if not 1 <= self.max_changed_files <= 100: raise ValueError("MAX_CHANGED_FILES must be between 1 and 100") if not 1 <= self.max_changed_lines <= 10_000: diff --git a/src/lai_automation/worker.py b/src/lai_automation/worker.py index ece5ecb..360525e 100644 --- a/src/lai_automation/worker.py +++ b/src/lai_automation/worker.py @@ -87,41 +87,54 @@ class Worker: extra_environment=self.settings.test_environment, ) - timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ") - trajectory = ( - self.settings.workspace_root - / "trajectories" - / f"issue-{issue.number}-{timestamp}.traj.json" - ) - started_at = time.monotonic() - result = self.agent.run(current_issue, workspace.path, trajectory) - elapsed = time.monotonic() - started_at - if result.exit_status != "Submitted": - details = self._agent_failure_details(result, elapsed, trajectory) - raise RuntimeError( - f"Agent stopped with status {result.exit_status}: {details}" + validation_feedback = "" + result = None + summary = None + for attempt in range(1, self.settings.max_agent_attempts + 1): + timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ") + trajectory = ( + self.settings.workspace_root + / "trajectories" + / f"issue-{issue.number}-attempt-{attempt}-{timestamp}.traj.json" ) - - summary = workspace.summary() - normalized = workspace.normalize_changed_whitespace(summary) - if normalized: logger.info( - "Normalized trailing whitespace in %s changed file(s) for issue #%s", - len(normalized), + "Starting attempt %s/%s for issue #%s", + attempt, + self.settings.max_agent_attempts, issue.number, ) - summary = workspace.summary() - validate_changes(workspace, summary, self.settings) - workspace.check_diff(summary) - for command in self.settings.test_commands: - workspace.run_trusted( - command, - extra_environment=self.settings.test_environment, + started_at = time.monotonic() + result = self.agent.run( + current_issue, + workspace.path, + trajectory, + validation_feedback, ) + elapsed = time.monotonic() - started_at + try: + if result.exit_status != "Submitted": + details = self._agent_failure_details(result, elapsed, trajectory) + raise RuntimeError( + f"Agent stopped with status {result.exit_status}: {details}" + ) + summary = self._validate_agent_changes(workspace, issue.number) + break + except Exception as validation_error: + if attempt >= self.settings.max_agent_attempts: + raise + validation_feedback = redact(str(validation_error), 8_000) + logger.warning( + "Attempt %s/%s for issue #%s failed validation; preserving " + "workspace for repair attempt %s: %s", + attempt, + self.settings.max_agent_attempts, + issue.number, + attempt + 1, + validation_feedback, + ) - summary = workspace.summary() - validate_changes(workspace, summary, self.settings) - workspace.check_diff(summary) + if result is None or summary is None: + raise RuntimeError("Agent attempts ended without a validated change") commit = workspace.commit() workspace.push() @@ -182,6 +195,29 @@ class Worker: finally: workspace.cleanup() + def _validate_agent_changes(self, workspace: GitWorkspace, issue_number: int): + summary = workspace.summary() + normalized = workspace.normalize_changed_whitespace(summary) + if normalized: + logger.info( + "Normalized trailing whitespace in %s changed file(s) for issue #%s", + len(normalized), + issue_number, + ) + summary = workspace.summary() + validate_changes(workspace, summary, self.settings) + workspace.check_diff(summary) + for command in self.settings.test_commands: + workspace.run_trusted( + command, + extra_environment=self.settings.test_environment, + ) + + summary = workspace.summary() + validate_changes(workspace, summary, self.settings) + workspace.check_diff(summary) + return summary + def _agent_failure_details( self, result, diff --git a/tests/conftest.py b/tests/conftest.py index 885117d..4281a10 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -30,6 +30,7 @@ def settings(tmp_path: Path) -> Settings: agent_wall_time_seconds=1200, agent_step_limit=40, agent_cost_limit=5.0, + max_agent_attempts=1, agent_user="coder", setup_commands=("composer install",), test_commands=("php artisan test",), diff --git a/tests/test_agent.py b/tests/test_agent.py index dbb6f81..6468ea6 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -20,6 +20,21 @@ def test_task_uses_issue_text_not_github_parser() -> None: assert issue.body in task +def test_task_includes_validation_feedback_for_repair_attempt() -> None: + issue = Issue( + number=43, + title="Broken query", + body="Original report", + html_url="https://gitea.example.test/owner/lai/issues/43", + labels=(), + ) + + task = MiniSWEAgentRunner._task(issue, "PHP parse error on line 3") + + assert "previous attempt failed deterministic validation" in task + assert "PHP parse error on line 3" in task + + def test_system_prompt_requires_precise_diff_hygiene() -> None: assert "Never create backup files" in _SYSTEM_TEMPLATE assert "failure-report transport code" in _SYSTEM_TEMPLATE diff --git a/tests/test_config.py b/tests/test_config.py index f841551..cea6f0d 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -30,6 +30,7 @@ def test_settings_load_issue_workflow_defaults( assert settings.queued_label == "ai:queued" assert settings.running_label == "ai:running" assert settings.agent_model == "mistral/devstral-latest" + assert settings.max_agent_attempts == 3 assert settings.test_commands == ("npm run build", "php artisan test") assert settings.test_environment["DB_DATABASE"] == "lai_agent_test" assert settings.test_environment["ERP_DATABASE"] == "customer_agent_test" diff --git a/tests/test_worker.py b/tests/test_worker.py index a383624..4cc22aa 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -175,6 +175,50 @@ def test_agent_failure_details_identify_step_limit(settings) -> None: assert "issue-43.traj.json" in details +@patch("lai_automation.worker.GitWorkspace") +def test_failed_validation_is_given_to_repair_attempt( + GitWorkspace, + settings, +) -> None: + settings = settings.__class__( + **{**settings.__dict__, "max_agent_attempts": 2} + ) + issue = queued_issue() + running = Issue(**{**issue.__dict__, "labels": (Label(2, "ai:running"),)}) + gitea = Mock() + gitea.get_issue.return_value = issue + gitea.set_automation_state.side_effect = [running, running] + gitea.create_pull_request.return_value = PullRequest( + 18, + "https://gitea.example.test/owner/lai/pulls/18", + ) + agent = Mock() + agent.run.side_effect = [ + AgentResult("Submitted", "First", 4, 0.2), + AgentResult("Submitted", "Repaired", 2, 0.1), + ] + workspace = GitWorkspace.return_value + workspace.path = settings.workspace_root / "issue-7" / "repo" + workspace.branch = "ai/issue-7" + workspace.summary.return_value = ChangeSummary(("app/Fix.php",), 2, 0) + workspace.normalize_changed_whitespace.return_value = () + workspace.has_symlink.return_value = False + workspace.has_path.return_value = False + workspace.run_trusted.side_effect = [ + Mock(), + RuntimeError("PHP parse error on line 3"), + Mock(), + ] + workspace.commit.return_value = "abc123" + + Worker(settings, gitea, agent).process(issue, labels()) + + assert agent.run.call_count == 2 + assert "PHP parse error on line 3" in agent.run.call_args_list[1].args[3] + workspace.push.assert_called_once() + assert gitea.set_automation_state.call_args_list[-1].args[1] == "ai:ready" + + @patch("lai_automation.worker.GitWorkspace") def test_preflight_runs_setup_and_tests_without_model( GitWorkspace,