feat: improve worker diagnostics
Some checks failed
CI / test (push) Has been cancelled
CI / test (pull_request) Has been cancelled

This commit is contained in:
Christian Meyer
2026-08-31 17:03:40 +02:00
parent 407af8099b
commit 4b5ac5f9d7
8 changed files with 90 additions and 3 deletions

View File

@@ -11,6 +11,7 @@ GITEA_RUNNING_LABEL=ai:running
GITEA_READY_LABEL=ai:ready GITEA_READY_LABEL=ai:ready
GITEA_FAILED_LABEL=ai:failed GITEA_FAILED_LABEL=ai:failed
POLL_SECONDS=60 POLL_SECONDS=60
LOG_LEVEL=INFO
AGENT_MODEL=mistral/devstral-latest AGENT_MODEL=mistral/devstral-latest
MISTRAL_API_KEY= MISTRAL_API_KEY=

View File

@@ -44,6 +44,10 @@ Start the worker with Docker Compose. Use `lai-agent check` inside the image for
validation, `lai-agent preflight` for clean-clone setup and tests without a model call, or validation, `lai-agent preflight` for clean-clone setup and tests without a model call, or
`lai-agent once` for a single polling cycle. `lai-agent once` for a single polling cycle.
Set `LOG_LEVEL=DEBUG` to include redacted output from setup and test commands. Agent runs always
log their model, configured limits, call count, cost, exit status, and trajectory path. `INFO` is
the default for normal operation.
## Archived implementation ## Archived implementation
The previous database-polling Aider implementation is preserved unchanged in `old_2026-08-31/`. The previous database-polling Aider implementation is preserved unchanged in `old_2026-08-31/`.

View File

@@ -20,6 +20,7 @@ services:
AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-1200} AGENT_TIMEOUT_SECONDS: ${AGENT_TIMEOUT_SECONDS:-1200}
AGENT_STEP_LIMIT: ${AGENT_STEP_LIMIT:-50} AGENT_STEP_LIMIT: ${AGENT_STEP_LIMIT:-50}
AGENT_COST_LIMIT: ${AGENT_COST_LIMIT:-5} AGENT_COST_LIMIT: ${AGENT_COST_LIMIT:-5}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
AGENT_USER: coder AGENT_USER: coder
MISTRAL_API_KEY: ${MISTRAL_API_KEY:-} MISTRAL_API_KEY: ${MISTRAL_API_KEY:-}
POLL_SECONDS: ${POLL_SECONDS:-60} POLL_SECONDS: ${POLL_SECONDS:-60}

View File

@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import logging
from pathlib import Path from pathlib import Path
from minisweagent.agents.default import DefaultAgent from minisweagent.agents.default import DefaultAgent
@@ -9,6 +10,8 @@ from .config import Settings
from .models import AgentResult, Issue from .models import AgentResult, Issue
from .security import SafeLocalEnvironment from .security import SafeLocalEnvironment
logger = logging.getLogger(__name__)
_SYSTEM_TEMPLATE = """You are a careful coding agent working in one checked-out repository. _SYSTEM_TEMPLATE = """You are a careful coding agent working in one checked-out repository.
Solve only the supplied issue. Inspect existing code and tests before editing. Solve only the supplied issue. Inspect existing code and tests before editing.
Keep the patch minimal, add or update regression tests, and preserve public APIs Keep the patch minimal, add or update regression tests, and preserve public APIs
@@ -80,6 +83,17 @@ class MiniSWEAgentRunner:
workspace: Path, workspace: Path,
trajectory_path: Path, trajectory_path: Path,
) -> AgentResult: ) -> AgentResult:
logger.info(
"Starting agent for issue #%s: model=%s step_limit=%s cost_limit=%.2f "
"wall_time_limit=%ss command_timeout=%ss trajectory=%s",
issue.number,
self.settings.agent_model,
self.settings.agent_step_limit,
self.settings.agent_cost_limit,
self.settings.agent_wall_time_seconds,
self.settings.agent_command_timeout_seconds,
trajectory_path,
)
model_kwargs: dict[str, object] = { model_kwargs: dict[str, object] = {
"timeout": min(self.settings.agent_wall_time_seconds, 600), "timeout": min(self.settings.agent_wall_time_seconds, 600),
} }
@@ -117,12 +131,22 @@ class MiniSWEAgentRunner:
output_path=trajectory_path, output_path=trajectory_path,
) )
info = agent.run(self._task(issue)) info = agent.run(self._task(issue))
return AgentResult( result = AgentResult(
exit_status=str(info.get("exit_status") or "unknown"), exit_status=str(info.get("exit_status") or "unknown"),
submission=str(info.get("submission") or ""), submission=str(info.get("submission") or ""),
model_calls=agent.n_calls, model_calls=agent.n_calls,
cost=float(agent.cost), cost=float(agent.cost),
) )
logger.info(
"Agent finished for issue #%s: status=%s model_calls=%s cost=%.4f "
"trajectory=%s",
issue.number,
result.exit_status,
result.model_calls,
result.cost,
trajectory_path,
)
return result
@staticmethod @staticmethod
def _task(issue: Issue) -> str: def _task(issue: Issue) -> str:

View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import argparse import argparse
import logging import logging
import os
from .config import Settings from .config import Settings
from .worker import Worker from .worker import Worker
@@ -15,7 +16,7 @@ def main() -> None:
nargs="?", nargs="?",
default="worker", default="worker",
) )
parser.add_argument("--log-level", default="INFO") parser.add_argument("--log-level", default=os.getenv("LOG_LEVEL", "INFO"))
arguments = parser.parse_args() arguments = parser.parse_args()
logging.basicConfig( logging.basicConfig(
level=getattr(logging, arguments.log_level.upper(), logging.INFO), level=getattr(logging, arguments.log_level.upper(), logging.INFO),

View File

@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
import logging
import os import os
import pwd import pwd
import re import re
@@ -15,6 +16,8 @@ from .config import Settings
from .models import ChangeSummary from .models import ChangeSummary
from .security import redact from .security import redact
logger = logging.getLogger(__name__)
_TRAILING_WHITESPACE = re.compile(r"^(.*):(\d+): trailing whitespace\.$") _TRAILING_WHITESPACE = re.compile(r"^(.*):(\d+): trailing whitespace\.$")
@@ -65,6 +68,8 @@ class GitWorkspace:
extra_environment: Mapping[str, str] | None = None, extra_environment: Mapping[str, str] | None = None,
timeout: int = 1800, timeout: int = 1800,
) -> subprocess.CompletedProcess[str]: ) -> subprocess.CompletedProcess[str]:
safe_command = redact(command, 1_000)
logger.info("Running trusted command: %s", safe_command)
environment, identity = self._safe_process_context(extra_environment) environment, identity = self._safe_process_context(extra_environment)
result = subprocess.run( result = subprocess.run(
command, command,
@@ -83,7 +88,15 @@ class GitWorkspace:
8_000, 8_000,
) )
detail = f":\n{output}" if output else "" detail = f":\n{output}" if output else ""
logger.error("Trusted command failed: %s%s", safe_command, detail)
raise RuntimeError(f"Command failed ({command}){detail}") raise RuntimeError(f"Command failed ({command}){detail}")
logger.info("Trusted command completed: %s", safe_command)
if result.stdout or result.stderr:
logger.debug(
"Trusted command output (%s):\n%s",
safe_command,
redact("\n".join(filter(None, (result.stdout, result.stderr))), 8_000),
)
return result return result
def summary(self) -> ChangeSummary: def summary(self) -> ChangeSummary:

View File

@@ -93,9 +93,14 @@ class Worker:
/ "trajectories" / "trajectories"
/ f"issue-{issue.number}-{timestamp}.traj.json" / f"issue-{issue.number}-{timestamp}.traj.json"
) )
started_at = time.monotonic()
result = self.agent.run(current_issue, workspace.path, trajectory) result = self.agent.run(current_issue, workspace.path, trajectory)
elapsed = time.monotonic() - started_at
if result.exit_status != "Submitted": if result.exit_status != "Submitted":
raise RuntimeError(f"Agent stopped with status {result.exit_status}") details = self._agent_failure_details(result, elapsed, trajectory)
raise RuntimeError(
f"Agent stopped with status {result.exit_status}: {details}"
)
summary = workspace.summary() summary = workspace.summary()
normalized = workspace.normalize_changed_whitespace(summary) normalized = workspace.normalize_changed_whitespace(summary)
@@ -177,6 +182,27 @@ class Worker:
finally: finally:
workspace.cleanup() workspace.cleanup()
def _agent_failure_details(
self,
result,
elapsed: float,
trajectory,
) -> str:
limits: list[str] = []
if result.model_calls >= self.settings.agent_step_limit:
limits.append(f"step limit reached ({self.settings.agent_step_limit})")
if self.settings.agent_cost_limit and result.cost >= self.settings.agent_cost_limit:
limits.append(f"cost limit reached ({self.settings.agent_cost_limit:.2f})")
if elapsed >= self.settings.agent_wall_time_seconds:
limits.append(
f"wall-time limit reached ({self.settings.agent_wall_time_seconds}s)"
)
reason = ", ".join(limits) or "limit source not reported by mini-SWE-agent"
return (
f"{reason}; model_calls={result.model_calls}; cost={result.cost:.4f}; "
f"elapsed={elapsed:.1f}s; trajectory={trajectory}"
)
def _pull_request_body( def _pull_request_body(
self, self,
issue: Issue, issue: Issue,

View File

@@ -1,5 +1,6 @@
from __future__ import annotations from __future__ import annotations
from pathlib import Path
from unittest.mock import Mock, call, patch from unittest.mock import Mock, call, patch
from lai_automation.models import ( from lai_automation.models import (
@@ -158,6 +159,22 @@ def test_run_once_returns_false_without_work(settings) -> None:
assert Worker(settings, gitea, Mock()).run_once() is False assert Worker(settings, gitea, Mock()).run_once() is False
def test_agent_failure_details_identify_step_limit(settings) -> None:
worker = Worker(settings, Mock(), Mock())
result = AgentResult("LimitsExceeded", "", settings.agent_step_limit, 0.2)
details = worker._agent_failure_details(
result,
elapsed=180.0,
trajectory=Path("/workspaces/trajectories/issue-43.traj.json"),
)
assert f"step limit reached ({settings.agent_step_limit})" in details
assert "model_calls=40" in details
assert "cost=0.2000" in details
assert "issue-43.traj.json" in details
@patch("lai_automation.worker.GitWorkspace") @patch("lai_automation.worker.GitWorkspace")
def test_preflight_runs_setup_and_tests_without_model( def test_preflight_runs_setup_and_tests_without_model(
GitWorkspace, GitWorkspace,