Make agent validation self-correcting
This commit is contained in:
@@ -35,6 +35,7 @@ def settings(**overrides) -> Settings:
|
||||
"poll_seconds": 60,
|
||||
"max_changed_files": 12,
|
||||
"max_changed_lines": 800,
|
||||
"max_agent_attempts": 2,
|
||||
"stale_claim_minutes": 90,
|
||||
"dry_run": True,
|
||||
}
|
||||
@@ -61,5 +62,12 @@ def test_default_commands_initialize_disposable_laravel_schema(monkeypatch) -> N
|
||||
|
||||
config = Settings.from_env()
|
||||
|
||||
assert "npm ci" in config.test_commands
|
||||
assert "npm run build" in config.test_commands
|
||||
assert "php artisan migrate:fresh --force" in config.test_commands
|
||||
assert config.test_commands[-1] == "php artisan test"
|
||||
|
||||
|
||||
def test_agent_attempts_are_bounded() -> None:
|
||||
with pytest.raises(ValueError, match="MAX_AGENT_ATTEMPTS"):
|
||||
settings(max_agent_attempts=4).validate()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
from lai_automation.models import FailureReport
|
||||
from lai_automation.prompt import build_task, redact
|
||||
from lai_automation.prompt import build_retry_task, build_task, redact
|
||||
|
||||
|
||||
def test_redact_removes_common_secrets_and_personal_markers() -> None:
|
||||
@@ -51,3 +51,11 @@ def test_redact_removes_extended_credentials() -> None:
|
||||
assert "eyJhbGci" not in result
|
||||
assert "database-password" not in result
|
||||
assert "private-material" not in result
|
||||
|
||||
|
||||
def test_retry_task_contains_failure_without_secrets() -> None:
|
||||
task = build_retry_task(RuntimeError("Missing import; token=secret-value"))
|
||||
|
||||
assert "Missing import" in task
|
||||
assert "secret-value" not in task
|
||||
assert "Do not remove or weaken tests" in task
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from unittest.mock import Mock
|
||||
from pathlib import Path
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from test_config import settings
|
||||
|
||||
from lai_automation.git import ChangeSummary
|
||||
from lai_automation.models import FailureReport
|
||||
from lai_automation.worker import AutomationWorker
|
||||
|
||||
@@ -49,8 +51,49 @@ def test_database_credentials_are_only_passed_to_test_process() -> None:
|
||||
assert results[0]["returncode"] == 0
|
||||
environment = workspace.run.call_args.kwargs["extra_environment"]
|
||||
assert environment["AGENT_TEST_DB_ENABLED"] == "true"
|
||||
assert environment["APP_ENV"] == "testing"
|
||||
assert environment["AI_MISTRAL_API_KEY"] == "automation-test-key"
|
||||
assert environment["TEST_DB_USER"] == "test_runner"
|
||||
assert environment["TEST_DB_PASSWORD"] == "test-secret"
|
||||
assert environment["DB_DATABASE"] == "lai_agent_test"
|
||||
assert environment["ERP_DATABASE"] == "customer_agent_test"
|
||||
assert environment["COCKPIT_CONFIG_DATABASE"] == "cockpit_agent_test"
|
||||
|
||||
|
||||
def test_failed_validation_gets_one_correction_before_pr() -> None:
|
||||
report = FailureReport(
|
||||
id=13,
|
||||
chat_message_id=1,
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List products",
|
||||
response=None,
|
||||
sql_query="SELECT missing_column FROM invoice_row",
|
||||
error_message="Unknown column",
|
||||
context={},
|
||||
)
|
||||
config = settings(mode="pr", dry_run=False, gitea_token="token")
|
||||
worker = AutomationWorker(config)
|
||||
worker.reports = Mock()
|
||||
worker.reports.claim_next.return_value = report
|
||||
worker.agent = Mock()
|
||||
worker.agent.run.side_effect = ["first change", "corrected change"]
|
||||
worker.gitea = Mock()
|
||||
worker._run_tests = Mock(side_effect=[RuntimeError("Missing import"), []])
|
||||
workspace = Mock()
|
||||
workspace.path = Path("/tmp/nonexistent-workspace")
|
||||
workspace.branch = "ai/failure-13"
|
||||
workspace.summary.return_value = ChangeSummary(("tests/Feature/FixTest.php",), 5, 0)
|
||||
workspace.commit.return_value = "commit-id"
|
||||
|
||||
with patch("lai_automation.worker.GitWorkspace", return_value=workspace):
|
||||
assert worker.run_once()
|
||||
|
||||
assert worker.agent.run.call_count == 2
|
||||
assert "Missing import" in worker.agent.run.call_args_list[1].args[1]
|
||||
assert worker._run_tests.call_count == 2
|
||||
workspace.push_branch.assert_called_once()
|
||||
worker.reports.complete.assert_called_once()
|
||||
|
||||
Reference in New Issue
Block a user