diff --git a/src/lai_automation/models.py b/src/lai_automation/models.py index 7162518..4dfc726 100644 --- a/src/lai_automation/models.py +++ b/src/lai_automation/models.py @@ -12,6 +12,8 @@ class FailureReport: source: str category: str user_feedback: str | None + feedback_type: str | None + expected_behavior: str | None prompt: str | None response: str | None sql_query: str | None @@ -29,6 +31,8 @@ class FailureReport: source=row["source"], category=row["category"], user_feedback=row.get("user_feedback"), + feedback_type=row.get("feedback_type"), + expected_behavior=row.get("expected_behavior"), prompt=row.get("prompt"), response=row.get("response"), sql_query=row.get("sql_query"), diff --git a/src/lai_automation/prompt.py b/src/lai_automation/prompt.py index e318cdc..10a6b3f 100644 --- a/src/lai_automation/prompt.py +++ b/src/lai_automation/prompt.py @@ -29,6 +29,8 @@ def redact(value: str | None) -> str: def build_task(report: FailureReport) -> str: prompt = redact(report.prompt) feedback = redact(report.user_feedback) + expected_behavior = redact(report.expected_behavior) + response = redact(report.response) error = redact(report.error_message) sql = redact(report.sql_query) @@ -44,6 +46,7 @@ Rules: - Do not commit or push; the orchestrator owns Git publication. Failure category: {report.category} +Feedback type: {report.feedback_type or 'not specified'} Source: {report.source} Model: {redact(str(report.context.get('model', 'unknown')))} Database type: {redact(str(report.context.get('database_type', 'unknown')))} @@ -54,6 +57,12 @@ User request: User feedback: {feedback or '[not available]'} +Expected behavior: +{expected_behavior or '[not available]'} + +Observed answer: +{response or '[not available]'} + Observed error: {error or '[not available]'} diff --git a/tests/test_prompt.py b/tests/test_prompt.py index a700cdd..36a15d7 100644 --- a/tests/test_prompt.py +++ b/tests/test_prompt.py @@ -19,8 +19,10 @@ def test_task_contains_constraints_and_redacted_report() -> None: source="user_feedback", category="incorrect_answer", user_feedback="password=secret-value", + feedback_type="wrong_join", + expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.", prompt="List locations for admin@example.com", - response=None, + response="The wrong product count was returned.", sql_query="SELECT * FROM company", error_message=None, context={"model": "mistral", "database_type": "customer"}, @@ -31,6 +33,9 @@ def test_task_contains_constraints_and_redacted_report() -> None: assert "admin@example.com" not in task assert "Do not commit or push" in task assert "never mock or stub the behavior being tested" in task + assert "Feedback type: wrong_join" in task + assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task + assert "The wrong product count was returned." in task def test_redact_removes_extended_credentials() -> None: diff --git a/tests/test_worker.py b/tests/test_worker.py index c50bf35..73154b1 100644 --- a/tests/test_worker.py +++ b/tests/test_worker.py @@ -13,6 +13,8 @@ def test_dry_run_generates_task_without_agent_or_git() -> None: source="automatic", category="sql_execution", user_feedback=None, + feedback_type=None, + expected_behavior=None, prompt="List locations", response=None, sql_query=None,