Include structured feedback in repair tasks
Some checks failed
CI / test (push) Has been cancelled

This commit is contained in:
2026-08-27 18:18:04 +02:00
parent efce42b967
commit 2ee2887eb9
4 changed files with 21 additions and 1 deletions

View File

@@ -12,6 +12,8 @@ class FailureReport:
source: str source: str
category: str category: str
user_feedback: str | None user_feedback: str | None
feedback_type: str | None
expected_behavior: str | None
prompt: str | None prompt: str | None
response: str | None response: str | None
sql_query: str | None sql_query: str | None
@@ -29,6 +31,8 @@ class FailureReport:
source=row["source"], source=row["source"],
category=row["category"], category=row["category"],
user_feedback=row.get("user_feedback"), user_feedback=row.get("user_feedback"),
feedback_type=row.get("feedback_type"),
expected_behavior=row.get("expected_behavior"),
prompt=row.get("prompt"), prompt=row.get("prompt"),
response=row.get("response"), response=row.get("response"),
sql_query=row.get("sql_query"), sql_query=row.get("sql_query"),

View File

@@ -29,6 +29,8 @@ def redact(value: str | None) -> str:
def build_task(report: FailureReport) -> str: def build_task(report: FailureReport) -> str:
prompt = redact(report.prompt) prompt = redact(report.prompt)
feedback = redact(report.user_feedback) feedback = redact(report.user_feedback)
expected_behavior = redact(report.expected_behavior)
response = redact(report.response)
error = redact(report.error_message) error = redact(report.error_message)
sql = redact(report.sql_query) sql = redact(report.sql_query)
@@ -44,6 +46,7 @@ Rules:
- Do not commit or push; the orchestrator owns Git publication. - Do not commit or push; the orchestrator owns Git publication.
Failure category: {report.category} Failure category: {report.category}
Feedback type: {report.feedback_type or 'not specified'}
Source: {report.source} Source: {report.source}
Model: {redact(str(report.context.get('model', 'unknown')))} Model: {redact(str(report.context.get('model', 'unknown')))}
Database type: {redact(str(report.context.get('database_type', 'unknown')))} Database type: {redact(str(report.context.get('database_type', 'unknown')))}
@@ -54,6 +57,12 @@ User request:
User feedback: User feedback:
{feedback or '[not available]'} {feedback or '[not available]'}
Expected behavior:
{expected_behavior or '[not available]'}
Observed answer:
{response or '[not available]'}
Observed error: Observed error:
{error or '[not available]'} {error or '[not available]'}

View File

@@ -19,8 +19,10 @@ def test_task_contains_constraints_and_redacted_report() -> None:
source="user_feedback", source="user_feedback",
category="incorrect_answer", category="incorrect_answer",
user_feedback="password=secret-value", user_feedback="password=secret-value",
feedback_type="wrong_join",
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
prompt="List locations for admin@example.com", prompt="List locations for admin@example.com",
response=None, response="The wrong product count was returned.",
sql_query="SELECT * FROM company", sql_query="SELECT * FROM company",
error_message=None, error_message=None,
context={"model": "mistral", "database_type": "customer"}, context={"model": "mistral", "database_type": "customer"},
@@ -31,6 +33,9 @@ def test_task_contains_constraints_and_redacted_report() -> None:
assert "admin@example.com" not in task assert "admin@example.com" not in task
assert "Do not commit or push" in task assert "Do not commit or push" in task
assert "never mock or stub the behavior being tested" in task assert "never mock or stub the behavior being tested" in task
assert "Feedback type: wrong_join" in task
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
assert "The wrong product count was returned." in task
def test_redact_removes_extended_credentials() -> None: def test_redact_removes_extended_credentials() -> None:

View File

@@ -13,6 +13,8 @@ def test_dry_run_generates_task_without_agent_or_git() -> None:
source="automatic", source="automatic",
category="sql_execution", category="sql_execution",
user_feedback=None, user_feedback=None,
feedback_type=None,
expected_behavior=None,
prompt="List locations", prompt="List locations",
response=None, response=None,
sql_query=None, sql_query=None,