This commit is contained in:
@@ -12,6 +12,8 @@ class FailureReport:
|
||||
source: str
|
||||
category: str
|
||||
user_feedback: str | None
|
||||
feedback_type: str | None
|
||||
expected_behavior: str | None
|
||||
prompt: str | None
|
||||
response: str | None
|
||||
sql_query: str | None
|
||||
@@ -29,6 +31,8 @@ class FailureReport:
|
||||
source=row["source"],
|
||||
category=row["category"],
|
||||
user_feedback=row.get("user_feedback"),
|
||||
feedback_type=row.get("feedback_type"),
|
||||
expected_behavior=row.get("expected_behavior"),
|
||||
prompt=row.get("prompt"),
|
||||
response=row.get("response"),
|
||||
sql_query=row.get("sql_query"),
|
||||
|
||||
@@ -29,6 +29,8 @@ def redact(value: str | None) -> str:
|
||||
def build_task(report: FailureReport) -> str:
|
||||
prompt = redact(report.prompt)
|
||||
feedback = redact(report.user_feedback)
|
||||
expected_behavior = redact(report.expected_behavior)
|
||||
response = redact(report.response)
|
||||
error = redact(report.error_message)
|
||||
sql = redact(report.sql_query)
|
||||
|
||||
@@ -44,6 +46,7 @@ Rules:
|
||||
- Do not commit or push; the orchestrator owns Git publication.
|
||||
|
||||
Failure category: {report.category}
|
||||
Feedback type: {report.feedback_type or 'not specified'}
|
||||
Source: {report.source}
|
||||
Model: {redact(str(report.context.get('model', 'unknown')))}
|
||||
Database type: {redact(str(report.context.get('database_type', 'unknown')))}
|
||||
@@ -54,6 +57,12 @@ User request:
|
||||
User feedback:
|
||||
{feedback or '[not available]'}
|
||||
|
||||
Expected behavior:
|
||||
{expected_behavior or '[not available]'}
|
||||
|
||||
Observed answer:
|
||||
{response or '[not available]'}
|
||||
|
||||
Observed error:
|
||||
{error or '[not available]'}
|
||||
|
||||
|
||||
@@ -19,8 +19,10 @@ def test_task_contains_constraints_and_redacted_report() -> None:
|
||||
source="user_feedback",
|
||||
category="incorrect_answer",
|
||||
user_feedback="password=secret-value",
|
||||
feedback_type="wrong_join",
|
||||
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
|
||||
prompt="List locations for admin@example.com",
|
||||
response=None,
|
||||
response="The wrong product count was returned.",
|
||||
sql_query="SELECT * FROM company",
|
||||
error_message=None,
|
||||
context={"model": "mistral", "database_type": "customer"},
|
||||
@@ -31,6 +33,9 @@ def test_task_contains_constraints_and_redacted_report() -> None:
|
||||
assert "admin@example.com" not in task
|
||||
assert "Do not commit or push" in task
|
||||
assert "never mock or stub the behavior being tested" in task
|
||||
assert "Feedback type: wrong_join" in task
|
||||
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
|
||||
assert "The wrong product count was returned." in task
|
||||
|
||||
|
||||
def test_redact_removes_extended_credentials() -> None:
|
||||
|
||||
@@ -13,6 +13,8 @@ def test_dry_run_generates_task_without_agent_or_git() -> None:
|
||||
source="automatic",
|
||||
category="sql_execution",
|
||||
user_feedback=None,
|
||||
feedback_type=None,
|
||||
expected_behavior=None,
|
||||
prompt="List locations",
|
||||
response=None,
|
||||
sql_query=None,
|
||||
|
||||
Reference in New Issue
Block a user