This commit is contained in:
@@ -12,6 +12,8 @@ class FailureReport:
|
|||||||
source: str
|
source: str
|
||||||
category: str
|
category: str
|
||||||
user_feedback: str | None
|
user_feedback: str | None
|
||||||
|
feedback_type: str | None
|
||||||
|
expected_behavior: str | None
|
||||||
prompt: str | None
|
prompt: str | None
|
||||||
response: str | None
|
response: str | None
|
||||||
sql_query: str | None
|
sql_query: str | None
|
||||||
@@ -29,6 +31,8 @@ class FailureReport:
|
|||||||
source=row["source"],
|
source=row["source"],
|
||||||
category=row["category"],
|
category=row["category"],
|
||||||
user_feedback=row.get("user_feedback"),
|
user_feedback=row.get("user_feedback"),
|
||||||
|
feedback_type=row.get("feedback_type"),
|
||||||
|
expected_behavior=row.get("expected_behavior"),
|
||||||
prompt=row.get("prompt"),
|
prompt=row.get("prompt"),
|
||||||
response=row.get("response"),
|
response=row.get("response"),
|
||||||
sql_query=row.get("sql_query"),
|
sql_query=row.get("sql_query"),
|
||||||
|
|||||||
@@ -29,6 +29,8 @@ def redact(value: str | None) -> str:
|
|||||||
def build_task(report: FailureReport) -> str:
|
def build_task(report: FailureReport) -> str:
|
||||||
prompt = redact(report.prompt)
|
prompt = redact(report.prompt)
|
||||||
feedback = redact(report.user_feedback)
|
feedback = redact(report.user_feedback)
|
||||||
|
expected_behavior = redact(report.expected_behavior)
|
||||||
|
response = redact(report.response)
|
||||||
error = redact(report.error_message)
|
error = redact(report.error_message)
|
||||||
sql = redact(report.sql_query)
|
sql = redact(report.sql_query)
|
||||||
|
|
||||||
@@ -44,6 +46,7 @@ Rules:
|
|||||||
- Do not commit or push; the orchestrator owns Git publication.
|
- Do not commit or push; the orchestrator owns Git publication.
|
||||||
|
|
||||||
Failure category: {report.category}
|
Failure category: {report.category}
|
||||||
|
Feedback type: {report.feedback_type or 'not specified'}
|
||||||
Source: {report.source}
|
Source: {report.source}
|
||||||
Model: {redact(str(report.context.get('model', 'unknown')))}
|
Model: {redact(str(report.context.get('model', 'unknown')))}
|
||||||
Database type: {redact(str(report.context.get('database_type', 'unknown')))}
|
Database type: {redact(str(report.context.get('database_type', 'unknown')))}
|
||||||
@@ -54,6 +57,12 @@ User request:
|
|||||||
User feedback:
|
User feedback:
|
||||||
{feedback or '[not available]'}
|
{feedback or '[not available]'}
|
||||||
|
|
||||||
|
Expected behavior:
|
||||||
|
{expected_behavior or '[not available]'}
|
||||||
|
|
||||||
|
Observed answer:
|
||||||
|
{response or '[not available]'}
|
||||||
|
|
||||||
Observed error:
|
Observed error:
|
||||||
{error or '[not available]'}
|
{error or '[not available]'}
|
||||||
|
|
||||||
|
|||||||
@@ -19,8 +19,10 @@ def test_task_contains_constraints_and_redacted_report() -> None:
|
|||||||
source="user_feedback",
|
source="user_feedback",
|
||||||
category="incorrect_answer",
|
category="incorrect_answer",
|
||||||
user_feedback="password=secret-value",
|
user_feedback="password=secret-value",
|
||||||
|
feedback_type="wrong_join",
|
||||||
|
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
|
||||||
prompt="List locations for admin@example.com",
|
prompt="List locations for admin@example.com",
|
||||||
response=None,
|
response="The wrong product count was returned.",
|
||||||
sql_query="SELECT * FROM company",
|
sql_query="SELECT * FROM company",
|
||||||
error_message=None,
|
error_message=None,
|
||||||
context={"model": "mistral", "database_type": "customer"},
|
context={"model": "mistral", "database_type": "customer"},
|
||||||
@@ -31,6 +33,9 @@ def test_task_contains_constraints_and_redacted_report() -> None:
|
|||||||
assert "admin@example.com" not in task
|
assert "admin@example.com" not in task
|
||||||
assert "Do not commit or push" in task
|
assert "Do not commit or push" in task
|
||||||
assert "never mock or stub the behavior being tested" in task
|
assert "never mock or stub the behavior being tested" in task
|
||||||
|
assert "Feedback type: wrong_join" in task
|
||||||
|
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
|
||||||
|
assert "The wrong product count was returned." in task
|
||||||
|
|
||||||
|
|
||||||
def test_redact_removes_extended_credentials() -> None:
|
def test_redact_removes_extended_credentials() -> None:
|
||||||
|
|||||||
@@ -13,6 +13,8 @@ def test_dry_run_generates_task_without_agent_or_git() -> None:
|
|||||||
source="automatic",
|
source="automatic",
|
||||||
category="sql_execution",
|
category="sql_execution",
|
||||||
user_feedback=None,
|
user_feedback=None,
|
||||||
|
feedback_type=None,
|
||||||
|
expected_behavior=None,
|
||||||
prompt="List locations",
|
prompt="List locations",
|
||||||
response=None,
|
response=None,
|
||||||
sql_query=None,
|
sql_query=None,
|
||||||
|
|||||||
Reference in New Issue
Block a user