This commit is contained in:
@@ -43,6 +43,21 @@ schema/semantics, and fix SQL-generation guidance rather than rewriting one unkn
|
|||||||
to another guessed column. TenantDatabaseService records the explicit source type in
|
to another guessed column. TenantDatabaseService records the explicit source type in
|
||||||
database.connections.<connection>.tenant_type; use that metadata instead of database-name
|
database.connections.<connection>.tenant_type; use that metadata instead of database-name
|
||||||
patterns or heuristics.
|
patterns or heuristics.
|
||||||
|
"""
|
||||||
|
sql_semantics = ''
|
||||||
|
if report.category in {'incorrect_answer', 'sql_execution'}:
|
||||||
|
sql_semantics = """
|
||||||
|
SQL repair guidance:
|
||||||
|
When the SQL is syntactically valid but uses the wrong business calculation, prefer adding or
|
||||||
|
correcting tracked domain knowledge in config/*_semantics.php so the schema prompt teaches the
|
||||||
|
model the right joins, filters, and formula. Do not hardcode one user's question or article ID
|
||||||
|
in SQLGeneratorService, and do not add post-generation replacement of guessed SQL.
|
||||||
|
|
||||||
|
For regression coverage of prompt guidance, expose the relevant protected schema/business-
|
||||||
|
context builder through a test subclass and assert that its real output contains the required
|
||||||
|
formula and rules. Do not mock AIService to return the known-bad SQL and then expect prompt
|
||||||
|
guidance to rewrite that mocked response. Keep database fixture tests only when production code
|
||||||
|
itself deterministically builds and executes the SQL under test.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository.
|
return f"""You are fixing failure report #{report.id} in the Laravel AI Chat repository.
|
||||||
@@ -80,15 +95,18 @@ Observed error:
|
|||||||
Generated SQL, if relevant:
|
Generated SQL, if relevant:
|
||||||
{sql or '[not available]'}
|
{sql or '[not available]'}
|
||||||
{tenant_scope}
|
{tenant_scope}
|
||||||
|
{sql_semantics}
|
||||||
|
|
||||||
Run focused tests while working. Leave the repository with the fix and regression test applied.
|
Run focused tests while working. Leave the repository with the fix and regression test applied.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
|
||||||
def build_retry_task(exception: Exception) -> str:
|
def build_retry_task(report: FailureReport, exception: Exception) -> str:
|
||||||
failure = redact(str(exception))[-6_000:]
|
failure = redact(str(exception))[-6_000:]
|
||||||
|
|
||||||
return f"""The previous implementation failed isolated validation.
|
return build_task(report) + f"""
|
||||||
|
|
||||||
|
The previous implementation failed isolated validation.
|
||||||
|
|
||||||
Correct the existing changes using this test output:
|
Correct the existing changes using this test output:
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,7 @@ class AutomationWorker:
|
|||||||
attempt,
|
attempt,
|
||||||
)
|
)
|
||||||
task = build_retry_task(
|
task = build_retry_task(
|
||||||
|
report,
|
||||||
RuntimeError(
|
RuntimeError(
|
||||||
"No file changes were applied. Make the smallest concrete code and regression-test edits now."
|
"No file changes were applied. Make the smallest concrete code and regression-test edits now."
|
||||||
)
|
)
|
||||||
@@ -76,7 +77,7 @@ class AutomationWorker:
|
|||||||
report.id,
|
report.id,
|
||||||
attempt,
|
attempt,
|
||||||
)
|
)
|
||||||
task = build_retry_task(exception)
|
task = build_retry_task(report, exception)
|
||||||
|
|
||||||
summary = workspace.summary()
|
summary = workspace.summary()
|
||||||
metadata["changes"] = asdict(summary)
|
metadata["changes"] = asdict(summary)
|
||||||
|
|||||||
@@ -54,12 +54,53 @@ def test_redact_removes_extended_credentials() -> None:
|
|||||||
|
|
||||||
|
|
||||||
def test_retry_task_contains_failure_without_secrets() -> None:
|
def test_retry_task_contains_failure_without_secrets() -> None:
|
||||||
task = build_retry_task(RuntimeError("Missing import; token=secret-value"))
|
report = FailureReport(
|
||||||
|
id=8,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Use a weighted average",
|
||||||
|
prompt="Average sale price",
|
||||||
|
response="Zero",
|
||||||
|
sql_query="SELECT AVG(list_price)",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
task = build_retry_task(report, RuntimeError("Missing import; token=secret-value"))
|
||||||
|
|
||||||
assert "Missing import" in task
|
assert "Missing import" in task
|
||||||
assert "secret-value" not in task
|
assert "secret-value" not in task
|
||||||
assert "Do not remove or weaken tests" in task
|
assert "Do not remove or weaken tests" in task
|
||||||
assert "unrelated production routing" in task
|
assert "unrelated production routing" in task
|
||||||
|
assert "Use a weighted average" in task
|
||||||
|
assert "SELECT AVG(list_price)" in task
|
||||||
|
|
||||||
|
|
||||||
|
def test_incorrect_answer_task_guides_semantics_level_repair() -> None:
|
||||||
|
report = FailureReport(
|
||||||
|
id=42,
|
||||||
|
chat_message_id=1,
|
||||||
|
source="user_feedback",
|
||||||
|
category="incorrect_answer",
|
||||||
|
user_feedback="Wrong result",
|
||||||
|
feedback_type="wrong_result",
|
||||||
|
expected_behavior="Sum net values and divide by quantity",
|
||||||
|
prompt="Average sale price for article 9031",
|
||||||
|
response="0.00",
|
||||||
|
sql_query="SELECT AVG(Vk_preis) FROM FK_Artikel",
|
||||||
|
error_message=None,
|
||||||
|
context={"database_type": "erp"},
|
||||||
|
)
|
||||||
|
|
||||||
|
task = build_task(report)
|
||||||
|
|
||||||
|
assert "config/*_semantics.php" in task
|
||||||
|
assert "Do not hardcode" in task
|
||||||
|
assert "protected schema/business-" in task
|
||||||
|
assert "context builder" in task
|
||||||
|
assert "known-bad SQL" in task
|
||||||
|
|
||||||
|
|
||||||
def test_customer_database_task_explains_tenant_scope() -> None:
|
def test_customer_database_task_explains_tenant_scope() -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user