368 lines
14 KiB
Python
368 lines
14 KiB
Python
from lai_automation.models import FailureReport
|
|
from lai_automation.prompt import build_retry_task, build_task, redact, repair_files
|
|
|
|
|
|
def test_redact_removes_common_secrets_and_personal_markers() -> None:
|
|
value = "password=hunter2 token: abc123 admin@example.com 10.2.3.4 Bearer secret.jwt"
|
|
result = redact(value)
|
|
assert "hunter2" not in result
|
|
assert "abc123" not in result
|
|
assert "admin@example.com" not in result
|
|
assert "10.2.3.4" not in result
|
|
assert "secret.jwt" not in result
|
|
|
|
|
|
def test_task_contains_constraints_and_redacted_report() -> None:
|
|
report = FailureReport(
|
|
id=7,
|
|
chat_message_id=91,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="password=secret-value",
|
|
feedback_type="wrong_join",
|
|
expected_behavior="Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3.",
|
|
prompt="List locations for admin@example.com",
|
|
response="The wrong product count was returned.",
|
|
sql_query="SELECT * FROM company",
|
|
error_message=None,
|
|
context={"model": "mistral", "database_type": "customer"},
|
|
)
|
|
task = build_task(report)
|
|
assert "failure report #7" in task
|
|
assert "secret-value" not in task
|
|
assert "admin@example.com" not in task
|
|
assert "Do not commit or push" in task
|
|
assert "never mock or stub the behavior being tested" in task
|
|
assert "Repository ownership map:" in task
|
|
assert "config/erp_semantics.php owns ERP joins" in task
|
|
assert "Tests must contain assertions that execute" in task
|
|
assert "Do not expose protected production methods publicly" in task
|
|
assert "Feedback type: wrong_join" in task
|
|
assert "Join FK_Artikel with FK_AuftragPos and filter Auftragkennung = 3." in task
|
|
assert "The wrong product count was returned." in task
|
|
|
|
|
|
def test_redact_removes_extended_credentials() -> None:
|
|
private_key = "-----BEGIN PRIVATE KEY-----\nprivate-material\n-----END PRIVATE KEY-----"
|
|
value = (
|
|
"AKIAIOSFODNN7EXAMPLE "
|
|
"eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxMjM0In0.signature "
|
|
"mysql://admin:database-password@db.internal/lai "
|
|
f"{private_key}"
|
|
)
|
|
result = redact(value)
|
|
assert "AKIAIOSFODNN7EXAMPLE" not in result
|
|
assert "eyJhbGci" not in result
|
|
assert "database-password" not in result
|
|
assert "private-material" not in result
|
|
|
|
|
|
def test_retry_task_contains_failure_without_secrets() -> None:
|
|
report = FailureReport(
|
|
id=8,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong result",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="Use a weighted average",
|
|
prompt="Average sale price",
|
|
response="Zero",
|
|
sql_query="SELECT AVG(list_price)",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
task = build_retry_task(report, RuntimeError("Missing import; token=secret-value"))
|
|
|
|
assert "Missing import" in task
|
|
assert "secret-value" not in task
|
|
assert "Do not remove or weaken tests" in task
|
|
assert "unrelated production routing" in task
|
|
assert "Use a weighted average" in task
|
|
assert "SELECT AVG(list_price)" in task
|
|
|
|
|
|
def test_incorrect_answer_task_guides_semantics_level_repair() -> None:
|
|
report = FailureReport(
|
|
id=42,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong result",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="Sum net values and divide by quantity",
|
|
prompt="Average sale price for article 9031",
|
|
response="0.00",
|
|
sql_query="SELECT AVG(Vk_preis) FROM FK_Artikel",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "config/*_semantics.php" in task
|
|
assert "Do not hardcode" in task
|
|
assert "protected schema/business-" in task
|
|
assert "context builder" in task
|
|
assert "known-bad SQL" in task
|
|
assert "compare every alias.column" in task
|
|
assert "Never replace one missing column" in task
|
|
assert "Never require or copy production rows" in task
|
|
assert repair_files(report) == (
|
|
"config/erp_semantics.php",
|
|
"tests/Unit/SQLGeneratorBusinessContextTest.php",
|
|
)
|
|
|
|
|
|
def test_task_derives_exact_sql_literal_contract() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong result",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
"FK_Auftrag nach Auftragskennung 2 filtern, System_created nutzen, "
|
|
"obsolet und status storniert ausfiltern"
|
|
),
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query="SELECT Datum_erfassung",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "Exact SQL literal contract" in task
|
|
assert "a.AuftragsKennung = 2" in task
|
|
assert "a.System_created" in task
|
|
assert "COALESCE(a.obsolet, 0) = 0" in task
|
|
assert "COALESCE(a.bStatus_storniert, 0) = 0" in task
|
|
assert "Do not add an LS lookup or bStatus_gebucht" in task
|
|
|
|
|
|
def test_task_derives_exact_negative_contract_for_removal_only_report() -> None:
|
|
report = FailureReport(
|
|
id=5,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong delivery note count",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss " a.bStatus_gebucht = 1 " entfernt werden. '
|
|
"Einfach nur das aus der SQL entfernen"
|
|
),
|
|
prompt="Delivery notes by month",
|
|
response="0 rows",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "Exact SQL removal contract" in task
|
|
assert "a.bStatus_gebucht = 1" in task
|
|
assert "negative assertion" in task
|
|
assert "forbidden identifier is a negative requirement" in task
|
|
assert (
|
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
|
in task
|
|
)
|
|
assert "copy them exactly, including aliases and qualifiers" in task
|
|
assert "explicitly a removal-only repair" in task
|
|
assert "Preserve every other selected field" in task
|
|
assert "Exact SQL literal contract" not in task
|
|
|
|
|
|
def test_non_removal_only_contract_protects_correct_reference_and_exercises_rule() -> None:
|
|
report = FailureReport(
|
|
id=6,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Die Abfrage stimmt nicht",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
|
"weil Lieferscheine nicht gebucht werden."
|
|
),
|
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
|
response="0 Ergebnisse",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "protected correct baseline" in task
|
|
assert "preserve it exactly" in task
|
|
assert "positively assert the new rule" in task
|
|
assert "negative assertion alone is vacuous" in task
|
|
assert "without embedding the complete forbidden predicate" in task
|
|
assert "Do not add or restate date-column choices" in task
|
|
assert "every table in its tables list" in task
|
|
assert "supplied to businessContext()" in task
|
|
assert "explicitly a removal-only repair" not in task
|
|
|
|
|
|
def test_retry_task_keeps_failure_from_start_of_long_output() -> None:
|
|
report = FailureReport(
|
|
id=43,
|
|
chat_message_id=1,
|
|
source="automatic",
|
|
category="sql_execution",
|
|
user_feedback=None,
|
|
feedback_type=None,
|
|
expected_behavior=None,
|
|
prompt="Top products",
|
|
response=None,
|
|
sql_query="SELECT a.missing FROM orders a",
|
|
error_message="Unknown column",
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_retry_task(report, RuntimeError("DECISIVE ASSERTION\n" + "x" * 11_000))
|
|
|
|
assert "DECISIVE ASSERTION" in task
|
|
|
|
|
|
def test_customer_database_task_explains_tenant_scope() -> None:
|
|
report = FailureReport(
|
|
id=13,
|
|
chat_message_id=1,
|
|
source="automatic",
|
|
category="sql_execution",
|
|
user_feedback=None,
|
|
feedback_type=None,
|
|
expected_behavior=None,
|
|
prompt="What did this customer buy most?",
|
|
response=None,
|
|
sql_query="SELECT * FROM invoice_row WHERE InvoiceCreator = 'tenant'",
|
|
error_message="Unknown column InvoiceCreator",
|
|
context={"database_type": "customer"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "already selected one tenant database" in task
|
|
assert "must not" in task
|
|
assert "guessed column" in task
|
|
assert "tenant_type" in task
|
|
|
|
|
|
def test_unresolved_placeholder_task_guides_real_answer_path() -> None:
|
|
report = FailureReport(
|
|
id=49,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Token was not restored",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="Return the database value",
|
|
prompt="Average price",
|
|
response="Price: __LAI_UNKNOWN_VALUE__",
|
|
sql_query="SELECT AVG(price)",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "AIService::generateDatabaseResponse" in task
|
|
assert "exact token map" in task
|
|
assert "HTTP-backed AIService test" in task
|
|
assert "local database-result fallback" in task
|
|
|
|
|
|
def test_cross_database_report_guides_staged_source_routing() -> None:
|
|
report = FailureReport(
|
|
id=56,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong hours",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="Join cockpit_config.time_registration to customer manager and ERP positions",
|
|
prompt="Hours in April",
|
|
response="0 hours",
|
|
sql_query="SELECT SUM(hours) FROM LX_AUFGABE",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
task = build_task(report)
|
|
|
|
assert "tables from multiple databases" in task
|
|
assert "staged-query workflows" in task
|
|
assert "Do not put an impossible cross-connection join" in task
|
|
assert "preserving tenant isolation" in task
|
|
assert "editing only config/*_semantics.php or tests is an invalid repair" in task
|
|
assert "describe the data" in task
|
|
assert "flow, connection used at each stage" in task
|
|
assert repair_files(report) == (
|
|
"app/Http/Controllers/ChatController.php",
|
|
"app/Services/TenantDatabaseService.php",
|
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
|
)
|
|
|
|
|
|
def test_presentation_and_route_feedback_is_routed_to_real_owners() -> None:
|
|
report = FailureReport(
|
|
id=7,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
|
|
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
|
|
),
|
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
|
response="2024-01: 38, 2024-02: 95",
|
|
sql_query="SELECT Monat, COUNT(*) FROM FK_Auftrag GROUP BY Monat",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
|
|
assert repair_files(report) == (
|
|
"app/Services/AIService.php",
|
|
"app/Http/Controllers/ChatController.php",
|
|
"app/Services/WebSearchService.php",
|
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
|
"tests/Unit/WebSearchServiceTest.php",
|
|
)
|
|
task = build_task(report)
|
|
assert "real answer-formatting change" in task
|
|
assert "exact reported user prompt" in task
|
|
assert '"cockpit:" must force the database path' in task
|
|
assert '"www:" must force public web research' in task
|
|
assert '"direct:" must' in task
|
|
assert "including Ollama/Qwen" in task
|
|
assert "Do not pass private conversation/database context" in task
|
|
assert "AIService::determineRoute() must not be called" in task
|
|
assert "resolveRoutePrefix(string $message): array" in task
|
|
assert "[$explicitRoute, $message] = $this->resolveRoutePrefix($message)" in task
|
|
assert "Do not add prefix handling to resolveDatabaseSource()" in task
|
|
assert "parent::resolveRoutePrefix($message)" in task
|
|
assert "three tests that call processMessage() itself" in task
|
|
assert "setLaravelSession()" in task
|
|
assert "generateWebResearchResponse()" in task
|
|
assert "columns named exactly Monat and Anzahl_Lieferscheine" in task
|
|
assert 'exact substring "allen bereitgestellten Zeilen"' in task
|
|
assert "substr_count($userPrompt" in task
|
|
assert "heuristics must return false" in task
|
|
assert "persisted ChatMessage.message equals the cleaned question exactly" in task
|
|
assert "$response->getData(true)" in task
|
|
assert "Do not write “we can't fully test processMessage”" in task
|
|
assert "one ordinary result row with multiple" in task
|
|
assert "duplicate method declaration" in task
|
|
assert "All three prefix behaviors" in task
|
|
assert "config/*_semantics.php" in task
|