925 lines
31 KiB
Python
925 lines
31 KiB
Python
from dataclasses import replace
|
|
from unittest.mock import Mock
|
|
|
|
from test_config import settings
|
|
|
|
from lai_automation.git import ChangeSummary
|
|
from lai_automation.models import FailureReport
|
|
from lai_automation.policy import (
|
|
evaluate_changes,
|
|
evaluate_expected_behavior,
|
|
evaluate_removal_prevention,
|
|
evaluate_removal_scope,
|
|
evaluate_report_scope,
|
|
evaluate_requested_behavior_scope,
|
|
evaluate_routed_files,
|
|
)
|
|
|
|
|
|
def test_policy_accepts_small_application_change() -> None:
|
|
result = evaluate_changes(
|
|
ChangeSummary(("app/Services/Example.php", "tests/Feature/ExampleTest.php"), 40, 5),
|
|
settings(),
|
|
)
|
|
assert result.allowed
|
|
|
|
|
|
def test_policy_rejects_secrets_and_vendor_paths() -> None:
|
|
result = evaluate_changes(
|
|
ChangeSummary((".env", "vendor/foo.php", ".aider.tags.cache.v4/cache.db"), 3, 0),
|
|
settings(),
|
|
)
|
|
assert not result.allowed
|
|
assert len(result.errors) == 3
|
|
|
|
|
|
def test_policy_rejects_test_only_repair() -> None:
|
|
result = evaluate_changes(
|
|
ChangeSummary(("tests/Unit/ExampleTest.php",), 20, 2),
|
|
settings(),
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert "production behavior was not fixed" in result.errors[0]
|
|
|
|
|
|
def test_direct_main_blocks_validator_and_migrations() -> None:
|
|
config = replace(
|
|
settings(),
|
|
mode="direct-main",
|
|
dry_run=False,
|
|
allow_direct_main=True,
|
|
gitea_token="token",
|
|
)
|
|
result = evaluate_changes(
|
|
ChangeSummary(("app/Services/SQLValidatorService.php", "database/migrations/new.php"), 10, 2),
|
|
config,
|
|
)
|
|
assert not result.allowed
|
|
|
|
|
|
def test_policy_rejects_symlinks(tmp_path) -> None:
|
|
(tmp_path / "link.php").symlink_to("/etc/passwd")
|
|
result = evaluate_changes(ChangeSummary(("link.php",), 1, 0), settings(), tmp_path)
|
|
assert not result.allowed
|
|
|
|
|
|
def test_cross_database_report_requires_application_orchestration() -> None:
|
|
report = FailureReport(
|
|
id=56,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Use cockpit_config.time_registration",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="Join cockpit_config manager_id and FK_AuftragPos",
|
|
prompt="Hours in April",
|
|
response="0 hours",
|
|
sql_query="SELECT 0",
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
rejected = evaluate_report_scope(
|
|
report,
|
|
ChangeSummary(("config/erp_semantics.php", "tests/Unit/FixTest.php"), 20, 2),
|
|
)
|
|
accepted = evaluate_report_scope(
|
|
report,
|
|
ChangeSummary(("app/Http/Controllers/ChatController.php", "tests/Unit/FixTest.php"), 20, 2),
|
|
)
|
|
|
|
assert not rejected.allowed
|
|
assert "staged orchestration" in rejected.errors[0]
|
|
assert accepted.allowed
|
|
|
|
|
|
def test_expected_behavior_requires_explicit_technical_terms_in_added_code() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong result",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
"FK_Auftrag nach AuftragsKennung 2 filtern und System_created verwenden; "
|
|
"obsolet und storniert ausfiltern"
|
|
),
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query=None,
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
rejected = evaluate_expected_behavior(
|
|
report,
|
|
"Use FK_Auftrag.Datum_erfassung and status storniert",
|
|
)
|
|
accepted = evaluate_expected_behavior(
|
|
report,
|
|
"SELECT DATE_FORMAT(System_created, '%Y-%m') FROM FK_Auftrag "
|
|
"WHERE AuftragsKennung = 2 AND COALESCE(obsolet, 0) = 0 "
|
|
"AND COALESCE(bStatus_storniert, 0) = 0",
|
|
)
|
|
|
|
assert not rejected.allowed
|
|
assert "AuftragsKennung" in rejected.errors[0]
|
|
assert "System_created" in rejected.errors[0]
|
|
assert accepted.allowed
|
|
|
|
|
|
def test_expected_behavior_rejects_negated_required_date_column() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="System_created verwenden",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query=None,
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
result = evaluate_expected_behavior(
|
|
report,
|
|
"Lieferscheine niemals mit System_created filtern",
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert "contradicts expected identifier System_created" in result.errors[0]
|
|
|
|
|
|
def test_expected_behavior_allows_rejecting_an_alternative_column_on_same_table() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="FK_Auftrag nach System_created filtern",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query=None,
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
result = evaluate_expected_behavior(
|
|
report,
|
|
"FK_Auftrag nach System_created filtern, nicht nach FK_Auftrag.Datum_erfassung",
|
|
)
|
|
|
|
assert result.allowed
|
|
|
|
|
|
def test_expected_behavior_does_not_treat_later_never_as_table_negation() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="FK_Auftrag nach System_created filtern",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query=None,
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
result = evaluate_expected_behavior(
|
|
report,
|
|
"FK_Auftrag nach System_created filtern und Datum_erfassung niemals verwenden",
|
|
)
|
|
|
|
assert result.allowed
|
|
|
|
|
|
def test_expected_behavior_rejects_unrequested_filter_copied_from_bad_sql() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="System_created verwenden und status storniert ausfiltern",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query="WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
result = evaluate_expected_behavior(
|
|
report,
|
|
"System_created AND bStatus_gebucht = 1 AND COALESCE(bStatus_storniert, 0) = 0",
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert "copied bStatus_gebucht" in result.errors[-1]
|
|
|
|
|
|
def test_expected_behavior_requires_one_complete_reference_select() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
"FK_Auftrag nach Auftragskennung 2 filtern und System_created verwenden; "
|
|
"obsolet und status storniert ausfiltern"
|
|
),
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query="SELECT * WHERE Bezeichnung_Kurz = 'LS' AND Datum_erfassung >= '2024-01-01'",
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
split_rules_and_bad_sql = (
|
|
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
|
|
"COALESCE(bStatus_storniert, 0) = 0\n"
|
|
"SELECT * FROM FK_Auftrag a WHERE Bezeichnung_Kurz = 'LS' "
|
|
"AND a.System_created >= '2024-01-01' "
|
|
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
|
)
|
|
complete_sql = (
|
|
"AuftragsKennung = 2 System_created COALESCE(obsolet, 0) = 0 "
|
|
"COALESCE(bStatus_storniert, 0) = 0\n"
|
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
|
"AND a.System_created >= '2024-01-01' "
|
|
"AND COALESCE(a.obsolet, 0) = 0 AND COALESCE(a.bStatus_storniert, 0) = 0"
|
|
)
|
|
|
|
rejected = evaluate_expected_behavior(report, split_rules_and_bad_sql)
|
|
accepted = evaluate_expected_behavior(report, complete_sql)
|
|
|
|
assert not rejected.allowed
|
|
assert any("one reference SELECT" in error for error in rejected.errors)
|
|
assert any("LS lookup" in error for error in rejected.errors)
|
|
assert accepted.allowed
|
|
|
|
|
|
def test_expected_behavior_detects_lowercase_document_code_without_observed_sql() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query=None,
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
rejected = evaluate_expected_behavior(
|
|
report,
|
|
"SELECT * FROM FK_Auftrag WHERE Bezeichnung_Kurz = 'LS'",
|
|
)
|
|
accepted = evaluate_expected_behavior(
|
|
report,
|
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2",
|
|
)
|
|
|
|
assert not rejected.allowed
|
|
assert any("AuftragsKennung = 2" in error for error in rejected.errors)
|
|
assert accepted.allowed
|
|
|
|
|
|
def test_expected_behavior_rejects_lookup_join_for_authoritative_direct_code() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="FK_Auftrag nach Auftragskennung 2 filtern",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query=None,
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
|
|
result = evaluate_expected_behavior(
|
|
report,
|
|
"SELECT * FROM FK_Auftrag a JOIN FK_Nummerkreis n ON n.Kennung = a.AuftragsKennung "
|
|
"WHERE a.AuftragsKennung = 2",
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert any("joined FK_Nummerkreis" in error for error in result.errors)
|
|
|
|
|
|
def test_expected_behavior_rejects_invented_obsolete_prefix_and_incomplete_assertions() -> None:
|
|
report = FailureReport(
|
|
id=4,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback=None,
|
|
feedback_type="wrong_result",
|
|
expected_behavior="Auftragskennung 2, obsolet und status storniert ausfiltern",
|
|
prompt="Delivery notes by month",
|
|
response=None,
|
|
sql_query="SELECT 0",
|
|
error_message=None,
|
|
context={},
|
|
)
|
|
invented = (
|
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
|
"AND COALESCE(bStatus_obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
|
|
"$this->assertStringContainsString('AuftragsKennung = 2', $context);"
|
|
)
|
|
complete = (
|
|
"SELECT * FROM FK_Auftrag WHERE AuftragsKennung = 2 "
|
|
"AND COALESCE(obsolet, 0) = 0 AND COALESCE(bStatus_storniert, 0) = 0\n"
|
|
"$this->assertStringContainsString('AuftragsKennung = 2', $context);\n"
|
|
"$this->assertStringContainsString('COALESCE(obsolet, 0) = 0', $context);\n"
|
|
"$this->assertStringContainsString('COALESCE(bStatus_storniert, 0) = 0', $context);"
|
|
)
|
|
|
|
rejected = evaluate_expected_behavior(report, invented)
|
|
accepted = evaluate_expected_behavior(report, complete)
|
|
|
|
assert not rejected.allowed
|
|
assert any("obsolete rows" in error for error in rejected.errors)
|
|
assert any("Focused regression assertions" in error for error in rejected.errors)
|
|
assert accepted.allowed
|
|
|
|
|
|
def test_routed_file_policy_rejects_unexpected_edits() -> None:
|
|
result = evaluate_routed_files(
|
|
ChangeSummary(("config/erp_semantics.php", "app/Services/AIService.php"), 10, 0),
|
|
("config/erp_semantics.php", "tests/Unit/SQLGeneratorBusinessContextTest.php"),
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert "app/Services/AIService.php" in result.errors[0]
|
|
|
|
|
|
def test_requested_behavior_scope_requires_all_presentation_and_routing_owners() -> None:
|
|
report = FailureReport(
|
|
id=7,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Die Antwort sollte als Tabelle angezeigt werden",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
"Die Daten sind korrekt. cockpit: muss immer die Datenbank ausführen und "
|
|
"www: muss im Web recherchieren; direct: muss nur das Modell fragen."
|
|
),
|
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
|
response="2024-01: 38",
|
|
sql_query="SELECT COUNT(*) FROM FK_Auftrag",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
incomplete = evaluate_requested_behavior_scope(
|
|
report,
|
|
ChangeSummary(("tests/Feature/AIPrivacyBoundaryTest.php",), 10, 0),
|
|
"cockpit: www: Markdown-Tabelle",
|
|
)
|
|
complete = evaluate_requested_behavior_scope(
|
|
report,
|
|
ChangeSummary((
|
|
"app/Services/AIService.php",
|
|
"app/Http/Controllers/ChatController.php",
|
|
"app/Services/WebSearchService.php",
|
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
|
"tests/Unit/WebSearchServiceTest.php",
|
|
), 30, 2),
|
|
(
|
|
"Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024\n"
|
|
"Markdown-Tabelle\ncount($rows) > 1\nsingle_row keeps sentence format\n"
|
|
"resolveRoutePrefix\n['database', 'Frage']\n['research', 'Frage']\n"
|
|
"['direct', 'Frage']\n"
|
|
"[null, 'Frage']\n[$explicitRoute, $message] = "
|
|
"$this->resolveRoutePrefix($message);\n$route = $explicitRoute ?? $fallback;\n"
|
|
"generateWebResearchResponse($message, 'ollama');\n"
|
|
"generateDirectResponse($message, 'ollama', []);\n"
|
|
"$controller->processMessage($cockpitRequest);\n"
|
|
"$ai->expects($this->never())->method('determineRoute');\n"
|
|
"function test_cockpit_prefix(): void {\n"
|
|
"$tenant->method('requiresCustomerDatabase')->willReturn(false);\n"
|
|
"$tenant->method('requiresConfigDatabase')->willReturn(false);\n"
|
|
"$tenant->method('requiresErpDatabase')->willReturn(false);\n}\n"
|
|
"$controller->processMessage($wwwRequest);\n"
|
|
"$ai->expects($this->never())->method('determineRoute');\n"
|
|
"$controller->processMessage($directRequest);\n"
|
|
"$ai->expects($this->never())->method('determineRoute');\n"
|
|
"$this->assertSame(3, substr_count($userPrompt, '\"Monat\"'));\n"
|
|
"$this->assertSame(3, substr_count($userPrompt, '\"Anzahl_Lieferscheine\"'));\n"
|
|
"cockpit:\nwww:\ndirect:"
|
|
),
|
|
)
|
|
|
|
assert not incomplete.allowed
|
|
assert any("AIService.php" in error for error in incomplete.errors)
|
|
assert any("ChatController.php" in error for error in incomplete.errors)
|
|
assert not complete.errors
|
|
assert complete.allowed
|
|
|
|
|
|
def test_requested_behavior_scope_rejects_dead_prefix_source_tests() -> None:
|
|
report = FailureReport(
|
|
id=7,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Als Tabelle anzeigen",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
|
|
prompt="Anzahl pro Monat",
|
|
response="Text",
|
|
sql_query="SELECT 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
added = """
|
|
Markdown-Tabelle
|
|
count($rows) > 1
|
|
single_row keeps sentence format
|
|
resolveRoutePrefix
|
|
['database', 'Frage']
|
|
['direct', 'Frage']
|
|
[null, 'Frage']
|
|
[$explicitRoute, $message] = $this->resolveRoutePrefix($message);
|
|
$route = $explicitRoute ?? $fallback;
|
|
$controller->processMessage($cockpitRequest);
|
|
$ai->expects($this->never())->method('determineRoute');
|
|
$controller->processMessage($wwwRequest);
|
|
$ai->expects($this->never())->method('determineRoute');
|
|
$controller->resolveSource('cockpit: Frage', []);
|
|
return $this->resolveRoutePrefix($message);
|
|
public function determineRouteForMessage() { return $this->determineRoute($message, $this->defaultModel); }
|
|
www:
|
|
cockpit:
|
|
Anzahl pro Monat
|
|
"""
|
|
|
|
result = evaluate_requested_behavior_scope(
|
|
report,
|
|
ChangeSummary((
|
|
"app/Services/AIService.php",
|
|
"app/Http/Controllers/ChatController.php",
|
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
|
), 40, 2),
|
|
added,
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert any("do not test resolveDatabaseSource" in error for error in result.errors)
|
|
|
|
|
|
def test_requested_behavior_scope_rejects_helper_only_prefix_tests() -> None:
|
|
report = FailureReport(
|
|
id=7,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Als Tabelle anzeigen",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="cockpit: nutzt die Datenbank; www: nutzt das Modell direkt",
|
|
prompt="Anzahl pro Monat",
|
|
response="Text",
|
|
sql_query="SELECT 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
added = """
|
|
Anzahl pro Monat
|
|
Markdown-Tabelle
|
|
count($rows) > 1
|
|
single_row stays sentence formatted
|
|
resolveRoutePrefix
|
|
['database', 'Frage']
|
|
['direct', 'Frage']
|
|
[null, 'Frage']
|
|
[$explicitRoute, $message] = $this->resolveRoutePrefix($originalMessage);
|
|
$route = $explicitRoute ?? $fallback;
|
|
cockpit:
|
|
www:
|
|
"""
|
|
|
|
result = evaluate_requested_behavior_scope(
|
|
report,
|
|
ChangeSummary((
|
|
"app/Services/AIService.php",
|
|
"app/Http/Controllers/ChatController.php",
|
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
|
"tests/Unit/ChatControllerSourceResolutionTest.php",
|
|
), 40, 2),
|
|
added,
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert any("helper-only tests" in error for error in result.errors)
|
|
|
|
|
|
def test_requested_behavior_scope_rejects_incorrect_table_instruction_assertion() -> None:
|
|
report = FailureReport(
|
|
id=7,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Als Tabelle anzeigen",
|
|
feedback_type="wrong_result",
|
|
expected_behavior="",
|
|
prompt="Anzahl pro Monat",
|
|
response="Text",
|
|
sql_query="SELECT 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
added = """
|
|
Anzahl pro Monat
|
|
Markdown-Tabelle
|
|
count($rows) > 1
|
|
single_row stays sentence formatted
|
|
$this->assertStringContainsString('alle bereitgestellten Zeilen', $systemPrompt);
|
|
"""
|
|
|
|
result = evaluate_requested_behavior_scope(
|
|
report,
|
|
ChangeSummary((
|
|
"app/Services/AIService.php",
|
|
"tests/Feature/AIPrivacyBoundaryTest.php",
|
|
), 20, 2),
|
|
added,
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert any("exact grammatical substring" in error for error in result.errors)
|
|
|
|
|
|
def test_expected_behavior_treats_removed_predicate_as_negative_requirement() -> None:
|
|
report = FailureReport(
|
|
id=5,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong delivery note count",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
|
|
"Einfach nur das aus der SQL entfernen"
|
|
),
|
|
prompt="Delivery notes by month",
|
|
response="0 rows",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
accepted = (
|
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
|
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
|
)
|
|
still_present = (
|
|
"SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1\n"
|
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
|
)
|
|
|
|
assert evaluate_expected_behavior(report, accepted).allowed
|
|
rejected_sql = evaluate_expected_behavior(report, still_present)
|
|
rejected_test = evaluate_expected_behavior(
|
|
report,
|
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2",
|
|
)
|
|
near_match = evaluate_expected_behavior(
|
|
report,
|
|
"SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2\n"
|
|
"$this->assertStringNotContainsString('bStatus_gebucht = 1', $context);",
|
|
)
|
|
|
|
assert not rejected_sql.allowed
|
|
assert any("forbidden fragment" in error for error in rejected_sql.errors)
|
|
assert not rejected_test.allowed
|
|
assert any("negatively assert" in error for error in rejected_test.errors)
|
|
assert not near_match.allowed
|
|
assert any("near-match with a missing qualifier" in error for error in near_match.errors)
|
|
assert any(
|
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);"
|
|
in error
|
|
for error in near_match.errors
|
|
)
|
|
|
|
|
|
def test_removal_prevention_rejects_vacuous_test_and_full_forbidden_rule(tmp_path) -> None:
|
|
report = FailureReport(
|
|
id=6,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Die Abfrage stimmt nicht",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
|
"weil Lieferscheine nicht gebucht werden."
|
|
),
|
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
|
response="0 Ergebnisse",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
source_file = "config/erp_semantics.php"
|
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
|
reference_sql = (
|
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
|
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
|
)
|
|
|
|
def semantics(metric: str = "", sql: str = reference_sql) -> str:
|
|
return f"""<?php
|
|
return [
|
|
'metrics' => [
|
|
{metric}
|
|
],
|
|
'examples' => [[
|
|
'tables' => ['FK_Auftrag'],
|
|
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
|
'sql' => <<<'SQL'
|
|
{sql}
|
|
SQL,
|
|
]],
|
|
];
|
|
"""
|
|
|
|
original_source = semantics()
|
|
bad_metric = """ 'Lieferscheine' => [
|
|
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
|
|
'rules' => [
|
|
'Lieferscheine werden nicht gebucht, daher niemals a.bStatus_gebucht = 1 verwenden.',
|
|
],
|
|
],"""
|
|
original_test = """<?php
|
|
$context = $service->businessContext(['FK_Auftrag'], 'erp');
|
|
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
|
"""
|
|
bad_test = original_test + (
|
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
|
)
|
|
(tmp_path / "config").mkdir()
|
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
|
(tmp_path / source_file).write_text(semantics(bad_metric))
|
|
(tmp_path / test_file).write_text(bad_test)
|
|
workspace = Mock()
|
|
workspace.path = tmp_path
|
|
workspace.head_file.side_effect = lambda filename: {
|
|
source_file: original_source,
|
|
test_file: original_test,
|
|
}.get(filename)
|
|
summary = ChangeSummary((source_file, test_file), 3, 0)
|
|
|
|
result = evaluate_removal_prevention(report, summary, workspace)
|
|
|
|
assert not result.allowed
|
|
assert any("must not embed the complete forbidden predicate" in error for error in result.errors)
|
|
assert any("context-builder test is vacuous" in error for error in result.errors)
|
|
|
|
|
|
def test_removal_prevention_rejects_unrelated_new_semantics(tmp_path) -> None:
|
|
report = FailureReport(
|
|
id=6,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong delivery note count",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
|
"weil Lieferscheine nicht gebucht werden."
|
|
),
|
|
prompt="Delivery notes by month",
|
|
response="0 rows",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
source_file = "config/erp_semantics.php"
|
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
|
original_source = """<?php
|
|
return ['examples' => [[
|
|
'question' => 'Delivery notes by month',
|
|
'sql' => <<<'SQL'
|
|
SELECT * FROM FK_Auftrag a WHERE a.AuftragsKennung = 2;
|
|
SQL,
|
|
]]];
|
|
"""
|
|
current_source = original_source.replace(
|
|
"return [",
|
|
"""return [
|
|
'metrics' => ['Lieferscheine' => [
|
|
'tables' => ['FK_Auftrag'],
|
|
'rules' => [
|
|
'Lieferscheine werden nicht gebucht; bStatus_gebucht niemals verwenden.',
|
|
'Verwende Datum_erfassung oder System_created als Datumsspalte.',
|
|
],
|
|
]],""",
|
|
)
|
|
original_test = "<?php\n"
|
|
current_test = original_test + (
|
|
"$this->assertStringContainsString('Lieferscheine werden nicht gebucht; "
|
|
"bStatus_gebucht niemals verwenden.', $context);\n"
|
|
"$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
|
)
|
|
(tmp_path / "config").mkdir()
|
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
|
(tmp_path / source_file).write_text(current_source)
|
|
(tmp_path / test_file).write_text(current_test)
|
|
workspace = Mock()
|
|
workspace.path = tmp_path
|
|
workspace.head_file.side_effect = lambda filename: {
|
|
source_file: original_source,
|
|
test_file: original_test,
|
|
}.get(filename)
|
|
|
|
result = evaluate_removal_prevention(
|
|
report,
|
|
ChangeSummary((source_file, test_file), 7, 0),
|
|
workspace,
|
|
)
|
|
|
|
assert not result.allowed
|
|
assert any("unrelated business guidance" in error for error in result.errors)
|
|
assert any("Datum_erfassung" in error for error in result.errors)
|
|
|
|
|
|
def test_removal_prevention_accepts_exercised_rule_and_protects_reference(tmp_path) -> None:
|
|
report = FailureReport(
|
|
id=6,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Die Abfrage stimmt nicht",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss "a.bStatus_gebucht = 1" aus dem SQL entfernt werden, '
|
|
"weil Lieferscheine nicht gebucht werden."
|
|
),
|
|
prompt="Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024",
|
|
response="0 Ergebnisse",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
source_file = "config/erp_semantics.php"
|
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
|
reference_sql = (
|
|
"SELECT COUNT(*) FROM FK_Auftrag a WHERE a.AuftragsKennung = 2 "
|
|
"AND COALESCE(a.bStatus_storniert, 0) = 0;"
|
|
)
|
|
|
|
def semantics(metric: str = "", sql: str = reference_sql) -> str:
|
|
return f"""<?php
|
|
return [
|
|
'metrics' => [
|
|
{metric}
|
|
],
|
|
'examples' => [[
|
|
'tables' => ['FK_Auftrag'],
|
|
'question' => 'Gib mir eine Anzahl von Lieferscheinen pro Monat in 2024',
|
|
'sql' => <<<'SQL'
|
|
{sql}
|
|
SQL,
|
|
]],
|
|
];
|
|
"""
|
|
|
|
rule = (
|
|
"Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine "
|
|
"niemals verwenden."
|
|
)
|
|
metric = f""" 'Lieferscheine' => [
|
|
'tables' => ['FK_Auftrag', 'FK_Nummerkreis'],
|
|
'rules' => [
|
|
'{rule}',
|
|
],
|
|
],"""
|
|
original_source = semantics()
|
|
original_test = """<?php
|
|
$context = $service->businessContext(['FK_Auftrag'], 'erp');
|
|
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
|
"""
|
|
current_test = """<?php
|
|
$context = $service->businessContext(['FK_Auftrag', 'FK_Nummerkreis'], 'erp');
|
|
$this->assertStringContainsString('AuftragsKennung = 2', $context);
|
|
$this->assertStringContainsString('Lieferscheine werden nicht gebucht; bStatus_gebucht fuer Lieferscheine niemals verwenden.', $context);
|
|
$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);
|
|
"""
|
|
(tmp_path / "config").mkdir()
|
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
|
(tmp_path / source_file).write_text(semantics(metric))
|
|
(tmp_path / test_file).write_text(current_test)
|
|
workspace = Mock()
|
|
workspace.path = tmp_path
|
|
workspace.head_file.side_effect = lambda filename: {
|
|
source_file: original_source,
|
|
test_file: original_test,
|
|
}.get(filename)
|
|
summary = ChangeSummary((source_file, test_file), 4, 1)
|
|
|
|
accepted = evaluate_removal_prevention(report, summary, workspace)
|
|
|
|
assert accepted.allowed
|
|
|
|
(tmp_path / source_file).write_text(semantics(
|
|
metric,
|
|
reference_sql.replace("AuftragsKennung = 2", "AuftragsKennung = 4"),
|
|
))
|
|
regressed = evaluate_removal_prevention(report, summary, workspace)
|
|
|
|
assert not regressed.allowed
|
|
assert any("protected baseline" in error for error in regressed.errors)
|
|
|
|
|
|
def test_removal_only_scope_allows_only_exact_predicate_removal_and_added_test(tmp_path) -> None:
|
|
report = FailureReport(
|
|
id=5,
|
|
chat_message_id=1,
|
|
source="user_feedback",
|
|
category="incorrect_answer",
|
|
user_feedback="Wrong delivery note count",
|
|
feedback_type="wrong_result",
|
|
expected_behavior=(
|
|
'Es muss "a.bStatus_gebucht = 1" entfernt werden. '
|
|
"Einfach nur das aus der SQL entfernen"
|
|
),
|
|
prompt="Delivery notes by month in 2024",
|
|
response="0 rows",
|
|
sql_query="SELECT * FROM FK_Auftrag a WHERE a.bStatus_gebucht = 1",
|
|
error_message=None,
|
|
context={"database_type": "erp"},
|
|
)
|
|
original_sql = (
|
|
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
|
|
"WHERE a.AuftragsKennung = 2 AND a.bStatus_gebucht = 1 "
|
|
"AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
|
|
)
|
|
repaired_sql = (
|
|
"SELECT DATE_FORMAT(a.System_created, '%Y-%m') FROM FK_Auftrag a "
|
|
"WHERE a.AuftragsKennung = 2 AND COALESCE(a.obsolet, 0) = 0 GROUP BY 1;"
|
|
)
|
|
|
|
def semantics(sql: str) -> str:
|
|
return f"""<?php
|
|
return [
|
|
'examples' => [[
|
|
'question' => 'Delivery notes by month in 2024',
|
|
'sql' => <<<'SQL'
|
|
{sql}
|
|
SQL,
|
|
]],
|
|
];
|
|
"""
|
|
|
|
source_file = "config/erp_semantics.php"
|
|
test_file = "tests/Unit/SQLGeneratorBusinessContextTest.php"
|
|
(tmp_path / "config").mkdir()
|
|
(tmp_path / "tests/Unit").mkdir(parents=True)
|
|
(tmp_path / source_file).write_text(semantics(repaired_sql))
|
|
previous_test = "<?php\n$this->assertStringContainsString('System_created', $context);\n"
|
|
(tmp_path / test_file).write_text(
|
|
previous_test
|
|
+ "$this->assertStringNotContainsString('a.bStatus_gebucht = 1', $context);\n"
|
|
)
|
|
workspace = Mock()
|
|
workspace.path = tmp_path
|
|
workspace.head_file.side_effect = lambda filename: {
|
|
source_file: semantics(original_sql),
|
|
test_file: previous_test,
|
|
}.get(filename)
|
|
summary = ChangeSummary((source_file, test_file), 2, 1)
|
|
|
|
result = evaluate_removal_scope(report, summary, workspace)
|
|
|
|
assert result.allowed
|
|
|
|
(tmp_path / source_file).write_text(semantics(repaired_sql.replace(
|
|
"System_created",
|
|
"Datum_erfassung",
|
|
)))
|
|
regression = evaluate_removal_scope(report, summary, workspace)
|
|
|
|
assert not regression.allowed
|
|
assert any("outside the explicitly forbidden" in error for error in regression.errors)
|