test: Lit-Mapping validation — Dict vs Control Library comparison
8 test cases with deliberately wrong legal basis assignments: - Cookie tracking on lit. f (should be lit. a) - Analytics on lit. b (should be lit. a) - Newsletter on lit. f (should be lit. a) - Klarna without Art. 22 - Session recording on lit. f - 2 correct cases (should NOT trigger findings) Runs both hardcoded dict AND Control Library query, compares results. If Control Library passes all → dict can be removed. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Benjamin Admin
@@ -0,0 +1,316 @@
|
||||
"""
|
||||
Lit-Mapping Validation Test — verifies that BOTH the hardcoded dict AND
|
||||
the Control Library detect the same legal basis errors.
|
||||
|
||||
If both produce the same results, we can safely delete the dict.
|
||||
|
||||
Test cases use deliberately WRONG legal basis assignments that are
|
||||
common mistakes on real websites.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
# Add parent to path for imports
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
# TEST CASES — Deliberately wrong DSE text blocks
|
||||
# ═══════════════════════════════════════════════════════════════
|
||||
|
||||
TEST_CASES = [
|
||||
{
|
||||
"id": "cookie_tracking_wrong_litf",
|
||||
"description": "Cookie-Tracking auf lit. f statt lit. a",
|
||||
"dse_text": (
|
||||
"Wir setzen Cookies und aehnliche Tracking-Technologien ein, "
|
||||
"um die Nutzung unserer Website zu analysieren. Die Verarbeitung "
|
||||
"erfolgt auf Grundlage unseres berechtigten Interesses gemaess "
|
||||
"Art. 6 Abs. 1 lit. f DSGVO an der Optimierung unseres Angebots."
|
||||
),
|
||||
"expected_finding": True,
|
||||
"expected_purpose": "cookie_tracking",
|
||||
"correct_basis": "lit. a (Einwilligung)",
|
||||
"wrong_basis": "lit. f",
|
||||
},
|
||||
{
|
||||
"id": "analytics_wrong_litb",
|
||||
"description": "Google Analytics auf lit. b (Vertragserfuellung) statt lit. a",
|
||||
"dse_text": (
|
||||
"Wir nutzen Google Analytics zur Webanalyse. Die Datenverarbeitung "
|
||||
"erfolgt auf Basis der Vertragserfuellung gemaess Art. 6 Abs. 1 lit. b DSGVO, "
|
||||
"da die Analyse fuer die Erbringung unserer Dienste erforderlich ist."
|
||||
),
|
||||
"expected_finding": True,
|
||||
"expected_purpose": "web_analytics",
|
||||
"correct_basis": "lit. a (Einwilligung)",
|
||||
"wrong_basis": "lit. b",
|
||||
},
|
||||
{
|
||||
"id": "newsletter_wrong_litf",
|
||||
"description": "Newsletter auf lit. f statt lit. a",
|
||||
"dse_text": (
|
||||
"Wir versenden regelmaessig Newsletter mit Informationen zu unseren Produkten. "
|
||||
"Die Verarbeitung Ihrer E-Mail-Adresse erfolgt auf Grundlage unseres "
|
||||
"berechtigten Interesses gemaess Art. 6 Abs. 1 lit. f DSGVO an der "
|
||||
"Direktwerbung fuer eigene aehnliche Produkte."
|
||||
),
|
||||
"expected_finding": True,
|
||||
"expected_purpose": "marketing_email",
|
||||
"correct_basis": "lit. a (Einwilligung)",
|
||||
"wrong_basis": "lit. f",
|
||||
},
|
||||
{
|
||||
"id": "remarketing_wrong_litf",
|
||||
"description": "Remarketing/Retargeting auf lit. f statt lit. a",
|
||||
"dse_text": (
|
||||
"Wir setzen Remarketing-Technologien ein, um Ihnen auf anderen Websites "
|
||||
"personalisierte Werbung anzuzeigen. Die Verarbeitung basiert auf unserem "
|
||||
"berechtigten Interesse an effektiver Werbung (Art. 6 Abs. 1 lit. f DSGVO)."
|
||||
),
|
||||
"expected_finding": True,
|
||||
"expected_purpose": "remarketing",
|
||||
"correct_basis": "lit. a (Einwilligung)",
|
||||
"wrong_basis": "lit. f",
|
||||
},
|
||||
{
|
||||
"id": "klarna_missing_art22",
|
||||
"description": "Klarna Bonitaetspruefung ohne Art. 22 Hinweis",
|
||||
"dse_text": (
|
||||
"Bei Auswahl der Zahlungsart Rechnung ueber Klarna wird eine "
|
||||
"Bonitaetspruefung durchgefuehrt. Klarna AB, Stockholm, Schweden, "
|
||||
"uebermittelt Ihre Daten an Auskunfteien. Rechtsgrundlage ist "
|
||||
"Art. 6 Abs. 1 lit. b DSGVO (Vertragserfuellung)."
|
||||
),
|
||||
"expected_finding": True,
|
||||
"expected_purpose": "credit_check",
|
||||
"correct_basis": "lit. b/f + Art. 22 DSGVO Hinweis",
|
||||
"wrong_basis": "(fehlt)",
|
||||
},
|
||||
{
|
||||
"id": "session_recording_wrong_litf",
|
||||
"description": "Session Recording (Hotjar) auf lit. f statt lit. a",
|
||||
"dse_text": (
|
||||
"Wir nutzen Hotjar zur Analyse des Nutzerverhaltens mittels Session Recording "
|
||||
"und Heatmaps. Die Aufzeichnung der Nutzersitzungen erfolgt auf Grundlage "
|
||||
"unseres berechtigten Interesses (Art. 6 Abs. 1 lit. f DSGVO)."
|
||||
),
|
||||
"expected_finding": True,
|
||||
"expected_purpose": "session_recording",
|
||||
"correct_basis": "lit. a (Einwilligung)",
|
||||
"wrong_basis": "lit. f",
|
||||
},
|
||||
{
|
||||
"id": "payment_correct_litb",
|
||||
"description": "Zahlung korrekt auf lit. b — sollte KEIN Finding sein",
|
||||
"dse_text": (
|
||||
"Die Verarbeitung Ihrer Zahlungsdaten durch unseren Zahlungsdienstleister "
|
||||
"Stripe erfolgt auf Grundlage der Vertragserfuellung gemaess "
|
||||
"Art. 6 Abs. 1 lit. b DSGVO."
|
||||
),
|
||||
"expected_finding": False,
|
||||
"expected_purpose": None,
|
||||
"correct_basis": "lit. b (Vertragserfuellung)",
|
||||
"wrong_basis": None,
|
||||
},
|
||||
{
|
||||
"id": "analytics_correct_lita",
|
||||
"description": "Analytics korrekt auf lit. a — sollte KEIN Finding sein",
|
||||
"dse_text": (
|
||||
"Wir setzen Google Analytics nur mit Ihrer ausdruecklichen Einwilligung "
|
||||
"gemaess Art. 6 Abs. 1 lit. a DSGVO ein. Sie koennen Ihre Einwilligung "
|
||||
"jederzeit widerrufen."
|
||||
),
|
||||
"expected_finding": False,
|
||||
"expected_purpose": None,
|
||||
"correct_basis": "lit. a (Einwilligung)",
|
||||
"wrong_basis": None,
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
def test_hardcoded_dict():
|
||||
"""Test the hardcoded CORRECT_BASIS dict against test cases."""
|
||||
from compliance.services.legal_basis_validator import validate_legal_bases
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST 1: Hartkodiertes Dict (legal_basis_validator.py)")
|
||||
print("=" * 70)
|
||||
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
for tc in TEST_CASES:
|
||||
findings = validate_legal_bases(tc["dse_text"])
|
||||
has_finding = len(findings) > 0
|
||||
|
||||
if has_finding == tc["expected_finding"]:
|
||||
status = "PASS"
|
||||
passed += 1
|
||||
else:
|
||||
status = "FAIL"
|
||||
failed += 1
|
||||
|
||||
print(f" [{status}] {tc['id']}: {tc['description']}")
|
||||
if has_finding:
|
||||
for f in findings:
|
||||
print(f" → {f.text[:80]}")
|
||||
elif tc["expected_finding"]:
|
||||
print(f" → ERWARTET: Finding fuer {tc['expected_purpose']}, aber KEINS gefunden")
|
||||
|
||||
print(f"\n Ergebnis: {passed} bestanden, {failed} fehlgeschlagen\n")
|
||||
return passed, failed
|
||||
|
||||
|
||||
def test_control_library():
|
||||
"""Test the Control Library against the same test cases.
|
||||
|
||||
Queries canonical_controls for lit-mapping controls and checks
|
||||
if they would detect the same errors.
|
||||
"""
|
||||
try:
|
||||
import asyncpg
|
||||
except ImportError:
|
||||
print("\n SKIP: asyncpg nicht installiert — Control Library Test uebersprungen")
|
||||
return 0, 0
|
||||
|
||||
db_url = os.environ.get(
|
||||
"COMPLIANCE_DATABASE_URL",
|
||||
os.environ.get("DATABASE_URL", ""),
|
||||
)
|
||||
if not db_url:
|
||||
print("\n SKIP: Keine DATABASE_URL — Control Library Test uebersprungen")
|
||||
return 0, 0
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("TEST 2: Control Library (canonical_controls)")
|
||||
print("=" * 70)
|
||||
|
||||
async def _run():
|
||||
pool = await asyncpg.create_pool(db_url, min_size=1, max_size=2)
|
||||
passed = 0
|
||||
failed = 0
|
||||
|
||||
try:
|
||||
async with pool.acquire() as conn:
|
||||
# Fetch lit-mapping relevant controls
|
||||
controls = await conn.fetch("""
|
||||
SELECT control_id, title, objective, requirements
|
||||
FROM compliance.canonical_controls
|
||||
WHERE (
|
||||
title ILIKE '%einwilligung%tracking%'
|
||||
OR title ILIKE '%rechtsgrundlage%cookie%'
|
||||
OR title ILIKE '%consent%cookie%'
|
||||
OR title ILIKE '%einwilligung%cookie%'
|
||||
OR title ILIKE '%art. 22%'
|
||||
OR title ILIKE '%automatisierte%entscheidung%'
|
||||
OR requirements ILIKE '%lit. a%tracking%'
|
||||
OR requirements ILIKE '%einwilligung%analytics%'
|
||||
)
|
||||
AND release_state = 'published'
|
||||
LIMIT 50
|
||||
""")
|
||||
|
||||
print(f" Gefundene Lit-Mapping Controls: {len(controls)}")
|
||||
for c in controls[:10]:
|
||||
print(f" [{c['control_id']}] {c['title'][:60]}")
|
||||
|
||||
if not controls:
|
||||
print(" WARNUNG: Keine Lit-Mapping Controls in der DB!")
|
||||
return 0, 0
|
||||
|
||||
# For each test case, check if a control would catch it
|
||||
for tc in TEST_CASES:
|
||||
text_lower = tc["dse_text"].lower()
|
||||
matched_control = None
|
||||
|
||||
for c in controls:
|
||||
title_lower = (c["title"] or "").lower()
|
||||
req_lower = (c["requirements"] or "").lower()
|
||||
obj_lower = (c["objective"] or "").lower()
|
||||
|
||||
# Check if this control is relevant for this test case
|
||||
relevant = False
|
||||
if tc["expected_purpose"] == "cookie_tracking":
|
||||
relevant = "cookie" in title_lower or "tracking" in title_lower
|
||||
elif tc["expected_purpose"] == "web_analytics":
|
||||
relevant = "analytics" in title_lower or "tracking" in title_lower
|
||||
elif tc["expected_purpose"] == "marketing_email":
|
||||
relevant = "newsletter" in title_lower or "marketing" in title_lower
|
||||
elif tc["expected_purpose"] == "remarketing":
|
||||
relevant = "remarketing" in title_lower or "retargeting" in title_lower
|
||||
elif tc["expected_purpose"] == "credit_check":
|
||||
relevant = "art. 22" in title_lower or "bonitaet" in title_lower
|
||||
elif tc["expected_purpose"] == "session_recording":
|
||||
relevant = "recording" in title_lower or "heatmap" in title_lower
|
||||
|
||||
if relevant:
|
||||
# Check if the control requires consent (lit. a)
|
||||
requires_consent = (
|
||||
"einwilligung" in req_lower
|
||||
or "consent" in req_lower
|
||||
or "lit. a" in req_lower
|
||||
)
|
||||
if requires_consent and tc["expected_finding"]:
|
||||
matched_control = c
|
||||
break
|
||||
|
||||
has_match = matched_control is not None
|
||||
# For negative test cases (no finding expected), no match = correct
|
||||
if not tc["expected_finding"]:
|
||||
correct = not has_match
|
||||
else:
|
||||
correct = has_match
|
||||
|
||||
if correct:
|
||||
status = "PASS"
|
||||
passed += 1
|
||||
else:
|
||||
status = "FAIL"
|
||||
failed += 1
|
||||
|
||||
print(f" [{status}] {tc['id']}: {tc['description']}")
|
||||
if matched_control:
|
||||
print(f" → Control: [{matched_control['control_id']}] {matched_control['title'][:60]}")
|
||||
elif tc["expected_finding"]:
|
||||
print(f" → KEIN passender Control gefunden!")
|
||||
|
||||
finally:
|
||||
await pool.close()
|
||||
|
||||
print(f"\n Ergebnis: {passed} bestanden, {failed} fehlgeschlagen\n")
|
||||
return passed, failed
|
||||
|
||||
return asyncio.run(_run())
|
||||
|
||||
|
||||
def test_comparison():
|
||||
"""Compare results: Dict vs. Control Library."""
|
||||
print("\n" + "=" * 70)
|
||||
print("VERGLEICH: Dict vs. Control Library")
|
||||
print("=" * 70)
|
||||
|
||||
dict_passed, dict_failed = test_hardcoded_dict()
|
||||
ctrl_passed, ctrl_failed = test_control_library()
|
||||
|
||||
print("\n" + "=" * 70)
|
||||
print("ZUSAMMENFASSUNG")
|
||||
print("=" * 70)
|
||||
print(f" Dict: {dict_passed}/{dict_passed + dict_failed} bestanden")
|
||||
print(f" Control Library: {ctrl_passed}/{ctrl_passed + ctrl_failed} bestanden")
|
||||
|
||||
if ctrl_passed >= dict_passed and ctrl_failed == 0:
|
||||
print("\n ✓ Control Library deckt alle Faelle ab → Dict kann entfernt werden")
|
||||
elif ctrl_passed > 0:
|
||||
print("\n ⚠ Control Library deckt teilweise ab → Dict als Fallback behalten")
|
||||
else:
|
||||
print("\n ✗ Control Library deckt nichts ab → Dict wird noch gebraucht")
|
||||
|
||||
print("=" * 70)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
test_comparison()
|
||||
Reference in New Issue
Block a user