test(dse): adopt canonical v3 tests + criteria/GT/validation

Replace the reconstructed test_dse_agent.py with the canonical version and add
the companion unit tests (classification_gate, embedding_recall) covering the
recovered v3 modules. Include the curated DSE criteria backup + changelog
(legal-note rationale per control), the v1 validation writeup, and the
multi-company DSE ground-truth fulltexts (elli/eto/mercedes/safetykon) used
for threshold calibration.

18 DSE tests green offline (DB/embedding/LLM stubbed).

dev-only, no deploy.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>

backend-compliance/compliance/tests/test_dse_agent.py

@@ -1,96 +1,153 @@
-"""DSEAgent v3 (4-Layer: Regex-Boost / Keyword / BGE-M3-Recall / Semantic).
+"""DSE-Agent v3 — DB-Controls (doc_check_controls) via run_v3_pipeline +
+kuratierter Art-13-Regex-Boost (Layer 0). Volle Parität zu impressum/cookie.
 
-DB (_load_controls), Embedding-Service und LLM sind offline gestubbt → die Tests
-sind deterministisch und brauchen kein Netzwerk. Die reinen Schichten
-(compute_regex_boosts, embedding_recall-Reachability) werden direkt geprüft, die
-Result→Finding-Konvertierung über einen gestubbten run_v3_pipeline.
+Die Tests prüfen die deterministischen Bausteine (regex_boost/mcs) ohne DB und
+den Agent-Pfad mit gemocktem run_v3_pipeline (CI hat keine DB).
 """
+
+from __future__ import annotations
+
 import asyncio
 
 import compliance.services.specialist_agents.dse.agent as dse_agent
 from compliance.services.specialist_agents import REGISTRY, AgentInput
-from compliance.services.specialist_agents.dse._embedding_recall import (
-    embedding_recall,
-)
+from compliance.services.specialist_agents.dse.mcs import MCS, MC_IDS
 from compliance.services.specialist_agents.dse.regex_boost import (
+    boost_matches_db_mc,
     compute_regex_boosts,
+    criteria_on_topic,
 )
 
-_TELEMETRY = {
-    "layer_0_field_hits": 0, "layer_0_field_ids": [],
-    "layer_1_pass": 0, "embedding_passes": 0, "total_mcs": 1,
-    "sector_dropped": 0, "offtopic_dropped": 0,
-    "gate_excluded": 0, "organizational_checklist": [],
-}
+_DSE_SAMPLE = (
+    "Datenschutzerklaerung. Verantwortlich im Sinne der DSGVO ist die Muster "
+    "GmbH, Musterstrasse 1, 12345 Berlin. E-Mail: info@muster.de. "
+    "Datenschutzbeauftragter: dsb@muster.de. Zwecke der Verarbeitung und "
+    "Rechtsgrundlage Art. 6 Abs. 1 lit. f berechtigtes Interesse. Empfaenger "
+    "Ihrer Daten sind Auftragsverarbeiter. Speicherdauer der Daten richtet "
+    "sich nach Aufbewahrungsfristen. Sie haben das Recht auf Auskunft, das "
+    "Recht auf Berichtigung, das Recht auf Loeschung sowie ein "
+    "Widerspruchsrecht. Beschwerde bei der Aufsichtsbehoerde moeglich. Stand: "
+    "Januar 2026. ") * 3
 
 
-def _pipeline_stub(results):
-    async def _stub(text, scope):
-        return results, dict(_TELEMETRY, total_mcs=len(results))
-    return _stub
+# ── Registrierung ────────────────────────────────────────────────────────
+def test_dse_agent_registered():
+    agent = REGISTRY.get("dse")
+    assert agent is not None
+    assert agent.agent_version == "3.0"
+    assert agent.doc_type == "dse"
 
 
-def _evaluate(text, context=None):
-    return asyncio.run(dse_agent.DSEAgent().evaluate(
+def test_owned_mc_ids_match_checklist():
+    # owned_mc_ids = die Boost-Pattern-IDs (aus ART13_CHECKLIST gehoben).
+    assert MC_IDS == tuple(m.mc_id for m in MCS)
+    assert len(MC_IDS) >= 10  # mind. die 10 L1-Pflichtfelder + L2
+
+
+# ── Layer-0 Regex-Boost (deterministisch, ohne DB) ───────────────────────
+def test_regex_boost_detects_core_fields():
+    boosts = compute_regex_boosts(_DSE_SAMPLE)
+    # Die zentralen Art-13-Felder müssen erkannt werden.
+    for field in ("controller", "legal_basis", "rights", "complaint",
+                  "retention", "dse_version_date"):
+        assert field in boosts, f"{field} nicht erkannt: {sorted(boosts)}"
+
+
+def test_regex_boost_empty_on_short_text():
+    assert compute_regex_boosts("zu kurz") == set()
+
+
+def test_criteria_on_topic_accepts_dse_rejects_foreign():
+    dse_crit = ["Rechtsgrundlage gemäß Art. 6 DSGVO benannt",
+                "Speicherdauer und Löschfrist angegeben"]
+    assert criteria_on_topic(dse_crit) is True
+    foreign = ["Bestellbestätigung wird per E-Mail versendet",
+               "Versandkosten werden im Warenkorb angezeigt"]
+    assert criteria_on_topic(foreign) is False
+    # leere Kriterien → konservativ on-topic behalten
+    assert criteria_on_topic([]) is True
+
+
+def test_boost_matches_db_mc_third_country():
+    boosts = {"third_country", "controller"}
+    crit = ["Standardvertragsklauseln für Drittland benannt",
+            "Geeignete Garantien bei Übermittlung in ein Drittland"]
+    assert boost_matches_db_mc(boosts, crit) == "third_country"
+    # ohne passende Boosts → None
+    assert boost_matches_db_mc(set(), crit) is None
+
+
+# ── Agent-Pfad mit gemocktem run_v3_pipeline ─────────────────────────────
+def _mock_v3(results, telemetry=None):
+    async def _fake(text, scope, db_url="", skip_embedding=False):
+        return results, (telemetry or {
+            "total_mcs": len(results), "layer_0_field_hits": 0,
+            "layer_0_field_ids": [], "layer_0_boost_overrides": 0,
+            "sector_dropped": 0, "offtopic_dropped": 0})
+    return _fake
+
+
+def _run(text, context=None):
+    return asyncio.run(REGISTRY.get("dse").evaluate(
         AgentInput(doc_type="dse", text=text, context=context or {})))
 
 
-def test_dse_agent_registered_is_v3():
-    agent = REGISTRY.get("dse")
-    assert agent is not None and agent.agent_version == "3.0"
-
-
 def test_dse_short_text_skips():
-    out = _evaluate("zu kurz")
+    out = _run("zu kurz")
     assert out.confidence == 0.0
-    assert all(c.status == "skipped" for c in out.mc_coverage)
+    assert out.mc_coverage and all(
+        c.status == "skipped" for c in out.mc_coverage)
 
 
-def test_regex_boost_detects_core_fields():
-    text = ("Verantwortlicher im Sinne der DSGVO ist die Muster GmbH. "
-            "Rechtsgrundlage ist Art. 6. Speicherdauer der Daten. Beschwerde "
-            "bei der Aufsichtsbehoerde. ") * 2
-    hits = compute_regex_boosts(text, set())
-    assert {"controller", "legal_basis", "retention", "complaint"} & hits
+def test_dse_findings_from_failed_db_mc(monkeypatch):
+    results = [{
+        "control_id": "DATA-525-A17", "passed": False, "severity": "HIGH",
+        "label": "Berechtigte Interessen ausweisen", "regulation": None,
+        "article": None, "_pass_criteria": ["berechtigtes interesse benannt"],
+        "matched_text": "", "source": "keyword_match",
+    }, {
+        "control_id": "AUTH-2051-A11", "passed": True, "severity": "LOW",
+        "label": "Prägnante Form", "regulation": None, "article": None,
+        "_pass_criteria": [], "matched_text": "ok",
+    }]
+    monkeypatch.setattr(dse_agent, "run_v3_pipeline", _mock_v3(results))
+    out = _run(_DSE_SAMPLE, context={"skip_llm": True})
+    fids = {f.field_id for f in out.findings}
+    assert "DATA-525-A17" in fids       # failed → Finding
+    assert "AUTH-2051-A11" not in fids  # passed → kein Finding
+    f = next(f for f in out.findings if f.field_id == "DATA-525-A17")
+    assert f.severity == "HIGH"
+    assert f.norm == "DSGVO Art. 13/14"  # NULL-regulation → Fallback-Norm
+    assert len(f.action) < 410
 
 
-def test_embedding_recall_offline_returns_empty():
-    # Kein Embedding-Service (Unit) -> Reachability-Guard -> leeres Set, kein Hang.
-    got = asyncio.run(embedding_recall("x" * 200, ["DSE-X-1"]))
-    assert got == set()
-
-
-def test_evaluate_builds_finding_from_failed_db_mc(monkeypatch):
-    monkeypatch.setattr(dse_agent, "run_v3_pipeline", _pipeline_stub([{
-        "control_id": "DATA-RETENTION-1", "passed": False, "severity": "MEDIUM",
-        "label": "Speicherdauer der Daten", "regulation": "DSGVO", "article": "13",
-        "source": "keyword_match",
-    }]))
-    out = _evaluate("Datenschutzerklaerung " + "x" * 200, context={"skip_llm": True})
-    f = next((f for f in out.findings if f.field_id == "DATA-RETENTION-1"), None)
-    assert f is not None and f.severity == "MEDIUM"
-    assert f.action and len(f.action) <= 400
-
-
-def test_evaluate_passed_db_mc_no_finding(monkeypatch):
-    monkeypatch.setattr(dse_agent, "run_v3_pipeline", _pipeline_stub([{
-        "control_id": "PURPOSE-1", "passed": True, "severity": "MEDIUM",
-        "label": "Zwecke", "matched_text": "Zwecke der Verarbeitung",
-    }]))
-    out = _evaluate("Datenschutzerklaerung " + "x" * 200, context={"skip_llm": True})
-    assert "PURPOSE-1" not in [f.field_id for f in out.findings]
-    assert any(c.mc_id == "PURPOSE-1" and c.status == "ok" for c in out.mc_coverage)
-
-
-def test_evaluate_third_country_high_on_documented_transfer(monkeypatch):
-    monkeypatch.setattr(dse_agent, "run_v3_pipeline", _pipeline_stub([{
-        "control_id": "TRANSFER-1", "passed": False, "severity": "MEDIUM",
-        "label": "Drittlanduebermittlung", "regulation": "DSGVO", "article": "13",
-    }]))
-    out = _evaluate(
-        "Datenschutzerklaerung " + "x" * 200,
-        context={"skip_llm": True,
-                 "scan_context": {"third_country_transfer": "yes"}})
-    f = next((f for f in out.findings if f.field_id == "TRANSFER-1"), None)
-    assert f is not None and f.severity == "HIGH"
+def test_dse_third_country_override_to_high(monkeypatch):
+    # MEDIUM-Drittland-MC → HIGH bei dokumentiertem Transfer (scan_context).
+    results = [{
+        "control_id": "DATA-900-A01", "passed": False, "severity": "MEDIUM",
+        "label": "Drittlandtransfer Schutzgarantien benennen",
+        "regulation": None, "article": None,
+        "_pass_criteria": ["standardvertragsklauseln", "drittland garantien"],
+        "matched_text": "", "source": "keyword_match",
+    }]
+    monkeypatch.setattr(dse_agent, "run_v3_pipeline", _mock_v3(results))
+    out = _run(_DSE_SAMPLE, context={
+        "skip_llm": True,
+        "scan_context": {"third_country_transfer": "yes"}})
+    f = next(f for f in out.findings if f.field_id == "DATA-900-A01")
+    assert f.severity == "HIGH"
     assert f.severity_reason == "db_mc_failed_third_country_transfer"
+
+
+def test_dse_no_transfer_keeps_medium(monkeypatch):
+    results = [{
+        "control_id": "DATA-900-A01", "passed": False, "severity": "MEDIUM",
+        "label": "Drittlandtransfer Schutzgarantien benennen",
+        "regulation": None, "article": None,
+        "_pass_criteria": ["standardvertragsklauseln", "drittland garantien"],
+        "matched_text": "", "source": "keyword_match",
+    }]
+    monkeypatch.setattr(dse_agent, "run_v3_pipeline", _mock_v3(results))
+    out = _run(_DSE_SAMPLE, context={"skip_llm": True})
+    f = next(f for f in out.findings if f.field_id == "DATA-900-A01")
+    assert f.severity == "MEDIUM"