feat(compliance-check): unlock all 1874 MCs + close gap-table items

User: 'wir haben 1800 MCs erstellt um sie zu 10% zu nutzen — das ist
Schwachsinn'. Fixed all 6 gaps from the audit.

#1 max_controls=0 (was 20):
- agent_compliance_check_routes _check_single: passes max_controls=0 to
  check_document_with_controls -> ALL MCs evaluated per doc_type.
- 8 doc_types now use 1874 MCs instead of 160 (10x coverage).
- Regex matching is cheap (<1s per doc); LLM-enrich cap of 10 stays.

#2 LLM-verify fixed:
- llm_verify.py was getting 0/N parsed. Causes: qwen3 thinking-mode
  wrapped output in <think>...</think>, /api/generate doesn't enforce
  JSON, prompt didn't handle code-fence wrappers.
- Now uses /api/chat with format='json' (forces valid JSON).
- _parse_batch_response strips <think> tags, accepts {results:[...]}
  AND bare [...], adds richer regex-fallback parse, logs raw head on
  total parse failure for diagnosis.

#3 Loeschkonzept checklist (new):
- doc_checks/loeschkonzept_checks.py — 9 L1 + 7 L2 checks per DIN 66398
  + Art. 5(1)(e)/17/32 DSGVO: scope+responsibility, data categories,
  retention periods, legal basis refs (HGB/AO/BGB), deletion trigger,
  deletion process+technical+systems, deletion proof, exceptions +
  Art. 18 lock, review cycle, DSGVO references.
- runner.py registered for loeschkonzept/loeschung/loeschfristen.

#4 regulation backfill script:
- backend-compliance/scripts/backfill_mc_regulation.py — regex-detects
  DSGVO/TDDDG/TMG/BGB/HGB/AO/MStV/UWG/VSBG/PAngV/GwG/BDSG/EU-VO
  references in MC title+question+pass_criteria, UPDATEs regulation +
  article fields.
- Idempotent (only NULL rows), --dry-run flag, batched 200/UPDATE.
- Run inside container: docker exec bp-compliance-backend python3 \
    /app/scripts/backfill_mc_regulation.py

#5 MC alias-fallback:
- rag_document_checker._MC_ALIAS_FALLBACK maps doc_types without own
  MCs to a related set: nutzungsbedingungen->agb, social_media->dse,
  sub_processor/scc/tom_annex->avv, loeschfristen->loeschkonzept,
  eu_institution/dsb->dse.
- _load_controls retries with the alias when the primary query
  returns 0 rows.
- 14 additional doc_types now get MC coverage transparently.

#6 cross-domain auto-discovery:
- _autodiscover_missing builds a crawl plan: primary submitted base
  + up to 2 related domains sharing the owner SLD (e.g. BMW Group:
  bmw.de + bmwgroup.com + bmwgroup.jobs).
- Detection: regex over submitted texts for https?://...<owner>...
  hostnames distinct from the primary base.
- Each crawled base contributes documents + cmp_payloads to the
  discovery pool.

Net effect for BMW: 1874 MCs evaluated (90 from cookie alone, was
20), Loeschkonzept Pflichtangaben benoten-bar, LLM overturns false
regex FAILs, Joint-Controller policies on bmwgroup.jobs (Social
Media) jetzt entdeckbar. Same wins will apply to CRA-Compliance check.

backend-compliance/compliance/api/agent_compliance_check_routes.py

@@ -9,6 +9,7 @@ GET  /compliance/agent/compliance-check/{check_id} — poll status
 import asyncio
 import logging
 import os
+import re
 import uuid as _uuid
 from dataclasses import asdict
 from datetime import datetime, timezone
@@ -600,32 +601,65 @@ async def _autodiscover_missing(
             })
         return
 
-    base = max(bases, key=bases.get) + "/"
+    # Build crawl plan: primary base + any related domains mentioned in
+    # the submitted texts that share the owner's SLD. Example: BMW Group
+    # text mentions bmwgroup.com and bmwgroup.jobs in addition to bmw.de.
+    primary_base = max(bases, key=bases.get) + "/"
+    crawl_bases: list[str] = [primary_base]
+    primary_netloc = urlparse(primary_base).netloc.lower().lstrip("www.")
+    owner_token = primary_netloc.split(".")[0]  # 'bmw'
+
+    if owner_token and len(owner_token) >= 3:
+        domain_re = re.compile(
+            r"https?://([a-z0-9][a-z0-9\-]*\.)*" + re.escape(owner_token)
+            + r"[a-z0-9\-]*\.[a-z]{2,}",
+            re.IGNORECASE,
+        )
+        seen_bases = {primary_base}
+        for entry in doc_entries:
+            text = entry.get("text") or ""
+            for m in domain_re.finditer(text):
+                p = urlparse(m.group(0))
+                base = f"{p.scheme}://{p.netloc}/"
+                base_netloc = p.netloc.lower().lstrip("www.")
+                if base_netloc == primary_netloc:
+                    continue
+                if base in seen_bases:
+                    continue
+                seen_bases.add(base)
+                crawl_bases.append(base)
+                if len(crawl_bases) >= 3:
+                    break
+            if len(crawl_bases) >= 3:
+                break
+
     _update(
         check_id,
-        f"Suche fehlende Dokumente auf {urlparse(base).netloc}...",
+        f"Suche fehlende Dokumente auf {', '.join(urlparse(b).netloc for b in crawl_bases)}...",
         18,
     )
 
-    try:
+    discovered: list[dict] = []
-        async with httpx.AsyncClient(timeout=180.0) as client:
+    disc_payloads: list[dict] = []
-            resp = await client.post(
+    for base in crawl_bases:
-                f"{CONSENT_TESTER_URL}/dsi-discovery",
+        try:
-                json={"url": base, "max_documents": 15},
+            async with httpx.AsyncClient(timeout=180.0) as client:
-                timeout=180.0,
+                resp = await client.post(
-            )
+                    f"{CONSENT_TESTER_URL}/dsi-discovery",
-            if resp.status_code != 200:
+                    json={"url": base, "max_documents": 15},
-                logger.warning("auto-discovery: HTTP %d for %s", resp.status_code, base)
+                    timeout=180.0,
-                discovered: list[dict] = []
+                )
-                disc_payloads: list[dict] = []
+                if resp.status_code != 200:
-            else:
+                    logger.warning("auto-discovery: HTTP %d for %s",
-                disc_body = resp.json()
+                                   resp.status_code, base)
-                discovered = disc_body.get("documents", [])
+                    continue
-                disc_payloads = disc_body.get("cmp_payloads") or []
+                body = resp.json()
-    except Exception as e:
+                discovered.extend(body.get("documents", []) or [])
-        logger.warning("auto-discovery failed for %s: %s", base, e)
+                disc_payloads.extend(body.get("cmp_payloads") or [])
-        discovered = []
+                logger.info("auto-discovery on %s: %d docs",
-        disc_payloads = []
+                            base, len(body.get("documents", []) or []))
+        except Exception as e:
+            logger.warning("auto-discovery failed for %s: %s", base, e)
 
     # Classify each discovered doc into a canonical doc_type
     by_type: dict[str, dict] = {}
@@ -736,8 +770,12 @@ async def _check_single(
 
     # Master Control checks (top 20 by severity to avoid noise)
     try:
+        # max_controls=0 -> evaluate ALL MCs for this doc_type (DB has
+        # 1874 across 8 types; regex matching is cheap and dominates
+        # well under 1s per doc). Caps remain on the LLM-enrich step
+        # (top-10 FAILs) so cost stays bounded.
         mc_results = await check_document_with_controls(
-            text, doc_type, label, max_controls=20, use_agent=use_agent,
+            text, doc_type, label, max_controls=0, use_agent=use_agent,
         )
         if mc_results:
             for mc in mc_results:

backend-compliance/compliance/services/doc_checks/llm_verify.py

@@ -62,7 +62,13 @@ async def verify_failed_checks(
 async def _ask_llm_batch(
     text: str, checks: list[dict], doc_title: str,
 ) -> dict[str, dict]:
-    """Ask the LLM to verify ALL failed checks in a single call."""
+    """Ask the LLM to verify ALL failed checks in a single call.
+
+    Uses /api/chat with format='json' so Ollama enforces a valid JSON
+    response object — much more reliable than the previous /api/generate
+    + free-text approach which qwen3 often wrapped in <think>...</think>
+    reasoning tokens.
+    """
     checklist_lines = []
     for i, c in enumerate(checks, 1):
         checklist_lines.append(
@@ -70,69 +76,113 @@ async def _ask_llm_batch(
         )
     checklist_str = "\n".join(checklist_lines)
 
-    prompt = f"""/no_think
+    system_msg = (
-Pruefe ob der Dokumenttext die folgenden Anforderungen erfuellt.
+        "Du pruefst ob ein Dokument bestimmte Pflichtangaben enthaelt. "
-DOKUMENT: "{doc_title}"
+        "Antworte AUSSCHLIESSLICH mit einem JSON-Objekt: "
+        '{"results": [{"id": "<check-id>", "found": true|false, '
+        '"evidence": "<kurzes Zitat oder leer>"}]}. '
+        "Keine Erklaerungen, keine Reasoning-Tags, kein Markdown."
+    )
+    user_msg = (
+        f'DOKUMENT: "{doc_title}"\n\n'
+        f"ANFORDERUNGEN:\n{checklist_str}\n\n"
+        f"TEXT:\n{text}"
+    )
 
-ANFORDERUNGEN:
+    payload = {
-{checklist_str}
+        "model": OLLAMA_MODEL,
+        "messages": [
+            {"role": "system", "content": system_msg},
+            {"role": "user", "content": user_msg},
+        ],
+        "stream": False,
+        "format": "json",  # forces valid JSON output
+        "options": {"temperature": 0.0, "num_predict": 3000},
+    }
 
-TEXT:
+    async with httpx.AsyncClient(timeout=120.0) as client:
-{text}
+        resp = await client.post(f"{OLLAMA_URL}/api/chat", json=payload)
-
-Antworte NUR mit einem JSON-Array (keine Erklaerung). Fuer jede Anforderung:
-[{{"id": "check-id", "found": true/false, "evidence": "Kurzes Zitat (max 80 Zeichen) oder leer"}}]
-"""
-
-    async with httpx.AsyncClient(timeout=90.0) as client:
-        resp = await client.post(
-            f"{OLLAMA_URL}/api/generate",
-            json={
-                "model": OLLAMA_MODEL,
-                "prompt": prompt,
-                "stream": False,
-                "options": {"temperature": 0.0, "num_predict": 2000},
-            },
-        )
         resp.raise_for_status()
-        raw = resp.json().get("response", "")
+        data = resp.json()
+        raw = (data.get("message") or {}).get("content", "")
 
     return _parse_batch_response(raw, checks)
 
 
 def _parse_batch_response(raw: str, checks: list[dict]) -> dict[str, dict]:
-    """Parse batch LLM JSON array response."""
+    """Parse batch LLM response. Tolerates <think>…</think> wrappers,
+    code-fences, and either {results: [...]} or top-level [...]."""
     import json
     import re
 
     results: dict[str, dict] = {}
-    raw = raw.strip()
+    if not raw:
+        logger.info("LLM batch: empty response from model")
+        return results
 
-    # Extract JSON array from markdown code blocks
+    text = raw.strip()
-    m = re.search(r"```(?:json)?\s*(\[.*?\])\s*```", raw, re.DOTALL)
+    # Strip qwen3 thinking tags
+    text = re.sub(r"<think>.*?</think>", "", text, flags=re.DOTALL).strip()
+    # Strip markdown code fences
+    m = re.search(r"```(?:json)?\s*(.+?)\s*```", text, re.DOTALL)
     if m:
-        raw = m.group(1)
+        text = m.group(1).strip()
-    else:
-        m = re.search(r"\[.*\]", raw, re.DOTALL)
-        if m:
-            raw = m.group(0)
 
+    # Try parse as-is
+    parsed = None
     try:
-        items = json.loads(raw)
+        parsed = json.loads(text)
-        if isinstance(items, list):
-            for item in items:
-                cid = item.get("id", "")
-                if cid:
-                    results[cid] = {
-                        "found": bool(item.get("found", False)),
-                        "evidence": str(item.get("evidence", ""))[:150],
-                    }
     except (json.JSONDecodeError, ValueError):
-        # Fallback: extract individual JSON objects
+        # Try finding the first JSON object or array in the text
-        for m in re.finditer(r'\{[^}]*"id"\s*:\s*"([^"]+)"[^}]*"found"\s*:\s*(true|false)[^}]*\}', raw, re.DOTALL):
+        for pattern in (r"\{.*\}", r"\[.*\]"):
-            cid = m.group(1)
+            mm = re.search(pattern, text, re.DOTALL)
-            found = m.group(2) == "true"
+            if mm:
-            results[cid] = {"found": found, "evidence": ""}
+                try:
+                    parsed = json.loads(mm.group(0))
+                    break
+                except (json.JSONDecodeError, ValueError):
+                    continue
+
+    if parsed is None:
+        logger.info(
+            "LLM batch: 0/%d checks parsed (raw head: %r)",
+            len(checks), raw[:120],
+        )
+        return results
+
+    # Accept both {"results": [...]} (preferred) and bare list
+    items = None
+    if isinstance(parsed, dict):
+        for key in ("results", "checks", "items", "verifications"):
+            if isinstance(parsed.get(key), list):
+                items = parsed[key]
+                break
+    elif isinstance(parsed, list):
+        items = parsed
+
+    if not items:
+        # Final fallback: regex over individual id/found pairs
+        for mm in re.finditer(
+            r'\{[^}]*"id"\s*:\s*"([^"]+)"[^}]*"found"\s*:\s*(true|false)[^}]*\}',
+            raw, re.DOTALL,
+        ):
+            results[mm.group(1)] = {
+                "found": mm.group(2) == "true", "evidence": "",
+            }
+        logger.info("LLM batch: %d/%d checks parsed (regex fallback)",
+                    len(results), len(checks))
+        return results
+
+    for item in items:
+        if not isinstance(item, dict):
+            continue
+        cid = item.get("id", "")
+        if not cid:
+            continue
+        results[cid] = {
+            "found": bool(item.get("found", False)),
+            "evidence": str(item.get("evidence", ""))[:150],
+        }
 
     logger.info("LLM batch: %d/%d checks parsed", len(results), len(checks))
     return results

backend-compliance/compliance/services/doc_checks/loeschkonzept_checks.py

@@ -0,0 +1,191 @@
+"""
+Loeschkonzept checks — Art. 5(1)(e) DSGVO ('Speicherbegrenzung'),
+DIN 66398 (Leitlinie zur Entwicklung eines Loeschkonzepts).
+
+L1: Pflichtangabe vorhanden?
+L2: Pflichtangabe vollstaendig/korrekt?
+"""
+
+LOESCHKONZEPT_CHECKLIST = [
+    # ── L1: Geltungsbereich + Verantwortliche ─────────────────────────
+    {
+        "id": "scope_responsibility",
+        "label": "Geltungsbereich + Verantwortliche benannt",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"(?:geltungsbereich|anwendungsbereich)",
+            r"verantwortlich\w*\s+(?:fuer|für)\s+(?:das\s+)?l(?:oe|ö)schkonzept",
+            r"(?:datenschutzbeauftragt\w*|dpo|dsb)\s+(?:verantwort|zustaendig|zuständig)",
+        ],
+        "severity": "HIGH",
+        "hint": "DIN 66398 verlangt einen klaren Geltungsbereich (welche Systeme, Datenarten, Standorte) und die Benennung des Verantwortlichen fuer Erstellung + Wartung des Loeschkonzepts.",
+    },
+
+    # ── L1: Datenkategorien ───────────────────────────────────────────
+    {
+        "id": "data_categories",
+        "label": "Datenkategorien / Datenarten dokumentiert",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"(?:daten[\-\s]?kategori|datenart)\w*",
+            r"(?:kunden|mitarbeiter|interessent|bewerber|lieferant)daten",
+            r"personenbezogene?\s+daten[\-\s]?(?:art|kategori|gruppe)",
+        ],
+        "severity": "HIGH",
+        "hint": "Pro Datenkategorie (Kundenstammdaten, Vertragsdaten, Mitarbeiterdaten, Bewerber, Lieferantendaten, etc.) muss eine eigene Loeschfrist festgelegt werden.",
+    },
+    {
+        "id": "data_categories_specific",
+        "label": "Konkrete Datenarten benannt (>= 5)",
+        "level": 2, "parent": "data_categories",
+        "patterns": [
+            r"(?:stammdaten|kontaktdaten|vertragsdaten|abrechnungsdaten|"
+            r"protokoll\w+|log[\-\s]?daten|gesundheits|biometrische?|"
+            r"finanz|bewerbungsunterlagen|kreditdaten|tracking)",
+        ],
+        "severity": "MEDIUM",
+        "hint": "Beispiele aus der Praxis: Stammdaten, Kontaktdaten, Vertragsdaten, Abrechnungsdaten, Bewerbungsunterlagen, Log-Daten, Tracking-Cookies. Jeweils mit eigener Frist.",
+    },
+
+    # ── L1: Loeschfristen ─────────────────────────────────────────────
+    {
+        "id": "retention_periods",
+        "label": "Konkrete Loeschfristen festgelegt",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"l(?:oe|ö)schfrist",
+            r"aufbewahrungs(?:frist|dauer|pflicht)",
+            r"\d+\s*(?:jahr|monat|tag|woche)",
+            r"speicherdauer",
+        ],
+        "severity": "HIGH",
+        "hint": "Art. 5(1)(e) DSGVO + Art. 13(2)(a) DSGVO: Speicherdauer oder Kriterien fuer die Festlegung muessen pro Datenkategorie konkret benannt sein.",
+    },
+    {
+        "id": "legal_retention_basis",
+        "label": "Gesetzliche Aufbewahrungspflichten referenziert",
+        "level": 2, "parent": "retention_periods",
+        "patterns": [
+            r"§\s*257\s*hgb|§\s*147\s*ao|hgb\s+§\s*257|ao\s+§\s*147",
+            r"6\s+jahr\w*\s+\((?:hgb|handels)",
+            r"10\s+jahr\w*\s+\((?:ao|steuer)",
+            r"handelsrechtlich|steuerrechtlich",
+            r"§\s*195\s*bgb|verjaehrung\w*\s+\(bgb",
+        ],
+        "severity": "MEDIUM",
+        "hint": "Standardfristen referenzieren: §257 HGB (6 Jahre Buchungsbelege), §147 AO (10 Jahre Steuerunterlagen), §195 BGB (3 Jahre Verjaehrung). Ohne Referenz wirken die Fristen willkuerlich.",
+    },
+    {
+        "id": "deletion_trigger",
+        "label": "Loeschtrigger / Loeschanlass dokumentiert",
+        "level": 2, "parent": "retention_periods",
+        "patterns": [
+            r"l(?:oe|ö)sch[\-\s]?(?:trigger|anlass|grund|kriteri)",
+            r"(?:nach|bei)\s+(?:vertragsende|kuendigung|kündigung|abschluss)",
+            r"zweckwegfall|zweck\s+entfaellt",
+            r"einwilligungswiderruf",
+        ],
+        "severity": "MEDIUM",
+        "hint": "Pro Datenkategorie muss klar sein WANN die Frist beginnt: Vertragsende, Kuendigung, letzter Kontakt, Zweckwegfall, Einwilligungswiderruf. Nur 'X Jahre' ohne Startpunkt ist unscharf.",
+    },
+
+    # ── L1: Loeschprozess ─────────────────────────────────────────────
+    {
+        "id": "deletion_process",
+        "label": "Loeschprozess beschrieben",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"l(?:oe|ö)sch(?:prozess|vorgang|verfahren|workflow|routine)",
+            r"(?:wie|wann)\s+(?:wird|werden)\s+(?:die\s+daten\s+)?gel(?:oe|ö)scht",
+            r"automatisierte?\s+l(?:oe|ö)schung",
+        ],
+        "severity": "HIGH",
+        "hint": "Beschreiben wie Loeschung erfolgt: automatisch per Cron-Job, manuell durch Admin, Loeschungs-Workflow im CRM, Backup-Loeschung etc.",
+    },
+    {
+        "id": "deletion_technical",
+        "label": "Technische Loeschmethode benannt",
+        "level": 2, "parent": "deletion_process",
+        "patterns": [
+            r"(?:physisch\w*|sicher\w*)\s+l(?:oe|ö)schung",
+            r"(?:ueberschr|überschr)eiben\w*\s*(?:der|von)?\s*daten",
+            r"(?:dod[\-\s]?5220|nist[\-\s]?800|crypto[\-\s]?shredding)",
+            r"papierakten?\s*(?:vernicht|schreddern|verbrenn)",
+            r"datentraeger\w*\s+(?:zerstoer|vernicht|entmagnetis)",
+        ],
+        "severity": "LOW",
+        "hint": "Technische Standards nennen: DoD 5220.22-M (mehrfaches Ueberschreiben), NIST 800-88, Crypto-Shredding (Verschluesselung + Schluesselvernichtung), Papier per DIN 66399.",
+    },
+    {
+        "id": "deletion_systems",
+        "label": "Loeschung in allen relevanten Systemen",
+        "level": 2, "parent": "deletion_process",
+        "patterns": [
+            r"backup\w*\s+(?:l(?:oe|ö)sch|umfass|einbezogen|loeschr|löschr)",
+            r"archive?\w*\s+(?:l(?:oe|ö)sch|enthalten|inbegriffen|umfass)",
+            r"(?:crm|erp|datenbank|datawarehouse|dwh)\w*\s+l(?:oe|ö)sch",
+            r"(?:alle|saemtliche|sämtliche)\s+systeme",
+        ],
+        "severity": "LOW",
+        "hint": "Loeschung muss in ALLEN Systemen erfolgen: CRM, ERP, Backups, Archive, Data Warehouse, lokale Kopien. Backups die laenger als die Loeschfrist aufbewahrt werden sind kritisch (gerichtlich umstritten).",
+    },
+
+    # ── L1: Loeschnachweis ────────────────────────────────────────────
+    {
+        "id": "deletion_proof",
+        "label": "Loeschnachweis / Loeschprotokoll",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"l(?:oe|ö)sch[\-\s]?(?:nachweis|protokoll|dokumentation|log)",
+            r"(?:protokoll|dokument)\w*\s+(?:der|alle)\s+l(?:oe|ö)schung",
+            r"audit[\-\s]?trail.*l(?:oe|ö)sch",
+        ],
+        "severity": "MEDIUM",
+        "hint": "Art. 5(2) DSGVO (Rechenschaftspflicht): Loeschvorgaenge muessen nachweisbar sein. Mindestens: was, wann, durch wen, von wo. Ein Audit-Log oder Loeschprotokoll erfuellt das.",
+    },
+
+    # ── L1: Ausnahmen / Sperren ───────────────────────────────────────
+    {
+        "id": "deletion_exceptions",
+        "label": "Ausnahmen + Sperrung statt Loeschung",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"(?:einschraenkung|einschränkung)\s+der\s+verarbeitung|art\.?\s*18",
+            r"sperr\w+\s+(?:statt|anstelle)\s+l(?:oe|ö)sch",
+            r"l(?:oe|ö)sch(?:beschr|sperr|ausnahme|hindernis)",
+            r"(?:rechtsstreit|gerichtsverfahren|prozessrelevant)",
+        ],
+        "severity": "MEDIUM",
+        "hint": "Wenn Loeschung nicht moeglich ist (laufender Prozess, gesetzliche Aufbewahrung, Streitfall) muss stattdessen Sperrung/Einschraenkung (Art. 18 DSGVO) erfolgen. Sperrkonzept dokumentieren.",
+    },
+
+    # ── L1: Review-Zyklus ─────────────────────────────────────────────
+    {
+        "id": "review_cycle",
+        "label": "Review-Zyklus / regelmaessige Pruefung",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"(?:jaehrlich|jährlich|halbjaehrlich|halbjährlich|vierteljaehrlich|vierteljährlich|quartalsweise)\s+(?:gepr|review|aktualis)",
+            r"review[\-\s]?(?:zyklus|intervall|frist)",
+            r"naechste|nächste\s+(?:ueberpr|überpr|review)",
+            r"(?:loeschkonzept|löschkonzept)\s+(?:wird|muss)\s+(?:regelmaessig|regelmäßig|jaehrlich|jährlich)",
+        ],
+        "severity": "LOW",
+        "hint": "DIN 66398 + Praxis: Loeschkonzept jaehrlich (oder bei Systemaenderungen ausserplanmaessig) ueberpruefen. Frist explizit benennen ('jaehrlich') statt nur 'regelmaessig'.",
+    },
+
+    # ── L1: DSGVO-Verweise ────────────────────────────────────────────
+    {
+        "id": "gdpr_reference",
+        "label": "Rechtliche Grundlagen referenziert",
+        "level": 1, "parent": None,
+        "patterns": [
+            r"art\.?\s*5\s*(?:abs\.?\s*1\s*)?(?:lit\.?\s*)?e\s*dsgvo",
+            r"art\.?\s*17\s*dsgvo",
+            r"art\.?\s*32\s*dsgvo",
+            r"speicherbegrenzung",
+        ],
+        "severity": "LOW",
+        "hint": "Direkte Norm-Referenzen erhoehen Beweiskraft: Art. 5(1)(e) DSGVO (Speicherbegrenzung), Art. 17 (Recht auf Loeschung), Art. 32 (TOMs).",
+    },
+]

backend-compliance/compliance/services/doc_checks/runner.py

@@ -20,6 +20,7 @@ from .avv_checks import AVV_CHECKLIST
 from .scc_checks import SCC_CHECKLIST
 from .tom_annex_checks import TOM_ANNEX_CHECKLIST
 from .sub_processor_checks import SUB_PROCESSOR_LIST_CHECKLIST
+from .loeschkonzept_checks import LOESCHKONZEPT_CHECKLIST
 
 logger = logging.getLogger(__name__)
 
@@ -52,6 +53,10 @@ _CHECKLIST_MAP = {
     "sub_processor_list": (SUB_PROCESSOR_LIST_CHECKLIST, "Art. 28(3)(d) DSGVO"),
     "sub_processor": (SUB_PROCESSOR_LIST_CHECKLIST, "Art. 28(3)(d) DSGVO"),
     "unterauftragnehmer": (SUB_PROCESSOR_LIST_CHECKLIST, "Art. 28(3)(d) DSGVO"),
+    "loeschkonzept": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
+    "loeschung": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
+    "loeschfristen": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
+    "deletion_concept": (LOESCHKONZEPT_CHECKLIST, "Art. 5(1)(e) DSGVO / DIN 66398"),
 }
 
 

backend-compliance/compliance/services/rag_document_checker.py

@@ -241,8 +241,35 @@ def _map_doc_type(doc_type: str) -> str:
     return _DOC_TYPE_MAP.get(doc_type, doc_type)
 
 
+# Doc-types that have no own MCs but can borrow from a related set.
+# (DB currently covers: dse, cookie, loeschkonzept, widerruf, dsfa,
+#  avv, agb, impressum — total 1874 MCs across these.)
+_MC_ALIAS_FALLBACK = {
+    "nutzungsbedingungen": "agb",        # T&C overlap
+    "terms": "agb",
+    "terms_of_use": "agb",
+    "social_media": "dse",               # Joint-controller / Art. 26 is in DSE area
+    "joint_controller": "dse",
+    "sub_processor": "avv",              # Subprocessor list = AVV annex
+    "sub_processor_list": "avv",
+    "scc": "avv",                        # SCC = AVV-Vertragsklauseln
+    "standardvertragsklauseln": "avv",
+    "tom_annex": "avv",                  # TOM-Annex meist als AVV-Anlage
+    "tom": "avv",
+    "dpa": "avv",
+    "loeschung": "loeschkonzept",
+    "loeschfristen": "loeschkonzept",
+    "eu_institution": "dse",             # EU institution = DSE under VO 2018/1725
+    "dsb": "dse",                        # DSB info ist Teil der DSE
+}
+
+
 async def _load_controls(doc_type: str, db_url: str, limit: int) -> list[dict]:
-    """Load all doc_check_controls for a doc_type from PostgreSQL."""
+    """Load all doc_check_controls for a doc_type from PostgreSQL.
+
+    Falls back via _MC_ALIAS_FALLBACK when no MCs exist for the requested
+    type (e.g. 'nutzungsbedingungen' -> 'agb').
+    """
     try:
         import asyncpg
         db = db_url or os.getenv(
@@ -264,6 +291,10 @@ async def _load_controls(doc_type: str, db_url: str, limit: int) -> list[dict]:
             query += f" LIMIT {limit}"
 
         rows = await conn.fetch(query, doc_type)
+        if not rows and doc_type in _MC_ALIAS_FALLBACK:
+            fallback = _MC_ALIAS_FALLBACK[doc_type]
+            logger.info("No MCs for %s -> falling back to %s", doc_type, fallback)
+            rows = await conn.fetch(query, fallback)
         return [dict(r) for r in rows]
     except Exception as e:
         logger.warning("MC query failed: %s", e)

backend-compliance/scripts/backfill_mc_regulation.py

@@ -0,0 +1,180 @@
+"""
+Backfill the doc_check_controls.regulation + .article fields.
+
+The fields are currently NULL on all 1874 rows. Many MCs cite the
+relevant norm inline in title / check_question / pass_criteria
+(e.g. 'Art. 6 Abs. 1 lit. a DSGVO', '§ 25 Abs. 1 TDDDG'). We detect
+those with regex and UPDATE the row.
+
+Run inside the bp-compliance-backend container:
+    docker exec bp-compliance-backend python3 /app/scripts/backfill_mc_regulation.py [--dry-run]
+
+The script is idempotent: existing non-null regulation is never overwritten.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import os
+import re
+import sys
+
+import asyncpg
+
+
+# Ordered: first match wins. Each pattern captures (article_str, regulation_label).
+_PATTERNS: list[tuple[re.Pattern[str], str]] = [
+    # Art. X DSGVO / GDPR / EU 2016/679
+    (re.compile(
+        r"\b(?:art\.?|artikel)\s*"
+        r"(\d+[a-z]?(?:\s*(?:abs\.?|absatz)\s*\d+)?"
+        r"(?:\s*(?:lit\.?|litera|buchstabe)\s*[a-z])?"
+        r"(?:\s*satz\s*\d+)?)"
+        r"\s*(?:dsgvo|gdpr|vo\s*\(eu\)\s*2016/679|eu[\s-]?vo\s*2016/679)",
+        re.I,
+    ), "DSGVO"),
+    # § X TDDDG / TTDSG
+    (re.compile(
+        r"§\s*(\d+[a-z]?(?:\s*abs\.?\s*\d+)?(?:\s*(?:nr\.?|lit\.?)\s*\w+)?)\s*"
+        r"(?:tdddg|ttdsg|tkg)",
+        re.I,
+    ), "TDDDG"),
+    # § X TMG
+    (re.compile(
+        r"§\s*(\d+[a-z]?(?:\s*abs\.?\s*\d+)?(?:\s*nr\.?\s*\d+)?)\s*tmg\b",
+        re.I,
+    ), "TMG"),
+    # § X BGB
+    (re.compile(
+        r"§\s*(\d+[a-z]?(?:\s*abs\.?\s*\d+)?)\s*bgb\b",
+        re.I,
+    ), "BGB"),
+    # § X HGB
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*hgb\b",
+        re.I,
+    ), "HGB"),
+    # § X AO
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*ao\b",
+        re.I,
+    ), "AO"),
+    # § X MStV (Medienstaatsvertrag)
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*m(?:edien)?st(?:aat)?v\b",
+        re.I,
+    ), "MStV"),
+    # § X UWG
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*uwg\b",
+        re.I,
+    ), "UWG"),
+    # § X VSBG (Verbraucherstreitbeilegung)
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*vsbg\b",
+        re.I,
+    ), "VSBG"),
+    # § X PAngV
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*p(?:reis)?ang?v\b",
+        re.I,
+    ), "PAngV"),
+    # § X GwG
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*gwg\b",
+        re.I,
+    ), "GwG"),
+    # § X BDSG
+    (re.compile(
+        r"§\s*(\d+[a-z]?)\s*bdsg\b",
+        re.I,
+    ), "BDSG"),
+    # EU-VO 524/2013 (ODR), 2018/1725 (EU-DSGVO) etc.
+    (re.compile(
+        r"\bart\.?\s*(\d+)\s*(?:eu[\s-]?vo|vo|verordnung)\s*(?:\(eu\)\s*)?(\d+/\d+)",
+        re.I,
+    ), "EU-VO"),
+    # Norm names without numbers, last resort (set article=NULL)
+    (re.compile(r"\bdsgvo\b", re.I), "DSGVO"),
+    (re.compile(r"\btdddg\b|\bttdsg\b", re.I), "TDDDG"),
+    (re.compile(r"\btmg\b", re.I), "TMG"),
+    (re.compile(r"\bbgb\b", re.I), "BGB"),
+    (re.compile(r"\bmstv\b", re.I), "MStV"),
+    (re.compile(r"\buwg\b", re.I), "UWG"),
+    (re.compile(r"\bvsbg\b", re.I), "VSBG"),
+    (re.compile(r"\bgwg\b", re.I), "GwG"),
+    (re.compile(r"\bbdsg\b", re.I), "BDSG"),
+]
+
+
+def detect(text: str) -> tuple[str | None, str | None]:
+    """Return (regulation, article) for the first pattern that matches."""
+    if not text:
+        return None, None
+    for pat, label in _PATTERNS:
+        m = pat.search(text)
+        if m:
+            article = m.group(1) if m.groups() else None
+            if label == "EU-VO" and m.lastindex and m.lastindex >= 2:
+                article = f"Art. {m.group(1)} EU-VO {m.group(2)}"
+            elif article:
+                article = re.sub(r"\s+", " ", article).strip()
+            return label, article
+    return None, None
+
+
+async def main(dry_run: bool = False) -> None:
+    db = os.getenv("DATABASE_URL")
+    if not db:
+        print("DATABASE_URL not set", file=sys.stderr)
+        sys.exit(1)
+    conn = await asyncpg.connect(db)
+
+    rows = await conn.fetch(
+        "SELECT id, title, check_question, pass_criteria::text AS pc "
+        "FROM compliance.doc_check_controls "
+        "WHERE regulation IS NULL"
+    )
+    print(f"{len(rows)} MCs with NULL regulation")
+
+    updates: list[tuple[str | None, str | None, str]] = []
+    hits = {"DSGVO": 0, "TDDDG": 0, "TMG": 0, "BGB": 0, "MStV": 0,
+            "UWG": 0, "VSBG": 0, "EU-VO": 0, "HGB": 0, "AO": 0,
+            "PAngV": 0, "GwG": 0, "BDSG": 0}
+    no_match = 0
+    for r in rows:
+        combined = " ".join(filter(None, [
+            r["title"] or "", r["check_question"] or "", r["pc"] or "",
+        ]))
+        reg, art = detect(combined)
+        if reg:
+            hits[reg] = hits.get(reg, 0) + 1
+            updates.append((reg, art, str(r["id"])))
+        else:
+            no_match += 1
+
+    print(f"Detected: {sum(hits.values())} | no match: {no_match}")
+    for k, v in sorted(hits.items(), key=lambda x: -x[1]):
+        if v:
+            print(f"   {k:8s} {v:>5}")
+
+    if dry_run:
+        print("\nDRY RUN — no changes written. Re-run without --dry-run to apply.")
+        await conn.close()
+        return
+
+    # Apply updates in batches
+    BATCH = 200
+    for i in range(0, len(updates), BATCH):
+        chunk = updates[i:i + BATCH]
+        await conn.executemany(
+            "UPDATE compliance.doc_check_controls "
+            "SET regulation = $1, article = $2 WHERE id = $3::uuid",
+            chunk,
+        )
+    print(f"\nApplied {len(updates)} updates.")
+    await conn.close()
+
+
+if __name__ == "__main__":
+    asyncio.run(main(dry_run="--dry-run" in sys.argv))